From 5fab96b7ee5181e0164ccef6a135952f58726dad Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Tue, 14 Jan 2014 17:26:55 -0800 Subject: [PATCH 01/72] First import of AVX-JNI to git --- .gitignore | 6 + CHEATSHEET | 2 + Makefile | 43 ++ README | 41 ++ define-double.h | 154 ++++++ define-float.h | 155 ++++++ hmm_mask.cc | 484 ++++++++++++++++++ ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 146 ++++++ ...te_sting_utils_pairhmm_JNILoglessPairHMM.h | 61 +++ pairhmm-1-base.cc | 276 ++++++++++ pairhmm-template-kernel.cc | 399 +++++++++++++++ pairhmm-template-main.cc | 114 +++++ shift_template.c | 68 +++ template.h | 226 ++++++++ 14 files changed, 2175 insertions(+) create mode 100644 .gitignore create mode 100644 CHEATSHEET create mode 100644 Makefile create mode 100644 README create mode 100644 define-double.h create mode 100644 define-float.h create mode 100644 hmm_mask.cc create mode 100644 org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc create mode 100644 org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h create mode 100644 pairhmm-1-base.cc create mode 100644 pairhmm-template-kernel.cc create mode 100644 pairhmm-template-main.cc create mode 100644 shift_template.c create mode 100644 template.h diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..16e86f25d --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.svn +*.o +*.so +tests +.deps +hmm_Mohammad diff --git a/CHEATSHEET b/CHEATSHEET new file mode 100644 index 000000000..f61bfdd7f --- /dev/null +++ b/CHEATSHEET @@ -0,0 +1,2 @@ +/opt/intel/bin/icc pairhmm-template-main.cc -o pairhmm-template-main.o -O3 -W -Wall -lm -xAVX; ./pairhmm-template-main.o < mediumTest.in > med.out; head -n -5 med.out > medium.out; lua checkLikelihoods.lua medium.out ~/hmm/mediumTest.out 6e-5 + diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..df5dbc463 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +#OMPCFLAGS=-fopenmp +#OMPLDFLAGS=-lgomp + +#CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas +#CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas + +JAVA_ROOT=/opt/jdk1.7.0_25/ +JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JAVA_ROOT}/include -I${JAVA_ROOT}/include/linux + +CFLAGS=-g -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas -xAVX + +CXXFLAGS=$(CFLAGS) +CC=icc +CXX=icc + +LDFLAGS=-lm $(OMPLDFLAGS) + +#BIN:=pairhmm-1-base #pairhmm-2-omp pairhmm-3-hybrid-float-double pairhmm-4-hybrid-diagonal pairhmm-5-hybrid-diagonal-homogeneus pairhmm-6-onlythreediags pairhmm-7-presse pairhmm-8-sse #pairhmm-dev +BIN:=libJNILoglessPairHMM.so #pairhmm-2-omp pairhmm-3-hybrid-float-double pairhmm-4-hybrid-diagonal pairhmm-5-hybrid-diagonal-homogeneus pairhmm-6-onlythreediags pairhmm-7-presse pairhmm-8-sse #pairhmm-dev + +#SOURCES=pairhmm-1-base.cc input.cc +SOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc hmm_mask.cc +OBJECTS=$(SOURCES:.cc=.o) +DEPDIR=.deps +DF=$(DEPDIR)/$(*).d + +all: $(BIN) + +-include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) + +pairhmm-1-base: pairhmm-1-base.o input.o + $(CXX) -o $@ $^ $(LDFLAGS) + +libJNILoglessPairHMM.so: $(OBJECTS) + $(CXX) -shared -o libJNILoglessPairHMM.so $(OBJECTS) + +%.o: %.cc + @mkdir -p $(DEPDIR) + $(COMPILE.cpp) -MMD -MF $(DF) $(JNI_COMPILATION_FLAGS) $(CXXFLAGS) $(OUTPUT_OPTION) $< + + +clean: + rm -f $(BIN) *.o diff --git a/README b/README new file mode 100644 index 000000000..e16df5a8d --- /dev/null +++ b/README @@ -0,0 +1,41 @@ +How to compile: +/opt/intel/bin/icc pairhmm-template-main.cc -o pairhmm-template-main.o -O3 -W -Wall -lm -xAVX + +------------------------------------ +How to run: +./pairhmm-template-main.o < ./test_data/largeTest.in > large.out +./pairhmm-template-main.o < ./test_data/mediumTest.in > medium.out +------------------------------------- +Results: +------------------------------------- +MEDIUM FLOAT: +AVX Read Time: 8019020 +AVX Compute Time: 29270238 +AVX Write Time: 242170 +AVX Total Time: 37531428 +# Double called: 0 +------------------------------------- +MEDIUM HYBRID: +AVX Read Time: 7596629 +AVX Compute Time: 29979220 +AVX Write Time: 248485 +AVX Total Time: 37824334 +# Double called: 20329 +------------------------------------- +LARGE FLOAT: +AVX Read Time: 111888586 +AVX Compute Time: 408131806 +AVX Write Time: 3477892 +AVX Total Time: 523498284 +# Double called: 0 +------------------------------------- +LARGE HYBRID: +AVX Read Time: 109728994 +AVX Compute Time: 419475317 +AVX Write Time: 3575364 +AVX Total Time: 532779675 +# Double called: 310042 + + + + diff --git a/define-double.h b/define-double.h new file mode 100644 index 000000000..b2de7520a --- /dev/null +++ b/define-double.h @@ -0,0 +1,154 @@ +#include + +#ifdef PRECISION + #undef PRECISION + #undef MAIN_TYPE + #undef MAIN_TYPE_SIZE + #undef UNION_TYPE + #undef IF_128 + #undef IF_MAIN_TYPE + #undef SHIFT_CONST1 + #undef SHIFT_CONST2 + #undef SHIFT_CONST3 + #undef _128_TYPE + #undef _256_TYPE + #undef AVX_LENGTH + #undef MAVX_COUNT + #undef HAP_TYPE + #undef MASK_TYPE + #undef MASK_ALL_ONES + + #undef SET_VEC_ZERO + #undef VEC_OR + #undef VEC_ADD + #undef VEC_SUB + #undef VEC_MUL + #undef VEC_BLEND + #undef VEC_BLENDV + #undef VEC_CAST_256_128 + #undef VEC_EXTRACT_128 + #undef VEC_EXTRACT_UNIT + #undef VEC_SET1_VAL128 + #undef VEC_MOVE + #undef VEC_CAST_128_256 + #undef VEC_INSERT_VAL + #undef VEC_CVT_128_256 + #undef VEC_SET1_VAL + #undef VEC_POPCVT_CHAR + #undef VEC_LDPOPCVT_CHAR + #undef VEC_CMP_EQ + #undef VEC_SET_LSE + #undef SHIFT_HAP + #undef print256b + #undef MASK_VEC + #undef VEC_SSE_TO_AVX + #undef VEC_SHIFT_LEFT_1BIT + #undef MASK_ALL_ONES + #undef COMPARE_VECS + #undef _256_INT_TYPE +#endif + +#define PRECISION d +#define MAIN_TYPE double +#define MAIN_TYPE_SIZE 64 +#define UNION_TYPE mix_D +#define IF_128 IF_128d +#define IF_MAIN_TYPE IF_64 +#define SHIFT_CONST1 8 +#define SHIFT_CONST2 1 +#define SHIFT_CONST3 8 +#define _128_TYPE __m128d +#define _256_TYPE __m256d +#define _256_INT_TYPE __m256i +#define AVX_LENGTH 4 +#define MAVX_COUNT (MROWS+7)/AVX_LENGTH +#define HAP_TYPE __m128i +#define MASK_TYPE uint64_t +#define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFF +#define MASK_VEC MaskVec_D + +#define SET_VEC_ZERO(__vec) \ + __vec= _mm256_setzero_pd() + +#define VEC_OR(__v1, __v2) \ + _mm256_or_pd(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm256_add_pd(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm256_sub_pd(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm256_mul_pd(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm256_blend_pd(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm256_blendv_pd(__v1, __v2, __maskV) + +#define VEC_CAST_256_128(__v1) \ + _mm256_castpd256_pd128 (__v1) + +#define VEC_EXTRACT_128(__v1, __im) \ + _mm256_extractf128_pd (__v1, __im) + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi64(__v1, __im) + +#define VEC_SET1_VAL128(__val) \ + _mm_set1_pd(__val) + +#define VEC_MOVE(__v1, __val) \ + _mm_move_sd(__v1, __val) + +#define VEC_CAST_128_256(__v1) \ + _mm256_castpd128_pd256(__v1) + +#define VEC_INSERT_VAL(__v1, __val, __pos) \ + _mm256_insertf128_pd(__v1, __val, __pos) + +#define VEC_CVT_128_256(__v1) \ + _mm256_cvtepi32_pd(__v1) + +#define VEC_SET1_VAL(__val) \ + _mm256_set1_pd(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm256_cvtepi32_pd(_mm_set1_epi32(__ch)) + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm256_cvtepi32_pd(_mm_load_si128((__m128i const *)__addr)) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm256_cmp_pd(__v1, __v2, _CMP_EQ_OQ) + +#define VEC_SET_LSE(__val) \ + _mm256_set_pd(zero, zero, zero, __val); + +#define SHIFT_HAP(__v1, __val) \ + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) + +#define print256b(__v1) \ + print256bDP(__v1) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castpd128_pd256(__vsLow) ; \ + __vdst = _mm256_insertf128_pd(__vdst, __vsHigh, 1) ; + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi64(__vs, 1) + + +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + double* ptr1 = (double*) (&__v1) ; \ + double* ptr2 = (double*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Double Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ + } diff --git a/define-float.h b/define-float.h new file mode 100644 index 000000000..4cba9a179 --- /dev/null +++ b/define-float.h @@ -0,0 +1,155 @@ +#include + +#ifdef PRECISION + #undef PRECISION + #undef MAIN_TYPE + #undef MAIN_TYPE_SIZE + #undef UNION_TYPE + #undef IF_128 + #undef IF_MAIN_TYPE + #undef SHIFT_CONST1 + #undef SHIFT_CONST2 + #undef SHIFT_CONST3 + #undef _128_TYPE + #undef _256_TYPE + #undef AVX_LENGTH + #undef MAVX_COUNT + #undef HAP_TYPE + #undef MASK_TYPE + #undef MASK_ALL_ONES + + #undef SET_VEC_ZERO(__vec) + #undef VEC_OR(__v1, __v2) + #undef VEC_ADD(__v1, __v2) + #undef VEC_SUB(__v1, __v2) + #undef VEC_MUL(__v1, __v2) + #undef VEC_BLEND(__v1, __v2, __mask) + #undef VEC_BLENDV(__v1, __v2, __maskV) + #undef VEC_CAST_256_128(__v1) + #undef VEC_EXTRACT_128(__v1, __im) + #undef VEC_EXTRACT_UNIT(__v1, __im) + #undef VEC_SET1_VAL128(__val) + #undef VEC_MOVE(__v1, __val) + #undef VEC_CAST_128_256(__v1) + #undef VEC_INSERT_VAL(__v1, __val, __pos) + #undef VEC_CVT_128_256(__v1) + #undef VEC_SET1_VAL(__val) + #undef VEC_POPCVT_CHAR(__ch) + #undef VEC_LDPOPCVT_CHAR(__addr) + #undef VEC_CMP_EQ(__v1, __v2) + #undef VEC_SET_LSE(__val) + #undef SHIFT_HAP(__v1, __val) + #undef print256b(__v1) + #undef MASK_VEC + #undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) + #undef VEC_SHIFT_LEFT_1BIT(__vs) + #undef MASK_ALL_ONES + #undef COMPARE_VECS(__v1, __v2) + #undef _256_INT_TYPE +#endif + +#define PRECISION s + +#define MAIN_TYPE float +#define MAIN_TYPE_SIZE 32 +#define UNION_TYPE mix_F +#define IF_128 IF_128f +#define IF_MAIN_TYPE IF_32 +#define SHIFT_CONST1 12 +#define SHIFT_CONST2 3 +#define SHIFT_CONST3 4 +#define _128_TYPE __m128 +#define _256_TYPE __m256 +#define _256_INT_TYPE __m256i +#define AVX_LENGTH 8 +#define MAVX_COUNT (MROWS+7)/AVX_LENGTH +#define HAP_TYPE UNION_TYPE +#define MASK_TYPE uint32_t +#define MASK_ALL_ONES 0xFFFFFFFF +#define MASK_VEC MaskVec_F + +#define SET_VEC_ZERO(__vec) \ + __vec= _mm256_setzero_ps() + +#define VEC_OR(__v1, __v2) \ + _mm256_or_ps(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm256_add_ps(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm256_sub_ps(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm256_mul_ps(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm256_blend_ps(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm256_blendv_ps(__v1, __v2, __maskV) + +#define VEC_CAST_256_128(__v1) \ + _mm256_castps256_ps128 (__v1) + +#define VEC_EXTRACT_128(__v1, __im) \ + _mm256_extractf128_ps (__v1, __im) + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi32(__v1, __im) + +#define VEC_SET1_VAL128(__val) \ + _mm_set1_ps(__val) + +#define VEC_MOVE(__v1, __val) \ + _mm_move_ss(__v1, __val) + +#define VEC_CAST_128_256(__v1) \ + _mm256_castps128_ps256(__v1) + +#define VEC_INSERT_VAL(__v1, __val, __pos) \ + _mm256_insertf128_ps(__v1, __val, __pos) + +#define VEC_CVT_128_256(__v1) \ + _mm256_cvtepi32_ps(__v1.i) + +#define VEC_SET1_VAL(__val) \ + _mm256_set1_ps(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm256_cvtepi32_ps(_mm256_set1_epi32(__ch)) + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm256_cvtepi32_ps(_mm256_loadu_si256((__m256i const *)__addr)) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm256_cmp_ps(__v1, __v2, _CMP_EQ_OQ) + +#define VEC_SET_LSE(__val) \ + _mm256_set_ps(zero, zero, zero, zero, zero, zero, zero, __val); + +#define SHIFT_HAP(__v1, __val) \ + _vector_shift_lasts(__v1, __val.f); + +#define print256b(__v1) \ + print256bFP(__v1) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castps128_ps256(__vsLow) ; \ + __vdst = _mm256_insertf128_ps(__vdst, __vsHigh, 1) ; + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi32(__vs, 1) + +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + float* ptr1 = (float*) (&__v1) ; \ + float* ptr2 = (float*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Float Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ + } + diff --git a/hmm_mask.cc b/hmm_mask.cc new file mode 100644 index 000000000..aace7acf2 --- /dev/null +++ b/hmm_mask.cc @@ -0,0 +1,484 @@ +#include +#include +#include +#include + + +// /usr/intel/pkgs/icc/13.0.0e/bin/icc -o ed -O3 ed.cpp -xAVX -openmp -openmp-link static + + +#include + +#include +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std ; + +#define CHECK_MASK_CORRECTNESS + +template +string getBinaryStr (T val, int numBitsToWrite) { + + ostringstream oss ; + uint64_t mask = ((T) 0x1) << (numBitsToWrite-1) ; + for (int i=numBitsToWrite-1; i >= 0; --i) { + oss << ((val & mask) >> i) ; + mask >>= 1 ; + } + return oss.str() ; +} + + +typedef struct +{ + int rslen, haplen, *q, *i, *d, *c; + char *hap, *rs; +} testcase; + +int normalize(char c) +{ + return ((int) (c - 33)); +} + +class ConvertChar { + + static uint8_t conversionTable[255] ; + + +public: + + static void init() { + conversionTable['A'] = 0 ; + conversionTable['C'] = 1 ; + conversionTable['T'] = 2 ; + conversionTable['G'] = 3 ; + conversionTable['N'] = 4 ; + } + + static inline uint8_t get(uint8_t input) { + return conversionTable[input] ; + } + +} ; + +uint8_t ConvertChar::conversionTable[255] ; + + + +int read_testcase(testcase *tc, FILE* ifp) +{ + char *q, *i, *d, *c, *line = NULL; + int _q, _i, _d, _c; + int x, size = 0; + ssize_t read; + + read = getline(&line, (size_t *) &size, ifp); + if (read == -1) + return -1; + + + tc->hap = (char *) malloc(size); + tc->rs = (char *) malloc(size); + q = (char *) malloc(size); + i = (char *) malloc(size); + d = (char *) malloc(size); + c = (char *) malloc(size); + + if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) + return -1; + + tc->haplen = strlen(tc->hap); + tc->rslen = strlen(tc->rs); + tc->q = (int *) malloc(sizeof(int) * tc->rslen); + tc->i = (int *) malloc(sizeof(int) * tc->rslen); + tc->d = (int *) malloc(sizeof(int) * tc->rslen); + tc->c = (int *) malloc(sizeof(int) * tc->rslen); + + // Convert hap and rs to 3 bits + + /* + for (int ci=0; ci < tc->haplen; ++ci) + tc->hap[ci] = ConvertChar::get(tc->hap[ci]) ; + + for (int ci=0; ci < tc->rslen; ++ci) + tc->rs[ci] = ConvertChar::get(tc->rs[ci]) ; + */ + + for (x = 0; x < tc->rslen; x++) + { + _q = normalize(q[x]); + _i = normalize(i[x]); + _d = normalize(d[x]); + _c = normalize(c[x]); + tc->q[x] = (_q < 6) ? 6 : _q; + tc->i[x] = _i; + tc->d[x] = _d; + tc->c[x] = _c; + } + + + free(q); + free(i); + free(d); + free(c); + free(line); + + return 0; +} + + + + +#define ALIGN __attribute__ ((aligned(32))) + + +typedef union ALIGN { + //__m256i vi; + __m128 vf ; + __m128i vi ; + __m128d vd; + + uint8_t b[16]; + //uint16_t hw[8] ; + uint32_t w[4] ; + uint64_t dw[2] ; + + float f[4] ; + //double d[2] ; + +} v128 ; + +typedef union ALIGN { + __m256i vi; + __m256 vf ; + __m256d vd; + //__m128i vi128[2] ; + + uint8_t b[32]; + //uint16_t hw[16] ; + uint32_t w[8] ; + uint64_t dw[4] ; + + float f[8] ; + double d[4] ; + +} v256 ; + + +#define NUM_DISTINCT_CHARS 5 +#define AMBIG_CHAR 4 + +#define VEC_ENTRY_CNT 8 +#define VEC_LEN 256 +#define VTYPE vf +#define VTYPEI vi +#define VENTRY f +#define VENTRYI w +#define MTYPE uint32_t +#define MASK_ALL_ONES 0xFFFFFFFF + + +#define VECTOR v256 +#define VECTOR_SSE v128 + +#define SET_VEC_ZERO(__vec) \ + __vec= _mm256_setzero_ps() + +#define SET_VEC_ONES(__vec) \ + __vec = _mm256_set1_epi32(0xFFFFFFFF) + +#define VEC_OR(__v1, __v2) \ + _mm256_or_ps(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm256_add_ps(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm256_mul_ps(__v1, __v2) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm256_blendv_ps(__v1, __v2, __maskV) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castps128_ps256(__vsLow) ; \ + __vdst = _mm256_insertf128_ps(__vdst, __vsHigh, 1) ; + +#define VECS_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi32(__vs, 1) + + +#define VECS_SHIFT_RIGHT_WHOLE(__vs, __cnt) { \ + uint64_t __mask = ; \ + uint64_t __shiftWord = ((((uint64_t) 0x1) << __cnt) - 1 ) & __vs.dw[1] ; \ + __shiftWord <<= (64-cnt) ; \ + __vs = _mm_slri_epi64(__vs, __cnt) ; \ + __vs.dw[0] |= __shiftWord ; \ + } + + +#define GET_MASK_WORD_OLD(__mask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ + MTYPE __bitMask = (((MTYPE)0x1) << __shiftBy) - 1 ; \ + MTYPE __nextShiftOut = (__mask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ + __mask >>= __shiftBy ; \ + __mask |= __lastShiftOut ; \ + __lastShiftOut = __nextShiftOut ; \ + } + + +#define SET_MASK_WORD(__dstMask, __srcMask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ + MTYPE __bitMask = (((MTYPE)0x1) << __shiftBy) - 1 ; \ + MTYPE __nextShiftOut = (__srcMask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ + __dstMask = (__srcMask >> __shiftBy) | __lastShiftOut ; \ + __lastShiftOut = __nextShiftOut ; \ +} + + +void precompute_masks_avx(const testcase& tc, int COLS, int numMaskVecs, + MTYPE (*maskArr)[NUM_DISTINCT_CHARS]) { + + const int maskBitCnt = VEC_LEN / VEC_ENTRY_CNT ; + + for (int vi=0; vi < numMaskVecs; ++vi) { + for (int rs=0; rs < NUM_DISTINCT_CHARS; ++rs) { + maskArr[vi][rs] = 0 ; + } + maskArr[vi][AMBIG_CHAR] = MASK_ALL_ONES ; + } + + for (int col=1; col < COLS; ++col) { + int mIndex = (col-1) / maskBitCnt ; + int mOffset = (col-1) % maskBitCnt ; + MTYPE bitMask = ((MTYPE)0x1) << (maskBitCnt-1-mOffset) ; + + char hapChar = tc.hap[col-1] ; + + if (hapChar == AMBIG_CHAR) { + maskArr[mIndex][0] |= bitMask ; + maskArr[mIndex][1] |= bitMask ; + maskArr[mIndex][2] |= bitMask ; + maskArr[mIndex][3] |= bitMask ; + } + + //cout << hapChar << " " << mIndex << " " << getBinaryStr(bitMask, 32) + // << endl ; + //cout << getBinaryStr(maskArr[0][hapChar],32) << endl ; + //exit(0) ; + + maskArr[mIndex][ConvertChar::get(hapChar)] |= bitMask ; + + + // bit corresponding to col 1 will be the MSB of the mask 0 + // bit corresponding to col 2 will be the MSB-1 of the mask 0 + // ... + // bit corresponding to col 32 will be the LSB of the mask 0 + // bit corresponding to col 33 will be the MSB of the mask 1 + // ... + } + +} + + +void test_mask_computations (testcase& tc, int tcID, bool printDebug=false) { + + int ROWS = tc.rslen + 1 ; + int COLS = tc.haplen + 1 ; + + // only for testing + VECTOR mismatchData, matchData ; + //SET_VEC_ZERO(mismatchData.VTYPE) ; + //SET_VEC_ONES(matchData.VTYPEI) ; + for (int ei=0; ei < VEC_ENTRY_CNT; ++ei) { + matchData.VENTRY[ei] = 1.0 ; + mismatchData.VENTRY[ei] = 0.0 ; + } + + const int maskBitCnt = VEC_LEN / VEC_ENTRY_CNT ; + const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function + + MTYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; + precompute_masks_avx(tc, COLS, numMaskVecs, maskArr) ; + +#ifdef DEBUG + if (printDebug) { + cout << "The first 32 hap chars are: " ; + for (int i=0; i < 32; ++i) { + cout << tc.hap[i] ; + } + cout << endl ; + + cout << "Masks computed for A, C, T, G, N are: " << endl ; + cout << getBinaryStr(maskArr[0][0], 32) << endl ; + cout << getBinaryStr(maskArr[0][1], 32) << endl ; + cout << getBinaryStr(maskArr[0][2], 32) << endl ; + cout << getBinaryStr(maskArr[0][3], 32) << endl ; + cout << getBinaryStr(maskArr[0][4], 32) << endl ; + } +#endif // #ifdef DEBUG + + int beginRowIndex = 1 ; + while (beginRowIndex < ROWS) { + + int numRowsToProcess = min(VEC_ENTRY_CNT, ROWS - beginRowIndex) ; + + char rsArr[VEC_ENTRY_CNT] ; + for (int ri=0; ri < numRowsToProcess; ++ri) { + rsArr[ri] = ConvertChar::get(tc.rs[ri+beginRowIndex-1]) ; + } + + // Since there are no shift intrinsics in AVX, keep the masks in 2 SSE vectors + VECTOR_SSE currMaskVecLow ; // corresponding to entries 0-3 + VECTOR_SSE currMaskVecHigh ; // corresponding to entries 4-7 + + MTYPE lastMaskShiftOut[VEC_ENTRY_CNT] ; + for (int ei=0; ei < VEC_ENTRY_CNT; ++ei) + lastMaskShiftOut[ei] = 0 ; + + int col = 1 ; + int diag = 1 ; + for (int maskIndex=0; maskIndex < numMaskVecs; ++maskIndex) { + // set up the mask vectors for the next maskBitCnt columns + + // For AVX, maskBitCnt = 32 (so, the operation below is amortized over 32 cols) + for (int ei=0; ei < VEC_ENTRY_CNT/2; ++ei) { + SET_MASK_WORD(currMaskVecLow.VENTRYI[ei], maskArr[maskIndex][rsArr[ei]], + lastMaskShiftOut[ei], ei, maskBitCnt) ; + + int ei2 = ei + VEC_ENTRY_CNT/2 ; // the second entry index + SET_MASK_WORD(currMaskVecHigh.VENTRYI[ei], maskArr[maskIndex][rsArr[ei2]], + lastMaskShiftOut[ei2], ei2, maskBitCnt) ; + } + +#ifdef DEBUG + if (printDebug && maskIndex == 0) { + cout << "The masks for entry 1: " << endl + << getBinaryStr(maskArr[0][rsArr[1]], 32) << endl + << getBinaryStr(currMaskVecLow.VENTRYI[1], 32) << endl ; + + } +#endif // #ifdef DEBUG + + // iterate over mask bit indices and columns + for (int mbi=0; mbi < maskBitCnt && diag < COLS + ROWS -2; ++mbi, ++diag) { + + VECTOR maskV ; + VEC_SSE_TO_AVX(currMaskVecLow.VTYPE, currMaskVecHigh.VTYPE, maskV.VTYPE) ; + + VECTOR testData ; + testData.VTYPE = VEC_BLENDV(mismatchData.VTYPE, matchData.VTYPE, + maskV.VTYPE) ; + + VECS_SHIFT_LEFT_1BIT(currMaskVecLow.VTYPEI) ; + VECS_SHIFT_LEFT_1BIT(currMaskVecHigh.VTYPEI) ; + +#ifdef DEBUG + if (printDebug && maskIndex == 0) { + cout << "The mask for entry 1, mbi=" << mbi << ": " + << getBinaryStr(maskV.VENTRYI[1], 32) << endl ; + + } +#endif // #ifdef DEBUG + +#ifdef CHECK_MASK_CORRECTNESS + + int firstRowIndex = (diag < COLS) ? 0 : (diag - COLS) ; + int lastRowIndex = min(col-1, numRowsToProcess-1) ; + + for (int ri=firstRowIndex; ri <= lastRowIndex; ++ri) { + int currRow = beginRowIndex + ri ; + int currCol = col - ri + firstRowIndex ; + + char hapChar = tc.hap[currCol-1] ; + char rsChar = tc.rs[currRow-1] ; + + bool match = (hapChar == rsChar || hapChar == 'N' || rsChar == 'N') ; + + if ((bool) testData.VENTRYI[ri] != match) { + cout << "Error: Incorrect mask for tc " << tcID << ", diag = " << diag + << " (" << currRow << ", " << currCol << ")" << endl ; + + cout << "The chars are: " << hapChar << " and " << rsChar << endl ; + cout << "The selected value is: " << testData.VENTRYI[ri] << endl ; + + exit(0) ; + } + } +#endif // #ifdef CHECK_MASK_CORRECTNESS + + if (diag < COLS) + ++col ; + + } // mbi + } // maskIndex + + beginRowIndex += VEC_ENTRY_CNT ; + } // end of stripe + + //cout << "Finished validating entry " << endl ; +} + + +int main () { + + #define BATCH_SIZE 10000 + + ConvertChar::init() ; + + testcase* tcBatch = new testcase[BATCH_SIZE] ; + + int numBatches = 0 ; + int numRead = 0 ; + + const int DEBUG_TC = -1 ; + + FILE* ifp = stdin ; + + do { + numRead = 0 ; + + while (numRead < BATCH_SIZE) { + if (read_testcase(tcBatch+numRead, ifp) < 0) + break ; + + ++numRead ; + } + + if (numRead == 0) + break ; + + for (int ti=0; ti < numRead; ++ti) { + + int tcID = numBatches * BATCH_SIZE + ti ; + test_mask_computations(tcBatch[ti], tcID, tcID == DEBUG_TC) ; + } + + ++numBatches ; + } while (numRead == BATCH_SIZE) ; + + fclose(ifp) ; + + delete[] tcBatch ; + + return 0 ; +} diff --git a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc new file mode 100644 index 000000000..80065b8c9 --- /dev/null +++ b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -0,0 +1,146 @@ +#include +#include +#include +#include "org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "template.h" + +#include "define-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +using namespace std; + + +#define MM 0 +#define GapM 1 +#define MX 2 +#define XX 3 +#define MY 4 +#define YY 5 + +//#define DEBUG3 1 +#define DEBUG 1 + +template +string to_string(T obj) +{ + stringstream ss; + string ret_string; + ss.clear(); + ss << std::scientific << obj; + ss >> ret_string; + ss.clear(); + return ret_string; +} + +void debug_dump(string filename, string s, bool to_append, bool add_newline=true) +{ + ofstream fptr; + fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); + fptr << s; + if(add_newline) + fptr << "\n"; + fptr.close(); +} + +#define INT_STORE_ARRAY_SIZE 2048 +int insertionGOPIntArray[INT_STORE_ARRAY_SIZE]; +int deletionGOPIntArray[INT_STORE_ARRAY_SIZE]; +int overallGCPIntArray[INT_STORE_ARRAY_SIZE]; +int readQualsIntArray[INT_STORE_ARRAY_SIZE]; + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10( + JNIEnv* env, jobject thisObject, + jint readLength, jint haplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP, + jint hapStartIndex + ) +{ + jboolean is_copy = JNI_FALSE; + jbyte* readBasesArray = (env)->GetByteArrayElements(readBases, &is_copy); + jbyte* haplotypeBasesArray = (env)->GetByteArrayElements(haplotypeBases, &is_copy); + jbyte* readQualsArray = (env)->GetByteArrayElements(readQuals, &is_copy); + jbyte* insertionGOPArray = (env)->GetByteArrayElements(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (env)->GetByteArrayElements(deletionGOP, &is_copy); + jbyte* overallGCPArray = (env)->GetByteArrayElements(overallGCP, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); + assert(readLength < INT_STORE_ARRAY_SIZE); +#endif + for(unsigned i=0;i(&tc); + double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); +#ifdef DEBUG + debug_dump("return_values_jni.txt",to_string(result),true); +#endif + + + env->ReleaseByteArrayElements(overallGCP, overallGCPArray, JNI_ABORT); + env->ReleaseByteArrayElements(deletionGOP, deletionGOPArray, JNI_ABORT); + env->ReleaseByteArrayElements(insertionGOP, insertionGOPArray, JNI_ABORT); + env->ReleaseByteArrayElements(readQuals, readQualsArray, JNI_ABORT); + env->ReleaseByteArrayElements(haplotypeBases, haplotypeBasesArray, JNI_ABORT); + env->ReleaseByteArrayElements(readBases, readBasesArray, JNI_ABORT); + + return 0.0; +} + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeProbabilities +(JNIEnv* env, jclass thisObject, + jobjectArray transition, jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP + ) +{} + + JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitialize +(JNIEnv* env, jobject thisObject, + jint readMaxLength, jint haplotypeMaxLength) +{} + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializePriorsAndUpdateCells( + JNIEnv* env, jobject thisObject, + jboolean doInitialization, jint paddedReadLength, jint paddedHaplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jint hapStartIndex + ) + +{ return 0.0; } diff --git a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h new file mode 100644 index 000000000..5408d2f57 --- /dev/null +++ b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h @@ -0,0 +1,61 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM */ + +#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM +#define _Included_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM +#ifdef __cplusplus +extern "C" { +#endif +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_TRISTATE_CORRECTION +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_TRISTATE_CORRECTION 3.0 +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToMatch +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToMatch 0L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_indelToMatch +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_indelToMatch 1L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToInsertion +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToInsertion 2L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_insertionToInsertion +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_insertionToInsertion 3L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToDeletion +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToDeletion 4L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_deletionToDeletion +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_deletionToDeletion 5L +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniInitialize + * Signature: (II)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitialize + (JNIEnv *, jobject, jint, jint); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniInitializeProbabilities + * Signature: ([[D[B[B[B)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeProbabilities + (JNIEnv *, jclass, jobjectArray, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniInitializePriorsAndUpdateCells + * Signature: (ZII[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializePriorsAndUpdateCells + (JNIEnv *, jobject, jboolean, jint, jint, jbyteArray, jbyteArray, jbyteArray, jint); + + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniSubComputeReadLikelihoodGivenHaplotypeLog10 + * Signature: (II[B[B[B[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10 + (JNIEnv *, jobject, jint, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jint); + + +#ifdef __cplusplus +} +#endif +#endif diff --git a/pairhmm-1-base.cc b/pairhmm-1-base.cc new file mode 100644 index 000000000..dacd02536 --- /dev/null +++ b/pairhmm-1-base.cc @@ -0,0 +1,276 @@ +#include +#include +#include +#include + +#include + +#define MM 0 +#define GapM 1 +#define MX 2 +#define XX 3 +#define MY 4 +#define YY 5 + +//#define DEBUG + +/* + q: read quality + i: insertion penalty + d: deletion penalty + c: gap continuation penalty +*/ + +typedef struct +{ + int rslen, haplen, *q, *i, *d, *c; + char *hap, *rs; +} testcase; + +int normalize(char c) +{ + return ((int) (c - 33)); +} + +int read_testcase(testcase *tc) +{ + char *q, *i, *d, *c, *line = NULL; + int _q, _i, _d, _c; + int x, size = 0; + ssize_t read; + + read = getline(&line, (size_t *) &size, stdin); + if (read == -1) + return -1; + + tc->hap = (char *) malloc(size); + tc->rs = (char *) malloc(size); + q = (char *) malloc(size); + i = (char *) malloc(size); + d = (char *) malloc(size); + c = (char *) malloc(size); + + if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) + return -1; + + tc->haplen = strlen(tc->hap); + tc->rslen = strlen(tc->rs); + tc->q = (int *) malloc(sizeof(int) * tc->rslen); + tc->i = (int *) malloc(sizeof(int) * tc->rslen); + tc->d = (int *) malloc(sizeof(int) * tc->rslen); + tc->c = (int *) malloc(sizeof(int) * tc->rslen); + + for (x = 0; x < tc->rslen; x++) + { + _q = normalize(q[x]); + _i = normalize(i[x]); + _d = normalize(d[x]); + _c = normalize(c[x]); + tc->q[x] = (_q < 6) ? 6 : _q; + tc->i[x] = _i; + tc->d[x] = _d; + tc->c[x] = _c; + } + + free(q); + free(i); + free(d); + free(c); + free(line); + + return 0; +} + +template +struct Context{}; + +template<> +struct Context +{ + Context() + { + for (int x = 0; x < 128; x++) + ph2pr[x] = pow(10.0, -((double)x) / 10.0); + + INITIAL_CONSTANT = ldexp(1.0, 1020.0); + LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); + RESULT_THRESHOLD = 0.0; + } + + double LOG10(double v){ return log10(v); } + + static double _(double n){ return n; } + static double _(float n){ return ((double) n); } + double ph2pr[128]; + double INITIAL_CONSTANT; + double LOG10_INITIAL_CONSTANT; + double RESULT_THRESHOLD; +}; + +template<> +struct Context +{ + Context() + { + for (int x = 0; x < 128; x++) + ph2pr[x] = powf(10.f, -((float)x) / 10.f); + + INITIAL_CONSTANT = ldexpf(1.f, 120.f); + LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); + RESULT_THRESHOLD = ldexpf(1.f, -110.f); + } + + float LOG10(float v){ return log10f(v); } + + static float _(double n){ return ((float) n); } + static float _(float n){ return n; } + float ph2pr[128]; + float INITIAL_CONSTANT; + float LOG10_INITIAL_CONSTANT; + float RESULT_THRESHOLD; +}; + +template +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) +{ + int r, c; + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + + Context ctx; + + NUMBER M[ROWS][COLS]; + NUMBER X[ROWS][COLS]; + NUMBER Y[ROWS][COLS]; + NUMBER p[ROWS][6]; + + p[0][MM] = ctx._(0.0); + p[0][GapM] = ctx._(0.0); + p[0][MX] = ctx._(0.0); + p[0][XX] = ctx._(0.0); + p[0][MY] = ctx._(0.0); + p[0][YY] = ctx._(0.0); + for (r = 1; r < ROWS; r++) + { + int _i = tc->i[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; + p[r][MX] = ctx.ph2pr[_i]; + p[r][XX] = ctx.ph2pr[_c]; + p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + } + + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); + } + + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } + + NUMBER result = ctx._(0.0); + + for (r = 1; r < ROWS; r++) + for (c = 1; c < COLS; c++) + { + char _rs = tc->rs[r-1]; + char _hap = tc->hap[c-1]; + int _q = tc->q[r-1] & 127; + NUMBER distm = ctx.ph2pr[_q]; + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; + } + + for (c = 0; c < COLS; c++) + { + result += M[ROWS-1][c] + X[ROWS-1][c]; + } + + if (before_last_log != NULL) + *before_last_log = result; + + return result; //ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; +} + +#define BATCH_SIZE 10000 +#define RUN_HYBRID + +int main() +{ + testcase tc[BATCH_SIZE]; + float result[BATCH_SIZE], result_avxf; + double result_avxd; + struct timeval start, end; + long long aggregateTimeRead = 0L; + long long aggregateTimeCompute = 0L; + long long aggregateTimeWrite = 0L; + + bool noMoreData = false; + int count =0; + while (!noMoreData) + { + int read_count = BATCH_SIZE; + gettimeofday(&start, NULL); + for (int b=0;b(&tc[b]); + + #ifdef RUN_HYBRID + #define MIN_ACCEPTED 1e-28f + if (result_avxf < MIN_ACCEPTED) { + count++; + result_avxd = compute_full_prob(&tc[b]); + result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); + } + else + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); + #endif + + #ifndef RUN_HYBRID + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); + #endif + + } + gettimeofday(&end, NULL); + aggregateTimeCompute += ((end.tv_sec * 1000000 + end.tv_usec) - (start.tv_sec * 1000000 + start.tv_usec)); + + gettimeofday(&start, NULL); + for (int b=0;b +#include +#include + +/* +template +string getBinaryStr (T val, int numBitsToWrite) { + + ostringstream oss ; + uint64_t mask = ((T) 0x1) << (numBitsToWrite-1) ; + for (int i=numBitsToWrite-1; i >= 0; --i) { + oss << ((val & mask) >> i) ; + mask >>= 1 ; + } + return oss.str() ; +} +*/ + +void GEN_INTRINSIC(precompute_masks_, PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]) { + + const int maskBitCnt = MAIN_TYPE_SIZE ; + + for (int vi=0; vi < numMaskVecs; ++vi) { + for (int rs=0; rs < NUM_DISTINCT_CHARS; ++rs) { + maskArr[vi][rs] = 0 ; + } + maskArr[vi][AMBIG_CHAR] = MASK_ALL_ONES ; + } + + for (int col=1; col < COLS; ++col) { + int mIndex = (col-1) / maskBitCnt ; + int mOffset = (col-1) % maskBitCnt ; + MASK_TYPE bitMask = ((MASK_TYPE)0x1) << (maskBitCnt-1-mOffset) ; + + char hapChar = tc.hap[col-1] ; + + if (hapChar == AMBIG_CHAR) { + for (int ci=0; ci < NUM_DISTINCT_CHARS; ++ci) + maskArr[mIndex][ci] |= bitMask ; + } + + maskArr[mIndex][ConvertChar::get(hapChar)] |= bitMask ; + // bit corresponding to col 1 will be the MSB of the mask 0 + // bit corresponding to col 2 will be the MSB-1 of the mask 0 + // ... + // bit corresponding to col 32 will be the LSB of the mask 0 + // bit corresponding to col 33 will be the MSB of the mask 1 + // ... + } + +} + +void GEN_INTRINSIC(init_masks_for_row_, PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess) { + + for (int ri=0; ri < numRowsToProcess; ++ri) { + rsArr[ri] = ConvertChar::get(tc.rs[ri+beginRowIndex-1]) ; + } + + for (int ei=0; ei < AVX_LENGTH; ++ei) { + lastMaskShiftOut[ei] = 0 ; + } +} + +#define SET_MASK_WORD(__dstMask, __srcMask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ + MASK_TYPE __bitMask = (((MASK_TYPE)0x1) << __shiftBy) - 1 ; \ + MASK_TYPE __nextShiftOut = (__srcMask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ + __dstMask = (__srcMask >> __shiftBy) | __lastShiftOut ; \ + __lastShiftOut = __nextShiftOut ; \ +} + + +void GEN_INTRINSIC(update_masks_for_cols_, PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt) { + + for (int ei=0; ei < AVX_LENGTH/2; ++ei) { + SET_MASK_WORD(currMaskVecLow.masks[ei], maskArr[maskIndex][rsArr[ei]], + lastMaskShiftOut[ei], ei, maskBitCnt) ; + + int ei2 = ei + AVX_LENGTH/2 ; // the second entry index + SET_MASK_WORD(currMaskVecHigh.masks[ei], maskArr[maskIndex][rsArr[ei2]], + lastMaskShiftOut[ei2], ei2, maskBitCnt) ; + } + +} + +//void GEN_INTRINSIC(computeDistVec, PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen, const _256_TYPE& distmSel, int firstRowIndex, int lastRowIndex) { + +inline void GEN_INTRINSIC(computeDistVec, PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen) { + //#define computeDistVec() { + + _256_TYPE maskV ; + VEC_SSE_TO_AVX(currMaskVecLow.vecf, currMaskVecHigh.vecf, maskV) ; + + distmChosen = VEC_BLENDV(distm, _1_distm, maskV) ; + + /*COMPARE_VECS(distmChosen, distmSel, firstRowIndex, lastRowIndex) ;*/ + + VEC_SHIFT_LEFT_1BIT(currMaskVecLow.vec) ; + VEC_SHIFT_LEFT_1BIT(currMaskVecHigh.vec) ; +} + + +/* +template +struct HmmData { + int ROWS ; + int COLS ; + + NUMBER shiftOutM[MROWS+MCOLS+AVX_LENGTH], shiftOutX[MROWS+MCOLS+AVX_LENGTH], shiftOutY[MROWS+MCOLS+AVX_LENGTH] ; + Context ctx ; + testcase* tc ; + _256_TYPE p_MM[MAVX_COUNT], p_GAPM[MAVX_COUNT], p_MX[MAVX_COUNT], p_XX[MAVX_COUNT], p_MY[MAVX_COUNT], p_YY[MAVX_COUNT], distm1D[MAVX_COUNT] ; + _256_TYPE pGAPM, pMM, pMX, pXX, pMY, pYY ; + + UNION_TYPE M_t, M_t_1, M_t_2, X_t, X_t_1, X_t_2, Y_t, Y_t_1, Y_t_2, M_t_y, M_t_1_y ; + UNION_TYPE rs , rsN ; + _256_TYPE distmSel; + _256_TYPE distm, _1_distm; + +} ; +*/ + +template void GEN_INTRINSIC(initializeVectors, PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D) +{ + NUMBER zero = ctx._(0.0); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + for (int s=0;si[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + + *(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; + *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; + *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; + *(ptr_p_MY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + *(ptr_p_YY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + } + + NUMBER *ptr_distm1D = (NUMBER *)distm1D; + for (int r = 1; r < ROWS; r++) + { + int _q = tc->q[r-1] & 127; + ptr_distm1D[r-1] = ctx.ph2pr[_q]; + } +} + + +template inline void GEN_INTRINSIC(stripINITIALIZATION, PRECISION)( + int stripIdx, Context ctx, testcase *tc, _256_TYPE &pGAPM, _256_TYPE &pMM, _256_TYPE &pMX, _256_TYPE &pXX, _256_TYPE &pMY, _256_TYPE &pYY, + _256_TYPE &rs, UNION_TYPE &rsN, _256_TYPE &distm, _256_TYPE &_1_distm, _256_TYPE *distm1D, _256_TYPE N_packed256, _256_TYPE *p_MM , _256_TYPE *p_GAPM , + _256_TYPE *p_MX, _256_TYPE *p_XX , _256_TYPE *p_MY, _256_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM) +{ + int i = stripIdx; + pGAPM = p_GAPM[i]; + pMM = p_MM[i]; + pMX = p_MX[i]; + pXX = p_XX[i]; + pMY = p_MY[i]; + pYY = p_YY[i]; + + NUMBER zero = ctx._(0.0); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + + /* compare rs and N */ + //rs = VEC_LDPOPCVT_CHAR((tc->irs+i*AVX_LENGTH)); + //rsN.d = VEC_CMP_EQ(N_packed256, rs); + + distm = distm1D[i]; + _1_distm = VEC_SUB(packed1.d, distm); + + /* initialize M_t_2, M_t_1, X_t_2, X_t_1, Y_t_2, Y_t_1 */ + M_t_2.d = VEC_SET1_VAL(zero); + X_t_2.d = VEC_SET1_VAL(zero); + + if (i==0) { + M_t_1.d = VEC_SET1_VAL(zero); + X_t_1.d = VEC_SET1_VAL(zero); + Y_t_2.d = VEC_SET_LSE(init_Y); + Y_t_1.d = VEC_SET1_VAL(zero); + } + else { + X_t_1.d = VEC_SET_LSE(shiftOutX[AVX_LENGTH]); + M_t_1.d = VEC_SET_LSE(shiftOutM[AVX_LENGTH]); + Y_t_2.d = VEC_SET1_VAL(zero); + Y_t_1.d = VEC_SET1_VAL(zero); + } + M_t_1_y = M_t_1; +} + + + +inline _256_TYPE GEN_INTRINSIC(computeDISTM, PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, + _256_TYPE distm, _256_TYPE _1_distm) +{ + UNION_TYPE hapN, rshap; + _256_TYPE cond; + IF_32 shiftInHap; + + int *hap_ptr = tc->ihap; + + shiftInHap.i = (d NUMBER GEN_INTRINSIC(compute_full_prob_avx, PRECISION) (testcase *tc, NUMBER *before_last_log = NULL) +{ + _256_TYPE p_MM [MAVX_COUNT], p_GAPM [MAVX_COUNT], p_MX [MAVX_COUNT]; + _256_TYPE p_XX [MAVX_COUNT], p_MY [MAVX_COUNT], p_YY [MAVX_COUNT]; + _256_TYPE distm1D[MAVX_COUNT]; + NUMBER shiftOutM[MROWS+MCOLS+AVX_LENGTH], shiftOutX[MROWS+MCOLS+AVX_LENGTH], shiftOutY[MROWS+MCOLS+AVX_LENGTH]; + UNION_TYPE M_t, M_t_1, M_t_2, X_t, X_t_1, X_t_2, Y_t, Y_t_1, Y_t_2, M_t_y, M_t_1_y; + _256_TYPE pGAPM, pMM, pMX, pXX, pMY, pYY; + + struct timeval start, end; + NUMBER result_avx2; + Context ctx; + UNION_TYPE rs , rsN; + HAP_TYPE hap; + _256_TYPE distmSel, distmChosen ; + _256_TYPE distm, _1_distm; + + int r, c; + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + int AVX_COUNT = (ROWS+7)/8; + NUMBER zero = ctx._(0.0); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + _256_TYPE N_packed256 = VEC_POPCVT_CHAR('N'); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + int remainingRows = (ROWS-1) % AVX_LENGTH; + int strip_cnt = ((ROWS-1) / AVX_LENGTH) + (remainingRows!=0); + + const int maskBitCnt = MAIN_TYPE_SIZE ; + const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function + + MASK_TYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; + GEN_INTRINSIC(precompute_masks_, PRECISION)(*tc, COLS, numMaskVecs, maskArr) ; + + char rsArr[AVX_LENGTH] ; + MASK_TYPE lastMaskShiftOut[AVX_LENGTH] ; + + GEN_INTRINSIC(initializeVectors, PRECISION)(ROWS, COLS, shiftOutM, shiftOutX, shiftOutY, + ctx, tc, p_MM, p_GAPM, p_MX, p_XX, p_MY, p_YY, distm1D); + + //for (int __ii=0; __ii < 10; ++__ii) + for (int i=0;i +#include +#include +#include + +#include "template.h" + +#include "define-float.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +#include "define-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +#define BATCH_SIZE 10000 +//#define RUN_HYBRID + +uint8_t ConvertChar::conversionTable[255] ; +int thread_level_parallelism_enabled = false ; + +double getCurrClk() { + struct timeval tv ; + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} + +int main() +{ + testcase tc[BATCH_SIZE]; + float result[BATCH_SIZE], result_avxf; + double result_avxd; + //struct timeval start, end; + double lastClk = 0.0 ; + double aggregateTimeRead = 0.0; + double aggregateTimeCompute = 0.0; + double aggregateTimeWrite = 0.0; + + // Need to call it once to initialize the static array + ConvertChar::init() ; + + + char* ompEnvVar = getenv("OMP_NUM_THREADS") ; + if (ompEnvVar != NULL && ompEnvVar != "" && ompEnvVar != "1" ) { + thread_level_parallelism_enabled = true ; + } + + bool noMoreData = false; + int count =0; + while (!noMoreData) + { + int read_count = BATCH_SIZE; + + lastClk = getCurrClk() ; + for (int b=0;b(&tc[b]); + + #ifdef RUN_HYBRID + #define MIN_ACCEPTED 1e-28f + if (result_avxf < MIN_ACCEPTED) { + count++; + result_avxd = GEN_INTRINSIC(compute_full_prob_avx, d)(&tc[b]); + result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); + } + else + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); + #endif + + #ifndef RUN_HYBRID + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); + #endif + + } + //gettimeofday(&end, NULL); + aggregateTimeCompute += (getCurrClk() - lastClk) ; + //((end.tv_sec * 1000000 + end.tv_usec) - (start.tv_sec * 1000000 + start.tv_usec)); + + //gettimeofday(&start, NULL); + lastClk = getCurrClk() ; + for (int b=0;b +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#define MROWS 500 +#define MCOLS 1000 + +#define CAT(X,Y) X####Y +#define GEN_INTRINSIC(X,Y) CAT(X,Y) + +#define ALIGNED __attribute__((aligned(32))) + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256 ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED float ALIGNED f[8]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_F ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128 vecf ; + uint32_t masks[4] ; +} MaskVec_F ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128 ALIGNED f; +} ALIGNED IF_128f ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int ALIGNED i; + ALIGNED float ALIGNED f; +} ALIGNED IF_32 ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256d ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED double ALIGNED f[4]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_D ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128d vecf ; + uint64_t masks[2] ; +} MaskVec_D ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128d ALIGNED f; +} ALIGNED IF_128d ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int64_t ALIGNED i; + ALIGNED double ALIGNED f; +} ALIGNED IF_64 ALIGNED; + +template +struct Context{}; + +template<> +struct Context +{ + Context() + { + for (int x = 0; x < 128; x++) + ph2pr[x] = pow(10.0, -((double)x) / 10.0); + + INITIAL_CONSTANT = ldexp(1.0, 1020.0); + LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); + RESULT_THRESHOLD = 0.0; + } + + double LOG10(double v){ return log10(v); } + + static double _(double n){ return n; } + static double _(float n){ return ((double) n); } + double ph2pr[128]; + double INITIAL_CONSTANT; + double LOG10_INITIAL_CONSTANT; + double RESULT_THRESHOLD; +}; + +template<> +struct Context +{ + Context() + { + for (int x = 0; x < 128; x++) + { + ph2pr[x] = powf(10.f, -((float)x) / 10.f); + } + + INITIAL_CONSTANT = ldexpf(1.f, 120.f); + LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); + RESULT_THRESHOLD = ldexpf(1.f, -110.f); + } + + float LOG10(float v){ return log10f(v); } + + static float _(double n){ return ((float) n); } + static float _(float n){ return n; } + float ph2pr[128]; + float INITIAL_CONSTANT; + float LOG10_INITIAL_CONSTANT; + float RESULT_THRESHOLD; +}; + + +typedef struct +{ + int rslen, haplen, *q, *i, *d, *c; + char *hap, *rs; + int *ihap; + int *irs; +} testcase; + +int normalize(char c); + + +int read_testcase(testcase *tc) +{ + char *q, *i, *d, *c, *line = NULL; + int _q, _i, _d, _c; + int x, size = 0; + ssize_t read; + + read = getline(&line, (size_t *) &size, stdin); + if (read == -1) + return -1; + + tc->hap = (char *) malloc(size); + tc->rs = (char *) malloc(size); + q = (char *) malloc(size); + i = (char *) malloc(size); + d = (char *) malloc(size); + c = (char *) malloc(size); + tc->ihap = (int *) malloc(size*sizeof(int)); + tc->irs = (int *) malloc(size*sizeof(int)); + + + if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) + return -1; + + tc->haplen = strlen(tc->hap); + tc->rslen = strlen(tc->rs); + tc->q = (int *) malloc(sizeof(int) * tc->rslen); + tc->i = (int *) malloc(sizeof(int) * tc->rslen); + tc->d = (int *) malloc(sizeof(int) * tc->rslen); + tc->c = (int *) malloc(sizeof(int) * tc->rslen); + + tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); + tc->irs = (int *) malloc(tc->rslen*sizeof(int)); + + for (x = 0; x < tc->rslen; x++) + { + _q = normalize(q[x]); + _i = normalize(i[x]); + _d = normalize(d[x]); + _c = normalize(c[x]); + tc->q[x] = (_q < 6) ? 6 : _q; + tc->i[x] = _i; + tc->d[x] = _d; + tc->c[x] = _c; + tc->irs[x] = tc->rs[x]; + } + + for (x = 0; x < tc->haplen; x++) + tc->ihap[x] = tc->hap[x]; + + free(q); + free(i); + free(d); + free(c); + free(line); + + return 0; +} + +#define NUM_DISTINCT_CHARS 5 +#define AMBIG_CHAR 4 + +class ConvertChar { + + static uint8_t conversionTable[255] ; + +public: + + static void init() { + assert (NUM_DISTINCT_CHARS == 5) ; + assert (AMBIG_CHAR == 4) ; + + conversionTable['A'] = 0 ; + conversionTable['C'] = 1 ; + conversionTable['T'] = 2 ; + conversionTable['G'] = 3 ; + conversionTable['N'] = 4 ; + } + + static inline uint8_t get(uint8_t input) { + return conversionTable[input] ; + } + +} ; + + +#endif + + From 8240ea826ebe1f47ba7548cb8971f4d5803b0fbb Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Wed, 15 Jan 2014 10:48:58 -0800 Subject: [PATCH 02/72] Changes: 1. Added TRISTATE_CORRECTION in pairhmm-template-kernel.cc (function stripINITIALIZATION) 2. Added VEC_DIV macros to define-double.h and define-float.h 3. Edited initializeVectors to match Java C++ original: *(ptr_p_MY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; *(ptr_p_YY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; Modified: *(ptr_p_MY+r-1) = ctx.ph2pr[_d]; *(ptr_p_YY+r-1) = ctx.ph2pr[_c]; --- .gitignore | 2 ++ Makefile | 19 +++++++------ define-double.h | 4 +++ define-float.h | 4 +++ hmm_mask.cc | 3 +- ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 28 +++++++++---------- pairhmm-template-kernel.cc | 24 ++++++++++++++-- pairhmm-template-main.cc | 2 +- template.h | 15 ++++++++++ 9 files changed, 72 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 16e86f25d..dcc44daa4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ tests .deps hmm_Mohammad +pairhmm-template-main +*.swp diff --git a/Makefile b/Makefile index df5dbc463..f858a7241 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -#OMPCFLAGS=-fopenmp +OMPCFLAGS=-fopenmp #OMPLDFLAGS=-lgomp #CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas @@ -7,7 +7,7 @@ JAVA_ROOT=/opt/jdk1.7.0_25/ JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JAVA_ROOT}/include -I${JAVA_ROOT}/include/linux -CFLAGS=-g -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas -xAVX +CFLAGS=-O3 -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas -xAVX CXXFLAGS=$(CFLAGS) CC=icc @@ -16,11 +16,12 @@ CXX=icc LDFLAGS=-lm $(OMPLDFLAGS) #BIN:=pairhmm-1-base #pairhmm-2-omp pairhmm-3-hybrid-float-double pairhmm-4-hybrid-diagonal pairhmm-5-hybrid-diagonal-homogeneus pairhmm-6-onlythreediags pairhmm-7-presse pairhmm-8-sse #pairhmm-dev -BIN:=libJNILoglessPairHMM.so #pairhmm-2-omp pairhmm-3-hybrid-float-double pairhmm-4-hybrid-diagonal pairhmm-5-hybrid-diagonal-homogeneus pairhmm-6-onlythreediags pairhmm-7-presse pairhmm-8-sse #pairhmm-dev +BIN:=libJNILoglessPairHMM.so pairhmm-template-main #SOURCES=pairhmm-1-base.cc input.cc -SOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc hmm_mask.cc -OBJECTS=$(SOURCES:.cc=.o) +LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc hmm_mask.cc +SOURCES=$(LIBSOURCES) pairhmm-template-main.cc +LIBOBJECTS=$(LIBSOURCES:.cc=.o) DEPDIR=.deps DF=$(DEPDIR)/$(*).d @@ -28,11 +29,11 @@ all: $(BIN) -include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) -pairhmm-1-base: pairhmm-1-base.o input.o - $(CXX) -o $@ $^ $(LDFLAGS) +pairhmm-template-main: pairhmm-template-main.o hmm_mask.o + $(CXX) -fopenmp -o $@ $^ $(LDFLAGS) -libJNILoglessPairHMM.so: $(OBJECTS) - $(CXX) -shared -o libJNILoglessPairHMM.so $(OBJECTS) +libJNILoglessPairHMM.so: $(LIBOBJECTS) + $(CXX) -shared -o $@ $(LIBOBJECTS) %.o: %.cc @mkdir -p $(DEPDIR) diff --git a/define-double.h b/define-double.h index b2de7520a..5d31741d6 100644 --- a/define-double.h +++ b/define-double.h @@ -23,6 +23,7 @@ #undef VEC_ADD #undef VEC_SUB #undef VEC_MUL + #undef VEC_DIV #undef VEC_BLEND #undef VEC_BLENDV #undef VEC_CAST_256_128 @@ -82,6 +83,9 @@ #define VEC_MUL(__v1, __v2) \ _mm256_mul_pd(__v1, __v2) +#define VEC_DIV(__v1, __v2) \ + _mm256_div_pd(__v1, __v2) + #define VEC_BLEND(__v1, __v2, __mask) \ _mm256_blend_pd(__v1, __v2, __mask) diff --git a/define-float.h b/define-float.h index 4cba9a179..0e2822b2d 100644 --- a/define-float.h +++ b/define-float.h @@ -23,6 +23,7 @@ #undef VEC_ADD(__v1, __v2) #undef VEC_SUB(__v1, __v2) #undef VEC_MUL(__v1, __v2) + #undef VEC_DIV(__v1, __v2) #undef VEC_BLEND(__v1, __v2, __mask) #undef VEC_BLENDV(__v1, __v2, __maskV) #undef VEC_CAST_256_128(__v1) @@ -83,6 +84,9 @@ #define VEC_MUL(__v1, __v2) \ _mm256_mul_ps(__v1, __v2) +#define VEC_DIV(__v1, __v2) \ + _mm256_div_ps(__v1, __v2) + #define VEC_BLEND(__v1, __v2, __mask) \ _mm256_blend_ps(__v1, __v2, __mask) diff --git a/hmm_mask.cc b/hmm_mask.cc index aace7acf2..f128a39d9 100644 --- a/hmm_mask.cc +++ b/hmm_mask.cc @@ -438,7 +438,7 @@ void test_mask_computations (testcase& tc, int tcID, bool printDebug=false) { //cout << "Finished validating entry " << endl ; } - +#ifdef HMM_MASK_MAIN int main () { #define BATCH_SIZE 10000 @@ -482,3 +482,4 @@ int main () { return 0 ; } +#endif diff --git a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index 80065b8c9..29668a306 100644 --- a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -15,6 +15,10 @@ #include #include #include +using namespace std; +//#define DEBUG3 1 +#define DEBUG 1 + #include "template.h" @@ -22,7 +26,6 @@ #include "shift_template.c" #include "pairhmm-template-kernel.cc" -using namespace std; #define MM 0 @@ -32,22 +35,17 @@ using namespace std; #define MY 4 #define YY 5 -//#define DEBUG3 1 -#define DEBUG 1 - -template -string to_string(T obj) +class LoadTimeInitializer { - stringstream ss; - string ret_string; - ss.clear(); - ss << std::scientific << obj; - ss >> ret_string; - ss.clear(); - return ret_string; -} + public: + LoadTimeInitializer() //will be called when library is loaded + { + ConvertChar::init(); + } +}; +LoadTimeInitializer g_load_time_initializer; -void debug_dump(string filename, string s, bool to_append, bool add_newline=true) +void debug_dump(string filename, string s, bool to_append, bool add_newline) { ofstream fptr; fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); diff --git a/pairhmm-template-kernel.cc b/pairhmm-template-kernel.cc index dd71373b0..24bcf7d1a 100644 --- a/pairhmm-template-kernel.cc +++ b/pairhmm-template-kernel.cc @@ -157,8 +157,18 @@ template void GEN_INTRINSIC(initializeVectors, PRECISION)(int ROWS *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; - *(ptr_p_MY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; - *(ptr_p_YY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + *(ptr_p_MY+r-1) = ctx.ph2pr[_d]; + *(ptr_p_YY+r-1) = ctx.ph2pr[_c]; +#ifdef DEBUG3 + debug_dump("transitions_jni.txt",to_string(*(ptr_p_MM+r-1) ),true); + debug_dump("transitions_jni.txt",to_string(*(ptr_p_GAPM+r-1)),true); + debug_dump("transitions_jni.txt",to_string(*(ptr_p_MX+r-1) ),true); + debug_dump("transitions_jni.txt",to_string(*(ptr_p_XX+r-1) ),true); + debug_dump("transitions_jni.txt",to_string(*(ptr_p_MY+r-1) ),true); + debug_dump("transitions_jni.txt",to_string(*(ptr_p_YY+r-1) ),true); +#endif + //*(ptr_p_MY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + //*(ptr_p_YY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; } NUMBER *ptr_distm1D = (NUMBER *)distm1D; @@ -166,6 +176,9 @@ template void GEN_INTRINSIC(initializeVectors, PRECISION)(int ROWS { int _q = tc->q[r-1] & 127; ptr_distm1D[r-1] = ctx.ph2pr[_q]; +#ifdef DEBUG3 + debug_dump("priors_jni.txt",to_string(ptr_distm1D[r-1]),true); +#endif } } @@ -187,13 +200,18 @@ template inline void GEN_INTRINSIC(stripINITIALIZATION, PRECISION) NUMBER zero = ctx._(0.0); NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); +#define TRISTATE_CORRECTION_FACTOR 3.0 + UNION_TYPE packed3; packed3.d = VEC_SET1_VAL(TRISTATE_CORRECTION_FACTOR); /* compare rs and N */ //rs = VEC_LDPOPCVT_CHAR((tc->irs+i*AVX_LENGTH)); //rsN.d = VEC_CMP_EQ(N_packed256, rs); distm = distm1D[i]; - _1_distm = VEC_SUB(packed1.d, distm); + _1_distm = VEC_SUB(packed1.d, distm); +#ifndef DO_NOT_USE_TRISTATE_CORRECTION + distm = VEC_DIV(distm, packed3.d); +#endif /* initialize M_t_2, M_t_1, X_t_2, X_t_1, Y_t_2, Y_t_1 */ M_t_2.d = VEC_SET1_VAL(zero); diff --git a/pairhmm-template-main.cc b/pairhmm-template-main.cc index f20755b6b..87d50eb5b 100644 --- a/pairhmm-template-main.cc +++ b/pairhmm-template-main.cc @@ -16,7 +16,7 @@ #define BATCH_SIZE 10000 //#define RUN_HYBRID -uint8_t ConvertChar::conversionTable[255] ; +//uint8_t ConvertChar::conversionTable[255] ; int thread_level_parallelism_enabled = false ; double getCurrClk() { diff --git a/template.h b/template.h index ee8036a11..4c8a4b5e3 100644 --- a/template.h +++ b/template.h @@ -13,6 +13,7 @@ #include #include +#include #define MROWS 500 #define MCOLS 1000 @@ -132,6 +133,20 @@ typedef struct int *irs; } testcase; + +template +std::string to_string(T obj) +{ + stringstream ss; + string ret_string; + ss.clear(); + ss << std::scientific << obj; + ss >> ret_string; + ss.clear(); + return ret_string; +} +void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); + int normalize(char c); From e8a502277731e7cab4ada76b3a413bfc04aa6417 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Wed, 15 Jan 2014 11:07:09 -0800 Subject: [PATCH 03/72] 1. Added support for JNI integration for LoglessCaching PairHMM AVX implementation. 2. Contains lots of debug code 3. Only invokes JNI for subComputeReadLikelihoodGivenHaplotypeLog10 --- build.xml | 14 +- .../PairHMMLikelihoodCalculationEngine.java | 41 +- .../utils/pairhmm/JNILoglessPairHMM.java | 382 ++++++++++++++++++ .../sting/utils/pairhmm/LoglessPairHMM.java | 48 +++ 4 files changed, 459 insertions(+), 26 deletions(-) create mode 100644 protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java diff --git a/build.xml b/build.xml index fd0801bfb..943b082cb 100644 --- a/build.xml +++ b/build.xml @@ -270,18 +270,18 @@ - + - + + + - + + + diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index b1db23d74..0f2401df6 100644 --- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -81,25 +81,28 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation private final boolean noFpga; private final ThreadLocal pairHMMThreadLocal = new ThreadLocal() { - @Override - protected PairHMM initialValue() { - switch (hmmType) { - case EXACT: return new Log10PairHMM(true); - case ORIGINAL: return new Log10PairHMM(false); - case LOGLESS_CACHING: - if (noFpga || !CnyPairHMM.isAvailable()) - return new LoglessPairHMM(); - else - return new CnyPairHMM(); - case ARRAY_LOGLESS: - if (noFpga || !CnyPairHMM.isAvailable()) - return new ArrayLoglessPairHMM(); - else - return new CnyPairHMM(); - default: - throw new UserException.BadArgumentValue("pairHMM", "Specified pairHMM implementation is unrecognized or incompatible with the HaplotypeCaller. Acceptable options are ORIGINAL, EXACT, CACHING, LOGLESS_CACHING, and ARRAY_LOGLESS."); - } - } + @Override + protected PairHMM initialValue() { + switch (hmmType) { + case EXACT: return new Log10PairHMM(true); + case ORIGINAL: return new Log10PairHMM(false); + case LOGLESS_CACHING: + if (noFpga || !CnyPairHMM.isAvailable()) + { + //return new LoglessPairHMM(); + return new JNILoglessPairHMM(); + } + else + return new CnyPairHMM(); + case ARRAY_LOGLESS: + if (noFpga || !CnyPairHMM.isAvailable()) + return new ArrayLoglessPairHMM(); + else + return new CnyPairHMM(); + default: + throw new UserException.BadArgumentValue("pairHMM", "Specified pairHMM implementation is unrecognized or incompatible with the HaplotypeCaller. Acceptable options are ORIGINAL, EXACT, CACHING, LOGLESS_CACHING, and ARRAY_LOGLESS."); + } + } }; // Attempted to do as below, to avoid calling pairHMMThreadLocal.get() later on, but it resulted in a NullPointerException // private final PairHMM pairHMM = pairHMMThreadLocal.get(); diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java new file mode 100644 index 000000000..6a19baf55 --- /dev/null +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java @@ -0,0 +1,382 @@ +/* +* By downloading the PROGRAM you agree to the following terms of use: +* +* BROAD INSTITUTE - SOFTWARE LICENSE AGREEMENT - FOR ACADEMIC NON-COMMERCIAL RESEARCH PURPOSES ONLY +* +* This Agreement is made between the Broad Institute, Inc. with a principal address at 7 Cambridge Center, Cambridge, MA 02142 (BROAD) and the LICENSEE and is effective at the date the downloading is completed (EFFECTIVE DATE). +* +* WHEREAS, LICENSEE desires to license the PROGRAM, as defined hereinafter, and BROAD wishes to have this PROGRAM utilized in the public interest, subject only to the royalty-free, nonexclusive, nontransferable license rights of the United States Government pursuant to 48 CFR 52.227-14; and +* WHEREAS, LICENSEE desires to license the PROGRAM and BROAD desires to grant a license on the following terms and conditions. +* NOW, THEREFORE, in consideration of the promises and covenants made herein, the parties hereto agree as follows: +* +* 1. DEFINITIONS +* 1.1 PROGRAM shall mean copyright in the object code and source code known as GATK2 and related documentation, if any, as they exist on the EFFECTIVE DATE and can be downloaded from http://www.broadinstitute/GATK on the EFFECTIVE DATE. +* +* 2. LICENSE +* 2.1 Grant. Subject to the terms of this Agreement, BROAD hereby grants to LICENSEE, solely for academic non-commercial research purposes, a non-exclusive, non-transferable license to: (a) download, execute and display the PROGRAM and (b) create bug fixes and modify the PROGRAM. +* The LICENSEE may apply the PROGRAM in a pipeline to data owned by users other than the LICENSEE and provide these users the results of the PROGRAM provided LICENSEE does so for academic non-commercial purposes only. For clarification purposes, academic sponsored research is not a commercial use under the terms of this Agreement. +* 2.2 No Sublicensing or Additional Rights. LICENSEE shall not sublicense or distribute the PROGRAM, in whole or in part, without prior written permission from BROAD. LICENSEE shall ensure that all of its users agree to the terms of this Agreement. LICENSEE further agrees that it shall not put the PROGRAM on a network, server, or other similar technology that may be accessed by anyone other than the LICENSEE and its employees and users who have agreed to the terms of this agreement. +* 2.3 License Limitations. Nothing in this Agreement shall be construed to confer any rights upon LICENSEE by implication, estoppel, or otherwise to any computer software, trademark, intellectual property, or patent rights of BROAD, or of any other entity, except as expressly granted herein. LICENSEE agrees that the PROGRAM, in whole or part, shall not be used for any commercial purpose, including without limitation, as the basis of a commercial software or hardware product or to provide services. LICENSEE further agrees that the PROGRAM shall not be copied or otherwise adapted in order to circumvent the need for obtaining a license for use of the PROGRAM. +* +* 3. OWNERSHIP OF INTELLECTUAL PROPERTY +* LICENSEE acknowledges that title to the PROGRAM shall remain with BROAD. The PROGRAM is marked with the following BROAD copyright notice and notice of attribution to contributors. LICENSEE shall retain such notice on all copies. LICENSEE agrees to include appropriate attribution if any results obtained from use of the PROGRAM are included in any publication. +* Copyright 2012 Broad Institute, Inc. +* Notice of attribution: The GATK2 program was made available through the generosity of Medical and Population Genetics program at the Broad Institute, Inc. +* LICENSEE shall not use any trademark or trade name of BROAD, or any variation, adaptation, or abbreviation, of such marks or trade names, or any names of officers, faculty, students, employees, or agents of BROAD except as states above for attribution purposes. +* +* 4. INDEMNIFICATION +* LICENSEE shall indemnify, defend, and hold harmless BROAD, and their respective officers, faculty, students, employees, associated investigators and agents, and their respective successors, heirs and assigns, (Indemnitees), against any liability, damage, loss, or expense (including reasonable attorneys fees and expenses) incurred by or imposed upon any of the Indemnitees in connection with any claims, suits, actions, demands or judgments arising out of any theory of liability (including, without limitation, actions in the form of tort, warranty, or strict liability and regardless of whether such action has any factual basis) pursuant to any right or license granted under this Agreement. +* +* 5. NO REPRESENTATIONS OR WARRANTIES +* THE PROGRAM IS DELIVERED AS IS. BROAD MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE PROGRAM OR THE COPYRIGHT, EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, WHETHER OR NOT DISCOVERABLE. BROAD EXTENDS NO WARRANTIES OF ANY KIND AS TO PROGRAM CONFORMITY WITH WHATEVER USER MANUALS OR OTHER LITERATURE MAY BE ISSUED FROM TIME TO TIME. +* IN NO EVENT SHALL BROAD OR ITS RESPECTIVE DIRECTORS, OFFICERS, EMPLOYEES, AFFILIATED INVESTIGATORS AND AFFILIATES BE LIABLE FOR INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, INCLUDING, WITHOUT LIMITATION, ECONOMIC DAMAGES OR INJURY TO PROPERTY AND LOST PROFITS, REGARDLESS OF WHETHER BROAD SHALL BE ADVISED, SHALL HAVE OTHER REASON TO KNOW, OR IN FACT SHALL KNOW OF THE POSSIBILITY OF THE FOREGOING. +* +* 6. ASSIGNMENT +* This Agreement is personal to LICENSEE and any rights or obligations assigned by LICENSEE without the prior written consent of BROAD shall be null and void. +* +* 7. MISCELLANEOUS +* 7.1 Export Control. LICENSEE gives assurance that it will comply with all United States export control laws and regulations controlling the export of the PROGRAM, including, without limitation, all Export Administration Regulations of the United States Department of Commerce. Among other things, these laws and regulations prohibit, or require a license for, the export of certain types of software to specified countries. +* 7.2 Termination. LICENSEE shall have the right to terminate this Agreement for any reason upon prior written notice to BROAD. If LICENSEE breaches any provision hereunder, and fails to cure such breach within thirty (30) days, BROAD may terminate this Agreement immediately. Upon termination, LICENSEE shall provide BROAD with written assurance that the original and all copies of the PROGRAM have been destroyed, except that, upon prior written authorization from BROAD, LICENSEE may retain a copy for archive purposes. +* 7.3 Survival. The following provisions shall survive the expiration or termination of this Agreement: Articles 1, 3, 4, 5 and Sections 2.2, 2.3, 7.3, and 7.4. +* 7.4 Notice. Any notices under this Agreement shall be in writing, shall specifically refer to this Agreement, and shall be sent by hand, recognized national overnight courier, confirmed facsimile transmission, confirmed electronic mail, or registered or certified mail, postage prepaid, return receipt requested. All notices under this Agreement shall be deemed effective upon receipt. +* 7.5 Amendment and Waiver; Entire Agreement. This Agreement may be amended, supplemented, or otherwise modified only by means of a written instrument signed by all parties. Any waiver of any rights or failure to act in a specific instance shall relate only to such instance and shall not be construed as an agreement to waive any rights or fail to act in any other instance, whether or not similar. This Agreement constitutes the entire agreement among the parties with respect to its subject matter and supersedes prior agreements or understandings between the parties relating to its subject matter. +* 7.6 Binding Effect; Headings. This Agreement shall be binding upon and inure to the benefit of the parties and their respective permitted successors and assigns. All headings are for convenience only and shall not affect the meaning of any provision of this Agreement. +* 7.7 Governing Law. This Agreement shall be construed, governed, interpreted and applied in accordance with the internal laws of the Commonwealth of Massachusetts, U.S.A., without regard to conflict of laws principles. +*/ + +package org.broadinstitute.sting.utils.pairhmm; + +import com.google.java.contract.Ensures; +import com.google.java.contract.Requires; +import org.broadinstitute.sting.utils.QualityUtils; + +import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap; +import org.broadinstitute.sting.utils.haplotype.Haplotype; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; +import org.broadinstitute.variant.variantcontext.Allele; + +import java.util.List; +import java.util.Map; + + +/** + * Created with IntelliJ IDEA. + * User: rpoplin, carneiro + * Date: 10/16/12 + */ +public class JNILoglessPairHMM extends LoglessPairHMM { + static { + System.loadLibrary("JNILoglessPairHMM"); + } + + private static final boolean debug = true; //simulates ifdef + private static final boolean debug2 = false; + private static final boolean debug3 = false; + + @Override + protected void finalize() throws Throwable { + try{ + debugClose(); + }catch(Throwable t){ + throw t; + }finally{ + super.finalize(); + } + } + + private native void jniInitialize(final int readMaxLength, final int haplotypeMaxLength); + + private native static void jniInitializeProbabilities(final double[][] transition, final byte[] insertionGOP, final byte[] deletionGOP, final byte[] overallGCP); + + private native double jniInitializePriorsAndUpdateCells( + boolean doInitialization, + final int paddedReadLength, final int paddedHaplotypeLength, + final byte[] readBases, final byte[] haplotypeBases, final byte[] readQuals, + final int hapStartIndex); + + private native double jniSubComputeReadLikelihoodGivenHaplotypeLog10( + final int readLength, final int haplotypeLength, + final byte[] readBases, final byte[] haplotypeBases, final byte[] readQuals, + final byte[] insertionGOP, final byte[] deletionGOP, final byte[] overallGCP, + final int hapStartIndex); + + + + /** + * {@inheritDoc} + */ + @Override + public void initialize(final int readMaxLength, final int haplotypeMaxLength) + { + super.initialize(readMaxLength, haplotypeMaxLength); + if(debug3) + { + System.out.println("Java: alloc initialized readMaxLength : "+readMaxLength+" haplotypeMaxLength : "+haplotypeMaxLength); + debugDump("lengths_java.txt", String.format("%d %d\n",readMaxLength, haplotypeMaxLength), + true); + } + if(debug2) + jniInitialize(readMaxLength, haplotypeMaxLength); + } + + /** + * {@inheritDoc} + */ + @Override + public PerReadAlleleLikelihoodMap computeLikelihoods(final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap) + { + PerReadAlleleLikelihoodMap retValue = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + if(debug) + debugClose(); + return retValue; + } + //{ + + //// (re)initialize the pairHMM only if necessary + //final int readMaxLength = findMaxReadLength(reads); + //final int haplotypeMaxLength = findMaxHaplotypeLength(alleleHaplotypeMap); + //if (!initialized || readMaxLength > maxReadLength || haplotypeMaxLength > maxHaplotypeLength) + //{ initialize(readMaxLength, haplotypeMaxLength); } + + //if ( ! initialized ) + //throw new IllegalStateException("Must call initialize before calling computeReadLikelihoodGivenHaplotypeLog10"); + //final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap(); + //for(GATKSAMRecord read : reads){ + //final byte[] readBases = read.getReadBases(); + //final byte[] readQuals = read.getBaseQualities(); + //final byte[] readInsQuals = read.getBaseInsertionQualities(); + //final byte[] readDelQuals = read.getBaseDeletionQualities(); + //final byte[] overallGCP = GCPArrayMap.get(read); + + //// peak at the next haplotype in the list (necessary to get nextHaplotypeBases, which is required for caching in the array implementation) + //byte[] currentHaplotypeBases = null; + //boolean isFirstHaplotype = true; + //Allele currentAllele = null; + //double log10l; + //for (final Allele allele : alleleHaplotypeMap.keySet()){ + //final Haplotype haplotype = alleleHaplotypeMap.get(allele); + //final byte[] nextHaplotypeBases = haplotype.getBases(); + //if (currentHaplotypeBases != null) { + //log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, + //readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, nextHaplotypeBases); + //likelihoodMap.add(read, currentAllele, log10l); + //} + //// update the current haplotype + //currentHaplotypeBases = nextHaplotypeBases; + //currentAllele = allele; + //} + //// process the final haplotype + //if (currentHaplotypeBases != null) { + + //// there is no next haplotype, so pass null for nextHaplotypeBases. + //log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, + //readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, null); + //likelihoodMap.add(read, currentAllele, log10l); + //} + //} + //return likelihoodMap; + //} + + ///** + //* {@inheritDoc} + //*/ + //@Override + //protected final double computeReadLikelihoodGivenHaplotypeLog10( final byte[] haplotypeBases, + //final byte[] readBases, + //final byte[] readQuals, + //final byte[] insertionGOP, + //final byte[] deletionGOP, + //final byte[] overallGCP, + //final boolean recacheReadValues, + //final byte[] nextHaploytpeBases) { + + //if ( haplotypeBases == null ) throw new IllegalArgumentException("haplotypeBases cannot be null"); + //if ( haplotypeBases.length > maxHaplotypeLength ) throw new IllegalArgumentException("Haplotype bases is too long, got " + haplotypeBases.length + " but max is " + maxHaplotypeLength); + //if ( readBases == null ) throw new IllegalArgumentException("readBases cannot be null"); + //if ( readBases.length > maxReadLength ) throw new IllegalArgumentException("readBases is too long, got " + readBases.length + " but max is " + maxReadLength); + //if ( readQuals.length != readBases.length ) throw new IllegalArgumentException("Read bases and read quals aren't the same size: " + readBases.length + " vs " + readQuals.length); + //if ( insertionGOP.length != readBases.length ) throw new IllegalArgumentException("Read bases and read insertion quals aren't the same size: " + readBases.length + " vs " + insertionGOP.length); + //if ( deletionGOP.length != readBases.length ) throw new IllegalArgumentException("Read bases and read deletion quals aren't the same size: " + readBases.length + " vs " + deletionGOP.length); + //if ( overallGCP.length != readBases.length ) throw new IllegalArgumentException("Read bases and overall GCP aren't the same size: " + readBases.length + " vs " + overallGCP.length); + + //paddedReadLength = readBases.length + 1; + //paddedHaplotypeLength = haplotypeBases.length + 1; + //double result = subComputeReadLikelihoodGivenHaplotypeLog10(haplotypeBases, readBases, readQuals, insertionGOP, deletionGOP, overallGCP, 0, true, 0); + //if ( ! MathUtils.goodLog10Probability(result) ) + //throw new IllegalStateException("PairHMM Log Probability cannot be greater than 0: " + String.format("haplotype: %s, read: %s, result: %f", Arrays.toString(haplotypeBases), Arrays.toString(readBases), result)); + //// Warning: Careful if using the PairHMM in parallel! (this update has to be taken care of). + //// Warning: This assumes no downstream modification of the haplotype bases (saves us from copying the array). It is okay for the haplotype caller and the Unified Genotyper. + //// KG: seems pointless is not being used anywhere + //previousHaplotypeBases = haplotypeBases; + //return result; + //} + + /** + * {@inheritDoc} + */ + @Override + public double subComputeReadLikelihoodGivenHaplotypeLog10( final byte[] haplotypeBases, + final byte[] readBases, + final byte[] readQuals, + final byte[] insertionGOP, + final byte[] deletionGOP, + final byte[] overallGCP, + final int hapStartIndex, + final boolean recacheReadValues, + final int nextHapStartIndex) { + + + //System.out.println("#### START STACK TRACE ####"); + //for (StackTraceElement ste : Thread.currentThread().getStackTrace()) { + //System.out.println(ste); + //} + //System.out.println("#### END STACK TRACE ####"); + // + jniSubComputeReadLikelihoodGivenHaplotypeLog10(readBases.length, haplotypeBases.length, + readBases, haplotypeBases, readQuals, + insertionGOP, deletionGOP, overallGCP, + hapStartIndex); + + boolean doInitialization = (previousHaplotypeBases == null || previousHaplotypeBases.length != haplotypeBases.length); + if (doInitialization) { + final double initialValue = INITIAL_CONDITION / haplotypeBases.length; + // set the initial value (free deletions in the beginning) for the first row in the deletion matrix + for( int j = 0; j < paddedHaplotypeLength; j++ ) { + deletionMatrix[0][j] = initialValue; + } + } + + if ( ! constantsAreInitialized || recacheReadValues ) { + initializeProbabilities(transition, insertionGOP, deletionGOP, overallGCP); + if(debug3) + { + System.out.println("Java: initializeProbabilities lengths : "+insertionGOP.length+" padded "+paddedReadLength+" "+paddedHaplotypeLength); + for(int i=0;i filenameToWriter = new HashMap(); + protected void debugDump(String filename, String s, boolean toAppend) + { + try + { + File file = new File(filename); + if (!file.exists()) + file.createNewFile(); + BufferedWriter currWriter = filenameToWriter.get(filename); + if(currWriter == null) + { + FileWriter fw = new FileWriter(file, toAppend); + currWriter = new BufferedWriter(fw); + filenameToWriter.put(filename, currWriter); + } + currWriter.write(s); + } + catch(IOException e) + { + e.printStackTrace(); + } + } + protected void debugClose() + { + for(Map.Entry currEntry : filenameToWriter.entrySet()) + { + BufferedWriter currWriter = currEntry.getValue(); + try + { + currWriter.flush(); + currWriter.close(); + } + catch(IOException e) + { + e.printStackTrace(); + + } + } + filenameToWriter.clear(); + } /** * {@inheritDoc} */ From e90405cd1fd1f8d824194f91f971aa32da667cc7 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Thu, 16 Jan 2014 19:53:50 -0800 Subject: [PATCH 04/72] 1. Nested loops over reads and haplotypes moved to C++ through JNI 2. OpenMP support added 3. Using direct access to Java primitive arrays 4. Debug messages disabled --- .gitignore | 2 + Makefile | 17 +- convert_char.cc | 186 ++++++++ hmm_mask.cc | 60 +-- ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 417 +++++++++++++++--- ...te_sting_utils_pairhmm_JNILoglessPairHMM.h | 15 + pairhmm-1-base.cc | 324 ++++++-------- pairhmm-template-main.cc | 8 +- template.h | 79 +--- 9 files changed, 737 insertions(+), 371 deletions(-) create mode 100644 convert_char.cc diff --git a/.gitignore b/.gitignore index dcc44daa4..cb70b53a7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ tests hmm_Mohammad pairhmm-template-main *.swp +checker +reformat diff --git a/Makefile b/Makefile index f858a7241..70bf96dd3 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,11 @@ CXX=icc LDFLAGS=-lm $(OMPLDFLAGS) #BIN:=pairhmm-1-base #pairhmm-2-omp pairhmm-3-hybrid-float-double pairhmm-4-hybrid-diagonal pairhmm-5-hybrid-diagonal-homogeneus pairhmm-6-onlythreediags pairhmm-7-presse pairhmm-8-sse #pairhmm-dev -BIN:=libJNILoglessPairHMM.so pairhmm-template-main +BIN:=libJNILoglessPairHMM.so pairhmm-template-main checker #SOURCES=pairhmm-1-base.cc input.cc -LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc hmm_mask.cc -SOURCES=$(LIBSOURCES) pairhmm-template-main.cc +LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc convert_char.cc +SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc LIBOBJECTS=$(LIBSOURCES:.cc=.o) DEPDIR=.deps DF=$(DEPDIR)/$(*).d @@ -29,11 +29,14 @@ all: $(BIN) -include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) -pairhmm-template-main: pairhmm-template-main.o hmm_mask.o - $(CXX) -fopenmp -o $@ $^ $(LDFLAGS) +checker: pairhmm-1-base.o convert_char.o + $(CXX) $(OMPCFLAGS) -o $@ $^ $(LDFLAGS) + +pairhmm-template-main: pairhmm-template-main.o convert_char.o + $(CXX) $(OMPCFLAGS) -o $@ $^ $(LDFLAGS) libJNILoglessPairHMM.so: $(LIBOBJECTS) - $(CXX) -shared -o $@ $(LIBOBJECTS) + $(CXX) $(OMPCFLAGS) -shared -o $@ $(LIBOBJECTS) %.o: %.cc @mkdir -p $(DEPDIR) @@ -41,4 +44,4 @@ libJNILoglessPairHMM.so: $(LIBOBJECTS) clean: - rm -f $(BIN) *.o + rm -rf $(BIN) *.o $(DEPDIR) diff --git a/convert_char.cc b/convert_char.cc new file mode 100644 index 000000000..01c5a5137 --- /dev/null +++ b/convert_char.cc @@ -0,0 +1,186 @@ +#include "template.h" +uint8_t ConvertChar::conversionTable[255]; +using namespace std; + +int normalize(char c) +{ + return ((int) (c - 33)); +} + +int read_testcase(testcase *tc, FILE* ifp) +{ + char *q, *i, *d, *c, *line = NULL; + int _q, _i, _d, _c; + int x, size = 0; + ssize_t read; + + read = getline(&line, (size_t *) &size, ifp == 0 ? stdin : ifp); + if (read == -1) + return -1; + + + tc->hap = (char *) malloc(size); + tc->rs = (char *) malloc(size); + q = (char *) malloc(size); + i = (char *) malloc(size); + d = (char *) malloc(size); + c = (char *) malloc(size); + + if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) + return -1; + + tc->haplen = strlen(tc->hap); + tc->rslen = strlen(tc->rs); + assert(tc->rslen < MROWS); + tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); + tc->irs = (int *) malloc(tc->rslen*sizeof(int)); + + //tc->q = (int *) malloc(sizeof(int) * tc->rslen); + //tc->i = (int *) malloc(sizeof(int) * tc->rslen); + //tc->d = (int *) malloc(sizeof(int) * tc->rslen); + //tc->c = (int *) malloc(sizeof(int) * tc->rslen); + + for (x = 0; x < tc->rslen; x++) + { + _q = normalize(q[x]); + _i = normalize(i[x]); + _d = normalize(d[x]); + _c = normalize(c[x]); + tc->q[x] = (_q < 6) ? 6 : _q; + tc->i[x] = _i; + tc->d[x] = _d; + tc->c[x] = _c; + tc->irs[x] = tc->rs[x]; + } + for (x = 0; x < tc->haplen; x++) + tc->ihap[x] = tc->hap[x]; + + + free(q); + free(i); + free(d); + free(c); + free(line); + + return 0; +} + +unsigned MAX_LINE_LENGTH = 65536; +int convToInt(std::string s) +{ + int i; + std::istringstream strin(s); + strin >> i; + return i; +} + +void tokenize(std::ifstream& fptr, std::vector& tokens) +{ + int i = 0; + std::string tmp; + std::vector myVec; + vector line; + line.clear(); + line.resize(MAX_LINE_LENGTH); + vector tmpline; + tmpline.clear(); + tmpline.resize(MAX_LINE_LENGTH); + myVec.clear(); + + while(!fptr.eof()) + { + i = 0; + bool still_read_line = true; + unsigned line_position = 0; + while(still_read_line) + { + fptr.getline(&(tmpline[0]), MAX_LINE_LENGTH); + if(line_position + MAX_LINE_LENGTH > line.size()) + line.resize(2*line.size()); + for(unsigned i=0;i> std::skipws >> tmp; + if(tmp != "") + { + myVec.push_back(tmp); + ++i; + //std::cout < 0) + break; + } + tokens.clear(); + //std::cout << "Why "< tokens; + tokens.clear(); + tokenize(fptr, tokens); + if(tokens.size() == 0) + return -1; + tc->hap = new char[tokens[0].size()+2]; + tc->haplen = tokens[0].size(); + memcpy(tc->hap, tokens[0].c_str(), tokens[0].size()); + tc->rs = new char[tokens[1].size()+2]; + tc->rslen = tokens[1].size(); + //cout << "Lengths "<haplen <<" "<rslen<<"\n"; + memcpy(tc->rs, tokens[1].c_str(),tokens[1].size()); + assert(tokens.size() == 2 + 4*(tc->rslen)); + assert(tc->rslen < MROWS); + for(unsigned j=0;jrslen;++j) + tc->q[j] = convToInt(tokens[2+0*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->i[j] = convToInt(tokens[2+1*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->d[j] = convToInt(tokens[2+2*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->c[j] = convToInt(tokens[2+3*tc->rslen+j]); + + if(reformat) + { + ofstream ofptr; + ofptr.open("reformat/debug_dump.txt",first_call ? ios::out : ios::app); + assert(ofptr.is_open()); + ofptr << tokens[0] << " "; + ofptr << tokens[1] << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->q[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->i[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->d[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->c[j]+33)); + ofptr << " 0 false\n"; + + ofptr.close(); + first_call = false; + } + + + return tokens.size(); +} diff --git a/hmm_mask.cc b/hmm_mask.cc index f128a39d9..c335a0160 100644 --- a/hmm_mask.cc +++ b/hmm_mask.cc @@ -30,9 +30,7 @@ #include #include #include -#include -#include -#include +#include "template.h" using namespace std ; @@ -50,43 +48,14 @@ string getBinaryStr (T val, int numBitsToWrite) { return oss.str() ; } - -typedef struct -{ - int rslen, haplen, *q, *i, *d, *c; - char *hap, *rs; -} testcase; - int normalize(char c) { return ((int) (c - 33)); } -class ConvertChar { - - static uint8_t conversionTable[255] ; - - -public: - - static void init() { - conversionTable['A'] = 0 ; - conversionTable['C'] = 1 ; - conversionTable['T'] = 2 ; - conversionTable['G'] = 3 ; - conversionTable['N'] = 4 ; - } - - static inline uint8_t get(uint8_t input) { - return conversionTable[input] ; - } - -} ; uint8_t ConvertChar::conversionTable[255] ; - - int read_testcase(testcase *tc, FILE* ifp) { char *q, *i, *d, *c, *line = NULL; @@ -94,7 +63,7 @@ int read_testcase(testcase *tc, FILE* ifp) int x, size = 0; ssize_t read; - read = getline(&line, (size_t *) &size, ifp); + read = getline(&line, (size_t *) &size, ifp == 0 ? stdin : ifp); if (read == -1) return -1; @@ -111,20 +80,14 @@ int read_testcase(testcase *tc, FILE* ifp) tc->haplen = strlen(tc->hap); tc->rslen = strlen(tc->rs); - tc->q = (int *) malloc(sizeof(int) * tc->rslen); - tc->i = (int *) malloc(sizeof(int) * tc->rslen); - tc->d = (int *) malloc(sizeof(int) * tc->rslen); - tc->c = (int *) malloc(sizeof(int) * tc->rslen); + assert(tc->rslen < MROWS); + tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); + tc->irs = (int *) malloc(tc->rslen*sizeof(int)); - // Convert hap and rs to 3 bits - - /* - for (int ci=0; ci < tc->haplen; ++ci) - tc->hap[ci] = ConvertChar::get(tc->hap[ci]) ; - - for (int ci=0; ci < tc->rslen; ++ci) - tc->rs[ci] = ConvertChar::get(tc->rs[ci]) ; - */ + //tc->q = (int *) malloc(sizeof(int) * tc->rslen); + //tc->i = (int *) malloc(sizeof(int) * tc->rslen); + //tc->d = (int *) malloc(sizeof(int) * tc->rslen); + //tc->c = (int *) malloc(sizeof(int) * tc->rslen); for (x = 0; x < tc->rslen; x++) { @@ -136,8 +99,11 @@ int read_testcase(testcase *tc, FILE* ifp) tc->i[x] = _i; tc->d[x] = _d; tc->c[x] = _c; + tc->irs[x] = tc->rs[x]; } - + for (x = 0; x < tc->haplen; x++) + tc->ihap[x] = tc->hap[x]; + free(q); free(i); diff --git a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index 29668a306..b8298161e 100644 --- a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -2,6 +2,7 @@ #include #include #include "org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h" +#include #include #include @@ -12,12 +13,14 @@ #include #include + #include #include #include using namespace std; +//#define DEBUG 1 +//#define DEBUG0_1 1 //#define DEBUG3 1 -#define DEBUG 1 #include "template.h" @@ -42,6 +45,12 @@ class LoadTimeInitializer { ConvertChar::init(); } + jfieldID m_readBasesFID; + jfieldID m_readQualsFID; + jfieldID m_insertionGOPFID; + jfieldID m_deletionGOPFID; + jfieldID m_overallGCPFID; + jfieldID m_haplotypeBasesFID; }; LoadTimeInitializer g_load_time_initializer; @@ -55,73 +64,6 @@ void debug_dump(string filename, string s, bool to_append, bool add_newline) fptr.close(); } -#define INT_STORE_ARRAY_SIZE 2048 -int insertionGOPIntArray[INT_STORE_ARRAY_SIZE]; -int deletionGOPIntArray[INT_STORE_ARRAY_SIZE]; -int overallGCPIntArray[INT_STORE_ARRAY_SIZE]; -int readQualsIntArray[INT_STORE_ARRAY_SIZE]; - -JNIEXPORT jdouble JNICALL -Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10( - JNIEnv* env, jobject thisObject, - jint readLength, jint haplotypeLength, - jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, - jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP, - jint hapStartIndex - ) -{ - jboolean is_copy = JNI_FALSE; - jbyte* readBasesArray = (env)->GetByteArrayElements(readBases, &is_copy); - jbyte* haplotypeBasesArray = (env)->GetByteArrayElements(haplotypeBases, &is_copy); - jbyte* readQualsArray = (env)->GetByteArrayElements(readQuals, &is_copy); - jbyte* insertionGOPArray = (env)->GetByteArrayElements(insertionGOP, &is_copy); - jbyte* deletionGOPArray = (env)->GetByteArrayElements(deletionGOP, &is_copy); - jbyte* overallGCPArray = (env)->GetByteArrayElements(overallGCP, &is_copy); -#ifdef DEBUG - assert(readBasesArray && "readBasesArray not initialized in JNI"); - assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); - assert(readQualsArray && "readQualsArray not initialized in JNI"); - assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); - assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); - assert(overallGCPArray && "OverallGCP array not initialized in JNI"); - assert(readLength < INT_STORE_ARRAY_SIZE); -#endif - for(unsigned i=0;i(&tc); - double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); -#ifdef DEBUG - debug_dump("return_values_jni.txt",to_string(result),true); -#endif - - - env->ReleaseByteArrayElements(overallGCP, overallGCPArray, JNI_ABORT); - env->ReleaseByteArrayElements(deletionGOP, deletionGOPArray, JNI_ABORT); - env->ReleaseByteArrayElements(insertionGOP, insertionGOPArray, JNI_ABORT); - env->ReleaseByteArrayElements(readQuals, readQualsArray, JNI_ABORT); - env->ReleaseByteArrayElements(haplotypeBases, haplotypeBasesArray, JNI_ABORT); - env->ReleaseByteArrayElements(readBases, readBasesArray, JNI_ABORT); - - return 0.0; -} - JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeProbabilities (JNIEnv* env, jclass thisObject, jobjectArray transition, jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP @@ -142,3 +84,342 @@ Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializePrior ) { return 0.0; } + +#define DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY 1 + +#ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY +//Gets direct access to Java arrays +#define GET_BYTE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical +#define JNI_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical + +#else +//Likely makes copy of Java arrays to JNI C++ space +#define GET_BYTE_ARRAY_ELEMENTS env->GetByteArrayElements +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleaseByteArrayElements +#define JNI_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetDoubleArrayElements +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleaseDoubleArrayElements + +#endif //ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10( + JNIEnv* env, jobject thisObject, + jint readLength, jint haplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP, + jint hapStartIndex + ) +{ + jboolean is_copy = JNI_FALSE; + jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBases, &is_copy); + jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); + jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); + jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); + assert(readLength < MROWS); +#endif + testcase tc; + tc.rslen = readLength; + tc.haplen = haplotypeLength; + tc.rs = (char*)readBasesArray; + tc.hap = (char*)haplotypeBasesArray; + for(unsigned i=0;i(&tc); + double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); +#ifdef DEBUG + debug_dump("return_values_jni.txt",to_string(result),true); +#endif + + + RELEASE_BYTE_ARRAY_ELEMENTS(overallGCP, overallGCPArray, JNI_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(deletionGOP, deletionGOPArray, JNI_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(insertionGOP, insertionGOPArray, JNI_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(readQuals, readQualsArray, JNI_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBases, haplotypeBasesArray, JNI_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(readBases, readBasesArray, JNI_RELEASE_MODE); + + return 0.0; +} + +//Should be called only once for the whole Java process - initializes field ids for the classes JNIReadDataHolderClass +//and JNIHaplotypeDataHolderClass +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniGlobalInit + (JNIEnv* env, jobject thisObject, jclass readDataHolderClass, jclass haplotypeDataHolderClass) +{ + assert(readDataHolderClass); + assert(haplotypeDataHolderClass); + jfieldID fid; + fid = env->GetFieldID(readDataHolderClass, "readBases", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for readBases"); + g_load_time_initializer.m_readBasesFID = fid; + fid = env->GetFieldID(readDataHolderClass, "readQuals", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for readQuals"); + g_load_time_initializer.m_readQualsFID = fid; + fid = env->GetFieldID(readDataHolderClass, "insertionGOP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for insertionGOP"); + g_load_time_initializer.m_insertionGOPFID = fid; + fid = env->GetFieldID(readDataHolderClass, "deletionGOP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for deletionGOP"); + g_load_time_initializer.m_deletionGOPFID = fid; + fid = env->GetFieldID(readDataHolderClass, "overallGCP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for overallGCP"); + g_load_time_initializer.m_overallGCPFID = fid; + + fid = env->GetFieldID(haplotypeDataHolderClass, "haplotypeBases", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for haplotypeBases"); + g_load_time_initializer.m_haplotypeBasesFID = fid; +} + + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniComputeLikelihoods + (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, + jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +{ + jboolean is_copy = JNI_FALSE; + //To ensure, GET_BYTE_ARRAY_ELEMENTS is invoked only once for each haplotype, store bytearrays in a vector + vector > haplotypeBasesArrayVector; + haplotypeBasesArrayVector.clear(); + haplotypeBasesArrayVector.resize(numHaplotypes); +#ifdef DEBUG0_1 + cout << "JNI numReads "<GetObjectArrayElement(haplotypeDataArray, j); + jbyteArray haplotypeBases = (jbyteArray)env->GetObjectField(haplotypeObject, g_load_time_initializer.m_haplotypeBasesFID); +#ifdef DEBUG + assert(haplotypeBases && ("haplotypeBases is NULL at index : "+to_string(j)+"\n").c_str()); +#endif + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBases, &is_copy); +#ifdef DEBUG + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(env->GetArrayLength(haplotypeBases) < MCOLS); +#ifdef DEBUG0_1 + cout << "JNI haplotype length "<GetArrayLength(haplotypeBases)<<"\n"; +#endif +#endif + haplotypeBasesArrayVector[j] = make_pair(haplotypeBases, haplotypeBasesArray); +#ifdef DEBUG3 + for(unsigned k=0;kGetArrayLength(haplotypeBases);++k) + debug_dump("haplotype_bases_jni.txt",to_string((int)haplotypeBasesArray[k]),true); +#endif + } + + unsigned numTestCases = numReads*numHaplotypes; + vector tc_array; + tc_array.clear(); + tc_array.resize(numTestCases); + unsigned tc_idx = 0; + //Store arrays for release later + vector > > readBasesArrayVector; + readBasesArrayVector.clear(); + readBasesArrayVector.resize(numReads); + for(unsigned i=0;iGetObjectArrayElement(readDataArray, i); + jbyteArray readBases = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_readBasesFID); + jbyteArray insertionGOP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_insertionGOPFID); + jbyteArray deletionGOP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_deletionGOPFID); + jbyteArray overallGCP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_overallGCPFID); + jbyteArray readQuals = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_readQualsFID); + +#ifdef DEBUG + assert(readBases && ("readBases is NULL at index : "+to_string(i)+"\n").c_str()); + assert(insertionGOP && ("insertionGOP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(deletionGOP && ("deletionGOP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(overallGCP && ("overallGCP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(readQuals && ("readQuals is NULL at index : "+to_string(i)+"\n").c_str()); +#endif + jsize readLength = env->GetArrayLength(readBases); + + jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); //order of GET-RELEASE is important + jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); + jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); + jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "overallGCP array not initialized in JNI"); + assert(readLength < MROWS); + assert(readLength == env->GetArrayLength(readQuals)); + assert(readLength == env->GetArrayLength(insertionGOP)); + assert(readLength == env->GetArrayLength(deletionGOP)); + assert(readLength == env->GetArrayLength(overallGCP)); +#ifdef DEBUG0_1 + cout << "JNI read length "<GetArrayLength(haplotypeBasesArrayVector[j].first); + jbyte* haplotypeBasesArray = haplotypeBasesArrayVector[j].second; + tc_array[tc_idx].rslen = (int)readLength; + tc_array[tc_idx].haplen = (int)haplotypeLength; + tc_array[tc_idx].rs = (char*)readBasesArray; + tc_array[tc_idx].hap = (char*)haplotypeBasesArray; + //Can be avoided + for(unsigned k=0;kGetArrayLength(likelihoodArray) == numTestCases); +#pragma omp parallel for schedule (guided,2) private(tc_idx) num_threads(maxNumThreadsToUse) + for(tc_idx=0;tc_idx(&(tc_array[tc_idx])); + double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); + likelihoodDoubleArray[tc_idx] = result; + } +#ifdef DEBUG + for(tc_idx=0;tc_idx=0;--i)//note the order - reverse of GET + { + for(int j=readBasesArrayVector.size()-1;j>=0;--j) + RELEASE_BYTE_ARRAY_ELEMENTS(readBasesArrayVector[i][j].first, readBasesArrayVector[i][j].second, JNI_RELEASE_MODE); + readBasesArrayVector[i].clear(); + } + readBasesArrayVector.clear(); + + //Now release haplotype arrays + for(int j=numHaplotypes-1;j>=0;--j) //note the order - reverse of GET + RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBasesArrayVector[j].first, haplotypeBasesArrayVector[j].second, JNI_RELEASE_MODE); + haplotypeBasesArrayVector.clear(); + tc_array.clear(); +} + + + + + /* + protected final double computeReadLikelihoodGivenHaplotypeLog10( final byte[] haplotypeBases, + final byte[] readBases, + final byte[] readQuals, + final byte[] insertionGOP, + final byte[] deletionGOP, + final byte[] overallGCP, + final boolean recacheReadValues, + final byte[] nextHaploytpeBases) { + + paddedReadLength = readBases.length + 1; + paddedHaplotypeLength = haplotypeBases.length + 1; + double result = subComputeReadLikelihoodGivenHaplotypeLog10(haplotypeBases, readBases, readQuals, insertionGOP, deletionGOP, overallGCP, 0, true, 0); + if ( ! MathUtils.goodLog10Probability(result) ) + throw new IllegalStateException("PairHMM Log Probability cannot be greater than 0: " + String.format("haplotype: %s, read: %s, result: %f", Arrays.toString(haplotypeBases), Arrays.toString(readBases), result)); + // Warning: Careful if using the PairHMM in parallel! (this update has to be taken care of). + // Warning: This assumes no downstream modification of the haplotype bases (saves us from copying the array). It is okay for the haplotype caller and the Unified Genotyper. + // KG: seems pointless is not being used anywhere + previousHaplotypeBases = haplotypeBases; + return result; + return 0; + }*/ + +//public PerReadAlleleLikelihoodMap computeLikelihoods(final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap) + //{ + ////PerReadAlleleLikelihoodMap retValue = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + //// (re)initialize the pairHMM only if necessary + //final int readMaxLength = findMaxReadLength(reads); + //final int haplotypeMaxLength = findMaxHaplotypeLength(alleleHaplotypeMap); + //if (!initialized || readMaxLength > maxReadLength || haplotypeMaxLength > maxHaplotypeLength) + //{ initialize(readMaxLength, haplotypeMaxLength); } + + //if ( ! initialized ) + //throw new IllegalStateException("Must call initialize before calling jniComputeLikelihoods"); + //final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap(); + //jniComputeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap, likelihoodMap); + //for(GATKSAMRecord read : reads){ + //final byte[] readBases = read.getReadBases(); + //final byte[] readQuals = read.getBaseQualities(); + //final byte[] readInsQuals = read.getBaseInsertionQualities(); + //final byte[] readDelQuals = read.getBaseDeletionQualities(); + //final byte[] overallGCP = GCPArrayMap.get(read); + + //// peak at the next haplotype in the list (necessary to get nextHaplotypeBases, which is required for caching in the array implementation) + //byte[] currentHaplotypeBases = null; + //boolean isFirstHaplotype = true; + //Allele currentAllele = null; + //double log10l; + //for (final Allele allele : alleleHaplotypeMap.keySet()){ + //final Haplotype haplotype = alleleHaplotypeMap.get(allele); + //final byte[] nextHaplotypeBases = haplotype.getBases(); + //if (currentHaplotypeBases != null) { + //log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, + //readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, nextHaplotypeBases); + //likelihoodMap.add(read, currentAllele, log10l); + //} + //// update the current haplotype + //currentHaplotypeBases = nextHaplotypeBases; + //currentAllele = allele; + //} + //// process the final haplotype + //if (currentHaplotypeBases != null) { + + //// there is no next haplotype, so pass null for nextHaplotypeBases. + //log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, + //readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, null); + //likelihoodMap.add(read, currentAllele, log10l); + //} + //} + //if(debug) + //debugClose(); + //return likelihoodMap; + //} diff --git a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h index 5408d2f57..1fff9fa5a 100644 --- a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h +++ b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h @@ -54,6 +54,21 @@ JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILogless JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10 (JNIEnv *, jobject, jint, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jint); +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniGlobalInit + * Signature: (Ljava/lang/Class;Ljava/lang/Class;)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniGlobalInit + (JNIEnv *, jobject, jclass, jclass); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniComputeLikelihoods + * Signature: ([Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIReadDataHolderClass;[Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIHaplotypeDataHolderClass;[D)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); #ifdef __cplusplus } diff --git a/pairhmm-1-base.cc b/pairhmm-1-base.cc index dacd02536..90f95aeec 100644 --- a/pairhmm-1-base.cc +++ b/pairhmm-1-base.cc @@ -1,10 +1,6 @@ -#include -#include -#include -#include - -#include - +//#define DEBUG 1 +//#define DEBUG0_1 1 +//#define DEBUG3 1 #define MM 0 #define GapM 1 #define MX 2 @@ -12,123 +8,34 @@ #define MY 4 #define YY 5 -//#define DEBUG +#include +#include +#include +#include +#include +#include +#include "template.h" -/* - q: read quality - i: insertion penalty - d: deletion penalty - c: gap continuation penalty -*/ +//#include "define-float.h" +//#include "shift_template.c" +//#include "pairhmm-template-kernel.cc" -typedef struct +#include "define-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + + +using namespace std; +class LoadTimeInitializer { - int rslen, haplen, *q, *i, *d, *c; - char *hap, *rs; -} testcase; - -int normalize(char c) -{ - return ((int) (c - 33)); -} - -int read_testcase(testcase *tc) -{ - char *q, *i, *d, *c, *line = NULL; - int _q, _i, _d, _c; - int x, size = 0; - ssize_t read; - - read = getline(&line, (size_t *) &size, stdin); - if (read == -1) - return -1; - - tc->hap = (char *) malloc(size); - tc->rs = (char *) malloc(size); - q = (char *) malloc(size); - i = (char *) malloc(size); - d = (char *) malloc(size); - c = (char *) malloc(size); - - if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) - return -1; - - tc->haplen = strlen(tc->hap); - tc->rslen = strlen(tc->rs); - tc->q = (int *) malloc(sizeof(int) * tc->rslen); - tc->i = (int *) malloc(sizeof(int) * tc->rslen); - tc->d = (int *) malloc(sizeof(int) * tc->rslen); - tc->c = (int *) malloc(sizeof(int) * tc->rslen); - - for (x = 0; x < tc->rslen; x++) - { - _q = normalize(q[x]); - _i = normalize(i[x]); - _d = normalize(d[x]); - _c = normalize(c[x]); - tc->q[x] = (_q < 6) ? 6 : _q; - tc->i[x] = _i; - tc->d[x] = _d; - tc->c[x] = _c; - } - - free(q); - free(i); - free(d); - free(c); - free(line); - - return 0; -} - -template -struct Context{}; - -template<> -struct Context -{ - Context() - { - for (int x = 0; x < 128; x++) - ph2pr[x] = pow(10.0, -((double)x) / 10.0); - - INITIAL_CONSTANT = ldexp(1.0, 1020.0); - LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); - RESULT_THRESHOLD = 0.0; - } - - double LOG10(double v){ return log10(v); } - - static double _(double n){ return n; } - static double _(float n){ return ((double) n); } - double ph2pr[128]; - double INITIAL_CONSTANT; - double LOG10_INITIAL_CONSTANT; - double RESULT_THRESHOLD; + public: + LoadTimeInitializer() //will be called when library is loaded + { + ConvertChar::init(); + } }; +LoadTimeInitializer g_load_time_initializer; -template<> -struct Context -{ - Context() - { - for (int x = 0; x < 128; x++) - ph2pr[x] = powf(10.f, -((float)x) / 10.f); - - INITIAL_CONSTANT = ldexpf(1.f, 120.f); - LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); - RESULT_THRESHOLD = ldexpf(1.f, -110.f); - } - - float LOG10(float v){ return log10f(v); } - - static float _(double n){ return ((float) n); } - static float _(float n){ return n; } - float ph2pr[128]; - float INITIAL_CONSTANT; - float LOG10_INITIAL_CONSTANT; - float RESULT_THRESHOLD; -}; template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) @@ -139,10 +46,10 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) Context ctx; - NUMBER M[ROWS][COLS]; - NUMBER X[ROWS][COLS]; - NUMBER Y[ROWS][COLS]; - NUMBER p[ROWS][6]; + NUMBER M[MROWS][MCOLS]; + NUMBER X[MROWS][MCOLS]; + NUMBER Y[MROWS][MCOLS]; + NUMBER p[MROWS][6]; p[0][MM] = ctx._(0.0); p[0][GapM] = ctx._(0.0); @@ -159,8 +66,10 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; p[r][MX] = ctx.ph2pr[_i]; p[r][XX] = ctx.ph2pr[_c]; - p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; - p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + p[r][MY] = ctx.ph2pr[_d]; + p[r][YY] = ctx.ph2pr[_c]; + //p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + //p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; } for (c = 0; c < COLS; c++) @@ -188,6 +97,8 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) NUMBER distm = ctx.ph2pr[_q]; if (_rs == _hap || _rs == 'N' || _hap == 'N') distm = ctx._(1.0) - distm; + else + distm = distm/3; M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; @@ -201,76 +112,127 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) if (before_last_log != NULL) *before_last_log = result; - return result; //ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; + return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; } #define BATCH_SIZE 10000 #define RUN_HYBRID -int main() +int main(int argc, char** argv) { - testcase tc[BATCH_SIZE]; - float result[BATCH_SIZE], result_avxf; - double result_avxd; - struct timeval start, end; - long long aggregateTimeRead = 0L; - long long aggregateTimeCompute = 0L; - long long aggregateTimeWrite = 0L; + if(argc < 2) + { + cerr << "Needs path to input file as argument\n"; + exit(0); + } + bool use_old_read_testcase = false; + if(argc >= 3 && string(argv[2]) == "1") + use_old_read_testcase = true; - bool noMoreData = false; - int count =0; - while (!noMoreData) - { - int read_count = BATCH_SIZE; - gettimeofday(&start, NULL); - for (int b=0;b= 0) + { + double result_avxd = GEN_INTRINSIC(compute_full_prob_avx, d)(&tc); + double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); - gettimeofday(&start, NULL); - for (int b=0;b(&tc[b]); + cout << std::scientific << compute_full_prob(&tc) << " "< tokens; + ifptr.open(argv[1]); + assert(ifptr.is_open()); + while(1) + { + tokens.clear(); + if(read_mod_testcase(ifptr, &tc, false) < 0) + break; + //double result = 0; + double result_avxd = GEN_INTRINSIC(compute_full_prob_avx, d)(&tc); + double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); - #ifdef RUN_HYBRID - #define MIN_ACCEPTED 1e-28f - if (result_avxf < MIN_ACCEPTED) { - count++; - result_avxd = compute_full_prob(&tc[b]); - result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); - } - else - result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); - #endif + cout << std::scientific << compute_full_prob(&tc) << " "<(&tc[b]); - } +#ifdef RUN_HYBRID +#define MIN_ACCEPTED 1e-28f + if (result_avxf < MIN_ACCEPTED) { + count++; + result_avxd = compute_full_prob(&tc[b]); + result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); + } + else + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif - printf("AVX Read Time: %ld\n", aggregateTimeRead); - printf("AVX Compute Time: %ld\n", aggregateTimeCompute); - printf("AVX Write Time: %ld\n", aggregateTimeWrite); - printf("AVX Total Time: %ld\n", aggregateTimeRead + aggregateTimeCompute + aggregateTimeWrite); - printf("# Double called: %d\n", count); +#ifndef RUN_HYBRID + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif - return 0; + } + gettimeofday(&end, NULL); + aggregateTimeCompute += ((end.tv_sec * 1000000 + end.tv_usec) - (start.tv_sec * 1000000 + start.tv_usec)); + + gettimeofday(&start, NULL); + for (int b=0;b #include +#include +#include +#include +#include + #define MROWS 500 #define MCOLS 1000 @@ -127,18 +132,20 @@ struct Context typedef struct { - int rslen, haplen, *q, *i, *d, *c; + int rslen, haplen; + /*int *q, *i, *d, *c;*/ + int q[MROWS], i[MROWS], d[MROWS], c[MROWS]; char *hap, *rs; - int *ihap; - int *irs; + int *ihap; + int *irs; } testcase; template std::string to_string(T obj) { - stringstream ss; - string ret_string; + std::stringstream ss; + std::string ret_string; ss.clear(); ss << std::scientific << obj; ss >> ret_string; @@ -148,66 +155,8 @@ std::string to_string(T obj) void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); int normalize(char c); - - -int read_testcase(testcase *tc) -{ - char *q, *i, *d, *c, *line = NULL; - int _q, _i, _d, _c; - int x, size = 0; - ssize_t read; - - read = getline(&line, (size_t *) &size, stdin); - if (read == -1) - return -1; - - tc->hap = (char *) malloc(size); - tc->rs = (char *) malloc(size); - q = (char *) malloc(size); - i = (char *) malloc(size); - d = (char *) malloc(size); - c = (char *) malloc(size); - tc->ihap = (int *) malloc(size*sizeof(int)); - tc->irs = (int *) malloc(size*sizeof(int)); - - - if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) - return -1; - - tc->haplen = strlen(tc->hap); - tc->rslen = strlen(tc->rs); - tc->q = (int *) malloc(sizeof(int) * tc->rslen); - tc->i = (int *) malloc(sizeof(int) * tc->rslen); - tc->d = (int *) malloc(sizeof(int) * tc->rslen); - tc->c = (int *) malloc(sizeof(int) * tc->rslen); - - tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); - tc->irs = (int *) malloc(tc->rslen*sizeof(int)); - - for (x = 0; x < tc->rslen; x++) - { - _q = normalize(q[x]); - _i = normalize(i[x]); - _d = normalize(d[x]); - _c = normalize(c[x]); - tc->q[x] = (_q < 6) ? 6 : _q; - tc->i[x] = _i; - tc->d[x] = _d; - tc->c[x] = _c; - tc->irs[x] = tc->rs[x]; - } - - for (x = 0; x < tc->haplen; x++) - tc->ihap[x] = tc->hap[x]; - - free(q); - free(i); - free(d); - free(c); - free(line); - - return 0; -} +int read_testcase(testcase *tc, FILE* ifp); +int read_mod_testcase(std::ifstream& fptr, testcase* tc, bool reformat=false); #define NUM_DISTINCT_CHARS 5 #define AMBIG_CHAR 4 From 90938b86100dcfa8c63f26ac7b0c6ca6ec0f0dad Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Thu, 16 Jan 2014 19:58:04 -0800 Subject: [PATCH 05/72] Minor typo in comments fixed --- ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 78 +------------------ 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index b8298161e..852caa00d 100644 --- a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -316,7 +316,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai jdouble* likelihoodDoubleArray = (jdouble*)GET_DOUBLE_ARRAY_ELEMENTS(likelihoodArray, &is_copy); assert(likelihoodDoubleArray && "likelihoodArray is NULL"); assert(env->GetArrayLength(likelihoodArray) == numTestCases); -#pragma omp parallel for schedule (guided,2) private(tc_idx) num_threads(maxNumThreadsToUse) +#pragma omp parallel for schedule (dynamic,10) private(tc_idx) num_threads(maxNumThreadsToUse) for(tc_idx=0;tc_idx(&(tc_array[tc_idx])); @@ -347,79 +347,3 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai tc_array.clear(); } - - - - /* - protected final double computeReadLikelihoodGivenHaplotypeLog10( final byte[] haplotypeBases, - final byte[] readBases, - final byte[] readQuals, - final byte[] insertionGOP, - final byte[] deletionGOP, - final byte[] overallGCP, - final boolean recacheReadValues, - final byte[] nextHaploytpeBases) { - - paddedReadLength = readBases.length + 1; - paddedHaplotypeLength = haplotypeBases.length + 1; - double result = subComputeReadLikelihoodGivenHaplotypeLog10(haplotypeBases, readBases, readQuals, insertionGOP, deletionGOP, overallGCP, 0, true, 0); - if ( ! MathUtils.goodLog10Probability(result) ) - throw new IllegalStateException("PairHMM Log Probability cannot be greater than 0: " + String.format("haplotype: %s, read: %s, result: %f", Arrays.toString(haplotypeBases), Arrays.toString(readBases), result)); - // Warning: Careful if using the PairHMM in parallel! (this update has to be taken care of). - // Warning: This assumes no downstream modification of the haplotype bases (saves us from copying the array). It is okay for the haplotype caller and the Unified Genotyper. - // KG: seems pointless is not being used anywhere - previousHaplotypeBases = haplotypeBases; - return result; - return 0; - }*/ - -//public PerReadAlleleLikelihoodMap computeLikelihoods(final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap) - //{ - ////PerReadAlleleLikelihoodMap retValue = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); - //// (re)initialize the pairHMM only if necessary - //final int readMaxLength = findMaxReadLength(reads); - //final int haplotypeMaxLength = findMaxHaplotypeLength(alleleHaplotypeMap); - //if (!initialized || readMaxLength > maxReadLength || haplotypeMaxLength > maxHaplotypeLength) - //{ initialize(readMaxLength, haplotypeMaxLength); } - - //if ( ! initialized ) - //throw new IllegalStateException("Must call initialize before calling jniComputeLikelihoods"); - //final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap(); - //jniComputeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap, likelihoodMap); - //for(GATKSAMRecord read : reads){ - //final byte[] readBases = read.getReadBases(); - //final byte[] readQuals = read.getBaseQualities(); - //final byte[] readInsQuals = read.getBaseInsertionQualities(); - //final byte[] readDelQuals = read.getBaseDeletionQualities(); - //final byte[] overallGCP = GCPArrayMap.get(read); - - //// peak at the next haplotype in the list (necessary to get nextHaplotypeBases, which is required for caching in the array implementation) - //byte[] currentHaplotypeBases = null; - //boolean isFirstHaplotype = true; - //Allele currentAllele = null; - //double log10l; - //for (final Allele allele : alleleHaplotypeMap.keySet()){ - //final Haplotype haplotype = alleleHaplotypeMap.get(allele); - //final byte[] nextHaplotypeBases = haplotype.getBases(); - //if (currentHaplotypeBases != null) { - //log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, - //readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, nextHaplotypeBases); - //likelihoodMap.add(read, currentAllele, log10l); - //} - //// update the current haplotype - //currentHaplotypeBases = nextHaplotypeBases; - //currentAllele = allele; - //} - //// process the final haplotype - //if (currentHaplotypeBases != null) { - - //// there is no next haplotype, so pass null for nextHaplotypeBases. - //log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, - //readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, null); - //likelihoodMap.add(read, currentAllele, log10l); - //} - //} - //if(debug) - //debugClose(); - //return likelihoodMap; - //} From 532485ca599cc38fdf9c92a61be51a080f268c18 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Thu, 16 Jan 2014 20:26:41 -0800 Subject: [PATCH 06/72] Removed unnecessary files --- CHEATSHEET | 2 -- README | 41 ----------------------------------------- 2 files changed, 43 deletions(-) delete mode 100644 CHEATSHEET delete mode 100644 README diff --git a/CHEATSHEET b/CHEATSHEET deleted file mode 100644 index f61bfdd7f..000000000 --- a/CHEATSHEET +++ /dev/null @@ -1,2 +0,0 @@ -/opt/intel/bin/icc pairhmm-template-main.cc -o pairhmm-template-main.o -O3 -W -Wall -lm -xAVX; ./pairhmm-template-main.o < mediumTest.in > med.out; head -n -5 med.out > medium.out; lua checkLikelihoods.lua medium.out ~/hmm/mediumTest.out 6e-5 - diff --git a/README b/README deleted file mode 100644 index e16df5a8d..000000000 --- a/README +++ /dev/null @@ -1,41 +0,0 @@ -How to compile: -/opt/intel/bin/icc pairhmm-template-main.cc -o pairhmm-template-main.o -O3 -W -Wall -lm -xAVX - ------------------------------------- -How to run: -./pairhmm-template-main.o < ./test_data/largeTest.in > large.out -./pairhmm-template-main.o < ./test_data/mediumTest.in > medium.out -------------------------------------- -Results: -------------------------------------- -MEDIUM FLOAT: -AVX Read Time: 8019020 -AVX Compute Time: 29270238 -AVX Write Time: 242170 -AVX Total Time: 37531428 -# Double called: 0 -------------------------------------- -MEDIUM HYBRID: -AVX Read Time: 7596629 -AVX Compute Time: 29979220 -AVX Write Time: 248485 -AVX Total Time: 37824334 -# Double called: 20329 -------------------------------------- -LARGE FLOAT: -AVX Read Time: 111888586 -AVX Compute Time: 408131806 -AVX Write Time: 3477892 -AVX Total Time: 523498284 -# Double called: 0 -------------------------------------- -LARGE HYBRID: -AVX Read Time: 109728994 -AVX Compute Time: 419475317 -AVX Write Time: 3575364 -AVX Total Time: 532779675 -# Double called: 310042 - - - - From e6c6f8e313525e833c65b913d0fd5031cb855f19 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Thu, 16 Jan 2014 20:28:50 -0800 Subject: [PATCH 07/72] Renamed directory --- .gitignore => PairHMM_JNI/.gitignore | 0 Makefile => PairHMM_JNI/Makefile | 0 convert_char.cc => PairHMM_JNI/convert_char.cc | 0 define-double.h => PairHMM_JNI/define-double.h | 0 define-float.h => PairHMM_JNI/define-float.h | 0 hmm_mask.cc => PairHMM_JNI/hmm_mask.cc | 0 .../org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc | 0 .../org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h | 0 pairhmm-1-base.cc => PairHMM_JNI/pairhmm-1-base.cc | 0 .../pairhmm-template-kernel.cc | 0 pairhmm-template-main.cc => PairHMM_JNI/pairhmm-template-main.cc | 0 shift_template.c => PairHMM_JNI/shift_template.c | 0 template.h => PairHMM_JNI/template.h | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename .gitignore => PairHMM_JNI/.gitignore (100%) rename Makefile => PairHMM_JNI/Makefile (100%) rename convert_char.cc => PairHMM_JNI/convert_char.cc (100%) rename define-double.h => PairHMM_JNI/define-double.h (100%) rename define-float.h => PairHMM_JNI/define-float.h (100%) rename hmm_mask.cc => PairHMM_JNI/hmm_mask.cc (100%) rename org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc => PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc (100%) rename org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h => PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h (100%) rename pairhmm-1-base.cc => PairHMM_JNI/pairhmm-1-base.cc (100%) rename pairhmm-template-kernel.cc => PairHMM_JNI/pairhmm-template-kernel.cc (100%) rename pairhmm-template-main.cc => PairHMM_JNI/pairhmm-template-main.cc (100%) rename shift_template.c => PairHMM_JNI/shift_template.c (100%) rename template.h => PairHMM_JNI/template.h (100%) diff --git a/.gitignore b/PairHMM_JNI/.gitignore similarity index 100% rename from .gitignore rename to PairHMM_JNI/.gitignore diff --git a/Makefile b/PairHMM_JNI/Makefile similarity index 100% rename from Makefile rename to PairHMM_JNI/Makefile diff --git a/convert_char.cc b/PairHMM_JNI/convert_char.cc similarity index 100% rename from convert_char.cc rename to PairHMM_JNI/convert_char.cc diff --git a/define-double.h b/PairHMM_JNI/define-double.h similarity index 100% rename from define-double.h rename to PairHMM_JNI/define-double.h diff --git a/define-float.h b/PairHMM_JNI/define-float.h similarity index 100% rename from define-float.h rename to PairHMM_JNI/define-float.h diff --git a/hmm_mask.cc b/PairHMM_JNI/hmm_mask.cc similarity index 100% rename from hmm_mask.cc rename to PairHMM_JNI/hmm_mask.cc diff --git a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc similarity index 100% rename from org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc rename to PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc diff --git a/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h similarity index 100% rename from org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h rename to PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h diff --git a/pairhmm-1-base.cc b/PairHMM_JNI/pairhmm-1-base.cc similarity index 100% rename from pairhmm-1-base.cc rename to PairHMM_JNI/pairhmm-1-base.cc diff --git a/pairhmm-template-kernel.cc b/PairHMM_JNI/pairhmm-template-kernel.cc similarity index 100% rename from pairhmm-template-kernel.cc rename to PairHMM_JNI/pairhmm-template-kernel.cc diff --git a/pairhmm-template-main.cc b/PairHMM_JNI/pairhmm-template-main.cc similarity index 100% rename from pairhmm-template-main.cc rename to PairHMM_JNI/pairhmm-template-main.cc diff --git a/shift_template.c b/PairHMM_JNI/shift_template.c similarity index 100% rename from shift_template.c rename to PairHMM_JNI/shift_template.c diff --git a/template.h b/PairHMM_JNI/template.h similarity index 100% rename from template.h rename to PairHMM_JNI/template.h From d7ba1f1c287fb238e9cf5f67d07f4ebc316965de Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Thu, 16 Jan 2014 21:36:15 -0800 Subject: [PATCH 08/72] 1. Moved computeLikelihoods from PairHMM to native implementation 2. Disabled debug - debug code still left (hopefully, not part of bytecode) 3. Added directory PairHMM_JNI in the root which holds the C++ library that contains the PairHMM AVX implementation. See PairHMM_JNI/JNI_README first --- PairHMM_JNI/JNI_README | 1 + PairHMM_JNI/libJNILoglessPairHMM.so | Bin 0 -> 66472 bytes PairHMM_JNI/run.sh | 16 ++++++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 PairHMM_JNI/JNI_README create mode 100755 PairHMM_JNI/libJNILoglessPairHMM.so create mode 100755 PairHMM_JNI/run.sh diff --git a/PairHMM_JNI/JNI_README b/PairHMM_JNI/JNI_README new file mode 100644 index 000000000..2a02d41ce --- /dev/null +++ b/PairHMM_JNI/JNI_README @@ -0,0 +1 @@ +TEST diff --git a/PairHMM_JNI/libJNILoglessPairHMM.so b/PairHMM_JNI/libJNILoglessPairHMM.so new file mode 100755 index 0000000000000000000000000000000000000000..f7a3569bfa382d5e2f7267714e18d538eb9df1ee GIT binary patch literal 66472 zcmce<4SZD9wKhIs0z^e;u!vEyjCQo>_z?&QI3O9A$T@H(MhS`<9YYcznlF<{1Vsds zP>z$aX$$^sFSgXHw_dB2+lzgzMVo;PUw#EFilP!x+Irvn z`wzd9eb(B0uf6u#Yp=cb*Ws~j<>mn?DK_(Gx7}nT)G=1aWEl~;Zy7HLvTPGDy`@HVlo{G=E`eYfgYcvo@o7;PAi^#Nwx%=@*gDVBLo?q1fXpWu`J|`| z5MdVsn~Mo7p+Ux$3LKv%7JP=#t)p#b$h7?B?h6(1-T!+5o))e)=$WCJ?^fg^pa1I5 z%I7d-vg4VaGu!q|&Z_@&cJR6%{_yEzci-~+E$2Dky3kLW1$Z9CQ-bG4JR|Y&_YfXQ zyy<^@#``@8ndf#q<#;CH;cp?Hd3adT1Zj}G)l!;1e$eU0fZG7?jVtiSUBuxtDP&*gZoz%v}r zkMZ!A>}GgBUtshJon+@?`}MW;!XdPkKk53lPx&DX5*QG$4LOcOY}heeb9Jc zfN%~T8BYbCDm+v0q~iHUKRcwE)kg{6MCZC3wQT-3J*QdjJP;#(Et$m6#7n?i+d+@D z*>N{%PdO~5(ZJ8dKhO{VZw%GbyMEm}{qv9y`FU-tOGISo z(Keukt{7l4@brWK6_oHy^&NtCJQKbi27~-}EYnF>8YK)C@;t|uu|h|*u@mAqQqVca z*4CuM_vwMH4)JHAKeiuzU>`o;HpDjPd4qrv%mY7XqI0UB_T7?p=JdZqLx_4kqZ0-w zemD0+M}fg(`Zl8zndx7Jo)DikuQ#0u{m|(`gPn>0^ZSwikbdM=X2{Xw^~29IhJM-%{dDQ%Y>)KQ@7ns|r`({^)Ti7VVKC1m zpV0>Y_x0g_pwaJ|13DwQkQaYmgZ|P!^q2S3@2=~oz6<(k$Nw_Qb>-^ineu!Za-f`B z4STqj9TY!~v6qXt-ha9cIvIWF4C{x_KN;Nf1Z<#f4;lKf){tA4h{gEOj~ur5!)Inc_@jnBY^u@;n{vAc`b+!S z#=#4}wR&Ld03Gt5(MR8&??+E=2K_VXkJgWVwix}o+0d&>qaU4R$f4CJH_LE%ZWs0w z{XJJ_(B$)HM!m#fil`=o|EhlYJlm*m(@Z^Ko1rHY`r*@$^3PPSs($MGD#!{wyj7=P zX*f>jK}pU;=Lu6!jCRR_-1!^P5B-H8;xPqs_0F(N9ge?J&mQwq<%c6W`VkKVKjj`;pCXLfZ_Z@IXKE4D~af?h*Q_mudZL z??;{|`q57nc-E`q!QTA;JNOrJF#6Y0qmT#t(W@)_X)pS%q|$ zakVXYS5awh?W~|}MnP>su%c>yaBfvaK~YJ0b*Lm%6DkJXlJfb%novn;b+EFaq-sG~ zS#U;w6 ztq3iuEJnW387Otkm_j{qa9&MmX|S@YVs5aYc43iRl${sMoe(On4iy$u7w68(_+Mbn z;H!rx);nib=!V+j!caw(JS#MoRg-f^j}G2HcTuR=S5;N8D7PScR)IWgY%n`}^w?kk zas;CgOmS5xJ7*%qUS0$FE*w2R7%Z=-Dl0%F1Y1CYxyUu^24pO|Z6;oA^Mj=jYA{q% zRxH}5L^Mbh8t8$t%EDk#Np)pGsBnRZ3y!Th4PRcoFhqQ*kX~s~@xn?rbf~yAIIo~O z6f7$)t1d>X8@OsTZDz2lxRMctm5X}UHYr@zF5`bxyNuwhF-GmsKDrL|E@W2lpG%eH zQ%K)mx;F%8O)$#80p;sboL^co7mW~FPzC)9me-V-5fu*vLlwb!6;%(In97>!1x#Ti zhC1jZm?l_QP+Dpt(Aaug6+!u{5K)|rDlRVyo-w+YW_=Ga_(x~g_7>u4Ep>X5f2z8A zmf7VGm4M&dimPkNU8j}&EL@FK%mKK-K4=o55S5;M1 z<<2bB*&012c)RE@!MW&~|Ga%iPeiYoUmObNRR$rtibD1cvyR`JSSE-S6&rMlv)RSi z!qv59Fdr2UjRlXRCsbAyn|7tFp!$L8V9{+elTs`x$@Pt6rn|;u&`JjNBxv*?>ugP$ zTbNr`0t0&2n9Q2Wq5{~#p0tG(Foi`Ww@nP71PCPPM=uhTnTzCeMvtCWQC zHF{j30IEwW%JT~5mKJ9-$*fQx2_)GxyQP1OkQEZb!syL17m6(;U?@r-691w_8;7pL zLdGK$g}_RbRn$T+{uQ>qPph6*r_raiVuomt?_-P$W@W>wihj@GZ7xQ%a&%hQ&twJs z7(S{h76l7x7Rt)pjO=Wf>mnTo+x)NDeM%=gyEZR4Ybs3)i>fLX&0s(@0Lfx!TBv@H zK92?r1Pd1w7e2r)mtEF>U;CHFDr?JrpLVh^|Br^kUgS?}$?;}Oo~D)mWOOwY9Xc?f zq@r3Fm$5SX=d`3j?2Xq=nrIR9%!P1w{-2$f>0<0)q5to+NJSqsa;r)zs;YhEMYDws znO0m{TK&IC|9_KKE&7f=Vpxy=8X>02n$-LlR6WX z!aUFz7wqYu|4K#16_!?17bi!&e7e(4t^%hOIgT zZyl8xohkgDr01Npy}1*D@-4xeWd&8{a9jk(yvzoF6(h2(y1KBSd>$fnZxU|7Shg38 z5lBwd{aVZ&3T^YMiYfll<3%ZGjG$Q(Wo}fhgZptu~Q|JCeBm$ZS^ z9KNcv*@ijLv9Z;lBg9-@AjaQW)zCn+H~svH9DVaDo{wiZQPp9#2;i8XUW!A38E{>k4>olfrRhW($n%!?NFe=Z!(6{^K%qpp>&c2PV z4PB7$M>okb#jCp@>~+)Vs04g-f{L7z9aOS&rsqjIeY4Vqdhh(+j(*`RbW~2Fgb*>t zn*3Ir(FSB^7FWzO7%LeMRWB_mho@LbA2&y^kmQ>+Kxc-Ium;I^qlQqm>IX_HAF77Z zCC46-h| zb_7GN$#zLbz+&q?)8ISrYc-9`7e^eHoiWl_s9!GG*&p^MUi%H$mJbdR@($?r2! zNY?B$2c%l`{eP|{V>#_H)TFN$ZfffH;K^EEO$W zS~9n!Y~FR%6*jza-ebI$RF%&+Uds?(cLU=~pg-fxw4tIxk0%*5zYt?Gq0u5?d8o9I z&^SVs`t+G_24N8+2o|xp>&DoE#h4EjV8obPU9GP=0aadPn?7w?FcX=kEArH7!7-rRkid2oCv|>9G`Qypfj9 zg;_3nvxCDPD#WDVjE~LmxFAou~^9Xd`o=I#UwsEg<%-h`jt)EmFXfYUek#=8vo=|1>12Asa7PkIA>x(|Mh z0jC@L5vY{-xeR#0MxD+eU?Je-uhN1ymdZukLJQsyjleIp;Cbf6{H?IyuQQPdn=E+i zxsazV_{o;^Yb9HJ**6sPVTJY>K=5Ln;4+Tm7 z+AMg|GJl!{Z_1i+9TxmqW@3b$7CaQR=hxyA@1%{bm*g+af;YCJM4Z!tx1I$`x8N^G zQbC`x;CZIN{Ef2UdA`g1Wmxby{gM1lu;6(v$NXhk@Z94yf06}%sfk3GYr+4}f}dl- zkF?k4!-Ah~!B4Q@B?~^wf|o6L$%3C@!RK1=GcEWz7Q8X!h}ioq_#8|61s41*7JQ`z zZ*0wp*o7ASt(NplE%@6k_!SoXEDOHLg3q(ypSIvz-JkNBR zzt=4IyGa%87g_L~7W_O5 z-nPjlAnN~o3qH+)FR|dA7X1Ade7XhyfCcZe;7cv|Q5Jle1)pKTS6J{9EO=upTf}Br z@Ku)dk_8{K;Bzhb8Vi1o1z&5y-)F(|47B-MV8Jgmkq9d-_(c}{LJR(33x25uzu1D8 z>JFx9zZ(ccHx(|E_6~TJ>RQq^CdS>h(q&6rk&nNQky&^nyns35Z3+CYxQ#H~k$9_s zXAmAhxLLqc3DfnBuNCl(gi{Hx5%3toDA*Qn67bc8!GkTnRKS-L9!$7Wz!wvy3lhIi zz~>XD>k-ct@Y#gva>TO)Jb*A=z<7p$kN*NNcK>a0mw=BFK9{glzy}D^HHq5%EZ?Q_>Y7yAlxM2 z7YWmKi7yrKvxIpRIbJE?UlFDY6TeTuKPODrC7vtbCkWGJiDwD;al)4n&Jgg!gz37( zT>`EmOqV6@6!86o>8iwS0-j5lE=s(koAtkou#0e;fNvvw8R1p|&mep`;bs9(B}|tk zzE;3D5_S__Bj7QFuOQqc;HwE=NqDJ%FDLwC!j%HPm@r+G_iXb2Yd}-mw=BFrmGTn3itqFy7Y0IfcFxzgg^4c}@Uw)+6Rs5SuL#qX ziQgySpA)7F6VDa!6NGOhoF(AL3DcE{X9)OV!gOKcE&*2&o=n&&;QI;Fm5JK~JeM%1 zCh?9FqWuYH5pEOkZG`E{#9IYCgD_o~c(Z_~5~k}CUn}4n3Hu4J5%3tobXDR_0=}B? z&4iZ<_;SM230DgEV#0Ju;`a&oe8Mu}Tmhd=cn0At0S_Qd*CU=G;Nw37tPpky_$Xnz zAaSRF4-ls75w{6=FJZbI@s8u7{R!t1ZWHhh!gMj>tpeUg_%_1L0^UG)7U8u5ew{E~ ziTD};|B>))!c78xkuY6{_)-BsOZX1Lm4Fe7AMLdi9{A{!E}PWQ92$^l73R9h{LNlE zc?ArgO;Y#y@^D5+s++t3X>Fk)Qe?8D(Ph(aL+eVBOTP)bY)a&cFA;#=J;2Nzs$GteXf3I(IYm+jPDZ>hF{}kiX}1wywQtkFAL2FTZeRY^jq@BM>ka-U zuK`KhY%mbI7|C}>kQ|(zhw^`dn5~I6jwf3;>Kt!0Gde~tB?0D>>b`9A<@sj&Zui|W zOImjDX;>jyJ*B9PZW<)@eMPNzSK>`kHvzW-Z&GBXyNSudgErNRH(R2q1EYzAq)YV( zMLi~k*9j74@}4vcEoo|;z2jvy2D&19O?uy`mSmKxZ`9^w)TrK3VS{OKL*AtDdN<2g zN4QnRohqs8+_^-n>)jHfW&@eT8J^mujaXj_|?_! zPY?;*V&MMFvAij=D79t7qRaf}+G58a^Eaju7%~L!jg2HkaF_$&%0|*_*}Len5h(yy zH?MVGm+2c~Gfc)#YyczcQq&L>K1NQ)R)|80A#gWUFpukn-p2^&Se-z_h6`QlaNCaQH^d5hF?M^ zmxx;JPDiLfSvrANppU#aq^Qriod^T!+w6c7MACZqD1_*9V-X<9N>K{K`+#6L#|T*j z;SPjacUK~jd8lr^VAe(QFJFmay_+b@rP5) zy7niFfcl;(TT1e26zUI;&u(aTEFTA&*_nrYi7kG$)D2@Wc>&BV)YI+utKShf>UFw9 zoBi%sumtMAW*GMmx+wLwr$;GMA(&Vh4FktUMo9;K*7?ldXlce@mIu^YB2 z-|d#w%~GV)eT}UCt^E>zc&uAMC|<-Wk;OPu=NP$GUwg?0u;_&8NPrgi&(lche#xu9Hem z&zfqT^-%&|=;yMQ6_EQz$Uh4qHJcsA@k+hX z0bVD6nOmWw8H|UTR~q3`BV343TLck^q!PsCL?I$|M7PP>BZ%d2!K4yLj+b;Qo^I*2 zz{iDie!1j<$E9P3CHsd`^!N}dnwTP~+a=F?lJ}$PVN%IPT}Cn59g=9jMDlzrMHIB3 zeGggIS^8M}2IZjHUi<|bJ$V^Cuje7miEwzzk;w5*d5sFa*@CpOw&?kAf zRm)1mEz8j_?TY$EpyA8VkEPA^?hZuqCxyLILn1U#vwcG?+n8Q+h$9bau@f+v6j?3c z_A!~wl4l!Dr;ebx0Mj}jAF)zQt}XwTLegpIw4XbQ>NGRK79>CiXA_}=f0GK$uXncs z$)DMGA>au}0Heqlqt~k&;WzxE!RYt_)ci9k>`eSj@AviYT}X-)5B5osYf18go+PJl zlJp)bujxrVeUrF)k~Hf{()uPz>q)W}Ni3BiYl)^guw6cNQy+Y9D`hcY5CX~cjjwmx zT)l{kO7${zlyiFTM50=!B}(g^NL1{!L{22i&it~+DqjuD6j1lm{zmu1p4JO{Du-vI z%3}v7e$IL-C9wrUOuFT#(WPq3;l&rGW$w6H9h<^lM3hhUX9yx9UR`a=MLxwyKT}VDc|I1PWH{=UW{<82%9OHg0(s=1zS6rg7rNu1(hb5f|AfvC?LNDQM9WX z!doI^Yi|j=kKLjc9mCuKgFTQrkx54a;rVG87Ub|?$1JASG-G)f1?s!V=rEG5< zYIe%%^kmlfVH=XDtzUY`!}ACtXTJF9lk?-JI2jENp&j%*<~P~qv1uQz+k?r&SleFM=*9{SYg?b$ph+L z@E$e~{Pl8|?QFmMt=mLT4s9t0bpu|2hg4Trd+-W^( z-vN;*>Nlp6OP-Ic18ch!Jum=M+$oaheJQa`Dm(yhsSKaq_`)tV9CEDq4Wy@dj`U=a zkA2N7a`e;`+4BLiD1}D@>McgTr% z7oKf0)c%h7#;kBkFM<+p)TG2a3okHZ*nL@#_=E6``XVLuE!q2SXoRGKk};f*Zl}aH zrpw;XY7R(hoO&tz`otCtGN>K;b)!DKLu*7#L-XQ^OqNT@)4MJ+4arzLL~2hwz;AnkISP>Jh0_7rjtA7MInX zaNUWg`LYKg-n}j!Xk1I1=v7 zWx)JTeM(79g)2Szy2sJY{(Q*N&2&?$U2qWP8{V5XC$0l5T6=`85T4}Ljv^aKWEOzK z$hh_Q*=(=h1WP1E&TYLH>3$(HO>P4o(a$h?vL5|kj1Dmxf~Do@Wpp2gHIZI}O7Q29 zJR3}_=6IZ|>}DRx+bTJxZ;+C6gc&UW_dC0x$vV?B z=g32dB~9(XgJHp8PS^w9w;lD{!MdXE&)h0|x?aQ#;t~=2Y8a4y4Qv&Q{3>Fyj z2I=+LgaPfns%Dc9DUzxWY|+mLO7`tiL`ulsE!8(l(a%vceQ^PJuD$7IUk(pD3k8l2 zgq5>UpfVcG*k(!EYPN+K_ri~*3Ej~(Q1R|})W3&H%bt^&TVH35*$$DON=uQ@O_Tui z;+VC{YlQx82-sUg=g$lebIIy3+bv)O9Rh}8nbHTI7vpE@)&KOo%7Gt=m zvd)DR8D^<+REj)|A^jAqtUIMFOR{ffl^aFJnGsI?F?#aEoN&>PQRSkEs4~|B4084j zau~BU^mN#Zv$P}o*yR8j(v#H-B|G{6yPvGy;WTY<{1_&*=-F}h)01Da0|;ZWY>CUJ zH5#^HDR?pZg|=XSA`y?W-|7kHJc@*cNN6O8|FI`-{bIyTx5WKB;v6I4Tx^jF6aH|% zJ57lU`>X7Da{`vLI%k9@eJGzZQBrS`H?vi2h*&tljFHqXDH+#W~56eNU&%}GPbDSAr2P7XNU%(Bmjq&7pQt+Kj7jvl}Yaoga) zIk!SbZ%dQjZT{5I@xJh>4}DTNnUR8}w}84` z@qScwgH*Uf;f#OS&+)X%i46fYb!b5K4@D2~4|VcSy1dyx)J1zP*F{GKyzf`-%Ff(c zcPeGcVkLaR6SXc|;$6SGb=l`rY__G34y=o(B%;SY61}i4ibmRt$Y|HVX#8TI9WR5p z@E87#1Pz@2#0a|-8KUCZ;#+zmwdM!XSwp3>ZcW4E#FLK4mF@V`ZXdc^qItUGPa`g# zan`Vp{FvHAHfFMw;#Y$&h9wB0h{pQ-?dSj@>gtKT>hTkFu98QHHvxtZuIoXxvwC=S|5@L@A=nMw(!6L6qro8T2I-4Kvo42ONJah zLxwfOLwbfhks&LM#aeQ7gH;r9Dr6~Qww~j1J%_VTj+gWtZ(DO*ujlB%fe%^DO7E2; z6a@WWo2jwza>2QJ>J55oSMSsqtS+cTYK&5&^kw0VXqoeo7L77}4!nx>U;wgu>d})K-KIyMrAHqC>h(YB(LZE#RFD26 zvZ3s^82ySKoxpaM_N*R#w;ufxqbVD(*2d^MJ-S&Z9%1x<>d{d~7wORvJ-VFHFX++F zGCCX4hC0;)jnzjim_xy`O#Mn4CPUx)!qC4vX3fT0a2m8pRujGzZ%Yw)I4jrRwWzgr}6WN*ikw6=jAf!ayG=y?Fy^OXoA3(bx{`6ryZ`H3MZ{uN4wSP8`Bcq&GgsThW)rlXRbz54&{64ss(eN4Om zvqa*~jk6^63-15To-KtR03NeW?g%bB*fc1`_8Mm5*oUe(_sZGmj_Mm}m293@Z3wVG z80#u;j&)VGBIf1J3dTI&fdDI7*i?!v?K+h}xT=!jsjr?9aJO^d32f2E4&AXMb}0Xg z*r)ey(QbqAQHGx3$SFMoGBQKUZyz06zpwQHTI`lodD+ROV5n+sa^3q+c_7yLkoG1T z9IM<7ttel+j5-m%E?Yh53*Vkf^e&*K%?)h`kl*HuWT&VNE4oi5FodY9Mxi=tf!8$FMa9`F%J0EuN>J&ecKbh=;0r#ucjl@w8)U_ zqg?~0M;2c_K#jP7$&O5);)`|;^hLFcBa=qMpODlyoQRknQAVXqi{uT>iXO-4u>-#4 z&n2}%BYV*{jPuPSG1j+9o;C?HcD11rd8BZJ0rt*;75De?y8D@9(V zjI{BeCK4^t>Z@~5YiC0xDi}L)?^a)3;-urrQ>=L1=2W?GbFBN0=2-X8hFp@b>vTFA z9~%leo;W}1tJ~ip7k-$zHP-E1#psqV9!+mfvGi}`Tlvmf@*!S+BVOv?S^KVmX*Hu- z&8Sv0s>6)xFrzxmC@1FLf}nGaLD2agd5b!^xD?Jz_#$^;u;0Pyl|N9(aZ>hdo)KBs z&4On{8oSqm=8VY7ZnB2?%5gkhX!;?iq{L>)+ga7#KEzsWPB|KyARq*}=b|XJ!Fj0& zTCWj7(a31DqB5}+3ff1@YU@d>^t^xwtXiS*c{7$g$6-0U2ybPBnj6q=jv#7 zc=d8p^wqUJ=BxYSF<-cGx!BnK{IO|~N$GX_AB(l;*S{nBRC|77w`lzEI)TUuH^vCh z2seHz5}X)XA0z+R-;U{ZTo+>{A(qA%>y8cnm|k7LtS?p-y&D{lmnYZ=&TA`k!Qt_W zD<*(Lxp0GDeVKyrtE-&()U!KxuYZcnwfOGb{nAq_@D}rxZEo=mE&I0R^R6YQT6|qg z4)9Mi|9mSIz8fgqCKq-F3O~gJ#wACOj*;3H8q8tW=+mI~if3cn5%Ms=;}a$)n&)v|G1 z`cX*yiu98yvDx{X@Vp0c)bFHXqI?bUBeA|fZV&WBV}93WNR#5wu3CcM`*iJ_?cj|aEnrieVk66 z-NFmf--S+2PT#{;Yuw3x2L)NZvjZ`|EP6Jr z?NnIXX2sj-c>EyDQ~+};wF?daJHG7MW>^Ba0t;!=VZ^S*!LFH+u}{GS?_@bMBK4w* zISAPzIg!;uQF9`VqHp9N9lK+bKfL};8WD^b`m`C;XBSc|g6U3(pCE z_@y^>%ee2cmft>*Z`3yW$k5c|(<76vV}9awn~@XNyHZZr=$}Y8q*GDH;+b?@3<+IT z^-5S&$f&&%7PWzO$ADOim3=AF!1`iD9;r6#=pfM8v9H_f@OuuxG!DE}p=sRN5?!=A zZ+RM(;DZ+5qTR~!bSTBg=vXbj(Pi7&nYK|mcA@DSjUL4uqu8eq%^VY$BiXgA7Lp1| z-A$T~=aRi@wa}W_5sT?~PVZj7z0)uqvCD?PlinQop}HSQg~!kXMBhq4QM+J` zQ5HBLK!4>o|iKaEvb<;2qIGl!iT`inOw1#-bc5->`~{+$z&+@sI!4N5k8-b5Pq$* zelj}%-x?>gi|}pbWZDJ3t)9#wgm2GHW{2Y2OOy5C+xcoGZQHf?Ui!5}J3j5|C8^-8 zf#-@5MmqFxRg)e*lOe|Qq0ggWQHM(6A07HkF5yRq7B}(F+Spt7zSrWLoWBiF#EB@Z z4b)s4>vjr1G4}Z#yJJU&y&pR{G`kgl8rFgc$Jj4Y06m`zs688r-(dCVeG30?`yGey z{!A-zUBljqeLid-Qoq5}Enn26?}T}}C;eMeSVejWYg)|`;qk^!Is<@*He@goHETua z6SI=3ks&YZ?egjipe{vLt?fn{ILAkK!l0>p#Z0a44Z6(TXk1MDFilwsNLE`+!}V}Q zq?NfIu4w2$2t&|>1TX}Z4AeJPAn>Ul`qeIDO(+d(LUmi46VbY^6i5ALY}uei%bI+N z&EWb~C#w}5U3~<-#wn!R5}jDxkrG?vT&0)um|ormcfQ)e8U^lrbZzXNJ6GirIO(iu zZHb;c2Ws<(bNxOcvv=-nXkuLV(3+-LY|a5BdSfjSu{)p7&|4D}y61d?;-AkzL`&hd z>Cdw+WMNe+lRo^Is9ehzSIiX6+8n#~&Sy}uL~Q4s@z_0gK3~av(JSWsjRilqPc&Tj z&{cxm#z!wYloC^`yVGJ_)t{!d_|9>D*b;q^aJ6$+OJUdWc*+d5ao<`H1eY(fM9dj3 z+J%-Nz#Z-GztR!Z!*)w|PTpI4tqi~VPHC#V%V54QL`&Tm?N`-pS(J~2}` zIW*APf@1flAk8Ofv5%@xrm-B<=x*okmS{?K0vYYjy))GH`{@14iI$d)7o=~3!MY;- zU`lN9(3f*rZ$n43z1@z+8-ake7wsv!vl14%VA^~fm8#q9QoJv-UppE-Cs`Y(=ksU> zTa;BlFay>ADH{ihsBY)Vfoow4fLT2-LwJ+U=LS~75&*MqV8;l)3~lJZv=3o~(?5i( za!W<{j8hMvuhhd=GZ>=FuVpyy-V0v!({J})eRC?z$GuxxqOK#VK(}I z21@JO{~I!n!}>tH!r4xm04K2R64L)mCV;8?ngG%R-B$&Dqg>J1F#(6>M|3Gz5A53%l1Te2gishIG04{$s=9G1eg z{E2s98X`?B4kltPLM8k>Z39csr4-Nvl?eT6=M_`9a*$T1rPOU{PBhnb!azu%g+_ll zmxbvzf(_PVBPi}38=)8Um|h@kglN>joo|RngPCx$#FMc8U_@97*pBC&dW*d(5*a2! z2r71GL#`gDH;7>+I+&SOg4IDUg;i)mvS!)|qJ-UGXjq2~vWV{4_-!jIPTN7Mv**0b ziVM2|#Ss7%kmO^)!@%v?H-!nl?kX3N#uAU`RSi>l0A;k@N4mk%$)1G=SLpZ)r)euifIi+qs7s z_Cl?M`t-FFP9aiVm4&4kL(|diG+GynC`e$K0NB~6XmD^CDEtx~Jh^ZaUPY~-nCkZ} zKy-2?$KHk^2};9-G1wfjOcoXkXfQ`<$Rpu!v7~--_eS_4@nMI#e0C^40vlgo;@ZPm zbW?W3@8{Xbx|R-KWO@SYO3?5o6oSkJwW;#OceDcQ} zun#MfIMRv@75(&-?ERbL@pBlZZdN>}S!n_9NyqY1biRQ4mCPeZIHGcF zw`AWXc{e)hxmb<`q(J0$2PUxgztA9Hn{Co9k^C_Zh+B@HNDZi;%G(dY1FFsA!9n}S zA?&_=D0@yR;You7>ZAcUpgR~lON$3!QykmL7YEc21K=T?KNvf#`2(;!kv|yK$sdr8 z{T{h)gDc>TRjr#58Idi#mIRKK?5=k6u!yWSEB1{cB>p(y`5+KZ9e}kfdmx-Y09&c~ zcI=G!(s0BSN2E5%$39W)`xFlekRtbV@Q7Gs@xTtryQlVip8CP@ zV{SfsHY?ud#Umx}3CHr^p}LGZwp&Em_sHH|jwgQxtW>ud=brEBWbSj2)yQ@(vekwp z&k5Pvxug^)UEwuH+o$N~sAc;nSWDY}0w+dlr((|?bE|;ovq0GI>A(?zQ61O?_h8bc zjKbc2o(Cp6Z&X_#Tl9L$zJS{g~6tz;z+^mX!%V};A%SJwLk{c zy#ddMp-@0=&CY~uwJAP+R2Xag8vt(a3^X*?j>FX!#oivm0f8ZQ96mtn)^+1doun#u z9!9W3TQ=e}4zy)g)%!V-Ywpp7au0;^e(eQvbRX0p#x+~6;Y%Hon%k-C$6lPXYn8m; zI_kHfOj&IgYOSCt)LYYpQtwZNX$eH8LFqf~V2}Eq4uH!3nY{gL>;Tr@4T%e#^z6au zigcWv-o>Lp7vgm3IMmw95CuP?EB#P{Q1=N8p}z~$usp9OH{dhoKgyo8FLl9C@ z^nhzRV7Su7evY`Rf$ff{^g4W7{v2|6A`gg1yU+o!NxawblRv^H$lkXD>ORFWJ+9c} ziZ@>6=*gjY57eG5zoF+(IFvBX?I>zY@*L(yX@i&9(FQz$RkyE0wnJ!N);@%Tl8`Wt zq)hSY#>D=2Y5OU|7G>_njtd*UU9s;L4evoid-PK%Ybgb42hi}d(W{Q4M>^{NAbQFs6y!N3HqE3+IgW^Q8pbLxg6c1qaHJVLy~258{Z(x7AmQ1E1+)jKgZlsNk;XYx3i;?*860l`FU3O5M2~&4UxLC9RJBPCR9`CwUaubpJ z{%1j*rSL6I%LR+@Lur!dT^tzXQ73WFq6?=tw@cmwj-MQapcEVl!|g6{jYHdl60ydj z-|*|Xq=9o#qKI%1t??1a$?B)#2;v8d9ZQ<0aCxH(rx1@QuvW{rQ+Z4Fk-Z)^aw4fe zl8+t40XKLyk{w&apDLaWvgaU7*AN^_BmryzE1sxa(#ovbwJnsP)!(196&!90wPmn3_vCCJk5RLlAWS(9Ay$Rvg5YJ9wnt9P5TIK$^(&tjQtDQ`-!9e z0~#_g5E*O8papFrEfcp-kf7#^_&TA>U(4#j(CA(%e`QITc)R9{q~xsfDtU)YC@LOB=I#^_@`cDxD4j&Yj%IL;iOf_{9!TXpai{szB~NB!QxVmxlu z!DGHjpAPH>FEZC*HsL7pK^{d$2}&556yAwbc;^nKiK)XYNDJFKYU8{ni~R$#&wN6X z)em@nc7m{cW1#~fca}Km^szX?OVgLU2j;mYjdo9P+(o0Oc#k@Mas>MkGOeQejU2vS z@?h)oL@GudM?({L&M+>+{k89s;df9ZJOVOG@_y~8UrS5j`4)Zw@GfIqz8B*%ZhS}E zF)rt4igEb_#^qh|c8%^~E)M)D`p`M#I=F~~l(2Fg+EC%4Ri5C)XnY;UG-VKkqFje5 ziNZsxa2z-;w~BFD_qC#X2g0Ssp?;3b-{|8q@7)^XGKdcwgkvVxpMn=>^9=Hnkx#Zc1^JAi= zG158eqmVXyCbq4(2dlUiNFH|{%!9%s zA@jV!kU3n5%{VChbyYKti6&hLB?31gaz9*%c=eCTFYnOWzYYChMmTgS29B|#W`q}C zifS!}CxIif80bKHYFfbFjl&KQ&btBcVMoLJScHh*EW}z0{r7z1s87dT9xM@1rWpYh zUL3}6e8Ldpcd6S2sd!4Ee8S1g#yFo1;biNQtH1==kQ~JpmDK^V5Z?Xp?H03(K|O^Z zNy#am&6(c`!O4*YNhdNV;veY9JcNN0bA_n9Js!ZRr`yB{>rbhad6QAMyn$dwnT*OQ z1AE-a$xzq4fpjA$qk76fhz{2cVkq%Q)tfk}S%W%&zpvnZ4atxqRS;c=sjgAU{x03f zA>+{(GEmd;AYPgQb;aRgXt%7+z>H%vW*RUVryP&(q1b{HN?nU&8OUf#vRGLdD`++p z7ae0n7PXxUi$#JSRfn))rLo|_4sa~4s1*>U_3p%yYj6}vgDPv{?BjRZZGWNRdIv4? z*gm5q6axCBE!jVnyzQakl6n{?x83QQ8>vzMbYO9Lgbe_b=BS^6DTlTT{$XF%Y36a# zEnN#^S`+90LT9}w5JFAhO6KXRc39iA9u#oN81l|@XNV7_=mW)K^f>jD_LPXnLAQIv z3(metVbIXt1*k1z@hDGo0%}*aix8l9qHo6YySWGiU5vj684FP=$e5*fafpnfgluO< zQfuXi5`#M6dXels=&0Yv_E2xZ9294`HuvuTT#LDTQ{9@WQ z3d!g!i_Wi@oj2;bZhpqZfp2Rt$CUmF@+2@Tfo3gkrc2Z_4^ob6i7EiNrW z#nAPyL)YgW0h(KJN1#h|eVAJsS$2JzTX9EVvuPNd1Bq3hh`Z;-MW= z#7j~;)&kV-{*#_sJNjoR;sO+dAlUVJ>%SCki5_?F~B=m$0Q4pT$jpk)l`)X)Vu zb>EAJzk{YYiuo7u!u$77v^4D+^6=rSdTOnlJbZZ)s1V*nDdHChxC`MC7$Yx| zj^UCUeH{Ig8?N%COLbrxHwYiW?qKF_dN5gN#M{t_Se?M<1-PQQ{4&ui^4%_eLI7CE ze>+cw)pt^d!_%QByuB~M(M5;QM?F|=ajcW$A^&?!i&ft&#>*0{Ms+yq|4Z2MO<08D z5UQ_1MI=nAyRhsN$Fh%-f+=+~hrxil&0G?|3^ah-o|x%glOe0u!0A~GMO7E`@(8Yd z4wJ*B8L+seHY|=U&cJHVVjCB$xYQHilE4geu?kB9GjLW-iDV}N-e~oeVwF+dEZaMI z8E4WUTqni#&d?w%350Agz!)SWlh9C$V9PP8ZlL?;cznN5PTUHSV7wc_h2{!9sv zQoI=ZUgWTbPdm6!Sq_Z4ZNULyNLUE+F(k-rzD2h!^I%)%!M2=4>nG30vLPBe;aG4v zf+9{)>}?booH<1;fNhy)*p@cawzOn^XWEv9uq_F=a)&3uwhYzm z3;ByViT6ESQ6*0o?qJZJ)7Q?F$aH9`zCam>6ipER+DRyCTQ48^!w^;sM>rI9#9$?y z??Bb_>95g8#tNbXmNjoMeB^uwj5ZhIc~f~a6m?4#7#|S;Q)uxRP$|1tt~qB65HcAHEja8}qTq5ZId$Z%OJLu0i$kg_5?JOrkHf zIexNRNaIT=?|!X*9cdy)co3p}(YWy`ea8w#q{CS81My;v6{AFc?RqjsPh!)5lNwST z5-)I7;mwC`5>{7RL>&5y`n8rrBK+!v@X=5a?QM|I_P=7#Sz^)|Xp}ZV^9ocRO*pMP zY7FNamiC&rV0|{uh=gJ`?HJOu|4z$AAk;NJO;iK5i|6sJmn?8#x9y?nv;~yIVuL*F zkAmcTEaMbx)hSI8@vJ{zPzy0iEeD{D2G`mre@rBV@rgf#VT#kjw0M|rmNIGGFgE6z zV`G}NOt=fvF)EFZUvixJEyMJ7kQPFHTD&Y~TXBdyi-oWI1O5aV@tXdDlNZmHa0%7v zc>MA%X5&Z}nFQZ-zg-jg`SA~oTJc$k7Wd!!-WNy?c?uZ9Fr|0~%YQa$b8QXdA8un| zD-DZ!+XD7&aP1$;kUco8{Q=g0uoiq&@*d?XyO{}K?b*_sNl@*;x`O(imi{u=H_iJA z+W?961`OS&Lf4D0^1#+`)kIdm?ddSjA{~1R`ddw$!!7zCd?#RV@xj)!KJc)hr>~FV z28!B^^(jX^|KCa}d@m}tO9|f~d+^DlM}e5_lsx#*48Eqo&otoN*-x%OLItZe0d)%| zNt_DFSi1g6g!WBsq1y0n)RW@N0^U>HKRb>)?AUc!lCF4fg7g{=Gy7pvctBMWZx)l$Bo6YWB7pvb?l9p!H(hJaAR6dSQ$fy_C`9iW9ZP{2!CY* z2AnNb@1p%XzJfe4}N?ppEiwL>nn% z&_>FQo$;TZk*^)S`PzC$zSu_K3vHCgHp*ig<+F|Q*+xZdqaw47-WPlsZ6x@j-kg~y z_A_)cstAJPLLS9Z7;GPcHWi5gom*cyXKC*n^FS!Y*oA8hn_*?v+|ABFYMu-;&7K|=8w z^i9|nc=uvJVdZ$=gO&TvxX3;Wl{=2g9YE!jiyiF8*rX;d7>eZqb zqY76W{^JbvYE;gDH7hrUmAhJBTw!8ZqwsYCeyQsA7dV?LgF+t;*bm~(8NZklI!wOA&jC~T&?QFQIZV!e3C0&2q_hab zkwp9Q_Z&w2eeRq3?%?qbx{tUjcdOS0lX;TM8fkRlv8Dl25>AsG+3&uN1J*XKl#D{e zC`M>ge+LHYI?$S z6R%i3KdgOBx@4VfM8=K03?IV`s2u^dE3?@*TgGy_#Qgzm1PFS&Nx!beS=Tuz^aC-7 z1IoC^{74|aj;CkfL=Gw)>pG0faiIaRt`D2*K6kvYIo3DVz{s~^V@sh2)2tN`q#WIIt`1^lQ4E65&0H9uV zliW7PlYfCAIcY{m#$1)hXXvnUmxR86iyySSP=v1!u#u(^WymasX?aLWNdD+Xul)p=yWx-K5e?He~fY(z8J~#{L|EW@#V;=__pN4 zsp^O~xuAziC|;byk}&UPJr=uDZA;vy&{+(ti%(S!P=_qXMJBlr zDJ1Xa#h+l~NBaY$zHcEIbg3Q#Wu;cAPVIej^OpnseX}j zKg2u5edF^u*3=|9-rQW*yeE0hReVF0_gi`P4~{)(M%KA`>r_H3Fw9zk4cU5kx`A>c z)N;rd>OcUeQTR0#CdC~$_#pUL4))0JL}RlGUpz-WY{o2hv-S|*)jH}acN3XK+X$RG zEj0;G1I&J4#{WBDMa36naLx2D%&DcIB4#v?0BK7})+k~a-_<&ysRjv=CLeg!-YeTj z|K6)4k!+l`G|`WK0Sr%RtrV4&JS%nKz!XMOBakpO8qI+E;c`ktCY-KiUAnfX;T_Xa z-(OKn8+H$tH;DFF?{)#fZ%wfJGa_R~}{=a$SMe9Zi@T4>JC!L^T62w*D z0Y3XClH_+g1<6SUNg}kL8YBsjBs-S&qLt*j(@4rD$q6ROxIu*oZIT5bN#4=X?&w7_ zs~#%_9I~Ydd(bwh(QCNwXcLRCOD7k=%^b$}QNL>@jXX33Mdi^qu7*wB< z(_^-?`S`ufar>F&kL1Lf0JHBj?WJU*4NMe4B7EP}YpbmgM8cupr`2t`Oq&gSBIfl} z#oI9xSS@r;=0x))QJ{kPcl2vJm+0|zsKZ(<%c_gPBX6%cY$oEA6(js%ZNhJPv!Da} zsS=nq8Xs(CfdifOK}j}nbP_@dulqflmz=zX%z~4zkmI~$j=uq|GoH-B)?AUpjz+_eU*t4 z>T4tFaTCRUp&bIHIL;X-$+44b9};^zv60EYxQGH7w-~|ePYESFw+-*wzoSS*hjbS2 zVe~3JI?CvJMsqf8viOT%gQf#CaYPkv3%`H8lq!j{0(4CDntx`>jmEqZ(b{4Y!$THC z{2MBP`PZ@*=Dr*ASyO6{Q~DI9jNa4K>wFFcf=>p^Fvnm=RM8K12y}o7*Cg@XXuL68 zYlL(kwFs*b9(dju7uJ|@yvxWT;k#hWDMj$rL4;wf2(5P8Pz=LPVDRN&M%c993pp7L zev?k_ucKWsv0w7*q|^h3=|<>8=-1buMmv7e0?}k9GWn5P?OAdR^UD-i<0Eo^ChiM9 zCotMC&}RIyEUUrOh3I*#iGtO!ph+u?!df@3(VrhN!IwQmGF1OJ}k5@v;U=BJY2#tFfAy8L0= z)W|UO`Qz2ukOtKa@=BcoXQ3J&O;24!VKZC%l|1ed8MY`XT}V}EXN6uy6UqR!%w~Kc zq86X_(#`)9!v@sv2#IeavBX64iecyA=v?@M0wnhisQU1Xl{*QNG>3kei>z10c*fkD zQth_HMq1j2^Afsd#91R~9iz2ZAzkr7=JoDo;v*~FjMoAn*Q3AGT1IQbnH;(z0QH$3 zg4${}L)cGO80JB;FVwUWou!5+At5Pz?TgpDAW7r98=w9g%Y}_zr>pI?g4Vbvhmi+E zmH!MJJ(rPG3~0bwaG>A|o>X8*!!G=#$b@?9UowLaz&IN}hl!OV{iAC5d^S_!;(umx z?QA5!#YS}1ND1FNt7L+@^#Wt7xl->oPXpOL#gYZBDkzy5pt1svOyYdsraeIkAQ3Z? zF+mlH1RR;vD&Z8}LdL-x5Z~(5BXj{UBRjtK90WtZK~xK^Boe8+cp|;N6N<|Z1T!Ij z?323$#oeJx&=+{P4s^DVPCf;U`~>Kbh%O@N)NEls;#tmG=4bZ-ZUx>0{DRm_76b$f zV%WwICnfblz9Om>DDBv<^g;;8buFS05U9bzv7W*#!R){$QBg`O+ZA&_A6R8><~$td z5nFcSHK57WYRobAC)H#*3N%%LSK_0Ul467I5iepxnENR zkN9S)sCkd+5Zs`52%aNDu&UT73|f35^->f%1FL?OB$_6>$n@#qcg7J9Z` zbXqn+jqaV+dB8FU`Dn~_fxU}m9v)bj9iasRj%;SIR zAnhAbBW;yQ`x=mp=W<9ngQ?TXg_!j++W%-;gZQ^6bk10oFkGn6G?U4*#f#1aE3}1$ zu417s7K&m7J21bDiBGZ6S_lL;RgO_=XOk%k5{1x+Fyb>tJZou2G016b#t~wBMR(|$ zS>zv4TKn!Z4zlBsfi~pdI+5{r_~U4xZ^#Y1!@`-7u(_aZUqyCT8_NDM&kjSJ=y=9K zP^}r2!oFx^(lxjg!<3>MV2c{7DKNxn++S0E_IyC7gx+bewt|7qz*{cry#QZ zAmyu*cuDwM@wdN&MnVaXfnO|9#qTjh5}yR2_5ME9?eKNyV`SELlWt_(-5jMO7p%Gh zL3r{9KSs=G)Q2B|Tgow~XFueAWU!u>ydCG9uwcP2-C`jKM>Bc$rv3%w351tmC+cP= zmLp_u%<=dd;COw=zQ^&WGOVwET6?pip1=fI_I%0}M)fo88PVi8vWYDV+-piH$2KeW zi?s&ec`^`3pCG3BWWmyP6%S5c>G9f3#NmGfkx@4HIN=0hdqh^3q-j1u8%snUk@UEn za&MY8qAw|&QN?|pFg&`v6fBqH{QB>~w&KBOEZPI!ZY7NW)WI=*54Wv(1!@oe6M@Ye zA!i7DkG2Huhh?#LEQ{{0`5MOy0`~o28g1hEQ;kh~a_yZ&a?3^X<>-8sh59-^^1I-$JL(2E7h$Y;>Kb~@ATgylIb5c(_7CIi_^qem9h}+K`^}!%~=s=>DUL z4F}D815M%Je9!n_kGMRY%LA_H!Z>Y$E&$!drfn9F`qsZqGlVa!OW`{(IvOMQF2_$k z7Q>-%iAQNKvS2VZK|iW2&Q>hMSaJf3{t9**dGME;M}MFi3QG|g@eW1AtBF|1w(p_3 z^9bu9Vf7Ycu@f>rUkv$Jan)o|08b3k>7Y+2(%1Y=l*)bTEKxLQW5IMGJb9fyZrr>l z zFN4(IC@K!lNnBK3tW+YOrihjkbT^60BSX!Hdd*2#7dk6FY!54k=uj>lCbFiT5}Lk= z*qQ_F6QCctrrCk(gJu@lab8ole};t(ZZAK>_O zu^q}$i>yP69$_P(YDb~9#KMe)P?0*M*3SDWa3G*PkI44dD9rS@VvcVn3EBstt*`VZ zkwp@3J!z1bi^zDDFkGZ^qA!(aO)4e5sT>CnT7yYt1gTsDDmpI(q=Jtzz9SfXw;mJk4lfc5V zaonsB1UO1`ki=Ux=v#!aXE6bd%A*0iD;vHT!(%SXLd>tIx|6#+{L~wFeqmH6fKTIJS$KC;v-@sm}Nw)1iVCGpNZ0P&X+IO&%p) z*aFbxQQDtTh~_65YVvNXG*b-_smL%YJc(^FLCZlZefFZ&As2czOoJZv?}*Y)g$*jJ zOe!OcvL7n-P0$3a%Z%NojR!5dr;uTMhehvUi%w}Tnxr1Y3kwI+E-jTz z{}jbhfn4#amRzE;Av2d(yd*38Hrc#`Y(^%}!Sq^t;0ZA-V|-k8OS%o~-Wc*Qp3KJA zGiPBiL2nm976dD*<_G6iRTLDJlvjsJLN%e{Aj0zb!J1G>X?3u&prmR+Sy^yK zj;vJ7FD)MM0OHrTUOJY4|EH~bU16cE zG})YfOFLY7rR(yZ7`<&us$Ds=73Ff9t+>4Q+MA{qhXh_JE}tJ-Fse`Lt6Y;OyUJ_I z@`|fN(+aAKtI6j)s39psZCq3}DAtXxX%#i4MXvISkZXQ%$aS;qcg?G)a#a-<6isD5 z6!(4e-C9#n`hD}EKNMGmN-D~y-;&EBzfYM(#ihmnx4EkUkFvPdv)OMUgs@rgrxehB zSFy$x*WDz9AY2*<3q)d!B(`X+ADi8c*|OP9vzrjDKeYHzS_8zNr}Zg_*hkcBxitN; zm$ryd`e3d6SZiBvFCzBVUhkDE-m5;^tLDCE=FI+ly9?n*pF0ov&dixJXJ*cvIdf*d zZ-;czljvijFVYi?#TGVDjVDpAGuRzVq;BetXaV(M-X(f2zudag>WIc-mg)>nIw8dQ z!BlWjA{LJHG?4CzbYXi2`uxRts*L1|G?kaaL_AwSNQ$gZi%dsEhCeg8C~4;Y!^aw- zL>ygDS)tBg59wekbm^R)WGoUlXPeV~KGX{p=nAGT3rE^}Z?wX_UESVPe@Y}ex>G&g zXwph_N8**22ovoveP#!SJ3`-pWXVA_*9!Lb0}qoT70T*Lgsl`bG6c=4Os2w@U)~-} zMnhJ#Bbh>HyJl01rkOH_qE@OW7)>Q5dd?j4)6+~{n&xnOA`!bJ!(3-o>WG}@jaOL< z8k+*(l!sLTr3sczCu8el||*@yjYP4ey)wK``{*=_uenyqTMi?cpK7>j94frU2gu z901&k*Q-Xk5HWmNFdOUDEr4x+^qoTr@co|;568f_{1>1Hoc&AS@%2U%;6}jf0e1jy z+&et{5Z*~_1>6C+1CajH(x#_|ho|8!(Di_AfE)h+K4U-!_z>WfKcc;W(*TbG#sH^L z`vIE(j~+rA?c4X#@bKB=&=0_AfE!-{f57(vXS-3)G0@{^_!ht^NDlxW0-XKd!^7oR zb(;Y3a|>}j;C#RsU>I;K;6s2r0CxZ$0z3+M6!7fv==cAC9&i9~BjCRGhKFlV5B-PT z6dX6VED?@=&!`RGo??p0<)cgK5G^OaNfM0S&`SiG0xv)W+yHUkn&DwomktI>Uysxv z@IG1?2qD}ayx&1%MtN437*y2bbk|K8cVu{Yc+$8R@VvlNoLILDJ1vo-M9b1K?B5Mz+-%x z31s$JNTASc0QctGy&i+Sd2sUz<88za8chSwohaqz9=hFY8qja9u$f}Agrt)tGPw(YJoV;)wGgBv4}nc3 z>_E1Dk>2hA-qd*2A&-H%BHZ{dJUHKj{BAHYE=Go?kviz~tST`u1B7Q(NuMw-pqSg} zTY2PGPleI9=E$uEUa1RDxzTs*rCUAYjJ|hYy4Cn3guhRUaKC&Wkk5k#EDYXV(i57c zRIi*Wki2tw*eaQdG8n@E0b$^kJ*5`P6dtPoXzx;-HV+Yk1Ic9v@W+5ZLQ{(a$>n+A z*F4Vh(tT#R{Dt%Tk?+HB8=r#lkrTu187NVaiEEuIVy-h(q;kDW@$s!1r6P5!N6YTI zTZ~$C+g+!oliQ~{@2*rc&uyx?=bor$pWCxfaby*1Sv6|)=z)^8&UFT=xElj{MmPyU zV%>p;?Sj20y)hQU_}XLvw6uQ))`Lfx&Q3$}nu`4UkY5jerz$ImTDXTkXIyp~T^#yc zj9Ye7%-&W zS?iKBW!)%g%l47TVVy{z2Mm}+nzAxX#go?MiBIyleV_P|f@AGq9VWsWplIC|psOd7 zx}NADT+p489k&6LU^!6wU>X=AG2oPnu+}i}EhcO! zsen=hsRG3RUM`R7GVNggw{U(X_;*8zjD^JiIO8r=r*f6HU#mwMSEEX{R+Vh6PGF^c zt6Zx`DJv!Wk1CgbXLz_JyM6PJKo8mg+K|77@~K(JYug+&?)di6RJE#c+X}ub@n%rU zh3ZD2dHXBi`wQR?0pD%Ir$GNa@SVU9A#O~Du zSl3P@KRN7~wFzTO+T>nlJV>BZ>YKY;E~-FE4f5%cAKCh}OpiNOC$vtiDmS(O>@o0v zv|@nLo234KhC z&pM2c@y;m~6ExqaVrkuuea$$I04x*%(M@0V1brQ6aG`j|KZZbag&ls7GJMe4=jK=h6s$ zDJijfHMvgFm#WWhnNd#?KO>y}Fp!}`K_Y}#W zjr@Hy!;xPPP-M*^|2l8YA#&;e&~8ifCnUl&{LwD#U5u|u@ylRhGGnaxNjn5Pia>U* z2K29jp6s+i7q;rYKy|gFLwv6X-LsnS8*o03!!JD)Ug89(cM zWNQaYxmK!v-+%!x47OkAQuXUgOgq+Ud$UgO2iEKLAbUNPjez?^uW>Ub zQLn6QXD71Z@$f!?Z2;UREUjxx);Ym?FXTAjHY%xwG#(ovz`_5>wU49+KLP&T7qNdq zJT2Rc5oafIctfpY+D{!{+I^weHQKf1Lh(VV>-`JF{xNr#9h@ZYIQL&m0PdO$>-x@= z)uRwc!*3;C2*Vr79ix8Y{O(06iFAd-^>w!wCnRa2;SKn|!qWN0;+q_fdmL|;Rv#QA zJ{aS=tIYX$S=p0i;x}a!|Jyhh@EhE&AGn>rbC+(mrslso`4U9$nL?RQ;W@?m!4x9-#(5O~#d)qj zoag-ed4##+e2Rbnd=x6y5A^du*}1v-6CD%ILy?P`aSLoeBKJG+kYRX3hhy3F-<511 zBL5Z^Sa z_qj3RiP5gFm5C2VZ*d^8Z44FOKStL?V{)#;a}LrMyB3K$!0WQ}zUy$TcYF$%H4fL$ z&K83u10{be6Mr>aPfidY7_Q$>5Z`i9{4v*Cju$J$OQT(DCpaG|b?qMS92ny|R^fbZ zj0<>-(ZdtOma>V6|9hDWEdFUMnErU|6^K4uKJ8t%_)~cm;+x&BKfA>BI64}~4fA1~eywC)#kL0+ z=$4C*5a%=WGMvM(nc)o#;|yQecx~sT;hF*qq7&bG!fnl8CZ4B2j+{ADj!<`Hd zFg(ogb%q}@bXRlz4808JFl=Uc1H(AO+Ze86xQXF5hC3M^V0f6}>kL0+=&s@V8G0Gc zVc5*@28MBlw=rDDa1+CA40kd-!0<3bs!iJjif`U)5)iX`MSlhA-~6%EUtK+` z3V#LaH=q7E`d>XA^{2<56izK?4zaMI;c~NbVe>VYm^I!S?+mlb=dbcr`>V~$C6TbX zD40_COKUC>mxd!9!QNOZ7Hz*g+1C}b!qH^6nTkP2|DJC+jOD8AEKwct@Zz_a0(rI<{1iK>5XmmTl>Ipk2el*51$222r*F2EKgR2`hMnC^ z$Uv9V*p-On)W1dL=zRqpVPywhzs9Xbgvx9AujKL?{>s*VQx{~o4luj?{O_v_4L4F| zRuDLY3FBMJtX%OC98>1l#eX`ZZbUZWAf!$!-aR zsX+Rv^+zL~L?pX>xQ&*Sp;`t0dvK$~5@ezysX z5cMN0{{m=7;ZOH}(QXqdypQAZCM$-PyRKjN>t)bp*WbCvq~fA;j|5_o0_mqNui;;j zlU-i#SBJR#3tBOmgD$7x8@BRm*pFR9f<^G?j;Unc_|q3I#7mQHe%h3j_dP9vSVoP@ z3T9TEqGTU!B$7gx|A9+cnyM7pi}V1_t3>Iv~vo_WHB3@}f?(POqccnNC<6y*}pRo$2$9T)ZLldX|fK z3B69`;zy;=A#(AfgLnp z?QAaIEwug2#a9Sz7jy9*q3vBRemv}$nT?WhCl1B5oyyL1inCyM%xsj5J8@c6Ul4x| z?1U+4vL8-yF6MtB{3nE--?{V?g`T&$_)iKwA9L}OvXe4SdRYQb$}WGJUxs*d*j|9t z+9C4g^BIZHmrtF<=gTJmJk{%&j!0JcBH|8_X}_K~gQbx`L#Ao2O_r{$XJHF@3F^t%x$B)1PFpHGTcx&3-wp#L_& zfm2Pb&p%>D5yCq%`Z<&FbVDJ@VaUekC5g}IgSJy8Fmeu@HtKVV4(3CbV-UXdDN|xJ z{XE9&a|KOrF@BAW{_C1Q<2C&Qnm_N4HUD2~{*2f3FKPZZdisP^w%L!v)qqra2!uQFb@=hwiKzUg!S7UuID(L*1OvK*iWG8~ckaboksDt#N{|H}N| zXa2O0qu>}-SZ_6-7avn}G|wrd^_KBpQfM(Ae5hX&$-zU2Dj=2-9{M(DYv&54zk})N zMJ|N_#&2f)CWRJ%VtoBBh0x^Z{mN% z%*9^_`~}dHT^kgo_BR8JAG$ghe>d=2$5f7bK2ZdJSo62>xda1Ih|jzt_-l*czfuH$ zC-CE;=Yr)t%#Gb9>Bov^+5XFK+>tlAjM{_sg(r%Z!&iYP`K;_x3^aa|q-UmCU@y~~ ztbm&SE#Rp=+n+WiwU5h`j}<~;7{Oq25&UJq7n0lJBKRACr}iKFnqu6|{I^Q_vBI}T zA&$VFQh1K>M+Q}TnF8YFBK+Sjg7@G>+ys&DkDV*=%rp!5is0$<-$MQE2442-e#Jz~ zc}o%cpE3QBujJCd!}$7+T)c@FhlT2`DS}^E1RpGdzggnp58SEh-32>KVNVhI7lEh# z4t-tG>HdDgt?-*|a%cp;Q2Xgy9pWQcKTX9>exp-}es2*zuQ8uNTfLL8P!{520Z;WV zv$cPRq=)^vQZ+<=dx>Z%I!N>@ZTx2eUx@!bMevUm!P9STXug{*s$L{zD1~9CdY@%` zD8CIu&QghIqAZXCp7{7|eD+Ix79mHEGJXAB3PIn-QJ~-SAS*8{1fJTnxl3i$!#|*~ zMbb|c_p&`DgJ%c@pm6lsf(vKBKu2A?P zuIWLh_u2IA=SBFxQv~lhyKs9Nfv5KDV!fhoiz%#R{IgtdJ^TO)&r3dIMLmy~Zs&h8 zz2JTgDspl0Ifd)JQQ{$o#ftCJdz|5_3JS@@zy>fzOjKm8Vl!d&1BjmuY= zelxe9zOAS5RuTFNe2Cz|e(gTiA4QFI5O}KBvqfP%oKX#Yp?bei1V2;+UjoOw5T6T+ z;4dkHzoG~}Tm*kh5&TZzNeZC4t$|{Ym4C9fTw;P`--Yn zw|_n3TX=r0IBrJ>_M}9pCzVX~ zc64|{!m=9Lu3BO>HMX`{7E<$bQdcx&WrY*gjq*+aOTPJ&KSsujyl(tjJQDWSR@clF zlwn1~{YXvs3-vk?KX>lBN!Jr-o-eCeuw?F)0ZXT-8XeEnB40DgT2ALe0L!wLcXjt! zH%EFBAHnw!mJeN*}Uw z^3l(b6-$IL%00cIl%(Yq_{LiyDlPkwcfpLJbT70m;>eWaroGp)NP z(QXC%`ooRkK%3Q4r++}-((1E#Os(shTT`{_2mXz%sT#UIqOryAx4ztdQ%YSP(h>}` z1{+&ztU$nDlY13Hpn0Zc;p+e^)Zg!)VOepyUI6zbq;LTPW?*9rO10LaVAqnxh!(0d z-6*QV2h#ceKwlxkR_ny=BOX*;Q64AiV9|mnQTfRWsn%+(V zbC3ao8M@Bz%Um$gnruzBxATD7FQ@?N47v;>=K_f(tto#^?)?#sA>jO3H%Byv0)dP^ zYxW({o+NI+z#`-I`~1E-ug~kpyt1yFNfoyx8sn+fWLmfCwG{R_oUfvjDq4g28IY`Z%UzMxRg{d;wC>F9}YN%NjnR7@w8HeX&7Id2pjADb zOjw=4co<*r@Dk6dti`Tr7OqF4TZuY?(U?s=<%fS3yoaFG6OnVUz8>>!?z~3J@2zG5 z)X@Ds`Td8TNDopCjedb?aXe#E)Ffb-QjwU2M%R!73yDxe@`gMu4l=3axD;hLlETuo z!FPgbvdKubrXz@ztt--%jHG01$OvFbPTEv6-S%Xp8Y1ncBB5^Vdd|>73$>-e!nEp- zrfdx@(9IJ%l*KK2dZo%UkV&$St*KZN{$QR{Kx=umavl;GJls9i7;3WutSc!3?}kFA)2@27-r7c@fHauB-`=CpsdcE};$U||vJArU*2wA5U(#dp6!Y2# zY}e@<=*%^K1*el1&T$s%o@60EP%dy-g45#;8!E>gZU?3%8o#8qowhH~fj0Os`Qwz# zAHI&BNF-OiPkURPS)f$^^g& z?@ZUDt)b#Z3n1^x5`3#)q1h<6-nPao#~Jl6mwU4uj~O1zOy27y3Qq4pbtKVI(7KLT z9(wnhJMYrFiqqN)foJ}1*H+OFRC97i_mYv|@sGGDX|TaDN|T?GWL1=VVy4Vi5d zO&frmO*Af4?eEsT?(R)upWWL<+jHz7;p}GAhBvka9WUC&Es5H!kaWmKV!>wcMW}fy z1Z76cH=j7}P?`KY+ zKd0@5@qt9@cP0MOF?cN}*2M?iHQEp}0C*YR4gf+}_zY3Tj613oWJS zr1M$2F6(H=;Xu4#EwnP{szut+WB#kHseQ`O?vbT%uMMDD)+~^#+%10+AS^f(h(q%7X s8Wvi7j0zvoqbj=8Phpj{!Tk4Pp#yMj<#0mlR??&?32O&11(^3CIA2c literal 0 HcmV?d00001 diff --git a/PairHMM_JNI/run.sh b/PairHMM_JNI/run.sh new file mode 100755 index 000000000..8b8d44b54 --- /dev/null +++ b/PairHMM_JNI/run.sh @@ -0,0 +1,16 @@ +#!/bin/bash +rm -f *.txt +export GSA_ROOT_DIR=/home/karthikg/broad/gsa-unstable +#-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed +java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI -jar ${GSA_ROOT_DIR}/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ +-R /data/broad/samples/joint_variant_calling/broad_reference/Homo_sapiens_assembly19.fasta \ +-I /data/simulated/sim1M_pairs_final.bam \ +--dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ +-stand_call_conf 50.0 \ +-stand_emit_conf 10.0 \ +--pair_hmm_implementation JNI_LOGLESS_CACHING \ +-o output.raw.snps.indels.vcf + +#--pair_hmm_implementation JNI_LOGLESS_CACHING \ +#-I /data/simulated/sim1M_pairs_final.bam \ +#-I /data/broad/samples/joint_variant_calling/NA12878_low_coverage_alignment/NA12878.chrom11.ILLUMINA.bwa.CEU.low_coverage.20121211.bam \ From f1c772ceea77fdccf7744e0caacb99717f0009c6 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Thu, 16 Jan 2014 21:40:04 -0800 Subject: [PATCH 09/72] Same log message as before - forgot -a option 1. Moved computeLikelihoods from PairHMM to native implementation 2. Disabled debug - debug code still left (hopefully, not part of bytecode) 3. Added directory PairHMM_JNI in the root which holds the C++ library that contains the PairHMM AVX implementation. See PairHMM_JNI/JNI_README first --- PairHMM_JNI/.gitignore | 2 +- PairHMM_JNI/JNI_README | 34 ++- PairHMM_JNI/libJNILoglessPairHMM.so | Bin 66472 -> 77757 bytes ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 22 +- .../PairHMMLikelihoodCalculationEngine.java | 7 +- .../utils/pairhmm/JNILoglessPairHMM.java | 276 +++++++++++------- .../sting/utils/pairhmm/PairHMM.java | 16 +- 7 files changed, 238 insertions(+), 119 deletions(-) diff --git a/PairHMM_JNI/.gitignore b/PairHMM_JNI/.gitignore index cb70b53a7..9722ad970 100644 --- a/PairHMM_JNI/.gitignore +++ b/PairHMM_JNI/.gitignore @@ -1,6 +1,6 @@ .svn *.o -*.so +#*.so tests .deps hmm_Mohammad diff --git a/PairHMM_JNI/JNI_README b/PairHMM_JNI/JNI_README index 2a02d41ce..9fc3aa8d1 100644 --- a/PairHMM_JNI/JNI_README +++ b/PairHMM_JNI/JNI_README @@ -1 +1,33 @@ -TEST +Implementation overview: +Created a new Java class called JNILoglessPairHMM which extends LoglessPairHMM and +overrides functions from both LoglessPairHMM and PairHMM. +1. Constructor: Call base class constructors. Then, load the JNI library located in this +directory and call a global init function in the library to determine fields ids for the +members of classes JNIReadDataHolder and JNIHaplotypeDataHolders +2. initialize(): Override and do nothing as all matrix manipulation is done in the native +library. +3. computeLikelihoods(): Copies array references for readBases/quals etc and +haplotypeBases to arrays of JNIReadDataHolder and JNIHaplotypeDataHolders classes. Invokes +the JNI function to perform the computation and updates the likelihoodMap. + +Note: Lots of debug code still left in the files, however, debug code is not executed +(hopefully, not even compiled into bytecode) because the debug flags are set to false. + +On the C++ side, the primary function called is +Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniComputeLikelihoods. It +uses standard JNI calls to get and return data from/to the Java class JNILoglessPairHMM. +The last argument to the function is the maximum number of OpenMP threads to use while +computing PairHMM in C++. This option is set when the native function call is made from +JNILoglessPairHMM computeLikelihoods - currently it is set to 12 (no logical reason). + +Compiling: +Make sure you have icc (Intel C compiler) available. Currently, gcc does not seem to +support all AVX intrinsics. +Type 'make'. This should create a library called libJNILoglessPairHMM.so .Comment out +OMPCFLAGS if OpenMP is not desired. + +Running: +If libJNILoglessPairHMM.so is compiled using icc, make sure that the Intel Composer XE +libraries are in your LD_LIBRARY_PATH : +source /bin/compilervars.sh intel64 +See run.sh in this directory on how to invoke HaplotypeCaller with the native library diff --git a/PairHMM_JNI/libJNILoglessPairHMM.so b/PairHMM_JNI/libJNILoglessPairHMM.so index f7a3569bfa382d5e2f7267714e18d538eb9df1ee..435034f78de8b6b51001484dd15d775e3fdae892 100755 GIT binary patch literal 77757 zcmb?^3t$x0)&GVKunM{h1&kIo+Gx@ENC*fRkSuIuCd@)AK~T{Uk|4o^q{#-sBCjOK zxLKQ8@Uxa$wE7jTex(*IRip`IdHD)Z6v0OXky!!~v`PYm{r`UV&Tb|#Y4!VmtCN{? z&b{ZJd+xdCp8J?#NxF7>kC+%+%on-GGXZ{) zyF-GuSloHlu;4E42@-9K+?kfY%p<-$J|}Dz;CJrlOn#AjB;U`FPt;Fl`;ULEat=Uh zJFW?nrrLgsPVT+49r=aVtIXBY12xR&602-l6c_$!dN z=-&(QbStj;xUR=_DK7q2;#w%4+W)Zb{IQY|{mm5^o2?jk4ObN|1=k8(kzXPnF2r>o zuCfRc_i?x;;hJaSMw|B%^ZqN`D{9^8M1>qoeznYeu1bzELt zc_z+!y8*x)Ttjj7GYNN^_Zsv53*7I)buF%oanW{e!F2~N6;}=}{sOqz-uw;2^-J+& zvpt6UqqruU@c+e)cGuN!U{}b0gS#5E{_8x&&7ou0go?y9QTp z@g%(f`^rrw9J!5%Jec<@%==ZH@DVTx4--wmbTziL2(g>{$WHhSz(2t?3Re%4<}~lt zTQ-2(aE-?`7FQOodvOiNH33)W-%KE`!*!X3a5J9ow!nN$G2sVsUx4f9Cj2n&gK-t% za+|mrxDUaViEELGi`=U5u+#*SBglVqx7#kTHFi??PhsR&+xplV*U5}7GyexYCB7!N zBYr6wl=jj2_jjc43O^Kv*%dwkPJ-pG?Zm$YiMvWay_<61L&J5Iela?IS9m-4^n)i` zdpurLA#7Ge!M#7^S434YCH`d^rOE1_e>FOq&& zH}%>L$4Nb;{i$R6KY~ByoNmgQB*`|QZSm%_^+rG16}}7x-jyDzAy`-V;pphX9_Xa- zE0H(b3qsF*Y?-SiqS5R)j|)D?^t8m!G}FJ*jXjif(~eJflb*h$zpbxr!7CW2S9IE${*zX3a8eO;aO@9Bn52AsXn4+kCm*2|miN(`37$D4SlMf^^{=@GxQ zlRX^h#%}-8O@Fzz8+q!xNq^RCue2wn9JSCde=l^Shq1s@&b7ak=_2x7 zdc7w2_pz05AjdD$wC8HvS>F<~Uq|#X+QcW(VdIx3{$dO_nj{)#9n+pZ_uWGFyG8@syPl(X5CGY$3O?;ob!F74DVy+wcNV{10;AyVHlD2L^ynSQKB zvat>5roO*0^_Ip!9KS}>4xj0!Uta=0*_D2-=*IuQ3Z9gwHdnB>m7u@zcQx)^@i}bz z;hH5fT^i)(Z$~%nQazAk^r`Sj(>`Tto9z`aXSqqQNKC{|?A^$D#ME=M=_e!UYr64E z6T89B??%s=-T3*QX1OJ%Uy8Kjfo|&iJnGez-k#{Dy;8cd&u37P#+zr0e95>hJO1=?!?nFJh;`3|t_pb1-cGKS-XkXf=n82fm5?PFG zhN+*VPWtH!JurRApJc+NX8Njb?7s@abS2MNaeEFs03v;+(vKd1bx=U%4+Id<*8? zpIz=NC@jk^$tftEQ&g0lF-g^m?=Q?RE6W5%@p`l8&MTNwK5Kk&Q3>+gk)M;N6+Dn% zSTLu!IB!D1{QP-JPDx?0Z(&J3@+Bvs)S*LjW#a7F<%NaWC8fo)vUBDy$W!yuv$8Ws z`tr+sxjAL|nNyPf4@5KNYL|)io;1aG%l!OYUva5A#W##qQ!|s3v**rQ=*xGPmgX$X z%t@b;qfQx?ot}P6X?__rm|g^kSw}zv>GS4i4^5ts@0&bpF4%}H>4o`4`SX0~naL^H zbBo*Y1-ARM3!&C*UqMm6XrThpB&Akn$-|<|ZZF6$%uCN48DWw+Dtqp1;#3cEjPezu zSxV6^>fH1xWmyzyim#+l&7I;)PR^Z^Q=087%_;Desky8{ZhHEZS!z~xR#J9Wa&~&= z#3?D+cTAje%MZ&Ya)`W?dE%6^;kgtdeLle{Wu9dD6xGadR3@{ups2uy_7}<-UyA44 zoWk_X5!rKdgofuZb0mwXzeuU&&%56@r#+ph->9OT2l9mpcF|5|1ByzfqmiZzNA0p7 zC@RU#&MPP@$?@gR5o*sKR{kCQy!-_|;tPdo7UtzID2b%Gzp!{#PGPohPAN<}dtP~w zsOyA4^Bydimxr!bR$hc&STN6*Uzj~Rr_7gKlwVYqkKStH%FrcKvP<(zNXRW& z*s-)|r=V3Pb+Afw61GCSVA=oKrpe_Na2cLkn4c4A$Nv?F_TC|ED%zk!N8OPR-sr{u z$qwezSBcgtejppxJ-fK{fe5Cgylf6tZ?*!uu1v`^*)XTV2%;>%RQ9SoSV<`e`B73n z)ZZoff2*{JxwotQJFO#=WYf*UG0)B^fHZf{dypnXmy6pJ-_WFNR;&2I>_TDk+4-fV z#if}Ob7gZT56!+)c&O}I(Bgk?(&SOu-t7DHec4$h*{FYUZVtSg*+YFlm{|&#<>i}v z^3yZXtss1M+595->*5E8LB`~fC8hZhM_!au_CQ&7-W?O8swgPPbPs2yyN4!mSjv`3 z&~n+Vvo&dEZe~#dtl;jUDdi=3IT(1_)8-aq?8z&*W0V&qKq1*4n7`o6T%u1VCyy_l zHy?@~5AT;eJXZi^1;z8Sa%L6grwetKjZaR87n8of?3SF85|{%$Ww`|yp9*Fd;~92%)pKS>fMkEZHWZo9Miw*kj+IAzvl1n#0P z|7+KtJPe}EEkvTLI!hdI~iY1HzP%D7GjP zA6Unf{}d$UOkwxp5Folk)OsmEWJ8S6);yrEE&stXXEa8zy~n@$Bx$ob>jVOa4B|hZSQ;;?PN# z-+7Fh;w$`qqXb4KF$lJg5)(TQ6M`2C=`^cJw~RFrF1cCR=?MGVJLLDN2!h|@viykE zey_Tl#@umk!RfY~Tim~3L3T-gX<0ER-M)p{^OJ04KFp`l7|AJO8WEk_Mu!qi^wi1O zx2qaPM#gV>m^v5P;F=M_*~-dtbLP!P_EFMZV;-7on_ZgEu`L;a2)r7CgX{=bYR0U2 z!M|(q_Bl4~WTdb7f&6)3{e80|UDUyqakMH+=deC6hw;xS#$Q>*>O7wrZAC@;w?;Uk zpe$P$7M;isZ+SCl{*GAPMh@sDQt)Ar(y^m)+<8@EZ*3?Pq*;@Lf30aEd8mVe7>VaAQ4<7Dwryv;PU@ufR@ju3vDfz{- z&Aw7F0(M$hFc0&BTuy8z2@#@hgt0byB9uePqI9!fu${683Q8UpBR9 znn!S^kyDBy%?VN&LXOCD)ErG^L{%6iM4S5u>j^ncMMY*0GS-n(3D(eJLH{YrK}TWm zl44HTJI9C-oBTo1Ks3{L#`rQZANkKsJB$<5s2}V&?~It&513s>vUHXwS5A*QN5+vh zqovFK(zOIpY(Y^;p=ja4f>{Mcvu`RZw&96WQ}eN)bl&~uV-fB*-9mZ+>}Pl+tuHS( zF)>QIKNmAILdhcGJYQihq2Yu|8Cv5ql3dTRh;* zYZqcOqnGV><~|JRB^G-5Wd+#Th_yX$p;%|te~Hq2*oK>QmbdrM4`3&v zr|s`1&UfA8*a_%it2fg-8YzX%2Bi4Ko96;to$%||N&3i6_y|1mi+GVB!Y9pyH+I6; zn()Y@ozk1|#!mQJ6CP=FN^im&JK<|hcx0$kdK2E*314f%Bad}TZ^9cp;cHEJWK*Z~ zCcLo|zSe|i9+K&M*=#t~)Bc-d!DH(t`YW;E&8=TSTVTPPO(^h7E%=lu7o1tO;L|Ml z8VlZI!9Qui-)_OLwcu3?ew_uMYrzjO_myLStFqvCnCW{1$DR@SHCXWM{gK~(3m%3X z{WV(f)M?~rSn!Sr68B~c{sIfW#e%=kg0~%x7ZBy_W5GKtcx>fGe=Z9?vZqI4oCS}q z$mlQLf{&aVC2@!a&vT`bUy=nM7eV4a(t^Ltf={#Ht!G*l3!Z0YBEL)v{_+SC_vse= zl@|O=3qIb0pJTyaWx}3%U8(H6Yz^LPQ#{>NDG4h!Dg3KvwD1)pX~A7{ajv*6<``0*C} z5DPxtf={yGCs^5nTJV|$Kiz`&TJSS1_(>N091DK31z%#p zXIk(JEcl;U@JlWDJ1qE>7W@8!dSAD1yKn7W_%}Wx?NX!N*zf1r~g~1wYqP$qXoarg0HpUD=hdO z7JQ`z-(bP7u;BMw@Q+yVjTZb$3*NBcbql`Pf`8P4Z?WM07QF3?cmdJ=t1Wnk1s|~B zT^9Uf7JQrq|4R!#-h%&?1+P>bcNj+z(HYBoe9b2ObzYf< zC&F`>Gt?Ny@0wQ#Gu#X{2>3<9JqXtd_*ues!s`Y66yaFHYX$r`;huzR1pFvrhL)kF z0$xrSGTA~U0)B|FgYZlNml9@Z8Ojv!T*4O;P80Af!hHxQ3HWZpeF?`4_zuDs5q1eU zgYdQ$s8PT}2{XhDH3;~6!aNEVsul3nggM0strzfRggLbc ztrc*8!kkisY6N^CVTPumr2_6jm?3GXM8IcW0L)M{G*iG|5x$ggrhtzTW=I-J6YwFz z3`Ijp0zN>PA!sOGzjU4*Y7Y!mPn!VEz}&26mz>xAP8HwyR_!VEb>4FZ0V z@YRHC1^g^whM1xC0)C2c0^zj+ew^?%glh!+DB)`fFBR}|!apTkBH)Jz4z* zBb+JVxrDDLoF?E|gcAuT3HWZpHxP~&@EwG2B6BFA;JtjLrDTYK$szCC|#I8wC6!VTPQcS^+;xn4xB9y?~!0>><2Xz>gD7 zCtM@oM+x6fc&UJw6P`f0M8FRbRtV1&a4BJxaHfFg63!r;Cg53wClXE)@ZE$p!tnyW zgRqydOTZa~ClR&@cpTx$gqzQZ_9x6xGSn#Gp@bPih8hHXJz<89p;`f7O_(8LXuW_h zBg{}Sv{u0V2{S|t)d=`P!VC>VO9kA6@HE0D0zUIczzh*XGX?w=VTOjGOaUJwd^h1V z0UsjFP%xAv-~)si0*2xRyod0;gk1vOMVKL9$R^+|gc<6Enp;Kt6P`)9QNXVd&LP|& z;1>zcB3vurX9?#LUN7LM2*!2>4OLvk5O1@N&ZU6D|?(LxkrLo+;o` z!Ucpg1w5DVT*7Gro<;Zp!byN}7e9`7iofUTKg8RVs#;%OUEcL$72xG zkGQjNR#mAOvlMA>eD|Kh>N<~wQ zGHQQWyz@#}yHZgbqv$=`UW&K5!vpG(^ea4qUV3C(&rk3ebB}w*&aJbZS615iLwOs) z(l!+W_%1{8ub)M7NP02K|2-%>!;Kt|cW#vuZ;fPhUb&R{nNO+sywRQIp6b5SJ#C7z z?D&)L;;MdL)2kEc(Dl8VUYSsWCr#f5+)6wtfmI1LOy=*ktx`PM!ZppkKtR%^lA!6Q z75@goB9gp4%>qlBa#u&Xst3VWWG|2FOsbEP;yaVJM@d6ElKiI7kcP4;{!IxiUmu*H zYYDN6z9Au#Xnj+Hf_v3L=L)_?S#kD4XLW23yz}bZ?6n2TdLgl+;)~t@C&!%r1e$mW z!@d1$z@rn=ottMQ#Ncmlk8`ubxmn@!)EKbbkC?;*t*k?04>B#l812!s68d`dH3=Vq z2;3sz{*IDtHGze(^;;HR<+;cfJhQ}8?I6&%FP^Kb$%xI0T>bs>CDLVs;V^TjY z9i8||VD!b|y4#(bV{gWvzCXpE0b^0K!M#1YXOjc<5G+0VD!~g?kxbqy-InsL5*VF8 zJD{N$KZ0{m{a1+Alqf~tqWH^)i28F_z52VF{#?RFfvQX}M3Tyc{kWrtt-%dRR|$T+v-9yz9aEA9+>PGfgv04o zC!8x@u~Wp)H9f59-+J^zMjKq3;=gFy)A2|hR`hL(|7Syt(`Xw-52n=Wo&*Q3zPJV` zU{JXRfu~3Rf;DI+=QeOw0>fYYxvW(SaNCg>9$Of&E{Ob40Ih8iPHUzuh@ z0`nNxe9JUZqJuA!%is&!Ugi&85VB{*!pf!0JwV>9|7Cun_5nkhkFK|2ZQKpSAv zC=I0(w7sB-Dt}3zKs_2Ytc<=@K}WA#g@{{{jaHN}#(?IWfoxzE{iLFQV*KHBIPB3s zbBF7dy3b-2{~Fn(dhDf&o^`1sJ@trl`6^gO=r;6U(>fvz>DB8!{+Nhibl36`L-OkH zDE`WXxOk}EKQ=M@jKZL$)EXnJ0PLkV~i;x)Z60sePJLV~K- zD}lm<8&rK?(~muV^kG6MUQo3_9ufls-#TXVT1tTt;+Up?rt14uJ(N;wjQlDb*0=e! zgm^y$toqjXQ0k5c=O@;A^smCTZvB0aK03jTLi~jZ4!8cEhPx}}+wpx1 z{b8GN2(4pyVN1cY{X7YA|2T&hq3pA;EMcUhjf^BgbZH53?(jC%Uv3M2J);4s2YiyA z`mfTL#e7bKk7sYx;PctNG{5YvnqT%YHYB^B0{jl%m({ahZ&mbI)O*awsGzYQg+bLW zX_6O3Oww5Tb;J|}P8fEf#K8mwPIQRrz%kQO@Hfs9fKrYAFetN>)4xY4rDiEj|3u?A zQWUsPxAV`zly6_BQc`xpl|D&41`44i4k9L8g1Zq9y+i(*#&B5VQ1qipK)N{8(lHiE zE@u+`6H2omF}o397g$!$>`Gu!LXvP6c?n6hi;?`3W?Trll-v!bNJx`XOVf!{pc3&X zo=-!J<)@GiJmQQvR59g5#U_!(=3K#d%+a^>V+;cfN;)6Ks&qa*M?V!$8r+lOcNz0o z7J8(=XMzQv=`yAYLS@26m@}xotkgJTgh^d5sm5>Gd1l5NmzvadlDf-Ebr@%*DCi?_ zvPgm3NAlLl^z;pvV1(7-HZ~=!1j=Y_W<`geDWT3bCDZ`79W92N=z8!jBH1gV)d!C7 z$z)H;WaKTAZ4fz)9}Bf@N-*gZfpnwnBub$({0-I?pi2^LK7>IXk`igOKp>=`!YpidkTtZN=>A(0V#wI-Ub96G|1aOaZ|v1Bkne5<(C(A(8Ni~7#KdJutOD@ zr-LTe(dYsLZ!GJeFqRO}1S@a!6MBs*l!6FN6ntrFq6DQw77{wvQ((yAO9+G%WZh9$ z0>cz<%M_H+Okrw5V3^`Xq!5~@1q4mh%i9jzsEPHsLleQi9W-%-G6sf^!@R-LLd?@a z6HklA9{3Zi&{|A~C3Mh4Z=u(yCTdN(k$u9{!~~QMSxD%p34sC9_#6Tu1zC60gupPx z7MX%FcGQHxFvVJ=5Spk21Wl}!x94!9Cf4B&O|0qDK@$flV_^8Xq7Is1o(`H=B^rC+ zrDh;e13JucZ8X^i(IJdB!7Zu@3PLu<)Z?Zmu0ZLKg@lfp5Ev#s02WBGwmk){%`A*a zW1~z#89Qo1V3^_wq<|(o2@7y7f-05*>#h3MS-IrG4z>T7as`H8(XsYS-J$kRhz1_` z;}VML^*%``#wA-n~&S#-jyhs3Y`;L@3B+7!U%qMJ7#D7u*HyZPG9l z5s=GraSjEQnhA?L?w|oKO31`>o%yWb`3dvch35y%X9u3s z@VsnMLdn0NLTBJdfTZAt6ZK0){hU>wVot@fBhr2Yfh6U>+bKfSgHcLRJ7pc^tcy~H zfr3#IGEWy_cnS(u5DQIb5p|7-9%TT}6mt~+Fqct=>sHxKcxaz?9>4k3Ckyq*i7_gG|&b`(%o|GKE)PWmf=Wno_mLx#HIlGd-o& zTktk>c=Q_*C>7QAI7RXM5;FZzRSn}LCUS>sz4j2MeS-&0&}Xc6MDQlE-voWl zYECLU;rkQM*<~lZi+D~m)AcQ$)a|8Ddi+v|=?JXDQf2PcpmX^<2&JJdb83dz4Wd&{ zhcWk9M+#d-)WFE>M4{c0q8PVG3Ze{?VOBz>aTSrMf6``lt&}~Ox&?eiwu%1O=83we z8CBI1k|qX5pECY-lot3YwKLOrisndCIGEudd`i*niay1W`jUutoXdZNu#U|fkdq8r z)PCsE_(>>Y#T?X~2xd)t6Vximk}AVHXVqTLAOd4PTNQ6JX239DC5X(&{A_hRtZIAD z69|6L`)x)uGGT=$547Qe2Kih{?kX{LFvd~Nz~x(!S_xDO90uW8h)T488At}kw4n?Q z(VQb|%G_S}DS?+nL^)N&@$Nf}$0&3~eSFg_l(8FWQ)*d;u>vJ4fu8pZ-is*R@L!8( zSvJdKq^bq5Qj)0z)`)2~<_A-ZLyQG53jx1n?fkw#IZ#K+&WO5XQ!g=)*gqp-q@sVQ zBtqrN#y`C%e%C9fLrUGT9!g#7SS4|{65gfdel) zv&ga+S(MxZivF=ucO({J-lC)of9zH5CgcS(h;O;sY2Hug|Q=&21*gc9Dtu(s|n7MV;;Rqdi1NM=`WFLjt5h1wlP zl_+#83au6l7XCAmRxqan5uF~~R+2wf0K^o)ON}8!B++mkk${+YRDk3E=VU082Kg46 zN*bhD%8Q?4(tA0BU0qC>USz80qp2`q5dEj=T{QVkd9>`|zs#}OUQee}3|zE#HYEK5 zJU~8}E661zdn9=S$=gYm+%*`wF~Q`HfowlIf;rdXIA;TEx*dwas;zVLf*uiyqPHq_p&m-b zQM=klbt(FV?!d&`l)Xn)eW8mfW2`B6hmbG^2_1?)$$^w(WlBDd@j5rB*)syMwHP%T zRDFwDcMLl|jlI2neuhQuI~>a1MpbvmMYm-jFCwE^18R|@lpl_8a=12q`k7OssS|-F&m({HWK~3G|Tz(6j zrdQt*$cRyLcY5{RTIz?TwuO8dat9$x6_{YURaq4zYzj$O0 zs}*%ays3Lj_ot`qtT-RDc#-BGxaSd^M|szy?_BoTSetF>!#yiPG2yz?AHL4^uBbyJ z9RjhgwP#)EGPfNMy->R5|Gjc0zM}f*V7n6NtK-_?UV1jR{71?K{gexS=D_8`6^AQ6 z-TB%8jJ z;aOWHZvHX1L1tqaDgK`Q@w_d~jca_&wzLWMZE2HYx20wExC^%8fiIQzE%)uPWUK1$ zs`@!E7w`M_Fej;>qpGSuHbK*mV($UPoQEesFaH`hY{em&SZR8i#^&$&!isoABxcG+ znQ|v?sxFvVGE9;g7FaX9Au~KFGNd_JtR+VqbdDl!x90eb%rRT$aCOS@lFTu{nxjnS z7$$SXb;#k%2LCUr%qn2bdf?Aw>RvK+e8<%N75~7ENUiAOhRAK6t!SA8(jy+74xh3a z0zg+2;en0YJ`x_OLXvlo{JJFHK=PH6e1jyvK=R)t`67~SlDzvKG93OW$r~j3OUPz? zc2F>XNRl5Q`7e_E9?4rI`6rUBk^DcByq@GgNOCOPAei4s@}DI6S0qqIUi9YA{J=e~ZZ%Qp~EV*ep3L3;Ve z)D@o&QmI(9N3r*Et8Rw5bWhqpNaI@jf1`X;aPy|+?&i+QW6wg4biHmPoq@Y9bY;rU zbo&nX?$-3Go%8QS?^{(758IfLU?|uhX@+M|{D0;QJ23bHCHJ6eKjv}1HX|`S?$I0K z(pKIu34ViAul>L{_<|olIyi3Sj7e}e?$n*7*zE+Lfaii3?tCK?Pst#cNK~RSI@}vw?Eo9#RMuz)UD3tLEtdO z@-0&I1DNC09UIHflVO`@FDkYhb@p))g8NTZd*%<<^!F5di|>LAeV8LdFLa=!aI^2i z4Bh8&5sp*B81QQ2mDF9Odocie1+Tz_u#&oe{y?SfXsl*`PpxQm`5dbL^JY~qXkmRQ zx@!N>n`#h$A3PH-+54FN-7)S#QAfoiQ7?}^ASH< zKubNV>K}QX6OMZAN4=>>O9!{7(^4BByvSSid3k@#Ok-0{G!0QVNfzJ%jBvEE3C8Cz zRSmup+prWCnpZ_d+NigMs(uhlIBH$fSYfD|{UD&-XHCVi!Kw~YaV?5{i%{IH8M<~W z6xSjZcPkX9-5Q6{Q>oYz&k=GjVv(s*ahXuu0V=K$ifd6T!a{KmLUEH@6x~o$+nkkj zL|*+ELXX4`#K5TrN}WpGdAm~g<>-unZ;%u60ghGmuhrd66z6BEe$s0{Okb3I&s7K8(ZO{+peHTA91T8#IyY3n}gsRVEmLJQmx z)>7Xu`>9%Y2oq^TMP>UVp1OqGGyK{m;I3Vdh6Xj@8E6y68W;f1A9zzk&dQCfMB+Os zJH>*JH}EsGz!^ItvC)$Qg%`30YF%3_7A@=uE1Jw|WGMkm%CU**n(QClL(^}Y>|fNw zq3MgE-Sf@9egu(x6z_3feW62G=eBr+BDkVe85XUI78re@7MK+FroK~lonqgwayuDY zr}jEuf2=*hgStWG#(?$k(-evj|o~xDqVTz(3=6Rs0Qx*RMNQbRjo)B7g zyrx%-ZDTAn89Y;b(Q6n-H2qCS3ElgvHr(Jzg01svgRLbEpuF5tOv>}kxM6D%hfD%X zThE7aUtPlc`7eGa;5JuJ|1_{XF>O!q#EiqikMG-I9NLeq-!sKB!>RK!12Qs0{WA~u zt31;103*KoSasRCr4XoeeRPlbcE}ZMdC<6mJkIv3YC!o$3 zt!%Gq#0~jv?m&8sUbV9Ad>A39zIq6%qX(YEU-z=l?F=3JxGRpwxGUa1hsDZt{bYL8 z=WczE=ngm|;Y|O^y*uo#J9&lvY8*0+5A?mcuC>R6z@qDW=z(}((gPF5y6akcy6cR~ z0;7{LA5ru-T_8*dXhUMg2eSI5)ty1~xkm}aKCb9h2F0su#5|$)O3W1+mBdCx$NyC& z$fNiJyx}~J|5eJ!>K4C`o!>Y#juLp2`c(q2Qb)#5VGZ?lW!FzatzA_ms9^ByeLLM1 z;d9Q%&a>hb+hf(-?ZLKbwZXQ2RheX8(c*GeFX;z8p6y@ft~lDP=6;Z}GuY-@O>+I= zhvRBvEb#&It$JrYPlI!PYcGp^yq3b=$ zR_CJg_osy2foYg5>`_v;lzP0mjI-gGGXfjhSa3$5x@|ppW&~EXQ8Z1DJ%g(iP2cyt z5{3hBDQ#-%Yppg1+Nv5T2)E3OP?TQfx?J2EZVf*joyjZ54WhnKkh)hon7ug{i@4~&kh zIJzX*G^6qz(WjbbRJV!7_iqr0N&e~};S7KE$0EVmew9JWkMnLpS;q}QRuXEd4zlhz z@E4TT1bNRe58^LuW|Dg%=%xiYwuI6s>=r2Jx$k;&cd5B8-dwE3o{xu5#?^wW zEk1Z=v8+V>RSzwGPS)?C#j<`xZIU%o2d(;sRjYsI(m|{CiKnZQ2dxp+u74(GQ1uCc z+j?W%)|mRkMR6a-1ozEo#&s6fx?vxcfrm}|O8D^5VyVb(ihM~HIVw`*RhJIh(7DK~ z`VHC?Epjg=y%)v37sHORkNU<#^}Z55oUs&Tw%(b=Yeeg@bhEu$2W^nGy9$NMqMpea zv`N;E^^`^BdgG49)Q`V0t|=z?aKGpFi6(ux-%I#|p9^6@kc4G^&@t~tkqA}|0S{Jv}=d><9 z#g>!JsO5ebX+o^d#|NQ)Q+!JkRz@o+o3I{gTw|&cwG?$jPTAxX@RQ(YMF;utDa))2 zwieYYx$kMYUwU&9XVPj~*TkKR2~M4{4cB{6N98^mHXf??t>**O zEGqzxb=37Lsl;bcGf_C^2SWPm2EAlT?;d2@1=3UsQG=fi+92u?Y#n6k1MJ=no+{c& zAsd3BqCFJ(Jwm&++)sp{xjVGnlQeK4ZtlRicVUxb;yz`oRqtcJgMqBs*NlgH6tzLv z+w7WP#K_g$Q><(-S_LGM1`brDALI<$Aa!_Zkg4it3I>@~K73E)m+1)* z3M`;chZB1<6aO4eVAvCI!TVTFMxatuaT4xqkx79y!cZp#szu+JgmmnVH6H(_&2%Cd zd6szm8#dE)-4$OfndE;-sNAdH(2DD<$N$o1=^2kKsek5y8D?$ke$g-X%!I(`o0y+? z=`$wzWmlTyH~S~j^=;AgVYo)05ko?2X{F{D6*6nD`9*CQ7dxxxW8+N;RI$Dok+0Ni zB{~~ukAAG;kkgZR46d=~%aiFE_tn=eJeak@K_hs--o5akwjvHjaR42w-kn^uo1JMF zjblHWuG;KT%rS(03S{OO$sEzHW%ZCWP}*+PbvzgCRcnOR1W#F9$8)lKJ#&}oI)YaX zdMVlmK9KUDRAIUwD!Hf814Q2n!%$mA|EhD`$bIX(Vypd=iVfzPYOHF%Sj+BdSESXfpI+%L*SvSP;AXE zbnB}gcC8vifr1aas+be^=QD9fTM`shd|EYzet}PG#&8JX({p3kq4@OD z7&&~qUM-<-yYaru7i;w6NlCX{&0ERKDeiQ z)#QhJc|)W`M7+UsE-&DIRY@eGW(~Nz#jIpppzq7FU0z)W?n+?w`ZlCNaQwP4b1gWaaA+!a0E3-07p>5oBrlX+}!#H9=+9E zQ+8lYxngH+xUQl###y-?t1M{Iq8fL2JEVTq#cI_hmz_edaS83#*NrM`jtMSwt(N61 zk>w4%>(yq~$a~kr>x1vywK|i)Iahf@eceUVVK%>TZ8{=!_Rd{ZHKeumE3XL#ryoP2 zH`fypyzBWS*_zv+98(e-uxGrw@HIU;5X=lXdY>QU^`7^L~g5&W?1oP*_{M)$Z5*4M?9g^|(j zI+USrI>P8*4cFIi9T>L_4(pn@<1xWS{a(&wy-gcU$FfHCtw6xri}n=VS@R29i1_?) zP_5V=ucf}se(kJIJjdF&L}&N-g%|J#IM*}hTmC9e^UPV7=>4pF5^$bCp(AYkejBLL zm0(`iGr~$(1rni4yeWDxx_-~*dqi24J(J)Kkg~d`Aho$x^;{2M0L+@6Ng|qbJ=e1Y zo&cB)J(~yfpJ`Cu)CIVJZBmf_d$|Cn?(71{ z4}4!0{LOMjXU7B_mLHIKRGR5nY1|`zz~l5Z!egXk-W=%c5!UXcQe-{cVnxzl#Yw+g ztPcg-hU`cW6vkr0-`2zV*dTa{i02Q#1J@9!VR3K~>v7lo&(k-s^h|02O;CcnM{l`i zELRR36-G?Oj@oc-MGG8+0$ynJmor(I^bu^Zb{|1?xBCcL&=Of7e1vFJ?_F<-MuVGh zvBY!m{t!fX3iyuaU9!bq6^Tq2Aruw7t145{WP_MqqM4cLC0HHwQh0?LB&(&LAWHZR z-c9e2L=n+FtG{Vr#pyf9b?Wq&S#jYvU^oJx0g`<%fBMUrlofd1*`2?P2`;18cmZA{ zXm@>xh^g#6I`S`Ehv;@zePgI7#v9pd3^*9KXLs_ zI*Ga*x&|=Y^bI|U8s1j#zQ^?`GaQ0h3G?afDO^IO(v*d#7)saC<}zCsizrB7x&Zju zI?>>e(3|@?I(T&9#=n|YK{Zt#nFDfkCCA=~Aqhsqg)#UXu}tO{3utgh4&>p>MvDH? zJzEirga(}C^4W>dV4N+35H~KtIv!RlLv#7U3|{qcWAPj7O0e)66oS*aXp4lL>&t_$}v9@;9HxH_A3=x!t=_w>LtENh)1dyZD_x?|+e z!9>$fb<2NzV9_Ol|78~b%pVz5Lm%Sxjp+M}?rGSEz-sUZiq4&z{5oJPP`pO;yne5kqnscDd-1GSfn#$Tm>nh zESGp&%p&FtR78mR6C|*#mvO6XDFPM!8wKx+@o=x$&e|M@Mlwc-x=FNgA&TMV1YXmD z6nG2AzQ8bErBeOFZz+fu`QnUNmalKuzcelrnK!iaG~jo>%b|=M#17h}RPVBSN^`F% zxmon@nNg(qMwAC_&tPJl>S#LJaX$^~%ZmP$^?lUiLlnQ0$1eQSFr(b%n}lf=-Yvq* zLa2s{Js#|;VJ{0M7VEm*bTYq=Ez2{KK$)-KvRp!d4M- zOvVa){SX$3?tuNTu`>HZ$S&5iO?a+=JMMQWdYiJ*dmwlEZx%nWT-E=j`HQfme^T+^ zYf}=>DyQ+r*Lek-adqLom~f3%^%gC0n>V$w>~giBUL3rmU8GY0>mHbvc+`tc57g2A zg|m|X#FFCox-2CnzQc2ib}TKN0pnI4UDQ$!IUl(bNwF~&6^W_zy?86QubTRibHyKF z6smbTxIGQdM4oQK9vZjw$uIG1vSc-IAhKYOI0E=ETM!A`bDjQ>>A@dIAls!-3eG&) zh@>)`YHw3w?sXVrkz2KYtfe+NE1#qH0Plb=Ayd3vWbA$u)MY`IgB0b5Ld%#W@vx>h z`ffIN41yiAJl&o}3BOZ*NW20@w zREmj$<#uKR6)Zb8iL!Cbiq)aNUF)zZr$5EEqGCU<~&bzS-`Mr+z{A%Jmn*A`cdka1nN%g(htOOTqK$4L#qE;AWgqNl# zBpA1mPr+7^!T&HSQZG()VK3?eqyHN$gNI5U#(p>UkJHhG9`0g8Dt$C08t{5_Lux5S z?0!G$b)>v&D@KDM!l&RRGad^bsrXCaM$Y^4mBiDQGrbktE>TX?{&5ueJj8#WFRfu5 z`6u6JgqwNC^g7s3mG0pluvZz^Z;SJ$9xpvg*BYn!k%{ji;?Van9Eu+))834<(#O=a z`=T?^aJ66bY+*@Ev=-iTl;mugh>wJ9fSr7=Q$@wmlsk zqL(9IHx6;#-zmn{IPAceACn%C@6xq9Ms_1O#A%MEeoEq3nthw5Ll0Ywf6%OikF4q7 z{}fe!pTpxw;fCp0eF~^({w19bsM6je(9`S&&E&uhA%+lty6IrLWuRaN$wic=hA5Lqc3rJLpe%?^R9x z#JOS`T#Tk069a?iLmzeoV@l#y^fO_ZTB?DUSDIIDWIy>Nh?`&K;^yTLOes$M(UKjUU& zqJPoN&5AzPrKRFr#>>#Vns_$lgj;M_DuD;1`}f%KzRj6}a~`d+n*Oe``%CN%Q|Gms zeY>y!BtPCb(FgRN^PqwWGa9gZ%E)fr{B(i5f zaX8f>Pj=|(Qgg+zQO!|pb*kP&70R27S7GoLE8h%3JGMy8X^GqM8ZN7&2IfFBbW${G^@Qrq3UkdG19Q#l z$+vFCpn6(QsBRnHGC+x^N;l)=L%CGl7l^w1CMkhZsIEC;u62t2UC|dtpf4n$^N#@Y z!X%h04?gsT_sejo;q5rY0vC7Qx%^YAEgKBs3lz(d9lE|WRu;|`md%4dN?>prt(^wz z#e4rzQ-`uOdlLeFc7W3%O)rL>?ZY0OuzM{rwD<<`8brl5LmX-O)>!^GI@)*8B1?{# zC7}@TCvODMfD3k3W`wbQy>5eWV;w}-`K|$9HvZFV zs9cK!m5ZLuQQ*$7C~Q2KB~9(5v4S2H@9aR|SqVwv!%JcyLpezZ^^7|O9pP;on26|} zM{8WhW564Cv3QhctOc=2H(tOEy%T*iGz0S)@#1{wJ?L1dQbWcx*~OtUsuH@LC_KLw zggIy!l~Rv8E03@}5W~WFvBuP~|8p%OrFK%lH^%1m&ujK3&NFy2#nSm{=T$?HOpSDY zEzwbp2lF z;B>cHbhoXd>%-mB$+GLy-Dc6QM%iaI4lJF{hD2$?shwZFyD_< z|Dxg0!005e-!mNCJxONQ?+ex$2(9a2y zQ$($)*Q)?vP0)1tvCzvTS*!zg0QdXgH0lmPrZI=M*KQquW`VS?h;@tNGFIYAG+t-~%vXpV()bjWLD&Z^bXvqhKSv%j0HKA>IV-tF zW!NB!h79+Jheb4HoSHL^12q2j7gL7wsCVe|b3nzCADp5?d=*VW%p!hH4ip{?#T^7<#hPVh~j5lSLLzBY5zgVz?vBH1Ng#c)8CqMGav+eMctaKU-X zGi;c~rKcR^5c(z!)EiK62CNllz(P0!rp4e4SS^P^uf9v&jlpCOhrw-Ldoxe5-H@c} zHz4R)1Vhyq+0Yko{tKtr3X|Y*3vD>XwkQc_%@)~k&TgxULm%GM1EsYYfgA2+QEJ`c zo*6i+p#{=$^sBDyT9q$?>f2Qub;KdI(Y-SC(KZZ+y>RZ!XM+Pi0$)BF4Yd%y9HZ(M zhO5ryM}=|noF<(2NU;_{*Zqm+AEKq6a8|y^VGW-FiARwf7^QE)0b6KTC~^rBq|{E9 zzGXIi%WU|TbC4u@zLX77)q=o6oLGdt#AxXW5l=Ax!>#z9-J2ic9Z_Pi$iK@?Fyv7(AuMa!nS+ja0 zMxNn>)1J{AhKiH4+hM3XN+I}QaYhm65qnMw3~;D{U%+XfW6*IF;gbvoHGk|V9HHWS zkfTr;4b0&>kR1H*5{sBrAy8`+K3f>54TaP zz@$X2JC8|I+gNYnH!9xDZFa2{7U#)3YFJf`ck9wk%Q2?c}Cap(8amd4j#{PBTaQk`Pr(Q7EM=*4s z=b{q6-~?Zz={I0?^_%t%Q;Kx#E$DCMAr80bgBU7+!9ifldTEB>C)Wq?1_xF~Q@?Rm zj^j9j6Y@Bjtod(I6R`&VFc4E+O5z9NTz}meobBJ@eB>G=)UecqHv}+A;#5dYt#v*U zpnucrX*PVf=&>e_C#mP>W6}MLidCA_LyP0I)YHxt4Rmh$S758A{?l1q$37RBkFJUs zFZErf-^EoOq#vlHo^q~Oiv&vIDK+(9&gxZs*0-sNI9KxJSh!33Hn-T{S9Z7XI67A( zYUfWt9h5p_EZ6dP5ny_Es&PzJQOB3{{o0cS_) zyXZgCF$U7aD+X~KxhMKPNpSI=kS4MQ>IansSYwV2${r!hWfo^nun2zLW_wiTjMuu9kfP&7$_SA z6Gzb~dlp|Q8ik$@B(#e-YR|UWs|H{bqHW$X+h(tpx`%D^p>xG^qHR7@Qx7<+ zpTx6Jb*D|r6 zp_9>k*poC`{0H2&((GSpsaWI?>;H+Lr0kKe%>*VQ#%e{36~c0v?@}iWA8%gp1)kka zSc03cAbi#AyG2-;jIig`>qRd{75G{S9@wE~xuCP0Q&C12f_z~Y^Kpc+B;k_5Er`j9H zb3jstJ^-sxS`y#C7~+t@$F2wZ#w#ty@#G3!#tB_?S_N15;E&C^BMdPH()sE^ z;SmNQrQw4Q@A&>jWWQ?HAA~>GTl~}FmK{2a@psuSu3+iYgrhdM*&^$=d|iv-CSFau z&KnOGdroZ7G33V@2wacfp*YjueH{l9ZQNiO0>Tgyj6Kgn;EEP3s;nR_w-DnaCcg(! zJnpJqDL#5?ywwi#tu*5e!2DDjlCBqzm{XlJo)?eY#W0={8BkTuivze77T?8==D(`9+`^(6*9$9#-Ci`@cKMH5YLwV5((0)v$p`05bugUKOJg zY|h8tfFd?Ou!e`H}8KM{vYj6-0cF~k=V^X-4* zRgGrvVs?@Px^ChbF}fwWktjj>8Y&_WIQ{ zzraQRbqx-|K;6gIa{pd&Zr)x|`)TyGQn8=UO>&-oip7L8^={cxDODg{B>vQ}yB4m@M}>i!3^ z*|6KNqDr*Kri6I%!53+$v5dg57Ne0e2PDDxkp&p1d;aP$mO%o1coPSh{CzxFwtl1O zHtx?}O6}%@hNTB?*p{D+G=motIgD;$gFE24 z7@uO;lrV&X1%_=gCYv%4VJrue2z8uZ`B$D0H%=f0WRe2CCK+lGhIsY1hugl5nUW$b z&zW3^FmAR0WXX_OMVhBxB%AbiOKf0mBq1UbghvCnVEtCHiPnES;h@lgM=~Hu(62mlD z0t*a(;0k0B`m!d#Y~5jegocudZfBxsUA(!b$PY+g$8Tabh#_$cx4$D=u(`)&@l9&;2}bIgq9 zDE)2@@k$kPjIidI7g4{F2DDV|x1xz5*tVYICWCHF1tN6#Javf~DGZESF+Qbov4UY- zE1H57HrWs)Zo(%m#@Gex;q~wyuRmvF2qK1@;3?1J@cZ#RQG_*t*BO||n&0+&GKCK4%u7HvHbpcEqY{WWP%q5a zmWlI7_h9}T(b}`rK9%_)m{|9-mmQ}p z#gg5Bft~o0?sKwt=+B8^il2j~v-ZgKs$}OQ^-xVpIOSU$LosGSfpou2fi*!QkMir{ zGLP|Bv>88I%xdsx2|5*PqG2_yAD8aGX7-A$=yD&!<02QS*e3E_r25n085ZDm>9(dG zDPdzLgoMYa`Z)%Iv{z%W9PaOVWL!MBqN_>=f=}@t7UJ}L8w6rmXS^+}AdC}=No@7_ zZQ}w1(C5#Tr9&GuH|Q&NEP{oyi}A3KN@upFYk6=bFkoR+yU?nz&TLu69FzfWDYf__ z!~E+a-ulr2Ju3I`pE8gohHF<2=o7}G%)n7dp4y}IgDxv~94zS$J#ZIkuZr=E@@|W@ z+rnGvX{#<4cE2eh#2UfsNH)@GT=KifwZsQj3A!-`?Aq;@R!g!mAC1A<2wHEE@=dc$9!&UgnQp$-R z`{aI>hEFB@7=3{UN5N+Y`3$Clk)Hqq646BjpYk2dM?A|}&;0B@z^%kHKI7o5oXCQJ zU_rb$@*XYZ3!)l;GXD02EQEksQ7;MsgK{i1%M_6kA{{uwRFu-dcEy}A8KN>bbDj*n zBCyXTtOZZ5ZexxizUD0)`0Xe#VhYR>>F;pldj~j#ILagVL6E}2^;Y0v#}@EW;f3M8d{PIld6A^lDI`g_NhkM zwOSwnpP@gZr5?uTUm*`ejgME8w;70pxTv8d8kotjuj|L?-dTw8Ek!U(G+tmHxIEU0 z&Qaf?hBp2DlF39gpGBJirkfnZTN$&CS6v~QPg@m#B6lYw8I~VDa2din-U7J1Q|nzVMuJ9&>J8}44f7Vt2}BP zM@D)9(SMB>P zJm6c7WQ>MrpkQ}GFyk2(4Nis;F~uuITC+c+A5T;Qre0tw&x+&-Y4L$|^5{>k$RSY* zkAG}~_zR&rBR#^q8oWc998$V)b;f-{&9aQ9S0dgZ^i8Bsp{)Is8x`6c5%MANAcbIs zcCgSY7TWMDS&R?|IwU1D*1~In$?Fjcbs>eKB2fsP2n(4Z9pkmxJsysD?h|w=Kx{NBnVCF*f7|-$@b7DAVZ()%z!bjMn^vtm7J?e?K8GiB&3_plxoOLJ~ zft!EK|BM3V6e#>S%2JLw?dL%+6zItM+?1(z;#?HhV)3tdU?B(x-g#E1avkz`{rF~! z_%zaXmESmC3mlFStM*TwuN7fE{^R+#Yx-GCkk!PGxgv>wb>vOj8MiXA#i1YY#>~UE zEcTa;=Yi)*LL8NXntn?fqpT=^J1&~2I>pJAGZji|*gt1{5A zTRD$^JRmmm%=Y{4eqQ8zIJbfu-yev;*9#cSu42FKx=#CTG_M7yh8(z$aIquLveOoQ zzom0|?ksUW0IQ6`uLmI$UbEh#Bk2JSRs2htR9n-Hi(_a)xZ#^$9PD`g@ zCeY$bp}1I?tmiT%R_fb$-f0JFggtiz8pk<1_gK$AN6$W|fU{D8_bcc_+RsNTejd8J zwwetG%k#p9oP7Vn{8z)cJe|o`7tn=qIt5(-wu?>PG#G%h@@={y{{);F!Yd(>ksBY; zIv|Ea5fTqE=CWW2HBuhM7H2UQU@SR{O#%(OlRPNQ%_e^QPP|K^1O~rD74bwq7P9T1 z(%gCE^@Q+xi?G-U9rqVQK2}_f3q%1tVaT9^F`-Cbj!!I$^8%toij5?ie*q$S$sb3z z-@`YpaPH5mw|Nu4#6bnSqGvkTzaJAHWvvL}^I6+lAO%8DBZp>c9)Jve0s(Ov9p)=! zMh|sXehB#p;q*e-cL{7?3Gjdc4@((&&@fkP;~2EELU>(buT%@=;!b!P6(j^qd<#zW zJqnDd3Y)jLii$&WG8dH>Ylp~Z^c5{9_-+%GM~3nb{$G1v10H2{t@{o04L>0aDTi7V z#7{12P@GPFNdPgB3=oZ&lGtM{$B)U(gbYq*=9!tq=v7J&T(l(wdfH=o3PpO3t)AA$ z=34Ec6_r*W>ODoI{`SXW+S)eMpW;0pFSY92ckR9Q%r{@YNdPI&bMNDRl39DNz4qE` zuf6u#Ki{8hIWd+>r$$eIg&K$KOe-yzNVMSrI1}8hOxT`3y;cYNXjN_pOCNNzD31F( zo83oo_O&h`$(BHje~talvCycp`@`YKcQnWL`klPp^-A zhb({5bIsVaOkMGAi~HS{MenBO<>~OMjhnFrhCamW^k8pT%)?xIF2F(LYt@v~{5Tj= z08@J9o}|Ie9`Dnj7~V>Qoi0DnAcTQq0$Gf!Il?f0mASO#3s5E{aZyPO5_nkZoL9df z8(A&}2%bs7*b;YCw9|Ki@Hp1|xj0~7x%E6uTT?gxA2K*xX`-js(0xRh8*;c8ZTK$c zX5@$52^7ST1Fb<9LNMCt#!hiV@_HI_xXiQ$ADW`Ou0q0`*N~iRNX}p3vtYE#gQ#mD zYQ12N82L1+0EK8UgQ*}`xfKkzE);hZ5|c&mXPQ=Ch9L^ixN=)y0(6F(n1`=exvi;u z0^R>Ww^`wuCOqXHuoidU31^vloh7A$TNhFjONJ-kNs1C$sl;u**m>T7aks4O*ehy5 z1yk{pCBFr8Te%fxxKl(#!)Z|c8!8!o`wnE1B`;D`DG?!o(cr~oaCO5UfJ*LOhSMm8 zQ4QB1v(k`b_yNV`zhIQ2uKT&Jy5S#Fo>XuPS;6#!umqgTRR72DCE!KNQ|R#6vlhFD zFZy73p%Umf;7OHZ_!+}*Q*~DgagaQ=%u-4;R#&CIBP7W+c0ZMyK%#8BVm?m4hX49M z#I%h0@$=Wt&~WaJDG&3>b$Huc8zvLKMCO)cWLYpBNiR*Mg0~nKUu?{A!p9RC zBN~rpqQO}7)<_s};H8L1&0iA7jFvH~mO4AfAdY0p@v;`V@}(`vu|{0p#`DGVxUovhF)7t+Tycfrr1e}=gtws7==9d! zm?!i)I=!W_s9|jW`PPv@vFJ^aShOpV2sb5qI%1K$7V(ojc^*qu>yr;W8 zlFl@eHzynFjD~6C%z?ogWV|rdg+^ndCl)rSgV#ke#`5MSqcf2*s2ReX$fKWPIpHEt zv7A-uViGxpIxQVA=Tv0PcLZuYh58DIv5=OuEBn5vLqdMOLZ^AoI=?u&BdcvKHfqRo zXj*S1lu4u(E;O$(JEO6fDHSPsH3c)lm5Eq5l4^_v(`kHRoac9Y2Kan-nMHo2bE5`v zYv&+o^+lF8X?gxa2}D^KdJ{0ftAf2j^K?C{xjr6kOmrvVid%96CtVJM4#ay+@Paq!>JYrOYn|Cvy3b@XY9EMJ&+~ zj5T9zR!lGE`ViTJR2N58Td}Rs#^E1ZPVIBHe|%6AKVUoH{>|C!8gxvI zE}r6A?yOze>~6sKaCLDpzKGiLb>su?zB`+J2=IQuLxB4MUj;k__#WVGxCJAFvd7A9Lst&;h;-_z+;r{%p2%66yi028;o=1HSh-+JlZQJpg*ZS%CWi?*}Z!w=DMq z&IgSB2y}q=A3=U8@PN+)&i@(O1-KY+))dGIxCXEv@HW8rUPL}%=}Xz{T(o~a;BK_P z^q1g=-PJ6>-GDE@g3TP>?!0&?o4p^f|4*prn ze%Cvf8QRR!2@~leOF@2%AegZ01tL%sV5wo7sMUQgn?-YSK}TtP<3jTT@27!H2+doD z>;I^HeA$+9Iu$K*d+TORLEE$EP5C)~=aiM^5xAfM#FRYZxIlRGh+%=4nnw_)IXd0@ z|AeUU8J!%?ZtQ|M_fo5c<~B=u903 zt9f@c=rAm8Td7Wp+wRpD0BkDRGHGCnz7o;?$vQo~`^3RRPal3phwTu?$28kY2d1FN z3M9RI8+>It9r=KzPt)H)>=_-lk9O1V)8hA#_n)cS>@}nfduK>1 z_Ld!z9JxbUH;vyip?};q_kfNnetE_?-31k$5t(SP$Ecxq7y1e6$IZZ>*kkKQ_)OK0 z_aJ}Rp3nXKB*-ZL=g3c@68&)u6lDwO7p>G?wrQM>L8q0CAJ?ntb13H3dp8csYx{X9aI>b=KbxxH+P-h1Me+x2tdQV$8cep>t<7QfGkt~w}w;Xz~ z)@NRWjdKQbE@R{58vL8|fpObO`Iy)6-vp1l6LKCwTsNs6YKWKS0O<(b`hY;N?K7Jl z?k(elUhYBp^6wSeO*;zEdcXiaNqjFOzAyyM>(|kF+Q4|>>M$C}2ciK4wz!2#p5Q(u z)H6D?fcw%Y4!=R&^fvg`aeewLn8cMmp$ej*Kz6$r(o_9qgs{<&e^`V3GUV?dRdC;& zrf+Bghchw`b-F*}%y6Qocg_eWQlGGYs5bd*#ZdnB$lrmuz6jHe zvV(;{^vj;#<`I1ktGF1*L!#547IcSocy86LqF<+Dp4wTYKC(N?zZ3a`i0dz54p;Rx z0Wq50$r(>spR~J~vhr#uGX}mx;M+hhY&-F#k$Rf*$wrs{SJ>U@cF;yw!HhypJwvRP zqqoisE6615=1I&2!?@Q$j6*^q&^Ue>_#xnvgivW4=PYN(JjQClzsa*@yj(4io?eA~C!Uh+*M9{gw8;n(q38J+ zT0@{S2*H8c*981NT;6OYoGQBvacF?XOBngXluykXt!+!txY5@+hLl>$L;JBIT+bGI z1#1fAM`(Zca|itEz#p;UGoXJJ_@{s$L|p$P%+oHTbsm)Z-y+?KcTm@FMk#MOcfg35 zv2d9-sBhA@c=|CplHO^bsC^0V_65-NTKpi_tv4d_gjh)*gltIM1G+nZg86?e{Ry+l zuQ6d8Fb&PSR{sWpF1a>Nip2+z!UFxua!ziYxmMW?=2b|iZ7S8j3Gjq|Bh_&F%%uN^ zL>vECVSnBazMFyX1O5Pwj}RtR?jOf>N@4#T*L7Mo74M(ImCJF_KeTw!-#M*fV!cxh z|9$7-Y<4YV*DoM>g;2B(MFRUHw@ybb-ZCDox%JMbal#*c1?9_s;p~rgqafuI|5kDp zzZ##@$EmI@gfP(1azswoBST}Bod?bv(A0Je$Uf}m>7G*6SEnkQ&{%^UT&KAwYOqSC zH;~592&aEPQ=Gs@*!^ZKKu$nkRzF65bvN+G6<+yOo;N6eKk|o)Q z58@<-*VDu|2D-h9?<1r0rFmsH=y!mABjWm3i5~y7{wFE&>%Q4YsRbW2lAHbg{W?~K znC?_ArN4sY+bnMb4Xw7k9NW~sXh7}jx9hSg1@o~G0?RMD?Je&T)y&L(b_P41GTzU# z55TeuQ}ecQ18%TB2tD?D^>UJs`s04=Yt|ku><_YoCxL(Jd7QH$p406mh_i09yZ*`L z+UnZvsyR4Od&l*!uAvFqpgVxXk0!#U9-QQP<3jD&B+r-UXkRMveCtweSIL7eB;K98 z&-Kebm`#aYTD!Hd46@CJ9}Qq7iPM@J@b;E z&D4(0gpk8C=OOy#S(Nz`{LXT}KZ{82IG^H=obP$@eD}NO6K3;lihpM|3guL1Yu8kl z=Wq!gLMfI%!!_-E6u)TIR+rX?$k%g?&F<=St*v=(+`dWL4!7q>iS{pU58{uxJ@-u3 zzM@n78@dOtHu%0?i}>5R=iUHu%B=6%iO+U~jlm^)pbA57PFjO!owMv3+Z-ShY~ z?S0+z!ZhuR9*Te4^S0~PQ?*wnc(zS*-#gLsqmQ}!CwY!fbw4}F13dccThp{}mdrr> zcO@RM__xVm`n}0lA^NS-`6s;EFH0*C-|O}K)~o%L?CtAiCHI!Oe^&OHO&`98RvPyGAMcx~qdpr4uG*;MNO`b5vulilx5^xReAeu1?5 z!;+FGOWeONx#)|NwXaeezcJZB^v{zCeQzoK?kfd4-w4$aAHk0_@R0^S(!fU=_(%gE zY2g1G4d8mX2*zh+lLqfAY?{h1#NztLkRec5D2R}y4G4ltPr|iqYkImZeSs}K&z7#S zrSX~g0)DG)>9w|W(w4rJ(>Tf#VTdlKBhV2Wg}w(Rq|F(HecTYNg+zFYt{EX<%O}DN zj^Kzzgd@x!N2MZ&CoB+?SGhE8@VAD*)7F=Q!39}u8lAOJ_=E)7fy-q?;ae}0X}N30 zbr5Dm+fxdvz7Mk51f>sLO2634a9XwNIHz~cllgz)G>(k^p9l?XcXnaDbafnIHbWo7 zB@9LT<@J)svF!a`Q{S18!moQw#@J5Dlh8q|TFxv;U0z$Fno+5@m9Wz;thW=2xyCX;y1+; zH^&$K4ep^`R8_e^{Bs{$>pyVT>aVI={R@=-H3YdVc4;db8y6boD^^`M zlQ`xY3L{)$R}RYjFiz8a5SuEfKV0)Kh+Jniyuq%+tP%fzA`3)8*bF*6)ZCk-nW zInh+ej_QkdCDNG!Y+u|YTy1r@0FCma6KlToE!~-52Vf>8VV8>GIirYgUA)KF(G!h@ zFOP;bUso{QrTN0Q#8FDZOiB|pcyJ(%r#TA}Ch}5|Sda>EEE&saK6(~a^JOAH{|+ zTd+GCLeT_j!H-W=EvxCk12(=;qPsg1&rs)`fiM$1XiuW%VtE%huCwueX|AmfweL`9 zA0o7;Qsvb?OTjfEZxjA$Po>JMeUyT8xqy;i(JA;Tl&3wKN~?XEf;+e-lAr#S{EXJtGG3Ke z`$7fPzEIV#%B%LTx0P4>Oa;}xl**Uj-(H^Vh?=PO!D=6^psEKXB2XKwfi|bs@+Wv7 zuHX^HmKgN{w(>`KeX8JnX=`n`zp^l?ZM2nF`+f!K^0_sr`W0>qA|$`k|6V>f zP;fIDrzIGcI7J8WZ!JIfF-f4{dz5Jj_WE}t&syF%Ac+-xmCq>@3w!x**~;tBOG*W2 z*$UW_+CDC?>`NOmB!eLdI3*MKRraG0^z;D9ugV)--WZY;_WrV$KWHoO=kk6oZ>P7X ze`+hQ&avuFvi#3cemwq_{OUZSfy*0g7)o!de$}qupscn1YkzA1qiJh@D}dH2LGD+T zSMU$Wv6ffoN`qX!PYK2xR5=CzY%5>R`>1kaj8I06vjo)+h59c>ss5o-S<|oQfWQ*$^;@z=MBR{1&7;;#H@uCS9}DqrP0f3ScwJNT zSs~t|sd=gpKR$O}ScspXsd=RkKQVVcS%{ycsd=CfU!p1hUWlKpDZgEaFU_6D7UHL9 z%FhD{-P+%2%KsGNr{~Vc z3-KS;}!j3#s5!*{OOrLp?}6J`cfEx z;BTY%FDEfnhhrR7)`&Y|Hx`&AH zgN(-r5n&PI4=|oDdx<;5i2mn{SJ$x~X8PldzmWMn!}#ehNCeHR6h1Lt)@#`C4UC^} z!*6DM9plydo%RtV&q~Jsi)61|J3-=KxAA#^@h2ERjp<)weCdl4q2!zkM?`)=T{l(h zHXrchuljkMVt9(MSm38<#(tUJ!1!f^hnzE*Kkct6`~&b#{zUP4^ifH7sRWwxCu$S1Btq$D4dc6P_#KST z*zn(F{6@x;AE$7H@n2#5R*BY>{?+wrKjW4D?|#D&7^VM9OQ>JqC%L^$t}Pe%x3czm zc8$P~oX4XAKTRtilKd5)Ugl$;-#-WZ9N5XweUeh`1HQrd9ic+}o4_mEQ@jaIne2?O zD}w(F@KoB#@lBKYTv;9oC-ACC*{AA|kxWj(8f#5~}qL7qXLul?|k6z*dD z0XA5rpLc*4@|%*5{3(TsDH5Ngg8&3Y-wwQ!-sl+`;?wY*Li%q5PjX)Wx*;-@{rnhs ziIMc`_h&`$zX#r_UKbpZ6Mj1I!XD@#03j)X)*DeO+FBhSI2Y8Zm?{-OtqzHaE zI>H;b$n+7$H!=Rewd0CSRO&{ppX;GOF2XZluK zyN(HZtj}p-k074y1FXRUgZL+TT&s32zoO(jxda zfyX|`=6~)6p5#2vep@`N1eyLU@GP7KO#V343qNn;6Bc+2AxAec{aPN^^sE+zp8}7r z84->GPx9=r>Fu0zh<=7v&3+8aIT0=fp6HLT-WnMHY2cmoww~$z|11d&roW>I{k=u- zKVd!w^HKf#uAqnf*GtBO%>R?0kaD)#>~TO^@wnMVfRsio*?;QdU`p-zh zAzsi77NLKq2!1k-0G#+&0Z;NA;B^AMBZES)2>mXm-|;y~N6(^Dn2v?36aP;FUxxF9 zI<`+qjr&G`r+WPlN{q6Twjz9PX8Pl|6w=>cgx-S#3nzL0z6idr2z~?bBxkZu)~ovQ z;Ue@e7Qr7cf}eQ4vz!+f!FLtG|4-ma&wJl6q@eu!{UY=~2Ab zo~@rP>5mLZI@Mo0fp^mL{}y=o!)20w5dMq8@(Y~l|FH;uPZ2zQi;d(TWV==J{G|x} zOvAa}`XcxZz!QIs^^=6`6b={?@3-mauZ&N!f7l3G3g=wtZ2#v2@1(Z{z*D_@uaxCg zyBdnn2aDh{0*~>vvyjg}7NP$d)9<}e(jSmOdlGn(XD$252F9O1r-=MT@MaPGh9daQ zz>9iKS)tOy_XYjPbLRa;_-G-#z1UpWorvQ-xxrLM3#BsYOiyR0FQl1fWBawM&6ei2 zcGE4fi2cQ=!Sr`*?L= z_bsZPz^W!u&GOYtuL+ncMb)Tyz7}!WU(|BCucf3H8=9thQ+Kl0yfu#On4$`8?)}p;-zaU$1Gh{k(bUcdDU1tudS&sTDrETE}!SzZj2mTms6*T9;>Yxv+l~` z{asr*W`FyOYpvElrq&h|Z+FdtvD#fzGm5p;RBPr9tJ*TP@*ShiZJBC%Q*v`_MTNP( z=5V0h zY^_r-4sC7o=gYKZlCkDcTc)DI@)6XMoAn%pqO-B97rzX-T3-Ing?;2P~O5(eo4dP#M`%YIiw zVbz82`gIypf&3U$>^q~WG~QB-1&pu4U*WIw`F$0YH4vztDsE3S$1`o|T;uD+t9&u; zPqsqPcU~DnkvXYJmCh7nS$;)5 z1~cA`Z?-2*m`5T+LzbDPYlBI=pE?|9y*kw0(6FMVd0FH2*PB&7nnv0(m44IAbfpqE zn|R?hz5roHQmI4=v^G*X^V6NW!Az*jjN>j=v%06D;nRWDZOzxNI!(E&Aa^*zB1ZzP zn4!7_t274lNFiV>=|)(GT{biwZ`hBj=9!gPG|<3UC2W-RYAbQWs)~w6`P%PByj;7Y zMh;!NmA9R4v<-atHEdf?2fqe>HC#acRq!jKy^;7z^_qKpm7~J{;VYw8smssYw5FoC zLwIRCd|eVZ{YDxiu~=GmPHu3IWli~MrW!_V9iWagqacB63bZbV3#p4H(!xQ|%sE2q zVzM|X|DdhI`W>z_6UA-OcN7hBtn5e`j1n?aR$b5BMx<;)2 zM|OQYsO%&+3_zaueWx&~Vc!&Crb`oV$ETOpcLt*|&QfMC?kzVlkq6CGM2zi*1`PS7%bLv! zUlnVoS}X}2s?7IbHCdIza3hN-=EI75BJ^P<5;GxmHKx}T)J1}vp4#&UTFVA%6DOx} znh$6NS5z#Awu~q&{4y3p@T)dW%QkfeF)epTy3>)2XcM_8ERxdp9#a}fN2<^5(D=O(d=5iUJV9=$CSOJj9z-#K9N z$W&Cx)ldRvfe#-vhuY0RTaBr7ogbtltWK>L)VxUxl13+IZ*fb)sA&OV>m3ITG5ZlM z3Y}hq9H+Ap11ax^Msw3OuxYYwS~HK357_PxKc(cq(Lo))fWKc-S%VKoVUFpBL++J3 z&BhesSd|7^7ntjV^zpLRF45zszFBlME!7Q0*EnsNRN83?Ehc}fg<+Bq7ACf-0jOy; z_N-WPz}jfjI?^K2BMqf96^RrMt`B|Mm7lz2`_)9P8h)xfwUAoN(up6r1WX9x6L^0! zxm$615bTIzc^s9~Z(DQ$)?~409BaCenxAqb`HXB$G<+H*8ZT^;)llnK<5P0jsElc> zy;1F&rPt%wcU8b_39MStzA|Tqqi<`mKcd}gm8qrEpH zQ3aSF7SIt)Fa;frSZ>1|(KLb#%p~O3n0Zcz)~RzePeBJjWW}hmvZ_a{3C-?c`lhsr zkJ^}_1U?cMj;^l11}Ln{%qC2itxIdn?$%Iiw_J+nJd$#^k{c}ocpsbKZ`L>sLQ%O* zc)6EA{Wpo-Y{4c1E3tas9VZ+|b)YJe=yZ19b+RBvIlh3WW)q0%cw7bTBCE=5#(MQy zY>@Cl9yPE}OUdf=K^PCz8dz^@HO(+;Xy>QPQyttWncJySRmw3}RKqcgPerweeO%rf zgKFf7g^)5Y&MB;U*_ckY`*F~gNJmEYS4CYi6;Vf!(jddDug

`3 z)K5DKj`A=`h{N~9O(srRX;fO;ZsM6yzEVbuxZ3nh(d5nPLSH6}kD)^h>+ax95$M

N)P8c#tC~LTMP7I;S0QzfJ077eSi$iKzHrx< z6fHX0RZ;#FTy;>gCyj%bo^Cp?!I1*?7kLxqlMcsqE$z)$M@M=i>&ei_+>=w#0{;A& zqcN*tTf~v`89Y>5DZDT|aR$2=nqUpr?kQtIS*lSu-M-zxOjEBis!?^GxiaKVC0U_&&R6y8DAZh z%jrBKUJyXyi9PhlP7~)-Z5UDx$KkdvjNfcOtGc0HUQNQq2y98Ni{FgzX5-K<-P28j zy|EF8(R5v<(og4nmenbpN?R8Nv?h(BD@lv+8KbYx^GzLXY+5fm3C48r>~MI4Drrr3 zGDn;_w>pwdOq}*Y{(@DTwBO~?NDPPcb;@pA>&^9@gllfn!~v)>;oKx6v}HYm|G!Sv B5<36@ delta 28358 zcmbt-dz?*G8~;9+;TV^5j9~_YgG0k;%$UJ&TxK+CZ_a2!rQBaCN^%)e$-xZS9fx7j z2x$`Kl`>IplDsqSAq|z3(xlryu7g66^ZPz)uRZphdVjw^e#>Xiv!CzttoyUpde+*< z-XDK&-|&LHFg~U-UE(2ACsOXACGETT?PWS^P)Txgz=FRua5;qBQRW6myqkzr=-@=U zUQ4G&JS$j=CyRJ;nHwDOZX%MK6mHWc-zzP%-_oH-x|+&>u$n2B2mP-MSpGru$mHhD zS6mxz94_p*x;Cd7n z)vGS9Nx1TGb;8wElKdc&l+6E1(!W3-!!;0BFI>}bQ5b|P30FL>`*A&o>j_*>;u?Xg z9j>vsD0INppWjJRbI<{}p2al**WI`t$JGZHg@>f3l0^O25Latl&)_mbTQ=I#fR+4a z_xk{KG+;rSf#%@42iL>6rs6U}d)!RNbuX?*1W1iq6IU*-!MGm8Wy6(#%ZZD^Or1z0 z1r>xnWvS1~es921PV`)ooxmnp?Ec0=7F%6ast2A-p#iQ|xZ2=qgzFAm6spV$_c0vN z|I`upEd|X0O%wM-QpJ6WPWwrnaTA9tQ3!~5#r>Uvb`i9jphRk$?xS(PrmUh zqU9yg@@_r0^c=T@3SQ5NLZ>hyiSBH2{Wv96WIQS&<0xSXr8#<%xu^&QuSjinL~8T7 z1t#-iS;kRgw0c)0;m*~g8g*KMd4=j!zKTWHY1{ZOp_US%7E38M`;Tb0%q4mvxiroZ zEWqp2m{8iFWW0%lZDwMOP*{L$>SLB)_KBPG0+yZ@OO!tfz*y+|4dI<189=%;-JBxJpH%PZG9J zOBAhKq>u?c3J;*tlw)SD&Ko%{5hdhP4@8B!SeimPHjWnMEEFqP1&otI29+Ty8)@qU zX9|_9@%xbQUb&AeNt48CP>;Pw?8f!W_BT?(%dO|7SZrB<|v zYrT%3dOSSH+u{PB5B^Pmm%AN2u#-zNw5ACx|lO>EeN{n)4ndp1rXX@iuJgE|Rm%1CMC z!(%7pGFQB}DAd5xn>CAT%9g|*U)u0a_DEdAn31EWPJLub-pEHskDmb5iBCVdb#~(^ zmZJY8SX$b%JN2Dj^n3?PNQKu$ng{N9+@x_>8BGo;gN}KbL%(L2S6=->iB0L={ub= z9Q`Ez4j44`uGGG$%~UYm#ymA~`rxOOiFurbESou7B^RZ%w?uD!#Z${-oqE^SHErr! zSbBnQYf0P7R<^EPEZg3$5j)l{hW*mcRog0A;1%${tY-Vxb@h7-M{TTk`__I!XbHA) zJ<`Ms$N%YZqqscEgb`wa7q6#6+7zF-6mC$G#M*hkbIa)KCQ`k%)WNN zw8W&)(4;VN)|;4nY|<-La;wx*$SJ|kH$MS8i;~t0EU(>CEGSl~MCiN9+@!`mcA}HS zauQ$WCRX5nOkKhw$}DpeUKPRMSL~~f7Yjaqe|6j~_!SX&vEbtmMCcQb{G}BUf?~nP zi#ZL|;=wKWal74|uYtd4PUDZdAl)P=HAN7MFP9!L;l`en1mz~2HoQhSVZzPBwZeq^ zy@ocZW)k!;;gu%5mkGaa!ZS^{wAal!s#ul@cbM=#I!^V62WJx8r3?7qcoUv&!rdmk zuL(~y;dh(x6cc`r3D4N;cXO6XDw_nErU;4&mrZy-6P|0r`Q8+?+9c3S z5yqJCN)tZTgkLw|kD734pPO@({$nQGVZz5(wLcVGCc)#T2=ONT2@~!%;S)@Fq6vS} zgr}JBrv%RP&oBwZ-hqoVP52a3Lb3_ZGvT==e3}U#V#24J@KO8Rdi`qxq!U>qj59@; zVSu2MO!zYEv@#LR`$~oVUcq{RxobN+CImDq7&UYu?PJA)vI}wkZrBEU9 z{xW8&RdBmw=k5#`9B$W z?AN4_oAZ~6uS>iOyg%IL3<2boIgl> z4DqF$-$gvRlc5sMZzrC7n@}<5w-Dcu_`+MrKip;`0p!etW^;wL#M7Z{XcFgF5>HM| zXcXs{5>L)dD3|ju5>Gx%D3kMZiEm1L3g@3Co_v{*oAXnMCm$x{x`q71Z5}6poR^Tq z75+th3*syPBKgC_yNR#h{6OMc5?{{we#EyTzLfKQh$kN=RKof0#CwP@=6omO+Yn#q z=RkV`+7d9E^R0-#gZN3DZ$dmdEum4Ik0HJt@wuFjCZ2qkP$uWC#FNhwO5yyUOTl*_ z-p%>T#FOt5^1C>2h5&N)LlWmt6W@{e%A3>z-w;nuO{jwN$A~9qCREP(gT!|tzLfL3 zh)*NFg!9{pCm$zN%=s?xB0p!$#W^;Zm@m-0Z#QBxPlXDXq#rdVglT#DQ z<@}4pcPBoR^K*$O-zJp8`DckIpC;ty{1oDQ()t&2ao}+R$hiqgoc|Z`y@{{9!EJ!} zOyVmzKahBGZbIdp??*g2HK9_@_aUBqnNSJmyAz*Hd@<)c5#N{A-%ue3+7oa$Da_`4 zE8_1VeiG-K5KlfzXcXsTh$r79l*{>O;`g2C_*~*EIDd?I@;O4~oIgnXK;la|zl-=m#Fub>JMrXN zgo-)8g?Rt{1Qc>$BLU<~gl2PoE%6T$KM6c24>WhnfvD#fxg~jCX`VG)&S&mIBiui` z=K@TDlC1umHF(gpf64hh$3a)ht0M<{Iv2VnZ6G>W4mP_Ia7(`0!8SjEBEjLySi9cM z<(`O}6^`8lgr;h_+leVZ?~7`VTg%X_VV~@N)Y+myl%r=cVoLWTqr4_iKN-|ZR1f*Q z5ZJo7_gt%GD~suK)V~0B)Hm&zy3j)wm3CDg^NCO0GM)|5Q2>Yz5tQ;Ez7kaX>yL0a?R@?YUS`QTKLv}N66gzsi$3)i^-Q>s zmN7X{ccJPgPzel20r$y)`x3+KI+SH~x32HMD66OB;4)r-aA74DJ4D|rmz}oDfrWad z)r4qS&5m|>H}U+Yyc8Z1BwfBZA>R%I3jNo8T zqE8*`amc}JkK3ot^tj~UFpo!3_shWvo(_uo#o4CWfpib&kO7zaf-|8U9P3F$KBr+T z$ikIp>+rIl*F-@;*aU%=;a2T+80SM#3n9(dY6EvYOUEmWGNewnSNd$Lqm z;{=Z*OFiNXAnUZNcLfuYVSxJr!kYX6VNLIvLeF9TVAvvshlAyB%odK#ngGS)0G8)VkSnb!< zgYWU=wC5m}Q6$op)oYYHPbTRvd0WmmV}`su>xsGYwTrUtE4l2?I&xXKx2ztOlRlHZ zUr&vd$M4o#X|If8P!eaG%1PhIK_9x%_C1wZFVi>L7371S+xC`!BM&d~@95~x4tyE4rh*?7bG%=vqdO^OXYEiP<{NJgW z^hET;*Ih=1N=e}W6flyhmoSpAz#T zB?nyLCHe?2@SH#@(3l*dk!w=fpsNgsREfWx$~IjkiL3ZAmxzpV-M4j0bsa}!GLG9S zB~UR{hsqW%9D->oOWhZNSD%ln=BkTm2ET{WiBu@?NbYK>^J-O_)u^fCt1Iz}{Y}YH zU5VH1wvrzcBqim7R^POrZcoAOF?;2xztUt|_ABPj0zP*tf%{SE^qS$NR8!yhz2kU8 z>C-$-uw>v$U}l^n?bzLFx`jHLpe!{zg(t%0>S`$$@qEzt6VPAX;gA9Jop;qa|5XVk zG7?@W3ib&lL^UC+qiXw5AZjCLi){K+fJg+c)^iJ&`D`cpI(Npp`l^GO!xvb_ai6U` z+qttxTwis^(i7PB0QZ{&361)ytCwzYa)Al&QJ*5rz%C(MJ=oWKWXg)H@SgDAZ0|Zg z_c-SshbM39a%$oau_&Ri%Xs<2h1b!{CJpK-UBeF*s@LCCsK=`kp&I{9gAAuigW5vZ z@S&v+Ktxll26}HWefqrt&$WBivDe_tV08yLAlTzlPT(s+ZzQSYgW8@fnUQ_JORCn&-SiV)^QzZkd9f^vwJz&$P{X^&iX-7b3z zJX|ohC#~VIFyf%MkdpBu6XMf8k;}B+J7ILZ)p(-8!7dBtl%s#j^NvmLnihsDAloYQ zYV}ilxcjLy-3~ebR%Ko^-MZ*cyc{l*^Gn^B5~qAF2h($SYQek1MiHhqmdk#z`)t1` z`8Qp8b<#=|)t~LDRInspr`DhvQ(@$?3cF(4L$xC|^rRg>lL^FLLgST#50_zep&Qaa zK~%NM>cb9Nbg1RM`V8)z8xP;s)a&FLMUHgj?|^IREI5m0G(uOOPSAWQi0GWR5gJ(B zm*=}B&3y@LIo24;JlNA(;Fi)#_eZUn@0Oy`f|{i=9EQL87-YXdJ*^Zr!F9(hgO37* z6ez$gojl*~Mgj{tCHXT_ z3$&o@ai5)leEjBov?UiQ{iqle$VfmAcJ_j$<%EioqgGH`jHcpH38;>~pwHvT%^Ivd zf^v{kgme?DrF#ZhMzj(vCotIKxNZEENZqGiF-)D!AL#aXw4upxi6TK+J*0S# zRD=+-SLX|)U-3Q8X^Of(QtD309F-L4?}pNzM{iAxMW`G%x-*?(NsdCgnS*i zKzVn`Nr!GXs+?aICtnLG>X&|{>@TZa_D64D63Q$p^XM7$Wq2ReC&&BylwYe7oOd!0 zW9fUXFBE9Lr;K$kBTXQ@NZhGu38c!IL~}~e_5ID4^#8LnDXVsP(LIyrVnls--ZY~0 znNw?c4h0Zin>Hk*GnBXX9CbvXhex}NXhRDPyknFkZSRF?M-J949|1kaBlYxG zKmhqGB=4!q-z9k-$;GTwZH|2wQ-{0_WUVj0jk= zDq5fn*S=9~UVE2%Vmda;$u*wS&~?Rox9I{7dP{QsQiGpWZZg}&rY-r^I!Pqk8f&d9nS#yJYY#lEjlluhBj%VV_MYH3}jJ?fOzy_Cu6x zJ1Pg|u;M*1wYyyQ1L~$17C0RY=+nFsh^>VTQ*#2oTFB6s>PJ^rm~_jH?%*q30AeyA zeYNv>e{~johH5KGH`6}ZPIacP2;Q!?9L(!QZ2;T2X0CKGT2fJW<=D#eV)_SS-HIA3 z-5c<^9XWp0C!x}RqSE~XgWYf^21_my#pggml~(w=%_}X|;9#t&(q(e+SuEqXsM5MO zN}0*F{Z#3Ne3kAOu-}0J-EBZ%>>a4|*lwsZZTcxWKk4pL0@$r##AD*9rCs`&MjklP zfQlL?+b|Mn6e{Y2E@L_lUBi|ZBRoXI_2y4B4EUrmXO>&i7K%AxHVPv~hc@ola5z*( z!&g@v^1NH>sEvW9&>r-B4}UY!QXeLq4Hh&nip`GyWm+m6Rtn0z*a;Pk@^hMbEv=tW;3YBcm=%s%x3 z(par)&Kt^(4sGgBF^itoMT0>UHPfM>nM{fOMMEMGbt})5*!_BpR(cFqM2t;(jKk&_ z9rYNMcs_?b{PESIT0 zlif1}bM6k50L}6oPWZIuKTuQl{OBPrP9*U*7zoLSkvv?%li+hr|LKoN4 z#b*F*-=d3~lekP5Z$UIS4F?`PT=rNIrH&+a|{;IX~uhyA$syAcPp zfj_Y9vEAeTgLB%w*PqsQB0C%t%`1NjXGkqiv*lwenzTR-w#UQ$mV+Np`r{V*?fvp6 zAny9z*2Rx(wOB9T$Fd(=5j`mvN&NX#n~eLm{_2S z+r&}UO`o$SPmQ%+khji$Dh6@C+`4Si9IJKnXDn`NvGx6I_TkjC){3KB-^lyGYQ31r zMn2u&`tV`4>FE*HKM%1wGd%tWWK1F4?>O@bRWECB*8N!zWIZ_Oew-A?<77@z|A61- zQ|owSHP7R~VT`jN4#igjS#W7^7`&WA)W1sEnI88Bhjnt|O1MLHI)QA>Tc-FeXd zCMW55PQY%(a&5~A46|Yu9%jSrk>$YI6gE`*lxyGmY(M*wP_3JYm&My>(Z6VNQY+=) z@JiZJ24_Z9%HHp%$IuP|XP2~aCGGclOJ}z5%ibH#`S2(B?%~==F0*~FcuzPNEumuN z@5ia>@aq)4HsXqSbrEfPo}6?;@m`-b0cT`5Wh*<|JBtqu#r7>$x1%@Yc_*j$!ICY9 z_va*?$_ZpARl>zcti&3cbRBDGB3uLi;G_zy(uoy0feA^PPo0pcVZT{{+2HRk$P%p!fH=fK96mO$V5xp3X`7DlacbE$^F#*427v(eGfT9NR9i zN3Z*H=9NzGtf+f^wzGNgFzVRgy`YEl|H3w0R(&?|W^6E--Eit*GAE{dJ|Ng(xZW(o z(JY@&Z>W_03{%)E=fnTPkdWbZas;FnVZTrw0>U|EfY(Lh1 z9C+sh?}F)Hx1oS!_3>cHS!tn36&;vAK`H5uFIZ3}NHk@%(je!_n zz@7%{kM{)zr@{IM$2)QY_c_km`99M2XO8*|T=d`b&`edW4}MxD_PX)Ihh~j57B96- z-cF7`f>Q}}qXXu$KkTwU>8Q_p+WEp3Ob&|oaE|)3&v{SCXAAkfp()PWQGDJr)1#G- z^#sCBUjW+xpSo90x`-*kInPVc(G9ej%m2Ajv7y<1n*KD7SJ24tYkOzulZx#(`RFY% zDW#p1=Uw$v=b!c2PV&xALT4xGt}Jza5_NtOI-j?m53PpIr&dE|!J+84jmLC+Umu1^=+9kwx<%##h11rw#iE}EPb?^&u!`jgbya@o&O zaxkGCS`XWVi&NYAY^Q-?8L5zMd-G!8Iwj)hA`xD$9S@TRcR*#ZM+zpS>W*=<-L#{w zN@8f?a2JH`lXELkmGQ#sGrGr6gdgEbfi;I zi+tU{y)M&pn!wWzIq3*aAZXLXAJ_cRTTVJEd(SvuIESY4VS5?u;o*;iv;)Xe#@3V4 zHm4!72#iFB_`gPaiuxTt)Hv?5UBY=09s<#^#w8!-t@(VnIR5AAX$%P3TPxSjVY}>8 ze~@jT%j$Q&q+LqVIm}^oaLz#qz!UQ&l_}%PDXMno9%@DHZ^iKA&7o_(pj+fQ{4o|3ru`Utj~ny_MLn07D*Mfm-$cxt{ zvNz;>p%|7TCtZ=%Gw{ED$7#!PYI{s?cgWkSqIt`o#-xvjg~$=7)zg~!#-FCLr=2|O zk+&OahK3MgE2jNIy`t{UL8DZ)kaF$3qPGo>&Cs`BD(ZRP_{-yH8|$-Ofw@n2%W8th zr(S~z>*nh$g~`m>M|mpYPpADvS@FiJN|iRWym;uu{dN!f6<+ByRE`$aR zkowAa#N)~_uj>pZdq3ho;R`X zi3dV##m3f_ovg>K;ntHMvsJU&H{A>&rn1D{j~1F{8-Ne(-dgCNVz&|DG zZe`B}Th+P-(b+e(+^y`5U?=O$t?Zj%^Hx4Xzk$eux6FJnS}M}izTLu_Eb3%+Y-Ix$ zwecL?B1*Y}-^|2sEx3Ly(tqzA{29506)kFGU9pAjT(rm)|2D00_@M=6v}@YIx7o18 zZLMqHW(6S6zP)wR;#N`C2R5+sWgWZRzlmo2y>j4rDb$FJ?5DTLinHn2=4=|O##%@l z%@m}G6QBFM?2+Ynj_>m(N|k>d);S+~@|X!I7FOb>8>me#DmZkKF*dc@xA|P+Zy*|0 zYErG_xBKb#XMT&3cv1U?(xoC%p9MR&XvuCYZ*D!gfi-=ppY`51*<&wtvBtg0UVSMc zt`@}_(*xVVU#*nk5X#`UH`uY4GCIAuF&xGdao$^5^zc=i!4V`p7(Wo<$fH2i-E`kk zdw3)3QMB@bcitcy3u|w1dQn>gN~L;2Rth&$}u^7o_+zwI~&d)|q>QQSr#+gnkC z1nYiI4K`y%JO7XCkWzjr{H^`k({!)qQ$x`*=72Au?FOdag-yBO z?*dN^X7I0rr%|Bpk^>*{Cbq1nvgzuy71d@_vLcIEx>xh5?YYifIye})fKTtc{duhr? zXM5#%Z{^`>R#=<1j#*YFSi8Q?TCD8iSuElvQbMQK>Io6kM=jV&kIS{cUT4p&?C9ws z6zNx5+G3$d3@Os%b8X}6>@XBVLY6^_JxmNK((`ofL1ewEssA8aZX2CZ@#E&%Jx6=p z)N?cGw5lAvdATWcqo{ca)B5AC0N(c7v^2xrp#IV&N2xLBhfSQ8@o>h&t zEOT{Z|Do3~T=#mDru>3T>s3j=0E}O>Ax456C=H~e~&>;L@^#S z$9ShI#w5Uv^}-zIjf=L~7qzP!+D?09ExWWj)_*Tmm6oNKc%8An@&?R+`dR!isb_DV zSZDfQS=-m?qh&1awL6Nb9Z6j`ORdYRi1EGw@eXZ(?9VPYAZ`p2L30f#(dtK1=aE!s z0vgGHxuJeV=mUfXd$PvV2AyYuY=22FLGC2P7hdW)A+G*mBL^xsq#UXzx!7Db{*5H47#&sLF! z-{k$8c8vdalEwr>E6^|AHXvI%wSd!BLbu+ZaTE(L*!C)$`f6gwuaPF~ecoidBxSyD z!45ePRX5d54wqrw0he}#=N0u^3Oo4f&{|LFTww}JF22(`I)x1{Zfd=`hCN%{+4_Dl zd$+ieKL;w?9lS+RS8AP7YK%rG6!kl2)K_#SM3U9KZuFdytVp)a(6sSLgNn9@hG5{% zjUC56I?^Yo$4aWEM$>~m>R_8^r)ZIHgHMt>t_zKH;?QQFeud_q6$mYOO>j0YYVV75cx?`l>YP0^ zXxRtrMLT=Z}jvB(UPG#e?gOm6be;Axxs&orBmF&zL3D${4%<*QsHg_3WMT=tTG2inXpqR@kru!xKz?*ki zuf5FtZ^kL7sDf*6i3*~Pxk+otJCp#w*O*DT)o($bXHVj=EUzy8+C&b_&=lKWXYun) zt?LT*{hJ=^(d8^^LknwZ5o^1_-E0jtw`~~41nR_BBW%y~X(VCSJUR)P(ew*d+CJbnixsmp06B3!=P8$0}0~H$;bkk$Y*i-**lG-1c_|!j@z$_Fh-Gk!GNyim+mP5RmavR%|F_RN3yr(6S(D9-a>7eqfa5{!&E>53#%3`KP)sGz zi}r>5VTiVrP2AYRd9gNH-nx_=S8`j~!ZFQbR)VFI9m5NLaVa~z(QRG8*oBQ*-P=G) zyWqE%z)2ZKn=)-JZm9_hFfWnYtEe{>R7=n3cL+)U`6X=pTWQu4OWB&YS|>7~d{ak# z=nIVXxcGiZ4q)4+y-`S2LA!jsoQ2-H&$@95>#=F1RawGb+4QLO=rR`dw%59@ko9@{ z0jsZ&t$e$O^%i3%-p&}k2aTY>U+{UaU=J9CyRY4c#Ln)}O@;U@@S%I?_ZWQ8;3wEL zVv>04A?1RXlwNr1B14-aN|-oQ zDs;mzV?qmc@$y?K`P#u66T+}Y|`#CC1= zS&s*qt7NG4nIL}*M*e>6(PTQia0Q0~|QC2AiAkx%wQQQDJK zM{08`*BVB<=aYFos3|hE0noz91-5ZoK4QUzhMDT3*CEnw1w<;#jZ_juUKv`m7|J4$ zO}kS3R17Du`;=ghBRIc0sC5mp&F?lHz0rs@88?&<%IDVXRKCSX*@wq+?oc07`|(Y3 zo)OE-Z>s8bm`d~@l_=PA2v%S1i5FSYmX=PfDF!VzU6_l5%}zFDL$|cDZeGm%TUu`U zsKQe8T3t(zt+CtoSlHn0t69a4;WYJnKR)lLNvadKi74wUaElxT(R zLBFiU8ax=Z13Z<(_(>mmEayNE!;QVfjk z1RV@Yo25KE9$?(u9u7}H!gW4E1kfJ4Ai&SRe4wjAhk@<@UA-?HUI(9l6X*`m9iZQW z`u2yz9pEzz104*y`Z&_51wGJppb1~0z@QyK&w)+=?LZX(^+m%UJohaCl<@HP;czVe zYv8w_9Y9zAj0`|;g7&C~j4BZUOZ`mH1n>(%zXk1aIUKHot%48K13C;e8*~EbSkO(N z>p*va?g0H3^c?6p&{%(c)cgt}fGz}G4SM*Ga5xbV0kuopk$fhs?* zPE7}`Mb+d!u26H;G*U1$;MGXkxK=C(hml`Zup{;`aD~8GLFr9@4*=P?wxRFog*)~! z*cjN+wo<)9za1{W6s&K*A0*#7zurREqL{@E?EAs)-go%ZBcIu!EG=}|+X@9PZU6?6 zdVtg`_9wgS@pM<{PnPVz@m0`3%VEl}h#&b(z1s8!O#k)7r((vi96#wJ&9vdu04{!vozNcKp9_ zj3P?Q-X9hnkse@fQ+AB94MHzchw}75z5?=0JDXMBy%x8FlHT@?$jV#4q<4Oe`85|> zgSJKX#ZmUpEGS3!wCX6*pgPw*gM^iox5 z0*L>wsLERyt^ZJ!bTlPR5EAt0wOIDW(V}^x*dL#qZT=ey?)bEN_LO!VGQCfk^-!xC z>A00YW$h0&anfxDv@;>Jud2lc9ZIdSjAQIYi0L*FVtQkk{hnHE@!{s|=%IU~sSnd( zXG4nEwHmMeN1H10UFg+u88&c+WzM0=3E>;4}UK{}%Y92s{t^UjQEmyaf05 zrmWMakJp%h1}T1-t^KrBjb*sqR?ZH7disu;m~BgrRnKq}BGGKT0XV(*&R$f5CGBg@ zE*;5tdSD^jFbR&Lga_4N`JbiMC;&F(G~51J6DQr0m|py5ch=xdn@~QaMmDCz4L@vs zw!FQic^*bX;_2#HQj07E?mo>n?rBnE6K-9nsflYGrm=IHS&wyZTZiV1h=?7i9Kf&t z5DvTPz9ZXytT~%~%-%H}um+f0HvL4!FbC>>j7AC+eHir0jp!F`?4x5XoeP1L0-FuF z{g92FJ*GH+66s|kzWoIoyX$kc`D!Fy`0L;6@*eQ(-?1~FCk*Oh>F^)R@UqrYY0aqh zt);&lQ8!yjpVnF${aF*KpwZV>kR?sg%2yNS*T6mO^Lb=SA-t-NrN(=LZXVB=JI-Q`reN^DNAP%mST%X= z2%cUvMZt(qFP1X%^d>74KS=6T9gt`lG%4^`TbcPsg&ZctgYh~}m<|s^*0bmznzyQ? z{zRZ&q>zEOqX3KJVMh$>_d|>NyWO0ld7VP0gFTDL#@kS7dZLga+E%v)NjhL*+ka?o z-QZ#0{LsAqrq;UFAK~y*bXSYDJn<@? z3))N20fLU;6i(1Y0n8F~k)Ue?eMiupf*uj{l%N*`y(MV9PND#UCJWk2&;f#uAd38@ zi2|4<=psSa2>OnoI|V%==qW)j2zpDvOV9y=jsT_nrHKNVCFmkS*9iKK zpgRRUBIqeWF9>=|(0b{j0D>k9+Dp&@f{sY%uMtK81Tag`MS`vo^c_KW3VKA)Q-V@< z#u%ad;(e_9FJlu@yLr=ac?qhn;ND-Q(5dr3DHM~!m}aTnx_RmPrz8%r?!S&r=#=5@ zgzFy^ilLxA(lpK6DJ|`v#0FkWj^03?00O3m7cX{+F(wa#V)8h4 zF*(MVV+@L^C#Etv+L+EVJFN^^+Ntayp|MvQCw&YUBn2Jq$QU?9P#g1J{j(P869X4pLjm2C} zuIngsbDY1moTohMvO79apowMd#milyGX$Ga#twotR-Vi<7Ja2lH0Dc*f?US>UFlMP zoS2!%?dGT?vn5wjjOn^WfG4c%*p=j%1^e7wX~8~S*M&m;mKf&)JAVlyn`crN~x0l(<`Y-Xv0!xSWT5O$MGIh z)37#VwtslRGRbUsE4yJpx~|CIhWGFpwqqosM{r7p)l|n_l3_O0@%mM7x37-JNQMPf z!~J-9xnWDy1n_|kqpFTKgjKoC5WlzK#pi~ZnT0mIx4cA{Uv)fQB2#N*2I=00_hA=* z!j4@_wpOMy>-A*6qs&c$s?Z*H7PwB4^~vD)->cIXxD~IUbak#ug5M0~`jnfS^M;r8 zxWG5)c!R1}ZO(vz5-RM>c3n^I=Pq*txE+?@)`Hi|mxy6)NM7am-^=wDa7xa-o^}5t z*{_LWz-#7VB+6qk*~=DzKSH?WcK3G^_GKL2A_fzYmfz8u|DAa z{KEkxXdwiS{cd8bBJ>fs@dnXHxm1!y3VcC?{#+vifgAd-8tET!^Az*~VIb@>G7thI z!uLi7)gqtCxW#ip$=GFD1&b!A_N4k3EU{q0pMgx#(pGM zq;rh)F`&+gmVhbnP{HxKQt=x)KTG5<{US2BDKcm(B3No*1BMq^3$}_F34OOwhiGh4 zs4`xG57%L-KGLCz#s60a#tVE@4b-3dvM{2gPm2fzA_5-Y@~}|g#R6X=@NWd3S*im@ z*U5 z!V5%&QGY$0;et!yzs;HsitzVYxh}_Jq?9tZNI@q}IDQHLVWB$S8h9(%O2j7?nn-5|Q6qSVONL@4Qb)Bfq!s8*u%53vOO&qmSy~JCwCW;mL35HGlz4iOJ&RL-o`3Z(5^p1CB zB<5pEA@UOpJAhMhBR|35Knsz6Go>0hT$ z)=-ZY`89?@F18%k$yr^w0sKa)ZTWqAb&QWOWEEYaw4M{<;whqe#fttL7vt^!|NJ?I zrobuH$j>pP6}?--5@&f(CnY+u7S*o_u(eLw>7<`d#_D7yCyjT)0imxyM1GWEHy10P zK}@+uewCpo);iv!!bV`MJg5S%n4|Lst`!|cftvNfkDjRcBfre>65)K3J63gf!ML-; z7ALU#z+827Jzw25K z>=CKpk)La5RCEbNOHCHy$nP}_0ZwI&{9ePmJeuVYCykHt*nj`v&{tgiK}8+QZcD2! zX(Kz09QnkPlg5l3Gj7_%Cq_;hJ?4p#$*CC`ou%YX$?09viZY#+_{7vMT~h>56=A|t zW4L;9YL_nEs%vLtWc+XKjEt_;wYy|=`QO@IB1_UWrPKe`?wXQXJ?*ZY{x4~F?NqIn zEmJf8Z?(+m(v8iGO)jd8vBa`E4O{;GLl-2P%fCDEM=ZW-bUnsWhqZ|BK;O0a@q?&_ zmP1yS-?-@q(eakAEk&E#Tk3z%v5954rRd*HEeAgMAi*-!Qq(jFM@ygH(c*(4%`K&t zqW2Rl^^0;FSsWj%a$DM3i`KWdH0AkL`-BF2r%j`x+N~{FAB=Bp8DuG%eTV7u7e$*J zSq2xiY-nltf!4+{)ml`TX!(r29^a^FQ#(s$QFMDt@1mvcu}X+4)iHm&gUz`kj*V*3 z>4VrL%MuG~5f@iiCXj_8C&H6QK@&UfI(bZBkqNAmzef-4ndF+Qq@zI%?5A+>4 KNV#`_^nU<;-l?Ae diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index 852caa00d..59d00928e 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -18,6 +18,8 @@ #include #include using namespace std; + +#define ENABLE_ASSERTIONS 1 //#define DEBUG 1 //#define DEBUG0_1 1 //#define DEBUG3 1 @@ -189,7 +191,11 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai g_load_time_initializer.m_haplotypeBasesFID = fid; } - +//JNI function to invoke compute_full_prob_avx +//readDataArray - array of JNIReadDataHolderClass objects which contain the readBases, readQuals etc +//haplotypeDataArray - array of JNIHaplotypeDataHolderClass objects which contain the haplotypeBases +//likelihoodArray - array of doubles to return results back to Java. Memory allocated by Java prior to JNI call +//maxNumThreadsToUse - Max number of threads that OpenMP can use for the HMM computation JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniComputeLikelihoods (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) @@ -206,16 +212,16 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai { jobject haplotypeObject = env->GetObjectArrayElement(haplotypeDataArray, j); jbyteArray haplotypeBases = (jbyteArray)env->GetObjectField(haplotypeObject, g_load_time_initializer.m_haplotypeBasesFID); -#ifdef DEBUG +#ifdef ENABLE_ASSERTIONS assert(haplotypeBases && ("haplotypeBases is NULL at index : "+to_string(j)+"\n").c_str()); #endif jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBases, &is_copy); -#ifdef DEBUG +#ifdef ENABLE_ASSERTIONS assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); assert(env->GetArrayLength(haplotypeBases) < MCOLS); +#endif #ifdef DEBUG0_1 cout << "JNI haplotype length "<GetArrayLength(haplotypeBases)<<"\n"; -#endif #endif haplotypeBasesArrayVector[j] = make_pair(haplotypeBases, haplotypeBasesArray); #ifdef DEBUG3 @@ -243,7 +249,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai jbyteArray overallGCP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_overallGCPFID); jbyteArray readQuals = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_readQualsFID); -#ifdef DEBUG +#ifdef ENABLE_ASSERTIONS assert(readBases && ("readBases is NULL at index : "+to_string(i)+"\n").c_str()); assert(insertionGOP && ("insertionGOP is NULL at index : "+to_string(i)+"\n").c_str()); assert(deletionGOP && ("deletionGOP is NULL at index : "+to_string(i)+"\n").c_str()); @@ -257,7 +263,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); -#ifdef DEBUG +#ifdef ENABLE_ASSERTIONS assert(readBasesArray && "readBasesArray not initialized in JNI"); assert(readQualsArray && "readQualsArray not initialized in JNI"); assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); @@ -268,10 +274,10 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai assert(readLength == env->GetArrayLength(insertionGOP)); assert(readLength == env->GetArrayLength(deletionGOP)); assert(readLength == env->GetArrayLength(overallGCP)); +#endif #ifdef DEBUG0_1 cout << "JNI read length "<GetArrayLength(likelihoodArray) == numTestCases); +#endif #pragma omp parallel for schedule (dynamic,10) private(tc_idx) num_threads(maxNumThreadsToUse) for(tc_idx=0;tc_idx readDataHolderClass, Class haplotypeDataHolderClass); + + private static boolean isJNILoglessPairHMMLibraryLoaded = false; + + public JNILoglessPairHMM() + { + super(); + if(!isJNILoglessPairHMMLibraryLoaded) + { + System.loadLibrary("JNILoglessPairHMM"); + isJNILoglessPairHMMLibraryLoaded = true; + jniGlobalInit(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class); //need to do this only once } } - - private native void jniInitialize(final int readMaxLength, final int haplotypeMaxLength); - - private native static void jniInitializeProbabilities(final double[][] transition, final byte[] insertionGOP, final byte[] deletionGOP, final byte[] overallGCP); + //Used to test parts of the compute kernel separately + private native void jniInitialize(final int readMaxLength, final int haplotypeMaxLength); + private native static void jniInitializeProbabilities(final double[][] transition, final byte[] insertionGOP, final byte[] deletionGOP, final byte[] overallGCP); private native double jniInitializePriorsAndUpdateCells( boolean doInitialization, final int paddedReadLength, final int paddedHaplotypeLength, final byte[] readBases, final byte[] haplotypeBases, final byte[] readQuals, final int hapStartIndex); - private native double jniSubComputeReadLikelihoodGivenHaplotypeLog10( final int readLength, final int haplotypeLength, final byte[] readBases, final byte[] haplotypeBases, final byte[] readQuals, @@ -101,14 +119,15 @@ public class JNILoglessPairHMM extends LoglessPairHMM { final int hapStartIndex); - + //Used only when testing parts of the compute kernel /** * {@inheritDoc} */ @Override public void initialize(final int readMaxLength, final int haplotypeMaxLength) { - super.initialize(readMaxLength, haplotypeMaxLength); + if(debug) + super.initialize(readMaxLength, haplotypeMaxLength); if(debug3) { System.out.println("Java: alloc initialized readMaxLength : "+readMaxLength+" haplotypeMaxLength : "+haplotypeMaxLength); @@ -119,97 +138,142 @@ public class JNILoglessPairHMM extends LoglessPairHMM { jniInitialize(readMaxLength, haplotypeMaxLength); } + //Real compute kernel + private native void jniComputeLikelihoods(int numReads, int numHaplotypes, + JNIReadDataHolderClass[] readDataArray, JNIHaplotypeDataHolderClass[] haplotypeDataArray, + double[] likelihoodArray, int maxNumThreadsToUse); /** * {@inheritDoc} */ @Override public PerReadAlleleLikelihoodMap computeLikelihoods(final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap) { - PerReadAlleleLikelihoodMap retValue = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + // (re)initialize the pairHMM only if necessary + final int readMaxLength = debug ? findMaxReadLength(reads) : 0; + final int haplotypeMaxLength = debug ? findMaxHaplotypeLength(alleleHaplotypeMap) : 0; if(debug) + { + if (!initialized || readMaxLength > maxReadLength || haplotypeMaxLength > maxHaplotypeLength) + { initialize(readMaxLength, haplotypeMaxLength); } + if ( ! initialized ) + throw new IllegalStateException("Must call initialize before calling jniComputeLikelihoods in debug mode"); + } + int readListSize = reads.size(); + int alleleHaplotypeMapSize = alleleHaplotypeMap.size(); + if(debug0_1) + System.out.println("Java numReads "+readListSize+" numHaplotypes "+alleleHaplotypeMapSize); + JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; + int idx = 0; + for(GATKSAMRecord read : reads) + { + readDataArray[idx] = new JNIReadDataHolderClass(); + readDataArray[idx].readBases = read.getReadBases(); + readDataArray[idx].readQuals = read.getBaseQualities(); + readDataArray[idx].insertionGOP = read.getBaseInsertionQualities(); + readDataArray[idx].deletionGOP = read.getBaseDeletionQualities(); + readDataArray[idx].overallGCP = GCPArrayMap.get(read); + + if(debug0_1) + System.out.println("Java read length "+readDataArray[idx].readBases.length); + if(debug3) + { + for(int i=0;i currEntry : alleleHaplotypeMap.entrySet()) //order is important - access in same order always + { + haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); + haplotypeDataArray[idx].haplotypeBases = currEntry.getValue().getBases(); + if(debug0_1) + System.out.println("Java haplotype length "+haplotypeDataArray[idx].haplotypeBases.length); + if(debug3) + { + for(int i=0;i currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always + { + likelihoodMap.add(read, currEntry.getKey(), likelihoodArray[idx]); + ++idx; + } + if(debug) + { + //to compare values + likelihoodMap = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + //for floating point values, no exact equality + //check whether numbers are close in terms of abs_error or relative_error + //For very large values, relative_error is relevant + //For very small values, abs_error is relevant + boolean toDump = false; + for(int i=0;i 1e-5 && relative_error > 1e-5) + { + toDump = true; + break; + } + } + //if numbers are not close, then dump out the data that produced the inconsistency + if(toDump) + { + idx = 0; + System.out.println("Dump : Java numReads "+readListSize+" numHaplotypes "+alleleHaplotypeMapSize); + for(int i=0;i maxReadLength || haplotypeMaxLength > maxHaplotypeLength) - //{ initialize(readMaxLength, haplotypeMaxLength); } - - //if ( ! initialized ) - //throw new IllegalStateException("Must call initialize before calling computeReadLikelihoodGivenHaplotypeLog10"); - //final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap(); - //for(GATKSAMRecord read : reads){ - //final byte[] readBases = read.getReadBases(); - //final byte[] readQuals = read.getBaseQualities(); - //final byte[] readInsQuals = read.getBaseInsertionQualities(); - //final byte[] readDelQuals = read.getBaseDeletionQualities(); - //final byte[] overallGCP = GCPArrayMap.get(read); - - //// peak at the next haplotype in the list (necessary to get nextHaplotypeBases, which is required for caching in the array implementation) - //byte[] currentHaplotypeBases = null; - //boolean isFirstHaplotype = true; - //Allele currentAllele = null; - //double log10l; - //for (final Allele allele : alleleHaplotypeMap.keySet()){ - //final Haplotype haplotype = alleleHaplotypeMap.get(allele); - //final byte[] nextHaplotypeBases = haplotype.getBases(); - //if (currentHaplotypeBases != null) { - //log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, - //readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, nextHaplotypeBases); - //likelihoodMap.add(read, currentAllele, log10l); - //} - //// update the current haplotype - //currentHaplotypeBases = nextHaplotypeBases; - //currentAllele = allele; - //} - //// process the final haplotype - //if (currentHaplotypeBases != null) { - - //// there is no next haplotype, so pass null for nextHaplotypeBases. - //log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, - //readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, null); - //likelihoodMap.add(read, currentAllele, log10l); - //} - //} - //return likelihoodMap; - //} - - ///** - //* {@inheritDoc} - //*/ - //@Override - //protected final double computeReadLikelihoodGivenHaplotypeLog10( final byte[] haplotypeBases, - //final byte[] readBases, - //final byte[] readQuals, - //final byte[] insertionGOP, - //final byte[] deletionGOP, - //final byte[] overallGCP, - //final boolean recacheReadValues, - //final byte[] nextHaploytpeBases) { - - //if ( haplotypeBases == null ) throw new IllegalArgumentException("haplotypeBases cannot be null"); - //if ( haplotypeBases.length > maxHaplotypeLength ) throw new IllegalArgumentException("Haplotype bases is too long, got " + haplotypeBases.length + " but max is " + maxHaplotypeLength); - //if ( readBases == null ) throw new IllegalArgumentException("readBases cannot be null"); - //if ( readBases.length > maxReadLength ) throw new IllegalArgumentException("readBases is too long, got " + readBases.length + " but max is " + maxReadLength); - //if ( readQuals.length != readBases.length ) throw new IllegalArgumentException("Read bases and read quals aren't the same size: " + readBases.length + " vs " + readQuals.length); - //if ( insertionGOP.length != readBases.length ) throw new IllegalArgumentException("Read bases and read insertion quals aren't the same size: " + readBases.length + " vs " + insertionGOP.length); - //if ( deletionGOP.length != readBases.length ) throw new IllegalArgumentException("Read bases and read deletion quals aren't the same size: " + readBases.length + " vs " + deletionGOP.length); - //if ( overallGCP.length != readBases.length ) throw new IllegalArgumentException("Read bases and overall GCP aren't the same size: " + readBases.length + " vs " + overallGCP.length); - - //paddedReadLength = readBases.length + 1; - //paddedHaplotypeLength = haplotypeBases.length + 1; - //double result = subComputeReadLikelihoodGivenHaplotypeLog10(haplotypeBases, readBases, readQuals, insertionGOP, deletionGOP, overallGCP, 0, true, 0); - //if ( ! MathUtils.goodLog10Probability(result) ) - //throw new IllegalStateException("PairHMM Log Probability cannot be greater than 0: " + String.format("haplotype: %s, read: %s, result: %f", Arrays.toString(haplotypeBases), Arrays.toString(readBases), result)); - //// Warning: Careful if using the PairHMM in parallel! (this update has to be taken care of). - //// Warning: This assumes no downstream modification of the haplotype bases (saves us from copying the array). It is okay for the haplotype caller and the Unified Genotyper. - //// KG: seems pointless is not being used anywhere - //previousHaplotypeBases = haplotypeBases; - //return result; - //} /** * {@inheritDoc} @@ -232,10 +296,11 @@ public class JNILoglessPairHMM extends LoglessPairHMM { //} //System.out.println("#### END STACK TRACE ####"); // + if(debug1) jniSubComputeReadLikelihoodGivenHaplotypeLog10(readBases.length, haplotypeBases.length, - readBases, haplotypeBases, readQuals, - insertionGOP, deletionGOP, overallGCP, - hapStartIndex); + readBases, haplotypeBases, readQuals, + insertionGOP, deletionGOP, overallGCP, + hapStartIndex); boolean doInitialization = (previousHaplotypeBases == null || previousHaplotypeBases.length != haplotypeBases.length); if (doInitialization) { @@ -310,6 +375,9 @@ public class JNILoglessPairHMM extends LoglessPairHMM { // initialize the pBaseReadLog10 matrix for all combinations of read x haplotype bases // Abusing the fact that java initializes arrays with 0.0, so no need to fill in rows and columns below 2. + if(debug3) + System.out.println("hapStartIndex "+startIndex); + for (int i = 0; i < readBases.length; i++) { final byte x = readBases[i]; final byte qual = readQuals[i]; diff --git a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java index ff883c5ae..3b4498776 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java +++ b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java @@ -57,6 +57,8 @@ public abstract class PairHMM { ORIGINAL, /* Optimized version of the PairHMM which caches per-read computations and operations in real space to avoid costly sums of log10'ed likelihoods */ LOGLESS_CACHING, + /* Optimized AVX implementation of LOGLESS_CACHING called through JNI */ + JNI_LOGLESS_CACHING, /* Logless caching PairHMM that stores computations in 1D arrays instead of matrices, and which proceeds diagonally over the (read x haplotype) intersection matrix */ ARRAY_LOGLESS } @@ -70,6 +72,9 @@ public abstract class PairHMM { protected boolean doNotUseTristateCorrection = false; protected void doNotUseTristateCorrection() { doNotUseTristateCorrection = true; } + //debug array + protected double[] mLikelihoodArray; + /** * Initialize this PairHMM, making it suitable to run against a read and haplotype with given lengths * @@ -132,6 +137,8 @@ public abstract class PairHMM { if (!initialized || readMaxLength > maxReadLength || haplotypeMaxLength > maxHaplotypeLength) { initialize(readMaxLength, haplotypeMaxLength); } final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap(); + mLikelihoodArray = new double[reads.size()*alleleHaplotypeMap.size()]; + int idx = 0; for(GATKSAMRecord read : reads){ final byte[] readBases = read.getReadBases(); final byte[] readQuals = read.getBaseQualities(); @@ -144,12 +151,16 @@ public abstract class PairHMM { boolean isFirstHaplotype = true; Allele currentAllele = null; double log10l; - for (final Allele allele : alleleHaplotypeMap.keySet()){ - final Haplotype haplotype = alleleHaplotypeMap.get(allele); + //for (final Allele allele : alleleHaplotypeMap.keySet()){ + for (Map.Entry currEntry : alleleHaplotypeMap.entrySet()){ + //final Haplotype haplotype = alleleHaplotypeMap.get(allele); + final Allele allele = currEntry.getKey(); + final Haplotype haplotype = currEntry.getValue(); final byte[] nextHaplotypeBases = haplotype.getBases(); if (currentHaplotypeBases != null) { log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, nextHaplotypeBases); + mLikelihoodArray[idx++] = log10l; likelihoodMap.add(read, currentAllele, log10l); } // update the current haplotype @@ -163,6 +174,7 @@ public abstract class PairHMM { log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, null); likelihoodMap.add(read, currentAllele, log10l); + mLikelihoodArray[idx++] = log10l; } } return likelihoodMap; From d53e2fbe66849858ebc78191e43ecbf0eb772541 Mon Sep 17 00:00:00 2001 From: Intel Repocontact Date: Thu, 16 Jan 2014 21:55:04 -0800 Subject: [PATCH 10/72] Uncommenting download option in build.xml --- build.xml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/build.xml b/build.xml index 943b082cb..732beb568 100644 --- a/build.xml +++ b/build.xml @@ -270,21 +270,21 @@ - + - - - - + + - - - - + + From 25aecb96e098664e3ca472afdc8a93657ec52317 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Sat, 18 Jan 2014 11:07:23 -0800 Subject: [PATCH 11/72] Added support for dynamic selection between AVX and un-vectorized C++, still to include SSE code from Mohammad. Debug flags turned on in this commit. --- .gitignore | 13 ++ PairHMM_JNI/Makefile | 45 +++-- PairHMM_JNI/avx_function_instantiations.cc | 12 ++ PairHMM_JNI/avx_function_prototypes.h | 19 ++ PairHMM_JNI/baseline.cc | 85 +++++++++ PairHMM_JNI/headers.h | 24 +++ PairHMM_JNI/libJNILoglessPairHMM.so | Bin 77757 -> 96945 bytes ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 125 +++++++++---- PairHMM_JNI/pairhmm-1-base.cc | 177 +++++------------- PairHMM_JNI/pairhmm-template-kernel.cc | 10 +- PairHMM_JNI/pairhmm-template-main.cc | 5 +- PairHMM_JNI/run.sh | 9 +- PairHMM_JNI/template.h | 28 +-- PairHMM_JNI/{convert_char.cc => utils.cc} | 39 ++++ PairHMM_JNI/utils.h | 13 ++ .../utils/pairhmm/JNILoglessPairHMM.java | 7 +- 16 files changed, 399 insertions(+), 212 deletions(-) create mode 100644 PairHMM_JNI/avx_function_instantiations.cc create mode 100644 PairHMM_JNI/avx_function_prototypes.h create mode 100644 PairHMM_JNI/baseline.cc create mode 100644 PairHMM_JNI/headers.h rename PairHMM_JNI/{convert_char.cc => utils.cc} (84%) create mode 100644 PairHMM_JNI/utils.h diff --git a/.gitignore b/.gitignore index 65f111587..ede62ef81 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,16 @@ dump/ lib/ out/ /atlassian-ide-plugin.xml +kg_tmp/ +maven-ant-tasks-2.1.3.jar +null-sequenceGraph.25.0.0.raw_readthreading_graph.dot +null-sequenceGraph.25.0.1.cleaned_readthreading_graph.dot +null-sequenceGraph.25.0.1.initial_seqgraph.dot +null-sequenceGraph.3.0.0.raw_readthreading_graph.dot +null-sequenceGraph.3.0.1.cleaned_readthreading_graph.dot +null-sequenceGraph.3.0.1.initial_seqgraph.dot +org/ +package-list +resources/ +velocity.log + diff --git a/PairHMM_JNI/Makefile b/PairHMM_JNI/Makefile index 70bf96dd3..62dc3b0f6 100644 --- a/PairHMM_JNI/Makefile +++ b/PairHMM_JNI/Makefile @@ -7,40 +7,59 @@ OMPCFLAGS=-fopenmp JAVA_ROOT=/opt/jdk1.7.0_25/ JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JAVA_ROOT}/include -I${JAVA_ROOT}/include/linux -CFLAGS=-O3 -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas -xAVX - -CXXFLAGS=$(CFLAGS) +COMMON_COMPILATION_FLAGS=$(JNI_COMPILATION_FLAGS) -O3 -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas CC=icc CXX=icc LDFLAGS=-lm $(OMPLDFLAGS) -#BIN:=pairhmm-1-base #pairhmm-2-omp pairhmm-3-hybrid-float-double pairhmm-4-hybrid-diagonal pairhmm-5-hybrid-diagonal-homogeneus pairhmm-6-onlythreediags pairhmm-7-presse pairhmm-8-sse #pairhmm-dev -BIN:=libJNILoglessPairHMM.so pairhmm-template-main checker +BIN=libJNILoglessPairHMM.so pairhmm-template-main checker +#BIN=checker -#SOURCES=pairhmm-1-base.cc input.cc -LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc convert_char.cc -SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc -LIBOBJECTS=$(LIBSOURCES:.cc=.o) DEPDIR=.deps DF=$(DEPDIR)/$(*).d +#Common across libJNI and sandbox +COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc +#Part of libJNI +LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc $(COMMON_SOURCES) +SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc +LIBOBJECTS=$(LIBSOURCES:.cc=.o) +COMMON_OBJECTS=$(COMMON_SOURCES:.cc=.o) + + +#No vectorization for these files +NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc +#Use -xAVX for these files +AVX_SOURCES=avx_function_instantiations.cc +#Use -xSSE4.2 for these files +SSE_SOURCES= + +NO_VECTOR_OBJECTS=$(NO_VECTOR_SOURCES:.cc=.o) +AVX_OBJECTS=$(AVX_SOURCES:.cc=.o) +SSE_OBJECTS=$(SSE_SOURCES:.cc=.o) +$(NO_VECTOR_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) +$(AVX_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xAVX +$(SSE_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xSSE4.2 +OBJECTS=$(NO_VECTOR_OBJECTS) $(AVX_OBJECTS) $(SSE_OBJECTS) + all: $(BIN) -include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) -checker: pairhmm-1-base.o convert_char.o +checker: pairhmm-1-base.o $(COMMON_OBJECTS) $(CXX) $(OMPCFLAGS) -o $@ $^ $(LDFLAGS) -pairhmm-template-main: pairhmm-template-main.o convert_char.o +pairhmm-template-main: pairhmm-template-main.o $(COMMON_OBJECTS) $(CXX) $(OMPCFLAGS) -o $@ $^ $(LDFLAGS) libJNILoglessPairHMM.so: $(LIBOBJECTS) $(CXX) $(OMPCFLAGS) -shared -o $@ $(LIBOBJECTS) -%.o: %.cc + +$(OBJECTS): %.o: %.cc @mkdir -p $(DEPDIR) - $(COMPILE.cpp) -MMD -MF $(DF) $(JNI_COMPILATION_FLAGS) $(CXXFLAGS) $(OUTPUT_OPTION) $< + $(CXX) -c -MMD -MF $(DF) $(CXXFLAGS) $(OUTPUT_OPTION) $< clean: diff --git a/PairHMM_JNI/avx_function_instantiations.cc b/PairHMM_JNI/avx_function_instantiations.cc new file mode 100644 index 000000000..c29370561 --- /dev/null +++ b/PairHMM_JNI/avx_function_instantiations.cc @@ -0,0 +1,12 @@ +#include "template.h" + +#include "define-float.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +#include "define-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +template double compute_full_prob_avxd(testcase* tc, double* nextlog); +template float compute_full_prob_avxs(testcase* tc, float* nextlog); diff --git a/PairHMM_JNI/avx_function_prototypes.h b/PairHMM_JNI/avx_function_prototypes.h new file mode 100644 index 000000000..1836a1c37 --- /dev/null +++ b/PairHMM_JNI/avx_function_prototypes.h @@ -0,0 +1,19 @@ +void GEN_INTRINSIC(_vector_shift, PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); +void GEN_INTRINSIC(_vector_shift_last, PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); +void GEN_INTRINSIC(precompute_masks_, PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); +void GEN_INTRINSIC(init_masks_for_row_, PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess); +void GEN_INTRINSIC(update_masks_for_cols_, PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt); +void GEN_INTRINSIC(computeDistVec, PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen); +template void GEN_INTRINSIC(initializeVectors, PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D); +template void GEN_INTRINSIC(stripINITIALIZATION, PRECISION)( + int stripIdx, Context ctx, testcase *tc, _256_TYPE &pGAPM, _256_TYPE &pMM, _256_TYPE &pMX, _256_TYPE &pXX, _256_TYPE &pMY, _256_TYPE &pYY, + _256_TYPE &rs, UNION_TYPE &rsN, _256_TYPE &distm, _256_TYPE &_1_distm, _256_TYPE *distm1D, _256_TYPE N_packed256, _256_TYPE *p_MM , _256_TYPE *p_GAPM , + _256_TYPE *p_MX, _256_TYPE *p_XX , _256_TYPE *p_MY, _256_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM); +_256_TYPE GEN_INTRINSIC(computeDISTM, PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, + _256_TYPE distm, _256_TYPE _1_distm); +void GEN_INTRINSIC(computeMXY, PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, + UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, + _256_TYPE pMM, _256_TYPE pGAPM, _256_TYPE pMX, _256_TYPE pXX, _256_TYPE pMY, _256_TYPE pYY, _256_TYPE distmSel); +template NUMBER GEN_INTRINSIC(compute_full_prob_avx, PRECISION) (testcase *tc, NUMBER *before_last_log = NULL); + diff --git a/PairHMM_JNI/baseline.cc b/PairHMM_JNI/baseline.cc new file mode 100644 index 000000000..b953c4436 --- /dev/null +++ b/PairHMM_JNI/baseline.cc @@ -0,0 +1,85 @@ +#include "headers.h" +#include "template.h" + +template +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) +{ + int r, c; + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + + Context ctx; + + NUMBER M[MROWS][MCOLS]; + NUMBER X[MROWS][MCOLS]; + NUMBER Y[MROWS][MCOLS]; + NUMBER p[MROWS][6]; + + p[0][MM] = ctx._(0.0); + p[0][GapM] = ctx._(0.0); + p[0][MX] = ctx._(0.0); + p[0][XX] = ctx._(0.0); + p[0][MY] = ctx._(0.0); + p[0][YY] = ctx._(0.0); + for (r = 1; r < ROWS; r++) + { + int _i = tc->i[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; + p[r][MX] = ctx.ph2pr[_i]; + p[r][XX] = ctx.ph2pr[_c]; + p[r][MY] = ctx.ph2pr[_d]; + p[r][YY] = ctx.ph2pr[_c]; + //p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + //p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + } + + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); + } + + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } + + NUMBER result = ctx._(0.0); + + for (r = 1; r < ROWS; r++) + for (c = 1; c < COLS; c++) + { + char _rs = tc->rs[r-1]; + char _hap = tc->hap[c-1]; + int _q = tc->q[r-1] & 127; + NUMBER distm = ctx.ph2pr[_q]; + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; + else + distm = distm/3; + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; + } + + for (c = 0; c < COLS; c++) + { + result += M[ROWS-1][c] + X[ROWS-1][c]; + } + + if (before_last_log != NULL) + *before_last_log = result; + + return result; + //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; +} + +template double compute_full_prob(testcase* tc, double* nextbuf); +template float compute_full_prob(testcase* tc, float* nextbuf); + diff --git a/PairHMM_JNI/headers.h b/PairHMM_JNI/headers.h new file mode 100644 index 000000000..42485fc51 --- /dev/null +++ b/PairHMM_JNI/headers.h @@ -0,0 +1,24 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + diff --git a/PairHMM_JNI/libJNILoglessPairHMM.so b/PairHMM_JNI/libJNILoglessPairHMM.so index 435034f78de8b6b51001484dd15d775e3fdae892..9b6b8e6930e9764b187f019979a525dd50840252 100755 GIT binary patch literal 96945 zcmb@v3t$x0^*_F014acmSfEj{tnF$eQA`Mt$ATu1$V`}p6oOKXE+GjL$jfBILlJ=_ z$aGmtEn2nIqF?&0_3>@0*e_CQ6G(We5I_{cN5I!CQHfX}fRO*^bMMS%5|g(6zrWGR z%sJ=YbI(2Z+;h);%<@o%c3ZEgD2H`99k)0L&73GPsV2cl>`0U980F~WNOk%IauNOe(TWIE6Xy&t?ZYIEG zJ!2&3h{lumYAMclo*pxu^<-LJS)T^-#d|L^W0z-W+a-)SXpV#3z)O^3b3+;9r zJO=O6ao>sCjhilteb$HDx-K_gCRqqPC)nQk`g7br!<~-XbR&*|c<=5i1yaSm5O)^t zOx*dni*cvoz6*CU?ygrE5KD3Y*hUzQ_qjIMdcDPb`30Uq+|?%hD?B~8vvEI!dopfb zMYwOmJrnn8+{1C(uV(?Qu_3&+_wHVu=<#K@cBWi_L;A7fQb#J8;i9g2{SXaHeCz|J(jA|87-V<&!oKwJS2&j5bd6xpF4+kG z&-$h+5)n&hge%X|!&V8$M$6}Amy!Raj&*ZPynHyeK`y~RQSwLi#Wlhy+ojnwyjH-x zTEL&}wbYPst$a92(O#n6%Ot{T$3PEydjK7W>DR6AoPIsZp*#+=zNWYisM+C`Njv?e zpXm-?hI)6W&ol_u9sa6Ir94g5lD%0CM~T^vWk)1F)^v*Zh;~G#MkbcGVsK0@xwjrL0T4 z427d7zAhjOtTW5)eX+#1R!D@^?)y<+rq41JY2km_gZ=-ehyM6{5BS@Kotr|~9#_M! zbXTuOd(gv&u*2@s*P89yx>_=?GyB)(9`trC@T~9Zn$CLo%IrsVg%aOn%KDV4XLlDp zPlEsKPM*8W{<73e?>6~d=s^#E0h8|dtnWeo8%+MOUF7+whx9wl{*v0IzFW{==x4HK zO98XYe(_MW)K9JHPb_;FZQ`5Ej-G1P>x`)%M^|~^2)e7+f*$(I3q9CdiRlk(&2lYy zc7wUF+h?U9wJ48Qs%a0+Aql(X!%^FVT`|V`v6;sH*kqPl*QMOd9{l7Pv)ouSpv*!) z=k-_7?(m;mB%Er>IRoioLf^T{%>LWdMgOm(zWovLzBud}kq6Bot`t-MiKhNjP5k3M z=<@~WpMHB~mvQ7bkc;|s2PFfy$>*0n)b|wV-Lc0?KlhiXTMwhj7(?MHwpGB^ke$%b&~8@)tw=a z)Q8tX2-%&S=X;R=AouOSDWQpcJgr#e10SRn3cQZdC=7V>Mr_k2gB~@uk+}zp00MEfaQPCbB(>!Mr>E8D{*G7?z2f(B8a$--F-V z)kD4R>Y-n3HQVbs({D)bSSt~BXh9v%znlGiW|_pTHsjIz4b!5-Qi2kc%ZdJ^0nmIDaKj!yDsC)B-FP% zK8t!NcW4joYnXa#GVQQT7Vj8u>djgdvE=y}{(=3pX}wIi)J$L7L;s?AbSLK}NKZbF z6Ouub$%oTM;u95#u*yy7!Tw`=@H4*?=`rtH-I>oLJ?P=I>EGNtWcoV!a0H=G%9B+i z5veBrsvi8(xE|VNLl1WT&mQ{Ao*vR~Htl3x7e7NiUEa;SJG}>a`t^W!`2F`5mzMY| z0=eY@zu)06EGY~){Il@@DQ{seUvrBJ?=NupGw1uK7TjA{5hy61P?TFyQBdLV-NXDNkyPAP#Guy-@=l6{gr{jq6&XmZejVH;$na1WK}D@ zx2T|^A`2MB=kw1kDV$b0YeH#p8S+dm$j#RZ=M@wc&M7U;pIA7*phU?nD=G~vDl0&~ zq(qcDe0ZKr?4Mm(ROBx!FP-Jjoxd<&&Ckg8XN?LJR0Q&JD+;ovCHfsj`2`EhkZ1C= zTs1FaLQ=+j0aV~EYf|3SNdhfUfuvEhaw`h+Aayy!#@~Y6Vl{7CASo$tPHwqBP@Y>D zr~ng@g{(kI_N%28{wZYzCB>!r&=SfaYgtHOv}9OVifrHIlaZ2FRFGRvHp2xQunOb` z$OWjxioC*tl0f0?!n~=|0;9mg4{oxE(xQrttlR8pR5imtzufm ztXWb^CRw(N1g4iPC@jf`nN?I4qoRIV8SHX)Zefw*UjKL7JioMZR#8E>q}fHKxq<(? zU7AMOVURR>XyWGi1$lwea&=l@1npYQN=ow2owX=XkXBxvyC^G{HUjNsWZYa{P=W4~ zQ4EM#N5cFwO6L2ACrvB}Oqn$oY($ofqJmQcsp-fuI#6m_w>mdtT17TRnieQ4>drc)iNFek zNze$gCQVEBqsQF*{j!N1A}?j0G_4{=ylF_YAsS~1ts?e=5(YJwHjTVJFh@^oo-skY&O%{g_*e z*>rS((NhcH{otklU8B$ESR|UJbeojX z@I*gVTe`rHqR_|u1?AftC9J*&8Mz8p|F0^1A!PLA&*`FAsaN7BR| zB>fLPMGCqmyXZ(f(^;4Q-yJ+@1e7?t5K$DOHVn2IlSd;aEU85Bw=ij>-(ON%UJQR$ zh^UWkrDnk`OdEw_i>-mc-XZ?oXqY>VhD}Exg3DQmCri*x;caCN&*esY`-YOPh4eBY4d=`ARuqUC> zNw(s?M^{HBeEz&S1$pz>X)=m??ky32`mgk6Z|m=4g*M^;WA@*%(~&*e>A#74OshmQ z<&{=q__}j)QqqLdlKC*P2?(Z>Qt|{)QCM1%or@sOj4&f%?g$k#k&ZyRGhTX1VL}L( z7jnvSXVT4u6`Tb4D=N#%O3MQU`OqEk#DIL=w~DDP4Uat>S2aaq0av9 z3c;=fJRVD}sl^D*Bo zcEHnOOmtLKgeEB4gO{F z5v7QP7zm+u-Augyvoc4b!33X(f9Pgvj{4NnkSp^d&f6<(=gLzbR!xEA%G=i4{JEd z1*NmiRw*2bo?KK|f>1OMvkQdJH&@InEL%{4t`ix+gp^c6H|sZ03p6JNliYG}kuprH zK-grxN1D>CUqmYrLZk)1ch!;8th+V$knx(Fh*0e|qxg4`8oUDouw--c*^`-luZYI7 zO?3V)k$;5<{QrGxjo?~Rq!V?H`7Mk29&;2cOILaFx}S$wZ5D~xyJ*E?7m=&5xU5LF zu)kt{F;+TD7mR=bC5E7= zaxgp*!x|PBL|#mm{hZ?2d1U2f6&JFj4o|KWL%1z%9wv)eCYd_ghwPX&`O~rDAc|xz zmTL(+dsO$*R(wZV&ON&o%AOSL=l)BfDr`dCW(qZzxyXL*mAT3B9kulZVx=dhbaN~}(&SZM?Kc?k&-(!F_D{34Vj z5|#vt@(866DwC^5gcAu1ieQ+};)X%W0z^}}Q1Gk@L>*kT2dX6BF>%5Ke=;&n)YS14 z{KJP02gBPje5q3=J0{GSkv3lSCk;!M#mNk*N<#}ADE){!Y4u=~x$+~V4 zHof{dc9{Ezr2p1Nue`DlJ8#jBCL2B5MlV_NN9;&B9q&izy&N-5I@mk*|9KvEc6vKL zF>!(GAH@z?FUN5+eWH2J9!PN|7RcwQF8G#@mA2R}@YuSK zT+i9?)?O=#wKhDqIwRLc8=mLrt*g$4x6b~Pw!?u1+_D#>I2*pgN{px5 zhF@yKuQS^%3b?0i_*64JwujMsJ1)hBM@NrbSvEY|-nwSk@Z@D(Gi`XwhDe)Z!^06p zt}+`Q4lHslwBd17A#yFX;c>Jra;>!Cu|*%bR@?CQ^8!!U@Yte{TFv58Kt$u_)V!8Nu>umVjZ20GF_=z@rtqrf(@EdLT zOdGz=hSzNP9X7n(4>#HHQ*7xE+VEL6e6tPzGaKHp;iuW~tv3908@|nkzuks+T-#y) z{65FJVr+PR_hVhLHhhkS#52x@zuSg)+wk|;@bNah--b`L;b+?LqipzG8$Q*BpJl@< zHhi89pJl`6+we1Ncr(-!)R{K?Y+L#{HvGLde3=bD$A({M!x!4{OKo`bsFI+rwBhI3 z(yzARi){EOZ1`duew__(AAg^-;Y)4lYi;;48-Ak=f1eFsXTt|<_#HNUr48R?!_T+j z58ChxZ1`pyp5KC5mtn&%vXFST+VBt9@NG8yVjJFZU5EWYXv4?Y@Jnp?SR4K!8$Qm4 zUuMI*ZTRIje7p@`Wy2@h@HIC4C>wr-4WDYmue9M68(z2JvuyaF4L`$%UuDD3wBbWG z{2Uwp5gWeDhX17vztDzXZNo3M;eTbrue9MGwc%IW@V~auh+#hJVI}Z?)l{ zwc*=r`2Vuu9p*fe@y~i2KE{UsgAE^R!~fBSkF(*Qv*DGhFJg?(!QEIMgFg?xf={oj z8e+D#r#$jeA@!cM}q0$xry zn(#US-%q$V;ne~zC(MvPyi~w*3A5VaG6Byb3>h8anF79(Fhl!rmVl=c#+rg7oGRc< z!VK}li2@!^n4x{xE#NVP`w@;6@NmKm@xu-Q4<+m(+*a^t$=$G<`gu%PQYi60_Id8yjs8~2y+S$UMk>Ygc@3A+WniSX5gV+H&&VNMak4go(;m{Wsr z>v`7yS;B6@%>sUkFr9O_Nx+X1{xRV?0sn&VPYBlvcsXGY;dKJOpYXMWR|~kD@O6Zj z3V1HzpAs$;@GQddgl7u)PQup{&Jyrc!kh|(Qw5w!m{WjoqJYN}zJah?z+(vCNH|u& z!wGY05OxT7DB+t3w|*NX*KTp_Cxb=)^f5I~fHw*YF z!nuT-1pFxBS%m8Z{0qW)glh%7oNzwjbppPhZ~@`f0xl;!oA6Qr&n0{>;W7cwBFq6O zJX64T5-ud1CE%%q=Mqj8a3k2yi9g)dD_2m?3v~seq3W zUO>1^z()x$Bs^2VhX^ksoF(9WgzqPuD&Sp&A0V6v7*BC=yjOy~ryq7Zl$yFgul6P} z+^@E-=}X6r`rM5mK>su?8{aM{RpS(-bp-k

wBpMTwCQOD1 zQ>>*WOXrnlmN;`9zO1FP6V=g9zG{M?^EE*SDYnCG_HpBt_(S$#9t)(mL1tzy(kJxQ zd{lJ^DnFLKJ`h-v_XTLDWErk8NvXoc&yfU|uey#Lm^-Gq{t`?w`p@16YsVs45&27( zSqdvktn_^!`Z$p@zXadnVc#X9)BF>&EE9_C#zK2(zLQ)Vn=>&vbt39%!US@S>2qC$ z6>z@G%1aknz^fp$z&y|5ukUP_6koxIiQ~vD3>SxL$6i!o3nL0Tko0@8_q{L?t$Eop zOBr_KmzLm5LeAxurSKnTXWco&lx4Yd)~p<)FwZn2%S>Z{0!OK3aehewHq?s&fpE^# zbLX1oWX?qP`s;|VLP$%(7eJ`Ygywgt2A(tpHVfaVu*@s7pq~OPNP~sqIPSi~*Iu)&*-MIwSem7Fhq)k#0q>LPMUQ-^b+M%c?>1OcD$>$^ zXudlq^Uk?nGvmp~c!5pi*=Ej0-&oAE4Wkz#7jttQGtth1WV>%}(v+;y{DOH!#kNe} z7pBUo!J*nwu=tKfkt{sR1o#=G*K?UEpgQi)E@>ug6TaRx6Q7bunn;FR{HLoSJ@h$@=lHWZaQK7(aYx1Zo5WDZf-!PCDe9aiB5(?Mp3p+a4%#z=vR;U&FU< zkc~TX9242VvA;r<SVJFd&{x6mie!T8mNp(|RvxfqQEi6>3&S2ZEbGZ7}U zu3dV90p>f^B0hKms=%%pw*~g>{%s`30WSexxLXRBHmJ}o_#_<~kEJlb$SxkqR>$6k zvcMV3x8U1-WVq7OV9!i5GA&7o$w-TeF@-tGA(Hf0bqeWfNK??SzJ2EWVrKSh)rDV? zCox6tmvleny{HI&cZ0bq{UWpO2+k`yh@eu%XL4+I6gM2LVq3Ap=2W1+fhcPk&0Ddz z>dH!gT!`_M|AsmeZdc&}JA$i{^Kqa;lTl-3#Gf(ymYhTgLb?DOv#iJ&l&er(^QCNy z=(76_*;pRQqYtzMe$D9D4>_nC%5mBq7~ z@i+5+Iw9Rf=3Dq`VFDA7eAAhsI-~$2$26b1wxF5{QCcFYq~P3Hzh34-_Q(Co)ZdaW zR#y|@w~|Gt!BIc?fGz&!y_x^3qg1w&#b;qBOvI<=(7hGIyf5V+ft^{7Fa{{*?8%lT z?2~rci&^bo{)7Xo#o!{5)ezRp%W;-FLXD>QSS3K)(B&{~V^bYw6w}?9_khm?&7+Br z{u<%vuPBA(He26@_Kl}>{eA<_zw`uT#xHkBZk59t*0rVLvtcklDHAM?in6lOa!k>a zTPF>Z4R~ic`a)q!(8I6~jQ+ot$7~FM@)s3h=uyNw{+y!8@G)udV$Qb84$o%+zLgOz z%5_3=1@pxMTzD%Sb&I3u?kRKN)x)q^X2Anuz92dCOB8$!t*E4LIRv(ZDR<-RdKh+H zvNbF%Qs$WEV`XKo*^*_RJ9}P+PtPwtL4vV3&3hzU6o+}U>OarIr1Xn(8BRSPsj?ni zOtM|H`Z38{>ag{%uMp$Jf`Ab|A14Y0Op+S$)oQkXKLSUv7~h%f+oZ+%8s;l37y*bc zJ!2J(KCEqnY4pz)gqa}-GB~cU;sF32?{xG6!!eU&+>`&?iI5$Y08`1kF&OItNL~RwIEraWhiV5K5|QQZkQs+Xvwe9Vm>LPXHIh!cNYa4uTZ1*{u89E ztOBP2gV{jEzfws~7T;g(uPxvp%|VzBLaZwxr3uWs!@}d+y|VR7TY}p`2%>iM<@Oy%0+(L3Lg7i(;`MOHe09&-!#?{|}kU BX;J_H delta 32353 zcmb7t3tUvy_WwDfu(fmbsSF@>2zu#1q4A?pUY}z-OnLUdW(EOxFsutf9bd9KVjhVLajl^U!W>+WvzR1}6jUdVA(E>zpwHkc#x-4Fqd~oK^}*Et^bTw!Zu7`0g z!8H-rGF%j9#B(ZTfQ^EUpLCjCS&1#cYn0!cWn@{3o?ClF}})`plQwM9GzR|Kv|T&-|XXphT>#dx3gMLa?9I_arL1dSCm4uy{h zPXsquAYzzE>5#NlE|`N`41(k?T)l9al-1@4B~LcTj0e{j*L}G9;TnPKQCumw22zze zauDl;t2Hh?bVFo}&Ip<;=#!ueaLpF{)1c9~s6sLXHWri`HXGMsf$5|KkpjUaNs^kI zD&KWqbE(dw>>bg(?IeGxZX4GP7D>MK=6IPHb2?t29B&>Wzpwn_7vbIA3k^kui?KH2 z+T|YFH_(nuLmti2eW4uiiIA5l@AyVIywFM^7(wM%EG$pA$k_%O3XdwN9uX3e3k6dP zj#pE?BIq9V#5_YKD5o|+s2BQAi4x?|fJV^uKP8_eNy(K%IvvL#rL^fgn(Dj`o8;SQ z;aj5ei*HYPu96WNA(to_%_Fi>{CUbc>c$9poJem3aWAu<=Rlo_(>2JRN+oULd=4kl zE~-g6kJZj5im&nW4Oy^DNCmmAtZo@0Ph?69m8Jd>Q6FIZlH^^RyNizu;0q&lU1?;jzrX0@_R z=u~)EIHP6^kRu%DMVn=NIP@MIrMAhV=?o!Tv{?x#6(CRaEZw2r0#BsziIC1dF_Vex z%RRE6hU%pr=i?J1FJpb^I#q1SMA4&bJ$m$?VocVHF#I9UE_NiIjC7Dd$q( zCXJ{}I7?;95&bROkC~8OQQ`t4vdTmUDf1}VPslk0(`gUU@1e#*v1poFQL=P_e~)rR zMvLB)D7w5w2ruf+c(2i8N_9)$42#gytMeEp4|26^G*4I|(wjx5*&Z!EkoC%6;TtQe z_dwmcyp{TA)L0ainyo@KbDrpxIg0EPF(F1+tru3)kvoNtM57jXREZDPHuA1sRJHD1 zA2Ge8i?LM3%}8P7UDQ*eOzE^3Bh-l|e3^QycLc^EpB7_j$Sc*sz9H4QTr&@yh(bGM zc6db04`RfYIXIe*5l7*uUizexaGKhFai@wV<8n#bg<7FnNZiE{y|%q5$Eb^P z>e4bcQslv$%x5*7t&jH@Z;>ibbbP%oPU+rwzuM3i>)5zVW^>ldG=k+Ttaa84f6K30 zbXOUjrOXbB$odX>k=JEnCMp9@K^65->oIc`iXmMqreVDtS(JO!4@gOESJ$5zkt&ck zg=khh?*JqYRg%nMW)`7^Y>t?lDss7|Zto%Z!COpWbMd>>mnQq7+~h6WGa^AAPb7^M z-X?l@8z2UD-VUxoj*(Nq)s0r2UNZn6(*a9&&^CyYFdBnH|z1FL}|vP$qVw- z9Z$+p%8oWE%D0h0Aya4OPo6RUu?chY$3HRoiMf;W)wYkz(Um^YIkJ);-BZ~W(Dk&&h|@)EUaC*h9*%T~gEu3*|1#^sYUWUF|w3XJguj_831mf6k1_Go|rkh7O;5 zU($g2iht}tWqxEg^?_M(n3-^8TBkU*(>yssIUCU{h!khdm^)w^M8osvs3CLZ4(i*} zuq~V&nWos{Lo2(rUF4;F(=Lsb@61%$zw#k#Gg&<`PHw4$bnK!W?O;~}=gL{invN-6 z*@JT#_*=(b%GmZ@E6dx*%j(s9*-vdVMZQM~>>N><5kFa0k|WyrPRPm~kO>!6#~itJ z<%gYmn3Oqpf8A1+WXXg-FG0=NkWNi#dpBWbVBSnhGY@IXY=@2!%wildNwV@`LQG52 zMx3SMnzHg|LeDO`eM;lStdVWf)NNASL^&xFx`jI^M;UiJWP@vRJ+!_^k;%HcBw4^` zcUXwA(@c6v3`A~pcQ9(aXVh*LdqolpJI!qPi*L+Jw8E2xfF3qJKH zo@MJqDAfN zc$opuG~nwE`27aF+<<2p@NEX%k!ujdiv?K*w%mXp6$yNR(P-2|tpUfn(hyD>aIuqN zvN{7!g+`$AOOj>~_%eil)En^T2K4f4S0+W zf{r%eZ4G#i0k;6>^^eFDgCNe3AkToeGvEsh_+19Pz<^r~_(}ubu@QGjWd=cfqd;13 zz&jc6as%GkfNwM4T@84J0XLqi>}5F0Pi^1Lkf7R-AklyyHQ?P1c&!0XG2kZ+IGr-- zq0WG(Hq<|)nn6HkHF~Hw;AuJtdc%PCHsI0)3*)F@eGIs-0q-YpmVdB8kZwp2X~6p% zaEk%I&w$4p@Bs!q(SQ#$;HeiZy!_;;%^*lOBp77CZ3cX>0nawzb^|`zfDbX?ISn}S zA8HUxX%OJv*nsC5@N5IVz<@t!zzYocFay5QfDb2}mtT^~41y7c1nUj>NCRGOz#lT; z+YGq4Ic3rc11@gx81C3>5R5Sxsy5(akHMrz4ft4teysr)_nl07(twLSJj3fa?qI`j zyupxWNRVT|>kaq>1AfDRPcq=rMGNDoZ%sDfz6SiUhWdv>utD&+Awi@8pK8D@27H5;M)wi zxS?gz3IonZ9{-f~8U#xXhN=yCp#eW?z>5rctpQ(Qz)u?RXAF3q!ys5`5NHP6X~63Z z__GH5h5;`&;L?v4#!(ZO7;s;KJ4jG!5Cj_%h!YAXjWpoT8T2g%e6<0$6<+bxe!?zK zTZSyOAvhhe71sD3bf@gwZjs#WtMD1rHr;_J0aGZctImzkei!ky2X)mlemn6d;;R|I zg?KOGD;U3lcr)?kj9){%H}PeRf0lUKZ@LN?zl?Y|CAsq44j}Cp5#UQe4pW#zJnc7K z*^Hk?ydUxDjGsunKk)Bxk#5l{CXt_sGtCZ6s&T;+@pC7$jzTxE>+BcArFt^&rJ zh^Kw3%aO-`zuy2r8&p>g<9{VSiui2CUm~9NsjhU!pC_L7r>;cCpCX?2r7jEOPY_T0 zQCBeIKP5hvc!}{J6WlbT z8;G|OU(Wb7#J4BDjPcJB-+}l7#xEnjBdx!#JO(TxAf6O*7(a*jPQ+(3ej4$eiBD(z zMB=*;pUC(z#CIj$!uVmt-%Wfl;|CF+K)l5G{=|2q`PWr{i}e8lXus*IV|;hwlZdZn zd}rcm-|4Dmd^_Ulw$W9=_}0YtAikXOp~TaE)K$iKKjLX0>MCHoiTIv0|GDxQ@b@+V zsl?|n{#W8@AL`0x{3YV=B|e?;=ZUBNs4J22r--M0sLR6m6U5X0(-qA4Pl@kKyu|pA ziFfoPp#CQ70|cZKU&r`e#Ags+%lPfY(>~Ny&G;?E)Be*{!T1ftXA)n|_%*~2Aij+8 z&k}z>@db=u2Hr7cAOU#{SVRgo;&T{3hxjbwvl%~)_(8;{GkzlRgNaXM{21cx#9J6Y zjQAnM2Qz*U@k5E17~g-J#UV|heWt7aZ`KD$f%ccKI>vV=p7xclTE=%Kp7xWjYR0!C zp7xQh3dXl4p0j=ePuoUU8RPwkr|qJvfbphn4gjMF$Ya3YuY;$3qbrB;zYPZ3YsLYIZ{Cy1x*peva1pAtWgc!}{JI|vw0K>c5= z4-lV2d>!L=5kG#JohIrckxyl&-Eb)&KU%+_B zG6EhaAddlyh@V1y4&&z#Kb81w#!n-D8u96jpGf?4;uFDxvOue!t=RkdS1gikd3C9aaP6S43^SPR_Y_*!wJ^@*^2zKRNhVP}?Gj0?h73N@=9c zc`?5=)PLIob$AqrY&MrGZEbN5lGzF`)Mbpw7@6@<#;D=8rB~Ls#S;|e54GMC$ZhgM7?Z-awZZjz~YiqQq&(<`gD>Qo2M=J2$6ym$Rk41KkJi z587|;hL3=b_z3HVvPJu3Iy0+$Q4(e|)47%z%XDt!Mn0Q!IQhD*w6~S|0(BOxC5C%e zaV%?44%wU^+luGKzaK#*b}Ki~a~j&tVt4+nJUF&VLQby_)=q9ZgscKFjHL-m}(Php4TzN_3w+ADv$-I>nsGu$<{1K*o%#T$7;IL%Qu=ZGlZ z0cjV5maRpDI$V9xw?wPjom;JYk<}1qO=hvIH|T$BFuel|asJCzTx5;3pqh*O4M3Hw zxCb>fAo+TR`)H;!---#SPv*-OG(x7;n(4evSp2@bP|PEl4r^Cd)rB?n#gWOMWV}xt z8Zp|cwWD?}tO-7JGuS)!NotM})_A*flGWE%nrXGzoinVM5XV}rSTMQ z&tH^w^vU^M4rDribysINzsYp=wq_ug;u%)o4Cfbi(BS0j_mxKXv`xK9W6kFEG!E#| zK-v)Wr$aYGGOdvx-9n>~^Am9npbbDXX^Bi2Y_~>cxDRI)&yx-X|2DQ3>S1RGr2RhU zP24*vr^aVEzJr_ENu)rH1@peuLhoBz!Ebt>Vi@b8GdPB>#4y&+M+2ioG`h8$#Gq=l zP;?BN(~LUm^XF#dGDqZc`75*`AGE!_tr;$UqiWrlH6>}@sR*}>XKCMdG*MVEuLnNb?4^(1LJ=H?16)?R2dnr+ddMB;5+ z`l(UstKHEl>w4%NDYBjfp;4jD{TWGU#(4qv$*2huNcmX#o2U|~-GFsyoN3f3LlnjsLK@$2 zjk-n+atj)Degq+CyX%!>Xq0O$nmGP5c~P1YKSS{w8QXjtHGXOM-E)T5qDTXg z2^=Ot(;_hhpmqWILZhZpLllW2r13G=AXkJ9QKX{{MJm6hoPWp~xzWE#kxr8rr75?b zP+~{LI@Yr4509G73n;3#97cHoce5^~{lgM96p41htd&My5k-ncCUBT-k9q$RrWgWs z?G#KvgG@K}zzNiS7z9+?%T=~Es)#Z%gf!McgRR#KFxVHgH=P@+f2I@Ylk32&2UC?(Q2DWb_(#7ye~R=!6si}LxvJZ9AeV&P?lgA#I=G!1z{ zwphqmOTjvmeNHk5W>n&M#fUO71faGZDr{jstNjcgP_R7CHZ0?#sW8BEfYS`6FYoCB z&nA5V(%iO}%-$u0HoMc&L|Q@8W{! zJ>4LAqEWJq96it==>Z8=1UNmKZ2^;!vkfcrXv!kuKpploYsG#h(>ldg+#^_9enlBF z1`pSbNpoOty8J}Y3I#R?Bv%ia`Z=vzna-|OGJ!kJ5!g0hskhmR^R3y%C|H@jcqhC6 z%5YZ?@p54UAKhx8bL{iJIxfJFfzFN3)BS-59!fafC@_2kauW%HNkDfD1DqdcrX88H zKC_s6F&g_aH!omD+M%Fj|H4iTWfL26?1$kyIqIuTyTl~Q(nrD^Jldc{9fOo?+C5x? z-Xw%0tl3%|fhcj}e(L(qKM~=3E zx(kWihXxf#|6+4`*_^|D({@_x94J`OvX>1Y<)z3VAADq zCauW2o=hzNl6IV>eZF67k+iXBbhI-za(y0r-hx&?;(ZnCFim?}JrdzId=g|Sko8Hf zhKIqn+_h}4sr4sMOJfc~-Bwz{Fswuqv6R|Nc~(gH4>GZ1YvHTZj*W1TkA*ds%M&3v z4hgrYtw2V$Qt!u^rN!h-io)VNC$dhS+e>j@n{6xI$hJS&Jr38-)AENsD(o$7wD%+O zLFtn}q47ZF=WXdo*PvZI)|zT_{@d0Cb!^+aV>=6-ZGXCK2QHax2mb13>vG)Y{?wlP z>jS0M`B?`pcwtIh@DytkrTw_%!Fo5z_a?jEzS`f^euanXZSKz&_%OZ8Uo$eO*c>;= znHlGsRam2CxsQ<)xnEw+_GQ|MOUf(bx=hmT_3(l@FJEmrGj|Z?O4zN~!mnt1PrFE# zjK&_{3F`N#Ygv70L#TAvi;z$YzQ`H2`7HcNp-FWF7SKG1PIwByu9@v*3j1T9=3%LIm@SQ+||EXmB z1Vj34LwloFJTV^49>$yf9ds0}<~$|n^Dq|=B{3eZq)xMj#IJI(9~XBcaVQtho#cRg z1($!2!>k9BJcP?fQ1ZvQcqQ+1zu>@jUoaP^kl4n>eR%2LCxXE<%4lf4Eb7;x8U-VB)|MUE7~NUd>)DE$O1)spTwVYadR$yh{XO}yn)1T za`8}ZyeEleE`Ex{tG;(TsN$aE@<@`?4iVPJl6)3VUdzRQJqocKO$2ct5@&O92ynQ5 zip00MIF`goTs(}6KOpfxTHZon^u1=JFmMpdxho5*x3I4dVBg5;H1AjHM<$hu>-fU}&{|kFI z+RIiN;KX$_qu{1_UQ1i^5L@#HeQ^cjio|6Z5cJ;Z4EKR7clE%a_o7=5Y999QOdKng z9!x%gJ&UU|W&@&AL4%kpa8P;(g_=ZF(&j7`Fl992wzSgle~Lhh`}BPVqNOA%_6|k- zaC!!=`{cvv1HBHX4>cc7A7OgqhOneLlaY_=MV{e!$b{h_i%9w5=&fs z{1LL4@+^sWbMZAer+t5#SzpA((@Fde7k@$GkGc3RF1C~SEr`d!xeAiM!sTZ4V_3hN zX&04edrw9$7q{Qx!vqw zZeK7oicZsT^bHzt9A~fEMCx$VJpgF~zzit5kRJk1{(yOoe#BfL`MmF_s1!SWM)BX$ zI_x8=8akGG4O!{EGU{kI)7nC1(9zkZxI*Q_qj&Xt$X4uw18Q_m>cHt33B=PEDMo&@ zcT=8DyqR1%H%;-OGdtVz6UxHrf0{0wQRd7@#l3*t8a!nA>B?O*TFHL4(y+)Z=eI-B zT*`?jYD}BHR@TpK=kpD|3e;ZxN;xpIZQF-zI2!b|IseoKeeHHXe2}|Ak5TY!Ht*BDTrRe9a?<^z@A?|xS{br_Q^*ei4PgvzE}|EkR2yVPXb zd|3H&U%6?3O?lyi%ck;ADpyqPF_|2hO23Z=n-(2ZNx~=&nQ5i_(?7Q`t=7G^Td52joG8=a50$hz zvrP9MP^?Tx+4zZNBh3@$a#&nmSaLXF|hF)Qtg67*C` zpv@`6W#;w}xIOHt+ZJ1?`4LP%2W-w!q!v_kly6?<#qVyPBr#Q8q1(?Vd`W zlOyNeYvM>Aa4l_@aAX;Tt|_pht$a_pwz#|F0s#8n0{8tRtck8EH@PB}B+=%)t?g#k zVD=GV7KbGH7=Ks5YJ1)hW^ca>vlngwGLt=g1>?&e7P6s1SqS3iHt}?LW1$b!@Tn); z*^&?ZfbWmbY2zpv45w4ZsaeHOVz1E6)dSy)XqB+d4#dLe;SnC~_X_32(*yBUe;CFS zzWEul{DRVcNpdHr35zyruRpbE(DHXN``Qj%G$UUx4TG1}?$qF{wED`WOWMk)?FM~V zsC>)OOvc@Mk1Slj+R&{~-&d8Ig5I5N&So~}{V;q#mV^sI%S$knarX_wcCWfDd=cw3 z*_{)EZE0Urc3v7Gn_k(e+_$WgX~9lq=CWa?%y*QJmvwLiP(@MWR8~+#^pbcj6Sx+% zQ`F)^ObD$~7MX#XT7EUZFRK!*Ibk@B!7LQWs{Jg(=?EcM`y3{;BRhnhK;6zE7FldH zr)rd~)G_@}z-nu^pPU7Y8#7f+%-h;uL?m_}$V7xUd9mZY+Ld91go!&asG*=|YUl+=01 zhUv-GipFL>GBIvp8&iM!3d_(^92X`Y(in;LD-laJ_QRcqDB-ITrfflp3U6SXub^ox zm%bBp{;ZuwO3*BvK+^_Z`>c`ACjJBP^u-cXD_9hr>}Ty&7Cl7K4J-k?r1maN`Jscd zV^yr_v#rW!tL}FE&sES3kFs!}Liu3H%Rq<3Fgb23JYWAm(;Y#&3`{tE_@V4r^i^B# zZQFsLOthAMLRNF}Jt%m8D%NN!#_lbOY+Eo8yg>SeI z1liY)s=c&@+MaKME|t68kCiEPW%rxVcR$uJ? zw7I~X_u>3k@IP$kO{4R}A!oY>=MTl*MM|`zJ0z?+OksMjD+9#d+==O({ z0-%JGPh+};O|xQZ7lAco$NI|fa;4>(34Q9waA}p*LQc0dF#VmGmA=|in8!E38SW#+ z{+VgN%=zGQ-yvzY^P{MOCzmVltcf;#yGi+C%{|r!MA~>Vv}_Z%N=$!eQ6-&4Ygacb zos@3YTSAsdirowhDbnGzwqdif7>aKQSt==>d5K#jh7{?rS{t}o`9f*q=#2KeF`FR~;y`g?1 zpk+D~Z*suYcfedTP)Rz>dg>8EA0o80&nOyqr77v;#5(RoIpW%Ia-y_P5AI_eiJ#}< zr6lg*>0?Xy=n8^OJaCsONqBNjsa)4`VBgJT2F13O;&B4>8LdjMtzsOo6Yv3w;yfvK z)sOHS4OUTGAYk?P@6AdJ)!O0(GKHsmkO<$n_f?~sKT2NUK;S+lNA1<>Mdq@ovp_CV z?^E7ZlQSm5J38JDMsLMBpL`vs9rSF|R)%Pms1SScR_ivHwmW~Y;ftaQil$w}JubF0 zzim+NS|8u-6WB!GUq0K6|44aPPCkviFZQ>v{kiDrco5+SmY4VE{gp-QA89_Gb5r^& zXV>3t>eXLq@j@HZl?_Vg7utuSFfL?6kL^&Qv;i+E6JBWL=z}Etd|73SimN8+d^kzv zXUsxT8I)c-!^*1Pw6L3=hhc+UT}8vQ`0i&jELeEN?fcUz4AB_e%_puHW-BEyih4qo z)ra+wH(pTwdEre{>y66J4VkScA&I@Xid7J~yPn*3o*a^POYwU#!4$13_r92G+OtmC z{9?BVdgP1lkcOoF6jW48HSYXLOH!4KFLpBBzfK9>7}f5}Ry;43)&8~AI8EIDOjQPN z>@+oTJv!Cvht<-kgNgO$IE z^&Wm#SEX=$wtjUZ2GvEa%Ed1UF>B{wCf?aln-!#%b~zspZT+nMK($eL$d_(eex{yC zC8Lca_N+f^BNcZ5^NSU6?^KV$Xo!Wn?XSIk$xGA!5 zlu|CIl(nmsX5}4C&#Y6rl}BXFgr19i+aWp!8w>GN?C4vf*!plG4=!>z~V}WVbeIt@2#C)wFuG@@{!sQ=OuGQ*MdbPc7s%mhS%ed}70+YSfa=HKj~x z`EqB|GtVpiUT)v}r%a2a6_-J~H03nnIFWp!CrELhw|YVH3nX-CRDjWJ*aG`8SjpZp zD*i*Njp80wTbHvq{1_t7^vx<=oPx6`;LdA!f{is*$1>%MEv=LMkq*y;W)&}iV?C@W zg00IpSY0c2nS5Wy&itt!F()Iblccyo523)#WQ} zL+s8UwIi#Q&9B6SZd*-z)V^6>S9yKQHuK z==w_S)~=?btCZVYC?)&VXgX#{ zM~7DCzS_|=tyJ0g>UdN0QstjlA2Ur@txSF`%`~Y*+49=m0a>NJzpyzmAKTOZP)@%V zV+try{(h|k9?J?lXLDu~oR4QZ{Vy$!udKqu5dMbahf>^IX&G?i zauuJ?h))9V8t8=)$cE`+x}CnQeZblZ&Q4?zS_W`a7OBmJwzmISlmyzE_jAhe*KtIz zE4OW`oJh(~>B=9YEnIWhRshxEx_O%$p~_)dcGs=^8;XRAEH4$qN{Vh}F_ISrHD3+}lb+evD zC=1k0y!M6@j{g3PsGBrhYYvNyrIVl2`Ms%NsoHI5QCr~&+PO%Dy*)-17tLJ>D{tvm z{y^bbUOPozsanj(Bt#KWPCD)>4Lgjo;%n<3pK2YQO3;orj(hdIzejTRoHw41)qW?- z{aDUmUHuIWz>a3jda4~P(}r?YQ92*Ww6srGYzwrT%h_neFkJdTB>h$cKc=I9UKmdQ zZ4rWPN%h^Bgm4dxHKjDBQfd9$U8Y$jO26OY*4zkGhWOg=d8O~I(fC=O+g&Pu`{!<%b|Wk(KmQe@G{1gd$ehU&CfO&? zd^~@OrMD#|F;P;ouD4NkUoTN2{}@WY4-~5G{e8;w1#*sJ`J+h9lx55FTjkJ3L$U5w zLAkSL&Zl4d$(=G`PC{<3($D=UZVhW3avQHST4MVm&&5mKZu-r(63`K#RYh+1Sd5n$ zw8O!jQ8YF`8$oYjd(<0GOxm$)91FS;JI~#qJ3&7OtpdFO`Z?$=(8buD=VNl~jm`OP zZ0PKuUxAK&+3j%uj>rrEVct*#?EyODRkwQ*zH3+vx)`(sbR+0S&??Y?eJ}(^?4ZAs zL!e_(-p@aP9_Sg+-Jtd=w>zLYG6L-ZIsJvC>dyP&}ck?xfpaTXbI?I&|BX?4jS++mS@yJSI~{9 zfq+x6hi()Nx)Jn@;~cte2n^M^-8(@`E+M0EWCU6Q{&P?}{s(jM6&S|D=p~@^sQO0G z-Jm-`zXH7i8Wx6(LA!!RUxh=UT|tXL?V!6sKL-tf{ea)y?yaC<(W1A>i?WAF@`B)I zFJAAW7MSI>rmwrGFPrg$oT;cIx@pye8NCJHeMyPH*>6q~AO!}S(Nm;=X2B~=W@ID< zdj+OOw?MJnty`SLXS?8lMgXUi09rHx^$ZAX1T+Q&1*ST5BV?+b*@RpUvK+sle~^)m zvCn_8&(F>18&ZIe?_s6@U;e|16{i1RH?ql7_|e1-WP>mQ1<%Aada2t@KbdBxpA|Ht zKT3gR>GWG^QfYvhs_wZ!b5D>$zZK0_wNN*B%dOid2);C6RSSp*f(2K%VzMOMN`l}E{aqGHIko%oPPSx{85nFEmDqEM@Gmm_A9}pQHRY&1+?@)op3Ag1=wKH zhFlN55!nbldzagtj=1^TK=s>Za#GM40g@5(+kxsML2?`aLbDmUN`Z-Hhx$c`e3zQz zBL@d%qNA-x`dU&QAE-Kf)v!{nei66Rq2Tfsr2Y zbOz*gkkijInxFIEgFe$qj;%o(YX_UUO!adHc-=wu#};yIGUZM~Djr`8-l-m_RY@KR zc{;^k^i#V9%6GGblZ4<-2^T||so9ZS&t3O;G-!%nCc0bc-|e!bS5>%+!}da9Kiq8<#9=LPjZo31?OcIQCV+}1}O z7%FOiEd=rZzH_r}gfht~A^r5K`E)aNbEv%9e+1fU0oD!Nuv2sNQuABNSH=~gL*4+& z8WKq7!a9jjuMaA+I|F*VhSl)c7C^VMN(<#p=)?d0=nc02jx|KmsB zP;53>(0}T%Vw*b1B6o1;KV_&dC~g{^2s+;!`v4;!=E1l1;OTc7jr#Pnj7C1QDesVm zH3g&`51xJ)(r93j;4vYvp#QKr%>xwl-_S+}Wnr%<396Zeqv#+MsMpdjS4fdO#-1oA zs8(cvS&{|Xz)--pfQ5?l7Dj2bP_WR7jvy*dqWuyDd{M$e_1&E6cy<@gvynypKTlY~ zOw59w;cd5j76H|rIga*Yyk4?;!LRSc<$no2h<;fAbuYZS0lx?x?g&if;#ffw1nn#6 zP(jBDI#bXkg02#Dlb~-2x?j*^f}R!hnxMBibx47|gad*m2-;WBp@NPRbf%z71YITQ zCPCj4bibg-1U)P0H9>DPiWUx}-`hZl6*NK6zJd-Fbey0w1zjTODnU01`j(*k1wAI{ zSwXJ}dYdTCLV;t_ zf{qh(rl3m%T_xxyLEjQ|zo5qiJuB!nL2tAB3*;S02LcGOf+h&sSJ0t?juUjIpi2Z@ zCFmwW-x74cpk!O05h$+z5^S2kcaE8ng1WS2d4Jitd%xJ-OE%Rb3XfGG3k(os`zSN79`% zs-eMLN=Z)Zj_XeOUD8#0K&8i>RH_?PP_UFc7A*a;ZUqHPxnseqg-WxOo>1wjZcLEv z>iTZ-Aho2M9H36`CdchL-%Y+B3w4s5E00y3NpiPPecIP)MGN)QBsn2eUmtW@8?O3w zmlHztSyrc3HLJVa%}<}4t2?X9y36;3>a)8}^GxceNFJ)M0y?$$seZ{Yt1n+VjZ|XH zZPh8sa&YJky2nP)X>Btl*4#=hPnIJ?_16OFblY6@Q!=NoemV_g=Hh$6T=_0;PNzlw z>YyI*>_6Y%gKT2eZIs4R&C}@gLQ5voNM`6NA4D) zugVnyUZ8#gI5hr8o;?0Xt{qHu7^}wjl!HUkuW(uF6)wwBvwF%so9pb`>i74`E!B-Z zWvkauSKwbyxlefBA6z$2tnM*jr>Wtoau>h40GY|YYo`uMm1idD9g8R6pI(w#(mPgD z+*?}zJiRW)3mX%?Pc_x>k@P;*6!(?%KGhWe&mUtaxfk|1dWUMN?=PV(pRx2*{qBjtU)^D01RMIG)X(mf?>5!;RsX(M zPH=dZi1v0eemdW?M6`L5@wO(<5)A-u!p}0-8`d4l^bw}dG9zo{AR6l} z3ATyJKuPSJMpgMS^sJ>lx7NHdqE8`t;K}tcE=F zj|n{fQd9eMc1rr`mmEz!3P2}d8F(Z}5IA-M5(V8MdXo|9$6w~S-UM_M%Q6tSuD?>Z zuj}g**>*hxj|7JaN2l8;y0^G1lzecXNO*<7F#uRd6L_`2#Z98LK;S0@uHW@J=&@rm zR4)W^!q6825Br(}%>^FOjO$xGa60uMJ6%2SB7vt0oKA-*)N&D$Xwv zcz?oCvPh9Z8-YIx+>AHd2)2Rfc0T-&L!E_0pI34O{wU$p{!fSudWa0BiUd{6p-3rF zOf0&iUkd$lp&upmHGx+Ne3QWML6uMe#(v8Iy@u#DFb)z-@eq`W1o3)X);`UtD#xndr*UhO6 z50&EUEV>Cs=LuFCer{C#lhgHGzHFGZu-19Ya*XS)f-YxvaTX(!u^;pcqh><1F zSIuqJHM~D;h<()%yQ0S&&2V6Jr`OJP0#0ww@qF#vI)Qt>cJ4ZG=ErzmLYnd@B(>nU z=WFLi0LPG0_hiZ;DRf#*i9O#tw;gzs_s(&f(#OE*6sM*4aXltqRNu*zBV?Jq%P1=h z@_4-~6Mv5q(A}(5KRR zzL4$=NwcmCYo0HpOT;*%Ts&V$mnm@17t+mVFPC~>PbErf5oP%|d-2t8^cfS07AMP| zFQjXDWtpQyXkN+;Mc|(Aq}wcT&v(+D0FEq_J-#7w zXL{9`oI#%{!^3#Kp6{gF44m@yd?(${gtIr!`G@nAp6{faq&$e4C=^o9_tAX@JcQCK z+5YA}l!NCx=~}hA^E>G}04M#1chV`cf7_^O0`+_u-7`u^K!p4{eP$gII+o{~*7@JS zsm?uLMAt2X=alWy(l#XyuO2L7pPSg{xAYm+6}uGjtU|P@{sOvOHDj?*9&vB zvQs!FaL+f;{SKVm@_YkbbF530jprNa1_EdADvNoBVx;jf&!JAN0~NZ4_jNrc`>RQI zIU*{#H8<+{I=Ts5{Gt$hzK($vqQL(>5Ol{%$F1q{G;}FZsIN8KV6g|+- zdQdL+NzdaiHR%Dl8@@H1@PHgKVF~;uE&W&j^F#rv1)k{P;Z1?(iH=tw^jpQ>qUhZdF3^X+rXm^f*J5PQCQu0|O5eDz$6`VR7zdvY?K zlO>EK6}96IdA@w^J>YkG`CJuCrdAA>-D>HC(OE9t@h!W8B#*`gdV9q z$9Eq;e)F4Ij&NxVl_T{=pR zYTheZXTKjMw`|_4yUsF0Nh$x`P*M+Fl^IHHYACg-q3*i0!T&~&x;Ir#PWtb8CO4Fm z<(bk;cY!&dlB%;@svA#0&65&S6A}}Wx~J?}_^3QuR(Fn<7pf=c$zf{F1lgjzAKOOVk|PgPKc6cHDNncY zQPU^LE7Tt*$OqKQ5psqam@BtcXU~!o*lP~eb7SPY_Uy@(L*+fcPLjjq(DD40hInP6 z{zgJ&cJv^1=woup9-qhMh4P+~DRL_>_1JXzw0d@iyjeLHVNv%!A@|!8I8&}LEA86! zP+QEE$EfL3WwZK8zHC+g?9f{Ma4tMZSS0)J*_AI}H0^owNx7e_x);c|m0J<5)!0Sy KbhT`e9QS{C2kt%q diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index 59d00928e..432a528bb 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -1,44 +1,25 @@ +#include "headers.h" #include -#include -#include #include "org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -#include -#include -#include -using namespace std; #define ENABLE_ASSERTIONS 1 -//#define DEBUG 1 +#define DO_PROFILING 1 +#define DEBUG 1 //#define DEBUG0_1 1 //#define DEBUG3 1 - #include "template.h" +#include "utils.h" + + +#include "define-float.h" +#include "avx_function_prototypes.h" #include "define-double.h" -#include "shift_template.c" -#include "pairhmm-template-kernel.cc" +#include "avx_function_prototypes.h" - - -#define MM 0 -#define GapM 1 -#define MX 2 -#define XX 3 -#define MY 4 -#define YY 5 +using namespace std; class LoadTimeInitializer { @@ -46,6 +27,54 @@ class LoadTimeInitializer LoadTimeInitializer() //will be called when library is loaded { ConvertChar::init(); + m_sumNumReads = 0; + m_sumSquareNumReads = 0; + m_sumNumHaplotypes = 0; + m_sumSquareNumHaplotypes = 0; + m_sumNumTestcases = 0; + m_sumSquareNumTestcases = 0; + m_num_invocations = 0; + + if(is_avx_supported()) + { + cout << "Using AVX accelerated implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob_avxs; + g_compute_full_prob_double = compute_full_prob_avxd; + } + else + { + cout << "Using un-vectorized C++ implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob; + g_compute_full_prob_double = compute_full_prob; + } + cout.flush(); + //if(is_sse42_supported()) + //{ + //g_compute_full_prob_float = compute_full_prob_avxs; + //g_compute_full_prob_double = compute_full_prob_avxd; + //} + } + void print_profiling() + { + double mean_val; + cout <<"Invocations : "<(&(tc_array[tc_idx])); - double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); + float result_avxf = g_compute_full_prob_float(&(tc_array[tc_idx]), 0); + double result = 0; + if (result_avxf < MIN_ACCEPTED) { + double result_avxd = g_compute_full_prob_double(&(tc_array[tc_idx]), 0); + result = log10(result_avxd) - log10(ldexp(1.0, 1020.0)); + } + else + result = (double)(log10f(result_avxf) - log10f(ldexpf(1.f, 120.f))); + likelihoodDoubleArray[tc_idx] = result; } #ifdef DEBUG @@ -353,5 +389,14 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBasesArrayVector[j].first, haplotypeBasesArrayVector[j].second, JNI_RELEASE_MODE); haplotypeBasesArrayVector.clear(); tc_array.clear(); +#ifdef DO_PROFILING + g_load_time_initializer.m_sumNumReads += numReads; + g_load_time_initializer.m_sumSquareNumReads += numReads*numReads; + g_load_time_initializer.m_sumNumHaplotypes += numHaplotypes; + g_load_time_initializer.m_sumSquareNumHaplotypes += numHaplotypes*numHaplotypes; + g_load_time_initializer.m_sumNumTestcases += numTestCases; + g_load_time_initializer.m_sumSquareNumTestcases += numTestCases*numTestCases; + ++(g_load_time_initializer.m_num_invocations); +#endif } diff --git a/PairHMM_JNI/pairhmm-1-base.cc b/PairHMM_JNI/pairhmm-1-base.cc index 90f95aeec..a3b5e0ad5 100644 --- a/PairHMM_JNI/pairhmm-1-base.cc +++ b/PairHMM_JNI/pairhmm-1-base.cc @@ -1,29 +1,15 @@ //#define DEBUG 1 //#define DEBUG0_1 1 //#define DEBUG3 1 -#define MM 0 -#define GapM 1 -#define MX 2 -#define XX 3 -#define MY 4 -#define YY 5 - -#include -#include -#include -#include -#include -#include +#include "headers.h" #include "template.h" +#include "utils.h" -//#include "define-float.h" -//#include "shift_template.c" -//#include "pairhmm-template-kernel.cc" +#include "define-float.h" +#include "avx_function_prototypes.h" #include "define-double.h" -#include "shift_template.c" -#include "pairhmm-template-kernel.cc" - +#include "avx_function_prototypes.h" using namespace std; class LoadTimeInitializer @@ -36,85 +22,6 @@ class LoadTimeInitializer }; LoadTimeInitializer g_load_time_initializer; - -template -NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) -{ - int r, c; - int ROWS = tc->rslen + 1; - int COLS = tc->haplen + 1; - - Context ctx; - - NUMBER M[MROWS][MCOLS]; - NUMBER X[MROWS][MCOLS]; - NUMBER Y[MROWS][MCOLS]; - NUMBER p[MROWS][6]; - - p[0][MM] = ctx._(0.0); - p[0][GapM] = ctx._(0.0); - p[0][MX] = ctx._(0.0); - p[0][XX] = ctx._(0.0); - p[0][MY] = ctx._(0.0); - p[0][YY] = ctx._(0.0); - for (r = 1; r < ROWS; r++) - { - int _i = tc->i[r-1] & 127; - int _d = tc->d[r-1] & 127; - int _c = tc->c[r-1] & 127; - p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; - p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; - p[r][MX] = ctx.ph2pr[_i]; - p[r][XX] = ctx.ph2pr[_c]; - p[r][MY] = ctx.ph2pr[_d]; - p[r][YY] = ctx.ph2pr[_c]; - //p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; - //p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; - } - - for (c = 0; c < COLS; c++) - { - M[0][c] = ctx._(0.0); - X[0][c] = ctx._(0.0); - Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); - } - - for (r = 1; r < ROWS; r++) - { - M[r][0] = ctx._(0.0); - X[r][0] = X[r-1][0] * p[r][XX]; - Y[r][0] = ctx._(0.0); - } - - NUMBER result = ctx._(0.0); - - for (r = 1; r < ROWS; r++) - for (c = 1; c < COLS; c++) - { - char _rs = tc->rs[r-1]; - char _hap = tc->hap[c-1]; - int _q = tc->q[r-1] & 127; - NUMBER distm = ctx.ph2pr[_q]; - if (_rs == _hap || _rs == 'N' || _hap == 'N') - distm = ctx._(1.0) - distm; - else - distm = distm/3; - M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); - X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; - Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; - } - - for (c = 0; c < COLS; c++) - { - result += M[ROWS-1][c] + X[ROWS-1][c]; - } - - if (before_last_log != NULL) - *before_last_log = result; - - return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; -} - #define BATCH_SIZE 10000 #define RUN_HYBRID @@ -129,45 +36,55 @@ int main(int argc, char** argv) if(argc >= 3 && string(argv[2]) == "1") use_old_read_testcase = true; - testcase tc; - if(use_old_read_testcase) + if(true) { - FILE* fptr = fopen(argv[1],"r"); - while(!feof(fptr)) - { - if(read_testcase(&tc, fptr) >= 0) - { - double result_avxd = GEN_INTRINSIC(compute_full_prob_avx, d)(&tc); - double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); - - cout << std::scientific << compute_full_prob(&tc) << " "<; + g_compute_full_prob_float = GEN_INTRINSIC(compute_full_prob_avx, s); + } + else + { + g_compute_full_prob_double = compute_full_prob; + g_compute_full_prob_float = compute_full_prob; + } + + std::ifstream ifptr; + FILE* fptr = 0; + if(use_old_read_testcase) + { + fptr = fopen(argv[1],"r"); + assert(fptr); } else { - std::ifstream ifptr; - std::vector tokens; ifptr.open(argv[1]); assert(ifptr.is_open()); - while(1) - { - tokens.clear(); - if(read_mod_testcase(ifptr, &tc, false) < 0) - break; - //double result = 0; - double result_avxd = GEN_INTRINSIC(compute_full_prob_avx, d)(&tc); - double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); - - cout << std::scientific << compute_full_prob(&tc) << " "<(&tc); + baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); + cout << std::scientific << baseline_result << " "< +#include "headers.h" #include #include #include @@ -7,11 +8,11 @@ #include "define-float.h" #include "shift_template.c" -#include "pairhmm-template-kernel.cc" +#include "avx_function_prototypes.h" #include "define-double.h" #include "shift_template.c" -#include "pairhmm-template-kernel.cc" +#include "avx_function_prototypes.h" #define BATCH_SIZE 10000 //#define RUN_HYBRID diff --git a/PairHMM_JNI/run.sh b/PairHMM_JNI/run.sh index 8b8d44b54..495a3761b 100755 --- a/PairHMM_JNI/run.sh +++ b/PairHMM_JNI/run.sh @@ -3,8 +3,8 @@ rm -f *.txt export GSA_ROOT_DIR=/home/karthikg/broad/gsa-unstable #-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI -jar ${GSA_ROOT_DIR}/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ --R /data/broad/samples/joint_variant_calling/broad_reference/Homo_sapiens_assembly19.fasta \ --I /data/simulated/sim1M_pairs_final.bam \ +-R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ +-I /data/broad/samples/joint_variant_calling/NA12878_low_coverage_alignment/NA12878.chrom11.ILLUMINA.bwa.CEU.low_coverage.20121211.bam \ --dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -stand_call_conf 50.0 \ -stand_emit_conf 10.0 \ @@ -14,3 +14,8 @@ java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI -jar ${GSA_ROOT_DIR}/dist/ #--pair_hmm_implementation JNI_LOGLESS_CACHING \ #-I /data/simulated/sim1M_pairs_final.bam \ #-I /data/broad/samples/joint_variant_calling/NA12878_low_coverage_alignment/NA12878.chrom11.ILLUMINA.bwa.CEU.low_coverage.20121211.bam \ +#-R /data/broad/samples/joint_variant_calling/broad_reference/Homo_sapiens_assembly19.fasta \ +#-R /data/broad/samples/joint_variant_calling/broad_reference/ucsc.hg19.fasta \ +#-R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ +#--dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ +#--dbsnp /data/broad/samples/joint_variant_calling/dbSNP/dbsnp_138.hg19.vcf \ diff --git a/PairHMM_JNI/template.h b/PairHMM_JNI/template.h index e70784073..52a4e4650 100644 --- a/PairHMM_JNI/template.h +++ b/PairHMM_JNI/template.h @@ -1,24 +1,14 @@ #ifndef TEMPLATES_H_ #define TEMPLATES_H_ -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include -#include -#include +#include "headers.h" +#define MM 0 +#define GapM 1 +#define MX 2 +#define XX 3 +#define MY 4 +#define YY 5 #define MROWS 500 #define MCOLS 1000 @@ -130,6 +120,7 @@ struct Context }; + typedef struct { int rslen, haplen; @@ -182,7 +173,8 @@ public: return conversionTable[input] ; } -} ; +}; + #endif diff --git a/PairHMM_JNI/convert_char.cc b/PairHMM_JNI/utils.cc similarity index 84% rename from PairHMM_JNI/convert_char.cc rename to PairHMM_JNI/utils.cc index 01c5a5137..5b91fc7e9 100644 --- a/PairHMM_JNI/convert_char.cc +++ b/PairHMM_JNI/utils.cc @@ -1,7 +1,36 @@ +#include "headers.h" #include "template.h" + uint8_t ConvertChar::conversionTable[255]; +float (*g_compute_full_prob_float)(testcase *tc, float* before_last_log) = 0; +double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log) = 0; + using namespace std; +bool is_avx_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 28)&1) == 1; +} + +bool is_sse42_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 20)&1) == 1; +} + int normalize(char c) { return ((int) (c - 33)); @@ -184,3 +213,13 @@ int read_mod_testcase(ifstream& fptr, testcase* tc, bool reformat) return tokens.size(); } + +void debug_dump(string filename, string s, bool to_append, bool add_newline) +{ + ofstream fptr; + fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); + fptr << s; + if(add_newline) + fptr << "\n"; + fptr.close(); +} diff --git a/PairHMM_JNI/utils.h b/PairHMM_JNI/utils.h new file mode 100644 index 000000000..8c244333e --- /dev/null +++ b/PairHMM_JNI/utils.h @@ -0,0 +1,13 @@ +#ifndef PAIRHMM_UTIL_H +#define PAIRHMM_UTIL_H + +#define MIN_ACCEPTED 1e-28f +bool is_avx_supported(); +bool is_sse42_supported(); +extern float (*g_compute_full_prob_float)(testcase *tc, float *before_last_log); +extern double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log); +void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline); +template +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); + +#endif diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java index 58b8fc591..a80800f82 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java @@ -66,13 +66,13 @@ import java.util.Map; */ public class JNILoglessPairHMM extends LoglessPairHMM { - private static final boolean debug = false; //simulates ifdef + private static final boolean debug = true; //simulates ifdef private static final boolean debug0_1 = false; //simulates ifdef private static final boolean debug1 = false; //simulates ifdef private static final boolean debug2 = false; private static final boolean debug3 = false; private int numComputeLikelihoodCalls = 0; - + //Used to copy references to byteArrays to JNI from reads protected class JNIReadDataHolderClass { @@ -138,6 +138,7 @@ public class JNILoglessPairHMM extends LoglessPairHMM { jniInitialize(readMaxLength, haplotypeMaxLength); } + //Real compute kernel private native void jniComputeLikelihoods(int numReads, int numHaplotypes, JNIReadDataHolderClass[] readDataArray, JNIHaplotypeDataHolderClass[] haplotypeDataArray, @@ -160,6 +161,7 @@ public class JNILoglessPairHMM extends LoglessPairHMM { } int readListSize = reads.size(); int alleleHaplotypeMapSize = alleleHaplotypeMap.size(); + int numTestcases = readListSize*alleleHaplotypeMapSize; if(debug0_1) System.out.println("Java numReads "+readListSize+" numHaplotypes "+alleleHaplotypeMapSize); JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; @@ -275,6 +277,7 @@ public class JNILoglessPairHMM extends LoglessPairHMM { return likelihoodMap; } + /** * {@inheritDoc} */ From 7180c392af944a2bc47d842e4c9029eb9b7612fb Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Mon, 20 Jan 2014 08:03:42 -0800 Subject: [PATCH 12/72] 1. Integrated Mohammad's SSE4.2 code, Mustafa's bug fix and code to fix the SSE compilation warning. 2. Added code to dynamically select between AVX, SSE4.2 and normal C++ (in that order) 3. Created multiple files to compile with different compilation flags: avx_function_prototypes.cc is compiled with -xAVX while sse_function_instantiations.cc is compiled with -xSSE4.2 flag. 4. Added jniClose() and support in Java (HaplotypeCaller, PairHMMLikelihoodCalculationEngine) to call this function at the end of the program. 5. Removed debug code, kept assertions and profiling in C++ 6. Disabled OpenMP for now. --- PairHMM_JNI/.gitignore | 3 + PairHMM_JNI/Makefile | 6 +- PairHMM_JNI/avx_function_instantiations.cc | 6 + PairHMM_JNI/avx_function_prototypes.h | 19 - PairHMM_JNI/define-double.h | 54 +-- PairHMM_JNI/define-float.h | 4 +- PairHMM_JNI/define-sse-double.h | 128 +++++ PairHMM_JNI/define-sse-float.h | 127 +++++ PairHMM_JNI/headers.h | 1 + PairHMM_JNI/hmm_mask.cc | 451 ------------------ PairHMM_JNI/libJNILoglessPairHMM.so | Bin 96945 -> 0 bytes ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 104 ++-- ...te_sting_utils_pairhmm_JNILoglessPairHMM.h | 8 + PairHMM_JNI/pairhmm-1-base.cc | 82 +--- PairHMM_JNI/pairhmm-template-kernel.cc | 161 ++++--- PairHMM_JNI/pairhmm-template-main.cc | 74 ++- PairHMM_JNI/run.sh | 7 +- PairHMM_JNI/shift_template.c | 38 +- PairHMM_JNI/sse_function_instantiations.cc | 18 + PairHMM_JNI/template.h | 47 +- PairHMM_JNI/utils.cc | 35 +- PairHMM_JNI/utils.h | 19 +- PairHMM_JNI/vector_defs.h | 26 + PairHMM_JNI/vector_function_prototypes.h | 19 + .../haplotypecaller/HaplotypeCaller.java | 2 +- .../LikelihoodCalculationEngine.java | 2 + .../PairHMMLikelihoodCalculationEngine.java | 2 + .../utils/pairhmm/JNILoglessPairHMM.java | 38 +- .../sting/utils/pairhmm/PairHMM.java | 3 + 29 files changed, 708 insertions(+), 776 deletions(-) delete mode 100644 PairHMM_JNI/avx_function_prototypes.h create mode 100644 PairHMM_JNI/define-sse-double.h create mode 100644 PairHMM_JNI/define-sse-float.h delete mode 100644 PairHMM_JNI/hmm_mask.cc delete mode 100755 PairHMM_JNI/libJNILoglessPairHMM.so create mode 100644 PairHMM_JNI/sse_function_instantiations.cc create mode 100644 PairHMM_JNI/vector_defs.h create mode 100644 PairHMM_JNI/vector_function_prototypes.h diff --git a/PairHMM_JNI/.gitignore b/PairHMM_JNI/.gitignore index 9722ad970..a68330124 100644 --- a/PairHMM_JNI/.gitignore +++ b/PairHMM_JNI/.gitignore @@ -8,3 +8,6 @@ pairhmm-template-main *.swp checker reformat +subdir_checkout.sh +avx/ +sse/ diff --git a/PairHMM_JNI/Makefile b/PairHMM_JNI/Makefile index 62dc3b0f6..b86958a6c 100644 --- a/PairHMM_JNI/Makefile +++ b/PairHMM_JNI/Makefile @@ -1,4 +1,4 @@ -OMPCFLAGS=-fopenmp +#OMPCFLAGS=-fopenmp #OMPLDFLAGS=-lgomp #CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas @@ -20,7 +20,7 @@ DEPDIR=.deps DF=$(DEPDIR)/$(*).d #Common across libJNI and sandbox -COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc +COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc sse_function_instantiations.cc #Part of libJNI LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc $(COMMON_SOURCES) SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc @@ -33,7 +33,7 @@ NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc pa #Use -xAVX for these files AVX_SOURCES=avx_function_instantiations.cc #Use -xSSE4.2 for these files -SSE_SOURCES= +SSE_SOURCES=sse_function_instantiations.cc NO_VECTOR_OBJECTS=$(NO_VECTOR_SOURCES:.cc=.o) AVX_OBJECTS=$(AVX_SOURCES:.cc=.o) diff --git a/PairHMM_JNI/avx_function_instantiations.cc b/PairHMM_JNI/avx_function_instantiations.cc index c29370561..b236ddc8f 100644 --- a/PairHMM_JNI/avx_function_instantiations.cc +++ b/PairHMM_JNI/avx_function_instantiations.cc @@ -1,5 +1,11 @@ #include "template.h" +#undef SIMD_TYPE +#undef SIMD_TYPE_SSE + +#define SIMD_TYPE avx +#define SIMD_TYPE_AVX + #include "define-float.h" #include "shift_template.c" #include "pairhmm-template-kernel.cc" diff --git a/PairHMM_JNI/avx_function_prototypes.h b/PairHMM_JNI/avx_function_prototypes.h deleted file mode 100644 index 1836a1c37..000000000 --- a/PairHMM_JNI/avx_function_prototypes.h +++ /dev/null @@ -1,19 +0,0 @@ -void GEN_INTRINSIC(_vector_shift, PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); -void GEN_INTRINSIC(_vector_shift_last, PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); -void GEN_INTRINSIC(precompute_masks_, PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); -void GEN_INTRINSIC(init_masks_for_row_, PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess); -void GEN_INTRINSIC(update_masks_for_cols_, PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt); -void GEN_INTRINSIC(computeDistVec, PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen); -template void GEN_INTRINSIC(initializeVectors, PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D); -template void GEN_INTRINSIC(stripINITIALIZATION, PRECISION)( - int stripIdx, Context ctx, testcase *tc, _256_TYPE &pGAPM, _256_TYPE &pMM, _256_TYPE &pMX, _256_TYPE &pXX, _256_TYPE &pMY, _256_TYPE &pYY, - _256_TYPE &rs, UNION_TYPE &rsN, _256_TYPE &distm, _256_TYPE &_1_distm, _256_TYPE *distm1D, _256_TYPE N_packed256, _256_TYPE *p_MM , _256_TYPE *p_GAPM , - _256_TYPE *p_MX, _256_TYPE *p_XX , _256_TYPE *p_MY, _256_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, - UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM); -_256_TYPE GEN_INTRINSIC(computeDISTM, PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, - _256_TYPE distm, _256_TYPE _1_distm); -void GEN_INTRINSIC(computeMXY, PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, - UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, - _256_TYPE pMM, _256_TYPE pGAPM, _256_TYPE pMX, _256_TYPE pXX, _256_TYPE pMY, _256_TYPE pYY, _256_TYPE distmSel); -template NUMBER GEN_INTRINSIC(compute_full_prob_avx, PRECISION) (testcase *tc, NUMBER *before_last_log = NULL); - diff --git a/PairHMM_JNI/define-double.h b/PairHMM_JNI/define-double.h index 5d31741d6..79c6b323f 100644 --- a/PairHMM_JNI/define-double.h +++ b/PairHMM_JNI/define-double.h @@ -18,34 +18,34 @@ #undef MASK_TYPE #undef MASK_ALL_ONES - #undef SET_VEC_ZERO - #undef VEC_OR - #undef VEC_ADD - #undef VEC_SUB - #undef VEC_MUL - #undef VEC_DIV - #undef VEC_BLEND - #undef VEC_BLENDV - #undef VEC_CAST_256_128 - #undef VEC_EXTRACT_128 - #undef VEC_EXTRACT_UNIT - #undef VEC_SET1_VAL128 - #undef VEC_MOVE - #undef VEC_CAST_128_256 - #undef VEC_INSERT_VAL - #undef VEC_CVT_128_256 - #undef VEC_SET1_VAL - #undef VEC_POPCVT_CHAR - #undef VEC_LDPOPCVT_CHAR - #undef VEC_CMP_EQ - #undef VEC_SET_LSE - #undef SHIFT_HAP - #undef print256b + #undef SET_VEC_ZERO(__vec) + #undef VEC_OR(__v1, __v2) + #undef VEC_ADD(__v1, __v2) + #undef VEC_SUB(__v1, __v2) + #undef VEC_MUL(__v1, __v2) + #undef VEC_DIV(__v1, __v2) + #undef VEC_BLEND(__v1, __v2, __mask) + #undef VEC_BLENDV(__v1, __v2, __maskV) + #undef VEC_CAST_256_128(__v1) + #undef VEC_EXTRACT_128(__v1, __im) + #undef VEC_EXTRACT_UNIT(__v1, __im) + #undef VEC_SET1_VAL128(__val) + #undef VEC_MOVE(__v1, __val) + #undef VEC_CAST_128_256(__v1) + #undef VEC_INSERT_VAL(__v1, __val, __pos) + #undef VEC_CVT_128_256(__v1) + #undef VEC_SET1_VAL(__val) + #undef VEC_POPCVT_CHAR(__ch) + #undef VEC_LDPOPCVT_CHAR(__addr) + #undef VEC_CMP_EQ(__v1, __v2) + #undef VEC_SET_LSE(__val) + #undef SHIFT_HAP(__v1, __val) + #undef print256b(__v1) #undef MASK_VEC - #undef VEC_SSE_TO_AVX - #undef VEC_SHIFT_LEFT_1BIT + #undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) + #undef VEC_SHIFT_LEFT_1BIT(__vs) #undef MASK_ALL_ONES - #undef COMPARE_VECS + #undef COMPARE_VECS(__v1, __v2) #undef _256_INT_TYPE #endif @@ -83,7 +83,7 @@ #define VEC_MUL(__v1, __v2) \ _mm256_mul_pd(__v1, __v2) -#define VEC_DIV(__v1, __v2) \ +#define VEC_DIV(__v1, __v2) \ _mm256_div_pd(__v1, __v2) #define VEC_BLEND(__v1, __v2, __mask) \ diff --git a/PairHMM_JNI/define-float.h b/PairHMM_JNI/define-float.h index 0e2822b2d..25ebd1489 100644 --- a/PairHMM_JNI/define-float.h +++ b/PairHMM_JNI/define-float.h @@ -84,7 +84,7 @@ #define VEC_MUL(__v1, __v2) \ _mm256_mul_ps(__v1, __v2) -#define VEC_DIV(__v1, __v2) \ +#define VEC_DIV(__v1, __v2) \ _mm256_div_ps(__v1, __v2) #define VEC_BLEND(__v1, __v2, __mask) \ @@ -133,7 +133,7 @@ _mm256_set_ps(zero, zero, zero, zero, zero, zero, zero, __val); #define SHIFT_HAP(__v1, __val) \ - _vector_shift_lasts(__v1, __val.f); + _vector_shift_lastavxs(__v1, __val.f); #define print256b(__v1) \ print256bFP(__v1) diff --git a/PairHMM_JNI/define-sse-double.h b/PairHMM_JNI/define-sse-double.h new file mode 100644 index 000000000..e48325ba9 --- /dev/null +++ b/PairHMM_JNI/define-sse-double.h @@ -0,0 +1,128 @@ +#ifdef PRECISION + #undef PRECISION + #undef MAIN_TYPE + #undef MAIN_TYPE_SIZE + #undef UNION_TYPE + #undef IF_128 + #undef IF_MAIN_TYPE + #undef SHIFT_CONST1 + #undef SHIFT_CONST2 + #undef SHIFT_CONST3 + #undef _128_TYPE + #undef _256_TYPE + #undef AVX_LENGTH + #undef MAVX_COUNT + #undef HAP_TYPE + #undef MASK_TYPE + #undef MASK_ALL_ONES + + #undef VEC_EXTRACT_UNIT(__v1, __im) + #undef VEC_INSERT_UNIT(__v1,__ins,__im) + #undef SET_VEC_ZERO(__vec) + #undef VEC_OR(__v1, __v2) + #undef VEC_ADD(__v1, __v2) + #undef VEC_SUB(__v1, __v2) + #undef VEC_MUL(__v1, __v2) + #undef VEC_DIV(__v1, __v2) + #undef VEC_BLEND(__v1, __v2, __mask) + #undef VEC_BLENDV(__v1, __v2, __maskV) + #undef VEC_CAST_256_128(__v1) + #undef VEC_EXTRACT_128(__v1, __im) + #undef VEC_EXTRACT_UNIT(__v1, __im) + #undef VEC_SET1_VAL128(__val) + #undef VEC_MOVE(__v1, __val) + #undef VEC_CAST_128_256(__v1) + #undef VEC_INSERT_VAL(__v1, __val, __pos) + #undef VEC_CVT_128_256(__v1) + #undef VEC_SET1_VAL(__val) + #undef VEC_POPCVT_CHAR(__ch) + #undef VEC_LDPOPCVT_CHAR(__addr) + #undef VEC_CMP_EQ(__v1, __v2) + #undef VEC_SET_LSE(__val) + #undef SHIFT_HAP(__v1, __val) + #undef print256b(__v1) + #undef MASK_VEC + #undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) + #undef VEC_SHIFT_LEFT_1BIT(__vs) + #undef MASK_ALL_ONES + #undef COMPARE_VECS(__v1, __v2) + #undef _256_INT_TYPE + +#endif + +#define SSE +#define PRECISION d + +#define MAIN_TYPE double +#define MAIN_TYPE_SIZE 64 +#define UNION_TYPE mix_D128 +#define IF_128 IF_128d +#define IF_MAIN_TYPE IF_64 +#define SHIFT_CONST1 1 +#define SHIFT_CONST2 8 +#define SHIFT_CONST3 0 +#define _128_TYPE __m128d +#define _256_TYPE __m128d +#define _256_INT_TYPE __m128i +#define AVX_LENGTH 2 +#define MAVX_COUNT (MROWS+3)/AVX_LENGTH +#define HAP_TYPE __m128i +#define MASK_TYPE uint64_t +#define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFFL +#define MASK_VEC MaskVec_D128 + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi64(__v1, __im) + +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi64(__v1,__ins,__im) + +#define VEC_OR(__v1, __v2) \ + _mm_or_pd(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm_add_pd(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm_sub_pd(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm_mul_pd(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm_div_pd(__v1, __v2) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm_cmpeq_pd(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm_blend_pd(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm_blendv_pd(__v1, __v2, __maskV) + +#define SHIFT_HAP(__v1, __val) \ + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) + +#define VEC_CVT_128_256(__v1) \ + _mm_cvtepi32_pd(__v1) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_pd(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm_cvtepi32_pd(_mm_set1_epi32(__ch)) + +#define VEC_SET_LSE(__val) \ + _mm_set_pd(zero, __val); + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm_cvtepi32_pd(_mm_loadu_si128((__m128i const *)__addr)) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm_castsi128_pd(_mm_set_epi64(__vsHigh, __vsLow)) + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_si64(__vs, 1) + + diff --git a/PairHMM_JNI/define-sse-float.h b/PairHMM_JNI/define-sse-float.h new file mode 100644 index 000000000..f5758c74a --- /dev/null +++ b/PairHMM_JNI/define-sse-float.h @@ -0,0 +1,127 @@ +#ifdef PRECISION + #undef PRECISION + #undef MAIN_TYPE + #undef MAIN_TYPE_SIZE + #undef UNION_TYPE + #undef IF_128 + #undef IF_MAIN_TYPE + #undef SHIFT_CONST1 + #undef SHIFT_CONST2 + #undef SHIFT_CONST3 + #undef _128_TYPE + #undef _256_TYPE + #undef AVX_LENGTH + #undef MAVX_COUNT + #undef HAP_TYPE + #undef MASK_TYPE + #undef MASK_ALL_ONES + + #undef VEC_EXTRACT_UNIT(__v1, __im) + #undef VEC_INSERT_UNIT(__v1,__ins,__im) + #undef SET_VEC_ZERO(__vec) + #undef VEC_OR(__v1, __v2) + #undef VEC_ADD(__v1, __v2) + #undef VEC_SUB(__v1, __v2) + #undef VEC_MUL(__v1, __v2) + #undef VEC_DIV(__v1, __v2) + #undef VEC_BLEND(__v1, __v2, __mask) + #undef VEC_BLENDV(__v1, __v2, __maskV) + #undef VEC_CAST_256_128(__v1) + #undef VEC_EXTRACT_128(__v1, __im) + #undef VEC_EXTRACT_UNIT(__v1, __im) + #undef VEC_SET1_VAL128(__val) + #undef VEC_MOVE(__v1, __val) + #undef VEC_CAST_128_256(__v1) + #undef VEC_INSERT_VAL(__v1, __val, __pos) + #undef VEC_CVT_128_256(__v1) + #undef VEC_SET1_VAL(__val) + #undef VEC_POPCVT_CHAR(__ch) + #undef VEC_LDPOPCVT_CHAR(__addr) + #undef VEC_CMP_EQ(__v1, __v2) + #undef VEC_SET_LSE(__val) + #undef SHIFT_HAP(__v1, __val) + #undef print256b(__v1) + #undef MASK_VEC + #undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) + #undef VEC_SHIFT_LEFT_1BIT(__vs) + #undef MASK_ALL_ONES + #undef COMPARE_VECS(__v1, __v2) + #undef _256_INT_TYPE + +#endif + +#define SSE +#define PRECISION s + +#define MAIN_TYPE float +#define MAIN_TYPE_SIZE 32 +#define UNION_TYPE mix_F128 +#define IF_128 IF_128f +#define IF_MAIN_TYPE IF_32 +#define SHIFT_CONST1 3 +#define SHIFT_CONST2 4 +#define SHIFT_CONST3 0 +#define _128_TYPE __m128 +#define _256_TYPE __m128 +#define _256_INT_TYPE __m128i +#define AVX_LENGTH 4 +#define MAVX_COUNT (MROWS+3)/AVX_LENGTH +#define HAP_TYPE UNION_TYPE +#define MASK_TYPE uint32_t +#define MASK_ALL_ONES 0xFFFFFFFF +#define MASK_VEC MaskVec_F128 + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi32(__v1, __im) + +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi32(__v1,__ins,__im) + +#define VEC_OR(__v1, __v2) \ + _mm_or_ps(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm_add_ps(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm_sub_ps(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm_mul_ps(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm_div_ps(__v1, __v2) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm_cmpeq_ps(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm_blend_ps(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm_blendv_ps(__v1, __v2, __maskV) + +#define SHIFT_HAP(__v1, __val) \ + _vector_shift_lastsses(__v1, __val.f) + +#define VEC_CVT_128_256(__v1) \ + _mm_cvtepi32_ps(__v1.i) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_ps(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm_cvtepi32_ps(_mm_set1_epi32(__ch)) + +#define VEC_SET_LSE(__val) \ + _mm_set_ps(zero, zero, zero, __val); + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm_cvtepi32_ps(_mm_loadu_si128((__m128i const *)__addr)) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm_cvtpi32x2_ps(__vsLow, __vsHigh) + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_pi32(__vs, 1) + diff --git a/PairHMM_JNI/headers.h b/PairHMM_JNI/headers.h index 42485fc51..f502ff5ce 100644 --- a/PairHMM_JNI/headers.h +++ b/PairHMM_JNI/headers.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/PairHMM_JNI/hmm_mask.cc b/PairHMM_JNI/hmm_mask.cc deleted file mode 100644 index c335a0160..000000000 --- a/PairHMM_JNI/hmm_mask.cc +++ /dev/null @@ -1,451 +0,0 @@ -#include -#include -#include -#include - - -// /usr/intel/pkgs/icc/13.0.0e/bin/icc -o ed -O3 ed.cpp -xAVX -openmp -openmp-link static - - -#include - -#include -#include -#include -#include -#include -#include - - -#include -#include -#include -#include -#include - - -#include -#include -#include -#include -#include -#include -#include "template.h" - -using namespace std ; - -#define CHECK_MASK_CORRECTNESS - -template -string getBinaryStr (T val, int numBitsToWrite) { - - ostringstream oss ; - uint64_t mask = ((T) 0x1) << (numBitsToWrite-1) ; - for (int i=numBitsToWrite-1; i >= 0; --i) { - oss << ((val & mask) >> i) ; - mask >>= 1 ; - } - return oss.str() ; -} - -int normalize(char c) -{ - return ((int) (c - 33)); -} - - -uint8_t ConvertChar::conversionTable[255] ; - -int read_testcase(testcase *tc, FILE* ifp) -{ - char *q, *i, *d, *c, *line = NULL; - int _q, _i, _d, _c; - int x, size = 0; - ssize_t read; - - read = getline(&line, (size_t *) &size, ifp == 0 ? stdin : ifp); - if (read == -1) - return -1; - - - tc->hap = (char *) malloc(size); - tc->rs = (char *) malloc(size); - q = (char *) malloc(size); - i = (char *) malloc(size); - d = (char *) malloc(size); - c = (char *) malloc(size); - - if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) - return -1; - - tc->haplen = strlen(tc->hap); - tc->rslen = strlen(tc->rs); - assert(tc->rslen < MROWS); - tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); - tc->irs = (int *) malloc(tc->rslen*sizeof(int)); - - //tc->q = (int *) malloc(sizeof(int) * tc->rslen); - //tc->i = (int *) malloc(sizeof(int) * tc->rslen); - //tc->d = (int *) malloc(sizeof(int) * tc->rslen); - //tc->c = (int *) malloc(sizeof(int) * tc->rslen); - - for (x = 0; x < tc->rslen; x++) - { - _q = normalize(q[x]); - _i = normalize(i[x]); - _d = normalize(d[x]); - _c = normalize(c[x]); - tc->q[x] = (_q < 6) ? 6 : _q; - tc->i[x] = _i; - tc->d[x] = _d; - tc->c[x] = _c; - tc->irs[x] = tc->rs[x]; - } - for (x = 0; x < tc->haplen; x++) - tc->ihap[x] = tc->hap[x]; - - - free(q); - free(i); - free(d); - free(c); - free(line); - - return 0; -} - - - - -#define ALIGN __attribute__ ((aligned(32))) - - -typedef union ALIGN { - //__m256i vi; - __m128 vf ; - __m128i vi ; - __m128d vd; - - uint8_t b[16]; - //uint16_t hw[8] ; - uint32_t w[4] ; - uint64_t dw[2] ; - - float f[4] ; - //double d[2] ; - -} v128 ; - -typedef union ALIGN { - __m256i vi; - __m256 vf ; - __m256d vd; - //__m128i vi128[2] ; - - uint8_t b[32]; - //uint16_t hw[16] ; - uint32_t w[8] ; - uint64_t dw[4] ; - - float f[8] ; - double d[4] ; - -} v256 ; - - -#define NUM_DISTINCT_CHARS 5 -#define AMBIG_CHAR 4 - -#define VEC_ENTRY_CNT 8 -#define VEC_LEN 256 -#define VTYPE vf -#define VTYPEI vi -#define VENTRY f -#define VENTRYI w -#define MTYPE uint32_t -#define MASK_ALL_ONES 0xFFFFFFFF - - -#define VECTOR v256 -#define VECTOR_SSE v128 - -#define SET_VEC_ZERO(__vec) \ - __vec= _mm256_setzero_ps() - -#define SET_VEC_ONES(__vec) \ - __vec = _mm256_set1_epi32(0xFFFFFFFF) - -#define VEC_OR(__v1, __v2) \ - _mm256_or_ps(__v1, __v2) - -#define VEC_ADD(__v1, __v2) \ - _mm256_add_ps(__v1, __v2) - -#define VEC_MUL(__v1, __v2) \ - _mm256_mul_ps(__v1, __v2) - -#define VEC_BLENDV(__v1, __v2, __maskV) \ - _mm256_blendv_ps(__v1, __v2, __maskV) - -#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ - __vdst = _mm256_castps128_ps256(__vsLow) ; \ - __vdst = _mm256_insertf128_ps(__vdst, __vsHigh, 1) ; - -#define VECS_SHIFT_LEFT_1BIT(__vs) \ - __vs = _mm_slli_epi32(__vs, 1) - - -#define VECS_SHIFT_RIGHT_WHOLE(__vs, __cnt) { \ - uint64_t __mask = ; \ - uint64_t __shiftWord = ((((uint64_t) 0x1) << __cnt) - 1 ) & __vs.dw[1] ; \ - __shiftWord <<= (64-cnt) ; \ - __vs = _mm_slri_epi64(__vs, __cnt) ; \ - __vs.dw[0] |= __shiftWord ; \ - } - - -#define GET_MASK_WORD_OLD(__mask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ - MTYPE __bitMask = (((MTYPE)0x1) << __shiftBy) - 1 ; \ - MTYPE __nextShiftOut = (__mask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ - __mask >>= __shiftBy ; \ - __mask |= __lastShiftOut ; \ - __lastShiftOut = __nextShiftOut ; \ - } - - -#define SET_MASK_WORD(__dstMask, __srcMask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ - MTYPE __bitMask = (((MTYPE)0x1) << __shiftBy) - 1 ; \ - MTYPE __nextShiftOut = (__srcMask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ - __dstMask = (__srcMask >> __shiftBy) | __lastShiftOut ; \ - __lastShiftOut = __nextShiftOut ; \ -} - - -void precompute_masks_avx(const testcase& tc, int COLS, int numMaskVecs, - MTYPE (*maskArr)[NUM_DISTINCT_CHARS]) { - - const int maskBitCnt = VEC_LEN / VEC_ENTRY_CNT ; - - for (int vi=0; vi < numMaskVecs; ++vi) { - for (int rs=0; rs < NUM_DISTINCT_CHARS; ++rs) { - maskArr[vi][rs] = 0 ; - } - maskArr[vi][AMBIG_CHAR] = MASK_ALL_ONES ; - } - - for (int col=1; col < COLS; ++col) { - int mIndex = (col-1) / maskBitCnt ; - int mOffset = (col-1) % maskBitCnt ; - MTYPE bitMask = ((MTYPE)0x1) << (maskBitCnt-1-mOffset) ; - - char hapChar = tc.hap[col-1] ; - - if (hapChar == AMBIG_CHAR) { - maskArr[mIndex][0] |= bitMask ; - maskArr[mIndex][1] |= bitMask ; - maskArr[mIndex][2] |= bitMask ; - maskArr[mIndex][3] |= bitMask ; - } - - //cout << hapChar << " " << mIndex << " " << getBinaryStr(bitMask, 32) - // << endl ; - //cout << getBinaryStr(maskArr[0][hapChar],32) << endl ; - //exit(0) ; - - maskArr[mIndex][ConvertChar::get(hapChar)] |= bitMask ; - - - // bit corresponding to col 1 will be the MSB of the mask 0 - // bit corresponding to col 2 will be the MSB-1 of the mask 0 - // ... - // bit corresponding to col 32 will be the LSB of the mask 0 - // bit corresponding to col 33 will be the MSB of the mask 1 - // ... - } - -} - - -void test_mask_computations (testcase& tc, int tcID, bool printDebug=false) { - - int ROWS = tc.rslen + 1 ; - int COLS = tc.haplen + 1 ; - - // only for testing - VECTOR mismatchData, matchData ; - //SET_VEC_ZERO(mismatchData.VTYPE) ; - //SET_VEC_ONES(matchData.VTYPEI) ; - for (int ei=0; ei < VEC_ENTRY_CNT; ++ei) { - matchData.VENTRY[ei] = 1.0 ; - mismatchData.VENTRY[ei] = 0.0 ; - } - - const int maskBitCnt = VEC_LEN / VEC_ENTRY_CNT ; - const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function - - MTYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; - precompute_masks_avx(tc, COLS, numMaskVecs, maskArr) ; - -#ifdef DEBUG - if (printDebug) { - cout << "The first 32 hap chars are: " ; - for (int i=0; i < 32; ++i) { - cout << tc.hap[i] ; - } - cout << endl ; - - cout << "Masks computed for A, C, T, G, N are: " << endl ; - cout << getBinaryStr(maskArr[0][0], 32) << endl ; - cout << getBinaryStr(maskArr[0][1], 32) << endl ; - cout << getBinaryStr(maskArr[0][2], 32) << endl ; - cout << getBinaryStr(maskArr[0][3], 32) << endl ; - cout << getBinaryStr(maskArr[0][4], 32) << endl ; - } -#endif // #ifdef DEBUG - - int beginRowIndex = 1 ; - while (beginRowIndex < ROWS) { - - int numRowsToProcess = min(VEC_ENTRY_CNT, ROWS - beginRowIndex) ; - - char rsArr[VEC_ENTRY_CNT] ; - for (int ri=0; ri < numRowsToProcess; ++ri) { - rsArr[ri] = ConvertChar::get(tc.rs[ri+beginRowIndex-1]) ; - } - - // Since there are no shift intrinsics in AVX, keep the masks in 2 SSE vectors - VECTOR_SSE currMaskVecLow ; // corresponding to entries 0-3 - VECTOR_SSE currMaskVecHigh ; // corresponding to entries 4-7 - - MTYPE lastMaskShiftOut[VEC_ENTRY_CNT] ; - for (int ei=0; ei < VEC_ENTRY_CNT; ++ei) - lastMaskShiftOut[ei] = 0 ; - - int col = 1 ; - int diag = 1 ; - for (int maskIndex=0; maskIndex < numMaskVecs; ++maskIndex) { - // set up the mask vectors for the next maskBitCnt columns - - // For AVX, maskBitCnt = 32 (so, the operation below is amortized over 32 cols) - for (int ei=0; ei < VEC_ENTRY_CNT/2; ++ei) { - SET_MASK_WORD(currMaskVecLow.VENTRYI[ei], maskArr[maskIndex][rsArr[ei]], - lastMaskShiftOut[ei], ei, maskBitCnt) ; - - int ei2 = ei + VEC_ENTRY_CNT/2 ; // the second entry index - SET_MASK_WORD(currMaskVecHigh.VENTRYI[ei], maskArr[maskIndex][rsArr[ei2]], - lastMaskShiftOut[ei2], ei2, maskBitCnt) ; - } - -#ifdef DEBUG - if (printDebug && maskIndex == 0) { - cout << "The masks for entry 1: " << endl - << getBinaryStr(maskArr[0][rsArr[1]], 32) << endl - << getBinaryStr(currMaskVecLow.VENTRYI[1], 32) << endl ; - - } -#endif // #ifdef DEBUG - - // iterate over mask bit indices and columns - for (int mbi=0; mbi < maskBitCnt && diag < COLS + ROWS -2; ++mbi, ++diag) { - - VECTOR maskV ; - VEC_SSE_TO_AVX(currMaskVecLow.VTYPE, currMaskVecHigh.VTYPE, maskV.VTYPE) ; - - VECTOR testData ; - testData.VTYPE = VEC_BLENDV(mismatchData.VTYPE, matchData.VTYPE, - maskV.VTYPE) ; - - VECS_SHIFT_LEFT_1BIT(currMaskVecLow.VTYPEI) ; - VECS_SHIFT_LEFT_1BIT(currMaskVecHigh.VTYPEI) ; - -#ifdef DEBUG - if (printDebug && maskIndex == 0) { - cout << "The mask for entry 1, mbi=" << mbi << ": " - << getBinaryStr(maskV.VENTRYI[1], 32) << endl ; - - } -#endif // #ifdef DEBUG - -#ifdef CHECK_MASK_CORRECTNESS - - int firstRowIndex = (diag < COLS) ? 0 : (diag - COLS) ; - int lastRowIndex = min(col-1, numRowsToProcess-1) ; - - for (int ri=firstRowIndex; ri <= lastRowIndex; ++ri) { - int currRow = beginRowIndex + ri ; - int currCol = col - ri + firstRowIndex ; - - char hapChar = tc.hap[currCol-1] ; - char rsChar = tc.rs[currRow-1] ; - - bool match = (hapChar == rsChar || hapChar == 'N' || rsChar == 'N') ; - - if ((bool) testData.VENTRYI[ri] != match) { - cout << "Error: Incorrect mask for tc " << tcID << ", diag = " << diag - << " (" << currRow << ", " << currCol << ")" << endl ; - - cout << "The chars are: " << hapChar << " and " << rsChar << endl ; - cout << "The selected value is: " << testData.VENTRYI[ri] << endl ; - - exit(0) ; - } - } -#endif // #ifdef CHECK_MASK_CORRECTNESS - - if (diag < COLS) - ++col ; - - } // mbi - } // maskIndex - - beginRowIndex += VEC_ENTRY_CNT ; - } // end of stripe - - //cout << "Finished validating entry " << endl ; -} - -#ifdef HMM_MASK_MAIN -int main () { - - #define BATCH_SIZE 10000 - - ConvertChar::init() ; - - testcase* tcBatch = new testcase[BATCH_SIZE] ; - - int numBatches = 0 ; - int numRead = 0 ; - - const int DEBUG_TC = -1 ; - - FILE* ifp = stdin ; - - do { - numRead = 0 ; - - while (numRead < BATCH_SIZE) { - if (read_testcase(tcBatch+numRead, ifp) < 0) - break ; - - ++numRead ; - } - - if (numRead == 0) - break ; - - for (int ti=0; ti < numRead; ++ti) { - - int tcID = numBatches * BATCH_SIZE + ti ; - test_mask_computations(tcBatch[ti], tcID, tcID == DEBUG_TC) ; - } - - ++numBatches ; - } while (numRead == BATCH_SIZE) ; - - fclose(ifp) ; - - delete[] tcBatch ; - - return 0 ; -} -#endif diff --git a/PairHMM_JNI/libJNILoglessPairHMM.so b/PairHMM_JNI/libJNILoglessPairHMM.so deleted file mode 100755 index 9b6b8e6930e9764b187f019979a525dd50840252..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96945 zcmb@v3t$x0^*_F014acmSfEj{tnF$eQA`Mt$ATu1$V`}p6oOKXE+GjL$jfBILlJ=_ z$aGmtEn2nIqF?&0_3>@0*e_CQ6G(We5I_{cN5I!CQHfX}fRO*^bMMS%5|g(6zrWGR z%sJ=YbI(2Z+;h);%<@o%c3ZEgD2H`99k)0L&73GPsV2cl>`0U980F~WNOk%IauNOe(TWIE6Xy&t?ZYIEG zJ!2&3h{lumYAMclo*pxu^<-LJS)T^-#d|L^W0z-W+a-)SXpV#3z)O^3b3+;9r zJO=O6ao>sCjhilteb$HDx-K_gCRqqPC)nQk`g7br!<~-XbR&*|c<=5i1yaSm5O)^t zOx*dni*cvoz6*CU?ygrE5KD3Y*hUzQ_qjIMdcDPb`30Uq+|?%hD?B~8vvEI!dopfb zMYwOmJrnn8+{1C(uV(?Qu_3&+_wHVu=<#K@cBWi_L;A7fQb#J8;i9g2{SXaHeCz|J(jA|87-V<&!oKwJS2&j5bd6xpF4+kG z&-$h+5)n&hge%X|!&V8$M$6}Amy!Raj&*ZPynHyeK`y~RQSwLi#Wlhy+ojnwyjH-x zTEL&}wbYPst$a92(O#n6%Ot{T$3PEydjK7W>DR6AoPIsZp*#+=zNWYisM+C`Njv?e zpXm-?hI)6W&ol_u9sa6Ir94g5lD%0CM~T^vWk)1F)^v*Zh;~G#MkbcGVsK0@xwjrL0T4 z427d7zAhjOtTW5)eX+#1R!D@^?)y<+rq41JY2km_gZ=-ehyM6{5BS@Kotr|~9#_M! zbXTuOd(gv&u*2@s*P89yx>_=?GyB)(9`trC@T~9Zn$CLo%IrsVg%aOn%KDV4XLlDp zPlEsKPM*8W{<73e?>6~d=s^#E0h8|dtnWeo8%+MOUF7+whx9wl{*v0IzFW{==x4HK zO98XYe(_MW)K9JHPb_;FZQ`5Ej-G1P>x`)%M^|~^2)e7+f*$(I3q9CdiRlk(&2lYy zc7wUF+h?U9wJ48Qs%a0+Aql(X!%^FVT`|V`v6;sH*kqPl*QMOd9{l7Pv)ouSpv*!) z=k-_7?(m;mB%Er>IRoioLf^T{%>LWdMgOm(zWovLzBud}kq6Bot`t-MiKhNjP5k3M z=<@~WpMHB~mvQ7bkc;|s2PFfy$>*0n)b|wV-Lc0?KlhiXTMwhj7(?MHwpGB^ke$%b&~8@)tw=a z)Q8tX2-%&S=X;R=AouOSDWQpcJgr#e10SRn3cQZdC=7V>Mr_k2gB~@uk+}zp00MEfaQPCbB(>!Mr>E8D{*G7?z2f(B8a$--F-V z)kD4R>Y-n3HQVbs({D)bSSt~BXh9v%znlGiW|_pTHsjIz4b!5-Qi2kc%ZdJ^0nmIDaKj!yDsC)B-FP% zK8t!NcW4joYnXa#GVQQT7Vj8u>djgdvE=y}{(=3pX}wIi)J$L7L;s?AbSLK}NKZbF z6Ouub$%oTM;u95#u*yy7!Tw`=@H4*?=`rtH-I>oLJ?P=I>EGNtWcoV!a0H=G%9B+i z5veBrsvi8(xE|VNLl1WT&mQ{Ao*vR~Htl3x7e7NiUEa;SJG}>a`t^W!`2F`5mzMY| z0=eY@zu)06EGY~){Il@@DQ{seUvrBJ?=NupGw1uK7TjA{5hy61P?TFyQBdLV-NXDNkyPAP#Guy-@=l6{gr{jq6&XmZejVH;$na1WK}D@ zx2T|^A`2MB=kw1kDV$b0YeH#p8S+dm$j#RZ=M@wc&M7U;pIA7*phU?nD=G~vDl0&~ zq(qcDe0ZKr?4Mm(ROBx!FP-Jjoxd<&&Ckg8XN?LJR0Q&JD+;ovCHfsj`2`EhkZ1C= zTs1FaLQ=+j0aV~EYf|3SNdhfUfuvEhaw`h+Aayy!#@~Y6Vl{7CASo$tPHwqBP@Y>D zr~ng@g{(kI_N%28{wZYzCB>!r&=SfaYgtHOv}9OVifrHIlaZ2FRFGRvHp2xQunOb` z$OWjxioC*tl0f0?!n~=|0;9mg4{oxE(xQrttlR8pR5imtzufm ztXWb^CRw(N1g4iPC@jf`nN?I4qoRIV8SHX)Zefw*UjKL7JioMZR#8E>q}fHKxq<(? zU7AMOVURR>XyWGi1$lwea&=l@1npYQN=ow2owX=XkXBxvyC^G{HUjNsWZYa{P=W4~ zQ4EM#N5cFwO6L2ACrvB}Oqn$oY($ofqJmQcsp-fuI#6m_w>mdtT17TRnieQ4>drc)iNFek zNze$gCQVEBqsQF*{j!N1A}?j0G_4{=ylF_YAsS~1ts?e=5(YJwHjTVJFh@^oo-skY&O%{g_*e z*>rS((NhcH{otklU8B$ESR|UJbeojX z@I*gVTe`rHqR_|u1?AftC9J*&8Mz8p|F0^1A!PLA&*`FAsaN7BR| zB>fLPMGCqmyXZ(f(^;4Q-yJ+@1e7?t5K$DOHVn2IlSd;aEU85Bw=ij>-(ON%UJQR$ zh^UWkrDnk`OdEw_i>-mc-XZ?oXqY>VhD}Exg3DQmCri*x;caCN&*esY`-YOPh4eBY4d=`ARuqUC> zNw(s?M^{HBeEz&S1$pz>X)=m??ky32`mgk6Z|m=4g*M^;WA@*%(~&*e>A#74OshmQ z<&{=q__}j)QqqLdlKC*P2?(Z>Qt|{)QCM1%or@sOj4&f%?g$k#k&ZyRGhTX1VL}L( z7jnvSXVT4u6`Tb4D=N#%O3MQU`OqEk#DIL=w~DDP4Uat>S2aaq0av9 z3c;=fJRVD}sl^D*Bo zcEHnOOmtLKgeEB4gO{F z5v7QP7zm+u-Augyvoc4b!33X(f9Pgvj{4NnkSp^d&f6<(=gLzbR!xEA%G=i4{JEd z1*NmiRw*2bo?KK|f>1OMvkQdJH&@InEL%{4t`ix+gp^c6H|sZ03p6JNliYG}kuprH zK-grxN1D>CUqmYrLZk)1ch!;8th+V$knx(Fh*0e|qxg4`8oUDouw--c*^`-luZYI7 zO?3V)k$;5<{QrGxjo?~Rq!V?H`7Mk29&;2cOILaFx}S$wZ5D~xyJ*E?7m=&5xU5LF zu)kt{F;+TD7mR=bC5E7= zaxgp*!x|PBL|#mm{hZ?2d1U2f6&JFj4o|KWL%1z%9wv)eCYd_ghwPX&`O~rDAc|xz zmTL(+dsO$*R(wZV&ON&o%AOSL=l)BfDr`dCW(qZzxyXL*mAT3B9kulZVx=dhbaN~}(&SZM?Kc?k&-(!F_D{34Vj z5|#vt@(866DwC^5gcAu1ieQ+};)X%W0z^}}Q1Gk@L>*kT2dX6BF>%5Ke=;&n)YS14 z{KJP02gBPje5q3=J0{GSkv3lSCk;!M#mNk*N<#}ADE){!Y4u=~x$+~V4 zHof{dc9{Ezr2p1Nue`DlJ8#jBCL2B5MlV_NN9;&B9q&izy&N-5I@mk*|9KvEc6vKL zF>!(GAH@z?FUN5+eWH2J9!PN|7RcwQF8G#@mA2R}@YuSK zT+i9?)?O=#wKhDqIwRLc8=mLrt*g$4x6b~Pw!?u1+_D#>I2*pgN{px5 zhF@yKuQS^%3b?0i_*64JwujMsJ1)hBM@NrbSvEY|-nwSk@Z@D(Gi`XwhDe)Z!^06p zt}+`Q4lHslwBd17A#yFX;c>Jra;>!Cu|*%bR@?CQ^8!!U@Yte{TFv58Kt$u_)V!8Nu>umVjZ20GF_=z@rtqrf(@EdLT zOdGz=hSzNP9X7n(4>#HHQ*7xE+VEL6e6tPzGaKHp;iuW~tv3908@|nkzuks+T-#y) z{65FJVr+PR_hVhLHhhkS#52x@zuSg)+wk|;@bNah--b`L;b+?LqipzG8$Q*BpJl@< zHhi89pJl`6+we1Ncr(-!)R{K?Y+L#{HvGLde3=bD$A({M!x!4{OKo`bsFI+rwBhI3 z(yzARi){EOZ1`duew__(AAg^-;Y)4lYi;;48-Ak=f1eFsXTt|<_#HNUr48R?!_T+j z58ChxZ1`pyp5KC5mtn&%vXFST+VBt9@NG8yVjJFZU5EWYXv4?Y@Jnp?SR4K!8$Qm4 zUuMI*ZTRIje7p@`Wy2@h@HIC4C>wr-4WDYmue9M68(z2JvuyaF4L`$%UuDD3wBbWG z{2Uwp5gWeDhX17vztDzXZNo3M;eTbrue9MGwc%IW@V~auh+#hJVI}Z?)l{ zwc*=r`2Vuu9p*fe@y~i2KE{UsgAE^R!~fBSkF(*Qv*DGhFJg?(!QEIMgFg?xf={oj z8e+D#r#$jeA@!cM}q0$xry zn(#US-%q$V;ne~zC(MvPyi~w*3A5VaG6Byb3>h8anF79(Fhl!rmVl=c#+rg7oGRc< z!VK}li2@!^n4x{xE#NVP`w@;6@NmKm@xu-Q4<+m(+*a^t$=$G<`gu%PQYi60_Id8yjs8~2y+S$UMk>Ygc@3A+WniSX5gV+H&&VNMak4go(;m{Wsr z>v`7yS;B6@%>sUkFr9O_Nx+X1{xRV?0sn&VPYBlvcsXGY;dKJOpYXMWR|~kD@O6Zj z3V1HzpAs$;@GQddgl7u)PQup{&Jyrc!kh|(Qw5w!m{WjoqJYN}zJah?z+(vCNH|u& z!wGY05OxT7DB+t3w|*NX*KTp_Cxb=)^f5I~fHw*YF z!nuT-1pFxBS%m8Z{0qW)glh%7oNzwjbppPhZ~@`f0xl;!oA6Qr&n0{>;W7cwBFq6O zJX64T5-ud1CE%%q=Mqj8a3k2yi9g)dD_2m?3v~seq3W zUO>1^z()x$Bs^2VhX^ksoF(9WgzqPuD&Sp&A0V6v7*BC=yjOy~ryq7Zl$yFgul6P} z+^@E-=}X6r`rM5mK>su?8{aM{RpS(-bp-k

cZ-x*bL@^bRF-^(%PSLf5Xx1N{0S zk9PgXO8q&fQq>Zrj6PW5b`3#qSE}lw6utMCpSc}r?E@N+^eA3IFaLB~@7;Kfx+^Vb z=hoS-A=RdQ<7R@TV>$!~T#4kb971wPdKt=}2+Gd(X2v5sw@Qh(S{YqKmXbkQcG~o` z+tcotHeFfv1=rM7{eq^i^0a|xn*Oe)S9=`5YWg<4uSW^3@m#?)!9Lrn#hasjb!#6; zjbyQspy{WT;0BT3O5Twsrc;`kfI6p9^+xbES#YrIN^a;Nmwkt<*mE!JhP+j&+AQRB zxT*~&xC5j}05<{j)qLryepI}C;;Np@1jLLNFNa;#SK=jYS^GbukRkL{FZ{?xv?rsc zT?wucOqJj&!7W|i>{*C3(8*#v6n%|n2ei2fve6QqrFaFwPq>O zrFpjlGh%?gVNKS>DB<8{tKe~_+MuW3a*qi-p1Og^QAWFf{hIG+=) z{?(pVusz|5?fr`v9FBi{3sH@Ok`#@0+w6(E2AJzGQYoR7hmnN1W@Mp;1?>>5{&nyN zX0a#6@S;;Qje#`A(*bQhSHr24I>QpN-*VqS~Rl68FRp0)we?kDUGoJ2)k_2 z9wT3%Vpj=qH67%f9j^F#qc2Nf!AfX@Ec5=;-IQ6$GKG8z=vJGcwn#z}a(t>pZm&{E1}LQ9)HZNP0u!YHAsH8RbU!YT${@)uci2WpHou`&%jKKZ-~K0x-B z$ju%mGMc}J7&UdS>bp^D)nZRyM_?cmfw|D3Nc3Ce1T9w5IByg*!Sx|Y8z5<%6AGG; zJP$NcW!UE$p-00Fl%j7{YT&cq7)ZTBBVQSRf*p;c+joT?2Ua2WL!$zYCtW|1*507h zABk3iYef@ect3;DU!myPSH!rMufYT_d<>4kv<|BweR@NBFe;;_&a#z%+4$0YXe@DV zXg@g8wc;rBl#%>pTKkT4y~vY=F7)QRZZu4~Cp%p~Mcj-{KS6DwtsJ4!s)p8JT=LGe z*9axo>4QCn@NZQOvER622R{7=Y9`w=13RcMd>R%c z@NvL{YP^KvT=6>u1YCd36K^~T91wWrAf^T{gAJv7^F0daecy8}k3$q3UJujJg3<+O zh>M;>J++`T1uZE30gaBXLIHjUpUWEPgG`@3?zMN^4r4Hk7iJM7yH7uMTnEw^`K?yB z5jfFZvLp`OMc_m?84H}{uZ(sA93N)WwTkg!-2MY7<*%qWOKEu0a9q}~x*4OhOz=5}WztZ~m|KzLFHE9;NNEn< z0=OPt4Oy53K8@o}Dveb9QTUc+OlFVH+hB@>1aN6E^LPcqia&{b8)aNa)?g5C3`P}` zzeIe;S~y%QzQ#%z{D6Lp!Kne9@jQx!^&@01+atUtT&WU_HTId_BiMU}4WAWjye0_M zo{eCLTx0CiID`I+VQ{Ua4z^R>(!YZGoTOgZ-BDszjNwOKh(bSo7>N!d5gzZ$M>8I- z>Sy5*b-1|Ao|S-&x4vXWQ;t2uCfe*-4cvCL7#`?*Kw;p()A&rZ`oKHPc<%||kc^CF zvJE1q(GG6Fvs{Va>|s*lr(d8HI&QdS2Xx5Vk0$wen`yv%Bc9O2B3CC(Y^98$l=T?QEny%Z^K{b0 z`|wptXrNC-2@!>{1WOa58YjUG*(rdf2_gK|Ev6>UT_!Za1hN+54S>}Q#y+rsv}7$) zh&MBZD5O@Vpo}twcr#Ooc6$UVgeK~Mg(e#0V+YfKcP*aKL>6SR*~Dvw~T_o8CY1&7?4R{J5vzwZ@IBaU-Pkn_Zm_XJ-yaBMd z8n1%|q+J(DA>PasLKACc3d$&Xh&MBZ(8T>nkzD7ip@UwCdoi@J6bW3_m!N!G?f*i# zLMfMZu04}>s{Jn57;5kRV~3mwGdN>~+-B8311lgoo9(Dff5WWzqnD!Ii#-cvShf&p zIugB!oS-e1w8a)p+COOsorx6QM7#l@zC>LgK?)JIeQ11xYNNJg5{$%nW08VBeHXHRgLP-cMaD)5PHY4ZJT~>?xCr2Tuc%f{Xs-Mg3egb0sn&?HNcbDM=j^A?imF zN?VLz#5_XI#t7vfpd{B}w!k{gaIF}LHdKOmW%cTt5$Vdw2FY`j;D}g5M_ymy`(o4v zzoi%?6$Ex5ZWd!bLcOw@eOfR(DQ3JrV2FkgfoUh`>CR{kYacQrnq-n*eQB1+EmQdP zHBJR^npgsHtylw%X26sS-((iXhZGFUY}pJ9HBX#^5YZikwq~zp%*0e_?R7rq*SEP| zA3SKHp7S`ONQpO*{U_>cA3t~zl+hV~Iq}b7#OQ*LP4{jmY2gnc3RYB}5b-b7gCMTG zF3r15s%`mB#&7y@a~>|WOhIFexu18Yuw_IQjI1sc+8!x!#?6ufgEbjuBeuMTNEDs= zI=fl&K1?-3i;EqTg3)c0^z=4VRr91y3XMHu-0=xb^dqWgrm=}8Nm6^G_eBxrWCjPH zQFN!GPs5zWg&AQNOFjra8qUv5GZzE<4;r^Y9&=F&>9`^VLpD+=JyRNg zWdo9h@uKa_3RyuAM7XhzBcQ&YG229oJ*I!YbkM4`?YqmL+bD+*mDk}rBY z9djf$&I#2u^wSe*uA%Q_xc>Yl?o%Kz)<5AI+DQC|K;!)_Cj1hUzTj5ah`w9ZcdGgu zs=iy%_gLjB`Wy6ybTg_B39H4bi>mM9(k-VYs{W>mM7&kzYGf2*rN@Lcp5b}OL&ucY^*+ptSF(W11$=EQ=&y=@2ci1K%|7f zumv9NDB(SpFt($Fx(-%C*kzJcLkC)vFjt~?5RFX=dkIrm0#7lRbvVru4xv@eI#hSC z5+w|gtk!j)MF}wyy`E@Sbu)JPBX#%^O%UEqjI@%093vK5C5V_%tUU(lA}q!Ns@0M`OlL4Pe=mhpFR%qFiGx5@@h$5^r#?jBl$*2{tERm?vv#CGWQ=z zzCn`rkUZ5a8-;@TN|F;Kxt8Q)N&WfN%HlQTu$=I_eH%+ zBsqcPcuAf{@;j0|5fK)0_a*s9l01Us|CQvGlKkakAYU!X14({Dl6Ok-ZjuK}@~K~f zyhxHS0Upf%O!8Hd{4U9NO7bL0UQP0qlDv^*FUita<}|t;#w7GJ>6o}&Q~V*?e2Lg8 z9ru!~++=b%3)4fR&L@X6heh2pWwYEw@)=3KgXBY!+#<=bB%hSzQ6&EjWY-YvcfxFv zX{|DS%1`l*c*OHJJc8qD{tn7aiW(gFbG$1U!}WI*{asaWQuTdUD$PhfjK!ZDLCfHt zKxU$*tE19a&PdyqIw5LX>O|+Z)XCA?QnPz$`s7qipQ8A5CCjI0&dAVn5VfVFPgU0i z#%RH7Rq)VsERUtU0b4OLWvPcZQ;?LwEIiozRRQ(8PKeS=(JyNmg`BQP&byg2R_1(0 z<~%AMy`6$VzZVJV%!#6m`A{=zc`uQnfr*<$QTG8#%L&hgf@ZIl`@yPy07Imze=D{` z`(owWhi!-IPtcxfFxmrF8Yi3sXTxmJJa^%NOy#tl2mjlx)Yxww_?k>zB2&9Nr@kBp6OYu2K0e;sk_(Na-$MIV!6r8ymc}?Q zgcck2o-pMWY3cbSzbnZa$u3D=Bgq*gb0C4JHVMl8>(f`DIBCO0ofmx9JT@ZX)^Dl3etN$i0(f4lyYEFC+)dvL*R>lDA9pqa;rU z*~N1V)-eO_R^rS-DiBMN&e8S!L035K4xG5at_1cd5zia1f}dEIB`||_j+i~oFPFwK zLyR%J2aX!;4ki2bWFK<;o4kRd8wi| zDbBryDmZZ#p>H)_e1`?M8|&W%dnd}njv9;02aJ6_ML#5Ak4T4wV-gq8c|$<4Eo+nU zA5J@^7Tymxj^?R^l z(-3FdDEHxzNWw?1>Rm8KHF#UBnozIQw?!-7t*+&bz+ny)Sd8YyB&`0}t^a^B9Ex*4 zmge`K<-9vasoAFli#)OOQ#4EooNa-NGxZTMnR-zSN@{NnT#~5=Vqyu$Ded(*&f`|R zyUO=tBgH3pg(kKu-h=Z8D)pa7YtB7t)tT5pjH=()s_KPptPe$3o$vX)2J!d6^Wnxb za202RSS7)1Q`M)#$7Nv3)fkI}mB+hOQrXxpFPECI9mTmG4x$A#?>SX}-{+e6xzG8z z&-;1#;Er^fw|PN7U(F|#mt*P}o&04>yt+BU0~q0#aeiaKAc5~=pMDG@bHck~ zBCdwYT}u50r&9mb*vwF1kPGqwj#l+=)V(c`Y5vbt{kYG0HgGu{`WRJ@9;ybj$3WxR zL!n+Y=m0gCJ_Z_3ABrI*8^VFYrd1)K>V2cUP6-W=O|iG;V3cZvleU+8Bp zwf?MAtv@z)N~q`()N2D9dxQ-Rni!LbNFZkZBX+$%=Ymq5E0 zS_AzFBKc@O<9zy}7-5~;+-g-l?nqTeM5>~N#$KX@Cb#>%Z&h5cI1j4a9>>dzb-o~(f+|U}PI=}Ei097xE?P{Q*agRKV@#17t zm+=Dr14DusK$b!c=#HlX=T>8#3HoAfB_+JY zg)Zk_rT#1!pW#}U<~{0q_;w_PqmGEgRQi6Ls_d(J-*>I}BMeA2SF}6QU=i(f3w%05 zSn^BwhAdePte_R{qnO4AY%L`0$aVTXrU!qlnRQ5^6rOzo5xtpBb)HwE?v62jPxE(v zpm|#`^@j}Pjg`PGu_JE$We=#!8d(lfR2~g4W0HiAHN81-6I&Jb-q<BTX|AXWns`MehbKSDTTlOjQ|+|F#Ef@S9>Q8tcS z@mTnOuZwXgr$53lr#LTY-p^eR=R@N@Zv#$(sIG}ksqZqoFz=!{Y? zJhjuI%GtY7KdDNrOImu-!yP3k!Ah9sZfO|Qfb%_?hcu%1l*0WiNZB6kK^tJdTy%F| zOhQy2oQYO+JNNMTX#ms7M3*IXl|L z#^rB;CMHo2I($yVGa2YY4|d}qi83uN{G+0BtzfZdlyK7{JeV|Vdcc$Fsy>8c zXG(Aid|G7XJ^_B1tM_byGt#4-a6SC>MGQ+9zy*Ax25(Xlur7a&>+-Ie)u1p^Kq!jA zSV!&A0E+i3RyW>NT}$IgILQNuIItwnhbf_Z5c}YSd40=Roast-VbTC@JioQq@YU4K z&r;Dto$mxNdF?w)(fjtnR?{$8qt*wZ&sl^%2-t>UtkL?Qk+osiHr4td^npSUH=y4m z^yw>uv-+dGwNPK|rrd`h?VF107!IKOE0XyDFdxtdnjVIF<@7?_g?{K-aR*$GrW=z&gXcpZPE9|eBy2@L6qc)b4OewN zw4H$yUa(cH*mI_ksMPGl!UXU>{d97jmT+!Tu;`}NNeM+a!NSf%)+n5jRP;0mm=^1* zxrptB9T6xSiziM4(|UA@x+%e&n~X`p#W%HT-j5W0Zmg^NCFosEIEO=g=El#0NW6&% z?G{%u<|StkKJQlceuZF$IV4rIGh@{D?U;1I1zvD1|A=byQz=yqifxKX z*OdEm3IdbjOcH~E)(DHxJTb27PuU)babdh_)%MQ)pR@Voy2+uyIEOD-(&|en zF*uWgD%t-dou4b}s@3^5tMk`u=dybhl?j*N3~MlV#VZyUnJ%-7ej2o9OxvLN;{$&*^SQ zVR8%m{f1qi?shwZw7?J4;Nlc$U~Hl4|37?+ezM2)*lJy`Q;iKD0C|4V0D| zoDu_#VtBm^Jp}{I9nj|vG)2UkZr%<6)`V1*$HFg>6j^ByY~X$u+(F$T2sI4aUWax3 znFZ3mEbG`P=&+6{;w54otpJUy8f0qYC=GEAiZTqmiN*_$g!zi{!W!QqG6?&ih2CLV zXg+wE0SYa2&bcl!jId;Q1I#HIOqnMOiYUWxw#wATJ(S^-Z-L6knKE1`54%FXDAqbsNmd7n}$?!IxJfl>~%@=Q--JOJLVvJUrldc=cB?98jg8us>`^ ze}Usv^g@h)Tyw!Rj~gTCLVu@$`a%lUEY4ufB8)W)EedNEbsPqL`Yv@Z29tdp2DkZ~ ztz31vAyL(DK+v-ohN>@ipf6y(0;?`XiSW2Z4y?K?PQ=24lrYRA$X7G1{n$<)U>FdX*5Iz_+%2mC2~`Pg^` zDMvY!2}hG8M^x z&@#Dhiu5hB;ag_Iw|ooV@<}H}wjr<(D{QcrD9zbSl_8kZ^j!Fs*`{x4wtP!N@+r%= zEQD`q$LM$*qhroRQlaP2%|2509*0Y(LhF6b9f8ZI1P8c%`T)n2V742DGIj}&K?Zy&Nwu6Hs6{>ho3+ECU9N}CNE$3p!a>BWwwZ@JxeanchzD0#^*#d|A zj_Uo&RdcwrLWO(5(G@K?2R`OYELN$wu{e*;=Bhr(ts}g`Jk6+0P!P3Y*Xb`H(rd%5 zslkU@u`8tprx;qok%)f#DEe(M)Z1XFt6&pSzfdm1TPoj+FK=di-!M+_0JMC{Y zA1yQymf8kOHOBfv`J+U;OzPCMr!7%EnNx5H3(ltb{rVoeR}XT2wf2E?eL2jH~7Wzg|CVsRM^YQgBynjX!A zHKS1z4b0&>kR1G`La&^3Ay8`;J{u?9sjPVFQVY7-qoF#V_tQX@FX4+0pMgN-hw5p> zXYZ)>7chxBf2%Lyq>7_HoLgW@Etcx4UWdcsysiOW{j2iN7*c@!hnn63-kg6A}_^E%$+c{5%?Mnbg}S^ zSoDHd677q|4c{uP$2L~166r8j{6xH%V@15kZyf(SohBlQztVd$-Z(E_5URqF1#S^u z*C-UO^&@@?^oVfm9}2vvi19w^X8h6OQ)2PSFiRU{BnT9~H=%-X1g!5=1mEzqH@pGw zvlUCrjSgcZQn&okNJUa%uHgxy8mL`3n{S*qoP8bso^37Hzy&J~HENg63>}4Wcn21qn zWcVuUn?Y-s?r#i)R{b|wnE=keM0q^|>a9P=xVXnyh|xQ;zWl)7to0Mb$``ok zl+SgLZ#5lZrhc-c!%Rg&_7?QF$}opp^g#?2z+mATQxbFiG(qx{>jU_@8Y`#XlQ^ft zaRfU$SV`A{H>(NQR(ueM>9I<}yJ8=z{w(&QwzwX?774YW4<396CP|zMsopx*!y)=N zy@6)Kz4k|1IG%Vf%*Q6?Srw}|-lI$6H1BEGiY7WY{RG&m-nU(=>e=T)^U+nY?BLza z^t-s~gY*Lx@7Jys>ySW6_*(V;)3pi*K+rJT)PxVQZE`EzrE^=F*w9h-wsHG}D-w0{ zC!!8Yy@7AGwgR6S95I|*3i^mKn86O`;4mg;a!?!2fOZT6+Tjdn#~@zWf&pho`EK+d z=@>()VoNNJBlo0WdLmqW`l!}P!R*8~Y_^PQql&`4I3w?nX&_|4h zOi7e!G-XPnOrt4N5@i|!39*hk>#|FmV)( zvTw-{(J1tMAfa8v))3ofpKDbkuxbc4A=>5*vu*Zk-hFJF_gpL1i?(@B^&WDq!l@&) z>sBpc50-UqRjZD)YR;|6JJr2yW~6{HD|sJRcP45$*T^(j^4xoFaxj|hgFy!Elg;+Q zF4yE>4%;V(?UT>;$+z0)4eCPfYpG&8YnN-qipLdy(C8)c6{8)?JQM%tLRFiu5u zBNul6>~ihwRxY*?Lnov8 zuqSD>_;+~>Ky#kZyjbKA`vVCdChwEmyrD^mvCbgI3S+q}aD@wokDWqpIPY!265M(u|jgCeGhz0 z#gD`uFr;!KyikJ!Zc+6Cqt&2x3;Yhk0CaPT&aU2ak=f@}=eLUfAiRfx^#=3HZx~*U z*v-}(5Z#Lbg_V1Xt%b^+l07#x4V62K${j=Hv@2cg$7fKvH##a8)||VR;tIR-H>_N=j305x7jZZ$hkaVCuR5E^b3kIIJ^-sx zS|axgc4*AAT!QxWN zaZpdV?!jlH{rh^1LL?(;A|_wkjjJ|TEW_lXSwzHM+l>paSj0C;yop3>OY&%hR!iCs zMB#VLL^c+_OO!AQrPd?I0M7~6&-dZ;)Gfvfk!)%Ovb_)Wd`D)4Gbs}~$^0sk367~0 z>)r^D6c+u2MN~-4j1c2Ue1Jqt=3x=qw}qhjMd4j!#%Hr#Wj_6~rH04Kku6?i>!Rl@ z$oyL{?a*_SB~}*2dL_bR8H@e|Y8MrnLE_^PVm^u6Nwj3XKSEPTdjYhxU>BKj*t@ID z6C&A;+=pxdWTR14HN^0@8p8iXbJ^{6n&_RbXY0d*tByqJJMKadu@pVm+7Gtq%Z~8h zF|Q8~kVzuvsjU5n>DV~tbO~n$_pWgxMEF*0Hgn2~H9QQR!5GDrv3t`ce8&b_JP7e5 z82kPLC0Dg!QE&xmc}0ltuz1rV>Tz7fX?O)&WX9nRnBVpoZve&|o=NM(E7k;#8?TDj zM!r5TGN7w6Ru83Al;KBee0r--KZBn>nXcl%J^8Bma7gg;LQ&9CY*kyIzz;;w3MldJ z$PZjt8UB-#To1wAn4XLgGo)d91W)(xPk>ECzBJ4u#>ua=ehzP;5#)i3Bhc04h~A*5hzJ@;7LaD?}KzurPzqdbtJUR&=c2AnNuiHUk(bIe5j z#m)Q(YJ$FYGe3HopjU6^7f$2x+0^Lq`ry~OdW}kG-geij61Y#UjA2C{a&mRb%|#~G zW3eaNvBYC(oW-!ZuGjna()+GZ+UwLjq)@!u7r)Pq-LIhq9+eA==W*i*o-}~>_+;xF zd+>&pqfYIKW4KF8Xp1Sucx<|rwioI*v=nNMtz{pn1_n6S>)Wg9K8hR@jQm!SB9;uk z0o%|9PmBPn#WQu~0R?RmXQE>9G@hnts0IN|Z{>#-%)-M_nEQ)Et88r6;*~$s4(qV? zwcQxWcfE>s%0nc~V*DC7eL{2ub^>NUu;>+l#R0)BO7L%@=hTynNye%`jFClzS;X7_ zBZ?4~YEN?m(%@4rJL*X!wUs1T^|qHL`q4CCf|v4p7FJeCSR<8#p+6Lj)1g4+p(Rj1 zZqv&T(7_(X@*&kT)dGu)KlBpAu8fC-G( z(_e`(_JW+&zAdeNd+^e9Y`gre-!+KNwS59t8;kQ(d8-Hb4RgIW|w&2X7 znTSg?h-i-+Z7=YrSXyy>N#gJwT#gSfx{n* zRUj;8!N8~$-2pY4n?BvD|iM^fJ(72E1u-ud;JT|=| zWk{q96_g<~u0e|U7m{C=afj#kED#r`uV7+0PIG=o%JXwcH(E7*2~_ZuNT7Z|Rkc{aAEAc#N9+Fc74^8YRAS3H`ziuIQ@Lf#By+1ixVK00_jg&N!aQ z-@X$HO*oStbc_!TK%YNbkpXSc+|Y#4w<2DsxC}3gsB~s)xsC_3LIV~>v-@8* za_?A_0dD+?y>9;Xmbd=JfL_)6_)~|n#P+(C11@dHqT;|Ky4&IHRsL?bmCJz|=nm82 zE>d3=;~C}M7VUJjZ>6WLxr}xXlM~fIdxt?bPC~opZ`IVXrlB`T!H-j+-3+30~WxB)x7vgAv|3*2PrN_Okv)q52?<#IsGTip;VoX!b&Ml z=U|bl+OwJ|y<5tYBWuM&6nq0tbomg4pTIIf6@K-CnJceD_d%{F0JPj{%VKe|GQea7 z=g?TLZ{d3@YQW4$!30+%A|ot}G%42UT|@&{)8HjRDu5Z6BB8M!iecQqYWck1vcW_; z{UG)-VYmu@se=i5u}>Z(cN|T>3Vneel7P<+^2wosk(mGk646EUgJ4;?gZYSOIkn8s z?gQLPyyMust9lX(0)hqc+01985d3o3T2B*D#_2!ELI|i;4WbY*sKiEzOktH^bzl%J zzOL#fwky^=mO)hJX3pc`mj!mcXB~KQcLvi8@e51Bfv-S;mMJic)!*UB{dYn}%XD7< zFACPl8ohs%9b^$V^`U)H@dVR7${-m#2g2jlB5C-wb4X_&i*`|?D4>-!|GNo1M_?Vh zvsLsfLMIsejtMNi|4i`n1+RnmQiCPf7BM}S&XkttviutPl!2*|@S?Nhu|oG`YLvl` zOt0iM;k3ZwH%X$-6j&Rh1FVeEO52D+m!R9nO4<9U9_!8&#!Uo}Tx1gqX@57##2B%H zEcHVhuO%xh1WRTu>>(Jd6A8g8O+Lb!VwiMU14mK&=_a$$Mbi#^2ByH@N4HF&i~{U| zd4)fQA^Ffc;lF_#F>qKctnzcCuaI$#?7YyxYDxjuA*4a?5Hg4Q!a=PP4lO*Ib}5Rc z$*}#x%PbUBA-W*;$?{=KA{scyrUPLm8KWVZDA+a#X4FGwaxzwmBrwH2;>GOG=*N?k zkf}YG$`+9vAuWCtf;^V9@5muhcpkVi3U6ehMn-|f`v$NSKeXSSt26#bs9BcL^0MU( z!Y8dhg|dX>LWO>92{}}}NFi9E9W3-D3vFG;s+zil4oL~$YU8yK2yP-yq)?YwQ6CDS z6Cq(33D4S^QH)tT(Ej1UVmndBoOLYn1gh9_xZ6SYU8shG>~GtU@sIf9s=-%=4sK8% z7r~5z%{1c~A?{jf?D&rEG>**R#QKMkQ`j@X2Il}mV{hO`1u$Ut0UpV~f(jfl8tfLf zfinjXDiP^4gFQH2{mSpD3@oe zN(f@F+^^^-jXuyQC%U_zZf}1L%5-evgs%FqNaD)CaguGu<-)$qIN(#XZ$|Ye2<1X4 zzq%V*ATMJGH7MSC3qvCAE?ORbIMC{>)=$Ecb}MO|Z_G#eo$D9buY3R^;rkD3J#pG} ztcqhH1s~hJ#kDJZfPyA>RX+!LeL;NFBR-wmu6i3?%hv&iWBsc0BiCPxvH$hK{M$7B z9O|hie83$5{K{rXNPwdQ*izIF`Jzg23PA~;paA2qb1;Eq@)}3 zK+~|LhO&7EfuC9=yrbglA|uJH9Xn-{`5ioNTU+Uj>5LlF&BRX z857hqIOyQ>p4WmN558#lh{sCs-5|ap9H4C93cG-Td}RCxZGx4&7HnG_uKWtefA9?e zgoX{d{#prt_dDvB=Y@`898(i;h#s2e~FQ_=sn{FZHKtq*tIKacEZzW`?Zp2zxr z>bEA$uXwN;bA6ZnZFYw^=u8G0pK@^~-?p>e@q6Q4zR$u+E+$ymrjk`iK`EFoq}?43 zN-@J#;@PoY-8p^lNcy{Edhz{}`Tc_UdLb1vm$pDM#l>Q316NkDGj^Wm3+F)XIOBrA z=nJkUq}#to==3eZ@4*}0G+xXaJHA;^g8X83>QiRt_&Mxob>!F3tG>YY0Xi*D@v;lR zcJTot&Q+wks^6sB3{J!uJ$#~NMH=|X@{sUTqU**RXIUtO8pThrc&Z~+3ogXicMh9! z8cqZ7U_Li@|A%HMJX>h+TX<(zfW}p*cef9NKWRW@?vEb`HZ!q)db&dqVmXD z`L3)v8B3)@-`1+p06Q8eReICFd*Kx*Z&mQk>cEX(a6R6`q!>^M)YIzPlt zabQp+XMQB-H#koX9~^HSfhaAT6*D%8H}(P!PZY-$Fl?gkZVs_;pmQlfIJsHk1@%*c z0aPRzt!NB;5=4e1Ct=bp8>!5=uy#QrBP>>0EQWMu(TW0%mM2UWFM~z6TsSUr;l`(s z%@Sn1#pOLDByUrY=gH*{7MCJ&xdB|HEV<-@&l2&495<0Sg1~DCt)HJ4pQ7b^_p6v> zxh9^~oM$!f+4Ac<(rMnd1(>o!ANa0^vk4q=g~7E~4#C1{%Lr1`e+ZU>X-66yt~$?7 zVf+>XiPdkk4x5>-7^Z>^C@45El$$JZe8(o3=p^v4Y@7fqgi6MX4uWsQ(6r8I$jxN`RjOcQdLNetz0Qn%~T-X;* zIu6FpSdPc-s?HPUoI^QtDCgn8Krt(Ys5uaI=Wi$~-ssa<0CKZ4LsF4|43*#*ae|jj zyypYyN&>nlJfjlIxCna1M+3$95z*F7=xy5N$;VjF)FePQi;20|G z661l#S-Q~5aGa^fO!-dqyYaE%kG=(g#tkhd(vqX#vy~t$`3chjq2laRg($?xCmULF zf*>MQxp<+#sPI^F8D*3pm7HxE$;gFX4c8z^PH4gSk;Uax7^Ns{vsujfc+Xk}J*G4m}zKCXpk!aft$h+!Gy@vXYW_)0N;nC={a? z{-q8qg0Aip<=BXI1V`i9(th)&(}Vs!ri^mD>c9i>R^yqHy2dkQO`yEgR!YhrTJgXF zV&mC|LQBtFXvg!hGCnVS_B#QekL^7#8!W%PV_)N!IUhHEaL*27DtrsdkQu(dATuB% zGc-K&VE^h*o959QHbkq-zFi7|%4@}sU^kwN?L7=~HMT7<9zi2Dp6g%Jgz}ANpoxs& zunhf+wBYU0L>~lN*C(S$Y0KdnhwXe=Hu=(L+z*Z`!g?;1-uYcxhVI z7g1?dZ+@HB-jJamt%P2VTBHSnACu8b3Dd)w3MkN-%2H4d33g!&5DlyYs!#E35NK9K?Gl=jI8cIU}Br%xX2UwJN2uCUZswVk!a(F z`ih~GQR~>6GE}hf+&w$fs@lJGJ#v8+ui751=5230e@9*8`TjLoWM9=5>ss|tf9Ubt z<@IS*pSP-c?eVz;9?Dja&v|y02k`|<=#D5V49kG&zC74i9ri{uLmSSs z;LOmf^R?ia8Cr9mqT#I8S=?vP^nEYz`xI|mc}q)Qd$qYnShE@m!Xv96iqdOhuNIFc z3@W6J6^|?pKLhr6no5-Gku6x8_2r$%uP(&6R^^}zsUh*(oj9^JL~lBM5GU2^jb1ox zpV1V`gm*mST6Hs0W#IQ7>e3VTrBxlhDDCv24CetJ$X~(pPiUrra}hLW1glqYqbMV| zb_KV8(yE#tN~`+#p|s$t6=HMe$U_tG4MElC4>h*rRKF$qR7=jP^P=&C8w6r;>#*Gh6 z+zne(;!Z&0*T#Jt)i^z88}2<&NA&?3HeOa8sO1aPEGqzx_0;tmsl;bcGf}vfhlml< zUq9$YQ~I<)rd=RSwGg%O$e<0P9*t)Pnfd^`-Hl%t@1&4Tjp5>b6nPJ!y;>elB}34> z9a`RT8n_TQZ(!VR*yOmlkJxIf4zShSA9rmCMQ9AsAc;~V4JP^1r?L0CPzI9{p`)UvNvZNbE~$+djMx7gGU z%AOq*J)7QkJiKikf@0V5FW{zpdDzdxLd4(M@m2oQ86rJGfra$xaAJS(IUJKhBOZqf zKEQG^L)D^+lksGWOb)FThB`U4O7xA%NXPEDIz71gb$U3AJP)M@H@r?Sm{#@KLz9Cq z3YGix8_wW9hu>9wU3$h(A8L4JUXEGY`Um<)pPd*QJB;~>mp)^1P4%UEelM zAAx)9SurG>DX-RoqC#fvwVpB->{I_= zcW(n9RdMwJ-z9e;K*(-1*jPaq4H^W}4G9ngF$-BpG+;DXiKZ-JvytHD!|aBF4TvUC z)?^`1pU~PGY|zl!K4{YxTG|Gi$Zq%$Au4SVQ6r)xf*KVqAdvTe=3{rW*#%Kw-{0@O zzhuwcbI+VPGjrz5nU6ack10kf#qc$)KnqCT5XyVOlQ?4)_ z-ceU=@wI{faCwj_Q{C4ytp7y|U~Q`xih72%FAq%2e_>)q#+@f+MX!NkK%m8IqyVzO zn+mx6o5>|oM7tMlinRceiV5nSA{#g|!6o*PcuXo1kFyI-HCv`y#%iGzz5qI>|AF6( z56r-Y;~H8%d5-nNIb+Y?;f{SCTK6HzTH|~?5;~Mo)517sxTfu<1Cgd4D1YxnB=u|q z;Spn;f{C(>D(0D?cmF>5Q#Gsg(C8ZG6Q;X1Q-5b_!LSI*fR~WY$2@1m;|!O(5x_k| z3qJvYP_EdP$I+}utP{-j$BkRv&HX~t{ zhcd#ih}oD;_bXy-b@bEheQCh~kLlLDy|{Dh@d)!R7300<^~_K7zH{#z-cMp*^PY<_ zci?ArGnh~x`>qCeo3n9u!VKaals)o3ng937y&odHv4e1DVqf;Y6Z;;hcM!Gb-HNz> z!94va?sMX>f%pMdbjTFp@p{kc=iokOwUHhoXB~hhHY%Cnj-s7IsF!Wef_sL0L-Tpi zz&W0=j&|uB-A+%dc96?_9+ivnKZYr*aK|R;xZ!%Z)ZIa`9xh$o0|-M<2Lc#^G6Jq0 zYXR_)!F1P|N8h=P1_qH;-I}UBt-WnkXEeI%UD)u58ZEBFRx(JvO;6csOLBaIR-uHo@Rp9^vT?&)wETIhvEZqS^a$?uKl-Ij67a@U#t^54HJ;zU4huW-sTi zuA^t?V=C&r-uWMZXh$<4yt$i=yf(q%{QM(GezOq|Jl64Xn<+0O!iEkaefSZUInTSJ z?_ky1>K&iE5gF_C{wufJ`=i{=WfZS%^!$UAaMOFjao@n$?Obv7hZ?WTaX!*}#_?vP z$23&`A5Ysubnnm~@>tJY)vdYRRr_8u7(&YDC`F7Ju0DjCJ#$q%o^5@PTyqb6vX?a8 zO^!Drz3(`_jP(A_aT?sL`jZ~hSp8?7O+V2$yvIfg=VIn`c#!OI4QP%;dS7>(i==dr zqx{EkMB9 zv-)JsIje^0g0%Vce&H@d*3{>yUF&Mco}=98Sz}MHVOF5VO=CSV-l|zm!#pwSVy&O@ zZUipfZI8rweGQ&+Q-X1wR1%uLXK7Vyjj#rwtktqd=k@j4X4nE? z8ni~{P3oJpWv~RmY}NLRpg_#(9t`^c-8B3I+>mW&aHF2X&1D>JGZI9TZzec#!SK;$ z7!K3an`fyo9}9MS+V08g#`6swoc_5wKr`CE5vc{Y|JBB`us$eW=4|^+fF9Tr8KD14 zCV;4eO#tx&-))S)NS8Hs48WoJZjMK$iH^8Jh-jX*pLbBk$#xLed*(k!8E19_io|G6+b&vk$Sx%K-X8`%->FQv`kR)LJh2QVb3iPsTu zxIur48;Q1jG7VtX@;O3;p&6yYe;im6YqrO>k?<*Ljq#RM`P=k=m?F^K{K zVFF-h+gOD|!a3GsXyCqyo0<)z6;w>s?=8k--%L(KC%Pmk4NZ)}=CEn98a9Cja}9Y^JNBoxwG2UV#H|5ptC8t1$PwSa*I49(^_YV zJ@rcz^Go znYtHJvAGIwXHNT1=Flk_uIZV(sN+`fU>P=1^CuauGkEE3Kt|Q^h|Hmx*wZ+~bX z481Fky~_StXQsy^~s;+h$ZmN!+!y%s5S1}Xi{p904-ZM*E zalip)tW?j=GgoCA6b3RdLd9JfyK~X2kqj}IKTY6IBl(??-|6{XD!+^4cWLa-#JQRT z*KGc58K<1j?p&*68kq|@VyzFcm?P>W!oy5WKP|ydu$Rpf36oXIhtYJ zGY4-tPkp_75@z4B#$xkt?9cGuffG>YxYU?AuJjlz1gFQ~;D+>=xXfMYF$P4&Y;9Wt zosN14XBkvofOGvQcxTlN|M^0L(z`$1wMY0~ zYJN?xCzv8|kIB+Kvvq$N5i$Gm5m3q~muc0PYDHYe;Mm<6gXiszq#r%~05xy70hRoQ z2@ZcR9d2+ab8zfS=~MTYuicqU)zv);0uo#ZSQlC!x6i1V_6yU{DX!@zzdF*n9PEMX zOk1~FbzEnHE;3ad@!D}yfF=r}SGu8wpiy0ORjA%|j7TMQMHBOL(pDY64pgzO+llHL zMind7-WJ=OxsBh@GdL04%o0P>t63GAAbnv?{&Bw%S+H&Xz`#`@pd~75oRTpm0r{{nmr$X z;r+z%hU#tm7#+;=m5Z}IrYRq{(lZQSKgP&)`O1t0k7>-uufpnTR4qn3b@8vLD_gj2I!*=M7}VM zejBOXvOuQ{7$reE!_!;wEO1N)QY1kJ!&5_Nfi4+HC1F_+B8B;}^l*Lw{XVf(kUXIh z57Y9MX)HtY<0zalk5MkBa9xw6+)e>to;W2jks|KGNJc+`o-Urx!gdNbKPxF)MBMGC zB_>M9KHos}WqB-C8U1eGB57Jh;MsE$qZhH#^hB8*Prn(d^t&*Ne$y5*ypw*{l4+zs zV}l5+Y8HVyj|i+iBxz1c8bu^g7b^m56GWhK8UmS*wx$8h!G~ra8;x!d77*&sZ7Hly ze_L7yTWWSi>CLVzy-Gik^b=*;m9E!Q8^QrNvBH+N!j`W53GC?H`+as4CUVt^AUk@U zFr#W)_ceiLR2$NaUO6~+?cjN9Rs2WdKMMc)J{I%|)8N<|vY_jj1>Ms<7Gfb^-Ae$a zArRZPXg}Jsi?JW=n&t2++!GrR#(p&0e1Mu;+wa4ZO?Z+EE3yhD3ZwA~cF*h9J&my0 z)y78crnc_uP-+poO|P6WeEPQ>9E=r*o-d_>&Zyt(LVOjGxitFX^clW<4IINne{~q&GACr%qw@Ic}$v{|^_D ze+lGo05e(%@1YGsT><-}tb4L*eoN`5BP(;PZ_J4}ik{AXL+`A|;}g=?#v>~%qjMrY zpdN7SEaWBKpP7HscA*~1481Ga6*e;?`e(*_1~&EVBLnb1Loe$i?0RtaYDL)|iCkjZ zbL5)I3=Urhh&w$*_+n0nJy1J3;rzhR%oa;OrwCdpO+OLGWA-PI=g-M;vx) z^mCC2VeMD}ptWNfw=*)m_SeLZraV~m(hZD_C%4C&K7W@tEBAo+$i2rrrfGSv(?rxp zN(@s;_n+|Q z4I}cNj#=GG%xhTTU?sH!ciTH*5^FY43Ai>K0s@h%wN%!iuhw?pT|i)JwW~O$R@=%k z_1ZFy!DI->G-wxcOoO(OW16(t9Mhz&116)YE4!kRgcIyn6L(KrIjxGH#k?e?ewJ3n z&z#n+YlA|Ys@ftmt*>~`VO5-FHNXvTpums$`<1Bep7u6Syn|)&HY|&u({G^Xd|4dS z_3jSji3Q3q=0u>{t$$x&dgP-z`N)Vzd{ujQc^lqjc4v}{?LB=j7G7T|pFPLACv)s8 zx4T;y4=iuHi7A%1c`9dP)JLI6>Kgn+{gkiP;>AY2_td?N2U2&Fbfj!lVFc?Qy?>pn$G5=!h0fgL z{vj5)aoo#qK$Y%lP2MxM9iyoK9x_F=VoRrP%@fR0=!e%QqkwB#*0ELW<58f{AUg_wcsrz-0VT??sYGT0zWXm134Puf-8-I}QtsPmejnp=Us=HR7 z#tr6;JE1gawp{?8QFpyjn?%{6_fS2X?s}s>iPpZ^!U~pf_R^9LCemqLr|rzj*XZX! zh7~Vz_hR9+jh6K0SYN}zn*dlI>@~n(wlE#_E-&F3_l#Oz(!kXgp(Zy|V1!y&&ODp& zbR)wd)byMTiMS?{y$cSp{ghW@|I+)q{j|rVZ*O1gRoV}%^~}6A?iG#4l$CcF)1(c< zs6>%!lT3aKD|bqMD^H%1;>lA|EO~TYBtf-Lf)*|TA_)oFuD1Wn>$1P(&9(3LK%5Tm z;`Z0qdQ8UlqmV$%(QX;WG*hMoDOz~ql%y|lG%sqrl%6l|Ej(pP&zCaSBv0F5`vLDe z_M_hAb}V#FvA^yuYd?mR6WUK8<&m_~P#Lpz7-<<*TvHOM7q3E6N^kCkvtG>`UaG-e zgl*xUEYT-dP#%@VlNU?#JzIu-ZOJW+y#xNEh}#qS+8NtaGVi3L5z0SQTg-Q@u_H*6-R5G zHc<8MCP}M#MIpqSAFDD^AXDsz>9&^D8*yNGu9^=)JB;ne*LvCv?JqL9|ARL4DvA6$ z+X;XGRz=RNJdDM3k{o83H=)zCwC;I#WLzgLA2khQh7GB(lFhW@Y^$>dCOB6T2-@xS zWeei8aETGB=aqqwkSy1}3#1J`G*^6Yw|zI1>zr5LzGtm>d;6}no_8n4?WWXghcOdV z)n>p^Zo0=_{}M){h8q&zIo5DPQo{{N4KpOkd)(Qu{;!f5eT+tpw4TXL4cwzN*zvx` zDAS#?Cd0LjEK-KMo*D2A*9HYWa+d2YVVp9?_Mnd&eqAECxm%dA$E+ixao4gw0V^KN zrhx-B1L8ytF~eQGnyf*cS=ee?PseT~W`JoY(ySWZy3<|i<7WD=sb+?pb}Eta+ykQ2Em!x<)sC= zsYuW@p<^BsMfqDzmRhdP6Z#6E(_JsJ)%h1;)Q9zsLl-3pcOfmB9C8&FDcMuRpDW4+X=f9*E+oV0_N-uVhU?rXOlNmfp zuo8>=qtHs3-GbJ0GmbzDiw{P^TtcEV-Xi-;WVDZg-840LD{2yLuF z`t79Ag43CjElh>8tp6s-VQoJoHyAsgkb2&lRLbhxBm(ajk#apt8QW>Nph@;msc+c7 zmV{!gAQW#5-bd|cz4^soQkrLk3?Y(6ny`Us?H#bL6XOn3Q>bROLcPmsJya9^C118* zhIw^zc%;ggPn=_I#Rn7aa7V4drV$pa#`!2M+t=_Ca!q@Tj&hm0v-~&~?a^ib@n*cL z^(sQcc2TI++jZ}&nN{cEx11aTGyZPTs2BB_?g}_(%SRnHnA~>`_Pm=v>O~{Dq(*X! zu(Xs7h}E4QbmUkEhF=R84UMW5K(sx)RU{e{Un-JruHUhBlh=_x+Uq6~!g0~VYbrhjeznPe0Jfp83O^upDwWGcD4cbF>m4Wp^)6Rv4A888ZM z8O=RK{Wj((k6SSEzA-dQGK2OQz&p-n{u4F`@FmuJ>VVux^Av!a>o}6F0z(D(#)Hb3 z&CP@dG!ooP+YF{*n?WVtsX*Hd(!A%e)I!1^y7!p(uzru{-G@dU)Ki@vfsxN2bn*-u|6FSA-gookTApg%bLpoY|#Ihk( z9hM%;u|1(u&*}zCkLB2&fa2jviR}roxoj%~M%X)yl(~Jm8>mRYgLWuHVTZ!?%i~%$ zQ#s?AnN(mKrQ`Lm9SYcy;K8~i*9p$y9Cj$2)3aS70Xq~bF_b|~cKZtg(X(-w~%3P`wVvlvdXZ4*+v%vsz zEPzv_(=i+QU_H>Lx##J7vDX~5Y}oFScCA4U*c-IMt~JR4dy^cnZ;=D`Epou#A_wd( ze89d81OIGFdl?4)+^&fM`^GvU5^={F{aTOd`kNQ1q{U@;;u!@yHNFFuj_g#!_#V6t z$86X5I$_zij5o5OV5Pogd>L6b#Hb#xC(DLV?f7gsJl=<)>?j;>!vc)vV>WhBn&+^A ziNoiZ4A`P%r29&Yqd9c<@aJwL4(|j0xokjB+szBG?vRyt3LLiSfx#E1rh7lozv0Dh zg)+n-({&VPEO*ogFk^R(dX>zWyBX6$1nU5)%Wfq@wmKVkCl>BEe ztquc1?a83AVj(od)Fb(HSJ!A^)o4RSr`S+2n>JJkeK27c6s%eY6^`Gkv1;*IHI@*p z8bMgKY^p?9HL5(aYT3L%SOGD^2IOWV5urXA5w~bNnu*YZ%~<@|jxq+hQ7gkUZq$$p znbr+Pq9&sT1U6XQ?k3e>My;9R38U77Fj<>gUeEL9Kc^>7Wadc0=muhJAokvZ`Oi`J zO?ZN0Lk0FyL}C5HaRKJ$VeF%Tajf6Llm}ZaRPO-&>tOI4NfW%GLKr-@&5F$42Oioy zp)$L^gNz>z;-ftlD($g=O}(D)u|PWa(T0jed_zSu;~cd0ii^Fv4$zEq_G{UO3N!~M zKeR=U2f{#k&tXFa=_RtnH&pnhC2Ka2R$yPWzIyW(Joe2~$_*6?N)HAuk^wfEj13hu z4;`?fVo&!kPeKYT^*;7F6{db-iGdu)?ngf*jB3lXxU%! zg2{AqO`(d(%TI3 z_nK>J*|?C#hkQ@=JK}AXt!$nT<2ktTIH*+nZ?xgrZuM#!CE|WqJMMAXYxOCT<uvN__ zSk-F8D?_@+=A)a(3w^qEZ)<4+a;}Y!<~asTek% z#`2E!)k0lttABk)H)ezfW#Lf<--^3Jxu$MKP?Fm9bnhd7%$DU$>SQU2i)c)T_&QDJ>-Kv1tjEOfDr{9h=4~ zhXP3tQ{vUJtEN*Sfo|1oCJr#!5a-bs=>bkL0!2d+vps=U$Q7zrNUfB8!>&qE#2i)S% zGJ5ww{NXD`l0^Dp1k+tj@g2|}9Hg^lKUJ8n%YB)CJoNJ>{T!s959#Mvp>F<# zLbGoEZu&V)KPPAE#|~+ zXSqkj^DI%pWZ`%*DvCJ~hvr0d&@si8Cn5QB7?!RmWq)1@k*^? zy5-}Ff$6%0lPC0PtIp4Lo-(cKo$LGvT)RJ=t<$G`K)>-Pz&!VT1m~xmq~P11nsvDy z6#5*Y`G@J}6yZJxP0ZF4lyk87H{CUyq&vLaoP~0z!K){iYS<*RXmiXxiP{iM}CW%kB;ax9;v_uw{1-Aatf_FA*THWiL@6uyrqyAkeZ`&>X?dzwo~w z|M=b)7R;vZD}M$zhw;A~|F0wcLf!D4sQyC;;xH>mYK9eWmvfEDD0-RXg=F#cncRpr z)WiJhPD+@Cj#3EYB*ZH}h8Y3pK++*IY>kIHF?slbDG&Q3kDb_rJ(4*nHf)XDj((4f zX!qL)z`AsA|B`ovVt)ah7xTmIFeOUH-|?f6Q;Wl=QRQ0A)rwN6D``z`MeD(9%!OIH zlD#;*{WFv9{a3(qk4cx%TByt0TWG==>0LMHLe`luVkgnEF6=Fwsrz2~Nt`|J9gz9x z`Bh+BP6k$cB2H1*`^jBr?q^4N$5h8Fu#8xLVDEn6eh7zY2Hm#hNVZ=O%Yvyr%2RJl>B}C!YKT z8hL!3`+nr_<5mziB8V1`!kr_YmqUtJN8Oie#8CnXxx8BU3fJKMxpw83eBqF7_8@|Y(;7WTL3YG+(6H< z?#JjpL&uV`1rHFYPkEM6R;M(J7~28qvqJbdlJ2p4VVLl(Qr)^qs6JE8BWl%oJw&LF zUyB0L)mCDz<1^jO$8`yNT_1d=Td)t?&TuBCb+4{{56+>fW`(Td`%TbyZ>7F_YhCrV z=*Vlg@;*FoEA`!58wJt;BS)0fcW-SLNZyC%ZA~L&t3akv7f{uXNzPBopTRDlsolEA zXr~rU?9)dR_hwX`!Isf~;8hVR;(jLL`{(F`WY#ZWQ|p-wkTuI)TTgOjxzp?QFx?}b zpr2pjC+b(3RXf$Jsc)5cAtgr=cpU(nSF+sIEH?Z*789oj)$3jApIqu+p~f+>&GZEX zh<1{7e+M8K-e6DeZTkotY)lcS_*YY2xqy!?Siiu!?NU36wKkrW4KJ_H;6KDtHNtXH zk;m?174xEzsxQh|e=V|=yVq46)rf0aGuO|1RB~ zewzlf!SEc8>ejpnXAIN9nMJ!B!G%PK>TzC+7Ybw3DE)&CNR!kzJqp#1WJx|GLd9lc zts0wD@f{l7Iyx_!Y$-L4O!p5l7{K{5&~qF*wjHWsMoSrn)eIdJ-}K)%%n?U2tuN8Y z1BbXZQ;|%$oJo6I6Y9rLOj`kn=P2Q?)ATJ3g3(j8V6xQb}VvoaGQr zN^g&fHMPJwq4ouTq#hYZz23IAWJe>kJ23`#M&6n}-fj9tf;{PB? z1^6NUk5P|`|C2M|yFA?lg_wnQ`t(*FXn5Mg16!Uxh(IPrVEE9H?y+*1h^Y;%UUZLr zz#3pHJ97>>$Qav!&RtWXrM{yWdpp$1&rPm_x4XCa%9)qJT(=z+;VTGqwCA85Xs49l z47YtX%k&%yP|i21l+TSS&lPe*5^p4lsj<$PdZheye80$H;T%~bf`>!OOPD5E_ruWM z@hQ;=3CyA zPL^raIp$=+)nZN-)KQ;Og>>`f9#*YNrAQ|(&r|goRyb%dJkwR+ryrx?`UO zgX+gf9tu240aj-1ujU-YxHm>f~0Oy-dAH)vr*T-H*Tx zbG_I-08fGJE@smZ=_oSM?Q;JM<_=Sd3B#YK>TsQdzg0TyX9$~hIWPKMutRv}h0}Ei z|D7;shfTWqfcL+G@FC%ly$F9ym-ntNcVE|-JFx=}{@51yU@G}!FG)x2AF$rqa`0yx z_qrC&Lw=d(rNAP0ZSvi!xMe%`h<*m>yC(Xc(LsgWG1JjGIWj-AzjfcO2BmB2T$G)A zVk^Sf-@1A{K+Uc1{{R5H2fKIRDT{XxPSeT3p>a4FNt;SU_>u|Ens<*JiyKTcWK^Aq z?Cr>?!AmHodn-Pn$oKErSyx9jhTCvD(Gu)oPG~ZHM{(xmsd1-z@ zQHjG@k1ak1t0xtUp|57=!EM>a4SbLLo>o2k4!e>o@m)^3pomLmhMrKLp$-#qss>;7yzzDg!})&sKH55a54 zN6}Au1>Y<9HsqgeFI|*x&nzi&hLhW}#A?rXICyo4puZ%0ZsSGTbK75jdR~uxsNx=j z8k@7Cyu`9J-(F#JkfJ0ympMUPPFi9sdBC~YFwHR0Xr%aro00FZISfS(!`yqavJCl7 zXhnf-nc)@#o|i7QmFL^-vu9?9CZPCtRpi^l#a{|VD=oj}7Rww{5r!}`)@<2U?A6` zi*ofBmA{U->;d=RN(uq`z!L~WdytnFoqju+f2p=!+A}-E%{-D6NsNw-jK}B&H^TyY zuUJORals{J6=o?eEwD&0D8b<@xaF2b`Hmv1rKk|St}VZKnssr0`8Wf6LP3kOJio~4 zVCd=74L6Q6$k^m#E-EdxkMT2?j}l~n`JtJ2T4rTtnGH7w_32zP{_8A#&{pC|bU?8a zSOb_~7;8wmIl)&h7m4fV!_Gv&Z8+%ne|enX`^Yqo4+RY-MY-5N`O+8rPXq6Z{qr@8 z0T6)Z0Urta$1Pjakk4tPjoDNM^UVsT#Y5673gyr5;|G@urax$)%+l+hH#|^eIRusy zn@C9rH^Z+&k9--%ervN(uqfj87fboDGIKX5c z3a1m$4%h=&O@&q5+uH!x0k{t^tpxOVB`Okd7=?031Iz+! zt?BJeK{<2+<^d|Tkb~~mAs?8%=mD$-Y_10#MsfNLNFQ+FuK{C`4&WBRv__-@d^X@X zocyXh4Ssl!i}ngT0S$n)fJVS3z%syNfU5w{0X71T!@F_Kfa!o8fKI^dXOKU@I_%MN zA|3j!4$YnGUqF2VwgReG0pALKfSqmN2iW87?Ty7NREBoQ4cG~o2iUv|<%8#CZzEs0 zU-drtVb#fa9Qg%QFntw;MZ6Bcd4MB;qD%8|_T7qRnSS7t*WYYVu8A5HNgqJz^E`_Y z40=*S1d;+GE12q0DrffgqIi5)147gA?<73*Nqq)F(cF;|t*;%x199#M9x$v^c_3lE zmfoFLqK#Dpc_4MoAeGcCx^sp~I}V~#+Xcl=4igRt9{UB;mpRn$<1mtwbMYY~T0t{sSs>hcIMPiDum}P@z64-V}i7#a_&)q6UTdhV`T+1^BNN@tYA=9sc<1r2edr;FW8rQaMMXT1gxYLO0?BXT8u3k%F;G zT{mDo^$c9jV@M|pa(3Wey@AqUI;dYWKqy3nx(<(`^+NSz)kLfrpfLRht0RaXsQ-k| z1O8wLyc76+A@H@pdqUuwfNu%H(?J&vz;6Y<2T)y)6h--7Zu?@Dk*%gu9=~Su$_{K@ z>fvvV)6Y>g--=9BA19Gfr>%cIQNL&pfgb^UK@ff%_{5U^0G{c~STuH@n$%Oqed>W5 zWgh<1L4OYPWeBU+5j`t$BeN6h2C`NU!$(E|icDZl1Zx^Q8DnlJ4v>`iwjn)^j%0e* z08A|UchWocHFPq#D4z#`j}mx>#1HAqN#IrB>!2^+P*<|R;ajN}bu9$@b~sghTb!uO zs2oS2J1Btv6Wp#-fX@rUQ$3FZejf0R2&;F(Ns#q?3lO!uX5Cs=r>ONCf!xO`^<_r4 zpWGYL9;)YPBaa(Nax{T&Lr{8TAIa~k2fh_y^#r7p>7@g4F}>o(U1}Gl*G@i0zr=x} z<8VmlP9mK&^mXcbN{1};<@e)E>sM*iFQI>5x5`&(`dFG~669;ztRPmikjrm}6=Y)i zlZP(#)Y88C6OU(3;7~@x4erLobx*gevK; ztr@7^1t~-inIJk~!^NF4W0iKzK<<-Ke(jJywo=%$6XdLOd7FTb0^Yzd{D;c11$gy` zz>_R~S!pR+I)BVJJqEgy*h0UGvLopF`w(g&^)>xm2A&+VvRHtp0&-3W47Wd_xeP%3~k!)hk3f3B9Tsz&un@ACU^todR8D ze>&f07m`nVR;kar*1G#R+_vzMq0=ZwRVqvNusM zK=mQ;@xa#x;i(^*27EQ}X$XhugQ&^(>w}1#(yJ8l0esguglL|n5_}B^tKTD~V``>% z4^SzZ7~oTHCi(}^+l74UXXwOm5uX;|3nZW8z+Z$<0q9SHejeyoA*}8sdS=Fq%y9Yr zw`=iOI&NrW{k$gLs*JTNd_59I(9%817l?Uq4N2ZK%&V!TedOZpX`P%GSugEMU_Z-* zf*VEeOZ)Ap>~V!XjT8NkUX~V@3O*0IT=#%=5n$bp)M}&E1d@@}KYD;z%zvR6eAfts z;(_0X_dtz=6*fg3jW9e4r+9t@&ztG-Sbu(eePiG_oi#L{oorTllM8DnWk=4FZDw|| z7IxBuv>G6n6Z3Xu5MDh1eUP%z_$DfX>>l-L`hf$MD(dwVidL7d>Rh4MsY};(u28>E zs3>)5&)X~X!_=jx-(I0!jnUaYW-VW4zx&zm0j3D;>=!d*yr@|MjCWriL5aj85zGLY z2`EgHa5tOX;fJ+@Y9u=1SzdmGql@aJ0r;)JQ%@o<%6C2RwRqpRmFBIE!X_bhgYAKw0v$*q|0 zObf^#<+~m~M{p2^-0M&Dq=z&Qd+_-nJ-i7Ipodh=Co$-s&nC!s^53$)m^6LnkVQWK2I<4*ll12#_&EO)q(8!r;jz;D zq)9V1Z8fZiOV#ckq&%T&j}B5ctJ=+x%HP%HYa^BITEZOEYCj*Oyu-pDX^HaZ1NR9X z?tQXZqxqetK{N5w!O9n!XEe2gmF5UDZay3g^Z)4(t!K2dVW{?l(aMgY+BXxGT|@t* z!Od@mZPz?|lk#OWJ!#f$*F2t}td60ZzsGFXoJ>&GUr9HfE4OR*k5PU(oNitm{u@p6 z&C2TUT^NYF-x;)z->B>`Xn%jB@|r=r_eSMS7Cz4K|2wLelKC*6(mxxog+M^P-RPPD?~bocP$9VYFjl*+U*S*Wj#gtN1u53hN8H7pZqG}r!<=X)qE0} zd|gZ{A-=|&$@{J=!U$b zQ$A+l&H5i;W9#>|Z+t(Z_xnR!*F^mOnwws^MtSQR2-tPa7zF=!1U=k^-w_e#M-a(R zM^gA7Benk?8F6+bVXCjC@c&UJkqw@0lto}ZbQWe4Q`$vOp#wEE%cS|RT}+p+~1_lP*MT!3w-t~jb^>(I$+jnw110L>IT#f z_;85wzN-D_70P*4`>!jM$F&sxz4nymO`Y=gAg%j~h~GwP+hQVW2WxwD5w8x`0*^ZR z%@xY#A#n)5J46c>zZ(jse;#@}g1?CxcRE@*95n&q#%S%)Xyql+!;ShOztu;)qR*|0 zQ9g>H@VP6tX+RUM%mT|dFdVIk9IIWRJRF#>pK3HKGJ*KLM!Qdq_}KvM-&Ezm0PVlD z5x^!A-qZSG;Qehu_qPY#&kMR=7<6A6biX3#z9#7Y$)Nkq;+~vuy2`LJPZx&2 z>`MKNL)ad6v7>_!LIJZY1FerPbja)~69J4g*!890kC86BX!VXRax3XdxPg}!^+a?v z3PQLQ>>_u8F2P3WNaptvpTHbx>r1zzYIKr*Tf#3YaM1bOGlI zxIn-X0apsRPQa%Gd|tp81l%v+TLOM0-~|DrO$xiN6);i2=>pCbaDjj&0r1zzYIK&k*Sgm?+?M0p|+1K)@0KR|>dJz^4R!UceUw+%Mo;0)8am z1p%YUQ^$3!fQbT57jUkC3j{0?aHW9j1bj-s=LLL0!2JTgCE!N_UJx)kU8FByqJYx{ zoGai00ZRm2Dd0K*pAztS0bdYszkqKE_>q7Y1dKL|^aV^5aJqnV1zaFtiGV8wTqodD z0zNO`3j*#J@GSv967Yh6(X&MQ0wxMLUBI~lE)cLpz?A~76YwbkpBL~20rv~|mVh4# zctODE*&=-b69t?u;9LQTY5*r?zo|EkVsL6j$%7@OKQ5X2jo<9)|8C3F8DJbIsyM7(oY@pUF3gK^3)0V50U=% z?L33h(%=5wq>uKO z8nXSRj{8pXkM@@uvi)uUZqlEG^e2TX|J3o{LH?5`Cr_pSQ1$=+ar&u=-+lh67u2w> zP@~MAIrA1n!tA;Cj4`AnrX)@_OfV)*FeWD@8xrouOA;A)qmkjqr;Jg?7uX8(E9_2t z(V|=MzpP|&sl#aq#4anb5N=XRK_B#G5j&O^+bsn}jsxalT`*l31|31hF`DmMe?~Z#_BiW@Mim z3!atR?D-Tygv#trC6QipR1%#wyo;Jxh#NdBEx;?AN}_GCrLa7|*k)N=fVlDwOs!Un zZJE_p=CnXayG==?mq{HEfrBMhkP{0^58Rk+`Nc(6L@h-s_(^0*^JEs`ozO&UX)(QG z!@a0)_InMY55m9n#rTe8?4{<4$>#b4Q&0nM5T;?X7XLE7oG0)>5g|gvljC*CCk1#K z`pfuo9znuGIv$beiAKhkX-Hp%|Y? zci`4eD+A~b zmwL4yfnXuyN%(P4e1jNA8*UQmhiQL;o$@&X#7idAD*T}4L>|A314*tJd?6k<3lK!Y z9?19;f*uClDzuI+)8-uc@6jJ{%7~x{LANaJpeDu`EKcQ=8Pvhy8j1f({L6Bac8%tX zL#5xG!{cji0ztqPoPNNge!&PxC;QRBhY_$gvVZK0k5FX4*B7rUvOnvK*DA7~>WjxB z%MSuEJRE_ZUiK@2Pa{w&vcKqyAMEQ7`r?Nu(!Te_4^^by?u(C7q&@D7AErn<+ZPX` zEA42&g9t^ZNV^z-M*Zm(Y47^tV-#u6`r@xpq@C)EzfzI*r!PKMk#?moez+p-MPK|? zX#b-EQ7jyR)ePCb10P0UwL-SrzWA#Z*&h4iY1d8Pi}Z2;JgMOFr}m{PpY-;n=6gXj zXnn!uLu+FirGNRPGJOB?p=nP_ca`Y3WjgmWdcXX#-H^YcQLa{Y3p*p*<1)cV$&eg4 z3@3jz;}fG~2d!WH55wyer|1VIw_h`SfBDJ1Ao&&1?@RhN1VWYDdB(@Dx2Hq|vQrw3 zUk}p-ektLR-^D)|#W1oTImGbbBN+Ti(7z@4vBF7xR;n{&UG+;3a(wjHw3t+!#ckEbwxkQ_|ll z@CGseC;2}l@N!;K(z_-9Ao@Q@{sJ%QUy}UA{F>zdq2w>{l727>mdPJPKUUzk2hqf7>KZyR0JQANP z@UcPojRJ27!gmY&IDvm!$g>y*kob25(eD!YBLYvGFzKSbbwuA2M1L#rWXI$_1_KvY znN9St8+KuT1X{U=;fE;(asY9y0-S^P(rAeu6H^r%~X4#L-Ft@rRt}gq%eJ z|7VtNobs%wcPYSWxhtCYauzi1Rvp!D${@uRnGSl zJ$7NN!Y^G0_KR2k1s~PG@nTv;`HjHG2H~Zgavw{uoQ*+p9u6bV`$C>Z4Nq5&dnA9k zKPH%eQxN}ij1T$+QSTyKN?a7>_mkdWzbKL6`}@%u z;5UcCw=#VH@m+fu`pz);FT>ysC^VMepnADS;9F?|6IUzyRg@3ur7UJXNk5_l@Ny49Rc z=J&=}j&CjE_zn(~TLj(^RL*wbLzT}T1pO*OZxDPAgrR>U4F2;l`0vB<#(9=Y|HVM2EB+o(ML&-A^ z{YfbN%rN+S7#{M7bfr9-fG0V#MgPU#&x1^#2>MPp7tkQ&91I6LRJtR9zh2M;f+>uC zD9$Y6Sefq6!tnW{px6I`)6@Hmbe$3S#uxd08h00zM3_!A?S7XDc*<8tQ28%s^l{1- z;U}mfrc2NpMfs-*{NI2NmEV^cpR1L%LH$UV;A0T|ph57_VoD_xpV49PX5d5RD-U?4 zw}QVc&-IKC^tqKM*eufZ2>ND`FN46p$>_yR0Js_sWGFe4!{F^<@NK}8oQl)jF;%L3Vdr&J-UFW{-|yfr<3`8O3>?_9N)-+a)i-~n*cBZ6Su^t zwUr|~gQjQBGIp^;B<7J16{uZK2*JU zfY-ynZ4UB#I|QFPk*?H}t}uL}t_#ioCg4e)x}fsU3q!w6(6fa*$lbxDSClZg%Qz)F7Y4z@0^cn3T-NW6Fv3LNTFvPtel_r+hnzwE%egRou8j{Z&$KZ3M}enwJ4$)F zqWF}Z0?%HG#Um-t1xeq;=~r=}Ts=B;`!WC@syuH2p3+r5^8i9$1PBmf>z< zlEU}w;5gIb(5){AaO!s%A|>w2`nR4i*+&f zuj7jo)(0(#lZ~m0?=iAzwb)D9S?(29CnHImXiV}`P)TqSijHSS|1;ksY%uf1o}3gu z_T;2X#hz%K;^*Hl?Mq2M`67}}zNF+6lO~5xJ86>tS;%}(oG{T(5iI$H6#ui3u_sLq zpZ285mr8rmr0}ISY0{-iZBj~j$tR^;Qt}Dm>wD6KOV;<4@cEuNqvY79_BWIE&`;MGsi?m;t5t+qf9DI?d_`dd#xx}2~nCE9ZH^(`N zA77oB<4mE0$1}5&k^;UjlbuiU<)m25=1Ju?Iwsm&jJr^*3&OLuT$($jG6|_nwAyX? z<>sZn6aqv{mB(8Xu@S`SNuddIE+fHjgNMIpCi#mdK=8|vCn=@x_gBnwr&ui5<7=@l zTb4B0Vkx2X$?rDrFm$sq;

wBpMTwCQOD1 zQ>>*WOXrnlmN;`9zO1FP6V=g9zG{M?^EE*SDYnCG_HpBt_(S$#9t)(mL1tzy(kJxQ zd{lJ^DnFLKJ`h-v_XTLDWErk8NvXoc&yfU|uey#Lm^-Gq{t`?w`p@16YsVs45&27( zSqdvktn_^!`Z$p@zXadnVc#X9)BF>&EE9_C#zK2(zLQ)Vn=>&vbt39%!US@S>2qC$ z6>z@G%1aknz^fp$z&y|5ukUP_6koxIiQ~vD3>SxL$6i!o3nL0Tko0@8_q{L?t$Eop zOBr_KmzLm5LeAxurSKnTXWco&lx4Yd)~p<)FwZn2%S>Z{0!OK3aehewHq?s&fpE^# zbLX1oWX?qP`s;|VLP$%(7eJ`Ygywgt2A(tpHVfaVu*@s7pq~OPNP~sqIPSi~*Iu)&*-MIwSem7Fhq)k#0q>LPMUQ-^b+M%c?>1OcD$>$^ zXudlq^Uk?nGvmp~c!5pi*=Ej0-&oAE4Wkz#7jttQGtth1WV>%}(v+;y{DOH!#kNe} z7pBUo!J*nwu=tKfkt{sR1o#=G*K?UEpgQi)E@>ug6TaRx6Q7bunn;FR{HLoSJ@h$@=lHWZaQK7(aYx1Zo5WDZf-!PCDe9aiB5(?Mp3p+a4%#z=vR;U&FU< zkc~TX9242VvA;r<SVJFd&{x6mie!T8mNp(|RvxfqQEi6>3&S2ZEbGZ7}U zu3dV90p>f^B0hKms=%%pw*~g>{%s`30WSexxLXRBHmJ}o_#_<~kEJlb$SxkqR>$6k zvcMV3x8U1-WVq7OV9!i5GA&7o$w-TeF@-tGA(Hf0bqeWfNK??SzJ2EWVrKSh)rDV? zCox6tmvleny{HI&cZ0bq{UWpO2+k`yh@eu%XL4+I6gM2LVq3Ap=2W1+fhcPk&0Ddz z>dH!gT!`_M|AsmeZdc&}JA$i{^Kqa;lTl-3#Gf(ymYhTgLb?DOv#iJ&l&er(^QCNy z=(76_*;pRQqYtzMe$D9D4>_nC%5mBq7~ z@i+5+Iw9Rf=3Dq`VFDA7eAAhsI-~$2$26b1wxF5{QCcFYq~P3Hzh34-_Q(Co)ZdaW zR#y|@w~|Gt!BIc?fGz&!y_x^3qg1w&#b;qBOvI<=(7hGIyf5V+ft^{7Fa{{*?8%lT z?2~rci&^bo{)7Xo#o!{5)ezRp%W;-FLXD>QSS3K)(B&{~V^bYw6w}?9_khm?&7+Br z{u<%vuPBA(He26@_Kl}>{eA<_zw`uT#xHkBZk59t*0rVLvtcklDHAM?in6lOa!k>a zTPF>Z4R~ic`a)q!(8I6~jQ+ot$7~FM@)s3h=uyNw{+y!8@G)udV$Qb84$o%+zLgOz z%5_3=1@pxMTzD%Sb&I3u?kRKN)x)q^X2Anuz92dCOB8$!t*E4LIRv(ZDR<-RdKh+H zvNbF%Qs$WEV`XKo*^*_RJ9}P+PtPwtL4vV3&3hzU6o+}U>OarIr1Xn(8BRSPsj?ni zOtM|H`Z38{>ag{%uMp$Jf`Ab|A14Y0Op+S$)oQkXKLSUv7~h%f+oZ+%8s;l37y*bc zJ!2J(KCEqnY4pz)gqa}-GB~cU;sF32?{xG6!!eU&+>`&?iI5$Y08`1kF&OItNL~RwIEraWhiV5K5|QQZkQs+Xvwe9Vm>LPXHIh!cNYa4uTZ1*{u89E ztOBP2gV{jEzfws~7T;g(uPxvp%|VzBLaZwxr3uWs!@}d+y|VR7TY}p`2%>iM<@Oy%0+(L3Lg7i(;`MOHe09&-!#?{|}kU BX;J_H diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index 432a528bb..a2dbff785 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -5,20 +5,13 @@ #define ENABLE_ASSERTIONS 1 #define DO_PROFILING 1 -#define DEBUG 1 +//#define DEBUG 1 //#define DEBUG0_1 1 //#define DEBUG3 1 #include "template.h" #include "utils.h" - -#include "define-float.h" -#include "avx_function_prototypes.h" - -#include "define-double.h" -#include "avx_function_prototypes.h" - using namespace std; class LoadTimeInitializer @@ -33,32 +26,18 @@ class LoadTimeInitializer m_sumSquareNumHaplotypes = 0; m_sumNumTestcases = 0; m_sumSquareNumTestcases = 0; + m_maxNumTestcases = 0; m_num_invocations = 0; + m_filename_to_fptr.clear(); - if(is_avx_supported()) - { - cout << "Using AVX accelerated implementation of PairHMM\n"; - g_compute_full_prob_float = compute_full_prob_avxs; - g_compute_full_prob_double = compute_full_prob_avxd; - } - else - { - cout << "Using un-vectorized C++ implementation of PairHMM\n"; - g_compute_full_prob_float = compute_full_prob; - g_compute_full_prob_double = compute_full_prob; - } + initialize_function_pointers(); cout.flush(); - //if(is_sse42_supported()) - //{ - //g_compute_full_prob_float = compute_full_prob_avxs; - //g_compute_full_prob_double = compute_full_prob_avxd; - //} } void print_profiling() { double mean_val; cout <<"Invocations : "<::iterator mI = m_filename_to_fptr.find(filename); + ofstream* fptr = 0; + if(mI == m_filename_to_fptr.end()) + { + m_filename_to_fptr[filename] = new ofstream(); + fptr = m_filename_to_fptr[filename]; + fptr->open(filename.c_str(), to_append ? ios::app : ios::out); + assert(fptr->is_open()); + } + else + fptr = (*mI).second; + //ofstream fptr; + //fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); + (*fptr) << s; + if(add_newline) + (*fptr) << "\n"; + //fptr.close(); + } + void debug_close() + { + for(map::iterator mB = m_filename_to_fptr.begin(), mE = m_filename_to_fptr.end(); + mB != mE;mB++) + { + (*mB).second->close(); + delete (*mB).second; + } + m_filename_to_fptr.clear(); } jfieldID m_readBasesFID; jfieldID m_readQualsFID; @@ -89,8 +93,10 @@ class LoadTimeInitializer double m_sumSquareNumHaplotypes; double m_sumNumTestcases; double m_sumSquareNumTestcases; + unsigned m_maxNumTestcases; unsigned m_num_invocations; - + private: + map m_filename_to_fptr; }; LoadTimeInitializer g_load_time_initializer; @@ -174,10 +180,10 @@ Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadL tc.c[i] = (int)overallGCPArray[i]; } - double result_avxd = GEN_INTRINSIC(compute_full_prob_avx, d)(&tc); + double result_avxd = g_compute_full_prob_double(&tc, 0); double result = log10(result_avxd) - log10(ldexp(1.0, 1020)); #ifdef DEBUG - debug_dump("return_values_jni.txt",to_string(result),true); + g_load_time_initializer.debug_dump("return_values_jni.txt",to_string(result),true); #endif @@ -255,7 +261,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai haplotypeBasesArrayVector[j] = make_pair(haplotypeBases, haplotypeBasesArray); #ifdef DEBUG3 for(unsigned k=0;kGetArrayLength(haplotypeBases);++k) - debug_dump("haplotype_bases_jni.txt",to_string((int)haplotypeBasesArray[k]),true); + g_load_time_initializer.debug_dump("haplotype_bases_jni.txt",to_string((int)haplotypeBasesArray[k]),true); #endif } @@ -310,11 +316,11 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai #ifdef DEBUG3 for(unsigned j=0;j g_load_time_initializer.m_maxNumTestcases ? numTestCases + : g_load_time_initializer.m_maxNumTestcases; ++(g_load_time_initializer.m_num_invocations); #endif +#ifdef DEBUG + g_load_time_initializer.debug_close(); +#endif +} + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniClose + (JNIEnv* env, jobject thisObject) +{ +#ifdef DO_PROFILING + g_load_time_initializer.print_profiling(); +#endif +#ifdef DEBUG + g_load_time_initializer.debug_close(); +#endif } diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h index 1fff9fa5a..b50f6a5a9 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h @@ -70,6 +70,14 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniComputeLikelihoods (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniClose + (JNIEnv *, jobject); + #ifdef __cplusplus } #endif diff --git a/PairHMM_JNI/pairhmm-1-base.cc b/PairHMM_JNI/pairhmm-1-base.cc index a3b5e0ad5..3030b640e 100644 --- a/PairHMM_JNI/pairhmm-1-base.cc +++ b/PairHMM_JNI/pairhmm-1-base.cc @@ -5,12 +5,6 @@ #include "template.h" #include "utils.h" -#include "define-float.h" -#include "avx_function_prototypes.h" - -#include "define-double.h" -#include "avx_function_prototypes.h" - using namespace std; class LoadTimeInitializer { @@ -36,16 +30,7 @@ int main(int argc, char** argv) if(argc >= 3 && string(argv[2]) == "1") use_old_read_testcase = true; - if(true) - { - g_compute_full_prob_double = GEN_INTRINSIC(compute_full_prob_avx, d); - g_compute_full_prob_float = GEN_INTRINSIC(compute_full_prob_avx, s); - } - else - { - g_compute_full_prob_double = compute_full_prob; - g_compute_full_prob_float = compute_full_prob; - } + initialize_function_pointers(); std::ifstream ifptr; FILE* fptr = 0; @@ -86,70 +71,5 @@ int main(int argc, char** argv) else ifptr.close(); return 0; - -#if 0 - float result[BATCH_SIZE], result_avxf; - double result_avxd; - struct timeval start, end; - long long aggregateTimeRead = 0L; - long long aggregateTimeCompute = 0L; - long long aggregateTimeWrite = 0L; - - bool noMoreData = false; - int count =0; - while (!noMoreData) - { - int read_count = BATCH_SIZE; - gettimeofday(&start, NULL); - for (int b=0;b(&tc[b]); - -#ifdef RUN_HYBRID -#define MIN_ACCEPTED 1e-28f - if (result_avxf < MIN_ACCEPTED) { - count++; - result_avxd = compute_full_prob(&tc[b]); - result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); - } - else - result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); -#endif - -#ifndef RUN_HYBRID - result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); -#endif - - } - gettimeofday(&end, NULL); - aggregateTimeCompute += ((end.tv_sec * 1000000 + end.tv_usec) - (start.tv_sec * 1000000 + start.tv_usec)); - - gettimeofday(&start, NULL); - for (int b=0;b #include +//#define DEBUG +#define MUSTAFA +#define KARTHIK + /* template string getBinaryStr (T val, int numBitsToWrite) { @@ -17,8 +21,9 @@ string getBinaryStr (T val, int numBitsToWrite) { return oss.str() ; } */ +#ifdef MUSTAFA -void GEN_INTRINSIC(precompute_masks_, PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]) { +void GEN_INTRINSIC(GEN_INTRINSIC(precompute_masks_,SIMD_TYPE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]) { const int maskBitCnt = MAIN_TYPE_SIZE ; @@ -52,7 +57,7 @@ void GEN_INTRINSIC(precompute_masks_, PRECISION)(const testcase& tc, int COLS, i } -void GEN_INTRINSIC(init_masks_for_row_, PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess) { +void GEN_INTRINSIC(GEN_INTRINSIC(init_masks_for_row_,SIMD_TYPE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess) { for (int ri=0; ri < numRowsToProcess; ++ri) { rsArr[ri] = ConvertChar::get(tc.rs[ri+beginRowIndex-1]) ; @@ -63,6 +68,7 @@ void GEN_INTRINSIC(init_masks_for_row_, PRECISION)(const testcase& tc, char* rsA } } + #define SET_MASK_WORD(__dstMask, __srcMask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ MASK_TYPE __bitMask = (((MASK_TYPE)0x1) << __shiftBy) - 1 ; \ MASK_TYPE __nextShiftOut = (__srcMask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ @@ -71,33 +77,43 @@ void GEN_INTRINSIC(init_masks_for_row_, PRECISION)(const testcase& tc, char* rsA } -void GEN_INTRINSIC(update_masks_for_cols_, PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt) { +void GEN_INTRINSIC(GEN_INTRINSIC(update_masks_for_cols_, SIMD_TYPE), PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt) { for (int ei=0; ei < AVX_LENGTH/2; ++ei) { SET_MASK_WORD(currMaskVecLow.masks[ei], maskArr[maskIndex][rsArr[ei]], - lastMaskShiftOut[ei], ei, maskBitCnt) ; - + lastMaskShiftOut[ei], ei, maskBitCnt) ; + int ei2 = ei + AVX_LENGTH/2 ; // the second entry index SET_MASK_WORD(currMaskVecHigh.masks[ei], maskArr[maskIndex][rsArr[ei2]], - lastMaskShiftOut[ei2], ei2, maskBitCnt) ; + lastMaskShiftOut[ei2], ei2, maskBitCnt) ; } } + //void GEN_INTRINSIC(computeDistVec, PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen, const _256_TYPE& distmSel, int firstRowIndex, int lastRowIndex) { -inline void GEN_INTRINSIC(computeDistVec, PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen) { +inline void GEN_INTRINSIC(GEN_INTRINSIC(computeDistVec, SIMD_TYPE), PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen) { //#define computeDistVec() { _256_TYPE maskV ; VEC_SSE_TO_AVX(currMaskVecLow.vecf, currMaskVecHigh.vecf, maskV) ; +#ifdef DEBUGG + long long *temp1 = (long long *)(&maskV); + double *temp2 = (double *)(&distm); + double *temp3 = (double *)(&_1_distm); + printf("***\n%lx\n%lx\n%f\n%f\n%f\n%f\n***\n", temp1[0], temp1[1], temp2[0], temp2[1], temp3[0], temp3[1]); +#endif + distmChosen = VEC_BLENDV(distm, _1_distm, maskV) ; /*COMPARE_VECS(distmChosen, distmSel, firstRowIndex, lastRowIndex) ;*/ VEC_SHIFT_LEFT_1BIT(currMaskVecLow.vec) ; VEC_SHIFT_LEFT_1BIT(currMaskVecHigh.vec) ; + + _mm_empty(); } @@ -120,8 +136,9 @@ struct HmmData { } ; */ +#endif // MUSTAFA -template void GEN_INTRINSIC(initializeVectors, PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D) +template void GEN_INTRINSIC(GEN_INTRINSIC(initializeVectors, SIMD_TYPE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D) { NUMBER zero = ctx._(0.0); NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); @@ -157,18 +174,14 @@ template void GEN_INTRINSIC(initializeVectors, PRECISION)(int ROWS *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; - *(ptr_p_MY+r-1) = ctx.ph2pr[_d]; - *(ptr_p_YY+r-1) = ctx.ph2pr[_c]; -#ifdef DEBUG3 - debug_dump("transitions_jni.txt",to_string(*(ptr_p_MM+r-1) ),true); - debug_dump("transitions_jni.txt",to_string(*(ptr_p_GAPM+r-1)),true); - debug_dump("transitions_jni.txt",to_string(*(ptr_p_MX+r-1) ),true); - debug_dump("transitions_jni.txt",to_string(*(ptr_p_XX+r-1) ),true); - debug_dump("transitions_jni.txt",to_string(*(ptr_p_MY+r-1) ),true); - debug_dump("transitions_jni.txt",to_string(*(ptr_p_YY+r-1) ),true); -#endif - //*(ptr_p_MY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; - //*(ptr_p_YY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + #ifdef KARTHIK + *(ptr_p_MY+r-1) = ctx.ph2pr[_d]; + *(ptr_p_YY+r-1) = ctx.ph2pr[_c]; + #else + *(ptr_p_MY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + *(ptr_p_YY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + #endif + } NUMBER *ptr_distm1D = (NUMBER *)distm1D; @@ -176,14 +189,11 @@ template void GEN_INTRINSIC(initializeVectors, PRECISION)(int ROWS { int _q = tc->q[r-1] & 127; ptr_distm1D[r-1] = ctx.ph2pr[_q]; -#ifdef DEBUG3 - debug_dump("priors_jni.txt",to_string(ptr_distm1D[r-1]),true); -#endif } } -template inline void GEN_INTRINSIC(stripINITIALIZATION, PRECISION)( +template inline void GEN_INTRINSIC(GEN_INTRINSIC(stripINITIALIZATION, SIMD_TYPE), PRECISION)( int stripIdx, Context ctx, testcase *tc, _256_TYPE &pGAPM, _256_TYPE &pMM, _256_TYPE &pMX, _256_TYPE &pXX, _256_TYPE &pMY, _256_TYPE &pYY, _256_TYPE &rs, UNION_TYPE &rsN, _256_TYPE &distm, _256_TYPE &_1_distm, _256_TYPE *distm1D, _256_TYPE N_packed256, _256_TYPE *p_MM , _256_TYPE *p_GAPM , _256_TYPE *p_MX, _256_TYPE *p_XX , _256_TYPE *p_MY, _256_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, @@ -200,19 +210,18 @@ template inline void GEN_INTRINSIC(stripINITIALIZATION, PRECISION) NUMBER zero = ctx._(0.0); NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); -#define TRISTATE_CORRECTION_FACTOR 3.0 - UNION_TYPE packed3; packed3.d = VEC_SET1_VAL(TRISTATE_CORRECTION_FACTOR); - + UNION_TYPE packed3; packed3.d = VEC_SET1_VAL(3.0); /* compare rs and N */ - //rs = VEC_LDPOPCVT_CHAR((tc->irs+i*AVX_LENGTH)); - //rsN.d = VEC_CMP_EQ(N_packed256, rs); - +#ifndef MUSTAFA + rs = VEC_LDPOPCVT_CHAR((tc->irs+i*AVX_LENGTH)); + rsN.d = VEC_CMP_EQ(N_packed256, rs); +#endif distm = distm1D[i]; - _1_distm = VEC_SUB(packed1.d, distm); -#ifndef DO_NOT_USE_TRISTATE_CORRECTION - distm = VEC_DIV(distm, packed3.d); -#endif - + _1_distm = VEC_SUB(packed1.d, distm); + + #ifdef KARTHIK + distm = VEC_DIV(distm, packed3.d); + #endif /* initialize M_t_2, M_t_1, X_t_2, X_t_1, Y_t_2, Y_t_1 */ M_t_2.d = VEC_SET1_VAL(zero); X_t_2.d = VEC_SET1_VAL(zero); @@ -234,7 +243,7 @@ template inline void GEN_INTRINSIC(stripINITIALIZATION, PRECISION) -inline _256_TYPE GEN_INTRINSIC(computeDISTM, PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, +inline _256_TYPE GEN_INTRINSIC(GEN_INTRINSIC(computeDISTM, SIMD_TYPE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, _256_TYPE distm, _256_TYPE _1_distm) { UNION_TYPE hapN, rshap; @@ -263,12 +272,21 @@ inline _256_TYPE GEN_INTRINSIC(computeDISTM, PRECISION)(int d, int COLS, testcas } -inline void GEN_INTRINSIC(computeMXY, PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, +inline void GEN_INTRINSIC(GEN_INTRINSIC(computeMXY, SIMD_TYPE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, _256_TYPE pMM, _256_TYPE pGAPM, _256_TYPE pMX, _256_TYPE pXX, _256_TYPE pMY, _256_TYPE pYY, _256_TYPE distmSel) { /* Compute M_t <= distm * (p_MM*M_t_2 + p_GAPM*X_t_2 + p_GAPM*Y_t_2) */ M_t.d = VEC_MUL(VEC_ADD(VEC_ADD(VEC_MUL(M_t_2.d, pMM), VEC_MUL(X_t_2.d, pGAPM)), VEC_MUL(Y_t_2.d, pGAPM)), distmSel); + +#ifdef DEBUG + double *temp1 = (double *)(&pGAPM); + double *temp2 = (double *)(&pMM); + double *temp3 = (double *)(&distmSel); + printf("%f\n%f\n%f\n%f\n%f\n%f\n", temp1[0], temp1[1], temp2[0], temp2[1], temp3[0], temp3[1]); + //printf("%f\n%f\n%f\n%f\n", X_t_2.f[0], X_t_2.f[1], Y_t_2.f[0], Y_t_2.f[1]); + printf("%f\n%f\n----------------------------------------------------------------------------\n", M_t.f[0], M_t.f[1]); +#endif M_t_y = M_t; /* Compute X_t */ @@ -278,7 +296,7 @@ inline void GEN_INTRINSIC(computeMXY, PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_ Y_t.d = VEC_ADD(VEC_MUL(M_t_1_y.d, pMY) , VEC_MUL(Y_t_1.d, pYY)); } -template NUMBER GEN_INTRINSIC(compute_full_prob_avx, PRECISION) (testcase *tc, NUMBER *before_last_log = NULL) +template NUMBER GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_,SIMD_TYPE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL) { _256_TYPE p_MM [MAVX_COUNT], p_GAPM [MAVX_COUNT], p_MX [MAVX_COUNT]; _256_TYPE p_XX [MAVX_COUNT], p_MY [MAVX_COUNT], p_YY [MAVX_COUNT]; @@ -306,54 +324,51 @@ template NUMBER GEN_INTRINSIC(compute_full_prob_avx, PRECISION) (t int remainingRows = (ROWS-1) % AVX_LENGTH; int strip_cnt = ((ROWS-1) / AVX_LENGTH) + (remainingRows!=0); +#ifdef MUSTAFA const int maskBitCnt = MAIN_TYPE_SIZE ; const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function MASK_TYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; - GEN_INTRINSIC(precompute_masks_, PRECISION)(*tc, COLS, numMaskVecs, maskArr) ; + GEN_INTRINSIC(GEN_INTRINSIC(precompute_masks_,SIMD_TYPE), PRECISION)(*tc, COLS, numMaskVecs, maskArr) ; char rsArr[AVX_LENGTH] ; MASK_TYPE lastMaskShiftOut[AVX_LENGTH] ; - - GEN_INTRINSIC(initializeVectors, PRECISION)(ROWS, COLS, shiftOutM, shiftOutX, shiftOutY, +#endif + GEN_INTRINSIC(GEN_INTRINSIC(initializeVectors,SIMD_TYPE), PRECISION)(ROWS, COLS, shiftOutM, shiftOutX, shiftOutY, ctx, tc, p_MM, p_GAPM, p_MX, p_XX, p_MY, p_YY, distm1D); //for (int __ii=0; __ii < 10; ++__ii) for (int i=0;i NUMBER GEN_INTRINSIC(compute_full_prob_avx, PRECISION) (t int i = strip_cnt-1; { //STRIP_INITIALIZATION - GEN_INTRINSIC(stripINITIALIZATION, PRECISION)(i, ctx, tc, pGAPM, pMM, pMX, pXX, pMY, pYY, rs.d, rsN, distm, _1_distm, distm1D, N_packed256, p_MM , p_GAPM , + GEN_INTRINSIC(GEN_INTRINSIC(stripINITIALIZATION,SIMD_TYPE), PRECISION)(i, ctx, tc, pGAPM, pMM, pMX, pXX, pMY, pYY, rs.d, rsN, distm, _1_distm, distm1D, N_packed256, p_MM , p_GAPM , p_MX, p_XX , p_MY, p_YY, M_t_2, X_t_2, M_t_1, X_t_1, Y_t_2, Y_t_1, M_t_1_y, shiftOutX, shiftOutM); if (remainingRows==0) remainingRows=AVX_LENGTH; - - GEN_INTRINSIC(init_masks_for_row_, PRECISION)(*tc, rsArr, lastMaskShiftOut, - i*AVX_LENGTH+1, remainingRows) ; - +#ifdef MUSTAFA + GEN_INTRINSIC(GEN_INTRINSIC(init_masks_for_row_,SIMD_TYPE), PRECISION)(*tc, rsArr, lastMaskShiftOut, i*AVX_LENGTH+1, remainingRows) ; +#endif _256_TYPE sumM, sumX; sumM = VEC_SET1_VAL(zero); sumX = VEC_SET1_VAL(zero); @@ -382,24 +396,25 @@ template NUMBER GEN_INTRINSIC(compute_full_prob_avx, PRECISION) (t for (int d=1;d #include "headers.h" -#include -#include -#include #include "template.h" +#include "vector_defs.h" -#include "define-float.h" -#include "shift_template.c" -#include "avx_function_prototypes.h" -#include "define-double.h" -#include "shift_template.c" -#include "avx_function_prototypes.h" +#define BATCH_SIZE 10 +#define RUN_HYBRID -#define BATCH_SIZE 10000 -//#define RUN_HYBRID - -//uint8_t ConvertChar::conversionTable[255] ; int thread_level_parallelism_enabled = false ; double getCurrClk() { @@ -26,11 +15,45 @@ double getCurrClk() { return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; } +void print128b_F(__m128 x) +{ + float *p = (float *)(&x); + for (int i=3;i>=0;i--) + printf("%f ", *(p+i)); + printf("\n"); +} + +void print128b_D(__m128d x) +{ + double *p = (double *)(&x); + for (int i=1;i>=0;i--) + printf("%f ", *(p+i)); + printf("\n"); +} int main() { - - testcase tc[BATCH_SIZE]; +/* + IF_128f x; + x.f = _mm_set_ps(1.0, 2.0, 3.0, 4.0); + IF_32 shiftIn, shiftOut; + shiftIn.f = 5.0f; + print128b_F(x.f); + GEN_INTRINSIC(_vector_shift, s)(x, shiftIn, shiftOut); + print128b_F(x.f); + + IF_128d y; + y.f = _mm_set_pd(10.0, 11.0); + IF_64 shiftInd, shiftOutd; + shiftInd.f = 12.0; + print128b_D(y.f); + GEN_INTRINSIC(_vector_shift, d)(y, shiftInd, shiftOutd); + print128b_D(y.f); + + exit(0); +*/ + + testcase tc[BATCH_SIZE]; float result[BATCH_SIZE], result_avxf; double result_avxd; //struct timeval start, end; @@ -41,12 +64,12 @@ int main() // Need to call it once to initialize the static array ConvertChar::init() ; + - - char* ompEnvVar = getenv("OMP_NUM_THREADS") ; - if (ompEnvVar != NULL && ompEnvVar != "" && ompEnvVar != "1" ) { - thread_level_parallelism_enabled = true ; - } +// char* ompEnvVar = getenv("OMP_NUM_THREADS") ; +// if (ompEnvVar != NULL && ompEnvVar != "" && ompEnvVar != "1" ) { +// thread_level_parallelism_enabled = true ; +// } bool noMoreData = false; int count =0; @@ -56,7 +79,7 @@ int main() lastClk = getCurrClk() ; for (int b=0;b(&tc[b]); + result_avxf = GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_, SIMD_TYPE), s)(&tc[b]); #ifdef RUN_HYBRID #define MIN_ACCEPTED 1e-28f if (result_avxf < MIN_ACCEPTED) { + //printf("**************** RUNNING DOUBLE ******************\n"); count++; - result_avxd = GEN_INTRINSIC(compute_full_prob_avx, d)(&tc[b]); + result_avxd = GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_, SIMD_TYPE), d)(&tc[b]); result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); } else diff --git a/PairHMM_JNI/run.sh b/PairHMM_JNI/run.sh index 495a3761b..2bc45efad 100755 --- a/PairHMM_JNI/run.sh +++ b/PairHMM_JNI/run.sh @@ -2,18 +2,21 @@ rm -f *.txt export GSA_ROOT_DIR=/home/karthikg/broad/gsa-unstable #-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed -java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI -jar ${GSA_ROOT_DIR}/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ +java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ -R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ --I /data/broad/samples/joint_variant_calling/NA12878_low_coverage_alignment/NA12878.chrom11.ILLUMINA.bwa.CEU.low_coverage.20121211.bam \ +-I /data/broad/samples/joint_variant_calling/NA12878_high_coverage_alignment/NA12878.mapped.ILLUMINA.bwa.CEU.high_coverage_pcr_free.20130906.bam \ --dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -stand_call_conf 50.0 \ -stand_emit_conf 10.0 \ --pair_hmm_implementation JNI_LOGLESS_CACHING \ +-XL unmapped \ -o output.raw.snps.indels.vcf #--pair_hmm_implementation JNI_LOGLESS_CACHING \ #-I /data/simulated/sim1M_pairs_final.bam \ #-I /data/broad/samples/joint_variant_calling/NA12878_low_coverage_alignment/NA12878.chrom11.ILLUMINA.bwa.CEU.low_coverage.20121211.bam \ +#-I /data/broad/samples/joint_variant_calling/NA12878_high_coverage_alignment/NA12878.mapped.ILLUMINA.bwa.CEU.high_coverage_pcr_free.20130906.bam \ +#-R /data/broad/samples/joint_variant_calling/broad_reference/Homo_sapiens_assembly18.fasta \ #-R /data/broad/samples/joint_variant_calling/broad_reference/Homo_sapiens_assembly19.fasta \ #-R /data/broad/samples/joint_variant_calling/broad_reference/ucsc.hg19.fasta \ #-R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ diff --git a/PairHMM_JNI/shift_template.c b/PairHMM_JNI/shift_template.c index 90160dcb3..db4a4f6f7 100644 --- a/PairHMM_JNI/shift_template.c +++ b/PairHMM_JNI/shift_template.c @@ -1,9 +1,8 @@ #ifdef PRECISION -#include "template.h" +#ifdef SIMD_TYPE_AVX - -inline void GEN_INTRINSIC(_vector_shift, PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut) +inline void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift,SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut) { IF_128 xlow , xhigh; @@ -33,7 +32,8 @@ inline void GEN_INTRINSIC(_vector_shift, PRECISION) (UNION_TYPE &x, MAIN_TYPE sh x.d = VEC_INSERT_VAL(x.d, xhigh.f, 1); } -inline void GEN_INTRINSIC(_vector_shift_last, PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn) + +inline void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift_last, SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn) { IF_128 xlow , xhigh; @@ -44,10 +44,6 @@ inline void GEN_INTRINSIC(_vector_shift_last, PRECISION) (UNION_TYPE &x, MAIN_TY /* extract xlow[3] */ IF_128 shiftOutL128; shiftOutL128.i = _mm_srli_si128(xlow.i, SHIFT_CONST1); - /* extract xhigh[3] */ -// IF_MAIN_TYPE shiftOutH; -// shiftOutH.i = VEC_EXTRACT_UNIT(xhigh.i, SHIFT_CONST2); -// shiftOut = shiftOutH.f; /* shift xlow */ xlow.i = _mm_slli_si128 (xlow.i, SHIFT_CONST3); /* shift xhigh */ @@ -63,6 +59,32 @@ inline void GEN_INTRINSIC(_vector_shift_last, PRECISION) (UNION_TYPE &x, MAIN_TY x.d = VEC_INSERT_VAL(x.d, xhigh.f, 1); } +#endif + +#ifdef SIMD_TYPE_SSE + +inline void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift, SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut) +{ + IF_MAIN_TYPE tempIn, tempOut; + tempIn.f = shiftIn; + /* extratc H */ + tempOut.i = VEC_EXTRACT_UNIT(x.i, SHIFT_CONST1); + shiftOut = tempOut.f; + /* shift */ + x.i = _mm_slli_si128(x.i, SHIFT_CONST2); + /* insert L */ + x.i = VEC_INSERT_UNIT(x.i , tempIn.i, SHIFT_CONST3); +} + +inline void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift_last, SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn) +{ + IF_MAIN_TYPE temp; temp.f = shiftIn; + /* shift */ + x.i = _mm_slli_si128(x.i, SHIFT_CONST2); + /* insert L */ + x.i = VEC_INSERT_UNIT(x.i , temp.i, SHIFT_CONST3); +} #endif +#endif diff --git a/PairHMM_JNI/sse_function_instantiations.cc b/PairHMM_JNI/sse_function_instantiations.cc new file mode 100644 index 000000000..38686ada2 --- /dev/null +++ b/PairHMM_JNI/sse_function_instantiations.cc @@ -0,0 +1,18 @@ +#include "template.h" + +#undef SIMD_TYPE +#undef SIMD_TYPE_AVX + +#define SIMD_TYPE sse +#define SIMD_TYPE_SSE + +#include "define-sse-float.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +#include "define-sse-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +template double compute_full_prob_ssed(testcase* tc, double* nextlog); +template float compute_full_prob_sses(testcase* tc, float* nextlog); diff --git a/PairHMM_JNI/template.h b/PairHMM_JNI/template.h index 52a4e4650..0ba4f843d 100644 --- a/PairHMM_JNI/template.h +++ b/PairHMM_JNI/template.h @@ -25,12 +25,25 @@ typedef union __attribute__((aligned(32))) { ALIGNED __m256i ALIGNED i; } ALIGNED mix_F ALIGNED; +typedef union __attribute__((aligned(32))) { + ALIGNED __m128 ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED float ALIGNED f[4]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_F128 ALIGNED; + typedef union ALIGNED { __m128i vec ; __m128 vecf ; uint32_t masks[4] ; } MaskVec_F ; +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint32_t masks[2] ; +} MaskVec_F128 ; + typedef union ALIGNED { ALIGNED __m128i ALIGNED i; @@ -50,12 +63,25 @@ typedef union __attribute__((aligned(32))) { ALIGNED __m256i ALIGNED i; } ALIGNED mix_D ALIGNED; +typedef union __attribute__((aligned(32))) { + ALIGNED __m128d ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED double ALIGNED f[2]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_D128 ALIGNED; + typedef union ALIGNED { __m128i vec ; __m128d vecf ; uint64_t masks[2] ; } MaskVec_D ; +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint64_t masks[1] ; +} MaskVec_D128 ; + typedef union ALIGNED { ALIGNED __m128i ALIGNED i; @@ -120,7 +146,6 @@ struct Context }; - typedef struct { int rslen, haplen; @@ -131,24 +156,11 @@ typedef struct int *irs; } testcase; - -template -std::string to_string(T obj) -{ - std::stringstream ss; - std::string ret_string; - ss.clear(); - ss << std::scientific << obj; - ss >> ret_string; - ss.clear(); - return ret_string; -} -void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); - int normalize(char c); -int read_testcase(testcase *tc, FILE* ifp); -int read_mod_testcase(std::ifstream& fptr, testcase* tc, bool reformat=false); +int read_testcase(testcase *tc, FILE* ifp=0); + +#define MIN_ACCEPTED 1e-28f #define NUM_DISTINCT_CHARS 5 #define AMBIG_CHAR 4 @@ -176,7 +188,6 @@ public: }; - #endif diff --git a/PairHMM_JNI/utils.cc b/PairHMM_JNI/utils.cc index 5b91fc7e9..94e2144c0 100644 --- a/PairHMM_JNI/utils.cc +++ b/PairHMM_JNI/utils.cc @@ -1,5 +1,7 @@ #include "headers.h" #include "template.h" +#include "utils.h" +#include "vector_defs.h" uint8_t ConvertChar::conversionTable[255]; float (*g_compute_full_prob_float)(testcase *tc, float* before_last_log) = 0; @@ -31,6 +33,30 @@ bool is_sse42_supported() return ((ecx >> 20)&1) == 1; } +void initialize_function_pointers() +{ + //if(is_avx_supported()) + if(false) + { + cout << "Using AVX accelerated implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob_avxs; + g_compute_full_prob_double = compute_full_prob_avxd; + } + else + if(is_sse42_supported()) + { + cout << "Using SSE4.2 accelerated implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob_sses; + g_compute_full_prob_double = compute_full_prob_ssed; + } + else + { + cout << "Using un-vectorized C++ implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob; + g_compute_full_prob_double = compute_full_prob; + } +} + int normalize(char c) { return ((int) (c - 33)); @@ -214,12 +240,3 @@ int read_mod_testcase(ifstream& fptr, testcase* tc, bool reformat) return tokens.size(); } -void debug_dump(string filename, string s, bool to_append, bool add_newline) -{ - ofstream fptr; - fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); - fptr << s; - if(add_newline) - fptr << "\n"; - fptr.close(); -} diff --git a/PairHMM_JNI/utils.h b/PairHMM_JNI/utils.h index 8c244333e..8da8269b8 100644 --- a/PairHMM_JNI/utils.h +++ b/PairHMM_JNI/utils.h @@ -1,7 +1,22 @@ #ifndef PAIRHMM_UTIL_H #define PAIRHMM_UTIL_H -#define MIN_ACCEPTED 1e-28f + +template +std::string to_string(T obj) +{ + std::stringstream ss; + std::string ret_string; + ss.clear(); + ss << std::scientific << obj; + ss >> ret_string; + ss.clear(); + return ret_string; +} +void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); + +int read_mod_testcase(std::ifstream& fptr, testcase* tc, bool reformat=false); + bool is_avx_supported(); bool is_sse42_supported(); extern float (*g_compute_full_prob_float)(testcase *tc, float *before_last_log); @@ -9,5 +24,5 @@ extern double (*g_compute_full_prob_double)(testcase *tc, double* before_last_lo void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline); template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); - +void initialize_function_pointers(); #endif diff --git a/PairHMM_JNI/vector_defs.h b/PairHMM_JNI/vector_defs.h new file mode 100644 index 000000000..c89f7f932 --- /dev/null +++ b/PairHMM_JNI/vector_defs.h @@ -0,0 +1,26 @@ +#undef SIMD_TYPE +#undef SIMD_TYPE_AVX +#undef SIMD_TYPE_SSE + +#define SIMD_TYPE avx +#define SIMD_TYPE_AVX + +#include "define-float.h" +#include "vector_function_prototypes.h" + +#include "define-double.h" +#include "vector_function_prototypes.h" + +#undef SIMD_TYPE +#undef SIMD_TYPE_AVX + +#define SIMD_TYPE sse +#define SIMD_TYPE_SSE + +#include "define-sse-float.h" +#include "vector_function_prototypes.h" + +#include "define-sse-double.h" +#include "vector_function_prototypes.h" + + diff --git a/PairHMM_JNI/vector_function_prototypes.h b/PairHMM_JNI/vector_function_prototypes.h new file mode 100644 index 000000000..2593986a6 --- /dev/null +++ b/PairHMM_JNI/vector_function_prototypes.h @@ -0,0 +1,19 @@ +void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift,SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); +void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift_last,SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); +void GEN_INTRINSIC(GEN_INTRINSIC(precompute_masks_,SIMD_TYPE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); +void GEN_INTRINSIC(GEN_INTRINSIC(init_masks_for_row_,SIMD_TYPE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess); +void GEN_INTRINSIC(GEN_INTRINSIC(update_masks_for_cols_,SIMD_TYPE), PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt); +void GEN_INTRINSIC(GEN_INTRINSIC(computeDistVec,SIMD_TYPE), PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen); +template void GEN_INTRINSIC(GEN_INTRINSIC(initializeVectors,SIMD_TYPE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D); +template void GEN_INTRINSIC(GEN_INTRINSIC(stripINITIALIZATION,SIMD_TYPE), PRECISION)( + int stripIdx, Context ctx, testcase *tc, _256_TYPE &pGAPM, _256_TYPE &pMM, _256_TYPE &pMX, _256_TYPE &pXX, _256_TYPE &pMY, _256_TYPE &pYY, + _256_TYPE &rs, UNION_TYPE &rsN, _256_TYPE &distm, _256_TYPE &_1_distm, _256_TYPE *distm1D, _256_TYPE N_packed256, _256_TYPE *p_MM , _256_TYPE *p_GAPM , + _256_TYPE *p_MX, _256_TYPE *p_XX , _256_TYPE *p_MY, _256_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM); +_256_TYPE GEN_INTRINSIC(GEN_INTRINSIC(computeDISTM,SIMD_TYPE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, + _256_TYPE distm, _256_TYPE _1_distm); +void GEN_INTRINSIC(GEN_INTRINSIC(computeMXY,SIMD_TYPE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, + UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, + _256_TYPE pMM, _256_TYPE pGAPM, _256_TYPE pMX, _256_TYPE pXX, _256_TYPE pMY, _256_TYPE pYY, _256_TYPE distmSel); +template NUMBER GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_,SIMD_TYPE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL); + diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java index 82015d153..b785b8d21 100644 --- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java +++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java @@ -1086,7 +1086,7 @@ public class HaplotypeCaller extends ActiveRegionWalker, In referenceConfidenceModel.close(); //TODO remove the need to call close here for debugging, the likelihood output stream should be managed //TODO (open & close) at the walker, not the engine. - //likelihoodCalculationEngine.close(); + likelihoodCalculationEngine.close(); logger.info("Ran local assembly on " + result + " active regions"); } diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java index 0626f2268..04e64186f 100644 --- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java +++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/LikelihoodCalculationEngine.java @@ -89,4 +89,6 @@ public interface LikelihoodCalculationEngine { */ public Map computeReadLikelihoods(AssemblyResultSet assemblyResultSet, Map> perSampleReadList); + + public void close(); } diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index 0a82e1997..5ab49b531 100644 --- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -165,8 +165,10 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation } } + @Override public void close() { if ( likelihoodsStream != null ) likelihoodsStream.close(); + pairHMMThreadLocal.get().close(); } private void writeDebugLikelihoods(final GATKSAMRecord processedRead, final Haplotype haplotype, final double log10l){ diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java index a80800f82..ca3db5ed3 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java @@ -66,7 +66,8 @@ import java.util.Map; */ public class JNILoglessPairHMM extends LoglessPairHMM { - private static final boolean debug = true; //simulates ifdef + private static final boolean debug = false; //simulates ifdef + private static final boolean verify = debug || false; //simulates ifdef private static final boolean debug0_1 = false; //simulates ifdef private static final boolean debug1 = false; //simulates ifdef private static final boolean debug2 = false; @@ -90,6 +91,7 @@ public class JNILoglessPairHMM extends LoglessPairHMM { } private native void jniGlobalInit(Class readDataHolderClass, Class haplotypeDataHolderClass); + private native void jniClose(); private static boolean isJNILoglessPairHMMLibraryLoaded = false; @@ -104,6 +106,13 @@ public class JNILoglessPairHMM extends LoglessPairHMM { } } + @Override + public void close() + { + jniClose(); + debugClose(); + } + //Used to test parts of the compute kernel separately private native void jniInitialize(final int readMaxLength, final int haplotypeMaxLength); private native static void jniInitializeProbabilities(final double[][] transition, final byte[] insertionGOP, final byte[] deletionGOP, final byte[] overallGCP); @@ -126,7 +135,7 @@ public class JNILoglessPairHMM extends LoglessPairHMM { @Override public void initialize(final int readMaxLength, final int haplotypeMaxLength) { - if(debug) + if(verify) super.initialize(readMaxLength, haplotypeMaxLength); if(debug3) { @@ -150,14 +159,14 @@ public class JNILoglessPairHMM extends LoglessPairHMM { public PerReadAlleleLikelihoodMap computeLikelihoods(final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap) { // (re)initialize the pairHMM only if necessary - final int readMaxLength = debug ? findMaxReadLength(reads) : 0; - final int haplotypeMaxLength = debug ? findMaxHaplotypeLength(alleleHaplotypeMap) : 0; - if(debug) + final int readMaxLength = verify ? findMaxReadLength(reads) : 0; + final int haplotypeMaxLength = verify ? findMaxHaplotypeLength(alleleHaplotypeMap) : 0; + if(verify) { if (!initialized || readMaxLength > maxReadLength || haplotypeMaxLength > maxHaplotypeLength) { initialize(readMaxLength, haplotypeMaxLength); } if ( ! initialized ) - throw new IllegalStateException("Must call initialize before calling jniComputeLikelihoods in debug mode"); + throw new IllegalStateException("Must call initialize before calling jniComputeLikelihoods in debug/verify mode"); } int readListSize = reads.size(); int alleleHaplotypeMapSize = alleleHaplotypeMap.size(); @@ -221,7 +230,7 @@ public class JNILoglessPairHMM extends LoglessPairHMM { likelihoodMap.add(read, currEntry.getKey(), likelihoodArray[idx]); ++idx; } - if(debug) + if(verify) { //to compare values likelihoodMap = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); @@ -273,6 +282,7 @@ public class JNILoglessPairHMM extends LoglessPairHMM { } ++numComputeLikelihoodCalls; //if(numComputeLikelihoodCalls == 5) + //jniClose(); //System.exit(0); return likelihoodMap; } @@ -299,11 +309,11 @@ public class JNILoglessPairHMM extends LoglessPairHMM { //} //System.out.println("#### END STACK TRACE ####"); // - if(debug1) - jniSubComputeReadLikelihoodGivenHaplotypeLog10(readBases.length, haplotypeBases.length, - readBases, haplotypeBases, readQuals, - insertionGOP, deletionGOP, overallGCP, - hapStartIndex); + if(debug1) + jniSubComputeReadLikelihoodGivenHaplotypeLog10(readBases.length, haplotypeBases.length, + readBases, haplotypeBases, readQuals, + insertionGOP, deletionGOP, overallGCP, + hapStartIndex); boolean doInitialization = (previousHaplotypeBases == null || previousHaplotypeBases.length != haplotypeBases.length); if (doInitialization) { @@ -358,8 +368,8 @@ public class JNILoglessPairHMM extends LoglessPairHMM { ++numCalls; //if(numCalls == 100) //{ - //debugClose(); - //System.exit(0); + //debugClose(); + //System.exit(0); //} return Math.log10(finalSumProbabilities) - INITIAL_CONDITION_LOG10; } diff --git a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java index 3b4498776..7b09fe047 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java +++ b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java @@ -280,4 +280,7 @@ public abstract class PairHMM { return Math.min(haplotype1.length, haplotype2.length); } + + //Called at the end of all HC calls + public void close() { ; } } From f614d7b0d8e7b71d79655f510cd404f1672e6161 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Mon, 20 Jan 2014 08:51:53 -0800 Subject: [PATCH 13/72] 1. Enabled OpenMP 2. Enabled AVX - earlier commit had disabled AVX --- PairHMM_JNI/Makefile | 2 +- PairHMM_JNI/libJNILoglessPairHMM.so | Bin 0 -> 114104 bytes PairHMM_JNI/pairhmm-1-base.cc | 39 +++++++++++++++++++++++++--- PairHMM_JNI/utils.cc | 3 +-- 4 files changed, 38 insertions(+), 6 deletions(-) create mode 100755 PairHMM_JNI/libJNILoglessPairHMM.so diff --git a/PairHMM_JNI/Makefile b/PairHMM_JNI/Makefile index b86958a6c..e966caa90 100644 --- a/PairHMM_JNI/Makefile +++ b/PairHMM_JNI/Makefile @@ -1,4 +1,4 @@ -#OMPCFLAGS=-fopenmp +OMPCFLAGS=-fopenmp #OMPLDFLAGS=-lgomp #CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas diff --git a/PairHMM_JNI/libJNILoglessPairHMM.so b/PairHMM_JNI/libJNILoglessPairHMM.so new file mode 100755 index 0000000000000000000000000000000000000000..7a771922fbd614ed3955980b46f605347db8f67a GIT binary patch literal 114104 zcmb?^3w#vS_5X$q7!lk+0V84wHd-WB6Ou?^K@&(|Cd`HiQKF(t$O8g-HQDe`#6S{s zT-H*H*0#2ywd&7D%gxhCJaqXgbzK63_2u-qk|R=z#}nJ>r8mt&Y&6TWo>sn_kWbW4aQW^(b~%S2xfA!a z%$pr6)Q%fI+vlHr`{k#eZhhT9d~M8^`^l{a_h0#f>p|SrxEJFt#a)Eky6(lx-MGCb z;%Pkp2X~zbm*bg;dj{?}+_&Slt|#&G2NPIpK8x}6nD0?|-irG%+>hX%fct*jt8o7> z?&omxx(N4l+z;U%hx-cLyq>}RFz$PBPr-cw?h&|o{StQyUvROmyq>_FkNYXycjEpN zZtF_M3r(g9_b)6Yp10sG!2JO3AK}i#oq&4{ZeB0p{v+;M+`q$p7w*S#KZ^TB+`Mvd zFT{Nr0rOgc=K|cc+T|wB{umEnCGN?%FEt5e=5vGj{575xxM$&Jo3kHH!@UUiT-?iW z^LhdIdcNRF$Nd-a=5YKO&u4KL;!ZVjKjhILiJdTOk5xH zIm~=^e=Gqa3-@f?H{mY9{d3$t+*5Jq;pTO{e1!ha$J_?s(k4!JUNLeys-ZJnrAykT=@idwO-UB`rbB z1jAuiSHRO$KP3^=JFjRcjQF-UB-~8H!1dw<5+6u0PHwB+7Qe&L%#h&~2V*fApVuw@Bc5i8v zj8mu>T=Y|e;boO8OoYHkCJh{ z$^XS(^iXBWlhsZBD|$&k!PHy5*-tIKeLYax!}^7?K+8}53jLCLbCgNEm41qekLJLQ zD@FW8{AkLP(oLRcMY~_*$l5FuZZOk#^y^sJZW-raVWzP^J~PXWH1%MWTiA==9*P8_-!Ztrl?DI8 z%ZGZw|29*fub~~O=k*Ut#?dDK{k`a+t{41nm|9PIsD)n=?P&71^n40@ zh;K9NWtH39OaFb|)X&@tq~PmKIm3Es_agKQ@{j(7OgPs}KcN@-$M<5-8xaBuKBm7a zH~rgJy`+yc+iUGJvcMFis{pQDZLL9u}H_~ z9XY1`%hZ@-oA%Ib%46Bd;9lg<@5N6ZH}%|7DmhzxF6%{~Par{0a!%`|e|dWuX9t+& z&NcnX2D4&8tKVHH?Y7>+_oAOid$CU)LiQx*b7nhceI*$;oBH2x;;UOtJnVzl_FnY) zNiTX>*^8Z5o9$R``WeX`b6#OT7S#DXZu)_iN{L$wnAZu&Mf;CiDq*X=4w-Rj%KcsO z6MNCaF(jjZUOU4~4>@>M^ir>U@b5{VzFzRl(0r6L#gyNY^Q7n(m=FF@mKQA_jzzuL zNh|tGPvvIx5+`2*J7Kw5X1V1i|Ba^Js!hLP$UmA>2xpV#Z~7L^wH9Nt1aK+0R1%h%lEqPq$l-i*cGnFaHUDt!eN zQ;KsdD+?+e-rMqvvlh?tI_7)x%1X+sd8<8cwxS*t<)R&$WpWt0s)`>4d?NF6`udk@2Kr~d5XqgH- zv-rdiv#CV|#rf%3$rh8W@!o}n#Hne>G2T~(Ca6Fgs0-6)Rc2G9S-$dOHE)(LK0a?j zZiUxZkz3@eRP$Jay!7;0^VDo_cAPgm-kY8^eO7{Z=JZ+D|FCQ#hsaBrr_ZWP%A*kJ ziwVxEOpBLqAd$<=<)@nZ!%43qM_$ zTLfv?W2VHXqfbdsRC#@Fc{zkb=gdPjqyZqqSaz}Eg3?e9(c6$KQMe+@mh|pFAS+x;f>$l8^v-~wC1v?AT`Fz7WK&cI zXY+0EV;L1FV?|~#@a6iL7pS<(ydrqZqQauQnX`P!;Nb-~$+)bzGCgam9ZgdgWsjS?OX}{uD%+@kx0C zs4OZg&CbPy!E~h|PoAh^N`kS&>~UT3=D5g-3PyyboV(mMHkp$cIeKF{Qkv%jYj1fO z2gHg>!6tsZhCn(SaWt8mVRYv(oKTWmuFk52?Z+od?_XAE)vyOeK;xXUAcyb}re$G( zg?B`!=;WL1ojDItbwL4=TFwT^zHd1kKI6Qz5C-M>;wyc^7j-L}-7pz_0j{Hx?wRAQ zStk5SJ_<&P|NC5Mnh0X4yZ<|{&K5vqS5{G(TAF`zc|KZmNG4_Q3#tNM+9+jrwFXP7FTvs(}s)w zn+hU+AC3+mvtWYcyAK=D@W4*&T5<^Y^N^xQ;66_x2TF?kc! z{$YGWB6Fo;3d1u`YK~V^Gt<3VdgiojMRK*OnkSp^2S;Lgvk>Yt4ik+t{X1ta2?b?^ zW~&s9Lr*R)Duq|hlan_gsx_{0CI{Pva)=hv^2MwddROJ5qVgq`=x3n;P$Z(xdYt*Z1M*MYL+2mc?{3;ePQpVyuIr@CGOr&n^|G>bmnX01>6O)H zb}ibcg%-PzTty}2#byh~Uk@jjyLhR$vZ}nitio52PiGgO3@la)60uxh$B!NR?Uh|* zu2c)tv%Q_8Miyq);L=O#d^P|3rslfx_vek&z3`3~#@}T{x@WdMCG1IBZ{^|=tZbJp zNn}0a%PR^j#gyb$E)p8epE*4=h!+)Qr6w_y4=u!079@Bp3d*@cCzs>KCLm0}ik--g z6(}z!{m`wQO;%o3NfCYG*z2oARAft@hxl5gFPb^t112y4ZyMS`6wHh)-IBU6xmU3( zza=(TMtYOZo*IQL{4VJ#ZPHCOrCZ31EF^OrrUD4ueJ~I?-iHJp+l5@%om|*UGteTw zRUdYu9_!OB^>=EU>5FE@j}56ier$3#jrU|5$YxiYw2fYs+%5HYDw*l+LQU-y%2af5 z(Y&IP!fPwb9C)Ken6E_@rSr|#5cqTAoLjbPLnTN6KyS>y+P;O-e?*1v7nw`5g_A;#ylIpOM}0S$~xHsBZYUof0424IgcugBaZnpJJZB$m)i- z;GFrR1Vc-nxhA~78@|Paa~5<I+c6W-qq-(tc!3%jK^;r-q4Ehe0^ zs9Smy-ro)1V!}Da-O`)z{%-gd6J9z`rpI=8mv1?$ZFn5X30?Qw@YY!X64%=BI64u! z9<|}^-@mN4;dz$Nx}LS+?caoKu;Hb|GRG@6y#3pVdK*5?Vvgr_8$R2HZ?@rcZ20{) z{B1UTiw)0jiLA@8;pbXNJlkyeJR82g1VZ)n8BLp?lhSzQB74tk+ z7;sP7@Ns5(9M=$@#Udr!@aX8FE5(MVtyq_0!;_bFW!dmn-ywvc#kx8Z+c!#CUTF*f{u8~#cgzQu;W z%7!;=_@CPFZ8rSXHhjAcKgNc4~Gi`XohM#4_x7qOaairab zzr~i`QP64svu*eY8~#=sKGKH2-G+~{;rWfAbw%6ocUVX~N89jT8$Qm4&$Z!`ZTNXM ze2NX9Z^J7#e1Q$0Wy2TR@Hsa8d>ekQ4ZpyKUtq(Vp`@Ue+wco*>6hB@i){F68@||v zzu$&0vEkR+@aEAzL4DMQFSDgzZ^M_{@Xy-tKeypG*zgrL{3|wmr43(i!~1Ob?Kb>k z8@}0wH;V6yEJen);EjIiz8y~}lUv9&<+3@$+@a;DI3LD;0*lGXw+VBxJe6^hOf2ZN89jqHhi27zuJaRw&Cxy;Ztn*2W)u7hWFd>SvLF{8$QQ| zf6#`XYs3G-hF@UA|I&spx8WbM;g{O*>umUH8~$M%{(c+&5gUH34gYH!{!ttLQ5$}} z4gVV({#hITw>JC+8~%4T{3|y6V>W!f4gUule!C67-iB|s;h(VK_uKGK+VCwl{2y(2 z!-oHp4c}(NKW)Re+wjlW@D6i+c>!?G+VBxJ{QubSkv9BuHhh!~|GW(!ZNtA{!;iM% z|7^p@+3RKO`JZc)frR67EZQt$_Cs z?nk&G9l^N*ewFZrgtG+v65;-YQw01R;fn~z3HV9E0|-Y8__u^P zB?(3f_#wg<6Ltvr0m3fAZRdfwaus1tnS(6?zKif6!p#D%AUv3Gy?_@I=9DD3LBR6} z4z_$_RR6Dphk*MN<`gE_c8>M$LpYjni-6Bu4w!>Nuvx$- z3130DUcko)|Ag=c0e?uCQ<>m;0UsoMCE>LK-b45*!qo!aN%*IP%LTld@Myww1^g=E zs|jZb_$9()2&V}6Il{4o;{^O9;cEy-3;4H$IW-AJ3iu(y*AjLJ_yNM#5pMfNv_D}^ zS%NJBzKd`?;bs9>5FSgoUcd_pb7~UYAmDj~6A7;u@NI-S6$!2t@Jzzj6Rs9;2H|mp z%LP1{a5CY!0-iv4JmD+>k0m^TaEgG(5Ox!e6Yv#;ClZbp@NmMD2uBKdFkwzbf(`-q zCwwE}wzH!B38xTl5%8H2fKv%K3-~1A$%N|#e2nlE!W#tqA>lN_>jiv}a5~|&0^UP- zD&cAY?<72paJhgt6IKY%74WNsRl->Ueu;1f;S>QsM|e8nH~~LNSR))Q;NKGFR3sQF z;D-oj5_Smq0m3r~x1AB~PdJNki-7MUd=uej0ap;7Nw{9X3kh?|5!@i)d4#hGuNUxb zgl{IiR=_g}-$J-rz!`*R6D}9h5Ji_$?K1Mi? z@CE^YNI0MHdI29KTtIlOfcFqCBwQ`vorLETE*J1-!W@8ta|Qe=;UdCW0)C0`Lc%Em zeva@W!f^tAl5jEMXaWD0FsBT`NC7`YxRkI%zz+~EBiz<5+Mh6|2Ei5q-$nT6gqsCi zLAZi&y?_@I<`f{fLBR6}`v|WW@NI;v2(K0JOu~x^R|`0U@Djr10-j8GDdD*So{Zof7R&xSDW_ zfX@sEypnLUfKL)$MYtX?p5kJBq4@hI4IAZ9>gs)cI+{gzvDUh7uAa0%8l#A!f1H|) z50R9bNpVQ)@C{G`lm7CfQ4XUIdZQA!{8xC_0$0}J0e&wck9PGDrQxhosc8*U#_z9; zc8!F0QEKYL6uoZ%^wf?ajYzr*ub@|aytQvBUc+urowH+0p=)HVDc__NuyouE0er)e z{8vMzq!*+7pMtWZqlNL$jxAE+EmlU?$Z9f3%}%{J^_JAxvu;*a9)ENsD{@ZL*Tj_L zHC2CC(`#cmh-&&)Agf6atc$syPk*{&Ypr;7bgXSd;G-bzQj?tBt-8aV zY5Ka2OA+?yZ&jcs11Anaa|9;FP&rgz!q**xG)q8(7 zYHIq2g8D5Yb)@OXdL~lAntrlpBHLKe5B5x40HUBM{&hl6vfx73iF##aa0?9rglXOS#bf6J5ib;1r!e<5ZAU{yb-uM<^L0-K~s z`D)sj3)JJO>6>EKSZzG5d&di8Ax4qQ8M+mJ8}QqlfJ8WnC? z2@D&kePAelOm2Bhgu^$KiNIWx zCK63YPS6HO+5n42NsQ|xZHS}|v1mf7AAu&S{22wM9*ufdM&F{;9dgySBef(OI^5zU z12ZC5U<MdLOuZ=IwM}e_Kfk&EuoNM)m&{KNCr>Py=)AZt)EV#UlaDQoN zqwF;OYvOWVWe5*#%@HcCX>9XHp-caRP(r;vBF5;$rmty?{K6GEbl5>^G&?2-z`%h5 zNQ=oj>5A<8C0mxRehVPzukISnm;i{U?Gl77g!#Xr3Lb1 zav;@@g;Zao4{Q2qRo}1b!GwC_+0Q#V{Mj*Cb@hHNCfij@b8+u>-A9v%-I=Bv9Sy0_ zke(Qms_%n~QZzprU-SE7B4Mi!xkfpr1m;``m!cH4vb(DO8x`ldjzO-`&6Hn-$gb;l zF7W{9n*!ip9DToV4a$y5FCbEX(yRKnmTIUk21c2ff$)EDuIsz9#o?QoLr-~3kbMDTFhwUR^WiZ>q&4| z{TIWA(%ktm3g`n4xK>4B)DAuc)6x9W1!(Y+^eCENnu6w+{(weDSD^qeT*t~rTGsSxW2Muu^($y8cwxU$!QE{VH21VFz5+qr_u0C4$eT=Cd46BM~+M z|GKuNqeJ9S^rK2Z4y34M#EnQ2&m{W$lx9EXJH|?O*OiUT4xgsL?&4^q;!iT)QjCis zmy#z35|IEd4Q3vtKqcZ&G~Y%WkF+5j7(^MYZ^EZFn?x-fuGOEz+rSU#$E=~sScqa_ z{RlZC2MVtVSE~3UjfYqkyobMU`+!b-R-~~^5Ncyyffa&!&`ymqZa1kLBz2RW8f{E6 zsn1I4AMDf!V=8C}VCa%}WU@zPGBTFQHj13aPasqL8^V1!a^e#G9EyH17>aAv94BEHu$5 zAKRG*yf@$pP24lEizddiwE{^`zu83-%+o~^tJwVlL-SCmB{OQw61r%@4{pHMhw#$M zjLPGtCOQTPO)!D1g?IyCaW$5L1*9cwnL@moDTF3&ktrynOd;OP6!+t4BqD{-L@gj_ zV!eDkg$Fh9ES}IrHe|8c#C2?~K+;@jp2DywRE2rEXksbs4Vt*MYbRPCO14p&_DXO= zb_yV~o0=$UH8t_hMM4uyAZsDs0E9GA3Ko!deJF)^GgAmnXfg$5lsv?nnL=n{3{oW2 zd+In0EXBPX+NeeXSM4~IZ>xPgTPTpUYFC%qGijIF-vt{(?Qh@HDJQ~=4dM}SzD#Nm+D4?^(%aIeb<&w7CqKUfBB@IInk;0pZHvrU^sOt?# zA)>bTjlHNgYFjSBP>lB(ZL4-Le$!8wGfBj6kez}?7{k#G>=d?)sDk0> zMxpJIq8Oh9MIIQe$q>=Av71N~9rrrBS;8JnH3Q2_9Mk>b?bG$Nc2rf1iJKmnc-r{; zCp7xQRL@-FE}A4s9Zl{RM3|G|A8}gIor*pSa~2muJ2rJdPKsz%`=LkUkB00x5n^E9 zf@qhiENTHncgqmZRd*kVj6Wj~8P0)mz)~Us54sTHbN#lyuU$OK0S|!r7BXRNDIc`2 zkT#(n@%=_A7)v>^euUIYV2!}R?YxBXSPPiJWMI;3$fqHqi^!g}u-~0Z-~|y? z-Yn)BsWXi`p(4z2qFd*oj9-SzFm6G~N}%sJ!5gy~xVg%X8|D2d)ML(d#LgmUU&p#)wG0Lf+(r~Pg z(s24lC3d&cu~W-CGb0dluiEgD6TW)s-J*$<*aP+~tuo75WZ8=>O5Q<5e@|)nI2`fc z^0*9t_^3|9^7Wi&V(kS|=i9 zV-IzNzS;~$C>`4w+BO`)YLSVlt6zQ%$?Wc(r6xL2s58Py5`}I-p=$(#WpAfpl0u!P z=_fETICnDDHRhd@s%y;0>8`(g4H{<+CtYKjNO<4%mrwEj=1Ijh1~xYEnK0S_;(JAZ zOVxL%`WveLmQ}c-zd@nyr99pKWlE9`xxe7Q{qxx)AoTlNMWWyGnr}79L&-1n9p2;%5wCZ=5|cU zHqwkCb_pb9ItZUR|yO}ydA<#7Lrc7Pm)KFJYSOQN&X4Rl6wve-T0Ns z9Ru)wbY63gD8}XOABk~U#_(e5xR%}fe^&nAL{H=|?~cW)mYzSj-aQJ4=vZ zawn@o$*P&GCPinUK6=fNanu- z0@MEp^<2SC61)Te*AonA=!y(Ae zkvve6Z`dTrMKB@@MaNbYP1n4RO~k*g@jBdM~U;^CwOGCK|)UDPtm)=O4Q7 zY&5*}Hh)FF(H~Ob6-gu5beACk5(2y#IgJ4#ecwNc^qVDN4hUF4Q{wy?`5Yn|PP*>I z65F5wF~)n8&p&jrWRFCX~D%W%-QH<+DN%%X{#x^8DURj^vAzC77 z&&%v-B73j(!P=x*pF@W%b==ApFBaMR_O;iixk1(^>)Z92Bujhev}k_TC(F=79RG-n z#&DVBuboM9jKWZoY9={_<_gs(BSgz5?GT&NKh!6(_gbHvP<<{6WxtZyACuXWWa<8j zxyC}an6S?UqWzJ8W{4Ot_0HhsFwZXbKIw7xyQ`?=9j+k}**Ql+3(uLq@Yyt({ zVPAW+gIh|euB)e(^_Ky^qb1{+4-uTn&Js$mg~37RrH;TJ}pkwQ<7DEvZCr!vJ^cP6SxOh z3eZbE`UCfi*BUW2{hH$XCq0TXjp@QbCyl^6_N5^GhhIT@S=$bwE0~dg(h$&0RFha#DnWc&fiG=Ja!IYm*-l=P?YRWIV9>VkrXS@I9~+YcZBG;j!9j1>oHXOn&;$vGsyC&`;g_Db?5>8;0; zd{B~qPjZSR=SuQ$lHZl&6(o<6iZHH(uwtE_Km_|ZE!r!$O0Z%OVK@)655SIZga0X9E6xT;7GHx4_L2o7Mc=K| zeAq^53SGWFg|2t&Yd&!8HC&tOt1eRXX2rSNI8DWnC#Fosf8Jri9Y#3o>qL3jVP$a} z-z6T4EC(f=FVbNnfyAHE1wufv(QK1Zv?pc!8){%~Jev<5W2lDt>t89@IOf6|CrlBq zM?tB^eWGp>ZDgP%#m{vUCcq||bE#nr6&@hz!7rmlz9?fIb);|8KQk7J%s4Y(@iexx zF5NlorEefdLnp75jg;mNQ+TuJ$3>$vO@Sy6H95`~L@cdGyVgh1+ZFw!?YxH{v(x+A z;Tk$VpcyCv=J5*~yf&vV;15l@=C9^?f>a&^v7h&-IUc2AX1Ehf_A&y2uE zyXwC|i|tSwI>J4eKDrNA4i{5i=V4X96DxR)QMRprkKUxl9(L94go0K7)JQe9L1}0Y zSKM1%t1!^RgB|iMNAqG$pyAk!|AI0cigPbq`tCDahru!m+1{{xxR@^%)_F`krBX|X-btvxri-#%=N5eJeF16-# zq%T6%f8M6*MeVE)MOU2%JZ^*dJK_0o|>fM?KD?9{16T5uNEYcgvE2p1O~!24f{7JmJ&U(dwoU z4`75B439D1hE-|s#W-d_VWD|dROB4>woKIz;UtCH(0b!5tgPld1ZekJp*T<7AueAC z#kDKW%|dY#GIVVM6xS{lHvx*%CPb+no0Xc)(H{5SioF?u@SCOLvY@zwR9p)b*RIxd z2*oXd;%30+4Ap(kRr|jv)uSK7l3MJ$uQHn&sBkF_=bcK!XA?65zF{uN2RK~SzfgC# zLZ-zxsrm_z^NepW9Qp)R4jgRyFJ3AtzZwM)QV8-s=hHIO#c z!4~(pgWv<^@49N?R2`n!HxqV<1wv2YCYRc9#;GZs-+@Cd28_tDek;@tB(Ob=G zWGewI;?#ZZi=5$~*hkZEnBiaECqmO#K)dJLe1ixg`FK8~Jo>T-VVzr})tUy}p{gW? zs-guZ_SXWL9Uk|al~*gy{VI=3;3$Bz!8aIdl`#46i>CXv2)t+)p)#L9qKRXbx(;7o zW5phBa9*oAk9#11s+UG~H_*VOet(CDOWNISynz3}kbsGZbv_3*LSH`>j0MXrg=?k% z;5l~Ff7Bcwt@vGH1!;B!x|%Oj(-ULhD0tifMekDdIanyfDi}^zL@7Gf(A-kU11MXm z*%GC=->bqlF;XEu)Nn2WXCB1K1@?2~m_ZjZnl24uDLAvhDh7Xt+(I4`_Dk4yJ`Mf~ zhtrf-JO}e~ezRiHDpmhd^OvAWurzjuLy0}BoWhBe^NRC~($FygMtP&Ew`;LmJ?@su z%hjSru1krbC!GS=eV7(|)B~eM9i5-JYAaypir*7yD=GF(u2VX9D-CDB_%xR)HTQ?E z`))x}IJS^TOr`I|slEZK`#-ML&#=RoI|7|)um*an6+Vz5ANj?8L6)or7C?*kh~=|8 z-sW(CMNg};%BcfP5B^y4?36+&I`b$4Pnk`1o>Ri^z^uq7Z) z6d0d3gSxVb-H4Mks%u)a>TFis%@xBGck_}#7)h#JTG2_eOi$=&9m8U{7sJV5i1Msz?a+RK8JZQ(I!1FYAeQ^NEjI`gZsZ)(mX$nGin zNF>$wVz2=h3?O7AjOa8B(&DA*F)_xE$){+G$l$*R6=`H_!H#yJG4?H%!AZwG4v!P# ze>%F*Jv}&-mtTTJAN@PzP^u|num0rFo3j7Xe=ugj!4hr*BrE=MxRLX|0wwm8Z4JL> z>m|x5+CP^5&qMrocu)nS!rK}Zt?F-@UIzoB^e%UUzuM8H)@+UPxQ|yH<+vTC`H@MJ z#>ic;WKl)Bm*8}y7k;K&JZ1TtSev||-3ZM;8}U!G=687z#^}4%QwBv;8`|MZ_IhIX ztL`r4eSPcw80$AUkt`5?}NV^vY`3iJN2Z@NP>X`J1JWtcjyKM}O&|K3)8yqU!H3JjZkwZaOgrlT1zb#iY1u4?5}7X27R~HeO=khq?K} z7C0?E+DX@apJNYD^)G=7_(Jtxr^I4gVTy<+f;mC^64TG1C+M@v!_h+o`Xjkn9 z8b|C`Jiv(qU*de761Wqw0!}zLv`)mCxC9r1Y;fcGx7~)Pu6}WriXQ5G$A`(@fNK?f zKtF8NT?=c}`XN*}gHQnh&b1h8w0>x0?b;|(5Gp_+7-!M%5h@Ij!A!%4eYL;tJE$AZr-Tm|}GT0;Wc~>Mme=VM7bb*83<<1Jhu1 ziuxJ;IoBD}{mZXw)7*y@ePN`l_C@GjjXjG)kRm=)0!u>i86uh+TnUKdPb1uXOWFMy z!WrtkUUP2q4bJpq4_Y5`L8d=@B(#uy0d$VAaY&{=Z6rqSvn37-7M(pB zT8U*SLB-X5iojh4?|M2kR~8=99M#sK>V2^I1LZBm-Z)OSaiEdx>q__(-5wL526cDPQ@3OBOjX~aIp3pEW=}%hvbnLSO+w|gzEIs+ zVXoPeV6NGHxeY%F)zkVybz8CPh!Q`qcpVXGl~mm)h-?SMDS-;8uFW#n2F3Z7=nLb} z7vkUy#({Zp9L$x0?ErYkj0m-38zw1mf#+ST4pVJjFoa)FY*RA2#uQc-&KH)=iJcM{ zkwR;y!Fq6RIb`Zkw&rZr^v~G=P6ai+4653Pcv#rI78_h~jo22g*=opz8)N1hbi{9> zMehCBED42xKY1${P~EM*VdgLyWyByg>JLZJ%AgPp02l45&A?p9cy^m`Xv$Sw|*QI*iG>OMYQ z3gEJBNkaMgay_CSma<5g?6b?yJ0{U+3B27Hqo9)D?@C$`k!%mb=q{||Nk(MYCR zonNy$e}e`SMmEFapW22=sx)9vY*Rvq?D}e;Jk<3w19SSp-Kuc69=p43&}d@l`q!fC z_d^G#yUnJ%JuSLE+%26fyFT4*Hr?$u>2BLa*S`m{q3a)|yG_Pq5cc~8yFT6RHUw$D zAF2N3NzlN=IFCOq3Eb1-%&tEGtTPaLH^6#7an-$VX;vC2E!96G0vg5edOLaw2AJ8< z=XNwj$eP~S4gl7KR8_}ze{FQcL+j_rL?_H>-Y-`qDTB99rM&_WlSm0UwI zG>W1j!*ubooTiLb5#u;OV_Tys!}~iR!$;o$^%34gDdHRFnnAi@6zY+|&9^3RR;=M1out+S4P;tqfenRn&z!-Tc zqh%2>$g>w%IEvyaE}Yh?yBljv35OVwrJx08B63C}VwRYV=wP+jyF;K5&ExSvp?B^^ zyzZ)PhdFut(_kk}s(+jki;(afM?H24Eb32&2b>JA{xXIGsuUFVhi&LDaGZ)>gb|Qy z6qx3vCmdoIdW{C^2`HH3pT-LmKk6Y}(D#h|REF>&#+PiUlZ-`(QOF-9o)Wk@q4&Lj4TsTMFS@ z3gKJ6fp7V!iz3?*ScnxV*h`q^Y@x~!%xQWqd`qF}TUspN(wOkIV}?IF8o_in6-wta!>>hS!9;UlK_xpH zOD5S44vti);yxvuOK5O}bBVK@iy6xa=YrOnn5anKlGxq1sPHYD;c(wk-JiMY4s}(i za4#A>xPJkB%%@n~QE_828=cKnyPsPvc!hbIQJbJ3YQrwg|AI)b4Y#KH?`^{_j_RLb zXt5uJ^xIF-r@~OD!cf-;-;EVN1X(QzZ>fAYzFeAT8mba|Ivw*O7y1fE|j}U86Sa0f^85k0w2JVK_{)R!vQN-dh7}WgX<25~;=c2}=CK{N- zbs#zT&4gY#=|Z5^B7C+mP+!bw>QeJZvqwX99{0z-EKlt5PM?85n_D2g>k@C{FU%^UDOTd)+{gzwvs zy7d{O9!Z6{2B(N>pmxD*zID+>1ed6%S$fb!LBjvwgPh=D+c*VZb>0{u=&Zk43o%Mf z1=bh^sSOQi;e3KiFidey7!maI%~mG4V{FW_#>V|-mEiL*L*giaAs~3!8RoYQ(@%(G zD0;MbS)I*kR)&o5Ij zY6WM3jQx=A>lwFnny&_dFidH&7pVWRc$*6X9(^;GkZ;stcY2&V5!x?_Q)Bli?spW| zw3CYSq~bn_Rq|J@OaK@6S6&Z+dfTlS7k3%={}b|9>#@JX_5jyExyp&<5-d95J2Uti zO-Goif7RJxDv*%91^ulm$l(@!5JLqpSm?!+#9Tj}F8Rsz0eszyl~eauuG+~QN3a8e zm1oU=y&8*cxO;%OIZ}yzSM1v~oWb7BX4idJBB6$*DC_`XlEkTy>aKU)7odOB8)-J& zn|`pB zApKCq{kd!PdL&R{KUdvfy4LW3Gj<5o*!Qu`a3kEMb8D*D08w_gbGv~n67`Fxp$8!c2N$1~+&0~x9oLS{iR5;wDArTfN8>&*cg|aXe&-r%dB1 zQ#@rFPnqH=(|E|#0GXPFOkzPp{a!MaVq~JwVxr2{_zhbJtuYt|$_By2Q8db)6(dEX z(DQ+Wb`e`CY@0o(0oa6Sn>Wn1*{iwtux$>wRzD@$=78!x=vwnA-ZjjGV|QU$ z=SH>WgEq~%C1HoUyWNZw5N0Lp;p)yb4JT5W21}m1&u03=**+L#&_3C0AM6Tc`sc8H z=CFP8**^JJ`@BJ2$bBGHZ2#%#f%8iy8i#E!hfHu;`qK&i(?ZIF4C>M6i>~ihs zQ7*O-C+v~iqJim%u}&k#3Szm;cc}}8kDWDc(C%)<65L`1;j8A{EyBup zgq0Txqsr9NE;K^{EVjgB$8Y(C5IZeCiWG!cST3V=?d5XWL?OA-u?s$?^2cJ&6jC`6 zUa0;dH>moM@v2|D0e%Nz0J=FvXIF2%!0hv?^BYCK2j0WLdV~D-n{J>^?EdPFi0;LJ z!pi-Yt%b^cEqiWY7AkiJl{<#YX~SLY$EQ)bH##d9)SNrT#@<-$g^dxt7**i5A>+pk z^lDTN+lHjDa@;n=;tIR-7pz>kj304E8*w-)hkZ?~uR5E_b4Xl-$ck)~VhiEE> zyC~zie_(k6Z;>k2N1~0}F|bmiJe1?nzr<2+)!W^c!#N~~@dRVcbdxv66S2PjmE?U? z^;bn1GX(1@aK0zn=pz`PF!~F|Uy<>_*X>gSZ z-%PuZ(IH!ZX|vVE$}E)P&`ir_HZ5BmLQNIA&z{Z#e0o00T z%KiHkv`LhSip10SGetu+2xxj6zrA1<9$vv*lODU4jm=tAqHvt5!CK2U^sIm8Ye( zN8laP{nURUn*+NutLv!#z^0gJ^1&B9)L2F!vE68+%qGD&vY`{0tf&1o!q^Q7QaiS$ zc5L%sl!j&gf41Jh{q9St-IqbbQVlo2t4>(WfbW`ohz2^0XTVE#{gsVuU%7_A1yW(Z zA%k_I45@*%i~F$r(G)C@xY_WU0uf=%GzB6sS+D&oJA^Sw$|ME)n`Ee23jSUNBh_AWL>rc2{_F&8PQ`CA;KMBzs9}Ah;WH<mNE80;NTIl>VO$M42)Va9;b4- zoY4=BB!NC%*cm095e7cMP!YE;#u># z!F)12Hv+7NzwhcD(OARBHhn1lV=`Z0O^ktbeyq4X7Q=tAfzNhe@Y_oyIE+@IC$pVy zknF~Ro%orhr)2NYpAy3qfBuIZCrz&J#k=lnglZBx68Hl-##pp5-7iyM4VB2F{52_= z$C!dP;}5^E8a%3lPQ{vNSbZGCt&S-yx@HTy+q$*d-&6Cvc!)1`-fcAfkmvL2ew5!+NH7V=Yu#yXSvcEE2wC z5L%lYsosIzI0q(jJ;h<1p$5#1 z6ijeMA~M26yjii%rz0A;oCYrmQUT0Zjf92|iecQqYI)q>u)#z+eLwcjV7Ll?S%V3A zu}|(NcN`tQ41IwI{lRBD`CLNp;~mGEUA5C$ z5D+Yg&lWyIh2WRI9*b!P%Gma-EQEks(tc=-{87tMo2S>Uj^@_`b)74Z+b4BDJ{=s`8Dz>2U8{X1!)6%A@&f@ISv1xpe4DGtNMp2 zv71F??=~v0(gHEZH2q`EeZ*C}h*=S8{7mpR1CbyXfs|N-pKqbB>&Fy0A{JsiVpc-{ zr=Nj5aCxi~oukwdc6+dB++aBC>Q^N4be{7o{OFjWvCd{1avcB0TF==QNPvj){;-G!?E^%R+#Lzq~&DKW`pWULotsUO<-2C}k3 zumsk^9)http%Cm!laH{H2qskk zb%Or{Ib`6lSXkvzW0R0^o$S2Oz*WnKb-V?!6{E&H1uFm+gP_rzf^(D(21i!NS z6v`5g3l;jSCFBJ0B86auwzJT_tk4ED0*evi!2OaEywS#MDG=PWnoptnTTvehp%Wn? zlLYYglg5tkoKE4022M!7A3F704mQ|#4otj;huC4j>;pUkprh#(9StV@mvasv zR3g-AM#SJaZ{Cws1{Pa%IR(Kt0@SZ$@q&oAg8%*+76~JqPcNW11@ACb5}XL7bv++@ z3u4{&H=-TJBJvF+9mhM)hJ3sU5C5dhf8hZ?8$1W9i|6-ey#gIMTu(LxN7Q@}rfwV);*zFIX-u_8ZD8;x;Xl>{Vue0E! zuf(J8YVjD@!3U3rTzu;mV-*sb`H$)%|9+5-Lqab@i$#v4m&L=%zsr^%#V(iGo?y18 zWVR%kZH3JCjCc%8;lnI8v|eMa_a0g=)_XhS@mp|?PJCSTCfCC70R>FRRr@SdGS(Su7B!vF~vO8^5yYVL#*6H@3v|gPyQb zY~$lRf{_F~PoCmv9n9nuWVqGly0loF?2~kZ@@d!-LfK@*PX%J%QSmj2@ww16R>@*T z(lctW3Kz2huB)|X>8Pc+~+iZObkxsALg+Ld?$u) z#)c@{w!qF{aEFc0(EsrReXUq_KUDP@j<4VhJA{T!y8c$>Fm6O)@~Gl5jEib44uM0~ zPmI}+%;P+v#ePhYakfB=m*_k=CZr!Tl0oJNQ_fH2YJSe+^uRZ0PJEVCijB(^k#=Rk zu#eO@9LC^3&~z>P>%RXR+K;^m%(x$G?a!V?HrxK})m%@v?a#7%ZAUd^;PNpSdq5BE z%wjTF3w5ELyM2#?m0?WSu$>^Q@Gwfj{3G>_;2XaeVg|Q}XQ;n)OJ5L5|2vr;%i^Q0 z^Y`NPeF|nm?Y@L>Z!MfKqJ+Fn2(*rKbt27vp1#Xj`VR0$w;wHLp`CjHik}BHpL~ps z2kW~UwzxL*Ytl8xv-l|tTo_I>p$ouv@xdC-1*EuY|4lc{O%EKm4@EBcsP3Tfdtyp3 z+PDisia{yn+MCeIr(Rrq41P}5&U$PT?q~rR!DPqQImbHO@_K|3pi&u{ihwNS2*ltRGRH#GNG7)NmUA0vmWc7mul zBqwuGd9ikie8vbispPv=R3162-jy{cW2toL+x-=$tcO`YUls#Kie}siFNxFbq*fhk zO@?h`S#AfRAKF-C$9Z7Y`95|!eZxXIV?sH6fqJLUI{ojO>DO# z@`8-f9I}7+glT41h!>8Ger}6>fz-wyeg`5L7%#)5TVJ6vqk`Ibi43vGu~-}y{wBm? z16kZ|v8VtsSRou2xp3o7$Yu#L*y8e=u3Q$9%TSBUHRN&)xJX%Y$pxR);R`Qrf{I07 z5J%Gb`8ny=w0!qo6_YmCv@@FXjOIR5adl@p&E38PO90S^_`DYOhRF=frD`M=Vp|hQ zQU3*oyug&7;L0>O9CVJI!dL`>#JW9Nhs{h^3{$~|78L9o!%Z8UrQ*41*-7AG*%)8` zmV)C=bP#;wguX=x+cwE)R600pFev`v7`9xsr)Y54X^Uc!GGVC8;q0REw8<^Hv&*$3 zBf1==kW6`)1@b}2g|IK2)Ej}FwmC7;uG&0d&T}Z|9LjmfH&o1WA?h56stV?K!?`mt z^+axVW=JX$kfGwAAWoo?iTj){O^HPp&F2}R^b4Rf9C|kGt4sVJe9u{aZsnfGnzcatN7hpm7TT8yJ5?-qUs4l_r* zh>EjQT@BkY-XW-YJLMQ4_aLQ zWN}$zmX&NgDFhV?)-ZwvhHQlyU|lM4r|}GUF+7J3gL}?PL1g1Y?BRj~1bPBISvb01 zwBbV*W26uVU~sZ6m#FMVmH>;yOQ^CFneXVYQAS{rwjB0(_akDw!`QWQMifFthx}}H zmV!0DnvYs;zFF}vfo9e5z#YC2t7 z-*mdX8I%{>%SicC8y;AYY&vsqp!)Rr4m=+&=kxq0zZdYi$i5dwqhQ=rE8C3gm^CEh~#&C7zH^}U$*dTrcyXkCX-?zxDeTi`c znzrfept@$1Z{$J~>Hcfe_2a4jTf&Ln545tat_2U|$A(XOm|l1Px$_-?j4=J-(Ws6d zcoctASAGPySbfh$sWr#LQfvPGO=?GDx_%NcPp0nfNNqUrWBuhQ zWSSBfa9zXcKGOor$Mn$y(ZHk!rrns@(B3z-!5AKx7_a2P;$DjcVOl^N9X2J9Jt(E& zOg|-Wj}i!fNYU#Iir3JBf2pE=q*Bw;qQtf+I{vRKM;^r=-~$HXf0;6}x;H+I)xsz) z8U)RFObv~goY<(N#=IK_t=Z@5xj&EEKo@d2twuP&C+nUbJu5UUw zs4k1_YuY1SYib8Uk7oxrq}Cj5Q}fqgfM-Tv-8qVevqoodpGMOUIM44Z-0c;utpn`U=9**O zS||vQtbr&>uZz4~Jeo16kTy{~vNZhK*=sbFFxP{dvDWL!JBP3ABV228(S_82`0Y*{ zIU1=qpW2U;)(u7$-D$WfOL+t|IxQG+uhYXe%G!-Wp|Yqaa))h_wPQVH zQF)%IqhXCxeiqdl)^yLHr}l{^y=Tx1`{K~HY!&%KvwT}4%(E%?{O3*jp)c7eUlYCV zkyl`2e5-xLfYqU8nwFhE*fi?U@unRmXg9VX8xK0#BpRc!;q;33#@y2@K4;6xX4LWy zSWSqNX3wAA?TvjoQ}qa648xYlA=yR<5xy~6V`O|oUOQbK^?XGXxMmJvu^`mpk`SCaBQHi z*GVNlftrcJwLCL*bnzvocJ3#{%;^qyFdJ8r=De5p=ZOuORI~d4g``Yl*h@v(Mdn;Vq zWEr`d_c<%u6siIeNdpJU>Ib>QHcB0SKFn106Gg+!Dj)e-R6C0Fpfd=oXBWpy?Y<4{ z>ouD(acy?3O8f?!R346cw8({J`qq#MwVBYC(do)JUB>55v-FDhi#Uh|9E@ON5VYwqsP!L^R{#fUsoub1e2 zpwskYH6OatVvoT!_8qvCu5n*u!?Hu!t0QOx?=+?^JEX0Sf>9hq$7)QCFWJq`w3EiM zA5FK$>`}}yntcjn=169aP}j11NE#?@H{?2=3iYbTgw-^CZgU+^$?o;Ut)}Z}8a3>N zP#<_#%7apc=^jw>PN4^gzSRLkJuUiIBV5cWxR}eM&e2#gYCsKm+0+#G|JZvQ_^67j zfBY`F3jvbcjSV(d)CHjpuZ@Bhc{2+eNNAw3MM-LeU|s~Dki;ZVumK`TDC=f1?US^& z1{*cBJ}vag(}qghpb6|^K)|3_d8iDj<;G_d7E;*(`}|ef;_VeLjEpliV|N z?wK=ZX3m^>pGyalAgcpbR)f zb2)r-G+xKKJkOurpV^owk4OgZOr@|t`yLYPanBThis+%PA8zJUBXIU z8Vmo1XwjaBZ$!i*4Y|ax4BwDQ|H|-;az1K(FU^11>zIuQ1=Tbd2w^P4p9e zZ_PX6`yk>~-=*-_W;{x3!Gvw>+ZOmXCc<|j7R1|_JoY|C|8M`iQ*du+ChkJSUf)|0 z?|^z6sl9J6jeHH}>AuL1$f1(_0GBqKF`~!oyJVXUe|Tv$Um|7AfDSP#ndS-MP9&7e zmS@2|&Qn=?88qk|_j#1NevXNG^WtsnGG9jFV*HO`N)dcw|F1M$59fHA8S3Gj(sn=? zf^rbR5abfLwygueN7|gO3y-{YCkF<>#ch`2T@4+L#TP7?X2S+bNb5;NW_cTL$^XFO zyMTm)leh7X{C11);njjaJJFSl0LfkY-BKIK?RY2RG=2n~xVs#i+W20*+UkdgG+4g(WE?oPTg~96IVNZXStL&Xx@CDn=j1IvgcO%^^ zuRCb*{WAYZuPlLUVG}{T^f90QtYn1Jr_w5VU?zGHu)xE=RJhWhTGebbNp2|NO z>^sD`{!0YA*Y+NX-vn{}JM8<%cX+)!21V|)kkJVQm5K@51+^*+-;xL!VsX<}*>~W8 z!52j;YIK5IsDo+tc|`6-d%~p6tlCCd)Fr0DChEJ>EewI%!{MG9a=A-6(o+L3VE~xo zXyBZltl*B;(Js7|)&xU4tH}z5yde=ps1@5pB}VPHw%;q_DzQexAb_&WDqdZ-Rak3b z4uGk$MvD%mt;U)QV*pH@wS6!h;ic^u`2l)30tBo~%o1>eO~Q@25^jkmM6ItSylwuV z?YXcWj!7eD=&&I3o4k#8r?laHWwWH;Tn-ou6A+Eq0!=_^^hFpTWVYyUcUl1(uqWa` z|1VhqQU_WA@&n&3g1?Ga)OU=*p~fDGN1{o`L<5?pz-c&JU^HSebiT?coGN9e$aFZw zX%KfQ$Ndg*Wts1CR8y=cOUIc1avx)TB#eb={5$r-I(W(%9jrtxV1j#NE=v%xCo&gM z1i64t*ZJYfMzmUrn;JSAiqFGF%mOb&>|w5U*$DFMvJt{Z8$tHcMlkb*jUYrk0;GJSaGv5_zBhJa{LWcs&=_)HVx%ShUoC|^_x!B(PqfS-osl*Q+wA-O<9 z;uvUq-P`yO{rR?o$j8l$PnrrZ#&9qB8w?njijRa{dP-Q0G#iGNH^X$u7s3+kgyskh zQO4t!oyOxS8V_YA?o}Z>)pz=!$SbrJBD;jGpxtAk$z)(A1>!3s;JcjED2g1S&Q3dy zD&9GJm3Wc-=Xj6V!rs32MXHF<2cjSLPP8XfQ~@2ovWzd0(9C=)KsP zjJ=I#Q18Z_YWw;pLWnUJv)EU8r7hze2=RSA2X17<yem>1#eTH~HyrZ> zk-47OgvpxqTv1C%JB({+bfatwJjKT@;nK#mS1=Rh`{2wj--TrjIG~RiE8V-}!nN^s zO@KIzP~poJKG)+&mq7HFZ=>YfVCjpNKAZGSkiJOiixEDDM+;}~p$yss5A|{i%+|PY#K>)#64J)G@qlykKo?`$mHM*F5gv2!bDzrMqSt ze;yn(^U=Z3#}JpJ=u>@ZWL*DOS3C3+oq zoN3@Y3}4S+;`P8-kx>$eAMox!mix?#yvI*EK=oFq4HK^hKh|dL8}G zVEzkXM|&L>$vUo3yyk{<;TTLhl0*w{k%@0M9d?soyda>F5RL<;gP&-C^oK>7YO57` zYdai1B#gw1Ld=w2&$wAzD!Ez^YJ=G^0-bOjuoq7M1qnc>5iK#C>POPUSm>j zr30A9&zYED6*d*425=tCw~La5JB#kxXH81n{Vl?7J#S({lyw;0yR%N7G10b(L}y)hb81S2@-lG-*znG@6Q{ zJVH6vMJY%1WH{meOe%&6YjcD=tDwf1O6zt%ruUH`f3 zbo>Y7KLr1_ZWi=$NB;;nE$Dh-L3g!{f>=mb+k=3nAyC`WxW6rGBQ|=ue~b===eEIc zV}G0V=K$_0*^s++NFjDnncLVf`-I*2#`>p`y|J!FF=}@-w%vqW3uPUFuhqdKNNIyy3D-RL<;HQ&w-Iy?={ z#`TX5j$D^A2ThG*(ylxlqyrxu(z5VA$aW`Yc48+rF1tC-{Q|>aS}7h^+`2++oW9-=87>gOGnRm~kb%okuxa{q{$t?MiU}mhn1~ zl-X%VW(OTZOP6(X$BajZMLE|ELsCRSX9vB{7I4%Iq{Z2lm>0NRsGW)7*~`{m6H|`@ z2uO_Y0&MEV)qU_k&1RNIuj|2yIKHam=)B`z$K;eloQT@M$S{?3?+IT@3T9+SC%@FXYz5=bsTA?H zUGWG8ZC3;-!zUw6jy+x#WEQPpF-ZE$C7QEIC(07A!dvh$Uyr z=5%>j0!~+HFg8Im{Zmo7edleZMSSZPXs25c`Ob%zHjuenWDZtRo8jBq0+Z;jWDdA0 z4+4R}C06D&=u524IGhDcnYBn_%B&3%Q(?`O7|eu7OqDfNVydjw5>sPMl$aW8IWTd> zt%*ykDV$e7`0d6?af_}^k zUqflP?`#CcTUZuv#IpD$TP5FPfQe;sP*-@GktX5D#TXQh5|8a&g=sfmm7A}k@k*|0 zH{~_rWV9!qU2Na^d02SepEq-M+OGIf`)7IT1rIE5d&m^a+cK7m;rb&`Bz-L&AwQ9; zwKyng^PQU~cp!EUrK7U3zzECNNSKRRQTh|1poG!ClF)|zV#Puq(M$h1>i6ulgBayt zfqVZoXv{o+=Auhn@w+#<+B+AxKQ?09J@;UN8`qWm9#l?O!?<0ec48Fu$w5a@1GaP; zYuVR8b`KiA02%DAUoTd%-@}4-#d>jG^4MypyKEba#_1~aP)b4}<@+fm;st|=)0kNG zzQ}xqs7_DB*RhV?EijYhed@uKvP!aZR|tR*WkMlc6&|LQSJzD1XRN{Sn5ODBeLj-q z3E2WMY0<_M(4W?v;M%~tsZ`xndLACk8@E7dP;FZQyhH9fQg#cI!*i|Pr|vpZaSPYJ z#li}faCUP^2NUUB*J-@4>Qz2oj!VPA@D41THgZXCcG|1BI1&KMgP7&HkX#NA#*(I zxz4BUe0iOB`jp81m?=$2IgI&^s!--AVy!Ww-z4&#(bvi78I_ElQHki$bWwtmP6_Iz z1PG)gXuK}#HJ>Z%C0}w@lNaJN`xfmyw9e~@-gyiX43ETYa$)$UP@$P-Opu~pM$RZZ zBS-aOcJ8;NO_Ug zD^wVc3rZ)*sHvR!FH5IWMHA9+HPm@Ki?@lAmdJvFOIANoWGX}M$U01KnJ71M5O}GY z4?#PkcfPmI+i2gpUC8}9>d-3``H&$F8ORb zcdhen-TC4=@7uRUHZk_HP+?+<8|}D?%vf{iQy2}mG$gWdbW20xmWISF49U1%o%ELxwgpk;5(bz%k!HAM?RL8K zM`QW#E)j;DJC$gBKDx8q^cS;Uxdy3s)?0iD(DQhP%sc%vQ*)AgH z3K285(?~&Y$vVrnVc$9mC0IcySs8qfWL@+vT=FU7yr5+0A!@`48yK^*8P@f-$iu7( zC8AW=x|G_Wn&@B3W&7orQfG(9RQbHyW~Vja!vF7jLe^r_hzR8_c!bOLZka=_$!mBu z%%mN8?_tp%P4->);ON&YaD_fVSA(y0-YfCNm(g$e0NXOWZ!aFQ9XBj3MdxhAki&L| z=X?FVZ!a9OJ(^w8Xm*RRu9OIfrJZ&(q_hKRxzA4!B>doEe0sGZ6tGyXKW?IS*htN8&1PcSH*DI zVJfs$4QJISu4Xt57+sr&%bucbi|8nio`1u|xtt}L#ytk;9Vd$Z6E+BNiR4oF-Q{>} z;5-H3#&U@i3%^hSx$&Sdd}A%~fMQ)`H@6u~#x{dOxl@7L3}Sqju*yQ=56(O8J8awK zefyyy2W%|UqcHLdhn$QCf3fi(w$}Uh-H}_OK}>zyk_+A^9$?R2^s+D7HnZF9bxc5S zUgd;uxvi4)-W@B3G}*j6?i$jL=R-r9?cN>nL(q!sSUlulw0F8=NK-V$Bunp(jL2iy z1EbU$EANHj_hI9M6ti=00&_$r+QXiuLyVBIAju;!Fi~5Ys4AYyYA%If+_Y>l)$gL( zo4~Qv1!FzW#zOhxAR#szDBI5k8vg|p>`kDq!VJox*~Vw#g9?ki379=F_a?Ncy$P$) zvw^f?lK|Eo^RPDoLn3m-x?^51dlP1rwj(kAdlL$6Vq*r^9Sd!nW#QXa)`GLx^5Gku zyurrwM)5D8Mz6I}}!2yAwg`HgZG;M*ce# zLbyXg=MDuB|ByQrR$Tigi0e0UDCoCCVWilh01Y<#qC$JclH+(J1WSZ0RcUw7{81y! zc2C^k%k2p?RYqARx)>}_sx=Bj*frWixI^RbLNkcfMwqAWZ72e4;E3H_X-6_xWQF6CJ}ue-@~>_Vce?bV?ZCip_y@B!XmYVn}rP6 zr4&tnA%@ZtdfMfCPc$x~1AcQNpto`4d@MaAq?`kXEjD2Ak))}<_iaafsC>ByLF;u4 zmMnS5`>k|iGHl#X(V{j~#KT%LZ&?a{h7AGz3>$siP|*$tH&o0jWzk5(1{5|_ zs62IGe*~1PSrjcM8fEru6xjgVmPkBo8;crnTO!|kVVR^VY!k&|+O~Fm4HddC4;w0A?HHfjNI=O*2i)C+8mwv}w$gx8&umF9*D4cP`07c797QO1S}PDcA}sMyt3aS~Esy?5nD zLWtY?C$RU}R(O^ixt}5y`zf%vyQLX%G`3#P{SI)+{S;JqY^SiJB49g(YR9pi!iwz< z*4AbB3m#%S1w)%E-S$H?w0_$u1b3jYpeoqIDYsMf$X@NHu(p2PpFP3|_JD0ev7176 zcx7i-c67zn-T~_|UFYR-rt5s+b2uD(1=n3;bU5tb^5<~iWNNHCQ^#a;eA-cmC&$tn zw2=F2$2Fj1jjzuhF-G7%s_UQA2yF`53D@&na9rAJaUHv3-DP5&$nm1wqy3gTy;3LU z{V={mw;mtTwq|};(`I_~>o{tJKlC2>@%8eObCl)mG$WxCtc`xGMA*@uE&oBrI--#t|Z8O%mFD9Pl&NziH)! zzY*1l&7=5gH-?etvD}mPN~RIf(6wG;j2P@iUU;?s zQ;;VmUhSHP0aLs-B5ew3Q%F0Jc7?PnWHgb{3KqH>W8t7o|)l|o+Z zzrhFO%VgGd?8J)-eYwn~RC14qiDER#jJPBs2G&$u?yl43XP{kwopQ+t7cd;v|lU8+BStynXrjOr>`c2I)|eRx+somMsk)OEVqZ_b#jd!44HYc5F?6rojPkIo$Lw+ zSNJA9jyc@P8M<`}|Hr2rQH5^@&orVE-j2jGV-qrs z2&`0rNB&lf@knBnIvXR0lD?4XvjGYngm9s^xObo8L6gX0_zJ`_f8%_UlfSR-l=qDwh`)`{LxjD6t1g5FLndtkT(OT9S~bVSGfVqAji{3+Y%xWb=M;#x6zYAy5_>*+Lc{bKaZl>!axRMU@agMl;Koeeff++`!ztdNBl0F=EPC!1m zaTJ9c28bzcgr&M*thpZs-ihnisLIeKs&WFLP+A(W)5tCwFQd#NCJj$HijSXl8BwnqQ7;*|O+slZYMkELEJW2bHMF1$6t$_TS-`rc76I#<+5w?6HM>aw$EMw+ zfTM0VN#Lm8t!R$I^BVs5;UC|x!UEc)J$cW-a~S_k_&k(Vn0Xn3BL!O~ zkDqu7+a-U1Y{M?etrscinTEDE;ed5%+xjW@D0JQLG1qbTJzHT)w79?Gp|OXH%dJu5 z)}J~xEz`(u7}tQ>gJaE^2}bsATt@z(!+3W;cr8 znssG&=5*uh&Xc%H-Pb4nk;_G3n@0m{{V}+T^UAHioR0uVMaeJ?ma+A(siv1V<;LTl zIU68<$&UMwFnPaaxAwc~`FkNOhW(=bURQrFTC$mXtovK8l3_*gc@|j`(mrx{QifSt z|F%H3<7~voeT+&ztlV=i(s!l-#MN-32EtVXm$9C^7vj=B#l7`t!ygl8u-p}o$+rCM z)mU%Cbnjzg%?n#7&~|?allzD1P-;)a*SH)9|DY$~|By|12?e98qJpj&Zi&JA-uODi zMpzGs8RaUz$2y>(`w&)t{3g7BqhkECg0f_MtqQRfke|uH$9{}8^I({8Z@RJmb`+n( zy;?86Y=a0D!`2~#jIu(^eSBz)e9ws5?Rx)1WBwj&L&Ke$X}gV`yKpZ}iO6I_U86K2 zrH*ZPU3tlkXvoXzWE-AR$F{q!S|RN)a%ko2*ml>|Dx_?~Q|e-fY*5G;HUY&uG3oh1 z-ZR+sGpWs3!<}24=;ucOcgGc9z!uWK;pm7d;yxkbyO;QZt+bC}Q!9iFkTtLsA-{?*-k~Q-iS}_Fji<5U#u0Q*eg$Oz6?{A(?PILtF3Uo;mJJhmBlGJ+ z^dBNvjIdnH@`yd6V75oI_(Gy9ZbY(@ce_eLYO$b35IKc_Ev%que&$BuM>81rNTaX; zldk0bm(%0C(}LMy^c;>EYqz5_hUww>#ZA@VLJ>mR3vR>#!-yEhf1nC+Qhl8vD0W0k z`49=S%^_+vHm%~jD#m)=M@?JG$`S9m2jc?VHv>J#rDj{9DzT_3p;+B8p!kkYj>HBX zjZb@tgAiO2SBv*9To{Xx4$luP5Mt2Q7$Hn(9Uw1C;BAPHldNJ`HGmW3&jPZGu^hqT z+#$no*_eo1Y=V6NW+&eL9PDP?1=OE{YXs60_>N`FFhJ>-%(wWnk z9v3M@m9k}pL^$fvIbr=mf20BlN4X}q@zFwDVMyeMXy@s_02av!A%lso3wd0Twi8Dna3!EJgXS!jfqDI~5BnLy$h%>2o489Ptm|G36} z%riRrC<)I*;F?wI#0=Ear|YDn>S?cZYOy>q&rV6Q{#2Y9vE9>k@NAnKhdp!9IBu7z+(8Hnf5>sQA zeJJWXK8+dg8D0E6Pf92hS1%?(VYN-}IKXS%a~fxDF|hKcG?{LX3!j{EsNXb=#y5rzB2_TyA6ZY*n~&Q*yjJ3HtQJvEzX zq1mT`LZux4kbNrRD@66i4z+9PE7574DO_{$b=m1+&%>|NHSXiK(hJzjeK4JUvX{(`*xz8it$n~> zY~*XTq5r^?p(6&1+Z;3NGj6Y=hydrVT^3Mo5g-O|JTu9F|OYpqWtBJVu6 zMZ=B#u7myqz&&Nv5&-NbY#RgFM7Un}SJql_-wxvN>z86@B@e8sWA@A!EEDmJ5sn98 z!{j2kWagQjRa6`6D2R>3y}RF-7tmq70106%N>0V#_!j%AqTg!f(M#;Cu0mkhSv~m! z^iGZN(ilY2Y9JS4aUGHFz&%(Fmya*v@WS!PJE$5(xLPL>KAIed=N{as6Z{Z{FxctH zi*!!mP~F}um}mV9NU?q!TZ&DP34Qy-dhWW@={jS~!T#S<*kQQn3J$u(Uf|gy9y;=j z@3a4kYs)(Db!{uh6K!=L-kmcp26yK;slsWmppTH?D&-Mh3?z7yPAZ)}-K5jP7j6#$ zZW45Fp@gjyP$=fvZ65W)6d<#vHTQd>XFk$743iYN2M3a2Hm13F8)dv=tmpmoxP&Gy zt(0t#&+*)ViH;q{WR0Qd8|=pP*|GBw@T_socH^cZ(4zuKfV)?C+YMSypH@v+;%mo} z68(0C=OsAAYq$f4_-KW%{QRlLRs z#U#@Hyf;q-ApxW-%|+?}>|vxXZN5Q(qA>zQPZ6Lc0fk=7yKs1a9P%K(<7M28&<@mID(!G1rRpt|N%`mPl+bdkLM#_G) zHWhT%*+wt8G&v-*lwBE11x96&`~)0w~Gq#+uJw;kB{1-o|em~k)C^kuoDcm9POpk zwS{KNAr3b8WZix?i_L(oehQ^9u+Ia`Fst!NrZIoYjgSNH%BHZ);V79x?SP|V3QHf3 zswvbDIBKR)58&7|g_>f7&u&1gph0tu@Vi@#@D04h+rYgNM);Cu2QKcTE%hXVuYrY9 z@or^1_kN7TZ1;B9*Cv?SY9UL+ZlpL@kW@Cvgx3O`7!MAm!Z=DwGNjGK3@UmJ3{|Sg zRbYlfkRq^}?nxM90X^S9P0u6S&=6p$rXAP{>z?(Ev*A@849y2x*ViRX2r5m24iO@vb}FfpNi=Yg48D65JrTtwD+ zc=vpKmjvK#Go?a(Dm-F7^Z@G9xNfn7{M8a8qH1zL-fe19I* zF?mS~ygTSB6-s6k7g~VthFY@0u%H#p@2(}qea)bK57jwGY$E{=TqXhYKJnT!7J4oR zqi#>U7GlG5B^V1#r>IhM+_*^w%_R)uDK^6lG4x%Hv_o4ZIIAH@(yJ)B2FfXSK_P2f zgw&gh1lXdn;BBR;0^C<**k)HV8?U`!ndK@w#g-X2S)5|aJO_R@%$OQR4JwDot2ZVh z7#(8UiV?m!hHS8pN&unPC`_48;altiyFmj0c5^!f0Nlc?g;-KZQ#-2#a$7k-?4(OZ z*)Lfi+)^=x<-oEkbrWmzRBOnD+c9j|enNX=l+|!j6}u?#3hA!~!2Tp!C8t|hAGfC* z9$b|{qv|eYFHUu?ghdK4Xr4k`nDJ>WXxy?hht(m)9h2znhM{b6I0W_zS)Q^VK_^5<(WNpp zc7O=5xk!L5Cgd(}HJm>*KBx(%I)SVWm;cSI*LU2Z*tW~V5Raqka3$uE>Fn~8D@z^9g^EJ{w9psMhb;(8O#)sjE(JJt6CXISt}wj?U_v|nb=l_ z8p>Sf9wU~#tL5tZFP5&Rjlu`khN&wHauI{PvY;HU1Xmlb6zIW3p-7}GLdCG$uJd*Q zln^6?Ow_p6cxa4DDy@|g$4n&dSwRdmU5rzjF2+gIMT|0CL>actELUN?V7uOj?GhBL zJ>u<;pnY4Sk#4M#!^Iq+W!aL7Ft9V2zOrp=MkW!-p={E}G(n#LEovrUC3BRoKf*}% zB>vAL+-7PU;Fcmnm=dm~oCdXMe!!C%&p;k3A_#>hf`-d{NiS#8c7fFubf|=Bo8_*! zOp_&&`aWj*^E-m^piL{eOg+TKke6x8u(LzZoYw^S0DW|6V4&ctp;E>J+r=OQMA|Hk zimkK(eke++urao~Vb7LSBiQV6@Yt3pu;CkGB=k2^M!47)2-3}V`LV+LH%v3N znGvv}DXpWT!@H#%Fm}?rMp-57kFqt$3QX3Xg3(BYIv`9U%2JH}+?CWwF71_^iX+lI zM_L`HJA{+%=KorACY`T-WW zaaK0ci4G1X+4L7?c_tr9a1VZQ_CS1}@W%Mg+XHfOMMcO532q%{9~mtt?u;Mk zzLb%whB4cA zoSD~Iir3Dvi_*MyR-8rpbEW^$$1#MEr~!&1N>KzWf3))3lz)QqM=F1e^iNYUOfzFh zRPUE7`XuRpl;d25N>wPHv{tA^3dKbc#aM~DMu>{t1l4$j6VJw)8=)$g^nW6*@mSF8 zNr&>I;yZTWV%I(iE@vOizB~Ka_=v=f*nOx~DCh0CcH+=g*G{}}Z|8Ao(Txf?F8zP| z+6hb;bN|t;kdSZzpx^fjdb$s#$NMLE9m>`BLGUURzw01)6Ut}aKfxOxc+UebLGioi zAqe9$-188EdmeOg&qIj3=RxV?2&kaLwIAa1aJJ)b@M_MW-B3w&Mmy2*Hzcd09Je8B z4%{P)y(hS)!|94x=XBw0IUm7%AcB;2Ho7ACh~xv8OUzwo=Zj&nlN$5tXL?dJORQ&l zhhtLnvYWAagE|13@{@eD^Kp)k^DjEfFELj0mXS4(3LkE3PL{>J|B<(_z|wU(UaoEx zH{RrUwZ-Y$-*}N6Z3;Vp3J8Fwg|W}Klsr-0PZRCAt~Cf;I>e9 zPdHC%o+p1AN%^ev@F$ZH^2K;X%YZ6Te}rb@uBgJ+~SG# zCbZqT2f4D56WTaTPI#jc?{S7(PT=}YaBrQ+0XGh*w~%YG*PP_ORD{gV+sa96A9%`1 z?onuku{c`Vf(?wV*~)Vqp3VvGUxKz(#~TMe+~~l^<=rcFbptvBwu08_iNUJukHkc7 z=Z)wNS7WSaZofXP`JNx%0dWQ7sPV9t12wk!J-KA=7Wskc%_8{JsdOgPRk;objwhJ6 z$YX2KAL%c%QantyT62ZJ!pg$MY>zcn_^YfeSIm-G6NSIVS}puF))@H3S@hpQEXlN# zJTVbEG8QX%Hct^Kv1Z>?yp8#TE81D-F{O>AcLOJzJPFa*q{unt%m~gYV`rkdCDGW~ z&^4c2QkshGx(H#+J7UZ`Zp=Fc@k&#XHxR3|1tUr$yrwi)_&1fxx#p76T!caXij9YV zGsOAx{81?Ol1hdY4hDsztdjiIbn!n<^aU{@qlQpy!s=oPbKMfg{HKJrCnRin8al?? z@YesQIAGoo-_aCbd>%cUU*Uv(yr+`Wnz%&Wke8_2%uCc=_zZCzHaRpJWe1zDs za1h#{SBj4XVIu`@ZpO_aYd0d-X1c2?IT^~6iqqZom7FidJSl6$pN3#c%U!mCd5 zl@ykc!cKRWiMTPHis{3Y5i!86~M={5AZL9XL-&DR5%CybXRF zK`WatNygpOD$dOsmQvg#Az?T_zJRR3;*=Ele;@GK)0M8&vzb2G=BZQVAk&!B^97fh&vfR{}oAoj`!R?I0*tvHd)Gw6kChwNOEy+7DOlrGV z>eq@JyX%(={;%|lyH~!_FHt=|r+zuz!bZizz7=nP?ZVbUVR*4JC9EoqCl)^N4GUqL zX=GuKZypRve9u{J8w~v#{2#*hnuZkDcu**Jy+FX6(y9Q^IGA=E$H++|TPYf5)BM7& zvlf9|)CO$%pz%$k@hv`&mh@(<&SDm=R-8;mdxP`CM27(FbTzmdi{I*pZ#$)-8<=Cg z|9ZAfvxDP4X_V&r2<&kEGXtUE4if9OuN*8AvX+;GRd#16V|G3(<+Iyz@cm> z?8b}2{>TbVvn(n!0>UbHS?2IMAu3G__oU*A2vJ@Lj&C7!*wx8%@a?j=6_O>0%OtVW z$a<4zr^`-P@_W0T_y3PX=Cu92D`_H4W^zG!4;_s?MDto8O%$n1$qI=re74@)djwL+y1JyZl(G}gI zx}u@7*H{utnb8#uRbA0g)fEkGKtS~MLQ}>2s!-Jx4OLyyPzoy=7Q7dy#{;^em>;Vj z&Ya+^CZvbIYB;MKRs*pZcvqxY3@jPWt^wYa4HrEeECvo|B?G2rII9^PI5m#yX@uWb zj@0q8SWF2vV-=$s3xEw+0IYSmXS2_{foa8bo`&f>BYb5GHd-y6iM^})(GNas%tOET zIH*2kAC{e2%ydm0@}Ug_Av(^XcQbLwN70BPQH1yKbR@Y!_7kaO&)zNIHqqNPWjZ`yk*#bT@&6e3KKlra~eH5;oDUFW_zZo!@Or$zJZA#W@O8ki+!a^_)O zd`lGLn}=*(`iwZHyX#_Fo;TvVnm*t`-faTuo$ByH3L~xyT(ASGc$(xby7Vq_JHBaC+Vp3e)do zEk+T$uy#o1co*AZ;8ePSbLz%ONib5pGOlfe^_qZHmemryTB5n2$W*HZ#dfkxUfHac z6Kmn+LLyfb!-J#fcM79=1UB)kFshg&5JvT^X;eSF%Ba%FlPSQ7 zj83D9Dk|p?XjNgIJFV)+)afBUf3qsoe8^7PRY3;3N}3mGSjipo2@6g+8I7=*m_0%i z)J0E7rI=n8*ZH48sV4%d?5w!lU)b4B?Gx*;)Mltut2GMj7<#5BYclqG2&`&;=k0>& zmpA;sAWZKDM(*g?Yv2GlTr^4eJOUM*EfF$7*<4WvkR`Hu3bSK&1zcX#NfrtYjXrWr z%r*xXzi%nt-O{nW_@c!qErKl4vd$d_dwP+kwVlz*lfbsZ-c2i2g<{D}f?0Rv$TRXx!`USMi}Ty zX@!>*QG?Zsh>=KGL87f3hhWz9W-l~Pv@Jh)T8MGJGYatc=D zW=va{pONp#%Uig@{>^XNhxUZe&MB}j&R$%wcwyG!hcnXQw$GXqKh$K^BkazHtfK;W z+ZX5C=iHr;U|&e?=^4xI6YWDs733_)FUVV*{Xo=%BSsBP$sRglY#K39Bf3YkI6FTh zuV8Ub_RQ}kc1iD#wJ#Kj`aHrTs=SPa>31z%n3XR@@ACc=sTY5y)Lm)FS(cHvFe_{3 z^h6nfpZ^PqAw452LuHSQz#s7i#Lz9LMd^0K{I6qRh6~l~%75wo9GnAmu?G&CmXr6u zg8TDw7N#Q^1&a%o7Gyx{poR;U7A(%nUy!?Sao(aOOBO)y6LKEN%E-@81SW3w?6GNS zT1NJ=F;iz|6bL;`$jE-6U{O@}I7iqg+i!`Ej@Gn)mP^!-e^OLJHv_6|XLiZD(7V!i zDsoq)2|)E8)=#dk(0<5&R^z&KyeoJY?w^I0by8-peaaMjPr2Ti6k#ps`)qpu(ikD~ z^|R^y5{nrAIsdNIQJ}2F|B;ckcu`JHx-)0#{aG1ZxzP1S;*^TQ#P*cR0O}FUjqIgM z=4Rvi$>(%h|{kChWCqhd z>(*rZZL}Pkwk#v9ASZ9)#09e#WG>FiS|BqLv!&U~a?%#k+UKJU)wF_)ye0kfmx})% zJk)E?chyockX{p3}WJyFfimtgNYiWPXTrzxpi$v;O=4 zz8LvuX64+!Fe@H?v)<%h@L(FtP9{e=yJTs<^UpuIY{On1Y|rZc8=i0P{s+wwO(7dR z{0h>>OwG&B%E(5`M)l0gfGJtBu;AP28TT)JU_tuQCAnh@mKSI_nYjgdV;AQy$jQyf zMkmf5ZO=rnDuasXr_WMF7A#!4T#14F7KUlTlAQDfre;Rv7o<;|c>luu#c2x`XQEe_ zv2e-cv_%W^M%%?3Iu{h=EnHlXFVIt`+P^v4ZiZ$a+@F(^H3AB$n47PnOozy)`0p*4 z5uXriAK8^rN~$hUknosI&H*OuOn#AX8+RAN(M8{McLV3)9ljjmukDz)pAXfn*rRJxTv+ zfX-xy&y~T~8*Ep4(Hjhok*s=xc88{Y)<1Rbv;Jei`?LPBjS-Re!9R|~X{!VN;qCTb zhaam78t#A4uos|5dQD?G{p&*aeDRO(>a>BWX6A!`elZPZI!WXI>&{=s&-d+j_LQD3 z{xY7=mdA|Kgx&M9XZ~I3w_G1IsR5Zg57N{Q?w$qj&m0c|&RbkqRaZ-*aCcsA?- zN(Z!KpHu|~uKZNyNx&jN9cy^}j_zQx{SaQPh_ zDS&ps62KV1O~e;;bPNNX4e%&n!_tn95KP!M0}cah0h|oj4mcMudKuCW7z4NoFct79 zU@qV}z;eK0*z;ujQAYfH z+FybnU<=?S;(vwkSj@_O0_g&M%%6}i@F)Hi`~bDTA>CN^YuA^LX!iU+Li|5%tF2WQzp zeI-Kg*tuu#zL!}?i9wSkXx4{<3i`lscL0P{$BebWhn6aAvG(x_tLjY}`ct1TJa?RZ0tcHfC(3 zVqN-3t$B|-=~5u)quEy&Z)FC$pvHyQuKz^CFn9UU=n>o+rhWm!MgM`x1~ zX4kQ+2W78HONnoPj8y6TSK({n)<0z-$?)qW|F;FmmH(@LUOb#J&07CZAuSR z4AI5<`aX}b<&$#m0o$XHvl)K<_l!sAn{90$r4T{-db|p=DK#^TCTMLRje0&BtgG-?jb9`gnrY>!_}Wzc5ZeN6~yCGO;`kAffuCtIO#m@R!j3`r9eev%NhRfS(Ke zCD5M)K9@9qf^j#MkM(|iUr~#Z3#{Z&OoFvRLe+N&eUcUbWpdYciq~PmLXU7MV5fZg z18k%;fiD9-P(GG_jgqe%svd^~{u&s_AI0?YX% z@CgBUmh*AooxoSat-p=JHOqM-5M{DtJyubsDD`gwxknW0UO~4{7q&H6pu(bM*kI~a z3L3AnfcVmZp9_2m@C|V5tNr400`XaTC5^SIT~e>mXVAg|A_r_)>O!_NiXu!Qd6*-j<^KN$EJxb^Sg<784->^j5MM6uI(#Z0qXfM$gkfe<m@NH)!%ZTWsPjE_2a4BGP<8Q=HPODF(bC{~5xxR^Q{mQw7+wX# zI735aVE&>z%C06lqqhKbuit{2MEH?k1iFniCchmh4U-?@xtiZtAU~U5I{4LrAN5Kv z_TvW?xtiZ+%SU_0@mLA+>!0?!ry@f?yQJ;1l%@W0JXeU_V*dKaa&rP-&;vdN_^bf@ zanKh4KOX6A@zaZc#U?>lMNy>cPk~F=B2jO`a&-)gw6Ij2r5P4&YVtoD8c0n}HT4+n z_zr|G4$(IP?9k``@0bZYg8T8b(ckZ$Uz@;pF7U&EZ(x0W24Z*B4dzH#qJ-@s~`G`NB9o_eF66AEECFDBw*2j9|L!H+uGSivdUl8Mv|QI9Z=x`I)9_#+5JP$>-+J=93P3C+O2bUj(<_0pl#qShO%)UH!`&@!D)C z7+GMiPPM&dI{Kn$@l9I#kzYV;LOhf=2K`z6_l!r@C!|l>mG$PF%VWA^)~z2PY%98X z{618d9W{Fd%AQ85_QGc7mSluJ8M*|XKI?ih+r>Lli#a6D8)iL#b{dJ&U#HO$wcEydUo9kv_KWvUM#hbxhZ3wh(<;`IdgBBYq6KMuhsYoI>!}1Uy?9^XG+wwWFj2^Cp~n zI0M!DtaQ&q0E$%Jx{B$3ymgfhwG`7F5c zc!+eLWcmtGKm5|ibn6&`A6N00a$z5VNX!K&P%gW^wvtz3#LhEg^95sGf)dsK#W=t z2Ze?&Q~JgrUq4r!1LGcJ23t|DoS5>h_zU`RM5rI@RSNKXfNy8A)Q{<&3%v6M=sE4l z3+x1cM!Lr!0At_Uwu(1JXC-^QIe&dyE-u zMY<=W7jpEka^8qI{+RB$z^DJMyPcsvWC34R-?Kgp!}~J4ufqEVxIfFD)PZjNHd8-o zcdoW4WYO`&&ny5<(J>m9{U8C|@YG5lnh6;W0ULzME@@x(QjkT7_!w|+~zqZ18zBP?Mv z;s4&PZi}|TV!r8k;zmp4VE9H@aoy0=nU7nvNBQ!(P70>gX6gG_5T|~jOe;l19Z&S_ z)VzPq{iMb6pOz1T$GzNNyWHRU(*Z$S1`K#%fVO)8-KRpWz&{aY{qL}#SHlMUBTPFX z+`l%ge=>qzHwL_AXlI0bqYbt7g>P7od?Tpi8v|U|2mS8)Z|%Qcd*gZt-Fp29IRA4n zU%rUv;GoNcN%E5$=>FRc*1z8nbnymaN^Ye4_ctO?XOfUk|1=pE>%IZ7roFu%WHCZ9 zom6iLZ?kBR^Z_USI-VF{(MM2I%TjgRF2r zA7rf=s8#56Kc!p$IzaogemmUfbn9;iXf7+=FIue?1GJ;op>RLhcj+trwQc>Zj||W* z_uFWJ=MViEcrUfFGwy3Gwn6ZJ%NnOm0Q{lk6CU4W}U_Qmk6!APg$Q+1GIN_ z>kHRtmv!rF*JzJg>HeMdoaG;e_GUk;=bE7323s4$gUb3_+l`=C`dfiV+5YAlZR3DQ zxZfUN1&jY02&R7;I1A3-gp58PrX3Es74GUV>#;EHCF<%1+koHNg7({zi^H|k;dEcR zW{U+hz2&`sIr+Q8EWx9!^Rz z^{)d4cn1W%Ghq0m1GQf;jZY1*rvHNg|D^zb zKQ8e2(T4~4ZwT;@4)9M5@W%!ClLGwrE5BTsM8r-Jy@u@t;ukkmN{%@(@e`L%z=iR? z_@$~rJZ7H7?{Ve8%9i+TRs6A%D1Ml>6+cc-^B0*W^NaKK{KY9j%$XyuMet|N74Tb2 z{F!*$c1tR0$Xn(63(DU$qb2eGe|aV; zJq}FoSfw9R6`Z5sd73l@i>j;8+ExDmX{M z`3hz$xJtqG3O=de9~69E!F>w8q2Or+uP7KcL&dM)SOuplI7h+x3T7*~O2PFCKB?dz z6ntL6eG0yz;AsV~C>SX$7w+ z7#64ES8%L?Qx%+};CuzM6#s_i&yb0I99=_3eHh* zzJl2bu2OKlf=?>=2L+#3aG!#2D0o`ID+-3qQt>M|R>7$X&QWl_g4qhLQgFS3Pb&Ba z1)o=NpMq~Fcv`_L3Wj}G#joI41*a-FN5T0DW-GW#!SxD0so)X$7w+ z7?z;oS8%L?Qx%+}AgN3nPIu}9(ymWhn*ATyIX}vt^nc+D@uYFL-zxqmF~;ukw|r^w zf9JMIJ;a~7=u1fdt>dF7@xO=oTfUU|Z@XpEt@!UD{?x@^M*45RW74ho?;-w{FD3r( zjGuHX{(Fc&^}#PA{olDA@!#H4{4HNf{1YZjx)uLDrT;&^jPy@H`QyKb_*=e|`0qgd z!+#I)r)GT_>AwT@5C1*H-}2?ee=Fj@)r_Au*un>%GpA3VXpfpX=k5{q@ngr2{f_|mu3}YExvyu{+DMj%E>RV`(u}9FCgyr z@#)>r{}yrnvL#sy(ii9F2C`V3mllZn*G2e$ovSuBf5no5h4%v%5bm4-UjIj@7 zFCBaT(#2WnV-}}tV;3#VU!;vqUy+Sa5*FlXf(G|!<>Tg)Zk`2rmzR;XkO7n{H>*G! z%S+6(u>~2pmtt%tJb0Irj!RCou^EdNWaceglCfY>I>MShFilHakg+^1Be!4ygv`p& z#_|f5e25^ygJ~cq3&sa-LbioV7N;R-4r0M$tcY4hb3blT8Jm`~gqJ@Eh5TPXt_O#H z_&57F@?g*(PkV z$pEBh9ARE0Ogo8x{Hn&W;melxt8{s#lXoAU@JH0K={egOUh!&8lz z#J>0!KWBGn#-p$z4W^rrlMnvkBUN}4#+VVRK+FUFRIzFg!|fkFRrR|~*!r3v?|hiy zOhgeJ{^8Ae9TQ^jTeqJXzlmH62g95BpQ`4COlS{aU^fFQxE|0yd{H!gn2%A=YcdvC zsRPO>Koy?;;hp1TcoQB~^Jr$6K>2?Y5Z*pphBskBK!5zd1jY zq{7=(`I-4<#&4$Ut$_HK-#-KxOw1{sP53Te`RAWGZy1>-!$+GDD;8!r z6P^hOAE(BLaSzDwHYU#R7r;*AG%$IY=}%oUL{KL!QQ-sh*Dl_6Jubp2DZ6lVnc;a9 z+W!~e)wuuNjN6~CC0kMk1q2B2ins$r6l@m#NK#JV=2HA_z<(kB&3rTMT7Teriog0l zW%$UEAn^MI#_#v)DrXR+Guu)BmqD;LX8YJ3AEcS>UU$5%neAD3yj3&XsqXl`o%6xn z@%=QjUFnVw*39;zJ07{?^v6hd5Lyz`zWcuoLQ8Ji?e6#x&9uke@u8Y&XS?IWI_+n7 zyrG$Pu{++Tnf9(b9@=HvGrt!>+BKSKr~J^+gRg0({ppU6&`i719Y07j?L~L|wW$9? z{87RkgxP+xe*3=+!fd`-Z@c5K)6DwV9j_K2{E*bkK0V|=>zA&5(BVfaoU9fW*1mxJ zp}Jdo$RD-?mLBql)5(mt$Zx(3DD7TB-zC3UZ`fb4XxC|rl$|l_<8t81N1F|Y->*0M zOF%>UqXXpsslXfBVAT$qtbQ%$C~kvV2W@x&R5Z zV0FnpN~ZLWictpfmM;DLxx&97=&#d?R6dw|-VpdMeK745garS{&iTho#b>a>Hz~Vs z(oa=*bH31||DnRisd+S$zR={a@FxA^CjS8Xznc65=>Kl=SMyXR|MMn)g*W*}!1yv< zO#$?`E4(@1Wzv6N;p5aikjbC-#)$k;c$1#@f(ZTr^nWz@2hi^``Kx&rlRxj55&RY2 zrDOu^uIItE4)d+lX&Rc*4a=b z{BUea{DLq@BfeST&2`5Zg>O}ODV5w>PWq5Jk`Io>iQf~7-mdU$KlyuG;YSDHL;A{i zCj{UV72X+uuTuC#g@0PI*RIDPmGYbmpnq85weLwe&2`P^6h1=XO*z|vXMH!<#ceW} zc9Hb3OOsW-R%k6428!|86(6&_A{Bmu!kg>L%M?B?0N)=1kWW$oel+p08!e+`jH^_7 zZw0<5`(x7IH%8K%-4oOPXbOLy;xkFbyGX^mSm7TP@kVNi%Kn(?eM-?EQ1s^Tc1kqTcGfS<4M&j#REDtw*7)9&;4q{25Tyj!9*vm873H7LAU zj)^%#1jbC)dGP5eKfjJ9hxwm6R#Ms(pRvRvU32e}cthduSNQY*yeW^lzam=EoARUv z$n$gX=}Dfal{`h?mhqb743odPZzGU@dI0~ez3_is@ox#>f0}rtw;&*0LqjCKBmi&9 zW9}ablxI|{*#5P>l}md6!+ z%xp<0?pJ~HE>s-yv77m<_^eWRZHGj(NOjkKr||7r5^v`J060XvO0Ud*<1pYwzU4@I zw$uDAQ26HerN34JEl=Sa0`j>M_@454o1$;|jbs3?_?>`*@>DB*eq7;sg^pdh{lPW8 z;J*ocPw`Ic1%Eg2OmE^k$vsy^@{Gbaye$znv}gS75qt(}?T<-+3;JRFomKd1C3vd9 zqK-nCp5z(S3;xz#@UsPepw{A*jAK;!mJ0mU<3V>Xd^YxiZ|DX84)DyM+JN%<20Adr z$1IofG^q5ZDtzvbB*HA;4GP~JpeHW^k0`GC9q9#sTHs+^gJlMo`S6X1p84=-VNd)g z_JW_;3w~iQ_=g1^aOy%Eb+QPwCyK=o=KhUGX_B=wT-Z$aqcu*W;vTPkg@B3*I5{1GR={B;y9f zKdl$~b-**di2?FI*9-k6Mc<(6shMxd$S2yv>IY?nrk&gfyvR>w$4vTz3U6N`>0^|f z20Egoj|q@x8t~+wxLp#~D*A^7UU~e$E4|>`dcl9H_-jfYQ_h*_C{mts)y~$c`u+>x zMS4{}+g05EE%1JVDCZ8r=W6_$ijOTI-U*lr>8af32|V=kmy#FnQR45{z{B)i_4_06 zl;0kZ|L+U>NNsMwIQ^pHZ@0>XCMu4%V?m94a#gv+D10XHJ<0hn@N7ro0@}4oK@UEP zf4j=hM#aB+y<}`x{NLz>|CL_wH(~;)r+l~#cu|guzgfPE6uv#6K0X3`PwCnw@aRAE zlY&<(IWP1=|20g|iu_mYRH~w{2EHf$hkC(}#Z5tikFr<1_m95};CtdzsOZaoE9rPY z5P!`IZwt_ek4^d?NxEtYwDC9f%)g)){9k*)zuXJ{6X2Pymd!Fk-m}Bsa699L-tH7$ z&Ag}OE`k3)&0R}xBUu$Tg3WBu@B%jLIsNjsH{JBedl?xD%%PNiJ#RAReK zRoR`4STv8tAa=81hmhE@LP!WCHZ4{x`v)L2YY@^bK(pmL_fb`@x)s~qX^AY7_|&a) z&%Nh;Zk3OKrGNhtaKcl%6;S&AOZxfJH}v#>4e+;7KYWsj+rJAq;dv?PQF78Pz(eIJ z-Twbn;`vni3x(%T1>@iTrXJ5l0sLnL@LvE<^nWPr$?wHBlwR=h(7U(3z#%Ur{NDw5 zzWk1!p7#pidw|otgZ~P|^!+CEGneq^{}P}1_rKuZUlri_V*&j00{Gqo!N-NPGh!C_ zYru7U`99!8=ch9M;oq-|n|>_el^cOU@#}9AzL)WL?|w^<|0f&{e{m-9z|YYSzFR{} z&sV;!hkvgCZUIj8?@9UKqhza@k zuHxT66pa5f;6&$V8N2jPKK`xTx!7j~cs?(HmmGBOV^3%P9DU;KASyXQ6h_O*r0kR! z>yFNb?4)On7{jN7wNH<_sY_#@J>@-o7=NIMH^9O3LFf;Io-gHQt==io1?G)!@Tp;x z#8XOWuQR(<^O*euKF#59|I}pa6HTN3PiEmyFY#GkwQUmh|6z<>&lW58ksJ7L;Q47* z!3+-`8>KgZ`@w?~>tMfBu9h3s2CMRgILx`k zEok;TjNw6C=G@#EEym{5;thQV(O}SqSLzM0ZB`qbFaZSw$85m%${05skjz3?G z{N6mW!kE}LZ`=oqvF(Hy7aOCHls`06O9$0e_5tt9$-rqBG;_mwP+hkU`cVm#y8ESf z>^x<{6v`rlqo^vnA5tR7-d(*z3x=O-x7tB_b;BmhxkKGXsR3zZ&%trtdz2!>@(HVY z)5>J(+Y!Y>Zd9(d8(>DYN?|RvRh}#nI4=VKf&tgKH#SEsVd?<3HRe^pJz;NBbPus$ z>doB(QtFjb7!n0L(cdhhAGmg=ANSlr|G^LN9>JsxqkD717I?G4pQGn`bjwxmThuJTWP|7>7Q)wC7{IoFG#T8TKFiu)lxW>tgeTJx+^hEE6QC#q80M zs>ip<4jUcV6LcMBqXmPiQomXsu9hPUoAv_TLC5Cc*qQC^9i8+ZcAq?9wQ?Pot*BaI zEV>N*XAJE|(OVm4Dewb~%?|Bb{xmel8L$X7=>;1*axU2%Z-21*a&PYgb7=L>`mea0 z8kRG55z#0!oRE(eUM{Op%0b^7;puorRbdXcPz7-<23Eg0Ylq9q141JiPY;0nC zQ+X*?>1uPyMJlDb4vZP;Zs-j=r%-5^Z*~Y?l2l(oiLb6oXakOy*cifdn*}bnk$ZbE zo%;`a%qZ7DBW-`3CrBDn6A`JCjBD*vF2IRB$6Xfvm3WvaK$9y#Zt9%S-$_Cu$viO2 z-ON27(i!-o3^&jo+MR+;2rFmqEOaAmq{iNmD^HmwVQ;{c_1m2V73%owm<#{e1c7Pt zf&3dJ1{;t3Da?wCz!PGc8o0Y4S~#+M3ZJFiz>eK`QDz5p5yK|?bw?yo5cFX6xgr4H zk!(<%$Xw3thU9gvXo~D9SHpmVN9H0EFyx}>7T$A4$aWe`u{<$6IcZa2w+xVU&M6}S zq?F{GVg^%zrOy+BuFhGv6Kk5pqzmMgu%Mwd_^}n|DoP=KMocuUQl>n+Si5j~id7p&ZUp~5ga2BIDpEJV z|HvE7L5p44L15nwF1ax6UJ>Jb&TfffLt*55htVIWjQ)W%`ZJ2$1Pg;ummG6ROA9@G zF&BDX!!xnzLlpE7sX_s$qL-JXFfU*e6dKm8!#xI6V&7DLtu}1eDwLzLRPOWJUdW)B z^(qUOi-jNHAO*T4tuihE9t8*zr2T^nMT+&&n_J@msjhwDA&v7yL}Ym_MD~=Xngd5g z0+$krWuK3QLfF5VvdOL{rK$W5oyAC(r6*DxxfmlJfLm^{^5@}W*c454#S##!7N#p7+Z1+;#qp6guH0p4b`{WN# zc)ca@kL%L7>q&>Wdn=K}NsN!mvo_XIJl&2rdErMGJ@Z$JK`H|iQjH=MLyjg&*khdb z2FMRw*yyBKR8W-E6#}tfP1-;yoi1%4le0-+>y$fcw?+BTU6`_aJ$UW72M&}Uw))o$ zM|orrQv1a4RVVVv2(q-z@Y-8E3)hXTxP8NZqo~)IOqbzhe%aWrnSJeoP1+Me8HdW? zheti&2+tJ?4?Hy)LlQh4Z-}0@JMF3aFcIWAHrWiZJ#pcZa{aZ!<#g=cEclAH4Oo*? zSQ5>Gj3(_YRjp8vvH%y@d^w|PEb4hkSQE7nX-;0-U{@?ENe#WM4bv$SMGyrQ8ciuD ze++oBFq&8B~*G5uhvu*P)nXuEAte;`M01rMGug5nvOzswI}4GO>7**n%avQo z2WZSn2Cu@A1HsXTbD62b0WHqY#VN)GimANpl*N#^;Hm)^e0Fh#gGKI$&8bUY8D$O- zWnB3QoXYq{6S4}YvYzg6%Pb!)6Y5Huo6=ZL?gi?#b1lIn6*~n%dex65dKbT>p2OhU zoE6TTYUIlZ6?qhsOs--iBC^C>{$*TbDkyT-u###j_kV^@%ZZEn6IIlwDzXKg)*Sg| zWN7{@TTwpmh$k|Tz9RK~Jb&gv{DyA0oRPKZc5#G82P;Cu@B48KjyM#tYNmF{J=25_ zpzw8@}o==-T~EfQA*->V@VmO KOxM<&^8Wx= 3 && string(argv[2]) == "1") use_old_read_testcase = true; + unsigned chunk_size = 100; + if(argc >= 4) + chunk_size = strtol(argv[3],0,10); + initialize_function_pointers(); @@ -45,12 +55,24 @@ int main(int argc, char** argv) assert(ifptr.is_open()); } + vector tc_vector; + tc_vector.clear(); testcase tc; while(1) { int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); if(break_value < 0) break; + tc_vector.push_back(tc); + } + vector results_vec; + results_vec.clear(); + results_vec.resize(tc_vector.size()); + double start_time = getCurrClk(); +#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) + for(unsigned i=0;i(&tc); baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); - cout << std::scientific << baseline_result << " "< 1e-5 && rel_error > 1e-5) + cout << std::scientific << baseline_result << " "<; From 28891117e23522aa7e9f10b2e9bed973e46068d4 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Mon, 20 Jan 2014 11:07:44 -0800 Subject: [PATCH 14/72] Fixed bug in JNI interface release_array Disabled OpenMP --- PairHMM_JNI/Makefile | 3 +-- PairHMM_JNI/libJNILoglessPairHMM.so | Bin 114104 -> 102085 bytes ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/PairHMM_JNI/Makefile b/PairHMM_JNI/Makefile index e966caa90..f6dfbb1f0 100644 --- a/PairHMM_JNI/Makefile +++ b/PairHMM_JNI/Makefile @@ -1,5 +1,4 @@ -OMPCFLAGS=-fopenmp -#OMPLDFLAGS=-lgomp +#OMPCFLAGS=-fopenmp #CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas #CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas diff --git a/PairHMM_JNI/libJNILoglessPairHMM.so b/PairHMM_JNI/libJNILoglessPairHMM.so index 7a771922fbd614ed3955980b46f605347db8f67a..053b901cd7d16182b21054593096ab2ad1c2a0b2 100755 GIT binary patch literal 102085 zcmb?^4?t8^_WvMQRN9QCF1eN1Qd#r|L=!`bNznK3VdTH8EJVP>KnMoZ)=UGe*ElV^ ztF7DGuI<)tyK9@@&04p#KpFp4hMKEs7HZ}j(L~Dx73cT)-22|}K-lX0{Y^dYJLjHz z?z!ild+xb^W_&758`&)^%wc`P9d|hhrHquA6f+|+y^SWtk?6R{k>a@4@l!$D-K6O( zsY{XNtkmI%7G%U;glChPzbR#az*}hpGx0GZWonh{29R0NZpk;=Ez4P{g9QATsZ~4z zC_dG)*ac|}%iv?Bq~#~`kSCY#Q?8Q*JEkkldaTqccOS}`O%Qy(?~h&1-XIRgGiuBP z$7c2Q$7=7Zd2ia>YVB0f#U~H_{nOu)R|THG@CBbLJah0oj%Nm*JUrHCHeSl`Og0g} zL;8C>H=A$)(s(=?o?G!u!ef1YgO^{Mz(O;fjx^nT{|VBGcz%gz5uTxVp2G74oVUhh18jlTnCWRD z%=9Xxx7%6IM%pqS!TTr^?qmcn(jciTnl9pW%5L&;5Aj;TemEpCUZB;c?>` zgvb7Q62Njizp^9$%X_%v635aG23!SYUxyxx=Ag&N(!CdoCBAyC#5d6i;gg8LLwtRK zgf04;VNh2{RJ*_ZK;IdDADmTZ_$rfsLkIp7LEIUAw=U|P1tX-K#dJvc6wAbM89Jpi zKA&~L|0_6$E5OIGMlz@|`J96My&P9L<}0$-gC;%)De0q4BTF%jtP+EU_!^T>y-EMF z$$$AOiD(FusUJ>O(9=obW9gaYM7^r4*BWD z(O(778|9zBQ!;a1EYm}9Xw+N70tq*m`rj(}z@9nq@R=h1VonY3yxx9Y*wsZ{*z^CG z{X73{$vA}yzz1l^)58(f+e8Sg!*Sy!o!8r=i+ZDAf}QE-DokjiA3OBRZ=et2Q#$nT zCnkPM2YlZy^gN&o{j7yQ5PEKr4YJxhxC{ExLO*8XZOKms9~;7at?xq5ABpjDA^lLZ zf9p-BT7&ZZC|&5SAI2~BQ*8!4mi-?#?Y!ZfEblP=UNgFn^i7-$@ZmV*XN;+T5!?W_ z_-uD}-d|3W{}i*oqR9lGfT>THS+B+aTo-yv#6;hjJxmboz05IXiR4=&6GvPZdT21^ zZ~BcSwB&!a3;(`G1@KZT=r5<6^-4OftuB&A%k)XbE07X895{C1Bn z@Xw(goyp(BkQuq^PsS|!;(Q0g`OsUAzj2*X_b;rY|pO5a~=hvWJowYZsi*b6dY0veh zy_xE8oCiJovA9UK$I{OXyb!-~sl?BhiDOL{_J6PoKeO2MODjzVQ%pXQutQ-#Ur9v0 ziN71Ez>ks$OP(!ao-&DSN$)P~Dn|4R);pE$`JC=zylge?B-6Bq6tj+xq2$i=Jhuxz z$GfoG;a%V#5&i-C&z2Q7Vchal-39**mYqB*=`8&)y`q!#bww9^99db978cFSD)nTS zc(Sq_S@|>bJ&vqtNRX8?JDacB1^G|pIkM8rv&QBd1O9w;;28 zT$ba}tem34;xbQO*0i#Mf~?|_qN!QAMP*Y9@*L^e<=I(9C68uJEh);*&7WE7$@h>D z>C8v7$~^f6rCG(<`6bf}3$xP4s9MpZ1$m{VnZPI+8Cf%C=8r3zIzloZo0pxdKQym19~#QMTv*E`}*XTm5YQm~oyv%JXtOMJ4Ju z&tNt~&5V!FnlW{bC(m6{l07FgJ8fLHI&N@QTG|~Yd8JTfS|K179g5bb%`DFv6hA7@ zbMMp{U?Ym8733A>&Ge*Y#wTRWC~Cv!qjhLXZk8v%Fi&(;zUY|}JG1z~A!Z}<^9pj) zG7~K(nZvSXOe0QBMTudaB6L9s`aqqLHm-C$MH=TRE>LsEdE(=9re~LAc}lYLJ*8?6 zTac5MHg2jqK5Kkj*7*3Ww9L`t60*jQ9(Tu&D<(>avXpuBxY8jx6e6vh;JDJ%c=;C6 z%n&ptvn0PT--Esu${A6D_nhp4w9KJdGqQz-r?aqCM6_Q}YI!ps^-OQ06YU#TnEhCu zFt<+n$?QPU$TW1)xFKRdc=8Iere&9UvI_GGOY<-~@@LvnImL4v$SEu;#~5j65Lw@E z6SR#xU>ien3i7h09yp@@_l)8PXJuf_Vhm!W<&}8S#tg#{pIL^nJUf19R@Tg-lEQ3c zVi0H3YBN#&xI{=$X!YigG3g6BY}nX5G^8}|dpoT>-%*g8H@kS6qksb!2_Xsm?^|!l z(>5v-J$a>`97se>$Dy5)WWvl@`7?9j&q~V*(QR3DXK;Yivhxd2`^;H|_&d%sC@w3@ zGrgo}R#t&9rmVb@lA@B#(K%AW_(53{#n{i9iVD8Jv*U-MZywF_WQ{M*$}T7<%3%+f zw(t4T%o4yXH_zmgmzD{GrPLTkBjVH0nbJL#-jQ8goHrAGfKr>Li^dM(xI?Q#C4|kP z)L`MG%mFzRE-SC396n3Z*_d^}gevsJCqf(fVqC+ypq9Ms!mgbPWQD&;$Wn_+v+jkU zg+;mMFd8P=p3#XEH453JsJTu1w z)>*|xj2ucz1)KO`8e+Ne2<6D!jO5w|e?nn)u{y34)*3%p`kJC?Rtq~(1T@a5870Kf zG9wwdb2uZIaU0*ntg%xO*yUlmDzze3kp0kl=&a+i#$l4r@x+&U#31faHwR84Tm}M+ zQcg?s?q;2cK5|hpDE{x^_^5)Sso4e8-T$3eTMr;^DJm&-&&-`roQvKZkylVq`u~&v z|4&|-SY%~S%`brUa4DAc=+Gq*^EeraU_o~QfGS1_$MUNe;qz; zW7}%Me&uJ8x7w9Wfm&~a^99j8gFYuh4(cq<}=AN*mFC4k$giUb5T5k@k@q#QkJHUNz2mG#*7-TNUl~>b7U9(=yh4nI0TSfm5I(7 zJ=&F_H`NXTtmDa5*zZh!C=It`A_lP!3xV$~W7E(N+3be(|hQ zjI+>0Cy1!C&I4ztc_{aPvP)2h|yHVYb*_1{B7xDrNu3Eb77Wpr5Jg@pVu3%O9C3&N~Yv4;Bot{u!}X=uc?m&r1( zA0d%dq@z4Ja!J>*)mEQ{BJ~bhvDk&=$}cP~Fnc)u4mi2&^4VFXWyQrsC7!%oI=lEp zV6g@rj76v&KWNZ}n}{e}s!mHApVc;NWMaJrE?rRPs<}Uunp>DZoHwYu;2kfFzuo3% z=WIJ8>`dCM((*!Vq87~>%yz~Xm*iQBDaS14w7f6POUNq8 zE9Q{dK7Hb#JIX}#V{9H~3&iGnyDRDJ)Tm^}_eoc3lWwFb-3%6FC1ZvnAI-tG z5utcyVzE?^Lud#g@g@r8lI3mavpj_8*|58*rEn)azyNAyu4B}Q5m^bh#W_Z4>OCW} z2HiFY3`R_vb zuwik~2AurxFo!%XJ{A|I)??*K2`pOP+LbgGrj3s!vCNm{tak~?GHr57EM>8%Wl9PQ zOAOmV8Y`9bQbLLAE;v|PkY(9c(s;$$pu-UjIP-=%IQi@D_{2OXW&U4m`DOj`aS|Ht z_$-v)%`w-^C%+wUJcg6K2!~_Y z-<5cy1HQpLk8J9Ik2cTCquc{TL91R1jvgWLIOapE+CBqqc=HN@$c(e$&7*UHPqg82 z^c(u5*zorI>52{i(-1ZEZ^QHMkM)^k!}qt4NT=BFi8lOn8{Tch7u)dn+3>S%_z5=r zd>j6L8~#}v{vjK_+J=AFhF@yKPqE>b+wjFU{A)IRnGL_vhJVV2Uv0zBx8awX_dUXZ zGq1Xc+}$QUj?rPAZJ!1k9)=wH7&bg@#riba@Z@EEnr(Otn6^)odiMmzt6}({1?cEMla^HvG?Q_}Mmmj152EhQHp1f7XV- z(T1a;S+551{>a-ZbhzP!{1?}Z?fTs+3?LaJipTr`lPkle^LmEdt^3zvJLOD;q6~R zh_c~%_r&@{+wdtC66ru2euNDlXTzu3@QF73NE<%Ih970aD>l4p!)My?={Ec%8-BD6 zKgEXEZ20Lm{1_X)*oME?hM#T2XWH=dZFsx?f7XT{XQQvS;m6zXOKtdxHvDoMev%FU znhpPe4ZqTcw|{46wGBVnMqgvYKVrjgvf;CA_<9>Y+lJq5!%wy08*KO-8{V+t&5%Xp zHreobHu`28ewq#M7};k3kJ|8&HvDuO-etq*+wf5~{0tjD+J=A3h979dn^)CDZk!Eo zUO5!_L>qpljZcaVZ(cnUxrz<{xQ#y3hA*+w`nC=sv&^yuLU7I{T;Kjj-@&frcPH*I!CF z46Fk60`5kbA$Xuhz-J-=hZ9~Y;FE+S2rn1#5yA}31JwdPK$szUV7`F&5Tlt{3prgc*_tY6ScQ;VTKR6mSV)E-3@c1w4Z=L-jzl zfTt4fOL)G3A0*7sJWwp)v4k0t2c`%(o$ytJGX;DP;eLcu1bipqpAwD}@F2omiUy(u zd^6!|2)hJ)9pP&UI|SU1aDT#0t!)1lgrf;J2>4RMoCE{)0`5kbA$_1mz-Ph%|BUcT z0iPrsLwLD>j}T_)9;g=Z0m3&Bo-g1%gl{BVEa2^g2NIql;4Or2BAhAUb%bvwoFd?N z2*(nR6YxI?-$FQAz<(pm5I*1%@E-}^M%W?X7YW}^xapkef5Hsa0}TRxj&MBTdI3L8 zn4x)~M!-)HP9VHez$JtkiU*bpcn0AigsTNSmGB*e=L`5j!b1rc3wSKyM8Z=9oKBdb zdLUE4_Yl65aEgHMB+Sq}5GUY4gohK37Vyo4lL@;7d>!Gt2s;Gak1#{>K+{>#|AbQr zHwgGr!fwL#0`5lm9>O&OKGO~G2*N7`e3Eb~;pGB8LO6|ZwSW&09!Yq4RM*@Wu_+>P*5!ZiXu69zbk@Ja!nB%Dilxqy!l z&Ldnc-~)uG5uPvLJ%k@6TrA-2ggF5PrU-Zo;e5iG0$xXW2H_L|zeD&j!f^urC*cCZ z(E|P(;X=YL0soQkOu`NUzeu=_kE*8->oR4-E>f?JsYl6 zG=?d|c9%vw`(u2@t^359AM-l3tGboygf7Y+xL*dA*c)y;acJe?H5*$Ga!EC%W$+H$ho&bSb7C zMQ_#gMKQ&Ab?dt{y)uTAk*03|az68Yi({T;>P>TOs1&b`V09A$2nDpW%^XEPu6S1o zE>;1HXgy1s`Smu2GGEm<*z$LGlJB?WH*}H@orU-t7vy_oE7Gchl#k-3>ceAnEha+I zSH)xkevfKV^kp#$fb!u9o88X$hTjpa)#t8?bSHhB=6o-82ntoM+KnIyP>;c>`l3~@ zA!12&zSlc0RbRYnHo^yeYYAG?GvaJCR5m<@%Axuaz6qU zMY1Y}ZPfGw37ap-p>{O=NGCbetEQjqB!~T~=zBWJnU0)<&5C!i(2A^Vn)7&#vY>7$ zvL1fOo#lSS{jhs7ObCYBnf*+8hvq0G(c59r|DlSzBMuYshEWA37ctB z1M!}re+>Ik@FpyW*;JCtOe2wM`iht+q{g351%qi-$D9j`!l3hn?==0erk_vMPZ~Wq zy1YFnv_ymYutMv(Z=i7lW>Q7>C)6la$Mn>gNIX68xSW-7=$cghI|>v>&L4iq(K#gS zBh(IDB5)f)h-!qT3!2^VONy`0H6iL%V(yS6?jF{8GAPPoG&^vke7)EZM-4!0G!2fiK6N{<*OZ*i`+5JGeoq-5;TNmx&%qOFE_K z5mzbt_^TqFPcKG9>a7$VN$t>h+|D=r3E#aVtp)wnjL=1aZ`f_7hM6`XHNO5zn%Bk)2iOAoIb9EpHXUqJ=EIf zyA{1fjoqXqEfPG+uU7IUH_D}kZ=fZSU*HrvMDeZ=R7xxx&Y{hQsNn`*S-@Gj6=WKG zgOEN{s}}o#dODznZ&PZ|pw^bV)!1!H(h=t~F&OP_!eP{aO8<{qd%lO3bjrE#ZS<3t ze~PRXeV?L#s>XiZMy|wugl3|M0>&5{;H`kS0^a%CS;p8U`Y~WH@&P4Bk<)vW@EuC* z5v8`(EEQ1lTbDLKh50*0h2dwlFnA4WmiC~9f1@US?yNjY+d;Ly!Ll2Sl5bFE!FrYf zLD|8;7EnM?RX^^zwH>8mqYQprb}-agmWR*(NYo59i!`J^1;NI@A&e9?e}iPRSk4<^vI~`@#eyw_f{|DiNk4YHtvP=>$>#V)eo|xEL3jzp zH#bJnbdUU#$+t^dn?xF4OV!R!09m^h;!odhI}GaMKQ3CAkdi1FrdE^LLPz!27taC)H2G_FO5 zs`^Gu^SR$fGY?)+@w$vV&?pXkZ+Fdx&vY5rh>XgZ)$Dz5_uK5bQASYMS!K*hnd`FW zMjQLh+}C98$#2?f%#1YFOHt@z_hPW^1`(3aWYU1dS$P>&>C*Q<3)skpPT*?0eF#zD zOsav~h%QDVohIT6lFgty@bwA*SduN3WMnMKR*90vo8X4xtXJaK#E{f@_c)5v=lb9- zp-U1tJW(bo0ah+ZmxBc;$XZf}H zbu!sR8t`6;6q@)qc6w}@$Y5{zh78)!P7^HCP7@P3xP5(&LvTxGw3s!t)5IUZ4fvW6 zURs&)mt&?Tl2AKjA%Sd#cmrT@H5P#dD9Bn;h&Pi$Xkvz>pp242yqOfwA~h6H2u)N1 zf+m*BQxjL9cE~~k*$VLnAf$<>!2%S^Llol8q!5~zA}J`N(m519V9N$R6U8v|a1Rz{b$}!4XSZwjtBq#6&t8Uh&yATaBdXb`AjHY2MoHw&m3ecmJkeGU^VY=7CsKG5@dki$ zh_;Reg$TGlH+G@fXl=0sL*ZEyl@I(B({Wp2+invdO?neam%ke9jd#t=s1BJNPsZ=f zOjn0YnwYV`%#2jfs438S!=(@x@6mXF%Y2W*`*Y^I3-4LxdnDcm<9)&0nBs4tN~iB) zKvHnyG@6=(_BpF|Tq2MmY4?S4UTn(|qOK3+WVhuAIsY2U8Hk*O8U*^((m?0JQK(q) z;+4&-Z9tH^SOlrep00QYyNp7V&B*^;Mg^;6rdMMoJU>Dp@A#PFs_mM0e0=0Rdhh-k zf?1+R=&9imjQ0qzhAE@<%F8ktubC7X`r>c}u=^dT5_2(4I}N6szm;{RVqwCyqvAzS z&rrcdCQ%Bii$?Xu&i>Z=NJCJ>O5DMkjPSrn=X+OQF-o7jB+|m0NY^NR*%B@&JK*~g z&$VR-{7+JoYMI5AW}3dPWN9kukrF{&i0Okh?j(d6RO8eCMidVXa&fmxcpu6cX$-)a z2%*_WM7xYj?P!`H(NV@XM+7wj4)Q@ejs3)+*~)tyYGP^U8(-)c?Tx4(t*6$bi5erX z-Z7(n!?zg+57YR+pt7bIBWaq<4q`QbbxhI?r2?O@7bCVA%18el!Cx)^tTdGF*EU9D zC2kps8XgR~oCE8>ak_f;#!TeAeaoy20NpPG@IHOSH_l$;V+I<(fVBaSS0K>e&VlQU z4+IWSG;)A5-jPHa5tBE4<9u*)<3k9&&K|j-%9CUgc@h=9;73L0AhIzTjQ~C(254j& z8n`(JZURJ9FAW_u3@E^OJ&6j4+~)|cPg*9L>U_E{HqE3)6z&ZecT!u>b*3%mjfLvD zny5X3$hJi+J%U@boHO_O20x|NeiaUnJ$ov9SxNV%4vI{VjTofV8WBoxlahl~Q=?LQ zFaqkIeM`DGVvuP=gTqlEJkr==puURrO-k^i*%u2Qjh~vVA_k@FV5e4WFx243bAB(v zVM9lgIqbO7XAOvB$sSXMz*N=Dy@}b}Am~t4R;a2)f_Ba>dI&M5vHie>1wa}AcCu2P zZ=5;lcHX@6q~g5!6|o*nbG~s5!zLPxPdab*Gj|_Qc>n06>bw~{SUp>fXsnvHs`_SC z|3KBTaQoPjMbWV)tBm13g`#g#`TrJG|4=3M$5=s}bWU=|;m?>vFrIW@3jaQ&@NZrw z&_$EH%9NyS=)MtgLJ!VSTaejHA$hN{B=0pjhV;FrW%6&)Bz^RhU|O!Bv(=b&*iK`` zmlB|MjQ2Ul7Sv6`yKTD$>-iYc3zB6d@eYqMevQ#<>xM{&a{3D; zyjcxzR+5f8pZNpnlAz5aigVO9u7i`dm0YVN)z7+0soGxl6D5D07^%=N7kX(4!Nz{v zCd)q~(Q+%ocmt-5<&b*%^R}fj*Sn{`R>S`z*X-x9X8(_hh*NS@!#BciJ&BsWMl2;Q zN3QCA4Qsg2;&=jtSLJF_4Q?$v7;>-rr)^cyoeE=%-6~h}njW;T=HdFRbuNM#O3shb z@)f&ak+4r2vsi11AAU>e1RPA)Xr`ff^BJC8wZgjo28Ye=}B8!%7#pV zl7N=$CxNaeT9xPqyd#Ks1BZGnA!4N4kQI1|7|BsCN!p5DE{oQ)sI8*Y2Sr6<$>FSg z*v|1YWCcdIaja=8Duh`hi`KQFh3KzJ^d_RQP_=V>jvQOW2BK(nWm{3fafU3qybUcl zj+1B+0%MuO@>Z)8$?-21M-hCsHTkD3CxXs4#a=9V&1V(P%0fG@qv*%LLjsfh3Stus z%>30Dc|+yM>&Y&QWS9ApC&=t{X5S&Rzg~-M-XlQvMKU{<*+XRZ$IRX$v;XyP!Mq2v z2g~f0%>KK~UIuS~x{v(}*$FcHm&{%yvxmvjTbVsbW|uI#P-gD|SCoE(+3_-a9J9yB z>``xt>}qDm$?U<*zFlTdk)}NEwdkD_C#h2H_d&ZX7pcR77hc&C=uJn!ZuRi*NsLr683`aVu4E}ZkNO{e!r5% zV0dDS_Y`M`}l56}* zIF^kdjbi6fzjSlHA$cfiydvnkKPl+f$&3PIAn;M*yy>}|#cNMGA7T8{BSz}i+h>eq z52AbGj1!+xgREd2IgXVQ#I}(n8e2moaU{7NBqGv6G2@@1yxz>aQWihjzy@I2cEP#p z8&5hP*zSE?NVAo-CPE~LHF=#;8Y(`FJ^Tsv=TZp8Z zB)Fo5gSI1?}ngM zzpq%oKUdY=DH*;o#rni4ioQkBN5!dnN}{UYqp11_>}0zqY5H@l1?brs`g5FzRlPz? zWw+C!yvfBB@K2&clfT`^@rt!ppFMjay=?7ogsNdi-sGdmS1LA6H^S*`8Azt%rER?{ z&y!Vsr-*~kWpItg-{j1w`p#MTWf zLKJLqlkyWuxfY443nsP#V`PC$dx1A(fkmP~N+hebm1u<)P{ptna#6Vdhs$B_B7WltsQB$F5lKKpMDEcD0J@plEj@7gm{hop5F)QZiCssiv zz9G-7gxlQ@0pOJz!Ns`YQ`%Yb&t&#=X77{PA2R!5nf;-3+A6d6$n4jdZR{4MAD7uT zGkcfJexBKzWp+QAeHpVEd7+E3|tf1x9+c(LYuwi%`r+|ObX=Jf;{Ts^5f z-~(p9=AF+YcQ-Fe*zZGB9F07N=QfBXHysU*E7;_@G+p;NT!f>PU@b0> zL@P;KN_ODJL55)E8x>TNjPgE8?coS5e4|>??D9madU2DgPikg!GW7kFKO^Zw)fX|) zS&2@7eeX~BPL181?wi})rRpOiS4&>^IkCD4hn&L|-}pojz`omH&yxwzVy0>>E4yU_2(!E+F-n8x*+M*d06wUBlrFhX@ zUTmJIdM|iKY&X=pkWtpup?8$kLHThivFo8F=QBQfprm~`&C{Ht&T2SGO*&h0O>- z)I95Q?2VLNsY0J=3C9|v)%PtXL}3AF^n zjIX!oXw{dFjt-RfQELyNzCB9CS#-3bk7!c#am||Es3x6rR$?flbHBnw6MNe1(=q7N z7W8QVeX7Mfu_;#D62a|`ZGnuc&E@xH=ts2h|9Ie@BJN7pBW{B=-W3UZyA5iESiPa> zyI_B*w_yi#@?BBPfhDd}DmFx`SktpRCy2Jz9_X(6CilTeFNUNiN^ewZzkWaMy-{yLMY7`yBMJCt;lp;1jFpwYW-wHe|%s;R*uTX805*4(7vEG~iy}e$%9Lc(|*wTd%i0x7#7 z0%!QF^;~HvULw7pP%g~tC2+;ls>R(f$>KY3!KEU>h-cvy2b$=er#U^{ra5=jRP1-| zFq~^@$}Us%dTi+!3j`GKaJrcfJ&|ye0^KMQS z5Kvq_vdMT@=ngV&MGMS}NM^!oZ1lDC+H1ri5I5H%-;08=7(pSCL>njT*-*F%K>))K zpKTZ!{~>b7#GTPF)VbKrT|ph`8}+Y^1EMhQjDfo-Vq^+2I&ip*99X6CGT#=|QJRNM z;mxXVWI;jmq^J)qi4gTf#55jm-yZb0qMx+i_d7aJ@j9{NsCe&>gtPaI(e%MFn5Z;% zu-JN>jN=GwqT<#;l%nrcyh-v-!ZtRzVto`m9PT7wR)H@*tznYU^cIc!R=lFby`07d zvdYfv&Aj0dr~o%%Z}P@Npwxsdd(1K63dqYgU~8NQ2Z@Te7-FCIpgoD2t^kXkMDkJG=$zgV7l;%Yt4b8ye z113+@#<9lc1a^GP)}vh=^lP=}P2{6Jw)>D3M-@(PP-~pRonOwzd&Nx=XJvQJ6yAGX zwibA2MJjS5P&A@N?5Zd2aX!PrsCYBbhz}L-1cw%TK&?F&kx})fvnm2v2{mf%mzYex z)505sny?qj#t-ydPcWJj>g9?T7gf;lB^XT5U&T6jiW(Ov%Ni7K5s(4R8xy0(ekhBy z-EQHYbIASgKtfCvVnn*3F)md4jBE?fG{FvQ7Fy$K0R#((1wup~b2os0Nus^6i?1fyNBKinw&Vb0FzR_a8 z!Nj5IC)jN`jM3STJF(Aep2-B}-J(|b4W5=@ZyHY0xED5$V->XesY9YBQ3?5jw;KML zmUPh57poJ+=Yi`Qz!8Fg6j+XTcr%O!XJtB6YWP1CqoJMXoC1W;n~k9mScs6A#sm|$ zA{-@Y_pI^kW}M?033TL$SfEBvOiG~pd4S}AdeAfHjc}0<&-NTVx^U;>oYaT~-dM=$ zL2cZM%tqa~gEZ{nz#j6U{^+0*g*BqrsXw+gBk2$VfrrpOKZNtdL^>;z*$Fzv9zf8a z;)Ixwi5E^I_Uf3BvEHHS<>=FL7{u>rPL{EzA>4+wrXg1A5u-8!gE1)j!>*7)APB?3 z?i^Gw8}{1HAW#bWMXUoCbjVpX2a1xiLl(Fp7yz6CvPSzNoXd%P$$jZ zDmw{OWk&)N+4?qplMR)Azoqo|XaLYRlwJ^%7)WM?rqa>&^TzWcA4=~dUW88vwedDU zqt^$L+E~s0?S-n0_vjQ^OJE{d*=iPEpkdbia^xH~QUya`Wq1GcTwBeeyL($T|Bfut zfx<2#jp3~MH(MmNaTja;O)F5TqGs?53~bkCakDEQ@mYxH9`PPAc>qA zD(|3w#RVEhOb{0*{pJLar(zVUvA4rRMZg7V`gNK<6%|c|Y%g=NK%@@Y!~|f-31B8B zfZOE+5bzA3h)%`(B)DP*NJCN10DjZOVo$C846u#jR6QYVoa2(>QdauyMBKVmvC4(; z(=jq>=t=vWl`b)7(bL&fo1=I~i@?ZxWfz(~h?dIb+y7p(h^wew#8wqjQg*bE&Pyx> zit>!WUVzPbD*Om~j&Xt(vvcd+T5M}5T)S=~1DrZWY#B+1Wq^bB!-?<3);~OO7zj+I z;DW|zUypK4ABZ*6N6>F3lrjm5Z9h5L#y5n+YS#J8r=g|?jVbiX7*`M*Bam8CyjKQ} zaD19hhSqA_B*@@oR??w>9%LA)#(lsWD>g`KW1PsO{}(Tl(I*gmxEQy_MZodA71z*z zic;9(PS8Pts#!pEOQdlG zppnKp9n7%%Dd#h%;RtcvrX2e{;YKy`u@fO?Qo13|)`6n4DCA4dU5@q8`|G)Us)PyP zoU#9WCS9&Ce_bczee8-ce7{}Oa~YT8+a0%H^>Iq|4i^jC?#2uda$4k!2B!il&3j7> zOfd?fw}TFv@q1u||s&}GGiQOgk&A8)~ zw9fg=0MKQ4xoF;`#csqm0wUC;8s{@!L<77LfV~)a8|U+f>CBSOmSatNMv2{_C7qcE z@xO2`{8+Sg57??nyPS(^2>ME~7G$uxmGoP<2myT`>{d7z{(%IsC)A|V&PBf#WjAmv zV=o7*n(z&7xt+4D8G9?jJ=T}YQ6|_YbQnQ;UT)kmWVess!7#T6PpmLL^Z4W_{3&t zC=e8mFC^0Tb|PWRi5|NXk*?xajXro7M7oM14Wmd`QKVrM=_-me3?kJ+q@VK?XkPniv0-r{n!!O)+~42uvHY1upQEr zkJ7NDBTaZf+19FgBiK8d9)aE&&)yl&-kHqanatkFW$)x#z4H-OVQsvP4QfeS%mo!U z-%uVsH2zNXkk$`9q}|yZ_(i93ZEr8v=1%2e7eOv`5x$Yb?x0-kqRH%{Ty{~e)kQmn zTxJ&uxyoTRoy#N!nfF>44~LOPbKk`+9R-_*oJ#*py~j2tY?ph|z6{$&q30&C`FkD< zfm6!1V+aPyl?;6gw=JK)}`tc0B%u?JFT?t$QoncS>7u43+qGOkz$`S9k#tcTfmM~zrj z7>(~y>~9f*mVH9+3w2{w65Ank!Pv^fMEsp+w88& z;|am|xbZ8&_!JrM>V&af{)&y$e^?vFh>AjOaGGtn@xQmhJRq763k{vuOhCip#3p+8 z1+~$b1I~#JHU_ehq*J_+iZgBNJD2hu%j1E_7|0Cc%oQ*QvJK`vVIv$czpM(<=Cyp?$k$`y6&G)h8wc>pTdE>J@(tO6W#@zpy(vR) zNvLs8P;oPnd{tZo6a3c0J}^nTDw;3yTq(i_KyeR;zI167_#r#57lRk+#f4^lgBxoo z4At-7vHcirawF7BmN&j1pu^5NkCE8T=*@=ZV2%esolK$*f>0fSDFD!@is3O1=kxDE zi(+}h4N2_aZ4~P`MQ7lS#RR`i`xB!(jMvRQV1y(R^9@YFO?4rbm6-j&=%!Mtk(sT#WxK~*>-rAl@-TX7aX4ttBZV6f46itl;_?Nl*~7<+-!M?{3+(SXI5 zwk1pBUBF^PA75&ISB#uX&>V{#YothHDcPD;T+MgALRhL|HpgPV=%l-VaUjK?6A1>B>jz8Pt#XfJNyyMr6Om!&41 zD0#Q>t_(!XS5dpSA`g3<*c3WncHHU)Ec88?w}Qs);3bFtf;#rEoXFNgDjdb7vrUx2 z?MuD98><{h!F+?)8M{mwh%okmgOz)aUil^mgz*+chD=hRwSMQED4Y$$CfeG&T`cSELDqTqQx@aWJ!cE*knn7EICXKXFHZDmB*ZwV(-_- zL39$%X0;1_A`brMC)d6o&E_M^nO2Ws^v4&#p(mZWATPyRCG7fjIx(0&H79dEr9Fy^ zJ-;%})i!@e+h-Q+g4~X#9%p^y7kznHW{5`T_!vN!Tc9%aXwj|IvkzY@x z&2cjc*UDL;7JEvq{W_fA4aGL%apN}DRpG}Hqp=K2NJ+#^Yy=_4jktgD8`(`bGg462 z?0mHB=&vl%pGS34jDAv#RexvqQj(upl6-{{?}SQBzOV#)k0nl^Iw3ntJR2%;^MxhY zIV|zMy~Kk{EUB7-ma09E_HF2X<9k-m7#TnW4xXpNBg7?iYQ^{qm5T+VaXGt7FLZo&9`6-vBi@fL>l0fcE>Vxffjt|sbf3pJ1u zyb4sH0PVN19OT9?h!s1bzT^>7hB(T=YszrW)p$4l{uT=Pl4C^chxB4LzKh4~$CU?TN)k?<(c0iAg9Sqmoi^Q#3%*;G|;Re7IljhGRPb8m{N2%tgMm?)N;>1$@X5~gYGI2H56JD+(Hs!0g)cODKHe}e)o_lpA@@#7B(+pRLj-_d89 z_Z86$O+Uy%#g=GEhdjNwOZG0Su2|3gL3mu$#9UlU^Ym2t8ynu)h8o`5*exMwY=gw` z7*#4Jf|R$y@Fj6?_hd*cc6;#MGDwK8T7Mv$5`zqS4YnD_@4U}16dK!->UG@X>y0sg zrZf%Opkbj4Bko4LP8 zZ^(VFcel#z{HYgNW3cAg-j@Z}%6yIWrPXoI)`=!+~+qeszVbL0HcAUj%l`yDO zS~$&|XYJWiq^fCXl;XYh%v;gWB<}jr6&(5>){DEltEAZ;g4x0&%n)A6H0A#Re?~5+ zQ8O)o9jhh15<>%aR`D-YCd~Wv9yRLl(ivHymfm z=G6pD{Czlr=CcN?t(W|h7L9M-5{Ylj>AN#{d1o)oQsI|kDFr@cQcn^v!6pDMs^Oc@ zfQ`-69m*46tieDD+j?M9_5lkKPyW=G%)_@c@Qz((?EbSB5V01fyP3)wz9PH{u<@u_ z0|B+7PSk+XWm_!@i@h~4k!dk?R@S4xv8px(pq(tulE(x86xdf{mVzg}0GfnzOtbsH zTPX)QMP?Ce9K$5;uZDoIvM7`{E`NhfZ)as^Q40}xvM+wK2So|`K z5nfd@Z!q>Xl<-qZc&jm2cwF)lL?*tmHM4 zE~w+|Xg|6)J6&{b)e4vbf3MLp1)-@T6bu}MA+dJ??;<;7;ILR&eQ?ZVX(d=tdoKjW3`D!Eqrt z1|Ad>1iq2SCi#Z1H>RL-ERK!uTVp?P@j0_a5WikD|GaU^5@{}9_%+~F(pjwm4`Y_7 zXPLkRvaJD#;HeI?Z8sG0AJKvB+c*ouIo-2@elPCTI z3H)ckk6b}Nj;e7u|5zKix?+ErzUiS3_oJ-*1^XQim!me&TatwCV^1_*6uR>aMP z@@w0b-$lqQk80QGqT0W*Sb;2-EQ>9a#oiD}PlbtD?1KH-H~-q<{?|l&i9yl75m&#i z<^?lDHy~)a*&P@F?K8(Exqnh8O~OjH!Yd8#ID1&WdV?SGkCg7R_mUkk^C4Jc^qE>NBL{yEE*xtqEEt9|M z>y2+$!79&Y=&c#}!83$eyfMM?{rqLDtHR%mz!_I;uwXoI=rq&dN63EaP~dY2D1HY3 z#t)e>LY}DELv?4yhK(6_Qd282bT!J`U3d zem(%aot1ZrhC^~P7mXJ?t0-rDDS8cWxE?4PkCJ8h`2*gOA!Dg@=o`Q1=j;6i+vmw- zAV|@SxBo`V*}&YgBaMl$jTFo6Af&_fDp6cqD*Rka`i7SZ&60CNCBMO?LLWQ&0W`63 zje^KS!YbgvT|9UviZ;fX`WxXRU}jH<7xuT;gND2dYNPxWbS0>bYmnKvTAUmNG;3P7 zSgf>I3~kS1Hd!QCEcRej1WJVCf*t}qXlp0P42#PN5K=r-kU`{f>`$g3i^=5{aFMcP zlMAk0<2U;F6&S3F#iu{5KM9t9oD+Q#qPl%pSmHYkNANR&B{$jVaMuJ&cIYFvRs`_4 zSqW!)tgJu3MKW0AtZidzqwx(t#zA4M=U_K4b13L$x?;>BemOEcH}l8^yD_{`#!+G# zO0@AVTZC3dixGm$Ul?43Fd*aLso2pP?^;|XekJfYEe=Ox+-DV>qua(D{EJa&a>Jbf zYs@wC*8@M1CMuY6nuCD2pYs3=>>GrNI2wZ$I4d6#_KaTXM9wQMIok~S8yYmkGzA7X zZ#?E6f-YL+V0aS6H{xO_6xZ*JWW_f*3cy+1W}Y z1D-50LG^3#G8t`cyY}=xsJNSifS?Yg_{~Pn{JhmP{*T~NWO3OOitSRT87yZ_kagZ_PwY%-pMV?Z6Uz=`v+ z01C1|-OEq*syyTZSIqkn>Vk79>?m0&eu~F`*46!T$mMUIWo$sB{bzesxgdt|Zzv?q zds~`*)a{)ZL39+*%Eqc_Bq)yqnzS&z>b2JML0?Ijz9a=`_ku6s5a&O6nY-d>n7iV` zbM9bWntnX3>Pxr2LmX(NB9=-!d!~E53+dDSDLy?ZWq{@C(y5 z{gsM_1|_yZ;r-02YRc>NF~LY+1Bd(``*~654#l^Y8dZF6Q7eWIom*F1dh?hFcg04u z)m2rEX8F%PvKhai?0mijtyOb2`dja>@wfJ>Dkfii#n8Fvm-y*N|Jf^Q-4%zM)SO)j zoBge>CCsin_+(T~SX=ytVv9doi46II>hMze(aNnRrrye`x3cQ3tR^d~$;xW7vRsdw z#a%E|!O-6?U$O71DmQrFrV&Q)yj&~eT!OGf@ivKaVtf` z^_w$zTF})!&MU!<2*^qr8++JWjbm87Y9&;Iq_`)l(yQXG5lPc6B3U_HB*hwjy|3 z4L|EtlN!6-U2))I_vt-p;XCp54Cg|e97E-Z8YeGm69W#0KZcgwsZ{8Pg}r<9zn897_j zoaT(2efZ8$v|4)xPcxp@X#bgcvJrIyo|yNFY~K^}Wc!F(BU_~QU3`j7t9$vXzKeH? zw*m2emx*TAy&Tqe(J_Hre{0nGu)2eVQJ;qScTR4?a~3wWYA2O}mo+=9`SQd(smLy> zTp_C*W>q=hs=li_Rym+o-!-8scZAig?-})R7;0F&llsQXqMg-zIb*BJ?7cJdmWkeD z?PhoZsUgu=l8cb0PYTJ;=_3 zj@F3IsH<(6*Ibv~GVcU?PIjY~^QqN^xJw`Hhx$$NZCzLyYNYJKI;e4(sYbL?v<)R? zms7yc{rd~|^5qNGS?g~ptWk14)^fhi$U%@rt7(bGZ%RkktsfF~5*oig>RgzA!sHEj zK889fcha!&vS?>LU!Z2$0C23Ot`|!szKoWM%C#Ish>-rKzAH@W-F;2F08OP3)xW>* zD$x#qOJ7qTV7JwOqHr^XtoH{Bw^QVg32oDIJ{N-KY|?U$)4+weIentG!X}fWzF@B{ z+R1ST16j7ybt`%eRjm^CHm%xk8M&Hsf{pcungAkc;6Pd9AiM7>slyX}O;x{~-`8yN z!CRx6QDp`OgRpuIalBOStmjy-Scipbz4Ph8=WvMal_T3LMmD|eJ@B?Qym0>XQMjp$ z9Gu@_<6$iaz8bs5^aPmtm(!=ii7lDqaE$g1egQ6cC+kV~Rf;B#LCPK(<69;Sb&PM3 z7#m|i$KhC=>Rt05od_nLr&7JE-lOTdE53SajCX}ld4_&V3!bz1L9h3uXFT*&-OG^Bfex%}nGd1=ITx0i#XVW$AtgD@~cl^Rg8o__++;jG7 z3!`8ZdoZx--0_9mIGDE3ICi7!7MUZ8B?fX#A)6%>iQ^O4|*&j#omXYMHPa z{|TGxctsAcmme@)hkrod6`?V(OUi>kvj#XCAtkP(ctR?QC)stZwb-h) zTJ!?HK8eZcmk50E&n_TjlNMSs6#sldMr`Y7U+;gw>W)!Xujjc)*bw$?1!qMcz6rO5 z)ufM04mCz{X8W!TXTwVvvn6cyRxwgb^%b~+3ETOP7Cb*0KWpl(Vi>69tYy50QS*8x z(p3Fb()v=mV=S<=>YIh>^{v;Cr|O@W!veSB%#nd__qxT8ZFUx_9aw^aKj2B%;_E50 z|4COBt3di{F;YajuFC5?n1ucd6Rs6TBkouZrx};jBFS*2A-|N*(nZAyy+0`!WORV{gHov??a$oOX|9VWl=lf#vYD3MGi~I4{2GwRiol7 zYx19SWdPBuDvr5mW<653SgzdT>#-KgSnPkTrL$6eOIEgm1~K%uU$I9|iVbylMJob4 z?%iwpt4l$LD{}rPihT8?eWIwU3906NwHi6-dfEhkqEc^Ju>Wo@6Cx`DVHKNeg0&Sb zVb03;ab^$ce8m*MHoo*kn7;)L1&g}cVWmxB{wJOij2T-mA!jtNY;;u|lp2jZQJHfV zEMY=5E~O!T-6b0BTDwv-&-H2?(_0^SsflTy2l7_pkItw}Z3lg&pDNTYc2fhC!eLs_-|*rZSEYHyvig9VqJ zGGqEBtiEpjlUE%L^Z&B6HPU~gbYEm$?Uk+#b+uuo=OX>#u5EQWEq(WgvDlhZTmhu( zi%zYC9O#_aSsjW;Z9%)AylQut|F@-kBmMhJ&qmg{b6k!7zm_)RE!OpI-NBZ=8^Y4{ zRi_x$0+7XXzs;iwaIe`2nO_Kg~S0U7fZLl~rq2qY3$~*&~15L*vZ5#4U2-q7H(Ep$z zPwI|Cp8UY~Ey3TcSBz~e$6>=hiSJd#;Gd0lfgZ!2HwkgqIJE(5<}|FM;U{dPf5-FG zkZgqPc70!WubI!Ha zRL`r`Afj_9Cce)DuZq5fe`q4!kI^mrQglbiMXVGOnJ!|@F`J8cD^9e_RYkS>Cro;k z?Sr@Q#gPxZMYTwKNqlRsx9SO7j>Dff>D6Mvpw|*1*ps~kO!sjt%&*gZw9tLHFLUjx zt9_hsscSd%_&V#84x$b#x)1Db0&ZpTG2x9~7QQ0Kg@xwN;4ox{82^5lj4%z;W&Aee zGG3<3FrC9gX33D+nMC5+rO7-2W{j)4C?NZ)C8x89#6H)=E@TYv7)#FdY0Ql;%ro z;!$@BhJohut;9rwn%%y+7X@pB7a4;v0Kmc3(V_znfsm}zeD`B>>kN5w?sC^2A#=`G zT{0ixoBI>Ne5bFCISX5JXy8j6)|*jPVA~lG>G+9isB<#TW_-QhSDo+StHQ&Z)4juY zsh8om9qv*$au_?1F&metWe)x^A^ZYTH>KimxO$n2-RVo+zR`CnI}WS*92Y6W?3CLA zAPfT`b~49Ag7R)j$=Boz=X)vP>Ar{>{4#dEs;^UPk6{0;;o^+T?!%3yv60G-2K*)! zzwu+8CvL*#&GS4_({lEx6=x%yi{1iIao`;OsjQss*qQR5IKJ86QeLC#{N1OBy7eu6 z)o6!E6s%YQ6$?n`x8DTf5}7+t=0-{=PC_mTB}yntLMZ}rORjE{Yo^SbEh#4nNS|-j zp-z#AXG4hT5>ag-+)}IsLM(Mb?Q&$O_^RB~cOV?V7yW$c2)Q?B==h%ZrzLk_H%^Pi z<=@`F!c(sX*JbDty)yLFUN{I&?SrjLF=YXPFTCsI+g+hnKN*bmO@+UF=1!&rslAyVRZJaaMG$))(MV-zZ z{T-$Q<;>s=D3F>JmX(#=$Q*^#iA>GP+d^n#+$_laNSe+d1%y!K|853;%2v1>wC@a+O6y;6YQwI)-T=_GpyyUsHj7D zi3wE9pC(oQ4<1jj1;dX?)7^8;FOCkL`{-y0WvI(m_O-q&GGSQwu7qI=cZKj_;RC3J zyR4|>_iY#e^tH@)k3Nt%Ec_KaF5}v;6ZZa;mYy+SkT4yvI8A&4-$KRp8z^L@R&^;7hdD_1W`ZnNF+X$#+BIQuC^vMX07M7P<-efkD$LeR5~ z`Sg8(T+wFx(Wi&$+^`E0hnX!zp)V3mw#_!|Q_P1U?u0fQ^sI2M60i3%FvE>zI0lIk z-ZBH1fS^7+oDd6g=RLHu6 zhGT+CDm|W&>ez0Dqswr_7y;}8kL~R;z+uC|d!hL%JwXIuLl=&2!!e!AiXud)Md5t8 zD2vZ08dZ=dbkeY`Tpcemv?!A9goPsH61uDFjgZ^u0OpBP1|~$sO~+^flt=UJl10Lu zM|Z>XM#yFrciUM5!_W7LkXAm6(uBK+&uv?bFfGdQ{5b<-QL*AJ44E9oXTo$o)06m& zPZ#)7J~vP`(ot8d9Ayp4QQf8-8x9y@jvHY#l|*&8a%_lJj=Gs}BtF~{4=``WKJ(gZ z&=W!fLj0vJh1MCUORJ$v9j;J|!X^ZYL$cY)}E_TYQei~`@VGGWYsUQ(O*Z*1Xo&AHnb_L( ziQW8S+W5X%k4}h;-!Nf5a?Q8%g5R5kZsUqaM@Mc*n~$!>He+`&4!(g74pjy5mY!a_ zI{_oZ*5yA*sQimK=9bx#i0>Fyj`*kbRtzUzv`;@j=h4t0`-Y?#9unmKv{|q$X2A@Z z6}i*k|35Ck|5EU82Qlsu>g957w|9LFIZ3MgSEg%6R_0~CKQH(Ydb+%y^v!uRD%!pw z3Rw{uofrHGd%*E?kQci*Gk@WBp!h2E8&TQ^1G4+n4#oHPK7iM9i0l0RI0cEEMW-+ZHJ&bgO|Et1Hf=pX#Z@Ycb%k4?2ueTq( z>u{TGX4<DKQoSNv)RURIp_zUKk@Wx_V zg^=BS>yf3(p$=D@qY`sxh_DS|Z0>k0KU=U-$L4+@2rJK+0{5;wpA{?5t->P0cvmeu zW0@Ii%2&F!hk!qd2fTzmC+~T8qV!cxwpiLa@~S6_>TKU2q$LlYd?o2aflid#Ca1~E!~%+ZhJdgu^jOc*2G(}CVtMciSIF>#F{ul*SI^8 zCE+N-m=g|s#qfy2^ct_KjaM;vCD*if6}RHtvOAFmb^F=7utMpXwCC7J8@4embbsU48`JpQ7&&Ex!Tq1S>8TlcJ#XM#PT*S z``C|Ac2~=k-QznjX8PiQEw}|6IL+&6S)imN);nSORW@%FYuHDzoL#e#6w42-u~$}Y zXU*VS%x-c?D7gG2twOwDG;vlEYu;a>T*)d_;y-X%+&40_)b`ZUw5rV{S6Q=(05MQ% z5L#D_n|b9;E;C*;uf^z^ifQVd_h3TU9ohsoslO(6(4X0v8jr< z-omBg+H854nI5=E-@ zM)F%kxik4jnLLw{$ulXDJUS|-TjAlLS#p3#ii6f4<-OJJ%6p|fHE&lN*y(Iv((&$w zHd{=`A#iX_B<74)g>4xjB-2Oco zRxFdlRTV3j$zN{2|NQ45oxiJn7y_b}cT3-w1r?HOpmvhLu7MSvl!@zw&WZ~66m*S@ zh1|%32J1&Lj>$X5TJTo5H`6sD(k|tR0VtCS@LAR#?5e z$Z3VFD7a+v6C$Dt#J~lXIwH*@Vt=VZH6+D;*lP$2*EIEd& z#v^!&v(&to5wzBWmV)2jleY`Pb*|mgv3o=NwvLxKw0%4^au-vt8X;6nd8;*H#!2(q zcfW?xsFaFC_KlTNkyJ`WQYjS4lsN=lQVp=C3`z8Ef~V`ZHZD^ws8T!qE{ zF=)4&MgIhSDkPAaPs9fyr9i>am+iTEuTYI2MAJqKj_G(qw2^}wk}%K#ak0?~>32}Z z6{Rl=wg{4H{|WT(37Wy!35C?t)=y_vKO_sDrdT6V#`YS?Xl&j|_7<;gAXAb%H@*|sBCf|K5)rGi^kFXJSIQD{&0Nc?T(AT1C>HV2iT~4`G%9HTPdQV$&O%5N zZNzVYZ|}bA4Y%FG^Tr#$!KRI*%yl$?v5jz8+sCd?n4oH=I9Q+>SRyTDA(3(|$*_(uWDjo__VD)q z7JGQiDof2o+B)s4umr|@ud+N_4Tw>XrD7tL0bNxS*&)zXGf^5^t1Pt>sbh$%pGd_= z*OrOW!n8Ee+;5+7&z0eKpb}tDCku-UYjRvfITveGTTrJ_V zy$Z$vaxjN|2kY3>5k@TQx9_-ah2FkL?mL*>ehv#V~8z`{xSPeySZZy zirPIMgEZt$gcz{W_VFE&P00vP*=#C8633QuNFZ$VuPj?>w71!&V=SP`qkV;C6T`Rd zTsdZ!rETZ!V|wwte@v&fZD-;b^g%mw#~g@hn{6AjD+U}sOOkCnvm*~-|BPw^&oW`W z`$eb0ELK#~#h`$@7FbBkGR$|Hr#3OD#`5Ji7&CF#0`rb_mpZPZVC*%%n8Io7S~v&0 zedG|xw`=PXa2asdf>4{>wSWM|t_9357`qm_)UJgGFzf__?rP?sHMSVL7BGt9;tkd& ziwD`YaO=8G1oz#wu*$Nj7btFAScTPcQT>+fq6p4e#Qu-=38|Yc%yD?C+{h4~y1j`0 zHnn#FJ3nIJZ?i>V`$Agk<_3CNL$Q5f?8?Z-CNZ$uYiwUY{j=EG_ohO&R;AuSnQH1K zx#QfsFadiP5UaLHjlwo1lbC1kg3i4Qp>3^~sl5w&`w;A1K!oNdj@Cd+?1*X0Tsi8a z;90J!CJy-g_b!BT?*e9>7t6g1a9zi}3oA!;AWCBsM{ywJyLTa!dlz)>T|nTwxp!gZ zs1FgixrxI^pS=s$$-N6KK9312#MXf;2C8C_7j%oPd8u}|cY$KSa}@2|yFkr_tFvH| zurevgsA8eus3GAVao_FeBrAooOOjf&QmVK>-7Sc0t^-M^was#vzo~)Z>Dt70|3@** zcMG-VR%)#_qJfE?4Xce19p70?eMFta)JLdc{+5VpiP;j-$-&!MOLIh-#56~UVg3qB zMMM!XE5a%w8W1L-{1Z9M->k@sZWY&8n%Zn*ue(R5?2%~O7^o>zFOCpVP1Tw#O0i+G zKqy*q|*6;`wX+gh)?2a6L)X{Yd}DH#}CdU$jDCzkiy+4vBXozfww zu+%Z1K!x2t<_)SacPFN#2pc#xg??R3K&-2V?^rrCm|JPE?V@#{5lTw;k9p71mT40m z9;jN|GH)2O2UI~jrtRZc(q4syc|r4z$X;wWP?VdP4$@oKDWFucwm)#fPwl#pFK~!J z26QI_x;ODo&YMeo9WaZqWOL8WnB zx)+tirDKaqLkv&AR1Ex{9=gr$I`r@%rO`eKD#r&gci_elbA^LcP^c_^uSO$HuSOF- z?zUhO+--5|I@XEMXhdPRg(^(U!%?alL|tPJQm};0dW6tuoy0?hH!7HJDGG=rEcmKFihehK_Wdv)wO3W zdRCRuCczkbB>KI(Ep+a-2yL?sSGz6r_G0X|fDAJ|xzB>?uV%Z_mqPC3E(~td@~XN| zWH4$;s2gm^Itx`cmU>TY(86vDrR1>N0&C!ME7slIZGk`(tOh3N*=f1!2!CL4Q5Niwg z=4I^eb?nix+rkR@$8HPNU1LK;I5tF>y5q1>%-C(g=!R7L@3tU$wcCQ^`F3 zQ}aNoieFKL7p1+9b|d2To%^R0xw zJ&wvMF|~)&E@CrzSdcBioBeh zWlpPVqYE2-1!jI_&0g`w99E%EN2<40HXalKT+Mm{{0V@I0PzvhDm?~{Bv*|dFJ3h2 z$%N%CF*tH$Ub_fAE=~fPH-2d16y|&#U+4+}3sq4EP$=fAR-p0W@qOm#R;V9*r4qx; zvsezwd?Uvk-r`-)F^?PFhO%hWhTViRsc6$Kdk`?K%|fI_AuS4NCDN*pR)vfqGDaa| zfCP`&y1%2i9tT+YL8p?Rk`b`52iem1<597n$uX1rnI12V#^#HUP(|jAT)@P})hTDV zsP={}_;sOr_u-OLE}~&|tw`ij@OyPUh@@7@qU-CxiyD1}ETwc(508&#GO0|sB0L@{ zSUO@Kl#a5S1*R&zY_4FZBD^da>^!`KFK|tF@l)u9duD0v&^Y^dWfx= zJtSVI*6ShQnHLH%A$;=(UztDdjR#lQ7CnI_?4b8(FU&EPOc??rNHhfmtkfZ`vWa3kMUgi;xh@#avWFW_nnrNWR|n+})b(N!kRcH^hZ zhT6^1$B$X!%f6XkddgPTH^206pxSeCuGtdz37=8NKs@yboQvX))A`{^hdH&AuG4TW zdXJA&#GOW%u*PG|IY_)sU+r=F@a3!|ltU$^#<-z?nDSO=sxQrJ7h{BG$8~BHWylgm zISEh*Efv_`P+U}A=Bgk_UP9&xTlwL$`20MHpbMR{+Z_F-IrZR@V^WH?;`yibHq+m{{c90Nmj{p@X8D9SfvTMuaaMI7KKmMB?Pyk9u`$} zF=634L>Iokfq2D-aq~mzP|A=4x~7etm~mNxEe)G04zy&Q+TF`oM>^UdNT(AeL zrk~r)NA`i{Zkst8U)xLDlVihoue%pyf!EnkV#m?4&hN>YZT<)Qaa^|EJ|ywsZ^}Tn zm%-uovA_8(z5v4&hcAs2gczN?}B5szm0P>e=`vo#4@2l+BfA9M9&%rEa z1w{Y7q50?N$>!>g%AdL_qRQZNF9}M@JZ*EQMFn+#j?~zXvlE|k28}$b+Wm9n@6#3p zu7eXT5UzT-%p1A8AtCeYgud6&hd(CHY*p4gCeLSmyavmAIMDi-SS7<23-sNeL*>3U z8$#_4zZ^Q2{)j)r|2ezx3Nl7lO$}XhDkTOhf^m&Vjj$OIGtRYqkHtR`?sI7UxGi`A zM@`)GB4kBegNm^YkRN(VnDM%K?Ojkz{Kn5lPPE%9*XZTnSinL})CLrgxoQ<=K0Y^J zchnrc$MwnQ=6m*Hn;P!W%-mz{*bTd=LKL!DnMvc)8rgR@R##kwj=ZW-_Tg!b?7JK5 z6w*p9gplmJ8ygf-_Tg!b@kF*LWITntyaSV>pA|oYT|xLl!dmX^;?jWYvxM<`63V~C z7Slfs^|JVyVDZQ~2QGihJOiCtBX|I>nAWc$U-+74jRmTE^b>si29MBxNi5%~C(Zbu z#ohdvXDqV&I>_uxa#x7huWjj6c` zC7-&-RT0{NB{+h}n+Vv$iV&U8TqXSI1`{4`6*^$X`Qm@GyY06GVfGl7!y)tf9Wcf) zeVmxPs}5AiLTGR4RXC6s9?$gm*CI`duRRpij%3Ln5@E5~M61T8R(zPmypi`yQB<2rq}jdlxQ%MNFGJB?wGJ z9EcapjMxvzixhYZ;u8#36IBObNATwXX=1EEG=8b7vR0-oM#1+&?Ic#d#InE@pox`_ zu_q}1`f_vCOW^Z!^ix$j<$b<84H;v%rL&|leJ*l{CS}PB4YxJJIAQyOJyL^=qh3>c z_~;^TRcPer=;!JG5+qWSLPrzb82W??+DmLjXf3_ynm*sO4k;_ z4zCzww$KT&P#Cy5bUFjiVd2k&%iM7S|A#f&G0*6*QIehs$20AjC_M9?iJJvG=4pKn z?3ky+@m%~!!Bw_IH(t>8nhE5S@%N2dg*C;_0CDt7ygq`(%`Coeza(9w~kA+pGvlMwl z^FrN{klAjIu74rDSJ+oC6bab)LYaWgFH{pQs{g%szi7+v#rs8#zZdTpHUD0|KM2bT zhLy0Z*pKuU|2r~^_8^Ta^JP1QN%e}k;}r@34Xr6H?mOh-^U315*e;pimO48j_iM5% z?5`%@@G-ikoatHj zC3fSqAFG(sg1eg8o0R_xinC`8%rMuEo*{S(WKX`Bkg%i3RJTie=1y_g3no{@Ju^_o zu5+-r3P;R>+hI=G;Wfby;9Wb6t^@e*fm2N{EwI6K+GMX7hS;B=y-j05Uu^X2PH-Z>(h!42?sj>+I=o_sAH$!#5$~?_yyIC? zX_%RLua1!)?uTt&Y}L9`Qc-s9sSR*rKkSuv161CWlLmm@hCOfLt%x`1{>ugv?&Cok zK6WYAczEUr3y~WDST0x=DuWXTh_In@1zfW53}Y3|#&jFlM&^#}GvDQBus#JIpe!m* zB;%wK?Nnj68fEkfjn&^GvNTo~;?N?-;o=N)_uPAsCT<^ssX02z;rf6t79n-p!+%B7 zD8tn|>F`nI;P?{uKCU=_9$OZF8N};$PT;zoSI^t}+U>6Qu%q#+A+4WW?Qq|I3r`~b z0kJXte8TSf)a>|@2a;yF<9@vFysIf5EV{NY0UhjCECQJ8PWv&pI!=kl6+5_k=VlmG zynP4ZFo@u7J3M$9@V1>UzNmB);FiJ4CUVz(7=>XR?M%i2Fig|2Sh`1Jq6kL%b@nK| zavc-Krz%h~SY|qp^0=py3(&l89+%c6WQvTUoD;cI6NVkeWL+a*8|=aK*`c%G@TB>& z9p}0q$z)1t;h^+q?b;VMTiNbGwZHz^D;SDg`iGcfB^{*kaAcf(%lyq@+_q3Hmj&Ahs{ zfk>{WsE)b{uv|1QW&EM&oUxcnggm zL00jm*?7~e-ZV#$dvpy*l7kt#wp4I915NY(0Y17R{DnK&0@^YEHiW2_oO$Net$e(X zq2KCH`|s~VN4L7FUGO!>JgVwd;c}SM_Mx|_iJ+t)BMemkmJz7_1G=~w17YeTSd{3S zjEvaLxK4oq*m*^W0>%NEgc+^owXb8&9Jd>o%b$YaVB=&JGl5ywDzVw}C?;1i!CF~S zCAMLTnN-u^Y50zZJI*X8TCaQyUQVnEhT*6cMFay#Ad)I8M9L6`idR{@m}nIlf?@os ztpA!Jh4h_b2@gyE4#-Ar(3Q379b^K7lmTCk_X0<#h3#@kmW`hINgGH+(nAf2K;Jx%`zv_H5@_hxU~|}} zPN6~EbpW`eVhI3muHa66hQYo4uBKhzz$MN;LrwBDyxrBr1aO%s`i>U7#SIsMG~ka@ z*iBfPuOXLaOB2(=tj8BQ=6i0s3Vh<)2*yT`s%t13a8z7FA%dgk8VU&h!@8;%-xfrj1nS%Awb zmSw}-z}nWGT-xGEUkga8lfE%<%9FlbKF5feHHfcDB1|NKRjyIu*zd*#-gH6OV(@|% zC~hD@d%AI~xTGVI&Wz>ZEcZp(m{ogUF@ULymn5iiXTm@V6xQ5uBvajsn~8oDad0 z(+SGY{K|?G6f3C86YQ87W9$))VaIk$a8e_$Z=$~HKp3*!D}cL&z_K-6fTk`1*2N?5 z$V$2Z^U7F~qrMEf)m3$(7qyFPFHW$#o)15JYs?oz&hdhpX^q%{M2FaqV-DMDC6%ye zS)LFoCWI;#n{&{PG=bgBbb%GObpSRskdH9Msv(bM0x7odh_mvKOM#=%N@aq5nv_dy zY2i>iX@<}<=BoeXEUQK8C1fALaf=4^5?5vs9;Zr#4yqJ72qMg;QccqECs^Jog@D6`Xg_@9z2x+moSLxJKs2HYZWb(i~PbR6f6fZ{h zP-Y-c4aNke24jNMV8kd5MwHpImDMS97*yCNP+=m(8n<}+02EgfbBpD3xR`w^m?p{! z)D7mfk+5(#2y<2w;4>KSQo}&5#U?6 z48p&e$_mg|TbbqEP{3}bwgTKF38S+nVMAMOWiohc2ws`=svUGk%~k&lwW2f+3Iubx zC$QVJ2pk|zGHO%|5GyxPD7o@iaUy|8)x)sWjV0mR4~I;=$GeR zitiC#<-H7}2v=N$eunQLV@FI&!oF^s`_@|$FSTJksVfL)QWM|onunW_ck(_1-dC|7 zBWE1BPZWy@T!F@Y72*{Bd)V^AsgWG>a*XMA;LN?w${cm8RaEMzTN8q$zexHYeF8%a zi5jj#M5_=X${(Zr7UiF={E^BZFa5Jr3bTw9lGXbaD*Phpf0W~3g-TZ_p43*TB?`qw z6Gd3Dx>m4?-3E1dg%i`}wO2t_uqg0kLhGU6c{AQEiB9a>iHl-~B)RNEFgtG_V&Nka z7itfo(4nGt;M$6J2V7h62ukO_@!ASZD08FIWN=961oZjdL7@9o0^VQ2>r@8bC&8;! ze6Ew=O)3|>zk)YL@SX@>qT+K;L@>s0xF;eM_eAL8o`_I+PlS@kaS%b9YaiIhG1jYX ze}Y%D2k(JMsx#inwm%_TebqSIs@ZTaE%v71nh%^1*`H18L zms2d*eZKdF$1-H@obveXU>iJ72#C*Oq{79@4%PhqQgJDX|aJ zH`s^SkqbOR_z2}g9kSn%*w>03Bu_Yo={RKncJU)vvVNlnyR5{rH0KY?=41ZwatE!S zsikmZ{_q0_K;=!>f>U&1Aw8S$R?Ht7=h2@=RxWg&zHu&M{@0$Tw;@TtBj}u066fh5 zEEV9GN=x1c@E;f8pXcdM&9P!34M*Sqo6gghB7MKiaP1uZ*bpiBYCdn!`CJR~tLC{a z5ks#*&T+*gcPzS^x&hQ|ZWo;RfCNZ^hV7^@&y7j&b7d&(;%%JNZU;>{2|hc4y>i$(4WGyB#z&QfM~BBV0sb3= z?H-LcHhiej#xw2@>FOSJMhrn%JC3^ZkbA#-)ord;Y_7ayh;hW-;lBH7>so1nv-k?k{ZYrq^*>F3zDn)2yrka$2L!L1ASqGmw zoNWIc*pfuoktSwIuXAAHnFUhbf^|Na>WG4s{au`RehT%7b#biCo9}u9lg=^N z#mGr#dpIYZv31edvuN&Uft@GTDR5#d9ea8a!F<>I=DQA??>Yev*QFz0;B1|oW3FFU zB>Y>}$+_i?+Dpcx+AB6Orf@JS995f0Uri@}aBL_7BQZ({b3xcr zE@5@0gz3MMFy=`KwPzq>yiafA{o=@aOJd)y#PYM)Cb1G{_7mOfIPHmpfMyr2lENMG zxGD^#;b_HuT<+Vk31Wx2Dj5Oou8uDb1b0k%qr5A4JgyMI1_pdZx(;QADb!73uJq^{ zHiDKhL{)4Ov#DKc?3Fd!Md4#p%d>c8chzhn%Y>3;9DZZ~lHp7&wx@{#Am2E`j$MX4 z!Y+w0lE@!-!fa-&$_iPvEGd?Vt6~!~E(^@OmX>m&yTVPv3IB$SYUYZ&#K`NKE-O&# ztJ$13?S&uHoP{;f%nar;JI&GOV9l+5hxd}@nDgJ@gy@?IneT9ZQ=LnPqG{YjKH#Vl z*}#Nq!4`>vt5z_XW6t{w|HsTYZ){FGk0oUjb4U7igpr})k_CZ@Pn--EMB$Rz#Ly=1 z03Z7t=8pF^z(~iH4k$&&Z}!BmmmDCLVhXM;k^@9N$M~la|2*U4{00SPj=8WZTFm0Z z)fml615M1y;?axYQQWth&NuZgKD^0aP@pg(@oF z*`v8o6?!$6u;3jL2k+-ts8TvIt;#KQW{vP0N|oBSiTY?(Wrb))u9_z5768?!R5h_F zLiska)wn8}q>Akzb89oHV$08>qa2IXSx%GTpe#P}VRpSC(U$^kceS`$%RjmV--pVC znOA69dj{O^-_~irc}p5RrWt~-cLD)H465rQVi9N)7>IER6>m;1ea2E zc5~i`)G}VJys1m%UVScLJj$l$EpN^YH7asrWD~v08zP_ox8*Ex22NAhDi^K`!6wzp zDp9tss&%6NdG@@e_z>6Mu37#guD>d&&8(J zPME&$neT$Vdl;cUr}0WF7A@$;F`rv70;1!TmCw^LNFiCocU#DFSd$gtSUSnmtOeX& z#1}MBVF!!22S5ZzFm4=k4i1Ae{G&m)1&lCrp)9i8+#rhrOJ zN3u|AV!0JIJGsD0#uiG47z0A*aOn<9h@v!&8G5;d_~lf}lJ5!dcST3-CUFdGFr6w= zy^Gs_;zb68A&*gu%x!}JVlEjR_6nbvUB!^|EO~If!`Wk&E$3!g3z*=fWQrT_#TJ--eQM>W?AQBXy)z{DD2zcoWFC1Sx5D~Yf?aqgRoeo&8CKny108;-lN zfG9lY^;wlTGLEvwe)lGdA?`|P;^>MoxPXYTTtFl^iG|~fSv%y?p*YQ+j&ykHoI|rY zINi>X8D{k{osRkI|H`F99A>YRVn%_L3J}xilty`&9ZQEoMVhP5a*LovYSc=hQIjf# zI+co;3yzf3N})X=x1t4EOmAYuhtlGhp6dZ1oKU$*Lr5H6r!rB8*M<6YOZ8b5A&1&k z5$rCZts;66klT0U_`NowOZaOe%EWsdUl;x@5pw+A7(p$MO(hZOcrT8uWBl%xL`4^z z*!q!4bUfg+_=9MJ{!h(*eeutvEHmBARmSB>b!@pU+AMr9YHb{0 zmHM-JM41>hV>5$D+s!@%tvp7|L~OMRAPx;<4J95vSSa^&vLzJ&t{0EE1< zrJ7~0-2J;4Ox`f|T=#^TO0aC=P$ZSiUHEJpk4-iU6}y>vG~=2O<{0_HRRE;K{s#1v zeibX`D34x|h}g8?>H6Eyx3ESAIkLX4417ps>OF02VTR>t;}|$GX@%3q1GOpSL{@~G zOd1D|RW8XXlI?ydmi!Ah6|2fFg;HcpQozm(`eeOfZw5U+3YOmMwhHVh>{r=J!O4M| zJx(>tm)NmE5Ma3zO0Y?Ye=24y7oDN@WdQ>HHQZvAQ>}Q4-U)cI(1|rp$}g8Tx#kHX zLXH>x4O^cP&v%^@QQQ`U#VRMIn~R$|S2>NvO)hlK;@yMLc<7|Z_Gnz+#AHYbHE)gR z@S%aXQ{zJOFvwR3a^u)D7^y}cw_J12NGxog>G{z~Cb+w& zO6=yYfJ{U8%KfZ&;A^5O_-Hmh;%m|-S|RLb`$~V5;J)(n308!G|5!+0XF??|;Xm8A z>@x^y$j^80tgdF1?pr34$F`82f?~kOl8mCf!qSyR*|RcAvP*2m#ThHD*IsKK8wg)e zSZd8J$Suvy$jf~&I}2{>t@9Jd2Bd2s`_m!mXg}K4+!E{jJCc&D8Kj<-y~287ac;rA(M!jTADdP%cHE>)Vxq?xlqJk9D9J7^%`GgL`?F*(_r6$bhRD>#FpngP zvoo@8UzU+qB3bu(zlH1E3v%tvL*er5;*7k!xwDgH0zUp9B!#T(ylhoGG67%250FB? zk`}ew5A%1YzycSd=`DZhy$H&FvKWAZrWF?7>s(x1n308Sl;)N$E6s+~K@6SCN^|o{ zoJASA#Y^(@osj#a!h7?wOG=V~NticpQf8)>U9f!O&2zI$g&Zbj7u;LAB)Wf^-;4IbBrFK=ugxQY-21Zga+eeqX4wmu zEzZmKmV)<<#3>PliVc)YztH1Y8U@Sp7iO1~&Zah}4CUlzvE__~z*;H15Y-#4vkRBy zWm#Fl_hy$`=Oo&#IfcbmWm`Pmr-!pes(^*=aky~0K&^O;~y>^1th|MxsTv(Vl z&P&{Q6>T^~IVJwgIVUm6VZF{l8eD$h>9NwT5Xzkp?}KUh5q9~`$GTN#t7zz;6E7u zYyAG)-*LEG0Cg7jPy70+04=xl z^;Haqe^y`LUO>z2zP@vpYFaur)h)!i#Uj8wz%sxJz-qt=sE9g1J75D~8ej`xDPSjH z6=2b=eSO#`p_Kvd2doA>4p;}MqXN4C?SR#{_4RE5Oi#jlKz;>SN9FR+R2pC}U@0pb z`_`)fEr9iaR={S!7Qk@qjOhfNK!`J{S;G(?a1~%LU=3gl&b00aj0ZdpXuSvdLVjZa z8v&C6Ly_+Wz%)RM6MO-d0oEel>5GwnEQJ;U)&j=hhMFYI08|6k6BYxHFGREeHUoA6 z9tPA(Ko`@oEr3-B*9mCHnrYcG27E*yADjC6q5zA2)7Li>unsVd z_IlV)L*ePN3>x#x}is-8nTJ}l^xvWL} zHvHq`Mtv6KSIe5oQ9P1$+_d9))r=n1oZ?I8;aT)X?Y&kO6}A!J z8bNoJUwYtL+X8&4!t3QsM7FQP0r+G1bBaHIzE1(420C>JqtEr>8#hsCRgzP;N$`kV zTnpr0!Q`tVf3NB2wzQEU0yQ#5<~fAv@gZ0mlc-a#&Z7*gO#$U!sgmaL!kmphj8e&Eym@Raktz%K+o9&Y_= zmNP0EtwpCDRuwEN6xR}74c}hzN>(VOPNp{joe|^v$_v|D6!1yFcY&P#COQZI`rByq zb&u(?X+@VCeck$@dJ>|kmKCiUu2nVMNY!vX9h!_DQ*K&6v}(x4;Ky|4w;t(lNs)Z% z_ac4#CgA@d@D*`=eZ5pDiYMp}mFfRUDe-6q(g{UIOF_p9xA9}SvF%N`9`qrr`sLpI zd1)O)C@(E3`PPk0HVOZB#GeVe>2T}EP1C7p8X@^wL1D2qE?ReCW}R{GR+RM3jZFb3;*H$v)*C(AKya z1v;4NqBOg-6r}uZ0bTuu{_@9qYXkn=^nv(;z@PHNA4d37z#m)ydHpv2_+4PIZ1po26;1Td73dcZrmFTcm;4H*Eal$*`DSuyQQE`PKbOn?`7q*7$nIYbwTN&K zc-}drp*-|Uz2y*4R{dmQfQ{az>sO%#U#y1j(tZ0K@QW;R{8y3H*Vn-8`s>v1IDQZ% z6|Xhr__1LpegTd49y71OAjBp87r< zqHrAec!oA~wa~4eK9Ws-Kp)BAq`OeX_f2mC@Eo@v23;%M`V3UAs#Et6(T%CWn^g3q zvkKw&EB^FA6nWvV4gVF0zZZv7 zi;yY(WhyHbi$j>uVCq4pX&E|Xxu%b$Yoxxsq3a=wSzo@b>mmKeM1|_hHyn7#GD2Ve zJW!)y)$bJ+$E)J`ns~l01fxSdF#{qqArcU@`_u4CgatRrqR%4Ggk-^I7d|{hedNJ( zD#bie0}<-Sda3|EbT#^8W=s8;?^VDbUJv^X^E~>85V;HG`z}NvOWV3vGv6)Ut93*d z`Fq9^f7z|VaVGIK4^hnWIw~GBOAW@z@)dM^{0#tTUezBxB&=VmgtO2mqfi)V1*r&(R0}4spQKFwE5X6RmZ)Y0o6>mEkW0!SlP&Z9#wisn$1wZyHB#3wkM9b6M#5vt?V5 zYn=A0%jns4*|wnf$7!3x>FEf6E@eu&Vc+kkX@c(RbWstTx$avHD z53FEKqlT>IvdXM^DR|AsO0tHTsyNWbsNQ9+gn_b(_TIJpNE*R$N%*qrngMN{~Bz1%oP0B;9G!sffHE0`tYxG?LSS!x0|#t zO~cEFYEN7Obmt|ebC(AHHpJ9%Y4F()lY4mZE|%)=hY$bL@Zf{PuX*@VZ6ove#HC~5 z?7o!fx==oU9}2W56MavH1`T~K_}ObbvH*XL{_F9+{zrm8g31v;5xze7LnAzqed?E< zCByXANZ9!&3A>I+X#HA3<7vb*{05A)p*i6PCdF?7*30-a{Aqsvd;R>yetxb;`o^#H z^Z&-r|F3@j-zz@{WBfhASq}U#Y!$!AaLE~Tn)rzm7I2}Q#m@r0&mVet@mr!C9Ln<7 zr|4s(CVtARaah4$%WPR5v_$xeR{@O{OPh~BqmC9$k$wZ;DGU(oJdkK@gCfMiBY#GE z=ll8!=}%YTQ9r_O@X@RM7TO~CeX0CLz9jblKToTYQ~w-yDEW9>!6OQuQE({7ocN7W z@LC0@DL6;LI~B}PaJho(6?|O5=M;QV!43uAR`7^|XA~S7FOt_rDR`}d(-fSe;GGKQ zD7aj~^$I?&;ByMTs9=YJZ!362!7~aDwQ1rvO2KOtoTlI$1@BZaN5SO^u2=AJ1)o#! zMFl$)d|SaI3Z7AL=q#1Kg4ZfIO~E+|-l<@Yg3A?Lui)bfKBwS|3U(;?wt`0#Jfq-H zju7!1rQo#+PE&A>f_EyIqu_D{*DLtAg3l@VqJkX?zOCR91>I7obLZbN&KfraEal#GJ-ZJuYGZD2Aso*h5BrTI!j9uH4G7Iy$+0>BCzYku4=(G)nC^B+~750n; zNko*w8$dT43Qhtv;opdF*gFPnP!WPvc*E{6=)?g}6UK;d*h2>71u6V7Ji{3A4LfBf zaHMU-H|#3|8uk=|h@Z-q;T8MDXxn(=D|^PUYv!v62LDDl1Ktiww3Q6MVJ{i5NTo#n zNyFe@p|x~4jJy$N*k=Yb>@y>MBfgRUrGD`Zd(VJ|eaQHb4$`0+~b8t?$-Lw$cnJOft-hj03Z{cS+Z z1@-$G=^M!Pa4^16fAQh;X_^78egv%It@p7JPT%Cw9mS2K?UTvF-Z^+lK_%R}-UGb96bd|0F4EZtqe}RJ`jrf{P z#@B2zzJGoB$M5iqZ&C3rD!zYsfB!3f@s06f3d!Ta0G)p zqaXEs9Sor|`p5qGV9n_F`r~!Y=+FA&O`6e9^~VqOjNAL;FY)v*{qZ51(O>k(5A*Z~ z{qe&!L%;XOU#b~;yFWfuGxTwP{0NVp?T;U+8Tzw7-mDpVu|M9T8G5fjK1?(8S${m{ zJ8ffpQNkUpU9K7W$M_)@`~N7HLtshymp|K=u6@?mKeXIKAB5KDUq06%V30PjeAo{J1t=d5qnK{`oc<|r z^(9F2@^7>o+AGBGQhLT{k1G_NERnInbPQWTwJMaR(B znF_x|;f;CMR)t^ThhK=n3HjOJhkt;0%-XhGDN|f6+0q^YK2W_I;cvQHhG#qB&ya_% zt0evx60NOQ={}*-%~klfM7oh$osxeezdR2`KJ{B9=0_@g5R4PztqT8Jh5wntwnHyFJnzTPQzG4g{mXWN z|E$ljH&yX#Zwmat{;o^lFVjM7V~n5zo=|lB`@u87kA+^fJ}!eB?_H|=UsL$%M`ieQ3A7Ieq5s(+_>q`r2*l?&fd@Y?O2SGVXc+=O z(5}9B5IPSJg0CF}-wM2_N56VGqww*5^*aTMn&H#^@V@{)P`>UT1YaTWDCaNx=eJ>y z@H+>=cMgL8U=aKlgWxR?V79x?3ds-m=ku2;@Y3!3ECHV7Q2k4ZHROM*!grlhp_Sfy zU*UW6CB9R={#4;Lzw#f04mwbIW&tnCUxl|SI^4%e{w*q9gZ}Rap|gDu{5y(H`X))Z zMbZCqknmT5N%GVCYZ(yP5x;rB2jc&qRrq+na{E+-hrZ30@x}QSIA_3+CVfrmC4+u} zfmh`jFQaMCDtwn8KL>zk`Skv$4A`K;TfzhLxnvOhnnCdYB=C?QKYjZF@Z>*6^^Xlo zKTpI!fc(U(a<(dd90IRAJ|GqNu|9soxlGaN@=N!%LFk-N;d_-H6O+XtHWm97P{jj& z3BZ$ot6%w-iSUtHz0%vNyU?nEXTG#AB&lRYk^AFF$D-smUg6J)@XF%@END0E7dH6y zQ{#ar{bWVISC!{oz>|J;nIvLWv`R&IHIFQGW`|kN$DZ*c> zwY((##=h-m2ch$!q7zdj!-J&wEgDT*<+Hx$mE5YgS{Cq3w@9UH)bA?b1Lf=2Dtt?U zq+o>KJ4pCzt_)1)fkE&O0Z)G7m40hbj5dn!kpHhGqDA398idZ*gWxZ}O4aXCNr3y2 z`Lii}{0Gu6_G`nr6!<{puwfAVop>6uwBcM72drXA7K3i75-6PWbHoe7DZ_d6iY@yjR>s z<`f$h#6@1%cYu4YaCgd$^IOf|$GIJK#<~f{( zkI5KrZz^I}gh=ILuYcn`W=?0+9kJT~hFj#KqLNbVHqCkW{FKsZ^7gO9l+rj}#F&^I z8|z%Ucx9=){xLbjk&=;^66bU{rWI%NIx$B+d>#TRrA2v(nJJ~Qu|AiZC1x^Gro)l4 zIB}tKVT^NOtkbdJwv=hkjAF-fPYOPCrW+TqF>nk5+hYREq1f=`H1NI&Ov%qE zN=zw1CS&8|1%8F<0>41g?EoLVWEU|+^H5p3Eu}=XlmXH)-ML^fF8<5TMg$`Z2>e}B zLvpb(&J^6WmsuKHQks~8GWO$ur9K^^fNCzuc98HCBTdLa77|7XZ-!%UNGi<8T9})k z&C40(#o5LE$}Z}T81IFO&bwo;=QXVTZaZAyEi2=)!+>R#>}foU6$3fB#U-UqYW7L7 zv9XhfTa%uW|~8A9`X*8~yHqtr^+U`rf6SqImXop*}s zQJtviZ15~}m3?ym79BX07t0F|eM`(&ml`AoQxXPOgT;rGNVdO!^Daq&t`)7@D8Y>7 zE1WsY3Nm>CEAN3V%_u;Pi|bg~$xKRePR`AX$;m6sC@sp!yf3?S(v;~_Q5lmbvxnqO z(>cq~2Nx9=E_Q-eR$`W8q0?Kb$kAhgqw2UHLn40BCzAZC9K7MXe;;h+=7CEO(io@%a(#bJS zXX%pS!UvqVffd({LoF2-79*^G(D^Bac_oglfoT6mYG}~oy?8723XIB^B7 zlZC=owBR<6T1?JN&d<#ynb_FbP=NSwz--(U8#|TSP;Td4$j!Enzuy#0S+-c+DZGG} zk9+PFo}0ToyCA{%<^ryekA?K!Js9&P{h41Xn4cV5RGe*S44GfQeCFmR+oqyY{%(WT z+h-*g=VJS@tsv`;BJ3#6o}HbSSEAkPG}!PeE}`B;O?mrvHluxfPnp7kwqthkpa}YUpeAt z>+C!fFf6qE?EI4KQWQaew#{DAM-9lfjpFPdJ+p|zXL0zVsPKUt*mzzKy9(4biVQ=M z690;pbV`uS6x2%*c@e~ZAWKx$ICRE}+)}?1pe5&Rb+YsQHeGRnPI(af0l6PAgcT(+ zAZ?E^IG~FwVGoy5numeNfDyQZ=JFJ9kITpls{~&ATED(V(qLlydcGOwqV#E&GW{fT~iQ53mO^ zFRrWtS`GuM=(}|)Q(>92d*EPg0Dr*#09~W^#(kFl_3CQ@lc(bIFX$`t(N`>&6Faku z;l>cmk$k;#X$C(8|N5LvY1K@nrKuIbYC}24lv-j5yKcMj<%gq?78U^-!Cv-B6 z0UzcI?~a|0PZ^+L#!YsXEGsH1EXMRE+LF@BcmaHAF^qDh|Iqn~k7oUE%VRP|jTwt` zF(S&9)<;V2^{}Ej;&UWt8hyA&5&a-zeGMO5$SvrfCEvzy!vf5tVA%Ps4Ptq&HqW*Q z6U_4+&LqeDxeF6e#DmE#3Kw!4JK3-^?3mKtu2o0(CN(?1`9 zY+Ly?53)AU1SeMmx@p;rVq;1&b8+x4HzzmK*E*$GqC*yTUq&&iDbTQu-8NpJ=ca;J zJbJ1TqqWS!Wk$d1K_mNyw14Y(j%|^gc%v9ylIO*DnyDMurY~59uoS&jILBLshP6nB z{tubWll4@3gY4olwg{vZ9aFLka{_mp(`7{H1AGR=Fc1dX5mK+Bx@k^06ETGO8Aks2 z?1`ZmFRFTtNnTOKoC8?uFGD^x@3oh|uX038&vY1*GCIWAICO~OgE;boTHao;zs!B# z>3kXcI|(M18wxR18_=RL>h!zS%|H={v7C=DIrUc^ko)gGqlb(H&?X#84bUbq`l&F! zO%UTaW($~Ik52d=YbW1-Vzl472f9C;JyuJKG@+-J+?QMQK#9K|Oevh2lees7$w0o5 z-~5dGvc;G80<4*-@q$?Wa!8{C#UxDiL08KH80;!OY?s@<#d!H{>lShR%n=p$w=`ya zRBJ_%VEDV5gWeT7iq0V81>dg7JL@Gh%s?Tg^5@*if?R*q5J-;M(G4;&5nu*m;B+*F zIK_8b(?3z@qwmzNuU8+F8YnWLU^MJOl(kxdL`su|riHe}N0^LuY$#gHyZQ{XU)$Se%tp89lQ4 z2%Nxk5;=An1Q3e4{{Vo~TZ=IlSP+mM*7^4`2LiWNuXXo5G`X?tE*hY}=nb6o8ZZYE zqo(=2i5v4JWKwfxi>oHsy2Vt)z>16g+^h5@UWBA5x@(m^~Y-FPyZO?f1fC3~yOoXkLrc6vj%vAY3A? zHov(#(O2Y%05s&j%T%{V&kmgV`}^B5%_=QVEn)Px_+%5BlbfA~m8xhCJaqXgbzK63_2u-qk|R=z#}nJ>r8mt&Y&6TWo>sn_kWbW4aQW^(b~%S2xfA!a z%$pr6)Q%fI+vlHr`{k#eZhhT9d~M8^`^l{a_h0#f>p|SrxEJFt#a)Eky6(lx-MGCb z;%Pkp2X~zbm*bg;dj{?}+_&Slt|#&G2NPIpK8x}6nD0?|-irG%+>hX%fct*jt8o7> z?&omxx(N4l+z;U%hx-cLyq>}RFz$PBPr-cw?h&|o{StQyUvROmyq>_FkNYXycjEpN zZtF_M3r(g9_b)6Yp10sG!2JO3AK}i#oq&4{ZeB0p{v+;M+`q$p7w*S#KZ^TB+`Mvd zFT{Nr0rOgc=K|cc+T|wB{umEnCGN?%FEt5e=5vGj{575xxM$&Jo3kHH!@UUiT-?iW z^LhdIdcNRF$Nd-a=5YKO&u4KL;!ZVjKjhILiJdTOk5xH zIm~=^e=Gqa3-@f?H{mY9{d3$t+*5Jq;pTO{e1!ha$J_?s(k4!JUNLeys-ZJnrAykT=@idwO-UB`rbB z1jAuiSHRO$KP3^=JFjRcjQF-UB-~8H!1dw<5+6u0PHwB+7Qe&L%#h&~2V*fApVuw@Bc5i8v zj8mu>T=Y|e;boO8OoYHkCJh{ z$^XS(^iXBWlhsZBD|$&k!PHy5*-tIKeLYax!}^7?K+8}53jLCLbCgNEm41qekLJLQ zD@FW8{AkLP(oLRcMY~_*$l5FuZZOk#^y^sJZW-raVWzP^J~PXWH1%MWTiA==9*P8_-!Ztrl?DI8 z%ZGZw|29*fub~~O=k*Ut#?dDK{k`a+t{41nm|9PIsD)n=?P&71^n40@ zh;K9NWtH39OaFb|)X&@tq~PmKIm3Es_agKQ@{j(7OgPs}KcN@-$M<5-8xaBuKBm7a zH~rgJy`+yc+iUGJvcMFis{pQDZLL9u}H_~ z9XY1`%hZ@-oA%Ib%46Bd;9lg<@5N6ZH}%|7DmhzxF6%{~Par{0a!%`|e|dWuX9t+& z&NcnX2D4&8tKVHH?Y7>+_oAOid$CU)LiQx*b7nhceI*$;oBH2x;;UOtJnVzl_FnY) zNiTX>*^8Z5o9$R``WeX`b6#OT7S#DXZu)_iN{L$wnAZu&Mf;CiDq*X=4w-Rj%KcsO z6MNCaF(jjZUOU4~4>@>M^ir>U@b5{VzFzRl(0r6L#gyNY^Q7n(m=FF@mKQA_jzzuL zNh|tGPvvIx5+`2*J7Kw5X1V1i|Ba^Js!hLP$UmA>2xpV#Z~7L^wH9Nt1aK+0R1%h%lEqPq$l-i*cGnFaHUDt!eN zQ;KsdD+?+e-rMqvvlh?tI_7)x%1X+sd8<8cwxS*t<)R&$WpWt0s)`>4d?NF6`udk@2Kr~d5XqgH- zv-rdiv#CV|#rf%3$rh8W@!o}n#Hne>G2T~(Ca6Fgs0-6)Rc2G9S-$dOHE)(LK0a?j zZiUxZkz3@eRP$Jay!7;0^VDo_cAPgm-kY8^eO7{Z=JZ+D|FCQ#hsaBrr_ZWP%A*kJ ziwVxEOpBLqAd$<=<)@nZ!%43qM_$ zTLfv?W2VHXqfbdsRC#@Fc{zkb=gdPjqyZqqSaz}Eg3?e9(c6$KQMe+@mh|pFAS+x;f>$l8^v-~wC1v?AT`Fz7WK&cI zXY+0EV;L1FV?|~#@a6iL7pS<(ydrqZqQauQnX`P!;Nb-~$+)bzGCgam9ZgdgWsjS?OX}{uD%+@kx0C zs4OZg&CbPy!E~h|PoAh^N`kS&>~UT3=D5g-3PyyboV(mMHkp$cIeKF{Qkv%jYj1fO z2gHg>!6tsZhCn(SaWt8mVRYv(oKTWmuFk52?Z+od?_XAE)vyOeK;xXUAcyb}re$G( zg?B`!=;WL1ojDItbwL4=TFwT^zHd1kKI6Qz5C-M>;wyc^7j-L}-7pz_0j{Hx?wRAQ zStk5SJ_<&P|NC5Mnh0X4yZ<|{&K5vqS5{G(TAF`zc|KZmNG4_Q3#tNM+9+jrwFXP7FTvs(}s)w zn+hU+AC3+mvtWYcyAK=D@W4*&T5<^Y^N^xQ;66_x2TF?kc! z{$YGWB6Fo;3d1u`YK~V^Gt<3VdgiojMRK*OnkSp^2S;Lgvk>Yt4ik+t{X1ta2?b?^ zW~&s9Lr*R)Duq|hlan_gsx_{0CI{Pva)=hv^2MwddROJ5qVgq`=x3n;P$Z(xdYt*Z1M*MYL+2mc?{3;ePQpVyuIr@CGOr&n^|G>bmnX01>6O)H zb}ibcg%-PzTty}2#byh~Uk@jjyLhR$vZ}nitio52PiGgO3@la)60uxh$B!NR?Uh|* zu2c)tv%Q_8Miyq);L=O#d^P|3rslfx_vek&z3`3~#@}T{x@WdMCG1IBZ{^|=tZbJp zNn}0a%PR^j#gyb$E)p8epE*4=h!+)Qr6w_y4=u!079@Bp3d*@cCzs>KCLm0}ik--g z6(}z!{m`wQO;%o3NfCYG*z2oARAft@hxl5gFPb^t112y4ZyMS`6wHh)-IBU6xmU3( zza=(TMtYOZo*IQL{4VJ#ZPHCOrCZ31EF^OrrUD4ueJ~I?-iHJp+l5@%om|*UGteTw zRUdYu9_!OB^>=EU>5FE@j}56ier$3#jrU|5$YxiYw2fYs+%5HYDw*l+LQU-y%2af5 z(Y&IP!fPwb9C)Ken6E_@rSr|#5cqTAoLjbPLnTN6KyS>y+P;O-e?*1v7nw`5g_A;#ylIpOM}0S$~xHsBZYUof0424IgcugBaZnpJJZB$m)i- z;GFrR1Vc-nxhA~78@|Paa~5<I+c6W-qq-(tc!3%jK^;r-q4Ehe0^ zs9Smy-ro)1V!}Da-O`)z{%-gd6J9z`rpI=8mv1?$ZFn5X30?Qw@YY!X64%=BI64u! z9<|}^-@mN4;dz$Nx}LS+?caoKu;Hb|GRG@6y#3pVdK*5?Vvgr_8$R2HZ?@rcZ20{) z{B1UTiw)0jiLA@8;pbXNJlkyeJR82g1VZ)n8BLp?lhSzQB74tk+ z7;sP7@Ns5(9M=$@#Udr!@aX8FE5(MVtyq_0!;_bFW!dmn-ywvc#kx8Z+c!#CUTF*f{u8~#cgzQu;W z%7!;=_@CPFZ8rSXHhjAcKgNc4~Gi`XohM#4_x7qOaairab zzr~i`QP64svu*eY8~#=sKGKH2-G+~{;rWfAbw%6ocUVX~N89jT8$Qm4&$Z!`ZTNXM ze2NX9Z^J7#e1Q$0Wy2TR@Hsa8d>ekQ4ZpyKUtq(Vp`@Ue+wco*>6hB@i){F68@||v zzu$&0vEkR+@aEAzL4DMQFSDgzZ^M_{@Xy-tKeypG*zgrL{3|wmr43(i!~1Ob?Kb>k z8@}0wH;V6yEJen);EjIiz8y~}lUv9&<+3@$+@a;DI3LD;0*lGXw+VBxJe6^hOf2ZN89jqHhi27zuJaRw&Cxy;Ztn*2W)u7hWFd>SvLF{8$QQ| zf6#`XYs3G-hF@UA|I&spx8WbM;g{O*>umUH8~$M%{(c+&5gUH34gYH!{!ttLQ5$}} z4gVV({#hITw>JC+8~%4T{3|y6V>W!f4gUule!C67-iB|s;h(VK_uKGK+VCwl{2y(2 z!-oHp4c}(NKW)Re+wjlW@D6i+c>!?G+VBxJ{QubSkv9BuHhh!~|GW(!ZNtA{!;iM% z|7^p@+3RKO`JZc)frR67EZQt$_Cs z?nk&G9l^N*ewFZrgtG+v65;-YQw01R;fn~z3HV9E0|-Y8__u^P zB?(3f_#wg<6Ltvr0m3fAZRdfwaus1tnS(6?zKif6!p#D%AUv3Gy?_@I=9DD3LBR6} z4z_$_RR6Dphk*MN<`gE_c8>M$LpYjni-6Bu4w!>Nuvx$- z3130DUcko)|Ag=c0e?uCQ<>m;0UsoMCE>LK-b45*!qo!aN%*IP%LTld@Myww1^g=E zs|jZb_$9()2&V}6Il{4o;{^O9;cEy-3;4H$IW-AJ3iu(y*AjLJ_yNM#5pMfNv_D}^ zS%NJBzKd`?;bs9>5FSgoUcd_pb7~UYAmDj~6A7;u@NI-S6$!2t@Jzzj6Rs9;2H|mp z%LP1{a5CY!0-iv4JmD+>k0m^TaEgG(5Ox!e6Yv#;ClZbp@NmMD2uBKdFkwzbf(`-q zCwwE}wzH!B38xTl5%8H2fKv%K3-~1A$%N|#e2nlE!W#tqA>lN_>jiv}a5~|&0^UP- zD&cAY?<72paJhgt6IKY%74WNsRl->Ueu;1f;S>QsM|e8nH~~LNSR))Q;NKGFR3sQF z;D-oj5_Smq0m3r~x1AB~PdJNki-7MUd=uej0ap;7Nw{9X3kh?|5!@i)d4#hGuNUxb zgl{IiR=_g}-$J-rz!`*R6D}9h5Ji_$?K1Mi? z@CE^YNI0MHdI29KTtIlOfcFqCBwQ`vorLETE*J1-!W@8ta|Qe=;UdCW0)C0`Lc%Em zeva@W!f^tAl5jEMXaWD0FsBT`NC7`YxRkI%zz+~EBiz<5+Mh6|2Ei5q-$nT6gqsCi zLAZi&y?_@I<`f{fLBR6}`v|WW@NI;v2(K0JOu~x^R|`0U@Djr10-j8GDdD*So{Zof7R&xSDW_ zfX@sEypnLUfKL)$MYtX?p5kJBq4@hI4IAZ9>gs)cI+{gzvDUh7uAa0%8l#A!f1H|) z50R9bNpVQ)@C{G`lm7CfQ4XUIdZQA!{8xC_0$0}J0e&wck9PGDrQxhosc8*U#_z9; zc8!F0QEKYL6uoZ%^wf?ajYzr*ub@|aytQvBUc+urowH+0p=)HVDc__NuyouE0er)e z{8vMzq!*+7pMtWZqlNL$jxAE+EmlU?$Z9f3%}%{J^_JAxvu;*a9)ENsD{@ZL*Tj_L zHC2CC(`#cmh-&&)Agf6atc$syPk*{&Ypr;7bgXSd;G-bzQj?tBt-8aV zY5Ka2OA+?yZ&jcs11Anaa|9;FP&rgz!q**xG)q8(7 zYHIq2g8D5Yb)@OXdL~lAntrlpBHLKe5B5x40HUBM{&hl6vfx73iF##aa0?9rglXOS#bf6J5ib;1r!e<5ZAU{yb-uM<^L0-K~s z`D)sj3)JJO>6>EKSZzG5d&di8Ax4qQ8M+mJ8}QqlfJ8WnC? z2@D&kePAelOm2Bhgu^$KiNIWx zCK63YPS6HO+5n42NsQ|xZHS}|v1mf7AAu&S{22wM9*ufdM&F{;9dgySBef(OI^5zU z12ZC5U<MdLOuZ=IwM}e_Kfk&EuoNM)m&{KNCr>Py=)AZt)EV#UlaDQoN zqwF;OYvOWVWe5*#%@HcCX>9XHp-caRP(r;vBF5;$rmty?{K6GEbl5>^G&?2-z`%h5 zNQ=oj>5A<8C0mxRehVPzukISnm;i{U?Gl77g!#Xr3Lb1 zav;@@g;Zao4{Q2qRo}1b!GwC_+0Q#V{Mj*Cb@hHNCfij@b8+u>-A9v%-I=Bv9Sy0_ zke(Qms_%n~QZzprU-SE7B4Mi!xkfpr1m;``m!cH4vb(DO8x`ldjzO-`&6Hn-$gb;l zF7W{9n*!ip9DToV4a$y5FCbEX(yRKnmTIUk21c2ff$)EDuIsz9#o?QoLr-~3kbMDTFhwUR^WiZ>q&4| z{TIWA(%ktm3g`n4xK>4B)DAuc)6x9W1!(Y+^eCENnu6w+{(weDSD^qeT*t~rTGsSxW2Muu^($y8cwxU$!QE{VH21VFz5+qr_u0C4$eT=Cd46BM~+M z|GKuNqeJ9S^rK2Z4y34M#EnQ2&m{W$lx9EXJH|?O*OiUT4xgsL?&4^q;!iT)QjCis zmy#z35|IEd4Q3vtKqcZ&G~Y%WkF+5j7(^MYZ^EZFn?x-fuGOEz+rSU#$E=~sScqa_ z{RlZC2MVtVSE~3UjfYqkyobMU`+!b-R-~~^5Ncyyffa&!&`ymqZa1kLBz2RW8f{E6 zsn1I4AMDf!V=8C}VCa%}WU@zPGBTFQHj13aPasqL8^V1!a^e#G9EyH17>aAv94BEHu$5 zAKRG*yf@$pP24lEizddiwE{^`zu83-%+o~^tJwVlL-SCmB{OQw61r%@4{pHMhw#$M zjLPGtCOQTPO)!D1g?IyCaW$5L1*9cwnL@moDTF3&ktrynOd;OP6!+t4BqD{-L@gj_ zV!eDkg$Fh9ES}IrHe|8c#C2?~K+;@jp2DywRE2rEXksbs4Vt*MYbRPCO14p&_DXO= zb_yV~o0=$UH8t_hMM4uyAZsDs0E9GA3Ko!deJF)^GgAmnXfg$5lsv?nnL=n{3{oW2 zd+In0EXBPX+NeeXSM4~IZ>xPgTPTpUYFC%qGijIF-vt{(?Qh@HDJQ~=4dM}SzD#Nm+D4?^(%aIeb<&w7CqKUfBB@IInk;0pZHvrU^sOt?# zA)>bTjlHNgYFjSBP>lB(ZL4-Le$!8wGfBj6kez}?7{k#G>=d?)sDk0> zMxpJIq8Oh9MIIQe$q>=Av71N~9rrrBS;8JnH3Q2_9Mk>b?bG$Nc2rf1iJKmnc-r{; zCp7xQRL@-FE}A4s9Zl{RM3|G|A8}gIor*pSa~2muJ2rJdPKsz%`=LkUkB00x5n^E9 zf@qhiENTHncgqmZRd*kVj6Wj~8P0)mz)~Us54sTHbN#lyuU$OK0S|!r7BXRNDIc`2 zkT#(n@%=_A7)v>^euUIYV2!}R?YxBXSPPiJWMI;3$fqHqi^!g}u-~0Z-~|y? z-Yn)BsWXi`p(4z2qFd*oj9-SzFm6G~N}%sJ!5gy~xVg%X8|D2d)ML(d#LgmUU&p#)wG0Lf+(r~Pg z(s24lC3d&cu~W-CGb0dluiEgD6TW)s-J*$<*aP+~tuo75WZ8=>O5Q<5e@|)nI2`fc z^0*9t_^3|9^7Wi&V(kS|=i9 zV-IzNzS;~$C>`4w+BO`)YLSVlt6zQ%$?Wc(r6xL2s58Py5`}I-p=$(#WpAfpl0u!P z=_fETICnDDHRhd@s%y;0>8`(g4H{<+CtYKjNO<4%mrwEj=1Ijh1~xYEnK0S_;(JAZ zOVxL%`WveLmQ}c-zd@nyr99pKWlE9`xxe7Q{qxx)AoTlNMWWyGnr}79L&-1n9p2;%5wCZ=5|cU zHqwkCb_pb9ItZUR|yO}ydA<#7Lrc7Pm)KFJYSOQN&X4Rl6wve-T0Ns z9Ru)wbY63gD8}XOABk~U#_(e5xR%}fe^&nAL{H=|?~cW)mYzSj-aQJ4=vZ zawn@o$*P&GCPinUK6=fNanu- z0@MEp^<2SC61)Te*AonA=!y(Ae zkvve6Z`dTrMKB@@MaNbYP1n4RO~k*g@jBdM~U;^CwOGCK|)UDPtm)=O4Q7 zY&5*}Hh)FF(H~Ob6-gu5beACk5(2y#IgJ4#ecwNc^qVDN4hUF4Q{wy?`5Yn|PP*>I z65F5wF~)n8&p&jrWRFCX~D%W%-QH<+DN%%X{#x^8DURj^vAzC77 z&&%v-B73j(!P=x*pF@W%b==ApFBaMR_O;iixk1(^>)Z92Bujhev}k_TC(F=79RG-n z#&DVBuboM9jKWZoY9={_<_gs(BSgz5?GT&NKh!6(_gbHvP<<{6WxtZyACuXWWa<8j zxyC}an6S?UqWzJ8W{4Ot_0HhsFwZXbKIw7xyQ`?=9j+k}**Ql+3(uLq@Yyt({ zVPAW+gIh|euB)e(^_Ky^qb1{+4-uTn&Js$mg~37RrH;TJ}pkwQ<7DEvZCr!vJ^cP6SxOh z3eZbE`UCfi*BUW2{hH$XCq0TXjp@QbCyl^6_N5^GhhIT@S=$bwE0~dg(h$&0RFha#DnWc&fiG=Ja!IYm*-l=P?YRWIV9>VkrXS@I9~+YcZBG;j!9j1>oHXOn&;$vGsyC&`;g_Db?5>8;0; zd{B~qPjZSR=SuQ$lHZl&6(o<6iZHH(uwtE_Km_|ZE!r!$O0Z%OVK@)655SIZga0X9E6xT;7GHx4_L2o7Mc=K| zeAq^53SGWFg|2t&Yd&!8HC&tOt1eRXX2rSNI8DWnC#Fosf8Jri9Y#3o>qL3jVP$a} z-z6T4EC(f=FVbNnfyAHE1wufv(QK1Zv?pc!8){%~Jev<5W2lDt>t89@IOf6|CrlBq zM?tB^eWGp>ZDgP%#m{vUCcq||bE#nr6&@hz!7rmlz9?fIb);|8KQk7J%s4Y(@iexx zF5NlorEefdLnp75jg;mNQ+TuJ$3>$vO@Sy6H95`~L@cdGyVgh1+ZFw!?YxH{v(x+A z;Tk$VpcyCv=J5*~yf&vV;15l@=C9^?f>a&^v7h&-IUc2AX1Ehf_A&y2uE zyXwC|i|tSwI>J4eKDrNA4i{5i=V4X96DxR)QMRprkKUxl9(L94go0K7)JQe9L1}0Y zSKM1%t1!^RgB|iMNAqG$pyAk!|AI0cigPbq`tCDahru!m+1{{xxR@^%)_F`krBX|X-btvxri-#%=N5eJeF16-# zq%T6%f8M6*MeVE)MOU2%JZ^*dJK_0o|>fM?KD?9{16T5uNEYcgvE2p1O~!24f{7JmJ&U(dwoU z4`75B439D1hE-|s#W-d_VWD|dROB4>woKIz;UtCH(0b!5tgPld1ZekJp*T<7AueAC z#kDKW%|dY#GIVVM6xS{lHvx*%CPb+no0Xc)(H{5SioF?u@SCOLvY@zwR9p)b*RIxd z2*oXd;%30+4Ap(kRr|jv)uSK7l3MJ$uQHn&sBkF_=bcK!XA?65zF{uN2RK~SzfgC# zLZ-zxsrm_z^NepW9Qp)R4jgRyFJ3AtzZwM)QV8-s=hHIO#c z!4~(pgWv<^@49N?R2`n!HxqV<1wv2YCYRc9#;GZs-+@Cd28_tDek;@tB(Ob=G zWGewI;?#ZZi=5$~*hkZEnBiaECqmO#K)dJLe1ixg`FK8~Jo>T-VVzr})tUy}p{gW? zs-guZ_SXWL9Uk|al~*gy{VI=3;3$Bz!8aIdl`#46i>CXv2)t+)p)#L9qKRXbx(;7o zW5phBa9*oAk9#11s+UG~H_*VOet(CDOWNISynz3}kbsGZbv_3*LSH`>j0MXrg=?k% z;5l~Ff7Bcwt@vGH1!;B!x|%Oj(-ULhD0tifMekDdIanyfDi}^zL@7Gf(A-kU11MXm z*%GC=->bqlF;XEu)Nn2WXCB1K1@?2~m_ZjZnl24uDLAvhDh7Xt+(I4`_Dk4yJ`Mf~ zhtrf-JO}e~ezRiHDpmhd^OvAWurzjuLy0}BoWhBe^NRC~($FygMtP&Ew`;LmJ?@su z%hjSru1krbC!GS=eV7(|)B~eM9i5-JYAaypir*7yD=GF(u2VX9D-CDB_%xR)HTQ?E z`))x}IJS^TOr`I|slEZK`#-ML&#=RoI|7|)um*an6+Vz5ANj?8L6)or7C?*kh~=|8 z-sW(CMNg};%BcfP5B^y4?36+&I`b$4Pnk`1o>Ri^z^uq7Z) z6d0d3gSxVb-H4Mks%u)a>TFis%@xBGck_}#7)h#JTG2_eOi$=&9m8U{7sJV5i1Msz?a+RK8JZQ(I!1FYAeQ^NEjI`gZsZ)(mX$nGin zNF>$wVz2=h3?O7AjOa8B(&DA*F)_xE$){+G$l$*R6=`H_!H#yJG4?H%!AZwG4v!P# ze>%F*Jv}&-mtTTJAN@PzP^u|num0rFo3j7Xe=ugj!4hr*BrE=MxRLX|0wwm8Z4JL> z>m|x5+CP^5&qMrocu)nS!rK}Zt?F-@UIzoB^e%UUzuM8H)@+UPxQ|yH<+vTC`H@MJ z#>ic;WKl)Bm*8}y7k;K&JZ1TtSev||-3ZM;8}U!G=687z#^}4%QwBv;8`|MZ_IhIX ztL`r4eSPcw80$AUkt`5?}NV^vY`3iJN2Z@NP>X`J1JWtcjyKM}O&|K3)8yqU!H3JjZkwZaOgrlT1zb#iY1u4?5}7X27R~HeO=khq?K} z7C0?E+DX@apJNYD^)G=7_(Jtxr^I4gVTy<+f;mC^64TG1C+M@v!_h+o`Xjkn9 z8b|C`Jiv(qU*de761Wqw0!}zLv`)mCxC9r1Y;fcGx7~)Pu6}WriXQ5G$A`(@fNK?f zKtF8NT?=c}`XN*}gHQnh&b1h8w0>x0?b;|(5Gp_+7-!M%5h@Ij!A!%4eYL;tJE$AZr-Tm|}GT0;Wc~>Mme=VM7bb*83<<1Jhu1 ziuxJ;IoBD}{mZXw)7*y@ePN`l_C@GjjXjG)kRm=)0!u>i86uh+TnUKdPb1uXOWFMy z!WrtkUUP2q4bJpq4_Y5`L8d=@B(#uy0d$VAaY&{=Z6rqSvn37-7M(pB zT8U*SLB-X5iojh4?|M2kR~8=99M#sK>V2^I1LZBm-Z)OSaiEdx>q__(-5wL526cDPQ@3OBOjX~aIp3pEW=}%hvbnLSO+w|gzEIs+ zVXoPeV6NGHxeY%F)zkVybz8CPh!Q`qcpVXGl~mm)h-?SMDS-;8uFW#n2F3Z7=nLb} z7vkUy#({Zp9L$x0?ErYkj0m-38zw1mf#+ST4pVJjFoa)FY*RA2#uQc-&KH)=iJcM{ zkwR;y!Fq6RIb`Zkw&rZr^v~G=P6ai+4653Pcv#rI78_h~jo22g*=opz8)N1hbi{9> zMehCBED42xKY1${P~EM*VdgLyWyByg>JLZJ%AgPp02l45&A?p9cy^m`Xv$Sw|*QI*iG>OMYQ z3gEJBNkaMgay_CSma<5g?6b?yJ0{U+3B27Hqo9)D?@C$`k!%mb=q{||Nk(MYCR zonNy$e}e`SMmEFapW22=sx)9vY*Rvq?D}e;Jk<3w19SSp-Kuc69=p43&}d@l`q!fC z_d^G#yUnJ%JuSLE+%26fyFT4*Hr?$u>2BLa*S`m{q3a)|yG_Pq5cc~8yFT6RHUw$D zAF2N3NzlN=IFCOq3Eb1-%&tEGtTPaLH^6#7an-$VX;vC2E!96G0vg5edOLaw2AJ8< z=XNwj$eP~S4gl7KR8_}ze{FQcL+j_rL?_H>-Y-`qDTB99rM&_WlSm0UwI zG>W1j!*ubooTiLb5#u;OV_Tys!}~iR!$;o$^%34gDdHRFnnAi@6zY+|&9^3RR;=M1out+S4P;tqfenRn&z!-Tc zqh%2>$g>w%IEvyaE}Yh?yBljv35OVwrJx08B63C}VwRYV=wP+jyF;K5&ExSvp?B^^ zyzZ)PhdFut(_kk}s(+jki;(afM?H24Eb32&2b>JA{xXIGsuUFVhi&LDaGZ)>gb|Qy z6qx3vCmdoIdW{C^2`HH3pT-LmKk6Y}(D#h|REF>&#+PiUlZ-`(QOF-9o)Wk@q4&Lj4TsTMFS@ z3gKJ6fp7V!iz3?*ScnxV*h`q^Y@x~!%xQWqd`qF}TUspN(wOkIV}?IF8o_in6-wta!>>hS!9;UlK_xpH zOD5S44vti);yxvuOK5O}bBVK@iy6xa=YrOnn5anKlGxq1sPHYD;c(wk-JiMY4s}(i za4#A>xPJkB%%@n~QE_828=cKnyPsPvc!hbIQJbJ3YQrwg|AI)b4Y#KH?`^{_j_RLb zXt5uJ^xIF-r@~OD!cf-;-;EVN1X(QzZ>fAYzFeAT8mba|Ivw*O7y1fE|j}U86Sa0f^85k0w2JVK_{)R!vQN-dh7}WgX<25~;=c2}=CK{N- zbs#zT&4gY#=|Z5^B7C+mP+!bw>QeJZvqwX99{0z-EKlt5PM?85n_D2g>k@C{FU%^UDOTd)+{gzwvs zy7d{O9!Z6{2B(N>pmxD*zID+>1ed6%S$fb!LBjvwgPh=D+c*VZb>0{u=&Zk43o%Mf z1=bh^sSOQi;e3KiFidey7!maI%~mG4V{FW_#>V|-mEiL*L*giaAs~3!8RoYQ(@%(G zD0;MbS)I*kR)&o5Ij zY6WM3jQx=A>lwFnny&_dFidH&7pVWRc$*6X9(^;GkZ;stcY2&V5!x?_Q)Bli?spW| zw3CYSq~bn_Rq|J@OaK@6S6&Z+dfTlS7k3%={}b|9>#@JX_5jyExyp&<5-d95J2Uti zO-Goif7RJxDv*%91^ulm$l(@!5JLqpSm?!+#9Tj}F8Rsz0eszyl~eauuG+~QN3a8e zm1oU=y&8*cxO;%OIZ}yzSM1v~oWb7BX4idJBB6$*DC_`XlEkTy>aKU)7odOB8)-J& zn|`pB zApKCq{kd!PdL&R{KUdvfy4LW3Gj<5o*!Qu`a3kEMb8D*D08w_gbGv~n67`Fxp$8!c2N$1~+&0~x9oLS{iR5;wDArTfN8>&*cg|aXe&-r%dB1 zQ#@rFPnqH=(|E|#0GXPFOkzPp{a!MaVq~JwVxr2{_zhbJtuYt|$_By2Q8db)6(dEX z(DQ+Wb`e`CY@0o(0oa6Sn>Wn1*{iwtux$>wRzD@$=78!x=vwnA-ZjjGV|QU$ z=SH>WgEq~%C1HoUyWNZw5N0Lp;p)yb4JT5W21}m1&u03=**+L#&_3C0AM6Tc`sc8H z=CFP8**^JJ`@BJ2$bBGHZ2#%#f%8iy8i#E!hfHu;`qK&i(?ZIF4C>M6i>~ihs zQ7*O-C+v~iqJim%u}&k#3Szm;cc}}8kDWDc(C%)<65L`1;j8A{EyBup zgq0Txqsr9NE;K^{EVjgB$8Y(C5IZeCiWG!cST3V=?d5XWL?OA-u?s$?^2cJ&6jC`6 zUa0;dH>moM@v2|D0e%Nz0J=FvXIF2%!0hv?^BYCK2j0WLdV~D-n{J>^?EdPFi0;LJ z!pi-Yt%b^cEqiWY7AkiJl{<#YX~SLY$EQ)bH##d9)SNrT#@<-$g^dxt7**i5A>+pk z^lDTN+lHjDa@;n=;tIR-7pz>kj304E8*w-)hkZ?~uR5E_b4Xl-$ck)~VhiEE> zyC~zie_(k6Z;>k2N1~0}F|bmiJe1?nzr<2+)!W^c!#N~~@dRVcbdxv66S2PjmE?U? z^;bn1GX(1@aK0zn=pz`PF!~F|Uy<>_*X>gSZ z-%PuZ(IH!ZX|vVE$}E)P&`ir_HZ5BmLQNIA&z{Z#e0o00T z%KiHkv`LhSip10SGetu+2xxj6zrA1<9$vv*lODU4jm=tAqHvt5!CK2U^sIm8Ye( zN8laP{nURUn*+NutLv!#z^0gJ^1&B9)L2F!vE68+%qGD&vY`{0tf&1o!q^Q7QaiS$ zc5L%sl!j&gf41Jh{q9St-IqbbQVlo2t4>(WfbW`ohz2^0XTVE#{gsVuU%7_A1yW(Z zA%k_I45@*%i~F$r(G)C@xY_WU0uf=%GzB6sS+D&oJA^Sw$|ME)n`Ee23jSUNBh_AWL>rc2{_F&8PQ`CA;KMBzs9}Ah;WH<mNE80;NTIl>VO$M42)Va9;b4- zoY4=BB!NC%*cm095e7cMP!YE;#u># z!F)12Hv+7NzwhcD(OARBHhn1lV=`Z0O^ktbeyq4X7Q=tAfzNhe@Y_oyIE+@IC$pVy zknF~Ro%orhr)2NYpAy3qfBuIZCrz&J#k=lnglZBx68Hl-##pp5-7iyM4VB2F{52_= z$C!dP;}5^E8a%3lPQ{vNSbZGCt&S-yx@HTy+q$*d-&6Cvc!)1`-fcAfkmvL2ew5!+NH7V=Yu#yXSvcEE2wC z5L%lYsosIzI0q(jJ;h<1p$5#1 z6ijeMA~M26yjii%rz0A;oCYrmQUT0Zjf92|iecQqYI)q>u)#z+eLwcjV7Ll?S%V3A zu}|(NcN`tQ41IwI{lRBD`CLNp;~mGEUA5C$ z5D+Yg&lWyIh2WRI9*b!P%Gma-EQEks(tc=-{87tMo2S>Uj^@_`b)74Z+b4BDJ{=s`8Dz>2U8{X1!)6%A@&f@ISv1xpe4DGtNMp2 zv71F??=~v0(gHEZH2q`EeZ*C}h*=S8{7mpR1CbyXfs|N-pKqbB>&Fy0A{JsiVpc-{ zr=Nj5aCxi~oukwdc6+dB++aBC>Q^N4be{7o{OFjWvCd{1avcB0TF==QNPvj){;-G!?E^%R+#Lzq~&DKW`pWULotsUO<-2C}k3 zumsk^9)http%Cm!laH{H2qskk zb%Or{Ib`6lSXkvzW0R0^o$S2Oz*WnKb-V?!6{E&H1uFm+gP_rzf^(D(21i!NS z6v`5g3l;jSCFBJ0B86auwzJT_tk4ED0*evi!2OaEywS#MDG=PWnoptnTTvehp%Wn? zlLYYglg5tkoKE4022M!7A3F704mQ|#4otj;huC4j>;pUkprh#(9StV@mvasv zR3g-AM#SJaZ{Cws1{Pa%IR(Kt0@SZ$@q&oAg8%*+76~JqPcNW11@ACb5}XL7bv++@ z3u4{&H=-TJBJvF+9mhM)hJ3sU5C5dhf8hZ?8$1W9i|6-ey#gIMTu(LxN7Q@}rfwV);*zFIX-u_8ZD8;x;Xl>{Vue0E! zuf(J8YVjD@!3U3rTzu;mV-*sb`H$)%|9+5-Lqab@i$#v4m&L=%zsr^%#V(iGo?y18 zWVR%kZH3JCjCc%8;lnI8v|eMa_a0g=)_XhS@mp|?PJCSTCfCC70R>FRRr@SdGS(Su7B!vF~vO8^5yYVL#*6H@3v|gPyQb zY~$lRf{_F~PoCmv9n9nuWVqGly0loF?2~kZ@@d!-LfK@*PX%J%QSmj2@ww16R>@*T z(lctW3Kz2huB)|X>8Pc+~+iZObkxsALg+Ld?$u) z#)c@{w!qF{aEFc0(EsrReXUq_KUDP@j<4VhJA{T!y8c$>Fm6O)@~Gl5jEib44uM0~ zPmI}+%;P+v#ePhYakfB=m*_k=CZr!Tl0oJNQ_fH2YJSe+^uRZ0PJEVCijB(^k#=Rk zu#eO@9LC^3&~z>P>%RXR+K;^m%(x$G?a!V?HrxK})m%@v?a#7%ZAUd^;PNpSdq5BE z%wjTF3w5ELyM2#?m0?WSu$>^Q@Gwfj{3G>_;2XaeVg|Q}XQ;n)OJ5L5|2vr;%i^Q0 z^Y`NPeF|nm?Y@L>Z!MfKqJ+Fn2(*rKbt27vp1#Xj`VR0$w;wHLp`CjHik}BHpL~ps z2kW~UwzxL*Ytl8xv-l|tTo_I>p$ouv@xdC-1*EuY|4lc{O%EKm4@EBcsP3Tfdtyp3 z+PDisia{yn+MCeIr(Rrq41P}5&U$PT?q~rR!DPqQImbHO@_K|3pi&u{ihwNS2*ltRGRH#GNG7)NmUA0vmWc7mul zBqwuGd9ikie8vbispPv=R3162-jy{cW2toL+x-=$tcO`YUls#Kie}siFNxFbq*fhk zO@?h`S#AfRAKF-C$9Z7Y`95|!eZxXIV?sH6fqJLUI{ojO>DO# z@`8-f9I}7+glT41h!>8Ger}6>fz-wyeg`5L7%#)5TVJ6vqk`Ibi43vGu~-}y{wBm? z16kZ|v8VtsSRou2xp3o7$Yu#L*y8e=u3Q$9%TSBUHRN&)xJX%Y$pxR);R`Qrf{I07 z5J%Gb`8ny=w0!qo6_YmCv@@FXjOIR5adl@p&E38PO90S^_`DYOhRF=frD`M=Vp|hQ zQU3*oyug&7;L0>O9CVJI!dL`>#JW9Nhs{h^3{$~|78L9o!%Z8UrQ*41*-7AG*%)8` zmV)C=bP#;wguX=x+cwE)R600pFev`v7`9xsr)Y54X^Uc!GGVC8;q0REw8<^Hv&*$3 zBf1==kW6`)1@b}2g|IK2)Ej}FwmC7;uG&0d&T}Z|9LjmfH&o1WA?h56stV?K!?`mt z^+axVW=JX$kfGwAAWoo?iTj){O^HPp&F2}R^b4Rf9C|kGt4sVJe9u{aZsnfGnzcatN7hpm7TT8yJ5?-qUs4l_r* zh>EjQT@BkY-XW-YJLMQ4_aLQ zWN}$zmX&NgDFhV?)-ZwvhHQlyU|lM4r|}GUF+7J3gL}?PL1g1Y?BRj~1bPBISvb01 zwBbV*W26uVU~sZ6m#FMVmH>;yOQ^CFneXVYQAS{rwjB0(_akDw!`QWQMifFthx}}H zmV!0DnvYs;zFF}vfo9e5z#YC2t7 z-*mdX8I%{>%SicC8y;AYY&vsqp!)Rr4m=+&=kxq0zZdYi$i5dwqhQ=rE8C3gm^CEh~#&C7zH^}U$*dTrcyXkCX-?zxDeTi`c znzrfept@$1Z{$J~>Hcfe_2a4jTf&Ln545tat_2U|$A(XOm|l1Px$_-?j4=J-(Ws6d zcoctASAGPySbfh$sWr#LQfvPGO=?GDx_%NcPp0nfNNqUrWBuhQ zWSSBfa9zXcKGOor$Mn$y(ZHk!rrns@(B3z-!5AKx7_a2P;$DjcVOl^N9X2J9Jt(E& zOg|-Wj}i!fNYU#Iir3JBf2pE=q*Bw;qQtf+I{vRKM;^r=-~$HXf0;6}x;H+I)xsz) z8U)RFObv~goY<(N#=IK_t=Z@5xj&EEKo@d2twuP&C+nUbJu5UUw zs4k1_YuY1SYib8Uk7oxrq}Cj5Q}fqgfM-Tv-8qVevqoodpGMOUIM44Z-0c;utpn`U=9**O zS||vQtbr&>uZz4~Jeo16kTy{~vNZhK*=sbFFxP{dvDWL!JBP3ABV228(S_82`0Y*{ zIU1=qpW2U;)(u7$-D$WfOL+t|IxQG+uhYXe%G!-Wp|Yqaa))h_wPQVH zQF)%IqhXCxeiqdl)^yLHr}l{^y=Tx1`{K~HY!&%KvwT}4%(E%?{O3*jp)c7eUlYCV zkyl`2e5-xLfYqU8nwFhE*fi?U@unRmXg9VX8xK0#BpRc!;q;33#@y2@K4;6xX4LWy zSWSqNX3wAA?TvjoQ}qa648xYlA=yR<5xy~6V`O|oUOQbK^?XGXxMmJvu^`mpk`SCaBQHi z*GVNlftrcJwLCL*bnzvocJ3#{%;^qyFdJ8r=De5p=ZOuORI~d4g``Yl*h@v(Mdn;Vq zWEr`d_c<%u6siIeNdpJU>Ib>QHcB0SKFn106Gg+!Dj)e-R6C0Fpfd=oXBWpy?Y<4{ z>ouD(acy?3O8f?!R346cw8({J`qq#MwVBYC(do)JUB>55v-FDhi#Uh|9E@ON5VYwqsP!L^R{#fUsoub1e2 zpwskYH6OatVvoT!_8qvCu5n*u!?Hu!t0QOx?=+?^JEX0Sf>9hq$7)QCFWJq`w3EiM zA5FK$>`}}yntcjn=169aP}j11NE#?@H{?2=3iYbTgw-^CZgU+^$?o;Ut)}Z}8a3>N zP#<_#%7apc=^jw>PN4^gzSRLkJuUiIBV5cWxR}eM&e2#gYCsKm+0+#G|JZvQ_^67j zfBY`F3jvbcjSV(d)CHjpuZ@Bhc{2+eNNAw3MM-LeU|s~Dki;ZVumK`TDC=f1?US^& z1{*cBJ}vag(}qghpb6|^K)|3_d8iDj<;G_d7E;*(`}|ef;_VeLjEpliV|N z?wK=ZX3m^>pGyalAgcpbR)f zb2)r-G+xKKJkOurpV^owk4OgZOr@|t`yLYPanBThis+%PA8zJUBXIU z8Vmo1XwjaBZ$!i*4Y|ax4BwDQ|H|-;az1K(FU^11>zIuQ1=Tbd2w^P4p9e zZ_PX6`yk>~-=*-_W;{x3!Gvw>+ZOmXCc<|j7R1|_JoY|C|8M`iQ*du+ChkJSUf)|0 z?|^z6sl9J6jeHH}>AuL1$f1(_0GBqKF`~!oyJVXUe|Tv$Um|7AfDSP#ndS-MP9&7e zmS@2|&Qn=?88qk|_j#1NevXNG^WtsnGG9jFV*HO`N)dcw|F1M$59fHA8S3Gj(sn=? zf^rbR5abfLwygueN7|gO3y-{YCkF<>#ch`2T@4+L#TP7?X2S+bNb5;NW_cTL$^XFO zyMTm)leh7X{C11);njjaJJFSl0LfkY-BKIK?RY2RG=2n~xVs#i+W20*+UkdgG+4g(WE?oPTg~96IVNZXStL&Xx@CDn=j1IvgcO%^^ zuRCb*{WAYZuPlLUVG}{T^f90QtYn1Jr_w5VU?zGHu)xE=RJhWhTGebbNp2|NO z>^sD`{!0YA*Y+NX-vn{}JM8<%cX+)!21V|)kkJVQm5K@51+^*+-;xL!VsX<}*>~W8 z!52j;YIK5IsDo+tc|`6-d%~p6tlCCd)Fr0DChEJ>EewI%!{MG9a=A-6(o+L3VE~xo zXyBZltl*B;(Js7|)&xU4tH}z5yde=ps1@5pB}VPHw%;q_DzQexAb_&WDqdZ-Rak3b z4uGk$MvD%mt;U)QV*pH@wS6!h;ic^u`2l)30tBo~%o1>eO~Q@25^jkmM6ItSylwuV z?YXcWj!7eD=&&I3o4k#8r?laHWwWH;Tn-ou6A+Eq0!=_^^hFpTWVYyUcUl1(uqWa` z|1VhqQU_WA@&n&3g1?Ga)OU=*p~fDGN1{o`L<5?pz-c&JU^HSebiT?coGN9e$aFZw zX%KfQ$Ndg*Wts1CR8y=cOUIc1avx)TB#eb={5$r-I(W(%9jrtxV1j#NE=v%xCo&gM z1i64t*ZJYfMzmUrn;JSAiqFGF%mOb&>|w5U*$DFMvJt{Z8$tHcMlkb*jUYrk0;GJSaGv5_zBhJa{LWcs&=_)HVx%ShUoC|^_x!B(PqfS-osl*Q+wA-O<9 z;uvUq-P`yO{rR?o$j8l$PnrrZ#&9qB8w?njijRa{dP-Q0G#iGNH^X$u7s3+kgyskh zQO4t!oyOxS8V_YA?o}Z>)pz=!$SbrJBD;jGpxtAk$z)(A1>!3s;JcjED2g1S&Q3dy zD&9GJm3Wc-=Xj6V!rs32MXHF<2cjSLPP8XfQ~@2ovWzd0(9C=)KsP zjJ=I#Q18Z_YWw;pLWnUJv)EU8r7hze2=RSA2X17<yem>1#eTH~HyrZ> zk-47OgvpxqTv1C%JB({+bfatwJjKT@;nK#mS1=Rh`{2wj--TrjIG~RiE8V-}!nN^s zO@KIzP~poJKG)+&mq7HFZ=>YfVCjpNKAZGSkiJOiixEDDM+;}~p$yss5A|{i%+|PY#K>)#64J)G@qlykKo?`$mHM*F5gv2!bDzrMqSt ze;yn(^U=Z3#}JpJ=u>@ZWL*DOS3C3+oq zoN3@Y3}4S+;`P8-kx>$eAMox!mix?#yvI*EK=oFq4HK^hKh|dL8}G zVEzkXM|&L>$vUo3yyk{<;TTLhl0*w{k%@0M9d?soyda>F5RL<;gP&-C^oK>7YO57` zYdai1B#gw1Ld=w2&$wAzD!Ez^YJ=G^0-bOjuoq7M1qnc>5iK#C>POPUSm>j zr30A9&zYED6*d*425=tCw~La5JB#kxXH81n{Vl?7J#S({lyw;0yR%N7G10b(L}y)hb81S2@-lG-*znG@6Q{ zJVH6vMJY%1WH{meOe%&6YjcD=tDwf1O6zt%ruUH`f3 zbo>Y7KLr1_ZWi=$NB;;nE$Dh-L3g!{f>=mb+k=3nAyC`WxW6rGBQ|=ue~b===eEIc zV}G0V=K$_0*^s++NFjDnncLVf`-I*2#`>p`y|J!FF=}@-w%vqW3uPUFuhqdKNNIyy3D-RL<;HQ&w-Iy?={ z#`TX5j$D^A2ThG*(ylxlqyrxu(z5VA$aW`Yc48+rF1tC-{Q|>aS}7h^+`2++oW9-=87>gOGnRm~kb%okuxa{q{$t?MiU}mhn1~ zl-X%VW(OTZOP6(X$BajZMLE|ELsCRSX9vB{7I4%Iq{Z2lm>0NRsGW)7*~`{m6H|`@ z2uO_Y0&MEV)qU_k&1RNIuj|2yIKHam=)B`z$K;eloQT@M$S{?3?+IT@3T9+SC%@FXYz5=bsTA?H zUGWG8ZC3;-!zUw6jy+x#WEQPpF-ZE$C7QEIC(07A!dvh$Uyr z=5%>j0!~+HFg8Im{Zmo7edleZMSSZPXs25c`Ob%zHjuenWDZtRo8jBq0+Z;jWDdA0 z4+4R}C06D&=u524IGhDcnYBn_%B&3%Q(?`O7|eu7OqDfNVydjw5>sPMl$aW8IWTd> zt%*ykDV$e7`0d6?af_}^k zUqflP?`#CcTUZuv#IpD$TP5FPfQe;sP*-@GktX5D#TXQh5|8a&g=sfmm7A}k@k*|0 zH{~_rWV9!qU2Na^d02SepEq-M+OGIf`)7IT1rIE5d&m^a+cK7m;rb&`Bz-L&AwQ9; zwKyng^PQU~cp!EUrK7U3zzECNNSKRRQTh|1poG!ClF)|zV#Puq(M$h1>i6ulgBayt zfqVZoXv{o+=Auhn@w+#<+B+AxKQ?09J@;UN8`qWm9#l?O!?<0ec48Fu$w5a@1GaP; zYuVR8b`KiA02%DAUoTd%-@}4-#d>jG^4MypyKEba#_1~aP)b4}<@+fm;st|=)0kNG zzQ}xqs7_DB*RhV?EijYhed@uKvP!aZR|tR*WkMlc6&|LQSJzD1XRN{Sn5ODBeLj-q z3E2WMY0<_M(4W?v;M%~tsZ`xndLACk8@E7dP;FZQyhH9fQg#cI!*i|Pr|vpZaSPYJ z#li}faCUP^2NUUB*J-@4>Qz2oj!VPA@D41THgZXCcG|1BI1&KMgP7&HkX#NA#*(I zxz4BUe0iOB`jp81m?=$2IgI&^s!--AVy!Ww-z4&#(bvi78I_ElQHki$bWwtmP6_Iz z1PG)gXuK}#HJ>Z%C0}w@lNaJN`xfmyw9e~@-gyiX43ETYa$)$UP@$P-Opu~pM$RZZ zBS-aOcJ8;NO_Ug zD^wVc3rZ)*sHvR!FH5IWMHA9+HPm@Ki?@lAmdJvFOIANoWGX}M$U01KnJ71M5O}GY z4?#PkcfPmI+i2gpUC8}9>d-3``H&$F8ORb zcdhen-TC4=@7uRUHZk_HP+?+<8|}D?%vf{iQy2}mG$gWdbW20xmWISF49U1%o%ELxwgpk;5(bz%k!HAM?RL8K zM`QW#E)j;DJC$gBKDx8q^cS;Uxdy3s)?0iD(DQhP%sc%vQ*)AgH z3K285(?~&Y$vVrnVc$9mC0IcySs8qfWL@+vT=FU7yr5+0A!@`48yK^*8P@f-$iu7( zC8AW=x|G_Wn&@B3W&7orQfG(9RQbHyW~Vja!vF7jLe^r_hzR8_c!bOLZka=_$!mBu z%%mN8?_tp%P4->);ON&YaD_fVSA(y0-YfCNm(g$e0NXOWZ!aFQ9XBj3MdxhAki&L| z=X?FVZ!a9OJ(^w8Xm*RRu9OIfrJZ&(q_hKRxzA4!B>doEe0sGZ6tGyXKW?IS*htN8&1PcSH*DI zVJfs$4QJISu4Xt57+sr&%bucbi|8nio`1u|xtt}L#ytk;9Vd$Z6E+BNiR4oF-Q{>} z;5-H3#&U@i3%^hSx$&Sdd}A%~fMQ)`H@6u~#x{dOxl@7L3}Sqju*yQ=56(O8J8awK zefyyy2W%|UqcHLdhn$QCf3fi(w$}Uh-H}_OK}>zyk_+A^9$?R2^s+D7HnZF9bxc5S zUgd;uxvi4)-W@B3G}*j6?i$jL=R-r9?cN>nL(q!sSUlulw0F8=NK-V$Bunp(jL2iy z1EbU$EANHj_hI9M6ti=00&_$r+QXiuLyVBIAju;!Fi~5Ys4AYyYA%If+_Y>l)$gL( zo4~Qv1!FzW#zOhxAR#szDBI5k8vg|p>`kDq!VJox*~Vw#g9?ki379=F_a?Ncy$P$) zvw^f?lK|Eo^RPDoLn3m-x?^51dlP1rwj(kAdlL$6Vq*r^9Sd!nW#QXa)`GLx^5Gku zyurrwM)5D8Mz6I}}!2yAwg`HgZG;M*ce# zLbyXg=MDuB|ByQrR$Tigi0e0UDCoCCVWilh01Y<#qC$JclH+(J1WSZ0RcUw7{81y! zc2C^k%k2p?RYqARx)>}_sx=Bj*frWixI^RbLNkcfMwqAWZ72e4;E3H_X-6_xWQF6CJ}ue-@~>_Vce?bV?ZCip_y@B!XmYVn}rP6 zr4&tnA%@ZtdfMfCPc$x~1AcQNpto`4d@MaAq?`kXEjD2Ak))}<_iaafsC>ByLF;u4 zmMnS5`>k|iGHl#X(V{j~#KT%LZ&?a{h7AGz3>$siP|*$tH&o0jWzk5(1{5|_ zs62IGe*~1PSrjcM8fEru6xjgVmPkBo8;crnTO!|kVVR^VY!k&|+O~Fm4HddC4;w0A?HHfjNI=O*2i)C+8mwv}w$gx8&umF9*D4cP`07c797QO1S}PDcA}sMyt3aS~Esy?5nD zLWtY?C$RU}R(O^ixt}5y`zf%vyQLX%G`3#P{SI)+{S;JqY^SiJB49g(YR9pi!iwz< z*4AbB3m#%S1w)%E-S$H?w0_$u1b3jYpeoqIDYsMf$X@NHu(p2PpFP3|_JD0ev7176 zcx7i-c67zn-T~_|UFYR-rt5s+b2uD(1=n3;bU5tb^5<~iWNNHCQ^#a;eA-cmC&$tn zw2=F2$2Fj1jjzuhF-G7%s_UQA2yF`53D@&na9rAJaUHv3-DP5&$nm1wqy3gTy;3LU z{V={mw;mtTwq|};(`I_~>o{tJKlC2>@%8eObCl)mG$WxCtc`xGMA*@uE&oBrI--#t|Z8O%mFD9Pl&NziH)! zzY*1l&7=5gH-?etvD}mPN~RIf(6wG;j2P@iUU;?s zQ;;VmUhSHP0aLs-B5ew3Q%F0Jc7?PnWHgb{3KqH>W8t7o|)l|o+Z zzrhFO%VgGd?8J)-eYwn~RC14qiDER#jJPBs2G&$u?yl43XP{kwopQ+t7cd;v|lU8+BStynXrjOr>`c2I)|eRx+somMsk)OEVqZ_b#jd!44HYc5F?6rojPkIo$Lw+ zSNJA9jyc@P8M<`}|Hr2rQH5^@&orVE-j2jGV-qrs z2&`0rNB&lf@knBnIvXR0lD?4XvjGYngm9s^xObo8L6gX0_zJ`_f8%_UlfSR-l=qDwh`)`{LxjD6t1g5FLndtkT(OT9S~bVSGfVqAji{3+Y%xWb=M;#x6zYAy5_>*+Lc{bKaZl>!axRMU@agMl;Koeeff++`!ztdNBl0F=EPC!1m zaTJ9c28bzcgr&M*thpZs-ihnisLIeKs&WFLP+A(W)5tCwFQd#NCJj$HijSXl8BwnqQ7;*|O+slZYMkELEJW2bHMF1$6t$_TS-`rc76I#<+5w?6HM>aw$EMw+ zfTM0VN#Lm8t!R$I^BVs5;UC|x!UEc)J$cW-a~S_k_&k(Vn0Xn3BL!O~ zkDqu7+a-U1Y{M?etrscinTEDE;ed5%+xjW@D0JQLG1qbTJzHT)w79?Gp|OXH%dJu5 z)}J~xEz`(u7}tQ>gJaE^2}bsATt@z(!+3W;cr8 znssG&=5*uh&Xc%H-Pb4nk;_G3n@0m{{V}+T^UAHioR0uVMaeJ?ma+A(siv1V<;LTl zIU68<$&UMwFnPaaxAwc~`FkNOhW(=bURQrFTC$mXtovK8l3_*gc@|j`(mrx{QifSt z|F%H3<7~voeT+&ztlV=i(s!l-#MN-32EtVXm$9C^7vj=B#l7`t!ygl8u-p}o$+rCM z)mU%Cbnjzg%?n#7&~|?allzD1P-;)a*SH)9|DY$~|By|12?e98qJpj&Zi&JA-uODi zMpzGs8RaUz$2y>(`w&)t{3g7BqhkECg0f_MtqQRfke|uH$9{}8^I({8Z@RJmb`+n( zy;?86Y=a0D!`2~#jIu(^eSBz)e9ws5?Rx)1WBwj&L&Ke$X}gV`yKpZ}iO6I_U86K2 zrH*ZPU3tlkXvoXzWE-AR$F{q!S|RN)a%ko2*ml>|Dx_?~Q|e-fY*5G;HUY&uG3oh1 z-ZR+sGpWs3!<}24=;ucOcgGc9z!uWK;pm7d;yxkbyO;QZt+bC}Q!9iFkTtLsA-{?*-k~Q-iS}_Fji<5U#u0Q*eg$Oz6?{A(?PILtF3Uo;mJJhmBlGJ+ z^dBNvjIdnH@`yd6V75oI_(Gy9ZbY(@ce_eLYO$b35IKc_Ev%que&$BuM>81rNTaX; zldk0bm(%0C(}LMy^c;>EYqz5_hUww>#ZA@VLJ>mR3vR>#!-yEhf1nC+Qhl8vD0W0k z`49=S%^_+vHm%~jD#m)=M@?JG$`S9m2jc?VHv>J#rDj{9DzT_3p;+B8p!kkYj>HBX zjZb@tgAiO2SBv*9To{Xx4$luP5Mt2Q7$Hn(9Uw1C;BAPHldNJ`HGmW3&jPZGu^hqT z+#$no*_eo1Y=V6NW+&eL9PDP?1=OE{YXs60_>N`FFhJ>-%(wWnk z9v3M@m9k}pL^$fvIbr=mf20BlN4X}q@zFwDVMyeMXy@s_02av!A%lso3wd0Twi8Dna3!EJgXS!jfqDI~5BnLy$h%>2o489Ptm|G36} z%riRrC<)I*;F?wI#0=Ear|YDn>S?cZYOy>q&rV6Q{#2Y9vE9>k@NAnKhdp!9IBu7z+(8Hnf5>sQA zeJJWXK8+dg8D0E6Pf92hS1%?(VYN-}IKXS%a~fxDF|hKcG?{LX3!j{EsNXb=#y5rzB2_TyA6ZY*n~&Q*yjJ3HtQJvEzX zq1mT`LZux4kbNrRD@66i4z+9PE7574DO_{$b=m1+&%>|NHSXiK(hJzjeK4JUvX{(`*xz8it$n~> zY~*XTq5r^?p(6&1+Z;3NGj6Y=hydrVT^3Mo5g-O|JTu9F|OYpqWtBJVu6 zMZ=B#u7myqz&&Nv5&-NbY#RgFM7Un}SJql_-wxvN>z86@B@e8sWA@A!EEDmJ5sn98 z!{j2kWagQjRa6`6D2R>3y}RF-7tmq70106%N>0V#_!j%AqTg!f(M#;Cu0mkhSv~m! z^iGZN(ilY2Y9JS4aUGHFz&%(Fmya*v@WS!PJE$5(xLPL>KAIed=N{as6Z{Z{FxctH zi*!!mP~F}um}mV9NU?q!TZ&DP34Qy-dhWW@={jS~!T#S<*kQQn3J$u(Uf|gy9y;=j z@3a4kYs)(Db!{uh6K!=L-kmcp26yK;slsWmppTH?D&-Mh3?z7yPAZ)}-K5jP7j6#$ zZW45Fp@gjyP$=fvZ65W)6d<#vHTQd>XFk$743iYN2M3a2Hm13F8)dv=tmpmoxP&Gy zt(0t#&+*)ViH;q{WR0Qd8|=pP*|GBw@T_socH^cZ(4zuKfV)?C+YMSypH@v+;%mo} z68(0C=OsAAYq$f4_-KW%{QRlLRs z#U#@Hyf;q-ApxW-%|+?}>|vxXZN5Q(qA>zQPZ6Lc0fk=7yKs1a9P%K(<7M28&<@mID(!G1rRpt|N%`mPl+bdkLM#_G) zHWhT%*+wt8G&v-*lwBE11x96&`~)0w~Gq#+uJw;kB{1-o|em~k)C^kuoDcm9POpk zwS{KNAr3b8WZix?i_L(oehQ^9u+Ia`Fst!NrZIoYjgSNH%BHZ);V79x?SP|V3QHf3 zswvbDIBKR)58&7|g_>f7&u&1gph0tu@Vi@#@D04h+rYgNM);Cu2QKcTE%hXVuYrY9 z@or^1_kN7TZ1;B9*Cv?SY9UL+ZlpL@kW@Cvgx3O`7!MAm!Z=DwGNjGK3@UmJ3{|Sg zRbYlfkRq^}?nxM90X^S9P0u6S&=6p$rXAP{>z?(Ev*A@849y2x*ViRX2r5m24iO@vb}FfpNi=Yg48D65JrTtwD+ zc=vpKmjvK#Go?a(Dm-F7^Z@G9xNfn7{M8a8qH1zL-fe19I* zF?mS~ygTSB6-s6k7g~VthFY@0u%H#p@2(}qea)bK57jwGY$E{=TqXhYKJnT!7J4oR zqi#>U7GlG5B^V1#r>IhM+_*^w%_R)uDK^6lG4x%Hv_o4ZIIAH@(yJ)B2FfXSK_P2f zgw&gh1lXdn;BBR;0^C<**k)HV8?U`!ndK@w#g-X2S)5|aJO_R@%$OQR4JwDot2ZVh z7#(8UiV?m!hHS8pN&unPC`_48;altiyFmj0c5^!f0Nlc?g;-KZQ#-2#a$7k-?4(OZ z*)Lfi+)^=x<-oEkbrWmzRBOnD+c9j|enNX=l+|!j6}u?#3hA!~!2Tp!C8t|hAGfC* z9$b|{qv|eYFHUu?ghdK4Xr4k`nDJ>WXxy?hht(m)9h2znhM{b6I0W_zS)Q^VK_^5<(WNpp zc7O=5xk!L5Cgd(}HJm>*KBx(%I)SVWm;cSI*LU2Z*tW~V5Raqka3$uE>Fn~8D@z^9g^EJ{w9psMhb;(8O#)sjE(JJt6CXISt}wj?U_v|nb=l_ z8p>Sf9wU~#tL5tZFP5&Rjlu`khN&wHauI{PvY;HU1Xmlb6zIW3p-7}GLdCG$uJd*Q zln^6?Ow_p6cxa4DDy@|g$4n&dSwRdmU5rzjF2+gIMT|0CL>actELUN?V7uOj?GhBL zJ>u<;pnY4Sk#4M#!^Iq+W!aL7Ft9V2zOrp=MkW!-p={E}G(n#LEovrUC3BRoKf*}% zB>vAL+-7PU;Fcmnm=dm~oCdXMe!!C%&p;k3A_#>hf`-d{NiS#8c7fFubf|=Bo8_*! zOp_&&`aWj*^E-m^piL{eOg+TKke6x8u(LzZoYw^S0DW|6V4&ctp;E>J+r=OQMA|Hk zimkK(eke++urao~Vb7LSBiQV6@Yt3pu;CkGB=k2^M!47)2-3}V`LV+LH%v3N znGvv}DXpWT!@H#%Fm}?rMp-57kFqt$3QX3Xg3(BYIv`9U%2JH}+?CWwF71_^iX+lI zM_L`HJA{+%=KorACY`T-WW zaaK0ci4G1X+4L7?c_tr9a1VZQ_CS1}@W%Mg+XHfOMMcO532q%{9~mtt?u;Mk zzLb%whB4cA zoSD~Iir3Dvi_*MyR-8rpbEW^$$1#MEr~!&1N>KzWf3))3lz)QqM=F1e^iNYUOfzFh zRPUE7`XuRpl;d25N>wPHv{tA^3dKbc#aM~DMu>{t1l4$j6VJw)8=)$g^nW6*@mSF8 zNr&>I;yZTWV%I(iE@vOizB~Ka_=v=f*nOx~DCh0CcH+=g*G{}}Z|8Ao(Txf?F8zP| z+6hb;bN|t;kdSZzpx^fjdb$s#$NMLE9m>`BLGUURzw01)6Ut}aKfxOxc+UebLGioi zAqe9$-188EdmeOg&qIj3=RxV?2&kaLwIAa1aJJ)b@M_MW-B3w&Mmy2*Hzcd09Je8B z4%{P)y(hS)!|94x=XBw0IUm7%AcB;2Ho7ACh~xv8OUzwo=Zj&nlN$5tXL?dJORQ&l zhhtLnvYWAagE|13@{@eD^Kp)k^DjEfFELj0mXS4(3LkE3PL{>J|B<(_z|wU(UaoEx zH{RrUwZ-Y$-*}N6Z3;Vp3J8Fwg|W}Klsr-0PZRCAt~Cf;I>e9 zPdHC%o+p1AN%^ev@F$ZH^2K;X%YZ6Te}rb@uBgJ+~SG# zCbZqT2f4D56WTaTPI#jc?{S7(PT=}YaBrQ+0XGh*w~%YG*PP_ORD{gV+sa96A9%`1 z?onuku{c`Vf(?wV*~)Vqp3VvGUxKz(#~TMe+~~l^<=rcFbptvBwu08_iNUJukHkc7 z=Z)wNS7WSaZofXP`JNx%0dWQ7sPV9t12wk!J-KA=7Wskc%_8{JsdOgPRk;objwhJ6 z$YX2KAL%c%QantyT62ZJ!pg$MY>zcn_^YfeSIm-G6NSIVS}puF))@H3S@hpQEXlN# zJTVbEG8QX%Hct^Kv1Z>?yp8#TE81D-F{O>AcLOJzJPFa*q{unt%m~gYV`rkdCDGW~ z&^4c2QkshGx(H#+J7UZ`Zp=Fc@k&#XHxR3|1tUr$yrwi)_&1fxx#p76T!caXij9YV zGsOAx{81?Ol1hdY4hDsztdjiIbn!n<^aU{@qlQpy!s=oPbKMfg{HKJrCnRin8al?? z@YesQIAGoo-_aCbd>%cUU*Uv(yr+`Wnz%&Wke8_2%uCc=_zZCzHaRpJWe1zDs za1h#{SBj4XVIu`@ZpO_aYd0d-X1c2?IT^~6iqqZom7FidJSl6$pN3#c%U!mCd5 zl@ykc!cKRWiMTPHis{3Y5i!86~M={5AZL9XL-&DR5%CybXRF zK`WatNygpOD$dOsmQvg#Az?T_zJRR3;*=Ele;@GK)0M8&vzb2G=BZQVAk&!B^97fh&vfR{}oAoj`!R?I0*tvHd)Gw6kChwNOEy+7DOlrGV z>eq@JyX%(={;%|lyH~!_FHt=|r+zuz!bZizz7=nP?ZVbUVR*4JC9EoqCl)^N4GUqL zX=GuKZypRve9u{J8w~v#{2#*hnuZkDcu**Jy+FX6(y9Q^IGA=E$H++|TPYf5)BM7& zvlf9|)CO$%pz%$k@hv`&mh@(<&SDm=R-8;mdxP`CM27(FbTzmdi{I*pZ#$)-8<=Cg z|9ZAfvxDP4X_V&r2<&kEGXtUE4if9OuN*8AvX+;GRd#16V|G3(<+Iyz@cm> z?8b}2{>TbVvn(n!0>UbHS?2IMAu3G__oU*A2vJ@Lj&C7!*wx8%@a?j=6_O>0%OtVW z$a<4zr^`-P@_W0T_y3PX=Cu92D`_H4W^zG!4;_s?MDto8O%$n1$qI=re74@)djwL+y1JyZl(G}gI zx}u@7*H{utnb8#uRbA0g)fEkGKtS~MLQ}>2s!-Jx4OLyyPzoy=7Q7dy#{;^em>;Vj z&Ya+^CZvbIYB;MKRs*pZcvqxY3@jPWt^wYa4HrEeECvo|B?G2rII9^PI5m#yX@uWb zj@0q8SWF2vV-=$s3xEw+0IYSmXS2_{foa8bo`&f>BYb5GHd-y6iM^})(GNas%tOET zIH*2kAC{e2%ydm0@}Ug_Av(^XcQbLwN70BPQH1yKbR@Y!_7kaO&)zNIHqqNPWjZ`yk*#bT@&6e3KKlra~eH5;oDUFW_zZo!@Or$zJZA#W@O8ki+!a^_)O zd`lGLn}=*(`iwZHyX#_Fo;TvVnm*t`-faTuo$ByH3L~xyT(ASGc$(xby7Vq_JHBaC+Vp3e)do zEk+T$uy#o1co*AZ;8ePSbLz%ONib5pGOlfe^_qZHmemryTB5n2$W*HZ#dfkxUfHac z6Kmn+LLyfb!-J#fcM79=1UB)kFshg&5JvT^X;eSF%Ba%FlPSQ7 zj83D9Dk|p?XjNgIJFV)+)afBUf3qsoe8^7PRY3;3N}3mGSjipo2@6g+8I7=*m_0%i z)J0E7rI=n8*ZH48sV4%d?5w!lU)b4B?Gx*;)Mltut2GMj7<#5BYclqG2&`&;=k0>& zmpA;sAWZKDM(*g?Yv2GlTr^4eJOUM*EfF$7*<4WvkR`Hu3bSK&1zcX#NfrtYjXrWr z%r*xXzi%nt-O{nW_@c!qErKl4vd$d_dwP+kwVlz*lfbsZ-c2i2g<{D}f?0Rv$TRXx!`USMi}Ty zX@!>*QG?Zsh>=KGL87f3hhWz9W-l~Pv@Jh)T8MGJGYatc=D zW=va{pONp#%Uig@{>^XNhxUZe&MB}j&R$%wcwyG!hcnXQw$GXqKh$K^BkazHtfK;W z+ZX5C=iHr;U|&e?=^4xI6YWDs733_)FUVV*{Xo=%BSsBP$sRglY#K39Bf3YkI6FTh zuV8Ub_RQ}kc1iD#wJ#Kj`aHrTs=SPa>31z%n3XR@@ACc=sTY5y)Lm)FS(cHvFe_{3 z^h6nfpZ^PqAw452LuHSQz#s7i#Lz9LMd^0K{I6qRh6~l~%75wo9GnAmu?G&CmXr6u zg8TDw7N#Q^1&a%o7Gyx{poR;U7A(%nUy!?Sao(aOOBO)y6LKEN%E-@81SW3w?6GNS zT1NJ=F;iz|6bL;`$jE-6U{O@}I7iqg+i!`Ej@Gn)mP^!-e^OLJHv_6|XLiZD(7V!i zDsoq)2|)E8)=#dk(0<5&R^z&KyeoJY?w^I0by8-peaaMjPr2Ti6k#ps`)qpu(ikD~ z^|R^y5{nrAIsdNIQJ}2F|B;ckcu`JHx-)0#{aG1ZxzP1S;*^TQ#P*cR0O}FUjqIgM z=4Rvi$>(%h|{kChWCqhd z>(*rZZL}Pkwk#v9ASZ9)#09e#WG>FiS|BqLv!&U~a?%#k+UKJU)wF_)ye0kfmx})% zJk)E?chyockX{p3}WJyFfimtgNYiWPXTrzxpi$v;O=4 zz8LvuX64+!Fe@H?v)<%h@L(FtP9{e=yJTs<^UpuIY{On1Y|rZc8=i0P{s+wwO(7dR z{0h>>OwG&B%E(5`M)l0gfGJtBu;AP28TT)JU_tuQCAnh@mKSI_nYjgdV;AQy$jQyf zMkmf5ZO=rnDuasXr_WMF7A#!4T#14F7KUlTlAQDfre;Rv7o<;|c>luu#c2x`XQEe_ zv2e-cv_%W^M%%?3Iu{h=EnHlXFVIt`+P^v4ZiZ$a+@F(^H3AB$n47PnOozy)`0p*4 z5uXriAK8^rN~$hUknosI&H*OuOn#AX8+RAN(M8{McLV3)9ljjmukDz)pAXfn*rRJxTv+ zfX-xy&y~T~8*Ep4(Hjhok*s=xc88{Y)<1Rbv;Jei`?LPBjS-Re!9R|~X{!VN;qCTb zhaam78t#A4uos|5dQD?G{p&*aeDRO(>a>BWX6A!`elZPZI!WXI>&{=s&-d+j_LQD3 z{xY7=mdA|Kgx&M9XZ~I3w_G1IsR5Zg57N{Q?w$qj&m0c|&RbkqRaZ-*aCcsA?- zN(Z!KpHu|~uKZNyNx&jN9cy^}j_zQx{SaQPh_ zDS&ps62KV1O~e;;bPNNX4e%&n!_tn95KP!M0}cah0h|oj4mcMudKuCW7z4NoFct79 zU@qV}z;eK0*z;ujQAYfH z+FybnU<=?S;(vwkSj@_O0_g&M%%6}i@F)Hi`~bDTA>CN^YuA^LX!iU+Li|5%tF2WQzp zeI-Kg*tuu#zL!}?i9wSkXx4{<3i`lscL0P{$BebWhn6aAvG(x_tLjY}`ct1TJa?RZ0tcHfC(3 zVqN-3t$B|-=~5u)quEy&Z)FC$pvHyQuKz^CFn9UU=n>o+rhWm!MgM`x1~ zX4kQ+2W78HONnoPj8y6TSK({n)<0z-$?)qW|F;FmmH(@LUOb#J&07CZAuSR z4AI5<`aX}b<&$#m0o$XHvl)K<_l!sAn{90$r4T{-db|p=DK#^TCTMLRje0&BtgG-?jb9`gnrY>!_}Wzc5ZeN6~yCGO;`kAffuCtIO#m@R!j3`r9eev%NhRfS(Ke zCD5M)K9@9qf^j#MkM(|iUr~#Z3#{Z&OoFvRLe+N&eUcUbWpdYciq~PmLXU7MV5fZg z18k%;fiD9-P(GG_jgqe%svd^~{u&s_AI0?YX% z@CgBUmh*AooxoSat-p=JHOqM-5M{DtJyubsDD`gwxknW0UO~4{7q&H6pu(bM*kI~a z3L3AnfcVmZp9_2m@C|V5tNr400`XaTC5^SIT~e>mXVAg|A_r_)>O!_NiXu!Qd6*-j<^KN$EJxb^Sg<784->^j5MM6uI(#Z0qXfM$gkfe<m@NH)!%ZTWsPjE_2a4BGP<8Q=HPODF(bC{~5xxR^Q{mQw7+wX# zI735aVE&>z%C06lqqhKbuit{2MEH?k1iFniCchmh4U-?@xtiZtAU~U5I{4LrAN5Kv z_TvW?xtiZ+%SU_0@mLA+>!0?!ry@f?yQJ;1l%@W0JXeU_V*dKaa&rP-&;vdN_^bf@ zanKh4KOX6A@zaZc#U?>lMNy>cPk~F=B2jO`a&-)gw6Ij2r5P4&YVtoD8c0n}HT4+n z_zr|G4$(IP?9k``@0bZYg8T8b(ckZ$Uz@;pF7U&EZ(x0W24Z*B4dzH#qJ-@s~`G`NB9o_eF66AEECFDBw*2j9|L!H+uGSivdUl8Mv|QI9Z=x`I)9_#+5JP$>-+J=93P3C+O2bUj(<_0pl#qShO%)UH!`&@!D)C z7+GMiPPM&dI{Kn$@l9I#kzYV;LOhf=2K`z6_l!r@C!|l>mG$PF%VWA^)~z2PY%98X z{618d9W{Fd%AQ85_QGc7mSluJ8M*|XKI?ih+r>Lli#a6D8)iL#b{dJ&U#HO$wcEydUo9kv_KWvUM#hbxhZ3wh(<;`IdgBBYq6KMuhsYoI>!}1Uy?9^XG+wwWFj2^Cp~n zI0M!DtaQ&q0E$%Jx{B$3ymgfhwG`7F5c zc!+eLWcmtGKm5|ibn6&`A6N00a$z5VNX!K&P%gW^wvtz3#LhEg^95sGf)dsK#W=t z2Ze?&Q~JgrUq4r!1LGcJ23t|DoS5>h_zU`RM5rI@RSNKXfNy8A)Q{<&3%v6M=sE4l z3+x1cM!Lr!0At_Uwu(1JXC-^QIe&dyE-u zMY<=W7jpEka^8qI{+RB$z^DJMyPcsvWC34R-?Kgp!}~J4ufqEVxIfFD)PZjNHd8-o zcdoW4WYO`&&ny5<(J>m9{U8C|@YG5lnh6;W0ULzME@@x(QjkT7_!w|+~zqZ18zBP?Mv z;s4&PZi}|TV!r8k;zmp4VE9H@aoy0=nU7nvNBQ!(P70>gX6gG_5T|~jOe;l19Z&S_ z)VzPq{iMb6pOz1T$GzNNyWHRU(*Z$S1`K#%fVO)8-KRpWz&{aY{qL}#SHlMUBTPFX z+`l%ge=>qzHwL_AXlI0bqYbt7g>P7od?Tpi8v|U|2mS8)Z|%Qcd*gZt-Fp29IRA4n zU%rUv;GoNcN%E5$=>FRc*1z8nbnymaN^Ye4_ctO?XOfUk|1=pE>%IZ7roFu%WHCZ9 zom6iLZ?kBR^Z_USI-VF{(MM2I%TjgRF2r zA7rf=s8#56Kc!p$IzaogemmUfbn9;iXf7+=FIue?1GJ;op>RLhcj+trwQc>Zj||W* z_uFWJ=MViEcrUfFGwy3Gwn6ZJ%NnOm0Q{lk6CU4W}U_Qmk6!APg$Q+1GIN_ z>kHRtmv!rF*JzJg>HeMdoaG;e_GUk;=bE7323s4$gUb3_+l`=C`dfiV+5YAlZR3DQ zxZfUN1&jY02&R7;I1A3-gp58PrX3Es74GUV>#;EHCF<%1+koHNg7({zi^H|k;dEcR zW{U+hz2&`sIr+Q8EWx9!^Rz z^{)d4cn1W%Ghq0m1GQf;jZY1*rvHNg|D^zb zKQ8e2(T4~4ZwT;@4)9M5@W%!ClLGwrE5BTsM8r-Jy@u@t;ukkmN{%@(@e`L%z=iR? z_@$~rJZ7H7?{Ve8%9i+TRs6A%D1Ml>6+cc-^B0*W^NaKK{KY9j%$XyuMet|N74Tb2 z{F!*$c1tR0$Xn(63(DU$qb2eGe|aV; zJq}FoSfw9R6`Z5sd73l@i>j;8+ExDmX{M z`3hz$xJtqG3O=de9~69E!F>w8q2Or+uP7KcL&dM)SOuplI7h+x3T7*~O2PFCKB?dz z6ntL6eG0yz;AsV~C>SX$7w+ z7#64ES8%L?Qx%+};CuzM6#s_i&yb0I99=_3eHh* zzJl2bu2OKlf=?>=2L+#3aG!#2D0o`ID+-3qQt>M|R>7$X&QWl_g4qhLQgFS3Pb&Ba z1)o=NpMq~Fcv`_L3Wj}G#joI41*a-FN5T0DW-GW#!SxD0so)X$7w+ z7?z;oS8%L?Qx%+}AgN3nPIu}9(ymWhn*ATyIX}vt^nc+D@uYFL-zxqmF~;ukw|r^w zf9JMIJ;a~7=u1fdt>dF7@xO=oTfUU|Z@XpEt@!UD{?x@^M*45RW74ho?;-w{FD3r( zjGuHX{(Fc&^}#PA{olDA@!#H4{4HNf{1YZjx)uLDrT;&^jPy@H`QyKb_*=e|`0qgd z!+#I)r)GT_>AwT@5C1*H-}2?ee=Fj@)r_Au*un>%GpA3VXpfpX=k5{q@ngr2{f_|mu3}YExvyu{+DMj%E>RV`(u}9FCgyr z@#)>r{}yrnvL#sy(ii9F2C`V3mllZn*G2e$ovSuBf5no5h4%v%5bm4-UjIj@7 zFCBaT(#2WnV-}}tV;3#VU!;vqUy+Sa5*FlXf(G|!<>Tg)Zk`2rmzR;XkO7n{H>*G! z%S+6(u>~2pmtt%tJb0Irj!RCou^EdNWaceglCfY>I>MShFilHakg+^1Be!4ygv`p& z#_|f5e25^ygJ~cq3&sa-LbioV7N;R-4r0M$tcY4hb3blT8Jm`~gqJ@Eh5TPXt_O#H z_&57F@?g*(PkV z$pEBh9ARE0Ogo8x{Hn&W;melxt8{s#lXoAU@JH0K={egOUh!&8lz z#J>0!KWBGn#-p$z4W^rrlMnvkBUN}4#+VVRK+FUFRIzFg!|fkFRrR|~*!r3v?|hiy zOhgeJ{^8Ae9TQ^jTeqJXzlmH62g95BpQ`4COlS{aU^fFQxE|0yd{H!gn2%A=YcdvC zsRPO>Koy?;;hp1TcoQB~^Jr$6K>2?Y5Z*pphBskBK!5zd1jY zq{7=(`I-4<#&4$Ut$_HK-#-KxOw1{sP53Te`RAWGZy1>-!$+GDD;8!r z6P^hOAE(BLaSzDwHYU#R7r;*AG%$IY=}%oUL{KL!QQ-sh*Dl_6Jubp2DZ6lVnc;a9 z+W!~e)wuuNjN6~CC0kMk1q2B2ins$r6l@m#NK#JV=2HA_z<(kB&3rTMT7Teriog0l zW%$UEAn^MI#_#v)DrXR+Guu)BmqD;LX8YJ3AEcS>UU$5%neAD3yj3&XsqXl`o%6xn z@%=QjUFnVw*39;zJ07{?^v6hd5Lyz`zWcuoLQ8Ji?e6#x&9uke@u8Y&XS?IWI_+n7 zyrG$Pu{++Tnf9(b9@=HvGrt!>+BKSKr~J^+gRg0({ppU6&`i719Y07j?L~L|wW$9? z{87RkgxP+xe*3=+!fd`-Z@c5K)6DwV9j_K2{E*bkK0V|=>zA&5(BVfaoU9fW*1mxJ zp}Jdo$RD-?mLBql)5(mt$Zx(3DD7TB-zC3UZ`fb4XxC|rl$|l_<8t81N1F|Y->*0M zOF%>UqXXpsslXfBVAT$qtbQ%$C~kvV2W@x&R5Z zV0FnpN~ZLWictpfmM;DLxx&97=&#d?R6dw|-VpdMeK745garS{&iTho#b>a>Hz~Vs z(oa=*bH31||DnRisd+S$zR={a@FxA^CjS8Xznc65=>Kl=SMyXR|MMn)g*W*}!1yv< zO#$?`E4(@1Wzv6N;p5aikjbC-#)$k;c$1#@f(ZTr^nWz@2hi^``Kx&rlRxj55&RY2 zrDOu^uIItE4)d+lX&Rc*4a=b z{BUea{DLq@BfeST&2`5Zg>O}ODV5w>PWq5Jk`Io>iQf~7-mdU$KlyuG;YSDHL;A{i zCj{UV72X+uuTuC#g@0PI*RIDPmGYbmpnq85weLwe&2`P^6h1=XO*z|vXMH!<#ceW} zc9Hb3OOsW-R%k6428!|86(6&_A{Bmu!kg>L%M?B?0N)=1kWW$oel+p08!e+`jH^_7 zZw0<5`(x7IH%8K%-4oOPXbOLy;xkFbyGX^mSm7TP@kVNi%Kn(?eM-?EQ1s^Tc1kqTcGfS<4M&j#REDtw*7)9&;4q{25Tyj!9*vm873H7LAU zj)^%#1jbC)dGP5eKfjJ9hxwm6R#Ms(pRvRvU32e}cthduSNQY*yeW^lzam=EoARUv z$n$gX=}Dfal{`h?mhqb743odPZzGU@dI0~ez3_is@ox#>f0}rtw;&*0LqjCKBmi&9 zW9}ablxI|{*#5P>l}md6!+ z%xp<0?pJ~HE>s-yv77m<_^eWRZHGj(NOjkKr||7r5^v`J060XvO0Ud*<1pYwzU4@I zw$uDAQ26HerN34JEl=Sa0`j>M_@454o1$;|jbs3?_?>`*@>DB*eq7;sg^pdh{lPW8 z;J*ocPw`Ic1%Eg2OmE^k$vsy^@{Gbaye$znv}gS75qt(}?T<-+3;JRFomKd1C3vd9 zqK-nCp5z(S3;xz#@UsPepw{A*jAK;!mJ0mU<3V>Xd^YxiZ|DX84)DyM+JN%<20Adr z$1IofG^q5ZDtzvbB*HA;4GP~JpeHW^k0`GC9q9#sTHs+^gJlMo`S6X1p84=-VNd)g z_JW_;3w~iQ_=g1^aOy%Eb+QPwCyK=o=KhUGX_B=wT-Z$aqcu*W;vTPkg@B3*I5{1GR={B;y9f zKdl$~b-**di2?FI*9-k6Mc<(6shMxd$S2yv>IY?nrk&gfyvR>w$4vTz3U6N`>0^|f z20Egoj|q@x8t~+wxLp#~D*A^7UU~e$E4|>`dcl9H_-jfYQ_h*_C{mts)y~$c`u+>x zMS4{}+g05EE%1JVDCZ8r=W6_$ijOTI-U*lr>8af32|V=kmy#FnQR45{z{B)i_4_06 zl;0kZ|L+U>NNsMwIQ^pHZ@0>XCMu4%V?m94a#gv+D10XHJ<0hn@N7ro0@}4oK@UEP zf4j=hM#aB+y<}`x{NLz>|CL_wH(~;)r+l~#cu|guzgfPE6uv#6K0X3`PwCnw@aRAE zlY&<(IWP1=|20g|iu_mYRH~w{2EHf$hkC(}#Z5tikFr<1_m95};CtdzsOZaoE9rPY z5P!`IZwt_ek4^d?NxEtYwDC9f%)g)){9k*)zuXJ{6X2Pymd!Fk-m}Bsa699L-tH7$ z&Ag}OE`k3)&0R}xBUu$Tg3WBu@B%jLIsNjsH{JBedl?xD%%PNiJ#RAReK zRoR`4STv8tAa=81hmhE@LP!WCHZ4{x`v)L2YY@^bK(pmL_fb`@x)s~qX^AY7_|&a) z&%Nh;Zk3OKrGNhtaKcl%6;S&AOZxfJH}v#>4e+;7KYWsj+rJAq;dv?PQF78Pz(eIJ z-Twbn;`vni3x(%T1>@iTrXJ5l0sLnL@LvE<^nWPr$?wHBlwR=h(7U(3z#%Ur{NDw5 zzWk1!p7#pidw|otgZ~P|^!+CEGneq^{}P}1_rKuZUlri_V*&j00{Gqo!N-NPGh!C_ zYru7U`99!8=ch9M;oq-|n|>_el^cOU@#}9AzL)WL?|w^<|0f&{e{m-9z|YYSzFR{} z&sV;!hkvgCZUIj8?@9UKqhza@k zuHxT66pa5f;6&$V8N2jPKK`xTx!7j~cs?(HmmGBOV^3%P9DU;KASyXQ6h_O*r0kR! z>yFNb?4)On7{jN7wNH<_sY_#@J>@-o7=NIMH^9O3LFf;Io-gHQt==io1?G)!@Tp;x z#8XOWuQR(<^O*euKF#59|I}pa6HTN3PiEmyFY#GkwQUmh|6z<>&lW58ksJ7L;Q47* z!3+-`8>KgZ`@w?~>tMfBu9h3s2CMRgILx`k zEok;TjNw6C=G@#EEym{5;thQV(O}SqSLzM0ZB`qbFaZSw$85m%${05skjz3?G z{N6mW!kE}LZ`=oqvF(Hy7aOCHls`06O9$0e_5tt9$-rqBG;_mwP+hkU`cVm#y8ESf z>^x<{6v`rlqo^vnA5tR7-d(*z3x=O-x7tB_b;BmhxkKGXsR3zZ&%trtdz2!>@(HVY z)5>J(+Y!Y>Zd9(d8(>DYN?|RvRh}#nI4=VKf&tgKH#SEsVd?<3HRe^pJz;NBbPus$ z>doB(QtFjb7!n0L(cdhhAGmg=ANSlr|G^LN9>JsxqkD717I?G4pQGn`bjwxmThuJTWP|7>7Q)wC7{IoFG#T8TKFiu)lxW>tgeTJx+^hEE6QC#q80M zs>ip<4jUcV6LcMBqXmPiQomXsu9hPUoAv_TLC5Cc*qQC^9i8+ZcAq?9wQ?Pot*BaI zEV>N*XAJE|(OVm4Dewb~%?|Bb{xmel8L$X7=>;1*axU2%Z-21*a&PYgb7=L>`mea0 z8kRG55z#0!oRE(eUM{Op%0b^7;puorRbdXcPz7-<23Eg0Ylq9q141JiPY;0nC zQ+X*?>1uPyMJlDb4vZP;Zs-j=r%-5^Z*~Y?l2l(oiLb6oXakOy*cifdn*}bnk$ZbE zo%;`a%qZ7DBW-`3CrBDn6A`JCjBD*vF2IRB$6Xfvm3WvaK$9y#Zt9%S-$_Cu$viO2 z-ON27(i!-o3^&jo+MR+;2rFmqEOaAmq{iNmD^HmwVQ;{c_1m2V73%owm<#{e1c7Pt zf&3dJ1{;t3Da?wCz!PGc8o0Y4S~#+M3ZJFiz>eK`QDz5p5yK|?bw?yo5cFX6xgr4H zk!(<%$Xw3thU9gvXo~D9SHpmVN9H0EFyx}>7T$A4$aWe`u{<$6IcZa2w+xVU&M6}S zq?F{GVg^%zrOy+BuFhGv6Kk5pqzmMgu%Mwd_^}n|DoP=KMocuUQl>n+Si5j~id7p&ZUp~5ga2BIDpEJV z|HvE7L5p44L15nwF1ax6UJ>Jb&TfffLt*55htVIWjQ)W%`ZJ2$1Pg;ummG6ROA9@G zF&BDX!!xnzLlpE7sX_s$qL-JXFfU*e6dKm8!#xI6V&7DLtu}1eDwLzLRPOWJUdW)B z^(qUOi-jNHAO*T4tuihE9t8*zr2T^nMT+&&n_J@msjhwDA&v7yL}Ym_MD~=Xngd5g z0+$krWuK3QLfF5VvdOL{rK$W5oyAC(r6*DxxfmlJfLm^{^5@}W*c454#S##!7N#p7+Z1+;#qp6guH0p4b`{WN# zc)ca@kL%L7>q&>Wdn=K}NsN!mvo_XIJl&2rdErMGJ@Z$JK`H|iQjH=MLyjg&*khdb z2FMRw*yyBKR8W-E6#}tfP1-;yoi1%4le0-+>y$fcw?+BTU6`_aJ$UW72M&}Uw))o$ zM|orrQv1a4RVVVv2(q-z@Y-8E3)hXTxP8NZqo~)IOqbzhe%aWrnSJeoP1+Me8HdW? zheti&2+tJ?4?Hy)LlQh4Z-}0@JMF3aFcIWAHrWiZJ#pcZa{aZ!<#g=cEclAH4Oo*? zSQ5>Gj3(_YRjp8vvH%y@d^w|PEb4hkSQE7nX-;0-U{@?ENe#WM4bv$SMGyrQ8ciuD ze++oBFq&8B~*G5uhvu*P)nXuEAte;`M01rMGug5nvOzswI}4GO>7**n%avQo z2WZSn2Cu@A1HsXTbD62b0WHqY#VN)GimANpl*N#^;Hm)^e0Fh#gGKI$&8bUY8D$O- zWnB3QoXYq{6S4}YvYzg6%Pb!)6Y5Huo6=ZL?gi?#b1lIn6*~n%dex65dKbT>p2OhU zoE6TTYUIlZ6?qhsOs--iBC^C>{$*TbDkyT-u###j_kV^@%ZZEn6IIlwDzXKg)*Sg| zWN7{@TTwpmh$k|Tz9RK~Jb&gv{DyA0oRPKZc5#G82P;Cu@B48KjyM#tYNmF{J=25_ zpzw8@}o==-T~EfQA*->V@VmO KOxM<&^8WxGetPrimitiveArrayCritical #define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical -#define JNI_RELEASE_MODE JNI_ABORT +#define JNI_RELEASE_MODE 0 #define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical #define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical @@ -384,7 +384,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai //Release read arrays first for(int i=readBasesArrayVector.size()-1;i>=0;--i)//note the order - reverse of GET { - for(int j=readBasesArrayVector.size()-1;j>=0;--j) + for(int j=readBasesArrayVector[i].size()-1;j>=0;--j) RELEASE_BYTE_ARRAY_ELEMENTS(readBasesArrayVector[i][j].first, readBasesArrayVector[i][j].second, JNI_RELEASE_MODE); readBasesArrayVector[i].clear(); } From 0170d4f3d5d4f8d182074fb968302c3e5da4163a Mon Sep 17 00:00:00 2001 From: mozdal Date: Tue, 21 Jan 2014 09:30:15 -0800 Subject: [PATCH 15/72] Got rid of the MMX instructions in the SSE version of the code. Handling the mask operations in a class, which is defined for each version of SSE and AVX implementations separately. --- PairHMM_JNI/avx_function_instantiations.cc | 1 + PairHMM_JNI/define-double.h | 30 +++++++++++++++++ PairHMM_JNI/define-float.h | 29 ++++++++++++++++ PairHMM_JNI/define-sse-double.h | 31 +++++++++++++++-- PairHMM_JNI/define-sse-float.h | 30 +++++++++++++++-- PairHMM_JNI/pairhmm-1-base.cc | 39 ++++++++++++++++++++-- PairHMM_JNI/pairhmm-template-kernel.cc | 37 +++++++++----------- PairHMM_JNI/utils.cc | 3 +- PairHMM_JNI/vector_defs.h | 1 + 9 files changed, 169 insertions(+), 32 deletions(-) diff --git a/PairHMM_JNI/avx_function_instantiations.cc b/PairHMM_JNI/avx_function_instantiations.cc index b236ddc8f..4118fc5cf 100644 --- a/PairHMM_JNI/avx_function_instantiations.cc +++ b/PairHMM_JNI/avx_function_instantiations.cc @@ -16,3 +16,4 @@ template double compute_full_prob_avxd(testcase* tc, double* nextlog); template float compute_full_prob_avxs(testcase* tc, float* nextlog); + diff --git a/PairHMM_JNI/define-double.h b/PairHMM_JNI/define-double.h index 79c6b323f..502b919fe 100644 --- a/PairHMM_JNI/define-double.h +++ b/PairHMM_JNI/define-double.h @@ -47,6 +47,7 @@ #undef MASK_ALL_ONES #undef COMPARE_VECS(__v1, __v2) #undef _256_INT_TYPE + #undef BITMASK_VEC #endif #define PRECISION d @@ -156,3 +157,32 @@ } \ } \ } + +class BitMaskVec_double { + + MASK_VEC low_, high_ ; + _256_TYPE combined_ ; + +public: + + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } + + inline const _256_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_double diff --git a/PairHMM_JNI/define-float.h b/PairHMM_JNI/define-float.h index 25ebd1489..3cc57ec38 100644 --- a/PairHMM_JNI/define-float.h +++ b/PairHMM_JNI/define-float.h @@ -47,6 +47,7 @@ #undef MASK_ALL_ONES #undef COMPARE_VECS(__v1, __v2) #undef _256_INT_TYPE + #undef BITMASK_VEC #endif #define PRECISION s @@ -157,3 +158,31 @@ } \ } +class BitMaskVec_float { + + MASK_VEC low_, high_ ; + _256_TYPE combined_ ; + +public: + + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } + + inline const _256_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_float diff --git a/PairHMM_JNI/define-sse-double.h b/PairHMM_JNI/define-sse-double.h index e48325ba9..a30b2e5f5 100644 --- a/PairHMM_JNI/define-sse-double.h +++ b/PairHMM_JNI/define-sse-double.h @@ -47,7 +47,7 @@ #undef MASK_ALL_ONES #undef COMPARE_VECS(__v1, __v2) #undef _256_INT_TYPE - + #undef BITMASK_VEC #endif #define SSE @@ -69,7 +69,7 @@ #define HAP_TYPE __m128i #define MASK_TYPE uint64_t #define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFFL -#define MASK_VEC MaskVec_D128 +#define MASK_VEC MaskVec_D #define VEC_EXTRACT_UNIT(__v1, __im) \ _mm_extract_epi64(__v1, __im) @@ -123,6 +123,31 @@ __vdst = _mm_castsi128_pd(_mm_set_epi64(__vsHigh, __vsLow)) #define VEC_SHIFT_LEFT_1BIT(__vs) \ - __vs = _mm_slli_si64(__vs, 1) + __vs = _mm_slli_epi64(__vs, 1) +class BitMaskVec_sse_double { + + MASK_VEC combined_ ; + +public: + + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } + + inline const _256_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_sse_double + diff --git a/PairHMM_JNI/define-sse-float.h b/PairHMM_JNI/define-sse-float.h index f5758c74a..6612b28e6 100644 --- a/PairHMM_JNI/define-sse-float.h +++ b/PairHMM_JNI/define-sse-float.h @@ -47,7 +47,7 @@ #undef MASK_ALL_ONES #undef COMPARE_VECS(__v1, __v2) #undef _256_INT_TYPE - + #undef BITMASK_VEC #endif #define SSE @@ -69,7 +69,7 @@ #define HAP_TYPE UNION_TYPE #define MASK_TYPE uint32_t #define MASK_ALL_ONES 0xFFFFFFFF -#define MASK_VEC MaskVec_F128 +#define MASK_VEC MaskVec_F #define VEC_EXTRACT_UNIT(__v1, __im) \ _mm_extract_epi32(__v1, __im) @@ -123,5 +123,29 @@ __vdst = _mm_cvtpi32x2_ps(__vsLow, __vsHigh) #define VEC_SHIFT_LEFT_1BIT(__vs) \ - __vs = _mm_slli_pi32(__vs, 1) + __vs = _mm_slli_epi32(__vs, 1) +class BitMaskVec_sse_float { + + MASK_VEC combined_ ; + +public: + + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } + + inline const _256_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_sse_float diff --git a/PairHMM_JNI/pairhmm-1-base.cc b/PairHMM_JNI/pairhmm-1-base.cc index 3030b640e..3941c92de 100644 --- a/PairHMM_JNI/pairhmm-1-base.cc +++ b/PairHMM_JNI/pairhmm-1-base.cc @@ -19,6 +19,12 @@ LoadTimeInitializer g_load_time_initializer; #define BATCH_SIZE 10000 #define RUN_HYBRID +double getCurrClk() { + struct timeval tv ; + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} + int main(int argc, char** argv) { if(argc < 2) @@ -29,6 +35,10 @@ int main(int argc, char** argv) bool use_old_read_testcase = false; if(argc >= 3 && string(argv[2]) == "1") use_old_read_testcase = true; + unsigned chunk_size = 100; + if(argc >= 4) + chunk_size = strtol(argv[3],0,10); + initialize_function_pointers(); @@ -45,12 +55,24 @@ int main(int argc, char** argv) assert(ifptr.is_open()); } + vector tc_vector; + tc_vector.clear(); testcase tc; while(1) { int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); if(break_value < 0) break; + tc_vector.push_back(tc); + } + vector results_vec; + results_vec.clear(); + results_vec.resize(tc_vector.size()); + double start_time = getCurrClk(); +#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) + for(unsigned i=0;i(&tc); baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); - cout << std::scientific << baseline_result << " "< 1e-5 && rel_error > 1e-5) + cout << std::scientific << baseline_result << " "< NUMBER GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_,SIM GEN_INTRINSIC(GEN_INTRINSIC(init_masks_for_row_,SIMD_TYPE), PRECISION)(*tc, rsArr, lastMaskShiftOut, i*AVX_LENGTH+1, AVX_LENGTH) ; #endif // Since there are no shift intrinsics in AVX, keep the masks in 2 SSE vectors - MASK_VEC currMaskVecLow ; // corresponding to lower half - MASK_VEC currMaskVecHigh ; // corresponding to upper half + + BITMASK_VEC bitMaskVec ; for (int d=1;d NUMBER GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_,SIM sumX = VEC_SET1_VAL(zero); // Since there are no shift intrinsics in AVX, keep the masks in 2 SSE vectors - MASK_VEC currMaskVecLow ; // corresponding to lower half - MASK_VEC currMaskVecHigh ; // corresponding to upper half + BITMASK_VEC bitMaskVec ; for (int d=1;d; diff --git a/PairHMM_JNI/vector_defs.h b/PairHMM_JNI/vector_defs.h index c89f7f932..6574f4a76 100644 --- a/PairHMM_JNI/vector_defs.h +++ b/PairHMM_JNI/vector_defs.h @@ -17,6 +17,7 @@ #define SIMD_TYPE sse #define SIMD_TYPE_SSE + #include "define-sse-float.h" #include "vector_function_prototypes.h" From 88c08e78e77ecc52eb1af08c670de7b81ed06b5b Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Tue, 21 Jan 2014 09:57:14 -0800 Subject: [PATCH 16/72] 1. Inserted #define in sandbox pairhmm-template-main.cc 2. Wrapped _mm_empty() with ifdef SIMD_TYPE_SSE 3. OpenMP disabled 4. Added code for initializing PairHMM's data inside initializePairHMM - not used yet --- PairHMM_JNI/libJNILoglessPairHMM.so | Bin 102085 -> 102053 bytes PairHMM_JNI/pairhmm-1-base.cc | 76 ++++++++++-------- PairHMM_JNI/pairhmm-template-kernel.cc | 4 +- PairHMM_JNI/pairhmm-template-main.cc | 12 ++- PairHMM_JNI/utils.cc | 1 + PairHMM_JNI/vector_defs.h | 3 + PairHMM_JNI/vector_function_prototypes.h | 20 ++--- .../PairHMMLikelihoodCalculationEngine.java | 2 +- .../utils/pairhmm/JNILoglessPairHMM.java | 47 +++++++++++ .../sting/utils/pairhmm/PairHMM.java | 12 +++ 10 files changed, 126 insertions(+), 51 deletions(-) diff --git a/PairHMM_JNI/libJNILoglessPairHMM.so b/PairHMM_JNI/libJNILoglessPairHMM.so index 053b901cd7d16182b21054593096ab2ad1c2a0b2..e18552b6b9147e44fc72618991b21b560945e537 100755 GIT binary patch delta 12981 zcmZvD4O~>!*8bTeDAY!Ahs3?AAn)p>}dO^io2qd

|cY1a&#|FiZP>gfBP-%r+go@ej1_gQQ2 zwa<*;+|ho|j`rn!S@@Rdq}bgSCehVqCCMZ(vF-71uTS4=i4gqWEnE5+q!gu-l%kN* zmd|GxEWH%+Ps@j}c_F3IeMXP1Ss!`u`$$vTz5N5vIxhceKg6{DtkK@kK8&$k{%waB z@G~P|AS>kS0%ov!ektGumd~>T-(dlqb^M08__yNcfrnmTl090xSktv=tTMEpn3W{0 z3zMW^=^@GU8$Z}-jB)-rMGo-EeSNZp8-k87v;A~X0b?)o48suyDtN3>`cY}=2~zCH zP)X_qe`dZUcm^xCpAJ?!gp9=uUJps?#%%nwIn4N>FD`8ZdA2J3>p6=J8+Ysh)pbQt zT@Ud3KNSAX1AH5n_{`WaW57t|p(rm8DILU5J>LQ~T&0GnR1H%Uabf%%*c26O!M;!d<)oAu;D7!%p1V21lvQ! zTKEmHd%^Znu~r_DfcS%rRI%6`U;`4gx;`p4pRWTu7;LnPE#wEl&Ida{#g_9+V7GxC zsA4O*ArbKh8>?b%JO%6>u<L#mL#2f5juurL2#2f5tXWyoVs(U}vaU z#2f4#u(MPw;tjU%5Up;uibcG^W)0CA=c?Go(SOBz9$z~o%s5}AdQ|E&ylP08Zh=Zg zyoVy*7-yl1MZCd=57p|PQ?ZCQ*i^7fR4n2Rb|u(lDi-kuyBF*WDi-k`hIoTrrD73p zumQugx))U};th5%*tIGa@di5|>^c>Tc!S*rc7uvVyoV#+U|&+Ph&R|fU^l5)#2aki z;aXj(ibcG^W)0UGUs17*WB!WwX1;cKn6X@?dQ|EbUNt;S$5krgos4*6oYzz=;te)D zS*zQoVi9k!sbK%1Vi9k!E5Ytiv4}U=yj^gs zynwKQuuI@sgz21c*9$y_aA(3cfhQ5Bv)NrKa2jE}XdnUtkMJu1K}U+aP!L0@0bX4Y z0fFNP)8XRI6}TT^BVnt+y$E+DY!SFCVLJBRW`TnVcOz^PxFg~2g!KYz3W%U{$}I`v zZaZLlLvVXqDFQbMn+P`we3dZm2DeM#Cc<>0x$6Z!PxukSHi6F&rnAjmDe!T^QH09{ zF8G9q-b54%;xJ)4_T2dbA0*tDaIV1n2-5-RwhFw9a6iHpf!`qfC}Fd}uM+N0*d*{K z!gN%+^@Ize))Ns!gd`fQCQPTM+tVV}0AVxXMu8U)rt{P75_lHjIKuS;Pazyn*e38K z!h;A`3YA93q^yWgp&y83mivyFyUN*`w<>O*eY-@!gL(FEdqBX zJdChe;9$bT37Z7&NI03Wz6IlZqclW3Mua39+|>Zn8SVD`A=Ut43*knAuM$on>=L+% z@Cd^70-q;Lr@7lE@EO9R2v-VxobYJE<$qv&Z`3D5&_VAm6b%j&9!of1;Ddxy3FivD zk1)LgxUB;3BAiCpBJdl8>1Due7Wh@d>4Z%JZz4RNu)aVL>xq~^ge354!V?L5?$QL8 z5~eo>ccZ`y2xky>2|SDNB*OIqPa!;+uub4egfj_O3Yfm<&K4H46+ zfh6!<225`)ZqM%&ft!SL2{#ISmGDf$E`gf})BB9OUf}bD^9b7nK0|mm;Yxvz6P|iuE6^UKSS6m@Gim&2wMbxgYdJ2%>utlcp+hv zz?%p!!uVJOf>=+)bJRc*cs1e0ggtk}8X%lcxKZE*gz3G=?Gktv;iZJ@1)f598DX2i zlL#*-Tq$rG;T0Gk5fH=(BAzE)DDY6i^fKkn2Tt8^Eg-eT|6+?tO5Iq$T;p|#Q@gzF zda^KiFgi%7#s5jkoV;RI>W1WVXe%w(rj{kYaMvU?Z^F_}E$coN_vvL(=`irebbQ;G zo~hTR2r0!I;sDcx+teZFmj35c|5LbSQ&Vz2KRP$4XAx#XuL{Wl=qWvkah7*Q^I^UT zb;2;e{BHM-S;sV&@7mX=&S5NpH;;QBaX_OZ3f& z|HYh&kHSxXeq%y+)``0#(D>olIWkH$o)u8ROC}6(`y&pU-_r#{Se6siu^vw#}!#tqIW?;2I&J8{dO&My`ujIJiNz|9HYozlYHaf;>r6d-aC^VsmL`XpHbvz z75UZ&kRMUxZ6w_`>!9e@3h#%#v}4M2J4vopDk z7^@(?B;nR2+8_UIK1;JG2`%I0Gee^9v^^FS%xN^5Ht!Y<{HLLPYVjwEkj_uej9@YR zmziB6D$UZ;)V3L|oQU9`pf8##2EOg}sn3Cj%@1bNO zI!FWJK0B=A95C@`QcH9T`GMKX23V=V+hWeiXD?EoenaoW-=X&ghdyPxTNcyui-a>o za?bd;Gh(giK7;X=do&W&?x3PKv1-yw{AUUmD1|@6%OR!Z4(^)ME!3=5HY>_P&gOP6 zxQD5V@=1LbDaufoam5z>xDvle)hPS1x-iNKb&s)1j{v1d_l<6tcQgV)>T1`2Ghb&8&XT$owgc)G@e*rhmy}zZ1 zC%04NRFW?$@_v##DDr+~cSMtXUXja4Zayb^XA~%UN7BDh^yf(bT+#0;d+(?BA)i&` zi6rk+nDMK+MUP?7gQHsH}-A^oJH->kOppBL)S{$c-Q z;c~_j?McsF(y~!@xS~3CW z>MJD&Sa)tI4QGBlyEKng^19L(wuE0R{g|ES@0LBn((GTB1#4K6y=il>|HCT}nXuE} z6tCblk8azG#jwQ@)Ni5Eue;K3uF`L_ zFuJ~W-D~xXZML71H+Rr@U*t318p{&-J8z}2NBPaS(ltL9+AWnyjFs9~?u=z@0pGVP zijCr~UA>TzUw6G8Xj>ym&by1Fvg{Ra_wr*a_=oR&8C}#7pp($w8qgT<%KiO{E`vGHL^LhKav#0sSeP1xGJ>%W}jQR7w?H^Nc7+;k* z*XkFQr+7bew(6_tBfO&_cG(REc1ONILza1%bCtfrh0n6i)dgk3hGtiLpgPBAueCW( zW>;7VwCGp391XfGuOs3`eeG>mKDd+F8_P*+HLNIiIa5izQ=>XXbu%FYP-5%BN2`tqMj%-qffnaK=uwRb^a>b?VOgq2P0wLQCVkc zPI4>>xBf$AT%kBg?Qu6fI94U|Le|7A9QNR7R>lB={7{Xaa4SD~A`6ITvmaRAfrOh%$xT5k-#h7p$Q|iW@w{_{WqvIj&D7<#@PNBBs%`OyAwX8 zAaMgC7D>|R_%rSt9&JzQn)C|oXNTUodU2VJ9IRd}g6E#bi|iv~&X@fqE4y*A*cWd1 z9lT-W)ukdI?tkIKc2G}zH)8kL?=jZH{!8^(f2^ORk9It~w+!VfuLxdH!f=mjs@n)t zlKd3^<)a~*%YWznYL}oKZ?8>fk=$Kdz;^Rz>c)qU!RKz&lh6sU>BV)HLzsPwUiUsf zRX35%;5|MrXMgAKe;mzn?B9NTfUzHVNqsB3&5IkNQ1st#=*i~tFB@b-;&iH8TAcpE z-+=#Tf8mp#{P3|X@3S+}7Og01`h9;DwYw;4(V#~ae&&!&Ph?l*lU2%} z7Ui$ks_@!s&j&&qU5;g-6ibIv{>=0pozsE(hY=wvURqEATl9OZH11y&aE-nKNmWYCT;2A5~F_CiV`dGkRD{ zQ0X8Z^JSmzOa8#Xck~;r%Id4mCyPW=(kIxLez{n~p5slYBblCeI5WCnB$Y{|xE#$= zybemS4Yw$hz7(7B&hl4^J*(jj$D6Q7v5UMT#d@@&6qmmUOFm#74ahRcJyvRj^omBl z^tz~_17$K_5$no@RnnVJy(ztV+|WX+uu^jrNg_}-)5@$EkkY%uL;ZXi&ZoH|{H1kb zS!HHbnW;VHoX!yfm`6oE+1XiDuGQWGFA2D5E0d^@T2UR5fYqpuSSnlV#8N>4^>b?U z-_jDWlG_K$q;Cm`!;TihRR@(sKe5c~XgQ3-4npPhqsoaD){81<5@kMR3YE!ON||&C z*Y}sna+)w5fUu*7T?s0U<(jiA!}G6EnXGp|dz)l^LRh*a0cP=6&yMLjBn~R{TE``EdJev-Vc|cx*ln1CrQnQd0y|$lpsS)}x;Xym6$`@~^OYWn!QhXqM4fETn&I{Sstkf9-TeBW&=2s)tJoy;WmD5o zXajitmB`5bvrH)S*`mx>;-1R<6s64Pz~H~;-2dSLKR)yDF!X`vE2o2={yZh9Z~fe$ zoa1v-f~Fs9KEO+V>=phBbmc*3<4yBskYS@0C8GIh{^^frjJGhr;cz^5YFQnffT?BG z^kUH5-+rL^2^LtNM31@H|CrtXY97;{O_Zb&WrKn?T%jO$ck-1#?O-+bF4y#oy~g`o zFX`TIve!EzNM9ROj*B9?;0laF*p0N z=j@Abni#9%<-cBL&Gr?y@-^%m`<360FwF~T+*7^Q9{GDd{hm(6-Oc^-Cm=A7j23~J zje7*<XEiqW>YKW}JtXgZ9rzjRNh>1KC`UcS8rhFE?X_&Oh8?Y$(usv@*kO5@o?T;QHMs^hUBf2GM>?|s zojm<1ly|(#jreN#2l;Mi_DJBwII$-Kj@BfGu*6OUu1K%<8dh&3tRFU#sgKuN3Tu8G zpUm(f#x%m~JppST>GifeBuRzHN+#X|%3&A6R>Bs++F<)*3D(1of_1^>z&64zhxNdg z!j`9by|w=62Yc2ZUtijAa}7j2tRFrENl(B5tPOTQY+*Wd*m77uyemlK;TTpAyByX8 zTMBE2-3@Djt%Yrb4Z;5}dtm#M&GLF@2ckbV%Nkfc?AE{nNwVVREQnm#Yp|B-crrY( z74{HpKCBi`?t;yMHDlNBhpmL&jwdg~XQv_^CJ4J7);b@rfSvFFu)E1F1s{wF!Pde` z%e>ypuqIe-L1zfKDMdpM>?ka{$`yDrJvzdc!q&s?hIPT#!s?$#Dqu~pAqGqsb`-1? zHU~BzwiMO_dkfaQ78AsP7T9!*XWN2)A$Z_d@%qyrHNZdwJdhn?x8&;sG14{J-^|`e5Ub7X@viA}?su}CoM8L;O_4jrvNzf8nyOgL z5@TMBqcI<7E~18A88S!i6wkV7v~%R%@$9j#+6UUvL({JHnZyff>ZYG}>IbeFE#D>6R<^PGG$?A-VDdvf!UjV2|NBsuS1*O;~4n z$xxOg4^Lzk1aoa73+vwhZ+LPnzP8JOn>C2O-%Avc1Bon6GrEh?ZBP=k82cA^y%tq_ zda53zP<<`OMEv%UxL)ci2wo1MNiFK3xhqBMuDEZ`2T1mBDuSYd9&GZEJ{w=u`t&F90 zUT-C+wu>fq0;M2CzBmjM8!q1-hKZ%ip~Eq;%QeZv*#tjBWt6YE_a9o;eyiWB@^%YL z4{ZB=TD^S3!un|LSmdrL>^&ABf1bi-u*#a~5lm)Gs%aR>o(^QkWaD@?FfbrQnOuOJ zF`gv_ntaeCZz1^SuP@7|0fVp6e>CB0vzcMMhhGnQ@C2q0vukB*S#GeV~Cz_b&xl@ zruq1@@+I=x?31xE#Pyv|mSn?Iki(M|uMs{u4li8fb-0{Da;i^8X%yGnJ~>lfLUOK8 z-sO|$;=G~We)3L|^^YmeQ4YkV!5Kkvq}y5GFkN}RR;h9<$yq)b zl~P=pcq1bx^W~Y4k;hd}vw_C$J(Zy-RdluDACq)az1%=fR{ESgBSakc@jp1)H z%n-&s1DaGbGne5@oFe)3EF5&6-{h;a*cdinj?IHxi##rm;qNYL0%tS& pI|{jY4j|aM41Z4{Z<&iWX8C{SVyyC-z|K|6Tb)M&0`?l6TYo9s7 z_D#)eH#INm%=)bD5+7A#VG@13R+3Bt6WbjBj(YT?mTWg;Bce<7saz7K(RxDC8-1Y zv+%`%)0y3II#6j4G#oS75ES2*Ir(XGi1EB9F5ZC})+z1lIExAyQU91)chxh>W4!5a zg%9-v)Z3dEoA`{V5Pm*NtHTN(e$0F=SifkkF6bdAVaC;9`+$8)#isI`VCR8tuVOQK zcytI~57w$;v-udXN5M`|v3YzY*t=k-sMr#|2W;mUtu9l=mhnr`A;u4eC`q7wv`nSi zxgjRRctEA*snqq{8WX|~phu00t>cTq)`O+qyq^ZX1#D=n*68o0$R;Y|K7q>?JRvs3 zs8y+Ym8#H`*z@7)&TE$xUTCjd`T3x7$P36^K`+#ky zVl(+ou=BunP_fxOJPz>(8=+!xGk`q`wv&p*%>ni<*e)uzjPC*4xwlr=L&e(pC9r8= zd#TteZs;At-vk?_Vx8Oy_5j!z6^nR-tq0p%#UkEdL*uo^ekxWP`Y7K0c|v@MF+rv3 zRq8;#Fg`>#NTnj)@rZZ4RyRb&BHm!ngB_+~5pS@5eYCm}Di-ku+Xw73Di-kuI}hwA z6^nTHLA=4Hs#wGu>`}1eR4n2R_Ac0oDi-ku+qth+m#$(FZ?I`#GgK_%-52o&J6**h z-e3=aovC6GZ?N@XXQ^1k8*FGlt#OWum4-cv_gtRPFT^-crRr7ce7>+>i0)aHig@=! zy!&Z&3sfxP4fZ_P=T$7?4c4!}R<~HiBHm#8fL*F$5pS^bz`m$r5%2zpH`td|EaDCJ zDA-q2EaDCJF4z?+7V!q#d4N{8QpF}C~0bK?`)f-pl5E~mg6!c?WjIdtd4mKj_+;T~R zXhWEu7+eia6oEj(Ccub!gSWT$^^bfIFfLQ zz_!aobR;5A5Z@D~W6+f?@EO9L31Kkr6+8Q+)6}GA|&DP4q-Y)T@8(54-hsJt`m3-VLDG;PJv$|98I`N;N^s42-^i- zLO7OinZVByjw4(GY;#A>B!UiISDtW~OgNr!w!mWv_aU4qa5CY(gi{3`Oqh;mmqp-y zg!>aV3mi>&0AZ8BT?r=;);D5&cVq`51`;6&hc<-i%yu>WE%pFm3*kC}TM)Jqb_%Q^ zJeY8m!1p!4bh^9j0^cH>NVrVkYlMdqF8LedyCW|XK?lDpPdI!}csSu~fzJ?5BAhAk zal-US;7S$vbHd4lEdn1TOb-Vxv%q@^rw}#?yo2ya!g`w^wh}Rl2ua{~2#+S*aGxgl zFT(T$;i?mO4dF3_odUl`cr4*6ftM2=N7yd#62jvNmkInV;R%FoC4!hq1U+`R@&ukt zcoN}kfyWX~Bb+I4GGTfkait18nDAu67J>T_oMYv4hYlLTG zd~549e5#ffrVh<3`CR``*8p8CPbhT2p>I>8NlMDE%F(!;;?yqj zekSB4q@jV7RPeKP{J3Q^lU60%g|C#OO)5^1@0+ChHQ3ro#bK}DI;A*r0Sx+MJ^1=z z?UQt9YIO(IfYIPGsqgtEKOgb^O$wLT6l=z@!*c@Kuf|O1Q6Zr-T1pczPEH%RPw`Bs zC58#O?{E)W$uvHHJ6=ng&DaE9KVp!Eo#b7TJFvx$l;k2G3`ug5~ zVNL~y(N7A$IVz0x3@H2T}T=jgWV7Jp{+aE;~!UpG2Z)9fq$>F7(E&@cFZQX|z6v@$wyoBV1iu^sgMeE%pM=A0+l1C}>@clwA zA=#|Ru_X6Wz?olAl)O10*{X`Gg|h`2;7$!48VPf%J8XuKPsH{TrBr;flPRe9Cn)j*Ozz;{ioBIinb9uf zZKOG+FfQma-5-C|pQTw82Jhk}GlIH=Qp{*sP>_w;aIN1a9DG+?GU1;8ToD%VlQY8E zDE|A5))C{#X{^#qH{tZftpNStKJ}+Apyz@w(2tpSnfa71h#Xrfj@S8^nbB?fQW(zC zd<%Vd)v5a5#y8CjXRq=@Gh0Wj!p5n8O*jQl5l*82r1}Rxc>B-f_t1YLISx@A+oJz= zd{9`ajq;DTTf(MHeWAzoD12if1Kv&Tbo=s#bO@N)cys z!ffr-O0l9$h8a;{(T^zfiBOHQ6T1teT*u->!KG2=DlIxIEkgg%B4;|hKl6Ii%7!Z5 z_pw*ezX{&9M+4*pVtbZQt^}BwtkI6C^*O$S0J$V+6_HEAlRq>(7hUi*1VDmGo~F{VmduDS8`4ev;&~ioA&A z9g2K-ubA8Q-H=Z!aw^G%ihNg@+ewm7DRLCa3lzB)vH?r`nDmp19{7o4=iFdl_Nn9Z z1v!jObi_Y@Ny}zAjIVs(oUP=ySC+D`9cx|-)CARS5M!T&Ihdw2UgNN@?%K>yeaJ*o zZiHXJP$&A|%71+$nsMIl%?S29AM)l(cAeM08O09rzwq---lZUlP3BVyjI0x1R$vMK zeVyoeAB;Cl|?<-T7JFgGj^YURQxo1*6~$wpoUFyTz)&y_o?;!Ot{lO z61P?nuCax5J7EeFCh~yNX|{LQV8Frz)5mc8iQEc<_w6D*u1Of*w}jC>0f(dlO;SP2 z^uzVZJKgTpRXOp9Oedv%9lYw_Q`+Y!?JNJ${#!zx_Gng7Acp-goZ2l=+9fFMURT=f z7Dkuk(EYoLvE7bS^4m{n+UN4=n}@TB{Qb?z>=}M*bBd}eKq2}nk zNncE_25PFKmfmEw_w@Nrd~2<#?z^;s)h@Rc3-r^9(9#Cbtmda>&*am)5EHa~q5@v5 zU7?>)Vs(E3yNg}|)HdoX?PQzu+s$Nqq;F~<8=OAdjvtk&1ZvZZZS>Nlw#l%IM$o=eT}t2w;Fc%9X*UE1VVdvobUznbb+`Xe>fY^kl$ueO=~bWPU1 z)|VKytGMGl<#s0*6y34Yz)gl-G#=WA9>>a87|*lX4NHIUtG%@JZ@(IAmj2h;50~D> zRX6?ZnxF2qKEjd<_T915(2a(XPHOQJtKG8nf?w@-OEK1mqS+MvFXa3qOO7h2xMRnF zBiu)74hM%GVbs&FXq#`RVTXi;-eI-5hW)8PL;E)c6vgvJA9rGXx%}}E+)O`wyoELK z6}w&RYd+WBjRkXVPqHQBOrV@LETBkwb85f$K$TO0D5p-)N>23)Jvk*xP9<6&~`ARi{T;)W`@`^`$n z`FgSw(zch7J(*r?rhbs;5@8Jr-(=YDB62n=D-1;N?Ym4i$|Ciu zFJF}Jgmq2fHph@I=ZYW6bZ?SO_o3^`T|8%GRVE?guG9F+?Pzd3 zU~Hh{_x;0tQ3B#W-SpJaV$`dw)#v~<4A&^9J@R1+6JFrIf7(~$^9Ju)u^9DuV?_!} z_Z2@?IhxscyU$A4fAQU)bz!-V?>^hZ zSUoSSYC=UXsE%Z3`R?lW>~;QCwQQI;jS80*$G_;W#}UVipa1HE?`2tEoar)DE2^4) z$D^utA62a<=){80MOFJ8lhFO>n0E9e>#}yDBo&tWVJb0y7Ut{89-~QZ0AE`_=}d8= z-i>FYx0)Q|kLNH=&;-ZvFXuDXlExHAC(oD0k zMxX>{q;IklsMewcc6d2--~NVA;CsSb0vq-D$S)BSN@{b{C)lyu_BT-f*Qx$>NBu*q zjr&WmaUqqaQfCkApAXePZB2EcO(}qlKD8RuKeVXWZ@wy8?4s^#vR-a|soCIyqWw;c z{&@Y9f5j5GiKU*1EUH`OH|9q(a!Fe+3Up%~jgm=Y$J%&}) zzphICqX<5#f7w{X(Eg19MNz!lSDnJ1{~I$!wo;W{xIddL5>Cl~&avpLMH==T|M7GL z3*W1Zn`g&3=5EVrIE6O34oSzDJk!qq)5RvK<3}kiJP^#_9 z*p*I*=zM!NS)|%YJW{PkYuL(?S8TYbNiP+XY&29-9}iQVEvDp2bw#Br)%)#A$N8yb z$ER&KlRc4MoJs@ark6GdYfyOX0WqL=57f>|+wQ~yX$Q0rQ_D4g;F+hwZQyoMr%ENH z+^?(j?3Q9D@ej|rM|r-cxX~6Ffvt{8i9B|oq>B>{_0W@H)I%R{J)BJS5Kb5{-%cGr zs)r?jL;GJ1DC)(VzJ4k6x$9I9t6X#L(3JIY(J7JyIF1*e9o8oHxkvSm(nR%6kM<~s zH_i@ibMDzk@;}O9SDtb%%+UQllGi?u>WyOfqI1#gOTP77hT(O*&>ddvW3;ZSv*C-Z z?gdA{Hx`Y54}7~P*u#-_euxj-!#7^&$;R?8FLZqBUA%KX)Ywdt>e*bk`<6ApusTBW z0lYPjYcEdK{4tk5cX7Jm)DYD3;-FU6RhMme5fuEHNc z^`|dEjdKzM7U>4_r>%^-0s+?mv`4u!rjl9zq|$L;rF^#cQTp z{h)<$`_+huikT*&;~tUJsmLVm^Gkt z)trEg<8!P5(~j0};kmzb2>lOqWufEojCm`-ke`aWP(Pf1^2-_H6%24N6w6L377qkT z#rson9M`vXY^i^S`8(rjnTveSJMOr$nEpy{Ng7-n8?fps1v%^-pZDt~_JyPQbv?tG z5ptt2tj##LdvJihBC-Ty1?czFa{(sw*PMW93GEz*ZWtI#;irGQ$lh}7{{3|gdxQ7B z)tSBOn10K|*e0I$#}#(oG5dD5hThNpd5CG=NahXuS2%+1Wiz&(=ih(3Yu+dXrqxgp zm>gUqFc*i4z;sOV1f~>fK>I}g;M1;}eZ$2$Xyq6FPSM;R#;;Yp%5wOJjUzS9Bl+FN z7*@@@H<_7*Pi^WDT^r%e*N`O2h#_ApM>Jmd03U#dx!M5V`nG&eQ;p^?tK-E7I!)WE za7nTjt}gu`K$5T#6ARZ>mhBap)om{=V?_qM3Bi1Qim@X^TpP+07#pSuENJ&nenY$R zfs7?+hV5GCtjv7K|=K-?Eu$2~- zf9%IvGL~6h>CYxI&Am7|_(@jBzAgXrNj6TyzAjI0$wD=3hn%Hn*V&5lR0Er)VbkQ@ ztyqtiQV$B{U3ZQVFNOb*ueD-N`wxm1x2NA>K7+N9^MY8*@}5DgT}zui!tK6}ZCnNG zgF8y<IdP=8NSCzgWc{Eu=*ixcjJ?il!^3=$CE%d>;l+4*wwHlu-&m2%U}~> z?XVfJRj@g*PS_&Y>}0pQ!WZpe&-&sGW(h8?gD8Xb!S|p#*hJWpXWZ_cu$d{)VY6X< z@U&0|n*rMZn*%G2M8~jt*ln;T*b3Mx*dY8DvlF&E*)+F%mOt9V=E63>uJgA^k{P$r zSr8W3>#(M2SQ%DqhTR973aiD+?XVfJdffgyVe?=&V&$1L5dj@02)hy1JP*%*EwKRD zZDf~#55$CED`4wjufR%6(XXu)1Y8us(FvP~-IuovE7PMPY!Pf3>^4|CYz1rsY$L4n z0#<0igkck5&9E7;sjx+`PT1S9`W2WU1~kE@V7!vGXcvS9{|o<1*&P?Ab(l#oR#NJA zN5E!paJvVtOAN-R{^BET9GAfJn5n~fRPe|lk}(nh?G8!s1kWFf|s|BrkEa-v)bnGGmE*OB#N#+`}( zZ!g_(KWvl3I-{43kI4ft$Nx&nI^mwa*kZt1w%dFn&d{&`UYf{A^Ub^lh`(SYFD6*<;%J zGEMUYZldw>#cr79csZ;)8l}i_-I>j}bDZMrGZ&i$pYHf~0e8@9`DAy5Bu8!{=k9WF z4>TPhPw2r$qv>`s8S)(%Uv0eGohpxsWF~ojPZq;`tiAepk}v-^}bk zY;}2Y6lRGrE2C-5S^snwQNvw%dzSoLG;6K7K1*&M!v?mw{+J&vn(o>wv*c+ptS9DD z93$qkJqEp3$me2^*Ryg$EPISb zy)|6SUT>9ejbgo9gnBTh8$Fu!_BVO3NnSh}{g{R*?k0H;!72}| zl5Y}R@A(a`^>S1ysUG-&yf_v2+6VF;GG_TEnFDgv7?`B;#ba>iAuQEn5n&6N;ASnQ zO!+3T(YrKnLdZ6g^pL~H!TqQ_W*iG=rh&B-NP_Sp9p!LM@E$s zpZ7d+jy#6sJdeE9BhSInL9HX?l_Z-7Dq|yZ;-f*iCpl5xGZ9tjCHWG0rFy(@l8R53 z$7`N!m<0JHkBp)xKI8FlMV-7UTOlKl>62J5V?ujnC`uK5n(&WFIs|VEb+X>m$$TM7 z(mCox3YE&_(-zjWPt`8~$p5d2Ly?!d`7(HqLYL$CPb3i;*~hQFnd=Vyr6Y{)=O ztds9$2y*dMhQF|ojnf$Z#6rHQm}AqS>C3H|3~yq#$$Mtv%#!|+kI!Vo*qgE;3*Dy5 zU9uSdilQ9f5b3WXj-)A9Qat|f;kw=UhbOX@tH?r`JQ>~cI5v8E$~$0 diff --git a/PairHMM_JNI/pairhmm-1-base.cc b/PairHMM_JNI/pairhmm-1-base.cc index 3941c92de..a2d5dfa88 100644 --- a/PairHMM_JNI/pairhmm-1-base.cc +++ b/PairHMM_JNI/pairhmm-1-base.cc @@ -58,47 +58,53 @@ int main(int argc, char** argv) vector tc_vector; tc_vector.clear(); testcase tc; + double total_time = 0; while(1) { int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); + if(break_value >= 0) + tc_vector.push_back(tc); + if(tc_vector.size() == BATCH_SIZE || (break_value < 0 && tc_vector.size() > 0)) + { + vector results_vec; + results_vec.clear(); + results_vec.resize(tc_vector.size()); + double start_time = getCurrClk(); +#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) + for(unsigned i=0;i(&tc); + baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); + double abs_error = fabs(baseline_result-results_vec[i]); + double rel_error = (baseline_result != 0) ? fabs(abs_error/baseline_result) : 0; + if(abs_error > 1e-5 && rel_error > 1e-5) + cout << std::scientific << baseline_result << " "< results_vec; - results_vec.clear(); - results_vec.resize(tc_vector.size()); - double start_time = getCurrClk(); -#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) - for(unsigned i=0;i(&tc); - baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); - double abs_error = fabs(baseline_result-results_vec[i]); - double rel_error = (baseline_result != 0) ? fabs(abs_error/baseline_result) : 0; - if(abs_error > 1e-5 && rel_error > 1e-5) - cout << std::scientific << baseline_result << " "< void GEN_INTRINSIC(GEN_INTRINSIC(initializeVectors,SIMD_TYPE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D); -template void GEN_INTRINSIC(GEN_INTRINSIC(stripINITIALIZATION,SIMD_TYPE), PRECISION)( +inline void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift,SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); +inline void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift_last,SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); +inline void GEN_INTRINSIC(GEN_INTRINSIC(precompute_masks_,SIMD_TYPE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); +inline void GEN_INTRINSIC(GEN_INTRINSIC(init_masks_for_row_,SIMD_TYPE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess); +inline void GEN_INTRINSIC(GEN_INTRINSIC(update_masks_for_cols_,SIMD_TYPE), PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt); +inline void GEN_INTRINSIC(GEN_INTRINSIC(computeDistVec,SIMD_TYPE), PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen); +template inline void GEN_INTRINSIC(GEN_INTRINSIC(initializeVectors,SIMD_TYPE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D); +template inline void GEN_INTRINSIC(GEN_INTRINSIC(stripINITIALIZATION,SIMD_TYPE), PRECISION)( int stripIdx, Context ctx, testcase *tc, _256_TYPE &pGAPM, _256_TYPE &pMM, _256_TYPE &pMX, _256_TYPE &pXX, _256_TYPE &pMY, _256_TYPE &pYY, _256_TYPE &rs, UNION_TYPE &rsN, _256_TYPE &distm, _256_TYPE &_1_distm, _256_TYPE *distm1D, _256_TYPE N_packed256, _256_TYPE *p_MM , _256_TYPE *p_GAPM , _256_TYPE *p_MX, _256_TYPE *p_XX , _256_TYPE *p_MY, _256_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM); -_256_TYPE GEN_INTRINSIC(GEN_INTRINSIC(computeDISTM,SIMD_TYPE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, +inline _256_TYPE GEN_INTRINSIC(GEN_INTRINSIC(computeDISTM,SIMD_TYPE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, _256_TYPE distm, _256_TYPE _1_distm); -void GEN_INTRINSIC(GEN_INTRINSIC(computeMXY,SIMD_TYPE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, +inline void GEN_INTRINSIC(GEN_INTRINSIC(computeMXY,SIMD_TYPE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, _256_TYPE pMM, _256_TYPE pGAPM, _256_TYPE pMX, _256_TYPE pXX, _256_TYPE pMY, _256_TYPE pYY, _256_TYPE distmSel); template NUMBER GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_,SIMD_TYPE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL); diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index 5ab49b531..caa880b41 100644 --- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -332,7 +332,7 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation } // initialize arrays to hold the probabilities of being in the match, insertion and deletion cases - pairHMMThreadLocal.get().initialize(X_METRIC_LENGTH, Y_METRIC_LENGTH); + pairHMMThreadLocal.get().initialize(haplotypes, perSampleReadList, X_METRIC_LENGTH, Y_METRIC_LENGTH); } diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java index ca3db5ed3..ed628f064 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java @@ -57,6 +57,7 @@ import org.broadinstitute.variant.variantcontext.Allele; import java.util.List; import java.util.Map; +import java.util.HashMap; /** @@ -147,6 +148,52 @@ public class JNILoglessPairHMM extends LoglessPairHMM { jniInitialize(readMaxLength, haplotypeMaxLength); } + private native void jniInitialize(final int numHaplotypes, final int numTotalReads, JNIHaplotypeDataHolderClass[] haplotypeDataArray, + JNIReadDataHolderClass[] readDataArray); + HashMap sampleToIndex = new HashMap(); + //Used to transfer data to JNI + /** + * {@inheritDoc} + */ + @Override + public void initialize( final List haplotypes, final Map> perSampleReadList, final int readMaxLength, final int haplotypeMaxLength ) { + if(verify) + super.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); + /* + int numHaplotypes = haplotypes.size(); + JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; + int idx = 0; + for(final Haplotype currHaplotype : haplotypes) + { + haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); + haplotypeDataArray[idx].haplotypeBases = currHaplotype.getBases(); + ++idx; + } + sampleToIndex.clear(); + int totalNumReads = 0; + for(final Map.Entry> currEntry : perSampleReadList) + { + sampleToIndex.put(currEntry.getKey(), totalNumReads); + totalNumReads += currEntry.getValue().size(); + } + + JNIHaplotypeDataHolderClass[] readDataArray = new JNIReadDataHolderClass[totalNumReads]; + idx = 0; + for(final Map.Entry> currEntry : perSampleReadList) + { + for(GATKSAMRecord read : currEntry.getValue()) + { + readDataArray[idx] = new JNIReadDataHolderClass(); + readDataArray[idx].readBases = read.getReadBases(); + readDataArray[idx].readQuals = read.getBaseQualities(); + readDataArray[idx].insertionGOP = read.getBaseInsertionQualities(); + readDataArray[idx].deletionGOP = read.getBaseDeletionQualities(); + readDataArray[idx].overallGCP = GCPArrayMap.get(read); + ++idx; + } + } + jniInitialize(numHaplotypes, numTotalReads, haplotypeDataArray, readDataArray);*/ + } //Real compute kernel private native void jniComputeLikelihoods(int numReads, int numHaplotypes, diff --git a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java index 7b09fe047..08a49cfb7 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java +++ b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java @@ -100,6 +100,18 @@ public abstract class PairHMM { initialized = true; } + /** + * Initialize this PairHMM, making it suitable to run against a read and haplotype with given lengths + * This function is used by the JNI implementations to transfer all data once to the native code + * @param haplotypes the list of haplotypes + * @param perSampleReadList map from sample name to list of reads + * @param haplotypeMaxLength the max length of haplotypes we want to use with this PairHMM + * @param readMaxLength the max length of reads we want to use with this PairHMM + */ + public void initialize( final List haplotypes, final Map> perSampleReadList, final int readMaxLength, final int haplotypeMaxLength ) { + initialize(readMaxLength, haplotypeMaxLength); + } + protected int findMaxReadLength(final List reads) { int listMaxReadLength = 0; for(GATKSAMRecord read : reads){ From 1b1c0c8e7618b7b8ae4c37745aea8f3e1601fdaf Mon Sep 17 00:00:00 2001 From: mozdal Date: Tue, 21 Jan 2014 11:47:30 -0800 Subject: [PATCH 17/72] Split the inner loop to avoid the overhead incurred when -fPIC flag is enabled. --- PairHMM_JNI/pairhmm-template-kernel.cc | 80 +++++++++++++++----------- PairHMM_JNI/utils.cc | 2 +- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/PairHMM_JNI/pairhmm-template-kernel.cc b/PairHMM_JNI/pairhmm-template-kernel.cc index 44d205869..66dc557aa 100644 --- a/PairHMM_JNI/pairhmm-template-kernel.cc +++ b/PairHMM_JNI/pairhmm-template-kernel.cc @@ -345,33 +345,38 @@ template NUMBER GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_,SIM BITMASK_VEC bitMaskVec ; - for (int d=1;d NUMBER GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_,SIM // Since there are no shift intrinsics in AVX, keep the masks in 2 SSE vectors BITMASK_VEC bitMaskVec ; - for (int d=1;d; From 868a8394f7fc9b67277a8095e085a6610ca9f401 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Tue, 21 Jan 2014 15:01:09 -0800 Subject: [PATCH 18/72] Deleted libJNILoglessPairHMM.so from git tracking --- PairHMM_JNI/.gitignore | 2 +- PairHMM_JNI/libJNILoglessPairHMM.so | Bin 102331 -> 0 bytes 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100755 PairHMM_JNI/libJNILoglessPairHMM.so diff --git a/PairHMM_JNI/.gitignore b/PairHMM_JNI/.gitignore index a68330124..ce903a6fa 100644 --- a/PairHMM_JNI/.gitignore +++ b/PairHMM_JNI/.gitignore @@ -1,6 +1,6 @@ .svn *.o -#*.so +*.so tests .deps hmm_Mohammad diff --git a/PairHMM_JNI/libJNILoglessPairHMM.so b/PairHMM_JNI/libJNILoglessPairHMM.so deleted file mode 100755 index eed82186343d5ec4ae68941ccd7c95e5e6b55cbe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 102331 zcmb?^4M0>?+W#O}RN9QCF1nS}QdwkyXlf(HDCoU97%3)hCL&-`AS^SWmLC{k-Nxx< zTW#Ccc6GPjddph3)m^vM0%iOv6)nq;EY!?9l1cUEiWws>S4Yr>P-mV5u zN4J+4ZGdp(Kx=#lbnmg@pgkVZ(kwh^*i=uQTHjJsJp z0w`WpMzQwy7?#1uypxxov}4{JK3D%j7HofCWz=Kd&2kT*oY4d!=Rf_i>e&;_;kZVR zn`qlSsln|j^?VVlf7kNXiMWSG?E132=1!!{4?Ai#8rjMhwE-!3vkWF_20Pu zh>M>qaVfZ-!j*t48W%syaXo{p1XmKS%W+M^#m{0~iQ>s-d)&A$!#xYv3%GuX>qT7V zXAmB`;VQ(n&_v=s0oQa~Ww@@vH3rvRxSqtt&v0D7#q|WP|H9?L^&4D%Tz|sF&qQ3g zxcU+>KBc&);p%51+T8~OPRDg0t|)`{sBwSAxG%xI0N4GvuEoW1l#FWzt|_>B;o@f{ zuBGD1W*dR~%f|DcaeoO{Hm-XO+>gCo0mkqj%WUvPbi#jKCwLRx!+bFAH?@OJ&>Sz_ zO@wjpgZrKBaJK-CvEXjNb8iE_-nie6yE(3I#q(9Th8Z;TmSQ}_nh4ANX5+cP1;@w9 z?O-S13|v3QH5Hc~S5I6+aJB!8GZ3RJgop7w+JL)Q@DEs?e}!i^u2ftzapmJ0i7OKq zKLh2h&EH}`H{u#6?P(x1T;&GgS=@htYZ0zVxaQ*;kIRLNpINvD;QAe| zcwE-cJOC?j{o0EBH_zd=D{QOVIdCNc(M{;Fq~~QpGcNgHSmLL>EAd6f;2Mg-Lwq9* zgOAC7JDiH+zWt{!_&dTMy0T+@r6FGoSO_!aKL{Zm$?4Kby|dwj)U%3)#ivN#Y*(RU zI+F8gC-T2UaJU9?>efpE)rOq!p??osAKNsQJ#VwM82D`5$)996T9V;t&tT9HU$sFp z)*1XO4f)g7N<@8_ya&2T{`yBGY}$e4M7?8Vz2>;P7vZo6~ zcG6$@up9NS`&f#1TrTg&5Y%XAbK%^Wr#JFhXGoOYS{n%I`G1Vx~ zkJ5?V`eK~YKC78{;A8sval@a}E=qhOI}o31h5q1QBVlvAG@;#;Wx78T`?PoO6bq%bMHDzq}K>4eimf{E5O2SJ}#zNjcSq{J2i+Q*Y?+ zXs7=xoy6x2@Mqy~?d&<%h*wp)vcfcD-2Dc3@<$tS$+Z7S6JH?tlMFrIH1rY4CvYZz zPxy03_S7J!BmBTl{QPDkJ~TceD>UU#ggvQe+Vc{>+VFD^`iuTQ&9G;*B-_63#BZ6S z-DL1sZ~rj-D(N2*n`X%Wl^L(v*=Khr{aa@Ezr$xpM?dlNIpmjeJ{+R>5z z-3e!6FW8m#nWh^8j0TJ}@Kv8ld>#CWpF^F* zr3X4`*YlnD=P|?nRmQlM(rvb7sJA2ihZ+8yR3teKHEmOk`68{|eDP{0?LC3wjJIW~ zAqRT!bEuQ}Gq{udDy0+rT*M*T(_!?NsprE+oKIRN>x-5*+j_LCqxNQYGEVO|{JGKa zH^UsZi{NLyRmONU>wOdt#MiwbD=ahW-P4KxAMGU0EH&a%ogrYFA?I57q41yM5>aR1 z@57z=xX}_}>a$(UM+TGSw#4w~X-yKDX3%@jX48LWN#oo+JTV~#@Jg+Tt25ul_&&lFrR({^&xwgzPrJ3V%AIbB0bBjmjXL&ri z9$V%wbMn(lCuG_l$;>V+C@S&hX3i+d&(AC>E}Wj3Q&=)RKi4)Ut28UKu=tV8>BWUv zIeD`@-aIcEai8@_W{Ed1-;-ICl~+8opdfS1IMrSFNPe!zlLm~Enwt6Oth@;&(??3- z<8!le+8e z?$X?BZ(*@I!8?e}P}Ab$GasEk*PH7qF3y^pmX$IgOPw$%GbQEj;#?05nNk3VMTelZ zDYHs52gZ-i_1-`IQHT*mQu1>Pa%Xu{(&7^`A1!Rf=b?3IN=~LXuOL@+RG#RWVym$D zK_OwI@^bTYQqqQ+Lehq1K01RqH5ny_c?;16#pna|(Ub|EbgDGLTa>S6Pw>XaXV1(k z&h!>%<#|17Hd~OLk}_esnx2^+mzf@)nUXekLPF;Fu@mn8X~jedQI5B3bG!{74FtSKN%e;8kvGlnlM-l2ybqF=8P|y7kvGeV$}XB~1Erv_ z6l0`~LuCGEpP+Bt4c{1?ou8W}?Z6TBzZVogC^Hpf7Gn@2Ew|X4GHw`#_^c9)lV|>8Rj8cXs?R^vxr=-pur( z%&h$U!ff_{;rrg7EGz-Sa&iqhxhZLISZa-7G%`K~ohieU=kBbcqTE@C1Jv4ZT{Lze z#~oS~Dj|Fhr3Q%@Wemt62wAzsrHEOQ&mycHCRCv}ekiPwC&o2`3u?*DD(F0@Kv%?@ zgiN*2lX*WBEhx+}hS4x7Ca(|?=+cm6ju=n|ioy`!&GND=P;s8@JcQ7^8F|^`CwPZK zhUg;Eh{AkNO4=wZntl-?Jt`y=cUY;6gD4uGU?^ua&j=YqGRGrG%E-YeDTMw+gS5JE zJCC@;lYfdkT~auCU?+}u2`4(IoukDWj&`Zf{;0WdFh(Iq;}DLmg#oKk`kF4Hmq@U8!{~Pg_bnuW)JNQ4OI7W9O z70NCw`Tvn~630KOMlP3*RF1-pOs;kCClq8AsS`Z#*7!j())dY#TiAgi zU~y*6C?STHk;x#OBN)MrTjdVT96ue2T`s08kD0N8?Z?)`U>%n^0h4^TH{RnFgScJY z95_P}GLT?+I4v={8+9W4$U(*6_`j#)qw@==XXVp&|94rfJ%GHWu-N08l{2v@2faBm zH$UI=|5N_|Pg!YLWMxgy%ZK-HDVF)joH?0AaIZox*t~NyOXF-FFBV+z$M^({4J;gt zTqrFR(fdDzG6#M0R5Hc(Su*%|P*CGHb-*CEGRwzC(#f4ImNI zbx0})qv5N0|ISxiYo885L%oHM<<5fWA6p!Jp}E4tdBl^#@j5Gu8Obn?+ zAh{|NoilcV*KBqr+N|Tm8D<{B{hzF2RB5b^Jjm0`=g@#Pj1e+nh=6{QBl#!W2_+3f znJXTdAj;(rTWIm1ftmk4HAE<3#+Sfd4SGhq1zt$WpR$k(6}lwN@`yFmf9~2rT$YAL zTw9$?_xcGMnMK;`lP#BY?OSd3StwI)XBAUiNUywtqI{!=oulUbSZ;1%{&?Bo?nHLH zaQ-%%pB;{ z05Se(gC4cOKyCM`RyAdfL|;N2x6`?0WRBaasRdE?`U zjL7q*LJ#bH7<;^;dQluYn7Ypx+NtuMOO)rTyE7fF+^FQy|D>Y_I^J#3QSARx#YaVH z)HH4gvVY|2UigX(iy^HCwjs}GPoB|9kI*?TX%xp(hmC5-{T~fa{=D%6<1LK>ssrP9 z_8DJSC->8J)>{ytsi}+98!*GJOi@-`+ z>tBgwIr6ub499bX1&5b%z{n3@@KEBVQYgW^Oe*D?ygbQEJb6rE>?&lKR*Qf;=?t{cT2NbxDl zk@t1&@DAfWYjr#P3gevR)^_+fUCOCzhp#f;uQ#>Bo3Jfe>cis_^aAq}X~7#u4g|?z z!Q+*A=o4kZ<5h9!6K%m;&#m{j;C~+C#&-ZLc%Bt9KSM3}ekKz4Bny6s1+Q4}E(<=* zf;ZpMlAK||Pqgq)v*0IL@G~v=Us~`*7W@?5TjTSs!y@x(c7Cc_1w|-KjKT{X$`KL$=-g;iaVZr0*M(7h|!Q;qI z=o4+hV@S`pG{uaDy!N*zf zV=VZg7W`NXKFNYlwcr&Cew+oLX2IWY!Dm?T);K@Sf`7omKhuJrV8Itz@aY!(91H$I z3%<;P&#>U1w%{jQ@KqN4Ll*oC7W~5&{0a+xss;a=1wYM#Uv0r>S@7#D_~{mWwFRGT z!Ed$Tjg&=D>n!+O3;#X~euf2KZ^1ue!Rr?MObfozg3q(yn=JT8EqL4LR{wv@f{(P| z^DTIX1z%vnM_KT*Ecj>(zQ}^_Z^8e{f{(M{i!Jz}7JP{XpJc(8TJVYmKih&&v*71g z@EI2TTnm1h1^>7OKhuIYj+hB*kp(}`!av7?pKrmJS@2~R{L>cvA`8CCf`8J2f5C#U zu;5o%@QW?@*DUy_Ecn$Hyk^0#v*4ex;Hxcop9R0wf?s06*IDp>3x1yk|EvXHZ^8eU z1+QE1RTg}s1^=7{-($i~KU(l<7W@heKEr}vX~9pk;QwsF&$QrQw&05_ z`2V)xmGZA5^;cnKeFTe_8nW_h? z1bjQ;-h|5pd=p`&=D{KX_a)4fJUC6j*AVVQI8DG;67EYlNx)qQ|D14~fX_t$=2A2m zE#T9HuP5vf@Cm|9;e$2-A0gb2aAPyue}Hf_;d%k@A4cfG2OBSl{wEwqxL&~H2{T0x)(LnF zVW#H6Y5|WR%#=L1TEO=Z9z=MBfCmz0Y96c-@a=@}CR`@qn+OjfTqNMWgqgAjrwRBP z!ovus3HVCFOwof$0`5ZCNjOfx=fVLGCmb!{(}WWVI|O`!FjMoOO~6M8GbIl;o)`U3 zIEiq*fcFq~5v~*PcETeFR||L};gN(_3-}$v$%I!3_-(=|gsTMnI^j`-%LM!gDl6OI;eG2v9g4go((cpPDyfTt5? zDjsY+C;Fdo8sT~Yk0;F3J6I>+F@(nxt`_hJ!c4(~s|9=yVW!@}6#^becp~8{0pCve zLBeGMzKQT8!bJk^OPHy6aGHRxAv~FInt-n){7b?~0`5ZiA;NJ2KGy~C6vEL0K27*x z!VUqSAk0)eXcO=e!kL5{&x-yhJdJR@fcFs2B3vin?S!Wjt`_h{!r6pZ3-}$vIfPdT z_-(>WwS!dxex2|P!es*f6X8b)7YX=9!kmDD(**n+;XJ}=0)B?@qlA+L{3PMW2*(Nd zal-k8qXk?{xPY)jz>gB1Mc5|b>4XajH#Uj>CtO6hUcloCGer*833v?QV#3t|9zobc zc(s7xe-ic?o|K>YzgS?YrH`?5{ z9aU=1hb!d`Val+5o@jeNj8>(*I!w{J?f~7@(z6z<8}SHw@v+Ta-@;?qL#`>?HqEg2 zt1$FS)X$;5iBQ1X7wkVbG`$-2?*e68OFi?6ZJVUVo6Lgteq|KkN_S0kJ?NS=VWP6= zs~0f+C|a{yTM|=*N0;`ATdRoSq~z8%16fA8e`(Cqy!%pYn=8blt);3FiG%{)IR-`1 zPAR@MLW)_yWZJ}%2EEQAP|{UxvxUB|1A4$hukU~kn}y`-m(YE(6)BZ1)Q{>WYr|tS zcT9w$t%*qk{4UL+Xv6+?#*fj+%1V`=2e=_c))9gsiKABLhEsco=sA?D7+SiJIeGGp}CSgy) zHo8=QJf~{A;6DnUgy)b8u3Z-_xkHz$&(~`x=EfjqKtmsnU)q5jFiarF=Wzjh1!T3 z_KJ6qXO_>4iL`lpkx6(MY726|v;)l{X-tL%O=z_pG|^;Ocd4+W z{uCRdZBi-^+AI2_N=eq^5YG~IEDh}bZ+k%k#-{d}{yMP9+BdG2%}UMT2*tNt^iYcP zdqs=rqiE@UBJEEuMP}-&5FJVD(0N?;zXTF~ct?5*`l|^kjDpy(&$#Qxy&iY{KRPATE%l$w@q zYE9F9igs3w-Ksd32pOfnV=fJTd6sRTF>66#_m>}C+ttfV6?X?hfxDA?PIm(VmG(*d;8+Q zqo3S)-&3@r9a6OYYV6mo>`LqoG!sP>2*%ieYz49v$j;ltGWvU>AA{B+?^AOWIdeb> z|4@lNq0}@RrGiRc^9%JbVcuR*VfcA>7@`I(JDV;N{;lf#%wF*oeFxS02gz>GfB7xS zEZW2}U?@2n+zt*Xs%oda1KM!LSvXsEl^hLqmKovmc8Ho`W^oVMPeHT8_bX>kD&c!6 zQM9xfE!}IhlpP>6-GPy!=53Z@mdbe}Om?B-Tq?vuDL9E)k&I)fTASlL&E^CI-Purb z6j4I)&x=vqnpgfw<5Pwnj^UPYP+9lJ>*8~}qD_g3)OqNPjl$K1fe2k9edx`_3Q6iOQl8WXM~?s z#H^OoIx96=-)~S~lhn7Z)JXkbQWb{SgBWc4z=YeU@>Y+Vz2YjY)@AH}8n8YYHbJNz zu>-0gm{b9`1zn7r44TL*$Tptgz~5_Ron(7KvQe;PTO&&9Z$KJ~vtEf`A469CZ>Lb4 zG1rf930soE=8ZB~39xd(`ZPp>!L)>E#!5&7 zo(hP&;%XgKUL(0$_H7z5pV?LD7;!JQ*CKk|~mdI!X@lWN-*e+yf3_ ziE3bBf?9b)8-*oS;|@#w^XfL1NMUdJ2N!Q?V+odNV~Kk?xc$BEGG=Ku4=rX5Z7lIy zNCUn)gqL2X|M8?@2`6etJINqhA)Ww)Ea8O+a8R`55Kjh&u*7)DK^-NBcrrMi#$AsC zhp`k9;{IY27864TbtG=vWNVGXKZWJda|}@{oU{}wEk8&u&F2Vsd&y1>~GI?S84jX!rT6(C&FLb7V#}2Ni@Q>2pvLw0V*?&!kEJCvAO98IdBIh$jHl zL$viCaEOHKGyOd@8?7ypU?@G)X?);2!H4QeezY^-=IU3JvjprylKVv*Q@SJHpN8))9o)^uFDf$kkwEHgy zBo)^`M^l|>pS|+KD+Ds6Z6HK>v6UiJeIrE4YNZH0e;uOq2PL5zi9W5=-@bS>Dpq`W zWbWm93H`Zj{s|!Hdd>+DvkM?!I7#h4OalW-hnAG7t^&<;L3SBSywU^CR{rzJ|y)_ z6&w^2rJ%ZKR9|H8XReRjNQzjAtEDw)J{S`r zH2a8Xhu+1CrVA1sr8k@q+(u%)ENW zjP(!Stsgv2=l`6>nx@}L*Ce?GtNH6t_woSc#iX zqK3D$IPCrFzO_5Lr(-6v-?e>qDuAwI0ldc;@vXf_dQ5-)DR>+3cmx9dZSTKP-z0E= zqCo*pe_JxSiI}qWTl+7!H9UmW>tB~vd745ZPotubH>08pP+3n#BY=;H0UAU-8Pe?i zw*n%nhZ`Lf4-{ZLo<;>k9M|2J!d+5SsCL?9vC?$He#Ssqemz$Ta|3Ani`avqY*Iw zoIA$&A_f{hG$y{8rX?JtY|k}@5RAsEX{V}fQ?>V1 z4GXtjrY?$xHCaUr_bC)@tIGektJ()Dxp!d&aoV2Yio>6ALm@chfh7KYNa5eyG@uJJ ze9AQEZgk(sIAI6-=H=|B9`~tiyKdFMlBc zT1Wo}$JnCU3_M%5Yp|Y=A-`Z*N*3Sn7~O}_Yf^-yUqVvDClZBqlYS+tX*i5s#KZ5Q zg~;1C*lh3c1*GEd{^ds0_B&B^;KxFyo2ocUY{Z&JgX z6z3`XQ-36%6V^PT*hlZ?I@r0p_y)yUH@lBgxu@i3O5R2>Qej^%^imR98v1gZEbp8| z%dH6gRk${mL+Y8&T9?XP@1FTu4gXlK*)L+v{;`VOTS`>Jx4>_`L*3eXv6M6exvB-+ zSi^-D$B9s0m8(g$WoOCJPg^7fb~HCji01}Kuv(;9iDl?n{WphE5FUh3x{vC6yVI9mU_%rb323=~ z5@;sodb~u};~7cBU+{8|B}9(Y8>HY%#7K#9Nzz*MhoeGb9gA8j`dp&LlEYr{uvKC$ zNWrnK602K_3T0lFMQdBpLUq4HZzUQFRjb5eN<1qz5JjsiT8j#a<7Lqmt!N?9DbXSY z#xjTH%~nTK;;W`ak$ko`Ih^H0(%EX*Q|K%hwv1KSD+;W#J_0HDkiaB<@h>qj|L30= zc|+vLOJ$eY*k%62Ws>|1$#+Zgog^RTW|2Q}lq5e!@?c4}le|%q&mi%^sCSb*NRk`h z1^LgC{3(Vz$OA}DkmTJYS4i?)S^8>{2TJnWB5?9ph0*Y z$#IhGCHYoKUID2f|CQvsBzYpqk&>Ju$CGI_mWBz_Ei^1{47V)dV2g3v= z@jY4fJXXy`6{2`FtN!paVU_eXqFoP@+yb`)IgI2rlDtfk?^X0f~L@n9Merw2c!1zEvwC~=I` zpy9C4fGk7x%^{W|vfK(55HMY8TlCjMv~<$0lEqhv;+VD_2=3aJ)Aq@Ge7_Rf>=YuQ z64aWqQGYB{d@k+y0qyAPS|ExW?fC+u0ELvKOWX63VDEKrtFlRYpOC~&D5yU!S$^Ki zqUb*yG@HDFER(RvF|D2{Dpn-(@tP}kY2D)K+t;01n_%OUd_`T`xiSK_AZPWiw zo&1B}!!x!iG7jP9OI!(=`fzj}hC>0K`BKegzim_z{wyxD*}yY61^9$LNI2bms79KJ zl=JBaT}Uf8j~W?<)N=Ev(c#EJH;)<@v3XQ_7bKN1Dv2*(eu(^F*T|yOXag(#BW z@)wxgG-##dzDIILx8=SD!7+{gqK)Wpyk^B5{mxo8V(^x~BJ3W90%%Yy458n!pMI8@ zEy=Hud`ObJk=(LRkh@8;M)CnkK8ecpdP)APjOmY({D~y*B)LYCe<{ghN&ZNZ|3dPg zBspA??;v@%Bv+AKF3CgXG}@hH?AlWEUy+<6$z3J+>(wA*E0g5$B&UGfW) z!X3UvEpKvoBUP=aQPnb<*ql`DF!fJ$exUjz`r9keFBlPr6Mj%*w~g`7>*`RoQIYGU ztOx8^-GoEW;fg=~N;~#}YR+Mu{-LtF0WX0{Q^)v|yK)439pijGuW)O)O}PRF%Yd#; zjou#PeCbzUr}Ae+V>>f5Qo*}ts$9MWR}EDEwzyjHN0f-_uoK|i=(&lnX}R@>rDAxU z_u4VOo{@Mt-PJb6=Z?gayDNsaXa*HUGrWBiAG*tj%@bAYf#`_sh8hP5C5`QRM_Jb* zKXxT{6Rc!^%FhVoJcPG-Zu{u-ZoEl#o-e+>mCx;Lnte6)MoO+#Vb7F=lMT`8d!`Vg zumIGrL9oPflE$T@U@JonKg4dHi<4V^gf=Yy`JEb_Y`{ zx0ODSs-19$f9yqginw=-7I7!M@!m-I+nq2g)anUC-wXdsz7sp3Q|^r-1)jK3Dc>Be zVolHPoG98_bEK>4pVA8>y$G6q)< z`Le6x?}lxg;uh8Ut><<3>OCzjIP#SY_tcu&oWFtJu%FKa z8^E$@MwN_rJm)Rroof>FaaF^{4i*ynGSn*G858LL2~^+=pSg)E4aG;K=OgNcdHqp@ z;u%%qY?u`B1EkG8R@|ajKuska_2Hlnt4jvVd_y}R;husILxRL1oGhTAIC^B!ahI?ibli*<7zdGz z8;`Lu)-r0Z7cYUhxfc0u6r4prE;N#8{X`ubiZCG<;P~Nlbbav0L=Nu#aWout9(HqI zrj4{M+SmGaQ5a{&AYByEhYB^?N%(dz3b0CDqpU@>)aGGBd82AO3ksfGQ6E}zfe+|} zhU0D9gZ@^u)7JBTU-egfcI-GRzDbb?_TF)BZBPs*DmOb=Y&}lF>j-S3;?zNuqU}|D zPI)F_8yj4{DGCt|XA&^0AQqo-W0G-eXWg{7;u9tA=QP%zRd(cW#tDbu6ObnSO`doN zK4!qCKR#r@<qHy1I;vLju-`De-*o`u1N^hIn)f9%$BV&JzQ zxzP+XYlqQFhE#FF;`}az*_HtIlA}_cUlku`S4O#gDC9;n@bUqZCu-wZV{?Mu65{*}VX4Q7o~xbw@|_@Fo`Vz21RnZkF!!_orZ>_|m!1d2wS6}#%r z1NNsl7!_YC8u5YRn`m>#9#Ly9M5I=JVXuq;DWO`e`2v&45AN`KVJ7T_vhn@BHxrEJ zgnG5&!$B2vd@%+S>{q@Kk)qlG&XRh?R|sU#?Td*~V?U5ZT2Hs|VA3b5c2ccr5nKK4 z*t3|_?2DPs^3;m|x`!}Dg(DEVW6vv^POph{JDcql`D~(=jw3PJ$BdjgsxLEAjqNA$ zp2~x64IBT(nB9vVcn!!gftk*&VZOOc3E!t+GFG&yNTU0xJ~)(fzt?PtQ_+UHeIrpa z2@Tm~G-N*-f-*cagVw=s>!?KK6PR6tuR>Sqh2-gTd&TB(2|=%~Ijug%PLFoS=EOu9 zc6DndnC41QLkV=d1^l!ybQ2cF`z2;^GcaMfjaPOYMy7hhgnBMRy&6Nk?NU9ZJb>Bl zeM7Hwwz|Alv1EMoLby0whW$bFVK3ymB+*AOb1GC0^^H6BTTC2o?HhI*UdCwb$GzC+ zHQvbt7u=zi2XwwI!QM22q&|crxMc@gz3`Z*NmN4lkgbM)>UJLW_QrZf@p}=vt^l`+ zgcMkgcSJLc27ARAm{i~Jffx;KR7doYF|P8QFHlaO#p=PW_X4p&({CUTdpLN2 za%evWsG-6e(d)Dy+nVY;hD6{Yv@ZbVyfKmXibQsThOq|_{NHmz%)`Wspb>jrOvqX9 zc59{R(^5FZ@99pav!)>3hPS35SL+s|G6RD%DB8o0kV7B|!;9Uys9+BKwT(ld6zq#! z2O;Q~y>c!LC3S}`2t#lH1O<@B`Xd%1dBCZ}NznLLXkw_7a(2p20$0h2;Dc;^tGy|P zMxSIF{qJ-D*c(RAj~N`M4^*Z9 zH`!QA@Ii{Q)GVSv{p?9{rRRpNY0$4fZ(o{V=71 zaNcHK`KLVQ_xhD>W|!^A(-4#FPo0i{FYpD?XNvPfB>kt|;iuim_3X+zv!L7ASbC-U zt_d<{YaPQ%>@M*hmDAQ)5%D?Xx!F-U9ip*BNJ;pD(}q7{yy~Avuf_Qb??TRMQGD)? zZsR1CBWDP_UE~aL4&fRz1l;6SxAqA9_7V8)t0Itbkx}msZ(^L~M6tl(MVvvj8k9k`bnOy6`%r0GL3Aa%QNwoADI~XqYiUUA*Zx zCV*TOqfm{#3lS;;A;_)Wd;M00J@w2W?=%jOHKem?~PQ^uK4Cd zDrSHb6y*#MFhXpw&i9=OdO1MR`YJFTmnFrO!gSFw1Qfu8wEL?32(9GY1 zPAL!hw)plD7BH$4Tf~VB$Uf|(Ar{HpIkhZ6gV*f-;jlXIi}sT;wU>f@q$ zt2=fJz7Y_iI;-tZ`H&6pL;&_;5N+(w9%nFfo-f6k^qdm=q1$!X)n4WmkZsM0W~R0Eajgi2yFNA>>*mAHNoDx+^w z*%vP`Q$t@|lZulZ8m2gGiLgudEa)e?#n3ZB zc`{Fl4XXHF#*!Dn?8Z?J%*r(QJveZS~sLpw}HLdDsDlvqh02Hwx)nd%3wbTy7M$>_U9; z+#q(Wp;h=Nl-aYNs`VU(nZF-mj(ZsXY++!)FmJfr81Fd6r)mq(l15`aHU(dVsxBD| zGJ>Bmy`yQN#Qs`ELz0R~PNYiaLi7+;dSZj&#rJTuRJ#S4dTRLbRIC>pUQ*scGQy7? zF=+PBjEcFfbv@#-s?|cLhTrLF2yJFq4aK{$@5%V7HTfn*Ejn@@^U6QTn+W=!Yi}&P5d1v_ zf6{?qtN!I%X8yP~+}*YfXJ8}PTkrSp+hDvPDuaiH-q%b-!{WpydRN!RXv_f@#0DD^ z*+}v!zDUKMvh$rQ`Hbape-QeUpr2lk39-BhE91qaW#=;|Kc!y<9dK811WoA&w(}c8 zFwY6=4S@M&Rj}5J$1Qx^D;{z1_LRO|6u|7usop>M+L!Pp-BhhHRXdwd?V70KWFqCN zI0z=>y@9r4l5~_eUFN-3qz{1N9u9r!(k$>}c3w|Gck+t^&Dv%c)=(I#|9QvuA1LQS z@|Gx1d{1V;&c29|*hRkpE2-I-;{nhn8MMK1A^}VTfJT)MkFnXG#mjQUU@ULAA&DKl zEn*#~XiVI(nBcc*f2KF1<6YbXMoJ&R#(%U2PjNJ#C8!LYq(Xm}0^O!SfASSwT;ZEpD9#j_Z-^v9UucLV zK#?3<`WUOoHJ6A~O_4)Qk$nu2MCf-IA_-6=hpB!B93d^&BvqWWSBSk|KL^nP1eV9$q3QDh+1 zvNB*lFH&C~VtR{A-NA&fW;$)9g+hqW`$lB><{R}_*j)iuJQnu7!4(-OeSt(BMCXB_x}gKk==-QMheOrosX8?0S%<; zi&33aqpwtB%^U1qYVvbalP^)?9F{1uMEa#A*n2DyM0G-Ume?99anq$G*f}im4{M1R zpEv7h1X|YiCEB;S>+yHZp3zf)2p+vigGY!%=(LLdUoQ4>)Kn3&B zeiO?I<|$^x?!S^;L9(^22~aU; zM@{m5+63eMB=!{EVPYcf_BH*2uFPOw1dr9{0Rw;Hvt;=_x<%V(yia^czE2e6;zepzbD50IZq0#LB~QD zM%;&d!E-ep=F;da)^H2o7y5h74OtgfCA)5yb(EnFNaHu+s!Q=57<~B+)Mt8jso2Ax zdXY7@R6pJGs+LVSKK20Eon4AQ>9BF1LL|dsGm@yJwPHS_zMCV$@!cUt+RCdr8sKuG z8R)pfAnV`5y8Pt_=*@05+B37GhMaRimI z2D7c-pzTZzgy#6hoVG8Ohj$LbEfsu^guMJvNFB>T2y$&4RKqu)0qcLI?NFWoa}6d+ z*w%xPlJ{ANc*>`}Bn{urz%zE4vHQ0qgf0H4sqCYefwxU9!{U zFvXh#6GT&}y`m2N%~jqNZ!;IFMJ{x_PGOVbG5SCbUg5+#m{Wb&K{%>nM>7O&x{tM)ggTrItmB;lQq2p2+0$_nMYJm|hw801!I{UjJ z1S}DODmaXODXM1NWB&ygnJAb-j6b}8Sk4|}%Jd8IAV(~FZ7V9!r$RBk537bGz1noe zGvZ;DR030;%k4KO%Hw)p!TvcyuHFyDIoz1IO4arCpz2#-1tD=31OD6#?nt>*1C&Ubed|Q+t-0z{~emtaInMKaTW}~AaSG#1;550 zdnJ-N8^(ZkN@O7l-g4-XR1t z?0>2WURQoNOxyYpb{fjq7~^Kr7ag|Q9JZQZPsx&G(jU~H6t?pYL1hPhXs$1@{w>sh zP&B8P7tc6Qw?W*xjuW@;TX@s3#c=iHP5MAswW3;-e?*pF0kZykS$>cxF?hbXndN_L zDUWI|JR_>jW3e1rtocn*?I>BSP~5slT8dq=Kl{f2v^)RxAil()Xy1yXU)S+~o%k-p zYcNf!ZvhV-JFtDC@*MXH;FL25)$q^le=op~DIF>u<<`z)L9fOh;`?>HIVX`G8xnprWNYZs5Xl}eULEU(l6oaHyoLg8B zssCEY#;#VZU_GaXO^wv2v=_x6(83IY3^Y9qZ!GX5R&PLU9tYDJQk{4Mg5zWM(Vx3x z52GEb^K(y6WeZ*p($7BE?}gi9C$9mg3=Wpy5OWK*#Gy1|hW+m)HvQCVbX~mU(@tQ% zQ)6*x54wJ-pFwBfaH~5Or^y znrG=#akRl|3^?|w3LEmNSLwQ@|8@Gl@*voFKG8gn_`U%f?=NrT{Y8r5*p^5dRaB zAKTddQHMMT$kUQIBH?WECS3X|;FP`<_QDAU`79YDC_ zZk(SlByLC3s8pPI_rNn7Zr>ctk>}M|{P7L`7Tr(;NdI+QXmcnbuEGoH z?s!*Qagx=6gXh=qGKi@RwvF{|Y!#Y-V-8r}i6oc&B_D}e#VHX{HA-T87l*eD`KrGs zzFh^cJfEsHr{V|CkY@431jqNYSFx`0Kob&YbOIKP7xhJS)0Q0|FFY3f3<`?h0f6%( zsAGiMEB2!dp(t+T{S3aZ__>)Zl#GFrVuP8PI2+WmqvS(r7U~t!QYaYvK7Kw9(+7S& z0J7~B_lSl=a|#!Y7kiy3r+*-N4No{8C>oEFCHVOR9tWXdX>`~dzvt)g`8nI?O=BWR z)%3Ssr{`=Ywd6#@Q20iY8FrA;;dqrOE)Erb=5~I|Lxo03Z>Z$AI8-P)+Jz=ItXGhE zNH`vF@Ls-nCyGACnff3Gm*Rt)O&1SnpW^ePAsiq2)R#d7x84=RhIQi2LC|eZ>!yeo zOcDLsikLm#3MtZ;QuhAYP-GsZ+yN<4mn=%b zv1|NBAHM>FEf?|WNAGvS^G|W2cOt7hgoP!(({KVm6Igtkg%4*9a=~s4Pn2<#7>*LH z|Cud<1*63X!Qn3qEoaY><&NCgqQCd0wLMXHEbW1!kA9uoeHUg<#3^M5k*Y<1{w>Clj9C=6-7 zc+5Qn9rVgUh$M=CC-sFakJ-rGp?j|82ScOvjW+P{Q9(!`5`T8VN%0)9KA2aHT(;pM+3bVQ~ zarx5&HgrGEdie)!*QY{S!)jrt;3pP`M-aNGXPGMLkSaE_`1&}OYJa(c7`y`uj48AHI)ep_TcXF6*aTgDS(t5I1iUP8uCV*_PcnjTXey*SYa4Mxz7;q-t2%JDs1B4jbwPP$>tpyG z*n}?v=N9^>oxRwC`?56NFMj!wfSVm%eTyLSp18YXqv6R1!D!y_o%Fc8v0*hkQCpY zDcV;q--8iE?*m%dQdy51%Hsu1N|;tTv-x6+e@vLR%vOXP&p!u$U5mazSS*`=m8<-# zFjx5p7hEm1DcY%jVp!NxD!*`PyIuC=4_v!jTs5a|(EO{R*GPZQyYPL2(f)b2bp7iDE^4&6s@uv-1r_9eqp-0 zpHg05uf*0XJfB%vgffcH&l^U9Z!L9WGjII^cJ-prDT;ps?W*|K(l&Y_y12H+bNjdm zSNRsS*io5=#s$t#-G<*$wm*9w>Z{pX0?m`E1I;}u9h8nQ8rqk9f`6VNaQ>PaSNZWq zHT#o4TkN%?(e*Re1!jDuTTcT}lWHeM%gt1+37HeB4| z-1rrT4Rnz(`?K5e3MVzYSuJmgv@iKv4;a(`JYUk_I8HyU?##aO>gXE1OXVKC=knuZ zuk);Zi62}k_<5)5zaIU)$>o|Dc~yDQtRqb@O|y!vkNZPy3Q^*!9Ugf z^SzS2Gc|j=n%$I|eF)zjidJjR;cCLw9342fKsKWG#>W@DEZg_^0@*&I*2@;Dy_bH^ zrq!uyA)vZ|F?z1Pduv7NH2?9`~^VYMR%L^Xs3=J$AcujtbG zJyz{?TrYcN`SWxo_RkXGo`I~3-vqRS-?3AEAbRQZ>)>O2s(4-pD|#j{_u_%TjR(I9 zY%4&&u?N|Cu+e(a8MQTM7c|voon7z^dro$vJA1#`g*a0m9f0`_^(|dk5o)CD!djSd zxnV}MQnU>vWtUUI&jN=F4)Wo1)>#ucTTrcJ?{a5vH_4-Lt8JEMqG)OiOSvC$P}UdZM|0++Piui zegU2ep=#i8?=_+wfwR30dqCXIz&8cksAOFrSg?mG?;^C@o&A|mG<&N%`xG5qsGHp@ zYA1X$G3s;n+LFB-cW{v9d+YE}i>lTLf16PiFr8e@{)UYWgqi>*>EJ+_;~=Z|8fn9C zdK;!*nb+HB^3efNO{g*zgF$#bhd3T8_EvGMmv6*^wa)(JpbL1J?UN(hCq_1-?FdBM zY92g)@;ij7)NH)R!v@3#4tzCsyAcVH0;FM*U=Z8!xXm`!Kj?Xc;JvJ8jK4xOaUAaK zk#YXz!coWhmx!@34tyMrq4j#QZ~eOrCYXX2Ci~XBOBZ&Pf4Oj+Za&T<7rv zU+>BYd2C_r%EzV{t*&{ZN5r|&{^57BJn=GijPuDMHO^;@Q}A_b!Vx-L!_SGS;cRh* z+b0@kSi$WRtwNN?%owZfq7C1Q)5EHBSMjliNX~G#%*MOoWz63aw)rZVwI%!W9W4oa_>UMoTaTYP z^;I%0bZ2j1{-$J$6buJXBK%2RF_zwp16K7f#H9F&$oVv!0o_1(;tRfO%UDR;FcUXp z%wV&!qs(pXGDLwJ=R1}TkT!NyQgUFv;}wyHt(@c~cXzt-p6VrW0scSo*xD- zpb~Q9UIoFNl8=S~dc^^3g982yY#tL;kv8t0bP~@miCpq*&mDooJ&%EV1G#IDmPGAE zG4O3}V_y_1B3$-WIt&mp=I zo|zJh?|GkK7OwdHtjks2l9b|~iuW*;tTV-a|2fw`SNb=n}&K^T2pOJ$>cqO{hsf_ zYF*tYp9>uJd>a-x>v)rDVSy*q3tT^WElNTB|IrG#%z&91IJIx*@^ z2Z)%p6UUxSLmAP8!1t5S1YVguzpAz-C1YppjZZk%1cV1Hn>@cR(44_VNT0^AK+tnB zGO)<=S!C^|v%PCjwWlevw&pPh0__tH^h&pjwVUSm*-66w$iO$AvyrtU&-O-+P;&)K z`W)4U>^gYUrh=$X!U!XlVd+up@=!$2v$fgt`vjrj+1~Gm2^ntfI?fZdN4sJnR9h3~ zIfP1kc$)FoiW*70?%YkvRpf%*Lp)MAqV22At&e*mx^*Qr2r|II2-vHp!#!k}w} zc`#^<&go%vP7kAVdKjJ4!|0qIvU3<{U^uLOlibc%7*y;_%A4VvKH-~iMS3cxG2x%a z{KY+$K(yN_qK|#a;`3C%AtDyaFviGl1d(Cl#JShO~@sfir zIq%rO;KIxft zvNU;rDNf#BDU$cKRlFlQHIjeu)L!#`#f6CT_E8ZCa8tL|)=Zod1bx|R$^T{@?g;7~ z2dO%&DG*F5#p)pxq*ep_<01hf=v1Y8oWE`Jtto0ynf;y0a5+3>clDgJp8VJZGXM4VjgcCM{AKi;D;2<9PMO895c@Q=bWr(SD-8^}Jt?T;9 zX9KB|Ux}kvbVsbhot@^VUYuRVZlI$$P)e%0Z1O7(^3*(>akSR;aK@>?vdPB-dna#5 zqlA+gAJw{2GCmDF;doibtPPVlRFU5`AmhV;AF&G|>3CQ`^IV7w{Mz#=qDGjfC9>9a zmE*fwSGeP3?a1y(u4+fl@ADY~M<35Y5nawlLgI26nMCwroM~XGFUX3;?Z{qHh{Fg( z`%s{dh=jvD=Oe3RxjGsHtL~pWptREwKqNZJuyhI!pYh=bJbcH*t{#b9f4m+2$i8GP zBNRqVBTdbX7tu96jIJ@9bY2gmYY@Ri*Yq&D#u$l+Y_dz>%qWjt-K13fOFK(`gJ=aW z^-JNH?m;x75JG$n>Jx(xKBV|s_rs*0)vkQPQE^N`b)gNi@OrqK+b8xJ5&gujqfhKd zBBpV7QqjJ8SkXRx2xqcef*Vd^>?fQI@*l5(5(DB2u+x&TCwT3*q0BCL@f>$YlSA** z@xv`Hzk08d-Qu>ti*K(Ee{Y=sx;?4Bp664q!k4;wt6MNektEpPP3wXmcLL?QbNn46 z{8G1CjvtV?Dh*r@Mynqlg_!$N@g-kZc}*hO!$N|z57pfbYWb0{)T=V^&S`3-`r(me zZCfh86>NVuDI70^a5x|96uf7|0UxFO`2S^ET7{k|Mo6}%Q68uh1i#@bQTH9g@3P#OE zOiinaO^C4(qDjcb)LZ2H{r2APFbt53&Hp>+dH&z~z^uL3yZ72_@4fcgYhT~d&T)xY zrCAu8^yd>!&+3?f`(`|QF{!kPD}KrDSDY2+b<`+46~`{%Q(o?uFk{yF{(Jj6&#r27 zdU!9D&Q8v0)dWZqU7oX8_&hwFClF)g+gSNFT>9dr&mw(uq%TJL5{1tusoE6PRQYbT z47o)3Jmp3j&P<7@_9IqEM6H3aNw(Gtww&wR8}P!3qkJ1~K%a<%h?u-ha--C64*f{U z4VWcNz7Cs{qkn||E@%I)6ptR2;;~0zq1qmWyQb_>G0tc0QC38Te747?%>8r8VO&R5 zaS`frJ!mI;CoJ<>wf+}T7wg}+jT6D9ir$d^_J14^TiPqyZ%Yyt)qd90-uu;L@6UOB z!WOo#{VXPR_boF2c4F9~hbLm^cZA1QeqLW0lQbrJZ_=3bz2W?^@CT^$z19@-Gq2h( zAn9*|OdNKOiGI$G%fhO6WBf6*t#=X_B+LY?mGU@I%3};pHBugbw#Ym2$BuC^cF$Ed zzdSC{`DTH4-j8jb^RO)SiW6qXxFr_5XPNcSCm@Yi+dQi+ki;b+xBKS(Y@v7JPwk%9 z559-_#t_@M)w*Y$ZWB^_rNgsCXE+nX&G*jxsU&EsIB_)yMuXrKLGUUNjAp>l0D|rB zg@R%jD2C&-pQQLzPz-0lQG+RtsyJ~QC?X%rVg<#ofnp3A%vTsVY!KBjP#q1b5uSAs zpo%&I(FnGe(WvkZ^S%0yY@S`{%0^=j)iEx0qTTbY>3{CFc@igLrNh6zkMd-Dc^C8i zvb}H`93GokGQ{~xwiYOfWcU8ggYUfqYxf8&i#3I4JzVT)(_oJtCT3l%6W2G0F#F#7 z7&hapdT0B4=rPu;TansfyXn0)zQb_)J&d&1tV@dRuuXf9b~NbOCcXEnK(6kvedWF5 zbXM<$1Y!mbQRs_76Yj8$c@Oh;h&#Q*79vR}trf4iqg^;AG8}`%3vaoB?=l=#Bfu;X zpwhq}G90{O8Qi}j^3>d*&^!9z=tRaCYej^Sv&?xSQ@}OUsE`du4aanqRAwS0HLx8E zN4MdKHv-rNzRtiOHypgjny)gGL;yB?;pj0ObI7bHLWEip&6i8E`TOX0735K!G;C|u zC5j9!iJ?0wU4&dgcWt8)at9s2Jj$sqK_pzo%|z!4$`ko^#S-Dpr@Q%SBV?>hD@Ep-=sPG%}nNRVy3{C@wb|)l8%OY z=R1&q(%26Gw91Zi}aISAl1W1{C(7a(PdTVGzh`-dW&_Y9X zYb|uE!xLd~c#D4;^!|@-1f6E|^`lxM8w3ixnlhCO9dan~2bzkrOfW}54wSSKL z@ecd2G3~Jm4&rA&4>$JX-BJbMy=CJM@n%vbc6}M!?l4=1-TvlHPoj7oo_0BR(B6AB zN-b=!?ciAlPJ(?02NR+ZGs%-NrH4zZeaJ_W_dj|5%{OsOk{esm-nN^6n{i$L{D-H< zBvwyfj9l|=O4uv&(T!aB@WhzvjK%1BY;*UOz!m@<90CjBtwMcvUsAtgZu_chlDvNs z$MW25PJDOK8~MaWD@G5$wa>Y@@ZpFMdv$U=4>fWd-+Zj;&c~wu{FvPa|Nr?Z{NE4$ z(?N`Tn)+!L^$%e^nz`|nKGj;y4(Urh-+hMq6~n*N0kPl>fxPeE2hMpMGxW{-H? zLgd9hkeNSmds81X!?T?|BWC6hDj+B`ooAs_&pi~1|M?c9K1N)>F2?b|9pT6&h96)1 z*e&3Mi-4HU(XdGxql`!w(Q*~L8{r<>-1Y!oVC2vx;nrrt3u(A`pydoUpNm`kw)Mf4 z?Ah7_z_o5pa*1FTlbQ4`Sau5kxuVU2Ovlo`PWzH)JCm0l=sbGQ@ebR(j3b;8+rq>! zr*-ei&WsGqAWvWVT+gcYOuuNeNVoT*PY~$6D1w47=?8y|+VRBNsLHLy$O`wwskL$Q zhBC;w_$E&0Ig9BAGK9fd;YQMpsw&1QkDgyEPZ>Oa!u%KBSWFQJ*|P|59Ntk5PX`09 z#(Rfvb%@=wvkQB299|d*zC~C$KZ6q!SSlYKxKw^dES0zWm&&zuAhB>A*sHH$r2fJk&o9 z5DVqlZG(X_ZbtI9kR!Xt$8xuOs(QdJa~+E&-Fe1Rm5Gely#j6bia?!bqN*B5)GLYv z>%PtK?dXOg^loOpv6K%4B3GJN!U$h!YW}sr)R;0QrpDADF?FU?iK#QyN=&^eQDW*% z*7ynKS`a+_$G4Gnj8+5x*Ua_!lgj`CcyKBXph50_#RNnz?R&A*#}sdO~61 zjaOa9t75#8OYwV4+7Va~!?A0hllFG!nR~EWe6VCuihCbEw|cv;RnWo&rH{lhF)33! z7p1R<6zLoB6Y)d2#Eeszmd=mv5i~&9M=mL8tT)2q4HBk4EMYvPlX%O|B<#bKU4;-v znC$mH*i6=<|IGtMadyYfK1b_q5I2cioDQO?MBoth} zolz-XFq$~(iKY5;C||N__eK8;7XJrDW|lghxId$2Gs$`D1VD@$A(tLh74pi9Xy(3X z-hjy#>Z#d(UBASL_F6-YqRqBJfIBAL`D#r9 zv%^EP9Z%f(YFz>s?8Pb&)|>Wo{Ry*~T!d;ryYA2YIgsSWsq=oU_O^2!GR6I8T-XbM zHO+o2v}LOh)y_4AGUFYS))dxD>4mG&C*&BeR*|!gC-8QQz`@nH&2Xu>Hd}rVtl|8R zIz#f$cOJ<<(_yph*;C!A?Kx20vEY`NgP3zp&UmGBZB%_EOBAU#8p$__a%b|}W%5i) zCeNfq^60n7L8YIAR>=V(DGu7dlK*n2C;z$5rTKe1z)n}^iakfFJ8bcLj)8-zF{eV1 zcvGa1Od}<5(JB*XlK#Z&{E1ik^WG{`W_tdVr6gf`I{!fD8~MjN*W_bW?xy@BoyB{O zBjwmVCz0~kxZbfgYI`K5j1||IAXfK96;dcWmtH*A8S-i;Yw%eHs80~9`*77H$kqL4 zJHL1FLy#`m+c^dSQOo5+FQ%|mbK#-(PZ~wW`)(emz-A0ih@fvKOrKj zKyJ!^h29!bZ=@jbQZ&B@>4@KRqPnBqx@VW*`>$w2FOlUVV*3IdUG%+z9RTDU>R2|R zGu#i*@urw}yAahFDHI!05hd$%8Ow5-{4^>CYC}X_XwqRVOP0F2&i>{AUrg z-j9}o-`ShL7s7R+)3Rq@b?1&f&sKN5IXh-Am(Xe=g^H@5>Ed>b|zLb7hLCeM(J$QFnxKS`&uirRp~dh8z% z>RRp}s24hzJE#_VWdpZ+^oJcDeUsf=DRen^SW*AVw@|!#HMS36a)H|ipq52BF7!&B zwa_b-T}S%^Q4t_B(c|!}_2DCUP|pXre;}~fz?{&$vcSMO%LBa=`sk`i6c`o&nH9C0 zfapS??Vep?H^DCG^~fUJGU9YYgueG`_Qr6uiBpv6w-6LD6V2RI-$Nwetll{}fT7^O)dTY(|)bhALDS1Yc@$I9f$~ z5K$wq7$#|?2{o9wrwa;pcFZem4V9u+*uhlwiRw|?3|3=SoHmak^(C{hl$mnfVO-<5 z5sR%NmbYv@w;6b433=vi;MFs8cbA;NCIj@^lkTB4N!xcOR&8kU&ioko$Z5cL_T2N5 zFWu75xu|vAYvQg7hq%~$cZ}(HGjGx^OUIjQCcR=!b_-)W8AHB~H?t<~ipPET;WSpR^n8@;sR~EukLFel8x~eA6ZHrdR7G9jf|5L5uNoy^2 zQ`u;+ie1H=0&b^lI0`yv*0Ucg3NvVHiTvbSfV7tD0r9 zRTe<5mA6v3~zK2c9d?gnmxupWvN>I~nD@W?s2!p|MEO@rb ztrc*o&cngm1;jQAY^vap4(pC2&P^52o^WAfMY!$vq=tPJRxV*;O_Ev{`zkJAHB-m@ z@0I=?6^*mXM*Wx4B!CNX?w~29)e@n&oZE|>oW=jrN3tB%4SR%b+ngpKSc@mQ|L=S=zOCD`zg)@ z?WdTCeG#1(mR1Sjd!G9#l9#;b2*XZVNKwFkirWMBQ+$N|6ph$B(a~;Sa-g$r=}t;) z$K*9Jtv)%(3*1tnV{P)a2#C!UyTZ7o0<)#}TVRJiVEK5VC;ErnDDjsNM&9nD@Nqp8 zTU@clBBEo#nk&wSi3Lr)a|3o)h!i+zEM2nb*Xi@VJ^^wI)?F5fS53xm&-$Lq`YcX}$CSSWuhwyF_M4?Dc z%~Zt~EcK=?4#h42Qw+rmN0Z6Ik@^Kot0|R3cxY!+-@ge*jVYdk`3JDAh)TD98Kx(q zwpc{TK^;lBO(c#TU(NKdMb%2^i^p9JtinF&-xd$p(SGeRtezxie1tbG7KFeTX>aX( z+wv+H5K{`&t;e8emri;cdiKsqFHz6>EI*L6y4YF3@Mxost?GmCWF~a15OcttqOGva zGA(HAS1cWFo9K~2)!N~{Y0`dB1?~8bHxo#EE!NHj&AVgzu$MqlZeco1L8dF!s!*M3 z6Nuucs#%5;<9}bo%ha5M_JH(3j04o`<=lXvIu@KdnTK3(^T+p%ZERf4{4cTj1I7k6 z>B(GIr2_g51#D>(wyMMPiNlvM1^YnG>yGPAVbSnPTmK8TKS%}pi51$o81bN7-)e@p~;|VXIl%>C=mZh*3 zlU@>T?GvSfjUFYngA0&7y2x4F>2-GOHf~a_64zGV#hwWwyEZ(nL-h>9l$|BY1T5bhd zi>)AQ{aZoSax2ISO0zK1-ri!SXC{;Pw@*%^i%7JObGFcm?DkaL- zjQ7->DRbX4*F3_lAZVeIINAeVDov@BW9Nk>m}!5TjfUbNH-#8;zt}5psoYuvynhzh z*bTy&c#!B5Kupv_e{0|+U^mFV-kVPfJD~UL9|=BYZ}=uIkZ9|j{5~aMz~q*8wz9{2 z)6Q|2Y3KHkN;FK^9qIlP*y~T7!Pi=OF*a`5_2DXz)xX0*o?9K>Tpg3)PWMrpuPCJF z$1B(zX0~Bh7QV*xs(D)d6z$62X@nL9t%RF+&N|8ca#H_(hqp$IV{s}-jalDNr-HVN zu`c8s!w!6C)ASSMVUbV&3P-i@M{a~4UpKA%h`f9--<(m?K^Hbc3(R6~-Sgs&IjT~h zgH&(#Za*plcv|%&_;E*wz61~-ZLQJc;Yjw>>51aSc0HA_qAeaD;4p7k0^0&7C(WB) zH=$6xi*PYiR*D>9Qfr2uu8u{ zM!>>;WJ^DUpUOi_j+s2f^mxfOHiNt&HqLPY)s8!0Kjxy|OSZ5B;(q0WxD=L)cv#&l z7P%DsV*UXin5~gT*S`lZ>h#sJlrl*@Ix&{XEM>x#(TP|zla7Q3q@(;+fvJfuUnJP6 zj4n?FJL`Ax1+FzOdBV5$h7f!s(2f%s8`tR+5PV7euac6bCc3S69xQNtyaZQ|gR~xv zEQlnSEx}q0*a4;{nhgJm670lPQ}|HZCUGUX`vu&p4om#Qr2c(if5I(Ja%71lwBs_i zE=UjVJK0Oo_N9&HQ8veQIHiEk3*kOF)O0f5wo$|?&ZFa518wj0{zNQGR$Lt8^ls!9 z8FNij7|6~yTbd#$OxUNQ)7Kb5nZuC~T~x<>^U{1h+}6q-5U-au>fzv-7cem)d?V=t z^O92o@!*PT(vw)ie#(%ockq9FfjM^Vo8gPhvB_`7;5XyVSp4R{IS2T%W5nB!5ubjH z`0QhIa?MLW!T(3OcJtB~_|w6kzwze~f8OEG@m%wgi@6T-lD+(Sg+Hejn5VTKV;&co zr_~(eEwpgdA6p?EO~+Ql!R0xAe`Z52+sxc|OYf2lGxN#5iBn{jSTLCOn!=(;2|Jt; z*7bqSY*~9UoS^J&w5_EV@P8VGa1p4e*0-5#Qhz5XmKg&702P*Wq3JOgQiU zp&)Q_iwPz9aB~gFn`7Td3G4n~q1jTeCqfpH96f01pt#tFk2liWc!8S1VUB$s$z>e> zz>Wr#`0(_b`K~g8%7BcwMhJZY*Y*f144Jh#a5)}cYtn2tzqfLf-5h)Bq$RQZlf`8p z*~)b^9i zIY|74zWP)2;Y(x5C22ME`y&_J`lv z0ee$R`U8G6HeOurjVd>_qub0i7q-o8L+imY@!Vu{;eIUIerPklbr3Z7+03zRx#o=h zxi)+!yyvTK@VWp>>=ato#r?Sp%>QCPh0EzXL!IkCDF@jSDp=DmK~6F2c$0QoGneFqtn_j?ZNzxO_KFPO!Ygy_GkTkl0rwn%q)zu~Ez zQVySQMM$#yv<)V5NY5|PdfAV&6QB7p8u^r3-@VA+du<5Z04G`?T#ay<*>yAefNh@xvwpNQ2U}U=ZYcx6JCV>Lw4bn zWQ?x5I=U8mB?e27v$i8O!d5`cc-QkimMTTK525vEHQ@yub+eupAuDG!s~9@~u_lO* zZ5%LfxCe>}_b!+>-GJ(|c^}d%KCys>x+&EtAal)Htn`0qzV?JUcE9KC56#P-$2K|K zIqKeT-m}kd7@5_zRMRrHv+v$sTX_{a@|x|k56{@nzI%IvLRzVX5R!fO_GX2YeR#(9 zL?YW1GLgbvu?J_r-Ys_QAFZ=Mbx(YhKmUfGh@UwtcI(M=Un}Y1$4?VL>j20cO!ifZ z*s$+JOx(^?XYuI2@#w#U7)M1n^AovX?Ud;L4uCN`zTev4{w_8I*}^`OpB*`P5nml~ ze~hK(Rrx5^nkk}eWO;oE`yrau2+hSJkA7ZM%&vG=UqpP}Rmj%T{hrE*W~})U#N0x_ z9#(|tdg?0SM>m+XzFp{mxfe_R!|t=+7J}JpSPsX`8+XAN!}PT?Z*K#rkcEi8va4`# zGdhv!AF4;16kmG;svXIaKP19pvx!!XO}6-ui+L08^)hX-Y4<*;9jEtqEDKzj>hwOso}l9V<>s2-fzJ=oPu1v@ z_r<;pWQ^UG&XUITy~rV&lqEkR+SUr=gzXFVNF6ecdR^MfpKjvTM#OxGexClbAh9$# zVj|JoBOX;j`-rWKsHYcQ^M~lF_98@7Q^YJ*9?6PS>1q<}@JdEz3!M-Pg@Icm<}mO= z7XCcA%zNI!|8b3W%u_mSl;o$P@tg6~6#V8tHETZXm?!mxuw$N##&7zQQ}CPprHOq<0k);+Yfsxehi>;Z`O8_B z!D3%ND#Bk7=x8rMI?zs;-y~mtmB@4o9nAR_o%!6N%UmHh#(9RgB@ZRHHZXW!=G2Do0z zT)k+rDPc$GX>6i~rcDKb$~gWZZ7SmHMDxZDx-00b)Ty1xTdPjZPSwY`dEKr4&B`VI7n^{r=1yvbfXkSavmj!qQtuA1Yu{rTJiV3q`cjv5MW`|=qyFvXq~QGlH8r<*v8*v_6hsy zH^l;O|4q4ot-q-yT+;Y!@qS6uuf_W%+kY+IFKPX?e18;{6$~q3SFs=IEBPfdi}oOm zD)VK#g-P|CdCzkc02*2|+kAJ+)$R{U7Gb+)l27XFq`dQFSJ+*+8wA3E04AtcLaj@qi<|Tml;#Qlbhm}FL3-*{f<4yC@=X=g#j8A*42{xEco9sEm5c?yvw`m6G zi;adomSxDVG{m5hd+zjmb$G=NLxvAU_@2Li#|X-MGo@i>4xC@%e%$2cR;_0oQpbMW zM7XgZciU0`?=2tS3xMHNFOFErd6^OSe>R(Ne-Y9MuuHM?lSgu4@nOM{jEFqUUI={tA($ zu^MqNtWz@<89~3tj8brT?}MfJFAmSkMDrY{ZT;WTFv@XJ5f2_y;owUWQ?TE%i#+*C zLU=FHJGhtVg^RZSPP^x&MLv6UX#3k&JA6sE*>EUDoC(BXM_f{L8XGUaiZ!~+4xaV+ z62YIRWyM7v|EnqnSmaBO;x^04ypPB?F9F7sXXm3x1>*-^OtgZ|;VvdR!xvs30j|;W zwve-)Cr}RlL!>z0g;`^kNY9aP34{FFMLKp_B2|3U17)%~6;4dXba5ATJ2II9%g8PA zh!S@Cl1E{i=UI%#PGfh3XFn#+j-3I2ADA!OC59ci!zS^2sC>Zq2ExitD;B8e=Rsz| zM1_x6_)LXQRro}O=K*Mj&s6wS;El!#tL|kcC?<^#mb`v4 z3~V!;$RmJtJVRG?eNBM!>jj9vMS$*Pu3>VDei4igP_nea*wV6NpkQ)SJMwUd_lW-^%efS z4Ph_ZO^NSBk%bbqD>ucwuAM)xqBX61Z}0{}bZqNTMNDX+`3hWuL6^gvaS**r-2}=9 zGQvQ(zd{9S{2N`|{DCly6Ihh!i;Rre&AeU$0oYkh5Cx0_IZ1Qd%^MD2^M(@}C@Y>| zBl5P25g2Aw#Rv?OA=oO5&7E=tCTC73=qTg)zFB5DvHG>sH*g*-0~ggG0XXVK5y9XQ zh-5FW9}zEbITni-6RRRaFoa)_^qreVN7tAUM z0K`3#yFh#`wHXkUinq+3k0tu!u^dvo@7aqDtJm$t6eI=W#B5GoA=pMCo+f>~y5EBEZ!`ScYI_|?ZI`uz2)ObH`9r~; zZjcckh(UyN6gXUp@jB~-byX>92$j!UXk=AXo^ECg)YI)&ek+lZv7bV*Pss!0$zr4+ z$a13$oNg0Y$8DMTp5uneESR}3mQJ&I?T>_`l8mxv5sN|+C0NpZax_}Xo>|;#Lk&0c zV{aAp9X71lvAqU5W*=_zNOreSz1e)}Q^JzmJE^vC-C`&duZFgr;#-KU`r3fSCWR!# z7T;TIAsunI>P`y;3Y%;C5W-VcE5dDMYOr#L(|oVr$)3T|I-87|EiFt06CMA^H7~nm zE?C98n%NW!I4WmTfZ(W`O#y+Uel|r2j>g#(6*!t^Q<%(ADP7>$0%GQSVfQryJ2&zycUz+ZP2%Sp?aA^Opoq2V`&FD0%!&XS#JP1 zISUl3gzl6f1wp}iG>izw5?6;HxQfhJh>{6X3w+7fhX@8xvc8rWQC@Iujt2po=!X%# zTqNo%2I#pmebCJtr_fa^4na~<_K`dE&xZw*n`=evs%A-~OMuON5?CNESl;YGNO5CF zOSgzBIyKyL&;=iE?7&@q*h?bs?Z8%&P;eTL_`IV7E1P7RS9J{B(Se0c!N0ny1IwQ5 z`LMsle_clla={N);<^s5beb)Wa$e2B9s^CFXe-I{?hbYxuqL)7f=}F1^LO?j@Ndo} z-)74;))=6;76Jvuo9#V+wS`?oky;Q0Gi11b0|9n2znG~`>4NTDiq}<4!ECv^8$sqw zco~RhhJZ!42R};=aQYDOs!~O;tz7`H>Cb?~ezeS`4>p)%55aSHGXjZ<0W-CvfR`z? zcvHaHyCvuN^8!{MFmVeFR3(Bq%{O1M7pq#h6(iY;iQGi?A&X0&Hc;pkyBs zC7bMt{tHs>28%wU+EsX&;cc3MRtqvc4rg~^(+N-4k=O=34Q9Z=cX zCD1Kx0w`(Q+8|&0m~RByQZC+#0u$e-y#s!jfQh)K!@^fJn?6B6uMkLXVg+khp`;*V zX_Bf46V^B+k84u5^}&ZT+niRl2z7*tXaf#nOI3u})I(mWcDzFE1nL|}3)PDm-*d11P3U5JGjA3$U$00C9x}%DEUmOy4f+ zV%0UngO7n;2`8%s>gFJwq7T}f32^dyra?gg6n7Ib4E~RhUlDE#Vm0AjPWUX!~7Ib7?$nxwdxP#Rr%65@2dn#0&cufNx zP2vrkFug0zz_2?m6&aaeN&uKCQQZods};02Hw&<(O8{S=1QsE#TZ=_*0+WY1Sq>y5 zi2BFjIWcf~$3=O0$AKM*GM8>n!Ls%;NlLdHR!RP)5w>)wF4?F zRPgT0Bp=XZ)L`2rB^14ACKkaRbcHbS0k=@xAcM@PDtoj>w*AxW511u{GSrrj0%2$_ zQbzT#>T{vHHdBmIt}GhFAFRJ*mt!&X*I7eE?bP1xh4=htRc zwIZRV+bDQ`oz@9%mYje}li`G#l8TFJ4Ign5qY8WuLyRi0klslpg+V<`Au1?Xx;Ib0 zs0rVDesvWkvC2ejaN z&!1*_nR>2;xx26TT5Q}%gbZcRR<;N%Gt}a9PKl zA-FK$cHGy2PZi^%Qe8<0dvJSe2(Hz@(b$FIxYxnyNe$=s2;}_$htQeh3IL~AT;Pf{ z?$Z#r0=$B)q?{_rHLt>$Zx`+W&{_9a+-?;zaK-IOA<|zg{SQBib}3O~Rft#>B3$|7 zmEWTLbCf?u`4gpozDi-fkwU6^zgmS~BK;4S`%}CFOrbw&$fdnv8)$@yDoIx zb%A>}c;Ab|ld2o{y=?H#`-A5UE_=aUFSc=s5uSw+xDdm$Is)gMD^VRVEw3&oQ1Uqq zB8ls$ksVC*Ty6U!yqZ01KSWfWK~J^)5!vjoUEqt*Y`EVSdslERiQN-jZTH~pJb%DA zPS4vt8P&*WdNuC#NUz3K9O>1#IwQT>$`?}{bK6T=7x^;Kd-@ji+vawyx(1s+C=&=% zdx}4O{P~DKXP&j!USO)mttuPHuy;aulC1uN>)*fv&A{n@xjI(Sezon--FDBx_OpY| z|L@0)7TD$?@^Lss9&oLSeT-ghA7e)@@Dt9T2>z%`0CqY1+p)vsQO6h^mjL{wzlgQwe;&^Zrv9^Z)L7 z|6eb(1|N~oioBpAl>Co4 z-y2_va|Sw_0ag}2o`RagvGwifF+44M0n}7)8=Uwc3?_Paz@;X74|8RtDDu~c#0m3} zX2|6PaKHrlVH?e49K!D=Ra{UHTY{717s|m#$qr6>w}7UcBtN;3oqJSOH}-k=#4VDZ z2P=-)YXmjjM#&)c3&b^b-yoWg8ett+dSA{%up3wsv~1y1 zC-0d>&*o|%U^|#tswiDk`ETg2F|jr=U27^9{yI~e@Yk6#g}>fJW-;GpN)`S_6Z45F zFH<7?;=KJYz?LLhMVgpWz1D%bX_i4n8x{dAzH&stvj1Mr8$E(b#G*MC?#=h$OCHJb z*y6}}X?ry1rD1Ct8yd}f+AyZAui|HH3FUNjWmUsl(%-D`aylBvHi0*$nV&U_)AXg+ zp@u6rlY(%N1ROP+Nkh#v|Lx=`1O_h@2J%eUfGZ9Ni@g#i{#ZiGV-j{h$>|M_OMWhn z%eOiE_c|-iV8?{_ax4}3Hgi%JmXKNAV{b9;v4@4^OTvDLc5}^g1hjkF|M5x~x|^3O zj)h^*1uoA<5pCRxB3k6F-^^)g9=crMZQZO6!BhBmOv13zTeF3QJ>P@OfLTH^yTDr` z(#A|IW*##tMIc0qtUx4OEn+4;yg{ij4-6v-d0Tgqp50sT6P1V{HJe!+^F6Jbnf-;H z`pwKf7ATW>(e=I1{7zqM6&XwV$Fs<k zy5KlDQ9SusLcnK{nZ4BvhHPaVbh^*;_5ozBE|Qg;BF{=QEmfFE>sfHoMtV(KM9#q2 zZczpPVyKslAh&O_#7_H9mW#U$Z00?0i4vgr45}xitOoJ|m*N`4Frk~@W5yYFOq54b z=H#^*xTX3TO#k#|N+DZw)nFm>!2kb*?EeXwluk&pYLJ&CWH>2FZRLcftJnf9hW&;@ z1!H+j=vS&yte)V5B0@`3-9j^8H4!245o`=f=#Lx^|A)}RhDyctBy36ER)K(GrD6e~ ziZC=f&c-tgdLN~t4MhwM&xVBHqDf$%2-UEgYPjMII?%D=H2mh~U$DKP=Xpt@Ka60r zX!EpJyfGRd;c~8lOO7>DI;Yt5!mx}u+H8s>w_U3S`N){z==DJij^ueopG45V> z0;=FRHPf5w*tojB1WHln1rDX8uta%4GgDr_z@Pr;*m z0;Xvy+c1_Av36LM2~t=}#1i6n+OU+^g{8y>3_kp(uo<@iiy&M|{CMSU*cFRbiKWDQ zUd6;M3meMgNUr&29Fi2jScQf=o)seIA2;{GqQ^RbhM*@?5x7bi!Tz|uX1rjWT_XN| z@ryNMVD^jOUGc&VidB-7vGz!cAji3VY;$Sh{0HoJri$6`8d1CNVPd}=!Q?jpKtWFL z%jL)VS^nime&VTHbk!nDpel)r)7xBr1U2NSR^;f@mLGS|cIlEyLfE35bHSg0Z0GM;up205Ok`g~8^4#Ye6( z8Viuo#sVbg?-5?kQDdu}f9C!S79KHG&!xv&EIYPwZ4m(jmmazD*oB41229Fv)iGen zQC!0X9$|=L=`oB;kC@8R{})s%mK?)?#@R~ovW+j#2=|aBM~oeIQVs<*lmugQ$q__9 zbIB1iXg@)Uwg3G~j`yOHFzJ9}>04`AZ8bj=OO8?qaf}@}UP2>)kYQ&(v}FgD99t4a z_$$t{2}#w7YyE-4f+f{vvRCL#uhf!Wp(T}_0bTqDj0aI+9N%NszZl`ME-uINn>l{| zgE+nJ6AIEN6=cnXZoGhfFoAj>+Gs+T@HbA7WBH~DZNlF)Ayd5HKA~Cow@;8``PK>4 z;pmAcB;tKi#oMVXF_!m9qN0O!w!fjHo4huK%bRtP4N#TTk&#^-|C%#i!YU`Un}`8? zV`Lk2C$(W@yi}CiBWpQ0g_@1jpeVTsH!@RVsb`up(uAy~UdB`2afA!r=ID z0Tw=mHvK(cW8w2!bx^$=nZcg(o{8~t;j>44uHdv#o?DECPq52kgaNvl^pFp(ej@vV zLwW6oDtPDt6f#5QnoB;wlIJ$oKGrj%x3T!BGcAYUE<%;uWP@zIqdSQ-%WAAp+NAu<@G55snA2&~aLxZ?YepmcH zt056%yToQqnXH%RYhmZQjpD9?vPd7_QTy&72K+=_Ytmk(`Am2|dQ zC2V1-Asope{8p9-+g3NpM{Vm8U}u{EqWJJ~i%is5WcQWJI6I30#itIqO9R2AoQeBE z->nX~^Sgb+vTK4O9s?i`P0&X;-wT0b?`rSH*I3i=(QzzAo64tK zF;{n{zw9eXzOR+SgJaZ56(_^{yOO;4O2C=^mG5)$;@*9`Yik*$ryqq&%x<)S5uO!U z#rZ{LYl?H`XO-rZ+Db~Y)>yy(b?fBe@P$QX*1W>Jvb?PPya#f!;kMqs*g1Iw$`U=8 z4oSxb(YEH5S{L7)oNUb^_3WJ0)|;)9uPZBZm6ny{72X$n|Fr8SXB1AJ7Uw1=cG}?N z@(N3HO3Lzz3K!jxI>3D()|w?Ubt%jvsgj(m>^oOx<(Ep<2fUxbb>63PJ&=c@RXHVD z`T2_$q{;*W{9j56**W<+s(54qfru|5g+V1PYIhLkpH6`VE<|&n{H6C2CnFcdVc zsN_D^@{*#gY-FP>uWV&m4x|oZ=vrBpmtX2C&dMuUQBdH5+$R^^m!DHwnhH!(N=lsD zt>qN1y8hNhIb}i)lXD91D_apeIL&F+dDevZ_;^kGtWu(cd`49X*$is7{^F8tVPv8A z3-Umt2||q=H-N8!*a67TYHPKhW~>~OgZ3O$XbQMj@o zJ*Tv60kt`0C^s*gEoU+W)=J@psNQT{P_!~X+sX>QFQ?49&}p~k7L{0;4Ixg{=wC3N zP?29SUSOe%PUH)u8BhU}zX-1b6@e7LKzf6lvEY`at7)rp++{^2H{a|^apmUa=euO4 zq)d#cs1R>}k>UEkzr?nn1Za;m_w6kl#bCff9BwU{FFg{+>{@@66x5p%CeG*^0RYF7UXA@mJWmu zeu41!&JW1DiZ6r%?!XF%H&DL?lt%zRq5v6k;9tU_oJTa&MPcaKL%E|)PP_Ci0pv&EhDr22R>g8|BLdA zmS^QVVVjM_x9fg4>rQ4zIR|*@yY06>{KSI2IkyG!M)%ZG>)<~~j_3;6;o&KBUw>;! zX?{*2dNzt-Ne)y=K~~wnX6Gzld7mqLWkGRV+3GT_D7Uz*BrdPiRaBf)2qVrKZ_R~O zl|w-cXhTv&Tv;nulQ~g;ki7RWQ%JBCBM& zRlK3oRaTOfS5_*}x87>~`gE%in`N-Ps3?Eh0CD40tl<#lCyUvJJ4Zs10UW8*9t!%0SKLWryywISZbQ&U;UBBq!-m&1 z<}VEQtBzVEwjxcm&zm+5>uKSr7cBp(C+ z!_yzgf5??#bDu_L)4@&gHU0fOr&A87qp;hi_V?EScHPq7UpW?j%#u70*oA#j7jT@- zicNIsSk;RM%m+*atOQI2oQ{gf1hfMd17-l01C{~S0@eV=JNx@vQCW$AhX7LnPXT5E z>Zrg5Ks#XS?fv~tfYxu~JzzYbj>>HS%m8c#ECXx4f+6_h*kme89DU&46`)eSn7mH5_<71=yW|d?CMmfZG8r-|6pjfrW?jomz(2y( z0@|@0nplQ>L2IW1)&OP#HUbs{wgNT-UI1(ZjE(}ofOfz>zzjgkO2h-K1^ft*a~hRM zk8?K3NUyIJc)*7Lh4iuOsT^~q&jWVVfiJ-BM?oLZf~!6!P5>Uz4%qOs{{AV%{};*! ze(N6){|dYZOb2Y(4Z48E9sT|Du)(GQFoWJ3X-eSi}IyZ_SPe-yCxuLuv=br^K8 z5~v+PJpr}>rUTZ#(%(M|@3Dw2o{$Gp@6tk6TSh&0Zjx2IGGcUiEt^Q-`(zPd^kYXD zzz6}BH9P*-Vh#$$=6@>T%D{ga@QILrjpy36@s^rU>4@=#Nr!clE*-Itm~e$tUHld_AT(V<* zge@f-|7%tJX2w4WrJ~|*GTP81VJb?6E-A+AYe~_n+DD9{Y*Fn)r4V1CZwh?`#PN{N z<8ao4&n~$2$CwW41$=D`mCdgmzL8GYR&x@=mp z6-HmTag?5nXsTt!s)lP-4L3$LTz`iqqsLd6Hjb(Z-4ynS&ipna{rtNmU-}8j-E`oO z0&kz$-`_`e26TrI6RZ9jrNplpNGAdrodr5pxQ&P9#Sb$xop%Z5L!W;m%8jOJifWhragk~T-q*n)b1o+H^ zTfZ)Vo~XxQdT|K#>GY0+-bBzVCcTp|5)J;)6u&5c>4=gJJNJ2%S871H4{K}OiUJ+U zbWxgJS_)GBnn1VX#=-K(dg}nbbr}3n;I{|ik0bm?z}KTbx<89Qz8^nemqAuVRb=W< z@Wl$zZpIhuG%v?z>a5N2@rERSmT4yQmq|I6wvfIX@hc+qtpL0AW&a#8p&yPy9_rEt zmsb<$9s<4w_%>)HeH_>xkY|zOn81D&RfICwDAoP=K3(qyW_+y5=4bS@eR_qy(ZtSH z|M-%kiT=3~{X#v)Pnlpve-?uvS$39+f&Teca%oZ8!{0ww$o}~_;-7L4E{A$VI0}3t z=xQhr-AlS6`Q8l~TTS(HFW$ouoC!j7PNrhYgMeCsgy3&3v=!p|cuG>InQ6XEvjLZMszeI%Rw zkUo;ZnO?ez56$dt>4G+LXE`>#Wi5hU4_?1M*+zjV#2Auw?>EU2Mub3r62qN zF&$k8`AvkKpq4-Lm&(;trV zl#OsquMKX4o{rfoZ*{XzY!L`CSU_Wt!jOQgQ42dEV&pXY_e@q+k$QT!edg0V;Z zVg^KJLL?w)52i6V5ter{Ouvt?E2bH4;~_p9_?idN-;!-2{)C3H#o8lW*a5%NC;rQ5U)_Dlf(#Lol% z_(P}{O0Ie+U+KUftq78@Z{UTJuQI$ph4*cYqqDNnONQwS;Mgr%)*hq@59jqNE9JOU zhN}sKeV&R}&x>#`APhVm2v@%ae9(ft#D0QN$2Xt*n@R6P>%G*jf&>D43$#Db&BjnP zqEX4kF#HK=w(1RJA!g_l7SJ-EO`uZ-I$dlU!_W~*=JU*_m0x@}RDS?eE;XMdFf^aZ z5Q@eoqrRkW543w#K7SAHN6aVXrw(+cZwrzirJvYt_29^%V@e;)Wg zvaKHGi}SweyMV_$ub3SCw0vEU2yFFjz3Z5Xy#UB%SMNGul{1$+;C6x3CFKi|how&b zXMyO2okR6pGVpcJp#KZV7wa<{_|`uSU!QE}m1Gj{+u;5*eb9t(*=>e=-GIPD^#Mur zKjsSwsSL@P{nRzupF*Ar`CFWJHq-%6^B7atHCp>v)BZT^xv{?qf#(+yJ3{{c4XrPSdtT)3Yb~e}`Ju60M*m~ zSaitvMeyG-!yBS)2{GREKXz3}%tZL6nQ*Wjw&Sv}zh5@>;mft3UJj<7xcpi;U$~qv|8cp=8y&VS znn8XSP4^qorca{7o|r()FDKCb+=Qp%Ke-7O-Mym}ob&b`Y3^T#~Oqi*0oK~mP z{e*7%!&vRN`VDY@q?`VItmZM%{jAAUHpnhf{jqgK8&M%y{sw0^Ai$>^;i@ce2F zBfl_4F^2T}AG;#NG6DW?n3A+PfZq;$_p^|YM?$U!raHv*!R1k1@TD2xDU4{^M8^ zNUR?RqTX@0!+CMs^se#RZz5*EeJ0ZMhw<7|l*QBLvHj+-|FA4NXwm*Nitd)nT0#(J zq_Q7L#>R()UuRmTJrJ0$ABKcH=mg>yA*Sc`upfn*u<8ClsOe=>*uRIF9x;XeJ?u7M ze!~f*K7H&tUHdQ7*qtWrtZ8h;DDBbFKzEHcT^JYkc(`fLxUe(fCg0ewy)4yVj~)BR zv0+EYPF+7v+r&IRI&LzYJ>!UOh~V$9BY^g2qW`gokWo!xPe6dhBm9r)=OKgx{taR6 zu%`z6PlWw)s`3-`tEW6$A}yOGY-p7*zDL5^3lbW?Mm)oBz%w?<4VV$?0O(iGxB=AjxeUb=3OM*|L3TU)g+H8D`Ix3zi{RX}#Ui#%o z0*y%#;^2*sk>17r{v!HUtMI5F;ioylr%(C6{;$%1R{4#5N$mfRe&@@23C{6rNl+Eoh1 zDR`@bixpg^V4;HR6x^iX;|l&t!DkeFQNh;~d{@DX3XZpF;<-w}I0bK2aIu2R6f9J5 zor0Sbd|bg_DfolECi;NuGZO2KCod{M#I6ns~~iwcgn ztMnC&Q}9*=7c00-!9oSsDY!|&#})jQg3l=UqJpm}_^yH%6&&wS=_?qg;H?TSR&beu zg$k}yaFc?MEBGq~pHc8d1z%I}T?H>HIDVl@U%@y9Z&h%yf-;oR--Kw37A&~g8oOxm z-P5eI;%3F&Xq^$CFe83u!c1%IUD#EZgu^ESfBmd!+V$Bvxmhdo%kuM<-;Dp&g)53m z%dCOe)rBtNZkUxl2>pcum#!+vcV*|576+5aD{%*-K0gcp)2V84rE3bxvX%pul}Nb4 zaA8AwPTYNkE8~{0%*)TdJ}+B~Tai_|LW|2@Q;1j+mX&BC49?AzVyE*Uj|=Zga`Lkn zLAi?a%d|KiTGHamaEW-Tsfg)6rtt02dPU zn9R@7;<(Mb6f8(^zZ=0xg6V;p;4P~l&yA=>NCiJ}B59e-a_pLpa~BnGv!@}M{~W}< z7#eMZP(`LTL^=#OMd1yg8x95I08RKe;v4pk0h?8XFcsdgI}AFrfTsy##5e3A1M=P= zJ`B$=Mts9gnFkzc8}SYM%7BJFg&^Wl*)zOiE{e8|5#O+947gZDF!(pZ8SqX}qOD~3 z4SUIe#VRH8PZ|dQ3aw?rVdRZC!#*>hVV@c48}W_&-yam;u=fmT*oTZy8#6dQ#fVuP zg?~PVJ!?QC9r;%BYPbn$6b8oMtLz&C)*6PVie%KgMfo2Hil3?Ut^s2XsDKI>@eFJ^ z9D(T@_O}7C%rWRO(l?MB;b45D{u-41Za`}g0jm*F!A*dH@!PJUPty$8WDpkEq2*T( zd|>?iugmxbJfy}8D!Pyr!%g_V@gEpJ@fI21fZ0JA4)SU{RD46ep0-UQS?y_?q%%kO z27V0rF))9EgCULh-70>!iXU8G!SVM5#qU$``&9hk@WKA)g5n$F#WSBX{$a!)6~Mn{ z*zxma{tdw}%FRgM$k!V|>1%3yrrBlu2D}e^jQ9q83oir9&luk|sQ8DCf>lI}cm{kg zDE{iCNh0LxB$>X!r;0vAeoo^BX&K2hsQA??e&Bp8X;}Fa@SKBNrWAmL(}-Uk^fJh+ zalOV!JCK6qb{R4(C_<1|q#e}61er(gk|ArA0^^i-7XFQLQ)rFj-r>@Z{jQAv<0*oi z|0g(o|BGQ9Vc^c_M+0AnL8y%WaWFniGy1*3cwIC4v%z?iX7p2o@uU3X_QCkk{{Ce! zK3p^Ui^2FY{{CPveynEb_rdsanxVG`<0CXf9}mVyYKER2j34jUpM&vc&CrX3@fOX{ zcZ2a!&|{MVQNkUjU8Wg&D)42PcDZKgpTYQO&Cn}@@mQQN^hLmnFbzuE&;x_;|Kgv& z2|z>sVl<=O4#s~)Guq=|JXdcAKa|T*SW>~|&-SHj@AnT5jlEq!grM~Wm(R5VKeT+< z4}=U;J{(Rl-SUNS1U&b`9ik2JZ?qfQE5tvo^o-FSR|8Kv+M)m=aE`=(rJxg~9kvaw z-~T1>W^JD82MuaJ7x$R7KJxf5JJ&WGW?XF@NKn z!SIlWB~Fr{Rp z1b(=BH^OhR%J6I_d<=O=RQT^mw6;;DyIQ52r|>U}bYt*&zQOrDrNSRq;ng&-76Rjh z{A-Ca-NzN3I~2Y_;f;D(sqjrf`2SG&9YOfL3g4mdY;Sye75=codnH;k>d~0jk5_o3 z9*b`uoUaA2|A#Bj<-oK28z#woSrwf%#3R4i%6?PBKW&@Bmj&Sse*EhTLHv{l@$<$A z{Jf+1X`3whG3+LTzOil*oZreI`e9IL!{v7}@Iv0M8BG5s;*sCVpnMf5d|eRU;Kx`$ z3FfCdh@U4!x#@oDHNwB6=mhtJ9|J#`di5tV zq*0zR5LDtbpOJX(^Wn2l;rsHX-w6L>;0@g(-^kzJjDY{!2zc)AWx86yVEVUVP&^#{ zB_rUAfG3^W^^#7pVyb3@@QowjpAq52Qr)?=PMOH z^`gZ0srj3y1RaFmF7eG0Xm6QuxyfZw;!KZ=i!7j-UI1CqLyXyj9V8ScC_j zh6)~o{_jShvwH;m-wZmNC7m`2wEhvob4v;Nu>|p-2K;dRKdQoK29@V&5gz(BM@H#gEFbRfXpj%fs@c!pEs;`hn%00;Zqfz$ZLk@$13pSv~Fdm8T8{u z2)__`@?WgvBvH}-`3T{EKLWmO1boj3_|a2jz1L>R1p7iITrBX4i2#reypW$DeY;VF zA36_TGXfpWjaxun_Z1Ws;=akOk}}O*QdU~FGB-EQt+`wa((ky-mF!GQce&uR5Axl< zAmCMYk?THj7nnNXN0* zE|OVzmo3HNGJH(Na1W#+c14I(F7^5~-XrF6O}RTk`~Tn;w<%MS)9g0Q_3g!JWjDy% zx14EZvv?7rGc_T>b^r1;W%ByP)GS9@mNRXZ%i*}8B!|~|ISSzO6G$s7&Ud=g$`TR+ zE*EpU8OiN%q%C)*yVB!b=?N~!UEfT*!Ik>Ww3(L@S;P|*{3+k$zY5oRzay~Rde3Q$C zy+SVc>eUH1x?F|4bq?1r;?Bb?2)Hv9SxuXR3=|A5P2@FUhRcpC4oiyGP;i~e%i|pm z5X+W0#Qc1sawCX49IMh@Y4eA#lFyQ}tTZVr!_~uQ3Ta@O@`hS8AqV6fU;;A>vWlH)rO0H$EO~)mk-9){IO%qP4_K(lN(%*K%CE zmy?4CMivnG^QMO665?HHxa-bcmQY&eOhXw5allfa15rRVm*zM~c&3piWFQ*}Bg8<4 z6K+Z_%F0g9E6CyHi1OmZ3@ougU22dVNl6%7jT9eJBH8}p z&AT)Wx>mGqqXe^7t#;+EEOhe%RNli{mQ{!v7uTV(lZi`q&B)7-&&@B&Dl5)%e>bNr zZq|&MsEipi*hBKB=iHU(gNsXwmb*YJ+nMc1cMVi(D)gbhu11Jc>dYM}4yzzPuP|pM zR^qkNQgZM`_@8;xNc@A=7v>*vK81h#J=p4)8waJkdbO(hSbFOAja91bu>wkX`A;E;LnMp<9IRQ4lM=A*&b{mS#Irzv(VWOk9-goWEen z64%VQS$LgRHY48UDqB%f^nDj@K*e?6P)j96B?ucFba7fyeyJmSC_1>28d`Hi>je5P z1$~ublqztRt3dLK4rSP+OBbMh4oP}M)rh)6Y2b$CJR`@h8F4e>W{X6qLRc}uMR}`o3X_a)9^eZ11W50{nCIMFnSzh z5Xu_Y{-XPaLihLI6j1*_i9?m!SGn%Wxi7C2#-F&M8dskq8!$X(i*H5@mDa*6L$v*k zIpM}3g*i+I0J;e?g@s@k$^jk39GAK=7@-1tczCn(Q-?H}#c8E5Ht_k1RTygo6_X2h zHKTdBa2<~*JWB4%J zWeAMFsShnK!;V!J$8ddFko+K1qC4=5#|)7hc8?2&>mKI}=&u1IuNVV0=&VwuV*Pqk zG)>yfK|(f6MJvvSSOxA&E30Achapw;`MQ?bu+P~^aL6`dyWd zv+)rY^q2+cF;>axodqRuV<6^8z0q}l7C#7tjS&>4FFvay>N&VbjA|IsluRotDIIRq zD92<0!i!!E{Y!zIPjaA`?!x2}Y&-}uXP1UbMCnjL$;AkKu#Wf=O%a)YW8i6I{EJS{ zxQ*c(X8*pOFb5w7Km(gK!&SPnxVWeUla*)_N;l&L@MR@1v6U7?7breKH8_u-s-H@s zpKb`yPr^>3db(e<Ibyc=OqbMWU#bsOEeUmJZX!+s6lJ;*Bz zE{}m0anoIxRKW=JGh4-~JT1kx1ar-c9j;`@;zj96etG}&Ni)nna4u}73*%UKR$=a7 zUWPMOWfuYR!}r#KBL~BxW{XXTFLmeP99>>+o;%RkB%kP)g&CJsf}$R76vo~gFVKNg zQ!5_*Re`~myJ)4+z53B3b$jd*^6r`uA0Sphxq?NDR>SAhn4EoPYnDYG)Wil}e6E?h zp;|%k@jqmSPF7agO|rwrcwso*=$M&PlskMUI7dc=R=}qJLp}QI_={ALF>x!ZiR1eFg9Ry&n%|!JqAC(rw||O@Ng3T+!Yp)e;v+NiBUJ-^ z+F-f)qEqxT_U95VEFlzO`gK_Q#HiE%)7lm7Bn$+>ev%MGdB&O;A0!fAd?BK3KuRI0 z20!1Ky*sYGyS7*HmB97d+nt@A3EjA}@xv6vd5`HxP5b~z#H)0;kpQHiNe?yvM2)^c zoUjiI-mXc~ArE`F*xB$hPz3RRVr>%=5vSE|dpuPx7%mUjdGTFs{C(qo_iwvQrr8C2 zFx@+dqm?S^fk!IDyEVsZ4CbKoQU4wc=COyu#CZ`Djt(Mv_ark0Rs9@g!?EPfx^4`e z7WQASM^?#_I^qX0v6gzs^VWriP#mLbgw$GWCZtOE5;MMt#YOYg0w(e3^p=(Y=9P8D&p5P*s7&=$#i73toIkXo#lNtO^-jU)vZ&Q zrw|8kS}f>^*o)b@);g`uDFRsJp<|COkc-xUqk?z!^}ccRB`gcWRmt9K$a5mfxGUIl zH6Ec{J;?wyDO#zTd6h0uk@a9T`JBS=Xo>&?P&eBE3Lk6edu4DtJo7A5=!TvzfO(3>m6q=9#SL5FCX-brua97Mi?HAy6shI^Te`fRvk06U5tth0lD3ZT z067uTL6|>4m`4s_4roQ?$yG#OXUgJ-KZ=&|r95F)FjQZRHzdT&=kR1G Date: Wed, 22 Jan 2014 10:52:41 -0800 Subject: [PATCH 19/72] Added support to transfer haplotypes once per region to the JNI Re-use transferred haplotypes (stored in GlobalRef) across calls to computeLikelihoods --- PairHMM_JNI/LoadTimeInitializer.cc | 76 ++++++ PairHMM_JNI/LoadTimeInitializer.h | 37 +++ PairHMM_JNI/Makefile | 13 +- ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 228 ++++++++---------- ...te_sting_utils_pairhmm_JNILoglessPairHMM.h | 57 +++-- PairHMM_JNI/pairhmm-1-base.cc | 20 +- PairHMM_JNI/pairhmm-template-main.cc | 7 +- PairHMM_JNI/run.sh | 4 +- PairHMM_JNI/utils.cc | 5 + PairHMM_JNI/utils.h | 2 + .../PairHMMLikelihoodCalculationEngine.java | 8 + .../utils/pairhmm/JNILoglessPairHMM.java | 130 +++++----- .../sting/utils/pairhmm/PairHMM.java | 8 + 13 files changed, 359 insertions(+), 236 deletions(-) create mode 100644 PairHMM_JNI/LoadTimeInitializer.cc create mode 100644 PairHMM_JNI/LoadTimeInitializer.h diff --git a/PairHMM_JNI/LoadTimeInitializer.cc b/PairHMM_JNI/LoadTimeInitializer.cc new file mode 100644 index 000000000..946451952 --- /dev/null +++ b/PairHMM_JNI/LoadTimeInitializer.cc @@ -0,0 +1,76 @@ +#include "LoadTimeInitializer.h" +#include "utils.h" +using namespace std; + +LoadTimeInitializer g_load_time_initializer; + +LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loaded +{ + ConvertChar::init(); + m_sumNumReads = 0; + m_sumSquareNumReads = 0; + m_sumNumHaplotypes = 0; + m_sumSquareNumHaplotypes = 0; + m_sumNumTestcases = 0; + m_sumSquareNumTestcases = 0; + m_maxNumTestcases = 0; + m_num_invocations = 0; + m_compute_time = 0; + m_data_transfer_time = 0; + + m_filename_to_fptr.clear(); + + initialize_function_pointers(); + cout.flush(); +} + +void LoadTimeInitializer::print_profiling() +{ + double mean_val; + cout << "Compute time "<::iterator mI = m_filename_to_fptr.find(filename); + ofstream* fptr = 0; + if(mI == m_filename_to_fptr.end()) + { + m_filename_to_fptr[filename] = new ofstream(); + fptr = m_filename_to_fptr[filename]; + fptr->open(filename.c_str(), to_append ? ios::app : ios::out); + assert(fptr->is_open()); + } + else + fptr = (*mI).second; + //ofstream fptr; + //fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); + (*fptr) << s; + if(add_newline) + (*fptr) << "\n"; + //fptr.close(); +} +void LoadTimeInitializer::debug_close() +{ + for(map::iterator mB = m_filename_to_fptr.begin(), mE = m_filename_to_fptr.end(); + mB != mE;mB++) + { + (*mB).second->close(); + delete (*mB).second; + } + m_filename_to_fptr.clear(); +} + diff --git a/PairHMM_JNI/LoadTimeInitializer.h b/PairHMM_JNI/LoadTimeInitializer.h new file mode 100644 index 000000000..8a03ea0f4 --- /dev/null +++ b/PairHMM_JNI/LoadTimeInitializer.h @@ -0,0 +1,37 @@ +#ifndef LOAD_TIME_INITIALIZER_H +#define LOAD_TIME_INITIALIZER_H +#include "headers.h" +#include +class LoadTimeInitializer +{ + public: + LoadTimeInitializer(); //will be called when library is loaded + void print_profiling(); + void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); + void debug_close(); + + jfieldID m_readBasesFID; + jfieldID m_readQualsFID; + jfieldID m_insertionGOPFID; + jfieldID m_deletionGOPFID; + jfieldID m_overallGCPFID; + jfieldID m_haplotypeBasesFID; + //used to compute avg, variance of #testcases + double m_sumNumReads; + double m_sumSquareNumReads; + double m_sumNumHaplotypes; + double m_sumSquareNumHaplotypes; + double m_sumNumTestcases; + double m_sumSquareNumTestcases; + unsigned m_maxNumTestcases; + unsigned m_num_invocations; + //timing + double m_compute_time; + double m_data_transfer_time; + private: + std::map m_filename_to_fptr; +}; +extern LoadTimeInitializer g_load_time_initializer; + + +#endif diff --git a/PairHMM_JNI/Makefile b/PairHMM_JNI/Makefile index f6dfbb1f0..7506279ef 100644 --- a/PairHMM_JNI/Makefile +++ b/PairHMM_JNI/Makefile @@ -1,4 +1,5 @@ -#OMPCFLAGS=-fopenmp +OMPCFLAGS=-fopenmp +OMPLFLAGS=-fopenmp #-openmp-link static #CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas #CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas @@ -19,7 +20,7 @@ DEPDIR=.deps DF=$(DEPDIR)/$(*).d #Common across libJNI and sandbox -COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc sse_function_instantiations.cc +COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc sse_function_instantiations.cc LoadTimeInitializer.cc #Part of libJNI LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc $(COMMON_SOURCES) SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc @@ -28,7 +29,7 @@ COMMON_OBJECTS=$(COMMON_SOURCES:.cc=.o) #No vectorization for these files -NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc +NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc LoadTimeInitializer.cc #Use -xAVX for these files AVX_SOURCES=avx_function_instantiations.cc #Use -xSSE4.2 for these files @@ -47,13 +48,13 @@ all: $(BIN) -include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) checker: pairhmm-1-base.o $(COMMON_OBJECTS) - $(CXX) $(OMPCFLAGS) -o $@ $^ $(LDFLAGS) + $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) pairhmm-template-main: pairhmm-template-main.o $(COMMON_OBJECTS) - $(CXX) $(OMPCFLAGS) -o $@ $^ $(LDFLAGS) + $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) libJNILoglessPairHMM.so: $(LIBOBJECTS) - $(CXX) $(OMPCFLAGS) -shared -o $@ $(LIBOBJECTS) + $(CXX) $(OMPLFLAGS) -shared -o $@ $(LIBOBJECTS) $(OBJECTS): %.o: %.cc diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index a746bc265..741957fc0 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -11,96 +11,10 @@ #include "template.h" #include "utils.h" +#include "LoadTimeInitializer.h" using namespace std; -class LoadTimeInitializer -{ - public: - LoadTimeInitializer() //will be called when library is loaded - { - ConvertChar::init(); - m_sumNumReads = 0; - m_sumSquareNumReads = 0; - m_sumNumHaplotypes = 0; - m_sumSquareNumHaplotypes = 0; - m_sumNumTestcases = 0; - m_sumSquareNumTestcases = 0; - m_maxNumTestcases = 0; - m_num_invocations = 0; - m_filename_to_fptr.clear(); - - initialize_function_pointers(); - cout.flush(); - } - void print_profiling() - { - double mean_val; - cout <<"Invocations : "<::iterator mI = m_filename_to_fptr.find(filename); - ofstream* fptr = 0; - if(mI == m_filename_to_fptr.end()) - { - m_filename_to_fptr[filename] = new ofstream(); - fptr = m_filename_to_fptr[filename]; - fptr->open(filename.c_str(), to_append ? ios::app : ios::out); - assert(fptr->is_open()); - } - else - fptr = (*mI).second; - //ofstream fptr; - //fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); - (*fptr) << s; - if(add_newline) - (*fptr) << "\n"; - //fptr.close(); - } - void debug_close() - { - for(map::iterator mB = m_filename_to_fptr.begin(), mE = m_filename_to_fptr.end(); - mB != mE;mB++) - { - (*mB).second->close(); - delete (*mB).second; - } - m_filename_to_fptr.clear(); - } - jfieldID m_readBasesFID; - jfieldID m_readQualsFID; - jfieldID m_insertionGOPFID; - jfieldID m_deletionGOPFID; - jfieldID m_overallGCPFID; - jfieldID m_haplotypeBasesFID; - //used to compute avg, variance of #testcases - double m_sumNumReads; - double m_sumSquareNumReads; - double m_sumNumHaplotypes; - double m_sumSquareNumHaplotypes; - double m_sumNumTestcases; - double m_sumSquareNumTestcases; - unsigned m_maxNumTestcases; - unsigned m_num_invocations; - private: - map m_filename_to_fptr; -}; -LoadTimeInitializer g_load_time_initializer; - - JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeProbabilities (JNIEnv* env, jclass thisObject, jobjectArray transition, jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP @@ -128,7 +42,7 @@ Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializePrior //Gets direct access to Java arrays #define GET_BYTE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical #define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical -#define JNI_RELEASE_MODE 0 +#define JNI_RO_RELEASE_MODE JNI_ABORT #define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical #define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical @@ -136,7 +50,7 @@ Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializePrior //Likely makes copy of Java arrays to JNI C++ space #define GET_BYTE_ARRAY_ELEMENTS env->GetByteArrayElements #define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleaseByteArrayElements -#define JNI_RELEASE_MODE JNI_ABORT +#define JNI_RO_RELEASE_MODE JNI_ABORT #define GET_DOUBLE_ARRAY_ELEMENTS env->GetDoubleArrayElements #define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleaseDoubleArrayElements @@ -187,12 +101,12 @@ Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadL #endif - RELEASE_BYTE_ARRAY_ELEMENTS(overallGCP, overallGCPArray, JNI_RELEASE_MODE); - RELEASE_BYTE_ARRAY_ELEMENTS(deletionGOP, deletionGOPArray, JNI_RELEASE_MODE); - RELEASE_BYTE_ARRAY_ELEMENTS(insertionGOP, insertionGOPArray, JNI_RELEASE_MODE); - RELEASE_BYTE_ARRAY_ELEMENTS(readQuals, readQualsArray, JNI_RELEASE_MODE); - RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBases, haplotypeBasesArray, JNI_RELEASE_MODE); - RELEASE_BYTE_ARRAY_ELEMENTS(readBases, readBasesArray, JNI_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(overallGCP, overallGCPArray, JNI_RO_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(deletionGOP, deletionGOPArray, JNI_RO_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(insertionGOP, insertionGOPArray, JNI_RO_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(readQuals, readQualsArray, JNI_RO_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBases, haplotypeBasesArray, JNI_RO_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(readBases, readBasesArray, JNI_RO_RELEASE_MODE); return 0.0; } @@ -226,6 +140,46 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai g_load_time_initializer.m_haplotypeBasesFID = fid; } +//Since the list of haplotypes against which the reads are evaluated in PairHMM is the same for a region, +//transfer the list only once +vector > g_haplotypeBasesArrayVector; +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeHaplotypes + (JNIEnv * env, jobject thisObject, jint numHaplotypes, jobjectArray haplotypeDataArray) +{ + jboolean is_copy = JNI_FALSE; + //To ensure, GET_BYTE_ARRAY_ELEMENTS is invoked only once for each haplotype, store bytearrays in a vector + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + haplotypeBasesArrayVector.clear(); + haplotypeBasesArrayVector.resize(numHaplotypes); + for(unsigned j=0;jGetObjectArrayElement(haplotypeDataArray, j); + jbyteArray haplotypeBases = (jbyteArray)env->GetObjectField(haplotypeObject, g_load_time_initializer.m_haplotypeBasesFID); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBases && ("haplotypeBases is NULL at index : "+to_string(j)+"\n").c_str()); +#endif + //Need a global reference as this will be accessed across multiple JNI calls to JNIComputeLikelihoods() + jbyteArray haplotypeBasesGlobalRef = (jbyteArray)env->NewGlobalRef(haplotypeBases); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBasesGlobalRef && ("Could not get global ref to haplotypeBases at index : "+to_string(j)+"\n").c_str()); +#endif + env->DeleteLocalRef(haplotypeBases); //free the local reference + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBasesGlobalRef, &is_copy); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(env->GetArrayLength(haplotypeBasesGlobalRef) < MCOLS); +#endif +#ifdef DEBUG0_1 + cout << "JNI haplotype length "<GetArrayLength(haplotypeBasesGlobalRef)<<"\n"; +#endif + haplotypeBasesArrayVector[j] = make_pair(haplotypeBasesGlobalRef, haplotypeBasesArray); +#ifdef DEBUG3 + for(unsigned k=0;kGetArrayLength(haplotypeBases);++k) + g_load_time_initializer.debug_dump("haplotype_bases_jni.txt",to_string((int)haplotypeBasesArray[k]),true); +#endif + } +} + //JNI function to invoke compute_full_prob_avx //readDataArray - array of JNIReadDataHolderClass objects which contain the readBases, readQuals etc //haplotypeDataArray - array of JNIHaplotypeDataHolderClass objects which contain the haplotypeBases @@ -235,37 +189,16 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) { - jboolean is_copy = JNI_FALSE; - //To ensure, GET_BYTE_ARRAY_ELEMENTS is invoked only once for each haplotype, store bytearrays in a vector - vector > haplotypeBasesArrayVector; - haplotypeBasesArrayVector.clear(); - haplotypeBasesArrayVector.resize(numHaplotypes); #ifdef DEBUG0_1 cout << "JNI numReads "<GetObjectArrayElement(haplotypeDataArray, j); - jbyteArray haplotypeBases = (jbyteArray)env->GetObjectField(haplotypeObject, g_load_time_initializer.m_haplotypeBasesFID); -#ifdef ENABLE_ASSERTIONS - assert(haplotypeBases && ("haplotypeBases is NULL at index : "+to_string(j)+"\n").c_str()); -#endif - jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBases, &is_copy); -#ifdef ENABLE_ASSERTIONS - assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); - assert(env->GetArrayLength(haplotypeBases) < MCOLS); -#endif -#ifdef DEBUG0_1 - cout << "JNI haplotype length "<GetArrayLength(haplotypeBases)<<"\n"; -#endif - haplotypeBasesArrayVector[j] = make_pair(haplotypeBases, haplotypeBasesArray); -#ifdef DEBUG3 - for(unsigned k=0;kGetArrayLength(haplotypeBases);++k) - g_load_time_initializer.debug_dump("haplotype_bases_jni.txt",to_string((int)haplotypeBasesArray[k]),true); -#endif - } + double start_time = 0; + //haplotype vector from earlier store - note the reference to vector, not copying + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + jboolean is_copy = JNI_FALSE; unsigned numTestCases = numReads*numHaplotypes; + //vector to store results vector tc_array; tc_array.clear(); tc_array.resize(numTestCases); @@ -274,6 +207,9 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai vector > > readBasesArrayVector; readBasesArrayVector.clear(); readBasesArrayVector.resize(numReads); +#ifdef DO_PROFILING + start_time = getCurrClk(); +#endif for(unsigned i=0;iGetArrayLength(likelihoodArray) == numTestCases); #endif +#ifdef DO_PROFILING + start_time = getCurrClk(); +#endif #pragma omp parallel for schedule (dynamic,10) private(tc_idx) num_threads(maxNumThreadsToUse) for(tc_idx=0;tc_idx=0;--i)//note the order - reverse of GET { for(int j=readBasesArrayVector[i].size()-1;j>=0;--j) - RELEASE_BYTE_ARRAY_ELEMENTS(readBasesArrayVector[i][j].first, readBasesArrayVector[i][j].second, JNI_RELEASE_MODE); + RELEASE_BYTE_ARRAY_ELEMENTS(readBasesArrayVector[i][j].first, readBasesArrayVector[i][j].second, JNI_RO_RELEASE_MODE); readBasesArrayVector[i].clear(); } readBasesArrayVector.clear(); - - //Now release haplotype arrays - for(int j=numHaplotypes-1;j>=0;--j) //note the order - reverse of GET - RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBasesArrayVector[j].first, haplotypeBasesArrayVector[j].second, JNI_RELEASE_MODE); - haplotypeBasesArrayVector.clear(); +#ifdef DO_PROFILING + g_load_time_initializer.m_data_transfer_time += (getCurrClk()-start_time); +#endif tc_array.clear(); #ifdef DO_PROFILING g_load_time_initializer.m_sumNumReads += numReads; @@ -411,6 +356,21 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai #endif } +//Release haplotypes at the end of a region +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniFinalizeRegion + (JNIEnv * env, jobject thisObject) +{ + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + //Now release haplotype arrays + for(int j=haplotypeBasesArrayVector.size()-1;j>=0;--j) //note the order - reverse of GET + { + RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBasesArrayVector[j].first, haplotypeBasesArrayVector[j].second, JNI_RO_RELEASE_MODE); + env->DeleteGlobalRef(haplotypeBasesArrayVector[j].first); //free the global reference + } + haplotypeBasesArrayVector.clear(); +} + + JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniClose (JNIEnv* env, jobject thisObject) { diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h index b50f6a5a9..06ee99d7d 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h @@ -21,6 +21,34 @@ extern "C" { #define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToDeletion 4L #undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_deletionToDeletion #define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_deletionToDeletion 5L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug 0L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_verify +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_verify 0L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug0_1 +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug0_1 0L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug1 +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug1 0L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug2 +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug2 0L +#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug3 +#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug3 0L +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniGlobalInit + * Signature: (Ljava/lang/Class;Ljava/lang/Class;)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniGlobalInit + (JNIEnv *, jobject, jclass, jclass); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniClose + (JNIEnv *, jobject); + /* * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM * Method: jniInitialize @@ -45,39 +73,38 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializePriorsAndUpdateCells (JNIEnv *, jobject, jboolean, jint, jint, jbyteArray, jbyteArray, jbyteArray, jint); - /* * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM * Method: jniSubComputeReadLikelihoodGivenHaplotypeLog10 * Signature: (II[B[B[B[B[B[BI)D */ JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10 - (JNIEnv *, jobject, jint, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jint); + (JNIEnv *, jobject, jint, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jint); /* * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniGlobalInit - * Signature: (Ljava/lang/Class;Ljava/lang/Class;)V + * Method: jniInitializeHaplotypes + * Signature: (I[Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIHaplotypeDataHolderClass;)V */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniGlobalInit - (JNIEnv *, jobject, jclass, jclass); +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniFinalizeRegion + (JNIEnv *, jobject); /* * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM * Method: jniComputeLikelihoods - * Signature: ([Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIReadDataHolderClass;[Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIHaplotypeDataHolderClass;[D)V + * Signature: (II[Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIReadDataHolderClass;[Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIHaplotypeDataHolderClass;[DI)V */ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniComputeLikelihoods (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniClose - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniClose - (JNIEnv *, jobject); - #ifdef __cplusplus } #endif diff --git a/PairHMM_JNI/pairhmm-1-base.cc b/PairHMM_JNI/pairhmm-1-base.cc index a2d5dfa88..1a2dc9a7c 100644 --- a/PairHMM_JNI/pairhmm-1-base.cc +++ b/PairHMM_JNI/pairhmm-1-base.cc @@ -4,26 +4,15 @@ #include "headers.h" #include "template.h" #include "utils.h" +#include "LoadTimeInitializer.h" using namespace std; -class LoadTimeInitializer -{ - public: - LoadTimeInitializer() //will be called when library is loaded - { - ConvertChar::init(); - } -}; -LoadTimeInitializer g_load_time_initializer; + #define BATCH_SIZE 10000 #define RUN_HYBRID -double getCurrClk() { - struct timeval tv ; - gettimeofday(&tv, NULL); - return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; -} + int main(int argc, char** argv) { @@ -39,9 +28,6 @@ int main(int argc, char** argv) if(argc >= 4) chunk_size = strtol(argv[3],0,10); - - initialize_function_pointers(); - std::ifstream ifptr; FILE* fptr = 0; if(use_old_read_testcase) diff --git a/PairHMM_JNI/pairhmm-template-main.cc b/PairHMM_JNI/pairhmm-template-main.cc index 1c2ca585e..0ab53d378 100644 --- a/PairHMM_JNI/pairhmm-template-main.cc +++ b/PairHMM_JNI/pairhmm-template-main.cc @@ -10,14 +10,9 @@ #define BATCH_SIZE 10000 #define RUN_HYBRID +double getCurrClk(); int thread_level_parallelism_enabled = false ; -double getCurrClk() { - struct timeval tv ; - gettimeofday(&tv, NULL); - return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; -} - void print128b_F(__m128 x) { float *p = (float *)(&x); diff --git a/PairHMM_JNI/run.sh b/PairHMM_JNI/run.sh index de69ba5fc..eafb7ff79 100755 --- a/PairHMM_JNI/run.sh +++ b/PairHMM_JNI/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -rm -f *.txt -export GSA_ROOT_DIR=/home/karthikg/broad/gsa-unstable +rm -f *.txt *.log +GSA_ROOT_DIR=/home/karthikg/broad/gsa-unstable #-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ -R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ diff --git a/PairHMM_JNI/utils.cc b/PairHMM_JNI/utils.cc index 2288377c4..5ed79e29d 100644 --- a/PairHMM_JNI/utils.cc +++ b/PairHMM_JNI/utils.cc @@ -240,3 +240,8 @@ int read_mod_testcase(ifstream& fptr, testcase* tc, bool reformat) return tokens.size(); } +double getCurrClk() { + struct timeval tv ; + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} diff --git a/PairHMM_JNI/utils.h b/PairHMM_JNI/utils.h index 8da8269b8..3f7c2ff69 100644 --- a/PairHMM_JNI/utils.h +++ b/PairHMM_JNI/utils.h @@ -1,6 +1,7 @@ #ifndef PAIRHMM_UTIL_H #define PAIRHMM_UTIL_H +#include "template.h" template std::string to_string(T obj) @@ -25,4 +26,5 @@ void debug_dump(std::string filename, std::string s, bool to_append, bool add_ne template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); void initialize_function_pointers(); +double getCurrClk(); #endif diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index caa880b41..b91805a62 100644 --- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -335,6 +335,11 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation pairHMMThreadLocal.get().initialize(haplotypes, perSampleReadList, X_METRIC_LENGTH, Y_METRIC_LENGTH); } + private void finalizePairHMM() + { + pairHMMThreadLocal.get().finalizeRegion(); + } + @Override public Map computeReadLikelihoods( final AssemblyResultSet assemblyResultSet, final Map> perSampleReadList ) { @@ -352,6 +357,9 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation map.filterPoorlyModelledReads(EXPECTED_ERROR_RATE_PER_BASE); stratifiedReadMap.put(sampleEntry.getKey(), map); } + + //Used mostly by the JNI implementation(s) to free arrays + finalizePairHMM(); return stratifiedReadMap; } diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java index ed628f064..f926e4bd6 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java @@ -68,8 +68,8 @@ import java.util.HashMap; public class JNILoglessPairHMM extends LoglessPairHMM { private static final boolean debug = false; //simulates ifdef - private static final boolean verify = debug || false; //simulates ifdef - private static final boolean debug0_1 = false; //simulates ifdef + private static final boolean verify = debug || true; //simulates ifdef + private static final boolean debug0_1 = true; //simulates ifdef private static final boolean debug1 = false; //simulates ifdef private static final boolean debug2 = false; private static final boolean debug3 = false; @@ -148,10 +148,10 @@ public class JNILoglessPairHMM extends LoglessPairHMM { jniInitialize(readMaxLength, haplotypeMaxLength); } - private native void jniInitialize(final int numHaplotypes, final int numTotalReads, JNIHaplotypeDataHolderClass[] haplotypeDataArray, - JNIReadDataHolderClass[] readDataArray); - HashMap sampleToIndex = new HashMap(); - //Used to transfer data to JNI + private HashMap haplotypeToHaplotypeListIdxMap = new HashMap(); + private native void jniInitializeHaplotypes(final int numHaplotypes, JNIHaplotypeDataHolderClass[] haplotypeDataArray); + //Used to transfer data to JNI + //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region /** * {@inheritDoc} */ @@ -159,7 +159,6 @@ public class JNILoglessPairHMM extends LoglessPairHMM { public void initialize( final List haplotypes, final Map> perSampleReadList, final int readMaxLength, final int haplotypeMaxLength ) { if(verify) super.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); - /* int numHaplotypes = haplotypes.size(); JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; int idx = 0; @@ -167,32 +166,22 @@ public class JNILoglessPairHMM extends LoglessPairHMM { { haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); haplotypeDataArray[idx].haplotypeBases = currHaplotype.getBases(); + haplotypeToHaplotypeListIdxMap.put(currHaplotype, idx); ++idx; } - sampleToIndex.clear(); - int totalNumReads = 0; - for(final Map.Entry> currEntry : perSampleReadList) - { - sampleToIndex.put(currEntry.getKey(), totalNumReads); - totalNumReads += currEntry.getValue().size(); - } + jniInitializeHaplotypes(numHaplotypes, haplotypeDataArray); + } - JNIHaplotypeDataHolderClass[] readDataArray = new JNIReadDataHolderClass[totalNumReads]; - idx = 0; - for(final Map.Entry> currEntry : perSampleReadList) - { - for(GATKSAMRecord read : currEntry.getValue()) - { - readDataArray[idx] = new JNIReadDataHolderClass(); - readDataArray[idx].readBases = read.getReadBases(); - readDataArray[idx].readQuals = read.getBaseQualities(); - readDataArray[idx].insertionGOP = read.getBaseInsertionQualities(); - readDataArray[idx].deletionGOP = read.getBaseDeletionQualities(); - readDataArray[idx].overallGCP = GCPArrayMap.get(read); - ++idx; - } - } - jniInitialize(numHaplotypes, numTotalReads, haplotypeDataArray, readDataArray);*/ + private native void jniFinalizeRegion(); + //Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not + //accessing Java memory directly, still important to release memory from C++ + /** + * {@inheritDoc} + */ + @Override + public void finalizeRegion() + { + jniFinalizeRegion(); } //Real compute kernel @@ -216,10 +205,10 @@ public class JNILoglessPairHMM extends LoglessPairHMM { throw new IllegalStateException("Must call initialize before calling jniComputeLikelihoods in debug/verify mode"); } int readListSize = reads.size(); - int alleleHaplotypeMapSize = alleleHaplotypeMap.size(); - int numTestcases = readListSize*alleleHaplotypeMapSize; + int numHaplotypes = alleleHaplotypeMap.size(); + int numTestcases = readListSize*numHaplotypes; if(debug0_1) - System.out.println("Java numReads "+readListSize+" numHaplotypes "+alleleHaplotypeMapSize); + System.out.println("Java numReads "+readListSize+" numHaplotypes "+numHaplotypes); JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; int idx = 0; for(GATKSAMRecord read : reads) @@ -246,39 +235,68 @@ public class JNILoglessPairHMM extends LoglessPairHMM { } ++idx; } - JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[alleleHaplotypeMapSize]; - idx = 0; - for (Map.Entry currEntry : alleleHaplotypeMap.entrySet()) //order is important - access in same order always + + JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; + if(verify) { - haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); - haplotypeDataArray[idx].haplotypeBases = currEntry.getValue().getBases(); - if(debug0_1) - System.out.println("Java haplotype length "+haplotypeDataArray[idx].haplotypeBases.length); - if(debug3) + idx = 0; + for (Map.Entry currEntry : alleleHaplotypeMap.entrySet()) //order is important - access in same order always { - for(int i=0;i currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always { - likelihoodMap.add(read, currEntry.getKey(), likelihoodArray[idx]); + //Since the order of haplotypes in the List and alleleHaplotypeMap is different, + //get idx of current haplotype in the list and use this idx to get the right likelihoodValue + idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); + likelihoodMap.add(read, currEntry.getKey(), likelihoodArray[readIdx + idxInsideHaplotypeList]); ++idx; } + readIdx += numHaplotypes; + } if(verify) { + //re-order values in likelihoodArray + double[] tmpArray = new double[numHaplotypes]; + idx = 0; + idxInsideHaplotypeList = 0; + readIdx = 0; + for(GATKSAMRecord read : reads) + { + for(int j=0;j currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always + { + idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); + likelihoodArray[idx] = tmpArray[idxInsideHaplotypeList]; + ++idx; + } + readIdx += numHaplotypes; + } //to compare values likelihoodMap = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); //for floating point values, no exact equality @@ -304,23 +322,23 @@ public class JNILoglessPairHMM extends LoglessPairHMM { if(toDump) { idx = 0; - System.out.println("Dump : Java numReads "+readListSize+" numHaplotypes "+alleleHaplotypeMapSize); + System.out.println("Dump : Java numReads "+readListSize+" numHaplotypes "+numHaplotypes); for(int i=0;i Date: Wed, 22 Jan 2014 22:57:32 -0800 Subject: [PATCH 20/72] 1. Converted q,i,d,c in C++ from int* to char* 2. Use clock_gettime to measure performance 3. Disabled OpenMP 4. Moved LoadTimeInitializer to different file --- PairHMM_JNI/LoadTimeInitializer.cc | 4 +- PairHMM_JNI/LoadTimeInitializer.h | 4 +- PairHMM_JNI/Makefile | 8 ++-- ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 40 +++++++++---------- PairHMM_JNI/pairhmm-1-base.cc | 13 ++++-- PairHMM_JNI/run.sh | 6 +-- PairHMM_JNI/template.h | 3 +- PairHMM_JNI/utils.cc | 31 ++++++++++---- PairHMM_JNI/utils.h | 1 + .../utils/pairhmm/JNILoglessPairHMM.java | 4 +- 10 files changed, 66 insertions(+), 48 deletions(-) diff --git a/PairHMM_JNI/LoadTimeInitializer.cc b/PairHMM_JNI/LoadTimeInitializer.cc index 946451952..534043371 100644 --- a/PairHMM_JNI/LoadTimeInitializer.cc +++ b/PairHMM_JNI/LoadTimeInitializer.cc @@ -27,8 +27,8 @@ LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loa void LoadTimeInitializer::print_profiling() { double mean_val; - cout << "Compute time "< m_filename_to_fptr; }; diff --git a/PairHMM_JNI/Makefile b/PairHMM_JNI/Makefile index 7506279ef..9b461a833 100644 --- a/PairHMM_JNI/Makefile +++ b/PairHMM_JNI/Makefile @@ -1,5 +1,5 @@ -OMPCFLAGS=-fopenmp -OMPLFLAGS=-fopenmp #-openmp-link static +#OMPCFLAGS=-fopenmp +#OMPLFLAGS=-fopenmp #-openmp-link static #CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas #CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas @@ -11,7 +11,7 @@ COMMON_COMPILATION_FLAGS=$(JNI_COMPILATION_FLAGS) -O3 -W -Wall -pedantic $(OMPCF CC=icc CXX=icc -LDFLAGS=-lm $(OMPLDFLAGS) +LDFLAGS=-lm -lrt $(OMPLDFLAGS) BIN=libJNILoglessPairHMM.so pairhmm-template-main checker #BIN=checker @@ -54,7 +54,7 @@ pairhmm-template-main: pairhmm-template-main.o $(COMMON_OBJECTS) $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) libJNILoglessPairHMM.so: $(LIBOBJECTS) - $(CXX) $(OMPLFLAGS) -shared -o $@ $(LIBOBJECTS) + $(CXX) $(OMPLFLAGS) -shared -o $@ $(LIBOBJECTS) ${LDFLAGS} $(OBJECTS): %.o: %.cc diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index 741957fc0..d575b8271 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -208,7 +208,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai readBasesArrayVector.clear(); readBasesArrayVector.resize(numReads); #ifdef DO_PROFILING - start_time = getCurrClk(); + start_time = get_time(); #endif for(unsigned i=0;iGetArrayLength(likelihoodArray) == numTestCases); #endif #ifdef DO_PROFILING - start_time = getCurrClk(); + start_time = get_time(); #endif #pragma omp parallel for schedule (dynamic,10) private(tc_idx) num_threads(maxNumThreadsToUse) for(tc_idx=0;tc_idx tc_vector; tc_vector.clear(); testcase tc; - double total_time = 0; + uint64_t total_time = 0; while(1) { int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); @@ -55,7 +55,7 @@ int main(int argc, char** argv) vector results_vec; results_vec.clear(); results_vec.resize(tc_vector.size()); - double start_time = getCurrClk(); + get_time(); #pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) for(unsigned i=0;iihap = (int *) malloc(tc->haplen*sizeof(int)); tc->irs = (int *) malloc(tc->rslen*sizeof(int)); - //tc->q = (int *) malloc(sizeof(int) * tc->rslen); - //tc->i = (int *) malloc(sizeof(int) * tc->rslen); - //tc->d = (int *) malloc(sizeof(int) * tc->rslen); - //tc->c = (int *) malloc(sizeof(int) * tc->rslen); + tc->q = (char *) malloc(sizeof(char) * tc->rslen); + tc->i = (char *) malloc(sizeof(char) * tc->rslen); + tc->d = (char *) malloc(sizeof(char) * tc->rslen); + tc->c = (char *) malloc(sizeof(char) * tc->rslen); for (x = 0; x < tc->rslen; x++) { @@ -199,18 +199,22 @@ int read_mod_testcase(ifstream& fptr, testcase* tc, bool reformat) memcpy(tc->hap, tokens[0].c_str(), tokens[0].size()); tc->rs = new char[tokens[1].size()+2]; tc->rslen = tokens[1].size(); + tc->q = new char[tc->rslen]; + tc->i = new char[tc->rslen]; + tc->d = new char[tc->rslen]; + tc->c = new char[tc->rslen]; //cout << "Lengths "<haplen <<" "<rslen<<"\n"; memcpy(tc->rs, tokens[1].c_str(),tokens[1].size()); assert(tokens.size() == 2 + 4*(tc->rslen)); assert(tc->rslen < MROWS); for(unsigned j=0;jrslen;++j) - tc->q[j] = convToInt(tokens[2+0*tc->rslen+j]); + tc->q[j] = (char)convToInt(tokens[2+0*tc->rslen+j]); for(unsigned j=0;jrslen;++j) - tc->i[j] = convToInt(tokens[2+1*tc->rslen+j]); + tc->i[j] = (char)convToInt(tokens[2+1*tc->rslen+j]); for(unsigned j=0;jrslen;++j) - tc->d[j] = convToInt(tokens[2+2*tc->rslen+j]); + tc->d[j] = (char)convToInt(tokens[2+2*tc->rslen+j]); for(unsigned j=0;jrslen;++j) - tc->c[j] = convToInt(tokens[2+3*tc->rslen+j]); + tc->c[j] = (char)convToInt(tokens[2+3*tc->rslen+j]); if(reformat) { @@ -245,3 +249,14 @@ double getCurrClk() { gettimeofday(&tv, NULL); return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; } + +uint64_t get_time(struct timespec* store_struct) +{ + static struct timespec start_time; + struct timespec curr_time; + struct timespec* ptr = (store_struct == 0) ? &curr_time : store_struct; + clock_gettime(CLOCK_REALTIME, ptr); + uint64_t diff_time = (ptr->tv_sec-start_time.tv_sec)*1000000000+(ptr->tv_nsec-start_time.tv_nsec); + start_time = *ptr; + return diff_time; +} diff --git a/PairHMM_JNI/utils.h b/PairHMM_JNI/utils.h index 3f7c2ff69..cc020ba40 100644 --- a/PairHMM_JNI/utils.h +++ b/PairHMM_JNI/utils.h @@ -27,4 +27,5 @@ template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); void initialize_function_pointers(); double getCurrClk(); +uint64_t get_time(struct timespec* x=0); #endif diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java index f926e4bd6..6cbd3510c 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java @@ -68,8 +68,8 @@ import java.util.HashMap; public class JNILoglessPairHMM extends LoglessPairHMM { private static final boolean debug = false; //simulates ifdef - private static final boolean verify = debug || true; //simulates ifdef - private static final boolean debug0_1 = true; //simulates ifdef + private static final boolean verify = debug || false; //simulates ifdef + private static final boolean debug0_1 = false; //simulates ifdef private static final boolean debug1 = false; //simulates ifdef private static final boolean debug2 = false; private static final boolean debug3 = false; From 81bdfbd00df712f82b88f2547ee43cce17927d2a Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Fri, 24 Jan 2014 16:29:35 -0800 Subject: [PATCH 21/72] Temporary commit before moving to new native library --- .gitignore | 3 +- PairHMM_JNI/LoadTimeInitializer.cc | 7 + PairHMM_JNI/LoadTimeInitializer.h | 4 + ...ing_utils_pairhmm_DebugJNILoglessPairHMM.h | 71 +++ ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 10 + ...sting_utils_pairhmm_VectorLoglessPairHMM.h | 79 +++ PairHMM_JNI/run.sh | 6 +- PairHMM_JNI/utils.cc | 16 +- PairHMM_JNI/utils.h | 9 +- .../PairHMMLikelihoodCalculationEngine.java | 6 +- .../utils/pairhmm/DebugJNILoglessPairHMM.java | 496 ++++++++++++++++++ .../utils/pairhmm/JNILoglessPairHMM.java | 471 +---------------- .../sting/utils/pairhmm/LoglessPairHMM.java | 50 +- .../utils/pairhmm/VectorLoglessPairHMM.java | 198 +++++++ .../sting/utils/pairhmm/PairHMM.java | 5 +- 15 files changed, 902 insertions(+), 529 deletions(-) create mode 100644 PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h create mode 100644 PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h create mode 100644 protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java create mode 100644 protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java diff --git a/.gitignore b/.gitignore index ede62ef81..b640999cc 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,5 @@ org/ package-list resources/ velocity.log - +perf +verify diff --git a/PairHMM_JNI/LoadTimeInitializer.cc b/PairHMM_JNI/LoadTimeInitializer.cc index 534043371..ae545d881 100644 --- a/PairHMM_JNI/LoadTimeInitializer.cc +++ b/PairHMM_JNI/LoadTimeInitializer.cc @@ -13,6 +13,10 @@ LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loa m_sumSquareNumHaplotypes = 0; m_sumNumTestcases = 0; m_sumSquareNumTestcases = 0; + m_sumReadLengths = 0; + m_sumHaplotypeLengths = 0; + m_sumProductReadLengthHaplotypeLength = 0; + m_sumSquareProductReadLengthHaplotypeLength = 0; m_maxNumTestcases = 0; m_num_invocations = 0; m_compute_time = 0; @@ -40,6 +44,9 @@ void LoadTimeInitializer::print_profiling() mean_val = m_sumNumTestcases/m_num_invocations; cout << "numtestcases\t"< +/* Header for class org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM */ + +#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM +#define _Included_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM +#ifdef __cplusplus +extern "C" { +#endif +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_TRISTATE_CORRECTION +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_TRISTATE_CORRECTION 3.0 +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToMatch +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToMatch 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_indelToMatch +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_indelToMatch 1L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToInsertion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToInsertion 2L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_insertionToInsertion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_insertionToInsertion 3L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToDeletion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToDeletion 4L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_deletionToDeletion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_deletionToDeletion 5L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_verify +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_verify 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug0_1 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug0_1 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug1 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug1 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug2 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug2 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug3 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug3 0L +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitialize + * Signature: (II)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitialize + (JNIEnv *, jobject, jint, jint); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitializeProbabilities + * Signature: ([[D[B[B[B)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializeProbabilities + (JNIEnv *, jclass, jobjectArray, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitializePriorsAndUpdateCells + * Signature: (ZII[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializePriorsAndUpdateCells + (JNIEnv *, jobject, jboolean, jint, jint, jbyteArray, jbyteArray, jbyteArray, jint); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniSubComputeReadLikelihoodGivenHaplotypeLog10 + * Signature: (II[B[B[B[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10 + (JNIEnv *, jobject, jint, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index d575b8271..a4cd63bc4 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -176,6 +176,9 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai #ifdef DEBUG3 for(unsigned k=0;kGetArrayLength(haplotypeBases);++k) g_load_time_initializer.debug_dump("haplotype_bases_jni.txt",to_string((int)haplotypeBasesArray[k]),true); +#endif +#ifdef DO_PROFILING + g_load_time_initializer.m_sumHaplotypeLengths += env->GetArrayLength(haplotypeBasesGlobalRef); #endif } } @@ -272,6 +275,10 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai tc_array[tc_idx].i = (char*)insertionGOPArray; tc_array[tc_idx].d = (char*)deletionGOPArray; tc_array[tc_idx].c = (char*)overallGCPArray; +#ifdef DO_PROFILING + g_load_time_initializer.m_sumProductReadLengthHaplotypeLength += (readLength*haplotypeLength); + g_load_time_initializer.m_sumSquareProductReadLengthHaplotypeLength += ((readLength*haplotypeLength)*(readLength*haplotypeLength)); +#endif ++tc_idx; } //Release read arrays at end because they are used by compute_full_prob @@ -283,6 +290,9 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai readBasesArrayVector[i][2] = make_pair(insertionGOP, insertionGOPArray); readBasesArrayVector[i][3] = make_pair(deletionGOP, deletionGOPArray); readBasesArrayVector[i][4] = make_pair(overallGCP, overallGCPArray); +#ifdef DO_PROFILING + g_load_time_initializer.m_sumReadLengths += readLength; +#endif } #ifdef DO_PROFILING g_load_time_initializer.m_data_transfer_time += get_time(); diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h new file mode 100644 index 000000000..099751b79 --- /dev/null +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h @@ -0,0 +1,79 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM */ + +#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM +#define _Included_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM +#ifdef __cplusplus +extern "C" { +#endif +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_TRISTATE_CORRECTION +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_TRISTATE_CORRECTION 3.0 +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToMatch +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToMatch 0L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_indelToMatch +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_indelToMatch 1L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToInsertion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToInsertion 2L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_insertionToInsertion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_insertionToInsertion 3L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToDeletion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToDeletion 4L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_deletionToDeletion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_deletionToDeletion 5L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_sse42Mask +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_sse42Mask 1LL +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_avxMask +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_avxMask 2LL +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniGlobalInit + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGlobalInit + (JNIEnv *, jobject, jclass, jclass, jlong); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniInitializeHaplotypes + * Signature: (I[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniFinalizeRegion + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniComputeLikelihoods + * Signature: (II[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIReadDataHolderClass;[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/PairHMM_JNI/run.sh b/PairHMM_JNI/run.sh index 1e82092e8..65a53289a 100755 --- a/PairHMM_JNI/run.sh +++ b/PairHMM_JNI/run.sh @@ -2,17 +2,17 @@ rm -f *.txt *.log GSA_ROOT_DIR=/home/karthikg/broad/gsa-unstable #-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed -java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ +java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI/ -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ -R /data/broad/samples/joint_variant_calling/broad_reference/human_g1k_v37_decoy.fasta \ -I /data/broad/samples/joint_variant_calling/NA12878_low_coverage_alignment/NA12878.chrom11.ILLUMINA.bwa.CEU.low_coverage.20121211.bam \ --dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -stand_call_conf 50.0 \ -stand_emit_conf 10.0 \ --pair_hmm_implementation JNI_LOGLESS_CACHING \ --o output.raw.snps.indels.vcf +-o output.raw.snps.indels.vcf -#-XL unmapped \ #--pair_hmm_implementation JNI_LOGLESS_CACHING \ +#-XL unmapped \ #-I /data/simulated/sim1M_pairs_final.bam \ #-I /data/broad/samples/joint_variant_calling/NA12878_low_coverage_alignment/NA12878.chrom11.ILLUMINA.bwa.CEU.low_coverage.20121211.bam \ #-I /data/broad/samples/joint_variant_calling/NA12878_high_coverage_alignment/NA12878.mapped.ILLUMINA.bwa.CEU.high_coverage_pcr_free.20130906.bam \ diff --git a/PairHMM_JNI/utils.cc b/PairHMM_JNI/utils.cc index da6509763..c33bd0eda 100644 --- a/PairHMM_JNI/utils.cc +++ b/PairHMM_JNI/utils.cc @@ -33,17 +33,27 @@ bool is_sse42_supported() return ((ecx >> 20)&1) == 1; } -void initialize_function_pointers() +uint64_t get_machine_capabilities() +{ + uint64_t machine_mask = 0ull; + if(is_avx_supported()) + machine_mask |= (1 << AVX_CUSTOM_IDX); + if(is_sse42_supported()) + machine_mask |= (1 << SSE42_CUSTOM_IDX); + return machine_mask; +} + +void initialize_function_pointers(uint64_t mask) { //if(false) - if(is_avx_supported()) + if(is_avx_supported() && (mask & (1<< AVX_CUSTOM_IDX))) { cout << "Using AVX accelerated implementation of PairHMM\n"; g_compute_full_prob_float = compute_full_prob_avxs; g_compute_full_prob_double = compute_full_prob_avxd; } else - if(is_sse42_supported()) + if(is_sse42_supported() && (mask & (1<< SSE42_CUSTOM_IDX))) { cout << "Using SSE4.2 accelerated implementation of PairHMM\n"; g_compute_full_prob_float = compute_full_prob_sses; diff --git a/PairHMM_JNI/utils.h b/PairHMM_JNI/utils.h index cc020ba40..a09c0204c 100644 --- a/PairHMM_JNI/utils.h +++ b/PairHMM_JNI/utils.h @@ -25,7 +25,14 @@ extern double (*g_compute_full_prob_double)(testcase *tc, double* before_last_lo void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline); template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); -void initialize_function_pointers(); +void initialize_function_pointers(uint64_t mask=0xFFFFFFFFFFFFFFFFull); double getCurrClk(); uint64_t get_time(struct timespec* x=0); + +enum ProcessorCapabilitiesEnum +{ + SSE42_CUSTOM_IDX=0, + AVX_CUSTOM_IDX +}; +uint64_t get_machine_capabilities(); #endif diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index b91805a62..66d8ee33c 100644 --- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -91,8 +91,10 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation return new LoglessPairHMM(); else return new CnyPairHMM(); - case JNI_LOGLESS_CACHING: - return new JNILoglessPairHMM(); + case VECTOR_LOGLESS_CACHING: + return new VectorLoglessPairHMM(); + case DEBUG_JNI_LOGLESS_CACHING: + return new DebugJNILoglessPairHMM(hmmType); case ARRAY_LOGLESS: if (noFpga || !CnyPairHMM.isAvailable()) return new ArrayLoglessPairHMM(); diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java new file mode 100644 index 000000000..294c5c451 --- /dev/null +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java @@ -0,0 +1,496 @@ +/* +* By downloading the PROGRAM you agree to the following terms of use: +* +* BROAD INSTITUTE - SOFTWARE LICENSE AGREEMENT - FOR ACADEMIC NON-COMMERCIAL RESEARCH PURPOSES ONLY +* +* This Agreement is made between the Broad Institute, Inc. with a principal address at 7 Cambridge Center, Cambridge, MA 02142 (BROAD) and the LICENSEE and is effective at the date the downloading is completed (EFFECTIVE DATE). +* +* WHEREAS, LICENSEE desires to license the PROGRAM, as defined hereinafter, and BROAD wishes to have this PROGRAM utilized in the public interest, subject only to the royalty-free, nonexclusive, nontransferable license rights of the United States Government pursuant to 48 CFR 52.227-14; and +* WHEREAS, LICENSEE desires to license the PROGRAM and BROAD desires to grant a license on the following terms and conditions. +* NOW, THEREFORE, in consideration of the promises and covenants made herein, the parties hereto agree as follows: +* +* 1. DEFINITIONS +* 1.1 PROGRAM shall mean copyright in the object code and source code known as GATK2 and related documentation, if any, as they exist on the EFFECTIVE DATE and can be downloaded from http://www.broadinstitute/GATK on the EFFECTIVE DATE. +* +* 2. LICENSE +* 2.1 Grant. Subject to the terms of this Agreement, BROAD hereby grants to LICENSEE, solely for academic non-commercial research purposes, a non-exclusive, non-transferable license to: (a) download, execute and display the PROGRAM and (b) create bug fixes and modify the PROGRAM. +* The LICENSEE may apply the PROGRAM in a pipeline to data owned by users other than the LICENSEE and provide these users the results of the PROGRAM provided LICENSEE does so for academic non-commercial purposes only. For clarification purposes, academic sponsored research is not a commercial use under the terms of this Agreement. +* 2.2 No Sublicensing or Additional Rights. LICENSEE shall not sublicense or distribute the PROGRAM, in whole or in part, without prior written permission from BROAD. LICENSEE shall ensure that all of its users agree to the terms of this Agreement. LICENSEE further agrees that it shall not put the PROGRAM on a network, server, or other similar technology that may be accessed by anyone other than the LICENSEE and its employees and users who have agreed to the terms of this agreement. +* 2.3 License Limitations. Nothing in this Agreement shall be construed to confer any rights upon LICENSEE by implication, estoppel, or otherwise to any computer software, trademark, intellectual property, or patent rights of BROAD, or of any other entity, except as expressly granted herein. LICENSEE agrees that the PROGRAM, in whole or part, shall not be used for any commercial purpose, including without limitation, as the basis of a commercial software or hardware product or to provide services. LICENSEE further agrees that the PROGRAM shall not be copied or otherwise adapted in order to circumvent the need for obtaining a license for use of the PROGRAM. +* +* 3. OWNERSHIP OF INTELLECTUAL PROPERTY +* LICENSEE acknowledges that title to the PROGRAM shall remain with BROAD. The PROGRAM is marked with the following BROAD copyright notice and notice of attribution to contributors. LICENSEE shall retain such notice on all copies. LICENSEE agrees to include appropriate attribution if any results obtained from use of the PROGRAM are included in any publication. +* Copyright 2012 Broad Institute, Inc. +* Notice of attribution: The GATK2 program was made available through the generosity of Medical and Population Genetics program at the Broad Institute, Inc. +* LICENSEE shall not use any trademark or trade name of BROAD, or any variation, adaptation, or abbreviation, of such marks or trade names, or any names of officers, faculty, students, employees, or agents of BROAD except as states above for attribution purposes. +* +* 4. INDEMNIFICATION +* LICENSEE shall indemnify, defend, and hold harmless BROAD, and their respective officers, faculty, students, employees, associated investigators and agents, and their respective successors, heirs and assigns, (Indemnitees), against any liability, damage, loss, or expense (including reasonable attorneys fees and expenses) incurred by or imposed upon any of the Indemnitees in connection with any claims, suits, actions, demands or judgments arising out of any theory of liability (including, without limitation, actions in the form of tort, warranty, or strict liability and regardless of whether such action has any factual basis) pursuant to any right or license granted under this Agreement. +* +* 5. NO REPRESENTATIONS OR WARRANTIES +* THE PROGRAM IS DELIVERED AS IS. BROAD MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE PROGRAM OR THE COPYRIGHT, EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, WHETHER OR NOT DISCOVERABLE. BROAD EXTENDS NO WARRANTIES OF ANY KIND AS TO PROGRAM CONFORMITY WITH WHATEVER USER MANUALS OR OTHER LITERATURE MAY BE ISSUED FROM TIME TO TIME. +* IN NO EVENT SHALL BROAD OR ITS RESPECTIVE DIRECTORS, OFFICERS, EMPLOYEES, AFFILIATED INVESTIGATORS AND AFFILIATES BE LIABLE FOR INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, INCLUDING, WITHOUT LIMITATION, ECONOMIC DAMAGES OR INJURY TO PROPERTY AND LOST PROFITS, REGARDLESS OF WHETHER BROAD SHALL BE ADVISED, SHALL HAVE OTHER REASON TO KNOW, OR IN FACT SHALL KNOW OF THE POSSIBILITY OF THE FOREGOING. +* +* 6. ASSIGNMENT +* This Agreement is personal to LICENSEE and any rights or obligations assigned by LICENSEE without the prior written consent of BROAD shall be null and void. +* +* 7. MISCELLANEOUS +* 7.1 Export Control. LICENSEE gives assurance that it will comply with all United States export control laws and regulations controlling the export of the PROGRAM, including, without limitation, all Export Administration Regulations of the United States Department of Commerce. Among other things, these laws and regulations prohibit, or require a license for, the export of certain types of software to specified countries. +* 7.2 Termination. LICENSEE shall have the right to terminate this Agreement for any reason upon prior written notice to BROAD. If LICENSEE breaches any provision hereunder, and fails to cure such breach within thirty (30) days, BROAD may terminate this Agreement immediately. Upon termination, LICENSEE shall provide BROAD with written assurance that the original and all copies of the PROGRAM have been destroyed, except that, upon prior written authorization from BROAD, LICENSEE may retain a copy for archive purposes. +* 7.3 Survival. The following provisions shall survive the expiration or termination of this Agreement: Articles 1, 3, 4, 5 and Sections 2.2, 2.3, 7.3, and 7.4. +* 7.4 Notice. Any notices under this Agreement shall be in writing, shall specifically refer to this Agreement, and shall be sent by hand, recognized national overnight courier, confirmed facsimile transmission, confirmed electronic mail, or registered or certified mail, postage prepaid, return receipt requested. All notices under this Agreement shall be deemed effective upon receipt. +* 7.5 Amendment and Waiver; Entire Agreement. This Agreement may be amended, supplemented, or otherwise modified only by means of a written instrument signed by all parties. Any waiver of any rights or failure to act in a specific instance shall relate only to such instance and shall not be construed as an agreement to waive any rights or fail to act in any other instance, whether or not similar. This Agreement constitutes the entire agreement among the parties with respect to its subject matter and supersedes prior agreements or understandings between the parties relating to its subject matter. +* 7.6 Binding Effect; Headings. This Agreement shall be binding upon and inure to the benefit of the parties and their respective permitted successors and assigns. All headings are for convenience only and shall not affect the meaning of any provision of this Agreement. +* 7.7 Governing Law. This Agreement shall be construed, governed, interpreted and applied in accordance with the internal laws of the Commonwealth of Massachusetts, U.S.A., without regard to conflict of laws principles. +*/ + +package org.broadinstitute.sting.utils.pairhmm; + +import com.google.java.contract.Ensures; +import com.google.java.contract.Requires; +import org.broadinstitute.sting.utils.QualityUtils; + +import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap; +import org.broadinstitute.sting.utils.haplotype.Haplotype; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; +import org.broadinstitute.variant.variantcontext.Allele; +import org.broadinstitute.sting.utils.exceptions.UserException; + + +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.File; +import java.io.FileWriter; +import java.io.BufferedWriter; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; + + +/** + * Created with IntelliJ IDEA. + * User: rpoplin, carneiro + * Date: 10/16/12 + */ +public class DebugJNILoglessPairHMM extends LoglessPairHMM { + + private static final boolean debug = false; //simulates ifdef + private static final boolean verify = debug || false; //simulates ifdef + private static final boolean debug0_1 = false; //simulates ifdef + private static final boolean debug1 = false; //simulates ifdef + private static final boolean debug2 = false; + private static final boolean debug3 = false; + + //Debugging stats + private int numCalls = 0; + private int numComputeLikelihoodCalls = 0; + protected HashMap filenameToWriter = new HashMap(); + + private JNILoglessPairHMM jniPairHMM = null; + public DebugJNILoglessPairHMM(final PairHMM.HMM_IMPLEMENTATION hmmType) { + super(); + switch(hmmType) { + case VECTOR_LOGLESS_CACHING: + jniPairHMM = new VectorLoglessPairHMM(); + break; + default: + throw new UserException.BadArgumentValue("pairHMM","Specified JNIPairHMM implementation is unrecognized or incompatible with the HaplotypeCaller. Acceptable options are VECTOR_LOGLESS_CACHING"); + } + } + + @Override + public void close() + { + jniPairHMM.close(); + debugClose(); + } + + //Used only when testing parts of the compute kernel + /** + * {@inheritDoc} + */ + @Override + public void initialize( final int readMaxLength, final int haplotypeMaxLength ) { + if(verify) + super.initialize(readMaxLength, haplotypeMaxLength); + if(debug3) + { + System.out.println("Java: alloc initialized readMaxLength : "+readMaxLength+" haplotypeMaxLength : "+haplotypeMaxLength); + debugDump("lengths_java.txt", String.format("%d %d\n",readMaxLength, haplotypeMaxLength), + true); + } + if(debug2) + jniInitialize(readMaxLength, haplotypeMaxLength); + } + + private HashMap haplotypeToHaplotypeListIdxMap = null; + //Used to transfer data to JNI + //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region + /** + * {@inheritDoc} + */ + @Override + public void initialize( final List haplotypes, final Map> perSampleReadList, + final int readMaxLength, final int haplotypeMaxLength ) { + if(verify) + super.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); + jniPairHMM.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); + haplotypeToHaplotypeListIdxMap = jniPairHMM.getHaplotypeToHaplotypeListIdxMap(); + } + + /** + * {@inheritDoc} + */ + @Override + public void finalizeRegion() + { + jniPairHMM.finalizeRegion(); + } + + /** + * {@inheritDoc} + */ + @Override + public PerReadAlleleLikelihoodMap computeLikelihoods( final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap ) { + // (re)initialize the pairHMM only if necessary + final int readMaxLength = verify ? findMaxReadLength(reads) : 0; + final int haplotypeMaxLength = verify ? findMaxHaplotypeLength(alleleHaplotypeMap) : 0; + if(verify) + { + if (!initialized || readMaxLength > maxReadLength || haplotypeMaxLength > maxHaplotypeLength) + { initialize(readMaxLength, haplotypeMaxLength); } + if ( ! initialized ) + throw new IllegalStateException("Must call initialize before calling jniComputeLikelihoods in debug/verify mode"); + } + int readListSize = reads.size(); + int numHaplotypes = alleleHaplotypeMap.size(); + int numTestcases = readListSize*numHaplotypes; + if(debug0_1) + System.out.println("Java numReads "+readListSize+" numHaplotypes "+numHaplotypes); + int idx = 0; + for(GATKSAMRecord read : reads) + { + byte [] overallGCP = GCPArrayMap.get(read); + if(debug0_1) + System.out.println("Java read length "+read.getReadBases().length); + if(debug3) + { + for(int i=0;i currEntry : alleleHaplotypeMap.entrySet()) //order is important - access in same order always + { + byte[] haplotypeBases = currEntry.getValue().getBases(); + if(debug0_1) + System.out.println("Java haplotype length "+haplotypeBases.length); + if(debug3) + { + for(int i=0;i currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always + { + idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); + likelihoodArray[idx] = tmpArray[idxInsideHaplotypeList]; + ++idx; + } + readIdx += numHaplotypes; + } + //for floating point values, no exact equality + //check whether numbers are close in terms of abs_error or relative_error + //For very large values, relative_error is relevant + //For very small values, abs_error is relevant + boolean toDump = false; + for(int i=0;i 1e-5 && relative_error > 1e-5) + { + toDump = true; + break; + } + } + //if numbers are not close, then dump out the data that produced the inconsistency + if(toDump) + { + idx = 0; + System.out.println("Dump : Java numReads "+readListSize+" numHaplotypes "+numHaplotypes); + for(GATKSAMRecord read : reads) + { + byte [] overallGCP = GCPArrayMap.get(read); + for (Map.Entry currEntry : alleleHaplotypeMap.entrySet()) //order is important - access in same order always + { + byte[] haplotypeBases = currEntry.getValue().getBases(); + debugDump("debug_dump.txt",new String(haplotypeBases)+" ",true); + debugDump("debug_dump.txt",new String(read.getReadBases())+" ",true); + for(int k=0;k currEntry : filenameToWriter.entrySet()) { + BufferedWriter currWriter = currEntry.getValue(); + try + { + currWriter.flush(); + currWriter.close(); + } + catch(IOException e) + { + e.printStackTrace(); + + } + } + filenameToWriter.clear(); + } +} diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java index 6cbd3510c..1c7adf4c3 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java @@ -46,17 +46,8 @@ package org.broadinstitute.sting.utils.pairhmm; -import com.google.java.contract.Ensures; -import com.google.java.contract.Requires; -import org.broadinstitute.sting.utils.QualityUtils; - -import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap; import org.broadinstitute.sting.utils.haplotype.Haplotype; -import org.broadinstitute.sting.utils.sam.GATKSAMRecord; -import org.broadinstitute.variant.variantcontext.Allele; -import java.util.List; -import java.util.Map; import java.util.HashMap; @@ -65,464 +56,6 @@ import java.util.HashMap; * User: rpoplin, carneiro * Date: 10/16/12 */ -public class JNILoglessPairHMM extends LoglessPairHMM { - - private static final boolean debug = false; //simulates ifdef - private static final boolean verify = debug || false; //simulates ifdef - private static final boolean debug0_1 = false; //simulates ifdef - private static final boolean debug1 = false; //simulates ifdef - private static final boolean debug2 = false; - private static final boolean debug3 = false; - private int numComputeLikelihoodCalls = 0; - - //Used to copy references to byteArrays to JNI from reads - protected class JNIReadDataHolderClass - { - public byte[] readBases = null; - public byte[] readQuals = null; - public byte[] insertionGOP = null; - public byte[] deletionGOP = null; - public byte[] overallGCP = null; - } - - //Used to copy references to byteArrays to JNI from haplotypes - protected class JNIHaplotypeDataHolderClass - { - public byte[] haplotypeBases = null; - } - - private native void jniGlobalInit(Class readDataHolderClass, Class haplotypeDataHolderClass); - private native void jniClose(); - - private static boolean isJNILoglessPairHMMLibraryLoaded = false; - - public JNILoglessPairHMM() - { - super(); - if(!isJNILoglessPairHMMLibraryLoaded) - { - System.loadLibrary("JNILoglessPairHMM"); - isJNILoglessPairHMMLibraryLoaded = true; - jniGlobalInit(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class); //need to do this only once - } - } - - @Override - public void close() - { - jniClose(); - debugClose(); - } - - //Used to test parts of the compute kernel separately - private native void jniInitialize(final int readMaxLength, final int haplotypeMaxLength); - private native static void jniInitializeProbabilities(final double[][] transition, final byte[] insertionGOP, final byte[] deletionGOP, final byte[] overallGCP); - private native double jniInitializePriorsAndUpdateCells( - boolean doInitialization, - final int paddedReadLength, final int paddedHaplotypeLength, - final byte[] readBases, final byte[] haplotypeBases, final byte[] readQuals, - final int hapStartIndex); - private native double jniSubComputeReadLikelihoodGivenHaplotypeLog10( - final int readLength, final int haplotypeLength, - final byte[] readBases, final byte[] haplotypeBases, final byte[] readQuals, - final byte[] insertionGOP, final byte[] deletionGOP, final byte[] overallGCP, - final int hapStartIndex); - - - //Used only when testing parts of the compute kernel - /** - * {@inheritDoc} - */ - @Override - public void initialize(final int readMaxLength, final int haplotypeMaxLength) - { - if(verify) - super.initialize(readMaxLength, haplotypeMaxLength); - if(debug3) - { - System.out.println("Java: alloc initialized readMaxLength : "+readMaxLength+" haplotypeMaxLength : "+haplotypeMaxLength); - debugDump("lengths_java.txt", String.format("%d %d\n",readMaxLength, haplotypeMaxLength), - true); - } - if(debug2) - jniInitialize(readMaxLength, haplotypeMaxLength); - } - - private HashMap haplotypeToHaplotypeListIdxMap = new HashMap(); - private native void jniInitializeHaplotypes(final int numHaplotypes, JNIHaplotypeDataHolderClass[] haplotypeDataArray); - //Used to transfer data to JNI - //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region - /** - * {@inheritDoc} - */ - @Override - public void initialize( final List haplotypes, final Map> perSampleReadList, final int readMaxLength, final int haplotypeMaxLength ) { - if(verify) - super.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); - int numHaplotypes = haplotypes.size(); - JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; - int idx = 0; - for(final Haplotype currHaplotype : haplotypes) - { - haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); - haplotypeDataArray[idx].haplotypeBases = currHaplotype.getBases(); - haplotypeToHaplotypeListIdxMap.put(currHaplotype, idx); - ++idx; - } - jniInitializeHaplotypes(numHaplotypes, haplotypeDataArray); - } - - private native void jniFinalizeRegion(); - //Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not - //accessing Java memory directly, still important to release memory from C++ - /** - * {@inheritDoc} - */ - @Override - public void finalizeRegion() - { - jniFinalizeRegion(); - } - - //Real compute kernel - private native void jniComputeLikelihoods(int numReads, int numHaplotypes, - JNIReadDataHolderClass[] readDataArray, JNIHaplotypeDataHolderClass[] haplotypeDataArray, - double[] likelihoodArray, int maxNumThreadsToUse); - /** - * {@inheritDoc} - */ - @Override - public PerReadAlleleLikelihoodMap computeLikelihoods(final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap) - { - // (re)initialize the pairHMM only if necessary - final int readMaxLength = verify ? findMaxReadLength(reads) : 0; - final int haplotypeMaxLength = verify ? findMaxHaplotypeLength(alleleHaplotypeMap) : 0; - if(verify) - { - if (!initialized || readMaxLength > maxReadLength || haplotypeMaxLength > maxHaplotypeLength) - { initialize(readMaxLength, haplotypeMaxLength); } - if ( ! initialized ) - throw new IllegalStateException("Must call initialize before calling jniComputeLikelihoods in debug/verify mode"); - } - int readListSize = reads.size(); - int numHaplotypes = alleleHaplotypeMap.size(); - int numTestcases = readListSize*numHaplotypes; - if(debug0_1) - System.out.println("Java numReads "+readListSize+" numHaplotypes "+numHaplotypes); - JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; - int idx = 0; - for(GATKSAMRecord read : reads) - { - readDataArray[idx] = new JNIReadDataHolderClass(); - readDataArray[idx].readBases = read.getReadBases(); - readDataArray[idx].readQuals = read.getBaseQualities(); - readDataArray[idx].insertionGOP = read.getBaseInsertionQualities(); - readDataArray[idx].deletionGOP = read.getBaseDeletionQualities(); - readDataArray[idx].overallGCP = GCPArrayMap.get(read); - - if(debug0_1) - System.out.println("Java read length "+readDataArray[idx].readBases.length); - if(debug3) - { - for(int i=0;i currEntry : alleleHaplotypeMap.entrySet()) //order is important - access in same order always - { - haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); - haplotypeDataArray[idx].haplotypeBases = currEntry.getValue().getBases(); - if(debug0_1) - System.out.println("Java haplotype length "+haplotypeDataArray[idx].haplotypeBases.length); - if(debug3) - { - for(int i=0;i currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always - { - //Since the order of haplotypes in the List and alleleHaplotypeMap is different, - //get idx of current haplotype in the list and use this idx to get the right likelihoodValue - idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); - likelihoodMap.add(read, currEntry.getKey(), likelihoodArray[readIdx + idxInsideHaplotypeList]); - ++idx; - } - readIdx += numHaplotypes; - } - if(verify) - { - //re-order values in likelihoodArray - double[] tmpArray = new double[numHaplotypes]; - idx = 0; - idxInsideHaplotypeList = 0; - readIdx = 0; - for(GATKSAMRecord read : reads) - { - for(int j=0;j currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always - { - idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); - likelihoodArray[idx] = tmpArray[idxInsideHaplotypeList]; - ++idx; - } - readIdx += numHaplotypes; - } - //to compare values - likelihoodMap = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); - //for floating point values, no exact equality - //check whether numbers are close in terms of abs_error or relative_error - //For very large values, relative_error is relevant - //For very small values, abs_error is relevant - boolean toDump = false; - for(int i=0;i 1e-5 && relative_error > 1e-5) - { - toDump = true; - break; - } - } - //if numbers are not close, then dump out the data that produced the inconsistency - if(toDump) - { - idx = 0; - System.out.println("Dump : Java numReads "+readListSize+" numHaplotypes "+numHaplotypes); - for(int i=0;i getHaplotypeToHaplotypeListIdxMap(); } diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/LoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/LoglessPairHMM.java index 8f5a13ba3..125389217 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/LoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/LoglessPairHMM.java @@ -50,13 +50,6 @@ import com.google.java.contract.Ensures; import com.google.java.contract.Requires; import org.broadinstitute.sting.utils.QualityUtils; -import java.io.File; -import java.io.FileWriter; -import java.io.BufferedWriter; -import java.util.Map; -import java.util.HashMap; -import java.io.IOException; - /** * Created with IntelliJ IDEA. * User: rpoplin, carneiro @@ -76,48 +69,7 @@ public class LoglessPairHMM extends N2MemoryPairHMM { protected static final int matchToDeletion = 4; protected static final int deletionToDeletion = 5; - protected static int numCalls = 0; - protected HashMap filenameToWriter = new HashMap(); - protected void debugDump(String filename, String s, boolean toAppend) - { - try - { - File file = new File(filename); - if (!file.exists()) - file.createNewFile(); - BufferedWriter currWriter = filenameToWriter.get(filename); - if(currWriter == null) - { - FileWriter fw = new FileWriter(file, toAppend); - currWriter = new BufferedWriter(fw); - filenameToWriter.put(filename, currWriter); - } - currWriter.write(s); - } - catch(IOException e) - { - e.printStackTrace(); - } - } - - protected void debugClose() - { - for(Map.Entry currEntry : filenameToWriter.entrySet()) - { - BufferedWriter currWriter = currEntry.getValue(); - try - { - currWriter.flush(); - currWriter.close(); - } - catch(IOException e) - { - e.printStackTrace(); - - } - } - filenameToWriter.clear(); - } + /** * {@inheritDoc} */ diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java new file mode 100644 index 000000000..9d6812c0c --- /dev/null +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java @@ -0,0 +1,198 @@ +/* +* By downloading the PROGRAM you agree to the following terms of use: +* +* BROAD INSTITUTE - SOFTWARE LICENSE AGREEMENT - FOR ACADEMIC NON-COMMERCIAL RESEARCH PURPOSES ONLY +* +* This Agreement is made between the Broad Institute, Inc. with a principal address at 7 Cambridge Center, Cambridge, MA 02142 (BROAD) and the LICENSEE and is effective at the date the downloading is completed (EFFECTIVE DATE). +* +* WHEREAS, LICENSEE desires to license the PROGRAM, as defined hereinafter, and BROAD wishes to have this PROGRAM utilized in the public interest, subject only to the royalty-free, nonexclusive, nontransferable license rights of the United States Government pursuant to 48 CFR 52.227-14; and +* WHEREAS, LICENSEE desires to license the PROGRAM and BROAD desires to grant a license on the following terms and conditions. +* NOW, THEREFORE, in consideration of the promises and covenants made herein, the parties hereto agree as follows: +* +* 1. DEFINITIONS +* 1.1 PROGRAM shall mean copyright in the object code and source code known as GATK2 and related documentation, if any, as they exist on the EFFECTIVE DATE and can be downloaded from http://www.broadinstitute/GATK on the EFFECTIVE DATE. +* +* 2. LICENSE +* 2.1 Grant. Subject to the terms of this Agreement, BROAD hereby grants to LICENSEE, solely for academic non-commercial research purposes, a non-exclusive, non-transferable license to: (a) download, execute and display the PROGRAM and (b) create bug fixes and modify the PROGRAM. +* The LICENSEE may apply the PROGRAM in a pipeline to data owned by users other than the LICENSEE and provide these users the results of the PROGRAM provided LICENSEE does so for academic non-commercial purposes only. For clarification purposes, academic sponsored research is not a commercial use under the terms of this Agreement. +* 2.2 No Sublicensing or Additional Rights. LICENSEE shall not sublicense or distribute the PROGRAM, in whole or in part, without prior written permission from BROAD. LICENSEE shall ensure that all of its users agree to the terms of this Agreement. LICENSEE further agrees that it shall not put the PROGRAM on a network, server, or other similar technology that may be accessed by anyone other than the LICENSEE and its employees and users who have agreed to the terms of this agreement. +* 2.3 License Limitations. Nothing in this Agreement shall be construed to confer any rights upon LICENSEE by implication, estoppel, or otherwise to any computer software, trademark, intellectual property, or patent rights of BROAD, or of any other entity, except as expressly granted herein. LICENSEE agrees that the PROGRAM, in whole or part, shall not be used for any commercial purpose, including without limitation, as the basis of a commercial software or hardware product or to provide services. LICENSEE further agrees that the PROGRAM shall not be copied or otherwise adapted in order to circumvent the need for obtaining a license for use of the PROGRAM. +* +* 3. OWNERSHIP OF INTELLECTUAL PROPERTY +* LICENSEE acknowledges that title to the PROGRAM shall remain with BROAD. The PROGRAM is marked with the following BROAD copyright notice and notice of attribution to contributors. LICENSEE shall retain such notice on all copies. LICENSEE agrees to include appropriate attribution if any results obtained from use of the PROGRAM are included in any publication. +* Copyright 2012 Broad Institute, Inc. +* Notice of attribution: The GATK2 program was made available through the generosity of Medical and Population Genetics program at the Broad Institute, Inc. +* LICENSEE shall not use any trademark or trade name of BROAD, or any variation, adaptation, or abbreviation, of such marks or trade names, or any names of officers, faculty, students, employees, or agents of BROAD except as states above for attribution purposes. +* +* 4. INDEMNIFICATION +* LICENSEE shall indemnify, defend, and hold harmless BROAD, and their respective officers, faculty, students, employees, associated investigators and agents, and their respective successors, heirs and assigns, (Indemnitees), against any liability, damage, loss, or expense (including reasonable attorneys fees and expenses) incurred by or imposed upon any of the Indemnitees in connection with any claims, suits, actions, demands or judgments arising out of any theory of liability (including, without limitation, actions in the form of tort, warranty, or strict liability and regardless of whether such action has any factual basis) pursuant to any right or license granted under this Agreement. +* +* 5. NO REPRESENTATIONS OR WARRANTIES +* THE PROGRAM IS DELIVERED AS IS. BROAD MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE PROGRAM OR THE COPYRIGHT, EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, WHETHER OR NOT DISCOVERABLE. BROAD EXTENDS NO WARRANTIES OF ANY KIND AS TO PROGRAM CONFORMITY WITH WHATEVER USER MANUALS OR OTHER LITERATURE MAY BE ISSUED FROM TIME TO TIME. +* IN NO EVENT SHALL BROAD OR ITS RESPECTIVE DIRECTORS, OFFICERS, EMPLOYEES, AFFILIATED INVESTIGATORS AND AFFILIATES BE LIABLE FOR INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, INCLUDING, WITHOUT LIMITATION, ECONOMIC DAMAGES OR INJURY TO PROPERTY AND LOST PROFITS, REGARDLESS OF WHETHER BROAD SHALL BE ADVISED, SHALL HAVE OTHER REASON TO KNOW, OR IN FACT SHALL KNOW OF THE POSSIBILITY OF THE FOREGOING. +* +* 6. ASSIGNMENT +* This Agreement is personal to LICENSEE and any rights or obligations assigned by LICENSEE without the prior written consent of BROAD shall be null and void. +* +* 7. MISCELLANEOUS +* 7.1 Export Control. LICENSEE gives assurance that it will comply with all United States export control laws and regulations controlling the export of the PROGRAM, including, without limitation, all Export Administration Regulations of the United States Department of Commerce. Among other things, these laws and regulations prohibit, or require a license for, the export of certain types of software to specified countries. +* 7.2 Termination. LICENSEE shall have the right to terminate this Agreement for any reason upon prior written notice to BROAD. If LICENSEE breaches any provision hereunder, and fails to cure such breach within thirty (30) days, BROAD may terminate this Agreement immediately. Upon termination, LICENSEE shall provide BROAD with written assurance that the original and all copies of the PROGRAM have been destroyed, except that, upon prior written authorization from BROAD, LICENSEE may retain a copy for archive purposes. +* 7.3 Survival. The following provisions shall survive the expiration or termination of this Agreement: Articles 1, 3, 4, 5 and Sections 2.2, 2.3, 7.3, and 7.4. +* 7.4 Notice. Any notices under this Agreement shall be in writing, shall specifically refer to this Agreement, and shall be sent by hand, recognized national overnight courier, confirmed facsimile transmission, confirmed electronic mail, or registered or certified mail, postage prepaid, return receipt requested. All notices under this Agreement shall be deemed effective upon receipt. +* 7.5 Amendment and Waiver; Entire Agreement. This Agreement may be amended, supplemented, or otherwise modified only by means of a written instrument signed by all parties. Any waiver of any rights or failure to act in a specific instance shall relate only to such instance and shall not be construed as an agreement to waive any rights or fail to act in any other instance, whether or not similar. This Agreement constitutes the entire agreement among the parties with respect to its subject matter and supersedes prior agreements or understandings between the parties relating to its subject matter. +* 7.6 Binding Effect; Headings. This Agreement shall be binding upon and inure to the benefit of the parties and their respective permitted successors and assigns. All headings are for convenience only and shall not affect the meaning of any provision of this Agreement. +* 7.7 Governing Law. This Agreement shall be construed, governed, interpreted and applied in accordance with the internal laws of the Commonwealth of Massachusetts, U.S.A., without regard to conflict of laws principles. +*/ + +package org.broadinstitute.sting.utils.pairhmm; + +import com.google.java.contract.Ensures; +import com.google.java.contract.Requires; +import org.broadinstitute.sting.utils.QualityUtils; + +import org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap; +import org.broadinstitute.sting.utils.haplotype.Haplotype; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; +import org.broadinstitute.variant.variantcontext.Allele; + +import java.util.List; +import java.util.Map; +import java.util.HashMap; + + +/** + * Created with IntelliJ IDEA. + * User: rpoplin, carneiro + * Date: 10/16/12 + */ +public class VectorLoglessPairHMM extends JNILoglessPairHMM { + + //For machine capabilities + public static final long sse42Mask = 1; + public static final long avxMask = 2; + + //Used to copy references to byteArrays to JNI from reads + protected class JNIReadDataHolderClass { + public byte[] readBases = null; + public byte[] readQuals = null; + public byte[] insertionGOP = null; + public byte[] deletionGOP = null; + public byte[] overallGCP = null; + } + + //Used to copy references to byteArrays to JNI from haplotypes + protected class JNIHaplotypeDataHolderClass { + public byte[] haplotypeBases = null; + } + + public native long jniGetMachineType(); + private native void jniClose(); + private native void jniGlobalInit(Class readDataHolderClass, Class haplotypeDataHolderClass, long mask); + + private static boolean isVectorLoglessPairHMMLibraryLoaded = false; + public VectorLoglessPairHMM() { + super(); + if(!isVectorLoglessPairHMMLibraryLoaded) { + System.loadLibrary("VectorLoglessPairHMM"); + isVectorLoglessPairHMMLibraryLoaded = true; + jniGlobalInit(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, 0xFFFFFFFFFFFFFFFFl); //need to do this only once + } + } + + @Override + public void close() + { + jniClose(); + } + + private native void jniInitializeHaplotypes(final int numHaplotypes, JNIHaplotypeDataHolderClass[] haplotypeDataArray); + //Hold the mapping between haplotype and index in the list of Haplotypes passed to initialize + //Use this mapping in computeLikelihoods to find the likelihood value corresponding to a given Haplotype + private HashMap haplotypeToHaplotypeListIdxMap = new HashMap(); + @Override + public HashMap getHaplotypeToHaplotypeListIdxMap() { return haplotypeToHaplotypeListIdxMap; } + //Used to transfer data to JNI + //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region + + /** + * {@inheritDoc} + */ + @Override + public void initialize( final List haplotypes, final Map> perSampleReadList, + final int readMaxLength, final int haplotypeMaxLength ) { + int numHaplotypes = haplotypes.size(); + JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; + int idx = 0; + haplotypeToHaplotypeListIdxMap.clear(); + for(final Haplotype currHaplotype : haplotypes) + { + haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); + haplotypeDataArray[idx].haplotypeBases = currHaplotype.getBases(); + haplotypeToHaplotypeListIdxMap.put(currHaplotype, idx); + ++idx; + } + jniInitializeHaplotypes(numHaplotypes, haplotypeDataArray); + } + + private native void jniFinalizeRegion(); + //Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not + //accessing Java memory directly, still important to release memory from C++ + /** + * {@inheritDoc} + */ + @Override + public void finalizeRegion() + { + jniFinalizeRegion(); + } + + //Real compute kernel + private native void jniComputeLikelihoods(int numReads, int numHaplotypes, JNIReadDataHolderClass[] readDataArray, + JNIHaplotypeDataHolderClass[] haplotypeDataArray, double[] likelihoodArray, int maxNumThreadsToUse); + /** + * {@inheritDoc} + */ + @Override + public PerReadAlleleLikelihoodMap computeLikelihoods( final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap ) { + int readListSize = reads.size(); + int numHaplotypes = alleleHaplotypeMap.size(); + int numTestcases = readListSize*numHaplotypes; + JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; + int idx = 0; + for(GATKSAMRecord read : reads) + { + readDataArray[idx] = new JNIReadDataHolderClass(); + readDataArray[idx].readBases = read.getReadBases(); + readDataArray[idx].readQuals = read.getBaseQualities(); + readDataArray[idx].insertionGOP = read.getBaseInsertionQualities(); + readDataArray[idx].deletionGOP = read.getBaseDeletionQualities(); + readDataArray[idx].overallGCP = GCPArrayMap.get(read); + ++idx; + } + + JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; + mLikelihoodArray = new double[readListSize*numHaplotypes]; //to store results + //for(reads) + // for(haplotypes) + // compute_full_prob() + jniComputeLikelihoods(readListSize, numHaplotypes, readDataArray, haplotypeDataArray, mLikelihoodArray, 12); + + final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap(); + idx = 0; + int idxInsideHaplotypeList = 0; + int readIdx = 0; + for(GATKSAMRecord read : reads) + { + for (Map.Entry currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always + { + //Since the order of haplotypes in the List and alleleHaplotypeMap is different, + //get idx of current haplotype in the list and use this idx to get the right likelihoodValue + idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); + likelihoodMap.add(read, currEntry.getKey(), mLikelihoodArray[readIdx + idxInsideHaplotypeList]); + ++idx; + } + readIdx += numHaplotypes; + } + return likelihoodMap; + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java index 3a211ba2a..8b2123751 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java +++ b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java @@ -58,7 +58,9 @@ public abstract class PairHMM { /* Optimized version of the PairHMM which caches per-read computations and operations in real space to avoid costly sums of log10'ed likelihoods */ LOGLESS_CACHING, /* Optimized AVX implementation of LOGLESS_CACHING called through JNI */ - JNI_LOGLESS_CACHING, + VECTOR_LOGLESS_CACHING, + /* Debugging for any JNI implementation of LOGLESS_CACHING */ + DEBUG_JNI_LOGLESS_CACHING, /* Logless caching PairHMM that stores computations in 1D arrays instead of matrices, and which proceeds diagonally over the (read x haplotype) intersection matrix */ ARRAY_LOGLESS } @@ -301,6 +303,7 @@ public abstract class PairHMM { return Math.min(haplotype1.length, haplotype2.length); } + public double[] getLikelihoodArray() { return mLikelihoodArray; } //Called at the end of all HC calls public void close() { ; } } From e7598dde8b1f0ab49e51a61c71bf3c1c626ea56b Mon Sep 17 00:00:00 2001 From: mghodrat Date: Sun, 26 Jan 2014 11:36:06 -0800 Subject: [PATCH 22/72] Clean up --- PairHMM_JNI/avx_function_instantiations.cc | 8 +- PairHMM_JNI/baseline.cc | 8 +- PairHMM_JNI/define-double.h | 236 +++--- PairHMM_JNI/define-float.h | 229 +++--- PairHMM_JNI/define-sse-double.h | 171 +++-- PairHMM_JNI/define-sse-float.h | 173 +++-- ...e_sting_utils_pairhmm_JNILoglessPairHMM.cc | 6 +- PairHMM_JNI/pairhmm-template-kernel.cc | 689 ++++++++---------- PairHMM_JNI/pairhmm-template-main.cc | 169 ++--- PairHMM_JNI/shift_template.c | 136 ++-- PairHMM_JNI/sse_function_instantiations.cc | 8 +- PairHMM_JNI/template.h | 6 +- PairHMM_JNI/utils.cc | 4 +- PairHMM_JNI/vector_defs.h | 24 +- PairHMM_JNI/vector_function_prototypes.h | 32 +- 15 files changed, 863 insertions(+), 1036 deletions(-) diff --git a/PairHMM_JNI/avx_function_instantiations.cc b/PairHMM_JNI/avx_function_instantiations.cc index 4118fc5cf..8f0de827d 100644 --- a/PairHMM_JNI/avx_function_instantiations.cc +++ b/PairHMM_JNI/avx_function_instantiations.cc @@ -1,10 +1,10 @@ #include "template.h" -#undef SIMD_TYPE -#undef SIMD_TYPE_SSE +#undef SIMD_ENGINE +#undef SIMD_ENGINE_SSE -#define SIMD_TYPE avx -#define SIMD_TYPE_AVX +#define SIMD_ENGINE avx +#define SIMD_ENGINE_AVX #include "define-float.h" #include "shift_template.c" diff --git a/PairHMM_JNI/baseline.cc b/PairHMM_JNI/baseline.cc index b953c4436..2f80acdb0 100644 --- a/PairHMM_JNI/baseline.cc +++ b/PairHMM_JNI/baseline.cc @@ -10,10 +10,10 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) Context ctx; - NUMBER M[MROWS][MCOLS]; - NUMBER X[MROWS][MCOLS]; - NUMBER Y[MROWS][MCOLS]; - NUMBER p[MROWS][6]; + NUMBER M[ROWS][COLS]; + NUMBER X[ROWS][COLS]; + NUMBER Y[ROWS][COLS]; + NUMBER p[ROWS][6]; p[0][MM] = ctx._(0.0); p[0][GapM] = ctx._(0.0); diff --git a/PairHMM_JNI/define-double.h b/PairHMM_JNI/define-double.h index 502b919fe..83589a13d 100644 --- a/PairHMM_JNI/define-double.h +++ b/PairHMM_JNI/define-double.h @@ -1,53 +1,51 @@ #include #ifdef PRECISION - #undef PRECISION - #undef MAIN_TYPE - #undef MAIN_TYPE_SIZE - #undef UNION_TYPE - #undef IF_128 - #undef IF_MAIN_TYPE - #undef SHIFT_CONST1 - #undef SHIFT_CONST2 - #undef SHIFT_CONST3 - #undef _128_TYPE - #undef _256_TYPE - #undef AVX_LENGTH - #undef MAVX_COUNT - #undef HAP_TYPE - #undef MASK_TYPE - #undef MASK_ALL_ONES +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES - #undef SET_VEC_ZERO(__vec) - #undef VEC_OR(__v1, __v2) - #undef VEC_ADD(__v1, __v2) - #undef VEC_SUB(__v1, __v2) - #undef VEC_MUL(__v1, __v2) - #undef VEC_DIV(__v1, __v2) - #undef VEC_BLEND(__v1, __v2, __mask) - #undef VEC_BLENDV(__v1, __v2, __maskV) - #undef VEC_CAST_256_128(__v1) - #undef VEC_EXTRACT_128(__v1, __im) - #undef VEC_EXTRACT_UNIT(__v1, __im) - #undef VEC_SET1_VAL128(__val) - #undef VEC_MOVE(__v1, __val) - #undef VEC_CAST_128_256(__v1) - #undef VEC_INSERT_VAL(__v1, __val, __pos) - #undef VEC_CVT_128_256(__v1) - #undef VEC_SET1_VAL(__val) - #undef VEC_POPCVT_CHAR(__ch) - #undef VEC_LDPOPCVT_CHAR(__addr) - #undef VEC_CMP_EQ(__v1, __v2) - #undef VEC_SET_LSE(__val) - #undef SHIFT_HAP(__v1, __val) - #undef print256b(__v1) - #undef MASK_VEC - #undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) - #undef VEC_SHIFT_LEFT_1BIT(__vs) - #undef MASK_ALL_ONES - #undef COMPARE_VECS(__v1, __v2) - #undef _256_INT_TYPE - #undef BITMASK_VEC +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC #endif #define PRECISION d @@ -60,128 +58,122 @@ #define SHIFT_CONST2 1 #define SHIFT_CONST3 8 #define _128_TYPE __m128d -#define _256_TYPE __m256d +#define SIMD_TYPE __m256d #define _256_INT_TYPE __m256i #define AVX_LENGTH 4 -#define MAVX_COUNT (MROWS+7)/AVX_LENGTH #define HAP_TYPE __m128i #define MASK_TYPE uint64_t #define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFF #define MASK_VEC MaskVec_D #define SET_VEC_ZERO(__vec) \ - __vec= _mm256_setzero_pd() + __vec= _mm256_setzero_pd() #define VEC_OR(__v1, __v2) \ - _mm256_or_pd(__v1, __v2) + _mm256_or_pd(__v1, __v2) #define VEC_ADD(__v1, __v2) \ - _mm256_add_pd(__v1, __v2) + _mm256_add_pd(__v1, __v2) #define VEC_SUB(__v1, __v2) \ - _mm256_sub_pd(__v1, __v2) + _mm256_sub_pd(__v1, __v2) #define VEC_MUL(__v1, __v2) \ - _mm256_mul_pd(__v1, __v2) + _mm256_mul_pd(__v1, __v2) -#define VEC_DIV(__v1, __v2) \ - _mm256_div_pd(__v1, __v2) +#define VEC_DIV(__v1, __v2) \ + _mm256_div_pd(__v1, __v2) #define VEC_BLEND(__v1, __v2, __mask) \ - _mm256_blend_pd(__v1, __v2, __mask) + _mm256_blend_pd(__v1, __v2, __mask) #define VEC_BLENDV(__v1, __v2, __maskV) \ - _mm256_blendv_pd(__v1, __v2, __maskV) + _mm256_blendv_pd(__v1, __v2, __maskV) -#define VEC_CAST_256_128(__v1) \ - _mm256_castpd256_pd128 (__v1) +#define VEC_CAST_256_128(__v1) \ + _mm256_castpd256_pd128 (__v1) -#define VEC_EXTRACT_128(__v1, __im) \ - _mm256_extractf128_pd (__v1, __im) +#define VEC_EXTRACT_128(__v1, __im) \ + _mm256_extractf128_pd (__v1, __im) -#define VEC_EXTRACT_UNIT(__v1, __im) \ - _mm_extract_epi64(__v1, __im) +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi64(__v1, __im) -#define VEC_SET1_VAL128(__val) \ - _mm_set1_pd(__val) +#define VEC_SET1_VAL128(__val) \ + _mm_set1_pd(__val) -#define VEC_MOVE(__v1, __val) \ - _mm_move_sd(__v1, __val) +#define VEC_MOVE(__v1, __val) \ + _mm_move_sd(__v1, __val) -#define VEC_CAST_128_256(__v1) \ - _mm256_castpd128_pd256(__v1) +#define VEC_CAST_128_256(__v1) \ + _mm256_castpd128_pd256(__v1) -#define VEC_INSERT_VAL(__v1, __val, __pos) \ - _mm256_insertf128_pd(__v1, __val, __pos) +#define VEC_INSERT_VAL(__v1, __val, __pos) \ + _mm256_insertf128_pd(__v1, __val, __pos) -#define VEC_CVT_128_256(__v1) \ - _mm256_cvtepi32_pd(__v1) +#define VEC_CVT_128_256(__v1) \ + _mm256_cvtepi32_pd(__v1) -#define VEC_SET1_VAL(__val) \ - _mm256_set1_pd(__val) +#define VEC_SET1_VAL(__val) \ + _mm256_set1_pd(__val) -#define VEC_POPCVT_CHAR(__ch) \ - _mm256_cvtepi32_pd(_mm_set1_epi32(__ch)) +#define VEC_POPCVT_CHAR(__ch) \ + _mm256_cvtepi32_pd(_mm_set1_epi32(__ch)) #define VEC_LDPOPCVT_CHAR(__addr) \ - _mm256_cvtepi32_pd(_mm_load_si128((__m128i const *)__addr)) + _mm256_cvtepi32_pd(_mm_load_si128((__m128i const *)__addr)) -#define VEC_CMP_EQ(__v1, __v2) \ - _mm256_cmp_pd(__v1, __v2, _CMP_EQ_OQ) +#define VEC_CMP_EQ(__v1, __v2) \ + _mm256_cmp_pd(__v1, __v2, _CMP_EQ_OQ) #define VEC_SET_LSE(__val) \ - _mm256_set_pd(zero, zero, zero, __val); + _mm256_set_pd(zero, zero, zero, __val); -#define SHIFT_HAP(__v1, __val) \ - __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) +#define SHIFT_HAP(__v1, __val) \ + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) -#define print256b(__v1) \ - print256bDP(__v1) +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castpd128_pd256(__vsLow) ; \ +__vdst = _mm256_insertf128_pd(__vdst, __vsHigh, 1) ; -#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ - __vdst = _mm256_castpd128_pd256(__vsLow) ; \ - __vdst = _mm256_insertf128_pd(__vdst, __vsHigh, 1) ; - -#define VEC_SHIFT_LEFT_1BIT(__vs) \ - __vs = _mm_slli_epi64(__vs, 1) +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi64(__vs, 1) -#define COMPARE_VECS(__v1, __v2, __first, __last) { \ - double* ptr1 = (double*) (&__v1) ; \ - double* ptr2 = (double*) (&__v2) ; \ - for (int ei=__first; ei <= __last; ++ei) { \ - if (ptr1[ei] != ptr2[ei]) { \ - std::cout << "Double Mismatch at " << ei << ": " \ - << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ - exit(0) ; \ - } \ - } \ - } +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + double* ptr1 = (double*) (&__v1) ; \ + double* ptr2 = (double*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Double Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ +} class BitMaskVec_double { - MASK_VEC low_, high_ ; - _256_TYPE combined_ ; + MASK_VEC low_, high_ ; + SIMD_TYPE combined_ ; -public: - - inline MASK_TYPE& getLowEntry(int index) { - return low_.masks[index] ; - } - inline MASK_TYPE& getHighEntry(int index) { - return high_.masks[index] ; - } - - inline const _256_TYPE& getCombinedMask() { - VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + public: + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } - return combined_ ; - } - - inline void shift_left_1bit() { - VEC_SHIFT_LEFT_1BIT(low_.vec) ; - VEC_SHIFT_LEFT_1BIT(high_.vec) ; - } + inline const SIMD_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } } ; diff --git a/PairHMM_JNI/define-float.h b/PairHMM_JNI/define-float.h index 3cc57ec38..87b2b01f3 100644 --- a/PairHMM_JNI/define-float.h +++ b/PairHMM_JNI/define-float.h @@ -1,53 +1,51 @@ #include #ifdef PRECISION - #undef PRECISION - #undef MAIN_TYPE - #undef MAIN_TYPE_SIZE - #undef UNION_TYPE - #undef IF_128 - #undef IF_MAIN_TYPE - #undef SHIFT_CONST1 - #undef SHIFT_CONST2 - #undef SHIFT_CONST3 - #undef _128_TYPE - #undef _256_TYPE - #undef AVX_LENGTH - #undef MAVX_COUNT - #undef HAP_TYPE - #undef MASK_TYPE - #undef MASK_ALL_ONES +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES - #undef SET_VEC_ZERO(__vec) - #undef VEC_OR(__v1, __v2) - #undef VEC_ADD(__v1, __v2) - #undef VEC_SUB(__v1, __v2) - #undef VEC_MUL(__v1, __v2) - #undef VEC_DIV(__v1, __v2) - #undef VEC_BLEND(__v1, __v2, __mask) - #undef VEC_BLENDV(__v1, __v2, __maskV) - #undef VEC_CAST_256_128(__v1) - #undef VEC_EXTRACT_128(__v1, __im) - #undef VEC_EXTRACT_UNIT(__v1, __im) - #undef VEC_SET1_VAL128(__val) - #undef VEC_MOVE(__v1, __val) - #undef VEC_CAST_128_256(__v1) - #undef VEC_INSERT_VAL(__v1, __val, __pos) - #undef VEC_CVT_128_256(__v1) - #undef VEC_SET1_VAL(__val) - #undef VEC_POPCVT_CHAR(__ch) - #undef VEC_LDPOPCVT_CHAR(__addr) - #undef VEC_CMP_EQ(__v1, __v2) - #undef VEC_SET_LSE(__val) - #undef SHIFT_HAP(__v1, __val) - #undef print256b(__v1) - #undef MASK_VEC - #undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) - #undef VEC_SHIFT_LEFT_1BIT(__vs) - #undef MASK_ALL_ONES - #undef COMPARE_VECS(__v1, __v2) - #undef _256_INT_TYPE - #undef BITMASK_VEC +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC #endif #define PRECISION s @@ -61,127 +59,122 @@ #define SHIFT_CONST2 3 #define SHIFT_CONST3 4 #define _128_TYPE __m128 -#define _256_TYPE __m256 +#define SIMD_TYPE __m256 #define _256_INT_TYPE __m256i #define AVX_LENGTH 8 -#define MAVX_COUNT (MROWS+7)/AVX_LENGTH #define HAP_TYPE UNION_TYPE #define MASK_TYPE uint32_t #define MASK_ALL_ONES 0xFFFFFFFF #define MASK_VEC MaskVec_F #define SET_VEC_ZERO(__vec) \ - __vec= _mm256_setzero_ps() + __vec= _mm256_setzero_ps() #define VEC_OR(__v1, __v2) \ - _mm256_or_ps(__v1, __v2) + _mm256_or_ps(__v1, __v2) #define VEC_ADD(__v1, __v2) \ - _mm256_add_ps(__v1, __v2) + _mm256_add_ps(__v1, __v2) #define VEC_SUB(__v1, __v2) \ - _mm256_sub_ps(__v1, __v2) + _mm256_sub_ps(__v1, __v2) #define VEC_MUL(__v1, __v2) \ - _mm256_mul_ps(__v1, __v2) + _mm256_mul_ps(__v1, __v2) -#define VEC_DIV(__v1, __v2) \ - _mm256_div_ps(__v1, __v2) +#define VEC_DIV(__v1, __v2) \ + _mm256_div_ps(__v1, __v2) #define VEC_BLEND(__v1, __v2, __mask) \ - _mm256_blend_ps(__v1, __v2, __mask) + _mm256_blend_ps(__v1, __v2, __mask) #define VEC_BLENDV(__v1, __v2, __maskV) \ - _mm256_blendv_ps(__v1, __v2, __maskV) + _mm256_blendv_ps(__v1, __v2, __maskV) -#define VEC_CAST_256_128(__v1) \ - _mm256_castps256_ps128 (__v1) +#define VEC_CAST_256_128(__v1) \ + _mm256_castps256_ps128 (__v1) #define VEC_EXTRACT_128(__v1, __im) \ - _mm256_extractf128_ps (__v1, __im) + _mm256_extractf128_ps (__v1, __im) -#define VEC_EXTRACT_UNIT(__v1, __im) \ - _mm_extract_epi32(__v1, __im) +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi32(__v1, __im) -#define VEC_SET1_VAL128(__val) \ - _mm_set1_ps(__val) +#define VEC_SET1_VAL128(__val) \ + _mm_set1_ps(__val) -#define VEC_MOVE(__v1, __val) \ - _mm_move_ss(__v1, __val) +#define VEC_MOVE(__v1, __val) \ + _mm_move_ss(__v1, __val) #define VEC_CAST_128_256(__v1) \ - _mm256_castps128_ps256(__v1) + _mm256_castps128_ps256(__v1) #define VEC_INSERT_VAL(__v1, __val, __pos) \ - _mm256_insertf128_ps(__v1, __val, __pos) + _mm256_insertf128_ps(__v1, __val, __pos) #define VEC_CVT_128_256(__v1) \ - _mm256_cvtepi32_ps(__v1.i) + _mm256_cvtepi32_ps(__v1.i) -#define VEC_SET1_VAL(__val) \ - _mm256_set1_ps(__val) +#define VEC_SET1_VAL(__val) \ + _mm256_set1_ps(__val) #define VEC_POPCVT_CHAR(__ch) \ - _mm256_cvtepi32_ps(_mm256_set1_epi32(__ch)) + _mm256_cvtepi32_ps(_mm256_set1_epi32(__ch)) -#define VEC_LDPOPCVT_CHAR(__addr) \ - _mm256_cvtepi32_ps(_mm256_loadu_si256((__m256i const *)__addr)) +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm256_cvtepi32_ps(_mm256_loadu_si256((__m256i const *)__addr)) #define VEC_CMP_EQ(__v1, __v2) \ - _mm256_cmp_ps(__v1, __v2, _CMP_EQ_OQ) + _mm256_cmp_ps(__v1, __v2, _CMP_EQ_OQ) -#define VEC_SET_LSE(__val) \ - _mm256_set_ps(zero, zero, zero, zero, zero, zero, zero, __val); +#define VEC_SET_LSE(__val) \ + _mm256_set_ps(zero, zero, zero, zero, zero, zero, zero, __val); -#define SHIFT_HAP(__v1, __val) \ - _vector_shift_lastavxs(__v1, __val.f); +#define SHIFT_HAP(__v1, __val) \ + _vector_shift_lastavxs(__v1, __val.f); -#define print256b(__v1) \ - print256bFP(__v1) - -#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ - __vdst = _mm256_castps128_ps256(__vsLow) ; \ - __vdst = _mm256_insertf128_ps(__vdst, __vsHigh, 1) ; +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castps128_ps256(__vsLow) ; \ +__vdst = _mm256_insertf128_ps(__vdst, __vsHigh, 1) ; -#define VEC_SHIFT_LEFT_1BIT(__vs) \ - __vs = _mm_slli_epi32(__vs, 1) +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi32(__vs, 1) -#define COMPARE_VECS(__v1, __v2, __first, __last) { \ - float* ptr1 = (float*) (&__v1) ; \ - float* ptr2 = (float*) (&__v2) ; \ - for (int ei=__first; ei <= __last; ++ei) { \ - if (ptr1[ei] != ptr2[ei]) { \ - std::cout << "Float Mismatch at " << ei << ": " \ - << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ - exit(0) ; \ - } \ - } \ - } +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + float* ptr1 = (float*) (&__v1) ; \ + float* ptr2 = (float*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Float Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ +} class BitMaskVec_float { - MASK_VEC low_, high_ ; - _256_TYPE combined_ ; + MASK_VEC low_, high_ ; + SIMD_TYPE combined_ ; -public: - - inline MASK_TYPE& getLowEntry(int index) { - return low_.masks[index] ; - } - inline MASK_TYPE& getHighEntry(int index) { - return high_.masks[index] ; - } - - inline const _256_TYPE& getCombinedMask() { - VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + public: - return combined_ ; - } - - inline void shift_left_1bit() { - VEC_SHIFT_LEFT_1BIT(low_.vec) ; - VEC_SHIFT_LEFT_1BIT(high_.vec) ; - } + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } } ; diff --git a/PairHMM_JNI/define-sse-double.h b/PairHMM_JNI/define-sse-double.h index a30b2e5f5..d781d55f3 100644 --- a/PairHMM_JNI/define-sse-double.h +++ b/PairHMM_JNI/define-sse-double.h @@ -1,53 +1,51 @@ #ifdef PRECISION - #undef PRECISION - #undef MAIN_TYPE - #undef MAIN_TYPE_SIZE - #undef UNION_TYPE - #undef IF_128 - #undef IF_MAIN_TYPE - #undef SHIFT_CONST1 - #undef SHIFT_CONST2 - #undef SHIFT_CONST3 - #undef _128_TYPE - #undef _256_TYPE - #undef AVX_LENGTH - #undef MAVX_COUNT - #undef HAP_TYPE - #undef MASK_TYPE - #undef MASK_ALL_ONES +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES - #undef VEC_EXTRACT_UNIT(__v1, __im) - #undef VEC_INSERT_UNIT(__v1,__ins,__im) - #undef SET_VEC_ZERO(__vec) - #undef VEC_OR(__v1, __v2) - #undef VEC_ADD(__v1, __v2) - #undef VEC_SUB(__v1, __v2) - #undef VEC_MUL(__v1, __v2) - #undef VEC_DIV(__v1, __v2) - #undef VEC_BLEND(__v1, __v2, __mask) - #undef VEC_BLENDV(__v1, __v2, __maskV) - #undef VEC_CAST_256_128(__v1) - #undef VEC_EXTRACT_128(__v1, __im) - #undef VEC_EXTRACT_UNIT(__v1, __im) - #undef VEC_SET1_VAL128(__val) - #undef VEC_MOVE(__v1, __val) - #undef VEC_CAST_128_256(__v1) - #undef VEC_INSERT_VAL(__v1, __val, __pos) - #undef VEC_CVT_128_256(__v1) - #undef VEC_SET1_VAL(__val) - #undef VEC_POPCVT_CHAR(__ch) - #undef VEC_LDPOPCVT_CHAR(__addr) - #undef VEC_CMP_EQ(__v1, __v2) - #undef VEC_SET_LSE(__val) - #undef SHIFT_HAP(__v1, __val) - #undef print256b(__v1) - #undef MASK_VEC - #undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) - #undef VEC_SHIFT_LEFT_1BIT(__vs) - #undef MASK_ALL_ONES - #undef COMPARE_VECS(__v1, __v2) - #undef _256_INT_TYPE - #undef BITMASK_VEC +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_INSERT_UNIT(__v1,__ins,__im) +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC #endif #define SSE @@ -62,90 +60,87 @@ #define SHIFT_CONST2 8 #define SHIFT_CONST3 0 #define _128_TYPE __m128d -#define _256_TYPE __m128d +#define SIMD_TYPE __m128d #define _256_INT_TYPE __m128i #define AVX_LENGTH 2 -#define MAVX_COUNT (MROWS+3)/AVX_LENGTH #define HAP_TYPE __m128i #define MASK_TYPE uint64_t #define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFFL #define MASK_VEC MaskVec_D #define VEC_EXTRACT_UNIT(__v1, __im) \ - _mm_extract_epi64(__v1, __im) + _mm_extract_epi64(__v1, __im) -#define VEC_INSERT_UNIT(__v1,__ins,__im) \ - _mm_insert_epi64(__v1,__ins,__im) +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi64(__v1,__ins,__im) #define VEC_OR(__v1, __v2) \ - _mm_or_pd(__v1, __v2) + _mm_or_pd(__v1, __v2) #define VEC_ADD(__v1, __v2) \ - _mm_add_pd(__v1, __v2) + _mm_add_pd(__v1, __v2) #define VEC_SUB(__v1, __v2) \ - _mm_sub_pd(__v1, __v2) + _mm_sub_pd(__v1, __v2) #define VEC_MUL(__v1, __v2) \ - _mm_mul_pd(__v1, __v2) + _mm_mul_pd(__v1, __v2) #define VEC_DIV(__v1, __v2) \ - _mm_div_pd(__v1, __v2) + _mm_div_pd(__v1, __v2) #define VEC_CMP_EQ(__v1, __v2) \ - _mm_cmpeq_pd(__v1, __v2) + _mm_cmpeq_pd(__v1, __v2) #define VEC_BLEND(__v1, __v2, __mask) \ - _mm_blend_pd(__v1, __v2, __mask) + _mm_blend_pd(__v1, __v2, __mask) #define VEC_BLENDV(__v1, __v2, __maskV) \ - _mm_blendv_pd(__v1, __v2, __maskV) + _mm_blendv_pd(__v1, __v2, __maskV) #define SHIFT_HAP(__v1, __val) \ - __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) #define VEC_CVT_128_256(__v1) \ - _mm_cvtepi32_pd(__v1) + _mm_cvtepi32_pd(__v1) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_pd(__val) -#define VEC_SET1_VAL(__val) \ - _mm_set1_pd(__val) - #define VEC_POPCVT_CHAR(__ch) \ - _mm_cvtepi32_pd(_mm_set1_epi32(__ch)) + _mm_cvtepi32_pd(_mm_set1_epi32(__ch)) #define VEC_SET_LSE(__val) \ - _mm_set_pd(zero, __val); + _mm_set_pd(zero, __val); #define VEC_LDPOPCVT_CHAR(__addr) \ - _mm_cvtepi32_pd(_mm_loadu_si128((__m128i const *)__addr)) + _mm_cvtepi32_pd(_mm_loadu_si128((__m128i const *)__addr)) #define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ - __vdst = _mm_castsi128_pd(_mm_set_epi64(__vsHigh, __vsLow)) + __vdst = _mm_castsi128_pd(_mm_set_epi64(__vsHigh, __vsLow)) #define VEC_SHIFT_LEFT_1BIT(__vs) \ - __vs = _mm_slli_epi64(__vs, 1) + __vs = _mm_slli_epi64(__vs, 1) class BitMaskVec_sse_double { - MASK_VEC combined_ ; + MASK_VEC combined_ ; + public: + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } -public: - - inline MASK_TYPE& getLowEntry(int index) { - return combined_.masks[index] ; - } - inline MASK_TYPE& getHighEntry(int index) { - return combined_.masks[AVX_LENGTH/2+index] ; - } - - inline const _256_TYPE& getCombinedMask() { - return combined_.vecf ; - } - - inline void shift_left_1bit() { - VEC_SHIFT_LEFT_1BIT(combined_.vec) ; - } + inline const SIMD_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } } ; diff --git a/PairHMM_JNI/define-sse-float.h b/PairHMM_JNI/define-sse-float.h index 6612b28e6..7516e6dbf 100644 --- a/PairHMM_JNI/define-sse-float.h +++ b/PairHMM_JNI/define-sse-float.h @@ -1,53 +1,51 @@ #ifdef PRECISION - #undef PRECISION - #undef MAIN_TYPE - #undef MAIN_TYPE_SIZE - #undef UNION_TYPE - #undef IF_128 - #undef IF_MAIN_TYPE - #undef SHIFT_CONST1 - #undef SHIFT_CONST2 - #undef SHIFT_CONST3 - #undef _128_TYPE - #undef _256_TYPE - #undef AVX_LENGTH - #undef MAVX_COUNT - #undef HAP_TYPE - #undef MASK_TYPE - #undef MASK_ALL_ONES +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES - #undef VEC_EXTRACT_UNIT(__v1, __im) - #undef VEC_INSERT_UNIT(__v1,__ins,__im) - #undef SET_VEC_ZERO(__vec) - #undef VEC_OR(__v1, __v2) - #undef VEC_ADD(__v1, __v2) - #undef VEC_SUB(__v1, __v2) - #undef VEC_MUL(__v1, __v2) - #undef VEC_DIV(__v1, __v2) - #undef VEC_BLEND(__v1, __v2, __mask) - #undef VEC_BLENDV(__v1, __v2, __maskV) - #undef VEC_CAST_256_128(__v1) - #undef VEC_EXTRACT_128(__v1, __im) - #undef VEC_EXTRACT_UNIT(__v1, __im) - #undef VEC_SET1_VAL128(__val) - #undef VEC_MOVE(__v1, __val) - #undef VEC_CAST_128_256(__v1) - #undef VEC_INSERT_VAL(__v1, __val, __pos) - #undef VEC_CVT_128_256(__v1) - #undef VEC_SET1_VAL(__val) - #undef VEC_POPCVT_CHAR(__ch) - #undef VEC_LDPOPCVT_CHAR(__addr) - #undef VEC_CMP_EQ(__v1, __v2) - #undef VEC_SET_LSE(__val) - #undef SHIFT_HAP(__v1, __val) - #undef print256b(__v1) - #undef MASK_VEC - #undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) - #undef VEC_SHIFT_LEFT_1BIT(__vs) - #undef MASK_ALL_ONES - #undef COMPARE_VECS(__v1, __v2) - #undef _256_INT_TYPE - #undef BITMASK_VEC +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_INSERT_UNIT(__v1,__ins,__im) +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC #endif #define SSE @@ -62,89 +60,88 @@ #define SHIFT_CONST2 4 #define SHIFT_CONST3 0 #define _128_TYPE __m128 -#define _256_TYPE __m128 +#define SIMD_TYPE __m128 #define _256_INT_TYPE __m128i #define AVX_LENGTH 4 -#define MAVX_COUNT (MROWS+3)/AVX_LENGTH +//#define MAVX_COUNT (MROWS+3)/AVX_LENGTH #define HAP_TYPE UNION_TYPE #define MASK_TYPE uint32_t #define MASK_ALL_ONES 0xFFFFFFFF #define MASK_VEC MaskVec_F #define VEC_EXTRACT_UNIT(__v1, __im) \ - _mm_extract_epi32(__v1, __im) + _mm_extract_epi32(__v1, __im) -#define VEC_INSERT_UNIT(__v1,__ins,__im) \ - _mm_insert_epi32(__v1,__ins,__im) +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi32(__v1,__ins,__im) #define VEC_OR(__v1, __v2) \ - _mm_or_ps(__v1, __v2) + _mm_or_ps(__v1, __v2) #define VEC_ADD(__v1, __v2) \ - _mm_add_ps(__v1, __v2) + _mm_add_ps(__v1, __v2) #define VEC_SUB(__v1, __v2) \ - _mm_sub_ps(__v1, __v2) + _mm_sub_ps(__v1, __v2) #define VEC_MUL(__v1, __v2) \ - _mm_mul_ps(__v1, __v2) + _mm_mul_ps(__v1, __v2) #define VEC_DIV(__v1, __v2) \ - _mm_div_ps(__v1, __v2) + _mm_div_ps(__v1, __v2) #define VEC_CMP_EQ(__v1, __v2) \ - _mm_cmpeq_ps(__v1, __v2) + _mm_cmpeq_ps(__v1, __v2) #define VEC_BLEND(__v1, __v2, __mask) \ - _mm_blend_ps(__v1, __v2, __mask) + _mm_blend_ps(__v1, __v2, __mask) #define VEC_BLENDV(__v1, __v2, __maskV) \ - _mm_blendv_ps(__v1, __v2, __maskV) + _mm_blendv_ps(__v1, __v2, __maskV) #define SHIFT_HAP(__v1, __val) \ - _vector_shift_lastsses(__v1, __val.f) + _vector_shift_lastsses(__v1, __val.f) #define VEC_CVT_128_256(__v1) \ - _mm_cvtepi32_ps(__v1.i) + _mm_cvtepi32_ps(__v1.i) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_ps(__val) -#define VEC_SET1_VAL(__val) \ - _mm_set1_ps(__val) - #define VEC_POPCVT_CHAR(__ch) \ - _mm_cvtepi32_ps(_mm_set1_epi32(__ch)) + _mm_cvtepi32_ps(_mm_set1_epi32(__ch)) #define VEC_SET_LSE(__val) \ - _mm_set_ps(zero, zero, zero, __val); + _mm_set_ps(zero, zero, zero, __val); #define VEC_LDPOPCVT_CHAR(__addr) \ - _mm_cvtepi32_ps(_mm_loadu_si128((__m128i const *)__addr)) + _mm_cvtepi32_ps(_mm_loadu_si128((__m128i const *)__addr)) #define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ - __vdst = _mm_cvtpi32x2_ps(__vsLow, __vsHigh) + __vdst = _mm_cvtpi32x2_ps(__vsLow, __vsHigh) #define VEC_SHIFT_LEFT_1BIT(__vs) \ - __vs = _mm_slli_epi32(__vs, 1) + __vs = _mm_slli_epi32(__vs, 1) class BitMaskVec_sse_float { - MASK_VEC combined_ ; + MASK_VEC combined_ ; -public: - - inline MASK_TYPE& getLowEntry(int index) { - return combined_.masks[index] ; - } - inline MASK_TYPE& getHighEntry(int index) { - return combined_.masks[AVX_LENGTH/2+index] ; - } - - inline const _256_TYPE& getCombinedMask() { - return combined_.vecf ; - } - - inline void shift_left_1bit() { - VEC_SHIFT_LEFT_1BIT(combined_.vec) ; - } + public: + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } } ; diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc index d575b8271..4f754d019 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc @@ -79,7 +79,7 @@ Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadL assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); assert(overallGCPArray && "OverallGCP array not initialized in JNI"); - assert(readLength < MROWS); + //assert(readLength < MROWS); #endif testcase tc; tc.rslen = readLength; @@ -167,7 +167,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBasesGlobalRef, &is_copy); #ifdef ENABLE_ASSERTIONS assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); - assert(env->GetArrayLength(haplotypeBasesGlobalRef) < MCOLS); + //assert(env->GetArrayLength(haplotypeBasesGlobalRef) < MCOLS); #endif #ifdef DEBUG0_1 cout << "JNI haplotype length "<GetArrayLength(haplotypeBasesGlobalRef)<<"\n"; @@ -240,7 +240,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); assert(overallGCPArray && "overallGCP array not initialized in JNI"); - assert(readLength < MROWS); + //assert(readLength < MROWS); assert(readLength == env->GetArrayLength(readQuals)); assert(readLength == env->GetArrayLength(insertionGOP)); assert(readLength == env->GetArrayLength(deletionGOP)); diff --git a/PairHMM_JNI/pairhmm-template-kernel.cc b/PairHMM_JNI/pairhmm-template-kernel.cc index 66dc557aa..9f59d7eeb 100644 --- a/PairHMM_JNI/pairhmm-template-kernel.cc +++ b/PairHMM_JNI/pairhmm-template-kernel.cc @@ -4,435 +4,338 @@ #include #include -//#define DEBUG -#define MUSTAFA -#define KARTHIK -/* -template -string getBinaryStr (T val, int numBitsToWrite) { - - ostringstream oss ; - uint64_t mask = ((T) 0x1) << (numBitsToWrite-1) ; - for (int i=numBitsToWrite-1; i >= 0; --i) { - oss << ((val & mask) >> i) ; - mask >>= 1 ; - } - return oss.str() ; -} -*/ -#ifdef MUSTAFA +void CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]) { + const int maskBitCnt = MAIN_TYPE_SIZE ; -void GEN_INTRINSIC(GEN_INTRINSIC(precompute_masks_,SIMD_TYPE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]) { - - const int maskBitCnt = MAIN_TYPE_SIZE ; - - for (int vi=0; vi < numMaskVecs; ++vi) { - for (int rs=0; rs < NUM_DISTINCT_CHARS; ++rs) { - maskArr[vi][rs] = 0 ; + for (int vi=0; vi < numMaskVecs; ++vi) { + for (int rs=0; rs < NUM_DISTINCT_CHARS; ++rs) { + maskArr[vi][rs] = 0 ; + } + maskArr[vi][AMBIG_CHAR] = MASK_ALL_ONES ; } - maskArr[vi][AMBIG_CHAR] = MASK_ALL_ONES ; - } - - for (int col=1; col < COLS; ++col) { - int mIndex = (col-1) / maskBitCnt ; - int mOffset = (col-1) % maskBitCnt ; - MASK_TYPE bitMask = ((MASK_TYPE)0x1) << (maskBitCnt-1-mOffset) ; - char hapChar = ConvertChar::get(tc.hap[col-1]); + for (int col=1; col < COLS; ++col) { + int mIndex = (col-1) / maskBitCnt ; + int mOffset = (col-1) % maskBitCnt ; + MASK_TYPE bitMask = ((MASK_TYPE)0x1) << (maskBitCnt-1-mOffset) ; - if (hapChar == AMBIG_CHAR) { - for (int ci=0; ci < NUM_DISTINCT_CHARS; ++ci) - maskArr[mIndex][ci] |= bitMask ; - } + char hapChar = ConvertChar::get(tc.hap[col-1]); - maskArr[mIndex][hapChar] |= bitMask ; - // bit corresponding to col 1 will be the MSB of the mask 0 - // bit corresponding to col 2 will be the MSB-1 of the mask 0 - // ... - // bit corresponding to col 32 will be the LSB of the mask 0 - // bit corresponding to col 33 will be the MSB of the mask 1 - // ... - } + if (hapChar == AMBIG_CHAR) { + for (int ci=0; ci < NUM_DISTINCT_CHARS; ++ci) + maskArr[mIndex][ci] |= bitMask ; + } + + maskArr[mIndex][hapChar] |= bitMask ; + // bit corresponding to col 1 will be the MSB of the mask 0 + // bit corresponding to col 2 will be the MSB-1 of the mask 0 + // ... + // bit corresponding to col 32 will be the LSB of the mask 0 + // bit corresponding to col 33 will be the MSB of the mask 1 + // ... + } } -void GEN_INTRINSIC(GEN_INTRINSIC(init_masks_for_row_,SIMD_TYPE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess) { +void CONCAT(CONCAT(init_masks_for_row_,SIMD_ENGINE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess) { - for (int ri=0; ri < numRowsToProcess; ++ri) { - rsArr[ri] = ConvertChar::get(tc.rs[ri+beginRowIndex-1]) ; - } + for (int ri=0; ri < numRowsToProcess; ++ri) { + rsArr[ri] = ConvertChar::get(tc.rs[ri+beginRowIndex-1]) ; + } - for (int ei=0; ei < AVX_LENGTH; ++ei) { - lastMaskShiftOut[ei] = 0 ; - } + for (int ei=0; ei < AVX_LENGTH; ++ei) { + lastMaskShiftOut[ei] = 0 ; + } } - #define SET_MASK_WORD(__dstMask, __srcMask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ - MASK_TYPE __bitMask = (((MASK_TYPE)0x1) << __shiftBy) - 1 ; \ - MASK_TYPE __nextShiftOut = (__srcMask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ - __dstMask = (__srcMask >> __shiftBy) | __lastShiftOut ; \ - __lastShiftOut = __nextShiftOut ; \ + MASK_TYPE __bitMask = (((MASK_TYPE)0x1) << __shiftBy) - 1 ; \ + MASK_TYPE __nextShiftOut = (__srcMask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ + __dstMask = (__srcMask >> __shiftBy) | __lastShiftOut ; \ + __lastShiftOut = __nextShiftOut ; \ } -void GEN_INTRINSIC(GEN_INTRINSIC(update_masks_for_cols_, SIMD_TYPE), PRECISION)(int maskIndex, BITMASK_VEC& bitMaskVec, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, int maskBitCnt) { +void CONCAT(CONCAT(update_masks_for_cols_,SIMD_ENGINE), PRECISION)(int maskIndex, BITMASK_VEC& bitMaskVec, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, int maskBitCnt) { - for (int ei=0; ei < AVX_LENGTH/2; ++ei) { - SET_MASK_WORD(bitMaskVec.getLowEntry(ei), maskArr[maskIndex][rsArr[ei]], - lastMaskShiftOut[ei], ei, maskBitCnt) ; - - int ei2 = ei + AVX_LENGTH/2 ; // the second entry index - SET_MASK_WORD(bitMaskVec.getHighEntry(ei), maskArr[maskIndex][rsArr[ei2]], - lastMaskShiftOut[ei2], ei2, maskBitCnt) ; - } + for (int ei=0; ei < AVX_LENGTH/2; ++ei) { + SET_MASK_WORD(bitMaskVec.getLowEntry(ei), maskArr[maskIndex][rsArr[ei]], + lastMaskShiftOut[ei], ei, maskBitCnt) ; + + int ei2 = ei + AVX_LENGTH/2 ; // the second entry index + SET_MASK_WORD(bitMaskVec.getHighEntry(ei), maskArr[maskIndex][rsArr[ei2]], + lastMaskShiftOut[ei2], ei2, maskBitCnt) ; + } } -//void GEN_INTRINSIC(computeDistVec, PRECISION) (BITMASK_VEC& bitMaskVec, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen, const _256_TYPE& distmSel, int firstRowIndex, int lastRowIndex) { +inline void CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (BITMASK_VEC& bitMaskVec, SIMD_TYPE& distm, SIMD_TYPE& _1_distm, SIMD_TYPE& distmChosen) { -inline void GEN_INTRINSIC(GEN_INTRINSIC(computeDistVec, SIMD_TYPE), PRECISION) (BITMASK_VEC& bitMaskVec, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen) { - //#define computeDistVec() { + distmChosen = VEC_BLENDV(distm, _1_distm, bitMaskVec.getCombinedMask()) ; -#ifdef DEBUGG - long long *temp1 = (long long *)(&maskV); - double *temp2 = (double *)(&distm); - double *temp3 = (double *)(&_1_distm); - printf("***\n%lx\n%lx\n%f\n%f\n%f\n%f\n***\n", temp1[0], temp1[1], temp2[0], temp2[1], temp3[0], temp3[1]); -#endif + bitMaskVec.shift_left_1bit() ; +} - distmChosen = VEC_BLENDV(distm, _1_distm, bitMaskVec.getCombinedMask()) ; - - /*COMPARE_VECS(distmChosen, distmSel, firstRowIndex, lastRowIndex) ;*/ - - bitMaskVec.shift_left_1bit() ; -} - - -/* -template -struct HmmData { - int ROWS ; - int COLS ; - - NUMBER shiftOutM[MROWS+MCOLS+AVX_LENGTH], shiftOutX[MROWS+MCOLS+AVX_LENGTH], shiftOutY[MROWS+MCOLS+AVX_LENGTH] ; - Context ctx ; - testcase* tc ; - _256_TYPE p_MM[MAVX_COUNT], p_GAPM[MAVX_COUNT], p_MX[MAVX_COUNT], p_XX[MAVX_COUNT], p_MY[MAVX_COUNT], p_YY[MAVX_COUNT], distm1D[MAVX_COUNT] ; - _256_TYPE pGAPM, pMM, pMX, pXX, pMY, pYY ; - - UNION_TYPE M_t, M_t_1, M_t_2, X_t, X_t_1, X_t_2, Y_t, Y_t_1, Y_t_2, M_t_y, M_t_1_y ; - UNION_TYPE rs , rsN ; - _256_TYPE distmSel; - _256_TYPE distm, _1_distm; - -} ; -*/ -#endif // MUSTAFA - -template void GEN_INTRINSIC(GEN_INTRINSIC(initializeVectors, SIMD_TYPE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D) +template void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, SIMD_TYPE *p_MM, SIMD_TYPE *p_GAPM, SIMD_TYPE *p_MX, SIMD_TYPE *p_XX, SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, SIMD_TYPE *distm1D) { - NUMBER zero = ctx._(0.0); - NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); - for (int s=0;shaplen); + for (int s=0;si[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + + *(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; + *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; + *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; + *(ptr_p_MY+r-1) = ctx.ph2pr[_d]; + *(ptr_p_YY+r-1) = ctx.ph2pr[_c]; + } + + NUMBER *ptr_distm1D = (NUMBER *)distm1D; + for (int r = 1; r < ROWS; r++) + { + int _q = tc->q[r-1] & 127; + ptr_distm1D[r-1] = ctx.ph2pr[_q]; + } +} + + +template inline void CONCAT(CONCAT(stripeINITIALIZATION,SIMD_ENGINE), PRECISION)( + int stripeIdx, Context ctx, testcase *tc, SIMD_TYPE &pGAPM, SIMD_TYPE &pMM, SIMD_TYPE &pMX, SIMD_TYPE &pXX, SIMD_TYPE &pMY, SIMD_TYPE &pYY, + SIMD_TYPE &rs, UNION_TYPE &rsN, SIMD_TYPE &distm, SIMD_TYPE &_1_distm, SIMD_TYPE *distm1D, SIMD_TYPE N_packed256, SIMD_TYPE *p_MM , SIMD_TYPE *p_GAPM , + SIMD_TYPE *p_MX, SIMD_TYPE *p_XX , SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM) +{ + int i = stripeIdx; + pGAPM = p_GAPM[i]; + pMM = p_MM[i]; + pMX = p_MX[i]; + pXX = p_XX[i]; + pMY = p_MY[i]; + pYY = p_YY[i]; + + NUMBER zero = ctx._(0.0); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + UNION_TYPE packed3; packed3.d = VEC_SET1_VAL(3.0); + + distm = distm1D[i]; + _1_distm = VEC_SUB(packed1.d, distm); + + distm = VEC_DIV(distm, packed3.d); + + /* initialize M_t_2, M_t_1, X_t_2, X_t_1, Y_t_2, Y_t_1 */ + M_t_2.d = VEC_SET1_VAL(zero); + X_t_2.d = VEC_SET1_VAL(zero); + + if (i==0) { + M_t_1.d = VEC_SET1_VAL(zero); + X_t_1.d = VEC_SET1_VAL(zero); + Y_t_2.d = VEC_SET_LSE(init_Y); + Y_t_1.d = VEC_SET1_VAL(zero); + } + else { + X_t_1.d = VEC_SET_LSE(shiftOutX[AVX_LENGTH]); + M_t_1.d = VEC_SET_LSE(shiftOutM[AVX_LENGTH]); + Y_t_2.d = VEC_SET1_VAL(zero); + Y_t_1.d = VEC_SET1_VAL(zero); + } + M_t_1_y = M_t_1; +} + +inline SIMD_TYPE CONCAT(CONCAT(computeDISTM,SIMD_ENGINE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, SIMD_TYPE rs, UNION_TYPE rsN, SIMD_TYPE N_packed256, + SIMD_TYPE distm, SIMD_TYPE _1_distm) +{ + UNION_TYPE hapN, rshap; + SIMD_TYPE cond; + IF_32 shiftInHap; + + int *hap_ptr = tc->ihap; + + shiftInHap.i = (d NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL) +{ + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + int MAVX_COUNT = (ROWS+AVX_LENGTH-1)/AVX_LENGTH; + + SIMD_TYPE p_MM [MAVX_COUNT], p_GAPM [MAVX_COUNT], p_MX [MAVX_COUNT]; + SIMD_TYPE p_XX [MAVX_COUNT], p_MY [MAVX_COUNT], p_YY [MAVX_COUNT]; + SIMD_TYPE distm1D[MAVX_COUNT]; + NUMBER shiftOutM[ROWS+COLS+AVX_LENGTH], shiftOutX[ROWS+COLS+AVX_LENGTH], shiftOutY[ROWS+COLS+AVX_LENGTH]; + UNION_TYPE M_t, M_t_1, M_t_2, X_t, X_t_1, X_t_2, Y_t, Y_t_1, Y_t_2, M_t_y, M_t_1_y; + SIMD_TYPE pGAPM, pMM, pMX, pXX, pMY, pYY; + + struct timeval start, end; + NUMBER result_avx2; + Context ctx; + UNION_TYPE rs , rsN; + HAP_TYPE hap; + SIMD_TYPE distmSel, distmChosen ; + SIMD_TYPE distm, _1_distm; + + int r, c; + NUMBER zero = ctx._(0.0); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + SIMD_TYPE N_packed256 = VEC_POPCVT_CHAR('N'); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + int remainingRows = (ROWS-1) % AVX_LENGTH; + int stripe_cnt = ((ROWS-1) / AVX_LENGTH) + (remainingRows!=0); + + const int maskBitCnt = MAIN_TYPE_SIZE ; + const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function + + MASK_TYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; + CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(*tc, COLS, numMaskVecs, maskArr) ; + + char rsArr[AVX_LENGTH] ; + MASK_TYPE lastMaskShiftOut[AVX_LENGTH] ; + CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(ROWS, COLS, shiftOutM, shiftOutX, shiftOutY, + ctx, tc, p_MM, p_GAPM, p_MX, p_XX, p_MY, p_YY, distm1D); + + for (int i=0;ii[r-1] & 127; - int _d = tc->d[r-1] & 127; - int _c = tc->c[r-1] & 127; + int numMaskBitsToProcess = std::min(MAIN_TYPE_SIZE, COLS+remainingRows-1-begin_d) ; + CONCAT(CONCAT(update_masks_for_cols_,SIMD_ENGINE),PRECISION)((begin_d-1)/MAIN_TYPE_SIZE, bitMaskVec, maskArr, rsArr, lastMaskShiftOut, maskBitCnt) ; - *(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; - *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; - *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; - *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; - #ifdef KARTHIK - *(ptr_p_MY+r-1) = ctx.ph2pr[_d]; - *(ptr_p_YY+r-1) = ctx.ph2pr[_c]; - #else - *(ptr_p_MY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; - *(ptr_p_YY+r-1) = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; - #endif - + for (int mbi=0; mbi < numMaskBitsToProcess; ++mbi) { + + CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (bitMaskVec, distm, _1_distm, distmChosen) ; + int ShiftIdx = begin_d + mbi +AVX_LENGTH; + + CONCAT(CONCAT(computeMXY,SIMD_ENGINE), PRECISION)(M_t, X_t, Y_t, M_t_y, M_t_2, X_t_2, Y_t_2, M_t_1, X_t_1, M_t_1_y, Y_t_1, + pMM, pGAPM, pMX, pXX, pMY, pYY, distmChosen); + + sumM = VEC_ADD(sumM, M_t.d); + CONCAT(CONCAT(_vector_shift_last,SIMD_ENGINE), PRECISION)(M_t, shiftOutM[ShiftIdx]); + + sumX = VEC_ADD(sumX, X_t.d); + CONCAT(CONCAT(_vector_shift_last,SIMD_ENGINE), PRECISION)(X_t, shiftOutX[ShiftIdx]); + + CONCAT(CONCAT(_vector_shift_last,SIMD_ENGINE), PRECISION)(Y_t_1, shiftOutY[ShiftIdx]); + + M_t_2 = M_t_1; M_t_1 = M_t; X_t_2 = X_t_1; X_t_1 = X_t; + Y_t_2 = Y_t_1; Y_t_1 = Y_t; M_t_1_y = M_t_y; + + } } - - NUMBER *ptr_distm1D = (NUMBER *)distm1D; - for (int r = 1; r < ROWS; r++) - { - int _q = tc->q[r-1] & 127; - ptr_distm1D[r-1] = ctx.ph2pr[_q]; - } -} - - -template inline void GEN_INTRINSIC(GEN_INTRINSIC(stripINITIALIZATION, SIMD_TYPE), PRECISION)( - int stripIdx, Context ctx, testcase *tc, _256_TYPE &pGAPM, _256_TYPE &pMM, _256_TYPE &pMX, _256_TYPE &pXX, _256_TYPE &pMY, _256_TYPE &pYY, - _256_TYPE &rs, UNION_TYPE &rsN, _256_TYPE &distm, _256_TYPE &_1_distm, _256_TYPE *distm1D, _256_TYPE N_packed256, _256_TYPE *p_MM , _256_TYPE *p_GAPM , - _256_TYPE *p_MX, _256_TYPE *p_XX , _256_TYPE *p_MY, _256_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, - UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM) -{ - int i = stripIdx; - pGAPM = p_GAPM[i]; - pMM = p_MM[i]; - pMX = p_MX[i]; - pXX = p_XX[i]; - pMY = p_MY[i]; - pYY = p_YY[i]; - - NUMBER zero = ctx._(0.0); - NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); - UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); - UNION_TYPE packed3; packed3.d = VEC_SET1_VAL(3.0); - /* compare rs and N */ -#ifndef MUSTAFA - rs = VEC_LDPOPCVT_CHAR((tc->irs+i*AVX_LENGTH)); - rsN.d = VEC_CMP_EQ(N_packed256, rs); -#endif - distm = distm1D[i]; - _1_distm = VEC_SUB(packed1.d, distm); - - #ifdef KARTHIK - distm = VEC_DIV(distm, packed3.d); - #endif - /* initialize M_t_2, M_t_1, X_t_2, X_t_1, Y_t_2, Y_t_1 */ - M_t_2.d = VEC_SET1_VAL(zero); - X_t_2.d = VEC_SET1_VAL(zero); - - if (i==0) { - M_t_1.d = VEC_SET1_VAL(zero); - X_t_1.d = VEC_SET1_VAL(zero); - Y_t_2.d = VEC_SET_LSE(init_Y); - Y_t_1.d = VEC_SET1_VAL(zero); - } - else { - X_t_1.d = VEC_SET_LSE(shiftOutX[AVX_LENGTH]); - M_t_1.d = VEC_SET_LSE(shiftOutM[AVX_LENGTH]); - Y_t_2.d = VEC_SET1_VAL(zero); - Y_t_1.d = VEC_SET1_VAL(zero); - } - M_t_1_y = M_t_1; -} - - - -inline _256_TYPE GEN_INTRINSIC(GEN_INTRINSIC(computeDISTM, SIMD_TYPE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, - _256_TYPE distm, _256_TYPE _1_distm) -{ - UNION_TYPE hapN, rshap; - _256_TYPE cond; - IF_32 shiftInHap; - - int *hap_ptr = tc->ihap; - - shiftInHap.i = (d NUMBER GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_,SIMD_TYPE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL) -{ - _256_TYPE p_MM [MAVX_COUNT], p_GAPM [MAVX_COUNT], p_MX [MAVX_COUNT]; - _256_TYPE p_XX [MAVX_COUNT], p_MY [MAVX_COUNT], p_YY [MAVX_COUNT]; - _256_TYPE distm1D[MAVX_COUNT]; - NUMBER shiftOutM[MROWS+MCOLS+AVX_LENGTH], shiftOutX[MROWS+MCOLS+AVX_LENGTH], shiftOutY[MROWS+MCOLS+AVX_LENGTH]; - UNION_TYPE M_t, M_t_1, M_t_2, X_t, X_t_1, X_t_2, Y_t, Y_t_1, Y_t_2, M_t_y, M_t_1_y; - _256_TYPE pGAPM, pMM, pMX, pXX, pMY, pYY; - - struct timeval start, end; - NUMBER result_avx2; - Context ctx; - UNION_TYPE rs , rsN; - HAP_TYPE hap; - _256_TYPE distmSel, distmChosen ; - _256_TYPE distm, _1_distm; - - int r, c; - int ROWS = tc->rslen + 1; - int COLS = tc->haplen + 1; - int AVX_COUNT = (ROWS+7)/8; - NUMBER zero = ctx._(0.0); - UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); - _256_TYPE N_packed256 = VEC_POPCVT_CHAR('N'); - NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); - int remainingRows = (ROWS-1) % AVX_LENGTH; - int strip_cnt = ((ROWS-1) / AVX_LENGTH) + (remainingRows!=0); - -#ifdef MUSTAFA - const int maskBitCnt = MAIN_TYPE_SIZE ; - const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function - - MASK_TYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; - GEN_INTRINSIC(GEN_INTRINSIC(precompute_masks_,SIMD_TYPE), PRECISION)(*tc, COLS, numMaskVecs, maskArr) ; - - char rsArr[AVX_LENGTH] ; - MASK_TYPE lastMaskShiftOut[AVX_LENGTH] ; -#endif - GEN_INTRINSIC(GEN_INTRINSIC(initializeVectors,SIMD_TYPE), PRECISION)(ROWS, COLS, shiftOutM, shiftOutX, shiftOutY, - ctx, tc, p_MM, p_GAPM, p_MX, p_XX, p_MY, p_YY, distm1D); - - //for (int __ii=0; __ii < 10; ++__ii) - for (int i=0;i=0;i--) - printf("%f ", *(p+i)); - printf("\n"); -} - -void print128b_D(__m128d x) -{ - double *p = (double *)(&x); - for (int i=1;i>=0;i--) - printf("%f ", *(p+i)); - printf("\n"); -} int main() { -/* - IF_128f x; - x.f = _mm_set_ps(1.0, 2.0, 3.0, 4.0); - IF_32 shiftIn, shiftOut; - shiftIn.f = 5.0f; - print128b_F(x.f); - GEN_INTRINSIC(_vector_shift, s)(x, shiftIn, shiftOut); - print128b_F(x.f); + testcase* tc = new testcase[BATCH_SIZE]; + float result[BATCH_SIZE], result_avxf; + double result_avxd; + double lastClk = 0.0 ; + double aggregateTimeRead = 0.0; + double aggregateTimeCompute = 0.0; + double aggregateTimeWrite = 0.0; - IF_128d y; - y.f = _mm_set_pd(10.0, 11.0); - IF_64 shiftInd, shiftOutd; - shiftInd.f = 12.0; - print128b_D(y.f); - GEN_INTRINSIC(_vector_shift, d)(y, shiftInd, shiftOutd); - print128b_D(y.f); + // Need to call it once to initialize the static array + ConvertChar::init() ; - exit(0); -*/ + // char* ompEnvVar = getenv("OMP_NUM_THREADS") ; + // if (ompEnvVar != NULL && ompEnvVar != "" && ompEnvVar != "1" ) { + // thread_level_parallelism_enabled = true ; + // } - testcase* tc = new testcase[BATCH_SIZE]; - float result[BATCH_SIZE], result_avxf; - double result_avxd; - //struct timeval start, end; - double lastClk = 0.0 ; - double aggregateTimeRead = 0.0; - double aggregateTimeCompute = 0.0; - double aggregateTimeWrite = 0.0; + bool noMoreData = false; + int count =0; + while (!noMoreData) + { + int read_count = BATCH_SIZE; - // Need to call it once to initialize the static array - ConvertChar::init() ; - + lastClk = getCurrClk() ; + for (int b=0;b(&tc[b]); - //gettimeofday(&start, NULL); - lastClk = getCurrClk() ; - -//#pragma omp parallel for schedule(dynamic) if(thread_level_parallelism_enabled) - for (int b=0;b(&tc[b]); - - #ifdef RUN_HYBRID - #define MIN_ACCEPTED 1e-28f - if (result_avxf < MIN_ACCEPTED) { - //printf("**************** RUNNING DOUBLE ******************\n"); - count++; - result_avxd = GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_, SIMD_TYPE), d)(&tc[b]); - result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); - } - else - result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); - #endif - - #ifndef RUN_HYBRID - result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); - #endif - - } - //gettimeofday(&end, NULL); - aggregateTimeCompute += (getCurrClk() - lastClk) ; - //((end.tv_sec * 1000000 + end.tv_usec) - (start.tv_sec * 1000000 + start.tv_usec)); - - //gettimeofday(&start, NULL); - lastClk = getCurrClk() ; - //for (int b=0;b(&tc[b]); + result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); + } + else + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif +#ifndef RUN_HYBRID + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif } + aggregateTimeCompute += (getCurrClk() - lastClk) ; + lastClk = getCurrClk() ; + for (int b=0;bhaplen = strlen(tc->hap); tc->rslen = strlen(tc->rs); - assert(tc->rslen < MROWS); + //assert(tc->rslen < MROWS); tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); tc->irs = (int *) malloc(tc->rslen*sizeof(int)); @@ -206,7 +206,7 @@ int read_mod_testcase(ifstream& fptr, testcase* tc, bool reformat) //cout << "Lengths "<haplen <<" "<rslen<<"\n"; memcpy(tc->rs, tokens[1].c_str(),tokens[1].size()); assert(tokens.size() == 2 + 4*(tc->rslen)); - assert(tc->rslen < MROWS); + //assert(tc->rslen < MROWS); for(unsigned j=0;jrslen;++j) tc->q[j] = (char)convToInt(tokens[2+0*tc->rslen+j]); for(unsigned j=0;jrslen;++j) diff --git a/PairHMM_JNI/vector_defs.h b/PairHMM_JNI/vector_defs.h index 7958480f2..80b48ae99 100644 --- a/PairHMM_JNI/vector_defs.h +++ b/PairHMM_JNI/vector_defs.h @@ -1,9 +1,9 @@ -#undef SIMD_TYPE -#undef SIMD_TYPE_AVX -#undef SIMD_TYPE_SSE +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX +#undef SIMD_ENGINE_SSE -#define SIMD_TYPE avx -#define SIMD_TYPE_AVX +#define SIMD_ENGINE avx +#define SIMD_ENGINE_AVX #include "define-float.h" #include "vector_function_prototypes.h" @@ -11,11 +11,11 @@ #include "define-double.h" #include "vector_function_prototypes.h" -#undef SIMD_TYPE -#undef SIMD_TYPE_AVX +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX -#define SIMD_TYPE sse -#define SIMD_TYPE_SSE +#define SIMD_ENGINE sse +#define SIMD_ENGINE_SSE #include "define-sse-float.h" @@ -24,7 +24,7 @@ #include "define-sse-double.h" #include "vector_function_prototypes.h" -#undef SIMD_TYPE -#undef SIMD_TYPE_AVX -#undef SIMD_TYPE_SSE +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX +#undef SIMD_ENGINE_SSE diff --git a/PairHMM_JNI/vector_function_prototypes.h b/PairHMM_JNI/vector_function_prototypes.h index ce9cc2abc..67a2667e1 100644 --- a/PairHMM_JNI/vector_function_prototypes.h +++ b/PairHMM_JNI/vector_function_prototypes.h @@ -1,19 +1,19 @@ -inline void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift,SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); -inline void GEN_INTRINSIC(GEN_INTRINSIC(_vector_shift_last,SIMD_TYPE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); -inline void GEN_INTRINSIC(GEN_INTRINSIC(precompute_masks_,SIMD_TYPE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); -inline void GEN_INTRINSIC(GEN_INTRINSIC(init_masks_for_row_,SIMD_TYPE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess); -inline void GEN_INTRINSIC(GEN_INTRINSIC(update_masks_for_cols_,SIMD_TYPE), PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt); -inline void GEN_INTRINSIC(GEN_INTRINSIC(computeDistVec,SIMD_TYPE), PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, _256_TYPE& distm, _256_TYPE& _1_distm, _256_TYPE& distmChosen); -template inline void GEN_INTRINSIC(GEN_INTRINSIC(initializeVectors,SIMD_TYPE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, _256_TYPE *p_MM, _256_TYPE *p_GAPM, _256_TYPE *p_MX, _256_TYPE *p_XX, _256_TYPE *p_MY, _256_TYPE *p_YY, _256_TYPE *distm1D); -template inline void GEN_INTRINSIC(GEN_INTRINSIC(stripINITIALIZATION,SIMD_TYPE), PRECISION)( - int stripIdx, Context ctx, testcase *tc, _256_TYPE &pGAPM, _256_TYPE &pMM, _256_TYPE &pMX, _256_TYPE &pXX, _256_TYPE &pMY, _256_TYPE &pYY, - _256_TYPE &rs, UNION_TYPE &rsN, _256_TYPE &distm, _256_TYPE &_1_distm, _256_TYPE *distm1D, _256_TYPE N_packed256, _256_TYPE *p_MM , _256_TYPE *p_GAPM , - _256_TYPE *p_MX, _256_TYPE *p_XX , _256_TYPE *p_MY, _256_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, +inline void CONCAT(CONCAT(_vector_shift,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); +inline void CONCAT(CONCAT(_vector_shift_last,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); +inline void CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); +inline void CONCAT(CONCAT(init_masks_for_row_,SIMD_ENGINE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess); +inline void CONCAT(CONCAT(update_masks_for_cols_,SIMD_ENGINE), PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt); +inline void CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, SIMD_TYPE& distm, SIMD_TYPE& _1_distm, SIMD_TYPE& distmChosen); +template inline void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, SIMD_TYPE *p_MM, SIMD_TYPE *p_GAPM, SIMD_TYPE *p_MX, SIMD_TYPE *p_XX, SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, SIMD_TYPE *distm1D); +template inline void CONCAT(CONCAT(stripINITIALIZATION,SIMD_ENGINE), PRECISION)( + int stripIdx, Context ctx, testcase *tc, SIMD_TYPE &pGAPM, SIMD_TYPE &pMM, SIMD_TYPE &pMX, SIMD_TYPE &pXX, SIMD_TYPE &pMY, SIMD_TYPE &pYY, + SIMD_TYPE &rs, UNION_TYPE &rsN, SIMD_TYPE &distm, SIMD_TYPE &_1_distm, SIMD_TYPE *distm1D, SIMD_TYPE N_packed256, SIMD_TYPE *p_MM , SIMD_TYPE *p_GAPM , + SIMD_TYPE *p_MX, SIMD_TYPE *p_XX , SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM); -inline _256_TYPE GEN_INTRINSIC(GEN_INTRINSIC(computeDISTM,SIMD_TYPE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, _256_TYPE rs, UNION_TYPE rsN, _256_TYPE N_packed256, - _256_TYPE distm, _256_TYPE _1_distm); -inline void GEN_INTRINSIC(GEN_INTRINSIC(computeMXY,SIMD_TYPE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, +inline SIMD_TYPE CONCAT(CONCAT(computeDISTM,SIMD_ENGINE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, SIMD_TYPE rs, UNION_TYPE rsN, SIMD_TYPE N_packed256, + SIMD_TYPE distm, SIMD_TYPE _1_distm); +inline void CONCAT(CONCAT(computeMXY,SIMD_ENGINE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, - _256_TYPE pMM, _256_TYPE pGAPM, _256_TYPE pMX, _256_TYPE pXX, _256_TYPE pMY, _256_TYPE pYY, _256_TYPE distmSel); -template NUMBER GEN_INTRINSIC(GEN_INTRINSIC(compute_full_prob_,SIMD_TYPE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL); + SIMD_TYPE pMM, SIMD_TYPE pGAPM, SIMD_TYPE pMX, SIMD_TYPE pXX, SIMD_TYPE pMY, SIMD_TYPE pYY, SIMD_TYPE distmSel); +template NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL); From 018e9e2c5f3ea7e1bb531c12be2371da02e6a440 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Sun, 26 Jan 2014 19:18:12 -0800 Subject: [PATCH 23/72] 1. Cleaned up code 2. Split into DebugJNILoglessPairHMM and VectorLoglessPairHMM with base class JNILoglessPairHMM. DebugJNILoglessPairHMM can, in principle, invoke any other child class of JNILoglessPairHMM. 3. Added more profiling code for Java parts of LoglessPairHMM --- PairHMM_JNI/Makefile | 8 +- PairHMM_JNI/jni_common.h | 32 ++++ PairHMM_JNI/jnidebug.h | 166 ++++++++++++++++++ ...ng_utils_pairhmm_DebugJNILoglessPairHMM.cc | 151 ++++++++++++++++ ...te_sting_utils_pairhmm_JNILoglessPairHMM.h | 111 ------------ ...ing_utils_pairhmm_VectorLoglessPairHMM.cc} | 139 +++------------ PairHMM_JNI/run.sh | 10 +- .../PairHMMLikelihoodCalculationEngine.java | 4 +- .../utils/pairhmm/DebugJNILoglessPairHMM.java | 2 +- .../utils/pairhmm/JNILoglessPairHMM.java | 1 + .../utils/pairhmm/VectorLoglessPairHMM.java | 14 +- .../sting/utils/pairhmm/PairHMM.java | 39 ++-- 12 files changed, 423 insertions(+), 254 deletions(-) create mode 100644 PairHMM_JNI/jni_common.h create mode 100644 PairHMM_JNI/jnidebug.h create mode 100644 PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc delete mode 100644 PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h rename PairHMM_JNI/{org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc => org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc} (72%) diff --git a/PairHMM_JNI/Makefile b/PairHMM_JNI/Makefile index 9b461a833..e207dd4cc 100644 --- a/PairHMM_JNI/Makefile +++ b/PairHMM_JNI/Makefile @@ -13,7 +13,7 @@ CXX=icc LDFLAGS=-lm -lrt $(OMPLDFLAGS) -BIN=libJNILoglessPairHMM.so pairhmm-template-main checker +BIN=libVectorLoglessPairHMM.so pairhmm-template-main checker #BIN=checker DEPDIR=.deps @@ -22,14 +22,14 @@ DF=$(DEPDIR)/$(*).d #Common across libJNI and sandbox COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc sse_function_instantiations.cc LoadTimeInitializer.cc #Part of libJNI -LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc $(COMMON_SOURCES) +LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc $(COMMON_SOURCES) SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc LIBOBJECTS=$(LIBSOURCES:.cc=.o) COMMON_OBJECTS=$(COMMON_SOURCES:.cc=.o) #No vectorization for these files -NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc LoadTimeInitializer.cc +NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc LoadTimeInitializer.cc #Use -xAVX for these files AVX_SOURCES=avx_function_instantiations.cc #Use -xSSE4.2 for these files @@ -53,7 +53,7 @@ checker: pairhmm-1-base.o $(COMMON_OBJECTS) pairhmm-template-main: pairhmm-template-main.o $(COMMON_OBJECTS) $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) -libJNILoglessPairHMM.so: $(LIBOBJECTS) +libVectorLoglessPairHMM.so: $(LIBOBJECTS) $(CXX) $(OMPLFLAGS) -shared -o $@ $(LIBOBJECTS) ${LDFLAGS} diff --git a/PairHMM_JNI/jni_common.h b/PairHMM_JNI/jni_common.h new file mode 100644 index 000000000..d1ce86e9c --- /dev/null +++ b/PairHMM_JNI/jni_common.h @@ -0,0 +1,32 @@ +#ifndef JNI_COMMON_H +#define JNI_COMMON_H + +#include +#define ENABLE_ASSERTIONS 1 +#define DO_PROFILING 1 +//#define DEBUG 1 +//#define DEBUG0_1 1 +//#define DEBUG3 1 + + +#define DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY 1 + +#ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY +//Gets direct access to Java arrays +#define GET_BYTE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical + +#else +//Likely makes copy of Java arrays to JNI C++ space +#define GET_BYTE_ARRAY_ELEMENTS env->GetByteArrayElements +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleaseByteArrayElements +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetDoubleArrayElements +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleaseDoubleArrayElements + +#endif //ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY + +#endif //ifndef JNI_COMMON_H diff --git a/PairHMM_JNI/jnidebug.h b/PairHMM_JNI/jnidebug.h new file mode 100644 index 000000000..7002b3637 --- /dev/null +++ b/PairHMM_JNI/jnidebug.h @@ -0,0 +1,166 @@ +#ifndef JNI_DEBUG_H +#define JNI_DEBUG_H + +template +class DataHolder +{ +#define INIT_MATRIX(X) \ + X = new NUMBER*[m_paddedMaxReadLength]; \ + for(int i=0;i ctx; + for (int r = 1; r <= length;r++) //in original code, r < ROWS (where ROWS = paddedReadLength) + { + int _i = insertionGOP[r-1]; //insertionGOP + int _d = deletionGOP[r-1]; //deletionGOP + int _c = overallGCP[r-1]; //overallGCP + m_transition[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; //lines 161-162 + m_transition[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; //line 163 + m_transition[r][MX] = ctx.ph2pr[_i]; //164 + m_transition[r][XX] = ctx.ph2pr[_c]; //165 + m_transition[r][MY] = ctx.ph2pr[_d];//last row seems different, compared to line 166 + m_transition[r][YY] = ctx.ph2pr[_c];//same as above for line 167 + //m_transition[r][MY] = (r == length) ? ctx._(1.0) : ctx.ph2pr[_d];//last row seems different, compared to line 166 + //m_transition[r][YY] = (r == length) ? ctx._(1.0) : ctx.ph2pr[_c];//same as above for line 167 +#ifdef DEBUG3 + for(int j=0;j<6;++j) + debug_dump("transitions_jni.txt", to_string(m_transition[r][j]),true); +#endif + } + ++g_num_prob_init; + } + bool m_is_initialized; + int m_readMaxLength; + int m_haplotypeMaxLength; + int m_paddedMaxReadLength; + int m_paddedMaxHaplotypeLength; + NUMBER** m_matchMatrix; + NUMBER** m_insertionMatrix; + NUMBER** m_deletionMatrix; + NUMBER** m_prior; + NUMBER (*m_transition)[6]; +}; +extern DataHolder g_double_dataholder; + +template +NUMBER compute_full_prob(testcase *tc, NUMBER** M, NUMBER** X, NUMBER** Y, NUMBER (*p)[6], + bool do_initialization, jint hapStartIndex, NUMBER *before_last_log = NULL) +{ + int r, c; + int ROWS = tc->rslen + 1; //ROWS = paddedReadLength + int COLS = tc->haplen + 1; //COLS = paddedHaplotypeLength + + Context ctx; + //////NOTES + ////ctx.ph2pr[quality]; //This quantity is QualityUtils.qualToErrorProb(quality) + ////1-ctx.ph2pr[quality]; //This corresponds to QualityUtils.qualToProb(quality); + + //Initialization + if(do_initialization) + { + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); //code from 87-90 in LoglessPairHMM + } + + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + //deletionMatrix row 0 in above nest is initialized in the Java code + //However, insertionMatrix column 0 is not initialized in Java code, could it be that + //values are re-used from a previous iteration? + //Why even do this, X[0][0] = 0 from above loop nest, X[idx][0] = 0 from this computation + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } + } + + for (r = 1; r < ROWS; r++) + for (c = hapStartIndex+1; c < COLS; c++) + { + //The following lines correspond to initializePriors() + char _rs = tc->rs[r-1]; //line 137 + char _hap = tc->hap[c-1]; //line 140 + //int _q = tc->q[r-1] & 127; //line 138 - q is the quality (qual), should be byte hence int ANDed with 0xFF + int _q = tc->q[r-1]; //line 138 - q is the quality (qual), should be byte hence int ANDed with 0xFF + NUMBER distm = ctx.ph2pr[_q]; //This quantity is QualityUtils.qualToErrorProb(_q) + //The assumption here is that doNotUseTristateCorrection is true + //TOASK + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; //This is the quantity QualityUtils.qualToProb(qual) + else + distm = distm/3; +#ifdef DEBUG3 + debug_dump("priors_jni.txt",to_string(distm),true); +#endif + + //Computation inside updateCell + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; +#ifdef DEBUG3 + debug_dump("matrices_jni.txt",to_string(M[r][c]),true); + debug_dump("matrices_jni.txt",to_string(X[r][c]),true); + debug_dump("matrices_jni.txt",to_string(Y[r][c]),true); +#endif + } + + NUMBER result = ctx._(0.0); + for (c = 0; c < COLS; c++) + result += M[ROWS-1][c] + X[ROWS-1][c]; + + if (before_last_log != NULL) + *before_last_log = result; + +#ifdef DEBUG + debug_dump("return_values_jni.txt",to_string(ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT),true); +#endif + return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; +} + +#endif diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc new file mode 100644 index 000000000..30f118c31 --- /dev/null +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc @@ -0,0 +1,151 @@ +#include "headers.h" +#include "jni_common.h" +#include "org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +#include "jnidebug.h" +DataHolder g_double_dataholder; + +using namespace std; + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitialize +(JNIEnv* env, jobject thisObject, + jint readMaxLength, jint haplotypeMaxLength) +{ + static int g_num_init_calls = 0; +#ifdef DEBUG3 + cout << "Entered alloc initialized .. readMaxLength "<GetArrayLength(insertionGOP); +#ifdef DEBUG3 + cout << "Entered initializeProbabilities .. length "<GetByteArrayElements(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (env)->GetByteArrayElements(deletionGOP, &is_copy); + jbyte* overallGCPArray = (env)->GetByteArrayElements(overallGCP, &is_copy); +#ifdef DEBUG + if(insertionGOPArray == 0) + cerr << "insertionGOP array not initialized in JNI\n"; + ////assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + if(deletionGOPArray == 0) + cerr << "deletionGOP array not initialized in JNI\n"; + ////assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); +#endif + + g_double_dataholder.initializeProbabilities(length, insertionGOPArray, deletionGOPArray, overallGCPArray); + + env->ReleaseByteArrayElements(overallGCP, overallGCPArray, JNI_ABORT); + env->ReleaseByteArrayElements(deletionGOP, deletionGOPArray, JNI_ABORT); + env->ReleaseByteArrayElements(insertionGOP, insertionGOPArray, JNI_ABORT); +} + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializePriorsAndUpdateCells( + JNIEnv* env, jobject thisObject, + jboolean doInitialization, jint paddedReadLength, jint paddedHaplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jint hapStartIndex + ) +{ +#ifdef DEBUG3 + cout << "Entered mainCompute .. doInitialization "<<(doInitialization == JNI_TRUE)<<" hapStartIndex "<GetByteArrayElements(readBases, &is_copy); + jbyte* haplotypeBasesArray = (env)->GetByteArrayElements(haplotypeBases, &is_copy); + jbyte* readQualsArray = (env)->GetByteArrayElements(readQuals, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); +#endif + testcase tc; + + tc.rslen = paddedReadLength-1; + tc.haplen = paddedHaplotypeLength-1; + + tc.rs = (char*)readBasesArray; + tc.hap = (char*)haplotypeBasesArray; + tc.q = (char*)readQualsArray; //TOASK - q is now char* + + compute_full_prob(&tc, g_double_dataholder.m_matchMatrix, g_double_dataholder.m_insertionMatrix, + g_double_dataholder.m_deletionMatrix, g_double_dataholder.m_transition, + doInitialization == JNI_TRUE, hapStartIndex, NULL); + + env->ReleaseByteArrayElements(readBases, readBasesArray, JNI_ABORT); + env->ReleaseByteArrayElements(haplotypeBases, haplotypeBasesArray, JNI_ABORT); + env->ReleaseByteArrayElements(readQuals, readQualsArray, JNI_ABORT); + return 0.0; +} + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10( + JNIEnv* env, jobject thisObject, + jint readLength, jint haplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP, + jint hapStartIndex + ) +{ + jboolean is_copy = JNI_FALSE; + jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBases, &is_copy); + jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); + jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); + jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); + assert(readLength < MROWS); +#endif + testcase tc; + tc.rslen = readLength; + tc.haplen = haplotypeLength; + tc.rs = (char*)readBasesArray; + tc.hap = (char*)haplotypeBasesArray; + for(unsigned i=0;i -/* Header for class org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM */ - -#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM -#define _Included_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM -#ifdef __cplusplus -extern "C" { -#endif -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_TRISTATE_CORRECTION -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_TRISTATE_CORRECTION 3.0 -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToMatch -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToMatch 0L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_indelToMatch -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_indelToMatch 1L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToInsertion -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToInsertion 2L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_insertionToInsertion -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_insertionToInsertion 3L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToDeletion -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_matchToDeletion 4L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_deletionToDeletion -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_deletionToDeletion 5L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug 0L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_verify -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_verify 0L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug0_1 -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug0_1 0L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug1 -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug1 0L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug2 -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug2 0L -#undef org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug3 -#define org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_debug3 0L -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniGlobalInit - * Signature: (Ljava/lang/Class;Ljava/lang/Class;)V - */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniGlobalInit - (JNIEnv *, jobject, jclass, jclass); - -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniClose - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniClose - (JNIEnv *, jobject); - -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniInitialize - * Signature: (II)V - */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitialize - (JNIEnv *, jobject, jint, jint); - -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniInitializeProbabilities - * Signature: ([[D[B[B[B)V - */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeProbabilities - (JNIEnv *, jclass, jobjectArray, jbyteArray, jbyteArray, jbyteArray); - -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniInitializePriorsAndUpdateCells - * Signature: (ZII[B[B[BI)D - */ -JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializePriorsAndUpdateCells - (JNIEnv *, jobject, jboolean, jint, jint, jbyteArray, jbyteArray, jbyteArray, jint); - -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniSubComputeReadLikelihoodGivenHaplotypeLog10 - * Signature: (II[B[B[B[B[B[BI)D - */ -JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10 - (JNIEnv *, jobject, jint, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jint); - -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniInitializeHaplotypes - * Signature: (I[Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIHaplotypeDataHolderClass;)V - */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeHaplotypes - (JNIEnv *, jobject, jint, jobjectArray); - -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniFinalizeRegion - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniFinalizeRegion - (JNIEnv *, jobject); - -/* - * Class: org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM - * Method: jniComputeLikelihoods - * Signature: (II[Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIReadDataHolderClass;[Lorg/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM/JNIHaplotypeDataHolderClass;[DI)V - */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniComputeLikelihoods - (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc similarity index 72% rename from PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc rename to PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index a4cd63bc4..54df4265f 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -1,120 +1,16 @@ #include "headers.h" -#include -#include "org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM.h" - - -#define ENABLE_ASSERTIONS 1 -#define DO_PROFILING 1 -//#define DEBUG 1 -//#define DEBUG0_1 1 -//#define DEBUG3 1 - +#include "jni_common.h" +#include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" #include "template.h" #include "utils.h" #include "LoadTimeInitializer.h" using namespace std; -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeProbabilities -(JNIEnv* env, jclass thisObject, - jobjectArray transition, jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP - ) -{} - - JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitialize -(JNIEnv* env, jobject thisObject, - jint readMaxLength, jint haplotypeMaxLength) -{} - -JNIEXPORT jdouble JNICALL -Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializePriorsAndUpdateCells( - JNIEnv* env, jobject thisObject, - jboolean doInitialization, jint paddedReadLength, jint paddedHaplotypeLength, - jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, - jint hapStartIndex - ) - -{ return 0.0; } - -#define DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY 1 - -#ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY -//Gets direct access to Java arrays -#define GET_BYTE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical -#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical -#define JNI_RO_RELEASE_MODE JNI_ABORT -#define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical -#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical - -#else -//Likely makes copy of Java arrays to JNI C++ space -#define GET_BYTE_ARRAY_ELEMENTS env->GetByteArrayElements -#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleaseByteArrayElements -#define JNI_RO_RELEASE_MODE JNI_ABORT -#define GET_DOUBLE_ARRAY_ELEMENTS env->GetDoubleArrayElements -#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleaseDoubleArrayElements - -#endif //ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY - -JNIEXPORT jdouble JNICALL -Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10( - JNIEnv* env, jobject thisObject, - jint readLength, jint haplotypeLength, - jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, - jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP, - jint hapStartIndex - ) -{ - jboolean is_copy = JNI_FALSE; - jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); - jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBases, &is_copy); - jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); - jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); - jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); - jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); -#ifdef DEBUG - assert(readBasesArray && "readBasesArray not initialized in JNI"); - assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); - assert(readQualsArray && "readQualsArray not initialized in JNI"); - assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); - assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); - assert(overallGCPArray && "OverallGCP array not initialized in JNI"); - assert(readLength < MROWS); -#endif - testcase tc; - tc.rslen = readLength; - tc.haplen = haplotypeLength; - tc.rs = (char*)readBasesArray; - tc.hap = (char*)haplotypeBasesArray; - for(unsigned i=0;i > g_haplotypeBasesArrayVector; -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniInitializeHaplotypes +vector g_haplotypeBasesLengths; +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes (JNIEnv * env, jobject thisObject, jint numHaplotypes, jobjectArray haplotypeDataArray) { jboolean is_copy = JNI_FALSE; //To ensure, GET_BYTE_ARRAY_ELEMENTS is invoked only once for each haplotype, store bytearrays in a vector vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; haplotypeBasesArrayVector.clear(); + g_haplotypeBasesLengths.clear(); haplotypeBasesArrayVector.resize(numHaplotypes); + g_haplotypeBasesLengths.resize(numHaplotypes); + jsize haplotypeBasesLength = 0; for(unsigned j=0;jGetObjectArrayElement(haplotypeDataArray, j); @@ -165,20 +65,22 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai #endif env->DeleteLocalRef(haplotypeBases); //free the local reference jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBasesGlobalRef, &is_copy); + haplotypeBasesLength = env->GetArrayLength(haplotypeBasesGlobalRef); #ifdef ENABLE_ASSERTIONS assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); - assert(env->GetArrayLength(haplotypeBasesGlobalRef) < MCOLS); + assert(haplotypeBasesLength < MCOLS); #endif #ifdef DEBUG0_1 - cout << "JNI haplotype length "<GetArrayLength(haplotypeBasesGlobalRef)<<"\n"; + cout << "JNI haplotype length "<GetArrayLength(haplotypeBases);++k) + for(unsigned k=0;kGetArrayLength(haplotypeBasesGlobalRef); + g_load_time_initializer.m_sumHaplotypeLengths += haplotypeBasesLength; #endif } } @@ -188,7 +90,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai //haplotypeDataArray - array of JNIHaplotypeDataHolderClass objects which contain the haplotypeBases //likelihoodArray - array of doubles to return results back to Java. Memory allocated by Java prior to JNI call //maxNumThreadsToUse - Max number of threads that OpenMP can use for the HMM computation -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniComputeLikelihoods +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) { @@ -265,7 +167,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai for(unsigned j=0;jGetArrayLength(haplotypeBasesArrayVector[j].first); + jsize haplotypeLength = (jsize)g_haplotypeBasesLengths[j]; jbyte* haplotypeBasesArray = haplotypeBasesArrayVector[j].second; tc_array[tc_idx].rslen = (int)readLength; tc_array[tc_idx].haplen = (int)haplotypeLength; @@ -363,7 +265,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai } //Release haplotypes at the end of a region -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniFinalizeRegion +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniFinalizeRegion (JNIEnv * env, jobject thisObject) { vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; @@ -373,11 +275,12 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPai RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBasesArrayVector[j].first, haplotypeBasesArrayVector[j].second, JNI_RO_RELEASE_MODE); env->DeleteGlobalRef(haplotypeBasesArrayVector[j].first); //free the global reference } - haplotypeBasesArrayVector.clear(); + haplotypeBasesArrayVector.clear(); + g_haplotypeBasesLengths.clear(); } -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniClose +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose (JNIEnv* env, jobject thisObject) { #ifdef DO_PROFILING diff --git a/PairHMM_JNI/run.sh b/PairHMM_JNI/run.sh index 65a53289a..a1eacdefa 100755 --- a/PairHMM_JNI/run.sh +++ b/PairHMM_JNI/run.sh @@ -1,14 +1,20 @@ #!/bin/bash rm -f *.txt *.log GSA_ROOT_DIR=/home/karthikg/broad/gsa-unstable +pair_hmm_implementation="VECTOR_LOGLESS_CACHING"; +if [ "$#" -ge 1 ]; +then + pair_hmm_implementation=$1; +fi + #-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI/ -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ +--dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -R /data/broad/samples/joint_variant_calling/broad_reference/human_g1k_v37_decoy.fasta \ -I /data/broad/samples/joint_variant_calling/NA12878_low_coverage_alignment/NA12878.chrom11.ILLUMINA.bwa.CEU.low_coverage.20121211.bam \ ---dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -stand_call_conf 50.0 \ -stand_emit_conf 10.0 \ ---pair_hmm_implementation JNI_LOGLESS_CACHING \ +--pair_hmm_implementation $pair_hmm_implementation \ -o output.raw.snps.indels.vcf #--pair_hmm_implementation JNI_LOGLESS_CACHING \ diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index 66d8ee33c..c534764a5 100644 --- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -93,8 +93,8 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation return new CnyPairHMM(); case VECTOR_LOGLESS_CACHING: return new VectorLoglessPairHMM(); - case DEBUG_JNI_LOGLESS_CACHING: - return new DebugJNILoglessPairHMM(hmmType); + case DEBUG_VECTOR_LOGLESS_CACHING: + return new DebugJNILoglessPairHMM(PairHMM.HMM_IMPLEMENTATION.VECTOR_LOGLESS_CACHING); case ARRAY_LOGLESS: if (noFpga || !CnyPairHMM.isAvailable()) return new ArrayLoglessPairHMM(); diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java index 294c5c451..55491bcd8 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java @@ -76,7 +76,7 @@ import java.io.IOException; public class DebugJNILoglessPairHMM extends LoglessPairHMM { private static final boolean debug = false; //simulates ifdef - private static final boolean verify = debug || false; //simulates ifdef + private static final boolean verify = debug || true; //simulates ifdef private static final boolean debug0_1 = false; //simulates ifdef private static final boolean debug1 = false; //simulates ifdef private static final boolean debug2 = false; diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java index 1c7adf4c3..9b0772941 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java @@ -58,4 +58,5 @@ import java.util.HashMap; */ public abstract class JNILoglessPairHMM extends LoglessPairHMM { public abstract HashMap getHaplotypeToHaplotypeListIdxMap(); + protected long setupTime = 0; } diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java index 9d6812c0c..6207c9679 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java @@ -70,6 +70,7 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { //For machine capabilities public static final long sse42Mask = 1; public static final long avxMask = 2; + public static final long enableAll = 0xFFFFFFFFFFFFFFFFl; //Used to copy references to byteArrays to JNI from reads protected class JNIReadDataHolderClass { @@ -95,13 +96,15 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { if(!isVectorLoglessPairHMMLibraryLoaded) { System.loadLibrary("VectorLoglessPairHMM"); isVectorLoglessPairHMMLibraryLoaded = true; - jniGlobalInit(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, 0xFFFFFFFFFFFFFFFFl); //need to do this only once + jniGlobalInit(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once } } @Override public void close() { + System.out.println("Time spent in setup for JNI call : "+setupTime); + super.close(); jniClose(); } @@ -154,6 +157,8 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { */ @Override public PerReadAlleleLikelihoodMap computeLikelihoods( final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap ) { + if(doProfiling) + startTime = System.nanoTime(); int readListSize = reads.size(); int numHaplotypes = alleleHaplotypeMap.size(); int numTestcases = readListSize*numHaplotypes; @@ -170,12 +175,13 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { ++idx; } - JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; mLikelihoodArray = new double[readListSize*numHaplotypes]; //to store results + if(doProfiling) + setupTime += (System.nanoTime() - startTime); //for(reads) // for(haplotypes) // compute_full_prob() - jniComputeLikelihoods(readListSize, numHaplotypes, readDataArray, haplotypeDataArray, mLikelihoodArray, 12); + jniComputeLikelihoods(readListSize, numHaplotypes, readDataArray, null, mLikelihoodArray, 12); final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap(); idx = 0; @@ -193,6 +199,8 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { } readIdx += numHaplotypes; } + if(doProfiling) + computeTime += (System.nanoTime() - startTime); return likelihoodMap; } } diff --git a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java index 8b2123751..4ab17d5f3 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java +++ b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java @@ -59,8 +59,8 @@ public abstract class PairHMM { LOGLESS_CACHING, /* Optimized AVX implementation of LOGLESS_CACHING called through JNI */ VECTOR_LOGLESS_CACHING, - /* Debugging for any JNI implementation of LOGLESS_CACHING */ - DEBUG_JNI_LOGLESS_CACHING, + /* Debugging for vector implementation of LOGLESS_CACHING */ + DEBUG_VECTOR_LOGLESS_CACHING, /* Logless caching PairHMM that stores computations in 1D arrays instead of matrices, and which proceeds diagonally over the (read x haplotype) intersection matrix */ ARRAY_LOGLESS } @@ -77,6 +77,11 @@ public abstract class PairHMM { //debug array protected double[] mLikelihoodArray; + //profiling information + protected static final boolean doProfiling = true; + protected long computeTime = 0; + protected long startTime = 0; + /** * Initialize this PairHMM, making it suitable to run against a read and haplotype with given lengths * @@ -107,7 +112,7 @@ public abstract class PairHMM { */ public void finalizeRegion() { - ; + ; } /** @@ -119,7 +124,7 @@ public abstract class PairHMM { * @param readMaxLength the max length of reads we want to use with this PairHMM */ public void initialize( final List haplotypes, final Map> perSampleReadList, final int readMaxLength, final int haplotypeMaxLength ) { - initialize(readMaxLength, haplotypeMaxLength); + initialize(readMaxLength, haplotypeMaxLength); } protected int findMaxReadLength(final List reads) { @@ -152,6 +157,8 @@ public abstract class PairHMM { * said read coming from the said haplotype under the provided error model */ public PerReadAlleleLikelihoodMap computeLikelihoods(final List reads, final Map alleleHaplotypeMap, final Map GCPArrayMap) { + if(doProfiling) + startTime = System.nanoTime(); // (re)initialize the pairHMM only if necessary final int readMaxLength = findMaxReadLength(reads); @@ -159,8 +166,8 @@ public abstract class PairHMM { if (!initialized || readMaxLength > maxReadLength || haplotypeMaxLength > maxHaplotypeLength) { initialize(readMaxLength, haplotypeMaxLength); } final PerReadAlleleLikelihoodMap likelihoodMap = new PerReadAlleleLikelihoodMap(); - mLikelihoodArray = new double[reads.size()*alleleHaplotypeMap.size()]; - int idx = 0; + mLikelihoodArray = new double[reads.size()*alleleHaplotypeMap.size()]; + int idx = 0; for(GATKSAMRecord read : reads){ final byte[] readBases = read.getReadBases(); final byte[] readQuals = read.getBaseQualities(); @@ -173,16 +180,16 @@ public abstract class PairHMM { boolean isFirstHaplotype = true; Allele currentAllele = null; double log10l; - //for (final Allele allele : alleleHaplotypeMap.keySet()){ + //for (final Allele allele : alleleHaplotypeMap.keySet()){ for (Map.Entry currEntry : alleleHaplotypeMap.entrySet()){ - //final Haplotype haplotype = alleleHaplotypeMap.get(allele); - final Allele allele = currEntry.getKey(); - final Haplotype haplotype = currEntry.getValue(); + //final Haplotype haplotype = alleleHaplotypeMap.get(allele); + final Allele allele = currEntry.getKey(); + final Haplotype haplotype = currEntry.getValue(); final byte[] nextHaplotypeBases = haplotype.getBases(); if (currentHaplotypeBases != null) { log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, nextHaplotypeBases); - mLikelihoodArray[idx++] = log10l; + mLikelihoodArray[idx++] = log10l; likelihoodMap.add(read, currentAllele, log10l); } // update the current haplotype @@ -196,9 +203,11 @@ public abstract class PairHMM { log10l = computeReadLikelihoodGivenHaplotypeLog10(currentHaplotypeBases, readBases, readQuals, readInsQuals, readDelQuals, overallGCP, isFirstHaplotype, null); likelihoodMap.add(read, currentAllele, log10l); - mLikelihoodArray[idx++] = log10l; + mLikelihoodArray[idx++] = log10l; } } + if(doProfiling) + computeTime += (System.nanoTime() - startTime); return likelihoodMap; } @@ -305,5 +314,9 @@ public abstract class PairHMM { public double[] getLikelihoodArray() { return mLikelihoodArray; } //Called at the end of all HC calls - public void close() { ; } + public void close() + { + if(doProfiling) + System.out.println("Total compute time in PairHMM computeLikelihoods() : "+computeTime); + } } From 85a748860e37c22f95936ba04eb59950b93611f1 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Mon, 27 Jan 2014 14:32:44 -0800 Subject: [PATCH 24/72] 1. Added more profiling code 2. Modified JNI_README --- PairHMM_JNI/JNI_README | 46 ++++++++------- PairHMM_JNI/LoadTimeInitializer.cc | 5 ++ PairHMM_JNI/LoadTimeInitializer.h | 5 +- ...ng_utils_pairhmm_DebugJNILoglessPairHMM.cc | 2 +- ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 22 +++++++- ...sting_utils_pairhmm_VectorLoglessPairHMM.h | 4 +- PairHMM_JNI/run.sh | 4 +- PairHMM_JNI/utils.cc | 2 +- PairHMM_JNI/utils.h | 4 +- .../utils/pairhmm/DebugJNILoglessPairHMM.java | 2 +- .../utils/pairhmm/JNILoglessPairHMM.java | 1 + .../utils/pairhmm/VectorLoglessPairHMM.java | 56 +++++++++++++------ .../sting/utils/pairhmm/PairHMM.java | 9 ++- 13 files changed, 115 insertions(+), 47 deletions(-) diff --git a/PairHMM_JNI/JNI_README b/PairHMM_JNI/JNI_README index 9fc3aa8d1..3eb797ab6 100644 --- a/PairHMM_JNI/JNI_README +++ b/PairHMM_JNI/JNI_README @@ -1,33 +1,41 @@ Implementation overview: -Created a new Java class called JNILoglessPairHMM which extends LoglessPairHMM and +Created a new Java class called VectorLoglessPairHMM which extends LoglessPairHMM and overrides functions from both LoglessPairHMM and PairHMM. -1. Constructor: Call base class constructors. Then, load the JNI library located in this +1. Constructor: Call base class constructors. Then, load the native library located in this directory and call a global init function in the library to determine fields ids for the -members of classes JNIReadDataHolder and JNIHaplotypeDataHolders -2. initialize(): Override and do nothing as all matrix manipulation is done in the native -library. -3. computeLikelihoods(): Copies array references for readBases/quals etc and -haplotypeBases to arrays of JNIReadDataHolder and JNIHaplotypeDataHolders classes. Invokes -the JNI function to perform the computation and updates the likelihoodMap. +members of classes JNIReadDataHolder and JNIHaplotypeDataHolders. +2. When the library is loaded, it initializes two global function pointers to point to the +function implementation that is supported on the machine on which the program is being +run. The two pointers are for float and double respectively. This initialization is done +only once for the whole program. +3. initialize(): To initialized the region for PairHMM. Pass haplotype bases to native +code through the JNIHaplotypeDataHolders class. Since the haplotype list is common across +multiple samples in computeReadLikelihoods(), we can store the haplotype bases to the +native code once and re-use across multiple samples. +4. computeLikelihoods(): Copies array references for readBases/quals etc to array of +JNIReadDataHolder objects. Invokes the JNI function to perform the computation and +updates the likelihoodMap. -Note: Lots of debug code still left in the files, however, debug code is not executed -(hopefully, not even compiled into bytecode) because the debug flags are set to false. +Note: Debug code has been moved to a separate class DebugJNILoglessPairHMM.java. On the C++ side, the primary function called is -Java_org_broadinstitute_sting_utils_pairhmm_JNILoglessPairHMM_jniComputeLikelihoods. It -uses standard JNI calls to get and return data from/to the Java class JNILoglessPairHMM. -The last argument to the function is the maximum number of OpenMP threads to use while -computing PairHMM in C++. This option is set when the native function call is made from -JNILoglessPairHMM computeLikelihoods - currently it is set to 12 (no logical reason). +Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods. It +uses standard JNI calls to get and return data from/to the Java class +VectorLoglessPairHMM. The last argument to the function is the maximum number of OpenMP +threads to use while computing PairHMM in C++. This option is set when the native function +call is made from JNILoglessPairHMM computeLikelihoods - currently it is set to 12 (no +logical reason). +Note: OpenMP has been disabled for now. Compiling: Make sure you have icc (Intel C compiler) available. Currently, gcc does not seem to support all AVX intrinsics. -Type 'make'. This should create a library called libJNILoglessPairHMM.so .Comment out -OMPCFLAGS if OpenMP is not desired. +Type 'make'. This should create a library called libVectorLoglessPairHMM.so Running: -If libJNILoglessPairHMM.so is compiled using icc, make sure that the Intel Composer XE +If libVectorLoglessPairHMM.so is compiled using icc, make sure that the Intel Composer XE libraries are in your LD_LIBRARY_PATH : source /bin/compilervars.sh intel64 -See run.sh in this directory on how to invoke HaplotypeCaller with the native library +See run.sh in this directory on how to invoke HaplotypeCaller with the native library. The +argument -Djava.library.path is needed if the native implementation is selected, else +unnecessary. diff --git a/PairHMM_JNI/LoadTimeInitializer.cc b/PairHMM_JNI/LoadTimeInitializer.cc index ae545d881..4188ea3fb 100644 --- a/PairHMM_JNI/LoadTimeInitializer.cc +++ b/PairHMM_JNI/LoadTimeInitializer.cc @@ -12,6 +12,7 @@ LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loa m_sumNumHaplotypes = 0; m_sumSquareNumHaplotypes = 0; m_sumNumTestcases = 0; + m_sumNumDoubleTestcases = 0; m_sumSquareNumTestcases = 0; m_sumReadLengths = 0; m_sumHaplotypeLengths = 0; @@ -19,8 +20,10 @@ LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loa m_sumSquareProductReadLengthHaplotypeLength = 0; m_maxNumTestcases = 0; m_num_invocations = 0; + m_compute_time = 0; m_data_transfer_time = 0; + m_bytes_copied = 0; m_filename_to_fptr.clear(); @@ -47,6 +50,8 @@ void LoadTimeInitializer::print_profiling() mean_val = m_sumProductReadLengthHaplotypeLength/m_sumNumTestcases; cout <<"productReadLengthHaplotypeLength\t"< m_filename_to_fptr; }; diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc index 30f118c31..98ebcde92 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc @@ -117,7 +117,7 @@ Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniSubCompute assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); assert(overallGCPArray && "OverallGCP array not initialized in JNI"); - assert(readLength < MROWS); + //assert(readLength < MROWS); #endif testcase tc; tc.rslen = readLength; diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index 5e77403f4..e837e5765 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -7,9 +7,15 @@ using namespace std; +JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType + (JNIEnv* env, jobject thisObject) +{ + return (jlong)get_machine_capabilities(); +} + //Should be called only once for the whole Java process - initializes field ids for the classes JNIReadDataHolderClass //and JNIHaplotypeDataHolderClass -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGlobalInit +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask (JNIEnv* env, jobject thisObject, jclass readDataHolderClass, jclass haplotypeDataHolderClass, jlong mask) { assert(readDataHolderClass); @@ -34,6 +40,12 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless fid = env->GetFieldID(haplotypeDataHolderClass, "haplotypeBases", "[B"); assert(fid && "JNI pairHMM: Could not get FID for haplotypeBases"); g_load_time_initializer.m_haplotypeBasesFID = fid; + if(mask != ENABLE_ALL_HARDWARE_FEATURES) + { + cout << "Using user supplied hardware mask to re-initialize function pointers for PairHMM\n"; + initialize_function_pointers((uint64_t)mask); + cout.flush(); + } } //Since the list of haplotypes against which the reads are evaluated in PairHMM is the same for a region, @@ -81,6 +93,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless #endif #ifdef DO_PROFILING g_load_time_initializer.m_sumHaplotypeLengths += haplotypeBasesLength; + g_load_time_initializer.m_bytes_copied += (is_copy ? haplotypeBasesLength : 0); #endif } } @@ -139,6 +152,9 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DO_PROFILING + g_load_time_initializer.m_bytes_copied += (is_copy ? readLength*5 : 0); +#endif #ifdef ENABLE_ASSERTIONS assert(readBasesArray && "readBasesArray not initialized in JNI"); assert(readQualsArray && "readQualsArray not initialized in JNI"); @@ -206,6 +222,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless assert(env->GetArrayLength(likelihoodArray) == numTestCases); #endif #ifdef DO_PROFILING + g_load_time_initializer.m_bytes_copied += (is_copy ? numTestCases*sizeof(double) : 0); start_time = get_time(); #endif #pragma omp parallel for schedule (dynamic,10) private(tc_idx) num_threads(maxNumThreadsToUse) @@ -216,6 +233,9 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless if (result_avxf < MIN_ACCEPTED) { double result_avxd = g_compute_full_prob_double(&(tc_array[tc_idx]), 0); result = log10(result_avxd) - log10(ldexp(1.0, 1020.0)); +#ifdef DO_PROFILING + ++(g_load_time_initializer.m_sumNumDoubleTestcases); +#endif } else result = (double)(log10f(result_avxf) - log10f(ldexpf(1.f, 120.f))); diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h index 099751b79..3b07665f6 100644 --- a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h +++ b/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h @@ -43,10 +43,10 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless /* * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM - * Method: jniGlobalInit + * Method: jniInitializeClassFieldsAndMachineMask * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V */ -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGlobalInit +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask (JNIEnv *, jobject, jclass, jclass, jlong); /* diff --git a/PairHMM_JNI/run.sh b/PairHMM_JNI/run.sh index a1eacdefa..e66ce543f 100755 --- a/PairHMM_JNI/run.sh +++ b/PairHMM_JNI/run.sh @@ -10,8 +10,8 @@ fi #-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI/ -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ --dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ --R /data/broad/samples/joint_variant_calling/broad_reference/human_g1k_v37_decoy.fasta \ --I /data/broad/samples/joint_variant_calling/NA12878_low_coverage_alignment/NA12878.chrom11.ILLUMINA.bwa.CEU.low_coverage.20121211.bam \ +-R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ +-I /data/simulated/sim1M_pairs_final.bam \ -stand_call_conf 50.0 \ -stand_emit_conf 10.0 \ --pair_hmm_implementation $pair_hmm_implementation \ diff --git a/PairHMM_JNI/utils.cc b/PairHMM_JNI/utils.cc index 8eb417d94..dcc7835df 100644 --- a/PairHMM_JNI/utils.cc +++ b/PairHMM_JNI/utils.cc @@ -45,7 +45,7 @@ uint64_t get_machine_capabilities() void initialize_function_pointers(uint64_t mask) { - //if(false) + //mask = 0; if(is_avx_supported() && (mask & (1<< AVX_CUSTOM_IDX))) { cout << "Using AVX accelerated implementation of PairHMM\n"; diff --git a/PairHMM_JNI/utils.h b/PairHMM_JNI/utils.h index a09c0204c..783d75ae5 100644 --- a/PairHMM_JNI/utils.h +++ b/PairHMM_JNI/utils.h @@ -25,14 +25,16 @@ extern double (*g_compute_full_prob_double)(testcase *tc, double* before_last_lo void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline); template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); -void initialize_function_pointers(uint64_t mask=0xFFFFFFFFFFFFFFFFull); double getCurrClk(); uint64_t get_time(struct timespec* x=0); +//bit 0 is sse4.2, bit 1 is AVX enum ProcessorCapabilitiesEnum { SSE42_CUSTOM_IDX=0, AVX_CUSTOM_IDX }; +#define ENABLE_ALL_HARDWARE_FEATURES 0xFFFFFFFFFFFFFFFFull uint64_t get_machine_capabilities(); +void initialize_function_pointers(uint64_t mask=ENABLE_ALL_HARDWARE_FEATURES); #endif diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java index 55491bcd8..7e50f0c30 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java @@ -277,7 +277,7 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { } ++numComputeLikelihoodCalls; //if(numComputeLikelihoodCalls == 5) - //jniPairHMM.jniClose(); + //jniPairHMM.close(); //System.exit(0); return likelihoodMap; } diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java index 9b0772941..f039cc295 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java @@ -59,4 +59,5 @@ import java.util.HashMap; public abstract class JNILoglessPairHMM extends LoglessPairHMM { public abstract HashMap getHaplotypeToHaplotypeListIdxMap(); protected long setupTime = 0; + } diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java index 6207c9679..192ebacc1 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java @@ -86,37 +86,45 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { public byte[] haplotypeBases = null; } + /** + * Return 64-bit mask representing machine capabilities + * Bit 0 is LSB, bit 63 MSB + * Bit 0 represents sse4.2 availability + * Bit 1 represents AVX availability + */ public native long jniGetMachineType(); - private native void jniClose(); - private native void jniGlobalInit(Class readDataHolderClass, Class haplotypeDataHolderClass, long mask); + + /** + * Function to initialize the fields of JNIReadDataHolderClass and JNIHaplotypeDataHolderClass from JVM. + * C++ codegets FieldIDs for these classes once and re-uses these IDs for the remainder of the program. Field IDs do not + * change per JVM session + * @param readDataHolderClass class type of JNIReadDataHolderClass + * @param haplotypeDataHolderClass class type of JNIHaplotypeDataHolderClass + * @param mask mask is a 64 bit integer identical to the one received from jniGetMachineType(). Users can disable usage of some hardware features by zeroing some bits in the mask + * */ + private native void jniInitializeClassFieldsAndMachineMask(Class readDataHolderClass, Class haplotypeDataHolderClass, long mask); private static boolean isVectorLoglessPairHMMLibraryLoaded = false; + //The constructor is called only once inside PairHMMLikelihoodCalculationEngine public VectorLoglessPairHMM() { super(); + //Load the library and initialize the FieldIDs if(!isVectorLoglessPairHMMLibraryLoaded) { System.loadLibrary("VectorLoglessPairHMM"); isVectorLoglessPairHMMLibraryLoaded = true; - jniGlobalInit(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once + jniInitializeClassFieldsAndMachineMask(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once } } - @Override - public void close() - { - System.out.println("Time spent in setup for JNI call : "+setupTime); - super.close(); - jniClose(); - } - private native void jniInitializeHaplotypes(final int numHaplotypes, JNIHaplotypeDataHolderClass[] haplotypeDataArray); //Hold the mapping between haplotype and index in the list of Haplotypes passed to initialize //Use this mapping in computeLikelihoods to find the likelihood value corresponding to a given Haplotype private HashMap haplotypeToHaplotypeListIdxMap = new HashMap(); @Override public HashMap getHaplotypeToHaplotypeListIdxMap() { return haplotypeToHaplotypeListIdxMap; } + //Used to transfer data to JNI //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region - /** * {@inheritDoc} */ @@ -136,10 +144,12 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { } jniInitializeHaplotypes(numHaplotypes, haplotypeDataArray); } - + /** + * Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not + * accessing Java memory directly, still important to release memory from C++ + */ private native void jniFinalizeRegion(); - //Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not - //accessing Java memory directly, still important to release memory from C++ + /** * {@inheritDoc} */ @@ -149,7 +159,9 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { jniFinalizeRegion(); } - //Real compute kernel + /** + * Real compute kernel + */ private native void jniComputeLikelihoods(int numReads, int numHaplotypes, JNIReadDataHolderClass[] readDataArray, JNIHaplotypeDataHolderClass[] haplotypeDataArray, double[] likelihoodArray, int maxNumThreadsToUse); /** @@ -203,4 +215,16 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { computeTime += (System.nanoTime() - startTime); return likelihoodMap; } + + /** + * Print final profiling information from native code + */ + public native void jniClose(); + @Override + public void close() + { + System.out.println("Time spent in setup for JNI call : "+(setupTime*1e-9)); + super.close(); + jniClose(); + } } diff --git a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java index 4ab17d5f3..254945af4 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java +++ b/public/java/src/org/broadinstitute/sting/utils/pairhmm/PairHMM.java @@ -312,11 +312,16 @@ public abstract class PairHMM { return Math.min(haplotype1.length, haplotype2.length); } + /** + * Return the results of the computeLikelihoods function + */ public double[] getLikelihoodArray() { return mLikelihoodArray; } - //Called at the end of all HC calls + /** + * Called at the end of the program to close files, print profiling information etc + */ public void close() { if(doProfiling) - System.out.println("Total compute time in PairHMM computeLikelihoods() : "+computeTime); + System.out.println("Total compute time in PairHMM computeLikelihoods() : "+(computeTime*1e-9)); } } From 2c0d70c86328519a571ea1a183c8f1f808420095 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Mon, 27 Jan 2014 14:52:59 -0800 Subject: [PATCH 25/72] Moved vector JNI code to public/c++/VectorPairHMM --- {PairHMM_JNI => public/c++/VectorPairHMM}/.gitignore | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/JNI_README | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/LoadTimeInitializer.cc | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/LoadTimeInitializer.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/Makefile | 0 .../c++/VectorPairHMM}/avx_function_instantiations.cc | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/baseline.cc | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/define-double.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/define-float.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/define-sse-double.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/define-sse-float.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/headers.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/jni_common.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/jnidebug.h | 0 ...g_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc | 0 ...rg_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h | 0 ...org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc | 0 .../org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/pairhmm-1-base.cc | 0 .../c++/VectorPairHMM}/pairhmm-template-kernel.cc | 0 .../c++/VectorPairHMM}/pairhmm-template-main.cc | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/run.sh | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/shift_template.c | 0 .../c++/VectorPairHMM}/sse_function_instantiations.cc | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/template.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/utils.cc | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/utils.h | 0 {PairHMM_JNI => public/c++/VectorPairHMM}/vector_defs.h | 0 .../c++/VectorPairHMM}/vector_function_prototypes.h | 0 29 files changed, 0 insertions(+), 0 deletions(-) rename {PairHMM_JNI => public/c++/VectorPairHMM}/.gitignore (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/JNI_README (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/LoadTimeInitializer.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/LoadTimeInitializer.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/Makefile (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/avx_function_instantiations.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/baseline.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/define-double.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/define-float.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/define-sse-double.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/define-sse-float.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/headers.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/jni_common.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/jnidebug.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/pairhmm-1-base.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/pairhmm-template-kernel.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/pairhmm-template-main.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/run.sh (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/shift_template.c (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/sse_function_instantiations.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/template.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/utils.cc (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/utils.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/vector_defs.h (100%) rename {PairHMM_JNI => public/c++/VectorPairHMM}/vector_function_prototypes.h (100%) diff --git a/PairHMM_JNI/.gitignore b/public/c++/VectorPairHMM/.gitignore similarity index 100% rename from PairHMM_JNI/.gitignore rename to public/c++/VectorPairHMM/.gitignore diff --git a/PairHMM_JNI/JNI_README b/public/c++/VectorPairHMM/JNI_README similarity index 100% rename from PairHMM_JNI/JNI_README rename to public/c++/VectorPairHMM/JNI_README diff --git a/PairHMM_JNI/LoadTimeInitializer.cc b/public/c++/VectorPairHMM/LoadTimeInitializer.cc similarity index 100% rename from PairHMM_JNI/LoadTimeInitializer.cc rename to public/c++/VectorPairHMM/LoadTimeInitializer.cc diff --git a/PairHMM_JNI/LoadTimeInitializer.h b/public/c++/VectorPairHMM/LoadTimeInitializer.h similarity index 100% rename from PairHMM_JNI/LoadTimeInitializer.h rename to public/c++/VectorPairHMM/LoadTimeInitializer.h diff --git a/PairHMM_JNI/Makefile b/public/c++/VectorPairHMM/Makefile similarity index 100% rename from PairHMM_JNI/Makefile rename to public/c++/VectorPairHMM/Makefile diff --git a/PairHMM_JNI/avx_function_instantiations.cc b/public/c++/VectorPairHMM/avx_function_instantiations.cc similarity index 100% rename from PairHMM_JNI/avx_function_instantiations.cc rename to public/c++/VectorPairHMM/avx_function_instantiations.cc diff --git a/PairHMM_JNI/baseline.cc b/public/c++/VectorPairHMM/baseline.cc similarity index 100% rename from PairHMM_JNI/baseline.cc rename to public/c++/VectorPairHMM/baseline.cc diff --git a/PairHMM_JNI/define-double.h b/public/c++/VectorPairHMM/define-double.h similarity index 100% rename from PairHMM_JNI/define-double.h rename to public/c++/VectorPairHMM/define-double.h diff --git a/PairHMM_JNI/define-float.h b/public/c++/VectorPairHMM/define-float.h similarity index 100% rename from PairHMM_JNI/define-float.h rename to public/c++/VectorPairHMM/define-float.h diff --git a/PairHMM_JNI/define-sse-double.h b/public/c++/VectorPairHMM/define-sse-double.h similarity index 100% rename from PairHMM_JNI/define-sse-double.h rename to public/c++/VectorPairHMM/define-sse-double.h diff --git a/PairHMM_JNI/define-sse-float.h b/public/c++/VectorPairHMM/define-sse-float.h similarity index 100% rename from PairHMM_JNI/define-sse-float.h rename to public/c++/VectorPairHMM/define-sse-float.h diff --git a/PairHMM_JNI/headers.h b/public/c++/VectorPairHMM/headers.h similarity index 100% rename from PairHMM_JNI/headers.h rename to public/c++/VectorPairHMM/headers.h diff --git a/PairHMM_JNI/jni_common.h b/public/c++/VectorPairHMM/jni_common.h similarity index 100% rename from PairHMM_JNI/jni_common.h rename to public/c++/VectorPairHMM/jni_common.h diff --git a/PairHMM_JNI/jnidebug.h b/public/c++/VectorPairHMM/jnidebug.h similarity index 100% rename from PairHMM_JNI/jnidebug.h rename to public/c++/VectorPairHMM/jnidebug.h diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc similarity index 100% rename from PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc rename to public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h similarity index 100% rename from PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h rename to public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc similarity index 100% rename from PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc rename to public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc diff --git a/PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h similarity index 100% rename from PairHMM_JNI/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h rename to public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h diff --git a/PairHMM_JNI/pairhmm-1-base.cc b/public/c++/VectorPairHMM/pairhmm-1-base.cc similarity index 100% rename from PairHMM_JNI/pairhmm-1-base.cc rename to public/c++/VectorPairHMM/pairhmm-1-base.cc diff --git a/PairHMM_JNI/pairhmm-template-kernel.cc b/public/c++/VectorPairHMM/pairhmm-template-kernel.cc similarity index 100% rename from PairHMM_JNI/pairhmm-template-kernel.cc rename to public/c++/VectorPairHMM/pairhmm-template-kernel.cc diff --git a/PairHMM_JNI/pairhmm-template-main.cc b/public/c++/VectorPairHMM/pairhmm-template-main.cc similarity index 100% rename from PairHMM_JNI/pairhmm-template-main.cc rename to public/c++/VectorPairHMM/pairhmm-template-main.cc diff --git a/PairHMM_JNI/run.sh b/public/c++/VectorPairHMM/run.sh similarity index 100% rename from PairHMM_JNI/run.sh rename to public/c++/VectorPairHMM/run.sh diff --git a/PairHMM_JNI/shift_template.c b/public/c++/VectorPairHMM/shift_template.c similarity index 100% rename from PairHMM_JNI/shift_template.c rename to public/c++/VectorPairHMM/shift_template.c diff --git a/PairHMM_JNI/sse_function_instantiations.cc b/public/c++/VectorPairHMM/sse_function_instantiations.cc similarity index 100% rename from PairHMM_JNI/sse_function_instantiations.cc rename to public/c++/VectorPairHMM/sse_function_instantiations.cc diff --git a/PairHMM_JNI/template.h b/public/c++/VectorPairHMM/template.h similarity index 100% rename from PairHMM_JNI/template.h rename to public/c++/VectorPairHMM/template.h diff --git a/PairHMM_JNI/utils.cc b/public/c++/VectorPairHMM/utils.cc similarity index 100% rename from PairHMM_JNI/utils.cc rename to public/c++/VectorPairHMM/utils.cc diff --git a/PairHMM_JNI/utils.h b/public/c++/VectorPairHMM/utils.h similarity index 100% rename from PairHMM_JNI/utils.h rename to public/c++/VectorPairHMM/utils.h diff --git a/PairHMM_JNI/vector_defs.h b/public/c++/VectorPairHMM/vector_defs.h similarity index 100% rename from PairHMM_JNI/vector_defs.h rename to public/c++/VectorPairHMM/vector_defs.h diff --git a/PairHMM_JNI/vector_function_prototypes.h b/public/c++/VectorPairHMM/vector_function_prototypes.h similarity index 100% rename from PairHMM_JNI/vector_function_prototypes.h rename to public/c++/VectorPairHMM/vector_function_prototypes.h From a15137a667233ef68e451cfcb06a374c327858fa Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Mon, 27 Jan 2014 14:56:46 -0800 Subject: [PATCH 26/72] Modified run.sh --- public/c++/VectorPairHMM/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/c++/VectorPairHMM/run.sh b/public/c++/VectorPairHMM/run.sh index e66ce543f..a6930678b 100755 --- a/public/c++/VectorPairHMM/run.sh +++ b/public/c++/VectorPairHMM/run.sh @@ -8,7 +8,7 @@ then fi #-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed -java -Djava.library.path=${GSA_ROOT_DIR}/PairHMM_JNI/ -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ +java -Djava.library.path=${GSA_ROOT_DIR}/public/c++/VectorPairHMM -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ --dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ -I /data/simulated/sim1M_pairs_final.bam \ From 0c63d6264f499feadc2beefb46045dfdde5f12d3 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Mon, 27 Jan 2014 15:34:58 -0800 Subject: [PATCH 27/72] 1. Added synchronization block around loadLibrary in VectorLoglessPairHMM 2. Edited Makefile to use static libraries where possible --- .../sting/utils/pairhmm/VectorLoglessPairHMM.java | 14 ++++++++------ public/c++/VectorPairHMM/Makefile | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java index 192ebacc1..ef11600f0 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java @@ -104,15 +104,17 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { * */ private native void jniInitializeClassFieldsAndMachineMask(Class readDataHolderClass, Class haplotypeDataHolderClass, long mask); - private static boolean isVectorLoglessPairHMMLibraryLoaded = false; + private static Boolean isVectorLoglessPairHMMLibraryLoaded = false; //The constructor is called only once inside PairHMMLikelihoodCalculationEngine public VectorLoglessPairHMM() { super(); - //Load the library and initialize the FieldIDs - if(!isVectorLoglessPairHMMLibraryLoaded) { - System.loadLibrary("VectorLoglessPairHMM"); - isVectorLoglessPairHMMLibraryLoaded = true; - jniInitializeClassFieldsAndMachineMask(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once + synchronized(isVectorLoglessPairHMMLibraryLoaded) { + //Load the library and initialize the FieldIDs + if(!isVectorLoglessPairHMMLibraryLoaded) { + System.loadLibrary("VectorLoglessPairHMM"); + isVectorLoglessPairHMMLibraryLoaded = true; + jniInitializeClassFieldsAndMachineMask(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once + } } } diff --git a/public/c++/VectorPairHMM/Makefile b/public/c++/VectorPairHMM/Makefile index e207dd4cc..6eefc3393 100644 --- a/public/c++/VectorPairHMM/Makefile +++ b/public/c++/VectorPairHMM/Makefile @@ -54,7 +54,7 @@ pairhmm-template-main: pairhmm-template-main.o $(COMMON_OBJECTS) $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) libVectorLoglessPairHMM.so: $(LIBOBJECTS) - $(CXX) $(OMPLFLAGS) -shared -o $@ $(LIBOBJECTS) ${LDFLAGS} + $(CXX) $(OMPLFLAGS) -shared -o $@ $(LIBOBJECTS) ${LDFLAGS} -Wl,-Bstatic -limf -lsvml -lirng -Wl,-Bdynamic #-lintlc $(OBJECTS): %.o: %.cc From 5c7427e48c75164400425ebb825c6e096b610f8a Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Wed, 29 Jan 2014 12:10:29 -0800 Subject: [PATCH 28/72] Temporary commit containing debug profiling code - commented out --- .../c++/VectorPairHMM/LoadTimeInitializer.cc | 22 ++++++ .../c++/VectorPairHMM/LoadTimeInitializer.h | 6 ++ public/c++/VectorPairHMM/jni_common.h | 1 + ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 52 +++++++++++++- public/c++/VectorPairHMM/pairhmm-1-base.cc | 72 ++++++++++++++----- public/c++/VectorPairHMM/utils.cc | 12 +++- 6 files changed, 142 insertions(+), 23 deletions(-) diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/c++/VectorPairHMM/LoadTimeInitializer.cc index 4188ea3fb..a1843141e 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.cc +++ b/public/c++/VectorPairHMM/LoadTimeInitializer.cc @@ -86,3 +86,25 @@ void LoadTimeInitializer::debug_close() m_filename_to_fptr.clear(); } +void LoadTimeInitializer::dump_sandbox(unsigned haplotypeLength, unsigned readLength, char* haplotypeBasesArray, testcase& tc) +{ + ofstream& dumpFptr = m_sandbox_fptr; + for(unsigned k=0;k +#include "template.h" class LoadTimeInitializer { public: @@ -10,6 +11,10 @@ class LoadTimeInitializer void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); void debug_close(); + void dump_sandbox(unsigned haplotypeLength, unsigned readLength, char* haplotypeBasesArray, testcase& tc); + void open_sandbox() { m_sandbox_fptr.open("sandbox.txt", std::ios::app); } + void close_sandbox() { m_sandbox_fptr.close(); } + jfieldID m_readBasesFID; jfieldID m_readQualsFID; jfieldID m_insertionGOPFID; @@ -37,6 +42,7 @@ class LoadTimeInitializer uint64_t m_bytes_copied; private: std::map m_filename_to_fptr; + std::ofstream m_sandbox_fptr; }; extern LoadTimeInitializer g_load_time_initializer; diff --git a/public/c++/VectorPairHMM/jni_common.h b/public/c++/VectorPairHMM/jni_common.h index d1ce86e9c..1e23f66ee 100644 --- a/public/c++/VectorPairHMM/jni_common.h +++ b/public/c++/VectorPairHMM/jni_common.h @@ -7,6 +7,7 @@ //#define DEBUG 1 //#define DEBUG0_1 1 //#define DEBUG3 1 +/*#define DUMP_TO_SANDBOX 1*/ #define DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY 1 diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index e837e5765..ba31da420 100644 --- a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc +++ b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -5,6 +5,16 @@ #include "utils.h" #include "LoadTimeInitializer.h" +char* all_ptrs[] = { + "TCAAACCGAAATAAAGGCCAGTATATCCATATCCTTCCCATAAATGTTGATGGAAGAATTATTTGGAAGCCATATAGAATGAAATGACTCTATACACAAATTAAAACACAAAAACGTACTCAAAATAGTCCAGAGACTACAACTTCAAATGCAAAACTATAAATAATCTAAAAGAAAACCTAAGAGACATTC", + "GTCCAGAGACTACAACTTCAAATGCAAAACTATAAATAATCTAACAGAAAACCTAAGAGACATTC", + ">D?@BAEEEEDEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE@EEEEEEEEEEDEEEEEE?", + "IIIIIIIIIIIIIIIIIIIIIIIIHHHHIIIIIIIIIIIIIIIIIIHHHHIIIIIIIIIIIIIIN", + "IIIIIIIIIIIIIIIIIIIIIIIIHHHHIIIIIIIIIIIIIIIIIIHHHHIIIIIIIIIIIIIIN", + "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" +}; +char all_arrays[6][16384]; + using namespace std; JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType @@ -46,6 +56,17 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless initialize_function_pointers((uint64_t)mask); cout.flush(); } +#if 0 + for(unsigned i=0;i<6;++i) + { + unsigned length = strlen(all_ptrs[i]); + for(unsigned j=0;j<16384;++j) + all_arrays[i][j] = all_ptrs[i][j%length]; + } + for(unsigned i=2;i<6;++i) + for(unsigned j=0;j<16384;++j) + all_arrays[i][j] = all_arrays[i][j]-33; +#endif } //Since the list of haplotypes against which the reads are evaluated in PairHMM is the same for a region, @@ -127,6 +148,9 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless readBasesArrayVector.resize(numReads); #ifdef DO_PROFILING start_time = get_time(); +#endif +#ifdef DUMP_TO_SANDBOX + g_load_time_initializer.open_sandbox(); #endif for(unsigned i=0;i tc_vector; tc_vector.clear(); - testcase tc; + tc_vector.resize(BATCH_SIZE+4); + vector results_vec; + results_vec.clear(); + results_vec.resize(tc_vector.size()); + vector baseline_results; + baseline_results.clear(); + baseline_results.resize(tc_vector.size()); + + bool all_ok = true; uint64_t total_time = 0; + uint64_t baseline_time = 0; + unsigned total_count = 0; + unsigned num_testcases = 0; + //unsigned curr_batch_size = rand()%BATCH_SIZE + 4; //min batch size + unsigned curr_batch_size = BATCH_SIZE; while(1) { - int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); + int break_value = use_old_read_testcase ? read_testcase(&(tc_vector[num_testcases]), fptr) : + read_mod_testcase(ifptr,&(tc_vector[num_testcases]),true); if(break_value >= 0) - tc_vector.push_back(tc); - if(tc_vector.size() == BATCH_SIZE || (break_value < 0 && tc_vector.size() > 0)) + ++num_testcases; + if(num_testcases == curr_batch_size || (break_value < 0 && num_testcases > 0)) { - vector results_vec; - results_vec.clear(); - results_vec.resize(tc_vector.size()); get_time(); #pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) - for(unsigned i=0;i(&tc); - baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); + testcase& tc = tc_vector[i]; + float result_avxf = compute_full_prob(&tc); + double result = 0; + if (result_avxf < MIN_ACCEPTED) { + double result_avxd = compute_full_prob(&tc); + result = log10(result_avxd) - log10(ldexp(1.0, 1020.0)); + } + else + result = (double)(log10f(result_avxf) - log10f(ldexpf(1.f, 120.f))); + baseline_results[i] = result; + } + baseline_time += get_time(); + for(unsigned i=0;i 1e-5 && rel_error > 1e-5) - cout << std::scientific << baseline_result << " "<hap, tc->rs, q, i, d, c) != 6) return -1; + tc->haplen = strlen(tc->hap); tc->rslen = strlen(tc->rs); + assert(strlen(q) == tc->rslen); + assert(strlen(i) == tc->rslen); + assert(strlen(d) == tc->rslen); + assert(strlen(c) == tc->rslen); //assert(tc->rslen < MROWS); tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); tc->irs = (int *) malloc(tc->rslen*sizeof(int)); @@ -111,7 +116,8 @@ int read_testcase(testcase *tc, FILE* ifp) _i = normalize(i[x]); _d = normalize(d[x]); _c = normalize(c[x]); - tc->q[x] = (_q < 6) ? 6 : _q; + tc->q[x] = (_q < 6) ? 6 : _q; + //tc->q[x] = _q; tc->i[x] = _i; tc->d[x] = _d; tc->c[x] = _c; @@ -120,13 +126,15 @@ int read_testcase(testcase *tc, FILE* ifp) for (x = 0; x < tc->haplen; x++) tc->ihap[x] = tc->hap[x]; - + free(q); free(i); free(d); free(c); free(line); + + return 0; } From 6d4d7766335d983cfcf8379c4f0762a908d87c4d Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Thu, 30 Jan 2014 12:08:06 -0800 Subject: [PATCH 29/72] Includes code for all debug code for obtaining profiling info --- .../c++/VectorPairHMM/LoadTimeInitializer.cc | 6 +- .../c++/VectorPairHMM/LoadTimeInitializer.h | 2 +- public/c++/VectorPairHMM/Makefile | 2 +- ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 38 ++++- public/c++/VectorPairHMM/pairhmm-1-base.cc | 143 ++++++++++-------- public/c++/VectorPairHMM/utils.cc | 9 ++ public/c++/VectorPairHMM/utils.h | 1 + 7 files changed, 127 insertions(+), 74 deletions(-) diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/c++/VectorPairHMM/LoadTimeInitializer.cc index a1843141e..d403ff892 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.cc +++ b/public/c++/VectorPairHMM/LoadTimeInitializer.cc @@ -86,11 +86,13 @@ void LoadTimeInitializer::debug_close() m_filename_to_fptr.clear(); } -void LoadTimeInitializer::dump_sandbox(unsigned haplotypeLength, unsigned readLength, char* haplotypeBasesArray, testcase& tc) +void LoadTimeInitializer::dump_sandbox(testcase& tc) { + unsigned haplotypeLength = tc.haplen; + unsigned readLength = tc.rslen; ofstream& dumpFptr = m_sandbox_fptr; for(unsigned k=0;k=0;--i)//note the order - reverse of GET { diff --git a/public/c++/VectorPairHMM/pairhmm-1-base.cc b/public/c++/VectorPairHMM/pairhmm-1-base.cc index 805a46a0a..811a67b8c 100644 --- a/public/c++/VectorPairHMM/pairhmm-1-base.cc +++ b/public/c++/VectorPairHMM/pairhmm-1-base.cc @@ -8,14 +8,13 @@ using namespace std; - -#define BATCH_SIZE 5 #define RUN_HYBRID - - +vector results_vec; +vector tc_vector; int main(int argc, char** argv) { +#define BATCH_SIZE 5 if(argc < 2) { cerr << "Needs path to input file as argument\n"; @@ -41,16 +40,12 @@ int main(int argc, char** argv) assert(ifptr.is_open()); } - vector tc_vector; tc_vector.clear(); tc_vector.resize(BATCH_SIZE+4); - vector results_vec; results_vec.clear(); - results_vec.resize(tc_vector.size()); vector baseline_results; baseline_results.clear(); - baseline_results.resize(tc_vector.size()); - + bool all_ok = true; uint64_t total_time = 0; uint64_t baseline_time = 0; @@ -58,76 +53,92 @@ int main(int argc, char** argv) unsigned num_testcases = 0; //unsigned curr_batch_size = rand()%BATCH_SIZE + 4; //min batch size unsigned curr_batch_size = BATCH_SIZE; + uint64_t product = 0; + + testcase tc_in; + int break_value = 0; + tc_vector.clear(); while(1) { - int break_value = use_old_read_testcase ? read_testcase(&(tc_vector[num_testcases]), fptr) : - read_mod_testcase(ifptr,&(tc_vector[num_testcases]),true); + break_value = use_old_read_testcase ? read_testcase(&tc_in, fptr) : + read_mod_testcase(ifptr, &tc_in, true); + tc_vector.push_back(tc_in); if(break_value >= 0) ++num_testcases; - if(num_testcases == curr_batch_size || (break_value < 0 && num_testcases > 0)) - { - get_time(); -#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) - for(unsigned i=0;i(&tc); - double result = 0; - if (result_avxf < MIN_ACCEPTED) { - double result_avxd = compute_full_prob(&tc); - result = log10(result_avxd) - log10(ldexp(1.0, 1020.0)); - } - else - result = (double)(log10f(result_avxf) - log10f(ldexpf(1.f, 120.f))); - baseline_results[i] = result; - } - baseline_time += get_time(); - for(unsigned i=0;i 1e-5 && rel_error > 1e-5) - { - cout << "Line "< 0)) + { + results_vec.resize(tc_vector.size()); + baseline_results.resize(tc_vector.size()); + + get_time(); +#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) + for(unsigned i=0;i(&tc); + double result = 0; + if (result_avxf < MIN_ACCEPTED) { + double result_avxd = compute_full_prob(&tc); + result = log10(result_avxd) - log10(ldexp(1.0, 1020.0)); + } + else + result = (double)(log10f(result_avxf) - log10f(ldexpf(1.f, 120.f))); + baseline_results[i] = result; + } + baseline_time += get_time(); + for(unsigned i=0;i 1e-5 && rel_error > 1e-5) + { + cout << "Line "< NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); double getCurrClk(); uint64_t get_time(struct timespec* x=0); +uint64_t diff_time(struct timespec& prev_time); //bit 0 is sse4.2, bit 1 is AVX enum ProcessorCapabilitiesEnum From 24f8aef3442300e746eadebfce90f4b4faea4e34 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Tue, 4 Feb 2014 16:27:29 -0800 Subject: [PATCH 30/72] Contains profiling, exception tracking, PAPI code Contains Sandbox Java --- public/c++/VectorPairHMM/.gitignore | 1 + .../c++/VectorPairHMM/LoadTimeInitializer.cc | 4 +- .../c++/VectorPairHMM/LoadTimeInitializer.h | 10 +- public/c++/VectorPairHMM/Makefile | 19 +- public/c++/VectorPairHMM/Sandbox.cc | 79 +++++ public/c++/VectorPairHMM/Sandbox.h | 71 +++++ public/c++/VectorPairHMM/Sandbox.java | 278 ++++++++++++++++++ .../Sandbox_JNIHaplotypeDataHolderClass.h | 13 + .../Sandbox_JNIReadDataHolderClass.h | 13 + public/c++/VectorPairHMM/baseline.cc | 96 +++--- public/c++/VectorPairHMM/headers.h | 15 + public/c++/VectorPairHMM/jni_common.h | 4 +- ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 82 +----- public/c++/VectorPairHMM/pairhmm-1-base.cc | 15 +- public/c++/VectorPairHMM/utils.cc | 188 +++++++++++- public/c++/VectorPairHMM/utils.h | 3 + 16 files changed, 751 insertions(+), 140 deletions(-) create mode 100644 public/c++/VectorPairHMM/Sandbox.cc create mode 100644 public/c++/VectorPairHMM/Sandbox.h create mode 100644 public/c++/VectorPairHMM/Sandbox.java create mode 100644 public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h create mode 100644 public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h diff --git a/public/c++/VectorPairHMM/.gitignore b/public/c++/VectorPairHMM/.gitignore index ce903a6fa..b034f9461 100644 --- a/public/c++/VectorPairHMM/.gitignore +++ b/public/c++/VectorPairHMM/.gitignore @@ -6,6 +6,7 @@ tests hmm_Mohammad pairhmm-template-main *.swp +*.class checker reformat subdir_checkout.sh diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/c++/VectorPairHMM/LoadTimeInitializer.cc index d403ff892..fb640ef88 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.cc +++ b/public/c++/VectorPairHMM/LoadTimeInitializer.cc @@ -86,7 +86,7 @@ void LoadTimeInitializer::debug_close() m_filename_to_fptr.clear(); } -void LoadTimeInitializer::dump_sandbox(testcase& tc) +void LoadTimeInitializer::dump_sandbox(testcase& tc, unsigned tc_idx, unsigned numReads, unsigned numHaplotypes) { unsigned haplotypeLength = tc.haplen; unsigned readLength = tc.rslen; @@ -108,5 +108,7 @@ void LoadTimeInitializer::dump_sandbox(testcase& tc) dumpFptr<<" "; for(unsigned k=0;kGetStringUTFChars(fileNameString, 0); + do_compute((char*)fileName); + env->ReleaseStringUTFChars(fileNameString, fileName); +} + diff --git a/public/c++/VectorPairHMM/Sandbox.h b/public/c++/VectorPairHMM/Sandbox.h new file mode 100644 index 000000000..4ac1ea24c --- /dev/null +++ b/public/c++/VectorPairHMM/Sandbox.h @@ -0,0 +1,71 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox */ + +#ifndef _Included_Sandbox +#define _Included_Sandbox +#ifdef __cplusplus +extern "C" { +#endif +#undef Sandbox_enableAll +#define Sandbox_enableAll -1LL +/* + * Class: Sandbox + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_Sandbox_jniGetMachineType + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeClassFieldsAndMachineMask + (JNIEnv *, jobject, jclass, jclass, jlong); + +/* + * Class: Sandbox + * Method: jniInitializeHaplotypes + * Signature: (I[LSandbox/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: Sandbox + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniFinalizeRegion + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniComputeLikelihoods + * Signature: (II[LSandbox/JNIReadDataHolderClass;[LSandbox/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); + +/* + * Class: Sandbox + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniClose + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: doEverythingNative + * Signature: ([B)V + */ +JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative + (JNIEnv *, jobject, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/c++/VectorPairHMM/Sandbox.java b/public/c++/VectorPairHMM/Sandbox.java new file mode 100644 index 000000000..c41a276c2 --- /dev/null +++ b/public/c++/VectorPairHMM/Sandbox.java @@ -0,0 +1,278 @@ +import java.util.List; +import java.util.LinkedList; +import java.util.Map; +import java.util.HashMap; +import java.io.File; +import java.util.Scanner; +import java.io.IOException; +import java.io.FileNotFoundException; +import java.io.InputStreamReader; + +public class Sandbox { + + private long setupTime = 0; + private long computeTime = 0; + //Used to copy references to byteArrays to JNI from reads + protected class JNIReadDataHolderClass { + public byte[] readBases = null; + public byte[] readQuals = null; + public byte[] insertionGOP = null; + public byte[] deletionGOP = null; + public byte[] overallGCP = null; + } + + //Used to copy references to byteArrays to JNI from haplotypes + protected class JNIHaplotypeDataHolderClass { + public byte[] haplotypeBases = null; + } + + /** + * Return 64-bit mask representing machine capabilities + * Bit 0 is LSB, bit 63 MSB + * Bit 0 represents sse4.2 availability + * Bit 1 represents AVX availability + */ + public native long jniGetMachineType(); + public static final long enableAll = 0xFFFFFFFFFFFFFFFFl; + + + /** + * Function to initialize the fields of JNIReadDataHolderClass and JNIHaplotypeDataHolderClass from JVM. + * C++ codegets FieldIDs for these classes once and re-uses these IDs for the remainder of the program. Field IDs do not + * change per JVM session + * @param readDataHolderClass class type of JNIReadDataHolderClass + * @param haplotypeDataHolderClass class type of JNIHaplotypeDataHolderClass + * @param mask mask is a 64 bit integer identical to the one received from jniGetMachineType(). Users can disable usage of some hardware features by zeroing some bits in the mask + * */ + private native void jniInitializeClassFieldsAndMachineMask(Class readDataHolderClass, Class haplotypeDataHolderClass, long mask); + + private static Boolean isVectorLoglessPairHMMLibraryLoaded = false; + //The constructor is called only once inside PairHMMLikelihoodCalculationEngine + public Sandbox() { + synchronized(isVectorLoglessPairHMMLibraryLoaded) { + //Load the library and initialize the FieldIDs + if(!isVectorLoglessPairHMMLibraryLoaded) { + System.loadLibrary("VectorLoglessPairHMM"); + isVectorLoglessPairHMMLibraryLoaded = true; + jniInitializeClassFieldsAndMachineMask(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once + } + } + } + + private native void jniInitializeHaplotypes(final int numHaplotypes, JNIHaplotypeDataHolderClass[] haplotypeDataArray); + + //Used to transfer data to JNI + //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region + public void initialize(final List haplotypes) { + int numHaplotypes = haplotypes.size(); + JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; + int idx = 0; + for(final JNIHaplotypeDataHolderClass currHaplotype : haplotypes) + { + haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); + haplotypeDataArray[idx].haplotypeBases = currHaplotype.haplotypeBases; + ++idx; + } + jniInitializeHaplotypes(numHaplotypes, haplotypeDataArray); + } + /** + * Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not + * accessing Java memory directly, still important to release memory from C++ + */ + private native void jniFinalizeRegion(); + + + public void finalizeRegion() + { + jniFinalizeRegion(); + } + + /** + * Real compute kernel + */ + private native void jniComputeLikelihoods(int numReads, int numHaplotypes, JNIReadDataHolderClass[] readDataArray, + JNIHaplotypeDataHolderClass[] haplotypeDataArray, double[] likelihoodArray, int maxNumThreadsToUse); + + public void computeLikelihoods(final List reads, final List haplotypes) { + //System.out.println("Region : "+reads.size()+" x "+haplotypes.size()); + long startTime = System.nanoTime(); + int readListSize = reads.size(); + int numHaplotypes = haplotypes.size(); + int numTestcases = readListSize*numHaplotypes; + JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; + int idx = 0; + for(JNIReadDataHolderClass read : reads) + { + readDataArray[idx] = new JNIReadDataHolderClass(); + readDataArray[idx].readBases = read.readBases; + readDataArray[idx].readQuals = read.readQuals; + readDataArray[idx].insertionGOP = read.insertionGOP; + readDataArray[idx].deletionGOP = read.deletionGOP; + readDataArray[idx].overallGCP = read.overallGCP; + ++idx; + } + + double[] mLikelihoodArray = new double[readListSize*numHaplotypes]; //to store results + setupTime += (System.nanoTime() - startTime); + //for(reads) + // for(haplotypes) + // compute_full_prob() + jniComputeLikelihoods(readListSize, numHaplotypes, readDataArray, null, mLikelihoodArray, 12); + + computeTime += (System.nanoTime() - startTime); + } + + /** + * Print final profiling information from native code + */ + public native void jniClose(); + public void close() + { + System.out.println("Time spent in setup for JNI call : "+(setupTime*1e-9)+" compute time : "+(computeTime*1e-9)); + jniClose(); + } + + public void parseSandboxFile(String filename) + { + File file = new File(filename); + Scanner input = null; + try + { + input = new Scanner(file); + } + catch(FileNotFoundException e) + { + System.err.println("File "+filename+" cannot be found/read"); + return; + } + int idx = 0; + int numReads = 0; + int numHaplotypes = 0; + int readIdx = 0, testCaseIdx = 0, haplotypeIdx = 0; + LinkedList haplotypeList = new LinkedList(); + LinkedList readList = new LinkedList(); + + byte[][] byteArray = new byte[6][]; + boolean firstLine = true; + String[] currTokens = new String[8]; + while(input.hasNextLine()) + { + String line = input.nextLine(); + Scanner lineScanner = new Scanner(line); + idx = 0; + while(lineScanner.hasNext()) + currTokens[idx++] = lineScanner.next(); + if(idx == 0) + break; + assert(idx >= 6); + //start of new region + if(idx == 8) + { + if(!firstLine) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + try + { + numReads = Integer.parseInt(currTokens[6]); + } + catch(NumberFormatException e) + { + numReads = 1; + } + try + { + numHaplotypes = Integer.parseInt(currTokens[7]); + } + catch(NumberFormatException e) + { + numHaplotypes = 1; + } + haplotypeIdx = readIdx = testCaseIdx = 0; + readList.clear(); + haplotypeList.clear(); + } + if(haplotypeIdx < numHaplotypes) + { + JNIHaplotypeDataHolderClass X = new JNIHaplotypeDataHolderClass(); + X.haplotypeBases = currTokens[0].getBytes(); + haplotypeList.add(X); + } + if(testCaseIdx%numHaplotypes == 0) + { + JNIReadDataHolderClass X = new JNIReadDataHolderClass(); + X.readBases = currTokens[1].getBytes(); + for(int i=2;i<6;++i) + { + byteArray[i] = currTokens[i].getBytes(); + for(int j=0;j 0 && readList.size() > 0) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + + close(); + input.close(); + } + + private native void doEverythingNative(String filename); + + public static void main(String[] args) + { + if(args.length <= 0) + { + System.err.println("Needs 1 argument - "); + System.exit(-1); + } + //// Get runtime + //java.lang.Runtime rt = java.lang.Runtime.getRuntime(); + //// Start a new process: UNIX command ls + //String cmd = "/home/karthikg/broad/gsa-unstable/public/c++/VectorPairHMM/checker "+args[0]; + //try + //{ + //System.out.println(cmd); + //java.lang.Process p = rt.exec(cmd); + //try + //{ + //p.waitFor(); + //java.io.InputStream is = p.getInputStream(); + //java.io.BufferedReader reader = new java.io.BufferedReader(new InputStreamReader(is)); + //// And print each line + //String s = null; + //while ((s = reader.readLine()) != null) { + //System.out.println(s); + //} + //is.close(); + //} + //catch(InterruptedException e) + //{ + //System.err.println(e); + //} + //} + //catch(IOException e) + //{ + //System.err.println(e); + //} + Sandbox t = new Sandbox(); + t.doEverythingNative(args[0]); + //t.parseSandboxFile(args[0]); + } +} diff --git a/public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h b/public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h new file mode 100644 index 000000000..7f78f0178 --- /dev/null +++ b/public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIHaplotypeDataHolderClass */ + +#ifndef _Included_Sandbox_JNIHaplotypeDataHolderClass +#define _Included_Sandbox_JNIHaplotypeDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h b/public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h new file mode 100644 index 000000000..a9312ff3b --- /dev/null +++ b/public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIReadDataHolderClass */ + +#ifndef _Included_Sandbox_JNIReadDataHolderClass +#define _Included_Sandbox_JNIReadDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/c++/VectorPairHMM/baseline.cc b/public/c++/VectorPairHMM/baseline.cc index 2f80acdb0..d92a21ecf 100644 --- a/public/c++/VectorPairHMM/baseline.cc +++ b/public/c++/VectorPairHMM/baseline.cc @@ -1,28 +1,28 @@ #include "headers.h" #include "template.h" - +extern uint64_t exceptions_array[128]; template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) { - int r, c; - int ROWS = tc->rslen + 1; - int COLS = tc->haplen + 1; + int r, c; + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; - Context ctx; + Context ctx; - NUMBER M[ROWS][COLS]; - NUMBER X[ROWS][COLS]; - NUMBER Y[ROWS][COLS]; - NUMBER p[ROWS][6]; + NUMBER M[ROWS][COLS]; + NUMBER X[ROWS][COLS]; + NUMBER Y[ROWS][COLS]; + NUMBER p[ROWS][6]; - p[0][MM] = ctx._(0.0); - p[0][GapM] = ctx._(0.0); - p[0][MX] = ctx._(0.0); - p[0][XX] = ctx._(0.0); - p[0][MY] = ctx._(0.0); - p[0][YY] = ctx._(0.0); - for (r = 1; r < ROWS; r++) - { + p[0][MM] = ctx._(0.0); + p[0][GapM] = ctx._(0.0); + p[0][MX] = ctx._(0.0); + p[0][XX] = ctx._(0.0); + p[0][MY] = ctx._(0.0); + p[0][YY] = ctx._(0.0); + for (r = 1; r < ROWS; r++) + { int _i = tc->i[r-1] & 127; int _d = tc->d[r-1] & 127; int _c = tc->c[r-1] & 127; @@ -34,50 +34,62 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) p[r][YY] = ctx.ph2pr[_c]; //p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; //p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; - } + } - for (c = 0; c < COLS; c++) - { + for (c = 0; c < COLS; c++) + { M[0][c] = ctx._(0.0); X[0][c] = ctx._(0.0); Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); - } + } - for (r = 1; r < ROWS; r++) - { + for (r = 1; r < ROWS; r++) + { M[r][0] = ctx._(0.0); X[r][0] = X[r-1][0] * p[r][XX]; Y[r][0] = ctx._(0.0); - } + } - NUMBER result = ctx._(0.0); + NUMBER result = ctx._(0.0); - for (r = 1; r < ROWS; r++) + for (r = 1; r < ROWS; r++) for (c = 1; c < COLS; c++) { - char _rs = tc->rs[r-1]; - char _hap = tc->hap[c-1]; - int _q = tc->q[r-1] & 127; - NUMBER distm = ctx.ph2pr[_q]; - if (_rs == _hap || _rs == 'N' || _hap == 'N') + fexcept_t flagp; + char _rs = tc->rs[r-1]; + char _hap = tc->hap[c-1]; + int _q = tc->q[r-1] & 127; + NUMBER distm = ctx.ph2pr[_q]; + if (_rs == _hap || _rs == 'N' || _hap == 'N') distm = ctx._(1.0) - distm; - else - distm = distm/3; - M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); - X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; - Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; + else + distm = distm/3; + + + //feclearexcept(FE_ALL_EXCEPT); + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + //M[r][c] = (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //feclearexcept(FE_ALL_EXCEPT); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //feclearexcept(FE_ALL_EXCEPT); + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); } - for (c = 0; c < COLS; c++) - { + for (c = 0; c < COLS; c++) + { result += M[ROWS-1][c] + X[ROWS-1][c]; - } + } - if (before_last_log != NULL) + if (before_last_log != NULL) *before_last_log = result; - return result; - //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; + return result; + //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; } template double compute_full_prob(testcase* tc, double* nextbuf); diff --git a/public/c++/VectorPairHMM/headers.h b/public/c++/VectorPairHMM/headers.h index f502ff5ce..feb687660 100644 --- a/public/c++/VectorPairHMM/headers.h +++ b/public/c++/VectorPairHMM/headers.h @@ -1,3 +1,6 @@ +#ifndef COMMON_HEADERS_H +#define COMMON_HEADERS_H + #include #include #include @@ -22,4 +25,16 @@ #include #include #include +#include +#define STORE_FP_EXCEPTIONS(flagp, exceptions_array) \ + fegetexceptflag(&flagp, FE_OVERFLOW | FE_UNDERFLOW | FE_DIVBYZERO | FE_INVALID | __FE_DENORM); \ + exceptions_array[FE_INVALID] += ((flagp & FE_INVALID)); \ + exceptions_array[__FE_DENORM] += ((flagp & __FE_DENORM) >> 1); \ + exceptions_array[FE_DIVBYZERO] += ((flagp & FE_DIVBYZERO) >> 2); \ + exceptions_array[FE_OVERFLOW] += ((flagp & FE_OVERFLOW) >> 3); \ + exceptions_array[FE_UNDERFLOW] += ((flagp & FE_UNDERFLOW) >> 4); \ + feclearexcept(FE_ALL_EXCEPT); + + +#endif diff --git a/public/c++/VectorPairHMM/jni_common.h b/public/c++/VectorPairHMM/jni_common.h index 1e23f66ee..4c63a0411 100644 --- a/public/c++/VectorPairHMM/jni_common.h +++ b/public/c++/VectorPairHMM/jni_common.h @@ -2,9 +2,9 @@ #define JNI_COMMON_H #include -#define ENABLE_ASSERTIONS 1 +/*#define ENABLE_ASSERTIONS 1*/ #define DO_PROFILING 1 -//#define DEBUG 1 +/*#define DEBUG 1*/ //#define DEBUG0_1 1 //#define DEBUG3 1 /*#define DUMP_TO_SANDBOX 1*/ diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index e31eba232..c39d2ec3f 100644 --- a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc +++ b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -5,16 +5,6 @@ #include "utils.h" #include "LoadTimeInitializer.h" -char* all_ptrs[] = { - "TCAAACCGAAATAAAGGCCAGTATATCCATATCCTTCCCATAAATGTTGATGGAAGAATTATTTGGAAGCCATATAGAATGAAATGACTCTATACACAAATTAAAACACAAAAACGTACTCAAAATAGTCCAGAGACTACAACTTCAAATGCAAAACTATAAATAATCTAAAAGAAAACCTAAGAGACATTC", - "GTCCAGAGACTACAACTTCAAATGCAAAACTATAAATAATCTAACAGAAAACCTAAGAGACATTC", - ">D?@BAEEEEDEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE@EEEEEEEEEEDEEEEEE?", - "IIIIIIIIIIIIIIIIIIIIIIIIHHHHIIIIIIIIIIIIIIIIIIHHHHIIIIIIIIIIIIIIN", - "IIIIIIIIIIIIIIIIIIIIIIIIHHHHIIIIIIIIIIIIIIIIIIHHHHIIIIIIIIIIIIIIN", - "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" -}; -char all_arrays[6][16384]; - using namespace std; JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType @@ -56,17 +46,6 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless initialize_function_pointers((uint64_t)mask); cout.flush(); } -#if 0 - for(unsigned i=0;i<6;++i) - { - unsigned length = strlen(all_ptrs[i]); - for(unsigned j=0;j<16384;++j) - all_arrays[i][j] = all_ptrs[i][j%length]; - } - for(unsigned i=2;i<6;++i) - for(unsigned j=0;j<16384;++j) - all_arrays[i][j] = all_arrays[i][j]-33; -#endif } //Since the list of haplotypes against which the reads are evaluated in PairHMM is the same for a region, @@ -211,44 +190,18 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless jbyte* haplotypeBasesArray = haplotypeBasesArrayVector[j].second; tc_array[tc_idx].rslen = (int)readLength; tc_array[tc_idx].haplen = (int)haplotypeLength; -#if 0 tc_array[tc_idx].hap = (char*)haplotypeBasesArray; tc_array[tc_idx].rs = (char*)readBasesArray; tc_array[tc_idx].q = (char*)readQualsArray; tc_array[tc_idx].i = (char*)insertionGOPArray; tc_array[tc_idx].d = (char*)deletionGOPArray; tc_array[tc_idx].c = (char*)overallGCPArray; -#endif - //#define MEMCPY_HACK -#ifdef MEMCPY_HACK - tc_array[tc_idx].hap = new char[haplotypeLength]; - tc_array[tc_idx].rs = new char[readLength]; - tc_array[tc_idx].q = new char[readLength]; - tc_array[tc_idx].i = new char[readLength]; - tc_array[tc_idx].d = new char[readLength]; - tc_array[tc_idx].c = new char[readLength]; - memcpy(tc_array[tc_idx].hap, haplotypeBasesArray, haplotypeLength); - memcpy(tc_array[tc_idx].rs, readBasesArray, readLength); - memcpy(tc_array[tc_idx].q, readQualsArray, readLength); - memcpy(tc_array[tc_idx].i, insertionGOPArray, readLength); - memcpy(tc_array[tc_idx].d, deletionGOPArray, readLength); - memcpy(tc_array[tc_idx].c, overallGCPArray, readLength); -#endif -#if 0 - tc_array[tc_idx].hap = (char*)all_arrays[0]; - tc_array[tc_idx].rs = (char*)all_arrays[1]; - tc_array[tc_idx].q = (char*)all_arrays[2]; - tc_array[tc_idx].i = (char*)all_arrays[3]; - tc_array[tc_idx].d = (char*)all_arrays[4]; - tc_array[tc_idx].c = (char*)all_arrays[5]; -#endif - -#ifdef DUMP_TO_SANDBOX - g_load_time_initializer.dump_sandbox(tc_array[tc_idx]); -#endif #ifdef DO_PROFILING g_load_time_initializer.m_sumProductReadLengthHaplotypeLength += (readLength*haplotypeLength); g_load_time_initializer.m_sumSquareProductReadLengthHaplotypeLength += ((readLength*haplotypeLength)*(readLength*haplotypeLength)); +#endif +#ifdef DUMP_TO_SANDBOX + g_load_time_initializer.dump_sandbox(tc_array[tc_idx], tc_idx, numReads, numHaplotypes); #endif ++tc_idx; } @@ -265,9 +218,7 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless g_load_time_initializer.m_sumReadLengths += readLength; #endif } -#ifdef DUMP_TO_SANDBOX - g_load_time_initializer.close_sandbox(); -#endif + #ifdef DO_PROFILING g_load_time_initializer.m_data_transfer_time += get_time(); #endif @@ -296,16 +247,6 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless } else result = (double)(log10f(result_avxf) - log10f(ldexpf(1.f, 120.f))); -#if 0 - double result = 0; - testcase& tc = tc_array[tc_idx]; - for(unsigned k=0;k=0;--i)//note the order - reverse of GET { @@ -360,6 +289,9 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless #ifdef DEBUG g_load_time_initializer.debug_close(); #endif +#ifdef DUMP_TO_SANDBOX + g_load_time_initializer.close_sandbox(); +#endif } //Release haplotypes at the end of a region diff --git a/public/c++/VectorPairHMM/pairhmm-1-base.cc b/public/c++/VectorPairHMM/pairhmm-1-base.cc index 811a67b8c..c7a85e647 100644 --- a/public/c++/VectorPairHMM/pairhmm-1-base.cc +++ b/public/c++/VectorPairHMM/pairhmm-1-base.cc @@ -2,16 +2,11 @@ //#define DEBUG0_1 1 //#define DEBUG3 1 #include "headers.h" -#include "template.h" #include "utils.h" #include "LoadTimeInitializer.h" - using namespace std; - #define RUN_HYBRID -vector results_vec; -vector tc_vector; int main(int argc, char** argv) { #define BATCH_SIZE 5 @@ -20,6 +15,8 @@ int main(int argc, char** argv) cerr << "Needs path to input file as argument\n"; exit(0); } + do_compute(argv[1]); +#if 0 bool use_old_read_testcase = false; if(argc >= 3 && string(argv[2]) == "1") use_old_read_testcase = true; @@ -39,7 +36,8 @@ int main(int argc, char** argv) ifptr.open(argv[1]); assert(ifptr.is_open()); } - + vector results_vec; + vector tc_vector; tc_vector.clear(); tc_vector.resize(BATCH_SIZE+4); results_vec.clear(); @@ -58,6 +56,7 @@ int main(int argc, char** argv) testcase tc_in; int break_value = 0; tc_vector.clear(); + g_load_time_initializer.open_sandbox(); while(1) { break_value = use_old_read_testcase ? read_testcase(&tc_in, fptr) : @@ -137,7 +136,7 @@ int main(int argc, char** argv) tc_vector.clear(); if(all_ok) cout << "All outputs acceptable\n"; - cout << "Total vector time "<< ((double)total_time)/1e9 << " baseline time "<rslen); assert(strlen(c) == tc->rslen); //assert(tc->rslen < MROWS); - tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); - tc->irs = (int *) malloc(tc->rslen*sizeof(int)); + //tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); + //tc->irs = (int *) malloc(tc->rslen*sizeof(int)); tc->q = (char *) malloc(sizeof(char) * tc->rslen); tc->i = (char *) malloc(sizeof(char) * tc->rslen); @@ -121,10 +121,10 @@ int read_testcase(testcase *tc, FILE* ifp) tc->i[x] = _i; tc->d[x] = _d; tc->c[x] = _c; - tc->irs[x] = tc->rs[x]; + //tc->irs[x] = tc->rs[x]; } - for (x = 0; x < tc->haplen; x++) - tc->ihap[x] = tc->hap[x]; + //for (x = 0; x < tc->haplen; x++) + //tc->ihap[x] = tc->hap[x]; free(q); @@ -286,4 +286,182 @@ uint64_t diff_time(struct timespec& prev_time) return (uint64_t)((curr_time.tv_sec-prev_time.tv_sec)*1000000000+(curr_time.tv_nsec-prev_time.tv_nsec)); } +//#define USE_PAPI +#ifdef USE_PAPI +#include "papi.h" +#define NUM_PAPI_COUNTERS 4 +#endif +uint64_t exceptions_array[128]; +void do_compute(char* filename) +{ + memset(exceptions_array, 0, 128*sizeof(uint64_t)); + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + //assert(feenableexcept(FE_DIVBYZERO | FE_INVALID) >= 0); +#ifdef USE_PAPI + PAPI_num_counters(); + //int events[NUM_PAPI_COUNTERS] = { PAPI_TOT_INS, PAPI_TOT_CYC, PAPI_L1_DCM, PAPI_L1_ICM, PAPI_L3_TCM, PAPI_TLB_DM, PAPI_TLB_IM }; + //char* eventnames[NUM_PAPI_COUNTERS]= { "instructions", "cycles", "l1d_misses", "l1i_misses", "l3_misses", "dtlb_misses", "itlb_misses" }; + //long long values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0, 0, 0, 0 }; + //long long accum_values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0, 0, 0, 0 }; + //int events[NUM_PAPI_COUNTERS] = { PAPI_TOT_INS, PAPI_TOT_CYC, PAPI_L1_ICM }; + //char* eventnames[NUM_PAPI_COUNTERS]= { "instructions", "cycles", "l1i_misses"}; + //assert(PAPI_event_name_to_code("PERF_COUNT_HW_STALLED_CYCLES_FRONTEND",&(events[2])) == PAPI_OK); + int events[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + //assert(PAPI_event_name_to_code("ICACHE:IFETCH_STALL",&(events[2])) == PAPI_OK); + //assert(PAPI_event_name_to_code("MACHINE_CLEARS:e",&(events[3])) == PAPI_OK); + char* eventnames[NUM_PAPI_COUNTERS]= { "instructions", "cycles", "ifetch_stall", "store_misses" }; + assert(PAPI_event_name_to_code("ix86arch::INSTRUCTION_RETIRED",&(events[0])) == PAPI_OK); + assert(PAPI_event_name_to_code("UNHALTED_REFERENCE_CYCLES",&(events[1])) == PAPI_OK); + assert(PAPI_event_name_to_code("ICACHE:IFETCH_STALL", &(events[2])) == PAPI_OK); + assert(PAPI_event_name_to_code("perf::L1-DCACHE-STORE-MISSES", &(events[3])) == PAPI_OK); + long long values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + long long accum_values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + +#endif +#define BATCH_SIZE 100000 + bool use_old_read_testcase = true; + unsigned chunk_size = 100; + std::ifstream ifptr; + FILE* fptr = 0; + if(use_old_read_testcase) + { + fptr = fopen(filename,"r"); + if(fptr == 0) + cerr << "Could not open file "< tc_vector; + tc_vector.clear(); + vector results_vec; + results_vec.clear(); + vector baseline_results; + baseline_results.clear(); + + bool all_ok = true; + uint64_t total_time = 0; + uint64_t baseline_time = 0; + unsigned total_count = 0; + unsigned num_testcases = 0; + //unsigned curr_batch_size = rand()%BATCH_SIZE + 4; //min batch size + unsigned curr_batch_size = BATCH_SIZE; + + testcase tc_in; + int break_value = 0; + while(1) + { + break_value = use_old_read_testcase ? read_testcase(&tc_in, fptr) : + read_mod_testcase(ifptr, &tc_in, true); + tc_vector.push_back(tc_in); + if(break_value >= 0) + ++num_testcases; + if(num_testcases == curr_batch_size || (break_value < 0 && num_testcases > 0)) + { + results_vec.resize(tc_vector.size()); + baseline_results.resize(tc_vector.size()); + + get_time(); +#ifdef USE_PAPI + assert(PAPI_start_counters(events, NUM_PAPI_COUNTERS) == PAPI_OK); +#endif +#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) + for(unsigned i=0;i(&tc); + double result = 0; + if (result_avxf < MIN_ACCEPTED) { + double result_avxd = compute_full_prob(&tc); + result = log10(result_avxd) - log10(ldexp(1.0, 1020.0)); + } + else + result = (double)(log10f(result_avxf) - log10f(ldexpf(1.f, 120.f))); + baseline_results[i] = result; + } + baseline_time += get_time(); + for(unsigned i=0;i 1e-5 && rel_error > 1e-5) + { + cout << "Line "< Date: Wed, 5 Feb 2014 17:09:57 -0800 Subject: [PATCH 31/72] 1. Whew, finally debugged the source of performance issues with PairHMM JNI. See copied text from email below. 2. This commit contains all the code used in profiling, detecting FP exceptions, dumping intermediate results. All flagged off using ifdefs, but it's there. --------------Text from email As we discussed before, it's the denormal numbers that are causing the slowdown - the core executes some microcode uops (called FP assists) when denormal numbers are detected for FP operations (even un-vectorized code). The C++ compiler by default enables flush to zero (FTZ) - when set, the hardware simply converts denormal numbers to 0. The Java binary (executable provided by Oracle, not the native library) seems to be compiled without FTZ (sensible choice, they want to be conservative). Hence, the JNI invocation sees a large slowdown. Disabling FTZ in C++ slows down the C++ sandbox performance to the JNI version (fortunately, the reverse also holds :)). Not sure how to show the overhead for these FP assists easily - measured a couple of counters. FP_ASSISTS:ANY - shows number of uops executed as part of the FP assists. When FTZ is enabled, this is 0 (both C++ and JNI), when FTZ is disabled this value is around 203540557 (both C++ and JNI) IDQ:MS_UOPS_CYCLES - shows the number of cycles the decoder was issuing uops when the microcode sequencing engine was busy. When FTZ is enabled, this is around 1.77M cycles (both C++ and JNI), when FTZ is disabled this value is around 4.31B cycles (both C++ and JNI). This number is still small with respect to total cycles (~40B), but it only reflects the cycles in the decode stage. The total overhead of the microcode assist ops could be larger. As suggested by Mustafa, I compared intermediate values (matrices M,X,Y) and final output of compute_full_prob. The values produced by C++ and Java are identical to the last bit (as long as both use FTZ or no-FTZ). Comparing the outputs of compute_full_prob for the cases no-FTZ and FTZ, there are differences for very small values (denormal numbers). Examples: Diff values 1.952970E-33 1.952967E-33 Diff values 1.135071E-32 1.135070E-32 Diff values 1.135071E-32 1.135070E-32 Diff values 1.135071E-32 1.135070E-32 For this test case (low coverage NA12878), all these values would be recomputed using the double precision version. Enabling FTZ should be fine. -------------------End text from email --- .../c++/VectorPairHMM/LoadTimeInitializer.cc | 4 ++ public/c++/VectorPairHMM/Makefile | 7 +- public/c++/VectorPairHMM/baseline.cc | 21 +++--- public/c++/VectorPairHMM/headers.h | 11 ++- public/c++/VectorPairHMM/utils.cc | 70 +++++++++++++++---- public/c++/VectorPairHMM/utils.h | 1 + 6 files changed, 87 insertions(+), 27 deletions(-) diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/c++/VectorPairHMM/LoadTimeInitializer.cc index fb640ef88..9b56fa9ad 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.cc +++ b/public/c++/VectorPairHMM/LoadTimeInitializer.cc @@ -7,6 +7,10 @@ LoadTimeInitializer g_load_time_initializer; LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loaded { ConvertChar::init(); +#ifndef DISABLE_FTZ + //Very important to get good performance - enable FTZ, converts denormals to 0 + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); +#endif m_sumNumReads = 0; m_sumSquareNumReads = 0; m_sumNumHaplotypes = 0; diff --git a/public/c++/VectorPairHMM/Makefile b/public/c++/VectorPairHMM/Makefile index 35b38e291..0b564d033 100644 --- a/public/c++/VectorPairHMM/Makefile +++ b/public/c++/VectorPairHMM/Makefile @@ -13,15 +13,18 @@ CXX=icc LDFLAGS=-lm -lrt $(OMPLDFLAGS) -#USE_PAPI=1 PAPI_DIR=/home/karthikg/softwares/papi-5.3.0 ifdef USE_PAPI ifeq ($(USE_PAPI),1) - COMMON_COMPILATION_FLAGS+=-I$(PAPI_DIR)/include + COMMON_COMPILATION_FLAGS+=-I$(PAPI_DIR)/include -DUSE_PAPI LDFLAGS+=-L$(PAPI_DIR)/lib -lpapi endif endif +ifdef DISABLE_FTZ + COMMON_COMPILATION_FLAGS+=-DDISABLE_FTZ -no-ftz +endif + BIN=libVectorLoglessPairHMM.so pairhmm-template-main checker #BIN=checker diff --git a/public/c++/VectorPairHMM/baseline.cc b/public/c++/VectorPairHMM/baseline.cc index d92a21ecf..eb233d5c3 100644 --- a/public/c++/VectorPairHMM/baseline.cc +++ b/public/c++/VectorPairHMM/baseline.cc @@ -1,6 +1,7 @@ #include "headers.h" #include "template.h" -extern uint64_t exceptions_array[128]; +#include "utils.h" + template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) { @@ -66,18 +67,22 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) distm = distm/3; - //feclearexcept(FE_ALL_EXCEPT); + //feclearexcept(FE_ALL_EXCEPT); M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); - //M[r][c] = (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); - //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); - //feclearexcept(FE_ALL_EXCEPT); + //feclearexcept(FE_ALL_EXCEPT); X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; - //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); - //feclearexcept(FE_ALL_EXCEPT); + //feclearexcept(FE_ALL_EXCEPT); Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; - //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //CONVERT_AND_PRINT(M[r][c]); + //CONVERT_AND_PRINT(X[r][c]); + //CONVERT_AND_PRINT(Y[r][c]); + } for (c = 0; c < COLS; c++) diff --git a/public/c++/VectorPairHMM/headers.h b/public/c++/VectorPairHMM/headers.h index feb687660..9e4600136 100644 --- a/public/c++/VectorPairHMM/headers.h +++ b/public/c++/VectorPairHMM/headers.h @@ -27,14 +27,19 @@ #include #include -#define STORE_FP_EXCEPTIONS(flagp, exceptions_array) \ - fegetexceptflag(&flagp, FE_OVERFLOW | FE_UNDERFLOW | FE_DIVBYZERO | FE_INVALID | __FE_DENORM); \ +extern uint64_t exceptions_array[128]; +extern FILE* g_debug_fptr; +#define STORE_FP_EXCEPTIONS(flagp, exceptions_array) \ + fegetexceptflag(&flagp, FE_ALL_EXCEPT | __FE_DENORM); \ exceptions_array[FE_INVALID] += ((flagp & FE_INVALID)); \ exceptions_array[__FE_DENORM] += ((flagp & __FE_DENORM) >> 1); \ exceptions_array[FE_DIVBYZERO] += ((flagp & FE_DIVBYZERO) >> 2); \ exceptions_array[FE_OVERFLOW] += ((flagp & FE_OVERFLOW) >> 3); \ exceptions_array[FE_UNDERFLOW] += ((flagp & FE_UNDERFLOW) >> 4); \ - feclearexcept(FE_ALL_EXCEPT); + feclearexcept(FE_ALL_EXCEPT | __FE_DENORM); +#define CONVERT_AND_PRINT(X) \ + g_converter.f = (X); \ + fwrite(&(g_converter.i),4,1,g_debug_fptr); \ #endif diff --git a/public/c++/VectorPairHMM/utils.cc b/public/c++/VectorPairHMM/utils.cc index 4625537bd..419b2d4ce 100644 --- a/public/c++/VectorPairHMM/utils.cc +++ b/public/c++/VectorPairHMM/utils.cc @@ -45,7 +45,7 @@ uint64_t get_machine_capabilities() void initialize_function_pointers(uint64_t mask) { - //mask = 0; + //mask = 0ull; if(is_avx_supported() && (mask & (1<< AVX_CUSTOM_IDX))) { cout << "Using AVX accelerated implementation of PairHMM\n"; @@ -287,16 +287,23 @@ uint64_t diff_time(struct timespec& prev_time) } //#define USE_PAPI +//#define COUNT_EXCEPTIONS +//#define CHECK_RESULTS +#define CHECK_UNDERFLOW 1 #ifdef USE_PAPI #include "papi.h" #define NUM_PAPI_COUNTERS 4 #endif +IF_32 g_converter; +FILE* g_debug_fptr = 0; uint64_t exceptions_array[128]; void do_compute(char* filename) { - memset(exceptions_array, 0, 128*sizeof(uint64_t)); - _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + //g_debug_fptr = fopen("/mnt/app_hdd/scratch/karthikg/dump.log","w"); + //assert(g_debug_fptr); + for(unsigned i=0;i<128;++i) + exceptions_array[i] = 0ull; //assert(feenableexcept(FE_DIVBYZERO | FE_INVALID) >= 0); #ifdef USE_PAPI PAPI_num_counters(); @@ -310,11 +317,11 @@ void do_compute(char* filename) int events[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; //assert(PAPI_event_name_to_code("ICACHE:IFETCH_STALL",&(events[2])) == PAPI_OK); //assert(PAPI_event_name_to_code("MACHINE_CLEARS:e",&(events[3])) == PAPI_OK); - char* eventnames[NUM_PAPI_COUNTERS]= { "instructions", "cycles", "ifetch_stall", "store_misses" }; + char* eventnames[NUM_PAPI_COUNTERS]= { "instructions", "cycles", "fp_assists", "idq_ms_cycles" }; assert(PAPI_event_name_to_code("ix86arch::INSTRUCTION_RETIRED",&(events[0])) == PAPI_OK); assert(PAPI_event_name_to_code("UNHALTED_REFERENCE_CYCLES",&(events[1])) == PAPI_OK); - assert(PAPI_event_name_to_code("ICACHE:IFETCH_STALL", &(events[2])) == PAPI_OK); - assert(PAPI_event_name_to_code("perf::L1-DCACHE-STORE-MISSES", &(events[3])) == PAPI_OK); + assert(PAPI_event_name_to_code("FP_ASSIST:ANY", &(events[2])) == PAPI_OK); + assert(PAPI_event_name_to_code("IDQ:MS_UOPS_CYCLES", &(events[3])) == PAPI_OK); long long values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; long long accum_values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; @@ -353,6 +360,9 @@ void do_compute(char* filename) testcase tc_in; int break_value = 0; + uint64_t fp_single_exceptions_reexecute = 0; + uint64_t fp_single_exceptions_continue = 0; + uint64_t num_double_executions = 0; while(1) { break_value = use_old_read_testcase ? read_testcase(&tc_in, fptr) : @@ -373,13 +383,38 @@ void do_compute(char* filename) for(unsigned i=0;i Date: Thu, 6 Feb 2014 09:32:56 -0800 Subject: [PATCH 32/72] 1. Moved break statement in utils.cc to correct position 2. Tested sandbox with regions 3. Lots of profiling code from previous commit exists --- .../c++/VectorPairHMM/LoadTimeInitializer.cc | 3 ++ public/c++/VectorPairHMM/Sandbox.java | 4 +- public/c++/VectorPairHMM/pairhmm-1-base.cc | 7 +-- public/c++/VectorPairHMM/utils.cc | 54 ++++++++++--------- 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/c++/VectorPairHMM/LoadTimeInitializer.cc index 9b56fa9ad..7a4e46161 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.cc +++ b/public/c++/VectorPairHMM/LoadTimeInitializer.cc @@ -10,6 +10,9 @@ LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loa #ifndef DISABLE_FTZ //Very important to get good performance - enable FTZ, converts denormals to 0 _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + cout << "FTZ enabled - may decrease accuracy if denormal numbers encountered\n"; +#else + cout << "FTZ is not set - may slow down performance if denormal numbers encountered\n"; #endif m_sumNumReads = 0; m_sumSquareNumReads = 0; diff --git a/public/c++/VectorPairHMM/Sandbox.java b/public/c++/VectorPairHMM/Sandbox.java index c41a276c2..81a57c0a0 100644 --- a/public/c++/VectorPairHMM/Sandbox.java +++ b/public/c++/VectorPairHMM/Sandbox.java @@ -272,7 +272,7 @@ public class Sandbox { //System.err.println(e); //} Sandbox t = new Sandbox(); - t.doEverythingNative(args[0]); - //t.parseSandboxFile(args[0]); + //t.doEverythingNative(args[0]); + t.parseSandboxFile(args[0]); } } diff --git a/public/c++/VectorPairHMM/pairhmm-1-base.cc b/public/c++/VectorPairHMM/pairhmm-1-base.cc index c7a85e647..2c80f2a2e 100644 --- a/public/c++/VectorPairHMM/pairhmm-1-base.cc +++ b/public/c++/VectorPairHMM/pairhmm-1-base.cc @@ -9,14 +9,14 @@ using namespace std; int main(int argc, char** argv) { -#define BATCH_SIZE 5 +#define BATCH_SIZE 10000 if(argc < 2) { cerr << "Needs path to input file as argument\n"; exit(0); } do_compute(argv[1]); -#if 0 + return 0; bool use_old_read_testcase = false; if(argc >= 3 && string(argv[2]) == "1") use_old_read_testcase = true; @@ -56,7 +56,6 @@ int main(int argc, char** argv) testcase tc_in; int break_value = 0; tc_vector.clear(); - g_load_time_initializer.open_sandbox(); while(1) { break_value = use_old_read_testcase ? read_testcase(&tc_in, fptr) : @@ -144,8 +143,6 @@ int main(int argc, char** argv) fclose(fptr); else ifptr.close(); - g_load_time_initializer.close_sandbox(); -#endif return 0; } diff --git a/public/c++/VectorPairHMM/utils.cc b/public/c++/VectorPairHMM/utils.cc index 419b2d4ce..3d3a17042 100644 --- a/public/c++/VectorPairHMM/utils.cc +++ b/public/c++/VectorPairHMM/utils.cc @@ -79,9 +79,13 @@ int read_testcase(testcase *tc, FILE* ifp) int x, size = 0; ssize_t read; - read = getline(&line, (size_t *) &size, ifp == 0 ? stdin : ifp); + + read = getline(&line, (size_t *) &size, ifp == 0 ? stdin : ifp); if (read == -1) - return -1; + { + free(line); + return -1; + } tc->hap = (char *) malloc(size); @@ -326,7 +330,7 @@ void do_compute(char* filename) long long accum_values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; #endif -#define BATCH_SIZE 100000 +#define BATCH_SIZE 10000 bool use_old_read_testcase = true; unsigned chunk_size = 100; std::ifstream ifptr; @@ -396,25 +400,25 @@ void do_compute(char* filename) #ifdef CHECK_UNDERFLOW if (result_avxf < MIN_ACCEPTED) #else - if(false) + if(false) #endif - { + { #ifdef COUNT_EXCEPTIONS - if(fp_exception) - ++fp_single_exceptions_reexecute; + if(fp_exception) + ++fp_single_exceptions_reexecute; #endif - double result_avxd = g_compute_full_prob_double(&(tc_vector[i]), 0); - result = log10(result_avxd) - log10(ldexp(1.0, 1020.0)); - ++num_double_executions; - } - else - { + double result_avxd = g_compute_full_prob_double(&(tc_vector[i]), 0); + result = log10(result_avxd) - log10(ldexp(1.0, 1020.0)); + ++num_double_executions; + } + else + { #ifdef COUNT_EXCEPTIONS - if(fp_exception) - ++fp_single_exceptions_continue; + if(fp_exception) + ++fp_single_exceptions_continue; #endif - result = (double)(log10f(result_avxf) - log10f(ldexpf(1.f, 120.f))); - } + result = (double)(log10f(result_avxf) - log10f(ldexpf(1.f, 120.f))); + } results_vec[i] = result; } #ifdef USE_PAPI @@ -459,12 +463,12 @@ void do_compute(char* filename) #endif for(unsigned i=0;i Date: Thu, 6 Feb 2014 11:01:33 -0800 Subject: [PATCH 33/72] 1. Enabled FTZ in LoadTimeInitializer.cc 2. Added Sandbox.java for testing 3. Moved compute to utils.cc (inside library) 4. Added flag for disabling FTZ in Makefile --- public/c++/VectorPairHMM/.gitignore | 1 + .../c++/VectorPairHMM/LoadTimeInitializer.cc | 9 + public/c++/VectorPairHMM/Makefile | 15 +- public/c++/VectorPairHMM/Sandbox.cc | 79 +++++ public/c++/VectorPairHMM/Sandbox.h | 71 +++++ public/c++/VectorPairHMM/Sandbox.java | 278 ++++++++++++++++++ .../Sandbox_JNIHaplotypeDataHolderClass.h | 13 + .../Sandbox_JNIReadDataHolderClass.h | 13 + public/c++/VectorPairHMM/pairhmm-1-base.cc | 80 +---- public/c++/VectorPairHMM/utils.cc | 117 +++++++- public/c++/VectorPairHMM/utils.h | 1 + 11 files changed, 589 insertions(+), 88 deletions(-) create mode 100644 public/c++/VectorPairHMM/Sandbox.cc create mode 100644 public/c++/VectorPairHMM/Sandbox.h create mode 100644 public/c++/VectorPairHMM/Sandbox.java create mode 100644 public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h create mode 100644 public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h diff --git a/public/c++/VectorPairHMM/.gitignore b/public/c++/VectorPairHMM/.gitignore index ce903a6fa..b034f9461 100644 --- a/public/c++/VectorPairHMM/.gitignore +++ b/public/c++/VectorPairHMM/.gitignore @@ -6,6 +6,7 @@ tests hmm_Mohammad pairhmm-template-main *.swp +*.class checker reformat subdir_checkout.sh diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/c++/VectorPairHMM/LoadTimeInitializer.cc index 4188ea3fb..4a09542ae 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.cc +++ b/public/c++/VectorPairHMM/LoadTimeInitializer.cc @@ -7,6 +7,15 @@ LoadTimeInitializer g_load_time_initializer; LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loaded { ConvertChar::init(); +#ifndef DISABLE_FTZ + //Very important to get good performance on Intel processors + //Function: enabling FTZ converts denormals to 0 in hardware + //Denormals cause microcode to insert uops into the core causing big slowdown + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + cout << "FTZ enabled - may decrease accuracy if denormal numbers encountered\n"; +#else + cout << "FTZ is not set - may slow down performance if denormal numbers encountered\n"; +#endif m_sumNumReads = 0; m_sumSquareNumReads = 0; m_sumNumHaplotypes = 0; diff --git a/public/c++/VectorPairHMM/Makefile b/public/c++/VectorPairHMM/Makefile index 6eefc3393..57be10154 100644 --- a/public/c++/VectorPairHMM/Makefile +++ b/public/c++/VectorPairHMM/Makefile @@ -12,6 +12,9 @@ CC=icc CXX=icc LDFLAGS=-lm -lrt $(OMPLDFLAGS) +ifdef DISABLE_FTZ + COMMON_COMPILATION_FLAGS+=-DDISABLE_FTZ -no-ftz +endif BIN=libVectorLoglessPairHMM.so pairhmm-template-main checker #BIN=checker @@ -22,14 +25,14 @@ DF=$(DEPDIR)/$(*).d #Common across libJNI and sandbox COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc sse_function_instantiations.cc LoadTimeInitializer.cc #Part of libJNI -LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc $(COMMON_SOURCES) +LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc Sandbox.cc $(COMMON_SOURCES) SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc LIBOBJECTS=$(LIBSOURCES:.cc=.o) COMMON_OBJECTS=$(COMMON_SOURCES:.cc=.o) #No vectorization for these files -NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc LoadTimeInitializer.cc +NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc LoadTimeInitializer.cc Sandbox.cc #Use -xAVX for these files AVX_SOURCES=avx_function_instantiations.cc #Use -xSSE4.2 for these files @@ -43,7 +46,7 @@ $(AVX_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xAVX $(SSE_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xSSE4.2 OBJECTS=$(NO_VECTOR_OBJECTS) $(AVX_OBJECTS) $(SSE_OBJECTS) -all: $(BIN) +all: $(BIN) Sandbox.class -include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) @@ -54,13 +57,15 @@ pairhmm-template-main: pairhmm-template-main.o $(COMMON_OBJECTS) $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) libVectorLoglessPairHMM.so: $(LIBOBJECTS) - $(CXX) $(OMPLFLAGS) -shared -o $@ $(LIBOBJECTS) ${LDFLAGS} -Wl,-Bstatic -limf -lsvml -lirng -Wl,-Bdynamic #-lintlc + $(CXX) $(OMPLFLAGS) -shared -static-intel -o $@ $(LIBOBJECTS) ${LDFLAGS} $(OBJECTS): %.o: %.cc @mkdir -p $(DEPDIR) $(CXX) -c -MMD -MF $(DF) $(CXXFLAGS) $(OUTPUT_OPTION) $< +Sandbox.class: Sandbox.java + javac Sandbox.java clean: - rm -rf $(BIN) *.o $(DEPDIR) + rm -rf $(BIN) *.o $(DEPDIR) *.class diff --git a/public/c++/VectorPairHMM/Sandbox.cc b/public/c++/VectorPairHMM/Sandbox.cc new file mode 100644 index 000000000..7c10e0620 --- /dev/null +++ b/public/c++/VectorPairHMM/Sandbox.cc @@ -0,0 +1,79 @@ +#include "Sandbox.h" +#include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" +#include "utils.h" +#include "jni_common.h" +/* + * Class: Sandbox + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_Sandbox_jniGetMachineType + (JNIEnv * env, jobject thisObj) +{ + return 0; +} + +/* + * Class: Sandbox + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeClassFieldsAndMachineMask + (JNIEnv* env, jobject thisObject, jclass readDataHolderClass, jclass haplotypeDataHolderClass, jlong mask) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask(env, thisObject, readDataHolderClass, + haplotypeDataHolderClass, mask); +} + +/* + * Class: Sandbox + * Method: jniInitializeHaplotypes + * Signature: (I[LSandbox/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeHaplotypes + (JNIEnv * env, jobject thisObject, jint numHaplotypes, jobjectArray haplotypeDataArray) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes(env, thisObject, numHaplotypes, haplotypeDataArray); +} + +/* + * Class: Sandbox + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniFinalizeRegion + (JNIEnv * env, jobject thisObject) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniFinalizeRegion(env, thisObject); +} + + +/* + * Class: Sandbox + * Method: jniComputeLikelihoods + * Signature: (II[LSandbox/JNIReadDataHolderClass;[LSandbox/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniComputeLikelihoods + (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, + jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods(env, thisObject, + numReads, numHaplotypes, readDataArray, haplotypeDataArray, likelihoodArray, maxNumThreadsToUse); +} +/* + * Class: Sandbox + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniClose + (JNIEnv* env, jobject thisObject) +{ Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose(env, thisObject); } + +JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative + (JNIEnv* env, jobject thisObject, jstring fileNameString) +{ + const char* fileName = env->GetStringUTFChars(fileNameString, 0); + do_compute((char*)fileName); + env->ReleaseStringUTFChars(fileNameString, fileName); +} + diff --git a/public/c++/VectorPairHMM/Sandbox.h b/public/c++/VectorPairHMM/Sandbox.h new file mode 100644 index 000000000..4ac1ea24c --- /dev/null +++ b/public/c++/VectorPairHMM/Sandbox.h @@ -0,0 +1,71 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox */ + +#ifndef _Included_Sandbox +#define _Included_Sandbox +#ifdef __cplusplus +extern "C" { +#endif +#undef Sandbox_enableAll +#define Sandbox_enableAll -1LL +/* + * Class: Sandbox + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_Sandbox_jniGetMachineType + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeClassFieldsAndMachineMask + (JNIEnv *, jobject, jclass, jclass, jlong); + +/* + * Class: Sandbox + * Method: jniInitializeHaplotypes + * Signature: (I[LSandbox/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: Sandbox + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniFinalizeRegion + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniComputeLikelihoods + * Signature: (II[LSandbox/JNIReadDataHolderClass;[LSandbox/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); + +/* + * Class: Sandbox + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniClose + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: doEverythingNative + * Signature: ([B)V + */ +JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative + (JNIEnv *, jobject, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/c++/VectorPairHMM/Sandbox.java b/public/c++/VectorPairHMM/Sandbox.java new file mode 100644 index 000000000..81a57c0a0 --- /dev/null +++ b/public/c++/VectorPairHMM/Sandbox.java @@ -0,0 +1,278 @@ +import java.util.List; +import java.util.LinkedList; +import java.util.Map; +import java.util.HashMap; +import java.io.File; +import java.util.Scanner; +import java.io.IOException; +import java.io.FileNotFoundException; +import java.io.InputStreamReader; + +public class Sandbox { + + private long setupTime = 0; + private long computeTime = 0; + //Used to copy references to byteArrays to JNI from reads + protected class JNIReadDataHolderClass { + public byte[] readBases = null; + public byte[] readQuals = null; + public byte[] insertionGOP = null; + public byte[] deletionGOP = null; + public byte[] overallGCP = null; + } + + //Used to copy references to byteArrays to JNI from haplotypes + protected class JNIHaplotypeDataHolderClass { + public byte[] haplotypeBases = null; + } + + /** + * Return 64-bit mask representing machine capabilities + * Bit 0 is LSB, bit 63 MSB + * Bit 0 represents sse4.2 availability + * Bit 1 represents AVX availability + */ + public native long jniGetMachineType(); + public static final long enableAll = 0xFFFFFFFFFFFFFFFFl; + + + /** + * Function to initialize the fields of JNIReadDataHolderClass and JNIHaplotypeDataHolderClass from JVM. + * C++ codegets FieldIDs for these classes once and re-uses these IDs for the remainder of the program. Field IDs do not + * change per JVM session + * @param readDataHolderClass class type of JNIReadDataHolderClass + * @param haplotypeDataHolderClass class type of JNIHaplotypeDataHolderClass + * @param mask mask is a 64 bit integer identical to the one received from jniGetMachineType(). Users can disable usage of some hardware features by zeroing some bits in the mask + * */ + private native void jniInitializeClassFieldsAndMachineMask(Class readDataHolderClass, Class haplotypeDataHolderClass, long mask); + + private static Boolean isVectorLoglessPairHMMLibraryLoaded = false; + //The constructor is called only once inside PairHMMLikelihoodCalculationEngine + public Sandbox() { + synchronized(isVectorLoglessPairHMMLibraryLoaded) { + //Load the library and initialize the FieldIDs + if(!isVectorLoglessPairHMMLibraryLoaded) { + System.loadLibrary("VectorLoglessPairHMM"); + isVectorLoglessPairHMMLibraryLoaded = true; + jniInitializeClassFieldsAndMachineMask(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once + } + } + } + + private native void jniInitializeHaplotypes(final int numHaplotypes, JNIHaplotypeDataHolderClass[] haplotypeDataArray); + + //Used to transfer data to JNI + //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region + public void initialize(final List haplotypes) { + int numHaplotypes = haplotypes.size(); + JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; + int idx = 0; + for(final JNIHaplotypeDataHolderClass currHaplotype : haplotypes) + { + haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); + haplotypeDataArray[idx].haplotypeBases = currHaplotype.haplotypeBases; + ++idx; + } + jniInitializeHaplotypes(numHaplotypes, haplotypeDataArray); + } + /** + * Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not + * accessing Java memory directly, still important to release memory from C++ + */ + private native void jniFinalizeRegion(); + + + public void finalizeRegion() + { + jniFinalizeRegion(); + } + + /** + * Real compute kernel + */ + private native void jniComputeLikelihoods(int numReads, int numHaplotypes, JNIReadDataHolderClass[] readDataArray, + JNIHaplotypeDataHolderClass[] haplotypeDataArray, double[] likelihoodArray, int maxNumThreadsToUse); + + public void computeLikelihoods(final List reads, final List haplotypes) { + //System.out.println("Region : "+reads.size()+" x "+haplotypes.size()); + long startTime = System.nanoTime(); + int readListSize = reads.size(); + int numHaplotypes = haplotypes.size(); + int numTestcases = readListSize*numHaplotypes; + JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; + int idx = 0; + for(JNIReadDataHolderClass read : reads) + { + readDataArray[idx] = new JNIReadDataHolderClass(); + readDataArray[idx].readBases = read.readBases; + readDataArray[idx].readQuals = read.readQuals; + readDataArray[idx].insertionGOP = read.insertionGOP; + readDataArray[idx].deletionGOP = read.deletionGOP; + readDataArray[idx].overallGCP = read.overallGCP; + ++idx; + } + + double[] mLikelihoodArray = new double[readListSize*numHaplotypes]; //to store results + setupTime += (System.nanoTime() - startTime); + //for(reads) + // for(haplotypes) + // compute_full_prob() + jniComputeLikelihoods(readListSize, numHaplotypes, readDataArray, null, mLikelihoodArray, 12); + + computeTime += (System.nanoTime() - startTime); + } + + /** + * Print final profiling information from native code + */ + public native void jniClose(); + public void close() + { + System.out.println("Time spent in setup for JNI call : "+(setupTime*1e-9)+" compute time : "+(computeTime*1e-9)); + jniClose(); + } + + public void parseSandboxFile(String filename) + { + File file = new File(filename); + Scanner input = null; + try + { + input = new Scanner(file); + } + catch(FileNotFoundException e) + { + System.err.println("File "+filename+" cannot be found/read"); + return; + } + int idx = 0; + int numReads = 0; + int numHaplotypes = 0; + int readIdx = 0, testCaseIdx = 0, haplotypeIdx = 0; + LinkedList haplotypeList = new LinkedList(); + LinkedList readList = new LinkedList(); + + byte[][] byteArray = new byte[6][]; + boolean firstLine = true; + String[] currTokens = new String[8]; + while(input.hasNextLine()) + { + String line = input.nextLine(); + Scanner lineScanner = new Scanner(line); + idx = 0; + while(lineScanner.hasNext()) + currTokens[idx++] = lineScanner.next(); + if(idx == 0) + break; + assert(idx >= 6); + //start of new region + if(idx == 8) + { + if(!firstLine) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + try + { + numReads = Integer.parseInt(currTokens[6]); + } + catch(NumberFormatException e) + { + numReads = 1; + } + try + { + numHaplotypes = Integer.parseInt(currTokens[7]); + } + catch(NumberFormatException e) + { + numHaplotypes = 1; + } + haplotypeIdx = readIdx = testCaseIdx = 0; + readList.clear(); + haplotypeList.clear(); + } + if(haplotypeIdx < numHaplotypes) + { + JNIHaplotypeDataHolderClass X = new JNIHaplotypeDataHolderClass(); + X.haplotypeBases = currTokens[0].getBytes(); + haplotypeList.add(X); + } + if(testCaseIdx%numHaplotypes == 0) + { + JNIReadDataHolderClass X = new JNIReadDataHolderClass(); + X.readBases = currTokens[1].getBytes(); + for(int i=2;i<6;++i) + { + byteArray[i] = currTokens[i].getBytes(); + for(int j=0;j 0 && readList.size() > 0) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + + close(); + input.close(); + } + + private native void doEverythingNative(String filename); + + public static void main(String[] args) + { + if(args.length <= 0) + { + System.err.println("Needs 1 argument - "); + System.exit(-1); + } + //// Get runtime + //java.lang.Runtime rt = java.lang.Runtime.getRuntime(); + //// Start a new process: UNIX command ls + //String cmd = "/home/karthikg/broad/gsa-unstable/public/c++/VectorPairHMM/checker "+args[0]; + //try + //{ + //System.out.println(cmd); + //java.lang.Process p = rt.exec(cmd); + //try + //{ + //p.waitFor(); + //java.io.InputStream is = p.getInputStream(); + //java.io.BufferedReader reader = new java.io.BufferedReader(new InputStreamReader(is)); + //// And print each line + //String s = null; + //while ((s = reader.readLine()) != null) { + //System.out.println(s); + //} + //is.close(); + //} + //catch(InterruptedException e) + //{ + //System.err.println(e); + //} + //} + //catch(IOException e) + //{ + //System.err.println(e); + //} + Sandbox t = new Sandbox(); + //t.doEverythingNative(args[0]); + t.parseSandboxFile(args[0]); + } +} diff --git a/public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h b/public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h new file mode 100644 index 000000000..7f78f0178 --- /dev/null +++ b/public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIHaplotypeDataHolderClass */ + +#ifndef _Included_Sandbox_JNIHaplotypeDataHolderClass +#define _Included_Sandbox_JNIHaplotypeDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h b/public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h new file mode 100644 index 000000000..a9312ff3b --- /dev/null +++ b/public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIReadDataHolderClass */ + +#ifndef _Included_Sandbox_JNIReadDataHolderClass +#define _Included_Sandbox_JNIReadDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/c++/VectorPairHMM/pairhmm-1-base.cc b/public/c++/VectorPairHMM/pairhmm-1-base.cc index aaf45848b..6cfd522b8 100644 --- a/public/c++/VectorPairHMM/pairhmm-1-base.cc +++ b/public/c++/VectorPairHMM/pairhmm-1-base.cc @@ -8,12 +8,6 @@ using namespace std; - -#define BATCH_SIZE 10000 -#define RUN_HYBRID - - - int main(int argc, char** argv) { if(argc < 2) @@ -24,82 +18,12 @@ int main(int argc, char** argv) bool use_old_read_testcase = false; if(argc >= 3 && string(argv[2]) == "1") use_old_read_testcase = true; - unsigned chunk_size = 100; + unsigned chunk_size = 10000; if(argc >= 4) chunk_size = strtol(argv[3],0,10); - std::ifstream ifptr; - FILE* fptr = 0; - if(use_old_read_testcase) - { - fptr = fopen(argv[1],"r"); - assert(fptr); - } - else - { - ifptr.open(argv[1]); - assert(ifptr.is_open()); - } + do_compute(argv[1], use_old_read_testcase, chunk_size); - vector tc_vector; - tc_vector.clear(); - testcase tc; - uint64_t total_time = 0; - while(1) - { - int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); - if(break_value >= 0) - tc_vector.push_back(tc); - if(tc_vector.size() == BATCH_SIZE || (break_value < 0 && tc_vector.size() > 0)) - { - vector results_vec; - results_vec.clear(); - results_vec.resize(tc_vector.size()); - get_time(); -#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) - for(unsigned i=0;i(&tc); - baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); - double abs_error = fabs(baseline_result-results_vec[i]); - double rel_error = (baseline_result != 0) ? fabs(abs_error/baseline_result) : 0; - if(abs_error > 1e-5 && rel_error > 1e-5) - cout << std::scientific << baseline_result << " "<haplen = strlen(tc->hap); tc->rslen = strlen(tc->rs); //assert(tc->rslen < MROWS); - tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); - tc->irs = (int *) malloc(tc->rslen*sizeof(int)); + //tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); + //tc->irs = (int *) malloc(tc->rslen*sizeof(int)); tc->q = (char *) malloc(sizeof(char) * tc->rslen); tc->i = (char *) malloc(sizeof(char) * tc->rslen); @@ -115,10 +115,10 @@ int read_testcase(testcase *tc, FILE* ifp) tc->i[x] = _i; tc->d[x] = _d; tc->c[x] = _c; - tc->irs[x] = tc->rs[x]; + //tc->irs[x] = tc->rs[x]; } - for (x = 0; x < tc->haplen; x++) - tc->ihap[x] = tc->hap[x]; + //for (x = 0; x < tc->haplen; x++) + //tc->ihap[x] = tc->hap[x]; free(q); @@ -270,3 +270,110 @@ uint64_t get_time(struct timespec* store_struct) start_time = *ptr; return diff_time; } + +//#define CHECK_VALUES 1 +#define BATCH_SIZE 10000 +#define RUN_HYBRID +void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size) +{ + FILE* fptr = 0; + ifstream ifptr; + if(use_old_read_testcase) + { + fptr = fopen(filename,"r"); + assert(fptr); + } + else + { + ifptr.open(filename); + assert(ifptr.is_open()); + } + vector tc_vector; + tc_vector.clear(); + testcase tc; + uint64_t vector_compute_time = 0; + uint64_t baseline_compute_time = 0; + uint64_t num_double_calls = 0; + bool all_ok = true; +#ifndef CHECK_VALUES + all_ok = false; +#endif + while(1) + { + int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); + if(break_value >= 0) + tc_vector.push_back(tc); + if(tc_vector.size() == BATCH_SIZE || (break_value < 0 && tc_vector.size() > 0)) + { + vector results_vec; + vector baseline_results_vec; + results_vec.clear(); + baseline_results_vec.clear(); + results_vec.resize(tc_vector.size()); + baseline_results_vec.resize(tc_vector.size()); + get_time(); +#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) + for(unsigned i=0;i(&tc); + baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); + baseline_results_vec[i] = baseline_result; + } + baseline_compute_time += get_time(); + for(unsigned i=0;i 1e-5 && rel_error > 1e-5) + { + cout << std::scientific << baseline_result << " "< Date: Thu, 6 Feb 2014 14:35:32 -0800 Subject: [PATCH 34/72] 1. Split main JNI function into initializeTestcases, compute_testcases and releaseReads 2. FTZ enabled 3. Cleaner profiling code --- .../c++/VectorPairHMM/LoadTimeInitializer.cc | 111 +++++++---- .../c++/VectorPairHMM/LoadTimeInitializer.h | 37 ++-- public/c++/VectorPairHMM/baseline.cc | 2 +- public/c++/VectorPairHMM/headers.h | 1 + public/c++/VectorPairHMM/jni_common.h | 4 +- ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 176 +++++++++--------- public/c++/VectorPairHMM/pairhmm-1-base.cc | 2 - public/c++/VectorPairHMM/utils.cc | 26 +-- public/c++/VectorPairHMM/utils.h | 2 +- 9 files changed, 211 insertions(+), 150 deletions(-) diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/c++/VectorPairHMM/LoadTimeInitializer.cc index b8ff2ddf8..fd2d1df09 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.cc +++ b/public/c++/VectorPairHMM/LoadTimeInitializer.cc @@ -1,6 +1,19 @@ #include "LoadTimeInitializer.h" #include "utils.h" using namespace std; +char* LoadTimeInitializerStatsNames[] = +{ + "num_regions", + "num_reads", + "num_haplotypes", + "num_testcases", + "num_double_invocations", + "haplotype_length", + "readlength", + "product_read_length_haplotype_length", + "dummy" +}; + LoadTimeInitializer g_load_time_initializer; @@ -16,25 +29,23 @@ LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loa #else cout << "FTZ is not set - may slow down performance if denormal numbers encountered\n"; #endif - m_sumNumReads = 0; - m_sumSquareNumReads = 0; - m_sumNumHaplotypes = 0; - m_sumSquareNumHaplotypes = 0; - m_sumNumTestcases = 0; - m_sumNumDoubleTestcases = 0; - m_sumSquareNumTestcases = 0; - m_sumReadLengths = 0; - m_sumHaplotypeLengths = 0; - m_sumProductReadLengthHaplotypeLength = 0; - m_sumSquareProductReadLengthHaplotypeLength = 0; - m_maxNumTestcases = 0; - m_num_invocations = 0; - + //Profiling: times for compute and transfer (either bytes copied or pointers copied) m_compute_time = 0; m_data_transfer_time = 0; m_bytes_copied = 0; + //Initialize profiling counters + for(unsigned i=0;i C++) "<open(filename.c_str(), to_append ? ios::app : ios::out); assert(fptr->is_open()); } @@ -121,3 +157,12 @@ void LoadTimeInitializer::dump_sandbox(testcase& tc, unsigned tc_idx, unsigned n dumpFptr << " "<< numReads << " "< #include "template.h" + +enum LoadTimeInitializerStatsEnum +{ + NUM_REGIONS_IDX=0, + NUM_READS_IDX, + NUM_HAPLOTYPES_IDX, + NUM_TESTCASES_IDX, + NUM_DOUBLE_INVOCATIONS_IDX, + HAPLOTYPE_LENGTH_IDX, + READ_LENGTH_IDX, + PRODUCT_READ_LENGTH_HAPLOTYPE_LENGTH_IDX, + TOTAL_NUMBER_STATS +}; +extern char* LoadTimeInitializerStatsNames[]; + class LoadTimeInitializer { public: @@ -21,20 +36,8 @@ class LoadTimeInitializer jfieldID m_deletionGOPFID; jfieldID m_overallGCPFID; jfieldID m_haplotypeBasesFID; - //used to compute avg, variance of #testcases - double m_sumNumReads; - double m_sumSquareNumReads; - double m_sumNumHaplotypes; - double m_sumSquareNumHaplotypes; - double m_sumNumTestcases; - double m_sumSquareNumTestcases; - uint64_t m_sumNumDoubleTestcases; - uint64_t m_sumReadLengths; - uint64_t m_sumHaplotypeLengths; - uint64_t m_sumProductReadLengthHaplotypeLength; - double m_sumSquareProductReadLengthHaplotypeLength; - unsigned m_maxNumTestcases; - unsigned m_num_invocations; + //profiling - update stats + void update_stat(LoadTimeInitializerStatsEnum stat_idx, uint64_t value); //timing in nanoseconds uint64_t m_compute_time; uint64_t m_data_transfer_time; @@ -42,7 +45,13 @@ class LoadTimeInitializer uint64_t m_bytes_copied; private: std::map m_filename_to_fptr; + std::set m_written_files_set; std::ofstream m_sandbox_fptr; + //used to compute various stats + uint64_t m_sum_stats[TOTAL_NUMBER_STATS]; + double m_sum_square_stats[TOTAL_NUMBER_STATS]; + uint64_t m_min_stats[TOTAL_NUMBER_STATS]; + uint64_t m_max_stats[TOTAL_NUMBER_STATS]; }; extern LoadTimeInitializer g_load_time_initializer; diff --git a/public/c++/VectorPairHMM/baseline.cc b/public/c++/VectorPairHMM/baseline.cc index eb233d5c3..268f32f00 100644 --- a/public/c++/VectorPairHMM/baseline.cc +++ b/public/c++/VectorPairHMM/baseline.cc @@ -3,7 +3,7 @@ #include "utils.h" template -NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log = NULL) +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) { int r, c; int ROWS = tc->rslen + 1; diff --git a/public/c++/VectorPairHMM/headers.h b/public/c++/VectorPairHMM/headers.h index 9e4600136..48bd4d836 100644 --- a/public/c++/VectorPairHMM/headers.h +++ b/public/c++/VectorPairHMM/headers.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/public/c++/VectorPairHMM/jni_common.h b/public/c++/VectorPairHMM/jni_common.h index 4c63a0411..1cffea1cc 100644 --- a/public/c++/VectorPairHMM/jni_common.h +++ b/public/c++/VectorPairHMM/jni_common.h @@ -5,8 +5,8 @@ /*#define ENABLE_ASSERTIONS 1*/ #define DO_PROFILING 1 /*#define DEBUG 1*/ -//#define DEBUG0_1 1 -//#define DEBUG3 1 +/*#define DEBUG0_1 1*/ +/*#define DEBUG3 1*/ /*#define DUMP_TO_SANDBOX 1*/ diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index c39d2ec3f..0f5219dd4 100644 --- a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc +++ b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -92,45 +92,20 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless g_load_time_initializer.debug_dump("haplotype_bases_jni.txt",to_string((int)haplotypeBasesArray[k]),true); #endif #ifdef DO_PROFILING - g_load_time_initializer.m_sumHaplotypeLengths += haplotypeBasesLength; + g_load_time_initializer.update_stat(HAPLOTYPE_LENGTH_IDX, haplotypeBasesLength); g_load_time_initializer.m_bytes_copied += (is_copy ? haplotypeBasesLength : 0); #endif } } -//JNI function to invoke compute_full_prob_avx -//readDataArray - array of JNIReadDataHolderClass objects which contain the readBases, readQuals etc -//haplotypeDataArray - array of JNIHaplotypeDataHolderClass objects which contain the haplotypeBases -//likelihoodArray - array of doubles to return results back to Java. Memory allocated by Java prior to JNI call -//maxNumThreadsToUse - Max number of threads that OpenMP can use for the HMM computation -JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods - (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, - jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +inline JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeTestcasesVector + (JNIEnv* env, jint numReads, jint numHaplotypes, jobjectArray& readDataArray, + vector > >& readBasesArrayVector, vector& tc_array) { -#ifdef DEBUG0_1 - cout << "JNI numReads "< >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; - jboolean is_copy = JNI_FALSE; - - unsigned numTestCases = numReads*numHaplotypes; - //vector to store results - vector tc_array; - tc_array.clear(); - tc_array.resize(numTestCases); unsigned tc_idx = 0; - //Store arrays for release later - vector > > readBasesArrayVector; - readBasesArrayVector.clear(); - readBasesArrayVector.resize(numReads); -#ifdef DO_PROFILING - start_time = get_time(); -#endif -#ifdef DUMP_TO_SANDBOX - g_load_time_initializer.open_sandbox(); -#endif for(unsigned i=0;i& tc_array, unsigned numTestCases, double* likelihoodDoubleArray, + unsigned maxNumThreadsToUse) +{ +#pragma omp parallel for schedule (dynamic,10000) num_threads(maxNumThreadsToUse) + for(unsigned tc_idx=0;tc_idx > >& readBasesArrayVector) +{ + //Release read arrays first + for(int i=readBasesArrayVector.size()-1;i>=0;--i)//note the order - reverse of GET + { + for(int j=readBasesArrayVector[i].size()-1;j>=0;--j) + RELEASE_BYTE_ARRAY_ELEMENTS(readBasesArrayVector[i][j].first, readBasesArrayVector[i][j].second, JNI_RO_RELEASE_MODE); + readBasesArrayVector[i].clear(); + } + readBasesArrayVector.clear(); +} + +//JNI function to invoke compute_full_prob_avx +//readDataArray - array of JNIReadDataHolderClass objects which contain the readBases, readQuals etc +//haplotypeDataArray - array of JNIHaplotypeDataHolderClass objects which contain the haplotypeBases +//likelihoodArray - array of doubles to return results back to Java. Memory allocated by Java prior to JNI call +//maxNumThreadsToUse - Max number of threads that OpenMP can use for the HMM computation +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods + (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, + jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +{ +#ifdef DEBUG0_1 + cout << "JNI numReads "< tc_array; + tc_array.clear(); + tc_array.resize(numTestCases); + //Store read arrays for release later + vector > > readBasesArrayVector; + readBasesArrayVector.clear(); + readBasesArrayVector.resize(numReads); +#ifdef DUMP_TO_SANDBOX + g_load_time_initializer.open_sandbox(); +#endif +#ifdef DO_PROFILING + get_time(&start_time); +#endif + //Copy byte array references from Java memory into vector of testcase structs + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeTestcasesVector(env, + numReads, numHaplotypes, readDataArray, readBasesArrayVector, tc_array); +#ifdef DO_PROFILING + g_load_time_initializer.m_data_transfer_time += diff_time(start_time); #endif jdouble* likelihoodDoubleArray = (jdouble*)GET_DOUBLE_ARRAY_ELEMENTS(likelihoodArray, &is_copy); @@ -230,65 +270,29 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless #endif #ifdef DO_PROFILING g_load_time_initializer.m_bytes_copied += (is_copy ? numTestCases*sizeof(double) : 0); - struct timespec prev_time; - clock_gettime(CLOCK_REALTIME, &prev_time); + get_time(&start_time); #endif -#pragma omp parallel for schedule (dynamic,10) private(tc_idx) num_threads(maxNumThreadsToUse) - for(tc_idx=0;tc_idx=0;--i)//note the order - reverse of GET - { - for(int j=readBasesArrayVector[i].size()-1;j>=0;--j) - RELEASE_BYTE_ARRAY_ELEMENTS(readBasesArrayVector[i][j].first, readBasesArrayVector[i][j].second, JNI_RO_RELEASE_MODE); - readBasesArrayVector[i].clear(); - } - readBasesArrayVector.clear(); + RELEASE_DOUBLE_ARRAY_ELEMENTS(likelihoodArray, likelihoodDoubleArray, 0); //release mode 0, copy back results to Java memory (if copy made) + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniReleaseReadArrays(env, readBasesArrayVector); #ifdef DO_PROFILING - g_load_time_initializer.m_data_transfer_time += get_time(); + g_load_time_initializer.m_data_transfer_time += diff_time(start_time); + g_load_time_initializer.update_stat(NUM_REGIONS_IDX, 1); + g_load_time_initializer.update_stat(NUM_READS_IDX, numReads); + g_load_time_initializer.update_stat(NUM_HAPLOTYPES_IDX, numHaplotypes); + g_load_time_initializer.update_stat(NUM_TESTCASES_IDX, numTestCases); #endif tc_array.clear(); -#ifdef DO_PROFILING - g_load_time_initializer.m_sumNumReads += numReads; - g_load_time_initializer.m_sumSquareNumReads += numReads*numReads; - g_load_time_initializer.m_sumNumHaplotypes += numHaplotypes; - g_load_time_initializer.m_sumSquareNumHaplotypes += numHaplotypes*numHaplotypes; - g_load_time_initializer.m_sumNumTestcases += numTestCases; - g_load_time_initializer.m_sumSquareNumTestcases += numTestCases*numTestCases; - g_load_time_initializer.m_maxNumTestcases = numTestCases > g_load_time_initializer.m_maxNumTestcases ? numTestCases - : g_load_time_initializer.m_maxNumTestcases; - ++(g_load_time_initializer.m_num_invocations); -#endif -#ifdef DEBUG - g_load_time_initializer.debug_close(); -#endif #ifdef DUMP_TO_SANDBOX g_load_time_initializer.close_sandbox(); #endif diff --git a/public/c++/VectorPairHMM/pairhmm-1-base.cc b/public/c++/VectorPairHMM/pairhmm-1-base.cc index a552aecca..8b686d5ea 100644 --- a/public/c++/VectorPairHMM/pairhmm-1-base.cc +++ b/public/c++/VectorPairHMM/pairhmm-1-base.cc @@ -14,8 +14,6 @@ int main(int argc, char** argv) cerr << "Needs path to input file as argument\n"; exit(0); } - do_compute(argv[1]); - return 0; bool use_old_read_testcase = false; if(argc >= 3 && string(argv[2]) == "1") use_old_read_testcase = true; diff --git a/public/c++/VectorPairHMM/utils.cc b/public/c++/VectorPairHMM/utils.cc index b13c84459..9974c5ace 100644 --- a/public/c++/VectorPairHMM/utils.cc +++ b/public/c++/VectorPairHMM/utils.cc @@ -2,6 +2,7 @@ #include "template.h" #include "utils.h" #include "vector_defs.h" +#include "LoadTimeInitializer.h" uint8_t ConvertChar::conversionTable[255]; float (*g_compute_full_prob_float)(testcase *tc, float* before_last_log) = 0; @@ -271,15 +272,9 @@ double getCurrClk() { return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; } -uint64_t get_time(struct timespec* store_struct) +void get_time(struct timespec* store_struct) { - static struct timespec start_time; - struct timespec curr_time; - struct timespec* ptr = (store_struct == 0) ? &curr_time : store_struct; - clock_gettime(CLOCK_REALTIME, ptr); - uint64_t diff_time = (ptr->tv_sec-start_time.tv_sec)*1000000000+(ptr->tv_nsec-start_time.tv_nsec); - start_time = *ptr; - return diff_time; + clock_gettime(CLOCK_REALTIME, store_struct); } uint64_t diff_time(struct timespec& prev_time) @@ -289,6 +284,7 @@ uint64_t diff_time(struct timespec& prev_time) return (uint64_t)((curr_time.tv_sec-prev_time.tv_sec)*1000000000+(curr_time.tv_nsec-prev_time.tv_nsec)); } +//#define DUMP_COMPUTE_VALUES 1 #define CHECK_VALUES 1 #define BATCH_SIZE 10000 #define RUN_HYBRID @@ -329,7 +325,8 @@ void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size) baseline_results_vec.clear(); results_vec.resize(tc_vector.size()); baseline_results_vec.resize(tc_vector.size()); - get_time(); + struct timespec start_time; + get_time(&start_time); #pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) for(unsigned i=0;i NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); double getCurrClk(); -uint64_t get_time(struct timespec* x=0); +void get_time(struct timespec* x); uint64_t diff_time(struct timespec& prev_time); //bit 0 is sse4.2, bit 1 is AVX From 7815c30df8c618ccaceffb663f42c87ee052ab3a Mon Sep 17 00:00:00 2001 From: mghodrat Date: Thu, 6 Feb 2014 20:13:06 -0800 Subject: [PATCH 35/72] Adding comments to pairhmm-template-kernel --- .../VectorPairHMM/pairhmm-template-kernel.cc | 68 +++++++++++-------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/public/c++/VectorPairHMM/pairhmm-template-kernel.cc b/public/c++/VectorPairHMM/pairhmm-template-kernel.cc index 9f59d7eeb..a2111b6b6 100644 --- a/public/c++/VectorPairHMM/pairhmm-template-kernel.cc +++ b/public/c++/VectorPairHMM/pairhmm-template-kernel.cc @@ -79,6 +79,12 @@ inline void CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (BITMASK_VEC& bitMaskVec.shift_left_1bit() ; } +/* + * This function: + * 1- Intializes probability values p_MM, p_XX, P_YY, p_MX, p_GAPM and pack them into vectors (SSE or AVX) + * 2- Precompute parts of "distm" which only depeneds on a row number and pack it into vector + */ + template void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, SIMD_TYPE *p_MM, SIMD_TYPE *p_GAPM, SIMD_TYPE *p_MX, SIMD_TYPE *p_XX, SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, SIMD_TYPE *distm1D) { NUMBER zero = ctx._(0.0); @@ -126,6 +132,11 @@ template void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECIS } } +/* + * This function handles pre-stripe computation: + * 1- Retrieve probaility vectors from memory + * 2- Initialize M, X, Y vectors with all 0's (for the first stripe) and shifting the last row from previous stripe for the rest + */ template inline void CONCAT(CONCAT(stripeINITIALIZATION,SIMD_ENGINE), PRECISION)( int stripeIdx, Context ctx, testcase *tc, SIMD_TYPE &pGAPM, SIMD_TYPE &pMM, SIMD_TYPE &pMX, SIMD_TYPE &pXX, SIMD_TYPE &pMY, SIMD_TYPE &pYY, @@ -170,34 +181,9 @@ template inline void CONCAT(CONCAT(stripeINITIALIZATION,SIMD_ENGIN M_t_1_y = M_t_1; } -inline SIMD_TYPE CONCAT(CONCAT(computeDISTM,SIMD_ENGINE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, SIMD_TYPE rs, UNION_TYPE rsN, SIMD_TYPE N_packed256, - SIMD_TYPE distm, SIMD_TYPE _1_distm) -{ - UNION_TYPE hapN, rshap; - SIMD_TYPE cond; - IF_32 shiftInHap; - - int *hap_ptr = tc->ihap; - - shiftInHap.i = (d NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL) { int ROWS = tc->rslen + 1; int COLS = tc->haplen + 1; int MAVX_COUNT = (ROWS+AVX_LENGTH-1)/AVX_LENGTH; + /* Probaility arrays */ SIMD_TYPE p_MM [MAVX_COUNT], p_GAPM [MAVX_COUNT], p_MX [MAVX_COUNT]; SIMD_TYPE p_XX [MAVX_COUNT], p_MY [MAVX_COUNT], p_YY [MAVX_COUNT]; + + /* For distm precomputation */ SIMD_TYPE distm1D[MAVX_COUNT]; + + /* Carries the values from each stripe to the next stripe */ NUMBER shiftOutM[ROWS+COLS+AVX_LENGTH], shiftOutX[ROWS+COLS+AVX_LENGTH], shiftOutY[ROWS+COLS+AVX_LENGTH]; + + /* The vector to keep the anti-diagonals of M, X, Y*/ + /* Current: M_t, X_t, Y_t */ + /* Previous: M_t_1, X_t_1, Y_t_1 */ + /* Previous to previous: M_t_2, X_t_2, Y_t_2 */ UNION_TYPE M_t, M_t_1, M_t_2, X_t, X_t_1, X_t_2, Y_t, Y_t_1, Y_t_2, M_t_y, M_t_1_y; + + /* Probality vectors */ SIMD_TYPE pGAPM, pMM, pMX, pXX, pMY, pYY; struct timeval start, end; @@ -247,11 +256,14 @@ template NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRE const int maskBitCnt = MAIN_TYPE_SIZE ; const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function + /* Mask precomputation for distm*/ MASK_TYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(*tc, COLS, numMaskVecs, maskArr) ; char rsArr[AVX_LENGTH] ; MASK_TYPE lastMaskShiftOut[AVX_LENGTH] ; + + /* Precompute initialization for probabilities and shift vector*/ CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(ROWS, COLS, shiftOutM, shiftOutX, shiftOutY, ctx, tc, p_MM, p_GAPM, p_MX, p_XX, p_MY, p_YY, distm1D); From dc44b64ad87d04ae529e419aa5217861584b849c Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Fri, 7 Feb 2014 13:13:59 -0800 Subject: [PATCH 36/72] 1. Added support for building the PairHMM vector library into build.xml. The library is compiled using makefile and copied into the directory: build/java/classes/org/broadinstitute/sting/utils/pairhmm/ 2. Bundled the library into StingUtils.jar. Unpacked and loaded at runtime without the need to set java.library.path Caveats: Platform independence has probably been thrown out of the window. Assumptions: a. make command exists at /usr/bin/make b. rsync command exists at /usr/bin/rsync c. icc is in the PATH of the user --- build.xml | 125 ++++++++++-------- .../utils/pairhmm/VectorLoglessPairHMM.java | 103 ++++++++++++++- 2 files changed, 172 insertions(+), 56 deletions(-) diff --git a/build.xml b/build.xml index 732beb568..2493553fc 100644 --- a/build.xml +++ b/build.xml @@ -64,6 +64,7 @@ + @@ -270,21 +271,21 @@ - + - - + + - - + + @@ -601,14 +602,14 @@ - + @@ -648,9 +649,23 @@ - + + + + + + + + + + + + + + + @@ -682,9 +697,9 @@ - + - + @@ -693,7 +708,7 @@ - + @@ -815,7 +830,7 @@ docletpathref="doclet.classpath" classpathref="external.dependencies" classpath="${java.classes}" - maxmemory="2g" + maxmemory="2g" additionalparam="${gatkdocs.include.hidden.arg} -private -build-timestamp "${build.timestamp}" -absolute-version ${build.version} -quiet"> @@ -974,10 +989,10 @@ - - - - + + + + @@ -990,7 +1005,7 @@ - + @@ -1149,31 +1164,31 @@ - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1220,7 +1235,7 @@ - + @@ -1266,13 +1281,13 @@ - + @@ -1308,7 +1323,7 @@ - + @@ -1442,7 +1457,7 @@ - + diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java index ef11600f0..aebe3cf95 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java @@ -59,6 +59,15 @@ import java.util.List; import java.util.Map; import java.util.HashMap; +//For loading library from jar +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + + /** * Created with IntelliJ IDEA. @@ -111,7 +120,28 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { synchronized(isVectorLoglessPairHMMLibraryLoaded) { //Load the library and initialize the FieldIDs if(!isVectorLoglessPairHMMLibraryLoaded) { - System.loadLibrary("VectorLoglessPairHMM"); + try + { + //Try loading from Java's library path first + //Useful if someone builds his/her own library and wants to override the bundled + //implementation without modifying the Java code + System.loadLibrary("VectorLoglessPairHMM"); + } + catch(UnsatisfiedLinkError ule) + { + //Could not load from Java's library path - try unpacking from jar + try + { + logger.info("libVectorLoglessPairHMM not found in JVM library path - trying to unpack from StingUtils.jar"); + loadLibraryFromJar("/org/broadinstitute/sting/utils/pairhmm/libVectorLoglessPairHMM.so"); + } + catch(IOException ioe) + { + //Throw the UnsatisfiedLinkError to make it clear to the user what failed + throw ule; + } + } + isVectorLoglessPairHMMLibraryLoaded = true; jniInitializeClassFieldsAndMachineMask(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once } @@ -229,4 +259,75 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { super.close(); jniClose(); } + + //Copied from http://frommyplayground.com/how-to-load-native-jni-library-from-jar + /** + * Loads library from current JAR archive + * + * The file from JAR is copied into system temporary directory and then loaded. The temporary file is deleted after exiting. + * Method uses String as filename because the pathname is "abstract", not system-dependent. + * + * @param filename The filename inside JAR as absolute path (beginning with '/'), e.g. /package/File.ext + * @throws IOException If temporary file creation or read/write operation fails + * @throws IllegalArgumentException If source file (param path) does not exist + * @throws IllegalArgumentException If the path is not absolute or if the filename is shorter than three characters (restriction of {@see File#createTempFile(java.lang.String, java.lang.String)}). + */ + public static void loadLibraryFromJar(String path) throws IOException { + + if (!path.startsWith("/")) { + throw new IllegalArgumentException("The path to be absolute (start with '/')."); + } + + // Obtain filename from path + String[] parts = path.split("/"); + String filename = (parts.length > 1) ? parts[parts.length - 1] : null; + + // Split filename to prexif and suffix (extension) + String prefix = ""; + String suffix = null; + if (filename != null) { + parts = filename.split("\\.", 2); + prefix = parts[0]; + suffix = (parts.length > 1) ? "."+parts[parts.length - 1] : null; // Thanks, davs! :-) + } + + // Check if the filename is okay + if (filename == null || prefix.length() < 3) { + throw new IllegalArgumentException("The filename has to be at least 3 characters long."); + } + + // Prepare temporary file + File temp = File.createTempFile(prefix, suffix); + //System.out.println("Temp lib file "+temp.getAbsolutePath()); + temp.deleteOnExit(); + + if (!temp.exists()) { + throw new FileNotFoundException("File " + temp.getAbsolutePath() + " does not exist."); + } + + // Prepare buffer for data copying + byte[] buffer = new byte[1024]; + int readBytes; + + // Open and check input stream + InputStream is = VectorLoglessPairHMM.class.getResourceAsStream(path); + if (is == null) { + throw new FileNotFoundException("File " + path + " was not found inside JAR."); + } + + // Open output stream and copy data between source file in JAR and the temporary file + OutputStream os = new FileOutputStream(temp); + try { + while ((readBytes = is.read(buffer)) != -1) { + os.write(buffer, 0, readBytes); + } + } finally { + // If read/write fails, close streams safely before throwing an exception + os.close(); + is.close(); + } + + // Finally, load the library + System.load(temp.getAbsolutePath()); + } } From 20a46e4098469c47fd124a5610f49e3f928759fc Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Fri, 7 Feb 2014 15:19:55 -0800 Subject: [PATCH 37/72] Check only for SSE 4.1 (rather than SSE 4.2) when trying to use the SSE implementation of PairHMM --- .../utils/pairhmm/VectorLoglessPairHMM.java | 5 +++-- public/c++/VectorPairHMM/utils.cc | 18 ++++++++++++++++-- public/c++/VectorPairHMM/utils.h | 3 ++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java index aebe3cf95..29d3a729d 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java @@ -77,8 +77,9 @@ import java.io.OutputStream; public class VectorLoglessPairHMM extends JNILoglessPairHMM { //For machine capabilities - public static final long sse42Mask = 1; - public static final long avxMask = 2; + public static final long sse41Mask = 1; + public static final long sse42Mask = 2; + public static final long avxMask = 4; public static final long enableAll = 0xFFFFFFFFFFFFFFFFl; //Used to copy references to byteArrays to JNI from reads diff --git a/public/c++/VectorPairHMM/utils.cc b/public/c++/VectorPairHMM/utils.cc index 9974c5ace..8fe20234e 100644 --- a/public/c++/VectorPairHMM/utils.cc +++ b/public/c++/VectorPairHMM/utils.cc @@ -22,6 +22,18 @@ bool is_avx_supported() return ((ecx >> 28)&1) == 1; } +bool is_sse41_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 19)&1) == 1; +} + bool is_sse42_supported() { int ecx = 0, edx = 0, ebx = 0; @@ -41,6 +53,8 @@ uint64_t get_machine_capabilities() machine_mask |= (1 << AVX_CUSTOM_IDX); if(is_sse42_supported()) machine_mask |= (1 << SSE42_CUSTOM_IDX); + if(is_sse41_supported()) + machine_mask |= (1 << SSE41_CUSTOM_IDX); return machine_mask; } @@ -54,9 +68,9 @@ void initialize_function_pointers(uint64_t mask) g_compute_full_prob_double = compute_full_prob_avxd; } else - if(is_sse42_supported() && (mask & (1<< SSE42_CUSTOM_IDX))) + if(is_sse41_supported() && (mask & (1<< SSE41_CUSTOM_IDX))) { - cout << "Using SSE4.2 accelerated implementation of PairHMM\n"; + cout << "Using SSE4.1 accelerated implementation of PairHMM\n"; g_compute_full_prob_float = compute_full_prob_sses; g_compute_full_prob_double = compute_full_prob_ssed; } diff --git a/public/c++/VectorPairHMM/utils.h b/public/c++/VectorPairHMM/utils.h index 092132473..501b48fbc 100644 --- a/public/c++/VectorPairHMM/utils.h +++ b/public/c++/VectorPairHMM/utils.h @@ -32,7 +32,8 @@ uint64_t diff_time(struct timespec& prev_time); //bit 0 is sse4.2, bit 1 is AVX enum ProcessorCapabilitiesEnum { - SSE42_CUSTOM_IDX=0, + SSE41_CUSTOM_IDX=0, + SSE42_CUSTOM_IDX, AVX_CUSTOM_IDX }; #define ENABLE_ALL_HARDWARE_FEATURES 0xFFFFFFFFFFFFFFFFull From a03d83579b812316d739025f86c8d87ac3bbf5e9 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Fri, 7 Feb 2014 23:22:05 -0800 Subject: [PATCH 38/72] Matrices in baseline C++ (no vector) implementation of PairHMM are now allocated on heap using "new". Stack allocation led to program crashes for large matrix sizes. --- public/c++/VectorPairHMM/baseline.cc | 179 +++++++++++++++------------ 1 file changed, 100 insertions(+), 79 deletions(-) diff --git a/public/c++/VectorPairHMM/baseline.cc b/public/c++/VectorPairHMM/baseline.cc index 268f32f00..232df83f2 100644 --- a/public/c++/VectorPairHMM/baseline.cc +++ b/public/c++/VectorPairHMM/baseline.cc @@ -1,100 +1,121 @@ #include "headers.h" #include "template.h" #include "utils.h" - +using namespace std; template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) { - int r, c; - int ROWS = tc->rslen + 1; - int COLS = tc->haplen + 1; + int r, c; + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; - Context ctx; + Context ctx; - NUMBER M[ROWS][COLS]; - NUMBER X[ROWS][COLS]; - NUMBER Y[ROWS][COLS]; - NUMBER p[ROWS][6]; + //allocate on heap in way that simulates a 2D array. Having a 2D array instead of + //a straightforward array of pointers ensures that all data lies 'close' in memory, increasing + //the chance of being stored together in the cache. Also, prefetchers can learn memory access + //patterns for 2D arrays, not possible for array of pointers + NUMBER* common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //pointers to within the allocated buffer + NUMBER** common_pointer_buffer = new NUMBER*[4*ROWS]; + NUMBER* ptr = common_buffer; + unsigned i = 0; + for(i=0;i<3*ROWS;++i, ptr+=COLS) + common_pointer_buffer[i] = ptr; + for(;i<4*ROWS;++i, ptr+=6) + common_pointer_buffer[i] = ptr; - p[0][MM] = ctx._(0.0); - p[0][GapM] = ctx._(0.0); - p[0][MX] = ctx._(0.0); - p[0][XX] = ctx._(0.0); - p[0][MY] = ctx._(0.0); - p[0][YY] = ctx._(0.0); - for (r = 1; r < ROWS; r++) - { - int _i = tc->i[r-1] & 127; - int _d = tc->d[r-1] & 127; - int _c = tc->c[r-1] & 127; - p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; - p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; - p[r][MX] = ctx.ph2pr[_i]; - p[r][XX] = ctx.ph2pr[_c]; - p[r][MY] = ctx.ph2pr[_d]; - p[r][YY] = ctx.ph2pr[_c]; - //p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; - //p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; - } - - for (c = 0; c < COLS; c++) - { - M[0][c] = ctx._(0.0); - X[0][c] = ctx._(0.0); - Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); - } - - for (r = 1; r < ROWS; r++) - { - M[r][0] = ctx._(0.0); - X[r][0] = X[r-1][0] * p[r][XX]; - Y[r][0] = ctx._(0.0); - } - - NUMBER result = ctx._(0.0); - - for (r = 1; r < ROWS; r++) - for (c = 1; c < COLS; c++) - { - fexcept_t flagp; - char _rs = tc->rs[r-1]; - char _hap = tc->hap[c-1]; - int _q = tc->q[r-1] & 127; - NUMBER distm = ctx.ph2pr[_q]; - if (_rs == _hap || _rs == 'N' || _hap == 'N') - distm = ctx._(1.0) - distm; - else - distm = distm/3; + //NUMBER M[ROWS][COLS]; + //NUMBER X[ROWS][COLS]; + //NUMBER Y[ROWS][COLS]; + //NUMBER p[ROWS][6]; + NUMBER** M = common_pointer_buffer; + NUMBER** X = M + ROWS; + NUMBER** Y = X + ROWS; + NUMBER** p = Y + ROWS; - //feclearexcept(FE_ALL_EXCEPT); - M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); - //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + p[0][MM] = ctx._(0.0); + p[0][GapM] = ctx._(0.0); + p[0][MX] = ctx._(0.0); + p[0][XX] = ctx._(0.0); + p[0][MY] = ctx._(0.0); + p[0][YY] = ctx._(0.0); - //feclearexcept(FE_ALL_EXCEPT); - X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; - //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + for (r = 1; r < ROWS; r++) + { + int _i = tc->i[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; + p[r][MX] = ctx.ph2pr[_i]; + p[r][XX] = ctx.ph2pr[_c]; + p[r][MY] = ctx.ph2pr[_d]; + p[r][YY] = ctx.ph2pr[_c]; + //p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + //p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + } + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); + } - //feclearexcept(FE_ALL_EXCEPT); - Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; - //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } - //CONVERT_AND_PRINT(M[r][c]); - //CONVERT_AND_PRINT(X[r][c]); - //CONVERT_AND_PRINT(Y[r][c]); + NUMBER result = ctx._(0.0); - } + for (r = 1; r < ROWS; r++) + for (c = 1; c < COLS; c++) + { + fexcept_t flagp; + char _rs = tc->rs[r-1]; + char _hap = tc->hap[c-1]; + int _q = tc->q[r-1] & 127; + NUMBER distm = ctx.ph2pr[_q]; + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; + else + distm = distm/3; - for (c = 0; c < COLS; c++) - { - result += M[ROWS-1][c] + X[ROWS-1][c]; - } - if (before_last_log != NULL) - *before_last_log = result; + //feclearexcept(FE_ALL_EXCEPT); + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); - return result; - //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; + //feclearexcept(FE_ALL_EXCEPT); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //feclearexcept(FE_ALL_EXCEPT); + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //CONVERT_AND_PRINT(M[r][c]); + //CONVERT_AND_PRINT(X[r][c]); + //CONVERT_AND_PRINT(Y[r][c]); + + } + for (c = 0; c < COLS; c++) + { + result += M[ROWS-1][c] + X[ROWS-1][c]; + } + + if (before_last_log != NULL) + *before_last_log = result; + + delete common_pointer_buffer; + delete common_buffer; + + return result; + //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; } template double compute_full_prob(testcase* tc, double* nextbuf); From d081c19178bfd8fdfb78eaf7643b7179444b16b2 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Sun, 9 Feb 2014 18:05:35 -0800 Subject: [PATCH 39/72] Minor: added support in C++ sandbox to choose implementation and check from command line --- public/c++/VectorPairHMM/pairhmm-1-base.cc | 26 ++++++++++-- public/c++/VectorPairHMM/utils.cc | 49 ++++++++++------------ public/c++/VectorPairHMM/utils.h | 2 +- 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/public/c++/VectorPairHMM/pairhmm-1-base.cc b/public/c++/VectorPairHMM/pairhmm-1-base.cc index 8b686d5ea..78dbe296d 100644 --- a/public/c++/VectorPairHMM/pairhmm-1-base.cc +++ b/public/c++/VectorPairHMM/pairhmm-1-base.cc @@ -18,10 +18,28 @@ int main(int argc, char** argv) if(argc >= 3 && string(argv[2]) == "1") use_old_read_testcase = true; unsigned chunk_size = 10000; - if(argc >= 4) - chunk_size = strtol(argv[3],0,10); - - do_compute(argv[1], use_old_read_testcase, chunk_size); + bool do_check = true; + uint64_t mask = ~(0ull); + for(int i=3;i; } else - if(is_sse41_supported() && (mask & (1<< SSE41_CUSTOM_IDX))) + if(is_sse41_supported() && (mask & ((1<< SSE41_CUSTOM_IDX) | (1<; @@ -299,10 +299,9 @@ uint64_t diff_time(struct timespec& prev_time) } //#define DUMP_COMPUTE_VALUES 1 -#define CHECK_VALUES 1 #define BATCH_SIZE 10000 #define RUN_HYBRID -void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size) +void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size, bool do_check) { FILE* fptr = 0; ifstream ifptr; @@ -322,10 +321,7 @@ void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size) uint64_t vector_compute_time = 0; uint64_t baseline_compute_time = 0; uint64_t num_double_calls = 0; - bool all_ok = true; -#ifndef CHECK_VALUES - all_ok = false; -#endif + bool all_ok = do_check ? true : false; while(1) { int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); @@ -360,29 +356,30 @@ void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size) results_vec[i] = result; } vector_compute_time += diff_time(start_time); -#ifdef CHECK_VALUES - get_time(&start_time); + if(do_check) + { + get_time(&start_time); #pragma omp parallel for schedule(dynamic,chunk_size) - for(unsigned i=0;i(&tc); - baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); - baseline_results_vec[i] = baseline_result; - } - baseline_compute_time += diff_time(start_time); - for(unsigned i=0;i 1e-5 && rel_error > 1e-5) + for(unsigned i=0;i(&tc); + baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); + baseline_results_vec[i] = baseline_result; + } + baseline_compute_time += diff_time(start_time); + for(unsigned i=0;i 1e-5 && rel_error > 1e-5) + { + cout << std::scientific << baseline_result << " "< Date: Mon, 10 Feb 2014 10:11:03 -0800 Subject: [PATCH 40/72] Fixed denominator in profiling --- public/c++/VectorPairHMM/LoadTimeInitializer.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/c++/VectorPairHMM/LoadTimeInitializer.cc index fd2d1df09..f9db149e6 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.cc +++ b/public/c++/VectorPairHMM/LoadTimeInitializer.cc @@ -74,7 +74,11 @@ void LoadTimeInitializer::print_profiling() denominator = m_sum_stats[NUM_REGIONS_IDX]; break; case HAPLOTYPE_LENGTH_IDX: + denominator = m_sum_stats[NUM_HAPLOTYPES_IDX]; + break; case READ_LENGTH_IDX: + denominator = m_sum_stats[NUM_READS_IDX]; + break; case PRODUCT_READ_LENGTH_HAPLOTYPE_LENGTH_IDX: denominator = m_sum_stats[NUM_TESTCASES_IDX]; break; From a058e96c34e597b2d9bb1ec2455409432d7e7284 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Tue, 25 Feb 2014 21:44:20 -0800 Subject: [PATCH 41/72] Now has PAPI values --- .../haplotypecaller/HaplotypeCaller.java | 2 +- .../utils/pairhmm/DebugJNILoglessPairHMM.java | 123 +++++++++++------- public/c++/VectorPairHMM/.gitignore | 2 + .../c++/VectorPairHMM/LoadTimeInitializer.cc | 6 + .../c++/VectorPairHMM/LoadTimeInitializer.h | 5 + public/c++/VectorPairHMM/Sandbox.cc | 4 +- public/c++/VectorPairHMM/baseline.cc | 28 +++- ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 55 ++++++-- public/c++/VectorPairHMM/utils.cc | 86 +++++++++--- 9 files changed, 229 insertions(+), 82 deletions(-) diff --git a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java index b785b8d21..f9f13ffc6 100644 --- a/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java +++ b/protected/java/src/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java @@ -1086,7 +1086,7 @@ public class HaplotypeCaller extends ActiveRegionWalker, In referenceConfidenceModel.close(); //TODO remove the need to call close here for debugging, the likelihood output stream should be managed //TODO (open & close) at the walker, not the engine. - likelihoodCalculationEngine.close(); + likelihoodCalculationEngine.close(); logger.info("Ran local assembly on " + result + " active regions"); } diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java index 7e50f0c30..dc0ee23f7 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java +++ b/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java @@ -75,8 +75,9 @@ import java.io.IOException; */ public class DebugJNILoglessPairHMM extends LoglessPairHMM { + private static final boolean dumpSandboxOnly = false; //simulates ifdef private static final boolean debug = false; //simulates ifdef - private static final boolean verify = debug || true; //simulates ifdef + private static final boolean verify = !dumpSandboxOnly && (debug || true); //simulates ifdef private static final boolean debug0_1 = false; //simulates ifdef private static final boolean debug1 = false; //simulates ifdef private static final boolean debug2 = false; @@ -134,9 +135,11 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { public void initialize( final List haplotypes, final Map> perSampleReadList, final int readMaxLength, final int haplotypeMaxLength ) { if(verify) + { super.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); - jniPairHMM.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); - haplotypeToHaplotypeListIdxMap = jniPairHMM.getHaplotypeToHaplotypeListIdxMap(); + jniPairHMM.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); + haplotypeToHaplotypeListIdxMap = jniPairHMM.getHaplotypeToHaplotypeListIdxMap(); + } } /** @@ -145,7 +148,8 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { @Override public void finalizeRegion() { - jniPairHMM.finalizeRegion(); + if(!dumpSandboxOnly) + jniPairHMM.finalizeRegion(); } /** @@ -204,46 +208,61 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { ++idx; } } - jniPairHMM.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); - double[] likelihoodArray = jniPairHMM.getLikelihoodArray(); - //to compare values - final PerReadAlleleLikelihoodMap likelihoodMap = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + double[] likelihoodArray = null; + PerReadAlleleLikelihoodMap likelihoodMap = null; if(verify) { - //re-order values in likelihoodArray - double[] tmpArray = new double[numHaplotypes]; - idx = 0; - int idxInsideHaplotypeList = 0; - int readIdx = 0; - for(GATKSAMRecord read : reads) + jniPairHMM.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + likelihoodArray = jniPairHMM.getLikelihoodArray(); + //to compare values + likelihoodMap = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + } + else + { + likelihoodMap = new PerReadAlleleLikelihoodMap(); + likelihoodArray = new double[numTestcases]; + for(int i=0;i currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always + //re-order values in likelihoodArray + double[] tmpArray = new double[numHaplotypes]; + idx = 0; + int idxInsideHaplotypeList = 0; + int readIdx = 0; + for(GATKSAMRecord read : reads) { - idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); - likelihoodArray[idx] = tmpArray[idxInsideHaplotypeList]; - ++idx; + for(int j=0;j currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always + { + idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); + likelihoodArray[idx] = tmpArray[idxInsideHaplotypeList]; + ++idx; + } + readIdx += numHaplotypes; } - readIdx += numHaplotypes; - } - //for floating point values, no exact equality - //check whether numbers are close in terms of abs_error or relative_error - //For very large values, relative_error is relevant - //For very small values, abs_error is relevant - boolean toDump = false; - for(int i=0;i 1e-5 && relative_error > 1e-5) + //for floating point values, no exact equality + //check whether numbers are close in terms of abs_error or relative_error + //For very large values, relative_error is relevant + //For very small values, abs_error is relevant + for(int i=0;i 1e-6 && relative_error > 1e-6) + { + toDump = true; + break; + } } } //if numbers are not close, then dump out the data that produced the inconsistency @@ -251,24 +270,40 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { { idx = 0; System.out.println("Dump : Java numReads "+readListSize+" numHaplotypes "+numHaplotypes); + boolean firstLine = true; for(GATKSAMRecord read : reads) { byte [] overallGCP = GCPArrayMap.get(read); + byte[] tmpByteArray = new byte[read.getReadBases().length]; for (Map.Entry currEntry : alleleHaplotypeMap.entrySet()) //order is important - access in same order always { byte[] haplotypeBases = currEntry.getValue().getBases(); debugDump("debug_dump.txt",new String(haplotypeBases)+" ",true); debugDump("debug_dump.txt",new String(read.getReadBases())+" ",true); for(int k=0;k m_filename_to_fptr; std::set m_written_files_set; @@ -52,6 +55,8 @@ class LoadTimeInitializer double m_sum_square_stats[TOTAL_NUMBER_STATS]; uint64_t m_min_stats[TOTAL_NUMBER_STATS]; uint64_t m_max_stats[TOTAL_NUMBER_STATS]; + unsigned m_buffer_size; + uint64_t* m_buffer; }; extern LoadTimeInitializer g_load_time_initializer; diff --git a/public/c++/VectorPairHMM/Sandbox.cc b/public/c++/VectorPairHMM/Sandbox.cc index 7c10e0620..8fb0d0b67 100644 --- a/public/c++/VectorPairHMM/Sandbox.cc +++ b/public/c++/VectorPairHMM/Sandbox.cc @@ -73,7 +73,9 @@ JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative (JNIEnv* env, jobject thisObject, jstring fileNameString) { const char* fileName = env->GetStringUTFChars(fileNameString, 0); - do_compute((char*)fileName); + char local_array[800]; + strncpy(local_array, fileName, 200); env->ReleaseStringUTFChars(fileNameString, fileName); + do_compute(local_array, true, 10000, false); } diff --git a/public/c++/VectorPairHMM/baseline.cc b/public/c++/VectorPairHMM/baseline.cc index 232df83f2..290a3a4a2 100644 --- a/public/c++/VectorPairHMM/baseline.cc +++ b/public/c++/VectorPairHMM/baseline.cc @@ -1,6 +1,8 @@ #include "headers.h" #include "template.h" #include "utils.h" +#include "LoadTimeInitializer.h" + using namespace std; template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) @@ -10,12 +12,28 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) int COLS = tc->haplen + 1; Context ctx; - + //#define USE_STACK_ALLOCATION 1 +#ifdef USE_STACK_ALLOCATION + NUMBER M[ROWS][COLS]; + NUMBER X[ROWS][COLS]; + NUMBER Y[ROWS][COLS]; + NUMBER p[ROWS][6]; +#else //allocate on heap in way that simulates a 2D array. Having a 2D array instead of //a straightforward array of pointers ensures that all data lies 'close' in memory, increasing //the chance of being stored together in the cache. Also, prefetchers can learn memory access //patterns for 2D arrays, not possible for array of pointers + //bool locally_allocated = false; + //NUMBER* common_buffer = 0; NUMBER* common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //unsigned curr_size = sizeof(NUMBER)*(3*ROWS*COLS + ROWS*6); + //if(true) + //{ + //common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //locally_allocated = true; + //} + //else + //common_buffer = (NUMBER*)(g_load_time_initializer.get_buffer()); //pointers to within the allocated buffer NUMBER** common_pointer_buffer = new NUMBER*[4*ROWS]; NUMBER* ptr = common_buffer; @@ -25,14 +43,11 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) for(;i<4*ROWS;++i, ptr+=6) common_pointer_buffer[i] = ptr; - //NUMBER M[ROWS][COLS]; - //NUMBER X[ROWS][COLS]; - //NUMBER Y[ROWS][COLS]; - //NUMBER p[ROWS][6]; NUMBER** M = common_pointer_buffer; NUMBER** X = M + ROWS; NUMBER** Y = X + ROWS; NUMBER** p = Y + ROWS; +#endif p[0][MM] = ctx._(0.0); @@ -111,8 +126,11 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) if (before_last_log != NULL) *before_last_log = result; +#ifndef USE_STACK_ALLOCATION delete common_pointer_buffer; + //if(locally_allocated) delete common_buffer; +#endif return result; //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index 0f5219dd4..2d8dbaf3d 100644 --- a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc +++ b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -191,25 +191,32 @@ inline JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_Vector } } +//#define DO_WARMUP +//#define DO_REPEAT_PROFILING //Do compute over vector of testcase structs inline void compute_testcases(vector& tc_array, unsigned numTestCases, double* likelihoodDoubleArray, unsigned maxNumThreadsToUse) { -#pragma omp parallel for schedule (dynamic,10000) num_threads(maxNumThreadsToUse) - for(unsigned tc_idx=0;tc_idxGetArrayLength(likelihoodArray) == numTestCases); #endif +#ifdef DO_WARMUP + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + for(unsigned i=0;iGetArrayLength(haplotypeBasesArrayVector[i].first); + for(unsigned j=0;jGetArrayLength(readBasesArrayVector[i][j].first); + for(unsigned k=0;k Date: Wed, 26 Feb 2014 10:53:51 -0800 Subject: [PATCH 42/72] Followed Khalid's suggestion for packing libVectorLoglessCaching into the jar file with Maven --- .gitignore | 36 ++++ ...GraphBasedLikelihoodCalculationEngine.java | 4 + .../haplotypecaller/HaplotypeCaller.java | 2 +- .../PairHMMLikelihoodCalculationEngine.java | 3 +- .../RandomLikelihoodCalculationEngine.java | 5 + .../utils/pairhmm/DebugJNILoglessPairHMM.java | 4 +- .../utils/pairhmm/JNILoglessPairHMM.java | 0 .../utils/pairhmm/VectorLoglessPairHMM.java | 2 - public/c++/VectorPairHMM/Makefile | 8 +- ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 2 - public/c++/VectorPairHMM/run.sh | 3 +- public/c++/VectorPairHMM/utils.cc | 7 +- public/c++/VectorPairHMM/utils.h | 7 + public/c++/pom.xml | 167 ++++++++++++++++++ public/sting-root/pom.xml | 6 +- 15 files changed, 240 insertions(+), 16 deletions(-) rename protected/{java/src => gatk-protected/src/main/java}/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java (99%) rename protected/{java/src => gatk-protected/src/main/java}/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java (100%) rename protected/{java/src => gatk-protected/src/main/java}/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java (99%) create mode 100644 public/c++/pom.xml diff --git a/.gitignore b/.gitignore index a2ff166e5..3ecbc1485 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /*.bed *~ /*.vcf +/*.vcf.idx /*.txt /*.csh /.* @@ -40,3 +41,38 @@ perf verify maven-metadata-local.xml dependency-reduced-pom.xml +private/gatk-private/private/ +private/gatk-private/public/ +private/gatk-private/target/ +private/na12878kb-utils/public/ +private/na12878kb-utils/target/ +private/queue-private/private/ +private/queue-private/public/ +private/queue-private/target/ +private/scala/ +private/testdata +protected/gatk-protected/private/ +protected/gatk-protected/public/ +protected/gatk-protected/target/ +public/external-example/target/ +public/gatk-framework/private/ +public/gatk-framework/public/ +public/gatk-framework/target/ +public/gatk-package/private/ +public/gatk-package/public/ +public/gatk-package/target/ +public/gatk-queue-extgen/public/ +public/gatk-queue-extgen/target/ +public/gsalib/public/ +public/gsalib/target/ +public/queue-framework/private/ +public/queue-framework/public/ +public/queue-framework/target/ +public/queue-package/private/ +public/queue-package/public/ +public/queue-package/target/ +public/scala/ +public/sting-utils/public/ +public/sting-utils/target/ +public/testdata +target/ diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GraphBasedLikelihoodCalculationEngine.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GraphBasedLikelihoodCalculationEngine.java index 8a35ccb05..8b37e265d 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GraphBasedLikelihoodCalculationEngine.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GraphBasedLikelihoodCalculationEngine.java @@ -155,4 +155,8 @@ public class GraphBasedLikelihoodCalculationEngine implements LikelihoodCalculat } } } + + @Override + public void close() { + } } diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java index 276103277..7844741df 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java @@ -1043,7 +1043,7 @@ public class HaplotypeCaller extends ActiveRegionWalker, In referenceConfidenceModel.close(); //TODO remove the need to call close here for debugging, the likelihood output stream should be managed //TODO (open & close) at the walker, not the engine. - likelihoodCalculationEngine.close(); + likelihoodCalculationEngine.close(); logger.info("Ran local assembly on " + result + " active regions"); } diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index c534764a5..344366869 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -170,9 +170,10 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation @Override public void close() { if ( likelihoodsStream != null ) likelihoodsStream.close(); - pairHMMThreadLocal.get().close(); + pairHMMThreadLocal.get().close(); } + private void writeDebugLikelihoods(final GATKSAMRecord processedRead, final Haplotype haplotype, final double log10l){ if ( WRITE_LIKELIHOODS_TO_FILE ) { likelihoodsStream.printf("%s %s %s %s %s %s %f%n", diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/RandomLikelihoodCalculationEngine.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/RandomLikelihoodCalculationEngine.java index b8dba7b86..d5d424ca9 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/RandomLikelihoodCalculationEngine.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/RandomLikelihoodCalculationEngine.java @@ -79,4 +79,9 @@ public class RandomLikelihoodCalculationEngine implements LikelihoodCalculationE return result; } + + @Override + public void close() { + } + } diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java similarity index 99% rename from protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java rename to protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java index dc0ee23f7..beaca115a 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java @@ -55,7 +55,7 @@ import org.broadinstitute.sting.utils.haplotype.Haplotype; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.broadinstitute.variant.variantcontext.Allele; import org.broadinstitute.sting.utils.exceptions.UserException; - +import static org.broadinstitute.sting.utils.pairhmm.PairHMMModel.*; import java.util.List; import java.util.Map; @@ -258,7 +258,7 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { relative_error = 0; else relative_error = Math.abs(abs_error/mLikelihoodArray[i]); - if(abs_error > 1e-6 && relative_error > 1e-6) + if(abs_error > 1e-5 && relative_error > 1e-5) { toDump = true; break; diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java similarity index 100% rename from protected/java/src/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java rename to protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java similarity index 99% rename from protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java rename to protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java index 29d3a729d..50600d850 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java @@ -67,8 +67,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; - - /** * Created with IntelliJ IDEA. * User: rpoplin, carneiro diff --git a/public/c++/VectorPairHMM/Makefile b/public/c++/VectorPairHMM/Makefile index 269cecbd2..f88b38401 100644 --- a/public/c++/VectorPairHMM/Makefile +++ b/public/c++/VectorPairHMM/Makefile @@ -58,7 +58,7 @@ $(AVX_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xAVX $(SSE_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xSSE4.2 OBJECTS=$(NO_VECTOR_OBJECTS) $(AVX_OBJECTS) $(SSE_OBJECTS) -all: $(BIN) Sandbox.class +all: $(BIN) Sandbox.class copied_lib -include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) @@ -79,5 +79,11 @@ $(OBJECTS): %.o: %.cc Sandbox.class: Sandbox.java javac Sandbox.java +copied_lib: libVectorLoglessPairHMM.so +ifdef OUTPUT_DIR + mkdir -p $(OUTPUT_DIR) + rsync -a libVectorLoglessPairHMM.so $(OUTPUT_DIR)/ +endif + clean: rm -rf $(BIN) *.o $(DEPDIR) *.class diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index 2d8dbaf3d..cd1656e7c 100644 --- a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc +++ b/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -191,8 +191,6 @@ inline JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_Vector } } -//#define DO_WARMUP -//#define DO_REPEAT_PROFILING //Do compute over vector of testcase structs inline void compute_testcases(vector& tc_array, unsigned numTestCases, double* likelihoodDoubleArray, unsigned maxNumThreadsToUse) diff --git a/public/c++/VectorPairHMM/run.sh b/public/c++/VectorPairHMM/run.sh index a6930678b..b52dddc46 100755 --- a/public/c++/VectorPairHMM/run.sh +++ b/public/c++/VectorPairHMM/run.sh @@ -8,7 +8,8 @@ then fi #-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed -java -Djava.library.path=${GSA_ROOT_DIR}/public/c++/VectorPairHMM -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ +#java -Djava.library.path=${GSA_ROOT_DIR}/public/c++/VectorPairHMM -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ +java -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ --dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ -I /data/simulated/sim1M_pairs_final.bam \ diff --git a/public/c++/VectorPairHMM/utils.cc b/public/c++/VectorPairHMM/utils.cc index 832335d88..387a6b4a0 100644 --- a/public/c++/VectorPairHMM/utils.cc +++ b/public/c++/VectorPairHMM/utils.cc @@ -306,15 +306,12 @@ uint64_t diff_time(struct timespec& prev_time) return (uint64_t)((curr_time.tv_sec-prev_time.tv_sec)*1000000000+(curr_time.tv_nsec-prev_time.tv_nsec)); } -#define DUMP_COMPUTE_VALUES 1 -//#define DO_REPEATS + #ifdef USE_PAPI #include "papi.h" #define NUM_PAPI_COUNTERS 4 #endif -#define BATCH_SIZE 10000 -#define RUN_HYBRID void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size, bool do_check) { FILE* fptr = 0; @@ -370,7 +367,7 @@ void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size, #endif get_time(&start_time); #pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) -#ifdef DO_REPEATS +#ifdef DO_REPEAT_PROFILING for(unsigned z=0;z<10;++z) #endif { diff --git a/public/c++/VectorPairHMM/utils.h b/public/c++/VectorPairHMM/utils.h index 98115e283..c1288d9e5 100644 --- a/public/c++/VectorPairHMM/utils.h +++ b/public/c++/VectorPairHMM/utils.h @@ -40,4 +40,11 @@ enum ProcessorCapabilitiesEnum uint64_t get_machine_capabilities(); void initialize_function_pointers(uint64_t mask=ENABLE_ALL_HARDWARE_FEATURES); void do_compute(char* filename, bool use_old_read_testcase=true, unsigned chunk_size=10000, bool do_check=true); + +//#define DO_WARMUP +//#define DO_REPEAT_PROFILING +//#define DUMP_COMPUTE_VALUES 1 +#define BATCH_SIZE 10000 +#define RUN_HYBRID + #endif diff --git a/public/c++/pom.xml b/public/c++/pom.xml new file mode 100644 index 000000000..c64f81ede --- /dev/null +++ b/public/c++/pom.xml @@ -0,0 +1,167 @@ + + + 4.0.0 + + + org.broadinstitute.sting + sting-root + 2.8-SNAPSHOT + ../../public/sting-root + + + pairhmm-native + pom + PairHMM Native Libraries + + Builds a platform dependent version of pairhmm. During install, copies it into sting-utils. + + + UTF-8 + ${sourceEncoding} + ${sourceEncoding} + ${project.basedir}/../.. + ${sting.basedir}/public/sting-utils + + ${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm + + + + x86_64 + so + -Wall + -dynamiclib + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + display-info + + validate + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + + exec + + compile + + make + VectorPairHMM + + ${pairhmm.build.architecture} + ${pairhmm.build.extension} + ${pairhmm.build.cxxflags} + ${pairhmm.build.ldflags} + ${project.build.directory} + + + + + + + + + org.apache.maven.plugins + maven-install-plugin + + true + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + default-install + + copy-resources + + install + + ${pairhmm.resources.directory} + + + ${project.build.directory} + + **/* + + + + + + + + + + + com.google.code.sortpom + maven-sortpom-plugin + + false + custom_1 + \n + ${sourceEncoding} + true + scope + 4 + false + + + + + + + + + mac_x86_64 + + + mac + x86_64 + + + + mac_x86_64 + jnilib + -dynamiclib + + + + + linux_x86_64 + + + linux + amd64 + + + + linux_x86_64 + jnilib + -fPIC + -shared + + + + + diff --git a/public/sting-root/pom.xml b/public/sting-root/pom.xml index 84edd9be5..39227e4ba 100644 --- a/public/sting-root/pom.xml +++ b/public/sting-root/pom.xml @@ -332,7 +332,11 @@ maven-assembly-plugin 2.4 - + + org.apache.maven.plugins + maven-enforcer-plugin + 1.3.1 + + ${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm + + + + x86_64 + so + -Wall + -dynamiclib + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + display-info + + validate + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + + exec + + compile + + make + VectorPairHMM + + ${pairhmm.build.architecture} + ${pairhmm.build.extension} + ${pairhmm.build.cxxflags} + ${pairhmm.build.ldflags} + ${project.build.directory} + + + + + + + + + org.apache.maven.plugins + maven-install-plugin + + true + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + default-install + + copy-resources + + install + + ${pairhmm.resources.directory} + + + ${project.build.directory} + + **/* + + + + + + + + + + + com.google.code.sortpom + maven-sortpom-plugin + + false + custom_1 + \n + ${sourceEncoding} + true + scope + 4 + false + + + + + + + + + mac_x86_64 + + + mac + x86_64 + + + + mac_x86_64 + jnilib + -dynamiclib + + + + + linux_x86_64 + + + linux + amd64 + + + + linux_x86_64 + jnilib + -fPIC + -shared + + + + + diff --git a/public/sting-root/pom.xml b/public/sting-root/pom.xml index 84edd9be5..39227e4ba 100644 --- a/public/sting-root/pom.xml +++ b/public/sting-root/pom.xml @@ -332,7 +332,11 @@ maven-assembly-plugin 2.4 - + + org.apache.maven.plugins + maven-enforcer-plugin + 1.3.1 + ${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm @@ -64,7 +64,7 @@ compile make - VectorPairHMM + . ${pairhmm.build.architecture} ${pairhmm.build.extension} From c645725fc3ae392d186947cb5b3ea58f8ca5038e Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Wed, 26 Feb 2014 15:17:15 -0800 Subject: [PATCH 48/72] 1. Renamed directory structure from public/c++/VectorPairHMM to public/VectorPairHMM/src/main/c++ as per Khalid's suggestion 2. Use java.home in public/VectorPairHMM/pom.xml to pass environment variable JRE_HOME to the make process. This is needed because the Makefile needs to compile JNI code with the flag -I/../include (among others). Assuming that the Maven build process uses a JDK (and not just a JRE), the variable java.home points to the JRE inside maven. 3. Dropped all pretense at cross-platform compatibility. Removed Mac profile from pom.xml for VectorPairHMM --- public/{c++ => }/VectorPairHMM/pom.xml | 32 +++---------------- .../src/main/c++}/.gitignore | 0 .../src/main/c++}/JNI_README | 0 .../src/main/c++}/LoadTimeInitializer.cc | 0 .../src/main/c++}/LoadTimeInitializer.h | 0 .../src/main/c++}/Makefile | 4 +-- .../src/main/c++}/Sandbox.cc | 0 .../src/main/c++}/Sandbox.h | 0 .../src/main/c++}/Sandbox.java | 0 .../Sandbox_JNIHaplotypeDataHolderClass.h | 0 .../c++}/Sandbox_JNIReadDataHolderClass.h | 0 .../main/c++}/avx_function_instantiations.cc | 0 .../src/main/c++}/baseline.cc | 0 .../src/main/c++}/define-double.h | 0 .../src/main/c++}/define-float.h | 0 .../src/main/c++}/define-sse-double.h | 0 .../src/main/c++}/define-sse-float.h | 0 .../src/main/c++}/headers.h | 0 .../src/main/c++}/jni_common.h | 0 .../src/main/c++}/jnidebug.h | 0 ...ng_utils_pairhmm_DebugJNILoglessPairHMM.cc | 0 ...ing_utils_pairhmm_DebugJNILoglessPairHMM.h | 0 ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 0 ...sting_utils_pairhmm_VectorLoglessPairHMM.h | 0 .../src/main/c++}/pairhmm-1-base.cc | 0 .../src/main/c++}/pairhmm-template-kernel.cc | 0 .../src/main/c++}/pairhmm-template-main.cc | 0 .../src/main/c++}/run.sh | 0 .../src/main/c++}/shift_template.c | 0 .../main/c++}/sse_function_instantiations.cc | 0 .../src/main/c++}/template.h | 0 .../src/main/c++}/utils.cc | 0 .../src/main/c++}/utils.h | 0 .../src/main/c++}/vector_defs.h | 0 .../main/c++}/vector_function_prototypes.h | 0 35 files changed, 7 insertions(+), 29 deletions(-) rename public/{c++ => }/VectorPairHMM/pom.xml (80%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/.gitignore (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/JNI_README (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/LoadTimeInitializer.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/LoadTimeInitializer.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Makefile (95%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox.java (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox_JNIHaplotypeDataHolderClass.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox_JNIReadDataHolderClass.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/avx_function_instantiations.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/baseline.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/define-double.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/define-float.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/define-sse-double.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/define-sse-float.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/headers.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/jni_common.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/jnidebug.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/pairhmm-1-base.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/pairhmm-template-kernel.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/pairhmm-template-main.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/run.sh (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/shift_template.c (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/sse_function_instantiations.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/template.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/utils.cc (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/utils.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/vector_defs.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/vector_function_prototypes.h (100%) diff --git a/public/c++/VectorPairHMM/pom.xml b/public/VectorPairHMM/pom.xml similarity index 80% rename from public/c++/VectorPairHMM/pom.xml rename to public/VectorPairHMM/pom.xml index ce0692799..b310148d6 100644 --- a/public/c++/VectorPairHMM/pom.xml +++ b/public/VectorPairHMM/pom.xml @@ -6,20 +6,20 @@ org.broadinstitute.sting sting-root 2.8-SNAPSHOT - ../../../public/sting-root + ../../public/sting-root VectorPairHMM pom Vectorized PairHMM native libraries - Builds a platform dependent version of pairhmm. During install, copies it into sting-utils. + Builds a GNU/Linux x86_64 version of VectorPairHMM. During install, copies it into sting-utils. UTF-8 ${sourceEncoding} ${sourceEncoding} - ${project.basedir}/../../.. + ${project.basedir}/../.. ${sting.basedir}/public/sting-utils ${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm @@ -33,8 +33,6 @@ --> x86_64 so - -Wall - -dynamiclib @@ -64,12 +62,10 @@ compile make - . + src/main/c++ ${pairhmm.build.architecture} - ${pairhmm.build.extension} - ${pairhmm.build.cxxflags} - ${pairhmm.build.ldflags} + ${java.home} ${project.build.directory} @@ -132,21 +128,6 @@ - - mac_x86_64 - - - mac - x86_64 - - - - mac_x86_64 - jnilib - -dynamiclib - - - linux_x86_64 @@ -157,9 +138,6 @@ linux_x86_64 - jnilib - -fPIC - -shared diff --git a/public/c++/VectorPairHMM/.gitignore b/public/VectorPairHMM/src/main/c++/.gitignore similarity index 100% rename from public/c++/VectorPairHMM/.gitignore rename to public/VectorPairHMM/src/main/c++/.gitignore diff --git a/public/c++/VectorPairHMM/JNI_README b/public/VectorPairHMM/src/main/c++/JNI_README similarity index 100% rename from public/c++/VectorPairHMM/JNI_README rename to public/VectorPairHMM/src/main/c++/JNI_README diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc similarity index 100% rename from public/c++/VectorPairHMM/LoadTimeInitializer.cc rename to public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.h b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h similarity index 100% rename from public/c++/VectorPairHMM/LoadTimeInitializer.h rename to public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h diff --git a/public/c++/VectorPairHMM/Makefile b/public/VectorPairHMM/src/main/c++/Makefile similarity index 95% rename from public/c++/VectorPairHMM/Makefile rename to public/VectorPairHMM/src/main/c++/Makefile index f88b38401..e1e443d2e 100644 --- a/public/c++/VectorPairHMM/Makefile +++ b/public/VectorPairHMM/src/main/c++/Makefile @@ -4,8 +4,8 @@ #CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas #CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas -JAVA_ROOT=/opt/jdk1.7.0_25/ -JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JAVA_ROOT}/include -I${JAVA_ROOT}/include/linux +JRE_HOME?=/opt/jdk1.7.0_25/ +JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JRE_HOME}/../include -I${JRE_HOME}/../include/linux COMMON_COMPILATION_FLAGS=$(JNI_COMPILATION_FLAGS) -O3 -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas CC=icc diff --git a/public/c++/VectorPairHMM/Sandbox.cc b/public/VectorPairHMM/src/main/c++/Sandbox.cc similarity index 100% rename from public/c++/VectorPairHMM/Sandbox.cc rename to public/VectorPairHMM/src/main/c++/Sandbox.cc diff --git a/public/c++/VectorPairHMM/Sandbox.h b/public/VectorPairHMM/src/main/c++/Sandbox.h similarity index 100% rename from public/c++/VectorPairHMM/Sandbox.h rename to public/VectorPairHMM/src/main/c++/Sandbox.h diff --git a/public/c++/VectorPairHMM/Sandbox.java b/public/VectorPairHMM/src/main/c++/Sandbox.java similarity index 100% rename from public/c++/VectorPairHMM/Sandbox.java rename to public/VectorPairHMM/src/main/c++/Sandbox.java diff --git a/public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h similarity index 100% rename from public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h rename to public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h diff --git a/public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h similarity index 100% rename from public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h rename to public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h diff --git a/public/c++/VectorPairHMM/avx_function_instantiations.cc b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc similarity index 100% rename from public/c++/VectorPairHMM/avx_function_instantiations.cc rename to public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc diff --git a/public/c++/VectorPairHMM/baseline.cc b/public/VectorPairHMM/src/main/c++/baseline.cc similarity index 100% rename from public/c++/VectorPairHMM/baseline.cc rename to public/VectorPairHMM/src/main/c++/baseline.cc diff --git a/public/c++/VectorPairHMM/define-double.h b/public/VectorPairHMM/src/main/c++/define-double.h similarity index 100% rename from public/c++/VectorPairHMM/define-double.h rename to public/VectorPairHMM/src/main/c++/define-double.h diff --git a/public/c++/VectorPairHMM/define-float.h b/public/VectorPairHMM/src/main/c++/define-float.h similarity index 100% rename from public/c++/VectorPairHMM/define-float.h rename to public/VectorPairHMM/src/main/c++/define-float.h diff --git a/public/c++/VectorPairHMM/define-sse-double.h b/public/VectorPairHMM/src/main/c++/define-sse-double.h similarity index 100% rename from public/c++/VectorPairHMM/define-sse-double.h rename to public/VectorPairHMM/src/main/c++/define-sse-double.h diff --git a/public/c++/VectorPairHMM/define-sse-float.h b/public/VectorPairHMM/src/main/c++/define-sse-float.h similarity index 100% rename from public/c++/VectorPairHMM/define-sse-float.h rename to public/VectorPairHMM/src/main/c++/define-sse-float.h diff --git a/public/c++/VectorPairHMM/headers.h b/public/VectorPairHMM/src/main/c++/headers.h similarity index 100% rename from public/c++/VectorPairHMM/headers.h rename to public/VectorPairHMM/src/main/c++/headers.h diff --git a/public/c++/VectorPairHMM/jni_common.h b/public/VectorPairHMM/src/main/c++/jni_common.h similarity index 100% rename from public/c++/VectorPairHMM/jni_common.h rename to public/VectorPairHMM/src/main/c++/jni_common.h diff --git a/public/c++/VectorPairHMM/jnidebug.h b/public/VectorPairHMM/src/main/c++/jnidebug.h similarity index 100% rename from public/c++/VectorPairHMM/jnidebug.h rename to public/VectorPairHMM/src/main/c++/jnidebug.h diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc similarity index 100% rename from public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc rename to public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h similarity index 100% rename from public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h rename to public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc similarity index 100% rename from public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc rename to public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h similarity index 100% rename from public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h rename to public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h diff --git a/public/c++/VectorPairHMM/pairhmm-1-base.cc b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc similarity index 100% rename from public/c++/VectorPairHMM/pairhmm-1-base.cc rename to public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc diff --git a/public/c++/VectorPairHMM/pairhmm-template-kernel.cc b/public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc similarity index 100% rename from public/c++/VectorPairHMM/pairhmm-template-kernel.cc rename to public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc diff --git a/public/c++/VectorPairHMM/pairhmm-template-main.cc b/public/VectorPairHMM/src/main/c++/pairhmm-template-main.cc similarity index 100% rename from public/c++/VectorPairHMM/pairhmm-template-main.cc rename to public/VectorPairHMM/src/main/c++/pairhmm-template-main.cc diff --git a/public/c++/VectorPairHMM/run.sh b/public/VectorPairHMM/src/main/c++/run.sh similarity index 100% rename from public/c++/VectorPairHMM/run.sh rename to public/VectorPairHMM/src/main/c++/run.sh diff --git a/public/c++/VectorPairHMM/shift_template.c b/public/VectorPairHMM/src/main/c++/shift_template.c similarity index 100% rename from public/c++/VectorPairHMM/shift_template.c rename to public/VectorPairHMM/src/main/c++/shift_template.c diff --git a/public/c++/VectorPairHMM/sse_function_instantiations.cc b/public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc similarity index 100% rename from public/c++/VectorPairHMM/sse_function_instantiations.cc rename to public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc diff --git a/public/c++/VectorPairHMM/template.h b/public/VectorPairHMM/src/main/c++/template.h similarity index 100% rename from public/c++/VectorPairHMM/template.h rename to public/VectorPairHMM/src/main/c++/template.h diff --git a/public/c++/VectorPairHMM/utils.cc b/public/VectorPairHMM/src/main/c++/utils.cc similarity index 100% rename from public/c++/VectorPairHMM/utils.cc rename to public/VectorPairHMM/src/main/c++/utils.cc diff --git a/public/c++/VectorPairHMM/utils.h b/public/VectorPairHMM/src/main/c++/utils.h similarity index 100% rename from public/c++/VectorPairHMM/utils.h rename to public/VectorPairHMM/src/main/c++/utils.h diff --git a/public/c++/VectorPairHMM/vector_defs.h b/public/VectorPairHMM/src/main/c++/vector_defs.h similarity index 100% rename from public/c++/VectorPairHMM/vector_defs.h rename to public/VectorPairHMM/src/main/c++/vector_defs.h diff --git a/public/c++/VectorPairHMM/vector_function_prototypes.h b/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h similarity index 100% rename from public/c++/VectorPairHMM/vector_function_prototypes.h rename to public/VectorPairHMM/src/main/c++/vector_function_prototypes.h From 2d0ce45bb056ea32321c01587b6ae8ddce372d52 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Thu, 27 Feb 2014 10:12:23 -0800 Subject: [PATCH 49/72] Moved JNI_README --- .../{src/main/c++/JNI_README => README.md} | 25 +++++++++----- public/VectorPairHMM/pom.xml | 34 +++---------------- public/VectorPairHMM/src/main/c++/run.sh | 4 +-- 3 files changed, 22 insertions(+), 41 deletions(-) rename public/VectorPairHMM/{src/main/c++/JNI_README => README.md} (62%) diff --git a/public/VectorPairHMM/src/main/c++/JNI_README b/public/VectorPairHMM/README.md similarity index 62% rename from public/VectorPairHMM/src/main/c++/JNI_README rename to public/VectorPairHMM/README.md index 3eb797ab6..69ef76c50 100644 --- a/public/VectorPairHMM/src/main/c++/JNI_README +++ b/public/VectorPairHMM/README.md @@ -9,9 +9,9 @@ function implementation that is supported on the machine on which the program is run. The two pointers are for float and double respectively. This initialization is done only once for the whole program. 3. initialize(): To initialized the region for PairHMM. Pass haplotype bases to native -code through the JNIHaplotypeDataHolders class. Since the haplotype list is common across -multiple samples in computeReadLikelihoods(), we can store the haplotype bases to the -native code once and re-use across multiple samples. +code through the JNIHaplotypeDataHolders class. Since the haplotype list is common across multiple +samples in computeReadLikelihoods(), we can store the haplotype bases to the native code once and +re-use across multiple samples. 4. computeLikelihoods(): Copies array references for readBases/quals etc to array of JNIReadDataHolder objects. Invokes the JNI function to perform the computation and updates the likelihoodMap. @@ -33,9 +33,16 @@ support all AVX intrinsics. Type 'make'. This should create a library called libVectorLoglessPairHMM.so Running: -If libVectorLoglessPairHMM.so is compiled using icc, make sure that the Intel Composer XE -libraries are in your LD_LIBRARY_PATH : -source /bin/compilervars.sh intel64 -See run.sh in this directory on how to invoke HaplotypeCaller with the native library. The -argument -Djava.library.path is needed if the native implementation is selected, else -unnecessary. +The default implementation of PairHMM is still LOGLESS_CACHING in HaplotypeCaller.java. To use the +native version, use the command line argument "--pair_hmm_implementation VECTOR_LOGLESS_CACHING" +(see run.sh in src/main/c++). +The native library is bundled with the StingUtils jar file. When HaplotypeCaller is invoked with the +VectorLoglessPairHMM implementation (see run.sh in the directory src/main/c++), then the library is +unpacked from the jar file, copied to the /tmp directory (with a unique id) and loaded by the Java class +VectorLoglessPairHMM in the constructor (if it has not been loaded already). +The default library can be overridden by using the -Djava.library.path argument for the JVM to pass +the path to the library. If the library libVectorLoglessPairHMM.so can be found in +java.library.path, then it is loaded and the 'packed' library is not used. +See run.sh in this directory on how to invoke HaplotypeCaller with the vector implementation of +PairHMM. The argument -Djava.library.path is needed if you wish to override the default packed +library, else unnecessary. diff --git a/public/VectorPairHMM/pom.xml b/public/VectorPairHMM/pom.xml index b310148d6..41bb73211 100644 --- a/public/VectorPairHMM/pom.xml +++ b/public/VectorPairHMM/pom.xml @@ -13,7 +13,7 @@ pom Vectorized PairHMM native libraries - Builds a GNU/Linux x86_64 version of VectorPairHMM. During install, copies it into sting-utils. + Builds a GNU/Linux x86_64 library of VectorPairHMM using icc (Intel C++ compiler). During install, copies it into sting-utils. Neither tested nor expected to work on any other platform. UTF-8 @@ -23,20 +23,12 @@ ${sting.basedir}/public/sting-utils ${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm - - - - x86_64 - so - + + + org.apache.maven.plugins maven-enforcer-plugin @@ -64,7 +56,6 @@ make src/main/c++ - ${pairhmm.build.architecture} ${java.home} ${project.build.directory} @@ -125,21 +116,4 @@ - - - - - linux_x86_64 - - - linux - amd64 - - - - linux_x86_64 - - - - diff --git a/public/VectorPairHMM/src/main/c++/run.sh b/public/VectorPairHMM/src/main/c++/run.sh index b52dddc46..7cc240c9d 100755 --- a/public/VectorPairHMM/src/main/c++/run.sh +++ b/public/VectorPairHMM/src/main/c++/run.sh @@ -7,8 +7,8 @@ then pair_hmm_implementation=$1; fi -#-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed -#java -Djava.library.path=${GSA_ROOT_DIR}/public/c++/VectorPairHMM -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ +#-Djava.library.path is needed if you wish to override the default 'packed' library +#java -Djava.library.path=${GSA_ROOT_DIR}/public/VectorPairHMM/src/main/c++ -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ java -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ --dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ From 37526dfad510108281936c0c7c52887b1f193dd4 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Fri, 28 Feb 2014 08:59:55 -0800 Subject: [PATCH 50/72] 1. Added the catch UnsatisfiedLinkError exception in PairHMMLikelihoodCalculationEngine.java to fall back to LOGLESS_CACHING in case the native library could not be loaded. Made VECTOR_LOGLESS_CACHING as the default implementation. 2. Updated the README with Mauricio's comments 3. baseline.cc is used within the library - if the machine supports neither AVX nor SSE4.1, the native library falls back to un-vectorized C++ in baseline.cc. 4. pairhmm-1-base.cc: This is not part of the library, but is being heavily used for debugging/profiling. Can I request that we keep it there for now? In the next release, we can delete it from the repository. 5. I agree with Mauricio about the ifdefs. I am sure you already know, but just to reassure you the debug code is not compiled into the library (because of the ifdefs) and will not affect performance. --- .../haplotypecaller/HaplotypeCaller.java | 2 +- .../PairHMMLikelihoodCalculationEngine.java | 62 ++++++------ .../utils/pairhmm/VectorLoglessPairHMM.java | 7 +- public/VectorPairHMM/README.md | 97 ++++++++++++------- ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 7 +- 5 files changed, 106 insertions(+), 69 deletions(-) diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java index 7844741df..654359060 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java @@ -426,7 +426,7 @@ public class HaplotypeCaller extends ActiveRegionWalker, In */ @Hidden @Argument(fullName = "pair_hmm_implementation", shortName = "pairHMM", doc = "The PairHMM implementation to use for genotype likelihood calculations", required = false) - public PairHMM.HMM_IMPLEMENTATION pairHMM = PairHMM.HMM_IMPLEMENTATION.LOGLESS_CACHING; + public PairHMM.HMM_IMPLEMENTATION pairHMM = PairHMM.HMM_IMPLEMENTATION.VECTOR_LOGLESS_CACHING; @Hidden @Argument(fullName="keepRG", shortName="keepRG", doc="Only use read from this read group when making calls (but use all reads to build the assembly)", required = false) diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index 344366869..0c984b4c3 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -81,29 +81,37 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation private final boolean noFpga; private final ThreadLocal pairHMMThreadLocal = new ThreadLocal() { - @Override - protected PairHMM initialValue() { - switch (hmmType) { - case EXACT: return new Log10PairHMM(true); - case ORIGINAL: return new Log10PairHMM(false); - case LOGLESS_CACHING: - if (noFpga || !CnyPairHMM.isAvailable()) - return new LoglessPairHMM(); - else - return new CnyPairHMM(); - case VECTOR_LOGLESS_CACHING: - return new VectorLoglessPairHMM(); - case DEBUG_VECTOR_LOGLESS_CACHING: - return new DebugJNILoglessPairHMM(PairHMM.HMM_IMPLEMENTATION.VECTOR_LOGLESS_CACHING); - case ARRAY_LOGLESS: - if (noFpga || !CnyPairHMM.isAvailable()) - return new ArrayLoglessPairHMM(); - else - return new CnyPairHMM(); - default: - throw new UserException.BadArgumentValue("pairHMM", "Specified pairHMM implementation is unrecognized or incompatible with the HaplotypeCaller. Acceptable options are ORIGINAL, EXACT, CACHING, LOGLESS_CACHING, and ARRAY_LOGLESS."); - } - } + @Override + protected PairHMM initialValue() { + switch (hmmType) { + case EXACT: return new Log10PairHMM(true); + case ORIGINAL: return new Log10PairHMM(false); + case LOGLESS_CACHING: + if (noFpga || !CnyPairHMM.isAvailable()) + return new LoglessPairHMM(); + else + return new CnyPairHMM(); + case VECTOR_LOGLESS_CACHING: + try + { + return new VectorLoglessPairHMM(); + } + catch(UnsatisfiedLinkError ule) + { + logger.info("Failed to load native library for VectorLoglessPairHMM - using Java implementation of LOGLESS_CACHING"); + return new LoglessPairHMM(); + } + case DEBUG_VECTOR_LOGLESS_CACHING: + return new DebugJNILoglessPairHMM(PairHMM.HMM_IMPLEMENTATION.VECTOR_LOGLESS_CACHING); + case ARRAY_LOGLESS: + if (noFpga || !CnyPairHMM.isAvailable()) + return new ArrayLoglessPairHMM(); + else + return new CnyPairHMM(); + default: + throw new UserException.BadArgumentValue("pairHMM", "Specified pairHMM implementation is unrecognized or incompatible with the HaplotypeCaller. Acceptable options are ORIGINAL, EXACT, CACHING, LOGLESS_CACHING, and ARRAY_LOGLESS."); + } + } }; // Attempted to do as below, to avoid calling pairHMMThreadLocal.get() later on, but it resulted in a NullPointerException // private final PairHMM pairHMM = pairHMMThreadLocal.get(); @@ -340,7 +348,7 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation private void finalizePairHMM() { - pairHMMThreadLocal.get().finalizeRegion(); + pairHMMThreadLocal.get().finalizeRegion(); } @@ -360,9 +368,9 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation map.filterPoorlyModelledReads(EXPECTED_ERROR_RATE_PER_BASE); stratifiedReadMap.put(sampleEntry.getKey(), map); } - - //Used mostly by the JNI implementation(s) to free arrays - finalizePairHMM(); + + //Used mostly by the JNI implementation(s) to free arrays + finalizePairHMM(); return stratifiedReadMap; } diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java index 50600d850..a32768c20 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java @@ -97,8 +97,9 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { /** * Return 64-bit mask representing machine capabilities * Bit 0 is LSB, bit 63 MSB - * Bit 0 represents sse4.2 availability - * Bit 1 represents AVX availability + * Bit 0 represents sse4.1 availability + * Bit 1 represents sse4.2 availability + * Bit 2 represents AVX availability */ public native long jniGetMachineType(); @@ -125,6 +126,7 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { //Useful if someone builds his/her own library and wants to override the bundled //implementation without modifying the Java code System.loadLibrary("VectorLoglessPairHMM"); + logger.info("libVectorLoglessPairHMM found in JVM library path"); } catch(UnsatisfiedLinkError ule) { @@ -133,6 +135,7 @@ public class VectorLoglessPairHMM extends JNILoglessPairHMM { { logger.info("libVectorLoglessPairHMM not found in JVM library path - trying to unpack from StingUtils.jar"); loadLibraryFromJar("/org/broadinstitute/sting/utils/pairhmm/libVectorLoglessPairHMM.so"); + logger.info("libVectorLoglessPairHMM unpacked successfully from StingUtils.jar"); } catch(IOException ioe) { diff --git a/public/VectorPairHMM/README.md b/public/VectorPairHMM/README.md index 69ef76c50..85cc0a04a 100644 --- a/public/VectorPairHMM/README.md +++ b/public/VectorPairHMM/README.md @@ -2,47 +2,70 @@ Implementation overview: Created a new Java class called VectorLoglessPairHMM which extends LoglessPairHMM and overrides functions from both LoglessPairHMM and PairHMM. 1. Constructor: Call base class constructors. Then, load the native library located in this -directory and call a global init function in the library to determine fields ids for the -members of classes JNIReadDataHolder and JNIHaplotypeDataHolders. -2. When the library is loaded, it initializes two global function pointers to point to the -function implementation that is supported on the machine on which the program is being -run. The two pointers are for float and double respectively. This initialization is done -only once for the whole program. -3. initialize(): To initialized the region for PairHMM. Pass haplotype bases to native -code through the JNIHaplotypeDataHolders class. Since the haplotype list is common across multiple -samples in computeReadLikelihoods(), we can store the haplotype bases to the native code once and -re-use across multiple samples. +directory and call an init function (with suffix 'jniInitializeClassFieldsAndMachineMask') in the +library to determine fields ids for the members of classes JNIReadDataHolder and +JNIHaplotypeDataHolders. The native code stores the field ids (struct offsets) for the classes and +re-uses them for subsequent computations. Optionally, the user can disable the vector +implementation, by using the 'mask' argument (see comments for a more detailed explanation). +2. When the library is loaded, it invokes the constructor of the class LoadTimeInitializer (because +a global variable g_load_time_initializer is declared in the library). This constructor +(LoadTimeInitializer.cc) can be used to perform various initializations. Currently, it initializes +two global function pointers to point to the function implementation that is supported on the +machine (AVX/SSE/un-vectorized) on which the program is being run. The two pointers are for float +and double respectively. The global function pointers are declared in utils.cc and are assigned in +the function initialize_function_pointers() defined in utils.cc and invoked from the constructor of +LoadTimeInitializer. +Other initializations in LoadTimeInitializer: +* ConvertChar::init - sets some masks for the vector implementation +* FTZ for performance +* stat counters = 0 +* debug structs (which are never used in non-debug mode) +This initialization is done only once for the whole program. +3. initialize(): To initialize the region for PairHMM. Pass haplotype bases to native code through +the JNIHaplotypeDataHolder class. Since the haplotype list is common across multiple samples in +computeReadLikelihoods(), we can pass the haplotype bases to the native code once and re-use across +multiple samples. 4. computeLikelihoods(): Copies array references for readBases/quals etc to array of -JNIReadDataHolder objects. Invokes the JNI function to perform the computation and -updates the likelihoodMap. +JNIReadDataHolder objects. Invokes the JNI function to perform the computation and updates the +likelihoodMap. +The JNI function copies the byte array references into an array of testcase structs and invokes the +compute_full_prob function through the function pointers initialized earlier. +The primary native function called is +Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods. It uses +standard JNI calls to get and return data from/to the Java class VectorLoglessPairHMM. The last +argument to the function is the maximum number of OpenMP threads to use while computing PairHMM in +C++. This option is set when the native function call is made from JNILoglessPairHMM +computeLikelihoods - currently it is set to 12 (no logical reason). +Note: OpenMP has been disabled for now - insufficient #testcases per call to computeLikelihoods() to +justify multi-threading. +5. finalizeRegion(): Releases the haplotype arrays initialized in step 3 - should be called at the +end of every region (line 351 in PairHMMLikelihoodCalculationEngine). Note: Debug code has been moved to a separate class DebugJNILoglessPairHMM.java. -On the C++ side, the primary function called is -Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods. It -uses standard JNI calls to get and return data from/to the Java class -VectorLoglessPairHMM. The last argument to the function is the maximum number of OpenMP -threads to use while computing PairHMM in C++. This option is set when the native function -call is made from JNILoglessPairHMM computeLikelihoods - currently it is set to 12 (no -logical reason). -Note: OpenMP has been disabled for now. - Compiling: -Make sure you have icc (Intel C compiler) available. Currently, gcc does not seem to -support all AVX intrinsics. -Type 'make'. This should create a library called libVectorLoglessPairHMM.so +Make sure you have icc (Intel C compiler) available. Currently, gcc does not seem to support all AVX +intrinsics. +This native library is called libVectorLoglessPairHMM.so +Using Maven: +Type 'mvn install' in this directory - this will build the library (by invoking 'make') and copy the +native library to the directory +${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm +The GATK maven build process (when run) will bundle the library into the StingUtils jar file from +the copied directory. +Simple build: +cd src/main/c++ +make Running: -The default implementation of PairHMM is still LOGLESS_CACHING in HaplotypeCaller.java. To use the -native version, use the command line argument "--pair_hmm_implementation VECTOR_LOGLESS_CACHING" -(see run.sh in src/main/c++). -The native library is bundled with the StingUtils jar file. When HaplotypeCaller is invoked with the -VectorLoglessPairHMM implementation (see run.sh in the directory src/main/c++), then the library is -unpacked from the jar file, copied to the /tmp directory (with a unique id) and loaded by the Java class -VectorLoglessPairHMM in the constructor (if it has not been loaded already). -The default library can be overridden by using the -Djava.library.path argument for the JVM to pass -the path to the library. If the library libVectorLoglessPairHMM.so can be found in -java.library.path, then it is loaded and the 'packed' library is not used. -See run.sh in this directory on how to invoke HaplotypeCaller with the vector implementation of -PairHMM. The argument -Djava.library.path is needed if you wish to override the default packed -library, else unnecessary. +The default implementation of PairHMM is now VECTOR_LOGLESS_CACHING in HaplotypeCaller.java. To use +the Java version, use the command line argument "--pair_hmm_implementation LOGLESS_CACHING". (see +run.sh in src/main/c++). +The native library is bundled with the StingUtils jar file. When HaplotypeCaller is invoked, then +the library is unpacked from the jar file, copied to the /tmp directory (with a unique id) and +loaded by the Java class VectorLoglessPairHMM in the constructor (if it has not been loaded +already). +The default library can be overridden by using the -Djava.library.path argument (see +src/main/c++/run.sh for an example) for the JVM to pass the path to the library. If the library +libVectorLoglessPairHMM.so can be found in java.library.path, then it is loaded and the 'packed' +library is not used. diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index cd1656e7c..c28c24df1 100644 --- a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -98,6 +98,8 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless } } +//Create a vector of testcases for computation - copy the references to bytearrays read/readQuals etc into the appropriate +//testcase struct inline JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeTestcasesVector (JNIEnv* env, jint numReads, jint numHaplotypes, jobjectArray& readDataArray, vector > >& readBasesArrayVector, vector& tc_array) @@ -272,12 +274,13 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless g_load_time_initializer.m_data_transfer_time += diff_time(start_time); #endif + //Get double array where results are stored (to pass back to java) jdouble* likelihoodDoubleArray = (jdouble*)GET_DOUBLE_ARRAY_ELEMENTS(likelihoodArray, &is_copy); #ifdef ENABLE_ASSERTIONS assert(likelihoodDoubleArray && "likelihoodArray is NULL"); assert(env->GetArrayLength(likelihoodArray) == numTestCases); #endif -#ifdef DO_WARMUP +#ifdef DO_WARMUP //ignore - only for crazy profiling vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; for(unsigned i=0;i Date: Fri, 28 Feb 2014 16:08:28 -0800 Subject: [PATCH 51/72] 1. Changed logger.info to logger.warn in PairHMMLikelihoodCalculationEngine.java 2. Committing the right set of files after rebase --- .../PairHMMLikelihoodCalculationEngine.java | 2 +- .../utils/pairhmm/DebugJNILoglessPairHMM.java | 16 +- .../src/main/c++/LoadTimeInitializer.cc | 5 +- .../src/main/c++/LoadTimeInitializer.h | 5 +- public/VectorPairHMM/src/main/c++/Makefile | 2 +- public/VectorPairHMM/src/main/c++/baseline.cc | 5 +- .../src/main/c++/pairhmm-template-kernel.cc | 3 +- public/VectorPairHMM/src/main/c++/run.sh | 4 +- public/VectorPairHMM/src/main/c++/template.h | 173 ++++++++++++++---- public/VectorPairHMM/src/main/c++/utils.cc | 11 +- .../utils/pairhmm/libVectorLoglessPairHMM.so | Bin 388054 -> 443803 bytes 11 files changed, 165 insertions(+), 61 deletions(-) diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index 0c984b4c3..0f2c61290 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -98,7 +98,7 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation } catch(UnsatisfiedLinkError ule) { - logger.info("Failed to load native library for VectorLoglessPairHMM - using Java implementation of LOGLESS_CACHING"); + logger.warn("Failed to load native library for VectorLoglessPairHMM - using Java implementation of LOGLESS_CACHING"); return new LoglessPairHMM(); } case DEBUG_VECTOR_LOGLESS_CACHING: diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java index beaca115a..ea93ebe4a 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java @@ -449,21 +449,7 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { }) @Ensures("constantsAreInitialized") protected static void initializeProbabilities(final double[][] transition, final byte[] insertionGOP, final byte[] deletionGOP, final byte[] overallGCP) { - for (int i = 0; i < insertionGOP.length; i++) { - final int qualIndexGOP = Math.min(insertionGOP[i] + deletionGOP[i], Byte.MAX_VALUE); - transition[i+1][matchToMatch] = QualityUtils.qualToProb((byte) qualIndexGOP); - transition[i+1][indelToMatch] = QualityUtils.qualToProb(overallGCP[i]); - transition[i+1][matchToInsertion] = QualityUtils.qualToErrorProb(insertionGOP[i]); - transition[i+1][insertionToInsertion] = QualityUtils.qualToErrorProb(overallGCP[i]); - transition[i+1][matchToDeletion] = QualityUtils.qualToErrorProb(deletionGOP[i]); - transition[i+1][deletionToDeletion] = QualityUtils.qualToErrorProb(overallGCP[i]); - - //TODO it seems that it is not always the case that matchToMatch + matchToDeletion + matchToInsertion == 1. - //TODO We have detected cases of 1.00002 which can cause problems downstream. This are typically masked - //TODO by the fact that we always add a indelToMatch penalty to all PairHMM likelihoods (~ -0.1) - //TODO This is in fact not well justified and although it does not have any effect (since is equally added to all - //TODO haplotypes likelihoods) perhaps we should just remove it eventually and fix this != 1.0 issue here. - } + PairHMMModel.qualToTransProbs(transition,insertionGOP,deletionGOP,overallGCP); } /** diff --git a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc index 0fec4c8da..3aacefe72 100644 --- a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc +++ b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc @@ -14,7 +14,6 @@ char* LoadTimeInitializerStatsNames[] = "dummy" }; - LoadTimeInitializer g_load_time_initializer; LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loaded @@ -54,6 +53,10 @@ LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loa initialize_function_pointers(); + //Initialize static members of class + Context::initializeStaticMembers(); + Context::initializeStaticMembers(); + cout.flush(); } diff --git a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h index 778ae221f..1834405a5 100644 --- a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h +++ b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h @@ -22,7 +22,10 @@ class LoadTimeInitializer { public: LoadTimeInitializer(); //will be called when library is loaded - ~LoadTimeInitializer() { delete m_buffer; } + ~LoadTimeInitializer() + { + delete m_buffer; + } void print_profiling(); void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); void debug_close(); diff --git a/public/VectorPairHMM/src/main/c++/Makefile b/public/VectorPairHMM/src/main/c++/Makefile index e1e443d2e..da91149e9 100644 --- a/public/VectorPairHMM/src/main/c++/Makefile +++ b/public/VectorPairHMM/src/main/c++/Makefile @@ -4,7 +4,7 @@ #CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas #CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas -JRE_HOME?=/opt/jdk1.7.0_25/ +JRE_HOME?=/opt/jdk1.7.0_25/jre JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JRE_HOME}/../include -I${JRE_HOME}/../include/linux COMMON_COMPILATION_FLAGS=$(JNI_COMPILATION_FLAGS) -O3 -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas diff --git a/public/VectorPairHMM/src/main/c++/baseline.cc b/public/VectorPairHMM/src/main/c++/baseline.cc index 290a3a4a2..66aeacbc4 100644 --- a/public/VectorPairHMM/src/main/c++/baseline.cc +++ b/public/VectorPairHMM/src/main/c++/baseline.cc @@ -2,8 +2,8 @@ #include "template.h" #include "utils.h" #include "LoadTimeInitializer.h" - using namespace std; + template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) { @@ -62,7 +62,8 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) int _i = tc->i[r-1] & 127; int _d = tc->d[r-1] & 127; int _c = tc->c[r-1] & 127; - p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + //p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(p[r][MM], _i, _d); p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; p[r][MX] = ctx.ph2pr[_i]; p[r][XX] = ctx.ph2pr[_c]; diff --git a/public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc b/public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc index a2111b6b6..77394f457 100644 --- a/public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc +++ b/public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc @@ -116,7 +116,8 @@ template void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECIS int _d = tc->d[r-1] & 127; int _c = tc->c[r-1] & 127; - *(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + //*(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(*(ptr_p_MM+r-1), _i, _d); *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; diff --git a/public/VectorPairHMM/src/main/c++/run.sh b/public/VectorPairHMM/src/main/c++/run.sh index 7cc240c9d..e821497f1 100755 --- a/public/VectorPairHMM/src/main/c++/run.sh +++ b/public/VectorPairHMM/src/main/c++/run.sh @@ -8,8 +8,8 @@ then fi #-Djava.library.path is needed if you wish to override the default 'packed' library -#java -Djava.library.path=${GSA_ROOT_DIR}/public/VectorPairHMM/src/main/c++ -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ -java -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ +#java -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ +java -Djava.library.path=${GSA_ROOT_DIR}/public/VectorPairHMM/src/main/c++ -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ --dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ -I /data/simulated/sim1M_pairs_final.bam \ diff --git a/public/VectorPairHMM/src/main/c++/template.h b/public/VectorPairHMM/src/main/c++/template.h index f91c2300e..3e52854a6 100644 --- a/public/VectorPairHMM/src/main/c++/template.h +++ b/public/VectorPairHMM/src/main/c++/template.h @@ -94,57 +94,158 @@ typedef union ALIGNED ALIGNED double ALIGNED f; } ALIGNED IF_64 ALIGNED; -template -struct Context{}; + +#define MAX_QUAL 254 +#define MAX_JACOBIAN_TOLERANCE 8.0 +#define JACOBIAN_LOG_TABLE_STEP 0.0001 +#define JACOBIAN_LOG_TABLE_INV_STEP (1.0 / JACOBIAN_LOG_TABLE_STEP) +#define MAXN 70000 +#define LOG10_CACHE_SIZE (4*MAXN) // we need to be able to go up to 2*(2N) when calculating some of the coefficients +#define JACOBIAN_LOG_TABLE_SIZE ((int) (MAX_JACOBIAN_TOLERANCE / JACOBIAN_LOG_TABLE_STEP) + 1) + +template +struct ContextBase +{ + public: + NUMBER ph2pr[128]; + NUMBER INITIAL_CONSTANT; + NUMBER LOG10_INITIAL_CONSTANT; + NUMBER RESULT_THRESHOLD; + + static bool staticMembersInitializedFlag; + static NUMBER jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; + static NUMBER matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; + + static void initializeStaticMembers() + { + if(!staticMembersInitializedFlag) + { + //Order of calls important - Jacobian first, then MatchToMatch + initializeJacobianLogTable(); + initializeMatchToMatchProb(); + staticMembersInitializedFlag = true; + } + } + + static void deleteStaticMembers() + { + if(staticMembersInitializedFlag) + { + staticMembersInitializedFlag = false; + } + } + + //Called only once during library load - don't bother to optimize with single precision fp + static void initializeJacobianLogTable() + { + for (int k = 0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { + jacobianLogTable[k] = (NUMBER)(log10(1.0 + pow(10.0, -((double) k) * JACOBIAN_LOG_TABLE_STEP))); + } + } + + //Called only once per library load - don't bother optimizing with single fp + static void initializeMatchToMatchProb() + { + double LN10 = log(10); + double INV_LN10 = 1.0/LN10; + for (int i = 0, offset = 0; i <= MAX_QUAL; offset += ++i) + for (int j = 0; j <= i; j++) { + double log10Sum = approximateLog10SumLog10(-0.1*i, -0.1*j); + double matchToMatchLog10 = + log1p(-std::min(1.0,pow(10,log10Sum))) * INV_LN10; + matchToMatchProb[offset + j] = (NUMBER)(pow(10,matchToMatchLog10)); + } + } + //Called during computation - use single precision where possible + static int fastRound(NUMBER d) { + return (d > ((NUMBER)0.0)) ? (int) (d + ((NUMBER)0.5)) : (int) (d - ((NUMBER)0.5)); + } + //Called during computation - use single precision where possible + static NUMBER approximateLog10SumLog10(NUMBER small, NUMBER big) { + // make sure small is really the smaller value + if (small > big) { + NUMBER t = big; + big = small; + small = t; + } + + if (isinf(small) == -1 || isinf(big) == -1) + return big; + + NUMBER diff = big - small; + if (diff >= ((NUMBER)MAX_JACOBIAN_TOLERANCE)) + return big; + + // OK, so |y-x| < tol: we use the following identity then: + // we need to compute log10(10^x + 10^y) + // By Jacobian logarithm identity, this is equal to + // max(x,y) + log10(1+10^-abs(x-y)) + // we compute the second term as a table lookup with integer quantization + // we have pre-stored correction for 0,0.1,0.2,... 10.0 + int ind = fastRound((NUMBER)(diff * ((NUMBER)JACOBIAN_LOG_TABLE_INV_STEP))); // hard rounding + return big + jacobianLogTable[ind]; + } +}; + +template +struct Context : public ContextBase +{}; template<> -struct Context +struct Context : public ContextBase { - Context() - { - for (int x = 0; x < 128; x++) - ph2pr[x] = pow(10.0, -((double)x) / 10.0); + Context():ContextBase() + { + for (int x = 0; x < 128; x++) + ph2pr[x] = pow(10.0, -((double)x) / 10.0); - INITIAL_CONSTANT = ldexp(1.0, 1020.0); - LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); - RESULT_THRESHOLD = 0.0; - } + INITIAL_CONSTANT = ldexp(1.0, 1020.0); + LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); + RESULT_THRESHOLD = 0.0; + } - double LOG10(double v){ return log10(v); } + double LOG10(double v){ return log10(v); } + inline double POW(double b, double e) { return pow(b,e); } - static double _(double n){ return n; } - static double _(float n){ return ((double) n); } - double ph2pr[128]; - double INITIAL_CONSTANT; - double LOG10_INITIAL_CONSTANT; - double RESULT_THRESHOLD; + static double _(double n){ return n; } + static double _(float n){ return ((double) n); } }; template<> -struct Context +struct Context : public ContextBase { - Context() - { - for (int x = 0; x < 128; x++) - { - ph2pr[x] = powf(10.f, -((float)x) / 10.f); - } + Context() : ContextBase() + { + for (int x = 0; x < 128; x++) + { + ph2pr[x] = powf(10.f, -((float)x) / 10.f); + } - INITIAL_CONSTANT = ldexpf(1.f, 120.f); - LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); - RESULT_THRESHOLD = ldexpf(1.f, -110.f); - } + INITIAL_CONSTANT = ldexpf(1.f, 120.f); + LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); + RESULT_THRESHOLD = ldexpf(1.f, -110.f); + } - float LOG10(float v){ return log10f(v); } + float LOG10(float v){ return log10f(v); } + inline float POW(float b, float e) { return powf(b,e); } - static float _(double n){ return ((float) n); } - static float _(float n){ return n; } - float ph2pr[128]; - float INITIAL_CONSTANT; - float LOG10_INITIAL_CONSTANT; - float RESULT_THRESHOLD; + static float _(double n){ return ((float) n); } + static float _(float n){ return n; } }; +#define SET_MATCH_TO_MATCH_PROB(output, insQual, delQual) \ +{ \ + int minQual = delQual; \ + int maxQual = insQual; \ + if (insQual <= delQual) \ + { \ + minQual = insQual; \ + maxQual = delQual; \ + } \ + (output) = (MAX_QUAL < maxQual) ? \ + ((NUMBER)1.0) - ctx.POW(((NUMBER)10), ctx.approximateLog10SumLog10(((NUMBER)-0.1)*minQual, ((NUMBER)-0.1)*maxQual)) \ + : ctx.matchToMatchProb[((maxQual * (maxQual + 1)) >> 1) + minQual]; \ +} typedef struct { diff --git a/public/VectorPairHMM/src/main/c++/utils.cc b/public/VectorPairHMM/src/main/c++/utils.cc index 387a6b4a0..35871948c 100644 --- a/public/VectorPairHMM/src/main/c++/utils.cc +++ b/public/VectorPairHMM/src/main/c++/utils.cc @@ -3,12 +3,21 @@ #include "utils.h" #include "vector_defs.h" #include "LoadTimeInitializer.h" +using namespace std; +//static members from ConvertChar uint8_t ConvertChar::conversionTable[255]; +//Global function pointers in utils.h float (*g_compute_full_prob_float)(testcase *tc, float* before_last_log) = 0; double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log) = 0; +//Static members in ContextBase +bool ContextBase::staticMembersInitializedFlag = false; +double ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +double ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; +bool ContextBase::staticMembersInitializedFlag = false; +float ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +float ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; -using namespace std; bool is_avx_supported() { diff --git a/public/sting-utils/src/main/resources/org/broadinstitute/sting/utils/pairhmm/libVectorLoglessPairHMM.so b/public/sting-utils/src/main/resources/org/broadinstitute/sting/utils/pairhmm/libVectorLoglessPairHMM.so index 07fcc79d041fa0459f608a5618fe9cffb6495398..17cec8be400bc0847030f66d645762ac3c3e3ea2 100644 GIT binary patch delta 120747 zcma&P3tZFH|3AJBFjR0uLGeBm3@_QdeL_)V3N>`tDhzKaikg}!c}q!8 zI^5_(d(+&=2C79Wb|vsYmDQ)#yY0JKzlyBI2w$jD=^0kzlOzQ~UJ%)a@xIW~ltGoQ zIGjI-cp4yz=V{LEdYbcizOa-G;}YS!$CIPNM-@Aty8P+)D?} zGtZm+(QheVj{W6@#T4x^7;Gzy?gbveZ1InG5qyKY68As2-@zRN-4+P{z)jZz+;?%m zk2@ImcHHr}=isKR0QYAql~r{H565xW7m49f^*8Q9+#?A{S4*K2f-qD(=RtlCcSGR$ z2>0Wjgs)8qa` z@D%)jyEpDY0@9^}-e16%CGISqTS{019pCi^9!}u?Q>7ZkxC(sU$K3=xOd@>gwAh7NY_#ETudRQui$PB z#TO8IaMN`I_hH=SxQF2mhWv5dbfw^KjprhSEf5Ca-il{C?o`}r;0p-pnu+@o?mM_Y z!tKUA0k`jp+JL8*aZkn_frtl%WEsLnxL4+#IIS6>){(R1d8P&hDyM@Il&-ZRl}vSr zlKEJ$Qdlb_w1J34!DCQu8Ty38)FMXR2cHeX>Vj`2WX~cDCBtEDByD;eM7q|tg6$!~ z<_P1&voDAe4`N zJhwymp?EeRG~u2uaE=K3BkU!f!x1{fvoGi=9zOOF2m^30#+`**T#CI`?J+bg)E2~4 zgd1^Z;$ADb!3ZP7^NR?V;?BkWH109DJK*k!`wQIZxanGfdj{?&aleFn2UUKQ8Bfb` z)AcD2*rx`cZ{q$%IT6&wdJIo`+!?ri*XwvVi8~wjQ-b^o;VZbm7T6d3A0A%CP5pqb zTez37Cza|gght$xaJR?(8g9C7<1WK}o{xn@sZ`r|plT>W3O3=c1)rWLxH<>}fj1-6 zBFqG-ioTX%<;*1cmm@{D^*>U{wl{hAx=f-AwGe6 zDqWBBAS9Ucr9X1))4zuU?t7kzN-kDa3Zn%*HOiZHd)0A7i9{5oxNe(n+ZuD| z6yvN@L@PrF3R5*!CH&5NWVPULP%fzcf`+ydPo1VT;Wn~pvcTnJpP0pj?R1*baCvy3 z+K{*AgP=xAZb+~-wkz!6qV7tU{xNRKvyx+9>Gp+i-ms7Js(L*96#>iUA)ZatIJ?$} zQfF22G_wLZzgARY?n|uBJJeKPEP5feVZl1C=u7yQ7@mOTXb+`t=o6i@u0JfN>4-9r zB{EP5J9ItMnrAq7kFqv&(hDO1!&IRvPh%k$8Qq{2uG5d_Q!hGvf0U10BTO zn0<)Xdz4Z2dbesLQmHKB(G6T%6%F@i9=1&>uGcZj>k%F%#9k52a9BiBZq<9LRY*&o zm#i(E^;N+%jKx$nnOl?>v>p3d{A!>Ms+4MA&{5t5qNY{xc%W!m8+fWdbK6lA;+^Z1 zJ=$li2}ij&LFDZ)+$s$IklCJlhOb4#Q8QGe0^V{eUr`q}6?`Vobpj8_2-8mx`W9hd zgVLZM`!40lhPACaQ6EmfYAg`8bANldMjjUqpL_6O{tv;I9(b6)=WC`zoKFy`Uqqip zUh>|4KPXJ8(na>j7QIe?y$e$bJjrWwndnIgU-5?gt{u1Vy3b9HDxJeRK2h);r+rmj zAO@z~sUkl}kFFO`$E*|gs1Sw=XiKh>3+7$+tA~Sv)TKk7tl_G_2#Obm0qn z5iLWA^`g>!(TYVymQLZxIEB6zDNv)P|1L5pcvTeVGjo**_1n~GBh0E^Qa-5Prj>zJ zTV0js10KuQp6_BppiEa@Q%=_JXtnI+)_j>bhcT7J>3+4;A9YHZsCbX$P zI|Z*7{1a$d($5^k?HC0Av=}E7_AsJ-&G@z!;}}J-B=QL9sCl}+7ZE%!DYL`d)cII= za6&U>LwLs~Sstc@RW%PUitOiRDp$hWSicuA->jvKpemIWXyU(>QiN2^317?9pZLt`a6TakxuiM0nR5JhI@hu{iJmEQ+ z+|^Y1!cI4yE;>&)-fF=vKBO{7&YRE2{N9Q-{J|}8Rx07eQKxDkH&^Nd^?`wulfufnk_}~Gi%OY~?5q*tD zRl35_Z0PUG;kscdw}j1HzZs;1D3Rd?kCRY~hrb}CJnL6-?Bs#!ElP`)S9Q)Jx8!vB z8W!^;kvZ2NN60yP0*VkE~^+g*gH3{Z-A&TS~Fct{T5|p z0~Q&*!cQ-+kV%wDm1M>aIWYk9xM>(ig}zC(U+@LVkyd5yj3bu4I8x@DC$?WMkFX5zsBl@ z+)#E6N4XhaWqikyk5r}m4YMBKT(?(g(!7mztndxJ8g_7H3LpDREQ0jIcxDPjyT2<& z-Av*8zU1CR-BVA?68@0Dtf^E%EqJ!|UvoPabYZ%_zQyhW>PQM~w;{aoFbpI1ar_Uu)zH1BBjEaK*U;`hYlRNyx` zJ??8{(a{q`uI7tcOZRiRPNEw)MH>rs)ex~l)bq8g@^*{1R*PTUc(f3;T!B~-W$_ZI z{y_n$21?gGoaZG$oVSRP)>oG0K07pC;L2y8D%D`Ip78K8HmOC}hQpzLfl};^N zs$W(*g)~yewKTLU6P1t~!hOOcYP>E|WKDBP+1j#A^WQ|e`F<<9{@}xC7*urnU}OGX zo*li&)g)oO@{GtWPeauXj!5mR8YNQuACGWWncujP^|(kY*DtL(zE~fKSZG(eW(M-C z=5FBFS678TeO(X}B#6$v6XP8ftw7A7Sv;Dm1#jP`{~G61$UQY)0S;CEpR#v_2t9jr1tEhR0aw4>mqHh)4HdA<3f=E71Sn}-SmV9f0 zEM-mIMy-B`q87slBeG)>vHpNv%CxRfS=_2)hE6QzeNB*#mLjo5G^IgA`wUG?=H`e2 z3oXd5t$16bs>-^=aW3N1^&>LJ){EFq&$lV3?4IcnmVB-Mx^13Go; zJ9*MWYOjr)mN71MatZ~ODU(P2huYZxf7Zr^L+@Ya@}|3kqDTKn(w*a`P8&IGd`8Nc z*G7$*GBtV9_-W%uP8>gb%;;w(j(p`mv+@5OMY$25svR+P#%mKtWKMpg>xkDzPJLB* zr&pws-gBw0h94VOBgi}I4r;uqc`vP664%6Qn)m&$LE%c%UXAmdhl1K|TGe|;P-A9& z;^bF489J**aGQiOJ8v46ct&ldBu9*y=@TIvW4XN%~QCWkbIZ9MS|pYhb@9JgHTIO7nh=l%@T2%Bp_RR_Zy# z7NAd2Kh@)&-ef!m$6BiYVF;v)$_>>%`1c>c?9yay#WU zJS#tr0iyUw;i(abgokq8>(9T=`AR=NUu=m~`SA%y_|W8Rj72prU%{qXuW&)RpP=d= z&R6>JGlhO|lZOrEU*r0EKmPP}&d2)k=LK&y_zA8FL7X3dSMUjbeDDpPfXR;!7re!f zZzlM3&Rc7$LJ+DfzX3)1d zL*8WLz%&2%<r2{wu*(`SF_tuZrcp!0Kxorw{}SfsZc{yv~m=6?~*0UoLpP zAMX)-?8CfO&Ke}ztWGl2tMDBA0+to63_Zy zx)2md0_uL_1YhXKXA0iw$Ildei65UO_)4_1i@_|=C=u6=f}GQAL+;M6TIG!KPdQEAJ5B=?}Z70!6%^l z$8}opaen-H!6*3f*933!TM9#lm5bpCCl=>3)2;;K%v#k%G_kfhJ>5{00^FM?U{g?@ah;GKT_ zFu|Aj@uLM_D)ZF-nL_prCG5)p#^ z_)i6|^W(n~e54=0S@3#4ew*N9+qL28`oL6f))DlzE?1(A8$Bh1c_fJC1834Z+Qei=-)hV|13$+Qc7UaBlz4$@GBp|=S#deiess)Zzk6N zRGYG(;1LlDAHl1{4hY$CNCgO%_&XA>^YK)ESrF+H`0-AO-!B;|k$9ZX)?B5J;LH7Z zUjK+z(DFfd62~meSfRw@IHu-uN<6-^QFE0@JdU<%u2PA|!LaYLs>&rnD<6%}Bk|Zf zuDQGtFSd)BtWx4*YqV9WDv56|@hUcOq5L!$bdUtW(umem;&l?=N8%$Tet^X5CH_T; zkCpiJn)-)=K@yCUBBYC1i=0d5<0SoQl7523&yskP#Lp8vOWz_1UE&u@ z{5XkUBJr6L|E|Q()Qj~$HNkR8kR?TUPvRFz{QDB0EAbm8ex=0!pTw`P;Za=r_L=W0 zs1fi#$@1}q67TYn2%Qq|miQ8h-z)K@68{_VzUnTQ1P7%E9*GxU^I%f1#Q!DfS4zA` z;;SV7l*C(A(lmNn5(G>9S&5I7`1=w+PI6M2*Gu}koms8>!*05MXNxb;3ACs0!JigmgbCo~ft@z^5gUj|p&r7_j z^@CnuO_c<}l3<|3>m>dqiI0@{K@zW*_#qM>EAc})&-ciBFLD5fX2b z_>mHCk@yUWAJp3Vpen{lf^;duScxAe@#7^vQ{pE`{7i{|RpPUJJe7ZK=kV69)K`;|=K1bqp691OOM@szL z60etdyTr#z{L-5Ghk`*8ydy=3llWYTPmp*;;!P62LgFnF|DNDk`hz6F2U3J|iT_aI z$4UGuiO-by)e=8b;y;r3tZ1?RrzZGV5-gG;9d?`inNc>k4@0IuhiLaFS zZzR4-;=h%6t15=N0r&RJk|0>(w@AEB;=hylNQp0$c)i5`An~!Bx3ccHRT3Da2tP`E zoW%bm@d*St+E{V^S_}vmeQ{sR1@znlV zlAzQlK)6Wa%OpNm;txptN{Rnn;`1f`ki@SC&-{qo{FMYsU*^is}WaSS8?6*FAyhq|s`bdOciT_*TD<%Gn#8*lDKaBVFKl+PFTvQWY z_QXGeCH|bm>m>f7#79c}C5hKd{AG!cwMv4klE5JG*Cakp;%`WNg2dmHc$36eNxVhy zZ2f;r5)6_e+?M!siN7cD<0PK#WD8lQ#M6&(_^z1}k00*v)jxWHB%t3U@m-4~zOKaQ zN_>dKuatP-kKmB4e2IVXLoP^vy(G{{h6*IUzQh+wJpImz?{Z3fgv6Ifd_##ZC7#Mp zPDek6;=9VF2;$F-naCsY;uiuK@0ED^Wgg#EDe=C)K_gj}#9Qe{Qfe--(?J>1OMI}z zw~~0B#79efq{PQayk6qlaGtk+tR$eHk?~!o?DN6i&+DR3_6fm9%4m(sl?b0S6!IC~ zLpX?VF2luyYZ1<3cq?J5sESO6za<<@IGy1QgliMFFuay<9l{9=e?Yh{wSR?y5zB}` zDODAEh8GhKC9Gq30pWUtRSeH2tR-A|A4JR)!aBkph9?jXBV5YxD8ls#I~g8AxB=CF zMFAsTAR?S_KEwS8M-a|sxDVlmgtHj#LAVj&OolrXjwGDUa67_{30oL$MYsv!1csXs zwl*cgz=--pG$X8MxDMgwgmnyS2)7`tV)*taz%2<^-lH10LRe4O!|++cQG`nwK1sM0 zVJE{!fvw|O6H&m3{iG00IG^D?gkuQjGF(iUCW?wIhPM)aoNy+?-x7``oX+qD!fgp# z7+y=b9pMCqKlmiZsv6gx2m>RQkwOQ;dWIJhrfs1L9m5LdZK!nq9hA^bnWSq%3e{1o9# zhC36EBb?4~JHp)wTNrLdxCh|`hMQQ4h$q6pi28(k64oenK zKkjM5m3NpA5KbWMVfZXzBjHkpPZI7!*vYW$C}6~X!ij|Q8Qw!UiEu8%#e|T5LPifoA7glD{nI&AZ#J*VR!=J=LwfGJc{rOgq^_F>X;!!yhua=Q+R=JD&c&F z`w<>UIG5o*gkK_@#c&V8X@oNw?o4p5Z!#UnZ<$SVMR?VHLx-*8qA;gf_%5q91}`qeQ< ziO3+LfGO-JJeqJm!+QviA)L!_G2yX|uBU;nxY5 zGCYd#6v9>~BZd$$m52g{Um!e8zd(37;e3Ys5ne$!m*GBy-y@vG za1X+HgfkiLOn4>XbcWjzexI;~;Z}sL9}tnih$cjQNZ7z|eZs2<>lv;?cr{@i!y3XL z5mqsL`(xmb30GccK0r91u!rHZgx3%*W%wkpb=+DaoQyb13ZD=zV0b^_PYLHUyoc~Q z!nq6=6aI{F7Q zp5euWHxkw{ynyik5mqrgoAB3!E3YvhAY4G$!|(*c-w-Zkc+|&M5Z@BvWW*3s*hIL1 z;TH&RCY;Z3Kf+rG=Q7-f@OOl>816y%d%~H(2-!u;c#}Qo&%79wDaScYQ(ekd@wvF3 zrjOi#S-r}?jKN<#+5R@BzBql9DZ7_yCGfNmlcQJ5WicvmF?zYl(R@d4jLPhY*^B@N zH+~tT%%0Oc*@3vm>TnmR1$Z=dFQ4-F&Y($nR1Y-{+r49~wk7ps*w5~jiKwc9NN8FU zsK<*e)k8XIxiN}oPL$@r%}oJwpH{0oD}Cop)6Bi9{4mehya~oxlYLHfrYWa-T9CK> zRm#q`XzJkJdP*bf^8w43$0)B_TLw%)=&Df7Jc=* zRT?Vm!f5ghlaNw#nWrScy??X0>iN1nDGoxskmsIzbIx?OT8I7(Jj# z1_psO*+V~B7NaE2FI3OmbZ`EvnmThXQf5OP@5Crm7Czmu|2t52gt!q^Wzr_@F{%3} zc!w*y7PfAqfvC$3lcQ&Ob$xAY=~b;RbeBb=(muII`!@Z_B8Z@^7DbmczwwI~$Y{jq zrDTkk+@$Vf@V=>R$hmFJJqKyFJ2E-`;`9oWZB;bA(E9YetX{EOL4d52!24GPY4wcu4y2~EFJvsL!XB|?36+p) zYy-1qxzYCJ;eK&qFt1r-Dr=fo(%OSjf(l|#78W zT_TbYt;?~boD15w&M-d+iibhhJC6s1A-wp`tk}8PF{%N!iR$1KdxvOqhR19>X|yG- zR(Y=>YStX&9L6)VV>Z2$+cQ8tUfHL#h`DWLdA*7t#g>Mc$cSB4aR4h!G{He58a3~a zP1@!A)!_~PB%e!+M%D1lUzr=DTwl>eJx6K#UR1ybKq>FF)^FV!>7i9v%c4; z-bcrX3yDVheL;Qt-k;Xbdt+0>5xNRNbuRsRA%;lpLi$@$vpr}ye&quF0yTq1wQ7!}UAL`eV3$AEr;mkt1~D5$;*vlu}6KF~WEZluwR9Ll3gFPma)%NAUEb zEG0xf9D&vfCi@c~@(5YU)2oVp=k4YSNoA%DXEYTK^WwIJ)hH zu9qs#Y^(MjLfun^U>E4+p~ex$VHKYqXDu`kzs-aGaRtg-+fu@g(9S&(ao`aqzRxn+ z?Er}}0g$)>iT}sNyb-!>CUGkgTOFa-;vsLuo&w-f62$TJ3 zHqKQH1+imnZVm45hvPThe!sRlWX&JQw0&B%#nk?yQaQJw67yjj^)$^7eP{w@JzLxnZk^y%lc!zg)9-s)=aq}|9e#$E3iD6-+b1Jt;}l~N`Ai+40dCCa)lrl?bt z$PEd#7PdzTo>hi#Xs`Klmtx=WxMsmFW%Gs@_0Ua6H{=IuwiYXMzD|jHwfNzR9qNbs zD`&rcQr&b@lY&m_fKC}P%7AYUwJn^1!TC)ml6Q3b3eWKVq+tl`y=skuWMmO#EU0)C zMvHH!htq6s2!!X^lh89Rne3aR*DK$A+t3=+657eO9m$w3Fq`PQ+}IikJ&cr*$@ab& zrWZz=2-`|h?9b~>8OO}DE?|}t;>@wl_hr!)$;0iV->61)UUIVsS9z6L%nF zg2Y|HQ=EMrskPOPKGpH%GoG#9i&T*44rCw%j&AO`_g`e+9y21lB*od(oxtz>)8{if>T;N_&v9M;g?J>vys&(8$7Rb<*N#O{xt z2bHkzn)f?I%Hf9x*hl}4<{Z^n{8(qRh_G%!*+%kz-!= zms3S!A!&8=jAlL6+xW-->}!kl)YraZouQ_?J?L%jNA$LHu+~T0Ht>9Mwp%kcNqc`! z^89;QXwWIYqPc=nVU?{^t!6v7?Hz(Kh;>30a^^MfpjQefx>e3G#s*V)&=IMN(xqKW zLjnd88iS}X)YYiaDcYaH4Q6}uQ5eN2sk1Gq07v`TLk?mjiUSpT7!CCnpk~7sb@EN?2$^3j5jh}4oG2HmF zao7%ROB_LEQN`iab)fARq}~_Vvo$`D6{pfJoP|LRR3}hf_T!~2c4&<9`4257JXy#x zs$wMxy)^^{%18(?*=I#2SG&ydo;hVS)#|*DalP^yh0tH{2va#uDC_5SPPQFH3;bis z**&d=$@ZJ~(D%&n=M=#nnm;5)d2(x`i2viP^?{jbkPO}0Y{FbF8Nl_Qtx=d2`*21E za=kV+3ZYH$=*x)V-x-PK5v~tr8M!z(Mme#yQOmpEvF!cA*qL!`;o^PjCq@=Rr_qlw zqh>OlCv2o+ONyj7n4mcXsD^T!^Tq*B1nzRolrd9hD*7kDo5Z3-yu_j$GTHk@Vi{0@ zxgleo4r$D5g{5V8COk|+<@xQ61~wvG?PgZvy>r%?iaY@(+b-`vTPQbAR1IRu>sj*W z2uR5v1D2BC%%ViN2gN9Lwl%4{o<*gU^em;f2>(Y)FY%OKVd_b-ETtg?q?GyrODVPH zDRo6kA0nkjEGnfG%Tl_Y_TMSFH?uMwBC##Wz*5>xKuYNcU@4_#EK0;DX)#KN?M(vS zLYTb0W$)4cDL8s2pa}xKUATTCOQKJl$yS;@GekA-lb8jbQG=3}@8J)SqWYw*404tx4O?jJ*Q{v;rC)dN)#| z1UR44_e1mq?ROV2@LRB-kaJ;Ltl9Pta{EOrmZCksRfag*)-9s0gT-F3_c3Lu^OdH@ z3#zNnHR9VPM4KF&qYao0z0Z8B=!&+hA5(rUI;{D5v+{ZIDD_aK&W>K1jsLePc}J!i zEAp+b4w_##DqdGN^|ML~cQ18qWtclk^Tt<-)&0GeibIEl zOo>tE?`&jUSfA|(y)lAzgWd%AnRfQCYQvUC(-FFN6)F@WjzD8+1X>$g?7hMyAn2F> zpu#2IV?4qVK@TgrsvWZX+yo@qJ%x6_s=M5%*73Os@!HT_b1TfZ0rT89zS#^ih!#c% zFA8BL9sL`OF-a_jB#6^h7UPy@hdf>?RYM2X=Z7w}urnvyow z(>lnWT~DgJ5OYg&!16Cq?~lQAAFHuwLq@z#@KCc|^uG0fz6e<)_?!zc<#_k8RLRr? zGUZ?r%F|R*oCj#OMPq{~qCXNo-a+}iL_aWz^05AMpUH+4u{$)e|3}WEnx>_>*h5YA zExAM;?j(mO22Ik?;REVvDhG3*qhBe_c1P)-sv$PA;$!3MQ&c3jHpWw$!OF|Kr#_MR zH8I;~RXWwKgZ)Pd?v(LF7OUZ|NNBhyi zoKEqhX=H}>|2|iodmFh_}$al znrRVmUUb@p(X>&kO$`bGeJ+p|m}9k1MzjH%@g|;vn$gqi7+i)vOJhtoGXn1`a0%v@ ze3y5ZckCy~8FQ+A(D&Yyi(=bFi|xQGULz@mFs-j6p|$@rCGnTnk{&}0_{?cEz}Eh> zlyHQmQ)9$`O7*dXReVK#9Dksd_Cf&NLejNx-L9moRc`#EZ+0Jr<$H|ARBXuWl5Kap zAXe9C@9D3U&cBY<%==7P`D+{N49ZBhQ$sW1*|m7KCB>3+1yi#RhR~K2Sc89n0$VN% zd3VUo*cG%Do9uIAy?e-(h-=E;73^&+8Fz5whKCfYLc!XSFfYwjc_*$_`u)~CrWaGh za-Ao_3{k?goz4<6c>8^#EctC zv_tD@LeD^Yjf5Tn=tYv=)K8RCr5#$u2)#JcE6L}^2|!OzdeKU!vZkF%&`}F@=<95> z&+4@|8QWDR>dM-rQnYRX+QEC3T!=XDtly%ylx&~zLjPiDZuzE-Q%c?1-5hdv56CSOH)Q{e^`NAVpYwrPbnqSJBt2uu@>t;&mq`K-yB3=YaN7&`s^d0y0G;3D#Y00GtL@xKXQ3)g^yys z@H0^hd{nFk{{)x!D{vJPQ42nrdFp$_zC^5}*8o<)ZhMjN*86Y+Wa8sMWTKUSCPuDO zt{;ec=C5@W4b?zvf=uMvyARcYT?BgOP1Y0zw3VYxvp@~q!IP$n(?GyB$=+9n9gM~v z-28)W>h$`U$eW36<(xF*n=K_I%e7LosA1t6TwA6=Vu=zp5u zKT*Cp)X>@-QMLxNDiqaIh~&Kusn9481LLU(04GSYk30$+_NYpV5HQp0odp||xv|Iv zUI`Z9AHHCF@l<;+e*ksrbDN_vV5_{L%vQJe&{|~jF-Fe%NJ%-oPIGdta_#Vw>M$j` z{Ao?u2g;c8bj{=slwIYCn&c0a&?9Yo-hZFh7FAJlwkw#bk)F|yvNhZyGuS5YUzu#6 zOgYldT8QK;PTZk91!7qtT5K6ym4KHDm?HqSh<7FGog~can^YD$4Xa<%tH!}zioFe5 zF4;aQxTe*fC(Y~~!KU~>r(>6rw<+u>O^-IzOnp!3aBx;r+)ns6y=#xEnco*dS?73n+4Mk1xJRT}=OZ!(Vzv?T-7 zuBfM#lu_G#?-<4MXGG+P4&z73E-&ctzBrsQpub#I+|Whyn7W< z#J;iKpKG2C-c5MM+)pa&+2c-nT+JTc^tg;Y;#8h>zi!XnQDciF#a5MKyWGWT9B8t? zh=oOegLeqEX|_v=I&Q4{0&|8J4Yt16;yWH4(i)hK#51aF@kD5@E?0IO@2vTGxpEVM zeYw)!)46LJTHO@COPfRA)Alx|a%9AtY<+{hp*7Dsue#J`&btI;C@Qz)!>||GK)j!=;-u5g`xAS<$YsmHX zw1n;TT_*$>e=0)X6|c$0v}7E-{e^L7p>vus`Q|QqAD$M6T(+aK*n90o6z@`~V~?dJ zMYnfb&w`-K2121H3E8lstY;jt!@K#tYOBu_{EVCWiZX8RHJ?l+a#O#v$lZo>Q;k>< zS}vPH77}>8WQrHUIwB&rTWwX z&9Y_6!P7sgla$=Q_o(YAL(e>^Ihw01ITNX|*p<)Dbbicr5w-cYO*ww1N&P#EA3ha2 zw^*tB&#=euy+bw8HQLl3%hRJN>Pf-Ljyc_M`T?HKtIo2asg-Hn(SyZYQgU2YdfmtQ`Hvto}dcac_&plaag$2izvxS&Y-k z?GNI`8GG5-G8pb{QQx~1z0=-1Ip6{xQ^(nqrQVhym(C;q-@Z*vt5V+oNFUbJ#tV^i zgWjobR@7C_dRu8scBNj0vFEx)l)X#Fmn&Z7Q7ST6ShFQ#9GGmomu#OaUOT7P=*PdU z*eg0|RxVb4s(3tB3m#4t%hn!rJft(*Ct`|)M@D*Ic#Fz`(X}F12|b&lIl4$0c6Nkj z)*@y5*|F*oQRiOL%z0bcdakE>fO7L(_w;boP%_rF=J>1A&(f5w_l6;z zbKCgPPgo6H(Hl<2$4;b11809!4NLH(gC1yf!fB|^djQ(rk8;4#pua-NJ>Se4?9=`D zVclHP9qQA40m6zYY>dVUUBw;W{6poV6pLA86mjswR{D{Z2%nV@veE%o%r>$z5>}M< z7v`zkD;q96rFk(&Id#FT4pIyk2W!4usMs$K(7c(W9J|<3b9;eu_hLuuVmK2kR}_1o zv{5d-w>~P^`{x2tu7k{NkQZ`alYJPz#*lMtT2HD6gZHs)*5Gu$`UkQ_Tj%7jBnnvw zD|tREOMO-binI(~&E5sdmzSD$Uy6w53QQ=2zT&V{6MCwNGGDxLc%YM9#|q_>Sy zeW^*G4}A)q*+WesTEnXM;(TP;(QCxtaN<^3%!$R`V9lBLhx3)8l>!wI85X;8C!C%l3hmvA&0qBAf7Xo>dS4FXRk!m4Q7>yL>*ZyuC;*TI=0g{WyNW* zHwm>-C^jnHw4P*?mDZte?d``s6Y zopS$))7)e57DRL~h0FI>un4!UD+s&R%ne)kw^DMYNyC50Zq*y!H_$HbJ7GEW^?mOI zrRqxjQ&gQsZQ?#-S(UNK+rs8>g6QiAk1M*ZdDfBGLt|S}LMo1#PZw1O6l5_P*>q^Qbj1QcPB|IG#qqH7yCdU>kwD)Bz@=`RlT}oi0J%^Ny+0H=s z9bJy+Cgo=M?j(GNW@!~Bm*k96_q{Zy`yO&gwU&Kbr(N=QB>u7IJN1i<*`CVe3}=_! z?t8iwOl;pmb&GzpohsdIcNowju2XX(ood=UQ3hNa6itd)zc@^4Q^syneA$$glnf_g zP|W1^IG)-Pd|%mf?I|m_m{Udi=xHnHuU1TUM)xA!|&)j*JfKz2_l%$oCs_=!Hio`N}PQoFIx_YP()I6S&fZnoQ5YgkH9+KRXi&y zolmn<}M5v z?%9*qk<2~sjDso{onRfsaTUEj^^agz(M*fWRWxRBnfvd-=aJ7Cm+R!LnBCO&m(}jq zQg^xsrXF$?^(wwy&wV#+OC{>eosxRowWC-6;sRISC;FEZK)3M@wYw^|biI07E4TGr zap(aQLFK+RZ}-h3?pqxregT|bcXn+Y7+{Rl!0~Qn?y0w41MRED5uXX=LpT3|a%jF# z{z)j`SRXho#(kKf9Xgse&4w#!eFR-VA>(KJ9Ve4C@J-}drPVZBf%Ex#0b|>r*@s3+o)HB6}sAm&rmf_ z9~|*G(1hwi?zy^{o_M5wx%o75cjDHhQuh<7$KC6SHyRMPo~x+&psTK;m|^?e_s-lP zpFXd4?U>v6zS=!2^HQ+OI4kphmiygf#c;6;!EW`j19&){)3c7b-S;fJTsvx;cd6hH5paVd8cXfn+BJ#6 z$gB(b^<1F)avF6HXLocDNh6qRgp)=ErY)usC~KT)%xpBXlFEs+V_YSesW|a{AiR*c zxJqCVG;U#bxEy}ceIne8%2~nc@6^kY*VXQBgZ^^uc%uKEOw{Xk;vs*+h^-=2-a6Kf zWf(a?u49#dSGR&j1!}}U=)-!!WsFEW=sGoqjIBt!@9u^1!Wff^UKNvCOv3xFqKLE- zSJ7*!yU@Zk4Y5HP)!l!$j)wi^9;9|J%t5JpkI$th)`k8|sHbeCDDoL0Nb?z6#8)beg#%UWK1sSbDhYFTJph~#znpbDYEYguUQkX zaU|_DDj6EQl1q3cTfg8cAymuMm`B1dCui>pL=hYos?Cb!@CrG3>SLm>Ox~bSI@cSgnq{HSeqoA2&=HdrTQSO+~l$rlR|LOwlxhx-tev?o_*s zyC|u)OO*al_{V^tWr`O;n-_pD^on;`+TcC&@wr~i@`c!5DWd_Dl3I7IiW<30tH$x zKD@{ga`HFGX>nk-M^r}89yFWXk{y9~bKPFZEX%7P3{|03$JO{U=0+ZC9T_*lwqK3f z0r>jcfLnD9Y|IM((KUMv0J^$zr_2+Co|3b8*$<;f|@vEs=+68G?I{asIqBLx&$6F4IPyD&U zeJrB8hvFMg+5c2_p|^#gMEnD@X8a*R2EIDCq8!C_owUx1s*R;yIeGq3_leg|rDW_$ zMzP!DhFXLL+}uyYGdWk+qYvqMi?SR~+-y zbXT8gWNH;R?ce2UGQF8++SlW(SP@W{L$1xY6DZryhnTX9bndn=PnmKV8^3nmXCFBe z_M@VFIfD|qbWP5M{WY~4F5x+6LjrQ1dOID5qB>K2m3G0~=sEOulBQ0_7g0^|SEh7M zK}X|_k`k~hx&6(4j_})k7Kf`YNsiF=aiH1Sj=QyZlDKK zxZ9DatEM-qZ|dSOW_tlRbH?^UWzL}SC+76?ql;R2BzO>~O;DL2U z!u*TNc$K5R;(L_~dz~DbzXsX2vo46SlN_jMt}ZYTvnzYap;^`-*pQwW26Y(Y@94HX zlZzBwMSa`N#k>-hLsJDpM<%=(;lNTI?Ir30+NH)yaR|Hon!|pcM zymCZ&^KR4NTqavxseE*|Q4>m&Eg~+5AHcdlCl}RAVd-7vx4RJ$?A6@9o!%1NWoZ{M zwQV=tQ*PbuP`{LNf)@oA0rNWOr|xx3szi28jui!D1~0hbui>>F10Q-4z3sVhjwHq` zynmIj%3J3|fG=O88ViZAZ!AzY+|z3+Zz}ul_0qJQtkk{VOoLa>w)cCtW2s_(WksE< z+n!0;M-@_;5IS^}E><~}cQ?Iz|8=#M>+|&ms*3C13)`qefiT)9nB&*co`iPEJQzr{ z^DNm}mM|`e{1if#T)mUe4qK@0wM$0hAW94;W;^o?J9h>?nuwQryw$-KxSRMKA{ED@ z$wmJJp?Rp4;JGwzV^SgCz_h7L?e3u>SpX5%ctv6(54IqTE9NjftjncvZrC~l4_DJ)S#wWeujv?fxu>u>l2S?S%@P~-G)rt^ z96EHbVS8X|g@v13UBJVQncSqK5+SdG9Ora{`yP2jweca}(p^KUKbtJ#r4o*M?oU2=HU`?TOu;!nu zW3M~#s{-a;*tE%;6rhf5kV(-_Ouedh-=6M4icWF z6@%B6f^ZeRJm{3m`0}7j?iGXoasNJeyCv^JfciJ9feeyoP&ij?$IKP=K8H6j8cO6> zY-r346WI-8*<|g5_*g%qPi`;8qjt$6>H+BNJOT6wur+;-P1^2zgNphfE^{eVGDf~d zXi4@(SNn=M^laCfH+2jDUW6LLcYa!$vOOMCfCp+>57Y|o<)sIyo94}aPF+9G5u~o0 zw=lq!B|(8ZKtGv#nnYqARUPfmUiPVB5nc+xeGGwm*jM!2U_aC;!mbq^#MK}&z7 zm%65W_pYJ~5}?_X6#HY`p|7TXf2o%Gjyj-zV9fUyg4KhAG~eCNGu2Zsi7MHQbGviH zP%M1C70)=g+fo%xCucM6N2|PLFXvsar|zOTcrUM=Ry|HL;a=WytvXc`elPE=R{g5x z^SgOzI(47%^e?Mpo+-c}&3DJ0c%li01?$!$c(6NOO33kCu(9ZtMr@Ba=iB3%=+B9^ zq(c0yO!wfT+X1xTupKfLoeAQ5-z5mA2HnYfI!xWTMLG=CG)H@n$%g-vvbgHF zJ9+Q+RyPQl37_OlCTC(THO$Qaiob|!XL0T3nWUYOQiwfu%ALuUQa~H(INEY7x|4Sy zOx?UqQa+2~$o(&KE4geXH)o~E&wcBXvkM(e(xBFnO!i-hicQE4>_gAwF;Jm<>7-83%IbjPo7TnEi)OgK$W6LFFzS+XO(8WUC_0!uxq|Hc&@f$-!`5 zP{JInsG_;e;Q@mKgYNLDXC3Bv92j^>$1$sAQFUO^rKaq_pd8dQj?|~saK9${qT4~_ zevZIpn1S*k!Wf|Wm$$7q*;Cfz*vUS?{G3(PI?xx|tn`ZX6Y%Y%e(EDyp;!%Ebu zAmJseg6Jn*z>2DMUAlyjlCS-YAlN=N$u5Xs>QNX(qQXGufxFM6KrcbM$V(h*zosz ztUIOHeutxx5|$@c7RSDYEVzub=LOcH&t~=Lv*-U@MV+RW2V&EG>M?x2KJ|DY_THn= zy(V|Fp|sdwj@s=mIz~IqQ3u@L96Lh~r`=y1`->hfx!*r_h+cYkyTgv%z{~m#ciGrG zv}x-eJhs@KnTGvJYi62_7xq=L1n`>C=Ntdx}>RFeuS@ECs+*KB*6-~UtfTndB zJEUI4c38iA9@mbDe)~Ms_VlJeryIAiV?SreKkmCjj$*yFx|F3oxA7@8JMTDF95jYj z&$Q9a&O2g80Q#SIxcGU;o#Kc%p-d-1)>|EeIt%TeP+fFxL`Olba1<1z6NTPWk~0Owu7E4#qPGFB&gbDeHb>yV;3_Zm-r}IwMUuSK-Ch+*{veX14gQ@qrA`#e{QN7Z zH&?dS{#U7+2}M!rhQjK9m-^E7he}-m_;9H)Ke7VRNe^B(SYc*O`qf?0a2*xU>Yg>} zu=_;AjZ`{3mtzdxl0elYe9T?G^WGwtvHL`9=(L@9&SmU`Gmlfr=EExYOx+eItDDK| z$iH01NK(bQ$MFPrrx{nEdI?7WvyR<#e=}o;`H-XN= z618s6B2zfwEYdFgvnkHIP~+HTLe`nW-61F7g{5e#9rV^wNy^Z8;EA4X4|(EIz(;%H zrfpWPEgEH`(EeAO*o9TmCOJayzuRQEP!^szNGKP|o=6!dPqdz?u>?lt5#8U2g zD?B`0UCG0BDpBeOo_PL8xzydx0Q#5ufhYbflzsj02ciA1Qs)UpQR;Vv-hY>Rf>0Kv z9xap~lp3DM3M4#{6=s%|yCN&0@I)#d^F(+cYjb#{fw+e~k=!qrRRnX!+mp92&tXpJ zo}+6X{R0L{I2wyOZCDN)%lg(fSEuu1UmK2ei)veE{6l@5O?nvK*`zl!Y&$nRCi1-6 z{b|}gco@!K+;dVdxr%zG{_P%C9McBYLtRDDH~}+;r|xqvPsIW8wA3=1N!g5vVU^9O zcf(feXn4h;W%K%oq+QsF-u(X<9KkYZ#X6Sc?h`m~oui8|P_8GY{)XXrP!Y!JVMpDY zi??J#14ozhhI;Aalsh5y3>{kDQoGaBF49va2KR*2d#)lwYNe|vJ+0h**RsoPNIhoD z$%NCmj24{Bp`Tn(yDd2AKULd&u#)g;E2fUN=06I=5n3rf-1_8VG`>-w3!i^3$YqRW z=aBom-(aTrai!3Elk0^rJp((@^y%^AG@YQoUT3(QalQp2+9L4wr@vf7VUt`|*u zv;?D!-DCyL!U|JXeV1#ZP6FAw4~sUJag=U9ee8Ch)K%lY;Og5p>WGdgrz^u4bwLO6 zw5xAalt+)J8?KD7sB84J-<8og>b}8s%33>WzX1!>EzAk->ew>Og{|+SYWKB4=ddBu z?_N5VU)wF@Byf-yzhm=1_t-f)2Vb6H--yeZ9^QS@F4J2&^JlgDtF-fQUboBGGxdmj zTk3r{SK29ehqRkm|4d9h?#De!JO%h{63ecn z?-Am3Hk~@;EqrN$!ixKUxsCUCx~HYBrfjfNT2~)qLW#Q!r?gl)SnzF_wo^{916vEl z_~JfWx7c#5^p+x=Pn@jU&;LK#-UlwqV*4N8d6xB0Nzam?W|F9DuK$um3q^|6C6@%F z8{n>yTAF2{lA>9u4+@mqx<#cu*43o0-t4AbyXD17jem=Z{{a7sCjMVU1vM4_;rD%J zp5;$<@9*B%?|yi_*k_)ZGiPSboSAdx%$a#O^EG9vt>|I`6wIf=jTdlDnaT$zm`{U? zAh;XGCLH4f1M4Z!7&N#*0oTZTRHEQLjj1kG|CXsrw2BGd(-3VFU`=?+SD@fM4QvI0 zDL0Irh@)r;K%30hcsm<-PXm2ZU<670Xz>v1dM=xMLx~TIEUy^hgz$VykfjqG`>rhdL^`|FrdrMQ1}mJ3ElPj z+&qZ;$y;bUdRN@|Eiw6ZA+h84+w$*(5MLg|3ft(;CT{yRxzuwmVI}9hlS${ejE0PZ z)5Y8aMe}9{qy%Hjv(|?EMW4Sh<$E_@(jl~gFYYkKf4?r2ZU}|QTO^`lQqFjsDVIFY zPrgjWUo7hK#*{^iDIcJhZlr*1e3^~5r-d@oq1dwk*;fdd>r+m_b9Z1J}iP43wNX^}dj>B0t@3IbBxQXef=aAcnAQ#P#AD46LZ{6Zg1 z7b${Z4GvM36l^>`oN^qyyR*=jZ~?ji?JuPG*9FwQe3gW~PSAB3Un$Y{WD#6?>UphZ z2V)!J`rL{p&0Z*4gnBI!EkZi{W>RiwFbY*!qas4zYt04{&(kUJ0g5W#(m=&7Xs|$s zkcQ3zbW~vu8ZI&o^8I!C~6xgA2k)GtywIC=YWc?N8x>H8ZW1VL!0*2VZTw#A@3$BU>|B z**0e_!0ZJPrgO)`vQZuzQ6kU(1WF>LXq>YIr!p7B!kbgC)WY$A+y($`)7Qm1SMqhc zyVHD_pNU|xPQGT)*6uz7fJi3}L3%p!Ha7{x^7O=XB$E0O=kAn=AkC+^VCA2uVbXO6 z&PPo4H3e1{x8K{%Z3tv9z>Mwx+_&5YC-SW%Oz<|~rMZ?-^FSFv1WuA@`JV$KG@pah z+~7Q%vI{o-PS7(O7+r1NEBbs|vKS~} znH)ATFo|2aG;Ri++a|g}j&=I8$K!+# zY}NEbZ9K1G{t_ioh0A&QzCvg)qy1D0G}J*U2R)X_n|-3jVlJ*;K=?&<(Twi}el%k< z`PX{hNcqKw8dYED!@8&u%!jKmxWf)8$e7l+WCN}Sk|=X`73GH%7xALRfY*>>7@wi) ziixH{{xk@lD+@fg0@{2~G;9uLKdAk)KM1j96u?cPRG$AFB#dxV5JPX;!@^4;6}%9m zzZTbf7rYdHe78+#x_aaR);gY8%IXZ94-;ZcMEYh5YX)W&BWxTy~5%q(e8vligt+_;v07pLP_n*re8fbAoi0LUntgZ7Y7b{yqtFcPQYt(lQ{79s$y8Hp9z zFQ7q-s3un8aU!OPl{j9+ENwIrM~RpwM&eM!{N6}R*f9wZSy3&TxiGP1LqHnghpo(p zXMoU(SEiC$nm^m<*OGqQw8zwMB*$yxI%Z!H>S2Rr5i8)0UT5-)XF3wiRbYa0#>MeCn7xsWcU`2(r zl~*)uAgwphPC|x?E)&Q_Rj{FeTDg7YWs7p%sd!rW4$n|`8+f3D8F7Ary1>2IhzD&s zjEvMAKr%Ro!FK(v@BzWLN)%Xa<0_N~j?2M=oIqS_qCn_q=V{B2AwXz#%~=nmw8V+- zU5_nxLT|~~+zK1(#+z!)c(fvM6^0hd!k&8=_S_BLzW_zp)wrxX@_Tv|hMO94M!2mj z5lCEuxm2vq$(#h1fK~rXPI0tlq}%TcEU?z__RThOD&k#`Vm7>Bi8P?YT5A7n(SsfnZQk zYq&p%bWv-}1);z%Tv}dSD=&N(Tgl|eLcQVQy`c?$O`X79S=gJ0rm~V|4SC@vh0tMQ z$QDEqDhq98V4nKRUolTLsxKIrG{-=23+!CKfXGAkW!k*5Ow23pwP+MHfm8tgPzjEL z4-R6S>$G`gssCD{N?pKGe?^;DzW3KQB)|99=9LZp*#c;TKc81-88iL)=F_ZzOn*MF z%rY+YPbHXH0So=byiyfEgEpU*Y52L)Lc8oxH^kf$hrzv$26vHVwlPJjgxWgWm>g0` z13axoC7L~(!D4Q-A`IT(=-)Ihqn3CUjlue?W3Lx!Vp+D9V#sld2#B$&^&#dwpk9iIm zD^M{v#`^~kvJSm!X;Q_UwiZuvz{#T#&$m~rUw|Cp+g!kotupoCgKU73&EXO|z_#Rr zUPpN;m%t_F2R;YjND`GL;^FTOIP)mE;*`<|C$qmU z5%~C2s3}j;?i3wrY~EsVN8ib)!(6hNhWB_N1v8;t#yU~AP&^^n*5ct)S2eBHHDi@} zrU&+9<0(ye?eHn(2byg(uN?!QTpGM~+D2eHY4F;i8HE-?gaFU1RCV9sGkAoUMrq!h;yLk(KHB=fb9*V<^vND^1 z@{`HECmHTNi{Rd~#~~Q=oE_#(tNpNNgJYxdh8o?Iw|znE0G5lLR?!X!jIBnrU13V;U1Su9I1Z77~w_IzSNDwiKk zYupcaC9Z<|tuZqz1!)btk|J zSfB!Vh7Dr zv&(%q=P=x3cb@JV zhnUG+j{8C3$dG5tI40v_&3mrq1La0UV)Fq8gQtP=lDu$r3pMmn)=gC&VnNdS&(x0} zVzZRk@jJq{X53~Y-GN)4at|nZ%I&u^#ubX?%noElJ@2tOM`AfL5?B9^#Od&nA^11b zjwWtiC8v#8*IPIq|7?d?*Q+5ot2A;qi~-I-9P`iQHv?OpEBMX8PC7vE#PRK+@;|MQ z;Y!$lCO?3~pNf)kLUC@6D0Chf`fkEQcjS>?dF0LX<8bvZx4K`V!vweJFab_p_o5f^RFel}Oi9u%K|6CbV#heObD!%^HQRC;QMSm#Tq^i=cg zEL_ZrJ4v*P;jSM$UBfJn_M>Le{cZ$I-(GgeToSk5g z=~Pb_1^eBJb0*ididCG+^{ptvdZ*R2^5O2}M_9D`RY0Gb*%_$v>GtNh@f8>cPdQT# z)ZsG$ztAPd0kKEn4U&g`oBK2_4Ibfc6BCnv$`59mEwOrUj5`Jz^GH`(UaO~ELkeP? zk38i{E2LtKE%D=cq66Lwc(FArEP2kzN_%av=ZRlNKJuT3T`5Z8UHIGk9AEC z#Wy*&pp(t$Y+U!V&6a-xa8~Y0qUBuOK~ZNGtGq5g{GH@PTEM1#RoQK@%PoH zar}w9pYQ~qD5kKhmW*2v{%@hQSo946(&aP+0IWgx0_zj+#u{(~_b~+~p@VU**s~}i z!DRID7QX=ca7di1(^;@a8PV+Bgdkw+?*pwp&B8-Y9D9vJFFF|4mg3T zHS}G-2M3qVFv#ZP)os#mj7+;aWMtYU*?f!$*XJ_*b|&&AU(w-V?^8=JQ}MwKdo?bS zK8Lq}l&hS|uN(1Ve5nHgq+OvX$ArrS_$vXWU2JtJS~d<3!HZ&4%y@D5T?L9E*DhIQ z`)2$ESg*C>ynRWYKam8-9ZxLocri9-UJxo`HK(*ZKbKT(+)jXYg2yxJG0z0?)lzJH zU5s;U3}y!SS*5kLT+7GIqLZ_5&x+3S18tXs-?zlL?YJZ9G7e@V!s>3uNcL`!S$ z6Jap^o{Xd0dsB>cGrOZl#MSI*436mF*YBY1PGL;MfobRYv>63LMmsTuYH*lOEAU^A zhy${@1UD5v=}z9Evt28csTxBwXW}6)`lD`~;GqkdjVT@@J<>cO^qB1loj6o(H^I}2 z9x0v<^hooV=`q`*{xyt+c8QB9nzpEVj4l_!8T|-+?7rWmv{(GF?Hhg;8xrRpD zaO-6_8jn3je)A)OjTz+ti9RLNN}+2Cd-XtF^LIDG5Gf^q@b%G_)q9j>H zJ(FmgcWxH(hVv$RCXa?}h$fk>Ci6sjA7mYpNj^`M=VqG1@j`-nJ>7pL_q6(j(2o(9 zdiznJ7G|8N8E4j0+c8h!UjCqbmFXa^r~0o#AXPBt-BRD%0F=pHGU+zgO@Q_WAgQUu=9#FDje=S7#`Xk(nhRzj zm&1_ID3@a(m}`kRTrgjuYqozQn2#Z@nP4tO1Ozi?-$uzCGk&`o^9UQ#Djm4!{$F+B zBP@~2=aY{xQxCeo`*-rW8h5uhlh23Levh(2Qs!dyl}A}xgJh;Y`WMpKd5?3XAH=hB zWE%)*=g2@!F6W~bzmdyJwl>P;DC$tiWx6!+U&-YRJ@#imlEg|=Q4CaDfuu}`;^EGeCAXe%rhInmGDLC0@>~D0 zD7I*#808pc{5VFo;i8zzy{CyJ2AtAV6w{?Z|EVaZ?)cM3O+>LBu4!pAFx){N3gzaz5^%NZzCpP_P6Sg z2xfjf1}g9T7k?u=oAMb=oZ~5XJ>jf9}e!z4n80z(r3W7d;|PX-Jz_Gz%nYE_Ysz)DV*1P%X8s`?+sOvb;V1Clay^&VBzH5ngh5W%v z)MzliM=Q~j4J%P~(&Ma?w7QG>&f~1heN{*mE6dIg@|9&4nJ~Tg_Ehs9XGv0aoqAss z3ufW!qfu;#y3N7-@4AK?WI!RWv#Y8+$=XRQMExR)^=?C-CXC6r1KrL49`#@pyZ>HZ zNt{>bMCRum)z%hfznkLe0}V(~Q!Fe5-mMe^HO;+>!>;2*5!d|Sob?h!Vq+zT(&%0} zW^=Z%Ia}hxTrp0Tv5}|_9h^MbaJic?=FW~7R}vi2XNF)&*FDb!aj|j%2!%s8vpA+@ zgdQ8Prf~cWFj|z1lh~ofwyg7*^omZutid^dNom6_OZGH-?D;TRh7lA7bDkI&Gi#Gbn#v)pDx;}lOQJ+!-% zG72Pj3_j5yZc=h5L2FsL;NvG)N3it$Pp~c{?(0OgSxnq28wZ}im7eHOY9l@|Gdl=c zaq_i^@|eg&Jb}{y-AnX>4VhZs*str=cb;Iq`en;7+Yz>axV3V?b9r*W$`Bff0h4nq zt|VH(a>sEwU`DANFsD*I@&p?wfrA2vvk|O=I&L`Y+BTHN0OmGb#>PCL0#}r%i-)tZ z_Y%jFl^2ib-dC`B(?M-vW4-&)LC?7HS7B}KO=l#%A9SGs6_<5Yeh;_zVS49BVPRLe z1;W(gbe;`IzEJ358w+W_7N3N1)XOjqQaz1{l@sOgpu|#jyNx~kV76@DGf{RV=2>M} zy*a>|Y&@V^I^5JSBu>uB4T-^|aPPBt*UlU22z23})!|Pvv;241$F#nN$3?a}_es`% z%#ZCk<*6@Y@{jwWj-nGH-^c_PTI60c{f5g%Yd|eN8I@dz^2QjZA_WZ)_YaMiStmH6_C68dC)Z+I>u(2TXg%OxNR^G3+iDfg`0(E{Y zGxzF%Do4vtof79wVX>~{DETScq3+q?L|Moe@9j?No>(@tbuBLnlvO*&v95O;Q5I2d zjqZAGY#b|Oz-xSpJ;++AkK!SL>|>u|_fsWiKgAMNb36`5o_LCx)ibXm6dcd)#UOiKPnotD5p?3Fr!1V9^Pau1wBf7cq5>Pck7q@vaQNT7byYLhxEiP;{qp7BmFyr? z96X;bNBDp|iaxuRkFT!D$tPsTc5g^)vd{)(uA~yMQ|U(^gW4T4?^ZhInrZ8yVj5n2 z-M1JF3*d`E-W2c0+Dj(d)vy#C^m<#Y1=}DAgPVF8noQOLwV#7^l13WBwGJ0 zGLXcPanw%q6TB(-`wHG)GS1miQ+U7q4Rs)@q7H-0*q94SGg8?%`9c&Kt1HWE`)9{xH5=F3!V5=O*sL)W+G)B`(pyDLE{c-iG|0hh@?` zM*O^uAL~3Lb}yh67fd@G6S04Ke-J_B=94s#5I3WL0<+~&dGaVM#{&F5#!sgYumu(q zq3oZA>(;e9H5Svr0vKvg%R{8(@Ckhsu3FzngG?TE600+x>aaylKbt5JP1wVv|^30Ra0{XrONaQg56;HeeeZ!x)4}!>Tl@&hEP+(aORT`UNctG4&0u8^U zaB(R@aMIbrA6Wui5A#C-0#kn_ply_HyiE06mxrPu*Cq}ung$yn zQp1hgFUtzwSVVky`%7|WADDFHXO|J5$8gd3kqSePbh{c3fxUU2jzfeoUKQnkngRKM;F2~oIa4_FQZMDlv?~r`& zA$jy!d31$Frta0R@kXDbMjz)?>RXAZ-%{z6k4k-i6dBG@h6|iZ-T$s-_$`&Xzb9Z% zD4a}{Sm&b2!Se|GcVw!dV|mmM-5pvkDxp$j8L5@fluF0JM1o8QDC9-UQ;vZy`4l(; z(NHHBpI87Qtx2bAO+X-+iNlTOV0;G@%m*upZ$K)3mJx*#W1kvJNUoATv^ zQ&#YhWRXW*huQJGoKO>uWTGz(L#v!{Rt{LRmN*>@T`gy}g+3}9`#wr}9p_}@N=kzb zKKK!i4TWEp&9^X{((uJ69pwTKlas42%g_FT|0{BQ5h{Wm$%;C3CbkMIj^ZD2;l^L+ z$O9;Hh)#982LAGe3G;+;$j8-sc@#=dt>8D-H3DoD00Rg>8Ff`a;FLon#8Ls#hoeB5 zgaYRq03Z&|1id8yI?bmtDo|sB<%|)NB?V50Af=i@r$X_K zOLUGB9(WGK3^q8EmO-?w5;S51OY771$+Y@%m3EMH&X@&?qOMJf%v*PF} zxGG;Z@9~A2ONkMRVK$@o+wW{4)bz779VgRT{0&;StR5JxJ4b_zffCkNK*boQc{Y|>4E zz7jwh%`_l|ZZ&8Ml_{VCIjVAOKF=-3rm`PoDM+^i)xhH%q!1n_xHvN&Li6`ycAZ=< zC-0*S`zFT8b5DR^`zRLG8Qf)uk$$&_Kz5PJNiW;5EngxlNhNeYAfez_1=feas@TR4 zxBFp&S?DH_j=%;wT})HM#AE|ijjbxZNi1z5{)(U@EYEP-vP9>=gz zf@2qypUy~V*X~i|Lb~`z0s53O{wgp(Ax~Msi$zsQ^wN;sF2l{9F!~Y3gVZVv*eOeW z)mtd4C#M`l>>_~&_<03^hNL=`BI6JRuyjJ}_&tGCs-l5YeFQpQ0-&1!Eo`WoRwU}Q z%~z)Z|7lI2X4E#R36z9Relbc$kuff!MxO5ifw0n~!4IX9WH$seX#$YQTz6CY z3M?guVMW6pt4f#Q18gQJUZ3=V{GL&#y4Ra1zKh(8B z>n`5)TCb69+V6)aT&wxzPGAdCF>nlo2r%5)XFXirR}xDMQ=CJ7groZpK=o71oz2vs z`Ed|<4KxsXifaxGCA6>jGOp6II%|JpD4FvLt&iQ6z>eN{0OPz17^LFB#28{1;uZw3 znaI0cySb8j|NAV3Gd6;YMDHWCT5uCWD|qd}I+yT&PAOp}Q77#hzll$PS-E2p6}q8;n^0y5jVmN;ENqoij?!={iza5p z3O@Pcajd;U4a0X}xobj+>w8{Ri*rCackwQsb`oAgP{C9}{QzdO~#YT&Y1;mU_H>rn0N5u-4 zv+2rGz&9>M#&M?gEk#oC)@&&<3lUA1BB`A5u@pfaXeBa^zc;Q#;%Sm-x)h1x0F6tL zP`v+kDMCoWbwoGW$OSUkgQgQ$w}!}|gqWtT!NN8DWo}#HEAa+nr(jn;USafCKYE^Z zZAGIHB#BZtK98NU?AvM`V(-i5u`6@%s7?Z6TC?zRtcJb7!i^$%aXiIB9lfEx^#bcY zl-g?HD_ty`(s>%UF?svnsBbWU0V9?Vn@B-}yR{%O6+_r>s9z6fJ)$cqm}7-e_=o{th2y(43yhaTau> zM!|@-8gmgXu4rWd)ke)3&$_iHvVgcD_3HNVth0X<(lxUhAnL}m9TJ|KC$KLS;y!ud zm#7NdgL%ziaip4y$&BBNx7mmjOvGaH!uG_JvXOIR^O+H3CT4^eFaa73ohCr1f_af> z6tGzEAHTYt{C@!={C|~od@a0AFyJD8v1(o9FBovCzhJ^|T(Q+~r zNBTbcI>CFsi}PsCZkRForLNf7YE_Gt5wWnT)1nr>3N35Fm?=8?O=I*sTY~NbA`nc# zvTXiXS;`-4INK-zT%KTj6WOpx9aDFdt$SG2gJIaWkr+Myt{##~L<6mI+w-g8#s|#OZ-4s49z@EgE z(YWKv&F8U(F|jxmbn;C|+HB-)b)_Dzz0^6vwO3;e=t4ebvAC^%@a{_mpVwmq9p@v6 z`xsg_hCo;nm8eextnO@LNpTSq=v&I)+_mCC?)0!N6EOk(n`+?VB#Oi;Tb=qc8`P1Q z2cshtSv8ZXflQ{8uGXssFSEhPG=3o)Y4m903DTKUP0$p^2-2c~6Uyl&CL7mKzd(=$ zuA$kOnnv}(b0x1HU_mq3qBmkLuuozG+fi4;2zDkKUM2WHgEUx$)W1w(rq+2#lO5~Q zZ{(IvVvH$N8*-N=al4RSWH=8KNt!7jXn<_rhE68K6v$6R)7)OEowVbcxiWW0|oE9oFOXo{c|*{n)k9PBrnl(}tB- z6?wVc+jSYE18%d6aikLr5b5_E(^Y+jNKJBA_25=;on;iZJKuj2*6u?W>dD%jfiSGy zE8gYS?iaIh?Ok3?{hinWuE+qq;3G)hr@>9w3O)2bz2a0%&@2F3##Vg-xNMuV%m!aK zugz_kH%7gk!Zzt!*VL<(0x-_=gtcofp8qP0cafoRbHr9M-kA-KL&c8fW83GVgM<|KL#2^KogJ_mX~#Y zW=|`=tJBKDKf8txs9}w>X+V4W^k(__hRTY1z-eQ~08!(s&j3Xipupa>)p^x>n&?Ac zE*yB6Qd;tq?ntWlZby>HR5BJSL`G4E#;xnBqvz{Y`&8Di;|@;XxA@n7Ti9Ikou}a& zder;&<$Cq=sjP2@V(JjrbfeAvR*3f#)QnOeuC7;4Pi1rT#}Js9%1rtn5pbun`}MKN z_;o4^(NDTmukK7`J@ii@sVt|juqGXkGS z!hA+wQfiepnZC(%&0Bw`dOUB`oDlCpAeb_p^-5Ze=w=8;`~d>=&@bK(0STuE^7K4| zyQmR&fxH{Sy_2YuTvj+l-ib0(Zyh+TUYHI)i8qlzV`@H_guaA%4O{UTk(ce@4)?%4 zc>IWO!6NLxAPN0aeVVEcp9oIR$>4*kTa-hKGJ8iOs``2UPNRjs3H1`9Y%Y9}s-80< zYJ|^Ud5-~;cSDp9pAR^~2VCA1cpU*}Yp~vTD2Wh`#3o8l|?K!OS6!lT?2i8k-65>>1y12&C7FGwQQ5Snr4LX@Htd(aRg4 zf+ILK9E|4pU4R8^o0QUr5Q5G(5zqtzH+&htK#Vgr~l6#0` z`j&VK8bp4ZnK!)vL>})eC+hE57hH|1-x-x&-{SR1o4g4V-M|V3?Q|PH$HX10_p7(u zskFJi%R`>(5SvRKGm||e=@QlVX0i}zW|F#gChL8_gHsKkalGLD41En^SWWmG(PnSX zarNpKUY91C{&+X?$`=O`o6U63(o{<84lS@)O~(^EM}{{xUf6WlSG0cOVxnO0YSH zkHi1VaCA?#O>oaoN8IM&@%Tm3RQvC833Rr3n^$p&zvbYundd?PS^g%F&d=s=^Qj)* zpD>xo-*kcty&*i)q|Tm&y|Y+X-;zv|&R>@fUqU|<_Que!HjqpHCeY-4oj4C)_vOeo z=Rtb{3%wc|>mF~5qmS6ZO91cazgMGd&aa8B@DNP%HPP7S%%Gk?!8;ul(4J9;2!K z7ku7s#(p2oV&I+$D66Se3^}vLx~I>gKnTUd338%Q!6D4(}N!d~cIi6#z(1N8}PgiAnH53AdmX&%eGdMaZ2 zC>#MW8cKnW0{&l80GqnBn@7mr2Qo}=C5TfTC?;^LcP}~)ul79N%Hcqd7GVw!@Vg0< zo`!eYPuSpfRuRs=R^UWarXHZTfg|Y+*8Boqo(*qdw)1F`A8;)u<1~&ST#c5aYopyG zrRa!b^1?R(3-rCpvGR-#+Zx>)y-*3?zf&>2y)n)$bTVQu^2l#T1DOrwUZQ zOD?98G{y9?)w#p!oM1|ga~45So`o|f=_nY7XO4g%4%%uefs_bxbE1vihREj*VIy%j znlrcbD&O;Z18Fl1Z|bcJ>W17&8$;k@ICw!>Bb5I$&7i{RV2G?Fp8(zZou1x{3d05$ z*hlVneCFNM87D8veTu4EBbdyCwk6=jHihWHvN)n9`Yq%rmtjHPk<_Kn5#NyHg2OzR zwUD?MX!B$<&MI)OUV@C?>R%u*J#-Ans+_{cc9A9Gf;=DIH5eQqQ5Y_x1%U_^od|gu z-q)s7s7bH~&)aEEN}+ZwPmBG3Sbe;>+d?qrh9RTJfc zY9^ac$Yu|=@#MsOIbnyKSS=^*kdrFGRI>RZy1{@C(t=RakK*u5K`r$~t!zGu@Btld zJ=R=RXdieJ2bv3O$;^rwE#EY(r+JHZOQ{(TMJ`7&MgK_Ah{Oi&3wzNv45yjcVyW3J z#}};9%b8V};INI?MK&MiC7`rJVrTj}!q9YQ8X&7A)T;t}#}5FwkgAp^c0(W5fO##j zjZMbsDSSKh07m)?Cx}uvP;7;4{uN>D8b{OSr&T^W+Z2yS#c@PbV=DmLY;y8NNW{9KcTKt%-8{ z?yPb-^B))-=ob@!uNGhj&T&CDM?Q!*IiXrMZziJRTnsih3$7qFxn`n#IuHM}^|q}Q zZ1__^!qls+VA|h>e+A4l6EQXQc)5;$Y;#wj-WAAq6uYF=HEH*NG+ua9qC5QxtNj3* zceOxDYYXxn!J99wk4Qr~fhgQwBqtPFyit-V=$&@ zm|?i#kdon10DIpB;4mCdV_-ZpYrP{XXX3HpS6*I3s9Xi0Y^(gf;a%mGk_8N5F`U9W~ zNn;P{04mt?6eQ<)UjjOKgG02-qw`QcPEqNO`{}}H%o$!GSY!&Cga8^)i&hwem*Ry2 z-Bm5%{~-d9@X4le#Kkldk%WgC6Ngqwz{5ohm7lto_`S+)XBjsk?3G6hT-JI?PF-*w zK?Tmk1tQ~AJ(#p?7U^j_F*QBK8#1*-ENq2@XAe{KmtZF2Y-)mMiR|2bvX?C@dCm^f($T`D@OsW z&*oy;Tq7rJmJ`p(31{TQdO7KgoaB*5!{6hGY`#L`Uf~s-UO{r^5-`lI#k_WvPw0JL zMH(vAAXy+nu(B*TOJE9wNzHjKC4#MhYbs|j$-jU(GGzx)k@wF|IQfvA$-roqN7o8b zbHRt{EJrf%4jRx*)L@$Yr4^i@NX+b`B5@xtq}srJ98hU;AvP#9rGS36keneOjj3fQ z_9}W1;gn*`c~=lPn!p7#W@(5Sat@ohm44viVLp!zuDK{DLuTN1HFMB5s9kbGJ>F%Py8<&3(u(DTE08WqfgHaN;ssMWTJ|;ii4+E8gKUu<`%(N!@_~SWgOpiD=*(y3 zg5ylK*T~8H;Zy@(if5rR)f|S*IE{Zeid3A#KLqnp%+S?%rj)12)Nb;~$0kdkOfE#u zlMo36|KpCdCaFV2Dn?gLIkrC_=WEDaqec$gE89y^7f6L#)V}~yfhcmGQ^hAaE|C_J za1qjkH20b-^n(a^z2qa`7UaVmpM>yeC`N_{tW42)U)#&Y37GadFG|b9u=jX7VWeu} z1pIHfG7O{F20-wp*Ei^IO&lh-O^JkX@!+fQkSh^Fu1v)!;9^DL{N7XWI|O|c4Ost+ zR&C^-L_pz#c|SsYh`*-=2F}p}kt+~@4-SSC%w-*5ETFN3o$b6qXd47DM2t0}TF!t1 z04dQckH&E^WDLzD!k6jZ8E|TV6f~t$^Tp7laS3Vju~QKUk*n8U$_nzDF*rsO?Stb` z_GNi&33~0h(#vweA!6SHR}aV!1wDeKMv;jaNHTp+vMsFqsE!ODMKY=4E*_O~GkH#boL&+|4^||L^qeSM~|9 zQ$Rf@^zFxy$@?K?fY{L{SHEu{dWdNATikhuwgb$d4La<%RGxQ}OP~ZJa~_cA@1?y6 zm*MDi7}=aV8f4J%S|NH~r_Bo(P^#pMYE&QAPTlt=>m&6qSN+~%{iRK1YWQ2MTib(W zsEN}<_Y(X}#$4~LgX)yGa8bz5W$HTsklx*=u6_%r4eVv=FK;n(+lPLeVaEZHAyCV3 zbT4HvtIBNF>)HFkwcHZ^+@;pc zW5MdUdCVY<*`pqqgXV7Cqu!pwI!eL2)cfYLE`e{Nxg2kez)G}= zA}!8hOa29|2j=Y+h_~zzh!>R;;vR!;?$L05hyWS#$t>f|+76uY_=$C;cAzsdneU?; z`DGKLLr?Az=uj@2+o@%Jtn-q$u8iY+WULS8Rd8ej&Z=#|`8|HH`Gtw3R?eINjR;1V z828K&@1M7;e@NzigWKQ_B0|Hnj}2kZjx43+b$PulrFyXJLM|*1`?Q@P}QZ z-`=E4c9p6x&0{@+9|1I{<&N?uw9G3(Ij$1*>v^nWuc;`s34Y_h!ar&o;qM}u1nO`X zFmlvgo1zXP)RkMQ`HER5znueu+FS~_>H@Kl*HfatHlOtlSOT~vr4A|=gibGMLg+Z~ zXFBVMt+NZ>jo@N+t-|gbn?qbh2Kq-4Xdrtp6zRRP1xXmT-gBiI%y)okV$H<|Vykih z@7Yb>t4@@t85!)+`$|A`ZW9%zyFa70Fc*5mwy3)^*wFSy|S}e-U`%=@=o7Od55**san?gJgvstakF|ZlMRj8S4{PWfzf7IMTCP@ zJd9WgIx=qMdvR7ADa&zL=hGhI#>*KUyqOpQF|Ns?CfCpuGWWe-tiGPby5HwT&3G2k ziXV9L8Sm4@>bfk}HTVds!YOHLHl2{w@80d|uURZ!S~5s|;%)Y})M}8r^KE8sa~8@l zJo-W6qMfSr4i2n+8?Fv|hxKSb0mvXY=tOH#heGVS-HpA3QQ?%WW8z{o1|D!mKlm6s z4ABip?r)Z<;dTVf%sQd=E3y~JSygC*E#q7(?<`e4{SI5u|DUNQ zT}EULesv>V1M1Qqgh}#mow3cERUv1(IT4TY<0m**5$n9@Jzb>cEMN%<)a@uIU?L=d zdxJ?wc4Akkx94U=&nqhR)9IeIjJ1p@#C}!ZM`3}x>ivvzIAJvnzV!g`XWT?}o<^kr z(H;=q_qM1w%GqCf#iP!4vU$pv1djtAnOTq~#X#?lv^|0?FUWR;2bjY0ylGULJF+{X zQNYLa#tZm0*jGRd@^0dla}UABSkUP~w0Guzc_4D+dB4v?4wqr&1Zo7*(SUppqBa}8 z3BG#}%7LlaaCnMIedk>qer;Kx9(|YPnC1WncCcQjhpYRS<4rhEdo|W&7>H09@C{t? zf=T_0i}eWV`Z(Pf@z3k&$LlRFbge5m1W89-5QbTgtNsfy6-R`ty%(}9X@W=n$3oUU zZpLV;i@%d3sJ>0#(yBV29D;#__e z-Wwai#%{w$qfM&G&HCK6h(?9mFb40V-7GkGvNlTWHrMy29H78@a~w7eGmYLq6|0Nf ztVg@2d5J-19jFY()5}2j%S$G;%#FR%RSuIXzsE-DlMtBn9*YP!ahg)%fTyAw;3t52e4`3kIeguFI`yS&_N3G{QvEI) z`^zVWFx{K>v?SP34Z9GYmP98(%rNDdgH^mk<1Z(NUnpD&SRL%a8fPv~d}W9l_yK#k z%}W4E!^JEPL*fwi=?`%0ODDv1!}j^C5mj{bX`<{Y(&u~IT$bk=$|8s}4JCMx{_3a0 zHo5Ub099L?f_*B$K577~4sMd{|CUHj(~{e8EFbZr_E}Bpoe$W((szYw+Yi}uQc8q6 z?L)Rek|NYwAF_TOJ}Kaq5N#UEsM|jKdGzdG2CKt9Vk6s!HA}@LCxfk@7))6^ss$gh zptk4po8|}Co*bl}{fH$>eTS-+MeIK5AA{7}i`jkZ40_o)NX=fv`rkDgLy;)@!XS0~ zBHVQF4Bla###+@JG>V5EFzuyQSwY9h2Smg{0N|X9;}i%t@J6`WX)!x01%wm!#Zv#D z)kPn(Z=}JG=UP=}W|DoQI%Nq<81g0-s@M~J6%X9eV3^LIqw}C+qnUAd!}_}erli3l z*beO813`33hbPqfC2X>EBwC&P2_}G>R`t)HuikbxS9ulSu)Qxt zP1~Q;?>=S8(wbO7!v5AKBphiKBs_#g9g*<(FhRoIc=wTTKMyw{;b$VE842GUCP?_f zP{RK4ko6?b=8`NXf#%Z~8pmGMvnJi0O6@&q6t3}EKof<_kn^-jUA~M(-1Fg+H0A?K zZO))~o;9i7Wh_$p`CE)Mm%;cnlExdWzi6tQxC}Azh>kGy0_qutD*bnueXu8kcHz68 zx};IG8I5vUL4u&Sk~w;V2YF(gdvQtR5EFjVOdg)7(5Dm6!^QK z05!NN1uQ7-5BaKcq{dg5Y}9PdS4Saoh_43U`G)gV{7Ay~JHB#4V*Me`5FgGD8*%pV z;e1@+%#Qm*oMof_1wklm12RN=Il^e4!CnO`=x4ny)#oC=J(d zM&~Dqzs0Fl&j_4jKV!r1x%xyy{dxl@)lX?JaK22V6+J`sOSCyp*)mS%t;Y`jI3ET2Dk;EvrAZ@ z>x$3P<-eAc6NLaioLm~{xjy<=Bo!=4M-209X;<&5duG36CoqbiL=8eY=jfRYmz0 z$9i3Nq?~x8-9-td+t-dmI&#=^a=O4Lg=fj5EL$BrJQq>oMaLpOtfHQJL6W!Y%lJ*=cO5WX+~a}k(Y*vABAD!M`4)wQ5YtE zg%oNb8MV@IxfGQa>k>ubJ6EDLRe@McJ?Y+XG7qUwy1ze3P4r&)3fgrBas6+21_w~V zHk&cxaNz+2{~EZX!rr-__(x8tv&c#7iGyI19!)2G3s!ONp->Lz7`fe6Iq97I3^dmG zz1+q)y(~~3eF@v0mti%=e`*;vBu-zJPt$=y&Qj2J_n75`{WR8+v7_LCG2KxQoXOe5 zQqX-Xk?kBdqwTP^&#gi}Wj(PKMk@Aj>uHCVc1PC}Pf-ntrGV-vRXhqTg7h0e=U74< zg@AoM@e)#V(nTuH<2r6r0`V5W?YEG)Hcuo{ZEl0J5Pbuj1r^%3P~ohF9PJiz)|=2GhE*GWTq-dm{$@EVfngi+;v0`We6fnyHk3 z$5k&jaFwvz;+L03yGjW2(#+s0gn4OZUYeO!y8oW3WV2{URR`sdnCdf3X4RShlBr&| z`j~1mT$QMAe#2D%1y3pe7fkhE@Dwz@|2Uoh2w!BYxtl>D!m>W_G86UJKgKQ*O{h&McB>anK&_K=2f(v*n)+#X3m7RzRsUIzpYtj)CrnB4F0m zN7%T*gbsT{&|i z$O}ZZuvqv5&?@J~nyzAnSNkSdd)_fQ9oA{L;Vj$H}9@2{6%y{IfV5 z=N~0%k!*s~&oUIdP_-y;LFc)nmjKJQe48Qh-GHe6e)5?s%!jE1AE!Wbm zbP&VeNpC2B5bF4%NJ(yqdQB4XO7a16T+650Y(prS#&ae9t5Ao>1dRO{66~c|l-{D6 z5B3FcX(3{K5wd-d_WrmJAlw(Alo~1m4&nAIRPCLF5moK2) z%|p~5B9j22T2ZC2dA;RLDeWFy)0~Hh7Tiw|RW3p=_n81h$MO0Kfg)(dCnz*67&;)y z90>FNkK>z-_>zbz2O0nZ8@Mh4CC|7vy+7$a38y_<6QPy8VO$@D2q;?x3-S%kueo9Urs&bE~3`OB%m;jI84@I=%2$prybdO6l0lW zPp?Ek@svRWKIXWAPm$T{Jo6lfpj@6ub+$qkJ?S}^v=d6^{ZQ33szckvi=cd?+L=xr zLTYCo0;G2C<*cSVBA{n{UdA2jEx3847wMG^K*|oTm_31F$o2v#`ULU@Rt$r1ivC zh^&E^1tqR0&VoXBo2#1$LoJ3z0mNj8yiEQ28zb zVk+k_(mfGgMf`<81-v9~LR*Ah2`3&zXzYbhGCUR>r^xyWZ-KpmlqQ%EIN?M?nk?3Y z&;qBIo1ur6Kw`|@npFw6GF$b`0Cp#_Pvxz(5agur-{v8GT7;xGbIf z#J5Okw}EpoAtue-4ky!XGde;{i>`sICBm9Do-cW!pr)hqxPmHJFs)2eP@yJ5K_#UW zjx_k?Ua#<&G9t|R51izjt=3#)A$P?!Friw#jCE1|nA2plD5C0-KVr14}+&|+#;WmYqDQN%yp4~KVQ>Oo0Zi8lk(*G~m?LXr-<^PP`{xfcS>VM2`>Zxn2 z*Z+jc`Y&n9WZZEu_`z9Rkm4pV`);lY)sniVX=w8_CG$<_hCHz@Y%AC1t|>yngdIO9 z>-7c&({ZnWLhS2^B}D`JVyXPLA6G6*5dRd9Pu>PVD38BWET4zO7S(|6))uf0dL_<% zHPI`TTD<;y#quFhg9)Tq;yR-}s7*iUZ<~>k^t#{cm6#TxaWra`lT#bD$|@=yS|w_= zlMq1tYOkr4x%3XT(hr)CJ>_>=Www?a%eBfk=?$faBS*4~>O3zZl1Wu`Y(cD}h?2Hp z4n2r~eFv#ubbiBLjLO}lH&O}}9wLpB2S}s5iQ+|s0HJr%DDlKU5NsBzBb3H_NNMbV zlA-&x=8wo5Gq}=N%^&`U`6Flte*|yBgZdJP@U0R%!&9LEDQNjrBczG|CGi*mizGdd z5gCP&2i36j4^_i>QA~qsn9kpQs$qiZnx-3)ruitoH32pAn$?86N}(B^$9>nex*4aT z8%hGr?^MG~O!Yc-c@7TwLWRgJ&S5+B(zlD$#Vc5p{!TxWdSC@hlS==ZJ8~r})F*{y zW3c?x=sS>cZWbK>uEGH<()bse0H+4o#S<6bOzl2Ig!?YyPdN2O!M_m}k%}w_`mUmdgUwD|K2PrWi%Y?r|*Neq&axq7wVwDu_yXZ z=I{>eqb5f_&%s$9ZmlTFd{uJr*XlQ7JXq%S!ot7w%PBYnn1=O(j z{S1-npT1^&0(K$zP%YKutFVz zV{-22>i)0Mp1+7ZWX__h;wC?P&{Q5by%w9z_P!tT7kl@2`IB4WaHPT>^oq!c3=UF5 zg5MOGE+UgTXtDr?pQ0U@?ZMhR!qtY?L>g*u4@zv!U#1Rx?sIkCYBsq0!{XillN@T{ z9=Z4wbS2`jnJ9E36rzgm7nZA)tJwq6W6M8Y!}>@6Z8_%~Izl2>qYJC|!^f3d&p?{s zTc1(tM1voPns|tG7Xrj$%p7@?CsqWFTCTpnhCTQUc2ZENe>i{OpiT`=ECn4BK--WB zgsTX?fp=0V<$Ba?>S^5mMwME@p{DI_kN7=c&8uL|ilA>kQ)}0-Zg<7-)EYzC^4#EW z*ag1;%cspc@K2wL4*d8Ns;%h2jsO9&W*zt-j}sl(>r>Hz?RXq%qKzGRoa50t(7*xg z-EZ^A#tu9p3iEYf_wyo?=)g(=Ozc1%*j;<)9e7ftaW2@>n!ijPSo#TdU`N&bJ!>Dm zMnw7t;R*AFTztgWi|F-nl@5dzQVvF`e%2xR23O#P0wrrf*o+3R|#Fip)UVk!DlvMKjc5u0cTKQ{$) zY%W|#GS1mCa|gASusd(X0`9sQ`&H&r&-DsJ>Z=lVI7Y#dh$qw=roM{eOW0t2c`HfX zQNlXe3R_7!{Z>y^b2!pp0#7kdjJ3 z*Uf!r(eofazF6kKI^152&6nnkm4v)0-X8Fw5DIbSF=s+z~cV9#V zHka&0K+wIgB@i96AMk8VzKq2_$f8DUXLsxW+EP+uwzE}jdlNvJy$ZnlVX}z49|?Xt zn4{ZRO31H|9MyuJ6_Ly7dCYzYz}T-d8#)6peFw92uBHUkK~EX3J3L}<3(t!V|2r)t zwQ>i0NI%6O=px>nEvWN|_S4T#|I?I=^{^I;le*)TDd#l6tX} zRSX`76wghp$U~rQjK_=39h|}+eP{5;z`6Vp`3@eiTV?Y{@O_e$+oGI}k;3ScMDy}> zy4uH~FO9$pOo(mVv3-H4Zy&tE?<5)z&s$jglSh_KC#mWlTphOkPHy=gmdASERgb;H z3cJU1)A2A$znfmS*&{r|9C^sxiC!Zd@KlNHQm<~@#~zk;-d1b(u`=nWn`+^H_EhMb zH^~D9XYm(7%Yd-}S3;BPSmCYa(YWG*qBq>i9eRLWVfypqC3Wy2)>*#|f#gGQNLg2> zeszeAmK=4ep@Kz99qUwU1+G5ddrfs#;BJm_FG%Xo6#)FxHMPxQHblSYc}X2{m_F;;-?_(a-+2?37aY!2ufD^MbW=K`DM7mO7< zPpZK5n2cG?QHTFZgt3KX4~hlQN55dKL~$TR1`$uC5ksTDFtv2!*Q%4-dSeZFiQXNMz zR&I{r01$b&%u}{UR*#X?z)FmjemrM=WDgEd5jj-=Ld=qx#%!noV0IYp zK;Lqpc>eU?1)$nWjFrcplXANpV;?a6;O8WD-Eo$yPfwQAsVCTT`s~q?y6XfB)gM7X zI>{~!?lf8=4SO~pBc1UAE;aMVz=!!Gaxfm??P2`kAIl%z_d2B9UZ>a`sehu~H%0*D z9i@$tHjjY4OYjEPK7j{{fC&r+?bD+q_23!Ip_4{Qx$V5HhV^(o5$RCa=}Tz$p?G>n zwrq+f3;8%8{wPsW`nty=>s*x9PJI#itv^VtvZsD0L-iGQ2 zA<&|R_0wgXEQQ~$$A4jrWaxrTL@hWwxw)Ih&~Qp5({?VV2+AGhItM3`#X zCaL^H?#WtqNRpngsCk!hXrR0@_vB@k=_f56lKV;>%Vg5dt!nLcxYZSIRqwt5-{iyJ zs6%hC9#Y0u_0=1!r)1lzetd)FNZ)M94Zn%`ruBAQT#LzRU9rk;v$v$2uhqG?;nO<* zYjx>u4B@+r)t_#&_~_R>9P9=D82Ai-L|X9xsn_sF&`kaaeu+Q2+wlOM0^nsE&0_{W z;>nG#XJ1Rw)`G>irEb!m0<|lXW=rh~)Ep*d$~pPO7n`w$K$jNgt6`GVLmHj0CQH&& zQtNzmjU@GOHHvs?ojUbD}TMcJ&w`A3nby~vY}cb=zZV;qzfaEcL-wz9IPD~7y#dItbQtcFv84UvYuMRlNFZS|-W@!-a_jj(;q zKGGE&4R{ix+=^Ooz{q^d~ zaOsAA*V0UwcTe zzC1+wv)+L~i^%`S-J8eN)cx(_r!!QN9CMN!M8+gZhFT|*OgZK$WS+?!(mCc#$+$y^ z+mIwtG`CM>mK^hVa}Y8mgzl#EyVg0Y+i-I~pXYhLzwaO4{d&E-_I0iI`@PoMYwfky zK8_L|7aeXX;u2h;RKUp~HzRV;n=x7#DDtZ$;@;Df<7OhxZ;UWe)CSeG z7%NN?Iad&Iuf_`1qF59~je~O91XE-@lr|`8F@gFR!)F5ZK~dC1>LZ5FBSC-x#qLMor}2K*}@Fr$zm>Q4oy(VO`1!~i@4_VX!%zzcOH!| z6L7XEcnrStuN7B-9pgfs3H@ncgryZjo@fk+9N- z_$mRyww0@N7P;VqSNvn!+%|MiBy!;fuM-ZpxFmEDyyCEOc( zQRotHy`3BcB6!d@ZS<<$+9oJ)v@I#fpMf&hz zdZ;wEs2$g9t1v|B;sV#@g}S+PquAV~A@_Exuz@Ll;V@m$Gvl4N3455(-=g}%!8tq@ z*5p=&39S@6chDn|{~cm*aLD9(_yzuu7lfv+Q70(V5bz7+}ZN*L4E$kutuWI2ULWT8O{s+lrxb%h!IQ4GKsq4FiHk@CW(AdmWu5X#Z zRL)6pfjHS7p@Z`0-v2xnKVJ=jK#nxu?B{u)@9FpB|MOV>ICQPaf7NkAn{81kJ^sg+ z*_u=B5o$TbUZJs7c`{ph%P!V$o;q8Ra^!r&gf)dSHMeQLq#Cy|S}0MLx33&4J#SoN z@#P1k9DlxGKXWu1Bl4a7`voDimw!()flMja^YZytj(^WPE!lRvac)KXDsDeFu)I(D z_y*SYp4%(D>dH+B6G}Pb5hD2(KO-}xD&~vltPXS3 zf{fZP0{m>!{Aw@gHV?e-%<1ZL3&Af(c3lV_cD<<4Q-AQuTius{cbxo}$?f?zY5AL+ zJ1zwuK1mxZUSN|Zc)Z1NC3unDl=|t2_k9sYt_}a`PvB}66^KSR(DDb)kyJmG> zphSxzqvSCVE-yU(E(HOKxm)YTfrkfN9@`QXD2@*CK+5aWw~xF zxWk8qhk5Y!$1G;I1J^|Uxi}O2<-p+Z`QZ5-8hf7ycmA_qCpUUUJ`A+E4Bl5UdFxsw z?XlI&x7iS?MO0s^pqHq0PQrC?x9fwqhJZ^SPgr&v{6JWr{o}wx1U-7(2e<5bGuC90 zO`3L3J?SIx-hNx>-9@?l)2ka#z?bdY)^-pojA?Up+jH=v=Q356Xpmy__<^s$Z`S%e zOPP)amFN{|v^1yyJj5sR+IVp3v9|I;@NP+= zvDt`M*UtM~3?6yvp*jrl>Y%72I`DV>+e>DlTvmO~a8buOYOBM#7N%%$`1LLoO~8+} za#6;Kkuc^W`jHI9;s{&?y>IF(z>*QbB(N8fsdcJy;57+qjr^J9l`r8 z%?WY=cX(vo!x_9v`q#zI;F^Pq$L+zF90+(mls0Hy)NEJq29BK@%%NA<(NXLU9;x(g zmJ@&$`K3ME;sK%d>tP=H3JWvpdx2X9?<;A51ghj~$9jTyGroDF4F1xmRb2Xjr@X%C z)gI;Y&MVdZz}HV5zd8ryA;(*G8wB34>erKIXi$EUdDc+yuo&4UItTnd+V&X%uB_U@ zDcl1AzM~gtd?C2b$@$|W0#s6C#W?Wi9loBdMtgAf`jv^`r?xEDEJL}U$K&o(!58k3 z)ftI+S?`u-W`N(6d4~C*T>f!u>^$&6uQcmdgFCFM6&3(q@m^K;o#6Q~8B3OToTF4v zsI|7D1Qp0W{xM`N93s1wJPtt%Rr@Zq*aSW$W=q#v^pd@IE#knl>{~usfC^(?NYnR# zTV7gFa1~rN#e3;N@WpR?k1Iq4n)afOQQ#Z=M_2PkxkFyZyp!OMTwYeYNE=it-j&!< zi3p@6-vDqJ~;JpD>^}ITvbRS&qeJ0V9UDFYUthZ75WOv>0QFtE!sqfT?wt$aM&@TgD~ zJoTyxxcixaJ~hD=R!J=^z#CTV{lb>^@KE~|)xnqk`LaVhaCJt(Ei3RcGq=8Usn$IE zTwhrqLS@;C$m(c-bab~#cHm!po?bbD1_W^u+b9m77MP{I1MhJ zpv?cW3__ztsmc1KcmJ%HLEy88bpMnB2lc9F*Mq@V*e{(x;nz zOY+X0;7e4`M@Pcn?}+1qaPUDNdX>n~0Kcv&-iN@?I!07ar2L3){U~q?;cO+%Cs2x- zygvcKRNaD$LJy=?x!S9V77fsi8#f2_k zE`WDlXnxQVTyv{+R5rNy^y^>;G)UWZ_K2I{T~|bS|A_|prkI=E2d^+RGW{j;DZ?+B zo_q{pfW`GhG3~+o=>wmGhuz!}Om8C<*QMY!_@tJT<}RVU?xx^;@Ru+8WV}N{#ibFp zU%;=gdfP1)4N%>RIaLh4EAwP_0Jy{WO`VNg=cxBhdHA3M%^z%h*%*9}t3?JDDrK7A zz*>FGA&kEIsiFy5q&{{jy9&6IouZ31t#FZDk6PffUr(xf0z4$xDcu@;NlC|q0q~EB zDs0mL{MF}O8H2zzs~7FJ18=jXZb?6Sg)-IVEx{M~M$cc5a_z@!vmL>^Y)sm2xd_jH zhh1?u+d;VNSWtQ&JyNxv>F5exO>XJXpZ3VwbbV*=$~Weo?}Tzq?8+zIz|F!In2Bfu z-#HcSLRv~TF6F?7u01?Q)MNAA4U_au1C(4{4_z*iSsyDGrr z)VE7r`vu^EH9fBvph1Gp_lF07m%PaJUIc%6&9s$E!QZVlyGj=>+1%`-tHAH?v)wWd4^Nxi|0^1(snW${JNVl5_M;PNkN3OU>;ivtEYY$n z8Z29-Y8?*llsI#Yl=irB759VS(~BC;rYm3glY(B6;HUlw7}*Q{`DU#L9tWSf^u2OW zC<6RmT^bq(AywxsxQ+_)6V4Ay1P^=DdWk2>1(7X=q=By<{PaL1t?+4C-%N1cQtx{hd8>N;Ej&pOv`zYqLlBreUagWpT<@AI5qfzKam+yOt+L0R88 z0u{t`aV&cPp~ku99p=Kp_jc)1E%=g(!{ZWY1A>O0c@Caf@UiV}a9PtH`(K0G?Ax9> zgeFwo40sRTt7}$7ExN<$UfRC^yyllVk#r@Lxm>jS3w+%6{Z}2)ppb%2g{9z!l(T&r zFF^vS&4{x?w>fIxPU@HQz~w#cH=2MuUhg;J3j+Kkrv{pXd-wEw`4T-;T>orY1$@EX zOH0$>uWI)BZVm7b#ittm34c{!{5C7__mT7F>BO zKt1UIZJ}FALK6rTqMlG7|D9R3bRUwd|U!}EW{ z%#I%((-wVf@}xH$TD+d#U>F=iuKW1+1Fy0vbe96<+TLr;2ZOt+ULB~39y*LXwR<>t z){w#J3h?mHPn!FJ%WhgF(WNz{tXljy@Gsd%ljx*V5AQ=JgTEs|JbfA}a=A0p-SGT> zocw1feZFfJeDItNhj9yx(=F*3-X3*sKKSm^h^`4J_q+I_^I~x0&n=CrE^muT^u-&yifK{7l zf;+n`M@G;dG|2g|4cu|u?10zc(msL1c7o3cPM*0I31rUR>OJ5k7gOeSLb+zd;D&#I z->i1uUkR@0x_kLyaQ`D$F4Cu+e0;+vN5OUYTBZ;U3~%+WV+?qJZf@mD^fs54ZBfQS zC~mvfehw;dm^SoP0{EMw(dG>h;JaVgDHZ&E*V8U^8?CBXx-K0&_x7j&8F>C*J@1_b z_x!7H30iG0&PkV}gR(2iA;$_K z)SBUXoW7~VG_85B1f2U~{7MUleAxh@5t*aj?0@o=6?z~zUZbf5eD}r+baDo7eBB|q1^gYt zUGH`20718JV$>G|_(_Ys+`xy;;_Q78AisO5XJ>H1lODI|7;kwZaZ$#v@L#p|Oi#YT z@nu{0qz>mx)3#2naB&>p_+9Y~$Z=?I7rHvYnS? z9S=>tznw3v*YR=YF^<>t&2gNd3gur7Yw__wza6~KRExzb+nv1hg`oLgpOriL1;f6K ztbBeKZx%3eP9_(|&#Ps3rpBb*{6SWL$gY0sq5%=VjJs zLmwXIU5~zP<|~ij3%VsdD*7XW&vEuQoosT1?{xM3)yOGF_|EcUKI$t+_}A|%+*TfO zh~&Au&BjjL63H$}B!8f9_W`4Z9OYM(ZjkIuI?4~a+GzX{>nQ%O3T=062#n%496EO1 zQyay{MfIt9%l#N{zV=9@%OA)1&yy@8XNjZvkX0AjtHww3ra`9y$DNDjWmhA2+t?lF zjkCKXN3TAPEB>~)dhT()rYc{V-o!SB_d3FLbQ}=FzqEN!(s)G-zrFdES|)KZ{K%S1 zcVB-N!`t`h6TYU}3Ep(crq(UGoZuhbJ~Jk1_6feGrJ}3T{u6u}zx?%%YbW?Qt#^)n zBRI*wyC0w4#Nj00W9XY2{YIVSOLn)MH8J=kKdai9oJpxC`I+g1RD<506DsUn-4pnu18Tujl!Q4`uO_PxJirZGSko)bhN~!@!mKVl}@iZd`aqZ8e{8=KhEE z_SootI-%^-Ma@6{Bd(8`a)_Egoq1?t>NGWfMAP-kq@`-!ILB+1`F1sbHQ6^bB2vv; zpJ`XYl~?orve}O=-ca+6PaUh2YrMsh+X~v}U=cYS!slOndU-;$xjw$QXdCvd-I%U+}bbfK4i#9J~ z)A`cE_k*o2r1OP!4|MW;n9ir}YdaR$FsUaip{%MAY5jHhp-T4eAG zD{osAC(qzJEg5+uZa@a_wqRPxu}K+xdefIhdjm3fk@JTeYeO>lW=V(F&p435Hyqla zSGY=z{_6rw!F-cU+?VvW%;aah3r&1#pUF?J zAa4}hIg{rUp`wD;nZ#_<+J%E`GRf&SpYHNznPhQpfwjZe3{qJYFF*V&gIvy;NE&G} z$f{~C)gx0f$inKQlG+~0AWj9AEiY`#AYmzoCXQZ~LB32^dXG1nnL$dv4yzwAJcF3D zvE~MPWsr9FUZhuOn?Yig)pwt%pF!Sj6xpt_$RGvsTK&9>)5*<{_y%=z(}|H=-%THH zrjv8(fjhF()5!&oj?O2d)5*5|0b_UWN++#{?yz1PlupuD*BGIi2mW%QY{{5(@-DOg z7*(Hi60ZE}WV5zYIx&70+h=Ffbdo%A>c^Ox>7?P)BBN|kI_clgc+rP^H4#s3JE+bh zHQ713?dG1B)x@r9eA8tKYO*|MrX_z+O{Q_RhY7c+$&n7TU-wk0iF?ydXLn6klQ@yv zix0!pq)TMy?%rN%(y?9b*D-C?r0-u_&(vjXmoF!H(rx;b6Os))u{$3&t^OpQSRL6ox_cL%?3~m*Xo(F^rjGW#oBCH8DS6$v zhuN((lG|H(YfMZUS(G?y_W5;bq*r^3+Ab5*$fw9@>2aOYNK$OnSXY}gGWYc7uh;%c zRg$3>=g*&gE0q)+op8tIL@H^VDynv8eJW}5VN3m8lTyj^^W%!Ac%+il&X+?yWvQg) zgN7elmZXqR{?~P~yD4P)q|Qn8;!;SJzAtJw3rQibo4eGKPfH;UJ0wPq_DUg#ia%rp z+oq7qjb_h3FGwNMotnKB|4b$?e2%&gPgEw8)$0!rIK3^IJR3agn)U2tvhj0rOt5z{ z(d=}8Q?Yq6IW{|U;Z~DmQgK^_flYFf$cxga^UkZ2$chz>Zq3@AL@r!EHq~xX5?Spt z$}x9P68SL5wn1E*BqE7<*JhnGiBvU?^_~7Mk#yZVd*852iKP4J*6;frP9*skiaHNc zu1F-gPqf>{jZP#gw-W6Vw?wkw{nWbq>m-uLiwAmKD@-6p_j0@|-cBHc$MkYm#3qnU z*Uo&}8tncZ9iPmwprUW{GibBff=9;1wV z+~E|dC@^m$uYHOfznpw8@@pIs^ys_4-t9Ot%I3tXJ#lg5>CCe#yDf1f{^^pnNwea} zs!dmS4_3sHSH`E}%Is0@Y=7~1r8qLlUw&)K+gS1;@XPg9mt%?90N<3dBe6uM*s|_U zP%K$)8QCFeVl0^uawhYTXDk`;KxtBcccWMm-fvH@-QrlbfSe@pwSvATWt=2WKXjjZ zci%};BJQ+Zr#eZNjoLrk!S^IFYjCb;3OPyK7k+MX+~y<^hc5qGQhI{SJaNHyz@I0G z$*20GPNke6+Q1KQ>xG>ln(=$YTNj@ok^8oWR~deSEUnx%IkdwG@UC@UV~D@|{N??I#1NxRR@t^L zF{Dsl8^>eh z;6a~-cRxmv2mIZvjj>T=knPvHL+3`3KWnPSIJJ!;4K{d$)cL)VtX+zxbc!mCAnVZyfDNql1wetRIVt9 zBzG$1ALY`Ikj$DruC!f#geaUUO?mEdgj`Oa>3_)h2$`#h$@9MuLHh5hk?FHGg4EcM z_oi>J2vTCXdDIY#2y&;-sOo;2!=!>&s$j?F!z8R$ojNxL944bKzaMH->oAdyOp0A} z?-1E>+^AmeF6ANO9WY??gpr4cd_Z^c7ug{;dk>N?19ql$J9Lng&F)c9Ve&zeFwaBu zXY+%^VPn^)>bwI)w7F`~sS^i?_|<|pNpla7x53>-*V-K*7RF=GeJ%Qf)KeU8-YV@6 zB7PxgwqWTWB>MR1gsUEZkkaCeD=wygkU0NFqfaU??@P51001?_sZ%Nf0oSnRj&qG_~`cyzD3D(h7^*>+{< z*Sp8U$*#vOfIJwozWhV8t#n zbAVg0{r#O}Qpr$_?YNyJcktek9rAXN{v`{W%=F(uxHGPnvV}{+cCzWfjmurTY$u`Xb~K!wy^SPg8r!B0 z-A1leH%)E)d@K3*a{HrbzpccjL!$q{k}YJ<#-)p`g0_&{=N5NLt8F2>53gUW+q;?6 z+3Ps8UW?5n`jv<8h=ff<{kq!HM2}6R4k=2{486Eor3#E6MaD^GL+`C1im1xmkoLfCMhn zZMKpxA_u!|+N}%lCn@!}=MPGnPg+Xb_LCLPCEY%$gN8fLA^D>chMCTrMT9Tc*6lrM zCdurx_|DXqQ%Pc#)!}cN}r(St+o}l+h$5Y)RcI_lA>NNuj;w&C(9Q zc8TxmcfBX6y8Wr;=j@JLWwo%@miLlo%Dr#C*08$RNc}r}Fk#==h@&;-zr$Tc*T+@Z zVygZf`gh;#1(of(@&|PkfU$ z(pBHkx>Psp4X@oPd487kfv&_wetruDhF#|;RjuUY`hyJt(Ns6=W*69pE;(2@P_L`z zQ{AvlZo9=c$H!xU!JkllqJH~b?XP^VPr`wz`ieC-p&Q-{GOIrJ{Zn9QkSY|qQe}AK zxKZMNiksTakZ3dwh||Bx?69f7Yfe!weIiv;=Yw%RbiUcM7h13EW@ylI=<@9j#v=qpzBM1e|sWmcOr z#H9bb$Qc2ShJfnO)$^8-n`ek~wt~AA(9=-C!|SLZE#}$$vt)g6cAv5#eGL8wq02+t zPp)&0+@F1R@R}vjWHrfg%6jy7H$2!6F2xedY%jWs!v133sQs zG%zIU3|+tln#W%u8!K zU(!)ovahwPK9LIwRCJbJCD(6Qd~Eu$l_AmZ^o!XT-n$!o%v^XmO3qJXkrhyO{OhcX|N){obd>ZoNjf z=B3W88`Rp+(BJ8m;IIBT%m#BNbAZFCM2qT%hU#_YFADZ#5kY>_ESt&K$zQJ`dmjwT zH{52w(_6t`RdIg$t?Pt0t(^Agbq7NQzth#1;cs|WG;7ztV?rC+<8pyu`csd|sv!iNl!g^P(x}XN}=Hn^Ij>bJwE1Bwcpi`N&9i4~OJIcNsP`=rDQpR_4^A^IU_!jOwQ9)~};U zo-O&y!+)GXk9h~(vhYruSn_z&s79v_()AdJoa(R5ls5$ND!XX_93dKGqa`|bU&&)A>;Vz=2G`%$ z{Kk$9vbDHMrgErzs=i_c)i3UD?si_lOuwfU;OkMbpCLc`So!*IOE9(t?3{`qQYG0W6~@$u9EOC z)-w+zI~nvS=<3C<#0NFx(Zb8?PrYOdzAyDZcG+j@HNwwrDc<48)^$ths`tf9n_egP zRoi^)SMOkGkmeQHFvNzRInZe@>Z&-eK$z^eU4*8UAhvBAH&t1~K%0h>jT)9CHfi4ess_t=*v=T&YsMp)@BohpPu2q}ce!WL_ zubz8i)0m!0Lx433!W%@Jhu3IFC;mo>N^ksU0qwdVEseV<;{r)9d#ZC-3z*0wk56ApCq!A^~le&JPmqv zB@K8Yj=cFNDYeRP+4nA6Uvr)z;Qpa8Gaixdb~m0CA7MR?f$kfl+*tE5nPDDa7B{n{ zA(4vuFFHOX_A%KqF8z;DrtYr#1`UAj6`SSQKueq!NQ%~FGJjj>TDgyfa<-OKFe0U` z=35&Q6+T5l!_tq*TH;x|#}U4;twFy4U8TOe*BWz_X+79 zy=6c5r@g^{F7$|sNvC%|Az9HaQ~UQ~lhF%$uHTGNub&X>@v1kxF@0Q_yrUse;S==iqRpNKPf6vATWjwa-q4_HpvzYL)s%lqmY#XK(8tcrphrNL z_Hi0)@{H`L*P(x_3#>sZ=pnn0HBmex1C38x)~#@3y&Ur78Rza4H}5!78xU?x`tmmEtcD{<09D@+oa#XfoJ*MlBKrl?ipnI^8J!HftvsEK_uOh zFTvZ~OQEtgr>ZS7sm<&3$NPUE-lFQ)!~aLs>4=j4ztmqLd>V3@wM9#|j}&*UBM`XJ zCgJ!8>XYGD(b7Zk8-!1T_|8@$HzQ8fL2SaU9w~OKBamah(Bq#7f7xMG9gz{<`;6l! z{3+BKtzP+92ouGxd@Sx*S9#%ii7w=2iO%m;iB9pRMCb4h_8#^D_NhdtM!5>*z9^TY zT!wNf%B3imqC6bsz9^TYT!wN1%JcJJ`LK_OMY$a1GL#EYu6>J~C|9Fg{jo%;Q@}wA zhy2_Uo#u6kE(Q)Ea0r1z2olIpu6+S#l!v2Sg>nVT6)0DrT$=+OqFj!0DaxfN zm!doz<-RDFqg;k^0m?OKqz0YPd{Ckg4l+38qY)akQjJ!Iz##+RS~-HckLy8ODOIu%R` z3%4%SIl$C)N_7gD)~ZzJ2NS?TYM1I{uo%lyogAjrAb0}oHUcNWz94WFtS0Obf}LUK z5I6|-0)g{k#<1-OZVWq(SRYsxVtrt55f=b6hlL@w1uO-zywg83^tNdy2r>Fd=L+f*Zh2BDgc`I)cZ*J|cJ-tSW3jV!dDqi1mWqMcgDx8y zmaquKy1*_Vb_gsNasDt;PB};^xA{u&s!-g~cJ(19lU!<6s3!1g?NJgY86cGguOWd%*4^ za0;vlfoow_ut)@VfL%iHFxYFvErQ8lA&8a1Vh}5bX%OoR%SW6FCWVC~)&ZtQtOBOg zBG3;et%QK;5MkFwt$5p)*2RuSZi1q zVy$6eh_!}=A=VnU-U8!ag1~985_60tf*Zg#Be*JTGlHwaHY2zyY%_wZ!ZstgDr^}l z|A^Qzu#bojMXV((2(hNHAjF!&f)Hy83qq_ZEC{itFnKm?Z| zFc4OTz(80T0s~=X2n>Xkq4F89M~L-?Jwp5n#EM~a5nq7Vxv&Dn&V>~qb}p;{v2$Ss zh#jZIkDCbgfZarbMF=iJ@D$i<1W$pzM(`BaYXnb$y+-g9*lWZNgIz*w2iPUV`yoCb zvA!@ZVtrv+#QMUti1meO5$g-nB37xu4>f`vU}_}rMFK5?6)+8g6)+8g6)+8g6)+8g z6)+8A<**pU%4j@RL@kzGU%D*g+2H#b_yt%9{0mqNECH4QbwH!io;s}%m5NcR1W9Nx zT~d+2w*ZI)zF&bz;9CSl0^d>~68H)bV}w`{5b?goT%btOSQ&?3%y0`2!R)i*)%Fp##0lp#wvJz7L9Yrj-N&>GL981Tf@fk?sQg!+~0046qQW z21@b!9(><|1O0$f{5}#Dut^1hQXqR*q*Fm~0ESX5EYfMr@N@>|0u?|38V~}M1H*ws zfU43W-Ce2!ebJzBpg%AMs0ONOIZy-42Wo)|3!n}aaQLK|G>4|kQCLt05O{?Ql0bPKC2F1Eu%7OA4z!nIo4L=8zTcJW=D9{&}0F<{b z#u|hM0`CHSf%(9ECvfB!v@O;(K^`ek3sgH7>y*-3xI{Q<0@MQKz^3Frq@0YiXN?Bzm+BM(sSQ>?oHl#MJ_ z>QvTn9EFM~j)em-9H;^MjzdL2*?3e;1J>gT)J1}A=qXUM8<*M!J==qdfHB8WUJpD5 zR|u#*fhzzEIf*MPgMTc_fr2=66c`hyL`!kvcM2_sqv{%Z0Pb{3Ze$ zBk&f+6sW$3p4%b8L-Y*4SVRqMLNO0Rfq20O%$53x|BA7;MMr>^KtG@ZPzA)_LK0|| zI0?mxybxmz3Ki0#FXr07HPev>Zr( zk-NYe3!)?P*r4G~s6SkWgwCkAAp(I8jgiO&0Zq`L4roa865WtaxODc|Q&MyU$LomJ z89kMQcfutFO65oh3%as z0SC$^prybVU?DJn3KDfkMN^Ro=m3-hRWs25VE95boa&3P*#-IrVkmo|9H<3`01GKD z#g*s-zf~nVrB(r96B6}BL$;s+K-D(145;Oh0GJQd0tKN+0F(ly{m?*Q6QCbZ4pae$ z07HO&Ks7K3sMv`i>W^}u3s4PI0R2vu4Af1+iQ){l?m!t{L_&dDY_(!2$9Aj`sIG@6 z=K%Q0uvw+p08dXKJ+M6J52HnjegY$ZsqwBsW4M$Gl6s1^V`PFlb<9mnLyQChtfK-C zSWG6KKZXeXPpbRCB7V}tpwSw_p*vK@eH7c9t6Wjip7WJSjJYXhl0iMyj}58Bq&FN5 zRkT1=2~5|EOvj(P6vX_b=b+37dc@-*u3|;W^NzmHe$h6(lMnexr|rV8auX<^hvQuQ zrYG0jT++K?D039=?Fm^Ra6)$ao3KKl(`Tr7OHbMfH}0Ofc;fziDV)Se`;(42 zFK7YX3%VTV;;TKlQC1R5u8xJIZbPbmpCteuE8s1Mt_6yh_uz(GNaocE|L~(D?L{XE zK5x!0;@((D2KM*M{?UOB&3ZUAVS(a}nAA9^eg?Yir-WE#1S!y6kS7G^;;G%aEtMsW z`se2VGqEl9I@(|8ouJ?SNvCtc3AzG_V=5rAs5`xk?p!aaWPr%`P7(KqRB})h^uCDe zSw*t4kpK;oBSs*`^%q!3#jPor;`g*+4q^{P8?NOlRg`q(9I8r8g{Q>alO_^p3tFCq za{tfR5(znrijoeT8V*LoxV{x7ZMm|l5@VAAXn@~W%$RDD?h_Ay#1#LtcPY>#excul z9{vmcE%eZzbUHp_GfbZ1A}+a_r0-xKkh^elK*8T#YlL%jjgCnG%5#58LiMfCwLj^$ z(4!3bxUtnG^-XSqT;Uu60wObW-oG3rGBekAm18-IVP}>iSdx6 z4WVO2%V|Rd7HFrg%lG$f4N1Qa=H|G>Hh6Huwfvy%w1w{Xlb!>;6ZA>YLqy!3CK&a0 zH6^>9ZL#N-*8gXcPS7n)f7I#396rixTlOckWXyNo%{g5HumV zo3+6D)dXQNMus^ht1Wr!?St+|{FIAcg}Knff1$639{Q8s3H}G5ht%VSSxGwB<>=3U zY#;6XL+C+>mEl}`hj7VOlKOV$RWJ-JA-;)-A}lj7Tj+k}&WH8RbvaobiMa9rl=z}V z1uyY(!nLU*8Cp4orIF(X6}KYXt~!#XX49%-8K{O$qKNjmH87F-XS!ZwdKUCt=(2JVXbsDO9#VtbSyy6ZEJot+8eCFcNx#aS zaPAL(0d}n7{;pgd8%bm9(1l{`nDMe9njf-IY9X2*zEE0OG(TdYxREtC!bZ|=Fyu<2 z`Jq1)(&m=n$6bQa#CiEa_3GHt`^m9gq>Bza=;Sbm?$RCAV>>C{(#cSL%!0qGk61vf zPgp>!S3_>BuU-im)yMo$NUO<0x=?p)m%poq>N(Igolygv&PbV(VT4LNlU17QLDwFlX4WeN2M%-TX&|d_+uJ|v&J#4BE-d-Qet`iG% zt1q!>6$OV$%AcCa7B`xJmiwcezR8J29k_M%C34{%DR-s5qz{>fp$rPaHPP2+_(8A9 z3g~L+KMzR&bj$6SakxO@F6|9Na$&xB0b=NkP|v4BqM1*JLHx z-T3GII1ak}7rGC07p9Acw?$=G>S1%CJ3#lvxwubTuD_kcz495BL@!ou%aJD7yREmA zm{zWi7ez(6`&B3I6WmQ!K=i#)%yq;9UoQ*9E6T9WAgBIVfuA|hW1@<=T}>oQEBC_d zqGhyU-ii;m;hHzaViX6_kBYbn#C_Trb_q9|N)#q@Wmp{WOZu8OlPs)!6GVYm%zPvj zbB-Ke4Oi+~Gf77~pZd7F@OCOiN%=vi-vH<#(CG&U;&zVTR(P?-NPLBE4?0?CjJ}`y zTv~HU1Ci;qV(wXUiEEu8yvf?rjAJdN-Er%6!s%RC_;qx;g~U}Py;aODZXu~F3c<;) z7Lwjd6<%^3Xb3y>Qw>uo^cVU7=rO<0r$N5}{rjr*cLsFrFZ3|zh0xJ|kMer;n~OLU z5&*TtGb*Wz5@ZXsPEbhofvDe7z? zTQBZx(PV?9n?;OJI7s4cCh21#QB;-mttsheCFyS?8PGs7u<^v~?pM2Ox?l6W?s>!W zX4hNYESg+FY*uFr%d^OK#@)iy9SK$lSBlmv#Tz8Q%R_57JiN>?1Bd*gp|;R{|5puF z{>Lr-j~W_>2F0L3-_Lk@$+MtGK#!q4`gTRTS#Y=PC4K5SVR6s@MOZKBxi9o1HM(9L zh{MZYbY9>jfpwtttuhO5JZR^q2KFklK6`w0fc5H$kHp zWZwie6?#f5eNz-P>*#x)_dOqYKJ@(4^O5J{u3E1%UT3|sysmf~x~cr{_4DWK(*JSC zS-+LEpZ^Cst{-JO#@5*LTfV}a_%_CLh!x)t@yKsOto*+)#;mKq?5Xnq&oS=(t)IW^ zrc(Nkx%_{8jDOj2)^CizRi*!Ma{oQXC3v9?`p?JscYUW_SC-%D|I7RM-^TdghFJN( zy^sHcG5)t9R#wul694P__}|9(--cNEZMo7fVEQqpYxqNaC{TzomeF54&klNj7eE)h z!#_p+aDNLG-}fLKqtQV6-P#yJ1$^!0UT>|(x22wjaKWu5rpjLj2z!2i|4h;+{q9lV zs~5hzef;&pcPqc)(tfw{%k>_Hl^>^8l^pbf^1B64rdPh3GE%+p-O4YZg4g#8y&qLZ z|Gmt2ecbg-*Rx%-x|T2UYLr`5`h5o~m)``29#rD|+DPgs|I^n0_ZyE9q(gZVpTmS; zD8qjk%GvC(r_?t(gm&%wp$w;jXsywsUOx=wcNgr}A;Rwe&B1J-ulc*@o8fwVH@tE# zqP4_PDb+jv2Ltr?lla>J{c!jGet>=&pKte!pH3&k_4yySlHOQ!DB|!MugLq|P@scW zm41C$|I?8eX_$lh(fe-?#s552$2w9LfDdCrzu-01NmAP+45YBQm`iq&3{{qZcGZk;K+mnu?R3RwmSH%@r+-PV5=R^Q35CGnpOwCa?(Yz9<|e5sau(a7 z7p?zUnKkrK=-*#7`VqI2-ao#dJG~yA8e>o67(c8Fu7|O86AW8hu0vN`wE%i9=XQ3L zbhbHzbG1AE37bw1x&D1c|DoYS=y$ttMO`Hgl;WnCK%T5NxY2_Soi%jRU+7NI1;5aH zK`-q3&phLx=R^0Pi5D6^J)!vnpf~X>>Gs2$D=pZH0@*Jrh=Ohj{rk^Cw!;4mbO%rF zsi$PK(x(}cc0-4ummf6eT@^TSK_4UvvK|hRnUw%;i0Cd5q65Rvr9Lo=?r$E2p zSE4(MI?E5L--M2Tp0(7Srh)9BJ$?(`oFIW5P04}E#43D-2f zw3p<9(fCj!$*YD~uZu;mB!a`ncx(Urz!IrH@11jkc2Deel#pX_@j4%`Uo<(U|PI1=wh){ua$4n{F%Rmeh~qOSpO6E zd4a*-)O3hm!Rx&K$oXv0-!d#)+00H@=-~_2>A8v)Je}Rq^%hHp1|m-yHmI(evJ@Q_!x?(Sw{ilR5X^87!)S(S46_+NVwlIUjG_J;e)I|; zCq3*Lx-s--IFjKEiqry`5X^87!)S(S46_+NVwlIUjG=`J$#B@H^dPWj=*G~S;Yfxv zfcnrtCImCw!!Vj*8pCXcj~M1LEMsV~RG>d>7}_&*W9ZFrB*Pg?4fP8GnGnoy55s7N zX$-R&K4O^1u#BO_GS&cw_6*$^dNUl!aKnM$%y19GXohJFvl%{On8&b;p~Z66 z0EYGq-57c^9LaEolAQ!H3}(28VKl=uhS>}sG0bCF#!&yw4z$5$g`pw#4BZ%dGaSiK zIfI=9G7M(8hha3sG=|v>A2G~hSjNy|CA$I)?HRf;^kz8HfJ(s(CIm7JX1Ir8G{ZE8 z*$f{s%wt%_&|(#907H9*ZVbH{j?^Q@e+Cl*83r@l!!Vj*8pCXcj~M1LEMsV~nl*r- zJwrE!-V8@l>o8cpdd8_IFKBA91{Z8wp$EeO z4977HU>L$M?1NCBAuL}=Hv!53w)Lltok7{>4b!zhMv3{x1MVVK46 zW`R&&zu+Mgav2sd6tFQiXDDT8$va)w?E6%2hCj$=6Ov(hkG<}x9GVGzTu z48s^6U>LhJps{cBsy<2}3zU+b{atl&%0`PZ7o{{HkFc z@Y8q4j+G3L*Z4|aR^Td5kqqM0(D zKsN2eS+xcZCmC#$q`CDOk)w&fe#ew@)fNh>oc+Ye*^+hLx>xoY|TxE!%RUN%o+ zU$uN6U5?maXFxYA-^42dxFW3Z}IUf@y7IuA@H$OD0%CSPY@Aj0v*ZhFtXPB7YxSPE^8m^_NI1;%z{$ zYU?61;;O{dV^!5N#6@uK3ni_pl)s=1sc(?)a!aW!w=;Ol?F_P-ONU%n;AD{XUui{t z^&$EJ06X-R%N9v2Ym^r+EPsch9)>~pMY(99J36yS;#xz{!;oI%V#rY!(2l#ZNYcE9 zW|^Vra%V%4{wK{*3$LG+T&KknCnFU%Yq6x3vV4wx=O_>fB!coe^`mYqD4$$E>L!Bn z$@QadiZ9K7uNcln_`}F-l5y<QaGzdKs41Z$%Q7ANA^j z^11h;UPDkm*{D9ayig>lDJY+GT>|M|55p3I5q5x=Rz{Ywfb1hHALpUtb~4zI}=_h3Te-8=#uNR9}p3 z1dt{Q58`GAO6+?G*zL)R1)KDNl?A5k)-Tuh>h*s%>?m}aFGkC~Mm}Zvl0m=SN{589p5f(I>$A{_rS)OG~7gX2hSL*-%H!Y)clm?g^ z3h0-~^oI=|PgIvNy}UuL&}m1?KV%DG{@tj*pteBHmJ-ABN$~$QcI|Oe6<4}?;PMcZ z9(3s$1i1qU5*~H~!%L700R;k~QSgCer)OYz7+@L(7cgwP@v&lv+UuBHgosT71_6^s zg=P(sj?VchH{pnb~IoH=&FmpOA%2Eg1@`Q zOI3_&?WLsC|5~#h7zTq;m?wtMi6JrJJ<%`_40o{xN<|c&dcc>$Vj32|0el7cQ{Z0! zod3&vK$WP;7=%rVPJ;iu@FysX(mn>E?hWPU2R!;u0L#sL!R+i`oh(Al===VJq3;Q^{HNW6A9K7FZ%fF*0=1gU*0~X9}yv$eBH2 zTW$`?ZFrCz41Z84IH|Ry;$_Q`Zv)?N;f;o{32hsiPlozFL?Z?^r=tkiRPC_iBQ#=) z=~uAxJJ`VttZt`~jk~CQNV@1J&~pL!Uf`ydJUU_i1Hcnz&ff^aTOb%ba)k(-vhbO} zKeKRCOSumvfj@^sYrx;8L>!$fcu(~T2yqa2QNis5@Dwe!zV)!vzw8ulip^Ml9ZlUl zGUN^JM;>vy`KKOs+=O~X5|Ot5n1ho$MY2Yic?QifeG+v@|7)(~su*t0h?RRsq zBQJ>IFbcy39)})r^m?Uk(u8a1^+pnLOsi39CUCcNFTd|2&X&Ym7e%89k$+Lmmg+5h zKkx$3^sTSIev zgGC<-dXUyO+>FDw0_QbPQ~XZg`HOz{N9l@9wkf3ZvYGvMExFae;|52D9|0aVxVA=L z@B#OLn<3*#!dt4d7s2KqdekXyk!sVz|Kp=HtKCp6-Wcv*K+D#k;4z3D0^SDvCBT0R z+y!pR;5hI!^cegj!kc#i(D1Dvq0DYyJ?2IeJZ@bp%Hs)Oom#8(Cn$|QI@yf{D%jcn{ zKjEwOOHJIU!B&7#^F65Xt}XCp7u2k?)bN8v?2Xw;FhQMznslL>t10um zHc4A*1`^I{%tnM!GhOr3U1FC4ZTZ*oZ?)en%){NQF|;k;|SM1gR)(CeCQv3+zAEbP+CLlaUL@on&oOFs{zd^DSa5y-gF_S zTS>wSZ9EAsbx*TO3^~|5c`Kw{kW zd^PWCE%b^>;UBBQ7NMA~TlU_ALnde!c+|qJIK1yPA4i(*fN2Uf4VGDG^4{>JxDz-&xGwOn8&|3%)3|;D-fInCZIFz{$%9{Lf`@haQz_&QaR0qjnaHnnv}i47gl5bhl?(9}vuEbBq8t z&(|1yB=C#@aQy!n1~c2goZ+us#_&wSn~Qh>;k*YC!(!Znja3@IQeBB{T~jBI5uVT! z!AYq&T;-4bCr%MCDD;&*toqZH-K<k)^c9b7jN6M$^Lu4LtaZt(bP!8#*fX6xuo|l8lX)!;y&S^;t!cuj^U@02P zfo`3y6@B@2^4TOQ20VNajCA|JZ5iR(%cJ6@xf^*L{K*C}oJ7zYfv4^$x>+kQ23{fp z=RUJV!;QD8M`0j}%gq=#SGPJCnsQwikxG72C^eaIb}I?D_F%;|hj30;ig%u=nUY#W zxT`B_l{pKf{MAoWqIc*+<y9}sp5@>{PulQd z+|3IflY<&mSJo*MxRw7#s=(9m3(*+{&r0BGF92K+XS9@a9c#Yyi=1{EZ>Pe?3bncZ zru7ux2z`jmATEw0xl6cAzx)(<+`>J-a)T49tnMaaQL6*i6V4rQmo-Z627hv!gka2k z3_NFbf7iNc-9mu~|4oc@C3W6$k@e)D?jgLGrh>S2XS0xSPIU$syh&ri;Bg7nlY`n0 z+yj49N!zD!Rd;8%;>73R$yr@fTqd?sxcfG7^)NWT@-}!Fcs;%nH~1*v4Y(Wg#07Pm z##Pasbm~r?bLp3d z=}cdOyTz#S>0aQe+JGkT%N@9_Gm;Zu;4{v2sc>Y3L7jo5vZ|wN58=<>Riruj9emvS z`J0}hTa6kbvm-Hxxtgfd6A*K)5qvZ7`lVt%2mZsrV@KdP@Dsq}DZw#Dsg6C_Emikv zNpWdEeNK~-Wot~`*^*FXE~xqSqJhP@JxDlv-tbrXY*P3lOC>&qRP>EPYTxS-ua)>U zgmYROuxp+aG4&?+^OltWjpfXre629vD>Sa^W(SI$Dn8dE-AZ$oSW9Q@br=(Q20{H^ z%h7zu-@8#W&(=2pk0B0EQ|ZMgG|{vD9B>zSFW_->cNq8x!EaICuLUvbx*Gz~abh?| z{lKjTcpjaC{!_n4tC{f zytI=FTGR9_R>T2MsfEfY|Fno&S^`iH5zej!aSeHzO1}>Sf8H1V2Jq*NM(3~QtSi4? zk^3|C=R#h7-=emj_!l~3D>*9x&k@c;@_cJZ=D~;utqIn0;V^3?J?>($WR@#mK!57I zxbGs0&V4n1M->l>|MiYgzT)rvp%W^dBy^RTD&(A~RkJnOd63A(x?a|UG5@x3p#Rl& z&6xNTGm^jFv0LtR-E{!VXbgfZqv%jrA9`}oX}dJwT*`)?_us;QxUvAgUR_!}|p{z4;Bob+ure%cIVp0>3>S z{8;14pSRg5p12Q8DQo6E0;S<|;Q)6jhQ(JD_;SYeF#M`HEB!5< zu}c{gJHJ*-H^2BfTBwEeA<{P@mQr2e6Owo*;l;EamcPD*NVSnetw_Hhhu8EWag>i#*@Jq7_6YpDzb9wVGno8KW0nVa-tf64QduD(NsoU=N7HQ^jb7^_Z`7N2mQAY#@8 zu@(H@8)Db+|3;fhya%!ykTBz@+s_GS@7!799ly+sTOS&>8IM~9F_v(y=OjwZ)7-S! z1Hb!I(Zlcc;`TY=6vi#|aw{RY8m4#94HpBk+XYYIsw*d)%f!Q18cK$7HGy!c;aPk} zu0`>Xtcmnq;CW=gSbD%nCiWYSHR6Pk42=+9GS*}qCY&v0Fa~+>eH`$ZRkAY)=j$(r z(g3@v5dKceJu1N(!{1~A4`zu3R7Ne8KLnn^$}}ko)HcGof9kPNG8)eTkM1rEamGyu zWv?hm!?$^alb)}rycPR;2t?~ez=gmLf9G}@Q(O5tSmdBO8 zop5$GW^E6h)M8y*juNr7wfb&PxL#~baSeynwft|JxPuzrM^uWVXg`u&^5?y zCiCljuYhFAvbowwu7_l6k^Da496}1Sp2_!N!ylAsjh~z0_CEMCSWhH1O51NY>k=7o z$r#;+2s#whCcj~rTLt6n@|2Zyrd_Z$6m|hm1CMDrrLG(;{Pk9~cOqPCEFa2eTttf` zeNjlR1n#~rKoV&^)+ovUBdv@^p$XQydypp@y}yX0Nz)j{Ig0YT!I3*4Hc?)x3V4Ir zY=O<%M#;zNjN^}^wCSU6f1(qPB7aacW+3?yY?&Kr5+O?c?Mk%!LNQg+;4XI8-PzR$TP5cZ#a)%kvh@`0-2pB62g zGkM;GG1Dun{4%0-pqam70fUP*m?dzL)1ji47%Rk;it3@+@4oJwa5`!*Ub7rgS5&Xa z&i$41MNzqAK~;zta|~Syi>jHyr^xrt<6!*B){G!Tbv*%B*nZk$Y z(weIAb$dr$S9jmdz5UZiop2CQn&hnTOOH8a{VME0<}??Gnj61wS*<_qKb@|1nOj?C zW5=A)0Y7%!X|4M=J}|rSE$8(>#8x(U{J>$g%PN;Gm_4O>QC0QQ+KG!7EL%{yaKZi6 zb8cB!Id8mQaop)~b(8!{maB(nlgFLE1#T^ja?byy&UBd0W%$cn`VCp9M|RyGoC8I@ znvM99++{2N+bM5>f}+a61rt8tg#72Q6iFbTUPM=pE;m zSASVuqNAdEc%2&#Wq171850QFq5BkfdDeZ;xhqiCSj=2AD+*$ZSFp+KeeXMMTUb3i Ne|Ywv|Lojd^mjnEO#A=< delta 82669 zcmb@vdt6k-|3ALFfRTc`m?+*CT@5eQDq19JtU~VUqGXzAY9zgTlcAAeX`xFZy;;{a z@<^y@Vd*|}~;ZWa!hk%!mcL3Bs#w&cZQiw` z^-B(f|M*F_xl?j>T6&LN@Yyj^bRQJzgwpSbkdC>y{siYyoCk0g;fz4sb_jpONe4B{ zA2?sYc^}S~aXyao4V-~viArT-*L(1{F2Ew(k8?H7aS}(DjYNU2;`*8-JB#bP0#_i+ zm1J~jz_}P_CeBEq1D80I4>gS<|K!)Ij}U&VJg(wB_WfxbouMjgyXpIDI(PQ20DT56;0j*W=uZ za}&-daSp}V8)pdOy^oNN6+Ec_dqrGLMwpM&NIBX*5bQ~WR-D%brXU&T_X1aokb?V> zOD}}S1^1W;PaupJ*A%>sGZb-WN!&$TuNC+=>fFr$H*xO6*$Jl`=Q*5JI0MI0aj^^G zMx5bLO2;akZE*ff07plht#E#d^Ie>u;iTg-&av!DrFu?;Q3!t*#>J><_&WmUmkg;? zP7z)}_@21dii`=~#%U6~4q>IZrr;Tz-{AD)d>E&E>=fWD3T-$YbOFVw2=>x7j)^$8 zl4pKLT-E@e4P(q##-FG}s4hae_(^VOxZcit9k27Z*(s4hfJ5dy4BZ z64w&fuizXcasQUC)5P_3gfVEFOm#ck09=of;&jDzvA~arFb5%ZTHr_&7Y`tOCqRm@ zFT#c5Is{<>&bI^(1k@DcafWl_Q>mr|BnU_1Y%cIHg#W>5$Jrle2F~s{AHul|=d(EJ zn1}OKoKN9gM@<`R##P{06u3aB!TCDQEr|7CSO;89P)^+UfNdtO`rxFa4ClvmAsx$c z{UXjc1$PQzA?X$eW^@r(xvg?PF!egtSo5qaC{oI430ov zEufop>IyMxan!KCXrzujnYTu*Txrt3@iTgnM~>1rqD#-uV4swwN_ZcaNY)2@6#QM; zOPv|x7yQCfQ%<9n>TsnZwq?&$_(MwU3QGKfP+CTF6^<$%s9sZ|qFUP4qH?6VGK;H5 zWOnH96Gd@y9H`mu2+r$VoL4pF;T{C6h?BS;G%7nvM2*W<@NDx#IiJK^x3{XiF>l^L z25}+0>AZ`1!~p*?TAVT{pVC?RF7lyX8P)g7eSjpd5(U@@J#<`&wU%d_)`OSRCHSGJAGK=6?>wGMsMSY__W4;k(sV$_10t8IcX@In zkF7GmGcqG8SNdunzCXM~}1N$f8s6BIoPo-r-s-kxiYp+!aa ziVg}?>ABXNcRkHZox}rD!dptjPZRoo#8|+P3>>AyCpEg?vIUuv^75eeYA%d8=|iqH zP={{>pT8`KFEveS)Kb|P71Oa;*m*_Rrw0faY@H7RsIx z^`N;BhrjehcB7Vke-T=1#B#HoOI3dOG}fqOcQ^v-W^%j6(o*@V*~7LZ(RX?l36sAN zCk0u(Lo{mDC|(Meh_8h+pmx%IbZ@qVOMLc9I%f`LfCVt=zB|imMOG z^Q>}7+p^=$K+bOnO~OLoLpx9fy*_2r0}oF${=iEVsOR}GF3t~X&Y?(7nbl--b@`$q z6NEZfP5h%dq#Ax@5dz3XgRgKBpK)kI=&GJR85o-(Onw98L-RymD$k` zx6TNxPciK2m?w%~Zcuhbcd?BU&=u4Z^Wbc#t#!h>0*yQxc@wW%!qt_9^6;+c{ft+5 z;ar?neU9Fzp==y?uV;76INv9nXU0jcrb=ilRVtdbY_+XD@7EgP_%npG z303F$+h|R)Rnd+bLF=nx7Y>&SD+IH*z9wdstz z4@%R4b>3z~rLyISE~mDoqfF1M=Mi4VvY#@`s8KZSp48-js|;z^<@uF} z&HPx|z18d}v}EiPT155r!g_VWdIMEi4M#+JGT1#RR_G#rmhx)zmbS4N3nX>@hv#Ax zW&REWmE|rLf90@dIu43bd9U#J0e{pbR_GDlHHpQ>){UNz|207?l^WiOZB#}v-Ia?3 z@nYTVeN$P|zRUBkiH;}_>WDt5Je5x;mNJ3bW}+PA|B{0IP?cyMy)d&t{4`~Gla{tn zF;0>;@LJ^v^Y~q?408Gl9^RFX_X11B8jb_n-bY)Lx6_MNdJ-3__K4-2ZYI~0CPq{Q zmKNywW-47DE-(uJ_<-_i)0U0@EyjlH2c?((;fX4`S;JAE8=N!QjW@^_zb)#T6l4i| zg?Z{<;d2AmhAjcH!0~>h9ML~)GX^E|>4hm2^D^f1Ky?t5!A{y0k79{hcIqhX!255~a@2rQ>Y97@0!3SwuS~ zMzZQ1WnkxrZ8d*zKG2{U=xZurlIR$tNcJ=eK?8l980fsYu}g}zK2<#96pvVp^mJsQ zG;ARfBr@B~S`R(^YJGZkaY|H4SG;aW zm!UDGmzOqbqkQ$mh|-?lHfo`CPnypzN{20L)M{N_l2%QyRq6cyHfp}kJg`xtb(74e z)JmhF>Fd@HZKhEwQ^zS?hQ)g)2G3b;|W&I%U!Drpl_}%@x=1ICUe%HoT?s%%RpGcG2b%71z zU!%Gr6o!i@Ai@!raT<(yBj1ql+CazR=UUn>%11@Zb7Jbr!<|H4Yn z7X|Uzf-e?4wLd#@gO1@T3KUlqhJ6nuFQzf|z9dwH8GKvdl;P!%uZT~iyx z>pv6YaKyb$nkKdk%7S>kxOLtb#OM4*dTdw|{jC7o{Qtmv|A9aB4}4V+pRtQqz(+jt z4{SlH0)m=NT7HVQ$SBp}```1BzDXTfI# z@uvl!6~x~bd`=LrE7P$2Rnvn6ZH2Yx2lHjT24z9~Z4rM}5Fh$6k6#|dM+ko7INQBO z)LaN$K?&LlzA}h62;LjS_YnM{AilTYs{%Z)ze<%P1ipZP+8>8m@HIhvn&4}L_;kUm z#@}1948ey7@wO}>&;%To7L*cw-P>%z2wCAxN-DBuEP4R|wu5 z#Fq;`Er?eMhm#(}M+iQH@o4|R@YpOAWd$X02|g!?uN3_BAl@tZ{2>02;EQCQ`oBsD zie&+KpWqh-@il@k3*u`9zbc4VeIiCd5FZYn+26(>L4-))3gVj!zA}h!D|l}ZZxH;U zAijs-tHyWc_8;hi-a_CDO3+X6H9>r`;A?|;tKd}=?i~dq1s`6|^ZZrggg{p>puvYX z76cy?#Agd$AH?ShJ}!vQ6TC5qpJNk(gdo8@!6yar3k7cu;ui@%Er?$#`1ByYOz^gh zAi)YD$O__D2|g!?UnBVGLHv5b=Lhkd1z*H@TmA5J3qf&Eg582&6vY25__84WnBZ3h z@uvk}eh-iSuMvWc_Xx1LD0o*8e_QaCL42rKig|7CDR_MlKThy*L420rjSV~> zf4M@C&>&#LFJJIULHsbZXdYO% z2k~tMUm3(31n&*vdkFr}y}V7;TL`M|6{z|N-WS9t3%(|ZHw(Tth_?z}6|^as#(3CY zc#vSENT3Vi(*++B#OI5}r#^^ZDEPP_ewE;j#FPC8hM!9a5`q$}5E*9#@qs&}tROyc zCzKP!2kwNXOFSEYfg1*uxWS;8d{`3GJv@te4IKK|qWSHQ%^f8Z1T zfzOxtcj}9eml7rZjKmj@uyNN&HI@a7{vknB#5n#u2N-4yx65- z(kzMZBE`><_y;9^y2N*D;BBgWNzlDPpemC1z7k(7@ktWDNaBY{e3`^QEAguskM^gg z&Xfd!rw7!>GG8t!dPR!AQQ~tY-X-xfB)(GO-{khsrCv!eTS{<9;tM3cO5zJ8-Y4;M zCB8=D#l{-%|5{0~TuPvNrok9jN_@D)ua|h8#BY%J7>WN*;`Psn^*?pNHc1dCCD~K6XJ&Tu5 zB_7W?`9b}!mjv`SY~YBK_znRQp;6-X5}zRP9VI?V;(1lL&z42ka{#m|!Xha^6yfw$fB*7WoSf$9+{LB7QIlK3Kt?=A7g65mJS7fJl1j2Gjt zOcFdMC0Hf#2@+o}@$|GUaBP(L{s9u9OX3Gee5FkiJRu3Z5^s|DLlQqw;;STnki`2W zK3U>x1aG6he^L_EN(lx_yehq66qqGGT;eSfuao#85+5V+L#6eHN-qhlQi3>%A1?7m ziBFUG1c`r2;*%u)>3SahZIL{!j>JDB@#zvjTH-S#evHItN&Hxe&mo@dkGf!- zB$zHGm>}`_5}zUQMH2tK#1~8a3lhIb;%)zu1Z9%oMTuV}@&A_ia*5B9_>B_(lEk|t zeiG;T_^XrzFG~r$5pXlj|uQp|2#>c4+y}CH$Ip&PU2@t@r@GC+mZ_tBtBn?pCs{fz}KfS zOM?GM3DP9KNaE8a-Y)SO67P`sEQz1jz}r+gk|3}uCHZuTf43olDqrG@CB8`F-;?-a ziC-Y`ix`jkQ{O9+piD}zP~ulfe2K)DOZ@v1zfs~#CEg|Ri|X4S5h^9YVkv=F;+IJL zA&Flq@l_JPOyYeK|DoVn{xy=|BPl_x#Ft6DYHY(O_(bBvC4Pm(>m>fa5+5^GtpBMC zK9vM|DZwgHr^@QsvUy2O7g@%a+}y~Gzu{0|aeEb%Uhw=I$cTO~o6#JeSamBf1_ zzFgwBOZ-NO-y!iX&hzzur6j1768tFfUWwl=@rNXSkHl9=yjS9V62HH`{Sl!?5*&~c z)Jpu%60dr;VH6ye_;86oBJnzjKNjGr|6?S!jyHjm*I(oHH2LZk0l&JxSZh;ghL6J`E6i24Ca^=QF&Ca7)5D41YzKHa)5{7+y`d72!07KPKFoa1z5y2-BuVwUOZk zgxdn!s`ZSROGG=0pksJ8;r4`848KXZ1L2xG)B;lp>j_seJc)2DVK2iI33nvyVt6d! zPK3)D9zobZ^KW$-BZd$`4!ydV;emv^5YA_~FX0CX=P=xha2(+bhPx5&N;r+-j)c1r zPGYz<;qHWu3^ya(gY3Ur&xpoEJVX(63~LBKOjyNm-6`Nl2-nF@G&S!WN;l~K)F#Hu^+7zkI zV0bm*#|ft~{4wFagp(LvLO7AIk>Le|ZT*PQGh!|g{R!(Bo=tcFVHLw~5`Kbk%`IjF zgp&wYF+7Q|iLjU9iG&9db}>Ab@F2qF437Y|WhE0)#)u&lA%$=;!vhIFNjRV3zJv!8 z&SAJ0VKdc8MiNoRh+Pz66yaiq zw-SDaa6ZGE2#+S5!|+#x(+OuVyqfSB!f6bDOn5BeB!-s|ewMJ2;RQA##u1@s#9YGT z3F{c1O?U!f6~k{5evWX>O=bgxGYD5PJc;o0guM(;Bs`I@i{Y_^GYOY7Y#Twu3q+JL zVhG`X5iVwUAmJAY=QG@w@V^P?Fx-nU%|_K340j{^65%w4I})BmIEmrbgkL6XBy8*4 zjEHO^^ejSS!jlQ>7}gM;LRiIc-3j1V2-nmy8z7uRxQgL(gkL4>W%wlFsf1k&A0hl2 z;c{S`zw>?~a)~Ho5q1%tM!1;at%P4EoX_wk!fz1HVfZV;(+OuVyqfR~!f6bDO!!U0 zNenL`Oq(Iq##-d>@4SGBJR?EvXSVMR|VHLx5cmqG{UBWfj zsREY?7Za{x_#EN)2zwbmNq7Na7sE#gD}-(3jMz`aLL$l--bJ{Ca52MM3BOM`pW#h} zO9|&N{1xFvgfkdkO?WZkG=@JWyo7KP!%GN%0Box^GGYM{ODTe$;kksD5!Nw0oA8H( zRSdsL_#?tK*O(0uE+bsU@Fc>^340lyNcdyIE{4Yv{)El{<%}3X#0rW~#_$lrD+w1f zJdp5z3FkB1m++^Aa~SSLcopFchPx5|jBpym9SN@{oWyWz!k@GG-^hq&M696*dWIVl zUQ1ZVu!is#gjK)@*+Hvjvm>nkVS~zC=*rdjy=>8395_bh_fP-RfK6`u8Pmw8r%f~$ z^iM}zRc?gY+5fqNhSng29?#>`7{D3eCSVuGPbClB@&5E{^o8_8eEzm zKQ#5s_OQRs8&v9NP2+ZMdr{kg1`aYP=wF7Us!_-(w>6@FaEKSFDKh%c0fXWyiq$+_ zyY5uc6KeHWN~gKGn#ill;<=`FlPG^jo*~Cv=+6!FeRqY5v)Mr7$u~^7G54vEs|O9r zP)CQ5{Rk&J^dVm&EOzL-%sYr8+b>!isi@!YW=ERd|6Z+zDm%wOU4c@(gs8tbu0~G9 zZ(6MOIe_X` zIbYT^j<`sLjlAGBC?krWXz{rhk)079BvqNU$-B(zp-H|El~0R1b(siJ{B^Ul-ywf9 zZJhUtRu_3zr%`Di*`@u@VH6TX*b$vlQ~Y+TQx`~Sv>U(C3)I}K9%%G^r_6n?&Zhqp z(gIJ6CGoFm)n@xrLkyKH&_Z94)DfxdVYxZl&BcY;etZ+v$*eXd_@e$o1pD#t8Q~ad zi1GO;J1{7O%AC27)!%nckRe1OT9)q+7^`E25~8GOo;t6LQaUs@o~O(<8{()0thd^q zSBjL_CvQ=+phzU}O*u!g`Wbwqs4E?F7>lNGM4$eNYDg_|SsRsX2)~RrIE0@k|8N`G z`up@@Xx&aH{2~k%mp3)x8&)=_6`P!d>QhTVu?1ahq%9OB09YbdB zG$`E`Ew(*hNtJ&!+-$EfJ9K9IQ{fm5ab%>)RIC1DAg^v)?f27-7qsvE!R#RH1CK#j z@xg;3uOi&@!Hl@yP?2HwY;}05qldwoS!K1KG})7vseHM}F+UF_hjOpmux4GmGx-^srJ!GiGny()d#SkLl<0Cqf;OJZQaF>_N$wJ@*CM)vH?xQ3qRrq zgK}_ryn45C|HrW*b>AD5hd%D4opXY!mu%3(rYgZb_i@Xn4<92g!hrlg0X662-)uYX zs?np7n-TbnYsfJay0mkChK*VsVc&0|W;u?QL8pGkV=h2ov7-thQhfS`58}_4;}`Py z6Ipz!j+}sg&Wi-&ZE}hic#57p1?nfKAlHKwJ>&#W@B}KJAm-i#G&P$Y58c8qNKpr? z{SVZkHBYej4k~R=LDMvA$RgK*mfEw~puD=Gg{{BH#@XjQ;`-X7;jDJQ@5$fMfM^hG zx{QC;^qgs2_37iRho{PWqX6ft+@Cb z5)a|xQe*^iI*D6w@uE>A@6P2faQWjTkLL0jB#wZXe1zHY1oLy%=MVY)+g>cL$47(k zkL!-Cyk8x0i5X$8AJ0spc{4{}%BZRD`Ja`P)!iFq-#{roQeI!(Nz>$K<%87|)SoMLs~^$4bwKIcUrwSz zl+VySvS;0=dJF%-|v>QeVM-y8O#(zih_5Z1i3E z(eL-&-AU#VYViFF5t)1MfoME}W}k9({kxi{b}EG%rmBBaYBwY`T5%pV_*EJ3b$3l- zrSkIE4{DlKDht2vtUkYP)7NF8njzbiao?rJ-r9O^!=4YqN0nXQJ)-`6UEOy*)gks3 z2Ib)`zjRIb2F|%j1+sVc>49rlf8j?69Q`MJ0?7%IIKs|-0&nsCv=EEkqpbeEosDL7 zV<-)ru)&CCwr__=x7fFuGmm2J5VMiZhmKkLREIe()iE~VMeP)qc_XZXZOoeO3k~U% z+P>M4hR|xCXD}mt(W;$N8Sk=UVoqY>rG^BeleIZsRkAiNEdGodkzX*h3%A(aS4`SJ z_PwaRxYMMKbG@h?x9dfdc0#2urb2n;hxWD@L}hcK)$xKM+-LFl{ib}sT3a{|(3c(Sa@^wpks0>0;!L zaI2#!yQ;udOS-ad4WGJkkaMI;?HuS&JoNgL{vT0{11QE;n*XWUsLo2htA!17yY-h^ ztK&YF(2J`kbd|iXmQ`ybZE0Bw?y6^vvmz-qsnsD`8Y--)0c(B7Ql{48h(I<`_gfs} z_Sae*z4v2D5psE;c1*G=#$4bsLe|LQ=(o@8-)c$x1t>(Dw_9me(Iq4oKeJA(=$*1q z*u+vp9OY%-X-(Xtb>5+tv9D)cnX3K%g=YSgxw)QCu=13Bodr*+esBD?4i zXj)(K@FTlxFAvh{B0^A|G1k4c2-3&xA#TjLknmv+1G**s@MZ0oBUInEyOpS|?S|~3 z$j$c-bG&d6tDf|eBr@zC=IHn9Fh_4M>B|k#Q;;1JM^SU35J5K7Ytye$?B09L{t9#A z9%cI0r$emvbISj1ZP9TRs`4$jXI?*lF}>$%e%qbJ`@*F(+V+s`mf8MOs{N-= zjgV7SG(uJw^iXNBAH%q-pi#Alk0G=DfYrXn5W|EQ%u3C+R<@GO@XuXCkC_X$X`&K@bPTiXlzte2@S?w3B_ET8BGMy>$`>hVHA*JxTcK%MpOwGJb_C>Wo|F%_Q z4PCX2R-U@B3>0UGqu(*J!;%0TnFfpw9~XKIf1{G463jM7s5v zb9<|kif#;=8}WFO=4MjJWQwRN81u!B%4`>~!`(-27tYFq&duA>e2NziqZ}W$kS8 z_5R{e!kS8vLDD# zP-*APK-Ym;3??3~C9-&_LD}JHKgqw5x!o-AV4tTTGbzICm=R;~yRC^;dHZOJ)A@eo z@sjEzME2pT3SEe-pViA^|9OZb;;gxFXKs75eXsAoJb{)g4o7ywx3`R*#o4p?P$SPL z^4JFk<*n^)C#7=9*CcU7epw%3M%=vx>CEHJua6K}fJ@Yg6~y?z1-beStKoRA_!(rU z-qEtdZqD{$?DPbFWAy3wEUF%H9^27*qM5~c=+Y7t$(|BJcMf5v2~Z0ax@Kho{|p>| z)SNk8XRbIK;)`ZUqQ7TJela_S#9;cZj(3?eXX%i~td3aHEoZ`CNvOQKv*jooVZVoI zP4wlhG*?t%0Ppa9^EH*`p$ry1MbEN-nt+u3AYdu`IV?%^LrZXLRoS|UjwPj>^em^p z7XN)tU3gA=No-GvV>xXlAm#KGu$0r44OF=3rHc*9cgV@cl2T4_ET{2=|E5kQEZIW? zSo9Pl%c&;;DW_OqDW{1%rzXhBy0dl2UyBUNOFKIZ*u5wy2WP(|bU~;ugvVb)a{Isp zv)x-TJwi2WeEiP%{qZiW2HU*{729_{ZAnBFf7qSX27Js{jpg&WZETK>ckwW=OHy6% zEnirQ;ZDb8p5wA=Q%?GtqR((6XD-;K5zM$v z0YPDB=71sIkJ_*I&I@$=esJ z8N62cXx}&5`P6IJjPR1W*mcT>zxHbU1`38Dwr9RUS-!ue?eu;%?ZmwC9PPp!1^9tB zu&-#N78>YDvF{j~8zPQCV+{jZ8@JWh3(6Tm-=;wnF7dTtv~He()8i_yXvgk6KM7gx zn@>AW{`l*D9hM(+4B<%Eg{GM?lr7u%*o(GcGWy96L_Csjx-$MiEA<5BjRS8Pn?VcB z#LhnE18CGMYfwc7|3&aLL9CLdqsB0g)O~8D_;^Y_7W zKXL{%R#UWmrg4Y;FJG500|}}~@NX`_yyBa;HXvzCs#2MR%5;Dt4gj>;4cM58{@klp z9q*y+_*p+{B`$r_R|OJ&eF*U8=YBu--N;x}xmsGNR#8`J$RrwbBl$0-fIyEp+@Y?g zdN2c;yH@FUFjjwGJu#ljMH^11p#fV8J1NgY%FKhWJ>;g!z!nmq$M+3D^)Sqsi1Wa1 z#Ch^lE@hT7<1?k*FL{H9vYgkmoQF8_?#5B}k-pwRbU3Hm1<~Z05xdbUrRtaHn2V$z zYj3Ir>C$uQAtmzH4C5P2xR4diT)00MyEDJ~CW6F5f8OZZDC)njNyYOi%YGf$<=}sD zD^`PgFniHKFEb&srzJn3M$u>rpH~SxG(pqzQ{`WWa%}~u=$o3{Xc#XB?HmJbkZMPU zWrIE+id8x8LLP~J2k7;;a21wESFggkTqzghw&f~^V+`&BVLW+A_X>ml~b^t2N_x4O9u@SDb*l))pRd0jOuvQsybgQ`PYJEZc0^zcSRub3aw~S@aG&Uy8RuI< zxaug0-h$UYe=$_4`DbbhKS(qRjSf_r#-&X&akmWc0OO zp{zMND(L~r4+gBKvTpyl0hk@5!iz#5@HIoT`L~(;JFp9$czEij^8+l2{@k`^I~MoH zeODFJvDO;<$IAF)y*m|94$u`xx*ip}2tb#Pbj@C&tUuPHQ*RNE`Qx+8c_IRc7en#l zu`B#rkB(=AiUiWPNW>$6czTL=T6yZXHa){JP&eu@)|t1@@Biam1NAzbzBa`ROVI5^ zkG@7G#D@2nRm!s8Vjg{h2xdGhD7&saQj!S--p>-tFf6sgH4%K6{gXl8w^ zg#Z4q`mXZW@9oAEm-+q4_FCNfn30t2Y>nTBTx)lb8KMxrS*S+*(Sn)yNP1hY+L5Bu ze&0K!V0*0Z5~}6jmN;HD^)wP~t50P4SXujf8{6i4#PjRL&HqNHKA`hI_ayqZKGAX3 zgjA%1==JSk9qoaJ@`Zw^K7vklMjF%f<#CJYN|3A+akS?)w#ub^Zv(&oHLD z&zIXVe(1F@jI{y}8tV0r0?~!04a<;XLqIv}&;=;X`H}#|Y{5p<>;MJJ8LskWgR9Oa zr^>Pymii2_&k*bE{{=5ZcPhj+m;nmmccT!OKWs?Es?&3s5?vMhWPA8H-Vaz4~mDmeh7V<59--%JZoyW1FSDJ$V8=vKO`EJL=q z!8GHpaur=Bp2>}(;r7W#igF^(xEQsc=E)j!dWLuE+q(=?(=m;wp6$mQ3~7>T%JYi0Nkfj^F8zdqSK&T zZ-}AR`3YtLv%ik+B9-?svg=36)nnylR&Yn)z z+`m+L=uFps-!I{FN zmknfv+HCMq1^WmokgdX{NTq`J_mU<7)DSFaY4QypEBS!p(`l zO}iNAQ)H(k{?S4+s8o6Sj~<#0OO&_&=&WrjvSViX_7Y{oAKle%<-{MuA6`$D2o&*+ z_bG2)M4I!%d?iB63Z4GGGU03+&3B8Hx6Zb0kt#&2)WwUD@@;pVMtN6>ZpUghvjnX` zugL{kfn`2=v2y%utopQ~_6<;blq6rwW1E@rqQ+0N3p_)m#)iRFQsaCN&A!D{Ff^u5 z6^Q%bhswLYHp!cq;5aP7XG26-;;u_q=5Pr?X-vrYQO7SspM2g#~eq0 zR3~Bu6sRz>>&2zYvg%gq50!7KJ8LRSl;5j6Xj(2&&@}36ivC<{&FY0pzjNKxdz62l z>k(pl!=Su>u7~a90;nmd^=s#tu~@QZzF?r66W<35MI8sl`QEL+Hu?&1joF`~OlOzd z>2fl=^w8x*c8N_gHvam2`JGy28kuUZO|@T&cbP_+9ZzFvF%;7`b!vfIF`YLxX~wMK zX`_8msB-DNA>tG|lpaQB*4h(^(5x#_PMzx3zl<<} z;1Mv-Awth3O7w-NY`-g1t{c8zSa{L5hlOW-+Yzz~`!?eWI}inKBRwISU>c7bU3hrs zw`@uGI{Il7z1af~Prk_4|2-sh_J4+pN0XRd1cEq>#O=Ab0~aSGlAK-~g8YwfAomxu z&s-{wqkkrn_=ZvII--A=N?Q6PkrqeTdkjU6 zC1l5lU#(xK*nrlX=s^r1z9Hmt!>gdCK<7z3wx7iRK(VSMgtN-Rwuly$@u*b12=fjC;_@ z8oRf_+#M^_BdO{s;TC6JAC+$!czRTHj*a;n$`@DK^gId`uy2cFRyejS4VI8Qc+_mM z&kFbTWLo+etle3nI*WQjxNnU^sk>rT|D_DR+P>8jNNdKkWwZSpJhi3!&$xHS#-*=R znRT^&v+3_*csd4HLN0KxnlM-S;%bM8;cHR;*+tZ~DrLrMebjk7uSDT>x~;lVeN<_D zt)nJo2OG zSCVh^(=04fX5V;5)2~Q5a-*MmzS6Sx@r;pFLo98riC3nbqghMu+lPG4Z|1&`a0PJn z02mqfp2&?{r(?dzup_%5_vFYm=zI!F@0;=tgyi-QD&@6pZRSAS<4#IcA2*HSb`Qi| z0Acl1=B4uty6Rhj8HnmfIcfr07T>F-F=;tdC~~xtmLAYzwUd_Tp+y;bbC&vLW!KHe zG!ML^gw$Cvf2Gxp(Y#!!d|o$9llYEutFD7)SAo*@*2A`4%plN^&$A248}-ur_M@1- zwFM%V=7Cs=>|z|=ASgVV+mGsI^!c2~=h|XSjj&LweM=dxto99y zSg^7PK?=;)QcR+}oS+b26sYI}+Ah7xrrRufaJym#k;(1bha zkREs1sFM`)o%T)4fM$D**NJC$$$d5U5*I~rPJg1pA69X(wX=6)`-XC5nZLCr z|8{AI=kR|Jc>OJHHE0aVB4HM)8-rQK?A$^%GCMa=seDHCxVz%DZmFTkjzWrTDsY8* zZs`iWlqCBLadHL`muxAo?x*LFx&f@ z?T?%711$E0f-PhcYVG{lQ7A3CQ2VJGPvG{#$LdlYDKQquG(A~$y@-cX9)Z^_(cOSZ41_kq#T(QhNU)$Syv$@VSOXvy}HS_H5(dPf*zBnAaRPRmH8 z)v=O_=RP?j){x--ct&gwmFMu$U7meMw|Xw!t?^inZgra;pIqrS-JG?9{_OP3n9|Rf znt2f$JE4}$ot6r}-e#$|sdq1bbJpey%pl*KweTmpaL1ZjEWN|8y0%!sxq&lc{hqgv zp0heisJ%Sa+TdWo2qH?2&He^jWYVlb7}I<)XY1UiaUCD zEE0(Vwra7Sm7c|sexcP>0TbyVjhonDl+DTnud^}uZ_~|@Z6m2qDEN5OM$EP ztc>pEPqo<0%Hy$!f}#1S-Q0sxVk*^Cw1{3*l!aV#Y}Y`mk=j69=$pgk%V)Hh#|wR` zGb@7)h3NQqSz8^;$Q1vk1X9+m%>`Ta=EOp>4(;sgh@b3W=7e#<%*esb$id7AT|@Rm z!(s%6#eTEBu+m0`l(zRG6GKVeJJe!UZ?3+Hv5w*BtqFOP#EGCO>E+#C;G4RdfS9OjsJjqq@L z#Gkkp*6~(UsL%cOTTu}ghucHX53@&953`4!v-myM#I;m+?QGgO`rDqcI#}UFPry;w z!r>X<@Dd{NWJvRqo|4wOG%JnVNaXQc=s~YZ`2s$IMKP*`fB#j0(N%woRN7YYFxAgl5yt zS@;u6s=X{JnKiF6%z8afibz!f)sPF-u~98HbNRN&n7wMxoha*NJ(c;Xa8LEoDpoYl z;mLmnG-E(4<(h5FpARHO`F#P^h#*E_ECwD|%|;wbZM~f2Mya2QFc(zlJYAul3gtGn zYS5383H`CL0riMfuO4!t-nRVy7W26NQ(^EpDEi3D8Q6u@nG8?>?s$7KK2Y~Mmk z@mj5&O|D{0SnSvTZZ3Xkdi{{Y3`m|E1BSW8ikUWDXsBnVj?GaBSJHIou9&Hd=;wZW zrmi_n*&Arec2^AQHZv5hQ&^1j2%R~wafGA1=#rIQA`zBT_~V4V7~#}Ja6>d!F<+1$ zahpgZ>1`W9e+|vurqaKfs_(OfU`GhL#+T#I$~00uGs;OnnMAm(hH2p55UF~!MWo5a zsfE|489jHfpg@T|Cq^IjTxx#6a~tKOWC(L8$5WGIPN{)b8o{1VL$R_zqZMe?;Mq6& z0yJ##s&AK0)2e$kqa=vDfvI4sOV?@D52sManVlu&q#0|Tq+IO$!(C#)((Lc1O;tWM z!#OffW?xBe4ll-`W0>o5tksov(W&(s*X>fXPTgOh_&V7{YN1~{zY*K9WF;EvOg@8K z<{_R*m>aB?z$ey(GUbrPWE@o|f^(Oq5Y+c;)0_(k;*PyK8Y#1%b zpqEUU1x_+In_E3oH}dhSzcaMtmT+1#b=JqL$Rx%Ms0vt|=b;Jhln>OZ2Fs&`CHmf3 z!^36e6vF7zc({yyS;mcT16enEnXC@BZc9T*xr`jZGJD|(mRWWJyhi_VyP&hg%XO|S z=Hc2LuG2|BXX8y!==ytmOS_6(VQ*{(skb*BSh6>CrQaWgiS!79u}=`ZyWh4tQb>%ZEE3ZZpfIumQh56gPm7b#QMFM;|)||G6f^y zJl#qig|~HU(|EXnHNi~XGTyH`r;+uAZY_;;w4@G&W$Sl;H|k^7uy>#ZJ%#rnJvT92 zNYRPcuBbgXU#n5$W@}k_X><#9$6j6tsI7-?p?=jt!V6h4W@QaBaaW8_Kjk)!Prv9X z8FSWia16%JOD)uUZ8@Y6R#Hsi%qUR<(Fb`^q6Sf-22r90QKAM>tmLe(kls?TBgvY$ zh1ym-uZT6gPBgrZHGKWTfh`q0chW05BQ3M9)XdXxVfoY!^a>P~;c#Q_zG(>04?;c@RgV5+NO^Z>tNoC{ro=E9DQW!F%k%Zd^ zNqt&_kcQj6LUZY?7h>UOXCH}>Q0xa=X7Dt5;nuId9M2Pt5ML! zC*A7#umrrT^v##8)VI_jBdmsR*0xcP3DaDEvNSwaJulYxEVe73eE}tM_Bn=YY`-2H zh``bRizgxJ`LEIqvFdnD^CwHs#j3M3t5Zrxc2tklj7%wA-BJCr=4^6l%TDTn+4SF_ zVV20Lqj`ZgIr4FZ>0vqxR%Rh+*yJcC_|Gzb)_E~)a@b*bwrXbXByXK4(M{G6VcoT z%pUp0yWlF?x#yyLZtEiZQOI{os<2z~O7wR`VAF%4WN38`76 zh~NGD$_2Oz>-_;+=8)*Z&ZCsRE|8?yYbj>!?b2CY)P~2Y_pwJ)$$Gz{x}Duw1_qln zkE6E-JFSV>q*+CAoHII9_(Ll$wqctzi|R^`c0vDU)Ro?OUER{Ar#^5vDZr%Q*g!Uu zjO_((K}AqN#k`tHp{}qBCBs$?(aE@ZL?lmbRH2Ld81o;5m=C-_8>=vT;C>Q}z2p#! zrkfL1!?Z=s31cl@tD6^AW#iPmFc)=Gt!`nMmop2)$~m(%%*C0dVP%|I5mwHb6=B6- z%mu!*X=PM8p`Yy@xhtk(Tgg;V5spS=W%AU~e$r0ud@m+8OXfb$ZR{s$X-wODt8+mF+yJ+huEfrfxjxChN(tRJM z9%js#h>&cEv){WzoW1Ace%FpZ41Jsayn{ax|G^D_IdSXM>hn6!)zM5DjMd2tP{>A~ zqa==%LoAs#B+#gESG*b32nt~KQ5piF23%nli)vJ`g~l9~oKg7I58G%;)!`nJZ#3oU z!V2HP54{R*hHK~f@UtJ2t$ZyDRwkib@KrRoz=x5AV?jtUJ#Wrx-q<~GM(i%y1nJSs^M6Ng5Io~q zeDtixa(6fGEYG^1zd7ryyW;-WehX!HX}F)K+~~Hg+FcR*+L2J&0Ey^p&a~aYhDNv< z`zxuL6>Pf*x{i{<>z?IDFJd}B9O|(jJx;flF6?G4WcMbxSjvVv-4@>T+&sG1UC}jr zo9C&!2i&nUrmW|Cj&C3RmC$A2p6bc_>0*Cs<|(fF9No-YZJB$_6?gTP%v!nuM>~=Cb`P3~n;5sL-?*#p*u1FT zMpg};+wvJx=CBfY)Y+A8poKhd;_J(}lvf#r4K&vp-drfZXq=a_y}0$yw%vADL}l-B zR~WZ%E%M}V-&f?$oEfu=WxkIxZ=OKKzTw$9`Fd*R4Ym;mB}+Yc2{zn?4;JE3U29<1@sVk6{qpbnHDuLG_&Z?Ung z(RwsoviAmJAuHZ>RG@~|JC1#zqn^XrUJ=ievwc5nyoP33U*C4eQMx^nLeXbX$m`=d zk$uf-!z@9iFW3cJ9{?fHCtc@lp zY=AOH>##W~0?J7R6iLVvBgy;NTaa9}@Wz+{qvPMEb;nBPu^lB;9<(PpSnP*Jvi;DM zxq1CnDRYx)7xb(xx#&9E4AtC7!d@xuh2k}L+6%p&f;}uOY9@{2{o^?|j`!<82e~J+{6>SBKYw%9MH zwZjpVY$r<+$VH5d=WRD<9j7#BJyqMU*5`6+RLpiYHh<4Gln2VsvU0L2xGQ>Q9|`0@ z4`ooCr2*Xy6-RMmvrl_6qza%F_*~{;EkxgKp}Fsd=lSfPg+gkn%>iY?7n@Akhux;Q z?Wf?~qGPVo#zp9CPhM2+Y&}LRwoWUwv!BCm+23uQQr}rZZt$(sO>-lRZ0q!9vI9*$ z3i&77o6m08J1v6&#qY$%1=({vZ4*T9n~BA}Z}fiWNNl9;*UtCh@i7JxyALUF;WmU_ zIyMv|i8-?n%i{L^)opCFoTV{>VzTis1}2t1q#LPdbm7@EfW3rlRE%S6vIf@2|6f+b zHY|yu8Y=$(UlC(aQL74Xg7Lv7Ky{w3xfc5?thOE-Hd<)y>8{Arh2H0WJ5Lw!7+p7S z2_Le-OAXUwLc581wBa_8wtBIA&Y@w3HESBiFT#=>3bEahMjN4dQOjskm2fe|+eN#+ z8^{ULrVIk{RE9EX`_Z>?4L%YDa@hl5vCzCHwMWD>Rz(N4LV59PeXg~ zl{zi(ZCGz6;I?48j?E0Hbdnx_Ch6VVX2e$N+}oattIvBcsD%C?tc z4;kHqOtF2fx^u+i4ispRGcVqc_W^wEpT=;!XWN=?g0OA9lD4g>wc(amQho@Rc@g3U=~L>? z@l@|&&d`t1#%ozyRq_(Ud zYMvO=aDr&9$U|@7wvJD)FnMxg4(`DC?22T|=9VE6MI z4w3d+-O|V!mR;1+NHz`HYIQ3jh4)$!DemXjM6wZD8?`2~oaV|}-TFuuPqsc%+|O^0 zEa%MT$YR7X7gY26`DH>sn^tIJ5I5o6gR$oY!jHI3dAb)gRi4{6-MDZ(LJR$G~`U)XJXAXsVQEaj0 zuIM%a@3RZXn=LC5=s~Y)X?%=7nr18|wVuXeJrw#C&(;Y=@c7sno z>LmQtnAmghS5L+6Cx7Kk!2_8I9 zcJ$zZLUI!n-o}FmjyF*4Y4GScW;ZmDuT5j|Y&zbY@JHojwRt)xv!Fa~V;DB%USXSj zQrW`1&u!gGG9!BMP>Tl-r0B#{dhjq+Ja_=(HW|iXxzu*dP3%3c)REmpz-b|*hd)j) zqR>N$enYmDgX2#nNYO$(l|Vt<731is#5g>aScs<*KabgBBo%+A|Lis;ryuid9z04irordGb8p0TN`aPvPCamPrX~)mMxmA=U%38e|7tdWEQ%x0{qmXdVT35L+}W1 zIvy>i#9zRV3^!pEsm$NvkBY5I@SPC)t_*$3;s_F$9LWa!vU1AS6ukUQAF|kKpCf$} z#+juyIVY(9;Y%?42K)U3wOy0(YsLZ(|F(tY?K}DOrNXH{(a${hAqW1m;rpyufcpkJ zvtJJ}X}|QaUxd>4QU*HvWts8v1${)|wG{LfvZoI`Jn8Jc0Rip|?=snF$mrWDB*POS z_SvuBAU74S1A-G;eB7bw z^$O^;gT#CCDwXpo=)A0*H@#75)hKnR(vPg_Xw9K1rC(ds({1L=EBLgB>3fP;a0_oR zUUcEZdo@;P=%rL=kD64+D69^f)6dKBk+H_9_95XobvR>i>T$+t=YK;&{0IW?9F8-@ zq~a4}di-YHjOyFl8t4-};e+SqweZue|Dd^f?fuC`x9Hm)Nt)w>Y;&`l`twiOlJd{s z*8ga-_Ukj~jL0aw42CfJeuP6E+1FHfO~9IX9iREI9@Eh`Lq?v~4RMS+t;43lS#(e; zVn>a_o!9u7VUE1hI_NoTwa?Unh{HEvPV21pf9bXJ@XCYV!As^q$?$5qEgT}|F9vz5MjgD(ZuVl1JvW{t4euc4=2w^W=7 z9p-H7vWEDq&L=U_b`Epodhu}>mbsw3MFXZG(&rLQP~!24ZQp{dTX^c&{L zG1=Gz3c=hv5;Bb34Ag)=-o(G@w8D_X@EZ1YA^HMP7O>fFYi_`|BJW`C9!&Y+cTp6N zeji10=xx^+dI!&K8)br(ptnyAbBx=G40xjpGov3y!y@o4Ahh1NBh+f9xy_@7IeJv0 zvXy9vydT4dI`S&RhdY{A(zvA#rXNMysCe|#v~+}XLm%$zoff8zJAXbRziwFW^?t*0 zFKgq@QQP&tN`D`u7JT8F3PF1B%LLxQ#}?DC;_E}~qd~dXSwq`&6#11Hija4Ws4>}B zDdHVmM&(@zznr4g4cFtZO*c3W!I+5amy@+)LT}(Yy%LvE)*Inwm{H;-?d%s}wfN2r z6_?E7|Izj)a8XtN|M0zc7Epm16ciM6WYKXW+zJH@6ciMV(iD`8zABozkfB+bgIj@s zk)p@SjLM42g?c!=)V3bWBZDWS;lu+;b7g@9+QnKhNu#*Xzvt-p_XS z^`3h!y&Oit7X>*f5{Dz;)--<$#`sC7hIm22CnRyf9W*w-p<3g}zvSw~+yjYd9Po@P zt3~XsooHPo)1sa9oq$;0M#SgtPb^5m+YvvS@;`@C?xol+Cl;J{>vD;X`S@!no!pXT zuAYShCR6lh=)I-58{>h!36KP8Up;MAG z3MrF9%ISYfIn`lvR=MTx_P{yi;Tio+EytI;6HF=hBgiVZ63i)Q7ay0y+b2ctow5AV zFf@JrP1cZd{J{wRVFUlLz#Nau57|g;ONJ@sB;0PcTke}Cc}6!2zTNx<<5;Ydt6#}#FeF43?k2{;0_Io{vIHQx2;(~rNkk2mZf zO^v*i)rwxWv+Boiafr046PdSXHYE7#(duhmJ^!krW~k>sRXs4U0klHbitN3gkMGj6 zPGljqOtg4hjZ1Dp; z9H-Hx)$0;*P)A&d^?rY-I6jY}{tyRDHtr0Uf{}!l)U5S{R}pNrCycsEM`zXR3aKYN z1zLli@E{22388+!D@+)7m{pIEquk<9yxJ|4Sk#m9le{}DdQuMQco#=${i{3t1H=a1 z;ZN+hC*?j$uMF0Dq@4AC(-Fq!HjU36tcP=l^MEtM`QU&Q#O>c5VGQ!@|4m1TEbtsU zL8jACeB*)jI>PJd2v>z)r;czlb#HR03|B3p*iVJlDSbD#1Um^WulIQ4^Zxv#qC zKIJnoZ#?dECt$1ZY_%JtaEWfveUS}??L*3%``sX3$@xF^f$g169Cn<6M z$4(CQp1KSQ0d;W6N{|2E!%_P5+D!i)DV4@=kpL;ZHnI(=ayO67`s?VrK=Gcli8)eb zYh=TI31LjY*xUkND4^7mQT$EwcmBIa8qFRXEeCdwLLU=*>v$~I>hTtk5{D!43WlNk zIcmypPu)*dY?u8D&boD}!m)GX@KWMG?7h))EBOd38!fjVu*qaj+hj%veZIh7+81V1 z%0W}gZaTUmA@b&oUoQoltUjAe*8f3>wHbp=V2wHiL{tAM{e5=7L%=&_QdSfGjxVZK zT~~l>wDYds{5&J-3h)x1l;uNjznN9u`8po@03=6)KH$wewuVjc;akJ{`2pz7-0crD|1^1m z2cIzGJvA(USlHq;`60QKeVrz^l}p$Of~~BYpqRCKN{((;NFm@`u`21^vX(ixPa3z5 zj3kU(uD&$C+{|*Hk_R_)N5~R#_hUy{HOY3QFUvHV%Hd3`ICLnPV`UszIc*o8ys@rh zb{bcD<|a|lXBgL=E@7UQ_5kK`;!;K;5v-7kkarhF>e;;OarlfW|+lNmQ=DDG4R-0=+4^glC9 zwU>x<76gYaBhfVX%FuB?Ct`aDiCzG;5^d7l%|B6M(_n4$O@3uct0m#H^!GCkVvMqi zSFeVSt0nF015I-ms!9V*c&Uji#hcRh4@4)yxkdLl4csDb5)7LUCSuzu2o%$-S z4uOCO^6xjX!GMI;a9u23P}XF?*cxsYvLWHbUt4EnEjNPPD}*HYU2(oSJw*J)H2+(a z1MG)Y(?5@=KP`tqE-~_+X~9vHDasyXi^vOM1?$P=Egq~>n2{xvGjWT1J+fGDqBn#> zYy%fZxtYGKP#=gpW0id&m;qxBX*N7Nn$Lbcp8?WSam;D*Z9K5JTDX_e2Fq2Hx!}@? zReFqP)f7ePq`XhW#1rl|cpi5K=c%cfnCnx6Yn}+WzLVhEM{$@x5iJZwk>VhthBwSn z5K&VC@MtqG*U|F_(}K!sypoI6RVsU_sby=PkwfG`5%8?1U47n%9y_jJPE%Jry;I<5 ziope6#l}X>a7ZC!W~y!|cm;_#JW;(X7>V`Rj|!P-oZMQ@VV%dx?T75|Mgdt)O^(UD z#mRMfEyHoGb9~W&q}|Fejg5Y56vn-3bWs^8HL!ZFCiWsV=?eqUdkN2@p#>-DPr*;R;OgJFE%S}ApvNFF3au?n{;UrWn9e2UfvC*-+2MzV{#t4oY zM)!s?LT%1t1$rD>x?HMdGbUgIFFgHZf^2DJ$4ET2)`acpkTIE`bfUWXK3U1mOpv<= z7n`idGEAvYR-irQkM>OFb%l__5I=QjAvjY@%X(+XzH;-iplVya$x!}kmNrqg+O}Zr zR%ACG&{2c(phk0gC!PtEzll4~VI8DE&5!MrADG2T~x^2C&}qe$QI_hizN2^BsrWC;KNDs)9tJXE`Nn! zc_jqE&!+HWF(2S)-Ho+>R-P#rGWM)&MGo1Ym0LBZo}hohoCr#7tk64pgt4k;<^D~m z8Rx0ZdQFytnn~zzDK+u@tn8V|@@^SsLZ`?bWIvV!XhGujRzgwBoUAE!i?jEq$nosO zsj`XPoFaE&qo>NvWji}P1+2%W%FWP;j+rV4pi7-SRepHjHYe4Jsy|Utu@n3X7C%v| z!-nE)M75Exx=mGh9N*)ErxKLN{JTk7*RH;%UiSS|`2$(^jS|AJA^fGXzoyGC$jTv! z{pWdkX=l4Hy8bS0VW*(?W?ZA||EMMyH6nVhB)a~4;9|(YcC?o}>n^y0*`@aKMBU4{ zz-i(GbiHu#d5}C=x34ysO?p922&<-n2;Up-jzWS4jXsZs$GlF^IVVwHf57vqT%m5^OWC{+o(yl|DRDuK$I zIs~z+gu*()eohFU4MNhmEffL_O3ajZxu{K8_Ai*zQjhUS#U0jcrrg#eoomtv<+y69 zx{grE3H;f(_R>6RmhbX-F;d|(=j$}Ho&gd~pK-$Yakxs1lTTvw2^@12*T+qx=2H{kj(E_8K@|lY~HI3F`Tf*K!wuey~YrZd%jQmGioxq)*>>B z`AX+_JziEJxltGJ#yJwPfWm{R(|&pZl0Frs4hRq9wfMN)J0|N+zFQdyU@EtxF7aU% znlD~n^0ueb^Cb))UWa=P-R?Y-_xRV%_H#GgkYKSKQ%?9$dj3ktmM)b(q;(C2O8)deqWQl&uqxe@?nn(QlZ zGMo|4jpk_5g6&iz&`EhAR*9}7I0XQsOgar(=+ZY2Qs|k<650VuLp%&Bq4?9_Pu~M0$saN1i)inmU?8{)?jXjD=&EY3 z;reeHi=mgE`VV@BX(3JN_HT6kdNDj3I_@BCH^!iBTK*QwESSPJP<{p4LjIVKXxn%( zhLI_Uu`HWm!^J*yj~IT&o67g+fN|sqwk)wSICo;2sZfnTpqEe z0!PVzE!I?Co?}DY^|V?$ZEankL>9r+Rbu4V1#BA+7>2%7hRLh%P32WNlzNuB;q@l4 z(46sNhElE;Gb)COk+r4;Kd7T?wGdM2v{7eHLBja3gu}a(c<_U2P2A$H>hSva#=~oe zHoQg?;CbdzzNTjQ8HfQimYLG>dAj0aA#XbJ7V>+H%_!4pkWCd!zwNpW753}?c}44} z0HH!q%wltWz5$6RSS;4MaQ_v)-LoBsYh!#}tFGVI#gW%ByfT6DbuGr%SDc7a>ENF&ox1Ow(FYjr#$$sv^UN48&(Hrv5<1??>^9f_OWCR~H_9o(*cLK!$8q z%gp^eGflaT(sq-IH6~|oQVkuZr?nGau?|q3+E3MIKh>uLbaim3`aZx=+x>dGMlDfP z>{U;3wMz9AS4FC)D36uAQ1a4+l2Xux(QQ^;PC$XBD%yD zEl6XiMSYoQ&rII$R>Hf2=M5(BkH}a84%mS_`GEJuwt|NPXXt;qS|C44#|`h`Y}EY! z|JuX-&mG+34SG1#1xhzek|;hXFxdc1lA=uLMphCh?G`3Vr?J?EqFl^nwotXfNL;N% z7>K{NP`0|;N!YukuX_J_hyV2Sl2~zgjr=TPew8$ zNt5rftVMEY{|X4jZl#A4NU3bjQwQ$f;gv`J@N~g;N?i0$^rbXRzj)7YUX-j_kGaRj zJ}ZaXFj&h31)UFNo#>qKc3ph#mAc)9e8{Iwzi4ye9AqIj7wWNBhlk^T+FVFO8Eh`B z+gTXZLY<>w@`@d49yCd#2xF4AmQ9@?w_zW@Cbw)tDaE^Uw)Zu;ou>sx>Sk-~+H3M* zCA^l|Uzb0zQSrk{4#h!TTS9Bfqmrk#0rcYgJ~O@e4iQs+E5y=-eR6bT=EYL98Ay04 znR&c0qR_tGXgYdzKB)>_YG&fn}QQEkAR~-U zJTP=~Vre={qW?Lv6PDu?r#6~1c`sMPu`Zd1c}pe-+xBcA%`RhrXuro|}!5#GjMqMz4R<$p7`0{_OZ{a!|sywy4muDl=mqv&hfEcAyFrle(pdEuXUvnbY z-oyV7nS;p49&YLo|KAf%ltcH;7O+K&>{~E*hfw( zt=L>Slu%R<)Rt0so4cqOG-^cIb9r)~))Y*%D^!of)Z0K)@0*@3wj)pO+tPyY zA%~q16Lkr_b~^LQmjjwnHlSPcSyy?FVIuC^lyVzO7}#0HM&!%ATP9QFOeq)R@z@sb zB6y23@3Qyu<&Y+nv8L25F?Y)j<;$|%pFEK+{V3<>Js*evFTG}U-#qMUF8zlH>yJ1A;OgV4HpUat4uwZQ~????RVI; zsj?roe=XX2huNp#75UQp^1L~|zxN74CHI+BM;aO{zQgXo0_NH5aTv8>7j1$a^y` z#^Ff{PBy-X0Q4lG9T4(=Q*PI;>wTdM^@MJAflxMtMv>4JL!G7iLS-b>A41C^WGj8J z&`Ttwhc7+t*yhtI4rdd($MUmlaSE*^7N?E3q7%$#G}gwZpqahY5I;;Nc0vG~2I6x6 zab&pjOT;CBIG=D%C|I?QB(PMa!9CDVFWBRE|3Oie&ER4wOXY`J42#l`+uD!;0mr*pB4WpapaG%gC3 z$!&B~p!oSR*`mvB;$laZ$sKi9AzZyo?$G*6m|7ecl8p0PO~_k>IV~Jveh;ba49W3b{dG_ocOJFI?7EO^o4cwRxfYJasSoo9TcJw zV7MVF^dSH)H5pdd70Io2f$;udksQ+ZADFLa$m+NW0ylJp425fktd4vH6axY>WOY#R zt;^+}x?I`C#x9qwy0N&(U5<4wKU{pjT#nRzCb^jB3b{k%7ZOTG?w;74m)m`7g+WIJ zOeMe+j+jI{LZ?x>XHBI~_Ay9a8pk8`s>SgV3`|=gcOIG4*uZlSFn}C7&7`NR-C{=6Ubb{O;JZ}32=WV`Qf-ccvMCPdMRm9$> zviT=j5LeoxwD7J2g7DeV8X>@oi!rrFVzO(EFGllkixu>aJc)P~7~V%Gn55I!#=0sK z@;2F_bIFpB$Hu-ThbeX|Tk;l8Ui!z%O5c*Zw7Gqc$A!GvfxN~Oh&X&u7G5OZVgI~^ z9kGR0dj6A&XF+NE@dO8t*B$Hb>&(z}P-^x$^mnuvj)swRu1c0}xxu_I$g4Sy1!5AjPV7?jMb6rKVnMTDmVa2=Y-q^ zlpDA!Rf$d&GF4u<%G;_Et2x*{oUJnH3p-GxA_br?83D3YUW|ITU*);0cR4CALA|R` zdGtja=oPBGWcBX2%JWn2R;s)d_0CqQGU*dLP%KuN>42;-to-81wV&e|l!wH;vF5~r zxfVwp>IWTiez_V%bOMV`&BV9HP;`c3&rxDQJigKy@0V2Y%4{I^4Nt(QKwqSU%j$&2 zj+^U`T0AswnrhB=lNan;j(1^iC_bQw&h?jXvCNH+E$&7R;e&=xV+-;`e4OTKn44&I z^n##Tp7!md3_>9&MR51cFTp=4I+pUL=BlGF)$;}&t@@=4-r}hx%h7!n&u=Pg3---+ z?5B#u`3v?XI6Cu29nIbIbb*%k(od##?6!a6y)MJ_^xL*r>* z@we1>IZ9k??mD@ZGQFBH2q<@I*tT`_@T;0#StnatoVnFV;a;t(u;-Sl@EBLHvNzVt zoyQEJs8GXuj@M0m44hi>3ICvR4#!`ER==Gl`(=IEJ|wFbvX19?JYla#E%VtRM<^x# zuwfhIwmu~{$l7i+T1f>jSSX`s{ll_1$er4}0+qyEoac|DGqk?bmm0VJ`2NHuIgn*; zk`2m{JIt{G(d~1Wd6vqpmBTk#r&77SPix9GvU^pv`;TMgO*Xbv?yRh;VcRz&p~l^0 zOTLhUl&iPdu#K38pDUFE0*yR(5YBgbu@mvOAdrqr*IAn{u(M?Q4c6xixl@a!AmWpo zu{*Db9Qfj{YIyby)v%r$?%1ab^)Zx|yLi#Vmm?`HLbjZx=zszaI3`|6^=>O6xfarzX4O+W{3JVKm8jvv+#$(JvKc#O$*2K7UgPU z!7Pho$aVJlM%h0wM$`S}HdQSVp3$Tm`Zqg^v{0)5X0;pTE}g#vF?aXQOCpAsq42;g z+4Oh4EP3=6dvTN8F)#ztJdF2{7fHDyEwit|pCQ-SH=E?vo!d3EAM@YryK(#Nm7;&R z>4c#5W zYBk@&uhmQF?Kdf*Tcbz0;^>A_%f-Y!eDVoCBj`@Ex{Z=_yv9@6&x5}?6 zD_gTOTV<=q%dKgVo%YDxxWSr#g*_Fqty$t%a>rIhh!<8%U<6I@Zp<{AVf=K_pVV4E zxg3$fE>E*Q=ejA?@RPr0WJm)gan3fC8bj)lU@1@fRHEgIMZHi?Kc&XswCE+o0izFh zN6D+~pReSc9v}Y_EG>xs4gV6p0bHHc0UzsY{RhU2_%7VS0-lJ!@#m#@5SWyE&9UMV zEB#s?F~UrS;Ll-*7O0=fNyrO#9H1PSx2FOto&1{}^u3PlezMf_A4gl#z*op}1Ez6R zk#g)3+qFXu9s2uj3X5kh!kwV=sBoQ4DhTYphep$$pM4L&-dH|}L%oXG;lq={cPJ&W zeMp|i(V=|G?bjoTudRQF6fe2zONUaECfZB(nP-?UShlG9tnMo7 zGg8h~@Nv{^gav!!IY5V0>uN~|!Xau&pL0kdswF8iQF5?f^wJL*g^)i1Lv`{u{-~4p zhbou1&Xm}TyX3*jhi%#JU6@n9&=%iAOvxHaA3J;+*E2@;#4{^v1zcc9+-I0x_@Nh` zX5nR#5kF>_j$y%dE@$2eWFfod?jE-SWhv`2<~@er1KBgX@ixa=P=c{!FndTgwr#4ZKaV|zAJ*?PhD;8vD# zuYQgTVWvHDbLD&$3*I9?rPKtl`FrFXWoZEO*egHecfe6gQ#a9!EvP;;=NaT|&j2=j zuRN^PxCUG_awZh(pWBdXYxcul*|$Yt0|}JctTybQ; z-HdI+ip%95P5(r*N-6qxYj(O^4(xdz_jrTlFlfFP`E@EDyJuF#AswUFs|0)-Jq3@d zQ-4Ro^dev^S?nf+t1r>3}%r5$%df5`Cy$qMglah|i>pOX>GPNUn^*hx6ybkR1 z@8r-n$2!OoUQ@w~W1c&4iPoe4HeTC5@4zm7C;PW80TscvAh>}6{MxMG9+e_`!Z*Pz z=#V@_d9eeVen@VwWSwP856MBM2(px0tM)Vl&a&Ny*gk};FOOb&hWfL))E85SrVShJ*S!;H1zeTl ze&ncxd6)SdHP2AFHJEMqUXJuuHdDp+sYuB6eUXHJUaG*~gdV!#d^KF*egF$LJpVBA9J;%2|pzqAcv1yjwT&g$gva z-`pQ3EVw)yFR1;4Eh^E;y{G_8_g$&t30#NHQLnol<7i6?iNWJ`JSERK0vFg1RMW2L zmc7`Zn{vC>MlN*9cNoVE?3Mln+o-ShUecRo-;_hTr+_{Wd;ZY=F5HRXL569@HN04B z!$_E^W| zQ_*IoMZeFlt$4jH^&VslM^zpgS}L@u7xBtl?^k=WH~*1?Y`L75>AC_2IqcC%>6WD` z1m<&0F^m9SH~oQUF`7(jXr{{R79nIT996z)^ax6uly1!g^Ewu-N!c_yD=xp4Ue!TT^PrrEtnUU+d%=m0C8gUB{3a#zAjeb^tRbP#R7EJHVo!TY;Mc0s zH7Hqq*Qvs%!GJOz-_}5B5y8+&A=j_1^OY9-?4RuBrU@ zU)2&e3!Y}&K=}y`>(1=ea$99?IIFCdd&K{Vb#&0@P&v4SJiPP^?HBBnMkzWyFp6_C z3~?%+G<6!m1@Qu-HM*Uu>dV5}sM~Uf;nQ(%vU<+uPz8^7BQC8~q3>WBDR(NcFL5am zYZ0{Rf0(RD+f$+Vwj@F^L2056)X=)Bm?xN=4NKUgh$iW~^6FC^qZ&E^|Io{8~ zKvhx8!O(81lA41SRhY^Hdd781#m}%1YAXLJ2lv!gppIPy?>7|m-%#hK8}3-Qj8|GIx})o4Hdj#| z(S7oX%(g0uRhPL=E<3L%`Lem{b5t`t1Fysuqv+$G%VjpbvPjm={!C_WM&)(gkq>3| zrcnvfeTs`Ojmj)t_@^>^$W8HY>-QxiwStvhMNKGV}6K zI_pAk5$U0X>T2GV**Fhnq3&K$aM_<8%8NQhw}p)}D^qm7Zw8nBU{*2=5nVT+i7mru zalz#j3}0rxO+9g3NX*{b*5hy$I0n^FQG=7x4vfN_fir3i-`BwVmP%h;kGEyEuB8%? z5DY!tfxT6id^_O^XP2D!!Et(lU2(H#=S$YL`_qGPw#semxxKeL;rnEJ&(;Q7NU$ok zV&0!-tn2MPn-=H=Yu>Gtp6wiOfwl8g&E6Af7x(T=t=@YgwQ}#yv^#sNQhZOpg;=#x z`q(N+=(e@nal#Wg(d1|25PpZjJuna&Ou}bVj_*B?c75;dxkwh)#!qRcGa@vd{FIUwc`Lb(RS^DV@6MFy zS60gGf}fHa{4Ftd)<%~x!c);M!m}wS5j+Qk4K@g-`zx{ShL9k_Fpltc>RyBy-UF|t zdOi&yyT1~qJF-G9yXLRFA?psUklBJZ%2u5>F8T#1HXqkA49#%3#=PB<;%CQ(aOuYETp3{Kodun)Z1haKgB6Z5& zGTYt>b>;dssw-&TeQ)CmZTEQ}YN2ZA27Axdx&m|OGE7ugU?b4U1+}_@F+|{*Ycgx! z8FghVXPrXhn1)6zwHmGx*LSG9TMMBVI-{->b0KTD64ed~UWEYW6Dgr;^DxIHisdfiEeQjj8PFh)yU1v*lq*nXd9LnLYN9@|4bbNoE@!Qo?o7 zmt}VDA>~TnIhXis2+Mz{BR1Til;&{g{U(R#w*e^IA9C^lJ>Zbb{^_Y)l{^0O2l&`7gCh;h z{_vjeq2?sA$s3CvFns+FnRz^{4ASM|V&uchKAr7?%q)*6L%PRafOx$cVp40!;@=yR zv4%@EWWn!p*{VmBJF>DqiTQU>0*vt0ke|f9?xQ5>W}TJU`+XH37T8xApbOCMUhJ!M z)Sap1y#BhkaPdc9psA z_QL;HK+s1iJCx=V%JxJlCl#f99xHxS2~wK>r)+MzCL#lwhUT2&N2G<|wmdy>!J>vKk15YS!3u^c5lY7=*uG)POvUmD>-U&qQEDG%>5nOGl`{{s?8lT|w(Q4g zVr5R4cTSQP;@R090(wu`kU^XU5z@?-j@@`db_p-L8eV=PSlTBwK|0g4+c1DcZsAUr zW|B(J`znQ`avHQjT*X~eF$)!hnpA8C>AucWnvM_Gp+iWxH@_yU-eB>PdvloX)sS`8 zRL;W8J_%<*s4AZc6-;)y%G>u*?Oqh_DMMRLDhLt)yc*^*O_=Y6oT?I|sr0(9@}{Pez*P`VRmEFXv8XB?^9=HRow1sZFW14Hs$+b-Eah>fv!c8_ ztZdm7IxB||YSI**J_xV4iX zAG2r{3Rhg7=eVRY*F@!*Q7MQo`)Qi;mq8aeFt}`QmeR_oWcOibURHimK8+~b@`|$8 zs0%7r*sJrE_ztVPqb&6~n3#Lb^!oaa^uEtolW8HPV4mk<@UZB73$l*+N@R!Vda~WD znk;G|WIL&{KSMTgfilDYEQ(CxIG<%n@=4b@Q!}hIAZzkGSA?@;3zYS`)_WATXrXdP z7r0wt^A{--+icp^KtapJJkN|UrhiTOSoil%g{^x{xvbOSV%6))r@B#gg(c)D@94hz zR$+hSC>wNcxLBDB%CkEZ_V8j*%C;l$i$U?n#cO$>Ox&ihhQpohdMaD8vUf--RzamhoHYx1OB2XT|#iZq+ytPqb#ucDk{6b+{SAdd;i;)Zz zR;n=fm7v_-ps3RiwU_iOGPGKc)gOa*VDI55XVl?ZL-&NRSpDVxU zX01|Kr?pD=mi>#>G)*ke@g}yvjjWw^H{T;#Lr>-car{M zmbZ)aUuOxsiJ!xo?IHdmwto+Xx)ZJjmDTQ1#>z^}Yiw*eDBu4TR8~~3jFFY8^UJR7 zQ+k`c=bsP4Q9n^3QdUp&V3W=Vu}6MU#&|Uaxe9X`bZeg9y~v7xQJ!e}*|{Lx`hIdQ zsI2A}B}mpSIUB^F(qj%(&3)j6Lc)S{;`gU3(%($~8#D%c}{E4~eSs;bmB~%RK|=_;oe3(Nby|5u z*11jxm6iUg#LK#ghl5!4870s+@i56m#}Vv2kM|x7GfxGv&c7+wyx&A!7l~B6#P=~W5AcduN(y)_{HECf?o)JA@~;XE1v`p{9^F4!A}Q29sKlX z?%7<<;V=Y&LI`9+AQ=MT5D14rI1Jdq&jvpo{21^p;5*ac#=>ELc{cbl;9J0#z^@z) zd*GLXUkZK<_!9V)2&5g6CZlKnFht zd<*zy@GB9BN(8bJfz0lMl!^xLQOLzWJ{B?qU?&a%h==__h*SbRPDFqQ!%hF99fEmi^LtqpJBq%382BXmTdJVeykcBJF zgDtp<#gK(_=E4@78Ll`7w&CorLl>?RdbUjb8R2F@*$B4+${paogz`YRpP`rm_YV|b zgQMeZK8EtcaOKb&30H#?XQx272738$0dQ-f8wGa=x@mBip*IK418x;`yTWaI4f&r0 z1AoK7e_-HmD9(p#4)+0+d%*32@)L09pg03gfm;gYAh?at?GJ~&oURn;JD@ik&K+(g zbggh(p_>4A9D0-Bs-c$y=Lc8v8uC8^2F}31Fc>%k#c6Q0P+S7n7H&P1`@tQ7@>sYl zP@D_r1-BZyVQ|Nw9|!$o&>atV19}VLeBjnVw-?+V=#GH90KF_YJzS9u${}zEVW2Mz z9E9>{xIdxzGF(%*x1rn_?rSIyhC2zx$#8d|w-~N9oE`e%(6>W38Lkp~nQ#(ZA#^Qp zrO=ImtAK7goD+JsZ2XzwN?(D(0ubke30FA}2v?a6gsVhKQEjQ53xunD6&}Ia=fGpQ zkD;Fg{X@`=g4+h&D7bCVje^?--6*(i(2auI2Hmc&BL7!G*#mAB4D^SAT~O`;w-L%c z;5I_J2i!&|_ki07rWw!)P_*9unx zUB6dB$boV-lyl(1U|>Cz+rq7ea$C67P;Lvi8nSKSRztZh+-ex`f}0EdE6|+_7Xtk? z(Di{Ug02r-5p;dvil9rM(kp_l4_py+^)~!v!N3J5XTi0Gfw!UD6mBt;o5C%Ia#Og) zP;Lsh7|KoI7DM+AbSJ}|gzjWGGxQ6gE5T($SAxrit^}72T?sB5x)NMA^leTkr^8i1 zIUP=dfov!{p_mEhgkmO~6N;H|PAF!=IiZ{hR|(x@I6LW&h3?a^@eJ@k!12Hdz=^;~ zz{$YpfK!3ffSLD(xU!$e#S0*0!3YdU>9~dgX&ev+qzoVoNY4UcK$-%C0ckog6F371 zeQ74E-EQb$(_=n_&974OOLtufxK0PA12ciMfyFs@T`~Bab9(Mw*Gga}uo5UOzUxZC zi?OB4?z*ah;VbUC{4np10fqyUfeFBLU^-BG2ltq6nStp*3vjkOP6@T+q5^~ppo8$^ zyDkfk_^AX&0G+^OVE89@U4_6HUvG3=2^L^DFb0?c%mmK1(ZxZ8qy-EC(}8wiAuzoH zA;feFCm^SCE_z--vb1V;EG zK)>I0*(Tw0d*Qgq2F3ta0+WHIz(SxCSPXP;1%to{UH7@ryND+OF%L1%4)Yx3daS`rT<8l%mr-`()tp5EuqJf%fO17>cByQsb%w#ynr+%I<&wXVthS zb;La|8yNm#jmvC-$1l~ms)2UcPYH${U?#8WBj#4bU?BnkOa>MLrPpg*6~JPk6IhuC{Z7!&hljuzU?wmVm;)>Z76U7QTY;6p z3ZTu23kNQw0(jUNFC^zU#Fv>!tNfXP3>4zTb9?DT>E zrMNiW2T22T0xf3{fWGj+fx-i<0Hy<_zmNsd-~$VR;lL6?d~u}oQOMl@Jr?&8ntPyA zMi1K`@_ID+z-%K%TfjnKA+Q)&V#7r#F6=-%@Ho)y1_M9~(0u?50xiIFUi!#4jEfc(XMVNI%pr z{*zx$enOIGn~m>eJi#vSF|9`XSM5%s#`=V22RYyP`mrBy7Bq zXeqZ6bg3BxuSc;*%MJZlll_LPktM7C?OiWD%r>Hvt*)T21U&}V`iG)e&jW_u!9=K^ zjB~(T2*E&tc@G=vV1Xe3C6Loh%He*irSbZa9zm4IF}pn&1h zCNMm%FQs5CoAaGvgs$|pyX?pB3?J!^ufEG(Ib>MWAsPBH(2?|%;a+Av>bT_PF69H_ z+|_{F^=$oN!=qlsP4sUfmAoSK_?Otxas%~$_$`Dx2I1m0=vvvj!-fHJZ+7jlA<``k zL74sVUDor6A#w6r+-&{q-wC$|^wI~=PlI0a0QwEkiyP4?hdi1h&p|JiSif?tQ8<4$ zG+~DC4G+1E2dDBgl-lnN{*NrgjkA$DCG#rK9gXNApl=q+dEXn_yPd{OH7sqS8$F6B zEU{B`(-k*Rr<*`#sW9l>Mu0-TcBwG}z(AhKv*H*|B zqa5mOJ=t?V7~VGpAb5V;kc|-5NA_fGel&FNI0wYzM;cnAf|dn3jkhEuir&?OZ8(go z`o)ii;l^HQe8P{EH9cnNEDsxx(3Lh)r$EjDz2pJ(BG8K)(W4;07W9=z*!-UieLG&} z*A0V25xf9;A#^Rc)<4vpb@|y4xZ-DnzN3#fQdt<)H=|OHMuHsZvo!@*I4QI@rGYpJ z#B2!YPldBFKO0g_dxV7;R4@JO;cWNMhIz&;)PR^>WrKe)G?#7p82&l-Gz=6)bRp={ zk1Ab19KA4pMBfH_^$-7+KL)xJbPEZJUjXnj=qq7AoM>8w4aGGo>Ymv{H!J5QvzKnx%9qS0-K^r5^ap&|&C`a6 zkuRA&b+ZZ^@F=KyjAUaRXj5>l{fHg_`f<=Jsre8;DjGhZ7hl7i1LH~krEaS4&a67$ z9a-dibr$)aJyRdrhAlc_cw`vJ@ZQ;gN!}cTIq6S~hwFJm^vj?}TtW$?7D@alrSyoj z(Q~Q-+`>Sw#21@Pz$!$`Gq-qEBPpPaC4x;ZufI zO?)uWPWGumfdF%WQsKs)Icn(3rkpnTxorlovP})kJB@lxH_kQ~L?LhVW4f?SrwuX6 zjOLV>k3E`&#I}cF#H57{NWsnry&Ck!skRXGF{aWTaYk115PBhgpFo9OEWjYd@l=fxZ>=n0l#6sg?zLW%n93;FvEmaDlPc>$lwd}{=438_LMg(GTl!not79=2rY6|En51@|+Jz3E8n>uic|- zJpuG|T*UGY};R0(uwv3O$klo#-1Z_K3lU zJ%7OvY1#&vl_Y}_s{bTJ?Tl+K7?PXLu^<-)!=yspRXaBGcf%;tX%Huk29Zo&4JL74 z>d4|cj&F~$l+-ZUsHUZXz7=$OC0YLlUU)fbh>?rfymCV%d*`UZlU?3#aA%kQFa+o< zqia~Ri-zbn#bat*VbGPvd6^T@ctqhc3vTO}8a56B9js$94*@}n^)f55UhV`b5TtZJ z23G=pjGis2H3YM&i-wN2mH$CqAqDZH48^fkQt1QeBS7E(0QwZrD?zUxGP1J}^y&xD zSAlLG$5R9o4Dlm{&0x5z42{f7*dfrRMszCo7la4d#EL5f(EMJmL5xihaSL(qWg38*eq1R$eZrPSFVKgNTeGYQi{)enoVyrCZp!H<|+ z!(ON|L`Nn}d9Wdh+yw|8pVqJ(j0e32^onUU>{69sLL0y74F$3wFh_^vnqI>q|1vz^ z?gnmB8VQjV4;0g62*JuT?XqyQvXKy3$_KqrwKTOEC7vEbh~v|1%HFwSI4iqdMs;^S zUsJZAhVE!!n3Gk*o~t$ZyWPNz<0UkmwT4vN2#n(*Ud40{CMn4jn-?uhNKe06e`E-&OzclJDkAYEQW0PHm(Z*gF z!)#q#cF<)|6l0VJBA8$HgKRu0cc39rHs1B(6c{W~7pK(A;-r*t_6`tkSJWH)1L z;|*N9uVIVbj6K?R#Tdl7?%$rqfj(({*-1BJbKS#f=qa-?$+u9b>LpGV=sBR*Pc$m; z`CPs%sF@MzQG`xzc4=8>Po#&DdWncltf|R3+IWnvcd*GOWB;}RXzfeCMKb_xDaONS zeoeta#+Z(PzJDh>Xfg)b(r_*9X;?NX>#{(v-u-WSKIqN|(AR+OcmRDH=#`+y!KU<* z+PIN@>WTdJ)C~Gpr*jz$^8*5;$GF8E^e{3cTFD|@27qqa!#0|YOKqER?YH;e?cO2K z@wrY{4C#s=1>yqeDWHFX)YN`NuL0e8y2f=1Qv~ftbRT5;)+;rxy`40w+EI1|J=5== zpXse=z&KV44&$Kgkky&<;&TW9pT%amhHmc~{p3Y4H(@^*LuUg0(+{5`j)n)ow z)~NFC?kW79*3juU{v(w>MPQb|IVy%oIf76maFsw+9bNU~p?KUKs1&}7y&WykK1!BP z>565(r|YRCuhy=ID32h^51TB{|VJ&{51V9#66lx_4m9Am{Ibd`*Jn)L@Kk1ZH?YRv=U1{^bY(-k00nK zKyrn!x7!-qHqRMJQyOF~|3|qsvxA@qtxQ)5S{5Y_QSY^$I>DDUZD(w&*ax%DK=0xN zm5(#grqk6LDv=vxUHWj!8DCqN2y{Pz!vu~MI8ESOflCCg7PwyE4uMAmo)LIOV68x} z8ESais%VqK1fie6VFJeroF;Itz$F4#3tTU7hrlBO&j`FCuvVbg3nBmRaM+BY`ctv2XK(E;%05)+ECa|BtVFJeroF;Itz$F4# z3tTU7hrlBO&j`FCuvVbgOKJdYZN){Hz6OC2=oycATUH=SAo3*MhT1)m?Ut7z%+s51x_KPHYQ6D<_KIUFkfJi zz?A}52`m=4Mqr7+wE{N_+$M03z(WF$0eKsDS`aP>bPBv7utuO>3eq^92?PTq$skz*2!*1=Y6QBA!sR0{KwyZ#t^#`rj1m|pFhO9Fz+{0V1f~eg4ieR`NMMOTyTCOc@oGvf zfW9pa)eGNM+kxcODq7xC@&1T*Kv{TK$-pR1#xD>3Mb%`rdxxUPFns zk2gNl!a@xuei}Oblx1Cvfh;@TXl|+vw%wPrx8o6p7%}+PFnt<37jF#nF4oa31nUq2 z^{o9sq)bT@m8@aWqwGHejbY8A5>+~oy))3*yP1@%j?1NF13N{86hTM(Fj{7F?Wg|$yGKIE;< z401-QAZt<*OmxENSJeA>A9xkjOr@0lm-CNzEm# z*xjd6Oj<}<>1#;$k+eeCknSsKrLG~prKA-;qCcuJby6!yD}EY{P$&6GTKQ{8_m@xr z*|;QQH^nlCElo0p+3ISLkfhv@9}uGmPXBj|;z3uCH!-3)5N4z?T%LQ#N!nOyvY?l! zHKB#1O_gaVOor@p*b_sIVS|%JJ>}Z?dy^}gqzKUlX!Q3u{oeum9CWf*QN#|yo=wZs zu0ruCgh-anSb|b3idNTN6ZEV`bXwOS`Rqn?9oi3~=L`A=!v1_g4bc%?!UR5aM2a&wwFI7%Lp~O#1;U|TiG+{?w06`xzK`^EW zMhjtR4>v@$hqZpp7&b&(5Q`Cf&7<^Z8+z0XODQsHUExZ^2tU4D3k>3Pe{87>Wy>Bj zc573&vc|eQqBYi%=!}r>Iz;3R0?U8b1tURaNUH@s7IG9IhoEbDGf~iQG@`#F=u#5*pf2yf z7IY6mN0WjdT9i|QVG)cx;jxxCUyBsd8q413I4D4C8+o9W>aC6FS|QRFH`NtKkrhUi zqSDEsEbnn+w{bCIwbCL&uvqxCM_9xz3jW(6=*Jq-HJ`NQ(*{1lBRf25^Mo<1$GZ(n zji#q9w>H+>1HD(CKmaQmDGtWs8zsesMgf^C=rs7K;E9jR6T`Nm5xFGpzLAPlum#YQC zeVFQjM(^HKRq$&>A1&zCMs%$#Wv^CK{#y~C93dYeklEkn9SXWk=??b!YiMvKk)x|(qOp(Kor&s}Y$+SmkZDo03yaB(=z9b`p%MLiLAMCHS$OoTpqm@fF9~{#N*`0#H#-F* zqLBx*{hy*9-iQtor%R1IFbTSIJzG2qeSBsk{v(24*oZzz(6bxSM+tiR2BxIYU}meD zU|K|968z#udh-O`snJCWzb@#d8eJq1?iBQD zC3ZD5n)FU##VY9OLhXA&&k=M#LH|L}iv?XP1}6pG(ujT)^!lM`htNEmeBi-J;uOkS zQfWyNvs1`7N)kz7v$ZVO|M34Pvy(bg3}&6_@vs zDz&qXluny$YDJ$a)_FBCz#CFrHJYlVg<7QtwO0Sg(b z6bhOL_(CYBS8#>v$IcOYO5tk z<#^Rz;bhgLmBRnhBO*aWC;XwHryp;aAfF4mQ_y2XZ=@wuxOhrx5%Cb|(O*h##MmJs zXcqxOLi68YL9b{;Zxj#v_YaPT{j3^ZZ7E5MMILB{&)0X5gfp;BB8STszI-8t@1AM440o^&pd5x zB|oLY^D4|$Ay0+(RM^PDwwF)8uDR3-cHf^Pg07~=efrOWexIUVoe^}La>Rccze>6=bAx)X$?pW+w>cP=JW)bK793vo ze6v<^HU0aWTB^$SM#`g??QGM^{{?W#LU>bERWqGK+{E{UZu0qt9)Bz7=0$2iohn;8 zEv!|DHWLeS{CA10$+~5-M=SrpR;rJdOjXw|3REI-yXOk7WD;Cq`Ag8fQK*W=N{lw} zJV9FWUtCv*;UmSx-bgoG(2Lzvo0Td^Y3%)pXeZtVn@sN)xu*?6HwrbWKn;VIG&O>5 zwu`V~WJW(TmQr{SC#$94|0(P0gQKdh_}zrP0YtisflZ3k4Z{cmiJOEC5Jev_3bBZ5 zYc0^$Y^J1b92_>pg2bk~V;$oM)P*fRKL*(-M5MGXP*_?>x>2V(?TlRpvBOmA3Nuk! zItJ|wmbOFBx#xGC$@Bh^clUR`?>+C_`#v^nH-NJVXScGu#cm1#Hw(T*nJ8g(fYxDrRe`$a*H8(#t|N- z=e5`Pt8C9V*RAQG#bvoUw0(#0AnjG+*kRj()k!!z9-Au~3mAgu31>_3zlmoBcu)Xt zUScLvT*&IrV(pwcxTy|3rk2G^Pm9_J*e28I!FV_a00H#6i|^N=>OMpM@-6dM4B0ni z4)yO77yF#}nJBlj+(fVCF&_8eZ)N&|p|%U4Mm%F(LpKU|ijHE6Y_VKKIBUz|wwkrl z4ZP%;>fM=OFyuSRf0KVA9uwv}5~hP_&l+1*D(2ane+K>{(#j5({+j41d{ZKIw1av)n#FzMsLUpKhQyj2sdZU;};W2-weR1>P-1!`#%G~gULPPiFDPry{EYeLD$ zbnz$hdvVbEvzBm{N_hr;2>c_sahO6}^iRVKf5Bs=nQ$JTL|VME`j;A9WxnqV`d{F_ zDMJTsFG;^=V)O;DaOOl@Rn8E7=_U!!zN32sa0hWX@M{_HEbuV!R;y=%=!pRD68v@* z9u&knyd46G&&88Cr3$ybz|~cv2yvP@0)w=5~#+s3JchqgKG&c3BQz#=U!k4*B<4pk){$0TgG(0Yi%D8nVnI;kU;GX?vH?nw+n8E_7kWp?GrsEB+&J9pvO~f7rH|a(gr66a>}y^ z{g7~z)_)guj+Wyw@H<=Nb3qPj;#@mX)2YXjg%^dC1xMU+M6=Z6txn-S_z2De%t4YR zRFDWHhpeQ(qSjiumU?U@jZEV2fO5OTa`@Y9FZyHd5pQukat3Dm2sc}y)z zea!=cY6{^}&f2NR5u}C@!9&8ck#P1gj?uBc{M<_VU$wpJ|8S3ZQ@<72G)8BIyKNKY z;hu<)v)R#ucbIT9cqd>QVt)jGz*E)F1CJjRGxic?rF)UgB(wEmrhw-;#BEwX@a$Ik z%-{Iub_ICxGjYq7*NF|#gVn|{+%y`5^Coh?XA`+V`0MDM2vL-SKsyn52#eid)&pFX=DaKb`Fap3FO;J z!c8770)CCy@y!!FHZt1$R|)442;*wcfm?Op5A=v0{wgcClY&oDfv6B7V}>!|kw8)hx!%l6gNkRF9BsU;~7kp?z&M{vP-pPwHGCoX>+WZgRkox(NQFXNUJc zgPR>sGT02 zlCJt5rmMckH={#hsCW_F0yk7XQ!NVqjlZK{AP806q0F#W8K?AxY@r< z%Yq~2nXV&NNAg|E;n8u1b#RgS0H;8Oc=0LW>{Sls+uE#M2>x$LN#G~Z+?Ei|{AsL{ zDT6BYJ;G^vXMd^AttEmvk&r+9{Ds>wxAP~o9-o4~l;>(P*bF-Y^OvBBhwMf4&!p-gqn%%qEnR|$AN(`J?4zUdoQ~Wv- zHRPAtY{Zg%L}qIv!@_TUEkeveMnr_H-h-L%qLQKXt6}JgBTv)F-=;3s5aOWxU-GQ}nx3DnfRwul|;VTBG@|(`T{)ny}Qx>8B z)bg0FET2C=>W=NzJIkW=`B!>%&R5pZ==L4bb#Cty^hM9c1*)s7-JSV_zOlSotrunfacgz{#ofBU!gcy}MdtENbKKtjy4(HBAiXuY;S}aZ;3u40sU=Xp7cKD%g-%nU7lb1hF(?ne=CF? A4*&oF From 7cd23543a1d7ddd067ac8c673757c8a67277b44e Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Mon, 3 Mar 2014 09:04:00 -0800 Subject: [PATCH 52/72] Added public license text to all C++ files --- .../src/main/c++/LoadTimeInitializer.cc | 25 +++++++++++++++++++ .../src/main/c++/LoadTimeInitializer.h | 25 +++++++++++++++++++ public/VectorPairHMM/src/main/c++/Makefile | 25 +++++++++++++++++++ public/VectorPairHMM/src/main/c++/Sandbox.cc | 25 +++++++++++++++++++ public/VectorPairHMM/src/main/c++/Sandbox.h | 25 +++++++++++++++++++ .../VectorPairHMM/src/main/c++/Sandbox.java | 25 +++++++++++++++++++ .../main/c++/avx_function_instantiations.cc | 25 +++++++++++++++++++ public/VectorPairHMM/src/main/c++/baseline.cc | 25 +++++++++++++++++++ .../src/main/c++/define-double.h | 25 +++++++++++++++++++ .../VectorPairHMM/src/main/c++/define-float.h | 25 +++++++++++++++++++ .../src/main/c++/define-sse-double.h | 25 +++++++++++++++++++ .../src/main/c++/define-sse-float.h | 25 +++++++++++++++++++ public/VectorPairHMM/src/main/c++/headers.h | 25 +++++++++++++++++++ .../VectorPairHMM/src/main/c++/jni_common.h | 25 +++++++++++++++++++ public/VectorPairHMM/src/main/c++/jnidebug.h | 25 +++++++++++++++++++ ...ng_utils_pairhmm_DebugJNILoglessPairHMM.cc | 25 +++++++++++++++++++ ...ing_utils_pairhmm_DebugJNILoglessPairHMM.h | 25 +++++++++++++++++++ ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 25 +++++++++++++++++++ ...sting_utils_pairhmm_VectorLoglessPairHMM.h | 25 +++++++++++++++++++ .../src/main/c++/pairhmm-1-base.cc | 25 +++++++++++++++++++ .../src/main/c++/pairhmm-template-kernel.cc | 25 +++++++++++++++++++ .../src/main/c++/pairhmm-template-main.cc | 25 +++++++++++++++++++ .../src/main/c++/shift_template.c | 25 +++++++++++++++++++ .../main/c++/sse_function_instantiations.cc | 25 +++++++++++++++++++ public/VectorPairHMM/src/main/c++/template.h | 25 +++++++++++++++++++ public/VectorPairHMM/src/main/c++/utils.cc | 25 +++++++++++++++++++ public/VectorPairHMM/src/main/c++/utils.h | 25 +++++++++++++++++++ .../VectorPairHMM/src/main/c++/vector_defs.h | 25 +++++++++++++++++++ .../src/main/c++/vector_function_prototypes.h | 25 +++++++++++++++++++ 29 files changed, 725 insertions(+) diff --git a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc index 3aacefe72..0e3026f65 100644 --- a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc +++ b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "LoadTimeInitializer.h" #include "utils.h" using namespace std; diff --git a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h index 1834405a5..e7d9af626 100644 --- a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h +++ b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifndef LOAD_TIME_INITIALIZER_H #define LOAD_TIME_INITIALIZER_H #include "headers.h" diff --git a/public/VectorPairHMM/src/main/c++/Makefile b/public/VectorPairHMM/src/main/c++/Makefile index da91149e9..7296b7026 100644 --- a/public/VectorPairHMM/src/main/c++/Makefile +++ b/public/VectorPairHMM/src/main/c++/Makefile @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #OMPCFLAGS=-fopenmp #OMPLFLAGS=-fopenmp #-openmp-link static diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.cc b/public/VectorPairHMM/src/main/c++/Sandbox.cc index 8fb0d0b67..985b19ae9 100644 --- a/public/VectorPairHMM/src/main/c++/Sandbox.cc +++ b/public/VectorPairHMM/src/main/c++/Sandbox.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "Sandbox.h" #include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" #include "utils.h" diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.h b/public/VectorPairHMM/src/main/c++/Sandbox.h index 4ac1ea24c..486a1c095 100644 --- a/public/VectorPairHMM/src/main/c++/Sandbox.h +++ b/public/VectorPairHMM/src/main/c++/Sandbox.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + /* DO NOT EDIT THIS FILE - it is machine generated */ #include /* Header for class Sandbox */ diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.java b/public/VectorPairHMM/src/main/c++/Sandbox.java index 81a57c0a0..ee55ccbac 100644 --- a/public/VectorPairHMM/src/main/c++/Sandbox.java +++ b/public/VectorPairHMM/src/main/c++/Sandbox.java @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + import java.util.List; import java.util.LinkedList; import java.util.Map; diff --git a/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc index 8f0de827d..6d90d5070 100644 --- a/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc +++ b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "template.h" #undef SIMD_ENGINE diff --git a/public/VectorPairHMM/src/main/c++/baseline.cc b/public/VectorPairHMM/src/main/c++/baseline.cc index 66aeacbc4..d6085e661 100644 --- a/public/VectorPairHMM/src/main/c++/baseline.cc +++ b/public/VectorPairHMM/src/main/c++/baseline.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "template.h" #include "utils.h" diff --git a/public/VectorPairHMM/src/main/c++/define-double.h b/public/VectorPairHMM/src/main/c++/define-double.h index 83589a13d..2067d369c 100644 --- a/public/VectorPairHMM/src/main/c++/define-double.h +++ b/public/VectorPairHMM/src/main/c++/define-double.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include #ifdef PRECISION diff --git a/public/VectorPairHMM/src/main/c++/define-float.h b/public/VectorPairHMM/src/main/c++/define-float.h index 87b2b01f3..318f78280 100644 --- a/public/VectorPairHMM/src/main/c++/define-float.h +++ b/public/VectorPairHMM/src/main/c++/define-float.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include #ifdef PRECISION diff --git a/public/VectorPairHMM/src/main/c++/define-sse-double.h b/public/VectorPairHMM/src/main/c++/define-sse-double.h index d781d55f3..2d271a854 100644 --- a/public/VectorPairHMM/src/main/c++/define-sse-double.h +++ b/public/VectorPairHMM/src/main/c++/define-sse-double.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifdef PRECISION #undef PRECISION #undef MAIN_TYPE diff --git a/public/VectorPairHMM/src/main/c++/define-sse-float.h b/public/VectorPairHMM/src/main/c++/define-sse-float.h index 7516e6dbf..20af947dd 100644 --- a/public/VectorPairHMM/src/main/c++/define-sse-float.h +++ b/public/VectorPairHMM/src/main/c++/define-sse-float.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifdef PRECISION #undef PRECISION #undef MAIN_TYPE diff --git a/public/VectorPairHMM/src/main/c++/headers.h b/public/VectorPairHMM/src/main/c++/headers.h index 48bd4d836..4a0d89b57 100644 --- a/public/VectorPairHMM/src/main/c++/headers.h +++ b/public/VectorPairHMM/src/main/c++/headers.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifndef COMMON_HEADERS_H #define COMMON_HEADERS_H diff --git a/public/VectorPairHMM/src/main/c++/jni_common.h b/public/VectorPairHMM/src/main/c++/jni_common.h index 1cffea1cc..23c323246 100644 --- a/public/VectorPairHMM/src/main/c++/jni_common.h +++ b/public/VectorPairHMM/src/main/c++/jni_common.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifndef JNI_COMMON_H #define JNI_COMMON_H diff --git a/public/VectorPairHMM/src/main/c++/jnidebug.h b/public/VectorPairHMM/src/main/c++/jnidebug.h index 7002b3637..7fcab2a51 100644 --- a/public/VectorPairHMM/src/main/c++/jnidebug.h +++ b/public/VectorPairHMM/src/main/c++/jnidebug.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifndef JNI_DEBUG_H #define JNI_DEBUG_H diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc index 98ebcde92..8a3f8b5bc 100644 --- a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "jni_common.h" #include "org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h" diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h index 9e099dae0..18c2b0394 100644 --- a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + /* DO NOT EDIT THIS FILE - it is machine generated */ #include /* Header for class org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM */ diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index c28c24df1..0b54c8a81 100644 --- a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "jni_common.h" #include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h index 3b07665f6..d820b4b26 100644 --- a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + /* DO NOT EDIT THIS FILE - it is machine generated */ #include /* Header for class org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM */ diff --git a/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc index 78dbe296d..7ff219b88 100644 --- a/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc +++ b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + //#define DEBUG 1 //#define DEBUG0_1 1 //#define DEBUG3 1 diff --git a/public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc b/public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc index 77394f457..8c62b3c09 100644 --- a/public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc +++ b/public/VectorPairHMM/src/main/c++/pairhmm-template-kernel.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifdef PRECISION #include diff --git a/public/VectorPairHMM/src/main/c++/pairhmm-template-main.cc b/public/VectorPairHMM/src/main/c++/pairhmm-template-main.cc index cbcb8556d..5d9a05830 100644 --- a/public/VectorPairHMM/src/main/c++/pairhmm-template-main.cc +++ b/public/VectorPairHMM/src/main/c++/pairhmm-template-main.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "template.h" #include "vector_defs.h" diff --git a/public/VectorPairHMM/src/main/c++/shift_template.c b/public/VectorPairHMM/src/main/c++/shift_template.c index 9750e1d8b..c591c6882 100644 --- a/public/VectorPairHMM/src/main/c++/shift_template.c +++ b/public/VectorPairHMM/src/main/c++/shift_template.c @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifdef PRECISION #ifdef SIMD_ENGINE_AVX diff --git a/public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc b/public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc index f3fc110e0..550272494 100644 --- a/public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc +++ b/public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "template.h" #undef SIMD_ENGINE diff --git a/public/VectorPairHMM/src/main/c++/template.h b/public/VectorPairHMM/src/main/c++/template.h index 3e52854a6..ce4dbfc86 100644 --- a/public/VectorPairHMM/src/main/c++/template.h +++ b/public/VectorPairHMM/src/main/c++/template.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifndef TEMPLATES_H_ #define TEMPLATES_H_ diff --git a/public/VectorPairHMM/src/main/c++/utils.cc b/public/VectorPairHMM/src/main/c++/utils.cc index 35871948c..9f83cffa2 100644 --- a/public/VectorPairHMM/src/main/c++/utils.cc +++ b/public/VectorPairHMM/src/main/c++/utils.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "template.h" #include "utils.h" diff --git a/public/VectorPairHMM/src/main/c++/utils.h b/public/VectorPairHMM/src/main/c++/utils.h index c1288d9e5..ce9ecd8c9 100644 --- a/public/VectorPairHMM/src/main/c++/utils.h +++ b/public/VectorPairHMM/src/main/c++/utils.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifndef PAIRHMM_UTIL_H #define PAIRHMM_UTIL_H diff --git a/public/VectorPairHMM/src/main/c++/vector_defs.h b/public/VectorPairHMM/src/main/c++/vector_defs.h index 80b48ae99..2aca9565f 100644 --- a/public/VectorPairHMM/src/main/c++/vector_defs.h +++ b/public/VectorPairHMM/src/main/c++/vector_defs.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #undef SIMD_ENGINE #undef SIMD_ENGINE_AVX #undef SIMD_ENGINE_SSE diff --git a/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h b/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h index 67a2667e1..c0fddc394 100644 --- a/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h +++ b/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + inline void CONCAT(CONCAT(_vector_shift,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); inline void CONCAT(CONCAT(_vector_shift_last,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); inline void CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); From a893765ae2aebaf8420962e6f5b1f1ba6ee1e556 Mon Sep 17 00:00:00 2001 From: Karthik Gururaj Date: Mon, 3 Mar 2014 09:11:02 -0800 Subject: [PATCH 53/72] Added license to Makefile --- public/VectorPairHMM/src/main/c++/Makefile | 40 +++++++++--------- .../utils/pairhmm/libVectorLoglessPairHMM.so | Bin 443803 -> 443803 bytes 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/public/VectorPairHMM/src/main/c++/Makefile b/public/VectorPairHMM/src/main/c++/Makefile index 7296b7026..354bca0bb 100644 --- a/public/VectorPairHMM/src/main/c++/Makefile +++ b/public/VectorPairHMM/src/main/c++/Makefile @@ -1,26 +1,26 @@ -/*Copyright (c) 2012 The Broad Institute +#Copyright (c) 2012 The Broad Institute -*Permission is hereby granted, free of charge, to any person -*obtaining a copy of this software and associated documentation -*files (the "Software"), to deal in the Software without -*restriction, including without limitation the rights to use, -*copy, modify, merge, publish, distribute, sublicense, and/or sell -*copies of the Software, and to permit persons to whom the -*Software is furnished to do so, subject to the following -*conditions: +#Permission is hereby granted, free of charge, to any person +#obtaining a copy of this software and associated documentation +#files (the "Software"), to deal in the Software without +#restriction, including without limitation the rights to use, +#copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the +#Software is furnished to do so, subject to the following +#conditions: -*The above copyright notice and this permission notice shall be -*included in all copies or substantial portions of the Software. +#The above copyright notice and this permission notice shall be +#included in all copies or substantial portions of the Software. -*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR -*THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +#THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# #OMPCFLAGS=-fopenmp diff --git a/public/sting-utils/src/main/resources/org/broadinstitute/sting/utils/pairhmm/libVectorLoglessPairHMM.so b/public/sting-utils/src/main/resources/org/broadinstitute/sting/utils/pairhmm/libVectorLoglessPairHMM.so index 17cec8be400bc0847030f66d645762ac3c3e3ea2..2620557f0cf685ba9e238d5db4bded7d2d28aad2 100644 GIT binary patch delta 239 zcmbQ;Ej_zidcy)nk&BPthfJL^C+(B@r=;%Xi7wll3wSqgVdP&N=fuFk;L*!#WW&Vp zBJ}3}{~o=z|LmB6LZ+8(nHYB2L8OI2(q<6pQcByBNS zaj6`m>0}2WX*4+=Na|0n1Cn~1=Pq?HVN98P@1{0m=4P&28LW(IlZ)>+Gl?^9{&9aZ zFXPDK+FQftlRabv*qdm0As~l AXaE2J delta 239 zcmbQ;Ej_zidcy)nk*cUi$uB-=o*|pB)oW$n>%;6T>c5h_o Date: Tue, 25 Feb 2014 21:44:20 -0800 Subject: [PATCH 54/72] Added vectorized PairHMM implementation by Mohammad and Mustafa into the Maven build of GATK. C++ code has PAPI calls for reading hardware counters Followed Khalid's suggestion for packing libVectorLoglessCaching into the jar file with Maven Native library part of git repo 1. Renamed directory structure from public/c++/VectorPairHMM to public/VectorPairHMM/src/main/c++ as per Khalid's suggestion 2. Use java.home in public/VectorPairHMM/pom.xml to pass environment variable JRE_HOME to the make process. This is needed because the Makefile needs to compile JNI code with the flag -I/../include (among others). Assuming that the Maven build process uses a JDK (and not just a JRE), the variable java.home points to the JRE inside maven. 3. Dropped all pretense at cross-platform compatibility. Removed Mac profile from pom.xml for VectorPairHMM Moved JNI_README 1. Added the catch UnsatisfiedLinkError exception in PairHMMLikelihoodCalculationEngine.java to fall back to LOGLESS_CACHING in case the native library could not be loaded. Made VECTOR_LOGLESS_CACHING as the default implementation. 2. Updated the README with Mauricio's comments 3. baseline.cc is used within the library - if the machine supports neither AVX nor SSE4.1, the native library falls back to un-vectorized C++ in baseline.cc. 4. pairhmm-1-base.cc: This is not part of the library, but is being heavily used for debugging/profiling. Can I request that we keep it there for now? In the next release, we can delete it from the repository. 5. I agree with Mauricio about the ifdefs. I am sure you already know, but just to reassure you the debug code is not compiled into the library (because of the ifdefs) and will not affect performance. 1. Changed logger.info to logger.warn in PairHMMLikelihoodCalculationEngine.java 2. Committing the right set of files after rebase Added public license text to all C++ files Added license to Makefile Add package info to Sandbox.java --- .gitignore | 14 - ...GraphBasedLikelihoodCalculationEngine.java | 4 + .../haplotypecaller/HaplotypeCaller.java | 4 +- .../PairHMMLikelihoodCalculationEngine.java | 65 ++-- .../RandomLikelihoodCalculationEngine.java | 5 + .../utils/pairhmm/DebugJNILoglessPairHMM.java | 141 ++++---- .../utils/pairhmm/JNILoglessPairHMM.java | 0 .../utils/pairhmm/VectorLoglessPairHMM.java | 9 +- public/VectorPairHMM/README.md | 71 ++++ public/VectorPairHMM/pom.xml | 119 +++++++ .../src/main/c++}/.gitignore | 2 + .../src/main/c++}/LoadTimeInitializer.cc | 36 +- .../src/main/c++}/LoadTimeInitializer.h | 33 ++ .../src/main/c++}/Makefile | 37 +- .../src/main/c++}/Sandbox.cc | 29 +- .../src/main/c++}/Sandbox.h | 25 ++ .../src/main/c++}/Sandbox.java | 27 ++ .../Sandbox_JNIHaplotypeDataHolderClass.h | 0 .../c++}/Sandbox_JNIReadDataHolderClass.h | 0 .../main/c++/avx_function_instantiations.cc | 44 +++ .../src/main/c++}/baseline.cc | 56 ++- .../src/main/c++}/define-double.h | 25 ++ .../src/main/c++}/define-float.h | 25 ++ .../src/main/c++}/define-sse-double.h | 25 ++ .../src/main/c++}/define-sse-float.h | 25 ++ .../src/main/c++}/headers.h | 25 ++ .../VectorPairHMM/src/main/c++/jni_common.h | 58 ++++ .../src/main/c++}/jnidebug.h | 25 ++ ...ng_utils_pairhmm_DebugJNILoglessPairHMM.cc | 25 ++ ...ing_utils_pairhmm_DebugJNILoglessPairHMM.h | 25 ++ ...ting_utils_pairhmm_VectorLoglessPairHMM.cc | 83 ++++- ...sting_utils_pairhmm_VectorLoglessPairHMM.h | 25 ++ .../src/main/c++/pairhmm-1-base.cc | 70 ++++ .../src/main/c++}/pairhmm-template-kernel.cc | 28 +- .../src/main/c++}/pairhmm-template-main.cc | 25 ++ .../src/main/c++}/run.sh | 5 +- .../src/main/c++}/shift_template.c | 25 ++ .../main/c++/sse_function_instantiations.cc | 43 +++ public/VectorPairHMM/src/main/c++/template.h | 320 ++++++++++++++++++ .../src/main/c++}/utils.cc | 123 +++++-- .../src/main/c++}/utils.h | 32 ++ .../VectorPairHMM/src/main/c++/vector_defs.h | 55 +++ .../main/c++}/vector_function_prototypes.h | 25 ++ public/c++/VectorPairHMM/JNI_README | 41 --- .../avx_function_instantiations.cc | 19 -- public/c++/VectorPairHMM/jni_common.h | 33 -- public/c++/VectorPairHMM/pairhmm-1-base.cc | 45 --- .../sse_function_instantiations.cc | 18 - public/c++/VectorPairHMM/template.h | 194 ----------- public/c++/VectorPairHMM/vector_defs.h | 30 -- public/sting-root/pom.xml | 6 +- .../utils/pairhmm/libVectorLoglessPairHMM.so | Bin 0 -> 443803 bytes 52 files changed, 1686 insertions(+), 538 deletions(-) rename protected/{java/src => gatk-protected/src/main/java}/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java (85%) rename protected/{java/src => gatk-protected/src/main/java}/org/broadinstitute/sting/utils/pairhmm/JNILoglessPairHMM.java (100%) rename protected/{java/src => gatk-protected/src/main/java}/org/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM.java (98%) create mode 100644 public/VectorPairHMM/README.md create mode 100644 public/VectorPairHMM/pom.xml rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/.gitignore (88%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/LoadTimeInitializer.cc (79%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/LoadTimeInitializer.h (58%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Makefile (66%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox.cc (68%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox.h (60%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox.java (89%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox_JNIHaplotypeDataHolderClass.h (100%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/Sandbox_JNIReadDataHolderClass.h (100%) create mode 100644 public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/baseline.cc (65%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/define-double.h (82%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/define-float.h (82%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/define-sse-double.h (77%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/define-sse-float.h (77%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/headers.h (56%) create mode 100644 public/VectorPairHMM/src/main/c++/jni_common.h rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/jnidebug.h (84%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc (85%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h (78%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc (84%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h (78%) create mode 100644 public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/pairhmm-template-kernel.cc (92%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/pairhmm-template-main.cc (70%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/run.sh (82%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/shift_template.c (71%) create mode 100644 public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc create mode 100644 public/VectorPairHMM/src/main/c++/template.h rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/utils.cc (70%) rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/utils.h (52%) create mode 100644 public/VectorPairHMM/src/main/c++/vector_defs.h rename public/{c++/VectorPairHMM => VectorPairHMM/src/main/c++}/vector_function_prototypes.h (71%) delete mode 100644 public/c++/VectorPairHMM/JNI_README delete mode 100644 public/c++/VectorPairHMM/avx_function_instantiations.cc delete mode 100644 public/c++/VectorPairHMM/jni_common.h delete mode 100644 public/c++/VectorPairHMM/pairhmm-1-base.cc delete mode 100644 public/c++/VectorPairHMM/sse_function_instantiations.cc delete mode 100644 public/c++/VectorPairHMM/template.h delete mode 100644 public/c++/VectorPairHMM/vector_defs.h create mode 100644 public/sting-utils/src/main/resources/org/broadinstitute/sting/utils/pairhmm/libVectorLoglessPairHMM.so diff --git a/.gitignore b/.gitignore index a2ff166e5..ac3b931eb 100644 --- a/.gitignore +++ b/.gitignore @@ -24,19 +24,5 @@ dump/ lib/ out/ /atlassian-ide-plugin.xml -kg_tmp/ -maven-ant-tasks-2.1.3.jar -null-sequenceGraph.25.0.0.raw_readthreading_graph.dot -null-sequenceGraph.25.0.1.cleaned_readthreading_graph.dot -null-sequenceGraph.25.0.1.initial_seqgraph.dot -null-sequenceGraph.3.0.0.raw_readthreading_graph.dot -null-sequenceGraph.3.0.1.cleaned_readthreading_graph.dot -null-sequenceGraph.3.0.1.initial_seqgraph.dot -org/ -package-list -resources/ -velocity.log -perf -verify maven-metadata-local.xml dependency-reduced-pom.xml diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GraphBasedLikelihoodCalculationEngine.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GraphBasedLikelihoodCalculationEngine.java index 8a35ccb05..8b37e265d 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GraphBasedLikelihoodCalculationEngine.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/GraphBasedLikelihoodCalculationEngine.java @@ -155,4 +155,8 @@ public class GraphBasedLikelihoodCalculationEngine implements LikelihoodCalculat } } } + + @Override + public void close() { + } } diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java index 74164415e..33dfa54ce 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/HaplotypeCaller.java @@ -433,7 +433,7 @@ public class HaplotypeCaller extends ActiveRegionWalker, In */ @Hidden @Argument(fullName = "pair_hmm_implementation", shortName = "pairHMM", doc = "The PairHMM implementation to use for genotype likelihood calculations", required = false) - public PairHMM.HMM_IMPLEMENTATION pairHMM = PairHMM.HMM_IMPLEMENTATION.LOGLESS_CACHING; + public PairHMM.HMM_IMPLEMENTATION pairHMM = PairHMM.HMM_IMPLEMENTATION.VECTOR_LOGLESS_CACHING; @Hidden @Argument(fullName="keepRG", shortName="keepRG", doc="Only use read from this read group when making calls (but use all reads to build the assembly)", required = false) @@ -1051,7 +1051,7 @@ public class HaplotypeCaller extends ActiveRegionWalker, In referenceConfidenceModel.close(); //TODO remove the need to call close here for debugging, the likelihood output stream should be managed //TODO (open & close) at the walker, not the engine. - likelihoodCalculationEngine.close(); + likelihoodCalculationEngine.close(); logger.info("Ran local assembly on " + result + " active regions"); } diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java index 9de532dcf..66755542f 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/PairHMMLikelihoodCalculationEngine.java @@ -80,29 +80,37 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation private final boolean noFpga; private final ThreadLocal pairHMMThreadLocal = new ThreadLocal() { - @Override - protected PairHMM initialValue() { - switch (hmmType) { - case EXACT: return new Log10PairHMM(true); - case ORIGINAL: return new Log10PairHMM(false); - case LOGLESS_CACHING: - if (noFpga || !CnyPairHMM.isAvailable()) - return new LoglessPairHMM(); - else - return new CnyPairHMM(); - case VECTOR_LOGLESS_CACHING: - return new VectorLoglessPairHMM(); - case DEBUG_VECTOR_LOGLESS_CACHING: - return new DebugJNILoglessPairHMM(PairHMM.HMM_IMPLEMENTATION.VECTOR_LOGLESS_CACHING); - case ARRAY_LOGLESS: - if (noFpga || !CnyPairHMM.isAvailable()) - return new ArrayLoglessPairHMM(); - else - return new CnyPairHMM(); - default: - throw new UserException.BadArgumentValue("pairHMM", "Specified pairHMM implementation is unrecognized or incompatible with the HaplotypeCaller. Acceptable options are ORIGINAL, EXACT, CACHING, LOGLESS_CACHING, and ARRAY_LOGLESS."); - } - } + @Override + protected PairHMM initialValue() { + switch (hmmType) { + case EXACT: return new Log10PairHMM(true); + case ORIGINAL: return new Log10PairHMM(false); + case LOGLESS_CACHING: + if (noFpga || !CnyPairHMM.isAvailable()) + return new LoglessPairHMM(); + else + return new CnyPairHMM(); + case VECTOR_LOGLESS_CACHING: + try + { + return new VectorLoglessPairHMM(); + } + catch(UnsatisfiedLinkError ule) + { + logger.warn("Failed to load native library for VectorLoglessPairHMM - using Java implementation of LOGLESS_CACHING"); + return new LoglessPairHMM(); + } + case DEBUG_VECTOR_LOGLESS_CACHING: + return new DebugJNILoglessPairHMM(PairHMM.HMM_IMPLEMENTATION.VECTOR_LOGLESS_CACHING); + case ARRAY_LOGLESS: + if (noFpga || !CnyPairHMM.isAvailable()) + return new ArrayLoglessPairHMM(); + else + return new CnyPairHMM(); + default: + throw new UserException.BadArgumentValue("pairHMM", "Specified pairHMM implementation is unrecognized or incompatible with the HaplotypeCaller. Acceptable options are ORIGINAL, EXACT, CACHING, LOGLESS_CACHING, and ARRAY_LOGLESS."); + } + } }; // Attempted to do as below, to avoid calling pairHMMThreadLocal.get() later on, but it resulted in a NullPointerException // private final PairHMM pairHMM = pairHMMThreadLocal.get(); @@ -169,9 +177,10 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation @Override public void close() { if ( likelihoodsStream != null ) likelihoodsStream.close(); - pairHMMThreadLocal.get().close(); + pairHMMThreadLocal.get().close(); } + private void writeDebugLikelihoods(final GATKSAMRecord processedRead, final Haplotype haplotype, final double log10l){ if ( WRITE_LIKELIHOODS_TO_FILE ) { likelihoodsStream.printf("%s %s %s %s %s %s %f%n", @@ -338,7 +347,7 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation private void finalizePairHMM() { - pairHMMThreadLocal.get().finalizeRegion(); + pairHMMThreadLocal.get().finalizeRegion(); } @@ -358,9 +367,9 @@ public class PairHMMLikelihoodCalculationEngine implements LikelihoodCalculation map.filterPoorlyModelledReads(EXPECTED_ERROR_RATE_PER_BASE); stratifiedReadMap.put(sampleEntry.getKey(), map); } - - //Used mostly by the JNI implementation(s) to free arrays - finalizePairHMM(); + + //Used mostly by the JNI implementation(s) to free arrays + finalizePairHMM(); return stratifiedReadMap; } diff --git a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/RandomLikelihoodCalculationEngine.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/RandomLikelihoodCalculationEngine.java index b8dba7b86..d5d424ca9 100644 --- a/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/RandomLikelihoodCalculationEngine.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/gatk/walkers/haplotypecaller/RandomLikelihoodCalculationEngine.java @@ -79,4 +79,9 @@ public class RandomLikelihoodCalculationEngine implements LikelihoodCalculationE return result; } + + @Override + public void close() { + } + } diff --git a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java similarity index 85% rename from protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java rename to protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java index 7e50f0c30..ea93ebe4a 100644 --- a/protected/java/src/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java +++ b/protected/gatk-protected/src/main/java/org/broadinstitute/sting/utils/pairhmm/DebugJNILoglessPairHMM.java @@ -55,7 +55,7 @@ import org.broadinstitute.sting.utils.haplotype.Haplotype; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.broadinstitute.variant.variantcontext.Allele; import org.broadinstitute.sting.utils.exceptions.UserException; - +import static org.broadinstitute.sting.utils.pairhmm.PairHMMModel.*; import java.util.List; import java.util.Map; @@ -75,8 +75,9 @@ import java.io.IOException; */ public class DebugJNILoglessPairHMM extends LoglessPairHMM { + private static final boolean dumpSandboxOnly = false; //simulates ifdef private static final boolean debug = false; //simulates ifdef - private static final boolean verify = debug || true; //simulates ifdef + private static final boolean verify = !dumpSandboxOnly && (debug || true); //simulates ifdef private static final boolean debug0_1 = false; //simulates ifdef private static final boolean debug1 = false; //simulates ifdef private static final boolean debug2 = false; @@ -134,9 +135,11 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { public void initialize( final List haplotypes, final Map> perSampleReadList, final int readMaxLength, final int haplotypeMaxLength ) { if(verify) + { super.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); - jniPairHMM.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); - haplotypeToHaplotypeListIdxMap = jniPairHMM.getHaplotypeToHaplotypeListIdxMap(); + jniPairHMM.initialize(haplotypes, perSampleReadList, readMaxLength, haplotypeMaxLength); + haplotypeToHaplotypeListIdxMap = jniPairHMM.getHaplotypeToHaplotypeListIdxMap(); + } } /** @@ -145,7 +148,8 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { @Override public void finalizeRegion() { - jniPairHMM.finalizeRegion(); + if(!dumpSandboxOnly) + jniPairHMM.finalizeRegion(); } /** @@ -204,46 +208,61 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { ++idx; } } - jniPairHMM.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); - double[] likelihoodArray = jniPairHMM.getLikelihoodArray(); - //to compare values - final PerReadAlleleLikelihoodMap likelihoodMap = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + double[] likelihoodArray = null; + PerReadAlleleLikelihoodMap likelihoodMap = null; if(verify) { - //re-order values in likelihoodArray - double[] tmpArray = new double[numHaplotypes]; - idx = 0; - int idxInsideHaplotypeList = 0; - int readIdx = 0; - for(GATKSAMRecord read : reads) + jniPairHMM.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + likelihoodArray = jniPairHMM.getLikelihoodArray(); + //to compare values + likelihoodMap = super.computeLikelihoods(reads, alleleHaplotypeMap, GCPArrayMap); + } + else + { + likelihoodMap = new PerReadAlleleLikelihoodMap(); + likelihoodArray = new double[numTestcases]; + for(int i=0;i currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always + //re-order values in likelihoodArray + double[] tmpArray = new double[numHaplotypes]; + idx = 0; + int idxInsideHaplotypeList = 0; + int readIdx = 0; + for(GATKSAMRecord read : reads) { - idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); - likelihoodArray[idx] = tmpArray[idxInsideHaplotypeList]; - ++idx; + for(int j=0;j currEntry : alleleHaplotypeMap.entrySet())//order is important - access in same order always + { + idxInsideHaplotypeList = haplotypeToHaplotypeListIdxMap.get(currEntry.getValue()); + likelihoodArray[idx] = tmpArray[idxInsideHaplotypeList]; + ++idx; + } + readIdx += numHaplotypes; } - readIdx += numHaplotypes; - } - //for floating point values, no exact equality - //check whether numbers are close in terms of abs_error or relative_error - //For very large values, relative_error is relevant - //For very small values, abs_error is relevant - boolean toDump = false; - for(int i=0;i 1e-5 && relative_error > 1e-5) + //for floating point values, no exact equality + //check whether numbers are close in terms of abs_error or relative_error + //For very large values, relative_error is relevant + //For very small values, abs_error is relevant + for(int i=0;i 1e-5 && relative_error > 1e-5) + { + toDump = true; + break; + } } } //if numbers are not close, then dump out the data that produced the inconsistency @@ -251,24 +270,40 @@ public class DebugJNILoglessPairHMM extends LoglessPairHMM { { idx = 0; System.out.println("Dump : Java numReads "+readListSize+" numHaplotypes "+numHaplotypes); + boolean firstLine = true; for(GATKSAMRecord read : reads) { byte [] overallGCP = GCPArrayMap.get(read); + byte[] tmpByteArray = new byte[read.getReadBases().length]; for (Map.Entry currEntry : alleleHaplotypeMap.entrySet()) //order is important - access in same order always { byte[] haplotypeBases = currEntry.getValue().getBases(); debugDump("debug_dump.txt",new String(haplotypeBases)+" ",true); debugDump("debug_dump.txt",new String(read.getReadBases())+" ",true); for(int k=0;k + + 4.0.0 + + + org.broadinstitute.sting + sting-root + 2.8-SNAPSHOT + ../../public/sting-root + + + VectorPairHMM + pom + Vectorized PairHMM native libraries + + Builds a GNU/Linux x86_64 library of VectorPairHMM using icc (Intel C++ compiler). During install, copies it into sting-utils. Neither tested nor expected to work on any other platform. + + + UTF-8 + ${sourceEncoding} + ${sourceEncoding} + ${project.basedir}/../.. + ${sting.basedir}/public/sting-utils + + ${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + display-info + + validate + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + + exec + + compile + + make + src/main/c++ + + ${java.home} + ${project.build.directory} + + + + + + + + + org.apache.maven.plugins + maven-install-plugin + + true + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + default-install + + copy-resources + + install + + ${pairhmm.resources.directory} + + + ${project.build.directory} + + **/* + + + + + + + + + + + com.google.code.sortpom + maven-sortpom-plugin + + false + custom_1 + \n + ${sourceEncoding} + true + scope + 4 + false + + + + + diff --git a/public/c++/VectorPairHMM/.gitignore b/public/VectorPairHMM/src/main/c++/.gitignore similarity index 88% rename from public/c++/VectorPairHMM/.gitignore rename to public/VectorPairHMM/src/main/c++/.gitignore index b034f9461..d791ffd80 100644 --- a/public/c++/VectorPairHMM/.gitignore +++ b/public/VectorPairHMM/src/main/c++/.gitignore @@ -12,3 +12,5 @@ reformat subdir_checkout.sh avx/ sse/ +triplicate.sh + diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.cc b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc similarity index 79% rename from public/c++/VectorPairHMM/LoadTimeInitializer.cc rename to public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc index f9db149e6..0e3026f65 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.cc +++ b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "LoadTimeInitializer.h" #include "utils.h" using namespace std; @@ -14,7 +39,6 @@ char* LoadTimeInitializerStatsNames[] = "dummy" }; - LoadTimeInitializer g_load_time_initializer; LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loaded @@ -47,7 +71,17 @@ LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loa m_filename_to_fptr.clear(); m_written_files_set.clear(); + //Common buffer - 8MB + unsigned size = 1024*1024; + m_buffer = new uint64_t[size]; + m_buffer_size = size*sizeof(uint64_t); + initialize_function_pointers(); + + //Initialize static members of class + Context::initializeStaticMembers(); + Context::initializeStaticMembers(); + cout.flush(); } diff --git a/public/c++/VectorPairHMM/LoadTimeInitializer.h b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h similarity index 58% rename from public/c++/VectorPairHMM/LoadTimeInitializer.h rename to public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h index 0eb63849a..e7d9af626 100644 --- a/public/c++/VectorPairHMM/LoadTimeInitializer.h +++ b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifndef LOAD_TIME_INITIALIZER_H #define LOAD_TIME_INITIALIZER_H #include "headers.h" @@ -22,6 +47,10 @@ class LoadTimeInitializer { public: LoadTimeInitializer(); //will be called when library is loaded + ~LoadTimeInitializer() + { + delete m_buffer; + } void print_profiling(); void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); void debug_close(); @@ -43,6 +72,8 @@ class LoadTimeInitializer uint64_t m_data_transfer_time; //bytes copied uint64_t m_bytes_copied; + unsigned get_buffer_size() { return m_buffer_size; } + char* get_buffer() { return (char*)m_buffer; } private: std::map m_filename_to_fptr; std::set m_written_files_set; @@ -52,6 +83,8 @@ class LoadTimeInitializer double m_sum_square_stats[TOTAL_NUMBER_STATS]; uint64_t m_min_stats[TOTAL_NUMBER_STATS]; uint64_t m_max_stats[TOTAL_NUMBER_STATS]; + unsigned m_buffer_size; + uint64_t* m_buffer; }; extern LoadTimeInitializer g_load_time_initializer; diff --git a/public/c++/VectorPairHMM/Makefile b/public/VectorPairHMM/src/main/c++/Makefile similarity index 66% rename from public/c++/VectorPairHMM/Makefile rename to public/VectorPairHMM/src/main/c++/Makefile index 269cecbd2..354bca0bb 100644 --- a/public/c++/VectorPairHMM/Makefile +++ b/public/VectorPairHMM/src/main/c++/Makefile @@ -1,11 +1,36 @@ +#Copyright (c) 2012 The Broad Institute + +#Permission is hereby granted, free of charge, to any person +#obtaining a copy of this software and associated documentation +#files (the "Software"), to deal in the Software without +#restriction, including without limitation the rights to use, +#copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the +#Software is furnished to do so, subject to the following +#conditions: + +#The above copyright notice and this permission notice shall be +#included in all copies or substantial portions of the Software. + +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +#THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + + #OMPCFLAGS=-fopenmp #OMPLFLAGS=-fopenmp #-openmp-link static #CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas #CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas -JAVA_ROOT=/opt/jdk1.7.0_25/ -JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JAVA_ROOT}/include -I${JAVA_ROOT}/include/linux +JRE_HOME?=/opt/jdk1.7.0_25/jre +JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JRE_HOME}/../include -I${JRE_HOME}/../include/linux COMMON_COMPILATION_FLAGS=$(JNI_COMPILATION_FLAGS) -O3 -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas CC=icc @@ -58,7 +83,7 @@ $(AVX_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xAVX $(SSE_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xSSE4.2 OBJECTS=$(NO_VECTOR_OBJECTS) $(AVX_OBJECTS) $(SSE_OBJECTS) -all: $(BIN) Sandbox.class +all: $(BIN) Sandbox.class copied_lib -include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) @@ -79,5 +104,11 @@ $(OBJECTS): %.o: %.cc Sandbox.class: Sandbox.java javac Sandbox.java +copied_lib: libVectorLoglessPairHMM.so +ifdef OUTPUT_DIR + mkdir -p $(OUTPUT_DIR) + rsync -a libVectorLoglessPairHMM.so $(OUTPUT_DIR)/ +endif + clean: rm -rf $(BIN) *.o $(DEPDIR) *.class diff --git a/public/c++/VectorPairHMM/Sandbox.cc b/public/VectorPairHMM/src/main/c++/Sandbox.cc similarity index 68% rename from public/c++/VectorPairHMM/Sandbox.cc rename to public/VectorPairHMM/src/main/c++/Sandbox.cc index 7c10e0620..985b19ae9 100644 --- a/public/c++/VectorPairHMM/Sandbox.cc +++ b/public/VectorPairHMM/src/main/c++/Sandbox.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "Sandbox.h" #include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" #include "utils.h" @@ -73,7 +98,9 @@ JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative (JNIEnv* env, jobject thisObject, jstring fileNameString) { const char* fileName = env->GetStringUTFChars(fileNameString, 0); - do_compute((char*)fileName); + char local_array[800]; + strncpy(local_array, fileName, 200); env->ReleaseStringUTFChars(fileNameString, fileName); + do_compute(local_array, true, 10000, false); } diff --git a/public/c++/VectorPairHMM/Sandbox.h b/public/VectorPairHMM/src/main/c++/Sandbox.h similarity index 60% rename from public/c++/VectorPairHMM/Sandbox.h rename to public/VectorPairHMM/src/main/c++/Sandbox.h index 4ac1ea24c..486a1c095 100644 --- a/public/c++/VectorPairHMM/Sandbox.h +++ b/public/VectorPairHMM/src/main/c++/Sandbox.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + /* DO NOT EDIT THIS FILE - it is machine generated */ #include /* Header for class Sandbox */ diff --git a/public/c++/VectorPairHMM/Sandbox.java b/public/VectorPairHMM/src/main/c++/Sandbox.java similarity index 89% rename from public/c++/VectorPairHMM/Sandbox.java rename to public/VectorPairHMM/src/main/c++/Sandbox.java index 81a57c0a0..d6b7c2eae 100644 --- a/public/c++/VectorPairHMM/Sandbox.java +++ b/public/VectorPairHMM/src/main/c++/Sandbox.java @@ -1,3 +1,30 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.broadinstitute.sting.utils.vectorpairhmm; + import java.util.List; import java.util.LinkedList; import java.util.Map; diff --git a/public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h similarity index 100% rename from public/c++/VectorPairHMM/Sandbox_JNIHaplotypeDataHolderClass.h rename to public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h diff --git a/public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h similarity index 100% rename from public/c++/VectorPairHMM/Sandbox_JNIReadDataHolderClass.h rename to public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h diff --git a/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc new file mode 100644 index 000000000..6d90d5070 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc @@ -0,0 +1,44 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "template.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_SSE + +#define SIMD_ENGINE avx +#define SIMD_ENGINE_AVX + +#include "define-float.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +#include "define-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +template double compute_full_prob_avxd(testcase* tc, double* nextlog); +template float compute_full_prob_avxs(testcase* tc, float* nextlog); + diff --git a/public/c++/VectorPairHMM/baseline.cc b/public/VectorPairHMM/src/main/c++/baseline.cc similarity index 65% rename from public/c++/VectorPairHMM/baseline.cc rename to public/VectorPairHMM/src/main/c++/baseline.cc index 232df83f2..d6085e661 100644 --- a/public/c++/VectorPairHMM/baseline.cc +++ b/public/VectorPairHMM/src/main/c++/baseline.cc @@ -1,7 +1,34 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "template.h" #include "utils.h" +#include "LoadTimeInitializer.h" using namespace std; + template NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) { @@ -10,12 +37,28 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) int COLS = tc->haplen + 1; Context ctx; - + //#define USE_STACK_ALLOCATION 1 +#ifdef USE_STACK_ALLOCATION + NUMBER M[ROWS][COLS]; + NUMBER X[ROWS][COLS]; + NUMBER Y[ROWS][COLS]; + NUMBER p[ROWS][6]; +#else //allocate on heap in way that simulates a 2D array. Having a 2D array instead of //a straightforward array of pointers ensures that all data lies 'close' in memory, increasing //the chance of being stored together in the cache. Also, prefetchers can learn memory access //patterns for 2D arrays, not possible for array of pointers + //bool locally_allocated = false; + //NUMBER* common_buffer = 0; NUMBER* common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //unsigned curr_size = sizeof(NUMBER)*(3*ROWS*COLS + ROWS*6); + //if(true) + //{ + //common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //locally_allocated = true; + //} + //else + //common_buffer = (NUMBER*)(g_load_time_initializer.get_buffer()); //pointers to within the allocated buffer NUMBER** common_pointer_buffer = new NUMBER*[4*ROWS]; NUMBER* ptr = common_buffer; @@ -25,14 +68,11 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) for(;i<4*ROWS;++i, ptr+=6) common_pointer_buffer[i] = ptr; - //NUMBER M[ROWS][COLS]; - //NUMBER X[ROWS][COLS]; - //NUMBER Y[ROWS][COLS]; - //NUMBER p[ROWS][6]; NUMBER** M = common_pointer_buffer; NUMBER** X = M + ROWS; NUMBER** Y = X + ROWS; NUMBER** p = Y + ROWS; +#endif p[0][MM] = ctx._(0.0); @@ -47,7 +87,8 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) int _i = tc->i[r-1] & 127; int _d = tc->d[r-1] & 127; int _c = tc->c[r-1] & 127; - p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + //p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(p[r][MM], _i, _d); p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; p[r][MX] = ctx.ph2pr[_i]; p[r][XX] = ctx.ph2pr[_c]; @@ -111,8 +152,11 @@ NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) if (before_last_log != NULL) *before_last_log = result; +#ifndef USE_STACK_ALLOCATION delete common_pointer_buffer; + //if(locally_allocated) delete common_buffer; +#endif return result; //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; diff --git a/public/c++/VectorPairHMM/define-double.h b/public/VectorPairHMM/src/main/c++/define-double.h similarity index 82% rename from public/c++/VectorPairHMM/define-double.h rename to public/VectorPairHMM/src/main/c++/define-double.h index 83589a13d..2067d369c 100644 --- a/public/c++/VectorPairHMM/define-double.h +++ b/public/VectorPairHMM/src/main/c++/define-double.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include #ifdef PRECISION diff --git a/public/c++/VectorPairHMM/define-float.h b/public/VectorPairHMM/src/main/c++/define-float.h similarity index 82% rename from public/c++/VectorPairHMM/define-float.h rename to public/VectorPairHMM/src/main/c++/define-float.h index 87b2b01f3..318f78280 100644 --- a/public/c++/VectorPairHMM/define-float.h +++ b/public/VectorPairHMM/src/main/c++/define-float.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include #ifdef PRECISION diff --git a/public/c++/VectorPairHMM/define-sse-double.h b/public/VectorPairHMM/src/main/c++/define-sse-double.h similarity index 77% rename from public/c++/VectorPairHMM/define-sse-double.h rename to public/VectorPairHMM/src/main/c++/define-sse-double.h index d781d55f3..2d271a854 100644 --- a/public/c++/VectorPairHMM/define-sse-double.h +++ b/public/VectorPairHMM/src/main/c++/define-sse-double.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifdef PRECISION #undef PRECISION #undef MAIN_TYPE diff --git a/public/c++/VectorPairHMM/define-sse-float.h b/public/VectorPairHMM/src/main/c++/define-sse-float.h similarity index 77% rename from public/c++/VectorPairHMM/define-sse-float.h rename to public/VectorPairHMM/src/main/c++/define-sse-float.h index 7516e6dbf..20af947dd 100644 --- a/public/c++/VectorPairHMM/define-sse-float.h +++ b/public/VectorPairHMM/src/main/c++/define-sse-float.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifdef PRECISION #undef PRECISION #undef MAIN_TYPE diff --git a/public/c++/VectorPairHMM/headers.h b/public/VectorPairHMM/src/main/c++/headers.h similarity index 56% rename from public/c++/VectorPairHMM/headers.h rename to public/VectorPairHMM/src/main/c++/headers.h index 48bd4d836..4a0d89b57 100644 --- a/public/c++/VectorPairHMM/headers.h +++ b/public/VectorPairHMM/src/main/c++/headers.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifndef COMMON_HEADERS_H #define COMMON_HEADERS_H diff --git a/public/VectorPairHMM/src/main/c++/jni_common.h b/public/VectorPairHMM/src/main/c++/jni_common.h new file mode 100644 index 000000000..23c323246 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/jni_common.h @@ -0,0 +1,58 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef JNI_COMMON_H +#define JNI_COMMON_H + +#include +/*#define ENABLE_ASSERTIONS 1*/ +#define DO_PROFILING 1 +/*#define DEBUG 1*/ +/*#define DEBUG0_1 1*/ +/*#define DEBUG3 1*/ +/*#define DUMP_TO_SANDBOX 1*/ + + +#define DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY 1 + +#ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY +//Gets direct access to Java arrays +#define GET_BYTE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical + +#else +//Likely makes copy of Java arrays to JNI C++ space +#define GET_BYTE_ARRAY_ELEMENTS env->GetByteArrayElements +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleaseByteArrayElements +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetDoubleArrayElements +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleaseDoubleArrayElements + +#endif //ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY + +#endif //ifndef JNI_COMMON_H diff --git a/public/c++/VectorPairHMM/jnidebug.h b/public/VectorPairHMM/src/main/c++/jnidebug.h similarity index 84% rename from public/c++/VectorPairHMM/jnidebug.h rename to public/VectorPairHMM/src/main/c++/jnidebug.h index 7002b3637..7fcab2a51 100644 --- a/public/c++/VectorPairHMM/jnidebug.h +++ b/public/VectorPairHMM/src/main/c++/jnidebug.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifndef JNI_DEBUG_H #define JNI_DEBUG_H diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc similarity index 85% rename from public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc rename to public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc index 98ebcde92..8a3f8b5bc 100644 --- a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "jni_common.h" #include "org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h" diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h similarity index 78% rename from public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h rename to public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h index 9e099dae0..18c2b0394 100644 --- a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + /* DO NOT EDIT THIS FILE - it is machine generated */ #include /* Header for class org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM */ diff --git a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc similarity index 84% rename from public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc rename to public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc index 0f5219dd4..0b54c8a81 100644 --- a/public/c++/VectorPairHMM/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "jni_common.h" #include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" @@ -98,6 +123,8 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLogless } } +//Create a vector of testcases for computation - copy the references to bytearrays read/readQuals etc into the appropriate +//testcase struct inline JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeTestcasesVector (JNIEnv* env, jint numReads, jint numHaplotypes, jobjectArray& readDataArray, vector > >& readBasesArrayVector, vector& tc_array) @@ -195,21 +222,26 @@ inline JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_Vector inline void compute_testcases(vector& tc_array, unsigned numTestCases, double* likelihoodDoubleArray, unsigned maxNumThreadsToUse) { -#pragma omp parallel for schedule (dynamic,10000) num_threads(maxNumThreadsToUse) - for(unsigned tc_idx=0;tc_idxGetArrayLength(likelihoodArray) == numTestCases); #endif +#ifdef DO_WARMUP //ignore - only for crazy profiling + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + for(unsigned i=0;iGetArrayLength(haplotypeBasesArrayVector[i].first); + for(unsigned j=0;jGetArrayLength(readBasesArrayVector[i][j].first); + for(unsigned k=0;k /* Header for class org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM */ diff --git a/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc new file mode 100644 index 000000000..7ff219b88 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc @@ -0,0 +1,70 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +//#define DEBUG 1 +//#define DEBUG0_1 1 +//#define DEBUG3 1 +#include "headers.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +using namespace std; + +int main(int argc, char** argv) +{ +#define BATCH_SIZE 10000 + if(argc < 2) + { + cerr << "Needs path to input file as argument\n"; + exit(0); + } + bool use_old_read_testcase = false; + if(argc >= 3 && string(argv[2]) == "1") + use_old_read_testcase = true; + unsigned chunk_size = 10000; + bool do_check = true; + uint64_t mask = ~(0ull); + for(int i=3;i @@ -116,7 +141,8 @@ template void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECIS int _d = tc->d[r-1] & 127; int _c = tc->c[r-1] & 127; - *(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + //*(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(*(ptr_p_MM+r-1), _i, _d); *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; diff --git a/public/c++/VectorPairHMM/pairhmm-template-main.cc b/public/VectorPairHMM/src/main/c++/pairhmm-template-main.cc similarity index 70% rename from public/c++/VectorPairHMM/pairhmm-template-main.cc rename to public/VectorPairHMM/src/main/c++/pairhmm-template-main.cc index cbcb8556d..5d9a05830 100644 --- a/public/c++/VectorPairHMM/pairhmm-template-main.cc +++ b/public/VectorPairHMM/src/main/c++/pairhmm-template-main.cc @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "template.h" #include "vector_defs.h" diff --git a/public/c++/VectorPairHMM/run.sh b/public/VectorPairHMM/src/main/c++/run.sh similarity index 82% rename from public/c++/VectorPairHMM/run.sh rename to public/VectorPairHMM/src/main/c++/run.sh index a6930678b..e821497f1 100755 --- a/public/c++/VectorPairHMM/run.sh +++ b/public/VectorPairHMM/src/main/c++/run.sh @@ -7,8 +7,9 @@ then pair_hmm_implementation=$1; fi -#-Djava.library.path is needed if you are using JNI_LOGLESS_CACHING, else not needed -java -Djava.library.path=${GSA_ROOT_DIR}/public/c++/VectorPairHMM -jar $GSA_ROOT_DIR/dist/GenomeAnalysisTK.jar -T HaplotypeCaller \ +#-Djava.library.path is needed if you wish to override the default 'packed' library +#java -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ +java -Djava.library.path=${GSA_ROOT_DIR}/public/VectorPairHMM/src/main/c++ -jar $GSA_ROOT_DIR/target/GenomeAnalysisTK.jar -T HaplotypeCaller \ --dbsnp /data/broad/samples/joint_variant_calling/dbSNP/00-All.vcf \ -R /opt/Genomics/ohsu/dnapipeline/humanrefgenome/human_g1k_v37.fasta \ -I /data/simulated/sim1M_pairs_final.bam \ diff --git a/public/c++/VectorPairHMM/shift_template.c b/public/VectorPairHMM/src/main/c++/shift_template.c similarity index 71% rename from public/c++/VectorPairHMM/shift_template.c rename to public/VectorPairHMM/src/main/c++/shift_template.c index 9750e1d8b..c591c6882 100644 --- a/public/c++/VectorPairHMM/shift_template.c +++ b/public/VectorPairHMM/src/main/c++/shift_template.c @@ -1,3 +1,28 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #ifdef PRECISION #ifdef SIMD_ENGINE_AVX diff --git a/public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc b/public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc new file mode 100644 index 000000000..550272494 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/sse_function_instantiations.cc @@ -0,0 +1,43 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "template.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX + +#define SIMD_ENGINE sse +#define SIMD_ENGINE_SSE + +#include "define-sse-float.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +#include "define-sse-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +template double compute_full_prob_ssed(testcase* tc, double* nextlog); +template float compute_full_prob_sses(testcase* tc, float* nextlog); diff --git a/public/VectorPairHMM/src/main/c++/template.h b/public/VectorPairHMM/src/main/c++/template.h new file mode 100644 index 000000000..ce4dbfc86 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/template.h @@ -0,0 +1,320 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef TEMPLATES_H_ +#define TEMPLATES_H_ + +#include "headers.h" + +#define MM 0 +#define GapM 1 +#define MX 2 +#define XX 3 +#define MY 4 +#define YY 5 + +//#define MROWS 500 +//#define MCOLS 1000 + +#define CAT(X,Y) X####Y +#define CONCAT(X,Y) CAT(X,Y) + +#define ALIGNED __attribute__((aligned(32))) + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256 ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED float ALIGNED f[8]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_F ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m128 ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED float ALIGNED f[4]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_F128 ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128 vecf ; + uint32_t masks[4] ; +} MaskVec_F ; + +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint32_t masks[2] ; +} MaskVec_F128 ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128 ALIGNED f; +} ALIGNED IF_128f ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int ALIGNED i; + ALIGNED float ALIGNED f; +} ALIGNED IF_32 ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256d ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED double ALIGNED f[4]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_D ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m128d ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED double ALIGNED f[2]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_D128 ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128d vecf ; + uint64_t masks[2] ; +} MaskVec_D ; + +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint64_t masks[1] ; +} MaskVec_D128 ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128d ALIGNED f; +} ALIGNED IF_128d ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int64_t ALIGNED i; + ALIGNED double ALIGNED f; +} ALIGNED IF_64 ALIGNED; + + +#define MAX_QUAL 254 +#define MAX_JACOBIAN_TOLERANCE 8.0 +#define JACOBIAN_LOG_TABLE_STEP 0.0001 +#define JACOBIAN_LOG_TABLE_INV_STEP (1.0 / JACOBIAN_LOG_TABLE_STEP) +#define MAXN 70000 +#define LOG10_CACHE_SIZE (4*MAXN) // we need to be able to go up to 2*(2N) when calculating some of the coefficients +#define JACOBIAN_LOG_TABLE_SIZE ((int) (MAX_JACOBIAN_TOLERANCE / JACOBIAN_LOG_TABLE_STEP) + 1) + +template +struct ContextBase +{ + public: + NUMBER ph2pr[128]; + NUMBER INITIAL_CONSTANT; + NUMBER LOG10_INITIAL_CONSTANT; + NUMBER RESULT_THRESHOLD; + + static bool staticMembersInitializedFlag; + static NUMBER jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; + static NUMBER matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; + + static void initializeStaticMembers() + { + if(!staticMembersInitializedFlag) + { + //Order of calls important - Jacobian first, then MatchToMatch + initializeJacobianLogTable(); + initializeMatchToMatchProb(); + staticMembersInitializedFlag = true; + } + } + + static void deleteStaticMembers() + { + if(staticMembersInitializedFlag) + { + staticMembersInitializedFlag = false; + } + } + + //Called only once during library load - don't bother to optimize with single precision fp + static void initializeJacobianLogTable() + { + for (int k = 0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { + jacobianLogTable[k] = (NUMBER)(log10(1.0 + pow(10.0, -((double) k) * JACOBIAN_LOG_TABLE_STEP))); + } + } + + //Called only once per library load - don't bother optimizing with single fp + static void initializeMatchToMatchProb() + { + double LN10 = log(10); + double INV_LN10 = 1.0/LN10; + for (int i = 0, offset = 0; i <= MAX_QUAL; offset += ++i) + for (int j = 0; j <= i; j++) { + double log10Sum = approximateLog10SumLog10(-0.1*i, -0.1*j); + double matchToMatchLog10 = + log1p(-std::min(1.0,pow(10,log10Sum))) * INV_LN10; + matchToMatchProb[offset + j] = (NUMBER)(pow(10,matchToMatchLog10)); + } + } + //Called during computation - use single precision where possible + static int fastRound(NUMBER d) { + return (d > ((NUMBER)0.0)) ? (int) (d + ((NUMBER)0.5)) : (int) (d - ((NUMBER)0.5)); + } + //Called during computation - use single precision where possible + static NUMBER approximateLog10SumLog10(NUMBER small, NUMBER big) { + // make sure small is really the smaller value + if (small > big) { + NUMBER t = big; + big = small; + small = t; + } + + if (isinf(small) == -1 || isinf(big) == -1) + return big; + + NUMBER diff = big - small; + if (diff >= ((NUMBER)MAX_JACOBIAN_TOLERANCE)) + return big; + + // OK, so |y-x| < tol: we use the following identity then: + // we need to compute log10(10^x + 10^y) + // By Jacobian logarithm identity, this is equal to + // max(x,y) + log10(1+10^-abs(x-y)) + // we compute the second term as a table lookup with integer quantization + // we have pre-stored correction for 0,0.1,0.2,... 10.0 + int ind = fastRound((NUMBER)(diff * ((NUMBER)JACOBIAN_LOG_TABLE_INV_STEP))); // hard rounding + return big + jacobianLogTable[ind]; + } +}; + +template +struct Context : public ContextBase +{}; + +template<> +struct Context : public ContextBase +{ + Context():ContextBase() + { + for (int x = 0; x < 128; x++) + ph2pr[x] = pow(10.0, -((double)x) / 10.0); + + INITIAL_CONSTANT = ldexp(1.0, 1020.0); + LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); + RESULT_THRESHOLD = 0.0; + } + + double LOG10(double v){ return log10(v); } + inline double POW(double b, double e) { return pow(b,e); } + + static double _(double n){ return n; } + static double _(float n){ return ((double) n); } +}; + +template<> +struct Context : public ContextBase +{ + Context() : ContextBase() + { + for (int x = 0; x < 128; x++) + { + ph2pr[x] = powf(10.f, -((float)x) / 10.f); + } + + INITIAL_CONSTANT = ldexpf(1.f, 120.f); + LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); + RESULT_THRESHOLD = ldexpf(1.f, -110.f); + } + + float LOG10(float v){ return log10f(v); } + inline float POW(float b, float e) { return powf(b,e); } + + static float _(double n){ return ((float) n); } + static float _(float n){ return n; } +}; + +#define SET_MATCH_TO_MATCH_PROB(output, insQual, delQual) \ +{ \ + int minQual = delQual; \ + int maxQual = insQual; \ + if (insQual <= delQual) \ + { \ + minQual = insQual; \ + maxQual = delQual; \ + } \ + (output) = (MAX_QUAL < maxQual) ? \ + ((NUMBER)1.0) - ctx.POW(((NUMBER)10), ctx.approximateLog10SumLog10(((NUMBER)-0.1)*minQual, ((NUMBER)-0.1)*maxQual)) \ + : ctx.matchToMatchProb[((maxQual * (maxQual + 1)) >> 1) + minQual]; \ +} + +typedef struct +{ + int rslen, haplen; + /*int *q, *i, *d, *c;*/ + /*int q[MROWS], i[MROWS], d[MROWS], c[MROWS];*/ + char *q, *i, *d, *c; + char *hap, *rs; + int *ihap; + int *irs; +} testcase; + +int normalize(char c); +int read_testcase(testcase *tc, FILE* ifp=0); + + +#define MIN_ACCEPTED 1e-28f +#define NUM_DISTINCT_CHARS 5 +#define AMBIG_CHAR 4 + +class ConvertChar { + + static uint8_t conversionTable[255] ; + +public: + + static void init() { + assert (NUM_DISTINCT_CHARS == 5) ; + assert (AMBIG_CHAR == 4) ; + + conversionTable['A'] = 0 ; + conversionTable['C'] = 1 ; + conversionTable['T'] = 2 ; + conversionTable['G'] = 3 ; + conversionTable['N'] = 4 ; + } + + static inline uint8_t get(uint8_t input) { + return conversionTable[input] ; + } + +}; + + +#endif + + diff --git a/public/c++/VectorPairHMM/utils.cc b/public/VectorPairHMM/src/main/c++/utils.cc similarity index 70% rename from public/c++/VectorPairHMM/utils.cc rename to public/VectorPairHMM/src/main/c++/utils.cc index 76cee7327..9f83cffa2 100644 --- a/public/c++/VectorPairHMM/utils.cc +++ b/public/VectorPairHMM/src/main/c++/utils.cc @@ -1,14 +1,48 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + #include "headers.h" #include "template.h" #include "utils.h" #include "vector_defs.h" #include "LoadTimeInitializer.h" +using namespace std; +//static members from ConvertChar uint8_t ConvertChar::conversionTable[255]; +//Global function pointers in utils.h float (*g_compute_full_prob_float)(testcase *tc, float* before_last_log) = 0; double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log) = 0; +//Static members in ContextBase +bool ContextBase::staticMembersInitializedFlag = false; +double ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +double ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; +bool ContextBase::staticMembersInitializedFlag = false; +float ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +float ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; -using namespace std; bool is_avx_supported() { @@ -61,6 +95,7 @@ uint64_t get_machine_capabilities() void initialize_function_pointers(uint64_t mask) { //mask = 0ull; + //mask = (1 << SSE41_CUSTOM_IDX); if(is_avx_supported() && (mask & (1<< AVX_CUSTOM_IDX))) { cout << "Using AVX accelerated implementation of PairHMM\n"; @@ -286,6 +321,13 @@ double getCurrClk() { return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; } +inline unsigned long long rdtsc(void) +{ + unsigned hi, lo; + __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); + return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 ); +} + void get_time(struct timespec* store_struct) { clock_gettime(CLOCK_REALTIME, store_struct); @@ -298,9 +340,12 @@ uint64_t diff_time(struct timespec& prev_time) return (uint64_t)((curr_time.tv_sec-prev_time.tv_sec)*1000000000+(curr_time.tv_nsec-prev_time.tv_nsec)); } -//#define DUMP_COMPUTE_VALUES 1 -#define BATCH_SIZE 10000 -#define RUN_HYBRID + +#ifdef USE_PAPI +#include "papi.h" +#define NUM_PAPI_COUNTERS 4 +#endif + void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size, bool do_check) { FILE* fptr = 0; @@ -321,7 +366,22 @@ void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size, uint64_t vector_compute_time = 0; uint64_t baseline_compute_time = 0; uint64_t num_double_calls = 0; + unsigned num_testcases = 0; bool all_ok = do_check ? true : false; +#ifdef USE_PAPI + uint32_t all_mask = (0); + uint32_t no_usr_mask = (1 << 16); //bit 16 user mode, bit 17 kernel mode + uint32_t no_kernel_mask = (1 << 17); //bit 16 user mode, bit 17 kernel mode + PAPI_num_counters(); + int events[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + char* eventnames[NUM_PAPI_COUNTERS]= { "cycles", "itlb_walk_cycles", "dtlb_load_walk_cycles", "dtlb_store_walk_cycles" }; + assert(PAPI_event_name_to_code("UNHALTED_REFERENCE_CYCLES:u=1:k=1",&(events[0])) == PAPI_OK); + assert(PAPI_event_name_to_code("ITLB_MISSES:WALK_DURATION", &(events[1])) == PAPI_OK); + assert(PAPI_event_name_to_code("DTLB_LOAD_MISSES:WALK_DURATION", &(events[2])) == PAPI_OK); + assert(PAPI_event_name_to_code("DTLB_STORE_MISSES:WALK_DURATION", &(events[3])) == PAPI_OK); + long long values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + long long accum_values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; +#endif while(1) { int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); @@ -336,26 +396,42 @@ void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size, results_vec.resize(tc_vector.size()); baseline_results_vec.resize(tc_vector.size()); struct timespec start_time; +#ifdef USE_PAPI + assert(PAPI_start_counters(events, NUM_PAPI_COUNTERS) == PAPI_OK); +#endif get_time(&start_time); #pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) - for(unsigned i=0;i/bin/compilervars.sh intel64 -See run.sh in this directory on how to invoke HaplotypeCaller with the native library. The -argument -Djava.library.path is needed if the native implementation is selected, else -unnecessary. diff --git a/public/c++/VectorPairHMM/avx_function_instantiations.cc b/public/c++/VectorPairHMM/avx_function_instantiations.cc deleted file mode 100644 index 8f0de827d..000000000 --- a/public/c++/VectorPairHMM/avx_function_instantiations.cc +++ /dev/null @@ -1,19 +0,0 @@ -#include "template.h" - -#undef SIMD_ENGINE -#undef SIMD_ENGINE_SSE - -#define SIMD_ENGINE avx -#define SIMD_ENGINE_AVX - -#include "define-float.h" -#include "shift_template.c" -#include "pairhmm-template-kernel.cc" - -#include "define-double.h" -#include "shift_template.c" -#include "pairhmm-template-kernel.cc" - -template double compute_full_prob_avxd(testcase* tc, double* nextlog); -template float compute_full_prob_avxs(testcase* tc, float* nextlog); - diff --git a/public/c++/VectorPairHMM/jni_common.h b/public/c++/VectorPairHMM/jni_common.h deleted file mode 100644 index 1cffea1cc..000000000 --- a/public/c++/VectorPairHMM/jni_common.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef JNI_COMMON_H -#define JNI_COMMON_H - -#include -/*#define ENABLE_ASSERTIONS 1*/ -#define DO_PROFILING 1 -/*#define DEBUG 1*/ -/*#define DEBUG0_1 1*/ -/*#define DEBUG3 1*/ -/*#define DUMP_TO_SANDBOX 1*/ - - -#define DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY 1 - -#ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY -//Gets direct access to Java arrays -#define GET_BYTE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical -#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical -#define JNI_RO_RELEASE_MODE JNI_ABORT -#define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical -#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical - -#else -//Likely makes copy of Java arrays to JNI C++ space -#define GET_BYTE_ARRAY_ELEMENTS env->GetByteArrayElements -#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleaseByteArrayElements -#define JNI_RO_RELEASE_MODE JNI_ABORT -#define GET_DOUBLE_ARRAY_ELEMENTS env->GetDoubleArrayElements -#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleaseDoubleArrayElements - -#endif //ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY - -#endif //ifndef JNI_COMMON_H diff --git a/public/c++/VectorPairHMM/pairhmm-1-base.cc b/public/c++/VectorPairHMM/pairhmm-1-base.cc deleted file mode 100644 index 78dbe296d..000000000 --- a/public/c++/VectorPairHMM/pairhmm-1-base.cc +++ /dev/null @@ -1,45 +0,0 @@ -//#define DEBUG 1 -//#define DEBUG0_1 1 -//#define DEBUG3 1 -#include "headers.h" -#include "utils.h" -#include "LoadTimeInitializer.h" -using namespace std; - -int main(int argc, char** argv) -{ -#define BATCH_SIZE 10000 - if(argc < 2) - { - cerr << "Needs path to input file as argument\n"; - exit(0); - } - bool use_old_read_testcase = false; - if(argc >= 3 && string(argv[2]) == "1") - use_old_read_testcase = true; - unsigned chunk_size = 10000; - bool do_check = true; - uint64_t mask = ~(0ull); - for(int i=3;i(testcase* tc, double* nextlog); -template float compute_full_prob_sses(testcase* tc, float* nextlog); diff --git a/public/c++/VectorPairHMM/template.h b/public/c++/VectorPairHMM/template.h deleted file mode 100644 index f91c2300e..000000000 --- a/public/c++/VectorPairHMM/template.h +++ /dev/null @@ -1,194 +0,0 @@ -#ifndef TEMPLATES_H_ -#define TEMPLATES_H_ - -#include "headers.h" - -#define MM 0 -#define GapM 1 -#define MX 2 -#define XX 3 -#define MY 4 -#define YY 5 - -//#define MROWS 500 -//#define MCOLS 1000 - -#define CAT(X,Y) X####Y -#define CONCAT(X,Y) CAT(X,Y) - -#define ALIGNED __attribute__((aligned(32))) - -typedef union __attribute__((aligned(32))) { - ALIGNED __m256 ALIGNED d; - ALIGNED __m128i ALIGNED s[2]; - ALIGNED float ALIGNED f[8]; - ALIGNED __m256i ALIGNED i; -} ALIGNED mix_F ALIGNED; - -typedef union __attribute__((aligned(32))) { - ALIGNED __m128 ALIGNED d; - ALIGNED __m64 ALIGNED s[2]; - ALIGNED float ALIGNED f[4]; - ALIGNED __m128i ALIGNED i; -} ALIGNED mix_F128 ALIGNED; - -typedef union ALIGNED { - __m128i vec ; - __m128 vecf ; - uint32_t masks[4] ; -} MaskVec_F ; - -typedef union ALIGNED { - __m64 vec ; - __m64 vecf ; - uint32_t masks[2] ; -} MaskVec_F128 ; - -typedef union ALIGNED -{ - ALIGNED __m128i ALIGNED i; - ALIGNED __m128 ALIGNED f; -} ALIGNED IF_128f ALIGNED; - -typedef union ALIGNED -{ - ALIGNED int ALIGNED i; - ALIGNED float ALIGNED f; -} ALIGNED IF_32 ALIGNED; - -typedef union __attribute__((aligned(32))) { - ALIGNED __m256d ALIGNED d; - ALIGNED __m128i ALIGNED s[2]; - ALIGNED double ALIGNED f[4]; - ALIGNED __m256i ALIGNED i; -} ALIGNED mix_D ALIGNED; - -typedef union __attribute__((aligned(32))) { - ALIGNED __m128d ALIGNED d; - ALIGNED __m64 ALIGNED s[2]; - ALIGNED double ALIGNED f[2]; - ALIGNED __m128i ALIGNED i; -} ALIGNED mix_D128 ALIGNED; - -typedef union ALIGNED { - __m128i vec ; - __m128d vecf ; - uint64_t masks[2] ; -} MaskVec_D ; - -typedef union ALIGNED { - __m64 vec ; - __m64 vecf ; - uint64_t masks[1] ; -} MaskVec_D128 ; - -typedef union ALIGNED -{ - ALIGNED __m128i ALIGNED i; - ALIGNED __m128d ALIGNED f; -} ALIGNED IF_128d ALIGNED; - -typedef union ALIGNED -{ - ALIGNED int64_t ALIGNED i; - ALIGNED double ALIGNED f; -} ALIGNED IF_64 ALIGNED; - -template -struct Context{}; - -template<> -struct Context -{ - Context() - { - for (int x = 0; x < 128; x++) - ph2pr[x] = pow(10.0, -((double)x) / 10.0); - - INITIAL_CONSTANT = ldexp(1.0, 1020.0); - LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); - RESULT_THRESHOLD = 0.0; - } - - double LOG10(double v){ return log10(v); } - - static double _(double n){ return n; } - static double _(float n){ return ((double) n); } - double ph2pr[128]; - double INITIAL_CONSTANT; - double LOG10_INITIAL_CONSTANT; - double RESULT_THRESHOLD; -}; - -template<> -struct Context -{ - Context() - { - for (int x = 0; x < 128; x++) - { - ph2pr[x] = powf(10.f, -((float)x) / 10.f); - } - - INITIAL_CONSTANT = ldexpf(1.f, 120.f); - LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); - RESULT_THRESHOLD = ldexpf(1.f, -110.f); - } - - float LOG10(float v){ return log10f(v); } - - static float _(double n){ return ((float) n); } - static float _(float n){ return n; } - float ph2pr[128]; - float INITIAL_CONSTANT; - float LOG10_INITIAL_CONSTANT; - float RESULT_THRESHOLD; -}; - - -typedef struct -{ - int rslen, haplen; - /*int *q, *i, *d, *c;*/ - /*int q[MROWS], i[MROWS], d[MROWS], c[MROWS];*/ - char *q, *i, *d, *c; - char *hap, *rs; - int *ihap; - int *irs; -} testcase; - -int normalize(char c); -int read_testcase(testcase *tc, FILE* ifp=0); - - -#define MIN_ACCEPTED 1e-28f -#define NUM_DISTINCT_CHARS 5 -#define AMBIG_CHAR 4 - -class ConvertChar { - - static uint8_t conversionTable[255] ; - -public: - - static void init() { - assert (NUM_DISTINCT_CHARS == 5) ; - assert (AMBIG_CHAR == 4) ; - - conversionTable['A'] = 0 ; - conversionTable['C'] = 1 ; - conversionTable['T'] = 2 ; - conversionTable['G'] = 3 ; - conversionTable['N'] = 4 ; - } - - static inline uint8_t get(uint8_t input) { - return conversionTable[input] ; - } - -}; - - -#endif - - diff --git a/public/c++/VectorPairHMM/vector_defs.h b/public/c++/VectorPairHMM/vector_defs.h deleted file mode 100644 index 80b48ae99..000000000 --- a/public/c++/VectorPairHMM/vector_defs.h +++ /dev/null @@ -1,30 +0,0 @@ -#undef SIMD_ENGINE -#undef SIMD_ENGINE_AVX -#undef SIMD_ENGINE_SSE - -#define SIMD_ENGINE avx -#define SIMD_ENGINE_AVX - -#include "define-float.h" -#include "vector_function_prototypes.h" - -#include "define-double.h" -#include "vector_function_prototypes.h" - -#undef SIMD_ENGINE -#undef SIMD_ENGINE_AVX - -#define SIMD_ENGINE sse -#define SIMD_ENGINE_SSE - - -#include "define-sse-float.h" -#include "vector_function_prototypes.h" - -#include "define-sse-double.h" -#include "vector_function_prototypes.h" - -#undef SIMD_ENGINE -#undef SIMD_ENGINE_AVX -#undef SIMD_ENGINE_SSE - diff --git a/public/sting-root/pom.xml b/public/sting-root/pom.xml index 171eb7620..549a99ae6 100644 --- a/public/sting-root/pom.xml +++ b/public/sting-root/pom.xml @@ -335,7 +335,11 @@ maven-assembly-plugin 2.4 - + + org.apache.maven.plugins + maven-enforcer-plugin + 1.3.1 + + ${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + display-info + + validate + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + + exec + + compile + + make + src/main/c++ + + ${java.home} + ${project.build.directory} + + + + + + + + + org.apache.maven.plugins + maven-install-plugin + + true + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + default-install + + copy-resources + + install + + ${pairhmm.resources.directory} + + + ${project.build.directory} + + **/* + + + + + + + + + + + com.google.code.sortpom + maven-sortpom-plugin + + false + custom_1 + \n + ${sourceEncoding} + true + scope + 4 + false + + + + + diff --git a/public/VectorPairHMM/src/main/c++/.gitignore b/public/VectorPairHMM/src/main/c++/.gitignore new file mode 100644 index 000000000..d791ffd80 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/.gitignore @@ -0,0 +1,16 @@ +.svn +*.o +*.so +tests +.deps +hmm_Mohammad +pairhmm-template-main +*.swp +*.class +checker +reformat +subdir_checkout.sh +avx/ +sse/ +triplicate.sh + diff --git a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc new file mode 100644 index 000000000..0e3026f65 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc @@ -0,0 +1,206 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "LoadTimeInitializer.h" +#include "utils.h" +using namespace std; +char* LoadTimeInitializerStatsNames[] = +{ + "num_regions", + "num_reads", + "num_haplotypes", + "num_testcases", + "num_double_invocations", + "haplotype_length", + "readlength", + "product_read_length_haplotype_length", + "dummy" +}; + +LoadTimeInitializer g_load_time_initializer; + +LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loaded +{ + ConvertChar::init(); +#ifndef DISABLE_FTZ + //Very important to get good performance on Intel processors + //Function: enabling FTZ converts denormals to 0 in hardware + //Denormals cause microcode to insert uops into the core causing big slowdown + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + cout << "FTZ enabled - may decrease accuracy if denormal numbers encountered\n"; +#else + cout << "FTZ is not set - may slow down performance if denormal numbers encountered\n"; +#endif + //Profiling: times for compute and transfer (either bytes copied or pointers copied) + m_compute_time = 0; + m_data_transfer_time = 0; + m_bytes_copied = 0; + + //Initialize profiling counters + for(unsigned i=0;i::initializeStaticMembers(); + Context::initializeStaticMembers(); + + cout.flush(); +} + +void LoadTimeInitializer::print_profiling() +{ + double mean = 0; + double variance = 0; + uint64_t denominator = 1; + cout << "Time spent in compute_testcases "< C++) "<::iterator mI = m_filename_to_fptr.find(filename); + ofstream* fptr = 0; + if(mI == m_filename_to_fptr.end()) + { + m_filename_to_fptr[filename] = new ofstream(); + fptr = m_filename_to_fptr[filename]; + //File never seen before + if(m_written_files_set.find(filename) == m_written_files_set.end()) + { + to_append = false; + m_written_files_set.insert(filename); + } + fptr->open(filename.c_str(), to_append ? ios::app : ios::out); + assert(fptr->is_open()); + } + else + fptr = (*mI).second; + //ofstream fptr; + //fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); + (*fptr) << s; + if(add_newline) + (*fptr) << "\n"; + //fptr.close(); +} +void LoadTimeInitializer::debug_close() +{ + for(map::iterator mB = m_filename_to_fptr.begin(), mE = m_filename_to_fptr.end(); + mB != mE;mB++) + { + (*mB).second->close(); + delete (*mB).second; + } + m_filename_to_fptr.clear(); +} + +void LoadTimeInitializer::dump_sandbox(testcase& tc, unsigned tc_idx, unsigned numReads, unsigned numHaplotypes) +{ + unsigned haplotypeLength = tc.haplen; + unsigned readLength = tc.rslen; + ofstream& dumpFptr = m_sandbox_fptr; + for(unsigned k=0;k +#include "template.h" + +enum LoadTimeInitializerStatsEnum +{ + NUM_REGIONS_IDX=0, + NUM_READS_IDX, + NUM_HAPLOTYPES_IDX, + NUM_TESTCASES_IDX, + NUM_DOUBLE_INVOCATIONS_IDX, + HAPLOTYPE_LENGTH_IDX, + READ_LENGTH_IDX, + PRODUCT_READ_LENGTH_HAPLOTYPE_LENGTH_IDX, + TOTAL_NUMBER_STATS +}; +extern char* LoadTimeInitializerStatsNames[]; + +class LoadTimeInitializer +{ + public: + LoadTimeInitializer(); //will be called when library is loaded + ~LoadTimeInitializer() + { + delete m_buffer; + } + void print_profiling(); + void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); + void debug_close(); + + void dump_sandbox(testcase& tc, unsigned tc_idx, unsigned numReads, unsigned numHaplotypes); + void open_sandbox() { m_sandbox_fptr.open("sandbox.txt", std::ios::app); } + void close_sandbox() { m_sandbox_fptr.close(); } + + jfieldID m_readBasesFID; + jfieldID m_readQualsFID; + jfieldID m_insertionGOPFID; + jfieldID m_deletionGOPFID; + jfieldID m_overallGCPFID; + jfieldID m_haplotypeBasesFID; + //profiling - update stats + void update_stat(LoadTimeInitializerStatsEnum stat_idx, uint64_t value); + //timing in nanoseconds + uint64_t m_compute_time; + uint64_t m_data_transfer_time; + //bytes copied + uint64_t m_bytes_copied; + unsigned get_buffer_size() { return m_buffer_size; } + char* get_buffer() { return (char*)m_buffer; } + private: + std::map m_filename_to_fptr; + std::set m_written_files_set; + std::ofstream m_sandbox_fptr; + //used to compute various stats + uint64_t m_sum_stats[TOTAL_NUMBER_STATS]; + double m_sum_square_stats[TOTAL_NUMBER_STATS]; + uint64_t m_min_stats[TOTAL_NUMBER_STATS]; + uint64_t m_max_stats[TOTAL_NUMBER_STATS]; + unsigned m_buffer_size; + uint64_t* m_buffer; +}; +extern LoadTimeInitializer g_load_time_initializer; + +#define SIZE_PER_TESTCASE 6*10000 +#define SIZE_PER_BUFFER 10000 + +#endif diff --git a/public/VectorPairHMM/src/main/c++/Makefile b/public/VectorPairHMM/src/main/c++/Makefile new file mode 100644 index 000000000..354bca0bb --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Makefile @@ -0,0 +1,114 @@ +#Copyright (c) 2012 The Broad Institute + +#Permission is hereby granted, free of charge, to any person +#obtaining a copy of this software and associated documentation +#files (the "Software"), to deal in the Software without +#restriction, including without limitation the rights to use, +#copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the +#Software is furnished to do so, subject to the following +#conditions: + +#The above copyright notice and this permission notice shall be +#included in all copies or substantial portions of the Software. + +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +#THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + + +#OMPCFLAGS=-fopenmp +#OMPLFLAGS=-fopenmp #-openmp-link static + +#CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas +#CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas + +JRE_HOME?=/opt/jdk1.7.0_25/jre +JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JRE_HOME}/../include -I${JRE_HOME}/../include/linux + +COMMON_COMPILATION_FLAGS=$(JNI_COMPILATION_FLAGS) -O3 -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas +CC=icc +CXX=icc + +LDFLAGS=-lm -lrt $(OMPLDFLAGS) +ifdef DISABLE_FTZ + COMMON_COMPILATION_FLAGS+=-DDISABLE_FTZ -no-ftz +endif + +PAPI_DIR=/home/karthikg/softwares/papi-5.3.0 +ifdef USE_PAPI + ifeq ($(USE_PAPI),1) + COMMON_COMPILATION_FLAGS+=-I$(PAPI_DIR)/include -DUSE_PAPI + LDFLAGS+=-L$(PAPI_DIR)/lib -lpapi + endif +endif + +ifdef DISABLE_FTZ + COMMON_COMPILATION_FLAGS+=-DDISABLE_FTZ -no-ftz +endif + +BIN=libVectorLoglessPairHMM.so pairhmm-template-main checker +#BIN=checker + +DEPDIR=.deps +DF=$(DEPDIR)/$(*).d + +#Common across libJNI and sandbox +COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc sse_function_instantiations.cc LoadTimeInitializer.cc +#Part of libJNI +LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc Sandbox.cc $(COMMON_SOURCES) +SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc +LIBOBJECTS=$(LIBSOURCES:.cc=.o) +COMMON_OBJECTS=$(COMMON_SOURCES:.cc=.o) + + +#No vectorization for these files +NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc LoadTimeInitializer.cc Sandbox.cc +#Use -xAVX for these files +AVX_SOURCES=avx_function_instantiations.cc +#Use -xSSE4.2 for these files +SSE_SOURCES=sse_function_instantiations.cc + +NO_VECTOR_OBJECTS=$(NO_VECTOR_SOURCES:.cc=.o) +AVX_OBJECTS=$(AVX_SOURCES:.cc=.o) +SSE_OBJECTS=$(SSE_SOURCES:.cc=.o) +$(NO_VECTOR_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) +$(AVX_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xAVX +$(SSE_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xSSE4.2 +OBJECTS=$(NO_VECTOR_OBJECTS) $(AVX_OBJECTS) $(SSE_OBJECTS) + +all: $(BIN) Sandbox.class copied_lib + +-include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) + +checker: pairhmm-1-base.o $(COMMON_OBJECTS) + $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) + +pairhmm-template-main: pairhmm-template-main.o $(COMMON_OBJECTS) + $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) + +libVectorLoglessPairHMM.so: $(LIBOBJECTS) + $(CXX) $(OMPLFLAGS) -shared -static-intel -o $@ $(LIBOBJECTS) ${LDFLAGS} + + +$(OBJECTS): %.o: %.cc + @mkdir -p $(DEPDIR) + $(CXX) -c -MMD -MF $(DF) $(CXXFLAGS) $(OUTPUT_OPTION) $< + +Sandbox.class: Sandbox.java + javac Sandbox.java + +copied_lib: libVectorLoglessPairHMM.so +ifdef OUTPUT_DIR + mkdir -p $(OUTPUT_DIR) + rsync -a libVectorLoglessPairHMM.so $(OUTPUT_DIR)/ +endif + +clean: + rm -rf $(BIN) *.o $(DEPDIR) *.class diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.cc b/public/VectorPairHMM/src/main/c++/Sandbox.cc new file mode 100644 index 000000000..985b19ae9 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox.cc @@ -0,0 +1,106 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "Sandbox.h" +#include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" +#include "utils.h" +#include "jni_common.h" +/* + * Class: Sandbox + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_Sandbox_jniGetMachineType + (JNIEnv * env, jobject thisObj) +{ + return 0; +} + +/* + * Class: Sandbox + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeClassFieldsAndMachineMask + (JNIEnv* env, jobject thisObject, jclass readDataHolderClass, jclass haplotypeDataHolderClass, jlong mask) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask(env, thisObject, readDataHolderClass, + haplotypeDataHolderClass, mask); +} + +/* + * Class: Sandbox + * Method: jniInitializeHaplotypes + * Signature: (I[LSandbox/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeHaplotypes + (JNIEnv * env, jobject thisObject, jint numHaplotypes, jobjectArray haplotypeDataArray) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes(env, thisObject, numHaplotypes, haplotypeDataArray); +} + +/* + * Class: Sandbox + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniFinalizeRegion + (JNIEnv * env, jobject thisObject) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniFinalizeRegion(env, thisObject); +} + + +/* + * Class: Sandbox + * Method: jniComputeLikelihoods + * Signature: (II[LSandbox/JNIReadDataHolderClass;[LSandbox/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniComputeLikelihoods + (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, + jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods(env, thisObject, + numReads, numHaplotypes, readDataArray, haplotypeDataArray, likelihoodArray, maxNumThreadsToUse); +} +/* + * Class: Sandbox + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniClose + (JNIEnv* env, jobject thisObject) +{ Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose(env, thisObject); } + +JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative + (JNIEnv* env, jobject thisObject, jstring fileNameString) +{ + const char* fileName = env->GetStringUTFChars(fileNameString, 0); + char local_array[800]; + strncpy(local_array, fileName, 200); + env->ReleaseStringUTFChars(fileNameString, fileName); + do_compute(local_array, true, 10000, false); +} + diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.h b/public/VectorPairHMM/src/main/c++/Sandbox.h new file mode 100644 index 000000000..486a1c095 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox.h @@ -0,0 +1,96 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox */ + +#ifndef _Included_Sandbox +#define _Included_Sandbox +#ifdef __cplusplus +extern "C" { +#endif +#undef Sandbox_enableAll +#define Sandbox_enableAll -1LL +/* + * Class: Sandbox + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_Sandbox_jniGetMachineType + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeClassFieldsAndMachineMask + (JNIEnv *, jobject, jclass, jclass, jlong); + +/* + * Class: Sandbox + * Method: jniInitializeHaplotypes + * Signature: (I[LSandbox/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: Sandbox + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniFinalizeRegion + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniComputeLikelihoods + * Signature: (II[LSandbox/JNIReadDataHolderClass;[LSandbox/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); + +/* + * Class: Sandbox + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniClose + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: doEverythingNative + * Signature: ([B)V + */ +JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative + (JNIEnv *, jobject, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.java b/public/VectorPairHMM/src/main/c++/Sandbox.java new file mode 100644 index 000000000..d6b7c2eae --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox.java @@ -0,0 +1,305 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.broadinstitute.sting.utils.vectorpairhmm; + +import java.util.List; +import java.util.LinkedList; +import java.util.Map; +import java.util.HashMap; +import java.io.File; +import java.util.Scanner; +import java.io.IOException; +import java.io.FileNotFoundException; +import java.io.InputStreamReader; + +public class Sandbox { + + private long setupTime = 0; + private long computeTime = 0; + //Used to copy references to byteArrays to JNI from reads + protected class JNIReadDataHolderClass { + public byte[] readBases = null; + public byte[] readQuals = null; + public byte[] insertionGOP = null; + public byte[] deletionGOP = null; + public byte[] overallGCP = null; + } + + //Used to copy references to byteArrays to JNI from haplotypes + protected class JNIHaplotypeDataHolderClass { + public byte[] haplotypeBases = null; + } + + /** + * Return 64-bit mask representing machine capabilities + * Bit 0 is LSB, bit 63 MSB + * Bit 0 represents sse4.2 availability + * Bit 1 represents AVX availability + */ + public native long jniGetMachineType(); + public static final long enableAll = 0xFFFFFFFFFFFFFFFFl; + + + /** + * Function to initialize the fields of JNIReadDataHolderClass and JNIHaplotypeDataHolderClass from JVM. + * C++ codegets FieldIDs for these classes once and re-uses these IDs for the remainder of the program. Field IDs do not + * change per JVM session + * @param readDataHolderClass class type of JNIReadDataHolderClass + * @param haplotypeDataHolderClass class type of JNIHaplotypeDataHolderClass + * @param mask mask is a 64 bit integer identical to the one received from jniGetMachineType(). Users can disable usage of some hardware features by zeroing some bits in the mask + * */ + private native void jniInitializeClassFieldsAndMachineMask(Class readDataHolderClass, Class haplotypeDataHolderClass, long mask); + + private static Boolean isVectorLoglessPairHMMLibraryLoaded = false; + //The constructor is called only once inside PairHMMLikelihoodCalculationEngine + public Sandbox() { + synchronized(isVectorLoglessPairHMMLibraryLoaded) { + //Load the library and initialize the FieldIDs + if(!isVectorLoglessPairHMMLibraryLoaded) { + System.loadLibrary("VectorLoglessPairHMM"); + isVectorLoglessPairHMMLibraryLoaded = true; + jniInitializeClassFieldsAndMachineMask(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once + } + } + } + + private native void jniInitializeHaplotypes(final int numHaplotypes, JNIHaplotypeDataHolderClass[] haplotypeDataArray); + + //Used to transfer data to JNI + //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region + public void initialize(final List haplotypes) { + int numHaplotypes = haplotypes.size(); + JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; + int idx = 0; + for(final JNIHaplotypeDataHolderClass currHaplotype : haplotypes) + { + haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); + haplotypeDataArray[idx].haplotypeBases = currHaplotype.haplotypeBases; + ++idx; + } + jniInitializeHaplotypes(numHaplotypes, haplotypeDataArray); + } + /** + * Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not + * accessing Java memory directly, still important to release memory from C++ + */ + private native void jniFinalizeRegion(); + + + public void finalizeRegion() + { + jniFinalizeRegion(); + } + + /** + * Real compute kernel + */ + private native void jniComputeLikelihoods(int numReads, int numHaplotypes, JNIReadDataHolderClass[] readDataArray, + JNIHaplotypeDataHolderClass[] haplotypeDataArray, double[] likelihoodArray, int maxNumThreadsToUse); + + public void computeLikelihoods(final List reads, final List haplotypes) { + //System.out.println("Region : "+reads.size()+" x "+haplotypes.size()); + long startTime = System.nanoTime(); + int readListSize = reads.size(); + int numHaplotypes = haplotypes.size(); + int numTestcases = readListSize*numHaplotypes; + JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; + int idx = 0; + for(JNIReadDataHolderClass read : reads) + { + readDataArray[idx] = new JNIReadDataHolderClass(); + readDataArray[idx].readBases = read.readBases; + readDataArray[idx].readQuals = read.readQuals; + readDataArray[idx].insertionGOP = read.insertionGOP; + readDataArray[idx].deletionGOP = read.deletionGOP; + readDataArray[idx].overallGCP = read.overallGCP; + ++idx; + } + + double[] mLikelihoodArray = new double[readListSize*numHaplotypes]; //to store results + setupTime += (System.nanoTime() - startTime); + //for(reads) + // for(haplotypes) + // compute_full_prob() + jniComputeLikelihoods(readListSize, numHaplotypes, readDataArray, null, mLikelihoodArray, 12); + + computeTime += (System.nanoTime() - startTime); + } + + /** + * Print final profiling information from native code + */ + public native void jniClose(); + public void close() + { + System.out.println("Time spent in setup for JNI call : "+(setupTime*1e-9)+" compute time : "+(computeTime*1e-9)); + jniClose(); + } + + public void parseSandboxFile(String filename) + { + File file = new File(filename); + Scanner input = null; + try + { + input = new Scanner(file); + } + catch(FileNotFoundException e) + { + System.err.println("File "+filename+" cannot be found/read"); + return; + } + int idx = 0; + int numReads = 0; + int numHaplotypes = 0; + int readIdx = 0, testCaseIdx = 0, haplotypeIdx = 0; + LinkedList haplotypeList = new LinkedList(); + LinkedList readList = new LinkedList(); + + byte[][] byteArray = new byte[6][]; + boolean firstLine = true; + String[] currTokens = new String[8]; + while(input.hasNextLine()) + { + String line = input.nextLine(); + Scanner lineScanner = new Scanner(line); + idx = 0; + while(lineScanner.hasNext()) + currTokens[idx++] = lineScanner.next(); + if(idx == 0) + break; + assert(idx >= 6); + //start of new region + if(idx == 8) + { + if(!firstLine) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + try + { + numReads = Integer.parseInt(currTokens[6]); + } + catch(NumberFormatException e) + { + numReads = 1; + } + try + { + numHaplotypes = Integer.parseInt(currTokens[7]); + } + catch(NumberFormatException e) + { + numHaplotypes = 1; + } + haplotypeIdx = readIdx = testCaseIdx = 0; + readList.clear(); + haplotypeList.clear(); + } + if(haplotypeIdx < numHaplotypes) + { + JNIHaplotypeDataHolderClass X = new JNIHaplotypeDataHolderClass(); + X.haplotypeBases = currTokens[0].getBytes(); + haplotypeList.add(X); + } + if(testCaseIdx%numHaplotypes == 0) + { + JNIReadDataHolderClass X = new JNIReadDataHolderClass(); + X.readBases = currTokens[1].getBytes(); + for(int i=2;i<6;++i) + { + byteArray[i] = currTokens[i].getBytes(); + for(int j=0;j 0 && readList.size() > 0) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + + close(); + input.close(); + } + + private native void doEverythingNative(String filename); + + public static void main(String[] args) + { + if(args.length <= 0) + { + System.err.println("Needs 1 argument - "); + System.exit(-1); + } + //// Get runtime + //java.lang.Runtime rt = java.lang.Runtime.getRuntime(); + //// Start a new process: UNIX command ls + //String cmd = "/home/karthikg/broad/gsa-unstable/public/c++/VectorPairHMM/checker "+args[0]; + //try + //{ + //System.out.println(cmd); + //java.lang.Process p = rt.exec(cmd); + //try + //{ + //p.waitFor(); + //java.io.InputStream is = p.getInputStream(); + //java.io.BufferedReader reader = new java.io.BufferedReader(new InputStreamReader(is)); + //// And print each line + //String s = null; + //while ((s = reader.readLine()) != null) { + //System.out.println(s); + //} + //is.close(); + //} + //catch(InterruptedException e) + //{ + //System.err.println(e); + //} + //} + //catch(IOException e) + //{ + //System.err.println(e); + //} + Sandbox t = new Sandbox(); + //t.doEverythingNative(args[0]); + t.parseSandboxFile(args[0]); + } +} diff --git a/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h new file mode 100644 index 000000000..7f78f0178 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIHaplotypeDataHolderClass */ + +#ifndef _Included_Sandbox_JNIHaplotypeDataHolderClass +#define _Included_Sandbox_JNIHaplotypeDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h new file mode 100644 index 000000000..a9312ff3b --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIReadDataHolderClass */ + +#ifndef _Included_Sandbox_JNIReadDataHolderClass +#define _Included_Sandbox_JNIReadDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc new file mode 100644 index 000000000..6d90d5070 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc @@ -0,0 +1,44 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "template.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_SSE + +#define SIMD_ENGINE avx +#define SIMD_ENGINE_AVX + +#include "define-float.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +#include "define-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +template double compute_full_prob_avxd(testcase* tc, double* nextlog); +template float compute_full_prob_avxs(testcase* tc, float* nextlog); + diff --git a/public/VectorPairHMM/src/main/c++/baseline.cc b/public/VectorPairHMM/src/main/c++/baseline.cc new file mode 100644 index 000000000..d6085e661 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/baseline.cc @@ -0,0 +1,167 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +using namespace std; + +template +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) +{ + int r, c; + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + + Context ctx; + //#define USE_STACK_ALLOCATION 1 +#ifdef USE_STACK_ALLOCATION + NUMBER M[ROWS][COLS]; + NUMBER X[ROWS][COLS]; + NUMBER Y[ROWS][COLS]; + NUMBER p[ROWS][6]; +#else + //allocate on heap in way that simulates a 2D array. Having a 2D array instead of + //a straightforward array of pointers ensures that all data lies 'close' in memory, increasing + //the chance of being stored together in the cache. Also, prefetchers can learn memory access + //patterns for 2D arrays, not possible for array of pointers + //bool locally_allocated = false; + //NUMBER* common_buffer = 0; + NUMBER* common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //unsigned curr_size = sizeof(NUMBER)*(3*ROWS*COLS + ROWS*6); + //if(true) + //{ + //common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //locally_allocated = true; + //} + //else + //common_buffer = (NUMBER*)(g_load_time_initializer.get_buffer()); + //pointers to within the allocated buffer + NUMBER** common_pointer_buffer = new NUMBER*[4*ROWS]; + NUMBER* ptr = common_buffer; + unsigned i = 0; + for(i=0;i<3*ROWS;++i, ptr+=COLS) + common_pointer_buffer[i] = ptr; + for(;i<4*ROWS;++i, ptr+=6) + common_pointer_buffer[i] = ptr; + + NUMBER** M = common_pointer_buffer; + NUMBER** X = M + ROWS; + NUMBER** Y = X + ROWS; + NUMBER** p = Y + ROWS; +#endif + + + p[0][MM] = ctx._(0.0); + p[0][GapM] = ctx._(0.0); + p[0][MX] = ctx._(0.0); + p[0][XX] = ctx._(0.0); + p[0][MY] = ctx._(0.0); + p[0][YY] = ctx._(0.0); + + for (r = 1; r < ROWS; r++) + { + int _i = tc->i[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + //p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(p[r][MM], _i, _d); + p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; + p[r][MX] = ctx.ph2pr[_i]; + p[r][XX] = ctx.ph2pr[_c]; + p[r][MY] = ctx.ph2pr[_d]; + p[r][YY] = ctx.ph2pr[_c]; + //p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + //p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + } + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); + } + + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } + + NUMBER result = ctx._(0.0); + + for (r = 1; r < ROWS; r++) + for (c = 1; c < COLS; c++) + { + fexcept_t flagp; + char _rs = tc->rs[r-1]; + char _hap = tc->hap[c-1]; + int _q = tc->q[r-1] & 127; + NUMBER distm = ctx.ph2pr[_q]; + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; + else + distm = distm/3; + + + //feclearexcept(FE_ALL_EXCEPT); + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //feclearexcept(FE_ALL_EXCEPT); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //feclearexcept(FE_ALL_EXCEPT); + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //CONVERT_AND_PRINT(M[r][c]); + //CONVERT_AND_PRINT(X[r][c]); + //CONVERT_AND_PRINT(Y[r][c]); + + } + for (c = 0; c < COLS; c++) + { + result += M[ROWS-1][c] + X[ROWS-1][c]; + } + + if (before_last_log != NULL) + *before_last_log = result; + +#ifndef USE_STACK_ALLOCATION + delete common_pointer_buffer; + //if(locally_allocated) + delete common_buffer; +#endif + + return result; + //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; +} + +template double compute_full_prob(testcase* tc, double* nextbuf); +template float compute_full_prob(testcase* tc, float* nextbuf); + diff --git a/public/VectorPairHMM/src/main/c++/define-double.h b/public/VectorPairHMM/src/main/c++/define-double.h new file mode 100644 index 000000000..2067d369c --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-double.h @@ -0,0 +1,205 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define PRECISION d +#define MAIN_TYPE double +#define MAIN_TYPE_SIZE 64 +#define UNION_TYPE mix_D +#define IF_128 IF_128d +#define IF_MAIN_TYPE IF_64 +#define SHIFT_CONST1 8 +#define SHIFT_CONST2 1 +#define SHIFT_CONST3 8 +#define _128_TYPE __m128d +#define SIMD_TYPE __m256d +#define _256_INT_TYPE __m256i +#define AVX_LENGTH 4 +#define HAP_TYPE __m128i +#define MASK_TYPE uint64_t +#define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFF +#define MASK_VEC MaskVec_D + +#define SET_VEC_ZERO(__vec) \ + __vec= _mm256_setzero_pd() + +#define VEC_OR(__v1, __v2) \ + _mm256_or_pd(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm256_add_pd(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm256_sub_pd(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm256_mul_pd(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm256_div_pd(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm256_blend_pd(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm256_blendv_pd(__v1, __v2, __maskV) + +#define VEC_CAST_256_128(__v1) \ + _mm256_castpd256_pd128 (__v1) + +#define VEC_EXTRACT_128(__v1, __im) \ + _mm256_extractf128_pd (__v1, __im) + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi64(__v1, __im) + +#define VEC_SET1_VAL128(__val) \ + _mm_set1_pd(__val) + +#define VEC_MOVE(__v1, __val) \ + _mm_move_sd(__v1, __val) + +#define VEC_CAST_128_256(__v1) \ + _mm256_castpd128_pd256(__v1) + +#define VEC_INSERT_VAL(__v1, __val, __pos) \ + _mm256_insertf128_pd(__v1, __val, __pos) + +#define VEC_CVT_128_256(__v1) \ + _mm256_cvtepi32_pd(__v1) + +#define VEC_SET1_VAL(__val) \ + _mm256_set1_pd(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm256_cvtepi32_pd(_mm_set1_epi32(__ch)) + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm256_cvtepi32_pd(_mm_load_si128((__m128i const *)__addr)) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm256_cmp_pd(__v1, __v2, _CMP_EQ_OQ) + +#define VEC_SET_LSE(__val) \ + _mm256_set_pd(zero, zero, zero, __val); + +#define SHIFT_HAP(__v1, __val) \ + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castpd128_pd256(__vsLow) ; \ +__vdst = _mm256_insertf128_pd(__vdst, __vsHigh, 1) ; + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi64(__vs, 1) + + +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + double* ptr1 = (double*) (&__v1) ; \ + double* ptr2 = (double*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Double Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ +} + +class BitMaskVec_double { + + MASK_VEC low_, high_ ; + SIMD_TYPE combined_ ; + + public: + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_double diff --git a/public/VectorPairHMM/src/main/c++/define-float.h b/public/VectorPairHMM/src/main/c++/define-float.h new file mode 100644 index 000000000..318f78280 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-float.h @@ -0,0 +1,206 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define PRECISION s + +#define MAIN_TYPE float +#define MAIN_TYPE_SIZE 32 +#define UNION_TYPE mix_F +#define IF_128 IF_128f +#define IF_MAIN_TYPE IF_32 +#define SHIFT_CONST1 12 +#define SHIFT_CONST2 3 +#define SHIFT_CONST3 4 +#define _128_TYPE __m128 +#define SIMD_TYPE __m256 +#define _256_INT_TYPE __m256i +#define AVX_LENGTH 8 +#define HAP_TYPE UNION_TYPE +#define MASK_TYPE uint32_t +#define MASK_ALL_ONES 0xFFFFFFFF +#define MASK_VEC MaskVec_F + +#define SET_VEC_ZERO(__vec) \ + __vec= _mm256_setzero_ps() + +#define VEC_OR(__v1, __v2) \ + _mm256_or_ps(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm256_add_ps(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm256_sub_ps(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm256_mul_ps(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm256_div_ps(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm256_blend_ps(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm256_blendv_ps(__v1, __v2, __maskV) + +#define VEC_CAST_256_128(__v1) \ + _mm256_castps256_ps128 (__v1) + +#define VEC_EXTRACT_128(__v1, __im) \ + _mm256_extractf128_ps (__v1, __im) + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi32(__v1, __im) + +#define VEC_SET1_VAL128(__val) \ + _mm_set1_ps(__val) + +#define VEC_MOVE(__v1, __val) \ + _mm_move_ss(__v1, __val) + +#define VEC_CAST_128_256(__v1) \ + _mm256_castps128_ps256(__v1) + +#define VEC_INSERT_VAL(__v1, __val, __pos) \ + _mm256_insertf128_ps(__v1, __val, __pos) + +#define VEC_CVT_128_256(__v1) \ + _mm256_cvtepi32_ps(__v1.i) + +#define VEC_SET1_VAL(__val) \ + _mm256_set1_ps(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm256_cvtepi32_ps(_mm256_set1_epi32(__ch)) + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm256_cvtepi32_ps(_mm256_loadu_si256((__m256i const *)__addr)) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm256_cmp_ps(__v1, __v2, _CMP_EQ_OQ) + +#define VEC_SET_LSE(__val) \ + _mm256_set_ps(zero, zero, zero, zero, zero, zero, zero, __val); + +#define SHIFT_HAP(__v1, __val) \ + _vector_shift_lastavxs(__v1, __val.f); + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castps128_ps256(__vsLow) ; \ +__vdst = _mm256_insertf128_ps(__vdst, __vsHigh, 1) ; + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi32(__vs, 1) + +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + float* ptr1 = (float*) (&__v1) ; \ + float* ptr2 = (float*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Float Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ +} + +class BitMaskVec_float { + + MASK_VEC low_, high_ ; + SIMD_TYPE combined_ ; + + public: + + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_float diff --git a/public/VectorPairHMM/src/main/c++/define-sse-double.h b/public/VectorPairHMM/src/main/c++/define-sse-double.h new file mode 100644 index 000000000..2d271a854 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-sse-double.h @@ -0,0 +1,173 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_INSERT_UNIT(__v1,__ins,__im) +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define SSE +#define PRECISION d + +#define MAIN_TYPE double +#define MAIN_TYPE_SIZE 64 +#define UNION_TYPE mix_D128 +#define IF_128 IF_128d +#define IF_MAIN_TYPE IF_64 +#define SHIFT_CONST1 1 +#define SHIFT_CONST2 8 +#define SHIFT_CONST3 0 +#define _128_TYPE __m128d +#define SIMD_TYPE __m128d +#define _256_INT_TYPE __m128i +#define AVX_LENGTH 2 +#define HAP_TYPE __m128i +#define MASK_TYPE uint64_t +#define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFFL +#define MASK_VEC MaskVec_D + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi64(__v1, __im) + +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi64(__v1,__ins,__im) + +#define VEC_OR(__v1, __v2) \ + _mm_or_pd(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm_add_pd(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm_sub_pd(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm_mul_pd(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm_div_pd(__v1, __v2) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm_cmpeq_pd(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm_blend_pd(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm_blendv_pd(__v1, __v2, __maskV) + +#define SHIFT_HAP(__v1, __val) \ + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) + +#define VEC_CVT_128_256(__v1) \ + _mm_cvtepi32_pd(__v1) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_pd(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm_cvtepi32_pd(_mm_set1_epi32(__ch)) + +#define VEC_SET_LSE(__val) \ + _mm_set_pd(zero, __val); + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm_cvtepi32_pd(_mm_loadu_si128((__m128i const *)__addr)) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm_castsi128_pd(_mm_set_epi64(__vsHigh, __vsLow)) + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi64(__vs, 1) + + +class BitMaskVec_sse_double { + + MASK_VEC combined_ ; + public: + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_sse_double + diff --git a/public/VectorPairHMM/src/main/c++/define-sse-float.h b/public/VectorPairHMM/src/main/c++/define-sse-float.h new file mode 100644 index 000000000..20af947dd --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-sse-float.h @@ -0,0 +1,173 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_INSERT_UNIT(__v1,__ins,__im) +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define SSE +#define PRECISION s + +#define MAIN_TYPE float +#define MAIN_TYPE_SIZE 32 +#define UNION_TYPE mix_F128 +#define IF_128 IF_128f +#define IF_MAIN_TYPE IF_32 +#define SHIFT_CONST1 3 +#define SHIFT_CONST2 4 +#define SHIFT_CONST3 0 +#define _128_TYPE __m128 +#define SIMD_TYPE __m128 +#define _256_INT_TYPE __m128i +#define AVX_LENGTH 4 +//#define MAVX_COUNT (MROWS+3)/AVX_LENGTH +#define HAP_TYPE UNION_TYPE +#define MASK_TYPE uint32_t +#define MASK_ALL_ONES 0xFFFFFFFF +#define MASK_VEC MaskVec_F + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi32(__v1, __im) + +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi32(__v1,__ins,__im) + +#define VEC_OR(__v1, __v2) \ + _mm_or_ps(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm_add_ps(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm_sub_ps(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm_mul_ps(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm_div_ps(__v1, __v2) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm_cmpeq_ps(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm_blend_ps(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm_blendv_ps(__v1, __v2, __maskV) + +#define SHIFT_HAP(__v1, __val) \ + _vector_shift_lastsses(__v1, __val.f) + +#define VEC_CVT_128_256(__v1) \ + _mm_cvtepi32_ps(__v1.i) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_ps(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm_cvtepi32_ps(_mm_set1_epi32(__ch)) + +#define VEC_SET_LSE(__val) \ + _mm_set_ps(zero, zero, zero, __val); + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm_cvtepi32_ps(_mm_loadu_si128((__m128i const *)__addr)) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm_cvtpi32x2_ps(__vsLow, __vsHigh) + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi32(__vs, 1) + +class BitMaskVec_sse_float { + + MASK_VEC combined_ ; + + public: + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_sse_float diff --git a/public/VectorPairHMM/src/main/c++/headers.h b/public/VectorPairHMM/src/main/c++/headers.h new file mode 100644 index 000000000..4a0d89b57 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/headers.h @@ -0,0 +1,71 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef COMMON_HEADERS_H +#define COMMON_HEADERS_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern uint64_t exceptions_array[128]; +extern FILE* g_debug_fptr; +#define STORE_FP_EXCEPTIONS(flagp, exceptions_array) \ + fegetexceptflag(&flagp, FE_ALL_EXCEPT | __FE_DENORM); \ + exceptions_array[FE_INVALID] += ((flagp & FE_INVALID)); \ + exceptions_array[__FE_DENORM] += ((flagp & __FE_DENORM) >> 1); \ + exceptions_array[FE_DIVBYZERO] += ((flagp & FE_DIVBYZERO) >> 2); \ + exceptions_array[FE_OVERFLOW] += ((flagp & FE_OVERFLOW) >> 3); \ + exceptions_array[FE_UNDERFLOW] += ((flagp & FE_UNDERFLOW) >> 4); \ + feclearexcept(FE_ALL_EXCEPT | __FE_DENORM); + +#define CONVERT_AND_PRINT(X) \ + g_converter.f = (X); \ + fwrite(&(g_converter.i),4,1,g_debug_fptr); \ + +#endif diff --git a/public/VectorPairHMM/src/main/c++/jni_common.h b/public/VectorPairHMM/src/main/c++/jni_common.h new file mode 100644 index 000000000..23c323246 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/jni_common.h @@ -0,0 +1,58 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef JNI_COMMON_H +#define JNI_COMMON_H + +#include +/*#define ENABLE_ASSERTIONS 1*/ +#define DO_PROFILING 1 +/*#define DEBUG 1*/ +/*#define DEBUG0_1 1*/ +/*#define DEBUG3 1*/ +/*#define DUMP_TO_SANDBOX 1*/ + + +#define DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY 1 + +#ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY +//Gets direct access to Java arrays +#define GET_BYTE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical + +#else +//Likely makes copy of Java arrays to JNI C++ space +#define GET_BYTE_ARRAY_ELEMENTS env->GetByteArrayElements +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleaseByteArrayElements +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetDoubleArrayElements +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleaseDoubleArrayElements + +#endif //ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY + +#endif //ifndef JNI_COMMON_H diff --git a/public/VectorPairHMM/src/main/c++/jnidebug.h b/public/VectorPairHMM/src/main/c++/jnidebug.h new file mode 100644 index 000000000..7fcab2a51 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/jnidebug.h @@ -0,0 +1,191 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef JNI_DEBUG_H +#define JNI_DEBUG_H + +template +class DataHolder +{ +#define INIT_MATRIX(X) \ + X = new NUMBER*[m_paddedMaxReadLength]; \ + for(int i=0;i ctx; + for (int r = 1; r <= length;r++) //in original code, r < ROWS (where ROWS = paddedReadLength) + { + int _i = insertionGOP[r-1]; //insertionGOP + int _d = deletionGOP[r-1]; //deletionGOP + int _c = overallGCP[r-1]; //overallGCP + m_transition[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; //lines 161-162 + m_transition[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; //line 163 + m_transition[r][MX] = ctx.ph2pr[_i]; //164 + m_transition[r][XX] = ctx.ph2pr[_c]; //165 + m_transition[r][MY] = ctx.ph2pr[_d];//last row seems different, compared to line 166 + m_transition[r][YY] = ctx.ph2pr[_c];//same as above for line 167 + //m_transition[r][MY] = (r == length) ? ctx._(1.0) : ctx.ph2pr[_d];//last row seems different, compared to line 166 + //m_transition[r][YY] = (r == length) ? ctx._(1.0) : ctx.ph2pr[_c];//same as above for line 167 +#ifdef DEBUG3 + for(int j=0;j<6;++j) + debug_dump("transitions_jni.txt", to_string(m_transition[r][j]),true); +#endif + } + ++g_num_prob_init; + } + bool m_is_initialized; + int m_readMaxLength; + int m_haplotypeMaxLength; + int m_paddedMaxReadLength; + int m_paddedMaxHaplotypeLength; + NUMBER** m_matchMatrix; + NUMBER** m_insertionMatrix; + NUMBER** m_deletionMatrix; + NUMBER** m_prior; + NUMBER (*m_transition)[6]; +}; +extern DataHolder g_double_dataholder; + +template +NUMBER compute_full_prob(testcase *tc, NUMBER** M, NUMBER** X, NUMBER** Y, NUMBER (*p)[6], + bool do_initialization, jint hapStartIndex, NUMBER *before_last_log = NULL) +{ + int r, c; + int ROWS = tc->rslen + 1; //ROWS = paddedReadLength + int COLS = tc->haplen + 1; //COLS = paddedHaplotypeLength + + Context ctx; + //////NOTES + ////ctx.ph2pr[quality]; //This quantity is QualityUtils.qualToErrorProb(quality) + ////1-ctx.ph2pr[quality]; //This corresponds to QualityUtils.qualToProb(quality); + + //Initialization + if(do_initialization) + { + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); //code from 87-90 in LoglessPairHMM + } + + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + //deletionMatrix row 0 in above nest is initialized in the Java code + //However, insertionMatrix column 0 is not initialized in Java code, could it be that + //values are re-used from a previous iteration? + //Why even do this, X[0][0] = 0 from above loop nest, X[idx][0] = 0 from this computation + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } + } + + for (r = 1; r < ROWS; r++) + for (c = hapStartIndex+1; c < COLS; c++) + { + //The following lines correspond to initializePriors() + char _rs = tc->rs[r-1]; //line 137 + char _hap = tc->hap[c-1]; //line 140 + //int _q = tc->q[r-1] & 127; //line 138 - q is the quality (qual), should be byte hence int ANDed with 0xFF + int _q = tc->q[r-1]; //line 138 - q is the quality (qual), should be byte hence int ANDed with 0xFF + NUMBER distm = ctx.ph2pr[_q]; //This quantity is QualityUtils.qualToErrorProb(_q) + //The assumption here is that doNotUseTristateCorrection is true + //TOASK + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; //This is the quantity QualityUtils.qualToProb(qual) + else + distm = distm/3; +#ifdef DEBUG3 + debug_dump("priors_jni.txt",to_string(distm),true); +#endif + + //Computation inside updateCell + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; +#ifdef DEBUG3 + debug_dump("matrices_jni.txt",to_string(M[r][c]),true); + debug_dump("matrices_jni.txt",to_string(X[r][c]),true); + debug_dump("matrices_jni.txt",to_string(Y[r][c]),true); +#endif + } + + NUMBER result = ctx._(0.0); + for (c = 0; c < COLS; c++) + result += M[ROWS-1][c] + X[ROWS-1][c]; + + if (before_last_log != NULL) + *before_last_log = result; + +#ifdef DEBUG + debug_dump("return_values_jni.txt",to_string(ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT),true); +#endif + return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; +} + +#endif diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc new file mode 100644 index 000000000..8a3f8b5bc --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc @@ -0,0 +1,176 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "jni_common.h" +#include "org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +#include "jnidebug.h" +DataHolder g_double_dataholder; + +using namespace std; + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitialize +(JNIEnv* env, jobject thisObject, + jint readMaxLength, jint haplotypeMaxLength) +{ + static int g_num_init_calls = 0; +#ifdef DEBUG3 + cout << "Entered alloc initialized .. readMaxLength "<GetArrayLength(insertionGOP); +#ifdef DEBUG3 + cout << "Entered initializeProbabilities .. length "<GetByteArrayElements(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (env)->GetByteArrayElements(deletionGOP, &is_copy); + jbyte* overallGCPArray = (env)->GetByteArrayElements(overallGCP, &is_copy); +#ifdef DEBUG + if(insertionGOPArray == 0) + cerr << "insertionGOP array not initialized in JNI\n"; + ////assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + if(deletionGOPArray == 0) + cerr << "deletionGOP array not initialized in JNI\n"; + ////assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); +#endif + + g_double_dataholder.initializeProbabilities(length, insertionGOPArray, deletionGOPArray, overallGCPArray); + + env->ReleaseByteArrayElements(overallGCP, overallGCPArray, JNI_ABORT); + env->ReleaseByteArrayElements(deletionGOP, deletionGOPArray, JNI_ABORT); + env->ReleaseByteArrayElements(insertionGOP, insertionGOPArray, JNI_ABORT); +} + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializePriorsAndUpdateCells( + JNIEnv* env, jobject thisObject, + jboolean doInitialization, jint paddedReadLength, jint paddedHaplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jint hapStartIndex + ) +{ +#ifdef DEBUG3 + cout << "Entered mainCompute .. doInitialization "<<(doInitialization == JNI_TRUE)<<" hapStartIndex "<GetByteArrayElements(readBases, &is_copy); + jbyte* haplotypeBasesArray = (env)->GetByteArrayElements(haplotypeBases, &is_copy); + jbyte* readQualsArray = (env)->GetByteArrayElements(readQuals, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); +#endif + testcase tc; + + tc.rslen = paddedReadLength-1; + tc.haplen = paddedHaplotypeLength-1; + + tc.rs = (char*)readBasesArray; + tc.hap = (char*)haplotypeBasesArray; + tc.q = (char*)readQualsArray; //TOASK - q is now char* + + compute_full_prob(&tc, g_double_dataholder.m_matchMatrix, g_double_dataholder.m_insertionMatrix, + g_double_dataholder.m_deletionMatrix, g_double_dataholder.m_transition, + doInitialization == JNI_TRUE, hapStartIndex, NULL); + + env->ReleaseByteArrayElements(readBases, readBasesArray, JNI_ABORT); + env->ReleaseByteArrayElements(haplotypeBases, haplotypeBasesArray, JNI_ABORT); + env->ReleaseByteArrayElements(readQuals, readQualsArray, JNI_ABORT); + return 0.0; +} + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10( + JNIEnv* env, jobject thisObject, + jint readLength, jint haplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP, + jint hapStartIndex + ) +{ + jboolean is_copy = JNI_FALSE; + jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBases, &is_copy); + jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); + jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); + jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); + //assert(readLength < MROWS); +#endif + testcase tc; + tc.rslen = readLength; + tc.haplen = haplotypeLength; + tc.rs = (char*)readBasesArray; + tc.hap = (char*)haplotypeBasesArray; + for(unsigned i=0;i +/* Header for class org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM */ + +#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM +#define _Included_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM +#ifdef __cplusplus +extern "C" { +#endif +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_TRISTATE_CORRECTION +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_TRISTATE_CORRECTION 3.0 +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToMatch +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToMatch 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_indelToMatch +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_indelToMatch 1L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToInsertion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToInsertion 2L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_insertionToInsertion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_insertionToInsertion 3L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToDeletion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToDeletion 4L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_deletionToDeletion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_deletionToDeletion 5L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_verify +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_verify 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug0_1 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug0_1 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug1 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug1 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug2 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug2 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug3 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug3 0L +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitialize + * Signature: (II)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitialize + (JNIEnv *, jobject, jint, jint); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitializeProbabilities + * Signature: ([[D[B[B[B)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializeProbabilities + (JNIEnv *, jclass, jobjectArray, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitializePriorsAndUpdateCells + * Signature: (ZII[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializePriorsAndUpdateCells + (JNIEnv *, jobject, jboolean, jint, jint, jbyteArray, jbyteArray, jbyteArray, jint); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniSubComputeReadLikelihoodGivenHaplotypeLog10 + * Signature: (II[B[B[B[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10 + (JNIEnv *, jobject, jint, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc new file mode 100644 index 000000000..0b54c8a81 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -0,0 +1,382 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "jni_common.h" +#include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" + +using namespace std; + +JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType + (JNIEnv* env, jobject thisObject) +{ + return (jlong)get_machine_capabilities(); +} + +//Should be called only once for the whole Java process - initializes field ids for the classes JNIReadDataHolderClass +//and JNIHaplotypeDataHolderClass +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask + (JNIEnv* env, jobject thisObject, jclass readDataHolderClass, jclass haplotypeDataHolderClass, jlong mask) +{ + assert(readDataHolderClass); + assert(haplotypeDataHolderClass); + jfieldID fid; + fid = env->GetFieldID(readDataHolderClass, "readBases", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for readBases"); + g_load_time_initializer.m_readBasesFID = fid; + fid = env->GetFieldID(readDataHolderClass, "readQuals", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for readQuals"); + g_load_time_initializer.m_readQualsFID = fid; + fid = env->GetFieldID(readDataHolderClass, "insertionGOP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for insertionGOP"); + g_load_time_initializer.m_insertionGOPFID = fid; + fid = env->GetFieldID(readDataHolderClass, "deletionGOP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for deletionGOP"); + g_load_time_initializer.m_deletionGOPFID = fid; + fid = env->GetFieldID(readDataHolderClass, "overallGCP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for overallGCP"); + g_load_time_initializer.m_overallGCPFID = fid; + + fid = env->GetFieldID(haplotypeDataHolderClass, "haplotypeBases", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for haplotypeBases"); + g_load_time_initializer.m_haplotypeBasesFID = fid; + if(mask != ENABLE_ALL_HARDWARE_FEATURES) + { + cout << "Using user supplied hardware mask to re-initialize function pointers for PairHMM\n"; + initialize_function_pointers((uint64_t)mask); + cout.flush(); + } +} + +//Since the list of haplotypes against which the reads are evaluated in PairHMM is the same for a region, +//transfer the list only once +vector > g_haplotypeBasesArrayVector; +vector g_haplotypeBasesLengths; +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes + (JNIEnv * env, jobject thisObject, jint numHaplotypes, jobjectArray haplotypeDataArray) +{ + jboolean is_copy = JNI_FALSE; + //To ensure, GET_BYTE_ARRAY_ELEMENTS is invoked only once for each haplotype, store bytearrays in a vector + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + haplotypeBasesArrayVector.clear(); + g_haplotypeBasesLengths.clear(); + haplotypeBasesArrayVector.resize(numHaplotypes); + g_haplotypeBasesLengths.resize(numHaplotypes); + jsize haplotypeBasesLength = 0; + for(unsigned j=0;jGetObjectArrayElement(haplotypeDataArray, j); + jbyteArray haplotypeBases = (jbyteArray)env->GetObjectField(haplotypeObject, g_load_time_initializer.m_haplotypeBasesFID); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBases && ("haplotypeBases is NULL at index : "+to_string(j)+"\n").c_str()); +#endif + //Need a global reference as this will be accessed across multiple JNI calls to JNIComputeLikelihoods() + jbyteArray haplotypeBasesGlobalRef = (jbyteArray)env->NewGlobalRef(haplotypeBases); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBasesGlobalRef && ("Could not get global ref to haplotypeBases at index : "+to_string(j)+"\n").c_str()); +#endif + env->DeleteLocalRef(haplotypeBases); //free the local reference + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBasesGlobalRef, &is_copy); + haplotypeBasesLength = env->GetArrayLength(haplotypeBasesGlobalRef); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + //assert(haplotypeBasesLength < MCOLS); +#endif +#ifdef DEBUG0_1 + cout << "JNI haplotype length "< > >& readBasesArrayVector, vector& tc_array) +{ + jboolean is_copy = JNI_FALSE; + //haplotype vector from earlier store - note the reference to vector, not copying + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + unsigned tc_idx = 0; + for(unsigned i=0;iGetObjectArrayElement(readDataArray, i); + jbyteArray readBases = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_readBasesFID); + jbyteArray insertionGOP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_insertionGOPFID); + jbyteArray deletionGOP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_deletionGOPFID); + jbyteArray overallGCP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_overallGCPFID); + jbyteArray readQuals = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_readQualsFID); + +#ifdef ENABLE_ASSERTIONS + assert(readBases && ("readBases is NULL at index : "+to_string(i)+"\n").c_str()); + assert(insertionGOP && ("insertionGOP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(deletionGOP && ("deletionGOP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(overallGCP && ("overallGCP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(readQuals && ("readQuals is NULL at index : "+to_string(i)+"\n").c_str()); +#endif + jsize readLength = env->GetArrayLength(readBases); + + jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); //order of GET-RELEASE is important + jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); + jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); + jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DO_PROFILING + g_load_time_initializer.m_bytes_copied += (is_copy ? readLength*5 : 0); + g_load_time_initializer.update_stat(READ_LENGTH_IDX, readLength); +#endif +#ifdef ENABLE_ASSERTIONS + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "overallGCP array not initialized in JNI"); + //assert(readLength < MROWS); + assert(readLength == env->GetArrayLength(readQuals)); + assert(readLength == env->GetArrayLength(insertionGOP)); + assert(readLength == env->GetArrayLength(deletionGOP)); + assert(readLength == env->GetArrayLength(overallGCP)); +#endif +#ifdef DEBUG0_1 + cout << "JNI read length "<& tc_array, unsigned numTestCases, double* likelihoodDoubleArray, + unsigned maxNumThreadsToUse) +{ +#ifdef DO_REPEAT_PROFILING + for(unsigned i=0;i<10;++i) +#endif + { +#pragma omp parallel for schedule (dynamic,10000) num_threads(maxNumThreadsToUse) + for(unsigned tc_idx=0;tc_idx > >& readBasesArrayVector) +{ + //Release read arrays first + for(int i=readBasesArrayVector.size()-1;i>=0;--i)//note the order - reverse of GET + { + for(int j=readBasesArrayVector[i].size()-1;j>=0;--j) + RELEASE_BYTE_ARRAY_ELEMENTS(readBasesArrayVector[i][j].first, readBasesArrayVector[i][j].second, JNI_RO_RELEASE_MODE); + readBasesArrayVector[i].clear(); + } + readBasesArrayVector.clear(); +} + + +#ifdef DO_WARMUP +uint64_t g_sum = 0; +#endif +//JNI function to invoke compute_full_prob_avx +//readDataArray - array of JNIReadDataHolderClass objects which contain the readBases, readQuals etc +//haplotypeDataArray - array of JNIHaplotypeDataHolderClass objects which contain the haplotypeBases +//likelihoodArray - array of doubles to return results back to Java. Memory allocated by Java prior to JNI call +//maxNumThreadsToUse - Max number of threads that OpenMP can use for the HMM computation +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods + (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, + jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +{ +#ifdef DEBUG0_1 + cout << "JNI numReads "< tc_array; + tc_array.clear(); + tc_array.resize(numTestCases); + //Store read arrays for release later + vector > > readBasesArrayVector; + readBasesArrayVector.clear(); + readBasesArrayVector.resize(numReads); +#ifdef DUMP_TO_SANDBOX + g_load_time_initializer.open_sandbox(); +#endif +#ifdef DO_PROFILING + get_time(&start_time); +#endif + //Copy byte array references from Java memory into vector of testcase structs + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeTestcasesVector(env, + numReads, numHaplotypes, readDataArray, readBasesArrayVector, tc_array); +#ifdef DO_PROFILING + g_load_time_initializer.m_data_transfer_time += diff_time(start_time); +#endif + + //Get double array where results are stored (to pass back to java) + jdouble* likelihoodDoubleArray = (jdouble*)GET_DOUBLE_ARRAY_ELEMENTS(likelihoodArray, &is_copy); +#ifdef ENABLE_ASSERTIONS + assert(likelihoodDoubleArray && "likelihoodArray is NULL"); + assert(env->GetArrayLength(likelihoodArray) == numTestCases); +#endif +#ifdef DO_WARMUP //ignore - only for crazy profiling + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + for(unsigned i=0;iGetArrayLength(haplotypeBasesArrayVector[i].first); + for(unsigned j=0;jGetArrayLength(readBasesArrayVector[i][j].first); + for(unsigned k=0;k >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + //Now release haplotype arrays + for(int j=haplotypeBasesArrayVector.size()-1;j>=0;--j) //note the order - reverse of GET + { + RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBasesArrayVector[j].first, haplotypeBasesArrayVector[j].second, JNI_RO_RELEASE_MODE); + env->DeleteGlobalRef(haplotypeBasesArrayVector[j].first); //free the global reference + } + haplotypeBasesArrayVector.clear(); + g_haplotypeBasesLengths.clear(); +} + + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose + (JNIEnv* env, jobject thisObject) +{ +#ifdef DO_PROFILING + g_load_time_initializer.print_profiling(); +#endif +#ifdef DEBUG + g_load_time_initializer.debug_close(); +#endif +} + diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h new file mode 100644 index 000000000..d820b4b26 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h @@ -0,0 +1,104 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM */ + +#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM +#define _Included_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM +#ifdef __cplusplus +extern "C" { +#endif +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_TRISTATE_CORRECTION +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_TRISTATE_CORRECTION 3.0 +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToMatch +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToMatch 0L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_indelToMatch +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_indelToMatch 1L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToInsertion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToInsertion 2L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_insertionToInsertion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_insertionToInsertion 3L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToDeletion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToDeletion 4L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_deletionToDeletion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_deletionToDeletion 5L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_sse42Mask +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_sse42Mask 1LL +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_avxMask +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_avxMask 2LL +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask + (JNIEnv *, jobject, jclass, jclass, jlong); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniInitializeHaplotypes + * Signature: (I[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniFinalizeRegion + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniComputeLikelihoods + * Signature: (II[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIReadDataHolderClass;[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc new file mode 100644 index 000000000..7ff219b88 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc @@ -0,0 +1,70 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +//#define DEBUG 1 +//#define DEBUG0_1 1 +//#define DEBUG3 1 +#include "headers.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +using namespace std; + +int main(int argc, char** argv) +{ +#define BATCH_SIZE 10000 + if(argc < 2) + { + cerr << "Needs path to input file as argument\n"; + exit(0); + } + bool use_old_read_testcase = false; + if(argc >= 3 && string(argv[2]) == "1") + use_old_read_testcase = true; + unsigned chunk_size = 10000; + bool do_check = true; + uint64_t mask = ~(0ull); + for(int i=3;i +#include +#include + + +void CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]) { + + const int maskBitCnt = MAIN_TYPE_SIZE ; + + for (int vi=0; vi < numMaskVecs; ++vi) { + for (int rs=0; rs < NUM_DISTINCT_CHARS; ++rs) { + maskArr[vi][rs] = 0 ; + } + maskArr[vi][AMBIG_CHAR] = MASK_ALL_ONES ; + } + + for (int col=1; col < COLS; ++col) { + int mIndex = (col-1) / maskBitCnt ; + int mOffset = (col-1) % maskBitCnt ; + MASK_TYPE bitMask = ((MASK_TYPE)0x1) << (maskBitCnt-1-mOffset) ; + + char hapChar = ConvertChar::get(tc.hap[col-1]); + + if (hapChar == AMBIG_CHAR) { + for (int ci=0; ci < NUM_DISTINCT_CHARS; ++ci) + maskArr[mIndex][ci] |= bitMask ; + } + + maskArr[mIndex][hapChar] |= bitMask ; + // bit corresponding to col 1 will be the MSB of the mask 0 + // bit corresponding to col 2 will be the MSB-1 of the mask 0 + // ... + // bit corresponding to col 32 will be the LSB of the mask 0 + // bit corresponding to col 33 will be the MSB of the mask 1 + // ... + } + +} + +void CONCAT(CONCAT(init_masks_for_row_,SIMD_ENGINE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess) { + + for (int ri=0; ri < numRowsToProcess; ++ri) { + rsArr[ri] = ConvertChar::get(tc.rs[ri+beginRowIndex-1]) ; + } + + for (int ei=0; ei < AVX_LENGTH; ++ei) { + lastMaskShiftOut[ei] = 0 ; + } +} + +#define SET_MASK_WORD(__dstMask, __srcMask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ + MASK_TYPE __bitMask = (((MASK_TYPE)0x1) << __shiftBy) - 1 ; \ + MASK_TYPE __nextShiftOut = (__srcMask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ + __dstMask = (__srcMask >> __shiftBy) | __lastShiftOut ; \ + __lastShiftOut = __nextShiftOut ; \ +} + + +void CONCAT(CONCAT(update_masks_for_cols_,SIMD_ENGINE), PRECISION)(int maskIndex, BITMASK_VEC& bitMaskVec, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, int maskBitCnt) { + + for (int ei=0; ei < AVX_LENGTH/2; ++ei) { + SET_MASK_WORD(bitMaskVec.getLowEntry(ei), maskArr[maskIndex][rsArr[ei]], + lastMaskShiftOut[ei], ei, maskBitCnt) ; + + int ei2 = ei + AVX_LENGTH/2 ; // the second entry index + SET_MASK_WORD(bitMaskVec.getHighEntry(ei), maskArr[maskIndex][rsArr[ei2]], + lastMaskShiftOut[ei2], ei2, maskBitCnt) ; + } + +} + + +inline void CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (BITMASK_VEC& bitMaskVec, SIMD_TYPE& distm, SIMD_TYPE& _1_distm, SIMD_TYPE& distmChosen) { + + distmChosen = VEC_BLENDV(distm, _1_distm, bitMaskVec.getCombinedMask()) ; + + bitMaskVec.shift_left_1bit() ; +} + +/* + * This function: + * 1- Intializes probability values p_MM, p_XX, P_YY, p_MX, p_GAPM and pack them into vectors (SSE or AVX) + * 2- Precompute parts of "distm" which only depeneds on a row number and pack it into vector + */ + +template void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, SIMD_TYPE *p_MM, SIMD_TYPE *p_GAPM, SIMD_TYPE *p_MX, SIMD_TYPE *p_XX, SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, SIMD_TYPE *distm1D) +{ + NUMBER zero = ctx._(0.0); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + for (int s=0;si[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + + //*(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(*(ptr_p_MM+r-1), _i, _d); + *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; + *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; + *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; + *(ptr_p_MY+r-1) = ctx.ph2pr[_d]; + *(ptr_p_YY+r-1) = ctx.ph2pr[_c]; + } + + NUMBER *ptr_distm1D = (NUMBER *)distm1D; + for (int r = 1; r < ROWS; r++) + { + int _q = tc->q[r-1] & 127; + ptr_distm1D[r-1] = ctx.ph2pr[_q]; + } +} + +/* + * This function handles pre-stripe computation: + * 1- Retrieve probaility vectors from memory + * 2- Initialize M, X, Y vectors with all 0's (for the first stripe) and shifting the last row from previous stripe for the rest + */ + +template inline void CONCAT(CONCAT(stripeINITIALIZATION,SIMD_ENGINE), PRECISION)( + int stripeIdx, Context ctx, testcase *tc, SIMD_TYPE &pGAPM, SIMD_TYPE &pMM, SIMD_TYPE &pMX, SIMD_TYPE &pXX, SIMD_TYPE &pMY, SIMD_TYPE &pYY, + SIMD_TYPE &rs, UNION_TYPE &rsN, SIMD_TYPE &distm, SIMD_TYPE &_1_distm, SIMD_TYPE *distm1D, SIMD_TYPE N_packed256, SIMD_TYPE *p_MM , SIMD_TYPE *p_GAPM , + SIMD_TYPE *p_MX, SIMD_TYPE *p_XX , SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM) +{ + int i = stripeIdx; + pGAPM = p_GAPM[i]; + pMM = p_MM[i]; + pMX = p_MX[i]; + pXX = p_XX[i]; + pMY = p_MY[i]; + pYY = p_YY[i]; + + NUMBER zero = ctx._(0.0); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + UNION_TYPE packed3; packed3.d = VEC_SET1_VAL(3.0); + + distm = distm1D[i]; + _1_distm = VEC_SUB(packed1.d, distm); + + distm = VEC_DIV(distm, packed3.d); + + /* initialize M_t_2, M_t_1, X_t_2, X_t_1, Y_t_2, Y_t_1 */ + M_t_2.d = VEC_SET1_VAL(zero); + X_t_2.d = VEC_SET1_VAL(zero); + + if (i==0) { + M_t_1.d = VEC_SET1_VAL(zero); + X_t_1.d = VEC_SET1_VAL(zero); + Y_t_2.d = VEC_SET_LSE(init_Y); + Y_t_1.d = VEC_SET1_VAL(zero); + } + else { + X_t_1.d = VEC_SET_LSE(shiftOutX[AVX_LENGTH]); + M_t_1.d = VEC_SET_LSE(shiftOutM[AVX_LENGTH]); + Y_t_2.d = VEC_SET1_VAL(zero); + Y_t_1.d = VEC_SET1_VAL(zero); + } + M_t_1_y = M_t_1; +} + +/* + * This function is the main compute kernel to compute M, X and Y + */ + +inline void CONCAT(CONCAT(computeMXY,SIMD_ENGINE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, + UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, + SIMD_TYPE pMM, SIMD_TYPE pGAPM, SIMD_TYPE pMX, SIMD_TYPE pXX, SIMD_TYPE pMY, SIMD_TYPE pYY, SIMD_TYPE distmSel) +{ + /* Compute M_t <= distm * (p_MM*M_t_2 + p_GAPM*X_t_2 + p_GAPM*Y_t_2) */ + M_t.d = VEC_MUL(VEC_ADD(VEC_ADD(VEC_MUL(M_t_2.d, pMM), VEC_MUL(X_t_2.d, pGAPM)), VEC_MUL(Y_t_2.d, pGAPM)), distmSel); + //M_t.d = VEC_MUL( VEC_ADD(VEC_MUL(M_t_2.d, pMM), VEC_MUL(VEC_ADD(X_t_2.d, Y_t_2.d), pGAPM)), distmSel); + + M_t_y = M_t; + + /* Compute X_t */ + X_t.d = VEC_ADD(VEC_MUL(M_t_1.d, pMX) , VEC_MUL(X_t_1.d, pXX)); + + /* Compute Y_t */ + Y_t.d = VEC_ADD(VEC_MUL(M_t_1_y.d, pMY) , VEC_MUL(Y_t_1.d, pYY)); +} + +/* + * This is the main compute function. It operates on the matrix in s stripe manner. + * The stripe height is determined by the SIMD engine type. + * Stripe height: "AVX float": 8, "AVX double": 4, "SSE float": 4, "SSE double": 2 + * For each stripe the operations are anti-diagonal based. + * Each anti-diagonal (M_t, Y_t, X_t) depends on the two previous anti-diagonals (M_t_2, X_t_2, Y_t_2, M_t_1, X_t_1, Y_t_1). + * Each stripe (except the fist one) depends on the last row of the previous stripe. + * The last stripe computation handles the addition of the last row of M and X, that's the reason for loop spliting. + */ + +template NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL) +{ + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + int MAVX_COUNT = (ROWS+AVX_LENGTH-1)/AVX_LENGTH; + + /* Probaility arrays */ + SIMD_TYPE p_MM [MAVX_COUNT], p_GAPM [MAVX_COUNT], p_MX [MAVX_COUNT]; + SIMD_TYPE p_XX [MAVX_COUNT], p_MY [MAVX_COUNT], p_YY [MAVX_COUNT]; + + /* For distm precomputation */ + SIMD_TYPE distm1D[MAVX_COUNT]; + + /* Carries the values from each stripe to the next stripe */ + NUMBER shiftOutM[ROWS+COLS+AVX_LENGTH], shiftOutX[ROWS+COLS+AVX_LENGTH], shiftOutY[ROWS+COLS+AVX_LENGTH]; + + /* The vector to keep the anti-diagonals of M, X, Y*/ + /* Current: M_t, X_t, Y_t */ + /* Previous: M_t_1, X_t_1, Y_t_1 */ + /* Previous to previous: M_t_2, X_t_2, Y_t_2 */ + UNION_TYPE M_t, M_t_1, M_t_2, X_t, X_t_1, X_t_2, Y_t, Y_t_1, Y_t_2, M_t_y, M_t_1_y; + + /* Probality vectors */ + SIMD_TYPE pGAPM, pMM, pMX, pXX, pMY, pYY; + + struct timeval start, end; + NUMBER result_avx2; + Context ctx; + UNION_TYPE rs , rsN; + HAP_TYPE hap; + SIMD_TYPE distmSel, distmChosen ; + SIMD_TYPE distm, _1_distm; + + int r, c; + NUMBER zero = ctx._(0.0); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + SIMD_TYPE N_packed256 = VEC_POPCVT_CHAR('N'); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + int remainingRows = (ROWS-1) % AVX_LENGTH; + int stripe_cnt = ((ROWS-1) / AVX_LENGTH) + (remainingRows!=0); + + const int maskBitCnt = MAIN_TYPE_SIZE ; + const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function + + /* Mask precomputation for distm*/ + MASK_TYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; + CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(*tc, COLS, numMaskVecs, maskArr) ; + + char rsArr[AVX_LENGTH] ; + MASK_TYPE lastMaskShiftOut[AVX_LENGTH] ; + + /* Precompute initialization for probabilities and shift vector*/ + CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(ROWS, COLS, shiftOutM, shiftOutX, shiftOutY, + ctx, tc, p_MM, p_GAPM, p_MX, p_XX, p_MY, p_YY, distm1D); + + for (int i=0;i(&tc[b]); + +#ifdef RUN_HYBRID +#define MIN_ACCEPTED 1e-28f + if (result_avxf < MIN_ACCEPTED) { + count++; + result_avxd = CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), d)(&tc[b]); + result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); + } + else + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif + +#ifndef RUN_HYBRID + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif + } + aggregateTimeCompute += (getCurrClk() - lastClk) ; + lastClk = getCurrClk() ; + for (int b=0;b(testcase* tc, double* nextlog); +template float compute_full_prob_sses(testcase* tc, float* nextlog); diff --git a/public/VectorPairHMM/src/main/c++/template.h b/public/VectorPairHMM/src/main/c++/template.h new file mode 100644 index 000000000..ce4dbfc86 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/template.h @@ -0,0 +1,320 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef TEMPLATES_H_ +#define TEMPLATES_H_ + +#include "headers.h" + +#define MM 0 +#define GapM 1 +#define MX 2 +#define XX 3 +#define MY 4 +#define YY 5 + +//#define MROWS 500 +//#define MCOLS 1000 + +#define CAT(X,Y) X####Y +#define CONCAT(X,Y) CAT(X,Y) + +#define ALIGNED __attribute__((aligned(32))) + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256 ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED float ALIGNED f[8]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_F ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m128 ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED float ALIGNED f[4]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_F128 ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128 vecf ; + uint32_t masks[4] ; +} MaskVec_F ; + +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint32_t masks[2] ; +} MaskVec_F128 ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128 ALIGNED f; +} ALIGNED IF_128f ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int ALIGNED i; + ALIGNED float ALIGNED f; +} ALIGNED IF_32 ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256d ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED double ALIGNED f[4]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_D ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m128d ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED double ALIGNED f[2]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_D128 ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128d vecf ; + uint64_t masks[2] ; +} MaskVec_D ; + +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint64_t masks[1] ; +} MaskVec_D128 ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128d ALIGNED f; +} ALIGNED IF_128d ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int64_t ALIGNED i; + ALIGNED double ALIGNED f; +} ALIGNED IF_64 ALIGNED; + + +#define MAX_QUAL 254 +#define MAX_JACOBIAN_TOLERANCE 8.0 +#define JACOBIAN_LOG_TABLE_STEP 0.0001 +#define JACOBIAN_LOG_TABLE_INV_STEP (1.0 / JACOBIAN_LOG_TABLE_STEP) +#define MAXN 70000 +#define LOG10_CACHE_SIZE (4*MAXN) // we need to be able to go up to 2*(2N) when calculating some of the coefficients +#define JACOBIAN_LOG_TABLE_SIZE ((int) (MAX_JACOBIAN_TOLERANCE / JACOBIAN_LOG_TABLE_STEP) + 1) + +template +struct ContextBase +{ + public: + NUMBER ph2pr[128]; + NUMBER INITIAL_CONSTANT; + NUMBER LOG10_INITIAL_CONSTANT; + NUMBER RESULT_THRESHOLD; + + static bool staticMembersInitializedFlag; + static NUMBER jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; + static NUMBER matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; + + static void initializeStaticMembers() + { + if(!staticMembersInitializedFlag) + { + //Order of calls important - Jacobian first, then MatchToMatch + initializeJacobianLogTable(); + initializeMatchToMatchProb(); + staticMembersInitializedFlag = true; + } + } + + static void deleteStaticMembers() + { + if(staticMembersInitializedFlag) + { + staticMembersInitializedFlag = false; + } + } + + //Called only once during library load - don't bother to optimize with single precision fp + static void initializeJacobianLogTable() + { + for (int k = 0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { + jacobianLogTable[k] = (NUMBER)(log10(1.0 + pow(10.0, -((double) k) * JACOBIAN_LOG_TABLE_STEP))); + } + } + + //Called only once per library load - don't bother optimizing with single fp + static void initializeMatchToMatchProb() + { + double LN10 = log(10); + double INV_LN10 = 1.0/LN10; + for (int i = 0, offset = 0; i <= MAX_QUAL; offset += ++i) + for (int j = 0; j <= i; j++) { + double log10Sum = approximateLog10SumLog10(-0.1*i, -0.1*j); + double matchToMatchLog10 = + log1p(-std::min(1.0,pow(10,log10Sum))) * INV_LN10; + matchToMatchProb[offset + j] = (NUMBER)(pow(10,matchToMatchLog10)); + } + } + //Called during computation - use single precision where possible + static int fastRound(NUMBER d) { + return (d > ((NUMBER)0.0)) ? (int) (d + ((NUMBER)0.5)) : (int) (d - ((NUMBER)0.5)); + } + //Called during computation - use single precision where possible + static NUMBER approximateLog10SumLog10(NUMBER small, NUMBER big) { + // make sure small is really the smaller value + if (small > big) { + NUMBER t = big; + big = small; + small = t; + } + + if (isinf(small) == -1 || isinf(big) == -1) + return big; + + NUMBER diff = big - small; + if (diff >= ((NUMBER)MAX_JACOBIAN_TOLERANCE)) + return big; + + // OK, so |y-x| < tol: we use the following identity then: + // we need to compute log10(10^x + 10^y) + // By Jacobian logarithm identity, this is equal to + // max(x,y) + log10(1+10^-abs(x-y)) + // we compute the second term as a table lookup with integer quantization + // we have pre-stored correction for 0,0.1,0.2,... 10.0 + int ind = fastRound((NUMBER)(diff * ((NUMBER)JACOBIAN_LOG_TABLE_INV_STEP))); // hard rounding + return big + jacobianLogTable[ind]; + } +}; + +template +struct Context : public ContextBase +{}; + +template<> +struct Context : public ContextBase +{ + Context():ContextBase() + { + for (int x = 0; x < 128; x++) + ph2pr[x] = pow(10.0, -((double)x) / 10.0); + + INITIAL_CONSTANT = ldexp(1.0, 1020.0); + LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); + RESULT_THRESHOLD = 0.0; + } + + double LOG10(double v){ return log10(v); } + inline double POW(double b, double e) { return pow(b,e); } + + static double _(double n){ return n; } + static double _(float n){ return ((double) n); } +}; + +template<> +struct Context : public ContextBase +{ + Context() : ContextBase() + { + for (int x = 0; x < 128; x++) + { + ph2pr[x] = powf(10.f, -((float)x) / 10.f); + } + + INITIAL_CONSTANT = ldexpf(1.f, 120.f); + LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); + RESULT_THRESHOLD = ldexpf(1.f, -110.f); + } + + float LOG10(float v){ return log10f(v); } + inline float POW(float b, float e) { return powf(b,e); } + + static float _(double n){ return ((float) n); } + static float _(float n){ return n; } +}; + +#define SET_MATCH_TO_MATCH_PROB(output, insQual, delQual) \ +{ \ + int minQual = delQual; \ + int maxQual = insQual; \ + if (insQual <= delQual) \ + { \ + minQual = insQual; \ + maxQual = delQual; \ + } \ + (output) = (MAX_QUAL < maxQual) ? \ + ((NUMBER)1.0) - ctx.POW(((NUMBER)10), ctx.approximateLog10SumLog10(((NUMBER)-0.1)*minQual, ((NUMBER)-0.1)*maxQual)) \ + : ctx.matchToMatchProb[((maxQual * (maxQual + 1)) >> 1) + minQual]; \ +} + +typedef struct +{ + int rslen, haplen; + /*int *q, *i, *d, *c;*/ + /*int q[MROWS], i[MROWS], d[MROWS], c[MROWS];*/ + char *q, *i, *d, *c; + char *hap, *rs; + int *ihap; + int *irs; +} testcase; + +int normalize(char c); +int read_testcase(testcase *tc, FILE* ifp=0); + + +#define MIN_ACCEPTED 1e-28f +#define NUM_DISTINCT_CHARS 5 +#define AMBIG_CHAR 4 + +class ConvertChar { + + static uint8_t conversionTable[255] ; + +public: + + static void init() { + assert (NUM_DISTINCT_CHARS == 5) ; + assert (AMBIG_CHAR == 4) ; + + conversionTable['A'] = 0 ; + conversionTable['C'] = 1 ; + conversionTable['T'] = 2 ; + conversionTable['G'] = 3 ; + conversionTable['N'] = 4 ; + } + + static inline uint8_t get(uint8_t input) { + return conversionTable[input] ; + } + +}; + + +#endif + + diff --git a/public/VectorPairHMM/src/main/c++/utils.cc b/public/VectorPairHMM/src/main/c++/utils.cc new file mode 100644 index 000000000..9f83cffa2 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/utils.cc @@ -0,0 +1,493 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "template.h" +#include "utils.h" +#include "vector_defs.h" +#include "LoadTimeInitializer.h" +using namespace std; + +//static members from ConvertChar +uint8_t ConvertChar::conversionTable[255]; +//Global function pointers in utils.h +float (*g_compute_full_prob_float)(testcase *tc, float* before_last_log) = 0; +double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log) = 0; +//Static members in ContextBase +bool ContextBase::staticMembersInitializedFlag = false; +double ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +double ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; +bool ContextBase::staticMembersInitializedFlag = false; +float ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +float ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; + + +bool is_avx_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 28)&1) == 1; +} + +bool is_sse41_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 19)&1) == 1; +} + +bool is_sse42_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 20)&1) == 1; +} + +uint64_t get_machine_capabilities() +{ + uint64_t machine_mask = 0ull; + if(is_avx_supported()) + machine_mask |= (1 << AVX_CUSTOM_IDX); + if(is_sse42_supported()) + machine_mask |= (1 << SSE42_CUSTOM_IDX); + if(is_sse41_supported()) + machine_mask |= (1 << SSE41_CUSTOM_IDX); + return machine_mask; +} + +void initialize_function_pointers(uint64_t mask) +{ + //mask = 0ull; + //mask = (1 << SSE41_CUSTOM_IDX); + if(is_avx_supported() && (mask & (1<< AVX_CUSTOM_IDX))) + { + cout << "Using AVX accelerated implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob_avxs; + g_compute_full_prob_double = compute_full_prob_avxd; + } + else + if(is_sse41_supported() && (mask & ((1<< SSE41_CUSTOM_IDX) | (1<; + g_compute_full_prob_double = compute_full_prob_ssed; + } + else + { + cout << "Using un-vectorized C++ implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob; + g_compute_full_prob_double = compute_full_prob; + } +} + +int normalize(char c) +{ + return ((int) (c - 33)); +} + +int read_testcase(testcase *tc, FILE* ifp) +{ + char *q, *i, *d, *c, *line = NULL; + int _q, _i, _d, _c; + int x, size = 0; + ssize_t read; + + + read = getline(&line, (size_t *) &size, ifp == 0 ? stdin : ifp); + if (read == -1) + { + free(line); + return -1; + } + + + tc->hap = (char *) malloc(size); + tc->rs = (char *) malloc(size); + q = (char *) malloc(size); + i = (char *) malloc(size); + d = (char *) malloc(size); + c = (char *) malloc(size); + + if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) + return -1; + + + tc->haplen = strlen(tc->hap); + tc->rslen = strlen(tc->rs); + assert(strlen(q) == tc->rslen); + assert(strlen(i) == tc->rslen); + assert(strlen(d) == tc->rslen); + assert(strlen(c) == tc->rslen); + //assert(tc->rslen < MROWS); + //tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); + //tc->irs = (int *) malloc(tc->rslen*sizeof(int)); + + tc->q = (char *) malloc(sizeof(char) * tc->rslen); + tc->i = (char *) malloc(sizeof(char) * tc->rslen); + tc->d = (char *) malloc(sizeof(char) * tc->rslen); + tc->c = (char *) malloc(sizeof(char) * tc->rslen); + + for (x = 0; x < tc->rslen; x++) + { + _q = normalize(q[x]); + _i = normalize(i[x]); + _d = normalize(d[x]); + _c = normalize(c[x]); + tc->q[x] = (_q < 6) ? 6 : _q; + //tc->q[x] = _q; + tc->i[x] = _i; + tc->d[x] = _d; + tc->c[x] = _c; + //tc->irs[x] = tc->rs[x]; + } + //for (x = 0; x < tc->haplen; x++) + //tc->ihap[x] = tc->hap[x]; + + free(q); + free(i); + free(d); + free(c); + free(line); + + + + return 0; +} + +unsigned MAX_LINE_LENGTH = 65536; +int convToInt(std::string s) +{ + int i; + std::istringstream strin(s); + strin >> i; + return i; +} + +void tokenize(std::ifstream& fptr, std::vector& tokens) +{ + int i = 0; + std::string tmp; + std::vector myVec; + vector line; + line.clear(); + line.resize(MAX_LINE_LENGTH); + vector tmpline; + tmpline.clear(); + tmpline.resize(MAX_LINE_LENGTH); + myVec.clear(); + + while(!fptr.eof()) + { + i = 0; + bool still_read_line = true; + unsigned line_position = 0; + while(still_read_line) + { + fptr.getline(&(tmpline[0]), MAX_LINE_LENGTH); + if(line_position + MAX_LINE_LENGTH > line.size()) + line.resize(2*line.size()); + for(unsigned i=0;i> std::skipws >> tmp; + if(tmp != "") + { + myVec.push_back(tmp); + ++i; + //std::cout < 0) + break; + } + tokens.clear(); + //std::cout << "Why "< tokens; + tokens.clear(); + tokenize(fptr, tokens); + if(tokens.size() == 0) + return -1; + tc->hap = new char[tokens[0].size()+2]; + tc->haplen = tokens[0].size(); + memcpy(tc->hap, tokens[0].c_str(), tokens[0].size()); + tc->rs = new char[tokens[1].size()+2]; + tc->rslen = tokens[1].size(); + tc->q = new char[tc->rslen]; + tc->i = new char[tc->rslen]; + tc->d = new char[tc->rslen]; + tc->c = new char[tc->rslen]; + //cout << "Lengths "<haplen <<" "<rslen<<"\n"; + memcpy(tc->rs, tokens[1].c_str(),tokens[1].size()); + assert(tokens.size() == 2 + 4*(tc->rslen)); + //assert(tc->rslen < MROWS); + for(unsigned j=0;jrslen;++j) + tc->q[j] = (char)convToInt(tokens[2+0*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->i[j] = (char)convToInt(tokens[2+1*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->d[j] = (char)convToInt(tokens[2+2*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->c[j] = (char)convToInt(tokens[2+3*tc->rslen+j]); + + if(reformat) + { + ofstream ofptr; + ofptr.open("reformat/debug_dump.txt",first_call ? ios::out : ios::app); + assert(ofptr.is_open()); + ofptr << tokens[0] << " "; + ofptr << tokens[1] << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->q[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->i[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->d[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->c[j]+33)); + ofptr << " 0 false\n"; + + ofptr.close(); + first_call = false; + } + + + return tokens.size(); +} + +double getCurrClk() { + struct timeval tv ; + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} + +inline unsigned long long rdtsc(void) +{ + unsigned hi, lo; + __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); + return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 ); +} + +void get_time(struct timespec* store_struct) +{ + clock_gettime(CLOCK_REALTIME, store_struct); +} + +uint64_t diff_time(struct timespec& prev_time) +{ + struct timespec curr_time; + clock_gettime(CLOCK_REALTIME, &curr_time); + return (uint64_t)((curr_time.tv_sec-prev_time.tv_sec)*1000000000+(curr_time.tv_nsec-prev_time.tv_nsec)); +} + + +#ifdef USE_PAPI +#include "papi.h" +#define NUM_PAPI_COUNTERS 4 +#endif + +void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size, bool do_check) +{ + FILE* fptr = 0; + ifstream ifptr; + if(use_old_read_testcase) + { + fptr = fopen(filename,"r"); + assert(fptr); + } + else + { + ifptr.open(filename); + assert(ifptr.is_open()); + } + vector tc_vector; + tc_vector.clear(); + testcase tc; + uint64_t vector_compute_time = 0; + uint64_t baseline_compute_time = 0; + uint64_t num_double_calls = 0; + unsigned num_testcases = 0; + bool all_ok = do_check ? true : false; +#ifdef USE_PAPI + uint32_t all_mask = (0); + uint32_t no_usr_mask = (1 << 16); //bit 16 user mode, bit 17 kernel mode + uint32_t no_kernel_mask = (1 << 17); //bit 16 user mode, bit 17 kernel mode + PAPI_num_counters(); + int events[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + char* eventnames[NUM_PAPI_COUNTERS]= { "cycles", "itlb_walk_cycles", "dtlb_load_walk_cycles", "dtlb_store_walk_cycles" }; + assert(PAPI_event_name_to_code("UNHALTED_REFERENCE_CYCLES:u=1:k=1",&(events[0])) == PAPI_OK); + assert(PAPI_event_name_to_code("ITLB_MISSES:WALK_DURATION", &(events[1])) == PAPI_OK); + assert(PAPI_event_name_to_code("DTLB_LOAD_MISSES:WALK_DURATION", &(events[2])) == PAPI_OK); + assert(PAPI_event_name_to_code("DTLB_STORE_MISSES:WALK_DURATION", &(events[3])) == PAPI_OK); + long long values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + long long accum_values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; +#endif + while(1) + { + int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); + if(break_value >= 0) + tc_vector.push_back(tc); + if(tc_vector.size() == BATCH_SIZE || (break_value < 0 && tc_vector.size() > 0)) + { + vector results_vec; + vector baseline_results_vec; + results_vec.clear(); + baseline_results_vec.clear(); + results_vec.resize(tc_vector.size()); + baseline_results_vec.resize(tc_vector.size()); + struct timespec start_time; +#ifdef USE_PAPI + assert(PAPI_start_counters(events, NUM_PAPI_COUNTERS) == PAPI_OK); +#endif + get_time(&start_time); +#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) +#ifdef DO_REPEAT_PROFILING + for(unsigned z=0;z<10;++z) +#endif + { + for(unsigned i=0;i(&tc); + baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); + baseline_results_vec[i] = baseline_result; + } + baseline_compute_time += diff_time(start_time); + for(unsigned i=0;i 1e-5 && rel_error > 1e-5) + { + cout << std::scientific << baseline_result << " "< +std::string to_string(T obj) +{ + std::stringstream ss; + std::string ret_string; + ss.clear(); + ss << std::scientific << obj; + ss >> ret_string; + ss.clear(); + return ret_string; +} +void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); + +int read_mod_testcase(std::ifstream& fptr, testcase* tc, bool reformat=false); + +bool is_avx_supported(); +bool is_sse42_supported(); +extern float (*g_compute_full_prob_float)(testcase *tc, float *before_last_log); +extern double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log); +void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline); +template +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); +double getCurrClk(); +void get_time(struct timespec* x); +uint64_t diff_time(struct timespec& prev_time); + +//bit 0 is sse4.2, bit 1 is AVX +enum ProcessorCapabilitiesEnum +{ + SSE41_CUSTOM_IDX=0, + SSE42_CUSTOM_IDX, + AVX_CUSTOM_IDX +}; +#define ENABLE_ALL_HARDWARE_FEATURES 0xFFFFFFFFFFFFFFFFull +uint64_t get_machine_capabilities(); +void initialize_function_pointers(uint64_t mask=ENABLE_ALL_HARDWARE_FEATURES); +void do_compute(char* filename, bool use_old_read_testcase=true, unsigned chunk_size=10000, bool do_check=true); + +//#define DO_WARMUP +//#define DO_REPEAT_PROFILING +//#define DUMP_COMPUTE_VALUES 1 +#define BATCH_SIZE 10000 +#define RUN_HYBRID + +#endif diff --git a/public/VectorPairHMM/src/main/c++/vector_defs.h b/public/VectorPairHMM/src/main/c++/vector_defs.h new file mode 100644 index 000000000..2aca9565f --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/vector_defs.h @@ -0,0 +1,55 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX +#undef SIMD_ENGINE_SSE + +#define SIMD_ENGINE avx +#define SIMD_ENGINE_AVX + +#include "define-float.h" +#include "vector_function_prototypes.h" + +#include "define-double.h" +#include "vector_function_prototypes.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX + +#define SIMD_ENGINE sse +#define SIMD_ENGINE_SSE + + +#include "define-sse-float.h" +#include "vector_function_prototypes.h" + +#include "define-sse-double.h" +#include "vector_function_prototypes.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX +#undef SIMD_ENGINE_SSE + diff --git a/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h b/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h new file mode 100644 index 000000000..c0fddc394 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h @@ -0,0 +1,44 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +inline void CONCAT(CONCAT(_vector_shift,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); +inline void CONCAT(CONCAT(_vector_shift_last,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); +inline void CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); +inline void CONCAT(CONCAT(init_masks_for_row_,SIMD_ENGINE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess); +inline void CONCAT(CONCAT(update_masks_for_cols_,SIMD_ENGINE), PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt); +inline void CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, SIMD_TYPE& distm, SIMD_TYPE& _1_distm, SIMD_TYPE& distmChosen); +template inline void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, SIMD_TYPE *p_MM, SIMD_TYPE *p_GAPM, SIMD_TYPE *p_MX, SIMD_TYPE *p_XX, SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, SIMD_TYPE *distm1D); +template inline void CONCAT(CONCAT(stripINITIALIZATION,SIMD_ENGINE), PRECISION)( + int stripIdx, Context ctx, testcase *tc, SIMD_TYPE &pGAPM, SIMD_TYPE &pMM, SIMD_TYPE &pMX, SIMD_TYPE &pXX, SIMD_TYPE &pMY, SIMD_TYPE &pYY, + SIMD_TYPE &rs, UNION_TYPE &rsN, SIMD_TYPE &distm, SIMD_TYPE &_1_distm, SIMD_TYPE *distm1D, SIMD_TYPE N_packed256, SIMD_TYPE *p_MM , SIMD_TYPE *p_GAPM , + SIMD_TYPE *p_MX, SIMD_TYPE *p_XX , SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM); +inline SIMD_TYPE CONCAT(CONCAT(computeDISTM,SIMD_ENGINE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, SIMD_TYPE rs, UNION_TYPE rsN, SIMD_TYPE N_packed256, + SIMD_TYPE distm, SIMD_TYPE _1_distm); +inline void CONCAT(CONCAT(computeMXY,SIMD_ENGINE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, + UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, + SIMD_TYPE pMM, SIMD_TYPE pGAPM, SIMD_TYPE pMX, SIMD_TYPE pXX, SIMD_TYPE pMY, SIMD_TYPE pYY, SIMD_TYPE distmSel); +template NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL); + diff --git a/public/sting-root/pom.xml b/public/sting-root/pom.xml index 171eb7620..549a99ae6 100644 --- a/public/sting-root/pom.xml +++ b/public/sting-root/pom.xml @@ -335,7 +335,11 @@ maven-assembly-plugin 2.4 - + + org.apache.maven.plugins + maven-enforcer-plugin + 1.3.1 + + ${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + display-info + + validate + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + + exec + + compile + + make + src/main/c++ + + ${java.home} + ${project.build.directory} + + + + + + + + + org.apache.maven.plugins + maven-install-plugin + + true + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + default-install + + copy-resources + + install + + ${pairhmm.resources.directory} + + + ${project.build.directory} + + **/* + + + + + + + + + + + com.google.code.sortpom + maven-sortpom-plugin + + false + custom_1 + \n + ${sourceEncoding} + true + scope + 4 + false + + + + + diff --git a/public/VectorPairHMM/src/main/c++/.gitignore b/public/VectorPairHMM/src/main/c++/.gitignore new file mode 100644 index 000000000..d791ffd80 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/.gitignore @@ -0,0 +1,16 @@ +.svn +*.o +*.so +tests +.deps +hmm_Mohammad +pairhmm-template-main +*.swp +*.class +checker +reformat +subdir_checkout.sh +avx/ +sse/ +triplicate.sh + diff --git a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc new file mode 100644 index 000000000..0e3026f65 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc @@ -0,0 +1,206 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "LoadTimeInitializer.h" +#include "utils.h" +using namespace std; +char* LoadTimeInitializerStatsNames[] = +{ + "num_regions", + "num_reads", + "num_haplotypes", + "num_testcases", + "num_double_invocations", + "haplotype_length", + "readlength", + "product_read_length_haplotype_length", + "dummy" +}; + +LoadTimeInitializer g_load_time_initializer; + +LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loaded +{ + ConvertChar::init(); +#ifndef DISABLE_FTZ + //Very important to get good performance on Intel processors + //Function: enabling FTZ converts denormals to 0 in hardware + //Denormals cause microcode to insert uops into the core causing big slowdown + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + cout << "FTZ enabled - may decrease accuracy if denormal numbers encountered\n"; +#else + cout << "FTZ is not set - may slow down performance if denormal numbers encountered\n"; +#endif + //Profiling: times for compute and transfer (either bytes copied or pointers copied) + m_compute_time = 0; + m_data_transfer_time = 0; + m_bytes_copied = 0; + + //Initialize profiling counters + for(unsigned i=0;i::initializeStaticMembers(); + Context::initializeStaticMembers(); + + cout.flush(); +} + +void LoadTimeInitializer::print_profiling() +{ + double mean = 0; + double variance = 0; + uint64_t denominator = 1; + cout << "Time spent in compute_testcases "< C++) "<::iterator mI = m_filename_to_fptr.find(filename); + ofstream* fptr = 0; + if(mI == m_filename_to_fptr.end()) + { + m_filename_to_fptr[filename] = new ofstream(); + fptr = m_filename_to_fptr[filename]; + //File never seen before + if(m_written_files_set.find(filename) == m_written_files_set.end()) + { + to_append = false; + m_written_files_set.insert(filename); + } + fptr->open(filename.c_str(), to_append ? ios::app : ios::out); + assert(fptr->is_open()); + } + else + fptr = (*mI).second; + //ofstream fptr; + //fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); + (*fptr) << s; + if(add_newline) + (*fptr) << "\n"; + //fptr.close(); +} +void LoadTimeInitializer::debug_close() +{ + for(map::iterator mB = m_filename_to_fptr.begin(), mE = m_filename_to_fptr.end(); + mB != mE;mB++) + { + (*mB).second->close(); + delete (*mB).second; + } + m_filename_to_fptr.clear(); +} + +void LoadTimeInitializer::dump_sandbox(testcase& tc, unsigned tc_idx, unsigned numReads, unsigned numHaplotypes) +{ + unsigned haplotypeLength = tc.haplen; + unsigned readLength = tc.rslen; + ofstream& dumpFptr = m_sandbox_fptr; + for(unsigned k=0;k +#include "template.h" + +enum LoadTimeInitializerStatsEnum +{ + NUM_REGIONS_IDX=0, + NUM_READS_IDX, + NUM_HAPLOTYPES_IDX, + NUM_TESTCASES_IDX, + NUM_DOUBLE_INVOCATIONS_IDX, + HAPLOTYPE_LENGTH_IDX, + READ_LENGTH_IDX, + PRODUCT_READ_LENGTH_HAPLOTYPE_LENGTH_IDX, + TOTAL_NUMBER_STATS +}; +extern char* LoadTimeInitializerStatsNames[]; + +class LoadTimeInitializer +{ + public: + LoadTimeInitializer(); //will be called when library is loaded + ~LoadTimeInitializer() + { + delete m_buffer; + } + void print_profiling(); + void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); + void debug_close(); + + void dump_sandbox(testcase& tc, unsigned tc_idx, unsigned numReads, unsigned numHaplotypes); + void open_sandbox() { m_sandbox_fptr.open("sandbox.txt", std::ios::app); } + void close_sandbox() { m_sandbox_fptr.close(); } + + jfieldID m_readBasesFID; + jfieldID m_readQualsFID; + jfieldID m_insertionGOPFID; + jfieldID m_deletionGOPFID; + jfieldID m_overallGCPFID; + jfieldID m_haplotypeBasesFID; + //profiling - update stats + void update_stat(LoadTimeInitializerStatsEnum stat_idx, uint64_t value); + //timing in nanoseconds + uint64_t m_compute_time; + uint64_t m_data_transfer_time; + //bytes copied + uint64_t m_bytes_copied; + unsigned get_buffer_size() { return m_buffer_size; } + char* get_buffer() { return (char*)m_buffer; } + private: + std::map m_filename_to_fptr; + std::set m_written_files_set; + std::ofstream m_sandbox_fptr; + //used to compute various stats + uint64_t m_sum_stats[TOTAL_NUMBER_STATS]; + double m_sum_square_stats[TOTAL_NUMBER_STATS]; + uint64_t m_min_stats[TOTAL_NUMBER_STATS]; + uint64_t m_max_stats[TOTAL_NUMBER_STATS]; + unsigned m_buffer_size; + uint64_t* m_buffer; +}; +extern LoadTimeInitializer g_load_time_initializer; + +#define SIZE_PER_TESTCASE 6*10000 +#define SIZE_PER_BUFFER 10000 + +#endif diff --git a/public/VectorPairHMM/src/main/c++/Makefile b/public/VectorPairHMM/src/main/c++/Makefile new file mode 100644 index 000000000..354bca0bb --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Makefile @@ -0,0 +1,114 @@ +#Copyright (c) 2012 The Broad Institute + +#Permission is hereby granted, free of charge, to any person +#obtaining a copy of this software and associated documentation +#files (the "Software"), to deal in the Software without +#restriction, including without limitation the rights to use, +#copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the +#Software is furnished to do so, subject to the following +#conditions: + +#The above copyright notice and this permission notice shall be +#included in all copies or substantial portions of the Software. + +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +#THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + + +#OMPCFLAGS=-fopenmp +#OMPLFLAGS=-fopenmp #-openmp-link static + +#CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas +#CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas + +JRE_HOME?=/opt/jdk1.7.0_25/jre +JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JRE_HOME}/../include -I${JRE_HOME}/../include/linux + +COMMON_COMPILATION_FLAGS=$(JNI_COMPILATION_FLAGS) -O3 -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas +CC=icc +CXX=icc + +LDFLAGS=-lm -lrt $(OMPLDFLAGS) +ifdef DISABLE_FTZ + COMMON_COMPILATION_FLAGS+=-DDISABLE_FTZ -no-ftz +endif + +PAPI_DIR=/home/karthikg/softwares/papi-5.3.0 +ifdef USE_PAPI + ifeq ($(USE_PAPI),1) + COMMON_COMPILATION_FLAGS+=-I$(PAPI_DIR)/include -DUSE_PAPI + LDFLAGS+=-L$(PAPI_DIR)/lib -lpapi + endif +endif + +ifdef DISABLE_FTZ + COMMON_COMPILATION_FLAGS+=-DDISABLE_FTZ -no-ftz +endif + +BIN=libVectorLoglessPairHMM.so pairhmm-template-main checker +#BIN=checker + +DEPDIR=.deps +DF=$(DEPDIR)/$(*).d + +#Common across libJNI and sandbox +COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc sse_function_instantiations.cc LoadTimeInitializer.cc +#Part of libJNI +LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc Sandbox.cc $(COMMON_SOURCES) +SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc +LIBOBJECTS=$(LIBSOURCES:.cc=.o) +COMMON_OBJECTS=$(COMMON_SOURCES:.cc=.o) + + +#No vectorization for these files +NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc LoadTimeInitializer.cc Sandbox.cc +#Use -xAVX for these files +AVX_SOURCES=avx_function_instantiations.cc +#Use -xSSE4.2 for these files +SSE_SOURCES=sse_function_instantiations.cc + +NO_VECTOR_OBJECTS=$(NO_VECTOR_SOURCES:.cc=.o) +AVX_OBJECTS=$(AVX_SOURCES:.cc=.o) +SSE_OBJECTS=$(SSE_SOURCES:.cc=.o) +$(NO_VECTOR_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) +$(AVX_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xAVX +$(SSE_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xSSE4.2 +OBJECTS=$(NO_VECTOR_OBJECTS) $(AVX_OBJECTS) $(SSE_OBJECTS) + +all: $(BIN) Sandbox.class copied_lib + +-include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) + +checker: pairhmm-1-base.o $(COMMON_OBJECTS) + $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) + +pairhmm-template-main: pairhmm-template-main.o $(COMMON_OBJECTS) + $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) + +libVectorLoglessPairHMM.so: $(LIBOBJECTS) + $(CXX) $(OMPLFLAGS) -shared -static-intel -o $@ $(LIBOBJECTS) ${LDFLAGS} + + +$(OBJECTS): %.o: %.cc + @mkdir -p $(DEPDIR) + $(CXX) -c -MMD -MF $(DF) $(CXXFLAGS) $(OUTPUT_OPTION) $< + +Sandbox.class: Sandbox.java + javac Sandbox.java + +copied_lib: libVectorLoglessPairHMM.so +ifdef OUTPUT_DIR + mkdir -p $(OUTPUT_DIR) + rsync -a libVectorLoglessPairHMM.so $(OUTPUT_DIR)/ +endif + +clean: + rm -rf $(BIN) *.o $(DEPDIR) *.class diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.cc b/public/VectorPairHMM/src/main/c++/Sandbox.cc new file mode 100644 index 000000000..985b19ae9 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox.cc @@ -0,0 +1,106 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "Sandbox.h" +#include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" +#include "utils.h" +#include "jni_common.h" +/* + * Class: Sandbox + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_Sandbox_jniGetMachineType + (JNIEnv * env, jobject thisObj) +{ + return 0; +} + +/* + * Class: Sandbox + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeClassFieldsAndMachineMask + (JNIEnv* env, jobject thisObject, jclass readDataHolderClass, jclass haplotypeDataHolderClass, jlong mask) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask(env, thisObject, readDataHolderClass, + haplotypeDataHolderClass, mask); +} + +/* + * Class: Sandbox + * Method: jniInitializeHaplotypes + * Signature: (I[LSandbox/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeHaplotypes + (JNIEnv * env, jobject thisObject, jint numHaplotypes, jobjectArray haplotypeDataArray) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes(env, thisObject, numHaplotypes, haplotypeDataArray); +} + +/* + * Class: Sandbox + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniFinalizeRegion + (JNIEnv * env, jobject thisObject) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniFinalizeRegion(env, thisObject); +} + + +/* + * Class: Sandbox + * Method: jniComputeLikelihoods + * Signature: (II[LSandbox/JNIReadDataHolderClass;[LSandbox/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniComputeLikelihoods + (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, + jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods(env, thisObject, + numReads, numHaplotypes, readDataArray, haplotypeDataArray, likelihoodArray, maxNumThreadsToUse); +} +/* + * Class: Sandbox + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniClose + (JNIEnv* env, jobject thisObject) +{ Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose(env, thisObject); } + +JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative + (JNIEnv* env, jobject thisObject, jstring fileNameString) +{ + const char* fileName = env->GetStringUTFChars(fileNameString, 0); + char local_array[800]; + strncpy(local_array, fileName, 200); + env->ReleaseStringUTFChars(fileNameString, fileName); + do_compute(local_array, true, 10000, false); +} + diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.h b/public/VectorPairHMM/src/main/c++/Sandbox.h new file mode 100644 index 000000000..486a1c095 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox.h @@ -0,0 +1,96 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox */ + +#ifndef _Included_Sandbox +#define _Included_Sandbox +#ifdef __cplusplus +extern "C" { +#endif +#undef Sandbox_enableAll +#define Sandbox_enableAll -1LL +/* + * Class: Sandbox + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_Sandbox_jniGetMachineType + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeClassFieldsAndMachineMask + (JNIEnv *, jobject, jclass, jclass, jlong); + +/* + * Class: Sandbox + * Method: jniInitializeHaplotypes + * Signature: (I[LSandbox/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: Sandbox + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniFinalizeRegion + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniComputeLikelihoods + * Signature: (II[LSandbox/JNIReadDataHolderClass;[LSandbox/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); + +/* + * Class: Sandbox + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniClose + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: doEverythingNative + * Signature: ([B)V + */ +JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative + (JNIEnv *, jobject, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.java b/public/VectorPairHMM/src/main/c++/Sandbox.java new file mode 100644 index 000000000..d6b7c2eae --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox.java @@ -0,0 +1,305 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.broadinstitute.sting.utils.vectorpairhmm; + +import java.util.List; +import java.util.LinkedList; +import java.util.Map; +import java.util.HashMap; +import java.io.File; +import java.util.Scanner; +import java.io.IOException; +import java.io.FileNotFoundException; +import java.io.InputStreamReader; + +public class Sandbox { + + private long setupTime = 0; + private long computeTime = 0; + //Used to copy references to byteArrays to JNI from reads + protected class JNIReadDataHolderClass { + public byte[] readBases = null; + public byte[] readQuals = null; + public byte[] insertionGOP = null; + public byte[] deletionGOP = null; + public byte[] overallGCP = null; + } + + //Used to copy references to byteArrays to JNI from haplotypes + protected class JNIHaplotypeDataHolderClass { + public byte[] haplotypeBases = null; + } + + /** + * Return 64-bit mask representing machine capabilities + * Bit 0 is LSB, bit 63 MSB + * Bit 0 represents sse4.2 availability + * Bit 1 represents AVX availability + */ + public native long jniGetMachineType(); + public static final long enableAll = 0xFFFFFFFFFFFFFFFFl; + + + /** + * Function to initialize the fields of JNIReadDataHolderClass and JNIHaplotypeDataHolderClass from JVM. + * C++ codegets FieldIDs for these classes once and re-uses these IDs for the remainder of the program. Field IDs do not + * change per JVM session + * @param readDataHolderClass class type of JNIReadDataHolderClass + * @param haplotypeDataHolderClass class type of JNIHaplotypeDataHolderClass + * @param mask mask is a 64 bit integer identical to the one received from jniGetMachineType(). Users can disable usage of some hardware features by zeroing some bits in the mask + * */ + private native void jniInitializeClassFieldsAndMachineMask(Class readDataHolderClass, Class haplotypeDataHolderClass, long mask); + + private static Boolean isVectorLoglessPairHMMLibraryLoaded = false; + //The constructor is called only once inside PairHMMLikelihoodCalculationEngine + public Sandbox() { + synchronized(isVectorLoglessPairHMMLibraryLoaded) { + //Load the library and initialize the FieldIDs + if(!isVectorLoglessPairHMMLibraryLoaded) { + System.loadLibrary("VectorLoglessPairHMM"); + isVectorLoglessPairHMMLibraryLoaded = true; + jniInitializeClassFieldsAndMachineMask(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once + } + } + } + + private native void jniInitializeHaplotypes(final int numHaplotypes, JNIHaplotypeDataHolderClass[] haplotypeDataArray); + + //Used to transfer data to JNI + //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region + public void initialize(final List haplotypes) { + int numHaplotypes = haplotypes.size(); + JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; + int idx = 0; + for(final JNIHaplotypeDataHolderClass currHaplotype : haplotypes) + { + haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); + haplotypeDataArray[idx].haplotypeBases = currHaplotype.haplotypeBases; + ++idx; + } + jniInitializeHaplotypes(numHaplotypes, haplotypeDataArray); + } + /** + * Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not + * accessing Java memory directly, still important to release memory from C++ + */ + private native void jniFinalizeRegion(); + + + public void finalizeRegion() + { + jniFinalizeRegion(); + } + + /** + * Real compute kernel + */ + private native void jniComputeLikelihoods(int numReads, int numHaplotypes, JNIReadDataHolderClass[] readDataArray, + JNIHaplotypeDataHolderClass[] haplotypeDataArray, double[] likelihoodArray, int maxNumThreadsToUse); + + public void computeLikelihoods(final List reads, final List haplotypes) { + //System.out.println("Region : "+reads.size()+" x "+haplotypes.size()); + long startTime = System.nanoTime(); + int readListSize = reads.size(); + int numHaplotypes = haplotypes.size(); + int numTestcases = readListSize*numHaplotypes; + JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; + int idx = 0; + for(JNIReadDataHolderClass read : reads) + { + readDataArray[idx] = new JNIReadDataHolderClass(); + readDataArray[idx].readBases = read.readBases; + readDataArray[idx].readQuals = read.readQuals; + readDataArray[idx].insertionGOP = read.insertionGOP; + readDataArray[idx].deletionGOP = read.deletionGOP; + readDataArray[idx].overallGCP = read.overallGCP; + ++idx; + } + + double[] mLikelihoodArray = new double[readListSize*numHaplotypes]; //to store results + setupTime += (System.nanoTime() - startTime); + //for(reads) + // for(haplotypes) + // compute_full_prob() + jniComputeLikelihoods(readListSize, numHaplotypes, readDataArray, null, mLikelihoodArray, 12); + + computeTime += (System.nanoTime() - startTime); + } + + /** + * Print final profiling information from native code + */ + public native void jniClose(); + public void close() + { + System.out.println("Time spent in setup for JNI call : "+(setupTime*1e-9)+" compute time : "+(computeTime*1e-9)); + jniClose(); + } + + public void parseSandboxFile(String filename) + { + File file = new File(filename); + Scanner input = null; + try + { + input = new Scanner(file); + } + catch(FileNotFoundException e) + { + System.err.println("File "+filename+" cannot be found/read"); + return; + } + int idx = 0; + int numReads = 0; + int numHaplotypes = 0; + int readIdx = 0, testCaseIdx = 0, haplotypeIdx = 0; + LinkedList haplotypeList = new LinkedList(); + LinkedList readList = new LinkedList(); + + byte[][] byteArray = new byte[6][]; + boolean firstLine = true; + String[] currTokens = new String[8]; + while(input.hasNextLine()) + { + String line = input.nextLine(); + Scanner lineScanner = new Scanner(line); + idx = 0; + while(lineScanner.hasNext()) + currTokens[idx++] = lineScanner.next(); + if(idx == 0) + break; + assert(idx >= 6); + //start of new region + if(idx == 8) + { + if(!firstLine) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + try + { + numReads = Integer.parseInt(currTokens[6]); + } + catch(NumberFormatException e) + { + numReads = 1; + } + try + { + numHaplotypes = Integer.parseInt(currTokens[7]); + } + catch(NumberFormatException e) + { + numHaplotypes = 1; + } + haplotypeIdx = readIdx = testCaseIdx = 0; + readList.clear(); + haplotypeList.clear(); + } + if(haplotypeIdx < numHaplotypes) + { + JNIHaplotypeDataHolderClass X = new JNIHaplotypeDataHolderClass(); + X.haplotypeBases = currTokens[0].getBytes(); + haplotypeList.add(X); + } + if(testCaseIdx%numHaplotypes == 0) + { + JNIReadDataHolderClass X = new JNIReadDataHolderClass(); + X.readBases = currTokens[1].getBytes(); + for(int i=2;i<6;++i) + { + byteArray[i] = currTokens[i].getBytes(); + for(int j=0;j 0 && readList.size() > 0) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + + close(); + input.close(); + } + + private native void doEverythingNative(String filename); + + public static void main(String[] args) + { + if(args.length <= 0) + { + System.err.println("Needs 1 argument - "); + System.exit(-1); + } + //// Get runtime + //java.lang.Runtime rt = java.lang.Runtime.getRuntime(); + //// Start a new process: UNIX command ls + //String cmd = "/home/karthikg/broad/gsa-unstable/public/c++/VectorPairHMM/checker "+args[0]; + //try + //{ + //System.out.println(cmd); + //java.lang.Process p = rt.exec(cmd); + //try + //{ + //p.waitFor(); + //java.io.InputStream is = p.getInputStream(); + //java.io.BufferedReader reader = new java.io.BufferedReader(new InputStreamReader(is)); + //// And print each line + //String s = null; + //while ((s = reader.readLine()) != null) { + //System.out.println(s); + //} + //is.close(); + //} + //catch(InterruptedException e) + //{ + //System.err.println(e); + //} + //} + //catch(IOException e) + //{ + //System.err.println(e); + //} + Sandbox t = new Sandbox(); + //t.doEverythingNative(args[0]); + t.parseSandboxFile(args[0]); + } +} diff --git a/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h new file mode 100644 index 000000000..7f78f0178 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIHaplotypeDataHolderClass */ + +#ifndef _Included_Sandbox_JNIHaplotypeDataHolderClass +#define _Included_Sandbox_JNIHaplotypeDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h new file mode 100644 index 000000000..a9312ff3b --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIReadDataHolderClass */ + +#ifndef _Included_Sandbox_JNIReadDataHolderClass +#define _Included_Sandbox_JNIReadDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc new file mode 100644 index 000000000..6d90d5070 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc @@ -0,0 +1,44 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "template.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_SSE + +#define SIMD_ENGINE avx +#define SIMD_ENGINE_AVX + +#include "define-float.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +#include "define-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +template double compute_full_prob_avxd(testcase* tc, double* nextlog); +template float compute_full_prob_avxs(testcase* tc, float* nextlog); + diff --git a/public/VectorPairHMM/src/main/c++/baseline.cc b/public/VectorPairHMM/src/main/c++/baseline.cc new file mode 100644 index 000000000..d6085e661 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/baseline.cc @@ -0,0 +1,167 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +using namespace std; + +template +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) +{ + int r, c; + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + + Context ctx; + //#define USE_STACK_ALLOCATION 1 +#ifdef USE_STACK_ALLOCATION + NUMBER M[ROWS][COLS]; + NUMBER X[ROWS][COLS]; + NUMBER Y[ROWS][COLS]; + NUMBER p[ROWS][6]; +#else + //allocate on heap in way that simulates a 2D array. Having a 2D array instead of + //a straightforward array of pointers ensures that all data lies 'close' in memory, increasing + //the chance of being stored together in the cache. Also, prefetchers can learn memory access + //patterns for 2D arrays, not possible for array of pointers + //bool locally_allocated = false; + //NUMBER* common_buffer = 0; + NUMBER* common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //unsigned curr_size = sizeof(NUMBER)*(3*ROWS*COLS + ROWS*6); + //if(true) + //{ + //common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //locally_allocated = true; + //} + //else + //common_buffer = (NUMBER*)(g_load_time_initializer.get_buffer()); + //pointers to within the allocated buffer + NUMBER** common_pointer_buffer = new NUMBER*[4*ROWS]; + NUMBER* ptr = common_buffer; + unsigned i = 0; + for(i=0;i<3*ROWS;++i, ptr+=COLS) + common_pointer_buffer[i] = ptr; + for(;i<4*ROWS;++i, ptr+=6) + common_pointer_buffer[i] = ptr; + + NUMBER** M = common_pointer_buffer; + NUMBER** X = M + ROWS; + NUMBER** Y = X + ROWS; + NUMBER** p = Y + ROWS; +#endif + + + p[0][MM] = ctx._(0.0); + p[0][GapM] = ctx._(0.0); + p[0][MX] = ctx._(0.0); + p[0][XX] = ctx._(0.0); + p[0][MY] = ctx._(0.0); + p[0][YY] = ctx._(0.0); + + for (r = 1; r < ROWS; r++) + { + int _i = tc->i[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + //p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(p[r][MM], _i, _d); + p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; + p[r][MX] = ctx.ph2pr[_i]; + p[r][XX] = ctx.ph2pr[_c]; + p[r][MY] = ctx.ph2pr[_d]; + p[r][YY] = ctx.ph2pr[_c]; + //p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + //p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + } + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); + } + + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } + + NUMBER result = ctx._(0.0); + + for (r = 1; r < ROWS; r++) + for (c = 1; c < COLS; c++) + { + fexcept_t flagp; + char _rs = tc->rs[r-1]; + char _hap = tc->hap[c-1]; + int _q = tc->q[r-1] & 127; + NUMBER distm = ctx.ph2pr[_q]; + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; + else + distm = distm/3; + + + //feclearexcept(FE_ALL_EXCEPT); + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //feclearexcept(FE_ALL_EXCEPT); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //feclearexcept(FE_ALL_EXCEPT); + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //CONVERT_AND_PRINT(M[r][c]); + //CONVERT_AND_PRINT(X[r][c]); + //CONVERT_AND_PRINT(Y[r][c]); + + } + for (c = 0; c < COLS; c++) + { + result += M[ROWS-1][c] + X[ROWS-1][c]; + } + + if (before_last_log != NULL) + *before_last_log = result; + +#ifndef USE_STACK_ALLOCATION + delete common_pointer_buffer; + //if(locally_allocated) + delete common_buffer; +#endif + + return result; + //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; +} + +template double compute_full_prob(testcase* tc, double* nextbuf); +template float compute_full_prob(testcase* tc, float* nextbuf); + diff --git a/public/VectorPairHMM/src/main/c++/define-double.h b/public/VectorPairHMM/src/main/c++/define-double.h new file mode 100644 index 000000000..2067d369c --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-double.h @@ -0,0 +1,205 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define PRECISION d +#define MAIN_TYPE double +#define MAIN_TYPE_SIZE 64 +#define UNION_TYPE mix_D +#define IF_128 IF_128d +#define IF_MAIN_TYPE IF_64 +#define SHIFT_CONST1 8 +#define SHIFT_CONST2 1 +#define SHIFT_CONST3 8 +#define _128_TYPE __m128d +#define SIMD_TYPE __m256d +#define _256_INT_TYPE __m256i +#define AVX_LENGTH 4 +#define HAP_TYPE __m128i +#define MASK_TYPE uint64_t +#define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFF +#define MASK_VEC MaskVec_D + +#define SET_VEC_ZERO(__vec) \ + __vec= _mm256_setzero_pd() + +#define VEC_OR(__v1, __v2) \ + _mm256_or_pd(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm256_add_pd(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm256_sub_pd(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm256_mul_pd(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm256_div_pd(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm256_blend_pd(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm256_blendv_pd(__v1, __v2, __maskV) + +#define VEC_CAST_256_128(__v1) \ + _mm256_castpd256_pd128 (__v1) + +#define VEC_EXTRACT_128(__v1, __im) \ + _mm256_extractf128_pd (__v1, __im) + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi64(__v1, __im) + +#define VEC_SET1_VAL128(__val) \ + _mm_set1_pd(__val) + +#define VEC_MOVE(__v1, __val) \ + _mm_move_sd(__v1, __val) + +#define VEC_CAST_128_256(__v1) \ + _mm256_castpd128_pd256(__v1) + +#define VEC_INSERT_VAL(__v1, __val, __pos) \ + _mm256_insertf128_pd(__v1, __val, __pos) + +#define VEC_CVT_128_256(__v1) \ + _mm256_cvtepi32_pd(__v1) + +#define VEC_SET1_VAL(__val) \ + _mm256_set1_pd(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm256_cvtepi32_pd(_mm_set1_epi32(__ch)) + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm256_cvtepi32_pd(_mm_load_si128((__m128i const *)__addr)) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm256_cmp_pd(__v1, __v2, _CMP_EQ_OQ) + +#define VEC_SET_LSE(__val) \ + _mm256_set_pd(zero, zero, zero, __val); + +#define SHIFT_HAP(__v1, __val) \ + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castpd128_pd256(__vsLow) ; \ +__vdst = _mm256_insertf128_pd(__vdst, __vsHigh, 1) ; + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi64(__vs, 1) + + +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + double* ptr1 = (double*) (&__v1) ; \ + double* ptr2 = (double*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Double Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ +} + +class BitMaskVec_double { + + MASK_VEC low_, high_ ; + SIMD_TYPE combined_ ; + + public: + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_double diff --git a/public/VectorPairHMM/src/main/c++/define-float.h b/public/VectorPairHMM/src/main/c++/define-float.h new file mode 100644 index 000000000..318f78280 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-float.h @@ -0,0 +1,206 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define PRECISION s + +#define MAIN_TYPE float +#define MAIN_TYPE_SIZE 32 +#define UNION_TYPE mix_F +#define IF_128 IF_128f +#define IF_MAIN_TYPE IF_32 +#define SHIFT_CONST1 12 +#define SHIFT_CONST2 3 +#define SHIFT_CONST3 4 +#define _128_TYPE __m128 +#define SIMD_TYPE __m256 +#define _256_INT_TYPE __m256i +#define AVX_LENGTH 8 +#define HAP_TYPE UNION_TYPE +#define MASK_TYPE uint32_t +#define MASK_ALL_ONES 0xFFFFFFFF +#define MASK_VEC MaskVec_F + +#define SET_VEC_ZERO(__vec) \ + __vec= _mm256_setzero_ps() + +#define VEC_OR(__v1, __v2) \ + _mm256_or_ps(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm256_add_ps(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm256_sub_ps(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm256_mul_ps(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm256_div_ps(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm256_blend_ps(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm256_blendv_ps(__v1, __v2, __maskV) + +#define VEC_CAST_256_128(__v1) \ + _mm256_castps256_ps128 (__v1) + +#define VEC_EXTRACT_128(__v1, __im) \ + _mm256_extractf128_ps (__v1, __im) + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi32(__v1, __im) + +#define VEC_SET1_VAL128(__val) \ + _mm_set1_ps(__val) + +#define VEC_MOVE(__v1, __val) \ + _mm_move_ss(__v1, __val) + +#define VEC_CAST_128_256(__v1) \ + _mm256_castps128_ps256(__v1) + +#define VEC_INSERT_VAL(__v1, __val, __pos) \ + _mm256_insertf128_ps(__v1, __val, __pos) + +#define VEC_CVT_128_256(__v1) \ + _mm256_cvtepi32_ps(__v1.i) + +#define VEC_SET1_VAL(__val) \ + _mm256_set1_ps(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm256_cvtepi32_ps(_mm256_set1_epi32(__ch)) + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm256_cvtepi32_ps(_mm256_loadu_si256((__m256i const *)__addr)) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm256_cmp_ps(__v1, __v2, _CMP_EQ_OQ) + +#define VEC_SET_LSE(__val) \ + _mm256_set_ps(zero, zero, zero, zero, zero, zero, zero, __val); + +#define SHIFT_HAP(__v1, __val) \ + _vector_shift_lastavxs(__v1, __val.f); + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castps128_ps256(__vsLow) ; \ +__vdst = _mm256_insertf128_ps(__vdst, __vsHigh, 1) ; + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi32(__vs, 1) + +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + float* ptr1 = (float*) (&__v1) ; \ + float* ptr2 = (float*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Float Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ +} + +class BitMaskVec_float { + + MASK_VEC low_, high_ ; + SIMD_TYPE combined_ ; + + public: + + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_float diff --git a/public/VectorPairHMM/src/main/c++/define-sse-double.h b/public/VectorPairHMM/src/main/c++/define-sse-double.h new file mode 100644 index 000000000..2d271a854 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-sse-double.h @@ -0,0 +1,173 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_INSERT_UNIT(__v1,__ins,__im) +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define SSE +#define PRECISION d + +#define MAIN_TYPE double +#define MAIN_TYPE_SIZE 64 +#define UNION_TYPE mix_D128 +#define IF_128 IF_128d +#define IF_MAIN_TYPE IF_64 +#define SHIFT_CONST1 1 +#define SHIFT_CONST2 8 +#define SHIFT_CONST3 0 +#define _128_TYPE __m128d +#define SIMD_TYPE __m128d +#define _256_INT_TYPE __m128i +#define AVX_LENGTH 2 +#define HAP_TYPE __m128i +#define MASK_TYPE uint64_t +#define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFFL +#define MASK_VEC MaskVec_D + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi64(__v1, __im) + +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi64(__v1,__ins,__im) + +#define VEC_OR(__v1, __v2) \ + _mm_or_pd(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm_add_pd(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm_sub_pd(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm_mul_pd(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm_div_pd(__v1, __v2) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm_cmpeq_pd(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm_blend_pd(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm_blendv_pd(__v1, __v2, __maskV) + +#define SHIFT_HAP(__v1, __val) \ + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) + +#define VEC_CVT_128_256(__v1) \ + _mm_cvtepi32_pd(__v1) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_pd(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm_cvtepi32_pd(_mm_set1_epi32(__ch)) + +#define VEC_SET_LSE(__val) \ + _mm_set_pd(zero, __val); + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm_cvtepi32_pd(_mm_loadu_si128((__m128i const *)__addr)) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm_castsi128_pd(_mm_set_epi64(__vsHigh, __vsLow)) + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi64(__vs, 1) + + +class BitMaskVec_sse_double { + + MASK_VEC combined_ ; + public: + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_sse_double + diff --git a/public/VectorPairHMM/src/main/c++/define-sse-float.h b/public/VectorPairHMM/src/main/c++/define-sse-float.h new file mode 100644 index 000000000..20af947dd --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-sse-float.h @@ -0,0 +1,173 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_INSERT_UNIT(__v1,__ins,__im) +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define SSE +#define PRECISION s + +#define MAIN_TYPE float +#define MAIN_TYPE_SIZE 32 +#define UNION_TYPE mix_F128 +#define IF_128 IF_128f +#define IF_MAIN_TYPE IF_32 +#define SHIFT_CONST1 3 +#define SHIFT_CONST2 4 +#define SHIFT_CONST3 0 +#define _128_TYPE __m128 +#define SIMD_TYPE __m128 +#define _256_INT_TYPE __m128i +#define AVX_LENGTH 4 +//#define MAVX_COUNT (MROWS+3)/AVX_LENGTH +#define HAP_TYPE UNION_TYPE +#define MASK_TYPE uint32_t +#define MASK_ALL_ONES 0xFFFFFFFF +#define MASK_VEC MaskVec_F + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi32(__v1, __im) + +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi32(__v1,__ins,__im) + +#define VEC_OR(__v1, __v2) \ + _mm_or_ps(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm_add_ps(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm_sub_ps(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm_mul_ps(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm_div_ps(__v1, __v2) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm_cmpeq_ps(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm_blend_ps(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm_blendv_ps(__v1, __v2, __maskV) + +#define SHIFT_HAP(__v1, __val) \ + _vector_shift_lastsses(__v1, __val.f) + +#define VEC_CVT_128_256(__v1) \ + _mm_cvtepi32_ps(__v1.i) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_ps(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm_cvtepi32_ps(_mm_set1_epi32(__ch)) + +#define VEC_SET_LSE(__val) \ + _mm_set_ps(zero, zero, zero, __val); + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm_cvtepi32_ps(_mm_loadu_si128((__m128i const *)__addr)) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm_cvtpi32x2_ps(__vsLow, __vsHigh) + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi32(__vs, 1) + +class BitMaskVec_sse_float { + + MASK_VEC combined_ ; + + public: + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_sse_float diff --git a/public/VectorPairHMM/src/main/c++/headers.h b/public/VectorPairHMM/src/main/c++/headers.h new file mode 100644 index 000000000..4a0d89b57 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/headers.h @@ -0,0 +1,71 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef COMMON_HEADERS_H +#define COMMON_HEADERS_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern uint64_t exceptions_array[128]; +extern FILE* g_debug_fptr; +#define STORE_FP_EXCEPTIONS(flagp, exceptions_array) \ + fegetexceptflag(&flagp, FE_ALL_EXCEPT | __FE_DENORM); \ + exceptions_array[FE_INVALID] += ((flagp & FE_INVALID)); \ + exceptions_array[__FE_DENORM] += ((flagp & __FE_DENORM) >> 1); \ + exceptions_array[FE_DIVBYZERO] += ((flagp & FE_DIVBYZERO) >> 2); \ + exceptions_array[FE_OVERFLOW] += ((flagp & FE_OVERFLOW) >> 3); \ + exceptions_array[FE_UNDERFLOW] += ((flagp & FE_UNDERFLOW) >> 4); \ + feclearexcept(FE_ALL_EXCEPT | __FE_DENORM); + +#define CONVERT_AND_PRINT(X) \ + g_converter.f = (X); \ + fwrite(&(g_converter.i),4,1,g_debug_fptr); \ + +#endif diff --git a/public/VectorPairHMM/src/main/c++/jni_common.h b/public/VectorPairHMM/src/main/c++/jni_common.h new file mode 100644 index 000000000..23c323246 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/jni_common.h @@ -0,0 +1,58 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef JNI_COMMON_H +#define JNI_COMMON_H + +#include +/*#define ENABLE_ASSERTIONS 1*/ +#define DO_PROFILING 1 +/*#define DEBUG 1*/ +/*#define DEBUG0_1 1*/ +/*#define DEBUG3 1*/ +/*#define DUMP_TO_SANDBOX 1*/ + + +#define DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY 1 + +#ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY +//Gets direct access to Java arrays +#define GET_BYTE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical + +#else +//Likely makes copy of Java arrays to JNI C++ space +#define GET_BYTE_ARRAY_ELEMENTS env->GetByteArrayElements +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleaseByteArrayElements +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetDoubleArrayElements +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleaseDoubleArrayElements + +#endif //ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY + +#endif //ifndef JNI_COMMON_H diff --git a/public/VectorPairHMM/src/main/c++/jnidebug.h b/public/VectorPairHMM/src/main/c++/jnidebug.h new file mode 100644 index 000000000..7fcab2a51 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/jnidebug.h @@ -0,0 +1,191 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef JNI_DEBUG_H +#define JNI_DEBUG_H + +template +class DataHolder +{ +#define INIT_MATRIX(X) \ + X = new NUMBER*[m_paddedMaxReadLength]; \ + for(int i=0;i ctx; + for (int r = 1; r <= length;r++) //in original code, r < ROWS (where ROWS = paddedReadLength) + { + int _i = insertionGOP[r-1]; //insertionGOP + int _d = deletionGOP[r-1]; //deletionGOP + int _c = overallGCP[r-1]; //overallGCP + m_transition[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; //lines 161-162 + m_transition[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; //line 163 + m_transition[r][MX] = ctx.ph2pr[_i]; //164 + m_transition[r][XX] = ctx.ph2pr[_c]; //165 + m_transition[r][MY] = ctx.ph2pr[_d];//last row seems different, compared to line 166 + m_transition[r][YY] = ctx.ph2pr[_c];//same as above for line 167 + //m_transition[r][MY] = (r == length) ? ctx._(1.0) : ctx.ph2pr[_d];//last row seems different, compared to line 166 + //m_transition[r][YY] = (r == length) ? ctx._(1.0) : ctx.ph2pr[_c];//same as above for line 167 +#ifdef DEBUG3 + for(int j=0;j<6;++j) + debug_dump("transitions_jni.txt", to_string(m_transition[r][j]),true); +#endif + } + ++g_num_prob_init; + } + bool m_is_initialized; + int m_readMaxLength; + int m_haplotypeMaxLength; + int m_paddedMaxReadLength; + int m_paddedMaxHaplotypeLength; + NUMBER** m_matchMatrix; + NUMBER** m_insertionMatrix; + NUMBER** m_deletionMatrix; + NUMBER** m_prior; + NUMBER (*m_transition)[6]; +}; +extern DataHolder g_double_dataholder; + +template +NUMBER compute_full_prob(testcase *tc, NUMBER** M, NUMBER** X, NUMBER** Y, NUMBER (*p)[6], + bool do_initialization, jint hapStartIndex, NUMBER *before_last_log = NULL) +{ + int r, c; + int ROWS = tc->rslen + 1; //ROWS = paddedReadLength + int COLS = tc->haplen + 1; //COLS = paddedHaplotypeLength + + Context ctx; + //////NOTES + ////ctx.ph2pr[quality]; //This quantity is QualityUtils.qualToErrorProb(quality) + ////1-ctx.ph2pr[quality]; //This corresponds to QualityUtils.qualToProb(quality); + + //Initialization + if(do_initialization) + { + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); //code from 87-90 in LoglessPairHMM + } + + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + //deletionMatrix row 0 in above nest is initialized in the Java code + //However, insertionMatrix column 0 is not initialized in Java code, could it be that + //values are re-used from a previous iteration? + //Why even do this, X[0][0] = 0 from above loop nest, X[idx][0] = 0 from this computation + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } + } + + for (r = 1; r < ROWS; r++) + for (c = hapStartIndex+1; c < COLS; c++) + { + //The following lines correspond to initializePriors() + char _rs = tc->rs[r-1]; //line 137 + char _hap = tc->hap[c-1]; //line 140 + //int _q = tc->q[r-1] & 127; //line 138 - q is the quality (qual), should be byte hence int ANDed with 0xFF + int _q = tc->q[r-1]; //line 138 - q is the quality (qual), should be byte hence int ANDed with 0xFF + NUMBER distm = ctx.ph2pr[_q]; //This quantity is QualityUtils.qualToErrorProb(_q) + //The assumption here is that doNotUseTristateCorrection is true + //TOASK + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; //This is the quantity QualityUtils.qualToProb(qual) + else + distm = distm/3; +#ifdef DEBUG3 + debug_dump("priors_jni.txt",to_string(distm),true); +#endif + + //Computation inside updateCell + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; +#ifdef DEBUG3 + debug_dump("matrices_jni.txt",to_string(M[r][c]),true); + debug_dump("matrices_jni.txt",to_string(X[r][c]),true); + debug_dump("matrices_jni.txt",to_string(Y[r][c]),true); +#endif + } + + NUMBER result = ctx._(0.0); + for (c = 0; c < COLS; c++) + result += M[ROWS-1][c] + X[ROWS-1][c]; + + if (before_last_log != NULL) + *before_last_log = result; + +#ifdef DEBUG + debug_dump("return_values_jni.txt",to_string(ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT),true); +#endif + return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; +} + +#endif diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc new file mode 100644 index 000000000..8a3f8b5bc --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc @@ -0,0 +1,176 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "jni_common.h" +#include "org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +#include "jnidebug.h" +DataHolder g_double_dataholder; + +using namespace std; + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitialize +(JNIEnv* env, jobject thisObject, + jint readMaxLength, jint haplotypeMaxLength) +{ + static int g_num_init_calls = 0; +#ifdef DEBUG3 + cout << "Entered alloc initialized .. readMaxLength "<GetArrayLength(insertionGOP); +#ifdef DEBUG3 + cout << "Entered initializeProbabilities .. length "<GetByteArrayElements(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (env)->GetByteArrayElements(deletionGOP, &is_copy); + jbyte* overallGCPArray = (env)->GetByteArrayElements(overallGCP, &is_copy); +#ifdef DEBUG + if(insertionGOPArray == 0) + cerr << "insertionGOP array not initialized in JNI\n"; + ////assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + if(deletionGOPArray == 0) + cerr << "deletionGOP array not initialized in JNI\n"; + ////assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); +#endif + + g_double_dataholder.initializeProbabilities(length, insertionGOPArray, deletionGOPArray, overallGCPArray); + + env->ReleaseByteArrayElements(overallGCP, overallGCPArray, JNI_ABORT); + env->ReleaseByteArrayElements(deletionGOP, deletionGOPArray, JNI_ABORT); + env->ReleaseByteArrayElements(insertionGOP, insertionGOPArray, JNI_ABORT); +} + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializePriorsAndUpdateCells( + JNIEnv* env, jobject thisObject, + jboolean doInitialization, jint paddedReadLength, jint paddedHaplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jint hapStartIndex + ) +{ +#ifdef DEBUG3 + cout << "Entered mainCompute .. doInitialization "<<(doInitialization == JNI_TRUE)<<" hapStartIndex "<GetByteArrayElements(readBases, &is_copy); + jbyte* haplotypeBasesArray = (env)->GetByteArrayElements(haplotypeBases, &is_copy); + jbyte* readQualsArray = (env)->GetByteArrayElements(readQuals, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); +#endif + testcase tc; + + tc.rslen = paddedReadLength-1; + tc.haplen = paddedHaplotypeLength-1; + + tc.rs = (char*)readBasesArray; + tc.hap = (char*)haplotypeBasesArray; + tc.q = (char*)readQualsArray; //TOASK - q is now char* + + compute_full_prob(&tc, g_double_dataholder.m_matchMatrix, g_double_dataholder.m_insertionMatrix, + g_double_dataholder.m_deletionMatrix, g_double_dataholder.m_transition, + doInitialization == JNI_TRUE, hapStartIndex, NULL); + + env->ReleaseByteArrayElements(readBases, readBasesArray, JNI_ABORT); + env->ReleaseByteArrayElements(haplotypeBases, haplotypeBasesArray, JNI_ABORT); + env->ReleaseByteArrayElements(readQuals, readQualsArray, JNI_ABORT); + return 0.0; +} + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10( + JNIEnv* env, jobject thisObject, + jint readLength, jint haplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP, + jint hapStartIndex + ) +{ + jboolean is_copy = JNI_FALSE; + jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBases, &is_copy); + jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); + jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); + jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); + //assert(readLength < MROWS); +#endif + testcase tc; + tc.rslen = readLength; + tc.haplen = haplotypeLength; + tc.rs = (char*)readBasesArray; + tc.hap = (char*)haplotypeBasesArray; + for(unsigned i=0;i +/* Header for class org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM */ + +#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM +#define _Included_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM +#ifdef __cplusplus +extern "C" { +#endif +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_TRISTATE_CORRECTION +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_TRISTATE_CORRECTION 3.0 +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToMatch +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToMatch 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_indelToMatch +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_indelToMatch 1L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToInsertion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToInsertion 2L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_insertionToInsertion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_insertionToInsertion 3L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToDeletion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToDeletion 4L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_deletionToDeletion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_deletionToDeletion 5L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_verify +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_verify 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug0_1 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug0_1 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug1 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug1 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug2 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug2 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug3 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug3 0L +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitialize + * Signature: (II)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitialize + (JNIEnv *, jobject, jint, jint); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitializeProbabilities + * Signature: ([[D[B[B[B)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializeProbabilities + (JNIEnv *, jclass, jobjectArray, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitializePriorsAndUpdateCells + * Signature: (ZII[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializePriorsAndUpdateCells + (JNIEnv *, jobject, jboolean, jint, jint, jbyteArray, jbyteArray, jbyteArray, jint); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniSubComputeReadLikelihoodGivenHaplotypeLog10 + * Signature: (II[B[B[B[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10 + (JNIEnv *, jobject, jint, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc new file mode 100644 index 000000000..0b54c8a81 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -0,0 +1,382 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "jni_common.h" +#include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" + +using namespace std; + +JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType + (JNIEnv* env, jobject thisObject) +{ + return (jlong)get_machine_capabilities(); +} + +//Should be called only once for the whole Java process - initializes field ids for the classes JNIReadDataHolderClass +//and JNIHaplotypeDataHolderClass +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask + (JNIEnv* env, jobject thisObject, jclass readDataHolderClass, jclass haplotypeDataHolderClass, jlong mask) +{ + assert(readDataHolderClass); + assert(haplotypeDataHolderClass); + jfieldID fid; + fid = env->GetFieldID(readDataHolderClass, "readBases", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for readBases"); + g_load_time_initializer.m_readBasesFID = fid; + fid = env->GetFieldID(readDataHolderClass, "readQuals", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for readQuals"); + g_load_time_initializer.m_readQualsFID = fid; + fid = env->GetFieldID(readDataHolderClass, "insertionGOP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for insertionGOP"); + g_load_time_initializer.m_insertionGOPFID = fid; + fid = env->GetFieldID(readDataHolderClass, "deletionGOP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for deletionGOP"); + g_load_time_initializer.m_deletionGOPFID = fid; + fid = env->GetFieldID(readDataHolderClass, "overallGCP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for overallGCP"); + g_load_time_initializer.m_overallGCPFID = fid; + + fid = env->GetFieldID(haplotypeDataHolderClass, "haplotypeBases", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for haplotypeBases"); + g_load_time_initializer.m_haplotypeBasesFID = fid; + if(mask != ENABLE_ALL_HARDWARE_FEATURES) + { + cout << "Using user supplied hardware mask to re-initialize function pointers for PairHMM\n"; + initialize_function_pointers((uint64_t)mask); + cout.flush(); + } +} + +//Since the list of haplotypes against which the reads are evaluated in PairHMM is the same for a region, +//transfer the list only once +vector > g_haplotypeBasesArrayVector; +vector g_haplotypeBasesLengths; +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes + (JNIEnv * env, jobject thisObject, jint numHaplotypes, jobjectArray haplotypeDataArray) +{ + jboolean is_copy = JNI_FALSE; + //To ensure, GET_BYTE_ARRAY_ELEMENTS is invoked only once for each haplotype, store bytearrays in a vector + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + haplotypeBasesArrayVector.clear(); + g_haplotypeBasesLengths.clear(); + haplotypeBasesArrayVector.resize(numHaplotypes); + g_haplotypeBasesLengths.resize(numHaplotypes); + jsize haplotypeBasesLength = 0; + for(unsigned j=0;jGetObjectArrayElement(haplotypeDataArray, j); + jbyteArray haplotypeBases = (jbyteArray)env->GetObjectField(haplotypeObject, g_load_time_initializer.m_haplotypeBasesFID); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBases && ("haplotypeBases is NULL at index : "+to_string(j)+"\n").c_str()); +#endif + //Need a global reference as this will be accessed across multiple JNI calls to JNIComputeLikelihoods() + jbyteArray haplotypeBasesGlobalRef = (jbyteArray)env->NewGlobalRef(haplotypeBases); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBasesGlobalRef && ("Could not get global ref to haplotypeBases at index : "+to_string(j)+"\n").c_str()); +#endif + env->DeleteLocalRef(haplotypeBases); //free the local reference + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBasesGlobalRef, &is_copy); + haplotypeBasesLength = env->GetArrayLength(haplotypeBasesGlobalRef); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + //assert(haplotypeBasesLength < MCOLS); +#endif +#ifdef DEBUG0_1 + cout << "JNI haplotype length "< > >& readBasesArrayVector, vector& tc_array) +{ + jboolean is_copy = JNI_FALSE; + //haplotype vector from earlier store - note the reference to vector, not copying + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + unsigned tc_idx = 0; + for(unsigned i=0;iGetObjectArrayElement(readDataArray, i); + jbyteArray readBases = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_readBasesFID); + jbyteArray insertionGOP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_insertionGOPFID); + jbyteArray deletionGOP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_deletionGOPFID); + jbyteArray overallGCP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_overallGCPFID); + jbyteArray readQuals = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_readQualsFID); + +#ifdef ENABLE_ASSERTIONS + assert(readBases && ("readBases is NULL at index : "+to_string(i)+"\n").c_str()); + assert(insertionGOP && ("insertionGOP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(deletionGOP && ("deletionGOP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(overallGCP && ("overallGCP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(readQuals && ("readQuals is NULL at index : "+to_string(i)+"\n").c_str()); +#endif + jsize readLength = env->GetArrayLength(readBases); + + jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); //order of GET-RELEASE is important + jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); + jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); + jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DO_PROFILING + g_load_time_initializer.m_bytes_copied += (is_copy ? readLength*5 : 0); + g_load_time_initializer.update_stat(READ_LENGTH_IDX, readLength); +#endif +#ifdef ENABLE_ASSERTIONS + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "overallGCP array not initialized in JNI"); + //assert(readLength < MROWS); + assert(readLength == env->GetArrayLength(readQuals)); + assert(readLength == env->GetArrayLength(insertionGOP)); + assert(readLength == env->GetArrayLength(deletionGOP)); + assert(readLength == env->GetArrayLength(overallGCP)); +#endif +#ifdef DEBUG0_1 + cout << "JNI read length "<& tc_array, unsigned numTestCases, double* likelihoodDoubleArray, + unsigned maxNumThreadsToUse) +{ +#ifdef DO_REPEAT_PROFILING + for(unsigned i=0;i<10;++i) +#endif + { +#pragma omp parallel for schedule (dynamic,10000) num_threads(maxNumThreadsToUse) + for(unsigned tc_idx=0;tc_idx > >& readBasesArrayVector) +{ + //Release read arrays first + for(int i=readBasesArrayVector.size()-1;i>=0;--i)//note the order - reverse of GET + { + for(int j=readBasesArrayVector[i].size()-1;j>=0;--j) + RELEASE_BYTE_ARRAY_ELEMENTS(readBasesArrayVector[i][j].first, readBasesArrayVector[i][j].second, JNI_RO_RELEASE_MODE); + readBasesArrayVector[i].clear(); + } + readBasesArrayVector.clear(); +} + + +#ifdef DO_WARMUP +uint64_t g_sum = 0; +#endif +//JNI function to invoke compute_full_prob_avx +//readDataArray - array of JNIReadDataHolderClass objects which contain the readBases, readQuals etc +//haplotypeDataArray - array of JNIHaplotypeDataHolderClass objects which contain the haplotypeBases +//likelihoodArray - array of doubles to return results back to Java. Memory allocated by Java prior to JNI call +//maxNumThreadsToUse - Max number of threads that OpenMP can use for the HMM computation +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods + (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, + jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +{ +#ifdef DEBUG0_1 + cout << "JNI numReads "< tc_array; + tc_array.clear(); + tc_array.resize(numTestCases); + //Store read arrays for release later + vector > > readBasesArrayVector; + readBasesArrayVector.clear(); + readBasesArrayVector.resize(numReads); +#ifdef DUMP_TO_SANDBOX + g_load_time_initializer.open_sandbox(); +#endif +#ifdef DO_PROFILING + get_time(&start_time); +#endif + //Copy byte array references from Java memory into vector of testcase structs + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeTestcasesVector(env, + numReads, numHaplotypes, readDataArray, readBasesArrayVector, tc_array); +#ifdef DO_PROFILING + g_load_time_initializer.m_data_transfer_time += diff_time(start_time); +#endif + + //Get double array where results are stored (to pass back to java) + jdouble* likelihoodDoubleArray = (jdouble*)GET_DOUBLE_ARRAY_ELEMENTS(likelihoodArray, &is_copy); +#ifdef ENABLE_ASSERTIONS + assert(likelihoodDoubleArray && "likelihoodArray is NULL"); + assert(env->GetArrayLength(likelihoodArray) == numTestCases); +#endif +#ifdef DO_WARMUP //ignore - only for crazy profiling + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + for(unsigned i=0;iGetArrayLength(haplotypeBasesArrayVector[i].first); + for(unsigned j=0;jGetArrayLength(readBasesArrayVector[i][j].first); + for(unsigned k=0;k >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + //Now release haplotype arrays + for(int j=haplotypeBasesArrayVector.size()-1;j>=0;--j) //note the order - reverse of GET + { + RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBasesArrayVector[j].first, haplotypeBasesArrayVector[j].second, JNI_RO_RELEASE_MODE); + env->DeleteGlobalRef(haplotypeBasesArrayVector[j].first); //free the global reference + } + haplotypeBasesArrayVector.clear(); + g_haplotypeBasesLengths.clear(); +} + + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose + (JNIEnv* env, jobject thisObject) +{ +#ifdef DO_PROFILING + g_load_time_initializer.print_profiling(); +#endif +#ifdef DEBUG + g_load_time_initializer.debug_close(); +#endif +} + diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h new file mode 100644 index 000000000..d820b4b26 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h @@ -0,0 +1,104 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM */ + +#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM +#define _Included_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM +#ifdef __cplusplus +extern "C" { +#endif +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_TRISTATE_CORRECTION +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_TRISTATE_CORRECTION 3.0 +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToMatch +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToMatch 0L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_indelToMatch +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_indelToMatch 1L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToInsertion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToInsertion 2L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_insertionToInsertion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_insertionToInsertion 3L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToDeletion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToDeletion 4L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_deletionToDeletion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_deletionToDeletion 5L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_sse42Mask +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_sse42Mask 1LL +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_avxMask +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_avxMask 2LL +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask + (JNIEnv *, jobject, jclass, jclass, jlong); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniInitializeHaplotypes + * Signature: (I[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniFinalizeRegion + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniComputeLikelihoods + * Signature: (II[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIReadDataHolderClass;[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc new file mode 100644 index 000000000..7ff219b88 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc @@ -0,0 +1,70 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +//#define DEBUG 1 +//#define DEBUG0_1 1 +//#define DEBUG3 1 +#include "headers.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +using namespace std; + +int main(int argc, char** argv) +{ +#define BATCH_SIZE 10000 + if(argc < 2) + { + cerr << "Needs path to input file as argument\n"; + exit(0); + } + bool use_old_read_testcase = false; + if(argc >= 3 && string(argv[2]) == "1") + use_old_read_testcase = true; + unsigned chunk_size = 10000; + bool do_check = true; + uint64_t mask = ~(0ull); + for(int i=3;i +#include +#include + + +void CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]) { + + const int maskBitCnt = MAIN_TYPE_SIZE ; + + for (int vi=0; vi < numMaskVecs; ++vi) { + for (int rs=0; rs < NUM_DISTINCT_CHARS; ++rs) { + maskArr[vi][rs] = 0 ; + } + maskArr[vi][AMBIG_CHAR] = MASK_ALL_ONES ; + } + + for (int col=1; col < COLS; ++col) { + int mIndex = (col-1) / maskBitCnt ; + int mOffset = (col-1) % maskBitCnt ; + MASK_TYPE bitMask = ((MASK_TYPE)0x1) << (maskBitCnt-1-mOffset) ; + + char hapChar = ConvertChar::get(tc.hap[col-1]); + + if (hapChar == AMBIG_CHAR) { + for (int ci=0; ci < NUM_DISTINCT_CHARS; ++ci) + maskArr[mIndex][ci] |= bitMask ; + } + + maskArr[mIndex][hapChar] |= bitMask ; + // bit corresponding to col 1 will be the MSB of the mask 0 + // bit corresponding to col 2 will be the MSB-1 of the mask 0 + // ... + // bit corresponding to col 32 will be the LSB of the mask 0 + // bit corresponding to col 33 will be the MSB of the mask 1 + // ... + } + +} + +void CONCAT(CONCAT(init_masks_for_row_,SIMD_ENGINE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess) { + + for (int ri=0; ri < numRowsToProcess; ++ri) { + rsArr[ri] = ConvertChar::get(tc.rs[ri+beginRowIndex-1]) ; + } + + for (int ei=0; ei < AVX_LENGTH; ++ei) { + lastMaskShiftOut[ei] = 0 ; + } +} + +#define SET_MASK_WORD(__dstMask, __srcMask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ + MASK_TYPE __bitMask = (((MASK_TYPE)0x1) << __shiftBy) - 1 ; \ + MASK_TYPE __nextShiftOut = (__srcMask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ + __dstMask = (__srcMask >> __shiftBy) | __lastShiftOut ; \ + __lastShiftOut = __nextShiftOut ; \ +} + + +void CONCAT(CONCAT(update_masks_for_cols_,SIMD_ENGINE), PRECISION)(int maskIndex, BITMASK_VEC& bitMaskVec, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, int maskBitCnt) { + + for (int ei=0; ei < AVX_LENGTH/2; ++ei) { + SET_MASK_WORD(bitMaskVec.getLowEntry(ei), maskArr[maskIndex][rsArr[ei]], + lastMaskShiftOut[ei], ei, maskBitCnt) ; + + int ei2 = ei + AVX_LENGTH/2 ; // the second entry index + SET_MASK_WORD(bitMaskVec.getHighEntry(ei), maskArr[maskIndex][rsArr[ei2]], + lastMaskShiftOut[ei2], ei2, maskBitCnt) ; + } + +} + + +inline void CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (BITMASK_VEC& bitMaskVec, SIMD_TYPE& distm, SIMD_TYPE& _1_distm, SIMD_TYPE& distmChosen) { + + distmChosen = VEC_BLENDV(distm, _1_distm, bitMaskVec.getCombinedMask()) ; + + bitMaskVec.shift_left_1bit() ; +} + +/* + * This function: + * 1- Intializes probability values p_MM, p_XX, P_YY, p_MX, p_GAPM and pack them into vectors (SSE or AVX) + * 2- Precompute parts of "distm" which only depeneds on a row number and pack it into vector + */ + +template void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, SIMD_TYPE *p_MM, SIMD_TYPE *p_GAPM, SIMD_TYPE *p_MX, SIMD_TYPE *p_XX, SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, SIMD_TYPE *distm1D) +{ + NUMBER zero = ctx._(0.0); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + for (int s=0;si[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + + //*(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(*(ptr_p_MM+r-1), _i, _d); + *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; + *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; + *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; + *(ptr_p_MY+r-1) = ctx.ph2pr[_d]; + *(ptr_p_YY+r-1) = ctx.ph2pr[_c]; + } + + NUMBER *ptr_distm1D = (NUMBER *)distm1D; + for (int r = 1; r < ROWS; r++) + { + int _q = tc->q[r-1] & 127; + ptr_distm1D[r-1] = ctx.ph2pr[_q]; + } +} + +/* + * This function handles pre-stripe computation: + * 1- Retrieve probaility vectors from memory + * 2- Initialize M, X, Y vectors with all 0's (for the first stripe) and shifting the last row from previous stripe for the rest + */ + +template inline void CONCAT(CONCAT(stripeINITIALIZATION,SIMD_ENGINE), PRECISION)( + int stripeIdx, Context ctx, testcase *tc, SIMD_TYPE &pGAPM, SIMD_TYPE &pMM, SIMD_TYPE &pMX, SIMD_TYPE &pXX, SIMD_TYPE &pMY, SIMD_TYPE &pYY, + SIMD_TYPE &rs, UNION_TYPE &rsN, SIMD_TYPE &distm, SIMD_TYPE &_1_distm, SIMD_TYPE *distm1D, SIMD_TYPE N_packed256, SIMD_TYPE *p_MM , SIMD_TYPE *p_GAPM , + SIMD_TYPE *p_MX, SIMD_TYPE *p_XX , SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM) +{ + int i = stripeIdx; + pGAPM = p_GAPM[i]; + pMM = p_MM[i]; + pMX = p_MX[i]; + pXX = p_XX[i]; + pMY = p_MY[i]; + pYY = p_YY[i]; + + NUMBER zero = ctx._(0.0); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + UNION_TYPE packed3; packed3.d = VEC_SET1_VAL(3.0); + + distm = distm1D[i]; + _1_distm = VEC_SUB(packed1.d, distm); + + distm = VEC_DIV(distm, packed3.d); + + /* initialize M_t_2, M_t_1, X_t_2, X_t_1, Y_t_2, Y_t_1 */ + M_t_2.d = VEC_SET1_VAL(zero); + X_t_2.d = VEC_SET1_VAL(zero); + + if (i==0) { + M_t_1.d = VEC_SET1_VAL(zero); + X_t_1.d = VEC_SET1_VAL(zero); + Y_t_2.d = VEC_SET_LSE(init_Y); + Y_t_1.d = VEC_SET1_VAL(zero); + } + else { + X_t_1.d = VEC_SET_LSE(shiftOutX[AVX_LENGTH]); + M_t_1.d = VEC_SET_LSE(shiftOutM[AVX_LENGTH]); + Y_t_2.d = VEC_SET1_VAL(zero); + Y_t_1.d = VEC_SET1_VAL(zero); + } + M_t_1_y = M_t_1; +} + +/* + * This function is the main compute kernel to compute M, X and Y + */ + +inline void CONCAT(CONCAT(computeMXY,SIMD_ENGINE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, + UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, + SIMD_TYPE pMM, SIMD_TYPE pGAPM, SIMD_TYPE pMX, SIMD_TYPE pXX, SIMD_TYPE pMY, SIMD_TYPE pYY, SIMD_TYPE distmSel) +{ + /* Compute M_t <= distm * (p_MM*M_t_2 + p_GAPM*X_t_2 + p_GAPM*Y_t_2) */ + M_t.d = VEC_MUL(VEC_ADD(VEC_ADD(VEC_MUL(M_t_2.d, pMM), VEC_MUL(X_t_2.d, pGAPM)), VEC_MUL(Y_t_2.d, pGAPM)), distmSel); + //M_t.d = VEC_MUL( VEC_ADD(VEC_MUL(M_t_2.d, pMM), VEC_MUL(VEC_ADD(X_t_2.d, Y_t_2.d), pGAPM)), distmSel); + + M_t_y = M_t; + + /* Compute X_t */ + X_t.d = VEC_ADD(VEC_MUL(M_t_1.d, pMX) , VEC_MUL(X_t_1.d, pXX)); + + /* Compute Y_t */ + Y_t.d = VEC_ADD(VEC_MUL(M_t_1_y.d, pMY) , VEC_MUL(Y_t_1.d, pYY)); +} + +/* + * This is the main compute function. It operates on the matrix in s stripe manner. + * The stripe height is determined by the SIMD engine type. + * Stripe height: "AVX float": 8, "AVX double": 4, "SSE float": 4, "SSE double": 2 + * For each stripe the operations are anti-diagonal based. + * Each anti-diagonal (M_t, Y_t, X_t) depends on the two previous anti-diagonals (M_t_2, X_t_2, Y_t_2, M_t_1, X_t_1, Y_t_1). + * Each stripe (except the fist one) depends on the last row of the previous stripe. + * The last stripe computation handles the addition of the last row of M and X, that's the reason for loop spliting. + */ + +template NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL) +{ + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + int MAVX_COUNT = (ROWS+AVX_LENGTH-1)/AVX_LENGTH; + + /* Probaility arrays */ + SIMD_TYPE p_MM [MAVX_COUNT], p_GAPM [MAVX_COUNT], p_MX [MAVX_COUNT]; + SIMD_TYPE p_XX [MAVX_COUNT], p_MY [MAVX_COUNT], p_YY [MAVX_COUNT]; + + /* For distm precomputation */ + SIMD_TYPE distm1D[MAVX_COUNT]; + + /* Carries the values from each stripe to the next stripe */ + NUMBER shiftOutM[ROWS+COLS+AVX_LENGTH], shiftOutX[ROWS+COLS+AVX_LENGTH], shiftOutY[ROWS+COLS+AVX_LENGTH]; + + /* The vector to keep the anti-diagonals of M, X, Y*/ + /* Current: M_t, X_t, Y_t */ + /* Previous: M_t_1, X_t_1, Y_t_1 */ + /* Previous to previous: M_t_2, X_t_2, Y_t_2 */ + UNION_TYPE M_t, M_t_1, M_t_2, X_t, X_t_1, X_t_2, Y_t, Y_t_1, Y_t_2, M_t_y, M_t_1_y; + + /* Probality vectors */ + SIMD_TYPE pGAPM, pMM, pMX, pXX, pMY, pYY; + + struct timeval start, end; + NUMBER result_avx2; + Context ctx; + UNION_TYPE rs , rsN; + HAP_TYPE hap; + SIMD_TYPE distmSel, distmChosen ; + SIMD_TYPE distm, _1_distm; + + int r, c; + NUMBER zero = ctx._(0.0); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + SIMD_TYPE N_packed256 = VEC_POPCVT_CHAR('N'); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + int remainingRows = (ROWS-1) % AVX_LENGTH; + int stripe_cnt = ((ROWS-1) / AVX_LENGTH) + (remainingRows!=0); + + const int maskBitCnt = MAIN_TYPE_SIZE ; + const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function + + /* Mask precomputation for distm*/ + MASK_TYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; + CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(*tc, COLS, numMaskVecs, maskArr) ; + + char rsArr[AVX_LENGTH] ; + MASK_TYPE lastMaskShiftOut[AVX_LENGTH] ; + + /* Precompute initialization for probabilities and shift vector*/ + CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(ROWS, COLS, shiftOutM, shiftOutX, shiftOutY, + ctx, tc, p_MM, p_GAPM, p_MX, p_XX, p_MY, p_YY, distm1D); + + for (int i=0;i(&tc[b]); + +#ifdef RUN_HYBRID +#define MIN_ACCEPTED 1e-28f + if (result_avxf < MIN_ACCEPTED) { + count++; + result_avxd = CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), d)(&tc[b]); + result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); + } + else + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif + +#ifndef RUN_HYBRID + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif + } + aggregateTimeCompute += (getCurrClk() - lastClk) ; + lastClk = getCurrClk() ; + for (int b=0;b(testcase* tc, double* nextlog); +template float compute_full_prob_sses(testcase* tc, float* nextlog); diff --git a/public/VectorPairHMM/src/main/c++/template.h b/public/VectorPairHMM/src/main/c++/template.h new file mode 100644 index 000000000..ce4dbfc86 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/template.h @@ -0,0 +1,320 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef TEMPLATES_H_ +#define TEMPLATES_H_ + +#include "headers.h" + +#define MM 0 +#define GapM 1 +#define MX 2 +#define XX 3 +#define MY 4 +#define YY 5 + +//#define MROWS 500 +//#define MCOLS 1000 + +#define CAT(X,Y) X####Y +#define CONCAT(X,Y) CAT(X,Y) + +#define ALIGNED __attribute__((aligned(32))) + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256 ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED float ALIGNED f[8]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_F ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m128 ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED float ALIGNED f[4]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_F128 ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128 vecf ; + uint32_t masks[4] ; +} MaskVec_F ; + +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint32_t masks[2] ; +} MaskVec_F128 ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128 ALIGNED f; +} ALIGNED IF_128f ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int ALIGNED i; + ALIGNED float ALIGNED f; +} ALIGNED IF_32 ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256d ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED double ALIGNED f[4]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_D ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m128d ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED double ALIGNED f[2]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_D128 ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128d vecf ; + uint64_t masks[2] ; +} MaskVec_D ; + +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint64_t masks[1] ; +} MaskVec_D128 ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128d ALIGNED f; +} ALIGNED IF_128d ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int64_t ALIGNED i; + ALIGNED double ALIGNED f; +} ALIGNED IF_64 ALIGNED; + + +#define MAX_QUAL 254 +#define MAX_JACOBIAN_TOLERANCE 8.0 +#define JACOBIAN_LOG_TABLE_STEP 0.0001 +#define JACOBIAN_LOG_TABLE_INV_STEP (1.0 / JACOBIAN_LOG_TABLE_STEP) +#define MAXN 70000 +#define LOG10_CACHE_SIZE (4*MAXN) // we need to be able to go up to 2*(2N) when calculating some of the coefficients +#define JACOBIAN_LOG_TABLE_SIZE ((int) (MAX_JACOBIAN_TOLERANCE / JACOBIAN_LOG_TABLE_STEP) + 1) + +template +struct ContextBase +{ + public: + NUMBER ph2pr[128]; + NUMBER INITIAL_CONSTANT; + NUMBER LOG10_INITIAL_CONSTANT; + NUMBER RESULT_THRESHOLD; + + static bool staticMembersInitializedFlag; + static NUMBER jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; + static NUMBER matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; + + static void initializeStaticMembers() + { + if(!staticMembersInitializedFlag) + { + //Order of calls important - Jacobian first, then MatchToMatch + initializeJacobianLogTable(); + initializeMatchToMatchProb(); + staticMembersInitializedFlag = true; + } + } + + static void deleteStaticMembers() + { + if(staticMembersInitializedFlag) + { + staticMembersInitializedFlag = false; + } + } + + //Called only once during library load - don't bother to optimize with single precision fp + static void initializeJacobianLogTable() + { + for (int k = 0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { + jacobianLogTable[k] = (NUMBER)(log10(1.0 + pow(10.0, -((double) k) * JACOBIAN_LOG_TABLE_STEP))); + } + } + + //Called only once per library load - don't bother optimizing with single fp + static void initializeMatchToMatchProb() + { + double LN10 = log(10); + double INV_LN10 = 1.0/LN10; + for (int i = 0, offset = 0; i <= MAX_QUAL; offset += ++i) + for (int j = 0; j <= i; j++) { + double log10Sum = approximateLog10SumLog10(-0.1*i, -0.1*j); + double matchToMatchLog10 = + log1p(-std::min(1.0,pow(10,log10Sum))) * INV_LN10; + matchToMatchProb[offset + j] = (NUMBER)(pow(10,matchToMatchLog10)); + } + } + //Called during computation - use single precision where possible + static int fastRound(NUMBER d) { + return (d > ((NUMBER)0.0)) ? (int) (d + ((NUMBER)0.5)) : (int) (d - ((NUMBER)0.5)); + } + //Called during computation - use single precision where possible + static NUMBER approximateLog10SumLog10(NUMBER small, NUMBER big) { + // make sure small is really the smaller value + if (small > big) { + NUMBER t = big; + big = small; + small = t; + } + + if (isinf(small) == -1 || isinf(big) == -1) + return big; + + NUMBER diff = big - small; + if (diff >= ((NUMBER)MAX_JACOBIAN_TOLERANCE)) + return big; + + // OK, so |y-x| < tol: we use the following identity then: + // we need to compute log10(10^x + 10^y) + // By Jacobian logarithm identity, this is equal to + // max(x,y) + log10(1+10^-abs(x-y)) + // we compute the second term as a table lookup with integer quantization + // we have pre-stored correction for 0,0.1,0.2,... 10.0 + int ind = fastRound((NUMBER)(diff * ((NUMBER)JACOBIAN_LOG_TABLE_INV_STEP))); // hard rounding + return big + jacobianLogTable[ind]; + } +}; + +template +struct Context : public ContextBase +{}; + +template<> +struct Context : public ContextBase +{ + Context():ContextBase() + { + for (int x = 0; x < 128; x++) + ph2pr[x] = pow(10.0, -((double)x) / 10.0); + + INITIAL_CONSTANT = ldexp(1.0, 1020.0); + LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); + RESULT_THRESHOLD = 0.0; + } + + double LOG10(double v){ return log10(v); } + inline double POW(double b, double e) { return pow(b,e); } + + static double _(double n){ return n; } + static double _(float n){ return ((double) n); } +}; + +template<> +struct Context : public ContextBase +{ + Context() : ContextBase() + { + for (int x = 0; x < 128; x++) + { + ph2pr[x] = powf(10.f, -((float)x) / 10.f); + } + + INITIAL_CONSTANT = ldexpf(1.f, 120.f); + LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); + RESULT_THRESHOLD = ldexpf(1.f, -110.f); + } + + float LOG10(float v){ return log10f(v); } + inline float POW(float b, float e) { return powf(b,e); } + + static float _(double n){ return ((float) n); } + static float _(float n){ return n; } +}; + +#define SET_MATCH_TO_MATCH_PROB(output, insQual, delQual) \ +{ \ + int minQual = delQual; \ + int maxQual = insQual; \ + if (insQual <= delQual) \ + { \ + minQual = insQual; \ + maxQual = delQual; \ + } \ + (output) = (MAX_QUAL < maxQual) ? \ + ((NUMBER)1.0) - ctx.POW(((NUMBER)10), ctx.approximateLog10SumLog10(((NUMBER)-0.1)*minQual, ((NUMBER)-0.1)*maxQual)) \ + : ctx.matchToMatchProb[((maxQual * (maxQual + 1)) >> 1) + minQual]; \ +} + +typedef struct +{ + int rslen, haplen; + /*int *q, *i, *d, *c;*/ + /*int q[MROWS], i[MROWS], d[MROWS], c[MROWS];*/ + char *q, *i, *d, *c; + char *hap, *rs; + int *ihap; + int *irs; +} testcase; + +int normalize(char c); +int read_testcase(testcase *tc, FILE* ifp=0); + + +#define MIN_ACCEPTED 1e-28f +#define NUM_DISTINCT_CHARS 5 +#define AMBIG_CHAR 4 + +class ConvertChar { + + static uint8_t conversionTable[255] ; + +public: + + static void init() { + assert (NUM_DISTINCT_CHARS == 5) ; + assert (AMBIG_CHAR == 4) ; + + conversionTable['A'] = 0 ; + conversionTable['C'] = 1 ; + conversionTable['T'] = 2 ; + conversionTable['G'] = 3 ; + conversionTable['N'] = 4 ; + } + + static inline uint8_t get(uint8_t input) { + return conversionTable[input] ; + } + +}; + + +#endif + + diff --git a/public/VectorPairHMM/src/main/c++/utils.cc b/public/VectorPairHMM/src/main/c++/utils.cc new file mode 100644 index 000000000..9f83cffa2 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/utils.cc @@ -0,0 +1,493 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "template.h" +#include "utils.h" +#include "vector_defs.h" +#include "LoadTimeInitializer.h" +using namespace std; + +//static members from ConvertChar +uint8_t ConvertChar::conversionTable[255]; +//Global function pointers in utils.h +float (*g_compute_full_prob_float)(testcase *tc, float* before_last_log) = 0; +double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log) = 0; +//Static members in ContextBase +bool ContextBase::staticMembersInitializedFlag = false; +double ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +double ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; +bool ContextBase::staticMembersInitializedFlag = false; +float ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +float ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; + + +bool is_avx_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 28)&1) == 1; +} + +bool is_sse41_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 19)&1) == 1; +} + +bool is_sse42_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 20)&1) == 1; +} + +uint64_t get_machine_capabilities() +{ + uint64_t machine_mask = 0ull; + if(is_avx_supported()) + machine_mask |= (1 << AVX_CUSTOM_IDX); + if(is_sse42_supported()) + machine_mask |= (1 << SSE42_CUSTOM_IDX); + if(is_sse41_supported()) + machine_mask |= (1 << SSE41_CUSTOM_IDX); + return machine_mask; +} + +void initialize_function_pointers(uint64_t mask) +{ + //mask = 0ull; + //mask = (1 << SSE41_CUSTOM_IDX); + if(is_avx_supported() && (mask & (1<< AVX_CUSTOM_IDX))) + { + cout << "Using AVX accelerated implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob_avxs; + g_compute_full_prob_double = compute_full_prob_avxd; + } + else + if(is_sse41_supported() && (mask & ((1<< SSE41_CUSTOM_IDX) | (1<; + g_compute_full_prob_double = compute_full_prob_ssed; + } + else + { + cout << "Using un-vectorized C++ implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob; + g_compute_full_prob_double = compute_full_prob; + } +} + +int normalize(char c) +{ + return ((int) (c - 33)); +} + +int read_testcase(testcase *tc, FILE* ifp) +{ + char *q, *i, *d, *c, *line = NULL; + int _q, _i, _d, _c; + int x, size = 0; + ssize_t read; + + + read = getline(&line, (size_t *) &size, ifp == 0 ? stdin : ifp); + if (read == -1) + { + free(line); + return -1; + } + + + tc->hap = (char *) malloc(size); + tc->rs = (char *) malloc(size); + q = (char *) malloc(size); + i = (char *) malloc(size); + d = (char *) malloc(size); + c = (char *) malloc(size); + + if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) + return -1; + + + tc->haplen = strlen(tc->hap); + tc->rslen = strlen(tc->rs); + assert(strlen(q) == tc->rslen); + assert(strlen(i) == tc->rslen); + assert(strlen(d) == tc->rslen); + assert(strlen(c) == tc->rslen); + //assert(tc->rslen < MROWS); + //tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); + //tc->irs = (int *) malloc(tc->rslen*sizeof(int)); + + tc->q = (char *) malloc(sizeof(char) * tc->rslen); + tc->i = (char *) malloc(sizeof(char) * tc->rslen); + tc->d = (char *) malloc(sizeof(char) * tc->rslen); + tc->c = (char *) malloc(sizeof(char) * tc->rslen); + + for (x = 0; x < tc->rslen; x++) + { + _q = normalize(q[x]); + _i = normalize(i[x]); + _d = normalize(d[x]); + _c = normalize(c[x]); + tc->q[x] = (_q < 6) ? 6 : _q; + //tc->q[x] = _q; + tc->i[x] = _i; + tc->d[x] = _d; + tc->c[x] = _c; + //tc->irs[x] = tc->rs[x]; + } + //for (x = 0; x < tc->haplen; x++) + //tc->ihap[x] = tc->hap[x]; + + free(q); + free(i); + free(d); + free(c); + free(line); + + + + return 0; +} + +unsigned MAX_LINE_LENGTH = 65536; +int convToInt(std::string s) +{ + int i; + std::istringstream strin(s); + strin >> i; + return i; +} + +void tokenize(std::ifstream& fptr, std::vector& tokens) +{ + int i = 0; + std::string tmp; + std::vector myVec; + vector line; + line.clear(); + line.resize(MAX_LINE_LENGTH); + vector tmpline; + tmpline.clear(); + tmpline.resize(MAX_LINE_LENGTH); + myVec.clear(); + + while(!fptr.eof()) + { + i = 0; + bool still_read_line = true; + unsigned line_position = 0; + while(still_read_line) + { + fptr.getline(&(tmpline[0]), MAX_LINE_LENGTH); + if(line_position + MAX_LINE_LENGTH > line.size()) + line.resize(2*line.size()); + for(unsigned i=0;i> std::skipws >> tmp; + if(tmp != "") + { + myVec.push_back(tmp); + ++i; + //std::cout < 0) + break; + } + tokens.clear(); + //std::cout << "Why "< tokens; + tokens.clear(); + tokenize(fptr, tokens); + if(tokens.size() == 0) + return -1; + tc->hap = new char[tokens[0].size()+2]; + tc->haplen = tokens[0].size(); + memcpy(tc->hap, tokens[0].c_str(), tokens[0].size()); + tc->rs = new char[tokens[1].size()+2]; + tc->rslen = tokens[1].size(); + tc->q = new char[tc->rslen]; + tc->i = new char[tc->rslen]; + tc->d = new char[tc->rslen]; + tc->c = new char[tc->rslen]; + //cout << "Lengths "<haplen <<" "<rslen<<"\n"; + memcpy(tc->rs, tokens[1].c_str(),tokens[1].size()); + assert(tokens.size() == 2 + 4*(tc->rslen)); + //assert(tc->rslen < MROWS); + for(unsigned j=0;jrslen;++j) + tc->q[j] = (char)convToInt(tokens[2+0*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->i[j] = (char)convToInt(tokens[2+1*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->d[j] = (char)convToInt(tokens[2+2*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->c[j] = (char)convToInt(tokens[2+3*tc->rslen+j]); + + if(reformat) + { + ofstream ofptr; + ofptr.open("reformat/debug_dump.txt",first_call ? ios::out : ios::app); + assert(ofptr.is_open()); + ofptr << tokens[0] << " "; + ofptr << tokens[1] << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->q[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->i[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->d[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->c[j]+33)); + ofptr << " 0 false\n"; + + ofptr.close(); + first_call = false; + } + + + return tokens.size(); +} + +double getCurrClk() { + struct timeval tv ; + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} + +inline unsigned long long rdtsc(void) +{ + unsigned hi, lo; + __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); + return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 ); +} + +void get_time(struct timespec* store_struct) +{ + clock_gettime(CLOCK_REALTIME, store_struct); +} + +uint64_t diff_time(struct timespec& prev_time) +{ + struct timespec curr_time; + clock_gettime(CLOCK_REALTIME, &curr_time); + return (uint64_t)((curr_time.tv_sec-prev_time.tv_sec)*1000000000+(curr_time.tv_nsec-prev_time.tv_nsec)); +} + + +#ifdef USE_PAPI +#include "papi.h" +#define NUM_PAPI_COUNTERS 4 +#endif + +void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size, bool do_check) +{ + FILE* fptr = 0; + ifstream ifptr; + if(use_old_read_testcase) + { + fptr = fopen(filename,"r"); + assert(fptr); + } + else + { + ifptr.open(filename); + assert(ifptr.is_open()); + } + vector tc_vector; + tc_vector.clear(); + testcase tc; + uint64_t vector_compute_time = 0; + uint64_t baseline_compute_time = 0; + uint64_t num_double_calls = 0; + unsigned num_testcases = 0; + bool all_ok = do_check ? true : false; +#ifdef USE_PAPI + uint32_t all_mask = (0); + uint32_t no_usr_mask = (1 << 16); //bit 16 user mode, bit 17 kernel mode + uint32_t no_kernel_mask = (1 << 17); //bit 16 user mode, bit 17 kernel mode + PAPI_num_counters(); + int events[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + char* eventnames[NUM_PAPI_COUNTERS]= { "cycles", "itlb_walk_cycles", "dtlb_load_walk_cycles", "dtlb_store_walk_cycles" }; + assert(PAPI_event_name_to_code("UNHALTED_REFERENCE_CYCLES:u=1:k=1",&(events[0])) == PAPI_OK); + assert(PAPI_event_name_to_code("ITLB_MISSES:WALK_DURATION", &(events[1])) == PAPI_OK); + assert(PAPI_event_name_to_code("DTLB_LOAD_MISSES:WALK_DURATION", &(events[2])) == PAPI_OK); + assert(PAPI_event_name_to_code("DTLB_STORE_MISSES:WALK_DURATION", &(events[3])) == PAPI_OK); + long long values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + long long accum_values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; +#endif + while(1) + { + int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); + if(break_value >= 0) + tc_vector.push_back(tc); + if(tc_vector.size() == BATCH_SIZE || (break_value < 0 && tc_vector.size() > 0)) + { + vector results_vec; + vector baseline_results_vec; + results_vec.clear(); + baseline_results_vec.clear(); + results_vec.resize(tc_vector.size()); + baseline_results_vec.resize(tc_vector.size()); + struct timespec start_time; +#ifdef USE_PAPI + assert(PAPI_start_counters(events, NUM_PAPI_COUNTERS) == PAPI_OK); +#endif + get_time(&start_time); +#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) +#ifdef DO_REPEAT_PROFILING + for(unsigned z=0;z<10;++z) +#endif + { + for(unsigned i=0;i(&tc); + baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); + baseline_results_vec[i] = baseline_result; + } + baseline_compute_time += diff_time(start_time); + for(unsigned i=0;i 1e-5 && rel_error > 1e-5) + { + cout << std::scientific << baseline_result << " "< +std::string to_string(T obj) +{ + std::stringstream ss; + std::string ret_string; + ss.clear(); + ss << std::scientific << obj; + ss >> ret_string; + ss.clear(); + return ret_string; +} +void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); + +int read_mod_testcase(std::ifstream& fptr, testcase* tc, bool reformat=false); + +bool is_avx_supported(); +bool is_sse42_supported(); +extern float (*g_compute_full_prob_float)(testcase *tc, float *before_last_log); +extern double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log); +void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline); +template +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); +double getCurrClk(); +void get_time(struct timespec* x); +uint64_t diff_time(struct timespec& prev_time); + +//bit 0 is sse4.2, bit 1 is AVX +enum ProcessorCapabilitiesEnum +{ + SSE41_CUSTOM_IDX=0, + SSE42_CUSTOM_IDX, + AVX_CUSTOM_IDX +}; +#define ENABLE_ALL_HARDWARE_FEATURES 0xFFFFFFFFFFFFFFFFull +uint64_t get_machine_capabilities(); +void initialize_function_pointers(uint64_t mask=ENABLE_ALL_HARDWARE_FEATURES); +void do_compute(char* filename, bool use_old_read_testcase=true, unsigned chunk_size=10000, bool do_check=true); + +//#define DO_WARMUP +//#define DO_REPEAT_PROFILING +//#define DUMP_COMPUTE_VALUES 1 +#define BATCH_SIZE 10000 +#define RUN_HYBRID + +#endif diff --git a/public/VectorPairHMM/src/main/c++/vector_defs.h b/public/VectorPairHMM/src/main/c++/vector_defs.h new file mode 100644 index 000000000..2aca9565f --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/vector_defs.h @@ -0,0 +1,55 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX +#undef SIMD_ENGINE_SSE + +#define SIMD_ENGINE avx +#define SIMD_ENGINE_AVX + +#include "define-float.h" +#include "vector_function_prototypes.h" + +#include "define-double.h" +#include "vector_function_prototypes.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX + +#define SIMD_ENGINE sse +#define SIMD_ENGINE_SSE + + +#include "define-sse-float.h" +#include "vector_function_prototypes.h" + +#include "define-sse-double.h" +#include "vector_function_prototypes.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX +#undef SIMD_ENGINE_SSE + diff --git a/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h b/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h new file mode 100644 index 000000000..c0fddc394 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h @@ -0,0 +1,44 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +inline void CONCAT(CONCAT(_vector_shift,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); +inline void CONCAT(CONCAT(_vector_shift_last,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); +inline void CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); +inline void CONCAT(CONCAT(init_masks_for_row_,SIMD_ENGINE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess); +inline void CONCAT(CONCAT(update_masks_for_cols_,SIMD_ENGINE), PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt); +inline void CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, SIMD_TYPE& distm, SIMD_TYPE& _1_distm, SIMD_TYPE& distmChosen); +template inline void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, SIMD_TYPE *p_MM, SIMD_TYPE *p_GAPM, SIMD_TYPE *p_MX, SIMD_TYPE *p_XX, SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, SIMD_TYPE *distm1D); +template inline void CONCAT(CONCAT(stripINITIALIZATION,SIMD_ENGINE), PRECISION)( + int stripIdx, Context ctx, testcase *tc, SIMD_TYPE &pGAPM, SIMD_TYPE &pMM, SIMD_TYPE &pMX, SIMD_TYPE &pXX, SIMD_TYPE &pMY, SIMD_TYPE &pYY, + SIMD_TYPE &rs, UNION_TYPE &rsN, SIMD_TYPE &distm, SIMD_TYPE &_1_distm, SIMD_TYPE *distm1D, SIMD_TYPE N_packed256, SIMD_TYPE *p_MM , SIMD_TYPE *p_GAPM , + SIMD_TYPE *p_MX, SIMD_TYPE *p_XX , SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM); +inline SIMD_TYPE CONCAT(CONCAT(computeDISTM,SIMD_ENGINE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, SIMD_TYPE rs, UNION_TYPE rsN, SIMD_TYPE N_packed256, + SIMD_TYPE distm, SIMD_TYPE _1_distm); +inline void CONCAT(CONCAT(computeMXY,SIMD_ENGINE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, + UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, + SIMD_TYPE pMM, SIMD_TYPE pGAPM, SIMD_TYPE pMX, SIMD_TYPE pXX, SIMD_TYPE pMY, SIMD_TYPE pYY, SIMD_TYPE distmSel); +template NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL); + diff --git a/public/sting-root/pom.xml b/public/sting-root/pom.xml index 171eb7620..549a99ae6 100644 --- a/public/sting-root/pom.xml +++ b/public/sting-root/pom.xml @@ -335,7 +335,11 @@ maven-assembly-plugin 2.4 - + + org.apache.maven.plugins + maven-enforcer-plugin + 1.3.1 + + ${sting-utils.basedir}/src/main/resources/org/broadinstitute/sting/utils/pairhmm + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + display-info + + validate + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + + exec + + compile + + make + src/main/c++ + + ${java.home} + ${project.build.directory} + + + + + + + + + org.apache.maven.plugins + maven-install-plugin + + true + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + default-install + + copy-resources + + install + + ${pairhmm.resources.directory} + + + ${project.build.directory} + + **/* + + + + + + + + + + + com.google.code.sortpom + maven-sortpom-plugin + + false + custom_1 + \n + ${sourceEncoding} + true + scope + 4 + false + + + + + diff --git a/public/VectorPairHMM/src/main/c++/.gitignore b/public/VectorPairHMM/src/main/c++/.gitignore new file mode 100644 index 000000000..d791ffd80 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/.gitignore @@ -0,0 +1,16 @@ +.svn +*.o +*.so +tests +.deps +hmm_Mohammad +pairhmm-template-main +*.swp +*.class +checker +reformat +subdir_checkout.sh +avx/ +sse/ +triplicate.sh + diff --git a/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc new file mode 100644 index 000000000..0e3026f65 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/LoadTimeInitializer.cc @@ -0,0 +1,206 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "LoadTimeInitializer.h" +#include "utils.h" +using namespace std; +char* LoadTimeInitializerStatsNames[] = +{ + "num_regions", + "num_reads", + "num_haplotypes", + "num_testcases", + "num_double_invocations", + "haplotype_length", + "readlength", + "product_read_length_haplotype_length", + "dummy" +}; + +LoadTimeInitializer g_load_time_initializer; + +LoadTimeInitializer::LoadTimeInitializer() //will be called when library is loaded +{ + ConvertChar::init(); +#ifndef DISABLE_FTZ + //Very important to get good performance on Intel processors + //Function: enabling FTZ converts denormals to 0 in hardware + //Denormals cause microcode to insert uops into the core causing big slowdown + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + cout << "FTZ enabled - may decrease accuracy if denormal numbers encountered\n"; +#else + cout << "FTZ is not set - may slow down performance if denormal numbers encountered\n"; +#endif + //Profiling: times for compute and transfer (either bytes copied or pointers copied) + m_compute_time = 0; + m_data_transfer_time = 0; + m_bytes_copied = 0; + + //Initialize profiling counters + for(unsigned i=0;i::initializeStaticMembers(); + Context::initializeStaticMembers(); + + cout.flush(); +} + +void LoadTimeInitializer::print_profiling() +{ + double mean = 0; + double variance = 0; + uint64_t denominator = 1; + cout << "Time spent in compute_testcases "< C++) "<::iterator mI = m_filename_to_fptr.find(filename); + ofstream* fptr = 0; + if(mI == m_filename_to_fptr.end()) + { + m_filename_to_fptr[filename] = new ofstream(); + fptr = m_filename_to_fptr[filename]; + //File never seen before + if(m_written_files_set.find(filename) == m_written_files_set.end()) + { + to_append = false; + m_written_files_set.insert(filename); + } + fptr->open(filename.c_str(), to_append ? ios::app : ios::out); + assert(fptr->is_open()); + } + else + fptr = (*mI).second; + //ofstream fptr; + //fptr.open(filename.c_str(), to_append ? ofstream::app : ofstream::out); + (*fptr) << s; + if(add_newline) + (*fptr) << "\n"; + //fptr.close(); +} +void LoadTimeInitializer::debug_close() +{ + for(map::iterator mB = m_filename_to_fptr.begin(), mE = m_filename_to_fptr.end(); + mB != mE;mB++) + { + (*mB).second->close(); + delete (*mB).second; + } + m_filename_to_fptr.clear(); +} + +void LoadTimeInitializer::dump_sandbox(testcase& tc, unsigned tc_idx, unsigned numReads, unsigned numHaplotypes) +{ + unsigned haplotypeLength = tc.haplen; + unsigned readLength = tc.rslen; + ofstream& dumpFptr = m_sandbox_fptr; + for(unsigned k=0;k +#include "template.h" + +enum LoadTimeInitializerStatsEnum +{ + NUM_REGIONS_IDX=0, + NUM_READS_IDX, + NUM_HAPLOTYPES_IDX, + NUM_TESTCASES_IDX, + NUM_DOUBLE_INVOCATIONS_IDX, + HAPLOTYPE_LENGTH_IDX, + READ_LENGTH_IDX, + PRODUCT_READ_LENGTH_HAPLOTYPE_LENGTH_IDX, + TOTAL_NUMBER_STATS +}; +extern char* LoadTimeInitializerStatsNames[]; + +class LoadTimeInitializer +{ + public: + LoadTimeInitializer(); //will be called when library is loaded + ~LoadTimeInitializer() + { + delete m_buffer; + } + void print_profiling(); + void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); + void debug_close(); + + void dump_sandbox(testcase& tc, unsigned tc_idx, unsigned numReads, unsigned numHaplotypes); + void open_sandbox() { m_sandbox_fptr.open("sandbox.txt", std::ios::app); } + void close_sandbox() { m_sandbox_fptr.close(); } + + jfieldID m_readBasesFID; + jfieldID m_readQualsFID; + jfieldID m_insertionGOPFID; + jfieldID m_deletionGOPFID; + jfieldID m_overallGCPFID; + jfieldID m_haplotypeBasesFID; + //profiling - update stats + void update_stat(LoadTimeInitializerStatsEnum stat_idx, uint64_t value); + //timing in nanoseconds + uint64_t m_compute_time; + uint64_t m_data_transfer_time; + //bytes copied + uint64_t m_bytes_copied; + unsigned get_buffer_size() { return m_buffer_size; } + char* get_buffer() { return (char*)m_buffer; } + private: + std::map m_filename_to_fptr; + std::set m_written_files_set; + std::ofstream m_sandbox_fptr; + //used to compute various stats + uint64_t m_sum_stats[TOTAL_NUMBER_STATS]; + double m_sum_square_stats[TOTAL_NUMBER_STATS]; + uint64_t m_min_stats[TOTAL_NUMBER_STATS]; + uint64_t m_max_stats[TOTAL_NUMBER_STATS]; + unsigned m_buffer_size; + uint64_t* m_buffer; +}; +extern LoadTimeInitializer g_load_time_initializer; + +#define SIZE_PER_TESTCASE 6*10000 +#define SIZE_PER_BUFFER 10000 + +#endif diff --git a/public/VectorPairHMM/src/main/c++/Makefile b/public/VectorPairHMM/src/main/c++/Makefile new file mode 100644 index 000000000..354bca0bb --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Makefile @@ -0,0 +1,114 @@ +#Copyright (c) 2012 The Broad Institute + +#Permission is hereby granted, free of charge, to any person +#obtaining a copy of this software and associated documentation +#files (the "Software"), to deal in the Software without +#restriction, including without limitation the rights to use, +#copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the +#Software is furnished to do so, subject to the following +#conditions: + +#The above copyright notice and this permission notice shall be +#included in all copies or substantial portions of the Software. + +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +#THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + + +#OMPCFLAGS=-fopenmp +#OMPLFLAGS=-fopenmp #-openmp-link static + +#CFLAGS=-O2 -std=c++11 -W -Wall -march=corei7-avx -Wa,-q -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas +#CFLAGS=-O2 -W -Wall -march=corei7 -mfpmath=sse -msse4.2 -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas + +JRE_HOME?=/opt/jdk1.7.0_25/jre +JNI_COMPILATION_FLAGS=-D_REENTRANT -fPIC -I${JRE_HOME}/../include -I${JRE_HOME}/../include/linux + +COMMON_COMPILATION_FLAGS=$(JNI_COMPILATION_FLAGS) -O3 -W -Wall -pedantic $(OMPCFLAGS) -Wno-unknown-pragmas +CC=icc +CXX=icc + +LDFLAGS=-lm -lrt $(OMPLDFLAGS) +ifdef DISABLE_FTZ + COMMON_COMPILATION_FLAGS+=-DDISABLE_FTZ -no-ftz +endif + +PAPI_DIR=/home/karthikg/softwares/papi-5.3.0 +ifdef USE_PAPI + ifeq ($(USE_PAPI),1) + COMMON_COMPILATION_FLAGS+=-I$(PAPI_DIR)/include -DUSE_PAPI + LDFLAGS+=-L$(PAPI_DIR)/lib -lpapi + endif +endif + +ifdef DISABLE_FTZ + COMMON_COMPILATION_FLAGS+=-DDISABLE_FTZ -no-ftz +endif + +BIN=libVectorLoglessPairHMM.so pairhmm-template-main checker +#BIN=checker + +DEPDIR=.deps +DF=$(DEPDIR)/$(*).d + +#Common across libJNI and sandbox +COMMON_SOURCES=utils.cc avx_function_instantiations.cc baseline.cc sse_function_instantiations.cc LoadTimeInitializer.cc +#Part of libJNI +LIBSOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc Sandbox.cc $(COMMON_SOURCES) +SOURCES=$(LIBSOURCES) pairhmm-template-main.cc pairhmm-1-base.cc +LIBOBJECTS=$(LIBSOURCES:.cc=.o) +COMMON_OBJECTS=$(COMMON_SOURCES:.cc=.o) + + +#No vectorization for these files +NO_VECTOR_SOURCES=org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc pairhmm-template-main.cc pairhmm-1-base.cc utils.cc baseline.cc LoadTimeInitializer.cc Sandbox.cc +#Use -xAVX for these files +AVX_SOURCES=avx_function_instantiations.cc +#Use -xSSE4.2 for these files +SSE_SOURCES=sse_function_instantiations.cc + +NO_VECTOR_OBJECTS=$(NO_VECTOR_SOURCES:.cc=.o) +AVX_OBJECTS=$(AVX_SOURCES:.cc=.o) +SSE_OBJECTS=$(SSE_SOURCES:.cc=.o) +$(NO_VECTOR_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) +$(AVX_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xAVX +$(SSE_OBJECTS): CXXFLAGS=$(COMMON_COMPILATION_FLAGS) -xSSE4.2 +OBJECTS=$(NO_VECTOR_OBJECTS) $(AVX_OBJECTS) $(SSE_OBJECTS) + +all: $(BIN) Sandbox.class copied_lib + +-include $(addprefix $(DEPDIR)/,$(SOURCES:.cc=.d)) + +checker: pairhmm-1-base.o $(COMMON_OBJECTS) + $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) + +pairhmm-template-main: pairhmm-template-main.o $(COMMON_OBJECTS) + $(CXX) $(OMPLFLAGS) -o $@ $^ $(LDFLAGS) + +libVectorLoglessPairHMM.so: $(LIBOBJECTS) + $(CXX) $(OMPLFLAGS) -shared -static-intel -o $@ $(LIBOBJECTS) ${LDFLAGS} + + +$(OBJECTS): %.o: %.cc + @mkdir -p $(DEPDIR) + $(CXX) -c -MMD -MF $(DF) $(CXXFLAGS) $(OUTPUT_OPTION) $< + +Sandbox.class: Sandbox.java + javac Sandbox.java + +copied_lib: libVectorLoglessPairHMM.so +ifdef OUTPUT_DIR + mkdir -p $(OUTPUT_DIR) + rsync -a libVectorLoglessPairHMM.so $(OUTPUT_DIR)/ +endif + +clean: + rm -rf $(BIN) *.o $(DEPDIR) *.class diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.cc b/public/VectorPairHMM/src/main/c++/Sandbox.cc new file mode 100644 index 000000000..985b19ae9 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox.cc @@ -0,0 +1,106 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "Sandbox.h" +#include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" +#include "utils.h" +#include "jni_common.h" +/* + * Class: Sandbox + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_Sandbox_jniGetMachineType + (JNIEnv * env, jobject thisObj) +{ + return 0; +} + +/* + * Class: Sandbox + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeClassFieldsAndMachineMask + (JNIEnv* env, jobject thisObject, jclass readDataHolderClass, jclass haplotypeDataHolderClass, jlong mask) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask(env, thisObject, readDataHolderClass, + haplotypeDataHolderClass, mask); +} + +/* + * Class: Sandbox + * Method: jniInitializeHaplotypes + * Signature: (I[LSandbox/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeHaplotypes + (JNIEnv * env, jobject thisObject, jint numHaplotypes, jobjectArray haplotypeDataArray) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes(env, thisObject, numHaplotypes, haplotypeDataArray); +} + +/* + * Class: Sandbox + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniFinalizeRegion + (JNIEnv * env, jobject thisObject) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniFinalizeRegion(env, thisObject); +} + + +/* + * Class: Sandbox + * Method: jniComputeLikelihoods + * Signature: (II[LSandbox/JNIReadDataHolderClass;[LSandbox/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniComputeLikelihoods + (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, + jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +{ + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods(env, thisObject, + numReads, numHaplotypes, readDataArray, haplotypeDataArray, likelihoodArray, maxNumThreadsToUse); +} +/* + * Class: Sandbox + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniClose + (JNIEnv* env, jobject thisObject) +{ Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose(env, thisObject); } + +JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative + (JNIEnv* env, jobject thisObject, jstring fileNameString) +{ + const char* fileName = env->GetStringUTFChars(fileNameString, 0); + char local_array[800]; + strncpy(local_array, fileName, 200); + env->ReleaseStringUTFChars(fileNameString, fileName); + do_compute(local_array, true, 10000, false); +} + diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.h b/public/VectorPairHMM/src/main/c++/Sandbox.h new file mode 100644 index 000000000..486a1c095 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox.h @@ -0,0 +1,96 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox */ + +#ifndef _Included_Sandbox +#define _Included_Sandbox +#ifdef __cplusplus +extern "C" { +#endif +#undef Sandbox_enableAll +#define Sandbox_enableAll -1LL +/* + * Class: Sandbox + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_Sandbox_jniGetMachineType + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeClassFieldsAndMachineMask + (JNIEnv *, jobject, jclass, jclass, jlong); + +/* + * Class: Sandbox + * Method: jniInitializeHaplotypes + * Signature: (I[LSandbox/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: Sandbox + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniFinalizeRegion + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: jniComputeLikelihoods + * Signature: (II[LSandbox/JNIReadDataHolderClass;[LSandbox/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); + +/* + * Class: Sandbox + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_Sandbox_jniClose + (JNIEnv *, jobject); + +/* + * Class: Sandbox + * Method: doEverythingNative + * Signature: ([B)V + */ +JNIEXPORT void JNICALL Java_Sandbox_doEverythingNative + (JNIEnv *, jobject, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/Sandbox.java b/public/VectorPairHMM/src/main/c++/Sandbox.java new file mode 100644 index 000000000..d6b7c2eae --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox.java @@ -0,0 +1,305 @@ +/* +* Copyright (c) 2012 The Broad Institute +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +* THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +package org.broadinstitute.sting.utils.vectorpairhmm; + +import java.util.List; +import java.util.LinkedList; +import java.util.Map; +import java.util.HashMap; +import java.io.File; +import java.util.Scanner; +import java.io.IOException; +import java.io.FileNotFoundException; +import java.io.InputStreamReader; + +public class Sandbox { + + private long setupTime = 0; + private long computeTime = 0; + //Used to copy references to byteArrays to JNI from reads + protected class JNIReadDataHolderClass { + public byte[] readBases = null; + public byte[] readQuals = null; + public byte[] insertionGOP = null; + public byte[] deletionGOP = null; + public byte[] overallGCP = null; + } + + //Used to copy references to byteArrays to JNI from haplotypes + protected class JNIHaplotypeDataHolderClass { + public byte[] haplotypeBases = null; + } + + /** + * Return 64-bit mask representing machine capabilities + * Bit 0 is LSB, bit 63 MSB + * Bit 0 represents sse4.2 availability + * Bit 1 represents AVX availability + */ + public native long jniGetMachineType(); + public static final long enableAll = 0xFFFFFFFFFFFFFFFFl; + + + /** + * Function to initialize the fields of JNIReadDataHolderClass and JNIHaplotypeDataHolderClass from JVM. + * C++ codegets FieldIDs for these classes once and re-uses these IDs for the remainder of the program. Field IDs do not + * change per JVM session + * @param readDataHolderClass class type of JNIReadDataHolderClass + * @param haplotypeDataHolderClass class type of JNIHaplotypeDataHolderClass + * @param mask mask is a 64 bit integer identical to the one received from jniGetMachineType(). Users can disable usage of some hardware features by zeroing some bits in the mask + * */ + private native void jniInitializeClassFieldsAndMachineMask(Class readDataHolderClass, Class haplotypeDataHolderClass, long mask); + + private static Boolean isVectorLoglessPairHMMLibraryLoaded = false; + //The constructor is called only once inside PairHMMLikelihoodCalculationEngine + public Sandbox() { + synchronized(isVectorLoglessPairHMMLibraryLoaded) { + //Load the library and initialize the FieldIDs + if(!isVectorLoglessPairHMMLibraryLoaded) { + System.loadLibrary("VectorLoglessPairHMM"); + isVectorLoglessPairHMMLibraryLoaded = true; + jniInitializeClassFieldsAndMachineMask(JNIReadDataHolderClass.class, JNIHaplotypeDataHolderClass.class, enableAll); //need to do this only once + } + } + } + + private native void jniInitializeHaplotypes(final int numHaplotypes, JNIHaplotypeDataHolderClass[] haplotypeDataArray); + + //Used to transfer data to JNI + //Since the haplotypes are the same for all calls to computeLikelihoods within a region, transfer the haplotypes only once to the JNI per region + public void initialize(final List haplotypes) { + int numHaplotypes = haplotypes.size(); + JNIHaplotypeDataHolderClass[] haplotypeDataArray = new JNIHaplotypeDataHolderClass[numHaplotypes]; + int idx = 0; + for(final JNIHaplotypeDataHolderClass currHaplotype : haplotypes) + { + haplotypeDataArray[idx] = new JNIHaplotypeDataHolderClass(); + haplotypeDataArray[idx].haplotypeBases = currHaplotype.haplotypeBases; + ++idx; + } + jniInitializeHaplotypes(numHaplotypes, haplotypeDataArray); + } + /** + * Tell JNI to release arrays - really important if native code is directly accessing Java memory, if not + * accessing Java memory directly, still important to release memory from C++ + */ + private native void jniFinalizeRegion(); + + + public void finalizeRegion() + { + jniFinalizeRegion(); + } + + /** + * Real compute kernel + */ + private native void jniComputeLikelihoods(int numReads, int numHaplotypes, JNIReadDataHolderClass[] readDataArray, + JNIHaplotypeDataHolderClass[] haplotypeDataArray, double[] likelihoodArray, int maxNumThreadsToUse); + + public void computeLikelihoods(final List reads, final List haplotypes) { + //System.out.println("Region : "+reads.size()+" x "+haplotypes.size()); + long startTime = System.nanoTime(); + int readListSize = reads.size(); + int numHaplotypes = haplotypes.size(); + int numTestcases = readListSize*numHaplotypes; + JNIReadDataHolderClass[] readDataArray = new JNIReadDataHolderClass[readListSize]; + int idx = 0; + for(JNIReadDataHolderClass read : reads) + { + readDataArray[idx] = new JNIReadDataHolderClass(); + readDataArray[idx].readBases = read.readBases; + readDataArray[idx].readQuals = read.readQuals; + readDataArray[idx].insertionGOP = read.insertionGOP; + readDataArray[idx].deletionGOP = read.deletionGOP; + readDataArray[idx].overallGCP = read.overallGCP; + ++idx; + } + + double[] mLikelihoodArray = new double[readListSize*numHaplotypes]; //to store results + setupTime += (System.nanoTime() - startTime); + //for(reads) + // for(haplotypes) + // compute_full_prob() + jniComputeLikelihoods(readListSize, numHaplotypes, readDataArray, null, mLikelihoodArray, 12); + + computeTime += (System.nanoTime() - startTime); + } + + /** + * Print final profiling information from native code + */ + public native void jniClose(); + public void close() + { + System.out.println("Time spent in setup for JNI call : "+(setupTime*1e-9)+" compute time : "+(computeTime*1e-9)); + jniClose(); + } + + public void parseSandboxFile(String filename) + { + File file = new File(filename); + Scanner input = null; + try + { + input = new Scanner(file); + } + catch(FileNotFoundException e) + { + System.err.println("File "+filename+" cannot be found/read"); + return; + } + int idx = 0; + int numReads = 0; + int numHaplotypes = 0; + int readIdx = 0, testCaseIdx = 0, haplotypeIdx = 0; + LinkedList haplotypeList = new LinkedList(); + LinkedList readList = new LinkedList(); + + byte[][] byteArray = new byte[6][]; + boolean firstLine = true; + String[] currTokens = new String[8]; + while(input.hasNextLine()) + { + String line = input.nextLine(); + Scanner lineScanner = new Scanner(line); + idx = 0; + while(lineScanner.hasNext()) + currTokens[idx++] = lineScanner.next(); + if(idx == 0) + break; + assert(idx >= 6); + //start of new region + if(idx == 8) + { + if(!firstLine) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + try + { + numReads = Integer.parseInt(currTokens[6]); + } + catch(NumberFormatException e) + { + numReads = 1; + } + try + { + numHaplotypes = Integer.parseInt(currTokens[7]); + } + catch(NumberFormatException e) + { + numHaplotypes = 1; + } + haplotypeIdx = readIdx = testCaseIdx = 0; + readList.clear(); + haplotypeList.clear(); + } + if(haplotypeIdx < numHaplotypes) + { + JNIHaplotypeDataHolderClass X = new JNIHaplotypeDataHolderClass(); + X.haplotypeBases = currTokens[0].getBytes(); + haplotypeList.add(X); + } + if(testCaseIdx%numHaplotypes == 0) + { + JNIReadDataHolderClass X = new JNIReadDataHolderClass(); + X.readBases = currTokens[1].getBytes(); + for(int i=2;i<6;++i) + { + byteArray[i] = currTokens[i].getBytes(); + for(int j=0;j 0 && readList.size() > 0) + { + initialize(haplotypeList); + computeLikelihoods(readList, haplotypeList); + finalizeRegion(); + } + + close(); + input.close(); + } + + private native void doEverythingNative(String filename); + + public static void main(String[] args) + { + if(args.length <= 0) + { + System.err.println("Needs 1 argument - "); + System.exit(-1); + } + //// Get runtime + //java.lang.Runtime rt = java.lang.Runtime.getRuntime(); + //// Start a new process: UNIX command ls + //String cmd = "/home/karthikg/broad/gsa-unstable/public/c++/VectorPairHMM/checker "+args[0]; + //try + //{ + //System.out.println(cmd); + //java.lang.Process p = rt.exec(cmd); + //try + //{ + //p.waitFor(); + //java.io.InputStream is = p.getInputStream(); + //java.io.BufferedReader reader = new java.io.BufferedReader(new InputStreamReader(is)); + //// And print each line + //String s = null; + //while ((s = reader.readLine()) != null) { + //System.out.println(s); + //} + //is.close(); + //} + //catch(InterruptedException e) + //{ + //System.err.println(e); + //} + //} + //catch(IOException e) + //{ + //System.err.println(e); + //} + Sandbox t = new Sandbox(); + //t.doEverythingNative(args[0]); + t.parseSandboxFile(args[0]); + } +} diff --git a/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h new file mode 100644 index 000000000..7f78f0178 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox_JNIHaplotypeDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIHaplotypeDataHolderClass */ + +#ifndef _Included_Sandbox_JNIHaplotypeDataHolderClass +#define _Included_Sandbox_JNIHaplotypeDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h b/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h new file mode 100644 index 000000000..a9312ff3b --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/Sandbox_JNIReadDataHolderClass.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class Sandbox_JNIReadDataHolderClass */ + +#ifndef _Included_Sandbox_JNIReadDataHolderClass +#define _Included_Sandbox_JNIReadDataHolderClass +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc new file mode 100644 index 000000000..6d90d5070 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/avx_function_instantiations.cc @@ -0,0 +1,44 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "template.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_SSE + +#define SIMD_ENGINE avx +#define SIMD_ENGINE_AVX + +#include "define-float.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +#include "define-double.h" +#include "shift_template.c" +#include "pairhmm-template-kernel.cc" + +template double compute_full_prob_avxd(testcase* tc, double* nextlog); +template float compute_full_prob_avxs(testcase* tc, float* nextlog); + diff --git a/public/VectorPairHMM/src/main/c++/baseline.cc b/public/VectorPairHMM/src/main/c++/baseline.cc new file mode 100644 index 000000000..d6085e661 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/baseline.cc @@ -0,0 +1,167 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +using namespace std; + +template +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log) +{ + int r, c; + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + + Context ctx; + //#define USE_STACK_ALLOCATION 1 +#ifdef USE_STACK_ALLOCATION + NUMBER M[ROWS][COLS]; + NUMBER X[ROWS][COLS]; + NUMBER Y[ROWS][COLS]; + NUMBER p[ROWS][6]; +#else + //allocate on heap in way that simulates a 2D array. Having a 2D array instead of + //a straightforward array of pointers ensures that all data lies 'close' in memory, increasing + //the chance of being stored together in the cache. Also, prefetchers can learn memory access + //patterns for 2D arrays, not possible for array of pointers + //bool locally_allocated = false; + //NUMBER* common_buffer = 0; + NUMBER* common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //unsigned curr_size = sizeof(NUMBER)*(3*ROWS*COLS + ROWS*6); + //if(true) + //{ + //common_buffer = new NUMBER[3*ROWS*COLS + ROWS*6]; + //locally_allocated = true; + //} + //else + //common_buffer = (NUMBER*)(g_load_time_initializer.get_buffer()); + //pointers to within the allocated buffer + NUMBER** common_pointer_buffer = new NUMBER*[4*ROWS]; + NUMBER* ptr = common_buffer; + unsigned i = 0; + for(i=0;i<3*ROWS;++i, ptr+=COLS) + common_pointer_buffer[i] = ptr; + for(;i<4*ROWS;++i, ptr+=6) + common_pointer_buffer[i] = ptr; + + NUMBER** M = common_pointer_buffer; + NUMBER** X = M + ROWS; + NUMBER** Y = X + ROWS; + NUMBER** p = Y + ROWS; +#endif + + + p[0][MM] = ctx._(0.0); + p[0][GapM] = ctx._(0.0); + p[0][MX] = ctx._(0.0); + p[0][XX] = ctx._(0.0); + p[0][MY] = ctx._(0.0); + p[0][YY] = ctx._(0.0); + + for (r = 1; r < ROWS; r++) + { + int _i = tc->i[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + //p[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(p[r][MM], _i, _d); + p[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; + p[r][MX] = ctx.ph2pr[_i]; + p[r][XX] = ctx.ph2pr[_c]; + p[r][MY] = ctx.ph2pr[_d]; + p[r][YY] = ctx.ph2pr[_c]; + //p[r][MY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_d]; + //p[r][YY] = (r == ROWS - 1) ? ctx._(1.0) : ctx.ph2pr[_c]; + } + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); + } + + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } + + NUMBER result = ctx._(0.0); + + for (r = 1; r < ROWS; r++) + for (c = 1; c < COLS; c++) + { + fexcept_t flagp; + char _rs = tc->rs[r-1]; + char _hap = tc->hap[c-1]; + int _q = tc->q[r-1] & 127; + NUMBER distm = ctx.ph2pr[_q]; + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; + else + distm = distm/3; + + + //feclearexcept(FE_ALL_EXCEPT); + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //feclearexcept(FE_ALL_EXCEPT); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //feclearexcept(FE_ALL_EXCEPT); + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; + //STORE_FP_EXCEPTIONS(flagp, exceptions_array); + + //CONVERT_AND_PRINT(M[r][c]); + //CONVERT_AND_PRINT(X[r][c]); + //CONVERT_AND_PRINT(Y[r][c]); + + } + for (c = 0; c < COLS; c++) + { + result += M[ROWS-1][c] + X[ROWS-1][c]; + } + + if (before_last_log != NULL) + *before_last_log = result; + +#ifndef USE_STACK_ALLOCATION + delete common_pointer_buffer; + //if(locally_allocated) + delete common_buffer; +#endif + + return result; + //return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; +} + +template double compute_full_prob(testcase* tc, double* nextbuf); +template float compute_full_prob(testcase* tc, float* nextbuf); + diff --git a/public/VectorPairHMM/src/main/c++/define-double.h b/public/VectorPairHMM/src/main/c++/define-double.h new file mode 100644 index 000000000..2067d369c --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-double.h @@ -0,0 +1,205 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define PRECISION d +#define MAIN_TYPE double +#define MAIN_TYPE_SIZE 64 +#define UNION_TYPE mix_D +#define IF_128 IF_128d +#define IF_MAIN_TYPE IF_64 +#define SHIFT_CONST1 8 +#define SHIFT_CONST2 1 +#define SHIFT_CONST3 8 +#define _128_TYPE __m128d +#define SIMD_TYPE __m256d +#define _256_INT_TYPE __m256i +#define AVX_LENGTH 4 +#define HAP_TYPE __m128i +#define MASK_TYPE uint64_t +#define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFF +#define MASK_VEC MaskVec_D + +#define SET_VEC_ZERO(__vec) \ + __vec= _mm256_setzero_pd() + +#define VEC_OR(__v1, __v2) \ + _mm256_or_pd(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm256_add_pd(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm256_sub_pd(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm256_mul_pd(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm256_div_pd(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm256_blend_pd(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm256_blendv_pd(__v1, __v2, __maskV) + +#define VEC_CAST_256_128(__v1) \ + _mm256_castpd256_pd128 (__v1) + +#define VEC_EXTRACT_128(__v1, __im) \ + _mm256_extractf128_pd (__v1, __im) + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi64(__v1, __im) + +#define VEC_SET1_VAL128(__val) \ + _mm_set1_pd(__val) + +#define VEC_MOVE(__v1, __val) \ + _mm_move_sd(__v1, __val) + +#define VEC_CAST_128_256(__v1) \ + _mm256_castpd128_pd256(__v1) + +#define VEC_INSERT_VAL(__v1, __val, __pos) \ + _mm256_insertf128_pd(__v1, __val, __pos) + +#define VEC_CVT_128_256(__v1) \ + _mm256_cvtepi32_pd(__v1) + +#define VEC_SET1_VAL(__val) \ + _mm256_set1_pd(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm256_cvtepi32_pd(_mm_set1_epi32(__ch)) + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm256_cvtepi32_pd(_mm_load_si128((__m128i const *)__addr)) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm256_cmp_pd(__v1, __v2, _CMP_EQ_OQ) + +#define VEC_SET_LSE(__val) \ + _mm256_set_pd(zero, zero, zero, __val); + +#define SHIFT_HAP(__v1, __val) \ + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castpd128_pd256(__vsLow) ; \ +__vdst = _mm256_insertf128_pd(__vdst, __vsHigh, 1) ; + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi64(__vs, 1) + + +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + double* ptr1 = (double*) (&__v1) ; \ + double* ptr2 = (double*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Double Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ +} + +class BitMaskVec_double { + + MASK_VEC low_, high_ ; + SIMD_TYPE combined_ ; + + public: + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_double diff --git a/public/VectorPairHMM/src/main/c++/define-float.h b/public/VectorPairHMM/src/main/c++/define-float.h new file mode 100644 index 000000000..318f78280 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-float.h @@ -0,0 +1,206 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define PRECISION s + +#define MAIN_TYPE float +#define MAIN_TYPE_SIZE 32 +#define UNION_TYPE mix_F +#define IF_128 IF_128f +#define IF_MAIN_TYPE IF_32 +#define SHIFT_CONST1 12 +#define SHIFT_CONST2 3 +#define SHIFT_CONST3 4 +#define _128_TYPE __m128 +#define SIMD_TYPE __m256 +#define _256_INT_TYPE __m256i +#define AVX_LENGTH 8 +#define HAP_TYPE UNION_TYPE +#define MASK_TYPE uint32_t +#define MASK_ALL_ONES 0xFFFFFFFF +#define MASK_VEC MaskVec_F + +#define SET_VEC_ZERO(__vec) \ + __vec= _mm256_setzero_ps() + +#define VEC_OR(__v1, __v2) \ + _mm256_or_ps(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm256_add_ps(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm256_sub_ps(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm256_mul_ps(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm256_div_ps(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm256_blend_ps(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm256_blendv_ps(__v1, __v2, __maskV) + +#define VEC_CAST_256_128(__v1) \ + _mm256_castps256_ps128 (__v1) + +#define VEC_EXTRACT_128(__v1, __im) \ + _mm256_extractf128_ps (__v1, __im) + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi32(__v1, __im) + +#define VEC_SET1_VAL128(__val) \ + _mm_set1_ps(__val) + +#define VEC_MOVE(__v1, __val) \ + _mm_move_ss(__v1, __val) + +#define VEC_CAST_128_256(__v1) \ + _mm256_castps128_ps256(__v1) + +#define VEC_INSERT_VAL(__v1, __val, __pos) \ + _mm256_insertf128_ps(__v1, __val, __pos) + +#define VEC_CVT_128_256(__v1) \ + _mm256_cvtepi32_ps(__v1.i) + +#define VEC_SET1_VAL(__val) \ + _mm256_set1_ps(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm256_cvtepi32_ps(_mm256_set1_epi32(__ch)) + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm256_cvtepi32_ps(_mm256_loadu_si256((__m256i const *)__addr)) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm256_cmp_ps(__v1, __v2, _CMP_EQ_OQ) + +#define VEC_SET_LSE(__val) \ + _mm256_set_ps(zero, zero, zero, zero, zero, zero, zero, __val); + +#define SHIFT_HAP(__v1, __val) \ + _vector_shift_lastavxs(__v1, __val.f); + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm256_castps128_ps256(__vsLow) ; \ +__vdst = _mm256_insertf128_ps(__vdst, __vsHigh, 1) ; + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi32(__vs, 1) + +#define COMPARE_VECS(__v1, __v2, __first, __last) { \ + float* ptr1 = (float*) (&__v1) ; \ + float* ptr2 = (float*) (&__v2) ; \ + for (int ei=__first; ei <= __last; ++ei) { \ + if (ptr1[ei] != ptr2[ei]) { \ + std::cout << "Float Mismatch at " << ei << ": " \ + << ptr1[ei] << " vs. " << ptr2[ei] << std::endl ; \ + exit(0) ; \ + } \ + } \ +} + +class BitMaskVec_float { + + MASK_VEC low_, high_ ; + SIMD_TYPE combined_ ; + + public: + + inline MASK_TYPE& getLowEntry(int index) { + return low_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return high_.masks[index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + VEC_SSE_TO_AVX(low_.vecf, high_.vecf, combined_) ; + return combined_ ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(low_.vec) ; + VEC_SHIFT_LEFT_1BIT(high_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_float diff --git a/public/VectorPairHMM/src/main/c++/define-sse-double.h b/public/VectorPairHMM/src/main/c++/define-sse-double.h new file mode 100644 index 000000000..2d271a854 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-sse-double.h @@ -0,0 +1,173 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_INSERT_UNIT(__v1,__ins,__im) +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define SSE +#define PRECISION d + +#define MAIN_TYPE double +#define MAIN_TYPE_SIZE 64 +#define UNION_TYPE mix_D128 +#define IF_128 IF_128d +#define IF_MAIN_TYPE IF_64 +#define SHIFT_CONST1 1 +#define SHIFT_CONST2 8 +#define SHIFT_CONST3 0 +#define _128_TYPE __m128d +#define SIMD_TYPE __m128d +#define _256_INT_TYPE __m128i +#define AVX_LENGTH 2 +#define HAP_TYPE __m128i +#define MASK_TYPE uint64_t +#define MASK_ALL_ONES 0xFFFFFFFFFFFFFFFFL +#define MASK_VEC MaskVec_D + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi64(__v1, __im) + +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi64(__v1,__ins,__im) + +#define VEC_OR(__v1, __v2) \ + _mm_or_pd(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm_add_pd(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm_sub_pd(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm_mul_pd(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm_div_pd(__v1, __v2) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm_cmpeq_pd(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm_blend_pd(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm_blendv_pd(__v1, __v2, __maskV) + +#define SHIFT_HAP(__v1, __val) \ + __v1 = _mm_insert_epi32(_mm_slli_si128(__v1, 4), __val.i, 0) + +#define VEC_CVT_128_256(__v1) \ + _mm_cvtepi32_pd(__v1) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_pd(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm_cvtepi32_pd(_mm_set1_epi32(__ch)) + +#define VEC_SET_LSE(__val) \ + _mm_set_pd(zero, __val); + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm_cvtepi32_pd(_mm_loadu_si128((__m128i const *)__addr)) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm_castsi128_pd(_mm_set_epi64(__vsHigh, __vsLow)) + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi64(__vs, 1) + + +class BitMaskVec_sse_double { + + MASK_VEC combined_ ; + public: + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_sse_double + diff --git a/public/VectorPairHMM/src/main/c++/define-sse-float.h b/public/VectorPairHMM/src/main/c++/define-sse-float.h new file mode 100644 index 000000000..20af947dd --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/define-sse-float.h @@ -0,0 +1,173 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifdef PRECISION +#undef PRECISION +#undef MAIN_TYPE +#undef MAIN_TYPE_SIZE +#undef UNION_TYPE +#undef IF_128 +#undef IF_MAIN_TYPE +#undef SHIFT_CONST1 +#undef SHIFT_CONST2 +#undef SHIFT_CONST3 +#undef _128_TYPE +#undef SIMD_TYPE +#undef AVX_LENGTH +#undef HAP_TYPE +#undef MASK_TYPE +#undef MASK_ALL_ONES + +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_INSERT_UNIT(__v1,__ins,__im) +#undef SET_VEC_ZERO(__vec) +#undef VEC_OR(__v1, __v2) +#undef VEC_ADD(__v1, __v2) +#undef VEC_SUB(__v1, __v2) +#undef VEC_MUL(__v1, __v2) +#undef VEC_DIV(__v1, __v2) +#undef VEC_BLEND(__v1, __v2, __mask) +#undef VEC_BLENDV(__v1, __v2, __maskV) +#undef VEC_CAST_256_128(__v1) +#undef VEC_EXTRACT_128(__v1, __im) +#undef VEC_EXTRACT_UNIT(__v1, __im) +#undef VEC_SET1_VAL128(__val) +#undef VEC_MOVE(__v1, __val) +#undef VEC_CAST_128_256(__v1) +#undef VEC_INSERT_VAL(__v1, __val, __pos) +#undef VEC_CVT_128_256(__v1) +#undef VEC_SET1_VAL(__val) +#undef VEC_POPCVT_CHAR(__ch) +#undef VEC_LDPOPCVT_CHAR(__addr) +#undef VEC_CMP_EQ(__v1, __v2) +#undef VEC_SET_LSE(__val) +#undef SHIFT_HAP(__v1, __val) +#undef MASK_VEC +#undef VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) +#undef VEC_SHIFT_LEFT_1BIT(__vs) +#undef MASK_ALL_ONES +#undef COMPARE_VECS(__v1, __v2) +#undef _256_INT_TYPE +#undef BITMASK_VEC +#endif + +#define SSE +#define PRECISION s + +#define MAIN_TYPE float +#define MAIN_TYPE_SIZE 32 +#define UNION_TYPE mix_F128 +#define IF_128 IF_128f +#define IF_MAIN_TYPE IF_32 +#define SHIFT_CONST1 3 +#define SHIFT_CONST2 4 +#define SHIFT_CONST3 0 +#define _128_TYPE __m128 +#define SIMD_TYPE __m128 +#define _256_INT_TYPE __m128i +#define AVX_LENGTH 4 +//#define MAVX_COUNT (MROWS+3)/AVX_LENGTH +#define HAP_TYPE UNION_TYPE +#define MASK_TYPE uint32_t +#define MASK_ALL_ONES 0xFFFFFFFF +#define MASK_VEC MaskVec_F + +#define VEC_EXTRACT_UNIT(__v1, __im) \ + _mm_extract_epi32(__v1, __im) + +#define VEC_INSERT_UNIT(__v1,__ins,__im) \ + _mm_insert_epi32(__v1,__ins,__im) + +#define VEC_OR(__v1, __v2) \ + _mm_or_ps(__v1, __v2) + +#define VEC_ADD(__v1, __v2) \ + _mm_add_ps(__v1, __v2) + +#define VEC_SUB(__v1, __v2) \ + _mm_sub_ps(__v1, __v2) + +#define VEC_MUL(__v1, __v2) \ + _mm_mul_ps(__v1, __v2) + +#define VEC_DIV(__v1, __v2) \ + _mm_div_ps(__v1, __v2) + +#define VEC_CMP_EQ(__v1, __v2) \ + _mm_cmpeq_ps(__v1, __v2) + +#define VEC_BLEND(__v1, __v2, __mask) \ + _mm_blend_ps(__v1, __v2, __mask) + +#define VEC_BLENDV(__v1, __v2, __maskV) \ + _mm_blendv_ps(__v1, __v2, __maskV) + +#define SHIFT_HAP(__v1, __val) \ + _vector_shift_lastsses(__v1, __val.f) + +#define VEC_CVT_128_256(__v1) \ + _mm_cvtepi32_ps(__v1.i) + +#define VEC_SET1_VAL(__val) \ + _mm_set1_ps(__val) + +#define VEC_POPCVT_CHAR(__ch) \ + _mm_cvtepi32_ps(_mm_set1_epi32(__ch)) + +#define VEC_SET_LSE(__val) \ + _mm_set_ps(zero, zero, zero, __val); + +#define VEC_LDPOPCVT_CHAR(__addr) \ + _mm_cvtepi32_ps(_mm_loadu_si128((__m128i const *)__addr)) + +#define VEC_SSE_TO_AVX(__vsLow, __vsHigh, __vdst) \ + __vdst = _mm_cvtpi32x2_ps(__vsLow, __vsHigh) + +#define VEC_SHIFT_LEFT_1BIT(__vs) \ + __vs = _mm_slli_epi32(__vs, 1) + +class BitMaskVec_sse_float { + + MASK_VEC combined_ ; + + public: + inline MASK_TYPE& getLowEntry(int index) { + return combined_.masks[index] ; + } + inline MASK_TYPE& getHighEntry(int index) { + return combined_.masks[AVX_LENGTH/2+index] ; + } + + inline const SIMD_TYPE& getCombinedMask() { + return combined_.vecf ; + } + + inline void shift_left_1bit() { + VEC_SHIFT_LEFT_1BIT(combined_.vec) ; + } + +} ; + +#define BITMASK_VEC BitMaskVec_sse_float diff --git a/public/VectorPairHMM/src/main/c++/headers.h b/public/VectorPairHMM/src/main/c++/headers.h new file mode 100644 index 000000000..4a0d89b57 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/headers.h @@ -0,0 +1,71 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef COMMON_HEADERS_H +#define COMMON_HEADERS_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern uint64_t exceptions_array[128]; +extern FILE* g_debug_fptr; +#define STORE_FP_EXCEPTIONS(flagp, exceptions_array) \ + fegetexceptflag(&flagp, FE_ALL_EXCEPT | __FE_DENORM); \ + exceptions_array[FE_INVALID] += ((flagp & FE_INVALID)); \ + exceptions_array[__FE_DENORM] += ((flagp & __FE_DENORM) >> 1); \ + exceptions_array[FE_DIVBYZERO] += ((flagp & FE_DIVBYZERO) >> 2); \ + exceptions_array[FE_OVERFLOW] += ((flagp & FE_OVERFLOW) >> 3); \ + exceptions_array[FE_UNDERFLOW] += ((flagp & FE_UNDERFLOW) >> 4); \ + feclearexcept(FE_ALL_EXCEPT | __FE_DENORM); + +#define CONVERT_AND_PRINT(X) \ + g_converter.f = (X); \ + fwrite(&(g_converter.i),4,1,g_debug_fptr); \ + +#endif diff --git a/public/VectorPairHMM/src/main/c++/jni_common.h b/public/VectorPairHMM/src/main/c++/jni_common.h new file mode 100644 index 000000000..23c323246 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/jni_common.h @@ -0,0 +1,58 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef JNI_COMMON_H +#define JNI_COMMON_H + +#include +/*#define ENABLE_ASSERTIONS 1*/ +#define DO_PROFILING 1 +/*#define DEBUG 1*/ +/*#define DEBUG0_1 1*/ +/*#define DEBUG3 1*/ +/*#define DUMP_TO_SANDBOX 1*/ + + +#define DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY 1 + +#ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY +//Gets direct access to Java arrays +#define GET_BYTE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetPrimitiveArrayCritical +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleasePrimitiveArrayCritical + +#else +//Likely makes copy of Java arrays to JNI C++ space +#define GET_BYTE_ARRAY_ELEMENTS env->GetByteArrayElements +#define RELEASE_BYTE_ARRAY_ELEMENTS env->ReleaseByteArrayElements +#define JNI_RO_RELEASE_MODE JNI_ABORT +#define GET_DOUBLE_ARRAY_ELEMENTS env->GetDoubleArrayElements +#define RELEASE_DOUBLE_ARRAY_ELEMENTS env->ReleaseDoubleArrayElements + +#endif //ifdef DIRECT_ACCESS_TO_JAVA_HEAP_MEMORY + +#endif //ifndef JNI_COMMON_H diff --git a/public/VectorPairHMM/src/main/c++/jnidebug.h b/public/VectorPairHMM/src/main/c++/jnidebug.h new file mode 100644 index 000000000..7fcab2a51 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/jnidebug.h @@ -0,0 +1,191 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef JNI_DEBUG_H +#define JNI_DEBUG_H + +template +class DataHolder +{ +#define INIT_MATRIX(X) \ + X = new NUMBER*[m_paddedMaxReadLength]; \ + for(int i=0;i ctx; + for (int r = 1; r <= length;r++) //in original code, r < ROWS (where ROWS = paddedReadLength) + { + int _i = insertionGOP[r-1]; //insertionGOP + int _d = deletionGOP[r-1]; //deletionGOP + int _c = overallGCP[r-1]; //overallGCP + m_transition[r][MM] = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; //lines 161-162 + m_transition[r][GapM] = ctx._(1.0) - ctx.ph2pr[_c]; //line 163 + m_transition[r][MX] = ctx.ph2pr[_i]; //164 + m_transition[r][XX] = ctx.ph2pr[_c]; //165 + m_transition[r][MY] = ctx.ph2pr[_d];//last row seems different, compared to line 166 + m_transition[r][YY] = ctx.ph2pr[_c];//same as above for line 167 + //m_transition[r][MY] = (r == length) ? ctx._(1.0) : ctx.ph2pr[_d];//last row seems different, compared to line 166 + //m_transition[r][YY] = (r == length) ? ctx._(1.0) : ctx.ph2pr[_c];//same as above for line 167 +#ifdef DEBUG3 + for(int j=0;j<6;++j) + debug_dump("transitions_jni.txt", to_string(m_transition[r][j]),true); +#endif + } + ++g_num_prob_init; + } + bool m_is_initialized; + int m_readMaxLength; + int m_haplotypeMaxLength; + int m_paddedMaxReadLength; + int m_paddedMaxHaplotypeLength; + NUMBER** m_matchMatrix; + NUMBER** m_insertionMatrix; + NUMBER** m_deletionMatrix; + NUMBER** m_prior; + NUMBER (*m_transition)[6]; +}; +extern DataHolder g_double_dataholder; + +template +NUMBER compute_full_prob(testcase *tc, NUMBER** M, NUMBER** X, NUMBER** Y, NUMBER (*p)[6], + bool do_initialization, jint hapStartIndex, NUMBER *before_last_log = NULL) +{ + int r, c; + int ROWS = tc->rslen + 1; //ROWS = paddedReadLength + int COLS = tc->haplen + 1; //COLS = paddedHaplotypeLength + + Context ctx; + //////NOTES + ////ctx.ph2pr[quality]; //This quantity is QualityUtils.qualToErrorProb(quality) + ////1-ctx.ph2pr[quality]; //This corresponds to QualityUtils.qualToProb(quality); + + //Initialization + if(do_initialization) + { + for (c = 0; c < COLS; c++) + { + M[0][c] = ctx._(0.0); + X[0][c] = ctx._(0.0); + Y[0][c] = ctx.INITIAL_CONSTANT / (tc->haplen); //code from 87-90 in LoglessPairHMM + } + + for (r = 1; r < ROWS; r++) + { + M[r][0] = ctx._(0.0); + //deletionMatrix row 0 in above nest is initialized in the Java code + //However, insertionMatrix column 0 is not initialized in Java code, could it be that + //values are re-used from a previous iteration? + //Why even do this, X[0][0] = 0 from above loop nest, X[idx][0] = 0 from this computation + X[r][0] = X[r-1][0] * p[r][XX]; + Y[r][0] = ctx._(0.0); + } + } + + for (r = 1; r < ROWS; r++) + for (c = hapStartIndex+1; c < COLS; c++) + { + //The following lines correspond to initializePriors() + char _rs = tc->rs[r-1]; //line 137 + char _hap = tc->hap[c-1]; //line 140 + //int _q = tc->q[r-1] & 127; //line 138 - q is the quality (qual), should be byte hence int ANDed with 0xFF + int _q = tc->q[r-1]; //line 138 - q is the quality (qual), should be byte hence int ANDed with 0xFF + NUMBER distm = ctx.ph2pr[_q]; //This quantity is QualityUtils.qualToErrorProb(_q) + //The assumption here is that doNotUseTristateCorrection is true + //TOASK + if (_rs == _hap || _rs == 'N' || _hap == 'N') + distm = ctx._(1.0) - distm; //This is the quantity QualityUtils.qualToProb(qual) + else + distm = distm/3; +#ifdef DEBUG3 + debug_dump("priors_jni.txt",to_string(distm),true); +#endif + + //Computation inside updateCell + M[r][c] = distm * (M[r-1][c-1] * p[r][MM] + X[r-1][c-1] * p[r][GapM] + Y[r-1][c-1] * p[r][GapM]); + X[r][c] = M[r-1][c] * p[r][MX] + X[r-1][c] * p[r][XX]; + Y[r][c] = M[r][c-1] * p[r][MY] + Y[r][c-1] * p[r][YY]; +#ifdef DEBUG3 + debug_dump("matrices_jni.txt",to_string(M[r][c]),true); + debug_dump("matrices_jni.txt",to_string(X[r][c]),true); + debug_dump("matrices_jni.txt",to_string(Y[r][c]),true); +#endif + } + + NUMBER result = ctx._(0.0); + for (c = 0; c < COLS; c++) + result += M[ROWS-1][c] + X[ROWS-1][c]; + + if (before_last_log != NULL) + *before_last_log = result; + +#ifdef DEBUG + debug_dump("return_values_jni.txt",to_string(ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT),true); +#endif + return ctx.LOG10(result) - ctx.LOG10_INITIAL_CONSTANT; +} + +#endif diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc new file mode 100644 index 000000000..8a3f8b5bc --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.cc @@ -0,0 +1,176 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "jni_common.h" +#include "org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +#include "jnidebug.h" +DataHolder g_double_dataholder; + +using namespace std; + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitialize +(JNIEnv* env, jobject thisObject, + jint readMaxLength, jint haplotypeMaxLength) +{ + static int g_num_init_calls = 0; +#ifdef DEBUG3 + cout << "Entered alloc initialized .. readMaxLength "<GetArrayLength(insertionGOP); +#ifdef DEBUG3 + cout << "Entered initializeProbabilities .. length "<GetByteArrayElements(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (env)->GetByteArrayElements(deletionGOP, &is_copy); + jbyte* overallGCPArray = (env)->GetByteArrayElements(overallGCP, &is_copy); +#ifdef DEBUG + if(insertionGOPArray == 0) + cerr << "insertionGOP array not initialized in JNI\n"; + ////assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + if(deletionGOPArray == 0) + cerr << "deletionGOP array not initialized in JNI\n"; + ////assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); +#endif + + g_double_dataholder.initializeProbabilities(length, insertionGOPArray, deletionGOPArray, overallGCPArray); + + env->ReleaseByteArrayElements(overallGCP, overallGCPArray, JNI_ABORT); + env->ReleaseByteArrayElements(deletionGOP, deletionGOPArray, JNI_ABORT); + env->ReleaseByteArrayElements(insertionGOP, insertionGOPArray, JNI_ABORT); +} + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializePriorsAndUpdateCells( + JNIEnv* env, jobject thisObject, + jboolean doInitialization, jint paddedReadLength, jint paddedHaplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jint hapStartIndex + ) +{ +#ifdef DEBUG3 + cout << "Entered mainCompute .. doInitialization "<<(doInitialization == JNI_TRUE)<<" hapStartIndex "<GetByteArrayElements(readBases, &is_copy); + jbyte* haplotypeBasesArray = (env)->GetByteArrayElements(haplotypeBases, &is_copy); + jbyte* readQualsArray = (env)->GetByteArrayElements(readQuals, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); +#endif + testcase tc; + + tc.rslen = paddedReadLength-1; + tc.haplen = paddedHaplotypeLength-1; + + tc.rs = (char*)readBasesArray; + tc.hap = (char*)haplotypeBasesArray; + tc.q = (char*)readQualsArray; //TOASK - q is now char* + + compute_full_prob(&tc, g_double_dataholder.m_matchMatrix, g_double_dataholder.m_insertionMatrix, + g_double_dataholder.m_deletionMatrix, g_double_dataholder.m_transition, + doInitialization == JNI_TRUE, hapStartIndex, NULL); + + env->ReleaseByteArrayElements(readBases, readBasesArray, JNI_ABORT); + env->ReleaseByteArrayElements(haplotypeBases, haplotypeBasesArray, JNI_ABORT); + env->ReleaseByteArrayElements(readQuals, readQualsArray, JNI_ABORT); + return 0.0; +} + +JNIEXPORT jdouble JNICALL +Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10( + JNIEnv* env, jobject thisObject, + jint readLength, jint haplotypeLength, + jbyteArray readBases, jbyteArray haplotypeBases, jbyteArray readQuals, + jbyteArray insertionGOP, jbyteArray deletionGOP, jbyteArray overallGCP, + jint hapStartIndex + ) +{ + jboolean is_copy = JNI_FALSE; + jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBases, &is_copy); + jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); + jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); + jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DEBUG + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "OverallGCP array not initialized in JNI"); + //assert(readLength < MROWS); +#endif + testcase tc; + tc.rslen = readLength; + tc.haplen = haplotypeLength; + tc.rs = (char*)readBasesArray; + tc.hap = (char*)haplotypeBasesArray; + for(unsigned i=0;i +/* Header for class org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM */ + +#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM +#define _Included_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM +#ifdef __cplusplus +extern "C" { +#endif +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_TRISTATE_CORRECTION +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_TRISTATE_CORRECTION 3.0 +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToMatch +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToMatch 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_indelToMatch +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_indelToMatch 1L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToInsertion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToInsertion 2L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_insertionToInsertion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_insertionToInsertion 3L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToDeletion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_matchToDeletion 4L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_deletionToDeletion +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_deletionToDeletion 5L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_verify +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_verify 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug0_1 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug0_1 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug1 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug1 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug2 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug2 0L +#undef org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug3 +#define org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_debug3 0L +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitialize + * Signature: (II)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitialize + (JNIEnv *, jobject, jint, jint); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitializeProbabilities + * Signature: ([[D[B[B[B)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializeProbabilities + (JNIEnv *, jclass, jobjectArray, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniInitializePriorsAndUpdateCells + * Signature: (ZII[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniInitializePriorsAndUpdateCells + (JNIEnv *, jobject, jboolean, jint, jint, jbyteArray, jbyteArray, jbyteArray, jint); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM + * Method: jniSubComputeReadLikelihoodGivenHaplotypeLog10 + * Signature: (II[B[B[B[B[B[BI)D + */ +JNIEXPORT jdouble JNICALL Java_org_broadinstitute_sting_utils_pairhmm_DebugJNILoglessPairHMM_jniSubComputeReadLikelihoodGivenHaplotypeLog10 + (JNIEnv *, jobject, jint, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc new file mode 100644 index 000000000..0b54c8a81 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.cc @@ -0,0 +1,382 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "jni_common.h" +#include "org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h" +#include "template.h" +#include "utils.h" +#include "LoadTimeInitializer.h" + +using namespace std; + +JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType + (JNIEnv* env, jobject thisObject) +{ + return (jlong)get_machine_capabilities(); +} + +//Should be called only once for the whole Java process - initializes field ids for the classes JNIReadDataHolderClass +//and JNIHaplotypeDataHolderClass +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask + (JNIEnv* env, jobject thisObject, jclass readDataHolderClass, jclass haplotypeDataHolderClass, jlong mask) +{ + assert(readDataHolderClass); + assert(haplotypeDataHolderClass); + jfieldID fid; + fid = env->GetFieldID(readDataHolderClass, "readBases", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for readBases"); + g_load_time_initializer.m_readBasesFID = fid; + fid = env->GetFieldID(readDataHolderClass, "readQuals", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for readQuals"); + g_load_time_initializer.m_readQualsFID = fid; + fid = env->GetFieldID(readDataHolderClass, "insertionGOP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for insertionGOP"); + g_load_time_initializer.m_insertionGOPFID = fid; + fid = env->GetFieldID(readDataHolderClass, "deletionGOP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for deletionGOP"); + g_load_time_initializer.m_deletionGOPFID = fid; + fid = env->GetFieldID(readDataHolderClass, "overallGCP", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for overallGCP"); + g_load_time_initializer.m_overallGCPFID = fid; + + fid = env->GetFieldID(haplotypeDataHolderClass, "haplotypeBases", "[B"); + assert(fid && "JNI pairHMM: Could not get FID for haplotypeBases"); + g_load_time_initializer.m_haplotypeBasesFID = fid; + if(mask != ENABLE_ALL_HARDWARE_FEATURES) + { + cout << "Using user supplied hardware mask to re-initialize function pointers for PairHMM\n"; + initialize_function_pointers((uint64_t)mask); + cout.flush(); + } +} + +//Since the list of haplotypes against which the reads are evaluated in PairHMM is the same for a region, +//transfer the list only once +vector > g_haplotypeBasesArrayVector; +vector g_haplotypeBasesLengths; +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes + (JNIEnv * env, jobject thisObject, jint numHaplotypes, jobjectArray haplotypeDataArray) +{ + jboolean is_copy = JNI_FALSE; + //To ensure, GET_BYTE_ARRAY_ELEMENTS is invoked only once for each haplotype, store bytearrays in a vector + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + haplotypeBasesArrayVector.clear(); + g_haplotypeBasesLengths.clear(); + haplotypeBasesArrayVector.resize(numHaplotypes); + g_haplotypeBasesLengths.resize(numHaplotypes); + jsize haplotypeBasesLength = 0; + for(unsigned j=0;jGetObjectArrayElement(haplotypeDataArray, j); + jbyteArray haplotypeBases = (jbyteArray)env->GetObjectField(haplotypeObject, g_load_time_initializer.m_haplotypeBasesFID); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBases && ("haplotypeBases is NULL at index : "+to_string(j)+"\n").c_str()); +#endif + //Need a global reference as this will be accessed across multiple JNI calls to JNIComputeLikelihoods() + jbyteArray haplotypeBasesGlobalRef = (jbyteArray)env->NewGlobalRef(haplotypeBases); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBasesGlobalRef && ("Could not get global ref to haplotypeBases at index : "+to_string(j)+"\n").c_str()); +#endif + env->DeleteLocalRef(haplotypeBases); //free the local reference + jbyte* haplotypeBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(haplotypeBasesGlobalRef, &is_copy); + haplotypeBasesLength = env->GetArrayLength(haplotypeBasesGlobalRef); +#ifdef ENABLE_ASSERTIONS + assert(haplotypeBasesArray && "haplotypeBasesArray not initialized in JNI"); + //assert(haplotypeBasesLength < MCOLS); +#endif +#ifdef DEBUG0_1 + cout << "JNI haplotype length "< > >& readBasesArrayVector, vector& tc_array) +{ + jboolean is_copy = JNI_FALSE; + //haplotype vector from earlier store - note the reference to vector, not copying + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + unsigned tc_idx = 0; + for(unsigned i=0;iGetObjectArrayElement(readDataArray, i); + jbyteArray readBases = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_readBasesFID); + jbyteArray insertionGOP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_insertionGOPFID); + jbyteArray deletionGOP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_deletionGOPFID); + jbyteArray overallGCP = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_overallGCPFID); + jbyteArray readQuals = (jbyteArray)env->GetObjectField(readObject, g_load_time_initializer.m_readQualsFID); + +#ifdef ENABLE_ASSERTIONS + assert(readBases && ("readBases is NULL at index : "+to_string(i)+"\n").c_str()); + assert(insertionGOP && ("insertionGOP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(deletionGOP && ("deletionGOP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(overallGCP && ("overallGCP is NULL at index : "+to_string(i)+"\n").c_str()); + assert(readQuals && ("readQuals is NULL at index : "+to_string(i)+"\n").c_str()); +#endif + jsize readLength = env->GetArrayLength(readBases); + + jbyte* readBasesArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readBases, &is_copy); //order of GET-RELEASE is important + jbyte* readQualsArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(readQuals, &is_copy); + jbyte* insertionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(insertionGOP, &is_copy); + jbyte* deletionGOPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(deletionGOP, &is_copy); + jbyte* overallGCPArray = (jbyte*)GET_BYTE_ARRAY_ELEMENTS(overallGCP, &is_copy); +#ifdef DO_PROFILING + g_load_time_initializer.m_bytes_copied += (is_copy ? readLength*5 : 0); + g_load_time_initializer.update_stat(READ_LENGTH_IDX, readLength); +#endif +#ifdef ENABLE_ASSERTIONS + assert(readBasesArray && "readBasesArray not initialized in JNI"); + assert(readQualsArray && "readQualsArray not initialized in JNI"); + assert(insertionGOPArray && "insertionGOP array not initialized in JNI"); + assert(deletionGOPArray && "deletionGOP array not initialized in JNI"); + assert(overallGCPArray && "overallGCP array not initialized in JNI"); + //assert(readLength < MROWS); + assert(readLength == env->GetArrayLength(readQuals)); + assert(readLength == env->GetArrayLength(insertionGOP)); + assert(readLength == env->GetArrayLength(deletionGOP)); + assert(readLength == env->GetArrayLength(overallGCP)); +#endif +#ifdef DEBUG0_1 + cout << "JNI read length "<& tc_array, unsigned numTestCases, double* likelihoodDoubleArray, + unsigned maxNumThreadsToUse) +{ +#ifdef DO_REPEAT_PROFILING + for(unsigned i=0;i<10;++i) +#endif + { +#pragma omp parallel for schedule (dynamic,10000) num_threads(maxNumThreadsToUse) + for(unsigned tc_idx=0;tc_idx > >& readBasesArrayVector) +{ + //Release read arrays first + for(int i=readBasesArrayVector.size()-1;i>=0;--i)//note the order - reverse of GET + { + for(int j=readBasesArrayVector[i].size()-1;j>=0;--j) + RELEASE_BYTE_ARRAY_ELEMENTS(readBasesArrayVector[i][j].first, readBasesArrayVector[i][j].second, JNI_RO_RELEASE_MODE); + readBasesArrayVector[i].clear(); + } + readBasesArrayVector.clear(); +} + + +#ifdef DO_WARMUP +uint64_t g_sum = 0; +#endif +//JNI function to invoke compute_full_prob_avx +//readDataArray - array of JNIReadDataHolderClass objects which contain the readBases, readQuals etc +//haplotypeDataArray - array of JNIHaplotypeDataHolderClass objects which contain the haplotypeBases +//likelihoodArray - array of doubles to return results back to Java. Memory allocated by Java prior to JNI call +//maxNumThreadsToUse - Max number of threads that OpenMP can use for the HMM computation +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods + (JNIEnv* env, jobject thisObject, jint numReads, jint numHaplotypes, + jobjectArray readDataArray, jobjectArray haplotypeDataArray, jdoubleArray likelihoodArray, jint maxNumThreadsToUse) +{ +#ifdef DEBUG0_1 + cout << "JNI numReads "< tc_array; + tc_array.clear(); + tc_array.resize(numTestCases); + //Store read arrays for release later + vector > > readBasesArrayVector; + readBasesArrayVector.clear(); + readBasesArrayVector.resize(numReads); +#ifdef DUMP_TO_SANDBOX + g_load_time_initializer.open_sandbox(); +#endif +#ifdef DO_PROFILING + get_time(&start_time); +#endif + //Copy byte array references from Java memory into vector of testcase structs + Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeTestcasesVector(env, + numReads, numHaplotypes, readDataArray, readBasesArrayVector, tc_array); +#ifdef DO_PROFILING + g_load_time_initializer.m_data_transfer_time += diff_time(start_time); +#endif + + //Get double array where results are stored (to pass back to java) + jdouble* likelihoodDoubleArray = (jdouble*)GET_DOUBLE_ARRAY_ELEMENTS(likelihoodArray, &is_copy); +#ifdef ENABLE_ASSERTIONS + assert(likelihoodDoubleArray && "likelihoodArray is NULL"); + assert(env->GetArrayLength(likelihoodArray) == numTestCases); +#endif +#ifdef DO_WARMUP //ignore - only for crazy profiling + vector >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + for(unsigned i=0;iGetArrayLength(haplotypeBasesArrayVector[i].first); + for(unsigned j=0;jGetArrayLength(readBasesArrayVector[i][j].first); + for(unsigned k=0;k >& haplotypeBasesArrayVector = g_haplotypeBasesArrayVector; + //Now release haplotype arrays + for(int j=haplotypeBasesArrayVector.size()-1;j>=0;--j) //note the order - reverse of GET + { + RELEASE_BYTE_ARRAY_ELEMENTS(haplotypeBasesArrayVector[j].first, haplotypeBasesArrayVector[j].second, JNI_RO_RELEASE_MODE); + env->DeleteGlobalRef(haplotypeBasesArrayVector[j].first); //free the global reference + } + haplotypeBasesArrayVector.clear(); + g_haplotypeBasesLengths.clear(); +} + + +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose + (JNIEnv* env, jobject thisObject) +{ +#ifdef DO_PROFILING + g_load_time_initializer.print_profiling(); +#endif +#ifdef DEBUG + g_load_time_initializer.debug_close(); +#endif +} + diff --git a/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h new file mode 100644 index 000000000..d820b4b26 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM.h @@ -0,0 +1,104 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM */ + +#ifndef _Included_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM +#define _Included_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM +#ifdef __cplusplus +extern "C" { +#endif +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_TRISTATE_CORRECTION +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_TRISTATE_CORRECTION 3.0 +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToMatch +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToMatch 0L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_indelToMatch +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_indelToMatch 1L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToInsertion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToInsertion 2L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_insertionToInsertion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_insertionToInsertion 3L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToDeletion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_matchToDeletion 4L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_deletionToDeletion +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_deletionToDeletion 5L +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_sse42Mask +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_sse42Mask 1LL +#undef org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_avxMask +#define org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_avxMask 2LL +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniGetMachineType + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniGetMachineType + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniClose + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniClose + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniInitializeClassFieldsAndMachineMask + * Signature: (Ljava/lang/Class;Ljava/lang/Class;J)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeClassFieldsAndMachineMask + (JNIEnv *, jobject, jclass, jclass, jlong); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniInitializeHaplotypes + * Signature: (I[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIHaplotypeDataHolderClass;)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniInitializeHaplotypes + (JNIEnv *, jobject, jint, jobjectArray); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniFinalizeRegion + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniFinalizeRegion + (JNIEnv *, jobject); + +/* + * Class: org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM + * Method: jniComputeLikelihoods + * Signature: (II[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIReadDataHolderClass;[Lorg/broadinstitute/sting/utils/pairhmm/VectorLoglessPairHMM/JNIHaplotypeDataHolderClass;[DI)V + */ +JNIEXPORT void JNICALL Java_org_broadinstitute_sting_utils_pairhmm_VectorLoglessPairHMM_jniComputeLikelihoods + (JNIEnv *, jobject, jint, jint, jobjectArray, jobjectArray, jdoubleArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc new file mode 100644 index 000000000..7ff219b88 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/pairhmm-1-base.cc @@ -0,0 +1,70 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +//#define DEBUG 1 +//#define DEBUG0_1 1 +//#define DEBUG3 1 +#include "headers.h" +#include "utils.h" +#include "LoadTimeInitializer.h" +using namespace std; + +int main(int argc, char** argv) +{ +#define BATCH_SIZE 10000 + if(argc < 2) + { + cerr << "Needs path to input file as argument\n"; + exit(0); + } + bool use_old_read_testcase = false; + if(argc >= 3 && string(argv[2]) == "1") + use_old_read_testcase = true; + unsigned chunk_size = 10000; + bool do_check = true; + uint64_t mask = ~(0ull); + for(int i=3;i +#include +#include + + +void CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]) { + + const int maskBitCnt = MAIN_TYPE_SIZE ; + + for (int vi=0; vi < numMaskVecs; ++vi) { + for (int rs=0; rs < NUM_DISTINCT_CHARS; ++rs) { + maskArr[vi][rs] = 0 ; + } + maskArr[vi][AMBIG_CHAR] = MASK_ALL_ONES ; + } + + for (int col=1; col < COLS; ++col) { + int mIndex = (col-1) / maskBitCnt ; + int mOffset = (col-1) % maskBitCnt ; + MASK_TYPE bitMask = ((MASK_TYPE)0x1) << (maskBitCnt-1-mOffset) ; + + char hapChar = ConvertChar::get(tc.hap[col-1]); + + if (hapChar == AMBIG_CHAR) { + for (int ci=0; ci < NUM_DISTINCT_CHARS; ++ci) + maskArr[mIndex][ci] |= bitMask ; + } + + maskArr[mIndex][hapChar] |= bitMask ; + // bit corresponding to col 1 will be the MSB of the mask 0 + // bit corresponding to col 2 will be the MSB-1 of the mask 0 + // ... + // bit corresponding to col 32 will be the LSB of the mask 0 + // bit corresponding to col 33 will be the MSB of the mask 1 + // ... + } + +} + +void CONCAT(CONCAT(init_masks_for_row_,SIMD_ENGINE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess) { + + for (int ri=0; ri < numRowsToProcess; ++ri) { + rsArr[ri] = ConvertChar::get(tc.rs[ri+beginRowIndex-1]) ; + } + + for (int ei=0; ei < AVX_LENGTH; ++ei) { + lastMaskShiftOut[ei] = 0 ; + } +} + +#define SET_MASK_WORD(__dstMask, __srcMask, __lastShiftOut, __shiftBy, __maskBitCnt){ \ + MASK_TYPE __bitMask = (((MASK_TYPE)0x1) << __shiftBy) - 1 ; \ + MASK_TYPE __nextShiftOut = (__srcMask & __bitMask) << (__maskBitCnt - __shiftBy) ; \ + __dstMask = (__srcMask >> __shiftBy) | __lastShiftOut ; \ + __lastShiftOut = __nextShiftOut ; \ +} + + +void CONCAT(CONCAT(update_masks_for_cols_,SIMD_ENGINE), PRECISION)(int maskIndex, BITMASK_VEC& bitMaskVec, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, int maskBitCnt) { + + for (int ei=0; ei < AVX_LENGTH/2; ++ei) { + SET_MASK_WORD(bitMaskVec.getLowEntry(ei), maskArr[maskIndex][rsArr[ei]], + lastMaskShiftOut[ei], ei, maskBitCnt) ; + + int ei2 = ei + AVX_LENGTH/2 ; // the second entry index + SET_MASK_WORD(bitMaskVec.getHighEntry(ei), maskArr[maskIndex][rsArr[ei2]], + lastMaskShiftOut[ei2], ei2, maskBitCnt) ; + } + +} + + +inline void CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (BITMASK_VEC& bitMaskVec, SIMD_TYPE& distm, SIMD_TYPE& _1_distm, SIMD_TYPE& distmChosen) { + + distmChosen = VEC_BLENDV(distm, _1_distm, bitMaskVec.getCombinedMask()) ; + + bitMaskVec.shift_left_1bit() ; +} + +/* + * This function: + * 1- Intializes probability values p_MM, p_XX, P_YY, p_MX, p_GAPM and pack them into vectors (SSE or AVX) + * 2- Precompute parts of "distm" which only depeneds on a row number and pack it into vector + */ + +template void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, SIMD_TYPE *p_MM, SIMD_TYPE *p_GAPM, SIMD_TYPE *p_MX, SIMD_TYPE *p_XX, SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, SIMD_TYPE *distm1D) +{ + NUMBER zero = ctx._(0.0); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + for (int s=0;si[r-1] & 127; + int _d = tc->d[r-1] & 127; + int _c = tc->c[r-1] & 127; + + //*(ptr_p_MM+r-1) = ctx._(1.0) - ctx.ph2pr[(_i + _d) & 127]; + SET_MATCH_TO_MATCH_PROB(*(ptr_p_MM+r-1), _i, _d); + *(ptr_p_GAPM+r-1) = ctx._(1.0) - ctx.ph2pr[_c]; + *(ptr_p_MX+r-1) = ctx.ph2pr[_i]; + *(ptr_p_XX+r-1) = ctx.ph2pr[_c]; + *(ptr_p_MY+r-1) = ctx.ph2pr[_d]; + *(ptr_p_YY+r-1) = ctx.ph2pr[_c]; + } + + NUMBER *ptr_distm1D = (NUMBER *)distm1D; + for (int r = 1; r < ROWS; r++) + { + int _q = tc->q[r-1] & 127; + ptr_distm1D[r-1] = ctx.ph2pr[_q]; + } +} + +/* + * This function handles pre-stripe computation: + * 1- Retrieve probaility vectors from memory + * 2- Initialize M, X, Y vectors with all 0's (for the first stripe) and shifting the last row from previous stripe for the rest + */ + +template inline void CONCAT(CONCAT(stripeINITIALIZATION,SIMD_ENGINE), PRECISION)( + int stripeIdx, Context ctx, testcase *tc, SIMD_TYPE &pGAPM, SIMD_TYPE &pMM, SIMD_TYPE &pMX, SIMD_TYPE &pXX, SIMD_TYPE &pMY, SIMD_TYPE &pYY, + SIMD_TYPE &rs, UNION_TYPE &rsN, SIMD_TYPE &distm, SIMD_TYPE &_1_distm, SIMD_TYPE *distm1D, SIMD_TYPE N_packed256, SIMD_TYPE *p_MM , SIMD_TYPE *p_GAPM , + SIMD_TYPE *p_MX, SIMD_TYPE *p_XX , SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM) +{ + int i = stripeIdx; + pGAPM = p_GAPM[i]; + pMM = p_MM[i]; + pMX = p_MX[i]; + pXX = p_XX[i]; + pMY = p_MY[i]; + pYY = p_YY[i]; + + NUMBER zero = ctx._(0.0); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + UNION_TYPE packed3; packed3.d = VEC_SET1_VAL(3.0); + + distm = distm1D[i]; + _1_distm = VEC_SUB(packed1.d, distm); + + distm = VEC_DIV(distm, packed3.d); + + /* initialize M_t_2, M_t_1, X_t_2, X_t_1, Y_t_2, Y_t_1 */ + M_t_2.d = VEC_SET1_VAL(zero); + X_t_2.d = VEC_SET1_VAL(zero); + + if (i==0) { + M_t_1.d = VEC_SET1_VAL(zero); + X_t_1.d = VEC_SET1_VAL(zero); + Y_t_2.d = VEC_SET_LSE(init_Y); + Y_t_1.d = VEC_SET1_VAL(zero); + } + else { + X_t_1.d = VEC_SET_LSE(shiftOutX[AVX_LENGTH]); + M_t_1.d = VEC_SET_LSE(shiftOutM[AVX_LENGTH]); + Y_t_2.d = VEC_SET1_VAL(zero); + Y_t_1.d = VEC_SET1_VAL(zero); + } + M_t_1_y = M_t_1; +} + +/* + * This function is the main compute kernel to compute M, X and Y + */ + +inline void CONCAT(CONCAT(computeMXY,SIMD_ENGINE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, + UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, + SIMD_TYPE pMM, SIMD_TYPE pGAPM, SIMD_TYPE pMX, SIMD_TYPE pXX, SIMD_TYPE pMY, SIMD_TYPE pYY, SIMD_TYPE distmSel) +{ + /* Compute M_t <= distm * (p_MM*M_t_2 + p_GAPM*X_t_2 + p_GAPM*Y_t_2) */ + M_t.d = VEC_MUL(VEC_ADD(VEC_ADD(VEC_MUL(M_t_2.d, pMM), VEC_MUL(X_t_2.d, pGAPM)), VEC_MUL(Y_t_2.d, pGAPM)), distmSel); + //M_t.d = VEC_MUL( VEC_ADD(VEC_MUL(M_t_2.d, pMM), VEC_MUL(VEC_ADD(X_t_2.d, Y_t_2.d), pGAPM)), distmSel); + + M_t_y = M_t; + + /* Compute X_t */ + X_t.d = VEC_ADD(VEC_MUL(M_t_1.d, pMX) , VEC_MUL(X_t_1.d, pXX)); + + /* Compute Y_t */ + Y_t.d = VEC_ADD(VEC_MUL(M_t_1_y.d, pMY) , VEC_MUL(Y_t_1.d, pYY)); +} + +/* + * This is the main compute function. It operates on the matrix in s stripe manner. + * The stripe height is determined by the SIMD engine type. + * Stripe height: "AVX float": 8, "AVX double": 4, "SSE float": 4, "SSE double": 2 + * For each stripe the operations are anti-diagonal based. + * Each anti-diagonal (M_t, Y_t, X_t) depends on the two previous anti-diagonals (M_t_2, X_t_2, Y_t_2, M_t_1, X_t_1, Y_t_1). + * Each stripe (except the fist one) depends on the last row of the previous stripe. + * The last stripe computation handles the addition of the last row of M and X, that's the reason for loop spliting. + */ + +template NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL) +{ + int ROWS = tc->rslen + 1; + int COLS = tc->haplen + 1; + int MAVX_COUNT = (ROWS+AVX_LENGTH-1)/AVX_LENGTH; + + /* Probaility arrays */ + SIMD_TYPE p_MM [MAVX_COUNT], p_GAPM [MAVX_COUNT], p_MX [MAVX_COUNT]; + SIMD_TYPE p_XX [MAVX_COUNT], p_MY [MAVX_COUNT], p_YY [MAVX_COUNT]; + + /* For distm precomputation */ + SIMD_TYPE distm1D[MAVX_COUNT]; + + /* Carries the values from each stripe to the next stripe */ + NUMBER shiftOutM[ROWS+COLS+AVX_LENGTH], shiftOutX[ROWS+COLS+AVX_LENGTH], shiftOutY[ROWS+COLS+AVX_LENGTH]; + + /* The vector to keep the anti-diagonals of M, X, Y*/ + /* Current: M_t, X_t, Y_t */ + /* Previous: M_t_1, X_t_1, Y_t_1 */ + /* Previous to previous: M_t_2, X_t_2, Y_t_2 */ + UNION_TYPE M_t, M_t_1, M_t_2, X_t, X_t_1, X_t_2, Y_t, Y_t_1, Y_t_2, M_t_y, M_t_1_y; + + /* Probality vectors */ + SIMD_TYPE pGAPM, pMM, pMX, pXX, pMY, pYY; + + struct timeval start, end; + NUMBER result_avx2; + Context ctx; + UNION_TYPE rs , rsN; + HAP_TYPE hap; + SIMD_TYPE distmSel, distmChosen ; + SIMD_TYPE distm, _1_distm; + + int r, c; + NUMBER zero = ctx._(0.0); + UNION_TYPE packed1; packed1.d = VEC_SET1_VAL(1.0); + SIMD_TYPE N_packed256 = VEC_POPCVT_CHAR('N'); + NUMBER init_Y = ctx.INITIAL_CONSTANT / (tc->haplen); + int remainingRows = (ROWS-1) % AVX_LENGTH; + int stripe_cnt = ((ROWS-1) / AVX_LENGTH) + (remainingRows!=0); + + const int maskBitCnt = MAIN_TYPE_SIZE ; + const int numMaskVecs = (COLS+ROWS+maskBitCnt-1)/maskBitCnt ; // ceil function + + /* Mask precomputation for distm*/ + MASK_TYPE maskArr[numMaskVecs][NUM_DISTINCT_CHARS] ; + CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(*tc, COLS, numMaskVecs, maskArr) ; + + char rsArr[AVX_LENGTH] ; + MASK_TYPE lastMaskShiftOut[AVX_LENGTH] ; + + /* Precompute initialization for probabilities and shift vector*/ + CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(ROWS, COLS, shiftOutM, shiftOutX, shiftOutY, + ctx, tc, p_MM, p_GAPM, p_MX, p_XX, p_MY, p_YY, distm1D); + + for (int i=0;i(&tc[b]); + +#ifdef RUN_HYBRID +#define MIN_ACCEPTED 1e-28f + if (result_avxf < MIN_ACCEPTED) { + count++; + result_avxd = CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), d)(&tc[b]); + result[b] = log10(result_avxd) - log10(ldexp(1.0, 1020.f)); + } + else + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif + +#ifndef RUN_HYBRID + result[b] = log10f(result_avxf) - log10f(ldexpf(1.f, 120.f)); +#endif + } + aggregateTimeCompute += (getCurrClk() - lastClk) ; + lastClk = getCurrClk() ; + for (int b=0;b(testcase* tc, double* nextlog); +template float compute_full_prob_sses(testcase* tc, float* nextlog); diff --git a/public/VectorPairHMM/src/main/c++/template.h b/public/VectorPairHMM/src/main/c++/template.h new file mode 100644 index 000000000..ce4dbfc86 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/template.h @@ -0,0 +1,320 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef TEMPLATES_H_ +#define TEMPLATES_H_ + +#include "headers.h" + +#define MM 0 +#define GapM 1 +#define MX 2 +#define XX 3 +#define MY 4 +#define YY 5 + +//#define MROWS 500 +//#define MCOLS 1000 + +#define CAT(X,Y) X####Y +#define CONCAT(X,Y) CAT(X,Y) + +#define ALIGNED __attribute__((aligned(32))) + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256 ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED float ALIGNED f[8]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_F ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m128 ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED float ALIGNED f[4]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_F128 ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128 vecf ; + uint32_t masks[4] ; +} MaskVec_F ; + +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint32_t masks[2] ; +} MaskVec_F128 ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128 ALIGNED f; +} ALIGNED IF_128f ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int ALIGNED i; + ALIGNED float ALIGNED f; +} ALIGNED IF_32 ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m256d ALIGNED d; + ALIGNED __m128i ALIGNED s[2]; + ALIGNED double ALIGNED f[4]; + ALIGNED __m256i ALIGNED i; +} ALIGNED mix_D ALIGNED; + +typedef union __attribute__((aligned(32))) { + ALIGNED __m128d ALIGNED d; + ALIGNED __m64 ALIGNED s[2]; + ALIGNED double ALIGNED f[2]; + ALIGNED __m128i ALIGNED i; +} ALIGNED mix_D128 ALIGNED; + +typedef union ALIGNED { + __m128i vec ; + __m128d vecf ; + uint64_t masks[2] ; +} MaskVec_D ; + +typedef union ALIGNED { + __m64 vec ; + __m64 vecf ; + uint64_t masks[1] ; +} MaskVec_D128 ; + +typedef union ALIGNED +{ + ALIGNED __m128i ALIGNED i; + ALIGNED __m128d ALIGNED f; +} ALIGNED IF_128d ALIGNED; + +typedef union ALIGNED +{ + ALIGNED int64_t ALIGNED i; + ALIGNED double ALIGNED f; +} ALIGNED IF_64 ALIGNED; + + +#define MAX_QUAL 254 +#define MAX_JACOBIAN_TOLERANCE 8.0 +#define JACOBIAN_LOG_TABLE_STEP 0.0001 +#define JACOBIAN_LOG_TABLE_INV_STEP (1.0 / JACOBIAN_LOG_TABLE_STEP) +#define MAXN 70000 +#define LOG10_CACHE_SIZE (4*MAXN) // we need to be able to go up to 2*(2N) when calculating some of the coefficients +#define JACOBIAN_LOG_TABLE_SIZE ((int) (MAX_JACOBIAN_TOLERANCE / JACOBIAN_LOG_TABLE_STEP) + 1) + +template +struct ContextBase +{ + public: + NUMBER ph2pr[128]; + NUMBER INITIAL_CONSTANT; + NUMBER LOG10_INITIAL_CONSTANT; + NUMBER RESULT_THRESHOLD; + + static bool staticMembersInitializedFlag; + static NUMBER jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; + static NUMBER matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; + + static void initializeStaticMembers() + { + if(!staticMembersInitializedFlag) + { + //Order of calls important - Jacobian first, then MatchToMatch + initializeJacobianLogTable(); + initializeMatchToMatchProb(); + staticMembersInitializedFlag = true; + } + } + + static void deleteStaticMembers() + { + if(staticMembersInitializedFlag) + { + staticMembersInitializedFlag = false; + } + } + + //Called only once during library load - don't bother to optimize with single precision fp + static void initializeJacobianLogTable() + { + for (int k = 0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { + jacobianLogTable[k] = (NUMBER)(log10(1.0 + pow(10.0, -((double) k) * JACOBIAN_LOG_TABLE_STEP))); + } + } + + //Called only once per library load - don't bother optimizing with single fp + static void initializeMatchToMatchProb() + { + double LN10 = log(10); + double INV_LN10 = 1.0/LN10; + for (int i = 0, offset = 0; i <= MAX_QUAL; offset += ++i) + for (int j = 0; j <= i; j++) { + double log10Sum = approximateLog10SumLog10(-0.1*i, -0.1*j); + double matchToMatchLog10 = + log1p(-std::min(1.0,pow(10,log10Sum))) * INV_LN10; + matchToMatchProb[offset + j] = (NUMBER)(pow(10,matchToMatchLog10)); + } + } + //Called during computation - use single precision where possible + static int fastRound(NUMBER d) { + return (d > ((NUMBER)0.0)) ? (int) (d + ((NUMBER)0.5)) : (int) (d - ((NUMBER)0.5)); + } + //Called during computation - use single precision where possible + static NUMBER approximateLog10SumLog10(NUMBER small, NUMBER big) { + // make sure small is really the smaller value + if (small > big) { + NUMBER t = big; + big = small; + small = t; + } + + if (isinf(small) == -1 || isinf(big) == -1) + return big; + + NUMBER diff = big - small; + if (diff >= ((NUMBER)MAX_JACOBIAN_TOLERANCE)) + return big; + + // OK, so |y-x| < tol: we use the following identity then: + // we need to compute log10(10^x + 10^y) + // By Jacobian logarithm identity, this is equal to + // max(x,y) + log10(1+10^-abs(x-y)) + // we compute the second term as a table lookup with integer quantization + // we have pre-stored correction for 0,0.1,0.2,... 10.0 + int ind = fastRound((NUMBER)(diff * ((NUMBER)JACOBIAN_LOG_TABLE_INV_STEP))); // hard rounding + return big + jacobianLogTable[ind]; + } +}; + +template +struct Context : public ContextBase +{}; + +template<> +struct Context : public ContextBase +{ + Context():ContextBase() + { + for (int x = 0; x < 128; x++) + ph2pr[x] = pow(10.0, -((double)x) / 10.0); + + INITIAL_CONSTANT = ldexp(1.0, 1020.0); + LOG10_INITIAL_CONSTANT = log10(INITIAL_CONSTANT); + RESULT_THRESHOLD = 0.0; + } + + double LOG10(double v){ return log10(v); } + inline double POW(double b, double e) { return pow(b,e); } + + static double _(double n){ return n; } + static double _(float n){ return ((double) n); } +}; + +template<> +struct Context : public ContextBase +{ + Context() : ContextBase() + { + for (int x = 0; x < 128; x++) + { + ph2pr[x] = powf(10.f, -((float)x) / 10.f); + } + + INITIAL_CONSTANT = ldexpf(1.f, 120.f); + LOG10_INITIAL_CONSTANT = log10f(INITIAL_CONSTANT); + RESULT_THRESHOLD = ldexpf(1.f, -110.f); + } + + float LOG10(float v){ return log10f(v); } + inline float POW(float b, float e) { return powf(b,e); } + + static float _(double n){ return ((float) n); } + static float _(float n){ return n; } +}; + +#define SET_MATCH_TO_MATCH_PROB(output, insQual, delQual) \ +{ \ + int minQual = delQual; \ + int maxQual = insQual; \ + if (insQual <= delQual) \ + { \ + minQual = insQual; \ + maxQual = delQual; \ + } \ + (output) = (MAX_QUAL < maxQual) ? \ + ((NUMBER)1.0) - ctx.POW(((NUMBER)10), ctx.approximateLog10SumLog10(((NUMBER)-0.1)*minQual, ((NUMBER)-0.1)*maxQual)) \ + : ctx.matchToMatchProb[((maxQual * (maxQual + 1)) >> 1) + minQual]; \ +} + +typedef struct +{ + int rslen, haplen; + /*int *q, *i, *d, *c;*/ + /*int q[MROWS], i[MROWS], d[MROWS], c[MROWS];*/ + char *q, *i, *d, *c; + char *hap, *rs; + int *ihap; + int *irs; +} testcase; + +int normalize(char c); +int read_testcase(testcase *tc, FILE* ifp=0); + + +#define MIN_ACCEPTED 1e-28f +#define NUM_DISTINCT_CHARS 5 +#define AMBIG_CHAR 4 + +class ConvertChar { + + static uint8_t conversionTable[255] ; + +public: + + static void init() { + assert (NUM_DISTINCT_CHARS == 5) ; + assert (AMBIG_CHAR == 4) ; + + conversionTable['A'] = 0 ; + conversionTable['C'] = 1 ; + conversionTable['T'] = 2 ; + conversionTable['G'] = 3 ; + conversionTable['N'] = 4 ; + } + + static inline uint8_t get(uint8_t input) { + return conversionTable[input] ; + } + +}; + + +#endif + + diff --git a/public/VectorPairHMM/src/main/c++/utils.cc b/public/VectorPairHMM/src/main/c++/utils.cc new file mode 100644 index 000000000..9f83cffa2 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/utils.cc @@ -0,0 +1,493 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#include "headers.h" +#include "template.h" +#include "utils.h" +#include "vector_defs.h" +#include "LoadTimeInitializer.h" +using namespace std; + +//static members from ConvertChar +uint8_t ConvertChar::conversionTable[255]; +//Global function pointers in utils.h +float (*g_compute_full_prob_float)(testcase *tc, float* before_last_log) = 0; +double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log) = 0; +//Static members in ContextBase +bool ContextBase::staticMembersInitializedFlag = false; +double ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +double ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; +bool ContextBase::staticMembersInitializedFlag = false; +float ContextBase::jacobianLogTable[JACOBIAN_LOG_TABLE_SIZE]; +float ContextBase::matchToMatchProb[((MAX_QUAL + 1) * (MAX_QUAL + 2)) >> 1]; + + +bool is_avx_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 28)&1) == 1; +} + +bool is_sse41_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 19)&1) == 1; +} + +bool is_sse42_supported() +{ + int ecx = 0, edx = 0, ebx = 0; + __asm__("cpuid" + : "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (1) + ); + return ((ecx >> 20)&1) == 1; +} + +uint64_t get_machine_capabilities() +{ + uint64_t machine_mask = 0ull; + if(is_avx_supported()) + machine_mask |= (1 << AVX_CUSTOM_IDX); + if(is_sse42_supported()) + machine_mask |= (1 << SSE42_CUSTOM_IDX); + if(is_sse41_supported()) + machine_mask |= (1 << SSE41_CUSTOM_IDX); + return machine_mask; +} + +void initialize_function_pointers(uint64_t mask) +{ + //mask = 0ull; + //mask = (1 << SSE41_CUSTOM_IDX); + if(is_avx_supported() && (mask & (1<< AVX_CUSTOM_IDX))) + { + cout << "Using AVX accelerated implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob_avxs; + g_compute_full_prob_double = compute_full_prob_avxd; + } + else + if(is_sse41_supported() && (mask & ((1<< SSE41_CUSTOM_IDX) | (1<; + g_compute_full_prob_double = compute_full_prob_ssed; + } + else + { + cout << "Using un-vectorized C++ implementation of PairHMM\n"; + g_compute_full_prob_float = compute_full_prob; + g_compute_full_prob_double = compute_full_prob; + } +} + +int normalize(char c) +{ + return ((int) (c - 33)); +} + +int read_testcase(testcase *tc, FILE* ifp) +{ + char *q, *i, *d, *c, *line = NULL; + int _q, _i, _d, _c; + int x, size = 0; + ssize_t read; + + + read = getline(&line, (size_t *) &size, ifp == 0 ? stdin : ifp); + if (read == -1) + { + free(line); + return -1; + } + + + tc->hap = (char *) malloc(size); + tc->rs = (char *) malloc(size); + q = (char *) malloc(size); + i = (char *) malloc(size); + d = (char *) malloc(size); + c = (char *) malloc(size); + + if (sscanf(line, "%s %s %s %s %s %s\n", tc->hap, tc->rs, q, i, d, c) != 6) + return -1; + + + tc->haplen = strlen(tc->hap); + tc->rslen = strlen(tc->rs); + assert(strlen(q) == tc->rslen); + assert(strlen(i) == tc->rslen); + assert(strlen(d) == tc->rslen); + assert(strlen(c) == tc->rslen); + //assert(tc->rslen < MROWS); + //tc->ihap = (int *) malloc(tc->haplen*sizeof(int)); + //tc->irs = (int *) malloc(tc->rslen*sizeof(int)); + + tc->q = (char *) malloc(sizeof(char) * tc->rslen); + tc->i = (char *) malloc(sizeof(char) * tc->rslen); + tc->d = (char *) malloc(sizeof(char) * tc->rslen); + tc->c = (char *) malloc(sizeof(char) * tc->rslen); + + for (x = 0; x < tc->rslen; x++) + { + _q = normalize(q[x]); + _i = normalize(i[x]); + _d = normalize(d[x]); + _c = normalize(c[x]); + tc->q[x] = (_q < 6) ? 6 : _q; + //tc->q[x] = _q; + tc->i[x] = _i; + tc->d[x] = _d; + tc->c[x] = _c; + //tc->irs[x] = tc->rs[x]; + } + //for (x = 0; x < tc->haplen; x++) + //tc->ihap[x] = tc->hap[x]; + + free(q); + free(i); + free(d); + free(c); + free(line); + + + + return 0; +} + +unsigned MAX_LINE_LENGTH = 65536; +int convToInt(std::string s) +{ + int i; + std::istringstream strin(s); + strin >> i; + return i; +} + +void tokenize(std::ifstream& fptr, std::vector& tokens) +{ + int i = 0; + std::string tmp; + std::vector myVec; + vector line; + line.clear(); + line.resize(MAX_LINE_LENGTH); + vector tmpline; + tmpline.clear(); + tmpline.resize(MAX_LINE_LENGTH); + myVec.clear(); + + while(!fptr.eof()) + { + i = 0; + bool still_read_line = true; + unsigned line_position = 0; + while(still_read_line) + { + fptr.getline(&(tmpline[0]), MAX_LINE_LENGTH); + if(line_position + MAX_LINE_LENGTH > line.size()) + line.resize(2*line.size()); + for(unsigned i=0;i> std::skipws >> tmp; + if(tmp != "") + { + myVec.push_back(tmp); + ++i; + //std::cout < 0) + break; + } + tokens.clear(); + //std::cout << "Why "< tokens; + tokens.clear(); + tokenize(fptr, tokens); + if(tokens.size() == 0) + return -1; + tc->hap = new char[tokens[0].size()+2]; + tc->haplen = tokens[0].size(); + memcpy(tc->hap, tokens[0].c_str(), tokens[0].size()); + tc->rs = new char[tokens[1].size()+2]; + tc->rslen = tokens[1].size(); + tc->q = new char[tc->rslen]; + tc->i = new char[tc->rslen]; + tc->d = new char[tc->rslen]; + tc->c = new char[tc->rslen]; + //cout << "Lengths "<haplen <<" "<rslen<<"\n"; + memcpy(tc->rs, tokens[1].c_str(),tokens[1].size()); + assert(tokens.size() == 2 + 4*(tc->rslen)); + //assert(tc->rslen < MROWS); + for(unsigned j=0;jrslen;++j) + tc->q[j] = (char)convToInt(tokens[2+0*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->i[j] = (char)convToInt(tokens[2+1*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->d[j] = (char)convToInt(tokens[2+2*tc->rslen+j]); + for(unsigned j=0;jrslen;++j) + tc->c[j] = (char)convToInt(tokens[2+3*tc->rslen+j]); + + if(reformat) + { + ofstream ofptr; + ofptr.open("reformat/debug_dump.txt",first_call ? ios::out : ios::app); + assert(ofptr.is_open()); + ofptr << tokens[0] << " "; + ofptr << tokens[1] << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->q[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->i[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->d[j]+33)); + ofptr << " "; + for(unsigned j=0;jrslen;++j) + ofptr << ((char)(tc->c[j]+33)); + ofptr << " 0 false\n"; + + ofptr.close(); + first_call = false; + } + + + return tokens.size(); +} + +double getCurrClk() { + struct timeval tv ; + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} + +inline unsigned long long rdtsc(void) +{ + unsigned hi, lo; + __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); + return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 ); +} + +void get_time(struct timespec* store_struct) +{ + clock_gettime(CLOCK_REALTIME, store_struct); +} + +uint64_t diff_time(struct timespec& prev_time) +{ + struct timespec curr_time; + clock_gettime(CLOCK_REALTIME, &curr_time); + return (uint64_t)((curr_time.tv_sec-prev_time.tv_sec)*1000000000+(curr_time.tv_nsec-prev_time.tv_nsec)); +} + + +#ifdef USE_PAPI +#include "papi.h" +#define NUM_PAPI_COUNTERS 4 +#endif + +void do_compute(char* filename, bool use_old_read_testcase, unsigned chunk_size, bool do_check) +{ + FILE* fptr = 0; + ifstream ifptr; + if(use_old_read_testcase) + { + fptr = fopen(filename,"r"); + assert(fptr); + } + else + { + ifptr.open(filename); + assert(ifptr.is_open()); + } + vector tc_vector; + tc_vector.clear(); + testcase tc; + uint64_t vector_compute_time = 0; + uint64_t baseline_compute_time = 0; + uint64_t num_double_calls = 0; + unsigned num_testcases = 0; + bool all_ok = do_check ? true : false; +#ifdef USE_PAPI + uint32_t all_mask = (0); + uint32_t no_usr_mask = (1 << 16); //bit 16 user mode, bit 17 kernel mode + uint32_t no_kernel_mask = (1 << 17); //bit 16 user mode, bit 17 kernel mode + PAPI_num_counters(); + int events[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + char* eventnames[NUM_PAPI_COUNTERS]= { "cycles", "itlb_walk_cycles", "dtlb_load_walk_cycles", "dtlb_store_walk_cycles" }; + assert(PAPI_event_name_to_code("UNHALTED_REFERENCE_CYCLES:u=1:k=1",&(events[0])) == PAPI_OK); + assert(PAPI_event_name_to_code("ITLB_MISSES:WALK_DURATION", &(events[1])) == PAPI_OK); + assert(PAPI_event_name_to_code("DTLB_LOAD_MISSES:WALK_DURATION", &(events[2])) == PAPI_OK); + assert(PAPI_event_name_to_code("DTLB_STORE_MISSES:WALK_DURATION", &(events[3])) == PAPI_OK); + long long values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; + long long accum_values[NUM_PAPI_COUNTERS] = { 0, 0, 0, 0 }; +#endif + while(1) + { + int break_value = use_old_read_testcase ? read_testcase(&tc, fptr) : read_mod_testcase(ifptr,&tc,true); + if(break_value >= 0) + tc_vector.push_back(tc); + if(tc_vector.size() == BATCH_SIZE || (break_value < 0 && tc_vector.size() > 0)) + { + vector results_vec; + vector baseline_results_vec; + results_vec.clear(); + baseline_results_vec.clear(); + results_vec.resize(tc_vector.size()); + baseline_results_vec.resize(tc_vector.size()); + struct timespec start_time; +#ifdef USE_PAPI + assert(PAPI_start_counters(events, NUM_PAPI_COUNTERS) == PAPI_OK); +#endif + get_time(&start_time); +#pragma omp parallel for schedule(dynamic,chunk_size) num_threads(12) +#ifdef DO_REPEAT_PROFILING + for(unsigned z=0;z<10;++z) +#endif + { + for(unsigned i=0;i(&tc); + baseline_result = log10(baseline_result) - log10(ldexp(1.0, 1020.0)); + baseline_results_vec[i] = baseline_result; + } + baseline_compute_time += diff_time(start_time); + for(unsigned i=0;i 1e-5 && rel_error > 1e-5) + { + cout << std::scientific << baseline_result << " "< +std::string to_string(T obj) +{ + std::stringstream ss; + std::string ret_string; + ss.clear(); + ss << std::scientific << obj; + ss >> ret_string; + ss.clear(); + return ret_string; +} +void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline=true); + +int read_mod_testcase(std::ifstream& fptr, testcase* tc, bool reformat=false); + +bool is_avx_supported(); +bool is_sse42_supported(); +extern float (*g_compute_full_prob_float)(testcase *tc, float *before_last_log); +extern double (*g_compute_full_prob_double)(testcase *tc, double* before_last_log); +void debug_dump(std::string filename, std::string s, bool to_append, bool add_newline); +template +NUMBER compute_full_prob(testcase *tc, NUMBER *before_last_log=0); +double getCurrClk(); +void get_time(struct timespec* x); +uint64_t diff_time(struct timespec& prev_time); + +//bit 0 is sse4.2, bit 1 is AVX +enum ProcessorCapabilitiesEnum +{ + SSE41_CUSTOM_IDX=0, + SSE42_CUSTOM_IDX, + AVX_CUSTOM_IDX +}; +#define ENABLE_ALL_HARDWARE_FEATURES 0xFFFFFFFFFFFFFFFFull +uint64_t get_machine_capabilities(); +void initialize_function_pointers(uint64_t mask=ENABLE_ALL_HARDWARE_FEATURES); +void do_compute(char* filename, bool use_old_read_testcase=true, unsigned chunk_size=10000, bool do_check=true); + +//#define DO_WARMUP +//#define DO_REPEAT_PROFILING +//#define DUMP_COMPUTE_VALUES 1 +#define BATCH_SIZE 10000 +#define RUN_HYBRID + +#endif diff --git a/public/VectorPairHMM/src/main/c++/vector_defs.h b/public/VectorPairHMM/src/main/c++/vector_defs.h new file mode 100644 index 000000000..2aca9565f --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/vector_defs.h @@ -0,0 +1,55 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX +#undef SIMD_ENGINE_SSE + +#define SIMD_ENGINE avx +#define SIMD_ENGINE_AVX + +#include "define-float.h" +#include "vector_function_prototypes.h" + +#include "define-double.h" +#include "vector_function_prototypes.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX + +#define SIMD_ENGINE sse +#define SIMD_ENGINE_SSE + + +#include "define-sse-float.h" +#include "vector_function_prototypes.h" + +#include "define-sse-double.h" +#include "vector_function_prototypes.h" + +#undef SIMD_ENGINE +#undef SIMD_ENGINE_AVX +#undef SIMD_ENGINE_SSE + diff --git a/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h b/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h new file mode 100644 index 000000000..c0fddc394 --- /dev/null +++ b/public/VectorPairHMM/src/main/c++/vector_function_prototypes.h @@ -0,0 +1,44 @@ +/*Copyright (c) 2012 The Broad Institute + +*Permission is hereby granted, free of charge, to any person +*obtaining a copy of this software and associated documentation +*files (the "Software"), to deal in the Software without +*restriction, including without limitation the rights to use, +*copy, modify, merge, publish, distribute, sublicense, and/or sell +*copies of the Software, and to permit persons to whom the +*Software is furnished to do so, subject to the following +*conditions: + +*The above copyright notice and this permission notice shall be +*included in all copies or substantial portions of the Software. + +*THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +*EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +*OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +*NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +*HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +*WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +*FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +*THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +inline void CONCAT(CONCAT(_vector_shift,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn, MAIN_TYPE &shiftOut); +inline void CONCAT(CONCAT(_vector_shift_last,SIMD_ENGINE), PRECISION) (UNION_TYPE &x, MAIN_TYPE shiftIn); +inline void CONCAT(CONCAT(precompute_masks_,SIMD_ENGINE), PRECISION)(const testcase& tc, int COLS, int numMaskVecs, MASK_TYPE (*maskArr)[NUM_DISTINCT_CHARS]); +inline void CONCAT(CONCAT(init_masks_for_row_,SIMD_ENGINE), PRECISION)(const testcase& tc, char* rsArr, MASK_TYPE* lastMaskShiftOut, int beginRowIndex, int numRowsToProcess); +inline void CONCAT(CONCAT(update_masks_for_cols_,SIMD_ENGINE), PRECISION)(int maskIndex, MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, MASK_TYPE (*maskArr) [NUM_DISTINCT_CHARS], char* rsArr, MASK_TYPE* lastMaskShiftOut, MASK_TYPE maskBitCnt); +inline void CONCAT(CONCAT(computeDistVec,SIMD_ENGINE), PRECISION) (MASK_VEC& currMaskVecLow, MASK_VEC& currMaskVecHigh, SIMD_TYPE& distm, SIMD_TYPE& _1_distm, SIMD_TYPE& distmChosen); +template inline void CONCAT(CONCAT(initializeVectors,SIMD_ENGINE), PRECISION)(int ROWS, int COLS, NUMBER* shiftOutM, NUMBER *shiftOutX, NUMBER *shiftOutY, Context ctx, testcase *tc, SIMD_TYPE *p_MM, SIMD_TYPE *p_GAPM, SIMD_TYPE *p_MX, SIMD_TYPE *p_XX, SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, SIMD_TYPE *distm1D); +template inline void CONCAT(CONCAT(stripINITIALIZATION,SIMD_ENGINE), PRECISION)( + int stripIdx, Context ctx, testcase *tc, SIMD_TYPE &pGAPM, SIMD_TYPE &pMM, SIMD_TYPE &pMX, SIMD_TYPE &pXX, SIMD_TYPE &pMY, SIMD_TYPE &pYY, + SIMD_TYPE &rs, UNION_TYPE &rsN, SIMD_TYPE &distm, SIMD_TYPE &_1_distm, SIMD_TYPE *distm1D, SIMD_TYPE N_packed256, SIMD_TYPE *p_MM , SIMD_TYPE *p_GAPM , + SIMD_TYPE *p_MX, SIMD_TYPE *p_XX , SIMD_TYPE *p_MY, SIMD_TYPE *p_YY, UNION_TYPE &M_t_2, UNION_TYPE &X_t_2, UNION_TYPE &M_t_1, UNION_TYPE &X_t_1, + UNION_TYPE &Y_t_2, UNION_TYPE &Y_t_1, UNION_TYPE &M_t_1_y, NUMBER* shiftOutX, NUMBER* shiftOutM); +inline SIMD_TYPE CONCAT(CONCAT(computeDISTM,SIMD_ENGINE), PRECISION)(int d, int COLS, testcase * tc, HAP_TYPE &hap, SIMD_TYPE rs, UNION_TYPE rsN, SIMD_TYPE N_packed256, + SIMD_TYPE distm, SIMD_TYPE _1_distm); +inline void CONCAT(CONCAT(computeMXY,SIMD_ENGINE), PRECISION)(UNION_TYPE &M_t, UNION_TYPE &X_t, UNION_TYPE &Y_t, UNION_TYPE &M_t_y, + UNION_TYPE M_t_2, UNION_TYPE X_t_2, UNION_TYPE Y_t_2, UNION_TYPE M_t_1, UNION_TYPE X_t_1, UNION_TYPE M_t_1_y, UNION_TYPE Y_t_1, + SIMD_TYPE pMM, SIMD_TYPE pGAPM, SIMD_TYPE pMX, SIMD_TYPE pXX, SIMD_TYPE pMY, SIMD_TYPE pYY, SIMD_TYPE distmSel); +template NUMBER CONCAT(CONCAT(compute_full_prob_,SIMD_ENGINE), PRECISION) (testcase *tc, NUMBER *before_last_log = NULL); + diff --git a/public/sting-root/pom.xml b/public/sting-root/pom.xml index 171eb7620..549a99ae6 100644 --- a/public/sting-root/pom.xml +++ b/public/sting-root/pom.xml @@ -335,7 +335,11 @@ maven-assembly-plugin 2.4 - + + org.apache.maven.plugins + maven-enforcer-plugin + 1.3.1 +

cZ-x*bL@^bRF-^(%PSLf5Xx1N{0S zk9PgXO8q&fQq>Zrj6PW5b`3#qSE}lw6utMCpSc}r?E@N+^eA3IFaLB~@7;Kfx+^Vb z=hoS-A=RdQ<7R@TV>$!~T#4kb971wPdKt=}2+Gd(X2v5sw@Qh(S{YqKmXbkQcG~o` z+tcotHeFfv1=rM7{eq^i^0a|xn*Oe)S9=`5YWg<4uSW^3@m#?)!9Lrn#hasjb!#6; zjbyQspy{WT;0BT3O5Twsrc;`kfI6p9^+xbES#YrIN^a;Nmwkt<*mE!JhP+j&+AQRB zxT*~&xC5j}05<{j)qLryepI}C;;Np@1jLLNFNa;#SK=jYS^GbukRkL{FZ{?xv?rsc zT?wucOqJj&!7W|i>{*C3(8*#v6n%|n2ei2fve6QqrFaFwPq>O zrFpjlGh%?gVNKS>DB<8{tKe~_+MuW3a*qi-p1Og^QAWFf{hIG+=) z{?(pVusz|5?fr`v9FBi{3sH@Ok`#@0+w6(E2AJzGQYoR7hmnN1W@Mp;1?>>5{&nyN zX0a#6@S;;Qje#`A(*bQhSHr24I>QpN-*VqS~Rl68FRp0)we?kDUGoJ2)k_2 z9wT3%Vpj=qH67%f9j^F#qc2Nf!AfX@Ec5=;-IQ6$GKG8z=vJGcwn#z}a(t>pZm&{E1}LQ9)HZNP0u!YHAsH8RbU!YT${@)uci2WpHou`&%jKKZ-~K0x-B z$ju%mGMc}J7&UdS>bp^D)nZRyM_?cmfw|D3Nc3Ce1T9w5IByg*!Sx|Y8z5<%6AGG; zJP$NcW!UE$p-00Fl%j7{YT&cq7)ZTBBVQSRf*p;c+joT?2Ua2WL!$zYCtW|1*507h zABk3iYef@ect3;DU!myPSH!rMufYT_d<>4kv<|BweR@NBFe;;_&a#z%+4$0YXe@DV zXg@g8wc;rBl#%>pTKkT4y~vY=F7)QRZZu4~Cp%p~Mcj-{KS6DwtsJ4!s)p8JT=LGe z*9axo>4QCn@NZQOvER622R{7=Y9`w=13RcMd>R%c z@NvL{YP^KvT=6>u1YCd36K^~T91wWrAf^T{gAJv7^F0daecy8}k3$q3UJujJg3<+O zh>M;>J++`T1uZE30gaBXLIHjUpUWEPgG`@3?zMN^4r4Hk7iJM7yH7uMTnEw^`K?yB z5jfFZvLp`OMc_m?84H}{uZ(sA93N)WwTkg!-2MY7<*%qWOKEu0a9q}~x*4OhOz=5}WztZ~m|KzLFHE9;NNEn< z0=OPt4Oy53K8@o}Dveb9QTUc+OlFVH+hB@>1aN6E^LPcqia&{b8)aNa)?g5C3`P}` zzeIe;S~y%QzQ#%z{D6Lp!Kne9@jQx!^&@01+atUtT&WU_HTId_BiMU}4WAWjye0_M zo{eCLTx0CiID`I+VQ{Ua4z^R>(!YZGoTOgZ-BDszjNwOKh(bSo7>N!d5gzZ$M>8I- z>Sy5*b-1|Ao|S-&x4vXWQ;t2uCfe*-4cvCL7#`?*Kw;p()A&rZ`oKHPc<%||kc^CF zvJE1q(GG6Fvs{Va>|s*lr(d8HI&QdS2Xx5Vk0$wen`yv%Bc9O2B3CC(Y^98$l=T?QEny%Z^K{b0 z`|wptXrNC-2@!>{1WOa58YjUG*(rdf2_gK|Ev6>UT_!Za1hN+54S>}Q#y+rsv}7$) zh&MBZD5O@Vpo}twcr#Ooc6$UVgeK~Mg(e#0V+YfKcP*aKL>6SR*~Dvw~T_o8CY1&7?4R{J5vzwZ@IBaU-Pkn_Zm_XJ-yaBMd z8n1%|q+J(DA>PasLKACc3d$&Xh&MBZ(8T>nkzD7ip@UwCdoi@J6bW3_m!N!G?f*i# zLMfMZu04}>s{Jn57;5kRV~3mwGdN>~+-B8311lgoo9(Dff5WWzqnD!Ii#-cvShf&p zIugB!oS-e1w8a)p+COOsorx6QM7#l@zC>LgK?)JIeQ11xYNNJg5{$%nW08VBeHXHRgLP-cMaD)5PHY4ZJT~>?xCr2Tuc%f{Xs-Mg3egb0sn&?HNcbDM=j^A?imF zN?VLz#5_XI#t7vfpd{B}w!k{gaIF}LHdKOmW%cTt5$Vdw2FY`j;D}g5M_ymy`(o4v zzoi%?6$Ex5ZWd!bLcOw@eOfR(DQ3JrV2FkgfoUh`>CR{kYacQrnq-n*eQB1+EmQdP zHBJR^npgsHtylw%X26sS-((iXhZGFUY}pJ9HBX#^5YZikwq~zp%*0e_?R7rq*SEP| zA3SKHp7S`ONQpO*{U_>cA3t~zl+hV~Iq}b7#OQ*LP4{jmY2gnc3RYB}5b-b7gCMTG zF3r15s%`mB#&7y@a~>|WOhIFexu18Yuw_IQjI1sc+8!x!#?6ufgEbjuBeuMTNEDs= zI=fl&K1?-3i;EqTg3)c0^z=4VRr91y3XMHu-0=xb^dqWgrm=}8Nm6^G_eBxrWCjPH zQFN!GPs5zWg&AQNOFjra8qUv5GZzE<4;r^Y9&=F&>9`^VLpD+=JyRNg zWdo9h@uKa_3RyuAM7XhzBcQ&YG229oJ*I!YbkM4`?YqmL+bD+*mDk}rBY z9djf$&I#2u^wSe*uA%Q_xc>Yl?o%Kz)<5AI+DQC|K;!)_Cj1hUzTj5ah`w9ZcdGgu zs=iy%_gLjB`Wy6ybTg_B39H4bi>mM9(k-VYs{W>mM7&kzYGf2*rN@Lcp5b}OL&ucY^*+ptSF(W11$=EQ=&y=@2ci1K%|7f zumv9NDB(SpFt($Fx(-%C*kzJcLkC)vFjt~?5RFX=dkIrm0#7lRbvVru4xv@eI#hSC z5+w|gtk!j)MF}wyy`E@Sbu)JPBX#%^O%UEqjI@%093vK5C5V_%tUU(lA}q!Ns@0M`OlL4Pe=mhpFR%qFiGx5@@h$5^r#?jBl$*2{tERm?vv#CGWQ=z zzCn`rkUZ5a8-;@TN|F;Kxt8Q)N&WfN%HlQTu$=I_eH%+ zBsqcPcuAf{@;j0|5fK)0_a*s9l01Us|CQvGlKkakAYU!X14({Dl6Ok-ZjuK}@~K~f zyhxHS0Upf%O!8Hd{4U9NO7bL0UQP0qlDv^*FUita<}|t;#w7GJ>6o}&Q~V*?e2Lg8 z9ru!~++=b%3)4fR&L@X6heh2pWwYEw@)=3KgXBY!+#<=bB%hSzQ6&EjWY-YvcfxFv zX{|DS%1`l*c*OHJJc8qD{tn7aiW(gFbG$1U!}WI*{asaWQuTdUD$PhfjK!ZDLCfHt zKxU$*tE19a&PdyqIw5LX>O|+Z)XCA?QnPz$`s7qipQ8A5CCjI0&dAVn5VfVFPgU0i z#%RH7Rq)VsERUtU0b4OLWvPcZQ;?LwEIiozRRQ(8PKeS=(JyNmg`BQP&byg2R_1(0 z<~%AMy`6$VzZVJV%!#6m`A{=zc`uQnfr*<$QTG8#%L&hgf@ZIl`@yPy07Imze=D{` z`(owWhi!-IPtcxfFxmrF8Yi3sXTxmJJa^%NOy#tl2mjlx)Yxww_?k>zB2&9Nr@kBp6OYu2K0e;sk_(Na-$MIV!6r8ymc}?Q zgcck2o-pMWY3cbSzbnZa$u3D=Bgq*gb0C4JHVMl8>(f`DIBCO0ofmx9JT@ZX)^Dl3etN$i0(f4lyYEFC+)dvL*R>lDA9pqa;rU z*~N1V)-eO_R^rS-DiBMN&e8S!L035K4xG5at_1cd5zia1f}dEIB`||_j+i~oFPFwK zLyR%J2aX!;4ki2bWFK<;o4kRd8wi| zDbBryDmZZ#p>H)_e1`?M8|&W%dnd}njv9;02aJ6_ML#5Ak4T4wV-gq8c|$<4Eo+nU zA5J@^7Tymxj^?R^l z(-3FdDEHxzNWw?1>Rm8KHF#UBnozIQw?!-7t*+&bz+ny)Sd8YyB&`0}t^a^B9Ex*4 zmge`K<-9vasoAFli#)OOQ#4EooNa-NGxZTMnR-zSN@{NnT#~5=Vqyu$Ded(*&f`|R zyUO=tBgH3pg(kKu-h=Z8D)pa7YtB7t)tT5pjH=()s_KPptPe$3o$vX)2J!d6^Wnxb za202RSS7)1Q`M)#$7Nv3)fkI}mB+hOQrXxpFPECI9mTmG4x$A#?>SX}-{+e6xzG8z z&-;1#;Er^fw|PN7U(F|#mt*P}o&04>yt+BU0~q0#aeiaKAc5~=pMDG@bHck~ zBCdwYT}u50r&9mb*vwF1kPGqwj#l+=)V(c`Y5vbt{kYG0HgGu{`WRJ@9;ybj$3WxR zL!n+Y=m0gCJ_Z_3ABrI*8^VFYrd1)K>V2cUP6-W=O|iG;V3cZvleU+8Bp zwf?MAtv@z)N~q`()N2D9dxQ-Rni!LbNFZkZBX+$%=Ymq5E0 zS_AzFBKc@O<9zy}7-5~;+-g-l?nqTeM5>~N#$KX@Cb#>%Z&h5cI1j4a9>>dzb-o~(f+|U}PI=}Ei097xE?P{Q*agRKV@#17t zm+=Dr14DusK$b!c=#HlX=T>8#3HoAfB_+JY zg)Zk_rT#1!pW#}U<~{0q_;w_PqmGEgRQi6Ls_d(J-*>I}BMeA2SF}6QU=i(f3w%05 zSn^BwhAdePte_R{qnO4AY%L`0$aVTXrU!qlnRQ5^6rOzo5xtpBb)HwE?v62jPxE(v zpm|#`^@j}Pjg`PGu_JE$We=#!8d(lfR2~g4W0HiAHN81-6I&Jb-q<BTX|AXWns`MehbKSDTTlOjQ|+|F#Ef@S9>Q8tcS z@mTnOuZwXgr$53lr#LTY-p^eR=R@N@Zv#$(sIG}ksqZqoFz=!{Y? zJhjuI%GtY7KdDNrOImu-!yP3k!Ah9sZfO|Qfb%_?hcu%1l*0WiNZB6kK^tJdTy%F| zOhQy2oQYO+JNNMTX#ms7M3*IXl|L z#^rB;CMHo2I($yVGa2YY4|d}qi83uN{G+0BtzfZdlyK7{JeV|Vdcc$Fsy>8c zXG(Aid|G7XJ^_B1tM_byGt#4-a6SC>MGQ+9zy*Ax25(Xlur7a&>+-Ie)u1p^Kq!jA zSV!&A0E+i3RyW>NT}$IgILQNuIItwnhbf_Z5c}YSd40=Roast-VbTC@JioQq@YU4K z&r;Dto$mxNdF?w)(fjtnR?{$8qt*wZ&sl^%2-t>UtkL?Qk+osiHr4td^npSUH=y4m z^yw>uv-+dGwNPK|rrd`h?VF107!IKOE0XyDFdxtdnjVIF<@7?_g?{K-aR*$GrW=z&gXcpZPE9|eBy2@L6qc)b4OewN zw4H$yUa(cH*mI_ksMPGl!UXU>{d97jmT+!Tu;`}NNeM+a!NSf%)+n5jRP;0mm=^1* zxrptB9T6xSiziM4(|UA@x+%e&n~X`p#W%HT-j5W0Zmg^NCFosEIEO=g=El#0NW6&% z?G{%u<|StkKJQlceuZF$IV4rIGh@{D?U;1I1zvD1|A=byQz=yqifxKX z*OdEm3IdbjOcH~E)(DHxJTb27PuU)babdh_)%MQ)pR@Voy2+uyIEOD-(&|en zF*uWgD%t-dou4b}s@3^5tMk`u=dybhl?j*N3~MlV#VZyUnJ%-7ej2o9OxvLN;{$&*^SQ zVR8%m{f1qi?shwZw7?J4;Nlc$U~Hl4|37?+ezM2)*lJy`Q;iKD0C|4V0D| zoDu_#VtBm^Jp}{I9nj|vG)2UkZr%<6)`V1*$HFg>6j^ByY~X$u+(F$T2sI4aUWax3 znFZ3mEbG`P=&+6{;w54otpJUy8f0qYC=GEAiZTqmiN*_$g!zi{!W!QqG6?&ih2CLV zXg+wE0SYa2&bcl!jId;Q1I#HIOqnMOiYUWxw#wATJ(S^-Z-L6knKE1`54%FXDAqbsNmd7n}$?!IxJfl>~%@=Q--JOJLVvJUrldc=cB?98jg8us>`^ ze}Usv^g@h)Tyw!Rj~gTCLVu@$`a%lUEY4ufB8)W)EedNEbsPqL`Yv@Z29tdp2DkZ~ ztz31vAyL(DK+v-ohN>@ipf6y(0;?`XiSW2Z4y?K?PQ=24lrYRA$X7G1{n$<)U>FdX*5Iz_+%2mC2~`Pg^` zDMvY!2}hG8M^x z&@#Dhiu5hB;ag_Iw|ooV@<}H}wjr<(D{QcrD9zbSl_8kZ^j!Fs*`{x4wtP!N@+r%= zEQD`q$LM$*qhroRQlaP2%|2509*0Y(LhF6b9f8ZI1P8c%`T)n2V742DGIj}&K?Zy&Nwu6Hs6{>ho3+ECU9N}CNE$3p!a>BWwwZ@JxeanchzD0#^*#d|A zj_Uo&RdcwrLWO(5(G@K?2R`OYELN$wu{e*;=Bhr(ts}g`Jk6+0P!P3Y*Xb`H(rd%5 zslkU@u`8tprx;qok%)f#DEe(M)Z1XFt6&pSzfdm1TPoj+FK=di-!M+_0JMC{Y zA1yQymf8kOHOBfv`J+U;OzPCMr!7%EnNx5H3(ltb{rVoeR}XT2wf2E?eL2jH~7Wzg|CVsRM^YQgBynjX!A zHKS1z4b0&>kR1G`La&^3Ay8`;J{u?9sjPVFQVY7-qoF#V_tQX@FX4+0pMgN-hw5p> zXYZ)>7chxBf2%Lyq>7_HoLgW@Etcx4UWdcsysiOW{j2iN7*c@!hnn63-kg6A}_^E%$+c{5%?Mnbg}S^ zSoDHd677q|4c{uP$2L~166r8j{6xH%V@15kZyf(SohBlQztVd$-Z(E_5URqF1#S^u z*C-UO^&@@?^oVfm9}2vvi19w^X8h6OQ)2PSFiRU{BnT9~H=%-X1g!5=1mEzqH@pGw zvlUCrjSgcZQn&okNJUa%uHgxy8mL`3n{S*qoP8bso^37Hzy&J~HENg63>}4Wcn21qn zWcVuUn?Y-s?r#i)R{b|wnE=keM0q^|>a9P=xVXnyh|xQ;zWl)7to0Mb$``ok zl+SgLZ#5lZrhc-c!%Rg&_7?QF$}opp^g#?2z+mATQxbFiG(qx{>jU_@8Y`#XlQ^ft zaRfU$SV`A{H>(NQR(ueM>9I<}yJ8=z{w(&QwzwX?774YW4<396CP|zMsopx*!y)=N zy@6)Kz4k|1IG%Vf%*Q6?Srw}|-lI$6H1BEGiY7WY{RG&m-nU(=>e=T)^U+nY?BLza z^t-s~gY*Lx@7Jys>ySW6_*(V;)3pi*K+rJT)PxVQZE`EzrE^=F*w9h-wsHG}D-w0{ zC!!8Yy@7AGwgR6S95I|*3i^mKn86O`;4mg;a!?!2fOZT6+Tjdn#~@zWf&pho`EK+d z=@>()VoNNJBlo0WdLmqW`l!}P!R*8~Y_^PQql&`4I3w?nX&_|4h zOi7e!G-XPnOrt4N5@i|!39*hk>#|FmV)( zvTw-{(J1tMAfa8v))3ofpKDbkuxbc4A=>5*vu*Zk-hFJF_gpL1i?(@B^&WDq!l@&) z>sBpc50-UqRjZD)YR;|6JJr2yW~6{HD|sJRcP45$*T^(j^4xoFaxj|hgFy!Elg;+Q zF4yE>4%;V(?UT>;$+z0)4eCPfYpG&8YnN-qipLdy(C8)c6{8)?JQM%tLRFiu5u zBNul6>~ihwRxY*?Lnov8 zuqSD>_;+~>Ky#kZyjbKA`vVCdChwEmyrD^mvCbgI3S+q}aD@wokDWqpIPY!265M(u|jgCeGhz0 z#gD`uFr;!KyikJ!Zc+6Cqt&2x3;Yhk0CaPT&aU2ak=f@}=eLUfAiRfx^#=3HZx~*U z*v-}(5Z#Lbg_V1Xt%b^+l07#x4V62K${j=Hv@2cg$7fKvH##a8)||VR;tIR-H>_N=j305x7jZZ$hkaVCuR5E^b3kIIJ^-sx zS|axgc4*AAT!QxWN zaZpdV?!jlH{rh^1LL?(;A|_wkjjJ|TEW_lXSwzHM+l>paSj0C;yop3>OY&%hR!iCs zMB#VLL^c+_OO!AQrPd?I0M7~6&-dZ;)Gfvfk!)%Ovb_)Wd`D)4Gbs}~$^0sk367~0 z>)r^D6c+u2MN~-4j1c2Ue1Jqt=3x=qw}qhjMd4j!#%Hr#Wj_6~rH04Kku6?i>!Rl@ z$oyL{?a*_SB~}*2dL_bR8H@e|Y8MrnLE_^PVm^u6Nwj3XKSEPTdjYhxU>BKj*t@ID z6C&A;+=pxdWTR14HN^0@8p8iXbJ^{6n&_RbXY0d*tByqJJMKadu@pVm+7Gtq%Z~8h zF|Q8~kVzuvsjU5n>DV~tbO~n$_pWgxMEF*0Hgn2~H9QQR!5GDrv3t`ce8&b_JP7e5 z82kPLC0Dg!QE&xmc}0ltuz1rV>Tz7fX?O)&WX9nRnBVpoZve&|o=NM(E7k;#8?TDj zM!r5TGN7w6Ru83Al;KBee0r--KZBn>nXcl%J^8Bma7gg;LQ&9CY*kyIzz;;w3MldJ z$PZjt8UB-#To1wAn4XLgGo)d91W)(xPk>ECzBJ4u#>ua=ehzP;5#)i3Bhc04h~A*5hzJ@;7LaD?}KzurPzqdbtJUR&=c2AnNuiHUk(bIe5j z#m)Q(YJ$FYGe3HopjU6^7f$2x+0^Lq`ry~OdW}kG-geij61Y#UjA2C{a&mRb%|#~G zW3eaNvBYC(oW-!ZuGjna()+GZ+UwLjq)@!u7r)Pq-LIhq9+eA==W*i*o-}~>_+;xF zd+>&pqfYIKW4KF8Xp1Sucx<|rwioI*v=nNMtz{pn1_n6S>)Wg9K8hR@jQm!SB9;uk z0o%|9PmBPn#WQu~0R?RmXQE>9G@hnts0IN|Z{>#-%)-M_nEQ)Et88r6;*~$s4(qV? zwcQxWcfE>s%0nc~V*DC7eL{2ub^>NUu;>+l#R0)BO7L%@=hTynNye%`jFClzS;X7_ zBZ?4~YEN?m(%@4rJL*X!wUs1T^|qHL`q4CCf|v4p7FJeCSR<8#p+6Lj)1g4+p(Rj1 zZqv&T(7_(X@*&kT)dGu)KlBpAu8fC-G( z(_e`(_JW+&zAdeNd+^e9Y`gre-!+KNwS59t8;kQ(d8-Hb4RgIW|w&2X7 znTSg?h-i-+Z7=YrSXyy>N#gJwT#gSfx{n* zRUj;8!N8~$-2pY4n?BvD|iM^fJ(72E1u-ud;JT|=| zWk{q96_g<~u0e|U7m{C=afj#kED#r`uV7+0PIG=o%JXwcH(E7*2~_ZuNT7Z|Rkc{aAEAc#N9+Fc74^8YRAS3H`ziuIQ@Lf#By+1ixVK00_jg&N!aQ z-@X$HO*oStbc_!TK%YNbkpXSc+|Y#4w<2DsxC}3gsB~s)xsC_3LIV~>v-@8* za_?A_0dD+?y>9;Xmbd=JfL_)6_)~|n#P+(C11@dHqT;|Ky4&IHRsL?bmCJz|=nm82 zE>d3=;~C}M7VUJjZ>6WLxr}xXlM~fIdxt?bPC~opZ`IVXrlB`T!H-j+-3+30~WxB)x7vgAv|3*2PrN_Okv)q52?<#IsGTip;VoX!b&Ml z=U|bl+OwJ|y<5tYBWuM&6nq0tbomg4pTIIf6@K-CnJceD_d%{F0JPj{%VKe|GQea7 z=g?TLZ{d3@YQW4$!30+%A|ot}G%42UT|@&{)8HjRDu5Z6BB8M!iecQqYWck1vcW_; z{UG)-VYmu@se=i5u}>Z(cN|T>3Vneel7P<+^2wosk(mGk646EUgJ4;?gZYSOIkn8s z?gQLPyyMust9lX(0)hqc+01985d3o3T2B*D#_2!ELI|i;4WbY*sKiEzOktH^bzl%J zzOL#fwky^=mO)hJX3pc`mj!mcXB~KQcLvi8@e51Bfv-S;mMJic)!*UB{dYn}%XD7< zFACPl8ohs%9b^$V^`U)H@dVR7${-m#2g2jlB5C-wb4X_&i*`|?D4>-!|GNo1M_?Vh zvsLsfLMIsejtMNi|4i`n1+RnmQiCPf7BM}S&XkttviutPl!2*|@S?Nhu|oG`YLvl` zOt0iM;k3ZwH%X$-6j&Rh1FVeEO52D+m!R9nO4<9U9_!8&#!Uo}Tx1gqX@57##2B%H zEcHVhuO%xh1WRTu>>(Jd6A8g8O+Lb!VwiMU14mK&=_a$$Mbi#^2ByH@N4HF&i~{U| zd4)fQA^Ffc;lF_#F>qKctnzcCuaI$#?7YyxYDxjuA*4a?5Hg4Q!a=PP4lO*Ib}5Rc z$*}#x%PbUBA-W*;$?{=KA{scyrUPLm8KWVZDA+a#X4FGwaxzwmBrwH2;>GOG=*N?k zkf}YG$`+9vAuWCtf;^V9@5muhcpkVi3U6ehMn-|f`v$NSKeXSSt26#bs9BcL^0MU( z!Y8dhg|dX>LWO>92{}}}NFi9E9W3-D3vFG;s+zil4oL~$YU8yK2yP-yq)?YwQ6CDS z6Cq(33D4S^QH)tT(Ej1UVmndBoOLYn1gh9_xZ6SYU8shG>~GtU@sIf9s=-%=4sK8% z7r~5z%{1c~A?{jf?D&rEG>**R#QKMkQ`j@X2Il}mV{hO`1u$Ut0UpV~f(jfl8tfLf zfinjXDiP^4gFQH2{mSpD3@oe zN(f@F+^^^-jXuyQC%U_zZf}1L%5-evgs%FqNaD)CaguGu<-)$qIN(#XZ$|Ye2<1X4 zzq%V*ATMJGH7MSC3qvCAE?ORbIMC{>)=$Ecb}MO|Z_G#eo$D9buY3R^;rkD3J#pG} ztcqhH1s~hJ#kDJZfPyA>RX+!LeL;NFBR-wmu6i3?%hv&iWBsc0BiCPxvH$hK{M$7B z9O|hie83$5{K{rXNPwdQ*izIF`Jzg23PA~;paA2qb1;Eq@)}3 zK+~|LhO&7EfuC9=yrbglA|uJH9Xn-{`5ioNTU+Uj>5LlF&BRX z857hqIOyQ>p4WmN558#lh{sCs-5|ap9H4C93cG-Td}RCxZGx4&7HnG_uKWtefA9?e zgoX{d{#prt_dDvB=Y@`898(i;h#s2e~FQ_=sn{FZHKtq*tIKacEZzW`?Zp2zxr z>bEA$uXwN;bA6ZnZFYw^=u8G0pK@^~-?p>e@q6Q4zR$u+E+$ymrjk`iK`EFoq}?43 zN-@J#;@PoY-8p^lNcy{Edhz{}`Tc_UdLb1vm$pDM#l>Q316NkDGj^Wm3+F)XIOBrA z=nJkUq}#to==3eZ@4*}0G+xXaJHA;^g8X83>QiRt_&Mxob>!F3tG>YY0Xi*D@v;lR zcJTot&Q+wks^6sB3{J!uJ$#~NMH=|X@{sUTqU**RXIUtO8pThrc&Z~+3ogXicMh9! z8cqZ7U_Li@|A%HMJX>h+TX<(zfW}p*cef9NKWRW@?vEb`HZ!q)db&dqVmXD z`L3)v8B3)@-`1+p06Q8eReICFd*Kx*Z&mQk>cEX(a6R6`q!>^M)YIzPlt zabQp+XMQB-H#koX9~^HSfhaAT6*D%8H}(P!PZY-$Fl?gkZVs_;pmQlfIJsHk1@%*c z0aPRzt!NB;5=4e1Ct=bp8>!5=uy#QrBP>>0EQWMu(TW0%mM2UWFM~z6TsSUr;l`(s z%@Sn1#pOLDByUrY=gH*{7MCJ&xdB|HEV<-@&l2&495<0Sg1~DCt)HJ4pQ7b^_p6v> zxh9^~oM$!f+4Ac<(rMnd1(>o!ANa0^vk4q=g~7E~4#C1{%Lr1`e+ZU>X-66yt~$?7 zVf+>XiPdkk4x5>-7^Z>^C@45El$$JZe8(o3=p^v4Y@7fqgi6MX4uWsQ(6r8I$jxN`RjOcQdLNetz0Qn%~T-X;* zIu6FpSdPc-s?HPUoI^QtDCgn8Krt(Ys5uaI=Wi$~-ssa<0CKZ4LsF4|43*#*ae|jj zyypYyN&>nlJfjlIxCna1M+3$95z*F7=xy5N$;VjF)FePQi;20|G z661l#S-Q~5aGa^fO!-dqyYaE%kG=(g#tkhd(vqX#vy~t$`3chjq2laRg($?xCmULF zf*>MQxp<+#sPI^F8D*3pm7HxE$;gFX4c8z^PH4gSk;Uax7^Ns{vsujfc+Xk}J*G4m}zKCXpk!aft$h+!Gy@vXYW_)0N;nC={a? z{-q8qg0Aip<=BXI1V`i9(th)&(}Vs!ri^mD>c9i>R^yqHy2dkQO`yEgR!YhrTJgXF zV&mC|LQBtFXvg!hGCnVS_B#QekL^7#8!W%PV_)N!IUhHEaL*27DtrsdkQu(dATuB% zGc-K&VE^h*o959QHbkq-zFi7|%4@}sU^kwN?L7=~HMT7<9zi2Dp6g%Jgz}ANpoxs& zunhf+wBYU0L>~lN*C(S$Y0KdnhwXe=Hu=(L+z*Z`!g?;1-uYcxhVI z7g1?dZ+@HB-jJamt%P2VTBHSnACu8b3Dd)w3MkN-%2H4d33g!&5DlyYs!#E35NK9K?Gl=jI8cIU}Br%xX2UwJN2uCUZswVk!a(F z`ih~GQR~>6GE}hf+&w$fs@lJGJ#v8+ui751=5230e@9*8`TjLoWM9=5>ss|tf9Ubt z<@IS*pSP-c?eVz;9?Dja&v|y02k`|<=#D5V49kG&zC74i9ri{uLmSSs z;LOmf^R?ia8Cr9mqT#I8S=?vP^nEYz`xI|mc}q)Qd$qYnShE@m!Xv96iqdOhuNIFc z3@W6J6^|?pKLhr6no5-Gku6x8_2r$%uP(&6R^^}zsUh*(oj9^JL~lBM5GU2^jb1ox zpV1V`gm*mST6Hs0W#IQ7>e3VTrBxlhDDCv24CetJ$X~(pPiUrra}hLW1glqYqbMV| zb_KV8(yE#tN~`+#p|s$t6=HMe$U_tG4MElC4>h*rRKF$qR7=jP^P=&C8w6r;>#*Gh6 z+zne(;!Z&0*T#Jt)i^z88}2<&NA&?3HeOa8sO1aPEGqzx_0;tmsl;bcGf}vfhlml< zUq9$YQ~I<)rd=RSwGg%O$e<0P9*t)Pnfd^`-Hl%t@1&4Tjp5>b6nPJ!y;>elB}34> z9a`RT8n_TQZ(!VR*yOmlkJxIf4zShSA9rmCMQ9AsAc;~V4JP^1r?L0CPzI9{p`)UvNvZNbE~$+djMx7gGU z%AOq*J)7QkJiKikf@0V5FW{zpdDzdxLd4(M@m2oQ86rJGfra$xaAJS(IUJKhBOZqf zKEQG^L)D^+lksGWOb)FThB`U4O7xA%NXPEDIz71gb$U3AJP)M@H@r?Sm{#@KLz9Cq z3YGix8_wW9hu>9wU3$h(A8L4JUXEGY`Um<)pPd*QJB;~>mp)^1P4%UEelM zAAx)9SurG>DX-RoqC#fvwVpB->{I_= zcW(n9RdMwJ-z9e;K*(-1*jPaq4H^W}4G9ngF$-BpG+;DXiKZ-JvytHD!|aBF4TvUC z)?^`1pU~PGY|zl!K4{YxTG|Gi$Zq%$Au4SVQ6r)xf*KVqAdvTe=3{rW*#%Kw-{0@O zzhuwcbI+VPGjrz5nU6ack10kf#qc$)KnqCT5XyVOlQ?4)_ z-ceU=@wI{faCwj_Q{C4ytp7y|U~Q`xih72%FAq%2e_>)q#+@f+MX!NkK%m8IqyVzO zn+mx6o5>|oM7tMlinRceiV5nSA{#g|!6o*PcuXo1kFyI-HCv`y#%iGzz5qI>|AF6( z56r-Y;~H8%d5-nNIb+Y?;f{SCTK6HzTH|~?5;~Mo)517sxTfu<1Cgd4D1YxnB=u|q z;Spn;f{C(>D(0D?cmF>5Q#Gsg(C8ZG6Q;X1Q-5b_!LSI*fR~WY$2@1m;|!O(5x_k| z3qJvYP_EdP$I+}utP{-j$BkRv&HX~t{ zhcd#ih}oD;_bXy-b@bEheQCh~kLlLDy|{Dh@d)!R7300<^~_K7zH{#z-cMp*^PY<_ zci?ArGnh~x`>qCeo3n9u!VKaals)o3ng937y&odHv4e1DVqf;Y6Z;;hcM!Gb-HNz> z!94va?sMX>f%pMdbjTFp@p{kc=iokOwUHhoXB~hhHY%Cnj-s7IsF!Wef_sL0L-Tpi zz&W0=j&|uB-A+%dc96?_9+ivnKZYr*aK|R;xZ!%Z)ZIa`9xh$o0|-M<2Lc#^G6Jq0 zYXR_)!F1P|N8h=P1_qH;-I}UBt-WnkXEeI%UD)u58ZEBFRx(JvO;6csOLBaIR-uHo@Rp9^vT?&)wETIhvEZqS^a$?uKl-Ij67a@U#t^54HJ;zU4huW-sTi zuA^t?V=C&r-uWMZXh$<4yt$i=yf(q%{QM(GezOq|Jl64Xn<+0O!iEkaefSZUInTSJ z?_ky1>K&iE5gF_C{wufJ`=i{=WfZS%^!$UAaMOFjao@n$?Obv7hZ?WTaX!*}#_?vP z$23&`A5Ysubnnm~@>tJY)vdYRRr_8u7(&YDC`F7Ju0DjCJ#$q%o^5@PTyqb6vX?a8 zO^!Drz3(`_jP(A_aT?sL`jZ~hSp8?7O+V2$yvIfg=VIn`c#!OI4QP%;dS7>(i==dr zqx{EkMB9 zv-)JsIje^0g0%Vce&H@d*3{>yUF&Mco}=98Sz}MHVOF5VO=CSV-l|zm!#pwSVy&O@ zZUipfZI8rweGQ&+Q-X1wR1%uLXK7Vyjj#rwtktqd=k@j4X4nE? z8ni~{P3oJpWv~RmY}NLRpg_#(9t`^c-8B3I+>mW&aHF2X&1D>JGZI9TZzec#!SK;$ z7!K3an`fyo9}9MS+V08g#`6swoc_5wKr`CE5vc{Y|JBB`us$eW=4|^+fF9Tr8KD14 zCV;4eO#tx&-))S)NS8Hs48WoJZjMK$iH^8Jh-jX*pLbBk$#xLed*(k!8E19_io|G6+b&vk$Sx%K-X8`%->FQv`kR)LJh2QVb3iPsTu zxIur48;Q1jG7VtX@;O3;p&6yYe;im6YqrO>k?<*Ljq#RM`P=k=m?F^K{K zVFF-h+gOD|!a3GsXyCqyo0<)z6;w>s?=8k--%L(KC%Pmk4NZ)}=CEn98a9Cja}9Y^JNBoxwG2UV#H|5ptC8t1$PwSa*I49(^_YV zJ@rcz^Go znYtHJvAGIwXHNT1=Flk_uIZV(sN+`fU>P=1^CuauGkEE3Kt|Q^h|Hmx*wZ+~bX z481Fky~_StXQsy^~s;+h$ZmN!+!y%s5S1}Xi{p904-ZM*E zalip)tW?j=GgoCA6b3RdLd9JfyK~X2kqj}IKTY6IBl(??-|6{XD!+^4cWLa-#JQRT z*KGc58K<1j?p&*68kq|@VyzFcm?P>W!oy5WKP|ydu$Rpf36oXIhtYJ zGY4-tPkp_75@z4B#$xkt?9cGuffG>YxYU?AuJjlz1gFQ~;D+>=xXfMYF$P4&Y;9Wt zosN14XBkvofOGvQcxTlN|M^0L(z`$1wMY0~ zYJN?xCzv8|kIB+Kvvq$N5i$Gm5m3q~muc0PYDHYe;Mm<6gXiszq#r%~05xy70hRoQ z2@ZcR9d2+ab8zfS=~MTYuicqU)zv);0uo#ZSQlC!x6i1V_6yU{DX!@zzdF*n9PEMX zOk1~FbzEnHE;3ad@!D}yfF=r}SGu8wpiy0ORjA%|j7TMQMHBOL(pDY64pgzO+llHL zMind7-WJ=OxsBh@GdL04%o0P>t63GAAbnv?{&Bw%S+H&Xz`#`@pd~75oRTpm0r{{nmr$X z;r+z%hU#tm7#+;=m5Z}IrYRq{(lZQSKgP&)`O1t0k7>-uufpnTR4qn3b@8vLD_gj2I!*=M7}VM zejBOXvOuQ{7$reE!_!;wEO1N)QY1kJ!&5_Nfi4+HC1F_+B8B;}^l*Lw{XVf(kUXIh z57Y9MX)HtY<0zalk5MkBa9xw6+)e>to;W2jks|KGNJc+`o-Urx!gdNbKPxF)MBMGC zB_>M9KHos}WqB-C8U1eGB57Jh;MsE$qZhH#^hB8*Prn(d^t&*Ne$y5*ypw*{l4+zs zV}l5+Y8HVyj|i+iBxz1c8bu^g7b^m56GWhK8UmS*wx$8h!G~ra8;x!d77*&sZ7Hly ze_L7yTWWSi>CLVzy-Gik^b=*;m9E!Q8^QrNvBH+N!j`W53GC?H`+as4CUVt^AUk@U zFr#W)_ceiLR2$NaUO6~+?cjN9Rs2WdKMMc)J{I%|)8N<|vY_jj1>Ms<7Gfb^-Ae$a zArRZPXg}Jsi?JW=n&t2++!GrR#(p&0e1Mu;+wa4ZO?Z+EE3yhD3ZwA~cF*h9J&my0 z)y78crnc_uP-+poO|P6WeEPQ>9E=r*o-d_>&Zyt(LVOjGxitFX^clW<4IINne{~q&GACr%qw@Ic}$v{|^_D ze+lGo05e(%@1YGsT><-}tb4L*eoN`5BP(;PZ_J4}ik{AXL+`A|;}g=?#v>~%qjMrY zpdN7SEaWBKpP7HscA*~1481Ga6*e;?`e(*_1~&EVBLnb1Loe$i?0RtaYDL)|iCkjZ zbL5)I3=Urhh&w$*_+n0nJy1J3;rzhR%oa;OrwCdpO+OLGWA-PI=g-M;vx) z^mCC2VeMD}ptWNfw=*)m_SeLZraV~m(hZD_C%4C&K7W@tEBAo+$i2rrrfGSv(?rxp zN(@s;_n+|Q z4I}cNj#=GG%xhTTU?sH!ciTH*5^FY43Ai>K0s@h%wN%!iuhw?pT|i)JwW~O$R@=%k z_1ZFy!DI->G-wxcOoO(OW16(t9Mhz&116)YE4!kRgcIyn6L(KrIjxGH#k?e?ewJ3n z&z#n+YlA|Ys@ftmt*>~`VO5-FHNXvTpums$`<1Bep7u6Syn|)&HY|&u({G^Xd|4dS z_3jSji3Q3q=0u>{t$$x&dgP-z`N)Vzd{ujQc^lqjc4v}{?LB=j7G7T|pFPLACv)s8 zx4T;y4=iuHi7A%1c`9dP)JLI6>Kgn+{gkiP;>AY2_td?N2U2&Fbfj!lVFc?Qy?>pn$G5=!h0fgL z{vj5)aoo#qK$Y%lP2MxM9iyoK9x_F=VoRrP%@fR0=!e%QqkwB#*0ELW<58f{AUg_wcsrz-0VT??sYGT0zWXm134Puf-8-I}QtsPmejnp=Us=HR7 z#tr6;JE1gawp{?8QFpyjn?%{6_fS2X?s}s>iPpZ^!U~pf_R^9LCemqLr|rzj*XZX! zh7~Vz_hR9+jh6K0SYN}zn*dlI>@~n(wlE#_E-&F3_l#Oz(!kXgp(Zy|V1!y&&ODp& zbR)wd)byMTiMS?{y$cSp{ghW@|I+)q{j|rVZ*O1gRoV}%^~}6A?iG#4l$CcF)1(c< zs6>%!lT3aKD|bqMD^H%1;>lA|EO~TYBtf-Lf)*|TA_)oFuD1Wn>$1P(&9(3LK%5Tm z;`Z0qdQ8UlqmV$%(QX;WG*hMoDOz~ql%y|lG%sqrl%6l|Ej(pP&zCaSBv0F5`vLDe z_M_hAb}V#FvA^yuYd?mR6WUK8<&m_~P#Lpz7-<<*TvHOM7q3E6N^kCkvtG>`UaG-e zgl*xUEYT-dP#%@VlNU?#JzIu-ZOJW+y#xNEh}#qS+8NtaGVi3L5z0SQTg-Q@u_H*6-R5G zHc<8MCP}M#MIpqSAFDD^AXDsz>9&^D8*yNGu9^=)JB;ne*LvCv?JqL9|ARL4DvA6$ z+X;XGRz=RNJdDM3k{o83H=)zCwC;I#WLzgLA2khQh7GB(lFhW@Y^$>dCOB6T2-@xS zWeei8aETGB=aqqwkSy1}3#1J`G*^6Yw|zI1>zr5LzGtm>d;6}no_8n4?WWXghcOdV z)n>p^Zo0=_{}M){h8q&zIo5DPQo{{N4KpOkd)(Qu{;!f5eT+tpw4TXL4cwzN*zvx` zDAS#?Cd0LjEK-KMo*D2A*9HYWa+d2YVVp9?_Mnd&eqAECxm%dA$E+ixao4gw0V^KN zrhx-B1L8ytF~eQGnyf*cS=ee?PseT~W`JoY(ySWZy3<|i<7WD=sb+?pb}Eta+ykQ2Em!x<)sC= zsYuW@p<^BsMfqDzmRhdP6Z#6E(_JsJ)%h1;)Q9zsLl-3pcOfmB9C8&FDcMuRpDW4+X=f9*E+oV0_N-uVhU?rXOlNmfp zuo8>=qtHs3-GbJ0GmbzDiw{P^TtcEV-Xi-;WVDZg-840LD{2yLuF z`t79Ag43CjElh>8tp6s-VQoJoHyAsgkb2&lRLbhxBm(ajk#apt8QW>Nph@;msc+c7 zmV{!gAQW#5-bd|cz4^soQkrLk3?Y(6ny`Us?H#bL6XOn3Q>bROLcPmsJya9^C118* zhIw^zc%;ggPn=_I#Rn7aa7V4drV$pa#`!2M+t=_Ca!q@Tj&hm0v-~&~?a^ib@n*cL z^(sQcc2TI++jZ}&nN{cEx11aTGyZPTs2BB_?g}_(%SRnHnA~>`_Pm=v>O~{Dq(*X! zu(Xs7h}E4QbmUkEhF=R84UMW5K(sx)RU{e{Un-JruHUhBlh=_x+Uq6~!g0~VYbrhjeznPe0Jfp83O^upDwWGcD4cbF>m4Wp^)6Rv4A888ZM z8O=RK{Wj((k6SSEzA-dQGK2OQz&p-n{u4F`@FmuJ>VVux^Av!a>o}6F0z(D(#)Hb3 z&CP@dG!ooP+YF{*n?WVtsX*Hd(!A%e)I!1^y7!p(uzru{-G@dU)Ki@vfsxN2bn*-u|6FSA-gookTApg%bLpoY|#Ihk( z9hM%;u|1(u&*}zCkLB2&fa2jviR}roxoj%~M%X)yl(~Jm8>mRYgLWuHVTZ!?%i~%$ zQ#s?AnN(mKrQ`Lm9SYcy;K8~i*9p$y9Cj$2)3aS70Xq~bF_b|~cKZtg(X(-w~%3P`wVvlvdXZ4*+v%vsz zEPzv_(=i+QU_H>Lx##J7vDX~5Y}oFScCA4U*c-IMt~JR4dy^cnZ;=D`Epou#A_wd( ze89d81OIGFdl?4)+^&fM`^GvU5^={F{aTOd`kNQ1q{U@;;u!@yHNFFuj_g#!_#V6t z$86X5I$_zij5o5OV5Pogd>L6b#Hb#xC(DLV?f7gsJl=<)>?j;>!vc)vV>WhBn&+^A ziNoiZ4A`P%r29&Yqd9c<@aJwL4(|j0xokjB+szBG?vRyt3LLiSfx#E1rh7lozv0Dh zg)+n-({&VPEO*ogFk^R(dX>zWyBX6$1nU5)%Wfq@wmKVkCl>BEe ztquc1?a83AVj(od)Fb(HSJ!A^)o4RSr`S+2n>JJkeK27c6s%eY6^`Gkv1;*IHI@*p z8bMgKY^p?9HL5(aYT3L%SOGD^2IOWV5urXA5w~bNnu*YZ%~<@|jxq+hQ7gkUZq$$p znbr+Pq9&sT1U6XQ?k3e>My;9R38U77Fj<>gUeEL9Kc^>7Wadc0=muhJAokvZ`Oi`J zO?ZN0Lk0FyL}C5HaRKJ$VeF%Tajf6Llm}ZaRPO-&>tOI4NfW%GLKr-@&5F$42Oioy zp)$L^gNz>z;-ftlD($g=O}(D)u|PWa(T0jed_zSu;~cd0ii^Fv4$zEq_G{UO3N!~M zKeR=U2f{#k&tXFa=_RtnH&pnhC2Ka2R$yPWzIyW(Joe2~$_*6?N)HAuk^wfEj13hu z4;`?fVo&!kPeKYT^*;7F6{db-iGdu)?ngf*jB3lXxU%! zg2{AqO`(d(%TI3 z_nK>J*|?C#hkQ@=JK}AXt!$nT<2ktTIH*+nZ?xgrZuM#!CE|WqJMMAXYxOCT<uvN__ zSk-F8D?_@+=A)a(3w^qEZ)<4+a;}Y!<~asTek% z#`2E!)k0lttABk)H)ezfW#Lf<--^3Jxu$MKP?Fm9bnhd7%$DU$>SQU2i)c)T_&QDJ>-Kv1tjEOfDr{9h=4~ zhXP3tQ{vUJtEN*Sfo|1oCJr#!5a-bs=>bkL0!2d+vps=U$Q7zrNUfB8!>&qE#2i)S% zGJ5ww{NXD`l0^Dp1k+tj@g2|}9Hg^lKUJ8n%YB)CJoNJ>{T!s959#Mvp>F<# zLbGoEZu&V)KPPAE#|~+ zXSqkj^DI%pWZ`%*DvCJ~hvr0d&@si8Cn5QB7?!RmWq)1@k*^? zy5-}Ff$6%0lPC0PtIp4Lo-(cKo$LGvT)RJ=t<$G`K)>-Pz&!VT1m~xmq~P11nsvDy z6#5*Y`G@J}6yZJxP0ZF4lyk87H{CUyq&vLaoP~0z!K){iYS<*RXmiXxiP{iM}CW%kB;ax9;v_uw{1-Aatf_FA*THWiL@6uyrqyAkeZ`&>X?dzwo~w z|M=b)7R;vZD}M$zhw;A~|F0wcLf!D4sQyC;;xH>mYK9eWmvfEDD0-RXg=F#cncRpr z)WiJhPD+@Cj#3EYB*ZH}h8Y3pK++*IY>kIHF?slbDG&Q3kDb_rJ(4*nHf)XDj((4f zX!qL)z`AsA|B`ovVt)ah7xTmIFeOUH-|?f6Q;Wl=QRQ0A)rwN6D``z`MeD(9%!OIH zlD#;*{WFv9{a3(qk4cx%TByt0TWG==>0LMHLe`luVkgnEF6=Fwsrz2~Nt`|J9gz9x z`Bh+BP6k$cB2H1*`^jBr?q^4N$5h8Fu#8xLVDEn6eh7zY2Hm#hNVZ=O%Yvyr%2RJl>B}C!YKT z8hL!3`+nr_<5mziB8V1`!kr_YmqUtJN8Oie#8CnXxx8BU3fJKMxpw83eBqF7_8@|Y(;7WTL3YG+(6H< z?#JjpL&uV`1rHFYPkEM6R;M(J7~28qvqJbdlJ2p4VVLl(Qr)^qs6JE8BWl%oJw&LF zUyB0L)mCDz<1^jO$8`yNT_1d=Td)t?&TuBCb+4{{56+>fW`(Td`%TbyZ>7F_YhCrV z=*Vlg@;*FoEA`!58wJt;BS)0fcW-SLNZyC%ZA~L&t3akv7f{uXNzPBopTRDlsolEA zXr~rU?9)dR_hwX`!Isf~;8hVR;(jLL`{(F`WY#ZWQ|p-wkTuI)TTgOjxzp?QFx?}b zpr2pjC+b(3RXf$Jsc)5cAtgr=cpU(nSF+sIEH?Z*789oj)$3jApIqu+p~f+>&GZEX zh<1{7e+M8K-e6DeZTkotY)lcS_*YY2xqy!?Siiu!?NU36wKkrW4KJ_H;6KDtHNtXH zk;m?174xEzsxQh|e=V|=yVq46)rf0aGuO|1RB~ zewzlf!SEc8>ejpnXAIN9nMJ!B!G%PK>TzC+7Ybw3DE)&CNR!kzJqp#1WJx|GLd9lc zts0wD@f{l7Iyx_!Y$-L4O!p5l7{K{5&~qF*wjHWsMoSrn)eIdJ-}K)%%n?U2tuN8Y z1BbXZQ;|%$oJo6I6Y9rLOj`kn=P2Q?)ATJ3g3(j8V6xQb}VvoaGQr zN^g&fHMPJwq4ouTq#hYZz23IAWJe>kJ23`#M&6n}-fj9tf;{PB? z1^6NUk5P|`|C2M|yFA?lg_wnQ`t(*FXn5Mg16!Uxh(IPrVEE9H?y+*1h^Y;%UUZLr zz#3pHJ97>>$Qav!&RtWXrM{yWdpp$1&rPm_x4XCa%9)qJT(=z+;VTGqwCA85Xs49l z47YtX%k&%yP|i21l+TSS&lPe*5^p4lsj<$PdZheye80$H;T%~bf`>!OOPD5E_ruWM z@hQ;=3CyA zPL^raIp$=+)nZN-)KQ;Og>>`f9#*YNrAQ|(&r|goRyb%dJkwR+ryrx?`UO zgX+gf9tu240aj-1ujU-YxHm>f~0Oy-dAH)vr*T-H*Tx zbG_I-08fGJE@smZ=_oSM?Q;JM<_=Sd3B#YK>TsQdzg0TyX9$~hIWPKMutRv}h0}Ei z|D7;shfTWqfcL+G@FC%ly$F9ym-ntNcVE|-JFx=}{@51yU@G}!FG)x2AF$rqa`0yx z_qrC&Lw=d(rNAP0ZSvi!xMe%`h<*m>yC(Xc(LsgWG1JjGIWj-AzjfcO2BmB2T$G)A zVk^Sf-@1A{K+Uc1{{R5H2fKIRDT{XxPSeT3p>a4FNt;SU_>u|Ens<*JiyKTcWK^Aq z?Cr>?!AmHodn-Pn$oKErSyx9jhTCvD(Gu)oPG~ZHM{(xmsd1-z@ zQHjG@k1ak1t0xtUp|57=!EM>a4SbLLo>o2k4!e>o@m)^3pomLmhMrKLp$-#qss>;7yzzDg!})&sKH55a54 zN6}Au1>Y<9HsqgeFI|*x&nzi&hLhW}#A?rXICyo4puZ%0ZsSGTbK75jdR~uxsNx=j z8k@7Cyu`9J-(F#JkfJ0ympMUPPFi9sdBC~YFwHR0Xr%aro00FZISfS(!`yqavJCl7 zXhnf-nc)@#o|i7QmFL^-vu9?9CZPCtRpi^l#a{|VD=oj}7Rww{5r!}`)@<2U?A6` zi*ofBmA{U->;d=RN(uq`z!L~WdytnFoqju+f2p=!+A}-E%{-D6NsNw-jK}B&H^TyY zuUJORals{J6=o?eEwD&0D8b<@xaF2b`Hmv1rKk|St}VZKnssr0`8Wf6LP3kOJio~4 zVCd=74L6Q6$k^m#E-EdxkMT2?j}l~n`JtJ2T4rTtnGH7w_32zP{_8A#&{pC|bU?8a zSOb_~7;8wmIl)&h7m4fV!_Gv&Z8+%ne|enX`^Yqo4+RY-MY-5N`O+8rPXq6Z{qr@8 z0T6)Z0Urta$1Pjakk4tPjoDNM^UVsT#Y5673gyr5;|G@urax$)%+l+hH#|^eIRusy zn@C9rH^Z+&k9--%ervN(uqfj87fboDGIKX5c z3a1m$4%h=&O@&q5+uH!x0k{t^tpxOVB`Okd7=?031Iz+! zt?BJeK{<2+<^d|Tkb~~mAs?8%=mD$-Y_10#MsfNLNFQ+FuK{C`4&WBRv__-@d^X@X zocyXh4Ssl!i}ngT0S$n)fJVS3z%syNfU5w{0X71T!@F_Kfa!o8fKI^dXOKU@I_%MN zA|3j!4$YnGUqF2VwgReG0pALKfSqmN2iW87?Ty7NREBoQ4cG~o2iUv|<%8#CZzEs0 zU-drtVb#fa9Qg%QFntw;MZ6Bcd4MB;qD%8|_T7qRnSS7t*WYYVu8A5HNgqJz^E`_Y z40=*S1d;+GE12q0DrffgqIi5)147gA?<73*Nqq)F(cF;|t*;%x199#M9x$v^c_3lE zmfoFLqK#Dpc_4MoAeGcCx^sp~I}V~#+Xcl=4igRt9{UB;mpRn$<1mtwbMYY~T0t{sSs>hcIMPiDum}P@z64-V}i7#a_&)q6UTdhV`T+1^BNN@tYA=9sc<1r2edr;FW8rQaMMXT1gxYLO0?BXT8u3k%F;G zT{mDo^$c9jV@M|pa(3Wey@AqUI;dYWKqy3nx(<(`^+NSz)kLfrpfLRht0RaXsQ-k| z1O8wLyc76+A@H@pdqUuwfNu%H(?J&vz;6Y<2T)y)6h--7Zu?@Dk*%gu9=~Su$_{K@ z>fvvV)6Y>g--=9BA19Gfr>%cIQNL&pfgb^UK@ff%_{5U^0G{c~STuH@n$%Oqed>W5 zWgh<1L4OYPWeBU+5j`t$BeN6h2C`NU!$(E|icDZl1Zx^Q8DnlJ4v>`iwjn)^j%0e* z08A|UchWocHFPq#D4z#`j}mx>#1HAqN#IrB>!2^+P*<|R;ajN}bu9$@b~sghTb!uO zs2oS2J1Btv6Wp#-fX@rUQ$3FZejf0R2&;F(Ns#q?3lO!uX5Cs=r>ONCf!xO`^<_r4 zpWGYL9;)YPBaa(Nax{T&Lr{8TAIa~k2fh_y^#r7p>7@g4F}>o(U1}Gl*G@i0zr=x} z<8VmlP9mK&^mXcbN{1};<@e)E>sM*iFQI>5x5`&(`dFG~669;ztRPmikjrm}6=Y)i zlZP(#)Y88C6OU(3;7~@x4erLobx*gevK; ztr@7^1t~-inIJk~!^NF4W0iKzK<<-Ke(jJywo=%$6XdLOd7FTb0^Yzd{D;c11$gy` zz>_R~S!pR+I)BVJJqEgy*h0UGvLopF`w(g&^)>xm2A&+VvRHtp0&-3W47Wd_xeP%3~k!)hk3f3B9Tsz&un@ACU^todR8D ze>&f07m`nVR;kar*1G#R+_vzMq0=ZwRVqvNusM zK=mQ;@xa#x;i(^*27EQ}X$XhugQ&^(>w}1#(yJ8l0esguglL|n5_}B^tKTD~V``>% z4^SzZ7~oTHCi(}^+l74UXXwOm5uX;|3nZW8z+Z$<0q9SHejeyoA*}8sdS=Fq%y9Yr zw`=iOI&NrW{k$gLs*JTNd_59I(9%817l?Uq4N2ZK%&V!TedOZpX`P%GSugEMU_Z-* zf*VEeOZ)Ap>~V!XjT8NkUX~V@3O*0IT=#%=5n$bp)M}&E1d@@}KYD;z%zvR6eAfts z;(_0X_dtz=6*fg3jW9e4r+9t@&ztG-Sbu(eePiG_oi#L{oorTllM8DnWk=4FZDw|| z7IxBuv>G6n6Z3Xu5MDh1eUP%z_$DfX>>l-L`hf$MD(dwVidL7d>Rh4MsY};(u28>E zs3>)5&)X~X!_=jx-(I0!jnUaYW-VW4zx&zm0j3D;>=!d*yr@|MjCWriL5aj85zGLY z2`EgHa5tOX;fJ+@Y9u=1SzdmGql@aJ0r;)JQ%@o<%6C2RwRqpRmFBIE!X_bhgYAKw0v$*q|0 zObf^#<+~m~M{p2^-0M&Dq=z&Qd+_-nJ-i7Ipodh=Co$-s&nC!s^53$)m^6LnkVQWK2I<4*ll12#_&EO)q(8!r;jz;D zq)9V1Z8fZiOV#ckq&%T&j}B5ctJ=+x%HP%HYa^BITEZOEYCj*Oyu-pDX^HaZ1NR9X z?tQXZqxqetK{N5w!O9n!XEe2gmF5UDZay3g^Z)4(t!K2dVW{?l(aMgY+BXxGT|@t* z!Od@mZPz?|lk#OWJ!#f$*F2t}td60ZzsGFXoJ>&GUr9HfE4OR*k5PU(oNitm{u@p6 z&C2TUT^NYF-x;)z->B>`Xn%jB@|r=r_eSMS7Cz4K|2wLelKC*6(mxxog+M^P-RPPD?~bocP$9VYFjl*+U*S*Wj#gtN1u53hN8H7pZqG}r!<=X)qE0} zd|gZ{A-=|&$@{J=!U$b zQ$A+l&H5i;W9#>|Z+t(Z_xnR!*F^mOnwws^MtSQR2-tPa7zF=!1U=k^-w_e#M-a(R zM^gA7Benk?8F6+bVXCjC@c&UJkqw@0lto}ZbQWe4Q`$vOp#wEE%cS|RT}+p+~1_lP*MT!3w-t~jb^>(I$+jnw110L>IT#f z_;85wzN-D_70P*4`>!jM$F&sxz4nymO`Y=gAg%j~h~GwP+hQVW2WxwD5w8x`0*^ZR z%@xY#A#n)5J46c>zZ(jse;#@}g1?CxcRE@*95n&q#%S%)Xyql+!;ShOztu;)qR*|0 zQ9g>H@VP6tX+RUM%mT|dFdVIk9IIWRJRF#>pK3HKGJ*KLM!Qdq_}KvM-&Ezm0PVlD z5x^!A-qZSG;Qehu_qPY#&kMR=7<6A6biX3#z9#7Y$)Nkq;+~vuy2`LJPZx&2 z>`MKNL)ad6v7>_!LIJZY1FerPbja)~69J4g*!890kC86BX!VXRax3XdxPg}!^+a?v z3PQLQ>>_u8F2P3WNaptvpTHbx>r1zzYIKr*Tf#3YaM1bOGlI zxIn-X0apsRPQa%Gd|tp81l%v+TLOM0-~|DrO$xiN6);i2=>pCbaDjj&0r1zzYIK&k*Sgm?+?M0p|+1K)@0KR|>dJz^4R!UceUw+%Mo;0)8am z1p%YUQ^$3!fQbT57jUkC3j{0?aHW9j1bj-s=LLL0!2JTgCE!N_UJx)kU8FByqJYx{ zoGai00ZRm2Dd0K*pAztS0bdYszkqKE_>q7Y1dKL|^aV^5aJqnV1zaFtiGV8wTqodD z0zNO`3j*#J@GSv967Yh6(X&MQ0wxMLUBI~lE)cLpz?A~76YwbkpBL~20rv~|mVh4# zctODE*&=-b69t?u;9LQTY5*r?zo|EkVsL6j$%7@OKQ5X2jo<9)|8C3F8DJbIsyM7(oY@pUF3gK^3)0V50U=% z?L33h(%=5wq>uKO z8nXSRj{8pXkM@@uvi)uUZqlEG^e2TX|J3o{LH?5`Cr_pSQ1$=+ar&u=-+lh67u2w> zP@~MAIrA1n!tA;Cj4`AnrX)@_OfV)*FeWD@8xrouOA;A)qmkjqr;Jg?7uX8(E9_2t z(V|=MzpP|&sl#aq#4anb5N=XRK_B#G5j&O^+bsn}jsxalT`*l31|31hF`DmMe?~Z#_BiW@Mim z3!atR?D-Tygv#trC6QipR1%#wyo;Jxh#NdBEx;?AN}_GCrLa7|*k)N=fVlDwOs!Un zZJE_p=CnXayG==?mq{HEfrBMhkP{0^58Rk+`Nc(6L@h-s_(^0*^JEs`ozO&UX)(QG z!@a0)_InMY55m9n#rTe8?4{<4$>#b4Q&0nM5T;?X7XLE7oG0)>5g|gvljC*CCk1#K z`pfuo9znuGIv$beiAKhkX-Hp%|Y? zci`4eD+A~b zmwL4yfnXuyN%(P4e1jNA8*UQmhiQL;o$@&X#7idAD*T}4L>|A314*tJd?6k<3lK!Y z9?19;f*uClDzuI+)8-uc@6jJ{%7~x{LANaJpeDu`EKcQ=8Pvhy8j1f({L6Bac8%tX zL#5xG!{cji0ztqPoPNNge!&PxC;QRBhY_$gvVZK0k5FX4*B7rUvOnvK*DA7~>WjxB z%MSuEJRE_ZUiK@2Pa{w&vcKqyAMEQ7`r?Nu(!Te_4^^by?u(C7q&@D7AErn<+ZPX` zEA42&g9t^ZNV^z-M*Zm(Y47^tV-#u6`r@xpq@C)EzfzI*r!PKMk#?moez+p-MPK|? zX#b-EQ7jyR)ePCb10P0UwL-SrzWA#Z*&h4iY1d8Pi}Z2;JgMOFr}m{PpY-;n=6gXj zXnn!uLu+FirGNRPGJOB?p=nP_ca`Y3WjgmWdcXX#-H^YcQLa{Y3p*p*<1)cV$&eg4 z3@3jz;}fG~2d!WH55wyer|1VIw_h`SfBDJ1Ao&&1?@RhN1VWYDdB(@Dx2Hq|vQrw3 zUk}p-ektLR-^D)|#W1oTImGbbBN+Ti(7z@4vBF7xR;n{&UG+;3a(wjHw3t+!#ckEbwxkQ_|ll z@CGseC;2}l@N!;K(z_-9Ao@Q@{sJ%QUy}UA{F>zdq2w>{l727>mdPJPKUUzk2hqf7>KZyR0JQANP z@UcPojRJ27!gmY&IDvm!$g>y*kob25(eD!YBLYvGFzKSbbwuA2M1L#rWXI$_1_KvY znN9St8+KuT1X{U=;fE;(asY9y0-S^P(rAeu6H^r%~X4#L-Ft@rRt}gq%eJ z|7VtNobs%wcPYSWxhtCYauzi1Rvp!D${@uRnGSl zJ$7NN!Y^G0_KR2k1s~PG@nTv;`HjHG2H~Zgavw{uoQ*+p9u6bV`$C>Z4Nq5&dnA9k zKPH%eQxN}ij1T$+QSTyKN?a7>_mkdWzbKL6`}@%u z;5UcCw=#VH@m+fu`pz);FT>ysC^VMepnADS;9F?|6IUzyRg@3ur7UJXNk5_l@Ny49Rc z=J&=}j&CjE_zn(~TLj(^RL*wbLzT}T1pO*OZxDPAgrR>U4F2;l`0vB<#(9=Y|HVM2EB+o(ML&-A^ z{YfbN%rN+S7#{M7bfr9-fG0V#MgPU#&x1^#2>MPp7tkQ&91I6LRJtR9zh2M;f+>uC zD9$Y6Sefq6!tnW{px6I`)6@Hmbe$3S#uxd08h00zM3_!A?S7XDc*<8tQ28%s^l{1- z;U}mfrc2NpMfs-*{NI2NmEV^cpR1L%LH$UV;A0T|ph57_VoD_xpV49PX5d5RD-U?4 zw}QVc&-IKC^tqKM*eufZ2>ND`FN46p$>_yR0Js_sWGFe4!{F^<@NK}8oQl)jF;%L3Vdr&J-UFW{-|yfr<3`8O3>?_9N)-+a)i-~n*cBZ6Su^t zwUr|~gQjQBGIp^;B<7J16{uZK2*JU zfY-ynZ4UB#I|QFPk*?H}t}uL}t_#ioCg4e)x}fsU3q!w6(6fa*$lbxDSClZg%Qz)F7Y4z@0^cn3T-NW6Fv3LNTFvPtel_r+hnzwE%egRou8j{Z&$KZ3M}enwJ4$)F zqWF}Z0?%HG#Um-t1xeq;=~r=}Ts=B;`!WC@syuH2p3+r5^8i9$1PBmf>z< zlEU}w;5gIb(5){AaO!s%A|>w2`nR4i*+&f zuj7jo)(0(#lZ~m0?=iAzwb)D9S?(29CnHImXiV}`P)TqSijHSS|1;ksY%uf1o}3gu z_T;2X#hz%K;^*Hl?Mq2M`67}}zNF+6lO~5xJ86>tS;%}(oG{T(5iI$H6#ui3u_sLq zpZ285mr8rmr0}ISY0{-iZBj~j$tR^;Qt}Dm>wD6KOV;<4@cEuNqvY79_BWIE&`;MGsi?m;t5t+qf9DI?d_`dd#xx}2~nCE9ZH^(`N zA77oB<4mE0$1}5&k^;UjlbuiU<)m25=1Ju?Iwsm&jJr^*3&OLuT$($jG6|_nwAyX? z<>sZn6aqv{mB(8Xu@S`SNuddIE+fHjgNMIpCi#mdK=8|vCn=@x_gBnwr&ui5<7=@l zTb4B0Vkx2X$?rDrFm$sq;