1193 lines
42 KiB
C
1193 lines
42 KiB
C
/*
|
|
* Copyright (c) 2017-2023 Genome Research Ltd.
|
|
* Author(s): James Bonfield
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are met:
|
|
*
|
|
* 1. Redistributions of source code must retain the above copyright notice,
|
|
* this list of conditions and the following disclaimer.
|
|
*
|
|
* 2. Redistributions in binary form must reproduce the above
|
|
* copyright notice, this list of conditions and the following
|
|
* disclaimer in the documentation and/or other materials provided
|
|
* with the distribution.
|
|
*
|
|
* 3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
|
|
* Institute nor the names of its contributors may be used to endorse
|
|
* or promote products derived from this software without specific
|
|
* prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
|
|
* IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
|
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
|
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
|
|
* LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
/*
|
|
* This is an AVX512 implementation of the 32-way interleaved 16-bit rANS.
|
|
* For now it only contains an order-0 implementation. The AVX2 code may
|
|
* be used for order-1.
|
|
*/
|
|
|
|
#include "config.h"
|
|
|
|
#if defined(__x86_64__) && defined(HAVE_AVX512)
|
|
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#include <limits.h>
|
|
#include <x86intrin.h>
|
|
|
|
#include "rANS_word.h"
|
|
#include "rANS_static4x16.h"
|
|
#define ROT32_SIMD
|
|
#include "rANS_static16_int.h"
|
|
#include "varint.h"
|
|
#include "utils.h"
|
|
|
|
#ifndef USE_GATHER
|
|
|
|
// Speds with Downfall mitigation Off/On and Zen4.
|
|
// <------ AVX512 ---------> <------- AVX2 -------->
|
|
// -o4: IntelOff IntelOn AMDZen4
|
|
// gcc7 544/1673 562/1711 448/1818 649/1515 645/1525 875/1624
|
|
// gcc13 557/1672 576/1711 582/1761 630/1623 629/1652 866/1762
|
|
// clang10 541/1547 564/1630 807/1912 620/1456 637/1481 837/1606
|
|
// clang16 533/1431 555/1510 890/1611 629/1370 627/1406 996/1432
|
|
//
|
|
// Zen4 encode is particularly slow with gcc, but AVX2 encode is
|
|
// faster and we use that already.
|
|
static inline __m512i _mm512_i32gather_epi32x(__m512i idx, void *v, int size) {
|
|
uint32_t *b = (uint32_t *)v;
|
|
|
|
#ifndef __clang__
|
|
volatile
|
|
#endif
|
|
int c[16] __attribute__((aligned(32)));
|
|
|
|
//_mm512_store_si512((__m512i *)c, idx); // equivalent, but slower
|
|
_mm256_store_si256((__m256i *)(c), _mm512_castsi512_si256(idx));
|
|
_mm256_store_si256((__m256i *)(c+8), _mm512_extracti64x4_epi64(idx, 1));
|
|
|
|
__m128i x0 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[0]]), b[c[1]], 1);
|
|
__m128i x1 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[2]]), b[c[3]], 1);
|
|
__m128i x2 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[4]]), b[c[5]], 1);
|
|
__m128i x3 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[6]]), b[c[7]], 1);
|
|
|
|
__m128i x01 = _mm_unpacklo_epi64(x0, x1);
|
|
__m128i x23 = _mm_unpacklo_epi64(x2, x3);
|
|
__m256i y0 =_mm256_castsi128_si256(x01);
|
|
|
|
__m128i x4 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[ 8]]), b[c[ 9]], 1);
|
|
__m128i x5 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[10]]), b[c[11]], 1);
|
|
__m128i x6 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[12]]), b[c[13]], 1);
|
|
__m128i x7 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[14]]), b[c[15]], 1);
|
|
|
|
__m128i x45 = _mm_unpacklo_epi64(x4, x5);
|
|
__m128i x67 = _mm_unpacklo_epi64(x6, x7);
|
|
__m256i y1 =_mm256_castsi128_si256(x45);
|
|
|
|
y0 = _mm256_inserti128_si256(y0, x23, 1);
|
|
y1 = _mm256_inserti128_si256(y1, x67, 1);
|
|
|
|
return _mm512_inserti64x4(_mm512_castsi256_si512(y0), y1, 1);
|
|
}
|
|
|
|
// 32-bit indices, 8-bit quantities into 32-bit lanes
|
|
static inline __m512i _mm512_i32gather_epi32x1(__m512i idx, void *v) {
|
|
uint8_t *b = (uint8_t *)v;
|
|
volatile int c[16] __attribute__((aligned(32)));
|
|
|
|
//_mm512_store_si512((__m512i *)c, idx); // equivalent, but slower
|
|
_mm256_store_si256((__m256i *)(c), _mm512_castsi512_si256(idx));
|
|
_mm256_store_si256((__m256i *)(c+8), _mm512_extracti64x4_epi64(idx, 1));
|
|
|
|
return _mm512_set_epi32(b[c[15]], b[c[14]], b[c[13]], b[c[12]],
|
|
b[c[11]], b[c[10]], b[c[9]], b[c[8]],
|
|
b[c[7]], b[c[6]], b[c[5]], b[c[4]],
|
|
b[c[3]], b[c[2]], b[c[1]], b[c[0]]);
|
|
}
|
|
|
|
#else
|
|
// real gathers
|
|
#define _mm512_i32gather_epi32x _mm512_i32gather_epi32
|
|
#endif
|
|
|
|
unsigned char *rans_compress_O0_32x16_avx512(unsigned char *in,
|
|
unsigned int in_size,
|
|
unsigned char *out,
|
|
unsigned int *out_size) {
|
|
unsigned char *cp, *out_end;
|
|
RansEncSymbol syms[256];
|
|
RansState ransN[32] __attribute__((aligned(64)));
|
|
uint8_t* ptr;
|
|
uint32_t F[256+MAGIC] = {0};
|
|
int i, j, tab_size = 0, x, z;
|
|
// -20 for order/size/meta
|
|
uint32_t bound = rans_compress_bound_4x16(in_size,0)-20;
|
|
|
|
if (!out) {
|
|
*out_size = bound;
|
|
out = malloc(*out_size);
|
|
}
|
|
if (!out || bound > *out_size)
|
|
return NULL;
|
|
|
|
// If "out" isn't word aligned, tweak out_end/ptr to ensure it is.
|
|
// We already added more round in bound to allow for this.
|
|
if (((size_t)out)&1)
|
|
bound--;
|
|
ptr = out_end = out + bound;
|
|
|
|
if (in_size == 0)
|
|
goto empty;
|
|
|
|
// Compute statistics
|
|
if (hist8(in, in_size, F) < 0)
|
|
return NULL;
|
|
|
|
// Normalise so frequences sum to power of 2
|
|
uint32_t fsum = in_size;
|
|
uint32_t max_val = round2(fsum);
|
|
if (max_val > TOTFREQ)
|
|
max_val = TOTFREQ;
|
|
|
|
if (normalise_freq(F, fsum, max_val) < 0)
|
|
return NULL;
|
|
fsum=max_val;
|
|
|
|
cp = out;
|
|
cp += encode_freq(cp, F);
|
|
tab_size = cp-out;
|
|
//write(2, out+4, cp-(out+4));
|
|
|
|
if (normalise_freq(F, fsum, TOTFREQ) < 0)
|
|
return NULL;
|
|
|
|
// Encode statistics and build lookup tables for SIMD encoding.
|
|
uint32_t SB[256], SA[256], SD[256], SC[256];
|
|
for (x = j = 0; j < 256; j++) {
|
|
if (F[j]) {
|
|
RansEncSymbolInit(&syms[j], x, F[j], TF_SHIFT);
|
|
SB[j] = syms[j].x_max;
|
|
SA[j] = syms[j].rcp_freq;
|
|
SD[j] = (syms[j].cmpl_freq<<0) | ((syms[j].rcp_shift-32)<<16);
|
|
SC[j] = syms[j].bias;
|
|
x += F[j];
|
|
}
|
|
}
|
|
|
|
for (z = 0; z < 32; z++)
|
|
RansEncInit(&ransN[z]);
|
|
|
|
z = i = in_size&(32-1);
|
|
while (z-- > 0)
|
|
RansEncPutSymbol(&ransN[z], &ptr, &syms[in[in_size-(i-z)]]);
|
|
|
|
#define LOAD512(a,b) \
|
|
__m512i a##1 = _mm512_load_si512((__m512i *)&b[0]); \
|
|
__m512i a##2 = _mm512_load_si512((__m512i *)&b[16]);
|
|
|
|
#define STORE512(a,b) \
|
|
_mm512_store_si512((__m256i *)&b[0], a##1); \
|
|
_mm512_store_si512((__m256i *)&b[16], a##2);
|
|
|
|
LOAD512(Rv, ransN);
|
|
|
|
uint16_t *ptr16 = (uint16_t *)ptr;
|
|
for (i=(in_size &~(32-1)); i>0; i-=32) {
|
|
uint8_t *c = &in[i-32];
|
|
|
|
// GATHER versions
|
|
// Much faster now we have an efficient loadu mechanism in place,
|
|
// BUT...
|
|
// Try this for avx2 variant too? Better way to populate the mm256
|
|
// regs for mix of avx2 and avx512 opcodes.
|
|
__m256i c12 = _mm256_loadu_si256((__m256i const *)c);
|
|
__m512i c1 = _mm512_cvtepu8_epi32(_mm256_extracti128_si256(c12,0));
|
|
__m512i c2 = _mm512_cvtepu8_epi32(_mm256_extracti128_si256(c12,1));
|
|
#define SET512(a,b) \
|
|
__m512i a##1 = _mm512_i32gather_epi32x(c1, b, 4); \
|
|
__m512i a##2 = _mm512_i32gather_epi32x(c2, b, 4)
|
|
|
|
SET512(xmax, SB);
|
|
|
|
uint16_t gt_mask1 = _mm512_cmpgt_epi32_mask(Rv1, xmax1);
|
|
int pc1 = _mm_popcnt_u32(gt_mask1);
|
|
__m512i Rp1 = _mm512_and_si512(Rv1, _mm512_set1_epi32(0xffff));
|
|
__m512i Rp2 = _mm512_and_si512(Rv2, _mm512_set1_epi32(0xffff));
|
|
uint16_t gt_mask2 = _mm512_cmpgt_epi32_mask(Rv2, xmax2);
|
|
SET512(SDv, SD);
|
|
int pc2 = _mm_popcnt_u32(gt_mask2);
|
|
|
|
Rp1 = _mm512_maskz_compress_epi32(gt_mask1, Rp1);
|
|
Rp2 = _mm512_maskz_compress_epi32(gt_mask2, Rp2);
|
|
|
|
_mm512_mask_cvtepi32_storeu_epi16(ptr16-pc2, (1<<pc2)-1, Rp2);
|
|
ptr16 -= pc2;
|
|
_mm512_mask_cvtepi32_storeu_epi16(ptr16-pc1, (1<<pc1)-1, Rp1);
|
|
ptr16 -= pc1;
|
|
|
|
SET512(rfv, SA);
|
|
Rv1 = _mm512_mask_srli_epi32(Rv1, gt_mask1, Rv1, 16);
|
|
Rv2 = _mm512_mask_srli_epi32(Rv2, gt_mask2, Rv2, 16);
|
|
|
|
// interleaved form of this, helps on icc a bit
|
|
//rfv1 = _mm512_mulhi_epu32(Rv1, rfv1);
|
|
//rfv2 = _mm512_mulhi_epu32(Rv2, rfv2);
|
|
|
|
// Alternatives here:
|
|
// SHIFT right/left instead of AND: (very marginally slower)
|
|
// rf1_hm = _mm512_and_epi32(
|
|
// rf1_hm, _mm512_set1_epi64((uint64_t)0xffffffff00000000));
|
|
// vs
|
|
// rf1_hm = _mm512_srli_epi64(rf1_hm, 32);
|
|
// rf1_hm = _mm512_slli_epi64(rf1_hm, 32);
|
|
__m512i rf1_hm = _mm512_mul_epu32(_mm512_srli_epi64(Rv1, 32),
|
|
_mm512_srli_epi64(rfv1, 32));
|
|
__m512i rf2_hm = _mm512_mul_epu32(_mm512_srli_epi64(Rv2, 32),
|
|
_mm512_srli_epi64(rfv2, 32));
|
|
__m512i rf1_lm = _mm512_srli_epi64(_mm512_mul_epu32(Rv1, rfv1), 32);
|
|
__m512i rf2_lm = _mm512_srli_epi64(_mm512_mul_epu32(Rv2, rfv2), 32);
|
|
rf1_hm = _mm512_and_epi32(rf1_hm,
|
|
_mm512_set1_epi64((uint64_t)0xffffffff<<32));
|
|
rf2_hm = _mm512_and_epi32(rf2_hm,
|
|
_mm512_set1_epi64((uint64_t)0xffffffff<<32));
|
|
rfv1 = _mm512_or_epi32(rf1_lm, rf1_hm);
|
|
rfv2 = _mm512_or_epi32(rf2_lm, rf2_hm);
|
|
|
|
// Or a pure masked blend approach, sadly slower.
|
|
// rfv1 = _mm512_mask_blend_epi32(0x5555,
|
|
// _mm512_mul_epu32(
|
|
// _mm512_srli_epi64(Rv1, 32),
|
|
// _mm512_srli_epi64(rfv1, 32)),
|
|
// _mm512_srli_epi64(
|
|
// _mm512_mul_epu32(Rv1, rfv1),
|
|
// 32));
|
|
// rfv2 = _mm512_mask_blend_epi32(0xaaaa,
|
|
// _mm512_srli_epi64(
|
|
// _mm512_mul_epu32(Rv2, rfv2),
|
|
// 32),
|
|
// _mm512_mul_epu32(
|
|
// _mm512_srli_epi64(Rv2, 32),
|
|
// _mm512_srli_epi64(rfv2, 32)));
|
|
|
|
SET512(biasv, SC);
|
|
__m512i shiftv1 = _mm512_srli_epi32(SDv1, 16);
|
|
__m512i shiftv2 = _mm512_srli_epi32(SDv2, 16);
|
|
|
|
__m512i qv1 = _mm512_srlv_epi32(rfv1, shiftv1);
|
|
__m512i qv2 = _mm512_srlv_epi32(rfv2, shiftv2);
|
|
|
|
qv1 = _mm512_mullo_epi32(qv1,
|
|
_mm512_and_si512(SDv1,_mm512_set1_epi32(0xffff)));
|
|
qv1 = _mm512_add_epi32(qv1, biasv1);
|
|
Rv1 = _mm512_add_epi32(Rv1, qv1);
|
|
|
|
qv2 = _mm512_mullo_epi32(qv2,
|
|
_mm512_and_si512(SDv2, _mm512_set1_epi32(0xffff)));
|
|
qv2 = _mm512_add_epi32(qv2, biasv2);
|
|
Rv2 = _mm512_add_epi32(Rv2, qv2);
|
|
}
|
|
ptr = (uint8_t *)ptr16;
|
|
STORE512(Rv, ransN);
|
|
|
|
for (z = 32-1; z >= 0; z--)
|
|
RansEncFlush(&ransN[z], &ptr);
|
|
|
|
empty:
|
|
// Finalise block size and return it
|
|
*out_size = (out_end - ptr) + tab_size;
|
|
|
|
memmove(out + tab_size, ptr, out_end-ptr);
|
|
|
|
return out;
|
|
}
|
|
|
|
unsigned char *rans_uncompress_O0_32x16_avx512(unsigned char *in,
|
|
unsigned int in_size,
|
|
unsigned char *out,
|
|
unsigned int out_sz) {
|
|
if (in_size < 32*4) // 32-states at least
|
|
return NULL;
|
|
|
|
if (out_sz >= INT_MAX)
|
|
return NULL; // protect against some overflow cases
|
|
|
|
/* Load in the static tables */
|
|
unsigned char *cp = in, *out_free = NULL;
|
|
unsigned char *cp_end = in + in_size;
|
|
int i;
|
|
uint32_t s3[TOTFREQ] __attribute__((aligned(64))); // For TF_SHIFT <= 12
|
|
|
|
if (!out)
|
|
out_free = out = malloc(out_sz);
|
|
if (!out)
|
|
return NULL;
|
|
|
|
// Precompute reverse lookup of frequency.
|
|
uint32_t F[256] = {0}, fsum;
|
|
int fsz = decode_freq(cp, cp_end, F, &fsum);
|
|
if (!fsz)
|
|
goto err;
|
|
cp += fsz;
|
|
|
|
normalise_freq_shift(F, fsum, TOTFREQ);
|
|
|
|
// Build symbols; fixme, do as part of decode, see the _d variant
|
|
if (rans_F_to_s3(F, TF_SHIFT, s3))
|
|
goto err;
|
|
|
|
if (cp_end - cp < 32 * 4)
|
|
goto err;
|
|
|
|
int z;
|
|
RansState Rv[32] __attribute__((aligned(64)));
|
|
for (z = 0; z < 32; z++) {
|
|
RansDecInit(&Rv[z], &cp);
|
|
if (Rv[z] < RANS_BYTE_L)
|
|
goto err;
|
|
}
|
|
|
|
uint16_t *sp = (uint16_t *)cp;
|
|
|
|
int out_end = (out_sz&~(32-1));
|
|
const uint32_t mask = (1u << TF_SHIFT)-1;
|
|
|
|
__m512i maskv = _mm512_set1_epi32(mask); // set mask in all lanes
|
|
__m512i R1 = _mm512_load_epi32(&Rv[0]);
|
|
__m512i R2 = _mm512_load_epi32(&Rv[16]);
|
|
|
|
// Start of the first loop iteration, which we do move to the end of the
|
|
// loop for the next cycle so we can remove some of the instr. latency.
|
|
__m512i masked1 = _mm512_and_epi32(R1, maskv);
|
|
__m512i masked2 = _mm512_and_epi32(R2, maskv);
|
|
__m512i S1 = _mm512_i32gather_epi32x(masked1, (int *)s3, sizeof(*s3));
|
|
__m512i S2 = _mm512_i32gather_epi32x(masked2, (int *)s3, sizeof(*s3));
|
|
|
|
uint8_t overflow[64+64] = {0};
|
|
for (i=0; i < out_end; i+=32) {
|
|
//for (z = 0; z < 16; z++) {
|
|
|
|
// Protect against running off the end of in buffer.
|
|
// We copy it to a worst-case local buffer when near the end.
|
|
if ((uint8_t *)sp+64 > cp_end) {
|
|
memmove(overflow, sp, cp_end - (uint8_t *)sp);
|
|
sp = (uint16_t *)overflow;
|
|
cp_end = overflow + sizeof(overflow);
|
|
}
|
|
|
|
//uint32_t S = s3[R[z] & mask];
|
|
__m512i renorm_words1 = _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)sp)); // next 16 words
|
|
|
|
//uint16_t f = S>>(TF_SHIFT+8), b = (S>>8) & mask;
|
|
__m512i f1 = _mm512_srli_epi32(S1, TF_SHIFT+8);
|
|
__m512i f2 = _mm512_srli_epi32(S2, TF_SHIFT+8);
|
|
__m512i b1 = _mm512_and_epi32(_mm512_srli_epi32(S1, 8), maskv);
|
|
__m512i b2 = _mm512_and_epi32(_mm512_srli_epi32(S2, 8), maskv);
|
|
|
|
//R[z] = f * (R[z] >> TF_SHIFT) + b;
|
|
// approx 10 cycle latency on mullo.
|
|
R1 = _mm512_add_epi32(
|
|
_mm512_mullo_epi32(
|
|
_mm512_srli_epi32(R1, TF_SHIFT), f1), b1);
|
|
__mmask16 renorm_mask1, renorm_mask2;
|
|
renorm_mask1=_mm512_cmplt_epu32_mask(R1, _mm512_set1_epi32(RANS_BYTE_L));
|
|
R2 = _mm512_add_epi32(
|
|
_mm512_mullo_epi32(
|
|
_mm512_srli_epi32(R2, TF_SHIFT), f2), b2);
|
|
|
|
// renorm. this is the interesting part:
|
|
renorm_mask2=_mm512_cmplt_epu32_mask(R2, _mm512_set1_epi32(RANS_BYTE_L));
|
|
// advance by however many words we actually read
|
|
sp += _mm_popcnt_u32(renorm_mask1);
|
|
__m512i renorm_words2 = _mm512_cvtepu16_epi32(_mm256_loadu_si256(
|
|
(const __m256i *)sp));
|
|
|
|
// select masked only
|
|
__m512i renorm_vals1, renorm_vals2;
|
|
|
|
renorm_vals1 = _mm512_mask_expand_epi32(R1, renorm_mask1, renorm_words1);
|
|
renorm_vals2 = _mm512_mask_expand_epi32(R2, renorm_mask2, renorm_words2);
|
|
|
|
// For start of next loop iteration. This has been moved here
|
|
// (and duplicated to before the loop starts) so we can do something
|
|
// with the latency period of gather, such as finishing up the
|
|
// renorm offset and writing the results.
|
|
__m512i S1_ = S1; // temporary copy for use in out[]=S later
|
|
__m512i S2_ = S2;
|
|
|
|
masked1 = _mm512_and_epi32(renorm_vals1, maskv);
|
|
S1 = _mm512_i32gather_epi32x(masked1, (int *)s3, sizeof(*s3));
|
|
masked2 = _mm512_and_epi32(renorm_vals2, maskv);
|
|
S2 = _mm512_i32gather_epi32x(masked2, (int *)s3, sizeof(*s3));
|
|
|
|
R1 = _mm512_mask_slli_epi32(R1, renorm_mask1, R1, 16);
|
|
R2 = _mm512_mask_slli_epi32(R2, renorm_mask2, R2, 16);
|
|
|
|
__m512i m16 = _mm512_set1_epi32(0xffff);
|
|
renorm_vals1 = _mm512_maskz_and_epi32(renorm_mask1, renorm_vals1, m16);
|
|
renorm_vals2 = _mm512_maskz_and_epi32(renorm_mask2, renorm_vals2, m16);
|
|
|
|
// advance by however many words we actually read
|
|
sp += _mm_popcnt_u32(renorm_mask2);
|
|
|
|
R1 = _mm512_add_epi32(R1, renorm_vals1);
|
|
R2 = _mm512_add_epi32(R2, renorm_vals2);
|
|
|
|
//out[i+z] = S;
|
|
_mm_storeu_si128((__m128i *)(out+i), _mm512_cvtepi32_epi8(S1_));
|
|
_mm_storeu_si128((__m128i *)(out+i+16), _mm512_cvtepi32_epi8(S2_));
|
|
}
|
|
|
|
_mm512_store_epi32(&Rv[ 0], R1);
|
|
_mm512_store_epi32(&Rv[16], R2);
|
|
|
|
for (z = out_sz & (32-1); z-- > 0; )
|
|
out[out_end + z] = s3[Rv[z] & mask];
|
|
|
|
return out;
|
|
|
|
err:
|
|
free(out_free);
|
|
return NULL;
|
|
}
|
|
|
|
#define TBUF8
|
|
#ifdef TBUF8
|
|
// 15% quicker overall O1 decode now due to rot32_simd below.
|
|
|
|
// NB: This uses AVX2 though and we could rewrite using AVX512 for
|
|
// further speed gains.
|
|
static inline void transpose_and_copy(uint8_t *out, int iN[32],
|
|
uint8_t t[32][32]) {
|
|
// int z;
|
|
// for (z = 0; z < 32; z++) {
|
|
// int k;
|
|
// for (k = 0; k < 32; k++)
|
|
// out[iN[z]+k] = t[k][z];
|
|
// iN[z] += 32;
|
|
// }
|
|
|
|
rot32_simd(t, out, iN);
|
|
}
|
|
|
|
#else
|
|
// Implemented using AVX512 gathers.
|
|
// This is faster than a naive scalar implementation, but doesn't beat the
|
|
// AVX2 vectorised 32x32 transpose function.
|
|
static inline void transpose_and_copy_avx512(uint8_t *out, int iN[32],
|
|
uint32_t t32[32][32]) {
|
|
int z;
|
|
// for (z = 0; z < 32; z++) {
|
|
// int k;
|
|
// for (k = 0; k < 32; k++)
|
|
// out[iN[z]+k] = t32[k][z];
|
|
// iN[z] += 32;
|
|
// }
|
|
|
|
|
|
__m512i v1 = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8,
|
|
7, 6, 5, 4, 3, 2, 1, 0);
|
|
v1 = _mm512_slli_epi32(v1, 5);
|
|
|
|
for (z = 0; z < 32; z++) {
|
|
__m512i t1 = _mm512_i32gather_epi32x(v1, &t32[ 0][z], 4);
|
|
__m512i t2 = _mm512_i32gather_epi32x(v1, &t32[16][z], 4);
|
|
_mm_storeu_si128((__m128i*)(&out[iN[z] ]), _mm512_cvtepi32_epi8(t1));
|
|
_mm_storeu_si128((__m128i*)(&out[iN[z]+16]), _mm512_cvtepi32_epi8(t2));
|
|
iN[z] += 32;
|
|
}
|
|
}
|
|
#endif // TBUF
|
|
|
|
unsigned char *rans_compress_O1_32x16_avx512(unsigned char *in,
|
|
unsigned int in_size,
|
|
unsigned char *out,
|
|
unsigned int *out_size) {
|
|
unsigned char *cp, *out_end, *out_free = NULL;
|
|
unsigned int tab_size;
|
|
uint32_t bound = rans_compress_bound_4x16(in_size,1)-20;
|
|
int z;
|
|
RansState ransN[32] __attribute__((aligned(64)));
|
|
|
|
if (in_size < 32) // force O0 instead
|
|
return NULL;
|
|
|
|
if (!out) {
|
|
*out_size = bound;
|
|
out = malloc(*out_size);
|
|
}
|
|
if (!out || bound > *out_size)
|
|
return NULL;
|
|
|
|
if (((size_t)out)&1)
|
|
bound--;
|
|
out_end = out + bound;
|
|
|
|
RansEncSymbol (*syms)[256] = htscodecs_tls_alloc(256 * (sizeof(*syms)));
|
|
if (!syms) {
|
|
free(out_free);
|
|
return NULL;
|
|
}
|
|
|
|
cp = out;
|
|
int shift = encode_freq1(in, in_size, 32, syms, &cp);
|
|
if (shift < 0) {
|
|
free(out_free);
|
|
htscodecs_tls_free(syms);
|
|
return NULL;
|
|
}
|
|
tab_size = cp - out;
|
|
|
|
for (z = 0; z < 32; z++)
|
|
RansEncInit(&ransN[z]);
|
|
|
|
uint8_t* ptr = out_end;
|
|
|
|
int iN[32] __attribute__((aligned(64)));
|
|
int isz4 = in_size/32;
|
|
for (z = 0; z < 32; z++)
|
|
iN[z] = (z+1)*isz4-2;
|
|
|
|
uint32_t lN[32] __attribute__((aligned(64)));
|
|
for (z = 0; z < 32; z++)
|
|
lN[z] = in[iN[z]+1];
|
|
|
|
// Deal with the remainder
|
|
z = 32-1;
|
|
lN[z] = in[in_size-1];
|
|
for (iN[z] = in_size-2; iN[z] > 32*isz4-2; iN[z]--) {
|
|
unsigned char c = in[iN[z]];
|
|
RansEncPutSymbol(&ransN[z], &ptr, &syms[c][lN[z]]);
|
|
lN[z] = c;
|
|
}
|
|
|
|
LOAD512(Rv, ransN);
|
|
|
|
uint16_t *ptr16 = (uint16_t *)ptr;
|
|
LOAD512(iN, iN);
|
|
LOAD512(last, lN);
|
|
|
|
__m512i c1 = _mm512_i32gather_epi32x1(iN1, in);
|
|
__m512i c2 = _mm512_i32gather_epi32x1(iN2, in);
|
|
|
|
// We cache the next 64-bytes locally and transpose.
|
|
// This means we can load 32 ints from t32[x] with load instructions
|
|
// instead of gathers. The copy, transpose and expand is easier done
|
|
// in scalar code.
|
|
#define BATCH 64
|
|
uint8_t t32[BATCH][32] __attribute__((aligned(64)));
|
|
int next_batch;
|
|
if (iN[0] > BATCH) {
|
|
int i, j;
|
|
for (i = 0; i < BATCH; i++)
|
|
// memcpy(c[i], &in[iN[i]-32], 32); fast mode
|
|
for (j = 0; j < 32; j++)
|
|
t32[BATCH-1-i][j] = in[iN[j]-1-i];
|
|
next_batch = BATCH;
|
|
} else {
|
|
next_batch = -1;
|
|
}
|
|
|
|
for (; iN[0] >= 0; iN[0]--) {
|
|
// Note, consider doing the same approach for the AVX2 encoder.
|
|
// Maybe we can also get gather working well there?
|
|
// Gather here is still a major latency bottleneck, consuming
|
|
// around 40% of CPU cycles overall.
|
|
|
|
// FIXME: maybe we need to cope with in[31] read over-flow
|
|
// on loop cycles 0, 1, 2 where gather reads 32-bits instead of
|
|
// 8 bits. Use set instead there on c2?
|
|
|
|
// index into syms[0][0] array, used for x_max, rcp_freq, and bias
|
|
__m512i vidx1 = _mm512_slli_epi32(c1, 8);
|
|
__m512i vidx2 = _mm512_slli_epi32(c2, 8);
|
|
vidx1 = _mm512_add_epi32(vidx1, last1);
|
|
vidx2 = _mm512_add_epi32(vidx2, last2);
|
|
vidx1 = _mm512_slli_epi32(vidx1, 2);
|
|
vidx2 = _mm512_slli_epi32(vidx2, 2);
|
|
|
|
// ------------------------------------------------------------
|
|
// for (z = NX-1; z >= 0; z--) {
|
|
// if (ransN[z] >= x_max[z]) {
|
|
// *--ptr16 = ransN[z] & 0xffff;
|
|
// ransN[z] >>= 16;
|
|
// }
|
|
// }
|
|
|
|
#define SET512x(a,x) \
|
|
__m512i a##1 = _mm512_i32gather_epi32x(vidx1, &syms[0][0].x, 4); \
|
|
__m512i a##2 = _mm512_i32gather_epi32x(vidx2, &syms[0][0].x, 4)
|
|
|
|
// Start of next loop, moved here to remove latency.
|
|
// last[z] = c[z]
|
|
// iN[z]--
|
|
// c[z] = in[iN[z]]
|
|
last1 = c1;
|
|
last2 = c2;
|
|
iN1 = _mm512_sub_epi32(iN1, _mm512_set1_epi32(1));
|
|
iN2 = _mm512_sub_epi32(iN2, _mm512_set1_epi32(1));
|
|
|
|
// Code below is equivalent to this:
|
|
// c1 = _mm512_i32gather_epi32(iN1, in, 1);
|
|
// c2 = _mm512_i32gather_epi32(iN2, in, 1);
|
|
|
|
// Better when we have a power of 2
|
|
if (next_batch >= 0) {
|
|
if (--next_batch < 0 && iN[0] > BATCH) {
|
|
// Load 32 BATCH blocks of data.
|
|
// Executed once every BATCH cycles
|
|
int i, j;
|
|
uint8_t c[32][BATCH];
|
|
iN[0] += BATCH;
|
|
for (j = 0; j < 32; j++) {
|
|
iN[j] -= BATCH;
|
|
memcpy(c[j], &in[iN[j]-BATCH], BATCH);
|
|
}
|
|
// transpose matrix from 32xBATCH to BATCHx32
|
|
for (j = 0; j < 32; j++) {
|
|
for (i = 0; i < BATCH; i+=16) {
|
|
int z;
|
|
for (z = 0; z < 16; z++)
|
|
t32[i+z][j] = c[j][i+z];
|
|
}
|
|
}
|
|
next_batch = BATCH-1;
|
|
}
|
|
if (next_batch >= 0) {
|
|
// Copy from our of our pre-loaded BATCHx32 tables
|
|
// Executed every cycles
|
|
__m128i c1_ = _mm_load_si128((__m128i *)&t32[next_batch][0]);
|
|
__m128i c2_ = _mm_load_si128((__m128i *)&t32[next_batch][16]);
|
|
c1 = _mm512_cvtepu8_epi32(c1_);
|
|
c2 = _mm512_cvtepu8_epi32(c2_);
|
|
}
|
|
}
|
|
|
|
if (next_batch < 0 && iN[0]) {
|
|
// no pre-loaded data as within BATCHx32 of input end
|
|
c1 = _mm512_i32gather_epi32x1(iN1, in);
|
|
c2 = _mm512_i32gather_epi32x1(iN2, in);
|
|
|
|
// Speeds up clang, even though not needed any more.
|
|
// Harmless to leave here.
|
|
c1 = _mm512_and_si512(c1, _mm512_set1_epi32(0xff));
|
|
c2 = _mm512_and_si512(c2, _mm512_set1_epi32(0xff));
|
|
}
|
|
// End of "equivalent to" code block
|
|
|
|
SET512x(xmax, x_max); // high latency
|
|
|
|
uint16_t gt_mask1 = _mm512_cmpgt_epi32_mask(Rv1, xmax1);
|
|
int pc1 = _mm_popcnt_u32(gt_mask1);
|
|
__m512i Rp1 = _mm512_and_si512(Rv1, _mm512_set1_epi32(0xffff));
|
|
__m512i Rp2 = _mm512_and_si512(Rv2, _mm512_set1_epi32(0xffff));
|
|
uint16_t gt_mask2 = _mm512_cmpgt_epi32_mask(Rv2, xmax2);
|
|
SET512x(SDv, cmpl_freq); // good
|
|
int pc2 = _mm_popcnt_u32(gt_mask2);
|
|
|
|
Rp1 = _mm512_maskz_compress_epi32(gt_mask1, Rp1);
|
|
Rp2 = _mm512_maskz_compress_epi32(gt_mask2, Rp2);
|
|
|
|
_mm512_mask_cvtepi32_storeu_epi16(ptr16-pc2, (1<<pc2)-1, Rp2);
|
|
ptr16 -= pc2;
|
|
_mm512_mask_cvtepi32_storeu_epi16(ptr16-pc1, (1<<pc1)-1, Rp1);
|
|
ptr16 -= pc1;
|
|
|
|
Rv1 = _mm512_mask_srli_epi32(Rv1, gt_mask1, Rv1, 16);
|
|
Rv2 = _mm512_mask_srli_epi32(Rv2, gt_mask2, Rv2, 16);
|
|
|
|
// ------------------------------------------------------------
|
|
// uint32_t q = (uint32_t) (((uint64_t)ransN[z] * rcp_freq[z])
|
|
// >> rcp_shift[z]);
|
|
// ransN[z] = ransN[z] + bias[z] + q * cmpl_freq[z];
|
|
SET512x(rfv, rcp_freq); // good-ish
|
|
|
|
__m512i rf1_hm = _mm512_mul_epu32(_mm512_srli_epi64(Rv1, 32),
|
|
_mm512_srli_epi64(rfv1, 32));
|
|
__m512i rf2_hm = _mm512_mul_epu32(_mm512_srli_epi64(Rv2, 32),
|
|
_mm512_srli_epi64(rfv2, 32));
|
|
__m512i rf1_lm = _mm512_srli_epi64(_mm512_mul_epu32(Rv1, rfv1), 32);
|
|
__m512i rf2_lm = _mm512_srli_epi64(_mm512_mul_epu32(Rv2, rfv2), 32);
|
|
|
|
const __m512i top32 = _mm512_set1_epi64((uint64_t)0xffffffff00000000);
|
|
rf1_hm = _mm512_and_epi32(rf1_hm, top32);
|
|
rf2_hm = _mm512_and_epi32(rf2_hm, top32);
|
|
rfv1 = _mm512_or_epi32(rf1_lm, rf1_hm);
|
|
rfv2 = _mm512_or_epi32(rf2_lm, rf2_hm);
|
|
|
|
SET512x(biasv, bias); // good
|
|
__m512i shiftv1 = _mm512_srli_epi32(SDv1, 16);
|
|
__m512i shiftv2 = _mm512_srli_epi32(SDv2, 16);
|
|
|
|
shiftv1 = _mm512_sub_epi32(shiftv1, _mm512_set1_epi32(32));
|
|
shiftv2 = _mm512_sub_epi32(shiftv2, _mm512_set1_epi32(32));
|
|
|
|
__m512i qv1 = _mm512_srlv_epi32(rfv1, shiftv1);
|
|
__m512i qv2 = _mm512_srlv_epi32(rfv2, shiftv2);
|
|
|
|
const __m512i bot16 = _mm512_set1_epi32(0xffff);
|
|
qv1 = _mm512_mullo_epi32(qv1, _mm512_and_si512(SDv1, bot16));
|
|
qv2 = _mm512_mullo_epi32(qv2, _mm512_and_si512(SDv2, bot16));
|
|
|
|
qv1 = _mm512_add_epi32(qv1, biasv1);
|
|
Rv1 = _mm512_add_epi32(Rv1, qv1);
|
|
|
|
qv2 = _mm512_add_epi32(qv2, biasv2);
|
|
Rv2 = _mm512_add_epi32(Rv2, qv2);
|
|
}
|
|
|
|
STORE512(Rv, ransN);
|
|
STORE512(last, lN);
|
|
|
|
ptr = (uint8_t *)ptr16;
|
|
|
|
for (z = 32-1; z>=0; z--)
|
|
RansEncPutSymbol(&ransN[z], &ptr, &syms[0][lN[z]]);
|
|
|
|
for (z = 32-1; z >= 0; z--)
|
|
RansEncFlush(&ransN[z], &ptr);
|
|
|
|
// Finalise block size and return it
|
|
*out_size = (out_end - ptr) + tab_size;
|
|
|
|
// cp = out;
|
|
// *cp++ = (in_size>> 0) & 0xff;
|
|
// *cp++ = (in_size>> 8) & 0xff;
|
|
// *cp++ = (in_size>>16) & 0xff;
|
|
// *cp++ = (in_size>>24) & 0xff;
|
|
|
|
memmove(out + tab_size, ptr, out_end-ptr);
|
|
|
|
htscodecs_tls_free(syms);
|
|
return out;
|
|
}
|
|
|
|
#define NX 32
|
|
unsigned char *rans_uncompress_O1_32x16_avx512(unsigned char *in,
|
|
unsigned int in_size,
|
|
unsigned char *out,
|
|
unsigned int out_sz) {
|
|
if (in_size < NX*4) // 4-states at least
|
|
return NULL;
|
|
|
|
if (out_sz >= INT_MAX)
|
|
return NULL; // protect against some overflow cases
|
|
|
|
/* Load in the static tables */
|
|
unsigned char *cp = in, *cp_end = in+in_size, *out_free = NULL;
|
|
unsigned char *c_freq = NULL;
|
|
|
|
uint32_t (*s3)[TOTFREQ_O1] = htscodecs_tls_alloc(256*TOTFREQ_O1*4);
|
|
if (!s3)
|
|
return NULL;
|
|
uint32_t (*s3F)[TOTFREQ_O1_FAST] = (uint32_t (*)[TOTFREQ_O1_FAST])s3;
|
|
|
|
if (!out)
|
|
out_free = out = malloc(out_sz);
|
|
|
|
if (!out)
|
|
goto err;
|
|
|
|
//fprintf(stderr, "out_sz=%d\n", out_sz);
|
|
|
|
// compressed header? If so uncompress it
|
|
unsigned char *tab_end = NULL;
|
|
unsigned char *c_freq_end = cp_end;
|
|
unsigned int shift = *cp >> 4;
|
|
if (*cp++ & 1) {
|
|
uint32_t u_freq_sz, c_freq_sz;
|
|
cp += var_get_u32(cp, cp_end, &u_freq_sz);
|
|
cp += var_get_u32(cp, cp_end, &c_freq_sz);
|
|
if (c_freq_sz > cp_end - cp)
|
|
goto err;
|
|
tab_end = cp + c_freq_sz;
|
|
if (!(c_freq = rans_uncompress_O0_4x16(cp, c_freq_sz, NULL,
|
|
u_freq_sz)))
|
|
goto err;
|
|
cp = c_freq;
|
|
c_freq_end = c_freq + u_freq_sz;
|
|
}
|
|
|
|
// Decode order-0 symbol list; avoids needing in order-1 tables
|
|
cp += decode_freq1(cp, c_freq_end, shift, s3, s3F, NULL, NULL);
|
|
|
|
if (tab_end)
|
|
cp = tab_end;
|
|
free(c_freq);
|
|
c_freq = NULL;
|
|
|
|
if (cp_end - cp < NX * 4)
|
|
goto err;
|
|
|
|
RansState R[NX] __attribute__((aligned(64)));
|
|
uint8_t *ptr = cp, *ptr_end = in + in_size;
|
|
int z;
|
|
for (z = 0; z < NX; z++) {
|
|
RansDecInit(&R[z], &ptr);
|
|
if (R[z] < RANS_BYTE_L)
|
|
goto err;
|
|
}
|
|
|
|
int isz4 = out_sz/NX;
|
|
int iN[NX], lN[NX] __attribute__((aligned(64))) = {0};
|
|
for (z = 0; z < NX; z++)
|
|
iN[z] = z*isz4;
|
|
|
|
uint16_t *sp = (uint16_t *)ptr;
|
|
const uint32_t mask = (1u << shift)-1;
|
|
|
|
__m512i _maskv = _mm512_set1_epi32(mask);
|
|
LOAD512(_Rv, R);
|
|
LOAD512(_Lv, lN);
|
|
|
|
#ifdef TBUF8
|
|
union {
|
|
unsigned char tbuf[32][32];
|
|
uint64_t tbuf64[32][4];
|
|
} u __attribute__((aligned(32)));
|
|
#else
|
|
uint32_t tbuf[32][32];
|
|
#endif
|
|
|
|
unsigned int tidx = 0;
|
|
|
|
if (shift == TF_SHIFT_O1) {
|
|
isz4 -= 64;
|
|
for (; iN[0] < isz4 && (uint8_t *)sp+64 < ptr_end; ) {
|
|
// m[z] = R[z] & mask;
|
|
__m512i _masked1 = _mm512_and_si512(_Rv1, _maskv);
|
|
__m512i _masked2 = _mm512_and_si512(_Rv2, _maskv);
|
|
|
|
// S[z] = s3[lN[z]][m[z]];
|
|
_Lv1 = _mm512_slli_epi32(_Lv1, TF_SHIFT_O1);
|
|
_Lv2 = _mm512_slli_epi32(_Lv2, TF_SHIFT_O1);
|
|
|
|
_masked1 = _mm512_add_epi32(_masked1, _Lv1);
|
|
_masked2 = _mm512_add_epi32(_masked2, _Lv2);
|
|
|
|
// This is the biggest bottleneck
|
|
__m512i _Sv1 = _mm512_i32gather_epi32x(_masked1, (int *)&s3F[0][0],
|
|
sizeof(s3F[0][0]));
|
|
__m512i _Sv2 = _mm512_i32gather_epi32x(_masked2, (int *)&s3F[0][0],
|
|
sizeof(s3F[0][0]));
|
|
|
|
// f[z] = S[z]>>(TF_SHIFT_O1+8);
|
|
__m512i _fv1 = _mm512_srli_epi32(_Sv1, TF_SHIFT_O1+8);
|
|
__m512i _fv2 = _mm512_srli_epi32(_Sv2, TF_SHIFT_O1+8);
|
|
|
|
// b[z] = (S[z]>>8) & mask;
|
|
__m512i _bv1 = _mm512_and_si512(_mm512_srli_epi32(_Sv1,8), _maskv);
|
|
__m512i _bv2 = _mm512_and_si512(_mm512_srli_epi32(_Sv2,8), _maskv);
|
|
|
|
// s[z] = S[z] & 0xff;
|
|
__m512i _sv1 = _mm512_and_si512(_Sv1, _mm512_set1_epi32(0xff));
|
|
__m512i _sv2 = _mm512_and_si512(_Sv2, _mm512_set1_epi32(0xff));
|
|
|
|
// A maximum frequency of 4096 doesn't fit in our s3 array.
|
|
// as it's 12 bit + 12 bit + 8 bit. It wraps around to zero.
|
|
// (We don't have this issue for TOTFREQ_O1_FAST.)
|
|
//
|
|
// Solution 1 is to change to spec to forbid freq of 4096.
|
|
// Easy hack is to add an extra symbol so it sums correctly.
|
|
// => 572 MB/s on q40 (deskpro).
|
|
//
|
|
// Solution 2 implemented here is to look for the wrap around
|
|
// and fix it.
|
|
// => 556 MB/s on q40
|
|
// cope with max freq of 4096. Only 3% hit
|
|
__m512i max_freq = _mm512_set1_epi32(TOTFREQ_O1);
|
|
__m512i zero = _mm512_setzero_si512();
|
|
__mmask16 cmp1 = _mm512_cmpeq_epi32_mask(_fv1, zero);
|
|
__mmask16 cmp2 = _mm512_cmpeq_epi32_mask(_fv2, zero);
|
|
_fv1 = _mm512_mask_blend_epi32(cmp1, _fv1, max_freq);
|
|
_fv2 = _mm512_mask_blend_epi32(cmp2, _fv2, max_freq);
|
|
|
|
// R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
|
|
_Rv1 = _mm512_add_epi32(
|
|
_mm512_mullo_epi32(
|
|
_mm512_srli_epi32(_Rv1,TF_SHIFT_O1), _fv1), _bv1);
|
|
_Rv2 = _mm512_add_epi32(
|
|
_mm512_mullo_epi32(
|
|
_mm512_srli_epi32(_Rv2,TF_SHIFT_O1), _fv2), _bv2);
|
|
|
|
//for (z = 0; z < NX; z++) lN[z] = c[z];
|
|
_Lv1 = _sv1;
|
|
_Lv2 = _sv2;
|
|
|
|
// RansDecRenorm(&R[z], &ptr);
|
|
__m512i _renorm_mask1 = _mm512_xor_si512(_Rv1,
|
|
_mm512_set1_epi32(0x80000000));
|
|
__m512i _renorm_mask2 = _mm512_xor_si512(_Rv2,
|
|
_mm512_set1_epi32(0x80000000));
|
|
|
|
int _imask1 =_mm512_cmpgt_epi32_mask
|
|
(_mm512_set1_epi32(RANS_BYTE_L-0x80000000), _renorm_mask1);
|
|
int _imask2 = _mm512_cmpgt_epi32_mask
|
|
(_mm512_set1_epi32(RANS_BYTE_L-0x80000000), _renorm_mask2);
|
|
|
|
__m512i renorm_words1 = _mm512_cvtepu16_epi32
|
|
(_mm256_loadu_si256((const __m256i *)sp));
|
|
sp += _mm_popcnt_u32(_imask1);
|
|
|
|
__m512i renorm_words2 = _mm512_cvtepu16_epi32
|
|
(_mm256_loadu_si256((const __m256i *)sp));
|
|
sp += _mm_popcnt_u32(_imask2);
|
|
|
|
__m512i _renorm_vals1 =
|
|
_mm512_maskz_expand_epi32(_imask1, renorm_words1);
|
|
__m512i _renorm_vals2 =
|
|
_mm512_maskz_expand_epi32(_imask2, renorm_words2);
|
|
|
|
_Rv1 = _mm512_mask_slli_epi32(_Rv1, _imask1, _Rv1, 16);
|
|
_Rv2 = _mm512_mask_slli_epi32(_Rv2, _imask2, _Rv2, 16);
|
|
|
|
_Rv1 = _mm512_add_epi32(_Rv1, _renorm_vals1);
|
|
_Rv2 = _mm512_add_epi32(_Rv2, _renorm_vals2);
|
|
|
|
#ifdef TBUF8
|
|
_mm_storeu_si128((__m128i *)(&u.tbuf64[tidx][0]),
|
|
_mm512_cvtepi32_epi8(_Sv1)); // or _sv1?
|
|
_mm_storeu_si128((__m128i *)(&u.tbuf64[tidx][2]),
|
|
_mm512_cvtepi32_epi8(_Sv2));
|
|
#else
|
|
_mm512_storeu_si512((__m512i *)(&tbuf[tidx][ 0]), _sv1);
|
|
_mm512_storeu_si512((__m512i *)(&tbuf[tidx][16]), _sv2);
|
|
#endif
|
|
|
|
iN[0]++;
|
|
if (++tidx == 32) {
|
|
iN[0]-=32;
|
|
|
|
// We have tidx[x][y] which we want to store in
|
|
// memory in out[y][z] instead. This is an unrolled
|
|
// transposition.
|
|
#ifdef TBUF8
|
|
transpose_and_copy(out, iN, u.tbuf);
|
|
#else
|
|
transpose_and_copy_avx512(out, iN, tbuf);
|
|
#endif
|
|
tidx = 0;
|
|
}
|
|
}
|
|
isz4 += 64;
|
|
|
|
STORE512(_Rv, R);
|
|
STORE512(_Lv, lN);
|
|
ptr = (uint8_t *)sp;
|
|
|
|
if (1) {
|
|
iN[0]-=tidx;
|
|
int T;
|
|
for (z = 0; z < NX; z++)
|
|
for (T = 0; T < tidx; T++)
|
|
#ifdef TBUF8
|
|
out[iN[z]++] = u.tbuf[T][z];
|
|
#else
|
|
out[iN[z]++] = tbuf[T][z];
|
|
#endif
|
|
}
|
|
|
|
// Scalar version for close to the end of in[] array so we don't
|
|
// do SIMD loads beyond the end of the buffer
|
|
for (; iN[0] < isz4;) {
|
|
for (z = 0; z < NX; z++) {
|
|
uint32_t m = R[z] & ((1u<<TF_SHIFT_O1)-1);
|
|
uint32_t S = s3[lN[z]][m];
|
|
unsigned char c = S & 0xff;
|
|
out[iN[z]++] = c;
|
|
uint32_t F = S>>(TF_SHIFT_O1+8);
|
|
R[z] = (F?F:4096) * (R[z]>>TF_SHIFT_O1) +
|
|
((S>>8) & ((1u<<TF_SHIFT_O1)-1));
|
|
RansDecRenormSafe(&R[z], &ptr, ptr_end);
|
|
lN[z] = c;
|
|
}
|
|
}
|
|
|
|
// Remainder
|
|
z = NX-1;
|
|
for (; iN[z] < out_sz; ) {
|
|
uint32_t m = R[z] & ((1u<<TF_SHIFT_O1)-1);
|
|
uint32_t S = s3[lN[z]][m];
|
|
unsigned char c = S & 0xff;
|
|
out[iN[z]++] = c;
|
|
uint32_t F = S>>(TF_SHIFT_O1+8);
|
|
R[z] = (F?F:4096) * (R[z]>>TF_SHIFT_O1) +
|
|
((S>>8) & ((1u<<TF_SHIFT_O1)-1));
|
|
RansDecRenormSafe(&R[z], &ptr, ptr_end);
|
|
lN[z] = c;
|
|
}
|
|
} else {
|
|
// TF_SHIFT_O1_FAST. This is the most commonly used variant.
|
|
|
|
// SIMD version ends decoding early as it reads at most 64 bytes
|
|
// from input via 4 vectorised loads.
|
|
isz4 -= 64;
|
|
for (; iN[0] < isz4 && (uint8_t *)sp+64 < ptr_end; ) {
|
|
// m[z] = R[z] & mask;
|
|
__m512i _masked1 = _mm512_and_si512(_Rv1, _maskv);
|
|
__m512i _masked2 = _mm512_and_si512(_Rv2, _maskv);
|
|
|
|
// S[z] = s3[lN[z]][m[z]];
|
|
_Lv1 = _mm512_slli_epi32(_Lv1, TF_SHIFT_O1_FAST);
|
|
_Lv2 = _mm512_slli_epi32(_Lv2, TF_SHIFT_O1_FAST);
|
|
|
|
_masked1 = _mm512_add_epi32(_masked1, _Lv1);
|
|
_masked2 = _mm512_add_epi32(_masked2, _Lv2);
|
|
|
|
// This is the biggest bottleneck
|
|
__m512i _Sv1 = _mm512_i32gather_epi32x(_masked1, (int *)&s3F[0][0],
|
|
sizeof(s3F[0][0]));
|
|
__m512i _Sv2 = _mm512_i32gather_epi32x(_masked2, (int *)&s3F[0][0],
|
|
sizeof(s3F[0][0]));
|
|
|
|
// f[z] = S[z]>>(TF_SHIFT_O1+8);
|
|
__m512i _fv1 = _mm512_srli_epi32(_Sv1, TF_SHIFT_O1_FAST+8);
|
|
__m512i _fv2 = _mm512_srli_epi32(_Sv2, TF_SHIFT_O1_FAST+8);
|
|
|
|
// b[z] = (S[z]>>8) & mask;
|
|
__m512i _bv1 = _mm512_and_si512(_mm512_srli_epi32(_Sv1,8), _maskv);
|
|
__m512i _bv2 = _mm512_and_si512(_mm512_srli_epi32(_Sv2,8), _maskv);
|
|
|
|
// s[z] = S[z] & 0xff;
|
|
__m512i _sv1 = _mm512_and_si512(_Sv1, _mm512_set1_epi32(0xff));
|
|
__m512i _sv2 = _mm512_and_si512(_Sv2, _mm512_set1_epi32(0xff));
|
|
|
|
// R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
|
|
_Rv1 = _mm512_add_epi32(
|
|
_mm512_mullo_epi32(
|
|
_mm512_srli_epi32(_Rv1,TF_SHIFT_O1_FAST),
|
|
_fv1), _bv1);
|
|
_Rv2 = _mm512_add_epi32(
|
|
_mm512_mullo_epi32(
|
|
_mm512_srli_epi32(_Rv2,TF_SHIFT_O1_FAST),
|
|
_fv2), _bv2);
|
|
|
|
//for (z = 0; z < NX; z++) lN[z] = c[z];
|
|
_Lv1 = _sv1;
|
|
_Lv2 = _sv2;
|
|
|
|
// RansDecRenorm(&R[z], &ptr);
|
|
__m512i _renorm_mask1 = _mm512_xor_si512(_Rv1,
|
|
_mm512_set1_epi32(0x80000000));
|
|
__m512i _renorm_mask2 = _mm512_xor_si512(_Rv2,
|
|
_mm512_set1_epi32(0x80000000));
|
|
|
|
int _imask1 =_mm512_cmpgt_epi32_mask
|
|
(_mm512_set1_epi32(RANS_BYTE_L-0x80000000), _renorm_mask1);
|
|
int _imask2 = _mm512_cmpgt_epi32_mask
|
|
(_mm512_set1_epi32(RANS_BYTE_L-0x80000000), _renorm_mask2);
|
|
|
|
__m512i renorm_words1 = _mm512_cvtepu16_epi32
|
|
(_mm256_loadu_si256((const __m256i *)sp));
|
|
sp += _mm_popcnt_u32(_imask1);
|
|
|
|
__m512i renorm_words2 = _mm512_cvtepu16_epi32
|
|
(_mm256_loadu_si256((const __m256i *)sp));
|
|
sp += _mm_popcnt_u32(_imask2);
|
|
|
|
__m512i _renorm_vals1 =
|
|
_mm512_maskz_expand_epi32(_imask1, renorm_words1);
|
|
__m512i _renorm_vals2 =
|
|
_mm512_maskz_expand_epi32(_imask2, renorm_words2);
|
|
|
|
_Rv1 = _mm512_mask_slli_epi32(_Rv1, _imask1, _Rv1, 16);
|
|
_Rv2 = _mm512_mask_slli_epi32(_Rv2, _imask2, _Rv2, 16);
|
|
|
|
_Rv1 = _mm512_add_epi32(_Rv1, _renorm_vals1);
|
|
_Rv2 = _mm512_add_epi32(_Rv2, _renorm_vals2);
|
|
|
|
#ifdef TBUF8
|
|
_mm_storeu_si128((__m128i *)(&u.tbuf64[tidx][0]),
|
|
_mm512_cvtepi32_epi8(_Sv1)); // or _sv1?
|
|
_mm_storeu_si128((__m128i *)(&u.tbuf64[tidx][2]),
|
|
_mm512_cvtepi32_epi8(_Sv2));
|
|
#else
|
|
_mm512_storeu_si512((__m512i *)(&tbuf[tidx][ 0]), _sv1);
|
|
_mm512_storeu_si512((__m512i *)(&tbuf[tidx][16]), _sv2);
|
|
#endif
|
|
|
|
iN[0]++;
|
|
if (++tidx == 32) {
|
|
iN[0]-=32;
|
|
#ifdef TBUF8
|
|
transpose_and_copy(out, iN, u.tbuf);
|
|
#else
|
|
transpose_and_copy_avx512(out, iN, tbuf);
|
|
#endif
|
|
tidx = 0;
|
|
}
|
|
}
|
|
isz4 += 64;
|
|
|
|
STORE512(_Rv, R);
|
|
STORE512(_Lv, lN);
|
|
ptr = (uint8_t *)sp;
|
|
|
|
if (1) {
|
|
iN[0]-=tidx;
|
|
int T;
|
|
for (z = 0; z < NX; z++)
|
|
for (T = 0; T < tidx; T++)
|
|
#ifdef TBUF8
|
|
out[iN[z]++] = u.tbuf[T][z];
|
|
#else
|
|
out[iN[z]++] = tbuf[T][z];
|
|
#endif
|
|
}
|
|
|
|
// Scalar version for close to the end of in[] array so we don't
|
|
// do SIMD loads beyond the end of the buffer
|
|
for (; iN[0] < isz4;) {
|
|
for (z = 0; z < NX; z++) {
|
|
uint32_t m = R[z] & ((1u<<TF_SHIFT_O1_FAST)-1);
|
|
uint32_t S = s3F[lN[z]][m];
|
|
unsigned char c = S & 0xff;
|
|
out[iN[z]++] = c;
|
|
R[z] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[z]>>TF_SHIFT_O1_FAST) +
|
|
((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
|
|
RansDecRenormSafe(&R[z], &ptr, ptr_end);
|
|
lN[z] = c;
|
|
}
|
|
}
|
|
|
|
// Remainder
|
|
z = NX-1;
|
|
for (; iN[z] < out_sz; ) {
|
|
uint32_t m = R[z] & ((1u<<TF_SHIFT_O1_FAST)-1);
|
|
uint32_t S = s3F[lN[z]][m];
|
|
unsigned char c = S & 0xff;
|
|
out[iN[z]++] = c;
|
|
R[z] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[z]>>TF_SHIFT_O1_FAST) +
|
|
((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
|
|
RansDecRenormSafe(&R[z], &ptr, ptr_end);
|
|
lN[z] = c;
|
|
}
|
|
}
|
|
|
|
htscodecs_tls_free(s3);
|
|
return out;
|
|
|
|
err:
|
|
htscodecs_tls_free(s3);
|
|
free(out_free);
|
|
free(c_freq);
|
|
|
|
return NULL;
|
|
}
|
|
#else // HAVE_AVX512
|
|
// Prevent "empty translation unit" errors when building without AVX512
|
|
const char *rANS_static32x16pr_avx512_disabled = "No AVX512";
|
|
#endif // HAVE_AVX512
|