diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index f18cca24d..ee814d7f3 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -93,6 +93,14 @@ extern "C" { #define HAS_SCALEROWDOWN4_AVX2 #endif +// The following are available for AVX512 clang x64 platforms: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \ + !defined(LIBYUV_ENABLE_ROWWIN) +#define HAS_SCALEADDROW_AVX512BW +#define HAS_SCALEADDCOLS_AVX512BW +#endif + // The following are available on Neon platforms: #if !defined(LIBYUV_DISABLE_NEON) && \ (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) @@ -798,12 +806,27 @@ void ScaleRowDown38_2_Box_Any_SSSE3(const uint8_t* src_ptr, void ScaleAddRow_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); +void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width); void ScaleAddRow_Any_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); void ScaleAddRow_Any_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); +void ScaleAddCols1_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr); +void ScaleAddCols2_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr); void ScaleFilterCols_SSSE3(uint8_t* dst_ptr, const uint8_t* src_ptr, diff --git a/source/scale.cc b/source/scale.cc index 114514197..fc89a5545 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -900,6 +900,18 @@ static int ScalePlaneBox(int src_width, } } #endif +#if defined(HAS_SCALEADDROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ScaleAddRow = ScaleAddRow_AVX512BW; + } +#endif +#if defined(HAS_SCALEADDCOLS_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0) { + ScaleAddCols = (dx & 0xffff) ? ScaleAddCols2_AVX512BW + : ((dx != 0x10000) ? ScaleAddCols1_AVX512BW + : ScaleAddCols0_C); + } +#endif #if defined(HAS_SCALEADDROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ScaleAddRow = ScaleAddRow_Any_NEON; diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index b2a289ed4..6c93933c8 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -1792,6 +1792,674 @@ void ScaleAddRow_AVX2(const uint8_t* src_ptr, } #endif // HAS_SCALEADDROW_AVX2 +#ifdef HAS_SCALEADDROW_AVX512BW +// Reads 32xN bytes and accumulates to 32xN shorts at a time, with masked tail. +void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width) { + uintptr_t temp; + asm volatile( + "sub $0x40,%2 \n" + "jl 2f \n" + + // 64 pixel loop. + LABELALIGN + "1: \n" + "vpmovzxbw (%0),%%zmm0 \n" + "vpmovzxbw 0x20(%0),%%zmm1 \n" + "lea 0x40(%0),%0 \n" + "vpaddusw (%1),%%zmm0,%%zmm0 \n" + "vpaddusw 0x40(%1),%%zmm1,%%zmm1 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "jl 3f \n" + "vpmovzxbw (%0),%%zmm0 \n" + "lea 0x20(%0),%0 \n" + "vpaddusw (%1),%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + + "3: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "mov $-1,%k3 \n" + "bzhi %k2,%k3,%k3 \n" + "kmovd %k3,%%k1 \n" + "vpmovzxbw (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpaddusw (%1),%%zmm0,%%zmm0%{%%k1%}%{z%}\n" + "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(src_width), // %2 + "=&r"(temp) // %3 + : + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_SCALEADDROW_AVX512BW + +#ifdef HAS_SCALEADDCOLS_AVX512BW +static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15}; + +void ScaleAddCols1_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr) { + int boxwidth = dx >> 16; + if (boxwidth < 1) { + boxwidth = 1; + } + int scaleval = 65536 / (boxwidth * boxheight); + src_ptr += (x >> 16); + intptr_t temp, mask; + asm volatile( + "mov $0x0000ffff,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm6 \n" + "vpbroadcastd %k[scaleval],%%zmm7 \n" + "cmp $4,%k[bw] \n" + "jg 20f \n" + + // Path 1: 1 <= boxwidth <= 4 (16 output pixels per loop). + "vpmovzxbd %[kSeq],%%zmm3 \n" + "vpslld $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpbroadcastw %k[bw],%%zmm2 \n" + "vpmullw %%zmm2,%%zmm3,%%zmm3 \n" + "mov $0x00010000,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm0 \n" + "mov $0x00030002,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm1 \n" + "vpaddw %%zmm0,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm1,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm0,%%zmm2,%%k2 \n" + "vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n" + "mov %k[bw],%k[temp] \n" + "shl $4,%k[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k4 \n" + "shl $1,%q[temp] \n" + "sub $0x10,%[dst_width] \n" + "jl 12f \n" + + LABELALIGN + "11: \n" + "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n" + "add %q[temp],%[src_ptr] \n" + "vmovdqa64 %%zmm4,%%zmm2 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n" + "vmovdqa64 %%zmm5,%%zmm3 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm2,%%zmm0 \n" + "vpandd %%zmm6,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm1 \n" + "vpandd %%zmm6,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm1,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm3,%%zmm2,%%zmm2 \n" + "vpmulld %%zmm7,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpmovdb %%zmm2,(%[dst_ptr]) \n" + "lea 0x10(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x10,%[dst_width] \n" + "jge 11b \n" + + "12: \n" + "add $0x10,%[dst_width] \n" + "jle 99f \n" + "mov %k[dst_width],%k[temp] \n" + "imul %k[bw],%k[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k4 \n" + "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n" + "vmovdqa64 %%zmm4,%%zmm2 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n" + "vmovdqa64 %%zmm5,%%zmm3 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm2,%%zmm0 \n" + "vpandd %%zmm6,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm1 \n" + "vpandd %%zmm6,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm1,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm3,%%zmm2,%%zmm2 \n" + "vpmulld %%zmm7,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask]\n" + "kmovw %k[mask],%%k1 \n" + "vpmovdb %%zmm2,(%[dst_ptr])%{%%k1%} \n" + "jmp 99f \n" + + // Path 2: 5 <= boxwidth <= 8 (8 output pixels per loop). + "20: \n" + "cmp $8,%k[bw] \n" + "jg 30f \n" + "vpmovzxbq %[kSeq],%%zmm3 \n" + "vpsllq $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $32,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpbroadcastw %k[bw],%%zmm2 \n" + "vpmullw %%zmm2,%%zmm3,%%zmm3 \n" + "movabs $0x0003000200010000,%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm0 \n" + "movabs $0x0007000600050004,%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm1 \n" + "vpaddw %%zmm0,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm1,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n" + "mov %k[bw],%k[temp] \n" + "shl $3,%k[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k4 \n" + "shl $1,%q[temp] \n" + "sub $0x8,%[dst_width] \n" + "jl 22f \n" + + LABELALIGN + "21: \n" + "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n" + "add %q[temp],%[src_ptr] \n" + "vmovdqa64 %%zmm4,%%zmm2 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm2 \n" + "vmovdqa64 %%zmm5,%%zmm3 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm2,%%zmm0 \n" + "vpandd %%zmm6,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm1 \n" + "vpandd %%zmm6,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm1,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm3,%%zmm2,%%zmm2 \n" + "vpsrlq $32,%%zmm2,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpmulld %%zmm7,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpmovqb %%zmm2,(%[dst_ptr]) \n" + "lea 0x8(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x8,%[dst_width] \n" + "jge 21b \n" + + "22: \n" + "add $0x8,%[dst_width] \n" + "jle 99f \n" + "mov %k[dst_width],%k[temp] \n" + "imul %k[bw],%k[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k4 \n" + "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n" + "vmovdqa64 %%zmm4,%%zmm2 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm2 \n" + "vmovdqa64 %%zmm5,%%zmm3 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm2,%%zmm0 \n" + "vpandd %%zmm6,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm1 \n" + "vpandd %%zmm6,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm1,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm3,%%zmm2,%%zmm2 \n" + "vpsrlq $32,%%zmm2,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpmulld %%zmm7,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask]\n" + "kmovw %k[mask],%%k1 \n" + "vpmovqb %%zmm2,(%[dst_ptr])%{%%k1%} \n" + "jmp 99f \n" + + // Path 3: boxwidth >= 9 (ZMM 32-word reduction per output pixel). + "30: \n" + "mov %k[bw],%k[temp] \n" + "and $0x1f,%k[temp] \n" + "mov $-1,%k[mask] \n" + "bzhi %k[temp],%k[mask],%k[mask] \n" + "kmovd %k[mask],%%k1 \n" + + LABELALIGN + "31: \n" + "vpxord %%zmm2,%%zmm2,%%zmm2 \n" + "mov %[src_ptr],%q[mask] \n" + "lea (%[src_ptr],%q[bw],2),%[src_ptr] \n" + "mov %q[bw],%q[temp] \n" + "sub $0x20,%q[temp] \n" + "jl 33f \n" + "32: \n" + "vmovdqu16 (%q[mask]),%%zmm0 \n" + "lea 0x40(%q[mask]),%q[mask] \n" + "vpsrld $16,%%zmm0,%%zmm1 \n" + "vpandd %%zmm6,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "sub $0x20,%q[temp] \n" + "jge 32b \n" + "33: \n" + "add $0x20,%q[temp] \n" + "jle 34f \n" + "vmovdqu16 (%q[mask]),%%zmm0%{%%k1%}%{z%}\n" + "vpsrld $16,%%zmm0,%%zmm1 \n" + "vpandd %%zmm6,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "34: \n" + "vextracti32x8 $1,%%zmm2,%%ymm0 \n" + "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" + "vextracti128 $1,%%ymm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpshufd $0xee,%%xmm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpshufd $0x55,%%xmm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpmulld %%xmm7,%%xmm2,%%xmm2 \n" + "vpsrld $16,%%xmm2,%%xmm2 \n" + "vmovd %%xmm2,%k[temp] \n" + "mov %b[temp],(%[dst_ptr]) \n" + "inc %[dst_ptr] \n" + "dec %[dst_width] \n" + "jg 31b \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [dst_width] "+r"(dst_width), // %[dst_width] + [temp] "=&r"(temp), // %[temp] + [mask] "=&r"(mask) // %[mask] + : [bw] "r"((intptr_t)boxwidth), // %[bw] + [scaleval] "r"(scaleval), // %[scaleval] + [kSeq] "m"(kSequence0_15) // %[kSeq] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "k1", "k2", "k3", "k4"); +} + +void ScaleAddCols2_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr) { + int minboxwidth = dx >> 16; + int scaletbl[2]; + scaletbl[0] = 65536 / ((minboxwidth < 1 ? 1 : minboxwidth) * boxheight); + scaletbl[1] = + 65536 / (((minboxwidth + 1) < 1 ? 1 : (minboxwidth + 1)) * boxheight); + intptr_t base_ix, temp, mask; + asm volatile( + "mov $0x0000ffff,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm20 \n" + "vpbroadcastd %[scale0],%%zmm22 \n" + "vpbroadcastd %[scale1],%%zmm23 \n" + "cmpl $1,%[minbw] \n" + "jl 30f \n" + "cmpl $3,%[minbw] \n" + "jg 20f \n" + + // Path 1: 1 <= minboxwidth <= 3 (16 output pixels per loop). + "vpmovzxbd %[kSeq],%%zmm16 \n" + "vpbroadcastd %k[dx],%%zmm17 \n" + "vpmulld %%zmm17,%%zmm16,%%zmm16 \n" + "vpaddd %%zmm17,%%zmm16,%%zmm17 \n" + "mov $0x00010000,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm18 \n" + "mov $0x00030002,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm19 \n" + "vpbroadcastd %[minbw],%%zmm21 \n" + "sub $0x10,%[dst_width] \n" + "jl 12f \n" + + LABELALIGN + "11: \n" + "movslq %k[x],%q[base_ix] \n" + "movzwl %w[x],%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm2 \n" + "sar $16,%q[base_ix] \n" + "lea (%k[x],%k[dx],8),%k[temp] \n" + "lea (%k[temp],%k[dx],8),%k[x] \n" + "movslq %k[x],%q[temp] \n" + "sar $16,%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k2 \n" + "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n" + + "vpaddd %%zmm16,%%zmm2,%%zmm3 \n" + "vpaddd %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm3 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpsubd %%zmm3,%%zmm2,%%zmm2 \n" + "vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n" + + "vpslld $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpslld $16,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm19,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n" + "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n" + + "vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm4,%%zmm0 \n" + "vpandd %%zmm20,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm5,%%zmm1 \n" + "vpandd %%zmm20,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm1,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" + + "vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n" + "vpmulld %%zmm2,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm4,%%zmm4 \n" + "vpmovdb %%zmm4,(%[dst_ptr]) \n" + "lea 0x10(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x10,%[dst_width] \n" + "jge 11b \n" + + "12: \n" + "add $0x10,%[dst_width] \n" + "jle 99f \n" + "movslq %k[x],%q[base_ix] \n" + "movzwl %w[x],%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm2 \n" + "sar $16,%q[base_ix] \n" + "mov %k[dst_width],%k[temp] \n" + "imul %k[dx],%k[temp] \n" + "add %k[x],%k[temp] \n" + "sar $16,%k[temp] \n" + "movslq %k[temp],%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k2 \n" + "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n" + + "vpaddd %%zmm16,%%zmm2,%%zmm3 \n" + "vpaddd %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm3 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpsubd %%zmm3,%%zmm2,%%zmm2 \n" + "vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n" + + "vpslld $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpslld $16,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm19,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n" + "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n" + + "vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm4,%%zmm0 \n" + "vpandd %%zmm20,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm5,%%zmm1 \n" + "vpandd %%zmm20,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm1,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" + + "vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n" + "vpmulld %%zmm2,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm4,%%zmm4 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask]\n" + "kmovw %k[mask],%%k1 \n" + "vpmovdb %%zmm4,(%[dst_ptr])%{%%k1%} \n" + "jmp 99f \n" + + // Path 2: 4 <= minboxwidth <= 7 (8 output pixels per loop). + "20: \n" + "cmpl $7,%[minbw] \n" + "jg 30f \n" + "vpmovzxbq %[kSeq],%%zmm16 \n" + "movslq %k[dx],%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm17 \n" + "vpmulld %%zmm17,%%zmm16,%%zmm16 \n" + "vpaddq %%zmm17,%%zmm16,%%zmm17 \n" + "movabs $0x0003000200010000,%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm18 \n" + "movabs $0x0007000600050004,%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm19 \n" + "movslq %[minbw],%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm21 \n" + "sub $0x8,%[dst_width] \n" + "jl 22f \n" + + LABELALIGN + "21: \n" + "movslq %k[x],%q[base_ix] \n" + "movzwl %w[x],%k[temp] \n" + "vpbroadcastq %q[temp],%%zmm2 \n" + "sar $16,%q[base_ix] \n" + "lea (%k[x],%k[dx],8),%k[x] \n" + "movslq %k[x],%q[temp] \n" + "sar $16,%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k2 \n" + "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n" + + "vpaddd %%zmm16,%%zmm2,%%zmm3 \n" + "vpaddd %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm3 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpsubd %%zmm3,%%zmm2,%%zmm2 \n" + "vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n" + + "vpsllq $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $32,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $16,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpsllq $32,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm19,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n" + + "vpermi2w %%zmm1,%%zmm0,%%zmm4 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm4,%%zmm0 \n" + "vpandd %%zmm20,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm5,%%zmm1 \n" + "vpandd %%zmm20,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm1,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" + "vpsrlq $32,%%zmm4,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + + "vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n" + "vpmulld %%zmm2,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm4,%%zmm4 \n" + "vpmovqb %%zmm4,(%[dst_ptr]) \n" + "lea 0x8(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x8,%[dst_width] \n" + "jge 21b \n" + + "22: \n" + "add $0x8,%[dst_width] \n" + "jle 99f \n" + "movslq %k[x],%q[base_ix] \n" + "movzwl %w[x],%k[temp] \n" + "vpbroadcastq %q[temp],%%zmm2 \n" + "sar $16,%q[base_ix] \n" + "mov %k[dst_width],%k[temp] \n" + "imul %k[dx],%k[temp] \n" + "add %k[x],%k[temp] \n" + "sar $16,%k[temp] \n" + "movslq %k[temp],%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k2 \n" + "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n" + + "vpaddd %%zmm16,%%zmm2,%%zmm3 \n" + "vpaddd %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm3 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpsubd %%zmm3,%%zmm2,%%zmm2 \n" + "vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n" + + "vpsllq $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $32,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $16,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpsllq $32,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm19,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n" + + "vpermi2w %%zmm1,%%zmm0,%%zmm4 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm4,%%zmm0 \n" + "vpandd %%zmm20,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm5,%%zmm1 \n" + "vpandd %%zmm20,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm1,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" + "vpsrlq $32,%%zmm4,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + + "vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n" + "vpmulld %%zmm2,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm4,%%zmm4 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask]\n" + "kmovw %k[mask],%%k1 \n" + "vpmovqb %%zmm4,(%[dst_ptr])%{%%k1%} \n" + "jmp 99f \n" + + // Path 3: minboxwidth >= 8 or minboxwidth < 1. + LABELALIGN + "30: \n" + "movslq %k[x],%q[base_ix] \n" + "sar $16,%q[base_ix] \n" + "add %k[dx],%k[x] \n" + "movslq %k[x],%q[temp] \n" + "sar $16,%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "cmp $1,%q[temp] \n" + "jge 31f \n" + "mov $1,%q[temp] \n" + "31: \n" + "vmovdqa32 %%xmm22,%%xmm3 \n" + "cmp %k[temp],%[minbw] \n" + "jge 32f \n" + "vmovdqa32 %%xmm23,%%xmm3 \n" + "32: \n" + "vpxord %%zmm2,%%zmm2,%%zmm2 \n" + "lea (%[src_ptr],%q[base_ix],2),%q[base_ix] \n" + "sub $0x20,%q[temp] \n" + "jl 34f \n" + "33: \n" + "vmovdqu16 (%q[base_ix]),%%zmm0 \n" + "lea 0x40(%q[base_ix]),%q[base_ix] \n" + "vpsrld $16,%%zmm0,%%zmm1 \n" + "vpandd %%zmm20,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "sub $0x20,%q[temp] \n" + "jge 33b \n" + "34: \n" + "add $0x20,%q[temp] \n" + "jle 35f \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "vmovdqu16 (%q[base_ix]),%%zmm0%{%%k1%}%{z%} \n" + "vpsrld $16,%%zmm0,%%zmm1 \n" + "vpandd %%zmm20,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "35: \n" + "vextracti32x8 $1,%%zmm2,%%ymm0 \n" + "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" + "vextracti128 $1,%%ymm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpshufd $0xee,%%xmm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpshufd $0x55,%%xmm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpmulld %%xmm3,%%xmm2,%%xmm2 \n" + "vpsrld $16,%%xmm2,%%xmm2 \n" + "vmovd %%xmm2,%k[temp] \n" + "mov %b[temp],(%[dst_ptr]) \n" + "inc %[dst_ptr] \n" + "dec %[dst_width] \n" + "jg 30b \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [dst_width] "+r"(dst_width), // %[dst_width] + [x] "+r"(x), // %[x] + [base_ix] "=&r"(base_ix), // %[base_ix] + [temp] "=&r"(temp), // %[temp] + [mask] "=&r"(mask) // %[mask] + : [dx] "r"(dx), // %[dx] + [minbw] "m"(minboxwidth), // %[minbw] + [scale0] "m"(scaletbl[0]), // %[scale0] + [scale1] "m"(scaletbl[1]), // %[scale1] + [kSeq] "m"(kSequence0_15) // %[kSeq] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1", + "k2", "k3", "k4"); +} +#endif // HAS_SCALEADDCOLS_AVX512BW + static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040};