From 7320155784bdfefb69db8edddca37ecacf7e5198 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Fri, 25 Sep 2026 14:52:12 -0700 Subject: [PATCH] Add ScaleAddRow_AVX512BW and ScaleAddCols1/2_AVX512BW ================================================== Processor / Architecture Before After Speedup -------------------------------------------------- Intel Granite Rapids 838 us 222 us 3.77x Intel Emerald Rapids 823 us 237 us 3.47x Intel Sapphire Rapids 884 us 241 us 3.67x AMD Turin (Zen 5) 684 us 119 us 5.75x AMD Genoa (Zen 4) 858 us 233 us 3.68x Intel Skylake-X 1421 us 319 us 4.45x Intel Ice Lake 1105 us 284 us 3.89x AMD Milan (Zen 3) 980 us 978 us 1.00x Intel Haswell 1356 us 1387 us 0.98x ================================================== Bug: None Change-Id: Id5473082cc2537234db9c53d53c38f2c9698d9d2 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8467771 Reviewed-by: richard winterton Commit-Queue: Frank Barchard --- include/libyuv/scale_row.h | 23 ++ source/scale.cc | 12 + source/scale_gcc.cc | 668 +++++++++++++++++++++++++++++++++++++ 3 files changed, 703 insertions(+) diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index f18cca24d..ee814d7f3 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -93,6 +93,14 @@ extern "C" { #define HAS_SCALEROWDOWN4_AVX2 #endif +// The following are available for AVX512 clang x64 platforms: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \ + !defined(LIBYUV_ENABLE_ROWWIN) +#define HAS_SCALEADDROW_AVX512BW +#define HAS_SCALEADDCOLS_AVX512BW +#endif + // The following are available on Neon platforms: #if !defined(LIBYUV_DISABLE_NEON) && \ (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) @@ -798,12 +806,27 @@ void ScaleRowDown38_2_Box_Any_SSSE3(const uint8_t* src_ptr, void ScaleAddRow_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); +void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width); void ScaleAddRow_Any_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); void ScaleAddRow_Any_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); +void ScaleAddCols1_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr); +void ScaleAddCols2_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr); void ScaleFilterCols_SSSE3(uint8_t* dst_ptr, const uint8_t* src_ptr, diff --git a/source/scale.cc b/source/scale.cc index 114514197..fc89a5545 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -900,6 +900,18 @@ static int ScalePlaneBox(int src_width, } } #endif +#if defined(HAS_SCALEADDROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ScaleAddRow = ScaleAddRow_AVX512BW; + } +#endif +#if defined(HAS_SCALEADDCOLS_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0) { + ScaleAddCols = (dx & 0xffff) ? ScaleAddCols2_AVX512BW + : ((dx != 0x10000) ? ScaleAddCols1_AVX512BW + : ScaleAddCols0_C); + } +#endif #if defined(HAS_SCALEADDROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ScaleAddRow = ScaleAddRow_Any_NEON; diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index b2a289ed4..6c93933c8 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -1792,6 +1792,674 @@ void ScaleAddRow_AVX2(const uint8_t* src_ptr, } #endif // HAS_SCALEADDROW_AVX2 +#ifdef HAS_SCALEADDROW_AVX512BW +// Reads 32xN bytes and accumulates to 32xN shorts at a time, with masked tail. +void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width) { + uintptr_t temp; + asm volatile( + "sub $0x40,%2 \n" + "jl 2f \n" + + // 64 pixel loop. + LABELALIGN + "1: \n" + "vpmovzxbw (%0),%%zmm0 \n" + "vpmovzxbw 0x20(%0),%%zmm1 \n" + "lea 0x40(%0),%0 \n" + "vpaddusw (%1),%%zmm0,%%zmm0 \n" + "vpaddusw 0x40(%1),%%zmm1,%%zmm1 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "jl 3f \n" + "vpmovzxbw (%0),%%zmm0 \n" + "lea 0x20(%0),%0 \n" + "vpaddusw (%1),%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + + "3: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "mov $-1,%k3 \n" + "bzhi %k2,%k3,%k3 \n" + "kmovd %k3,%%k1 \n" + "vpmovzxbw (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpaddusw (%1),%%zmm0,%%zmm0%{%%k1%}%{z%}\n" + "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(src_width), // %2 + "=&r"(temp) // %3 + : + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_SCALEADDROW_AVX512BW + +#ifdef HAS_SCALEADDCOLS_AVX512BW +static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15}; + +void ScaleAddCols1_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr) { + int boxwidth = dx >> 16; + if (boxwidth < 1) { + boxwidth = 1; + } + int scaleval = 65536 / (boxwidth * boxheight); + src_ptr += (x >> 16); + intptr_t temp, mask; + asm volatile( + "mov $0x0000ffff,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm6 \n" + "vpbroadcastd %k[scaleval],%%zmm7 \n" + "cmp $4,%k[bw] \n" + "jg 20f \n" + + // Path 1: 1 <= boxwidth <= 4 (16 output pixels per loop). + "vpmovzxbd %[kSeq],%%zmm3 \n" + "vpslld $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpbroadcastw %k[bw],%%zmm2 \n" + "vpmullw %%zmm2,%%zmm3,%%zmm3 \n" + "mov $0x00010000,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm0 \n" + "mov $0x00030002,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm1 \n" + "vpaddw %%zmm0,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm1,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm0,%%zmm2,%%k2 \n" + "vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n" + "mov %k[bw],%k[temp] \n" + "shl $4,%k[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k4 \n" + "shl $1,%q[temp] \n" + "sub $0x10,%[dst_width] \n" + "jl 12f \n" + + LABELALIGN + "11: \n" + "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n" + "add %q[temp],%[src_ptr] \n" + "vmovdqa64 %%zmm4,%%zmm2 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n" + "vmovdqa64 %%zmm5,%%zmm3 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm2,%%zmm0 \n" + "vpandd %%zmm6,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm1 \n" + "vpandd %%zmm6,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm1,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm3,%%zmm2,%%zmm2 \n" + "vpmulld %%zmm7,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpmovdb %%zmm2,(%[dst_ptr]) \n" + "lea 0x10(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x10,%[dst_width] \n" + "jge 11b \n" + + "12: \n" + "add $0x10,%[dst_width] \n" + "jle 99f \n" + "mov %k[dst_width],%k[temp] \n" + "imul %k[bw],%k[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k4 \n" + "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n" + "vmovdqa64 %%zmm4,%%zmm2 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n" + "vmovdqa64 %%zmm5,%%zmm3 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm2,%%zmm0 \n" + "vpandd %%zmm6,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm1 \n" + "vpandd %%zmm6,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm1,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm3,%%zmm2,%%zmm2 \n" + "vpmulld %%zmm7,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask]\n" + "kmovw %k[mask],%%k1 \n" + "vpmovdb %%zmm2,(%[dst_ptr])%{%%k1%} \n" + "jmp 99f \n" + + // Path 2: 5 <= boxwidth <= 8 (8 output pixels per loop). + "20: \n" + "cmp $8,%k[bw] \n" + "jg 30f \n" + "vpmovzxbq %[kSeq],%%zmm3 \n" + "vpsllq $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $32,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpbroadcastw %k[bw],%%zmm2 \n" + "vpmullw %%zmm2,%%zmm3,%%zmm3 \n" + "movabs $0x0003000200010000,%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm0 \n" + "movabs $0x0007000600050004,%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm1 \n" + "vpaddw %%zmm0,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm1,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n" + "mov %k[bw],%k[temp] \n" + "shl $3,%k[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k4 \n" + "shl $1,%q[temp] \n" + "sub $0x8,%[dst_width] \n" + "jl 22f \n" + + LABELALIGN + "21: \n" + "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n" + "add %q[temp],%[src_ptr] \n" + "vmovdqa64 %%zmm4,%%zmm2 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm2 \n" + "vmovdqa64 %%zmm5,%%zmm3 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm2,%%zmm0 \n" + "vpandd %%zmm6,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm1 \n" + "vpandd %%zmm6,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm1,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm3,%%zmm2,%%zmm2 \n" + "vpsrlq $32,%%zmm2,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpmulld %%zmm7,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpmovqb %%zmm2,(%[dst_ptr]) \n" + "lea 0x8(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x8,%[dst_width] \n" + "jge 21b \n" + + "22: \n" + "add $0x8,%[dst_width] \n" + "jle 99f \n" + "mov %k[dst_width],%k[temp] \n" + "imul %k[bw],%k[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k4 \n" + "vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n" + "vmovdqa64 %%zmm4,%%zmm2 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm2 \n" + "vmovdqa64 %%zmm5,%%zmm3 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm2,%%zmm0 \n" + "vpandd %%zmm6,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm1 \n" + "vpandd %%zmm6,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm1,%%zmm3,%%zmm3 \n" + "vpaddd %%zmm3,%%zmm2,%%zmm2 \n" + "vpsrlq $32,%%zmm2,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "vpmulld %%zmm7,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask]\n" + "kmovw %k[mask],%%k1 \n" + "vpmovqb %%zmm2,(%[dst_ptr])%{%%k1%} \n" + "jmp 99f \n" + + // Path 3: boxwidth >= 9 (ZMM 32-word reduction per output pixel). + "30: \n" + "mov %k[bw],%k[temp] \n" + "and $0x1f,%k[temp] \n" + "mov $-1,%k[mask] \n" + "bzhi %k[temp],%k[mask],%k[mask] \n" + "kmovd %k[mask],%%k1 \n" + + LABELALIGN + "31: \n" + "vpxord %%zmm2,%%zmm2,%%zmm2 \n" + "mov %[src_ptr],%q[mask] \n" + "lea (%[src_ptr],%q[bw],2),%[src_ptr] \n" + "mov %q[bw],%q[temp] \n" + "sub $0x20,%q[temp] \n" + "jl 33f \n" + "32: \n" + "vmovdqu16 (%q[mask]),%%zmm0 \n" + "lea 0x40(%q[mask]),%q[mask] \n" + "vpsrld $16,%%zmm0,%%zmm1 \n" + "vpandd %%zmm6,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "sub $0x20,%q[temp] \n" + "jge 32b \n" + "33: \n" + "add $0x20,%q[temp] \n" + "jle 34f \n" + "vmovdqu16 (%q[mask]),%%zmm0%{%%k1%}%{z%}\n" + "vpsrld $16,%%zmm0,%%zmm1 \n" + "vpandd %%zmm6,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "34: \n" + "vextracti32x8 $1,%%zmm2,%%ymm0 \n" + "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" + "vextracti128 $1,%%ymm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpshufd $0xee,%%xmm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpshufd $0x55,%%xmm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpmulld %%xmm7,%%xmm2,%%xmm2 \n" + "vpsrld $16,%%xmm2,%%xmm2 \n" + "vmovd %%xmm2,%k[temp] \n" + "mov %b[temp],(%[dst_ptr]) \n" + "inc %[dst_ptr] \n" + "dec %[dst_width] \n" + "jg 31b \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [dst_width] "+r"(dst_width), // %[dst_width] + [temp] "=&r"(temp), // %[temp] + [mask] "=&r"(mask) // %[mask] + : [bw] "r"((intptr_t)boxwidth), // %[bw] + [scaleval] "r"(scaleval), // %[scaleval] + [kSeq] "m"(kSequence0_15) // %[kSeq] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "k1", "k2", "k3", "k4"); +} + +void ScaleAddCols2_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr) { + int minboxwidth = dx >> 16; + int scaletbl[2]; + scaletbl[0] = 65536 / ((minboxwidth < 1 ? 1 : minboxwidth) * boxheight); + scaletbl[1] = + 65536 / (((minboxwidth + 1) < 1 ? 1 : (minboxwidth + 1)) * boxheight); + intptr_t base_ix, temp, mask; + asm volatile( + "mov $0x0000ffff,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm20 \n" + "vpbroadcastd %[scale0],%%zmm22 \n" + "vpbroadcastd %[scale1],%%zmm23 \n" + "cmpl $1,%[minbw] \n" + "jl 30f \n" + "cmpl $3,%[minbw] \n" + "jg 20f \n" + + // Path 1: 1 <= minboxwidth <= 3 (16 output pixels per loop). + "vpmovzxbd %[kSeq],%%zmm16 \n" + "vpbroadcastd %k[dx],%%zmm17 \n" + "vpmulld %%zmm17,%%zmm16,%%zmm16 \n" + "vpaddd %%zmm17,%%zmm16,%%zmm17 \n" + "mov $0x00010000,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm18 \n" + "mov $0x00030002,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm19 \n" + "vpbroadcastd %[minbw],%%zmm21 \n" + "sub $0x10,%[dst_width] \n" + "jl 12f \n" + + LABELALIGN + "11: \n" + "movslq %k[x],%q[base_ix] \n" + "movzwl %w[x],%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm2 \n" + "sar $16,%q[base_ix] \n" + "lea (%k[x],%k[dx],8),%k[temp] \n" + "lea (%k[temp],%k[dx],8),%k[x] \n" + "movslq %k[x],%q[temp] \n" + "sar $16,%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k2 \n" + "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n" + + "vpaddd %%zmm16,%%zmm2,%%zmm3 \n" + "vpaddd %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm3 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpsubd %%zmm3,%%zmm2,%%zmm2 \n" + "vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n" + + "vpslld $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpslld $16,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm19,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n" + "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n" + + "vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm4,%%zmm0 \n" + "vpandd %%zmm20,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm5,%%zmm1 \n" + "vpandd %%zmm20,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm1,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" + + "vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n" + "vpmulld %%zmm2,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm4,%%zmm4 \n" + "vpmovdb %%zmm4,(%[dst_ptr]) \n" + "lea 0x10(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x10,%[dst_width] \n" + "jge 11b \n" + + "12: \n" + "add $0x10,%[dst_width] \n" + "jle 99f \n" + "movslq %k[x],%q[base_ix] \n" + "movzwl %w[x],%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm2 \n" + "sar $16,%q[base_ix] \n" + "mov %k[dst_width],%k[temp] \n" + "imul %k[dx],%k[temp] \n" + "add %k[x],%k[temp] \n" + "sar $16,%k[temp] \n" + "movslq %k[temp],%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k2 \n" + "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n" + + "vpaddd %%zmm16,%%zmm2,%%zmm3 \n" + "vpaddd %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm3 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpsubd %%zmm3,%%zmm2,%%zmm2 \n" + "vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n" + + "vpslld $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpslld $16,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm19,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n" + "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n" + + "vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm4,%%zmm0 \n" + "vpandd %%zmm20,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm5,%%zmm1 \n" + "vpandd %%zmm20,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm1,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" + + "vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n" + "vpmulld %%zmm2,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm4,%%zmm4 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask]\n" + "kmovw %k[mask],%%k1 \n" + "vpmovdb %%zmm4,(%[dst_ptr])%{%%k1%} \n" + "jmp 99f \n" + + // Path 2: 4 <= minboxwidth <= 7 (8 output pixels per loop). + "20: \n" + "cmpl $7,%[minbw] \n" + "jg 30f \n" + "vpmovzxbq %[kSeq],%%zmm16 \n" + "movslq %k[dx],%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm17 \n" + "vpmulld %%zmm17,%%zmm16,%%zmm16 \n" + "vpaddq %%zmm17,%%zmm16,%%zmm17 \n" + "movabs $0x0003000200010000,%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm18 \n" + "movabs $0x0007000600050004,%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm19 \n" + "movslq %[minbw],%q[temp] \n" + "vpbroadcastq %q[temp],%%zmm21 \n" + "sub $0x8,%[dst_width] \n" + "jl 22f \n" + + LABELALIGN + "21: \n" + "movslq %k[x],%q[base_ix] \n" + "movzwl %w[x],%k[temp] \n" + "vpbroadcastq %q[temp],%%zmm2 \n" + "sar $16,%q[base_ix] \n" + "lea (%k[x],%k[dx],8),%k[x] \n" + "movslq %k[x],%q[temp] \n" + "sar $16,%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k2 \n" + "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n" + + "vpaddd %%zmm16,%%zmm2,%%zmm3 \n" + "vpaddd %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm3 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpsubd %%zmm3,%%zmm2,%%zmm2 \n" + "vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n" + + "vpsllq $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $32,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $16,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpsllq $32,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm19,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n" + + "vpermi2w %%zmm1,%%zmm0,%%zmm4 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm4,%%zmm0 \n" + "vpandd %%zmm20,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm5,%%zmm1 \n" + "vpandd %%zmm20,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm1,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" + "vpsrlq $32,%%zmm4,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + + "vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n" + "vpmulld %%zmm2,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm4,%%zmm4 \n" + "vpmovqb %%zmm4,(%[dst_ptr]) \n" + "lea 0x8(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x8,%[dst_width] \n" + "jge 21b \n" + + "22: \n" + "add $0x8,%[dst_width] \n" + "jle 99f \n" + "movslq %k[x],%q[base_ix] \n" + "movzwl %w[x],%k[temp] \n" + "vpbroadcastq %q[temp],%%zmm2 \n" + "sar $16,%q[base_ix] \n" + "mov %k[dst_width],%k[temp] \n" + "imul %k[dx],%k[temp] \n" + "add %k[x],%k[temp] \n" + "sar $16,%k[temp] \n" + "movslq %k[temp],%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "shr $32,%q[mask] \n" + "kmovd %k[mask],%%k2 \n" + "vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n" + + "vpaddd %%zmm16,%%zmm2,%%zmm3 \n" + "vpaddd %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrld $16,%%zmm3,%%zmm3 \n" + "vpsrld $16,%%zmm2,%%zmm2 \n" + "vpsubd %%zmm3,%%zmm2,%%zmm2 \n" + "vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n" + + "vpsllq $16,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $32,%%zmm3,%%zmm4 \n" + "vpord %%zmm4,%%zmm3,%%zmm3 \n" + "vpsllq $16,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpsllq $32,%%zmm2,%%zmm4 \n" + "vpord %%zmm4,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm4 \n" + "vpaddw %%zmm19,%%zmm3,%%zmm5 \n" + "vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n" + + "vpermi2w %%zmm1,%%zmm0,%%zmm4 \n" + "vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n" + "vpsrld $16,%%zmm4,%%zmm0 \n" + "vpandd %%zmm20,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm5,%%zmm1 \n" + "vpandd %%zmm20,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm1,%%zmm5,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" + "vpsrlq $32,%%zmm4,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm4,%%zmm4 \n" + + "vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n" + "vpmulld %%zmm2,%%zmm4,%%zmm4 \n" + "vpsrld $16,%%zmm4,%%zmm4 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask]\n" + "kmovw %k[mask],%%k1 \n" + "vpmovqb %%zmm4,(%[dst_ptr])%{%%k1%} \n" + "jmp 99f \n" + + // Path 3: minboxwidth >= 8 or minboxwidth < 1. + LABELALIGN + "30: \n" + "movslq %k[x],%q[base_ix] \n" + "sar $16,%q[base_ix] \n" + "add %k[dx],%k[x] \n" + "movslq %k[x],%q[temp] \n" + "sar $16,%q[temp] \n" + "sub %q[base_ix],%q[temp] \n" + "cmp $1,%q[temp] \n" + "jge 31f \n" + "mov $1,%q[temp] \n" + "31: \n" + "vmovdqa32 %%xmm22,%%xmm3 \n" + "cmp %k[temp],%[minbw] \n" + "jge 32f \n" + "vmovdqa32 %%xmm23,%%xmm3 \n" + "32: \n" + "vpxord %%zmm2,%%zmm2,%%zmm2 \n" + "lea (%[src_ptr],%q[base_ix],2),%q[base_ix] \n" + "sub $0x20,%q[temp] \n" + "jl 34f \n" + "33: \n" + "vmovdqu16 (%q[base_ix]),%%zmm0 \n" + "lea 0x40(%q[base_ix]),%q[base_ix] \n" + "vpsrld $16,%%zmm0,%%zmm1 \n" + "vpandd %%zmm20,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "sub $0x20,%q[temp] \n" + "jge 33b \n" + "34: \n" + "add $0x20,%q[temp] \n" + "jle 35f \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovd %k[mask],%%k1 \n" + "vmovdqu16 (%q[base_ix]),%%zmm0%{%%k1%}%{z%} \n" + "vpsrld $16,%%zmm0,%%zmm1 \n" + "vpandd %%zmm20,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm0,%%zmm2,%%zmm2 \n" + "35: \n" + "vextracti32x8 $1,%%zmm2,%%ymm0 \n" + "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" + "vextracti128 $1,%%ymm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpshufd $0xee,%%xmm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpshufd $0x55,%%xmm2,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpmulld %%xmm3,%%xmm2,%%xmm2 \n" + "vpsrld $16,%%xmm2,%%xmm2 \n" + "vmovd %%xmm2,%k[temp] \n" + "mov %b[temp],(%[dst_ptr]) \n" + "inc %[dst_ptr] \n" + "dec %[dst_width] \n" + "jg 30b \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [dst_width] "+r"(dst_width), // %[dst_width] + [x] "+r"(x), // %[x] + [base_ix] "=&r"(base_ix), // %[base_ix] + [temp] "=&r"(temp), // %[temp] + [mask] "=&r"(mask) // %[mask] + : [dx] "r"(dx), // %[dx] + [minbw] "m"(minboxwidth), // %[minbw] + [scale0] "m"(scaletbl[0]), // %[scale0] + [scale1] "m"(scaletbl[1]), // %[scale1] + [kSeq] "m"(kSequence0_15) // %[kSeq] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1", + "k2", "k3", "k4"); +} +#endif // HAS_SCALEADDCOLS_AVX512BW + static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040};