coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add ScaleAddRow_AVX512BW and ScaleAddCols1/2_AVX512BW

Browse Source 
==================================================
Processor / Architecture   Before    After Speedup
--------------------------------------------------
Intel Granite Rapids       838 us   222 us   3.77x
Intel Emerald Rapids       823 us   237 us   3.47x
Intel Sapphire Rapids      884 us   241 us   3.67x
AMD Turin (Zen 5)          684 us   119 us   5.75x
AMD Genoa (Zen 4)          858 us   233 us   3.68x
Intel Skylake-X           1421 us   319 us   4.45x
Intel Ice Lake            1105 us   284 us   3.89x
AMD Milan (Zen 3)          980 us   978 us   1.00x
Intel Haswell             1356 us  1387 us   0.98x
==================================================

Bug: None
Change-Id: Id5473082cc2537234db9c53d53c38f2c9698d9d2
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8467771
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard 2026-09-25 14:52:12 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent b5c764cfdd
commit 7320155784
3 changed files with 703 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
include/libyuv/scale_row.h
View File

@ -93,6 +93,14 @@ extern "C" {
#define HAS_SCALEROWDOWN4_AVX2
#endif
// The following are available for AVX512 clang x64 platforms:
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \
!defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_SCALEADDROW_AVX512BW
#define HAS_SCALEADDCOLS_AVX512BW
#endif
// The following are available on Neon platforms:
#if !defined(LIBYUV_DISABLE_NEON) && \
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
@ -798,12 +806,27 @@ void ScaleRowDown38_2_Box_Any_SSSE3(const uint8_t* src_ptr,
void ScaleAddRow_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);
void ScaleAddRow_Any_SSE2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);
void ScaleAddRow_Any_AVX2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);
void ScaleAddCols1_AVX512BW(int dst_width,
int boxheight,
int x,
int dx,
const uint16_t* src_ptr,
uint8_t* dst_ptr);
void ScaleAddCols2_AVX512BW(int dst_width,
int boxheight,
int x,
int dx,
const uint16_t* src_ptr,
uint8_t* dst_ptr);
void ScaleFilterCols_SSSE3(uint8_t* dst_ptr,
const uint8_t* src_ptr,

12
source/scale.cc
View File

@ -900,6 +900,18 @@ static int ScalePlaneBox(int src_width,
}
}
#endif
#if defined(HAS_SCALEADDROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ScaleAddRow = ScaleAddRow_AVX512BW;
}
#endif
#if defined(HAS_SCALEADDCOLS_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0) {
ScaleAddCols = (dx & 0xffff) ? ScaleAddCols2_AVX512BW
: ((dx != 0x10000) ? ScaleAddCols1_AVX512BW
: ScaleAddCols0_C);
}
#endif
#if defined(HAS_SCALEADDROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleAddRow = ScaleAddRow_Any_NEON;

668
source/scale_gcc.cc
View File

@ -1792,6 +1792,674 @@ void ScaleAddRow_AVX2(const uint8_t* src_ptr,
}
#endif // HAS_SCALEADDROW_AVX2
#ifdef HAS_SCALEADDROW_AVX512BW
// Reads 32xN bytes and accumulates to 32xN shorts at a time, with masked tail.
void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width) {
uintptr_t temp;
asm volatile(
"sub $0x40,%2 \n"
"jl 2f \n"
// 64 pixel loop.
LABELALIGN
"1: \n"
"vpmovzxbw (%0),%%zmm0 \n"
"vpmovzxbw 0x20(%0),%%zmm1 \n"
"lea 0x40(%0),%0 \n"
"vpaddusw (%1),%%zmm0,%%zmm0 \n"
"vpaddusw 0x40(%1),%%zmm1,%%zmm1 \n"
"vmovdqu16 %%zmm0,(%1) \n"
"vmovdqu16 %%zmm1,0x40(%1) \n"
"lea 0x80(%1),%1 \n"
"sub $0x40,%2 \n"
"jge 1b \n"
"2: \n"
"add $0x20,%2 \n"
"jl 3f \n"
"vpmovzxbw (%0),%%zmm0 \n"
"lea 0x20(%0),%0 \n"
"vpaddusw (%1),%%zmm0,%%zmm0 \n"
"vmovdqu16 %%zmm0,(%1) \n"
"lea 0x40(%1),%1 \n"
"sub $0x20,%2 \n"
"3: \n"
"add $0x20,%2 \n"
"jle 99f \n"
"mov $-1,%k3 \n"
"bzhi %k2,%k3,%k3 \n"
"kmovd %k3,%%k1 \n"
"vpmovzxbw (%0),%%zmm0%{%%k1%}%{z%} \n"
"vpaddusw (%1),%%zmm0,%%zmm0%{%%k1%}%{z%}\n"
"vmovdqu16 %%zmm0,(%1)%{%%k1%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(src_width), // %2
"=&r"(temp) // %3
:
: "memory", "cc", "xmm0", "xmm1", "k1");
}
#endif // HAS_SCALEADDROW_AVX512BW
#ifdef HAS_SCALEADDCOLS_AVX512BW
static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15};
void ScaleAddCols1_AVX512BW(int dst_width,
int boxheight,
int x,
int dx,
const uint16_t* src_ptr,
uint8_t* dst_ptr) {
int boxwidth = dx >> 16;
if (boxwidth < 1) {
boxwidth = 1;
}
int scaleval = 65536 / (boxwidth * boxheight);
src_ptr += (x >> 16);
intptr_t temp, mask;
asm volatile(
"mov $0x0000ffff,%k[temp] \n"
"vpbroadcastd %k[temp],%%zmm6 \n"
"vpbroadcastd %k[scaleval],%%zmm7 \n"
"cmp $4,%k[bw] \n"
"jg 20f \n"
// Path 1: 1 <= boxwidth <= 4 (16 output pixels per loop).
"vpmovzxbd %[kSeq],%%zmm3 \n"
"vpslld $16,%%zmm3,%%zmm4 \n"
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
"vpbroadcastw %k[bw],%%zmm2 \n"
"vpmullw %%zmm2,%%zmm3,%%zmm3 \n"
"mov $0x00010000,%k[temp] \n"
"vpbroadcastd %k[temp],%%zmm0 \n"
"mov $0x00030002,%k[temp] \n"
"vpbroadcastd %k[temp],%%zmm1 \n"
"vpaddw %%zmm0,%%zmm3,%%zmm4 \n"
"vpaddw %%zmm1,%%zmm3,%%zmm5 \n"
"vpcmpuw $6,%%zmm0,%%zmm2,%%k2 \n"
"vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n"
"mov %k[bw],%k[temp] \n"
"shl $4,%k[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovd %k[mask],%%k1 \n"
"shr $32,%q[mask] \n"
"kmovd %k[mask],%%k4 \n"
"shl $1,%q[temp] \n"
"sub $0x10,%[dst_width] \n"
"jl 12f \n"
LABELALIGN
"11: \n"
"vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
"add %q[temp],%[src_ptr] \n"
"vmovdqa64 %%zmm4,%%zmm2 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n"
"vmovdqa64 %%zmm5,%%zmm3 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
"vpsrld $16,%%zmm2,%%zmm0 \n"
"vpandd %%zmm6,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm3,%%zmm1 \n"
"vpandd %%zmm6,%%zmm3,%%zmm3 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
"vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
"vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm2,%%zmm2 \n"
"vpmovdb %%zmm2,(%[dst_ptr]) \n"
"lea 0x10(%[dst_ptr]),%[dst_ptr] \n"
"sub $0x10,%[dst_width] \n"
"jge 11b \n"
"12: \n"
"add $0x10,%[dst_width] \n"
"jle 99f \n"
"mov %k[dst_width],%k[temp] \n"
"imul %k[bw],%k[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovd %k[mask],%%k1 \n"
"shr $32,%q[mask] \n"
"kmovd %k[mask],%%k4 \n"
"vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
"vmovdqa64 %%zmm4,%%zmm2 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n"
"vmovdqa64 %%zmm5,%%zmm3 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
"vpsrld $16,%%zmm2,%%zmm0 \n"
"vpandd %%zmm6,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm3,%%zmm1 \n"
"vpandd %%zmm6,%%zmm3,%%zmm3 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
"vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
"vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm2,%%zmm2 \n"
"mov $-1,%q[mask] \n"
"bzhi %q[dst_width],%q[mask],%q[mask]\n"
"kmovw %k[mask],%%k1 \n"
"vpmovdb %%zmm2,(%[dst_ptr])%{%%k1%} \n"
"jmp 99f \n"
// Path 2: 5 <= boxwidth <= 8 (8 output pixels per loop).
"20: \n"
"cmp $8,%k[bw] \n"
"jg 30f \n"
"vpmovzxbq %[kSeq],%%zmm3 \n"
"vpsllq $16,%%zmm3,%%zmm4 \n"
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
"vpsllq $32,%%zmm3,%%zmm4 \n"
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
"vpbroadcastw %k[bw],%%zmm2 \n"
"vpmullw %%zmm2,%%zmm3,%%zmm3 \n"
"movabs $0x0003000200010000,%q[temp] \n"
"vpbroadcastq %q[temp],%%zmm0 \n"
"movabs $0x0007000600050004,%q[temp] \n"
"vpbroadcastq %q[temp],%%zmm1 \n"
"vpaddw %%zmm0,%%zmm3,%%zmm4 \n"
"vpaddw %%zmm1,%%zmm3,%%zmm5 \n"
"vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n"
"mov %k[bw],%k[temp] \n"
"shl $3,%k[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovd %k[mask],%%k1 \n"
"shr $32,%q[mask] \n"
"kmovd %k[mask],%%k4 \n"
"shl $1,%q[temp] \n"
"sub $0x8,%[dst_width] \n"
"jl 22f \n"
LABELALIGN
"21: \n"
"vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
"add %q[temp],%[src_ptr] \n"
"vmovdqa64 %%zmm4,%%zmm2 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm2 \n"
"vmovdqa64 %%zmm5,%%zmm3 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
"vpsrld $16,%%zmm2,%%zmm0 \n"
"vpandd %%zmm6,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm3,%%zmm1 \n"
"vpandd %%zmm6,%%zmm3,%%zmm3 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
"vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
"vpsrlq $32,%%zmm2,%%zmm0 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm2,%%zmm2 \n"
"vpmovqb %%zmm2,(%[dst_ptr]) \n"
"lea 0x8(%[dst_ptr]),%[dst_ptr] \n"
"sub $0x8,%[dst_width] \n"
"jge 21b \n"
"22: \n"
"add $0x8,%[dst_width] \n"
"jle 99f \n"
"mov %k[dst_width],%k[temp] \n"
"imul %k[bw],%k[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovd %k[mask],%%k1 \n"
"shr $32,%q[mask] \n"
"kmovd %k[mask],%%k4 \n"
"vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
"vmovdqa64 %%zmm4,%%zmm2 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm2 \n"
"vmovdqa64 %%zmm5,%%zmm3 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
"vpsrld $16,%%zmm2,%%zmm0 \n"
"vpandd %%zmm6,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm3,%%zmm1 \n"
"vpandd %%zmm6,%%zmm3,%%zmm3 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
"vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
"vpsrlq $32,%%zmm2,%%zmm0 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm2,%%zmm2 \n"
"mov $-1,%q[mask] \n"
"bzhi %q[dst_width],%q[mask],%q[mask]\n"
"kmovw %k[mask],%%k1 \n"
"vpmovqb %%zmm2,(%[dst_ptr])%{%%k1%} \n"
"jmp 99f \n"
// Path 3: boxwidth >= 9 (ZMM 32-word reduction per output pixel).
"30: \n"
"mov %k[bw],%k[temp] \n"
"and $0x1f,%k[temp] \n"
"mov $-1,%k[mask] \n"
"bzhi %k[temp],%k[mask],%k[mask] \n"
"kmovd %k[mask],%%k1 \n"
LABELALIGN
"31: \n"
"vpxord %%zmm2,%%zmm2,%%zmm2 \n"
"mov %[src_ptr],%q[mask] \n"
"lea (%[src_ptr],%q[bw],2),%[src_ptr] \n"
"mov %q[bw],%q[temp] \n"
"sub $0x20,%q[temp] \n"
"jl 33f \n"
"32: \n"
"vmovdqu16 (%q[mask]),%%zmm0 \n"
"lea 0x40(%q[mask]),%q[mask] \n"
"vpsrld $16,%%zmm0,%%zmm1 \n"
"vpandd %%zmm6,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"sub $0x20,%q[temp] \n"
"jge 32b \n"
"33: \n"
"add $0x20,%q[temp] \n"
"jle 34f \n"
"vmovdqu16 (%q[mask]),%%zmm0%{%%k1%}%{z%}\n"
"vpsrld $16,%%zmm0,%%zmm1 \n"
"vpandd %%zmm6,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"34: \n"
"vextracti32x8 $1,%%zmm2,%%ymm0 \n"
"vpaddd %%ymm0,%%ymm2,%%ymm2 \n"
"vextracti128 $1,%%ymm2,%%xmm0 \n"
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
"vpshufd $0xee,%%xmm2,%%xmm0 \n"
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
"vpshufd $0x55,%%xmm2,%%xmm0 \n"
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
"vpmulld %%xmm7,%%xmm2,%%xmm2 \n"
"vpsrld $16,%%xmm2,%%xmm2 \n"
"vmovd %%xmm2,%k[temp] \n"
"mov %b[temp],(%[dst_ptr]) \n"
"inc %[dst_ptr] \n"
"dec %[dst_width] \n"
"jg 31b \n"
"99: \n"
"vzeroupper \n"
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
[dst_width] "+r"(dst_width), // %[dst_width]
[temp] "=&r"(temp), // %[temp]
[mask] "=&r"(mask) // %[mask]
: [bw] "r"((intptr_t)boxwidth), // %[bw]
[scaleval] "r"(scaleval), // %[scaleval]
[kSeq] "m"(kSequence0_15) // %[kSeq]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "k1", "k2", "k3", "k4");
}
void ScaleAddCols2_AVX512BW(int dst_width,
int boxheight,
int x,
int dx,
const uint16_t* src_ptr,
uint8_t* dst_ptr) {
int minboxwidth = dx >> 16;
int scaletbl[2];
scaletbl[0] = 65536 / ((minboxwidth < 1 ? 1 : minboxwidth) * boxheight);
scaletbl[1] =
65536 / (((minboxwidth + 1) < 1 ? 1 : (minboxwidth + 1)) * boxheight);
intptr_t base_ix, temp, mask;
asm volatile(
"mov $0x0000ffff,%k[temp] \n"
"vpbroadcastd %k[temp],%%zmm20 \n"
"vpbroadcastd %[scale0],%%zmm22 \n"
"vpbroadcastd %[scale1],%%zmm23 \n"
"cmpl $1,%[minbw] \n"
"jl 30f \n"
"cmpl $3,%[minbw] \n"
"jg 20f \n"
// Path 1: 1 <= minboxwidth <= 3 (16 output pixels per loop).
"vpmovzxbd %[kSeq],%%zmm16 \n"
"vpbroadcastd %k[dx],%%zmm17 \n"
"vpmulld %%zmm17,%%zmm16,%%zmm16 \n"
"vpaddd %%zmm17,%%zmm16,%%zmm17 \n"
"mov $0x00010000,%k[temp] \n"
"vpbroadcastd %k[temp],%%zmm18 \n"
"mov $0x00030002,%k[temp] \n"
"vpbroadcastd %k[temp],%%zmm19 \n"
"vpbroadcastd %[minbw],%%zmm21 \n"
"sub $0x10,%[dst_width] \n"
"jl 12f \n"
LABELALIGN
"11: \n"
"movslq %k[x],%q[base_ix] \n"
"movzwl %w[x],%k[temp] \n"
"vpbroadcastd %k[temp],%%zmm2 \n"
"sar $16,%q[base_ix] \n"
"lea (%k[x],%k[dx],8),%k[temp] \n"
"lea (%k[temp],%k[dx],8),%k[x] \n"
"movslq %k[x],%q[temp] \n"
"sar $16,%q[temp] \n"
"sub %q[base_ix],%q[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovd %k[mask],%%k1 \n"
"shr $32,%q[mask] \n"
"kmovd %k[mask],%%k2 \n"
"vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
"vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
"vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm3,%%zmm3 \n"
"vpsrld $16,%%zmm2,%%zmm2 \n"
"vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
"vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n"
"vpslld $16,%%zmm3,%%zmm4 \n"
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
"vpslld $16,%%zmm2,%%zmm4 \n"
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
"vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
"vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
"vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n"
"vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
"vpsrld $16,%%zmm4,%%zmm0 \n"
"vpandd %%zmm20,%%zmm4,%%zmm4 \n"
"vpsrld $16,%%zmm5,%%zmm1 \n"
"vpandd %%zmm20,%%zmm5,%%zmm5 \n"
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
"vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
"vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
"vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
"vpsrld $16,%%zmm4,%%zmm4 \n"
"vpmovdb %%zmm4,(%[dst_ptr]) \n"
"lea 0x10(%[dst_ptr]),%[dst_ptr] \n"
"sub $0x10,%[dst_width] \n"
"jge 11b \n"
"12: \n"
"add $0x10,%[dst_width] \n"
"jle 99f \n"
"movslq %k[x],%q[base_ix] \n"
"movzwl %w[x],%k[temp] \n"
"vpbroadcastd %k[temp],%%zmm2 \n"
"sar $16,%q[base_ix] \n"
"mov %k[dst_width],%k[temp] \n"
"imul %k[dx],%k[temp] \n"
"add %k[x],%k[temp] \n"
"sar $16,%k[temp] \n"
"movslq %k[temp],%q[temp] \n"
"sub %q[base_ix],%q[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovd %k[mask],%%k1 \n"
"shr $32,%q[mask] \n"
"kmovd %k[mask],%%k2 \n"
"vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
"vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
"vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm3,%%zmm3 \n"
"vpsrld $16,%%zmm2,%%zmm2 \n"
"vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
"vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n"
"vpslld $16,%%zmm3,%%zmm4 \n"
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
"vpslld $16,%%zmm2,%%zmm4 \n"
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
"vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
"vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
"vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n"
"vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
"vpsrld $16,%%zmm4,%%zmm0 \n"
"vpandd %%zmm20,%%zmm4,%%zmm4 \n"
"vpsrld $16,%%zmm5,%%zmm1 \n"
"vpandd %%zmm20,%%zmm5,%%zmm5 \n"
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
"vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
"vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
"vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
"vpsrld $16,%%zmm4,%%zmm4 \n"
"mov $-1,%q[mask] \n"
"bzhi %q[dst_width],%q[mask],%q[mask]\n"
"kmovw %k[mask],%%k1 \n"
"vpmovdb %%zmm4,(%[dst_ptr])%{%%k1%} \n"
"jmp 99f \n"
// Path 2: 4 <= minboxwidth <= 7 (8 output pixels per loop).
"20: \n"
"cmpl $7,%[minbw] \n"
"jg 30f \n"
"vpmovzxbq %[kSeq],%%zmm16 \n"
"movslq %k[dx],%q[temp] \n"
"vpbroadcastq %q[temp],%%zmm17 \n"
"vpmulld %%zmm17,%%zmm16,%%zmm16 \n"
"vpaddq %%zmm17,%%zmm16,%%zmm17 \n"
"movabs $0x0003000200010000,%q[temp] \n"
"vpbroadcastq %q[temp],%%zmm18 \n"
"movabs $0x0007000600050004,%q[temp] \n"
"vpbroadcastq %q[temp],%%zmm19 \n"
"movslq %[minbw],%q[temp] \n"
"vpbroadcastq %q[temp],%%zmm21 \n"
"sub $0x8,%[dst_width] \n"
"jl 22f \n"
LABELALIGN
"21: \n"
"movslq %k[x],%q[base_ix] \n"
"movzwl %w[x],%k[temp] \n"
"vpbroadcastq %q[temp],%%zmm2 \n"
"sar $16,%q[base_ix] \n"
"lea (%k[x],%k[dx],8),%k[x] \n"
"movslq %k[x],%q[temp] \n"
"sar $16,%q[temp] \n"
"sub %q[base_ix],%q[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovd %k[mask],%%k1 \n"
"shr $32,%q[mask] \n"
"kmovd %k[mask],%%k2 \n"
"vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
"vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
"vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm3,%%zmm3 \n"
"vpsrld $16,%%zmm2,%%zmm2 \n"
"vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
"vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n"
"vpsllq $16,%%zmm3,%%zmm4 \n"
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
"vpsllq $32,%%zmm3,%%zmm4 \n"
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
"vpsllq $16,%%zmm2,%%zmm4 \n"
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
"vpsllq $32,%%zmm2,%%zmm4 \n"
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
"vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
"vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
"vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm4 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
"vpsrld $16,%%zmm4,%%zmm0 \n"
"vpandd %%zmm20,%%zmm4,%%zmm4 \n"
"vpsrld $16,%%zmm5,%%zmm1 \n"
"vpandd %%zmm20,%%zmm5,%%zmm5 \n"
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
"vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
"vpsrlq $32,%%zmm4,%%zmm0 \n"
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
"vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
"vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
"vpsrld $16,%%zmm4,%%zmm4 \n"
"vpmovqb %%zmm4,(%[dst_ptr]) \n"
"lea 0x8(%[dst_ptr]),%[dst_ptr] \n"
"sub $0x8,%[dst_width] \n"
"jge 21b \n"
"22: \n"
"add $0x8,%[dst_width] \n"
"jle 99f \n"
"movslq %k[x],%q[base_ix] \n"
"movzwl %w[x],%k[temp] \n"
"vpbroadcastq %q[temp],%%zmm2 \n"
"sar $16,%q[base_ix] \n"
"mov %k[dst_width],%k[temp] \n"
"imul %k[dx],%k[temp] \n"
"add %k[x],%k[temp] \n"
"sar $16,%k[temp] \n"
"movslq %k[temp],%q[temp] \n"
"sub %q[base_ix],%q[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovd %k[mask],%%k1 \n"
"shr $32,%q[mask] \n"
"kmovd %k[mask],%%k2 \n"
"vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
"vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
"vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
"vpsrld $16,%%zmm3,%%zmm3 \n"
"vpsrld $16,%%zmm2,%%zmm2 \n"
"vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
"vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n"
"vpsllq $16,%%zmm3,%%zmm4 \n"
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
"vpsllq $32,%%zmm3,%%zmm4 \n"
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
"vpsllq $16,%%zmm2,%%zmm4 \n"
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
"vpsllq $32,%%zmm2,%%zmm4 \n"
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
"vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
"vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
"vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm4 \n"
"vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
"vpsrld $16,%%zmm4,%%zmm0 \n"
"vpandd %%zmm20,%%zmm4,%%zmm4 \n"
"vpsrld $16,%%zmm5,%%zmm1 \n"
"vpandd %%zmm20,%%zmm5,%%zmm5 \n"
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
"vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
"vpsrlq $32,%%zmm4,%%zmm0 \n"
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
"vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
"vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
"vpsrld $16,%%zmm4,%%zmm4 \n"
"mov $-1,%q[mask] \n"
"bzhi %q[dst_width],%q[mask],%q[mask]\n"
"kmovw %k[mask],%%k1 \n"
"vpmovqb %%zmm4,(%[dst_ptr])%{%%k1%} \n"
"jmp 99f \n"
// Path 3: minboxwidth >= 8 or minboxwidth < 1.
LABELALIGN
"30: \n"
"movslq %k[x],%q[base_ix] \n"
"sar $16,%q[base_ix] \n"
"add %k[dx],%k[x] \n"
"movslq %k[x],%q[temp] \n"
"sar $16,%q[temp] \n"
"sub %q[base_ix],%q[temp] \n"
"cmp $1,%q[temp] \n"
"jge 31f \n"
"mov $1,%q[temp] \n"
"31: \n"
"vmovdqa32 %%xmm22,%%xmm3 \n"
"cmp %k[temp],%[minbw] \n"
"jge 32f \n"
"vmovdqa32 %%xmm23,%%xmm3 \n"
"32: \n"
"vpxord %%zmm2,%%zmm2,%%zmm2 \n"
"lea (%[src_ptr],%q[base_ix],2),%q[base_ix] \n"
"sub $0x20,%q[temp] \n"
"jl 34f \n"
"33: \n"
"vmovdqu16 (%q[base_ix]),%%zmm0 \n"
"lea 0x40(%q[base_ix]),%q[base_ix] \n"
"vpsrld $16,%%zmm0,%%zmm1 \n"
"vpandd %%zmm20,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"sub $0x20,%q[temp] \n"
"jge 33b \n"
"34: \n"
"add $0x20,%q[temp] \n"
"jle 35f \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovd %k[mask],%%k1 \n"
"vmovdqu16 (%q[base_ix]),%%zmm0%{%%k1%}%{z%} \n"
"vpsrld $16,%%zmm0,%%zmm1 \n"
"vpandd %%zmm20,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
"35: \n"
"vextracti32x8 $1,%%zmm2,%%ymm0 \n"
"vpaddd %%ymm0,%%ymm2,%%ymm2 \n"
"vextracti128 $1,%%ymm2,%%xmm0 \n"
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
"vpshufd $0xee,%%xmm2,%%xmm0 \n"
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
"vpshufd $0x55,%%xmm2,%%xmm0 \n"
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
"vpmulld %%xmm3,%%xmm2,%%xmm2 \n"
"vpsrld $16,%%xmm2,%%xmm2 \n"
"vmovd %%xmm2,%k[temp] \n"
"mov %b[temp],(%[dst_ptr]) \n"
"inc %[dst_ptr] \n"
"dec %[dst_width] \n"
"jg 30b \n"
"99: \n"
"vzeroupper \n"
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
[dst_width] "+r"(dst_width), // %[dst_width]
[x] "+r"(x), // %[x]
[base_ix] "=&r"(base_ix), // %[base_ix]
[temp] "=&r"(temp), // %[temp]
[mask] "=&r"(mask) // %[mask]
: [dx] "r"(dx), // %[dx]
[minbw] "m"(minboxwidth), // %[minbw]
[scale0] "m"(scaletbl[0]), // %[scale0]
[scale1] "m"(scaletbl[1]), // %[scale1]
[kSeq] "m"(kSequence0_15) // %[kSeq]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
"xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1",
"k2", "k3", "k4");
}
#endif // HAS_SCALEADDCOLS_AVX512BW
static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040,
0x0040, 0x0040, 0x0040, 0x0040};