mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
Add ScaleAddRow_AVX512BW and ScaleAddCols1/2_AVX512BW
================================================== Processor / Architecture Before After Speedup -------------------------------------------------- Intel Granite Rapids 838 us 222 us 3.77x Intel Emerald Rapids 823 us 237 us 3.47x Intel Sapphire Rapids 884 us 241 us 3.67x AMD Turin (Zen 5) 684 us 119 us 5.75x AMD Genoa (Zen 4) 858 us 233 us 3.68x Intel Skylake-X 1421 us 319 us 4.45x Intel Ice Lake 1105 us 284 us 3.89x AMD Milan (Zen 3) 980 us 978 us 1.00x Intel Haswell 1356 us 1387 us 0.98x ================================================== Bug: None Change-Id: Id5473082cc2537234db9c53d53c38f2c9698d9d2 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8467771 Reviewed-by: richard winterton <rrwinterton@gmail.com> Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard
libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent
b5c764cfdd
commit
7320155784
@ -93,6 +93,14 @@ extern "C" {
|
|||||||
#define HAS_SCALEROWDOWN4_AVX2
|
#define HAS_SCALEROWDOWN4_AVX2
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
// The following are available for AVX512 clang x64 platforms:
|
||||||
|
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||||
|
(defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \
|
||||||
|
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||||
|
#define HAS_SCALEADDROW_AVX512BW
|
||||||
|
#define HAS_SCALEADDCOLS_AVX512BW
|
||||||
|
#endif
|
||||||
|
|
||||||
// The following are available on Neon platforms:
|
// The following are available on Neon platforms:
|
||||||
#if !defined(LIBYUV_DISABLE_NEON) && \
|
#if !defined(LIBYUV_DISABLE_NEON) && \
|
||||||
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
|
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
|
||||||
@ -798,12 +806,27 @@ void ScaleRowDown38_2_Box_Any_SSSE3(const uint8_t* src_ptr,
|
|||||||
|
|
||||||
void ScaleAddRow_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
|
void ScaleAddRow_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
|
||||||
void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
|
void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
|
||||||
|
void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
|
||||||
|
uint16_t* dst_ptr,
|
||||||
|
int src_width);
|
||||||
void ScaleAddRow_Any_SSE2(const uint8_t* src_ptr,
|
void ScaleAddRow_Any_SSE2(const uint8_t* src_ptr,
|
||||||
uint16_t* dst_ptr,
|
uint16_t* dst_ptr,
|
||||||
int src_width);
|
int src_width);
|
||||||
void ScaleAddRow_Any_AVX2(const uint8_t* src_ptr,
|
void ScaleAddRow_Any_AVX2(const uint8_t* src_ptr,
|
||||||
uint16_t* dst_ptr,
|
uint16_t* dst_ptr,
|
||||||
int src_width);
|
int src_width);
|
||||||
|
void ScaleAddCols1_AVX512BW(int dst_width,
|
||||||
|
int boxheight,
|
||||||
|
int x,
|
||||||
|
int dx,
|
||||||
|
const uint16_t* src_ptr,
|
||||||
|
uint8_t* dst_ptr);
|
||||||
|
void ScaleAddCols2_AVX512BW(int dst_width,
|
||||||
|
int boxheight,
|
||||||
|
int x,
|
||||||
|
int dx,
|
||||||
|
const uint16_t* src_ptr,
|
||||||
|
uint8_t* dst_ptr);
|
||||||
|
|
||||||
void ScaleFilterCols_SSSE3(uint8_t* dst_ptr,
|
void ScaleFilterCols_SSSE3(uint8_t* dst_ptr,
|
||||||
const uint8_t* src_ptr,
|
const uint8_t* src_ptr,
|
||||||
|
|||||||
@ -900,6 +900,18 @@ static int ScalePlaneBox(int src_width,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
#if defined(HAS_SCALEADDROW_AVX512BW)
|
||||||
|
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||||
|
ScaleAddRow = ScaleAddRow_AVX512BW;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
#if defined(HAS_SCALEADDCOLS_AVX512BW)
|
||||||
|
if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0) {
|
||||||
|
ScaleAddCols = (dx & 0xffff) ? ScaleAddCols2_AVX512BW
|
||||||
|
: ((dx != 0x10000) ? ScaleAddCols1_AVX512BW
|
||||||
|
: ScaleAddCols0_C);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
#if defined(HAS_SCALEADDROW_NEON)
|
#if defined(HAS_SCALEADDROW_NEON)
|
||||||
if (TestCpuFlag(kCpuHasNEON)) {
|
if (TestCpuFlag(kCpuHasNEON)) {
|
||||||
ScaleAddRow = ScaleAddRow_Any_NEON;
|
ScaleAddRow = ScaleAddRow_Any_NEON;
|
||||||
|
|||||||
@ -1792,6 +1792,674 @@ void ScaleAddRow_AVX2(const uint8_t* src_ptr,
|
|||||||
}
|
}
|
||||||
#endif // HAS_SCALEADDROW_AVX2
|
#endif // HAS_SCALEADDROW_AVX2
|
||||||
|
|
||||||
|
#ifdef HAS_SCALEADDROW_AVX512BW
|
||||||
|
// Reads 32xN bytes and accumulates to 32xN shorts at a time, with masked tail.
|
||||||
|
void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
|
||||||
|
uint16_t* dst_ptr,
|
||||||
|
int src_width) {
|
||||||
|
uintptr_t temp;
|
||||||
|
asm volatile(
|
||||||
|
"sub $0x40,%2 \n"
|
||||||
|
"jl 2f \n"
|
||||||
|
|
||||||
|
// 64 pixel loop.
|
||||||
|
LABELALIGN
|
||||||
|
"1: \n"
|
||||||
|
"vpmovzxbw (%0),%%zmm0 \n"
|
||||||
|
"vpmovzxbw 0x20(%0),%%zmm1 \n"
|
||||||
|
"lea 0x40(%0),%0 \n"
|
||||||
|
"vpaddusw (%1),%%zmm0,%%zmm0 \n"
|
||||||
|
"vpaddusw 0x40(%1),%%zmm1,%%zmm1 \n"
|
||||||
|
"vmovdqu16 %%zmm0,(%1) \n"
|
||||||
|
"vmovdqu16 %%zmm1,0x40(%1) \n"
|
||||||
|
"lea 0x80(%1),%1 \n"
|
||||||
|
"sub $0x40,%2 \n"
|
||||||
|
"jge 1b \n"
|
||||||
|
|
||||||
|
"2: \n"
|
||||||
|
"add $0x20,%2 \n"
|
||||||
|
"jl 3f \n"
|
||||||
|
"vpmovzxbw (%0),%%zmm0 \n"
|
||||||
|
"lea 0x20(%0),%0 \n"
|
||||||
|
"vpaddusw (%1),%%zmm0,%%zmm0 \n"
|
||||||
|
"vmovdqu16 %%zmm0,(%1) \n"
|
||||||
|
"lea 0x40(%1),%1 \n"
|
||||||
|
"sub $0x20,%2 \n"
|
||||||
|
|
||||||
|
"3: \n"
|
||||||
|
"add $0x20,%2 \n"
|
||||||
|
"jle 99f \n"
|
||||||
|
"mov $-1,%k3 \n"
|
||||||
|
"bzhi %k2,%k3,%k3 \n"
|
||||||
|
"kmovd %k3,%%k1 \n"
|
||||||
|
"vpmovzxbw (%0),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vpaddusw (%1),%%zmm0,%%zmm0%{%%k1%}%{z%}\n"
|
||||||
|
"vmovdqu16 %%zmm0,(%1)%{%%k1%} \n"
|
||||||
|
|
||||||
|
"99: \n"
|
||||||
|
"vzeroupper \n"
|
||||||
|
: "+r"(src_ptr), // %0
|
||||||
|
"+r"(dst_ptr), // %1
|
||||||
|
"+r"(src_width), // %2
|
||||||
|
"=&r"(temp) // %3
|
||||||
|
:
|
||||||
|
: "memory", "cc", "xmm0", "xmm1", "k1");
|
||||||
|
}
|
||||||
|
#endif // HAS_SCALEADDROW_AVX512BW
|
||||||
|
|
||||||
|
#ifdef HAS_SCALEADDCOLS_AVX512BW
|
||||||
|
static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7,
|
||||||
|
8, 9, 10, 11, 12, 13, 14, 15};
|
||||||
|
|
||||||
|
void ScaleAddCols1_AVX512BW(int dst_width,
|
||||||
|
int boxheight,
|
||||||
|
int x,
|
||||||
|
int dx,
|
||||||
|
const uint16_t* src_ptr,
|
||||||
|
uint8_t* dst_ptr) {
|
||||||
|
int boxwidth = dx >> 16;
|
||||||
|
if (boxwidth < 1) {
|
||||||
|
boxwidth = 1;
|
||||||
|
}
|
||||||
|
int scaleval = 65536 / (boxwidth * boxheight);
|
||||||
|
src_ptr += (x >> 16);
|
||||||
|
intptr_t temp, mask;
|
||||||
|
asm volatile(
|
||||||
|
"mov $0x0000ffff,%k[temp] \n"
|
||||||
|
"vpbroadcastd %k[temp],%%zmm6 \n"
|
||||||
|
"vpbroadcastd %k[scaleval],%%zmm7 \n"
|
||||||
|
"cmp $4,%k[bw] \n"
|
||||||
|
"jg 20f \n"
|
||||||
|
|
||||||
|
// Path 1: 1 <= boxwidth <= 4 (16 output pixels per loop).
|
||||||
|
"vpmovzxbd %[kSeq],%%zmm3 \n"
|
||||||
|
"vpslld $16,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpbroadcastw %k[bw],%%zmm2 \n"
|
||||||
|
"vpmullw %%zmm2,%%zmm3,%%zmm3 \n"
|
||||||
|
"mov $0x00010000,%k[temp] \n"
|
||||||
|
"vpbroadcastd %k[temp],%%zmm0 \n"
|
||||||
|
"mov $0x00030002,%k[temp] \n"
|
||||||
|
"vpbroadcastd %k[temp],%%zmm1 \n"
|
||||||
|
"vpaddw %%zmm0,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpaddw %%zmm1,%%zmm3,%%zmm5 \n"
|
||||||
|
"vpcmpuw $6,%%zmm0,%%zmm2,%%k2 \n"
|
||||||
|
"vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n"
|
||||||
|
"mov %k[bw],%k[temp] \n"
|
||||||
|
"shl $4,%k[temp] \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
"shr $32,%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k4 \n"
|
||||||
|
"shl $1,%q[temp] \n"
|
||||||
|
"sub $0x10,%[dst_width] \n"
|
||||||
|
"jl 12f \n"
|
||||||
|
|
||||||
|
LABELALIGN
|
||||||
|
"11: \n"
|
||||||
|
"vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
|
||||||
|
"add %q[temp],%[src_ptr] \n"
|
||||||
|
"vmovdqa64 %%zmm4,%%zmm2 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n"
|
||||||
|
"vmovdqa64 %%zmm5,%%zmm3 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm0 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm3,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpmovdb %%zmm2,(%[dst_ptr]) \n"
|
||||||
|
"lea 0x10(%[dst_ptr]),%[dst_ptr] \n"
|
||||||
|
"sub $0x10,%[dst_width] \n"
|
||||||
|
"jge 11b \n"
|
||||||
|
|
||||||
|
"12: \n"
|
||||||
|
"add $0x10,%[dst_width] \n"
|
||||||
|
"jle 99f \n"
|
||||||
|
"mov %k[dst_width],%k[temp] \n"
|
||||||
|
"imul %k[bw],%k[temp] \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
"shr $32,%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k4 \n"
|
||||||
|
"vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
|
||||||
|
"vmovdqa64 %%zmm4,%%zmm2 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm2%{%%k2%}%{z%} \n"
|
||||||
|
"vmovdqa64 %%zmm5,%%zmm3 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm0 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm3,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm2 \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[dst_width],%q[mask],%q[mask]\n"
|
||||||
|
"kmovw %k[mask],%%k1 \n"
|
||||||
|
"vpmovdb %%zmm2,(%[dst_ptr])%{%%k1%} \n"
|
||||||
|
"jmp 99f \n"
|
||||||
|
|
||||||
|
// Path 2: 5 <= boxwidth <= 8 (8 output pixels per loop).
|
||||||
|
"20: \n"
|
||||||
|
"cmp $8,%k[bw] \n"
|
||||||
|
"jg 30f \n"
|
||||||
|
"vpmovzxbq %[kSeq],%%zmm3 \n"
|
||||||
|
"vpsllq $16,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpsllq $32,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpbroadcastw %k[bw],%%zmm2 \n"
|
||||||
|
"vpmullw %%zmm2,%%zmm3,%%zmm3 \n"
|
||||||
|
"movabs $0x0003000200010000,%q[temp] \n"
|
||||||
|
"vpbroadcastq %q[temp],%%zmm0 \n"
|
||||||
|
"movabs $0x0007000600050004,%q[temp] \n"
|
||||||
|
"vpbroadcastq %q[temp],%%zmm1 \n"
|
||||||
|
"vpaddw %%zmm0,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpaddw %%zmm1,%%zmm3,%%zmm5 \n"
|
||||||
|
"vpcmpuw $6,%%zmm1,%%zmm2,%%k3 \n"
|
||||||
|
"mov %k[bw],%k[temp] \n"
|
||||||
|
"shl $3,%k[temp] \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
"shr $32,%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k4 \n"
|
||||||
|
"shl $1,%q[temp] \n"
|
||||||
|
"sub $0x8,%[dst_width] \n"
|
||||||
|
"jl 22f \n"
|
||||||
|
|
||||||
|
LABELALIGN
|
||||||
|
"21: \n"
|
||||||
|
"vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
|
||||||
|
"add %q[temp],%[src_ptr] \n"
|
||||||
|
"vmovdqa64 %%zmm4,%%zmm2 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm2 \n"
|
||||||
|
"vmovdqa64 %%zmm5,%%zmm3 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm0 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm3,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrlq $32,%%zmm2,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpmovqb %%zmm2,(%[dst_ptr]) \n"
|
||||||
|
"lea 0x8(%[dst_ptr]),%[dst_ptr] \n"
|
||||||
|
"sub $0x8,%[dst_width] \n"
|
||||||
|
"jge 21b \n"
|
||||||
|
|
||||||
|
"22: \n"
|
||||||
|
"add $0x8,%[dst_width] \n"
|
||||||
|
"jle 99f \n"
|
||||||
|
"mov %k[dst_width],%k[temp] \n"
|
||||||
|
"imul %k[bw],%k[temp] \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
"shr $32,%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k4 \n"
|
||||||
|
"vmovdqu16 (%[src_ptr]),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vmovdqu16 0x40(%[src_ptr]),%%zmm1%{%%k4%}%{z%} \n"
|
||||||
|
"vmovdqa64 %%zmm4,%%zmm2 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm2 \n"
|
||||||
|
"vmovdqa64 %%zmm5,%%zmm3 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm3%{%%k3%}%{z%} \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm0 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm3,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm3,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrlq $32,%%zmm2,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpmulld %%zmm7,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm2 \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[dst_width],%q[mask],%q[mask]\n"
|
||||||
|
"kmovw %k[mask],%%k1 \n"
|
||||||
|
"vpmovqb %%zmm2,(%[dst_ptr])%{%%k1%} \n"
|
||||||
|
"jmp 99f \n"
|
||||||
|
|
||||||
|
// Path 3: boxwidth >= 9 (ZMM 32-word reduction per output pixel).
|
||||||
|
"30: \n"
|
||||||
|
"mov %k[bw],%k[temp] \n"
|
||||||
|
"and $0x1f,%k[temp] \n"
|
||||||
|
"mov $-1,%k[mask] \n"
|
||||||
|
"bzhi %k[temp],%k[mask],%k[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
|
||||||
|
LABELALIGN
|
||||||
|
"31: \n"
|
||||||
|
"vpxord %%zmm2,%%zmm2,%%zmm2 \n"
|
||||||
|
"mov %[src_ptr],%q[mask] \n"
|
||||||
|
"lea (%[src_ptr],%q[bw],2),%[src_ptr] \n"
|
||||||
|
"mov %q[bw],%q[temp] \n"
|
||||||
|
"sub $0x20,%q[temp] \n"
|
||||||
|
"jl 33f \n"
|
||||||
|
"32: \n"
|
||||||
|
"vmovdqu16 (%q[mask]),%%zmm0 \n"
|
||||||
|
"lea 0x40(%q[mask]),%q[mask] \n"
|
||||||
|
"vpsrld $16,%%zmm0,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm0,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"sub $0x20,%q[temp] \n"
|
||||||
|
"jge 32b \n"
|
||||||
|
"33: \n"
|
||||||
|
"add $0x20,%q[temp] \n"
|
||||||
|
"jle 34f \n"
|
||||||
|
"vmovdqu16 (%q[mask]),%%zmm0%{%%k1%}%{z%}\n"
|
||||||
|
"vpsrld $16,%%zmm0,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm6,%%zmm0,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"34: \n"
|
||||||
|
"vextracti32x8 $1,%%zmm2,%%ymm0 \n"
|
||||||
|
"vpaddd %%ymm0,%%ymm2,%%ymm2 \n"
|
||||||
|
"vextracti128 $1,%%ymm2,%%xmm0 \n"
|
||||||
|
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
|
||||||
|
"vpshufd $0xee,%%xmm2,%%xmm0 \n"
|
||||||
|
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
|
||||||
|
"vpshufd $0x55,%%xmm2,%%xmm0 \n"
|
||||||
|
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
|
||||||
|
"vpmulld %%xmm7,%%xmm2,%%xmm2 \n"
|
||||||
|
"vpsrld $16,%%xmm2,%%xmm2 \n"
|
||||||
|
"vmovd %%xmm2,%k[temp] \n"
|
||||||
|
"mov %b[temp],(%[dst_ptr]) \n"
|
||||||
|
"inc %[dst_ptr] \n"
|
||||||
|
"dec %[dst_width] \n"
|
||||||
|
"jg 31b \n"
|
||||||
|
|
||||||
|
"99: \n"
|
||||||
|
"vzeroupper \n"
|
||||||
|
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
|
||||||
|
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
|
||||||
|
[dst_width] "+r"(dst_width), // %[dst_width]
|
||||||
|
[temp] "=&r"(temp), // %[temp]
|
||||||
|
[mask] "=&r"(mask) // %[mask]
|
||||||
|
: [bw] "r"((intptr_t)boxwidth), // %[bw]
|
||||||
|
[scaleval] "r"(scaleval), // %[scaleval]
|
||||||
|
[kSeq] "m"(kSequence0_15) // %[kSeq]
|
||||||
|
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||||
|
"xmm7", "k1", "k2", "k3", "k4");
|
||||||
|
}
|
||||||
|
|
||||||
|
void ScaleAddCols2_AVX512BW(int dst_width,
|
||||||
|
int boxheight,
|
||||||
|
int x,
|
||||||
|
int dx,
|
||||||
|
const uint16_t* src_ptr,
|
||||||
|
uint8_t* dst_ptr) {
|
||||||
|
int minboxwidth = dx >> 16;
|
||||||
|
int scaletbl[2];
|
||||||
|
scaletbl[0] = 65536 / ((minboxwidth < 1 ? 1 : minboxwidth) * boxheight);
|
||||||
|
scaletbl[1] =
|
||||||
|
65536 / (((minboxwidth + 1) < 1 ? 1 : (minboxwidth + 1)) * boxheight);
|
||||||
|
intptr_t base_ix, temp, mask;
|
||||||
|
asm volatile(
|
||||||
|
"mov $0x0000ffff,%k[temp] \n"
|
||||||
|
"vpbroadcastd %k[temp],%%zmm20 \n"
|
||||||
|
"vpbroadcastd %[scale0],%%zmm22 \n"
|
||||||
|
"vpbroadcastd %[scale1],%%zmm23 \n"
|
||||||
|
"cmpl $1,%[minbw] \n"
|
||||||
|
"jl 30f \n"
|
||||||
|
"cmpl $3,%[minbw] \n"
|
||||||
|
"jg 20f \n"
|
||||||
|
|
||||||
|
// Path 1: 1 <= minboxwidth <= 3 (16 output pixels per loop).
|
||||||
|
"vpmovzxbd %[kSeq],%%zmm16 \n"
|
||||||
|
"vpbroadcastd %k[dx],%%zmm17 \n"
|
||||||
|
"vpmulld %%zmm17,%%zmm16,%%zmm16 \n"
|
||||||
|
"vpaddd %%zmm17,%%zmm16,%%zmm17 \n"
|
||||||
|
"mov $0x00010000,%k[temp] \n"
|
||||||
|
"vpbroadcastd %k[temp],%%zmm18 \n"
|
||||||
|
"mov $0x00030002,%k[temp] \n"
|
||||||
|
"vpbroadcastd %k[temp],%%zmm19 \n"
|
||||||
|
"vpbroadcastd %[minbw],%%zmm21 \n"
|
||||||
|
"sub $0x10,%[dst_width] \n"
|
||||||
|
"jl 12f \n"
|
||||||
|
|
||||||
|
LABELALIGN
|
||||||
|
"11: \n"
|
||||||
|
"movslq %k[x],%q[base_ix] \n"
|
||||||
|
"movzwl %w[x],%k[temp] \n"
|
||||||
|
"vpbroadcastd %k[temp],%%zmm2 \n"
|
||||||
|
"sar $16,%q[base_ix] \n"
|
||||||
|
"lea (%k[x],%k[dx],8),%k[temp] \n"
|
||||||
|
"lea (%k[temp],%k[dx],8),%k[x] \n"
|
||||||
|
"movslq %k[x],%q[temp] \n"
|
||||||
|
"sar $16,%q[temp] \n"
|
||||||
|
"sub %q[base_ix],%q[temp] \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
"shr $32,%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k2 \n"
|
||||||
|
"vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
|
||||||
|
|
||||||
|
"vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n"
|
||||||
|
|
||||||
|
"vpslld $16,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpslld $16,%%zmm2,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
|
||||||
|
"vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n"
|
||||||
|
"vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
|
||||||
|
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
|
||||||
|
"vpsrld $16,%%zmm4,%%zmm0 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrld $16,%%zmm5,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm5,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
|
||||||
|
|
||||||
|
"vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
|
||||||
|
"vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrld $16,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpmovdb %%zmm4,(%[dst_ptr]) \n"
|
||||||
|
"lea 0x10(%[dst_ptr]),%[dst_ptr] \n"
|
||||||
|
"sub $0x10,%[dst_width] \n"
|
||||||
|
"jge 11b \n"
|
||||||
|
|
||||||
|
"12: \n"
|
||||||
|
"add $0x10,%[dst_width] \n"
|
||||||
|
"jle 99f \n"
|
||||||
|
"movslq %k[x],%q[base_ix] \n"
|
||||||
|
"movzwl %w[x],%k[temp] \n"
|
||||||
|
"vpbroadcastd %k[temp],%%zmm2 \n"
|
||||||
|
"sar $16,%q[base_ix] \n"
|
||||||
|
"mov %k[dst_width],%k[temp] \n"
|
||||||
|
"imul %k[dx],%k[temp] \n"
|
||||||
|
"add %k[x],%k[temp] \n"
|
||||||
|
"sar $16,%k[temp] \n"
|
||||||
|
"movslq %k[temp],%q[temp] \n"
|
||||||
|
"sub %q[base_ix],%q[temp] \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
"shr $32,%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k2 \n"
|
||||||
|
"vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
|
||||||
|
|
||||||
|
"vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpcmpd $6,%%zmm21,%%zmm2,%%k4 \n"
|
||||||
|
|
||||||
|
"vpslld $16,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpslld $16,%%zmm2,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
|
||||||
|
"vpcmpuw $6,%%zmm18,%%zmm2,%%k2 \n"
|
||||||
|
"vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
|
||||||
|
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm4%{%%k2%}%{z%} \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
|
||||||
|
"vpsrld $16,%%zmm4,%%zmm0 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrld $16,%%zmm5,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm5,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
|
||||||
|
|
||||||
|
"vpblendmd %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
|
||||||
|
"vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrld $16,%%zmm4,%%zmm4 \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[dst_width],%q[mask],%q[mask]\n"
|
||||||
|
"kmovw %k[mask],%%k1 \n"
|
||||||
|
"vpmovdb %%zmm4,(%[dst_ptr])%{%%k1%} \n"
|
||||||
|
"jmp 99f \n"
|
||||||
|
|
||||||
|
// Path 2: 4 <= minboxwidth <= 7 (8 output pixels per loop).
|
||||||
|
"20: \n"
|
||||||
|
"cmpl $7,%[minbw] \n"
|
||||||
|
"jg 30f \n"
|
||||||
|
"vpmovzxbq %[kSeq],%%zmm16 \n"
|
||||||
|
"movslq %k[dx],%q[temp] \n"
|
||||||
|
"vpbroadcastq %q[temp],%%zmm17 \n"
|
||||||
|
"vpmulld %%zmm17,%%zmm16,%%zmm16 \n"
|
||||||
|
"vpaddq %%zmm17,%%zmm16,%%zmm17 \n"
|
||||||
|
"movabs $0x0003000200010000,%q[temp] \n"
|
||||||
|
"vpbroadcastq %q[temp],%%zmm18 \n"
|
||||||
|
"movabs $0x0007000600050004,%q[temp] \n"
|
||||||
|
"vpbroadcastq %q[temp],%%zmm19 \n"
|
||||||
|
"movslq %[minbw],%q[temp] \n"
|
||||||
|
"vpbroadcastq %q[temp],%%zmm21 \n"
|
||||||
|
"sub $0x8,%[dst_width] \n"
|
||||||
|
"jl 22f \n"
|
||||||
|
|
||||||
|
LABELALIGN
|
||||||
|
"21: \n"
|
||||||
|
"movslq %k[x],%q[base_ix] \n"
|
||||||
|
"movzwl %w[x],%k[temp] \n"
|
||||||
|
"vpbroadcastq %q[temp],%%zmm2 \n"
|
||||||
|
"sar $16,%q[base_ix] \n"
|
||||||
|
"lea (%k[x],%k[dx],8),%k[x] \n"
|
||||||
|
"movslq %k[x],%q[temp] \n"
|
||||||
|
"sar $16,%q[temp] \n"
|
||||||
|
"sub %q[base_ix],%q[temp] \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
"shr $32,%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k2 \n"
|
||||||
|
"vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
|
||||||
|
|
||||||
|
"vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n"
|
||||||
|
|
||||||
|
"vpsllq $16,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpsllq $32,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpsllq $16,%%zmm2,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsllq $32,%%zmm2,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
|
||||||
|
"vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
|
||||||
|
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm4 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
|
||||||
|
"vpsrld $16,%%zmm4,%%zmm0 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrld $16,%%zmm5,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm5,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrlq $32,%%zmm4,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
|
||||||
|
|
||||||
|
"vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
|
||||||
|
"vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrld $16,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpmovqb %%zmm4,(%[dst_ptr]) \n"
|
||||||
|
"lea 0x8(%[dst_ptr]),%[dst_ptr] \n"
|
||||||
|
"sub $0x8,%[dst_width] \n"
|
||||||
|
"jge 21b \n"
|
||||||
|
|
||||||
|
"22: \n"
|
||||||
|
"add $0x8,%[dst_width] \n"
|
||||||
|
"jle 99f \n"
|
||||||
|
"movslq %k[x],%q[base_ix] \n"
|
||||||
|
"movzwl %w[x],%k[temp] \n"
|
||||||
|
"vpbroadcastq %q[temp],%%zmm2 \n"
|
||||||
|
"sar $16,%q[base_ix] \n"
|
||||||
|
"mov %k[dst_width],%k[temp] \n"
|
||||||
|
"imul %k[dx],%k[temp] \n"
|
||||||
|
"add %k[x],%k[temp] \n"
|
||||||
|
"sar $16,%k[temp] \n"
|
||||||
|
"movslq %k[temp],%q[temp] \n"
|
||||||
|
"sub %q[base_ix],%q[temp] \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
"shr $32,%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k2 \n"
|
||||||
|
"vmovdqu16 (%[src_ptr],%q[base_ix],2),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vmovdqu16 0x40(%[src_ptr],%q[base_ix],2),%%zmm1%{%%k2%}%{z%} \n"
|
||||||
|
|
||||||
|
"vpaddd %%zmm16,%%zmm2,%%zmm3 \n"
|
||||||
|
"vpaddd %%zmm17,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsrld $16,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpsrld $16,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsubd %%zmm3,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpcmpq $6,%%zmm21,%%zmm2,%%k4 \n"
|
||||||
|
|
||||||
|
"vpsllq $16,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpsllq $32,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm3,%%zmm3 \n"
|
||||||
|
"vpsllq $16,%%zmm2,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpsllq $32,%%zmm2,%%zmm4 \n"
|
||||||
|
"vpord %%zmm4,%%zmm2,%%zmm2 \n"
|
||||||
|
"vpaddw %%zmm18,%%zmm3,%%zmm4 \n"
|
||||||
|
"vpaddw %%zmm19,%%zmm3,%%zmm5 \n"
|
||||||
|
"vpcmpuw $6,%%zmm19,%%zmm2,%%k3 \n"
|
||||||
|
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm4 \n"
|
||||||
|
"vpermi2w %%zmm1,%%zmm0,%%zmm5%{%%k3%}%{z%} \n"
|
||||||
|
"vpsrld $16,%%zmm4,%%zmm0 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrld $16,%%zmm5,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm5,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm5,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrlq $32,%%zmm4,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm4,%%zmm4 \n"
|
||||||
|
|
||||||
|
"vpblendmq %%zmm23,%%zmm22,%%zmm2%{%%k4%}\n"
|
||||||
|
"vpmulld %%zmm2,%%zmm4,%%zmm4 \n"
|
||||||
|
"vpsrld $16,%%zmm4,%%zmm4 \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[dst_width],%q[mask],%q[mask]\n"
|
||||||
|
"kmovw %k[mask],%%k1 \n"
|
||||||
|
"vpmovqb %%zmm4,(%[dst_ptr])%{%%k1%} \n"
|
||||||
|
"jmp 99f \n"
|
||||||
|
|
||||||
|
// Path 3: minboxwidth >= 8 or minboxwidth < 1.
|
||||||
|
LABELALIGN
|
||||||
|
"30: \n"
|
||||||
|
"movslq %k[x],%q[base_ix] \n"
|
||||||
|
"sar $16,%q[base_ix] \n"
|
||||||
|
"add %k[dx],%k[x] \n"
|
||||||
|
"movslq %k[x],%q[temp] \n"
|
||||||
|
"sar $16,%q[temp] \n"
|
||||||
|
"sub %q[base_ix],%q[temp] \n"
|
||||||
|
"cmp $1,%q[temp] \n"
|
||||||
|
"jge 31f \n"
|
||||||
|
"mov $1,%q[temp] \n"
|
||||||
|
"31: \n"
|
||||||
|
"vmovdqa32 %%xmm22,%%xmm3 \n"
|
||||||
|
"cmp %k[temp],%[minbw] \n"
|
||||||
|
"jge 32f \n"
|
||||||
|
"vmovdqa32 %%xmm23,%%xmm3 \n"
|
||||||
|
"32: \n"
|
||||||
|
"vpxord %%zmm2,%%zmm2,%%zmm2 \n"
|
||||||
|
"lea (%[src_ptr],%q[base_ix],2),%q[base_ix] \n"
|
||||||
|
"sub $0x20,%q[temp] \n"
|
||||||
|
"jl 34f \n"
|
||||||
|
"33: \n"
|
||||||
|
"vmovdqu16 (%q[base_ix]),%%zmm0 \n"
|
||||||
|
"lea 0x40(%q[base_ix]),%q[base_ix] \n"
|
||||||
|
"vpsrld $16,%%zmm0,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm0,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"sub $0x20,%q[temp] \n"
|
||||||
|
"jge 33b \n"
|
||||||
|
"34: \n"
|
||||||
|
"add $0x20,%q[temp] \n"
|
||||||
|
"jle 35f \n"
|
||||||
|
"mov $-1,%q[mask] \n"
|
||||||
|
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||||
|
"kmovd %k[mask],%%k1 \n"
|
||||||
|
"vmovdqu16 (%q[base_ix]),%%zmm0%{%%k1%}%{z%} \n"
|
||||||
|
"vpsrld $16,%%zmm0,%%zmm1 \n"
|
||||||
|
"vpandd %%zmm20,%%zmm0,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm1,%%zmm0,%%zmm0 \n"
|
||||||
|
"vpaddd %%zmm0,%%zmm2,%%zmm2 \n"
|
||||||
|
"35: \n"
|
||||||
|
"vextracti32x8 $1,%%zmm2,%%ymm0 \n"
|
||||||
|
"vpaddd %%ymm0,%%ymm2,%%ymm2 \n"
|
||||||
|
"vextracti128 $1,%%ymm2,%%xmm0 \n"
|
||||||
|
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
|
||||||
|
"vpshufd $0xee,%%xmm2,%%xmm0 \n"
|
||||||
|
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
|
||||||
|
"vpshufd $0x55,%%xmm2,%%xmm0 \n"
|
||||||
|
"vpaddd %%xmm0,%%xmm2,%%xmm2 \n"
|
||||||
|
"vpmulld %%xmm3,%%xmm2,%%xmm2 \n"
|
||||||
|
"vpsrld $16,%%xmm2,%%xmm2 \n"
|
||||||
|
"vmovd %%xmm2,%k[temp] \n"
|
||||||
|
"mov %b[temp],(%[dst_ptr]) \n"
|
||||||
|
"inc %[dst_ptr] \n"
|
||||||
|
"dec %[dst_width] \n"
|
||||||
|
"jg 30b \n"
|
||||||
|
|
||||||
|
"99: \n"
|
||||||
|
"vzeroupper \n"
|
||||||
|
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
|
||||||
|
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
|
||||||
|
[dst_width] "+r"(dst_width), // %[dst_width]
|
||||||
|
[x] "+r"(x), // %[x]
|
||||||
|
[base_ix] "=&r"(base_ix), // %[base_ix]
|
||||||
|
[temp] "=&r"(temp), // %[temp]
|
||||||
|
[mask] "=&r"(mask) // %[mask]
|
||||||
|
: [dx] "r"(dx), // %[dx]
|
||||||
|
[minbw] "m"(minboxwidth), // %[minbw]
|
||||||
|
[scale0] "m"(scaletbl[0]), // %[scale0]
|
||||||
|
[scale1] "m"(scaletbl[1]), // %[scale1]
|
||||||
|
[kSeq] "m"(kSequence0_15) // %[kSeq]
|
||||||
|
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
|
||||||
|
"xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1",
|
||||||
|
"k2", "k3", "k4");
|
||||||
|
}
|
||||||
|
#endif // HAS_SCALEADDCOLS_AVX512BW
|
||||||
|
|
||||||
static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040,
|
static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040,
|
||||||
0x0040, 0x0040, 0x0040, 0x0040};
|
0x0040, 0x0040, 0x0040, 0x0040};
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user