mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
[libyuv] AVX512, NEON, WASM and RVV alpha copy and mirror rows
1275x1650 (US Letter 150 DPI, 1000 iterations) speedup: CPU ARGBExtractAlpha ARGBCopyYToAlpha ARGBMirror RGB24Mirror ----------------------------------------------------------------------- Intel Granite Rapids 1.00x 1.00x 1.00x 1.04x Intel Emerald Rapids 1.01x 0.97x 1.03x 1.07x Intel Sapphire Rapids 0.97x 1.01x 1.08x 1.26x Intel Ice Lake 0.99x 1.02x 0.97x 1.08x Intel Skylake-X 1.04x 1.03x 1.01x NA Intel Haswell NA NA NA NA AMD Turin (Zen 5) 1.01x 1.10x 1.25x 1.31x AMD Genoa (Zen 4) 0.97x 1.04x 1.09x 1.42x AMD Milan (Zen 3) NA NA NA NA AMD Rome (Zen 2) NA NA NA NA NVIDIA Grace (V2) 1.57x 2.51x NA NA Google Axion 2 1.12x 3.17x NA NA Ampere Siryn 1.06x 1.70x NA NA ARM Neoverse-N1 1.49x 2.01x NA NA Google Axion (Maple) 1.47x 2.03x NA NA ARM Cortex-A53 1.06x 1.69x NA NA Add AVX512BW ARGBExtractAlphaRow, ARGBCopyYToAlphaRow and ARGBMirrorRow, and AVX512VBMI RGB24MirrorRow, in row_gcc.cc and as intrinsics in row_win.cc. They are x64 only. The remainder is done with a k-mask (bzhi) zero-masked load and a masked store, so they handle any width without an Any wrapper and never access memory outside the row. RGB24MirrorRow_AVX512VBMI loads and stores 3 full zmm vectors per 64 pixels using 3 vpermi2b. The one output byte that comes from the third input vector is merged with a masked vpbroadcastb. The 1 to 63 pixel tail rebases the same tables by 192 - 3 * r; indices >= 128 are taken from the third vector with vpmovb2m and a masked vpermb. Add ARGBCopyYToAlphaRow_NEON for Arm 32 bit (vld4 + vst4) and AArch64 (zip + sli), and rewrite AArch64 ARGBExtractAlphaRow_NEON with ldp + uzp2. Both prefetch source and destination, which is 2.5x faster on Neoverse-N1 and 1.2x to 1.5x faster on Cortex-A53 for 1280x720. Add WASM SIMD ARGBExtractAlphaRow and ARGBCopyYToAlphaRow, using a swizzle table for CopyYToAlpha. Rework the RVV versions to use unit stride loads with vnsrl and vzext.vf4. R=rrwinterton@gmail.com Bug: None Change-Id: I436f27d70a3db0a2b176c91eb789a9491133cfcb Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8523531 Reviewed-by: richard winterton <rrwinterton@gmail.com> Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard
parent
98697ab033
commit
1ddb18fb6b
@ -385,6 +385,10 @@ extern "C" {
|
||||
#define HAS_RGBTOYMATRIXROW_AVX512BW
|
||||
#define HAS_BLENDPLANEROW_AVX512BW
|
||||
#define HAS_INTERPOLATEROW_AVX512BW
|
||||
#define HAS_ARGBEXTRACTALPHAROW_AVX512BW
|
||||
#define HAS_ARGBCOPYYTOALPHAROW_AVX512BW
|
||||
#define HAS_ARGBMIRRORROW_AVX512BW
|
||||
#define HAS_RGB24MIRRORROW_AVX512VBMI
|
||||
#endif
|
||||
#define HAS_ARGBTOYMATRIXROW_AVX2
|
||||
#define HAS_RGBTOYMATRIXROW_AVX2
|
||||
@ -452,6 +456,10 @@ extern "C" {
|
||||
#define HAS_SPLITUVROW_AVX512BW
|
||||
#define HAS_RGBTOUVMATRIXROW_AVX512BW
|
||||
#define HAS_RGBTOYMATRIXROW_AVX512BW
|
||||
#define HAS_ARGBEXTRACTALPHAROW_AVX512BW
|
||||
#define HAS_ARGBCOPYYTOALPHAROW_AVX512BW
|
||||
#define HAS_ARGBMIRRORROW_AVX512BW
|
||||
#define HAS_RGB24MIRRORROW_AVX512VBMI
|
||||
#endif
|
||||
|
||||
// The following are available on Neon platforms:
|
||||
@ -465,6 +473,7 @@ extern "C" {
|
||||
#define HAS_ARGB4444TOARGBROW_NEON
|
||||
#define HAS_ARGB4444TOUVROW_NEON
|
||||
#define HAS_ARGB4444TOYROW_NEON
|
||||
#define HAS_ARGBCOPYYTOALPHAROW_NEON
|
||||
#define HAS_ARGBEXTRACTALPHAROW_NEON
|
||||
#define HAS_ARGBSETROW_NEON
|
||||
#define HAS_ARGBTOAB64ROW_NEON
|
||||
@ -937,6 +946,8 @@ extern "C" {
|
||||
#define HAS_ARGB4444TOUVMATRIXROW_WASMSIMD
|
||||
#define HAS_ARGB4444TOYMATRIXROW_WASMSIMD
|
||||
#define HAS_ARGBBLENDROW_WASMSIMD
|
||||
#define HAS_ARGBCOPYYTOALPHAROW_WASMSIMD
|
||||
#define HAS_ARGBEXTRACTALPHAROW_WASMSIMD
|
||||
#define HAS_ARGBMIRRORROW_WASMSIMD
|
||||
#define HAS_ARGBSHUFFLEROW_WASMSIMD
|
||||
#define HAS_ARGBTOARGB1555ROW_WASMSIMD
|
||||
@ -2770,6 +2781,7 @@ void MirrorSplitUVRow_C(const uint8_t* src_uv,
|
||||
void MirrorRow_16_C(const uint16_t* src, uint16_t* dst, int width);
|
||||
|
||||
void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBMirrorRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width);
|
||||
void ARGBMirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width);
|
||||
@ -2792,6 +2804,9 @@ void ARGBMirrorRow_Any_LASX(const uint8_t* src_ptr,
|
||||
void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
|
||||
uint8_t* dst_rgb24,
|
||||
int width);
|
||||
void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24,
|
||||
uint8_t* dst_rgb24,
|
||||
int width);
|
||||
void RGB24MirrorRow_NEON(const uint8_t* src_rgb24,
|
||||
uint8_t* dst_rgb24,
|
||||
int width);
|
||||
@ -3922,12 +3937,18 @@ void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb,
|
||||
void ARGBExtractAlphaRow_AVX2(const uint8_t* src_argb,
|
||||
uint8_t* dst_a,
|
||||
int width);
|
||||
void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb,
|
||||
uint8_t* dst_a,
|
||||
int width);
|
||||
void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_a,
|
||||
int width);
|
||||
void ARGBExtractAlphaRow_LSX(const uint8_t* src_argb,
|
||||
uint8_t* dst_a,
|
||||
int width);
|
||||
void ARGBExtractAlphaRow_WASMSIMD(const uint8_t* src_argb,
|
||||
uint8_t* dst_a,
|
||||
int width);
|
||||
void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
|
||||
uint8_t* dst_a,
|
||||
int width);
|
||||
@ -3943,10 +3964,16 @@ void ARGBExtractAlphaRow_Any_NEON(const uint8_t* src_ptr,
|
||||
void ARGBExtractAlphaRow_Any_LSX(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBExtractAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void ARGBCopyYToAlphaRow_C(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyYToAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyYToAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyYToAlphaRow_Any_SSE2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
@ -3954,6 +3981,12 @@ void ARGBCopyYToAlphaRow_Any_SSE2(const uint8_t* src_ptr,
|
||||
void ARGBCopyYToAlphaRow_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBCopyYToAlphaRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBCopyYToAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void SetRow_C(uint8_t* dst, uint8_t v8, int width);
|
||||
void SetRow_X86(uint8_t* dst, uint8_t v8, int width);
|
||||
|
||||
@ -3193,6 +3193,11 @@ int ARGBToI420Alpha(const uint8_t* src_argb,
|
||||
: ARGBExtractAlphaRow_Any_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBEXTRACTALPHAROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBExtractAlphaRow = ARGBExtractAlphaRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBEXTRACTALPHAROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_NEON
|
||||
@ -3205,6 +3210,13 @@ int ARGBToI420Alpha(const uint8_t* src_argb,
|
||||
: ARGBExtractAlphaRow_Any_LSX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBEXTRACTALPHAROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
ARGBExtractAlphaRow = IS_ALIGNED(width, 16)
|
||||
? ARGBExtractAlphaRow_WASMSIMD
|
||||
: ARGBExtractAlphaRow_Any_WASMSIMD;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBEXTRACTALPHAROW_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
ARGBExtractAlphaRow = ARGBExtractAlphaRow_RVV;
|
||||
|
||||
@ -2970,6 +2970,11 @@ int ARGBMirror(const uint8_t* src_argb,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBMIRRORROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBMirrorRow = ARGBMirrorRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBMIRRORROW_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
ARGBMirrorRow = ARGBMirrorRow_Any_LSX;
|
||||
@ -3041,6 +3046,11 @@ int RGB24Mirror(const uint8_t* src_rgb24,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGB24MIRRORROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
RGB24MirrorRow = RGB24MirrorRow_AVX512VBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGB24MIRRORROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
RGB24MirrorRow = RGB24MirrorRow_Any_WASMSIMD;
|
||||
@ -5754,6 +5764,11 @@ int ARGBExtractAlpha(const uint8_t* src_argb,
|
||||
: ARGBExtractAlphaRow_Any_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBEXTRACTALPHAROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBExtractAlphaRow = ARGBExtractAlphaRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBEXTRACTALPHAROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_NEON
|
||||
@ -5766,6 +5781,13 @@ int ARGBExtractAlpha(const uint8_t* src_argb,
|
||||
: ARGBExtractAlphaRow_Any_LSX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBEXTRACTALPHAROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
ARGBExtractAlphaRow = IS_ALIGNED(width, 16)
|
||||
? ARGBExtractAlphaRow_WASMSIMD
|
||||
: ARGBExtractAlphaRow_Any_WASMSIMD;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBEXTRACTALPHAROW_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
ARGBExtractAlphaRow = ARGBExtractAlphaRow_RVV;
|
||||
@ -5823,6 +5845,24 @@ int ARGBCopyYToAlpha(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOPYYTOALPHAROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOPYYTOALPHAROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBCopyYToAlphaRow = IS_ALIGNED(width, 16) ? ARGBCopyYToAlphaRow_NEON
|
||||
: ARGBCopyYToAlphaRow_Any_NEON;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOPYYTOALPHAROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
ARGBCopyYToAlphaRow = IS_ALIGNED(width, 16)
|
||||
? ARGBCopyYToAlphaRow_WASMSIMD
|
||||
: ARGBCopyYToAlphaRow_Any_WASMSIMD;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOPYYTOALPHAROW_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_RVV;
|
||||
|
||||
@ -160,6 +160,11 @@ static int ARGBRotate180(const uint8_t* src_argb,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBMIRRORROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBMirrorRow = ARGBMirrorRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBMIRRORROW_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
ARGBMirrorRow = ARGBMirrorRow_Any_LSX;
|
||||
|
||||
@ -1321,6 +1321,14 @@ ANY11(ARGBExtractAlphaRow_Any_NEON, ARGBExtractAlphaRow_NEON, 0, 4, 1, 15)
|
||||
#ifdef HAS_ARGBEXTRACTALPHAROW_LSX
|
||||
ANY11(ARGBExtractAlphaRow_Any_LSX, ARGBExtractAlphaRow_LSX, 0, 4, 1, 15)
|
||||
#endif
|
||||
#ifdef HAS_ARGBEXTRACTALPHAROW_WASMSIMD
|
||||
ANY11(ARGBExtractAlphaRow_Any_WASMSIMD,
|
||||
ARGBExtractAlphaRow_WASMSIMD,
|
||||
0,
|
||||
4,
|
||||
1,
|
||||
15)
|
||||
#endif
|
||||
#ifdef HAS_RAWTOARGBROW_WASMSIMD
|
||||
ANY11(RAWToARGBRow_Any_WASMSIMD, RAWToARGBRow_WASMSIMD, 0, 3, 4, 15)
|
||||
#endif
|
||||
@ -1387,6 +1395,17 @@ ANY11B(ARGBCopyYToAlphaRow_Any_AVX2, ARGBCopyYToAlphaRow_AVX2, 0, 1, 4, 15)
|
||||
#ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2
|
||||
ANY11B(ARGBCopyYToAlphaRow_Any_SSE2, ARGBCopyYToAlphaRow_SSE2, 0, 1, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_ARGBCOPYYTOALPHAROW_NEON
|
||||
ANY11B(ARGBCopyYToAlphaRow_Any_NEON, ARGBCopyYToAlphaRow_NEON, 0, 1, 4, 15)
|
||||
#endif
|
||||
#ifdef HAS_ARGBCOPYYTOALPHAROW_WASMSIMD
|
||||
ANY11B(ARGBCopyYToAlphaRow_Any_WASMSIMD,
|
||||
ARGBCopyYToAlphaRow_WASMSIMD,
|
||||
0,
|
||||
1,
|
||||
4,
|
||||
15)
|
||||
#endif
|
||||
#undef ANY11B
|
||||
|
||||
// Any 1 to 1 with parameter.
|
||||
|
||||
@ -5449,6 +5449,139 @@ void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
|
||||
}
|
||||
#endif // HAS_RGB24MIRRORROW_AVX2
|
||||
|
||||
#ifdef HAS_RGB24MIRRORROW_AVX512VBMI
|
||||
// Mirror 64 RGB24 pixels (192 bytes = 3 zmm) per loop. Loads and stores are
|
||||
// 3 full contiguous zmm vectors. Output vector 0 is a 2 source byte permute
|
||||
// of input vectors 1 and 2, and output vectors 1 and 2 are 2 source byte
|
||||
// permutes of input vectors 0 and 1. Output byte 65 (byte 1 of output
|
||||
// vector 1) is the only byte that comes from a third vector: byte 0 of input
|
||||
// vector 2, which is merged in with a masked broadcast.
|
||||
static const uint8_t kPermMirrorRGB24_0[64] = {
|
||||
125u, 126u, 127u, 122u, 123u, 124u, 119u, 120u, 121u, 116u, 117u,
|
||||
118u, 113u, 114u, 115u, 110u, 111u, 112u, 107u, 108u, 109u, 104u,
|
||||
105u, 106u, 101u, 102u, 103u, 98u, 99u, 100u, 95u, 96u, 97u,
|
||||
92u, 93u, 94u, 89u, 90u, 91u, 86u, 87u, 88u, 83u, 84u,
|
||||
85u, 80u, 81u, 82u, 77u, 78u, 79u, 74u, 75u, 76u, 71u,
|
||||
72u, 73u, 68u, 69u, 70u, 65u, 66u, 67u, 62u};
|
||||
static const uint8_t kPermMirrorRGB24_1[64] = {
|
||||
127u, 128u, 123u, 124u, 125u, 120u, 121u, 122u, 117u, 118u, 119u,
|
||||
114u, 115u, 116u, 111u, 112u, 113u, 108u, 109u, 110u, 105u, 106u,
|
||||
107u, 102u, 103u, 104u, 99u, 100u, 101u, 96u, 97u, 98u, 93u,
|
||||
94u, 95u, 90u, 91u, 92u, 87u, 88u, 89u, 84u, 85u, 86u,
|
||||
81u, 82u, 83u, 78u, 79u, 80u, 75u, 76u, 77u, 72u, 73u,
|
||||
74u, 69u, 70u, 71u, 66u, 67u, 68u, 63u, 64u};
|
||||
static const uint8_t kPermMirrorRGB24_2[64] = {
|
||||
65u, 60u, 61u, 62u, 57u, 58u, 59u, 54u, 55u, 56u, 51u, 52u, 53u,
|
||||
48u, 49u, 50u, 45u, 46u, 47u, 42u, 43u, 44u, 39u, 40u, 41u, 36u,
|
||||
37u, 38u, 33u, 34u, 35u, 30u, 31u, 32u, 27u, 28u, 29u, 24u, 25u,
|
||||
26u, 21u, 22u, 23u, 18u, 19u, 20u, 15u, 16u, 17u, 12u, 13u, 14u,
|
||||
9u, 10u, 11u, 6u, 7u, 8u, 3u, 4u, 5u, 0u, 1u, 2u};
|
||||
|
||||
// Byte mask with the low n bits set, clamped to [0, 64].
|
||||
static inline uint64_t RGB24MirrorMask(int n) {
|
||||
return n <= 0 ? 0ull : n >= 64 ? ~0ull : (1ull << n) - 1ull;
|
||||
}
|
||||
|
||||
void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24,
|
||||
uint8_t* dst_rgb24,
|
||||
int width) {
|
||||
int r = width & 63;
|
||||
ptrdiff_t temp_width = (ptrdiff_t)(width - r);
|
||||
if (temp_width > 0) {
|
||||
const uint8_t* src = src_rgb24 + (ptrdiff_t)width * 3 - 192;
|
||||
asm volatile(
|
||||
"vmovdqu8 %3,%%zmm20 \n"
|
||||
"vmovdqu8 %4,%%zmm21 \n"
|
||||
"vmovdqu8 %5,%%zmm22 \n"
|
||||
"kmovq %6,%%k1 \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vmovdqu8 (%0),%%zmm0 \n" // last 64 pixels
|
||||
"vmovdqu8 0x40(%0),%%zmm1 \n"
|
||||
"vmovdqu8 0x80(%0),%%zmm2 \n"
|
||||
"lea -0xc0(%0),%0 \n"
|
||||
"vmovdqa64 %%zmm20,%%zmm3 \n"
|
||||
"vmovdqa64 %%zmm21,%%zmm4 \n"
|
||||
"vmovdqa64 %%zmm22,%%zmm5 \n"
|
||||
"vpermi2b %%zmm2,%%zmm1,%%zmm3 \n" // out0 from in1:in2
|
||||
"vpermi2b %%zmm1,%%zmm0,%%zmm4 \n" // out1 from in0:in1
|
||||
"vpermi2b %%zmm1,%%zmm0,%%zmm5 \n" // out2 from in0:in1
|
||||
"vpbroadcastb %%xmm2,%%zmm4%{%%k1%} \n" // out1 byte 1 = in2[0]
|
||||
"vmovdqu8 %%zmm3,(%1) \n"
|
||||
"vmovdqu8 %%zmm4,0x40(%1) \n"
|
||||
"vmovdqu8 %%zmm5,0x80(%1) \n"
|
||||
"lea 0xc0(%1),%1 \n"
|
||||
"sub $0x40,%2 \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst_rgb24), // %1
|
||||
"+r"(temp_width) // %2
|
||||
: "m"(kPermMirrorRGB24_0), // %3
|
||||
"m"(kPermMirrorRGB24_1), // %4
|
||||
"m"(kPermMirrorRGB24_2), // %5
|
||||
"r"((uint64_t)2) // %6
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
|
||||
"xmm20", "xmm21", "xmm22", "k1");
|
||||
}
|
||||
// Remaining 1 to 63 pixels are the first r pixels of the source. Masked
|
||||
// loads and stores access only the 3 * r valid bytes. The tables, as
|
||||
// global byte offsets into a 192 byte block, are adjusted by the 192 - 3 * r
|
||||
// missing bytes. Bytes from the third vector (index >= 128) are selected
|
||||
// with a mask from the sign bit of the index.
|
||||
if (r) {
|
||||
int nb = r * 3;
|
||||
int off = 192 - nb;
|
||||
asm volatile(
|
||||
"kmovq %2,%%k1 \n"
|
||||
"kmovq %3,%%k2 \n"
|
||||
"kmovq %4,%%k3 \n"
|
||||
"vmovdqu8 (%0),%%zmm0%{%%k1%}%{z%} \n"
|
||||
"vmovdqu8 0x40(%0),%%zmm1%{%%k2%}%{z%}\n"
|
||||
"vmovdqu8 0x80(%0),%%zmm2%{%%k3%}%{z%}\n"
|
||||
"vpbroadcastb %5,%%zmm6 \n" // off
|
||||
"vpbroadcastb %6,%%zmm7 \n" // off - 64
|
||||
"vmovdqu8 %7,%%zmm3 \n"
|
||||
"vmovdqu8 %8,%%zmm4 \n"
|
||||
"vmovdqu8 %9,%%zmm5 \n"
|
||||
"vpsubb %%zmm7,%%zmm3,%%zmm3 \n"
|
||||
"vpsubb %%zmm6,%%zmm4,%%zmm4 \n"
|
||||
"vpsubb %%zmm6,%%zmm5,%%zmm5 \n"
|
||||
|
||||
"vpmovb2m %%zmm3,%%k4 \n" // out0
|
||||
"vmovdqa64 %%zmm3,%%zmm6 \n"
|
||||
"vpermi2b %%zmm1,%%zmm0,%%zmm6 \n"
|
||||
"vpermb %%zmm2,%%zmm3,%%zmm6%{%%k4%}\n"
|
||||
"vmovdqu8 %%zmm6,(%1)%{%%k1%} \n"
|
||||
"vpmovb2m %%zmm4,%%k4 \n" // out1
|
||||
"vmovdqa64 %%zmm4,%%zmm6 \n"
|
||||
"vpermi2b %%zmm1,%%zmm0,%%zmm6 \n"
|
||||
"vpermb %%zmm2,%%zmm4,%%zmm6%{%%k4%}\n"
|
||||
"vmovdqu8 %%zmm6,0x40(%1)%{%%k2%} \n"
|
||||
"vpmovb2m %%zmm5,%%k4 \n" // out2
|
||||
"vmovdqa64 %%zmm5,%%zmm6 \n"
|
||||
"vpermi2b %%zmm1,%%zmm0,%%zmm6 \n"
|
||||
"vpermb %%zmm2,%%zmm5,%%zmm6%{%%k4%}\n"
|
||||
"vmovdqu8 %%zmm6,0x80(%1)%{%%k3%} \n"
|
||||
"vzeroupper \n"
|
||||
:
|
||||
: "r"(src_rgb24), // %0
|
||||
"r"(dst_rgb24), // %1
|
||||
"r"(RGB24MirrorMask(nb)), // %2
|
||||
"r"(RGB24MirrorMask(nb - 64)), // %3
|
||||
"r"(RGB24MirrorMask(nb - 128)), // %4
|
||||
"r"(off), // %5
|
||||
"r"(off - 64), // %6
|
||||
"m"(kPermMirrorRGB24_0), // %7
|
||||
"m"(kPermMirrorRGB24_1), // %8
|
||||
"m"(kPermMirrorRGB24_2) // %9
|
||||
: "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||
"xmm7", "k1", "k2", "k3", "k4");
|
||||
}
|
||||
}
|
||||
#endif // HAS_RGB24MIRRORROW_AVX512VBMI
|
||||
|
||||
#ifdef HAS_ARGBMIRRORROW_SSE2
|
||||
|
||||
void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
|
||||
@ -5495,6 +5628,54 @@ void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
||||
}
|
||||
#endif // HAS_ARGBMIRRORROW_AVX2
|
||||
|
||||
#ifdef HAS_ARGBMIRRORROW_AVX512BW
|
||||
static const uint32_t kARGBShuffleMirror_AVX512[16] = {
|
||||
15u, 14u, 13u, 12u, 11u, 10u, 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
|
||||
|
||||
// Mirror 16 ARGB pixels per loop. The remaining 1 to 15 pixels are at the
|
||||
// start of src. They are loaded with a k mask, permuted with the reverse
|
||||
// index minus (16 - remaining), and stored with the same k mask.
|
||||
void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
||||
ptrdiff_t temp_width = (ptrdiff_t)(width);
|
||||
uintptr_t temp;
|
||||
asm volatile(
|
||||
"vmovdqu32 %4,%%zmm5 \n"
|
||||
"sub $0x10,%2 \n"
|
||||
"jl 2f \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vpermd (%0,%2,4),%%zmm5,%%zmm0 \n"
|
||||
"vmovdqu32 %%zmm0,(%1) \n"
|
||||
"lea 0x40(%1),%1 \n"
|
||||
"sub $0x10,%2 \n"
|
||||
"jge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"add $0x10,%2 \n"
|
||||
"jle 99f \n"
|
||||
"mov $-1,%k3 \n"
|
||||
"bzhi %k2,%k3,%k3 \n"
|
||||
"kmovw %k3,%%k1 \n"
|
||||
"mov $0x10,%k3 \n"
|
||||
"sub %k2,%k3 \n"
|
||||
"vpbroadcastd %k3,%%zmm1 \n"
|
||||
"vpsubd %%zmm1,%%zmm5,%%zmm1 \n"
|
||||
"vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
|
||||
"vpermd %%zmm0,%%zmm1,%%zmm0 \n"
|
||||
"vmovdqu32 %%zmm0,(%1)%{%%k1%} \n"
|
||||
|
||||
"99: \n"
|
||||
"vzeroupper \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(temp_width), // %2
|
||||
"=&r"(temp) // %3
|
||||
: "m"(kARGBShuffleMirror_AVX512) // %4
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm5", "k1");
|
||||
}
|
||||
#endif // HAS_ARGBMIRRORROW_AVX512BW
|
||||
|
||||
#ifdef HAS_SPLITUVROW_AVX2
|
||||
void SplitUVRow_AVX2(const uint8_t* src_uv,
|
||||
uint8_t* dst_u,
|
||||
@ -7722,6 +7903,60 @@ void ARGBExtractAlphaRow_AVX2(const uint8_t* src_argb,
|
||||
}
|
||||
#endif // HAS_ARGBEXTRACTALPHAROW_AVX2
|
||||
|
||||
#ifdef HAS_ARGBEXTRACTALPHAROW_AVX512BW
|
||||
void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb,
|
||||
uint8_t* dst_a,
|
||||
int width) {
|
||||
uintptr_t temp;
|
||||
asm volatile(
|
||||
"sub $0x20,%2 \n"
|
||||
"jl 2f \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vmovdqu32 (%0),%%zmm0 \n"
|
||||
"vmovdqu32 0x40(%0),%%zmm1 \n"
|
||||
"lea 0x80(%0),%0 \n"
|
||||
"vpsrld $0x18,%%zmm0,%%zmm0 \n"
|
||||
"vpsrld $0x18,%%zmm1,%%zmm1 \n"
|
||||
"vpmovdb %%zmm0,(%1) \n"
|
||||
"vpmovdb %%zmm1,0x10(%1) \n"
|
||||
"lea 0x20(%1),%1 \n"
|
||||
"sub $0x20,%2 \n"
|
||||
"jge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"add $0x20,%2 \n"
|
||||
"jle 99f \n"
|
||||
"cmp $0x10,%2 \n"
|
||||
"jl 3f \n"
|
||||
"vmovdqu32 (%0),%%zmm0 \n"
|
||||
"lea 0x40(%0),%0 \n"
|
||||
"vpsrld $0x18,%%zmm0,%%zmm0 \n"
|
||||
"vpmovdb %%zmm0,(%1) \n"
|
||||
"lea 0x10(%1),%1 \n"
|
||||
"sub $0x10,%2 \n"
|
||||
"jz 99f \n"
|
||||
|
||||
"3: \n"
|
||||
"mov $-1,%k3 \n"
|
||||
"bzhi %k2,%k3,%k3 \n"
|
||||
"kmovw %k3,%%k1 \n"
|
||||
"vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
|
||||
"vpsrld $0x18,%%zmm0,%%zmm0 \n"
|
||||
"vpmovdb %%zmm0,(%1)%{%%k1%} \n"
|
||||
|
||||
"99: \n"
|
||||
"vzeroupper \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_a), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(temp) // %3
|
||||
:
|
||||
: "memory", "cc", "xmm0", "xmm1", "k1");
|
||||
}
|
||||
#endif // HAS_ARGBEXTRACTALPHAROW_AVX512BW
|
||||
|
||||
#ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2
|
||||
// width in pixels
|
||||
void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
|
||||
@ -7789,6 +8024,64 @@ void ARGBCopyYToAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
||||
}
|
||||
#endif // HAS_ARGBCOPYYTOALPHAROW_AVX2
|
||||
|
||||
#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX512BW
|
||||
void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
||||
uintptr_t temp;
|
||||
asm volatile(
|
||||
"vpternlogd $0xff,%%zmm0,%%zmm0,%%zmm0 \n"
|
||||
"vpsrld $0x8,%%zmm0,%%zmm0 \n"
|
||||
"sub $0x20,%2 \n"
|
||||
"jl 2f \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vpmovzxbd (%0),%%zmm1 \n"
|
||||
"vpmovzxbd 0x10(%0),%%zmm2 \n"
|
||||
"lea 0x20(%0),%0 \n"
|
||||
"vpslld $0x18,%%zmm1,%%zmm1 \n"
|
||||
"vpslld $0x18,%%zmm2,%%zmm2 \n"
|
||||
"vpternlogd $0xf8,(%1),%%zmm0,%%zmm1 \n"
|
||||
"vpternlogd $0xf8,0x40(%1),%%zmm0,%%zmm2 \n"
|
||||
"vmovdqu32 %%zmm1,(%1) \n"
|
||||
"vmovdqu32 %%zmm2,0x40(%1) \n"
|
||||
"lea 0x80(%1),%1 \n"
|
||||
"sub $0x20,%2 \n"
|
||||
"jge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"add $0x20,%2 \n"
|
||||
"jle 99f \n"
|
||||
"cmp $0x10,%2 \n"
|
||||
"jl 3f \n"
|
||||
"vpmovzxbd (%0),%%zmm1 \n"
|
||||
"lea 0x10(%0),%0 \n"
|
||||
"vpslld $0x18,%%zmm1,%%zmm1 \n"
|
||||
"vpternlogd $0xf8,(%1),%%zmm0,%%zmm1 \n"
|
||||
"vmovdqu32 %%zmm1,(%1) \n"
|
||||
"lea 0x40(%1),%1 \n"
|
||||
"sub $0x10,%2 \n"
|
||||
"jz 99f \n"
|
||||
|
||||
"3: \n"
|
||||
"mov $-1,%k3 \n"
|
||||
"bzhi %k2,%k3,%k3 \n"
|
||||
"kmovw %k3,%%k1 \n"
|
||||
"vpmovzxbd (%0),%%zmm1%{%%k1%}%{z%} \n"
|
||||
"vpslld $0x18,%%zmm1,%%zmm1 \n"
|
||||
"vpternlogd $0xf8,(%1),%%zmm0,%%zmm1%{%%k1%}%{z%} \n"
|
||||
"vmovdqu32 %%zmm1,(%1)%{%%k1%} \n"
|
||||
|
||||
"99: \n"
|
||||
"vzeroupper \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(temp) // %3
|
||||
:
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "k1");
|
||||
}
|
||||
#endif // HAS_ARGBCOPYYTOALPHAROW_AVX512BW
|
||||
|
||||
#ifdef HAS_SETROW_X86
|
||||
void SetRow_X86(uint8_t* dst, uint8_t v8, int width) {
|
||||
size_t width_tmp = (size_t)(width >> 2);
|
||||
|
||||
@ -1834,6 +1834,26 @@ void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb,
|
||||
);
|
||||
}
|
||||
|
||||
void ARGBCopyYToAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"vld4.8 {d0, d1, d2, d3}, [%1] \n" // load 8 ARGB
|
||||
"pld [%0, #448] \n"
|
||||
"vld1.8 {d3}, [%0]! \n" // replace A with 8 Y
|
||||
"pld [%1, #448] \n"
|
||||
"vst4.8 {d0, d1, d2, d3}, [%1]! \n"
|
||||
"vld4.8 {d4, d5, d6, d7}, [%1] \n" // load 8 ARGB
|
||||
"vld1.8 {d7}, [%0]! \n" // replace A with 8 Y
|
||||
"vst4.8 {d4, d5, d6, d7}, [%1]! \n"
|
||||
"subs %2, %2, #16 \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "cc", "memory", "q0", "q1", "q2", "q3");
|
||||
}
|
||||
|
||||
// 8x1 pixels.
|
||||
void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_u,
|
||||
|
||||
@ -2721,10 +2721,15 @@ void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb,
|
||||
int width) {
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16
|
||||
"ldp q0, q1, [%0] \n" // load 8 ARGB pixels
|
||||
"ldp q2, q3, [%0, #32] \n" // load next 8 ARGB pixels
|
||||
"add %0, %0, #64 \n"
|
||||
"prfm pldl1keep, [%0, 448] \n"
|
||||
"uzp2 v0.8h, v0.8h, v1.8h \n"
|
||||
"uzp2 v2.8h, v2.8h, v3.8h \n"
|
||||
"uzp2 v0.16b, v0.16b, v2.16b \n"
|
||||
"subs %w2, %w2, #16 \n" // 16 processed per loop
|
||||
"st1 {v3.16b}, [%1], #16 \n" // store 16 A's.
|
||||
"str q0, [%1], #16 \n" // store 16 A's.
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_a), // %1
|
||||
@ -2734,6 +2739,36 @@ void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb,
|
||||
);
|
||||
}
|
||||
|
||||
void ARGBCopyYToAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldr q4, [%0], #16 \n"
|
||||
"ldp q0, q1, [%1] \n"
|
||||
"ldp q2, q3, [%1, #32] \n"
|
||||
"prfm pldl1keep, [%0, 448] \n"
|
||||
"prfm pldl1keep, [%1, 448] \n"
|
||||
"zip1 v6.16b, v4.16b, v4.16b \n"
|
||||
"zip2 v7.16b, v4.16b, v4.16b \n"
|
||||
"zip1 v4.8h, v6.8h, v6.8h \n"
|
||||
"zip2 v5.8h, v6.8h, v6.8h \n"
|
||||
"zip1 v6.8h, v7.8h, v7.8h \n"
|
||||
"zip2 v7.8h, v7.8h, v7.8h \n"
|
||||
"sli v0.4s, v4.4s, #24 \n"
|
||||
"sli v1.4s, v5.4s, #24 \n"
|
||||
"sli v2.4s, v6.4s, #24 \n"
|
||||
"sli v3.4s, v7.4s, #24 \n"
|
||||
"stp q0, q1, [%1] \n"
|
||||
"stp q2, q3, [%1, #32] \n"
|
||||
"add %1, %1, #64 \n"
|
||||
"subs %w2, %w2, #16 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
|
||||
}
|
||||
|
||||
// Coefficients expressed as negatives to allow 128
|
||||
struct RgbUVConstants {
|
||||
int8_t kRGBToU[4];
|
||||
|
||||
@ -2355,10 +2355,10 @@ void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
|
||||
int vl;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"vsetvli %[vl], %[w], e16, m2, ta, ma \n"
|
||||
"vsetvli %[vl], %[w], e16, m4, ta, ma \n"
|
||||
"vle32.v v8, (%[src_argb]) \n"
|
||||
"vnsrl.wi v16, v8, 16 \n"
|
||||
"vsetvli zero, zero, e8, m1, ta, ma \n"
|
||||
"vsetvli zero, zero, e8, m2, ta, ma \n"
|
||||
"vnsrl.wi v8, v16, 8 \n"
|
||||
"vse8.v v8, (%[dst_a]) \n"
|
||||
"sub %[w], %[w], %[vl] \n"
|
||||
@ -2371,7 +2371,8 @@ void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
|
||||
[w] "+r"(width), // %[w]
|
||||
[vl] "=&r"(vl) // %[vl]
|
||||
:
|
||||
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
|
||||
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
|
||||
"v15", "v16", "v17", "v18", "v19");
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -2379,23 +2380,28 @@ void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
|
||||
void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
|
||||
int vl;
|
||||
asm volatile(
|
||||
"addi %[dst], %[dst], 3 \n"
|
||||
|
||||
"1: \n"
|
||||
"vsetvli %[vl], %[w], e8, m4, ta, ma \n"
|
||||
"vle8.v v8, (%[src]) \n"
|
||||
"vsse8.v v8, (%[dst]), %[dst_stride] \n"
|
||||
"vsetvli %[vl], %[w], e32, m8, ta, ma \n"
|
||||
"vle8.v v16, (%[src]) \n"
|
||||
"vle32.v v8, (%[dst]) \n"
|
||||
"vzext.vf4 v24, v16 \n"
|
||||
"vand.vx v8, v8, %[mask] \n"
|
||||
"vsll.vi v24, v24, 24 \n"
|
||||
"vor.vv v8, v8, v24 \n"
|
||||
"vse32.v v8, (%[dst]) \n"
|
||||
"sub %[w], %[w], %[vl] \n"
|
||||
"add %[src], %[src], %[vl] \n"
|
||||
"slli %[vl], %[vl], 2 \n"
|
||||
"add %[dst], %[dst], %[vl] \n"
|
||||
"bgtz %[w], 1b \n"
|
||||
: [src] "+r"(src), // %[src]
|
||||
[dst] "+r"(dst), // %[dst]
|
||||
[w] "+r"(width), // %[w]
|
||||
[vl] "=&r"(vl) // %[vl]
|
||||
: [dst_stride] "r"(4) // %[dst_stride]
|
||||
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11");
|
||||
: [src] "+r"(src), // %[src]
|
||||
[dst] "+r"(dst), // %[dst]
|
||||
[w] "+r"(width), // %[w]
|
||||
[vl] "=&r"(vl) // %[vl]
|
||||
: [mask] "r"(0x00ffffffu) // %[mask]
|
||||
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
|
||||
"v15", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
|
||||
"v31");
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@ -1914,6 +1914,62 @@ void BlendPlaneRow_WASMSIMD(const uint8_t* src0,
|
||||
}
|
||||
#endif // HAS_BLENDPLANEROW_WASMSIMD
|
||||
|
||||
#ifdef HAS_ARGBEXTRACTALPHAROW_WASMSIMD
|
||||
void ARGBExtractAlphaRow_WASMSIMD(const uint8_t* src_argb,
|
||||
uint8_t* dst_a,
|
||||
int width) {
|
||||
do {
|
||||
asm("" : "+r"(src_argb), "+r"(dst_a));
|
||||
v128_t s0 = wasm_v128_load(src_argb);
|
||||
v128_t s1 = wasm_v128_load(src_argb + 16);
|
||||
v128_t s2 = wasm_v128_load(src_argb + 32);
|
||||
v128_t s3 = wasm_v128_load(src_argb + 48);
|
||||
s0 = wasm_u32x4_shr(s0, 24);
|
||||
s1 = wasm_u32x4_shr(s1, 24);
|
||||
s2 = wasm_u32x4_shr(s2, 24);
|
||||
s3 = wasm_u32x4_shr(s3, 24);
|
||||
v128_t a = wasm_u8x16_narrow_i16x8(wasm_i16x8_narrow_i32x4(s0, s1),
|
||||
wasm_i16x8_narrow_i32x4(s2, s3));
|
||||
wasm_v128_store(dst_a, a);
|
||||
src_argb += 64;
|
||||
dst_a += 16;
|
||||
width -= 16;
|
||||
} while (width > 0);
|
||||
}
|
||||
#endif // HAS_ARGBEXTRACTALPHAROW_WASMSIMD
|
||||
|
||||
#ifdef HAS_ARGBCOPYYTOALPHAROW_WASMSIMD
|
||||
// Swizzle Y[i] to the alpha byte of 32 bit lane i; 128 indices give 0.
|
||||
static const uint8_t kYToAlphaSwizzleTable[64] = {
|
||||
128, 128, 128, 0, 128, 128, 128, 1, 128, 128, 128, 2, 128, 128, 128, 3,
|
||||
128, 128, 128, 4, 128, 128, 128, 5, 128, 128, 128, 6, 128, 128, 128, 7,
|
||||
128, 128, 128, 8, 128, 128, 128, 9, 128, 128, 128, 10, 128, 128, 128, 11,
|
||||
128, 128, 128, 12, 128, 128, 128, 13, 128, 128, 128, 14, 128, 128, 128, 15};
|
||||
|
||||
void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) {
|
||||
v128_t rgb_mask = SplatConst32(0x00ffffff);
|
||||
v128_t idx0 = LoadConst(kYToAlphaSwizzleTable);
|
||||
v128_t idx1 = LoadConst(kYToAlphaSwizzleTable + 16);
|
||||
v128_t idx2 = LoadConst(kYToAlphaSwizzleTable + 32);
|
||||
v128_t idx3 = LoadConst(kYToAlphaSwizzleTable + 48);
|
||||
do {
|
||||
asm("" : "+r"(src), "+r"(dst));
|
||||
v128_t y = wasm_v128_load(src);
|
||||
v128_t d0 = wasm_v128_and(wasm_v128_load(dst), rgb_mask);
|
||||
v128_t d1 = wasm_v128_and(wasm_v128_load(dst + 16), rgb_mask);
|
||||
v128_t d2 = wasm_v128_and(wasm_v128_load(dst + 32), rgb_mask);
|
||||
v128_t d3 = wasm_v128_and(wasm_v128_load(dst + 48), rgb_mask);
|
||||
wasm_v128_store(dst, wasm_v128_or(d0, LIBYUV_WASM_SWIZZLE(y, idx0)));
|
||||
wasm_v128_store(dst + 16, wasm_v128_or(d1, LIBYUV_WASM_SWIZZLE(y, idx1)));
|
||||
wasm_v128_store(dst + 32, wasm_v128_or(d2, LIBYUV_WASM_SWIZZLE(y, idx2)));
|
||||
wasm_v128_store(dst + 48, wasm_v128_or(d3, LIBYUV_WASM_SWIZZLE(y, idx3)));
|
||||
src += 16;
|
||||
dst += 64;
|
||||
width -= 16;
|
||||
} while (width > 0);
|
||||
}
|
||||
#endif // HAS_ARGBCOPYYTOALPHAROW_WASMSIMD
|
||||
|
||||
#undef LIBYUV_WASM_SWIZZLE
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -2581,6 +2581,188 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0,
|
||||
}
|
||||
#endif // HAS_BLENDPLANEROW_AVX512BW
|
||||
|
||||
#ifdef HAS_ARGBEXTRACTALPHAROW_AVX512BW
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb,
|
||||
uint8_t* dst_a,
|
||||
int width) {
|
||||
while (width >= 32) {
|
||||
__m512i z0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
||||
__m512i z1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
||||
__m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24));
|
||||
__m128i a1 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z1, 24));
|
||||
_mm_storeu_si128((__m128i*)dst_a, a0);
|
||||
_mm_storeu_si128((__m128i*)(dst_a + 16), a1);
|
||||
src_argb += 128;
|
||||
dst_a += 32;
|
||||
width -= 32;
|
||||
}
|
||||
if (width >= 16) {
|
||||
__m512i z0 = _mm512_loadu_si512((const __m512i*)src_argb);
|
||||
__m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24));
|
||||
_mm_storeu_si128((__m128i*)dst_a, a0);
|
||||
src_argb += 64;
|
||||
dst_a += 16;
|
||||
width -= 16;
|
||||
}
|
||||
if (width > 0) {
|
||||
__mmask16 mask = (__mmask16)((1u << width) - 1u);
|
||||
__m512i z0 = _mm512_maskz_loadu_epi32(mask, src_argb);
|
||||
_mm512_mask_cvtepi32_storeu_epi8(dst_a, mask, _mm512_srli_epi32(z0, 24));
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // HAS_ARGBEXTRACTALPHAROW_AVX512BW
|
||||
|
||||
#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX512BW
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
||||
__m512i rgb_mask = _mm512_set1_epi32(0x00ffffff);
|
||||
while (width >= 32) {
|
||||
__m512i a0 = _mm512_slli_epi32(
|
||||
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)src)), 24);
|
||||
__m512i a1 = _mm512_slli_epi32(
|
||||
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)(src + 16))), 24);
|
||||
a0 = _mm512_ternarylogic_epi32(
|
||||
a0, rgb_mask, _mm512_loadu_si512((const __m512i*)dst), 0xf8);
|
||||
a1 = _mm512_ternarylogic_epi32(
|
||||
a1, rgb_mask, _mm512_loadu_si512((const __m512i*)(dst + 64)), 0xf8);
|
||||
_mm512_storeu_si512((__m512i*)dst, a0);
|
||||
_mm512_storeu_si512((__m512i*)(dst + 64), a1);
|
||||
src += 32;
|
||||
dst += 128;
|
||||
width -= 32;
|
||||
}
|
||||
if (width >= 16) {
|
||||
__m512i a0 = _mm512_slli_epi32(
|
||||
_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)src)), 24);
|
||||
a0 = _mm512_ternarylogic_epi32(
|
||||
a0, rgb_mask, _mm512_loadu_si512((const __m512i*)dst), 0xf8);
|
||||
_mm512_storeu_si512((__m512i*)dst, a0);
|
||||
src += 16;
|
||||
dst += 64;
|
||||
width -= 16;
|
||||
}
|
||||
if (width > 0) {
|
||||
__mmask16 mask = (__mmask16)((1u << width) - 1u);
|
||||
__m512i a0 = _mm512_slli_epi32(
|
||||
_mm512_maskz_cvtepu8_epi32(mask, _mm_maskz_loadu_epi8(mask, src)), 24);
|
||||
a0 = _mm512_ternarylogic_epi32(a0, rgb_mask,
|
||||
_mm512_maskz_loadu_epi32(mask, dst), 0xf8);
|
||||
_mm512_mask_storeu_epi32(dst, mask, a0);
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // HAS_ARGBCOPYYTOALPHAROW_AVX512BW
|
||||
|
||||
#ifdef HAS_ARGBMIRRORROW_AVX512BW
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
||||
const __m512i rev =
|
||||
_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
|
||||
const uint8_t* s = src + (ptrdiff_t)width * 4;
|
||||
while (width >= 16) {
|
||||
s -= 64;
|
||||
_mm512_storeu_si512(
|
||||
(__m512i*)dst,
|
||||
_mm512_permutexvar_epi32(rev, _mm512_loadu_si512((const __m512i*)s)));
|
||||
dst += 64;
|
||||
width -= 16;
|
||||
}
|
||||
if (width > 0) {
|
||||
// Remaining 1 to 15 pixels are at the start of src.
|
||||
__mmask16 mask = (__mmask16)((1u << width) - 1u);
|
||||
__m512i idx = _mm512_sub_epi32(rev, _mm512_set1_epi32(16 - width));
|
||||
__m512i v = _mm512_maskz_loadu_epi32(mask, src);
|
||||
_mm512_mask_storeu_epi32(dst, mask, _mm512_permutexvar_epi32(idx, v));
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // HAS_ARGBMIRRORROW_AVX512BW
|
||||
|
||||
#ifdef HAS_RGB24MIRRORROW_AVX512VBMI
|
||||
static const uint8_t kPermMirrorRGB24_0[64] = {
|
||||
125u, 126u, 127u, 122u, 123u, 124u, 119u, 120u, 121u, 116u, 117u,
|
||||
118u, 113u, 114u, 115u, 110u, 111u, 112u, 107u, 108u, 109u, 104u,
|
||||
105u, 106u, 101u, 102u, 103u, 98u, 99u, 100u, 95u, 96u, 97u,
|
||||
92u, 93u, 94u, 89u, 90u, 91u, 86u, 87u, 88u, 83u, 84u,
|
||||
85u, 80u, 81u, 82u, 77u, 78u, 79u, 74u, 75u, 76u, 71u,
|
||||
72u, 73u, 68u, 69u, 70u, 65u, 66u, 67u, 62u};
|
||||
static const uint8_t kPermMirrorRGB24_1[64] = {
|
||||
127u, 128u, 123u, 124u, 125u, 120u, 121u, 122u, 117u, 118u, 119u,
|
||||
114u, 115u, 116u, 111u, 112u, 113u, 108u, 109u, 110u, 105u, 106u,
|
||||
107u, 102u, 103u, 104u, 99u, 100u, 101u, 96u, 97u, 98u, 93u,
|
||||
94u, 95u, 90u, 91u, 92u, 87u, 88u, 89u, 84u, 85u, 86u,
|
||||
81u, 82u, 83u, 78u, 79u, 80u, 75u, 76u, 77u, 72u, 73u,
|
||||
74u, 69u, 70u, 71u, 66u, 67u, 68u, 63u, 64u};
|
||||
static const uint8_t kPermMirrorRGB24_2[64] = {
|
||||
65u, 60u, 61u, 62u, 57u, 58u, 59u, 54u, 55u, 56u, 51u, 52u, 53u,
|
||||
48u, 49u, 50u, 45u, 46u, 47u, 42u, 43u, 44u, 39u, 40u, 41u, 36u,
|
||||
37u, 38u, 33u, 34u, 35u, 30u, 31u, 32u, 27u, 28u, 29u, 24u, 25u,
|
||||
26u, 21u, 22u, 23u, 18u, 19u, 20u, 15u, 16u, 17u, 12u, 13u, 14u,
|
||||
9u, 10u, 11u, 6u, 7u, 8u, 3u, 4u, 5u, 0u, 1u, 2u};
|
||||
|
||||
// Byte mask with the low n bits set, clamped to [0, 64].
|
||||
static inline __mmask64 RGB24MirrorMask(int n) {
|
||||
return n <= 0 ? 0ull : n >= 64 ? ~0ull : (1ull << n) - 1ull;
|
||||
}
|
||||
|
||||
// Mirror 64 RGB24 pixels (3 full zmm vectors) per loop. The remaining 1 to
|
||||
// 63 pixels use masked loads and stores that access only valid bytes.
|
||||
LIBYUV_TARGET_AVX512VBMI
|
||||
void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24,
|
||||
uint8_t* dst_rgb24,
|
||||
int width) {
|
||||
const __m512i idx0 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_0);
|
||||
const __m512i idx1 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_1);
|
||||
const __m512i idx2 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_2);
|
||||
int r = width & 63;
|
||||
int w = width - r;
|
||||
if (w > 0) {
|
||||
const uint8_t* src = src_rgb24 + (ptrdiff_t)width * 3 - 192;
|
||||
do {
|
||||
__m512i a = _mm512_loadu_si512((const __m512i*)src);
|
||||
__m512i b = _mm512_loadu_si512((const __m512i*)(src + 64));
|
||||
__m512i c = _mm512_loadu_si512((const __m512i*)(src + 128));
|
||||
__m512i d0 = _mm512_permutex2var_epi8(b, idx0, c);
|
||||
__m512i d1 = _mm512_permutex2var_epi8(a, idx1, b);
|
||||
__m512i d2 = _mm512_permutex2var_epi8(a, idx2, b);
|
||||
d1 = _mm512_mask_broadcastb_epi8(d1, (__mmask64)2,
|
||||
_mm512_castsi512_si128(c));
|
||||
_mm512_storeu_si512((__m512i*)dst_rgb24, d0);
|
||||
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), d1);
|
||||
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), d2);
|
||||
src -= 192;
|
||||
dst_rgb24 += 192;
|
||||
w -= 64;
|
||||
} while (w > 0);
|
||||
}
|
||||
if (r) {
|
||||
int nb = r * 3;
|
||||
int off = 192 - nb;
|
||||
__mmask64 m0 = RGB24MirrorMask(nb);
|
||||
__mmask64 m1 = RGB24MirrorMask(nb - 64);
|
||||
__mmask64 m2 = RGB24MirrorMask(nb - 128);
|
||||
__m512i a = _mm512_maskz_loadu_epi8(m0, src_rgb24);
|
||||
__m512i b = _mm512_maskz_loadu_epi8(m1, src_rgb24 + 64);
|
||||
__m512i c = _mm512_maskz_loadu_epi8(m2, src_rgb24 + 128);
|
||||
__m512i t0 = _mm512_sub_epi8(idx0, _mm512_set1_epi8((char)(off - 64)));
|
||||
__m512i t1 = _mm512_sub_epi8(idx1, _mm512_set1_epi8((char)off));
|
||||
__m512i t2 = _mm512_sub_epi8(idx2, _mm512_set1_epi8((char)off));
|
||||
__m512i d0 = _mm512_mask_permutexvar_epi8(
|
||||
_mm512_permutex2var_epi8(a, t0, b), _mm512_movepi8_mask(t0), t0, c);
|
||||
__m512i d1 = _mm512_mask_permutexvar_epi8(
|
||||
_mm512_permutex2var_epi8(a, t1, b), _mm512_movepi8_mask(t1), t1, c);
|
||||
__m512i d2 = _mm512_mask_permutexvar_epi8(
|
||||
_mm512_permutex2var_epi8(a, t2, b), _mm512_movepi8_mask(t2), t2, c);
|
||||
_mm512_mask_storeu_epi8(dst_rgb24, m0, d0);
|
||||
_mm512_mask_storeu_epi8(dst_rgb24 + 64, m1, d1);
|
||||
_mm512_mask_storeu_epi8(dst_rgb24 + 128, m2, d2);
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // HAS_RGB24MIRRORROW_AVX512VBMI
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user