diff --git a/include/libyuv/row.h b/include/libyuv/row.h index b10d18218..a430db8e5 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -385,6 +385,10 @@ extern "C" { #define HAS_RGBTOYMATRIXROW_AVX512BW #define HAS_BLENDPLANEROW_AVX512BW #define HAS_INTERPOLATEROW_AVX512BW +#define HAS_ARGBEXTRACTALPHAROW_AVX512BW +#define HAS_ARGBCOPYYTOALPHAROW_AVX512BW +#define HAS_ARGBMIRRORROW_AVX512BW +#define HAS_RGB24MIRRORROW_AVX512VBMI #endif #define HAS_ARGBTOYMATRIXROW_AVX2 #define HAS_RGBTOYMATRIXROW_AVX2 @@ -452,6 +456,10 @@ extern "C" { #define HAS_SPLITUVROW_AVX512BW #define HAS_RGBTOUVMATRIXROW_AVX512BW #define HAS_RGBTOYMATRIXROW_AVX512BW +#define HAS_ARGBEXTRACTALPHAROW_AVX512BW +#define HAS_ARGBCOPYYTOALPHAROW_AVX512BW +#define HAS_ARGBMIRRORROW_AVX512BW +#define HAS_RGB24MIRRORROW_AVX512VBMI #endif // The following are available on Neon platforms: @@ -465,6 +473,7 @@ extern "C" { #define HAS_ARGB4444TOARGBROW_NEON #define HAS_ARGB4444TOUVROW_NEON #define HAS_ARGB4444TOYROW_NEON +#define HAS_ARGBCOPYYTOALPHAROW_NEON #define HAS_ARGBEXTRACTALPHAROW_NEON #define HAS_ARGBSETROW_NEON #define HAS_ARGBTOAB64ROW_NEON @@ -937,6 +946,8 @@ extern "C" { #define HAS_ARGB4444TOUVMATRIXROW_WASMSIMD #define HAS_ARGB4444TOYMATRIXROW_WASMSIMD #define HAS_ARGBBLENDROW_WASMSIMD +#define HAS_ARGBCOPYYTOALPHAROW_WASMSIMD +#define HAS_ARGBEXTRACTALPHAROW_WASMSIMD #define HAS_ARGBMIRRORROW_WASMSIMD #define HAS_ARGBSHUFFLEROW_WASMSIMD #define HAS_ARGBTOARGB1555ROW_WASMSIMD @@ -2770,6 +2781,7 @@ void MirrorSplitUVRow_C(const uint8_t* src_uv, void MirrorRow_16_C(const uint16_t* src, uint16_t* dst, int width); void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width); +void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width); void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width); void ARGBMirrorRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBMirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width); @@ -2792,6 +2804,9 @@ void ARGBMirrorRow_Any_LASX(const uint8_t* src_ptr, void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width); +void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width); void RGB24MirrorRow_NEON(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width); @@ -3922,12 +3937,18 @@ void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb, void ARGBExtractAlphaRow_AVX2(const uint8_t* src_argb, uint8_t* dst_a, int width); +void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_a, + int width); void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb, uint8_t* dst_a, int width); void ARGBExtractAlphaRow_LSX(const uint8_t* src_argb, uint8_t* dst_a, int width); +void ARGBExtractAlphaRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_a, + int width); void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb, uint8_t* dst_a, int width); @@ -3943,10 +3964,16 @@ void ARGBExtractAlphaRow_Any_NEON(const uint8_t* src_ptr, void ARGBExtractAlphaRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBExtractAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void ARGBCopyYToAlphaRow_C(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyYToAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width); +void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width); +void ARGBCopyYToAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width); +void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyYToAlphaRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -3954,6 +3981,12 @@ void ARGBCopyYToAlphaRow_Any_SSE2(const uint8_t* src_ptr, void ARGBCopyYToAlphaRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBCopyYToAlphaRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBCopyYToAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void SetRow_C(uint8_t* dst, uint8_t v8, int width); void SetRow_X86(uint8_t* dst, uint8_t v8, int width); diff --git a/source/convert.cc b/source/convert.cc index 39b7374f7..cc93155b1 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -3193,6 +3193,11 @@ int ARGBToI420Alpha(const uint8_t* src_argb, : ARGBExtractAlphaRow_Any_AVX2; } #endif +#if defined(HAS_ARGBEXTRACTALPHAROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBExtractAlphaRow = ARGBExtractAlphaRow_AVX512BW; + } +#endif #if defined(HAS_ARGBEXTRACTALPHAROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_NEON @@ -3205,6 +3210,13 @@ int ARGBToI420Alpha(const uint8_t* src_argb, : ARGBExtractAlphaRow_Any_LSX; } #endif +#if defined(HAS_ARGBEXTRACTALPHAROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBExtractAlphaRow = IS_ALIGNED(width, 16) + ? ARGBExtractAlphaRow_WASMSIMD + : ARGBExtractAlphaRow_Any_WASMSIMD; + } +#endif #if defined(HAS_ARGBEXTRACTALPHAROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBExtractAlphaRow = ARGBExtractAlphaRow_RVV; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index d0c09de37..6cde151f6 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -2970,6 +2970,11 @@ int ARGBMirror(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBMIRRORROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBMirrorRow = ARGBMirrorRow_AVX512BW; + } +#endif #if defined(HAS_ARGBMIRRORROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBMirrorRow = ARGBMirrorRow_Any_LSX; @@ -3041,6 +3046,11 @@ int RGB24Mirror(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGB24MIRRORROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + RGB24MirrorRow = RGB24MirrorRow_AVX512VBMI; + } +#endif #if defined(HAS_RGB24MIRRORROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { RGB24MirrorRow = RGB24MirrorRow_Any_WASMSIMD; @@ -5754,6 +5764,11 @@ int ARGBExtractAlpha(const uint8_t* src_argb, : ARGBExtractAlphaRow_Any_AVX2; } #endif +#if defined(HAS_ARGBEXTRACTALPHAROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBExtractAlphaRow = ARGBExtractAlphaRow_AVX512BW; + } +#endif #if defined(HAS_ARGBEXTRACTALPHAROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_NEON @@ -5766,6 +5781,13 @@ int ARGBExtractAlpha(const uint8_t* src_argb, : ARGBExtractAlphaRow_Any_LSX; } #endif +#if defined(HAS_ARGBEXTRACTALPHAROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBExtractAlphaRow = IS_ALIGNED(width, 16) + ? ARGBExtractAlphaRow_WASMSIMD + : ARGBExtractAlphaRow_Any_WASMSIMD; + } +#endif #if defined(HAS_ARGBEXTRACTALPHAROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBExtractAlphaRow = ARGBExtractAlphaRow_RVV; @@ -5823,6 +5845,24 @@ int ARGBCopyYToAlpha(const uint8_t* src_y, } } #endif +#if defined(HAS_ARGBCOPYYTOALPHAROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_AVX512BW; + } +#endif +#if defined(HAS_ARGBCOPYYTOALPHAROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBCopyYToAlphaRow = IS_ALIGNED(width, 16) ? ARGBCopyYToAlphaRow_NEON + : ARGBCopyYToAlphaRow_Any_NEON; + } +#endif +#if defined(HAS_ARGBCOPYYTOALPHAROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBCopyYToAlphaRow = IS_ALIGNED(width, 16) + ? ARGBCopyYToAlphaRow_WASMSIMD + : ARGBCopyYToAlphaRow_Any_WASMSIMD; + } +#endif #if defined(HAS_ARGBCOPYYTOALPHAROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_RVV; diff --git a/source/rotate_argb.cc b/source/rotate_argb.cc index 211119ae3..3220ab99e 100644 --- a/source/rotate_argb.cc +++ b/source/rotate_argb.cc @@ -160,6 +160,11 @@ static int ARGBRotate180(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBMIRRORROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBMirrorRow = ARGBMirrorRow_AVX512BW; + } +#endif #if defined(HAS_ARGBMIRRORROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBMirrorRow = ARGBMirrorRow_Any_LSX; diff --git a/source/row_any.cc b/source/row_any.cc index ad16d9670..43adb4758 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1321,6 +1321,14 @@ ANY11(ARGBExtractAlphaRow_Any_NEON, ARGBExtractAlphaRow_NEON, 0, 4, 1, 15) #ifdef HAS_ARGBEXTRACTALPHAROW_LSX ANY11(ARGBExtractAlphaRow_Any_LSX, ARGBExtractAlphaRow_LSX, 0, 4, 1, 15) #endif +#ifdef HAS_ARGBEXTRACTALPHAROW_WASMSIMD +ANY11(ARGBExtractAlphaRow_Any_WASMSIMD, + ARGBExtractAlphaRow_WASMSIMD, + 0, + 4, + 1, + 15) +#endif #ifdef HAS_RAWTOARGBROW_WASMSIMD ANY11(RAWToARGBRow_Any_WASMSIMD, RAWToARGBRow_WASMSIMD, 0, 3, 4, 15) #endif @@ -1387,6 +1395,17 @@ ANY11B(ARGBCopyYToAlphaRow_Any_AVX2, ARGBCopyYToAlphaRow_AVX2, 0, 1, 4, 15) #ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2 ANY11B(ARGBCopyYToAlphaRow_Any_SSE2, ARGBCopyYToAlphaRow_SSE2, 0, 1, 4, 7) #endif +#ifdef HAS_ARGBCOPYYTOALPHAROW_NEON +ANY11B(ARGBCopyYToAlphaRow_Any_NEON, ARGBCopyYToAlphaRow_NEON, 0, 1, 4, 15) +#endif +#ifdef HAS_ARGBCOPYYTOALPHAROW_WASMSIMD +ANY11B(ARGBCopyYToAlphaRow_Any_WASMSIMD, + ARGBCopyYToAlphaRow_WASMSIMD, + 0, + 1, + 4, + 15) +#endif #undef ANY11B // Any 1 to 1 with parameter. diff --git a/source/row_gcc.cc b/source/row_gcc.cc index fd56f4959..5d0d46060 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -5449,6 +5449,139 @@ void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24, } #endif // HAS_RGB24MIRRORROW_AVX2 +#ifdef HAS_RGB24MIRRORROW_AVX512VBMI +// Mirror 64 RGB24 pixels (192 bytes = 3 zmm) per loop. Loads and stores are +// 3 full contiguous zmm vectors. Output vector 0 is a 2 source byte permute +// of input vectors 1 and 2, and output vectors 1 and 2 are 2 source byte +// permutes of input vectors 0 and 1. Output byte 65 (byte 1 of output +// vector 1) is the only byte that comes from a third vector: byte 0 of input +// vector 2, which is merged in with a masked broadcast. +static const uint8_t kPermMirrorRGB24_0[64] = { + 125u, 126u, 127u, 122u, 123u, 124u, 119u, 120u, 121u, 116u, 117u, + 118u, 113u, 114u, 115u, 110u, 111u, 112u, 107u, 108u, 109u, 104u, + 105u, 106u, 101u, 102u, 103u, 98u, 99u, 100u, 95u, 96u, 97u, + 92u, 93u, 94u, 89u, 90u, 91u, 86u, 87u, 88u, 83u, 84u, + 85u, 80u, 81u, 82u, 77u, 78u, 79u, 74u, 75u, 76u, 71u, + 72u, 73u, 68u, 69u, 70u, 65u, 66u, 67u, 62u}; +static const uint8_t kPermMirrorRGB24_1[64] = { + 127u, 128u, 123u, 124u, 125u, 120u, 121u, 122u, 117u, 118u, 119u, + 114u, 115u, 116u, 111u, 112u, 113u, 108u, 109u, 110u, 105u, 106u, + 107u, 102u, 103u, 104u, 99u, 100u, 101u, 96u, 97u, 98u, 93u, + 94u, 95u, 90u, 91u, 92u, 87u, 88u, 89u, 84u, 85u, 86u, + 81u, 82u, 83u, 78u, 79u, 80u, 75u, 76u, 77u, 72u, 73u, + 74u, 69u, 70u, 71u, 66u, 67u, 68u, 63u, 64u}; +static const uint8_t kPermMirrorRGB24_2[64] = { + 65u, 60u, 61u, 62u, 57u, 58u, 59u, 54u, 55u, 56u, 51u, 52u, 53u, + 48u, 49u, 50u, 45u, 46u, 47u, 42u, 43u, 44u, 39u, 40u, 41u, 36u, + 37u, 38u, 33u, 34u, 35u, 30u, 31u, 32u, 27u, 28u, 29u, 24u, 25u, + 26u, 21u, 22u, 23u, 18u, 19u, 20u, 15u, 16u, 17u, 12u, 13u, 14u, + 9u, 10u, 11u, 6u, 7u, 8u, 3u, 4u, 5u, 0u, 1u, 2u}; + +// Byte mask with the low n bits set, clamped to [0, 64]. +static inline uint64_t RGB24MirrorMask(int n) { + return n <= 0 ? 0ull : n >= 64 ? ~0ull : (1ull << n) - 1ull; +} + +void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width) { + int r = width & 63; + ptrdiff_t temp_width = (ptrdiff_t)(width - r); + if (temp_width > 0) { + const uint8_t* src = src_rgb24 + (ptrdiff_t)width * 3 - 192; + asm volatile( + "vmovdqu8 %3,%%zmm20 \n" + "vmovdqu8 %4,%%zmm21 \n" + "vmovdqu8 %5,%%zmm22 \n" + "kmovq %6,%%k1 \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%0),%%zmm0 \n" // last 64 pixels + "vmovdqu8 0x40(%0),%%zmm1 \n" + "vmovdqu8 0x80(%0),%%zmm2 \n" + "lea -0xc0(%0),%0 \n" + "vmovdqa64 %%zmm20,%%zmm3 \n" + "vmovdqa64 %%zmm21,%%zmm4 \n" + "vmovdqa64 %%zmm22,%%zmm5 \n" + "vpermi2b %%zmm2,%%zmm1,%%zmm3 \n" // out0 from in1:in2 + "vpermi2b %%zmm1,%%zmm0,%%zmm4 \n" // out1 from in0:in1 + "vpermi2b %%zmm1,%%zmm0,%%zmm5 \n" // out2 from in0:in1 + "vpbroadcastb %%xmm2,%%zmm4%{%%k1%} \n" // out1 byte 1 = in2[0] + "vmovdqu8 %%zmm3,(%1) \n" + "vmovdqu8 %%zmm4,0x40(%1) \n" + "vmovdqu8 %%zmm5,0x80(%1) \n" + "lea 0xc0(%1),%1 \n" + "sub $0x40,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst_rgb24), // %1 + "+r"(temp_width) // %2 + : "m"(kPermMirrorRGB24_0), // %3 + "m"(kPermMirrorRGB24_1), // %4 + "m"(kPermMirrorRGB24_2), // %5 + "r"((uint64_t)2) // %6 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm20", "xmm21", "xmm22", "k1"); + } + // Remaining 1 to 63 pixels are the first r pixels of the source. Masked + // loads and stores access only the 3 * r valid bytes. The tables, as + // global byte offsets into a 192 byte block, are adjusted by the 192 - 3 * r + // missing bytes. Bytes from the third vector (index >= 128) are selected + // with a mask from the sign bit of the index. + if (r) { + int nb = r * 3; + int off = 192 - nb; + asm volatile( + "kmovq %2,%%k1 \n" + "kmovq %3,%%k2 \n" + "kmovq %4,%%k3 \n" + "vmovdqu8 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu8 0x40(%0),%%zmm1%{%%k2%}%{z%}\n" + "vmovdqu8 0x80(%0),%%zmm2%{%%k3%}%{z%}\n" + "vpbroadcastb %5,%%zmm6 \n" // off + "vpbroadcastb %6,%%zmm7 \n" // off - 64 + "vmovdqu8 %7,%%zmm3 \n" + "vmovdqu8 %8,%%zmm4 \n" + "vmovdqu8 %9,%%zmm5 \n" + "vpsubb %%zmm7,%%zmm3,%%zmm3 \n" + "vpsubb %%zmm6,%%zmm4,%%zmm4 \n" + "vpsubb %%zmm6,%%zmm5,%%zmm5 \n" + + "vpmovb2m %%zmm3,%%k4 \n" // out0 + "vmovdqa64 %%zmm3,%%zmm6 \n" + "vpermi2b %%zmm1,%%zmm0,%%zmm6 \n" + "vpermb %%zmm2,%%zmm3,%%zmm6%{%%k4%}\n" + "vmovdqu8 %%zmm6,(%1)%{%%k1%} \n" + "vpmovb2m %%zmm4,%%k4 \n" // out1 + "vmovdqa64 %%zmm4,%%zmm6 \n" + "vpermi2b %%zmm1,%%zmm0,%%zmm6 \n" + "vpermb %%zmm2,%%zmm4,%%zmm6%{%%k4%}\n" + "vmovdqu8 %%zmm6,0x40(%1)%{%%k2%} \n" + "vpmovb2m %%zmm5,%%k4 \n" // out2 + "vmovdqa64 %%zmm5,%%zmm6 \n" + "vpermi2b %%zmm1,%%zmm0,%%zmm6 \n" + "vpermb %%zmm2,%%zmm5,%%zmm6%{%%k4%}\n" + "vmovdqu8 %%zmm6,0x80(%1)%{%%k3%} \n" + "vzeroupper \n" + : + : "r"(src_rgb24), // %0 + "r"(dst_rgb24), // %1 + "r"(RGB24MirrorMask(nb)), // %2 + "r"(RGB24MirrorMask(nb - 64)), // %3 + "r"(RGB24MirrorMask(nb - 128)), // %4 + "r"(off), // %5 + "r"(off - 64), // %6 + "m"(kPermMirrorRGB24_0), // %7 + "m"(kPermMirrorRGB24_1), // %8 + "m"(kPermMirrorRGB24_2) // %9 + : "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "k1", "k2", "k3", "k4"); + } +} +#endif // HAS_RGB24MIRRORROW_AVX512VBMI + #ifdef HAS_ARGBMIRRORROW_SSE2 void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width) { @@ -5495,6 +5628,54 @@ void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBMIRRORROW_AVX2 +#ifdef HAS_ARGBMIRRORROW_AVX512BW +static const uint32_t kARGBShuffleMirror_AVX512[16] = { + 15u, 14u, 13u, 12u, 11u, 10u, 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u}; + +// Mirror 16 ARGB pixels per loop. The remaining 1 to 15 pixels are at the +// start of src. They are loaded with a k mask, permuted with the reverse +// index minus (16 - remaining), and stored with the same k mask. +void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + ptrdiff_t temp_width = (ptrdiff_t)(width); + uintptr_t temp; + asm volatile( + "vmovdqu32 %4,%%zmm5 \n" + "sub $0x10,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vpermd (%0,%2,4),%%zmm5,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x10,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%2 \n" + "jle 99f \n" + "mov $-1,%k3 \n" + "bzhi %k2,%k3,%k3 \n" + "kmovw %k3,%%k1 \n" + "mov $0x10,%k3 \n" + "sub %k2,%k3 \n" + "vpbroadcastd %k3,%%zmm1 \n" + "vpsubd %%zmm1,%%zmm5,%%zmm1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpermd %%zmm0,%%zmm1,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(temp_width), // %2 + "=&r"(temp) // %3 + : "m"(kARGBShuffleMirror_AVX512) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm5", "k1"); +} +#endif // HAS_ARGBMIRRORROW_AVX512BW + #ifdef HAS_SPLITUVROW_AVX2 void SplitUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_u, @@ -7722,6 +7903,60 @@ void ARGBExtractAlphaRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBEXTRACTALPHAROW_AVX2 +#ifdef HAS_ARGBEXTRACTALPHAROW_AVX512BW +void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_a, + int width) { + uintptr_t temp; + asm volatile( + "sub $0x20,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%0),%%zmm0 \n" + "vmovdqu32 0x40(%0),%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vpsrld $0x18,%%zmm0,%%zmm0 \n" + "vpsrld $0x18,%%zmm1,%%zmm1 \n" + "vpmovdb %%zmm0,(%1) \n" + "vpmovdb %%zmm1,0x10(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x20,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "cmp $0x10,%2 \n" + "jl 3f \n" + "vmovdqu32 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x18,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,(%1) \n" + "lea 0x10(%1),%1 \n" + "sub $0x10,%2 \n" + "jz 99f \n" + + "3: \n" + "mov $-1,%k3 \n" + "bzhi %k2,%k3,%k3 \n" + "kmovw %k3,%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpsrld $0x18,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_a), // %1 + "+r"(width), // %2 + "=&r"(temp) // %3 + : + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_ARGBEXTRACTALPHAROW_AVX512BW + #ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2 // width in pixels void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) { @@ -7789,6 +8024,64 @@ void ARGBCopyYToAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBCOPYYTOALPHAROW_AVX2 +#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX512BW +void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + uintptr_t temp; + asm volatile( + "vpternlogd $0xff,%%zmm0,%%zmm0,%%zmm0 \n" + "vpsrld $0x8,%%zmm0,%%zmm0 \n" + "sub $0x20,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vpmovzxbd (%0),%%zmm1 \n" + "vpmovzxbd 0x10(%0),%%zmm2 \n" + "lea 0x20(%0),%0 \n" + "vpslld $0x18,%%zmm1,%%zmm1 \n" + "vpslld $0x18,%%zmm2,%%zmm2 \n" + "vpternlogd $0xf8,(%1),%%zmm0,%%zmm1 \n" + "vpternlogd $0xf8,0x40(%1),%%zmm0,%%zmm2 \n" + "vmovdqu32 %%zmm1,(%1) \n" + "vmovdqu32 %%zmm2,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x20,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "cmp $0x10,%2 \n" + "jl 3f \n" + "vpmovzxbd (%0),%%zmm1 \n" + "lea 0x10(%0),%0 \n" + "vpslld $0x18,%%zmm1,%%zmm1 \n" + "vpternlogd $0xf8,(%1),%%zmm0,%%zmm1 \n" + "vmovdqu32 %%zmm1,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x10,%2 \n" + "jz 99f \n" + + "3: \n" + "mov $-1,%k3 \n" + "bzhi %k2,%k3,%k3 \n" + "kmovw %k3,%%k1 \n" + "vpmovzxbd (%0),%%zmm1%{%%k1%}%{z%} \n" + "vpslld $0x18,%%zmm1,%%zmm1 \n" + "vpternlogd $0xf8,(%1),%%zmm0,%%zmm1%{%%k1%}%{z%} \n" + "vmovdqu32 %%zmm1,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + "=&r"(temp) // %3 + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1"); +} +#endif // HAS_ARGBCOPYYTOALPHAROW_AVX512BW + #ifdef HAS_SETROW_X86 void SetRow_X86(uint8_t* dst, uint8_t v8, int width) { size_t width_tmp = (size_t)(width >> 2); diff --git a/source/row_neon.cc b/source/row_neon.cc index 0fff9ecd8..3dba9d423 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -1834,6 +1834,26 @@ void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb, ); } +void ARGBCopyYToAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) { + asm volatile( + "1: \n" + "vld4.8 {d0, d1, d2, d3}, [%1] \n" // load 8 ARGB + "pld [%0, #448] \n" + "vld1.8 {d3}, [%0]! \n" // replace A with 8 Y + "pld [%1, #448] \n" + "vst4.8 {d0, d1, d2, d3}, [%1]! \n" + "vld4.8 {d4, d5, d6, d7}, [%1] \n" // load 8 ARGB + "vld1.8 {d7}, [%0]! \n" // replace A with 8 Y + "vst4.8 {d4, d5, d6, d7}, [%1]! \n" + "subs %2, %2, #16 \n" + "bgt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "q0", "q1", "q2", "q3"); +} + // 8x1 pixels. void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb, uint8_t* dst_u, diff --git a/source/row_neon64.cc b/source/row_neon64.cc index 8b76f445c..8fea6df10 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -2721,10 +2721,15 @@ void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb, int width) { asm volatile( "1: \n" - "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 + "ldp q0, q1, [%0] \n" // load 8 ARGB pixels + "ldp q2, q3, [%0, #32] \n" // load next 8 ARGB pixels + "add %0, %0, #64 \n" "prfm pldl1keep, [%0, 448] \n" + "uzp2 v0.8h, v0.8h, v1.8h \n" + "uzp2 v2.8h, v2.8h, v3.8h \n" + "uzp2 v0.16b, v0.16b, v2.16b \n" "subs %w2, %w2, #16 \n" // 16 processed per loop - "st1 {v3.16b}, [%1], #16 \n" // store 16 A's. + "str q0, [%1], #16 \n" // store 16 A's. "b.gt 1b \n" : "+r"(src_argb), // %0 "+r"(dst_a), // %1 @@ -2734,6 +2739,36 @@ void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb, ); } +void ARGBCopyYToAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) { + asm volatile( + "1: \n" + "ldr q4, [%0], #16 \n" + "ldp q0, q1, [%1] \n" + "ldp q2, q3, [%1, #32] \n" + "prfm pldl1keep, [%0, 448] \n" + "prfm pldl1keep, [%1, 448] \n" + "zip1 v6.16b, v4.16b, v4.16b \n" + "zip2 v7.16b, v4.16b, v4.16b \n" + "zip1 v4.8h, v6.8h, v6.8h \n" + "zip2 v5.8h, v6.8h, v6.8h \n" + "zip1 v6.8h, v7.8h, v7.8h \n" + "zip2 v7.8h, v7.8h, v7.8h \n" + "sli v0.4s, v4.4s, #24 \n" + "sli v1.4s, v5.4s, #24 \n" + "sli v2.4s, v6.4s, #24 \n" + "sli v3.4s, v7.4s, #24 \n" + "stp q0, q1, [%1] \n" + "stp q2, q3, [%1, #32] \n" + "add %1, %1, #64 \n" + "subs %w2, %w2, #16 \n" + "b.gt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); +} + // Coefficients expressed as negatives to allow 128 struct RgbUVConstants { int8_t kRGBToU[4]; diff --git a/source/row_rvv.cc b/source/row_rvv.cc index f9cecec26..2206c53e9 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -2355,10 +2355,10 @@ void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb, int vl; asm volatile( "1: \n" - "vsetvli %[vl], %[w], e16, m2, ta, ma \n" + "vsetvli %[vl], %[w], e16, m4, ta, ma \n" "vle32.v v8, (%[src_argb]) \n" "vnsrl.wi v16, v8, 16 \n" - "vsetvli zero, zero, e8, m1, ta, ma \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" "vnsrl.wi v8, v16, 8 \n" "vse8.v v8, (%[dst_a]) \n" "sub %[w], %[w], %[vl] \n" @@ -2371,7 +2371,8 @@ void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb, [w] "+r"(width), // %[w] [vl] "=&r"(vl) // %[vl] : - : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17"); + : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19"); } #endif @@ -2379,23 +2380,28 @@ void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb, void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width) { int vl; asm volatile( - "addi %[dst], %[dst], 3 \n" - "1: \n" - "vsetvli %[vl], %[w], e8, m4, ta, ma \n" - "vle8.v v8, (%[src]) \n" - "vsse8.v v8, (%[dst]), %[dst_stride] \n" + "vsetvli %[vl], %[w], e32, m8, ta, ma \n" + "vle8.v v16, (%[src]) \n" + "vle32.v v8, (%[dst]) \n" + "vzext.vf4 v24, v16 \n" + "vand.vx v8, v8, %[mask] \n" + "vsll.vi v24, v24, 24 \n" + "vor.vv v8, v8, v24 \n" + "vse32.v v8, (%[dst]) \n" "sub %[w], %[w], %[vl] \n" "add %[src], %[src], %[vl] \n" "slli %[vl], %[vl], 2 \n" "add %[dst], %[dst], %[vl] \n" "bgtz %[w], 1b \n" - : [src] "+r"(src), // %[src] - [dst] "+r"(dst), // %[dst] - [w] "+r"(width), // %[w] - [vl] "=&r"(vl) // %[vl] - : [dst_stride] "r"(4) // %[dst_stride] - : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11"); + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [mask] "r"(0x00ffffffu) // %[mask] + : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31"); } #endif diff --git a/source/row_wasm.cc b/source/row_wasm.cc index 4376d912c..a68774edd 100644 --- a/source/row_wasm.cc +++ b/source/row_wasm.cc @@ -1914,6 +1914,62 @@ void BlendPlaneRow_WASMSIMD(const uint8_t* src0, } #endif // HAS_BLENDPLANEROW_WASMSIMD +#ifdef HAS_ARGBEXTRACTALPHAROW_WASMSIMD +void ARGBExtractAlphaRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_a, + int width) { + do { + asm("" : "+r"(src_argb), "+r"(dst_a)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + v128_t s2 = wasm_v128_load(src_argb + 32); + v128_t s3 = wasm_v128_load(src_argb + 48); + s0 = wasm_u32x4_shr(s0, 24); + s1 = wasm_u32x4_shr(s1, 24); + s2 = wasm_u32x4_shr(s2, 24); + s3 = wasm_u32x4_shr(s3, 24); + v128_t a = wasm_u8x16_narrow_i16x8(wasm_i16x8_narrow_i32x4(s0, s1), + wasm_i16x8_narrow_i32x4(s2, s3)); + wasm_v128_store(dst_a, a); + src_argb += 64; + dst_a += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGBEXTRACTALPHAROW_WASMSIMD + +#ifdef HAS_ARGBCOPYYTOALPHAROW_WASMSIMD +// Swizzle Y[i] to the alpha byte of 32 bit lane i; 128 indices give 0. +static const uint8_t kYToAlphaSwizzleTable[64] = { + 128, 128, 128, 0, 128, 128, 128, 1, 128, 128, 128, 2, 128, 128, 128, 3, + 128, 128, 128, 4, 128, 128, 128, 5, 128, 128, 128, 6, 128, 128, 128, 7, + 128, 128, 128, 8, 128, 128, 128, 9, 128, 128, 128, 10, 128, 128, 128, 11, + 128, 128, 128, 12, 128, 128, 128, 13, 128, 128, 128, 14, 128, 128, 128, 15}; + +void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { + v128_t rgb_mask = SplatConst32(0x00ffffff); + v128_t idx0 = LoadConst(kYToAlphaSwizzleTable); + v128_t idx1 = LoadConst(kYToAlphaSwizzleTable + 16); + v128_t idx2 = LoadConst(kYToAlphaSwizzleTable + 32); + v128_t idx3 = LoadConst(kYToAlphaSwizzleTable + 48); + do { + asm("" : "+r"(src), "+r"(dst)); + v128_t y = wasm_v128_load(src); + v128_t d0 = wasm_v128_and(wasm_v128_load(dst), rgb_mask); + v128_t d1 = wasm_v128_and(wasm_v128_load(dst + 16), rgb_mask); + v128_t d2 = wasm_v128_and(wasm_v128_load(dst + 32), rgb_mask); + v128_t d3 = wasm_v128_and(wasm_v128_load(dst + 48), rgb_mask); + wasm_v128_store(dst, wasm_v128_or(d0, LIBYUV_WASM_SWIZZLE(y, idx0))); + wasm_v128_store(dst + 16, wasm_v128_or(d1, LIBYUV_WASM_SWIZZLE(y, idx1))); + wasm_v128_store(dst + 32, wasm_v128_or(d2, LIBYUV_WASM_SWIZZLE(y, idx2))); + wasm_v128_store(dst + 48, wasm_v128_or(d3, LIBYUV_WASM_SWIZZLE(y, idx3))); + src += 16; + dst += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGBCOPYYTOALPHAROW_WASMSIMD + #undef LIBYUV_WASM_SWIZZLE #ifdef __cplusplus diff --git a/source/row_win.cc b/source/row_win.cc index b319973c1..077b4e077 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -2581,6 +2581,188 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0, } #endif // HAS_BLENDPLANEROW_AVX512BW +#ifdef HAS_ARGBEXTRACTALPHAROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_a, + int width) { + while (width >= 32) { + __m512i z0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i z1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24)); + __m128i a1 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z1, 24)); + _mm_storeu_si128((__m128i*)dst_a, a0); + _mm_storeu_si128((__m128i*)(dst_a + 16), a1); + src_argb += 128; + dst_a += 32; + width -= 32; + } + if (width >= 16) { + __m512i z0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m128i a0 = _mm512_cvtepi32_epi8(_mm512_srli_epi32(z0, 24)); + _mm_storeu_si128((__m128i*)dst_a, a0); + src_argb += 64; + dst_a += 16; + width -= 16; + } + if (width > 0) { + __mmask16 mask = (__mmask16)((1u << width) - 1u); + __m512i z0 = _mm512_maskz_loadu_epi32(mask, src_argb); + _mm512_mask_cvtepi32_storeu_epi8(dst_a, mask, _mm512_srli_epi32(z0, 24)); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBEXTRACTALPHAROW_AVX512BW + +#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + __m512i rgb_mask = _mm512_set1_epi32(0x00ffffff); + while (width >= 32) { + __m512i a0 = _mm512_slli_epi32( + _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)src)), 24); + __m512i a1 = _mm512_slli_epi32( + _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)(src + 16))), 24); + a0 = _mm512_ternarylogic_epi32( + a0, rgb_mask, _mm512_loadu_si512((const __m512i*)dst), 0xf8); + a1 = _mm512_ternarylogic_epi32( + a1, rgb_mask, _mm512_loadu_si512((const __m512i*)(dst + 64)), 0xf8); + _mm512_storeu_si512((__m512i*)dst, a0); + _mm512_storeu_si512((__m512i*)(dst + 64), a1); + src += 32; + dst += 128; + width -= 32; + } + if (width >= 16) { + __m512i a0 = _mm512_slli_epi32( + _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)src)), 24); + a0 = _mm512_ternarylogic_epi32( + a0, rgb_mask, _mm512_loadu_si512((const __m512i*)dst), 0xf8); + _mm512_storeu_si512((__m512i*)dst, a0); + src += 16; + dst += 64; + width -= 16; + } + if (width > 0) { + __mmask16 mask = (__mmask16)((1u << width) - 1u); + __m512i a0 = _mm512_slli_epi32( + _mm512_maskz_cvtepu8_epi32(mask, _mm_maskz_loadu_epi8(mask, src)), 24); + a0 = _mm512_ternarylogic_epi32(a0, rgb_mask, + _mm512_maskz_loadu_epi32(mask, dst), 0xf8); + _mm512_mask_storeu_epi32(dst, mask, a0); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBCOPYYTOALPHAROW_AVX512BW + +#ifdef HAS_ARGBMIRRORROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + const __m512i rev = + _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const uint8_t* s = src + (ptrdiff_t)width * 4; + while (width >= 16) { + s -= 64; + _mm512_storeu_si512( + (__m512i*)dst, + _mm512_permutexvar_epi32(rev, _mm512_loadu_si512((const __m512i*)s))); + dst += 64; + width -= 16; + } + if (width > 0) { + // Remaining 1 to 15 pixels are at the start of src. + __mmask16 mask = (__mmask16)((1u << width) - 1u); + __m512i idx = _mm512_sub_epi32(rev, _mm512_set1_epi32(16 - width)); + __m512i v = _mm512_maskz_loadu_epi32(mask, src); + _mm512_mask_storeu_epi32(dst, mask, _mm512_permutexvar_epi32(idx, v)); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBMIRRORROW_AVX512BW + +#ifdef HAS_RGB24MIRRORROW_AVX512VBMI +static const uint8_t kPermMirrorRGB24_0[64] = { + 125u, 126u, 127u, 122u, 123u, 124u, 119u, 120u, 121u, 116u, 117u, + 118u, 113u, 114u, 115u, 110u, 111u, 112u, 107u, 108u, 109u, 104u, + 105u, 106u, 101u, 102u, 103u, 98u, 99u, 100u, 95u, 96u, 97u, + 92u, 93u, 94u, 89u, 90u, 91u, 86u, 87u, 88u, 83u, 84u, + 85u, 80u, 81u, 82u, 77u, 78u, 79u, 74u, 75u, 76u, 71u, + 72u, 73u, 68u, 69u, 70u, 65u, 66u, 67u, 62u}; +static const uint8_t kPermMirrorRGB24_1[64] = { + 127u, 128u, 123u, 124u, 125u, 120u, 121u, 122u, 117u, 118u, 119u, + 114u, 115u, 116u, 111u, 112u, 113u, 108u, 109u, 110u, 105u, 106u, + 107u, 102u, 103u, 104u, 99u, 100u, 101u, 96u, 97u, 98u, 93u, + 94u, 95u, 90u, 91u, 92u, 87u, 88u, 89u, 84u, 85u, 86u, + 81u, 82u, 83u, 78u, 79u, 80u, 75u, 76u, 77u, 72u, 73u, + 74u, 69u, 70u, 71u, 66u, 67u, 68u, 63u, 64u}; +static const uint8_t kPermMirrorRGB24_2[64] = { + 65u, 60u, 61u, 62u, 57u, 58u, 59u, 54u, 55u, 56u, 51u, 52u, 53u, + 48u, 49u, 50u, 45u, 46u, 47u, 42u, 43u, 44u, 39u, 40u, 41u, 36u, + 37u, 38u, 33u, 34u, 35u, 30u, 31u, 32u, 27u, 28u, 29u, 24u, 25u, + 26u, 21u, 22u, 23u, 18u, 19u, 20u, 15u, 16u, 17u, 12u, 13u, 14u, + 9u, 10u, 11u, 6u, 7u, 8u, 3u, 4u, 5u, 0u, 1u, 2u}; + +// Byte mask with the low n bits set, clamped to [0, 64]. +static inline __mmask64 RGB24MirrorMask(int n) { + return n <= 0 ? 0ull : n >= 64 ? ~0ull : (1ull << n) - 1ull; +} + +// Mirror 64 RGB24 pixels (3 full zmm vectors) per loop. The remaining 1 to +// 63 pixels use masked loads and stores that access only valid bytes. +LIBYUV_TARGET_AVX512VBMI +void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width) { + const __m512i idx0 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_0); + const __m512i idx1 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_1); + const __m512i idx2 = _mm512_loadu_si512((const __m512i*)kPermMirrorRGB24_2); + int r = width & 63; + int w = width - r; + if (w > 0) { + const uint8_t* src = src_rgb24 + (ptrdiff_t)width * 3 - 192; + do { + __m512i a = _mm512_loadu_si512((const __m512i*)src); + __m512i b = _mm512_loadu_si512((const __m512i*)(src + 64)); + __m512i c = _mm512_loadu_si512((const __m512i*)(src + 128)); + __m512i d0 = _mm512_permutex2var_epi8(b, idx0, c); + __m512i d1 = _mm512_permutex2var_epi8(a, idx1, b); + __m512i d2 = _mm512_permutex2var_epi8(a, idx2, b); + d1 = _mm512_mask_broadcastb_epi8(d1, (__mmask64)2, + _mm512_castsi512_si128(c)); + _mm512_storeu_si512((__m512i*)dst_rgb24, d0); + _mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), d1); + _mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), d2); + src -= 192; + dst_rgb24 += 192; + w -= 64; + } while (w > 0); + } + if (r) { + int nb = r * 3; + int off = 192 - nb; + __mmask64 m0 = RGB24MirrorMask(nb); + __mmask64 m1 = RGB24MirrorMask(nb - 64); + __mmask64 m2 = RGB24MirrorMask(nb - 128); + __m512i a = _mm512_maskz_loadu_epi8(m0, src_rgb24); + __m512i b = _mm512_maskz_loadu_epi8(m1, src_rgb24 + 64); + __m512i c = _mm512_maskz_loadu_epi8(m2, src_rgb24 + 128); + __m512i t0 = _mm512_sub_epi8(idx0, _mm512_set1_epi8((char)(off - 64))); + __m512i t1 = _mm512_sub_epi8(idx1, _mm512_set1_epi8((char)off)); + __m512i t2 = _mm512_sub_epi8(idx2, _mm512_set1_epi8((char)off)); + __m512i d0 = _mm512_mask_permutexvar_epi8( + _mm512_permutex2var_epi8(a, t0, b), _mm512_movepi8_mask(t0), t0, c); + __m512i d1 = _mm512_mask_permutexvar_epi8( + _mm512_permutex2var_epi8(a, t1, b), _mm512_movepi8_mask(t1), t1, c); + __m512i d2 = _mm512_mask_permutexvar_epi8( + _mm512_permutex2var_epi8(a, t2, b), _mm512_movepi8_mask(t2), t2, c); + _mm512_mask_storeu_epi8(dst_rgb24, m0, d0); + _mm512_mask_storeu_epi8(dst_rgb24 + 64, m1, d1); + _mm512_mask_storeu_epi8(dst_rgb24 + 128, m2, d2); + } + _mm256_zeroupper(); +} +#endif // HAS_RGB24MIRRORROW_AVX512VBMI + #ifdef __cplusplus } // extern "C" } // namespace libyuv