From b9d572cc63f26641422c061364e86c5050fcac78 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Mon, 12 Oct 2026 17:12:52 -0700 Subject: [PATCH] Add 25 native k-masked AVX-512 row kernels R=rrwinterton@gmail.com Bug: None Change-Id: I2ee9b2c2d939f63f49d2f951f2d552ca6863d78b Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8564927 Commit-Queue: Frank Barchard Reviewed-by: richard winterton --- include/libyuv/convert.h | 26 + include/libyuv/planar_functions.h | 9 + include/libyuv/row.h | 142 +++ source/convert.cc | 251 +++++ source/convert_argb.cc | 141 ++- source/convert_from_argb.cc | 20 + source/convert_to_argb.cc | 21 + source/convert_to_i420.cc | 21 +- source/planar_functions.cc | 171 ++- source/rotate.cc | 11 +- source/row_gcc.cc | 1694 +++++++++++++++++++++++++++++ source/row_win.cc | 1457 +++++++++++++++++++++++++ source/scale.cc | 10 + source/scale_common.cc | 5 + unit_test/convert_test.cc | 2 + unit_test/planar_test.cc | 28 + 16 files changed, 4002 insertions(+), 7 deletions(-) diff --git a/include/libyuv/convert.h b/include/libyuv/convert.h index a09369783..adf35df7f 100644 --- a/include/libyuv/convert.h +++ b/include/libyuv/convert.h @@ -996,6 +996,32 @@ int ABGRToI420(const uint8_t* src_abgr, int width, int height); +// Convert AR30 to I420. +LIBYUV_API +int AR30ToI420(const uint8_t* src_ar30, + int src_stride_ar30, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height); + +// Convert AB30 to I420. +LIBYUV_API +int AB30ToI420(const uint8_t* src_ab30, + int src_stride_ab30, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height); + // BGRA little endian (argb in memory) to I422. LIBYUV_API int BGRAToI422(const uint8_t* src_bgra, diff --git a/include/libyuv/planar_functions.h b/include/libyuv/planar_functions.h index baef27306..b93b729df 100644 --- a/include/libyuv/planar_functions.h +++ b/include/libyuv/planar_functions.h @@ -609,6 +609,15 @@ void MirrorPlane(const uint8_t* src_y, int width, int height); +// Mirror a plane of 16-bit data. +LIBYUV_API +void MirrorPlane_16(const uint16_t* src_y, + int src_stride_y, + uint16_t* dst_y, + int dst_stride_y, + int width, + int height); + // Mirror a plane of UV data. LIBYUV_API void MirrorUVPlane(const uint8_t* src_uv, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index bf01f1c59..7f465fe38 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -389,6 +389,31 @@ extern "C" { #define HAS_ARGBCOPYYTOALPHAROW_AVX512BW #define HAS_ARGBMIRRORROW_AVX512BW #define HAS_RGB24MIRRORROW_AVX512VBMI +#define HAS_AYUVTOYROW_AVX512VBMI +#define HAS_AYUVTOUVROW_AVX512VBMI +#define HAS_AYUVTOVUROW_AVX512VBMI +#define HAS_AR30TOARGBROW_AVX512BW +#define HAS_AR30TOABGRROW_AVX512BW +#define HAS_AR30TOAB30ROW_AVX512BW +#define HAS_MERGEXR30ROW_10_AVX512BW +#define HAS_MIRRORROW_16_AVX512BW +#define HAS_MERGERGBROW_AVX512VBMI +#define HAS_RAWTORGBAROW_AVX512VBMI +#define HAS_NV12TOARGBROW_AVX512BW +#define HAS_NV21TOARGBROW_AVX512BW +#define HAS_NV12TORGB24ROW_AVX512VBMI +#define HAS_NV21TORGB24ROW_AVX512VBMI +#define HAS_I422ALPHATOARGBROW_AVX512BW +#define HAS_SWAPUVROW_AVX512BW +#define HAS_HALFMERGEUVROW_AVX512BW +#define HAS_INTERPOLATEROW_16_AVX512BW +#define HAS_ARGBATTENUATEROW_AVX512BW +#define HAS_ARGBMULTIPLYROW_AVX512BW +#define HAS_ARGBTOAR30ROW_AVX512BW +#define HAS_ABGRTOAR30ROW_AVX512BW +#define HAS_I400TOARGBROW_AVX512BW +#define HAS_ARGBGRAYROW_AVX512BW +#define HAS_ARGBBLENDROW_AVX512BW #endif #define HAS_ARGBTOYMATRIXROW_AVX2 #define HAS_RGBTOYMATRIXROW_AVX2 @@ -464,6 +489,31 @@ extern "C" { #define HAS_ARGBCOPYYTOALPHAROW_AVX512BW #define HAS_ARGBMIRRORROW_AVX512BW #define HAS_RGB24MIRRORROW_AVX512VBMI +#define HAS_AYUVTOYROW_AVX512VBMI +#define HAS_AYUVTOUVROW_AVX512VBMI +#define HAS_AYUVTOVUROW_AVX512VBMI +#define HAS_AR30TOARGBROW_AVX512BW +#define HAS_AR30TOABGRROW_AVX512BW +#define HAS_AR30TOAB30ROW_AVX512BW +#define HAS_MERGEXR30ROW_10_AVX512BW +#define HAS_MIRRORROW_16_AVX512BW +#define HAS_MERGERGBROW_AVX512VBMI +#define HAS_RAWTORGBAROW_AVX512VBMI +#define HAS_NV12TOARGBROW_AVX512BW +#define HAS_NV21TOARGBROW_AVX512BW +#define HAS_NV12TORGB24ROW_AVX512VBMI +#define HAS_NV21TORGB24ROW_AVX512VBMI +#define HAS_I422ALPHATOARGBROW_AVX512BW +#define HAS_SWAPUVROW_AVX512BW +#define HAS_HALFMERGEUVROW_AVX512BW +#define HAS_INTERPOLATEROW_16_AVX512BW +#define HAS_ARGBATTENUATEROW_AVX512BW +#define HAS_ARGBMULTIPLYROW_AVX512BW +#define HAS_ARGBTOAR30ROW_AVX512BW +#define HAS_ABGRTOAR30ROW_AVX512BW +#define HAS_I400TOARGBROW_AVX512BW +#define HAS_ARGBGRAYROW_AVX512BW +#define HAS_ARGBBLENDROW_AVX512BW #endif // The following are available on Neon platforms: @@ -2781,6 +2831,7 @@ void MirrorSplitUVRow_C(const uint8_t* src_uv, int width); void MirrorRow_16_C(const uint16_t* src, uint16_t* dst, int width); +void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width); void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width); void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width); @@ -3078,6 +3129,12 @@ void HalfMergeUVRow_AVX2(const uint8_t* src_u, int src_stride_v, uint8_t* dst_uv, int width); +void HalfMergeUVRow_AVX512BW(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); void SplitRGBRow_C(const uint8_t* src_rgb, uint8_t* dst_r, @@ -3150,6 +3207,11 @@ void MergeRGBRow_RVV(const uint8_t* src_r, const uint8_t* src_b, uint8_t* dst_rgb, int width); +void MergeRGBRow_AVX512VBMI(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width); void MergeRGBRow_Any_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -3397,6 +3459,12 @@ void MergeXR30Row_AVX2(const uint16_t* src_r, uint8_t* dst_ar30, int depth, int width); +void MergeXR30Row_10_AVX512BW(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + uint8_t* dst_ar30, + int depth, + int width); void MergeAR64Row_AVX2(const uint16_t* src_r, const uint16_t* src_g, const uint16_t* src_b, @@ -4152,6 +4220,9 @@ void ARGB4444ToARGBRow_LASX(const uint8_t* src_argb4444, void RGB24ToARGBRow_C(const uint8_t* src_rgb24, uint8_t* dst_argb, int width); void RAWToARGBRow_C(const uint8_t* src_raw, uint8_t* dst_argb, int width); void RAWToRGBARow_C(const uint8_t* src_raw, uint8_t* dst_rgba, int width); +void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width); void RAWToRGB24Row_C(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RGB565ToARGBRow_C(const uint8_t* src_rgb565, uint8_t* dst_argb, int width); void ARGB1555ToARGBRow_C(const uint8_t* src_argb1555, @@ -4161,9 +4232,18 @@ void ARGB4444ToARGBRow_C(const uint8_t* src_argb4444, uint8_t* dst_argb, int width); void AR30ToARGBRow_C(const uint8_t* src_ar30, uint8_t* dst_argb, int width); +void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width); void AR30ToABGRRow_C(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); +void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width); void ARGBToAR30Row_C(const uint8_t* src_argb, uint8_t* dst_ar30, int width); void AR30ToAB30Row_C(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); +void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width); void RGB24ToARGBRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -4286,7 +4366,9 @@ void ARGBToARGB4444Row_AVX2(const uint8_t* src_argb, uint8_t* dst_rgb, int width); void ABGRToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width); +void ABGRToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width); void ARGBToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width); +void ARGBToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width); void ARGBToRGB24Row_NEON(const uint8_t* src_argb, uint8_t* dst_rgb24, @@ -4864,6 +4946,13 @@ void I422AlphaToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void NV12ToARGBRow_SSSE3(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_argb, @@ -4874,6 +4963,11 @@ void NV12ToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void NV12ToRGB24Row_SSSE3(const uint8_t* src_y, const uint8_t* src_uv, uint8_t* dst_rgb24, @@ -4890,11 +4984,21 @@ void NV12ToRGB24Row_AVX2(const uint8_t* src_y, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width); +void NV12ToRGB24Row_AVX512VBMI(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); void NV21ToRGB24Row_AVX2(const uint8_t* src_y, const uint8_t* src_vu, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width); +void NV21ToRGB24Row_AVX512VBMI(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); void NV12ToRGB565Row_SSSE3(const uint8_t* src_y, const uint8_t* src_uv, uint8_t* dst_rgb565, @@ -4924,6 +5028,11 @@ void NV21ToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* vu_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void YUY2ToARGBRow_SSSE3(const uint8_t* yuy2_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, @@ -5384,6 +5493,10 @@ void I400ToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I400ToARGBRow_AVX512BW(const uint8_t* y_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, @@ -5426,6 +5539,10 @@ void ARGBBlendRow_SSSE3(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, int width); +void ARGBBlendRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); void ARGBBlendRow_NEON(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -5516,6 +5633,10 @@ void ARGBMultiplyRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); +void ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); void ARGBMultiplyRow_NEON(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -6386,16 +6507,26 @@ void SwapUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_vu, int width); void SwapUVRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width); void SwapUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void SwapUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_vu, int width); void SwapUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_vu, int width); void AYUVToYRow_C(const uint8_t* src_ayuv, uint8_t* dst_y, int width); +void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width); void AYUVToUVRow_C(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_uv, int width); +void AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); void AYUVToVURow_C(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, int width); +void AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); void AYUVToYRow_NEON(const uint8_t* src_ayuv, uint8_t* dst_y, int width); void AYUVToUVRow_NEON(const uint8_t* src_ayuv, int src_stride_ayuv, @@ -6550,6 +6681,9 @@ void ARGBAttenuateRow_SSSE3(const uint8_t* src_argb, void ARGBAttenuateRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width); +void ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); void ARGBAttenuateRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width); @@ -6598,6 +6732,9 @@ void ARGBUnattenuateRow_Any_AVX2(const uint8_t* src_ptr, void ARGBGrayRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width); +void ARGBGrayRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); void ARGBGrayRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_NEON_DotProd(const uint8_t* src_argb, uint8_t* dst_argb, @@ -6817,6 +6954,11 @@ void InterpolateRow_16_Any_AVX2(uint16_t* dst_ptr, ptrdiff_t src_stride, int width, int source_y_fraction); +void InterpolateRow_16_AVX512BW(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); void InterpolateRow_16_NEON(uint16_t* dst_ptr, const uint16_t* src_ptr, ptrdiff_t src_stride, diff --git a/source/convert.cc b/source/convert.cc index 075b3e869..c698cc967 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -2646,6 +2646,12 @@ int AYUVToNV12(const uint8_t* src_ayuv, } } #endif +#if defined(HAS_AYUVTOYROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + AYUVToUVRow = AYUVToUVRow_AVX512VBMI; + AYUVToYRow = AYUVToYRow_AVX512VBMI; + } +#endif #if defined(HAS_AYUVTOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -2726,6 +2732,12 @@ int AYUVToNV21(const uint8_t* src_ayuv, } } #endif +#if defined(HAS_AYUVTOYROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + AYUVToVURow = AYUVToVURow_AVX512VBMI; + AYUVToYRow = AYUVToYRow_AVX512VBMI; + } +#endif #if defined(HAS_AYUVTOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -3343,6 +3355,245 @@ int RGBAToI420(const uint8_t* src_rgba, &kRgbaI601Constants, width, height); } +static int AR30ToI420Matrix(const uint8_t* src_ar30, + int src_stride_ar30, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + const struct ArgbConstants* argbconstants, + int width, + int height) { + int y; + void (*AR30ToARGBRow)(const uint8_t* src_ar30, uint8_t* dst_argb, int width) = + AR30ToARGBRow_C; + void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width, + const struct ArgbConstants* c) = ARGBToYMatrixRow_C; + void (*ARGBToUVMatrixRow)(const uint8_t* src_argb, int src_stride_argb, + uint8_t* dst_u, uint8_t* dst_v, int width, + const struct ArgbConstants* c) = + ARGBToUVMatrixRow_C; + if (!src_ar30 || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || + height == 0 || height == INT_MIN) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_ar30 = src_ar30 + (ptrdiff_t)(height - 1) * src_stride_ar30; + src_stride_ar30 = -src_stride_ar30; + } +#if defined(HAS_AR30TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + AR30ToARGBRow = AR30ToARGBRow_AVX512BW; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_NEON; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) + if (TestCpuFlag(kCpuHasNeonI8MM)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + if (IS_ALIGNED(width, 2)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + if (IS_ALIGNED(width, 2)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } +#endif + + int row_stride = width * 4; + align_buffer_64(row, row_stride * 2); + if (!row) { + return -1; + } + for (y = 0; y < height - 1; y += 2) { + AR30ToARGBRow(src_ar30, row, width); + AR30ToARGBRow(src_ar30 + src_stride_ar30, row + row_stride, width); + ARGBToUVMatrixRow(row, row_stride, dst_u, dst_v, width, argbconstants); + ARGBToYMatrixRow(row, dst_y, width, argbconstants); + ARGBToYMatrixRow(row + row_stride, dst_y + dst_stride_y, width, + argbconstants); + src_ar30 += src_stride_ar30 * 2; + dst_y += dst_stride_y * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + AR30ToARGBRow(src_ar30, row, width); + ARGBToUVMatrixRow(row, 0, dst_u, dst_v, width, argbconstants); + ARGBToYMatrixRow(row, dst_y, width, argbconstants); + } + free_aligned_buffer_64(row); + return 0; +} + +// Convert AR30 to I420. +LIBYUV_API +int AR30ToI420(const uint8_t* src_ar30, + int src_stride_ar30, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + return AR30ToI420Matrix(src_ar30, src_stride_ar30, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, + &kArgbI601Constants, width, height); +} + +// Convert AB30 to I420. +LIBYUV_API +int AB30ToI420(const uint8_t* src_ab30, + int src_stride_ab30, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + return AR30ToI420Matrix(src_ab30, src_stride_ab30, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, + &kAbgrI601Constants, width, height); +} + // Enabled if 1 pass is available #if (defined(HAS_RGB24TOYROW_NEON) || defined(HAS_RGB24TOYROW_LSX) || \ defined(HAS_RGB24TOYROW_RVV)) diff --git a/source/convert_argb.cc b/source/convert_argb.cc index a889889c0..1893b208d 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -619,6 +619,14 @@ int I444ToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; + if (IS_ALIGNED(width, 32)) { + I444ToARGBRow = I444ToARGBRow_AVX512BW; + } + } +#endif #if defined(HAS_I444TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToARGBRow = I444ToARGBRow_Any_NEON; @@ -2429,6 +2437,11 @@ int I420AlphaToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I422AlphaToARGBRow = I422AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I422ALPHATOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422AlphaToARGBRow = I422AlphaToARGBRow_Any_NEON; @@ -2484,6 +2497,11 @@ int I420AlphaToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -2591,6 +2609,11 @@ int I422AlphaToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I422AlphaToARGBRow = I422AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I422ALPHATOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422AlphaToARGBRow = I422AlphaToARGBRow_Any_NEON; @@ -2646,6 +2669,11 @@ int I422AlphaToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -2790,6 +2818,11 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -3049,6 +3082,11 @@ int I010AlphaToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -3181,6 +3219,11 @@ int I210AlphaToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -3311,6 +3354,11 @@ int I410AlphaToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -3402,6 +3450,11 @@ int I400ToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I400TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I400ToARGBRow = I400ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I400TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I400ToARGBRow = I400ToARGBRow_Any_NEON; @@ -4055,6 +4108,11 @@ int RAWToRGBA(const uint8_t* src_raw, } } #endif +#if defined(HAS_RAWTORGBAROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + RAWToRGBARow = RAWToRGBARow_AVX512VBMI; + } +#endif #if defined(HAS_RAWTORGBAROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { RAWToRGBARow = RAWToRGBARow_Any_NEON; @@ -4347,8 +4405,15 @@ int AR30ToARGB(const uint8_t* src_ar30, height = 1; src_stride_ar30 = dst_stride_argb = 0; } + void (*AR30ToARGBRow)(const uint8_t* src_ar30, uint8_t* dst_argb, int width) = + AR30ToARGBRow_C; +#if defined(HAS_AR30TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + AR30ToARGBRow = AR30ToARGBRow_AVX512BW; + } +#endif for (y = 0; y < height; ++y) { - AR30ToARGBRow_C(src_ar30, dst_argb, width); + AR30ToARGBRow(src_ar30, dst_argb, width); src_ar30 += src_stride_ar30; dst_argb += dst_stride_argb; } @@ -4364,6 +4429,8 @@ int AR30ToABGR(const uint8_t* src_ar30, int width, int height) { int y; + void (*AR30ToABGRRow)(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) = + AR30ToABGRRow_C; if (!src_ar30 || !dst_abgr || width <= 0 || height == 0 || height == INT_MIN) { return -1; @@ -4381,8 +4448,13 @@ int AR30ToABGR(const uint8_t* src_ar30, height = 1; src_stride_ar30 = dst_stride_abgr = 0; } +#if defined(HAS_AR30TOABGRROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + AR30ToABGRRow = AR30ToABGRRow_AVX512BW; + } +#endif for (y = 0; y < height; ++y) { - AR30ToABGRRow_C(src_ar30, dst_abgr, width); + AR30ToABGRRow(src_ar30, dst_abgr, width); src_ar30 += src_stride_ar30; dst_abgr += dst_stride_abgr; } @@ -4398,6 +4470,8 @@ int AR30ToAB30(const uint8_t* src_ar30, int width, int height) { int y; + void (*AR30ToAB30Row)(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) = + AR30ToAB30Row_C; if (!src_ar30 || !dst_ab30 || width <= 0 || height == 0 || height == INT_MIN) { return -1; @@ -4415,8 +4489,13 @@ int AR30ToAB30(const uint8_t* src_ar30, height = 1; src_stride_ar30 = dst_stride_ab30 = 0; } +#if defined(HAS_AR30TOAB30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + AR30ToAB30Row = AR30ToAB30Row_AVX512BW; + } +#endif for (y = 0; y < height; ++y) { - AR30ToAB30Row_C(src_ar30, dst_ab30, width); + AR30ToAB30Row(src_ar30, dst_ab30, width); src_ar30 += src_stride_ar30; dst_ab30 += dst_stride_ab30; } @@ -4597,6 +4676,11 @@ int NV12ToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_NV12TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + NV12ToARGBRow = NV12ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_NV12TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { NV12ToARGBRow = NV12ToARGBRow_Any_NEON; @@ -4698,6 +4782,11 @@ int NV21ToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_NV21TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + NV21ToARGBRow = NV21ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_NV21TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { NV21ToARGBRow = NV21ToARGBRow_Any_NEON; @@ -4876,6 +4965,11 @@ int NV12ToRGB24Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_NV12TORGB24ROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + NV12ToRGB24Row = NV12ToRGB24Row_AVX512VBMI; + } +#endif #if defined(HAS_NV12TORGB24ROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { NV12ToRGB24Row = NV12ToRGB24Row_RVV; @@ -4953,6 +5047,11 @@ int NV21ToRGB24Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_NV21TORGB24ROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + NV21ToRGB24Row = NV21ToRGB24Row_AVX512VBMI; + } +#endif #if defined(HAS_NV21TORGB24ROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { NV21ToRGB24Row = NV21ToRGB24Row_RVV; @@ -7013,6 +7112,14 @@ static int I420ToARGBMatrixBilinear(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; + if (IS_ALIGNED(width, 32)) { + I444ToARGBRow = I444ToARGBRow_AVX512BW; + } + } +#endif #if defined(HAS_I444TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToARGBRow = I444ToARGBRow_Any_NEON; @@ -7163,6 +7270,14 @@ static int I422ToARGBMatrixLinear(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; + if (IS_ALIGNED(width, 32)) { + I444ToARGBRow = I444ToARGBRow_AVX512BW; + } + } +#endif #if defined(HAS_I444TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToARGBRow = I444ToARGBRow_Any_NEON; @@ -7955,6 +8070,11 @@ static int I420AlphaToARGBMatrixBilinear( } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -8175,6 +8295,11 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -8347,6 +8472,11 @@ static int I010AlphaToARGBMatrixBilinear( } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -8541,6 +8671,11 @@ static int I210AlphaToARGBMatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 09a07dc81..9d7c23fe8 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -577,6 +577,11 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -1374,6 +1379,11 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -3747,6 +3757,11 @@ int ABGRToAR30(const uint8_t* src_abgr, ABGRToAR30Row = ABGRToAR30Row_AVX2; } } +#endif +#if defined(HAS_ABGRTOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ABGRToAR30Row = ABGRToAR30Row_AVX512BW; + } #endif for (y = 0; y < height; ++y) { ABGRToAR30Row(src_abgr, dst_ar30, width); @@ -3806,6 +3821,11 @@ int ARGBToAR30(const uint8_t* src_argb, ARGBToAR30Row = ARGBToAR30Row_AVX2; } } +#endif +#if defined(HAS_ARGBTOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToAR30Row = ARGBToAR30Row_AVX512BW; + } #endif for (y = 0; y < height; ++y) { ARGBToAR30Row(src_argb, dst_ar30, width); diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index cea1e4499..36bbe6b46 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -65,6 +65,11 @@ static int NV16ToARGB(const uint8_t* src_y, } } #endif +#if defined(HAS_NV12TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + NV12ToARGBRow = NV12ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_NV12TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { NV12ToARGBRow = NV12ToARGBRow_Any_NEON; @@ -162,6 +167,14 @@ static int NV24ToARGB(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; + if (IS_ALIGNED(width, 32)) { + I444ToARGBRow = I444ToARGBRow_AVX512BW; + } + } +#endif #if defined(HAS_I444TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToARGBRow = I444ToARGBRow_Any_NEON; @@ -218,6 +231,14 @@ static int NV24ToARGB(const uint8_t* src_y, } } #endif +#if defined(HAS_SPLITUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + SplitUVRow = SplitUVRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + SplitUVRow = SplitUVRow_AVX512BW; + } + } +#endif #if defined(HAS_SPLITUVROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { SplitUVRow = SplitUVRow_Any_NEON; diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc index d43b31f7b..87c903d61 100644 --- a/source/convert_to_i420.cc +++ b/source/convert_to_i420.cc @@ -84,6 +84,14 @@ static int NV16ToI420(const uint8_t* src_y, } } #endif +#if defined(HAS_SPLITUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + SplitUVRow = SplitUVRow_Any_AVX512BW; + if (IS_ALIGNED(halfwidth, 64)) { + SplitUVRow = SplitUVRow_AVX512BW; + } + } +#endif #if defined(HAS_SPLITUVROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { SplitUVRow = SplitUVRow_Any_NEON; @@ -331,7 +339,18 @@ int ConvertToI420(const uint8_t* sample, dst_stride_u, dst_v, dst_stride_v, crop_width, inv_crop_height); break; - // TODO(fbarchard): Add AR30 and AB30 + case FOURCC_AR30: + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4; + r = AR30ToI420(src, src_width * 4, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, crop_width, + inv_crop_height); + break; + case FOURCC_AB30: + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4; + r = AB30ToI420(src, src_width * 4, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, crop_width, + inv_crop_height); + break; case FOURCC_I400: src = sample + (ptrdiff_t)src_width * crop_y + crop_x; r = I400ToI420(src, src_width, dst_y, dst_stride_y, dst_u, dst_stride_u, diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 0590febb9..6841be55e 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -1127,6 +1127,11 @@ void SwapUVPlane(const uint8_t* src_uv, } } #endif +#if defined(HAS_SWAPUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + SwapUVRow = SwapUVRow_AVX512BW; + } +#endif #if defined(HAS_SWAPUVROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { SwapUVRow = SwapUVRow_Any_NEON; @@ -1573,6 +1578,11 @@ void MergeRGBPlane(const uint8_t* src_r, } } #endif +#if defined(HAS_MERGERGBROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + MergeRGBRow = MergeRGBRow_AVX512VBMI; + } +#endif #if defined(HAS_MERGERGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { MergeRGBRow = MergeRGBRow_Any_NEON; @@ -2003,6 +2013,11 @@ void MergeXR30Plane(const uint16_t* src_r, } } #endif +#if defined(HAS_MERGEXR30ROW_10_AVX512BW) + if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL)) { + MergeXR30Row = MergeXR30Row_10_AVX512BW; + } +#endif #if defined(HAS_MERGEXR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { if (depth == 10) { @@ -2755,6 +2770,112 @@ void MirrorPlane(const uint8_t* src_y, } } +// Mirror a plane of 16-bit data. +LIBYUV_API +void MirrorPlane_16(const uint16_t* src_y, + int src_stride_y, + uint16_t* dst_y, + int dst_stride_y, + int width, + int height) { + int y; + void (*MirrorRow_16)(const uint16_t* src, uint16_t* dst, int width) = + MirrorRow_16_C; + if (width <= 0 || height == 0 || height == INT_MIN) { + return; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } +#if defined(HAS_MIRRORUVROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + MirrorRow_16 = reinterpret_cast( + MirrorUVRow_Any_NEON); + if (IS_ALIGNED(width, 32)) { + MirrorRow_16 = + reinterpret_cast( + MirrorUVRow_NEON); + } + } +#endif +#if defined(HAS_MIRRORUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + MirrorRow_16 = reinterpret_cast( + MirrorUVRow_Any_SSSE3); + if (IS_ALIGNED(width, 8)) { + MirrorRow_16 = + reinterpret_cast( + MirrorUVRow_SSSE3); + } + } +#endif +#if defined(HAS_MIRRORUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + MirrorRow_16 = reinterpret_cast( + MirrorUVRow_Any_AVX2); + if (IS_ALIGNED(width, 16)) { + MirrorRow_16 = + reinterpret_cast( + MirrorUVRow_AVX2); + } + } +#endif +#if defined(HAS_MIRRORROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MirrorRow_16 = MirrorRow_16_AVX512BW; + } +#endif +#if defined(HAS_MIRRORUVROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + MirrorRow_16 = reinterpret_cast( + MirrorUVRow_Any_LSX); + if (IS_ALIGNED(width, 8)) { + MirrorRow_16 = + reinterpret_cast( + MirrorUVRow_LSX); + } + } +#endif +#if defined(HAS_MIRRORUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MirrorRow_16 = reinterpret_cast( + MirrorUVRow_Any_LASX); + if (IS_ALIGNED(width, 16)) { + MirrorRow_16 = + reinterpret_cast( + MirrorUVRow_LASX); + } + } +#endif +#if defined(HAS_MIRRORUVROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + MirrorRow_16 = reinterpret_cast( + MirrorUVRow_RVV); + } +#endif +#if defined(HAS_MIRRORUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MirrorRow_16 = reinterpret_cast( + MirrorUVRow_Any_WASMSIMD); + if (IS_ALIGNED(width, 8)) { + MirrorRow_16 = + reinterpret_cast( + MirrorUVRow_WASMSIMD); + } + } +#endif + + // Mirror plane + for (y = 0; y < height; ++y) { + MirrorRow_16(src_y, dst_y, width); + src_y += src_stride_y; + dst_y += dst_stride_y; + } +} + // Mirror a plane of UV data. LIBYUV_API void MirrorUVPlane(const uint8_t* src_uv, @@ -2799,7 +2920,12 @@ void MirrorUVPlane(const uint8_t* src_uv, } } #endif -#if defined(HAS_MIRRORUVROW_AVX512BW) +#if defined(HAS_MIRRORROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MirrorUVRow = reinterpret_cast( + MirrorRow_16_AVX512BW); + } +#elif defined(HAS_MIRRORUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { MirrorUVRow = MirrorUVRow_Any_AVX512BW; if (IS_ALIGNED(width, 32)) { @@ -3133,6 +3259,11 @@ int ARGBBlend(const uint8_t* src_argb0, ARGBBlendRow = ARGBBlendRow_SSSE3; } #endif +#if defined(HAS_ARGBBLENDROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBBlendRow = ARGBBlendRow_AVX512BW; + } +#endif #if defined(HAS_ARGBBLENDROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBBlendRow = ARGBBlendRow_NEON; @@ -3479,6 +3610,11 @@ int ARGBMultiply(const uint8_t* src_argb0, } } #endif +#if defined(HAS_ARGBMULTIPLYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBMultiplyRow = ARGBMultiplyRow_AVX512BW; + } +#endif #if defined(HAS_ARGBMULTIPLYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBMultiplyRow = ARGBMultiplyRow_Any_NEON; @@ -4010,6 +4146,11 @@ int ARGBAttenuate(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBATTENUATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAttenuateRow = ARGBAttenuateRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_NEON; @@ -4133,6 +4274,11 @@ int ARGBGrayTo(const uint8_t* src_argb, ARGBGrayRow = ARGBGrayRow_SSSE3; } #endif +#if defined(HAS_ARGBGRAYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBGrayRow = ARGBGrayRow_AVX512BW; + } +#endif #if defined(HAS_ARGBGRAYROW_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { ARGBGrayRow = ARGBGrayRow_NEON; @@ -4188,6 +4334,11 @@ int ARGBGray(uint8_t* dst_argb, ARGBGrayRow = ARGBGrayRow_SSSE3; } #endif +#if defined(HAS_ARGBGRAYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBGrayRow = ARGBGrayRow_AVX512BW; + } +#endif #if defined(HAS_ARGBGRAYROW_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { ARGBGrayRow = ARGBGrayRow_NEON; @@ -4859,6 +5010,11 @@ int InterpolatePlane_16(const uint16_t* src0, } } #endif +#if defined(HAS_INTERPOLATEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow_16 = InterpolateRow_16_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow_16 = InterpolateRow_16_Any_NEON; @@ -6049,6 +6205,14 @@ int UYVYToNV12(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_SPLITUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + SplitUVRow = SplitUVRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + SplitUVRow = SplitUVRow_AVX512BW; + } + } +#endif #if defined(HAS_SPLITUVROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { SplitUVRow = SplitUVRow_Any_NEON; @@ -6206,6 +6370,11 @@ void HalfMergeUVPlane(const uint8_t* src_u, HalfMergeUVRow = HalfMergeUVRow_AVX2; } #endif +#if defined(HAS_HALFMERGEUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + HalfMergeUVRow = HalfMergeUVRow_AVX512BW; + } +#endif for (y = 0; y < height - 1; y += 2) { // Merge a row of U and V into a row of UV. diff --git a/source/rotate.cc b/source/rotate.cc index c155a0d90..d215533c6 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -222,6 +222,14 @@ void RotatePlane180(const uint8_t* src, } } #endif +#if defined(HAS_MIRRORROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MirrorRow = MirrorRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + MirrorRow = MirrorRow_AVX512BW; + } + } +#endif #if defined(HAS_MIRRORROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { MirrorRow = MirrorRow_Any_LSX; @@ -768,8 +776,7 @@ static void RotatePlane180_16(const uint16_t* src, int width, int height) { // Rotate by 180 is a vertical flip (negative height) and horizontal mirror. - MirrorUVPlane((const uint8_t*)src, src_stride * 2, (uint8_t*)dst, - dst_stride * 2, width, -height); + MirrorPlane_16(src, src_stride, dst, dst_stride, width, -height); } LIBYUV_API diff --git a/source/row_gcc.cc b/source/row_gcc.cc index cce7a9120..d27099e20 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -11767,6 +11767,1700 @@ void Convert8To8Row_AVX2(const uint8_t* src_y, } #endif // HAS_CONVERT16TO8ROW_AVX2 +// =========================================================================== +// Module 1: AYUV to Planar Y / UV / VU (AVX512VBMI) +// =========================================================================== + +#ifdef HAS_AYUVTOYROW_AVX512VBMI +static const uint8_t kPermAYUVToY32[64] = { + 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62, + 66, 70, 74, 78, 82, 86, 90, 94, 98, 102, 106, 110, 114, 118, 122, 126, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +void OMITFP AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, + uint8_t* dst_y, + int width) { + uintptr_t temp; + asm volatile( + "vmovdqu8 %[perm],%%zmm16 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%[src]),%%zmm0 \n" + "vmovdqu8 0x40(%[src]),%%zmm1 \n" + "lea 0x80(%[src]),%[src] \n" + "vmovdqa64 %%zmm16,%%zmm2 \n" + "vpermt2b %%zmm1,%%zmm2,%%zmm0 \n" + "vmovdqu8 %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovd %k[temp],%%k3 \n" + "kmovw %k[temp],%%k1 \n" + "kshiftrd $16,%%k3,%%k2 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src]),%%zmm1%{%%k2%}%{z%}\n" + "vpermt2b %%zmm1,%%zmm16,%%zmm0 \n" + "vmovdqu8 %%ymm0,(%[dst])%{%%k3%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_ayuv), [dst] "+r"(dst_y), [width] "+r"(width), + [temp] "=&r"(temp) + : [perm] "m"(kPermAYUVToY32) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "k1", "k2", "k3"); +} +#endif // HAS_AYUVTOYROW_AVX512VBMI + +#if defined(HAS_AYUVTOUVROW_AVX512VBMI) || defined(HAS_AYUVTOVUROW_AVX512VBMI) +static const uint8_t kPermAYUVToUV[64] = { + 1, 5, 0, 4, 9, 13, 8, 12, 17, 21, 16, 20, 25, + 29, 24, 28, 33, 37, 32, 36, 41, 45, 40, 44, 49, 53, + 48, 52, 57, 61, 56, 60, 65, 69, 64, 68, 73, 77, 72, + 76, 81, 85, 80, 84, 89, 93, 88, 92, 97, 101, 96, 100, + 105, 109, 104, 108, 113, 117, 112, 116, 121, 125, 120, 124}; + +static const uint8_t kPermAYUVToVU[64] = { + 0, 4, 1, 5, 8, 12, 9, 13, 16, 20, 17, 21, 24, + 28, 25, 29, 32, 36, 33, 37, 40, 44, 41, 45, 48, 52, + 49, 53, 56, 60, 57, 61, 64, 68, 65, 69, 72, 76, 73, + 77, 80, 84, 81, 85, 88, 92, 89, 93, 96, 100, 97, 101, + 104, 108, 105, 109, 112, 116, 113, 117, 120, 124, 121, 125}; + +static inline void AYUVToUVOrVURow_AVX512VBMI(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + const uint8_t* perm_table, + int width) { + uintptr_t temp1; + uintptr_t temp2; + asm volatile( + "vmovdqu8 (%[perm]),%%zmm16 \n" + "vpternlogd $0xff,%%zmm18,%%zmm18,%%zmm18 \n" + "vpsrlw $15,%%zmm18,%%zmm18 \n" + "vpsllw $8,%%zmm18,%%zmm19 \n" + "vpord %%zmm19,%%zmm18,%%zmm18 \n" // 0x0101 + "vpxord %%zmm19,%%zmm19,%%zmm19 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%[src]),%%zmm0 \n" + "vmovdqu8 0x40(%[src]),%%zmm1 \n" + "vmovdqu8 (%[src],%[stride]),%%zmm2 \n" + "vmovdqu8 0x40(%[src],%[stride]),%%zmm3 \n" + "lea 0x80(%[src]),%[src] \n" + "vpermt2b %%zmm1,%%zmm16,%%zmm0 \n" + "vpermt2b %%zmm3,%%zmm16,%%zmm2 \n" + "vpmaddubsw %%zmm18,%%zmm0,%%zmm0 \n" + "vpmaddubsw %%zmm18,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm2,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm19,%%zmm0,%%zmm0 \n" + "vpmovwb %%zmm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%[temp1] \n" + "bzhi %q[width],%[temp1],%[temp1] \n" + "kmovd %k[temp1],%%k3 \n" + "kmovw %k[temp1],%%k1 \n" + "kshiftrd $16,%%k3,%%k2 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src]),%%zmm1%{%%k2%}%{z%}\n" + "vmovdqu32 (%[src],%[stride]),%%zmm2%{%%k1%}%{z%}\n" + "vmovdqu32 0x40(%[src],%[stride]),%%zmm3%{%%k2%}%{z%}\n" + "vpermt2b %%zmm1,%%zmm16,%%zmm0 \n" + "vpermt2b %%zmm3,%%zmm16,%%zmm2 \n" + // If width is odd, duplicate pixel (width - 1) into pixel width in dword + // (width >> 1). + "mov %k[width],%k[temp1] \n" + "and $1,%k[temp1] \n" + "mov %k[width],%k[temp2] \n" + "shr $1,%k[temp2] \n" + "shlx %k[temp2],%k[temp1],%k[temp1] \n" + "kmovw %k[temp1],%%k4 \n" + "vpslld $8,%%zmm0,%%zmm1 \n" + "vpord %%zmm1,%%zmm0,%%zmm0%{%%k4%} \n" + "vpslld $8,%%zmm2,%%zmm3 \n" + "vpord %%zmm3,%%zmm2,%%zmm2%{%%k4%} \n" + "vpmaddubsw %%zmm18,%%zmm0,%%zmm0 \n" + "vpmaddubsw %%zmm18,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm2,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm19,%%zmm0,%%zmm0 \n" + "lea 1(%q[width]),%k[temp2] \n" + "and $-2,%k[temp2] \n" + "mov $-1,%[temp1] \n" + "bzhi %q[temp2],%[temp1],%[temp1] \n" + "kmovd %k[temp1],%%k3 \n" + "vpmovwb %%zmm0,(%[dst])%{%%k3%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_ayuv), [dst] "+r"(dst_uv), [width] "+r"(width), + [temp1] "=&r"(temp1), [temp2] "=&r"(temp2) + : [stride] "r"((ptrdiff_t)src_stride_ayuv), [perm] "r"(perm_table) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm18", + "xmm19", "k1", "k2", "k3", "k4"); +} +#endif // defined(HAS_AYUVTOUVROW_AVX512VBMI) || + // defined(HAS_AYUVTOVUROW_AVX512VBMI) + +#ifdef HAS_AYUVTOUVROW_AVX512VBMI +void OMITFP AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + AYUVToUVOrVURow_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_uv, kPermAYUVToUV, + width); +} +#endif // HAS_AYUVTOUVROW_AVX512VBMI + +#ifdef HAS_AYUVTOVUROW_AVX512VBMI +void OMITFP AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + AYUVToUVOrVURow_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_vu, kPermAYUVToVU, + width); +} +#endif // HAS_AYUVTOVUROW_AVX512VBMI + +// =========================================================================== +// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX512BW) +// =========================================================================== + +#ifdef HAS_AR30TOARGBROW_AVX512BW +void OMITFP AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width) { + static const uint32_t kC0000000 = 0xc0000000u; + static const uint32_t k300000FF = 0x300000ffu; + static const uint32_t k0C00FF00 = 0x0c00ff00u; + static const uint32_t k03FF0000 = 0x03ff0000u; + uintptr_t temp; + asm volatile( + "vpbroadcastd %[m0],%%zmm16 \n" + "vpbroadcastd %[m1],%%zmm17 \n" + "vpbroadcastd %[m2],%%zmm18 \n" + "vpbroadcastd %[m3],%%zmm19 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpsrld $2,%%zmm0,%%zmm1 \n" + "vpsrld $4,%%zmm0,%%zmm2 \n" + "vpsrld $6,%%zmm0,%%zmm3 \n" + "vpandd %%zmm16,%%zmm0,%%zmm0 \n" + "vpandd %%zmm17,%%zmm1,%%zmm1 \n" + "vpandd %%zmm18,%%zmm2,%%zmm2 \n" + "vpandd %%zmm19,%%zmm3,%%zmm3 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpord %%zmm3,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpsrld $2,%%zmm0,%%zmm1 \n" + "vpsrld $4,%%zmm0,%%zmm2 \n" + "vpsrld $6,%%zmm0,%%zmm3 \n" + "vpandd %%zmm16,%%zmm0,%%zmm0 \n" + "vpandd %%zmm17,%%zmm1,%%zmm1 \n" + "vpandd %%zmm18,%%zmm2,%%zmm2 \n" + "vpandd %%zmm19,%%zmm3,%%zmm3 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpord %%zmm3,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_ar30), [dst] "+r"(dst_argb), [width] "+r"(width), + [temp] "=&r"(temp) + : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00), + [m3] "m"(k03FF0000) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm17", + "xmm18", "xmm19", "k1"); +} +#endif // HAS_AR30TOARGBROW_AVX512BW + +#ifdef HAS_AR30TOABGRROW_AVX512BW +static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7, + 10, 9, 8, 11, 14, 13, 12, 15}; + +void OMITFP AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width) { + static const uint32_t kC0000000 = 0xc0000000u; + static const uint32_t k300000FF = 0x300000ffu; + static const uint32_t k0C00FF00 = 0x0c00ff00u; + static const uint32_t k03FF0000 = 0x03ff0000u; + uintptr_t temp; + asm volatile( + "vpbroadcastd %[m0],%%zmm16 \n" + "vpbroadcastd %[m1],%%zmm17 \n" + "vpbroadcastd %[m2],%%zmm18 \n" + "vpbroadcastd %[m3],%%zmm19 \n" + "vbroadcasti32x4 %[shuf],%%zmm20 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpsrld $2,%%zmm0,%%zmm1 \n" + "vpsrld $4,%%zmm0,%%zmm2 \n" + "vpsrld $6,%%zmm0,%%zmm3 \n" + "vpandd %%zmm16,%%zmm0,%%zmm0 \n" + "vpandd %%zmm17,%%zmm1,%%zmm1 \n" + "vpandd %%zmm18,%%zmm2,%%zmm2 \n" + "vpandd %%zmm19,%%zmm3,%%zmm3 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpord %%zmm3,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm20,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpsrld $2,%%zmm0,%%zmm1 \n" + "vpsrld $4,%%zmm0,%%zmm2 \n" + "vpsrld $6,%%zmm0,%%zmm3 \n" + "vpandd %%zmm16,%%zmm0,%%zmm0 \n" + "vpandd %%zmm17,%%zmm1,%%zmm1 \n" + "vpandd %%zmm18,%%zmm2,%%zmm2 \n" + "vpandd %%zmm19,%%zmm3,%%zmm3 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpord %%zmm3,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm20,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_ar30), [dst] "+r"(dst_abgr), [width] "+r"(width), + [temp] "=&r"(temp) + : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00), + [m3] "m"(k03FF0000), [shuf] "m"(kShufSwapRB16) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm17", + "xmm18", "xmm19", "xmm20", "k1"); +} +#endif // HAS_AR30TOABGRROW_AVX512BW + +#ifdef HAS_AR30TOAB30ROW_AVX512BW +void OMITFP AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width) { + static const uint32_t kC00FFC00 = 0xc00ffc00u; + static const uint32_t k3FF00000 = 0x3ff00000u; + static const uint32_t k000003FF = 0x000003ffu; + uintptr_t temp; + asm volatile( + "vpbroadcastd %[m0],%%zmm16 \n" + "vpbroadcastd %[m1],%%zmm17 \n" + "vpbroadcastd %[m2],%%zmm18 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpslld $20,%%zmm0,%%zmm1 \n" + "vpsrld $20,%%zmm0,%%zmm2 \n" + "vpandd %%zmm16,%%zmm0,%%zmm0 \n" + "vpandd %%zmm17,%%zmm1,%%zmm1 \n" + "vpandd %%zmm18,%%zmm2,%%zmm2 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpslld $20,%%zmm0,%%zmm1 \n" + "vpsrld $20,%%zmm0,%%zmm2 \n" + "vpandd %%zmm16,%%zmm0,%%zmm0 \n" + "vpandd %%zmm17,%%zmm1,%%zmm1 \n" + "vpandd %%zmm18,%%zmm2,%%zmm2 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_ar30), [dst] "+r"(dst_ab30), [width] "+r"(width), + [temp] "=&r"(temp) + : [m0] "m"(kC00FFC00), [m1] "m"(k3FF00000), [m2] "m"(k000003FF) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "xmm18", + "k1"); +} +#endif // HAS_AR30TOAB30ROW_AVX512BW + +#ifdef HAS_MERGEXR30ROW_10_AVX512BW +void OMITFP MergeXR30Row_10_AVX512BW(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + uint8_t* dst_ar30, + int depth, + int width) { + static const uint32_t kMax10 = 1023u; + static const uint32_t kAlpha30 = 0xc0000000u; + uintptr_t temp; + (void)depth; + asm volatile( + "vpbroadcastd %[max10],%%zmm16 \n" + "vpbroadcastd %[alpha],%%zmm17 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vpmovzxwd (%[r]),%%zmm0 \n" + "vpmovzxwd (%[g]),%%zmm1 \n" + "vpmovzxwd (%[b]),%%zmm2 \n" + "lea 0x20(%[r]),%[r] \n" + "lea 0x20(%[g]),%[g] \n" + "lea 0x20(%[b]),%[b] \n" + "vpminud %%zmm16,%%zmm0,%%zmm0 \n" + "vpminud %%zmm16,%%zmm1,%%zmm1 \n" + "vpminud %%zmm16,%%zmm2,%%zmm2 \n" + "vpslld $20,%%zmm0,%%zmm0 \n" + "vpslld $10,%%zmm1,%%zmm1 \n" + "vpord %%zmm17,%%zmm2,%%zmm2 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vpmovzxwd (%[r]),%%zmm0%{%%k1%}%{z%} \n" + "vpmovzxwd (%[g]),%%zmm1%{%%k1%}%{z%} \n" + "vpmovzxwd (%[b]),%%zmm2%{%%k1%}%{z%} \n" + "vpminud %%zmm16,%%zmm0,%%zmm0 \n" + "vpminud %%zmm16,%%zmm1,%%zmm1 \n" + "vpminud %%zmm16,%%zmm2,%%zmm2 \n" + "vpslld $20,%%zmm0,%%zmm0 \n" + "vpslld $10,%%zmm1,%%zmm1 \n" + "vpord %%zmm17,%%zmm2,%%zmm2 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [r] "+r"(src_r), [g] "+r"(src_g), [b] "+r"(src_b), [dst] "+r"(dst_ar30), + [width] "+r"(width), [temp] "=&r"(temp) + : [max10] "m"(kMax10), [alpha] "m"(kAlpha30) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "k1"); +} +#endif // HAS_MERGEXR30ROW_10_AVX512BW + +#if defined(HAS_ARGBTOAR30ROW_AVX512BW) || defined(HAS_ABGRTOAR30ROW_AVX512BW) +static inline void ARGBOrABGRToAR30Row_AVX512BW(const uint8_t* src, + uint8_t* dst, + const uvec8* shuf_rb, + int width) { + uintptr_t temp; + asm volatile( + "vbroadcasti32x4 (%[shuf]),%%zmm2 \n" + "vpbroadcastd %[mul_rb],%%zmm3 \n" + "vpbroadcastd %[mask_rb],%%zmm4 \n" + "vpbroadcastd %[mask_ag],%%zmm5 \n" + "vpbroadcastd %[mul_ag],%%zmm6 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpshufb %%zmm2,%%zmm0,%%zmm1 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm3,%%zmm1,%%zmm1 \n" + "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n" + "vpandd %%zmm4,%%zmm1,%%zmm1 \n" + "vpslld $10,%%zmm0,%%zmm0 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpshufb %%zmm2,%%zmm0,%%zmm1 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm3,%%zmm1,%%zmm1 \n" + "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n" + "vpandd %%zmm4,%%zmm1,%%zmm1 \n" + "vpslld $10,%%zmm0,%%zmm0 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : + [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [temp] "=&r"(temp) + : [shuf] "r"(shuf_rb), [mul_rb] "m"(kMulRB10), [mask_rb] "m"(kMaskRB10), + [mask_ag] "m"(kMaskAG10), [mul_ag] "m"(kMulAG10) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "k1"); +} +#endif + +#ifdef HAS_ARGBTOAR30ROW_AVX512BW +void OMITFP ARGBToAR30Row_AVX512BW(const uint8_t* src, + uint8_t* dst, + int width) { + ARGBOrABGRToAR30Row_AVX512BW(src, dst, &kShuffleRB30, width); +} +#endif // HAS_ARGBTOAR30ROW_AVX512BW + +#ifdef HAS_ABGRTOAR30ROW_AVX512BW +void OMITFP ABGRToAR30Row_AVX512BW(const uint8_t* src, + uint8_t* dst, + int width) { + ARGBOrABGRToAR30Row_AVX512BW(src, dst, &kShuffleBR30, width); +} +#endif // HAS_ABGRTOAR30ROW_AVX512BW + +// =========================================================================== +// Module 3: Planar / Channel Pack & Mirror / Interpolate_16 (AVX512VBMI / BW) +// =========================================================================== + +#ifdef HAS_MERGERGBROW_AVX512VBMI +static const uint8_t kPermMergeRGB64_0[64] = { + 0x00, 0x20, 0x40, 0x01, 0x21, 0x41, 0x02, 0x22, 0x42, 0x03, 0x23, + 0x43, 0x04, 0x24, 0x44, 0x05, 0x25, 0x45, 0x06, 0x26, 0x46, 0x07, + 0x27, 0x47, 0x08, 0x28, 0x48, 0x09, 0x29, 0x49, 0x0a, 0x2a, 0x4a, + 0x0b, 0x2b, 0x4b, 0x0c, 0x2c, 0x4c, 0x0d, 0x2d, 0x4d, 0x0e, 0x2e, + 0x4e, 0x0f, 0x2f, 0x4f, 0x10, 0x30, 0x50, 0x11, 0x31, 0x51, 0x12, + 0x32, 0x52, 0x13, 0x33, 0x53, 0x14, 0x34, 0x54, 0x15}; + +static const uint8_t kPermMergeRGB64_1[64] = { + 0x35, 0x55, 0x16, 0x36, 0x56, 0x17, 0x37, 0x57, 0x18, 0x38, 0x58, + 0x19, 0x39, 0x59, 0x1a, 0x3a, 0x5a, 0x1b, 0x3b, 0x5b, 0x1c, 0x3c, + 0x5c, 0x1d, 0x3d, 0x5d, 0x1e, 0x3e, 0x5e, 0x1f, 0x3f, 0x5f, 0x60, + 0x00, 0x20, 0x61, 0x01, 0x21, 0x62, 0x02, 0x22, 0x63, 0x03, 0x23, + 0x64, 0x04, 0x24, 0x65, 0x05, 0x25, 0x66, 0x06, 0x26, 0x67, 0x07, + 0x27, 0x68, 0x08, 0x28, 0x69, 0x09, 0x29, 0x6a, 0x0a}; + +static const uint8_t kPermMergeRGB64_2[64] = { + 0x6a, 0x2b, 0x4b, 0x6b, 0x2c, 0x4c, 0x6c, 0x2d, 0x4d, 0x6d, 0x2e, + 0x4e, 0x6e, 0x2f, 0x4f, 0x6f, 0x30, 0x50, 0x70, 0x31, 0x51, 0x71, + 0x32, 0x52, 0x72, 0x33, 0x53, 0x73, 0x34, 0x54, 0x74, 0x35, 0x55, + 0x75, 0x36, 0x56, 0x76, 0x37, 0x57, 0x77, 0x38, 0x58, 0x78, 0x39, + 0x59, 0x79, 0x3a, 0x5a, 0x7a, 0x3b, 0x5b, 0x7b, 0x3c, 0x5c, 0x7c, + 0x3d, 0x5d, 0x7d, 0x3e, 0x5e, 0x7e, 0x3f, 0x5f, 0x7f}; + +void OMITFP MergeRGBRow_AVX512VBMI(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width) { + uintptr_t temp1; + uintptr_t temp2; + asm volatile( + "vmovdqu8 %[m_rgb0],%%zmm16 \n" + "vmovdqu8 %[m_rgb1],%%zmm17 \n" + "vmovdqu8 %[m_rgb2],%%zmm18 \n" + "movabs $0xb6db6db600000000,%[temp1] \n" + "kmovq %[temp1],%%k2 \n" + "sub $0x40,%[width] \n" + "jl 2f \n" + + // 64-pixel loop (192 bytes out = 3 full 64B zmm stores, zero cache + // splits). + LABELALIGN + "1: \n" + "vmovdqu8 (%[r]),%%ymm0 \n" + "vinserti64x4 $1,(%[g]),%%zmm0,%%zmm0 \n" + "vmovdqu8 (%[b]),%%ymm2 \n" + "vinserti64x4 $1,0x20(%[r]),%%zmm2,%%zmm2 \n" + "vmovdqu8 0x20(%[g]),%%ymm4 \n" + "vinserti64x4 $1,0x20(%[b]),%%zmm4,%%zmm4 \n" + "lea 0x40(%[r]),%[r] \n" + "lea 0x40(%[g]),%[g] \n" + "lea 0x40(%[b]),%[b] \n" + "vmovdqa64 %%zmm0,%%zmm3 \n" + "vpermt2b %%zmm2,%%zmm16,%%zmm3 \n" + "vpermt2b %%zmm2,%%zmm17,%%zmm0 \n" + "vpermb %%zmm4,%%zmm17,%%zmm0%{%%k2%} \n" + "vpermt2b %%zmm4,%%zmm18,%%zmm2 \n" + "vmovdqu8 %%zmm3,(%[dst]) \n" + "vmovdqu8 %%zmm0,0x40(%[dst]) \n" + "vmovdqu8 %%zmm2,0x80(%[dst]) \n" + "lea 0xc0(%[dst]),%[dst] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "jl 3f \n" + "vmovdqu8 (%[r]),%%ymm0 \n" + "vinserti64x4 $1,(%[g]),%%zmm0,%%zmm0 \n" + "vmovdqu8 (%[b]),%%ymm2 \n" + "lea 0x20(%[r]),%[r] \n" + "lea 0x20(%[g]),%[g] \n" + "lea 0x20(%[b]),%[b] \n" + "vmovdqa64 %%zmm0,%%zmm3 \n" + "vpermt2b %%zmm2,%%zmm16,%%zmm3 \n" + "vpermt2b %%zmm2,%%zmm17,%%zmm0 \n" + "vmovdqu8 %%zmm3,(%[dst]) \n" + "vmovdqu8 %%ymm0,0x40(%[dst]) \n" + "lea 0x60(%[dst]),%[dst] \n" + "sub $0x20,%[width] \n" + + "3: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%[temp1] \n" + "bzhi %q[width],%[temp1],%[temp2] \n" + "kmovd %k[temp2],%%k1 \n" + "vmovdqu8 (%[r]),%%ymm0%{%%k1%}%{z%} \n" + "vmovdqu8 (%[g]),%%ymm1%{%%k1%}%{z%} \n" + "vmovdqu8 (%[b]),%%ymm2%{%%k1%}%{z%} \n" + "vinserti64x4 $1,%%ymm1,%%zmm0,%%zmm0 \n" + "lea (%q[width],%q[width],2),%k[width]\n" + "bzhi %q[width],%[temp1],%[temp2] \n" + "kmovq %[temp2],%%k3 \n" + "xor %k[temp2],%k[temp2] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp1],%[temp2] \n" + "bzhi %q[width],%[temp2],%[temp2] \n" + "kmovd %k[temp2],%%k4 \n" + "vmovdqa64 %%zmm0,%%zmm3 \n" + "vpermt2b %%zmm2,%%zmm16,%%zmm3 \n" + "vpermt2b %%zmm2,%%zmm17,%%zmm0 \n" + "vmovdqu8 %%zmm3,(%[dst])%{%%k3%} \n" + "vmovdqu8 %%ymm0,0x40(%[dst])%{%%k4%} \n" + + "99: \n" + "vzeroupper \n" + : [r] "+r"(src_r), [g] "+r"(src_g), [b] "+r"(src_b), [dst] "+r"(dst_rgb), + [width] "+r"(width), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2) + : [m_rgb0] "m"(kPermMergeRGB64_0), [m_rgb1] "m"(kPermMergeRGB64_1), + [m_rgb2] "m"(kPermMergeRGB64_2) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", + "xmm17", "xmm18", "k1", "k2", "k3", "k4"); +} +#endif // HAS_MERGERGBROW_AVX512VBMI + +#ifdef HAS_RAWTORGBAROW_AVX512VBMI +static const uint8_t kPermRAWToRGBA0[64] = { + 0, 2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6, 0, 11, 10, 9, + 0, 14, 13, 12, 0, 17, 16, 15, 0, 20, 19, 18, 0, 23, 22, 21, + 0, 26, 25, 24, 0, 29, 28, 27, 0, 32, 31, 30, 0, 35, 34, 33, + 0, 38, 37, 36, 0, 41, 40, 39, 0, 44, 43, 42, 0, 47, 46, 45}; + +static const uint8_t kPermRAWToRGBA1[64] = { + 0, 50, 49, 48, 0, 53, 52, 51, 0, 56, 55, 54, 0, 59, 58, 57, + 0, 62, 61, 60, 0, 65, 64, 63, 0, 68, 67, 66, 0, 71, 70, 69, + 0, 74, 73, 72, 0, 77, 76, 75, 0, 80, 79, 78, 0, 83, 82, 81, + 0, 86, 85, 84, 0, 89, 88, 87, 0, 92, 91, 90, 0, 95, 94, 93}; + +void OMITFP RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width) { + static const uint32_t kAlphaLow = 0x000000ffu; + uintptr_t temp1; + uintptr_t temp2; + asm volatile( + "vmovdqu8 %[p0],%%zmm16 \n" + "vmovdqu8 %[p1],%%zmm17 \n" + "vpbroadcastd %[alpha],%%zmm18 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%[src]),%%zmm0 \n" + "vmovdqu8 0x40(%[src]),%%ymm1 \n" + "lea 0x60(%[src]),%[src] \n" + "vpermb %%zmm0,%%zmm16,%%zmm2 \n" + "vpermt2b %%zmm1,%%zmm17,%%zmm0 \n" + "vpord %%zmm18,%%zmm2,%%zmm2 \n" + "vpord %%zmm18,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm2,(%[dst]) \n" + "vmovdqu32 %%zmm0,0x40(%[dst]) \n" + "lea 0x80(%[dst]),%[dst] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%[temp1] \n" + "bzhi %q[width],%[temp1],%[temp2] \n" + "kmovw %k[temp2],%%k1 \n" + "kmovd %k[temp2],%%k2 \n" + "kshiftrd $16,%%k2,%%k2 \n" + "lea (%q[width],%q[width],2),%k[width]\n" + "bzhi %q[width],%[temp1],%[temp2] \n" + "kmovq %[temp2],%%k3 \n" + "xor %k[temp2],%k[temp2] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp1],%[temp2] \n" + "bzhi %q[width],%[temp2],%[temp2] \n" + "kmovd %k[temp2],%%k4 \n" + "vmovdqu8 (%[src]),%%zmm0%{%%k3%}%{z%} \n" + "vmovdqu8 0x40(%[src]),%%ymm1%{%%k4%}%{z%}\n" + "vpermb %%zmm0,%%zmm16,%%zmm2 \n" + "vpermt2b %%zmm1,%%zmm17,%%zmm0 \n" + "vpord %%zmm18,%%zmm2,%%zmm2 \n" + "vpord %%zmm18,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm2,(%[dst])%{%%k1%} \n" + "vmovdqu32 %%zmm0,0x40(%[dst])%{%%k2%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_raw), [dst] "+r"(dst_rgba), [width] "+r"(width), + [temp1] "=&r"(temp1), [temp2] "=&r"(temp2) + : [p0] "m"(kPermRAWToRGBA0), [p1] "m"(kPermRAWToRGBA1), + [alpha] "m"(kAlphaLow) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "xmm18", "k1", + "k2", "k3", "k4"); +} +#endif // HAS_RAWTORGBAROW_AVX512VBMI + +#ifdef HAS_SWAPUVROW_AVX512BW +static const uint8_t kSwapUV16[16] = {1, 0, 3, 2, 5, 4, 7, 6, + 9, 8, 11, 10, 13, 12, 15, 14}; + +void OMITFP SwapUVRow_AVX512BW(const uint8_t* src_uv, + uint8_t* dst_vu, + int width) { + uintptr_t temp; + asm volatile( + "vbroadcasti32x4 %[swap],%%zmm16 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu16 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpshufb %%zmm16,%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpshufb %%zmm16,%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_uv), [dst] "+r"(dst_vu), [width] "+r"(width), + [temp] "=&r"(temp) + : [swap] "m"(kSwapUV16) + : "memory", "cc", "xmm0", "xmm16", "k1"); +} +#endif // HAS_SWAPUVROW_AVX512BW + +#ifdef HAS_MIRRORROW_16_AVX512BW +static const uint16_t kMirror16Idx[32] = { + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + +void OMITFP MirrorRow_16_AVX512BW(const uint16_t* src, + uint16_t* dst, + int width) { + uintptr_t temp1; + uintptr_t temp2; + asm volatile( + "vmovdqu16 %[rev],%%zmm16 \n" + "lea (%[src],%q[width],2),%[src] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "sub $0x40,%[src] \n" + "vmovdqu16 (%[src]),%%zmm0 \n" + "vpermw %%zmm0,%%zmm16,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst]) \n" + "add $0x40,%[dst] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $32,%k[temp2] \n" + "sub %k[width],%k[temp2] \n" + "vpbroadcastw %k[temp2],%%zmm17 \n" + "vpsubw %%zmm17,%%zmm16,%%zmm17 \n" + "mov $-1,%[temp1] \n" + "bzhi %q[width],%[temp1],%[temp1] \n" + "kmovd %k[temp1],%%k1 \n" + "neg %q[width] \n" + "vmovdqu16 (%[src],%q[width],2),%%zmm0%{%%k1%}%{z%}\n" + "vpermw %%zmm0,%%zmm17,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), + [temp1] "=&r"(temp1), [temp2] "=&r"(temp2) + : [rev] "m"(kMirror16Idx) + : "memory", "cc", "xmm0", "xmm16", "xmm17", "k1"); +} +#endif // HAS_MIRRORROW_16_AVX512BW + +#ifdef HAS_HALFMERGEUVROW_AVX512BW +void OMITFP HalfMergeUVRow_AVX512BW(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width) { + uintptr_t temp1; + uintptr_t temp2; + asm volatile( + "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" + "vpsrlw $15,%%zmm16,%%zmm16 \n" + "vpsllw $8,%%zmm16,%%zmm17 \n" + "vpord %%zmm17,%%zmm16,%%zmm16 \n" // 0x0101 + "vpxord %%zmm17,%%zmm17,%%zmm17 \n" // 0 + "vpternlogd $0xff,%%zmm18,%%zmm18,%%zmm18 \n" + "vpsrlw $8,%%zmm18,%%zmm18 \n" // 0x00ff + "sub $0x40,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%[u]),%%zmm0 \n" + "vmovdqu8 (%[u],%[su]),%%zmm1 \n" + "vmovdqu8 (%[v]),%%zmm2 \n" + "vmovdqu8 (%[v],%[sv]),%%zmm3 \n" + "lea 0x40(%[u]),%[u] \n" + "lea 0x40(%[v]),%[v] \n" + "vpmaddubsw %%zmm16,%%zmm0,%%zmm0 \n" + "vpmaddubsw %%zmm16,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm16,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm16,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm3,%%zmm2,%%zmm2 \n" + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm2,%%zmm2 \n" + "vpavgw %%zmm17,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm17,%%zmm2,%%zmm2 \n" + "vpsllw $8,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "je 99f \n" + "mov $-1,%[temp1] \n" + "bzhi %q[width],%[temp1],%[temp2] \n" + "kmovq %[temp2],%%k1 \n" + "vmovdqu8 (%[u]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu8 (%[u],%[su]),%%zmm1%{%%k1%}%{z%}\n" + "vmovdqu8 (%[v]),%%zmm2%{%%k1%}%{z%} \n" + "vmovdqu8 (%[v],%[sv]),%%zmm3%{%%k1%}%{z%}\n" + // If width is odd, duplicate byte (width - 1) into high byte of word + // (width >> 1). + "mov %k[width],%k[temp1] \n" + "and $1,%k[temp1] \n" + "mov %k[width],%k[temp2] \n" + "shr $1,%k[temp2] \n" + "shlx %k[temp2],%k[temp1],%k[temp1] \n" + "kmovd %k[temp1],%%k2 \n" + "vpandd %%zmm18,%%zmm0,%%zmm4 \n" + "vpmullw %%zmm16,%%zmm4,%%zmm0%{%%k2%} \n" + "vpandd %%zmm18,%%zmm1,%%zmm4 \n" + "vpmullw %%zmm16,%%zmm4,%%zmm1%{%%k2%} \n" + "vpandd %%zmm18,%%zmm2,%%zmm4 \n" + "vpmullw %%zmm16,%%zmm4,%%zmm2%{%%k2%} \n" + "vpandd %%zmm18,%%zmm3,%%zmm4 \n" + "vpmullw %%zmm16,%%zmm4,%%zmm3%{%%k2%} \n" + "vpmaddubsw %%zmm16,%%zmm0,%%zmm0 \n" + "vpmaddubsw %%zmm16,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm16,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm16,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm3,%%zmm2,%%zmm2 \n" + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm2,%%zmm2 \n" + "vpavgw %%zmm17,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm17,%%zmm2,%%zmm2 \n" + "vpsllw $8,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "lea 1(%q[width]),%k[temp2] \n" + "shr $1,%k[temp2] \n" + "mov $-1,%[temp1] \n" + "bzhi %q[temp2],%[temp1],%[temp1] \n" + "kmovd %k[temp1],%%k2 \n" + "vmovdqu16 %%zmm0,(%[dst])%{%%k2%} \n" + + "99: \n" + "vzeroupper \n" + : [u] "+r"(src_u), [v] "+r"(src_v), [dst] "+r"(dst_uv), + [width] "+r"(width), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2) + : [su] "r"((ptrdiff_t)src_stride_u), [sv] "r"((ptrdiff_t)src_stride_v) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", + "xmm17", "xmm18", "k1", "k2"); +} +#endif // HAS_HALFMERGEUVROW_AVX512BW + +#ifdef HAS_INTERPOLATEROW_16_AVX512BW +void OMITFP InterpolateRow_16_AVX512BW(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + int y1_fraction = source_y_fraction; + int y0_fraction = 256 - y1_fraction; + const uint16_t* src_ptr1 = src_ptr + src_stride; + int y_fractions = (y1_fraction << 16) + y0_fraction; + uintptr_t temp1; + uintptr_t temp2; + asm volatile( + "cmpl $0x0,%[y1] \n" + "je 10f \n" + "cmpl $0x80,%[y1] \n" + "je 20f \n" + + "vpbroadcastd %[y_frac],%%zmm16 \n" + "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n" + "vpsllw $15,%%zmm17,%%zmm17 \n" // 0x8000 + "mov $0x800080,%k[temp1] \n" + "vpbroadcastd %k[temp1],%%zmm18 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + // 32-pixel general blend loop (vpmaddwd + vpackusdw). + LABELALIGN + "1: \n" + "vmovdqu16 (%[src]),%%zmm0 \n" + "vmovdqu16 (%[src1]),%%zmm1 \n" + "lea 0x40(%[src]),%[src] \n" + "lea 0x40(%[src1]),%[src1] \n" + "vpunpcklwd %%zmm1,%%zmm0,%%zmm2 \n" + "vpunpckhwd %%zmm1,%%zmm0,%%zmm0 \n" + "vpsubw %%zmm17,%%zmm2,%%zmm2 \n" + "vpsubw %%zmm17,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" + "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm18,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm18,%%zmm0,%%zmm0 \n" + "vpsrad $8,%%zmm2,%%zmm2 \n" + "vpsrad $8,%%zmm0,%%zmm0 \n" + "vpackusdw %%zmm0,%%zmm2,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%[temp1] \n" + "bzhi %q[width],%[temp1],%[temp2] \n" + "kmovd %k[temp2],%%k1 \n" + "vmovdqu16 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 (%[src1]),%%zmm1%{%%k1%}%{z%} \n" + "vpunpcklwd %%zmm1,%%zmm0,%%zmm2 \n" + "vpunpckhwd %%zmm1,%%zmm0,%%zmm0 \n" + "vpsubw %%zmm17,%%zmm2,%%zmm2 \n" + "vpsubw %%zmm17,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" + "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm18,%%zmm2,%%zmm2 \n" + "vpaddd %%zmm18,%%zmm0,%%zmm0 \n" + "vpsrad $8,%%zmm2,%%zmm2 \n" + "vpsrad $8,%%zmm0,%%zmm0 \n" + "vpackusdw %%zmm0,%%zmm2,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n" + "jmp 99f \n" + + // y1 == 0 fast copy path (32 uint16_t per zmm). + "10: \n" + "sub $0x20,%[width] \n" + "jl 12f \n" LABELALIGN + "11: \n" + "vmovdqu16 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vmovdqu16 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x20,%[width] \n" + "jge 11b \n" + "12: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%[temp1] \n" + "bzhi %q[width],%[temp1],%[temp2] \n" + "kmovd %k[temp2],%%k1 \n" + "vmovdqu16 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n" + "jmp 99f \n" + + // y1 == 128 fast 50/50 average path (vpavgw on 32 uint16_t per zmm). + "20: \n" + "sub $0x20,%[width] \n" + "jl 22f \n" LABELALIGN + "21: \n" + "vmovdqu16 (%[src]),%%zmm0 \n" + "vpavgw (%[src1]),%%zmm0,%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "lea 0x40(%[src1]),%[src1] \n" + "vmovdqu16 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x20,%[width] \n" + "jge 21b \n" + "22: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%[temp1] \n" + "bzhi %q[width],%[temp1],%[temp2] \n" + "kmovd %k[temp2],%%k1 \n" + "vmovdqu16 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 (%[src1]),%%zmm1%{%%k1%}%{z%} \n" + "vpavgw %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst_ptr), [src] "+r"(src_ptr), [src1] "+r"(src_ptr1), + [width] "+r"(width), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2) + : [y1] "r"(y1_fraction), [y_frac] "r"(y_fractions) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "xmm18", + "k1"); +} +#endif // HAS_INTERPOLATEROW_16_AVX512BW + +// =========================================================================== +// Module 4: NV12 / NV21 & I422Alpha to ARGB / RGB24 (AVX512BW / AVX512VBMI) +// =========================================================================== + +#define READYUVNV12_AVX512BW \ + "vmovdqu16 (%[uv_buf]),%%ymm3 \n" \ + "lea 0x20(%[uv_buf]),%[uv_buf] \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm1 \n" \ + "vpmovzxbw (%[y_buf]),%%zmm4 \n" \ + "lea 0x20(%[y_buf]),%[y_buf] \n" \ + "vpsllw $8,%%zmm4,%%zmm3 \n" \ + "vpord %%zmm3,%%zmm4,%%zmm4 \n" \ + "vmovdqa64 %%zmm1,%%zmm3 \n" + +#define READYUVNV12_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "mov $-1,%[temp] \n" \ + "lea 1(%q[width]),%k[y_buf] \n" \ + "shr $1,%k[y_buf] \n" \ + "bzhi %q[y_buf],%[temp],%[y_buf] \n" \ + "kmovw %k[y_buf],%%k2 \n" \ + "vmovdqu16 (%[uv_buf]),%%ymm3%{%%k2%}%{z%} \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm1 \n" \ + "vpsllw $8,%%zmm4,%%zmm3 \n" \ + "vpord %%zmm3,%%zmm4,%%zmm4 \n" \ + "vmovdqa64 %%zmm1,%%zmm3 \n" + +#define READYUVNV21_AVX512BW \ + "vmovdqu16 (%[uv_buf]),%%ymm3 \n" \ + "lea 0x20(%[uv_buf]),%[uv_buf] \n" \ + "vpsllw $8,%%ymm3,%%ymm1 \n" \ + "vpsrlw $8,%%ymm3,%%ymm3 \n" \ + "vpor %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm1 \n" \ + "vpmovzxbw (%[y_buf]),%%zmm4 \n" \ + "lea 0x20(%[y_buf]),%[y_buf] \n" \ + "vpsllw $8,%%zmm4,%%zmm3 \n" \ + "vpord %%zmm3,%%zmm4,%%zmm4 \n" \ + "vmovdqa64 %%zmm1,%%zmm3 \n" + +#define READYUVNV21_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "mov $-1,%[temp] \n" \ + "lea 1(%q[width]),%k[y_buf] \n" \ + "shr $1,%k[y_buf] \n" \ + "bzhi %q[y_buf],%[temp],%[y_buf] \n" \ + "kmovw %k[y_buf],%%k2 \n" \ + "vmovdqu16 (%[uv_buf]),%%ymm3%{%%k2%}%{z%} \n" \ + "vpsllw $8,%%ymm3,%%ymm1 \n" \ + "vpsrlw $8,%%ymm3,%%ymm3 \n" \ + "vpor %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm1 \n" \ + "vpsllw $8,%%zmm4,%%zmm3 \n" \ + "vpord %%zmm3,%%zmm4,%%zmm4 \n" \ + "vmovdqa64 %%zmm1,%%zmm3 \n" + +#ifdef HAS_NV12TOARGBROW_AVX512BW +void OMITFP NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUVNV12_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUVNV12_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_NV12TOARGBROW_AVX512BW + +#ifdef HAS_NV21TOARGBROW_AVX512BW +void OMITFP NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* vu_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUVNV21_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUVNV21_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(vu_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_NV21TOARGBROW_AVX512BW + +#ifdef HAS_NV12TORGB24ROW_AVX512VBMI +void OMITFP NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vmovdqu32 %[kMaskBG],%%zmm20 \n" + "vmovdqu32 %[kMaskDST0],%%zmm21 \n" + "vmovdqu32 %[kMaskDST1],%%zmm22 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUVNV12_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_AVX512VBMI + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUVNV12_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_MASK_AVX512VBMI + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_rgb24] "+r"(dst_rgb24), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kMaskBG] "m"(kMaskBG), + [kMaskDST0] "m"(kMaskDST0), + [kMaskDST1] "m"(kMaskDST1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm20", "xmm21", "xmm22", "k1", "k2", "k3", "k4"); +} +#endif // HAS_NV12TORGB24ROW_AVX512VBMI + +#ifdef HAS_NV21TORGB24ROW_AVX512VBMI +void OMITFP NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* vu_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vmovdqu32 %[kMaskBG],%%zmm20 \n" + "vmovdqu32 %[kMaskDST0],%%zmm21 \n" + "vmovdqu32 %[kMaskDST1],%%zmm22 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUVNV21_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_AVX512VBMI + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUVNV21_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_MASK_AVX512VBMI + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(vu_buf), + [dst_rgb24] "+r"(dst_rgb24), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kMaskBG] "m"(kMaskBG), + [kMaskDST0] "m"(kMaskDST0), + [kMaskDST1] "m"(kMaskDST1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm20", "xmm21", "xmm22", "k1", "k2", "k3", "k4"); +} +#endif // HAS_NV21TORGB24ROW_AVX512VBMI + +#ifdef HAS_I422ALPHATOARGBROW_AVX512BW +void OMITFP I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV422_AVX512BW + "vpmovzxbw (%[a_buf]),%%zmm5 \n" + "lea 0x20(%[a_buf]),%[a_buf] \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV422_MASK_AVX512BW + "vpmovzxbw (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [a_buf] "+r"(a_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I422ALPHATOARGBROW_AVX512BW + +// =========================================================================== +// Module 5: I400ToARGB & ARGB Shaders (Attenuate, Multiply, Gray, Blend) +// =========================================================================== + +#ifdef HAS_I400TOARGBROW_AVX512BW +static const uint64_t kLinearizePackQ[8] = {0, 2, 4, 6, 1, 3, 5, 7}; +static const uint8_t kBcastGrayByte[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, + 8, 8, 8, 0x80, 12, 12, 12, 0x80}; + +void OMITFP I400ToARGBRow_AVX512BW(const uint8_t* y_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + "vpbroadcastw 96(%[yuv]),%%zmm16 \n" // kYToRgb + "vpbroadcastw 128(%[yuv]),%%zmm17 \n" // kYBiasToRgb + "vmovdqu64 %[lin],%%zmm18 \n" + "vbroadcasti32x4 %[bcast],%%zmm19 \n" + "vpternlogd $0xff,%%zmm20,%%zmm20,%%zmm20 \n" + "vpslld $24,%%zmm20,%%zmm20 \n" // 0xff000000 + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vpmovzxbw (%[y]),%%zmm0 \n" + "lea 0x20(%[y]),%[y] \n" + "vpsllw $8,%%zmm0,%%zmm1 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm16,%%zmm0,%%zmm0 \n" + "vpaddsw %%zmm17,%%zmm0,%%zmm0 \n" + "vpsraw $6,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n" + "vpermq %%zmm0,%%zmm18,%%zmm0 \n" + "vpmovzxbd %%xmm0,%%zmm1 \n" + "vextracti128 $1,%%ymm0,%%xmm0 \n" + "vpmovzxbd %%xmm0,%%zmm0 \n" + "vpshufb %%zmm19,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm19,%%zmm0,%%zmm0 \n" + "vpord %%zmm20,%%zmm1,%%zmm1 \n" + "vpord %%zmm20,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm1,(%[dst]) \n" + "vmovdqu32 %%zmm0,0x40(%[dst]) \n" + "lea 0x80(%[dst]),%[dst] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "kshiftrd $16,%%k1,%%k2 \n" + "vpmovzxbw (%[y]),%%zmm0%{%%k1%}%{z%} \n" + "vpsllw $8,%%zmm0,%%zmm1 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm16,%%zmm0,%%zmm0 \n" + "vpaddsw %%zmm17,%%zmm0,%%zmm0 \n" + "vpsraw $6,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n" + "vpermq %%zmm0,%%zmm18,%%zmm0 \n" + "vpmovzxbd %%xmm0,%%zmm1 \n" + "vextracti128 $1,%%ymm0,%%xmm0 \n" + "vpmovzxbd %%xmm0,%%zmm0 \n" + "vpshufb %%zmm19,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm19,%%zmm0,%%zmm0 \n" + "vpord %%zmm20,%%zmm1,%%zmm1 \n" + "vpord %%zmm20,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm1,(%[dst])%{%%k1%} \n" + "vmovdqu32 %%zmm0,0x40(%[dst])%{%%k2%} \n" + + "99: \n" + "vzeroupper \n" + : [y] "+r"(y_buf), [dst] "+r"(dst_argb), [width] "+r"(width), + [temp] "=&r"(temp) + : [yuv] "r"(yuvconstants), [lin] "m"(kLinearizePackQ), + [bcast] "m"(kBcastGrayByte) + : "memory", "cc", "xmm0", "xmm1", "xmm16", "xmm17", "xmm18", "xmm19", + "xmm20", "k1", "k2"); +} +#endif // HAS_I400TOARGBROW_AVX512BW + +#ifdef HAS_ARGBATTENUATEROW_AVX512BW +void OMITFP ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + uintptr_t temp; + asm volatile( + "vbroadcasti32x4 %[shuf],%%zmm16 \n" + "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n" + "vpslld $24,%%zmm17,%%zmm17 \n" // 0xff000000 + "vpternlogd $0xff,%%zmm18,%%zmm18,%%zmm18 \n" + "vpsrlw $8,%%zmm18,%%zmm18 \n" // 0x00ff + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm6 \n" + "lea 0x40(%[src]),%[src] \n" + "vpunpcklbw %%zmm17,%%zmm6,%%zmm0 \n" + "vpunpckhbw %%zmm17,%%zmm6,%%zmm1 \n" + "vpshufb %%zmm16,%%zmm0,%%zmm2 \n" + "vpshufb %%zmm16,%%zmm1,%%zmm3 \n" + "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmullw %%zmm3,%%zmm1,%%zmm1 \n" + "vpaddw %%zmm18,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm18,%%zmm1,%%zmm1 \n" + "vpsrlw $8,%%zmm0,%%zmm0 \n" + "vpsrlw $8,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpandd %%zmm17,%%zmm6,%%zmm1 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm6%{%%k1%}%{z%} \n" + "vpunpcklbw %%zmm17,%%zmm6,%%zmm0 \n" + "vpunpckhbw %%zmm17,%%zmm6,%%zmm1 \n" + "vpshufb %%zmm16,%%zmm0,%%zmm2 \n" + "vpshufb %%zmm16,%%zmm1,%%zmm3 \n" + "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmullw %%zmm3,%%zmm1,%%zmm1 \n" + "vpaddw %%zmm18,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm18,%%zmm1,%%zmm1 \n" + "vpsrlw $8,%%zmm0,%%zmm0 \n" + "vpsrlw $8,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpandd %%zmm17,%%zmm6,%%zmm1 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width), + [temp] "=&r"(temp) + : [shuf] "m"(kAttenuateShuffle_AVX2) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm6", "xmm16", + "xmm17", "xmm18", "k1"); +} +#endif // HAS_ARGBATTENUATEROW_AVX512BW + +#ifdef HAS_ARGBMULTIPLYROW_AVX512BW +void OMITFP ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + uintptr_t temp; + asm volatile( + "vpxord %%zmm16,%%zmm16,%%zmm16 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src0]),%%zmm1 \n" + "vmovdqu32 (%[src1]),%%zmm3 \n" + "lea 0x40(%[src0]),%[src0] \n" + "lea 0x40(%[src1]),%[src1] \n" + "vpunpcklbw %%zmm1,%%zmm16,%%zmm0 \n" + "vpunpckhbw %%zmm1,%%zmm16,%%zmm1 \n" + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm1,%%zmm1 \n" + "vpunpcklbw %%zmm16,%%zmm3,%%zmm2 \n" + "vpunpckhbw %%zmm16,%%zmm3,%%zmm3 \n" + "vpmulhrsw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmulhrsw %%zmm3,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src0]),%%zmm1%{%%k1%}%{z%} \n" + "vmovdqu32 (%[src1]),%%zmm3%{%%k1%}%{z%} \n" + "vpunpcklbw %%zmm1,%%zmm16,%%zmm0 \n" + "vpunpckhbw %%zmm1,%%zmm16,%%zmm1 \n" + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm1,%%zmm1 \n" + "vpunpcklbw %%zmm16,%%zmm3,%%zmm2 \n" + "vpunpckhbw %%zmm16,%%zmm3,%%zmm3 \n" + "vpmulhrsw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmulhrsw %%zmm3,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [width] "+r"(width), [temp] "=&r"(temp) + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "k1"); +} +#endif // HAS_ARGBMULTIPLYROW_AVX512BW + +#ifdef HAS_ARGBGRAYROW_AVX512BW +static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, + 8, 8, 8, 0x80, 12, 12, 12, 0x80}; + +void OMITFP ARGBGrayRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const uint32_t kCoeff = 0x004d961du; // {29, 150, 77, 0} + static const uint32_t kOne16 = 0x00010001u; + static const uint32_t kBias = 32896u; + uintptr_t temp; + asm volatile( + "vpbroadcastd %[coeff],%%zmm16 \n" + "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n" + "vpsllw $7,%%zmm17,%%zmm17 \n" + "vpbroadcastb %%xmm17,%%zmm17 \n" // 0x80808080 + "vpbroadcastd %[one16],%%zmm18 \n" + "vpbroadcastd %[bias],%%zmm19 \n" + "vbroadcasti32x4 %[shuf],%%zmm20 \n" + "vpternlogd $0xff,%%zmm21,%%zmm21,%%zmm21 \n" + "vpslld $24,%%zmm21,%%zmm21 \n" // 0xff000000 + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpsubb %%zmm17,%%zmm0,%%zmm1 \n" + "vpmaddubsw %%zmm1,%%zmm16,%%zmm1 \n" + "vpmaddwd %%zmm18,%%zmm1,%%zmm1 \n" + "vpaddd %%zmm19,%%zmm1,%%zmm1 \n" + "vpsrld $8,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm20,%%zmm1,%%zmm1 \n" + "vpandd %%zmm21,%%zmm0,%%zmm0 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpsubb %%zmm17,%%zmm0,%%zmm1 \n" + "vpmaddubsw %%zmm1,%%zmm16,%%zmm1 \n" + "vpmaddwd %%zmm18,%%zmm1,%%zmm1 \n" + "vpaddd %%zmm19,%%zmm1,%%zmm1 \n" + "vpsrld $8,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm20,%%zmm1,%%zmm1 \n" + "vpandd %%zmm21,%%zmm0,%%zmm0 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width), + [temp] "=&r"(temp) + : [coeff] "m"(kCoeff), [one16] "m"(kOne16), [bias] "m"(kBias), + [shuf] "m"(kShufGrayDword) + : "memory", "cc", "xmm0", "xmm1", "xmm16", "xmm17", "xmm18", "xmm19", + "xmm20", "xmm21", "k1"); +} +#endif // HAS_ARGBGRAYROW_AVX512BW + +#ifdef HAS_ARGBBLENDROW_AVX512BW +void OMITFP ARGBBlendRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + uintptr_t temp; + asm volatile( + "vbroadcasti32x4 %[shuf],%%zmm16 \n" + "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n" + "vpslld $24,%%zmm17,%%zmm17 \n" // 0xff000000 + "vpternlogd $0xff,%%zmm18,%%zmm18,%%zmm18 \n" + "vpsllw $8,%%zmm18,%%zmm18 \n" // 0xff00 + "vpternlogd $0xff,%%zmm19,%%zmm19,%%zmm19 \n" + "vpsrlw $8,%%zmm19,%%zmm19 \n" // 0x00ff + "vpternlogd $0xff,%%zmm20,%%zmm20,%%zmm20 \n" + "vpsrlw $15,%%zmm20,%%zmm20 \n" // 0x0001 + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src0]),%%zmm3 \n" + "vmovdqu32 (%[src1]),%%zmm2 \n" + "lea 0x40(%[src0]),%[src0] \n" + "lea 0x40(%[src1]),%[src1] \n" + "vmovdqa64 %%zmm3,%%zmm0 \n" + "vpxord %%zmm17,%%zmm3,%%zmm3 \n" + "vpshufb %%zmm16,%%zmm3,%%zmm3 \n" + "vmovdqa64 %%zmm2,%%zmm1 \n" + "vpandd %%zmm19,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm20,%%zmm3,%%zmm3 \n" + "vpmullw %%zmm3,%%zmm2,%%zmm2 \n" + "vpsrlw $8,%%zmm1,%%zmm1 \n" + "vpord %%zmm17,%%zmm0,%%zmm0 \n" + "vpmullw %%zmm3,%%zmm1,%%zmm1 \n" + "vpsrlw $8,%%zmm2,%%zmm2 \n" + "vpaddusb %%zmm2,%%zmm0,%%zmm0 \n" + "vpandd %%zmm18,%%zmm1,%%zmm1 \n" + "vpaddusb %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src0]),%%zmm3%{%%k1%}%{z%} \n" + "vmovdqu32 (%[src1]),%%zmm2%{%%k1%}%{z%} \n" + "vmovdqa64 %%zmm3,%%zmm0 \n" + "vpxord %%zmm17,%%zmm3,%%zmm3 \n" + "vpshufb %%zmm16,%%zmm3,%%zmm3 \n" + "vmovdqa64 %%zmm2,%%zmm1 \n" + "vpandd %%zmm19,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm20,%%zmm3,%%zmm3 \n" + "vpmullw %%zmm3,%%zmm2,%%zmm2 \n" + "vpsrlw $8,%%zmm1,%%zmm1 \n" + "vpord %%zmm17,%%zmm0,%%zmm0 \n" + "vpmullw %%zmm3,%%zmm1,%%zmm1 \n" + "vpsrlw $8,%%zmm2,%%zmm2 \n" + "vpaddusb %%zmm2,%%zmm0,%%zmm0 \n" + "vpandd %%zmm18,%%zmm1,%%zmm1 \n" + "vpaddusb %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [width] "+r"(width), [temp] "=&r"(temp) + : [shuf] "m"(kShuffleAlpha) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm17", + "xmm18", "xmm19", "xmm20", "k1"); +} +#endif // HAS_ARGBBLENDROW_AVX512BW + #endif // !defined(LIBYUV_DISABLE_X86) && // (defined(__x86_64__) || defined(__i386__)) && // !defined(LIBYUV_ENABLE_ROWWIN) diff --git a/source/row_win.cc b/source/row_win.cc index 90cfb4086..e00419249 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -2914,6 +2914,1463 @@ void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, } #endif // HAS_RGB24MIRRORROW_AVX512VBMI +// =========================================================================== +// Module 1: AYUV to Planar Y / UV / VU (AVX512VBMI) +// =========================================================================== + +#ifdef HAS_AYUVTOYROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI +void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width) { + static const uint8_t kPermAYUVToY32[64] = { + 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62, + 66, 70, 74, 78, 82, 86, 90, 94, 98, 102, 106, 110, 114, 118, 122, 126, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + const __m512i zmm_perm = _mm512_loadu_si512((const __m512i*)kPermAYUVToY32); + while (width >= 32) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_ayuv); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_ayuv + 64)); + src_ayuv += 128; + __m512i out = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); + _mm256_storeu_si256((__m256i*)dst_y, _mm512_castsi512_si256(out)); + dst_y += 32; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k1 = (__mmask16)m32; + __mmask16 k2 = (__mmask16)(m32 >> 16); + __m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_ayuv); + __m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_ayuv + 64); + __m512i out = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); + _mm256_mask_storeu_epi8(dst_y, (__mmask32)m32, _mm512_castsi512_si256(out)); + } + _mm256_zeroupper(); +} +#endif // HAS_AYUVTOYROW_AVX512VBMI + +#if defined(HAS_AYUVTOUVROW_AVX512VBMI) || defined(HAS_AYUVTOVUROW_AVX512VBMI) +static const uint8_t kPermAYUVToUV_Win[64] = { + 1, 5, 0, 4, 9, 13, 8, 12, 17, 21, 16, 20, 25, + 29, 24, 28, 33, 37, 32, 36, 41, 45, 40, 44, 49, 53, + 48, 52, 57, 61, 56, 60, 65, 69, 64, 68, 73, 77, 72, + 76, 81, 85, 80, 84, 89, 93, 88, 92, 97, 101, 96, 100, + 105, 109, 104, 108, 113, 117, 112, 116, 121, 125, 120, 124}; + +static const uint8_t kPermAYUVToVU_Win[64] = { + 0, 4, 1, 5, 8, 12, 9, 13, 16, 20, 17, 21, 24, + 28, 25, 29, 32, 36, 33, 37, 40, 44, 41, 45, 48, 52, + 49, 53, 56, 60, 57, 61, 64, 68, 65, 69, 72, 76, 73, + 77, 80, 84, 81, 85, 88, 92, 89, 93, 96, 100, 97, 101, + 104, 108, 105, 109, 112, 116, 113, 117, 120, 124, 121, 125}; + +LIBYUV_TARGET_AVX512VBMI +static inline void AYUVToUVOrVURow_Win_AVX512VBMI(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + const uint8_t* perm_table, + int width) { + const __m512i zmm_perm = _mm512_loadu_si512((const __m512i*)perm_table); + const __m512i zmm_0101 = _mm512_set1_epi16(0x0101); + const __m512i zmm_zero = _mm512_setzero_si512(); + while (width >= 32) { + __m512i s00 = _mm512_loadu_si512((const __m512i*)src_ayuv); + __m512i s01 = _mm512_loadu_si512((const __m512i*)(src_ayuv + 64)); + __m512i s10 = + _mm512_loadu_si512((const __m512i*)(src_ayuv + src_stride_ayuv)); + __m512i s11 = + _mm512_loadu_si512((const __m512i*)(src_ayuv + src_stride_ayuv + 64)); + src_ayuv += 128; + __m512i p0 = _mm512_permutex2var_epi8(s00, zmm_perm, s01); + __m512i p1 = _mm512_permutex2var_epi8(s10, zmm_perm, s11); + p0 = _mm512_maddubs_epi16(p0, zmm_0101); + p1 = _mm512_maddubs_epi16(p1, zmm_0101); + p0 = _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(p0, p1), 1), + zmm_zero); + _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(p0)); + dst_uv += 32; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k1 = (__mmask16)m32; + __mmask16 k2 = (__mmask16)(m32 >> 16); + __m512i s00 = _mm512_maskz_loadu_epi32(k1, src_ayuv); + __m512i s01 = _mm512_maskz_loadu_epi32(k2, src_ayuv + 64); + __m512i s10 = _mm512_maskz_loadu_epi32(k1, src_ayuv + src_stride_ayuv); + __m512i s11 = _mm512_maskz_loadu_epi32(k2, src_ayuv + src_stride_ayuv + 64); + __m512i p0 = _mm512_permutex2var_epi8(s00, zmm_perm, s01); + __m512i p1 = _mm512_permutex2var_epi8(s10, zmm_perm, s11); + __mmask16 k_odd = (__mmask16)((width & 1) << (width >> 1)); + p0 = _mm512_mask_or_epi32(p0, k_odd, p0, _mm512_slli_epi32(p0, 8)); + p1 = _mm512_mask_or_epi32(p1, k_odd, p1, _mm512_slli_epi32(p1, 8)); + p0 = _mm512_maddubs_epi16(p0, zmm_0101); + p1 = _mm512_maddubs_epi16(p1, zmm_0101); + p0 = _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(p0, p1), 1), + zmm_zero); + uint32_t out_bytes = (uint32_t)((width + 1) & ~1); + __mmask32 k_out = + out_bytes >= 32 ? ~0u : (__mmask32)((1u << out_bytes) - 1u); + _mm256_mask_storeu_epi8(dst_uv, k_out, _mm512_cvtepi16_epi8(p0)); + } + _mm256_zeroupper(); +} +#endif + +#ifdef HAS_AYUVTOUVROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI +void AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + AYUVToUVOrVURow_Win_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_uv, + kPermAYUVToUV_Win, width); +} +#endif // HAS_AYUVTOUVROW_AVX512VBMI + +#ifdef HAS_AYUVTOVUROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI +void AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + AYUVToUVOrVURow_Win_AVX512VBMI(src_ayuv, src_stride_ayuv, dst_vu, + kPermAYUVToVU_Win, width); +} +#endif // HAS_AYUVTOVUROW_AVX512VBMI + +// =========================================================================== +// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX512BW) +// =========================================================================== + +#ifdef HAS_AR30TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width) { + const __m512i m0 = _mm512_set1_epi32((int)0xc0000000u); + const __m512i m1 = _mm512_set1_epi32(0x300000ff); + const __m512i m2 = _mm512_set1_epi32(0x0c00ff00); + const __m512i m3 = _mm512_set1_epi32(0x03ff0000); + while (width >= 16) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_ar30); + src_ar30 += 64; + __m512i out = _mm512_or_si512( + _mm512_or_si512(_mm512_and_si512(v, m0), + _mm512_and_si512(_mm512_srli_epi32(v, 2), m1)), + _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2), + _mm512_and_si512(_mm512_srli_epi32(v, 6), m3))); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi32(k, src_ar30); + __m512i out = _mm512_or_si512( + _mm512_or_si512(_mm512_and_si512(v, m0), + _mm512_and_si512(_mm512_srli_epi32(v, 2), m1)), + _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2), + _mm512_and_si512(_mm512_srli_epi32(v, 6), m3))); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_AR30TOARGBROW_AVX512BW + +#ifdef HAS_AR30TOABGRROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width) { + static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7, + 10, 9, 8, 11, 14, 13, 12, 15}; + const __m512i m0 = _mm512_set1_epi32((int)0xc0000000u); + const __m512i m1 = _mm512_set1_epi32(0x300000ff); + const __m512i m2 = _mm512_set1_epi32(0x0c00ff00); + const __m512i m3 = _mm512_set1_epi32(0x03ff0000); + const __m512i shuf = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufSwapRB16)); + while (width >= 16) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_ar30); + src_ar30 += 64; + __m512i out = _mm512_or_si512( + _mm512_or_si512(_mm512_and_si512(v, m0), + _mm512_and_si512(_mm512_srli_epi32(v, 2), m1)), + _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2), + _mm512_and_si512(_mm512_srli_epi32(v, 6), m3))); + out = _mm512_shuffle_epi8(out, shuf); + _mm512_storeu_si512((__m512i*)dst_abgr, out); + dst_abgr += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi32(k, src_ar30); + __m512i out = _mm512_or_si512( + _mm512_or_si512(_mm512_and_si512(v, m0), + _mm512_and_si512(_mm512_srli_epi32(v, 2), m1)), + _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi32(v, 4), m2), + _mm512_and_si512(_mm512_srli_epi32(v, 6), m3))); + out = _mm512_shuffle_epi8(out, shuf); + _mm512_mask_storeu_epi32(dst_abgr, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_AR30TOABGRROW_AVX512BW + +#ifdef HAS_AR30TOAB30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width) { + const __m512i m0 = _mm512_set1_epi32((int)0xc00ffc00u); + const __m512i m1 = _mm512_set1_epi32(0x3ff00000); + const __m512i m2 = _mm512_set1_epi32(0x000003ff); + while (width >= 16) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_ar30); + src_ar30 += 64; + __m512i out = _mm512_or_si512( + _mm512_and_si512(v, m0), + _mm512_or_si512(_mm512_and_si512(_mm512_slli_epi32(v, 20), m1), + _mm512_and_si512(_mm512_srli_epi32(v, 20), m2))); + _mm512_storeu_si512((__m512i*)dst_ab30, out); + dst_ab30 += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi32(k, src_ar30); + __m512i out = _mm512_or_si512( + _mm512_and_si512(v, m0), + _mm512_or_si512(_mm512_and_si512(_mm512_slli_epi32(v, 20), m1), + _mm512_and_si512(_mm512_srli_epi32(v, 20), m2))); + _mm512_mask_storeu_epi32(dst_ab30, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_AR30TOAB30ROW_AVX512BW + +#ifdef HAS_MERGEXR30ROW_10_AVX512BW +LIBYUV_TARGET_AVX512BW +void MergeXR30Row_10_AVX512BW(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + uint8_t* dst_ar30, + int depth, + int width) { + (void)depth; + const __m512i max10 = _mm512_set1_epi32(1023); + const __m512i alpha = _mm512_set1_epi32((int)0xc0000000u); + while (width >= 16) { + __m512i r = _mm512_min_epu32( + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_r)), + max10); + __m512i g = _mm512_min_epu32( + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_g)), + max10); + __m512i b = _mm512_min_epu32( + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_b)), + max10); + src_r += 16; + src_g += 16; + src_b += 16; + __m512i out = _mm512_or_si512( + _mm512_or_si512(_mm512_slli_epi32(r, 20), _mm512_slli_epi32(g, 10)), + _mm512_or_si512(b, alpha)); + _mm512_storeu_si512((__m512i*)dst_ar30, out); + dst_ar30 += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i r = _mm512_min_epu32( + _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_r)), max10); + __m512i g = _mm512_min_epu32( + _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_g)), max10); + __m512i b = _mm512_min_epu32( + _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_b)), max10); + __m512i out = _mm512_or_si512( + _mm512_or_si512(_mm512_slli_epi32(r, 20), _mm512_slli_epi32(g, 10)), + _mm512_or_si512(b, alpha)); + _mm512_mask_storeu_epi32(dst_ar30, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_MERGEXR30ROW_10_AVX512BW + +#if defined(HAS_ARGBTOAR30ROW_AVX512BW) || defined(HAS_ABGRTOAR30ROW_AVX512BW) +static const uint8_t kShuffleRB30_Win[16] = {128u, 0u, 128u, 2u, 128u, 4u, + 128u, 6u, 128u, 8u, 128u, 10u, + 128u, 12u, 128u, 14u}; + +static const uint8_t kShuffleBR30_Win[16] = {128u, 2u, 128u, 0u, 128u, 6u, + 128u, 4u, 128u, 10u, 128u, 8u, + 128u, 14u, 128u, 12u}; + +LIBYUV_TARGET_AVX512BW +static inline void ARGBOrABGRToAR30Row_Win_AVX512BW(const uint8_t* src, + uint8_t* dst, + const uint8_t* shuf_rb, + int width) { + const __m512i zmm_shuf = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuf_rb)); + const __m512i zmm_mul_rb = _mm512_set1_epi32(1028 * 16 * 65536 + 1028); + const __m512i zmm_mask_rb = _mm512_set1_epi32(0x3ff003ff); + const __m512i zmm_mask_ag = _mm512_set1_epi32((int)0xc000ff00u); + const __m512i zmm_mul_ag = _mm512_set1_epi32(64 * 65536 + 1028); + while (width >= 16) { + __m512i v = _mm512_loadu_si512((const __m512i*)src); + src += 64; + __m512i rb = _mm512_and_si512( + _mm512_mulhi_epu16(_mm512_shuffle_epi8(v, zmm_shuf), zmm_mul_rb), + zmm_mask_rb); + __m512i ag = _mm512_slli_epi32( + _mm512_mulhi_epu16(_mm512_and_si512(v, zmm_mask_ag), zmm_mul_ag), 10); + _mm512_storeu_si512((__m512i*)dst, _mm512_or_si512(ag, rb)); + dst += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi32(k, src); + __m512i rb = _mm512_and_si512( + _mm512_mulhi_epu16(_mm512_shuffle_epi8(v, zmm_shuf), zmm_mul_rb), + zmm_mask_rb); + __m512i ag = _mm512_slli_epi32( + _mm512_mulhi_epu16(_mm512_and_si512(v, zmm_mask_ag), zmm_mul_ag), 10); + _mm512_mask_storeu_epi32(dst, k, _mm512_or_si512(ag, rb)); + } + _mm256_zeroupper(); +} +#endif + +#ifdef HAS_ARGBTOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + ARGBOrABGRToAR30Row_Win_AVX512BW(src, dst, kShuffleRB30_Win, width); +} +#endif // HAS_ARGBTOAR30ROW_AVX512BW + +#ifdef HAS_ABGRTOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ABGRToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + ARGBOrABGRToAR30Row_Win_AVX512BW(src, dst, kShuffleBR30_Win, width); +} +#endif // HAS_ABGRTOAR30ROW_AVX512BW + +// =========================================================================== +// Module 3: Planar / Channel Pack & Mirror / Interpolate_16 (AVX512VBMI / BW) +// =========================================================================== + +#ifdef HAS_MERGERGBROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI +void MergeRGBRow_AVX512VBMI(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width) { + static const uint8_t kPermMergeRGB64_0[64] = { + 0x00, 0x20, 0x40, 0x01, 0x21, 0x41, 0x02, 0x22, 0x42, 0x03, 0x23, + 0x43, 0x04, 0x24, 0x44, 0x05, 0x25, 0x45, 0x06, 0x26, 0x46, 0x07, + 0x27, 0x47, 0x08, 0x28, 0x48, 0x09, 0x29, 0x49, 0x0a, 0x2a, 0x4a, + 0x0b, 0x2b, 0x4b, 0x0c, 0x2c, 0x4c, 0x0d, 0x2d, 0x4d, 0x0e, 0x2e, + 0x4e, 0x0f, 0x2f, 0x4f, 0x10, 0x30, 0x50, 0x11, 0x31, 0x51, 0x12, + 0x32, 0x52, 0x13, 0x33, 0x53, 0x14, 0x34, 0x54, 0x15}; + static const uint8_t kPermMergeRGB64_1[64] = { + 0x35, 0x55, 0x16, 0x36, 0x56, 0x17, 0x37, 0x57, 0x18, 0x38, 0x58, + 0x19, 0x39, 0x59, 0x1a, 0x3a, 0x5a, 0x1b, 0x3b, 0x5b, 0x1c, 0x3c, + 0x5c, 0x1d, 0x3d, 0x5d, 0x1e, 0x3e, 0x5e, 0x1f, 0x3f, 0x5f, 0x60, + 0x00, 0x20, 0x61, 0x01, 0x21, 0x62, 0x02, 0x22, 0x63, 0x03, 0x23, + 0x64, 0x04, 0x24, 0x65, 0x05, 0x25, 0x66, 0x06, 0x26, 0x67, 0x07, + 0x27, 0x68, 0x08, 0x28, 0x69, 0x09, 0x29, 0x6a, 0x0a}; + static const uint8_t kPermMergeRGB64_2[64] = { + 0x6a, 0x2b, 0x4b, 0x6b, 0x2c, 0x4c, 0x6c, 0x2d, 0x4d, 0x6d, 0x2e, + 0x4e, 0x6e, 0x2f, 0x4f, 0x6f, 0x30, 0x50, 0x70, 0x31, 0x51, 0x71, + 0x32, 0x52, 0x72, 0x33, 0x53, 0x73, 0x34, 0x54, 0x74, 0x35, 0x55, + 0x75, 0x36, 0x56, 0x76, 0x37, 0x57, 0x77, 0x38, 0x58, 0x78, 0x39, + 0x59, 0x79, 0x3a, 0x5a, 0x7a, 0x3b, 0x5b, 0x7b, 0x3c, 0x5c, 0x7c, + 0x3d, 0x5d, 0x7d, 0x3e, 0x5e, 0x7e, 0x3f, 0x5f, 0x7f}; + const __m512i m_rgb0 = _mm512_loadu_si512((const __m512i*)kPermMergeRGB64_0); + const __m512i m_rgb1 = _mm512_loadu_si512((const __m512i*)kPermMergeRGB64_1); + const __m512i m_rgb2 = _mm512_loadu_si512((const __m512i*)kPermMergeRGB64_2); + while (width >= 64) { + __m512i rg0 = _mm512_inserti64x4( + _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_r)), + _mm256_loadu_si256((const __m256i*)src_g), 1); + __m512i br = _mm512_inserti64x4( + _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_b)), + _mm256_loadu_si256((const __m256i*)(src_r + 32)), 1); + __m512i gb1 = _mm512_inserti64x4( + _mm512_castsi256_si512( + _mm256_loadu_si256((const __m256i*)(src_g + 32))), + _mm256_loadu_si256((const __m256i*)(src_b + 32)), 1); + src_r += 64; + src_g += 64; + src_b += 64; + __m512i d0 = _mm512_permutex2var_epi8(rg0, m_rgb0, br); + __m512i d1 = + _mm512_mask_permutexvar_epi8(_mm512_permutex2var_epi8(rg0, m_rgb1, br), + 0xb6db6db600000000ull, m_rgb1, gb1); + __m512i d2 = _mm512_permutex2var_epi8(br, m_rgb2, gb1); + _mm512_storeu_si512((__m512i*)dst_rgb, d0); + _mm512_storeu_si512((__m512i*)(dst_rgb + 64), d1); + _mm512_storeu_si512((__m512i*)(dst_rgb + 128), d2); + dst_rgb += 192; + width -= 64; + } + if (width >= 32) { + __m512i rg = _mm512_inserti64x4( + _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_r)), + _mm256_loadu_si256((const __m256i*)src_g), 1); + __m512i b = + _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)src_b)); + src_r += 32; + src_g += 32; + src_b += 32; + __m512i d0 = _mm512_permutex2var_epi8(rg, m_rgb0, b); + __m512i d1 = _mm512_permutex2var_epi8(rg, m_rgb1, b); + _mm512_storeu_si512((__m512i*)dst_rgb, d0); + _mm256_storeu_si256((__m256i*)(dst_rgb + 64), _mm512_castsi512_si256(d1)); + dst_rgb += 96; + width -= 32; + } + if (width > 0) { + __mmask32 k_in = (__mmask32)((1u << width) - 1u); + __m512i rg = _mm512_inserti64x4( + _mm512_castsi256_si512(_mm256_maskz_loadu_epi8(k_in, src_r)), + _mm256_maskz_loadu_epi8(k_in, src_g), 1); + __m512i b = _mm512_castsi256_si512(_mm256_maskz_loadu_epi8(k_in, src_b)); + __m512i d0 = _mm512_permutex2var_epi8(rg, m_rgb0, b); + __m512i d1 = _mm512_permutex2var_epi8(rg, m_rgb1, b); + int dst_bytes = width * 3; + __mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; + __mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; + _mm512_mask_storeu_epi8(dst_rgb, k0, d0); + _mm256_mask_storeu_epi8(dst_rgb + 64, k1, _mm512_castsi512_si256(d1)); + } + _mm256_zeroupper(); +} +#endif // HAS_MERGERGBROW_AVX512VBMI + +#ifdef HAS_RAWTORGBAROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI +void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width) { + static const uint8_t kPermRAWToRGBA0[64] = { + 0, 2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6, 0, 11, 10, 9, + 0, 14, 13, 12, 0, 17, 16, 15, 0, 20, 19, 18, 0, 23, 22, 21, + 0, 26, 25, 24, 0, 29, 28, 27, 0, 32, 31, 30, 0, 35, 34, 33, + 0, 38, 37, 36, 0, 41, 40, 39, 0, 44, 43, 42, 0, 47, 46, 45}; + static const uint8_t kPermRAWToRGBA1[64] = { + 0, 50, 49, 48, 0, 53, 52, 51, 0, 56, 55, 54, 0, 59, 58, 57, + 0, 62, 61, 60, 0, 65, 64, 63, 0, 68, 67, 66, 0, 71, 70, 69, + 0, 74, 73, 72, 0, 77, 76, 75, 0, 80, 79, 78, 0, 83, 82, 81, + 0, 86, 85, 84, 0, 89, 88, 87, 0, 92, 91, 90, 0, 95, 94, 93}; + const __m512i p0 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGBA0); + const __m512i p1 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGBA1); + const __m512i alpha = _mm512_set1_epi32(0x000000ff); + while (width >= 32) { + __m512i s0 = _mm512_loadu_si512((const __m512i*)src_raw); + __m512i s1 = _mm512_castsi256_si512( + _mm256_loadu_si256((const __m256i*)(src_raw + 64))); + src_raw += 96; + __m512i d0 = _mm512_or_si512(_mm512_permutexvar_epi8(p0, s0), alpha); + __m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha); + _mm512_storeu_si512((__m512i*)dst_rgba, d0); + _mm512_storeu_si512((__m512i*)(dst_rgba + 64), d1); + dst_rgba += 128; + width -= 32; + } + if (width > 0) { + int src_bytes = width * 3; + __mmask64 ks0 = src_bytes >= 64 ? ~0ull : (1ull << src_bytes) - 1ull; + __mmask32 ks1 = src_bytes > 64 ? (1u << (src_bytes - 64)) - 1u : 0u; + __m512i s0 = _mm512_maskz_loadu_epi8(ks0, src_raw); + __m512i s1 = + _mm512_castsi256_si512(_mm256_maskz_loadu_epi8(ks1, src_raw + 64)); + __m512i d0 = _mm512_or_si512(_mm512_permutexvar_epi8(p0, s0), alpha); + __m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha); + uint32_t m32 = (1u << width) - 1u; + _mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, d0); + _mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16), d1); + } + _mm256_zeroupper(); +} +#endif // HAS_RAWTORGBAROW_AVX512VBMI + +#ifdef HAS_SWAPUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void SwapUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_vu, int width) { + static const uint8_t kSwapUV16[16] = {1, 0, 3, 2, 5, 4, 7, 6, + 9, 8, 11, 10, 13, 12, 15, 14}; + const __m512i swap = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kSwapUV16)); + while (width >= 32) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_uv); + src_uv += 64; + _mm512_storeu_si512((__m512i*)dst_vu, _mm512_shuffle_epi8(v, swap)); + dst_vu += 64; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi16(k, src_uv); + _mm512_mask_storeu_epi16(dst_vu, k, _mm512_shuffle_epi8(v, swap)); + } + _mm256_zeroupper(); +} +#endif // HAS_SWAPUVROW_AVX512BW + +#ifdef HAS_MIRRORROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW +void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) { + static const uint16_t kMirror16Idx[32] = { + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + const __m512i rev = _mm512_loadu_si512((const __m512i*)kMirror16Idx); + src += width; + while (width >= 32) { + src -= 32; + __m512i v = _mm512_loadu_si512((const __m512i*)src); + _mm512_storeu_si512((__m512i*)dst, _mm512_permutexvar_epi16(rev, v)); + dst += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i idx = _mm512_sub_epi16(rev, _mm512_set1_epi16((short)(32 - width))); + __m512i v = _mm512_maskz_loadu_epi16(k, src - width); + _mm512_mask_storeu_epi16(dst, k, _mm512_permutexvar_epi16(idx, v)); + } + _mm256_zeroupper(); +} +#endif // HAS_MIRRORROW_16_AVX512BW + +#ifdef HAS_HALFMERGEUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void HalfMergeUVRow_AVX512BW(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width) { + const __m512i c0101 = _mm512_set1_epi16(0x0101); + const __m512i czero = _mm512_setzero_si512(); + const __m512i c00ff = _mm512_set1_epi16(0x00ff); + while (width >= 64) { + __m512i u0 = _mm512_loadu_si512((const __m512i*)src_u); + __m512i u1 = _mm512_loadu_si512((const __m512i*)(src_u + src_stride_u)); + __m512i v0 = _mm512_loadu_si512((const __m512i*)src_v); + __m512i v1 = _mm512_loadu_si512((const __m512i*)(src_v + src_stride_v)); + src_u += 64; + src_v += 64; + u0 = _mm512_avg_epu16( + _mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(u0, c0101), + _mm512_maddubs_epi16(u1, c0101)), + 1), + czero); + v0 = _mm512_avg_epu16( + _mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(v0, c0101), + _mm512_maddubs_epi16(v1, c0101)), + 1), + czero); + _mm512_storeu_si512((__m512i*)dst_uv, + _mm512_or_si512(u0, _mm512_slli_epi16(v0, 8))); + dst_uv += 64; + width -= 64; + } + if (width > 0) { + __mmask64 k = width >= 64 ? ~0ull : (1ull << width) - 1ull; + __m512i u0 = _mm512_maskz_loadu_epi8(k, src_u); + __m512i u1 = _mm512_maskz_loadu_epi8(k, src_u + src_stride_u); + __m512i v0 = _mm512_maskz_loadu_epi8(k, src_v); + __m512i v1 = _mm512_maskz_loadu_epi8(k, src_v + src_stride_v); + __mmask32 k_odd = (__mmask32)((uint32_t)(width & 1) << (width >> 1)); + u0 = _mm512_mask_mullo_epi16(u0, k_odd, _mm512_and_si512(u0, c00ff), c0101); + u1 = _mm512_mask_mullo_epi16(u1, k_odd, _mm512_and_si512(u1, c00ff), c0101); + v0 = _mm512_mask_mullo_epi16(v0, k_odd, _mm512_and_si512(v0, c00ff), c0101); + v1 = _mm512_mask_mullo_epi16(v1, k_odd, _mm512_and_si512(v1, c00ff), c0101); + u0 = _mm512_avg_epu16( + _mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(u0, c0101), + _mm512_maddubs_epi16(u1, c0101)), + 1), + czero); + v0 = _mm512_avg_epu16( + _mm512_srli_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(v0, c0101), + _mm512_maddubs_epi16(v1, c0101)), + 1), + czero); + int out_words = (width + 1) >> 1; + __mmask32 k_out = + out_words >= 32 ? ~0u : (__mmask32)((1u << out_words) - 1u); + _mm512_mask_storeu_epi16(dst_uv, k_out, + _mm512_or_si512(u0, _mm512_slli_epi16(v0, 8))); + } + _mm256_zeroupper(); +} +#endif // HAS_HALFMERGEUVROW_AVX512BW + +#ifdef HAS_INTERPOLATEROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW +void InterpolateRow_16_AVX512BW(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + int y1 = source_y_fraction; + int y0 = 256 - y1; + const uint16_t* src_ptr1 = src_ptr + src_stride; + __m512i zmm_y = _mm512_set1_epi32((y1 << 16) | y0); + __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + __m512i zmm_round = _mm512_set1_epi32(8388736); // 0x800000 + 128 + int i; + + if (y1 == 0) { + for (i = 0; i <= width - 32; i += 32) { + _mm512_storeu_si512((__m512i*)(dst_ptr + i), + _mm512_loadu_si512((const __m512i*)(src_ptr + i))); + } + if (i < width) { + __mmask32 mask = (__mmask32)((1u << (width - i)) - 1u); + _mm512_mask_storeu_epi16(dst_ptr + i, mask, + _mm512_maskz_loadu_epi16(mask, src_ptr + i)); + } + } else if (y1 == 128) { + for (i = 0; i <= width - 32; i += 32) { + __m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i)); + __m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i)); + _mm512_storeu_si512((__m512i*)(dst_ptr + i), + _mm512_avg_epu16(row0, row1)); + } + if (i < width) { + __mmask32 mask = (__mmask32)((1u << (width - i)) - 1u); + __m512i row0 = _mm512_maskz_loadu_epi16(mask, src_ptr + i); + __m512i row1 = _mm512_maskz_loadu_epi16(mask, src_ptr1 + i); + _mm512_mask_storeu_epi16(dst_ptr + i, mask, _mm512_avg_epu16(row0, row1)); + } + } else { + for (i = 0; i <= width - 32; i += 32) { + __m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i)); + __m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i)); + __m512i row0l = _mm512_unpacklo_epi16(row0, row1); + __m512i row0h = _mm512_unpackhi_epi16(row0, row1); + row0l = _mm512_sub_epi16(row0l, zmm_8000); + row0h = _mm512_sub_epi16(row0h, zmm_8000); + __m512i resl = _mm512_madd_epi16(row0l, zmm_y); + __m512i resh = _mm512_madd_epi16(row0h, zmm_y); + resl = _mm512_add_epi32(resl, zmm_round); + resh = _mm512_add_epi32(resh, zmm_round); + resl = _mm512_srai_epi32(resl, 8); + resh = _mm512_srai_epi32(resh, 8); + _mm512_storeu_si512((__m512i*)(dst_ptr + i), + _mm512_packus_epi32(resl, resh)); + } + if (i < width) { + __mmask32 mask = (__mmask32)((1u << (width - i)) - 1u); + __m512i row0 = _mm512_maskz_loadu_epi16(mask, src_ptr + i); + __m512i row1 = _mm512_maskz_loadu_epi16(mask, src_ptr1 + i); + __m512i row0l = _mm512_unpacklo_epi16(row0, row1); + __m512i row0h = _mm512_unpackhi_epi16(row0, row1); + row0l = _mm512_sub_epi16(row0l, zmm_8000); + row0h = _mm512_sub_epi16(row0h, zmm_8000); + __m512i resl = _mm512_madd_epi16(row0l, zmm_y); + __m512i resh = _mm512_madd_epi16(row0h, zmm_y); + resl = _mm512_add_epi32(resl, zmm_round); + resh = _mm512_add_epi32(resh, zmm_round); + resl = _mm512_srai_epi32(resl, 8); + resh = _mm512_srai_epi32(resh, 8); + _mm512_mask_storeu_epi16(dst_ptr + i, mask, + _mm512_packus_epi32(resl, resh)); + } + } + _mm256_zeroupper(); +} +#endif // HAS_INTERPOLATEROW_16_AVX512BW + +// =========================================================================== +// Module 4: NV12 / NV21 & I422Alpha to ARGB / RGB24 (AVX512BW / AVX512VBMI) +// =========================================================================== + +#if defined(HAS_NV12TOARGBROW_AVX512BW) || \ + defined(HAS_NV21TOARGBROW_AVX512BW) || \ + defined(HAS_NV12TORGB24ROW_AVX512VBMI) || \ + defined(HAS_NV21TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I422ALPHATOARGBROW_AVX512BW) +static const uint16_t kSplitQuadWords_Win[32] = { + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, + 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15}; +static const uint64_t kUnpermuteAVX512_Win[8] = {0, 4, 1, 5, 2, 6, 3, 7}; +#endif + +#ifdef HAS_NV12TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + const __m512i zmm_kUVToB = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); + const __m512i zmm_kUVToG = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); + const __m512i zmm_kUVToR = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); + const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); + const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); + const __m512i zmm_128 = _mm512_set1_epi8((char)0x80); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i zmm3 = _mm512_permutexvar_epi16( + zmm_quad, + _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf))); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + uv_buf += 32; + y_buf += 32; + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6), + zmm5); + __m512i zmm1 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6), + zmm5); + __m512i zmm2 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), + zmm5); + __m512i bg = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra)); + _mm512_storeu_si512((__m512i*)(dst_argb + 64), + _mm512_unpackhi_epi16(bg, ra)); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i zmm3 = _mm512_permutexvar_epi16( + zmm_quad, + _mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf))); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6), + zmm5); + __m512i zmm1 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6), + zmm5); + __m512i zmm2 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), + zmm5); + __m512i bg = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, + _mm512_unpacklo_epi16(bg, ra)); + _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), + _mm512_unpackhi_epi16(bg, ra)); + } + _mm256_zeroupper(); +} +#endif // HAS_NV12TOARGBROW_AVX512BW + +#ifdef HAS_NV21TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* vu_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + const __m512i zmm_kUVToB = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); + const __m512i zmm_kUVToG = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); + const __m512i zmm_kUVToR = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); + const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); + const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); + const __m512i zmm_128 = _mm512_set1_epi8((char)0x80); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i zmm3 = _mm512_permutexvar_epi16( + zmm_quad, + _mm512_castsi256_si512(_mm256_or_si256( + _mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8), + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8)))); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + vu_buf += 32; + y_buf += 32; + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6), + zmm5); + __m512i zmm1 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6), + zmm5); + __m512i zmm2 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), + zmm5); + __m512i bg = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra)); + _mm512_storeu_si512((__m512i*)(dst_argb + 64), + _mm512_unpackhi_epi16(bg, ra)); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i zmm3 = _mm512_permutexvar_epi16( + zmm_quad, + _mm512_castsi256_si512(_mm256_or_si256( + _mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8), + _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8)))); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6), + zmm5); + __m512i zmm1 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6), + zmm5); + __m512i zmm2 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), + zmm5); + __m512i bg = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, + _mm512_unpacklo_epi16(bg, ra)); + _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), + _mm512_unpackhi_epi16(bg, ra)); + } + _mm256_zeroupper(); +} +#endif // HAS_NV21TOARGBROW_AVX512BW + +#if defined(HAS_NV12TORGB24ROW_AVX512VBMI) || \ + defined(HAS_NV21TORGB24ROW_AVX512VBMI) +static const uint8_t kMaskBG_NV[64] = { + 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, + 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, + 0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, + 0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, + 0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, + 0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77}; +static const uint8_t kMaskDST0_NV[64] = { + 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, + 0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, + 0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, + 0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, + 0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, + 0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a}; +static const uint8_t kMaskDST1_NV[64] = { + 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, + 0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, + 0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +#endif + +#ifdef HAS_NV12TORGB24ROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI +void NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + const __m512i zmm_kUVToB = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); + const __m512i zmm_kUVToG = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); + const __m512i zmm_kUVToR = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); + const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); + const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); + const __m512i zmm_128 = _mm512_set1_epi8((char)0x80); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win); + const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_NV); + const __m512i zmm_mask_DST0 = + _mm512_loadu_si512((const __m512i*)kMaskDST0_NV); + const __m512i zmm_mask_DST1 = + _mm512_loadu_si512((const __m512i*)kMaskDST1_NV); + while (width >= 32) { + __m512i zmm3 = _mm512_permutexvar_epi16( + zmm_quad, + _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf))); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + uv_buf += 32; + y_buf += 32; + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6); + __m512i zmm1 = _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6); + __m512i zmm2 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm1 = _mm512_packus_epi16(zmm1, zmm1); + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); + _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); + _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), + _mm512_castsi512_si256(zmm_dst1)); + dst_rgb24 += 96; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i zmm3 = _mm512_permutexvar_epi16( + zmm_quad, + _mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf))); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6); + __m512i zmm1 = _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6); + __m512i zmm2 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm1 = _mm512_packus_epi16(zmm1, zmm1); + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); + int dst_bytes = width * 3; + __mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; + __mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; + _mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0); + _mm256_mask_storeu_epi8(dst_rgb24 + 64, k1, + _mm512_castsi512_si256(zmm_dst1)); + } + _mm256_zeroupper(); +} +#endif // HAS_NV12TORGB24ROW_AVX512VBMI + +#ifdef HAS_NV21TORGB24ROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI +void NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* vu_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + const __m512i zmm_kUVToB = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); + const __m512i zmm_kUVToG = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); + const __m512i zmm_kUVToR = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); + const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); + const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); + const __m512i zmm_128 = _mm512_set1_epi8((char)0x80); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win); + const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_NV); + const __m512i zmm_mask_DST0 = + _mm512_loadu_si512((const __m512i*)kMaskDST0_NV); + const __m512i zmm_mask_DST1 = + _mm512_loadu_si512((const __m512i*)kMaskDST1_NV); + while (width >= 32) { + __m512i zmm3 = _mm512_permutexvar_epi16( + zmm_quad, + _mm512_castsi256_si512(_mm256_or_si256( + _mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8), + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8)))); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + vu_buf += 32; + y_buf += 32; + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6); + __m512i zmm1 = _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6); + __m512i zmm2 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm1 = _mm512_packus_epi16(zmm1, zmm1); + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); + _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); + _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), + _mm512_castsi512_si256(zmm_dst1)); + dst_rgb24 += 96; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i zmm3 = _mm512_permutexvar_epi16( + zmm_quad, + _mm512_castsi256_si512(_mm256_or_si256( + _mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8), + _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8)))); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6); + __m512i zmm1 = _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6); + __m512i zmm2 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm1 = _mm512_packus_epi16(zmm1, zmm1); + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); + int dst_bytes = width * 3; + __mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; + __mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; + _mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0); + _mm256_mask_storeu_epi8(dst_rgb24 + 64, k1, + _mm512_castsi512_si256(zmm_dst1)); + } + _mm256_zeroupper(); +} +#endif // HAS_NV21TORGB24ROW_AVX512VBMI + +#ifdef HAS_I422ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + const __m512i zmm_kUVToB = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); + const __m512i zmm_kUVToG = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); + const __m512i zmm_kUVToR = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); + const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); + const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); + const __m512i zmm_128 = _mm512_set1_epi8((char)0x80); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_Win); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win); + ptrdiff_t offset = v_buf - u_buf; + while (width >= 32) { + __m128i xu = _mm_loadu_si128((const __m128i*)u_buf); + __m128i xv = _mm_loadu_si128((const __m128i*)(u_buf + offset)); + u_buf += 16; + __m256i yuv = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_castsi256_si512(yuv)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + __m512i zmm5 = _mm512_packus_epi16( + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)a_buf)), + _mm512_setzero_si512()); + y_buf += 32; + a_buf += 32; + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6), + zmm5); + __m512i zmm1 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6), + zmm5); + __m512i zmm2 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), + zmm5); + __m512i bg = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra)); + _mm512_storeu_si512((__m512i*)(dst_argb + 64), + _mm512_unpackhi_epi16(bg, ra)); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf); + __m128i xv = _mm_maskz_loadu_epi8(k_uv, u_buf + offset); + __m256i yuv = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_castsi256_si512(yuv)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + __m512i zmm5 = _mm512_packus_epi16( + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, a_buf)), + _mm512_setzero_si512()); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), + zmm_kYBiasToRgb); + __m512i zmm0 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4), 6), + zmm5); + __m512i zmm1 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)), 6), + zmm5); + __m512i zmm2 = _mm512_packus_epi16( + _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), + zmm5); + __m512i bg = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, + _mm512_unpacklo_epi16(bg, ra)); + _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), + _mm512_unpackhi_epi16(bg, ra)); + } + _mm256_zeroupper(); +} +#endif // HAS_I422ALPHATOARGBROW_AVX512BW + +// =========================================================================== +// Module 5: I400ToARGB & ARGB Shaders (Attenuate, Multiply, Gray, Blend) +// =========================================================================== + +#ifdef HAS_I400TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I400ToARGBRow_AVX512BW(const uint8_t* y_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + static const uint64_t kLinearizePackQ[8] = {0, 2, 4, 6, 1, 3, 5, 7}; + static const uint8_t kBcastGrayByte[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, + 8, 8, 8, 0x80, 12, 12, 12, 0x80}; + const __m512i zmm_yg = + _mm512_set1_epi16(*(const int16_t*)yuvconstants->kYToRgb); + const __m512i zmm_ygb = + _mm512_set1_epi16(*(const int16_t*)yuvconstants->kYBiasToRgb); + const __m512i zmm_lin = _mm512_loadu_si512((const __m512i*)kLinearizePackQ); + const __m512i zmm_bcast = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kBcastGrayByte)); + const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u); + while (width >= 32) { + __m512i y16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + y_buf += 32; + y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8)); + y16 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6); + __m512i g = + _mm512_permutexvar_epi64(zmm_lin, _mm512_packus_epi16(y16, y16)); + __m256i g256 = _mm512_castsi512_si256(g); + __m512i d0 = _mm512_or_si512( + _mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)), + zmm_bcast), + zmm_alpha); + __m512i d1 = _mm512_or_si512( + _mm512_shuffle_epi8( + _mm512_cvtepu8_epi32(_mm256_extracti128_si256(g256, 1)), zmm_bcast), + zmm_alpha); + _mm512_storeu_si512((__m512i*)dst_argb, d0); + _mm512_storeu_si512((__m512i*)(dst_argb + 64), d1); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __m512i y16 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8)); + y16 = _mm512_srai_epi16( + _mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6); + __m512i g = + _mm512_permutexvar_epi64(zmm_lin, _mm512_packus_epi16(y16, y16)); + __m256i g256 = _mm512_castsi512_si256(g); + __m512i d0 = _mm512_or_si512( + _mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)), + zmm_bcast), + zmm_alpha); + __m512i d1 = _mm512_or_si512( + _mm512_shuffle_epi8( + _mm512_cvtepu8_epi32(_mm256_extracti128_si256(g256, 1)), zmm_bcast), + zmm_alpha); + _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, d0); + _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), d1); + } + _mm256_zeroupper(); +} +#endif // HAS_I400TOARGBROW_AVX512BW + +#ifdef HAS_ARGBATTENUATEROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const int8_t kAttenShuf16[16] = {6, -128, 6, -128, 6, -128, + -128, -128, 14, -128, 14, -128, + 14, -128, -128, -128}; + const __m512i zmm_shuf = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kAttenShuf16)); + const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u); + const __m512i zmm_bias = _mm512_set1_epi16(0x00ff); + while (width >= 16) { + __m512i s = _mm512_loadu_si512((const __m512i*)src_argb); + src_argb += 64; + __m512i lo = _mm512_unpacklo_epi8(s, zmm_alpha); + __m512i hi = _mm512_unpackhi_epi8(s, zmm_alpha); + lo = _mm512_srli_epi16( + _mm512_add_epi16( + _mm512_mullo_epi16(lo, _mm512_shuffle_epi8(lo, zmm_shuf)), + zmm_bias), + 8); + hi = _mm512_srli_epi16( + _mm512_add_epi16( + _mm512_mullo_epi16(hi, _mm512_shuffle_epi8(hi, zmm_shuf)), + zmm_bias), + 8); + __m512i out = _mm512_or_si512(_mm512_packus_epi16(lo, hi), + _mm512_and_si512(s, zmm_alpha)); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s = _mm512_maskz_loadu_epi32(k, src_argb); + __m512i lo = _mm512_unpacklo_epi8(s, zmm_alpha); + __m512i hi = _mm512_unpackhi_epi8(s, zmm_alpha); + lo = _mm512_srli_epi16( + _mm512_add_epi16( + _mm512_mullo_epi16(lo, _mm512_shuffle_epi8(lo, zmm_shuf)), + zmm_bias), + 8); + hi = _mm512_srli_epi16( + _mm512_add_epi16( + _mm512_mullo_epi16(hi, _mm512_shuffle_epi8(hi, zmm_shuf)), + zmm_bias), + 8); + __m512i out = _mm512_or_si512(_mm512_packus_epi16(lo, hi), + _mm512_and_si512(s, zmm_alpha)); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBATTENUATEROW_AVX512BW + +#ifdef HAS_ARGBMULTIPLYROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + const __m512i zero = _mm512_setzero_si512(); + while (width >= 16) { + __m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i s1 = _mm512_loadu_si512((const __m512i*)src_argb1); + src_argb += 64; + src_argb1 += 64; + __m512i lo = _mm512_mulhrs_epi16( + _mm512_srli_epi16(_mm512_unpacklo_epi8(zero, s0), 1), + _mm512_unpacklo_epi8(s1, zero)); + __m512i hi = _mm512_mulhrs_epi16( + _mm512_srli_epi16(_mm512_unpackhi_epi8(zero, s0), 1), + _mm512_unpackhi_epi8(s1, zero)); + _mm512_storeu_si512((__m512i*)dst_argb, _mm512_packus_epi16(lo, hi)); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s0 = _mm512_maskz_loadu_epi32(k, src_argb); + __m512i s1 = _mm512_maskz_loadu_epi32(k, src_argb1); + __m512i lo = _mm512_mulhrs_epi16( + _mm512_srli_epi16(_mm512_unpacklo_epi8(zero, s0), 1), + _mm512_unpacklo_epi8(s1, zero)); + __m512i hi = _mm512_mulhrs_epi16( + _mm512_srli_epi16(_mm512_unpackhi_epi8(zero, s0), 1), + _mm512_unpackhi_epi8(s1, zero)); + _mm512_mask_storeu_epi32(dst_argb, k, _mm512_packus_epi16(lo, hi)); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBMULTIPLYROW_AVX512BW + +#ifdef HAS_ARGBGRAYROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBGrayRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, + 8, 8, 8, 0x80, 12, 12, 12, 0x80}; + const __m512i c_coeff = _mm512_set1_epi32(0x004d961d); + const __m512i c_sub128 = _mm512_set1_epi8((char)0x80); + const __m512i c_one16 = _mm512_set1_epi16(1); + const __m512i c_bias = _mm512_set1_epi32(32896); + const __m512i c_shuf = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufGrayDword)); + const __m512i c_alpha = _mm512_set1_epi32((int)0xff000000u); + while (width >= 16) { + __m512i s = _mm512_loadu_si512((const __m512i*)src_argb); + src_argb += 64; + __m512i y16 = _mm512_maddubs_epi16(c_coeff, _mm512_sub_epi8(s, c_sub128)); + __m512i y32 = _mm512_srli_epi32( + _mm512_add_epi32(_mm512_madd_epi16(y16, c_one16), c_bias), 8); + __m512i out = _mm512_or_si512(_mm512_and_si512(s, c_alpha), + _mm512_shuffle_epi8(y32, c_shuf)); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s = _mm512_maskz_loadu_epi32(k, src_argb); + __m512i y16 = _mm512_maddubs_epi16(c_coeff, _mm512_sub_epi8(s, c_sub128)); + __m512i y32 = _mm512_srli_epi32( + _mm512_add_epi32(_mm512_madd_epi16(y16, c_one16), c_bias), 8); + __m512i out = _mm512_or_si512(_mm512_and_si512(s, c_alpha), + _mm512_shuffle_epi8(y32, c_shuf)); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBGRAYROW_AVX512BW + +#ifdef HAS_ARGBBLENDROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBBlendRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + static const uint8_t kShuffleAlpha16[16] = {3u, 0x80, 3u, 0x80, 7u, 0x80, + 7u, 0x80, 11u, 0x80, 11u, 0x80, + 15u, 0x80, 15u, 0x80}; + const __m512i zmm_shuf = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShuffleAlpha16)); + const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u); + const __m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00u); + const __m512i zmm_00ff = _mm512_set1_epi16(0x00ff); + const __m512i zmm_0001 = _mm512_set1_epi16(0x0001); + while (width >= 16) { + __m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i s1 = _mm512_loadu_si512((const __m512i*)src_argb1); + src_argb += 64; + src_argb1 += 64; + __m512i inv_a = _mm512_add_epi16( + _mm512_shuffle_epi8(_mm512_xor_si512(s0, zmm_alpha), zmm_shuf), + zmm_0001); + __m512i even = _mm512_srli_epi16( + _mm512_mullo_epi16(_mm512_and_si512(s1, zmm_00ff), inv_a), 8); + __m512i odd = _mm512_and_si512( + _mm512_mullo_epi16(_mm512_srli_epi16(s1, 8), inv_a), zmm_ff00); + __m512i out = _mm512_adds_epu8( + _mm512_adds_epu8(_mm512_or_si512(s0, zmm_alpha), even), odd); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s0 = _mm512_maskz_loadu_epi32(k, src_argb); + __m512i s1 = _mm512_maskz_loadu_epi32(k, src_argb1); + __m512i inv_a = _mm512_add_epi16( + _mm512_shuffle_epi8(_mm512_xor_si512(s0, zmm_alpha), zmm_shuf), + zmm_0001); + __m512i even = _mm512_srli_epi16( + _mm512_mullo_epi16(_mm512_and_si512(s1, zmm_00ff), inv_a), 8); + __m512i odd = _mm512_and_si512( + _mm512_mullo_epi16(_mm512_srli_epi16(s1, 8), inv_a), zmm_ff00); + __m512i out = _mm512_adds_epu8( + _mm512_adds_epu8(_mm512_or_si512(s0, zmm_alpha), even), odd); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBBLENDROW_AVX512BW + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale.cc b/source/scale.cc index ba06d42ea..98d417aa4 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -1275,6 +1275,11 @@ static int ScalePlaneBilinearDown_16(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_16_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_16_Any_NEON; @@ -1894,6 +1899,11 @@ static int ScalePlaneBilinearUp_16(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_16_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_16_Any_NEON; diff --git a/source/scale_common.cc b/source/scale_common.cc index 6c4c6dac1..f6633116f 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1523,6 +1523,11 @@ void ScalePlaneVertical_16(int src_height, } } #endif +#if defined(HAS_INTERPOLATEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_16_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_16_Any_NEON; diff --git a/unit_test/convert_test.cc b/unit_test/convert_test.cc index 337c77442..aaba43f29 100644 --- a/unit_test/convert_test.cc +++ b/unit_test/convert_test.cc @@ -880,6 +880,8 @@ TESTATOPLANAR(RAW, 3, 1, J444, 1, 1) TESTATOPLANAR(RGB24, 3, 1, I420, 2, 2) TESTATOPLANAR(RGB24, 3, 1, J420, 2, 2) TESTATOPLANAR(RGBA, 4, 1, I420, 2, 2) +TESTATOPLANAR(AR30, 4, 1, I420, 2, 2) +TESTATOPLANAR(AB30, 4, 1, I420, 2, 2) TESTATOPLANAR(UYVY, 2, 1, I420, 2, 2) TESTATOPLANAR(UYVY, 2, 1, I422, 2, 1) TESTATOPLANAR(YUY2, 2, 1, I420, 2, 2) diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 35dd3430c..861a3ebb9 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -855,6 +855,34 @@ TEST_F(LibYUVPlanarTest, ARGBMirror_Opt) { free_aligned_buffer_page_end(dst_pixels_c); } +TEST_F(LibYUVPlanarTest, MirrorPlane_16_Opt) { + align_buffer_page_end(src_pixels_8, benchmark_width_ * benchmark_height_ * 2); + align_buffer_page_end(dst_pixels_opt_8, + benchmark_width_ * benchmark_height_ * 2); + align_buffer_page_end(dst_pixels_c_8, + benchmark_width_ * benchmark_height_ * 2); + uint16_t* src_pixels = reinterpret_cast(src_pixels_8); + uint16_t* dst_pixels_opt = reinterpret_cast(dst_pixels_opt_8); + uint16_t* dst_pixels_c = reinterpret_cast(dst_pixels_c_8); + + MemRandomize(src_pixels_8, benchmark_width_ * benchmark_height_ * 2); + MaskCpuFlags(disable_cpu_flags_); + MirrorPlane_16(src_pixels, benchmark_width_, dst_pixels_c, benchmark_width_, + benchmark_width_, benchmark_height_); + MaskCpuFlags(benchmark_cpu_info_); + + for (int i = 0; i < benchmark_iterations_; ++i) { + MirrorPlane_16(src_pixels, benchmark_width_, dst_pixels_opt, + benchmark_width_, benchmark_width_, benchmark_height_); + } + for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) { + EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]); + } + free_aligned_buffer_page_end(src_pixels_8); + free_aligned_buffer_page_end(dst_pixels_opt_8); + free_aligned_buffer_page_end(dst_pixels_c_8); +} + TEST_F(LibYUVPlanarTest, MirrorPlane_Opt) { align_buffer_page_end(src_pixels, benchmark_width_ * benchmark_height_); align_buffer_page_end(dst_pixels_opt, benchmark_width_ * benchmark_height_);