diff --git a/README.chromium b/README.chromium index bf0c3c915..f1f089ed9 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1974 +Version: 1975 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/row.h b/include/libyuv/row.h index c7c67bd0f..bd748e2bf 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -415,6 +415,14 @@ extern "C" { #define HAS_ARGBBLENDROW_AVX512BW #define HAS_ARGBALPHABLENDROW_AVX512BW #define HAS_ARGBORROW_AVX512BW +#define HAS_ARGBTOYMATRIXROW_AVX512VNNI +#define HAS_ARGBTOUV444MATRIXROW_AVX512VNNI +#define HAS_ARGBTOUVMATRIXROW_AVX512VNNI +#define HAS_ARGBGRAYROW_AVX512VNNI +// AVX512VNNIVBMI requires both AVX512VNNI and AVX512VBMI. +#define HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI +#define HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI +#define HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI #endif #define HAS_ARGBALPHABLENDROW_SSSE3 #define HAS_ARGBORROW_SSE2 @@ -520,6 +528,14 @@ extern "C" { #define HAS_ARGBBLENDROW_AVX512BW #define HAS_ARGBALPHABLENDROW_AVX512BW #define HAS_ARGBORROW_AVX512BW +#define HAS_ARGBTOYMATRIXROW_AVX512VNNI +#define HAS_ARGBTOUV444MATRIXROW_AVX512VNNI +#define HAS_ARGBTOUVMATRIXROW_AVX512VNNI +#define HAS_ARGBGRAYROW_AVX512VNNI +// AVX512VNNIVBMI requires both AVX512VNNI and AVX512VBMI. +#define HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI +#define HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI +#define HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI #endif // The following are available on Neon platforms: @@ -2243,6 +2259,18 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c); +void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); void ARGBToUV444MatrixRow_C(const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, @@ -2389,6 +2417,14 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_y, int width, const struct ArgbConstants* c); +void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); void ARGBToYMatrixRow_Any_SSSE3(const uint8_t* src_argb, uint8_t* dst_y, int width, @@ -2401,6 +2437,14 @@ void ARGBToYMatrixRow_Any_AVX512BW(const uint8_t* src_argb, uint8_t* dst_y, int width, const struct ArgbConstants* c); +void ARGBToYMatrixRow_Any_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void ARGBToYMatrixRow_Any_AVX512VNNIVBMI(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); void RGBToYMatrixRow_NEON(const uint8_t* src_rgb, uint8_t* dst_y, @@ -2545,6 +2589,16 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c); +void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); void RGBToUV444MatrixRow_SSSE3(const uint8_t* src_rgb, uint8_t* dst_u, uint8_t* dst_v, @@ -2599,6 +2653,16 @@ void ARGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c); +void ARGBToUV444MatrixRow_Any_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); void RGBToUV444MatrixRow_Any_SSSE3(const uint8_t* src_rgb, uint8_t* dst_u, uint8_t* dst_v, @@ -7095,6 +7159,9 @@ void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, int width); +void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); void ARGBGrayRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_NEON_DotProd(const uint8_t* src_argb, uint8_t* dst_argb, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index 861435729..dc85ebbe1 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1974 +#define LIBYUV_VERSION 1975 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/convert.cc b/source/convert.cc index 79346f245..783071ce1 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -2904,6 +2904,22 @@ int ARGBToI420Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -3016,6 +3032,16 @@ int ARGBToI420Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -3146,6 +3172,22 @@ int ARGBToI420Alpha(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -3258,6 +3300,16 @@ int ARGBToI420Alpha(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -3515,6 +3567,22 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -3596,6 +3664,16 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON; diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 9c3d9d9dc..2337d54f7 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -84,6 +84,22 @@ int ARGBToI444Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -169,6 +185,22 @@ int ARGBToI444Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOUV444MATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON; @@ -286,6 +318,22 @@ int ARGBToI422Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -398,6 +446,16 @@ int ARGBToI422Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -495,6 +553,22 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -607,6 +681,16 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -797,6 +881,22 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -909,6 +1009,16 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -1093,6 +1203,22 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -1178,6 +1304,22 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOUV444MATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON; @@ -1366,6 +1508,22 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -1478,6 +1636,16 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -1654,6 +1822,22 @@ int ARGBToI400Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -1753,6 +1937,22 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -1813,6 +2013,16 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON; @@ -1953,6 +2163,22 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -2013,6 +2239,16 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON; @@ -2182,6 +2418,22 @@ int ARGBToP010Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -2294,6 +2546,16 @@ int ARGBToP010Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -2613,6 +2875,22 @@ int ARGBToP210Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -2725,6 +3003,16 @@ int ARGBToP210Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512VNNIVBMI; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; @@ -3029,6 +3317,22 @@ int ARGBToP410Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; @@ -3114,6 +3418,22 @@ int ARGBToP410Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNI; + if (IS_ALIGNED(width, 64)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI; + } + } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI; + if (IS_ALIGNED(width, 64)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI; + } + } +#endif #if defined(HAS_ARGBTOUV444MATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 4901686ad..dcf6b9227 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -4740,6 +4740,11 @@ int ARGBGrayTo(const uint8_t* src_argb, ARGBGrayRow = ARGBGrayRow_AVX512BW; } #endif +#if defined(HAS_ARGBGRAYROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBGrayRow = ARGBGrayRow_AVX512VNNI; + } +#endif #if defined(HAS_ARGBGRAYROW_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { ARGBGrayRow = ARGBGrayRow_NEON; @@ -4800,6 +4805,11 @@ int ARGBGray(uint8_t* dst_argb, ARGBGrayRow = ARGBGrayRow_AVX512BW; } #endif +#if defined(HAS_ARGBGRAYROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBGrayRow = ARGBGrayRow_AVX512VNNI; + } +#endif #if defined(HAS_ARGBGRAYROW_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { ARGBGrayRow = ARGBGrayRow_NEON; diff --git a/source/row_any.cc b/source/row_any.cc index 57806d43c..5dcf5325b 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -2544,6 +2544,18 @@ ANY12M(RGBToUV444MatrixRow_Any_AVX2, RGBToUV444MatrixRow_AVX2, 3, 31) #ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW ANY12M(ARGBToUV444MatrixRow_Any_AVX512BW, ARGBToUV444MatrixRow_AVX512BW, 4, 63) #endif +#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNI +ANY12M(ARGBToUV444MatrixRow_Any_AVX512VNNI, + ARGBToUV444MatrixRow_AVX512VNNI, + 4, + 63) +#endif +#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI +ANY12M(ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI, + ARGBToUV444MatrixRow_AVX512VNNIVBMI, + 4, + 63) +#endif #ifdef HAS_RGBTOUV444MATRIXROW_AVX512BW ANY12M(RGBToUV444MatrixRow_Any_AVX512BW, RGBToUV444MatrixRow_AVX512BW, 3, 63) #endif @@ -2634,6 +2646,15 @@ ANY11MC(ARGB4444ToYMatrixRow_Any_AVX2, ARGB4444ToYMatrixRow_AVX2, 2, 31) #ifdef HAS_ARGBTOYMATRIXROW_AVX512BW ANY11MC(ARGBToYMatrixRow_Any_AVX512BW, ARGBToYMatrixRow_AVX512BW, 4, 63) #endif +#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNI +ANY11MC(ARGBToYMatrixRow_Any_AVX512VNNI, ARGBToYMatrixRow_AVX512VNNI, 4, 63) +#endif +#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI +ANY11MC(ARGBToYMatrixRow_Any_AVX512VNNIVBMI, + ARGBToYMatrixRow_AVX512VNNIVBMI, + 4, + 63) +#endif #ifdef HAS_RGBTOYMATRIXROW_AVX512BW ANY11MC(RGBToYMatrixRow_Any_AVX512BW, RGBToYMatrixRow_AVX512BW, 3, 63) #endif diff --git a/source/row_gcc.cc b/source/row_gcc.cc index ee9f648ee..3296dad51 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -1910,21 +1910,31 @@ void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBTOYMATRIXROW_AVX2 -#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \ - defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \ - defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \ +#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) || \ + defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) || \ defined(HAS_RGBTOYMATRIXROW_AVX512BW) static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15}; #endif // defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || + // defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) || // defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || + // defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) || // defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || + // defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) || // defined(HAS_RGBTOYMATRIXROW_AVX512BW) -#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) +#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) static const uint32_t kPermdARGBToUV_AVX512BW[16] = { 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15}; -#endif // defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) +#endif // defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || + // defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) || + // defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) #ifdef HAS_ARGBTOYMATRIXROW_AVX512BW void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, @@ -1984,6 +1994,126 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBTOYMATRIXROW_AVX512BW +#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNI +void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + asm volatile( + "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" + "vpsllw $15,%%zmm16,%%zmm5 \n" + "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80 + "vbroadcasti32x4 0(%3),%%zmm4 \n" // kRGBToY (unsigned) + "vpmovzxwd 0x60(%3),%%zmm17 \n" // kAddY (32-bit) + "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" + "vpdpbusd %%zmm5,%%zmm4,%%zmm0 \n" // -128 * sum(c) + "vpsubd %%zmm0,%%zmm17,%%zmm17 \n" // kAddY + 128 * sum(c) + "vmovups %4,%%zmm16 \n" // kPermdARGBToY_AVX512BW + LABELALIGN + "1: \n" + "vmovups (%0),%%zmm0 \n" + "vmovups 0x40(%0),%%zmm1 \n" + "vmovups 0x80(%0),%%zmm2 \n" + "vmovups 0xc0(%0),%%zmm3 \n" + "lea 0x100(%0),%0 \n" + "vpxorq %%zmm5,%%zmm0,%%zmm0 \n" + "vpxorq %%zmm5,%%zmm1,%%zmm1 \n" + "vpxorq %%zmm5,%%zmm2,%%zmm2 \n" + "vpxorq %%zmm5,%%zmm3,%%zmm3 \n" + "vmovdqa64 %%zmm17,%%zmm18 \n" + "vmovdqa64 %%zmm17,%%zmm19 \n" + "vmovdqa64 %%zmm17,%%zmm20 \n" + "vmovdqa64 %%zmm17,%%zmm21 \n" + "vpdpbusd %%zmm0,%%zmm4,%%zmm18 \n" + "vpdpbusd %%zmm1,%%zmm4,%%zmm19 \n" + "vpdpbusd %%zmm2,%%zmm4,%%zmm20 \n" + "vpdpbusd %%zmm3,%%zmm4,%%zmm21 \n" + "vpackusdw %%zmm19,%%zmm18,%%zmm0 \n" + "vpackusdw %%zmm21,%%zmm20,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm16,%%zmm0 \n" + "vmovups %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(c), // %3 + "m"(kPermdARGBToY_AVX512BW) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); +} +#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNI + +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) || \ + defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) +// AVX512VNNIVBMI kernels use both AVX512VNNI (vpdpbusd) and AVX512VBMI +// (vpermt2b) and require both CPU features. There are no AVX512VBMI +// versions without AVX512VNNI. + +// vpermt2b indices selecting byte 1 of each int32 from 2 vectors. The 32 +// byte pattern is broadcast so each 256 bit half holds the same 32 results. +static const uint8_t kPermbDWordByte1_AVX512VNNIVBMI[32] = { + 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, + 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125}; +#endif + +#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI +// Requires AVX512VNNI and AVX512VBMI. +// The 32 bit sums are kAddY + dot(kRGBToY, argb) which is always in +// [0, 65535], so Y is byte 1 of each int32 and no saturation is needed. +void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + asm volatile( + "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" + "vpsllw $15,%%zmm16,%%zmm5 \n" + "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80 + "vbroadcasti32x4 0(%3),%%zmm4 \n" // kRGBToY (unsigned) + "vpmovzxwd 0x60(%3),%%zmm17 \n" // kAddY (32-bit) + "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" + "vpdpbusd %%zmm5,%%zmm4,%%zmm0 \n" // -128 * sum(c) + "vpsubd %%zmm0,%%zmm17,%%zmm17 \n" // kAddY + 128 * sum(c) + "vbroadcasti64x4 %4,%%zmm16 \n" // byte 1 of int32 + LABELALIGN + "1: \n" + "vpxorq (%0),%%zmm5,%%zmm0 \n" + "vmovdqa64 %%zmm17,%%zmm18 \n" + "vpdpbusd %%zmm0,%%zmm4,%%zmm18 \n" + "vpxorq 0x40(%0),%%zmm5,%%zmm1 \n" + "vmovdqa64 %%zmm17,%%zmm19 \n" + "vpdpbusd %%zmm1,%%zmm4,%%zmm19 \n" + "vpxorq 0x80(%0),%%zmm5,%%zmm2 \n" + "vmovdqa64 %%zmm17,%%zmm20 \n" + "vpdpbusd %%zmm2,%%zmm4,%%zmm20 \n" + "vpxorq 0xc0(%0),%%zmm5,%%zmm3 \n" + "vmovdqa64 %%zmm17,%%zmm21 \n" + "vpdpbusd %%zmm3,%%zmm4,%%zmm21 \n" + "lea 0x100(%0),%0 \n" + "vpermt2b %%zmm19,%%zmm16,%%zmm18 \n" // Y 0..31 + "vpermt2b %%zmm21,%%zmm16,%%zmm20 \n" // Y 32..63 + "vmovdqu64 %%ymm18,(%1) \n" + "vmovdqu64 %%ymm20,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(c), // %3 + "m"(kPermbDWordByte1_AVX512VNNIVBMI) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); +} +#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI + #ifdef HAS_RGBTOYMATRIXROW_AVX512BW void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, uint8_t* dst_y, @@ -2288,6 +2418,145 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW +#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNI +void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + asm volatile( + "vbroadcasti32x4 0x20(%4),%%zmm3 \n" // kRGBToU (signed) + "vbroadcasti32x4 0x40(%4),%%zmm4 \n" // kRGBToV (signed) + "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // -1 + "vpsllw $15,%%zmm16,%%zmm5 \n" // 0x8000 + "vmovups %5,%%zmm17 \n" + "sub %1,%2 \n" + + LABELALIGN + "1: \n" + "vmovups (%0),%%zmm18 \n" + "vmovups 0x40(%0),%%zmm19 \n" + "vmovups 0x80(%0),%%zmm20 \n" + "vmovups 0xc0(%0),%%zmm21 \n" + "lea 0x100(%0),%0 \n" + "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" + "vpxorq %%zmm1,%%zmm1,%%zmm1 \n" + "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" + "vpxorq %%zmm16,%%zmm16,%%zmm16 \n" + "vpdpbusd %%zmm3,%%zmm18,%%zmm0 \n" + "vpdpbusd %%zmm3,%%zmm19,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm20,%%zmm2 \n" + "vpdpbusd %%zmm3,%%zmm21,%%zmm16 \n" + "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates + "vpackssdw %%zmm16,%%zmm2,%%zmm2 \n" + "vpsubw %%zmm0,%%zmm5,%%zmm0 \n" + "vpsubw %%zmm2,%%zmm5,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates + "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate. + "vmovups %%zmm0,(%1) \n" + + "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" + "vpxorq %%zmm1,%%zmm1,%%zmm1 \n" + "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" + "vpxorq %%zmm16,%%zmm16,%%zmm16 \n" + "vpdpbusd %%zmm4,%%zmm18,%%zmm0 \n" + "vpdpbusd %%zmm4,%%zmm19,%%zmm1 \n" + "vpdpbusd %%zmm4,%%zmm20,%%zmm2 \n" + "vpdpbusd %%zmm4,%%zmm21,%%zmm16 \n" + "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates + "vpackssdw %%zmm16,%%zmm2,%%zmm2 \n" + "vpsubw %%zmm0,%%zmm5,%%zmm0 \n" + "vpsubw %%zmm2,%%zmm5,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates + "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate. + "vmovups %%zmm0,(%1,%2,1) \n" + "lea 0x40(%1),%1 \n" + "subl $0x40,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "r"(c), // %4 + "m"(kPermdARGBToY_AVX512BW) // %5 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); +} +#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNI + +#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI +// Requires AVX512VNNI and AVX512VBMI. +// U = (kAddUV - dot(kRGBToU, argb)) >> 8. The accumulators start at +// ~kAddUV so they hold ~(kAddUV - dot) and U is the inverse of byte 1 of each +// int32. The inverse is folded into the vpternlogd that merges the halves. +void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + asm volatile( + "vbroadcasti32x4 0x20(%4),%%zmm3 \n" // kRGBToU (signed) + "vbroadcasti32x4 0x40(%4),%%zmm4 \n" // kRGBToV (signed) + "vpmovzxwd 0x80(%4),%%zmm5 \n" // kAddUV (32-bit) + "vpternlogd $0x55,%%zmm5,%%zmm5,%%zmm5 \n" // ~kAddUV + "vbroadcasti64x4 %5,%%zmm16 \n" // byte 1 of int32 + "vpternlogd $0xff,%%zmm6,%%zmm6,%%zmm6 \n" + "vpxord %%zmm7,%%zmm7,%%zmm7 \n" + "vinserti64x4 $1,%%ymm6,%%zmm7,%%zmm6 \n" // high 256 bits select + "sub %1,%2 \n" + + LABELALIGN + "1: \n" + "vmovups (%0),%%zmm0 \n" + "vmovups 0x40(%0),%%zmm1 \n" + "vmovups 0x80(%0),%%zmm2 \n" + "vmovups 0xc0(%0),%%zmm7 \n" + "lea 0x100(%0),%0 \n" + "vmovdqa64 %%zmm5,%%zmm17 \n" + "vmovdqa64 %%zmm5,%%zmm18 \n" + "vmovdqa64 %%zmm5,%%zmm19 \n" + "vmovdqa64 %%zmm5,%%zmm20 \n" + "vmovdqa64 %%zmm5,%%zmm21 \n" + "vmovdqa64 %%zmm5,%%zmm22 \n" + "vmovdqa64 %%zmm5,%%zmm23 \n" + "vmovdqa64 %%zmm5,%%zmm24 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm17 \n" // U + "vpdpbusd %%zmm4,%%zmm0,%%zmm21 \n" // V + "vpdpbusd %%zmm3,%%zmm1,%%zmm18 \n" + "vpdpbusd %%zmm4,%%zmm1,%%zmm22 \n" + "vpdpbusd %%zmm3,%%zmm2,%%zmm19 \n" + "vpdpbusd %%zmm4,%%zmm2,%%zmm23 \n" + "vpdpbusd %%zmm3,%%zmm7,%%zmm20 \n" + "vpdpbusd %%zmm4,%%zmm7,%%zmm24 \n" + "vpermt2b %%zmm18,%%zmm16,%%zmm17 \n" // U 0..31 + "vpermt2b %%zmm20,%%zmm16,%%zmm19 \n" // U 32..63 + "vpermt2b %%zmm22,%%zmm16,%%zmm21 \n" // V 0..31 + "vpermt2b %%zmm24,%%zmm16,%%zmm23 \n" // V 32..63 + "vpternlogd $0x27,%%zmm6,%%zmm19,%%zmm17 \n" // ~(low 17, high 19) + "vpternlogd $0x27,%%zmm6,%%zmm23,%%zmm21 \n" // ~(low 21, high 23) + "vmovdqu64 %%zmm17,(%1) \n" + "vmovdqu64 %%zmm21,(%1,%2,1) \n" + "lea 0x40(%1),%1 \n" + "subl $0x40,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+rm"(width) // %3 + : "r"(c), // %4 + "m"(kPermbDWordByte1_AVX512VNNIVBMI) // %5 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", + "xmm23", "xmm24"); +} +#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI + #ifdef HAS_ARGBTOUVMATRIXROW_SSSE3 // ARGBARGB to AARRGGBB shuffle @@ -2590,6 +2859,280 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBTOUVMATRIXROW_AVX512BW +#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNI + +// 32x2 -> 16x1 ARGB pixels converted to 16 U and 16 V +// ARGBToUV does rounding average of 4 ARGB pixels + +void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + uintptr_t temp; + asm volatile( + "vbroadcasti32x4 0x20(%[c]),%%zmm4 \n" // RGBToU (signed) + "vbroadcasti32x4 0x40(%[c]),%%zmm5 \n" // RGBToV (signed) + "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" + "vpabsb %%zmm16,%%zmm20 \n" // 0x0101 + "vpsllw $15,%%zmm16,%%zmm17 \n" // 0x8000 + "vbroadcasti32x4 %[shuf],%%zmm21 \n" // kShuffleAARRGGBB + "vmovups %[permy],%%zmm18 \n" // kPermdARGBToY_AVX512BW + "vmovups %[permuv],%%zmm19 \n" // kPermdARGBToUV_AVX512BW + "sub %1,%2 \n" + "subl $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%0),%%zmm0 \n" // Read 32x2 ARGB Pixels + "vmovups 0x40(%0),%%zmm1 \n" + "vmovups 0x00(%0,%[stride],1),%%zmm2 \n" + "vmovups 0x40(%0,%[stride],1),%%zmm3 \n" + "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" // aarrggbb + "vpshufb %%zmm21,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm21,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm21,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" // 32x2 -> 16x2 + "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm0,%%zmm2,%%zmm0 \n" // 16x2 -> 16x1 + "vpaddw %%zmm1,%%zmm3,%%zmm1 \n" + "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" // 0 for vpavgw and + // vpdpbusd + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm1,%%zmm1 \n" + "vpavgw %%zmm2,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm2,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" // mutates + "vpermd %%zmm0,%%zmm19,%%zmm0 \n" // unscramble pixels + + "vpxorq %%zmm1,%%zmm1,%%zmm1 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm1 \n" // 16 U (negated) + "vpdpbusd %%zmm5,%%zmm0,%%zmm2 \n" // 16 V (negated) + "vpackssdw %%zmm2,%%zmm1,%%zmm0 \n" // mutates (U in lower, V + // in upper) + "vpsubw %%zmm0,%%zmm17,%%zmm0 \n" // 0x8000 - dot16 + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n" // mutates + "vpermd %%zmm0,%%zmm18,%%zmm0 \n" // unmutate + + "vmovdqu %%xmm0,(%1) \n" // Write 16 U's + "vextracti32x4 $0x1,%%zmm0,%%xmm0 \n" + "vmovdqu %%xmm0,0x00(%1,%2,1) \n" // Write 16 V's + + "lea 0x80(%0),%0 \n" + "lea 0x10(%1),%1 \n" + "subl $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "addl $0x20,%3 \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q3,%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "kshiftrd $16,%%k1,%%k2 \n" + "knotw %%k1,%%k3 \n" + "knotw %%k2,%%k4 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu32 0x00(%0,%[stride],1),%%zmm2%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%0,%[stride],1),%%zmm3%{%%k2%}%{z%} \n" + "vpshufd $0xa0,%%zmm0,%%zmm0%{%%k3%} \n" + "vpshufd $0xa0,%%zmm1,%%zmm1%{%%k4%} \n" + "vpshufd $0xa0,%%zmm2,%%zmm2%{%%k3%} \n" + "vpshufd $0xa0,%%zmm3,%%zmm3%{%%k4%} \n" + "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm21,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm21,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm21,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" + "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm0,%%zmm2,%%zmm0 \n" + "vpaddw %%zmm1,%%zmm3,%%zmm1 \n" + "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm1,%%zmm1 \n" + "vpavgw %%zmm2,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm2,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm19,%%zmm0 \n" + + "vpxorq %%zmm1,%%zmm1,%%zmm1 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm5,%%zmm0,%%zmm2 \n" + "vpackssdw %%zmm2,%%zmm1,%%zmm0 \n" + "vpsubw %%zmm0,%%zmm17,%%zmm0 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm18,%%zmm0 \n" + + "addl $1,%3 \n" + "shrl $1,%3 \n" + "mov $-1,%[temp] \n" + "bzhi %q3,%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu8 %%xmm0,(%1)%{%%k1%} \n" + "vextracti32x4 $0x1,%%zmm0,%%xmm0 \n" + "vmovdqu8 %%xmm0,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : [stride] "r"((ptrdiff_t)(src_stride_argb)), // %[stride] + [c] "r"(c), // %[c] + [shuf] "m"(kShuffleAARRGGBB), // %[shuf] + [permy] "m"(kPermdARGBToY_AVX512BW), // %[permy] + [permuv] "m"(kPermdARGBToUV_AVX512BW) // %[permuv] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4"); +} +#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNI + +#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI + +// Requires AVX512VNNI and AVX512VBMI. +// 32x2 -> 16x1 ARGB pixels converted to 16 U and 16 V +// ARGBToUV does rounding average of 4 ARGB pixels + +void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + uintptr_t temp; + asm volatile( + "vbroadcasti32x4 0x20(%[c]),%%zmm4 \n" // RGBToU (signed) + "vbroadcasti32x4 0x40(%[c]),%%zmm5 \n" // RGBToV (signed) + "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" + "vpabsb %%zmm16,%%zmm20 \n" // 0x0101 + "vpmovzxwd 0x80(%[c]),%%zmm17 \n" // kAddUV (32-bit) + "vpternlogd $0x55,%%zmm17,%%zmm17,%%zmm17 \n" // ~kAddUV + "vbroadcasti32x4 %[shuf],%%zmm21 \n" // kShuffleAARRGGBB + "vbroadcasti64x4 %[permb],%%zmm18 \n" // byte 1 of int32 + "vmovups %[permuv],%%zmm19 \n" // kPermdARGBToUV_AVX512BW + "sub %1,%2 \n" + "subl $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%0),%%zmm0 \n" // Read 32x2 ARGB Pixels + "vmovups 0x40(%0),%%zmm1 \n" + "vmovups 0x00(%0,%[stride],1),%%zmm2 \n" + "vmovups 0x40(%0,%[stride],1),%%zmm3 \n" + "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" // aarrggbb + "vpshufb %%zmm21,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm21,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm21,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" // 32x2 -> 16x2 + "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm0,%%zmm2,%%zmm0 \n" // 16x2 -> 16x1 + "vpaddw %%zmm1,%%zmm3,%%zmm1 \n" + "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" // 0 for vpavgw and + // vpdpbusd + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm1,%%zmm1 \n" + "vpavgw %%zmm2,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm2,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" // mutates + "vpermd %%zmm0,%%zmm19,%%zmm0 \n" // unscramble pixels + + "vmovdqa64 %%zmm17,%%zmm1 \n" // ~kAddUV + "vmovdqa64 %%zmm17,%%zmm3 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm1 \n" // ~(16 U) + "vpdpbusd %%zmm5,%%zmm0,%%zmm3 \n" // ~(16 V) + "vpermt2b %%zmm3,%%zmm18,%%zmm1 \n" // byte 1 of U and V + "vpternlogd $0x55,%%ymm1,%%ymm1,%%ymm1 \n" // invert + + "vmovdqu %%xmm1,(%1) \n" // Write 16 U's + "vextracti32x4 $0x1,%%ymm1,0x00(%1,%2,1) \n" // Write 16 V's + + "lea 0x80(%0),%0 \n" + "lea 0x10(%1),%1 \n" + "subl $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "addl $0x20,%3 \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q3,%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "kshiftrd $16,%%k1,%%k2 \n" + "knotw %%k1,%%k3 \n" + "knotw %%k2,%%k4 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu32 0x00(%0,%[stride],1),%%zmm2%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%0,%[stride],1),%%zmm3%{%%k2%}%{z%} \n" + "vpshufd $0xa0,%%zmm0,%%zmm0%{%%k3%} \n" + "vpshufd $0xa0,%%zmm1,%%zmm1%{%%k4%} \n" + "vpshufd $0xa0,%%zmm2,%%zmm2%{%%k3%} \n" + "vpshufd $0xa0,%%zmm3,%%zmm3%{%%k4%} \n" + "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm21,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm21,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm21,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" + "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm0,%%zmm2,%%zmm0 \n" + "vpaddw %%zmm1,%%zmm3,%%zmm1 \n" + "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm1,%%zmm1 \n" + "vpavgw %%zmm2,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm2,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm19,%%zmm0 \n" + + "vmovdqa64 %%zmm17,%%zmm1 \n" + "vmovdqa64 %%zmm17,%%zmm3 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm5,%%zmm0,%%zmm3 \n" + "vpermt2b %%zmm3,%%zmm18,%%zmm1 \n" + "vpternlogd $0x55,%%ymm1,%%ymm1,%%ymm1 \n" + + "addl $1,%3 \n" + "shrl $1,%3 \n" + "mov $-1,%[temp] \n" + "bzhi %q3,%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu8 %%xmm1,(%1)%{%%k1%} \n" + "vextracti32x4 $0x1,%%ymm1,%%xmm1 \n" + "vmovdqu8 %%xmm1,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : [stride] "r"((ptrdiff_t)(src_stride_argb)), // %[stride] + [c] "r"(c), // %[c] + [shuf] "m"(kShuffleAARRGGBB), // %[shuf] + [permb] "m"(kPermbDWordByte1_AVX512VNNIVBMI), // %[permb] + [permuv] "m"(kPermdARGBToUV_AVX512BW) // %[permuv] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4"); +} +#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI + #if defined(HAS_I422TOARGBROW_SSSE3) || defined(HAS_I422TOARGBROW_AVX2) // Read 8 UV from 444 @@ -13323,10 +13866,13 @@ void OMITFP ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBMULTIPLYROW_AVX512BW -#ifdef HAS_ARGBGRAYROW_AVX512BW +#if defined(HAS_ARGBGRAYROW_AVX512BW) || defined(HAS_ARGBGRAYROW_AVX512VNNI) static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, 8, 8, 8, 0x80, 12, 12, 12, 0x80}; +#endif // defined(HAS_ARGBGRAYROW_AVX512BW) || + // defined(HAS_ARGBGRAYROW_AVX512VNNI) +#ifdef HAS_ARGBGRAYROW_AVX512BW void OMITFP ARGBGrayRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, int width) { @@ -13392,6 +13938,64 @@ void OMITFP ARGBGrayRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBGRAYROW_AVX512BW +#ifdef HAS_ARGBGRAYROW_AVX512VNNI +void OMITFP ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const uint32_t kCoeff = 0x004d961du; // {29, 150, 77, 0} + static const uint32_t kBias = 32896u; + uintptr_t temp; + asm volatile( + "vpbroadcastd %[coeff],%%zmm16 \n" + "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n" + "vpslld $24,%%zmm17,%%zmm21 \n" // 0xff000000 + "vpsllw $7,%%zmm17,%%zmm17 \n" + "vpbroadcastb %%xmm17,%%zmm17 \n" // 0x80808080 + "vpbroadcastd %[bias],%%zmm19 \n" + "vbroadcasti32x4 %[shuf],%%zmm20 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpxorq %%zmm17,%%zmm0,%%zmm1 \n" + "vmovdqa64 %%zmm19,%%zmm2 \n" + "vpdpbusd %%zmm1,%%zmm16,%%zmm2 \n" + "vpsrld $8,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm20,%%zmm2,%%zmm2 \n" + "vpternlogd $0xf8,%%zmm21,%%zmm0,%%zmm2 \n" + "vmovdqu32 %%zmm2,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpxorq %%zmm17,%%zmm0,%%zmm1 \n" + "vmovdqa64 %%zmm19,%%zmm2 \n" + "vpdpbusd %%zmm1,%%zmm16,%%zmm2 \n" + "vpsrld $8,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm20,%%zmm2,%%zmm2 \n" + "vpternlogd $0xf8,%%zmm21,%%zmm0,%%zmm2 \n" + "vmovdqu32 %%zmm2,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width), + [temp] "=&r"(temp) + : [coeff] "m"(kCoeff), [bias] "m"(kBias), [shuf] "m"(kShufGrayDword) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm16", "xmm17", "xmm19", + "xmm20", "xmm21", "k1"); +} +#endif // HAS_ARGBGRAYROW_AVX512VNNI + #ifdef HAS_ARGBBLENDROW_AVX512BW void OMITFP ARGBBlendRow_AVX512BW(const uint8_t* src_argb, const uint8_t* src_argb1, diff --git a/source/row_win.cc b/source/row_win.cc index 88c6af8fc..6a08ebbab 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -37,11 +37,17 @@ extern "C" { __attribute__((target("avx512bw,avx512vl,avx512f"))) #define LIBYUV_TARGET_AVX512VBMI \ __attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f"))) +#define LIBYUV_TARGET_AVX512VNNI \ + __attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f"))) +#define LIBYUV_TARGET_AVX512VNNIVBMI \ + __attribute__((target("avx512vnni,avx512vbmi,avx512bw,avx512vl,avx512f"))) #else #define LIBYUV_TARGET_SSSE3 #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW #define LIBYUV_TARGET_AVX512VBMI +#define LIBYUV_TARGET_AVX512VNNI +#define LIBYUV_TARGET_AVX512VNNIVBMI #endif // defined(__clang__) || defined(__GNUC__) // Read 8 UV from 444 @@ -682,10 +688,15 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, } #endif // HAS_RAWTORGB24ROW_AVX512VBMI -#ifdef HAS_RGBTOYMATRIXROW_AVX512BW +#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) || \ + defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15}; +#endif +#ifdef HAS_RGBTOYMATRIXROW_AVX512BW LIBYUV_TARGET_AVX512BW void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, uint8_t* dst_y, @@ -770,6 +781,502 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, } #endif // HAS_RGBTOYMATRIXROW_AVX512BW +#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI +void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + __m512i zmm_80 = _mm512_set1_epi8((char)0x80); + __m512i zmm_coeff = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY)); + __m512i zmm_add = + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddY)); + + __m512i zmm_bias = + _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_coeff, zmm_80); + zmm_add = _mm512_sub_epi32(zmm_add, zmm_bias); + + while (width > 0) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); + __m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192)); + + zmm0 = _mm512_xor_si512(zmm0, zmm_80); + zmm1 = _mm512_xor_si512(zmm1, zmm_80); + zmm2 = _mm512_xor_si512(zmm2, zmm_80); + zmm3 = _mm512_xor_si512(zmm3, zmm_80); + + zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0); + zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1); + zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2); + zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3); + + zmm0 = _mm512_packus_epi32(zmm0, zmm1); + zmm2 = _mm512_packus_epi32(zmm2, zmm3); + + zmm0 = _mm512_srli_epi16(zmm0, 8); + zmm2 = _mm512_srli_epi16(zmm2, 8); + + zmm0 = _mm512_packus_epi16(zmm0, zmm2); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + + _mm512_storeu_si512((__m512i*)dst_y, zmm0); + + src_argb += 256; + dst_y += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNI + +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) || \ + defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) +// AVX512VNNIVBMI kernels use both AVX512VNNI (vpdpbusd) and AVX512VBMI +// (vpermt2b) and require both CPU features. There are no AVX512VBMI +// versions without AVX512VNNI. + +// vpermt2b indices selecting byte 1 of each int32 from 2 vectors. The 32 +// byte pattern is broadcast so each 256 bit half holds the same 32 results. +// Clang constant folds the indices and lowers _mm512_permutex2var_epi8 to a +// longer vpshufb + vpor + vpermd sequence, so hide the indices from it. +#if defined(__clang__) +#define LIBYUV_HIDE_CONSTANT(v) __asm__("" : "+v"(v)) +#else +#define LIBYUV_HIDE_CONSTANT(v) +#endif +static const uint8_t kPermbDWordByte1_AVX512VNNIVBMI[32] = { + 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, + 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125}; +#endif + +#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI +// Requires AVX512VNNI and AVX512VBMI. +// The 32 bit sums are kAddY + dot(kRGBToY, argb) which is always in +// [0, 65535], so Y is byte 1 of each int32 and no saturation is needed. +LIBYUV_TARGET_AVX512VNNIVBMI +void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + __m512i zmm_perm = _mm512_broadcast_i64x4( + _mm256_loadu_si256((const __m256i*)kPermbDWordByte1_AVX512VNNIVBMI)); + LIBYUV_HIDE_CONSTANT(zmm_perm); + __m512i zmm_80 = _mm512_set1_epi8((char)0x80); + __m512i zmm_coeff = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY)); + __m512i zmm_add = + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddY)); + + __m512i zmm_bias = + _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_coeff, zmm_80); + zmm_add = _mm512_sub_epi32(zmm_add, zmm_bias); + + while (width > 0) { + __m512i zmm0 = + _mm512_xor_si512(_mm512_loadu_si512((const __m512i*)src_argb), zmm_80); + __m512i zmm1 = _mm512_xor_si512( + _mm512_loadu_si512((const __m512i*)(src_argb + 64)), zmm_80); + __m512i zmm2 = _mm512_xor_si512( + _mm512_loadu_si512((const __m512i*)(src_argb + 128)), zmm_80); + __m512i zmm3 = _mm512_xor_si512( + _mm512_loadu_si512((const __m512i*)(src_argb + 192)), zmm_80); + + zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0); + zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1); + zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2); + zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3); + + zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // Y 0..31 + zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // Y 32..63 + + _mm256_storeu_si256((__m256i*)dst_y, _mm512_castsi512_si256(zmm0)); + _mm256_storeu_si256((__m256i*)(dst_y + 32), _mm512_castsi512_si256(zmm2)); + + src_argb += 256; + dst_y += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI + +#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI +void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + __m512i zmm_u = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); + __m512i zmm_v = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + + while (width > 0) { + __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i zmm_s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); + __m512i zmm_s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192)); + + __m512i zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_u); + __m512i zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_u); + __m512i zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_u); + __m512i zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_u); + + zmm0 = _mm512_packs_epi32(zmm0, zmm1); + zmm2 = _mm512_packs_epi32(zmm2, zmm3); + zmm0 = _mm512_sub_epi16(zmm_8000, zmm0); + zmm2 = _mm512_sub_epi16(zmm_8000, zmm2); + zmm0 = _mm512_srli_epi16(zmm0, 8); + zmm2 = _mm512_srli_epi16(zmm2, 8); + zmm0 = _mm512_packus_epi16(zmm0, zmm2); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + _mm512_storeu_si512((__m512i*)dst_u, zmm0); + + zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_v); + zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_v); + zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_v); + zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_v); + + zmm0 = _mm512_packs_epi32(zmm0, zmm1); + zmm2 = _mm512_packs_epi32(zmm2, zmm3); + zmm0 = _mm512_sub_epi16(zmm_8000, zmm0); + zmm2 = _mm512_sub_epi16(zmm_8000, zmm2); + zmm0 = _mm512_srli_epi16(zmm0, 8); + zmm2 = _mm512_srli_epi16(zmm2, 8); + zmm0 = _mm512_packus_epi16(zmm0, zmm2); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + _mm512_storeu_si512((__m512i*)dst_v, zmm0); + + src_argb += 256; + dst_u += 64; + dst_v += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNI + +#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI +// Requires AVX512VNNI and AVX512VBMI. +// U = (kAddUV - dot(kRGBToU, argb)) >> 8. The accumulators start at +// ~kAddUV so they hold ~(kAddUV - dot) and U is the inverse of byte 1 of each +// int32. The inverse is folded into the ternary logic that merges the halves. +LIBYUV_TARGET_AVX512VNNIVBMI +void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + __m512i zmm_perm = _mm512_broadcast_i64x4( + _mm256_loadu_si256((const __m256i*)kPermbDWordByte1_AVX512VNNIVBMI)); + LIBYUV_HIDE_CONSTANT(zmm_perm); + __m512i zmm_u = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); + __m512i zmm_v = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + __m512i zmm_add = _mm512_xor_si512( + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)), + _mm512_set1_epi32(-1)); // ~kAddUV + // Select low 256 bits from the first operand, high 256 bits from the second. + __m512i zmm_sel = + _mm512_inserti64x4(_mm512_setzero_si512(), _mm256_set1_epi32(-1), 1); + + while (width > 0) { + __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i zmm_s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); + __m512i zmm_s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192)); + + __m512i zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_u); + __m512i zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_u); + __m512i zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_u); + __m512i zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_u); + __m512i zmm4 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_v); + __m512i zmm5 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_v); + __m512i zmm6 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_v); + __m512i zmm7 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_v); + + zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // ~U 0..31 + zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // ~U 32..63 + zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); // ~V 0..31 + zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); // ~V 32..63 + zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm2, zmm_sel, 0x27); // U + zmm4 = _mm512_ternarylogic_epi32(zmm4, zmm6, zmm_sel, 0x27); // V + + _mm512_storeu_si512((__m512i*)dst_u, zmm0); + _mm512_storeu_si512((__m512i*)dst_v, zmm4); + + src_argb += 256; + dst_u += 64; + dst_v += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI + +#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI +void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + static const uint32_t kPermdARGBToUV_AVX512BW[16] = { + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15}; + static const uint8_t kShuffleAARRGGBB_AVX512[16] = { + 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15}; + __m512i zmm_u = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); + __m512i zmm_v = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + __m512i zmm_0101 = _mm512_set1_epi16(0x0101); + __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + __m512i zmm_zero = _mm512_setzero_si512(); + __m512i zmm_shuf = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512)); + __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + __m512i zmm_perm_uv = + _mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW); + + while (width >= 32) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i zmm2 = + _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb)); + __m512i zmm3 = + _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64)); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101); + zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101); + zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101); + zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101); + + zmm0 = _mm512_add_epi16(zmm0, zmm2); + zmm1 = _mm512_add_epi16(zmm1, zmm3); + + zmm0 = _mm512_srli_epi16(zmm0, 1); + zmm1 = _mm512_srli_epi16(zmm1, 1); + zmm0 = _mm512_avg_epu16(zmm0, zmm_zero); + zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); + + zmm0 = _mm512_packus_epi16(zmm0, zmm1); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + + zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u); + zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v); + zmm0 = _mm512_packs_epi32(zmm1, zmm2); + zmm0 = _mm512_sub_epi16(zmm_8000, zmm0); + zmm0 = _mm512_srli_epi16(zmm0, 8); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + + _mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0)); + _mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1)); + + src_argb += 128; + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + uint32_t mask = (1u << width) - 1u; + __mmask16 k1 = (__mmask16)mask; + __mmask16 k2 = (__mmask16)(mask >> 16); + __mmask16 k3 = ~k1; + __mmask16 k4 = ~k2; + __m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb); + __m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64); + __m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb); + __m512i zmm3 = + _mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64); + + zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0); + zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0); + zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0); + zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101); + zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101); + zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101); + zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101); + + zmm0 = _mm512_add_epi16(zmm0, zmm2); + zmm1 = _mm512_add_epi16(zmm1, zmm3); + + zmm0 = _mm512_srli_epi16(zmm0, 1); + zmm1 = _mm512_srli_epi16(zmm1, 1); + zmm0 = _mm512_avg_epu16(zmm0, zmm_zero); + zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); + + zmm0 = _mm512_packus_epi16(zmm0, zmm1); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + + zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u); + zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v); + zmm0 = _mm512_packs_epi32(zmm1, zmm2); + zmm0 = _mm512_sub_epi16(zmm_8000, zmm0); + zmm0 = _mm512_srli_epi16(zmm0, 8); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + + int halfwidth = (width + 1) >> 1; + __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u); + _mm_mask_storeu_epi8(dst_u, k_out, _mm512_castsi512_si128(zmm0)); + _mm_mask_storeu_epi8(dst_v, k_out, _mm512_extracti32x4_epi32(zmm0, 1)); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNI + +#ifdef HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI +// Requires AVX512VNNI and AVX512VBMI. +LIBYUV_TARGET_AVX512VNNIVBMI +void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + static const uint32_t kPermdARGBToUV_AVX512BW[16] = { + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15}; + static const uint8_t kShuffleAARRGGBB_AVX512[16] = { + 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15}; + __m512i zmm_u = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); + __m512i zmm_v = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + __m512i zmm_0101 = _mm512_set1_epi16(0x0101); + __m512i zmm_add = _mm512_xor_si512( + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)), + _mm512_set1_epi32(-1)); // ~kAddUV + __m512i zmm_zero = _mm512_setzero_si512(); + __m512i zmm_shuf = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512)); + __m512i zmm_perm = _mm512_broadcast_i64x4( + _mm256_loadu_si256((const __m256i*)kPermbDWordByte1_AVX512VNNIVBMI)); + LIBYUV_HIDE_CONSTANT(zmm_perm); + __m512i zmm_perm_uv = + _mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW); + + while (width >= 32) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i zmm2 = + _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb)); + __m512i zmm3 = + _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64)); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101); + zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101); + zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101); + zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101); + + zmm0 = _mm512_add_epi16(zmm0, zmm2); + zmm1 = _mm512_add_epi16(zmm1, zmm3); + + zmm0 = _mm512_srli_epi16(zmm0, 1); + zmm1 = _mm512_srli_epi16(zmm1, 1); + zmm0 = _mm512_avg_epu16(zmm0, zmm_zero); + zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); + + zmm0 = _mm512_packus_epi16(zmm0, zmm1); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + + zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U + zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V + zmm0 = _mm512_permutex2var_epi8(zmm1, zmm_perm, zmm2); + __m256i ymm_uv = + _mm256_xor_si256(_mm512_castsi512_si256(zmm0), _mm256_set1_epi32(-1)); + + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(ymm_uv)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(ymm_uv, 1)); + + src_argb += 128; + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + uint32_t mask = (1u << width) - 1u; + __mmask16 k1 = (__mmask16)mask; + __mmask16 k2 = (__mmask16)(mask >> 16); + __mmask16 k3 = ~k1; + __mmask16 k4 = ~k2; + __m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb); + __m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64); + __m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb); + __m512i zmm3 = + _mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64); + + zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0); + zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0); + zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0); + zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101); + zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101); + zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101); + zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101); + + zmm0 = _mm512_add_epi16(zmm0, zmm2); + zmm1 = _mm512_add_epi16(zmm1, zmm3); + + zmm0 = _mm512_srli_epi16(zmm0, 1); + zmm1 = _mm512_srli_epi16(zmm1, 1); + zmm0 = _mm512_avg_epu16(zmm0, zmm_zero); + zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); + + zmm0 = _mm512_packus_epi16(zmm0, zmm1); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + + zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U + zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V + zmm0 = _mm512_permutex2var_epi8(zmm1, zmm_perm, zmm2); + __m256i ymm_uv = + _mm256_xor_si256(_mm512_castsi512_si256(zmm0), _mm256_set1_epi32(-1)); + + int halfwidth = (width + 1) >> 1; + __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u); + _mm_mask_storeu_epi8(dst_u, k_out, _mm256_castsi256_si128(ymm_uv)); + _mm_mask_storeu_epi8(dst_v, k_out, _mm256_extracti128_si256(ymm_uv, 1)); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI + #ifdef HAS_ARGBTOUVMATRIXROW_AVX2 LIBYUV_TARGET_AVX2 void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, @@ -4319,6 +4826,43 @@ void ARGBGrayRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBGRAYROW_AVX512BW +#ifdef HAS_ARGBGRAYROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI +void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, + 8, 8, 8, 0x80, 12, 12, 12, 0x80}; + const __m512i c_coeff = _mm512_set1_epi32(0x004d961d); + const __m512i c_sub128 = _mm512_set1_epi8((char)0x80); + const __m512i c_bias = _mm512_set1_epi32(32896); + const __m512i c_shuf = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufGrayDword)); + const __m512i c_alpha = _mm512_set1_epi32((int)0xff000000u); + while (width >= 16) { + __m512i s = _mm512_loadu_si512((const __m512i*)src_argb); + src_argb += 64; + __m512i y32 = _mm512_srli_epi32( + _mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8); + __m512i out = _mm512_ternarylogic_epi32(_mm512_shuffle_epi8(y32, c_shuf), s, + c_alpha, 0xf8); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s = _mm512_maskz_loadu_epi32(k, src_argb); + __m512i y32 = _mm512_srli_epi32( + _mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8); + __m512i out = _mm512_ternarylogic_epi32(_mm512_shuffle_epi8(y32, c_shuf), s, + c_alpha, 0xf8); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBGRAYROW_AVX512VNNI + #ifdef HAS_ARGBBLENDROW_AVX512BW LIBYUV_TARGET_AVX512BW void ARGBBlendRow_AVX512BW(const uint8_t* src_argb,