diff --git a/.gn b/.gn index 856e1a649..43a31e13e 100644 --- a/.gn +++ b/.gn @@ -32,8 +32,10 @@ exec_script_allowlist = build_dotfile_settings.exec_script_allowlist + [ "//build_overrides/build.gni" ] default_args = { + android_static_analysis = "off" mac_sdk_min = "10.12" ios_deployment_target = "12.0" + # Use Siso instead of Ninja. use_siso = true } diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index 959f45442..cb5cbc827 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -34,13 +34,15 @@ extern "C" { #define HAS_HASHDJB2_SSE41 #define HAS_SSIM4X4ROW_SSE2 #define HAS_SSIMWINDOWROW_SSE2 +#if defined(VISUALC_HAS_AVX2) #define HAS_HAMMINGDISTANCE_AVX2 #define HAS_SUMSQUAREERROR_AVX2 #define HAS_HASHDJB2_AVX2 #define HAS_SSIM4X4ROW_AVX2 #define HAS_SSIMWINDOWROW_AVX2 #define HAS_ARGBDETECTROW_AVX2 -#if defined(__x86_64__) || defined(_M_X64) +#endif +#if defined(VISUALC_HAS_AVX512) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_HAMMINGDISTANCE_AVX512VNNI #define HAS_SUMSQUAREERROR_AVX512BW @@ -73,7 +75,8 @@ extern "C" { #define HAS_SSIMWINDOWROW_AVX2 #define HAS_ARGBDETECTROW_AVX2 #endif -#if defined(__x86_64__) && defined(CLANG_HAS_AVX512) +#if defined(__x86_64__) && \ + (defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512)) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_HAMMINGDISTANCE_AVX512VNNI #define HAS_SUMSQUAREERROR_AVX512BW diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index 227bf3bd1..d007f3d01 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -55,13 +55,42 @@ extern "C" { #endif // __clang__ // GCC >= 4.7.0 required for AVX2. -#if defined(__GNUC__) && !defined(LIBYUV_ENABLE_ROWWIN) && \ +#if defined(__GNUC__) && !defined(__clang__) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ (defined(__x86_64__) || defined(__i386__)) #if (__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 7)) #define GCC_HAS_AVX2 1 #endif // GNUC >= 4.7 #endif // __GNUC__ +// GCC >= 9.0.0 required for AVX512 (including VNNI and VBMI). +#if defined(__GNUC__) && !defined(__clang__) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && defined(__x86_64__) +#if (__GNUC__ >= 9) +#define GCC_HAS_AVX512 1 +#endif // GNUC >= 9 +#endif // __GNUC__ + +// Function target attributes so GCC enables zmm16..zmm31 and k1..k7 clobbers +// inside AVX512 functions without emitting AVX512 in non-AVX512 functions. +#if defined(GCC_HAS_AVX512) +#define LIBYUV_TARGET_AVX512BW \ + __attribute__((target("avx512bw,avx512vl,avx512f,bmi2"))) +#define LIBYUV_TARGET_AVX512VBMI \ + __attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f,bmi2"))) +#define LIBYUV_TARGET_AVX512VNNI \ + __attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f,bmi2"))) +#define LIBYUV_TARGET_AVX512VNNIVBMI \ + __attribute__(( \ + target("avx512vnni,avx512vbmi,avx512bw,avx512vl,avx512f,bmi2"))) +#elif !defined(LIBYUV_ENABLE_ROWWIN) && \ + !(defined(_MSC_VER) && !defined(__clang__)) +#define LIBYUV_TARGET_AVX512BW +#define LIBYUV_TARGET_AVX512VBMI +#define LIBYUV_TARGET_AVX512VNNI +#define LIBYUV_TARGET_AVX512VNNIVBMI +#endif + // clang >= 3.4.0 required for AVX2. #if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \ (defined(__x86_64__) || defined(__i386__)) @@ -70,7 +99,7 @@ extern "C" { #endif // clang >= 3.4 #endif // __clang__ -// clang >= 6.0.0 required for AVX512. +// clang >= 7.0.0 required for AVX512. #if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \ (defined(__x86_64__) || defined(__i386__)) // clang in xcode follows a different versioning scheme. @@ -80,13 +109,25 @@ extern "C" { #endif // clang >= 7 #endif // __clang__ -// Visual C 2012 required for AVX2. -#if (defined(_M_IX86) || defined(_M_X64)) && \ - (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN)) && \ - defined(_MSC_VER) && _MSC_VER >= 1700 +// Visual C 2012 (or clang >= 3.4 with LIBYUV_ENABLE_ROWWIN) required for AVX2. +#if ((defined(_M_IX86) || defined(_M_X64)) && !defined(__clang__) && \ + defined(_MSC_VER) && _MSC_VER >= 1700) || \ + (defined(LIBYUV_ENABLE_ROWWIN) && defined(__clang__) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((__clang_major__ > 3) || \ + (__clang_major__ == 3 && (__clang_minor__ >= 4)))) #define VISUALC_HAS_AVX2 1 #endif // VisualStudio >= 2012 +// Visual C 2019 (or clang >= 7 with LIBYUV_ENABLE_ROWWIN) required for AVX512. +#if (defined(_M_X64) || defined(__x86_64__)) && \ + ((!defined(__clang__) && defined(_MSC_VER) && _MSC_VER >= 1920) || \ + (defined(LIBYUV_ENABLE_ROWWIN) && defined(__clang__) && \ + (__clang_major__ >= 7) && !defined(__APPLE__))) +#define VISUALC_HAS_AVX512 1 +#endif // VisualStudio >= 2019 + #if defined(LIBYUV_ENABLE_SME) #undef LIBYUV_DISABLE_SME #endif diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index 4ac43d388..5caa3bdc9 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -29,6 +29,7 @@ extern "C" { defined(_M_IX86)) && \ ((defined(_MSC_VER) && !defined(__clang__)) || \ defined(LIBYUV_ENABLE_ROWWIN)) +#if defined(VISUALC_HAS_AVX2) #define HAS_TRANSPOSEWX8_AVX2 #define HAS_TRANSPOSEUVWX8_AVX2 #define HAS_TRANSPOSEWX8_16_AVX2 @@ -38,7 +39,8 @@ extern "C" { #define HAS_TRANSPOSEWXH_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 -#if defined(__x86_64__) || defined(_M_X64) +#endif +#if defined(VISUALC_HAS_AVX512) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW #define HAS_TRANSPOSEWX32_16_AVX512BW @@ -48,9 +50,10 @@ extern "C" { // The following are available for 32/64 bit GCC and clang, as inline assembly // in rotate_gcc.cc: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(__i386__)) && \ - !defined(LIBYUV_ENABLE_ROWWIN) && \ +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ (!defined(_MSC_VER) || defined(__clang__)) #define HAS_TRANSPOSEWX8_AVX2 #define HAS_TRANSPOSEUVWX8_AVX2 @@ -63,9 +66,10 @@ extern "C" { #define HAS_TRANSPOSEUVWX16_AVX2 #endif -// The following are available for AVX512 clang x64 platforms: -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) && \ +// The following are available for AVX512 GCC/clang x64 platforms: +#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ + (defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ (!defined(_MSC_VER) || defined(__clang__)) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 3fe8341b9..8f920f202 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -90,9 +90,9 @@ extern "C" { // The following are available on all x86 platforms, but // require VS2012, clang 3.4 or gcc 4.7. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ - defined(_M_IX86)) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \ + defined(GCC_HAS_AVX2)) #define HAS_ARGBMIRRORROW_AVX2 #define HAS_RGB24MIRRORROW_AVX2 #define HAS_ARGBTOUVMATRIXROW_AVX2 @@ -181,6 +181,7 @@ extern "C" { #define HAS_I422TOAR30ROW_SSSE3 #define HAS_MERGEARGBROW_SSE2 #define HAS_MERGEXRGBROW_SSE2 +#define HAS_MIRRORSPLITUVROW_SSSE3 #define HAS_MIRRORUVROW_SSSE3 #define HAS_NV21TOYUV24ROW_SSSE3 #define HAS_P210TOAR30ROW_SSSE3 @@ -188,11 +189,6 @@ extern "C" { #define HAS_P210TOARGBROW_SSSE3 #define HAS_P410TOAR30ROW_SSSE3 #define HAS_P410TOARGBROW_SSSE3 -#define HAS_RAWTOARGBROW_AVX2 -#define HAS_RGB24TOARGBROW_AVX2 -#define HAS_RGB565TOARGBROW_AVX2 -#define HAS_ARGB1555TOARGBROW_AVX2 -#define HAS_ARGB4444TOARGBROW_AVX2 #define HAS_SPLITARGBROW_SSE2 #define HAS_SPLITARGBROW_SSSE3 #define HAS_SPLITRGBROW_SSE41 @@ -233,11 +229,11 @@ extern "C" { #define HAS_ARGBTORGB565ROW_AVX2 #define HAS_ARGBTOARGB1555ROW_AVX2 #define HAS_ARGBTOARGB4444ROW_AVX2 -#define HAS_ARGBTOUVMATRIXROW_AVX2 -#define HAS_RGBTOUVMATRIXROW_AVX2 -#define HAS_RGB565TOUVMATRIXROW_AVX2 -#define HAS_ARGB1555TOUVMATRIXROW_AVX2 -#define HAS_ARGB4444TOUVMATRIXROW_AVX2 +#define HAS_RAWTOARGBROW_AVX2 +#define HAS_RGB24TOARGBROW_AVX2 +#define HAS_RGB565TOARGBROW_AVX2 +#define HAS_ARGB1555TOARGBROW_AVX2 +#define HAS_ARGB4444TOARGBROW_AVX2 #define HAS_ARGBTOUV444MATRIXROW_AVX2 #define HAS_RGBTOUV444MATRIXROW_AVX2 #define HAS_ARGBTOYMATRIXROW_AVX2 @@ -318,7 +314,6 @@ extern "C" { #define HAS_MERGEXR64ROW_AVX2 #define HAS_MERGEXRGB16TO8ROW_AVX2 #define HAS_MERGEXRGBROW_AVX2 -#define HAS_MIRRORUVROW_AVX2 #define HAS_MULTIPLYROW_16_AVX2 #define HAS_NV21TOYUV24ROW_AVX2 #define HAS_P210TOAR30ROW_AVX2 @@ -343,6 +338,10 @@ extern "C" { ((defined(_MSC_VER) && !defined(__clang__)) || \ defined(LIBYUV_ENABLE_ROWWIN)) #define HAS_RAWTORGB24ROW_SSSE3 +#define HAS_ARGBALPHABLENDROW_SSSE3 +#define HAS_ARGBORROW_SSE2 +#define HAS_MIRRORSPLITUVROW_SSSE3 +#if defined(VISUALC_HAS_AVX2) #define HAS_RAWTORGB24ROW_AVX2 #define HAS_RAWTOARGBROW_AVX2 #define HAS_I422TORGB24ROW_AVX2 @@ -360,6 +359,7 @@ extern "C" { #define HAS_I422TOARGB4444ROW_AVX2 #define HAS_NV12TORGB565ROW_AVX2 #define HAS_CONVERT8TO16ROW_AVX2 +#define HAS_CONVERT8TO8ROW_AVX2 #define HAS_I422TOAR30ROW_AVX2 #define HAS_AR30TOAB30ROW_AVX2 #define HAS_AR30TOABGRROW_AVX2 @@ -367,7 +367,108 @@ extern "C" { #define HAS_AYUVTOUVROW_AVX2 #define HAS_AYUVTOVUROW_AVX2 #define HAS_AYUVTOYROW_AVX2 -#if defined(__x86_64__) || defined(_M_X64) +#define HAS_ARGB1555TOUVROW_AVX2 +#define HAS_ARGB1555TOYROW_AVX2 +#define HAS_ARGB4444TOUVROW_AVX2 +#define HAS_ARGB4444TOYROW_AVX2 +#define HAS_ARGBAFFINEROW_AVX2 +#define HAS_ARGBCOLORMATRIXROW_AVX2 +#define HAS_ARGBCOLORTABLEROW_AVX2 +#define HAS_ARGBLUMACOLORTABLEROW_AVX2 +#define HAS_ARGBQUANTIZEROW_AVX2 +#define HAS_ARGBSEPIAROW_AVX2 +#define HAS_ARGBSETROW_AVX2 +#define HAS_ARGBSHADEROW_AVX2 +#define HAS_BYTETOFLOATROW_AVX2 +#define HAS_COPYROW_16_AVX2 +#define HAS_DETILEROW_16_AVX2 +#define HAS_DETILEROW_AVX2 +#define HAS_DETILESPLITUVROW_AVX2 +#define HAS_GAUSSROW_AVX2 +#define HAS_GAUSSROW_F32_AVX2 +#define HAS_RGB565TOUVROW_AVX2 +#define HAS_RGB565TOYROW_AVX2 +#define HAS_RGBCOLORTABLEROW_AVX2 +#define HAS_SETROW_AVX2 +#define HAS_SOBELROW_AVX2 +#define HAS_SOBELTOPLANEROW_AVX2 +#define HAS_SOBELXROW_AVX2 +#define HAS_SOBELXYROW_AVX2 +#define HAS_SOBELYROW_AVX2 +#define HAS_ARGBALPHABLENDROW_AVX2 +#define HAS_ARGBORROW_AVX2 +#define HAS_ARGBTOYMATRIXROW_AVX2 +#define HAS_RGBTOYMATRIXROW_AVX2 +#define HAS_RGB565TOYMATRIXROW_AVX2 +#define HAS_ARGB1555TOYMATRIXROW_AVX2 +#define HAS_ARGB4444TOYMATRIXROW_AVX2 +#define HAS_ARGBTOUV444MATRIXROW_AVX2 +#define HAS_RGBTOUV444MATRIXROW_AVX2 +#define HAS_J400TOARGBROW_AVX2 +#define HAS_MULTIPLYROW_16_AVX2 +#define HAS_SWAPUVROW_AVX2 +#define HAS_AB64TOARGBROW_AVX2 +#define HAS_AR64TOARGBROW_AVX2 +#define HAS_ARGBADDROW_AVX2 +#define HAS_ARGBBLENDROW_AVX2 +#define HAS_ARGBCOPYALPHAROW_AVX2 +#define HAS_ARGBGRAYROW_AVX2 +#define HAS_ARGBPOLYNOMIALROW_AVX2 +#define HAS_ARGBSUBTRACTROW_AVX2 +#define HAS_ARGBTOAB64ROW_AVX2 +#define HAS_ARGBTOAR64ROW_AVX2 +#define HAS_ARGBTORAWROW_AVX2 +#define HAS_ARGBTORGB24ROW_AVX2 +#define HAS_ARGBUNATTENUATEROW_AVX2 +#define HAS_COMPUTECUMULATIVESUMROW_AVX2 +#define HAS_CONVERT16TO8ROW_AVX2 +#define HAS_COPYROW_AVX +#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 +#define HAS_DETILETOYUY2_AVX2 +#define HAS_DIVIDEROW_16_AVX2 +#define HAS_GAUSSCOL_AVX2 +#define HAS_GAUSSCOL_F32_AVX2 +#define HAS_HALFMERGEUVROW_AVX2 +#define HAS_HALFROW_16_AVX2 +#define HAS_HALFROW_16TO8_AVX2 +#define HAS_HALFROW_AVX2 +#define HAS_HALFWIDTHROW_16TO8_AVX2 +#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2 +#define HAS_I422TOUYVYROW_AVX2 +#define HAS_I422TOYUY2ROW_AVX2 +#define HAS_I444TOARGBROW_AVX2 +#define HAS_I444TORGB24ROW_AVX2 +#define HAS_MERGEAR64ROW_AVX2 +#define HAS_MERGEARGB16TO8ROW_AVX2 +#define HAS_MERGEARGBROW_AVX2 +#define HAS_MERGERGBROW_AVX2 +#define HAS_MERGEUVROW_16_AVX2 +#define HAS_MERGEXR30ROW_AVX2 +#define HAS_MERGEXR64ROW_AVX2 +#define HAS_MERGEXRGB16TO8ROW_AVX2 +#define HAS_MERGEXRGBROW_AVX2 +#define HAS_NV12TORGB24ROW_AVX2 +#define HAS_NV21TORGB24ROW_AVX2 +#define HAS_NV21TOYUV24ROW_AVX2 +#define HAS_RAWTORGBAROW_AVX2 +#define HAS_SCALEMAXSAMPLES_AVX2 +#define HAS_SCALESAMPLES_AVX2 +#define HAS_SCALESUMSAMPLES_AVX2 +#define HAS_SPLITARGBROW_AVX2 +#define HAS_SPLITRGBROW_AVX2 +#define HAS_SPLITUVROW_16_AVX2 +#define HAS_SPLITUVROW_AVX2 +#define HAS_SPLITXRGBROW_AVX2 +#define HAS_UNPACKMT2T_AVX2 +#define HAS_UYVYTOUV422ROW_AVX2 +#define HAS_UYVYTOUVROW_AVX2 +#define HAS_UYVYTOYROW_AVX2 +#define HAS_YUY2TONVUVROW_AVX2 +#define HAS_YUY2TOUV422ROW_AVX2 +#define HAS_YUY2TOUVROW_AVX2 +#define HAS_YUY2TOYROW_AVX2 +#endif +#if defined(VISUALC_HAS_AVX512) #define HAS_RAWTOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW #define HAS_RAWTORGB24ROW_AVX512BW @@ -397,6 +498,7 @@ extern "C" { #define HAS_NV21TOARGBROW_AVX512BW #define HAS_NV12TORGB24ROW_AVX512VBMI #define HAS_NV21TORGB24ROW_AVX512VBMI +#define HAS_NV21TOYUV24ROW_AVX512VBMI #define HAS_I422ALPHATOARGBROW_AVX512BW #define HAS_SWAPUVROW_AVX512BW #define HAS_HALFMERGEUVROW_AVX512BW @@ -453,120 +555,6 @@ extern "C" { #define HAS_YUY2TOARGBROW_AVX512BW #define HAS_YUY2TOUVROW_AVX512BW #define HAS_YUY2TOYROW_AVX512BW -#endif -#define HAS_ARGB1555TOUVROW_AVX2 -#define HAS_ARGB1555TOYROW_AVX2 -#define HAS_ARGB4444TOUVROW_AVX2 -#define HAS_ARGB4444TOYROW_AVX2 -#define HAS_ARGBAFFINEROW_AVX2 -#define HAS_ARGBCOLORMATRIXROW_AVX2 -#define HAS_ARGBCOLORTABLEROW_AVX2 -#define HAS_ARGBLUMACOLORTABLEROW_AVX2 -#define HAS_ARGBQUANTIZEROW_AVX2 -#define HAS_ARGBSEPIAROW_AVX2 -#define HAS_ARGBSETROW_AVX2 -#define HAS_ARGBSHADEROW_AVX2 -#define HAS_BYTETOFLOATROW_AVX2 -#define HAS_COPYROW_16_AVX2 -#define HAS_DETILEROW_16_AVX2 -#define HAS_DETILEROW_AVX2 -#define HAS_DETILESPLITUVROW_AVX2 -#define HAS_GAUSSROW_AVX2 -#define HAS_GAUSSROW_F32_AVX2 -#define HAS_RGB565TOUVROW_AVX2 -#define HAS_RGB565TOYROW_AVX2 -#define HAS_RGBCOLORTABLEROW_AVX2 -#define HAS_SETROW_AVX2 -#define HAS_SOBELROW_AVX2 -#define HAS_SOBELTOPLANEROW_AVX2 -#define HAS_SOBELXROW_AVX2 -#define HAS_SOBELXYROW_AVX2 -#define HAS_SOBELYROW_AVX2 -#define HAS_ARGBALPHABLENDROW_SSSE3 -#define HAS_ARGBORROW_SSE2 -#define HAS_ARGBALPHABLENDROW_AVX2 -#define HAS_ARGBORROW_AVX2 -#define HAS_ARGBTOYMATRIXROW_AVX2 -#define HAS_RGBTOYMATRIXROW_AVX2 -#define HAS_RGB565TOYMATRIXROW_AVX2 -#define HAS_ARGB1555TOYMATRIXROW_AVX2 -#define HAS_ARGB4444TOYMATRIXROW_AVX2 -#define HAS_ARGBTOUVMATRIXROW_AVX2 -#define HAS_ARGBTOUV444MATRIXROW_AVX2 -#define HAS_RGBTOUV444MATRIXROW_AVX2 -#define HAS_J400TOARGBROW_AVX2 -#define HAS_I422TOARGBROW_AVX2 -#define HAS_NV12TOARGBROW_AVX2 -#define HAS_NV21TOARGBROW_AVX2 -#define HAS_MERGEUVROW_AVX2 -#define HAS_MIRRORROW_AVX2 -#define HAS_MIRRORUVROW_AVX2 -#define HAS_MIRRORSPLITUVROW_AVX2 -#define HAS_RGB24MIRRORROW_AVX2 -#define HAS_ARGBMIRRORROW_AVX2 -#define HAS_INTERPOLATEROW_AVX2 -#define HAS_INTERPOLATEROW_16_AVX2 -#define HAS_MULTIPLYROW_16_AVX2 -#define HAS_SWAPUVROW_AVX2 -#define HAS_AB64TOARGBROW_AVX2 -#define HAS_AR64TOARGBROW_AVX2 -#define HAS_ARGBADDROW_AVX2 -#define HAS_ARGBBLENDROW_AVX2 -#define HAS_ARGBCOPYALPHAROW_AVX2 -#define HAS_ARGBGRAYROW_AVX2 -#define HAS_ARGBPOLYNOMIALROW_AVX2 -#define HAS_ARGBSUBTRACTROW_AVX2 -#define HAS_ARGBTOAB64ROW_AVX2 -#define HAS_ARGBTOAR64ROW_AVX2 -#define HAS_ARGBTORAWROW_AVX2 -#define HAS_ARGBTORGB24ROW_AVX2 -#define HAS_ARGBUNATTENUATEROW_AVX2 -#define HAS_COMPUTECUMULATIVESUMROW_AVX2 -#define HAS_CONVERT16TO8ROW_AVX2 -#define HAS_COPYROW_AVX -#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 -#define HAS_DETILETOYUY2_AVX2 -#define HAS_GAUSSCOL_AVX2 -#define HAS_GAUSSCOL_F32_AVX2 -#define HAS_HALFROW_16_AVX2 -#define HAS_HALFROW_16TO8_AVX2 -#define HAS_HALFROW_AVX2 -#define HAS_HALFWIDTHROW_16TO8_AVX2 -#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2 -#define HAS_I422TOUYVYROW_AVX2 -#define HAS_I422TOYUY2ROW_AVX2 -#define HAS_I444TOARGBROW_AVX2 -#define HAS_I444TORGB24ROW_AVX2 -#define HAS_MERGEAR64ROW_AVX2 -#define HAS_MERGEARGB16TO8ROW_AVX2 -#define HAS_MERGEARGBROW_AVX2 -#define HAS_MERGERGBROW_AVX2 -#define HAS_MERGEUVROW_16_AVX2 -#define HAS_MERGEXR30ROW_AVX2 -#define HAS_MERGEXR64ROW_AVX2 -#define HAS_MERGEXRGB16TO8ROW_AVX2 -#define HAS_MERGEXRGBROW_AVX2 -#define HAS_NV12TORGB24ROW_AVX2 -#define HAS_NV21TORGB24ROW_AVX2 -#define HAS_NV21TOYUV24ROW_AVX2 -#define HAS_RAWTORGBAROW_AVX2 -#define HAS_SCALEMAXSAMPLES_AVX2 -#define HAS_SCALESAMPLES_AVX2 -#define HAS_SCALESUMSAMPLES_AVX2 -#define HAS_SPLITARGBROW_AVX2 -#define HAS_SPLITRGBROW_AVX2 -#define HAS_SPLITUVROW_16_AVX2 -#define HAS_SPLITUVROW_AVX2 -#define HAS_SPLITXRGBROW_AVX2 -#define HAS_UNPACKMT2T_AVX2 -#define HAS_UYVYTOUV422ROW_AVX2 -#define HAS_UYVYTOUVROW_AVX2 -#define HAS_UYVYTOYROW_AVX2 -#define HAS_YUY2TONVUVROW_AVX2 -#define HAS_YUY2TOUV422ROW_AVX2 -#define HAS_YUY2TOUVROW_AVX2 -#define HAS_YUY2TOYROW_AVX2 -#if defined(__x86_64__) || defined(_M_X64) #define HAS_ARGBTORAWROW_AVX512BW #define HAS_ARGBTORGB24ROW_AVX512BW #define HAS_ARGBTOUV444MATRIXROW_AVX512BW @@ -589,10 +577,10 @@ extern "C" { #endif #endif -// The following are available for AVX512 clang x64 platforms: -// TODO(fbarchard): Port to GCC and Visual C -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) +// The following are available for AVX512 GCC/clang x64 platforms: +#if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_GCC) && \ + defined(__x86_64__) && \ + (defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512)) #define HAS_COPYROW_AVX512BW #define HAS_RAWTOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW @@ -605,12 +593,6 @@ extern "C" { #define HAS_HALFROW_16TO8_AVX512BW #define HAS_HALFWIDTHROW_16TO8_AVX512BW #define HAS_MULTIPLYROW_16_AVX512BW -#endif - -// The following are available for AVX512 clang x64 platforms: -// TODO(fbarchard): Port to x86 -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) #define HAS_I422TOARGBROW_AVX512BW #define HAS_I422TOAR30ROW_AVX512BW #define HAS_ARGBSHUFFLEROW_AVX512BW @@ -648,6 +630,7 @@ extern "C" { #define HAS_NV21TOARGBROW_AVX512BW #define HAS_NV12TORGB24ROW_AVX512VBMI #define HAS_NV21TORGB24ROW_AVX512VBMI +#define HAS_NV21TOYUV24ROW_AVX512VBMI #define HAS_I422ALPHATOARGBROW_AVX512BW #define HAS_SWAPUVROW_AVX512BW #define HAS_HALFMERGEUVROW_AVX512BW @@ -2611,18 +2594,6 @@ void ARGBToYMatrixRow_Any_AVX2(const uint8_t* src_argb, uint8_t* dst_y, int width, const struct ArgbConstants* c); -void ARGBToYMatrixRow_Any_AVX512BW(const uint8_t* src_argb, - uint8_t* dst_y, - int width, - const struct ArgbConstants* c); -void ARGBToYMatrixRow_Any_AVX512VNNI(const uint8_t* src_argb, - uint8_t* dst_y, - int width, - const struct ArgbConstants* c); -void ARGBToYMatrixRow_Any_AVX512VNNIVBMI(const uint8_t* src_argb, - uint8_t* dst_y, - int width, - const struct ArgbConstants* c); void RGBToYMatrixRow_NEON(const uint8_t* src_rgb, uint8_t* dst_y, @@ -2726,10 +2697,6 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, uint8_t* dst_y, int width, const struct ArgbConstants* c); -void RGBToYMatrixRow_Any_AVX512BW(const uint8_t* src_rgb, - uint8_t* dst_y, - int width, - const struct ArgbConstants* c); void ARGBToYMatrixRow_Any_NEON_DotProd(const uint8_t* src_argb, uint8_t* dst_y, @@ -2826,21 +2793,6 @@ void ARGBToUV444MatrixRow_Any_AVX2(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c); -void ARGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_argb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c); -void ARGBToUV444MatrixRow_Any_AVX512VNNI(const uint8_t* src_argb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c); -void ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI(const uint8_t* src_argb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c); void RGBToUV444MatrixRow_Any_SSSE3(const uint8_t* src_rgb, uint8_t* dst_u, uint8_t* dst_v, @@ -2851,11 +2803,6 @@ void RGBToUV444MatrixRow_Any_AVX2(const uint8_t* src_rgb, uint8_t* dst_v, int width, const struct ArgbConstants* c); -void RGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_rgb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c); // Deprecated. void RGB565ToYRow_C(const uint8_t* src_rgb565, uint8_t* dst_y, int width); @@ -3041,6 +2988,18 @@ void MirrorSplitUVRow_AVX2(const uint8_t* src, uint8_t* dst_u, uint8_t* dst_v, int width); +void MirrorSplitUVRow_SSSE3(const uint8_t* src, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void MirrorSplitUVRow_Any_AVX2(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void MirrorSplitUVRow_Any_SSSE3(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void MirrorSplitUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, @@ -3147,10 +3106,6 @@ void SplitUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width); -void SplitUVRow_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_u, - uint8_t* dst_v, - int width); void SplitUVRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, @@ -3317,10 +3272,6 @@ void MergeUVRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); -void MergeUVRow_Any_AVX512BW(const uint8_t* y_buf, - const uint8_t* uv_buf, - uint8_t* dst_ptr, - int width); void MergeUVRow_Any_NEON(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, @@ -3374,6 +3325,18 @@ void HalfMergeUVRow_AVX512BW(const uint8_t* src_u, int src_stride_v, uint8_t* dst_uv, int width); +void HalfMergeUVRow_Any_SSSE3(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); +void HalfMergeUVRow_Any_AVX2(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); void SplitRGBRow_C(const uint8_t* src_rgb, uint8_t* dst_r, @@ -3917,10 +3880,6 @@ void MultiplyRow_16_AVX512BW(const uint16_t* src_y, uint16_t* dst_y, int scale, int width); -void MultiplyRow_16_Any_AVX512BW(const uint16_t* src_ptr, - uint16_t* dst_ptr, - int scale, - int width); void MultiplyRow_16_NEON(const uint16_t* src_y, uint16_t* dst_y, int scale, @@ -3991,10 +3950,6 @@ void Convert8To16Row_Any_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int bits, int width); -void Convert8To16Row_Any_AVX512BW(const uint8_t* src_ptr, - uint16_t* dst_ptr, - int bits, - int width); void Convert8To16Row_NEON(const uint8_t* src_y, uint16_t* dst_y, int bits, @@ -4036,10 +3991,6 @@ void Convert16To8Row_Any_AVX2(const uint16_t* src_ptr, uint8_t* dst_ptr, int scale, int width); -void Convert16To8Row_Any_AVX512BW(const uint16_t* src_ptr, - uint8_t* dst_ptr, - int scale, - int width); void Convert16To8Row_NEON(const uint16_t* src_y, uint8_t* dst_y, int scale, @@ -4117,11 +4068,6 @@ void HalfRow_16To8_Any_AVX2(const uint16_t* src_ptr, uint8_t* dst_ptr, int scale, int width); -void HalfRow_16To8_Any_AVX512BW(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int scale, - int width); void HalfRow_16To8_NEON(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, @@ -4188,11 +4134,6 @@ void HalfWidthRow_16To8_Any_AVX2(const uint16_t* src_ptr, uint8_t* dst_ptr, int scale, int width); -void HalfWidthRow_16To8_Any_AVX512BW(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int scale, - int width); void HalfWidthRow_16To8_NEON(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, @@ -4603,12 +4544,6 @@ void RAWToRGB24Row_Any_SSSE3(const uint8_t* src_ptr, void RAWToRGB24Row_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); -void RAWToRGB24Row_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); -void RAWToRGB24Row_Any_AVX512VBMI(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); void RGB565ToARGBRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -4867,9 +4802,6 @@ void J400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_C(const uint8_t* src_y, uint8_t* dst_argb, int width); -void J400ToARGBRow_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); void J400ToARGBRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -5453,6 +5385,10 @@ void NV21ToYUV24Row_AVX2(const uint8_t* src_y, const uint8_t* src_vu, uint8_t* dst_yuv24, int width); +void NV21ToYUV24Row_AVX512VBMI(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_yuv24, + int width); void NV21ToARGBRow_SSSE3(const uint8_t* y_buf, const uint8_t* vu_buf, @@ -7825,6 +7761,14 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, const float* poly, int width); +void ARGBPolynomialRow_Any_SSE2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const float* param, + int width); +void ARGBPolynomialRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const float* param, + int width); void ARGBPolynomialRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, const float* poly, diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index a4d34c515..e81d227d9 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -59,12 +59,9 @@ extern "C" { // The following are available on all x86 platforms, but // require VS2012, clang 3.4 or gcc 4.7. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \ - defined(GCC_HAS_AVX2) || \ - (defined(LIBYUV_ENABLE_ROWWIN) && \ - (defined(__x86_64__) || defined(__i386__) || \ - defined(_M_X64) || defined(_M_IX86)))) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \ + defined(GCC_HAS_AVX2)) #define HAS_SCALEADDROW_AVX2 #define HAS_SCALEARGBCOLS_AVX2 #define HAS_SCALEARGBCOLSUP2_AVX2 @@ -96,11 +93,10 @@ extern "C" { #endif // The following are available for AVX512 x64 platforms: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(_M_X64)) && \ - (defined(CLANG_HAS_AVX512) || \ - (defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(_M_X64)) && \ + (defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512) || \ + defined(VISUALC_HAS_AVX512)) #define HAS_SCALEADDROW_AVX512BW #define HAS_SCALEADDCOLS1_AVX512BW #define HAS_SCALEADDCOLS2_AVX512BW diff --git a/source/compare_gcc.cc b/source/compare_gcc.cc index f6c8817be..5828ceaa7 100644 --- a/source/compare_gcc.cc +++ b/source/compare_gcc.cc @@ -389,6 +389,7 @@ static uint64_t LowBytesMask64(int n) { #ifdef HAS_HAMMINGDISTANCE_AVX512BW // Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked // loads that read zeros, so any count is supported. +LIBYUV_TARGET_AVX512BW uint32_t HammingDistance_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -527,6 +528,7 @@ uint32_t HammingDistance_AVX512VNNI(const uint8_t* src_a, #ifdef HAS_SUMSQUAREERROR_AVX512BW // Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked // loads that read zeros, so any count is supported. +LIBYUV_TARGET_AVX512BW uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -596,6 +598,7 @@ uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, #ifdef HAS_SUMSQUAREERROR_AVX512VNNI // Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked // loads that read zeros, so any count is supported. +LIBYUV_TARGET_AVX512VNNI uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -811,6 +814,7 @@ static uint32_t Pow33(int n) { // Process 64 bytes per loop. The remainder of n = 1 to 63 bytes is loaded // into the high n bytes with a masked load ending at src + count, so the // same multipliers apply, and hash is multiplied by 33 ^ n. +LIBYUV_TARGET_AVX512BW uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) { const int n = count & 63; const uint64_t mask = n ? ~0ULL << (64 - n) : 0ULL; @@ -1150,6 +1154,7 @@ void Ssim4x4Row_AVX2(const uint8_t* src_a, // Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60 // pixels (1 to 15 tiles) with masked loads and stores. +LIBYUV_TARGET_AVX512BW void Ssim4x4Row_AVX512BW(const uint8_t* src_a, int stride_a, const uint8_t* src_b, @@ -1433,6 +1438,7 @@ float SsimWindowRow_AVX2(const uint32_t* row0, // Process 16 8x8 windows per loop and the remainder of 1 to 15 windows with // masked loads and masked accumulation. +LIBYUV_TARGET_AVX512BW float SsimWindowRow_AVX512BW(const uint32_t* row0, const uint32_t* row1, int num_windows) { diff --git a/source/convert.cc b/source/convert.cc index 50ea99484..e76ba7c34 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -291,10 +291,7 @@ static int I41xToI420(const uint16_t* src_y, #endif #if defined(HAS_CONVERT16TO8ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert16To8Row = Convert16To8Row_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - Convert16To8Row = Convert16To8Row_AVX512BW; - } + Convert16To8Row = Convert16To8Row_AVX512BW; } #endif #if defined(HAS_CONVERT16TO8ROW_RVV) @@ -345,10 +342,7 @@ static int I41xToI420(const uint16_t* src_y, #endif #if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_AVX512BW; - if (IS_ALIGNED(uv_width, 32)) { - HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX512BW; - } + HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX512BW; } #endif #if defined(HAS_HALFWIDTHROW_16TO8_RVV) @@ -466,10 +460,7 @@ static int I21xToI420(const uint16_t* src_y, #endif #if defined(HAS_CONVERT16TO8ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert16To8Row = Convert16To8Row_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - Convert16To8Row = Convert16To8Row_AVX512BW; - } + Convert16To8Row = Convert16To8Row_AVX512BW; } #endif #if defined(HAS_CONVERT16TO8ROW_RVV) @@ -519,10 +510,7 @@ static int I21xToI420(const uint16_t* src_y, #endif #if defined(HAS_HALFROW_16TO8_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - HalfRow_16To8 = HalfRow_16To8_Any_AVX512BW; - if (IS_ALIGNED(uv_width, 64)) { - HalfRow_16To8 = HalfRow_16To8_AVX512BW; - } + HalfRow_16To8 = HalfRow_16To8_AVX512BW; } #endif #if defined(HAS_HALFROW_16TO8_RVV) @@ -962,10 +950,7 @@ int I010ToNV12(const uint16_t* src_y, #endif #if defined(HAS_CONVERT16TO8ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert16To8Row = Convert16To8Row_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 64)) { - Convert16To8Row = Convert16To8Row_AVX512BW; - } + Convert16To8Row = Convert16To8Row_AVX512BW; } #endif #if defined(HAS_CONVERT16TO8ROW_HVX) @@ -985,17 +970,14 @@ int I010ToNV12(const uint16_t* src_y, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(halfwidth, 32)) { + if (IS_ALIGNED(halfwidth, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -1503,17 +1485,14 @@ int I422ToNV21(const uint8_t* src_y, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(halfwidth, 32)) { + if (IS_ALIGNED(halfwidth, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -2096,10 +2075,7 @@ static int Px10ToI420(const uint16_t* src_y, #endif #if defined(HAS_CONVERT16TO8ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert16To8Row = Convert16To8Row_Any_AVX512BW; - if (IS_ALIGNED(uv_packed, 64)) { - Convert16To8Row = Convert16To8Row_AVX512BW; - } + Convert16To8Row = Convert16To8Row_AVX512BW; } #endif #if defined(HAS_CONVERT16TO8ROW_RVV) @@ -2130,10 +2106,7 @@ static int Px10ToI420(const uint16_t* src_y, #endif #if defined(HAS_SPLITUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - SplitUVRow = SplitUVRow_Any_AVX512BW; - if (IS_ALIGNED(src_uv_width, 64)) { - SplitUVRow = SplitUVRow_AVX512BW; - } + SplitUVRow = SplitUVRow_AVX512BW; } #endif #if defined(HAS_SPLITUVROW_NEON) @@ -2262,10 +2235,7 @@ static int Px10ToI420(const uint16_t* src_y, #endif #if defined(HAS_HALFROW_16TO8_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Half16To8 = HalfRow_16To8_Any_AVX512BW; - if (IS_ALIGNED(dst_uv_width, 64)) { - Half16To8 = HalfRow_16To8_AVX512BW; - } + Half16To8 = HalfRow_16To8_AVX512BW; } #endif #if defined(HAS_HALFROW_16TO8_RVV) @@ -2315,10 +2285,7 @@ static int Px10ToI420(const uint16_t* src_y, #endif #if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Half16To8 = HalfWidthRow_16To8_Any_AVX512BW; - if (IS_ALIGNED(dst_uv_width, 32)) { - Half16To8 = HalfWidthRow_16To8_AVX512BW; - } + Half16To8 = HalfWidthRow_16To8_AVX512BW; } #endif #if defined(HAS_HALFWIDTHROW_16TO8_RVV) @@ -2919,26 +2886,17 @@ int ARGBToI420Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -3043,7 +3001,7 @@ int ARGBToI420Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -3187,26 +3145,17 @@ int ARGBToI420Alpha(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -3311,7 +3260,7 @@ int ARGBToI420Alpha(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -3598,26 +3547,17 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -3683,7 +3623,7 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, #if defined(HAS_ARGBTOUVMATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3; } } @@ -3691,7 +3631,7 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -3857,10 +3797,7 @@ int RGB24ToI420(const uint8_t* src_rgb24, #endif #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; - } + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUVMATRIXROW_SSSE3) @@ -4018,10 +3955,7 @@ int RGB24ToJ420(const uint8_t* src_rgb24, #endif #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; - } + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUVMATRIXROW_SSSE3) @@ -4178,10 +4112,7 @@ int RAWToI420(const uint8_t* src_raw, #endif #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; - } + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUVMATRIXROW_SSSE3) @@ -4329,10 +4260,7 @@ int RAWToJ420(const uint8_t* src_raw, #endif #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; - } + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUVMATRIXROW_SSSE3) @@ -4479,10 +4407,7 @@ static int RAWToI444Matrix(const uint8_t* src_raw, #endif #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; - } + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOYMATRIXROW_NEON) @@ -4549,10 +4474,7 @@ static int RAWToI444Matrix(const uint8_t* src_raw, #endif #if defined(HAS_RGBTOUV444MATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToUV444MatrixRow = RGBToUV444MatrixRow_AVX512BW; - } + RGBToUV444MatrixRow = RGBToUV444MatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUV444MATRIXROW_NEON) @@ -4974,10 +4896,7 @@ int RGB24ToJ400(const uint8_t* src_rgb24, #endif #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; - } + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOYMATRIXROW_NEON) @@ -5079,10 +4998,7 @@ int RAWToJ400(const uint8_t* src_raw, #endif #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; - } + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOYMATRIXROW_NEON) diff --git a/source/convert_argb.cc b/source/convert_argb.cc index e79dcb8ad..9799e0a79 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -872,7 +872,7 @@ int I444ToRGB24Matrix(const uint8_t* src_y, #if defined(HAS_I444TORGB24ROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I444ToRGB24Row = I444ToRGB24Row_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I444ToRGB24Row = I444ToRGB24Row_SSSE3; } } @@ -3710,10 +3710,7 @@ int J400ToARGB(const uint8_t* src_y, #endif #if defined(HAS_J400TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - J400ToARGBRow = J400ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - J400ToARGBRow = J400ToARGBRow_AVX512BW; - } + J400ToARGBRow = J400ToARGBRow_AVX512BW; } #endif #if defined(HAS_J400TOARGBROW_NEON) @@ -5446,6 +5443,11 @@ int NV21ToYUV24(const uint8_t* src_y, NV21ToYUV24Row = NV21ToYUV24Row_AVX2; } } +#endif +#if defined(HAS_NV21TOYUV24ROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + NV21ToYUV24Row = NV21ToYUV24Row_AVX512VBMI; + } #endif for (y = 0; y < height; ++y) { NV21ToYUV24Row(src_y, src_vu, dst_yuv24, width); @@ -5491,7 +5493,7 @@ int YUY2ToARGBMatrix(const uint8_t* src_yuy2, #if defined(HAS_YUY2TOARGBROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { YUY2ToARGBRow = YUY2ToARGBRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { YUY2ToARGBRow = YUY2ToARGBRow_SSSE3; } } @@ -5596,7 +5598,7 @@ int UYVYToARGBMatrix(const uint8_t* src_uyvy, #if defined(HAS_UYVYTOARGBROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { UYVYToARGBRow = UYVYToARGBRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { UYVYToARGBRow = UYVYToARGBRow_SSSE3; } } @@ -6216,7 +6218,7 @@ int I420ToRGB24Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB24ROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToRGB24Row = I422ToRGB24Row_SSSE3; } } @@ -6447,7 +6449,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB24ROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToRGB24Row = I422ToRGB24Row_SSSE3; } } @@ -7758,7 +7760,7 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y, #if defined(HAS_I444TORGB24ROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I444ToRGB24Row = I444ToRGB24Row_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I444ToRGB24Row = I444ToRGB24Row_SSSE3; } } @@ -9637,7 +9639,7 @@ static int I422ToRGB24MatrixLinear(const uint8_t* src_y, #if defined(HAS_I444TORGB24ROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I444ToRGB24Row = I444ToRGB24Row_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I444ToRGB24Row = I444ToRGB24Row_SSSE3; } } diff --git a/source/convert_from.cc b/source/convert_from.cc index c4f35cfea..6f5853bc2 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -775,10 +775,7 @@ static int I4xxToPxxx(const uint8_t* src_y, #endif #if defined(HAS_CONVERT8TO16ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert8To16Row = Convert8To16Row_Any_AVX512BW; - if (IS_ALIGNED(uv_width, 64)) { - Convert8To16Row = Convert8To16Row_AVX512BW; - } + Convert8To16Row = Convert8To16Row_AVX512BW; } #endif #if defined(HAS_CONVERT8TO16ROW_NEON) diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index cc7691314..8f71a0ad3 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -78,26 +78,17 @@ int ARGBToI444Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -179,26 +170,17 @@ int ARGBToI444Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW; - } + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI; - } + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI; - } + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOUV444MATRIXROW_NEON) @@ -312,26 +294,17 @@ int ARGBToI422Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -436,7 +409,7 @@ int ARGBToI422Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -547,26 +520,17 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -671,7 +635,7 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -732,17 +696,14 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(halfwidth, 32)) { + if (IS_ALIGNED(halfwidth, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -875,26 +836,17 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -999,7 +951,7 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -1060,17 +1012,14 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(halfwidth, 32)) { + if (IS_ALIGNED(halfwidth, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -1197,26 +1146,17 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -1298,26 +1238,17 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW; - } + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI; - } + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI; - } + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOUV444MATRIXROW_NEON) @@ -1377,17 +1308,14 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -1502,26 +1430,17 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -1626,7 +1545,7 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -1687,17 +1606,14 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(halfwidth, 32)) { + if (IS_ALIGNED(halfwidth, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -1816,26 +1732,17 @@ int ARGBToI400Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -1931,26 +1838,17 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -2003,7 +1901,7 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -2157,26 +2055,17 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -2229,7 +2118,7 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -2412,26 +2301,17 @@ int ARGBToP010Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -2536,7 +2416,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -2598,10 +2478,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb, #endif #if defined(HAS_CONVERT8TO16ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert8To16Row = Convert8To16Row_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - Convert8To16Row = Convert8To16Row_AVX512BW; - } + Convert8To16Row = Convert8To16Row_AVX512BW; } #endif #if defined(HAS_CONVERT8TO16ROW_NEON) @@ -2645,10 +2522,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb, #endif #if defined(HAS_MULTIPLYROW_16_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - MultiplyRow_16 = MultiplyRow_16_AVX512BW; - } + MultiplyRow_16 = MultiplyRow_16_AVX512BW; } #endif #if defined(HAS_MULTIPLYROW_16_NEON) @@ -2693,17 +2567,14 @@ int ARGBToP010Matrix(const uint8_t* src_argb, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(halfwidth, 32)) { + if (IS_ALIGNED(halfwidth, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -2869,26 +2740,17 @@ int ARGBToP210Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -2993,7 +2855,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } @@ -3055,10 +2917,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb, #endif #if defined(HAS_CONVERT8TO16ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert8To16Row = Convert8To16Row_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - Convert8To16Row = Convert8To16Row_AVX512BW; - } + Convert8To16Row = Convert8To16Row_AVX512BW; } #endif #if defined(HAS_CONVERT8TO16ROW_NEON) @@ -3102,10 +2961,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb, #endif #if defined(HAS_MULTIPLYROW_16_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - MultiplyRow_16 = MultiplyRow_16_AVX512BW; - } + MultiplyRow_16 = MultiplyRow_16_AVX512BW; } #endif #if defined(HAS_MULTIPLYROW_16_NEON) @@ -3150,17 +3006,14 @@ int ARGBToP210Matrix(const uint8_t* src_argb, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(halfwidth, 32)) { + if (IS_ALIGNED(halfwidth, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -3311,26 +3164,17 @@ int ARGBToP410Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -3412,26 +3256,17 @@ int ARGBToP410Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW; - } + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) if (TestCpuFlag(kCpuHasAVX512VNNI)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNI; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI; - } + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI) if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI; - } + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOUV444MATRIXROW_NEON) @@ -3492,10 +3327,7 @@ int ARGBToP410Matrix(const uint8_t* src_argb, #endif #if defined(HAS_CONVERT8TO16ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert8To16Row = Convert8To16Row_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - Convert8To16Row = Convert8To16Row_AVX512BW; - } + Convert8To16Row = Convert8To16Row_AVX512BW; } #endif #if defined(HAS_CONVERT8TO16ROW_NEON) @@ -3539,10 +3371,7 @@ int ARGBToP410Matrix(const uint8_t* src_argb, #endif #if defined(HAS_MULTIPLYROW_16_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - MultiplyRow_16 = MultiplyRow_16_AVX512BW; - } + MultiplyRow_16 = MultiplyRow_16_AVX512BW; } #endif #if defined(HAS_MULTIPLYROW_16_NEON) @@ -3587,17 +3416,14 @@ int ARGBToP410Matrix(const uint8_t* src_argb, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -4774,10 +4600,7 @@ static int RAWToNVMatrix(const uint8_t* src_raw, #endif #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; - } + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOYMATRIXROW_NEON) @@ -4875,17 +4698,14 @@ static int RAWToNVMatrix(const uint8_t* src_raw, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(halfwidth, 32)) { + if (IS_ALIGNED(halfwidth, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 64)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index d3b55687a..17842d285 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -240,10 +240,7 @@ static int NV24ToARGB(const uint8_t* src_y, #endif #if defined(HAS_SPLITUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - SplitUVRow = SplitUVRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - SplitUVRow = SplitUVRow_AVX512BW; - } + SplitUVRow = SplitUVRow_AVX512BW; } #endif #if defined(HAS_SPLITUVROW_NEON) diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc index 0d80d0c0d..e5a094df9 100644 --- a/source/convert_to_i420.cc +++ b/source/convert_to_i420.cc @@ -55,9 +55,11 @@ static int NV16ToI420(const uint8_t* src_y, if (height < 0) { height = -height; halfheight = (height + 1) >> 1; - src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + if (src_y) { + src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } src_uv = src_uv + (ptrdiff_t)(height - 1) * src_stride_uv; - src_stride_y = -src_stride_y; src_stride_uv = -src_stride_uv; } if (dst_y) { @@ -81,10 +83,7 @@ static int NV16ToI420(const uint8_t* src_y, #endif #if defined(HAS_SPLITUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - SplitUVRow = SplitUVRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 64)) { - SplitUVRow = SplitUVRow_AVX512BW; - } + SplitUVRow = SplitUVRow_AVX512BW; } #endif #if defined(HAS_SPLITUVROW_NEON) @@ -226,9 +225,11 @@ static int NV24ToI420(const uint8_t* src_y, if (height < 0) { height = -height; halfheight = (height + 1) >> 1; - src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + if (src_y) { + src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } src_uv = src_uv + (ptrdiff_t)(height - 1) * src_stride_uv; - src_stride_y = -src_stride_y; src_stride_uv = -src_stride_uv; } if (dst_y) { diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 0c8a42542..c30764490 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -232,10 +232,7 @@ void Convert16To8Plane(const uint16_t* src_y, #endif #if defined(HAS_CONVERT16TO8ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert16To8Row = Convert16To8Row_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - Convert16To8Row = Convert16To8Row_AVX512BW; - } + Convert16To8Row = Convert16To8Row_AVX512BW; } #endif #if defined(HAS_CONVERT16TO8ROW_HVX) @@ -309,10 +306,7 @@ void Convert8To16Plane( #endif #if defined(HAS_CONVERT8TO16ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - Convert8To16Row = Convert8To16Row_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - Convert8To16Row = Convert8To16Row_AVX512BW; - } + Convert8To16Row = Convert8To16Row_AVX512BW; } #endif #if defined(HAS_CONVERT8TO16ROW_NEON) @@ -745,10 +739,7 @@ void SplitUVPlane(const uint8_t* src_uv, #endif #if defined(HAS_SPLITUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - SplitUVRow = SplitUVRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - SplitUVRow = SplitUVRow_AVX512BW; - } + SplitUVRow = SplitUVRow_AVX512BW; } #endif #if defined(HAS_SPLITUVROW_NEON) @@ -839,17 +830,14 @@ void MergeUVPlane(const uint8_t* src_u, #if defined(HAS_MERGEUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow = MergeUVRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { MergeUVRow = MergeUVRow_AVX2; } } #endif #if defined(HAS_MERGEUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MergeUVRow = MergeUVRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - MergeUVRow = MergeUVRow_AVX512BW; - } + MergeUVRow = MergeUVRow_AVX512BW; } #endif #if defined(HAS_MERGEUVROW_NEON) @@ -1092,10 +1080,7 @@ void ConvertToMSBPlane_16(const uint16_t* src_y, #endif #if defined(HAS_MULTIPLYROW_16_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - MultiplyRow_16 = MultiplyRow_16_AVX512BW; - } + MultiplyRow_16 = MultiplyRow_16_AVX512BW; } #endif #if defined(HAS_MULTIPLYROW_16_NEON) @@ -4353,18 +4338,12 @@ int RAWToRGB24(const uint8_t* src_raw, #endif #if defined(HAS_RAWTORGB24ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RAWToRGB24Row = RAWToRGB24Row_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RAWToRGB24Row = RAWToRGB24Row_AVX512BW; - } + RAWToRGB24Row = RAWToRGB24Row_AVX512BW; } #endif #if defined(HAS_RAWTORGB24ROW_AVX512VBMI) if (TestCpuFlag(kCpuHasAVX512VBMI)) { - RAWToRGB24Row = RAWToRGB24Row_Any_AVX512VBMI; - if (IS_ALIGNED(width, 64)) { - RAWToRGB24Row = RAWToRGB24Row_AVX512VBMI; - } + RAWToRGB24Row = RAWToRGB24Row_AVX512VBMI; } #endif #if defined(HAS_RAWTORGB24ROW_NEON) @@ -5565,7 +5544,7 @@ int InterpolatePlane_16(const uint16_t* src0, #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow_16 = InterpolateRow_16_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { InterpolateRow_16 = InterpolateRow_16_AVX2; } } @@ -5981,10 +5960,17 @@ static int ARGBSobelize(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI) + if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI; } #endif #if defined(HAS_ARGBTOYMATRIXROW_NEON) @@ -6282,14 +6268,19 @@ int ARGBPolynomial(const uint8_t* src_argb, src_stride_argb = dst_stride_argb = 0; } #if defined(HAS_ARGBPOLYNOMIALROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 2)) { - ARGBPolynomialRow = ARGBPolynomialRow_SSE2; + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBPolynomialRow = ARGBPolynomialRow_Any_SSE2; + if (IS_ALIGNED(width, 2)) { + ARGBPolynomialRow = ARGBPolynomialRow_SSE2; + } } #endif #if defined(HAS_ARGBPOLYNOMIALROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2) && TestCpuFlag(kCpuHasFMA3) && - IS_ALIGNED(width, 2)) { - ARGBPolynomialRow = ARGBPolynomialRow_AVX2; + if (TestCpuFlag(kCpuHasAVX2) && TestCpuFlag(kCpuHasFMA3)) { + ARGBPolynomialRow = ARGBPolynomialRow_Any_AVX2; + if (IS_ALIGNED(width, 2)) { + ARGBPolynomialRow = ARGBPolynomialRow_AVX2; + } } #endif #if defined(HAS_ARGBPOLYNOMIALROW_NEON) @@ -6889,10 +6880,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #endif #if defined(HAS_SPLITUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - SplitUVRow = SplitUVRow_Any_AVX512BW; - if (IS_ALIGNED(awidth, 64)) { - SplitUVRow = SplitUVRow_AVX512BW; - } + SplitUVRow = SplitUVRow_AVX512BW; } #endif #if defined(HAS_SPLITUVROW_NEON) @@ -7068,13 +7056,19 @@ void HalfMergeUVPlane(const uint8_t* src_u, } #endif #if defined(HAS_HALFMERGEUVROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 16)) { - HalfMergeUVRow = HalfMergeUVRow_SSSE3; + if (TestCpuFlag(kCpuHasSSSE3)) { + HalfMergeUVRow = HalfMergeUVRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + HalfMergeUVRow = HalfMergeUVRow_SSSE3; + } } #endif #if defined(HAS_HALFMERGEUVROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(width, 32)) { - HalfMergeUVRow = HalfMergeUVRow_AVX2; + if (TestCpuFlag(kCpuHasAVX2)) { + HalfMergeUVRow = HalfMergeUVRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + HalfMergeUVRow = HalfMergeUVRow_AVX2; + } } #endif #if defined(HAS_HALFMERGEUVROW_AVX512BW) diff --git a/source/rotate.cc b/source/rotate.cc index 7b27333f1..db72843a2 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -559,9 +559,20 @@ void SplitRotateUV180(const uint8_t* src, MirrorSplitUVRow = MirrorSplitUVRow_NEON; } #endif +#if defined(HAS_MIRRORSPLITUVROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + MirrorSplitUVRow = MirrorSplitUVRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + MirrorSplitUVRow = MirrorSplitUVRow_SSSE3; + } + } +#endif #if defined(HAS_MIRRORSPLITUVROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(width, 32)) { - MirrorSplitUVRow = MirrorSplitUVRow_AVX2; + if (TestCpuFlag(kCpuHasAVX2)) { + MirrorSplitUVRow = MirrorSplitUVRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + MirrorSplitUVRow = MirrorSplitUVRow_AVX2; + } } #endif #if defined(HAS_MIRRORSPLITUVROW_AVX512BW) diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index 251054f2a..734638023 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -189,6 +189,7 @@ static const uint32_t kPermdTranspose_AVX512BW[32] = { // TODO(fbarchard): Use for ARGB (32 bit) and 16 bit channel transposes. // TODO(fbarchard): Consider removing in favor of TransposeWx16_Byte_AVX2, // which is within 10% of AVX512BW on most CPUs. +LIBYUV_TARGET_AVX512BW static void TransposeWx16_Byte_AVX512BW(const uint8_t* src, int src_stride, uint8_t* dst_a, @@ -748,6 +749,7 @@ void TransposeWx16_16_AVX2(const uint16_t* src, #if defined(HAS_TRANSPOSEWX32_16_AVX512BW) // Transpose 32 rows of 16 bit plane (arbitrary width) using 8x32 tiles in // zmm0..zmm15 with k1-masked right tail. +LIBYUV_TARGET_AVX512BW void TransposeWx32_16_AVX512BW(const uint16_t* src, int src_stride, uint16_t* dst, @@ -1553,6 +1555,7 @@ void TransposeWxH_32_AVX2(const uint8_t* src, #if defined(HAS_TRANSPOSEWX16_32_AVX512BW) // Transpose 16 rows of 32 bit plane (arbitrary width) using 4x16 ZMM vertical // tiles in zmm0..zmm7 with k1-masked right tail. +LIBYUV_TARGET_AVX512BW void TransposeWx16_32_AVX512BW(const uint8_t* src, int src_stride, uint8_t* dst, diff --git a/source/row_any.cc b/source/row_any.cc index 3ac640620..9c6d3ccb3 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -398,7 +398,7 @@ ANY31C(I422ToARGB1555Row_Any_SSSE3, I422ToARGB1555Row_SSSE3, 1, 0, 2, 7) ANY31C(I422ToRGB565Row_Any_SSSE3, I422ToRGB565Row_SSSE3, 1, 0, 2, 7) #endif #ifdef HAS_I422TORGB24ROW_SSSE3 -ANY31C(I422ToRGB24Row_Any_SSSE3, I422ToRGB24Row_SSSE3, 1, 0, 3, 15) +ANY31C(I422ToRGB24Row_Any_SSSE3, I422ToRGB24Row_SSSE3, 1, 0, 3, 7) #endif #ifdef HAS_I422TOAR30ROW_SSSE3 ANY31C(I422ToAR30Row_Any_SSSE3, I422ToAR30Row_SSSE3, 1, 0, 4, 7) @@ -410,7 +410,7 @@ ANY31C(I422ToAR30Row_Any_AVX2, I422ToAR30Row_AVX2, 1, 0, 4, 15) ANY31C(I444ToARGBRow_Any_SSSE3, I444ToARGBRow_SSSE3, 0, 0, 4, 7) #endif #ifdef HAS_I444TORGB24ROW_SSSE3 -ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 15) +ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 7) #endif #ifdef HAS_I422TORGB24ROW_AVX2 ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15) @@ -676,10 +676,7 @@ ANY31PT(MergeXRGB16To8Row_Any_NEON, ANY21(MergeUVRow_Any_SSE2, MergeUVRow_SSE2, 0, 1, 1, 2, 15) #endif #ifdef HAS_MERGEUVROW_AVX2 -ANY21(MergeUVRow_Any_AVX2, MergeUVRow_AVX2, 0, 1, 1, 2, 31) -#endif -#ifdef HAS_MERGEUVROW_AVX512BW -ANY21(MergeUVRow_Any_AVX512BW, MergeUVRow_AVX512BW, 0, 1, 1, 2, 31) +ANY21(MergeUVRow_Any_AVX2, MergeUVRow_AVX2, 0, 1, 1, 2, 15) #endif #ifdef HAS_MERGEUVROW_NEON ANY21(MergeUVRow_Any_NEON, MergeUVRow_NEON, 0, 1, 1, 2, 15) @@ -829,7 +826,7 @@ ANY21(MergeUVRow_Any_WASMSIMD, MergeUVRow_WASMSIMD, 0, 1, 1, 2, 15) memcpy(vin, src_yuy2 + np * SBPP, r * SBPP); \ memcpy(vin + (MASK + 1) * SBPP, src_yuy2 + stride_yuy2 + np * SBPP, \ r * SBPP); \ - ANY_SIMD(vin, (MASK + 1) * SBPP, vout, MASK + 1); \ + ANY_SIMD(vin, (MASK + 1) * SBPP, vout, (MASK + 1) * 2); \ memcpy(dst_uv + np * BPP, vout, r * BPP); \ } @@ -842,6 +839,46 @@ ANY21S(YUY2ToNVUVRow_Any_SSE2, YUY2ToNVUVRow_SSE2, 4, 2, 7) #ifdef HAS_YUY2TONVUVROW_AVX2 ANY21S(YUY2ToNVUVRow_Any_AVX2, YUY2ToNVUVRow_AVX2, 4, 2, 15) #endif +#undef ANY21S + +// Any 2 planes to 1 half-size merged UV row with stride (2 rows of U and V). +#define ANY21HMS(NAMEANY, ANY_SIMD, MASK) \ + void NAMEANY(const uint8_t* src_u, int src_stride_u, const uint8_t* src_v, \ + int src_stride_v, uint8_t* dst_uv, int width) { \ + SIMD_ALIGNED(uint8_t vin[64 * 4]); \ + static_assert(MASK + 1 <= sizeof(vin) / 4, "vin buffer too small"); \ + SIMD_ALIGNED(uint8_t vout[64]); \ + static_assert(SS(MASK + 1, 1) * 2 <= sizeof(vout), \ + "vout buffer too small"); \ + MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_u, src_stride_u, src_v, src_stride_v, dst_uv, n); \ + } \ + ptrdiff_t np = n; \ + memcpy(vin, src_u + np, r); \ + memcpy(vin + 64, src_u + src_stride_u + np, r); \ + memcpy(vin + 128, src_v + np, r); \ + memcpy(vin + 192, src_v + src_stride_v + np, r); \ + if (r & 1) { \ + vin[r] = vin[r - 1]; \ + vin[64 + r] = vin[64 + r - 1]; \ + vin[128 + r] = vin[128 + r - 1]; \ + vin[192 + r] = vin[192 + r - 1]; \ + } \ + ANY_SIMD(vin, 64, vin + 128, 64, vout, MASK + 1); \ + memcpy(dst_uv + (np >> 1) * 2, vout, SS(r, 1) * 2); \ + } + +#ifdef HAS_HALFMERGEUVROW_SSSE3 +ANY21HMS(HalfMergeUVRow_Any_SSSE3, HalfMergeUVRow_SSSE3, 15) +#endif +#ifdef HAS_HALFMERGEUVROW_AVX2 +ANY21HMS(HalfMergeUVRow_Any_AVX2, HalfMergeUVRow_AVX2, 31) +#endif +#undef ANY21HMS // Any 2 planes to 1 with yuvconstants #define ANY21C(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, SBPP2, BPP, MASK) \ @@ -1133,9 +1170,6 @@ ANY11(ABGRToAR30Row_Any_AVX2, ABGRToAR30Row_AVX2, 0, 4, 4, 7) ANY11(ARGBToAR30Row_Any_AVX2, ARGBToAR30Row_AVX2, 0, 4, 4, 7) #endif -#if defined(HAS_J400TOARGBROW_AVX512BW) -ANY11(J400ToARGBRow_Any_AVX512BW, J400ToARGBRow_AVX512BW, 0, 1, 4, 31) -#endif #if defined(HAS_J400TOARGBROW_AVX2) ANY11(J400ToARGBRow_Any_AVX2, J400ToARGBRow_AVX2, 0, 1, 4, 15) #endif @@ -1159,12 +1193,6 @@ ANY11(RAWToRGB24Row_Any_SSSE3, RAWToRGB24Row_SSSE3, 0, 3, 3, 7) #if defined(HAS_RAWTORGB24ROW_AVX2) ANY11(RAWToRGB24Row_Any_AVX2, RAWToRGB24Row_AVX2, 0, 3, 3, 31) #endif -#if defined(HAS_RAWTORGB24ROW_AVX512BW) -ANY11(RAWToRGB24Row_Any_AVX512BW, RAWToRGB24Row_AVX512BW, 0, 3, 3, 63) -#endif -#if defined(HAS_RAWTORGB24ROW_AVX512VBMI) -ANY11(RAWToRGB24Row_Any_AVX512VBMI, RAWToRGB24Row_AVX512VBMI, 0, 3, 3, 63) -#endif #if defined(HAS_RGB565TOARGBROW_AVX2) ANY11(RGB565ToARGBRow_Any_AVX2, RGB565ToARGBRow_AVX2, 0, 2, 4, 15) #endif @@ -1614,6 +1642,22 @@ ANY11P(ARGBOrRow_Any_LASX, ARGBOrRow_LASX, const uint32_t, 4, 4, 7) #ifdef HAS_ARGBORROW_WASMSIMD ANY11P(ARGBOrRow_Any_WASMSIMD, ARGBOrRow_WASMSIMD, const uint32_t, 4, 4, 3) #endif +#ifdef HAS_ARGBPOLYNOMIALROW_SSE2 +ANY11P(ARGBPolynomialRow_Any_SSE2, + ARGBPolynomialRow_SSE2, + const float*, + 4, + 4, + 1) +#endif +#ifdef HAS_ARGBPOLYNOMIALROW_AVX2 +ANY11P(ARGBPolynomialRow_Any_AVX2, + ARGBPolynomialRow_AVX2, + const float*, + 4, + 4, + 1) +#endif #ifdef HAS_ARGBPOLYNOMIALROW_NEON ANY11P(ARGBPolynomialRow_Any_NEON, ARGBPolynomialRow_NEON, @@ -1623,7 +1667,6 @@ ANY11P(ARGBPolynomialRow_Any_NEON, 1) #endif #undef ANY11P -#undef ANY11P // Any 1 to 1 with type #define ANY11T(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \ @@ -1738,15 +1781,6 @@ ANY11C(Convert16To8Row_Any_AVX2, uint8_t, 31) #endif -#ifdef HAS_CONVERT16TO8ROW_AVX512BW -ANY11C(Convert16To8Row_Any_AVX512BW, - Convert16To8Row_AVX512BW, - 2, - 1, - uint16_t, - uint8_t, - 63) -#endif #ifdef HAS_CONVERT16TO8ROW_NEON ANY11C(Convert16To8Row_Any_NEON, Convert16To8Row_NEON, @@ -1774,15 +1808,6 @@ ANY11C(Convert8To16Row_Any_AVX2, uint16_t, 31) #endif -#ifdef HAS_CONVERT8TO16ROW_AVX512BW -ANY11C(Convert8To16Row_Any_AVX512BW, - Convert8To16Row_AVX512BW, - 1, - 2, - uint8_t, - uint16_t, - 63) -#endif #ifdef HAS_CONVERT8TO16ROW_NEON ANY11C(Convert8To16Row_Any_NEON, Convert8To16Row_NEON, @@ -1792,15 +1817,6 @@ ANY11C(Convert8To16Row_Any_NEON, uint16_t, 15) #endif -#ifdef HAS_MULTIPLYROW_16_AVX512BW -ANY11C(MultiplyRow_16_Any_AVX512BW, - MultiplyRow_16_AVX512BW, - 2, - 2, - uint16_t, - uint16_t, - 63) -#endif #ifdef HAS_MULTIPLYROW_16_AVX2 ANY11C(MultiplyRow_16_Any_AVX2, MultiplyRow_16_AVX2, @@ -1959,8 +1975,8 @@ ANY11P16(HalfFloatRow_Any_LASX, HalfFloatRow_LASX, uint16_t, uint16_t, 2, 2, 15) } #if defined(HAS_YUY2TOARGBROW_SSSE3) -ANY11C(YUY2ToARGBRow_Any_SSSE3, YUY2ToARGBRow_SSSE3, 1, 4, 4, 15) -ANY11C(UYVYToARGBRow_Any_SSSE3, UYVYToARGBRow_SSSE3, 1, 4, 4, 15) +ANY11C(YUY2ToARGBRow_Any_SSSE3, YUY2ToARGBRow_SSSE3, 1, 4, 4, 7) +ANY11C(UYVYToARGBRow_Any_SSSE3, UYVYToARGBRow_SSSE3, 1, 4, 4, 7) #endif #if defined(HAS_YUY2TOARGBROW_AVX2) ANY11C(YUY2ToARGBRow_Any_AVX2, YUY2ToARGBRow_AVX2, 1, 4, 4, 31) @@ -2121,13 +2137,6 @@ ANY11HS(HalfRow_16To8_Any_SSSE3, HalfRow_16To8_SSSE3, uint16_t, uint8_t, 15) #ifdef HAS_HALFROW_16TO8_AVX2 ANY11HS(HalfRow_16To8_Any_AVX2, HalfRow_16To8_AVX2, uint16_t, uint8_t, 31) #endif -#ifdef HAS_HALFROW_16TO8_AVX512BW -ANY11HS(HalfRow_16To8_Any_AVX512BW, - HalfRow_16To8_AVX512BW, - uint16_t, - uint8_t, - 63) -#endif #ifdef HAS_HALFROW_16TO8_NEON ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15) #endif @@ -2172,13 +2181,6 @@ ANY11HWS(HalfWidthRow_16To8_Any_AVX2, uint8_t, 15) #endif -#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW -ANY11HWS(HalfWidthRow_16To8_Any_AVX512BW, - HalfWidthRow_16To8_AVX512BW, - uint16_t, - uint8_t, - 31) -#endif #ifdef HAS_HALFWIDTHROW_16TO8_NEON ANY11HWS(HalfWidthRow_16To8_Any_NEON, HalfWidthRow_16To8_NEON, @@ -2302,6 +2304,36 @@ ANY11MT(MirrorRow_16_Any_NEON, MirrorRow_16_NEON, uint16_t, 2, 15) #endif #undef ANY11MT +// Any 1 to 2 mirror split UV. +#define ANY12MR(NAMEANY, ANY_SIMD, MASK) \ + void NAMEANY(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, \ + int width) { \ + SIMD_ALIGNED(uint8_t vin[128]); \ + static_assert((MASK + 1) * 2 <= sizeof(vin), "vin buffer too small"); \ + SIMD_ALIGNED(uint8_t vout[64 * 2]); \ + static_assert(MASK + 1 <= sizeof(vout) / 2, "vout buffer too small"); \ + MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_uv + r * 2, dst_u, dst_v, n); \ + } \ + ptrdiff_t np = n; \ + memcpy(vin, src_uv, r * 2); \ + ANY_SIMD(vin, vout, vout + 64, MASK + 1); \ + memcpy(dst_u + np, vout + (MASK + 1 - r), r); \ + memcpy(dst_v + np, vout + 64 + (MASK + 1 - r), r); \ + } + +#ifdef HAS_MIRRORSPLITUVROW_SSSE3 +ANY12MR(MirrorSplitUVRow_Any_SSSE3, MirrorSplitUVRow_SSSE3, 7) +#endif +#ifdef HAS_MIRRORSPLITUVROW_AVX2 +ANY12MR(MirrorSplitUVRow_Any_AVX2, MirrorSplitUVRow_AVX2, 15) +#endif +#undef ANY12MR + // Any 1 plane. (memset) #define ANY1(NAMEANY, ANY_SIMD, T, BPP, MASK) \ void NAMEANY(uint8_t* dst_ptr, T v32, int width) { \ @@ -2371,9 +2403,6 @@ ANY1(ARGBSetRow_Any_LASX, ARGBSetRow_LASX, uint32_t, 4, 7) #ifdef HAS_SPLITUVROW_SSE2 ANY12(SplitUVRow_Any_SSE2, SplitUVRow_SSE2, 0, 2, 0, 15) #endif -#ifdef HAS_SPLITUVROW_AVX512BW -ANY12(SplitUVRow_Any_AVX512BW, SplitUVRow_AVX512BW, 0, 2, 0, 63) -#endif #ifdef HAS_SPLITUVROW_AVX2 ANY12(SplitUVRow_Any_AVX2, SplitUVRow_AVX2, 0, 2, 0, 31) #endif @@ -2628,21 +2657,21 @@ ANY12MS(ARGB1555ToUVMatrixRow_Any_NEON, ARGB1555ToUVMatrixRow_NEON, 0, 2, 15) ANY12MS(ARGB4444ToUVMatrixRow_Any_NEON, ARGB4444ToUVMatrixRow_NEON, 0, 2, 15) #endif #ifdef HAS_ARGBTOUVMATRIXROW_AVX2 -ANY12MS(ARGBToUVMatrixRow_Any_AVX2, ARGBToUVMatrixRow_AVX2, 0, 4, 31) +ANY12MS(ARGBToUVMatrixRow_Any_AVX2, ARGBToUVMatrixRow_AVX2, 0, 4, 15) ANY12MS(RGBToUVMatrixRow_Any_AVX2, RGBToUVMatrixRow_AVX2, 0, 3, 31) -ANY12MS(RGB565ToUVMatrixRow_Any_AVX2, RGB565ToUVMatrixRow_AVX2, 0, 2, 31) +ANY12MS(RGB565ToUVMatrixRow_Any_AVX2, RGB565ToUVMatrixRow_AVX2, 0, 2, 15) #ifdef HAS_ARGB1555TOARGBROW_AVX2 -ANY12MS(ARGB1555ToUVMatrixRow_Any_AVX2, ARGB1555ToUVMatrixRow_AVX2, 0, 2, 31) +ANY12MS(ARGB1555ToUVMatrixRow_Any_AVX2, ARGB1555ToUVMatrixRow_AVX2, 0, 2, 15) #endif #ifdef HAS_ARGB4444TOARGBROW_AVX2 -ANY12MS(ARGB4444ToUVMatrixRow_Any_AVX2, ARGB4444ToUVMatrixRow_AVX2, 0, 2, 31) +ANY12MS(ARGB4444ToUVMatrixRow_Any_AVX2, ARGB4444ToUVMatrixRow_AVX2, 0, 2, 15) #endif #endif #ifdef HAS_ARGBTOUVMATRIXROW_SSSE3 ANY12MS(ARGBToUVMatrixRow_Any_SSSE3, ARGBToUVMatrixRow_SSSE3, 0, 4, 7) #endif #ifdef HAS_RGBTOUVMATRIXROW_SSSE3 -ANY12MS(RGBToUVMatrixRow_Any_SSSE3, RGBToUVMatrixRow_SSSE3, 0, 3, 7) +ANY12MS(RGBToUVMatrixRow_Any_SSSE3, RGBToUVMatrixRow_SSSE3, 0, 3, 15) #endif #ifdef HAS_ARGBTOUV444MATRIXROW_AVX2 ANY12M(ARGBToUV444MatrixRow_Any_AVX2, ARGBToUV444MatrixRow_AVX2, 4, 31) @@ -2650,24 +2679,6 @@ ANY12M(ARGBToUV444MatrixRow_Any_AVX2, ARGBToUV444MatrixRow_AVX2, 4, 31) #ifdef HAS_RGBTOUV444MATRIXROW_AVX2 ANY12M(RGBToUV444MatrixRow_Any_AVX2, RGBToUV444MatrixRow_AVX2, 3, 31) #endif -#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW -ANY12M(ARGBToUV444MatrixRow_Any_AVX512BW, ARGBToUV444MatrixRow_AVX512BW, 4, 63) -#endif -#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNI -ANY12M(ARGBToUV444MatrixRow_Any_AVX512VNNI, - ARGBToUV444MatrixRow_AVX512VNNI, - 4, - 63) -#endif -#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI -ANY12M(ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI, - ARGBToUV444MatrixRow_AVX512VNNIVBMI, - 4, - 63) -#endif -#ifdef HAS_RGBTOUV444MATRIXROW_AVX512BW -ANY12M(RGBToUV444MatrixRow_Any_AVX512BW, RGBToUV444MatrixRow_AVX512BW, 3, 63) -#endif #ifdef HAS_ARGBTOUV444MATRIXROW_SSSE3 ANY12M(ARGBToUV444MatrixRow_Any_SSSE3, ARGBToUV444MatrixRow_SSSE3, 4, 15) #endif @@ -2752,21 +2763,6 @@ ANY11MC(ARGB1555ToYMatrixRow_Any_AVX2, ARGB1555ToYMatrixRow_AVX2, 2, 31) ANY11MC(ARGB4444ToYMatrixRow_Any_AVX2, ARGB4444ToYMatrixRow_AVX2, 2, 31) #endif #endif -#ifdef HAS_ARGBTOYMATRIXROW_AVX512BW -ANY11MC(ARGBToYMatrixRow_Any_AVX512BW, ARGBToYMatrixRow_AVX512BW, 4, 63) -#endif -#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNI -ANY11MC(ARGBToYMatrixRow_Any_AVX512VNNI, ARGBToYMatrixRow_AVX512VNNI, 4, 63) -#endif -#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI -ANY11MC(ARGBToYMatrixRow_Any_AVX512VNNIVBMI, - ARGBToYMatrixRow_AVX512VNNIVBMI, - 4, - 63) -#endif -#ifdef HAS_RGBTOYMATRIXROW_AVX512BW -ANY11MC(RGBToYMatrixRow_Any_AVX512BW, RGBToYMatrixRow_AVX512BW, 3, 63) -#endif #ifdef HAS_ARGBTOYMATRIXROW_NEON ANY11MC(ARGBToYMatrixRow_Any_NEON, ARGBToYMatrixRow_NEON, 4, 15) #endif diff --git a/source/row_common.cc b/source/row_common.cc index 951d3c927..48eda2282 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -2878,7 +2878,7 @@ void DetileToYUY2_C(const uint8_t* src_y, // packed into 1x4 blocks, whereas the upper bits are packed in normal raster // order. void UnpackMT2T_C(const uint8_t* src, uint16_t* dst, size_t size) { - for (size_t i = 0; i < size; i += 80) { + while (size >= 80) { const uint8_t* src_lower_bits = src; const uint8_t* src_upper_bits = src + 16; @@ -2892,6 +2892,7 @@ void UnpackMT2T_C(const uint8_t* src, uint16_t* dst, size_t size) { } src += 80; + size -= 80; } } @@ -3942,7 +3943,7 @@ void ByteToFloatRow_C(const uint8_t* src, float* dst, float scale, int width) { int i; for (i = 0; i < width; ++i) { float value = src[i] * scale; - dst[i] = value; + memcpy(&dst[i], &value, sizeof(value)); } } diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 876fdd274..3abe073b0 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -8,6 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include // For memcpy + #include "libyuv/convert_from_argb.h" // For ArgbConstants #include "libyuv/row.h" @@ -215,13 +217,17 @@ void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) { #endif // HAS_J400TOARGBROW_AVX2 #ifdef HAS_J400TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void J400ToARGBRow_AVX512BW(const uint8_t* src_y, uint8_t* dst_argb, int width) { + uintptr_t temp; asm volatile( "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // 0xffffffff "vpslld $0x18,%%zmm16,%%zmm16 \n" // 0xff000000 - "vmovdqu64 %3,%%zmm5 \n" + "vmovdqu64 %[shuf],%%zmm5 \n" + "sub $0x20,%2 \n" + "jl 2f \n" LABELALIGN "1: \n" @@ -236,13 +242,33 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y, "lea 0x20(%0),%0 \n" "lea 0x80(%1),%1 \n" "sub $0x20,%2 \n" - "jg 1b \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "kshiftrd $16,%%k1,%%k2 \n" + "vmovdqu8 (%0),%%ymm0%{%%k1%}%{z%} \n" + "vshufi32x4 $0x55,%%zmm0,%%zmm0,%%zmm1 \n" + "vshufi32x4 $0x00,%%zmm0,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm5,%%zmm1,%%zmm1 \n" + "vpord %%zmm16,%%zmm0,%%zmm0 \n" + "vpord %%zmm16,%%zmm1,%%zmm1 \n" + "vmovdqu32 %%zmm0,(%1)%{%%k1%} \n" + "vmovdqu32 %%zmm1,0x40(%1)%{%%k2%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_y), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "m"(kShuffleMaskJ400ToARGB) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm5", "xmm16"); + : "+r"(src_y), // %0 + "+r"(dst_argb), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : [shuf] "m"(kShuffleMaskJ400ToARGB) // %[shuf] + : "memory", "cc", "xmm0", "xmm1", "xmm5", "xmm16", "k1", "k2"); } #endif // HAS_J400TOARGBROW_AVX512BW @@ -432,10 +458,11 @@ static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, 5, 6, 6, 7}; // defined(HAS_RGBTOYMATRIXROW_AVX512BW) #ifdef HAS_RAWTOARGBROW_AVX512BW -void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, - uint8_t* dst_argb, - const uint32_t* shuffler, - int width) { +LIBYUV_TARGET_AVX512BW +static inline void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, + uint8_t* dst_argb, + const uint32_t* shuffler, + int width) { uintptr_t temp; uintptr_t mask; asm volatile( @@ -456,23 +483,20 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, "vmovdqu8 128(%[src_raw]),%%zmm2 \n" "lea 192(%[src_raw]),%[src_raw] \n" "vpermq %%zmm2,%%zmm23,%%zmm3 \n" - "vmovdqa64 %%zmm22,%%zmm18 \n" - "vpermi2q %%zmm2,%%zmm1,%%zmm18 \n" - "vmovdqa64 %%zmm21,%%zmm19 \n" - "vpermi2q %%zmm1,%%zmm0,%%zmm19 \n" + "vmovdqa64 %%zmm0,%%zmm19 \n" + "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n" + "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n" "vpermq %%zmm0,%%zmm20,%%zmm0 \n" - "vmovdqa64 %%zmm19,%%zmm1 \n" - "vmovdqa64 %%zmm18,%%zmm2 \n" "vpshufb %%zmm4,%%zmm0,%%zmm0 \n" - "vpshufb %%zmm4,%%zmm1,%%zmm1 \n" - "vpshufb %%zmm4,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm4,%%zmm19,%%zmm19 \n" + "vpshufb %%zmm4,%%zmm1,%%zmm2 \n" "vpshufb %%zmm4,%%zmm3,%%zmm3 \n" "vpord %%zmm16,%%zmm0,%%zmm0 \n" - "vpord %%zmm16,%%zmm1,%%zmm1 \n" + "vpord %%zmm16,%%zmm19,%%zmm19 \n" "vpord %%zmm16,%%zmm2,%%zmm2 \n" "vpord %%zmm16,%%zmm3,%%zmm3 \n" "vmovdqu32 %%zmm0,(%[dst_argb]) \n" - "vmovdqu32 %%zmm1,0x40(%[dst_argb]) \n" + "vmovdqu32 %%zmm19,0x40(%[dst_argb]) \n" "vmovdqu32 %%zmm2,0x80(%[dst_argb]) \n" "vmovdqu32 %%zmm3,0xc0(%[dst_argb]) \n" "lea 0x100(%[dst_argb]),%[dst_argb]\n" @@ -481,7 +505,7 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, "2: \n" "add $0x40,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[mask] \n" "kmovq %[mask],%%k1 \n" @@ -534,10 +558,11 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, [q2] "m"(kPermqRGB24ToARGB2_AVX512BW), // %[q2] [q3] "m"(kPermqRGB24ToARGB3_AVX512BW) // %[q3] : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", - "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1", "k2", "k3", - "k4", "k5"); + "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1", "k2", "k3", "k4", + "k5"); } +LIBYUV_TARGET_AVX512BW void RAWToARGBRow_AVX512BW(const uint8_t* src_raw, uint8_t* dst_argb, int width) { @@ -546,6 +571,7 @@ void RAWToARGBRow_AVX512BW(const uint8_t* src_raw, width); } +LIBYUV_TARGET_AVX512BW void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, uint8_t* dst_argb, int width) { @@ -579,10 +605,10 @@ void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { "sub $0x10,%2 \n" "jg 1b \n" "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_rgba), // %1 - "+r"(width) // %2 - : "m"(kShuffleMaskRAWToRGBA_AVX2[0]) // %3 + : "+r"(src_raw), // %0 + "+r"(dst_rgba), // %1 + "+r"(width) // %2 + : "m"(kShuffleMaskRAWToRGBA_AVX2) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm3", "xmm4", "xmm5"); } #endif // HAS_RAWTORGBAROW_AVX2 @@ -670,38 +696,43 @@ void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { "sub $32,%2 \n" "jg 1b \n" "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_rgb24), // %1 - "+r"(width) // %2 - : "m"(kShuffleMaskRAWToRGB24_AVX2[0]), // %3 - "m"(kPermd0_AVX2), // %4 - "m"(kPermd1_AVX2), // %5 - "m"(kPermd2_AVX2) // %6 + : "+r"(src_raw), // %0 + "+r"(dst_rgb24), // %1 + "+r"(width) // %2 + : "m"(kShuffleMaskRAWToRGB24_AVX2), // %3 + "m"(kPermd0_AVX2), // %4 + "m"(kPermd1_AVX2), // %5 + "m"(kPermd2_AVX2) // %6 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } #endif // HAS_RAWTORGB24ROW_AVX2 #ifdef HAS_RAWTORGB24ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { + uintptr_t temp; + uintptr_t mask; asm volatile( - "vmovdqu64 %3,%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW - "vmovdqu64 %4,%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW - "vmovdqu64 %5,%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW - "vmovdqu64 %6,%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW - "vbroadcasti64x4 %7,%%zmm4 \n" // shuffler (32 bytes) - "vmovdqu32 %8,%%zmm24 \n" // kPermdRGB24_0_AVX512BW - "vmovdqu32 %9,%%zmm25 \n" // kPermdRGB24_1_AVX512BW - "vmovdqu32 %10,%%zmm26 \n" // kPermdRGB24_2_AVX512BW + "vmovdqu64 %[q0],%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW + "vmovdqu64 %[q1],%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW + "vmovdqu64 %[q2],%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW + "vmovdqu64 %[q3],%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW + "vbroadcasti64x4 %[shuf],%%zmm4 \n" // shuffler (32 bytes) + "vmovdqu32 %[d0],%%zmm24 \n" // kPermdRGB24_0_AVX512BW + "vmovdqu32 %[d1],%%zmm25 \n" // kPermdRGB24_1_AVX512BW + "vmovdqu32 %[d2],%%zmm26 \n" // kPermdRGB24_2_AVX512BW + "sub $0x40,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" - "vmovdqu8 (%0),%%zmm0 \n" - "vmovdqu8 64(%0),%%zmm1 \n" - "vmovdqu8 128(%0),%%zmm2 \n" - "lea 192(%0),%0 \n" + "vmovdqu8 (%[src_raw]),%%zmm0 \n" + "vmovdqu8 64(%[src_raw]),%%zmm1 \n" + "vmovdqu8 128(%[src_raw]),%%zmm2 \n" + "lea 192(%[src_raw]),%[src_raw] \n" "vpermq %%zmm2,%%zmm23,%%zmm3 \n" "vmovdqa64 %%zmm0,%%zmm19 \n" "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n" @@ -714,66 +745,142 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, "vpermt2d %%zmm19,%%zmm24,%%zmm0 \n" "vpermt2d %%zmm2,%%zmm25,%%zmm19 \n" "vpermt2d %%zmm3,%%zmm26,%%zmm2 \n" - "vmovdqu8 %%zmm0,(%1) \n" - "vmovdqu8 %%zmm19,0x40(%1) \n" - "vmovdqu8 %%zmm2,0x80(%1) \n" - "lea 0xc0(%1),%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" + "vmovdqu8 %%zmm0,(%[dst_rgb24]) \n" + "vmovdqu8 %%zmm19,0x40(%[dst_rgb24]) \n" + "vmovdqu8 %%zmm2,0x80(%[dst_rgb24]) \n" + "lea 0xc0(%[dst_rgb24]),%[dst_rgb24]\n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "lea (%q[width],%q[width],2),%k[width] \n" + "bzhi %q[width],%[temp],%[mask] \n" + "kmovq %[mask],%%k1 \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k2 \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k3 \n" + "vmovdqu8 (%[src_raw]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu8 64(%[src_raw]),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu8 128(%[src_raw]),%%zmm2%{%%k3%}%{z%} \n" + "vpermq %%zmm2,%%zmm23,%%zmm3 \n" + "vmovdqa64 %%zmm0,%%zmm19 \n" + "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n" + "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n" + "vpermq %%zmm0,%%zmm20,%%zmm0 \n" + "vpshufb %%zmm4,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm4,%%zmm19,%%zmm19 \n" + "vpshufb %%zmm4,%%zmm1,%%zmm2 \n" + "vpshufb %%zmm4,%%zmm3,%%zmm3 \n" + "vpermt2d %%zmm19,%%zmm24,%%zmm0 \n" + "vpermt2d %%zmm2,%%zmm25,%%zmm19 \n" + "vpermt2d %%zmm3,%%zmm26,%%zmm2 \n" + "vmovdqu8 %%zmm0,(%[dst_rgb24])%{%%k1%} \n" + "vmovdqu8 %%zmm19,0x40(%[dst_rgb24])%{%%k2%} \n" + "vmovdqu8 %%zmm2,0x80(%[dst_rgb24])%{%%k3%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_rgb24), // %1 - "+r"(width) // %2 - : "m"(kPermqRGB24ToARGB0_AVX512BW), // %3 - "m"(kPermqRGB24ToARGB1_AVX512BW), // %4 - "m"(kPermqRGB24ToARGB2_AVX512BW), // %5 - "m"(kPermqRGB24ToARGB3_AVX512BW), // %6 - "m"(kShuffleMaskRAWToRGB24_AVX2[0]), // %7 - "m"(kPermdRGB24_0_AVX512BW), // %8 - "m"(kPermdRGB24_1_AVX512BW), // %9 - "m"(kPermdRGB24_2_AVX512BW) // %10 + : [src_raw] "+r"(src_raw), // %[src_raw] + [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] + [width] "+r"(width), // %[width] + [temp] "=&r"(temp), // %[temp] + [mask] "=&r"(mask) // %[mask] + : [q0] "m"(kPermqRGB24ToARGB0_AVX512BW), // %[q0] + [q1] "m"(kPermqRGB24ToARGB1_AVX512BW), // %[q1] + [q2] "m"(kPermqRGB24ToARGB2_AVX512BW), // %[q2] + [q3] "m"(kPermqRGB24ToARGB3_AVX512BW), // %[q3] + [shuf] "m"(kShuffleMaskRAWToRGB24_AVX2), // %[shuf] + [d0] "m"(kPermdRGB24_0_AVX512BW), // %[d0] + [d1] "m"(kPermdRGB24_1_AVX512BW), // %[d1] + [d2] "m"(kPermdRGB24_2_AVX512BW) // %[d2] : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm19", - "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26"); + "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26", "k1", + "k2", "k3"); } #endif // HAS_RAWTORGB24ROW_AVX512BW #ifdef HAS_RAWTORGB24ROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { + uintptr_t temp; + uintptr_t mask; asm volatile( "vmovdqu32 %[kPermRAWToRGB24_0],%%zmm20 \n" "vmovdqu32 %[kPermRAWToRGB24_1],%%zmm21 \n" "vmovdqu32 %[kPermRAWToRGB24_2],%%zmm22 \n" "vmovdqu32 %[kPermRAWToRGB24_3],%%zmm23 \n" + "sub $0x40,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" - "vmovdqu8 (%0),%%zmm0 \n" - "vmovdqu8 64(%0),%%zmm1 \n" - "vmovdqu8 128(%0),%%zmm2 \n" - "lea 192(%0),%0 \n" + "vmovdqu8 (%[src_raw]),%%zmm0 \n" + "vmovdqu8 64(%[src_raw]),%%zmm1 \n" + "vmovdqu8 128(%[src_raw]),%%zmm2 \n" + "lea 192(%[src_raw]),%[src_raw] \n" "vmovdqa64 %%zmm1,%%zmm3 \n" "vpermt2b %%zmm0,%%zmm21,%%zmm1 \n" "vpermt2b %%zmm3,%%zmm20,%%zmm0 \n" "vpermt2b %%zmm2,%%zmm23,%%zmm3 \n" "vpermt2b %%zmm2,%%zmm22,%%zmm1 \n" - "vmovdqu8 %%zmm0,(%1) \n" - "vmovdqu8 %%zmm1,0x40(%1) \n" - "vmovdqu8 %%zmm3,0x80(%1) \n" - "lea 0xc0(%1),%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" + "vmovdqu8 %%zmm0,(%[dst_rgb24]) \n" + "vmovdqu8 %%zmm1,0x40(%[dst_rgb24]) \n" + "vmovdqu8 %%zmm3,0x80(%[dst_rgb24]) \n" + "lea 0xc0(%[dst_rgb24]),%[dst_rgb24]\n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "lea (%q[width],%q[width],2),%k[width] \n" + "bzhi %q[width],%[temp],%[mask] \n" + "kmovq %[mask],%%k1 \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k2 \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k3 \n" + "vmovdqu8 (%[src_raw]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu8 64(%[src_raw]),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu8 128(%[src_raw]),%%zmm2%{%%k3%}%{z%} \n" + "vmovdqa64 %%zmm1,%%zmm3 \n" + "vpermt2b %%zmm0,%%zmm21,%%zmm1 \n" + "vpermt2b %%zmm3,%%zmm20,%%zmm0 \n" + "vpermt2b %%zmm2,%%zmm23,%%zmm3 \n" + "vpermt2b %%zmm2,%%zmm22,%%zmm1 \n" + "vmovdqu8 %%zmm0,(%[dst_rgb24])%{%%k1%} \n" + "vmovdqu8 %%zmm1,0x40(%[dst_rgb24])%{%%k2%} \n" + "vmovdqu8 %%zmm3,0x80(%[dst_rgb24])%{%%k3%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_rgb24), // %1 - "+r"(width) // %2 + : [src_raw] "+r"(src_raw), [dst_rgb24] "+r"(dst_rgb24), + [width] "+r"(width), [temp] "=&r"(temp), [mask] "=&r"(mask) : [kPermRAWToRGB24_0] "m"(kPermRAWToRGB24_0), [kPermRAWToRGB24_1] "m"(kPermRAWToRGB24_1), [kPermRAWToRGB24_2] "m"(kPermRAWToRGB24_2), [kPermRAWToRGB24_3] "m"(kPermRAWToRGB24_3) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm20", "xmm21", - "xmm22", "xmm23"); + "xmm22", "xmm23", "k1", "k2", "k3"); } #endif // HAS_RAWTORGB24ROW_AVX512VBMI @@ -879,6 +986,7 @@ void ARGB1555ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { #endif // HAS_ARGB1555TOARGBROW_AVX2 #ifdef HAS_ARGB1555TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { uintptr_t temp; asm volatile( @@ -924,7 +1032,7 @@ void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { "2: \n" "add $0x20,%2 \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%k[temp] \n" "bzhi %k2,%k[temp],%k[temp] \n" "kmovd %k[temp],%%k1 \n" @@ -1136,6 +1244,7 @@ void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { #endif // HAS_ARGBTORGB24ROW_AVX2 #if defined(HAS_ARGBTORGB24ROW_AVX512BW) || defined(HAS_ARGBTORAWROW_AVX512BW) +LIBYUV_TARGET_AVX512BW static inline void ARGBToRGBRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width, @@ -1172,7 +1281,7 @@ static inline void ARGBToRGBRow_AVX512BW(const uint8_t* src, "2: \n" "add $0x40,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" @@ -1220,8 +1329,7 @@ static inline void ARGBToRGBRow_AVX512BW(const uint8_t* src, [dst] "+r"(dst), // %[dst] [width] "+r"(width), // %[width] [temp] "=&r"(temp) // %[temp] - : [shuffler] "m"(shuffler), - [kPermdRGB24_0] "m"(kPermdRGB24_0_AVX512BW), + : [shuffler] "m"(shuffler), [kPermdRGB24_0] "m"(kPermdRGB24_0_AVX512BW), [kPermdRGB24_1] "m"(kPermdRGB24_1_AVX512BW), [kPermdRGB24_2] "m"(kPermdRGB24_2_AVX512BW) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", @@ -1231,6 +1339,7 @@ static inline void ARGBToRGBRow_AVX512BW(const uint8_t* src, // defined(HAS_ARGBTORAWROW_AVX512BW) #ifdef HAS_ARGBTORGB24ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void ARGBToRGB24Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRGB24[0]); } @@ -1283,6 +1392,7 @@ void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { #endif // HAS_ARGBTORAWROW_AVX2 #ifdef HAS_ARGBTORAWROW_AVX512BW +LIBYUV_TARGET_AVX512BW void ARGBToRAWRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRAW); } @@ -1335,6 +1445,7 @@ void ARGBToRGB565DitherRow_AVX2(const uint8_t* src, #endif // HAS_ARGBTORGB565DITHERROW_AVX2 #ifdef HAS_ARGBTORGB565DITHERROW_AVX512BW +LIBYUV_TARGET_AVX512BW void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src, uint8_t* dst, uint32_t dither4, @@ -1373,7 +1484,7 @@ void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src, "2: \n" "add $0x10,%2 \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%k[temp] \n" "bzhi %k2,%k[temp],%k[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -1438,6 +1549,7 @@ void ARGBToRGB565Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { #endif // HAS_ARGBTORGB565ROW_AVX2 #ifdef HAS_ARGBTORGB565ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void ARGBToRGB565Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { uintptr_t temp; asm volatile( @@ -1468,7 +1580,7 @@ void ARGBToRGB565Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { "2: \n" "add $0x10,%2 \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%k[temp] \n" "bzhi %k2,%k[temp],%k[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -1607,6 +1719,7 @@ static const uint32_t kMaskRB10 = 0x3ff003ff; static const uint32_t kMaskAG10 = 0xc000ff00; static const uint32_t kMulAG10 = 64 * 65536 + 1028; +#ifdef HAS_ARGBTOAR30ROW_SSSE3 void ARGBToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) { asm volatile( "movdqa %3,%%xmm2 \n" // shuffler for RB @@ -1645,7 +1758,9 @@ void ARGBToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) { "m"(kMulAG10) // %7 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } +#endif // HAS_ARGBTOAR30ROW_SSSE3 +#ifdef HAS_ABGRTOAR30ROW_SSSE3 void ABGRToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) { asm volatile( "movdqa %3,%%xmm2 \n" // shuffler for RB @@ -1684,6 +1799,7 @@ void ABGRToAR30Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) { "m"(kMulAG10) // %7 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } +#endif // HAS_ABGRTOAR30ROW_SSSE3 #ifdef HAS_ARGBTOAR30ROW_AVX2 void ARGBToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { @@ -1759,14 +1875,19 @@ void ABGRToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ABGRTOAR30ROW_AVX2 +#if defined(HAS_AB64TOARGBROW_SSSE3) || defined(HAS_AB64TOARGBROW_AVX2) static const uvec8 kShuffleARGBToABGR = {2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15}; +#endif +#if defined(HAS_ARGBTOAB64ROW_SSSE3) || defined(HAS_ARGBTOAB64ROW_AVX2) static const uvec8 kShuffleARGBToAB64Lo = {2, 2, 1, 1, 0, 0, 3, 3, 6, 6, 5, 5, 4, 4, 7, 7}; static const uvec8 kShuffleARGBToAB64Hi = {10, 10, 9, 9, 8, 8, 11, 11, 14, 14, 13, 13, 12, 12, 15, 15}; +#endif +#ifdef HAS_ARGBTOAR64ROW_SSSE3 void ARGBToAR64Row_SSSE3(const uint8_t* src_argb, uint16_t* dst_ar64, int width) { @@ -1788,7 +1909,9 @@ void ARGBToAR64Row_SSSE3(const uint8_t* src_argb, ::"memory", "cc", "xmm0", "xmm1"); } +#endif // HAS_ARGBTOAR64ROW_SSSE3 +#ifdef HAS_ARGBTOAB64ROW_SSSE3 void ARGBToAB64Row_SSSE3(const uint8_t* src_argb, uint16_t* dst_ab64, int width) { @@ -1813,7 +1936,9 @@ void ARGBToAB64Row_SSSE3(const uint8_t* src_argb, "m"(kShuffleARGBToAB64Hi) // %4 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } +#endif // HAS_ARGBTOAB64ROW_SSSE3 +#ifdef HAS_AR64TOARGBROW_SSSE3 void AR64ToARGBRow_SSSE3(const uint16_t* src_ar64, uint8_t* dst_argb, int width) { @@ -1835,7 +1960,9 @@ void AR64ToARGBRow_SSSE3(const uint16_t* src_ar64, ::"memory", "cc", "xmm0", "xmm1"); } +#endif // HAS_AR64TOARGBROW_SSSE3 +#ifdef HAS_AB64TOARGBROW_SSSE3 void AB64ToARGBRow_SSSE3(const uint16_t* src_ab64, uint8_t* dst_argb, int width) { @@ -1860,6 +1987,7 @@ void AB64ToARGBRow_SSSE3(const uint16_t* src_ab64, : "m"(kShuffleARGBToABGR) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2"); } +#endif // HAS_AB64TOARGBROW_SSSE3 #ifdef HAS_ARGBTOAR64ROW_AVX2 void ARGBToAR64Row_AVX2(const uint8_t* src_argb, @@ -2052,8 +2180,8 @@ void ARGBToYMatrixRow_SSSE3(const uint8_t* src_argb, "pcmpeqb %%xmm5,%%xmm5 \n" "psllw $15,%%xmm5 \n" "packsswb %%xmm5,%%xmm5 \n" - "movdqa 0(%3),%%xmm4 \n" - "movdqa 0x60(%3),%%xmm7 \n" + "movdqu 0(%3),%%xmm4 \n" + "movdqu 0x60(%3),%%xmm7 \n" "movdqa %%xmm4,%%xmm6 \n" "pmaddubsw %%xmm5,%%xmm6 \n" "phaddw %%xmm6,%%xmm6 \n" @@ -2120,27 +2248,33 @@ static const uint32_t kPermdARGBToUV_AVX512BW[16] = { // defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI) #ifdef HAS_ARGBTOYMATRIXROW_AVX512BW +LIBYUV_TARGET_AVX512BW void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_y, int width, const struct ArgbConstants* c) { + uintptr_t temp; asm volatile( "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" "vpsllw $15,%%zmm16,%%zmm5 \n" "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80 "vpabsw %%zmm16,%%zmm16 \n" // zmm16 = 1 - "vbroadcasti32x4 0(%3),%%zmm4 \n" - "vbroadcasti32x4 0x60(%3),%%zmm17 \n" + "vbroadcasti32x4 0(%[c]),%%zmm4 \n" + "vbroadcasti32x4 0x60(%[c]),%%zmm17 \n" "vpmaddubsw %%zmm5,%%zmm4,%%zmm18 \n" "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" "vpackssdw %%zmm18,%%zmm18,%%zmm18 \n" "vpsubw %%zmm18,%%zmm17,%%zmm17 \n" - "vmovups %4,%%zmm18 \n" LABELALIGN + "vmovdqu32 %[perm],%%zmm18 \n" + "sub $0x40,%[width] \n" + "jl 2f \n" + + LABELALIGN "1: \n" - "vmovups (%0),%%zmm0 \n" - "vmovups 0x40(%0),%%zmm1 \n" - "vmovups 0x80(%0),%%zmm2 \n" - "vmovups 0xc0(%0),%%zmm3 \n" + "vmovdqu8 (%[src_argb]),%%zmm0 \n" + "vmovdqu8 0x40(%[src_argb]),%%zmm1 \n" + "vmovdqu8 0x80(%[src_argb]),%%zmm2 \n" + "vmovdqu8 0xc0(%[src_argb]),%%zmm3 \n" "vpsubb %%zmm5,%%zmm0,%%zmm0 \n" "vpsubb %%zmm5,%%zmm1,%%zmm1 \n" "vpsubb %%zmm5,%%zmm2,%%zmm2 \n" @@ -2149,7 +2283,7 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, "vpmaddubsw %%zmm1,%%zmm4,%%zmm1 \n" "vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n" "vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n" - "lea 0x100(%0),%0 \n" + "lea 0x100(%[src_argb]),%[src_argb]\n" "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" @@ -2162,43 +2296,83 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" "vpermd %%zmm0,%%zmm18,%%zmm0 \n" - "vmovups %%zmm0,(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" + "vmovdqu8 %%zmm0,(%[dst_y]) \n" + "lea 0x40(%[dst_y]),%[dst_y] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "kshiftrq $16,%%k1,%%k2 \n" + "kshiftrq $32,%%k1,%%k3 \n" + "kshiftrq $48,%%k1,%%k4 \n" + "vmovdqu32 (%[src_argb]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src_argb]),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu32 0x80(%[src_argb]),%%zmm2%{%%k3%}%{z%} \n" + "vmovdqu32 0xc0(%[src_argb]),%%zmm3%{%%k4%}%{z%} \n" + "vpsubb %%zmm5,%%zmm0,%%zmm0 \n" + "vpsubb %%zmm5,%%zmm1,%%zmm1 \n" + "vpsubb %%zmm5,%%zmm2,%%zmm2 \n" + "vpsubb %%zmm5,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm0,%%zmm4,%%zmm0 \n" + "vpmaddubsw %%zmm1,%%zmm4,%%zmm1 \n" + "vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n" + "vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n" + "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" + "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" + "vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n" + "vpackssdw %%zmm3,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm17,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm18,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst_y])%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_argb), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(c), // %3 - "m"(kPermdARGBToY_AVX512BW) // %4 + : [src_argb] "+r"(src_argb), [dst_y] "+r"(dst_y), [width] "+r"(width), + [temp] "=&r"(temp) + : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", - "xmm17", "xmm18"); + "xmm17", "xmm18", "k1", "k2", "k3", "k4"); } #endif // HAS_ARGBTOYMATRIXROW_AVX512BW #ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb, uint8_t* dst_y, int width, const struct ArgbConstants* c) { + uintptr_t temp; asm volatile( "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" "vpsllw $15,%%zmm16,%%zmm5 \n" "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80 - "vbroadcasti32x4 0(%3),%%zmm4 \n" // kRGBToY (unsigned) - "vpmovzxwd 0x60(%3),%%zmm17 \n" // kAddY (32-bit) + "vbroadcasti32x4 0(%[c]),%%zmm4 \n" // kRGBToY (unsigned) + "vpmovzxwd 0x60(%[c]),%%zmm17 \n" // kAddY (32-bit) "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" "vpdpbusd %%zmm5,%%zmm4,%%zmm0 \n" // -128 * sum(c) "vpsubd %%zmm0,%%zmm17,%%zmm17 \n" // kAddY + 128 * sum(c) - "vmovups %4,%%zmm16 \n" // kPermdARGBToY_AVX512BW + "vmovdqu32 %[perm],%%zmm16 \n" // kPermdARGBToY_AVX512BW + "sub $0x40,%[width] \n" + "jl 2f \n" + LABELALIGN "1: \n" - "vmovups (%0),%%zmm0 \n" - "vmovups 0x40(%0),%%zmm1 \n" - "vmovups 0x80(%0),%%zmm2 \n" - "vmovups 0xc0(%0),%%zmm3 \n" - "lea 0x100(%0),%0 \n" + "vmovdqu8 (%[src_argb]),%%zmm0 \n" + "vmovdqu8 0x40(%[src_argb]),%%zmm1 \n" + "vmovdqu8 0x80(%[src_argb]),%%zmm2 \n" + "vmovdqu8 0xc0(%[src_argb]),%%zmm3 \n" + "lea 0x100(%[src_argb]),%[src_argb]\n" "vpxorq %%zmm5,%%zmm0,%%zmm0 \n" "vpxorq %%zmm5,%%zmm1,%%zmm1 \n" "vpxorq %%zmm5,%%zmm2,%%zmm2 \n" @@ -2217,18 +2391,51 @@ void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb, "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" "vpermd %%zmm0,%%zmm16,%%zmm0 \n" - "vmovups %%zmm0,(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" + "vmovdqu8 %%zmm0,(%[dst_y]) \n" + "lea 0x40(%[dst_y]),%[dst_y] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "kshiftrq $16,%%k1,%%k2 \n" + "kshiftrq $32,%%k1,%%k3 \n" + "kshiftrq $48,%%k1,%%k4 \n" + "vmovdqu32 (%[src_argb]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src_argb]),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu32 0x80(%[src_argb]),%%zmm2%{%%k3%}%{z%} \n" + "vmovdqu32 0xc0(%[src_argb]),%%zmm3%{%%k4%}%{z%} \n" + "vpxorq %%zmm5,%%zmm0,%%zmm0 \n" + "vpxorq %%zmm5,%%zmm1,%%zmm1 \n" + "vpxorq %%zmm5,%%zmm2,%%zmm2 \n" + "vpxorq %%zmm5,%%zmm3,%%zmm3 \n" + "vmovdqa64 %%zmm17,%%zmm18 \n" + "vmovdqa64 %%zmm17,%%zmm19 \n" + "vmovdqa64 %%zmm17,%%zmm20 \n" + "vmovdqa64 %%zmm17,%%zmm21 \n" + "vpdpbusd %%zmm0,%%zmm4,%%zmm18 \n" + "vpdpbusd %%zmm1,%%zmm4,%%zmm19 \n" + "vpdpbusd %%zmm2,%%zmm4,%%zmm20 \n" + "vpdpbusd %%zmm3,%%zmm4,%%zmm21 \n" + "vpackusdw %%zmm19,%%zmm18,%%zmm0 \n" + "vpackusdw %%zmm21,%%zmm20,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm16,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst_y])%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_argb), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(c), // %3 - "m"(kPermdARGBToY_AVX512BW) // %4 + : [src_argb] "+r"(src_argb), [dst_y] "+r"(dst_y), [width] "+r"(width), + [temp] "=&r"(temp) + : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", - "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4"); } #endif // HAS_ARGBTOYMATRIXROW_AVX512VNNI @@ -2250,105 +2457,142 @@ static const uint8_t kPermbDWordByte1_AVX512VNNIVBMI[32] = { // Requires AVX512VNNI and AVX512VBMI. // The 32 bit sums are kAddY + dot(kRGBToY, argb) which is always in // [0, 65535], so Y is byte 1 of each int32 and no saturation is needed. +LIBYUV_TARGET_AVX512VNNIVBMI void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, uint8_t* dst_y, int width, const struct ArgbConstants* c) { + uintptr_t temp; asm volatile( "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" "vpsllw $15,%%zmm16,%%zmm5 \n" "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80 - "vbroadcasti32x4 0(%3),%%zmm4 \n" // kRGBToY (unsigned) - "vpmovzxwd 0x60(%3),%%zmm17 \n" // kAddY (32-bit) + "vbroadcasti32x4 0(%[c]),%%zmm4 \n" // kRGBToY (unsigned) + "vpmovzxwd 0x60(%[c]),%%zmm17 \n" // kAddY (32-bit) "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" "vpdpbusd %%zmm5,%%zmm4,%%zmm0 \n" // -128 * sum(c) "vpsubd %%zmm0,%%zmm17,%%zmm17 \n" // kAddY + 128 * sum(c) - "vbroadcasti64x4 %4,%%zmm16 \n" // byte 1 of int32 + "vbroadcasti64x4 %[perm],%%zmm16 \n" // byte 1 of int32 + "sub $0x40,%[width] \n" + "jl 2f \n" + LABELALIGN "1: \n" - "vpxorq (%0),%%zmm5,%%zmm0 \n" + "vpxorq (%[src_argb]),%%zmm5,%%zmm0 \n" "vmovdqa64 %%zmm17,%%zmm18 \n" "vpdpbusd %%zmm0,%%zmm4,%%zmm18 \n" - "vpxorq 0x40(%0),%%zmm5,%%zmm1 \n" + "vpxorq 0x40(%[src_argb]),%%zmm5,%%zmm1 \n" "vmovdqa64 %%zmm17,%%zmm19 \n" "vpdpbusd %%zmm1,%%zmm4,%%zmm19 \n" - "vpxorq 0x80(%0),%%zmm5,%%zmm2 \n" + "vpxorq 0x80(%[src_argb]),%%zmm5,%%zmm2 \n" "vmovdqa64 %%zmm17,%%zmm20 \n" "vpdpbusd %%zmm2,%%zmm4,%%zmm20 \n" - "vpxorq 0xc0(%0),%%zmm5,%%zmm3 \n" + "vpxorq 0xc0(%[src_argb]),%%zmm5,%%zmm3 \n" "vmovdqa64 %%zmm17,%%zmm21 \n" "vpdpbusd %%zmm3,%%zmm4,%%zmm21 \n" - "lea 0x100(%0),%0 \n" + "lea 0x100(%[src_argb]),%[src_argb]\n" "vpermt2b %%zmm19,%%zmm16,%%zmm18 \n" // Y 0..31 "vpermt2b %%zmm21,%%zmm16,%%zmm20 \n" // Y 32..63 - "vmovdqu64 %%ymm18,(%1) \n" - "vmovdqu64 %%ymm20,0x20(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" + "vmovdqu8 %%ymm18,(%[dst_y]) \n" + "vmovdqu8 %%ymm20,0x20(%[dst_y]) \n" + "lea 0x40(%[dst_y]),%[dst_y] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "kshiftrq $16,%%k1,%%k2 \n" + "kshiftrq $32,%%k1,%%k3 \n" + "kshiftrq $48,%%k1,%%k4 \n" + "vmovdqu32 (%[src_argb]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src_argb]),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu32 0x80(%[src_argb]),%%zmm2%{%%k3%}%{z%} \n" + "vmovdqu32 0xc0(%[src_argb]),%%zmm3%{%%k4%}%{z%} \n" + "vpxorq %%zmm5,%%zmm0,%%zmm0 \n" + "vpxorq %%zmm5,%%zmm1,%%zmm1 \n" + "vpxorq %%zmm5,%%zmm2,%%zmm2 \n" + "vpxorq %%zmm5,%%zmm3,%%zmm3 \n" + "vmovdqa64 %%zmm17,%%zmm18 \n" + "vmovdqa64 %%zmm17,%%zmm19 \n" + "vmovdqa64 %%zmm17,%%zmm20 \n" + "vmovdqa64 %%zmm17,%%zmm21 \n" + "vpdpbusd %%zmm0,%%zmm4,%%zmm18 \n" + "vpdpbusd %%zmm1,%%zmm4,%%zmm19 \n" + "vpdpbusd %%zmm2,%%zmm4,%%zmm20 \n" + "vpdpbusd %%zmm3,%%zmm4,%%zmm21 \n" + "vpermt2b %%zmm19,%%zmm16,%%zmm18 \n" + "vpermt2b %%zmm21,%%zmm16,%%zmm20 \n" + "vmovdqu8 %%ymm18,(%[dst_y])%{%%k1%} \n" + "vmovdqu8 %%ymm20,0x20(%[dst_y])%{%%k3%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_argb), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(c), // %3 - "m"(kPermbDWordByte1_AVX512VNNIVBMI) // %4 + : [src_argb] "+r"(src_argb), [dst_y] "+r"(dst_y), [width] "+r"(width), + [temp] "=&r"(temp) + : [c] "r"(c), [perm] "m"(kPermbDWordByte1_AVX512VNNIVBMI) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", - "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4"); } #endif // HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI #ifdef HAS_RGBTOYMATRIXROW_AVX512BW +LIBYUV_TARGET_AVX512BW void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, uint8_t* dst_y, int width, const struct ArgbConstants* c) { + uintptr_t temp; + uintptr_t mask; asm volatile( - "vmovdqu64 %5,%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW - "vmovdqu64 %6,%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW - "vmovdqu64 %7,%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW - "vmovdqu64 %8,%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW - "vbroadcasti64x4 %9,%%zmm24 \n" // kShuffleMaskRGB24ToARGB + "vmovdqu64 %[q0],%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW + "vmovdqu64 %[q1],%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW + "vmovdqu64 %[q2],%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW + "vmovdqu64 %[q3],%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW + "vbroadcasti64x4 %[shuf],%%zmm24 \n" // kShuffleMaskRGB24ToARGB "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" "vpsllw $15,%%zmm16,%%zmm5 \n" "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80 "vpabsw %%zmm16,%%zmm16 \n" // zmm16 = 1 - "vbroadcasti32x4 0(%3),%%zmm4 \n" // zmm4 = kRGBToY - "vbroadcasti32x4 0x60(%3),%%zmm17 \n" // zmm17 = kAddY - "vpmaddubsw %%zmm5,%%zmm4,%%zmm18 \n" - "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" - "vpackssdw %%zmm18,%%zmm18,%%zmm18 \n" - "vpsubw %%zmm18,%%zmm17,%%zmm17 \n" // zmm17 = adjusted bias - "vmovups %4,%%zmm25 \n" // zmm25 = - // kPermdARGBToY_AVX512BW + "vbroadcasti32x4 0(%[c]),%%zmm4 \n" // zmm4 = kRGBToY + "vbroadcasti32x4 0x60(%[c]),%%zmm17 \n" // zmm17 = kAddY + "vpmaddubsw %%zmm5,%%zmm4,%%zmm19 \n" + "vpmaddwd %%zmm16,%%zmm19,%%zmm19 \n" + "vpackssdw %%zmm19,%%zmm19,%%zmm19 \n" + "vpsubw %%zmm19,%%zmm17,%%zmm17 \n" // zmm17 = adjusted bias + "vmovdqu32 %[perm],%%zmm25 \n" // kPermdARGBToY_AVX512BW + "sub $0x40,%[width] \n" + "jl 2f \n" + LABELALIGN "1: \n" - "vmovdqu8 (%0),%%zmm0 \n" - "vmovdqu8 64(%0),%%zmm1 \n" - "vmovdqu8 128(%0),%%zmm2 \n" - "lea 192(%0),%0 \n" + "vmovdqu8 (%[src_rgb]),%%zmm0 \n" + "vmovdqu8 64(%[src_rgb]),%%zmm1 \n" + "vmovdqu8 128(%[src_rgb]),%%zmm2 \n" + "lea 192(%[src_rgb]),%[src_rgb] \n" "vpermq %%zmm2,%%zmm23,%%zmm3 \n" - "vmovdqa64 %%zmm22,%%zmm18 \n" - "vpermi2q %%zmm2,%%zmm1,%%zmm18 \n" - "vmovdqa64 %%zmm21,%%zmm19 \n" - "vpermi2q %%zmm1,%%zmm0,%%zmm19 \n" + "vmovdqa64 %%zmm0,%%zmm19 \n" + "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n" + "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n" "vpermq %%zmm0,%%zmm20,%%zmm0 \n" - "vmovdqa64 %%zmm19,%%zmm1 \n" - "vmovdqa64 %%zmm18,%%zmm2 \n" "vpshufb %%zmm24,%%zmm0,%%zmm0 \n" - "vpshufb %%zmm24,%%zmm1,%%zmm1 \n" - "vpshufb %%zmm24,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm24,%%zmm19,%%zmm19 \n" + "vpshufb %%zmm24,%%zmm1,%%zmm2 \n" "vpshufb %%zmm24,%%zmm3,%%zmm3 \n" "vpsubb %%zmm5,%%zmm0,%%zmm0 \n" - "vpsubb %%zmm5,%%zmm1,%%zmm1 \n" + "vpsubb %%zmm5,%%zmm19,%%zmm19 \n" "vpsubb %%zmm5,%%zmm2,%%zmm2 \n" "vpsubb %%zmm5,%%zmm3,%%zmm3 \n" "vpmaddubsw %%zmm0,%%zmm4,%%zmm0 \n" - "vpmaddubsw %%zmm1,%%zmm4,%%zmm1 \n" + "vpmaddubsw %%zmm19,%%zmm4,%%zmm19 \n" "vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n" "vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n" "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" - "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" - "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm19,%%zmm19 \n" + "vpackssdw %%zmm19,%%zmm0,%%zmm0 \n" "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" "vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n" "vpackssdw %%zmm3,%%zmm2,%%zmm2 \n" @@ -2358,24 +2602,77 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" "vpermd %%zmm0,%%zmm25,%%zmm0 \n" - "vmovups %%zmm0,(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" + "vmovdqu8 %%zmm0,(%[dst_y]) \n" + "lea 0x40(%[dst_y]),%[dst_y] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[mask] \n" + "kmovq %[mask],%%k4 \n" + "lea (%q[width],%q[width],2),%k[width] \n" + "bzhi %q[width],%[temp],%[mask] \n" + "kmovq %[mask],%%k1 \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k2 \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k3 \n" + "vmovdqu8 (%[src_rgb]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu8 64(%[src_rgb]),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu8 128(%[src_rgb]),%%zmm2%{%%k3%}%{z%} \n" + "vpermq %%zmm2,%%zmm23,%%zmm3 \n" + "vmovdqa64 %%zmm0,%%zmm19 \n" + "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n" + "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n" + "vpermq %%zmm0,%%zmm20,%%zmm0 \n" + "vpshufb %%zmm24,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm24,%%zmm19,%%zmm19 \n" + "vpshufb %%zmm24,%%zmm1,%%zmm2 \n" + "vpshufb %%zmm24,%%zmm3,%%zmm3 \n" + "vpsubb %%zmm5,%%zmm0,%%zmm0 \n" + "vpsubb %%zmm5,%%zmm19,%%zmm19 \n" + "vpsubb %%zmm5,%%zmm2,%%zmm2 \n" + "vpsubb %%zmm5,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm0,%%zmm4,%%zmm0 \n" + "vpmaddubsw %%zmm19,%%zmm4,%%zmm19 \n" + "vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n" + "vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n" + "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm19,%%zmm19 \n" + "vpackssdw %%zmm19,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" + "vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n" + "vpackssdw %%zmm3,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm17,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm25,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst_y])%{%%k4%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_rgb), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(c), // %3 - "m"(kPermdARGBToY_AVX512BW), // %4 - "m"(kPermqRGB24ToARGB0_AVX512BW), // %5 - "m"(kPermqRGB24ToARGB1_AVX512BW), // %6 - "m"(kPermqRGB24ToARGB2_AVX512BW), // %7 - "m"(kPermqRGB24ToARGB3_AVX512BW), // %8 - "m"(kShuffleMaskRGB24ToARGB) // %9 + : [src_rgb] "+r"(src_rgb), [dst_y] "+r"(dst_y), [width] "+r"(width), + [temp] "=&r"(temp), [mask] "=&r"(mask) + : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW), + [q0] "m"(kPermqRGB24ToARGB0_AVX512BW), + [q1] "m"(kPermqRGB24ToARGB1_AVX512BW), + [q2] "m"(kPermqRGB24ToARGB2_AVX512BW), + [q3] "m"(kPermqRGB24ToARGB3_AVX512BW), + [shuf] "m"(kShuffleMaskRGB24ToARGB) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", - "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", - "xmm25"); + "xmm17", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", + "k1", "k2", "k3", "k4"); } #endif // HAS_RGBTOYMATRIXROW_AVX512BW @@ -2386,18 +2683,28 @@ void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb, int width, const struct ArgbConstants* c) { asm volatile( - "pcmpeqb %%xmm5,%%xmm5 \n" // 0x8000 - "psllw $15,%%xmm5 \n" - "movdqa 0x20(%4),%%xmm3 \n" // kRGBToU - "movdqa 0x40(%4),%%xmm4 \n" // kRGBToV + "movdqu 0x80(%4),%%xmm5 \n" // kAddUV + "movdqu 0x20(%4),%%xmm3 \n" // kRGBToU + "movdqu 0x40(%4),%%xmm4 \n" // kRGBToV "sub %1,%2 \n" LABELALIGN "1: \n" +#if defined(__x86_64__) + "movdqu (%0),%%xmm8 \n" + "movdqu 0x10(%0),%%xmm9 \n" + "movdqu 0x20(%0),%%xmm10 \n" + "movdqu 0x30(%0),%%xmm11 \n" + "movdqa %%xmm8,%%xmm0 \n" + "movdqa %%xmm9,%%xmm1 \n" + "movdqa %%xmm10,%%xmm2 \n" + "movdqa %%xmm11,%%xmm6 \n" +#else "movdqu (%0),%%xmm0 \n" "movdqu 0x10(%0),%%xmm1 \n" "movdqu 0x20(%0),%%xmm2 \n" "movdqu 0x30(%0),%%xmm6 \n" +#endif "pmaddubsw %%xmm3,%%xmm0 \n" "pmaddubsw %%xmm3,%%xmm1 \n" "pmaddubsw %%xmm3,%%xmm2 \n" @@ -2413,6 +2720,18 @@ void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb, "packuswb %%xmm6,%%xmm1 \n" "movdqu %%xmm1,(%1) \n" +#if defined(__x86_64__) + "pmaddubsw %%xmm4,%%xmm8 \n" + "pmaddubsw %%xmm4,%%xmm9 \n" + "pmaddubsw %%xmm4,%%xmm10 \n" + "pmaddubsw %%xmm4,%%xmm11 \n" + "phaddw %%xmm9,%%xmm8 \n" + "phaddw %%xmm11,%%xmm10 \n" + "movdqa %%xmm5,%%xmm1 \n" + "movdqa %%xmm5,%%xmm6 \n" + "psubw %%xmm8,%%xmm1 \n" + "psubw %%xmm10,%%xmm6 \n" +#else "movdqu (%0),%%xmm0 \n" "movdqu 0x10(%0),%%xmm1 \n" "movdqu 0x20(%0),%%xmm2 \n" @@ -2427,6 +2746,7 @@ void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb, "movdqa %%xmm5,%%xmm6 \n" "psubw %%xmm0,%%xmm1 \n" "psubw %%xmm2,%%xmm6 \n" +#endif "psrlw $0x8,%%xmm1 \n" "psrlw $0x8,%%xmm6 \n" "packuswb %%xmm6,%%xmm1 \n" @@ -2445,27 +2765,40 @@ void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb, "+rm"(width) // %3 #endif : "r"(c) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" +#if defined(__x86_64__) + , + "xmm8", "xmm9", "xmm10", "xmm11" +#endif + ); } #endif // HAS_ARGBTOUV444MATRIXROW_SSSE3 #ifdef HAS_ARGBTOUV444MATRIXROW_AVX2 - void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, int width, const struct ArgbConstants* c) { asm volatile( - "vmovdqa 0x20(%4),%%ymm3 \n" // kRGBToU - "vmovdqa 0x40(%4),%%ymm4 \n" // kRGBToV - "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // 0x8000 - "vpsllw $15,%%ymm5,%%ymm5 \n" - "vmovdqa %5,%%ymm7 \n" + "vmovdqu 0x20(%4),%%ymm3 \n" // kRGBToU + "vmovdqu 0x40(%4),%%ymm4 \n" // kRGBToV + "vmovdqu 0x80(%4),%%ymm5 \n" // kAddUV + "vmovdqu %5,%%ymm7 \n" "sub %1,%2 \n" LABELALIGN "1: \n" +#if defined(__x86_64__) + "vmovdqu (%0),%%ymm8 \n" + "vmovdqu 0x20(%0),%%ymm9 \n" + "vmovdqu 0x40(%0),%%ymm10 \n" + "vmovdqu 0x60(%0),%%ymm11 \n" + "vpmaddubsw %%ymm3,%%ymm8,%%ymm0 \n" + "vpmaddubsw %%ymm3,%%ymm9,%%ymm1 \n" + "vpmaddubsw %%ymm3,%%ymm10,%%ymm2 \n" + "vpmaddubsw %%ymm3,%%ymm11,%%ymm6 \n" +#else "vmovdqu (%0),%%ymm0 \n" "vmovdqu 0x20(%0),%%ymm1 \n" "vmovdqu 0x40(%0),%%ymm2 \n" @@ -2474,6 +2807,7 @@ void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb, "vpmaddubsw %%ymm3,%%ymm1,%%ymm1 \n" "vpmaddubsw %%ymm3,%%ymm2,%%ymm2 \n" "vpmaddubsw %%ymm3,%%ymm6,%%ymm6 \n" +#endif "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates "vphaddw %%ymm6,%%ymm2,%%ymm2 \n" "vpsubw %%ymm0,%%ymm5,%%ymm0 \n" @@ -2484,6 +2818,12 @@ void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb, "vpermd %%ymm0,%%ymm7,%%ymm0 \n" // unmutate. "vmovdqu %%ymm0,(%1) \n" +#if defined(__x86_64__) + "vpmaddubsw %%ymm4,%%ymm8,%%ymm0 \n" + "vpmaddubsw %%ymm4,%%ymm9,%%ymm1 \n" + "vpmaddubsw %%ymm4,%%ymm10,%%ymm2 \n" + "vpmaddubsw %%ymm4,%%ymm11,%%ymm6 \n" +#else "vmovdqu (%0),%%ymm0 \n" "vmovdqu 0x20(%0),%%ymm1 \n" "vmovdqu 0x40(%0),%%ymm2 \n" @@ -2492,6 +2832,7 @@ void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb, "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n" "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n" "vpmaddubsw %%ymm4,%%ymm6,%%ymm6 \n" +#endif "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates "vphaddw %%ymm6,%%ymm2,%%ymm2 \n" "vpsubw %%ymm0,%%ymm5,%%ymm0 \n" @@ -2517,111 +2858,163 @@ void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb, : "r"(c), // %4 "m"(kPermdARGBToY_AVX) // %5 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + "xmm7" +#if defined(__x86_64__) + , + "xmm8", "xmm9", "xmm10", "xmm11" +#endif + ); } #endif // HAS_ARGBTOUV444MATRIXROW_AVX2 #ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW - +LIBYUV_TARGET_AVX512BW void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, int width, const struct ArgbConstants* c) { + uintptr_t temp; asm volatile( - "vbroadcasti32x4 0x20(%4),%%zmm3 \n" // kRGBToU - "vbroadcasti32x4 0x40(%4),%%zmm4 \n" // kRGBToV + "vbroadcasti32x4 0x20(%[c]),%%zmm3 \n" // kRGBToU + "vbroadcasti32x4 0x40(%[c]),%%zmm4 \n" // kRGBToV "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // -1 - "vpsllw $15,%%zmm16,%%zmm5 \n" // 0x8000 - "vmovups %5,%%zmm17 \n" - "sub %1,%2 \n" + "vbroadcasti32x4 0x80(%[c]),%%zmm5 \n" // kAddUV + "vmovdqu32 %[perm],%%zmm17 \n" + "sub %[dst_u],%[dst_v] \n" + "sub $0x40,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" - "vmovups (%0),%%zmm0 \n" - "vmovups 0x40(%0),%%zmm1 \n" - "vmovups 0x80(%0),%%zmm2 \n" - "vmovups 0xc0(%0),%%zmm18 \n" - "vpmaddubsw %%zmm3,%%zmm0,%%zmm0 \n" - "vpmaddubsw %%zmm3,%%zmm1,%%zmm1 \n" - "vpmaddubsw %%zmm3,%%zmm2,%%zmm2 \n" - "vpmaddubsw %%zmm3,%%zmm18,%%zmm18 \n" + "vmovdqu8 (%[src_argb]),%%zmm18 \n" + "vmovdqu8 0x40(%[src_argb]),%%zmm19 \n" + "vmovdqu8 0x80(%[src_argb]),%%zmm20 \n" + "vmovdqu8 0xc0(%[src_argb]),%%zmm21 \n" + "lea 0x100(%[src_argb]),%[src_argb]\n" + "vpmaddubsw %%zmm3,%%zmm18,%%zmm0 \n" + "vpmaddubsw %%zmm3,%%zmm19,%%zmm1 \n" + "vpmaddubsw %%zmm3,%%zmm20,%%zmm2 \n" + "vpmaddubsw %%zmm3,%%zmm21,%%zmm6 \n" "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" - "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" + "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n" "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates - "vpackssdw %%zmm18,%%zmm2,%%zmm2 \n" - "vpsubw %%zmm5,%%zmm0,%%zmm0 \n" - "vpsubw %%zmm5,%%zmm2,%%zmm2 \n" + "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm5,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm5,%%zmm2,%%zmm2 \n" "vpsrlw $0x8,%%zmm0,%%zmm0 \n" "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate. - "vmovups %%zmm0,(%1) \n" + "vmovdqu8 %%zmm0,(%[dst_u]) \n" - "vmovups (%0),%%zmm0 \n" - "vmovups 0x40(%0),%%zmm1 \n" - "vmovups 0x80(%0),%%zmm2 \n" - "vmovups 0xc0(%0),%%zmm18 \n" - "vpmaddubsw %%zmm4,%%zmm0,%%zmm0 \n" - "vpmaddubsw %%zmm4,%%zmm1,%%zmm1 \n" - "vpmaddubsw %%zmm4,%%zmm2,%%zmm2 \n" - "vpmaddubsw %%zmm4,%%zmm18,%%zmm18 \n" + "vpmaddubsw %%zmm4,%%zmm18,%%zmm0 \n" + "vpmaddubsw %%zmm4,%%zmm19,%%zmm1 \n" + "vpmaddubsw %%zmm4,%%zmm20,%%zmm2 \n" + "vpmaddubsw %%zmm4,%%zmm21,%%zmm6 \n" "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" - "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" + "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n" "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates - "vpackssdw %%zmm18,%%zmm2,%%zmm2 \n" - "vpsubw %%zmm5,%%zmm0,%%zmm0 \n" - "vpsubw %%zmm5,%%zmm2,%%zmm2 \n" + "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm5,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm5,%%zmm2,%%zmm2 \n" "vpsrlw $0x8,%%zmm0,%%zmm0 \n" "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate. - "vmovups %%zmm0,(%1,%2,1) \n" - "lea 0x100(%0),%0 \n" - "lea 0x40(%1),%1 \n" - "subl $0x40,%3 \n" - "jg 1b \n" + "vmovdqu8 %%zmm0,(%[dst_u],%[dst_v],1) \n" + "lea 0x40(%[dst_u]),%[dst_u] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "kshiftrq $16,%%k1,%%k2 \n" + "kshiftrq $32,%%k1,%%k3 \n" + "kshiftrq $48,%%k1,%%k4 \n" + "vmovdqu32 (%[src_argb]),%%zmm18%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src_argb]),%%zmm19%{%%k2%}%{z%} \n" + "vmovdqu32 0x80(%[src_argb]),%%zmm20%{%%k3%}%{z%} \n" + "vmovdqu32 0xc0(%[src_argb]),%%zmm21%{%%k4%}%{z%} \n" + "vpmaddubsw %%zmm3,%%zmm18,%%zmm0 \n" + "vpmaddubsw %%zmm3,%%zmm19,%%zmm1 \n" + "vpmaddubsw %%zmm3,%%zmm20,%%zmm2 \n" + "vpmaddubsw %%zmm3,%%zmm21,%%zmm6 \n" + "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" + "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" + "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n" + "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" + "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm5,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm5,%%zmm2,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm17,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst_u])%{%%k1%} \n" + + "vpmaddubsw %%zmm4,%%zmm18,%%zmm0 \n" + "vpmaddubsw %%zmm4,%%zmm19,%%zmm1 \n" + "vpmaddubsw %%zmm4,%%zmm20,%%zmm2 \n" + "vpmaddubsw %%zmm4,%%zmm21,%%zmm6 \n" + "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" + "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" + "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n" + "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" + "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm5,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm5,%%zmm2,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm17,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst_u],%[dst_v],1)%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_argb), // %0 - "+r"(dst_u), // %1 - "+r"(dst_v), // %2 -#if defined(__i386__) - "+m"(width) // %3 -#else - "+rm"(width) // %3 -#endif - : "r"(c), // %4 - "m"(kPermdARGBToY_AVX512BW) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", - "xmm17", "xmm18"); + : [src_argb] "+r"(src_argb), [dst_u] "+r"(dst_u), [dst_v] "+r"(dst_v), + [width] "+r"(width), [temp] "=&r"(temp) + : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", + "k4"); } #endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW #ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, int width, const struct ArgbConstants* c) { + uintptr_t temp; asm volatile( - "vbroadcasti32x4 0x20(%4),%%zmm3 \n" // kRGBToU (signed) - "vbroadcasti32x4 0x40(%4),%%zmm4 \n" // kRGBToV (signed) - "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // -1 - "vpsllw $15,%%zmm16,%%zmm5 \n" // 0x8000 - "vmovups %5,%%zmm17 \n" - "sub %1,%2 \n" + "vbroadcasti32x4 0x20(%[c]),%%zmm3 \n" // kRGBToU (signed) + "vbroadcasti32x4 0x40(%[c]),%%zmm4 \n" // kRGBToV (signed) + "vbroadcasti32x4 0x80(%[c]),%%zmm5 \n" // kAddUV + "vmovdqu32 %[perm],%%zmm17 \n" + "sub %[dst_u],%[dst_v] \n" + "sub $0x40,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" - "vmovups (%0),%%zmm18 \n" - "vmovups 0x40(%0),%%zmm19 \n" - "vmovups 0x80(%0),%%zmm20 \n" - "vmovups 0xc0(%0),%%zmm21 \n" - "lea 0x100(%0),%0 \n" + "vmovdqu8 (%[src_argb]),%%zmm18 \n" + "vmovdqu8 0x40(%[src_argb]),%%zmm19 \n" + "vmovdqu8 0x80(%[src_argb]),%%zmm20 \n" + "vmovdqu8 0xc0(%[src_argb]),%%zmm21 \n" + "lea 0x100(%[src_argb]),%[src_argb]\n" "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" "vpxorq %%zmm1,%%zmm1,%%zmm1 \n" "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" @@ -2638,7 +3031,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate. - "vmovups %%zmm0,(%1) \n" + "vmovdqu8 %%zmm0,(%[dst_u]) \n" "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" "vpxorq %%zmm1,%%zmm1,%%zmm1 \n" @@ -2656,19 +3049,67 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate. - "vmovups %%zmm0,(%1,%2,1) \n" - "lea 0x40(%1),%1 \n" - "subl $0x40,%3 \n" - "jg 1b \n" + "vmovdqu8 %%zmm0,(%[dst_u],%[dst_v],1) \n" + "lea 0x40(%[dst_u]),%[dst_u] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "kshiftrq $16,%%k1,%%k2 \n" + "kshiftrq $32,%%k1,%%k3 \n" + "kshiftrq $48,%%k1,%%k4 \n" + "vmovdqu32 (%[src_argb]),%%zmm18%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src_argb]),%%zmm19%{%%k2%}%{z%} \n" + "vmovdqu32 0x80(%[src_argb]),%%zmm20%{%%k3%}%{z%} \n" + "vmovdqu32 0xc0(%[src_argb]),%%zmm21%{%%k4%}%{z%} \n" + "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" + "vpxorq %%zmm1,%%zmm1,%%zmm1 \n" + "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" + "vpxorq %%zmm16,%%zmm16,%%zmm16 \n" + "vpdpbusd %%zmm3,%%zmm18,%%zmm0 \n" + "vpdpbusd %%zmm3,%%zmm19,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm20,%%zmm2 \n" + "vpdpbusd %%zmm3,%%zmm21,%%zmm16 \n" + "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" + "vpackssdw %%zmm16,%%zmm2,%%zmm2 \n" + "vpsubw %%zmm0,%%zmm5,%%zmm0 \n" + "vpsubw %%zmm2,%%zmm5,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm17,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst_u])%{%%k1%} \n" + + "vpxorq %%zmm0,%%zmm0,%%zmm0 \n" + "vpxorq %%zmm1,%%zmm1,%%zmm1 \n" + "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" + "vpxorq %%zmm16,%%zmm16,%%zmm16 \n" + "vpdpbusd %%zmm4,%%zmm18,%%zmm0 \n" + "vpdpbusd %%zmm4,%%zmm19,%%zmm1 \n" + "vpdpbusd %%zmm4,%%zmm20,%%zmm2 \n" + "vpdpbusd %%zmm4,%%zmm21,%%zmm16 \n" + "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" + "vpackssdw %%zmm16,%%zmm2,%%zmm2 \n" + "vpsubw %%zmm0,%%zmm5,%%zmm0 \n" + "vpsubw %%zmm2,%%zmm5,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm17,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst_u],%[dst_v],1)%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_argb), // %0 - "+r"(dst_u), // %1 - "+r"(dst_v), // %2 - "+rm"(width) // %3 - : "r"(c), // %4 - "m"(kPermdARGBToY_AVX512BW) // %5 + : [src_argb] "+r"(src_argb), [dst_u] "+r"(dst_u), [dst_v] "+r"(dst_v), + [width] "+r"(width), [temp] "=&r"(temp) + : [c] "r"(c), [perm] "m"(kPermdARGBToY_AVX512BW) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", - "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4"); } #endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNI @@ -2677,29 +3118,33 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, // U = (kAddUV - dot(kRGBToU, argb)) >> 8. The accumulators start at // ~kAddUV so they hold ~(kAddUV - dot) and U is the inverse of byte 1 of each // int32. The inverse is folded into the vpternlogd that merges the halves. +LIBYUV_TARGET_AVX512VNNIVBMI void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, int width, const struct ArgbConstants* c) { + uintptr_t temp; asm volatile( - "vbroadcasti32x4 0x20(%4),%%zmm3 \n" // kRGBToU (signed) - "vbroadcasti32x4 0x40(%4),%%zmm4 \n" // kRGBToV (signed) - "vpmovzxwd 0x80(%4),%%zmm5 \n" // kAddUV (32-bit) + "vbroadcasti32x4 0x20(%[c]),%%zmm3 \n" // kRGBToU (signed) + "vbroadcasti32x4 0x40(%[c]),%%zmm4 \n" // kRGBToV (signed) + "vpmovzxwd 0x80(%[c]),%%zmm5 \n" // kAddUV (32-bit) "vpternlogd $0x55,%%zmm5,%%zmm5,%%zmm5 \n" // ~kAddUV - "vbroadcasti64x4 %5,%%zmm16 \n" // byte 1 of int32 + "vbroadcasti64x4 %[perm],%%zmm16 \n" // byte 1 of int32 "vpternlogd $0xff,%%zmm6,%%zmm6,%%zmm6 \n" "vpxord %%zmm7,%%zmm7,%%zmm7 \n" "vinserti64x4 $1,%%ymm6,%%zmm7,%%zmm6 \n" // high 256 bits select - "sub %1,%2 \n" + "sub %[dst_u],%[dst_v] \n" + "sub $0x40,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" - "vmovups (%0),%%zmm0 \n" - "vmovups 0x40(%0),%%zmm1 \n" - "vmovups 0x80(%0),%%zmm2 \n" - "vmovups 0xc0(%0),%%zmm7 \n" - "lea 0x100(%0),%0 \n" + "vmovdqu8 (%[src_argb]),%%zmm0 \n" + "vmovdqu8 0x40(%[src_argb]),%%zmm1 \n" + "vmovdqu8 0x80(%[src_argb]),%%zmm2 \n" + "vmovdqu8 0xc0(%[src_argb]),%%zmm7 \n" + "lea 0x100(%[src_argb]),%[src_argb]\n" "vmovdqa64 %%zmm5,%%zmm17 \n" "vmovdqa64 %%zmm5,%%zmm18 \n" "vmovdqa64 %%zmm5,%%zmm19 \n" @@ -2722,21 +3167,58 @@ void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "vpermt2b %%zmm24,%%zmm16,%%zmm23 \n" // V 32..63 "vpternlogd $0x27,%%zmm6,%%zmm19,%%zmm17 \n" // ~(low 17, high 19) "vpternlogd $0x27,%%zmm6,%%zmm23,%%zmm21 \n" // ~(low 21, high 23) - "vmovdqu64 %%zmm17,(%1) \n" - "vmovdqu64 %%zmm21,(%1,%2,1) \n" - "lea 0x40(%1),%1 \n" - "subl $0x40,%3 \n" - "jg 1b \n" + "vmovdqu8 %%zmm17,(%[dst_u]) \n" + "vmovdqu8 %%zmm21,(%[dst_u],%[dst_v],1) \n" + "lea 0x40(%[dst_u]),%[dst_u] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "kshiftrq $16,%%k1,%%k2 \n" + "kshiftrq $32,%%k1,%%k3 \n" + "kshiftrq $48,%%k1,%%k4 \n" + "vmovdqu32 (%[src_argb]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src_argb]),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu32 0x80(%[src_argb]),%%zmm2%{%%k3%}%{z%} \n" + "vmovdqu32 0xc0(%[src_argb]),%%zmm7%{%%k4%}%{z%} \n" + "vmovdqa64 %%zmm5,%%zmm17 \n" + "vmovdqa64 %%zmm5,%%zmm18 \n" + "vmovdqa64 %%zmm5,%%zmm19 \n" + "vmovdqa64 %%zmm5,%%zmm20 \n" + "vmovdqa64 %%zmm5,%%zmm21 \n" + "vmovdqa64 %%zmm5,%%zmm22 \n" + "vmovdqa64 %%zmm5,%%zmm23 \n" + "vmovdqa64 %%zmm5,%%zmm24 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm17 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm21 \n" + "vpdpbusd %%zmm3,%%zmm1,%%zmm18 \n" + "vpdpbusd %%zmm4,%%zmm1,%%zmm22 \n" + "vpdpbusd %%zmm3,%%zmm2,%%zmm19 \n" + "vpdpbusd %%zmm4,%%zmm2,%%zmm23 \n" + "vpdpbusd %%zmm3,%%zmm7,%%zmm20 \n" + "vpdpbusd %%zmm4,%%zmm7,%%zmm24 \n" + "vpermt2b %%zmm18,%%zmm16,%%zmm17 \n" + "vpermt2b %%zmm20,%%zmm16,%%zmm19 \n" + "vpermt2b %%zmm22,%%zmm16,%%zmm21 \n" + "vpermt2b %%zmm24,%%zmm16,%%zmm23 \n" + "vpternlogd $0x27,%%zmm6,%%zmm19,%%zmm17 \n" + "vpternlogd $0x27,%%zmm6,%%zmm23,%%zmm21 \n" + "vmovdqu8 %%zmm17,(%[dst_u])%{%%k1%} \n" + "vmovdqu8 %%zmm21,(%[dst_u],%[dst_v],1)%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_argb), // %0 - "+r"(dst_u), // %1 - "+r"(dst_v), // %2 - "+rm"(width) // %3 - : "r"(c), // %4 - "m"(kPermbDWordByte1_AVX512VNNIVBMI) // %5 + : [src_argb] "+r"(src_argb), [dst_u] "+r"(dst_u), [dst_v] "+r"(dst_v), + [width] "+r"(width), [temp] "=&r"(temp) + : [c] "r"(c), [perm] "m"(kPermbDWordByte1_AVX512VNNIVBMI) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", - "xmm23", "xmm24"); + "xmm23", "xmm24", "k1", "k2", "k3", "k4"); } #endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI @@ -2750,18 +3232,34 @@ static const lvec8 kShuffleAARRGGBB = { // 8x2 -> 4x1 ARGB pixels converted to 4 U and 4 V // ARGBToUV does rounding average of 4 ARGB pixels -void ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb, - int src_stride_argb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c) { +void OMITFP ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { +#if defined(__i386__) + uvec8 rgb_to_u; + uvec8 rgb_to_v; + uvec16 add_uv; + memcpy(&rgb_to_u, c->kRGBToU, sizeof(rgb_to_u)); + memcpy(&rgb_to_v, c->kRGBToV, sizeof(rgb_to_v)); + memcpy(&add_uv, c->kAddUV, sizeof(add_uv)); +#endif asm volatile( - "movdqa 0x20(%5),%%xmm4 \n" // RGBToU - "movdqa 0x40(%5),%%xmm5 \n" // RGBToV +#if defined(__i386__) + "movdqa %5,%%xmm4 \n" // RGBToU + "movdqa %6,%%xmm5 \n" // RGBToV + "pcmpeqb %%xmm6,%%xmm6 \n" // 0x0101 + "pabsb %%xmm6,%%xmm6 \n" + "movdqa %8,%%xmm7 \n" // kShuffleAARRGGBB +#else + "movdqu 0x20(%5),%%xmm4 \n" // RGBToU + "movdqu 0x40(%5),%%xmm5 \n" // RGBToV "pcmpeqb %%xmm6,%%xmm6 \n" // 0x0101 "pabsb %%xmm6,%%xmm6 \n" "movdqa %6,%%xmm7 \n" // kShuffleAARRGGBB +#endif "sub %1,%2 \n" "1: \n" @@ -2786,8 +3284,11 @@ void ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb, "pavgw %%xmm2,%%xmm1 \n" "packuswb %%xmm1,%%xmm0 \n" // mutates - "movdqa %%xmm6,%%xmm2 \n" - "psllw $15,%%xmm2 \n" // 0x8000 +#if defined(__i386__) + "movdqa %7,%%xmm2 \n" // kAddUV +#else + "movdqu 0x80(%5),%%xmm2 \n" // kAddUV +#endif "movdqa %%xmm0,%%xmm1 \n" "pmaddubsw %%xmm5,%%xmm1 \n" // 4 V "pmaddubsw %%xmm4,%%xmm0 \n" // 4 U @@ -2799,10 +3300,10 @@ void ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb, "pshufd $0x55,%%xmm2,%%xmm2 \n" // Copy V to low 4 bytes "movd %%xmm2,0x00(%1,%2,1) \n" // Write 4 V's - "lea 0x20(%0),%0 \n" - "lea 0x4(%1),%1 \n" - "subl $0x8,%3 \n" - "jg 1b \n" + "lea 0x20(%0),%0 \n" + "lea 0x4(%1),%1 \n" + "subl $0x8,%3 \n" + "jg 1b \n" : "+r"(src_argb), // %0 "+r"(dst_u), // %1 "+r"(dst_v), // %2 @@ -2812,8 +3313,15 @@ void ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb, "+rm"(width) // %3 #endif : "r"((ptrdiff_t)(src_stride_argb)), // %4 - "r"(c), // %5 - "m"(kShuffleAARRGGBB) // %6 +#if defined(__i386__) + "m"(rgb_to_u), // %5 + "m"(rgb_to_v), // %6 + "m"(add_uv), // %7 + "m"(kShuffleAARRGGBB) // %8 +#else + "r"(c), // %5 + "m"(kShuffleAARRGGBB) // %6 +#endif : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } @@ -2824,18 +3332,34 @@ void ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb, // 16x2 -> 8x1 ARGB pixels converted to 8 U and 8 V // ARGBToUV does rounding average of 4 ARGB pixels -void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, - int src_stride_argb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c) { +void OMITFP ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { +#if defined(__i386__) + uvec8 rgb_to_u; + uvec8 rgb_to_v; + uvec16 add_uv; + memcpy(&rgb_to_u, c->kRGBToU, sizeof(rgb_to_u)); + memcpy(&rgb_to_v, c->kRGBToV, sizeof(rgb_to_v)); + memcpy(&add_uv, c->kAddUV, sizeof(add_uv)); +#endif asm volatile( - "vbroadcasti128 0x20(%5),%%ymm4 \n" // RGBToU - "vbroadcasti128 0x40(%5),%%ymm5 \n" // RGBToV +#if defined(__i386__) + "vbroadcasti128 %5,%%ymm4 \n" // RGBToU + "vbroadcasti128 %6,%%ymm5 \n" // RGBToV + "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" // 0x0101 + "vpabsb %%ymm6,%%ymm6 \n" + "vmovdqa %8,%%ymm7 \n" // kShuffleAARRGGBB +#else + "vbroadcasti128 0x20(%5),%%ymm4 \n" // RGBToU + "vbroadcasti128 0x40(%5),%%ymm5 \n" // RGBToV "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" // 0x0101 "vpabsb %%ymm6,%%ymm6 \n" "vmovdqa %6,%%ymm7 \n" // kShuffleAARRGGBB +#endif "sub %1,%2 \n" "1: \n" @@ -2861,7 +3385,11 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" // mutates "vpermq $0xd8,%%ymm0,%%ymm0 \n" // 8 ARGB Pixels - "vpsllw $15,%%ymm6,%%ymm2 \n" // 0x8000 +#if defined(__i386__) + "vbroadcasti128 %7,%%ymm2 \n" // kAddUV +#else + "vmovdqu 0x80(%5),%%ymm2 \n" // kAddUV +#endif "vpmaddubsw %%ymm5,%%ymm0,%%ymm1 \n" // 8 V "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n" // 8 U "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // uuuuvvvv uuuuvvvv @@ -2887,8 +3415,15 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, "+rm"(width) // %3 #endif : "r"((ptrdiff_t)(src_stride_argb)), // %4 - "r"(c), // %5 - "m"(kShuffleAARRGGBB) // %6 +#if defined(__i386__) + "m"(rgb_to_u), // %5 + "m"(rgb_to_v), // %6 + "m"(add_uv), // %7 + "m"(kShuffleAARRGGBB) // %8 +#else + "r"(c), // %5 + "m"(kShuffleAARRGGBB) // %6 +#endif : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } @@ -2907,6 +3442,7 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, // 32x2 -> 16x1 ARGB pixels converted to 16 U and 16 V // ARGBToUV does rounding average of 4 ARGB pixels +LIBYUV_TARGET_AVX512BW void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, int src_stride_argb, uint8_t* dst_u, @@ -2919,7 +3455,7 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, "vbroadcasti32x4 0x40(%[c]),%%zmm5 \n" // RGBToV "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" "vpabsb %%zmm16,%%zmm20 \n" // 0x0101 - "vpsllw $15,%%zmm16,%%zmm17 \n" // 0x8000 + "vbroadcasti32x4 0x80(%[c]),%%zmm17 \n" // kAddUV "vbroadcasti32x4 %[shuf],%%zmm21 \n" // kShuffleAARRGGBB "vmovups %[permy],%%zmm18 \n" // kPermdARGBToY_AVX512BW "vmovups %[permuv],%%zmm19 \n" // kPermdARGBToUV_AVX512BW @@ -2973,7 +3509,7 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, "2: \n" "addl $0x20,%3 \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q3,%[temp],%[temp] \n" "kmovd %k[temp],%%k1 \n" @@ -3047,6 +3583,7 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, // 32x2 -> 16x1 ARGB pixels converted to 16 U and 16 V // ARGBToUV does rounding average of 4 ARGB pixels +LIBYUV_TARGET_AVX512VNNI void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, int src_stride_argb, uint8_t* dst_u, @@ -3059,7 +3596,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, "vbroadcasti32x4 0x40(%[c]),%%zmm5 \n" // RGBToV (signed) "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" "vpabsb %%zmm16,%%zmm20 \n" // 0x0101 - "vpsllw $15,%%zmm16,%%zmm17 \n" // 0x8000 + "vbroadcasti32x4 0x80(%[c]),%%zmm17 \n" // kAddUV "vbroadcasti32x4 %[shuf],%%zmm21 \n" // kShuffleAARRGGBB "vmovups %[permy],%%zmm18 \n" // kPermdARGBToY_AVX512BW "vmovups %[permuv],%%zmm19 \n" // kPermdARGBToUV_AVX512BW @@ -3113,7 +3650,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, "2: \n" "addl $0x20,%3 \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q3,%[temp],%[temp] \n" "kmovd %k[temp],%%k1 \n" @@ -3187,6 +3724,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, // 32x2 -> 16x1 ARGB pixels converted to 16 U and 16 V // ARGBToUV does rounding average of 4 ARGB pixels +LIBYUV_TARGET_AVX512VNNIVBMI void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, int src_stride_argb, uint8_t* dst_u, @@ -3250,7 +3788,7 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "2: \n" "addl $0x20,%3 \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q3,%[temp],%[temp] \n" "kmovd %k[temp],%%k1 \n" @@ -3345,14 +3883,14 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "movq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \ "lea 0x8(%[u_buf]),%[u_buf] \n" \ "punpcklwd %%xmm1,%%xmm3 \n" \ - "psraw $2,%%xmm3 \n" \ + "psrlw $2,%%xmm3 \n" \ "packuswb %%xmm3,%%xmm3 \n" \ "punpcklwd %%xmm3,%%xmm3 \n" \ "movdqu (%[y_buf]),%%xmm4 \n" \ "movdqa %%xmm4,%%xmm2 \n" \ "psllw $6,%%xmm4 \n" \ "psrlw $4,%%xmm2 \n" \ - "paddw %%xmm2,%%xmm4 \n" \ + "por %%xmm2,%%xmm4 \n" \ "lea 0x10(%[y_buf]),%[y_buf] \n" #define READYUVA210 \ @@ -3360,17 +3898,17 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "movq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \ "lea 0x8(%[u_buf]),%[u_buf] \n" \ "punpcklwd %%xmm1,%%xmm3 \n" \ - "psraw $2,%%xmm3 \n" \ + "psrlw $2,%%xmm3 \n" \ "packuswb %%xmm3,%%xmm3 \n" \ "punpcklwd %%xmm3,%%xmm3 \n" \ "movdqu (%[y_buf]),%%xmm4 \n" \ "movdqa %%xmm4,%%xmm2 \n" \ "psllw $6,%%xmm4 \n" \ "psrlw $4,%%xmm2 \n" \ - "paddw %%xmm2,%%xmm4 \n" \ + "por %%xmm2,%%xmm4 \n" \ "lea 0x10(%[y_buf]),%[y_buf] \n" \ "movdqu (%[a_buf]),%%xmm5 \n" \ - "psraw $2,%%xmm5 \n" \ + "psrlw $2,%%xmm5 \n" \ "packuswb %%xmm5,%%xmm5 \n" \ "lea 0x10(%[a_buf]),%[a_buf] \n" @@ -3379,8 +3917,8 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "movdqu (%[u_buf]),%%xmm3 \n" \ "movdqu 0x00(%[u_buf],%[v_buf],1),%%xmm2 \n" \ "lea 0x10(%[u_buf]),%[u_buf] \n" \ - "psraw $2,%%xmm3 \n" \ - "psraw $2,%%xmm2 \n" \ + "psrlw $2,%%xmm3 \n" \ + "psrlw $2,%%xmm2 \n" \ "movdqa %%xmm3,%%xmm1 \n" \ "punpcklwd %%xmm2,%%xmm3 \n" \ "punpckhwd %%xmm2,%%xmm1 \n" \ @@ -3389,7 +3927,7 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "movdqa %%xmm4,%%xmm2 \n" \ "psllw $6,%%xmm4 \n" \ "psrlw $4,%%xmm2 \n" \ - "paddw %%xmm2,%%xmm4 \n" \ + "por %%xmm2,%%xmm4 \n" \ "lea 0x10(%[y_buf]),%[y_buf] \n" // Read 8 UV from 444 10 bit. With 8 Alpha. @@ -3397,8 +3935,8 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "movdqu (%[u_buf]),%%xmm3 \n" \ "movdqu 0x00(%[u_buf],%[v_buf],1),%%xmm2 \n" \ "lea 0x10(%[u_buf]),%[u_buf] \n" \ - "psraw $2,%%xmm3 \n" \ - "psraw $2,%%xmm2 \n" \ + "psrlw $2,%%xmm3 \n" \ + "psrlw $2,%%xmm2 \n" \ "movdqa %%xmm3,%%xmm1 \n" \ "punpcklwd %%xmm2,%%xmm3 \n" \ "punpckhwd %%xmm2,%%xmm1 \n" \ @@ -3407,10 +3945,10 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "movdqa %%xmm4,%%xmm2 \n" \ "psllw $6,%%xmm4 \n" \ "psrlw $4,%%xmm2 \n" \ - "paddw %%xmm2,%%xmm4 \n" \ + "por %%xmm2,%%xmm4 \n" \ "lea 0x10(%[y_buf]),%[y_buf] \n" \ "movdqu (%[a_buf]),%%xmm5 \n" \ - "psraw $2,%%xmm5 \n" \ + "psrlw $2,%%xmm5 \n" \ "packuswb %%xmm5,%%xmm5 \n" \ "lea 0x10(%[a_buf]),%[a_buf] \n" @@ -3420,14 +3958,14 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "movq 0x00(%[u_buf],%[v_buf],1),%%xmm1 \n" \ "lea 0x8(%[u_buf]),%[u_buf] \n" \ "punpcklwd %%xmm1,%%xmm3 \n" \ - "psraw $0x4,%%xmm3 \n" \ + "psrlw $0x4,%%xmm3 \n" \ "packuswb %%xmm3,%%xmm3 \n" \ "punpcklwd %%xmm3,%%xmm3 \n" \ "movdqu (%[y_buf]),%%xmm4 \n" \ "movdqa %%xmm4,%%xmm2 \n" \ "psllw $4,%%xmm4 \n" \ "psrlw $8,%%xmm2 \n" \ - "paddw %%xmm2,%%xmm4 \n" \ + "por %%xmm2,%%xmm4 \n" \ "lea 0x10(%[y_buf]),%[y_buf] \n" // Read 4 UV from 422, upsample to 8 UV. With 8 Alpha. @@ -3517,13 +4055,13 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, #if defined(__x86_64__) #define YUVTORGB_SETUP(yuvconstants) \ "pcmpeqb %%xmm13,%%xmm13 \n" \ - "movdqa (%[yuvconstants]),%%xmm8 \n" \ + "movdqu (%[yuvconstants]),%%xmm8 \n" \ "pxor %%xmm12,%%xmm12 \n" \ - "movdqa 32(%[yuvconstants]),%%xmm9 \n" \ + "movdqu 32(%[yuvconstants]),%%xmm9 \n" \ "pavgb %%xmm12,%%xmm13 \n" \ - "movdqa 64(%[yuvconstants]),%%xmm10 \n" \ - "movdqa 96(%[yuvconstants]),%%xmm11 \n" \ - "movdqa 128(%[yuvconstants]),%%xmm12 \n" + "movdqu 64(%[yuvconstants]),%%xmm10 \n" \ + "movdqu 96(%[yuvconstants]),%%xmm11 \n" \ + "movdqu 128(%[yuvconstants]),%%xmm12 \n" #define YUVTORGB_SETUP_AR30(yuvconstants) \ YUVTORGB_SETUP(yuvconstants) \ @@ -3558,18 +4096,19 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, // Convert 8 pixels: 8 UV and 8 Y #define YUVTORGB16(yuvconstants) \ + "movdqu 96(%[yuvconstants]),%%xmm0 \n" \ + "pmulhuw %%xmm0,%%xmm4 \n" \ "pcmpeqb %%xmm0,%%xmm0 \n" \ "pabsb %%xmm0,%%xmm0 \n" \ "psllw $7,%%xmm0 \n" \ "psubb %%xmm0,%%xmm3 \n" \ - "pmulhuw 96(%[yuvconstants]),%%xmm4 \n" \ - "movdqa (%[yuvconstants]),%%xmm0 \n" \ - "movdqa 32(%[yuvconstants]),%%xmm1 \n" \ - "movdqa 64(%[yuvconstants]),%%xmm2 \n" \ + "movdqu (%[yuvconstants]),%%xmm0 \n" \ + "movdqu 32(%[yuvconstants]),%%xmm1 \n" \ + "movdqu 64(%[yuvconstants]),%%xmm2 \n" \ "pmaddubsw %%xmm3,%%xmm0 \n" \ "pmaddubsw %%xmm3,%%xmm1 \n" \ "pmaddubsw %%xmm3,%%xmm2 \n" \ - "movdqa 128(%[yuvconstants]),%%xmm3 \n" \ + "movdqu 128(%[yuvconstants]),%%xmm3 \n" \ "paddw %%xmm3,%%xmm4 \n" \ "paddsw %%xmm4,%%xmm0 \n" \ "paddsw %%xmm4,%%xmm2 \n" \ @@ -3577,18 +4116,19 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "movdqa %%xmm4,%%xmm1 \n" #define YUVTORGB16_AR30(yuvconstants) \ + "movdqu 96(%[yuvconstants]),%%xmm0 \n" \ + "pmulhuw %%xmm0,%%xmm4 \n" \ "pcmpeqb %%xmm0,%%xmm0 \n" \ "pabsb %%xmm0,%%xmm0 \n" \ "psllw $7,%%xmm0 \n" \ "psubb %%xmm0,%%xmm3 \n" \ - "pmulhuw 96(%[yuvconstants]),%%xmm4 \n" \ - "movdqa (%[yuvconstants]),%%xmm0 \n" \ - "movdqa 32(%[yuvconstants]),%%xmm1 \n" \ - "movdqa 64(%[yuvconstants]),%%xmm2 \n" \ + "movdqu (%[yuvconstants]),%%xmm0 \n" \ + "movdqu 32(%[yuvconstants]),%%xmm1 \n" \ + "movdqu 64(%[yuvconstants]),%%xmm2 \n" \ "pmaddubsw %%xmm3,%%xmm0 \n" \ "pmaddubsw %%xmm3,%%xmm1 \n" \ "pmaddubsw %%xmm3,%%xmm2 \n" \ - "movdqa 128(%[yuvconstants]),%%xmm3 \n" \ + "movdqu 128(%[yuvconstants]),%%xmm3 \n" \ "paddw %%xmm3,%%xmm4 \n" \ "pcmpeqb %%xmm3,%%xmm3 \n" \ "psrlw $14,%%xmm3 \n" \ @@ -3766,16 +4306,23 @@ void OMITFP I444AlphaToARGBRow_SSSE3(const uint8_t* y_buf, } #endif // HAS_I444ALPHATOARGBROW_SSSE3 +#ifdef HAS_I422TORGB24ROW_SSSE3 void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uint32_t shuf0[4]; + uint32_t shuf1[4]; + memcpy(shuf0, &kShuffleMaskARGBToRGB24[0], sizeof(shuf0)); + memcpy(shuf1, &kShuffleMaskARGBToRGB24[1], sizeof(shuf1)); +#endif asm volatile ( YUVTORGB_SETUP(yuvconstants) - "movdqa %[kShuffleMaskARGBToRGB24_1],%%xmm5 \n" - "movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm6 \n" + "movdqu %[kShuffleMaskARGBToRGB24_1],%%xmm5 \n" + "movdqu %[kShuffleMaskARGBToRGB24_0],%%xmm6 \n" "sub %[u_buf],%[v_buf] \n" LABELALIGN @@ -3795,23 +4342,36 @@ void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf, [width]"+rm"(width) // %[width] #endif : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] +#if defined(__i386__) + [kShuffleMaskARGBToRGB24_0]"m"(shuf0), + [kShuffleMaskARGBToRGB24_1]"m"(shuf1) +#else [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]), [kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1]) +#endif : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" ); } +#endif // HAS_I422TORGB24ROW_SSSE3 +#ifdef HAS_I444TORGB24ROW_SSSE3 void OMITFP I444ToRGB24Row_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uint32_t shuf0[4]; + uint32_t shuf1[4]; + memcpy(shuf0, &kShuffleMaskARGBToRGB24[0], sizeof(shuf0)); + memcpy(shuf1, &kShuffleMaskARGBToRGB24[1], sizeof(shuf1)); +#endif asm volatile ( YUVTORGB_SETUP(yuvconstants) - "movdqa %[kShuffleMaskARGBToRGB24_1],%%xmm5 \n" - "movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm6 \n" + "movdqu %[kShuffleMaskARGBToRGB24_1],%%xmm5 \n" + "movdqu %[kShuffleMaskARGBToRGB24_0],%%xmm6 \n" "sub %[u_buf],%[v_buf] \n" LABELALIGN @@ -3831,12 +4391,18 @@ void OMITFP I444ToRGB24Row_SSSE3(const uint8_t* y_buf, [width]"+rm"(width) // %[width] #endif : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] +#if defined(__i386__) + [kShuffleMaskARGBToRGB24_0]"m"(shuf0), + [kShuffleMaskARGBToRGB24_1]"m"(shuf1) +#else [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]), [kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1]) +#endif : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" ); } +#endif // HAS_I444TORGB24ROW_SSSE3 void OMITFP I422ToARGBRow_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, @@ -4609,13 +5175,13 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpermq $0xd8,%%ymm3,%%ymm3 \n" \ "vpermq $0xd8,%%ymm1,%%ymm1 \n" \ "vpunpcklwd %%ymm1,%%ymm3,%%ymm3 \n" \ - "vpsraw $2,%%ymm3,%%ymm3 \n" \ + "vpsrlw $2,%%ymm3,%%ymm3 \n" \ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \ "vmovdqu (%[y_buf]),%%ymm4 \n" \ "vpsllw $6,%%ymm4,%%ymm2 \n" \ "vpsrlw $4,%%ymm4,%%ymm4 \n" \ - "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" \ + "vpor %%ymm2,%%ymm4,%%ymm4 \n" \ "lea 0x20(%[y_buf]),%[y_buf] \n" // Read 8 UV from 210, upsample to 16 UV. With 16 Alpha. @@ -4626,16 +5192,16 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpermq $0xd8,%%ymm3,%%ymm3 \n" \ "vpermq $0xd8,%%ymm1,%%ymm1 \n" \ "vpunpcklwd %%ymm1,%%ymm3,%%ymm3 \n" \ - "vpsraw $2,%%ymm3,%%ymm3 \n" \ + "vpsrlw $2,%%ymm3,%%ymm3 \n" \ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \ "vmovdqu (%[y_buf]),%%ymm4 \n" \ "vpsllw $6,%%ymm4,%%ymm2 \n" \ "vpsrlw $4,%%ymm4,%%ymm4 \n" \ - "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" \ + "vpor %%ymm2,%%ymm4,%%ymm4 \n" \ "lea 0x20(%[y_buf]),%[y_buf] \n" \ "vmovdqu (%[a_buf]),%%ymm5 \n" \ - "vpsraw $2,%%ymm5,%%ymm5 \n" \ + "vpsrlw $2,%%ymm5,%%ymm5 \n" \ "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n" \ "lea 0x20(%[a_buf]),%[a_buf] \n" @@ -4644,15 +5210,15 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vmovdqu (%[u_buf]),%%ymm3 \n" \ "vmovdqu 0x00(%[u_buf],%[v_buf],1),%%ymm2 \n" \ "lea 0x20(%[u_buf]),%[u_buf] \n" \ - "vpsraw $2,%%ymm3,%%ymm3 \n" \ - "vpsraw $2,%%ymm2,%%ymm2 \n" \ + "vpsrlw $2,%%ymm3,%%ymm3 \n" \ + "vpsrlw $2,%%ymm2,%%ymm2 \n" \ "vpunpckhwd %%ymm2,%%ymm3,%%ymm1 \n" \ "vpunpcklwd %%ymm2,%%ymm3,%%ymm3 \n" \ "vpackuswb %%ymm1,%%ymm3,%%ymm3 \n" \ "vmovdqu (%[y_buf]),%%ymm4 \n" \ "vpsllw $6,%%ymm4,%%ymm2 \n" \ "vpsrlw $4,%%ymm4,%%ymm4 \n" \ - "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" \ + "vpor %%ymm2,%%ymm4,%%ymm4 \n" \ "lea 0x20(%[y_buf]),%[y_buf] \n" // Read 8 UV from 212 12 bit, upsample to 16 UV @@ -4663,13 +5229,13 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpermq $0xd8,%%ymm3,%%ymm3 \n" \ "vpermq $0xd8,%%ymm1,%%ymm1 \n" \ "vpunpcklwd %%ymm1,%%ymm3,%%ymm3 \n" \ - "vpsraw $0x4,%%ymm3,%%ymm3 \n" \ + "vpsrlw $0x4,%%ymm3,%%ymm3 \n" \ "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" \ "vmovdqu (%[y_buf]),%%ymm4 \n" \ "vpsllw $4,%%ymm4,%%ymm2 \n" \ "vpsrlw $8,%%ymm4,%%ymm4 \n" \ - "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" \ + "vpor %%ymm2,%%ymm4,%%ymm4 \n" \ "lea 0x20(%[y_buf]),%[y_buf] \n" // Read 16 UV from 410. With 16 Alpha. @@ -4677,18 +5243,18 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vmovdqu (%[u_buf]),%%ymm3 \n" \ "vmovdqu 0x00(%[u_buf],%[v_buf],1),%%ymm2 \n" \ "lea 0x20(%[u_buf]),%[u_buf] \n" \ - "vpsraw $2,%%ymm3,%%ymm3 \n" \ - "vpsraw $2,%%ymm2,%%ymm2 \n" \ + "vpsrlw $2,%%ymm3,%%ymm3 \n" \ + "vpsrlw $2,%%ymm2,%%ymm2 \n" \ "vpunpckhwd %%ymm2,%%ymm3,%%ymm1 \n" \ "vpunpcklwd %%ymm2,%%ymm3,%%ymm3 \n" \ "vpackuswb %%ymm1,%%ymm3,%%ymm3 \n" \ "vmovdqu (%[y_buf]),%%ymm4 \n" \ "vpsllw $6,%%ymm4,%%ymm2 \n" \ "vpsrlw $4,%%ymm4,%%ymm4 \n" \ - "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" \ + "vpor %%ymm2,%%ymm4,%%ymm4 \n" \ "lea 0x20(%[y_buf]),%[y_buf] \n" \ "vmovdqu (%[a_buf]),%%ymm5 \n" \ - "vpsraw $2,%%ymm5,%%ymm5 \n" \ + "vpsrlw $2,%%ymm5,%%ymm5 \n" \ "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n" \ "lea 0x20(%[a_buf]),%[a_buf] \n" @@ -4789,13 +5355,13 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, #if defined(__x86_64__) #define YUVTORGB_SETUP_AVX2(yuvconstants) \ "vpcmpeqb %%ymm13,%%ymm13,%%ymm13 \n" \ - "vmovdqa (%[yuvconstants]),%%ymm8 \n" \ + "vmovdqu (%[yuvconstants]),%%ymm8 \n" \ "vpabsb %%ymm13,%%ymm13 \n" \ - "vmovdqa 32(%[yuvconstants]),%%ymm9 \n" \ + "vmovdqu 32(%[yuvconstants]),%%ymm9 \n" \ "vpsllw $7,%%ymm13,%%ymm13 \n" \ - "vmovdqa 64(%[yuvconstants]),%%ymm10 \n" \ - "vmovdqa 96(%[yuvconstants]),%%ymm11 \n" \ - "vmovdqa 128(%[yuvconstants]),%%ymm12 \n" + "vmovdqu 64(%[yuvconstants]),%%ymm10 \n" \ + "vmovdqu 96(%[yuvconstants]),%%ymm11 \n" \ + "vmovdqu 128(%[yuvconstants]),%%ymm12 \n" #define YUVTORGB_SETUP_AVX512BW(yuvconstants) \ "vpternlogd $0xff,%%zmm13,%%zmm13,%%zmm13 \n" \ @@ -4807,7 +5373,6 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpbroadcastq 96(%[yuvconstants]),%%zmm11 \n" \ "vpbroadcastq 128(%[yuvconstants]),%%zmm12 \n" \ "vmovups (%[quadsplitperm]),%%zmm16 \n" \ - "vmovups (%[dquadsplitperm]),%%zmm17 \n" \ "vmovups (%[unperm]),%%zmm18 \n" #define YUVTORGB_SETUP_AR30_AVX2(yuvconstants) \ @@ -4850,7 +5415,7 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, #define YUVTORGB_REGS_AVX2 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", #define YUVTORGB_REGS_AVX512BW \ - "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm16", "xmm17", "xmm18", + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm16", "xmm18", #else // Convert 16 pixels: 16 UV and 16 Y. @@ -4863,13 +5428,13 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpsllw $7,%%ymm0,%%ymm0 \n" \ "vpsubb %%ymm0,%%ymm3,%%ymm3 \n" \ "vpmulhuw 96(%[yuvconstants]),%%ymm4,%%ymm4 \n" \ - "vmovdqa (%[yuvconstants]),%%ymm0 \n" \ - "vmovdqa 32(%[yuvconstants]),%%ymm1 \n" \ - "vmovdqa 64(%[yuvconstants]),%%ymm2 \n" \ + "vmovdqu (%[yuvconstants]),%%ymm0 \n" \ + "vmovdqu 32(%[yuvconstants]),%%ymm1 \n" \ + "vmovdqu 64(%[yuvconstants]),%%ymm2 \n" \ "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n" \ "vpmaddubsw %%ymm3,%%ymm1,%%ymm1 \n" \ "vpmaddubsw %%ymm3,%%ymm2,%%ymm2 \n" \ - "vmovdqa 128(%[yuvconstants]),%%ymm3 \n" \ + "vmovdqu 128(%[yuvconstants]),%%ymm3 \n" \ "vpaddw %%ymm4,%%ymm3,%%ymm4 \n" \ "vpaddsw %%ymm4,%%ymm0,%%ymm0 \n" \ "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \ @@ -4881,13 +5446,13 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpsllw $7,%%ymm0,%%ymm0 \n" \ "vpsubb %%ymm0,%%ymm3,%%ymm3 \n" \ "vpmulhuw 96(%[yuvconstants]),%%ymm4,%%ymm4 \n" \ - "vmovdqa (%[yuvconstants]),%%ymm0 \n" \ - "vmovdqa 32(%[yuvconstants]),%%ymm1 \n" \ - "vmovdqa 64(%[yuvconstants]),%%ymm2 \n" \ + "vmovdqu (%[yuvconstants]),%%ymm0 \n" \ + "vmovdqu 32(%[yuvconstants]),%%ymm1 \n" \ + "vmovdqu 64(%[yuvconstants]),%%ymm2 \n" \ "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n" \ "vpmaddubsw %%ymm3,%%ymm1,%%ymm1 \n" \ "vpmaddubsw %%ymm3,%%ymm2,%%ymm2 \n" \ - "vmovdqa 128(%[yuvconstants]),%%ymm3 \n" \ + "vmovdqu 128(%[yuvconstants]),%%ymm3 \n" \ "vpaddw %%ymm4,%%ymm3,%%ymm4 \n" \ "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n" \ "vpsrlw $14,%%ymm3,%%ymm3 \n" \ @@ -5188,6 +5753,12 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uint32_t shuf0[4]; + uint32_t shuf1[4]; + memcpy(shuf0, &kShuffleMaskARGBToRGB24[0], sizeof(shuf0)); + memcpy(shuf1, &kShuffleMaskARGBToRGB24[1], sizeof(shuf1)); +#endif asm volatile ( YUVTORGB_SETUP_AVX2(yuvconstants) "vbroadcasti128 %[kShuffleMaskARGBToRGB24_1],%%ymm5 \n" @@ -5212,8 +5783,13 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf, [width]"+rm"(width) // %[width] #endif : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] +#if defined(__i386__) + [kShuffleMaskARGBToRGB24_0]"m"(shuf0), + [kShuffleMaskARGBToRGB24_1]"m"(shuf1) +#else [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]), [kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1]) +#endif : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" ); @@ -5244,7 +5820,6 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf, static const uint16_t kSplitQuadWords[32] = { 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15}; -static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; static const uint64_t kUnpermuteAVX512[8] = {0, 4, 1, 5, 2, 6, 3, 7}; #endif @@ -5265,6 +5840,7 @@ static const uint16_t kPermAR30_1[32] = { // 32 pixels // 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 ARGB (128 // bytes). +LIBYUV_TARGET_AVX512BW void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -5289,7 +5865,7 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV422_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -5305,7 +5881,6 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf, [temp]"=&r"(temp) // %[temp] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm] - [dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm] [unperm]"r"(kUnpermuteAVX512) // %[unperm] : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", @@ -5344,6 +5919,7 @@ static const uint8_t kMaskDST1[64] = { // 32 pixels // 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96 // bytes). +LIBYUV_TARGET_AVX512VBMI void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -5370,7 +5946,7 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV422_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -5386,7 +5962,6 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, [temp]"=&r"(temp) // %[temp] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm] - [dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm] [unperm]"r"(kUnpermuteAVX512), // %[unperm] [kMaskBG]"m"(kMaskBG), // %[kMaskBG] [kMaskDST0]"m"(kMaskDST0), // %[kMaskDST0] @@ -5406,6 +5981,7 @@ static const uint64_t kStitchRGB24_1[8] = {13, 6, 14, 15, 0, 0, 0, 0}; // 32 pixels // 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96 // bytes). +LIBYUV_TARGET_AVX512BW void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -5433,7 +6009,7 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV422_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -5449,7 +6025,6 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf, [temp]"=&r"(temp) // %[temp] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm] - [dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm] [unperm]"r"(kUnpermuteAVX512), // %[unperm] [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]), [kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1]), @@ -5467,6 +6042,7 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf, // 32 pixels // 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 AR30 (128 // bytes). +LIBYUV_TARGET_AVX512BW void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -5498,7 +6074,7 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV422_MASK_AVX512BW YUVTORGB16_AVX512BW(yuvconstants) @@ -5514,7 +6090,6 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf, [temp]"=&r"(temp) // %[temp] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm] - [dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm] [unperm]"r"(kUnpermuteAVX512), // %[unperm] [permar30_0]"r"(kPermAR30_0), // %[permar30_0] [permar30_1]"r"(kPermAR30_1) // %[permar30_1] @@ -6374,8 +6949,8 @@ void I400ToARGBRow_SSE2(const uint8_t* y_buf, const struct YuvConstants* yuvconstants, int width) { asm volatile( - "movdqa 96(%3),%%xmm2 \n" // yg = 18997 = 1.164 - "movdqa 128(%3),%%xmm3 \n" // ygb = 1160 = 1.164 * 16 + "movdqu 96(%3),%%xmm2 \n" // yg = 18997 = 1.164 + "movdqu 128(%3),%%xmm3 \n" // ygb = 1160 = 1.164 * 16 "pcmpeqb %%xmm4,%%xmm4 \n" // 0xff000000 "pslld $0x18,%%xmm4 \n" @@ -6401,7 +6976,7 @@ void I400ToARGBRow_SSE2(const uint8_t* y_buf, "movdqu %%xmm1,0x10(%1) \n" "lea 0x20(%1),%1 \n" - "sub $0x8,%2 \n" + "subl $0x8,%2 \n" "jg 1b \n" : "+r"(y_buf), // %0 "+r"(dst_argb), // %1 @@ -6419,8 +6994,8 @@ void I400ToARGBRow_AVX2(const uint8_t* y_buf, const struct YuvConstants* yuvconstants, int width) { asm volatile( - "vmovdqa 96(%3),%%ymm2 \n" // yg = 18997 = 1.164 - "vmovdqa 128(%3),%%ymm3 \n" // ygb = -1160 = 1.164*16 + "vmovdqu 96(%3),%%ymm2 \n" // yg = 18997 = 1.164 + "vmovdqu 128(%3),%%ymm3 \n" // ygb = -1160 = 1.164*16 "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n" // 0xff000000 "vpslld $0x18,%%ymm4,%%ymm4 \n" @@ -6492,6 +7067,7 @@ static const uint16_t kMirrorUVPermute_AVX512[32] = { // defined(HAS_MIRRORUVROW_AVX512BW) #ifdef HAS_MIRRORROW_AVX512BW +LIBYUV_TARGET_AVX512BW void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); uintptr_t temp; @@ -6512,7 +7088,7 @@ void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { "2: \n" "add $0x40,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %[width],%[temp],%[temp] \n" "kmovq %[temp],%%k1 \n" @@ -6536,11 +7112,8 @@ void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { "99: \n" "vzeroupper \n" - : [dst] "+r"(dst), - [width] "+r"(temp_width), - [temp] "=&r"(temp) - : [src] "r"(src), - [shuf] "m"(kShuffleMirror), + : [dst] "+r"(dst), [width] "+r"(temp_width), [temp] "=&r"(temp) + : [src] "r"(src), [shuf] "m"(kShuffleMirror), [rev] "m"(kMirrorUVPermute_AVX512) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm5", "k1"); } @@ -6569,16 +7142,46 @@ void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_MIRRORROW_AVX2 -#if defined(HAS_MIRRORSPLITUVROW_AVX2) || defined(HAS_MIRRORSPLITUVROW_AVX512BW) +#if defined(HAS_MIRRORSPLITUVROW_SSSE3) || \ + defined(HAS_MIRRORSPLITUVROW_AVX2) || \ + defined(HAS_MIRRORSPLITUVROW_AVX512BW) // Shuffle table for reversing the bytes of UV channels. static const uvec8 kShuffleMirrorSplitUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u, 15u, 13u, 11u, 9u, 7u, 5u, 3u, 1u}; -#endif // defined(HAS_MIRRORSPLITUVROW_AVX2) || - // defined(HAS_MIRRORSPLITUVROW_AVX512BW) +#endif // defined(HAS_MIRRORSPLITUVROW_SSSE3) || ... + +#ifdef HAS_MIRRORSPLITUVROW_SSSE3 +void MirrorSplitUVRow_SSSE3(const uint8_t* src, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + ptrdiff_t temp_width = (ptrdiff_t)(width); + asm volatile( + "movdqa %4,%%xmm1 \n" + "sub %1,%2 \n" + + LABELALIGN + "1: \n" + "movdqu -0x10(%0,%3,2),%%xmm0 \n" + "pshufb %%xmm1,%%xmm0 \n" + "movq %%xmm0,(%1) \n" + "movhps %%xmm0,0x00(%1,%2,1) \n" + "lea 0x8(%1),%1 \n" + "sub $0x8,%3 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(temp_width) // %3 + : "m"(kShuffleMirrorSplitUV) // %4 + : "memory", "cc", "xmm0", "xmm1"); +} +#endif // HAS_MIRRORSPLITUVROW_SSSE3 #ifdef HAS_MIRRORSPLITUVROW_AVX512BW static const uint64_t kMirrorSplitUVPermute[8] = {6, 4, 2, 0, 7, 5, 3, 1}; +LIBYUV_TARGET_AVX512BW void MirrorSplitUVRow_AVX512BW(const uint8_t* src, uint8_t* dst_u, uint8_t* dst_v, @@ -6606,7 +7209,7 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src, "2: \n" "add $0x20,%3 \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q3,%[temp],%[temp] \n" "kmovd %k[temp],%%k1 \n" @@ -6717,6 +7320,7 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { #endif // HAS_MIRRORUVROW_AVX2 #ifdef HAS_MIRRORUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); uintptr_t temp; @@ -6735,7 +7339,7 @@ void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%k[temp] \n" "bzhi %k[width],%k[temp],%k[temp] \n" "kmovd %k[temp],%%k1 \n" @@ -6749,11 +7353,8 @@ void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { "99: \n" "vzeroupper \n" - : [dst_uv] "+r"(dst_uv), - [width] "+r"(temp_width), - [temp] "=&r"(temp) - : [src_uv] "r"(src_uv), - [rev] "m"(kMirrorUVPermute_AVX512) + : [dst_uv] "+r"(dst_uv), [width] "+r"(temp_width), [temp] "=&r"(temp) + : [src_uv] "r"(src_uv), [rev] "m"(kMirrorUVPermute_AVX512) : "memory", "cc", "xmm0", "xmm1", "xmm5", "k1"); } #endif // HAS_MIRRORUVROW_AVX512BW @@ -6773,7 +7374,7 @@ void RGB24MirrorRow_SSSE3(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); - src_rgb24 += width * 3 - 48; + src_rgb24 += temp_width * 3 - 48; asm volatile( "movdqa %3,%%xmm4 \n" "movdqa %4,%%xmm5 \n" @@ -6818,7 +7419,7 @@ void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); - src_rgb24 += width * 3 - 96; + src_rgb24 += temp_width * 3 - 96; asm volatile( "vbroadcasti128 %3,%%ymm4 \n" "vmovdqa %4,%%xmm5 \n" @@ -6890,6 +7491,7 @@ static inline uint64_t RGB24MirrorMask(int n) { return n <= 0 ? 0ull : n >= 64 ? ~0ull : (1ull << n) - 1ull; } +LIBYUV_TARGET_AVX512VBMI void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width) { @@ -7043,6 +7645,7 @@ static const uint32_t kARGBShuffleMirror_AVX512[16] = { // Mirror 16 ARGB pixels per loop. The remaining 1 to 15 pixels are at the // start of src. They are loaded with a k mask, permuted with the reverse // index minus (16 - remaining), and stored with the same k mask. +LIBYUV_TARGET_AVX512BW void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); uintptr_t temp; @@ -7125,21 +7728,25 @@ void SplitUVRow_AVX2(const uint8_t* src_uv, #ifdef HAS_SPLITUVROW_AVX512BW static const uint64_t kSplitUVPermute[8] = {0, 2, 4, 6, 1, 3, 5, 7}; +LIBYUV_TARGET_AVX512BW void SplitUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width) { + uintptr_t temp; asm volatile( "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" "vpsrlw $0x8,%%zmm5,%%zmm5 \n" - "vmovdqu64 %4,%%zmm4 \n" - "sub %1,%2 \n" + "vmovdqu64 %[perm],%%zmm4 \n" + "sub %[dst_u],%[dst_v] \n" + "sub $0x40,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" - "vmovdqu8 (%0),%%zmm0 \n" - "vmovdqu8 0x40(%0),%%zmm1 \n" - "lea 0x80(%0),%0 \n" + "vmovdqu8 (%[src_uv]),%%zmm0 \n" + "vmovdqu8 0x40(%[src_uv]),%%zmm1 \n" + "lea 0x80(%[src_uv]),%[src_uv] \n" "vpsrlw $0x8,%%zmm0,%%zmm2 \n" "vpsrlw $0x8,%%zmm1,%%zmm3 \n" "vpandd %%zmm5,%%zmm0,%%zmm0 \n" @@ -7148,18 +7755,39 @@ void SplitUVRow_AVX512BW(const uint8_t* src_uv, "vpackuswb %%zmm3,%%zmm2,%%zmm2 \n" "vpermq %%zmm0,%%zmm4,%%zmm0 \n" "vpermq %%zmm2,%%zmm4,%%zmm2 \n" - "vmovdqu8 %%zmm0,(%1) \n" - "vmovdqu8 %%zmm2,0x00(%1,%2,1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x40,%3 \n" - "jg 1b \n" + "vmovdqu8 %%zmm0,(%[dst_u]) \n" + "vmovdqu8 %%zmm2,(%[dst_u],%[dst_v],1) \n" + "lea 0x40(%[dst_u]),%[dst_u] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "kshiftrq $32,%%k1,%%k2 \n" + "vmovdqu16 (%[src_uv]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src_uv]),%%zmm1%{%%k2%}%{z%} \n" + "vpsrlw $0x8,%%zmm0,%%zmm2 \n" + "vpsrlw $0x8,%%zmm1,%%zmm3 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpandd %%zmm5,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm3,%%zmm2,%%zmm2 \n" + "vpermq %%zmm0,%%zmm4,%%zmm0 \n" + "vpermq %%zmm2,%%zmm4,%%zmm2 \n" + "vmovdqu8 %%zmm0,(%[dst_u])%{%%k1%} \n" + "vmovdqu8 %%zmm2,(%[dst_u],%[dst_v],1)%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_uv), // %0 - "+r"(dst_u), // %1 - "+r"(dst_v), // %2 - "+r"(width) // %3 - : "m"(kSplitUVPermute) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + : [src_uv] "+r"(src_uv), [dst_u] "+r"(dst_u), [dst_v] "+r"(dst_v), + [width] "+r"(width), [temp] "=&r"(temp) + : [perm] "m"(kSplitUVPermute) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", + "k2"); } #endif // HAS_SPLITUVROW_AVX512BW @@ -7357,9 +7985,6 @@ void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) { : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } - if (size > 0) { - UnpackMT2T_C(src, dst, size); - } } #endif // HAS_UNPACKMT2T_AVX2 @@ -7412,30 +8037,50 @@ void DetileSplitUVRow_AVX2(const uint8_t* src_uv, #endif // HAS_DETILESPLITUVROW_AVX2 #ifdef HAS_MERGEUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW void MergeUVRow_AVX512BW(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) { - asm volatile("sub %0,%1 \n" + uintptr_t temp; + asm volatile( + "sub %0,%1 \n" + "sub $0x20,%3 \n" + "jl 2f \n" - LABELALIGN - "1: \n" - "vpmovzxbw (%0),%%zmm0 \n" - "vpmovzxbw 0x00(%0,%1,1),%%zmm1 \n" - "lea 0x20(%0),%0 \n" - "vpsllw $0x8,%%zmm1,%%zmm1 \n" - "vporq %%zmm0,%%zmm1,%%zmm2 \n" - "vmovdqu64 %%zmm2,(%2) \n" - "lea 0x40(%2),%2 \n" - "sub $0x20,%3 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_u), // %0 - "+r"(src_v), // %1 - "+r"(dst_uv), // %2 - "+r"(width) // %3 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2"); + LABELALIGN + "1: \n" + "vpmovzxbw (%0),%%zmm0 \n" + "vpmovzxbw 0x00(%0,%1,1),%%zmm1 \n" + "lea 0x20(%0),%0 \n" + "vpsllw $0x8,%%zmm1,%%zmm1 \n" + "vporq %%zmm0,%%zmm1,%%zmm2 \n" + "vmovdqu16 %%zmm2,(%2) \n" + "lea 0x40(%2),%2 \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "jle 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k3,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vpmovzxbw (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmovzxbw 0x00(%0,%1,1),%%zmm1%{%%k1%}%{z%} \n" + "vpsllw $0x8,%%zmm1,%%zmm1 \n" + "vporq %%zmm0,%%zmm1,%%zmm2 \n" + "vmovdqu16 %%zmm2,(%2)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_u), // %0 + "+r"(src_v), // %1 + "+r"(dst_uv), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1"); } #endif // HAS_MERGEUVROW_AVX512BW @@ -7533,6 +8178,7 @@ void MergeUVRow_16_AVX2(const uint16_t* src_u, #endif // HAS_MERGEUVROW_16_AVX2 #ifdef HAS_MERGEUVROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW void MergeUVRow_16_AVX512BW(const uint16_t* src_u, const uint16_t* src_v, uint16_t* dst_uv, @@ -7568,7 +8214,7 @@ void MergeUVRow_16_AVX512BW(const uint16_t* src_u, "2: \n" "add $0x20,%3 \n" - "je 99f \n" + "jle 99f \n" "cmp $0x10,%3 \n" "jl 3f \n" "vpmovzxwd (%0),%%zmm0 \n" @@ -7580,7 +8226,7 @@ void MergeUVRow_16_AVX512BW(const uint16_t* src_u, "vmovdqu32 %%zmm0,(%2) \n" "lea 0x40(%2),%2 \n" "sub $0x10,%3 \n" - "je 99f \n" + "jle 99f \n" "3: \n" "mov $-1,%k[temp] \n" @@ -7686,29 +8332,57 @@ void MultiplyRow_16_AVX2(const uint16_t* src_y, #endif // HAS_MULTIPLYROW_16_AVX2 #ifdef HAS_MULTIPLYROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW void MultiplyRow_16_AVX512BW(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { - asm volatile("vpbroadcastw %3,%%zmm2 \n" + uintptr_t temp; + asm volatile( + "vpbroadcastw %[scale],%%zmm2 \n" + "sub $0x40,%2 \n" + "jl 2f \n" - // 64 pixels per loop. - LABELALIGN - "1: \n" - "vpmullw (%0),%%zmm2,%%zmm0 \n" - "vpmullw 0x40(%0),%%zmm2,%%zmm1 \n" - "add $0x80,%0 \n" - "vmovdqu16 %%zmm0,(%1) \n" - "vmovdqu16 %%zmm1,0x40(%1) \n" - "add $0x80,%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_y), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(scale) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm2"); + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vpmullw (%0),%%zmm2,%%zmm0 \n" + "vpmullw 0x40(%0),%%zmm2,%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "jle 99f \n" + "cmp $0x20,%2 \n" + "jl 3f \n" + "vpmullw (%0),%%zmm2,%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + "jle 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : [scale] "r"(scale) // %[scale] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1"); } #endif // HAS_MULTIPLYROW_16_AVX512BW @@ -7722,6 +8396,12 @@ void DivideRow_16_AVX2(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } asm volatile( "vmovd %3,%%xmm3 \n" "vpbroadcastw %%xmm3,%%ymm3 \n" @@ -7750,10 +8430,17 @@ void DivideRow_16_AVX2(const uint16_t* src_y, #endif // HAS_DIVIDEROW_16_AVX2 #ifdef HAS_DIVIDEROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW void DivideRow_16_AVX512BW(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } uintptr_t temp; asm volatile( "vpbroadcastw %[scale],%%zmm2 \n" @@ -7774,7 +8461,7 @@ void DivideRow_16_AVX512BW(const uint16_t* src_y, "2: \n" "add $0x40,%2 \n" - "je 99f \n" + "jle 99f \n" "cmp $0x20,%2 \n" "jl 3f \n" "vpmulhuw (%0),%%zmm2,%%zmm0 \n" @@ -7782,7 +8469,7 @@ void DivideRow_16_AVX512BW(const uint16_t* src_y, "vmovdqu16 %%zmm0,(%1) \n" "lea 0x40(%1),%1 \n" "sub $0x20,%2 \n" - "je 99f \n" + "jle 99f \n" "3: \n" "mov $-1,%k[temp] \n" @@ -7808,6 +8495,7 @@ void DivideRow_16_AVX512BW(const uint16_t* src_y, // 16384 = 10 bits // 4096 = 12 bits // 256 = 16 bits +#ifdef HAS_CONVERT16TO8ROW_SSSE3 void Convert16To8Row_SSSE3(const uint16_t* src_y, uint8_t* dst_y, int scale, @@ -7817,7 +8505,7 @@ void Convert16To8Row_SSSE3(const uint16_t* src_y, "punpcklwd %%xmm2,%%xmm2 \n" "pshufd $0x0,%%xmm2,%%xmm2 \n" - // 32 pixels per loop. + // 16 pixels per loop. LABELALIGN "1: \n" "movdqu (%0),%%xmm0 \n" @@ -7836,6 +8524,7 @@ void Convert16To8Row_SSSE3(const uint16_t* src_y, : "r"(scale) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2"); } +#endif // HAS_CONVERT16TO8ROW_SSSE3 #ifdef HAS_CONVERT16TO8ROW_AVX2 void Convert16To8Row_AVX2(const uint16_t* src_y, @@ -7870,33 +8559,64 @@ void Convert16To8Row_AVX2(const uint16_t* src_y, #endif // HAS_CONVERT16TO8ROW_AVX2 #ifdef HAS_CONVERT16TO8ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void Convert16To8Row_AVX512BW(const uint16_t* src_y, uint8_t* dst_y, int scale, int width) { - asm volatile("vpbroadcastw %3,%%zmm2 \n" + uintptr_t temp; + asm volatile( + "vpbroadcastw %[scale],%%zmm2 \n" + "sub $0x40,%2 \n" + "jl 2f \n" - // 64 pixels per loop. - LABELALIGN - "1: \n" - "vmovups (%0),%%zmm0 \n" - "vmovups 0x40(%0),%%zmm1 \n" - "add $0x80,%0 \n" - "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" - "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n" - "vpmovuswb %%zmm0,%%ymm0 \n" - "vpmovuswb %%zmm1,%%ymm1 \n" - "vmovups %%ymm0,(%1) \n" - "vmovups %%ymm1,0x20(%1) \n" - "add $0x40,%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_y), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(scale) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm2"); + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vmovdqu16 (%0),%%zmm0 \n" + "vmovdqu16 0x40(%0),%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n" + "vpmovuswb %%zmm0,%%ymm0 \n" + "vpmovuswb %%zmm1,%%ymm1 \n" + "vmovdqu8 %%ymm0,(%1) \n" + "vmovdqu8 %%ymm1,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "jle 99f \n" + "cmp $0x20,%2 \n" + "jl 3f \n" + "vmovdqu16 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmovuswb %%zmm0,%%ymm0 \n" + "vmovdqu8 %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x20,%2 \n" + "jle 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmovuswb %%zmm0,%%ymm0 \n" + "vmovdqu8 %%ymm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : [scale] "r"(scale) // %[scale] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1"); } #endif // HAS_CONVERT16TO8ROW_AVX512BW @@ -7951,9 +8671,9 @@ void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv, "movd %4,%%xmm2 \n" "punpcklwd %%xmm2,%%xmm2 \n" "pshufd $0x0,%%xmm2,%%xmm2 \n" - "pcmpeqw %%xmm5,%%xmm5 \n" + "pcmpeqd %%xmm5,%%xmm5 \n" "pcmpeqw %%xmm6,%%xmm6 \n" - "psrlw $15,%%xmm5 \n" + "psrld $16,%%xmm5 \n" "pmaddwd %%xmm6,%%xmm6 \n" "movdqa %5,%%xmm7 \n" @@ -7961,21 +8681,33 @@ void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv, LABELALIGN "1: \n" "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" "movdqu (%0,%3,2),%%xmm3 \n" - "movdqu 0x10(%0,%3,2),%%xmm4 \n" - "add $0x20,%0 \n" - "pmaddwd %%xmm5,%%xmm0 \n" - "pmaddwd %%xmm5,%%xmm1 \n" - "pmaddwd %%xmm5,%%xmm3 \n" - "pmaddwd %%xmm5,%%xmm4 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm3,%%xmm4 \n" + "pand %%xmm5,%%xmm0 \n" + "psrld $16,%%xmm1 \n" + "pand %%xmm5,%%xmm3 \n" + "psrld $16,%%xmm4 \n" + "paddd %%xmm1,%%xmm0 \n" + "paddd %%xmm4,%%xmm3 \n" "paddd %%xmm3,%%xmm0 \n" - "paddd %%xmm4,%%xmm1 \n" "paddd %%xmm6,%%xmm0 \n" - "paddd %%xmm6,%%xmm1 \n" "psrld $2,%%xmm0 \n" - "psrld $2,%%xmm1 \n" "pshufb %%xmm7,%%xmm0 \n" + "movdqu 0x10(%0),%%xmm1 \n" + "movdqu 0x10(%0,%3,2),%%xmm3 \n" + "add $0x20,%0 \n" + "movdqa %%xmm1,%%xmm4 \n" + "pand %%xmm5,%%xmm1 \n" + "psrld $16,%%xmm4 \n" + "paddd %%xmm4,%%xmm1 \n" + "movdqa %%xmm3,%%xmm4 \n" + "pand %%xmm5,%%xmm3 \n" + "psrld $16,%%xmm4 \n" + "paddd %%xmm4,%%xmm3 \n" + "paddd %%xmm3,%%xmm1 \n" + "paddd %%xmm6,%%xmm1 \n" + "psrld $2,%%xmm1 \n" "pshufb %%xmm7,%%xmm1 \n" "punpcklqdq %%xmm1,%%xmm0 \n" "pmulhuw %%xmm2,%%xmm0 \n" @@ -8041,8 +8773,9 @@ void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, "vmovd %4,%%xmm2 \n" "vpbroadcastw %%xmm2,%%ymm2 \n" "vpcmpeqw %%ymm5,%%ymm5,%%ymm5 \n" - "vpmaddwd %%ymm5,%%ymm5,%%ymm6 \n" "vpsrlw $15,%%ymm5,%%ymm5 \n" + "vpsllw $1,%%ymm5,%%ymm6 \n" + "vpsllw $15,%%ymm5,%%ymm7 \n" // 16 pixels per loop. LABELALIGN @@ -8052,6 +8785,10 @@ void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, "vmovdqu (%0,%3,2),%%ymm3 \n" "vmovdqu 0x20(%0,%3,2),%%ymm4 \n" "add $0x40,%0 \n" + "vpxor %%ymm7,%%ymm0,%%ymm0 \n" + "vpxor %%ymm7,%%ymm1,%%ymm1 \n" + "vpxor %%ymm7,%%ymm3,%%ymm3 \n" + "vpxor %%ymm7,%%ymm4,%%ymm4 \n" "vpmaddwd %%ymm5,%%ymm0,%%ymm0 \n" "vpmaddwd %%ymm5,%%ymm1,%%ymm1 \n" "vpmaddwd %%ymm5,%%ymm3,%%ymm3 \n" @@ -8077,7 +8814,8 @@ void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, "+r"(width) // %2 : "r"(src_uv_stride), // %3 "rm"(scale) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); } #endif // HAS_HALFWIDTHROW_16TO8_AVX2 @@ -8089,13 +8827,26 @@ void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv, int scale, int width) { int n = (width - 1) & ~15; + int r = (width - 1) & 15; + SIMD_ALIGNED(uint16_t pair[64]); + SIMD_ALIGNED(uint8_t out[16]); if (n > 0) { HalfWidthRow_16To8_AVX2(src_uv, src_uv_stride, dst_uv, scale, n); src_uv += n * 2; dst_uv += n; - width -= n; } - HalfWidthRow_16To8_Odd_C(src_uv, src_uv_stride, dst_uv, scale, width); + if (r > 0) { + HalfWidthRow_16To8_Any_AVX2(src_uv, src_uv_stride, dst_uv, scale, r); + src_uv += r * 2; + dst_uv += r; + } + memset(pair, 0, sizeof(pair)); + pair[0] = src_uv[0]; + pair[1] = src_uv[0]; + pair[32] = src_uv[src_uv_stride]; + pair[33] = src_uv[src_uv_stride]; + HalfWidthRow_16To8_AVX2(pair, 32, out, scale, 16); + dst_uv[0] = out[0]; } #endif // HAS_HALFWIDTHROW_16TO8_ODD_AVX2 @@ -8148,63 +8899,95 @@ void HalfRow_16_AVX2(const uint16_t* src_uv, #endif // HAS_HALFROW_16_AVX2 #ifdef HAS_HALFROW_16TO8_AVX512BW +LIBYUV_TARGET_AVX512BW void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, int scale, int width) { const uint16_t* src_uv1 = src_uv + src_uv_stride; - const int shift = 16 - __builtin_ctz((uint32_t)scale); - asm volatile("vpbroadcastw %4,%%zmm2 \n" + uintptr_t temp; + asm volatile( + "vpbroadcastw %[scale],%%zmm2 \n" + "sub $0x40,%[width] \n" + "jl 2f \n" - // 64 pixels per loop. - LABELALIGN - "1: \n" - "vmovdqu64 (%0),%%zmm0 \n" - "vmovdqu64 0x40(%0),%%zmm1 \n" - "vpavgw (%1),%%zmm0,%%zmm0 \n" - "vpavgw 0x40(%1),%%zmm1,%%zmm1 \n" - "add $0x80,%0 \n" - "add $0x80,%1 \n" - "vpsrlvw %%zmm2,%%zmm0,%%zmm0 \n" - "vpsrlvw %%zmm2,%%zmm1,%%zmm1 \n" - "vpmovuswb %%zmm0,(%2) \n" - "vpmovuswb %%zmm1,0x20(%2) \n" - "add $0x40,%2 \n" - "sub $0x40,%3 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_uv), // %0 - "+r"(src_uv1), // %1 - "+r"(dst_uv), // %2 - "+r"(width) // %3 - : "rm"(shift) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2"); + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vmovdqu64 (%[src0]),%%zmm0 \n" + "vmovdqu64 0x40(%[src0]),%%zmm1 \n" + "vpavgw (%[src1]),%%zmm0,%%zmm0 \n" + "vpavgw 0x40(%[src1]),%%zmm1,%%zmm1 \n" + "add $0x80,%[src0] \n" + "add $0x80,%[src1] \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n" + "vpmovuswb %%zmm0,(%[dst]) \n" + "vpmovuswb %%zmm1,0x20(%[dst]) \n" + "add $0x40,%[dst] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%q[temp] \n" + "bzhi %q[width],%q[temp],%q[temp] \n" + "kmovd %k[temp],%%k1 \n" + "shr $32,%q[temp] \n" + "kmovd %k[temp],%%k2 \n" + "vmovdqu16 (%[src0]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src0]),%%zmm1%{%%k2%}%{z%}\n" + "vmovdqu16 (%[src1]),%%zmm3%{%%k1%}%{z%} \n" + "vmovdqu16 0x40(%[src1]),%%zmm4%{%%k2%}%{z%}\n" + "vpavgw %%zmm3,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm4,%%zmm1,%%zmm1 \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n" + "vpmovuswb %%zmm0,(%[dst])%{%%k1%} \n" + "vpmovuswb %%zmm1,0x20(%[dst])%{%%k2%} \n" + + "99: \n" + "vzeroupper \n" + : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv), + [width] "+r"(width), [temp] "=&r"(temp) + : [scale] "r"(scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "k1", "k2"); } #endif // HAS_HALFROW_16TO8_AVX512BW #ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW +LIBYUV_TARGET_AVX512BW void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, int scale, int width) { const uint16_t* src_uv1 = src_uv + src_uv_stride; + uintptr_t temp; asm volatile( - "vpbroadcastw %4,%%zmm2 \n" + "vpbroadcastw %[scale],%%zmm2 \n" "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" - "vpmaddwd %%zmm5,%%zmm5,%%zmm6 \n" - "vpabsw %%zmm5,%%zmm5 \n" + "vpsrlw $15,%%zmm5,%%zmm5 \n" + "vpsllw $1,%%zmm5,%%zmm6 \n" + "vpsllw $15,%%zmm5,%%zmm7 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" // 32 pixels per loop. LABELALIGN "1: \n" - "vmovdqu64 (%0),%%zmm0 \n" - "vmovdqu64 0x40(%0),%%zmm1 \n" - "vmovdqu64 (%1),%%zmm3 \n" - "vmovdqu64 0x40(%1),%%zmm4 \n" - "add $0x80,%0 \n" - "add $0x80,%1 \n" + "vmovdqu64 (%[src0]),%%zmm0 \n" + "vmovdqu64 0x40(%[src0]),%%zmm1 \n" + "vmovdqu64 (%[src1]),%%zmm3 \n" + "vmovdqu64 0x40(%[src1]),%%zmm4 \n" + "add $0x80,%[src0] \n" + "add $0x80,%[src1] \n" + "vpxord %%zmm7,%%zmm0,%%zmm0 \n" + "vpxord %%zmm7,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm3,%%zmm3 \n" + "vpxord %%zmm7,%%zmm4,%%zmm4 \n" "vpmaddwd %%zmm5,%%zmm0,%%zmm0 \n" "vpmaddwd %%zmm5,%%zmm1,%%zmm1 \n" "vpmaddwd %%zmm5,%%zmm3,%%zmm3 \n" @@ -8219,17 +9002,51 @@ void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, "vpmovusdw %%zmm1,%%ymm1 \n" "vinserti64x4 $1,%%ymm1,%%zmm0,%%zmm0 \n" "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" - "vpmovuswb %%zmm0,(%2) \n" - "add $0x20,%2 \n" - "sub $0x20,%3 \n" - "jg 1b \n" + "vpmovuswb %%zmm0,(%[dst]) \n" + "add $0x20,%[dst] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "jle 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k[width],%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "kmovd %k[temp],%%k3 \n" + "shr $16,%k[temp] \n" + "kmovw %k[temp],%%k2 \n" + "vmovdqu32 (%[src0]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src0]),%%zmm1%{%%k2%}%{z%}\n" + "vmovdqu32 (%[src1]),%%zmm3%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src1]),%%zmm4%{%%k2%}%{z%}\n" + "vpxord %%zmm7,%%zmm0,%%zmm0 \n" + "vpxord %%zmm7,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm3,%%zmm3 \n" + "vpxord %%zmm7,%%zmm4,%%zmm4 \n" + "vpmaddwd %%zmm5,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm5,%%zmm1,%%zmm1 \n" + "vpmaddwd %%zmm5,%%zmm3,%%zmm3 \n" + "vpmaddwd %%zmm5,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm3,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm4,%%zmm1,%%zmm1 \n" + "vpaddd %%zmm6,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm6,%%zmm1,%%zmm1 \n" + "vpsrld $2,%%zmm0,%%zmm0 \n" + "vpsrld $2,%%zmm1,%%zmm1 \n" + "vpmovusdw %%zmm0,%%ymm0 \n" + "vpmovusdw %%zmm1,%%ymm1 \n" + "vinserti64x4 $1,%%ymm1,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmovuswb %%zmm0,(%[dst])%{%%k3%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_uv), // %0 - "+r"(src_uv1), // %1 - "+r"(dst_uv), // %2 - "+r"(width) // %3 - : "rm"(scale) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv), + [width] "+r"(width), [temp] "=&r"(temp) + : [scale] "r"(scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "k1", "k2", "k3"); } #endif // HAS_HALFWIDTHROW_16TO8_AVX512BW @@ -8238,6 +9055,7 @@ void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, // 1024 = 10 bits // 4096 = 12 bits // 65536 = 16 bits +#ifdef HAS_CONVERT8TO16ROW_SSE2 void Convert8To16Row_SSE2(const uint8_t* src_y, uint16_t* dst_y, int bits, @@ -8266,6 +9084,7 @@ void Convert8To16Row_SSE2(const uint8_t* src_y, : "r"(shift) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2"); } +#endif // HAS_CONVERT8TO16ROW_SSE2 #ifdef HAS_CONVERT8TO16ROW_AVX2 void Convert8To16Row_AVX2(const uint8_t* src_y, @@ -8300,37 +9119,67 @@ void Convert8To16Row_AVX2(const uint8_t* src_y, #endif // HAS_CONVERT8TO16ROW_AVX2 #ifdef HAS_CONVERT8TO16ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void Convert8To16Row_AVX512BW(const uint8_t* src_y, uint16_t* dst_y, int bits, int width) { const int shift = 16 - bits; + uintptr_t temp; asm volatile( - "vpbroadcastw %4,%%zmm2 \n" - "vmovd %3,%%xmm3 \n" + "vpbroadcastw %[c0101],%%zmm2 \n" + "vmovd %[shift],%%xmm3 \n" + "sub $0x40,%2 \n" + "jl 2f \n" // 64 pixels per loop. LABELALIGN "1: \n" - "vpmovzxbw (%0),%%zmm0 \n" - "vpmovzxbw 0x20(%0),%%zmm1 \n" - "add $0x40,%0 \n" - "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" - "vpmullw %%zmm2,%%zmm1,%%zmm1 \n" - "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n" - "vpsrlw %%xmm3,%%zmm1,%%zmm1 \n" - "vmovdqu16 %%zmm0,(%1) \n" - "vmovdqu16 %%zmm1,0x40(%1) \n" - "add $0x80,%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" + "vpmovzxbw (%0),%%zmm0 \n" + "vpmovzxbw 0x20(%0),%%zmm1 \n" + "lea 0x40(%0),%0 \n" + "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmullw %%zmm2,%%zmm1,%%zmm1 \n" + "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n" + "vpsrlw %%xmm3,%%zmm1,%%zmm1 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "jle 99f \n" + "cmp $0x20,%2 \n" + "jl 3f \n" + "vpmovzxbw (%0),%%zmm0 \n" + "lea 0x20(%0),%0 \n" + "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" + "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + "jle 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vpmovzxbw (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" + "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_y), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(shift), // %3 - "r"(0x0101) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : [shift] "r"(shift), // %[shift] + [c0101] "r"(0x0101) // %[c0101] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "k1"); } #endif // HAS_CONVERT8TO16ROW_AVX512BW @@ -8556,6 +9405,16 @@ void MergeRGBRow_AVX2(const uint8_t* src_r, asm volatile( "sub %0,%1 \n" "sub %0,%2 \n" +#if defined(__x86_64__) + "vbroadcasti128 (%5),%%ymm8 \n" + "vbroadcasti128 16(%5),%%ymm9 \n" + "vbroadcasti128 32(%5),%%ymm10 \n" + "vbroadcasti128 48(%5),%%ymm11 \n" + "vbroadcasti128 64(%5),%%ymm12 \n" + "vbroadcasti128 80(%5),%%ymm13 \n" + "vbroadcasti128 96(%5),%%ymm14 \n" + "vbroadcasti128 112(%5),%%ymm15 \n" +#endif LABELALIGN "1: \n" @@ -8564,6 +9423,26 @@ void MergeRGBRow_AVX2(const uint8_t* src_r, "vmovdqu (%0,%2,1),%%ymm5 \n" "lea 0x20(%0),%0 \n" +#if defined(__x86_64__) + "vpshufb %%ymm8,%%ymm3,%%ymm0 \n" + "vpshufb %%ymm9,%%ymm4,%%ymm6 \n" + "vpshufb %%ymm10,%%ymm5,%%ymm7 \n" + "vpor %%ymm6,%%ymm0,%%ymm0 \n" + "vpor %%ymm7,%%ymm0,%%ymm0 \n" + + "vpshufb %%ymm11,%%ymm3,%%ymm1 \n" + "vpshufb %%ymm12,%%ymm4,%%ymm6 \n" + "vpshufb %%ymm13,%%ymm5,%%ymm7 \n" + "vpor %%ymm6,%%ymm1,%%ymm1 \n" + "vpor %%ymm7,%%ymm1,%%ymm1 \n" + + "vpshufb %%ymm14,%%ymm3,%%ymm2 \n" + "vpshufb %%ymm15,%%ymm4,%%ymm6 \n" + "vbroadcasti128 128(%5),%%ymm7 \n" + "vpor %%ymm6,%%ymm2,%%ymm2 \n" + "vpshufb %%ymm7,%%ymm5,%%ymm7 \n" + "vpor %%ymm7,%%ymm2,%%ymm2 \n" +#else "vbroadcasti128 (%5),%%ymm6 \n" "vbroadcasti128 16(%5),%%ymm7 \n" "vpshufb %%ymm6,%%ymm3,%%ymm0 \n" @@ -8590,6 +9469,7 @@ void MergeRGBRow_AVX2(const uint8_t* src_r, "vpor %%ymm6,%%ymm2,%%ymm2 \n" "vpshufb %%ymm7,%%ymm5,%%ymm7 \n" "vpor %%ymm7,%%ymm2,%%ymm2 \n" +#endif "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm6 \n" "vperm2i128 $0x30,%%ymm0,%%ymm2,%%ymm7 \n" @@ -8608,7 +9488,12 @@ void MergeRGBRow_AVX2(const uint8_t* src_r, "+rm"(width) // %4 : "r"(&kMergeRGBShuffle[0]) // %5 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + "xmm7" +#if defined(__x86_64__) + , + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" +#endif + ); } #endif // HAS_MERGERGBROW_AVX2 @@ -8885,8 +9770,12 @@ void SplitARGBRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_b, uint8_t* dst_a, int width) { +#if defined(__i386__) + uint32_t shuf[4]; + memcpy(shuf, &kShuffleMaskARGBSplit, sizeof(shuf)); +#endif asm volatile( - "movdqa %6,%%xmm3 \n" + "movdqu %6,%%xmm3 \n" "sub %1,%2 \n" "sub %1,%3 \n" "sub %1,%4 \n" @@ -8917,10 +9806,11 @@ void SplitARGBRow_SSSE3(const uint8_t* src_argb, "+r"(dst_a), // %4 #if defined(__i386__) "+m"(width) // %5 + : "m"(shuf) // %6 #else - "+rm"(width) // %5 -#endif + "+rm"(width) // %5 : "m"(kShuffleMaskARGBSplit) // %6 +#endif : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } #endif // HAS_SPLITARGBROW_SSSE3 @@ -8972,11 +9862,17 @@ void SplitARGBRow_AVX2(const uint8_t* src_argb, uint8_t* dst_b, uint8_t* dst_a, int width) { +#if defined(__i386__) + uint32_t shuf[4]; + uint32_t perm[8]; + memcpy(shuf, &kShuffleMaskARGBSplit, sizeof(shuf)); + memcpy(perm, &kShuffleMaskARGBPermute, sizeof(perm)); +#endif asm volatile( "sub %1,%2 \n" "sub %1,%3 \n" "sub %1,%4 \n" - "vmovdqa %7,%%ymm3 \n" + "vmovdqu %7,%%ymm3 \n" "vbroadcasti128 %6,%%ymm4 \n" LABELALIGN @@ -9008,11 +9904,13 @@ void SplitARGBRow_AVX2(const uint8_t* src_argb, "+r"(dst_a), // %4 #if defined(__i386__) "+m"(width) // %5 + : "m"(shuf), // %6 + "m"(perm) // %7 #else - "+rm"(width) // %5 -#endif + "+rm"(width) // %5 : "m"(kShuffleMaskARGBSplit), // %6 "m"(kShuffleMaskARGBPermute) // %7 +#endif : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); } #endif // HAS_SPLITARGBROW_AVX2 @@ -9134,13 +10032,17 @@ void MergeAR64Row_AVX2(const uint16_t* src_r, int depth, int width) { int shift = 16 - depth; - int mask = (1 << depth) - 1; - mask = (mask << 16) + mask; + uint32_t mask16 = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u); + uint32_t mask = (mask16 << 16) | mask16; +#if defined(__i386__) + uint32_t perm[8]; + memcpy(perm, &MergeAR64Permute, sizeof(perm)); +#endif asm volatile( "sub %0,%1 \n" "sub %0,%2 \n" "sub %0,%3 \n" - "vmovdqa %8,%%ymm5 \n" + "vmovdqu %8,%%ymm5 \n" "vmovd %6,%%xmm6 \n" "vbroadcastss %7,%%ymm7 \n" @@ -9186,12 +10088,15 @@ void MergeAR64Row_AVX2(const uint16_t* src_r, "+r"(dst_ar64), // %4 #if defined(__i386__) "+m"(width) // %5 + : "m"(shift), // %6 + "m"(mask), // %7 + "m"(perm) // %8 #else - "+rm"(width) // %5 -#endif + "+rm"(width) // %5 : "m"(shift), // %6 "m"(mask), // %7 "m"(MergeAR64Permute) // %8 +#endif : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } @@ -9205,12 +10110,16 @@ void MergeXR64Row_AVX2(const uint16_t* src_r, int depth, int width) { int shift = 16 - depth; - int mask = (1 << depth) - 1; - mask = (mask << 16) + mask; + uint32_t mask16 = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u); + uint32_t mask = (mask16 << 16) | mask16; +#if defined(__i386__) + uint32_t perm[8]; + memcpy(perm, &MergeAR64Permute, sizeof(perm)); +#endif asm volatile( "sub %0,%1 \n" "sub %0,%2 \n" - "vmovdqa %7,%%ymm5 \n" + "vmovdqu %7,%%ymm5 \n" "vmovd %5,%%xmm6 \n" "vbroadcastss %6,%%ymm7 \n" @@ -9246,14 +10155,18 @@ void MergeXR64Row_AVX2(const uint16_t* src_r, "subl $0x10,%4 \n" "jg 1b \n" "vzeroupper \n" - : "+r"(src_r), // %0 - "+r"(src_g), // %1 - "+r"(src_b), // %2 - "+r"(dst_ar64), // %3 - "+r"(width) // %4 - : "m"(shift), // %5 - "m"(mask), // %6 + : "+r"(src_r), // %0 + "+r"(src_g), // %1 + "+r"(src_b), // %2 + "+r"(dst_ar64), // %3 + "+rm"(width) // %4 + : "m"(shift), // %5 + "m"(mask), // %6 +#if defined(__i386__) + "m"(perm) // %7 +#else "m"(MergeAR64Permute) // %7 +#endif : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } @@ -9270,12 +10183,18 @@ void MergeARGB16To8Row_AVX2(const uint16_t* src_r, int depth, int width) { int shift = depth - 8; +#if defined(__i386__) + uint32_t shuf[4]; + memcpy(shuf, &MergeARGB16To8Shuffle, sizeof(shuf)); +#endif asm volatile( "sub %0,%1 \n" "sub %0,%2 \n" "sub %0,%3 \n" "vbroadcasti128 %7,%%ymm5 \n" "vmovd %6,%%xmm6 \n" + "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n" + "vpsrlw $8,%%ymm7,%%ymm7 \n" LABELALIGN "1: \n" @@ -9287,6 +10206,10 @@ void MergeARGB16To8Row_AVX2(const uint16_t* src_r, "vpsrlw %%xmm6,%%ymm1,%%ymm1 \n" "vpsrlw %%xmm6,%%ymm2,%%ymm2 \n" "vpsrlw %%xmm6,%%ymm3,%%ymm3 \n" + "vpminuw %%ymm7,%%ymm0,%%ymm0 \n" + "vpminuw %%ymm7,%%ymm1,%%ymm1 \n" + "vpminuw %%ymm7,%%ymm2,%%ymm2 \n" + "vpminuw %%ymm7,%%ymm3,%%ymm3 \n" "vpackuswb %%ymm1,%%ymm2,%%ymm1 \n" // BG (planar) "vpackuswb %%ymm3,%%ymm0,%%ymm0 \n" // RA (planar) "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" // BG (interleave) @@ -9309,12 +10232,14 @@ void MergeARGB16To8Row_AVX2(const uint16_t* src_r, "+r"(dst_argb), // %4 #if defined(__i386__) "+m"(width) // %5 + : "m"(shift), // %6 + "m"(shuf) // %7 #else - "+rm"(width) // %5 -#endif + "+rm"(width) // %5 : "m"(shift), // %6 "m"(MergeARGB16To8Shuffle) // %7 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); +#endif + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7"); } #endif // HAS_MERGEARGB16TO8ROW_AVX2 @@ -9326,6 +10251,10 @@ void MergeXRGB16To8Row_AVX2(const uint16_t* src_r, int depth, int width) { int shift = depth - 8; +#if defined(__i386__) + uint32_t shuf[4]; + memcpy(shuf, &MergeARGB16To8Shuffle, sizeof(shuf)); +#endif asm volatile( "sub %0,%1 \n" "sub %0,%2 \n" @@ -9342,6 +10271,9 @@ void MergeXRGB16To8Row_AVX2(const uint16_t* src_r, "vpsrlw %%xmm6,%%ymm0,%%ymm0 \n" "vpsrlw %%xmm6,%%ymm1,%%ymm1 \n" "vpsrlw %%xmm6,%%ymm2,%%ymm2 \n" + "vpminuw %%ymm3,%%ymm0,%%ymm0 \n" + "vpminuw %%ymm3,%%ymm1,%%ymm1 \n" + "vpminuw %%ymm3,%%ymm2,%%ymm2 \n" "vpackuswb %%ymm1,%%ymm2,%%ymm1 \n" // BG (planar) "vpackuswb %%ymm3,%%ymm0,%%ymm0 \n" // RA (planar) "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" // BG (interleave) @@ -9357,14 +10289,18 @@ void MergeXRGB16To8Row_AVX2(const uint16_t* src_r, "subl $0x10,%4 \n" "jg 1b \n" "vzeroupper \n" - : "+r"(src_r), // %0 - "+r"(src_g), // %1 - "+r"(src_b), // %2 - "+r"(dst_argb), // %3 - "+r"(width) // %4 - : "m"(shift), // %5 + : "+r"(src_r), // %0 + "+r"(src_g), // %1 + "+r"(src_b), // %2 + "+r"(dst_argb), // %3 + "+rm"(width) // %4 + : "m"(shift), // %5 +#if defined(__i386__) + "m"(shuf) // %6 +#else "m"(MergeARGB16To8Shuffle) // %6 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); +#endif + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6"); } #endif // HAS_MERGEXRGB16TO8ROW_AVX2 @@ -9430,6 +10366,7 @@ void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) { #endif // HAS_COPYROW_AVX #ifdef HAS_COPYROW_AVX512BW +LIBYUV_TARGET_AVX512BW void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { uintptr_t temp; asm volatile( @@ -9449,7 +10386,7 @@ void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { "2: \n" "add $0x80,%2 \n" - "je 99f \n" + "jle 99f \n" "cmp $0x40,%2 \n" "jl 3f \n" "vmovups (%0),%%zmm0 \n" @@ -9457,7 +10394,7 @@ void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { "vmovups %%zmm0,(%1) \n" "lea 0x40(%1),%1 \n" "sub $0x40,%2 \n" - "je 99f \n" + "jle 99f \n" "3: \n" "mov $-1,%[temp] \n" @@ -9621,6 +10558,7 @@ void ARGBExtractAlphaRow_AVX2(const uint8_t* src_argb, #endif // HAS_ARGBEXTRACTALPHAROW_AVX2 #ifdef HAS_ARGBEXTRACTALPHAROW_AVX512BW +LIBYUV_TARGET_AVX512BW void ARGBExtractAlphaRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_a, int width) { @@ -9743,6 +10681,7 @@ void ARGBCopyYToAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { #endif // HAS_ARGBCOPYYTOALPHAROW_AVX2 #ifdef HAS_ARGBCOPYYTOALPHAROW_AVX512BW +LIBYUV_TARGET_AVX512BW void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { uintptr_t temp; asm volatile( @@ -10377,6 +11316,7 @@ void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy, #endif // HAS_YUY2TOYROW_AVX2 #ifdef HAS_YUY2TOYROW_AVX512BW +LIBYUV_TARGET_AVX512BW void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { uintptr_t temp; asm volatile( @@ -10396,7 +11336,7 @@ void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { "2: \n" "add $0x40,%2 \n" - "je 99f \n" + "jle 99f \n" "cmp $0x20,%2 \n" "jl 3f \n" "vmovdqu16 (%0),%%zmm0 \n" @@ -10404,7 +11344,7 @@ void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { "vpmovwb %%zmm0,(%1) \n" "lea 0x20(%1),%1 \n" "sub $0x20,%2 \n" - "je 99f \n" + "jle 99f \n" "3: \n" "mov $-1,%k[temp] \n" @@ -10425,6 +11365,7 @@ void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { #endif // HAS_YUY2TOYROW_AVX512BW #ifdef HAS_YUY2TOUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_u, @@ -10451,7 +11392,7 @@ void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, "2: \n" "add $0x20,%3 \n" - "je 99f \n" + "jle 99f \n" "add $0x1,%k3 \n" "shr $0x1,%k3 \n" "mov $-1,%k[temp] \n" @@ -10478,6 +11419,7 @@ void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, #endif // HAS_YUY2TOUVROW_AVX512BW #ifdef HAS_UYVYTOYROW_AVX512BW +LIBYUV_TARGET_AVX512BW void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { uintptr_t temp; asm volatile( @@ -10499,7 +11441,7 @@ void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { "2: \n" "add $0x40,%2 \n" - "je 99f \n" + "jle 99f \n" "cmp $0x20,%2 \n" "jl 3f \n" "vmovdqu16 (%0),%%zmm0 \n" @@ -10508,7 +11450,7 @@ void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { "vpmovwb %%zmm0,(%1) \n" "lea 0x20(%1),%1 \n" "sub $0x20,%2 \n" - "je 99f \n" + "jle 99f \n" "3: \n" "mov $-1,%k[temp] \n" @@ -10530,6 +11472,7 @@ void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { #endif // HAS_UYVYTOYROW_AVX512BW #ifdef HAS_UYVYTOUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, int stride_uyvy, uint8_t* dst_u, @@ -10555,7 +11498,7 @@ void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, "2: \n" "add $0x20,%3 \n" - "je 99f \n" + "jle 99f \n" "add $0x1,%k3 \n" "shr $0x1,%k3 \n" "mov $-1,%k[temp] \n" @@ -10604,6 +11547,9 @@ void ARGBBlendRow_AVX2(const uint8_t* src_argb, "vpsllw $0x8,%%ymm5,%%ymm5 \n" "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n" "vpslld $0x18,%%ymm4,%%ymm4 \n" +#if defined(__x86_64__) + "vbroadcasti128 %4,%%ymm8 \n" +#endif "sub $0x8,%3 \n" "jl 2f \n" @@ -10615,7 +11561,11 @@ void ARGBBlendRow_AVX2(const uint8_t* src_argb, "lea 0x20(%0),%0 \n" "lea 0x20(%1),%1 \n" "vpxor %%ymm4,%%ymm3,%%ymm0 \n" +#if defined(__x86_64__) + "vpshufb %%ymm8,%%ymm0,%%ymm0 \n" +#else "vpshufb %4,%%ymm0,%%ymm0 \n" +#endif "vpaddw %%ymm7,%%ymm0,%%ymm0 \n" "vpsrlw $0x8,%%ymm2,%%ymm1 \n" "vpand %%ymm6,%%ymm2,%%ymm2 \n" @@ -10639,7 +11589,11 @@ void ARGBBlendRow_AVX2(const uint8_t* src_argb, "lea 0x10(%0),%0 \n" "lea 0x10(%1),%1 \n" "vpxor %%xmm4,%%xmm3,%%xmm0 \n" +#if defined(__x86_64__) + "vpshufb %%xmm8,%%xmm0,%%xmm0 \n" +#else "vpshufb %4,%%xmm0,%%xmm0 \n" +#endif "vpaddw %%xmm7,%%xmm0,%%xmm0 \n" "vpsrlw $0x8,%%xmm2,%%xmm1 \n" "vpand %%xmm6,%%xmm2,%%xmm2 \n" @@ -10666,7 +11620,11 @@ void ARGBBlendRow_AVX2(const uint8_t* src_argb, "lea 0x4(%0),%0 \n" "lea 0x4(%1),%1 \n" "vpxor %%xmm4,%%xmm3,%%xmm0 \n" +#if defined(__x86_64__) + "vpshufb %%xmm8,%%xmm0,%%xmm0 \n" +#else "vpshufb %4,%%xmm0,%%xmm0 \n" +#endif "vpaddw %%xmm7,%%xmm0,%%xmm0 \n" "vpsrlw $0x8,%%xmm2,%%xmm1 \n" "vpand %%xmm6,%%xmm2,%%xmm2 \n" @@ -10690,7 +11648,12 @@ void ARGBBlendRow_AVX2(const uint8_t* src_argb, "+r"(width) // %3 : "m"(kShuffleAlpha[0]) // %4 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + "xmm7" +#if defined(__x86_64__) + , + "xmm8" +#endif + ); } #endif // HAS_ARGBBLENDROW_AVX2 @@ -10813,6 +11776,7 @@ void BlendPlaneRow_AVX2(const uint8_t* src0, // =((A2*C2)+(B2*(255-C2))+255)/256 // signed version of math // =(((A2-128)*C2)+((B2-128)*(255-C2))+32768+127)/256 +LIBYUV_TARGET_AVX512BW void BlendPlaneRow_AVX512BW(const uint8_t* src0, const uint8_t* src1, const uint8_t* alpha, @@ -10861,7 +11825,7 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0, "2: \n" "add $64,%4 \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q4,%[temp],%[temp] \n" "kmovq %[temp],%%k1 \n" @@ -11281,6 +12245,7 @@ void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width) { #endif // HAS_ARGBSEPIAROW_AVX2 #ifdef HAS_ARGBSEPIAROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) { uintptr_t temp; asm volatile( @@ -11320,7 +12285,7 @@ void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) { "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" @@ -11430,6 +12395,7 @@ void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb, #endif // HAS_ARGBCOLORMATRIXROW_AVX2 #ifdef HAS_ARGBCOLORMATRIXROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb, uint8_t* dst_argb, const int8_t* matrix_argb, @@ -11483,7 +12449,7 @@ void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" @@ -12148,13 +13114,18 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, asm volatile( "vpxor %%xmm5,%%xmm5,%%xmm5 \n" "vcvtsi2ssl %5,%%xmm5,%%xmm5 \n" - "vrcpss %%xmm5,%%xmm5,%%xmm4 \n" + "vpcmpeqd %%xmm4,%%xmm4,%%xmm4 \n" + "vpslld $25,%%xmm4,%%xmm4 \n" + "vpsrld $2,%%xmm4,%%xmm4 \n" + "vdivss %%xmm5,%%xmm4,%%xmm4 \n" "vpshufd $0x0,%%xmm4,%%xmm4 \n" "subl $0x4,%3 \n" "jl 19f \n" "vinserti128 $0x1,%%xmm4,%%ymm4,%%ymm4 \n" "cmpl $0x80,%5 \n" "ja 10f \n" + "cmpl $0x1,%5 \n" + "jle 10f \n" "vpcmpeqb %%xmm6,%%xmm6,%%xmm6 \n" "vpsrld $0x10,%%xmm6,%%xmm6 \n" @@ -12162,7 +13133,7 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vaddss %%xmm6,%%xmm5,%%xmm6 \n" "vmulss %%xmm4,%%xmm6,%%xmm6 \n" "vcvtps2dq %%xmm6,%%xmm6 \n" - "vpackssdw %%xmm6,%%xmm6,%%xmm6 \n" + "vpackusdw %%xmm6,%%xmm6,%%xmm6 \n" "vpbroadcastw %%xmm6,%%ymm6 \n" // 4 pixel small loop. @@ -12178,7 +13149,7 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vpaddd 0x00(%1,%4,4),%%ymm0,%%ymm0\n" "vpaddd 0x20(%1,%4,4),%%ymm1,%%ymm1\n" "lea 0x40(%1),%1 \n" - "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" "vextracti128 $0x1,%%ymm0,%%xmm1 \n" "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" @@ -12207,9 +13178,9 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vcvtdq2ps %%ymm1,%%ymm1 \n" "vmulps %%ymm4,%%ymm0,%%ymm0 \n" "vmulps %%ymm4,%%ymm1,%%ymm1 \n" - "vcvtps2dq %%ymm0,%%ymm0 \n" - "vcvtps2dq %%ymm1,%%ymm1 \n" - "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" + "vcvttps2dq %%ymm0,%%ymm0 \n" + "vcvttps2dq %%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" "vextracti128 $0x1,%%ymm0,%%xmm1 \n" "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" "vpshufd $0xd8,%%xmm0,%%xmm0 \n" @@ -12235,8 +13206,8 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" "vcvtdq2ps %%xmm0,%%xmm0 \n" "vmulps %%xmm4,%%xmm0,%%xmm0 \n" - "vcvtps2dq %%xmm0,%%xmm0 \n" - "vpackssdw %%xmm0,%%xmm0,%%xmm0 \n" + "vcvttps2dq %%xmm0,%%xmm0 \n" + "vpackusdw %%xmm0,%%xmm0,%%xmm0 \n" "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" "vmovd %%xmm0,(%2) \n" "lea 0x4(%2),%2 \n" @@ -12405,12 +13376,13 @@ void InterpolateRow_AVX2(uint8_t* dst_ptr, "+r"(width), // %2 "+r"(source_y_fraction) // %3 : "r"(src_stride) // %4 - : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm4", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm4", "xmm5"); } #endif // HAS_INTERPOLATEROW_AVX2 #ifdef HAS_INTERPOLATEROW_AVX512BW // Bilinear filter 64x2 -> 64x1 with masked tail for any width. +LIBYUV_TARGET_AVX512BW void InterpolateRow_AVX512BW(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -12555,16 +13527,9 @@ void InterpolateRow_16_AVX2(uint16_t* dst_ptr, "vmovd %3,%%xmm5 \n" "vpunpcklwd %%xmm0,%%xmm5,%%xmm5 \n" "vpbroadcastd %%xmm5,%%ymm5 \n" - "mov $0x80008000,%%eax \n" // 0x80008000 used to bias - // unsigned words to - // signed range for - // vpmaddwd. - "vmovd %%eax,%%xmm4 \n" - "vbroadcastss %%xmm4,%%ymm4 \n" - "mov $8388736,%%eax \n" // 32768 * 256 + 128 - // rounding constant. - "vmovd %%eax,%%xmm3 \n" - "vbroadcastss %%xmm3,%%ymm3 \n" + "vpcmpeqw %%ymm4,%%ymm4,%%ymm4 \n" + "vpsllw $15,%%ymm4,%%ymm4 \n" // 0x8000 unsigned bias + "vpsrlw $8,%%ymm4,%%ymm3 \n" // 0x00800080 = 8388736 LABELALIGN "1: \n" @@ -12612,7 +13577,7 @@ void InterpolateRow_16_AVX2(uint16_t* dst_ptr, "+r"(width), // %2 "+r"(source_y_fraction) // %3 : "r"(src_stride) // %4 - : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); } #endif // HAS_INTERPOLATEROW_16_AVX2 @@ -12675,6 +13640,7 @@ void ARGBShuffleRow_AVX2(const uint8_t* src_argb, #ifdef HAS_ARGBSHUFFLEROW_AVX512BW // For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. +LIBYUV_TARGET_AVX512BW void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, const uint8_t* shuffler, @@ -12700,7 +13666,7 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb, "2: \n" "add $0x20,%2 \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q2,%[temp],%[temp] \n" "kmovd %k[temp],%%k1 \n" @@ -12885,20 +13851,24 @@ void ARGBPolynomialRow_SSE2(const uint8_t* src_argb, "cvtdq2ps %%xmm4,%%xmm4 \n" "movdqa %%xmm0,%%xmm1 \n" "movdqa %%xmm4,%%xmm5 \n" - "mulps 0x10(%3),%%xmm0 \n" - "mulps 0x10(%3),%%xmm4 \n" - "addps (%3),%%xmm0 \n" - "addps (%3),%%xmm4 \n" + "movups 0x10(%3),%%xmm7 \n" + "mulps %%xmm7,%%xmm0 \n" + "mulps %%xmm7,%%xmm4 \n" + "movups (%3),%%xmm7 \n" + "addps %%xmm7,%%xmm0 \n" + "addps %%xmm7,%%xmm4 \n" "movdqa %%xmm1,%%xmm2 \n" "movdqa %%xmm5,%%xmm6 \n" "mulps %%xmm1,%%xmm2 \n" "mulps %%xmm5,%%xmm6 \n" "mulps %%xmm2,%%xmm1 \n" "mulps %%xmm6,%%xmm5 \n" - "mulps 0x20(%3),%%xmm2 \n" - "mulps 0x20(%3),%%xmm6 \n" - "mulps 0x30(%3),%%xmm1 \n" - "mulps 0x30(%3),%%xmm5 \n" + "movups 0x20(%3),%%xmm7 \n" + "mulps %%xmm7,%%xmm2 \n" + "mulps %%xmm7,%%xmm6 \n" + "movups 0x30(%3),%%xmm7 \n" + "mulps %%xmm7,%%xmm1 \n" + "mulps %%xmm7,%%xmm5 \n" "addps %%xmm2,%%xmm0 \n" "addps %%xmm6,%%xmm4 \n" "addps %%xmm1,%%xmm0 \n" @@ -12916,7 +13886,7 @@ void ARGBPolynomialRow_SSE2(const uint8_t* src_argb, "+r"(width) // %2 : "r"(poly) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6"); + "xmm6", "xmm7"); } #endif // HAS_ARGBPOLYNOMIALROW_SSE2 @@ -12944,7 +13914,7 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, "vfmadd231ps %%ymm3,%%ymm2,%%ymm0 \n" // result += C3 * X * X * // X "vcvttps2dq %%ymm0,%%ymm0 \n" - "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n" + "vpackssdw %%ymm0,%%ymm0,%%ymm0 \n" "vpermq $0xd8,%%ymm0,%%ymm0 \n" "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" "vmovq %%xmm0,(%1) \n" @@ -12956,12 +13926,11 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, "+r"(dst_argb), // %1 "+r"(width) // %2 : "r"(poly) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + : "memory", "cc", "xmm0", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } #endif // HAS_ARGBPOLYNOMIALROW_AVX2 -#if defined(HAS_HALFFLOATROW_AVX2) || defined(HAS_HALFFLOATROW_AVX512BW) +#if defined(HAS_HALFFLOATROW_AVX2) // 2^-112 moves the float exponent into half float position. Applied as a // separate multiply after scale so no denormal constant is formed when // scale < 2^-14 (see HalfFloatRow_C). @@ -13015,6 +13984,7 @@ void HalfFloatRow_AVX2(const uint16_t* src, #endif // HAS_HALFFLOATROW_AVX2 #ifdef HAS_HALFFLOATROW_AVX512BW +LIBYUV_TARGET_AVX512BW void HalfFloatRow_AVX512BW(const uint16_t* src, uint16_t* dst, float scale, @@ -13022,7 +13992,6 @@ void HalfFloatRow_AVX512BW(const uint16_t* src, uintptr_t temp; asm volatile( "vbroadcastss %[scale],%%zmm4 \n" - "vbroadcastss %[bias],%%zmm5 \n" "sub $0x10,%[width] \n" "jl 2f \n" @@ -13032,16 +14001,14 @@ void HalfFloatRow_AVX512BW(const uint16_t* src, "lea 0x20(%[src]),%[src] \n" "vcvtdq2ps %%zmm0,%%zmm0 \n" "vmulps %%zmm4,%%zmm0,%%zmm0 \n" - "vmulps %%zmm5,%%zmm0,%%zmm0 \n" - "vpsrld $0xd,%%zmm0,%%zmm0 \n" - "vpmovdw %%zmm0,(%[dst]) \n" + "vcvtps2ph $3,%%zmm0,(%[dst]) \n" "lea 0x20(%[dst]),%[dst] \n" "sub $0x10,%[width] \n" "jge 1b \n" "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" @@ -13049,16 +14016,14 @@ void HalfFloatRow_AVX512BW(const uint16_t* src, "vpmovzxwd (%[src]),%%zmm0%{%%k1%}%{z%} \n" "vcvtdq2ps %%zmm0,%%zmm0 \n" "vmulps %%zmm4,%%zmm0,%%zmm0 \n" - "vmulps %%zmm5,%%zmm0,%%zmm0 \n" - "vpsrld $0xd,%%zmm0,%%zmm0 \n" - "vpmovdw %%zmm0,(%[dst])%{%%k1%} \n" + "vcvtps2ph $3,%%zmm0,(%[dst])%{%%k1%} \n" "99: \n" "vzeroupper \n" : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [temp] "=&r"(temp) - : [scale] "m"(scale), [bias] "m"(kScaleBias) - : "memory", "cc", "xmm0", "xmm4", "xmm5", "k1"); + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm4", "k1"); } #endif // HAS_HALFFLOATROW_AVX512BW @@ -13071,11 +14036,15 @@ void HalfFloatRow_F16C(const uint16_t* src, "vbroadcastss %3, %%ymm4 \n" "sub %0,%1 \n" - // 16 pixel loop. + // 16 pixel loop (pure AVX1 + F16C). LABELALIGN "1: \n" - "vpmovzxwd (%0),%%ymm2 \n" // 16 shorts -> 16 ints - "vpmovzxwd 0x10(%0),%%ymm3 \n" + "vpmovzxwd (%0),%%xmm2 \n" // 16 shorts -> 16 ints + "vpmovzxwd 0x08(%0),%%xmm0 \n" + "vinsertf128 $1,%%xmm0,%%ymm2,%%ymm2 \n" + "vpmovzxwd 0x10(%0),%%xmm3 \n" + "vpmovzxwd 0x18(%0),%%xmm0 \n" + "vinsertf128 $1,%%xmm0,%%ymm3,%%ymm3 \n" "vcvtdq2ps %%ymm2,%%ymm2 \n" "vcvtdq2ps %%ymm3,%%ymm3 \n" "vmulps %%ymm2,%%ymm4,%%ymm2 \n" @@ -13091,12 +14060,8 @@ void HalfFloatRow_F16C(const uint16_t* src, : "+r"(src), // %0 "+r"(dst), // %1 "+r"(width) // %2 -#if defined(__x86_64__) - : "x"(scale) // %3 -#else - : "m"(scale) // %3 -#endif - : "memory", "cc", "xmm2", "xmm3", "xmm4"); + : "m"(scale) // %3 + : "memory", "cc", "xmm0", "xmm2", "xmm3", "xmm4"); } #endif // HAS_HALFFLOATROW_F16C @@ -13104,11 +14069,15 @@ void HalfFloatRow_F16C(const uint16_t* src, void HalfFloat1Row_F16C(const uint16_t* src, uint16_t* dst, float, int width) { asm volatile( "sub %0,%1 \n" - // 16 pixel loop. + // 16 pixel loop (pure AVX1 + F16C). LABELALIGN "1: \n" - "vpmovzxwd (%0),%%ymm2 \n" // 16 shorts -> 16 ints - "vpmovzxwd 0x10(%0),%%ymm3 \n" + "vpmovzxwd (%0),%%xmm2 \n" // 16 shorts -> 16 ints + "vpmovzxwd 0x08(%0),%%xmm0 \n" + "vinsertf128 $1,%%xmm0,%%ymm2,%%ymm2 \n" + "vpmovzxwd 0x10(%0),%%xmm3 \n" + "vpmovzxwd 0x18(%0),%%xmm0 \n" + "vinsertf128 $1,%%xmm0,%%ymm3,%%ymm3 \n" "vcvtdq2ps %%ymm2,%%ymm2 \n" "vcvtdq2ps %%ymm3,%%ymm3 \n" "vcvtps2ph $3, %%ymm2, %%xmm2 \n" @@ -13123,7 +14092,7 @@ void HalfFloat1Row_F16C(const uint16_t* src, uint16_t* dst, float, int width) { "+r"(dst), // %1 "+r"(width) // %2 : - : "memory", "cc", "xmm2", "xmm3"); + : "memory", "cc", "xmm0", "xmm2", "xmm3"); } #endif // HAS_HALFFLOATROW_F16C @@ -13918,138 +14887,103 @@ void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, int width, const uint8_t* luma, uint32_t lumacoeff) { - uint32_t coeff = lumacoeff & 0x00ffffffu; - uintptr_t pixel_temp; - uintptr_t table_temp; + uint32_t c_br = lumacoeff & 0x00ff00ffu; + uint32_t c_ga = (lumacoeff >> 8) & 0x000000ffu; asm volatile( - "vpbroadcastd %[coeff],%%ymm5 \n" - "vpcmpeqd %%xmm4,%%xmm4,%%xmm4 \n" - "vpsrlw $9,%%xmm4,%%xmm4 \n" - "vpsllw $8,%%xmm4,%%xmm4 \n" // 0x7f00 + "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrld $24,%%ymm6,%%ymm6 \n" // 0x000000ff +#if defined(__x86_64__) + "vpbroadcastd %[c_br],%%ymm8 \n" + "vpbroadcastd %[c_ga],%%ymm9 \n" + "vpsrld $1,%%ymm6,%%ymm10 \n" + "vpslld $8,%%ymm10,%%ymm10 \n" // 0x00007f00 + "vpsrld $6,%%ymm6,%%ymm11 \n" + "vpslld $3,%%ymm11,%%ymm11 \n" // 0x00000018 +#endif "subl $0x8,%[width] \n" "jl 2f \n" LABELALIGN "1: \n" "vmovdqu (%[src]),%%ymm0 \n" - "vpmaddubsw %%ymm5,%%ymm0,%%ymm1 \n" - "vphaddw %%ymm1,%%ymm1,%%ymm1 \n" - "vpermq $0xd8,%%ymm1,%%ymm1 \n" - "vpand %%xmm4,%%xmm1,%%xmm1 \n" - - "vpextrw $0,%%xmm1,%k[t1] \n" - "add %[luma],%[t1] \n" - "movzbl 0x00(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x00(%[dst]) \n" - "movzbl 0x01(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x01(%[dst]) \n" - "movzbl 0x02(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x02(%[dst]) \n" - "movzbl 0x03(%[src]),%k[t0] \n" - "mov %b[t0],0x03(%[dst]) \n" - - "vpextrw $1,%%xmm1,%k[t1] \n" - "add %[luma],%[t1] \n" - "movzbl 0x04(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x04(%[dst]) \n" - "movzbl 0x05(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x05(%[dst]) \n" - "movzbl 0x06(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x06(%[dst]) \n" - "movzbl 0x07(%[src]),%k[t0] \n" - "mov %b[t0],0x07(%[dst]) \n" - - "vpextrw $2,%%xmm1,%k[t1] \n" - "add %[luma],%[t1] \n" - "movzbl 0x08(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x08(%[dst]) \n" - "movzbl 0x09(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x09(%[dst]) \n" - "movzbl 0x0a(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x0a(%[dst]) \n" - "movzbl 0x0b(%[src]),%k[t0] \n" - "mov %b[t0],0x0b(%[dst]) \n" - - "vpextrw $3,%%xmm1,%k[t1] \n" - "add %[luma],%[t1] \n" - "movzbl 0x0c(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x0c(%[dst]) \n" - "movzbl 0x0d(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x0d(%[dst]) \n" - "movzbl 0x0e(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x0e(%[dst]) \n" - "movzbl 0x0f(%[src]),%k[t0] \n" - "mov %b[t0],0x0f(%[dst]) \n" - - "vpextrw $4,%%xmm1,%k[t1] \n" - "add %[luma],%[t1] \n" - "movzbl 0x10(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x10(%[dst]) \n" - "movzbl 0x11(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x11(%[dst]) \n" - "movzbl 0x12(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x12(%[dst]) \n" - "movzbl 0x13(%[src]),%k[t0] \n" - "mov %b[t0],0x13(%[dst]) \n" - - "vpextrw $5,%%xmm1,%k[t1] \n" - "add %[luma],%[t1] \n" - "movzbl 0x14(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x14(%[dst]) \n" - "movzbl 0x15(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x15(%[dst]) \n" - "movzbl 0x16(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x16(%[dst]) \n" - "movzbl 0x17(%[src]),%k[t0] \n" - "mov %b[t0],0x17(%[dst]) \n" - - "vpextrw $6,%%xmm1,%k[t1] \n" - "add %[luma],%[t1] \n" - "movzbl 0x18(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x18(%[dst]) \n" - "movzbl 0x19(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x19(%[dst]) \n" - "movzbl 0x1a(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x1a(%[dst]) \n" - "movzbl 0x1b(%[src]),%k[t0] \n" - "mov %b[t0],0x1b(%[dst]) \n" - - "vpextrw $7,%%xmm1,%k[t1] \n" - "add %[luma],%[t1] \n" - "movzbl 0x1c(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x1c(%[dst]) \n" - "movzbl 0x1d(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x1d(%[dst]) \n" - "movzbl 0x1e(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x1e(%[dst]) \n" - "movzbl 0x1f(%[src]),%k[t0] \n" - "mov %b[t0],0x1f(%[dst]) \n" - "lea 0x20(%[src]),%[src] \n" + "vpsllw $8,%%ymm0,%%ymm1 \n" + "vpsrlw $8,%%ymm1,%%ymm1 \n" // [B, R] + "vpsrlw $8,%%ymm0,%%ymm2 \n" // [G, A] +#if defined(__x86_64__) + "vpmaddwd %%ymm8,%%ymm1,%%ymm1 \n" + "vpmaddwd %%ymm9,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm2,%%ymm1,%%ymm1 \n" + "vpand %%ymm10,%%ymm1,%%ymm1 \n" +#else + "vpbroadcastd %[c_br],%%ymm3 \n" + "vpmaddwd %%ymm3,%%ymm1,%%ymm1 \n" + "vpbroadcastd %[c_ga],%%ymm3 \n" + "vpmaddwd %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm2,%%ymm1,%%ymm1 \n" + "vpsrld $1,%%ymm6,%%ymm3 \n" + "vpslld $8,%%ymm3,%%ymm3 \n" + "vpand %%ymm3,%%ymm1,%%ymm1 \n" +#endif + + "vpand %%ymm6,%%ymm0,%%ymm3 \n" + "vpor %%ymm1,%%ymm3,%%ymm3 \n" + "vpsrld $2,%%ymm3,%%ymm3 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[luma],%%ymm3,4),%%ymm2\n" +#if defined(__x86_64__) + "vpslld $3,%%ymm0,%%ymm4 \n" + "vpand %%ymm11,%%ymm4,%%ymm4 \n" +#else + "vpslld $30,%%ymm0,%%ymm4 \n" + "vpsrld $27,%%ymm4,%%ymm4 \n" +#endif + "vpsrlvd %%ymm4,%%ymm2,%%ymm2 \n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + + "vpsrld $8,%%ymm0,%%ymm3 \n" + "vpand %%ymm6,%%ymm3,%%ymm3 \n" + "vpor %%ymm1,%%ymm3,%%ymm3 \n" + "vpsrld $2,%%ymm3,%%ymm3 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[luma],%%ymm3,4),%%ymm5\n" +#if defined(__x86_64__) + "vpsrld $5,%%ymm0,%%ymm4 \n" + "vpand %%ymm11,%%ymm4,%%ymm4 \n" +#else + "vpslld $22,%%ymm0,%%ymm4 \n" + "vpsrld $30,%%ymm4,%%ymm4 \n" + "vpslld $3,%%ymm4,%%ymm4 \n" +#endif + "vpsrlvd %%ymm4,%%ymm5,%%ymm5 \n" + "vpand %%ymm6,%%ymm5,%%ymm5 \n" + "vpslld $8,%%ymm5,%%ymm5 \n" + "vpor %%ymm5,%%ymm2,%%ymm2 \n" + + "vpsrld $16,%%ymm0,%%ymm3 \n" + "vpand %%ymm6,%%ymm3,%%ymm3 \n" + "vpor %%ymm1,%%ymm3,%%ymm3 \n" + "vpsrld $2,%%ymm3,%%ymm3 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[luma],%%ymm3,4),%%ymm5\n" +#if defined(__x86_64__) + "vpsrld $13,%%ymm0,%%ymm4 \n" + "vpand %%ymm11,%%ymm4,%%ymm4 \n" +#else + "vpslld $14,%%ymm0,%%ymm4 \n" + "vpsrld $30,%%ymm4,%%ymm4 \n" + "vpslld $3,%%ymm4,%%ymm4 \n" +#endif + "vpsrlvd %%ymm4,%%ymm5,%%ymm5 \n" + "vpand %%ymm6,%%ymm5,%%ymm5 \n" + "vpslld $16,%%ymm5,%%ymm5 \n" + "vpor %%ymm5,%%ymm2,%%ymm2 \n" + + "vpsrld $24,%%ymm0,%%ymm0 \n" + "vpslld $24,%%ymm0,%%ymm0 \n" + "vpor %%ymm0,%%ymm2,%%ymm2 \n" + + "vmovdqu %%ymm2,(%[dst]) \n" "lea 0x20(%[dst]),%[dst] \n" "subl $0x8,%[width] \n" "jge 1b \n" @@ -14057,50 +14991,116 @@ void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, "2: \n" "addl $0x8,%[width] \n" "jle 99f \n" + "vpcmpeqd %%xmm5,%%xmm5,%%xmm5 \n" + "vpsrldq $12,%%xmm5,%%xmm5 \n" LABELALIGN "3: \n" "vmovd (%[src]),%%xmm0 \n" - "vpmaddubsw %%xmm5,%%xmm0,%%xmm1 \n" - "vphaddw %%xmm1,%%xmm1,%%xmm1 \n" - "vpand %%xmm4,%%xmm1,%%xmm1 \n" - "vpextrw $0,%%xmm1,%k[t1] \n" - "add %[luma],%[t1] \n" - "movzbl 0x00(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x00(%[dst]) \n" - "movzbl 0x01(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x01(%[dst]) \n" - "movzbl 0x02(%[src]),%k[t0] \n" - "movzbl (%[t1],%[t0],1),%k[t0] \n" - "mov %b[t0],0x02(%[dst]) \n" - "movzbl 0x03(%[src]),%k[t0] \n" - "mov %b[t0],0x03(%[dst]) \n" "lea 0x04(%[src]),%[src] \n" + "vpsllw $8,%%xmm0,%%xmm1 \n" + "vpsrlw $8,%%xmm1,%%xmm1 \n" + "vpsrlw $8,%%xmm0,%%xmm2 \n" +#if defined(__x86_64__) + "vpmaddwd %%xmm8,%%xmm1,%%xmm1 \n" + "vpmaddwd %%xmm9,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm2,%%xmm1,%%xmm1 \n" + "vpand %%xmm10,%%xmm1,%%xmm1 \n" +#else + "vpbroadcastd %[c_br],%%xmm3 \n" + "vpmaddwd %%xmm3,%%xmm1,%%xmm1 \n" + "vpbroadcastd %[c_ga],%%xmm3 \n" + "vpmaddwd %%xmm3,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm2,%%xmm1,%%xmm1 \n" + "vpsrld $1,%%xmm6,%%xmm3 \n" + "vpslld $8,%%xmm3,%%xmm3 \n" + "vpand %%xmm3,%%xmm1,%%xmm1 \n" +#endif + + "vpand %%xmm6,%%xmm0,%%xmm3 \n" + "vpor %%xmm1,%%xmm3,%%xmm3 \n" + "vpsrld $2,%%xmm3,%%xmm3 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[luma],%%xmm3,4),%%xmm2\n" +#if defined(__x86_64__) + "vpslld $3,%%xmm0,%%xmm4 \n" + "vpand %%xmm11,%%xmm4,%%xmm4 \n" +#else + "vpslld $30,%%xmm0,%%xmm4 \n" + "vpsrld $27,%%xmm4,%%xmm4 \n" +#endif + "vpsrlvd %%xmm4,%%xmm2,%%xmm2 \n" + "vpand %%xmm6,%%xmm2,%%xmm2 \n" + + "vpsrld $8,%%xmm0,%%xmm3 \n" + "vpand %%xmm6,%%xmm3,%%xmm3 \n" + "vpor %%xmm1,%%xmm3,%%xmm3 \n" + "vpsrld $2,%%xmm3,%%xmm3 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[luma],%%xmm3,4),%%xmm4\n" +#if defined(__x86_64__) + "vpsrld $5,%%xmm0,%%xmm3 \n" + "vpand %%xmm11,%%xmm3,%%xmm3 \n" +#else + "vpslld $22,%%xmm0,%%xmm3 \n" + "vpsrld $30,%%xmm3,%%xmm3 \n" + "vpslld $3,%%xmm3,%%xmm3 \n" +#endif + "vpsrlvd %%xmm3,%%xmm4,%%xmm4 \n" + "vpand %%xmm6,%%xmm4,%%xmm4 \n" + "vpslld $8,%%xmm4,%%xmm4 \n" + "vpor %%xmm4,%%xmm2,%%xmm2 \n" + + "vpsrld $16,%%xmm0,%%xmm3 \n" + "vpand %%xmm6,%%xmm3,%%xmm3 \n" + "vpor %%xmm1,%%xmm3,%%xmm3 \n" + "vpsrld $2,%%xmm3,%%xmm3 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[luma],%%xmm3,4),%%xmm4\n" +#if defined(__x86_64__) + "vpsrld $13,%%xmm0,%%xmm3 \n" + "vpand %%xmm11,%%xmm3,%%xmm3 \n" +#else + "vpslld $14,%%xmm0,%%xmm3 \n" + "vpsrld $30,%%xmm3,%%xmm3 \n" + "vpslld $3,%%xmm3,%%xmm3 \n" +#endif + "vpsrlvd %%xmm3,%%xmm4,%%xmm4 \n" + "vpand %%xmm6,%%xmm4,%%xmm4 \n" + "vpslld $16,%%xmm4,%%xmm4 \n" + "vpor %%xmm4,%%xmm2,%%xmm2 \n" + + "vpsrld $24,%%xmm0,%%xmm0 \n" + "vpslld $24,%%xmm0,%%xmm0 \n" + "vpor %%xmm0,%%xmm2,%%xmm2 \n" + + "vmovd %%xmm2,(%[dst]) \n" "lea 0x04(%[dst]),%[dst] \n" "subl $0x1,%[width] \n" "jg 3b \n" "99: \n" "vzeroupper \n" - : [src] "+r"(src_argb), [dst] "+r"(dst_argb), -#if defined(__i386__) - [width] "+m"(width), -#else - [width] "+rm"(width), + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+rm"(width) + : [luma] "r"(luma), [c_br] "m"(c_br), [c_ga] "m"(c_ga) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7" +#if defined(__x86_64__) + , + "xmm8", "xmm9", "xmm10", "xmm11" #endif - [t0] "=&q"(pixel_temp), [t1] "=&r"(table_temp) - : [luma] "r"(luma), [coeff] "m"(coeff) - : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5"); + ); } #endif // HAS_ARGBLUMACOLORTABLEROW_AVX2 +#if defined(HAS_NV21TOYUV24ROW_SSSE3) || defined(HAS_NV21TOYUV24ROW_AVX2) static const uvec8 kYUV24Shuffle[3] = { {8, 9, 0, 8, 9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12}, {9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15}, {2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15, 6, 14, 15, 7}}; +#endif +#ifdef HAS_NV21TOYUV24ROW_SSSE3 // Convert biplanar NV21 to packed YUV24 // NV21 has VU in memory for chroma. // YUV24 is VUY in memory @@ -14138,7 +15138,9 @@ void NV21ToYUV24Row_SSSE3(const uint8_t* src_y, : "r"(&kYUV24Shuffle[0]) // %4 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } +#endif // HAS_NV21TOYUV24ROW_SSSE3 +#ifdef HAS_NV21TOYUV24ROW_AVX2 // Convert biplanar NV21 to packed YUV24 // NV21 has VU in memory for chroma. // YUV24 is VUY in memory @@ -14179,52 +15181,100 @@ void NV21ToYUV24Row_AVX2(const uint8_t* src_y, : "r"(&kYUV24Shuffle[0]) // %4 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } +#endif // HAS_NV21TOYUV24ROW_AVX2 -#ifdef HAS_NV21ToYUV24ROW_AVX512 -// The following VMBI VEX256 code tests okay with the intelsde emulator. -static const lvec8 kYUV24Perm[3] = { - {32, 33, 0, 32, 33, 1, 34, 35, 2, 34, 35, 3, 36, 37, 4, 36, - 37, 5, 38, 39, 6, 38, 39, 7, 40, 41, 8, 40, 41, 9, 42, 43}, - {10, 42, 43, 11, 44, 45, 12, 44, 45, 13, 46, 47, 14, 46, 47, 15, - 48, 49, 16, 48, 49, 17, 50, 51, 18, 50, 51, 19, 52, 53, 20, 52}, - {53, 21, 54, 55, 22, 54, 55, 23, 56, 57, 24, 56, 57, 25, 58, 59, - 26, 58, 59, 27, 60, 61, 28, 60, 61, 29, 62, 63, 30, 62, 63, 31}}; +#ifdef HAS_NV21TOYUV24ROW_AVX512VBMI +static const uint8_t kNV21ToYUV24Perm_AVX512VBMI[3][64] = { + {64, 65, 0, 64, 65, 1, 66, 67, 2, 66, 67, 3, 68, 69, 4, 68, + 69, 5, 70, 71, 6, 70, 71, 7, 72, 73, 8, 72, 73, 9, 74, 75, + 10, 74, 75, 11, 76, 77, 12, 76, 77, 13, 78, 79, 14, 78, 79, 15, + 80, 81, 16, 80, 81, 17, 82, 83, 18, 82, 83, 19, 84, 85, 20, 84}, + {85, 21, 86, 87, 22, 86, 87, 23, 88, 89, 24, 88, 89, + 25, 90, 91, 26, 90, 91, 27, 92, 93, 28, 92, 93, 29, + 94, 95, 30, 94, 95, 31, 96, 97, 32, 96, 97, 33, 98, + 99, 34, 98, 99, 35, 100, 101, 36, 100, 101, 37, 102, 103, + 38, 102, 103, 39, 104, 105, 40, 104, 105, 41, 106, 107}, + {42, 106, 107, 43, 108, 109, 44, 108, 109, 45, 110, 111, 46, + 110, 111, 47, 112, 113, 48, 112, 113, 49, 114, 115, 50, 114, + 115, 51, 116, 117, 52, 116, 117, 53, 118, 119, 54, 118, 119, + 55, 120, 121, 56, 120, 121, 57, 122, 123, 58, 122, 123, 59, + 124, 125, 60, 124, 125, 61, 126, 127, 62, 126, 127, 63}}; -void NV21ToYUV24Row_AVX512(const uint8_t* src_y, - const uint8_t* src_vu, - uint8_t* dst_yuv24, - int width) { +LIBYUV_TARGET_AVX512VBMI +void NV21ToYUV24Row_AVX512VBMI(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_yuv24, + int width) { + uintptr_t temp; + uintptr_t mask; asm volatile( - "sub %0,%1 \n" - "vmovdqa32 (%4),%%ymm4 \n" // 3 shuffler constants - "vmovdqa32 32(%4),%%ymm5 \n" - "vmovdqa32 64(%4),%%ymm16 \n" LABELALIGN - "1: \n" - "vmovdqu (%0),%%ymm2 \n" // load 32 Y values - "vmovdqu (%0,%1),%%ymm3 \n" // load 16 VU values - "lea 32(%0),%0 \n" - "vmovdqa %%ymm2, %%ymm0 \n" - "vmovdqa %%ymm2, %%ymm1 \n" - "vpermt2b %%ymm3,%%ymm4,%%ymm0 \n" - "vpermt2b %%ymm3,%%ymm5,%%ymm1 \n" - "vpermt2b %%ymm3,%%ymm16,%%ymm2 \n" - "vmovdqu %%ymm0,(%2) \n" - "vmovdqu %%ymm1,32(%2) \n" - "vmovdqu %%ymm2,64(%2) \n" - "lea 96(%2),%2 \n" - "sub $32,%3 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_y), // %0 - "+r"(src_vu), // %1 - "+r"(dst_yuv24), // %2 - "+r"(width) // %3 - : "r"(&kYUV24Perm[0]) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm16"); -} + "sub %[src_y],%[src_vu] \n" + "vmovdqu8 (%[perm]),%%zmm4 \n" + "vmovdqu8 64(%[perm]),%%zmm5 \n" + "vmovdqu8 128(%[perm]),%%zmm16 \n" + "sub $0x40,%[width] \n" + "jl 2f \n" -#endif // HAS_NV21ToYUV24ROW_AVX512 + LABELALIGN + "1: \n" + "vmovdqu8 (%[src_y]),%%zmm2 \n" // load 64 Y values + "vmovdqu8 (%[src_y],%[src_vu],1),%%zmm3 \n" // load 32 VU pairs + "lea 0x40(%[src_y]),%[src_y] \n" + "vmovdqa64 %%zmm2,%%zmm0 \n" + "vmovdqa64 %%zmm2,%%zmm1 \n" + "vpermt2b %%zmm3,%%zmm4,%%zmm0 \n" + "vpermt2b %%zmm3,%%zmm5,%%zmm1 \n" + "vpermt2b %%zmm3,%%zmm16,%%zmm2 \n" + "vmovdqu8 %%zmm0,(%[dst_yuv24]) \n" + "vmovdqu8 %%zmm1,0x40(%[dst_yuv24]) \n" + "vmovdqu8 %%zmm2,0x80(%[dst_yuv24]) \n" + "lea 0xc0(%[dst_yuv24]),%[dst_yuv24] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "jle 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[mask] \n" + "kmovq %[mask],%%k1 \n" + "lea 1(%q[width]),%k[mask] \n" + "and $-2,%k[mask] \n" + "bzhi %q[mask],%[temp],%[mask] \n" + "kmovq %[mask],%%k2 \n" + "vmovdqu8 (%[src_y]),%%zmm2%{%%k1%}%{z%} \n" + "vmovdqu8 (%[src_y],%[src_vu],1),%%zmm3%{%%k2%}%{z%} \n" + "lea (%q[width],%q[width],2),%k[width] \n" + "bzhi %q[width],%[temp],%[mask] \n" + "kmovq %[mask],%%k1 \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k2 \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k3 \n" + "vmovdqa64 %%zmm2,%%zmm0 \n" + "vmovdqa64 %%zmm2,%%zmm1 \n" + "vpermt2b %%zmm3,%%zmm4,%%zmm0 \n" + "vpermt2b %%zmm3,%%zmm5,%%zmm1 \n" + "vpermt2b %%zmm3,%%zmm16,%%zmm2 \n" + "vmovdqu8 %%zmm0,(%[dst_yuv24])%{%%k1%} \n" + "vmovdqu8 %%zmm1,0x40(%[dst_yuv24])%{%%k2%} \n" + "vmovdqu8 %%zmm2,0x80(%[dst_yuv24])%{%%k3%} \n" + + "99: \n" + "vzeroupper \n" + : [src_y] "+r"(src_y), [src_vu] "+r"(src_vu), [dst_yuv24] "+r"(dst_yuv24), + [width] "+r"(width), [temp] "=&r"(temp), [mask] "=&r"(mask) + : [perm] "r"(&kNV21ToYUV24Perm_AVX512VBMI[0][0]) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "k1", "k2", "k3"); +} +#endif // HAS_NV21TOYUV24ROW_AVX512VBMI #if defined(HAS_SWAPUVROW_SSSE3) || defined(HAS_SWAPUVROW_AVX2) @@ -14283,6 +15333,7 @@ void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) { } #endif // HAS_SWAPUVROW_AVX2 +#ifdef HAS_HALFMERGEUVROW_SSSE3 void HalfMergeUVRow_SSSE3(const uint8_t* src_u, int src_stride_u, const uint8_t* src_v, @@ -14327,7 +15378,9 @@ void HalfMergeUVRow_SSSE3(const uint8_t* src_u, "r"((ptrdiff_t)(src_stride_v)) // %5 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); } +#endif // HAS_HALFMERGEUVROW_SSSE3 +#ifdef HAS_HALFMERGEUVROW_AVX2 void HalfMergeUVRow_AVX2(const uint8_t* src_u, int src_stride_u, const uint8_t* src_v, @@ -14373,26 +15426,7 @@ void HalfMergeUVRow_AVX2(const uint8_t* src_u, "r"((ptrdiff_t)(src_stride_v)) // %5 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); } - -void ClampFloatToZero_SSE2(const float* src_x, float* dst_y, int width) { - asm volatile( - "pxor %%xmm1,%%xmm1 \n" - - LABELALIGN - "1: \n" - "movd (%0),%%xmm0 \n" // load float - "maxss %%xmm1, %%xmm0 \n" // clamp to zero - "lea 0x4(%0),%0 \n" - "movd %%xmm0, (%1) \n" // store float - "lea 0x4(%1),%1 \n" - "sub $0x1,%2 \n" // 1 float per loop - "jg 1b \n" - : "+r"(src_x), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : - : "memory", "cc", "xmm0", "xmm1"); -} +#endif // HAS_HALFMERGEUVROW_AVX2 #ifdef HAS_CONVERT8TO8ROW_AVX2 void Convert8To8Row_AVX2(const uint8_t* src_y, @@ -14434,6 +15468,7 @@ void Convert8To8Row_AVX2(const uint8_t* src_y, #endif // HAS_CONVERT8TO8ROW_AVX2 #ifdef HAS_CONVERT8TO8ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void Convert8To8Row_AVX512BW(const uint8_t* src_y, uint8_t* dst_y, int scale, @@ -14465,7 +15500,7 @@ void Convert8To8Row_AVX512BW(const uint8_t* src_y, "2: \n" "add $0x40,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" @@ -14499,6 +15534,7 @@ static const uint8_t kPermAYUVToY32[64] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +LIBYUV_TARGET_AVX512VBMI void OMITFP AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width) { @@ -14522,7 +15558,7 @@ void OMITFP AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovd %k[temp],%%k3 \n" @@ -14557,6 +15593,7 @@ static const uint8_t kPermAYUVToVU[64] = { 77, 80, 84, 81, 85, 88, 92, 89, 93, 96, 100, 97, 101, 104, 108, 105, 109, 112, 116, 113, 117, 120, 124, 121, 125}; +LIBYUV_TARGET_AVX512VBMI static inline void AYUVToUVOrVURow_AVX512VBMI(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_uv, @@ -14595,7 +15632,7 @@ static inline void AYUVToUVOrVURow_AVX512VBMI(const uint8_t* src_ayuv, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp1] \n" "bzhi %q[width],%[temp1],%[temp1] \n" "kmovd %k[temp1],%%k3 \n" @@ -14643,6 +15680,7 @@ static inline void AYUVToUVOrVURow_AVX512VBMI(const uint8_t* src_ayuv, // defined(HAS_AYUVTOVUROW_AVX512VBMI) #ifdef HAS_AYUVTOUVROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI void OMITFP AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_uv, @@ -14653,6 +15691,7 @@ void OMITFP AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv, #endif // HAS_AYUVTOUVROW_AVX512VBMI #ifdef HAS_AYUVTOVUROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI void OMITFP AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, @@ -14812,6 +15851,7 @@ void OMITFP AR30ToARGBRow_AVX2(const uint8_t* src_ar30, #endif // HAS_AR30TOARGBROW_AVX2 #ifdef HAS_AR30TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_argb, int width) { @@ -14849,7 +15889,7 @@ void OMITFP AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -14894,6 +15934,9 @@ void OMITFP AR30ToABGRRow_AVX2(const uint8_t* src_ar30, "vpbroadcastd %[m0],%%ymm4 \n" "vpbroadcastd %[m1],%%ymm5 \n" "vpbroadcastd %[m2],%%ymm6 \n" +#if defined(__x86_64__) + "vpbroadcastd %[m3],%%ymm8 \n" +#endif "vbroadcasti128 %[shuf],%%ymm7 \n" LABELALIGN @@ -14907,8 +15950,12 @@ void OMITFP AR30ToABGRRow_AVX2(const uint8_t* src_ar30, "vpsrld $6,%%ymm0,%%ymm3 \n" "vpand %%ymm4,%%ymm0,%%ymm0 \n" "vpor %%ymm1,%%ymm0,%%ymm0 \n" +#if defined(__x86_64__) + "vpand %%ymm8,%%ymm3,%%ymm3 \n" +#else "vpbroadcastd %[m3],%%ymm1 \n" "vpand %%ymm1,%%ymm3,%%ymm3 \n" +#endif "vpor %%ymm3,%%ymm2,%%ymm2 \n" "vpor %%ymm2,%%ymm0,%%ymm0 \n" "vpshufb %%ymm7,%%ymm0,%%ymm0 \n" @@ -14921,11 +15968,17 @@ void OMITFP AR30ToABGRRow_AVX2(const uint8_t* src_ar30, : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00), [m3] "m"(k03FF0000), [shuf] "m"(kShufSwapRB16) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + "xmm7" +#if defined(__x86_64__) + , + "xmm8" +#endif + ); } #endif // HAS_AR30TOABGRROW_AVX2 #ifdef HAS_AR30TOABGRROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) { @@ -14965,7 +16018,7 @@ void OMITFP AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -15029,6 +16082,7 @@ void OMITFP AR30ToAB30Row_AVX2(const uint8_t* src_ar30, #endif // HAS_AR30TOAB30ROW_AVX2 #ifdef HAS_AR30TOAB30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) { @@ -15061,7 +16115,7 @@ void OMITFP AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -15086,6 +16140,7 @@ void OMITFP AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, #endif // HAS_AR30TOAB30ROW_AVX512BW #ifdef HAS_MERGEXR30ROW_10_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP MergeXR30Row_10_AVX512BW(const uint16_t* src_r, const uint16_t* src_g, const uint16_t* src_b, @@ -15125,7 +16180,7 @@ void OMITFP MergeXR30Row_10_AVX512BW(const uint16_t* src_r, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -15152,6 +16207,7 @@ void OMITFP MergeXR30Row_10_AVX512BW(const uint16_t* src_r, #endif // HAS_MERGEXR30ROW_10_AVX512BW #if defined(HAS_ARGBTOAR30ROW_AVX512BW) || defined(HAS_ABGRTOAR30ROW_AVX512BW) +LIBYUV_TARGET_AVX512BW static inline void ARGBOrABGRToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, const uvec8* shuf_rb, @@ -15184,7 +16240,7 @@ static inline void ARGBOrABGRToAR30Row_AVX512BW(const uint8_t* src, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -15210,6 +16266,7 @@ static inline void ARGBOrABGRToAR30Row_AVX512BW(const uint8_t* src, #endif #ifdef HAS_ARGBTOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP ARGBToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { @@ -15218,6 +16275,7 @@ void OMITFP ARGBToAR30Row_AVX512BW(const uint8_t* src, #endif // HAS_ARGBTOAR30ROW_AVX512BW #ifdef HAS_ABGRTOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP ABGRToAR30Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { @@ -15254,6 +16312,7 @@ static const uint8_t kPermMergeRGB64_2[64] = { 0x59, 0x79, 0x3a, 0x5a, 0x7a, 0x3b, 0x5b, 0x7b, 0x3c, 0x5c, 0x7c, 0x3d, 0x5d, 0x7d, 0x3e, 0x5e, 0x7e, 0x3f, 0x5f, 0x7f}; +LIBYUV_TARGET_AVX512VBMI void OMITFP MergeRGBRow_AVX512VBMI(const uint8_t* src_r, const uint8_t* src_g, const uint8_t* src_b, @@ -15314,7 +16373,7 @@ void OMITFP MergeRGBRow_AVX512VBMI(const uint8_t* src_r, "3: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp1] \n" "bzhi %q[width],%[temp1],%[temp2] \n" "kmovd %k[temp2],%%k1 \n" @@ -15360,6 +16419,7 @@ static const uint8_t kPermRAWToRGBA1[64] = { 0, 74, 73, 72, 0, 77, 76, 75, 0, 80, 79, 78, 0, 83, 82, 81, 0, 86, 85, 84, 0, 89, 88, 87, 0, 92, 91, 90, 0, 95, 94, 93}; +LIBYUV_TARGET_AVX512VBMI void OMITFP RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { @@ -15390,7 +16450,7 @@ void OMITFP RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp1] \n" "bzhi %q[width],%[temp1],%[temp2] \n" "kmovw %k[temp2],%%k1 \n" @@ -15428,6 +16488,7 @@ void OMITFP RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, static const uint8_t kSwapUV16[16] = {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14}; +LIBYUV_TARGET_AVX512BW void OMITFP SwapUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_vu, int width) { @@ -15449,7 +16510,7 @@ void OMITFP SwapUVRow_AVX512BW(const uint8_t* src_uv, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovd %k[temp],%%k1 \n" @@ -15471,9 +16532,11 @@ static const uint16_t kMirror16Idx[32] = { 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; +LIBYUV_TARGET_AVX512BW void OMITFP MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) { + intptr_t width64 = width; uintptr_t temp1; uintptr_t temp2; asm volatile( @@ -15494,7 +16557,7 @@ void OMITFP MirrorRow_16_AVX512BW(const uint16_t* src, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $32,%k[temp2] \n" "sub %k[width],%k[temp2] \n" "vpbroadcastw %k[temp2],%%zmm17 \n" @@ -15509,7 +16572,7 @@ void OMITFP MirrorRow_16_AVX512BW(const uint16_t* src, "99: \n" "vzeroupper \n" - : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width64), [temp1] "=&r"(temp1), [temp2] "=&r"(temp2) : [rev] "m"(kMirror16Idx) : "memory", "cc", "xmm0", "xmm16", "xmm17", "k1"); @@ -15517,6 +16580,7 @@ void OMITFP MirrorRow_16_AVX512BW(const uint16_t* src, #endif // HAS_MIRRORROW_16_AVX512BW #ifdef HAS_HALFMERGEUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP HalfMergeUVRow_AVX512BW(const uint8_t* src_u, int src_stride_u, const uint8_t* src_v, @@ -15563,7 +16627,7 @@ void OMITFP HalfMergeUVRow_AVX512BW(const uint8_t* src_u, "2: \n" "add $0x40,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp1] \n" "bzhi %q[width],%[temp1],%[temp2] \n" "kmovq %[temp2],%%k1 \n" @@ -15617,6 +16681,7 @@ void OMITFP HalfMergeUVRow_AVX512BW(const uint8_t* src_u, #endif // HAS_HALFMERGEUVROW_AVX512BW #ifdef HAS_INTERPOLATEROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP InterpolateRow_16_AVX512BW(uint16_t* dst_ptr, const uint16_t* src_ptr, ptrdiff_t src_stride, @@ -15667,7 +16732,7 @@ void OMITFP InterpolateRow_16_AVX512BW(uint16_t* dst_ptr, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp1] \n" "bzhi %q[width],%[temp1],%[temp2] \n" "kmovd %k[temp2],%%k1 \n" @@ -15700,7 +16765,7 @@ void OMITFP InterpolateRow_16_AVX512BW(uint16_t* dst_ptr, "jge 11b \n" "12: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp1] \n" "bzhi %q[width],%[temp1],%[temp2] \n" "kmovd %k[temp2],%%k1 \n" @@ -15723,7 +16788,7 @@ void OMITFP InterpolateRow_16_AVX512BW(uint16_t* dst_ptr, "jge 21b \n" "22: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp1] \n" "bzhi %q[width],%[temp1],%[temp2] \n" "kmovd %k[temp2],%%k1 \n" @@ -15805,6 +16870,7 @@ void OMITFP InterpolateRow_16_AVX512BW(uint16_t* dst_ptr, "vmovdqa64 %%zmm1,%%zmm3 \n" #ifdef HAS_NV12TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_argb, @@ -15827,7 +16893,7 @@ void OMITFP NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUVNV12_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -15842,7 +16908,6 @@ void OMITFP NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -15850,6 +16915,7 @@ void OMITFP NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, #endif // HAS_NV12TOARGBROW_AVX512BW #ifdef HAS_NV21TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, const uint8_t* vu_buf, uint8_t* dst_argb, @@ -15872,7 +16938,7 @@ void OMITFP NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUVNV21_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -15887,7 +16953,6 @@ void OMITFP NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -15895,6 +16960,7 @@ void OMITFP NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, #endif // HAS_NV21TOARGBROW_AVX512BW #ifdef HAS_NV12TORGB24ROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI void OMITFP NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_rgb24, @@ -15919,7 +16985,7 @@ void OMITFP NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUVNV12_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -15934,7 +17000,6 @@ void OMITFP NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [kMaskBG] "m"(kMaskBG), [kMaskDST0] "m"(kMaskDST0), @@ -15945,6 +17010,7 @@ void OMITFP NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, #endif // HAS_NV12TORGB24ROW_AVX512VBMI #ifdef HAS_NV21TORGB24ROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI void OMITFP NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, const uint8_t* vu_buf, uint8_t* dst_rgb24, @@ -15969,7 +17035,7 @@ void OMITFP NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUVNV21_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -15984,7 +17050,6 @@ void OMITFP NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [kMaskBG] "m"(kMaskBG), [kMaskDST0] "m"(kMaskDST0), @@ -15995,6 +17060,7 @@ void OMITFP NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, #endif // HAS_NV21TORGB24ROW_AVX512VBMI #ifdef HAS_I422ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -16022,7 +17088,7 @@ void OMITFP I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV422_MASK_AVX512BW "vpmovzxbw (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" @@ -16041,7 +17107,6 @@ void OMITFP I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -16057,6 +17122,7 @@ static const uint64_t kLinearizePackQ[8] = {0, 2, 4, 6, 1, 3, 5, 7}; static const uint8_t kBcastGrayByte[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, 8, 8, 8, 0x80, 12, 12, 12, 0x80}; +LIBYUV_TARGET_AVX512BW void OMITFP I400ToARGBRow_AVX512BW(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, @@ -16098,7 +17164,7 @@ void OMITFP I400ToARGBRow_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovd %k[temp],%%k1 \n" @@ -16133,6 +17199,7 @@ void OMITFP I400ToARGBRow_AVX512BW(const uint8_t* y_buf, #endif // HAS_I400TOARGBROW_AVX512BW #ifdef HAS_ARGBATTENUATEROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, int width) { @@ -16170,7 +17237,7 @@ void OMITFP ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -16201,6 +17268,7 @@ void OMITFP ARGBAttenuateRow_AVX512BW(const uint8_t* src_argb, #endif // HAS_ARGBATTENUATEROW_AVX512BW #ifdef HAS_ARGBMULTIPLYROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -16233,7 +17301,7 @@ void OMITFP ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -16260,6 +17328,7 @@ void OMITFP ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb, #endif // HAS_ARGBMULTIPLYROW_AVX512BW #ifdef HAS_ARGBGRAYROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP ARGBGrayRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, int width) { @@ -16299,7 +17368,7 @@ void OMITFP ARGBGrayRow_AVX512BW(const uint8_t* src_argb, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -16326,6 +17395,7 @@ void OMITFP ARGBGrayRow_AVX512BW(const uint8_t* src_argb, #endif // HAS_ARGBGRAYROW_AVX512BW #ifdef HAS_ARGBGRAYROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI void OMITFP ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, uint8_t* dst_argb, int width) { @@ -16360,7 +17430,7 @@ void OMITFP ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -16384,6 +17454,7 @@ void OMITFP ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, #endif // HAS_ARGBGRAYROW_AVX512VNNI #ifdef HAS_ARGBBLENDROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP ARGBBlendRow_AVX512BW(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -16429,7 +17500,7 @@ void OMITFP ARGBBlendRow_AVX512BW(const uint8_t* src_argb, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -16511,6 +17582,7 @@ void ARGBOrRow_AVX2(const uint8_t* src_argb, #endif // HAS_ARGBORROW_AVX2 #ifdef HAS_ARGBORROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP ARGBOrRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, uint32_t value, @@ -16533,7 +17605,7 @@ void OMITFP ARGBOrRow_AVX512BW(const uint8_t* src_argb, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -16734,6 +17806,7 @@ void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb, #endif // HAS_ARGBALPHABLENDROW_AVX2 #ifdef HAS_ARGBALPHABLENDROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -16747,7 +17820,6 @@ void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, "vpsubw %%zmm20,%%zmm18,%%zmm18 \n" // 0x7f80... "vpsubw %%zmm20,%%zmm18,%%zmm18 \n" // zmm18 = 0x7f81... "vpsllw $8,%%zmm20,%%zmm17 \n" // zmm17 = 0xff00... - "vpsrld $24,%%zmm20,%%zmm19 \n" // zmm19 = 255 in dwords "sub $0x10,%[width] \n" "jl 2f \n" @@ -16774,7 +17846,8 @@ void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, "vpord %%zmm5,%%zmm2,%%zmm2 \n" // sa_eff "vpord %%zmm5,%%zmm3,%%zmm7 \n" // den_eff - "vpmulld %%zmm19,%%zmm2,%%zmm2 \n" // sa_eff * 255 + "vpslld $8,%%zmm2,%%zmm6 \n" + "vpsubd %%zmm2,%%zmm6,%%zmm2 \n" // sa_eff * 255 "vcvtdq2ps %%zmm2,%%zmm2 \n" "vcvtdq2ps %%zmm7,%%zmm7 \n" @@ -16813,7 +17886,7 @@ void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, "2: \n" "add $0x10,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" "kmovw %k[temp],%%k1 \n" @@ -16835,7 +17908,8 @@ void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, "vpord %%zmm5,%%zmm2,%%zmm2 \n" "vpord %%zmm5,%%zmm3,%%zmm7 \n" - "vpmulld %%zmm19,%%zmm2,%%zmm2 \n" + "vpslld $8,%%zmm2,%%zmm6 \n" + "vpsubd %%zmm2,%%zmm6,%%zmm2 \n" "vcvtdq2ps %%zmm2,%%zmm2 \n" "vcvtdq2ps %%zmm7,%%zmm7 \n" @@ -16874,8 +17948,8 @@ void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb), [width] "+r"(width), [temp] "=&r"(temp) : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7", "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7", + "xmm16", "xmm17", "xmm18", "xmm20", "xmm21", "k1"); } #endif // HAS_ARGBALPHABLENDROW_AVX512BW @@ -17084,6 +18158,7 @@ void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, "vinserti64x4 $1,%%ymm1,%%zmm3,%%zmm3 \n" #ifdef HAS_I444TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I444ToARGBRow_AVX512BW(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -17108,7 +18183,7 @@ void OMITFP I444ToARGBRow_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV444_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -17124,7 +18199,6 @@ void OMITFP I444ToARGBRow_AVX512BW(const uint8_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -17132,6 +18206,7 @@ void OMITFP I444ToARGBRow_AVX512BW(const uint8_t* y_buf, #endif // HAS_I444TOARGBROW_AVX512BW #ifdef HAS_I444ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -17159,7 +18234,7 @@ void OMITFP I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV444_MASK_AVX512BW "vpmovzxbw (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" @@ -17178,7 +18253,6 @@ void OMITFP I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -17186,6 +18260,7 @@ void OMITFP I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, #endif // HAS_I444ALPHATOARGBROW_AVX512BW #ifdef HAS_I444TORGB24ROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI void OMITFP I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -17212,7 +18287,7 @@ void OMITFP I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV444_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -17228,7 +18303,6 @@ void OMITFP I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [kMaskBG] "m"(kMaskBG), [kMaskDST0] "m"(kMaskDST0), @@ -17239,6 +18313,7 @@ void OMITFP I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, #endif // HAS_I444TORGB24ROW_AVX512VBMI #ifdef HAS_I422TORGBAROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I422ToRGBARow_AVX512BW(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -17263,7 +18338,7 @@ void OMITFP I422ToRGBARow_AVX512BW(const uint8_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV422_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -17279,7 +18354,6 @@ void OMITFP I422ToRGBARow_AVX512BW(const uint8_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -17287,6 +18361,7 @@ void OMITFP I422ToRGBARow_AVX512BW(const uint8_t* y_buf, #endif // HAS_I422TORGBAROW_AVX512BW #ifdef HAS_YUY2TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP YUY2ToARGBRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, @@ -17313,7 +18388,7 @@ void OMITFP YUY2ToARGBRow_AVX512BW(const uint8_t* src_yuy2, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" @@ -17337,7 +18412,6 @@ void OMITFP YUY2ToARGBRow_AVX512BW(const uint8_t* src_yuy2, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [kShufY] "m"(kShuffleYUY2Y), [kShufUV] "m"(kShuffleYUY2UV) @@ -17347,6 +18421,7 @@ void OMITFP YUY2ToARGBRow_AVX512BW(const uint8_t* src_yuy2, #endif // HAS_YUY2TOARGBROW_AVX512BW #ifdef HAS_UYVYTOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP UYVYToARGBRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, @@ -17373,7 +18448,7 @@ void OMITFP UYVYToARGBRow_AVX512BW(const uint8_t* src_uyvy, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" "mov $-1,%[temp] \n" "bzhi %q[width],%[temp],%[temp] \n" @@ -17397,7 +18472,6 @@ void OMITFP UYVYToARGBRow_AVX512BW(const uint8_t* src_uyvy, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [kShufY] "m"(kShuffleUYVYY), [kShufUV] "m"(kShuffleUYVYUV) @@ -17407,6 +18481,7 @@ void OMITFP UYVYToARGBRow_AVX512BW(const uint8_t* src_uyvy, #endif // HAS_UYVYTOARGBROW_AVX512BW #ifdef HAS_I210TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I210ToARGBRow_AVX512BW(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -17431,7 +18506,7 @@ void OMITFP I210ToARGBRow_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV210_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -17447,7 +18522,6 @@ void OMITFP I210ToARGBRow_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -17455,6 +18529,7 @@ void OMITFP I210ToARGBRow_AVX512BW(const uint16_t* y_buf, #endif // HAS_I210TOARGBROW_AVX512BW #ifdef HAS_I210TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I210ToAR30Row_AVX512BW(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -17486,7 +18561,7 @@ void OMITFP I210ToAR30Row_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV210_MASK_AVX512BW YUVTORGB16_AVX512BW(yuvconstants) @@ -17502,7 +18577,6 @@ void OMITFP I210ToAR30Row_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [permar30_0] "r"(kPermAR30_0), [permar30_1] "r"(kPermAR30_1) @@ -17512,6 +18586,7 @@ void OMITFP I210ToAR30Row_AVX512BW(const uint16_t* y_buf, #endif // HAS_I210TOAR30ROW_AVX512BW #ifdef HAS_I210ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -17540,7 +18615,7 @@ void OMITFP I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV210_MASK_AVX512BW "vmovdqu16 (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" @@ -17560,7 +18635,6 @@ void OMITFP I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -17568,6 +18642,7 @@ void OMITFP I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, #endif // HAS_I210ALPHATOARGBROW_AVX512BW #ifdef HAS_I212TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I212ToARGBRow_AVX512BW(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -17592,7 +18667,7 @@ void OMITFP I212ToARGBRow_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV212_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -17608,7 +18683,6 @@ void OMITFP I212ToARGBRow_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -17616,6 +18690,7 @@ void OMITFP I212ToARGBRow_AVX512BW(const uint16_t* y_buf, #endif // HAS_I212TOARGBROW_AVX512BW #ifdef HAS_I212TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I212ToAR30Row_AVX512BW(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -17647,7 +18722,7 @@ void OMITFP I212ToAR30Row_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV212_MASK_AVX512BW YUVTORGB16_AVX512BW(yuvconstants) @@ -17663,7 +18738,6 @@ void OMITFP I212ToAR30Row_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [permar30_0] "r"(kPermAR30_0), [permar30_1] "r"(kPermAR30_1) @@ -17673,6 +18747,7 @@ void OMITFP I212ToAR30Row_AVX512BW(const uint16_t* y_buf, #endif // HAS_I212TOAR30ROW_AVX512BW #ifdef HAS_I410TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I410ToARGBRow_AVX512BW(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -17697,7 +18772,7 @@ void OMITFP I410ToARGBRow_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV410_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -17713,7 +18788,6 @@ void OMITFP I410ToARGBRow_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -17721,6 +18795,7 @@ void OMITFP I410ToARGBRow_AVX512BW(const uint16_t* y_buf, #endif // HAS_I410TOARGBROW_AVX512BW #ifdef HAS_I410TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I410ToAR30Row_AVX512BW(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -17752,7 +18827,7 @@ void OMITFP I410ToAR30Row_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV410_MASK_AVX512BW YUVTORGB16_AVX512BW(yuvconstants) @@ -17768,7 +18843,6 @@ void OMITFP I410ToAR30Row_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [permar30_0] "r"(kPermAR30_0), [permar30_1] "r"(kPermAR30_1) @@ -17778,6 +18852,7 @@ void OMITFP I410ToAR30Row_AVX512BW(const uint16_t* y_buf, #endif // HAS_I410TOAR30ROW_AVX512BW #ifdef HAS_I410ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -17806,7 +18881,7 @@ void OMITFP I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READYUV410_MASK_AVX512BW "vmovdqu16 (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" @@ -17826,7 +18901,6 @@ void OMITFP I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -17834,6 +18908,7 @@ void OMITFP I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, #endif // HAS_I410ALPHATOARGBROW_AVX512BW #ifdef HAS_P210TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP P210ToARGBRow_AVX512BW(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_argb, @@ -17857,7 +18932,7 @@ void OMITFP P210ToARGBRow_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READP210_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -17872,7 +18947,6 @@ void OMITFP P210ToARGBRow_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [kShufUV] "m"(kShuffleYUY2UV) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", @@ -17881,6 +18955,7 @@ void OMITFP P210ToARGBRow_AVX512BW(const uint16_t* y_buf, #endif // HAS_P210TOARGBROW_AVX512BW #ifdef HAS_P210TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP P210ToAR30Row_AVX512BW(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_ar30, @@ -17911,7 +18986,7 @@ void OMITFP P210ToAR30Row_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READP210_MASK_AVX512BW YUVTORGB16_AVX512BW(yuvconstants) @@ -17926,7 +19001,6 @@ void OMITFP P210ToAR30Row_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [permar30_0] "r"(kPermAR30_0), [permar30_1] "r"(kPermAR30_1), @@ -17938,6 +19012,7 @@ void OMITFP P210ToAR30Row_AVX512BW(const uint16_t* y_buf, #endif // HAS_P210TOAR30ROW_AVX512BW #ifdef HAS_P410TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP P410ToARGBRow_AVX512BW(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_argb, @@ -17960,7 +19035,7 @@ void OMITFP P410ToARGBRow_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READP410_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) @@ -17975,7 +19050,6 @@ void OMITFP P410ToARGBRow_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512) : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1", "k2"); @@ -17983,6 +19057,7 @@ void OMITFP P410ToARGBRow_AVX512BW(const uint16_t* y_buf, #endif // HAS_P410TOARGBROW_AVX512BW #ifdef HAS_P410TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW void OMITFP P410ToAR30Row_AVX512BW(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_ar30, @@ -18012,7 +19087,7 @@ void OMITFP P410ToAR30Row_AVX512BW(const uint16_t* y_buf, "2: \n" "add $0x20,%[width] \n" - "je 99f \n" + "jle 99f \n" READP410_MASK_AVX512BW YUVTORGB16_AVX512BW(yuvconstants) @@ -18027,7 +19102,6 @@ void OMITFP P410ToAR30Row_AVX512BW(const uint16_t* y_buf, [temp] "=&r"(temp) : [yuvconstants] "r"(yuvconstants), [quadsplitperm] "r"(kSplitQuadWords), - [dquadsplitperm] "r"(kSplitDoubleQuadWords), [unperm] "r"(kUnpermuteAVX512), [permar30_0] "r"(kPermAR30_0), [permar30_1] "r"(kPermAR30_1) diff --git a/source/row_hvx.cc b/source/row_hvx.cc index 1a8be0689..9ed5bfac1 100644 --- a/source/row_hvx.cc +++ b/source/row_hvx.cc @@ -988,6 +988,12 @@ void DivideRow_16_HVX(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } uint32_t scale32 = ((uint32_t)(uint16_t)scale << 16) | (uint16_t)scale; while (width > 0) { int w = width >= 64 ? 64 : width; diff --git a/source/row_neon.cc b/source/row_neon.cc index f1fc04f96..ae4782d6b 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -8,6 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include + #include "libyuv/convert_from_argb.h" // For ArgbConstants #include "libyuv/row.h" @@ -3819,6 +3821,12 @@ void DivideRow_16_NEON(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } asm volatile( "vdup.16 d8, %3 \n" "1: \n" diff --git a/source/row_neon64.cc b/source/row_neon64.cc index 9b9c039f6..e39360537 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -8,6 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include + #include "libyuv/convert_from_argb.h" #include "libyuv/row.h" @@ -5464,6 +5466,12 @@ void DivideRow_16_NEON(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } asm volatile( "dup v4.8h, %w3 \n" "1: \n" diff --git a/source/row_sve.cc b/source/row_sve.cc index 28c3b1a16..cb5f73993 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -8,6 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include + #include "libyuv/row_sve.h" #include "libyuv/compare_row.h" #include "libyuv/row.h" @@ -772,6 +774,12 @@ void DivideRow_16_SVE2(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } uint64_t vl; asm volatile( "cnth %x[vl] \n" diff --git a/source/row_win.cc b/source/row_win.cc index 69873fafb..ba6f7b56e 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -35,6 +35,7 @@ extern "C" { #define LIBYUV_TARGET_SSSE3 __attribute__((target("ssse3"))) #define LIBYUV_TARGET_AVX __attribute__((target("avx"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) +#define LIBYUV_TARGET_AVX2_FMA __attribute__((target("avx2,fma"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) #define LIBYUV_TARGET_AVX512VBMI \ @@ -47,6 +48,7 @@ extern "C" { #define LIBYUV_TARGET_SSSE3 #define LIBYUV_TARGET_AVX #define LIBYUV_TARGET_AVX2 +#define LIBYUV_TARGET_AVX2_FMA #define LIBYUV_TARGET_AVX512BW #define LIBYUV_TARGET_AVX512VBMI #define LIBYUV_TARGET_AVX512VNNI @@ -268,7 +270,8 @@ void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb, _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToU)); __m256i ymm_v = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToV)); - __m256i ymm5 = _mm256_set1_epi16((short)0x8000); + __m256i ymm5 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kAddUV)); __m256i perm_mask = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); while (width > 0) { @@ -742,7 +745,7 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, __m512i zmm_d1 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_1_AVX512BW); __m512i zmm_d2 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_2_AVX512BW); - while (width > 0) { + while (width >= 64) { __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw); __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128)); @@ -769,6 +772,44 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, dst_rgb24 += 192; width -= 64; } + if (width > 0) { + int nbytes = width * 3; + __mmask64 k0 = + (nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL); + __mmask64 k1 = (nbytes <= 64) ? 0 + : (nbytes >= 128) + ? ~(__mmask64)0 + : (__mmask64)((1ULL << (nbytes - 64)) - 1ULL); + __mmask64 k2 = + (nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL); + __m512i zmm_raw0 = _mm512_maskz_loadu_epi8(k0, src_raw); + __m512i zmm_raw1 = + k1 ? _mm512_maskz_loadu_epi8(k1, src_raw + 64) : _mm512_setzero_si512(); + __m512i zmm_raw2 = k2 ? _mm512_maskz_loadu_epi8(k2, src_raw + 128) + : _mm512_setzero_si512(); + + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); + __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); + __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + __m512i dst0 = _mm512_permutex2var_epi32(zmm0, zmm_d0, zmm1); + __m512i dst1 = _mm512_permutex2var_epi32(zmm1, zmm_d1, zmm2); + __m512i dst2 = _mm512_permutex2var_epi32(zmm2, zmm_d2, zmm3); + + _mm512_mask_storeu_epi8(dst_rgb24, k0, dst0); + if (k1) { + _mm512_mask_storeu_epi8(dst_rgb24 + 64, k1, dst1); + } + if (k2) { + _mm512_mask_storeu_epi8(dst_rgb24 + 128, k2, dst2); + } + } _mm256_zeroupper(); } #endif // HAS_RAWTORGB24ROW_AVX512BW @@ -783,7 +824,7 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, __m512i zmm_p2 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_2); __m512i zmm_p3 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_3); - while (width > 0) { + while (width >= 64) { __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw); __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128)); @@ -801,6 +842,35 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, dst_rgb24 += 192; width -= 64; } + if (width > 0) { + int nbytes = width * 3; + __mmask64 k0 = + (nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL); + __mmask64 k1 = (nbytes <= 64) ? 0 + : (nbytes >= 128) + ? ~(__mmask64)0 + : (__mmask64)((1ULL << (nbytes - 64)) - 1ULL); + __mmask64 k2 = + (nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL); + __m512i zmm_raw0 = _mm512_maskz_loadu_epi8(k0, src_raw); + __m512i zmm_raw1 = + k1 ? _mm512_maskz_loadu_epi8(k1, src_raw + 64) : _mm512_setzero_si512(); + __m512i zmm_raw2 = k2 ? _mm512_maskz_loadu_epi8(k2, src_raw + 128) + : _mm512_setzero_si512(); + + __m512i tmp1 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p1, zmm_raw0); + __m512i dst0 = _mm512_permutex2var_epi8(zmm_raw0, zmm_p0, zmm_raw1); + __m512i dst2 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p3, zmm_raw2); + __m512i dst1 = _mm512_permutex2var_epi8(tmp1, zmm_p2, zmm_raw2); + + _mm512_mask_storeu_epi8(dst_rgb24, k0, dst0); + if (k1) { + _mm512_mask_storeu_epi8(dst_rgb24 + 64, k1, dst1); + } + if (k2) { + _mm512_mask_storeu_epi8(dst_rgb24 + 128, k2, dst2); + } + } _mm256_zeroupper(); } #endif // HAS_RAWTORGB24ROW_AVX512VBMI @@ -849,7 +919,7 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias); zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias); - while (width > 0) { + while (width >= 64) { __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_rgb); __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_rgb + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_rgb + 128)); @@ -897,6 +967,47 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, dst_y += 64; width -= 64; } + if (width > 0) { + __mmask64 ky = (__mmask64)((1ULL << width) - 1ULL); + int nbytes = width * 3; + __mmask64 k0 = + (nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL); + __mmask64 k1 = (nbytes <= 64) ? 0 + : (nbytes >= 128) + ? ~(__mmask64)0 + : (__mmask64)((1ULL << (nbytes - 64)) - 1ULL); + __mmask64 k2 = + (nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL); + __m512i zmm_raw0 = _mm512_maskz_loadu_epi8(k0, src_rgb); + __m512i zmm_raw1 = + k1 ? _mm512_maskz_loadu_epi8(k1, src_rgb + 64) : _mm512_setzero_si512(); + __m512i zmm_raw2 = k2 ? _mm512_maskz_loadu_epi8(k2, src_rgb + 128) + : _mm512_setzero_si512(); + + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); + __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); + __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); + + zmm0 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm0, zmm_shuf), zmm_80); + zmm1 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm1, zmm_shuf), zmm_80); + zmm2 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm2, zmm_shuf), zmm_80); + zmm3 = _mm512_xor_si512(_mm512_shuffle_epi8(zmm3, zmm_shuf), zmm_80); + + zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one); + zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one); + zmm0 = _mm512_packs_epi32(zmm0, zmm1); + + zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one); + zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one); + zmm2 = _mm512_packs_epi32(zmm2, zmm3); + + zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8); + zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8); + + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); + _mm512_mask_storeu_epi8(dst_y, ky, zmm0); + } _mm256_zeroupper(); } #endif // HAS_RGBTOYMATRIXROW_AVX512BW @@ -919,7 +1030,7 @@ void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb, _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_coeff, zmm_80); zmm_add = _mm512_sub_epi32(zmm_add, zmm_bias); - while (width > 0) { + while (width >= 64) { __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb); __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); @@ -950,6 +1061,38 @@ void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb, dst_y += 64; width -= 64; } + if (width > 0) { + uint64_t mask = (1ULL << width) - 1ULL; + __mmask16 k0 = (__mmask16)mask; + __mmask16 k1 = (__mmask16)(mask >> 16); + __mmask16 k2 = (__mmask16)(mask >> 32); + __mmask16 k3 = (__mmask16)(mask >> 48); + __m512i zmm0 = + _mm512_xor_si512(_mm512_maskz_loadu_epi32(k0, src_argb), zmm_80); + __m512i zmm1 = + _mm512_xor_si512(k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64) + : _mm512_setzero_si512(), + zmm_80); + __m512i zmm2 = + _mm512_xor_si512(k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128) + : _mm512_setzero_si512(), + zmm_80); + __m512i zmm3 = + _mm512_xor_si512(k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192) + : _mm512_setzero_si512(), + zmm_80); + + zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0); + zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1); + zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2); + zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3); + + zmm0 = _mm512_srli_epi16(_mm512_packus_epi32(zmm0, zmm1), 8); + zmm2 = _mm512_srli_epi16(_mm512_packus_epi32(zmm2, zmm3), 8); + + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); + _mm512_mask_storeu_epi8(dst_y, (__mmask64)mask, zmm0); + } _mm256_zeroupper(); } #endif // HAS_ARGBTOYMATRIXROW_AVX512VNNI @@ -997,7 +1140,7 @@ void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_coeff, zmm_80); zmm_add = _mm512_sub_epi32(zmm_add, zmm_bias); - while (width > 0) { + while (width >= 64) { __m512i zmm0 = _mm512_xor_si512(_mm512_loadu_si512((const __m512i*)src_argb), zmm_80); __m512i zmm1 = _mm512_xor_si512( @@ -1022,6 +1165,42 @@ void ARGBToYMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, dst_y += 64; width -= 64; } + if (width > 0) { + uint64_t mask = (1ULL << width) - 1ULL; + __mmask16 k0 = (__mmask16)mask; + __mmask16 k1 = (__mmask16)(mask >> 16); + __mmask16 k2 = (__mmask16)(mask >> 32); + __mmask16 k3 = (__mmask16)(mask >> 48); + __m512i zmm0 = + _mm512_xor_si512(_mm512_maskz_loadu_epi32(k0, src_argb), zmm_80); + __m512i zmm1 = + _mm512_xor_si512(k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64) + : _mm512_setzero_si512(), + zmm_80); + __m512i zmm2 = + _mm512_xor_si512(k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128) + : _mm512_setzero_si512(), + zmm_80); + __m512i zmm3 = + _mm512_xor_si512(k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192) + : _mm512_setzero_si512(), + zmm_80); + + zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm0); + zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm1); + zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm2); + zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_coeff, zmm3); + + zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); + zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); + + _mm256_mask_storeu_epi8(dst_y, (__mmask32)mask, + _mm512_castsi512_si256(zmm0)); + if (mask >> 32) { + _mm256_mask_storeu_epi8(dst_y + 32, (__mmask32)(mask >> 32), + _mm512_castsi512_si256(zmm2)); + } + } _mm256_zeroupper(); } #endif // HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI @@ -1037,11 +1216,12 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); __m512i zmm_v = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); - __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + __m512i zmm_8000 = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV)); __m512i zmm_perm_y = _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); - while (width > 0) { + while (width >= 64) { __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb); __m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); __m512i zmm_s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); @@ -1082,6 +1262,44 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, dst_v += 64; width -= 64; } + if (width > 0) { + uint64_t mask = (1ULL << width) - 1ULL; + __mmask16 k0 = (__mmask16)mask; + __mmask16 k1 = (__mmask16)(mask >> 16); + __mmask16 k2 = (__mmask16)(mask >> 32); + __mmask16 k3 = (__mmask16)(mask >> 48); + __m512i zmm_s0 = _mm512_maskz_loadu_epi32(k0, src_argb); + __m512i zmm_s1 = k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64) + : _mm512_setzero_si512(); + __m512i zmm_s2 = k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128) + : _mm512_setzero_si512(); + __m512i zmm_s3 = k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192) + : _mm512_setzero_si512(); + + __m512i zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_u); + __m512i zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_u); + __m512i zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_u); + __m512i zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_u); + + zmm0 = _mm512_srli_epi16( + _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm0, zmm1)), 8); + zmm2 = _mm512_srli_epi16( + _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm2, zmm3)), 8); + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); + _mm512_mask_storeu_epi8(dst_u, (__mmask64)mask, zmm0); + + zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_v); + zmm1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s1, zmm_v); + zmm2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s2, zmm_v); + zmm3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s3, zmm_v); + + zmm0 = _mm512_srli_epi16( + _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm0, zmm1)), 8); + zmm2 = _mm512_srli_epi16( + _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(zmm2, zmm3)), 8); + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); + _mm512_mask_storeu_epi8(dst_v, (__mmask64)mask, zmm0); + } _mm256_zeroupper(); } #endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNI @@ -1109,7 +1327,7 @@ void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)), zmm_neg1); // ~kAddUV - while (width > 0) { + while (width >= 64) { __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb); __m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); __m512i zmm_s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); @@ -1142,6 +1360,41 @@ void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, dst_v += 64; width -= 64; } + if (width > 0) { + uint64_t mask = (1ULL << width) - 1ULL; + __mmask16 k0 = (__mmask16)mask; + __mmask16 k1 = (__mmask16)(mask >> 16); + __mmask16 k2 = (__mmask16)(mask >> 32); + __mmask16 k3 = (__mmask16)(mask >> 48); + __m512i zmm_s0 = _mm512_maskz_loadu_epi32(k0, src_argb); + __m512i zmm_s1 = k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64) + : _mm512_setzero_si512(); + __m512i zmm_s2 = k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128) + : _mm512_setzero_si512(); + __m512i zmm_s3 = k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192) + : _mm512_setzero_si512(); + + __m512i zmm0 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_u); + __m512i zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_u); + __m512i zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_u); + __m512i zmm3 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_u); + __m512i zmm4 = _mm512_dpbusd_epi32(zmm_add, zmm_s0, zmm_v); + __m512i zmm5 = _mm512_dpbusd_epi32(zmm_add, zmm_s1, zmm_v); + __m512i zmm6 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_v); + __m512i zmm7 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_v); + + zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); + zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); + zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); + zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); + zmm0 = _mm512_xor_si512( + _mm512_inserti64x4(zmm0, _mm512_castsi512_si256(zmm2), 1), zmm_neg1); + zmm4 = _mm512_xor_si512( + _mm512_inserti64x4(zmm4, _mm512_castsi512_si256(zmm6), 1), zmm_neg1); + + _mm512_mask_storeu_epi8(dst_u, (__mmask64)mask, zmm0); + _mm512_mask_storeu_epi8(dst_v, (__mmask64)mask, zmm4); + } _mm256_zeroupper(); } #endif // HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI @@ -1163,7 +1416,8 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, __m512i zmm_v = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); __m512i zmm_0101 = _mm512_set1_epi16(0x0101); - __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + __m512i zmm_8000 = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV)); __m512i zmm_zero = _mm512_setzero_si512(); __m512i zmm_shuf = _mm512_broadcast_i32x4( _mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512)); @@ -1414,7 +1668,8 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, __m256i ymm_shuf = _mm256_setr_epi8(0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15, 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15); - __m256i ymm_8000 = _mm256_set1_epi16((short)0x8000); + __m256i ymm_8000 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kAddUV)); __m256i ymm_zero = _mm256_setzero_si256(); while (width > 0) { @@ -1698,6 +1953,28 @@ void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) { } #endif // HAS_SWAPUVROW_AVX2 +#ifdef HAS_MIRRORSPLITUVROW_SSSE3 +LIBYUV_TARGET_SSSE3 +void MirrorSplitUVRow_SSSE3(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m128i xmm_shuf = + _mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1); + src_uv += (ptrdiff_t)width * 2; + while (width > 0) { + src_uv -= 16; + __m128i xmm0 = + _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)src_uv), xmm_shuf); + _mm_storel_epi64((__m128i*)dst_u, xmm0); + _mm_storel_epi64((__m128i*)dst_v, _mm_unpackhi_epi64(xmm0, xmm0)); + dst_u += 8; + dst_v += 8; + width -= 8; + } +} +#endif // HAS_MIRRORSPLITUVROW_SSSE3 + #ifdef HAS_MIRRORSPLITUVROW_AVX2 LIBYUV_TARGET_AVX2 void MirrorSplitUVRow_AVX2(const uint8_t* src_uv, @@ -1706,7 +1983,7 @@ void MirrorSplitUVRow_AVX2(const uint8_t* src_uv, int width) { __m256i ymm_shuf = _mm256_broadcastsi128_si256( _mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1)); - src_uv += width * 2; + src_uv += (ptrdiff_t)width * 2; while (width > 0) { src_uv -= 32; __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv); @@ -1733,7 +2010,7 @@ void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24, __m128i shuf1 = _mm_setr_epi8(13, 14, 15, 10, 11, 12, 7, 8, 9, 4, 5, 6, 1, 2, 3, -1); - src_rgb24 += width * 3 - 96; + src_rgb24 += (ptrdiff_t)width * 3 - 96; while (width > 0) { __m128i v0_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 0)); __m128i v0_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 15)); @@ -5204,6 +5481,8 @@ void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, const __m512i c_shuf = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)kShufGrayDword)); const __m512i c_alpha = _mm512_set1_epi32((int)0xff000000u); + // Avoid _mm512_ternarylogic_epi32 (unmodeled by LLVM MSan on RGBX inputs; + // see GEMINI.md). while (width >= 16) { __m512i s = _mm512_loadu_si512((const __m512i*)src_argb); src_argb += 64; @@ -5583,6 +5862,7 @@ void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width) { _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. ARGBSepiaRow_C(dst_argb, width); } } @@ -5622,6 +5902,7 @@ void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb, _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. ARGBColorMatrixRow_C(src_argb, dst_argb, matrix_argb, width); } } @@ -5658,6 +5939,7 @@ void ARGBQuantizeRow_AVX2(uint8_t* dst_argb, _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. ARGBQuantizeRow_C(dst_argb, scale, interval_size, interval_offset, width); } } @@ -5686,6 +5968,7 @@ void ARGBShadeRow_AVX2(const uint8_t* src_argb, _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. ARGBShadeRow_C(src_argb, dst_argb, width, value); } } @@ -5874,6 +6157,7 @@ void ARGBAffineRow_AVX2(const uint8_t* src_argb, _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. ARGBAffineRow_C(src_argb, src_argb_stride, dst_argb, uv_tail, width); } } @@ -5918,6 +6202,7 @@ void ARGBColorTableRow_AVX2(uint8_t* dst_argb, _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. ARGBColorTableRow_C(dst_argb, table_argb, width); } } @@ -5959,6 +6244,7 @@ void RGBColorTableRow_AVX2(uint8_t* dst_argb, _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. RGBColorTableRow_C(dst_argb, table_argb, width); } } @@ -5972,7 +6258,10 @@ void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, const uint8_t* luma, uint32_t lumacoeff) { if (width >= 8) { - const __m256i v_coeff = _mm256_set1_epi32((int)(lumacoeff & 0x00ffffffu)); + const __m256i c_br = _mm256_set1_epi32((int)(lumacoeff & 0x00ff00ffu)); + const __m256i c_ga = + _mm256_set1_epi32((int)((lumacoeff >> 8) & 0x000000ffu)); + const __m256i m_00ff = _mm256_set1_epi16(0x00ff); const __m256i m_ff = _mm256_set1_epi32(0x000000ff); const __m256i m_7f00 = _mm256_set1_epi32(0x00007f00); const __m256i m_18 = _mm256_set1_epi32(0x00000018); @@ -5980,11 +6269,10 @@ void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, while (width >= 8) { __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb); src_argb += 32; - __m256i y = _mm256_maddubs_epi16(v0, v_coeff); - y = _mm256_hadd_epi16(y, y); - y = _mm256_permute4x64_epi64(y, 0xd8); - y = _mm256_and_si256(_mm256_cvtepu16_epi32(_mm256_castsi256_si128(y)), - m_7f00); + __m256i y = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_and_si256(v0, m_00ff), c_br), + _mm256_madd_epi16(_mm256_srli_epi16(v0, 8), c_ga)); + y = _mm256_and_si256(y, m_7f00); __m256i b_idx = _mm256_srli_epi32(_mm256_or_si256(_mm256_and_si256(v0, m_ff), y), 2); @@ -6018,6 +6306,7 @@ void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. ARGBLumaColorTableRow_C(src_argb, dst_argb, width, luma, lumacoeff); } } @@ -6076,6 +6365,7 @@ void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width) { _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. GaussRow_C(src, dst, width); } } @@ -6103,6 +6393,7 @@ void GaussRow_F32_AVX2(const float* src, float* dst, int width) { _mm256_zeroupper(); } if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. GaussRow_F32_C(src, dst, width); } } @@ -6409,6 +6700,12 @@ void DivideRow_16_AVX512BW(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } const __m512i zmm_scale = _mm512_set1_epi16((short)scale); while (width >= 32) { __m512i v = _mm512_loadu_si512((const __m512i*)src_y); @@ -6651,15 +6948,12 @@ void HalfFloatRow_AVX512BW(const uint16_t* src, float scale, int width) { const __m512 vscale = _mm512_set1_ps(scale); - const __m512 vmult = _mm512_set1_ps(1.9259299444e-34f); while (width >= 16) { __m512i s32 = _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src)); src += 16; - __m512 f = - _mm512_mul_ps(_mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale), vmult); - __m256i h16 = - _mm512_cvtepi32_epi16(_mm512_srli_epi32(_mm512_castps_si512(f), 13)); + __m512 f = _mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale); + __m256i h16 = _mm512_cvtps_ph(f, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); _mm256_storeu_si256((__m256i*)dst, h16); dst += 16; width -= 16; @@ -6667,10 +6961,8 @@ void HalfFloatRow_AVX512BW(const uint16_t* src, if (width > 0) { __mmask16 k = (__mmask16)((1u << width) - 1u); __m512i s32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src)); - __m512 f = - _mm512_mul_ps(_mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale), vmult); - __m256i h16 = - _mm512_cvtepi32_epi16(_mm512_srli_epi32(_mm512_castps_si512(f), 13)); + __m512 f = _mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale); + __m256i h16 = _mm512_cvtps_ph(f, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); _mm256_mask_storeu_epi16(dst, k, h16); } _mm256_zeroupper(); @@ -7872,7 +8164,7 @@ void Convert16To8Row_AVX512BW(const uint16_t* src_y, int width) { const __m512i zmm_scale = _mm512_set1_epi16((short)scale); const __m512i zmm_255 = _mm512_set1_epi16(255); - do { + while (width >= 64) { __m512i y0 = _mm512_loadu_si512((const __m512i*)src_y); __m512i y1 = _mm512_loadu_si512((const __m512i*)(src_y + 32)); y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255); @@ -7882,7 +8174,21 @@ void Convert16To8Row_AVX512BW(const uint16_t* src_y, src_y += 64; dst_y += 64; width -= 64; - } while (width > 0); + } + if (width >= 32) { + __m512i y0 = _mm512_loadu_si512((const __m512i*)src_y); + y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255); + _mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(y0)); + src_y += 32; + dst_y += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i y0 = _mm512_maskz_loadu_epi16(k, src_y); + y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255); + _mm256_mask_storeu_epi8(dst_y, k, _mm512_cvtepi16_epi8(y0)); + } _mm256_zeroupper(); } #endif // HAS_CONVERT16TO8ROW_AVX512BW @@ -7924,7 +8230,7 @@ void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, int width) { const __m512i zmm_scale = _mm512_set1_epi16((short)scale); const __m512i zmm_255 = _mm512_set1_epi16(255); - do { + while (width >= 64) { __m512i r0 = _mm512_loadu_si512((const __m512i*)src_uv); __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); __m512i s0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride)); @@ -7939,7 +8245,22 @@ void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, src_uv += 64; dst_uv += 64; width -= 64; - } while (width > 0); + } + if (width > 0) { + uint64_t mask = (1ULL << width) - 1ULL; + __mmask32 k1 = (__mmask32)mask; + __mmask32 k2 = (__mmask32)(mask >> 32); + __m512i r0 = _mm512_maskz_loadu_epi16(k1, src_uv); + __m512i r1 = _mm512_maskz_loadu_epi16(k2, src_uv + 32); + __m512i s0 = _mm512_maskz_loadu_epi16(k1, src_uv + src_uv_stride); + __m512i s1 = _mm512_maskz_loadu_epi16(k2, src_uv + src_uv_stride + 32); + r0 = _mm512_avg_epu16(r0, s0); + r1 = _mm512_avg_epu16(r1, s1); + r0 = _mm512_min_epu16(_mm512_mulhi_epu16(r0, zmm_scale), zmm_255); + r1 = _mm512_min_epu16(_mm512_mulhi_epu16(r1, zmm_scale), zmm_255); + _mm256_mask_storeu_epi8(dst_uv, k1, _mm512_cvtepi16_epi8(r0)); + _mm256_mask_storeu_epi8(dst_uv + 32, k2, _mm512_cvtepi16_epi8(r1)); + } _mm256_zeroupper(); } #endif // HAS_HALFROW_16TO8_AVX512BW @@ -7955,7 +8276,7 @@ void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, const __m512i zmm_2 = _mm512_set1_epi32(2); const __m512i zmm_scale = _mm512_set1_epi16((short)scale); const __m512i zmm_255 = _mm512_set1_epi16(255); - do { + while (width >= 32) { __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv); __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); __m512i t0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride)); @@ -7981,7 +8302,34 @@ void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, src_uv += 64; dst_uv += 32; width -= 32; - } while (width > 0); + } + if (width > 0) { + uint32_t mask = (1u << width) - 1u; + __mmask16 k1 = (__mmask16)mask; + __mmask16 k2 = (__mmask16)(mask >> 16); + __mmask32 k3 = (__mmask32)mask; + __m512i s0 = _mm512_maskz_loadu_epi32(k1, src_uv); + __m512i s1 = _mm512_maskz_loadu_epi32(k2, src_uv + 32); + __m512i t0 = _mm512_maskz_loadu_epi32(k1, src_uv + src_uv_stride); + __m512i t1 = _mm512_maskz_loadu_epi32(k2, src_uv + src_uv_stride + 32); + __m512i sum0 = _mm512_add_epi32(_mm512_and_si512(s0, zmm_mask16), + _mm512_srli_epi32(s0, 16)); + sum0 = _mm512_add_epi32(sum0, _mm512_and_si512(t0, zmm_mask16)); + sum0 = _mm512_add_epi32(sum0, _mm512_srli_epi32(t0, 16)); + sum0 = _mm512_srli_epi32(_mm512_add_epi32(sum0, zmm_2), 2); + + __m512i sum1 = _mm512_add_epi32(_mm512_and_si512(s1, zmm_mask16), + _mm512_srli_epi32(s1, 16)); + sum1 = _mm512_add_epi32(sum1, _mm512_and_si512(t1, zmm_mask16)); + sum1 = _mm512_add_epi32(sum1, _mm512_srli_epi32(t1, 16)); + sum1 = _mm512_srli_epi32(_mm512_add_epi32(sum1, zmm_2), 2); + + __m512i avg16 = + _mm512_inserti64x4(_mm512_zextsi256_si512(_mm512_cvtepi32_epi16(sum0)), + _mm512_cvtepi32_epi16(sum1), 1); + avg16 = _mm512_min_epu16(_mm512_mulhi_epu16(avg16, zmm_scale), zmm_255); + _mm256_mask_storeu_epi8(dst_uv, k3, _mm512_cvtepi16_epi8(avg16)); + } _mm256_zeroupper(); } #endif // HAS_HALFWIDTHROW_16TO8_AVX512BW @@ -7992,7 +8340,7 @@ void SplitUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width) { - do { + while (width >= 64) { __m512i uv0 = _mm512_loadu_si512((const __m512i*)src_uv); __m512i uv1 = _mm512_loadu_si512((const __m512i*)(src_uv + 64)); _mm256_storeu_si256((__m256i*)dst_u, _mm512_cvtepi16_epi8(uv0)); @@ -8005,7 +8353,22 @@ void SplitUVRow_AVX512BW(const uint8_t* src_uv, dst_u += 64; dst_v += 64; width -= 64; - } while (width > 0); + } + if (width > 0) { + uint64_t mask = (1ULL << width) - 1ULL; + __mmask32 k0 = (__mmask32)mask; + __mmask32 k1 = (__mmask32)(mask >> 32); + __m512i uv0 = _mm512_maskz_loadu_epi16(k0, src_uv); + _mm256_mask_storeu_epi8(dst_u, k0, _mm512_cvtepi16_epi8(uv0)); + _mm256_mask_storeu_epi8(dst_v, k0, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8))); + if (k1) { + __m512i uv1 = _mm512_maskz_loadu_epi16(k1, src_uv + 64); + _mm256_mask_storeu_epi8(dst_u + 32, k1, _mm512_cvtepi16_epi8(uv1)); + _mm256_mask_storeu_epi8(dst_v + 32, k1, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8))); + } + } _mm256_zeroupper(); } #endif // HAS_SPLITUVROW_AVX512BW @@ -8017,7 +8380,7 @@ void Convert8To16Row_AVX512BW(const uint8_t* src_y, int bits, int width) { const __m128i shift128 = _mm_cvtsi32_si128(16 - bits); - do { + while (width >= 64) { __m256i y0_8 = _mm256_loadu_si256((const __m256i*)src_y); __m256i y1_8 = _mm256_loadu_si256((const __m256i*)(src_y + 32)); __m512i y0 = _mm512_cvtepu8_epi16(y0_8); @@ -8031,7 +8394,24 @@ void Convert8To16Row_AVX512BW(const uint8_t* src_y, src_y += 64; dst_y += 64; width -= 64; - } while (width > 0); + } + if (width >= 32) { + __m256i y0_8 = _mm256_loadu_si256((const __m256i*)src_y); + __m512i y0 = _mm512_cvtepu8_epi16(y0_8); + y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)), + shift128); + _mm512_storeu_si512((__m512i*)dst_y, y0); + src_y += 32; + dst_y += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i y0 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, src_y)); + y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)), + shift128); + _mm512_mask_storeu_epi16(dst_y, k, y0); + } _mm256_zeroupper(); } #endif // HAS_CONVERT8TO16ROW_AVX512BW @@ -8050,7 +8430,7 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y, const __m512i zmm_mask = _mm512_loadu_si512((const __m512i*)kShuffleMaskJ400ToARGB_AVX512); const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u); - do { + while (width >= 32) { __m512i z0 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)src_y)); __m512i z1 = @@ -8062,7 +8442,20 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y, src_y += 32; dst_argb += 128; width -= 32; - } while (width > 0); + } + if (width > 0) { + uint32_t mask = (1u << width) - 1u; + __mmask16 k0 = (__mmask16)mask; + __mmask16 k1 = (__mmask16)(mask >> 16); + __m512i z0 = _mm512_broadcast_i32x4(_mm_maskz_loadu_epi8(k0, src_y)); + z0 = _mm512_or_si512(_mm512_shuffle_epi8(z0, zmm_mask), zmm_alpha); + _mm512_mask_storeu_epi32(dst_argb, k0, z0); + if (k1) { + __m512i z1 = _mm512_broadcast_i32x4(_mm_maskz_loadu_epi8(k1, src_y + 16)); + z1 = _mm512_or_si512(_mm512_shuffle_epi8(z1, zmm_mask), zmm_alpha); + _mm512_mask_storeu_epi32(dst_argb + 64, k1, z1); + } + } _mm256_zeroupper(); } #endif // HAS_J400TOARGBROW_AVX512BW @@ -8073,16 +8466,25 @@ void MergeUVRow_AVX512BW(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) { - do { - __m512i u16 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_u)); - __m512i v16 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_v)); + while (width >= 32) { + __m512i u16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_u)); + __m512i v16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_v)); __m512i uv = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); _mm512_storeu_si512((__m512i*)dst_uv, uv); src_u += 32; src_v += 32; dst_uv += 64; width -= 32; - } while (width > 0); + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, src_u)); + __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, src_v)); + __m512i uv = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + _mm512_mask_storeu_epi16(dst_uv, k, uv); + } _mm256_zeroupper(); } #endif // HAS_MERGEUVROW_AVX512BW @@ -8094,7 +8496,7 @@ void MultiplyRow_16_AVX512BW(const uint16_t* src_y, int scale, int width) { const __m512i zmm_scale = _mm512_set1_epi16((short)scale); - do { + while (width >= 64) { __m512i z0 = _mm512_loadu_si512((const __m512i*)src_y); __m512i z1 = _mm512_loadu_si512((const __m512i*)(src_y + 32)); z0 = _mm512_mullo_epi16(z0, zmm_scale); @@ -8104,7 +8506,21 @@ void MultiplyRow_16_AVX512BW(const uint16_t* src_y, src_y += 64; dst_y += 64; width -= 64; - } while (width > 0); + } + if (width >= 32) { + __m512i z0 = _mm512_loadu_si512((const __m512i*)src_y); + z0 = _mm512_mullo_epi16(z0, zmm_scale); + _mm512_storeu_si512((__m512i*)dst_y, z0); + src_y += 32; + dst_y += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i z0 = _mm512_maskz_loadu_epi16(k, src_y); + z0 = _mm512_mullo_epi16(z0, zmm_scale); + _mm512_mask_storeu_epi16(dst_y, k, z0); + } _mm256_zeroupper(); } #endif // HAS_MULTIPLYROW_16_AVX512BW @@ -8129,7 +8545,7 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias); zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias); - while (width > 0) { + while (width >= 64) { __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb); __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); @@ -8158,6 +8574,41 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, dst_y += 64; width -= 64; } + if (width > 0) { + uint64_t mask = (1ULL << width) - 1ULL; + __mmask16 k0 = (__mmask16)mask; + __mmask16 k1 = (__mmask16)(mask >> 16); + __mmask16 k2 = (__mmask16)(mask >> 32); + __mmask16 k3 = (__mmask16)(mask >> 48); + __m512i zmm0 = + _mm512_xor_si512(_mm512_maskz_loadu_epi32(k0, src_argb), zmm_80); + __m512i zmm1 = + _mm512_xor_si512(k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64) + : _mm512_setzero_si512(), + zmm_80); + __m512i zmm2 = + _mm512_xor_si512(k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128) + : _mm512_setzero_si512(), + zmm_80); + __m512i zmm3 = + _mm512_xor_si512(k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192) + : _mm512_setzero_si512(), + zmm_80); + + zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one); + zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one); + zmm0 = _mm512_packs_epi32(zmm0, zmm1); + + zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one); + zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one); + zmm2 = _mm512_packs_epi32(zmm2, zmm3); + + zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8); + zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8); + + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); + _mm512_mask_storeu_epi8(dst_y, (__mmask64)mask, zmm0); + } _mm256_zeroupper(); } #endif // HAS_ARGBTOYMATRIXROW_AVX512BW @@ -8174,11 +8625,12 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, const __m512i zmm_v = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); const __m512i zmm_one = _mm512_set1_epi16(1); - const __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + const __m512i zmm_8000 = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV)); const __m512i zmm_perm_y = _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); - while (width > 0) { + while (width >= 64) { __m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb); __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); @@ -8217,6 +8669,44 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, dst_v += 64; width -= 64; } + if (width > 0) { + uint64_t mask = (1ULL << width) - 1ULL; + __mmask16 k0 = (__mmask16)mask; + __mmask16 k1 = (__mmask16)(mask >> 16); + __mmask16 k2 = (__mmask16)(mask >> 32); + __mmask16 k3 = (__mmask16)(mask >> 48); + __m512i s0 = _mm512_maskz_loadu_epi32(k0, src_argb); + __m512i s1 = k1 ? _mm512_maskz_loadu_epi32(k1, src_argb + 64) + : _mm512_setzero_si512(); + __m512i s2 = k2 ? _mm512_maskz_loadu_epi32(k2, src_argb + 128) + : _mm512_setzero_si512(); + __m512i s3 = k3 ? _mm512_maskz_loadu_epi32(k3, src_argb + 192) + : _mm512_setzero_si512(); + + __m512i u0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_u), zmm_one); + __m512i u1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_u), zmm_one); + __m512i u2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_u), zmm_one); + __m512i u3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_u), zmm_one); + __m512i u01 = _mm512_srli_epi16( + _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u0, u1)), 8); + __m512i u23 = _mm512_srli_epi16( + _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u2, u3)), 8); + _mm512_mask_storeu_epi8( + dst_u, (__mmask64)mask, + PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(u01, u23))); + + __m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one); + __m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one); + __m512i v2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_v), zmm_one); + __m512i v3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_v), zmm_one); + __m512i v01 = _mm512_srli_epi16( + _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v0, v1)), 8); + __m512i v23 = _mm512_srli_epi16( + _mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v2, v3)), 8); + _mm512_mask_storeu_epi8( + dst_v, (__mmask64)mask, + PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(v01, v23))); + } _mm256_zeroupper(); } #endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW @@ -8239,7 +8729,8 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); const __m512i zmm_0101 = _mm512_set1_epi16(0x0101); const __m512i zmm_one = _mm512_set1_epi16(1); - const __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + const __m512i zmm_8000 = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddUV)); const __m512i zmm_zero = _mm512_setzero_si512(); const __m512i zmm_shuf = _mm512_broadcast_i32x4( _mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512BW)); @@ -8391,7 +8882,7 @@ void DetileToYUY2_AVX2(const uint8_t* src_y, LIBYUV_TARGET_AVX2 void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) { const __m256i mask_00c0 = _mm256_set1_epi16(0x00c0); - for (size_t i = 0; i < size; i += 80) { + while (size >= 80) { __m256i low16 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src)); __m256i l0 = _mm256_and_si256(_mm256_slli_epi16(low16, 6), mask_00c0); __m256i l1 = _mm256_and_si256(_mm256_slli_epi16(low16, 4), mask_00c0); @@ -8427,6 +8918,7 @@ void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) { src += 80; dst += 64; + size -= 80; } _mm256_zeroupper(); } @@ -8459,6 +8951,7 @@ void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width) { } _mm256_zeroupper(); if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. ARGBGrayRow_C(src_argb, dst_argb, width); } } @@ -8589,6 +9082,94 @@ void HalfRow_16_AVX2(const uint16_t* src_uv, } #endif // HAS_HALFROW_16_AVX2 +#ifdef HAS_HALFMERGEUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void HalfMergeUVRow_AVX2(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width) { + const __m256i ymm_0101 = _mm256_set1_epi16(0x0101); + const __m256i ymm_zero = _mm256_setzero_si256(); + do { + __m256i u0 = _mm256_loadu_si256((const __m256i*)src_u); + __m256i v0 = _mm256_loadu_si256((const __m256i*)src_v); + __m256i u1 = _mm256_loadu_si256((const __m256i*)(src_u + src_stride_u)); + __m256i v1 = _mm256_loadu_si256((const __m256i*)(src_v + src_stride_v)); + u0 = _mm256_add_epi16(_mm256_maddubs_epi16(u0, ymm_0101), + _mm256_maddubs_epi16(u1, ymm_0101)); + v0 = _mm256_add_epi16(_mm256_maddubs_epi16(v0, ymm_0101), + _mm256_maddubs_epi16(v1, ymm_0101)); + u0 = _mm256_avg_epu16(_mm256_srli_epi16(u0, 1), ymm_zero); + v0 = _mm256_avg_epu16(_mm256_srli_epi16(v0, 1), ymm_zero); + u0 = _mm256_packus_epi16(u0, u0); + v0 = _mm256_packus_epi16(v0, v0); + _mm256_storeu_si256((__m256i*)dst_uv, _mm256_unpacklo_epi8(u0, v0)); + src_u += 32; + src_v += 32; + dst_uv += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFMERGEUVROW_AVX2 + +#ifdef HAS_CONVERT8TO8ROW_AVX2 +LIBYUV_TARGET_AVX2 +void Convert8To8Row_AVX2(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width) { + const __m256i ymm_scale = + _mm256_slli_epi16(_mm256_set1_epi16((short)scale), 8); + const __m256i ymm_bias = _mm256_set1_epi8((char)bias); + const __m256i ymm_zero = _mm256_setzero_si256(); + do { + __m256i y = _mm256_loadu_si256((const __m256i*)src_y); + __m256i lo = + _mm256_mulhi_epu16(_mm256_unpacklo_epi8(y, ymm_zero), ymm_scale); + __m256i hi = + _mm256_mulhi_epu16(_mm256_unpackhi_epi8(y, ymm_zero), ymm_scale); + __m256i out = _mm256_add_epi8(_mm256_packus_epi16(lo, hi), ymm_bias); + _mm256_storeu_si256((__m256i*)dst_y, out); + src_y += 32; + dst_y += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_CONVERT8TO8ROW_AVX2 + +#ifdef HAS_DIVIDEROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void DivideRow_16_AVX2(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } + const __m256i ymm_scale = _mm256_set1_epi16((short)scale); + while (width > 0) { + __m256i v0 = _mm256_mulhi_epu16(_mm256_loadu_si256((const __m256i*)src_y), + ymm_scale); + __m256i v1 = _mm256_mulhi_epu16( + _mm256_loadu_si256((const __m256i*)(src_y + 16)), ymm_scale); + _mm256_storeu_si256((__m256i*)dst_y, v0); + _mm256_storeu_si256((__m256i*)(dst_y + 16), v1); + src_y += 32; + dst_y += 32; + width -= 32; + } + _mm256_zeroupper(); +} +#endif // HAS_DIVIDEROW_16_AVX2 + #ifdef HAS_CONVERT16TO8ROW_AVX2 LIBYUV_TARGET_AVX2 void Convert16To8Row_AVX2(const uint16_t* src_y, @@ -8693,13 +9274,26 @@ void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv, int scale, int width) { int n = (width - 1) & ~15; + int r = (width - 1) & 15; + SIMD_ALIGNED(uint16_t pair[64]); + SIMD_ALIGNED(uint8_t out[16]); if (n > 0) { HalfWidthRow_16To8_AVX2(src_uv, src_uv_stride, dst_uv, scale, n); src_uv += n * 2; dst_uv += n; - width -= n; } - HalfWidthRow_16To8_Odd_C(src_uv, src_uv_stride, dst_uv, scale, width); + if (r > 0) { + HalfWidthRow_16To8_Any_AVX2(src_uv, src_uv_stride, dst_uv, scale, r); + src_uv += r * 2; + dst_uv += r; + } + memset(pair, 0, sizeof(pair)); + pair[0] = src_uv[0]; + pair[1] = src_uv[0]; + pair[32] = src_uv[src_uv_stride]; + pair[33] = src_uv[src_uv_stride]; + HalfWidthRow_16To8_AVX2(pair, 32, out, scale, 16); + dst_uv[0] = out[0]; } #endif // HAS_HALFWIDTHROW_16TO8_ODD_AVX2 @@ -8734,6 +9328,7 @@ void GaussCol_AVX2(const uint16_t* src0, } _mm256_zeroupper(); if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. GaussCol_C(src0, src1, src2, src3, src4, dst, width); } } @@ -8771,6 +9366,7 @@ void GaussCol_F32_AVX2(const float* src0, } _mm256_zeroupper(); if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. GaussCol_F32_C(src0, src1, src2, src3, src4, dst, width); } } @@ -8798,6 +9394,7 @@ float ScaleMaxSamples_AVX2(const float* src, m128 = _mm_max_ss(_mm_shuffle_ps(m128, m128, 1), m128); float fmax = _mm_cvtss_f32(m128); _mm256_zeroupper(); + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. for (; width > 0; --width) { float v = *src++; float vs = v * scale; @@ -8830,6 +9427,7 @@ float ScaleSumSamples_AVX2(const float* src, s128 = _mm_add_ss(s128, _mm_shuffle_ps(s128, s128, 1)); float fsum = _mm_cvtss_f32(s128); _mm256_zeroupper(); + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. for (; width > 0; --width) { float v = *src++; fsum += v * v; @@ -8852,6 +9450,7 @@ void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) { } _mm256_zeroupper(); if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. ScaleSamples_C(src, dst, scale, width); } } @@ -9111,8 +9710,7 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, res = _mm256_add_ps(res, _mm256_mul_ps(c2, x2)); res = _mm256_add_ps(res, _mm256_mul_ps(_mm256_mul_ps(c3, x), x2)); __m256i i32 = _mm256_cvttps_epi32(res); - __m256i u16 = - _mm256_permute4x64_epi64(_mm256_packus_epi32(i32, i32), 0xd8); + __m256i u16 = _mm256_permute4x64_epi64(_mm256_packs_epi32(i32, i32), 0xd8); __m128i u8 = _mm_packus_epi16(_mm256_castsi256_si128(u16), _mm256_castsi256_si128(u16)); _mm_storel_epi64((__m128i*)dst_argb, u8); @@ -9227,16 +9825,16 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, uint8_t* dst, int count) { __m128 xmm5 = _mm_cvtsi32_ss(_mm_setzero_ps(), area); - __m128 xmm4 = _mm_rcp_ss(xmm5); + __m128 xmm4 = _mm_div_ss(_mm_set_ss(1.0f), xmm5); xmm4 = _mm_shuffle_ps(xmm4, xmm4, 0); if (count >= 4) { __m256 ymm4 = _mm256_insertf128_ps(_mm256_castps128_ps256(xmm4), xmm4, 1); - if ((uint32_t)area <= 0x80u) { + if (area > 1 && area <= 0x80) { __m128 xmm6 = _mm_cvtepi32_ps(_mm_set1_epi32(0x0000ffff)); xmm6 = _mm_mul_ss(_mm_add_ss(xmm5, xmm6), xmm4); __m128i i6 = _mm_cvtps_epi32(xmm6); - i6 = _mm_packs_epi32(i6, i6); + i6 = _mm_packus_epi32(i6, i6); __m256i ymm6 = _mm256_broadcastw_epi16(i6); while (count >= 4) { __m256i ymm0 = _mm256_sub_epi32( @@ -9255,7 +9853,7 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, ymm1 = _mm256_add_epi32( ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8))); botleft += 16; - ymm0 = _mm256_mulhi_epu16(_mm256_packs_epi32(ymm0, ymm1), ymm6); + ymm0 = _mm256_mulhi_epu16(_mm256_packus_epi32(ymm0, ymm1), ymm6); __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0), _mm256_extracti128_si256(ymm0, 1)); _mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8)); @@ -9280,9 +9878,11 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, ymm1 = _mm256_add_epi32( ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8))); botleft += 16; - ymm0 = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm0), ymm4)); - ymm1 = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm1), ymm4)); - ymm0 = _mm256_packs_epi32(ymm0, ymm1); + ymm0 = + _mm256_cvttps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm0), ymm4)); + ymm1 = + _mm256_cvttps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm1), ymm4)); + ymm0 = _mm256_packus_epi32(ymm0, ymm1); __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0), _mm256_extracti128_si256(ymm0, 1)); _mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8)); @@ -9303,8 +9903,8 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, _mm_loadu_si128((const __m128i*)botleft)); botleft += 4; d0 = _mm_add_epi32(d0, d1); - d0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(d0), xmm4)); - d0 = _mm_packs_epi32(d0, d0); + d0 = _mm_cvttps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(d0), xmm4)); + d0 = _mm_packus_epi32(d0, d0); d0 = _mm_packus_epi16(d0, d0); int32_t out32 = _mm_cvtsi128_si32(d0); memcpy(dst, &out32, sizeof(out32)); @@ -9559,8 +10159,8 @@ void MergeAR64Row_AVX2(const uint16_t* src_r, int depth, int width) { const __m128i shift128 = _mm_cvtsi32_si128(16 - depth); - const __m256i ymm_max = - _mm256_set1_epi16((short)((1u << depth) - 1u)); + const uint32_t max_val = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u); + const __m256i ymm_max = _mm256_set1_epi16((short)max_val); do { __m256i r = _mm256_sll_epi16( _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_r), ymm_max), @@ -9610,8 +10210,8 @@ void MergeXR64Row_AVX2(const uint16_t* src_r, int depth, int width) { const __m128i shift128 = _mm_cvtsi32_si128(16 - depth); - const __m256i ymm_max = - _mm256_set1_epi16((short)((1u << depth) - 1u)); + const uint32_t max_val = (depth >= 16) ? 0xffffu : ((1u << depth) - 1u); + const __m256i ymm_max = _mm256_set1_epi16((short)max_val); const __m256i a = _mm256_set1_epi16((short)0xffff); do { __m256i r = _mm256_sll_epi16( @@ -9691,6 +10291,82 @@ void NV21ToYUV24Row_AVX2(const uint8_t* src_y, } #endif // HAS_NV21TOYUV24ROW_AVX2 +#ifdef HAS_NV21TOYUV24ROW_AVX512VBMI +LIBYUV_TARGET_AVX512VBMI +void NV21ToYUV24Row_AVX512VBMI(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_yuv24, + int width) { + static const uint8_t kNV21ToYUV24Perm_AVX512VBMI[3][64] = { + {64, 65, 0, 64, 65, 1, 66, 67, 2, 66, 67, 3, 68, 69, 4, 68, + 69, 5, 70, 71, 6, 70, 71, 7, 72, 73, 8, 72, 73, 9, 74, 75, + 10, 74, 75, 11, 76, 77, 12, 76, 77, 13, 78, 79, 14, 78, 79, 15, + 80, 81, 16, 80, 81, 17, 82, 83, 18, 82, 83, 19, 84, 85, 20, 84}, + {85, 21, 86, 87, 22, 86, 87, 23, 88, 89, 24, 88, 89, + 25, 90, 91, 26, 90, 91, 27, 92, 93, 28, 92, 93, 29, + 94, 95, 30, 94, 95, 31, 96, 97, 32, 96, 97, 33, 98, + 99, 34, 98, 99, 35, 100, 101, 36, 100, 101, 37, 102, 103, + 38, 102, 103, 39, 104, 105, 40, 104, 105, 41, 106, 107}, + {42, 106, 107, 43, 108, 109, 44, 108, 109, 45, 110, 111, 46, + 110, 111, 47, 112, 113, 48, 112, 113, 49, 114, 115, 50, 114, + 115, 51, 116, 117, 52, 116, 117, 53, 118, 119, 54, 118, 119, + 55, 120, 121, 56, 120, 121, 57, 122, 123, 58, 122, 123, 59, + 124, 125, 60, 124, 125, 61, 126, 127, 62, 126, 127, 63}}; + __m512i zmm_p0 = + _mm512_loadu_si512((const __m512i*)kNV21ToYUV24Perm_AVX512VBMI[0]); + __m512i zmm_p1 = + _mm512_loadu_si512((const __m512i*)kNV21ToYUV24Perm_AVX512VBMI[1]); + __m512i zmm_p2 = + _mm512_loadu_si512((const __m512i*)kNV21ToYUV24Perm_AVX512VBMI[2]); + LIBYUV_HIDE_CONSTANT(zmm_p0); + LIBYUV_HIDE_CONSTANT(zmm_p1); + LIBYUV_HIDE_CONSTANT(zmm_p2); + + while (width >= 64) { + __m512i zy = _mm512_loadu_si512((const __m512i*)src_y); + __m512i zvu = _mm512_loadu_si512((const __m512i*)src_vu); + __m512i dst0 = _mm512_permutex2var_epi8(zy, zmm_p0, zvu); + __m512i dst1 = _mm512_permutex2var_epi8(zy, zmm_p1, zvu); + __m512i dst2 = _mm512_permutex2var_epi8(zy, zmm_p2, zvu); + _mm512_storeu_si512((__m512i*)dst_yuv24, dst0); + _mm512_storeu_si512((__m512i*)(dst_yuv24 + 64), dst1); + _mm512_storeu_si512((__m512i*)(dst_yuv24 + 128), dst2); + src_y += 64; + src_vu += 64; + dst_yuv24 += 192; + width -= 64; + } + if (width > 0) { + __mmask64 ky = (__mmask64)((1ULL << width) - 1ULL); + int vu_bytes = (width + 1) & ~1; + __mmask64 kvu = (vu_bytes >= 64) ? ~(__mmask64)0 + : (__mmask64)((1ULL << vu_bytes) - 1ULL); + __m512i zy = _mm512_maskz_loadu_epi8(ky, src_y); + __m512i zvu = _mm512_maskz_loadu_epi8(kvu, src_vu); + __m512i dst0 = _mm512_permutex2var_epi8(zy, zmm_p0, zvu); + __m512i dst1 = _mm512_permutex2var_epi8(zy, zmm_p1, zvu); + __m512i dst2 = _mm512_permutex2var_epi8(zy, zmm_p2, zvu); + int nbytes = width * 3; + __mmask64 k0 = + (nbytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << nbytes) - 1ULL); + __mmask64 k1 = (nbytes <= 64) ? 0 + : (nbytes >= 128) + ? ~(__mmask64)0 + : (__mmask64)((1ULL << (nbytes - 64)) - 1ULL); + __mmask64 k2 = + (nbytes <= 128) ? 0 : (__mmask64)((1ULL << (nbytes - 128)) - 1ULL); + _mm512_mask_storeu_epi8(dst_yuv24, k0, dst0); + if (k1) { + _mm512_mask_storeu_epi8(dst_yuv24 + 64, k1, dst1); + } + if (k2) { + _mm512_mask_storeu_epi8(dst_yuv24 + 128, k2, dst2); + } + } + _mm256_zeroupper(); +} +#endif // HAS_NV21TOYUV24ROW_AVX512VBMI + #ifdef HAS_SPLITARGBROW_AVX2 LIBYUV_TARGET_AVX2 void SplitARGBRow_AVX2(const uint8_t* src_argb, diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index dc0383c37..4c239b7d9 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -415,6 +415,7 @@ void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr, #endif // HAS_SCALEROWDOWN2_16_AVX2 #ifdef HAS_SCALEROWDOWN2_AVX512BW +LIBYUV_TARGET_AVX512BW void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -455,6 +456,7 @@ void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr, : "memory", "cc", "xmm0", "k1"); } +LIBYUV_TARGET_AVX512BW void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -500,6 +502,7 @@ void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr, : "memory", "cc", "xmm0", "xmm4", "xmm5", "k1"); } +LIBYUV_TARGET_AVX512BW void ScaleRowDown2Box_AVX512BW(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -2200,6 +2203,7 @@ void ScaleAddRow_AVX2(const uint8_t* src_ptr, #ifdef HAS_SCALEADDROW_AVX512BW // Reads 32xN bytes and accumulates to 32xN shorts at a time, with masked tail. +LIBYUV_TARGET_AVX512BW void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) { @@ -2259,6 +2263,7 @@ static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7, #endif #ifdef HAS_SCALEADDCOLS1_AVX512BW +LIBYUV_TARGET_AVX512BW void ScaleAddCols1_AVX512BW(int dst_width, int boxheight, int x, @@ -2511,6 +2516,7 @@ void ScaleAddCols1_AVX512BW(int dst_width, #endif // HAS_SCALEADDCOLS1_AVX512BW #ifdef HAS_SCALEADDCOLS2_AVX512BW +LIBYUV_TARGET_AVX512BW void ScaleAddCols2_AVX512BW(int dst_width, int boxheight, int x, @@ -4246,6 +4252,7 @@ static const uint8_t kPackEvenBytes[64] = { // Scale UV columns by 3 box filter using AVX512VBMI. // Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop // iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels). +LIBYUV_TARGET_AVX512VBMI void ScaleUVAddCols3_AVX512VBMI(int dst_width, int scaleval, const uint16_t* src_uv, diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index cd4d98efd..5854b8903 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -2668,8 +2668,11 @@ static int TestHalfFloatPlane(int benchmark_width, int mask, int invert, int off) { - int i, j; - const int y_plane_size = benchmark_width * benchmark_height * 2; + if (benchmark_width <= 0 || benchmark_height <= 0) { + return 0; + } + const size_t y_plane_size = static_cast(benchmark_width) * + static_cast(benchmark_height) * 2; align_buffer_page_end(orig_y, y_plane_size + off); align_buffer_page_end(dst_c, y_plane_size); align_buffer_page_end(dst_opt, y_plane_size); @@ -2678,13 +2681,13 @@ static int TestHalfFloatPlane(int benchmark_width, memset(dst_c, 1, y_plane_size); memset(dst_opt, 2, y_plane_size); - for (i = 0; i < y_plane_size / 2; ++i) { + for (size_t i = 0; i < y_plane_size / 2; ++i) { reinterpret_cast(orig_y + off)[i] &= mask; } // Disable all optimizations. MaskCpuFlags(disable_cpu_flags); - for (j = 0; j < benchmark_iterations; j++) { + for (int j = 0; j < benchmark_iterations; j++) { HalfFloatPlane(reinterpret_cast(orig_y + off), benchmark_width * 2, reinterpret_cast(dst_c), benchmark_width * 2, scale, benchmark_width, @@ -2693,7 +2696,7 @@ static int TestHalfFloatPlane(int benchmark_width, // Enable optimizations. MaskCpuFlags(benchmark_cpu_info); - for (j = 0; j < benchmark_iterations; j++) { + for (int j = 0; j < benchmark_iterations; j++) { HalfFloatPlane(reinterpret_cast(orig_y + off), benchmark_width * 2, reinterpret_cast(dst_opt), benchmark_width * 2, scale, benchmark_width, @@ -2701,7 +2704,7 @@ static int TestHalfFloatPlane(int benchmark_width, } int max_diff = 0; - for (i = 0; i < y_plane_size / 2; ++i) { + for (size_t i = 0; i < y_plane_size / 2; ++i) { int abs_diff = abs(static_cast(reinterpret_cast(dst_c)[i]) - static_cast(reinterpret_cast(dst_opt)[i])); @@ -2838,9 +2841,11 @@ static float TestByteToFloat(int benchmark_width, int i, j; const int y_plane_size = benchmark_width * benchmark_height; - align_buffer_page_end(orig_y, y_plane_size * (1 + 4 + 4)); - float* dst_opt = reinterpret_cast(orig_y + y_plane_size); - float* dst_c = reinterpret_cast(orig_y + y_plane_size * 5); + align_buffer_page_end(orig_y, y_plane_size); + align_buffer_page_end(dst_opt_mem, y_plane_size * 4); + align_buffer_page_end(dst_c_mem, y_plane_size * 4); + float* dst_opt = reinterpret_cast(dst_opt_mem); + float* dst_c = reinterpret_cast(dst_c_mem); MemRandomize(orig_y, y_plane_size); memset(dst_c, 0, y_plane_size * 4); @@ -2865,6 +2870,8 @@ static float TestByteToFloat(int benchmark_width, } free_aligned_buffer_page_end(orig_y); + free_aligned_buffer_page_end(dst_opt_mem); + free_aligned_buffer_page_end(dst_c_mem); return max_diff; } @@ -4228,9 +4235,9 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Any) { #if defined(HAS_MULTIPLYROW_16_AVX512BW) if (has_avx512) { - MultiplyRow_16_Any_AVX512BW( - reinterpret_cast(src_pixels_y), - reinterpret_cast(dst_pixels_y_opt), 64, width); + MultiplyRow_16_AVX512BW(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), 64, + width); } else #endif #if defined(HAS_MULTIPLYROW_16_AVX2) @@ -6184,4 +6191,442 @@ TEST_F(LibYUVPlanarTest, MSanUninitializedInputsAndTails) { free_aligned_buffer_page_end(uv_src); } +// Regression test for RGBToUVMatrixRow_Any_SSSE3 MASK=15 (previously MASK=7 +// overread 24 bytes past src_rgb24 at width=24). +TEST_F(LibYUVPlanarTest, RGB24ToI420_Width24_OOB) { + const int kWidth = 24; + const int kHeight = 2; + align_buffer_page_end(src_rgb24, kWidth * 3 * kHeight); + align_buffer_page_end(dst_y_c, kWidth * kHeight); + align_buffer_page_end(dst_u_c, (kWidth / 2) * (kHeight / 2)); + align_buffer_page_end(dst_v_c, (kWidth / 2) * (kHeight / 2)); + align_buffer_page_end(dst_y_opt, kWidth * kHeight); + align_buffer_page_end(dst_u_opt, (kWidth / 2) * (kHeight / 2)); + align_buffer_page_end(dst_v_opt, (kWidth / 2) * (kHeight / 2)); + MemRandomize(src_rgb24, kWidth * 3 * kHeight); + + MaskCpuFlags(disable_cpu_flags_); + RGB24ToI420(src_rgb24, kWidth * 3, dst_y_c, kWidth, dst_u_c, kWidth / 2, + dst_v_c, kWidth / 2, kWidth, kHeight); + + const int kNoAvx512 = + benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VL | + kCpuHasAVX512VBMI | kCpuHasAVX512VNNI); + const int kSsse3Only = kNoAvx512 & ~kCpuHasAVX2; + const int kModes[3] = {benchmark_cpu_info_, kNoAvx512, kSsse3Only}; + for (int m = 0; m < 3; ++m) { + MaskCpuFlags(kModes[m]); + memset(dst_y_opt, 0, kWidth * kHeight); + memset(dst_u_opt, 0, (kWidth / 2) * (kHeight / 2)); + memset(dst_v_opt, 0, (kWidth / 2) * (kHeight / 2)); + RGB24ToI420(src_rgb24, kWidth * 3, dst_y_opt, kWidth, dst_u_opt, kWidth / 2, + dst_v_opt, kWidth / 2, kWidth, kHeight); + for (int i = 0; i < kWidth * kHeight; ++i) { + ASSERT_EQ(dst_y_c[i], dst_y_opt[i]); + } + for (int i = 0; i < (kWidth / 2) * (kHeight / 2); ++i) { + ASSERT_EQ(dst_u_c[i], dst_u_opt[i]); + ASSERT_EQ(dst_v_c[i], dst_v_opt[i]); + } + } + MaskCpuFlags(benchmark_cpu_info_); + + free_aligned_buffer_page_end(src_rgb24); + free_aligned_buffer_page_end(dst_y_c); + free_aligned_buffer_page_end(dst_u_c); + free_aligned_buffer_page_end(dst_v_c); + free_aligned_buffer_page_end(dst_y_opt); + free_aligned_buffer_page_end(dst_u_opt); + free_aligned_buffer_page_end(dst_v_opt); +} + +#ifdef ENABLE_ROW_TESTS +#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX2) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) || \ + defined(HAS_HALFROW_16TO8_AVX512BW) +// Regression test for HalfWidthRow_16To8 (>= 0x8000 unsigned MSB samples and +// exact page-end tail bounds) and HalfRow_16To8 (non-power-of-two scale). +TEST_F(LibYUVPlanarTest, Half16To8_HighBitAndNonPow2Scale) { + const int kWidths[] = {1, 7, 15, 24, 31, 33, 63, 65}; + for (size_t w_idx = 0; w_idx < sizeof(kWidths) / sizeof(kWidths[0]); + ++w_idx) { + const int w = kWidths[w_idx]; + align_buffer_page_end_16(src_uv_hw, w * 4); + align_buffer_page_end(dst_c, w); + align_buffer_page_end(dst_opt, w); + for (int i = 0; i < w * 4; ++i) { + src_uv_hw[i] = static_cast(0x8000u + ((i * 257u) & 0x7fffu)); + } + HalfWidthRow_16To8_C(src_uv_hw, w * 2, dst_c, 256, w); +#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + memset(dst_opt, 0x5a, w); + HalfWidthRow_16To8_AVX512BW(src_uv_hw, w * 2, dst_opt, 256, w); + for (int i = 0; i < w; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]) << "AVX512BW w=" << w << " i=" << i; + } + } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + memset(dst_opt, 0x5a, w); + HalfWidthRow_16To8_Any_AVX2(src_uv_hw, w * 2, dst_opt, 256, w); + for (int i = 0; i < w; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]) << "AVX2 w=" << w << " i=" << i; + } + } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + memset(dst_opt, 0x5a, w); + HalfWidthRow_16To8_Any_SSSE3(src_uv_hw, w * 2, dst_opt, 256, w); + for (int i = 0; i < w; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]) << "SSSE3 w=" << w << " i=" << i; + } + } +#endif + + // HalfRow_16To8 with non-power-of-two scale (e.g. 16320). + align_buffer_page_end_16(src_uv_hr, w * 2); + for (int i = 0; i < w * 2; ++i) { + src_uv_hr[i] = static_cast((i * 73u) & 1023u); + } + HalfRow_16To8_C(src_uv_hr, w, dst_c, 16320, w); +#if defined(HAS_HALFROW_16TO8_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + memset(dst_opt, 0x5a, w); + HalfRow_16To8_AVX512BW(src_uv_hr, w, dst_opt, 16320, w); + for (int i = 0; i < w; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]) + << "HalfRow_16To8_AVX512BW w=" << w << " i=" << i; + } + } +#endif + free_aligned_buffer_page_end_16(src_uv_hr); + free_aligned_buffer_page_end_16(src_uv_hw); + free_aligned_buffer_page_end(dst_c); + free_aligned_buffer_page_end(dst_opt); + } +} +#endif // HAS_HALFWIDTHROW_16TO8_* || HAS_HALFROW_16TO8_AVX512BW + +#if defined(HAS_UNPACKMT2T_AVX2) +// Regression test for UnpackMT2T_C and UnpackMT2T_AVX2 when size is not a +// multiple of the 80-byte (64-pixel) MT2T block size. +TEST_F(LibYUVPlanarTest, UnpackMT2T_PartialTail) { + if (!TestCpuFlag(kCpuHasAVX2)) { + return; + } + const size_t kSrcSize = 120; // 1 full 80-byte block + 40 trailing bytes + const size_t kDstCount = (kSrcSize / 80) * 64; + align_buffer_page_end(src, kSrcSize); + align_buffer_page_end_16(dst_c, kDstCount); + align_buffer_page_end_16(dst_opt, kDstCount); + MemRandomize(src, kSrcSize); + UnpackMT2T_C(src, dst_c, kSrcSize); + UnpackMT2T_AVX2(src, dst_opt, kSrcSize); + for (size_t i = 0; i < kDstCount; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]) << "i=" << i; + } + // Verify size < 80 performs zero out-of-bounds reads or writes. + UnpackMT2T_C(src + 80, dst_c, 40); + UnpackMT2T_AVX2(src + 80, dst_opt, 40); + for (size_t i = 0; i < kDstCount; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]) << "i=" << i; + } + free_aligned_buffer_page_end(src); + free_aligned_buffer_page_end_16(dst_c); + free_aligned_buffer_page_end_16(dst_opt); +} +#endif // HAS_UNPACKMT2T_AVX2 +#endif // ENABLE_ROW_TESTS + +// Regression test for ConvertToLSBPlane_16 with depth=16 (scale=65536). +TEST_F(LibYUVPlanarTest, ConvertToLSBPlane_16_Depth16) { + const int kWidth = 37; + const int kHeight = 4; + align_buffer_page_end_16(src_y, kWidth * kHeight); + align_buffer_page_end_16(dst_c, kWidth * kHeight); + align_buffer_page_end_16(dst_opt, kWidth * kHeight); + for (int i = 0; i < kWidth * kHeight; ++i) { + src_y[i] = static_cast(0x1234u + i * 251u); + } + MaskCpuFlags(disable_cpu_flags_); + ConvertToLSBPlane_16(src_y, kWidth, dst_c, kWidth, kWidth, kHeight, 16); + + const int kNoAvx512 = + benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VL); + MaskCpuFlags(benchmark_cpu_info_); + memset(dst_opt, 0, kWidth * kHeight * 2); + ConvertToLSBPlane_16(src_y, kWidth, dst_opt, kWidth, kWidth, kHeight, 16); + for (int i = 0; i < kWidth * kHeight; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]); + } + + MaskCpuFlags(kNoAvx512); + memset(dst_opt, 0, kWidth * kHeight * 2); + ConvertToLSBPlane_16(src_y, kWidth, dst_opt, kWidth, kWidth, kHeight, 16); + for (int i = 0; i < kWidth * kHeight; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]); + } + MaskCpuFlags(benchmark_cpu_info_); + + free_aligned_buffer_page_end_16(src_y); + free_aligned_buffer_page_end_16(dst_c); + free_aligned_buffer_page_end_16(dst_opt); +} + +// Regression test for ARGBPolynomial saturation on outputs >= 32768 / 65536, +// odd widths, and 4-byte-aligned (non-16-byte-aligned) polynomial arrays. +TEST_F(LibYUVPlanarTest, ARGBPolynomial_LargeOutputAndUnalignedPoly) { + const int kWidth = 37; + const int kHeight = 3; + align_buffer_page_end(src_argb, kWidth * 4 * kHeight); + align_buffer_page_end(dst_c, kWidth * 4 * kHeight); + align_buffer_page_end(dst_opt, kWidth * 4 * kHeight); + align_buffer_page_end(poly_buf, 17 * sizeof(float)); + // Offset by +1 float so poly is 4-byte aligned, not 16-byte aligned. + float* poly = reinterpret_cast(poly_buf) + 1; + for (int c = 0; c < 4; ++c) { + poly[0 + c] = 0.0f; + poly[4 + c] = 300.0f; // 255 * 300 = 76500 >= 65536 -> must clamp to 255 + poly[8 + c] = 1.0f; // 200^2 = 40000 in [32768, 65535] -> clamp to 255 + poly[12 + c] = 0.0f; + } + for (int i = 0; i < kWidth * 4 * kHeight; ++i) { + src_argb[i] = static_cast(i & 255); + } + + MaskCpuFlags(disable_cpu_flags_); + ARGBPolynomial(src_argb, kWidth * 4, dst_c, kWidth * 4, poly, kWidth, + kHeight); + + const int kSse2Only = + benchmark_cpu_info_ & ~(kCpuHasAVX2 | kCpuHasFMA3 | kCpuHasAVX512BW); + const int kModes[2] = {benchmark_cpu_info_, kSse2Only}; + for (int m = 0; m < 2; ++m) { + MaskCpuFlags(kModes[m]); + memset(dst_opt, 0, kWidth * 4 * kHeight); + ARGBPolynomial(src_argb, kWidth * 4, dst_opt, kWidth * 4, poly, kWidth, + kHeight); + for (int i = 0; i < kWidth * 4 * kHeight; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]) << "mode=" << m << " i=" << i; + } + } + MaskCpuFlags(benchmark_cpu_info_); + + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(dst_c); + free_aligned_buffer_page_end(dst_opt); + free_aligned_buffer_page_end(poly_buf); +} + +// Regression test for ARGBBlur / CumulativeSumToAverageRow_AVX2 when radius=0 +// or radius=1 produces small window areas (area == 1 or area == 2) on 0xff. +TEST_F(LibYUVPlanarTest, ARGBBlur_SmallRadiusWhite) { + const int kWidth = 37; + const int kHeight = 4; + const int kCumSumStride32 = (kWidth + 16) * 4; + align_buffer_page_end(src_argb, kWidth * 4 * kHeight); + align_buffer_page_end(dst_c, kWidth * 4 * kHeight); + align_buffer_page_end(dst_opt, kWidth * 4 * kHeight); + align_buffer_page_end(cumsum, kCumSumStride32 * (kHeight + 2) * 4); + memset(src_argb, 0xff, kWidth * 4 * kHeight); + + for (int radius = 1; radius <= 2; ++radius) { + MaskCpuFlags(disable_cpu_flags_); + ARGBBlur(src_argb, kWidth * 4, dst_c, kWidth * 4, + reinterpret_cast(cumsum), kCumSumStride32, kWidth, + kHeight, radius); + MaskCpuFlags(benchmark_cpu_info_); + memset(dst_opt, 0, kWidth * 4 * kHeight); + ARGBBlur(src_argb, kWidth * 4, dst_opt, kWidth * 4, + reinterpret_cast(cumsum), kCumSumStride32, kWidth, + kHeight, radius); + for (int i = 0; i < kWidth * 4 * kHeight; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]) << "radius=" << radius << " i=" << i; + } + } + + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(dst_c); + free_aligned_buffer_page_end(dst_opt); + free_aligned_buffer_page_end(cumsum); +} + +// Regression test for ARGBLumaColorTable with standard BT.601 green weight +// 150 (> 127) and bright pixels where R*77 + G*150 + B*29 > 32767. +TEST_F(LibYUVPlanarTest, ARGBLumaColorTable_BrightPixels) { + const int kWidth = 37; + const int kHeight = 3; + align_buffer_page_end(src_argb, kWidth * 4 * kHeight); + align_buffer_page_end(dst_c, kWidth * 4 * kHeight); + align_buffer_page_end(dst_opt, kWidth * 4 * kHeight); + align_buffer_page_end(luma, 32768); + for (int i = 0; i < 32768; ++i) { + luma[i] = static_cast(((i >> 8) * 3 + (i & 255)) & 255); + } + for (int i = 0; i < kWidth * kHeight; ++i) { + src_argb[i * 4 + 0] = static_cast(200 + (i % 56)); + src_argb[i * 4 + 1] = static_cast(200 + ((i * 3) % 56)); + src_argb[i * 4 + 2] = static_cast(200 + ((i * 7) % 56)); + src_argb[i * 4 + 3] = static_cast(i & 255); + } + + MaskCpuFlags(disable_cpu_flags_); + ARGBLumaColorTable(src_argb, kWidth * 4, dst_c, kWidth * 4, luma, kWidth, + kHeight); + MaskCpuFlags(benchmark_cpu_info_); + ARGBLumaColorTable(src_argb, kWidth * 4, dst_opt, kWidth * 4, luma, kWidth, + kHeight); + for (int i = 0; i < kWidth * 4 * kHeight; ++i) { + ASSERT_EQ(dst_c[i], dst_opt[i]) << "i=" << i; + } + + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(dst_c); + free_aligned_buffer_page_end(dst_opt); + free_aligned_buffer_page_end(luma); +} + +#if defined(ENABLE_ROW_TESTS) && !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) || \ + defined(__i386__)) +// Regression test for 2-byte-aligned (non-16/32-byte-aligned) Intel +// YuvConstants and ArgbConstants structs and custom non-0x8000 kAddUV offsets. +TEST_F(LibYUVPlanarTest, UnalignedMatrixConstantsAndCustomKAddUV) { + const int kWidth = 38; + const int kHeight = 4; + align_buffer_page_end(yuvc_buf, sizeof(YuvConstants) + 16); + align_buffer_page_end(argbc_buf, sizeof(ArgbConstants) + 16); + // Offset by +2 bytes (matches alignof == 2, misaligned for movdqa/vmovdqa). + YuvConstants* yuvc = reinterpret_cast(yuvc_buf + 2); + ArgbConstants* argbc = reinterpret_cast(argbc_buf + 2); + memcpy(yuvc, &kYuvI601Constants, sizeof(YuvConstants)); + memcpy(argbc, &kArgbI601Constants, sizeof(ArgbConstants)); + for (int i = 0; i < 16; ++i) { + argbc->kAddUV[i] = 0x7200u; // Non-default UV bias (!= 0x8000) + } + + align_buffer_page_end(src_argb, kWidth * 4 * kHeight); + align_buffer_page_end(y_c, kWidth * kHeight); + align_buffer_page_end(u_c, kWidth * kHeight); + align_buffer_page_end(v_c, kWidth * kHeight); + align_buffer_page_end(y_opt, kWidth * kHeight); + align_buffer_page_end(u_opt, kWidth * kHeight); + align_buffer_page_end(v_opt, kWidth * kHeight); + align_buffer_page_end(rgb_c, kWidth * 4 * kHeight); + align_buffer_page_end(rgb_opt, kWidth * 4 * kHeight); + MemRandomize(src_argb, kWidth * 4 * kHeight); + + MaskCpuFlags(disable_cpu_flags_); + ARGBToI444Matrix(src_argb, kWidth * 4, y_c, kWidth, u_c, kWidth, v_c, kWidth, + argbc, kWidth, kHeight); + I420ToARGBMatrix(y_c, kWidth, u_c, kWidth / 2, v_c, kWidth / 2, rgb_c, + kWidth * 4, yuvc, kWidth, kHeight); + + const int kNoAvx512 = + benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VL | + kCpuHasAVX512VBMI | kCpuHasAVX512VNNI); + const int kSsse3Only = kNoAvx512 & ~kCpuHasAVX2; + const int kModes[3] = {benchmark_cpu_info_, kNoAvx512, kSsse3Only}; + for (int m = 0; m < 3; ++m) { + MaskCpuFlags(kModes[m]); + ARGBToI444Matrix(src_argb, kWidth * 4, y_opt, kWidth, u_opt, kWidth, v_opt, + kWidth, argbc, kWidth, kHeight); + I420ToARGBMatrix(y_c, kWidth, u_c, kWidth / 2, v_c, kWidth / 2, rgb_opt, + kWidth * 4, yuvc, kWidth, kHeight); + for (int i = 0; i < kWidth * kHeight; ++i) { + ASSERT_EQ(y_c[i], y_opt[i]); + ASSERT_EQ(u_c[i], u_opt[i]) << "mode=" << m << " i=" << i; + ASSERT_EQ(v_c[i], v_opt[i]) << "mode=" << m << " i=" << i; + } + for (int i = 0; i < kWidth * 4 * kHeight; ++i) { + ASSERT_EQ(rgb_c[i], rgb_opt[i]) << "mode=" << m << " i=" << i; + } + } + + // Also test ARGBToI420Matrix with custom kAddUV and I400ToARGBMatrix with + // unaligned YuvConstants. + MaskCpuFlags(disable_cpu_flags_); + ARGBToI420Matrix(src_argb, kWidth * 4, y_c, kWidth, u_c, kWidth / 2, v_c, + kWidth / 2, argbc, kWidth, kHeight); + I400ToARGBMatrix(y_c, kWidth, rgb_c, kWidth * 4, yuvc, kWidth, kHeight); + for (int m = 0; m < 3; ++m) { + MaskCpuFlags(kModes[m]); + ARGBToI420Matrix(src_argb, kWidth * 4, y_opt, kWidth, u_opt, kWidth / 2, + v_opt, kWidth / 2, argbc, kWidth, kHeight); + I400ToARGBMatrix(y_c, kWidth, rgb_opt, kWidth * 4, yuvc, kWidth, kHeight); + for (int i = 0; i < (kWidth / 2) * (kHeight / 2); ++i) { + ASSERT_NEAR(u_c[i], u_opt[i], 1) << "mode=" << m << " i=" << i; + ASSERT_NEAR(v_c[i], v_opt[i], 1) << "mode=" << m << " i=" << i; + } + for (int i = 0; i < kWidth * 4 * kHeight; ++i) { + ASSERT_EQ(rgb_c[i], rgb_opt[i]) << "mode=" << m << " i=" << i; + } + } + MaskCpuFlags(benchmark_cpu_info_); + + free_aligned_buffer_page_end(yuvc_buf); + free_aligned_buffer_page_end(argbc_buf); + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(y_c); + free_aligned_buffer_page_end(u_c); + free_aligned_buffer_page_end(v_c); + free_aligned_buffer_page_end(y_opt); + free_aligned_buffer_page_end(u_opt); + free_aligned_buffer_page_end(v_opt); + free_aligned_buffer_page_end(rgb_c); + free_aligned_buffer_page_end(rgb_opt); +} +#endif // ENABLE_ROW_TESTS + +// Regression test for MergeAR64Plane depth=16 (UBSan shift) and +// MergeARGB16To8Plane depth=8 saturation on out-of-range >255 samples. +TEST_F(LibYUVPlanarTest, MergeAR64Depth16AndMergeARGB16To8Depth8Clamp) { + const int kWidth = 37; + const int kHeight = 2; + align_buffer_page_end_16(src_r, kWidth * kHeight); + align_buffer_page_end_16(src_g, kWidth * kHeight); + align_buffer_page_end_16(src_b, kWidth * kHeight); + align_buffer_page_end_16(src_a, kWidth * kHeight); + align_buffer_page_end_16(ar64_c, kWidth * 4 * kHeight); + align_buffer_page_end_16(ar64_opt, kWidth * 4 * kHeight); + align_buffer_page_end(argb_c, kWidth * 4 * kHeight); + align_buffer_page_end(argb_opt, kWidth * 4 * kHeight); + for (int i = 0; i < kWidth * kHeight; ++i) { + src_r[i] = static_cast(200u + (i * 19u)); + src_g[i] = static_cast(100u + (i * 37u)); + src_b[i] = static_cast(50u + (i * 53u)); + src_a[i] = static_cast(0x8000u + (i * 101u)); + } + + MaskCpuFlags(disable_cpu_flags_); + MergeAR64Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a, kWidth, + ar64_c, kWidth * 4, kWidth, kHeight, 16); + MergeARGB16To8Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a, + kWidth, argb_c, kWidth * 4, kWidth, kHeight, 8); + + MaskCpuFlags(benchmark_cpu_info_); + MergeAR64Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a, kWidth, + ar64_opt, kWidth * 4, kWidth, kHeight, 16); + MergeARGB16To8Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a, + kWidth, argb_opt, kWidth * 4, kWidth, kHeight, 8); + + for (int i = 0; i < kWidth * 4 * kHeight; ++i) { + ASSERT_EQ(ar64_c[i], ar64_opt[i]); + ASSERT_EQ(argb_c[i], argb_opt[i]); + } + + free_aligned_buffer_page_end_16(src_r); + free_aligned_buffer_page_end_16(src_g); + free_aligned_buffer_page_end_16(src_b); + free_aligned_buffer_page_end_16(src_a); + free_aligned_buffer_page_end_16(ar64_c); + free_aligned_buffer_page_end_16(ar64_opt); + free_aligned_buffer_page_end(argb_c); + free_aligned_buffer_page_end(argb_opt); +} + } // namespace libyuv diff --git a/util/psnr_main.cc b/util/psnr_main.cc index a11cd3fc4..45261805c 100644 --- a/util/psnr_main.cc +++ b/util/psnr_main.cc @@ -329,9 +329,12 @@ int main(int argc, const char* argv[]) { exit(1); } + if (num_rec <= 0) { + fclose(file_org); + exit(1); + } // Open all files to compare to - FILE** file_rec = new FILE*[num_rec]; - memset(file_rec, 0, num_rec * sizeof(FILE*)); // NOLINT + FILE** file_rec = new FILE*[num_rec](); for (int cur_rec = 0; cur_rec < num_rec; ++cur_rec) { file_rec[cur_rec] = fopen(argv[fileindex_rec + cur_rec], "rb"); if (file_rec[cur_rec] == NULL) {