From d088b8f2bd8f29badf5ba015a24d6c38dc69f23a Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Mon, 2 Nov 2026 10:21:09 -0800 Subject: [PATCH] [libyuv] Add GCC 15 SME support, SVE2/SME ports, and next 10 SIMD fixes 1. Enable ARMv9 SVE2 and SME compilation with GCC 15 and Clang across Android.bp, BUILD.gn, CMakeLists.txt, and linux.mk using portable LIBYUV_SVE_ATTR, LIBYUV_SME_ATTR, and STREAMING_COMPATIBLE macros. 2. Port 46 SVE2 kernels to streaming-compatible SVE_SC functions in row_sve.h and expose _SME entry points for processors with SME but no non-streaming SVE2. Upgrade 15 1x SVE2 kernels to 2x unrolled, use full-vector st1b in ARGBToRGB24Row/ARGBToRAWRow_SVE2, contiguous ld1h/st1h in HalfFloatRow_SVE_SC, and frecpe+frecps reciprocal in ARGBAlphaBlendRow_SVE_SC. 3. Add missing kCpuHasNEON runtime checks in planar_test.cc and define missing HAS_*_NEON macros in row.h for existing NEON kernels. 4. Handle odd width directly in AYUVToUVRow_SVE_SC and AYUVToVURow_SVE_SC and remove redundant width alignment guards in AYUVToNV12 and AYUVToNV21. 5. Implement TransposeWx8_NEON for AArch64 in rotate_neon64.cc so TransposePlane and RotatePlane90/270 cascade 16x -> 8x -> C tail. 6. Add _Any_ wrappers for 9 ARGB/float filter functions (ARGBGrayTo, ARGBGray, ARGBSepia, ARGBColorMatrix, ARGBQuantize, ARGBShade, GaussRow_F32, GaussCol_F32, and ARGBLumaColorTable). 7. Add HalfRow and HalfRow_16 wrappers for AVX2, NEON, SVE2, and SME delegating to InterpolateRow{,_16} with fraction 0x80. 8. Add YUY2ToUV422Row_AVX512BW and UYVYToUV422Row_AVX512BW in row_gcc.cc and row_win.cc. 9. Add HalfWidthRow_16To8_Odd_NEON in row_any.cc and wire into I41xToI420 and Px10ToI420 for odd chroma widths. 10. Fix inline asm clobbers and vzeroupper placement in TransposeWx32_16_AVX512BW and TransposeWx16_32_AVX512BW, and re-enable 36 passing unit tests. Test: libyuv_unittest Bug: 579659306 Change-Id: I6b808f8b03ed424d3af4bb7941c34af05c1cb774 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8701301 Commit-Queue: Frank Barchard Reviewed-by: richard winterton --- Android.bp | 5 + BUILD.gn | 8 +- CMakeLists.txt | 9 +- include/libyuv/compare_row.h | 4 +- include/libyuv/cpu_support.h | 21 +- include/libyuv/rotate_row.h | 7 +- include/libyuv/row.h | 192 ++- include/libyuv/row_sve.h | 2751 ++++++++++++++++++++++++++------ include/libyuv/scale_row.h | 4 +- linux.mk | 5 + source/compare_neon64.cc | 14 + source/convert.cc | 47 +- source/convert_argb.cc | 68 + source/convert_from_argb.cc | 20 + source/convert_to_argb.cc | 5 + source/convert_to_i420.cc | 10 + source/planar_functions.cc | 237 ++- source/rotate.cc | 4 - source/rotate_gcc.cc | 94 +- source/rotate_neon64.cc | 71 + source/rotate_sme.cc | 28 +- source/rotate_sve.cc | 16 +- source/row_any.cc | 180 ++- source/row_gcc.cc | 98 ++ source/row_neon64.cc | 14 + source/row_sme.cc | 514 ++---- source/row_sve.cc | 1181 +------------- source/row_win.cc | 56 + source/scale_sme.cc | 46 +- unit_test/convert_argb_test.cc | 10 +- unit_test/planar_test.cc | 120 +- unit_test/rotate_argb_test.cc | 8 +- unit_test/rotate_test.cc | 24 +- unit_test/scale_plane_test.cc | 16 +- 34 files changed, 3667 insertions(+), 2220 deletions(-) diff --git a/Android.bp b/Android.bp index aa953dade..186a1cbab 100644 --- a/Android.bp +++ b/Android.bp @@ -57,6 +57,8 @@ cc_library { "source/rotate_neon.cc", "source/rotate_neon64.cc", "source/rotate_rvv.cc", + "source/rotate_sme.cc", + "source/rotate_sve.cc", "source/row_any.cc", "source/row_common.cc", "source/row_gcc.cc", @@ -64,6 +66,8 @@ cc_library { "source/row_neon.cc", "source/row_neon64.cc", "source/row_rvv.cc", + "source/row_sme.cc", + "source/row_sve.cc", "source/scale.cc", "source/scale_any.cc", "source/scale_argb.cc", @@ -74,6 +78,7 @@ cc_library { "source/scale_neon64.cc", "source/scale_rgb.cc", "source/scale_rvv.cc", + "source/scale_sme.cc", "source/scale_uv.cc", "source/video_common.cc", ], diff --git a/BUILD.gn b/BUILD.gn index 3c37e0591..3735aa338 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -288,8 +288,12 @@ if (libyuv_use_sme) { use_libcxx_modules = false # SME is an Armv9-A feature, but use Armv8.5-A for compatibility with - # processors like Apple M4/M5 that support SME without SVE. - cflags = [ "-march=armv8.5-a+i8mm+sme" ] + # processors like Apple M4/M5/M6 that support SME without SVE. + if (is_clang) { + cflags = [ "-march=armv8.5-a+i8mm+sme" ] + } else { + cflags = [ "-march=armv8.5-a+i8mm+sve2+sme" ] + } } } diff --git a/CMakeLists.txt b/CMakeLists.txt index 83211a2f2..9c84dc1b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,7 +128,12 @@ if(NOT MSVC) # with Clang for Windows. set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) set(OLD_CMAKE_TRY_COMPILE_TARGET_TYPE ${CMAKE_TRY_COMPILE_TARGET_TYPE}) - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -march=armv8.5-a+i8mm+sme") + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(SME_COMPILER_FLAGS "-march=armv8.5-a+i8mm+sve2+sme") + else() + set(SME_COMPILER_FLAGS "-march=armv8.5-a+i8mm+sme") + endif() + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${SME_COMPILER_FLAGS}") set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) check_c_source_compiles(" __arm_locally_streaming void func(void) { } @@ -143,7 +148,7 @@ int main(void) { return 0; } ${ly_src_dir}/rotate_sme.cc ${ly_src_dir}/row_sme.cc ${ly_src_dir}/scale_sme.cc) - target_compile_options(${ly_lib_name}_sme PRIVATE -march=armv8.5-a+i8mm+sme) + target_compile_options(${ly_lib_name}_sme PRIVATE ${SME_COMPILER_FLAGS}) list(APPEND ly_lib_parts $) else() add_definitions(-DLIBYUV_DISABLE_SME) diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index cb5cbc827..a3c529128 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -112,8 +112,8 @@ extern "C" { #endif // The following are available for AArch64 SME: -#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ - defined(__aarch64__) +#if !defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__) #define HAS_SSIM4X4ROW_SME #define HAS_SSIMWINDOWROW_SME #endif diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index 1e3427808..d35276f9d 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -150,11 +150,24 @@ extern "C" { #undef LIBYUV_DISABLE_SME #endif -// Clang 18+ required for SME due to needing __arm_tpidr2_save from compiler-rt, -// only enabled on Linux and Android (both define __linux__) for now. -#if !defined(LIBYUV_DISABLE_SME) && defined(__aarch64__) && \ - defined(__linux__) && defined(__clang__) && (__clang_major__ >= 18) +// clang >= 11.0.0 or gcc >= 10.0.0 required for SVE2. +#if defined(__aarch64__) && !defined(LIBYUV_DISABLE_SVE) +#if (defined(__clang__) && (__clang_major__ < 11)) || \ + (!defined(__clang__) && defined(__GNUC__) && (__GNUC__ < 10)) +#define LIBYUV_DISABLE_SVE +#endif +#endif + +// Clang 18+ on Linux/Android, Apple Clang 17+ on macOS, or GCC 15+ on Linux +// required for SME (__arm_tpidr2_save from compiler-rt or libgcc). +#if !defined(LIBYUV_DISABLE_SME) && defined(__aarch64__) +#if defined(__clang__) && ((defined(__linux__) && (__clang_major__ >= 18)) || \ + (defined(__APPLE__) && (__clang_major__ >= 17))) #define CLANG_HAS_SME 1 +#elif defined(__linux__) && !defined(__clang__) && defined(__GNUC__) && \ + (__GNUC__ >= 15) +#define GCC_HAS_SME 1 +#endif #endif #ifdef __cplusplus diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index 31d78e610..fa75aa314 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -92,9 +92,8 @@ extern "C" { (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) #if defined(__aarch64__) #define HAS_TRANSPOSEWX16_NEON -#else -#define HAS_TRANSPOSEWX8_NEON #endif +#define HAS_TRANSPOSEWX8_NEON #define HAS_TRANSPOSEUVWX8_NEON #define HAS_TRANSPOSEWX8_16_NEON #define HAS_TRANSPOSEWX4_32_NEON @@ -106,8 +105,8 @@ extern "C" { #define HAS_TRANSPOSEWX4_32_SVE2 #endif -#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ - defined(__aarch64__) +#if !defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__) #define HAS_TRANSPOSEWXH_SME #define HAS_TRANSPOSEUVWXH_SME #define HAS_TRANSPOSEWXH_16_SME diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 67eb20cef..9f73b9be2 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -554,9 +554,11 @@ extern "C" { #define HAS_P410TOAR30ROW_AVX512BW #define HAS_P410TOARGBROW_AVX512BW #define HAS_UYVYTOARGBROW_AVX512BW +#define HAS_UYVYTOUV422ROW_AVX512BW #define HAS_UYVYTOUVROW_AVX512BW #define HAS_UYVYTOYROW_AVX512BW #define HAS_YUY2TOARGBROW_AVX512BW +#define HAS_YUY2TOUV422ROW_AVX512BW #define HAS_YUY2TOUVROW_AVX512BW #define HAS_YUY2TOYROW_AVX512BW #define HAS_ARGBTORAWROW_AVX512BW @@ -685,9 +687,11 @@ extern "C" { #define HAS_P410TOAR30ROW_AVX512BW #define HAS_P410TOARGBROW_AVX512BW #define HAS_UYVYTOARGBROW_AVX512BW +#define HAS_UYVYTOUV422ROW_AVX512BW #define HAS_UYVYTOUVROW_AVX512BW #define HAS_UYVYTOYROW_AVX512BW #define HAS_YUY2TOARGBROW_AVX512BW +#define HAS_YUY2TOUV422ROW_AVX512BW #define HAS_YUY2TOUVROW_AVX512BW #define HAS_YUY2TOYROW_AVX512BW #endif @@ -892,6 +896,8 @@ extern "C" { #define HAS_DIVIDEROW_16_SVE2 #define HAS_HALFFLOATROW_SVE2 #define HAS_HALFMERGEUVROW_SVE2 +#define HAS_HALFROW_16_SVE2 +#define HAS_HALFROW_SVE2 // Disabled: NEON version is faster for 128 bit vectors. // #define HAS_HALFROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors. @@ -916,6 +922,7 @@ extern "C" { #define HAS_I444ALPHATOARGBROW_SVE2 #define HAS_I444TOARGBROW_SVE2 #define HAS_I444TORGB24ROW_SVE2 +#define HAS_INTERPOLATEROW_16_SVE2 #define HAS_INTERPOLATEROW_SVE2 #define HAS_MERGEUVROW_SVE2 #define HAS_NV12TOARGBROW_SVE2 @@ -940,18 +947,30 @@ extern "C" { #endif // The following are available on AArch64 SME platforms: -#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ - defined(__aarch64__) +#if !defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__) +#define HAS_ARGB1555TOARGBROW_SME #define HAS_ARGBALPHABLENDROW_SME #define HAS_ARGBMULTIPLYROW_SME #define HAS_ARGBORROW_SME +#define HAS_ARGBTORAWROW_SME +#define HAS_ARGBTORGB24ROW_SME +#define HAS_ARGBTORGB565DITHERROW_SME +#define HAS_ARGBTORGB565ROW_SME #define HAS_ARGBTOUVMATRIXROW_SME #define HAS_ARGBTOYMATRIXROW_SME +#define HAS_AYUVTOUVROW_SME +#define HAS_AYUVTOVUROW_SME #define HAS_CONVERT16TO8ROW_SME #define HAS_CONVERT8TO16ROW_SME #define HAS_CONVERT8TO8ROW_SME #define HAS_COPYROW_SME +#define HAS_DIVIDEROW_16_SME +#define HAS_HALFFLOATROW_SME +#define HAS_HALFMERGEUVROW_SME +#define HAS_HALFROW_16_SME #define HAS_HALFROW_16TO8_SME +#define HAS_HALFROW_SME #define HAS_HALFWIDTHROW_16TO8_SME #define HAS_I210ALPHATOARGBROW_SME #define HAS_I210TOAR30ROW_SME @@ -986,14 +1005,25 @@ extern "C" { #define HAS_P210TOARGBROW_SME #define HAS_P410TOAR30ROW_SME #define HAS_P410TOARGBROW_SME +#define HAS_RAWTOARGBROW_SME +#define HAS_RAWTORGB24ROW_SME +#define HAS_RAWTORGBAROW_SME +#define HAS_RGB24TOARGBROW_SME +#define HAS_SPLITUVROW_SME #define HAS_UYVYTOARGBROW_SME +#define HAS_UYVYTOUVROW_SME +#define HAS_UYVYTOYROW_SME #define HAS_YUY2TOARGBROW_SME +#define HAS_YUY2TOUVROW_SME +#define HAS_YUY2TOYROW_SME #endif // The following are available on AArch64 platforms: #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #define HAS_GAUSSCOL_F32_NEON #define HAS_GAUSSROW_F32_NEON +#define HAS_SCALEMAXSAMPLES_NEON +#define HAS_SCALESAMPLES_NEON #define HAS_SCALESUMSAMPLES_NEON #endif @@ -3110,6 +3140,10 @@ void SplitUVRow_SVE2(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width); +void SplitUVRow_SME(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void SplitUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, @@ -3321,6 +3355,12 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u, int src_stride_v, uint8_t* dst_uv, int width); +void HalfMergeUVRow_SME(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); void HalfMergeUVRow_SSSE3(const uint8_t* src_u, int src_stride_u, @@ -3943,6 +3983,10 @@ void DivideRow_16_SVE2(const uint16_t* src_y, uint16_t* dst_y, int scale, int width); +void DivideRow_16_SME(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); void DivideRow_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int scale, @@ -4046,6 +4090,14 @@ void HalfRow_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int width); +void HalfRow_SME(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); +void HalfRow_SVE2(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); void HalfRow_16_C(const uint16_t* src_uv, ptrdiff_t src_uv_stride, @@ -4059,6 +4111,14 @@ void HalfRow_16_Any_AVX2(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst_ptr, int width); +void HalfRow_16_SME(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); +void HalfRow_16_SVE2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); void HalfRow_16To8_C(const uint16_t* src_uv, ptrdiff_t src_uv_stride, @@ -4131,6 +4191,11 @@ void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv, uint8_t* dst_uv, int scale, int width); +void HalfWidthRow_16To8_Odd_NEON(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, @@ -4441,6 +4506,7 @@ void RGB24ToARGBRow_NEON(const uint8_t* src_rgb24, void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24, uint8_t* dst_argb, int width); +void RGB24ToARGBRow_SME(const uint8_t* src_rgb24, uint8_t* dst_argb, int width); void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24, uint8_t* dst_argb, int width); void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24, uint8_t* dst_argb, @@ -4448,14 +4514,17 @@ void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24, void RGB24ToARGBRow_RVV(const uint8_t* src_rgb24, uint8_t* dst_argb, int width); void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width); void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width); +void RAWToARGBRow_SME(const uint8_t* src_raw, uint8_t* dst_argb, int width); void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width); +void RAWToRGBARow_SME(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width); void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width); void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width); void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); +void RAWToRGB24Row_SME(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); @@ -4477,6 +4546,9 @@ void ARGB1555ToARGBRow_NEON(const uint8_t* src_argb1555, void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555, uint8_t* dst_argb, int width); +void ARGB1555ToARGBRow_SME(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width); void ARGB1555ToARGBRow_LSX(const uint8_t* src_argb1555, uint8_t* dst_argb, int width); @@ -4677,14 +4749,19 @@ void ARGBToRGB24Row_NEON(const uint8_t* src_argb, void ARGBToRGB24Row_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb24, int width); +void ARGBToRGB24Row_SME(const uint8_t* src_argb, uint8_t* dst_rgb24, int width); void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width); void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_raw, int width); +void ARGBToRAWRow_SME(const uint8_t* src_argb, uint8_t* dst_raw, int width); void ARGBToRGB565Row_NEON(const uint8_t* src_argb, uint8_t* dst_rgb565, int width); void ARGBToRGB565Row_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb565, int width); +void ARGBToRGB565Row_SME(const uint8_t* src_argb, + uint8_t* dst_rgb565, + int width); void ARGBToARGB1555Row_NEON(const uint8_t* src_argb, uint8_t* dst_argb1555, int width); @@ -4699,6 +4776,10 @@ void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, uint32_t dither4, int width); +void ARGBToRGB565DitherRow_SME(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width); void ARGBToRGB565DitherRow_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, uint32_t dither4, @@ -6853,6 +6934,7 @@ void YUY2ToUV422Row_SSE2(const uint8_t* src_yuy2, int width); void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width); void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width); +void YUY2ToYRow_SME(const uint8_t* src_yuy2, uint8_t* dst_y, int width); void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width); void YUY2ToUVRow_NEON(const uint8_t* src_yuy2, int stride_yuy2, @@ -6864,23 +6946,42 @@ void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_u, uint8_t* dst_v, int width); +void YUY2ToUVRow_SME(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_u, uint8_t* dst_v, int width); +void YUY2ToUV422Row_AVX512BW(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width); +void UYVYToYRow_SME(const uint8_t* src_uyvy, uint8_t* dst_y, int width); void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width); void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, int stride_uyvy, uint8_t* dst_u, uint8_t* dst_v, int width); +void UYVYToUVRow_SME(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, int stride_uyvy, uint8_t* dst_u, uint8_t* dst_v, int width); +void UYVYToUV422Row_AVX512BW(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_uv, @@ -7151,6 +7252,10 @@ void AYUVToUVRow_SVE2(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_uv, int width); +void AYUVToUVRow_SME(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); void AYUVToVURow_NEON(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, @@ -7159,6 +7264,10 @@ void AYUVToVURow_SVE2(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, int width); +void AYUVToVURow_SME(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); void AYUVToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void AYUVToUVRow_Any_AVX2(const uint8_t* src_ptr, int src_stride, @@ -7173,18 +7282,10 @@ void AYUVToUVRow_Any_NEON(const uint8_t* src_ptr, int src_stride, uint8_t* dst_vu, int width); -void AYUVToUVRow_Any_SVE2(const uint8_t* src_ptr, - int src_stride, - uint8_t* dst_vu, - int width); void AYUVToVURow_Any_NEON(const uint8_t* src_ptr, int src_stride, uint8_t* dst_vu, int width); -void AYUVToVURow_Any_SVE2(const uint8_t* src_ptr, - int src_stride, - uint8_t* dst_vu, - int width); void I422ToYUY2Row_C(const uint8_t* src_y, const uint8_t* src_u, @@ -7374,6 +7475,12 @@ void ARGBGrayRow_NEON_DotProd(const uint8_t* src_argb, int width); void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width); +void ARGBGrayRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBGrayRow_Any_NEON_DotProd(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBGrayRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBGrayRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void ARGBSepiaRow_C(uint8_t* dst_argb, int width); void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width); @@ -7382,6 +7489,10 @@ void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width); void ARGBSepiaRow_NEON_DotProd(uint8_t* dst_argb, int width); void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width); void ARGBSepiaRow_LASX(uint8_t* dst_argb, int width); +void ARGBSepiaRow_Any_NEON(uint8_t* dst_ptr, int width); +void ARGBSepiaRow_Any_NEON_DotProd(uint8_t* dst_ptr, int width); +void ARGBSepiaRow_Any_LSX(uint8_t* dst_ptr, int width); +void ARGBSepiaRow_Any_LASX(uint8_t* dst_ptr, int width); void ARGBColorMatrixRow_C(const uint8_t* src_argb, uint8_t* dst_argb, @@ -7411,6 +7522,22 @@ void ARGBColorMatrixRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, const int8_t* matrix_argb, int width); +void ARGBColorMatrixRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const int8_t* matrix_argb, + int width); +void ARGBColorMatrixRow_Any_NEON_I8MM(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const int8_t* matrix_argb, + int width); +void ARGBColorMatrixRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const int8_t* matrix_argb, + int width); +void ARGBColorMatrixRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const int8_t* matrix_argb, + int width); void ARGBColorTableRow_C(uint8_t* dst_argb, const uint8_t* table_argb, @@ -7457,6 +7584,21 @@ void ARGBQuantizeRow_LASX(uint8_t* dst_argb, int interval_size, int interval_offset, int width); +void ARGBQuantizeRow_Any_NEON(uint8_t* dst_ptr, + int scale, + int interval_size, + int interval_offset, + int width); +void ARGBQuantizeRow_Any_LSX(uint8_t* dst_ptr, + int scale, + int interval_size, + int interval_offset, + int width); +void ARGBQuantizeRow_Any_LASX(uint8_t* dst_ptr, + int scale, + int interval_size, + int interval_offset, + int width); void ARGBShadeRow_C(const uint8_t* src_argb, uint8_t* dst_argb, @@ -7478,6 +7620,18 @@ void ARGBShadeRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width, uint32_t value); +void ARGBShadeRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width, + uint32_t value); +void ARGBShadeRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width, + uint32_t value); +void ARGBShadeRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width, + uint32_t value); // Used for blur. void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, @@ -7633,6 +7787,11 @@ void InterpolateRow_16_SME(uint16_t* dst_ptr, ptrdiff_t src_stride, int width, int source_y_fraction); +void InterpolateRow_16_SVE2(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); // Sobel images. void SobelXRow_C(const uint8_t* src_y0, @@ -7846,6 +8005,14 @@ void HalfFloat1Row_SVE2(const uint16_t* src, uint16_t* dst, float scale, int width); +void HalfFloatRow_SME(const uint16_t* src, + uint16_t* dst, + float scale, + int width); +void HalfFloat1Row_SME(const uint16_t* src, + uint16_t* dst, + float scale, + int width); void HalfFloatRow_LSX(const uint16_t* src, uint16_t* dst, float scale, @@ -7907,6 +8074,11 @@ void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb, int width, const uint8_t* luma, uint32_t lumacoeff); +void ARGBLumaColorTableRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width, + const uint8_t* luma, + uint32_t lumacoeff); float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width); float ScaleMaxSamples_AVX2(const float* src, diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 284b37007..942d0f4ae 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -18,16 +18,17 @@ namespace libyuv { extern "C" { #endif -#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) - -#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ +#if (!defined(LIBYUV_DISABLE_SVE) || \ + (!defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)))) && \ defined(__aarch64__) + +#if !defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__) #define STREAMING_COMPATIBLE __arm_streaming_compatible -#else // defined(LIBYUV_DISABLE_SME) || !defined(CLANG_HAS_SME) || - // !defined(__aarch64__) +#else #define STREAMING_COMPATIBLE -#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && - // defined(__aarch64__) +#endif #define YUVTORGB_SVE_SETUP \ "ld1rb {z28.b}, p0/z, [%[kUVCoeff], #0] \n" \ @@ -57,9 +58,27 @@ extern "C" { "trn1 z0.b, z0.b, z0.b \n" \ "prfm pldl1keep, [%[src_v], 448] \n" +// Read twice as much data from YUV, putting the first half of the Y data in +// z0.h and the second half in z1.h. +#define READYUV444_SVE_2X \ + "ld1b {z0.h}, p1/z, [%[src_y]] \n" \ + "ld1b {z1.h}, p2/z, [%[src_y], #1, mul vl] \n" \ + "ld1b {z2.h}, p1/z, [%[src_u]] \n" \ + "ld1b {z3.h}, p2/z, [%[src_u], #1, mul vl] \n" \ + "ld1b {z4.h}, p1/z, [%[src_v]] \n" \ + "ld1b {z5.h}, p2/z, [%[src_v], #1, mul vl] \n" \ + "incb %[src_y] \n" \ + "incb %[src_u] \n" \ + "incb %[src_v] \n" \ + "prfm pldl1keep, [%[src_y], 448] \n" \ + "prfm pldl1keep, [%[src_u], 128] \n" \ + "prfm pldl1keep, [%[src_v], 128] \n" \ + "trn1 z0.b, z0.b, z0.b \n" \ + "trn1 z1.b, z1.b, z1.b \n" + // Read twice as much data from YUV, putting the even elements from the Y data // in z0.h and odd elements in z1.h. -#define READYUV444_SVE_2X \ +#define READYUV444_RGB24_SVE_2X \ "ld1b {z0.b}, p1/z, [%[src_y]] \n" \ "ld1b {z2.b}, p1/z, [%[src_u]] \n" \ "ld1b {z3.b}, p1/z, [%[src_v]] \n" \ @@ -78,6 +97,14 @@ extern "C" { "prfm pldl1keep, [%[src_y], 448] \n" \ "trn1 z0.b, z0.b, z0.b \n" +#define READYUV400_SVE_2X \ + "ld1b {z0.h}, p1/z, [%[src_y]] \n" \ + "ld1b {z1.h}, p2/z, [%[src_y], #1, mul vl] \n" \ + "incb %[src_y] \n" \ + "prfm pldl1keep, [%[src_y], 448] \n" \ + "trn1 z0.b, z0.b, z0.b \n" \ + "trn1 z1.b, z1.b, z1.b \n" + #define READYUV422_SVE \ "ld1b {z0.h}, p1/z, [%[src_y]] \n" \ "ld1b {z1.s}, p1/z, [%[src_u]] \n" \ @@ -92,10 +119,33 @@ extern "C" { "trn1 z1.h, z1.h, z1.h \n" \ "trn1 z2.h, z2.h, z2.h \n" +// Read twice as much data from YUV, putting the first half of the Y data in +// z0.h and the second half in z1.h. U/V data is stored in z2.h/z3.h and +// z4.h/z5.h. +#define READYUV422_SVE_2X \ + "ld1b {z0.h}, p1/z, [%[src_y]] \n" \ + "ld1b {z1.h}, p2/z, [%[src_y], #1, mul vl] \n" \ + "ld1b {z2.s}, p1/z, [%[src_u]] \n" \ + "ld1b {z3.s}, p2/z, [%[src_u], #1, mul vl] \n" \ + "ld1b {z4.s}, p1/z, [%[src_v]] \n" \ + "ld1b {z5.s}, p2/z, [%[src_v], #1, mul vl] \n" \ + "incb %[src_y] \n" \ + "inch %[src_u] \n" \ + "inch %[src_v] \n" \ + "prfm pldl1keep, [%[src_y], 448] \n" \ + "prfm pldl1keep, [%[src_u], 128] \n" \ + "prfm pldl1keep, [%[src_v], 128] \n" \ + "trn1 z0.b, z0.b, z0.b \n" \ + "trn1 z1.b, z1.b, z1.b \n" \ + "trn1 z2.h, z2.h, z2.h \n" \ + "trn1 z3.h, z3.h, z3.h \n" \ + "trn1 z4.h, z4.h, z4.h \n" \ + "trn1 z5.h, z5.h, z5.h \n" + // Read twice as much data from YUV, putting the even elements from the Y data // in z0.h and odd elements in z1.h. U/V data is not duplicated, stored in // z2.h/z3.h. -#define READYUV422_SVE_2X \ +#define READYUV422_RGB24_SVE_2X \ "ld1b {z0.b}, p1/z, [%[src_y]] \n" \ "ld1b {z2.h}, p1/z, [%[src_u]] \n" \ "ld1b {z3.h}, p1/z, [%[src_v]] \n" \ @@ -126,24 +176,30 @@ extern "C" { "uqshrnb z2.b, z2.h, #2 \n" #define READI210_SVE_2X \ - "ld1h {z4.h}, p2/z, [%[src_y]] \n" \ - "ld1h {z5.h}, p3/z, [%[src_y], #1, mul vl] \n" \ - "ld1h {z2.h}, p1/z, [%[src_u]] \n" \ - "ld1h {z3.h}, p1/z, [%[src_v]] \n" \ + "ld1h {z6.h}, p1/z, [%[src_y]] \n" \ + "ld1h {z7.h}, p2/z, [%[src_y], #1, mul vl] \n" \ + "ld1h {z2.s}, p1/z, [%[src_u]] \n" \ + "ld1h {z3.s}, p2/z, [%[src_u], #1, mul vl] \n" \ + "ld1h {z4.s}, p1/z, [%[src_v]] \n" \ + "ld1h {z5.s}, p2/z, [%[src_v], #1, mul vl] \n" \ "incb %[src_y], all, mul #2 \n" \ - "uzp1 z6.h, z4.h, z5.h \n" \ - "uzp2 z5.h, z4.h, z5.h \n" \ "incb %[src_u] \n" \ "incb %[src_v] \n" \ "lsl z0.h, z6.h, #6 \n" \ - "lsl z1.h, z5.h, #6 \n" \ + "lsl z1.h, z7.h, #6 \n" \ + "trn1 z2.h, z2.h, z2.h \n" \ + "trn1 z3.h, z3.h, z3.h \n" \ + "trn1 z4.h, z4.h, z4.h \n" \ + "trn1 z5.h, z5.h, z5.h \n" \ "prfm pldl1keep, [%[src_y], 448] \n" \ "prfm pldl1keep, [%[src_u], 128] \n" \ "prfm pldl1keep, [%[src_v], 128] \n" \ "usra z0.h, z6.h, #4 \n" \ - "usra z1.h, z5.h, #4 \n" \ + "usra z1.h, z7.h, #4 \n" \ "uqshrnb z2.b, z2.h, #2 \n" \ - "uqshrnb z3.b, z3.h, #2 \n" + "uqshrnb z3.b, z3.h, #2 \n" \ + "uqshrnb z4.b, z4.h, #2 \n" \ + "uqshrnb z5.b, z5.h, #2 \n" #define READP210_SVE \ "ld1h {z0.h}, p1/z, [%[src_y]] \n" \ @@ -154,6 +210,20 @@ extern "C" { "prfm pldl1keep, [%[src_uv], 256] \n" \ "tbl z1.b, {z1.b}, z22.b \n" +#define READP210_SVE_2X \ + "ld1h {z0.h}, p1/z, [%[src_y]] \n" \ + "ld1h {z1.h}, p2/z, [%[src_y], #1, mul vl] \n" \ + "ld1w {z2.s}, p1/z, [%[src_uv]] \n" \ + "ld1w {z3.s}, p2/z, [%[src_uv], #1, mul vl] \n" \ + "incb %[src_y], all, mul #2 \n" \ + "incb %[src_uv], all, mul #2 \n" \ + "prfm pldl1keep, [%[src_y], 448] \n" \ + "prfm pldl1keep, [%[src_uv], 256] \n" \ + "trn2 z4.h, z2.h, z2.h \n" \ + "trn1 z2.h, z2.h, z2.h \n" \ + "trn2 z5.h, z3.h, z3.h \n" \ + "trn1 z3.h, z3.h, z3.h \n" + #define READI410_SVE \ "ld1h {z3.h}, p1/z, [%[src_y]] \n" \ "lsl z0.h, z3.h, #6 \n" \ @@ -169,6 +239,35 @@ extern "C" { "uqshrnb z1.b, z1.h, #2 \n" \ "uqshrnb z2.b, z2.h, #2 \n" +#define READI410_SVE_2X \ + "ld1h {z4.h}, p1/z, [%[src_y]] \n" \ + "ld1h {z5.h}, p2/z, [%[src_y], #1, mul vl] \n" \ + "incb %[src_y], all, mul #2 \n" \ + "lsl z0.h, z4.h, #6 \n" \ + "lsl z1.h, z5.h, #6 \n" \ + "usra z0.h, z4.h, #4 \n" \ + "usra z1.h, z5.h, #4 \n" \ + "ld1h {z2.h}, p1/z, [%[src_u]] \n" \ + "ld1h {z3.h}, p2/z, [%[src_u], #1, mul vl] \n" \ + "incb %[src_u], all, mul #2 \n" \ + "uqshrnb z2.b, z2.h, #2 \n" \ + "uqshrnb z3.b, z3.h, #2 \n" \ + "ld1h {z4.h}, p1/z, [%[src_v]] \n" \ + "ld1h {z5.h}, p2/z, [%[src_v], #1, mul vl] \n" \ + "incb %[src_v], all, mul #2 \n" \ + "prfm pldl1keep, [%[src_y], 448] \n" \ + "prfm pldl1keep, [%[src_u], 128] \n" \ + "prfm pldl1keep, [%[src_v], 128] \n" \ + "uqshrnb z4.b, z4.h, #2 \n" \ + "uqshrnb z5.b, z5.h, #2 \n" + +#define READA10_SVE_2X \ + "ld1h {z4.h}, p1/z, [%[src_a]] \n" \ + "ld1h {z5.h}, p2/z, [%[src_a], #1, mul vl] \n" \ + "incb %[src_a], all, mul #2 \n" \ + "uqshrnb z19.b, z4.h, #2 \n" \ + "uqshrnb z23.b, z5.h, #2 \n" + // We need different predicates for the UV components since we are reading // 32-bit (pairs of UV) elements rather than 16-bit Y elements. #define READP410_SVE \ @@ -198,6 +297,32 @@ extern "C" { "uqshrnb z1.b, z1.h, #4 \n" \ "uqshrnb z2.b, z2.h, #4 \n" +#define READI212_SVE_2X \ + "ld1h {z6.h}, p1/z, [%[src_y]] \n" \ + "ld1h {z7.h}, p2/z, [%[src_y], #1, mul vl] \n" \ + "ld1h {z2.s}, p1/z, [%[src_u]] \n" \ + "ld1h {z3.s}, p2/z, [%[src_u], #1, mul vl] \n" \ + "ld1h {z4.s}, p1/z, [%[src_v]] \n" \ + "ld1h {z5.s}, p2/z, [%[src_v], #1, mul vl] \n" \ + "incb %[src_y], all, mul #2 \n" \ + "incb %[src_u] \n" \ + "incb %[src_v] \n" \ + "lsl z0.h, z6.h, #4 \n" \ + "lsl z1.h, z7.h, #4 \n" \ + "trn1 z2.h, z2.h, z2.h \n" \ + "trn1 z3.h, z3.h, z3.h \n" \ + "trn1 z4.h, z4.h, z4.h \n" \ + "trn1 z5.h, z5.h, z5.h \n" \ + "prfm pldl1keep, [%[src_y], 448] \n" \ + "prfm pldl1keep, [%[src_u], 128] \n" \ + "prfm pldl1keep, [%[src_v], 128] \n" \ + "usra z0.h, z6.h, #8 \n" \ + "usra z1.h, z7.h, #8 \n" \ + "uqshrnb z2.b, z2.h, #4 \n" \ + "uqshrnb z3.b, z3.h, #4 \n" \ + "uqshrnb z4.b, z4.h, #4 \n" \ + "uqshrnb z5.b, z5.h, #4 \n" + #define I400TORGB_SVE \ "umulh z18.h, z24.h, z0.h \n" /* Y */ \ "movprfx z16, z18 \n" \ @@ -206,10 +331,38 @@ extern "C" { "usqadd z17.h, p0/m, z17.h, z6.h \n" /* G */ \ "usqadd z18.h, p0/m, z18.h, z5.h \n" /* R */ +#define I400TORGB_SVE_2X \ + "umulh z18.h, z24.h, z0.h \n" /* Y0 */ \ + "umulh z22.h, z24.h, z1.h \n" /* Y1 */ \ + "movprfx z16, z18 \n" \ + "usqadd z16.h, p0/m, z16.h, z4.h \n" /* B0 */ \ + "movprfx z20, z22 \n" \ + "usqadd z20.h, p0/m, z20.h, z4.h \n" /* B1 */ \ + "movprfx z17, z18 \n" \ + "usqadd z17.h, p0/m, z17.h, z6.h \n" /* G0 */ \ + "movprfx z21, z22 \n" \ + "usqadd z21.h, p0/m, z21.h, z6.h \n" /* G1 */ \ + "usqadd z18.h, p0/m, z18.h, z5.h \n" /* R0 */ \ + "usqadd z22.h, p0/m, z22.h, z5.h \n" /* R1 */ + +#define READNV_SVE_2X \ + "ld1b {z0.h}, p1/z, [%[src_y]] \n" /* Y0 */ \ + "ld1b {z1.h}, p2/z, [%[src_y], #1, mul vl]\n" /* Y1 */ \ + "ld1h {z2.s}, p1/z, [%[src_uv]] \n" /* UV0 */ \ + "ld1h {z3.s}, p2/z, [%[src_uv], #1, mul vl]\n" /* UV1 */ \ + "incb %[src_y] \n" \ + "incb %[src_uv] \n" \ + "prfm pldl1keep, [%[src_y], 448] \n" \ + "prfm pldl1keep, [%[src_uv], 256] \n" \ + "trn1 z0.b, z0.b, z0.b \n" /* Y0Y0 */ \ + "trn1 z1.b, z1.b, z1.b \n" /* Y1Y1 */ \ + "trn1 z2.h, z2.h, z2.h \n" /* UV0 */ \ + "trn1 z3.h, z3.h, z3.h \n" /* UV1 */ + // We need a different predicate for the UV component to handle the tail. // If there is a single element remaining then we want to load one Y element // but two UV elements. -#define READNV_SVE_2X \ +#define READNV_RGB24_SVE_2X \ "ld1b {z0.b}, p1/z, [%[src_y]] \n" /* Y0Y0 */ \ "ld1b {z2.b}, p2/z, [%[src_uv]] \n" /* U0V0 or V0U0 */ \ "incb %[src_y] \n" \ @@ -234,7 +387,33 @@ extern "C" { "uqsub z16.h, z16.h, z25.h \n" /* B */ \ "uqsub z18.h, z18.h, z27.h \n" /* R */ -#define I444TORGB_SVE_2X \ +// clang-format off +#define I4XXTORGB_SVE_2X(bt) \ + "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \ + "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \ + "umull" #bt " z6.h, z30.b, z2.b \n" \ + "umull" #bt " z7.h, z30.b, z3.b \n" \ + "umlal" #bt " z6.h, z31.b, z4.b \n" /* DG0 */\ + "umlal" #bt " z7.h, z31.b, z5.b \n" /* DG1 */\ + "umull" #bt " z2.h, z28.b, z2.b \n" /* DB0 */\ + "umull" #bt " z3.h, z28.b, z3.b \n" /* DB1 */\ + "umull" #bt " z4.h, z29.b, z4.b \n" /* DR0 */\ + "umull" #bt " z5.h, z29.b, z5.b \n" /* DR1 */\ + "add z17.h, z0.h, z26.h \n" /* G0 */ \ + "add z21.h, z1.h, z26.h \n" /* G1 */ \ + "add z16.h, z0.h, z2.h \n" /* B0 */ \ + "add z20.h, z1.h, z3.h \n" /* B1 */ \ + "add z18.h, z0.h, z4.h \n" /* R0 */ \ + "add z22.h, z1.h, z5.h \n" /* R1 */ \ + "uqsub z17.h, z17.h, z6.h \n" /* G0 */ \ + "uqsub z21.h, z21.h, z7.h \n" /* G1 */ \ + "uqsub z16.h, z16.h, z25.h \n" /* B0 */ \ + "uqsub z20.h, z20.h, z25.h \n" /* B1 */ \ + "uqsub z18.h, z18.h, z27.h \n" /* R0 */ \ + "uqsub z22.h, z22.h, z27.h \n" /* R1 */ +// clang-format on + +#define I444TORGB_RGB24_SVE_2X \ "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \ "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \ "umullb z6.h, z30.b, z2.b \n" \ @@ -275,7 +454,7 @@ extern "C" { // The U/V component multiplies do not need to be duplicated in I422, we just // need to combine them with Y0/Y1 correctly. -#define I422TORGB_SVE_2X \ +#define I422TORGB_RGB24_SVE_2X \ "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \ "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \ "umullb z6.h, z30.b, z2.b \n" \ @@ -297,7 +476,32 @@ extern "C" { "uqsub z22.h, z22.h, z27.h \n" /* R1 */ // clang-format off -#define NVTORGB_SVE_2X(bt_u, bt_v) \ +#define NVTORGB_SVE_2X(bt_u, bt_v) \ + "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \ + "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \ + "umull" #bt_u " z6.h, z30.b, z2.b \n" \ + "umull" #bt_u " z7.h, z30.b, z3.b \n" \ + "umull" #bt_u " z4.h, z28.b, z2.b \n" /* DB0 */ \ + "umull" #bt_u " z5.h, z28.b, z3.b \n" /* DB1 */ \ + "umlal" #bt_v " z6.h, z31.b, z2.b \n" /* DG0 */ \ + "umlal" #bt_v " z7.h, z31.b, z3.b \n" /* DG1 */ \ + "umull" #bt_v " z2.h, z29.b, z2.b \n" /* DR0 */ \ + "umull" #bt_v " z3.h, z29.b, z3.b \n" /* DR1 */ \ + \ + "add z17.h, z0.h, z26.h \n" /* G0 */ \ + "add z21.h, z1.h, z26.h \n" /* G1 */ \ + "add z16.h, z0.h, z4.h \n" /* B0 */ \ + "add z20.h, z1.h, z5.h \n" /* B1 */ \ + "add z18.h, z0.h, z2.h \n" /* R0 */ \ + "add z22.h, z1.h, z3.h \n" /* R1 */ \ + "uqsub z17.h, z17.h, z6.h \n" /* G0 */ \ + "uqsub z21.h, z21.h, z7.h \n" /* G1 */ \ + "uqsub z16.h, z16.h, z25.h \n" /* B0 */ \ + "uqsub z20.h, z20.h, z25.h \n" /* B1 */ \ + "uqsub z18.h, z18.h, z27.h \n" /* R0 */ \ + "uqsub z22.h, z22.h, z27.h \n" /* R1 */ + +#define NVTORGB_RGB24_SVE_2X(bt_u, bt_v) \ "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \ "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \ "umull" #bt_u " z6.h, z30.b, z2.b \n" \ @@ -354,15 +558,60 @@ extern "C" { "uqshrnb z20.b, z17.h, #6 \n" /* G0 */ \ "uqshrnt z20.b, z18.h, #6 \n" /* GR */ +#define RGBTORGBA8_SVE_2X \ + /* Inputs: B: z16.h/z20.h, G: z17.h/z21.h, R: z18.h/z22.h, A: z23.b */ \ + "mov z19.d, z23.d \n" /* A0 */ \ + "uqshrnt z19.b, z16.h, #6 \n" /* AB0 */ \ + "mov z16.d, z19.d \n" /* AB0 */ \ + "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \ + "uqshrnt z17.b, z18.h, #6 \n" /* GR0 */ \ + "mov z19.d, z23.d \n" /* A1 */ \ + "uqshrnt z19.b, z20.h, #6 \n" /* AB1 */ \ + "mov z20.d, z19.d \n" /* AB1 */ \ + "uqshrnb z21.b, z21.h, #6 \n" /* G1 */ \ + "uqshrnt z21.b, z22.h, #6 \n" /* GR1 */ + #define RGBTOARGB8_SVE_2X \ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \ + /* B: z20.h, G: z21.h, R: z22.h, A: z19.b */ \ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \ - "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \ - "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \ - "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \ + "uqshrnt z16.b, z17.h, #6 \n" /* BG0 */ \ + "trn1 z17.b, z18.b, z19.b \n" /* RA0 */ \ + "uqshrnb z20.b, z20.h, #6 \n" /* B1 */ \ + "uqshrnb z22.b, z22.h, #6 \n" /* R1 */ \ + "uqshrnt z20.b, z21.h, #6 \n" /* BG1 */ \ + "trn1 z21.b, z22.b, z19.b \n" /* RA1 */ + +#define RGBATOARGB8_SVE_2X \ + /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \ + /* B: z20.h, G: z21.h, R: z22.h, A: z23.b */ \ + "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \ + "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \ + "uqshrnt z16.b, z17.h, #6 \n" /* BG0 */ \ + "trn1 z17.b, z18.b, z19.b \n" /* RA0 */ \ + "uqshrnb z20.b, z20.h, #6 \n" /* B1 */ \ + "uqshrnb z22.b, z22.h, #6 \n" /* R1 */ \ + "uqshrnt z20.b, z21.h, #6 \n" /* BG1 */ \ + "trn1 z21.b, z22.b, z23.b \n" /* RA1 */ + +#define RGBTORGB8_SVE_2X \ + /* Inputs: B: z16.h, G: z17.h, R: z18.h */ \ + /* B: z20.h, G: z21.h, R: z22.h */ \ + "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \ + "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \ + "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \ + "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \ + "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \ "uqshrnt z18.b, z22.h, #6 \n" /* R1 */ +// clang-format off +#define STOREARGB8_SVE_2X(dst) \ + "st2h {z16.h, z17.h}, p1, [%[" #dst "]] \n" \ + "st2h {z20.h, z21.h}, p2, [%[" #dst "], #2, mul vl] \n" \ + "incb %[" #dst "], all, mul #4 \n" +// clang-format on + // Store AR30 elements. Inputs are 2.14 fixed point RGB. We expect z23 to be // populated with 0x3ff0 (0x3fff would also work) to saturate the R input // rather than needing a pair of shifts to saturate and then insert into the @@ -393,15 +642,11 @@ extern "C" { "sri z18.h, z17.h, #12 \n" /* 11rrrrrrrrrrgggg */ \ "sri z22.h, z21.h, #12 \n" /* 11rrrrrrrrrrgggg */ \ "lsl z17.h, z17.h, #4 \n" /* ggggggxxxxxx0000 */ \ - "lsl z19.h, z21.h, #4 \n" /* ggggggxxxxxx0000 */ \ + "lsl z21.h, z21.h, #4 \n" /* ggggggxxxxxx0000 */ \ "sri z17.h, z16.h, #6 \n" /* ggggggbbbbbbbbbb */ \ - "sri z19.h, z20.h, #6 \n" /* ggggggbbbbbbbbbb */ \ - "zip2 z16.h, z17.h, z19.h \n" \ - "zip1 z21.h, z17.h, z19.h \n" \ - "zip2 z17.h, z18.h, z22.h \n" \ - "zip1 z22.h, z18.h, z22.h \n" \ - "st2h {z21.h, z22.h}, p2, [%[dst_ar30]] \n" \ - "st2h {z16.h, z17.h}, p3, [%[dst_ar30], #2, mul vl] \n" \ + "sri z21.h, z20.h, #6 \n" /* ggggggbbbbbbbbbb */ \ + "st2h {z17.h, z18.h}, p1, [%[dst_ar30]] \n" \ + "st2h {z21.h, z22.h}, p2, [%[dst_ar30], #2, mul vl] \n" \ "incb %[dst_ar30], all, mul #4 \n" #define YUVTORGB_SVE_REGS \ @@ -409,6 +654,54 @@ extern "C" { "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", \ "z30", "z31", "p0", "p1", "p2", "p3" +static inline void I444ToARGBRow_SVE_SC(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) STREAMING_COMPATIBLE { + uint64_t vl; + asm volatile( + "cnth %[vl] \n" + "ptrue p0.b \n" // + YUVTORGB_SVE_SETUP + "dup z19.b, #255 \n" // A + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + // Run bulk of computation with an all-true predicate to avoid predicate + // generation overhead. + "ptrue p1.b \n" + "ptrue p2.b \n" + "1: \n" // + READYUV444_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUV444_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X + "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" + + "99: \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] + : "memory", "cc", YUVTORGB_SVE_REGS); +} + static inline void I444ToRGB24Row_SVE_SC( const uint8_t* src_y, const uint8_t* src_u, @@ -428,7 +721,7 @@ static inline void I444ToRGB24Row_SVE_SC( // generation overhead. "ptrue p1.b \n" "1: \n" // - READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X + READYUV444_RGB24_SVE_2X I444TORGB_RGB24_SVE_2X RGBTORGB8_SVE_2X "subs %w[width], %w[width], %w[vl] \n" "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n" "incb %[dst_rgb24], all, mul #3 \n" @@ -440,7 +733,7 @@ static inline void I444ToRGB24Row_SVE_SC( // Calculate a predicate for the final iteration to deal with the tail. "whilelt p1.b, wzr, %w[width] \n" // - READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X + READYUV444_RGB24_SVE_2X I444TORGB_RGB24_SVE_2X RGBTORGB8_SVE_2X "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n" "99: \n" @@ -465,7 +758,7 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y, "ptrue p0.b \n" "dup z19.b, #255 \n" // Alpha YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "mov z1.h, #128 \n" // U/V "umullb z6.h, z30.b, z1.b \n" "umullb z4.h, z28.b, z1.b \n" // DB @@ -478,22 +771,24 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y, // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + READYUV400_SVE_2X I400TORGB_SVE_2X RGBTOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUV400_SVE_2X I400TORGB_SVE_2X RGBTOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -513,31 +808,33 @@ static inline void I422ToARGBRow_SVE_SC(const uint8_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm volatile( - "cntb %[vl] \n" + "cnth %[vl] \n" "ptrue p0.b \n" // YUVTORGB_SVE_SETUP "dup z19.b, #255 \n" // Alpha - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X - "subs %w[width], %w[width], %w[vl] \n" - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" - "incb %[dst_argb], all, mul #4 \n" + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width] \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X + "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -570,7 +867,7 @@ static inline void I422ToRGB24Row_SVE_SC( // generation overhead. "ptrue p1.b \n" "1: \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X + READYUV422_RGB24_SVE_2X I422TORGB_RGB24_SVE_2X RGBTORGB8_SVE_2X "subs %w[width], %w[width], %w[vl] \n" "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_argb]] \n" "incb %[dst_argb], all, mul #3 \n" @@ -582,7 +879,7 @@ static inline void I422ToRGB24Row_SVE_SC( // Calculate a predicate for the final iteration to deal with the tail. "whilelt p1.b, wzr, %w[width] \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X + READYUV422_RGB24_SVE_2X I422TORGB_RGB24_SVE_2X RGBTORGB8_SVE_2X "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_argb]] \n" "99: \n" @@ -601,8 +898,7 @@ static inline void I422ToRGB24Row_SVE_SC( "sri z18.h, z17.h, #5 \n" /* rrrrrgggggg00000 */ \ "sri z22.h, z21.h, #5 \n" /* rrrrrgggggg00000 */ \ "sri z18.h, z16.h, #11 \n" /* rrrrrggggggbbbbb */ \ - "sri z22.h, z20.h, #11 \n" /* rrrrrggggggbbbbb */ \ - "mov z19.d, z22.d \n" + "sri z22.h, z20.h, #11 \n" /* rrrrrggggggbbbbb */ static inline void I422ToRGB565Row_SVE_SC( const uint8_t* src_y, @@ -613,36 +909,36 @@ static inline void I422ToRGB565Row_SVE_SC( int width) STREAMING_COMPATIBLE { uint64_t vl; asm volatile( - "cntb %[vl] \n" + "cnth %[vl] \n" "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X - "subs %w[width], %w[width], %w[vl] \n" // + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_TOP_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // RGB8TORGB565_SVE_FROM_TOP_2X - "st2h {z18.h, z19.h}, p1, [%[dst]] \n" + "st1h {z18.h}, p1, [%[dst]] \n" + "st1h {z22.h}, p2, [%[dst], #1, mul vl] \n" "incb %[dst], all, mul #2 \n" "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width] \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_TOP_2X RGB8TORGB565_SVE_FROM_TOP_2X - // Need to permute the data on the final iteration such that the - // predicates (.b) line up with the 16-bit element data. - "trn1 z20.b, z18.b, z19.b \n" - "trn2 z21.b, z18.b, z19.b \n" - "st2b {z20.b, z21.b}, p1, [%[dst]] \n" + "st1h {z18.h}, p1, [%[dst]] \n" + "st1h {z22.h}, p2, [%[dst], #1, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -675,36 +971,36 @@ static inline void I422ToARGB1555Row_SVE_SC( int width) STREAMING_COMPATIBLE { uint64_t vl; asm volatile( - "cntb %[vl] \n" + "cnth %[vl] \n" "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X - "subs %w[width], %w[width], %w[vl] \n" // + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_TOP_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // RGB8TOARGB1555_SVE_FROM_TOP_2X - "st2h {z0.h, z1.h}, p1, [%[dst]] \n" + "st1h {z0.h}, p1, [%[dst]] \n" + "st1h {z1.h}, p2, [%[dst], #1, mul vl] \n" "incb %[dst], all, mul #2 \n" "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width] \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_TOP_2X RGB8TOARGB1555_SVE_FROM_TOP_2X - // Need to permute the data on the final iteration such that the - // predicates (.b) line up with the 16-bit element data. - "trn1 z2.b, z0.b, z1.b \n" - "trn2 z3.b, z0.b, z1.b \n" - "st2b {z2.b, z3.b}, p1, [%[dst]] \n" + "st1h {z0.h}, p1, [%[dst]] \n" + "st1h {z1.h}, p2, [%[dst], #1, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -737,36 +1033,36 @@ static inline void I422ToARGB4444Row_SVE_SC( int width) STREAMING_COMPATIBLE { uint64_t vl; asm volatile( - "cntb %[vl] \n" + "cnth %[vl] \n" "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X - "subs %w[width], %w[width], %w[vl] \n" // + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_TOP_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // RGB8TOARGB4444_SVE_FROM_TOP_2X - "st2h {z0.h, z1.h}, p1, [%[dst]] \n" + "st1h {z0.h}, p1, [%[dst]] \n" + "st1h {z1.h}, p2, [%[dst], #1, mul vl] \n" "incb %[dst], all, mul #2 \n" "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width] \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_TOP_2X RGB8TOARGB4444_SVE_FROM_TOP_2X - // Need to permute the data on the final iteration such that the - // predicates (.b) line up with the 16-bit element data. - "trn1 z2.b, z0.b, z1.b \n" - "trn2 z3.b, z0.b, z1.b \n" - "st2b {z2.b, z3.b}, p1, [%[dst]] \n" + "st1h {z0.h}, p1, [%[dst]] \n" + "st1h {z1.h}, p2, [%[dst], #1, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -791,28 +1087,30 @@ static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y, "cnth %[vl] \n" "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "dup z19.b, #255 \n" // Alpha - "subs %w[width], %w[width], %w[vl] \n" + "dup z23.b, #255 \n" // Alpha + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READYUV422_SVE I4XXTORGB_SVE RGBTORGBA8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z19.h, z20.h}, p1, [%[dst_rgba]] \n" - "add %[dst_rgba], %[dst_rgba], %[vl], lsl #2 \n" + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTORGBA8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_rgba) "b.ge 1b \n" - // Calculate a predicate for the final iteration to deal with the tail. + // Calculate predicates for the final iteration to deal with the tail. "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" - "whilelt p1.h, wzr, %w[width] \n" // - READYUV422_SVE I4XXTORGB_SVE RGBTORGBA8_SVE - "st2h {z19.h, z20.h}, p1, [%[dst_rgba]] \n" + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) RGBTORGBA8_SVE_2X + "st2h {z16.h, z17.h}, p1, [%[dst_rgba]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_rgba], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -847,21 +1145,19 @@ static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y, // generation overhead. "ptrue p1.b \n" "ptrue p2.b \n" - "ptrue p3.b \n" "1: \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) STOREAR30_SVE_2X "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.ge 1b \n" - // Calculate a predicate for the final iteration to deal with the tail. + // Calculate predicates for the final iteration to deal with the tail. "2: \n" "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - "whilelt p1.b, wzr, %w[width] \n" - "whilelt p2.h, wzr, %w[width] \n" - "whilelt p3.h, %w[vl], %w[width] \n" // - READYUV422_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUV422_SVE_2X I4XXTORGB_SVE_2X(b) STOREAR30_SVE_2X "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -886,35 +1182,39 @@ static inline void I422AlphaToARGBRow_SVE_SC( int width) STREAMING_COMPATIBLE { uint64_t vl; asm volatile( - "cntb %[vl] \n" + "cnth %[vl] \n" "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // READYUV422_SVE_2X - "ld1b {z19.b}, p1/z, [%[src_a]] \n" - "add %[src_a], %[src_a], %[vl] \n" // Alpha - I422TORGB_SVE_2X RGBTOARGB8_SVE_2X - "subs %w[width], %w[width], %w[vl] \n" - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" - "incb %[dst_argb], all, mul #4 \n" + "ld1b {z19.h}, p1/z, [%[src_a]] \n" + "ld1b {z23.h}, p2/z, [%[src_a], #1, mul vl] \n" + "incb %[src_a] \n" // Alpha + I4XXTORGB_SVE_2X(b) RGBATOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width] \n" // + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // READYUV422_SVE_2X - "ld1b {z19.b}, p1/z, [%[src_a]] \n" // Alpha - I422TORGB_SVE_2X RGBTOARGB8_SVE_2X - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" + "ld1b {z19.h}, p1/z, [%[src_a]] \n" + "ld1b {z23.h}, p2/z, [%[src_a], #1, mul vl] \n" // Alpha + I4XXTORGB_SVE_2X(b) RGBATOARGB8_SVE_2X + "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -926,7 +1226,7 @@ static inline void I422AlphaToARGBRow_SVE_SC( [vl] "=&r"(vl) // %[vl] : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void I444AlphaToARGBRow_SVE_SC( @@ -942,32 +1242,36 @@ static inline void I444AlphaToARGBRow_SVE_SC( "cnth %[vl] \n" "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READYUV444_SVE + READYUV444_SVE_2X "ld1b {z19.h}, p1/z, [%[src_a]] \n" - "add %[src_a], %[src_a], %[vl] \n" // Alpha - I4XXTORGB_SVE RGBTOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + "ld1b {z23.h}, p2/z, [%[src_a], #1, mul vl] \n" + "incb %[src_a] \n" // Alpha + I4XXTORGB_SVE_2X(b) RGBATOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READYUV444_SVE - "ld1b {z19.h}, p1/z, [%[src_a]] \n" // Alpha - I4XXTORGB_SVE RGBTOARGB8_SVE + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUV444_SVE_2X + "ld1b {z19.h}, p1/z, [%[src_a]] \n" + "ld1b {z23.h}, p2/z, [%[src_a], #1, mul vl] \n" // Alpha + I4XXTORGB_SVE_2X(b) RGBATOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -979,7 +1283,7 @@ static inline void I444AlphaToARGBRow_SVE_SC( [vl] "=&r"(vl) // %[vl] : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y, @@ -988,12 +1292,12 @@ static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y, const struct YuvConstants* yuvconstants, int width) STREAMING_COMPATIBLE { uint64_t vl; - asm("cntb %0" : "=r"(vl)); + asm("cnth %0" : "=r"(vl)); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP "dup z19.b, #255 \n" // Alpha - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate @@ -1002,22 +1306,20 @@ static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y, "ptrue p2.b \n" "1: \n" // READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X - "subs %w[width], %w[width], %w[vl] \n" - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width] \n" - "add %w[width], %w[width], #1 \n" // Round up to even - "and %w[width], %w[width], #0xfffffffe \n" - "whilelt p2.b, wzr, %w[width] \n" // + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" + "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1027,7 +1329,7 @@ static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y, : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y, @@ -1036,12 +1338,12 @@ static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y, const struct YuvConstants* yuvconstants, int width) STREAMING_COMPATIBLE { uint64_t vl; - asm("cntb %0" : "=r"(vl)); + asm("cnth %0" : "=r"(vl)); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP "dup z19.b, #255 \n" // Alpha - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate @@ -1050,22 +1352,20 @@ static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y, "ptrue p2.b \n" "1: \n" // READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X - "subs %w[width], %w[width], %w[vl] \n" - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width] \n" - "add %w[width], %w[width], #1 \n" // Round up to even - "and %w[width], %w[width], #0xfffffffe \n" - "whilelt p2.b, wzr, %w[width] \n" // + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" + "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1075,7 +1375,7 @@ static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y, : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void NV12ToRGB24Row_SVE_SC( @@ -1098,7 +1398,7 @@ static inline void NV12ToRGB24Row_SVE_SC( "ptrue p1.b \n" "ptrue p2.b \n" "1: \n" // - READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X + READNV_RGB24_SVE_2X NVTORGB_RGB24_SVE_2X(b, t) RGBTORGB8_SVE_2X "subs %w[width], %w[width], %w[vl] \n" "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n" "incb %[dst_rgb24], all, mul #3 \n" @@ -1113,7 +1413,7 @@ static inline void NV12ToRGB24Row_SVE_SC( "add %w[width], %w[width], #1 \n" // Round up to even "and %w[width], %w[width], #0xfffffffe \n" "whilelt p2.b, wzr, %w[width] \n" // - READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X + READNV_RGB24_SVE_2X NVTORGB_RGB24_SVE_2X(b, t) RGBTORGB8_SVE_2X "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n" "99: \n" @@ -1124,7 +1424,7 @@ static inline void NV12ToRGB24Row_SVE_SC( : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void NV21ToRGB24Row_SVE_SC( @@ -1147,7 +1447,7 @@ static inline void NV21ToRGB24Row_SVE_SC( "ptrue p1.b \n" "ptrue p2.b \n" "1: \n" // - READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X + READNV_RGB24_SVE_2X NVTORGB_RGB24_SVE_2X(t, b) RGBTORGB8_SVE_2X "subs %w[width], %w[width], %w[vl] \n" "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n" "incb %[dst_rgb24], all, mul #3 \n" @@ -1162,7 +1462,7 @@ static inline void NV21ToRGB24Row_SVE_SC( "add %w[width], %w[width], #1 \n" // Round up to even "and %w[width], %w[width], #0xfffffffe \n" "whilelt p2.b, wzr, %w[width] \n" // - READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X + READNV_RGB24_SVE_2X NVTORGB_RGB24_SVE_2X(t, b) RGBTORGB8_SVE_2X "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n" "99: \n" @@ -1173,123 +1473,119 @@ static inline void NV21ToRGB24Row_SVE_SC( : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); + : "memory", "cc", YUVTORGB_SVE_REGS); } -#define READYUY2_SVE \ - "ld1w {z0.s}, p2/z, [%[src_yuy2]] \n" /* YUYV */ \ - "incb %[src_yuy2] \n" \ - "prfm pldl1keep, [%[src_yuy2], 448] \n" \ - "tbl z1.b, {z0.b}, z22.b \n" /* UVUV */ \ - "trn1 z0.b, z0.b, z0.b \n" /* YYYY */ +#define READYUY2_SVE_2X \ + "ld1w {z2.s}, p1/z, [%[src_yuy2]] \n" /* YUYV low */ \ + "ld1w {z3.s}, p2/z, [%[src_yuy2], #1, mul vl]\n" /* YUYV high */ \ + "incb %[src_yuy2], all, mul #2 \n" \ + "prfm pldl1keep, [%[src_yuy2], 448] \n" \ + "trn1 z0.b, z2.b, z2.b \n" /* Y0Y0 */ \ + "trn1 z1.b, z3.b, z3.b \n" /* Y1Y1 */ \ + "trn2 z4.h, z2.h, z2.h \n" /* YVYV0 */ \ + "trn1 z2.h, z2.h, z2.h \n" /* YUYU0 */ \ + "trn2 z5.h, z3.h, z3.h \n" /* YVYV1 */ \ + "trn1 z3.h, z3.h, z3.h \n" /* YUYU1 */ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) STREAMING_COMPATIBLE { - uint32_t nv_uv_start = 0x03010301U; - uint32_t nv_uv_step = 0x04040404U; uint64_t vl; asm("cnth %0" : "=r"(vl)); asm volatile( "ptrue p0.b \n" - "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n" "dup z19.b, #255 \n" // Alpha YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" - "ptrue p2.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READYUY2_SVE NVTORGB_SVE RGBTOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + READYUY2_SVE_2X I4XXTORGB_SVE_2X(t) RGBTOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. + // Calculate predicates for the final iteration to deal with the tail. "whilelt p1.h, wzr, %w[width] \n" - "add %w[width], %w[width], #1 \n" // Round up to even - "and %w[width], %w[width], #0xfffffffe \n" - "whilelt p2.h, wzr, %w[width] \n" // - READYUY2_SVE NVTORGB_SVE RGBTOARGB8_SVE + "whilelt p2.h, %w[vl], %w[width] \n" // + READYUY2_SVE_2X I4XXTORGB_SVE_2X(t) RGBTOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" - : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start] - [nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step] - : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); + : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] + : "memory", "cc", YUVTORGB_SVE_REGS); } -#define READUYVY_SVE \ - "ld1w {z0.s}, p2/z, [%[src_uyvy]] \n" /* UYVY */ \ - "incb %[src_uyvy] \n" \ - "prfm pldl1keep, [%[src_uyvy], 448] \n" \ - "tbl z1.b, {z0.b}, z22.b \n" /* UVUV */ \ - "trn2 z0.b, z0.b, z0.b \n" /* YYYY */ +#define READUYVY_SVE_2X \ + "ld1w {z2.s}, p1/z, [%[src_uyvy]] \n" /* UYVY low */ \ + "ld1w {z3.s}, p2/z, [%[src_uyvy], #1, mul vl]\n" /* UYVY high */ \ + "incb %[src_uyvy], all, mul #2 \n" \ + "prfm pldl1keep, [%[src_uyvy], 448] \n" \ + "trn2 z0.b, z2.b, z2.b \n" /* Y0Y0 */ \ + "trn2 z1.b, z3.b, z3.b \n" /* Y1Y1 */ \ + "trn2 z4.h, z2.h, z2.h \n" /* VYVY0 */ \ + "trn1 z2.h, z2.h, z2.h \n" /* UYUY0 */ \ + "trn2 z5.h, z3.h, z3.h \n" /* VYVY1 */ \ + "trn1 z3.h, z3.h, z3.h \n" /* UYUY1 */ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) STREAMING_COMPATIBLE { - uint32_t nv_uv_start = 0x02000200U; - uint32_t nv_uv_step = 0x04040404U; uint64_t vl; asm("cnth %0" : "=r"(vl)); asm volatile( "ptrue p0.b \n" - "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n" "dup z19.b, #255 \n" // Alpha YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" - "ptrue p2.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READUYVY_SVE NVTORGB_SVE RGBTOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + READUYVY_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. + // Calculate predicates for the final iteration to deal with the tail. "whilelt p1.h, wzr, %w[width] \n" - "add %w[width], %w[width], #1 \n" // Round up to even - "and %w[width], %w[width], #0xfffffffe \n" - "whilelt p2.h, wzr, %w[width] \n" // - READUYVY_SVE NVTORGB_SVE RGBTOARGB8_SVE + "whilelt p2.h, %w[vl], %w[width] \n" // + READUYVY_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" - : [src_uyvy] "+r"(src_uyvy), // %[src_yuy2] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start] - [nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step] - : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); + : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y, @@ -1304,27 +1600,29 @@ static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y, "ptrue p0.b \n" // YUVTORGB_SVE_SETUP "dup z19.b, #255 \n" // Alpha - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + READI210_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READI210_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1335,7 +1633,7 @@ static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y, : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void I210AlphaToARGBRow_SVE_SC( @@ -1351,33 +1649,29 @@ static inline void I210AlphaToARGBRow_SVE_SC( asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READI210_SVE - "ld1h {z19.h}, p1/z, [%[src_a]] \n" // - I4XXTORGB_SVE - "incb %[src_a] \n" // - RGBATOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + READI210_SVE_2X I4XXTORGB_SVE_2X(b) READA10_SVE_2X RGBATOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READI210_SVE - "ld1h {z19.h}, p1/z, [%[src_a]] \n" // - I4XXTORGB_SVE RGBATOARGB8_SVE + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READI210_SVE_2X I4XXTORGB_SVE_2X(b) READA10_SVE_2X RGBATOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1389,7 +1683,7 @@ static inline void I210AlphaToARGBRow_SVE_SC( : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y, @@ -1411,11 +1705,10 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y, // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" - "ptrue p2.h \n" - "ptrue p3.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READI210_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X + READI210_SVE_2X I4XXTORGB_SVE_2X(b) STOREAR30_SVE_2X "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.ge 1b \n" @@ -1423,13 +1716,10 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y, "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p2.h, wzr, %w[width] \n" - "whilelt p3.h, %w[vl], %w[width] \n" - "add %w[width], %w[width], #1 \n" // (width + 1) / 2 - "lsr %w[width], %w[width], #1 \n" - "whilelt p1.h, wzr, %w[width] \n" // - READI210_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READI210_SVE_2X I4XXTORGB_SVE_2X(b) STOREAR30_SVE_2X "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1441,7 +1731,7 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y, [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] [limit] "r"(limit) // %[limit] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } // P210 has 10 bits in msb of 16 bit NV12 style layout. @@ -1452,50 +1742,43 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - uint32_t nv_uv_start = 0x03010301U; - uint32_t nv_uv_step = 0x04040404U; asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n" "dup z19.b, #255 \n" // Alpha - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" - "ptrue p2.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READP210_SVE NVTORGB_SVE RGBTOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + READP210_SVE_2X I4XXTORGB_SVE_2X(t) RGBTOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. + // Calculate predicates for the final iteration to deal with the tail. "whilelt p1.h, wzr, %w[width] \n" - "add %w[width], %w[width], #1 \n" // Round up to even - "and %w[width], %w[width], #0xfffffffe \n" - "whilelt p2.h, wzr, %w[width] \n" // - READP210_SVE NVTORGB_SVE RGBTOARGB8_SVE + "whilelt p2.h, %w[vl], %w[width] \n" // + READP210_SVE_2X I4XXTORGB_SVE_2X(t) RGBTOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_uv] "+r"(src_uv), // %[src_uv] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start] - [nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step] - : "cc", "memory", YUVTORGB_SVE_REGS); + : [src_y] "+r"(src_y), // %[src_y] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y, @@ -1505,38 +1788,32 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - uint32_t nv_uv_start = 0x03010301U; - uint32_t nv_uv_step = 0x04040404U; - // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE. + // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE_2X. uint16_t limit = 0x3ff0; asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n" "dup z23.h, %w[limit] \n" - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" - "ptrue p2.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READP210_SVE NVTORGB_SVE - "subs %w[width], %w[width], %w[vl] \n" // - STOREAR30_SVE + READP210_SVE_2X I4XXTORGB_SVE_2X(t) STOREAR30_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. + // Calculate predicates for the final iteration to deal with the tail. "whilelt p1.h, wzr, %w[width] \n" - "add %w[width], %w[width], #1 \n" // Round up to even - "and %w[width], %w[width], #0xfffffffe \n" - "whilelt p2.h, wzr, %w[width] \n" // - READP210_SVE NVTORGB_SVE STOREAR30_SVE + "whilelt p2.h, %w[vl], %w[width] \n" // + READP210_SVE_2X I4XXTORGB_SVE_2X(t) STOREAR30_SVE_2X "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1546,10 +1823,8 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y, : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start] - [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step] [limit] "r"(limit) // %[limit] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y, @@ -1564,27 +1839,29 @@ static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y, "ptrue p0.b \n" // YUVTORGB_SVE_SETUP "dup z19.b, #255 \n" // Alpha - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READI410_SVE I4XXTORGB_SVE RGBTOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + READI410_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READI410_SVE I4XXTORGB_SVE RGBTOARGB8_SVE + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READI410_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1595,7 +1872,7 @@ static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y, : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void I410AlphaToARGBRow_SVE_SC( @@ -1611,33 +1888,29 @@ static inline void I410AlphaToARGBRow_SVE_SC( asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READI410_SVE - "ld1h {z19.h}, p1/z, [%[src_a]] \n" // - I4XXTORGB_SVE - "incb %[src_a] \n" // - RGBATOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + READI410_SVE_2X I4XXTORGB_SVE_2X(b) READA10_SVE_2X RGBATOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READI410_SVE - "ld1h {z19.h}, p1/z, [%[src_a]] \n" // - I4XXTORGB_SVE RGBATOARGB8_SVE + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READI410_SVE_2X I4XXTORGB_SVE_2X(b) READA10_SVE_2X RGBATOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1649,7 +1922,7 @@ static inline void I410AlphaToARGBRow_SVE_SC( : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y, @@ -1660,30 +1933,32 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE. + // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE_2X. uint16_t limit = 0x3ff0; asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP "dup z23.h, %w[limit] \n" - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READI410_SVE I4XXTORGB_SVE STOREAR30_SVE - "subs %w[width], %w[width], %w[vl] \n" + READI410_SVE_2X I4XXTORGB_SVE_2X(b) STOREAR30_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READI410_SVE I4XXTORGB_SVE STOREAR30_SVE + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READI410_SVE_2X I4XXTORGB_SVE_2X(b) STOREAR30_SVE_2X "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1695,7 +1970,7 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y, [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] [limit] "r"(limit) // %[limit] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y, @@ -1744,7 +2019,7 @@ static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y, [vl] "+r"(vl) // %[vl] : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y, @@ -1794,7 +2069,7 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y, : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] [limit] "r"(limit) // %[limit] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y, @@ -1805,30 +2080,32 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE. + // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE_2X. uint16_t limit = 0x3ff0; asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP "dup z23.h, %w[limit] \n" - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READI212_SVE I4XXTORGB_SVE STOREAR30_SVE - "subs %w[width], %w[width], %w[vl] \n" + READI212_SVE_2X I4XXTORGB_SVE_2X(b) STOREAR30_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READI212_SVE I4XXTORGB_SVE STOREAR30_SVE + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READI212_SVE_2X I4XXTORGB_SVE_2X(b) STOREAR30_SVE_2X "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1840,7 +2117,7 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y, [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] [limit] "r"(limit) // %[limit] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y, @@ -1855,27 +2132,29 @@ static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y, "ptrue p0.b \n" // YUVTORGB_SVE_SETUP "dup z19.b, #255 \n" // Alpha - "subs %w[width], %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. - "ptrue p1.h \n" + "ptrue p1.b \n" + "ptrue p2.b \n" "1: \n" // - READI212_SVE I4XXTORGB_SVE RGBTOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + READI212_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X + "subs %w[width], %w[width], %w[vl], lsl #1 \n" // + STOREARGB8_SVE_2X(dst_argb) "b.ge 1b \n" "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READI212_SVE I4XXTORGB_SVE RGBTOARGB8_SVE + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" // + READI212_SVE_2X I4XXTORGB_SVE_2X(b) RGBTOARGB8_SVE_2X "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "st2h {z20.h, z21.h}, p2, [%[dst_argb], #2, mul vl] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -1886,7 +2165,7 @@ static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y, : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + : "memory", "cc", YUVTORGB_SVE_REGS); } #define CONVERT8TO8_SVE \ @@ -1940,7 +2219,7 @@ static inline void Convert8To8Row_SVE_SC(const uint8_t* src_y, [vl] "=&r"(vl) // %[vl] : [scale] "r"(scale), // %[scale] [bias] "r"(bias) // %[bias] - : "cc", "memory", "z0", "z1", "z2", "z3", "p0", "p1"); + : "memory", "cc", "z0", "z1", "z2", "z3", "p0", "p1"); } // SVE constants are stored negated such that we can store 128 in int8_t. @@ -2116,15 +2395,11 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb, "z27", "p0", "p1", "p2", "p3", "p4", "p5"); } -#define ARGBTOYMATRIX_SVE(p1, p2, p3, p4, p5) \ - "ld1w {z0.s}, " p1 \ - "/z, [%[src]] \n" /* load 4*vl pixels */ \ - "ld1w {z1.s}, " p2 \ - "/z, [%[src], #1, mul vl] \n" \ - "ld1w {z2.s}, " p3 \ - "/z, [%[src], #2, mul vl] \n" \ - "ld1w {z3.s}, " p4 \ - "/z, [%[src], #3, mul vl] \n" \ +#define ARGBTOYMATRIX_SVE \ + "ld1w {z0.s}, p1/z, [%[src]] \n" \ + "ld1w {z1.s}, p2/z, [%[src], #1, mul vl] \n" \ + "ld1w {z2.s}, p3/z, [%[src], #2, mul vl] \n" \ + "ld1w {z3.s}, p4/z, [%[src], #3, mul vl] \n" \ "incb %[src], all, mul #4 \n" \ "mov z16.b, #0 \n" \ "mov z17.b, #0 \n" \ @@ -2139,8 +2414,7 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb, "addhnb z16.b, z16.h, z25.h \n" \ "addhnb z18.b, z18.h, z25.h \n" \ "uzp1 z20.b, z16.b, z18.b \n" \ - "st1b {z20.b}, " p5 \ - ", [%[dst_y]] \n" \ + "st1b {z20.b}, p5, [%[dst_y]] \n" \ "incb %[dst_y] \n" \ "subs %w[width], %w[width], %w[vl], lsl #2 \n" @@ -2161,8 +2435,12 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, "b.lt 2f \n" "ptrue p1.s \n" + "ptrue p2.s \n" + "ptrue p3.s \n" + "ptrue p4.s \n" + "ptrue p5.b \n" "1: \n" // - ARGBTOYMATRIX_SVE("p1", "p1", "p1", "p1", "p0") + ARGBTOYMATRIX_SVE "b.ge 1b \n" "2: \n" @@ -2175,65 +2453,61 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, "whilelt p3.s, %w[vl2], %w[width] \n" "whilelt p4.s, %w[vl3], %w[width] \n" "whilelt p5.b, wzr, %w[width] \n" // - ARGBTOYMATRIX_SVE("p1", "p2", "p3", "p4", "p5") + ARGBTOYMATRIX_SVE "b.gt 3b \n" "99: \n" - : [src] "+r"(src_argb), // %[src] - [dst_y] "+r"(dst_y), // %[dst_y] - [width] "+r"(width) // %[width] - : [c] "r"(c), // %[c] - [vl] "r"(vl), // %[vl] - [vl2] "r"(vl * 2), // %[vl2] - [vl3] "r"(vl * 3) // %[vl3] + : [src] "+r"(src_argb), // %[src] + [dst_y] "+r"(dst_y), // %[dst_y] + [width] "+r"(width) // %[width] + : [c] "r"(c), // %[c] + [vl] "r"(vl), // %[vl] + [vl2] "r"(vl * 2), // %[vl2] + [vl3] "r"(vl * 3) // %[vl3] : "cc", "memory", "z0", "z1", "z2", "z3", "z16", "z17", "z18", "z19", "z20", "z24", "z25", "p0", "p1", "p2", "p3", "p4", "p5"); } -#define SSIM4X4_SVE \ - "add %[tmp_a], %[src_a], %[stride_a] \n" \ - "add %[tmp_b], %[src_b], %[stride_b] \n" \ - "ld1w {z0.s}, p0/z, [%[src_a]] \n" \ - "ld1w {z1.s}, p0/z, [%[tmp_a]] \n" \ - "add %[tmp_a], %[tmp_a], %[stride_a] \n" \ - "ld1w {z4.s}, p0/z, [%[src_b]] \n" \ - "ld1w {z5.s}, p0/z, [%[tmp_b]] \n" \ - "add %[tmp_b], %[tmp_b], %[stride_b] \n" \ - "ld1w {z2.s}, p0/z, [%[tmp_a]] \n" \ - "add %[tmp_a], %[tmp_a], %[stride_a] \n" \ - "ld1w {z3.s}, p0/z, [%[tmp_a]] \n" \ - "ld1w {z6.s}, p0/z, [%[tmp_b]] \n" \ - "add %[tmp_b], %[tmp_b], %[stride_b] \n" \ - "ld1w {z7.s}, p0/z, [%[tmp_b]] \n" \ - "incb %[src_a] \n" \ - "incb %[src_b] \n" \ - "mov z16.s, #0 \n" \ - "mov z17.s, #0 \n" \ - "mov z18.s, #0 \n" \ - "mov z19.s, #0 \n" \ - "udot z16.s, z0.b, z20.b \n" \ - "udot z17.s, z4.b, z20.b \n" \ - "udot z18.s, z0.b, z0.b \n" \ - "udot z19.s, z0.b, z4.b \n" \ - "udot z16.s, z1.b, z20.b \n" \ - "udot z17.s, z5.b, z20.b \n" \ - "udot z18.s, z1.b, z1.b \n" \ - "udot z19.s, z1.b, z5.b \n" \ - "udot z16.s, z2.b, z20.b \n" \ - "udot z17.s, z6.b, z20.b \n" \ - "udot z18.s, z2.b, z2.b \n" \ - "udot z19.s, z2.b, z6.b \n" \ - "udot z16.s, z3.b, z20.b \n" \ - "udot z17.s, z7.b, z20.b \n" \ - "udot z18.s, z3.b, z3.b \n" \ - "udot z19.s, z3.b, z7.b \n" \ - "udot z18.s, z4.b, z4.b \n" \ - "udot z18.s, z5.b, z5.b \n" \ - "udot z18.s, z6.b, z6.b \n" \ - "udot z18.s, z7.b, z7.b \n" \ - "subs %w[width], %w[width], %w[vl] \n" \ - "st4w {z16.s, z17.s, z18.s, z19.s}, p0, [%[dst]]\n" \ - "incb %[dst], all, mul #4 \n" +#define SSIM4X4_SVE \ + "ld1b {z0.b}, p1/z, [%[src_a]] \n" \ + "ld1b {z1.b}, p1/z, [%[src_a], %[stride_a]] \n" \ + "ld1b {z2.b}, p1/z, [%[src_a], %[stride_a2]] \n" \ + "ld1b {z3.b}, p1/z, [%[src_a], %[stride_a3]] \n" \ + "ld1b {z4.b}, p1/z, [%[src_b]] \n" \ + "ld1b {z5.b}, p1/z, [%[src_b], %[stride_b]] \n" \ + "ld1b {z6.b}, p1/z, [%[src_b], %[stride_b2]] \n" \ + "ld1b {z7.b}, p1/z, [%[src_b], %[stride_b3]] \n" \ + "incb %[src_a] \n" \ + "incb %[src_b] \n" \ + "mov z16.s, #0 \n" \ + "mov z17.s, #0 \n" \ + "mov z18.s, #0 \n" \ + "mov z19.s, #0 \n" \ + "mov z21.s, #0 \n" \ + "udot z16.s, z0.b, z20.b \n" \ + "udot z17.s, z4.b, z20.b \n" \ + "udot z18.s, z0.b, z0.b \n" \ + "udot z21.s, z4.b, z4.b \n" \ + "udot z19.s, z0.b, z4.b \n" \ + "udot z16.s, z1.b, z20.b \n" \ + "udot z17.s, z5.b, z20.b \n" \ + "udot z18.s, z1.b, z1.b \n" \ + "udot z21.s, z5.b, z5.b \n" \ + "udot z19.s, z1.b, z5.b \n" \ + "udot z16.s, z2.b, z20.b \n" \ + "udot z17.s, z6.b, z20.b \n" \ + "udot z18.s, z2.b, z2.b \n" \ + "udot z21.s, z6.b, z6.b \n" \ + "udot z19.s, z2.b, z6.b \n" \ + "udot z16.s, z3.b, z20.b \n" \ + "udot z17.s, z7.b, z20.b \n" \ + "udot z18.s, z3.b, z3.b \n" \ + "udot z21.s, z7.b, z7.b \n" \ + "udot z19.s, z3.b, z7.b \n" \ + "add z18.s, z18.s, z21.s \n" \ + "subs %w[width], %w[width], %w[vl] \n" \ + "st4w {z16.s, z17.s, z18.s, z19.s}, p0, [%[dst]] \n" \ + "incb %[dst], all, mul #4 \n" static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a, int stride_a, @@ -2242,8 +2516,10 @@ static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a, uint32_t* dst, int width) STREAMING_COMPATIBLE { uint64_t vl; - const uint8_t* tmp_a; - const uint8_t* tmp_b; + ptrdiff_t stride_a2 = (ptrdiff_t)stride_a * 2; + ptrdiff_t stride_a3 = (ptrdiff_t)stride_a * 3; + ptrdiff_t stride_b2 = (ptrdiff_t)stride_b * 2; + ptrdiff_t stride_b3 = (ptrdiff_t)stride_b * 3; width >>= 2; asm volatile( "cntw %x[vl] \n" @@ -2252,6 +2528,7 @@ static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a, "b.lt 2f \n" "ptrue p0.s \n" + "ptrue p1.b \n" "1: \n" // SSIM4X4_SVE "b.ge 1b \n" @@ -2260,7 +2537,9 @@ static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a, "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" - "whilelt p0.s, wzr, %w[width] \n" // + "whilelt p0.s, wzr, %w[width] \n" + "lsl %w[vl], %w[width], #2 \n" + "whilelt p1.b, wzr, %w[vl] \n" // SSIM4X4_SVE "99: \n" @@ -2268,87 +2547,133 @@ static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a, [src_b] "+r"(src_b), // %[src_b] [dst] "+r"(dst), // %[dst] [width] "+r"(width), // %[width] - [vl] "=&r"(vl), // %[vl] - [tmp_a] "=&r"(tmp_a), // %[tmp_a] - [tmp_b] "=&r"(tmp_b) // %[tmp_b] + [vl] "=&r"(vl) // %[vl] : [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a] - [stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b] + [stride_a2] "r"(stride_a2), // %[stride_a2] + [stride_a3] "r"(stride_a3), // %[stride_a3] + [stride_b] "r"((ptrdiff_t)stride_b), // %[stride_b] + [stride_b2] "r"(stride_b2), // %[stride_b2] + [stride_b3] "r"(stride_b3) // %[stride_b3] : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", - "z17", "z18", "z19", "z20", "p0"); + "z17", "z18", "z19", "z20", "z21", "p0", "p1"); } +#define SSIMWINDOW_SVE \ + "ld1w {z0.s}, p2/z, [%[row0]] \n" \ + "ld1w {z1.s}, p3/z, [%[row0], #1, mul vl] \n" \ + "ld1w {z2.s}, p4/z, [%[row0], #2, mul vl] \n" \ + "ld1w {z3.s}, p5/z, [%[row0], #3, mul vl] \n" \ + "incb %[row0], all, mul #4 \n" \ + "ld1w {z4.s}, p2/z, [%[row0_next]] \n" \ + "ld1w {z5.s}, p3/z, [%[row0_next], #1, mul vl] \n" \ + "ld1w {z6.s}, p4/z, [%[row0_next], #2, mul vl] \n" \ + "ld1w {z7.s}, p5/z, [%[row0_next], #3, mul vl] \n" \ + "incb %[row0_next], all, mul #4 \n" \ + "add z0.s, z0.s, z4.s \n" \ + "add z1.s, z1.s, z5.s \n" \ + "add z2.s, z2.s, z6.s \n" \ + "add z3.s, z3.s, z7.s \n" \ + "ld1w {z16.s}, p2/z, [%[row1]] \n" \ + "ld1w {z17.s}, p3/z, [%[row1], #1, mul vl] \n" \ + "ld1w {z18.s}, p4/z, [%[row1], #2, mul vl] \n" \ + "ld1w {z19.s}, p5/z, [%[row1], #3, mul vl] \n" \ + "incb %[row1], all, mul #4 \n" \ + "ld1w {z4.s}, p2/z, [%[row1_next]] \n" \ + "ld1w {z5.s}, p3/z, [%[row1_next], #1, mul vl] \n" \ + "ld1w {z6.s}, p4/z, [%[row1_next], #2, mul vl] \n" \ + "ld1w {z7.s}, p5/z, [%[row1_next], #3, mul vl] \n" \ + "incb %[row1_next], all, mul #4 \n" \ + "add z16.s, z16.s, z4.s \n" \ + "add z17.s, z17.s, z5.s \n" \ + "add z18.s, z18.s, z6.s \n" \ + "add z19.s, z19.s, z7.s \n" \ + "add z0.s, z0.s, z16.s \n" \ + "add z1.s, z1.s, z17.s \n" \ + "add z2.s, z2.s, z18.s \n" \ + "add z3.s, z3.s, z19.s \n" \ + "uzp1 z4.s, z0.s, z1.s \n" \ + "uzp2 z5.s, z0.s, z1.s \n" \ + "uzp1 z6.s, z2.s, z3.s \n" \ + "uzp2 z7.s, z2.s, z3.s \n" \ + "uzp1 z0.s, z4.s, z6.s \n" \ + "uzp2 z2.s, z4.s, z6.s \n" \ + "uzp1 z1.s, z5.s, z7.s \n" \ + "uzp2 z3.s, z5.s, z7.s \n" \ + "mul z4.s, z0.s, z1.s \n" \ + "lsl z4.s, z4.s, #1 \n" \ + "mul z5.s, z0.s, z0.s \n" \ + "mla z5.s, p1/m, z1.s, z1.s \n" \ + "lsl z3.s, z3.s, #7 \n" \ + "sub z3.s, z3.s, z4.s \n" \ + "add z4.s, z4.s, z25.s \n" \ + "add z3.s, z3.s, z26.s \n" \ + "lsl z2.s, z2.s, #6 \n" \ + "sub z2.s, z2.s, z5.s \n" \ + "add z5.s, z5.s, z25.s \n" \ + "add z2.s, z2.s, z26.s \n" \ + "scvtf z4.s, p1/m, z4.s \n" \ + "scvtf z3.s, p1/m, z3.s \n" \ + "scvtf z5.s, p1/m, z5.s \n" \ + "scvtf z2.s, p1/m, z2.s \n" \ + "fmul z4.s, z4.s, z3.s \n" \ + "fmul z5.s, z5.s, z2.s \n" \ + "fdiv z4.s, p1/m, z4.s, z5.s \n" \ + "fadd z24.s, p0/m, z24.s, z4.s \n" + static inline float SsimWindowRow_SVE_SC(const uint32_t* row0, const uint32_t* row1, int num_windows) STREAMING_COMPATIBLE { uint64_t vl; + asm("cntw %x0" : "=r"(vl)); const uint32_t* row0_next = row0 + 4; const uint32_t* row1_next = row1 + 4; float total; asm volatile( - "cntw %x[vl] \n" "ptrue p1.s \n" "mov z24.s, #0 \n" "dup z25.s, %w[c1] \n" "dup z26.s, %w[c2] \n" - - "1: \n" - "whilelt p0.s, wzr, %w[num_windows] \n" - "ld4w {z0.s, z1.s, z2.s, z3.s}, p0/z, [%[row0]] \n" - "ld4w {z4.s, z5.s, z6.s, z7.s}, p0/z, [%[row0_next]] \n" - "ld4w {z16.s, z17.s, z18.s, z19.s}, p0/z, [%[row1]] \n" - "ld4w {z20.s, z21.s, z22.s, z23.s}, p0/z, [%[row1_next]] \n" - "incb %[row0], all, mul #4 \n" - "incb %[row0_next], all, mul #4 \n" - "incb %[row1], all, mul #4 \n" - "incb %[row1_next], all, mul #4 \n" - "add z0.s, z0.s, z4.s \n" - "add z1.s, z1.s, z5.s \n" - "add z2.s, z2.s, z6.s \n" - "add z3.s, z3.s, z7.s \n" - "add z16.s, z16.s, z20.s \n" - "add z17.s, z17.s, z21.s \n" - "add z18.s, z18.s, z22.s \n" - "add z19.s, z19.s, z23.s \n" - "add z0.s, z0.s, z16.s \n" - "add z1.s, z1.s, z17.s \n" - "add z2.s, z2.s, z18.s \n" - "add z3.s, z3.s, z19.s \n" - "mul z4.s, z0.s, z1.s \n" - "lsl z4.s, z4.s, #1 \n" - "mul z5.s, z0.s, z0.s \n" - "mla z5.s, p1/m, z1.s, z1.s \n" - "lsl z3.s, z3.s, #7 \n" - "sub z3.s, z3.s, z4.s \n" - "add z4.s, z4.s, z25.s \n" - "add z3.s, z3.s, z26.s \n" - "lsl z2.s, z2.s, #6 \n" - "sub z2.s, z2.s, z5.s \n" - "add z5.s, z5.s, z25.s \n" - "add z2.s, z2.s, z26.s \n" - "scvtf z4.s, p1/m, z4.s \n" - "scvtf z3.s, p1/m, z3.s \n" - "scvtf z5.s, p1/m, z5.s \n" - "scvtf z2.s, p1/m, z2.s \n" - "fmul z4.s, z4.s, z3.s \n" - "fmul z5.s, z5.s, z2.s \n" - "fdiv z4.s, p1/m, z4.s, z5.s \n" - "fadd z24.s, p0/m, z24.s, z4.s \n" "subs %w[num_windows], %w[num_windows], %w[vl] \n" - "b.gt 1b \n" + "b.lt 2f \n" + "ptrue p0.s \n" + "ptrue p2.s \n" + "ptrue p3.s \n" + "ptrue p4.s \n" + "ptrue p5.s \n" + "1: \n" // + SSIMWINDOW_SVE + "subs %w[num_windows], %w[num_windows], %w[vl] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[num_windows], %w[num_windows], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[num_windows] \n" + "lsl %w[num_windows], %w[num_windows], #2 \n" + "whilelt p2.s, wzr, %w[num_windows] \n" + "whilelt p3.s, %w[vl], %w[num_windows] \n" + "whilelt p4.s, %w[vl2], %w[num_windows] \n" + "whilelt p5.s, %w[vl3], %w[num_windows] \n" // + SSIMWINDOW_SVE + + "99: \n" "faddv %s[total], p1, z24.s \n" : [row0] "+r"(row0), // %[row0] [row0_next] "+r"(row0_next), // %[row0_next] [row1] "+r"(row1), // %[row1] [row1_next] "+r"(row1_next), // %[row1_next] [num_windows] "+r"(num_windows), // %[num_windows] - [vl] "=&r"(vl), // %[vl] [total] "=w"(total) // %[total] - : [c1] "r"(26634), // %[c1] + : [vl] "r"(vl), // %[vl] + [vl2] "r"(vl * 2), // %[vl2] + [vl3] "r"(vl * 3), // %[vl3] + [c1] "r"(26634), // %[c1] [c2] "r"(239708) // %[c2] : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", - "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", - "p0", "p1"); + "z17", "z18", "z19", "z24", "z25", "z26", "p0", "p1", "p2", "p3", "p4", + "p5"); return total; } @@ -2398,11 +2723,11 @@ static inline void ARGBOrRow_SVE_SC(const uint8_t* src_argb, : "memory", "cc", "z0", "z1", "z2", "p0", "p1"); } -#define ARGBALPHABLEND_SVE_2X(pred0, pred1) \ - "ld1w {z0.s}, " #pred0 "/z, [%[src0]] \n" \ - "ld1w {z1.s}, " #pred1 "/z, [%[src0], #1, mul vl] \n" \ - "ld1w {z2.s}, " #pred0 "/z, [%[src1]] \n" \ - "ld1w {z3.s}, " #pred1 "/z, [%[src1], #1, mul vl] \n" \ +#define ARGBALPHABLEND_SVE_2X \ + "ld1w {z0.s}, p1/z, [%[src0]] \n" \ + "ld1w {z1.s}, p2/z, [%[src0], #1, mul vl] \n" \ + "ld1w {z2.s}, p1/z, [%[src1]] \n" \ + "ld1w {z3.s}, p2/z, [%[src1], #1, mul vl] \n" \ "incb %[src0], all, mul #2 \n" \ "incb %[src1], all, mul #2 \n" \ "lsr z4.s, z0.s, #24 \n" \ @@ -2429,8 +2754,22 @@ static inline void ARGBOrRow_SVE_SC(const uint8_t* src_argb, "sub z7.s, z7.s, #1 \n" \ "asr z6.s, z6.s, #31 \n" \ "asr z7.s, z7.s, #31 \n" \ - "udiv z18.s, p0/m, z18.s, z16.s \n" \ - "udiv z19.s, p0/m, z19.s, z17.s \n" \ + "ucvtf z18.s, p0/m, z18.s \n" \ + "ucvtf z19.s, p0/m, z19.s \n" \ + "ucvtf z20.s, p0/m, z16.s \n" \ + "ucvtf z21.s, p0/m, z17.s \n" \ + "fadd z18.s, p0/m, z18.s, #0.5 \n" \ + "fadd z19.s, p0/m, z19.s, #0.5 \n" \ + "frecpe z22.s, z20.s \n" \ + "frecpe z23.s, z21.s \n" \ + "frecps z20.s, z20.s, z22.s \n" \ + "frecps z21.s, z21.s, z23.s \n" \ + "fmul z22.s, z22.s, z20.s \n" \ + "fmul z23.s, z23.s, z21.s \n" \ + "fmul z18.s, z18.s, z22.s \n" \ + "fmul z19.s, z19.s, z23.s \n" \ + "fcvtzu z18.s, p0/m, z18.s \n" \ + "fcvtzu z19.s, p0/m, z19.s \n" \ "mul z18.s, z18.s, z25.s \n" \ "mul z19.s, z19.s, z25.s \n" \ "orr z18.d, z18.d, z6.d \n" \ @@ -2463,8 +2802,8 @@ static inline void ARGBOrRow_SVE_SC(const uint8_t* src_argb, "lsl z17.s, z17.s, #24 \n" \ "orr z0.d, z0.d, z16.d \n" \ "orr z1.d, z1.d, z17.d \n" \ - "st1w {z0.s}, " #pred0 ", [%[dst]] \n" \ - "st1w {z1.s}, " #pred1 ", [%[dst], #1, mul vl] \n" \ + "st1w {z0.s}, p1, [%[dst]] \n" \ + "st1w {z1.s}, p2, [%[dst], #1, mul vl] \n" \ "incb %[dst], all, mul #2 \n" static inline void ARGBAlphaBlendRow_SVE_SC(const uint8_t* src_argb0, @@ -2476,13 +2815,15 @@ static inline void ARGBAlphaBlendRow_SVE_SC(const uint8_t* src_argb0, "cntw %x[vl] \n" "ptrue p0.b \n" "mov z24.h, #1 \n" - "mov z25.s, #0x01010101 \n" + "mov z25.b, #1 \n" "mov z26.s, #0x00ffffff \n" "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" - "1: \n" - ARGBALPHABLEND_SVE_2X(p0, p0) + "ptrue p1.s \n" + "ptrue p2.s \n" + "1: \n" // + ARGBALPHABLEND_SVE_2X "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.ge 1b \n" @@ -2491,8 +2832,8 @@ static inline void ARGBAlphaBlendRow_SVE_SC(const uint8_t* src_argb0, "b.eq 99f \n" "whilelt p1.s, wzr, %w[width] \n" - "whilelt p2.s, %w[vl], %w[width] \n" - ARGBALPHABLEND_SVE_2X(p1, p2) + "whilelt p2.s, %w[vl], %w[width] \n" // + ARGBALPHABLEND_SVE_2X "99: \n" : [src0] "+r"(src_argb0), // %[src0] @@ -2502,11 +2843,1359 @@ static inline void ARGBAlphaBlendRow_SVE_SC(const uint8_t* src_argb0, [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", - "z17", "z18", "z19", "z20", "z21", "z24", "z25", "z26", "p0", "p1", - "p2"); + "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", + "p0", "p1", "p2"); } -#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) +#define ARGBTORGB565_SVE \ + /* Inputs: \ + * z0: rrrrrxxxbbbbbxxx \ + * z1: xxxxxxxxggggggxx \ + * z3: 0000000000000011 (3, 0, 3, 0, ...) \ + * z4: 0000011111100000 \ + */ \ + "lsr z0.b, p0/m, z0.b, z3.b \n" \ + "lsl z1.h, z1.h, #3 \n" \ + "bsl z1.d, z1.d, z0.d, z4.d \n" + +static inline void ARGBToRGB565Row_SVE_SC(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) STREAMING_COMPATIBLE { + unsigned bsl_mask = 0x7e0; + uint64_t vl; + width *= 2; + asm volatile( + "mov z3.h, #3 \n" + "dup z4.h, %w[bsl_mask] \n" + + "cntb %[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA + "incb %[src], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl] \n" // + ARGBTORGB565_SVE + "st1b {z1.b}, p0, [%[dst]] \n" + "incb %[dst] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA + ARGBTORGB565_SVE + "st1b {z1.b}, p0, [%[dst]] \n" + + "99: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_rgb), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [bsl_mask] "r"(bsl_mask) // %[bsl_mask] + : "memory", "cc", "z0", "z1", "z3", "z4", "p0"); +} + +static inline void ARGBToRGB565DitherRow_SVE_SC(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width) + STREAMING_COMPATIBLE { + unsigned bsl_mask = 0x7e0; + uint64_t vl; + width *= 2; + asm volatile( + "mov z3.h, #3 \n" + "dup z4.h, %w[bsl_mask] \n" + "dup z2.s, %w[dither4] \n" + "zip1 z2.b, z2.b, z2.b \n" + + "cntb %[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA + "incb %[src], all, mul #2 \n" + "uqadd z0.b, z0.b, z2.b \n" + "uqadd z1.b, z1.b, z2.b \n" + "subs %w[width], %w[width], %w[vl] \n" // + ARGBTORGB565_SVE + "st1b {z1.b}, p0, [%[dst]] \n" + "incb %[dst] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA + "uqadd z0.b, z0.b, z2.b \n" + "uqadd z1.b, z1.b, z2.b \n" // + ARGBTORGB565_SVE + "st1b {z1.b}, p0, [%[dst]] \n" + + "99: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_rgb), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [bsl_mask] "r"(bsl_mask), // %[bsl_mask] + [dither4] "r"(dither4) // %[dither4] + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "p0"); +} + +#define ARGB1555TOARGB_SVE \ + /* Input: z1/z3.h = arrrrrgggggbbbbb */ \ + "lsl z0.h, z1.h, #3 \n" /* rrrgggggbbbbb000 */ \ + "lsl z2.h, z3.h, #3 \n" /* rrrgggggbbbbb000 */ \ + "asr z1.h, z1.h, #7 \n" /* aaaaaaaarrrrrggg */ \ + "asr z3.h, z3.h, #7 \n" /* aaaaaaaarrrrrggg */ \ + "lsl z0.b, p0/m, z0.b, z4.b \n" /* ggggg000bbbbb000 */ \ + "lsl z2.b, p0/m, z2.b, z4.b \n" /* ggggg000bbbbb000 */ \ + "sri z1.b, z1.b, #5 \n" /* aaaaaaaarrrrrrrr */ \ + "sri z3.b, z3.b, #5 \n" /* aaaaaaaarrrrrrrr */ \ + "sri z0.b, z0.b, #5 \n" /* ggggggggbbbbbbbb */ \ + "sri z2.b, z2.b, #5 \n" /* ggggggggbbbbbbbb */ + +static inline void ARGB1555ToARGBRow_SVE_SC(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width) STREAMING_COMPATIBLE { + uint64_t vl; + asm volatile( + "mov z4.h, #0x0300 \n" + "ptrue p0.b \n" + + "cnth %x[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + "1: \n" + "ld1h {z1.h}, p0/z, [%[src]] \n" + "ld1h {z3.h}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" // + ARGB1555TOARGB_SVE + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "st2h {z0.h, z1.h}, p0, [%[dst]] \n" + "st2h {z2.h, z3.h}, p0, [%[dst], #2, mul vl] \n" + "incb %[dst], all, mul #4 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.h, %w[vl], %w[width] \n" + "ld1h {z1.h}, p1/z, [%[src]] \n" + "ld1h {z3.h}, p2/z, [%[src], #1, mul vl] \n" // + ARGB1555TOARGB_SVE + "st2h {z0.h, z1.h}, p1, [%[dst]] \n" + "st2h {z2.h, z3.h}, p2, [%[dst], #2, mul vl] \n" + + "99: \n" + : [src] "+r"(src_argb1555), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2"); +} + +// clang-format off +#define AYUVTOUV_SVE(zU0, zV0, zU1, zV1) /* e.g. */ \ + "ld2h {z0.h, z1.h}, p0/z, [%[src0]] \n" /* VUVU.. YAYA.. */ \ + "ld2h {z1.h, z2.h}, p1/z, [%[src0], #2, mul vl] \n" /* VUVU.. YAYA.. */ \ + "ld2h {z2.h, z3.h}, p0/z, [%[src1]] \n" /* VUVU.. YAYA.. */ \ + "ld2h {z3.h, z4.h}, p1/z, [%[src1], #2, mul vl] \n" /* VUVU.. YAYA.. */ \ + "incb %[src0], all, mul #4 \n" \ + "incb %[src1], all, mul #4 \n" \ + "uaddlb z4.h, z0.b, z2.b \n" /* V */ \ + "uaddlt z5.h, z0.b, z2.b \n" /* U */ \ + "uaddlb z6.h, z1.b, z3.b \n" /* V */ \ + "uaddlt z7.h, z1.b, z3.b \n" /* U */ \ + "addp " #zU0 ".h, p0/m, " #zU0 ".h, " #zV0 ".h \n" /* UV */ \ + "addp " #zU1 ".h, p1/m, " #zU1 ".h, " #zV1 ".h \n" /* UV */ \ + "subs %w[width], %w[width], %w[vl] \n" \ + "urshr " #zU0 ".h, p0/m, " #zU0 ".h, #2 \n" /* U0V0 */ \ + "urshr " #zU1 ".h, p1/m, " #zU1 ".h, #2 \n" /* U0V0 */ \ + "st1b {" #zU0 ".h}, p0, [%[dst]] \n" \ + "st1b {" #zU1 ".h}, p1, [%[dst], #1, mul vl] \n" \ + "incb %[dst] \n" +// clang-format on + +// Filter 2 rows of AYUV UV's (444) into UV (420). +// AYUV is VUYA in memory. UV for NV12 is UV order in memory. +static inline void AYUVToUVRow_SVE_SC(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) STREAMING_COMPATIBLE { + // Output a row of UV values, filtering 2x2 rows of AYUV. + const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv; + if (width & 1) { + width &= ~1; + const uint8_t* last0 = src_ayuv + (ptrdiff_t)width * 4; + const uint8_t* last1 = src_ayuv1 + (ptrdiff_t)width * 4; + dst_uv[width] = (last0[1] + last1[1] + 1) >> 1; + dst_uv[width + 1] = (last0[0] + last1[0] + 1) >> 1; + if (width == 0) { + return; + } + } + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.h \n" + "ptrue p1.h \n" + "1: \n" + AYUVTOUV_SVE(z5, z4, z7, z6) + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "cnth %x[vl] \n" + "whilelt p0.h, wzr, %w[width] \n" // first row + "whilelt p1.h, %w[vl], %w[width] \n" // second row + AYUVTOUV_SVE(z5, z4, z7, z6) + + "99: \n" + : [src0] "+r"(src_ayuv), // %[src0] + [src1] "+r"(src_ayuv1), // %[src1] + [dst] "+r"(dst_uv), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0", + "p1"); +} + +// Filter 2 rows of AYUV UV's (444) into VU (420). +static inline void AYUVToVURow_SVE_SC(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) STREAMING_COMPATIBLE { + // Output a row of VU values, filtering 2x2 rows of AYUV. + const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv; + if (width & 1) { + width &= ~1; + const uint8_t* last0 = src_ayuv + (ptrdiff_t)width * 4; + const uint8_t* last1 = src_ayuv1 + (ptrdiff_t)width * 4; + dst_vu[width] = (last0[0] + last1[0] + 1) >> 1; + dst_vu[width + 1] = (last0[1] + last1[1] + 1) >> 1; + if (width == 0) { + return; + } + } + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.h \n" + "ptrue p1.h \n" + "1: \n" + AYUVTOUV_SVE(z4, z5, z6, z7) + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "cnth %x[vl] \n" + "whilelt p0.h, wzr, %w[width] \n" // first row + "whilelt p1.h, %w[vl], %w[width] \n" // second row + AYUVTOUV_SVE(z4, z5, z6, z7) + + "99: \n" + : [src0] "+r"(src_ayuv), // %[src0] + [src1] "+r"(src_ayuv1), // %[src1] + [dst] "+r"(dst_vu), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0", + "p1"); +} + +static inline void RAWToWXYZRow_SVE_SC(const uint8_t* src_raw, + uint8_t* dst_wxyz, + int width, + uint32_t idx_start, + uint32_t idx_step, + uint32_t alpha) STREAMING_COMPATIBLE { + uint64_t vl; + uint64_t vl_mul3; + uint32_t rem_mul3; + asm volatile( + "cntw %x[vl] \n" + "add %w[vl_mul3], %w[vl], %w[vl], lsl #1 \n" + "index z31.s, %w[idx_start], %w[idx_step] \n" + "dup z30.s, %w[alpha] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + // Run bulk of computation with the same predicates to avoid predicate + // generation overhead. We set up p1 to only load 3/4 of a vector. + "ptrue p0.s \n" + "whilelt p1.b, wzr, %w[vl_mul3] \n" + "1: \n" + "ld1b {z0.b}, p1/z, [%[src]] \n" + "add %[src], %[src], %x[vl_mul3] \n" + "ld1b {z1.b}, p1/z, [%[src]] \n" + "add %[src], %[src], %x[vl_mul3] \n" + "tbl z0.b, {z0.b}, z31.b \n" + "tbl z1.b, {z1.b}, z31.b \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "orr z0.d, z0.d, z30.d \n" + "orr z1.d, z1.d, z30.d \n" + "st1w {z0.s}, p0, [%[dst]] \n" + "st1w {z1.s}, p0, [%[dst], #1, mul vl] \n" + "incb %[dst], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + // Calculate a pair of predicates for the final iteration to deal with + // the tail. + "3: \n" + "add %w[rem_mul3], %w[width], %w[width], lsl #1 \n" + "whilelt p0.s, wzr, %w[width] \n" + "whilelt p1.b, wzr, %w[rem_mul3] \n" + "ld1b {z0.b}, p1/z, [%[src]] \n" + "add %[src], %[src], %x[vl_mul3] \n" + "tbl z0.b, {z0.b}, z31.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "orr z0.d, z0.d, z30.d \n" + "st1w {z0.s}, p0, [%[dst]] \n" + "incb %[dst] \n" + "b.gt 3b \n" + + "99: \n" + : [src] "+r"(src_raw), // %[src] + [dst] "+r"(dst_wxyz), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [vl_mul3] "=&r"(vl_mul3), // %[vl_mul3] + [rem_mul3] "=&r"(rem_mul3) // %[rem_mul3] + : [idx_start] "r"(idx_start), // %[idx_start] + [idx_step] "r"(idx_step), // %[idx_step] + [alpha] "r"(alpha) // %[alpha] + : "memory", "cc", "z0", "z1", "z30", "z31", "p0", "p1"); +} + +static inline void RAWToARGBRow_SVE_SC(const uint8_t* src_raw, + uint8_t* dst_argb, + int width) STREAMING_COMPATIBLE { + RAWToWXYZRow_SVE_SC(src_raw, dst_argb, width, 0xff000102U, 0x00030303U, + 0xff000000U); +} + +static inline void RAWToRGBARow_SVE_SC(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width) STREAMING_COMPATIBLE { + RAWToWXYZRow_SVE_SC(src_raw, dst_rgba, width, 0x000102ffU, 0x03030300U, + 0x000000ffU); +} + +static inline void RGB24ToARGBRow_SVE_SC(const uint8_t* src_rgb24, + uint8_t* dst_argb, + int width) STREAMING_COMPATIBLE { + RAWToWXYZRow_SVE_SC(src_rgb24, dst_argb, width, 0xff020100U, 0x00030303U, + 0xff000000U); +} + +static const uint8_t kRAWToRGB24Indices[] = { + 2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12, + 17, 16, 15, 20, 19, 18, 23, 22, 21, 26, 25, 24, 29, 28, 27, + 32, 31, 30, 35, 34, 33, 38, 37, 36, 41, 40, 39, 44, 43, 42, + 47, 46, 45, 50, 49, 48, 53, 52, 51, 56, 55, 54, 59, 58, 57, + 62, 61, 60, 65, 64, 63, 68, 67, 66, 71, 70, 69, 74, 73, 72, + 77, 76, 75, 80, 79, 78, 83, 82, 81, 86, 85, 84, 89, 88, 87, + 92, 91, 90, 95, 94, 93, 98, 97, 96, 101, 100, 99, 104, 103, 102, + 107, 106, 105, 110, 109, 108, 113, 112, 111, 116, 115, 114, 119, 118, 117, + 122, 121, 120, 125, 124, 123, 128, 127, 126, 131, 130, 129, 134, 133, 132, + 137, 136, 135, 140, 139, 138, 143, 142, 141, 146, 145, 144, 149, 148, 147, + 152, 151, 150, 155, 154, 153, 158, 157, 156, 161, 160, 159, 164, 163, 162, + 167, 166, 165, 170, 169, 168, 173, 172, 171, 176, 175, 174, 179, 178, 177, + 182, 181, 180, 185, 184, 183, 188, 187, 186, 191, 190, 189, 194, 193, 192, + 197, 196, 195, 200, 199, 198, 203, 202, 201, 206, 205, 204, 209, 208, 207, + 212, 211, 210, 215, 214, 213, 218, 217, 216, 221, 220, 219, 224, 223, 222, + 227, 226, 225, 230, 229, 228, 233, 232, 231, 236, 235, 234, 239, 238, 237, + 242, 241, 240, 245, 244, 243, 248, 247, 246, 251, 250, 249, 254, 253, 252}; + +static inline void RAWToRGB24Row_SVE_SC(const uint8_t* src_raw, + uint8_t* dst_rgb24, + int width) STREAMING_COMPATIBLE { + // width is in elements, convert to bytes. + width *= 3; + // we use the mul3 predicate pattern throughout to use the largest multiple + // of three number of lanes, for instance with a vector length of 16 bytes + // only the first 15 bytes will be used for load/store instructions. + uint32_t vl; + asm volatile( + "cntb %x[vl], mul3 \n" + "ptrue p0.b, mul3 \n" + "ld1b {z31.b}, p0/z, [%[kIndices]] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + // Run bulk of computation with the same predicate to avoid predicate + // generation overhead. + "1: \n" + "ld1b {z0.b}, p0/z, [%[src]] \n" + "add %[src], %[src], %x[vl] \n" + "tbl z0.b, {z0.b}, z31.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z0.b}, p0, [%[dst]] \n" + "add %[dst], %[dst], %x[vl] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + // Calculate a predicate for the final iteration to deal with the tail. + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z0.b}, p0/z, [%[src]] \n" + "tbl z0.b, {z0.b}, z31.b \n" + "st1b {z0.b}, p0, [%[dst]] \n" + + "99: \n" + : [src] "+r"(src_raw), // %[src] + [dst] "+r"(dst_rgb24), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [kIndices] "r"(kRAWToRGB24Indices) // %[kIndices] + : "memory", "cc", "z0", "z31", "p0"); +} + +static inline void ARGBToXYZRow_SVE_SC(const uint8_t* src_argb, + uint8_t* dst_xyz, + int width, + const uint8_t* indices) + STREAMING_COMPATIBLE { + uint64_t vl; + uint64_t vl_mul3; + uint32_t rem_mul3; + asm volatile( + "cntw %x[vl] \n" + "add %w[vl_mul3], %w[vl], %w[vl], lsl #1 \n" + "whilelt p1.b, wzr, %w[vl_mul3] \n" + "ld1b {z31.b}, p1/z, [%[indices]] \n" + "subs %w[width], %w[width], %w[vl_mul3] \n" + "b.lt 2f \n" + + // Run bulk of computation with an all-true predicate to avoid partial + // predicated store overhead. Ensure at least vl elements remain for the + // tail so full-vector stores never write past dst_xyz + width * 3. + "ptrue p0.b \n" + "1: \n" + "ld1w {z0.s}, p0/z, [%[src]] \n" + "ld1w {z1.s}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" + "tbl z0.b, {z0.b}, z31.b \n" + "tbl z1.b, {z1.b}, z31.b \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "st1b {z0.b}, p0, [%[dst]] \n" + "add %[dst], %[dst], %x[vl_mul3] \n" + "st1b {z1.b}, p0, [%[dst]] \n" + "add %[dst], %[dst], %x[vl_mul3] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl_mul3] \n" + "b.eq 99f \n" + + // Calculate predicates for the remaining iterations to deal with the + // tail. + "3: \n" + "add %w[rem_mul3], %w[width], %w[width], lsl #1 \n" + "whilelt p0.s, wzr, %w[width] \n" + "whilelt p1.b, wzr, %w[rem_mul3] \n" + "ld1w {z0.s}, p0/z, [%[src]] \n" + "incb %[src] \n" + "tbl z0.b, {z0.b}, z31.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z0.b}, p1, [%[dst]] \n" + "add %[dst], %[dst], %x[vl_mul3] \n" + "b.gt 3b \n" + + "99: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_xyz), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [vl_mul3] "=&r"(vl_mul3), // %[vl_mul3] + [rem_mul3] "=&r"(rem_mul3) // %[rem_mul3] + : [indices] "r"(indices) // %[indices] + : "memory", "cc", "z0", "z1", "z31", "p0", "p1"); +} + +static const uint8_t kARGBToRGB24RowIndices[] = { + 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, + 20, 21, 22, 24, 25, 26, 28, 29, 30, 32, 33, 34, 36, 37, 38, + 40, 41, 42, 44, 45, 46, 48, 49, 50, 52, 53, 54, 56, 57, 58, + 60, 61, 62, 64, 65, 66, 68, 69, 70, 72, 73, 74, 76, 77, 78, + 80, 81, 82, 84, 85, 86, 88, 89, 90, 92, 93, 94, 96, 97, 98, + 100, 101, 102, 104, 105, 106, 108, 109, 110, 112, 113, 114, 116, 117, 118, + 120, 121, 122, 124, 125, 126, 128, 129, 130, 132, 133, 134, 136, 137, 138, + 140, 141, 142, 144, 145, 146, 148, 149, 150, 152, 153, 154, 156, 157, 158, + 160, 161, 162, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178, + 180, 181, 182, 184, 185, 186, 188, 189, 190, 192, 193, 194, 196, 197, 198, + 200, 201, 202, 204, 205, 206, 208, 209, 210, 212, 213, 214, 216, 217, 218, + 220, 221, 222, 224, 225, 226, 228, 229, 230, 232, 233, 234, 236, 237, 238, + 240, 241, 242, 244, 245, 246, 248, 249, 250, 252, 253, 254, +}; + +static const uint8_t kARGBToRAWRowIndices[] = { + 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 18, 17, 16, + 22, 21, 20, 26, 25, 24, 30, 29, 28, 34, 33, 32, 38, 37, 36, + 42, 41, 40, 46, 45, 44, 50, 49, 48, 54, 53, 52, 58, 57, 56, + 62, 61, 60, 66, 65, 64, 70, 69, 68, 74, 73, 72, 78, 77, 76, + 82, 81, 80, 86, 85, 84, 90, 89, 88, 94, 93, 92, 98, 97, 96, + 102, 101, 100, 106, 105, 104, 110, 109, 108, 114, 113, 112, 118, 117, 116, + 122, 121, 120, 126, 125, 124, 130, 129, 128, 134, 133, 132, 138, 137, 136, + 142, 141, 140, 146, 145, 144, 150, 149, 148, 154, 153, 152, 158, 157, 156, + 162, 161, 160, 166, 165, 164, 170, 169, 168, 174, 173, 172, 178, 177, 176, + 182, 181, 180, 186, 185, 184, 190, 189, 188, 194, 193, 192, 198, 197, 196, + 202, 201, 200, 206, 205, 204, 210, 209, 208, 214, 213, 212, 218, 217, 216, + 222, 221, 220, 226, 225, 224, 230, 229, 228, 234, 233, 232, 238, 237, 236, + 242, 241, 240, 246, 245, 244, 250, 249, 248, 254, 253, 252, +}; + +static inline void ARGBToRGB24Row_SVE_SC(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) STREAMING_COMPATIBLE { + ARGBToXYZRow_SVE_SC(src_argb, dst_rgb, width, kARGBToRGB24RowIndices); +} + +static inline void ARGBToRAWRow_SVE_SC(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) STREAMING_COMPATIBLE { + ARGBToXYZRow_SVE_SC(src_argb, dst_rgb, width, kARGBToRAWRowIndices); +} + +static inline void CopyRow_SVE_SC(const uint8_t* src, + uint8_t* dst, + int width) STREAMING_COMPATIBLE { + int vl; + int vl2; + asm volatile( + "cntb %x[vl] \n" + "lsl %w[vl2], %w[vl], #1 \n" + "subs %w[width], %w[width], %w[vl2] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z0.b}, p0/z, [%[src], #0, mul vl] \n" + "ld1b {z1.b}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl2] \n" + "st1b {z0.b}, p0, [%[dst], #0, mul vl] \n" + "st1b {z1.b}, p0, [%[dst], #1, mul vl] \n" + "incb %[dst], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl2] \n" + "b.eq 99f \n" + + "3: \n" + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z0.b}, p0/z, [%[src]] \n" + "incb %[src] \n" + "st1b {z0.b}, p0, [%[dst]] \n" + "incb %[dst] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.gt 3b \n" + + "99: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [vl2] "=&r"(vl2) // %[vl2] + : + : "memory", "cc", "z0", "z1", "p0"); +} + +static inline void DivideRow_16_SVE_SC(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) STREAMING_COMPATIBLE { + if (scale == 65536) { // Divide by 1, so do copy. + if (src_y != dst_y) { + CopyRow_SVE_SC((const uint8_t*)src_y, (uint8_t*)dst_y, width * 2); + } + return; + } + uint64_t vl; + asm volatile( + "cnth %x[vl] \n" + "dup z0.h, %w[scale] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + // Run bulk of computation with the same predicates to avoid predicate + // generation overhead. + "ptrue p0.h \n" + "1: \n" + "ld1h {z1.h}, p0/z, [%[src]] \n" + "ld1h {z2.h}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" + "umulh z1.h, z1.h, z0.h \n" + "umulh z2.h, z2.h, z0.h \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "st1h {z1.h}, p0, [%[dst]] \n" + "st1h {z2.h}, p0, [%[dst], #1, mul vl] \n" + "incb %[dst], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + // Calculate a pair of predicates for the final iteration to deal with + // the tail. + "whilelt p0.h, wzr, %w[width] \n" + "whilelt p1.h, %w[vl], %w[width] \n" + "ld1h {z1.h}, p0/z, [%[src]] \n" + "ld1h {z2.h}, p1/z, [%[src], #1, mul vl] \n" + "umulh z1.h, z1.h, z0.h \n" + "umulh z2.h, z2.h, z0.h \n" + "st1h {z1.h}, p0, [%[dst]] \n" + "st1h {z2.h}, p1, [%[dst], #1, mul vl] \n" + + "99: \n" + : [src] "+r"(src_y), // %[src] + [dst] "+r"(dst_y), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [scale] "r"(scale) // %[scale] + : "memory", "cc", "z0", "z1", "z2", "p0", "p1"); +} + +#define HALFFLOAT_SVE \ + "lsr z1.s, z0.s, #16 \n" \ + "and z0.s, z0.s, #0xffff \n" \ + "lsr z3.s, z2.s, #16 \n" \ + "and z2.s, z2.s, #0xffff \n" \ + "scvtf z0.s, p2/m, z0.s \n" \ + "scvtf z1.s, p2/m, z1.s \n" \ + "scvtf z2.s, p2/m, z2.s \n" \ + "scvtf z3.s, p2/m, z3.s \n" \ + "fmul z0.s, z0.s, z4.s \n" \ + "fmul z1.s, z1.s, z4.s \n" \ + "fmul z2.s, z2.s, z4.s \n" \ + "fmul z3.s, z3.s, z4.s \n" \ + "fmul z0.s, z0.s, z5.s \n" \ + "fmul z1.s, z1.s, z5.s \n" \ + "fmul z2.s, z2.s, z5.s \n" \ + "fmul z3.s, z3.s, z5.s \n" \ + "uqshrnb z0.h, z0.s, #13 \n" \ + "uqshrnb z2.h, z2.s, #13 \n" \ + "uqshrnt z0.h, z1.s, #13 \n" \ + "uqshrnt z2.h, z3.s, #13 \n" + +static inline void HalfFloatRow_SVE_SC(const uint16_t* src, + uint16_t* dst, + float scale, + int width) STREAMING_COMPATIBLE { + uint64_t vl; + asm volatile( + "cnth %x[vl] \n" + "mov z4.s, %s[scale] \n" + "mov z5.s, %s[bias] \n" + "ptrue p2.s \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + // Run bulk of computation with all-true predicates to avoid predicate + // generation overhead. + "ptrue p0.h \n" + "1: \n" + "ld1h {z0.h}, p0/z, [%[src]] \n" + "ld1h {z2.h}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" // + HALFFLOAT_SVE + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "st1h {z0.h}, p0, [%[dst]] \n" + "st1h {z2.h}, p0, [%[dst], #1, mul vl] \n" + "incb %[dst], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p0.h, wzr, %w[width] \n" + "whilelt p1.h, %w[vl], %w[width] \n" + "ld1h {z0.h}, p0/z, [%[src]] \n" + "ld1h {z2.h}, p1/z, [%[src], #1, mul vl] \n" // + HALFFLOAT_SVE + "st1h {z0.h}, p0, [%[dst]] \n" + "st1h {z2.h}, p1, [%[dst], #1, mul vl] \n" + + "99: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [scale] "w"(scale), // %[scale] + [bias] "w"(1.9259299444e-34f) // %[bias] 2^-112 + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "p0", "p1", "p2"); +} + +static inline void HalfFloat1Row_SVE_SC(const uint16_t* src, + uint16_t* dst, + float scale, + int width) STREAMING_COMPATIBLE { + (void)scale; + uint64_t vl; + asm volatile( + "cnth %x[vl] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + // Run bulk of computation with all-true predicates to avoid predicate + // generation overhead. + "ptrue p0.h \n" + "1: \n" + "ld1h {z0.h}, p0/z, [%[src]] \n" + "ld1h {z1.h}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" + "ucvtf z0.h, p0/m, z0.h \n" + "ucvtf z1.h, p0/m, z1.h \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "st1h {z0.h}, p0, [%[dst]] \n" + "st1h {z1.h}, p0, [%[dst], #1, mul vl] \n" + "incb %[dst], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + // Calculate predicates for the final iteration to deal with the tail. + "whilelt p0.h, wzr, %w[width] \n" + "whilelt p1.h, %w[vl], %w[width] \n" + "ld1h {z0.h}, p0/z, [%[src]] \n" + "ld1h {z1.h}, p1/z, [%[src], #1, mul vl] \n" + "ucvtf z0.h, p0/m, z0.h \n" + "ucvtf z1.h, p0/m, z1.h \n" + "st1h {z0.h}, p0, [%[dst]] \n" + "st1h {z1.h}, p1, [%[dst], #1, mul vl] \n" + + "99: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "p0", "p1"); +} + +static inline void MergeUVRow_SVE_SC(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width) STREAMING_COMPATIBLE { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z1.b}, p0/z, [%[src_u]] \n" + "ld1b {z2.b}, p0/z, [%[src_v]] \n" + "incb %[src_u] \n" + "incb %[src_v] \n" + "subs %w[width], %w[width], %w[vl] \n" + "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" + "incb %[dst_uv], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z1.b}, p0/z, [%[src_u]] \n" + "ld1b {z2.b}, p0/z, [%[src_v]] \n" + "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" + + "99: \n" + : [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z1", "z2", "p0"); +} + +static inline void SplitUVRow_SVE_SC(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) STREAMING_COMPATIBLE { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n" + "incb %[src_uv], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z1.b}, p0, [%[dst_u]] \n" + "st1b {z2.b}, p0, [%[dst_v]] \n" + "incb %[dst_u] \n" + "incb %[dst_v] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n" + "st1b {z1.b}, p0, [%[dst_u]] \n" + "st1b {z2.b}, p0, [%[dst_v]] \n" + + "99: \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z1", "z2", "p0"); +} + +static inline void HalfMergeUVRow_SVE_SC(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width) STREAMING_COMPATIBLE { + const uint8_t* src_u1 = src_u + src_stride_u; + const uint8_t* src_v1 = src_v + src_stride_v; + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z1.b}, p0/z, [%[src_u]] \n" + "ld1b {z2.b}, p0/z, [%[src_u1]] \n" + "ld1b {z3.b}, p0/z, [%[src_v]] \n" + "ld1b {z4.b}, p0/z, [%[src_v1]] \n" + "incb %[src_u] \n" + "incb %[src_u1] \n" + "incb %[src_v] \n" + "incb %[src_v1] \n" + "uaddlb z5.h, z1.b, z2.b \n" + "uaddlb z6.h, z3.b, z4.b \n" + "uaddlt z1.h, z1.b, z2.b \n" + "uaddlt z3.h, z3.b, z4.b \n" + "add z1.h, z1.h, z5.h \n" + "add z3.h, z3.h, z6.h \n" + "rshrnb z1.b, z1.h, #2 \n" + "rshrnt z1.b, z3.h, #2 \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z1.b}, p0, [%[dst_uv]] \n" + "incb %[dst_uv] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z1.b}, p0/z, [%[src_u]] \n" + "ld1b {z2.b}, p0/z, [%[src_u1]] \n" + "ld1b {z3.b}, p0/z, [%[src_v]] \n" + "ld1b {z4.b}, p0/z, [%[src_v1]] \n" + "uaddlb z5.h, z1.b, z2.b \n" + "uaddlb z6.h, z3.b, z4.b \n" + "uaddlt z1.h, z1.b, z2.b \n" + "uaddlt z3.h, z3.b, z4.b \n" + "add z1.h, z1.h, z5.h \n" + "add z3.h, z3.h, z6.h \n" + "rshrnb z1.b, z1.h, #2 \n" + "rshrnt z1.b, z3.h, #2 \n" + "st1b {z1.b}, p0, [%[dst_uv]] \n" + + "99: \n" + : [src_u] "+r"(src_u), // %[src_u] + [src_u1] "+r"(src_u1), // %[src_u1] + [src_v] "+r"(src_v), // %[src_v] + [src_v1] "+r"(src_v1), // %[src_v1] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z1", "z2", "z3", "z4", "z5", "z6", "p0"); +} + +static inline void HalfRow_SVE_SC(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int width) STREAMING_COMPATIBLE { + const uint8_t* src_ptr1 = src_ptr + src_stride; + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "incb %[src_ptr] \n" + "incb %[src_ptr1] \n" + "subs %w[width], %w[width], %w[vl] \n" + "urhadd z2.b, p0/m, z2.b, z3.b \n" + "st1b {z2.b}, p0, [%[dst_ptr]] \n" + "incb %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "urhadd z2.b, p0/m, z2.b, z3.b \n" + "st1b {z2.b}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z2", "z3", "p0"); +} + +static inline void HalfRow_16_SVE_SC(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int width) STREAMING_COMPATIBLE { + const uint16_t* src_ptr1 = src_ptr + src_stride; + int vl; + asm volatile( + "cnth %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.h \n" + "1: \n" + "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" + "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" + "incb %[src_ptr] \n" + "incb %[src_ptr1] \n" + "urhadd z2.h, p0/m, z2.h, z3.h \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1h {z2.h}, p0, [%[dst_ptr]] \n" + "incb %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.h, wzr, %w[width] \n" + "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" + "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" + "urhadd z2.h, p0/m, z2.h, z3.h \n" + "st1h {z2.h}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z2", "z3", "p0"); +} + +static inline void InterpolateRow_SVE_SC(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) + STREAMING_COMPATIBLE { + int y1_fraction = source_y_fraction; + int y0_fraction = 256 - y1_fraction; + const uint8_t* src_ptr1 = src_ptr + src_stride; + + if (y0_fraction == 0) { + CopyRow_SVE_SC(src_ptr1, dst_ptr, width); + return; + } + if (y0_fraction == 128) { + HalfRow_SVE_SC(src_ptr, src_stride, dst_ptr, width); + return; + } + if (y0_fraction == 256) { + CopyRow_SVE_SC(src_ptr, dst_ptr, width); + return; + } + + int vl; + asm volatile( + "cntb %x[vl] \n" + "dup z0.b, %w[y0_fraction] \n" + "dup z1.b, %w[y1_fraction] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "incb %[src_ptr] \n" + "incb %[src_ptr1] \n" + "umullb z4.h, z2.b, z0.b \n" + "umullt z2.h, z2.b, z0.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "umlalb z4.h, z3.b, z1.b \n" + "umlalt z2.h, z3.b, z1.b \n" + "rshrnb z3.b, z4.h, #8 \n" + "rshrnt z3.b, z2.h, #8 \n" + "st1b {z3.b}, p0, [%[dst_ptr]] \n" + "incb %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "umullb z4.h, z2.b, z0.b \n" + "umullt z2.h, z2.b, z0.b \n" + "umlalb z4.h, z3.b, z1.b \n" + "umlalt z2.h, z3.b, z1.b \n" + "rshrnb z3.b, z4.h, #8 \n" + "rshrnt z3.b, z2.h, #8 \n" + "st1b {z3.b}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] + [y1_fraction] "r"(y1_fraction) // %[y1_fraction] + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "p0"); +} + +static inline void InterpolateRow_16_SVE_SC(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) + STREAMING_COMPATIBLE { + int y1_fraction = source_y_fraction; + int y0_fraction = 256 - y1_fraction; + const uint16_t* src_ptr1 = src_ptr + src_stride; + + if (y0_fraction == 0) { + CopyRow_SVE_SC((const uint8_t*)src_ptr1, (uint8_t*)dst_ptr, + width * sizeof(uint16_t)); + return; + } + if (y0_fraction == 128) { + HalfRow_16_SVE_SC(src_ptr, src_stride, dst_ptr, width); + return; + } + if (y0_fraction == 256) { + CopyRow_SVE_SC((const uint8_t*)src_ptr, (uint8_t*)dst_ptr, + width * sizeof(uint16_t)); + return; + } + + int vl; + asm volatile( + "cnth %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "dup z0.h, %w[y0_fraction] \n" + "dup z1.h, %w[y1_fraction] \n" + "b.lt 2f \n" + + "ptrue p0.h \n" + "1: \n" + "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" + "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" + "incb %[src_ptr] \n" + "incb %[src_ptr1] \n" + "umullb z4.s, z2.h, z0.h \n" + "umullt z2.s, z2.h, z0.h \n" + "subs %w[width], %w[width], %w[vl] \n" + "umlalb z4.s, z3.h, z1.h \n" + "umlalt z2.s, z3.h, z1.h \n" + "rshrnb z3.h, z4.s, #8 \n" + "rshrnt z3.h, z2.s, #8 \n" + "st1h {z3.h}, p0, [%[dst_ptr]] \n" + "incb %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.h, wzr, %w[width] \n" + "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" + "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" + "umullb z4.s, z2.h, z0.h \n" + "umullt z2.s, z2.h, z0.h \n" + "umlalb z4.s, z3.h, z1.h \n" + "umlalt z2.s, z3.h, z1.h \n" + "rshrnb z3.h, z4.s, #8 \n" + "rshrnt z3.h, z2.s, #8 \n" + "st1h {z3.h}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] + [y1_fraction] "r"(y1_fraction) // %[y1_fraction] + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "p0"); +} + +static inline void YUY2ToYRow_SVE_SC(const uint8_t* src_yuy2, + uint8_t* dst_y, + int width) STREAMING_COMPATIBLE { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" + "incb %[src_yuy2], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z0.b}, p0, [%[dst_y]] \n" + "incb %[dst_y] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" + "st1b {z0.b}, p0, [%[dst_y]] \n" + + "99: \n" + : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2] + [dst_y] "+r"(dst_y), // %[dst_y] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "p0"); +} + +static inline void UYVYToYRow_SVE_SC(const uint8_t* src_uyvy, + uint8_t* dst_y, + int width) STREAMING_COMPATIBLE { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" + "incb %[src_uyvy], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z1.b}, p0, [%[dst_y]] \n" + "incb %[dst_y] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" + "st1b {z1.b}, p0, [%[dst_y]] \n" + + "99: \n" + : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy] + [dst_y] "+r"(dst_y), // %[dst_y] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "p0"); +} + +static inline void YUY2ToUVRow_SVE_SC(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) STREAMING_COMPATIBLE { + const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2; + int vl; + int vl_half; + int width_half; + if (src_stride_yuy2 == 0) { + src_yuy2_1 = src_yuy2; + } + asm volatile( + "cntb %x[vl] \n" + "lsr %w[vl_half], %w[vl], #1 \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "whilelt p1.b, wzr, %w[vl_half] \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" + "ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n" + "incb %[src_yuy2], all, mul #2 \n" + "incb %[src_yuy2_1], all, mul #2 \n" + "urhadd z1.b, p0/m, z1.b, z5.b \n" + "uzp1 z0.b, z1.b, z1.b \n" + "uzp2 z2.b, z1.b, z1.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z0.b}, p1, [%[dst_u]] \n" + "st1b {z2.b}, p1, [%[dst_v]] \n" + "add %[dst_u], %[dst_u], %x[vl_half] \n" + "add %[dst_v], %[dst_v], %x[vl_half] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "add %w[width_half], %w[width], #1 \n" + "lsr %w[width_half], %w[width_half], #1 \n" + "whilelt p1.b, wzr, %w[width_half] \n" + "zip1 p0.b, p1.b, p1.b \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" + "ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n" + "urhadd z1.b, p0/m, z1.b, z5.b \n" + "uzp1 z0.b, z1.b, z1.b \n" + "uzp2 z2.b, z1.b, z1.b \n" + "st1b {z0.b}, p1, [%[dst_u]] \n" + "st1b {z2.b}, p1, [%[dst_v]] \n" + + "99: \n" + : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2] + [src_yuy2_1] "+r"(src_yuy2_1), // %[src_yuy2_1] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [vl_half] "=&r"(vl_half), // %[vl_half] + [width_half] "=&r"(width_half) // %[width_half] + : + : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1"); +} + +static inline void UYVYToUVRow_SVE_SC(const uint8_t* src_uyvy, + int src_stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) STREAMING_COMPATIBLE { + const uint8_t* src_uyvy_1 = src_uyvy + src_stride_uyvy; + int vl; + int vl_half; + int width_half; + if (src_stride_uyvy == 0) { + src_uyvy_1 = src_uyvy; + } + asm volatile( + "cntb %x[vl] \n" + "lsr %w[vl_half], %w[vl], #1 \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "whilelt p1.b, wzr, %w[vl_half] \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" + "ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n" + "incb %[src_uyvy], all, mul #2 \n" + "incb %[src_uyvy_1], all, mul #2 \n" + "urhadd z0.b, p0/m, z0.b, z4.b \n" + "uzp1 z1.b, z0.b, z0.b \n" + "uzp2 z2.b, z0.b, z0.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z1.b}, p1, [%[dst_u]] \n" + "st1b {z2.b}, p1, [%[dst_v]] \n" + "add %[dst_u], %[dst_u], %x[vl_half] \n" + "add %[dst_v], %[dst_v], %x[vl_half] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "add %w[width_half], %w[width], #1 \n" + "lsr %w[width_half], %w[width_half], #1 \n" + "whilelt p1.b, wzr, %w[width_half] \n" + "zip1 p0.b, p1.b, p1.b \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" + "ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n" + "urhadd z0.b, p0/m, z0.b, z4.b \n" + "uzp1 z1.b, z0.b, z0.b \n" + "uzp2 z2.b, z0.b, z0.b \n" + "st1b {z1.b}, p1, [%[dst_u]] \n" + "st1b {z2.b}, p1, [%[dst_v]] \n" + + "99: \n" + : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy] + [src_uyvy_1] "+r"(src_uyvy_1), // %[src_uyvy_1] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [vl_half] "=&r"(vl_half), // %[vl_half] + [width_half] "=&r"(width_half) // %[width_half] + : + : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1"); +} + +#endif // (!defined(LIBYUV_DISABLE_SVE) || (!defined(LIBYUV_DISABLE_SME) && + // (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)))) && + // defined(__aarch64__) #ifdef __cplusplus } // extern "C" diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 871d73dc6..94ef003b8 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -140,8 +140,8 @@ extern "C" { #endif // The following are available on AArch64 SME platforms: -#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ - defined(__aarch64__) +#if !defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__) #define HAS_SCALEARGBROWDOWN2_SME #define HAS_SCALEROWDOWN2_16_SME #define HAS_SCALEROWDOWN2_SME diff --git a/linux.mk b/linux.mk index f4d9c73b1..b458e84b7 100644 --- a/linux.mk +++ b/linux.mk @@ -39,6 +39,8 @@ LOCAL_OBJ_FILES := \ source/rotate_neon.o \ source/rotate_neon64.o \ source/rotate_rvv.o \ + source/rotate_sme.o \ + source/rotate_sve.o \ source/rotate_win.o \ source/row_any.o \ source/row_common.o \ @@ -49,6 +51,8 @@ LOCAL_OBJ_FILES := \ source/row_neon.o \ source/row_neon64.o \ source/row_rvv.o \ + source/row_sme.o \ + source/row_sve.o \ source/row_win.o \ source/scale.o \ source/scale_any.o \ @@ -61,6 +65,7 @@ LOCAL_OBJ_FILES := \ source/scale_neon64.o \ source/scale_rgb.o \ source/scale_rvv.o \ + source/scale_sme.o \ source/scale_uv.o \ source/scale_win.o \ source/video_common.o diff --git a/source/compare_neon64.cc b/source/compare_neon64.cc index c1b474158..a49e41c64 100644 --- a/source/compare_neon64.cc +++ b/source/compare_neon64.cc @@ -26,6 +26,14 @@ extern "C" { #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) +#if defined(__clang__) +#pragma clang attribute push(__attribute__((target("dotprod,i8mm"))), \ + apply_to = function) +#elif defined(__GNUC__) +#pragma GCC push_options +#pragma GCC target("+dotprod+i8mm") +#endif + // 256 bits at a time // uses short accumulator which restricts count to 131 KB uint32_t HammingDistance_NEON(const uint8_t* src_a, @@ -451,6 +459,12 @@ uint32_t ARGBDetectRow_NEON(const uint8_t* argb, int width) { return 0; } +#if defined(__clang__) +#pragma clang attribute pop +#elif defined(__GNUC__) +#pragma GCC pop_options +#endif + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/convert.cc b/source/convert.cc index 779a71c93..d2f7e78e3 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -357,6 +357,11 @@ static int I41xToI420(const uint16_t* src_y, } #endif } else { +#if defined(HAS_HALFWIDTHROW_16TO8_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_Odd_NEON; + } +#endif #if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \ defined(HAS_HALFWIDTHROW_16TO8_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { @@ -2159,6 +2164,11 @@ static int Px10ToI420(const uint16_t* src_y, SplitUVRow = SplitUVRow_SVE2; } #endif +#if defined(HAS_SPLITUVROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + SplitUVRow = SplitUVRow_SME; + } +#endif #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; @@ -2331,6 +2341,11 @@ static int Px10ToI420(const uint16_t* src_y, } #endif } else { +#if defined(HAS_HALFWIDTHROW_16TO8_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + Half16To8 = HalfWidthRow_16To8_Odd_NEON; + } +#endif #if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \ defined(HAS_HALFWIDTHROW_16TO8_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { @@ -2562,6 +2577,12 @@ int YUY2ToI420(const uint8_t* src_yuy2, YUY2ToUVRow = YUY2ToUVRow_SVE2; } #endif +#if defined(HAS_YUY2TOYROW_SME) && defined(HAS_YUY2TOUVROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + YUY2ToYRow = YUY2ToYRow_SME; + YUY2ToUVRow = YUY2ToUVRow_SME; + } +#endif #if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { YUY2ToYRow = YUY2ToYRow_Any_LSX; @@ -2675,6 +2696,12 @@ int UYVYToI420(const uint8_t* src_uyvy, UYVYToUVRow = UYVYToUVRow_SVE2; } #endif +#if defined(HAS_UYVYTOYROW_SME) && defined(HAS_UYVYTOUVROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + UYVYToYRow = UYVYToYRow_SME; + UYVYToUVRow = UYVYToUVRow_SME; + } +#endif #if defined(HAS_UYVYTOYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { UYVYToYRow = UYVYToYRow_Any_LSX; @@ -2773,10 +2800,12 @@ int AYUVToNV12(const uint8_t* src_ayuv, #endif #if defined(HAS_AYUVTOUVROW_SVE2) if (TestCpuFlag(kCpuHasSVE2)) { - AYUVToUVRow = AYUVToUVRow_Any_SVE2; - if (IS_ALIGNED(width, 2)) { - AYUVToUVRow = AYUVToUVRow_SVE2; - } + AYUVToUVRow = AYUVToUVRow_SVE2; + } +#endif +#if defined(HAS_AYUVTOUVROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + AYUVToUVRow = AYUVToUVRow_SME; } #endif @@ -2850,10 +2879,12 @@ int AYUVToNV21(const uint8_t* src_ayuv, #endif #if defined(HAS_AYUVTOVUROW_SVE2) if (TestCpuFlag(kCpuHasSVE2)) { - AYUVToVURow = AYUVToVURow_Any_SVE2; - if (IS_ALIGNED(width, 2)) { - AYUVToVURow = AYUVToVURow_SVE2; - } + AYUVToVURow = AYUVToVURow_SVE2; + } +#endif +#if defined(HAS_AYUVTOVUROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + AYUVToVURow = AYUVToVURow_SME; } #endif diff --git a/source/convert_argb.cc b/source/convert_argb.cc index d69224c16..be3d0a632 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -1257,6 +1257,14 @@ int I210ToAR30Matrix(const uint16_t* src_y, dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30; dst_stride_ar30 = -dst_stride_ar30; } + // Coalesce rows. + if (src_stride_y == width && src_stride_u * 2 == width && + src_stride_v * 2 == width && dst_stride_ar30 == width * 4 && + (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_ar30 = 0; + } #if defined(HAS_I210TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I210ToAR30Row = I210ToAR30Row_Any_NEON; @@ -1436,6 +1444,13 @@ int I410ToAR30Matrix(const uint16_t* src_y, dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30; dst_stride_ar30 = -dst_stride_ar30; } + // Coalesce rows. + if (src_stride_y == width && src_stride_u == width && src_stride_v == width && + dst_stride_ar30 == width * 4 && (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_ar30 = 0; + } #if defined(HAS_I410TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I410ToAR30Row = I410ToAR30Row_Any_NEON; @@ -2131,6 +2146,13 @@ int P210ToARGBMatrix(const uint16_t* src_y, dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb; dst_stride_argb = -dst_stride_argb; } + // Coalesce rows. + if (IS_ALIGNED(width, 2) && src_stride_y == width && src_stride_uv == width && + dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_uv = dst_stride_argb = 0; + } #if defined(HAS_P210TOARGBROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { P210ToARGBRow = P210ToARGBRow_Any_SSSE3; @@ -2204,6 +2226,13 @@ int P410ToARGBMatrix(const uint16_t* src_y, dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb; dst_stride_argb = -dst_stride_argb; } + // Coalesce rows. + if (src_stride_y == width && src_stride_uv == width * 2 && + dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_uv = dst_stride_argb = 0; + } #if defined(HAS_P410TOARGBROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { P410ToARGBRow = P410ToARGBRow_Any_SSSE3; @@ -2352,6 +2381,13 @@ int P210ToAR30Matrix(const uint16_t* src_y, dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30; dst_stride_ar30 = -dst_stride_ar30; } + // Coalesce rows. + if (IS_ALIGNED(width, 2) && src_stride_y == width && src_stride_uv == width && + dst_stride_ar30 == width * 4 && (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_uv = dst_stride_ar30 = 0; + } #if defined(HAS_P210TOAR30ROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { P210ToAR30Row = P210ToAR30Row_Any_SSSE3; @@ -2425,6 +2461,13 @@ int P410ToAR30Matrix(const uint16_t* src_y, dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30; dst_stride_ar30 = -dst_stride_ar30; } + // Coalesce rows. + if (src_stride_y == width && src_stride_uv == width * 2 && + dst_stride_ar30 == width * 4 && (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_uv = dst_stride_ar30 = 0; + } #if defined(HAS_P410TOAR30ROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { P410ToAR30Row = P410ToAR30Row_Any_SSSE3; @@ -4132,6 +4175,11 @@ int RGB24ToARGB(const uint8_t* src_rgb24, RGB24ToARGBRow = RGB24ToARGBRow_SVE2; } #endif +#if defined(HAS_RGB24TOARGBROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + RGB24ToARGBRow = RGB24ToARGBRow_SME; + } +#endif #if defined(HAS_RGB24TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { RGB24ToARGBRow = RGB24ToARGBRow_Any_LSX; @@ -4230,6 +4278,11 @@ int RAWToARGB(const uint8_t* src_raw, RAWToARGBRow = RAWToARGBRow_SVE2; } #endif +#if defined(HAS_RAWTOARGBROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + RAWToARGBRow = RAWToARGBRow_SME; + } +#endif #if defined(HAS_RAWTOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { RAWToARGBRow = RAWToARGBRow_Any_LSX; @@ -4322,6 +4375,11 @@ int RAWToRGBA(const uint8_t* src_raw, RAWToRGBARow = RAWToRGBARow_SVE2; } #endif +#if defined(HAS_RAWTORGBAROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + RAWToRGBARow = RAWToRGBARow_SME; + } +#endif #if defined(HAS_RAWTORGBAROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { RAWToRGBARow = RAWToRGBARow_RVV; @@ -4469,6 +4527,11 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555, ARGB1555ToARGBRow = ARGB1555ToARGBRow_SVE2; } #endif +#if defined(HAS_ARGB1555TOARGBROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_SME; + } +#endif #if defined(HAS_ARGB1555TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_LSX; @@ -7203,6 +7266,11 @@ int I420ToRGB565Dither(const uint8_t* src_y, ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SVE2; } #endif +#if defined(HAS_ARGBTORGB565DITHERROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SME; + } +#endif #if defined(HAS_ARGBTORGB565DITHERROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LSX; diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index f8057535d..5d42dc2c9 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -3817,6 +3817,11 @@ int ARGBToRGB24(const uint8_t* src_argb, ARGBToRGB24Row = ARGBToRGB24Row_SVE2; } #endif +#if defined(HAS_ARGBTORGB24ROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToRGB24Row = ARGBToRGB24Row_SME; + } +#endif #if defined(HAS_ARGBTORGB24ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToRGB24Row = ARGBToRGB24Row_Any_LSX; @@ -3907,6 +3912,11 @@ int ARGBToRAW(const uint8_t* src_argb, ARGBToRAWRow = ARGBToRAWRow_SVE2; } #endif +#if defined(HAS_ARGBTORAWROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToRAWRow = ARGBToRAWRow_SME; + } +#endif #if defined(HAS_ARGBTORAWROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToRAWRow = ARGBToRAWRow_Any_LSX; @@ -3996,6 +4006,11 @@ int ARGBToRGB565Dither(const uint8_t* src_argb, ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SVE2; } #endif +#if defined(HAS_ARGBTORGB565DITHERROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SME; + } +#endif #if defined(HAS_ARGBTORGB565DITHERROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LSX; @@ -4084,6 +4099,11 @@ int ARGBToRGB565(const uint8_t* src_argb, ARGBToRGB565Row = ARGBToRGB565Row_SVE2; } #endif +#if defined(HAS_ARGBTORGB565ROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToRGB565Row = ARGBToRGB565Row_SME; + } +#endif #if defined(HAS_ARGBTORGB565ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToRGB565Row = ARGBToRGB565Row_Any_LSX; diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index 02caa89ea..7d29fd392 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -256,6 +256,11 @@ static int NV24ToARGB(const uint8_t* src_y, SplitUVRow = SplitUVRow_SVE2; } #endif +#if defined(HAS_SPLITUVROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + SplitUVRow = SplitUVRow_SME; + } +#endif #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc index f1967f813..e1adc5fb5 100644 --- a/source/convert_to_i420.cc +++ b/source/convert_to_i420.cc @@ -100,6 +100,11 @@ static int NV16ToI420(const uint8_t* src_y, SplitUVRow = SplitUVRow_SVE2; } #endif +#if defined(HAS_SPLITUVROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + SplitUVRow = SplitUVRow_SME; + } +#endif #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; @@ -372,6 +377,11 @@ static int NV24ToI420(const uint8_t* src_y, SplitUVRow = SplitUVRow_SVE2; } #endif +#if defined(HAS_SPLITUVROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + SplitUVRow = SplitUVRow_SME; + } +#endif #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 3df4cc060..de5bfbff8 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -755,6 +755,11 @@ void SplitUVPlane(const uint8_t* src_uv, SplitUVRow = SplitUVRow_SVE2; } #endif +#if defined(HAS_SPLITUVROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + SplitUVRow = SplitUVRow_SME; + } +#endif #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; @@ -1178,6 +1183,11 @@ void ConvertToLSBPlane_16(const uint16_t* src_y, DivideRow = DivideRow_16_SVE2; } #endif +#if defined(HAS_DIVIDEROW_16_SME) + if (TestCpuFlag(kCpuHasSME)) { + DivideRow = DivideRow_16_SME; + } +#endif #if defined(HAS_DIVIDEROW_16_HVX) if (TestCpuFlag(kCpuHasHVX)) { DivideRow = DivideRow_16_HVX; @@ -2534,8 +2544,9 @@ int YUY2ToI422(const uint8_t* src_yuy2, } } #endif -#if defined(HAS_YUY2TOYROW_AVX512BW) +#if defined(HAS_YUY2TOYROW_AVX512BW) && defined(HAS_YUY2TOUV422ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToUV422Row = YUY2ToUV422Row_AVX512BW; YUY2ToYRow = YUY2ToYRow_AVX512BW; } #endif @@ -2554,6 +2565,11 @@ int YUY2ToI422(const uint8_t* src_yuy2, YUY2ToYRow = YUY2ToYRow_SVE2; } #endif +#if defined(HAS_YUY2TOYROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + YUY2ToYRow = YUY2ToYRow_SME; + } +#endif #if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUV422ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { YUY2ToYRow = YUY2ToYRow_Any_LSX; @@ -2647,8 +2663,9 @@ int UYVYToI422(const uint8_t* src_uyvy, } } #endif -#if defined(HAS_UYVYTOYROW_AVX512BW) +#if defined(HAS_UYVYTOYROW_AVX512BW) && defined(HAS_UYVYTOUV422ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { + UYVYToUV422Row = UYVYToUV422Row_AVX512BW; UYVYToYRow = UYVYToYRow_AVX512BW; } #endif @@ -2667,6 +2684,11 @@ int UYVYToI422(const uint8_t* src_uyvy, UYVYToYRow = UYVYToYRow_SVE2; } #endif +#if defined(HAS_UYVYTOYROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + UYVYToYRow = UYVYToYRow_SME; + } +#endif #if defined(HAS_UYVYTOYROW_LSX) && defined(HAS_UYVYTOUV422ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { UYVYToYRow = UYVYToYRow_Any_LSX; @@ -2766,6 +2788,11 @@ int YUY2ToY(const uint8_t* src_yuy2, YUY2ToYRow = YUY2ToYRow_SVE2; } #endif +#if defined(HAS_YUY2TOYROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + YUY2ToYRow = YUY2ToYRow_SME; + } +#endif #if defined(HAS_YUY2TOYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { YUY2ToYRow = YUY2ToYRow_Any_LSX; @@ -2857,6 +2884,11 @@ int UYVYToY(const uint8_t* src_uyvy, UYVYToYRow = UYVYToYRow_SVE2; } #endif +#if defined(HAS_UYVYTOYROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + UYVYToYRow = UYVYToYRow_SME; + } +#endif #if defined(HAS_UYVYTOYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { UYVYToYRow = UYVYToYRow_Any_LSX; @@ -3986,6 +4018,11 @@ int ARGBMultiply(const uint8_t* src_argb0, } } #endif +#if defined(HAS_ARGBMULTIPLYROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBMultiplyRow = ARGBMultiplyRow_SVE2; + } +#endif #if defined(HAS_ARGBMULTIPLYROW_SME) if (TestCpuFlag(kCpuHasSME)) { ARGBMultiplyRow = ARGBMultiplyRow_SME; @@ -4058,11 +4095,6 @@ int ARGBAdd(const uint8_t* src_argb0, height = 1; src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; } -#if defined(HAS_ARGBADDROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ARGBAddRow = ARGBAddRow_SSE2; - } -#endif #if defined(HAS_ARGBADDROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { ARGBAddRow = ARGBAddRow_Any_SSE2; @@ -4382,6 +4414,11 @@ int RAWToRGB24(const uint8_t* src_raw, RAWToRGB24Row = RAWToRGB24Row_SVE2; } #endif +#if defined(HAS_RAWTORGB24ROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + RAWToRGB24Row = RAWToRGB24Row_SME; + } +#endif #if defined(HAS_RAWTORGB24ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { RAWToRGB24Row = RAWToRGB24Row_Any_LSX; @@ -4813,23 +4850,35 @@ int ARGBGrayTo(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBGRAYROW_NEON) - if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_NEON; + if (TestCpuFlag(kCpuHasNEON)) { + ARGBGrayRow = ARGBGrayRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_NEON; + } } #endif #if defined(HAS_ARGBGRAYROW_NEON_DOTPROD) - if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_NEON_DotProd; + if (TestCpuFlag(kCpuHasNeonDotProd)) { + ARGBGrayRow = ARGBGrayRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_NEON_DotProd; + } } #endif #if defined(HAS_ARGBGRAYROW_LSX) - if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_LSX; + if (TestCpuFlag(kCpuHasLSX)) { + ARGBGrayRow = ARGBGrayRow_Any_LSX; + if (IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_LSX; + } } #endif #if defined(HAS_ARGBGRAYROW_LASX) - if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 16)) { - ARGBGrayRow = ARGBGrayRow_LASX; + if (TestCpuFlag(kCpuHasLASX)) { + ARGBGrayRow = ARGBGrayRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + ARGBGrayRow = ARGBGrayRow_LASX; + } } #endif @@ -4878,23 +4927,35 @@ int ARGBGray(uint8_t* dst_argb, } #endif #if defined(HAS_ARGBGRAYROW_NEON) - if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_NEON; + if (TestCpuFlag(kCpuHasNEON)) { + ARGBGrayRow = ARGBGrayRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_NEON; + } } #endif #if defined(HAS_ARGBGRAYROW_NEON_DOTPROD) - if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_NEON_DotProd; + if (TestCpuFlag(kCpuHasNeonDotProd)) { + ARGBGrayRow = ARGBGrayRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_NEON_DotProd; + } } #endif #if defined(HAS_ARGBGRAYROW_LSX) - if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_LSX; + if (TestCpuFlag(kCpuHasLSX)) { + ARGBGrayRow = ARGBGrayRow_Any_LSX; + if (IS_ALIGNED(width, 8)) { + ARGBGrayRow = ARGBGrayRow_LSX; + } } #endif #if defined(HAS_ARGBGRAYROW_LASX) - if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 16)) { - ARGBGrayRow = ARGBGrayRow_LASX; + if (TestCpuFlag(kCpuHasLASX)) { + ARGBGrayRow = ARGBGrayRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + ARGBGrayRow = ARGBGrayRow_LASX; + } } #endif @@ -4936,23 +4997,35 @@ int ARGBSepia(uint8_t* dst_argb, } #endif #if defined(HAS_ARGBSEPIAROW_NEON) - if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { - ARGBSepiaRow = ARGBSepiaRow_NEON; + if (TestCpuFlag(kCpuHasNEON)) { + ARGBSepiaRow = ARGBSepiaRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBSepiaRow = ARGBSepiaRow_NEON; + } } #endif #if defined(HAS_ARGBSEPIAROW_NEON_DOTPROD) - if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) { - ARGBSepiaRow = ARGBSepiaRow_NEON_DotProd; + if (TestCpuFlag(kCpuHasNeonDotProd)) { + ARGBSepiaRow = ARGBSepiaRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 8)) { + ARGBSepiaRow = ARGBSepiaRow_NEON_DotProd; + } } #endif #if defined(HAS_ARGBSEPIAROW_LSX) - if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) { - ARGBSepiaRow = ARGBSepiaRow_LSX; + if (TestCpuFlag(kCpuHasLSX)) { + ARGBSepiaRow = ARGBSepiaRow_Any_LSX; + if (IS_ALIGNED(width, 8)) { + ARGBSepiaRow = ARGBSepiaRow_LSX; + } } #endif #if defined(HAS_ARGBSEPIAROW_LASX) - if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 16)) { - ARGBSepiaRow = ARGBSepiaRow_LASX; + if (TestCpuFlag(kCpuHasLASX)) { + ARGBSepiaRow = ARGBSepiaRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + ARGBSepiaRow = ARGBSepiaRow_LASX; + } } #endif @@ -5004,23 +5077,35 @@ int ARGBColorMatrix(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBCOLORMATRIXROW_NEON) - if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { - ARGBColorMatrixRow = ARGBColorMatrixRow_NEON; + if (TestCpuFlag(kCpuHasNEON)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_NEON; + } } #endif #if defined(HAS_ARGBCOLORMATRIXROW_NEON_I8MM) - if (TestCpuFlag(kCpuHasNeonI8MM) && IS_ALIGNED(width, 8)) { - ARGBColorMatrixRow = ARGBColorMatrixRow_NEON_I8MM; + if (TestCpuFlag(kCpuHasNeonI8MM)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_Any_NEON_I8MM; + if (IS_ALIGNED(width, 8)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_NEON_I8MM; + } } #endif #if defined(HAS_ARGBCOLORMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) { - ARGBColorMatrixRow = ARGBColorMatrixRow_LSX; + if (TestCpuFlag(kCpuHasLSX)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_LSX; + } } #endif #if defined(HAS_ARGBCOLORMATRIXROW_LASX) - if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) { - ARGBColorMatrixRow = ARGBColorMatrixRow_LASX; + if (TestCpuFlag(kCpuHasLASX)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_LASX; + } } #endif for (y = 0; y < height; ++y) { @@ -5189,18 +5274,27 @@ int ARGBQuantize(uint8_t* dst_argb, } #endif #if defined(HAS_ARGBQUANTIZEROW_NEON) - if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { - ARGBQuantizeRow = ARGBQuantizeRow_NEON; + if (TestCpuFlag(kCpuHasNEON)) { + ARGBQuantizeRow = ARGBQuantizeRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBQuantizeRow = ARGBQuantizeRow_NEON; + } } #endif #if defined(HAS_ARGBQUANTIZEROW_LSX) - if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) { - ARGBQuantizeRow = ARGBQuantizeRow_LSX; + if (TestCpuFlag(kCpuHasLSX)) { + ARGBQuantizeRow = ARGBQuantizeRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBQuantizeRow = ARGBQuantizeRow_LSX; + } } #endif #if defined(HAS_ARGBQUANTIZEROW_LASX) - if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) { - ARGBQuantizeRow = ARGBQuantizeRow_LASX; + if (TestCpuFlag(kCpuHasLASX)) { + ARGBQuantizeRow = ARGBQuantizeRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBQuantizeRow = ARGBQuantizeRow_LASX; + } } #endif for (y = 0; y < height; ++y) { @@ -5402,18 +5496,27 @@ int ARGBShade(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBSHADEROW_NEON) - if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { - ARGBShadeRow = ARGBShadeRow_NEON; + if (TestCpuFlag(kCpuHasNEON)) { + ARGBShadeRow = ARGBShadeRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBShadeRow = ARGBShadeRow_NEON; + } } #endif #if defined(HAS_ARGBSHADEROW_LSX) - if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) { - ARGBShadeRow = ARGBShadeRow_LSX; + if (TestCpuFlag(kCpuHasLSX)) { + ARGBShadeRow = ARGBShadeRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBShadeRow = ARGBShadeRow_LSX; + } } #endif #if defined(HAS_ARGBSHADEROW_LASX) - if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) { - ARGBShadeRow = ARGBShadeRow_LASX; + if (TestCpuFlag(kCpuHasLASX)) { + ARGBShadeRow = ARGBShadeRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBShadeRow = ARGBShadeRow_LASX; + } } #endif @@ -5585,6 +5688,11 @@ int InterpolatePlane_16(const uint16_t* src0, } } #endif +#if defined(HAS_INTERPOLATEROW_16_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow_16 = InterpolateRow_16_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_16_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow_16 = InterpolateRow_16_SME; @@ -6418,6 +6526,11 @@ int HalfFloatPlane(const uint16_t* src_y, HalfFloatRow = scale == 1.0f ? HalfFloat1Row_SVE2 : HalfFloatRow_SVE2; } #endif +#if defined(HAS_HALFFLOATROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + HalfFloatRow = scale == 1.0f ? HalfFloat1Row_SME : HalfFloatRow_SME; + } +#endif #if defined(HAS_HALFFLOATROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { HalfFloatRow = HalfFloatRow_Any_LSX; @@ -6508,8 +6621,11 @@ int ARGBLumaColorTable(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBLUMACOLORTABLEROW_NEON) - if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 4)) { - ARGBLumaColorTableRow = ARGBLumaColorTableRow_NEON; + if (TestCpuFlag(kCpuHasNEON)) { + ARGBLumaColorTableRow = ARGBLumaColorTableRow_Any_NEON; + if (IS_ALIGNED(width, 4)) { + ARGBLumaColorTableRow = ARGBLumaColorTableRow_NEON; + } } #endif @@ -6813,6 +6929,11 @@ int YUY2ToNV12(const uint8_t* src_yuy2, YUY2ToYRow = YUY2ToYRow_SVE2; } #endif +#if defined(HAS_YUY2TOYROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + YUY2ToYRow = YUY2ToYRow_SME; + } +#endif #if defined(HAS_YUY2TOYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { YUY2ToYRow = YUY2ToYRow_Any_LSX; @@ -6940,6 +7061,11 @@ int UYVYToNV12(const uint8_t* src_uyvy, SplitUVRow = SplitUVRow_SVE2; } #endif +#if defined(HAS_SPLITUVROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + SplitUVRow = SplitUVRow_SME; + } +#endif #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; @@ -7102,6 +7228,11 @@ void HalfMergeUVPlane(const uint8_t* src_u, HalfMergeUVRow = HalfMergeUVRow_SVE2; } #endif +#if defined(HAS_HALFMERGEUVROW_SME) + if (TestCpuFlag(kCpuHasSME) && IS_ALIGNED(width, 2)) { + HalfMergeUVRow = HalfMergeUVRow_SME; + } +#endif #if defined(HAS_HALFMERGEUVROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { HalfMergeUVRow = HalfMergeUVRow_Any_SSSE3; diff --git a/source/rotate.cc b/source/rotate.cc index b0bbfab98..6574ac054 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -60,10 +60,8 @@ void TransposePlane(const uint8_t* src, void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) = TransposeWx16_C; #endif -#if !defined(HAS_TRANSPOSEWX16_NEON) void (*TransposeWx8)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) = TransposeWx8_C; -#endif #if defined(HAS_TRANSPOSEWX8_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -170,7 +168,6 @@ void TransposePlane(const uint8_t* src, } } #endif -#if !defined(HAS_TRANSPOSEWX16_NEON) // Work across the source in 8x8 tiles while (i >= 8) { TransposeWx8(src, src_stride, dst, dst_stride, width); @@ -178,7 +175,6 @@ void TransposePlane(const uint8_t* src, dst += 8; // Move over 8 columns. i -= 8; } -#endif if (i > 0) { TransposeWxH(src, src_stride, dst, dst_stride, width, i); diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index f7a78d51e..e7348cb30 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -1078,6 +1078,7 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src, "lea (%[dst],%[d],2),%[dst] \n" "sub $0x8,%[cnt] \n" "jg 1b \n" + "vzeroupper \n" : [src] "+r"(src_b), // %[src] [dst] "+r"(dst_b), // %[dst] [cnt] "+r"(cnt), // %[cnt] @@ -1222,6 +1223,7 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src, "lea (%[dst],%[d],2),%[dst] \n" "vmovdqu16 %%zmm14,(%[dst]) \n" "2: \n" + "vzeroupper \n" : [dst] "+r"(dst_b), // %[dst] [temp] "=&r"(temp) // %[temp] : [src] "r"(src_b), // %[src] @@ -1233,7 +1235,6 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src, "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "xmm16"); } - asm volatile("vzeroupper \n"); } #endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW) @@ -1915,16 +1916,16 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src, "add $0x10,%[src] \n" // Stage 1: 32-bit unpack across all 4 128-bit lanes - "vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n" - "vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n" - "vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n" - "vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n" + "vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n" + "vpunpckldq %%zmm3,%%zmm2,%%zmm17 \n" + "vpunpckhdq %%zmm1,%%zmm0,%%zmm18 \n" + "vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n" // Stage 2: 64-bit unpack across all 4 128-bit lanes - "vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n" - "vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n" - "vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n" - "vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n" + "vpunpcklqdq %%zmm17,%%zmm16,%%zmm0 \n" + "vpunpckhqdq %%zmm17,%%zmm16,%%zmm1 \n" + "vpunpcklqdq %%zmm19,%%zmm18,%%zmm2 \n" + "vpunpckhqdq %%zmm19,%%zmm18,%%zmm3 \n" // Store 4 full 512-bit (64 B) rows = 16 32-bit pixels each "vmovdqu32 %%zmm0,(%[dst]) \n" @@ -1935,14 +1936,15 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src, "lea (%[dst],%[d],2),%[dst] \n" "sub $0x4,%[cnt] \n" "jg 1b \n" + "vzeroupper \n" : [src] "+r"(src), // %[src] [dst] "+r"(dst), // %[dst] [cnt] "+r"(cnt), // %[cnt] [temp] "=&r"(temp) // %[temp] : [s] "r"(s), // %[s] [d] "r"(d) // %[d] - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6", "xmm7"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm17", + "xmm18", "xmm19"); width -= n; } if (width > 0) { @@ -1957,47 +1959,47 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src, "vmovdqu32 (%[temp],%[s]),%%xmm3%{%%k1%}%{z%} \n" "lea (%[temp],%[s],2),%[temp] \n" - "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $1,%%xmm8,%%zmm0,%%zmm0 \n" - "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $1,%%xmm8,%%zmm1,%%zmm1 \n" + "vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm20,%%zmm0,%%zmm0 \n" + "vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm20,%%zmm1,%%zmm1 \n" "lea (%[temp],%[s],2),%[temp] \n" - "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $1,%%xmm8,%%zmm2,%%zmm2 \n" - "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $1,%%xmm8,%%zmm3,%%zmm3 \n" + "vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm20,%%zmm2,%%zmm2 \n" + "vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm20,%%zmm3,%%zmm3 \n" "lea (%[temp],%[s],2),%[temp] \n" - "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $2,%%xmm8,%%zmm0,%%zmm0 \n" - "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $2,%%xmm8,%%zmm1,%%zmm1 \n" + "vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm20,%%zmm0,%%zmm0 \n" + "vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm20,%%zmm1,%%zmm1 \n" "lea (%[temp],%[s],2),%[temp] \n" - "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $2,%%xmm8,%%zmm2,%%zmm2 \n" - "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $2,%%xmm8,%%zmm3,%%zmm3 \n" + "vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm20,%%zmm2,%%zmm2 \n" + "vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm20,%%zmm3,%%zmm3 \n" "lea (%[temp],%[s],2),%[temp] \n" - "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $3,%%xmm8,%%zmm0,%%zmm0 \n" - "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $3,%%xmm8,%%zmm1,%%zmm1 \n" + "vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm20,%%zmm0,%%zmm0 \n" + "vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm20,%%zmm1,%%zmm1 \n" "lea (%[temp],%[s],2),%[temp] \n" - "vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $3,%%xmm8,%%zmm2,%%zmm2 \n" - "vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n" - "vinserti32x4 $3,%%xmm8,%%zmm3,%%zmm3 \n" + "vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm20,%%zmm2,%%zmm2 \n" + "vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm20,%%zmm3,%%zmm3 \n" - "vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n" - "vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n" - "vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n" - "vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n" + "vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n" + "vpunpckldq %%zmm3,%%zmm2,%%zmm17 \n" + "vpunpckhdq %%zmm1,%%zmm0,%%zmm18 \n" + "vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n" - "vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n" - "vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n" - "vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n" - "vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n" + "vpunpcklqdq %%zmm17,%%zmm16,%%zmm0 \n" + "vpunpckhqdq %%zmm17,%%zmm16,%%zmm1 \n" + "vpunpcklqdq %%zmm19,%%zmm18,%%zmm2 \n" + "vpunpckhqdq %%zmm19,%%zmm18,%%zmm3 \n" "vmovdqu32 %%zmm0,(%[dst]) \n" "cmp $1,%[width] \n" @@ -2008,6 +2010,7 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src, "lea (%[dst],%[d],2),%[dst] \n" "vmovdqu32 %%zmm2,(%[dst]) \n" "2: \n" + "vzeroupper \n" : [dst] "+r"(dst), // %[dst] [temp] "=&r"(temp) // %[temp] : [src] "r"(src), // %[src] @@ -2015,10 +2018,9 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src, [d] "r"(d), // %[d] [col_mask] "r"(col_mask), // %[col_mask] [width] "r"(width) // %[width] - : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6", "xmm7", "xmm8"); + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20"); } - asm volatile("vzeroupper \n"); } #endif // defined(HAS_TRANSPOSEWX16_32_AVX512BW) diff --git a/source/rotate_neon64.cc b/source/rotate_neon64.cc index 5844c0d28..4974934da 100644 --- a/source/rotate_neon64.cc +++ b/source/rotate_neon64.cc @@ -26,6 +26,77 @@ extern "C" { // This module is for GCC Neon armv8 64 bit. #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) +void TransposeWx8_NEON(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + const uint8_t* temp; + asm volatile( + "1: \n" + "mov %[temp], %[src] \n" + "ld1 {v0.8b}, [%[temp]], %[s] \n" + "ld1 {v1.8b}, [%[temp]], %[s] \n" + "ld1 {v2.8b}, [%[temp]], %[s] \n" + "ld1 {v3.8b}, [%[temp]], %[s] \n" + "ld1 {v4.8b}, [%[temp]], %[s] \n" + "ld1 {v5.8b}, [%[temp]], %[s] \n" + "ld1 {v6.8b}, [%[temp]], %[s] \n" + "ld1 {v7.8b}, [%[temp]] \n" + "add %[src], %[src], #8 \n" + + // Stage 1: 8-bit transpose + "trn1 v16.8b, v0.8b, v1.8b \n" + "trn2 v17.8b, v0.8b, v1.8b \n" + "trn1 v18.8b, v2.8b, v3.8b \n" + "trn2 v19.8b, v2.8b, v3.8b \n" + "trn1 v20.8b, v4.8b, v5.8b \n" + "trn2 v21.8b, v4.8b, v5.8b \n" + "trn1 v22.8b, v6.8b, v7.8b \n" + "trn2 v23.8b, v6.8b, v7.8b \n" + + // Stage 2: 16-bit transpose + "trn1 v0.4h, v16.4h, v18.4h \n" + "trn1 v1.4h, v17.4h, v19.4h \n" + "trn2 v2.4h, v16.4h, v18.4h \n" + "trn2 v3.4h, v17.4h, v19.4h \n" + "trn1 v4.4h, v20.4h, v22.4h \n" + "trn1 v5.4h, v21.4h, v23.4h \n" + "trn2 v6.4h, v20.4h, v22.4h \n" + "trn2 v7.4h, v21.4h, v23.4h \n" + + // Stage 3: 32-bit transpose + "trn1 v16.2s, v0.2s, v4.2s \n" + "trn1 v17.2s, v1.2s, v5.2s \n" + "trn1 v18.2s, v2.2s, v6.2s \n" + "trn1 v19.2s, v3.2s, v7.2s \n" + "trn2 v20.2s, v0.2s, v4.2s \n" + "trn2 v21.2s, v1.2s, v5.2s \n" + "trn2 v22.2s, v2.2s, v6.2s \n" + "trn2 v23.2s, v3.2s, v7.2s \n" + + "subs %w[w], %w[w], #8 \n" + "st1 {v16.8b}, [%[dst]], %[d] \n" + "st1 {v17.8b}, [%[dst]], %[d] \n" + "st1 {v18.8b}, [%[dst]], %[d] \n" + "st1 {v19.8b}, [%[dst]], %[d] \n" + "st1 {v20.8b}, [%[dst]], %[d] \n" + "st1 {v21.8b}, [%[dst]], %[d] \n" + "st1 {v22.8b}, [%[dst]], %[d] \n" + "st1 {v23.8b}, [%[dst]], %[d] \n" + "b.gt 1b \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} + void TransposeWx16_NEON(const uint8_t* src, int src_stride, uint8_t* dst, diff --git a/source/rotate_sme.cc b/source/rotate_sme.cc index 85d5bf604..2efa0ccbf 100644 --- a/source/rotate_sme.cc +++ b/source/rotate_sme.cc @@ -23,8 +23,16 @@ extern "C" { // asm statement, and other OS ABIs (e.g. Windows) differ in which registers // are callee-saved. List "memory" first, then "cc", then registers. -#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ - defined(__aarch64__) +#if !defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__) + +#if defined(__clang__) +#pragma clang attribute push( \ + __attribute__((target("arch=armv8.5-a+i8mm+sme"))), apply_to = function) +#elif defined(__GNUC__) +#pragma GCC push_options +#pragma GCC target("arch=armv8.5-a+i8mm+sve2+sme") +#endif __arm_locally_streaming __arm_new("za") void TransposeWxH_SME( const uint8_t* src, @@ -81,7 +89,7 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_SME( [dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride] [block_width] "r"(block_width), // %[block_width] [block_height] "r"(block_height) // %[block_height] - : "cc", "memory", "p0", "w12", "za"); + : "memory", "cc", "p0", "w12", "za"); src2 += vl; width2 -= vl; @@ -157,7 +165,7 @@ __arm_locally_streaming __arm_new("za") void TransposeUVWxH_SME( [dst_stride_b] "r"((ptrdiff_t)dst_stride_b), // %[dst_stride_b] [block_width] "r"(block_width * 2), // %[block_width] [block_height] "r"(block_height) // %[block_height] - : "cc", "memory", "p0", "w12", "za"); + : "memory", "cc", "p0", "w12", "za"); src2 += 2 * vl; width2 -= vl; @@ -225,7 +233,7 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_32_SME( [dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride] [block_width] "r"(block_width), // %[block_width] [block_height] "r"(block_height) // %[block_height] - : "cc", "memory", "p0", "w12", "za"); + : "memory", "cc", "p0", "w12", "za"); src2 += vl * 4; width2 -= vl; @@ -304,8 +312,14 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_16_SME( } while (height > 0); } -#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && - // defined(__aarch64__) +#if defined(__clang__) +#pragma clang attribute pop +#elif defined(__GNUC__) +#pragma GCC pop_options +#endif + +#endif // !defined(LIBYUV_DISABLE_SME) && (defined(CLANG_HAS_SME) || + // defined(GCC_HAS_SME)) && defined(__aarch64__) #ifdef __cplusplus } // extern "C" diff --git a/source/rotate_sve.cc b/source/rotate_sve.cc index f7dc3c14d..a20b0f2cc 100644 --- a/source/rotate_sve.cc +++ b/source/rotate_sve.cc @@ -18,6 +18,14 @@ extern "C" { #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) +#if defined(__clang__) +#pragma clang attribute push( \ + __attribute__((target("arch=armv8.5-a+i8mm+sve2"))), apply_to = function) +#elif defined(__GNUC__) +#pragma GCC push_options +#pragma GCC target("arch=armv8.5-a+i8mm+sve2") +#endif + // Transpose 32 bit values (ARGB) in Wx4 tiles using SVE2 with predication // for tail widths. void TransposeWx4_32_SVE2(const uint8_t* src, @@ -106,7 +114,7 @@ void TransposeWx4_32_SVE2(const uint8_t* src, [dst3] "+r"(dst3), // %[dst3] [width] "+r"(width) // %[width] : [s4] "r"(s4) // %[s4] - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0"); + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0"); } void TransposeWx8_16_SVE2(const uint16_t* src, @@ -271,6 +279,12 @@ void TransposeWx8_16_SVE2(const uint16_t* src, "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23"); } +#if defined(__clang__) +#pragma clang attribute pop +#elif defined(__GNUC__) +#pragma GCC pop_options +#endif + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/row_any.cc b/source/row_any.cc index 9036e0b13..7957c898e 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1453,6 +1453,18 @@ ANY11(ARGBToARGB1555Row_Any_WASMSIMD, ARGBToARGB1555Row_WASMSIMD, 0, 4, 2, 7) #ifdef HAS_ARGBTOARGB4444ROW_WASMSIMD ANY11(ARGBToARGB4444Row_Any_WASMSIMD, ARGBToARGB4444Row_WASMSIMD, 0, 4, 2, 7) #endif +#ifdef HAS_ARGBGRAYROW_NEON +ANY11(ARGBGrayRow_Any_NEON, ARGBGrayRow_NEON, 0, 4, 4, 7) +#endif +#ifdef HAS_ARGBGRAYROW_NEON_DOTPROD +ANY11(ARGBGrayRow_Any_NEON_DotProd, ARGBGrayRow_NEON_DotProd, 0, 4, 4, 7) +#endif +#ifdef HAS_ARGBGRAYROW_LSX +ANY11(ARGBGrayRow_Any_LSX, ARGBGrayRow_LSX, 0, 4, 4, 7) +#endif +#ifdef HAS_ARGBGRAYROW_LASX +ANY11(ARGBGrayRow_Any_LASX, ARGBGrayRow_LASX, 0, 4, 4, 15) +#endif #undef ANY11 // Any 1 to 1 blended. Destination is read, modify, write. @@ -1669,8 +1681,148 @@ ANY11P(ARGBPolynomialRow_Any_NEON, 4, 1) #endif +#ifdef HAS_ARGBCOLORMATRIXROW_NEON +ANY11P(ARGBColorMatrixRow_Any_NEON, + ARGBColorMatrixRow_NEON, + const int8_t*, + 4, + 4, + 7) +#endif +#ifdef HAS_ARGBCOLORMATRIXROW_NEON_I8MM +ANY11P(ARGBColorMatrixRow_Any_NEON_I8MM, + ARGBColorMatrixRow_NEON_I8MM, + const int8_t*, + 4, + 4, + 7) +#endif +#ifdef HAS_ARGBCOLORMATRIXROW_LSX +ANY11P(ARGBColorMatrixRow_Any_LSX, + ARGBColorMatrixRow_LSX, + const int8_t*, + 4, + 4, + 3) +#endif +#ifdef HAS_ARGBCOLORMATRIXROW_LASX +ANY11P(ARGBColorMatrixRow_Any_LASX, + ARGBColorMatrixRow_LASX, + const int8_t*, + 4, + 4, + 7) +#endif #undef ANY11P +// Any 1 in-place. +#define ANY1I(NAMEANY, ANY_SIMD, BPP, MASK) \ + void NAMEANY(uint8_t* dst_ptr, int width) { \ + SIMD_ALIGNED(uint8_t vout[(MASK + 1) * BPP]); \ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(dst_ptr, n); \ + } \ + ptrdiff_t np = n; \ + memcpy(vout, dst_ptr + np * BPP, r * BPP); \ + ANY_SIMD(vout, MASK + 1); \ + memcpy(dst_ptr + np * BPP, vout, r * BPP); \ + } + +#ifdef HAS_ARGBSEPIAROW_NEON +ANY1I(ARGBSepiaRow_Any_NEON, ARGBSepiaRow_NEON, 4, 7) +#endif +#ifdef HAS_ARGBSEPIAROW_NEON_DOTPROD +ANY1I(ARGBSepiaRow_Any_NEON_DotProd, ARGBSepiaRow_NEON_DotProd, 4, 7) +#endif +#ifdef HAS_ARGBSEPIAROW_LSX +ANY1I(ARGBSepiaRow_Any_LSX, ARGBSepiaRow_LSX, 4, 7) +#endif +#ifdef HAS_ARGBSEPIAROW_LASX +ANY1I(ARGBSepiaRow_Any_LASX, ARGBSepiaRow_LASX, 4, 15) +#endif +#undef ANY1I + +// Any 1 in-place with 3 int parameters. +#define ANY1I3(NAMEANY, ANY_SIMD, BPP, MASK) \ + void NAMEANY(uint8_t* dst_ptr, int p0, int p1, int p2, int width) { \ + SIMD_ALIGNED(uint8_t vout[(MASK + 1) * BPP]); \ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(dst_ptr, p0, p1, p2, n); \ + } \ + ptrdiff_t np = n; \ + memcpy(vout, dst_ptr + np * BPP, r * BPP); \ + ANY_SIMD(vout, p0, p1, p2, MASK + 1); \ + memcpy(dst_ptr + np * BPP, vout, r * BPP); \ + } + +#ifdef HAS_ARGBQUANTIZEROW_NEON +ANY1I3(ARGBQuantizeRow_Any_NEON, ARGBQuantizeRow_NEON, 4, 7) +#endif +#ifdef HAS_ARGBQUANTIZEROW_LSX +ANY1I3(ARGBQuantizeRow_Any_LSX, ARGBQuantizeRow_LSX, 4, 3) +#endif +#ifdef HAS_ARGBQUANTIZEROW_LASX +ANY1I3(ARGBQuantizeRow_Any_LASX, ARGBQuantizeRow_LASX, 4, 7) +#endif +#undef ANY1I3 + +// Any 1 to 1 with trailing parameter. +#define ANY11PT(NAMEANY, ANY_SIMD, T, SBPP, BPP, MASK) \ + void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width, T param) { \ + SIMD_ALIGNED(uint8_t vin[(MASK + 1) * SBPP]); \ + SIMD_ALIGNED(uint8_t vout[(MASK + 1) * BPP]); \ + MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr, dst_ptr, n, param); \ + } \ + ptrdiff_t np = n; \ + memcpy(vin, src_ptr + np * SBPP, r * SBPP); \ + ANY_SIMD(vin, vout, MASK + 1, param); \ + memcpy(dst_ptr + np * BPP, vout, r * BPP); \ + } + +#ifdef HAS_ARGBSHADEROW_NEON +ANY11PT(ARGBShadeRow_Any_NEON, ARGBShadeRow_NEON, uint32_t, 4, 4, 7) +#endif +#ifdef HAS_ARGBSHADEROW_LSX +ANY11PT(ARGBShadeRow_Any_LSX, ARGBShadeRow_LSX, uint32_t, 4, 4, 3) +#endif +#ifdef HAS_ARGBSHADEROW_LASX +ANY11PT(ARGBShadeRow_Any_LASX, ARGBShadeRow_LASX, uint32_t, 4, 4, 7) +#endif +#undef ANY11PT + +#ifdef HAS_ARGBLUMACOLORTABLEROW_NEON +void ARGBLumaColorTableRow_Any_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + const uint32_t lumacoeff) { + SIMD_ALIGNED(uint8_t vin[4 * 4]); + SIMD_ALIGNED(uint8_t vout[4 * 4]); + MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ + int r = width & 3; + int n = width & ~3; + if (n > 0) { + ARGBLumaColorTableRow_NEON(src_argb, dst_argb, n, luma, lumacoeff); + } + ptrdiff_t np = n; + memcpy(vin, src_argb + np * 4, r * 4); + ARGBLumaColorTableRow_NEON(vin, vout, 4, luma, lumacoeff); + memcpy(dst_argb + np * 4, vout, r * 4); +} +#endif + // Any 1 to 1 with type #define ANY11T(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \ void NAMEANY(const STYPE* src_ptr, DTYPE* dst_ptr, int width) { \ @@ -2190,6 +2342,25 @@ ANY11HWS(HalfWidthRow_16To8_Any_NEON, uint16_t, uint8_t, 7) +void HalfWidthRow_16To8_Odd_NEON(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + int n = width - 1; + if (n > 0) { + if (IS_ALIGNED(n, 8)) { + HalfWidthRow_16To8_NEON(src_uv, src_uv_stride, dst_uv, scale, n); + } else { + HalfWidthRow_16To8_Any_NEON(src_uv, src_uv_stride, dst_uv, scale, n); + } + src_uv += (ptrdiff_t)n * 2; + dst_uv += n; + } + const uint16_t* src_uv1 = src_uv + src_uv_stride; + uint32_t avg = (uint32_t)src_uv[0] + (uint32_t)src_uv1[0] + 1u; + dst_uv[0] = (uint8_t)((avg * (uint32_t)scale) >> 17); +} #endif #undef ANY11HWS @@ -2811,9 +2982,6 @@ ANY11MC(RGBToYMatrixRow_Any_LSX, RGBToYMatrixRow_LSX, 3, 15) #ifdef HAS_RGBTOYMATRIXROW_LASX ANY11MC(RGBToYMatrixRow_Any_LASX, RGBToYMatrixRow_LASX, 3, 31) #endif -#ifdef HAS_ARGBTOYMATRIXROW_RVV -ANY11MC(ARGBToYMatrixRow_Any_RVV, ARGBToYMatrixRow_RVV, 4, 15) -#endif #ifdef HAS_ARGBTOYMATRIXROW_WASMSIMD ANY11MC(ARGBToYMatrixRow_Any_WASMSIMD, ARGBToYMatrixRow_WASMSIMD, 4, 15) #endif @@ -2952,12 +3120,6 @@ ANY11S(AYUVToVURow_Any_AVX2, AYUVToVURow_AVX2, 0, 4, 15) ANY11S(AYUVToUVRow_Any_NEON, AYUVToUVRow_NEON, 0, 4, 15) ANY11S(AYUVToVURow_Any_NEON, AYUVToVURow_NEON, 0, 4, 15) #endif -#ifdef HAS_AYUVTOUVROW_SVE2 -ANY11S(AYUVToUVRow_Any_SVE2, AYUVToUVRow_SVE2, 0, 4, 1) -#endif -#ifdef HAS_AYUVTOVUROW_SVE2 -ANY11S(AYUVToVURow_Any_SVE2, AYUVToVURow_SVE2, 0, 4, 1) -#endif #undef ANY11S #define ANYDETILE(NAMEANY, ANY_SIMD, T, BPP, MASK) \ diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 5f13978ec..0ed9f7939 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -11431,6 +11431,56 @@ void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, } #endif // HAS_YUY2TOUVROW_AVX512BW +#ifdef HAS_YUY2TOUV422ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void YUY2ToUV422Row_AVX512BW(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + uintptr_t temp; + asm volatile( + "sub %1,%2 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x8,%%zmm0,%%zmm1 \n" + "vpsrld $0x18,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm1,(%1) \n" + "vpmovdb %%zmm0,0x00(%1,%2,1) \n" + "lea 0x10(%1),%1 \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "jle 99f \n" + "add $0x1,%k3 \n" + "shr $0x1,%k3 \n" + "mov $-1,%k[temp] \n" + "bzhi %k3,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpsrld $0x8,%%zmm0,%%zmm1 \n" + "vpsrld $0x18,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm1,(%1)%{%%k1%} \n" + "vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_YUY2TOUV422ROW_AVX512BW + #ifdef HAS_UYVYTOYROW_AVX512BW LIBYUV_TARGET_AVX512BW void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { @@ -11536,6 +11586,54 @@ void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, } #endif // HAS_UYVYTOUVROW_AVX512BW +#ifdef HAS_UYVYTOUV422ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void UYVYToUV422Row_AVX512BW(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + uintptr_t temp; + asm volatile( + "sub %1,%2 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpmovdb %%zmm0,(%1) \n" + "vpsrld $0x10,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,0x00(%1,%2,1) \n" + "lea 0x10(%1),%1 \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "jle 99f \n" + "add $0x1,%k3 \n" + "shr $0x1,%k3 \n" + "mov $-1,%k[temp] \n" + "bzhi %k3,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmovdb %%zmm0,(%1)%{%%k1%} \n" + "vpsrld $0x10,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "k1"); +} +#endif // HAS_UYVYTOUV422ROW_AVX512BW + #if defined(HAS_ARGBBLENDROW_AVX2) || defined(HAS_ARGBBLENDROW_AVX512BW) // Shuffle table for isolating alpha. SIMD_ALIGNED(static const uvec8 kShuffleAlpha[2]) = { diff --git a/source/row_neon64.cc b/source/row_neon64.cc index 14e6fd03d..c783ad5cd 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -32,6 +32,14 @@ extern "C" { // This module is for GCC Neon armv8 64 bit. #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) +#if defined(__clang__) +#pragma clang attribute push(__attribute__((target("dotprod,i8mm"))), \ + apply_to = function) +#elif defined(__GNUC__) +#pragma GCC push_options +#pragma GCC target("+dotprod+i8mm") +#endif + // v0.8h: Y // v1.16b: 8U, 8V @@ -6115,6 +6123,12 @@ void ARGBAffineRow_NEON(const uint8_t* src_argb, : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); } +#if defined(__clang__) +#pragma clang attribute pop +#elif defined(__GNUC__) +#pragma GCC pop_options +#endif + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/row_sme.cc b/source/row_sme.cc index f9b533ffb..4c20b5b87 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -10,6 +10,18 @@ #include "libyuv/compare_row.h" #include "libyuv/row.h" + +#if !defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__) +#if defined(__clang__) +#pragma clang attribute push( \ + __attribute__((target("arch=armv8.5-a+i8mm+sme"))), apply_to = function) +#elif defined(__GNUC__) +#pragma GCC push_options +#pragma GCC target("arch=armv8.5-a+i8mm+sve2+sme") +#endif +#endif + #include "libyuv/row_sve.h" #ifdef __cplusplus @@ -24,17 +36,8 @@ extern "C" { // asm statement, and other OS ABIs (e.g. Windows) differ in which registers // are callee-saved. List "memory" first, then "cc", then registers. -#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ - defined(__aarch64__) - -#define RGBTOARGB8_SVE_2X \ - /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \ - "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \ - "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \ - "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \ - "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \ - "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \ - "uqshrnt z18.b, z22.h, #6 \n" /* R1 */ +#if !defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__) __arm_locally_streaming void I444ToARGBRow_SME( const uint8_t* src_y, @@ -44,44 +47,7 @@ __arm_locally_streaming void I444ToARGBRow_SME( const struct YuvConstants* yuvconstants, int width) { // Streaming-SVE only, no use of ZA tile. - uint64_t vl; - asm volatile( - "cntb %[vl] \n" - "ptrue p0.b \n" // - YUVTORGB_SVE_SETUP - "dup z19.b, #255 \n" // A - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - // Run bulk of computation with an all-true predicate to avoid predicate - // generation overhead. - "ptrue p1.b \n" - "1: \n" // - READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X - "subs %w[width], %w[width], %w[vl] \n" - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" - "incb %[dst_argb], all, mul #4 \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width] \n" // - READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X - "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" - - "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_u] "+r"(src_u), // %[src_u] - [src_v] "+r"(src_v), // %[src_v] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + I444ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); } __arm_locally_streaming void I444ToRGB24Row_SME( @@ -465,51 +431,31 @@ __arm_locally_streaming void ARGBMultiplyRow_SME(const uint8_t* src_argb, [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : - : "memory", "cc", "z0", "z1", "z2", "p0", "p1"); + : "memory", "cc", "z0", "z1", "z2", "p0"); } __arm_locally_streaming void MergeUVRow_SME(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) { - // Streaming-SVE only, no use of ZA tile. - int vl; - asm volatile( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" + MergeUVRow_SVE_SC(src_u, src_v, dst_uv, width); +} - // Run bulk of computation with an all-true predicate to avoid predicate - // generation overhead. - "ptrue p0.b \n" - "1: \n" - "ld1b {z1.b}, p0/z, [%[src_u]] \n" - "ld1b {z2.b}, p0/z, [%[src_v]] \n" - "incb %[src_u] \n" - "incb %[src_v] \n" - "subs %w[width], %w[width], %w[vl] \n" - "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" - "incb %[dst_uv], all, mul #2 \n" - "b.ge 1b \n" +__arm_locally_streaming void SplitUVRow_SME(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + SplitUVRow_SVE_SC(src_uv, dst_u, dst_v, width); +} - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z1.b}, p0/z, [%[src_u]] \n" - "ld1b {z2.b}, p0/z, [%[src_v]] \n" - "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" - - "99: \n" - : [src_u] "+r"(src_u), // %[src_u] - [src_v] "+r"(src_v), // %[src_v] - [dst_uv] "+r"(dst_uv), // %[dst_uv] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "memory", "cc", "z1", "z2", "p0"); +__arm_locally_streaming void HalfMergeUVRow_SME(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width) { + HalfMergeUVRow_SVE_SC(src_u, src_stride_u, src_v, src_stride_v, dst_uv, + width); } __arm_locally_streaming void MergeUVRow_16_SME(const uint16_t* src_u, @@ -628,87 +574,14 @@ __arm_locally_streaming void Convert16To8Row_SME(const uint16_t* src_y, __arm_locally_streaming void CopyRow_SME(const uint8_t* src, uint8_t* dst, int width) { - // Streaming-SVE only, no use of ZA tile. - int vl; - asm volatile( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - // Run bulk of computation with an all-true predicate to avoid predicate - // generation overhead. - "ptrue p0.b \n" - "1: \n" - "ld1b {z0.b}, p0/z, [%[src]] \n" - "incb %[src] \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1b {z0.b}, p0, [%[dst]] \n" - "incb %[dst] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z0.b}, p0/z, [%[src]] \n" - "st1b {z0.b}, p0, [%[dst]] \n" - - "99: \n" - : [src] "+r"(src), // %[src] - [dst] "+r"(dst), // %[dst] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "memory", "cc", "z0", "p0"); + CopyRow_SVE_SC(src, dst, width); } -__arm_locally_streaming static void HalfRow_SME(uint8_t* dst_ptr, - const uint8_t* src_ptr, - ptrdiff_t src_stride, - int width) { - const uint8_t* src_ptr1 = src_ptr + src_stride; - - int vl; - asm volatile( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - // Run bulk of computation with an all-true predicate to avoid predicate - // generation overhead. - "ptrue p0.b \n" - "1: \n" - "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" - "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" - "incb %[src_ptr] \n" - "incb %[src_ptr1] \n" - "urhadd z2.b, p0/m, z2.b, z3.b \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1b {z2.b}, p0, [%[dst_ptr]] \n" - "incb %[dst_ptr] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" - "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" - "urhadd z2.b, p0/m, z2.b, z3.b \n" - "st1b {z2.b}, p0, [%[dst_ptr]] \n" - - "99: \n" - : [src_ptr] "+r"(src_ptr), // %[src_ptr] - [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] - [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "memory", "cc", "z2", "z3", "p0"); +__arm_locally_streaming void HalfRow_SME(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int width) { + HalfRow_SVE_SC(src_ptr, src_stride, dst_ptr, width); } __arm_locally_streaming void InterpolateRow_SME(uint8_t* dst_ptr, @@ -716,122 +589,14 @@ __arm_locally_streaming void InterpolateRow_SME(uint8_t* dst_ptr, ptrdiff_t src_stride, int width, int source_y_fraction) { - int y1_fraction = source_y_fraction; - int y0_fraction = 256 - y1_fraction; - const uint8_t* src_ptr1 = src_ptr + src_stride; - - if (y0_fraction == 0) { - CopyRow_SME(src_ptr1, dst_ptr, width); - return; - } - if (y0_fraction == 128) { - HalfRow_SME(dst_ptr, src_ptr, src_stride, width); - return; - } - if (y0_fraction == 256) { - CopyRow_SME(src_ptr, dst_ptr, width); - return; - } - - int vl; - asm volatile( - "cntb %x[vl] \n" - "dup z0.b, %w[y0_fraction] \n" - "dup z1.b, %w[y1_fraction] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - // Run bulk of computation with an all-true predicate to avoid predicate - // generation overhead. - "ptrue p0.b \n" - "1: \n" - "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" - "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" - "incb %[src_ptr] \n" - "incb %[src_ptr1] \n" - "umullb z4.h, z2.b, z0.b \n" - "umullt z2.h, z2.b, z0.b \n" - "subs %w[width], %w[width], %w[vl] \n" - "umlalb z4.h, z3.b, z1.b \n" - "umlalt z2.h, z3.b, z1.b \n" - "rshrnb z3.b, z4.h, #8 \n" - "rshrnt z3.b, z2.h, #8 \n" - "st1b {z3.b}, p0, [%[dst_ptr]] \n" - "incb %[dst_ptr] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" - "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" - "umullb z4.h, z2.b, z0.b \n" - "umullt z2.h, z2.b, z0.b \n" - "umlalb z4.h, z3.b, z1.b \n" - "umlalt z2.h, z3.b, z1.b \n" - "rshrnb z3.b, z4.h, #8 \n" - "rshrnt z3.b, z2.h, #8 \n" - "st1b {z3.b}, p0, [%[dst_ptr]] \n" - - "99: \n" - : [src_ptr] "+r"(src_ptr), // %[src_ptr] - [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] - [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] - [y1_fraction] "r"(y1_fraction) // %[y1_fraction] - : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "p0"); + InterpolateRow_SVE_SC(dst_ptr, src_ptr, src_stride, width, source_y_fraction); } -__arm_locally_streaming static void HalfRow_16_SME(uint16_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int width) { - const uint16_t* src_ptr1 = src_ptr + src_stride; - - int vl; - asm volatile( - "cnth %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - // Run bulk of computation with an all-true predicate to avoid predicate - // generation overhead. - "ptrue p0.h \n" - "1: \n" - "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" - "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" - "incb %[src_ptr] \n" - "incb %[src_ptr1] \n" - "urhadd z2.h, p0/m, z2.h, z3.h \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1h {z2.h}, p0, [%[dst_ptr]] \n" - "incb %[dst_ptr] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p0.h, wzr, %w[width] \n" - "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" - "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" - "urhadd z2.h, p0/m, z2.h, z3.h \n" - "st1h {z2.h}, p0, [%[dst_ptr]] \n" - - "99: \n" - : [src_ptr] "+r"(src_ptr), // %[src_ptr] - [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] - [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "memory", "cc", "z2", "z3", "p0"); +__arm_locally_streaming void HalfRow_16_SME(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int width) { + HalfRow_16_SVE_SC(src_ptr, src_stride, dst_ptr, width); } __arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr, @@ -839,77 +604,126 @@ __arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr, ptrdiff_t src_stride, int width, int source_y_fraction) { - int y1_fraction = source_y_fraction; - int y0_fraction = 256 - y1_fraction; - const uint16_t* src_ptr1 = src_ptr + src_stride; + InterpolateRow_16_SVE_SC(dst_ptr, src_ptr, src_stride, width, + source_y_fraction); +} - if (y0_fraction == 0) { - CopyRow_SME((const uint8_t*)src_ptr1, (uint8_t*)dst_ptr, - width * sizeof(uint16_t)); - return; - } - if (y0_fraction == 128) { - HalfRow_16_SME(dst_ptr, src_ptr, src_stride, width); - return; - } - if (y0_fraction == 256) { - CopyRow_SME((const uint8_t*)src_ptr, (uint8_t*)dst_ptr, - width * sizeof(uint16_t)); - return; - } +__arm_locally_streaming void DivideRow_16_SME(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + DivideRow_16_SVE_SC(src_y, dst_y, scale, width); +} - int vl; - asm volatile( - "cnth %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "dup z0.h, %w[y0_fraction] \n" - "dup z1.h, %w[y1_fraction] \n" - "b.lt 2f \n" +__arm_locally_streaming void HalfFloatRow_SME(const uint16_t* src, + uint16_t* dst, + float scale, + int width) { + HalfFloatRow_SVE_SC(src, dst, scale, width); +} - // Run bulk of computation with an all-true predicate to avoid predicate - // generation overhead. - "ptrue p0.h \n" - "1: \n" - "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" - "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" - "incb %[src_ptr] \n" - "incb %[src_ptr1] \n" - "umullb z4.s, z2.h, z0.h \n" - "umullt z2.s, z2.h, z0.h \n" - "subs %w[width], %w[width], %w[vl] \n" - "umlalb z4.s, z3.h, z1.h \n" - "umlalt z2.s, z3.h, z1.h \n" - "rshrnb z3.h, z4.s, #8 \n" - "rshrnt z3.h, z2.s, #8 \n" - "st1h {z3.h}, p0, [%[dst_ptr]] \n" - "incb %[dst_ptr] \n" - "b.ge 1b \n" +__arm_locally_streaming void HalfFloat1Row_SME(const uint16_t* src, + uint16_t* dst, + float scale, + int width) { + HalfFloat1Row_SVE_SC(src, dst, scale, width); +} - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" +__arm_locally_streaming void ARGBToRGB565Row_SME(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + ARGBToRGB565Row_SVE_SC(src_argb, dst_rgb, width); +} - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p0.h, wzr, %w[width] \n" - "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" - "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" - "umullb z4.s, z2.h, z0.h \n" - "umullt z2.s, z2.h, z0.h \n" - "umlalb z4.s, z3.h, z1.h \n" - "umlalt z2.s, z3.h, z1.h \n" - "rshrnb z3.h, z4.s, #8 \n" - "rshrnt z3.h, z2.s, #8 \n" - "st1h {z3.h}, p0, [%[dst_ptr]] \n" +__arm_locally_streaming void ARGBToRGB565DitherRow_SME(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width) { + ARGBToRGB565DitherRow_SVE_SC(src_argb, dst_rgb, dither4, width); +} - "99: \n" - : [src_ptr] "+r"(src_ptr), // %[src_ptr] - [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] - [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] - [y1_fraction] "r"(y1_fraction) // %[y1_fraction] - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0"); +__arm_locally_streaming void ARGB1555ToARGBRow_SME(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width) { + ARGB1555ToARGBRow_SVE_SC(src_argb1555, dst_argb, width); +} + +__arm_locally_streaming void AYUVToUVRow_SME(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + AYUVToUVRow_SVE_SC(src_ayuv, src_stride_ayuv, dst_uv, width); +} + +__arm_locally_streaming void AYUVToVURow_SME(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + AYUVToVURow_SVE_SC(src_ayuv, src_stride_ayuv, dst_vu, width); +} + +__arm_locally_streaming void RAWToARGBRow_SME(const uint8_t* src_raw, + uint8_t* dst_argb, + int width) { + RAWToARGBRow_SVE_SC(src_raw, dst_argb, width); +} + +__arm_locally_streaming void RAWToRGBARow_SME(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width) { + RAWToRGBARow_SVE_SC(src_raw, dst_rgba, width); +} + +__arm_locally_streaming void RGB24ToARGBRow_SME(const uint8_t* src_rgb24, + uint8_t* dst_argb, + int width) { + RGB24ToARGBRow_SVE_SC(src_rgb24, dst_argb, width); +} + +__arm_locally_streaming void RAWToRGB24Row_SME(const uint8_t* src_raw, + uint8_t* dst_rgb24, + int width) { + RAWToRGB24Row_SVE_SC(src_raw, dst_rgb24, width); +} + +__arm_locally_streaming void ARGBToRGB24Row_SME(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + ARGBToRGB24Row_SVE_SC(src_argb, dst_rgb, width); +} + +__arm_locally_streaming void ARGBToRAWRow_SME(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + ARGBToRAWRow_SVE_SC(src_argb, dst_rgb, width); +} + +__arm_locally_streaming void YUY2ToYRow_SME(const uint8_t* src_yuy2, + uint8_t* dst_y, + int width) { + YUY2ToYRow_SVE_SC(src_yuy2, dst_y, width); +} + +__arm_locally_streaming void UYVYToYRow_SME(const uint8_t* src_uyvy, + uint8_t* dst_y, + int width) { + UYVYToYRow_SVE_SC(src_uyvy, dst_y, width); +} + +__arm_locally_streaming void YUY2ToUVRow_SME(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + YUY2ToUVRow_SVE_SC(src_yuy2, src_stride_yuy2, dst_u, dst_v, width); +} + +__arm_locally_streaming void UYVYToUVRow_SME(const uint8_t* src_uyvy, + int src_stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + UYVYToUVRow_SVE_SC(src_uyvy, src_stride_uyvy, dst_u, dst_v, width); } __arm_locally_streaming void HalfRow_16To8_SME(const uint16_t* src_ptr, @@ -967,7 +781,7 @@ __arm_locally_streaming void HalfRow_16To8_SME(const uint16_t* src_ptr, [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [shift] "r"(shift) // %[shift] - : "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0"); + : "memory", "cc", "z2", "z3", "z31", "p0"); } __arm_locally_streaming void HalfWidthRow_16To8_SME(const uint16_t* src_ptr, @@ -1157,8 +971,14 @@ __arm_locally_streaming void ARGBAlphaBlendRow_SME(const uint8_t* src_argb0, } #endif // HAS_ARGBALPHABLENDROW_SME -#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && - // defined(__aarch64__) +#if defined(__clang__) +#pragma clang attribute pop +#elif defined(__GNUC__) +#pragma GCC pop_options +#endif + +#endif // !defined(LIBYUV_DISABLE_SME) && (defined(CLANG_HAS_SME) || + // defined(GCC_HAS_SME)) && defined(__aarch64__) #ifdef __cplusplus } // extern "C" diff --git a/source/row_sve.cc b/source/row_sve.cc index 79771cbb2..5bddaadec 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -8,12 +8,21 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include - -#include "libyuv/row_sve.h" #include "libyuv/compare_row.h" #include "libyuv/row.h" +#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) +#if defined(__clang__) +#pragma clang attribute push( \ + __attribute__((target("arch=armv8.5-a+i8mm+sve2"))), apply_to = function) +#elif defined(__GNUC__) +#pragma GCC push_options +#pragma GCC target("arch=armv8.5-a+i8mm+sve2") +#endif +#endif + +#include "libyuv/row_sve.h" + #ifdef __cplusplus namespace libyuv { extern "C" { @@ -28,68 +37,13 @@ extern "C" { #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) -#define RGBTOARGB8_SVE_2X \ - /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \ - "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \ - "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \ - "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \ - "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \ - "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \ - "uqshrnt z18.b, z22.h, #6 \n" /* R1 */ - -#define RGBTOARGB8_SVE_TOP_2X \ - /* Inputs: B: z16.h, G: z17.h, R: z18.h */ \ - "uqshl z16.h, p0/m, z16.h, #2 \n" /* B0 */ \ - "uqshl z17.h, p0/m, z17.h, #2 \n" /* G0 */ \ - "uqshl z18.h, p0/m, z18.h, #2 \n" /* R0 */ \ - "uqshl z20.h, p0/m, z20.h, #2 \n" /* B1 */ \ - "uqshl z21.h, p0/m, z21.h, #2 \n" /* G1 */ \ - "uqshl z22.h, p0/m, z22.h, #2 \n" /* R1 */ - void I444ToARGBRow_SVE2(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - uint64_t vl; - asm volatile( - "cnth %[vl] \n" - "ptrue p0.b \n" // - YUVTORGB_SVE_SETUP - "dup z19.b, #255 \n" // Alpha - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - // Run bulk of computation with an all-true predicate to avoid predicate - // generation overhead. - "ptrue p1.h \n" - "1: \n" // - READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE - "subs %w[width], %w[width], %w[vl] \n" - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width] \n" // - READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE - "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" - - "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_u] "+r"(src_u), // %[src_u] - [src_v] "+r"(src_v), // %[src_v] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] - : "cc", "memory", YUVTORGB_SVE_REGS); + I444ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); } void I444ToRGB24Row_SVE2(const uint8_t* src_y, @@ -248,265 +202,37 @@ void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb, ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c); } -#define ARGBTORGB565_SVE \ - /* Inputs: \ - * z0: rrrrrxxxbbbbbxxx \ - * z1: xxxxxxxxggggggxx \ - * z3: 0000000000000011 (3, 0, 3, 0, ...) \ - * z4: 0000011111100000 \ - */ \ - "lsr z0.b, p0/m, z0.b, z3.b \n" \ - "lsl z1.h, z1.h, #3 \n" \ - "bsl z1.d, z1.d, z0.d, z4.d \n" - void ARGBToRGB565Row_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { - unsigned bsl_mask = 0x7e0; - uint64_t vl; - width *= 2; - asm volatile( - "mov z3.h, #3 \n" - "dup z4.h, %w[bsl_mask] \n" - - "cntb %[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "1: \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA - "incb %[src], all, mul #2 \n" - "subs %w[width], %w[width], %w[vl] \n" // - ARGBTORGB565_SVE - "st1b {z1.b}, p0, [%[dst]] \n" - "incb %[dst] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "whilelt p0.b, wzr, %w[width] \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA - ARGBTORGB565_SVE - "st1b {z1.b}, p0, [%[dst]] \n" - - "99: \n" - : [src] "+r"(src_argb), // %[src] - [dst] "+r"(dst_rgb), // %[dst] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [bsl_mask] "r"(bsl_mask) // %[bsl_mask] - : "cc", "memory", "z0", "z1", "z3", "z4", "p0"); + ARGBToRGB565Row_SVE_SC(src_argb, dst_rgb, width); } void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, uint32_t dither4, int width) { - unsigned bsl_mask = 0x7e0; - uint64_t vl; - width *= 2; - asm volatile( - "mov z3.h, #3 \n" - "dup z4.h, %w[bsl_mask] \n" - "dup z2.s, %w[dither4] \n" - "zip1 z2.b, z2.b, z2.b \n" - - "cntb %[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "1: \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA - "incb %[src], all, mul #2 \n" - "uqadd z0.b, z0.b, z2.b \n" - "uqadd z1.b, z1.b, z2.b \n" - "subs %w[width], %w[width], %w[vl] \n" // - ARGBTORGB565_SVE - "st1b {z1.b}, p0, [%[dst]] \n" - "incb %[dst] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "whilelt p0.b, wzr, %w[width] \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA - "uqadd z0.b, z0.b, z2.b \n" - "uqadd z1.b, z1.b, z2.b \n" // - ARGBTORGB565_SVE - "st1b {z1.b}, p0, [%[dst]] \n" - - "99: \n" - : [src] "+r"(src_argb), // %[src] - [dst] "+r"(dst_rgb), // %[dst] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [bsl_mask] "r"(bsl_mask), // %[bsl_mask] - [dither4] "r"(dither4) // %[dither4] - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0"); + ARGBToRGB565DitherRow_SVE_SC(src_argb, dst_rgb, dither4, width); } -#define ARGB1555TOARGB \ - /* Input: z1/z3.h = arrrrrgggggbbbbb */ \ - "lsl z0.h, z1.h, #3 \n" /* rrrgggggbbbbb000 */ \ - "lsl z2.h, z3.h, #3 \n" /* rrrgggggbbbbb000 */ \ - "asr z1.h, z1.h, #7 \n" /* aaaaaaaarrrrrggg */ \ - "asr z3.h, z3.h, #7 \n" /* aaaaaaaarrrrrggg */ \ - "lsl z0.b, p0/m, z0.b, z4.b \n" /* ggggg000bbbbb000 */ \ - "lsl z2.b, p0/m, z2.b, z4.b \n" /* ggggg000bbbbb000 */ \ - "sri z1.b, z1.b, #5 \n" /* aaaaaaaarrrrrrrr */ \ - "sri z3.b, z3.b, #5 \n" /* aaaaaaaarrrrrrrr */ \ - "sri z0.b, z0.b, #5 \n" /* ggggggggbbbbbbbb */ \ - "sri z2.b, z2.b, #5 \n" /* ggggggggbbbbbbbb */ - void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555, uint8_t* dst_argb, int width) { - uint64_t vl; - asm volatile( - "mov z4.h, #0x0300 \n" - "ptrue p0.b \n" - - "cnth %x[vl] \n" - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.lt 2f \n" - - "1: \n" - "ld1h {z1.h}, p0/z, [%[src]] \n" - "ld1h {z3.h}, p0/z, [%[src], #1, mul vl] \n" - "incb %[src], all, mul #2 \n" // - ARGB1555TOARGB - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "st2h {z0.h, z1.h}, p0, [%[dst]] \n" - "st2h {z2.h, z3.h}, p0, [%[dst], #2, mul vl] \n" - "incb %[dst], all, mul #4 \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl], lsl #1 \n" - "b.eq 99f \n" - - "whilelt p1.h, wzr, %w[width] \n" - "whilelt p2.h, %w[vl], %w[width] \n" - "ld1h {z1.h}, p1/z, [%[src]] \n" - "ld1h {z3.h}, p2/z, [%[src], #1, mul vl] \n" // - ARGB1555TOARGB - "st2h {z0.h, z1.h}, p1, [%[dst]] \n" - "st2h {z2.h, z3.h}, p2, [%[dst], #2, mul vl] \n" - - "99: \n" - : [src] "+r"(src_argb1555), // %[src] - [dst] "+r"(dst_argb), // %[dst] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2"); + ARGB1555ToARGBRow_SVE_SC(src_argb1555, dst_argb, width); } -// clang-format off -#define AYUVTOUV_SVE(zU0, zV0, zU1, zV1) /* e.g. */ \ - "ld2h {z0.h, z1.h}, p0/z, [%[src0]] \n" /* VUVU.. YAYA.. */ \ - "ld2h {z1.h, z2.h}, p1/z, [%[src0], #2, mul vl] \n" /* VUVU.. YAYA.. */ \ - "ld2h {z2.h, z3.h}, p0/z, [%[src1]] \n" /* VUVU.. YAYA.. */ \ - "ld2h {z3.h, z4.h}, p1/z, [%[src1], #2, mul vl] \n" /* VUVU.. YAYA.. */ \ - "incb %[src0], all, mul #4 \n" \ - "incb %[src1], all, mul #4 \n" \ - "uaddlb z4.h, z0.b, z2.b \n" /* V */ \ - "uaddlt z5.h, z0.b, z2.b \n" /* U */ \ - "uaddlb z6.h, z1.b, z3.b \n" /* V */ \ - "uaddlt z7.h, z1.b, z3.b \n" /* U */ \ - "addp " #zU0 ".h, p0/m, " #zU0 ".h, " #zV0 ".h \n" /* UV */ \ - "addp " #zU1 ".h, p1/m, " #zU1 ".h, " #zV1 ".h \n" /* UV */ \ - "subs %w[width], %w[width], %w[vl] \n" \ - "urshr " #zU0 ".h, p0/m, " #zU0 ".h, #2 \n" /* U0V0 */ \ - "urshr " #zU1 ".h, p1/m, " #zU1 ".h, #2 \n" /* U0V0 */ \ - "st1b {" #zU0 ".h}, p0, [%[dst]] \n" \ - "st1b {" #zU1 ".h}, p1, [%[dst], #1, mul vl] \n" \ - "incb %[dst] \n" -// clang-format on - -// Filter 2 rows of AYUV UV's (444) into UV (420). -// AYUV is VUYA in memory. UV for NV12 is UV order in memory. void AYUVToUVRow_SVE2(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_uv, int width) { - // Output a row of UV values, filtering 2x2 rows of AYUV. - const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv; - int vl; - asm volatile ( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.h \n" - "ptrue p1.h \n" - "1: \n" - AYUVTOUV_SVE(z5, z4, z7, z6) - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "cnth %x[vl] \n" - "whilelt p0.h, wzr, %w[width] \n" // first row - "whilelt p1.h, %w[vl], %w[width] \n" // second row - AYUVTOUV_SVE(z5, z4, z7, z6) - - "99: \n" - : [src0]"+r"(src_ayuv), // %[src0] - [src1]"+r"(src_ayuv1), // %[src1] - [dst]"+r"(dst_uv), // %[dst] - [width]"+r"(width), // %[width] - [vl]"=&r"(vl) // %[vl] - : - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0", - "p1"); + AYUVToUVRow_SVE_SC(src_ayuv, src_stride_ayuv, dst_uv, width); } -// Filter 2 rows of AYUV UV's (444) into VU (420). void AYUVToVURow_SVE2(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, int width) { - // Output a row of VU values, filtering 2x2 rows of AYUV. - const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv; - int vl; - asm volatile ( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.h \n" - "ptrue p1.h \n" - "1: \n" - AYUVTOUV_SVE(z4, z5, z6, z7) - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "cnth %x[vl] \n" - "whilelt p0.h, wzr, %w[width] \n" // first row - "whilelt p1.h, %w[vl], %w[width] \n" // second row - AYUVTOUV_SVE(z4, z5, z6, z7) - - "99: \n" - : [src0]"+r"(src_ayuv), // %[src0] - [src1]"+r"(src_ayuv1), // %[src1] - [dst]"+r"(dst_vu), // %[dst] - [width]"+r"(width), // %[width] - [vl]"=&r"(vl) // %[vl] - : - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0", - "p1"); + AYUVToVURow_SVE_SC(src_ayuv, src_stride_ayuv, dst_vu, width); } void YUY2ToARGBRow_SVE2(const uint8_t* src_yuy2, @@ -523,434 +249,51 @@ void UYVYToARGBRow_SVE2(const uint8_t* src_uyvy, UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width); } -static inline void RAWToWXYZRow_SVE2(const uint8_t* src_raw, - uint8_t* dst_wxyz, - int width, - uint32_t idx_start, - uint32_t idx_step, - uint32_t alpha) { - uint32_t vl; - asm("cntw %x0" : "=r"(vl)); - uint32_t vl_mul3 = vl * 3; - uint32_t rem_mul3; - asm volatile( - "index z31.s, %w[idx_start], %w[idx_step] \n" - "dup z30.s, %w[alpha] \n" - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.lt 2f \n" - - // Run bulk of computation with the same predicates to avoid predicate - // generation overhead. We set up p1 to only load 3/4 of a vector. - "ptrue p0.s \n" - "whilelt p1.b, wzr, %w[vl_mul3] \n" - "1: \n" - "ld1b {z0.b}, p1/z, [%[src]] \n" - "add %[src], %[src], %x[vl_mul3] \n" - "ld1b {z1.b}, p1/z, [%[src]] \n" - "add %[src], %[src], %x[vl_mul3] \n" - "tbl z0.b, {z0.b}, z31.b \n" - "tbl z1.b, {z1.b}, z31.b \n" - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "orr z0.d, z0.d, z30.d \n" - "orr z1.d, z1.d, z30.d \n" - "st1w {z0.s}, p0, [%[dst]] \n" - "st1w {z1.s}, p0, [%[dst], #1, mul vl] \n" - "incb %[dst], all, mul #2 \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl], lsl #1 \n" - "b.eq 99f \n" - - // Calculate a pair of predicates for the final iteration to deal with - // the tail. - "3: \n" - "add %w[rem_mul3], %w[width], %w[width], lsl #1 \n" - "whilelt p0.s, wzr, %w[width] \n" - "whilelt p1.b, wzr, %w[rem_mul3] \n" - "ld1b {z0.b}, p1/z, [%[src]] \n" - "add %[src], %[src], %x[vl_mul3] \n" - "tbl z0.b, {z0.b}, z31.b \n" - "subs %w[width], %w[width], %w[vl] \n" - "orr z0.d, z0.d, z30.d \n" - "st1w {z0.s}, p0, [%[dst]] \n" - "incb %[dst] \n" - "b.gt 3b \n" - - "99: \n" - : [src] "+r"(src_raw), // %[src] - [dst] "+r"(dst_wxyz), // %[dst] - [width] "+r"(width), // %[width] - [vl_mul3] "+r"(vl_mul3), // %[vl_mul3] - [rem_mul3] "=&r"(rem_mul3) // %[rem_mul3] - : [idx_start] "r"(idx_start), // %[idx_start] - [idx_step] "r"(idx_step), // %[idx_step] - [alpha] "r"(alpha), // %[alpha] - [vl] "r"(vl) // %[vl] - : "cc", "memory", "z0", "z1", "z30", "z31", "p0", "p1"); -} - void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width) { - RAWToWXYZRow_SVE2(src_raw, dst_argb, width, 0xff000102U, 0x00030303U, - 0xff000000U); + RAWToARGBRow_SVE_SC(src_raw, dst_argb, width); } void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { - RAWToWXYZRow_SVE2(src_raw, dst_rgba, width, 0x000102ffU, 0x03030300U, - 0x000000ffU); + RAWToRGBARow_SVE_SC(src_raw, dst_rgba, width); } void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24, uint8_t* dst_argb, int width) { - RAWToWXYZRow_SVE2(src_rgb24, dst_argb, width, 0xff020100U, 0x00030303U, - 0xff000000U); + RGB24ToARGBRow_SVE_SC(src_rgb24, dst_argb, width); } -static const uint8_t kRAWToRGB24Indices[] = { - 2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12, - 17, 16, 15, 20, 19, 18, 23, 22, 21, 26, 25, 24, 29, 28, 27, - 32, 31, 30, 35, 34, 33, 38, 37, 36, 41, 40, 39, 44, 43, 42, - 47, 46, 45, 50, 49, 48, 53, 52, 51, 56, 55, 54, 59, 58, 57, - 62, 61, 60, 65, 64, 63, 68, 67, 66, 71, 70, 69, 74, 73, 72, - 77, 76, 75, 80, 79, 78, 83, 82, 81, 86, 85, 84, 89, 88, 87, - 92, 91, 90, 95, 94, 93, 98, 97, 96, 101, 100, 99, 104, 103, 102, - 107, 106, 105, 110, 109, 108, 113, 112, 111, 116, 115, 114, 119, 118, 117, - 122, 121, 120, 125, 124, 123, 128, 127, 126, 131, 130, 129, 134, 133, 132, - 137, 136, 135, 140, 139, 138, 143, 142, 141, 146, 145, 144, 149, 148, 147, - 152, 151, 150, 155, 154, 153, 158, 157, 156, 161, 160, 159, 164, 163, 162, - 167, 166, 165, 170, 169, 168, 173, 172, 171, 176, 175, 174, 179, 178, 177, - 182, 181, 180, 185, 184, 183, 188, 187, 186, 191, 190, 189, 194, 193, 192, - 197, 196, 195, 200, 199, 198, 203, 202, 201, 206, 205, 204, 209, 208, 207, - 212, 211, 210, 215, 214, 213, 218, 217, 216, 221, 220, 219, 224, 223, 222, - 227, 226, 225, 230, 229, 228, 233, 232, 231, 236, 235, 234, 239, 238, 237, - 242, 241, 240, 245, 244, 243, 248, 247, 246, 251, 250, 249, 254, 253, 252}; - void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { - // width is in elements, convert to bytes. - width *= 3; - // we use the mul3 predicate pattern throughout to use the largest multiple - // of three number of lanes, for instance with a vector length of 16 bytes - // only the first 15 bytes will be used for load/store instructions. - uint32_t vl; - asm volatile( - "cntb %x[vl], mul3 \n" - "ptrue p0.b, mul3 \n" - "ld1b {z31.b}, p0/z, [%[kIndices]] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - // Run bulk of computation with the same predicate to avoid predicate - // generation overhead. - "1: \n" - "ld1b {z0.b}, p0/z, [%[src]] \n" - "add %[src], %[src], %x[vl] \n" - "tbl z0.b, {z0.b}, z31.b \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1b {z0.b}, p0, [%[dst]] \n" - "add %[dst], %[dst], %x[vl] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z0.b}, p0/z, [%[src]] \n" - "tbl z0.b, {z0.b}, z31.b \n" - "st1b {z0.b}, p0, [%[dst]] \n" - - "99: \n" - : [src] "+r"(src_raw), // %[src] - [dst] "+r"(dst_rgb24), // %[dst] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [kIndices] "r"(kRAWToRGB24Indices) // %[kIndices] - : "cc", "memory", "z0", "z31", "p0"); + RAWToRGB24Row_SVE_SC(src_raw, dst_rgb24, width); } -static inline void ARGBToXYZRow_SVE2(const uint8_t* src_argb, - uint8_t* dst_xyz, - int width, - const uint8_t* indices) { - uint32_t vl; - asm("cntw %x0" : "=r"(vl)); - uint32_t vl_mul3 = vl * 3; - uint32_t rem_mul3; - asm volatile( - "whilelt p1.b, wzr, %w[vl_mul3] \n" - "ld1b {z31.b}, p1/z, [%[indices]] \n" - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.lt 2f \n" - - // Run bulk of computation with the same predicates to avoid predicate - // generation overhead. We set up p1 to only store 3/4 of a vector. - "ptrue p0.s \n" - "1: \n" - "ld1w {z0.s}, p0/z, [%[src]] \n" - "ld1w {z1.s}, p0/z, [%[src], #1, mul vl] \n" - "incb %[src], all, mul #2 \n" - "tbl z0.b, {z0.b}, z31.b \n" - "tbl z1.b, {z1.b}, z31.b \n" - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "st1b {z0.b}, p1, [%[dst]] \n" - "add %[dst], %[dst], %x[vl_mul3] \n" - "st1b {z1.b}, p1, [%[dst]] \n" - "add %[dst], %[dst], %x[vl_mul3] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl], lsl #1 \n" - "b.eq 99f \n" - - // Calculate predicates for the final iteration to deal with the tail. - "add %w[rem_mul3], %w[width], %w[width], lsl #1 \n" - "whilelt p0.s, wzr, %w[width] \n" - "whilelt p1.b, wzr, %w[rem_mul3] \n" - "whilelt p2.s, %w[vl], %w[width] \n" - "whilelt p3.b, %w[vl_mul3], %w[rem_mul3] \n" - "ld1w {z0.s}, p0/z, [%[src]] \n" - "ld1w {z1.s}, p2/z, [%[src], #1, mul vl] \n" - "tbl z0.b, {z0.b}, z31.b \n" - "tbl z1.b, {z1.b}, z31.b \n" - "st1b {z0.b}, p1, [%[dst]] \n" - "add %[dst], %[dst], %x[vl_mul3] \n" - "st1b {z1.b}, p3, [%[dst]] \n" - - "99: \n" - : [src] "+r"(src_argb), // %[src] - [dst] "+r"(dst_xyz), // %[dst] - [width] "+r"(width), // %[width] - [rem_mul3] "=&r"(rem_mul3) // %[rem_mul3] - : [indices] "r"(indices), // %[indices] - [vl_mul3] "r"(vl_mul3), // %[vl_mul3] - [vl] "r"(vl) // %[vl] - : "cc", "memory", "z0", "z1", "z31", "p0", "p1", "p2", "p3"); -} - -static const uint8_t kARGBToRGB24RowIndices[] = { - 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, - 20, 21, 22, 24, 25, 26, 28, 29, 30, 32, 33, 34, 36, 37, 38, - 40, 41, 42, 44, 45, 46, 48, 49, 50, 52, 53, 54, 56, 57, 58, - 60, 61, 62, 64, 65, 66, 68, 69, 70, 72, 73, 74, 76, 77, 78, - 80, 81, 82, 84, 85, 86, 88, 89, 90, 92, 93, 94, 96, 97, 98, - 100, 101, 102, 104, 105, 106, 108, 109, 110, 112, 113, 114, 116, 117, 118, - 120, 121, 122, 124, 125, 126, 128, 129, 130, 132, 133, 134, 136, 137, 138, - 140, 141, 142, 144, 145, 146, 148, 149, 150, 152, 153, 154, 156, 157, 158, - 160, 161, 162, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178, - 180, 181, 182, 184, 185, 186, 188, 189, 190, 192, 193, 194, 196, 197, 198, - 200, 201, 202, 204, 205, 206, 208, 209, 210, 212, 213, 214, 216, 217, 218, - 220, 221, 222, 224, 225, 226, 228, 229, 230, 232, 233, 234, 236, 237, 238, - 240, 241, 242, 244, 245, 246, 248, 249, 250, 252, 253, 254, -}; - -static const uint8_t kARGBToRAWRowIndices[] = { - 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 18, 17, 16, - 22, 21, 20, 26, 25, 24, 30, 29, 28, 34, 33, 32, 38, 37, 36, - 42, 41, 40, 46, 45, 44, 50, 49, 48, 54, 53, 52, 58, 57, 56, - 62, 61, 60, 66, 65, 64, 70, 69, 68, 74, 73, 72, 78, 77, 76, - 82, 81, 80, 86, 85, 84, 90, 89, 88, 94, 93, 92, 98, 97, 96, - 102, 101, 100, 106, 105, 104, 110, 109, 108, 114, 113, 112, 118, 117, 116, - 122, 121, 120, 126, 125, 124, 130, 129, 128, 134, 133, 132, 138, 137, 136, - 142, 141, 140, 146, 145, 144, 150, 149, 148, 154, 153, 152, 158, 157, 156, - 162, 161, 160, 166, 165, 164, 170, 169, 168, 174, 173, 172, 178, 177, 176, - 182, 181, 180, 186, 185, 184, 190, 189, 188, 194, 193, 192, 198, 197, 196, - 202, 201, 200, 206, 205, 204, 210, 209, 208, 214, 213, 212, 218, 217, 216, - 222, 221, 220, 226, 225, 224, 230, 229, 228, 234, 233, 232, 238, 237, 236, - 242, 241, 240, 246, 245, 244, 250, 249, 248, 254, 253, 252, -}; - void ARGBToRGB24Row_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { - ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRGB24RowIndices); + ARGBToRGB24Row_SVE_SC(src_argb, dst_rgb, width); } void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { - ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRAWRowIndices); + ARGBToRAWRow_SVE_SC(src_argb, dst_rgb, width); } void DivideRow_16_SVE2(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { - if (scale == 65536) { // Divide by 1, so do memcpy. - if (src_y != dst_y) { - memcpy(dst_y, src_y, (size_t)width * 2); - } - return; - } - uint64_t vl; - asm volatile( - "cnth %x[vl] \n" - "dup z0.h, %w[scale] \n" - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.lt 2f \n" - - // Run bulk of computation with the same predicates to avoid predicate - // generation overhead. - "ptrue p0.h \n" - "1: \n" - "ld1h {z1.h}, p0/z, [%[src]] \n" - "ld1h {z2.h}, p0/z, [%[src], #1, mul vl] \n" - "incb %[src], all, mul #2 \n" - "umulh z1.h, z1.h, z0.h \n" - "umulh z2.h, z2.h, z0.h \n" - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "st1h {z1.h}, p0, [%[dst]] \n" - "st1h {z2.h}, p0, [%[dst], #1, mul vl] \n" - "incb %[dst], all, mul #2 \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl], lsl #1 \n" - "b.eq 99f \n" - - // Calculate a pair of predicates for the final iteration to deal with - // the tail. - "whilelt p0.h, wzr, %w[width] \n" - "whilelt p1.h, %w[vl], %w[width] \n" - "ld1h {z1.h}, p0/z, [%[src]] \n" - "ld1h {z2.h}, p1/z, [%[src], #1, mul vl] \n" - "umulh z1.h, z1.h, z0.h \n" - "umulh z2.h, z2.h, z0.h \n" - "st1h {z1.h}, p0, [%[dst]] \n" - "st1h {z2.h}, p1, [%[dst], #1, mul vl] \n" - - "99: \n" - : [src] "+r"(src_y), // %[src] - [dst] "+r"(dst_y), // %[dst] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [scale] "r"(scale) // %[scale] - : "cc", "memory", "z0", "z1", "z2", "p0", "p1"); + DivideRow_16_SVE_SC(src_y, dst_y, scale, width); } -#define HALFFLOAT_SVE \ - "scvtf z0.s, p0/m, z0.s \n" \ - "scvtf z1.s, p0/m, z1.s \n" \ - "scvtf z2.s, p0/m, z2.s \n" \ - "scvtf z3.s, p0/m, z3.s \n" \ - "fmul z0.s, z0.s, z4.s \n" \ - "fmul z1.s, z1.s, z4.s \n" \ - "fmul z2.s, z2.s, z4.s \n" \ - "fmul z3.s, z3.s, z4.s \n" \ - "fmul z0.s, z0.s, z5.s \n" \ - "fmul z1.s, z1.s, z5.s \n" \ - "fmul z2.s, z2.s, z5.s \n" \ - "fmul z3.s, z3.s, z5.s \n" \ - "uqshrnb z0.h, z0.s, #13 \n" \ - "uqshrnb z1.h, z1.s, #13 \n" \ - "uqshrnb z2.h, z2.s, #13 \n" \ - "uqshrnb z3.h, z3.s, #13 \n" - void HalfFloatRow_SVE2(const uint16_t* src, uint16_t* dst, float scale, int width) { - uint64_t vl; - asm("cntw %x0" : "=r"(vl)); - asm volatile( - "mov z4.s, %s[scale] \n" - "mov z5.s, %s[bias] \n" - "subs %w[width], %w[width], %w[vl], lsl #2 \n" - "b.lt 2f \n" - - // Run bulk of computation with all-true predicates to avoid predicate - // generation overhead. - "ptrue p0.s \n" - "1: \n" - "ld1h {z0.s}, p0/z, [%[src]] \n" - "ld1h {z1.s}, p0/z, [%[src], #1, mul vl] \n" - "ld1h {z2.s}, p0/z, [%[src], #2, mul vl] \n" - "ld1h {z3.s}, p0/z, [%[src], #3, mul vl] \n" - "incb %[src], all, mul #2 \n" // - HALFFLOAT_SVE - "subs %w[width], %w[width], %w[vl], lsl #2 \n" - "st1h {z0.s}, p0, [%[dst]] \n" - "st1h {z1.s}, p0, [%[dst], #1, mul vl] \n" - "st1h {z2.s}, p0, [%[dst], #2, mul vl] \n" - "st1h {z3.s}, p0, [%[dst], #3, mul vl] \n" - "incb %[dst], all, mul #2 \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl], lsl #2 \n" - "b.eq 99f \n" - - // Calculate predicates for the final iteration to deal with the tail. - "whilelt p0.s, wzr, %w[width] \n" - "whilelt p1.s, %w[vl], %w[width] \n" - "whilelt p2.s, %w[vl2], %w[width] \n" - "whilelt p3.s, %w[vl3], %w[width] \n" - "ld1h {z0.s}, p0/z, [%[src]] \n" - "ld1h {z1.s}, p1/z, [%[src], #1, mul vl] \n" - "ld1h {z2.s}, p2/z, [%[src], #2, mul vl] \n" - "ld1h {z3.s}, p3/z, [%[src], #3, mul vl] \n" // - HALFFLOAT_SVE - "st1h {z0.s}, p0, [%[dst]] \n" - "st1h {z1.s}, p1, [%[dst], #1, mul vl] \n" - "st1h {z2.s}, p2, [%[dst], #2, mul vl] \n" - "st1h {z3.s}, p3, [%[dst], #3, mul vl] \n" - - "99: \n" - : [src] "+r"(src), // %[src] - [dst] "+r"(dst), // %[dst] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [vl2] "r"(vl * 2), // %[vl2] - [vl3] "r"(vl * 3), // %[vl3] - [scale] "w"(scale), // %[scale] - [bias] "w"(1.9259299444e-34f) // %[bias] 2^-112 - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "p0", "p1", "p2", - "p3"); + HalfFloatRow_SVE_SC(src, dst, scale, width); } void HalfFloat1Row_SVE2(const uint16_t* src, uint16_t* dst, float scale, int width) { - uint64_t vl; - asm volatile( - "cnth %x[vl] \n" - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.lt 2f \n" - - // Run bulk of computation with all-true predicates to avoid predicate - // generation overhead. - "ptrue p0.h \n" - "1: \n" - "ld1h {z0.h}, p0/z, [%[src]] \n" - "ld1h {z1.h}, p0/z, [%[src], #1, mul vl] \n" - "incb %[src], all, mul #2 \n" - "ucvtf z0.h, p0/m, z0.h \n" - "ucvtf z1.h, p0/m, z1.h \n" - "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "st1h {z0.h}, p0, [%[dst]] \n" - "st1h {z1.h}, p0, [%[dst], #1, mul vl] \n" - "incb %[dst], all, mul #2 \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl], lsl #1 \n" - "b.eq 99f \n" - - // Calculate predicates for the final iteration to deal with the tail. - "whilelt p0.h, wzr, %w[width] \n" - "whilelt p1.h, %w[vl], %w[width] \n" - "ld1h {z0.h}, p0/z, [%[src]] \n" - "ld1h {z1.h}, p1/z, [%[src], #1, mul vl] \n" - "ucvtf z0.h, p0/m, z0.h \n" - "ucvtf z1.h, p0/m, z1.h \n" - "st1h {z0.h}, p0, [%[dst]] \n" - "st1h {z1.h}, p1, [%[dst], #1, mul vl] \n" - - "99: \n" - : [src] "+r"(src), // %[src] - [dst] "+r"(dst), // %[dst] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "cc", "memory", "z0", "z1", "p0", "p1"); + HalfFloat1Row_SVE_SC(src, dst, scale, width); } void I210ToARGBRow_SVE2(const uint16_t* src_y, @@ -1073,80 +416,14 @@ void MergeUVRow_SVE2(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) { - int vl; - asm volatile( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "1: \n" - "ld1b {z1.b}, p0/z, [%[src_u]] \n" - "ld1b {z2.b}, p0/z, [%[src_v]] \n" - "incb %[src_u] \n" - "incb %[src_v] \n" - "subs %w[width], %w[width], %w[vl] \n" - "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" - "incb %[dst_uv], all, mul #2 \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z1.b}, p0/z, [%[src_u]] \n" - "ld1b {z2.b}, p0/z, [%[src_v]] \n" - "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" - - "99: \n" - : [src_u] "+r"(src_u), // %[src_u] - [src_v] "+r"(src_v), // %[src_v] - [dst_uv] "+r"(dst_uv), // %[dst_uv] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "memory", "cc", "z1", "z2", "p0"); + MergeUVRow_SVE_SC(src_u, src_v, dst_uv, width); } void SplitUVRow_SVE2(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width) { - int vl; - asm volatile( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "1: \n" - "ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n" - "incb %[src_uv], all, mul #2 \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1b {z1.b}, p0, [%[dst_u]] \n" - "st1b {z2.b}, p0, [%[dst_v]] \n" - "incb %[dst_u] \n" - "incb %[dst_v] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "whilelt p0.b, wzr, %w[width] \n" - "ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n" - "st1b {z1.b}, p0, [%[dst_u]] \n" - "st1b {z2.b}, p0, [%[dst_v]] \n" - - "99: \n" - : [src_uv] "+r"(src_uv), // %[src_uv] - [dst_u] "+r"(dst_u), // %[dst_u] - [dst_v] "+r"(dst_v), // %[dst_v] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "memory", "cc", "z1", "z2", "p0"); + SplitUVRow_SVE_SC(src_uv, dst_u, dst_v, width); } void HalfMergeUVRow_SVE2(const uint8_t* src_u, @@ -1155,106 +432,26 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u, int src_stride_v, uint8_t* dst_uv, int width) { - const uint8_t* src_u1 = src_u + src_stride_u; - const uint8_t* src_v1 = src_v + src_stride_v; - int vl; - asm volatile( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "1: \n" - "ld1b {z1.b}, p0/z, [%[src_u]] \n" - "ld1b {z2.b}, p0/z, [%[src_u1]] \n" - "ld1b {z3.b}, p0/z, [%[src_v]] \n" - "ld1b {z4.b}, p0/z, [%[src_v1]] \n" - "incb %[src_u] \n" - "incb %[src_u1] \n" - "incb %[src_v] \n" - "incb %[src_v1] \n" - "uaddlb z5.h, z1.b, z2.b \n" - "uaddlb z6.h, z3.b, z4.b \n" - "uaddlt z1.h, z1.b, z2.b \n" - "uaddlt z3.h, z3.b, z4.b \n" - "add z1.h, z1.h, z5.h \n" - "add z3.h, z3.h, z6.h \n" - "rshrnb z1.b, z1.h, #2 \n" - "rshrnt z1.b, z3.h, #2 \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1b {z1.b}, p0, [%[dst_uv]] \n" - "incb %[dst_uv] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z1.b}, p0/z, [%[src_u]] \n" - "ld1b {z2.b}, p0/z, [%[src_u1]] \n" - "ld1b {z3.b}, p0/z, [%[src_v]] \n" - "ld1b {z4.b}, p0/z, [%[src_v1]] \n" - "uaddlb z5.h, z1.b, z2.b \n" - "uaddlb z6.h, z3.b, z4.b \n" - "uaddlt z1.h, z1.b, z2.b \n" - "uaddlt z3.h, z3.b, z4.b \n" - "add z1.h, z1.h, z5.h \n" - "add z3.h, z3.h, z6.h \n" - "rshrnb z1.b, z1.h, #2 \n" - "rshrnt z1.b, z3.h, #2 \n" - "st1b {z1.b}, p0, [%[dst_uv]] \n" - - "99: \n" - : [src_u] "+r"(src_u), // %[src_u] - [src_u1] "+r"(src_u1), // %[src_u1] - [src_v] "+r"(src_v), // %[src_v] - [src_v1] "+r"(src_v1), // %[src_v1] - [dst_uv] "+r"(dst_uv), // %[dst_uv] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "cc", "memory", "z1", "z2", "z3", "z4", "z5", "z6", "p0"); + HalfMergeUVRow_SVE_SC(src_u, src_stride_u, src_v, src_stride_v, dst_uv, + width); } void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) { - int vl; - int vl2; - asm volatile( - "cntb %x[vl] \n" - "lsl %w[vl2], %w[vl], #1 \n" - "subs %w[width], %w[width], %w[vl2] \n" - "b.lt 2f \n" + CopyRow_SVE_SC(src, dst, width); +} - "ptrue p0.b \n" - "1: \n" - "ld1b {z0.b}, p0/z, [%[src], #0, mul vl] \n" - "ld1b {z1.b}, p0/z, [%[src], #1, mul vl] \n" - "incb %[src], all, mul #2 \n" - "subs %w[width], %w[width], %w[vl2] \n" - "st1b {z0.b}, p0, [%[dst], #0, mul vl] \n" - "st1b {z1.b}, p0, [%[dst], #1, mul vl] \n" - "incb %[dst], all, mul #2 \n" - "b.ge 1b \n" +void HalfRow_SVE2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int width) { + HalfRow_SVE_SC(src_ptr, src_stride, dst_ptr, width); +} - "2: \n" - "adds %w[width], %w[width], %w[vl2] \n" - "b.eq 99f \n" - - "3: \n" - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z0.b}, p0/z, [%[src]] \n" - "incb %[src] \n" - "st1b {z0.b}, p0, [%[dst]] \n" - "incb %[dst] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.gt 3b \n" - - "99: \n" - : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [vl] "=&r"(vl), - [vl2] "=&r"(vl2) - : - : "cc", "memory", "z0", "z1", "p0"); +void HalfRow_16_SVE2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int width) { + HalfRow_16_SVE_SC(src_ptr, src_stride, dst_ptr, width); } void InterpolateRow_SVE2(uint8_t* dst_ptr, @@ -1262,170 +459,24 @@ void InterpolateRow_SVE2(uint8_t* dst_ptr, ptrdiff_t src_stride, int width, int source_y_fraction) { - int y1_fraction = source_y_fraction; - int y0_fraction = 256 - y1_fraction; - const uint8_t* src_ptr1 = src_ptr + src_stride; + InterpolateRow_SVE_SC(dst_ptr, src_ptr, src_stride, width, source_y_fraction); +} - if (y0_fraction == 0) { - CopyRow_SVE2(src_ptr1, dst_ptr, width); - return; - } - if (y0_fraction == 256) { - CopyRow_SVE2(src_ptr, dst_ptr, width); - return; - } - if (y0_fraction == 128) { - int vl; - asm volatile( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "1: \n" - "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" - "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" - "incb %[src_ptr] \n" - "incb %[src_ptr1] \n" - "subs %w[width], %w[width], %w[vl] \n" - "urhadd z2.b, p0/m, z2.b, z3.b \n" - "st1b {z2.b}, p0, [%[dst_ptr]] \n" - "incb %[dst_ptr] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" - "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" - "urhadd z2.b, p0/m, z2.b, z3.b \n" - "st1b {z2.b}, p0, [%[dst_ptr]] \n" - - "99: \n" - : [src_ptr] "+r"(src_ptr), [src_ptr1] "+r"(src_ptr1), - [dst_ptr] "+r"(dst_ptr), [width] "+r"(width), [vl] "=&r"(vl) - : - : "cc", "memory", "z2", "z3", "p0"); - return; - } - - int vl; - asm volatile( - "cntb %x[vl] \n" - "dup z0.b, %w[y0_fraction] \n" - "dup z1.b, %w[y1_fraction] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "1: \n" - "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" - "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" - "incb %[src_ptr] \n" - "incb %[src_ptr1] \n" - "umullb z4.h, z2.b, z0.b \n" - "umullt z2.h, z2.b, z0.b \n" - "subs %w[width], %w[width], %w[vl] \n" - "umlalb z4.h, z3.b, z1.b \n" - "umlalt z2.h, z3.b, z1.b \n" - "rshrnb z3.b, z4.h, #8 \n" - "rshrnt z3.b, z2.h, #8 \n" - "st1b {z3.b}, p0, [%[dst_ptr]] \n" - "incb %[dst_ptr] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "whilelt p0.b, wzr, %w[width] \n" - "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" - "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" - "umullb z4.h, z2.b, z0.b \n" - "umullt z2.h, z2.b, z0.b \n" - "umlalb z4.h, z3.b, z1.b \n" - "umlalt z2.h, z3.b, z1.b \n" - "rshrnb z3.b, z4.h, #8 \n" - "rshrnt z3.b, z2.h, #8 \n" - "st1b {z3.b}, p0, [%[dst_ptr]] \n" - - "99: \n" - : [src_ptr] "+r"(src_ptr), // %[src_ptr] - [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] - [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] - [y1_fraction] "r"(y1_fraction) // %[y1_fraction] - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0"); +void InterpolateRow_16_SVE2(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + InterpolateRow_16_SVE_SC(dst_ptr, src_ptr, src_stride, width, + source_y_fraction); } void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { - int vl; - asm volatile( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "1: \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" - "incb %[src_yuy2], all, mul #2 \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1b {z0.b}, p0, [%[dst_y]] \n" - "incb %[dst_y] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "whilelt p0.b, wzr, %w[width] \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" - "st1b {z0.b}, p0, [%[dst_y]] \n" - - "99: \n" - : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2] - [dst_y] "+r"(dst_y), // %[dst_y] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "memory", "cc", "z0", "z1", "p0"); + YUY2ToYRow_SVE_SC(src_yuy2, dst_y, width); } void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { - int vl; - asm volatile( - "cntb %x[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "1: \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" - "incb %[src_uyvy], all, mul #2 \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1b {z1.b}, p0, [%[dst_y]] \n" - "incb %[dst_y] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "whilelt p0.b, wzr, %w[width] \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" - "st1b {z1.b}, p0, [%[dst_y]] \n" - - "99: \n" - : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy] - [dst_y] "+r"(dst_y), // %[dst_y] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : - : "memory", "cc", "z0", "z1", "p0"); + UYVYToYRow_SVE_SC(src_uyvy, dst_y, width); } void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2, @@ -1433,63 +484,7 @@ void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_u, uint8_t* dst_v, int width) { - const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2; - int vl; - int vl_half; - int width_half; - if (src_stride_yuy2 == 0) { - src_yuy2_1 = src_yuy2; - } - asm volatile( - "cntb %x[vl] \n" - "lsr %w[vl_half], %w[vl], #1 \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "whilelt p1.b, wzr, %w[vl_half] \n" - "1: \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" - "ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n" - "incb %[src_yuy2], all, mul #2 \n" - "incb %[src_yuy2_1], all, mul #2 \n" - "urhadd z1.b, p0/m, z1.b, z5.b \n" - "uzp1 z0.b, z1.b, z1.b \n" - "uzp2 z2.b, z1.b, z1.b \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1b {z0.b}, p1, [%[dst_u]] \n" - "st1b {z2.b}, p1, [%[dst_v]] \n" - "add %[dst_u], %[dst_u], %x[vl_half] \n" - "add %[dst_v], %[dst_v], %x[vl_half] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "add %w[width_half], %w[width], #1 \n" - "lsr %w[width_half], %w[width_half], #1 \n" - "whilelt p1.b, wzr, %w[width_half] \n" - "zip1 p0.b, p1.b, p1.b \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" - "ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n" - "urhadd z1.b, p0/m, z1.b, z5.b \n" - "uzp1 z0.b, z1.b, z1.b \n" - "uzp2 z2.b, z1.b, z1.b \n" - "st1b {z0.b}, p1, [%[dst_u]] \n" - "st1b {z2.b}, p1, [%[dst_v]] \n" - - "99: \n" - : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2] - [src_yuy2_1] "+r"(src_yuy2_1), // %[src_yuy2_1] - [dst_u] "+r"(dst_u), // %[dst_u] - [dst_v] "+r"(dst_v), // %[dst_v] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl), // %[vl] - [vl_half] "=&r"(vl_half), // %[vl_half] - [width_half] "=&r"(width_half) // %[width_half] - : - : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1"); + YUY2ToUVRow_SVE_SC(src_yuy2, src_stride_yuy2, dst_u, dst_v, width); } void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, @@ -1497,63 +492,7 @@ void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_u, uint8_t* dst_v, int width) { - const uint8_t* src_uyvy_1 = src_uyvy + src_stride_uyvy; - int vl; - int vl_half; - int width_half; - if (src_stride_uyvy == 0) { - src_uyvy_1 = src_uyvy; - } - asm volatile( - "cntb %x[vl] \n" - "lsr %w[vl_half], %w[vl], #1 \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - "ptrue p0.b \n" - "whilelt p1.b, wzr, %w[vl_half] \n" - "1: \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" - "ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n" - "incb %[src_uyvy], all, mul #2 \n" - "incb %[src_uyvy_1], all, mul #2 \n" - "urhadd z0.b, p0/m, z0.b, z4.b \n" - "uzp1 z1.b, z0.b, z0.b \n" - "uzp2 z2.b, z0.b, z0.b \n" - "subs %w[width], %w[width], %w[vl] \n" - "st1b {z1.b}, p1, [%[dst_u]] \n" - "st1b {z2.b}, p1, [%[dst_v]] \n" - "add %[dst_u], %[dst_u], %x[vl_half] \n" - "add %[dst_v], %[dst_v], %x[vl_half] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - "add %w[width_half], %w[width], #1 \n" - "lsr %w[width_half], %w[width_half], #1 \n" - "whilelt p1.b, wzr, %w[width_half] \n" - "zip1 p0.b, p1.b, p1.b \n" - "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" - "ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n" - "urhadd z0.b, p0/m, z0.b, z4.b \n" - "uzp1 z1.b, z0.b, z0.b \n" - "uzp2 z2.b, z0.b, z0.b \n" - "st1b {z1.b}, p1, [%[dst_u]] \n" - "st1b {z2.b}, p1, [%[dst_v]] \n" - - "99: \n" - : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy] - [src_uyvy_1] "+r"(src_uyvy_1), // %[src_uyvy_1] - [dst_u] "+r"(dst_u), // %[dst_u] - [dst_v] "+r"(dst_v), // %[dst_v] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl), // %[vl] - [vl_half] "=&r"(vl_half), // %[vl_half] - [width_half] "=&r"(width_half) // %[width_half] - : - : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1"); + UYVYToUVRow_SVE_SC(src_uyvy, src_stride_uyvy, dst_u, dst_v, width); } #ifdef HAS_CONVERT16TO8ROW_SVE2 @@ -1776,6 +715,12 @@ void ARGBAlphaBlendRow_SVE2(const uint8_t* src_argb0, } #endif // HAS_ARGBALPHABLENDROW_SVE2 +#if defined(__clang__) +#pragma clang attribute pop +#elif defined(__GNUC__) +#pragma GCC pop_options +#endif + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/row_win.cc b/source/row_win.cc index 928942a1a..18d5007fd 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -6778,6 +6778,35 @@ void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, } #endif // HAS_YUY2TOUVROW_AVX512BW +#ifdef HAS_YUY2TOUV422ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void YUY2ToUV422Row_AVX512BW(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + while (width >= 32) { + __m512i r0 = _mm512_loadu_si512((const __m512i*)src_yuy2); + src_yuy2 += 64; + _mm_storeu_si128((__m128i*)dst_u, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 8))); + _mm_storeu_si128((__m128i*)dst_v, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 24))); + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i r0 = _mm512_maskz_loadu_epi32(k, src_yuy2); + _mm_mask_storeu_epi8(dst_u, k, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 8))); + _mm_mask_storeu_epi8(dst_v, k, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 24))); + } + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOUV422ROW_AVX512BW + #ifdef HAS_UYVYTOYROW_AVX512BW LIBYUV_TARGET_AVX512BW void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { @@ -6832,6 +6861,33 @@ void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, } #endif // HAS_UYVYTOUVROW_AVX512BW +#ifdef HAS_UYVYTOUV422ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void UYVYToUV422Row_AVX512BW(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + while (width >= 32) { + __m512i r0 = _mm512_loadu_si512((const __m512i*)src_uyvy); + src_uyvy += 64; + _mm_storeu_si128((__m128i*)dst_u, _mm512_cvtepi32_epi8(r0)); + _mm_storeu_si128((__m128i*)dst_v, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 16))); + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i r0 = _mm512_maskz_loadu_epi32(k, src_uyvy); + _mm_mask_storeu_epi8(dst_u, k, _mm512_cvtepi32_epi8(r0)); + _mm_mask_storeu_epi8(dst_v, k, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 16))); + } + _mm256_zeroupper(); +} +#endif // HAS_UYVYTOUV422ROW_AVX512BW + #ifdef HAS_ARGBSEPIAROW_AVX512VNNI LIBYUV_TARGET_AVX512VNNI void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) { diff --git a/source/scale_sme.cc b/source/scale_sme.cc index 1dc5c17ba..f39dc85d8 100644 --- a/source/scale_sme.cc +++ b/source/scale_sme.cc @@ -22,8 +22,16 @@ extern "C" { // asm statement, and other OS ABIs (e.g. Windows) differ in which registers // are callee-saved. List "memory" first, then "cc", then registers. -#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ - defined(__aarch64__) +#if !defined(LIBYUV_DISABLE_SME) && \ + (defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__) + +#if defined(__clang__) +#pragma clang attribute push( \ + __attribute__((target("arch=armv8.5-a+i8mm+sme"))), apply_to = function) +#elif defined(__GNUC__) +#pragma GCC push_options +#pragma GCC target("arch=armv8.5-a+i8mm+sve2+sme") +#endif __arm_locally_streaming void ScaleRowDown2_SME(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -58,7 +66,7 @@ __arm_locally_streaming void ScaleRowDown2_SME(const uint8_t* src_ptr, : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst), // %[dst_ptr] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "p0"); } @@ -96,7 +104,7 @@ __arm_locally_streaming void ScaleRowDown2_16_SME(const uint16_t* src_ptr, : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst), // %[dst_ptr] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "p0"); } @@ -136,7 +144,7 @@ __arm_locally_streaming void ScaleRowDown2Linear_SME(const uint8_t* src_ptr, : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst), // %[dst_ptr] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "p0"); } @@ -176,7 +184,7 @@ __arm_locally_streaming void ScaleRowDown2Linear_16_SME(const uint16_t* src_ptr, : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst), // %[dst_ptr] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "p0"); } @@ -227,7 +235,7 @@ __arm_locally_streaming void ScaleRowDown2Box_SME(const uint8_t* src_ptr, [src2_ptr] "+r"(src2_ptr), // %[src2_ptr] [dst_ptr] "+r"(dst), // %[dst_ptr] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0"); } @@ -280,7 +288,7 @@ __arm_locally_streaming void ScaleRowDown2Box_16_SME(const uint16_t* src_ptr, [src2_ptr] "+r"(src2_ptr), // %[src2_ptr] [dst_ptr] "+r"(dst), // %[dst_ptr] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0"); } @@ -320,7 +328,7 @@ __arm_locally_streaming void ScaleUVRowDown2_SME(const uint8_t* src_uv, : [src_uv] "+r"(src_uv), // %[src_uv] [dst_uv] "+r"(dst_uv), // %[dst_uv] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "p0"); } @@ -361,7 +369,7 @@ __arm_locally_streaming void ScaleUVRowDown2Linear_SME(const uint8_t* src_uv, : [src_uv] "+r"(src_uv), // %[src_uv] [dst_uv] "+r"(dst_uv), // %[dst_uv] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "p0", "p1"); } @@ -413,7 +421,7 @@ __arm_locally_streaming void ScaleUVRowDown2Box_SME(const uint8_t* src_uv, [src2_uv] "+r"(src2_uv), // %[src2_uv] [dst_uv] "+r"(dst_uv), // %[dst_uv] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0", "p1"); @@ -454,7 +462,7 @@ __arm_locally_streaming void ScaleARGBRowDown2_SME(const uint8_t* src_argb, : [src_argb] "+r"(src_argb), // %[src_argb] [dst_argb] "+r"(dst_argb), // %[dst_argb] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "p0"); } @@ -496,7 +504,7 @@ __arm_locally_streaming void ScaleARGBRowDown2Linear_SME( : [src_argb] "+r"(src_argb), // %[src_argb] [dst_argb] "+r"(dst_argb), // %[dst_argb] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "p0", "p1"); } @@ -548,7 +556,7 @@ __arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb, [src2_argb] "+r"(src2_argb), // %[src2_argb] [dst_argb] "+r"(dst_argb), // %[dst_argb] [dst_width] "+r"(dst_width), // %[dst_width] - [vl] "=r"(vl) // %[vl] + [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0", "p1"); @@ -619,8 +627,14 @@ __arm_locally_streaming void ScaleUVAddCols3_SME(int dst_width, "p2", "p3"); } -#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && - // defined(__aarch64__) +#if defined(__clang__) +#pragma clang attribute pop +#elif defined(__GNUC__) +#pragma GCC pop_options +#endif + +#endif // !defined(LIBYUV_DISABLE_SME) && (defined(CLANG_HAS_SME) || + // defined(GCC_HAS_SME)) && defined(__aarch64__) #ifdef __cplusplus } // extern "C" diff --git a/unit_test/convert_argb_test.cc b/unit_test/convert_argb_test.cc index a2e1ad547..a12fd5df4 100644 --- a/unit_test/convert_argb_test.cc +++ b/unit_test/convert_argb_test.cc @@ -247,21 +247,15 @@ TESTBPTOP(P410, uint16_t, 2, 1, 1, I420, uint8_t, 1, 2, 2, 10, 1, 1) memset(dst_argb_c + OFF, 1, kStrideB * kHeight); \ memset(dst_argb_opt + OFF, 101, kStrideB * kHeight); \ MaskCpuFlags(disable_cpu_flags_); \ - double time0 = get_time(); \ FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \ src_v + OFF, kStrideUV, dst_argb_c + OFF, kStrideB, \ kWidth, NEG kHeight); \ - double time1 = get_time(); \ MaskCpuFlags(benchmark_cpu_info_); \ for (int i = 0; i < benchmark_iterations_; ++i) { \ FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \ src_v + OFF, kStrideUV, dst_argb_opt + OFF, \ kStrideB, kWidth, NEG kHeight); \ } \ - double time2 = get_time(); \ - printf(" %8d us C - %8d us OPT\n", \ - static_cast((time1 - time0) * 1e6), \ - static_cast((time2 - time1) * 1e6 / benchmark_iterations_)); \ for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \ ASSERT_EQ(dst_argb_c[i + OFF], dst_argb_opt[i + OFF]); \ } \ @@ -2982,6 +2976,7 @@ TEST_F(LibYUVConvertTest, P210ToAR30MatrixFilterOverflow) { #ifdef ENABLE_ROW_TESTS TEST_F(LibYUVConvertTest, TestARGBToUVMatrixRow_Opt) { +#if defined(HAS_ARGBTOUVMATRIXROW_NEON) const int kMaxWidth = 128; SIMD_ALIGNED(uint8_t orig_argb_pixels[kMaxWidth * 4 * 2]); SIMD_ALIGNED(uint8_t dest_u_c[kMaxWidth]); @@ -2993,7 +2988,6 @@ TEST_F(LibYUVConvertTest, TestARGBToUVMatrixRow_Opt) { orig_argb_pixels[i] = i * 43; } -#if defined(HAS_ARGBTOUVMATRIXROW_NEON) int has_neon = TestCpuFlag(kCpuHasNEON); if (has_neon) { for (int width = 1; width <= kMaxWidth; ++width) { @@ -3090,7 +3084,7 @@ TEST_F(LibYUVConvertTest, TestI400LargeSize) { free_aligned_buffer_page_end(orig_i400); } #endif // DISABLE_SLOW_TESTS -#endif // !defined(DISABLE_SLOW_TESTS) && \ +#endif // !defined(DISABLE_SLOW_TESTS) && // (defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__)) #endif // !defined(LEAN_TESTS) diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 6f0e42f2c..6439a9fce 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -4849,10 +4849,16 @@ float TestScaleMaxSamples(int benchmark_width, for (j = 0; j < benchmark_iterations; j++) { if (opt) { -#ifdef HAS_SCALESUMSAMPLES_NEON - max_opt = ScaleMaxSamples_NEON(reinterpret_cast(orig_y), - reinterpret_cast(dst_opt), scale, - kPixels); +#ifdef HAS_SCALEMAXSAMPLES_NEON + if (TestCpuFlag(kCpuHasNEON)) { + max_opt = ScaleMaxSamples_NEON(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else { + max_opt = ScaleMaxSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #elif defined(HAS_SCALEMAXSAMPLES_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { max_opt = ScaleMaxSamples_AVX2(reinterpret_cast(orig_y), @@ -4909,6 +4915,9 @@ float TestScaleSumSamples(int benchmark_width, float sum_c, sum_opt = 0.f; // NEON does multiple of 8, so round count up const int kPixels = (benchmark_width * benchmark_height + 7) & ~7; + if (kPixels <= 0) { + return 0.f; + } align_buffer_page_end(orig_y, kPixels * 4 * 3); uint8_t* dst_c = orig_y + kPixels * 4; uint8_t* dst_opt = orig_y + kPixels * 4 * 2; @@ -4928,9 +4937,15 @@ float TestScaleSumSamples(int benchmark_width, for (j = 0; j < benchmark_iterations; j++) { if (opt) { #ifdef HAS_SCALESUMSAMPLES_NEON - sum_opt = ScaleSumSamples_NEON(reinterpret_cast(orig_y), - reinterpret_cast(dst_opt), scale, - kPixels); + if (TestCpuFlag(kCpuHasNEON)) { + sum_opt = ScaleSumSamples_NEON(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else { + sum_opt = ScaleSumSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #elif defined(HAS_SCALESUMSAMPLES_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { sum_opt = ScaleSumSamples_AVX2(reinterpret_cast(orig_y), @@ -4997,6 +5012,9 @@ float TestScaleSamples(int benchmark_width, int i, j; // NEON does multiple of 8, so round count up const int kPixels = (benchmark_width * benchmark_height + 7) & ~7; + if (kPixels <= 0) { + return 0.f; + } align_buffer_page_end(orig_y, kPixels * 4 * 3); uint8_t* dst_c = orig_y + kPixels * 4; uint8_t* dst_opt = orig_y + kPixels * 4 * 2; @@ -5007,17 +5025,22 @@ float TestScaleSamples(int benchmark_width, for (i = 0; i < kPixels; ++i) { (reinterpret_cast(orig_y))[i] = sinf(static_cast(i) * 0.1f); } - memset(dst_c, 0, kPixels * 4); - memset(dst_opt, 1, kPixels * 4); + memset(dst_c, 0, static_cast(kPixels) * 4); + memset(dst_opt, 1, static_cast(kPixels) * 4); ScaleSamples_C(reinterpret_cast(orig_y), reinterpret_cast(dst_c), scale, kPixels); for (j = 0; j < benchmark_iterations; j++) { if (opt) { -#ifdef HAS_SCALESUMSAMPLES_NEON - ScaleSamples_NEON(reinterpret_cast(orig_y), - reinterpret_cast(dst_opt), scale, kPixels); +#ifdef HAS_SCALESAMPLES_NEON + if (TestCpuFlag(kCpuHasNEON)) { + ScaleSamples_NEON(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } else { + ScaleSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } #elif defined(HAS_SCALESAMPLES_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ScaleSamples_AVX2(reinterpret_cast(orig_y), @@ -5068,6 +5091,9 @@ float TestCopySamples(int benchmark_width, int i, j; // NEON does multiple of 16 floats, so round count up const int kPixels = (benchmark_width * benchmark_height + 15) & ~15; + if (kPixels <= 0) { + return 0.f; + } align_buffer_page_end(orig_y, kPixels * 4 * 3); uint8_t* dst_c = orig_y + kPixels * 4; uint8_t* dst_opt = orig_y + kPixels * 4 * 2; @@ -5078,16 +5104,20 @@ float TestCopySamples(int benchmark_width, for (i = 0; i < kPixels; ++i) { (reinterpret_cast(orig_y))[i] = sinf(static_cast(i) * 0.1f); } - memset(dst_c, 0, kPixels * 4); - memset(dst_opt, 1, kPixels * 4); + memset(dst_c, 0, static_cast(kPixels) * 4); + memset(dst_opt, 1, static_cast(kPixels) * 4); memcpy(reinterpret_cast(dst_c), reinterpret_cast(orig_y), - kPixels * 4); + static_cast(kPixels) * 4); for (j = 0; j < benchmark_iterations; j++) { if (opt) { #ifdef HAS_COPYROW_NEON - CopyRow_NEON(orig_y, dst_opt, kPixels * 4); + if (TestCpuFlag(kCpuHasNEON)) { + CopyRow_NEON(orig_y, dst_opt, kPixels * 4); + } else { + CopyRow_C(orig_y, dst_opt, kPixels * 4); + } #else CopyRow_C(orig_y, dst_opt, kPixels * 4); #endif @@ -5401,6 +5431,25 @@ TEST_F(LibYUVPlanarTest, HalfRow_Opt) { HalfRow_Any_AVX2(src_pixels, stride, dst_pixels_opt, width); } } else +#endif +#if defined(HAS_HALFROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + HalfRow_SME(src_pixels, stride, dst_pixels_opt, width); + } else +#endif +#if defined(HAS_HALFROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + HalfRow_SVE2(src_pixels, stride, dst_pixels_opt, width); + } else +#endif +#if defined(HAS_HALFROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + if (IS_ALIGNED(width, 16)) { + HalfRow_NEON(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_Any_NEON(src_pixels, stride, dst_pixels_opt, width); + } + } else #endif { HalfRow_C(src_pixels, stride, dst_pixels_opt, width); @@ -5437,6 +5486,25 @@ TEST_F(LibYUVPlanarTest, HalfRow_16_Opt) { HalfRow_16_Any_AVX2(src_pixels, stride, dst_pixels_opt, width); } } else +#endif +#if defined(HAS_HALFROW_16_SME) + if (TestCpuFlag(kCpuHasSME)) { + HalfRow_16_SME(src_pixels, stride, dst_pixels_opt, width); + } else +#endif +#if defined(HAS_HALFROW_16_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + HalfRow_16_SVE2(src_pixels, stride, dst_pixels_opt, width); + } else +#endif +#if defined(HAS_HALFROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + if (IS_ALIGNED(width, 8)) { + HalfRow_16_NEON(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_16_Any_NEON(src_pixels, stride, dst_pixels_opt, width); + } + } else #endif { HalfRow_16_C(src_pixels, stride, dst_pixels_opt, width); @@ -5470,6 +5538,12 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Odd_Opt) { HalfWidthRow_16To8_Odd_AVX2(src_pixels, stride, dst_pixels_opt, 16384, width); } else +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + HalfWidthRow_16To8_Odd_NEON(src_pixels, stride, dst_pixels_opt, 16384, + width); + } else #endif { HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_opt, 16384, @@ -5665,6 +5739,7 @@ TEST_F(LibYUVPlanarTest, NV21Copy) { TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32) { int i, j; const int y_plane_size = benchmark_width_ * benchmark_height_; + ASSERT_GT(y_plane_size, 0); align_buffer_page_end(orig_f, y_plane_size * 4); align_buffer_page_end(orig_y, y_plane_size * 2); @@ -5674,9 +5749,9 @@ TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32) { for (i = 0; i < y_plane_size; ++i) { ((float*)orig_f)[i] = (float)(i % 10000) * 3.14f; } - memset(orig_y, 1, y_plane_size * 2); - memset(dst_opt, 2, y_plane_size * 4); - memset(rec_opt, 3, y_plane_size * 2); + memset(orig_y, 1, static_cast(y_plane_size) * 2); + memset(dst_opt, 2, static_cast(y_plane_size) * 4); + memset(rec_opt, 3, static_cast(y_plane_size) * 2); ConvertFP32ToFP16Row_NEON((const float*)orig_f, (uint16_t*)orig_y, y_plane_size); @@ -5702,6 +5777,7 @@ TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32) { TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32Column) { int i, j; const int y_plane_size = benchmark_width_ * benchmark_height_; + ASSERT_GT(y_plane_size, 0); align_buffer_page_end(orig_f, y_plane_size * 4); align_buffer_page_end(orig_y, y_plane_size * 2); @@ -5711,9 +5787,9 @@ TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32Column) { for (i = 0; i < y_plane_size; ++i) { ((float*)orig_f)[i] = (float)(i % 10000) * 3.14f; } - memset(orig_y, 1, y_plane_size * 2); - memset(dst_opt, 2, y_plane_size * 4); - memset(rec_opt, 3, y_plane_size * 2); + memset(orig_y, 1, static_cast(y_plane_size) * 2); + memset(dst_opt, 2, static_cast(y_plane_size) * 4); + memset(rec_opt, 3, static_cast(y_plane_size) * 2); ConvertFP32ToFP16Row_NEON((const float*)orig_f, (uint16_t*)orig_y, y_plane_size); diff --git a/unit_test/rotate_argb_test.cc b/unit_test/rotate_argb_test.cc index 902aa8129..67ddc02db 100644 --- a/unit_test/rotate_argb_test.cc +++ b/unit_test/rotate_argb_test.cc @@ -170,28 +170,28 @@ TEST_F(LibYUVRotateTest, RotatePlane270_Opt) { disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_RotatePlane0_Odd) { +TEST_F(LibYUVRotateTest, RotatePlane0_Odd) { TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, benchmark_height_ + 1, kRotate0, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_RotatePlane90_Odd) { +TEST_F(LibYUVRotateTest, RotatePlane90_Odd) { TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, kRotate90, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_RotatePlane180_Odd) { +TEST_F(LibYUVRotateTest, RotatePlane180_Odd) { TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, benchmark_height_ + 1, kRotate180, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_RotatePlane270_Odd) { +TEST_F(LibYUVRotateTest, RotatePlane270_Odd) { TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, kRotate270, benchmark_iterations_, disable_cpu_flags_, diff --git a/unit_test/rotate_test.cc b/unit_test/rotate_test.cc index 42e71fa71..bc5369588 100644 --- a/unit_test/rotate_test.cc +++ b/unit_test/rotate_test.cc @@ -118,28 +118,28 @@ TEST_F(LibYUVRotateTest, I420Rotate270_Opt) { // TODO(fbarchard): Remove odd width tests. // Odd width tests work but disabled because they use C code and can be // tested by passing an odd width command line or environment variable. -TEST_F(LibYUVRotateTest, DISABLED_I420Rotate0_Odd) { +TEST_F(LibYUVRotateTest, I420Rotate0_Odd) { I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, benchmark_height_ + 1, kRotate0, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_I420Rotate90_Odd) { +TEST_F(LibYUVRotateTest, I420Rotate90_Odd) { I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, kRotate90, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_I420Rotate180_Odd) { +TEST_F(LibYUVRotateTest, I420Rotate180_Odd) { I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, benchmark_height_ + 1, kRotate180, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_I420Rotate270_Odd) { +TEST_F(LibYUVRotateTest, I420Rotate270_Odd) { I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, kRotate270, benchmark_iterations_, disable_cpu_flags_, @@ -323,28 +323,28 @@ TEST_F(LibYUVRotateTest, I444Rotate270_Opt) { // TODO(fbarchard): Remove odd width tests. // Odd width tests work but disabled because they use C code and can be // tested by passing an odd width command line or environment variable. -TEST_F(LibYUVRotateTest, DISABLED_I444Rotate0_Odd) { +TEST_F(LibYUVRotateTest, I444Rotate0_Odd) { I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, benchmark_height_ + 1, kRotate0, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_I444Rotate90_Odd) { +TEST_F(LibYUVRotateTest, I444Rotate90_Odd) { I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, kRotate90, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_I444Rotate180_Odd) { +TEST_F(LibYUVRotateTest, I444Rotate180_Odd) { I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, benchmark_height_ + 1, kRotate180, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_I444Rotate270_Odd) { +TEST_F(LibYUVRotateTest, I444Rotate270_Odd) { I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, kRotate270, benchmark_iterations_, disable_cpu_flags_, @@ -438,28 +438,28 @@ TEST_F(LibYUVRotateTest, NV12Rotate270_Opt) { disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate0_Odd) { +TEST_F(LibYUVRotateTest, NV12Rotate0_Odd) { NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, benchmark_height_ + 1, kRotate0, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate90_Odd) { +TEST_F(LibYUVRotateTest, NV12Rotate90_Odd) { NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, kRotate90, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate180_Odd) { +TEST_F(LibYUVRotateTest, NV12Rotate180_Odd) { NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, benchmark_height_ + 1, kRotate180, benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_); } -TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate270_Odd) { +TEST_F(LibYUVRotateTest, NV12Rotate270_Odd) { NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1, benchmark_height_ + 1, benchmark_width_ + 1, kRotate270, benchmark_iterations_, disable_cpu_flags_, diff --git a/unit_test/scale_plane_test.cc b/unit_test/scale_plane_test.cc index 012377d5c..cb43c0739 100644 --- a/unit_test/scale_plane_test.cc +++ b/unit_test/scale_plane_test.cc @@ -357,14 +357,14 @@ static int TestPlaneFilter_16(int src_width, #define DX(x, nom, denom) static_cast(((Abs(x) / nom + 1) / 2) * nom * 2) #define SX(x, nom, denom) static_cast(((x / nom + 1) / 2) * denom * 2) -#define TEST_FACTOR1(name, filter, nom, denom, max_diff) \ - TEST_F(LibYUVScaleTest, DISABLED_##ScalePlaneDownBy##name##_##filter##_16) { \ - int diff = TestPlaneFilter_16( \ - SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \ - DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \ - kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \ - benchmark_cpu_info_); \ - ASSERT_LE(diff, max_diff); \ +#define TEST_FACTOR1(name, filter, nom, denom, max_diff) \ + TEST_F(LibYUVScaleTest, ScalePlaneDownBy##name##_##filter##_16) { \ + int diff = TestPlaneFilter_16( \ + SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \ + DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \ + kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \ + benchmark_cpu_info_); \ + ASSERT_LE(diff, max_diff); \ } // Test a scale factor with all 4 filters. Expect unfiltered to be exact, but