From 149568401fc794586da2303b6fa711ee02c49978 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Wed, 5 Aug 2026 20:57:15 -0700 Subject: [PATCH] SVE2 row functions needed for multipass image functions - MergeUVRow_SVE2: Interleaves U and V - SplitUVRow_SVE2: De-interleaves UV - InterpolateRow_SVE2: Bilinear row interpolation - YUY2ToYRow_SVE2, YUY2ToUVRow_SVE2, UYVYToYRow_SVE2, UYVYToUVRow_SVE2 - HalfMergeUVRow_SVE2: 2:1 vertical averaging of U and V OnePlus 15 with Qualcomm Snapdragon 8 Elite Gen 5 Oryon V3 Benchmark (1280x720 1000x) | SVE2 | NEON | Result -------------------------------+--------+--------+------------------ TestInterpolatePlane | 119 ms | 164 ms | SVE2 is 1.38x win TestInterpolatePlane_16 | 138 ms | 172 ms | SVE2 is 1.25x win SplitUVPlane_Opt | 68 ms | 69 ms | SVE2 is 1.01x win MergeUVPlane_Opt | 40 ms | 40 ms | Tied SwapUVPlane_Opt | 72 ms | 72 ms | Tied HalfMergeUVPlane_Opt | 50 ms | 50 ms | Tied I420ToNV12_Opt | 25 ms | 26 ms | SVE2 is faster I420ToNV21_Opt | 26 ms | 26 ms | Tied YUY2ToNV12_Opt | 79 ms | 81 ms | SVE2 is faster NV21ToNV12_Opt | 36 ms | 35 ms | Tied Android420ToI420ToNV12_Opt | 53 ms | 53 ms | Tied Android420ToI420ToNV21_Opt | 35 ms | 33 ms | NEON is faster YUY2ToI420_Opt | 87 ms | 84 ms | NEON is faster UYVYToI420_Opt | 87 ms | 83 ms | NEON is faster YUY2ToI422_Opt | 101 ms | 97 ms | NEON is faster UYVYToI422_Opt | 101 ms | 97 ms | NEON is faster UYVYToNV12_Opt | 111 ms | 108 ms | NEON is faster ARGBToNV21_Opt | 279 ms | 279 ms | Tied ABGRToNV21_Opt | 279 ms | 280 ms | SVE2 is faster ARGBToNV12_Opt | 285 ms | 280 ms | NEON is faster ABGRToNV12_Opt | 284 ms | 281 ms | NEON is faster Bug: 42280902 Change-Id: I53f633fe453f2853f7bc9fe17280e938912c4e8f Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8196068 Reviewed-by: Justin Green Commit-Queue: Frank Barchard --- GEMINI.md | 1 + include/libyuv/row.h | 42 ++++ include/libyuv/row_sve.h | 51 ++--- source/convert.cc | 27 +++ source/convert_from_argb.cc | 95 ++++++++ source/planar_functions.cc | 65 ++++++ source/rotate.cc | 5 + source/rotate_argb.cc | 5 + source/row_sve.cc | 427 ++++++++++++++++++++++++++++++++++++ source/scale.cc | 10 + source/scale_argb.cc | 10 + source/scale_common.cc | 5 + source/scale_uv.cc | 10 + 13 files changed, 725 insertions(+), 28 deletions(-) diff --git a/GEMINI.md b/GEMINI.md index 3d071fe08..0a664f51a 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -52,6 +52,7 @@ and compiler compatibility. only**. 2. **Feature Macros**: Use the `HAS_` macros in `include/libyuv/row.h` to enable or disable specific AVX512 versions. +3. **MSVC Intrinsics (`row_win.cc`)**: When converting `row_gcc.cc` to `row_win.cc`, use `_mm512_permutex2var_epi8`, not `_mm512_permi2var_epi8` intrinsic. ## Changelist (CL) Format Guidelines diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 569bd6bad..d1cc9d257 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -579,8 +579,10 @@ extern "C" { #define HAS_AYUVTOUVROW_SVE2 #define HAS_AYUVTOVUROW_SVE2 #define HAS_CONVERT8TO8ROW_SVE2 +#define HAS_COPYROW_SVE2 #define HAS_DIVIDEROW_16_SVE2 #define HAS_HALFFLOATROW_SVE2 +#define HAS_HALFMERGEUVROW_SVE2 #define HAS_I210ALPHATOARGBROW_SVE2 #define HAS_I210TOAR30ROW_SVE2 #define HAS_I210TOARGBROW_SVE2 @@ -601,6 +603,8 @@ extern "C" { #define HAS_I444ALPHATOARGBROW_SVE2 #define HAS_I444TOARGBROW_SVE2 #define HAS_I444TORGB24ROW_SVE2 +#define HAS_INTERPOLATEROW_SVE2 +#define HAS_MERGEUVROW_SVE2 #define HAS_NV12TOARGBROW_SVE2 #define HAS_NV12TORGB24ROW_SVE2 #define HAS_NV21TOARGBROW_SVE2 @@ -613,8 +617,13 @@ extern "C" { #define HAS_RAWTORGB24ROW_SVE2 #define HAS_RAWTORGBAROW_SVE2 #define HAS_RGB24TOARGBROW_SVE2 +#define HAS_SPLITUVROW_SVE2 #define HAS_UYVYTOARGBROW_SVE2 +#define HAS_UYVYTOUVROW_SVE2 +#define HAS_UYVYTOYROW_SVE2 #define HAS_YUY2TOARGBROW_SVE2 +#define HAS_YUY2TOUVROW_SVE2 +#define HAS_YUY2TOYROW_SVE2 #endif // The following are available on AArch64 SME platforms: @@ -2708,6 +2717,10 @@ void SplitUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width); +void SplitUVRow_SVE2(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void SplitUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, @@ -2869,6 +2882,10 @@ void MergeUVRow_NEON(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width); +void MergeUVRow_SVE2(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width); void MergeUVRow_SME(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, @@ -2916,6 +2933,13 @@ void HalfMergeUVRow_NEON(const uint8_t* src_u, uint8_t* dst_uv, int width); +void HalfMergeUVRow_SVE2(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); + void HalfMergeUVRow_SSSE3(const uint8_t* src_u, int src_stride_u, const uint8_t* src_v, @@ -3605,6 +3629,7 @@ void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width); void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width); void CopyRow_ERMS(const uint8_t* src, uint8_t* dst, int width); void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width); +void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width); void CopyRow_SME(const uint8_t* src, uint8_t* dst, int width); void CopyRow_RVV(const uint8_t* src, uint8_t* dst, int count); void CopyRow_C(const uint8_t* src, uint8_t* dst, int count); @@ -5684,11 +5709,23 @@ void YUY2ToUV422Row_SSE2(const uint8_t* src_yuy2, uint8_t* dst_v, int width); void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width); +void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width); void YUY2ToUVRow_NEON(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_u, uint8_t* dst_v, int width); +void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width); +void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_uv, @@ -6275,6 +6312,11 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, ptrdiff_t src_stride, int dst_width, int source_y_fraction); +void InterpolateRow_SVE2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int dst_width, + int source_y_fraction); void InterpolateRow_SME(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index b39f59a79..7bfa5f4b0 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -2144,28 +2144,27 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb, "z27", "p0", "p1", "p2", "p3", "p4", "p5"); } -#define ARGBTOYMATRIX_SVE \ - "ld1w {z0.s}, p1/z, [%[src]] \n" /* load 4*vl pixels */\ - "ld1w {z1.s}, p2/z, [%[src], #1, mul vl] \n" \ - "ld1w {z2.s}, p3/z, [%[src], #2, mul vl] \n" \ - "ld1w {z3.s}, p4/z, [%[src], #3, mul vl] \n" \ - "incb %[src], all, mul #4 \n" \ - "fmov s16, wzr \n" \ - "fmov s17, wzr \n" \ - "fmov s18, wzr \n" \ - "fmov s19, wzr \n" \ - "udot z16.s, z0.b, z24.b \n" \ - "udot z17.s, z1.b, z24.b \n" \ - "udot z18.s, z2.b, z24.b \n" \ - "udot z19.s, z3.b, z24.b \n" \ - "uzp1 z16.h, z16.h, z17.h \n" \ - "uzp1 z18.h, z18.h, z19.h \n" \ - "add z16.h, z16.h, z25.h \n" \ - "add z18.h, z18.h, z25.h \n" \ - "uzp2 z16.b, z16.b, z18.b \n" \ - "st1b {z16.b}, p5, [%[dst_y]] \n" \ - "incb %[dst_y] \n" \ - "subs %w[width], %w[width], %w[vl], lsl #2 \n" +#define ARGBTOYMATRIX_SVE(p1, p2, p3, p4, p5) \ + "ld1w {z0.s}, " p1 "/z, [%[src]] \n" /* load 4*vl pixels */\ + "ld1w {z1.s}, " p2 "/z, [%[src], #1, mul vl] \n" \ + "ld1w {z2.s}, " p3 "/z, [%[src], #2, mul vl] \n" \ + "ld1w {z3.s}, " p4 "/z, [%[src], #3, mul vl] \n" \ + "incb %[src], all, mul #4 \n" \ + "fmov s16, wzr \n" \ + "fmov s17, wzr \n" \ + "fmov s18, wzr \n" \ + "fmov s19, wzr \n" \ + "udot z16.s, z0.b, z24.b \n" \ + "udot z17.s, z1.b, z24.b \n" \ + "udot z18.s, z2.b, z24.b \n" \ + "udot z19.s, z3.b, z24.b \n" \ + "uzp1 z16.h, z16.h, z17.h \n" \ + "uzp1 z18.h, z18.h, z19.h \n" \ + "addhnb z20.b, z16.h, z25.h \n" \ + "addhnt z20.b, z18.h, z25.h \n" \ + "st1b {z20.b}, " p5 ", [%[dst_y]] \n" \ + "incb %[dst_y] \n" \ + "subs %w[width], %w[width], %w[vl], lsl #2 \n" static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, uint8_t* dst_y, @@ -2184,12 +2183,8 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, "b.lt 2f \n" "ptrue p1.s \n" - "ptrue p2.s \n" - "ptrue p3.s \n" - "ptrue p4.s \n" - "ptrue p5.b \n" "1: \n" // - ARGBTOYMATRIX_SVE + ARGBTOYMATRIX_SVE("p1", "p1", "p1", "p1", "p0") "b.gt 1b \n" "2: \n" @@ -2202,7 +2197,7 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, "whilelt p3.s, %w[vl2], %w[width] \n" "whilelt p4.s, %w[vl3], %w[width] \n" "whilelt p5.b, wzr, %w[width] \n" // - ARGBTOYMATRIX_SVE + ARGBTOYMATRIX_SVE("p1", "p2", "p3", "p4", "p5") "b.gt 3b \n" "99: \n" diff --git a/source/convert.cc b/source/convert.cc index 0e023fcf1..36657c633 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -749,6 +749,11 @@ int I010ToNV12(const uint16_t* src_y, } } #endif +#if defined(HAS_MERGEUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + MergeUVRow = MergeUVRow_SVE2; + } +#endif #if defined(HAS_MERGEUVROW_SME) if (TestCpuFlag(kCpuHasSME)) { MergeUVRow = MergeUVRow_SME; @@ -1199,6 +1204,11 @@ int I422ToNV21(const uint8_t* src_y, } } #endif +#if defined(HAS_MERGEUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + MergeUVRow = MergeUVRow_SVE2; + } +#endif #if defined(HAS_MERGEUVROW_SME) if (TestCpuFlag(kCpuHasSME)) { MergeUVRow = MergeUVRow_SME; @@ -1233,6 +1243,11 @@ int I422ToNV21(const uint8_t* src_y, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME; @@ -1719,6 +1734,12 @@ int YUY2ToI420(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_SVE2) && defined(HAS_YUY2TOUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + YUY2ToYRow = YUY2ToYRow_SVE2; + YUY2ToUVRow = YUY2ToUVRow_SVE2; + } +#endif #if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { YUY2ToYRow = YUY2ToYRow_Any_LSX; @@ -1813,6 +1834,12 @@ int UYVYToI420(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_SVE2) && defined(HAS_UYVYTOUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + UYVYToYRow = UYVYToYRow_SVE2; + UYVYToUVRow = UYVYToUVRow_SVE2; + } +#endif #if defined(HAS_UYVYTOYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { UYVYToYRow = UYVYToYRow_Any_LSX; diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 96ab1bc9c..a633085a7 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -587,6 +587,11 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MERGEUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + MergeUVRow = MergeUVRow_SVE2; + } +#endif #if defined(HAS_MERGEUVROW_SME) if (TestCpuFlag(kCpuHasSME)) { MergeUVRow = MergeUVRow_SME; @@ -818,6 +823,11 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MERGEUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + MergeUVRow = MergeUVRow_SVE2; + } +#endif #if defined(HAS_MERGEUVROW_SME) if (TestCpuFlag(kCpuHasSME)) { MergeUVRow = MergeUVRow_SME; @@ -984,6 +994,24 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; @@ -1008,6 +1036,28 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) + if (TestCpuFlag(kCpuHasNeonI8MM)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + if (IS_ALIGNED(width, 2)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + if (IS_ALIGNED(width, 2)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME; + } + } +#endif #if defined(HAS_I422TOYUY2ROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { I422ToYUY2Row = I422ToYUY2Row_Any_AVX2; @@ -1097,6 +1147,24 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd; + } + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; @@ -1121,6 +1189,28 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) + if (TestCpuFlag(kCpuHasNeonI8MM)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + if (IS_ALIGNED(width, 2)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2; + } + } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + if (IS_ALIGNED(width, 2)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME; + } + } +#endif #if defined(HAS_I422TOUYVYROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { I422ToUYVYRow = I422ToUYVYRow_Any_AVX2; @@ -2294,6 +2384,11 @@ static int RAWToNVMatrix(const uint8_t* src_raw, } } #endif +#if defined(HAS_MERGEUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + MergeUVRow = MergeUVRow_SVE2; + } +#endif #if defined(HAS_MERGEUVROW_SME) if (TestCpuFlag(kCpuHasSME)) { MergeUVRow = MergeUVRow_SME; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index eeba46b12..0c8bc0665 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -80,6 +80,11 @@ void CopyPlane(const uint8_t* src_y, CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON; } #endif +#if defined(HAS_COPYROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + CopyRow = CopyRow_SVE2; + } +#endif #if defined(HAS_COPYROW_SME) if (TestCpuFlag(kCpuHasSME)) { CopyRow = CopyRow_SME; @@ -646,6 +651,11 @@ void SplitUVPlane(const uint8_t* src_uv, } } #endif +#if defined(HAS_SPLITUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + SplitUVRow = SplitUVRow_SVE2; + } +#endif #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; @@ -729,6 +739,11 @@ void MergeUVPlane(const uint8_t* src_u, } } #endif +#if defined(HAS_MERGEUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + MergeUVRow = MergeUVRow_SVE2; + } +#endif #if defined(HAS_MERGEUVROW_SME) if (TestCpuFlag(kCpuHasSME)) { MergeUVRow = MergeUVRow_SME; @@ -2295,6 +2310,11 @@ int YUY2ToI422(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + YUY2ToYRow = YUY2ToYRow_SVE2; + } +#endif #if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUV422ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { YUY2ToYRow = YUY2ToYRow_Any_LSX; @@ -2392,6 +2412,11 @@ int UYVYToI422(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + UYVYToYRow = UYVYToYRow_SVE2; + } +#endif #if defined(HAS_UYVYTOYROW_LSX) && defined(HAS_UYVYTOUV422ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { UYVYToYRow = UYVYToYRow_Any_LSX; @@ -2475,6 +2500,11 @@ int YUY2ToY(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + YUY2ToYRow = YUY2ToYRow_SVE2; + } +#endif for (y = 0; y < height; ++y) { YUY2ToYRow(src_yuy2, dst_y, width); @@ -2535,6 +2565,11 @@ int UYVYToY(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + UYVYToYRow = UYVYToYRow_SVE2; + } +#endif #if defined(HAS_UYVYTOYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { UYVYToYRow = UYVYToYRow_Any_LSX; @@ -4450,6 +4485,11 @@ int InterpolatePlane(const uint8_t* src0, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME; @@ -4919,6 +4959,11 @@ static int ARGBSobelize(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; @@ -5549,6 +5594,11 @@ int YUY2ToNV12(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + YUY2ToYRow = YUY2ToYRow_SVE2; + } +#endif #if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUV422ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { YUY2ToYRow = YUY2ToYRow_Any_LSX; @@ -5658,6 +5708,11 @@ int UYVYToNV12(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_SPLITUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + SplitUVRow = SplitUVRow_SVE2; + } +#endif #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; @@ -5688,6 +5743,11 @@ int UYVYToNV12(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME; @@ -5766,6 +5826,11 @@ void HalfMergeUVPlane(const uint8_t* src_u, HalfMergeUVRow = HalfMergeUVRow_NEON; } #endif +#if defined(HAS_HALFMERGEUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + HalfMergeUVRow = HalfMergeUVRow_SVE2; + } +#endif #if defined(HAS_HALFMERGEUVROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 16)) { HalfMergeUVRow = HalfMergeUVRow_SSSE3; diff --git a/source/rotate.cc b/source/rotate.cc index 60940f51f..93e92145f 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -232,6 +232,11 @@ void RotatePlane180(const uint8_t* src, CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON; } #endif +#if defined(HAS_COPYROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + CopyRow = CopyRow_SVE2; + } +#endif #if defined(HAS_COPYROW_SME) if (TestCpuFlag(kCpuHasSME)) { CopyRow = CopyRow_SME; diff --git a/source/rotate_argb.cc b/source/rotate_argb.cc index c2e2637a1..f6603cfd0 100644 --- a/source/rotate_argb.cc +++ b/source/rotate_argb.cc @@ -194,6 +194,11 @@ static int ARGBRotate180(const uint8_t* src_argb, CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_NEON : CopyRow_Any_NEON; } #endif +#if defined(HAS_COPYROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + CopyRow = CopyRow_SVE2; + } +#endif #if defined(HAS_COPYROW_SME) if (TestCpuFlag(kCpuHasSME)) { CopyRow = CopyRow_SME; diff --git a/source/row_sve.cc b/source/row_sve.cc index 5c7ddc425..749c0b7d5 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1047,6 +1047,433 @@ void Convert8To8Row_SVE2(const uint8_t* src_y, Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width); } +void MergeUVRow_SVE2(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width) { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z1.b}, p0/z, [%[src_u]] \n" + "ld1b {z2.b}, p0/z, [%[src_v]] \n" + "incb %[src_u] \n" + "incb %[src_v] \n" + "subs %w[width], %w[width], %w[vl] \n" + "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" + "incb %[dst_uv], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z1.b}, p0/z, [%[src_u]] \n" + "ld1b {z2.b}, p0/z, [%[src_v]] \n" + "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" + + "99: \n" + : [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z1", "z2", "p0"); +} + +void SplitUVRow_SVE2(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n" + "incb %[src_uv], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z1.b}, p0, [%[dst_u]] \n" + "st1b {z2.b}, p0, [%[dst_v]] \n" + "incb %[dst_u] \n" + "incb %[dst_v] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n" + "st1b {z1.b}, p0, [%[dst_u]] \n" + "st1b {z2.b}, p0, [%[dst_v]] \n" + + "99: \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z1", "z2", "p0"); +} + +void HalfMergeUVRow_SVE2(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width) { + const uint8_t* src_u1 = src_u + src_stride_u; + const uint8_t* src_v1 = src_v + src_stride_v; + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z1.b}, p0/z, [%[src_u]] \n" + "ld1b {z2.b}, p0/z, [%[src_u1]] \n" + "ld1b {z3.b}, p0/z, [%[src_v]] \n" + "ld1b {z4.b}, p0/z, [%[src_v1]] \n" + "incb %[src_u] \n" + "incb %[src_u1] \n" + "incb %[src_v] \n" + "incb %[src_v1] \n" + "urhadd z1.b, p0/m, z1.b, z2.b \n" + "urhadd z3.b, p0/m, z3.b, z4.b \n" + "mov z2.b, p0/m, z3.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" + "incb %[dst_uv], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z1.b}, p0/z, [%[src_u]] \n" + "ld1b {z2.b}, p0/z, [%[src_u1]] \n" + "ld1b {z3.b}, p0/z, [%[src_v]] \n" + "ld1b {z4.b}, p0/z, [%[src_v1]] \n" + "urhadd z1.b, p0/m, z1.b, z2.b \n" + "urhadd z3.b, p0/m, z3.b, z4.b \n" + "mov z2.b, p0/m, z3.b \n" + "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" + + "99: \n" + : [src_u] "+r"(src_u), // %[src_u] + [src_u1] "+r"(src_u1), // %[src_u1] + [src_v] "+r"(src_v), // %[src_v] + [src_v1] "+r"(src_v1), // %[src_v1] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "cc", "memory", "z1", "z2", "z3", "z4", "p0"); +} + +void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) { + uint64_t vl; + asm("cntb %x0" : "=r"(vl)); + asm volatile( + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + "ptrue p0.b \n" + "1: \n" + "ld1b {z0.b}, p0/z, [%[src]] \n" + "add %[src], %[src], %[vl] \n" + "st1b {z0.b}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.ge 1b \n" + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z0.b}, p0/z, [%[src]] \n" + "st1b {z0.b}, p0, [%[dst]] \n" + "99: \n" + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) + : [vl] "r"(vl) + : "cc", "memory", "z0", "p0"); +} + +void InterpolateRow_SVE2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + int y1_fraction = source_y_fraction; + int y0_fraction = 256 - y1_fraction; + const uint8_t* src_ptr1 = src_ptr + src_stride; + + if (y0_fraction == 0) { + CopyRow_SVE2(src_ptr1, dst_ptr, width); + return; + } + if (y0_fraction == 256) { + CopyRow_SVE2(src_ptr, dst_ptr, width); + return; + } + + int vl; + asm volatile( + "cntb %x[vl] \n" + "dup z0.b, %w[y0_fraction] \n" + "dup z1.b, %w[y1_fraction] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "incb %[src_ptr] \n" + "incb %[src_ptr1] \n" + "umullb z4.h, z2.b, z0.b \n" + "umullt z2.h, z2.b, z0.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "umlalb z4.h, z3.b, z1.b \n" + "umlalt z2.h, z3.b, z1.b \n" + "rshrnb z3.b, z4.h, #8 \n" + "rshrnt z3.b, z2.h, #8 \n" + "st1b {z3.b}, p0, [%[dst_ptr]] \n" + "incb %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "umullb z4.h, z2.b, z0.b \n" + "umullt z2.h, z2.b, z0.b \n" + "umlalb z4.h, z3.b, z1.b \n" + "umlalt z2.h, z3.b, z1.b \n" + "rshrnb z3.b, z4.h, #8 \n" + "rshrnt z3.b, z2.h, #8 \n" + "st1b {z3.b}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] + [y1_fraction] "r"(y1_fraction) // %[y1_fraction] + : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0"); +} + +void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" + "incb %[src_yuy2], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z0.b}, p0, [%[dst_y]] \n" + "incb %[dst_y] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" + "st1b {z0.b}, p0, [%[dst_y]] \n" + + "99: \n" + : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2] + [dst_y] "+r"(dst_y), // %[dst_y] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "p0"); +} + +void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" + "incb %[src_uyvy], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z1.b}, p0, [%[dst_y]] \n" + "incb %[dst_y] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" + "st1b {z1.b}, p0, [%[dst_y]] \n" + + "99: \n" + : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy] + [dst_y] "+r"(dst_y), // %[dst_y] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "p0"); +} + +void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2; + int vl; + int vl_half; + int width_half; + if (src_stride_yuy2 == 0) { + src_yuy2_1 = src_yuy2; + } + asm volatile( + "cntb %x[vl] \n" + "lsr %w[vl_half], %w[vl], #1 \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "whilelt p1.b, wzr, %w[vl_half] \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" + "ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n" + "incb %[src_yuy2], all, mul #2 \n" + "incb %[src_yuy2_1], all, mul #2 \n" + "urhadd z1.b, p0/m, z1.b, z5.b \n" + "uzp1 z0.b, z1.b, z1.b \n" + "uzp2 z2.b, z1.b, z1.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z0.b}, p1, [%[dst_u]] \n" + "st1b {z2.b}, p1, [%[dst_v]] \n" + "incp %[dst_u], p1.b \n" + "incp %[dst_v], p1.b \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "add %w[width_half], %w[width], #1 \n" + "lsr %w[width_half], %w[width_half], #1 \n" + "whilelt p1.b, wzr, %w[width_half] \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n" + "ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n" + "urhadd z1.b, p0/m, z1.b, z5.b \n" + "uzp1 z0.b, z1.b, z1.b \n" + "uzp2 z2.b, z1.b, z1.b \n" + "st1b {z0.b}, p1, [%[dst_u]] \n" + "st1b {z2.b}, p1, [%[dst_v]] \n" + + "99: \n" + : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2] + [src_yuy2_1] "+r"(src_yuy2_1), // %[src_yuy2_1] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [vl_half] "=&r"(vl_half), // %[vl_half] + [width_half] "=&r"(width_half) // %[width_half] + : + : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1"); +} + +void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, + int src_stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* src_uyvy_1 = src_uyvy + src_stride_uyvy; + int vl; + int vl_half; + int width_half; + if (src_stride_uyvy == 0) { + src_uyvy_1 = src_uyvy; + } + asm volatile( + "cntb %x[vl] \n" + "lsr %w[vl_half], %w[vl], #1 \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "whilelt p1.b, wzr, %w[vl_half] \n" + "1: \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" + "ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n" + "incb %[src_uyvy], all, mul #2 \n" + "incb %[src_uyvy_1], all, mul #2 \n" + "urhadd z0.b, p0/m, z0.b, z4.b \n" + "uzp1 z1.b, z0.b, z0.b \n" + "uzp2 z2.b, z0.b, z0.b \n" + "subs %w[width], %w[width], %w[vl] \n" + "st1b {z1.b}, p1, [%[dst_u]] \n" + "st1b {z2.b}, p1, [%[dst_v]] \n" + "incp %[dst_u], p1.b \n" + "incp %[dst_v], p1.b \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "add %w[width_half], %w[width], #1 \n" + "lsr %w[width_half], %w[width_half], #1 \n" + "whilelt p1.b, wzr, %w[width_half] \n" + "ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n" + "ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n" + "urhadd z0.b, p0/m, z0.b, z4.b \n" + "uzp1 z1.b, z0.b, z0.b \n" + "uzp2 z2.b, z0.b, z0.b \n" + "st1b {z1.b}, p1, [%[dst_u]] \n" + "st1b {z2.b}, p1, [%[dst_v]] \n" + + "99: \n" + : [src_uyvy] "+r"(src_uyvy), // %[src_uyvy] + [src_uyvy_1] "+r"(src_uyvy_1), // %[src_uyvy_1] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [vl_half] "=&r"(vl_half), // %[vl_half] + [width_half] "=&r"(width_half) // %[width_half] + : + : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1"); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/scale.cc b/source/scale.cc index 4b7b2d3bc..09be3c626 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -1093,6 +1093,11 @@ static int ScalePlaneBilinearDown(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME; @@ -1291,6 +1296,11 @@ static int ScalePlaneBilinearUp(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME; diff --git a/source/scale_argb.cc b/source/scale_argb.cc index 8758d79c2..1c376e93f 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -347,6 +347,11 @@ static int ScaleARGBBilinearDown(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME; @@ -462,6 +467,11 @@ static int ScaleARGBBilinearUp(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME; diff --git a/source/scale_common.cc b/source/scale_common.cc index e2447119b..1c9b005d0 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1652,6 +1652,11 @@ void ScalePlaneVertical(int src_height, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME; diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 43a464732..e20e708be 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -380,6 +380,11 @@ static int ScaleUVBilinearDown(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME; @@ -482,6 +487,11 @@ static int ScaleUVBilinearUp(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif #if defined(HAS_INTERPOLATEROW_SME) if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_SME;