From 2dd4257364d39c38d79465c4ddc4b93137fe729b Mon Sep 17 00:00:00 2001 From: George Steed Date: Mon, 10 Aug 2026 09:45:29 +0100 Subject: [PATCH] AArch64: Fix HalfMergeUVRow_SVE2 average calculation The existing SVE2 path only computes the average for a pair of input values instead of groups of four as done in the Neon implementation. The average of two values must be retained for the final odd element if present. Existing code just handles this by falling back to the C implementation, so mirror that for SVE2 as well. Reduction in time taken relative to the existing Neon code, the SVE2 code remains faster even with the correct averaging: Cortex-A510: -12.66% Lumex C1-Nano: -6.38% Cortex-A720: -9.73% Lumex C1-Pro: +0.99% Cortex-X4: -3.05% Lumex C1-Ultra: -18.98% Bug: 546962730 Change-Id: I7ab348e64a9d079b81d070cea5fd575243b037d2 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8252281 Reviewed-by: Wan-Teh Chang Reviewed-by: Justin Green Commit-Queue: Wan-Teh Chang --- include/libyuv/row.h | 2 +- source/planar_functions.cc | 5 ++++- source/row_sve.cc | 30 ++++++++++++++++++++---------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 736d2f9bc..98e332319 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -634,7 +634,7 @@ extern "C" { #define HAS_COPYROW_SVE2 #define HAS_DIVIDEROW_16_SVE2 #define HAS_HALFFLOATROW_SVE2 -// #define HAS_HALFMERGEUVROW_SVE2 // Disabled: Bug 546962730 +#define HAS_HALFMERGEUVROW_SVE2 // #define HAS_HALFROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors. // #define HAS_HALFWIDTHROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors. #define HAS_I210ALPHATOARGBROW_SVE2 diff --git a/source/planar_functions.cc b/source/planar_functions.cc index b7ae2a3e4..f03c8fa84 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -5921,7 +5921,10 @@ void HalfMergeUVPlane(const uint8_t* src_u, } #endif #if defined(HAS_HALFMERGEUVROW_SVE2) - if (TestCpuFlag(kCpuHasSVE2)) { + // The average of two values must be retained for the final odd element if + // present. Existing code just handles this by falling back to the C + // implementation, so mirror that for SVE2 as well. + if (TestCpuFlag(kCpuHasSVE2) && IS_ALIGNED(width, 2)) { HalfMergeUVRow = HalfMergeUVRow_SVE2; } #endif diff --git a/source/row_sve.cc b/source/row_sve.cc index 4aa66b1b2..f8fa227a6 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1151,12 +1151,17 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u, "incb %[src_u1] \n" "incb %[src_v] \n" "incb %[src_v1] \n" - "urhadd z1.b, p0/m, z1.b, z2.b \n" - "urhadd z3.b, p0/m, z3.b, z4.b \n" - "mov z2.b, p0/m, z3.b \n" + "uaddlb z5.h, z1.b, z2.b \n" + "uaddlb z6.h, z3.b, z4.b \n" + "uaddlt z1.h, z1.b, z2.b \n" + "uaddlt z3.h, z3.b, z4.b \n" + "add z1.h, z1.h, z5.h \n" + "add z3.h, z3.h, z6.h \n" + "rshrnb z1.b, z1.h, #2 \n" + "rshrnt z1.b, z3.h, #2 \n" "subs %w[width], %w[width], %w[vl] \n" - "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" - "incb %[dst_uv], all, mul #2 \n" + "st1b {z1.b}, p0, [%[dst_uv]] \n" + "incb %[dst_uv] \n" "b.ge 1b \n" "2: \n" @@ -1168,10 +1173,15 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u, "ld1b {z2.b}, p0/z, [%[src_u1]] \n" "ld1b {z3.b}, p0/z, [%[src_v]] \n" "ld1b {z4.b}, p0/z, [%[src_v1]] \n" - "urhadd z1.b, p0/m, z1.b, z2.b \n" - "urhadd z3.b, p0/m, z3.b, z4.b \n" - "mov z2.b, p0/m, z3.b \n" - "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" + "uaddlb z5.h, z1.b, z2.b \n" + "uaddlb z6.h, z3.b, z4.b \n" + "uaddlt z1.h, z1.b, z2.b \n" + "uaddlt z3.h, z3.b, z4.b \n" + "add z1.h, z1.h, z5.h \n" + "add z3.h, z3.h, z6.h \n" + "rshrnb z1.b, z1.h, #2 \n" + "rshrnt z1.b, z3.h, #2 \n" + "st1b {z1.b}, p0, [%[dst_uv]] \n" "99: \n" : [src_u] "+r"(src_u), // %[src_u] @@ -1182,7 +1192,7 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u, [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : - : "cc", "memory", "z1", "z2", "z3", "z4", "p0"); + : "cc", "memory", "z1", "z2", "z3", "z4", "z5", "z6", "p0"); } void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) {