From 9c5d5c1e55e8956d6bcd9cabddace4541abde2d3 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Fri, 7 Aug 2026 19:59:08 -0700 Subject: [PATCH] InterpolateRow_NEON/SVE2 and InterpolateRow_16_NEON/SVE2 use 2 registers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace rshrn2 with 2 8 byte registers and stp All benchmarks run at 1280×720, 1000 iterations (taskset c0): Function (1280x720 1000x) | OldNEON | NewNEON | OldSVE2 | NewSVE2 -------------------------------+---------+---------+---------+-------- TestInterpolatePlane (8-bit) | 76 ms | 66 ms | 125 ms | 34 ms TestInterpolatePlane_16 | 195 ms | 66 ms | 195 ms | 64 ms TestARGBInterpolate (50/50) | 104 ms | 104 ms | 128 ms | 89 ms ARGBInterpolate0_Opt (0%) | 133 ms | 128 ms | 148 ms | 145 ms ARGBInterpolate64_Opt (25%) | 211 ms | 181 ms | 244 ms | 222 ms ARGBInterpolate128_Opt (50%) | 182 ms | 178 ms | 278 ms | 176 ms ARGBInterpolate192_Opt (75%) | 211 ms | 173 ms | 223 ms | 209 ms ARGBInterpolate255_Opt (99%) | 211 ms | 183 ms | 315 ms | 221 ms ────── Function (1280x720 1000x) | OldNEON | NewNEON | OldSVE2 | NewSVE2 -------------------------------+---------+---------+---------+-------- TestInterpolatePlane (8-bit) | 164 ms | 124 ms | 119 ms | 50 ms TestInterpolatePlane_16 | 211 ms | 129 ms | 138 ms | 75 ms TestARGBInterpolate (50/50) | 159 ms | 159 ms | 210 ms | 166 ms ARGBInterpolate0_Opt (0%) | 92 ms | 92 ms | 148 ms | 138 ms ARGBInterpolate64_Opt (25%) | 223 ms | 223 ms | 232 ms | 232 ms ARGBInterpolate128_Opt (50%) | 130 ms | 130 ms | 232 ms | 195 ms ARGBInterpolate192_Opt (75%) | 223 ms | 223 ms | 232 ms | 232 ms ARGBInterpolate255_Opt (99%) | 223 ms | 223 ms | 232 ms | 232 ms Platform | NEON | SVE2 | Result --------------------+--------+--------+--------------- MediaTek Cortex-C1 | 47 ms | 34 ms | SVE2 1.38x win Qualcomm Oryon v3 | 87 ms | 73 ms | SVE2 1.19x win Testing: blaze test --config=android_arm64 //third_party/libyuv:libyuv_test --test_filter="*TestInterpolatePlane*" Bug: None Change-Id: I503ce5a1269da5e1c5950c6a2c5ce81ddf4aaca5 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8216829 Commit-Queue: Frank Barchard Reviewed-by: George Steed Reviewed-by: Wan-Teh Chang --- source/row_neon64.cc | 72 +++++++++++++++++++++++--------------- source/row_sve.cc | 82 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 113 insertions(+), 41 deletions(-) diff --git a/source/row_neon64.cc b/source/row_neon64.cc index ca879dd1b..6becaec87 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -3617,19 +3617,17 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, "dup v5.16b, %w4 \n" "dup v4.16b, %w5 \n" // General purpose row blend. - "1: \n" + "1: \n" "ld1 {v0.16b}, [%1], #16 \n" "ld1 {v1.16b}, [%2], #16 \n" "subs %w3, %w3, #16 \n" "umull v2.8h, v0.8b, v4.8b \n" - "prfm pldl1keep, [%1, 448] \n" "umull2 v3.8h, v0.16b, v4.16b \n" - "prfm pldl1keep, [%2, 448] \n" "umlal v2.8h, v1.8b, v5.8b \n" "umlal2 v3.8h, v1.16b, v5.16b \n" "rshrn v0.8b, v2.8h, #8 \n" - "rshrn2 v0.16b, v3.8h, #8 \n" - "st1 {v0.16b}, [%0], #16 \n" + "rshrn v1.8b, v3.8h, #8 \n" + "stp d0, d1, [%0], #16 \n" "b.gt 1b \n" "b 99f \n" @@ -3638,9 +3636,7 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, "ld1 {v0.16b}, [%1], #16 \n" "ld1 {v1.16b}, [%2], #16 \n" "subs %w3, %w3, #16 \n" - "prfm pldl1keep, [%1, 448] \n" "urhadd v0.16b, v0.16b, v1.16b \n" - "prfm pldl1keep, [%2, 448] \n" "st1 {v0.16b}, [%0], #16 \n" "b.gt 50b \n" "b 99f \n" @@ -3649,7 +3645,6 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, "100: \n" "ld1 {v0.16b}, [%1], #16 \n" "subs %w3, %w3, #16 \n" - "prfm pldl1keep, [%1, 448] \n" "st1 {v0.16b}, [%0], #16 \n" "b.gt 100b \n" @@ -3660,7 +3655,7 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, "+r"(dst_width) // %3 : "r"(y1_fraction), // %4 "r"(y0_fraction) // %5 - : "cc", "memory", "v0", "v1", "v3", "v4", "v5"); + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5"); } // Bilinear filter 8x2 -> 8x1 @@ -3681,21 +3676,44 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "dup v5.8h, %w4 \n" "dup v4.8h, %w5 \n" - // General purpose row blend. - "1: \n" - "ld1 {v0.8h}, [%1], #16 \n" - "ld1 {v1.8h}, [%2], #16 \n" - "subs %w3, %w3, #8 \n" - "umull v2.4s, v0.4h, v4.4h \n" - "prfm pldl1keep, [%1, 448] \n" - "umull2 v3.4s, v0.8h, v4.8h \n" - "prfm pldl1keep, [%2, 448] \n" - "umlal v2.4s, v1.4h, v5.4h \n" - "umlal2 v3.4s, v1.8h, v5.8h \n" - "rshrn v0.4h, v2.4s, #8 \n" - "rshrn2 v0.8h, v3.4s, #8 \n" - "st1 {v0.8h}, [%0], #16 \n" - "b.gt 1b \n" + "subs %w3, %w3, #16 \n" + "b.lt 2f \n" + + // 16-element unrolled loop (32 bytes). + "1: \n" + "ldp q0, q1, [%1], #32 \n" + "ldp q6, q7, [%2], #32 \n" + "subs %w3, %w3, #16 \n" + "umull v16.4s, v0.4h, v4.4h \n" + "umull2 v17.4s, v0.8h, v4.8h \n" + "umull v18.4s, v1.4h, v4.4h \n" + "umull2 v19.4s, v1.8h, v4.8h \n" + "umlal v16.4s, v6.4h, v5.4h \n" + "umlal2 v17.4s, v6.8h, v5.8h \n" + "umlal v18.4s, v7.4h, v5.4h \n" + "umlal2 v19.4s, v7.8h, v5.8h \n" + "rshrn v0.4h, v16.4s, #8 \n" + "rshrn v1.4h, v17.4s, #8 \n" + "rshrn v2.4h, v18.4s, #8 \n" + "rshrn v3.4h, v19.4s, #8 \n" + "stp d0, d1, [%0], #16 \n" + "stp d2, d3, [%0], #16 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w3, %w3, #16 \n" + "b.eq 99f \n" + + // 8-element tail (16 bytes). + "ld1 {v0.8h}, [%1] \n" + "ld1 {v1.8h}, [%2] \n" + "umull v16.4s, v0.4h, v4.4h \n" + "umull2 v17.4s, v0.8h, v4.8h \n" + "umlal v16.4s, v1.4h, v5.4h \n" + "umlal2 v17.4s, v1.8h, v5.8h \n" + "rshrn v0.4h, v16.4s, #8 \n" + "rshrn v1.4h, v17.4s, #8 \n" + "stp d0, d1, [%0] \n" "b 99f \n" // Blend 50 / 50. @@ -3703,9 +3721,7 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "ld1 {v0.8h}, [%1], #16 \n" "ld1 {v1.8h}, [%2], #16 \n" "subs %w3, %w3, #8 \n" - "prfm pldl1keep, [%1, 448] \n" "urhadd v0.8h, v0.8h, v1.8h \n" - "prfm pldl1keep, [%2, 448] \n" "st1 {v0.8h}, [%0], #16 \n" "b.gt 50b \n" "b 99f \n" @@ -3714,7 +3730,6 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "100: \n" "ld1 {v0.8h}, [%1], #16 \n" "subs %w3, %w3, #8 \n" - "prfm pldl1keep, [%1, 448] \n" "st1 {v0.8h}, [%0], #16 \n" "b.gt 100b \n" @@ -3725,7 +3740,8 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "+r"(dst_width) // %3 : "r"(y1_fraction), // %4 "r"(y0_fraction) // %5 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5"); + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19"); } // dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr diff --git a/source/row_sve.cc b/source/row_sve.cc index 4e465f7c4..1ce86940d 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1186,29 +1186,46 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u, } void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) { - uint64_t vl; - asm("cntb %x0" : "=r"(vl)); + int vl; + int vl2; asm volatile( - "subs %w[width], %w[width], %w[vl] \n" + "cntb %x[vl] \n" + "lsl %w[vl2], %w[vl], #1 \n" + "subs %w[width], %w[width], %w[vl2] \n" "b.lt 2f \n" + "ptrue p0.b \n" "1: \n" - "ld1b {z0.b}, p0/z, [%[src]] \n" - "add %[src], %[src], %[vl] \n" - "st1b {z0.b}, p0, [%[dst]] \n" - "add %[dst], %[dst], %[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.ge 1b \n" + "ld1b {z0.b}, p0/z, [%[src], #0, mul vl] \n" + "ld1b {z1.b}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl2] \n" + "st1b {z0.b}, p0, [%[dst], #0, mul vl] \n" + "st1b {z1.b}, p0, [%[dst], #1, mul vl] \n" + "incb %[dst], all, mul #2 \n" + "b.gt 1b \n" + "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl2] \n" "b.eq 99f \n" + + "3: \n" "whilelt p0.b, wzr, %w[width] \n" "ld1b {z0.b}, p0/z, [%[src]] \n" + "incb %[src] \n" "st1b {z0.b}, p0, [%[dst]] \n" + "incb %[dst] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.gt 3b \n" + "99: \n" - : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) - : [vl] "r"(vl) - : "cc", "memory", "z0", "p0"); + : [src] "+r"(src), + [dst] "+r"(dst), + [width] "+r"(width), + [vl] "=&r"(vl), + [vl2] "=&r"(vl2) + : + : "cc", "memory", "z0", "z1", "p0"); } void InterpolateRow_SVE2(uint8_t* dst_ptr, @@ -1228,6 +1245,45 @@ void InterpolateRow_SVE2(uint8_t* dst_ptr, CopyRow_SVE2(src_ptr, dst_ptr, width); return; } + if (y0_fraction == 128) { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "incb %[src_ptr] \n" + "incb %[src_ptr1] \n" + "subs %w[width], %w[width], %w[vl] \n" + "urhadd z2.b, p0/m, z2.b, z3.b \n" + "st1b {z2.b}, p0, [%[dst_ptr]] \n" + "incb %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "urhadd z2.b, p0/m, z2.b, z3.b \n" + "st1b {z2.b}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_ptr), + [src_ptr1] "+r"(src_ptr1), + [dst_ptr] "+r"(dst_ptr), + [width] "+r"(width), + [vl] "=&r"(vl) + : + : "cc", "memory", "z2", "z3", "p0"); + return; + } int vl; asm volatile(