mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
InterpolateRow_NEON/SVE2 and InterpolateRow_16_NEON/SVE2 use 2 registers
Replace rshrn2 with 2 8 byte registers and stp All benchmarks run at 1280×720, 1000 iterations (taskset c0): Function (1280x720 1000x) | OldNEON | NewNEON | OldSVE2 | NewSVE2 -------------------------------+---------+---------+---------+-------- TestInterpolatePlane (8-bit) | 76 ms | 66 ms | 125 ms | 34 ms TestInterpolatePlane_16 | 195 ms | 66 ms | 195 ms | 64 ms TestARGBInterpolate (50/50) | 104 ms | 104 ms | 128 ms | 89 ms ARGBInterpolate0_Opt (0%) | 133 ms | 128 ms | 148 ms | 145 ms ARGBInterpolate64_Opt (25%) | 211 ms | 181 ms | 244 ms | 222 ms ARGBInterpolate128_Opt (50%) | 182 ms | 178 ms | 278 ms | 176 ms ARGBInterpolate192_Opt (75%) | 211 ms | 173 ms | 223 ms | 209 ms ARGBInterpolate255_Opt (99%) | 211 ms | 183 ms | 315 ms | 221 ms ────── Function (1280x720 1000x) | OldNEON | NewNEON | OldSVE2 | NewSVE2 -------------------------------+---------+---------+---------+-------- TestInterpolatePlane (8-bit) | 164 ms | 124 ms | 119 ms | 50 ms TestInterpolatePlane_16 | 211 ms | 129 ms | 138 ms | 75 ms TestARGBInterpolate (50/50) | 159 ms | 159 ms | 210 ms | 166 ms ARGBInterpolate0_Opt (0%) | 92 ms | 92 ms | 148 ms | 138 ms ARGBInterpolate64_Opt (25%) | 223 ms | 223 ms | 232 ms | 232 ms ARGBInterpolate128_Opt (50%) | 130 ms | 130 ms | 232 ms | 195 ms ARGBInterpolate192_Opt (75%) | 223 ms | 223 ms | 232 ms | 232 ms ARGBInterpolate255_Opt (99%) | 223 ms | 223 ms | 232 ms | 232 ms Platform | NEON | SVE2 | Result --------------------+--------+--------+--------------- MediaTek Cortex-C1 | 47 ms | 34 ms | SVE2 1.38x win Qualcomm Oryon v3 | 87 ms | 73 ms | SVE2 1.19x win Testing: blaze test --config=android_arm64 //third_party/libyuv:libyuv_test --test_filter="*TestInterpolatePlane*" Bug: None Change-Id: I503ce5a1269da5e1c5950c6a2c5ce81ddf4aaca5 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8216829 Commit-Queue: Frank Barchard <fbarchard@google.com> Reviewed-by: George Steed <george.steed@arm.com> Reviewed-by: Wan-Teh Chang <wtc@google.com>
This commit is contained in:
Frank Barchard
parent
7db6591b96
commit
9c5d5c1e55
@ -3617,19 +3617,17 @@ void InterpolateRow_NEON(uint8_t* dst_ptr,
|
||||
"dup v5.16b, %w4 \n"
|
||||
"dup v4.16b, %w5 \n"
|
||||
// General purpose row blend.
|
||||
"1: \n"
|
||||
"1: \n"
|
||||
"ld1 {v0.16b}, [%1], #16 \n"
|
||||
"ld1 {v1.16b}, [%2], #16 \n"
|
||||
"subs %w3, %w3, #16 \n"
|
||||
"umull v2.8h, v0.8b, v4.8b \n"
|
||||
"prfm pldl1keep, [%1, 448] \n"
|
||||
"umull2 v3.8h, v0.16b, v4.16b \n"
|
||||
"prfm pldl1keep, [%2, 448] \n"
|
||||
"umlal v2.8h, v1.8b, v5.8b \n"
|
||||
"umlal2 v3.8h, v1.16b, v5.16b \n"
|
||||
"rshrn v0.8b, v2.8h, #8 \n"
|
||||
"rshrn2 v0.16b, v3.8h, #8 \n"
|
||||
"st1 {v0.16b}, [%0], #16 \n"
|
||||
"rshrn v1.8b, v3.8h, #8 \n"
|
||||
"stp d0, d1, [%0], #16 \n"
|
||||
"b.gt 1b \n"
|
||||
"b 99f \n"
|
||||
|
||||
@ -3638,9 +3636,7 @@ void InterpolateRow_NEON(uint8_t* dst_ptr,
|
||||
"ld1 {v0.16b}, [%1], #16 \n"
|
||||
"ld1 {v1.16b}, [%2], #16 \n"
|
||||
"subs %w3, %w3, #16 \n"
|
||||
"prfm pldl1keep, [%1, 448] \n"
|
||||
"urhadd v0.16b, v0.16b, v1.16b \n"
|
||||
"prfm pldl1keep, [%2, 448] \n"
|
||||
"st1 {v0.16b}, [%0], #16 \n"
|
||||
"b.gt 50b \n"
|
||||
"b 99f \n"
|
||||
@ -3649,7 +3645,6 @@ void InterpolateRow_NEON(uint8_t* dst_ptr,
|
||||
"100: \n"
|
||||
"ld1 {v0.16b}, [%1], #16 \n"
|
||||
"subs %w3, %w3, #16 \n"
|
||||
"prfm pldl1keep, [%1, 448] \n"
|
||||
"st1 {v0.16b}, [%0], #16 \n"
|
||||
"b.gt 100b \n"
|
||||
|
||||
@ -3660,7 +3655,7 @@ void InterpolateRow_NEON(uint8_t* dst_ptr,
|
||||
"+r"(dst_width) // %3
|
||||
: "r"(y1_fraction), // %4
|
||||
"r"(y0_fraction) // %5
|
||||
: "cc", "memory", "v0", "v1", "v3", "v4", "v5");
|
||||
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
|
||||
}
|
||||
|
||||
// Bilinear filter 8x2 -> 8x1
|
||||
@ -3681,21 +3676,44 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr,
|
||||
|
||||
"dup v5.8h, %w4 \n"
|
||||
"dup v4.8h, %w5 \n"
|
||||
// General purpose row blend.
|
||||
"1: \n"
|
||||
"ld1 {v0.8h}, [%1], #16 \n"
|
||||
"ld1 {v1.8h}, [%2], #16 \n"
|
||||
"subs %w3, %w3, #8 \n"
|
||||
"umull v2.4s, v0.4h, v4.4h \n"
|
||||
"prfm pldl1keep, [%1, 448] \n"
|
||||
"umull2 v3.4s, v0.8h, v4.8h \n"
|
||||
"prfm pldl1keep, [%2, 448] \n"
|
||||
"umlal v2.4s, v1.4h, v5.4h \n"
|
||||
"umlal2 v3.4s, v1.8h, v5.8h \n"
|
||||
"rshrn v0.4h, v2.4s, #8 \n"
|
||||
"rshrn2 v0.8h, v3.4s, #8 \n"
|
||||
"st1 {v0.8h}, [%0], #16 \n"
|
||||
"b.gt 1b \n"
|
||||
"subs %w3, %w3, #16 \n"
|
||||
"b.lt 2f \n"
|
||||
|
||||
// 16-element unrolled loop (32 bytes).
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%1], #32 \n"
|
||||
"ldp q6, q7, [%2], #32 \n"
|
||||
"subs %w3, %w3, #16 \n"
|
||||
"umull v16.4s, v0.4h, v4.4h \n"
|
||||
"umull2 v17.4s, v0.8h, v4.8h \n"
|
||||
"umull v18.4s, v1.4h, v4.4h \n"
|
||||
"umull2 v19.4s, v1.8h, v4.8h \n"
|
||||
"umlal v16.4s, v6.4h, v5.4h \n"
|
||||
"umlal2 v17.4s, v6.8h, v5.8h \n"
|
||||
"umlal v18.4s, v7.4h, v5.4h \n"
|
||||
"umlal2 v19.4s, v7.8h, v5.8h \n"
|
||||
"rshrn v0.4h, v16.4s, #8 \n"
|
||||
"rshrn v1.4h, v17.4s, #8 \n"
|
||||
"rshrn v2.4h, v18.4s, #8 \n"
|
||||
"rshrn v3.4h, v19.4s, #8 \n"
|
||||
"stp d0, d1, [%0], #16 \n"
|
||||
"stp d2, d3, [%0], #16 \n"
|
||||
"b.ge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"adds %w3, %w3, #16 \n"
|
||||
"b.eq 99f \n"
|
||||
|
||||
// 8-element tail (16 bytes).
|
||||
"ld1 {v0.8h}, [%1] \n"
|
||||
"ld1 {v1.8h}, [%2] \n"
|
||||
"umull v16.4s, v0.4h, v4.4h \n"
|
||||
"umull2 v17.4s, v0.8h, v4.8h \n"
|
||||
"umlal v16.4s, v1.4h, v5.4h \n"
|
||||
"umlal2 v17.4s, v1.8h, v5.8h \n"
|
||||
"rshrn v0.4h, v16.4s, #8 \n"
|
||||
"rshrn v1.4h, v17.4s, #8 \n"
|
||||
"stp d0, d1, [%0] \n"
|
||||
"b 99f \n"
|
||||
|
||||
// Blend 50 / 50.
|
||||
@ -3703,9 +3721,7 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr,
|
||||
"ld1 {v0.8h}, [%1], #16 \n"
|
||||
"ld1 {v1.8h}, [%2], #16 \n"
|
||||
"subs %w3, %w3, #8 \n"
|
||||
"prfm pldl1keep, [%1, 448] \n"
|
||||
"urhadd v0.8h, v0.8h, v1.8h \n"
|
||||
"prfm pldl1keep, [%2, 448] \n"
|
||||
"st1 {v0.8h}, [%0], #16 \n"
|
||||
"b.gt 50b \n"
|
||||
"b 99f \n"
|
||||
@ -3714,7 +3730,6 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr,
|
||||
"100: \n"
|
||||
"ld1 {v0.8h}, [%1], #16 \n"
|
||||
"subs %w3, %w3, #8 \n"
|
||||
"prfm pldl1keep, [%1, 448] \n"
|
||||
"st1 {v0.8h}, [%0], #16 \n"
|
||||
"b.gt 100b \n"
|
||||
|
||||
@ -3725,7 +3740,8 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr,
|
||||
"+r"(dst_width) // %3
|
||||
: "r"(y1_fraction), // %4
|
||||
"r"(y0_fraction) // %5
|
||||
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
|
||||
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
|
||||
"v17", "v18", "v19");
|
||||
}
|
||||
|
||||
// dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr
|
||||
|
||||
@ -1186,29 +1186,46 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u,
|
||||
}
|
||||
|
||||
void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) {
|
||||
uint64_t vl;
|
||||
asm("cntb %x0" : "=r"(vl));
|
||||
int vl;
|
||||
int vl2;
|
||||
asm volatile(
|
||||
"subs %w[width], %w[width], %w[vl] \n"
|
||||
"cntb %x[vl] \n"
|
||||
"lsl %w[vl2], %w[vl], #1 \n"
|
||||
"subs %w[width], %w[width], %w[vl2] \n"
|
||||
"b.lt 2f \n"
|
||||
|
||||
"ptrue p0.b \n"
|
||||
"1: \n"
|
||||
"ld1b {z0.b}, p0/z, [%[src]] \n"
|
||||
"add %[src], %[src], %[vl] \n"
|
||||
"st1b {z0.b}, p0, [%[dst]] \n"
|
||||
"add %[dst], %[dst], %[vl] \n"
|
||||
"subs %w[width], %w[width], %w[vl] \n"
|
||||
"b.ge 1b \n"
|
||||
"ld1b {z0.b}, p0/z, [%[src], #0, mul vl] \n"
|
||||
"ld1b {z1.b}, p0/z, [%[src], #1, mul vl] \n"
|
||||
"incb %[src], all, mul #2 \n"
|
||||
"subs %w[width], %w[width], %w[vl2] \n"
|
||||
"st1b {z0.b}, p0, [%[dst], #0, mul vl] \n"
|
||||
"st1b {z1.b}, p0, [%[dst], #1, mul vl] \n"
|
||||
"incb %[dst], all, mul #2 \n"
|
||||
"b.gt 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"adds %w[width], %w[width], %w[vl] \n"
|
||||
"adds %w[width], %w[width], %w[vl2] \n"
|
||||
"b.eq 99f \n"
|
||||
|
||||
"3: \n"
|
||||
"whilelt p0.b, wzr, %w[width] \n"
|
||||
"ld1b {z0.b}, p0/z, [%[src]] \n"
|
||||
"incb %[src] \n"
|
||||
"st1b {z0.b}, p0, [%[dst]] \n"
|
||||
"incb %[dst] \n"
|
||||
"subs %w[width], %w[width], %w[vl] \n"
|
||||
"b.gt 3b \n"
|
||||
|
||||
"99: \n"
|
||||
: [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width)
|
||||
: [vl] "r"(vl)
|
||||
: "cc", "memory", "z0", "p0");
|
||||
: [src] "+r"(src),
|
||||
[dst] "+r"(dst),
|
||||
[width] "+r"(width),
|
||||
[vl] "=&r"(vl),
|
||||
[vl2] "=&r"(vl2)
|
||||
:
|
||||
: "cc", "memory", "z0", "z1", "p0");
|
||||
}
|
||||
|
||||
void InterpolateRow_SVE2(uint8_t* dst_ptr,
|
||||
@ -1228,6 +1245,45 @@ void InterpolateRow_SVE2(uint8_t* dst_ptr,
|
||||
CopyRow_SVE2(src_ptr, dst_ptr, width);
|
||||
return;
|
||||
}
|
||||
if (y0_fraction == 128) {
|
||||
int vl;
|
||||
asm volatile(
|
||||
"cntb %x[vl] \n"
|
||||
"subs %w[width], %w[width], %w[vl] \n"
|
||||
"b.lt 2f \n"
|
||||
|
||||
"ptrue p0.b \n"
|
||||
"1: \n"
|
||||
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
||||
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
||||
"incb %[src_ptr] \n"
|
||||
"incb %[src_ptr1] \n"
|
||||
"subs %w[width], %w[width], %w[vl] \n"
|
||||
"urhadd z2.b, p0/m, z2.b, z3.b \n"
|
||||
"st1b {z2.b}, p0, [%[dst_ptr]] \n"
|
||||
"incb %[dst_ptr] \n"
|
||||
"b.ge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"adds %w[width], %w[width], %w[vl] \n"
|
||||
"b.eq 99f \n"
|
||||
|
||||
"whilelt p0.b, wzr, %w[width] \n"
|
||||
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
||||
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
||||
"urhadd z2.b, p0/m, z2.b, z3.b \n"
|
||||
"st1b {z2.b}, p0, [%[dst_ptr]] \n"
|
||||
|
||||
"99: \n"
|
||||
: [src_ptr] "+r"(src_ptr),
|
||||
[src_ptr1] "+r"(src_ptr1),
|
||||
[dst_ptr] "+r"(dst_ptr),
|
||||
[width] "+r"(width),
|
||||
[vl] "=&r"(vl)
|
||||
:
|
||||
: "cc", "memory", "z2", "z3", "p0");
|
||||
return;
|
||||
}
|
||||
|
||||
int vl;
|
||||
asm volatile(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user