diff --git a/source/row_neon64.cc b/source/row_neon64.cc index ca879dd1b..6becaec87 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -3617,19 +3617,17 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, "dup v5.16b, %w4 \n" "dup v4.16b, %w5 \n" // General purpose row blend. - "1: \n" + "1: \n" "ld1 {v0.16b}, [%1], #16 \n" "ld1 {v1.16b}, [%2], #16 \n" "subs %w3, %w3, #16 \n" "umull v2.8h, v0.8b, v4.8b \n" - "prfm pldl1keep, [%1, 448] \n" "umull2 v3.8h, v0.16b, v4.16b \n" - "prfm pldl1keep, [%2, 448] \n" "umlal v2.8h, v1.8b, v5.8b \n" "umlal2 v3.8h, v1.16b, v5.16b \n" "rshrn v0.8b, v2.8h, #8 \n" - "rshrn2 v0.16b, v3.8h, #8 \n" - "st1 {v0.16b}, [%0], #16 \n" + "rshrn v1.8b, v3.8h, #8 \n" + "stp d0, d1, [%0], #16 \n" "b.gt 1b \n" "b 99f \n" @@ -3638,9 +3636,7 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, "ld1 {v0.16b}, [%1], #16 \n" "ld1 {v1.16b}, [%2], #16 \n" "subs %w3, %w3, #16 \n" - "prfm pldl1keep, [%1, 448] \n" "urhadd v0.16b, v0.16b, v1.16b \n" - "prfm pldl1keep, [%2, 448] \n" "st1 {v0.16b}, [%0], #16 \n" "b.gt 50b \n" "b 99f \n" @@ -3649,7 +3645,6 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, "100: \n" "ld1 {v0.16b}, [%1], #16 \n" "subs %w3, %w3, #16 \n" - "prfm pldl1keep, [%1, 448] \n" "st1 {v0.16b}, [%0], #16 \n" "b.gt 100b \n" @@ -3660,7 +3655,7 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, "+r"(dst_width) // %3 : "r"(y1_fraction), // %4 "r"(y0_fraction) // %5 - : "cc", "memory", "v0", "v1", "v3", "v4", "v5"); + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5"); } // Bilinear filter 8x2 -> 8x1 @@ -3681,21 +3676,44 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "dup v5.8h, %w4 \n" "dup v4.8h, %w5 \n" - // General purpose row blend. - "1: \n" - "ld1 {v0.8h}, [%1], #16 \n" - "ld1 {v1.8h}, [%2], #16 \n" - "subs %w3, %w3, #8 \n" - "umull v2.4s, v0.4h, v4.4h \n" - "prfm pldl1keep, [%1, 448] \n" - "umull2 v3.4s, v0.8h, v4.8h \n" - "prfm pldl1keep, [%2, 448] \n" - "umlal v2.4s, v1.4h, v5.4h \n" - "umlal2 v3.4s, v1.8h, v5.8h \n" - "rshrn v0.4h, v2.4s, #8 \n" - "rshrn2 v0.8h, v3.4s, #8 \n" - "st1 {v0.8h}, [%0], #16 \n" - "b.gt 1b \n" + "subs %w3, %w3, #16 \n" + "b.lt 2f \n" + + // 16-element unrolled loop (32 bytes). + "1: \n" + "ldp q0, q1, [%1], #32 \n" + "ldp q6, q7, [%2], #32 \n" + "subs %w3, %w3, #16 \n" + "umull v16.4s, v0.4h, v4.4h \n" + "umull2 v17.4s, v0.8h, v4.8h \n" + "umull v18.4s, v1.4h, v4.4h \n" + "umull2 v19.4s, v1.8h, v4.8h \n" + "umlal v16.4s, v6.4h, v5.4h \n" + "umlal2 v17.4s, v6.8h, v5.8h \n" + "umlal v18.4s, v7.4h, v5.4h \n" + "umlal2 v19.4s, v7.8h, v5.8h \n" + "rshrn v0.4h, v16.4s, #8 \n" + "rshrn v1.4h, v17.4s, #8 \n" + "rshrn v2.4h, v18.4s, #8 \n" + "rshrn v3.4h, v19.4s, #8 \n" + "stp d0, d1, [%0], #16 \n" + "stp d2, d3, [%0], #16 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w3, %w3, #16 \n" + "b.eq 99f \n" + + // 8-element tail (16 bytes). + "ld1 {v0.8h}, [%1] \n" + "ld1 {v1.8h}, [%2] \n" + "umull v16.4s, v0.4h, v4.4h \n" + "umull2 v17.4s, v0.8h, v4.8h \n" + "umlal v16.4s, v1.4h, v5.4h \n" + "umlal2 v17.4s, v1.8h, v5.8h \n" + "rshrn v0.4h, v16.4s, #8 \n" + "rshrn v1.4h, v17.4s, #8 \n" + "stp d0, d1, [%0] \n" "b 99f \n" // Blend 50 / 50. @@ -3703,9 +3721,7 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "ld1 {v0.8h}, [%1], #16 \n" "ld1 {v1.8h}, [%2], #16 \n" "subs %w3, %w3, #8 \n" - "prfm pldl1keep, [%1, 448] \n" "urhadd v0.8h, v0.8h, v1.8h \n" - "prfm pldl1keep, [%2, 448] \n" "st1 {v0.8h}, [%0], #16 \n" "b.gt 50b \n" "b 99f \n" @@ -3714,7 +3730,6 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "100: \n" "ld1 {v0.8h}, [%1], #16 \n" "subs %w3, %w3, #8 \n" - "prfm pldl1keep, [%1, 448] \n" "st1 {v0.8h}, [%0], #16 \n" "b.gt 100b \n" @@ -3725,7 +3740,8 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "+r"(dst_width) // %3 : "r"(y1_fraction), // %4 "r"(y0_fraction) // %5 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5"); + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19"); } // dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr diff --git a/source/row_sve.cc b/source/row_sve.cc index 4e465f7c4..1ce86940d 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1186,29 +1186,46 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u, } void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) { - uint64_t vl; - asm("cntb %x0" : "=r"(vl)); + int vl; + int vl2; asm volatile( - "subs %w[width], %w[width], %w[vl] \n" + "cntb %x[vl] \n" + "lsl %w[vl2], %w[vl], #1 \n" + "subs %w[width], %w[width], %w[vl2] \n" "b.lt 2f \n" + "ptrue p0.b \n" "1: \n" - "ld1b {z0.b}, p0/z, [%[src]] \n" - "add %[src], %[src], %[vl] \n" - "st1b {z0.b}, p0, [%[dst]] \n" - "add %[dst], %[dst], %[vl] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.ge 1b \n" + "ld1b {z0.b}, p0/z, [%[src], #0, mul vl] \n" + "ld1b {z1.b}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" + "subs %w[width], %w[width], %w[vl2] \n" + "st1b {z0.b}, p0, [%[dst], #0, mul vl] \n" + "st1b {z1.b}, p0, [%[dst], #1, mul vl] \n" + "incb %[dst], all, mul #2 \n" + "b.gt 1b \n" + "2: \n" - "adds %w[width], %w[width], %w[vl] \n" + "adds %w[width], %w[width], %w[vl2] \n" "b.eq 99f \n" + + "3: \n" "whilelt p0.b, wzr, %w[width] \n" "ld1b {z0.b}, p0/z, [%[src]] \n" + "incb %[src] \n" "st1b {z0.b}, p0, [%[dst]] \n" + "incb %[dst] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.gt 3b \n" + "99: \n" - : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) - : [vl] "r"(vl) - : "cc", "memory", "z0", "p0"); + : [src] "+r"(src), + [dst] "+r"(dst), + [width] "+r"(width), + [vl] "=&r"(vl), + [vl2] "=&r"(vl2) + : + : "cc", "memory", "z0", "z1", "p0"); } void InterpolateRow_SVE2(uint8_t* dst_ptr, @@ -1228,6 +1245,45 @@ void InterpolateRow_SVE2(uint8_t* dst_ptr, CopyRow_SVE2(src_ptr, dst_ptr, width); return; } + if (y0_fraction == 128) { + int vl; + asm volatile( + "cntb %x[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.b \n" + "1: \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "incb %[src_ptr] \n" + "incb %[src_ptr1] \n" + "subs %w[width], %w[width], %w[vl] \n" + "urhadd z2.b, p0/m, z2.b, z3.b \n" + "st1b {z2.b}, p0, [%[dst_ptr]] \n" + "incb %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.b, wzr, %w[width] \n" + "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" + "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" + "urhadd z2.b, p0/m, z2.b, z3.b \n" + "st1b {z2.b}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_ptr), + [src_ptr1] "+r"(src_ptr1), + [dst_ptr] "+r"(dst_ptr), + [width] "+r"(width), + [vl] "=&r"(vl) + : + : "cc", "memory", "z2", "z3", "p0"); + return; + } int vl; asm volatile(