From 15f2ae7d703d4de38cac96710d307f3154e7c3e7 Mon Sep 17 00:00:00 2001 From: George Steed Date: Tue, 27 Aug 2024 17:43:25 +0100 Subject: [PATCH] [AArch64] Add SME impls of ScaleARGBRowDown2{,Linear,Box} Mostly just straightforward copies of the Neon code ported to Streaming-SVE, these follow the same pattern as the prior ScaleRowDown2 and ScaleUVRowDown2 SME kernels, but operating on 32-bit ARGB tuples rather than 8-bit data or 16-bit UV tuples. These is no benefit from this kernel when the SVE vector length is only 128 bits, so skip writing a non-streaming SVE implementation. Change-Id: I15600c2498cc592f5ea1d97b78fafec327de7947 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/6070783 Reviewed-by: Frank Barchard Reviewed-by: Justin Green --- include/libyuv/scale_row.h | 13 ++++ source/scale_argb.cc | 13 ++++ source/scale_sme.cc | 133 +++++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+) diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 3747c318e..7d97f541e 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -128,6 +128,7 @@ extern "C" { #define HAS_SCALEUVROWDOWN2_SME #define HAS_SCALEUVROWDOWN2LINEAR_SME #define HAS_SCALEUVROWDOWN2BOX_SME +#define HAS_SCALEARGBROWDOWN2_SME #endif #if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa) @@ -958,14 +959,26 @@ void ScaleARGBRowDown2_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width); +void ScaleARGBRowDown2_SME(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width); void ScaleARGBRowDown2Linear_NEON(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); +void ScaleARGBRowDown2Linear_SME(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width); void ScaleARGBRowDown2Box_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width); +void ScaleARGBRowDown2Box_SME(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width); void ScaleARGBRowDown2_RVV(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, diff --git a/source/scale_argb.cc b/source/scale_argb.cc index 0268b1afc..66082be8b 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -99,6 +99,14 @@ static void ScaleARGBDown2(int src_width, } } #endif +#if defined(HAS_SCALEARGBROWDOWN2_SME) + if (TestCpuFlag(kCpuHasSME)) { + ScaleARGBRowDown2 = filtering == kFilterNone ? ScaleARGBRowDown2_SME + : filtering == kFilterLinear + ? ScaleARGBRowDown2Linear_SME + : ScaleARGBRowDown2Box_SME; + } +#endif #if defined(HAS_SCALEARGBROWDOWN2_MSA) if (TestCpuFlag(kCpuHasMSA)) { ScaleARGBRowDown2 = @@ -202,6 +210,11 @@ static int ScaleARGBDown4Box(int src_width, } } #endif +#if defined(HAS_SCALEARGBROWDOWN2_SME) + if (TestCpuFlag(kCpuHasSME)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_SME; + } +#endif #if defined(HAS_SCALEARGBROWDOWN2_RVV) if (TestCpuFlag(kCpuHasRVV)) { ScaleARGBRowDown2 = ScaleARGBRowDown2Box_RVV; diff --git a/source/scale_sme.cc b/source/scale_sme.cc index fd364b316..9a4014f3d 100644 --- a/source/scale_sme.cc +++ b/source/scale_sme.cc @@ -283,6 +283,139 @@ __arm_locally_streaming void ScaleUVRowDown2Box_SME(const uint8_t* src_uv, #undef SCALEUVROWDOWN2BOX_SVE +__arm_locally_streaming void ScaleARGBRowDown2_SME(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + // Streaming-SVE only, no use of ZA tile. + (void)src_stride; + int vl; + asm volatile( + "cntw %x[vl] \n" + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "b.lt 2f \n" + + "1: \n" + "ptrue p0.b \n" + "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n" + "incb %[src_argb], all, mul #2 \n" + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "st1w {z1.s}, p0, [%[dst_argb]] \n" + "incb %[dst_argb] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[dst_width], %w[dst_width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[dst_width] \n" + "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n" + "st1w {z1.s}, p0, [%[dst_argb]] \n" + + "99: \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [dst_width] "+r"(dst_width), // %[dst_width] + [vl] "=r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "p0"); +} + +__arm_locally_streaming void ScaleARGBRowDown2Linear_SME( + const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + // Streaming-SVE only, no use of ZA tile. + (void)src_stride; + int vl; + asm volatile( + "cntw %x[vl] \n" + "ptrue p1.b \n" + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.s \n" + "1: \n" + "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n" + "incb %[src_argb], all, mul #2 \n" + "urhadd z0.b, p1/m, z0.b, z1.b \n" + "st1w {z0.s}, p0, [%[dst_argb]] \n" + "incb %[dst_argb], all, mul #1 \n" + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[dst_width], %w[dst_width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[dst_width] \n" + "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n" + "urhadd z0.b, p1/m, z0.b, z1.b \n" + "st1w {z0.s}, p0, [%[dst_argb]] \n" + + "99: \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [dst_width] "+r"(dst_width), // %[dst_width] + [vl] "=r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "p0", "p1"); +} + +#define SCALEARGBROWDOWN2BOX_SVE \ + "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n" \ + "ld2w {z2.s, z3.s}, p0/z, [%[src2_argb]] \n" \ + "incb %[src_argb], all, mul #2 \n" \ + "incb %[src2_argb], all, mul #2 \n" \ + "uaddlb z4.h, z0.b, z1.b \n" \ + "uaddlt z5.h, z0.b, z1.b \n" \ + "uaddlb z6.h, z2.b, z3.b \n" \ + "uaddlt z7.h, z2.b, z3.b \n" \ + "add z4.h, z4.h, z6.h \n" \ + "add z5.h, z5.h, z7.h \n" \ + "rshrnb z0.b, z4.h, #2 \n" \ + "rshrnt z0.b, z5.h, #2 \n" \ + "st1w {z0.s}, p0, [%[dst_argb]] \n" \ + "incb %[dst_argb], all, mul #1 \n" + +__arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + // Streaming-SVE only, no use of ZA tile. + const uint8_t* src2_argb = src_argb + src_stride; + int vl; + asm volatile( + "cntw %x[vl] \n" + "ptrue p1.b \n" + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.s \n" + "1: \n" // + SCALEARGBROWDOWN2BOX_SVE + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[dst_width], %w[dst_width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[dst_width] \n" // + SCALEARGBROWDOWN2BOX_SVE + + "99: \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [src2_argb] "+r"(src2_argb), // %[src2_argb] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [dst_width] "+r"(dst_width), // %[dst_width] + [vl] "=r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0", + "p1"); +} + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__)