From 0ddf3f7b903ccb62baf92b68645280fdff532fb8 Mon Sep 17 00:00:00 2001 From: George Steed Date: Thu, 16 May 2024 16:17:46 +0100 Subject: [PATCH] [AArch64] Add SVE2 implementation of I210ToARGBRow Observed reduction in runtime compared to the existing Neon code: Cortex-A510: -34.5% Cortex-A520: -6.5% Cortex-A715: -10.1% Cortex-A720: -13.9% Cortex-X2: -11.9% Cortex-X3: -11.6% Cortex-X4: -9.5% Cortex-X925: -11.5% Bug: b/42280942 Change-Id: Ie97dc3b5efd021ecfea14d4c477cc205191e09c3 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/6023037 Reviewed-by: Frank Barchard --- include/libyuv/row.h | 7 +++++ source/convert_argb.cc | 10 +++++++ source/row_sve.cc | 65 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 202b0b197..5a0c38fcb 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -550,6 +550,7 @@ extern "C" { #define HAS_BGRATOUVROW_SVE2 #define HAS_DIVIDEROW_16_SVE2 #define HAS_HALFFLOATROW_SVE2 +#define HAS_I210TOARGBROW_SVE2 #define HAS_I400TOARGBROW_SVE2 #define HAS_I422ALPHATOARGBROW_SVE2 #define HAS_I422TOARGB1555ROW_SVE2 @@ -1095,6 +1096,12 @@ void I210ToARGBRow_NEON(const uint16_t* src_y, uint8_t* rgb_buf, const struct YuvConstants* yuvconstants, int width); +void I210ToARGBRow_SVE2(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* rgb_buf, + const struct YuvConstants* yuvconstants, + int width); void I410ToARGBRow_NEON(const uint16_t* src_y, const uint16_t* src_u, const uint16_t* src_v, diff --git a/source/convert_argb.cc b/source/convert_argb.cc index 33f654595..fde7a304f 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -1444,6 +1444,11 @@ int I010ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I210TOARGBROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + I210ToARGBRow = I210ToARGBRow_SVE2; + } +#endif #if defined(HAS_I210TOARGBROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { I210ToARGBRow = I210ToARGBRow_Any_AVX2; @@ -1680,6 +1685,11 @@ int I210ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I210TOARGBROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + I210ToARGBRow = I210ToARGBRow_SVE2; + } +#endif #if defined(HAS_I210TOARGBROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { I210ToARGBRow = I210ToARGBRow_Any_AVX2; diff --git a/source/row_sve.cc b/source/row_sve.cc index 649d6b2a4..1f02d5430 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -78,6 +78,23 @@ extern "C" { "trn1 z0.b, z0.b, z0.b \n" /* YYYY */ \ "tbl z1.b, {z1.b}, z22.b \n" /* UVUV */ +#define READI210_SVE \ + "ld1h {z3.h}, p1/z, [%[src_y]] \n" \ + "lsl z0.h, z3.h, #6 \n" \ + "usra z0.h, z3.h, #4 \n" \ + "ld1h {z1.s}, p1/z, [%[src_u]] \n" \ + "ld1h {z2.s}, p1/z, [%[src_v]] \n" \ + "incb %[src_y] \n" \ + "inch %[src_u] \n" \ + "inch %[src_v] \n" \ + "prfm pldl1keep, [%[src_y], 448] \n" \ + "prfm pldl1keep, [%[src_u], 128] \n" \ + "prfm pldl1keep, [%[src_v], 128] \n" \ + "trn1 z1.h, z1.h, z1.h \n" \ + "trn1 z2.h, z2.h, z2.h \n" \ + "uqshrnb z1.b, z1.h, #2 \n" \ + "uqshrnb z2.b, z2.h, #2 \n" + #define READP210_SVE \ "ld1h {z0.h}, p1/z, [%[src_y]] \n" \ "ld1h {z1.h}, p2/z, [%[src_uv]] \n" \ @@ -1810,6 +1827,54 @@ void HalfFloat1Row_SVE2(const uint16_t* src, : "cc", "memory", "z0", "z1", "p0", "p1"); } +void I210ToARGBRow_SVE2(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uint64_t vl; + asm("cnth %0" : "=r"(vl)); + int width_last_y = width & (vl - 1); + width_last_y = width_last_y == 0 ? vl : width_last_y; + asm volatile( + "ptrue p0.b \n" YUVTORGB_SVE_SETUP + "dup z19.b, #255 \n" // A + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + // Run bulk of computation with an all-true predicate to avoid predicate + // generation overhead. + "ptrue p1.h \n" + "1: \n" // + READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE + "subs %w[width], %w[width], %w[vl] \n" + "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + // Calculate a predicate for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width_last_y] \n" // + READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE + "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + + "99: \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] + [width_last_y] "r"(width_last_y) // %[width_last_y] + : "cc", "memory", YUVTORGB_SVE_REGS); +} + // P210 has 10 bits in msb of 16 bit NV12 style layout. void P210ToARGBRow_SVE2(const uint16_t* src_y, const uint16_t* src_uv,