From d0da5a32989dc75dba663ab1cc27af3d7b798936 Mon Sep 17 00:00:00 2001
From: George Steed <george.steed@arm.com>
Date: Wed, 17 Apr 2024 10:03:28 +0100
Subject: [PATCH] [AArch64] Add SVE2 implementation of ARGB1555ToARGBRow

Avoiding LD4 and unrolling gives a good perf improvement for the little
core especially.

Observed reduction in runtime relative to the existing Neon code:

Cortex-A510: -69.7%
Cortex-A720:  -7.7%
  Cortex-X2: -41.9%
  Cortex-X4: -14.5%

Bug: libyuv:973
Change-Id: I4b3292fa23a6e866d761dfca035538cb09eba9bc
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/5522315
Reviewed-by: Frank Barchard <fbarchard@chromium.org>
Reviewed-by: Justin Green <greenjustin@google.com>
Commit-Queue: Frank Barchard <fbarchard@chromium.org>
---
 include/libyuv/row.h   |  4 ++++
 source/convert_argb.cc |  5 ++++
 source/row_sve.cc      | 54 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+)

diff --git a/include/libyuv/row.h b/include/libyuv/row.h
index 8b550246f..f1084dd8a 100644
--- a/include/libyuv/row.h
+++ b/include/libyuv/row.h
@@ -575,6 +575,7 @@ extern "C" {
 #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
 #define HAS_ABGRTOUVJROW_SVE2
 #define HAS_ABGRTOUVROW_SVE2
+#define HAS_ARGB1555TOARGBROW_SVE2
 #define HAS_ARGBTORGB565DITHERROW_SVE2
 #define HAS_ARGBTORGB565ROW_SVE2
 #define HAS_ARGBTOUVJROW_SVE2
@@ -3469,6 +3470,9 @@ void RGB565ToARGBRow_LASX(const uint8_t* src_rgb565,
 void ARGB1555ToARGBRow_NEON(const uint8_t* src_argb1555,
                             uint8_t* dst_argb,
                             int width);
+void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
+                            uint8_t* dst_argb,
+                            int width);
 void ARGB1555ToARGBRow_MSA(const uint8_t* src_argb1555,
                            uint8_t* dst_argb,
                            int width);
diff --git a/source/convert_argb.cc b/source/convert_argb.cc
index bc5478184..11c7f3cc4 100644
--- a/source/convert_argb.cc
+++ b/source/convert_argb.cc
@@ -3729,6 +3729,11 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555,
     }
   }
 #endif
+#if defined(HAS_ARGB1555TOARGBROW_SVE2)
+  if (TestCpuFlag(kCpuHasSVE2)) {
+    ARGB1555ToARGBRow = ARGB1555ToARGBRow_SVE2;
+  }
+#endif
 #if defined(HAS_ARGB1555TOARGBROW_MSA)
   if (TestCpuFlag(kCpuHasMSA)) {
     ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_MSA;
diff --git a/source/row_sve.cc b/source/row_sve.cc
index 5b03e5a56..b1d2cfe2f 100644
--- a/source/row_sve.cc
+++ b/source/row_sve.cc
@@ -597,6 +597,60 @@ void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb,
       : "cc", "memory", "z0", "z1", "z3", "z4", "p0");
 }
 
+#define ARGB1555TOARGB                                        \
+  /* Input: z1/z3.h = arrrrrgggggbbbbb */                     \
+  "lsl     z0.h, z1.h, #3          \n" /* rrrgggggbbbbb000 */ \
+  "lsl     z2.h, z3.h, #3          \n" /* rrrgggggbbbbb000 */ \
+  "asr     z1.h, z1.h, #7          \n" /* aaaaaaaarrrrrggg */ \
+  "asr     z3.h, z3.h, #7          \n" /* aaaaaaaarrrrrggg */ \
+  "lsl     z0.b, p0/m, z0.b, z4.b  \n" /* ggggg000bbbbb000 */ \
+  "lsl     z2.b, p0/m, z2.b, z4.b  \n" /* ggggg000bbbbb000 */ \
+  "sri     z1.b, z1.b, #5          \n" /* aaaaaaaarrrrrrrr */ \
+  "sri     z3.b, z3.b, #5          \n" /* aaaaaaaarrrrrrrr */ \
+  "sri     z0.b, z0.b, #5          \n" /* ggggggggbbbbbbbb */ \
+  "sri     z2.b, z2.b, #5          \n" /* ggggggggbbbbbbbb */
+
+void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
+                            uint8_t* dst_argb,
+                            int width) {
+  uint64_t vl;
+  asm("mov     z4.h, #0x0300                           \n"
+      "ptrue   p0.b                                    \n"
+
+      "cnth    %x[vl]                                  \n"
+      "subs    %w[width], %w[width], %w[vl], lsl #1    \n"
+      "b.lt    2f                                      \n"
+
+      "1:                                              \n"
+      "ld1h    {z1.h}, p0/z, [%[src]]                  \n"
+      "ld1h    {z3.h}, p0/z, [%[src], #1, mul vl]      \n"
+      "incb    %[src], all, mul #2                     \n" ARGB1555TOARGB
+      "subs    %w[width], %w[width], %w[vl], lsl #1    \n"
+      "st2h    {z0.h, z1.h}, p0, [%[dst]]              \n"
+      "st2h    {z2.h, z3.h}, p0, [%[dst], #2, mul vl]  \n"
+      "incb    %[dst], all, mul #4                     \n"
+      "b.ge    1b                                      \n"
+
+      "2:                                              \n"
+      "adds    %w[width], %w[width], %w[vl], lsl #1    \n"
+      "b.eq    99f                                     \n"
+
+      "whilelt p1.h, wzr, %w[width]                    \n"
+      "whilelt p2.h, %w[vl], %w[width]                 \n"
+      "ld1h    {z1.h}, p1/z, [%[src]]                  \n"
+      "ld1h    {z3.h}, p2/z, [%[src], #1, mul vl]      \n" ARGB1555TOARGB
+      "st2h    {z0.h, z1.h}, p1, [%[dst]]              \n"
+      "st2h    {z2.h, z3.h}, p2, [%[dst], #2, mul vl]  \n"
+
+      "99:                                             \n"
+      : [src] "+r"(src_argb1555),  // %[src]
+        [dst] "+r"(dst_argb),      // %[dst]
+        [width] "+r"(width),       // %[width]
+        [vl] "=&r"(vl)             // %[vl]
+      :
+      : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2");
+}
+
 #endif  // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
 
 #ifdef __cplusplus