mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2025-12-06 16:56:55 +08:00
b53b27d6bf
This is mostly identical to the existing I444ToARGBRow_SVE implementation, we just need to make sure to load the alpha component rather than hard-coding it to 255. Reduction in runtimes observed compared to the existing Neon code: Cortex-A510: -34.2% Cortex-A720: -17.6% Cortex-X2: -9.6% Bug: libyuv:973 Change-Id: Ief63965f6f1048ea24baf8f4037aabdd184e2925 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/5444425 Commit-Queue: Frank Barchard <fbarchard@chromium.org> Reviewed-by: Frank Barchard <fbarchard@chromium.org>
218 lines
10 KiB
C++
218 lines
10 KiB
C++
/*
|
|
* Copyright 2024 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/row.h"
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
|
|
|
#define READYUV444_SVE \
|
|
"ld1b {z0.h}, p1/z, [%[src_y]] \n" \
|
|
"ld1b {z1.h}, p1/z, [%[src_u]] \n" \
|
|
"ld1b {z2.h}, p1/z, [%[src_v]] \n" \
|
|
"add %[src_y], %[src_y], %[vl] \n" \
|
|
"add %[src_u], %[src_u], %[vl] \n" \
|
|
"add %[src_v], %[src_v], %[vl] \n" \
|
|
"prfm pldl1keep, [%[src_y], 448] \n" \
|
|
"prfm pldl1keep, [%[src_u], 448] \n" \
|
|
"trn1 z0.b, z0.b, z0.b \n" \
|
|
"prfm pldl1keep, [%[src_v], 448] \n"
|
|
|
|
#define READYUV422_SVE \
|
|
"ld1b {z0.h}, p1/z, [%[src_y]] \n" \
|
|
"ld1b {z1.s}, p1/z, [%[src_u]] \n" \
|
|
"ld1b {z2.s}, p1/z, [%[src_v]] \n" \
|
|
"inch %[src_y] \n" \
|
|
"incw %[src_u] \n" \
|
|
"incw %[src_v] \n" \
|
|
"prfm pldl1keep, [%[src_y], 448] \n" \
|
|
"prfm pldl1keep, [%[src_u], 128] \n" \
|
|
"prfm pldl1keep, [%[src_v], 128] \n" \
|
|
"trn1 z0.b, z0.b, z0.b \n" \
|
|
"trn1 z1.h, z1.h, z1.h \n" \
|
|
"trn1 z2.h, z2.h, z2.h \n"
|
|
|
|
#define YUVTORGB_SVE_SETUP \
|
|
"ld1rb {z28.h}, p0/z, [%[kUVCoeff], #0] \n" \
|
|
"ld1rb {z29.h}, p0/z, [%[kUVCoeff], #1] \n" \
|
|
"ld1rb {z30.h}, p0/z, [%[kUVCoeff], #2] \n" \
|
|
"ld1rb {z31.h}, p0/z, [%[kUVCoeff], #3] \n" \
|
|
"ld1rh {z24.h}, p0/z, [%[kRGBCoeffBias], #0] \n" \
|
|
"ld1rh {z25.h}, p0/z, [%[kRGBCoeffBias], #2] \n" \
|
|
"ld1rh {z26.h}, p0/z, [%[kRGBCoeffBias], #4] \n" \
|
|
"ld1rh {z27.h}, p0/z, [%[kRGBCoeffBias], #6] \n"
|
|
|
|
#define I4XXTORGB_SVE \
|
|
"umulh z0.h, z24.h, z0.h \n" /* Y */ \
|
|
"mul z6.h, z30.h, z1.h \n" \
|
|
"mul z4.h, z28.h, z1.h \n" /* DB */ \
|
|
"mul z5.h, z29.h, z2.h \n" /* DR */ \
|
|
"mla z6.h, p0/m, z31.h, z2.h \n" /* DG */ \
|
|
"add z17.h, z0.h, z26.h \n" /* G */ \
|
|
"add z16.h, z0.h, z4.h \n" /* B */ \
|
|
"add z18.h, z0.h, z5.h \n" /* R */ \
|
|
"uqsub z17.h, z17.h, z6.h \n" /* G */ \
|
|
"uqsub z16.h, z16.h, z25.h \n" /* B */ \
|
|
"uqsub z18.h, z18.h, z27.h \n" /* R */
|
|
|
|
// Convert from 2.14 fixed point RGB to 8 bit RGBA, interleaving as BG and RA
|
|
// pairs to allow us to use ST2 for storing rather than ST4.
|
|
#define RGBTORGBA8_SVE \
|
|
"uqshrnb z16.b, z16.h, #6 \n" \
|
|
"uqshrnb z18.b, z18.h, #6 \n" \
|
|
"uqshrnt z16.b, z17.h, #6 \n" \
|
|
"trn1 z17.b, z18.b, z19.b \n"
|
|
|
|
#define YUVTORGB_SVE_REGS \
|
|
"z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", \
|
|
"z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "p0", "p1"
|
|
|
|
void I444ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"cnth %[vl] \n"
|
|
"ptrue p0.b \n" YUVTORGB_SVE_SETUP
|
|
"dup z19.b, #255 \n" /* A */
|
|
"cmp %w[width], %w[vl] \n"
|
|
"b.le 2f \n"
|
|
|
|
// Run bulk of computation with an all-true predicate to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p1.h \n"
|
|
"1: \n" READYUV444_SVE
|
|
I4XXTORGB_SVE RGBTORGBA8_SVE
|
|
"sub %w[width], %w[width], %w[vl] \n"
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
"add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
|
|
"cmp %w[width], %w[vl] \n"
|
|
"b.gt 1b \n"
|
|
|
|
// Calculate a predicate for the final iteration to deal with the tail.
|
|
"2: \n"
|
|
"whilelt p1.h, wzr, %w[width] \n" READYUV444_SVE
|
|
I4XXTORGB_SVE RGBTORGBA8_SVE
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
: [src_y] "+r"(src_y), // %[src_y]
|
|
[src_u] "+r"(src_u), // %[src_u]
|
|
[src_v] "+r"(src_v), // %[src_v]
|
|
[dst_argb] "+r"(dst_argb), // %[dst_argb]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
|
|
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
|
|
: "cc", "memory", YUVTORGB_SVE_REGS);
|
|
}
|
|
|
|
void I422ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"cnth %[vl] \n"
|
|
"ptrue p0.b \n" YUVTORGB_SVE_SETUP
|
|
"dup z19.b, #255 \n" /* A */
|
|
"cmp %w[width], %w[vl] \n"
|
|
"b.le 2f \n"
|
|
|
|
// Run bulk of computation with an all-true predicate to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p1.h \n"
|
|
"sub %w[width], %w[width], %w[vl] \n"
|
|
"1: \n" READYUV422_SVE
|
|
I4XXTORGB_SVE RGBTORGBA8_SVE
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
"add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
|
|
"b.gt 1b \n"
|
|
"add %w[width], %w[width], %w[vl] \n"
|
|
|
|
// Calculate a predicate for the final iteration to deal with the tail.
|
|
"2: \n"
|
|
"whilelt p1.h, wzr, %w[width] \n" READYUV422_SVE
|
|
I4XXTORGB_SVE RGBTORGBA8_SVE
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
: [src_y] "+r"(src_y), // %[src_y]
|
|
[src_u] "+r"(src_u), // %[src_u]
|
|
[src_v] "+r"(src_v), // %[src_v]
|
|
[dst_argb] "+r"(dst_argb), // %[dst_argb]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
|
|
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
|
|
: "cc", "memory", YUVTORGB_SVE_REGS);
|
|
}
|
|
|
|
void I444AlphaToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
const uint8_t* src_a,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm("cnth %[vl] \n"
|
|
"ptrue p0.b \n" YUVTORGB_SVE_SETUP
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with an all-true predicate to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p1.h \n"
|
|
"1: \n" READYUV444_SVE
|
|
"ld1b {z19.h}, p1/z, [%[src_a]] \n" // A
|
|
"add %[src_a], %[src_a], %[vl] \n" I4XXTORGB_SVE
|
|
RGBTORGBA8_SVE
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
"add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate a predicate for the final iteration to deal with the tail.
|
|
"whilelt p1.h, wzr, %w[width] \n" READYUV444_SVE
|
|
"ld1b {z19.h}, p1/z, [%[src_a]] \n" // A
|
|
I4XXTORGB_SVE RGBTORGBA8_SVE
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
|
|
"99: \n"
|
|
: [src_y] "+r"(src_y), // %[src_y]
|
|
[src_u] "+r"(src_u), // %[src_u]
|
|
[src_v] "+r"(src_v), // %[src_v]
|
|
[src_a] "+r"(src_a), // %[src_a]
|
|
[dst_argb] "+r"(dst_argb), // %[dst_argb]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
|
|
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
|
|
: "cc", "memory", YUVTORGB_SVE_REGS);
|
|
}
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|