mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
47ba8e2160
Because each 8x8 SSIM window spans two 4x4 tiles horizontally, num_windows = (width - 5) >> 2, so aligned widths that are multiples of 64 (e.g. 64, 128, 640, 1280, 1920) always have remainder 15 (num_windows & 15 == 15). - Update SsimWindowRow_AVX512BW (compare_gcc.cc and compare_win.cc) to handle any remainder (1..15 windows) via k-masking (load_mask for the (rem + 1) * 4 tile words and win_mask for the rem window scores) and set win_simd_size = 1. - Add SsimWindowRow_SVE2, SsimWindowRow_SME, and SsimWindowRow_RVV with win_simd_size = 1 using predicated/vl-controlled loops so no C code (Ssim4x4Row_C or SsimWindowRow_C) is executed on AVX-512BW, SVE2, SME, or RVV CPUs for any image dimensions. - Remove LIBYUV_TARGET_AVX512BW __attribute__((target(...))) from compare_gcc.cc and GCC_HAS_AVX512 from cpu_support.h, using %%zmm0 directly in inline assembly. SSIM Performance (Before vs After): uArch / Target Before After Speedup -------------------------- --------- --------- ------- Coral RVV Verilator (64x16) 5302 cyc 2219 cyc 2.39x Coral RVV WindowRow (15w) 1854 cyc 308 cyc 6.02x Apple M6 SME (64x36 10kx) 16.41 ms 4.69 ms 3.50x Apple M6 SME (720p 1000x) 350.76 ms 66.09 ms 5.31x R=rrwinterton@gmail.com Bug: libyuv:352521931 Change-Id: Ic7d55184c60611af4a57d113cc4ee3d211debadf Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8621918 Commit-Queue: Frank Barchard <fbarchard@google.com> Reviewed-by: richard winterton <rrwinterton@gmail.com>
1760 lines
79 KiB
C++
1760 lines
79 KiB
C++
/*
|
|
* Copyright 2024 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/row_sve.h"
|
|
#include "libyuv/compare_row.h"
|
|
#include "libyuv/row.h"
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
// Note: every register written by an inline asm block (vector, general
|
|
// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
|
|
// listed in its clobber list, even when the ABI treats it as caller-saved.
|
|
// With LTO/LTCG the compiler may keep values live in any register across the
|
|
// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
|
|
// are callee-saved. List "memory" first, then "cc", then registers.
|
|
|
|
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
|
|
|
#define RGBTOARGB8_SVE_2X \
|
|
/* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
|
|
"uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
|
|
"uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \
|
|
"uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
|
|
"uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \
|
|
"uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \
|
|
"uqshrnt z18.b, z22.h, #6 \n" /* R1 */
|
|
|
|
#define RGBTOARGB8_SVE_TOP_2X \
|
|
/* Inputs: B: z16.h, G: z17.h, R: z18.h */ \
|
|
"uqshl z16.h, p0/m, z16.h, #2 \n" /* B0 */ \
|
|
"uqshl z17.h, p0/m, z17.h, #2 \n" /* G0 */ \
|
|
"uqshl z18.h, p0/m, z18.h, #2 \n" /* R0 */ \
|
|
"uqshl z20.h, p0/m, z20.h, #2 \n" /* B1 */ \
|
|
"uqshl z21.h, p0/m, z21.h, #2 \n" /* G1 */ \
|
|
"uqshl z22.h, p0/m, z22.h, #2 \n" /* R1 */
|
|
|
|
void I444ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"cnth %[vl] \n"
|
|
"ptrue p0.b \n" //
|
|
YUVTORGB_SVE_SETUP
|
|
"dup z19.b, #255 \n" // Alpha
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with an all-true predicate to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p1.h \n"
|
|
"1: \n" //
|
|
READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
"add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate a predicate for the final iteration to deal with the tail.
|
|
"whilelt p1.h, wzr, %w[width] \n" //
|
|
READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
|
|
"99: \n"
|
|
: [src_y] "+r"(src_y), // %[src_y]
|
|
[src_u] "+r"(src_u), // %[src_u]
|
|
[src_v] "+r"(src_v), // %[src_v]
|
|
[dst_argb] "+r"(dst_argb), // %[dst_argb]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
|
|
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
|
|
: "cc", "memory", YUVTORGB_SVE_REGS);
|
|
}
|
|
|
|
void I444ToRGB24Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I444ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_rgb24, yuvconstants, width);
|
|
}
|
|
|
|
void I400ToARGBRow_SVE2(const uint8_t* src_y,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I400ToARGBRow_SVE_SC(src_y, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToRGB24Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToRGB565Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb565,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToRGB565Row_SVE_SC(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToARGB1555Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb1555,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToARGB1555Row_SVE_SC(src_y, src_u, src_v, dst_argb1555, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I422ToARGB4444Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb4444,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToARGB4444Row_SVE_SC(src_y, src_u, src_v, dst_argb4444, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I422ToRGBARow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgba,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToRGBARow_SVE_SC(src_y, src_u, src_v, dst_rgba, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToAR30Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I422AlphaToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
const uint8_t* src_a,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I444AlphaToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
const uint8_t* src_a,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I444AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void NV12ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_uv,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
NV12ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void NV21ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_vu,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
NV21ToARGBRow_SVE_SC(src_y, src_vu, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void NV12ToRGB24Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_uv,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
NV12ToRGB24Row_SVE_SC(src_y, src_uv, dst_rgb24, yuvconstants, width);
|
|
}
|
|
|
|
void NV21ToRGB24Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_vu,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
NV21ToRGB24Row_SVE_SC(src_y, src_vu, dst_rgb24, yuvconstants, width);
|
|
}
|
|
|
|
void ARGBToUVMatrixRow_SVE2(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
int8_t uvconstants[8] = {(int8_t)c->kRGBToU[0], (int8_t)c->kRGBToU[1],
|
|
(int8_t)c->kRGBToU[2], (int8_t)c->kRGBToU[3],
|
|
(int8_t)c->kRGBToV[0], (int8_t)c->kRGBToV[1],
|
|
(int8_t)c->kRGBToV[2], (int8_t)c->kRGBToV[3]};
|
|
ARGBToUVMatrixRow_SVE_SC(src_argb, src_stride_argb, dst_u, dst_v, width,
|
|
uvconstants);
|
|
}
|
|
|
|
void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb,
|
|
uint8_t* dst_y,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c);
|
|
}
|
|
|
|
#define ARGBTORGB565_SVE \
|
|
/* Inputs: \
|
|
* z0: rrrrrxxxbbbbbxxx \
|
|
* z1: xxxxxxxxggggggxx \
|
|
* z3: 0000000000000011 (3, 0, 3, 0, ...) \
|
|
* z4: 0000011111100000 \
|
|
*/ \
|
|
"lsr z0.b, p0/m, z0.b, z3.b \n" \
|
|
"lsl z1.h, z1.h, #3 \n" \
|
|
"bsl z1.d, z1.d, z0.d, z4.d \n"
|
|
|
|
void ARGBToRGB565Row_SVE2(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
int width) {
|
|
unsigned bsl_mask = 0x7e0;
|
|
uint64_t vl;
|
|
width *= 2;
|
|
asm volatile(
|
|
"mov z3.h, #3 \n"
|
|
"dup z4.h, %w[bsl_mask] \n"
|
|
|
|
"cntb %[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
|
|
"incb %[src], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n" //
|
|
ARGBTORGB565_SVE
|
|
"st1b {z1.b}, p0, [%[dst]] \n"
|
|
"incb %[dst] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
|
|
ARGBTORGB565_SVE
|
|
"st1b {z1.b}, p0, [%[dst]] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_argb), // %[src]
|
|
[dst] "+r"(dst_rgb), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [bsl_mask] "r"(bsl_mask) // %[bsl_mask]
|
|
: "cc", "memory", "z0", "z1", "z3", "z4", "p0");
|
|
}
|
|
|
|
void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
uint32_t dither4,
|
|
int width) {
|
|
unsigned bsl_mask = 0x7e0;
|
|
uint64_t vl;
|
|
width *= 2;
|
|
asm volatile(
|
|
"mov z3.h, #3 \n"
|
|
"dup z4.h, %w[bsl_mask] \n"
|
|
"dup z2.s, %w[dither4] \n"
|
|
"zip1 z2.b, z2.b, z2.b \n"
|
|
|
|
"cntb %[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
|
|
"incb %[src], all, mul #2 \n"
|
|
"uqadd z0.b, z0.b, z2.b \n"
|
|
"uqadd z1.b, z1.b, z2.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n" //
|
|
ARGBTORGB565_SVE
|
|
"st1b {z1.b}, p0, [%[dst]] \n"
|
|
"incb %[dst] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
|
|
"uqadd z0.b, z0.b, z2.b \n"
|
|
"uqadd z1.b, z1.b, z2.b \n" //
|
|
ARGBTORGB565_SVE
|
|
"st1b {z1.b}, p0, [%[dst]] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_argb), // %[src]
|
|
[dst] "+r"(dst_rgb), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [bsl_mask] "r"(bsl_mask), // %[bsl_mask]
|
|
[dither4] "r"(dither4) // %[dither4]
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
|
|
}
|
|
|
|
#define ARGB1555TOARGB \
|
|
/* Input: z1/z3.h = arrrrrgggggbbbbb */ \
|
|
"lsl z0.h, z1.h, #3 \n" /* rrrgggggbbbbb000 */ \
|
|
"lsl z2.h, z3.h, #3 \n" /* rrrgggggbbbbb000 */ \
|
|
"asr z1.h, z1.h, #7 \n" /* aaaaaaaarrrrrggg */ \
|
|
"asr z3.h, z3.h, #7 \n" /* aaaaaaaarrrrrggg */ \
|
|
"lsl z0.b, p0/m, z0.b, z4.b \n" /* ggggg000bbbbb000 */ \
|
|
"lsl z2.b, p0/m, z2.b, z4.b \n" /* ggggg000bbbbb000 */ \
|
|
"sri z1.b, z1.b, #5 \n" /* aaaaaaaarrrrrrrr */ \
|
|
"sri z3.b, z3.b, #5 \n" /* aaaaaaaarrrrrrrr */ \
|
|
"sri z0.b, z0.b, #5 \n" /* ggggggggbbbbbbbb */ \
|
|
"sri z2.b, z2.b, #5 \n" /* ggggggggbbbbbbbb */
|
|
|
|
void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"mov z4.h, #0x0300 \n"
|
|
"ptrue p0.b \n"
|
|
|
|
"cnth %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
"1: \n"
|
|
"ld1h {z1.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z3.h}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n" //
|
|
ARGB1555TOARGB
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"st2h {z0.h, z1.h}, p0, [%[dst]] \n"
|
|
"st2h {z2.h, z3.h}, p0, [%[dst], #2, mul vl] \n"
|
|
"incb %[dst], all, mul #4 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p1.h, wzr, %w[width] \n"
|
|
"whilelt p2.h, %w[vl], %w[width] \n"
|
|
"ld1h {z1.h}, p1/z, [%[src]] \n"
|
|
"ld1h {z3.h}, p2/z, [%[src], #1, mul vl] \n" //
|
|
ARGB1555TOARGB
|
|
"st2h {z0.h, z1.h}, p1, [%[dst]] \n"
|
|
"st2h {z2.h, z3.h}, p2, [%[dst], #2, mul vl] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_argb1555), // %[src]
|
|
[dst] "+r"(dst_argb), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2");
|
|
}
|
|
|
|
// clang-format off
|
|
#define AYUVTOUV_SVE(zU0, zV0, zU1, zV1) /* e.g. */ \
|
|
"ld2h {z0.h, z1.h}, p0/z, [%[src0]] \n" /* VUVU.. YAYA.. */ \
|
|
"ld2h {z1.h, z2.h}, p1/z, [%[src0], #2, mul vl] \n" /* VUVU.. YAYA.. */ \
|
|
"ld2h {z2.h, z3.h}, p0/z, [%[src1]] \n" /* VUVU.. YAYA.. */ \
|
|
"ld2h {z3.h, z4.h}, p1/z, [%[src1], #2, mul vl] \n" /* VUVU.. YAYA.. */ \
|
|
"incb %[src0], all, mul #4 \n" \
|
|
"incb %[src1], all, mul #4 \n" \
|
|
"uaddlb z4.h, z0.b, z2.b \n" /* V */ \
|
|
"uaddlt z5.h, z0.b, z2.b \n" /* U */ \
|
|
"uaddlb z6.h, z1.b, z3.b \n" /* V */ \
|
|
"uaddlt z7.h, z1.b, z3.b \n" /* U */ \
|
|
"addp " #zU0 ".h, p0/m, " #zU0 ".h, " #zV0 ".h \n" /* UV */ \
|
|
"addp " #zU1 ".h, p1/m, " #zU1 ".h, " #zV1 ".h \n" /* UV */ \
|
|
"subs %w[width], %w[width], %w[vl] \n" \
|
|
"urshr " #zU0 ".h, p0/m, " #zU0 ".h, #2 \n" /* U0V0 */ \
|
|
"urshr " #zU1 ".h, p1/m, " #zU1 ".h, #2 \n" /* U0V0 */ \
|
|
"st1b {" #zU0 ".h}, p0, [%[dst]] \n" \
|
|
"st1b {" #zU1 ".h}, p1, [%[dst], #1, mul vl] \n" \
|
|
"incb %[dst] \n"
|
|
// clang-format on
|
|
|
|
// Filter 2 rows of AYUV UV's (444) into UV (420).
|
|
// AYUV is VUYA in memory. UV for NV12 is UV order in memory.
|
|
void AYUVToUVRow_SVE2(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
// Output a row of UV values, filtering 2x2 rows of AYUV.
|
|
const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
|
|
int vl;
|
|
asm volatile (
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"ptrue p1.h \n"
|
|
"1: \n"
|
|
AYUVTOUV_SVE(z5, z4, z7, z6)
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"cnth %x[vl] \n"
|
|
"whilelt p0.h, wzr, %w[width] \n" // first row
|
|
"whilelt p1.h, %w[vl], %w[width] \n" // second row
|
|
AYUVTOUV_SVE(z5, z4, z7, z6)
|
|
|
|
"99: \n"
|
|
: [src0]"+r"(src_ayuv), // %[src0]
|
|
[src1]"+r"(src_ayuv1), // %[src1]
|
|
[dst]"+r"(dst_uv), // %[dst]
|
|
[width]"+r"(width), // %[width]
|
|
[vl]"=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
|
|
"p1");
|
|
}
|
|
|
|
// Filter 2 rows of AYUV UV's (444) into VU (420).
|
|
void AYUVToVURow_SVE2(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_vu,
|
|
int width) {
|
|
// Output a row of VU values, filtering 2x2 rows of AYUV.
|
|
const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
|
|
int vl;
|
|
asm volatile (
|
|
"cntb %x[vl] \n"
|
|
"cmp %w[width], %w[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"ptrue p1.h \n"
|
|
"1: \n"
|
|
AYUVTOUV_SVE(z4, z5, z6, z7)
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"cnth %x[vl] \n"
|
|
"whilelt p0.h, wzr, %w[width] \n" // first row
|
|
"whilelt p1.h, %w[vl], %w[width] \n" // second row
|
|
AYUVTOUV_SVE(z4, z5, z6, z7)
|
|
|
|
"99: \n"
|
|
: [src0]"+r"(src_ayuv), // %[src0]
|
|
[src1]"+r"(src_ayuv1), // %[src1]
|
|
[dst]"+r"(dst_vu), // %[dst]
|
|
[width]"+r"(width), // %[width]
|
|
[vl]"=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
|
|
"p1");
|
|
}
|
|
|
|
void YUY2ToARGBRow_SVE2(const uint8_t* src_yuy2,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUY2ToARGBRow_SVE_SC(src_yuy2, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void UYVYToARGBRow_SVE2(const uint8_t* src_uyvy,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
static inline void RAWToWXYZRow_SVE2(const uint8_t* src_raw,
|
|
uint8_t* dst_wxyz,
|
|
int width,
|
|
uint32_t idx_start,
|
|
uint32_t idx_step,
|
|
uint32_t alpha) {
|
|
uint32_t vl;
|
|
asm("cntw %x0" : "=r"(vl));
|
|
uint32_t vl_mul3 = vl * 3;
|
|
uint32_t rem_mul3;
|
|
asm volatile(
|
|
"index z31.s, %w[idx_start], %w[idx_step] \n"
|
|
"dup z30.s, %w[alpha] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with the same predicates to avoid predicate
|
|
// generation overhead. We set up p1 to only load 3/4 of a vector.
|
|
"ptrue p0.s \n"
|
|
"whilelt p1.b, wzr, %w[vl_mul3] \n"
|
|
"1: \n"
|
|
"ld1b {z0.b}, p1/z, [%[src]] \n"
|
|
"add %[src], %[src], %x[vl_mul3] \n"
|
|
"ld1b {z1.b}, p1/z, [%[src]] \n"
|
|
"add %[src], %[src], %x[vl_mul3] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"tbl z1.b, {z1.b}, z31.b \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"orr z0.d, z0.d, z30.d \n"
|
|
"orr z1.d, z1.d, z30.d \n"
|
|
"st1w {z0.s}, p0, [%[dst]] \n"
|
|
"st1w {z1.s}, p0, [%[dst], #1, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate a pair of predicates for the final iteration to deal with
|
|
// the tail.
|
|
"3: \n"
|
|
"add %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
|
|
"whilelt p0.s, wzr, %w[width] \n"
|
|
"whilelt p1.b, wzr, %w[rem_mul3] \n"
|
|
"ld1b {z0.b}, p1/z, [%[src]] \n"
|
|
"add %[src], %[src], %x[vl_mul3] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"orr z0.d, z0.d, z30.d \n"
|
|
"st1w {z0.s}, p0, [%[dst]] \n"
|
|
"incb %[dst] \n"
|
|
"b.gt 3b \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_raw), // %[src]
|
|
[dst] "+r"(dst_wxyz), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl_mul3] "+r"(vl_mul3), // %[vl_mul3]
|
|
[rem_mul3] "=&r"(rem_mul3) // %[rem_mul3]
|
|
: [idx_start] "r"(idx_start), // %[idx_start]
|
|
[idx_step] "r"(idx_step), // %[idx_step]
|
|
[alpha] "r"(alpha), // %[alpha]
|
|
[vl] "r"(vl) // %[vl]
|
|
: "cc", "memory", "z0", "z1", "z30", "z31", "p0", "p1");
|
|
}
|
|
|
|
void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
|
|
RAWToWXYZRow_SVE2(src_raw, dst_argb, width, 0xff000102U, 0x00030303U,
|
|
0xff000000U);
|
|
}
|
|
|
|
void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
|
|
RAWToWXYZRow_SVE2(src_raw, dst_rgba, width, 0x000102ffU, 0x03030300U,
|
|
0x000000ffU);
|
|
}
|
|
|
|
void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
RAWToWXYZRow_SVE2(src_rgb24, dst_argb, width, 0xff020100U, 0x00030303U,
|
|
0xff000000U);
|
|
}
|
|
|
|
static const uint8_t kRAWToRGB24Indices[] = {
|
|
2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12,
|
|
17, 16, 15, 20, 19, 18, 23, 22, 21, 26, 25, 24, 29, 28, 27,
|
|
32, 31, 30, 35, 34, 33, 38, 37, 36, 41, 40, 39, 44, 43, 42,
|
|
47, 46, 45, 50, 49, 48, 53, 52, 51, 56, 55, 54, 59, 58, 57,
|
|
62, 61, 60, 65, 64, 63, 68, 67, 66, 71, 70, 69, 74, 73, 72,
|
|
77, 76, 75, 80, 79, 78, 83, 82, 81, 86, 85, 84, 89, 88, 87,
|
|
92, 91, 90, 95, 94, 93, 98, 97, 96, 101, 100, 99, 104, 103, 102,
|
|
107, 106, 105, 110, 109, 108, 113, 112, 111, 116, 115, 114, 119, 118, 117,
|
|
122, 121, 120, 125, 124, 123, 128, 127, 126, 131, 130, 129, 134, 133, 132,
|
|
137, 136, 135, 140, 139, 138, 143, 142, 141, 146, 145, 144, 149, 148, 147,
|
|
152, 151, 150, 155, 154, 153, 158, 157, 156, 161, 160, 159, 164, 163, 162,
|
|
167, 166, 165, 170, 169, 168, 173, 172, 171, 176, 175, 174, 179, 178, 177,
|
|
182, 181, 180, 185, 184, 183, 188, 187, 186, 191, 190, 189, 194, 193, 192,
|
|
197, 196, 195, 200, 199, 198, 203, 202, 201, 206, 205, 204, 209, 208, 207,
|
|
212, 211, 210, 215, 214, 213, 218, 217, 216, 221, 220, 219, 224, 223, 222,
|
|
227, 226, 225, 230, 229, 228, 233, 232, 231, 236, 235, 234, 239, 238, 237,
|
|
242, 241, 240, 245, 244, 243, 248, 247, 246, 251, 250, 249, 254, 253, 252};
|
|
|
|
void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
|
|
// width is in elements, convert to bytes.
|
|
width *= 3;
|
|
// we use the mul3 predicate pattern throughout to use the largest multiple
|
|
// of three number of lanes, for instance with a vector length of 16 bytes
|
|
// only the first 15 bytes will be used for load/store instructions.
|
|
uint32_t vl;
|
|
asm volatile(
|
|
"cntb %x[vl], mul3 \n"
|
|
"ptrue p0.b, mul3 \n"
|
|
"ld1b {z31.b}, p0/z, [%[kIndices]] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with the same predicate to avoid predicate
|
|
// generation overhead.
|
|
"1: \n"
|
|
"ld1b {z0.b}, p0/z, [%[src]] \n"
|
|
"add %[src], %[src], %x[vl] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z0.b}, p0, [%[dst]] \n"
|
|
"add %[dst], %[dst], %x[vl] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate a predicate for the final iteration to deal with the tail.
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z0.b}, p0/z, [%[src]] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"st1b {z0.b}, p0, [%[dst]] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_raw), // %[src]
|
|
[dst] "+r"(dst_rgb24), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [kIndices] "r"(kRAWToRGB24Indices) // %[kIndices]
|
|
: "cc", "memory", "z0", "z31", "p0");
|
|
}
|
|
|
|
static inline void ARGBToXYZRow_SVE2(const uint8_t* src_argb,
|
|
uint8_t* dst_xyz,
|
|
int width,
|
|
const uint8_t* indices) {
|
|
uint32_t vl;
|
|
asm("cntw %x0" : "=r"(vl));
|
|
uint32_t vl_mul3 = vl * 3;
|
|
uint32_t rem_mul3;
|
|
asm volatile(
|
|
"whilelt p1.b, wzr, %w[vl_mul3] \n"
|
|
"ld1b {z31.b}, p1/z, [%[indices]] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with the same predicates to avoid predicate
|
|
// generation overhead. We set up p1 to only store 3/4 of a vector.
|
|
"ptrue p0.s \n"
|
|
"1: \n"
|
|
"ld1w {z0.s}, p0/z, [%[src]] \n"
|
|
"ld1w {z1.s}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"tbl z1.b, {z1.b}, z31.b \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"st1b {z0.b}, p1, [%[dst]] \n"
|
|
"add %[dst], %[dst], %x[vl_mul3] \n"
|
|
"st1b {z1.b}, p1, [%[dst]] \n"
|
|
"add %[dst], %[dst], %x[vl_mul3] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate predicates for the final iteration to deal with the tail.
|
|
"add %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
|
|
"whilelt p0.s, wzr, %w[width] \n"
|
|
"whilelt p1.b, wzr, %w[rem_mul3] \n"
|
|
"whilelt p2.s, %w[vl], %w[width] \n"
|
|
"whilelt p3.b, %w[vl_mul3], %w[rem_mul3] \n"
|
|
"ld1w {z0.s}, p0/z, [%[src]] \n"
|
|
"ld1w {z1.s}, p2/z, [%[src], #1, mul vl] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"tbl z1.b, {z1.b}, z31.b \n"
|
|
"st1b {z0.b}, p1, [%[dst]] \n"
|
|
"add %[dst], %[dst], %x[vl_mul3] \n"
|
|
"st1b {z1.b}, p3, [%[dst]] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_argb), // %[src]
|
|
[dst] "+r"(dst_xyz), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[rem_mul3] "=&r"(rem_mul3) // %[rem_mul3]
|
|
: [indices] "r"(indices), // %[indices]
|
|
[vl_mul3] "r"(vl_mul3), // %[vl_mul3]
|
|
[vl] "r"(vl) // %[vl]
|
|
: "cc", "memory", "z0", "z1", "z31", "p0", "p1", "p2", "p3");
|
|
}
|
|
|
|
static const uint8_t kARGBToRGB24RowIndices[] = {
|
|
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18,
|
|
20, 21, 22, 24, 25, 26, 28, 29, 30, 32, 33, 34, 36, 37, 38,
|
|
40, 41, 42, 44, 45, 46, 48, 49, 50, 52, 53, 54, 56, 57, 58,
|
|
60, 61, 62, 64, 65, 66, 68, 69, 70, 72, 73, 74, 76, 77, 78,
|
|
80, 81, 82, 84, 85, 86, 88, 89, 90, 92, 93, 94, 96, 97, 98,
|
|
100, 101, 102, 104, 105, 106, 108, 109, 110, 112, 113, 114, 116, 117, 118,
|
|
120, 121, 122, 124, 125, 126, 128, 129, 130, 132, 133, 134, 136, 137, 138,
|
|
140, 141, 142, 144, 145, 146, 148, 149, 150, 152, 153, 154, 156, 157, 158,
|
|
160, 161, 162, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178,
|
|
180, 181, 182, 184, 185, 186, 188, 189, 190, 192, 193, 194, 196, 197, 198,
|
|
200, 201, 202, 204, 205, 206, 208, 209, 210, 212, 213, 214, 216, 217, 218,
|
|
220, 221, 222, 224, 225, 226, 228, 229, 230, 232, 233, 234, 236, 237, 238,
|
|
240, 241, 242, 244, 245, 246, 248, 249, 250, 252, 253, 254,
|
|
};
|
|
|
|
static const uint8_t kARGBToRAWRowIndices[] = {
|
|
2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 18, 17, 16,
|
|
22, 21, 20, 26, 25, 24, 30, 29, 28, 34, 33, 32, 38, 37, 36,
|
|
42, 41, 40, 46, 45, 44, 50, 49, 48, 54, 53, 52, 58, 57, 56,
|
|
62, 61, 60, 66, 65, 64, 70, 69, 68, 74, 73, 72, 78, 77, 76,
|
|
82, 81, 80, 86, 85, 84, 90, 89, 88, 94, 93, 92, 98, 97, 96,
|
|
102, 101, 100, 106, 105, 104, 110, 109, 108, 114, 113, 112, 118, 117, 116,
|
|
122, 121, 120, 126, 125, 124, 130, 129, 128, 134, 133, 132, 138, 137, 136,
|
|
142, 141, 140, 146, 145, 144, 150, 149, 148, 154, 153, 152, 158, 157, 156,
|
|
162, 161, 160, 166, 165, 164, 170, 169, 168, 174, 173, 172, 178, 177, 176,
|
|
182, 181, 180, 186, 185, 184, 190, 189, 188, 194, 193, 192, 198, 197, 196,
|
|
202, 201, 200, 206, 205, 204, 210, 209, 208, 214, 213, 212, 218, 217, 216,
|
|
222, 221, 220, 226, 225, 224, 230, 229, 228, 234, 233, 232, 238, 237, 236,
|
|
242, 241, 240, 246, 245, 244, 250, 249, 248, 254, 253, 252,
|
|
};
|
|
|
|
void ARGBToRGB24Row_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
|
|
ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRGB24RowIndices);
|
|
}
|
|
|
|
void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
|
|
ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRAWRowIndices);
|
|
}
|
|
|
|
void DivideRow_16_SVE2(const uint16_t* src_y,
|
|
uint16_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"dup z0.h, %w[scale] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with the same predicates to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z1.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n"
|
|
"umulh z1.h, z1.h, z0.h \n"
|
|
"umulh z2.h, z2.h, z0.h \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"st1h {z1.h}, p0, [%[dst]] \n"
|
|
"st1h {z2.h}, p0, [%[dst], #1, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate a pair of predicates for the final iteration to deal with
|
|
// the tail.
|
|
"whilelt p0.h, wzr, %w[width] \n"
|
|
"whilelt p1.h, %w[vl], %w[width] \n"
|
|
"ld1h {z1.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z2.h}, p1/z, [%[src], #1, mul vl] \n"
|
|
"umulh z1.h, z1.h, z0.h \n"
|
|
"umulh z2.h, z2.h, z0.h \n"
|
|
"st1h {z1.h}, p0, [%[dst]] \n"
|
|
"st1h {z2.h}, p1, [%[dst], #1, mul vl] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_y), // %[src]
|
|
[dst] "+r"(dst_y), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [scale] "r"(scale) // %[scale]
|
|
: "cc", "memory", "z0", "z1", "z2", "p0", "p1");
|
|
}
|
|
|
|
#define HALFFLOAT_SVE \
|
|
"scvtf z0.s, p0/m, z0.s \n" \
|
|
"scvtf z1.s, p0/m, z1.s \n" \
|
|
"scvtf z2.s, p0/m, z2.s \n" \
|
|
"scvtf z3.s, p0/m, z3.s \n" \
|
|
"fmul z0.s, z0.s, z4.s \n" \
|
|
"fmul z1.s, z1.s, z4.s \n" \
|
|
"fmul z2.s, z2.s, z4.s \n" \
|
|
"fmul z3.s, z3.s, z4.s \n" \
|
|
"fmul z0.s, z0.s, z5.s \n" \
|
|
"fmul z1.s, z1.s, z5.s \n" \
|
|
"fmul z2.s, z2.s, z5.s \n" \
|
|
"fmul z3.s, z3.s, z5.s \n" \
|
|
"uqshrnb z0.h, z0.s, #13 \n" \
|
|
"uqshrnb z1.h, z1.s, #13 \n" \
|
|
"uqshrnb z2.h, z2.s, #13 \n" \
|
|
"uqshrnb z3.h, z3.s, #13 \n"
|
|
|
|
void HalfFloatRow_SVE2(const uint16_t* src,
|
|
uint16_t* dst,
|
|
float scale,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm("cntw %x0" : "=r"(vl));
|
|
asm volatile(
|
|
"mov z4.s, %s[scale] \n"
|
|
"mov z5.s, %s[bias] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #2 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with all-true predicates to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p0.s \n"
|
|
"1: \n"
|
|
"ld1h {z0.s}, p0/z, [%[src]] \n"
|
|
"ld1h {z1.s}, p0/z, [%[src], #1, mul vl] \n"
|
|
"ld1h {z2.s}, p0/z, [%[src], #2, mul vl] \n"
|
|
"ld1h {z3.s}, p0/z, [%[src], #3, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n" //
|
|
HALFFLOAT_SVE
|
|
"subs %w[width], %w[width], %w[vl], lsl #2 \n"
|
|
"st1h {z0.s}, p0, [%[dst]] \n"
|
|
"st1h {z1.s}, p0, [%[dst], #1, mul vl] \n"
|
|
"st1h {z2.s}, p0, [%[dst], #2, mul vl] \n"
|
|
"st1h {z3.s}, p0, [%[dst], #3, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #2 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate predicates for the final iteration to deal with the tail.
|
|
"whilelt p0.s, wzr, %w[width] \n"
|
|
"whilelt p1.s, %w[vl], %w[width] \n"
|
|
"whilelt p2.s, %w[vl2], %w[width] \n"
|
|
"whilelt p3.s, %w[vl3], %w[width] \n"
|
|
"ld1h {z0.s}, p0/z, [%[src]] \n"
|
|
"ld1h {z1.s}, p1/z, [%[src], #1, mul vl] \n"
|
|
"ld1h {z2.s}, p2/z, [%[src], #2, mul vl] \n"
|
|
"ld1h {z3.s}, p3/z, [%[src], #3, mul vl] \n" //
|
|
HALFFLOAT_SVE
|
|
"st1h {z0.s}, p0, [%[dst]] \n"
|
|
"st1h {z1.s}, p1, [%[dst], #1, mul vl] \n"
|
|
"st1h {z2.s}, p2, [%[dst], #2, mul vl] \n"
|
|
"st1h {z3.s}, p3, [%[dst], #3, mul vl] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src), // %[src]
|
|
[dst] "+r"(dst), // %[dst]
|
|
[width] "+r"(width) // %[width]
|
|
: [vl] "r"(vl), // %[vl]
|
|
[vl2] "r"(vl * 2), // %[vl2]
|
|
[vl3] "r"(vl * 3), // %[vl3]
|
|
[scale] "w"(scale), // %[scale]
|
|
[bias] "w"(1.9259299444e-34f) // %[bias] 2^-112
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "p0", "p1", "p2",
|
|
"p3");
|
|
}
|
|
|
|
void HalfFloat1Row_SVE2(const uint16_t* src,
|
|
uint16_t* dst,
|
|
float scale,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with all-true predicates to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z0.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z1.h}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n"
|
|
"ucvtf z0.h, p0/m, z0.h \n"
|
|
"ucvtf z1.h, p0/m, z1.h \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"st1h {z0.h}, p0, [%[dst]] \n"
|
|
"st1h {z1.h}, p0, [%[dst], #1, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate predicates for the final iteration to deal with the tail.
|
|
"whilelt p0.h, wzr, %w[width] \n"
|
|
"whilelt p1.h, %w[vl], %w[width] \n"
|
|
"ld1h {z0.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z1.h}, p1/z, [%[src], #1, mul vl] \n"
|
|
"ucvtf z0.h, p0/m, z0.h \n"
|
|
"ucvtf z1.h, p0/m, z1.h \n"
|
|
"st1h {z0.h}, p0, [%[dst]] \n"
|
|
"st1h {z1.h}, p1, [%[dst], #1, mul vl] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src), // %[src]
|
|
[dst] "+r"(dst), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z0", "z1", "p0", "p1");
|
|
}
|
|
|
|
void I210ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I210ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I210AlphaToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
const uint16_t* src_a,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I210AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I210ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I210ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void P210ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_uv,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
P210ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void P210ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_uv,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
P210ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void I410ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I410ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I410AlphaToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
const uint16_t* src_a,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I410AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I410ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I410ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void P410ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_uv,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
P410ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void P410ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_uv,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
P410ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void I212ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I212ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void I212ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I212ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void Convert8To8Row_SVE2(const uint8_t* src_y,
|
|
uint8_t* dst_y,
|
|
int scale,
|
|
int bias,
|
|
int width) {
|
|
Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width);
|
|
}
|
|
|
|
void MergeUVRow_SVE2(const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_v]] \n"
|
|
"incb %[src_u] \n"
|
|
"incb %[src_v] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
|
|
"incb %[dst_uv], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_v]] \n"
|
|
"st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
|
|
|
|
"99: \n"
|
|
: [src_u] "+r"(src_u), // %[src_u]
|
|
[src_v] "+r"(src_v), // %[src_v]
|
|
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "memory", "cc", "z1", "z2", "p0");
|
|
}
|
|
|
|
void SplitUVRow_SVE2(const uint8_t* src_uv,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n"
|
|
"incb %[src_uv], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z1.b}, p0, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p0, [%[dst_v]] \n"
|
|
"incb %[dst_u] \n"
|
|
"incb %[dst_v] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n"
|
|
"st1b {z1.b}, p0, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p0, [%[dst_v]] \n"
|
|
|
|
"99: \n"
|
|
: [src_uv] "+r"(src_uv), // %[src_uv]
|
|
[dst_u] "+r"(dst_u), // %[dst_u]
|
|
[dst_v] "+r"(dst_v), // %[dst_v]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "memory", "cc", "z1", "z2", "p0");
|
|
}
|
|
|
|
void HalfMergeUVRow_SVE2(const uint8_t* src_u,
|
|
int src_stride_u,
|
|
const uint8_t* src_v,
|
|
int src_stride_v,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
const uint8_t* src_u1 = src_u + src_stride_u;
|
|
const uint8_t* src_v1 = src_v + src_stride_v;
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_u1]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_v]] \n"
|
|
"ld1b {z4.b}, p0/z, [%[src_v1]] \n"
|
|
"incb %[src_u] \n"
|
|
"incb %[src_u1] \n"
|
|
"incb %[src_v] \n"
|
|
"incb %[src_v1] \n"
|
|
"uaddlb z5.h, z1.b, z2.b \n"
|
|
"uaddlb z6.h, z3.b, z4.b \n"
|
|
"uaddlt z1.h, z1.b, z2.b \n"
|
|
"uaddlt z3.h, z3.b, z4.b \n"
|
|
"add z1.h, z1.h, z5.h \n"
|
|
"add z3.h, z3.h, z6.h \n"
|
|
"rshrnb z1.b, z1.h, #2 \n"
|
|
"rshrnt z1.b, z3.h, #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z1.b}, p0, [%[dst_uv]] \n"
|
|
"incb %[dst_uv] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_u1]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_v]] \n"
|
|
"ld1b {z4.b}, p0/z, [%[src_v1]] \n"
|
|
"uaddlb z5.h, z1.b, z2.b \n"
|
|
"uaddlb z6.h, z3.b, z4.b \n"
|
|
"uaddlt z1.h, z1.b, z2.b \n"
|
|
"uaddlt z3.h, z3.b, z4.b \n"
|
|
"add z1.h, z1.h, z5.h \n"
|
|
"add z3.h, z3.h, z6.h \n"
|
|
"rshrnb z1.b, z1.h, #2 \n"
|
|
"rshrnt z1.b, z3.h, #2 \n"
|
|
"st1b {z1.b}, p0, [%[dst_uv]] \n"
|
|
|
|
"99: \n"
|
|
: [src_u] "+r"(src_u), // %[src_u]
|
|
[src_u1] "+r"(src_u1), // %[src_u1]
|
|
[src_v] "+r"(src_v), // %[src_v]
|
|
[src_v1] "+r"(src_v1), // %[src_v1]
|
|
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z1", "z2", "z3", "z4", "z5", "z6", "p0");
|
|
}
|
|
|
|
void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) {
|
|
int vl;
|
|
int vl2;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"lsl %w[vl2], %w[vl], #1 \n"
|
|
"subs %w[width], %w[width], %w[vl2] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z0.b}, p0/z, [%[src], #0, mul vl] \n"
|
|
"ld1b {z1.b}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl2] \n"
|
|
"st1b {z0.b}, p0, [%[dst], #0, mul vl] \n"
|
|
"st1b {z1.b}, p0, [%[dst], #1, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl2] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"3: \n"
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z0.b}, p0/z, [%[src]] \n"
|
|
"incb %[src] \n"
|
|
"st1b {z0.b}, p0, [%[dst]] \n"
|
|
"incb %[dst] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.gt 3b \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [vl] "=&r"(vl),
|
|
[vl2] "=&r"(vl2)
|
|
:
|
|
: "cc", "memory", "z0", "z1", "p0");
|
|
}
|
|
|
|
void InterpolateRow_SVE2(uint8_t* dst_ptr,
|
|
const uint8_t* src_ptr,
|
|
ptrdiff_t src_stride,
|
|
int width,
|
|
int source_y_fraction) {
|
|
int y1_fraction = source_y_fraction;
|
|
int y0_fraction = 256 - y1_fraction;
|
|
const uint8_t* src_ptr1 = src_ptr + src_stride;
|
|
|
|
if (y0_fraction == 0) {
|
|
CopyRow_SVE2(src_ptr1, dst_ptr, width);
|
|
return;
|
|
}
|
|
if (y0_fraction == 256) {
|
|
CopyRow_SVE2(src_ptr, dst_ptr, width);
|
|
return;
|
|
}
|
|
if (y0_fraction == 128) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
|
"incb %[src_ptr] \n"
|
|
"incb %[src_ptr1] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"urhadd z2.b, p0/m, z2.b, z3.b \n"
|
|
"st1b {z2.b}, p0, [%[dst_ptr]] \n"
|
|
"incb %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
|
"urhadd z2.b, p0/m, z2.b, z3.b \n"
|
|
"st1b {z2.b}, p0, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_ptr), [src_ptr1] "+r"(src_ptr1),
|
|
[dst_ptr] "+r"(dst_ptr), [width] "+r"(width), [vl] "=&r"(vl)
|
|
:
|
|
: "cc", "memory", "z2", "z3", "p0");
|
|
return;
|
|
}
|
|
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"dup z0.b, %w[y0_fraction] \n"
|
|
"dup z1.b, %w[y1_fraction] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
|
"incb %[src_ptr] \n"
|
|
"incb %[src_ptr1] \n"
|
|
"umullb z4.h, z2.b, z0.b \n"
|
|
"umullt z2.h, z2.b, z0.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"umlalb z4.h, z3.b, z1.b \n"
|
|
"umlalt z2.h, z3.b, z1.b \n"
|
|
"rshrnb z3.b, z4.h, #8 \n"
|
|
"rshrnt z3.b, z2.h, #8 \n"
|
|
"st1b {z3.b}, p0, [%[dst_ptr]] \n"
|
|
"incb %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
|
"umullb z4.h, z2.b, z0.b \n"
|
|
"umullt z2.h, z2.b, z0.b \n"
|
|
"umlalb z4.h, z3.b, z1.b \n"
|
|
"umlalt z2.h, z3.b, z1.b \n"
|
|
"rshrnb z3.b, z4.h, #8 \n"
|
|
"rshrnt z3.b, z2.h, #8 \n"
|
|
"st1b {z3.b}, p0, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
|
|
[src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
|
|
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
|
|
[y1_fraction] "r"(y1_fraction) // %[y1_fraction]
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
|
|
}
|
|
|
|
void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
|
|
"incb %[src_yuy2], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z0.b}, p0, [%[dst_y]] \n"
|
|
"incb %[dst_y] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
|
|
"st1b {z0.b}, p0, [%[dst_y]] \n"
|
|
|
|
"99: \n"
|
|
: [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
|
|
[dst_y] "+r"(dst_y), // %[dst_y]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "memory", "cc", "z0", "z1", "p0");
|
|
}
|
|
|
|
void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
|
|
"incb %[src_uyvy], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z1.b}, p0, [%[dst_y]] \n"
|
|
"incb %[dst_y] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
|
|
"st1b {z1.b}, p0, [%[dst_y]] \n"
|
|
|
|
"99: \n"
|
|
: [src_uyvy] "+r"(src_uyvy), // %[src_uyvy]
|
|
[dst_y] "+r"(dst_y), // %[dst_y]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "memory", "cc", "z0", "z1", "p0");
|
|
}
|
|
|
|
void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2,
|
|
int src_stride_yuy2,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2;
|
|
int vl;
|
|
int vl_half;
|
|
int width_half;
|
|
if (src_stride_yuy2 == 0) {
|
|
src_yuy2_1 = src_yuy2;
|
|
}
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"lsr %w[vl_half], %w[vl], #1 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"whilelt p1.b, wzr, %w[vl_half] \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
|
|
"ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n"
|
|
"incb %[src_yuy2], all, mul #2 \n"
|
|
"incb %[src_yuy2_1], all, mul #2 \n"
|
|
"urhadd z1.b, p0/m, z1.b, z5.b \n"
|
|
"uzp1 z0.b, z1.b, z1.b \n"
|
|
"uzp2 z2.b, z1.b, z1.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z0.b}, p1, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p1, [%[dst_v]] \n"
|
|
"add %[dst_u], %[dst_u], %x[vl_half] \n"
|
|
"add %[dst_v], %[dst_v], %x[vl_half] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"add %w[width_half], %w[width], #1 \n"
|
|
"lsr %w[width_half], %w[width_half], #1 \n"
|
|
"whilelt p1.b, wzr, %w[width_half] \n"
|
|
"zip1 p0.b, p1.b, p1.b \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
|
|
"ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n"
|
|
"urhadd z1.b, p0/m, z1.b, z5.b \n"
|
|
"uzp1 z0.b, z1.b, z1.b \n"
|
|
"uzp2 z2.b, z1.b, z1.b \n"
|
|
"st1b {z0.b}, p1, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p1, [%[dst_v]] \n"
|
|
|
|
"99: \n"
|
|
: [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
|
|
[src_yuy2_1] "+r"(src_yuy2_1), // %[src_yuy2_1]
|
|
[dst_u] "+r"(dst_u), // %[dst_u]
|
|
[dst_v] "+r"(dst_v), // %[dst_v]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl), // %[vl]
|
|
[vl_half] "=&r"(vl_half), // %[vl_half]
|
|
[width_half] "=&r"(width_half) // %[width_half]
|
|
:
|
|
: "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
|
|
}
|
|
|
|
void UYVYToUVRow_SVE2(const uint8_t* src_uyvy,
|
|
int src_stride_uyvy,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const uint8_t* src_uyvy_1 = src_uyvy + src_stride_uyvy;
|
|
int vl;
|
|
int vl_half;
|
|
int width_half;
|
|
if (src_stride_uyvy == 0) {
|
|
src_uyvy_1 = src_uyvy;
|
|
}
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"lsr %w[vl_half], %w[vl], #1 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"whilelt p1.b, wzr, %w[vl_half] \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
|
|
"ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n"
|
|
"incb %[src_uyvy], all, mul #2 \n"
|
|
"incb %[src_uyvy_1], all, mul #2 \n"
|
|
"urhadd z0.b, p0/m, z0.b, z4.b \n"
|
|
"uzp1 z1.b, z0.b, z0.b \n"
|
|
"uzp2 z2.b, z0.b, z0.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z1.b}, p1, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p1, [%[dst_v]] \n"
|
|
"add %[dst_u], %[dst_u], %x[vl_half] \n"
|
|
"add %[dst_v], %[dst_v], %x[vl_half] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"add %w[width_half], %w[width], #1 \n"
|
|
"lsr %w[width_half], %w[width_half], #1 \n"
|
|
"whilelt p1.b, wzr, %w[width_half] \n"
|
|
"zip1 p0.b, p1.b, p1.b \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
|
|
"ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n"
|
|
"urhadd z0.b, p0/m, z0.b, z4.b \n"
|
|
"uzp1 z1.b, z0.b, z0.b \n"
|
|
"uzp2 z2.b, z0.b, z0.b \n"
|
|
"st1b {z1.b}, p1, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p1, [%[dst_v]] \n"
|
|
|
|
"99: \n"
|
|
: [src_uyvy] "+r"(src_uyvy), // %[src_uyvy]
|
|
[src_uyvy_1] "+r"(src_uyvy_1), // %[src_uyvy_1]
|
|
[dst_u] "+r"(dst_u), // %[dst_u]
|
|
[dst_v] "+r"(dst_v), // %[dst_v]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl), // %[vl]
|
|
[vl_half] "=&r"(vl_half), // %[vl_half]
|
|
[width_half] "=&r"(width_half) // %[width_half]
|
|
:
|
|
: "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
|
|
}
|
|
|
|
#ifdef HAS_CONVERT16TO8ROW_SVE2
|
|
// Disabled: NEON version is faster for 128 bit vectors.
|
|
void Convert16To8Row_SVE2(const uint16_t* src_y,
|
|
uint8_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
const int shift = 23 - __builtin_clz((int32_t)scale);
|
|
int vl;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"dup z31.h, %w[shift] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
|
|
"incb %[src_ptr] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"uqshl z2.h, p0/m, z2.h, z31.h \n"
|
|
"shrnb z2.b, z2.h, #8 \n"
|
|
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
|
|
"inch %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.h, wzr, %w[width] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
|
|
"uqshl z2.h, p0/m, z2.h, z31.h \n"
|
|
"shrnb z2.b, z2.h, #8 \n"
|
|
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_y), // %[src_ptr]
|
|
[dst_ptr] "+r"(dst_y), // %[dst_ptr]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [shift] "r"(shift) // %[shift]
|
|
: "cc", "memory", "z2", "z31", "p0");
|
|
}
|
|
#endif // HAS_CONVERT16TO8ROW_SVE2
|
|
|
|
#ifdef HAS_HALFROW_16TO8_SVE2
|
|
// Disabled: NEON version is faster for 128 bit vectors.
|
|
void HalfRow_16To8_SVE2(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int scale,
|
|
int width) {
|
|
const uint16_t* src_uv1 = src_uv + src_uv_stride;
|
|
const int shift = 23 - __builtin_clz((int32_t)scale);
|
|
int vl;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"dup z31.h, %w[shift] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
|
|
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
|
|
"incb %[src_ptr] \n"
|
|
"incb %[src_ptr1] \n"
|
|
"urhadd z2.h, p0/m, z2.h, z3.h \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"uqshl z2.h, p0/m, z2.h, z31.h \n"
|
|
"shrnb z2.b, z2.h, #8 \n"
|
|
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
|
|
"inch %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.h, wzr, %w[width] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
|
|
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
|
|
"urhadd z2.h, p0/m, z2.h, z3.h \n"
|
|
"uqshl z2.h, p0/m, z2.h, z31.h \n"
|
|
"shrnb z2.b, z2.h, #8 \n"
|
|
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_uv), // %[src_ptr]
|
|
[src_ptr1] "+r"(src_uv1), // %[src_ptr1]
|
|
[dst_ptr] "+r"(dst_uv), // %[dst_ptr]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [shift] "r"(shift) // %[shift]
|
|
: "cc", "memory", "z2", "z3", "z31", "p0");
|
|
}
|
|
#endif // HAS_HALFROW_16TO8_SVE2
|
|
|
|
#ifdef HAS_HALFWIDTHROW_16TO8_SVE2
|
|
// Disabled: NEON version is faster for 128 bit vectors.
|
|
void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int scale,
|
|
int width) {
|
|
const uint16_t* src_uv1 = src_uv + src_uv_stride;
|
|
const int shift = 23 - __builtin_clz((int32_t)scale);
|
|
int vl;
|
|
int vl_input;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"dup z31.h, %w[shift] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z0.h}, p0/z, [%[src_ptr]] \n"
|
|
"ld1h {z1.h}, p0/z, [%[src_ptr], #1, mul vl] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr1]] \n"
|
|
"ld1h {z3.h}, p0/z, [%[src_ptr1], #1, mul vl] \n"
|
|
"incb %[src_ptr], all, mul #4 \n"
|
|
"incb %[src_ptr1], all, mul #4 \n"
|
|
"uzp1 z4.h, z0.h, z1.h \n"
|
|
"uzp2 z5.h, z0.h, z1.h \n"
|
|
"uzp1 z6.h, z2.h, z3.h \n"
|
|
"uzp2 z7.h, z2.h, z3.h \n"
|
|
"uaddlb z0.s, z4.h, z5.h \n"
|
|
"uaddlt z1.s, z4.h, z5.h \n"
|
|
"uaddlb z2.s, z6.h, z7.h \n"
|
|
"uaddlt z3.s, z6.h, z7.h \n"
|
|
"add z0.s, z0.s, z2.s \n"
|
|
"add z1.s, z1.s, z3.s \n"
|
|
"rshrnb z0.h, z0.s, #2 \n"
|
|
"rshrnt z0.h, z1.s, #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"uqshl z0.h, p0/m, z0.h, z31.h \n"
|
|
"shrnb z0.b, z0.h, #8 \n"
|
|
"st1b {z0.h}, p0, [%[dst_ptr]] \n"
|
|
"inch %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"lsl %w[vl_input], %w[width], #1 \n"
|
|
"whilelt p0.h, wzr, %w[vl_input] \n"
|
|
"whilelt p1.h, %w[vl], %w[vl_input] \n"
|
|
"whilelt p2.h, wzr, %w[width] \n"
|
|
"ld1h {z0.h}, p0/z, [%[src_ptr]] \n"
|
|
"ld1h {z1.h}, p1/z, [%[src_ptr], #1, mul vl] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr1]] \n"
|
|
"ld1h {z3.h}, p1/z, [%[src_ptr1], #1, mul vl] \n"
|
|
"uzp1 z4.h, z0.h, z1.h \n"
|
|
"uzp2 z5.h, z0.h, z1.h \n"
|
|
"uzp1 z6.h, z2.h, z3.h \n"
|
|
"uzp2 z7.h, z2.h, z3.h \n"
|
|
"uaddlb z0.s, z4.h, z5.h \n"
|
|
"uaddlt z1.s, z4.h, z5.h \n"
|
|
"uaddlb z2.s, z6.h, z7.h \n"
|
|
"uaddlt z3.s, z6.h, z7.h \n"
|
|
"add z0.s, z0.s, z2.s \n"
|
|
"add z1.s, z1.s, z3.s \n"
|
|
"rshrnb z0.h, z0.s, #2 \n"
|
|
"rshrnt z0.h, z1.s, #2 \n"
|
|
"uqshl z0.h, p2/m, z0.h, z31.h \n"
|
|
"shrnb z0.b, z0.h, #8 \n"
|
|
"st1b {z0.h}, p2, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_uv), // %[src_ptr]
|
|
[src_ptr1] "+r"(src_uv1), // %[src_ptr1]
|
|
[dst_ptr] "+r"(dst_uv), // %[dst_ptr]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl), // %[vl]
|
|
[vl_input] "=&r"(vl_input) // %[vl_input]
|
|
: [shift] "r"(shift) // %[shift]
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31",
|
|
"p0", "p1", "p2");
|
|
}
|
|
#endif // HAS_HALFWIDTHROW_16TO8_SVE2
|
|
|
|
#ifdef HAS_SSIM4X4ROW_SVE2
|
|
void Ssim4x4Row_SVE2(const uint8_t* src_a,
|
|
int stride_a,
|
|
const uint8_t* src_b,
|
|
int stride_b,
|
|
uint32_t* dst,
|
|
int width) {
|
|
Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width);
|
|
}
|
|
#endif // HAS_SSIM4X4ROW_SVE2
|
|
|
|
#ifdef HAS_SSIMWINDOWROW_SVE2
|
|
float SsimWindowRow_SVE2(const uint32_t* row0,
|
|
const uint32_t* row1,
|
|
int num_windows) {
|
|
return SsimWindowRow_SVE_SC(row0, row1, num_windows);
|
|
}
|
|
#endif // HAS_SSIMWINDOWROW_SVE2
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|