mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
f249eaa72f
1. Fix multi-bit TestCpuFlag(A | B) checks in AYUVToNV12, AYUVToNV21, RAWToRGBA, NV12ToRGB24Matrix, NV21ToRGB24Matrix, MergeRGBPlane, and MergeXR30Plane to compare == (A | B) so AVX-512 VBMI/BW kernels are not dispatched when AVX512VBMI or AVX512VL is absent. 2. Fix 2-byte SVE2 heap buffer overflow on odd widths in I422ToARGB1555Row_SVE_SC and I422ToARGB4444Row_SVE_SC using trn1/trn2 + st2b under .b predicates. 3. Fix ARGBPolynomialRow_SSE2 to use packssdw before packuswb, and add missing vzeroupper to SplitRGBRow_AVX2. 4. Fix CalcFrameSsim to return 0.0 instead of 0.0 / 0 (NaN) when width <= 8 || height <= 8 so I420Ssim stays finite on small frames. 5. Fix negative-height vertical inversion in ConvertFromI420, SplitARGBPlane (when dst_a is NULL), I422Rotate, I010Rotate, and I210Rotate. 6. Fix horizontal flip (negative x/dx) in ScaleFilterCols_NEON (signed sbfx #16, #16 offset extraction) and ScaleAddCols1_16_C (shift x >>= 16 before indexing src_ptr). 7. Fix ConvertToARGB odd crop_x chroma alignment in NV12/NV21, src_u offset in I420/YV12/J420/H420/U420, and FOURCC_U422 calling U422ToARGB instead of H422ToARGB. 8. Remove dead 14-instruction block in ScaleRowUp2_Bilinear_16_SSE2. 9. Fix UVScale_16 for src_width == 1 (P010ToP410/P210ToP410), enable and fix column order in TransposeWx4_32_SVE2, fix ByteToFloatRow NEON Any BPP (3 -> 4), fix AB64ToARGBRow_Any_*, ScaleColsUp2_SSE2, ScaleUVRowDown2Box_SME, and UYVYToARGBRow_SME macro guards, fix SVE2 loop tail branches (b.lt/b.ge), and remove duplicate HAS_* macros. 10. Fix ConvertFP16ToFP32Column_NEON loop condition and unsigned b.lo underflow on widths < 8 or not multiples of 8. 11. Fix UYVYToNV12 IS_ALIGNED(awidth, ...) checks and WASM SIMD IS_ALIGNED(width, 16) checks in I420ToRGB24Matrix / I422ToRGB24Matrix. 12. Wire SplitUVRow_SVE2, SplitUVRow_LSX, and SplitUVRow_LASX in NV24ToARGB. 13. Fix MJpegDecoder heap buffer overflows in LoadFrame (reallocate databuf_[i] when scanlines_size changes), DecodeToBuffers, and DecodeToCallback (clamp scanlines_to_copy to lines_left and align skip to max_v_samp_factor). 14. Replace width & (vl - 1) tail calculations with post-loop remainder across 18 SVE2 functions in row_sve.h for non-power-of-2 SVE VL, and remove duplicate label 2: in UYVYToARGBRow_SVE_SC. 15. Handle width & 15 tail pixels in DetileToYUY2_C and validate IS_POWEROFTWO(tile_height) in DetileSplitUVPlane and DetileToYUY2. 16. Avoid signed 32-bit overflow in DivideRow_16_C and C16TO8, fix ClampFloatToZero_SSE2 pointer increment (lea 0x4) and loop counter (sub $0x1), and initialize xmm3 in ARGBCopyYToAlphaRow_SSE2. 17. Check max leaf 7 subleaf and max extended CPUID leaf in GetCpuFlags before querying subleaf 7:1 and extended leaf 0x80000021. 18. Free plane_u on error in I422ToNV21, and fix 4x/2x scratch row over-allocation in ScaleARGBBilinearDown and ScaleUVBilinearDown. 19. Add ARGBSetRow_RVV and ARGBShuffleRow_RVV in row_rvv.cc. 20. Fix NEON mask (15 -> 7) in ScaleUVRowUp2_Linear_Any_NEON / _16_Any_NEON, use SIMD InterpolateRow + SplitUVRow in NV16ToI420, wire SSSE3/SSE2 in ARGBToI400Matrix / ARGBToYUY2Matrix / ARGBToUYVYMatrix and RGB24Mirror, and split HalfFloatRow multiply into (src * scale) * 2^-112 to avoid 50x Intel denormal stalls. Bug: b/574714081 Test: libyuv_unittest Change-Id: Iec3ae5c58d52bee3ef13cf9b765ac84b733aa576 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8601931 Reviewed-by: richard winterton <rrwinterton@gmail.com> Commit-Queue: Frank Barchard <fbarchard@google.com>
1752 lines
79 KiB
C++
1752 lines
79 KiB
C++
/*
|
|
* Copyright 2024 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/row_sve.h"
|
|
#include "libyuv/compare_row.h"
|
|
#include "libyuv/row.h"
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
// Note: every register written by an inline asm block (vector, general
|
|
// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
|
|
// listed in its clobber list, even when the ABI treats it as caller-saved.
|
|
// With LTO/LTCG the compiler may keep values live in any register across the
|
|
// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
|
|
// are callee-saved. List "memory" first, then "cc", then registers.
|
|
|
|
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
|
|
|
#define RGBTOARGB8_SVE_2X \
|
|
/* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
|
|
"uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
|
|
"uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \
|
|
"uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
|
|
"uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \
|
|
"uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \
|
|
"uqshrnt z18.b, z22.h, #6 \n" /* R1 */
|
|
|
|
#define RGBTOARGB8_SVE_TOP_2X \
|
|
/* Inputs: B: z16.h, G: z17.h, R: z18.h */ \
|
|
"uqshl z16.h, p0/m, z16.h, #2 \n" /* B0 */ \
|
|
"uqshl z17.h, p0/m, z17.h, #2 \n" /* G0 */ \
|
|
"uqshl z18.h, p0/m, z18.h, #2 \n" /* R0 */ \
|
|
"uqshl z20.h, p0/m, z20.h, #2 \n" /* B1 */ \
|
|
"uqshl z21.h, p0/m, z21.h, #2 \n" /* G1 */ \
|
|
"uqshl z22.h, p0/m, z22.h, #2 \n" /* R1 */
|
|
|
|
void I444ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"cnth %[vl] \n"
|
|
"ptrue p0.b \n" //
|
|
YUVTORGB_SVE_SETUP
|
|
"dup z19.b, #255 \n" // Alpha
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with an all-true predicate to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p1.h \n"
|
|
"1: \n" //
|
|
READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
"add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate a predicate for the final iteration to deal with the tail.
|
|
"whilelt p1.h, wzr, %w[width] \n" //
|
|
READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
|
|
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
|
|
|
|
"99: \n"
|
|
: [src_y] "+r"(src_y), // %[src_y]
|
|
[src_u] "+r"(src_u), // %[src_u]
|
|
[src_v] "+r"(src_v), // %[src_v]
|
|
[dst_argb] "+r"(dst_argb), // %[dst_argb]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
|
|
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
|
|
: "cc", "memory", YUVTORGB_SVE_REGS);
|
|
}
|
|
|
|
void I444ToRGB24Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I444ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_rgb24, yuvconstants, width);
|
|
}
|
|
|
|
void I400ToARGBRow_SVE2(const uint8_t* src_y,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I400ToARGBRow_SVE_SC(src_y, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToRGB24Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToRGB565Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb565,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToRGB565Row_SVE_SC(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToARGB1555Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb1555,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToARGB1555Row_SVE_SC(src_y, src_u, src_v, dst_argb1555, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I422ToARGB4444Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb4444,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToARGB4444Row_SVE_SC(src_y, src_u, src_v, dst_argb4444, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I422ToRGBARow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgba,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToRGBARow_SVE_SC(src_y, src_u, src_v, dst_rgba, yuvconstants, width);
|
|
}
|
|
|
|
void I422ToAR30Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I422AlphaToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
const uint8_t* src_a,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I422AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I444AlphaToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
const uint8_t* src_a,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I444AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void NV12ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_uv,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
NV12ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void NV21ToARGBRow_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_vu,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
NV21ToARGBRow_SVE_SC(src_y, src_vu, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void NV12ToRGB24Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_uv,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
NV12ToRGB24Row_SVE_SC(src_y, src_uv, dst_rgb24, yuvconstants, width);
|
|
}
|
|
|
|
void NV21ToRGB24Row_SVE2(const uint8_t* src_y,
|
|
const uint8_t* src_vu,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
NV21ToRGB24Row_SVE_SC(src_y, src_vu, dst_rgb24, yuvconstants, width);
|
|
}
|
|
|
|
void ARGBToUVMatrixRow_SVE2(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
int8_t uvconstants[8] = {(int8_t)c->kRGBToU[0], (int8_t)c->kRGBToU[1],
|
|
(int8_t)c->kRGBToU[2], (int8_t)c->kRGBToU[3],
|
|
(int8_t)c->kRGBToV[0], (int8_t)c->kRGBToV[1],
|
|
(int8_t)c->kRGBToV[2], (int8_t)c->kRGBToV[3]};
|
|
ARGBToUVMatrixRow_SVE_SC(src_argb, src_stride_argb, dst_u, dst_v, width,
|
|
uvconstants);
|
|
}
|
|
|
|
void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb,
|
|
uint8_t* dst_y,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c);
|
|
}
|
|
|
|
#define ARGBTORGB565_SVE \
|
|
/* Inputs: \
|
|
* z0: rrrrrxxxbbbbbxxx \
|
|
* z1: xxxxxxxxggggggxx \
|
|
* z3: 0000000000000011 (3, 0, 3, 0, ...) \
|
|
* z4: 0000011111100000 \
|
|
*/ \
|
|
"lsr z0.b, p0/m, z0.b, z3.b \n" \
|
|
"lsl z1.h, z1.h, #3 \n" \
|
|
"bsl z1.d, z1.d, z0.d, z4.d \n"
|
|
|
|
void ARGBToRGB565Row_SVE2(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
int width) {
|
|
unsigned bsl_mask = 0x7e0;
|
|
uint64_t vl;
|
|
width *= 2;
|
|
asm volatile(
|
|
"mov z3.h, #3 \n"
|
|
"dup z4.h, %w[bsl_mask] \n"
|
|
|
|
"cntb %[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
|
|
"incb %[src], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n" //
|
|
ARGBTORGB565_SVE
|
|
"st1b {z1.b}, p0, [%[dst]] \n"
|
|
"incb %[dst] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
|
|
ARGBTORGB565_SVE
|
|
"st1b {z1.b}, p0, [%[dst]] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_argb), // %[src]
|
|
[dst] "+r"(dst_rgb), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [bsl_mask] "r"(bsl_mask) // %[bsl_mask]
|
|
: "cc", "memory", "z0", "z1", "z3", "z4", "p0");
|
|
}
|
|
|
|
void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
uint32_t dither4,
|
|
int width) {
|
|
unsigned bsl_mask = 0x7e0;
|
|
uint64_t vl;
|
|
width *= 2;
|
|
asm volatile(
|
|
"mov z3.h, #3 \n"
|
|
"dup z4.h, %w[bsl_mask] \n"
|
|
"dup z2.s, %w[dither4] \n"
|
|
"zip1 z2.b, z2.b, z2.b \n"
|
|
|
|
"cntb %[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
|
|
"incb %[src], all, mul #2 \n"
|
|
"uqadd z0.b, z0.b, z2.b \n"
|
|
"uqadd z1.b, z1.b, z2.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n" //
|
|
ARGBTORGB565_SVE
|
|
"st1b {z1.b}, p0, [%[dst]] \n"
|
|
"incb %[dst] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
|
|
"uqadd z0.b, z0.b, z2.b \n"
|
|
"uqadd z1.b, z1.b, z2.b \n" //
|
|
ARGBTORGB565_SVE
|
|
"st1b {z1.b}, p0, [%[dst]] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_argb), // %[src]
|
|
[dst] "+r"(dst_rgb), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [bsl_mask] "r"(bsl_mask), // %[bsl_mask]
|
|
[dither4] "r"(dither4) // %[dither4]
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
|
|
}
|
|
|
|
#define ARGB1555TOARGB \
|
|
/* Input: z1/z3.h = arrrrrgggggbbbbb */ \
|
|
"lsl z0.h, z1.h, #3 \n" /* rrrgggggbbbbb000 */ \
|
|
"lsl z2.h, z3.h, #3 \n" /* rrrgggggbbbbb000 */ \
|
|
"asr z1.h, z1.h, #7 \n" /* aaaaaaaarrrrrggg */ \
|
|
"asr z3.h, z3.h, #7 \n" /* aaaaaaaarrrrrggg */ \
|
|
"lsl z0.b, p0/m, z0.b, z4.b \n" /* ggggg000bbbbb000 */ \
|
|
"lsl z2.b, p0/m, z2.b, z4.b \n" /* ggggg000bbbbb000 */ \
|
|
"sri z1.b, z1.b, #5 \n" /* aaaaaaaarrrrrrrr */ \
|
|
"sri z3.b, z3.b, #5 \n" /* aaaaaaaarrrrrrrr */ \
|
|
"sri z0.b, z0.b, #5 \n" /* ggggggggbbbbbbbb */ \
|
|
"sri z2.b, z2.b, #5 \n" /* ggggggggbbbbbbbb */
|
|
|
|
void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"mov z4.h, #0x0300 \n"
|
|
"ptrue p0.b \n"
|
|
|
|
"cnth %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
"1: \n"
|
|
"ld1h {z1.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z3.h}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n" //
|
|
ARGB1555TOARGB
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"st2h {z0.h, z1.h}, p0, [%[dst]] \n"
|
|
"st2h {z2.h, z3.h}, p0, [%[dst], #2, mul vl] \n"
|
|
"incb %[dst], all, mul #4 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p1.h, wzr, %w[width] \n"
|
|
"whilelt p2.h, %w[vl], %w[width] \n"
|
|
"ld1h {z1.h}, p1/z, [%[src]] \n"
|
|
"ld1h {z3.h}, p2/z, [%[src], #1, mul vl] \n" //
|
|
ARGB1555TOARGB
|
|
"st2h {z0.h, z1.h}, p1, [%[dst]] \n"
|
|
"st2h {z2.h, z3.h}, p2, [%[dst], #2, mul vl] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_argb1555), // %[src]
|
|
[dst] "+r"(dst_argb), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2");
|
|
}
|
|
|
|
// clang-format off
|
|
#define AYUVTOUV_SVE(zU0, zV0, zU1, zV1) /* e.g. */ \
|
|
"ld2h {z0.h, z1.h}, p0/z, [%[src0]] \n" /* VUVU.. YAYA.. */ \
|
|
"ld2h {z1.h, z2.h}, p1/z, [%[src0], #2, mul vl] \n" /* VUVU.. YAYA.. */ \
|
|
"ld2h {z2.h, z3.h}, p0/z, [%[src1]] \n" /* VUVU.. YAYA.. */ \
|
|
"ld2h {z3.h, z4.h}, p1/z, [%[src1], #2, mul vl] \n" /* VUVU.. YAYA.. */ \
|
|
"incb %[src0], all, mul #4 \n" \
|
|
"incb %[src1], all, mul #4 \n" \
|
|
"uaddlb z4.h, z0.b, z2.b \n" /* V */ \
|
|
"uaddlt z5.h, z0.b, z2.b \n" /* U */ \
|
|
"uaddlb z6.h, z1.b, z3.b \n" /* V */ \
|
|
"uaddlt z7.h, z1.b, z3.b \n" /* U */ \
|
|
"addp " #zU0 ".h, p0/m, " #zU0 ".h, " #zV0 ".h \n" /* UV */ \
|
|
"addp " #zU1 ".h, p1/m, " #zU1 ".h, " #zV1 ".h \n" /* UV */ \
|
|
"subs %w[width], %w[width], %w[vl] \n" \
|
|
"urshr " #zU0 ".h, p0/m, " #zU0 ".h, #2 \n" /* U0V0 */ \
|
|
"urshr " #zU1 ".h, p1/m, " #zU1 ".h, #2 \n" /* U0V0 */ \
|
|
"st1b {" #zU0 ".h}, p0, [%[dst]] \n" \
|
|
"st1b {" #zU1 ".h}, p1, [%[dst], #1, mul vl] \n" \
|
|
"incb %[dst] \n"
|
|
// clang-format on
|
|
|
|
// Filter 2 rows of AYUV UV's (444) into UV (420).
|
|
// AYUV is VUYA in memory. UV for NV12 is UV order in memory.
|
|
void AYUVToUVRow_SVE2(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
// Output a row of UV values, filtering 2x2 rows of AYUV.
|
|
const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
|
|
int vl;
|
|
asm volatile (
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"ptrue p1.h \n"
|
|
"1: \n"
|
|
AYUVTOUV_SVE(z5, z4, z7, z6)
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"cnth %x[vl] \n"
|
|
"whilelt p0.h, wzr, %w[width] \n" // first row
|
|
"whilelt p1.h, %w[vl], %w[width] \n" // second row
|
|
AYUVTOUV_SVE(z5, z4, z7, z6)
|
|
|
|
"99: \n"
|
|
: [src0]"+r"(src_ayuv), // %[src0]
|
|
[src1]"+r"(src_ayuv1), // %[src1]
|
|
[dst]"+r"(dst_uv), // %[dst]
|
|
[width]"+r"(width), // %[width]
|
|
[vl]"=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
|
|
"p1");
|
|
}
|
|
|
|
// Filter 2 rows of AYUV UV's (444) into VU (420).
|
|
void AYUVToVURow_SVE2(const uint8_t* src_ayuv,
|
|
int src_stride_ayuv,
|
|
uint8_t* dst_vu,
|
|
int width) {
|
|
// Output a row of VU values, filtering 2x2 rows of AYUV.
|
|
const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
|
|
int vl;
|
|
asm volatile (
|
|
"cntb %x[vl] \n"
|
|
"cmp %w[width], %w[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"ptrue p1.h \n"
|
|
"1: \n"
|
|
AYUVTOUV_SVE(z4, z5, z6, z7)
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"cnth %x[vl] \n"
|
|
"whilelt p0.h, wzr, %w[width] \n" // first row
|
|
"whilelt p1.h, %w[vl], %w[width] \n" // second row
|
|
AYUVTOUV_SVE(z4, z5, z6, z7)
|
|
|
|
"99: \n"
|
|
: [src0]"+r"(src_ayuv), // %[src0]
|
|
[src1]"+r"(src_ayuv1), // %[src1]
|
|
[dst]"+r"(dst_vu), // %[dst]
|
|
[width]"+r"(width), // %[width]
|
|
[vl]"=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
|
|
"p1");
|
|
}
|
|
|
|
void YUY2ToARGBRow_SVE2(const uint8_t* src_yuy2,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
YUY2ToARGBRow_SVE_SC(src_yuy2, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void UYVYToARGBRow_SVE2(const uint8_t* src_uyvy,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
static inline void RAWToWXYZRow_SVE2(const uint8_t* src_raw,
|
|
uint8_t* dst_wxyz,
|
|
int width,
|
|
uint32_t idx_start,
|
|
uint32_t idx_step,
|
|
uint32_t alpha) {
|
|
uint32_t vl;
|
|
asm("cntw %x0" : "=r"(vl));
|
|
uint32_t vl_mul3 = vl * 3;
|
|
uint32_t rem_mul3;
|
|
asm volatile(
|
|
"index z31.s, %w[idx_start], %w[idx_step] \n"
|
|
"dup z30.s, %w[alpha] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with the same predicates to avoid predicate
|
|
// generation overhead. We set up p1 to only load 3/4 of a vector.
|
|
"ptrue p0.s \n"
|
|
"whilelt p1.b, wzr, %w[vl_mul3] \n"
|
|
"1: \n"
|
|
"ld1b {z0.b}, p1/z, [%[src]] \n"
|
|
"add %[src], %[src], %x[vl_mul3] \n"
|
|
"ld1b {z1.b}, p1/z, [%[src]] \n"
|
|
"add %[src], %[src], %x[vl_mul3] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"tbl z1.b, {z1.b}, z31.b \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"orr z0.d, z0.d, z30.d \n"
|
|
"orr z1.d, z1.d, z30.d \n"
|
|
"st1w {z0.s}, p0, [%[dst]] \n"
|
|
"st1w {z1.s}, p0, [%[dst], #1, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate a pair of predicates for the final iteration to deal with
|
|
// the tail.
|
|
"3: \n"
|
|
"add %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
|
|
"whilelt p0.s, wzr, %w[width] \n"
|
|
"whilelt p1.b, wzr, %w[rem_mul3] \n"
|
|
"ld1b {z0.b}, p1/z, [%[src]] \n"
|
|
"add %[src], %[src], %x[vl_mul3] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"orr z0.d, z0.d, z30.d \n"
|
|
"st1w {z0.s}, p0, [%[dst]] \n"
|
|
"incb %[dst] \n"
|
|
"b.gt 3b \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_raw), // %[src]
|
|
[dst] "+r"(dst_wxyz), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl_mul3] "+r"(vl_mul3), // %[vl_mul3]
|
|
[rem_mul3] "=&r"(rem_mul3) // %[rem_mul3]
|
|
: [idx_start] "r"(idx_start), // %[idx_start]
|
|
[idx_step] "r"(idx_step), // %[idx_step]
|
|
[alpha] "r"(alpha), // %[alpha]
|
|
[vl] "r"(vl) // %[vl]
|
|
: "cc", "memory", "z0", "z1", "z30", "z31", "p0", "p1");
|
|
}
|
|
|
|
void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
|
|
RAWToWXYZRow_SVE2(src_raw, dst_argb, width, 0xff000102U, 0x00030303U,
|
|
0xff000000U);
|
|
}
|
|
|
|
void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
|
|
RAWToWXYZRow_SVE2(src_raw, dst_rgba, width, 0x000102ffU, 0x03030300U,
|
|
0x000000ffU);
|
|
}
|
|
|
|
void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
RAWToWXYZRow_SVE2(src_rgb24, dst_argb, width, 0xff020100U, 0x00030303U,
|
|
0xff000000U);
|
|
}
|
|
|
|
static const uint8_t kRAWToRGB24Indices[] = {
|
|
2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12,
|
|
17, 16, 15, 20, 19, 18, 23, 22, 21, 26, 25, 24, 29, 28, 27,
|
|
32, 31, 30, 35, 34, 33, 38, 37, 36, 41, 40, 39, 44, 43, 42,
|
|
47, 46, 45, 50, 49, 48, 53, 52, 51, 56, 55, 54, 59, 58, 57,
|
|
62, 61, 60, 65, 64, 63, 68, 67, 66, 71, 70, 69, 74, 73, 72,
|
|
77, 76, 75, 80, 79, 78, 83, 82, 81, 86, 85, 84, 89, 88, 87,
|
|
92, 91, 90, 95, 94, 93, 98, 97, 96, 101, 100, 99, 104, 103, 102,
|
|
107, 106, 105, 110, 109, 108, 113, 112, 111, 116, 115, 114, 119, 118, 117,
|
|
122, 121, 120, 125, 124, 123, 128, 127, 126, 131, 130, 129, 134, 133, 132,
|
|
137, 136, 135, 140, 139, 138, 143, 142, 141, 146, 145, 144, 149, 148, 147,
|
|
152, 151, 150, 155, 154, 153, 158, 157, 156, 161, 160, 159, 164, 163, 162,
|
|
167, 166, 165, 170, 169, 168, 173, 172, 171, 176, 175, 174, 179, 178, 177,
|
|
182, 181, 180, 185, 184, 183, 188, 187, 186, 191, 190, 189, 194, 193, 192,
|
|
197, 196, 195, 200, 199, 198, 203, 202, 201, 206, 205, 204, 209, 208, 207,
|
|
212, 211, 210, 215, 214, 213, 218, 217, 216, 221, 220, 219, 224, 223, 222,
|
|
227, 226, 225, 230, 229, 228, 233, 232, 231, 236, 235, 234, 239, 238, 237,
|
|
242, 241, 240, 245, 244, 243, 248, 247, 246, 251, 250, 249, 254, 253, 252};
|
|
|
|
void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
|
|
// width is in elements, convert to bytes.
|
|
width *= 3;
|
|
// we use the mul3 predicate pattern throughout to use the largest multiple
|
|
// of three number of lanes, for instance with a vector length of 16 bytes
|
|
// only the first 15 bytes will be used for load/store instructions.
|
|
uint32_t vl;
|
|
asm volatile(
|
|
"cntb %x[vl], mul3 \n"
|
|
"ptrue p0.b, mul3 \n"
|
|
"ld1b {z31.b}, p0/z, [%[kIndices]] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with the same predicate to avoid predicate
|
|
// generation overhead.
|
|
"1: \n"
|
|
"ld1b {z0.b}, p0/z, [%[src]] \n"
|
|
"add %[src], %[src], %x[vl] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z0.b}, p0, [%[dst]] \n"
|
|
"add %[dst], %[dst], %x[vl] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate a predicate for the final iteration to deal with the tail.
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z0.b}, p0/z, [%[src]] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"st1b {z0.b}, p0, [%[dst]] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_raw), // %[src]
|
|
[dst] "+r"(dst_rgb24), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [kIndices] "r"(kRAWToRGB24Indices) // %[kIndices]
|
|
: "cc", "memory", "z0", "z31", "p0");
|
|
}
|
|
|
|
static inline void ARGBToXYZRow_SVE2(const uint8_t* src_argb,
|
|
uint8_t* dst_xyz,
|
|
int width,
|
|
const uint8_t* indices) {
|
|
uint32_t vl;
|
|
asm("cntw %x0" : "=r"(vl));
|
|
uint32_t vl_mul3 = vl * 3;
|
|
uint32_t rem_mul3;
|
|
asm volatile(
|
|
"whilelt p1.b, wzr, %w[vl_mul3] \n"
|
|
"ld1b {z31.b}, p1/z, [%[indices]] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with the same predicates to avoid predicate
|
|
// generation overhead. We set up p1 to only store 3/4 of a vector.
|
|
"ptrue p0.s \n"
|
|
"1: \n"
|
|
"ld1w {z0.s}, p0/z, [%[src]] \n"
|
|
"ld1w {z1.s}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"tbl z1.b, {z1.b}, z31.b \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"st1b {z0.b}, p1, [%[dst]] \n"
|
|
"add %[dst], %[dst], %x[vl_mul3] \n"
|
|
"st1b {z1.b}, p1, [%[dst]] \n"
|
|
"add %[dst], %[dst], %x[vl_mul3] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate predicates for the final iteration to deal with the tail.
|
|
"add %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
|
|
"whilelt p0.s, wzr, %w[width] \n"
|
|
"whilelt p1.b, wzr, %w[rem_mul3] \n"
|
|
"whilelt p2.s, %w[vl], %w[width] \n"
|
|
"whilelt p3.b, %w[vl_mul3], %w[rem_mul3] \n"
|
|
"ld1w {z0.s}, p0/z, [%[src]] \n"
|
|
"ld1w {z1.s}, p2/z, [%[src], #1, mul vl] \n"
|
|
"tbl z0.b, {z0.b}, z31.b \n"
|
|
"tbl z1.b, {z1.b}, z31.b \n"
|
|
"st1b {z0.b}, p1, [%[dst]] \n"
|
|
"add %[dst], %[dst], %x[vl_mul3] \n"
|
|
"st1b {z1.b}, p3, [%[dst]] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_argb), // %[src]
|
|
[dst] "+r"(dst_xyz), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[rem_mul3] "=&r"(rem_mul3) // %[rem_mul3]
|
|
: [indices] "r"(indices), // %[indices]
|
|
[vl_mul3] "r"(vl_mul3), // %[vl_mul3]
|
|
[vl] "r"(vl) // %[vl]
|
|
: "cc", "memory", "z0", "z1", "z31", "p0", "p1", "p2", "p3");
|
|
}
|
|
|
|
static const uint8_t kARGBToRGB24RowIndices[] = {
|
|
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18,
|
|
20, 21, 22, 24, 25, 26, 28, 29, 30, 32, 33, 34, 36, 37, 38,
|
|
40, 41, 42, 44, 45, 46, 48, 49, 50, 52, 53, 54, 56, 57, 58,
|
|
60, 61, 62, 64, 65, 66, 68, 69, 70, 72, 73, 74, 76, 77, 78,
|
|
80, 81, 82, 84, 85, 86, 88, 89, 90, 92, 93, 94, 96, 97, 98,
|
|
100, 101, 102, 104, 105, 106, 108, 109, 110, 112, 113, 114, 116, 117, 118,
|
|
120, 121, 122, 124, 125, 126, 128, 129, 130, 132, 133, 134, 136, 137, 138,
|
|
140, 141, 142, 144, 145, 146, 148, 149, 150, 152, 153, 154, 156, 157, 158,
|
|
160, 161, 162, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178,
|
|
180, 181, 182, 184, 185, 186, 188, 189, 190, 192, 193, 194, 196, 197, 198,
|
|
200, 201, 202, 204, 205, 206, 208, 209, 210, 212, 213, 214, 216, 217, 218,
|
|
220, 221, 222, 224, 225, 226, 228, 229, 230, 232, 233, 234, 236, 237, 238,
|
|
240, 241, 242, 244, 245, 246, 248, 249, 250, 252, 253, 254,
|
|
};
|
|
|
|
static const uint8_t kARGBToRAWRowIndices[] = {
|
|
2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 18, 17, 16,
|
|
22, 21, 20, 26, 25, 24, 30, 29, 28, 34, 33, 32, 38, 37, 36,
|
|
42, 41, 40, 46, 45, 44, 50, 49, 48, 54, 53, 52, 58, 57, 56,
|
|
62, 61, 60, 66, 65, 64, 70, 69, 68, 74, 73, 72, 78, 77, 76,
|
|
82, 81, 80, 86, 85, 84, 90, 89, 88, 94, 93, 92, 98, 97, 96,
|
|
102, 101, 100, 106, 105, 104, 110, 109, 108, 114, 113, 112, 118, 117, 116,
|
|
122, 121, 120, 126, 125, 124, 130, 129, 128, 134, 133, 132, 138, 137, 136,
|
|
142, 141, 140, 146, 145, 144, 150, 149, 148, 154, 153, 152, 158, 157, 156,
|
|
162, 161, 160, 166, 165, 164, 170, 169, 168, 174, 173, 172, 178, 177, 176,
|
|
182, 181, 180, 186, 185, 184, 190, 189, 188, 194, 193, 192, 198, 197, 196,
|
|
202, 201, 200, 206, 205, 204, 210, 209, 208, 214, 213, 212, 218, 217, 216,
|
|
222, 221, 220, 226, 225, 224, 230, 229, 228, 234, 233, 232, 238, 237, 236,
|
|
242, 241, 240, 246, 245, 244, 250, 249, 248, 254, 253, 252,
|
|
};
|
|
|
|
void ARGBToRGB24Row_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
|
|
ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRGB24RowIndices);
|
|
}
|
|
|
|
void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
|
|
ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRAWRowIndices);
|
|
}
|
|
|
|
void DivideRow_16_SVE2(const uint16_t* src_y,
|
|
uint16_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"dup z0.h, %w[scale] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with the same predicates to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z1.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n"
|
|
"umulh z1.h, z1.h, z0.h \n"
|
|
"umulh z2.h, z2.h, z0.h \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"st1h {z1.h}, p0, [%[dst]] \n"
|
|
"st1h {z2.h}, p0, [%[dst], #1, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate a pair of predicates for the final iteration to deal with
|
|
// the tail.
|
|
"whilelt p0.h, wzr, %w[width] \n"
|
|
"whilelt p1.h, %w[vl], %w[width] \n"
|
|
"ld1h {z1.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z2.h}, p1/z, [%[src], #1, mul vl] \n"
|
|
"umulh z1.h, z1.h, z0.h \n"
|
|
"umulh z2.h, z2.h, z0.h \n"
|
|
"st1h {z1.h}, p0, [%[dst]] \n"
|
|
"st1h {z2.h}, p1, [%[dst], #1, mul vl] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src_y), // %[src]
|
|
[dst] "+r"(dst_y), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [scale] "r"(scale) // %[scale]
|
|
: "cc", "memory", "z0", "z1", "z2", "p0", "p1");
|
|
}
|
|
|
|
#define HALFFLOAT_SVE \
|
|
"scvtf z0.s, p0/m, z0.s \n" \
|
|
"scvtf z1.s, p0/m, z1.s \n" \
|
|
"scvtf z2.s, p0/m, z2.s \n" \
|
|
"scvtf z3.s, p0/m, z3.s \n" \
|
|
"fmul z0.s, z0.s, z4.s \n" \
|
|
"fmul z1.s, z1.s, z4.s \n" \
|
|
"fmul z2.s, z2.s, z4.s \n" \
|
|
"fmul z3.s, z3.s, z4.s \n" \
|
|
"fmul z0.s, z0.s, z5.s \n" \
|
|
"fmul z1.s, z1.s, z5.s \n" \
|
|
"fmul z2.s, z2.s, z5.s \n" \
|
|
"fmul z3.s, z3.s, z5.s \n" \
|
|
"uqshrnb z0.h, z0.s, #13 \n" \
|
|
"uqshrnb z1.h, z1.s, #13 \n" \
|
|
"uqshrnb z2.h, z2.s, #13 \n" \
|
|
"uqshrnb z3.h, z3.s, #13 \n"
|
|
|
|
void HalfFloatRow_SVE2(const uint16_t* src,
|
|
uint16_t* dst,
|
|
float scale,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm("cntw %x0" : "=r"(vl));
|
|
asm volatile(
|
|
"mov z4.s, %s[scale] \n"
|
|
"mov z5.s, %s[bias] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #2 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with all-true predicates to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p0.s \n"
|
|
"1: \n"
|
|
"ld1h {z0.s}, p0/z, [%[src]] \n"
|
|
"ld1h {z1.s}, p0/z, [%[src], #1, mul vl] \n"
|
|
"ld1h {z2.s}, p0/z, [%[src], #2, mul vl] \n"
|
|
"ld1h {z3.s}, p0/z, [%[src], #3, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n" //
|
|
HALFFLOAT_SVE
|
|
"subs %w[width], %w[width], %w[vl], lsl #2 \n"
|
|
"st1h {z0.s}, p0, [%[dst]] \n"
|
|
"st1h {z1.s}, p0, [%[dst], #1, mul vl] \n"
|
|
"st1h {z2.s}, p0, [%[dst], #2, mul vl] \n"
|
|
"st1h {z3.s}, p0, [%[dst], #3, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #2 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate predicates for the final iteration to deal with the tail.
|
|
"whilelt p0.s, wzr, %w[width] \n"
|
|
"whilelt p1.s, %w[vl], %w[width] \n"
|
|
"whilelt p2.s, %w[vl2], %w[width] \n"
|
|
"whilelt p3.s, %w[vl3], %w[width] \n"
|
|
"ld1h {z0.s}, p0/z, [%[src]] \n"
|
|
"ld1h {z1.s}, p1/z, [%[src], #1, mul vl] \n"
|
|
"ld1h {z2.s}, p2/z, [%[src], #2, mul vl] \n"
|
|
"ld1h {z3.s}, p3/z, [%[src], #3, mul vl] \n" //
|
|
HALFFLOAT_SVE
|
|
"st1h {z0.s}, p0, [%[dst]] \n"
|
|
"st1h {z1.s}, p1, [%[dst], #1, mul vl] \n"
|
|
"st1h {z2.s}, p2, [%[dst], #2, mul vl] \n"
|
|
"st1h {z3.s}, p3, [%[dst], #3, mul vl] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src), // %[src]
|
|
[dst] "+r"(dst), // %[dst]
|
|
[width] "+r"(width) // %[width]
|
|
: [vl] "r"(vl), // %[vl]
|
|
[vl2] "r"(vl * 2), // %[vl2]
|
|
[vl3] "r"(vl * 3), // %[vl3]
|
|
[scale] "w"(scale), // %[scale]
|
|
[bias] "w"(1.9259299444e-34f) // %[bias] 2^-112
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "p0", "p1", "p2",
|
|
"p3");
|
|
}
|
|
|
|
void HalfFloat1Row_SVE2(const uint16_t* src,
|
|
uint16_t* dst,
|
|
float scale,
|
|
int width) {
|
|
uint64_t vl;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Run bulk of computation with all-true predicates to avoid predicate
|
|
// generation overhead.
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z0.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z1.h}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n"
|
|
"ucvtf z0.h, p0/m, z0.h \n"
|
|
"ucvtf z1.h, p0/m, z1.h \n"
|
|
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"st1h {z0.h}, p0, [%[dst]] \n"
|
|
"st1h {z1.h}, p0, [%[dst], #1, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
|
|
"b.eq 99f \n"
|
|
|
|
// Calculate predicates for the final iteration to deal with the tail.
|
|
"whilelt p0.h, wzr, %w[width] \n"
|
|
"whilelt p1.h, %w[vl], %w[width] \n"
|
|
"ld1h {z0.h}, p0/z, [%[src]] \n"
|
|
"ld1h {z1.h}, p1/z, [%[src], #1, mul vl] \n"
|
|
"ucvtf z0.h, p0/m, z0.h \n"
|
|
"ucvtf z1.h, p0/m, z1.h \n"
|
|
"st1h {z0.h}, p0, [%[dst]] \n"
|
|
"st1h {z1.h}, p1, [%[dst], #1, mul vl] \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src), // %[src]
|
|
[dst] "+r"(dst), // %[dst]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z0", "z1", "p0", "p1");
|
|
}
|
|
|
|
void I210ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I210ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I210AlphaToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
const uint16_t* src_a,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I210AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I210ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I210ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void P210ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_uv,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
P210ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void P210ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_uv,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
P210ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void I410ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I410ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void I410AlphaToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
const uint16_t* src_a,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I410AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
|
|
width);
|
|
}
|
|
|
|
void I410ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I410ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void P410ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_uv,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
P410ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void P410ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_uv,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
P410ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void I212ToAR30Row_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I212ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
|
|
}
|
|
|
|
void I212ToARGBRow_SVE2(const uint16_t* src_y,
|
|
const uint16_t* src_u,
|
|
const uint16_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
I212ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
|
}
|
|
|
|
void Convert8To8Row_SVE2(const uint8_t* src_y,
|
|
uint8_t* dst_y,
|
|
int scale,
|
|
int bias,
|
|
int width) {
|
|
Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width);
|
|
}
|
|
|
|
void MergeUVRow_SVE2(const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_v]] \n"
|
|
"incb %[src_u] \n"
|
|
"incb %[src_v] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
|
|
"incb %[dst_uv], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_v]] \n"
|
|
"st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
|
|
|
|
"99: \n"
|
|
: [src_u] "+r"(src_u), // %[src_u]
|
|
[src_v] "+r"(src_v), // %[src_v]
|
|
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "memory", "cc", "z1", "z2", "p0");
|
|
}
|
|
|
|
void SplitUVRow_SVE2(const uint8_t* src_uv,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n"
|
|
"incb %[src_uv], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z1.b}, p0, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p0, [%[dst_v]] \n"
|
|
"incb %[dst_u] \n"
|
|
"incb %[dst_v] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z1.b, z2.b}, p0/z, [%[src_uv]] \n"
|
|
"st1b {z1.b}, p0, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p0, [%[dst_v]] \n"
|
|
|
|
"99: \n"
|
|
: [src_uv] "+r"(src_uv), // %[src_uv]
|
|
[dst_u] "+r"(dst_u), // %[dst_u]
|
|
[dst_v] "+r"(dst_v), // %[dst_v]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "memory", "cc", "z1", "z2", "p0");
|
|
}
|
|
|
|
void HalfMergeUVRow_SVE2(const uint8_t* src_u,
|
|
int src_stride_u,
|
|
const uint8_t* src_v,
|
|
int src_stride_v,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
const uint8_t* src_u1 = src_u + src_stride_u;
|
|
const uint8_t* src_v1 = src_v + src_stride_v;
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_u1]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_v]] \n"
|
|
"ld1b {z4.b}, p0/z, [%[src_v1]] \n"
|
|
"incb %[src_u] \n"
|
|
"incb %[src_u1] \n"
|
|
"incb %[src_v] \n"
|
|
"incb %[src_v1] \n"
|
|
"uaddlb z5.h, z1.b, z2.b \n"
|
|
"uaddlb z6.h, z3.b, z4.b \n"
|
|
"uaddlt z1.h, z1.b, z2.b \n"
|
|
"uaddlt z3.h, z3.b, z4.b \n"
|
|
"add z1.h, z1.h, z5.h \n"
|
|
"add z3.h, z3.h, z6.h \n"
|
|
"rshrnb z1.b, z1.h, #2 \n"
|
|
"rshrnt z1.b, z3.h, #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z1.b}, p0, [%[dst_uv]] \n"
|
|
"incb %[dst_uv] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_u1]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_v]] \n"
|
|
"ld1b {z4.b}, p0/z, [%[src_v1]] \n"
|
|
"uaddlb z5.h, z1.b, z2.b \n"
|
|
"uaddlb z6.h, z3.b, z4.b \n"
|
|
"uaddlt z1.h, z1.b, z2.b \n"
|
|
"uaddlt z3.h, z3.b, z4.b \n"
|
|
"add z1.h, z1.h, z5.h \n"
|
|
"add z3.h, z3.h, z6.h \n"
|
|
"rshrnb z1.b, z1.h, #2 \n"
|
|
"rshrnt z1.b, z3.h, #2 \n"
|
|
"st1b {z1.b}, p0, [%[dst_uv]] \n"
|
|
|
|
"99: \n"
|
|
: [src_u] "+r"(src_u), // %[src_u]
|
|
[src_u1] "+r"(src_u1), // %[src_u1]
|
|
[src_v] "+r"(src_v), // %[src_v]
|
|
[src_v1] "+r"(src_v1), // %[src_v1]
|
|
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "cc", "memory", "z1", "z2", "z3", "z4", "z5", "z6", "p0");
|
|
}
|
|
|
|
void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) {
|
|
int vl;
|
|
int vl2;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"lsl %w[vl2], %w[vl], #1 \n"
|
|
"subs %w[width], %w[width], %w[vl2] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z0.b}, p0/z, [%[src], #0, mul vl] \n"
|
|
"ld1b {z1.b}, p0/z, [%[src], #1, mul vl] \n"
|
|
"incb %[src], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl2] \n"
|
|
"st1b {z0.b}, p0, [%[dst], #0, mul vl] \n"
|
|
"st1b {z1.b}, p0, [%[dst], #1, mul vl] \n"
|
|
"incb %[dst], all, mul #2 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl2] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"3: \n"
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z0.b}, p0/z, [%[src]] \n"
|
|
"incb %[src] \n"
|
|
"st1b {z0.b}, p0, [%[dst]] \n"
|
|
"incb %[dst] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.gt 3b \n"
|
|
|
|
"99: \n"
|
|
: [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [vl] "=&r"(vl),
|
|
[vl2] "=&r"(vl2)
|
|
:
|
|
: "cc", "memory", "z0", "z1", "p0");
|
|
}
|
|
|
|
void InterpolateRow_SVE2(uint8_t* dst_ptr,
|
|
const uint8_t* src_ptr,
|
|
ptrdiff_t src_stride,
|
|
int width,
|
|
int source_y_fraction) {
|
|
int y1_fraction = source_y_fraction;
|
|
int y0_fraction = 256 - y1_fraction;
|
|
const uint8_t* src_ptr1 = src_ptr + src_stride;
|
|
|
|
if (y0_fraction == 0) {
|
|
CopyRow_SVE2(src_ptr1, dst_ptr, width);
|
|
return;
|
|
}
|
|
if (y0_fraction == 256) {
|
|
CopyRow_SVE2(src_ptr, dst_ptr, width);
|
|
return;
|
|
}
|
|
if (y0_fraction == 128) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
|
"incb %[src_ptr] \n"
|
|
"incb %[src_ptr1] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"urhadd z2.b, p0/m, z2.b, z3.b \n"
|
|
"st1b {z2.b}, p0, [%[dst_ptr]] \n"
|
|
"incb %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
|
"urhadd z2.b, p0/m, z2.b, z3.b \n"
|
|
"st1b {z2.b}, p0, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_ptr), [src_ptr1] "+r"(src_ptr1),
|
|
[dst_ptr] "+r"(dst_ptr), [width] "+r"(width), [vl] "=&r"(vl)
|
|
:
|
|
: "cc", "memory", "z2", "z3", "p0");
|
|
return;
|
|
}
|
|
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"dup z0.b, %w[y0_fraction] \n"
|
|
"dup z1.b, %w[y1_fraction] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
|
"incb %[src_ptr] \n"
|
|
"incb %[src_ptr1] \n"
|
|
"umullb z4.h, z2.b, z0.b \n"
|
|
"umullt z2.h, z2.b, z0.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"umlalb z4.h, z3.b, z1.b \n"
|
|
"umlalt z2.h, z3.b, z1.b \n"
|
|
"rshrnb z3.b, z4.h, #8 \n"
|
|
"rshrnt z3.b, z2.h, #8 \n"
|
|
"st1b {z3.b}, p0, [%[dst_ptr]] \n"
|
|
"incb %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
|
|
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
|
|
"umullb z4.h, z2.b, z0.b \n"
|
|
"umullt z2.h, z2.b, z0.b \n"
|
|
"umlalb z4.h, z3.b, z1.b \n"
|
|
"umlalt z2.h, z3.b, z1.b \n"
|
|
"rshrnb z3.b, z4.h, #8 \n"
|
|
"rshrnt z3.b, z2.h, #8 \n"
|
|
"st1b {z3.b}, p0, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
|
|
[src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
|
|
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
|
|
[y1_fraction] "r"(y1_fraction) // %[y1_fraction]
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
|
|
}
|
|
|
|
void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
|
|
"incb %[src_yuy2], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z0.b}, p0, [%[dst_y]] \n"
|
|
"incb %[dst_y] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
|
|
"st1b {z0.b}, p0, [%[dst_y]] \n"
|
|
|
|
"99: \n"
|
|
: [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
|
|
[dst_y] "+r"(dst_y), // %[dst_y]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "memory", "cc", "z0", "z1", "p0");
|
|
}
|
|
|
|
void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
|
|
int vl;
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
|
|
"incb %[src_uyvy], all, mul #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z1.b}, p0, [%[dst_y]] \n"
|
|
"incb %[dst_y] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.b, wzr, %w[width] \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
|
|
"st1b {z1.b}, p0, [%[dst_y]] \n"
|
|
|
|
"99: \n"
|
|
: [src_uyvy] "+r"(src_uyvy), // %[src_uyvy]
|
|
[dst_y] "+r"(dst_y), // %[dst_y]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
:
|
|
: "memory", "cc", "z0", "z1", "p0");
|
|
}
|
|
|
|
void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2,
|
|
int src_stride_yuy2,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2;
|
|
int vl;
|
|
int vl_half;
|
|
int width_half;
|
|
if (src_stride_yuy2 == 0) {
|
|
src_yuy2_1 = src_yuy2;
|
|
}
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"lsr %w[vl_half], %w[vl], #1 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"whilelt p1.b, wzr, %w[vl_half] \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
|
|
"ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n"
|
|
"incb %[src_yuy2], all, mul #2 \n"
|
|
"incb %[src_yuy2_1], all, mul #2 \n"
|
|
"urhadd z1.b, p0/m, z1.b, z5.b \n"
|
|
"uzp1 z0.b, z1.b, z1.b \n"
|
|
"uzp2 z2.b, z1.b, z1.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z0.b}, p1, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p1, [%[dst_v]] \n"
|
|
"add %[dst_u], %[dst_u], %x[vl_half] \n"
|
|
"add %[dst_v], %[dst_v], %x[vl_half] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"add %w[width_half], %w[width], #1 \n"
|
|
"lsr %w[width_half], %w[width_half], #1 \n"
|
|
"whilelt p1.b, wzr, %w[width_half] \n"
|
|
"zip1 p0.b, p1.b, p1.b \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_yuy2]] \n"
|
|
"ld2b {z4.b, z5.b}, p0/z, [%[src_yuy2_1]] \n"
|
|
"urhadd z1.b, p0/m, z1.b, z5.b \n"
|
|
"uzp1 z0.b, z1.b, z1.b \n"
|
|
"uzp2 z2.b, z1.b, z1.b \n"
|
|
"st1b {z0.b}, p1, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p1, [%[dst_v]] \n"
|
|
|
|
"99: \n"
|
|
: [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
|
|
[src_yuy2_1] "+r"(src_yuy2_1), // %[src_yuy2_1]
|
|
[dst_u] "+r"(dst_u), // %[dst_u]
|
|
[dst_v] "+r"(dst_v), // %[dst_v]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl), // %[vl]
|
|
[vl_half] "=&r"(vl_half), // %[vl_half]
|
|
[width_half] "=&r"(width_half) // %[width_half]
|
|
:
|
|
: "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
|
|
}
|
|
|
|
void UYVYToUVRow_SVE2(const uint8_t* src_uyvy,
|
|
int src_stride_uyvy,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
const uint8_t* src_uyvy_1 = src_uyvy + src_stride_uyvy;
|
|
int vl;
|
|
int vl_half;
|
|
int width_half;
|
|
if (src_stride_uyvy == 0) {
|
|
src_uyvy_1 = src_uyvy;
|
|
}
|
|
asm volatile(
|
|
"cntb %x[vl] \n"
|
|
"lsr %w[vl_half], %w[vl], #1 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.b \n"
|
|
"whilelt p1.b, wzr, %w[vl_half] \n"
|
|
"1: \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
|
|
"ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n"
|
|
"incb %[src_uyvy], all, mul #2 \n"
|
|
"incb %[src_uyvy_1], all, mul #2 \n"
|
|
"urhadd z0.b, p0/m, z0.b, z4.b \n"
|
|
"uzp1 z1.b, z0.b, z0.b \n"
|
|
"uzp2 z2.b, z0.b, z0.b \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"st1b {z1.b}, p1, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p1, [%[dst_v]] \n"
|
|
"add %[dst_u], %[dst_u], %x[vl_half] \n"
|
|
"add %[dst_v], %[dst_v], %x[vl_half] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"add %w[width_half], %w[width], #1 \n"
|
|
"lsr %w[width_half], %w[width_half], #1 \n"
|
|
"whilelt p1.b, wzr, %w[width_half] \n"
|
|
"zip1 p0.b, p1.b, p1.b \n"
|
|
"ld2b {z0.b, z1.b}, p0/z, [%[src_uyvy]] \n"
|
|
"ld2b {z4.b, z5.b}, p0/z, [%[src_uyvy_1]] \n"
|
|
"urhadd z0.b, p0/m, z0.b, z4.b \n"
|
|
"uzp1 z1.b, z0.b, z0.b \n"
|
|
"uzp2 z2.b, z0.b, z0.b \n"
|
|
"st1b {z1.b}, p1, [%[dst_u]] \n"
|
|
"st1b {z2.b}, p1, [%[dst_v]] \n"
|
|
|
|
"99: \n"
|
|
: [src_uyvy] "+r"(src_uyvy), // %[src_uyvy]
|
|
[src_uyvy_1] "+r"(src_uyvy_1), // %[src_uyvy_1]
|
|
[dst_u] "+r"(dst_u), // %[dst_u]
|
|
[dst_v] "+r"(dst_v), // %[dst_v]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl), // %[vl]
|
|
[vl_half] "=&r"(vl_half), // %[vl_half]
|
|
[width_half] "=&r"(width_half) // %[width_half]
|
|
:
|
|
: "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
|
|
}
|
|
|
|
#ifdef HAS_CONVERT16TO8ROW_SVE2
|
|
// Disabled: NEON version is faster for 128 bit vectors.
|
|
void Convert16To8Row_SVE2(const uint16_t* src_y,
|
|
uint8_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
const int shift = 23 - __builtin_clz((int32_t)scale);
|
|
int vl;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"dup z31.h, %w[shift] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
|
|
"incb %[src_ptr] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"uqshl z2.h, p0/m, z2.h, z31.h \n"
|
|
"shrnb z2.b, z2.h, #8 \n"
|
|
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
|
|
"inch %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.h, wzr, %w[width] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
|
|
"uqshl z2.h, p0/m, z2.h, z31.h \n"
|
|
"shrnb z2.b, z2.h, #8 \n"
|
|
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_y), // %[src_ptr]
|
|
[dst_ptr] "+r"(dst_y), // %[dst_ptr]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [shift] "r"(shift) // %[shift]
|
|
: "cc", "memory", "z2", "z31", "p0");
|
|
}
|
|
#endif // HAS_CONVERT16TO8ROW_SVE2
|
|
|
|
#ifdef HAS_HALFROW_16TO8_SVE2
|
|
// Disabled: NEON version is faster for 128 bit vectors.
|
|
void HalfRow_16To8_SVE2(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int scale,
|
|
int width) {
|
|
const uint16_t* src_uv1 = src_uv + src_uv_stride;
|
|
const int shift = 23 - __builtin_clz((int32_t)scale);
|
|
int vl;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"dup z31.h, %w[shift] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
|
|
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
|
|
"incb %[src_ptr] \n"
|
|
"incb %[src_ptr1] \n"
|
|
"urhadd z2.h, p0/m, z2.h, z3.h \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"uqshl z2.h, p0/m, z2.h, z31.h \n"
|
|
"shrnb z2.b, z2.h, #8 \n"
|
|
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
|
|
"inch %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"whilelt p0.h, wzr, %w[width] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
|
|
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
|
|
"urhadd z2.h, p0/m, z2.h, z3.h \n"
|
|
"uqshl z2.h, p0/m, z2.h, z31.h \n"
|
|
"shrnb z2.b, z2.h, #8 \n"
|
|
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_uv), // %[src_ptr]
|
|
[src_ptr1] "+r"(src_uv1), // %[src_ptr1]
|
|
[dst_ptr] "+r"(dst_uv), // %[dst_ptr]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl) // %[vl]
|
|
: [shift] "r"(shift) // %[shift]
|
|
: "cc", "memory", "z2", "z3", "z31", "p0");
|
|
}
|
|
#endif // HAS_HALFROW_16TO8_SVE2
|
|
|
|
#ifdef HAS_HALFWIDTHROW_16TO8_SVE2
|
|
// Disabled: NEON version is faster for 128 bit vectors.
|
|
void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv,
|
|
ptrdiff_t src_uv_stride,
|
|
uint8_t* dst_uv,
|
|
int scale,
|
|
int width) {
|
|
const uint16_t* src_uv1 = src_uv + src_uv_stride;
|
|
const int shift = 23 - __builtin_clz((int32_t)scale);
|
|
int vl;
|
|
int vl_input;
|
|
asm volatile(
|
|
"cnth %x[vl] \n"
|
|
"dup z31.h, %w[shift] \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"b.lt 2f \n"
|
|
|
|
"ptrue p0.h \n"
|
|
"1: \n"
|
|
"ld1h {z0.h}, p0/z, [%[src_ptr]] \n"
|
|
"ld1h {z1.h}, p0/z, [%[src_ptr], #1, mul vl] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr1]] \n"
|
|
"ld1h {z3.h}, p0/z, [%[src_ptr1], #1, mul vl] \n"
|
|
"incb %[src_ptr], all, mul #4 \n"
|
|
"incb %[src_ptr1], all, mul #4 \n"
|
|
"uzp1 z4.h, z0.h, z1.h \n"
|
|
"uzp2 z5.h, z0.h, z1.h \n"
|
|
"uzp1 z6.h, z2.h, z3.h \n"
|
|
"uzp2 z7.h, z2.h, z3.h \n"
|
|
"uaddlb z0.s, z4.h, z5.h \n"
|
|
"uaddlt z1.s, z4.h, z5.h \n"
|
|
"uaddlb z2.s, z6.h, z7.h \n"
|
|
"uaddlt z3.s, z6.h, z7.h \n"
|
|
"add z0.s, z0.s, z2.s \n"
|
|
"add z1.s, z1.s, z3.s \n"
|
|
"rshrnb z0.h, z0.s, #2 \n"
|
|
"rshrnt z0.h, z1.s, #2 \n"
|
|
"subs %w[width], %w[width], %w[vl] \n"
|
|
"uqshl z0.h, p0/m, z0.h, z31.h \n"
|
|
"shrnb z0.b, z0.h, #8 \n"
|
|
"st1b {z0.h}, p0, [%[dst_ptr]] \n"
|
|
"inch %[dst_ptr] \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], %w[vl] \n"
|
|
"b.eq 99f \n"
|
|
|
|
"lsl %w[vl_input], %w[width], #1 \n"
|
|
"whilelt p0.h, wzr, %w[vl_input] \n"
|
|
"whilelt p1.h, %w[vl], %w[vl_input] \n"
|
|
"whilelt p2.h, wzr, %w[width] \n"
|
|
"ld1h {z0.h}, p0/z, [%[src_ptr]] \n"
|
|
"ld1h {z1.h}, p1/z, [%[src_ptr], #1, mul vl] \n"
|
|
"ld1h {z2.h}, p0/z, [%[src_ptr1]] \n"
|
|
"ld1h {z3.h}, p1/z, [%[src_ptr1], #1, mul vl] \n"
|
|
"uzp1 z4.h, z0.h, z1.h \n"
|
|
"uzp2 z5.h, z0.h, z1.h \n"
|
|
"uzp1 z6.h, z2.h, z3.h \n"
|
|
"uzp2 z7.h, z2.h, z3.h \n"
|
|
"uaddlb z0.s, z4.h, z5.h \n"
|
|
"uaddlt z1.s, z4.h, z5.h \n"
|
|
"uaddlb z2.s, z6.h, z7.h \n"
|
|
"uaddlt z3.s, z6.h, z7.h \n"
|
|
"add z0.s, z0.s, z2.s \n"
|
|
"add z1.s, z1.s, z3.s \n"
|
|
"rshrnb z0.h, z0.s, #2 \n"
|
|
"rshrnt z0.h, z1.s, #2 \n"
|
|
"uqshl z0.h, p2/m, z0.h, z31.h \n"
|
|
"shrnb z0.b, z0.h, #8 \n"
|
|
"st1b {z0.h}, p2, [%[dst_ptr]] \n"
|
|
|
|
"99: \n"
|
|
: [src_ptr] "+r"(src_uv), // %[src_ptr]
|
|
[src_ptr1] "+r"(src_uv1), // %[src_ptr1]
|
|
[dst_ptr] "+r"(dst_uv), // %[dst_ptr]
|
|
[width] "+r"(width), // %[width]
|
|
[vl] "=&r"(vl), // %[vl]
|
|
[vl_input] "=&r"(vl_input) // %[vl_input]
|
|
: [shift] "r"(shift) // %[shift]
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31",
|
|
"p0", "p1", "p2");
|
|
}
|
|
#endif // HAS_HALFWIDTHROW_16TO8_SVE2
|
|
|
|
#ifdef HAS_SSIM4X4ROW_SVE2
|
|
void Ssim4x4Row_SVE2(const uint8_t* src_a,
|
|
int stride_a,
|
|
const uint8_t* src_b,
|
|
int stride_b,
|
|
uint32_t* dst,
|
|
int width) {
|
|
Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width);
|
|
}
|
|
#endif // HAS_SSIM4X4ROW_SVE2
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|