coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity
libyuv/source/rotate_sve.cc
Frank Barchard 16af162cf9 [libyuv] Optimize RotatePlane_16 SIMD and compact 32-bit transposes 
Implement SIMD 16-bit plane rotation (TransposePlane_16,
RotatePlane90_16, RotatePlane180_16, RotatePlane270_16) used by
I010Rotate, I210Rotate, and I410Rotate, keeping tile sizes compact
with full-register writes and zero cross-lane shuffles:
- AVX2: 8x16 tile in ymm0..ymm15 with _Any tail (low-end x86 SIMD)
- AVX-512BW: 8x32 tile in zmm0..zmm15 with k1/k2 right/bottom masks
- NEON (ARM64/ARM32): 8x16 tile in v0..v23 / q0..q11 with _Any tail
- SME: 8xVL tile using ZA16 horizontal/vertical array slices
- RVV: 8xVL tile using widening unpacks with hardware vl tail
- WASM SIMD: 8x8 tile in v128 with _Any tail

Also update 32-bit ARGBRotate x86 transposes to compact vertical
in-lane tiles with full-register writes and zero cross-lane shuffles:
- TransposeWxH_32_AVX2: 4x8 tile in ymm0..ymm7 with vpmaskmovd tails
- TransposeWxH_32_AVX512BW: 4x16 tile in zmm0..zmm7 with k1/k2 tails

Add RotatePlane16_SplitMergeUV_Opt and RotatePlane16_SplitMergeUV_Odd
unit tests comparing 16-bit rotation against SplitUVPlane +
RotatePlane + MergeUVPlane across 0/90/180/270 degrees.

Performance on 1275x1650 (1000 iterations, before -> after):
CPU                     90 degree               180 degree
Coral RVV (Verilator)   7306 -> 1415c (5.16x)   2274 -> 719c (3.16x)
AMD Rome (Zen 2)        821 -> 215ms (3.82x)    259 -> 181ms (1.44x)
AMD Genoa (Zen 4)       483 -> 129ms (3.73x)    149 -> 111ms (1.34x)
AMD Turin (Zen 5)       321 -> 101ms (3.18x)    109 -> 78ms (1.40x)
AMD Milan (Zen 3)       505 -> 189ms (2.67x)    199 -> 118ms (1.69x)
Intel Skylake-X         1076 -> 447ms (2.41x)   353 -> 317ms (1.11x)
WASM SIMD (d8)          415 -> 203ms (2.04x)    370 -> 106ms (3.49x)
Intel Emerald Rapids    735 -> 422ms (1.74x)    238 -> 233ms (1.02x)
Intel Haswell           1120 -> 648ms (1.73x)   438 -> 393ms (1.11x)
Intel Ice Lake          774 -> 474ms (1.63x)    279 -> 290ms (0.96x)
ARM Neoverse-N1         864 -> 544ms (1.59x)    404 -> 266ms (1.52x)
Ampere Siryn            862 -> 576ms (1.50x)    568 -> 329ms (1.72x)
Google Axion (Maple)    554 -> 379ms (1.46x)    199 -> 142ms (1.41x)
Intel Sapphire Rapids   1126 -> 772ms (1.46x)   301 -> 288ms (1.04x)
Intel Granite Rapids    730 -> 565ms (1.29x)    280 -> 280ms (1.00x)
NVIDIA Grace            462 -> 365ms (1.26x)    192 -> 126ms (1.52x)
ARM Cortex-A53          28290 -> 24386ms (1.16x) 7178 -> 5075ms (1.41x)
Google Axion 2 (Tamar)  523 -> 453ms (1.15x)    217 -> 112ms (1.93x)

Test: libyuv_unittest --gtest_filter=*Rotate*
Bug: b/42280942
Change-Id: I91ba6cac22185b5d294fcb98344fdb8a23cc94ff
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8545294
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
2026-10-12 10:57:34 -07:00

280 lines
13 KiB
C++
Raw Blame History

/*
* Copyright 2026 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
// Transpose 32 bit values (ARGB) in Wx4 tiles using SVE2 with predication
// for tail widths.
void TransposeWx4_32_SVE2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t s4 = s * 4;
const ptrdiff_t d = dst_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
asm volatile(
"ptrue p0.s, vl4 \n"
"subs %w[width], %w[width], #4 \n"
"b.lt 2f \n"
// Main loop: full 4x4 blocks.
"1: \n"
"ld1w {z0.s}, p0/z, [%[src0]] \n"
"ld1w {z1.s}, p0/z, [%[src1]] \n"
"ld1w {z2.s}, p0/z, [%[src2]] \n"
"ld1w {z3.s}, p0/z, [%[src3]] \n"
"add %[src0], %[src0], %[s4] \n"
"add %[src1], %[src1], %[s4] \n"
"add %[src2], %[src2], %[s4] \n"
"add %[src3], %[src3], %[s4] \n"
"trn1 z4.s, z0.s, z1.s \n"
"trn2 z5.s, z0.s, z1.s \n"
"trn1 z6.s, z2.s, z3.s \n"
"trn2 z7.s, z2.s, z3.s \n"
"trn1 z0.d, z4.d, z6.d \n"
"trn2 z1.d, z4.d, z6.d \n"
"trn1 z2.d, z5.d, z7.d \n"
"trn2 z3.d, z5.d, z7.d \n"
"subs %w[width], %w[width], #4 \n"
"st1w {z0.s}, p0, [%[dst0]] \n"
"st1w {z1.s}, p0, [%[dst1]] \n"
"st1w {z2.s}, p0, [%[dst2]] \n"
"st1w {z3.s}, p0, [%[dst3]] \n"
"add %[dst0], %[dst0], #16 \n"
"add %[dst1], %[dst1], #16 \n"
"add %[dst2], %[dst2], #16 \n"
"add %[dst3], %[dst3], #16 \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], #4 \n"
"b.eq 3f \n"
// Tail loop: 1..3 remaining source rows / destination columns.
"cmp %w[width], #1 \n"
"csel %[src1], %[src0], %[src1], eq \n"
"cmp %w[width], #2 \n"
"csel %[src2], %[src1], %[src2], le \n"
"mov %[src3], %[src2] \n"
"ld1w {z0.s}, p0/z, [%[src0]] \n"
"ld1w {z1.s}, p0/z, [%[src1]] \n"
"ld1w {z2.s}, p0/z, [%[src2]] \n"
"ld1w {z3.s}, p0/z, [%[src3]] \n"
"trn1 z4.s, z0.s, z1.s \n"
"trn2 z5.s, z0.s, z1.s \n"
"trn1 z6.s, z2.s, z3.s \n"
"trn2 z7.s, z2.s, z3.s \n"
"trn1 z0.d, z4.d, z6.d \n"
"trn2 z1.d, z4.d, z6.d \n"
"trn1 z2.d, z5.d, z7.d \n"
"trn2 z3.d, z5.d, z7.d \n"
"whilelt p0.s, wzr, %w[width] \n"
"st1w {z0.s}, p0, [%[dst0]] \n"
"st1w {z1.s}, p0, [%[dst1]] \n"
"st1w {z2.s}, p0, [%[dst2]] \n"
"st1w {z3.s}, p0, [%[dst3]] \n"
"3: \n"
: [src0] "+r"(src), // %[src0]
[src1] "+r"(src1), // %[src1]
[src2] "+r"(src2), // %[src2]
[src3] "+r"(src3), // %[src3]
[dst0] "+r"(dst), // %[dst0]
[dst1] "+r"(dst1), // %[dst1]
[dst2] "+r"(dst2), // %[dst2]
[dst3] "+r"(dst3), // %[dst3]
[width] "+r"(width) // %[width]
: [s4] "r"(s4) // %[s4]
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
}
void TransposeWx8_16_SVE2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint16_t* temp;
asm volatile(
"ptrue p0.h, vl8 \n"
"subs %w[w], %w[w], #8 \n"
"b.lt 2f \n"
"1: \n"
"mov %[temp], %[src] \n"
"ld1h {z0.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z1.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z2.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z3.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z4.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z5.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z6.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z7.h}, p0/z, [%[temp]] \n"
"add %[src], %[src], #16 \n"
"trn1 z16.h, z0.h, z1.h \n"
"trn2 z17.h, z0.h, z1.h \n"
"trn1 z18.h, z2.h, z3.h \n"
"trn2 z19.h, z2.h, z3.h \n"
"trn1 z20.h, z4.h, z5.h \n"
"trn2 z21.h, z4.h, z5.h \n"
"trn1 z22.h, z6.h, z7.h \n"
"trn2 z23.h, z6.h, z7.h \n"
"trn1 z0.s, z16.s, z18.s \n"
"trn1 z1.s, z17.s, z19.s \n"
"trn2 z2.s, z16.s, z18.s \n"
"trn2 z3.s, z17.s, z19.s \n"
"trn1 z4.s, z20.s, z22.s \n"
"trn1 z5.s, z21.s, z23.s \n"
"trn2 z6.s, z20.s, z22.s \n"
"trn2 z7.s, z21.s, z23.s \n"
"trn1 z16.d, z0.d, z4.d \n"
"trn1 z17.d, z1.d, z5.d \n"
"trn1 z18.d, z2.d, z6.d \n"
"trn1 z19.d, z3.d, z7.d \n"
"trn2 z20.d, z0.d, z4.d \n"
"trn2 z21.d, z1.d, z5.d \n"
"trn2 z22.d, z2.d, z6.d \n"
"trn2 z23.d, z3.d, z7.d \n"
"st1h {z16.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z17.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z18.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z19.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z20.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z21.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z22.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z23.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"subs %w[w], %w[w], #8 \n"
"b.ge 1b \n"
"2: \n"
"adds %w[w], %w[w], #8 \n"
"b.eq 3f \n"
"whilelt p1.h, wzr, %w[w] \n"
"mov %[temp], %[src] \n"
"ld1h {z0.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z1.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z2.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z3.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z4.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z5.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z6.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z7.h}, p1/z, [%[temp]] \n"
"trn1 z16.h, z0.h, z1.h \n"
"trn2 z17.h, z0.h, z1.h \n"
"trn1 z18.h, z2.h, z3.h \n"
"trn2 z19.h, z2.h, z3.h \n"
"trn1 z20.h, z4.h, z5.h \n"
"trn2 z21.h, z4.h, z5.h \n"
"trn1 z22.h, z6.h, z7.h \n"
"trn2 z23.h, z6.h, z7.h \n"
"trn1 z0.s, z16.s, z18.s \n"
"trn1 z1.s, z17.s, z19.s \n"
"trn2 z2.s, z16.s, z18.s \n"
"trn2 z3.s, z17.s, z19.s \n"
"trn1 z4.s, z20.s, z22.s \n"
"trn1 z5.s, z21.s, z23.s \n"
"trn2 z6.s, z20.s, z22.s \n"
"trn2 z7.s, z21.s, z23.s \n"
"trn1 z16.d, z0.d, z4.d \n"
"trn1 z17.d, z1.d, z5.d \n"
"trn1 z18.d, z2.d, z6.d \n"
"trn1 z19.d, z3.d, z7.d \n"
"trn2 z20.d, z0.d, z4.d \n"
"trn2 z21.d, z1.d, z5.d \n"
"trn2 z22.d, z2.d, z6.d \n"
"trn2 z23.d, z3.d, z7.d \n"
"st1h {z16.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z17.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z18.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z19.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z20.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z21.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z22.h}, p0, [%[dst]] \n"
"3: \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[w] "+r"(width), // %[w]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "p0", "p1", "z0", "z1", "z2", "z3", "z4", "z5", "z6",
"z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23");
}
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
Reference in New Issue View Git Blame Copy Permalink