coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-29 02:17:13 +08:00
Code Issues Packages Projects Releases Wiki Activity
libyuv/source/rotate_neon.cc
Frank Barchard 16af162cf9 [libyuv] Optimize RotatePlane_16 SIMD and compact 32-bit transposes 
Implement SIMD 16-bit plane rotation (TransposePlane_16,
RotatePlane90_16, RotatePlane180_16, RotatePlane270_16) used by
I010Rotate, I210Rotate, and I410Rotate, keeping tile sizes compact
with full-register writes and zero cross-lane shuffles:
- AVX2: 8x16 tile in ymm0..ymm15 with _Any tail (low-end x86 SIMD)
- AVX-512BW: 8x32 tile in zmm0..zmm15 with k1/k2 right/bottom masks
- NEON (ARM64/ARM32): 8x16 tile in v0..v23 / q0..q11 with _Any tail
- SME: 8xVL tile using ZA16 horizontal/vertical array slices
- RVV: 8xVL tile using widening unpacks with hardware vl tail
- WASM SIMD: 8x8 tile in v128 with _Any tail

Also update 32-bit ARGBRotate x86 transposes to compact vertical
in-lane tiles with full-register writes and zero cross-lane shuffles:
- TransposeWxH_32_AVX2: 4x8 tile in ymm0..ymm7 with vpmaskmovd tails
- TransposeWxH_32_AVX512BW: 4x16 tile in zmm0..zmm7 with k1/k2 tails

Add RotatePlane16_SplitMergeUV_Opt and RotatePlane16_SplitMergeUV_Odd
unit tests comparing 16-bit rotation against SplitUVPlane +
RotatePlane + MergeUVPlane across 0/90/180/270 degrees.

Performance on 1275x1650 (1000 iterations, before -> after):
CPU                     90 degree               180 degree
Coral RVV (Verilator)   7306 -> 1415c (5.16x)   2274 -> 719c (3.16x)
AMD Rome (Zen 2)        821 -> 215ms (3.82x)    259 -> 181ms (1.44x)
AMD Genoa (Zen 4)       483 -> 129ms (3.73x)    149 -> 111ms (1.34x)
AMD Turin (Zen 5)       321 -> 101ms (3.18x)    109 -> 78ms (1.40x)
AMD Milan (Zen 3)       505 -> 189ms (2.67x)    199 -> 118ms (1.69x)
Intel Skylake-X         1076 -> 447ms (2.41x)   353 -> 317ms (1.11x)
WASM SIMD (d8)          415 -> 203ms (2.04x)    370 -> 106ms (3.49x)
Intel Emerald Rapids    735 -> 422ms (1.74x)    238 -> 233ms (1.02x)
Intel Haswell           1120 -> 648ms (1.73x)   438 -> 393ms (1.11x)
Intel Ice Lake          774 -> 474ms (1.63x)    279 -> 290ms (0.96x)
ARM Neoverse-N1         864 -> 544ms (1.59x)    404 -> 266ms (1.52x)
Ampere Siryn            862 -> 576ms (1.50x)    568 -> 329ms (1.72x)
Google Axion (Maple)    554 -> 379ms (1.46x)    199 -> 142ms (1.41x)
Intel Sapphire Rapids   1126 -> 772ms (1.46x)   301 -> 288ms (1.04x)
Intel Granite Rapids    730 -> 565ms (1.29x)    280 -> 280ms (1.00x)
NVIDIA Grace            462 -> 365ms (1.26x)    192 -> 126ms (1.52x)
ARM Cortex-A53          28290 -> 24386ms (1.16x) 7178 -> 5075ms (1.41x)
Google Axion 2 (Tamar)  523 -> 453ms (1.15x)    217 -> 112ms (1.93x)

Test: libyuv_unittest --gtest_filter=*Rotate*
Bug: b/42280942
Change-Id: I91ba6cac22185b5d294fcb98344fdb8a23cc94ff
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8545294
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
2026-10-12 10:57:34 -07:00

486 lines
21 KiB
C++
Raw Blame History

/*
* Copyright 2011 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
// Note: every register written by an inline asm block (vector, general
// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
// listed in its clobber list, even when the ABI treats it as caller-saved.
// With LTO/LTCG the compiler may keep values live in any register across the
// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
// are callee-saved. List "memory" first, then "cc", then registers.
#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
!defined(__aarch64__)
void TransposeWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
"sub %[width], #8 \n"
"1: \n"
"mov %[temp], %[src] \n"
"vld1.8 {d0}, [%[temp]], %[src_stride] \n"
"vld1.8 {d1}, [%[temp]], %[src_stride] \n"
"vld1.8 {d2}, [%[temp]], %[src_stride] \n"
"vld1.8 {d3}, [%[temp]], %[src_stride] \n"
"vld1.8 {d4}, [%[temp]], %[src_stride] \n"
"vld1.8 {d5}, [%[temp]], %[src_stride] \n"
"vld1.8 {d6}, [%[temp]], %[src_stride] \n"
"vld1.8 {d7}, [%[temp]] \n"
"add %[src], #8 \n"
"vtrn.8 d1, d0 \n"
"vtrn.8 d3, d2 \n"
"vtrn.8 d5, d4 \n"
"vtrn.8 d7, d6 \n"
"subs %[width], #8 \n"
"vtrn.16 d1, d3 \n"
"vtrn.16 d0, d2 \n"
"vtrn.16 d5, d7 \n"
"vtrn.16 d4, d6 \n"
"vtrn.32 d1, d5 \n"
"vtrn.32 d0, d4 \n"
"vtrn.32 d3, d7 \n"
"vtrn.32 d2, d6 \n"
"vrev16.8 q0, q0 \n"
"vrev16.8 q1, q1 \n"
"vrev16.8 q2, q2 \n"
"vrev16.8 q3, q3 \n"
"mov %[temp], %[dst] \n"
"vst1.8 {d1}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d0}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d3}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d2}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d5}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d4}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d7}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d6}, [%[temp]] \n"
"add %[dst], %[dst], %[dst_stride], lsl #3 \n"
"bge 1b \n"
: [temp] "=&r"(temp), // %[temp]
[src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[width] "+r"(width) // %[width]
: [src_stride] "r"(src_stride), // %[src_stride]
[dst_stride] "r"(dst_stride) // %[dst_stride]
: "memory", "cc", "q0", "q1", "q2", "q3");
}
void TransposeUVWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width) {
const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
"sub %[width], #8 \n"
"1: \n"
"mov %[temp], %[src] \n"
"vld2.8 {d0, d1}, [%[temp]], %[src_stride] \n"
"vld2.8 {d2, d3}, [%[temp]], %[src_stride] \n"
"vld2.8 {d4, d5}, [%[temp]], %[src_stride] \n"
"vld2.8 {d6, d7}, [%[temp]], %[src_stride] \n"
"vld2.8 {d16, d17}, [%[temp]], %[src_stride] \n"
"vld2.8 {d18, d19}, [%[temp]], %[src_stride] \n"
"vld2.8 {d20, d21}, [%[temp]], %[src_stride] \n"
"vld2.8 {d22, d23}, [%[temp]] \n"
"add %[src], #8*2 \n"
"vtrn.8 q1, q0 \n"
"vtrn.8 q3, q2 \n"
"vtrn.8 q9, q8 \n"
"vtrn.8 q11, q10 \n"
"subs %[width], #8 \n"
"vtrn.16 q1, q3 \n"
"vtrn.16 q0, q2 \n"
"vtrn.16 q9, q11 \n"
"vtrn.16 q8, q10 \n"
"vtrn.32 q1, q9 \n"
"vtrn.32 q0, q8 \n"
"vtrn.32 q3, q11 \n"
"vtrn.32 q2, q10 \n"
"vrev16.8 q0, q0 \n"
"vrev16.8 q1, q1 \n"
"vrev16.8 q2, q2 \n"
"vrev16.8 q3, q3 \n"
"vrev16.8 q8, q8 \n"
"vrev16.8 q9, q9 \n"
"vrev16.8 q10, q10 \n"
"vrev16.8 q11, q11 \n"
"mov %[temp], %[dst_a] \n"
"vst1.8 {d2}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d0}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d6}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d4}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d18}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d16}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d22}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d20}, [%[temp]] \n"
"add %[dst_a], %[dst_a], %[dst_stride_a], lsl #3 \n"
"mov %[temp], %[dst_b] \n"
"vst1.8 {d3}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d1}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d7}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d5}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d19}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d17}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d23}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d21}, [%[temp]] \n"
"add %[dst_b], %[dst_b], %[dst_stride_b], lsl #3 \n"
"bge 1b \n"
: [temp] "=&r"(temp), // %[temp]
[src] "+r"(src), // %[src]
[dst_a] "+r"(dst_a), // %[dst_a]
[dst_b] "+r"(dst_b), // %[dst_b]
[width] "+r"(width) // %[width]
: [src_stride] "r"(src_stride), // %[src_stride]
[dst_stride_a] "r"(dst_stride_a), // %[dst_stride_a]
[dst_stride_b] "r"(dst_stride_b) // %[dst_stride_b]
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
}
// Transpose 32 bit values (ARGB) in Wx4 tiles using vld4.32 with lane insertion
// and lane stores for 1..3 remainder rows.
void TransposeWx4_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
const ptrdiff_t d = dst_stride;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
asm volatile(
"subs %8, %8, #4 \n"
"blt 2f \n"
// Main loop transpose 4x4. Read a column, write a row.
"1: \n"
"vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0], %9 \n"
"vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1], %9 \n"
"vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2], %9 \n"
"vld4.32 {d1[1], d3[1], d5[1], d7[1]}, [%3], %9 \n"
"subs %8, %8, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"vst1.8 {q1}, [%5]! \n"
"vst1.8 {q2}, [%6]! \n"
"vst1.8 {q3}, [%7]! \n"
"bge 1b \n"
"2: \n"
"adds %8, %8, #4 \n"
"beq 5f \n"
// Remainder of 1, 2, or 3 source rows.
"vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0] \n"
"cmp %8, #2 \n"
"blt 4f \n"
"vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"vst1.32 {d2}, [%5] \n"
"vst1.32 {d4}, [%6] \n"
"vst1.32 {d6}, [%7] \n"
"b 5f \n"
"3: \n"
"vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d2}, [%5]! \n"
"vst1.32 {d4}, [%6]! \n"
"vst1.32 {d6}, [%7]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"vst1.32 {d3[0]}, [%5] \n"
"vst1.32 {d5[0]}, [%6] \n"
"vst1.32 {d7[0]}, [%7] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"vst1.32 {d2[0]}, [%5] \n"
"vst1.32 {d4[0]}, [%6] \n"
"vst1.32 {d6[0]}, [%7] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(dst3), // %7
"+r"(width) // %8
: "r"(s * 4) // %9
: "memory", "cc", "q0", "q1", "q2", "q3");
}
void TransposeWxH_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const ptrdiff_t d = dst_stride;
while (width >= 4) {
TransposeWx4_32_NEON(src, src_stride, dst, dst_stride, height);
src += 16;
dst += 4 * d;
width -= 4;
}
if (width == 3) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
asm volatile(
"subs %7, %7, #4 \n"
"blt 2f \n"
"1: \n"
"vld3.32 {d0[0], d2[0], d4[0]}, [%0], %8 \n"
"vld3.32 {d0[1], d2[1], d4[1]}, [%1], %8 \n"
"vld3.32 {d1[0], d3[0], d5[0]}, [%2], %8 \n"
"vld3.32 {d1[1], d3[1], d5[1]}, [%3], %8 \n"
"subs %7, %7, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"vst1.8 {q1}, [%5]! \n"
"vst1.8 {q2}, [%6]! \n"
"bge 1b \n"
"2: \n"
"adds %7, %7, #4 \n"
"beq 5f \n"
"vld3.32 {d0[0], d2[0], d4[0]}, [%0] \n"
"cmp %7, #2 \n"
"blt 4f \n"
"vld3.32 {d0[1], d2[1], d4[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"vst1.32 {d2}, [%5] \n"
"vst1.32 {d4}, [%6] \n"
"b 5f \n"
"3: \n"
"vld3.32 {d1[0], d3[0], d5[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d2}, [%5]! \n"
"vst1.32 {d4}, [%6]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"vst1.32 {d3[0]}, [%5] \n"
"vst1.32 {d5[0]}, [%6] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"vst1.32 {d2[0]}, [%5] \n"
"vst1.32 {d4[0]}, [%6] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(height) // %7
: "r"(s * 4) // %8
: "memory", "cc", "q0", "q1", "q2");
} else if (width == 2) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
asm volatile(
"subs %6, %6, #4 \n"
"blt 2f \n"
"1: \n"
"vld2.32 {d0[0], d2[0]}, [%0], %7 \n"
"vld2.32 {d0[1], d2[1]}, [%1], %7 \n"
"vld2.32 {d1[0], d3[0]}, [%2], %7 \n"
"vld2.32 {d1[1], d3[1]}, [%3], %7 \n"
"subs %6, %6, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"vst1.8 {q1}, [%5]! \n"
"bge 1b \n"
"2: \n"
"adds %6, %6, #4 \n"
"beq 5f \n"
"vld2.32 {d0[0], d2[0]}, [%0] \n"
"cmp %6, #2 \n"
"blt 4f \n"
"vld2.32 {d0[1], d2[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"vst1.32 {d2}, [%5] \n"
"b 5f \n"
"3: \n"
"vld2.32 {d1[0], d3[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d2}, [%5]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"vst1.32 {d3[0]}, [%5] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"vst1.32 {d2[0]}, [%5] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(height) // %6
: "r"(s * 4) // %7
: "memory", "cc", "q0", "q1");
} else if (width == 1) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
asm volatile(
"subs %5, %5, #4 \n"
"blt 2f \n"
"1: \n"
"vld1.32 {d0[0]}, [%0], %6 \n"
"vld1.32 {d0[1]}, [%1], %6 \n"
"vld1.32 {d1[0]}, [%2], %6 \n"
"vld1.32 {d1[1]}, [%3], %6 \n"
"subs %5, %5, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"bge 1b \n"
"2: \n"
"adds %5, %5, #4 \n"
"beq 5f \n"
"vld1.32 {d0[0]}, [%0] \n"
"cmp %5, #2 \n"
"blt 4f \n"
"vld1.32 {d0[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"b 5f \n"
"3: \n"
"vld1.32 {d1[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(height) // %5
: "r"(s * 4) // %6
: "memory", "cc", "q0");
}
}
void TransposeWx8_16_NEON(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint8_t* temp;
asm volatile(
"1: \n"
"mov %[temp], %[src] \n"
"vld1.8 {q0}, [%[temp]], %[s] \n"
"vld1.8 {q1}, [%[temp]], %[s] \n"
"vld1.8 {q2}, [%[temp]], %[s] \n"
"vld1.8 {q3}, [%[temp]], %[s] \n"
"vld1.8 {q8}, [%[temp]], %[s] \n"
"vld1.8 {q9}, [%[temp]], %[s] \n"
"vld1.8 {q10}, [%[temp]], %[s] \n"
"vld1.8 {q11}, [%[temp]] \n"
"add %[src], %[src], #16 \n"
// Stage 1: 16-bit transpose
"vtrn.16 q0, q1 \n"
"vtrn.16 q2, q3 \n"
"vtrn.16 q8, q9 \n"
"vtrn.16 q10, q11 \n"
// Stage 2: 32-bit transpose
"vtrn.32 q0, q2 \n"
"vtrn.32 q1, q3 \n"
"vtrn.32 q8, q10 \n"
"vtrn.32 q9, q11 \n"
// Stage 3: 64-bit swap
"vswp d1, d16 \n"
"vswp d3, d18 \n"
"vswp d5, d20 \n"
"vswp d7, d22 \n"
"subs %[w], %[w], #8 \n"
"vst1.8 {q0}, [%[dst]], %[d] \n"
"vst1.8 {q1}, [%[dst]], %[d] \n"
"vst1.8 {q2}, [%[dst]], %[d] \n"
"vst1.8 {q3}, [%[dst]], %[d] \n"
"vst1.8 {q8}, [%[dst]], %[d] \n"
"vst1.8 {q9}, [%[dst]], %[d] \n"
"vst1.8 {q10}, [%[dst]], %[d] \n"
"vst1.8 {q11}, [%[dst]], %[d] \n"
"bgt 1b \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[w] "+r"(width), // %[w]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
}
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
Reference in New Issue View Git Blame Copy Permalink