mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
eeb06a6451
Optimize 32-bit ARGBRotate (0, 90, 180, 270) across Intel x86/x64
architectures (SSE2, AVX2, AVX512BW), with matching kernels for ARM
(NEON, SVE2, SME), WASM, LoongArch (LSX, LASX), and RISC-V (RVV):
- Replace column-by-column ScaleARGBRowDownEven gather loops in
ARGBTranspose (90/270) with full-plane horizontal tile transposes:
- AVX512BW (TransposeWxH_32_AVX512BW / TransposeNx4_32_AVX512BW):
Transpose 16x16 blocks across zmm0..zmm31 using 4-stage
vpunpckldq/hdq, vpunpcklqdq/hqdq, and vshufi32x4 butterflies with
k-masked right and bottom tails, eliminating Any wrappers and scalar
C edge fallbacks.
- AVX2 (TransposeWxH_32_AVX2 / TransposeNx4_32_AVX2):
Transpose 8x8 blocks across ymm0..ymm15 using vpunpck* and
vperm2i128 with vpmaskmovd masked tails.
- SSE2 (Transpose4x4_32_SSE2 / Transpose4x4_32_Any_SSE2):
Transpose 4x4 blocks in xmm registers.
- Implement ARGBRotate180 directly via ARGBMirror with negative height,
avoiding intermediate row buffer allocations and extra copies.
- Implement ARGBRotate0 via ARGBCopy.
Speedup vs Before (1275x1650, US Letter 150 DPI, 1000 iterations):
CPU | 0 deg | 90 deg | 180 deg | 270 deg
---------------------|---------|---------|---------|--------
Intel Ice Lake | 1.02x | 1.34x | 1.18x | 1.33x
Intel Skylake-X | 1.02x | 1.20x | 1.07x | 1.13x
Intel Emerald Rapids | 1.01x | 1.07x | 1.00x | 1.07x
Intel AVX-512 vs C | 1.00x | 2.67x | 2.41x | 2.63x
R=rrwinterton@gmail.com
Bug: b/42280915
Test: LIBYUV_WIDTH=1275 LIBYUV_HEIGHT=1650 LIBYUV_REPEAT=1000 ./out/Release/libyuv_unittest --gtest_filter=*ARGBRotate*
Change-Id: Ib308a225c17e46eab083a9da4033598fcf0c303c
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8538986
Reviewed-by: richard winterton <rrwinterton@gmail.com>
118 lines
4.8 KiB
C++
118 lines
4.8 KiB
C++
/*
|
|
* Copyright 2026 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/rotate_row.h"
|
|
#include "libyuv/row.h"
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
|
|
|
// Transpose 32 bit values (ARGB) in Nx4 tiles using SVE2 with predication
|
|
// for tail widths.
|
|
void TransposeNx4_32_SVE2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t s4 = s * 4;
|
|
const ptrdiff_t d = dst_stride;
|
|
const uint8_t* src1 = src + s;
|
|
const uint8_t* src2 = src1 + s;
|
|
const uint8_t* src3 = src2 + s;
|
|
uint8_t* dst1 = dst + d;
|
|
uint8_t* dst2 = dst1 + d;
|
|
uint8_t* dst3 = dst2 + d;
|
|
asm volatile(
|
|
"ptrue p0.s, vl4 \n"
|
|
"subs %w[width], %w[width], #4 \n"
|
|
"b.lt 2f \n"
|
|
|
|
// Main loop: full 4x4 blocks.
|
|
"1: \n"
|
|
"ld1w {z0.s}, p0/z, [%[src0]] \n"
|
|
"ld1w {z1.s}, p0/z, [%[src1]] \n"
|
|
"ld1w {z2.s}, p0/z, [%[src2]] \n"
|
|
"ld1w {z3.s}, p0/z, [%[src3]] \n"
|
|
"add %[src0], %[src0], %[s4] \n"
|
|
"add %[src1], %[src1], %[s4] \n"
|
|
"add %[src2], %[src2], %[s4] \n"
|
|
"add %[src3], %[src3], %[s4] \n"
|
|
"trn1 z4.s, z0.s, z1.s \n"
|
|
"trn2 z5.s, z0.s, z1.s \n"
|
|
"trn1 z6.s, z2.s, z3.s \n"
|
|
"trn2 z7.s, z2.s, z3.s \n"
|
|
"trn1 z0.d, z4.d, z6.d \n"
|
|
"trn2 z1.d, z4.d, z6.d \n"
|
|
"trn1 z2.d, z5.d, z7.d \n"
|
|
"trn2 z3.d, z5.d, z7.d \n"
|
|
"subs %w[width], %w[width], #4 \n"
|
|
"st1w {z0.s}, p0, [%[dst0]] \n"
|
|
"st1w {z1.s}, p0, [%[dst1]] \n"
|
|
"st1w {z2.s}, p0, [%[dst2]] \n"
|
|
"st1w {z3.s}, p0, [%[dst3]] \n"
|
|
"add %[dst0], %[dst0], #16 \n"
|
|
"add %[dst1], %[dst1], #16 \n"
|
|
"add %[dst2], %[dst2], #16 \n"
|
|
"add %[dst3], %[dst3], #16 \n"
|
|
"b.ge 1b \n"
|
|
|
|
"2: \n"
|
|
"adds %w[width], %w[width], #4 \n"
|
|
"b.eq 3f \n"
|
|
|
|
// Tail loop: 1..3 remaining source rows / destination columns.
|
|
"cmp %w[width], #1 \n"
|
|
"csel %[src1], %[src0], %[src1], eq \n"
|
|
"cmp %w[width], #2 \n"
|
|
"csel %[src2], %[src1], %[src2], le \n"
|
|
"mov %[src3], %[src2] \n"
|
|
"ld1w {z0.s}, p0/z, [%[src0]] \n"
|
|
"ld1w {z1.s}, p0/z, [%[src1]] \n"
|
|
"ld1w {z2.s}, p0/z, [%[src2]] \n"
|
|
"ld1w {z3.s}, p0/z, [%[src3]] \n"
|
|
"trn1 z4.s, z0.s, z1.s \n"
|
|
"trn2 z5.s, z0.s, z1.s \n"
|
|
"trn1 z6.s, z2.s, z3.s \n"
|
|
"trn2 z7.s, z2.s, z3.s \n"
|
|
"trn1 z0.d, z4.d, z6.d \n"
|
|
"trn2 z1.d, z4.d, z6.d \n"
|
|
"trn1 z2.d, z5.d, z7.d \n"
|
|
"trn2 z3.d, z5.d, z7.d \n"
|
|
"whilelt p0.s, wzr, %w[width] \n"
|
|
"st1w {z0.s}, p0, [%[dst0]] \n"
|
|
"st1w {z1.s}, p0, [%[dst1]] \n"
|
|
"st1w {z2.s}, p0, [%[dst2]] \n"
|
|
"st1w {z3.s}, p0, [%[dst3]] \n"
|
|
"3: \n"
|
|
: [src0] "+r"(src), // %[src0]
|
|
[src1] "+r"(src1), // %[src1]
|
|
[src2] "+r"(src2), // %[src2]
|
|
[src3] "+r"(src3), // %[src3]
|
|
[dst0] "+r"(dst), // %[dst0]
|
|
[dst1] "+r"(dst1), // %[dst1]
|
|
[dst2] "+r"(dst2), // %[dst2]
|
|
[dst3] "+r"(dst3), // %[dst3]
|
|
[width] "+r"(width) // %[width]
|
|
: [s4] "r"(s4) // %[s4]
|
|
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
|
|
}
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|