coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity
libyuv/source/rotate_neon.cc
Frank Barchard eeb06a6451 [libyuv] Optimize ARGBRotate for Intel SSE2, AVX2, and AVX-512 
Optimize 32-bit ARGBRotate (0, 90, 180, 270) across Intel x86/x64
architectures (SSE2, AVX2, AVX512BW), with matching kernels for ARM
(NEON, SVE2, SME), WASM, LoongArch (LSX, LASX), and RISC-V (RVV):

- Replace column-by-column ScaleARGBRowDownEven gather loops in
  ARGBTranspose (90/270) with full-plane horizontal tile transposes:
  - AVX512BW (TransposeWxH_32_AVX512BW / TransposeNx4_32_AVX512BW):
    Transpose 16x16 blocks across zmm0..zmm31 using 4-stage
    vpunpckldq/hdq, vpunpcklqdq/hqdq, and vshufi32x4 butterflies with
    k-masked right and bottom tails, eliminating Any wrappers and scalar
    C edge fallbacks.
  - AVX2 (TransposeWxH_32_AVX2 / TransposeNx4_32_AVX2):
    Transpose 8x8 blocks across ymm0..ymm15 using vpunpck* and
    vperm2i128 with vpmaskmovd masked tails.
  - SSE2 (Transpose4x4_32_SSE2 / Transpose4x4_32_Any_SSE2):
    Transpose 4x4 blocks in xmm registers.
- Implement ARGBRotate180 directly via ARGBMirror with negative height,
  avoiding intermediate row buffer allocations and extra copies.
- Implement ARGBRotate0 via ARGBCopy.

Speedup vs Before (1275x1650, US Letter 150 DPI, 1000 iterations):
CPU                  | 0 deg   | 90 deg  | 180 deg | 270 deg
---------------------|---------|---------|---------|--------
Intel Ice Lake       | 1.02x   | 1.34x   | 1.18x   | 1.33x
Intel Skylake-X      | 1.02x   | 1.20x   | 1.07x   | 1.13x
Intel Emerald Rapids | 1.01x   | 1.07x   | 1.00x   | 1.07x
Intel AVX-512 vs C   | 1.00x   | 2.67x   | 2.41x   | 2.63x

R=rrwinterton@gmail.com

Bug: b/42280915
Test: LIBYUV_WIDTH=1275 LIBYUV_HEIGHT=1650 LIBYUV_REPEAT=1000 ./out/Release/libyuv_unittest --gtest_filter=*ARGBRotate*
Change-Id: Ib308a225c17e46eab083a9da4033598fcf0c303c
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8538986
Reviewed-by: richard winterton <rrwinterton@gmail.com>
2026-10-08 16:20:46 -07:00

428 lines
18 KiB
C++
Raw Blame History

/*
* Copyright 2011 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
// Note: every register written by an inline asm block (vector, general
// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
// listed in its clobber list, even when the ABI treats it as caller-saved.
// With LTO/LTCG the compiler may keep values live in any register across the
// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
// are callee-saved. List "memory" first, then "cc", then registers.
#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
!defined(__aarch64__)
void TransposeWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
"sub %[width], #8 \n"
"1: \n"
"mov %[temp], %[src] \n"
"vld1.8 {d0}, [%[temp]], %[src_stride] \n"
"vld1.8 {d1}, [%[temp]], %[src_stride] \n"
"vld1.8 {d2}, [%[temp]], %[src_stride] \n"
"vld1.8 {d3}, [%[temp]], %[src_stride] \n"
"vld1.8 {d4}, [%[temp]], %[src_stride] \n"
"vld1.8 {d5}, [%[temp]], %[src_stride] \n"
"vld1.8 {d6}, [%[temp]], %[src_stride] \n"
"vld1.8 {d7}, [%[temp]] \n"
"add %[src], #8 \n"
"vtrn.8 d1, d0 \n"
"vtrn.8 d3, d2 \n"
"vtrn.8 d5, d4 \n"
"vtrn.8 d7, d6 \n"
"subs %[width], #8 \n"
"vtrn.16 d1, d3 \n"
"vtrn.16 d0, d2 \n"
"vtrn.16 d5, d7 \n"
"vtrn.16 d4, d6 \n"
"vtrn.32 d1, d5 \n"
"vtrn.32 d0, d4 \n"
"vtrn.32 d3, d7 \n"
"vtrn.32 d2, d6 \n"
"vrev16.8 q0, q0 \n"
"vrev16.8 q1, q1 \n"
"vrev16.8 q2, q2 \n"
"vrev16.8 q3, q3 \n"
"mov %[temp], %[dst] \n"
"vst1.8 {d1}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d0}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d3}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d2}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d5}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d4}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d7}, [%[temp]], %[dst_stride] \n"
"vst1.8 {d6}, [%[temp]] \n"
"add %[dst], %[dst], %[dst_stride], lsl #3 \n"
"bge 1b \n"
: [temp] "=&r"(temp), // %[temp]
[src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[width] "+r"(width) // %[width]
: [src_stride] "r"(src_stride), // %[src_stride]
[dst_stride] "r"(dst_stride) // %[dst_stride]
: "memory", "cc", "q0", "q1", "q2", "q3");
}
void TransposeUVWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width) {
const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
"sub %[width], #8 \n"
"1: \n"
"mov %[temp], %[src] \n"
"vld2.8 {d0, d1}, [%[temp]], %[src_stride] \n"
"vld2.8 {d2, d3}, [%[temp]], %[src_stride] \n"
"vld2.8 {d4, d5}, [%[temp]], %[src_stride] \n"
"vld2.8 {d6, d7}, [%[temp]], %[src_stride] \n"
"vld2.8 {d16, d17}, [%[temp]], %[src_stride] \n"
"vld2.8 {d18, d19}, [%[temp]], %[src_stride] \n"
"vld2.8 {d20, d21}, [%[temp]], %[src_stride] \n"
"vld2.8 {d22, d23}, [%[temp]] \n"
"add %[src], #8*2 \n"
"vtrn.8 q1, q0 \n"
"vtrn.8 q3, q2 \n"
"vtrn.8 q9, q8 \n"
"vtrn.8 q11, q10 \n"
"subs %[width], #8 \n"
"vtrn.16 q1, q3 \n"
"vtrn.16 q0, q2 \n"
"vtrn.16 q9, q11 \n"
"vtrn.16 q8, q10 \n"
"vtrn.32 q1, q9 \n"
"vtrn.32 q0, q8 \n"
"vtrn.32 q3, q11 \n"
"vtrn.32 q2, q10 \n"
"vrev16.8 q0, q0 \n"
"vrev16.8 q1, q1 \n"
"vrev16.8 q2, q2 \n"
"vrev16.8 q3, q3 \n"
"vrev16.8 q8, q8 \n"
"vrev16.8 q9, q9 \n"
"vrev16.8 q10, q10 \n"
"vrev16.8 q11, q11 \n"
"mov %[temp], %[dst_a] \n"
"vst1.8 {d2}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d0}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d6}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d4}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d18}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d16}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d22}, [%[temp]], %[dst_stride_a] \n"
"vst1.8 {d20}, [%[temp]] \n"
"add %[dst_a], %[dst_a], %[dst_stride_a], lsl #3 \n"
"mov %[temp], %[dst_b] \n"
"vst1.8 {d3}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d1}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d7}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d5}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d19}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d17}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d23}, [%[temp]], %[dst_stride_b] \n"
"vst1.8 {d21}, [%[temp]] \n"
"add %[dst_b], %[dst_b], %[dst_stride_b], lsl #3 \n"
"bge 1b \n"
: [temp] "=&r"(temp), // %[temp]
[src] "+r"(src), // %[src]
[dst_a] "+r"(dst_a), // %[dst_a]
[dst_b] "+r"(dst_b), // %[dst_b]
[width] "+r"(width) // %[width]
: [src_stride] "r"(src_stride), // %[src_stride]
[dst_stride_a] "r"(dst_stride_a), // %[dst_stride_a]
[dst_stride_b] "r"(dst_stride_b) // %[dst_stride_b]
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
}
// Transpose 32 bit values (ARGB) in Nx4 tiles using vld4.32 with lane insertion
// and lane stores for 1..3 remainder rows.
void TransposeNx4_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
const ptrdiff_t d = dst_stride;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
asm volatile(
"subs %8, %8, #4 \n"
"blt 2f \n"
// Main loop transpose 4x4. Read a column, write a row.
"1: \n"
"vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0], %9 \n"
"vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1], %9 \n"
"vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2], %9 \n"
"vld4.32 {d1[1], d3[1], d5[1], d7[1]}, [%3], %9 \n"
"subs %8, %8, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"vst1.8 {q1}, [%5]! \n"
"vst1.8 {q2}, [%6]! \n"
"vst1.8 {q3}, [%7]! \n"
"bge 1b \n"
"2: \n"
"adds %8, %8, #4 \n"
"beq 5f \n"
// Remainder of 1, 2, or 3 source rows.
"vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0] \n"
"cmp %8, #2 \n"
"blt 4f \n"
"vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"vst1.32 {d2}, [%5] \n"
"vst1.32 {d4}, [%6] \n"
"vst1.32 {d6}, [%7] \n"
"b 5f \n"
"3: \n"
"vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d2}, [%5]! \n"
"vst1.32 {d4}, [%6]! \n"
"vst1.32 {d6}, [%7]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"vst1.32 {d3[0]}, [%5] \n"
"vst1.32 {d5[0]}, [%6] \n"
"vst1.32 {d7[0]}, [%7] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"vst1.32 {d2[0]}, [%5] \n"
"vst1.32 {d4[0]}, [%6] \n"
"vst1.32 {d6[0]}, [%7] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(dst3), // %7
"+r"(width) // %8
: "r"(s * 4) // %9
: "memory", "cc", "q0", "q1", "q2", "q3");
}
void TransposeWxH_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const ptrdiff_t d = dst_stride;
while (width >= 4) {
TransposeNx4_32_NEON(src, src_stride, dst, dst_stride, height);
src += 16;
dst += 4 * d;
width -= 4;
}
if (width == 3) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
asm volatile(
"subs %7, %7, #4 \n"
"blt 2f \n"
"1: \n"
"vld3.32 {d0[0], d2[0], d4[0]}, [%0], %8 \n"
"vld3.32 {d0[1], d2[1], d4[1]}, [%1], %8 \n"
"vld3.32 {d1[0], d3[0], d5[0]}, [%2], %8 \n"
"vld3.32 {d1[1], d3[1], d5[1]}, [%3], %8 \n"
"subs %7, %7, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"vst1.8 {q1}, [%5]! \n"
"vst1.8 {q2}, [%6]! \n"
"bge 1b \n"
"2: \n"
"adds %7, %7, #4 \n"
"beq 5f \n"
"vld3.32 {d0[0], d2[0], d4[0]}, [%0] \n"
"cmp %7, #2 \n"
"blt 4f \n"
"vld3.32 {d0[1], d2[1], d4[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"vst1.32 {d2}, [%5] \n"
"vst1.32 {d4}, [%6] \n"
"b 5f \n"
"3: \n"
"vld3.32 {d1[0], d3[0], d5[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d2}, [%5]! \n"
"vst1.32 {d4}, [%6]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"vst1.32 {d3[0]}, [%5] \n"
"vst1.32 {d5[0]}, [%6] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"vst1.32 {d2[0]}, [%5] \n"
"vst1.32 {d4[0]}, [%6] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(height) // %7
: "r"(s * 4) // %8
: "memory", "cc", "q0", "q1", "q2");
} else if (width == 2) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
asm volatile(
"subs %6, %6, #4 \n"
"blt 2f \n"
"1: \n"
"vld2.32 {d0[0], d2[0]}, [%0], %7 \n"
"vld2.32 {d0[1], d2[1]}, [%1], %7 \n"
"vld2.32 {d1[0], d3[0]}, [%2], %7 \n"
"vld2.32 {d1[1], d3[1]}, [%3], %7 \n"
"subs %6, %6, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"vst1.8 {q1}, [%5]! \n"
"bge 1b \n"
"2: \n"
"adds %6, %6, #4 \n"
"beq 5f \n"
"vld2.32 {d0[0], d2[0]}, [%0] \n"
"cmp %6, #2 \n"
"blt 4f \n"
"vld2.32 {d0[1], d2[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"vst1.32 {d2}, [%5] \n"
"b 5f \n"
"3: \n"
"vld2.32 {d1[0], d3[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d2}, [%5]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"vst1.32 {d3[0]}, [%5] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"vst1.32 {d2[0]}, [%5] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(height) // %6
: "r"(s * 4) // %7
: "memory", "cc", "q0", "q1");
} else if (width == 1) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
asm volatile(
"subs %5, %5, #4 \n"
"blt 2f \n"
"1: \n"
"vld1.32 {d0[0]}, [%0], %6 \n"
"vld1.32 {d0[1]}, [%1], %6 \n"
"vld1.32 {d1[0]}, [%2], %6 \n"
"vld1.32 {d1[1]}, [%3], %6 \n"
"subs %5, %5, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"bge 1b \n"
"2: \n"
"adds %5, %5, #4 \n"
"beq 5f \n"
"vld1.32 {d0[0]}, [%0] \n"
"cmp %5, #2 \n"
"blt 4f \n"
"vld1.32 {d0[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"b 5f \n"
"3: \n"
"vld1.32 {d1[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(height) // %5
: "r"(s * 4) // %6
: "memory", "cc", "q0");
}
}
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
Reference in New Issue View Git Blame Copy Permalink