libyuv/source/rotate_argb.cc
Frank Barchard 16af162cf9 [libyuv] Optimize RotatePlane_16 SIMD and compact 32-bit transposes
Implement SIMD 16-bit plane rotation (TransposePlane_16,
RotatePlane90_16, RotatePlane180_16, RotatePlane270_16) used by
I010Rotate, I210Rotate, and I410Rotate, keeping tile sizes compact
with full-register writes and zero cross-lane shuffles:
- AVX2: 8x16 tile in ymm0..ymm15 with _Any tail (low-end x86 SIMD)
- AVX-512BW: 8x32 tile in zmm0..zmm15 with k1/k2 right/bottom masks
- NEON (ARM64/ARM32): 8x16 tile in v0..v23 / q0..q11 with _Any tail
- SME: 8xVL tile using ZA16 horizontal/vertical array slices
- RVV: 8xVL tile using widening unpacks with hardware vl tail
- WASM SIMD: 8x8 tile in v128 with _Any tail

Also update 32-bit ARGBRotate x86 transposes to compact vertical
in-lane tiles with full-register writes and zero cross-lane shuffles:
- TransposeWxH_32_AVX2: 4x8 tile in ymm0..ymm7 with vpmaskmovd tails
- TransposeWxH_32_AVX512BW: 4x16 tile in zmm0..zmm7 with k1/k2 tails

Add RotatePlane16_SplitMergeUV_Opt and RotatePlane16_SplitMergeUV_Odd
unit tests comparing 16-bit rotation against SplitUVPlane +
RotatePlane + MergeUVPlane across 0/90/180/270 degrees.

Performance on 1275x1650 (1000 iterations, before -> after):
CPU                     90 degree               180 degree
Coral RVV (Verilator)   7306 -> 1415c (5.16x)   2274 -> 719c (3.16x)
AMD Rome (Zen 2)        821 -> 215ms (3.82x)    259 -> 181ms (1.44x)
AMD Genoa (Zen 4)       483 -> 129ms (3.73x)    149 -> 111ms (1.34x)
AMD Turin (Zen 5)       321 -> 101ms (3.18x)    109 -> 78ms (1.40x)
AMD Milan (Zen 3)       505 -> 189ms (2.67x)    199 -> 118ms (1.69x)
Intel Skylake-X         1076 -> 447ms (2.41x)   353 -> 317ms (1.11x)
WASM SIMD (d8)          415 -> 203ms (2.04x)    370 -> 106ms (3.49x)
Intel Emerald Rapids    735 -> 422ms (1.74x)    238 -> 233ms (1.02x)
Intel Haswell           1120 -> 648ms (1.73x)   438 -> 393ms (1.11x)
Intel Ice Lake          774 -> 474ms (1.63x)    279 -> 290ms (0.96x)
ARM Neoverse-N1         864 -> 544ms (1.59x)    404 -> 266ms (1.52x)
Ampere Siryn            862 -> 576ms (1.50x)    568 -> 329ms (1.72x)
Google Axion (Maple)    554 -> 379ms (1.46x)    199 -> 142ms (1.41x)
Intel Sapphire Rapids   1126 -> 772ms (1.46x)   301 -> 288ms (1.04x)
Intel Granite Rapids    730 -> 565ms (1.29x)    280 -> 280ms (1.00x)
NVIDIA Grace            462 -> 365ms (1.26x)    192 -> 126ms (1.52x)
ARM Cortex-A53          28290 -> 24386ms (1.16x) 7178 -> 5075ms (1.41x)
Google Axion 2 (Tamar)  523 -> 453ms (1.15x)    217 -> 112ms (1.93x)

Test: libyuv_unittest --gtest_filter=*Rotate*
Bug: b/42280942
Change-Id: I91ba6cac22185b5d294fcb98344fdb8a23cc94ff
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8545294
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
2026-10-12 10:57:34 -07:00

251 lines
7.9 KiB
C++

/*
* Copyright 2012 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_argb.h"
#include <limits.h>
#include "libyuv/convert.h"
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
#include "libyuv/rotate.h"
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
static int ARGBTranspose(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
defined(HAS_TRANSPOSEWXH_32_RVV)
void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width, int height) = NULL;
#endif
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
void (*TransposeWx16_32)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = NULL;
#endif
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
void (*TransposeWx8_32)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = NULL;
#endif
void (*TransposeWx4_32)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = NULL;
// Check stride is a multiple of 4.
if (src_stride_argb & 3) {
return -1;
}
#if defined(HAS_TRANSPOSEWX4_32_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
TransposeWx4_32 = TransposeWx4_32_Any_SSE2;
if (IS_ALIGNED(height, 4)) { // Width of dest.
TransposeWx4_32 = TransposeWx4_32_SSE2;
}
}
#endif
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
TransposeWx8_32 = TransposeWx8_32_Any_AVX2;
if (IS_ALIGNED(width, 4)) {
TransposeWx8_32 = TransposeWx8_32_AVX2;
}
}
#endif
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
TransposeWx16_32 = TransposeWx16_32_AVX512BW;
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
TransposeWxH_32 = TransposeWxH_32_NEON;
}
#endif
#if defined(HAS_TRANSPOSEWX4_32_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
TransposeWx4_32 = TransposeWx4_32_SVE2;
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_SME)
if (TestCpuFlag(kCpuHasSME)) {
TransposeWxH_32 = TransposeWxH_32_SME;
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
TransposeWxH_32 = TransposeWxH_32_RVV;
}
#endif
#if defined(HAS_TRANSPOSEWX4_32_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
TransposeWx4_32 = TransposeWx4_32_Any_LSX;
if (IS_ALIGNED(height, 4)) { // Width of dest.
TransposeWx4_32 = TransposeWx4_32_LSX;
}
}
#endif
#if defined(HAS_TRANSPOSEWX4_32_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
TransposeWx4_32 = TransposeWx4_32_Any_LASX;
if (IS_ALIGNED(height, 8)) { // Width of dest.
TransposeWx4_32 = TransposeWx4_32_LASX;
}
}
#endif
#if defined(HAS_TRANSPOSEWX4_32_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
TransposeWx4_32 = TransposeWx4_32_Any_WASMSIMD;
if (IS_ALIGNED(height, 4)) { // Width of dest.
TransposeWx4_32 = TransposeWx4_32_WASMSIMD;
}
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
defined(HAS_TRANSPOSEWXH_32_RVV)
if (TransposeWxH_32) {
TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
height);
return 0;
}
#endif
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
if (TransposeWx16_32) {
while (height >= 16) {
TransposeWx16_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width);
src_argb += 16 * (ptrdiff_t)src_stride_argb;
dst_argb += 16 * 4;
height -= 16;
}
}
#endif
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
if (TransposeWx8_32) {
while (height >= 8) {
TransposeWx8_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width);
src_argb += 8 * (ptrdiff_t)src_stride_argb;
dst_argb += 8 * 4;
height -= 8;
}
}
#endif
if (height > 0) {
if (TransposeWx4_32) {
while (width >= 4) {
TransposeWx4_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
height);
src_argb += 4 * 4;
dst_argb += 4 * (ptrdiff_t)dst_stride_argb;
width -= 4;
}
}
if (width > 0) {
TransposeWxH_32_C(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
}
}
return 0;
}
static int ARGBRotate90(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
// Rotate by 90 is a ARGBTranspose with the source read
// from bottom to top. So set the source pointer to the end
// of the buffer and flip the sign of the source stride.
src_argb += (ptrdiff_t)src_stride_argb * (height - 1);
src_stride_argb = -src_stride_argb;
return ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
}
static int ARGBRotate270(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
// Rotate by 270 is a ARGBTranspose with the destination written
// from bottom to top. So set the destination pointer to the end
// of the buffer and flip the sign of the destination stride.
dst_argb += (ptrdiff_t)dst_stride_argb * (width - 1);
dst_stride_argb = -dst_stride_argb;
return ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
}
static int ARGBRotate180(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
// Rotate by 180 is a vertical flip (negative height) and horizontal mirror.
return ARGBMirror(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
-height);
}
LIBYUV_API
int ARGBRotate(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height,
enum RotationMode mode) {
if (!src_argb || width <= 0 || height == 0 || height == INT_MIN ||
!dst_argb) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
switch (mode) {
case kRotate0:
// copy frame
return ARGBCopy(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
case kRotate90:
return ARGBRotate90(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
case kRotate270:
return ARGBRotate270(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
case kRotate180:
return ARGBRotate180(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
default:
break;
}
return -1;
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif