mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
Optimize 32-bit ARGBRotate (0, 90, 180, 270) across Intel x86/x64
architectures (SSE2, AVX2, AVX512BW), with matching kernels for ARM
(NEON, SVE2, SME), WASM, LoongArch (LSX, LASX), and RISC-V (RVV):
- Replace column-by-column ScaleARGBRowDownEven gather loops in
ARGBTranspose (90/270) with full-plane horizontal tile transposes:
- AVX512BW (TransposeWxH_32_AVX512BW / TransposeNx4_32_AVX512BW):
Transpose 16x16 blocks across zmm0..zmm31 using 4-stage
vpunpckldq/hdq, vpunpcklqdq/hqdq, and vshufi32x4 butterflies with
k-masked right and bottom tails, eliminating Any wrappers and scalar
C edge fallbacks.
- AVX2 (TransposeWxH_32_AVX2 / TransposeNx4_32_AVX2):
Transpose 8x8 blocks across ymm0..ymm15 using vpunpck* and
vperm2i128 with vpmaskmovd masked tails.
- SSE2 (Transpose4x4_32_SSE2 / Transpose4x4_32_Any_SSE2):
Transpose 4x4 blocks in xmm registers.
- Implement ARGBRotate180 directly via ARGBMirror with negative height,
avoiding intermediate row buffer allocations and extra copies.
- Implement ARGBRotate0 via ARGBCopy.
Speedup vs Before (1275x1650, US Letter 150 DPI, 1000 iterations):
CPU | 0 deg | 90 deg | 180 deg | 270 deg
---------------------|---------|---------|---------|--------
Intel Ice Lake | 1.02x | 1.34x | 1.18x | 1.33x
Intel Skylake-X | 1.02x | 1.20x | 1.07x | 1.13x
Intel Emerald Rapids | 1.01x | 1.07x | 1.00x | 1.07x
Intel AVX-512 vs C | 1.00x | 2.67x | 2.41x | 2.63x
R=rrwinterton@gmail.com
Bug: b/42280915
Test: LIBYUV_WIDTH=1275 LIBYUV_HEIGHT=1650 LIBYUV_REPEAT=1000 ./out/Release/libyuv_unittest --gtest_filter=*ARGBRotate*
Change-Id: Ib308a225c17e46eab083a9da4033598fcf0c303c
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8538986
Reviewed-by: richard winterton <rrwinterton@gmail.com>
220 lines
7.1 KiB
C++
220 lines
7.1 KiB
C++
/*
|
|
* Copyright 2012 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/rotate_argb.h"
|
|
|
|
#include <limits.h>
|
|
|
|
#include "libyuv/convert.h"
|
|
#include "libyuv/cpu_id.h"
|
|
#include "libyuv/planar_functions.h"
|
|
#include "libyuv/rotate.h"
|
|
#include "libyuv/rotate_row.h"
|
|
#include "libyuv/row.h"
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
static int ARGBTranspose(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_argb,
|
|
int dst_stride_argb,
|
|
int width,
|
|
int height) {
|
|
#if defined(HAS_TRANSPOSEWXH_32_AVX2) || \
|
|
defined(HAS_TRANSPOSEWXH_32_AVX512BW) || \
|
|
defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
|
|
defined(HAS_TRANSPOSEWXH_32_RVV)
|
|
void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst,
|
|
int dst_stride, int width, int height) = NULL;
|
|
#endif
|
|
void (*Transpose4x4_32)(const uint8_t* src, int src_stride, uint8_t* dst,
|
|
int dst_stride, int width) = NULL;
|
|
// Check stride is a multiple of 4.
|
|
if (src_stride_argb & 3) {
|
|
return -1;
|
|
}
|
|
#if defined(HAS_TRANSPOSE4X4_32_SSE2)
|
|
if (TestCpuFlag(kCpuHasSSE2)) {
|
|
Transpose4x4_32 = Transpose4x4_32_Any_SSE2;
|
|
if (IS_ALIGNED(height, 4)) { // Width of dest.
|
|
Transpose4x4_32 = Transpose4x4_32_SSE2;
|
|
}
|
|
}
|
|
#endif
|
|
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
|
|
if (TestCpuFlag(kCpuHasAVX2)) {
|
|
TransposeWxH_32 = TransposeWxH_32_AVX2;
|
|
}
|
|
#endif
|
|
#if defined(HAS_TRANSPOSEWXH_32_AVX512BW)
|
|
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
|
TransposeWxH_32 = TransposeWxH_32_AVX512BW;
|
|
}
|
|
#endif
|
|
#if defined(HAS_TRANSPOSEWXH_32_NEON)
|
|
if (TestCpuFlag(kCpuHasNEON)) {
|
|
TransposeWxH_32 = TransposeWxH_32_NEON;
|
|
}
|
|
#endif
|
|
#if defined(HAS_TRANSPOSENX4_32_SVE2)
|
|
if (TestCpuFlag(kCpuHasSVE2)) {
|
|
Transpose4x4_32 = TransposeNx4_32_SVE2;
|
|
}
|
|
#endif
|
|
#if defined(HAS_TRANSPOSEWXH_32_SME)
|
|
if (TestCpuFlag(kCpuHasSME)) {
|
|
TransposeWxH_32 = TransposeWxH_32_SME;
|
|
}
|
|
#endif
|
|
#if defined(HAS_TRANSPOSEWXH_32_RVV)
|
|
if (TestCpuFlag(kCpuHasRVV)) {
|
|
TransposeWxH_32 = TransposeWxH_32_RVV;
|
|
}
|
|
#endif
|
|
#if defined(HAS_TRANSPOSE4X4_32_LSX)
|
|
if (TestCpuFlag(kCpuHasLSX)) {
|
|
Transpose4x4_32 = Transpose4x4_32_Any_LSX;
|
|
if (IS_ALIGNED(height, 4)) { // Width of dest.
|
|
Transpose4x4_32 = Transpose4x4_32_LSX;
|
|
}
|
|
}
|
|
#endif
|
|
#if defined(HAS_TRANSPOSE4X4_32_LASX)
|
|
if (TestCpuFlag(kCpuHasLASX)) {
|
|
Transpose4x4_32 = Transpose4x4_32_Any_LASX;
|
|
if (IS_ALIGNED(height, 8)) { // Width of dest.
|
|
Transpose4x4_32 = Transpose4x4_32_LASX;
|
|
}
|
|
}
|
|
#endif
|
|
#if defined(HAS_TRANSPOSE4X4_32_WASMSIMD)
|
|
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
|
Transpose4x4_32 = Transpose4x4_32_Any_WASMSIMD;
|
|
if (IS_ALIGNED(height, 4)) { // Width of dest.
|
|
Transpose4x4_32 = Transpose4x4_32_WASMSIMD;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
#if defined(HAS_TRANSPOSEWXH_32_AVX2) || \
|
|
defined(HAS_TRANSPOSEWXH_32_AVX512BW) || \
|
|
defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
|
|
defined(HAS_TRANSPOSEWXH_32_RVV)
|
|
if (TransposeWxH_32) {
|
|
TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
|
|
height);
|
|
return 0;
|
|
}
|
|
#endif
|
|
if (Transpose4x4_32) {
|
|
while (width >= 4) {
|
|
Transpose4x4_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
|
height);
|
|
src_argb += 4 * 4;
|
|
dst_argb += 4 * (ptrdiff_t)dst_stride_argb;
|
|
width -= 4;
|
|
}
|
|
}
|
|
if (width > 0) {
|
|
TransposeWxH_32_C(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
|
width, height);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int ARGBRotate90(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_argb,
|
|
int dst_stride_argb,
|
|
int width,
|
|
int height) {
|
|
// Rotate by 90 is a ARGBTranspose with the source read
|
|
// from bottom to top. So set the source pointer to the end
|
|
// of the buffer and flip the sign of the source stride.
|
|
src_argb += (ptrdiff_t)src_stride_argb * (height - 1);
|
|
src_stride_argb = -src_stride_argb;
|
|
return ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
|
width, height);
|
|
}
|
|
|
|
static int ARGBRotate270(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_argb,
|
|
int dst_stride_argb,
|
|
int width,
|
|
int height) {
|
|
// Rotate by 270 is a ARGBTranspose with the destination written
|
|
// from bottom to top. So set the destination pointer to the end
|
|
// of the buffer and flip the sign of the destination stride.
|
|
dst_argb += (ptrdiff_t)dst_stride_argb * (width - 1);
|
|
dst_stride_argb = -dst_stride_argb;
|
|
return ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
|
width, height);
|
|
}
|
|
|
|
static int ARGBRotate180(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_argb,
|
|
int dst_stride_argb,
|
|
int width,
|
|
int height) {
|
|
// Rotate by 180 is a vertical flip (negative height) and horizontal mirror.
|
|
return ARGBMirror(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
|
|
-height);
|
|
}
|
|
|
|
LIBYUV_API
|
|
int ARGBRotate(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_argb,
|
|
int dst_stride_argb,
|
|
int width,
|
|
int height,
|
|
enum RotationMode mode) {
|
|
if (!src_argb || width <= 0 || height == 0 || height == INT_MIN ||
|
|
!dst_argb) {
|
|
return -1;
|
|
}
|
|
|
|
// Negative height means invert the image.
|
|
if (height < 0) {
|
|
height = -height;
|
|
src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
|
|
src_stride_argb = -src_stride_argb;
|
|
}
|
|
|
|
switch (mode) {
|
|
case kRotate0:
|
|
// copy frame
|
|
return ARGBCopy(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
|
width, height);
|
|
case kRotate90:
|
|
return ARGBRotate90(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
|
width, height);
|
|
case kRotate270:
|
|
return ARGBRotate270(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
|
width, height);
|
|
case kRotate180:
|
|
return ARGBRotate180(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
|
width, height);
|
|
default:
|
|
break;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|