mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
eeb06a6451
Optimize 32-bit ARGBRotate (0, 90, 180, 270) across Intel x86/x64
architectures (SSE2, AVX2, AVX512BW), with matching kernels for ARM
(NEON, SVE2, SME), WASM, LoongArch (LSX, LASX), and RISC-V (RVV):
- Replace column-by-column ScaleARGBRowDownEven gather loops in
ARGBTranspose (90/270) with full-plane horizontal tile transposes:
- AVX512BW (TransposeWxH_32_AVX512BW / TransposeNx4_32_AVX512BW):
Transpose 16x16 blocks across zmm0..zmm31 using 4-stage
vpunpckldq/hdq, vpunpcklqdq/hqdq, and vshufi32x4 butterflies with
k-masked right and bottom tails, eliminating Any wrappers and scalar
C edge fallbacks.
- AVX2 (TransposeWxH_32_AVX2 / TransposeNx4_32_AVX2):
Transpose 8x8 blocks across ymm0..ymm15 using vpunpck* and
vperm2i128 with vpmaskmovd masked tails.
- SSE2 (Transpose4x4_32_SSE2 / Transpose4x4_32_Any_SSE2):
Transpose 4x4 blocks in xmm registers.
- Implement ARGBRotate180 directly via ARGBMirror with negative height,
avoiding intermediate row buffer allocations and extra copies.
- Implement ARGBRotate0 via ARGBCopy.
Speedup vs Before (1275x1650, US Letter 150 DPI, 1000 iterations):
CPU | 0 deg | 90 deg | 180 deg | 270 deg
---------------------|---------|---------|---------|--------
Intel Ice Lake | 1.02x | 1.34x | 1.18x | 1.33x
Intel Skylake-X | 1.02x | 1.20x | 1.07x | 1.13x
Intel Emerald Rapids | 1.01x | 1.07x | 1.00x | 1.07x
Intel AVX-512 vs C | 1.00x | 2.67x | 2.41x | 2.63x
R=rrwinterton@gmail.com
Bug: b/42280915
Test: LIBYUV_WIDTH=1275 LIBYUV_HEIGHT=1650 LIBYUV_REPEAT=1000 ./out/Release/libyuv_unittest --gtest_filter=*ARGBRotate*
Change-Id: Ib308a225c17e46eab083a9da4033598fcf0c303c
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8538986
Reviewed-by: richard winterton <rrwinterton@gmail.com>
129 lines
5.0 KiB
C++
129 lines
5.0 KiB
C++
/*
|
|
* Copyright 2015 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/rotate.h"
|
|
#include "libyuv/rotate_row.h"
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
#define TANY(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \
|
|
void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
|
|
int dst_stride, int width) { \
|
|
int r = width & MASK; \
|
|
int n = width - r; \
|
|
if (n > 0) { \
|
|
TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
|
|
} \
|
|
ptrdiff_t np = n; \
|
|
TPOS_C(src + np, src_stride, dst + np * dst_stride, dst_stride, r); \
|
|
}
|
|
|
|
#ifdef HAS_TRANSPOSEWX8_NEON
|
|
TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEWX16_NEON
|
|
TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEWX8_SSSE3
|
|
TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEWX16_AVX2
|
|
TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEWX16_LSX
|
|
TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, TransposeWx16_C, 15)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEWX8_WASMSIMD
|
|
TANY(TransposeWx8_Any_WASMSIMD, TransposeWx8_WASMSIMD, TransposeWx8_C, 15)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEWX16_WASMSIMD
|
|
TANY(TransposeWx16_Any_WASMSIMD, TransposeWx16_WASMSIMD, TransposeWx16_C, 15)
|
|
#endif
|
|
#undef TANY
|
|
|
|
#define TUVANY(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \
|
|
void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst_a, \
|
|
int dst_stride_a, uint8_t* dst_b, int dst_stride_b, \
|
|
int width) { \
|
|
int r = width & MASK; \
|
|
int n = width - r; \
|
|
if (n > 0) { \
|
|
TPOS_SIMD(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, n); \
|
|
} \
|
|
ptrdiff_t np = n; \
|
|
TPOS_C(src + np * 2, src_stride, dst_a + np * dst_stride_a, dst_stride_a, \
|
|
dst_b + np * dst_stride_b, dst_stride_b, r); \
|
|
}
|
|
|
|
#ifdef HAS_TRANSPOSEUVWX8_NEON
|
|
TUVANY(TransposeUVWx8_Any_NEON, TransposeUVWx8_NEON, TransposeUVWx8_C, 7)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEUVWX8_SSE2
|
|
TUVANY(TransposeUVWx8_Any_SSE2, TransposeUVWx8_SSE2, TransposeUVWx8_C, 7)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEUVWX16_AVX2
|
|
TUVANY(TransposeUVWx16_Any_AVX2, TransposeUVWx16_AVX2, TransposeUVWx16_C, 7)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEUVWX16_LSX
|
|
TUVANY(TransposeUVWx16_Any_LSX, TransposeUVWx16_LSX, TransposeUVWx16_C, 7)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEUVWX8_WASMSIMD
|
|
TUVANY(TransposeUVWx8_Any_WASMSIMD,
|
|
TransposeUVWx8_WASMSIMD,
|
|
TransposeUVWx8_C,
|
|
7)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSEUVWX16_WASMSIMD
|
|
TUVANY(TransposeUVWx16_Any_WASMSIMD,
|
|
TransposeUVWx16_WASMSIMD,
|
|
TransposeUVWx16_C,
|
|
7)
|
|
#endif
|
|
#undef TUVANY
|
|
|
|
#define TANY32(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \
|
|
void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
|
|
int dst_stride, int width) { \
|
|
int r = width & MASK; \
|
|
int n = width - r; \
|
|
if (n > 0) { \
|
|
TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
|
|
} \
|
|
if (r > 0) { \
|
|
TPOS_C(src + (ptrdiff_t)n * src_stride, src_stride, \
|
|
dst + (ptrdiff_t)n * 4, dst_stride, 4, r); \
|
|
} \
|
|
}
|
|
|
|
#ifdef HAS_TRANSPOSE4X4_32_SSE2
|
|
TANY32(Transpose4x4_32_Any_SSE2, Transpose4x4_32_SSE2, TransposeWxH_32_C, 3)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSE4X4_32_LSX
|
|
TANY32(Transpose4x4_32_Any_LSX, Transpose4x4_32_LSX, TransposeWxH_32_C, 3)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSE4X4_32_LASX
|
|
TANY32(Transpose4x4_32_Any_LASX, Transpose4x4_32_LASX, TransposeWxH_32_C, 7)
|
|
#endif
|
|
#ifdef HAS_TRANSPOSE4X4_32_WASMSIMD
|
|
TANY32(Transpose4x4_32_Any_WASMSIMD,
|
|
Transpose4x4_32_WASMSIMD,
|
|
TransposeWxH_32_C,
|
|
3)
|
|
#endif
|
|
#undef TANY32
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|