mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
eeb06a6451
Optimize 32-bit ARGBRotate (0, 90, 180, 270) across Intel x86/x64
architectures (SSE2, AVX2, AVX512BW), with matching kernels for ARM
(NEON, SVE2, SME), WASM, LoongArch (LSX, LASX), and RISC-V (RVV):
- Replace column-by-column ScaleARGBRowDownEven gather loops in
ARGBTranspose (90/270) with full-plane horizontal tile transposes:
- AVX512BW (TransposeWxH_32_AVX512BW / TransposeNx4_32_AVX512BW):
Transpose 16x16 blocks across zmm0..zmm31 using 4-stage
vpunpckldq/hdq, vpunpcklqdq/hqdq, and vshufi32x4 butterflies with
k-masked right and bottom tails, eliminating Any wrappers and scalar
C edge fallbacks.
- AVX2 (TransposeWxH_32_AVX2 / TransposeNx4_32_AVX2):
Transpose 8x8 blocks across ymm0..ymm15 using vpunpck* and
vperm2i128 with vpmaskmovd masked tails.
- SSE2 (Transpose4x4_32_SSE2 / Transpose4x4_32_Any_SSE2):
Transpose 4x4 blocks in xmm registers.
- Implement ARGBRotate180 directly via ARGBMirror with negative height,
avoiding intermediate row buffer allocations and extra copies.
- Implement ARGBRotate0 via ARGBCopy.
Speedup vs Before (1275x1650, US Letter 150 DPI, 1000 iterations):
CPU | 0 deg | 90 deg | 180 deg | 270 deg
---------------------|---------|---------|---------|--------
Intel Ice Lake | 1.02x | 1.34x | 1.18x | 1.33x
Intel Skylake-X | 1.02x | 1.20x | 1.07x | 1.13x
Intel Emerald Rapids | 1.01x | 1.07x | 1.00x | 1.07x
Intel AVX-512 vs C | 1.00x | 2.67x | 2.41x | 2.63x
R=rrwinterton@gmail.com
Bug: b/42280915
Test: LIBYUV_WIDTH=1275 LIBYUV_HEIGHT=1650 LIBYUV_REPEAT=1000 ./out/Release/libyuv_unittest --gtest_filter=*ARGBRotate*
Change-Id: Ib308a225c17e46eab083a9da4033598fcf0c303c
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8538986
Reviewed-by: richard winterton <rrwinterton@gmail.com>
251 lines
9.7 KiB
C++
251 lines
9.7 KiB
C++
/*
|
|
* Copyright 2022 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Copyright (c) 2022 Loongson Technology Corporation Limited
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/rotate_row.h"
|
|
|
|
#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
|
|
#include "libyuv/loongson_intrinsics.h"
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
#define ILVLH_B(in0, in1, in2, in3, out0, out1, out2, out3) \
|
|
{ \
|
|
DUP2_ARG2(__lsx_vilvl_b, in1, in0, in3, in2, out0, out2); \
|
|
DUP2_ARG2(__lsx_vilvh_b, in1, in0, in3, in2, out1, out3); \
|
|
}
|
|
|
|
#define ILVLH_H(in0, in1, in2, in3, out0, out1, out2, out3) \
|
|
{ \
|
|
DUP2_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, out0, out2); \
|
|
DUP2_ARG2(__lsx_vilvh_h, in1, in0, in3, in2, out1, out3); \
|
|
}
|
|
|
|
#define ILVLH_W(in0, in1, in2, in3, out0, out1, out2, out3) \
|
|
{ \
|
|
DUP2_ARG2(__lsx_vilvl_w, in1, in0, in3, in2, out0, out2); \
|
|
DUP2_ARG2(__lsx_vilvh_w, in1, in0, in3, in2, out1, out3); \
|
|
}
|
|
|
|
#define ILVLH_D(in0, in1, in2, in3, out0, out1, out2, out3) \
|
|
{ \
|
|
DUP2_ARG2(__lsx_vilvl_d, in1, in0, in3, in2, out0, out2); \
|
|
DUP2_ARG2(__lsx_vilvh_d, in1, in0, in3, in2, out1, out3); \
|
|
}
|
|
|
|
#define LSX_ST_4(_dst0, _dst1, _dst2, _dst3, _dst, _stride, _stride2, \
|
|
_stride3, _stride4) \
|
|
{ \
|
|
__lsx_vst(_dst0, _dst, 0); \
|
|
__lsx_vstx(_dst1, _dst, _stride); \
|
|
__lsx_vstx(_dst2, _dst, _stride2); \
|
|
__lsx_vstx(_dst3, _dst, _stride3); \
|
|
_dst += _stride4; \
|
|
}
|
|
|
|
#define LSX_ST_2(_dst0, _dst1, _dst, _stride, _stride2) \
|
|
{ \
|
|
__lsx_vst(_dst0, _dst, 0); \
|
|
__lsx_vstx(_dst1, _dst, _stride); \
|
|
_dst += _stride2; \
|
|
}
|
|
|
|
void TransposeWx16_LSX(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
int x;
|
|
int len = width / 16;
|
|
uint8_t* s;
|
|
int src_stride2 = src_stride << 1;
|
|
int src_stride3 = src_stride + src_stride2;
|
|
int src_stride4 = src_stride2 << 1;
|
|
int dst_stride2 = dst_stride << 1;
|
|
int dst_stride3 = dst_stride + dst_stride2;
|
|
int dst_stride4 = dst_stride2 << 1;
|
|
__m128i src0, src1, src2, src3, dst0, dst1, dst2, dst3;
|
|
__m128i tmp0, tmp1, tmp2, tmp3;
|
|
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
|
|
__m128i res0, res1, res2, res3, res4, res5, res6, res7, res8, res9;
|
|
|
|
for (x = 0; x < len; x++) {
|
|
s = (uint8_t*)src;
|
|
src0 = __lsx_vld(s, 0);
|
|
src1 = __lsx_vldx(s, src_stride);
|
|
src2 = __lsx_vldx(s, src_stride2);
|
|
src3 = __lsx_vldx(s, src_stride3);
|
|
s += src_stride4;
|
|
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
|
|
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3);
|
|
src0 = __lsx_vld(s, 0);
|
|
src1 = __lsx_vldx(s, src_stride);
|
|
src2 = __lsx_vldx(s, src_stride2);
|
|
src3 = __lsx_vldx(s, src_stride3);
|
|
s += src_stride4;
|
|
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
|
|
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7);
|
|
ILVLH_W(reg0, reg4, reg1, reg5, res0, res1, res2, res3);
|
|
ILVLH_W(reg2, reg6, reg3, reg7, res4, res5, res6, res7);
|
|
src0 = __lsx_vld(s, 0);
|
|
src1 = __lsx_vldx(s, src_stride);
|
|
src2 = __lsx_vldx(s, src_stride2);
|
|
src3 = __lsx_vldx(s, src_stride3);
|
|
s += src_stride4;
|
|
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
|
|
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3);
|
|
src0 = __lsx_vld(s, 0);
|
|
src1 = __lsx_vldx(s, src_stride);
|
|
src2 = __lsx_vldx(s, src_stride2);
|
|
src3 = __lsx_vldx(s, src_stride3);
|
|
s += src_stride4;
|
|
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
|
|
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7);
|
|
res8 = __lsx_vilvl_w(reg4, reg0);
|
|
res9 = __lsx_vilvh_w(reg4, reg0);
|
|
ILVLH_D(res0, res8, res1, res9, dst0, dst1, dst2, dst3);
|
|
LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3,
|
|
dst_stride4);
|
|
res8 = __lsx_vilvl_w(reg5, reg1);
|
|
res9 = __lsx_vilvh_w(reg5, reg1);
|
|
ILVLH_D(res2, res8, res3, res9, dst0, dst1, dst2, dst3);
|
|
LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3,
|
|
dst_stride4);
|
|
res8 = __lsx_vilvl_w(reg6, reg2);
|
|
res9 = __lsx_vilvh_w(reg6, reg2);
|
|
ILVLH_D(res4, res8, res5, res9, dst0, dst1, dst2, dst3);
|
|
LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3,
|
|
dst_stride4);
|
|
res8 = __lsx_vilvl_w(reg7, reg3);
|
|
res9 = __lsx_vilvh_w(reg7, reg3);
|
|
ILVLH_D(res6, res8, res7, res9, dst0, dst1, dst2, dst3);
|
|
LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3,
|
|
dst_stride4);
|
|
src += 16;
|
|
}
|
|
}
|
|
|
|
void TransposeUVWx16_LSX(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int width) {
|
|
int x;
|
|
int len = width / 8;
|
|
uint8_t* s;
|
|
int src_stride2 = src_stride << 1;
|
|
int src_stride3 = src_stride + src_stride2;
|
|
int src_stride4 = src_stride2 << 1;
|
|
int dst_stride_a2 = dst_stride_a << 1;
|
|
int dst_stride_b2 = dst_stride_b << 1;
|
|
__m128i src0, src1, src2, src3, dst0, dst1, dst2, dst3;
|
|
__m128i tmp0, tmp1, tmp2, tmp3;
|
|
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
|
|
__m128i res0, res1, res2, res3, res4, res5, res6, res7, res8, res9;
|
|
|
|
for (x = 0; x < len; x++) {
|
|
s = (uint8_t*)src;
|
|
src0 = __lsx_vld(s, 0);
|
|
src1 = __lsx_vldx(s, src_stride);
|
|
src2 = __lsx_vldx(s, src_stride2);
|
|
src3 = __lsx_vldx(s, src_stride3);
|
|
s += src_stride4;
|
|
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
|
|
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3);
|
|
src0 = __lsx_vld(s, 0);
|
|
src1 = __lsx_vldx(s, src_stride);
|
|
src2 = __lsx_vldx(s, src_stride2);
|
|
src3 = __lsx_vldx(s, src_stride3);
|
|
s += src_stride4;
|
|
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
|
|
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7);
|
|
ILVLH_W(reg0, reg4, reg1, reg5, res0, res1, res2, res3);
|
|
ILVLH_W(reg2, reg6, reg3, reg7, res4, res5, res6, res7);
|
|
src0 = __lsx_vld(s, 0);
|
|
src1 = __lsx_vldx(s, src_stride);
|
|
src2 = __lsx_vldx(s, src_stride2);
|
|
src3 = __lsx_vldx(s, src_stride3);
|
|
s += src_stride4;
|
|
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
|
|
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3);
|
|
src0 = __lsx_vld(s, 0);
|
|
src1 = __lsx_vldx(s, src_stride);
|
|
src2 = __lsx_vldx(s, src_stride2);
|
|
src3 = __lsx_vldx(s, src_stride3);
|
|
s += src_stride4;
|
|
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
|
|
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7);
|
|
res8 = __lsx_vilvl_w(reg4, reg0);
|
|
res9 = __lsx_vilvh_w(reg4, reg0);
|
|
ILVLH_D(res0, res8, res1, res9, dst0, dst1, dst2, dst3);
|
|
LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2);
|
|
LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2);
|
|
res8 = __lsx_vilvl_w(reg5, reg1);
|
|
res9 = __lsx_vilvh_w(reg5, reg1);
|
|
ILVLH_D(res2, res8, res3, res9, dst0, dst1, dst2, dst3);
|
|
LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2);
|
|
LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2);
|
|
res8 = __lsx_vilvl_w(reg6, reg2);
|
|
res9 = __lsx_vilvh_w(reg6, reg2);
|
|
ILVLH_D(res4, res8, res5, res9, dst0, dst1, dst2, dst3);
|
|
LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2);
|
|
LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2);
|
|
res8 = __lsx_vilvl_w(reg7, reg3);
|
|
res9 = __lsx_vilvh_w(reg7, reg3);
|
|
ILVLH_D(res6, res8, res7, res9, dst0, dst1, dst2, dst3);
|
|
LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2);
|
|
LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2);
|
|
src += 16;
|
|
}
|
|
}
|
|
|
|
void Transpose4x4_32_LSX(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
int x;
|
|
int len = width / 4;
|
|
ptrdiff_t src_stride2 = src_stride * 2;
|
|
ptrdiff_t src_stride3 = src_stride + src_stride2;
|
|
ptrdiff_t src_stride4 = src_stride * 4;
|
|
ptrdiff_t dst_stride2 = dst_stride * 2;
|
|
ptrdiff_t dst_stride3 = dst_stride + dst_stride2;
|
|
__m128i src0, src1, src2, src3;
|
|
__m128i dst0, dst1, dst2, dst3;
|
|
|
|
for (x = 0; x < len; x++) {
|
|
src0 = __lsx_vld(src, 0);
|
|
src1 = __lsx_vldx(src, src_stride);
|
|
src2 = __lsx_vldx(src, src_stride2);
|
|
src3 = __lsx_vldx(src, src_stride3);
|
|
LSX_TRANSPOSE4x4_W(src0, src1, src2, src3, dst0, dst1, dst2, dst3);
|
|
__lsx_vst(dst0, dst, 0);
|
|
__lsx_vstx(dst1, dst, dst_stride);
|
|
__lsx_vstx(dst2, dst, dst_stride2);
|
|
__lsx_vstx(dst3, dst, dst_stride3);
|
|
dst += 16;
|
|
src += src_stride4;
|
|
}
|
|
}
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
|