coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity
libyuv/source/rotate_lsx.cc
Frank Barchard 16af162cf9 [libyuv] Optimize RotatePlane_16 SIMD and compact 32-bit transposes 
Implement SIMD 16-bit plane rotation (TransposePlane_16,
RotatePlane90_16, RotatePlane180_16, RotatePlane270_16) used by
I010Rotate, I210Rotate, and I410Rotate, keeping tile sizes compact
with full-register writes and zero cross-lane shuffles:
- AVX2: 8x16 tile in ymm0..ymm15 with _Any tail (low-end x86 SIMD)
- AVX-512BW: 8x32 tile in zmm0..zmm15 with k1/k2 right/bottom masks
- NEON (ARM64/ARM32): 8x16 tile in v0..v23 / q0..q11 with _Any tail
- SME: 8xVL tile using ZA16 horizontal/vertical array slices
- RVV: 8xVL tile using widening unpacks with hardware vl tail
- WASM SIMD: 8x8 tile in v128 with _Any tail

Also update 32-bit ARGBRotate x86 transposes to compact vertical
in-lane tiles with full-register writes and zero cross-lane shuffles:
- TransposeWxH_32_AVX2: 4x8 tile in ymm0..ymm7 with vpmaskmovd tails
- TransposeWxH_32_AVX512BW: 4x16 tile in zmm0..zmm7 with k1/k2 tails

Add RotatePlane16_SplitMergeUV_Opt and RotatePlane16_SplitMergeUV_Odd
unit tests comparing 16-bit rotation against SplitUVPlane +
RotatePlane + MergeUVPlane across 0/90/180/270 degrees.

Performance on 1275x1650 (1000 iterations, before -> after):
CPU                     90 degree               180 degree
Coral RVV (Verilator)   7306 -> 1415c (5.16x)   2274 -> 719c (3.16x)
AMD Rome (Zen 2)        821 -> 215ms (3.82x)    259 -> 181ms (1.44x)
AMD Genoa (Zen 4)       483 -> 129ms (3.73x)    149 -> 111ms (1.34x)
AMD Turin (Zen 5)       321 -> 101ms (3.18x)    109 -> 78ms (1.40x)
AMD Milan (Zen 3)       505 -> 189ms (2.67x)    199 -> 118ms (1.69x)
Intel Skylake-X         1076 -> 447ms (2.41x)   353 -> 317ms (1.11x)
WASM SIMD (d8)          415 -> 203ms (2.04x)    370 -> 106ms (3.49x)
Intel Emerald Rapids    735 -> 422ms (1.74x)    238 -> 233ms (1.02x)
Intel Haswell           1120 -> 648ms (1.73x)   438 -> 393ms (1.11x)
Intel Ice Lake          774 -> 474ms (1.63x)    279 -> 290ms (0.96x)
ARM Neoverse-N1         864 -> 544ms (1.59x)    404 -> 266ms (1.52x)
Ampere Siryn            862 -> 576ms (1.50x)    568 -> 329ms (1.72x)
Google Axion (Maple)    554 -> 379ms (1.46x)    199 -> 142ms (1.41x)
Intel Sapphire Rapids   1126 -> 772ms (1.46x)   301 -> 288ms (1.04x)
Intel Granite Rapids    730 -> 565ms (1.29x)    280 -> 280ms (1.00x)
NVIDIA Grace            462 -> 365ms (1.26x)    192 -> 126ms (1.52x)
ARM Cortex-A53          28290 -> 24386ms (1.16x) 7178 -> 5075ms (1.41x)
Google Axion 2 (Tamar)  523 -> 453ms (1.15x)    217 -> 112ms (1.93x)

Test: libyuv_unittest --gtest_filter=*Rotate*
Bug: b/42280942
Change-Id: I91ba6cac22185b5d294fcb98344fdb8a23cc94ff
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8545294
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
2026-10-12 10:57:34 -07:00

302 lines
12 KiB
C++
Raw Blame History

/*
* Copyright 2022 The LibYuv Project Authors. All rights reserved.
*
* Copyright (c) 2022 Loongson Technology Corporation Limited
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_row.h"
#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
#include "libyuv/loongson_intrinsics.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
#define ILVLH_B(in0, in1, in2, in3, out0, out1, out2, out3) \
{ \
DUP2_ARG2(__lsx_vilvl_b, in1, in0, in3, in2, out0, out2); \
DUP2_ARG2(__lsx_vilvh_b, in1, in0, in3, in2, out1, out3); \
}
#define ILVLH_H(in0, in1, in2, in3, out0, out1, out2, out3) \
{ \
DUP2_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, out0, out2); \
DUP2_ARG2(__lsx_vilvh_h, in1, in0, in3, in2, out1, out3); \
}
#define ILVLH_W(in0, in1, in2, in3, out0, out1, out2, out3) \
{ \
DUP2_ARG2(__lsx_vilvl_w, in1, in0, in3, in2, out0, out2); \
DUP2_ARG2(__lsx_vilvh_w, in1, in0, in3, in2, out1, out3); \
}
#define ILVLH_D(in0, in1, in2, in3, out0, out1, out2, out3) \
{ \
DUP2_ARG2(__lsx_vilvl_d, in1, in0, in3, in2, out0, out2); \
DUP2_ARG2(__lsx_vilvh_d, in1, in0, in3, in2, out1, out3); \
}
#define LSX_ST_4(_dst0, _dst1, _dst2, _dst3, _dst, _stride, _stride2, \
_stride3, _stride4) \
{ \
__lsx_vst(_dst0, _dst, 0); \
__lsx_vstx(_dst1, _dst, _stride); \
__lsx_vstx(_dst2, _dst, _stride2); \
__lsx_vstx(_dst3, _dst, _stride3); \
_dst += _stride4; \
}
#define LSX_ST_2(_dst0, _dst1, _dst, _stride, _stride2) \
{ \
__lsx_vst(_dst0, _dst, 0); \
__lsx_vstx(_dst1, _dst, _stride); \
_dst += _stride2; \
}
void TransposeWx16_LSX(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
int x;
int len = width / 16;
uint8_t* s;
int src_stride2 = src_stride << 1;
int src_stride3 = src_stride + src_stride2;
int src_stride4 = src_stride2 << 1;
int dst_stride2 = dst_stride << 1;
int dst_stride3 = dst_stride + dst_stride2;
int dst_stride4 = dst_stride2 << 1;
__m128i src0, src1, src2, src3, dst0, dst1, dst2, dst3;
__m128i tmp0, tmp1, tmp2, tmp3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
__m128i res0, res1, res2, res3, res4, res5, res6, res7, res8, res9;
for (x = 0; x < len; x++) {
s = (uint8_t*)src;
src0 = __lsx_vld(s, 0);
src1 = __lsx_vldx(s, src_stride);
src2 = __lsx_vldx(s, src_stride2);
src3 = __lsx_vldx(s, src_stride3);
s += src_stride4;
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3);
src0 = __lsx_vld(s, 0);
src1 = __lsx_vldx(s, src_stride);
src2 = __lsx_vldx(s, src_stride2);
src3 = __lsx_vldx(s, src_stride3);
s += src_stride4;
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7);
ILVLH_W(reg0, reg4, reg1, reg5, res0, res1, res2, res3);
ILVLH_W(reg2, reg6, reg3, reg7, res4, res5, res6, res7);
src0 = __lsx_vld(s, 0);
src1 = __lsx_vldx(s, src_stride);
src2 = __lsx_vldx(s, src_stride2);
src3 = __lsx_vldx(s, src_stride3);
s += src_stride4;
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3);
src0 = __lsx_vld(s, 0);
src1 = __lsx_vldx(s, src_stride);
src2 = __lsx_vldx(s, src_stride2);
src3 = __lsx_vldx(s, src_stride3);
s += src_stride4;
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7);
res8 = __lsx_vilvl_w(reg4, reg0);
res9 = __lsx_vilvh_w(reg4, reg0);
ILVLH_D(res0, res8, res1, res9, dst0, dst1, dst2, dst3);
LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3,
dst_stride4);
res8 = __lsx_vilvl_w(reg5, reg1);
res9 = __lsx_vilvh_w(reg5, reg1);
ILVLH_D(res2, res8, res3, res9, dst0, dst1, dst2, dst3);
LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3,
dst_stride4);
res8 = __lsx_vilvl_w(reg6, reg2);
res9 = __lsx_vilvh_w(reg6, reg2);
ILVLH_D(res4, res8, res5, res9, dst0, dst1, dst2, dst3);
LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3,
dst_stride4);
res8 = __lsx_vilvl_w(reg7, reg3);
res9 = __lsx_vilvh_w(reg7, reg3);
ILVLH_D(res6, res8, res7, res9, dst0, dst1, dst2, dst3);
LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3,
dst_stride4);
src += 16;
}
}
void TransposeUVWx16_LSX(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width) {
int x;
int len = width / 8;
uint8_t* s;
int src_stride2 = src_stride << 1;
int src_stride3 = src_stride + src_stride2;
int src_stride4 = src_stride2 << 1;
int dst_stride_a2 = dst_stride_a << 1;
int dst_stride_b2 = dst_stride_b << 1;
__m128i src0, src1, src2, src3, dst0, dst1, dst2, dst3;
__m128i tmp0, tmp1, tmp2, tmp3;
__m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7;
__m128i res0, res1, res2, res3, res4, res5, res6, res7, res8, res9;
for (x = 0; x < len; x++) {
s = (uint8_t*)src;
src0 = __lsx_vld(s, 0);
src1 = __lsx_vldx(s, src_stride);
src2 = __lsx_vldx(s, src_stride2);
src3 = __lsx_vldx(s, src_stride3);
s += src_stride4;
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3);
src0 = __lsx_vld(s, 0);
src1 = __lsx_vldx(s, src_stride);
src2 = __lsx_vldx(s, src_stride2);
src3 = __lsx_vldx(s, src_stride3);
s += src_stride4;
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7);
ILVLH_W(reg0, reg4, reg1, reg5, res0, res1, res2, res3);
ILVLH_W(reg2, reg6, reg3, reg7, res4, res5, res6, res7);
src0 = __lsx_vld(s, 0);
src1 = __lsx_vldx(s, src_stride);
src2 = __lsx_vldx(s, src_stride2);
src3 = __lsx_vldx(s, src_stride3);
s += src_stride4;
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3);
src0 = __lsx_vld(s, 0);
src1 = __lsx_vldx(s, src_stride);
src2 = __lsx_vldx(s, src_stride2);
src3 = __lsx_vldx(s, src_stride3);
s += src_stride4;
ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3);
ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7);
res8 = __lsx_vilvl_w(reg4, reg0);
res9 = __lsx_vilvh_w(reg4, reg0);
ILVLH_D(res0, res8, res1, res9, dst0, dst1, dst2, dst3);
LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2);
LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2);
res8 = __lsx_vilvl_w(reg5, reg1);
res9 = __lsx_vilvh_w(reg5, reg1);
ILVLH_D(res2, res8, res3, res9, dst0, dst1, dst2, dst3);
LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2);
LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2);
res8 = __lsx_vilvl_w(reg6, reg2);
res9 = __lsx_vilvh_w(reg6, reg2);
ILVLH_D(res4, res8, res5, res9, dst0, dst1, dst2, dst3);
LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2);
LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2);
res8 = __lsx_vilvl_w(reg7, reg3);
res9 = __lsx_vilvh_w(reg7, reg3);
ILVLH_D(res6, res8, res7, res9, dst0, dst1, dst2, dst3);
LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2);
LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2);
src += 16;
}
}
void TransposeWx4_32_LSX(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
int x;
int len = width / 4;
ptrdiff_t src_stride2 = src_stride * 2;
ptrdiff_t src_stride3 = src_stride + src_stride2;
ptrdiff_t src_stride4 = src_stride * 4;
ptrdiff_t dst_stride2 = dst_stride * 2;
ptrdiff_t dst_stride3 = dst_stride + dst_stride2;
__m128i src0, src1, src2, src3;
__m128i dst0, dst1, dst2, dst3;
for (x = 0; x < len; x++) {
src0 = __lsx_vld(src, 0);
src1 = __lsx_vldx(src, src_stride);
src2 = __lsx_vldx(src, src_stride2);
src3 = __lsx_vldx(src, src_stride3);
LSX_TRANSPOSE4x4_W(src0, src1, src2, src3, dst0, dst1, dst2, dst3);
__lsx_vst(dst0, dst, 0);
__lsx_vstx(dst1, dst, dst_stride);
__lsx_vstx(dst2, dst, dst_stride2);
__lsx_vstx(dst3, dst, dst_stride3);
dst += 16;
src += src_stride4;
}
}
void TransposeWx8_16_LSX(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint8_t* src_b = (const uint8_t*)src;
uint8_t* dst_b = (uint8_t*)dst;
while (width > 0) {
__m128i r0 = __lsx_vld(src_b, 0);
__m128i r1 = __lsx_vldx(src_b, s);
__m128i r2 = __lsx_vldx(src_b, s * 2);
__m128i r3 = __lsx_vldx(src_b, s * 3);
__m128i r4 = __lsx_vldx(src_b, s * 4);
__m128i r5 = __lsx_vldx(src_b, s * 5);
__m128i r6 = __lsx_vldx(src_b, s * 6);
__m128i r7 = __lsx_vldx(src_b, s * 7);
__m128i t0 = __lsx_vilvl_h(r1, r0);
__m128i t1 = __lsx_vilvh_h(r1, r0);
__m128i t2 = __lsx_vilvl_h(r3, r2);
__m128i t3 = __lsx_vilvh_h(r3, r2);
__m128i t4 = __lsx_vilvl_h(r5, r4);
__m128i t5 = __lsx_vilvh_h(r5, r4);
__m128i t6 = __lsx_vilvl_h(r7, r6);
__m128i t7 = __lsx_vilvh_h(r7, r6);
__m128i u0 = __lsx_vilvl_w(t2, t0);
__m128i u1 = __lsx_vilvh_w(t2, t0);
__m128i u2 = __lsx_vilvl_w(t3, t1);
__m128i u3 = __lsx_vilvh_w(t3, t1);
__m128i u4 = __lsx_vilvl_w(t6, t4);
__m128i u5 = __lsx_vilvh_w(t6, t4);
__m128i u6 = __lsx_vilvl_w(t7, t5);
__m128i u7 = __lsx_vilvh_w(t7, t5);
__lsx_vst(__lsx_vilvl_d(u4, u0), dst_b, 0);
__lsx_vstx(__lsx_vilvh_d(u4, u0), dst_b, d);
__lsx_vstx(__lsx_vilvl_d(u5, u1), dst_b, d * 2);
__lsx_vstx(__lsx_vilvh_d(u5, u1), dst_b, d * 3);
__lsx_vstx(__lsx_vilvl_d(u6, u2), dst_b, d * 4);
__lsx_vstx(__lsx_vilvh_d(u6, u2), dst_b, d * 5);
__lsx_vstx(__lsx_vilvl_d(u7, u3), dst_b, d * 6);
__lsx_vstx(__lsx_vilvh_d(u7, u3), dst_b, d * 7);
src_b += 16;
dst_b += d * 8;
width -= 8;
}
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
#endif // !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
Reference in New Issue View Git Blame Copy Permalink