coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity
libyuv/source/rotate_wasm.cc
Frank Barchard eeb06a6451 [libyuv] Optimize ARGBRotate for Intel SSE2, AVX2, and AVX-512 
Optimize 32-bit ARGBRotate (0, 90, 180, 270) across Intel x86/x64
architectures (SSE2, AVX2, AVX512BW), with matching kernels for ARM
(NEON, SVE2, SME), WASM, LoongArch (LSX, LASX), and RISC-V (RVV):

- Replace column-by-column ScaleARGBRowDownEven gather loops in
  ARGBTranspose (90/270) with full-plane horizontal tile transposes:
  - AVX512BW (TransposeWxH_32_AVX512BW / TransposeNx4_32_AVX512BW):
    Transpose 16x16 blocks across zmm0..zmm31 using 4-stage
    vpunpckldq/hdq, vpunpcklqdq/hqdq, and vshufi32x4 butterflies with
    k-masked right and bottom tails, eliminating Any wrappers and scalar
    C edge fallbacks.
  - AVX2 (TransposeWxH_32_AVX2 / TransposeNx4_32_AVX2):
    Transpose 8x8 blocks across ymm0..ymm15 using vpunpck* and
    vperm2i128 with vpmaskmovd masked tails.
  - SSE2 (Transpose4x4_32_SSE2 / Transpose4x4_32_Any_SSE2):
    Transpose 4x4 blocks in xmm registers.
- Implement ARGBRotate180 directly via ARGBMirror with negative height,
  avoiding intermediate row buffer allocations and extra copies.
- Implement ARGBRotate0 via ARGBCopy.

Speedup vs Before (1275x1650, US Letter 150 DPI, 1000 iterations):
CPU                  | 0 deg   | 90 deg  | 180 deg | 270 deg
---------------------|---------|---------|---------|--------
Intel Ice Lake       | 1.02x   | 1.34x   | 1.18x   | 1.33x
Intel Skylake-X      | 1.02x   | 1.20x   | 1.07x   | 1.13x
Intel Emerald Rapids | 1.01x   | 1.07x   | 1.00x   | 1.07x
Intel AVX-512 vs C   | 1.00x   | 2.67x   | 2.41x   | 2.63x

R=rrwinterton@gmail.com

Bug: b/42280915
Test: LIBYUV_WIDTH=1275 LIBYUV_HEIGHT=1650 LIBYUV_REPEAT=1000 ./out/Release/libyuv_unittest --gtest_filter=*ARGBRotate*
Change-Id: Ib308a225c17e46eab083a9da4033598fcf0c303c
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8538986
Reviewed-by: richard winterton <rrwinterton@gmail.com>
2026-10-08 16:20:46 -07:00

278 lines
9.8 KiB
C++
Raw Blame History

/*
* Copyright 2026 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
#include <wasm_simd128.h>
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
// Interleave the low or high halves of 2 vectors as 8, 16, 32 or 64 bit
// elements. These map to punpck on x86 and zip on Arm.
#define ZIPLO8(a, b) \
wasm_i8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, \
23)
#define ZIPHI8(a, b) \
wasm_i8x16_shuffle(a, b, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, \
30, 15, 31)
#define ZIPLO16(a, b) wasm_i16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11)
#define ZIPHI16(a, b) wasm_i16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15)
#define ZIPLO32(a, b) wasm_i32x4_shuffle(a, b, 0, 4, 1, 5)
#define ZIPHI32(a, b) wasm_i32x4_shuffle(a, b, 2, 6, 3, 7)
#define ZIPLO64(a, b) wasm_i64x2_shuffle(a, b, 0, 2)
#define ZIPHI64(a, b) wasm_i64x2_shuffle(a, b, 1, 3)
// Transpose 8 rows x 16 bytes. c[m] holds columns 2m and 2m + 1 of the 8 rows
// in its low and high 8 bytes.
static inline void Transpose8x16_WASMSIMD(const uint8_t* src,
ptrdiff_t s,
v128_t* c) {
v128_t r0 = wasm_v128_load(src);
v128_t r1 = wasm_v128_load(src + s);
v128_t r2 = wasm_v128_load(src + s * 2);
v128_t r3 = wasm_v128_load(src + s * 3);
v128_t r4 = wasm_v128_load(src + s * 4);
v128_t r5 = wasm_v128_load(src + s * 5);
v128_t r6 = wasm_v128_load(src + s * 6);
v128_t r7 = wasm_v128_load(src + s * 7);
// Bytes of row pairs. a0 is columns 0..7 of rows 0, 1.
v128_t a0 = ZIPLO8(r0, r1);
v128_t a1 = ZIPHI8(r0, r1);
v128_t a2 = ZIPLO8(r2, r3);
v128_t a3 = ZIPHI8(r2, r3);
v128_t a4 = ZIPLO8(r4, r5);
v128_t a5 = ZIPHI8(r4, r5);
v128_t a6 = ZIPLO8(r6, r7);
v128_t a7 = ZIPHI8(r6, r7);
// Words. b0 is columns 0..3 of rows 0..3.
v128_t b0 = ZIPLO16(a0, a2);
v128_t b1 = ZIPHI16(a0, a2);
v128_t b2 = ZIPLO16(a1, a3);
v128_t b3 = ZIPHI16(a1, a3);
v128_t b4 = ZIPLO16(a4, a6);
v128_t b5 = ZIPHI16(a4, a6);
v128_t b6 = ZIPLO16(a5, a7);
v128_t b7 = ZIPHI16(a5, a7);
// Dwords. c0 is columns 0, 1 of rows 0..7.
c[0] = ZIPLO32(b0, b4);
c[1] = ZIPHI32(b0, b4);
c[2] = ZIPLO32(b1, b5);
c[3] = ZIPHI32(b1, b5);
c[4] = ZIPLO32(b2, b6);
c[5] = ZIPHI32(b2, b6);
c[6] = ZIPLO32(b3, b7);
c[7] = ZIPHI32(b3, b7);
}
#if defined(HAS_TRANSPOSEWX8_WASMSIMD) || defined(HAS_TRANSPOSEUVWX8_WASMSIMD)
// Transpose 8 rows x 16 bytes. Even columns go to dst_a and odd columns go to
// dst_b. Width is a multiple of 16 bytes.
static void TransposeWx8_Byte_WASMSIMD(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int byte_width) {
const ptrdiff_t sa = dst_stride_a;
const ptrdiff_t sb = dst_stride_b;
while (byte_width > 0) {
v128_t c[8];
int m;
Transpose8x16_WASMSIMD(src, src_stride, c);
for (m = 0; m < 8; ++m) {
wasm_v128_store64_lane(dst_a + sa * m, c[m], 0);
wasm_v128_store64_lane(dst_b + sb * m, c[m], 1);
}
src += 16;
dst_a += sa * 8;
dst_b += sb * 8;
byte_width -= 16;
}
}
#endif // HAS_TRANSPOSEWX8_WASMSIMD || HAS_TRANSPOSEUVWX8_WASMSIMD
#if defined(HAS_TRANSPOSEWX8_WASMSIMD)
// Transpose 16x8. Width is a multiple of 16.
void TransposeWx8_WASMSIMD(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
TransposeWx8_Byte_WASMSIMD(src, src_stride, dst, dst_stride * 2,
dst + dst_stride, dst_stride * 2, width);
}
#endif // HAS_TRANSPOSEWX8_WASMSIMD
#if defined(HAS_TRANSPOSEUVWX8_WASMSIMD)
// Transpose UV 8x8. Width is a multiple of 8.
void TransposeUVWx8_WASMSIMD(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width) {
TransposeWx8_Byte_WASMSIMD(src, src_stride, dst_a, dst_stride_a, dst_b,
dst_stride_b, width * 2);
}
#endif // HAS_TRANSPOSEUVWX8_WASMSIMD
#if defined(HAS_TRANSPOSEWX16_WASMSIMD) || defined(HAS_TRANSPOSEUVWX16_WASMSIMD)
// Transpose 16 rows x 16 bytes as 2 halves of 8 rows, which are joined with
// 64 bit interleaves to store full 16 byte rows. Even columns go to dst_a and
// odd columns go to dst_b. Width is a multiple of 16 bytes.
static void TransposeWx16_Byte_WASMSIMD(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int byte_width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t sa = dst_stride_a;
const ptrdiff_t sb = dst_stride_b;
while (byte_width > 0) {
v128_t lo[8];
v128_t hi[8];
int m;
Transpose8x16_WASMSIMD(src, s, lo);
Transpose8x16_WASMSIMD(src + s * 8, s, hi);
for (m = 0; m < 8; ++m) {
wasm_v128_store(dst_a + sa * m, ZIPLO64(lo[m], hi[m]));
wasm_v128_store(dst_b + sb * m, ZIPHI64(lo[m], hi[m]));
}
src += 16;
dst_a += sa * 8;
dst_b += sb * 8;
byte_width -= 16;
}
}
#endif // HAS_TRANSPOSEWX16_WASMSIMD || HAS_TRANSPOSEUVWX16_WASMSIMD
#if defined(HAS_TRANSPOSEWX16_WASMSIMD)
// Transpose 16x16. Width is a multiple of 16.
void TransposeWx16_WASMSIMD(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
TransposeWx16_Byte_WASMSIMD(src, src_stride, dst, dst_stride * 2,
dst + dst_stride, dst_stride * 2, width);
}
#endif // HAS_TRANSPOSEWX16_WASMSIMD
#if defined(HAS_TRANSPOSEUVWX16_WASMSIMD)
// Transpose UV 8x16. Width is a multiple of 8.
void TransposeUVWx16_WASMSIMD(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width) {
TransposeWx16_Byte_WASMSIMD(src, src_stride, dst_a, dst_stride_a, dst_b,
dst_stride_b, width * 2);
}
#endif // HAS_TRANSPOSEUVWX16_WASMSIMD
#if defined(HAS_TRANSPOSE4X4_32_WASMSIMD)
// Transpose 32 bit values (ARGB) in 4x4 tiles, moving down 4 rows of source
// and across 4 columns of destination. Width is a multiple of 4.
void Transpose4x4_32_WASMSIMD(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
while (width >= 8) {
v128_t r0 = wasm_v128_load(src);
v128_t r1 = wasm_v128_load(src + s);
v128_t r2 = wasm_v128_load(src + s * 2);
v128_t r3 = wasm_v128_load(src + s * 3);
src += s * 4;
v128_t r4 = wasm_v128_load(src);
v128_t r5 = wasm_v128_load(src + s);
v128_t r6 = wasm_v128_load(src + s * 2);
v128_t r7 = wasm_v128_load(src + s * 3);
src += s * 4;
v128_t a0 = ZIPLO32(r0, r1);
v128_t a1 = ZIPHI32(r0, r1);
v128_t a2 = ZIPLO32(r2, r3);
v128_t a3 = ZIPHI32(r2, r3);
v128_t b0 = ZIPLO32(r4, r5);
v128_t b1 = ZIPHI32(r4, r5);
v128_t b2 = ZIPLO32(r6, r7);
v128_t b3 = ZIPHI32(r6, r7);
wasm_v128_store(dst, ZIPLO64(a0, a2));
wasm_v128_store(dst1, ZIPHI64(a0, a2));
wasm_v128_store(dst2, ZIPLO64(a1, a3));
wasm_v128_store(dst3, ZIPHI64(a1, a3));
wasm_v128_store(dst + 16, ZIPLO64(b0, b2));
wasm_v128_store(dst1 + 16, ZIPHI64(b0, b2));
wasm_v128_store(dst2 + 16, ZIPLO64(b1, b3));
wasm_v128_store(dst3 + 16, ZIPHI64(b1, b3));
dst += 32;
dst1 += 32;
dst2 += 32;
dst3 += 32;
width -= 8;
}
while (width > 0) {
v128_t r0 = wasm_v128_load(src);
v128_t r1 = wasm_v128_load(src + s);
v128_t r2 = wasm_v128_load(src + s * 2);
v128_t r3 = wasm_v128_load(src + s * 3);
v128_t a0 = ZIPLO32(r0, r1);
v128_t a1 = ZIPHI32(r0, r1);
v128_t a2 = ZIPLO32(r2, r3);
v128_t a3 = ZIPHI32(r2, r3);
wasm_v128_store(dst, ZIPLO64(a0, a2));
wasm_v128_store(dst1, ZIPHI64(a0, a2));
wasm_v128_store(dst2, ZIPLO64(a1, a3));
wasm_v128_store(dst3, ZIPHI64(a1, a3));
src += s * 4;
dst += 16;
dst1 += 16;
dst2 += 16;
dst3 += 16;
width -= 4;
}
}
#endif // HAS_TRANSPOSE4X4_32_WASMSIMD
#undef ZIPLO8
#undef ZIPHI8
#undef ZIPLO16
#undef ZIPHI16
#undef ZIPLO32
#undef ZIPHI32
#undef ZIPLO64
#undef ZIPHI64
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
Reference in New Issue View Git Blame Copy Permalink