mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
[libyuv] Optimize RotatePlane_16 SIMD and compact 32-bit transposes
Implement SIMD 16-bit plane rotation (TransposePlane_16, RotatePlane90_16, RotatePlane180_16, RotatePlane270_16) used by I010Rotate, I210Rotate, and I410Rotate, keeping tile sizes compact with full-register writes and zero cross-lane shuffles: - AVX2: 8x16 tile in ymm0..ymm15 with _Any tail (low-end x86 SIMD) - AVX-512BW: 8x32 tile in zmm0..zmm15 with k1/k2 right/bottom masks - NEON (ARM64/ARM32): 8x16 tile in v0..v23 / q0..q11 with _Any tail - SME: 8xVL tile using ZA16 horizontal/vertical array slices - RVV: 8xVL tile using widening unpacks with hardware vl tail - WASM SIMD: 8x8 tile in v128 with _Any tail Also update 32-bit ARGBRotate x86 transposes to compact vertical in-lane tiles with full-register writes and zero cross-lane shuffles: - TransposeWxH_32_AVX2: 4x8 tile in ymm0..ymm7 with vpmaskmovd tails - TransposeWxH_32_AVX512BW: 4x16 tile in zmm0..zmm7 with k1/k2 tails Add RotatePlane16_SplitMergeUV_Opt and RotatePlane16_SplitMergeUV_Odd unit tests comparing 16-bit rotation against SplitUVPlane + RotatePlane + MergeUVPlane across 0/90/180/270 degrees. Performance on 1275x1650 (1000 iterations, before -> after): CPU 90 degree 180 degree Coral RVV (Verilator) 7306 -> 1415c (5.16x) 2274 -> 719c (3.16x) AMD Rome (Zen 2) 821 -> 215ms (3.82x) 259 -> 181ms (1.44x) AMD Genoa (Zen 4) 483 -> 129ms (3.73x) 149 -> 111ms (1.34x) AMD Turin (Zen 5) 321 -> 101ms (3.18x) 109 -> 78ms (1.40x) AMD Milan (Zen 3) 505 -> 189ms (2.67x) 199 -> 118ms (1.69x) Intel Skylake-X 1076 -> 447ms (2.41x) 353 -> 317ms (1.11x) WASM SIMD (d8) 415 -> 203ms (2.04x) 370 -> 106ms (3.49x) Intel Emerald Rapids 735 -> 422ms (1.74x) 238 -> 233ms (1.02x) Intel Haswell 1120 -> 648ms (1.73x) 438 -> 393ms (1.11x) Intel Ice Lake 774 -> 474ms (1.63x) 279 -> 290ms (0.96x) ARM Neoverse-N1 864 -> 544ms (1.59x) 404 -> 266ms (1.52x) Ampere Siryn 862 -> 576ms (1.50x) 568 -> 329ms (1.72x) Google Axion (Maple) 554 -> 379ms (1.46x) 199 -> 142ms (1.41x) Intel Sapphire Rapids 1126 -> 772ms (1.46x) 301 -> 288ms (1.04x) Intel Granite Rapids 730 -> 565ms (1.29x) 280 -> 280ms (1.00x) NVIDIA Grace 462 -> 365ms (1.26x) 192 -> 126ms (1.52x) ARM Cortex-A53 28290 -> 24386ms (1.16x) 7178 -> 5075ms (1.41x) Google Axion 2 (Tamar) 523 -> 453ms (1.15x) 217 -> 112ms (1.93x) Test: libyuv_unittest --gtest_filter=*Rotate* Bug: b/42280942 Change-Id: I91ba6cac22185b5d294fcb98344fdb8a23cc94ff Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8545294 Reviewed-by: richard winterton <rrwinterton@gmail.com> Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard
libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent
eeb06a6451
commit
16af162cf9
@ -31,16 +31,16 @@ extern "C" {
|
||||
defined(LIBYUV_ENABLE_ROWWIN))))
|
||||
#define HAS_TRANSPOSEWX8_SSSE3
|
||||
#define HAS_TRANSPOSEUVWX8_SSE2
|
||||
#define HAS_TRANSPOSE4X4_32_SSE2
|
||||
#define HAS_TRANSPOSENX4_32_AVX2
|
||||
#define HAS_TRANSPOSEWXH_32_AVX2
|
||||
#define HAS_TRANSPOSEWX16_16_AVX2
|
||||
#define HAS_TRANSPOSEWX4_32_SSE2
|
||||
#define HAS_TRANSPOSEWX8_32_AVX2
|
||||
#define HAS_TRANSPOSEWX16_AVX2
|
||||
#define HAS_TRANSPOSEUVWX16_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define HAS_TRANSPOSEWX16_AVX512BW
|
||||
#define HAS_TRANSPOSEUVWX16_AVX512BW
|
||||
#define HAS_TRANSPOSENX4_32_AVX512BW
|
||||
#define HAS_TRANSPOSEWXH_32_AVX512BW
|
||||
#define HAS_TRANSPOSEWX32_16_AVX512BW
|
||||
#define HAS_TRANSPOSEWX16_32_AVX512BW
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@ -50,9 +50,9 @@ extern "C" {
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||
#define HAS_TRANSPOSEWX8_SSSE3
|
||||
#define HAS_TRANSPOSEUVWX8_SSE2
|
||||
#define HAS_TRANSPOSE4X4_32_SSE2
|
||||
#define HAS_TRANSPOSENX4_32_AVX2
|
||||
#define HAS_TRANSPOSEWXH_32_AVX2
|
||||
#define HAS_TRANSPOSEWX16_16_AVX2
|
||||
#define HAS_TRANSPOSEWX4_32_SSE2
|
||||
#define HAS_TRANSPOSEWX8_32_AVX2
|
||||
#define HAS_TRANSPOSEWX16_AVX2
|
||||
#define HAS_TRANSPOSEUVWX16_AVX2
|
||||
#endif
|
||||
@ -62,8 +62,8 @@ extern "C" {
|
||||
defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN)
|
||||
#define HAS_TRANSPOSEWX16_AVX512BW
|
||||
#define HAS_TRANSPOSEUVWX16_AVX512BW
|
||||
#define HAS_TRANSPOSENX4_32_AVX512BW
|
||||
#define HAS_TRANSPOSEWXH_32_AVX512BW
|
||||
#define HAS_TRANSPOSEWX32_16_AVX512BW
|
||||
#define HAS_TRANSPOSEWX16_32_AVX512BW
|
||||
#endif
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && \
|
||||
@ -74,33 +74,39 @@ extern "C" {
|
||||
#define HAS_TRANSPOSEWX8_NEON
|
||||
#endif
|
||||
#define HAS_TRANSPOSEUVWX8_NEON
|
||||
#define HAS_TRANSPOSENX4_32_NEON
|
||||
#define HAS_TRANSPOSEWX8_16_NEON
|
||||
#define HAS_TRANSPOSEWX4_32_NEON
|
||||
#define HAS_TRANSPOSEWXH_32_NEON
|
||||
#endif
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
||||
#define HAS_TRANSPOSENX4_32_SVE2
|
||||
#define HAS_TRANSPOSEWX8_16_SVE2
|
||||
#define HAS_TRANSPOSEWX4_32_SVE2
|
||||
#endif
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
|
||||
defined(__aarch64__)
|
||||
#define HAS_TRANSPOSEWXH_SME
|
||||
#define HAS_TRANSPOSEUVWXH_SME
|
||||
#define HAS_TRANSPOSEWXH_16_SME
|
||||
#define HAS_TRANSPOSEWXH_32_SME
|
||||
#endif
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
|
||||
#define HAS_TRANSPOSEWX16_LSX
|
||||
#define HAS_TRANSPOSEUVWX16_LSX
|
||||
#define HAS_TRANSPOSE4X4_32_LSX
|
||||
#define HAS_TRANSPOSEWX8_16_LSX
|
||||
#define HAS_TRANSPOSEWX4_32_LSX
|
||||
#endif
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
|
||||
#define HAS_TRANSPOSE4X4_32_LASX
|
||||
#define HAS_TRANSPOSEWX16_16_LASX
|
||||
#define HAS_TRANSPOSEWX4_32_LASX
|
||||
#endif
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
|
||||
#define HAS_TRANSPOSENX4_32_RVV
|
||||
#define HAS_TRANSPOSEWXH_16_RVV
|
||||
#define HAS_TRANSPOSEWX4_32_RVV
|
||||
#define HAS_TRANSPOSEWXH_32_RVV
|
||||
#endif
|
||||
|
||||
@ -109,7 +115,8 @@ extern "C" {
|
||||
#define HAS_TRANSPOSEUVWX8_WASMSIMD
|
||||
#define HAS_TRANSPOSEWX16_WASMSIMD
|
||||
#define HAS_TRANSPOSEUVWX16_WASMSIMD
|
||||
#define HAS_TRANSPOSE4X4_32_WASMSIMD
|
||||
#define HAS_TRANSPOSEWX8_16_WASMSIMD
|
||||
#define HAS_TRANSPOSEWX4_32_WASMSIMD
|
||||
#endif
|
||||
|
||||
void TransposeWxH_C(const uint8_t* src,
|
||||
@ -347,11 +354,78 @@ void TransposeWx8_16_C(const uint16_t* src,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx1_16_C(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx16_16_AVX2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx32_16_AVX512BW(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_16_NEON(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_16_SVE2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWxH_16_SME(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height);
|
||||
void TransposeWxH_16_RVV(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height);
|
||||
void TransposeWx8_16_LSX(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx16_16_LASX(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_16_WASMSIMD(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx16_16_Any_AVX2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_16_Any_NEON(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_16_Any_LSX(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx16_16_Any_LASX(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_16_Any_WASMSIMD(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeWxH_32_C(const uint8_t* src,
|
||||
int src_stride,
|
||||
@ -367,7 +441,7 @@ void TransposeWxH_32_SME(const uint8_t* src,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
void TransposeNx4_32_NEON(const uint8_t* src,
|
||||
void TransposeWx4_32_NEON(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -380,45 +454,31 @@ void TransposeWxH_32_NEON(const uint8_t* src,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
void TransposeNx4_32_SVE2(const uint8_t* src,
|
||||
void TransposeWx4_32_SVE2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void Transpose4x4_32_SSE2(const uint8_t* src,
|
||||
void TransposeWx4_32_SSE2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeNx4_32_AVX2(const uint8_t* src,
|
||||
void TransposeWx8_32_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeWxH_32_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height);
|
||||
void TransposeWx16_32_AVX512BW(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeNx4_32_AVX512BW(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeWxH_32_AVX512BW(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
void TransposeNx4_32_RVV(const uint8_t* src,
|
||||
void TransposeWx4_32_RVV(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -431,49 +491,55 @@ void TransposeWxH_32_RVV(const uint8_t* src,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
void Transpose4x4_32_LSX(const uint8_t* src,
|
||||
void TransposeWx4_32_LSX(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void Transpose4x4_32_LASX(const uint8_t* src,
|
||||
void TransposeWx4_32_LASX(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void Transpose4x4_32_WASMSIMD(const uint8_t* src,
|
||||
void TransposeWx4_32_WASMSIMD(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void Transpose4x4_32_C(const uint8_t* src,
|
||||
void TransposeWx4_32_C(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void Transpose4x4_32_Any_SSE2(const uint8_t* src,
|
||||
void TransposeWx4_32_Any_SSE2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void Transpose4x4_32_Any_LSX(const uint8_t* src,
|
||||
void TransposeWx8_32_Any_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeWx4_32_Any_LSX(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void Transpose4x4_32_Any_LASX(const uint8_t* src,
|
||||
void TransposeWx4_32_Any_LASX(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void Transpose4x4_32_Any_WASMSIMD(const uint8_t* src,
|
||||
void TransposeWx4_32_Any_WASMSIMD(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
|
||||
@ -412,6 +412,9 @@ extern "C" {
|
||||
#define HAS_INTERPOLATEROW_16_AVX2
|
||||
#define HAS_MULTIPLYROW_16_AVX2
|
||||
#define HAS_SWAPUVROW_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define HAS_MIRRORUVROW_AVX512BW
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// The following are available for AVX512 clang x64 platforms:
|
||||
@ -452,6 +455,7 @@ extern "C" {
|
||||
#define HAS_J400TOARGBROW_AVX512BW
|
||||
#define HAS_MERGEUVROW_AVX512BW
|
||||
#define HAS_MIRRORROW_AVX512BW
|
||||
#define HAS_MIRRORUVROW_AVX512BW
|
||||
#define HAS_MIRRORSPLITUVROW_AVX512BW
|
||||
#define HAS_SPLITUVROW_AVX512BW
|
||||
#define HAS_RGBTOUVMATRIXROW_AVX512BW
|
||||
@ -930,6 +934,7 @@ extern "C" {
|
||||
#define HAS_MERGERGBROW_RVV
|
||||
#define HAS_MERGEUVROW_RVV
|
||||
#define HAS_MERGEXRGBROW_RVV
|
||||
#define HAS_MIRRORUVROW_RVV
|
||||
#define HAS_NV12TOARGBROW_RVV
|
||||
#define HAS_NV12TORGB24ROW_RVV
|
||||
#define HAS_NV21TOARGBROW_RVV
|
||||
@ -2749,13 +2754,19 @@ void MirrorRow_Any_SSE2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void MirrorRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
void MirrorRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
void MirrorRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width);
|
||||
void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width);
|
||||
void MirrorUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_uv, int width);
|
||||
void MirrorUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_uv, int width);
|
||||
void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width);
|
||||
void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width);
|
||||
void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width);
|
||||
void MirrorUVRow_C(const uint8_t* src_uv, uint8_t* dst_uv, int width);
|
||||
void MirrorUVRow_Any_AVX512BW(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void MirrorUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
void MirrorUVRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
void MirrorUVRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
void MirrorUVRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
void MirrorUVRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
|
||||
@ -2785,6 +2785,7 @@ void MirrorUVPlane(const uint8_t* src_uv,
|
||||
#endif
|
||||
#if defined(HAS_MIRRORUVROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
MirrorUVRow = MirrorUVRow_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
MirrorUVRow = MirrorUVRow_SSSE3;
|
||||
}
|
||||
@ -2798,6 +2799,14 @@ void MirrorUVPlane(const uint8_t* src_uv,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MirrorUVRow = MirrorUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
MirrorUVRow = MirrorUVRow_AVX512BW;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORUVROW_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
MirrorUVRow = MirrorUVRow_Any_LSX;
|
||||
@ -2814,6 +2823,11 @@ void MirrorUVPlane(const uint8_t* src_uv,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORUVROW_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
MirrorUVRow = MirrorUVRow_RVV;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORUVROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
MirrorUVRow = MirrorUVRow_Any_WASMSIMD;
|
||||
|
||||
134
source/rotate.cc
134
source/rotate.cc
@ -618,11 +618,113 @@ static void TransposePlane_16(const uint16_t* src,
|
||||
int width,
|
||||
int height) {
|
||||
int i = height;
|
||||
#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV)
|
||||
void (*TransposeWxH_16)(const uint16_t* src, int src_stride, uint16_t* dst,
|
||||
int dst_stride, int width, int height) = NULL;
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
||||
void (*TransposeWx32_16)(const uint16_t* src, int src_stride, uint16_t* dst,
|
||||
int dst_stride, int width) = NULL;
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX)
|
||||
void (*TransposeWx16_16)(const uint16_t* src, int src_stride, uint16_t* dst,
|
||||
int dst_stride, int width) = NULL;
|
||||
#endif
|
||||
void (*TransposeWx8_16)(const uint16_t* src, int src_stride, uint16_t* dst,
|
||||
int dst_stride, int width) = TransposeWx8_16_C;
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX8_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
TransposeWx8_16 = TransposeWx8_16_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
TransposeWx8_16 = TransposeWx8_16_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX8_16_SVE2)
|
||||
if (TestCpuFlag(kCpuHasSVE2)) {
|
||||
TransposeWx8_16 = TransposeWx8_16_SVE2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_16_SME)
|
||||
if (TestCpuFlag(kCpuHasSME)) {
|
||||
TransposeWxH_16 = TransposeWxH_16_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_16_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
TransposeWxH_16 = TransposeWxH_16_RVV;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX16_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
TransposeWx16_16 = TransposeWx16_16_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
TransposeWx16_16 = TransposeWx16_16_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
TransposeWx32_16 = TransposeWx32_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX8_16_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
TransposeWx8_16 = TransposeWx8_16_Any_LSX;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
TransposeWx8_16 = TransposeWx8_16_LSX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX16_16_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
TransposeWx16_16 = TransposeWx16_16_Any_LASX;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
TransposeWx16_16 = TransposeWx16_16_LASX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
TransposeWx8_16 = TransposeWx8_16_Any_WASMSIMD;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
TransposeWx8_16 = TransposeWx8_16_WASMSIMD;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV)
|
||||
if (TransposeWxH_16) {
|
||||
TransposeWxH_16(src, src_stride, dst, dst_stride, width, height);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
||||
if (TransposeWx32_16) {
|
||||
while (i >= 32) {
|
||||
TransposeWx32_16(src, src_stride, dst, dst_stride, width);
|
||||
src += 32 * (ptrdiff_t)src_stride;
|
||||
dst += 32;
|
||||
i -= 32;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX)
|
||||
if (TransposeWx16_16) {
|
||||
while (i >= 16) {
|
||||
TransposeWx16_16(src, src_stride, dst, dst_stride, width);
|
||||
src += 16 * (ptrdiff_t)src_stride;
|
||||
dst += 16;
|
||||
i -= 16;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// Work across the source in 8x8 tiles
|
||||
while (i >= 8) {
|
||||
TransposeWx8_16_C(src, src_stride, dst, dst_stride, width);
|
||||
src += 8 * src_stride; // Go down 8 rows.
|
||||
dst += 8; // Move over 8 columns.
|
||||
TransposeWx8_16(src, src_stride, dst, dst_stride, width);
|
||||
src += 8 * (ptrdiff_t)src_stride; // Go down 8 rows.
|
||||
dst += 8; // Move over 8 columns.
|
||||
i -= 8;
|
||||
}
|
||||
|
||||
@ -665,29 +767,9 @@ static void RotatePlane180_16(const uint16_t* src,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
const uint16_t* src_bot = src + (ptrdiff_t)src_stride * (height - 1);
|
||||
uint16_t* dst_bot = dst + (ptrdiff_t)dst_stride * (height - 1);
|
||||
int half_height = (height + 1) >> 1;
|
||||
int y;
|
||||
|
||||
// Swap top and bottom row and mirror the content. Uses a temporary row.
|
||||
align_buffer_64(row, width * 2);
|
||||
uint16_t* row_tmp = (uint16_t*)row;
|
||||
assert(row);
|
||||
if (!row)
|
||||
return;
|
||||
|
||||
// Odd height will harmlessly mirror the middle row twice.
|
||||
for (y = 0; y < half_height; ++y) {
|
||||
CopyRow_16_C(src, row_tmp, width); // Copy top row into buffer
|
||||
MirrorRow_16_C(src_bot, dst, width); // Mirror bottom row into top row
|
||||
MirrorRow_16_C(row_tmp, dst_bot, width); // Mirror buffer into bottom row
|
||||
src += src_stride;
|
||||
dst += dst_stride;
|
||||
src_bot -= src_stride;
|
||||
dst_bot -= dst_stride;
|
||||
}
|
||||
free_aligned_buffer_64(row);
|
||||
// Rotate by 180 is a vertical flip (negative height) and horizontal mirror.
|
||||
MirrorUVPlane((const uint8_t*)src, src_stride * 2, (uint8_t*)dst,
|
||||
dst_stride * 2, width, -height);
|
||||
}
|
||||
|
||||
LIBYUV_API
|
||||
|
||||
@ -105,23 +105,71 @@ TUVANY(TransposeUVWx16_Any_WASMSIMD,
|
||||
} \
|
||||
}
|
||||
|
||||
#ifdef HAS_TRANSPOSE4X4_32_SSE2
|
||||
TANY32(Transpose4x4_32_Any_SSE2, Transpose4x4_32_SSE2, TransposeWxH_32_C, 3)
|
||||
#ifdef HAS_TRANSPOSEWX4_32_SSE2
|
||||
TANY32(TransposeWx4_32_Any_SSE2, TransposeWx4_32_SSE2, TransposeWxH_32_C, 3)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSE4X4_32_LSX
|
||||
TANY32(Transpose4x4_32_Any_LSX, Transpose4x4_32_LSX, TransposeWxH_32_C, 3)
|
||||
#ifdef HAS_TRANSPOSEWX4_32_LSX
|
||||
TANY32(TransposeWx4_32_Any_LSX, TransposeWx4_32_LSX, TransposeWxH_32_C, 3)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSE4X4_32_LASX
|
||||
TANY32(Transpose4x4_32_Any_LASX, Transpose4x4_32_LASX, TransposeWxH_32_C, 7)
|
||||
#ifdef HAS_TRANSPOSEWX4_32_LASX
|
||||
TANY32(TransposeWx4_32_Any_LASX, TransposeWx4_32_LASX, TransposeWxH_32_C, 7)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSE4X4_32_WASMSIMD
|
||||
TANY32(Transpose4x4_32_Any_WASMSIMD,
|
||||
Transpose4x4_32_WASMSIMD,
|
||||
#ifdef HAS_TRANSPOSEWX4_32_WASMSIMD
|
||||
TANY32(TransposeWx4_32_Any_WASMSIMD,
|
||||
TransposeWx4_32_WASMSIMD,
|
||||
TransposeWxH_32_C,
|
||||
3)
|
||||
#endif
|
||||
#undef TANY32
|
||||
|
||||
#define TANY32_SRC(NAMEANY, TPOS_SIMD, MASK, HEIGHT) \
|
||||
void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
|
||||
int dst_stride, int width) { \
|
||||
int r = width & MASK; \
|
||||
int n = width - r; \
|
||||
if (n > 0) { \
|
||||
TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
TransposeWxH_32_C(src + np * 4, src_stride, dst + np * dst_stride, \
|
||||
dst_stride, r, HEIGHT); \
|
||||
}
|
||||
|
||||
#ifdef HAS_TRANSPOSEWX8_32_AVX2
|
||||
TANY32_SRC(TransposeWx8_32_Any_AVX2, TransposeWx8_32_AVX2, 3, 8)
|
||||
#endif
|
||||
#undef TANY32_SRC
|
||||
|
||||
#define TANY16(NAMEANY, TPOS_SIMD, MASK, HEIGHT) \
|
||||
void NAMEANY(const uint16_t* src, int src_stride, uint16_t* dst, \
|
||||
int dst_stride, int width) { \
|
||||
int r = width & MASK; \
|
||||
int n = width - r; \
|
||||
if (n > 0) { \
|
||||
TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
TransposeWxH_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \
|
||||
r, HEIGHT); \
|
||||
}
|
||||
|
||||
#ifdef HAS_TRANSPOSEWX8_16_NEON
|
||||
TANY16(TransposeWx8_16_Any_NEON, TransposeWx8_16_NEON, 7, 8)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEWX16_16_AVX2
|
||||
TANY16(TransposeWx16_16_Any_AVX2, TransposeWx16_16_AVX2, 7, 16)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEWX8_16_LSX
|
||||
TANY16(TransposeWx8_16_Any_LSX, TransposeWx8_16_LSX, 7, 8)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEWX16_16_LASX
|
||||
TANY16(TransposeWx16_16_Any_LASX, TransposeWx16_16_LASX, 7, 16)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEWX8_16_WASMSIMD
|
||||
TANY16(TransposeWx8_16_Any_WASMSIMD, TransposeWx8_16_WASMSIMD, 7, 8)
|
||||
#endif
|
||||
#undef TANY16
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -30,35 +30,44 @@ static int ARGBTranspose(const uint8_t* src_argb,
|
||||
int dst_stride_argb,
|
||||
int width,
|
||||
int height) {
|
||||
#if defined(HAS_TRANSPOSEWXH_32_AVX2) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_AVX512BW) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
|
||||
#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_RVV)
|
||||
void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst,
|
||||
int dst_stride, int width, int height) = NULL;
|
||||
#endif
|
||||
void (*Transpose4x4_32)(const uint8_t* src, int src_stride, uint8_t* dst,
|
||||
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
||||
void (*TransposeWx16_32)(const uint8_t* src, int src_stride, uint8_t* dst,
|
||||
int dst_stride, int width) = NULL;
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
|
||||
void (*TransposeWx8_32)(const uint8_t* src, int src_stride, uint8_t* dst,
|
||||
int dst_stride, int width) = NULL;
|
||||
#endif
|
||||
void (*TransposeWx4_32)(const uint8_t* src, int src_stride, uint8_t* dst,
|
||||
int dst_stride, int width) = NULL;
|
||||
// Check stride is a multiple of 4.
|
||||
if (src_stride_argb & 3) {
|
||||
return -1;
|
||||
}
|
||||
#if defined(HAS_TRANSPOSE4X4_32_SSE2)
|
||||
#if defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
Transpose4x4_32 = Transpose4x4_32_Any_SSE2;
|
||||
TransposeWx4_32 = TransposeWx4_32_Any_SSE2;
|
||||
if (IS_ALIGNED(height, 4)) { // Width of dest.
|
||||
Transpose4x4_32 = Transpose4x4_32_SSE2;
|
||||
TransposeWx4_32 = TransposeWx4_32_SSE2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
TransposeWxH_32 = TransposeWxH_32_AVX2;
|
||||
TransposeWx8_32 = TransposeWx8_32_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 4)) {
|
||||
TransposeWx8_32 = TransposeWx8_32_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_32_AVX512BW)
|
||||
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
TransposeWxH_32 = TransposeWxH_32_AVX512BW;
|
||||
TransposeWx16_32 = TransposeWx16_32_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_32_NEON)
|
||||
@ -66,9 +75,9 @@ static int ARGBTranspose(const uint8_t* src_argb,
|
||||
TransposeWxH_32 = TransposeWxH_32_NEON;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSENX4_32_SVE2)
|
||||
#if defined(HAS_TRANSPOSEWX4_32_SVE2)
|
||||
if (TestCpuFlag(kCpuHasSVE2)) {
|
||||
Transpose4x4_32 = TransposeNx4_32_SVE2;
|
||||
TransposeWx4_32 = TransposeWx4_32_SVE2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_32_SME)
|
||||
@ -81,34 +90,32 @@ static int ARGBTranspose(const uint8_t* src_argb,
|
||||
TransposeWxH_32 = TransposeWxH_32_RVV;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSE4X4_32_LSX)
|
||||
#if defined(HAS_TRANSPOSEWX4_32_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
Transpose4x4_32 = Transpose4x4_32_Any_LSX;
|
||||
TransposeWx4_32 = TransposeWx4_32_Any_LSX;
|
||||
if (IS_ALIGNED(height, 4)) { // Width of dest.
|
||||
Transpose4x4_32 = Transpose4x4_32_LSX;
|
||||
TransposeWx4_32 = TransposeWx4_32_LSX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSE4X4_32_LASX)
|
||||
#if defined(HAS_TRANSPOSEWX4_32_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
Transpose4x4_32 = Transpose4x4_32_Any_LASX;
|
||||
TransposeWx4_32 = TransposeWx4_32_Any_LASX;
|
||||
if (IS_ALIGNED(height, 8)) { // Width of dest.
|
||||
Transpose4x4_32 = Transpose4x4_32_LASX;
|
||||
TransposeWx4_32 = TransposeWx4_32_LASX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSE4X4_32_WASMSIMD)
|
||||
#if defined(HAS_TRANSPOSEWX4_32_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
Transpose4x4_32 = Transpose4x4_32_Any_WASMSIMD;
|
||||
TransposeWx4_32 = TransposeWx4_32_Any_WASMSIMD;
|
||||
if (IS_ALIGNED(height, 4)) { // Width of dest.
|
||||
Transpose4x4_32 = Transpose4x4_32_WASMSIMD;
|
||||
TransposeWx4_32 = TransposeWx4_32_WASMSIMD;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_TRANSPOSEWXH_32_AVX2) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_AVX512BW) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
|
||||
#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_RVV)
|
||||
if (TransposeWxH_32) {
|
||||
TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
|
||||
@ -116,18 +123,42 @@ static int ARGBTranspose(const uint8_t* src_argb,
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
if (Transpose4x4_32) {
|
||||
while (width >= 4) {
|
||||
Transpose4x4_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
||||
height);
|
||||
src_argb += 4 * 4;
|
||||
dst_argb += 4 * (ptrdiff_t)dst_stride_argb;
|
||||
width -= 4;
|
||||
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
||||
if (TransposeWx16_32) {
|
||||
while (height >= 16) {
|
||||
TransposeWx16_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
||||
width);
|
||||
src_argb += 16 * (ptrdiff_t)src_stride_argb;
|
||||
dst_argb += 16 * 4;
|
||||
height -= 16;
|
||||
}
|
||||
}
|
||||
if (width > 0) {
|
||||
TransposeWxH_32_C(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
||||
width, height);
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
|
||||
if (TransposeWx8_32) {
|
||||
while (height >= 8) {
|
||||
TransposeWx8_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
||||
width);
|
||||
src_argb += 8 * (ptrdiff_t)src_stride_argb;
|
||||
dst_argb += 8 * 4;
|
||||
height -= 8;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (height > 0) {
|
||||
if (TransposeWx4_32) {
|
||||
while (width >= 4) {
|
||||
TransposeWx4_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
||||
height);
|
||||
src_argb += 4 * 4;
|
||||
dst_argb += 4 * (ptrdiff_t)dst_stride_argb;
|
||||
width -= 4;
|
||||
}
|
||||
}
|
||||
if (width > 0) {
|
||||
TransposeWxH_32_C(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
||||
width, height);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -166,7 +166,7 @@ static inline void Copy4(uint8_t* dst, const uint8_t* src) {
|
||||
}
|
||||
|
||||
// Transpose 32 bit values (ARGB)
|
||||
void Transpose4x4_32_C(const uint8_t* src,
|
||||
void TransposeWx4_32_C(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
|
||||
1359
source/rotate_gcc.cc
1359
source/rotate_gcc.cc
File diff suppressed because it is too large
Load Diff
@ -19,8 +19,8 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
// Transpose 32 bit values (ARGB) in 2x 4x4 tiles per step (4 cols x 8 rows ->
|
||||
// 4 rows x 8 cols), matching Transpose4x4_32_AVX2. Width is a multiple of 8.
|
||||
void Transpose4x4_32_LASX(const uint8_t* src,
|
||||
// 8 cols x 4 rows). Width is a multiple of 8.
|
||||
void TransposeWx4_32_LASX(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -95,6 +95,121 @@ void Transpose4x4_32_LASX(const uint8_t* src,
|
||||
"$xr7");
|
||||
}
|
||||
|
||||
void TransposeWx16_16_LASX(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
|
||||
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
|
||||
const uint8_t* src_b = (const uint8_t*)src;
|
||||
uint8_t* dst_b = (uint8_t*)dst;
|
||||
const uint8_t* t;
|
||||
asm volatile(
|
||||
"srli.w %[w], %[w], 3 \n"
|
||||
"beqz %[w], 2f \n"
|
||||
"1: \n"
|
||||
"move %[t], %[src] \n"
|
||||
"vld $vr0, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr1, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr2, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr3, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr4, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr5, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr6, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr7, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
|
||||
"vld $vr8, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr9, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr10, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr11, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr12, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr13, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr14, %[t], 0 \n"
|
||||
"add.d %[t], %[t], %[s] \n"
|
||||
"vld $vr15, %[t], 0 \n"
|
||||
"addi.d %[src], %[src], 16 \n"
|
||||
|
||||
"xvpermi.q $xr0, $xr8, 0x02 \n"
|
||||
"xvpermi.q $xr1, $xr9, 0x02 \n"
|
||||
"xvpermi.q $xr2, $xr10, 0x02 \n"
|
||||
"xvpermi.q $xr3, $xr11, 0x02 \n"
|
||||
"xvpermi.q $xr4, $xr12, 0x02 \n"
|
||||
"xvpermi.q $xr5, $xr13, 0x02 \n"
|
||||
"xvpermi.q $xr6, $xr14, 0x02 \n"
|
||||
"xvpermi.q $xr7, $xr15, 0x02 \n"
|
||||
|
||||
"xvilvl.h $xr8, $xr1, $xr0 \n"
|
||||
"xvilvh.h $xr9, $xr1, $xr0 \n"
|
||||
"xvilvl.h $xr10, $xr3, $xr2 \n"
|
||||
"xvilvh.h $xr11, $xr3, $xr2 \n"
|
||||
"xvilvl.h $xr12, $xr5, $xr4 \n"
|
||||
"xvilvh.h $xr13, $xr5, $xr4 \n"
|
||||
"xvilvl.h $xr14, $xr7, $xr6 \n"
|
||||
"xvilvh.h $xr15, $xr7, $xr6 \n"
|
||||
|
||||
"xvilvl.w $xr0, $xr10, $xr8 \n"
|
||||
"xvilvh.w $xr1, $xr10, $xr8 \n"
|
||||
"xvilvl.w $xr2, $xr11, $xr9 \n"
|
||||
"xvilvh.w $xr3, $xr11, $xr9 \n"
|
||||
"xvilvl.w $xr4, $xr14, $xr12 \n"
|
||||
"xvilvh.w $xr5, $xr14, $xr12 \n"
|
||||
"xvilvl.w $xr6, $xr15, $xr13 \n"
|
||||
"xvilvh.w $xr7, $xr15, $xr13 \n"
|
||||
|
||||
"xvilvl.d $xr8, $xr4, $xr0 \n"
|
||||
"xvilvh.d $xr9, $xr4, $xr0 \n"
|
||||
"xvilvl.d $xr10, $xr5, $xr1 \n"
|
||||
"xvilvh.d $xr11, $xr5, $xr1 \n"
|
||||
"xvilvl.d $xr12, $xr6, $xr2 \n"
|
||||
"xvilvh.d $xr13, $xr6, $xr2 \n"
|
||||
"xvilvl.d $xr14, $xr7, $xr3 \n"
|
||||
"xvilvh.d $xr15, $xr7, $xr3 \n"
|
||||
|
||||
"xvst $xr8, %[dst], 0 \n"
|
||||
"add.d %[dst], %[dst], %[d] \n"
|
||||
"xvst $xr9, %[dst], 0 \n"
|
||||
"add.d %[dst], %[dst], %[d] \n"
|
||||
"xvst $xr10, %[dst], 0 \n"
|
||||
"add.d %[dst], %[dst], %[d] \n"
|
||||
"xvst $xr11, %[dst], 0 \n"
|
||||
"add.d %[dst], %[dst], %[d] \n"
|
||||
"xvst $xr12, %[dst], 0 \n"
|
||||
"add.d %[dst], %[dst], %[d] \n"
|
||||
"xvst $xr13, %[dst], 0 \n"
|
||||
"add.d %[dst], %[dst], %[d] \n"
|
||||
"xvst $xr14, %[dst], 0 \n"
|
||||
"add.d %[dst], %[dst], %[d] \n"
|
||||
"xvst $xr15, %[dst], 0 \n"
|
||||
"add.d %[dst], %[dst], %[d] \n"
|
||||
"addi.w %[w], %[w], -1 \n"
|
||||
"bnez %[w], 1b \n"
|
||||
"2: \n"
|
||||
: [src] "+r"(src_b), // %[src]
|
||||
[dst] "+r"(dst_b), // %[dst]
|
||||
[w] "+r"(width), // %[w]
|
||||
[t] "=&r"(t) // %[t]
|
||||
: [s] "r"(s), // %[s]
|
||||
[d] "r"(d) // %[d]
|
||||
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
|
||||
"$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
|
||||
"$xr15");
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -212,7 +212,7 @@ void TransposeUVWx16_LSX(const uint8_t* src,
|
||||
}
|
||||
}
|
||||
|
||||
void Transpose4x4_32_LSX(const uint8_t* src,
|
||||
void TransposeWx4_32_LSX(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -242,6 +242,57 @@ void Transpose4x4_32_LSX(const uint8_t* src,
|
||||
}
|
||||
}
|
||||
|
||||
void TransposeWx8_16_LSX(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
|
||||
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
|
||||
const uint8_t* src_b = (const uint8_t*)src;
|
||||
uint8_t* dst_b = (uint8_t*)dst;
|
||||
while (width > 0) {
|
||||
__m128i r0 = __lsx_vld(src_b, 0);
|
||||
__m128i r1 = __lsx_vldx(src_b, s);
|
||||
__m128i r2 = __lsx_vldx(src_b, s * 2);
|
||||
__m128i r3 = __lsx_vldx(src_b, s * 3);
|
||||
__m128i r4 = __lsx_vldx(src_b, s * 4);
|
||||
__m128i r5 = __lsx_vldx(src_b, s * 5);
|
||||
__m128i r6 = __lsx_vldx(src_b, s * 6);
|
||||
__m128i r7 = __lsx_vldx(src_b, s * 7);
|
||||
|
||||
__m128i t0 = __lsx_vilvl_h(r1, r0);
|
||||
__m128i t1 = __lsx_vilvh_h(r1, r0);
|
||||
__m128i t2 = __lsx_vilvl_h(r3, r2);
|
||||
__m128i t3 = __lsx_vilvh_h(r3, r2);
|
||||
__m128i t4 = __lsx_vilvl_h(r5, r4);
|
||||
__m128i t5 = __lsx_vilvh_h(r5, r4);
|
||||
__m128i t6 = __lsx_vilvl_h(r7, r6);
|
||||
__m128i t7 = __lsx_vilvh_h(r7, r6);
|
||||
|
||||
__m128i u0 = __lsx_vilvl_w(t2, t0);
|
||||
__m128i u1 = __lsx_vilvh_w(t2, t0);
|
||||
__m128i u2 = __lsx_vilvl_w(t3, t1);
|
||||
__m128i u3 = __lsx_vilvh_w(t3, t1);
|
||||
__m128i u4 = __lsx_vilvl_w(t6, t4);
|
||||
__m128i u5 = __lsx_vilvh_w(t6, t4);
|
||||
__m128i u6 = __lsx_vilvl_w(t7, t5);
|
||||
__m128i u7 = __lsx_vilvh_w(t7, t5);
|
||||
|
||||
__lsx_vst(__lsx_vilvl_d(u4, u0), dst_b, 0);
|
||||
__lsx_vstx(__lsx_vilvh_d(u4, u0), dst_b, d);
|
||||
__lsx_vstx(__lsx_vilvl_d(u5, u1), dst_b, d * 2);
|
||||
__lsx_vstx(__lsx_vilvh_d(u5, u1), dst_b, d * 3);
|
||||
__lsx_vstx(__lsx_vilvl_d(u6, u2), dst_b, d * 4);
|
||||
__lsx_vstx(__lsx_vilvh_d(u6, u2), dst_b, d * 5);
|
||||
__lsx_vstx(__lsx_vilvl_d(u7, u3), dst_b, d * 6);
|
||||
__lsx_vstx(__lsx_vilvh_d(u7, u3), dst_b, d * 7);
|
||||
src_b += 16;
|
||||
dst_b += d * 8;
|
||||
width -= 8;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -177,9 +177,9 @@ void TransposeUVWx8_NEON(const uint8_t* src,
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
|
||||
}
|
||||
|
||||
// Transpose 32 bit values (ARGB) in Nx4 tiles using vld4.32 with lane insertion
|
||||
// Transpose 32 bit values (ARGB) in Wx4 tiles using vld4.32 with lane insertion
|
||||
// and lane stores for 1..3 remainder rows.
|
||||
void TransposeNx4_32_NEON(const uint8_t* src,
|
||||
void TransposeWx4_32_NEON(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -264,7 +264,7 @@ void TransposeWxH_32_NEON(const uint8_t* src,
|
||||
int height) {
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (width >= 4) {
|
||||
TransposeNx4_32_NEON(src, src_stride, dst, dst_stride, height);
|
||||
TransposeWx4_32_NEON(src, src_stride, dst, dst_stride, height);
|
||||
src += 16;
|
||||
dst += 4 * d;
|
||||
width -= 4;
|
||||
@ -419,6 +419,64 @@ void TransposeWxH_32_NEON(const uint8_t* src,
|
||||
}
|
||||
}
|
||||
|
||||
void TransposeWx8_16_NEON(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
|
||||
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
|
||||
const uint8_t* temp;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"mov %[temp], %[src] \n"
|
||||
"vld1.8 {q0}, [%[temp]], %[s] \n"
|
||||
"vld1.8 {q1}, [%[temp]], %[s] \n"
|
||||
"vld1.8 {q2}, [%[temp]], %[s] \n"
|
||||
"vld1.8 {q3}, [%[temp]], %[s] \n"
|
||||
"vld1.8 {q8}, [%[temp]], %[s] \n"
|
||||
"vld1.8 {q9}, [%[temp]], %[s] \n"
|
||||
"vld1.8 {q10}, [%[temp]], %[s] \n"
|
||||
"vld1.8 {q11}, [%[temp]] \n"
|
||||
"add %[src], %[src], #16 \n"
|
||||
|
||||
// Stage 1: 16-bit transpose
|
||||
"vtrn.16 q0, q1 \n"
|
||||
"vtrn.16 q2, q3 \n"
|
||||
"vtrn.16 q8, q9 \n"
|
||||
"vtrn.16 q10, q11 \n"
|
||||
|
||||
// Stage 2: 32-bit transpose
|
||||
"vtrn.32 q0, q2 \n"
|
||||
"vtrn.32 q1, q3 \n"
|
||||
"vtrn.32 q8, q10 \n"
|
||||
"vtrn.32 q9, q11 \n"
|
||||
|
||||
// Stage 3: 64-bit swap
|
||||
"vswp d1, d16 \n"
|
||||
"vswp d3, d18 \n"
|
||||
"vswp d5, d20 \n"
|
||||
"vswp d7, d22 \n"
|
||||
|
||||
"subs %[w], %[w], #8 \n"
|
||||
"vst1.8 {q0}, [%[dst]], %[d] \n"
|
||||
"vst1.8 {q1}, [%[dst]], %[d] \n"
|
||||
"vst1.8 {q2}, [%[dst]], %[d] \n"
|
||||
"vst1.8 {q3}, [%[dst]], %[d] \n"
|
||||
"vst1.8 {q8}, [%[dst]], %[d] \n"
|
||||
"vst1.8 {q9}, [%[dst]], %[d] \n"
|
||||
"vst1.8 {q10}, [%[dst]], %[d] \n"
|
||||
"vst1.8 {q11}, [%[dst]], %[d] \n"
|
||||
"bgt 1b \n"
|
||||
: [src] "+r"(src), // %[src]
|
||||
[dst] "+r"(dst), // %[dst]
|
||||
[w] "+r"(width), // %[w]
|
||||
[temp] "=&r"(temp) // %[temp]
|
||||
: [s] "r"(s), // %[s]
|
||||
[d] "r"(d) // %[d]
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
|
||||
}
|
||||
|
||||
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -232,9 +232,9 @@ void TransposeUVWx8_NEON(const uint8_t* src,
|
||||
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v30", "v31");
|
||||
}
|
||||
|
||||
// Transpose 32 bit values (ARGB) in Nx4 tiles using ld4 with lane insertion
|
||||
// Transpose 32 bit values (ARGB) in Wx4 tiles using ld4 with lane insertion
|
||||
// and lane stores for 1..3 remainder rows.
|
||||
void TransposeNx4_32_NEON(const uint8_t* src,
|
||||
void TransposeWx4_32_NEON(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -319,7 +319,7 @@ void TransposeWxH_32_NEON(const uint8_t* src,
|
||||
int height) {
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (width >= 4) {
|
||||
TransposeNx4_32_NEON(src, src_stride, dst, dst_stride, height);
|
||||
TransposeWx4_32_NEON(src, src_stride, dst, dst_stride, height);
|
||||
src += 16;
|
||||
dst += 4 * d;
|
||||
width -= 4;
|
||||
@ -474,6 +474,77 @@ void TransposeWxH_32_NEON(const uint8_t* src,
|
||||
}
|
||||
}
|
||||
|
||||
void TransposeWx8_16_NEON(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
|
||||
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
|
||||
const uint16_t* temp;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"mov %[temp], %[src] \n"
|
||||
"ld1 {v0.8h}, [%[temp]], %[s] \n"
|
||||
"ld1 {v1.8h}, [%[temp]], %[s] \n"
|
||||
"ld1 {v2.8h}, [%[temp]], %[s] \n"
|
||||
"ld1 {v3.8h}, [%[temp]], %[s] \n"
|
||||
"ld1 {v4.8h}, [%[temp]], %[s] \n"
|
||||
"ld1 {v5.8h}, [%[temp]], %[s] \n"
|
||||
"ld1 {v6.8h}, [%[temp]], %[s] \n"
|
||||
"ld1 {v7.8h}, [%[temp]] \n"
|
||||
"add %[src], %[src], #16 \n"
|
||||
|
||||
// Stage 1: 16-bit transpose
|
||||
"trn1 v16.8h, v0.8h, v1.8h \n"
|
||||
"trn2 v17.8h, v0.8h, v1.8h \n"
|
||||
"trn1 v18.8h, v2.8h, v3.8h \n"
|
||||
"trn2 v19.8h, v2.8h, v3.8h \n"
|
||||
"trn1 v20.8h, v4.8h, v5.8h \n"
|
||||
"trn2 v21.8h, v4.8h, v5.8h \n"
|
||||
"trn1 v22.8h, v6.8h, v7.8h \n"
|
||||
"trn2 v23.8h, v6.8h, v7.8h \n"
|
||||
|
||||
// Stage 2: 32-bit transpose
|
||||
"trn1 v0.4s, v16.4s, v18.4s \n"
|
||||
"trn1 v1.4s, v17.4s, v19.4s \n"
|
||||
"trn2 v2.4s, v16.4s, v18.4s \n"
|
||||
"trn2 v3.4s, v17.4s, v19.4s \n"
|
||||
"trn1 v4.4s, v20.4s, v22.4s \n"
|
||||
"trn1 v5.4s, v21.4s, v23.4s \n"
|
||||
"trn2 v6.4s, v20.4s, v22.4s \n"
|
||||
"trn2 v7.4s, v21.4s, v23.4s \n"
|
||||
|
||||
// Stage 3: 64-bit transpose
|
||||
"trn1 v16.2d, v0.2d, v4.2d \n"
|
||||
"trn1 v17.2d, v1.2d, v5.2d \n"
|
||||
"trn1 v18.2d, v2.2d, v6.2d \n"
|
||||
"trn1 v19.2d, v3.2d, v7.2d \n"
|
||||
"trn2 v20.2d, v0.2d, v4.2d \n"
|
||||
"trn2 v21.2d, v1.2d, v5.2d \n"
|
||||
"trn2 v22.2d, v2.2d, v6.2d \n"
|
||||
"trn2 v23.2d, v3.2d, v7.2d \n"
|
||||
|
||||
"subs %w[w], %w[w], #8 \n"
|
||||
"st1 {v16.8h}, [%[dst]], %[d] \n"
|
||||
"st1 {v17.8h}, [%[dst]], %[d] \n"
|
||||
"st1 {v18.8h}, [%[dst]], %[d] \n"
|
||||
"st1 {v19.8h}, [%[dst]], %[d] \n"
|
||||
"st1 {v20.8h}, [%[dst]], %[d] \n"
|
||||
"st1 {v21.8h}, [%[dst]], %[d] \n"
|
||||
"st1 {v22.8h}, [%[dst]], %[d] \n"
|
||||
"st1 {v23.8h}, [%[dst]], %[d] \n"
|
||||
"b.gt 1b \n"
|
||||
: [src] "+r"(src), // %[src]
|
||||
[dst] "+r"(dst), // %[dst]
|
||||
[w] "+r"(width), // %[w]
|
||||
[temp] "=&r"(temp) // %[temp]
|
||||
: [s] "r"(s), // %[s]
|
||||
[d] "r"(d) // %[d]
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
|
||||
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -26,11 +26,11 @@ extern "C" {
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef HAS_TRANSPOSENX4_32_RVV
|
||||
// Transpose 32 bit values (ARGB) in Nx4 tiles, moving down N rows of source
|
||||
// (VL at a time) and across N columns of destination (4 rows).
|
||||
#ifdef HAS_TRANSPOSEWX4_32_RVV
|
||||
// Transpose 32 bit values (ARGB) in Wx4 tiles, moving down W rows of source
|
||||
// (VL at a time) and across W columns of destination (4 rows).
|
||||
// Tuned for 512-bit VL (SiFive X280).
|
||||
void TransposeNx4_32_RVV(const uint8_t* src,
|
||||
void TransposeWx4_32_RVV(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -83,7 +83,7 @@ void TransposeNx4_32_RVV(const uint8_t* src,
|
||||
: RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12",
|
||||
"v13", "v14", "v15");
|
||||
}
|
||||
#endif // HAS_TRANSPOSE4X4_32_RVV
|
||||
#endif // HAS_TRANSPOSEWX4_32_RVV
|
||||
|
||||
#ifdef HAS_TRANSPOSEWXH_32_RVV
|
||||
// Transpose 32 bit plane of any width x height using RVV without scalar
|
||||
@ -95,7 +95,7 @@ void TransposeWxH_32_RVV(const uint8_t* src,
|
||||
int width,
|
||||
int height) {
|
||||
while (width >= 4) {
|
||||
TransposeNx4_32_RVV(src, src_stride, dst, dst_stride, height);
|
||||
TransposeWx4_32_RVV(src, src_stride, dst, dst_stride, height);
|
||||
src += 16;
|
||||
dst += 4 * (ptrdiff_t)dst_stride;
|
||||
width -= 4;
|
||||
@ -143,6 +143,130 @@ void TransposeWxH_32_RVV(const uint8_t* src,
|
||||
}
|
||||
#endif // HAS_TRANSPOSEWXH_32_RVV
|
||||
|
||||
#ifdef HAS_TRANSPOSEWXH_16_RVV
|
||||
void TransposeWxH_16_RVV(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
const ptrdiff_t s_stride = (ptrdiff_t)src_stride * 2;
|
||||
const ptrdiff_t d_stride = (ptrdiff_t)dst_stride * 2;
|
||||
while (width >= 4) {
|
||||
const uint16_t* s0 = src;
|
||||
const uint16_t* s1 = src + 1;
|
||||
const uint16_t* s2 = src + 2;
|
||||
const uint16_t* s3 = src + 3;
|
||||
uint8_t* d0 = (uint8_t*)dst;
|
||||
uint8_t* d1 = d0 + d_stride;
|
||||
uint8_t* d2 = d1 + d_stride;
|
||||
uint8_t* d3 = d2 + d_stride;
|
||||
int h = height;
|
||||
int vl;
|
||||
ptrdiff_t src_step;
|
||||
ptrdiff_t dst_step;
|
||||
asm volatile(
|
||||
"vsetvli %[vl], %[h], e16, m8, ta, ma \n"
|
||||
"slli %[d_step], %[vl], 1 \n"
|
||||
"mul %[s_step], %[vl], %[s_stride] \n"
|
||||
"sub %[h], %[h], %[vl] \n"
|
||||
"bltz %[h], 2f \n"
|
||||
"1: \n"
|
||||
"vlse16.v v0, (%[s0]), %[s_stride] \n"
|
||||
"vlse16.v v8, (%[s1]), %[s_stride] \n"
|
||||
"vlse16.v v16, (%[s2]), %[s_stride] \n"
|
||||
"vlse16.v v24, (%[s3]), %[s_stride] \n"
|
||||
"add %[s0], %[s0], %[s_step] \n"
|
||||
"add %[s1], %[s1], %[s_step] \n"
|
||||
"add %[s2], %[s2], %[s_step] \n"
|
||||
"add %[s3], %[s3], %[s_step] \n"
|
||||
"vse16.v v0, (%[d0]) \n"
|
||||
"vse16.v v8, (%[d1]) \n"
|
||||
"vse16.v v16, (%[d2]) \n"
|
||||
"vse16.v v24, (%[d3]) \n"
|
||||
"add %[d0], %[d0], %[d_step] \n"
|
||||
"add %[d1], %[d1], %[d_step] \n"
|
||||
"add %[d2], %[d2], %[d_step] \n"
|
||||
"add %[d3], %[d3], %[d_step] \n"
|
||||
"sub %[h], %[h], %[vl] \n"
|
||||
"bgez %[h], 1b \n"
|
||||
"2: \n"
|
||||
"add %[h], %[h], %[vl] \n"
|
||||
"beqz %[h], 3f \n"
|
||||
"vsetvli %[vl], %[h], e16, m8, ta, ma \n"
|
||||
"vlse16.v v0, (%[s0]), %[s_stride] \n"
|
||||
"vlse16.v v8, (%[s1]), %[s_stride] \n"
|
||||
"vlse16.v v16, (%[s2]), %[s_stride] \n"
|
||||
"vlse16.v v24, (%[s3]), %[s_stride] \n"
|
||||
"vse16.v v0, (%[d0]) \n"
|
||||
"vse16.v v8, (%[d1]) \n"
|
||||
"vse16.v v16, (%[d2]) \n"
|
||||
"vse16.v v24, (%[d3]) \n"
|
||||
"3: \n"
|
||||
: [h] "+r"(h), // %[h]
|
||||
[s0] "+r"(s0), // %[s0]
|
||||
[s1] "+r"(s1), // %[s1]
|
||||
[s2] "+r"(s2), // %[s2]
|
||||
[s3] "+r"(s3), // %[s3]
|
||||
[d0] "+r"(d0), // %[d0]
|
||||
[d1] "+r"(d1), // %[d1]
|
||||
[d2] "+r"(d2), // %[d2]
|
||||
[d3] "+r"(d3), // %[d3]
|
||||
[vl] "=&r"(vl), // %[vl]
|
||||
[s_step] "=&r"(src_step), // %[s_step]
|
||||
[d_step] "=&r"(dst_step) // %[d_step]
|
||||
: [s_stride] "r"(s_stride) // %[s_stride]
|
||||
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
|
||||
"v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
|
||||
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
|
||||
"v26", "v27", "v28", "v29", "v30", "v31");
|
||||
src += 4;
|
||||
dst += 4 * (ptrdiff_t)dst_stride;
|
||||
width -= 4;
|
||||
}
|
||||
while (width > 0) {
|
||||
const uint16_t* s_col = src;
|
||||
uint16_t* d_row = dst;
|
||||
int h = height;
|
||||
int vl;
|
||||
ptrdiff_t src_step;
|
||||
ptrdiff_t dst_step;
|
||||
asm volatile(
|
||||
"vsetvli %[vl], %[h], e16, m8, ta, ma \n"
|
||||
"slli %[d_step], %[vl], 1 \n"
|
||||
"mul %[s_step], %[vl], %[s_stride] \n"
|
||||
"sub %[h], %[h], %[vl] \n"
|
||||
"bltz %[h], 2f \n"
|
||||
"1: \n"
|
||||
"vlse16.v v8, (%[s]), %[s_stride] \n"
|
||||
"add %[s], %[s], %[s_step] \n"
|
||||
"vse16.v v8, (%[d]) \n"
|
||||
"add %[d], %[d], %[d_step] \n"
|
||||
"sub %[h], %[h], %[vl] \n"
|
||||
"bgez %[h], 1b \n"
|
||||
"2: \n"
|
||||
"add %[h], %[h], %[vl] \n"
|
||||
"beqz %[h], 3f \n"
|
||||
"vsetvli %[vl], %[h], e16, m8, ta, ma \n"
|
||||
"vlse16.v v8, (%[s]), %[s_stride] \n"
|
||||
"vse16.v v8, (%[d]) \n"
|
||||
"3: \n"
|
||||
: [h] "+r"(h), // %[h]
|
||||
[s] "+r"(s_col), // %[s]
|
||||
[d] "+r"(d_row), // %[d]
|
||||
[vl] "=&r"(vl), // %[vl]
|
||||
[s_step] "=&r"(src_step), // %[s_step]
|
||||
[d_step] "=&r"(dst_step) // %[d_step]
|
||||
: [s_stride] "r"(s_stride) // %[s_stride]
|
||||
: RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12",
|
||||
"v13", "v14", "v15");
|
||||
src += 1;
|
||||
dst += (ptrdiff_t)dst_stride;
|
||||
width -= 1;
|
||||
}
|
||||
}
|
||||
#endif // HAS_TRANSPOSEWXH_16_RVV
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -237,6 +237,73 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_32_SME(
|
||||
} while (height > 0);
|
||||
}
|
||||
|
||||
__arm_locally_streaming __arm_new("za") void TransposeWxH_16_SME(
|
||||
const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
int vl;
|
||||
asm("cnth %x0" : "=r"(vl));
|
||||
|
||||
do {
|
||||
const uint16_t* src2 = src;
|
||||
uint16_t* dst2 = dst;
|
||||
|
||||
// Process up to VL 16-bit rows per iteration of the outer loop.
|
||||
int block_height = height > vl ? vl : height;
|
||||
|
||||
int width2 = width;
|
||||
do {
|
||||
const uint16_t* src3 = src2;
|
||||
|
||||
// Process up to VL 16-bit columns per iteration of the inner loop.
|
||||
int block_width = width2 > vl ? vl : width2;
|
||||
|
||||
asm volatile(
|
||||
"mov w12, #0 \n"
|
||||
|
||||
// Create a predicate to handle loading partial rows.
|
||||
"whilelt p0.h, wzr, %w[block_width] \n"
|
||||
|
||||
// Load H <= VL rows into ZA0.H.
|
||||
"1: \n"
|
||||
"ld1h {za0h.h[w12, 0]}, p0/z, [%[src3]] \n"
|
||||
"add %[src3], %[src3], %[src_stride] \n"
|
||||
"add w12, w12, #1 \n"
|
||||
"cmp w12, %w[block_height] \n"
|
||||
"b.ne 1b \n"
|
||||
|
||||
// Create a predicate to handle storing partial columns.
|
||||
"whilelt p0.h, wzr, %w[block_height] \n"
|
||||
"mov w12, #0 \n"
|
||||
|
||||
// Store W <= VL columns from ZA0.H.
|
||||
"2: \n"
|
||||
"st1h {za0v.h[w12, 0]}, p0, [%[dst2]] \n"
|
||||
"add %[dst2], %[dst2], %[dst_stride] \n"
|
||||
"add w12, w12, #1 \n"
|
||||
"cmp w12, %w[block_width] \n"
|
||||
"b.ne 2b \n"
|
||||
: [src3] "+r"(src3), // %[src3]
|
||||
[dst2] "+r"(dst2) // %[dst2]
|
||||
: [src_stride] "r"((ptrdiff_t)src_stride * 2), // %[src_stride]
|
||||
[dst_stride] "r"((ptrdiff_t)dst_stride * 2), // %[dst_stride]
|
||||
[block_width] "r"(block_width), // %[block_width]
|
||||
[block_height] "r"(block_height) // %[block_height]
|
||||
: "memory", "cc", "w12", "p0", "za");
|
||||
|
||||
src2 += vl;
|
||||
width2 -= vl;
|
||||
} while (width2 > 0);
|
||||
|
||||
src += (ptrdiff_t)vl * src_stride;
|
||||
dst += vl;
|
||||
height -= vl;
|
||||
} while (height > 0);
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
|
||||
// defined(__aarch64__)
|
||||
|
||||
|
||||
@ -18,9 +18,9 @@ extern "C" {
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
||||
|
||||
// Transpose 32 bit values (ARGB) in Nx4 tiles using SVE2 with predication
|
||||
// Transpose 32 bit values (ARGB) in Wx4 tiles using SVE2 with predication
|
||||
// for tail widths.
|
||||
void TransposeNx4_32_SVE2(const uint8_t* src,
|
||||
void TransposeWx4_32_SVE2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -109,6 +109,168 @@ void TransposeNx4_32_SVE2(const uint8_t* src,
|
||||
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
|
||||
}
|
||||
|
||||
void TransposeWx8_16_SVE2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
|
||||
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
|
||||
const uint16_t* temp;
|
||||
asm volatile(
|
||||
"ptrue p0.h, vl8 \n"
|
||||
"subs %w[w], %w[w], #8 \n"
|
||||
"b.lt 2f \n"
|
||||
|
||||
"1: \n"
|
||||
"mov %[temp], %[src] \n"
|
||||
"ld1h {z0.h}, p0/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z1.h}, p0/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z2.h}, p0/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z3.h}, p0/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z4.h}, p0/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z5.h}, p0/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z6.h}, p0/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z7.h}, p0/z, [%[temp]] \n"
|
||||
"add %[src], %[src], #16 \n"
|
||||
|
||||
"trn1 z16.h, z0.h, z1.h \n"
|
||||
"trn2 z17.h, z0.h, z1.h \n"
|
||||
"trn1 z18.h, z2.h, z3.h \n"
|
||||
"trn2 z19.h, z2.h, z3.h \n"
|
||||
"trn1 z20.h, z4.h, z5.h \n"
|
||||
"trn2 z21.h, z4.h, z5.h \n"
|
||||
"trn1 z22.h, z6.h, z7.h \n"
|
||||
"trn2 z23.h, z6.h, z7.h \n"
|
||||
|
||||
"trn1 z0.s, z16.s, z18.s \n"
|
||||
"trn1 z1.s, z17.s, z19.s \n"
|
||||
"trn2 z2.s, z16.s, z18.s \n"
|
||||
"trn2 z3.s, z17.s, z19.s \n"
|
||||
"trn1 z4.s, z20.s, z22.s \n"
|
||||
"trn1 z5.s, z21.s, z23.s \n"
|
||||
"trn2 z6.s, z20.s, z22.s \n"
|
||||
"trn2 z7.s, z21.s, z23.s \n"
|
||||
|
||||
"trn1 z16.d, z0.d, z4.d \n"
|
||||
"trn1 z17.d, z1.d, z5.d \n"
|
||||
"trn1 z18.d, z2.d, z6.d \n"
|
||||
"trn1 z19.d, z3.d, z7.d \n"
|
||||
"trn2 z20.d, z0.d, z4.d \n"
|
||||
"trn2 z21.d, z1.d, z5.d \n"
|
||||
"trn2 z22.d, z2.d, z6.d \n"
|
||||
"trn2 z23.d, z3.d, z7.d \n"
|
||||
|
||||
"st1h {z16.h}, p0, [%[dst]] \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z17.h}, p0, [%[dst]] \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z18.h}, p0, [%[dst]] \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z19.h}, p0, [%[dst]] \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z20.h}, p0, [%[dst]] \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z21.h}, p0, [%[dst]] \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z22.h}, p0, [%[dst]] \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z23.h}, p0, [%[dst]] \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"subs %w[w], %w[w], #8 \n"
|
||||
"b.ge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"adds %w[w], %w[w], #8 \n"
|
||||
"b.eq 3f \n"
|
||||
"whilelt p1.h, wzr, %w[w] \n"
|
||||
"mov %[temp], %[src] \n"
|
||||
"ld1h {z0.h}, p1/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z1.h}, p1/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z2.h}, p1/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z3.h}, p1/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z4.h}, p1/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z5.h}, p1/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z6.h}, p1/z, [%[temp]] \n"
|
||||
"add %[temp], %[temp], %[s] \n"
|
||||
"ld1h {z7.h}, p1/z, [%[temp]] \n"
|
||||
|
||||
"trn1 z16.h, z0.h, z1.h \n"
|
||||
"trn2 z17.h, z0.h, z1.h \n"
|
||||
"trn1 z18.h, z2.h, z3.h \n"
|
||||
"trn2 z19.h, z2.h, z3.h \n"
|
||||
"trn1 z20.h, z4.h, z5.h \n"
|
||||
"trn2 z21.h, z4.h, z5.h \n"
|
||||
"trn1 z22.h, z6.h, z7.h \n"
|
||||
"trn2 z23.h, z6.h, z7.h \n"
|
||||
|
||||
"trn1 z0.s, z16.s, z18.s \n"
|
||||
"trn1 z1.s, z17.s, z19.s \n"
|
||||
"trn2 z2.s, z16.s, z18.s \n"
|
||||
"trn2 z3.s, z17.s, z19.s \n"
|
||||
"trn1 z4.s, z20.s, z22.s \n"
|
||||
"trn1 z5.s, z21.s, z23.s \n"
|
||||
"trn2 z6.s, z20.s, z22.s \n"
|
||||
"trn2 z7.s, z21.s, z23.s \n"
|
||||
|
||||
"trn1 z16.d, z0.d, z4.d \n"
|
||||
"trn1 z17.d, z1.d, z5.d \n"
|
||||
"trn1 z18.d, z2.d, z6.d \n"
|
||||
"trn1 z19.d, z3.d, z7.d \n"
|
||||
"trn2 z20.d, z0.d, z4.d \n"
|
||||
"trn2 z21.d, z1.d, z5.d \n"
|
||||
"trn2 z22.d, z2.d, z6.d \n"
|
||||
"trn2 z23.d, z3.d, z7.d \n"
|
||||
|
||||
"st1h {z16.h}, p0, [%[dst]] \n"
|
||||
"subs %w[w], %w[w], #1 \n"
|
||||
"b.eq 3f \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z17.h}, p0, [%[dst]] \n"
|
||||
"subs %w[w], %w[w], #1 \n"
|
||||
"b.eq 3f \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z18.h}, p0, [%[dst]] \n"
|
||||
"subs %w[w], %w[w], #1 \n"
|
||||
"b.eq 3f \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z19.h}, p0, [%[dst]] \n"
|
||||
"subs %w[w], %w[w], #1 \n"
|
||||
"b.eq 3f \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z20.h}, p0, [%[dst]] \n"
|
||||
"subs %w[w], %w[w], #1 \n"
|
||||
"b.eq 3f \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z21.h}, p0, [%[dst]] \n"
|
||||
"subs %w[w], %w[w], #1 \n"
|
||||
"b.eq 3f \n"
|
||||
"add %[dst], %[dst], %[d] \n"
|
||||
"st1h {z22.h}, p0, [%[dst]] \n"
|
||||
"3: \n"
|
||||
: [src] "+r"(src), // %[src]
|
||||
[dst] "+r"(dst), // %[dst]
|
||||
[w] "+r"(width), // %[w]
|
||||
[temp] "=&r"(temp) // %[temp]
|
||||
: [s] "r"(s), // %[s]
|
||||
[d] "r"(d) // %[d]
|
||||
: "memory", "cc", "p0", "p1", "z0", "z1", "z2", "z3", "z4", "z5", "z6",
|
||||
"z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23");
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -191,10 +191,10 @@ void TransposeUVWx16_WASMSIMD(const uint8_t* src,
|
||||
}
|
||||
#endif // HAS_TRANSPOSEUVWX16_WASMSIMD
|
||||
|
||||
#if defined(HAS_TRANSPOSE4X4_32_WASMSIMD)
|
||||
// Transpose 32 bit values (ARGB) in 4x4 tiles, moving down 4 rows of source
|
||||
#if defined(HAS_TRANSPOSEWX4_32_WASMSIMD)
|
||||
// Transpose 32 bit values (ARGB) in Wx4 tiles, moving down 4 rows of source
|
||||
// and across 4 columns of destination. Width is a multiple of 4.
|
||||
void Transpose4x4_32_WASMSIMD(const uint8_t* src,
|
||||
void TransposeWx4_32_WASMSIMD(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -258,7 +258,58 @@ void Transpose4x4_32_WASMSIMD(const uint8_t* src,
|
||||
width -= 4;
|
||||
}
|
||||
}
|
||||
#endif // HAS_TRANSPOSE4X4_32_WASMSIMD
|
||||
#endif // HAS_TRANSPOSEWX4_32_WASMSIMD
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD)
|
||||
void TransposeWx8_16_WASMSIMD(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (width > 0) {
|
||||
v128_t r0 = wasm_v128_load(src);
|
||||
v128_t r1 = wasm_v128_load(src + s);
|
||||
v128_t r2 = wasm_v128_load(src + s * 2);
|
||||
v128_t r3 = wasm_v128_load(src + s * 3);
|
||||
v128_t r4 = wasm_v128_load(src + s * 4);
|
||||
v128_t r5 = wasm_v128_load(src + s * 5);
|
||||
v128_t r6 = wasm_v128_load(src + s * 6);
|
||||
v128_t r7 = wasm_v128_load(src + s * 7);
|
||||
|
||||
v128_t t0 = ZIPLO16(r0, r1);
|
||||
v128_t t1 = ZIPHI16(r0, r1);
|
||||
v128_t t2 = ZIPLO16(r2, r3);
|
||||
v128_t t3 = ZIPHI16(r2, r3);
|
||||
v128_t t4 = ZIPLO16(r4, r5);
|
||||
v128_t t5 = ZIPHI16(r4, r5);
|
||||
v128_t t6 = ZIPLO16(r6, r7);
|
||||
v128_t t7 = ZIPHI16(r6, r7);
|
||||
|
||||
v128_t u0 = ZIPLO32(t0, t2);
|
||||
v128_t u1 = ZIPHI32(t0, t2);
|
||||
v128_t u2 = ZIPLO32(t1, t3);
|
||||
v128_t u3 = ZIPHI32(t1, t3);
|
||||
v128_t u4 = ZIPLO32(t4, t6);
|
||||
v128_t u5 = ZIPHI32(t4, t6);
|
||||
v128_t u6 = ZIPLO32(t5, t7);
|
||||
v128_t u7 = ZIPHI32(t5, t7);
|
||||
|
||||
wasm_v128_store(dst, ZIPLO64(u0, u4));
|
||||
wasm_v128_store(dst + d, ZIPHI64(u0, u4));
|
||||
wasm_v128_store(dst + d * 2, ZIPLO64(u1, u5));
|
||||
wasm_v128_store(dst + d * 3, ZIPHI64(u1, u5));
|
||||
wasm_v128_store(dst + d * 4, ZIPLO64(u2, u6));
|
||||
wasm_v128_store(dst + d * 5, ZIPHI64(u2, u6));
|
||||
wasm_v128_store(dst + d * 6, ZIPLO64(u3, u7));
|
||||
wasm_v128_store(dst + d * 7, ZIPHI64(u3, u7));
|
||||
src += 8;
|
||||
dst += d * 8;
|
||||
width -= 8;
|
||||
}
|
||||
}
|
||||
#endif // HAS_TRANSPOSEWX8_16_WASMSIMD
|
||||
|
||||
#undef ZIPLO8
|
||||
#undef ZIPHI8
|
||||
|
||||
@ -417,10 +417,302 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src,
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
||||
|
||||
#if defined(HAS_TRANSPOSE4X4_32_SSE2)
|
||||
#if defined(HAS_TRANSPOSEWX16_16_AVX2)
|
||||
LIBYUV_TARGET_AVX2
|
||||
void TransposeWx16_16_AVX2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (width > 0) {
|
||||
__m256i r0 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 8)), 1);
|
||||
__m256i r1 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 9)), 1);
|
||||
__m256i r2 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 10)), 1);
|
||||
__m256i r3 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 11)), 1);
|
||||
__m256i r4 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 4))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 12)), 1);
|
||||
__m256i r5 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 5))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 13)), 1);
|
||||
__m256i r6 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 6))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 14)), 1);
|
||||
__m256i r7 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 7))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 15)), 1);
|
||||
|
||||
__m256i t0 = _mm256_unpacklo_epi16(r0, r1);
|
||||
__m256i t1 = _mm256_unpackhi_epi16(r0, r1);
|
||||
__m256i t2 = _mm256_unpacklo_epi16(r2, r3);
|
||||
__m256i t3 = _mm256_unpackhi_epi16(r2, r3);
|
||||
__m256i t4 = _mm256_unpacklo_epi16(r4, r5);
|
||||
__m256i t5 = _mm256_unpackhi_epi16(r4, r5);
|
||||
__m256i t6 = _mm256_unpacklo_epi16(r6, r7);
|
||||
__m256i t7 = _mm256_unpackhi_epi16(r6, r7);
|
||||
|
||||
__m256i u0 = _mm256_unpacklo_epi32(t0, t2);
|
||||
__m256i u1 = _mm256_unpackhi_epi32(t0, t2);
|
||||
__m256i u2 = _mm256_unpacklo_epi32(t1, t3);
|
||||
__m256i u3 = _mm256_unpackhi_epi32(t1, t3);
|
||||
__m256i u4 = _mm256_unpacklo_epi32(t4, t6);
|
||||
__m256i u5 = _mm256_unpackhi_epi32(t4, t6);
|
||||
__m256i u6 = _mm256_unpacklo_epi32(t5, t7);
|
||||
__m256i u7 = _mm256_unpackhi_epi32(t5, t7);
|
||||
|
||||
_mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(u0, u4));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(u0, u4));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(u1, u5));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(u1, u5));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 4), _mm256_unpacklo_epi64(u2, u6));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 5), _mm256_unpackhi_epi64(u2, u6));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 6), _mm256_unpacklo_epi64(u3, u7));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 7), _mm256_unpackhi_epi64(u3, u7));
|
||||
src += 8;
|
||||
dst += d * 8;
|
||||
width -= 8;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWX16_16_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void TransposeWx32_16_AVX512BW(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (width >= 8) {
|
||||
__m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src)));
|
||||
__m512i r1 =
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s)));
|
||||
__m512i r2 =
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2)));
|
||||
__m512i r3 =
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3)));
|
||||
__m512i r4 =
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 4)));
|
||||
__m512i r5 =
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 5)));
|
||||
__m512i r6 =
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 6)));
|
||||
__m512i r7 =
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 7)));
|
||||
|
||||
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)),
|
||||
1);
|
||||
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)),
|
||||
1);
|
||||
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)),
|
||||
1);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)),
|
||||
1);
|
||||
r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 12)),
|
||||
1);
|
||||
r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 13)),
|
||||
1);
|
||||
r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 14)),
|
||||
1);
|
||||
r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 15)),
|
||||
1);
|
||||
|
||||
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 16)),
|
||||
2);
|
||||
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 17)),
|
||||
2);
|
||||
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 18)),
|
||||
2);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 19)),
|
||||
2);
|
||||
r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 20)),
|
||||
2);
|
||||
r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 21)),
|
||||
2);
|
||||
r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 22)),
|
||||
2);
|
||||
r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 23)),
|
||||
2);
|
||||
|
||||
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 24)),
|
||||
3);
|
||||
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 25)),
|
||||
3);
|
||||
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 26)),
|
||||
3);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 27)),
|
||||
3);
|
||||
r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 28)),
|
||||
3);
|
||||
r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 29)),
|
||||
3);
|
||||
r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 30)),
|
||||
3);
|
||||
r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 31)),
|
||||
3);
|
||||
|
||||
__m512i t0 = _mm512_unpacklo_epi16(r0, r1);
|
||||
__m512i t1 = _mm512_unpackhi_epi16(r0, r1);
|
||||
__m512i t2 = _mm512_unpacklo_epi16(r2, r3);
|
||||
__m512i t3 = _mm512_unpackhi_epi16(r2, r3);
|
||||
__m512i t4 = _mm512_unpacklo_epi16(r4, r5);
|
||||
__m512i t5 = _mm512_unpackhi_epi16(r4, r5);
|
||||
__m512i t6 = _mm512_unpacklo_epi16(r6, r7);
|
||||
__m512i t7 = _mm512_unpackhi_epi16(r6, r7);
|
||||
|
||||
__m512i u0 = _mm512_unpacklo_epi32(t0, t2);
|
||||
__m512i u1 = _mm512_unpackhi_epi32(t0, t2);
|
||||
__m512i u2 = _mm512_unpacklo_epi32(t1, t3);
|
||||
__m512i u3 = _mm512_unpackhi_epi32(t1, t3);
|
||||
__m512i u4 = _mm512_unpacklo_epi32(t4, t6);
|
||||
__m512i u5 = _mm512_unpackhi_epi32(t4, t6);
|
||||
__m512i u6 = _mm512_unpacklo_epi32(t5, t7);
|
||||
__m512i u7 = _mm512_unpackhi_epi32(t5, t7);
|
||||
|
||||
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(u0, u4));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(u0, u4));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(u1, u5));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(u1, u5));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 4), _mm512_unpacklo_epi64(u2, u6));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 5), _mm512_unpackhi_epi64(u2, u6));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 6), _mm512_unpacklo_epi64(u3, u7));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 7), _mm512_unpackhi_epi64(u3, u7));
|
||||
src += 8;
|
||||
dst += d * 8;
|
||||
width -= 8;
|
||||
}
|
||||
if (width > 0) {
|
||||
const __mmask8 col_mask = (1u << width) - 1;
|
||||
__m512i r0 = _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src));
|
||||
__m512i r1 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s));
|
||||
__m512i r2 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 2));
|
||||
__m512i r3 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 3));
|
||||
__m512i r4 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 4));
|
||||
__m512i r5 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 5));
|
||||
__m512i r6 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 6));
|
||||
__m512i r7 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 7));
|
||||
|
||||
r0 =
|
||||
_mm512_inserti32x4(r0, _mm_maskz_loadu_epi16(col_mask, src + s * 8), 1);
|
||||
r1 =
|
||||
_mm512_inserti32x4(r1, _mm_maskz_loadu_epi16(col_mask, src + s * 9), 1);
|
||||
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi16(col_mask, src + s * 10),
|
||||
1);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi16(col_mask, src + s * 11),
|
||||
1);
|
||||
r4 = _mm512_inserti32x4(r4, _mm_maskz_loadu_epi16(col_mask, src + s * 12),
|
||||
1);
|
||||
r5 = _mm512_inserti32x4(r5, _mm_maskz_loadu_epi16(col_mask, src + s * 13),
|
||||
1);
|
||||
r6 = _mm512_inserti32x4(r6, _mm_maskz_loadu_epi16(col_mask, src + s * 14),
|
||||
1);
|
||||
r7 = _mm512_inserti32x4(r7, _mm_maskz_loadu_epi16(col_mask, src + s * 15),
|
||||
1);
|
||||
|
||||
r0 = _mm512_inserti32x4(r0, _mm_maskz_loadu_epi16(col_mask, src + s * 16),
|
||||
2);
|
||||
r1 = _mm512_inserti32x4(r1, _mm_maskz_loadu_epi16(col_mask, src + s * 17),
|
||||
2);
|
||||
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi16(col_mask, src + s * 18),
|
||||
2);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi16(col_mask, src + s * 19),
|
||||
2);
|
||||
r4 = _mm512_inserti32x4(r4, _mm_maskz_loadu_epi16(col_mask, src + s * 20),
|
||||
2);
|
||||
r5 = _mm512_inserti32x4(r5, _mm_maskz_loadu_epi16(col_mask, src + s * 21),
|
||||
2);
|
||||
r6 = _mm512_inserti32x4(r6, _mm_maskz_loadu_epi16(col_mask, src + s * 22),
|
||||
2);
|
||||
r7 = _mm512_inserti32x4(r7, _mm_maskz_loadu_epi16(col_mask, src + s * 23),
|
||||
2);
|
||||
|
||||
r0 = _mm512_inserti32x4(r0, _mm_maskz_loadu_epi16(col_mask, src + s * 24),
|
||||
3);
|
||||
r1 = _mm512_inserti32x4(r1, _mm_maskz_loadu_epi16(col_mask, src + s * 25),
|
||||
3);
|
||||
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi16(col_mask, src + s * 26),
|
||||
3);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi16(col_mask, src + s * 27),
|
||||
3);
|
||||
r4 = _mm512_inserti32x4(r4, _mm_maskz_loadu_epi16(col_mask, src + s * 28),
|
||||
3);
|
||||
r5 = _mm512_inserti32x4(r5, _mm_maskz_loadu_epi16(col_mask, src + s * 29),
|
||||
3);
|
||||
r6 = _mm512_inserti32x4(r6, _mm_maskz_loadu_epi16(col_mask, src + s * 30),
|
||||
3);
|
||||
r7 = _mm512_inserti32x4(r7, _mm_maskz_loadu_epi16(col_mask, src + s * 31),
|
||||
3);
|
||||
|
||||
__m512i t0 = _mm512_unpacklo_epi16(r0, r1);
|
||||
__m512i t1 = _mm512_unpackhi_epi16(r0, r1);
|
||||
__m512i t2 = _mm512_unpacklo_epi16(r2, r3);
|
||||
__m512i t3 = _mm512_unpackhi_epi16(r2, r3);
|
||||
__m512i t4 = _mm512_unpacklo_epi16(r4, r5);
|
||||
__m512i t5 = _mm512_unpackhi_epi16(r4, r5);
|
||||
__m512i t6 = _mm512_unpacklo_epi16(r6, r7);
|
||||
__m512i t7 = _mm512_unpackhi_epi16(r6, r7);
|
||||
|
||||
__m512i u0 = _mm512_unpacklo_epi32(t0, t2);
|
||||
__m512i u1 = _mm512_unpackhi_epi32(t0, t2);
|
||||
__m512i u2 = _mm512_unpacklo_epi32(t1, t3);
|
||||
__m512i u3 = _mm512_unpackhi_epi32(t1, t3);
|
||||
__m512i u4 = _mm512_unpacklo_epi32(t4, t6);
|
||||
__m512i u5 = _mm512_unpackhi_epi32(t4, t6);
|
||||
__m512i u6 = _mm512_unpacklo_epi32(t5, t7);
|
||||
__m512i u7 = _mm512_unpackhi_epi32(t5, t7);
|
||||
|
||||
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(u0, u4));
|
||||
if (width > 1) {
|
||||
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(u0, u4));
|
||||
}
|
||||
if (width > 2) {
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 2),
|
||||
_mm512_unpacklo_epi64(u1, u5));
|
||||
}
|
||||
if (width > 3) {
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 3),
|
||||
_mm512_unpackhi_epi64(u1, u5));
|
||||
}
|
||||
if (width > 4) {
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 4),
|
||||
_mm512_unpacklo_epi64(u2, u6));
|
||||
}
|
||||
if (width > 5) {
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 5),
|
||||
_mm512_unpackhi_epi64(u2, u6));
|
||||
}
|
||||
if (width > 6) {
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 6),
|
||||
_mm512_unpacklo_epi64(u3, u7));
|
||||
}
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
// Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row.
|
||||
LIBYUV_TARGET_SSE2
|
||||
void Transpose4x4_32_SSE2(const uint8_t* src,
|
||||
void TransposeWx4_32_SSE2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -445,20 +737,18 @@ void Transpose4x4_32_SSE2(const uint8_t* src,
|
||||
width -= 4;
|
||||
}
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSE4X4_32_SSE2)
|
||||
#endif // defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
|
||||
#if defined(HAS_TRANSPOSENX4_32_AVX2)
|
||||
// Transpose 32 bit values (ARGB) in Nx4 tiles with 2 blocks of 4x4 at a time
|
||||
// in ymm0..ymm3 and _mm_maskstore_epi32 tail for arbitrary N.
|
||||
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
|
||||
LIBYUV_TARGET_AVX2
|
||||
void TransposeNx4_32_AVX2(const uint8_t* src,
|
||||
void TransposeWx8_32_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (width >= 8) {
|
||||
while (width > 0) {
|
||||
__m256i r0 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
|
||||
@ -471,208 +761,34 @@ void TransposeNx4_32_AVX2(const uint8_t* src,
|
||||
__m256i r3 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
|
||||
__m256i t0 = _mm256_unpacklo_epi32(r0, r1); // a e b f
|
||||
__m256i t1 = _mm256_unpacklo_epi32(r2, r3); // i m j n
|
||||
__m256i t2 = _mm256_unpackhi_epi32(r0, r1); // c g d h
|
||||
__m256i t3 = _mm256_unpackhi_epi32(r2, r3); // k o l p
|
||||
|
||||
__m256i t0 = _mm256_unpacklo_epi32(r0, r1);
|
||||
__m256i t1 = _mm256_unpacklo_epi32(r2, r3);
|
||||
__m256i t2 = _mm256_unpackhi_epi32(r0, r1);
|
||||
__m256i t3 = _mm256_unpackhi_epi32(r2, r3);
|
||||
|
||||
_mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3));
|
||||
src += s * 8;
|
||||
dst += 32;
|
||||
width -= 8;
|
||||
}
|
||||
if (width >= 4) {
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
|
||||
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
|
||||
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
|
||||
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
|
||||
_mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3));
|
||||
src += s * 4;
|
||||
dst += 16;
|
||||
src += 16;
|
||||
dst += d * 4;
|
||||
width -= 4;
|
||||
}
|
||||
if (width > 0) {
|
||||
const __m128i mask =
|
||||
_mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (width * 8)) - 1));
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
|
||||
__m128i r1 = (width > 1) ? _mm_loadu_si128((const __m128i*)(src + s)) : r0;
|
||||
__m128i r2 =
|
||||
(width > 2) ? _mm_loadu_si128((const __m128i*)(src + s * 2)) : r1;
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r2);
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r2);
|
||||
_mm_maskstore_epi32((int*)(dst), mask, _mm_unpacklo_epi64(t0, t1));
|
||||
_mm_maskstore_epi32((int*)(dst + d), mask, _mm_unpackhi_epi64(t0, t1));
|
||||
_mm_maskstore_epi32((int*)(dst + d * 2), mask, _mm_unpacklo_epi64(t2, t3));
|
||||
_mm_maskstore_epi32((int*)(dst + d * 3), mask, _mm_unpackhi_epi64(t2, t3));
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSENX4_32_AVX2)
|
||||
#endif // defined(HAS_TRANSPOSEWX8_32_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
LIBYUV_TARGET_AVX2
|
||||
void TransposeWxH_32_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (height >= 8) {
|
||||
const uint8_t* src_row = src;
|
||||
uint8_t* dst_col = dst;
|
||||
int w = width;
|
||||
while (w >= 8) {
|
||||
__m256i r0 = _mm256_loadu_si256((const __m256i*)(src_row));
|
||||
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_row + s));
|
||||
__m256i r2 = _mm256_loadu_si256((const __m256i*)(src_row + s * 2));
|
||||
__m256i r3 = _mm256_loadu_si256((const __m256i*)(src_row + s * 3));
|
||||
__m256i r4 = _mm256_loadu_si256((const __m256i*)(src_row + s * 4));
|
||||
__m256i r5 = _mm256_loadu_si256((const __m256i*)(src_row + s * 5));
|
||||
__m256i r6 = _mm256_loadu_si256((const __m256i*)(src_row + s * 6));
|
||||
__m256i r7 = _mm256_loadu_si256((const __m256i*)(src_row + s * 7));
|
||||
|
||||
__m256i t0 = _mm256_unpacklo_epi32(r0, r1);
|
||||
__m256i t1 = _mm256_unpackhi_epi32(r0, r1);
|
||||
__m256i t2 = _mm256_unpacklo_epi32(r2, r3);
|
||||
__m256i t3 = _mm256_unpackhi_epi32(r2, r3);
|
||||
__m256i t4 = _mm256_unpacklo_epi32(r4, r5);
|
||||
__m256i t5 = _mm256_unpackhi_epi32(r4, r5);
|
||||
__m256i t6 = _mm256_unpacklo_epi32(r6, r7);
|
||||
__m256i t7 = _mm256_unpackhi_epi32(r6, r7);
|
||||
|
||||
__m256i u0 = _mm256_unpacklo_epi64(t0, t2);
|
||||
__m256i u1 = _mm256_unpackhi_epi64(t0, t2);
|
||||
__m256i u2 = _mm256_unpacklo_epi64(t1, t3);
|
||||
__m256i u3 = _mm256_unpackhi_epi64(t1, t3);
|
||||
__m256i u4 = _mm256_unpacklo_epi64(t4, t6);
|
||||
__m256i u5 = _mm256_unpackhi_epi64(t4, t6);
|
||||
__m256i u6 = _mm256_unpacklo_epi64(t5, t7);
|
||||
__m256i u7 = _mm256_unpackhi_epi64(t5, t7);
|
||||
|
||||
_mm256_storeu_si256((__m256i*)(dst_col),
|
||||
_mm256_permute2x128_si256(u0, u4, 0x20));
|
||||
_mm256_storeu_si256((__m256i*)(dst_col + d),
|
||||
_mm256_permute2x128_si256(u1, u5, 0x20));
|
||||
_mm256_storeu_si256((__m256i*)(dst_col + d * 2),
|
||||
_mm256_permute2x128_si256(u2, u6, 0x20));
|
||||
_mm256_storeu_si256((__m256i*)(dst_col + d * 3),
|
||||
_mm256_permute2x128_si256(u3, u7, 0x20));
|
||||
_mm256_storeu_si256((__m256i*)(dst_col + d * 4),
|
||||
_mm256_permute2x128_si256(u0, u4, 0x31));
|
||||
_mm256_storeu_si256((__m256i*)(dst_col + d * 5),
|
||||
_mm256_permute2x128_si256(u1, u5, 0x31));
|
||||
_mm256_storeu_si256((__m256i*)(dst_col + d * 6),
|
||||
_mm256_permute2x128_si256(u2, u6, 0x31));
|
||||
_mm256_storeu_si256((__m256i*)(dst_col + d * 7),
|
||||
_mm256_permute2x128_si256(u3, u7, 0x31));
|
||||
src_row += 32;
|
||||
dst_col += 8 * d;
|
||||
w -= 8;
|
||||
}
|
||||
if (w >= 4) {
|
||||
TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, 8);
|
||||
src_row += 16;
|
||||
dst_col += 4 * d;
|
||||
w -= 4;
|
||||
}
|
||||
if (w > 0) {
|
||||
const __m128i col_mask =
|
||||
_mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (w * 8)) - 1));
|
||||
uint8_t* dst0 = dst_col;
|
||||
uint8_t* dst1 = (w > 1) ? dst0 + d : dst0;
|
||||
uint8_t* dst2 = (w > 2) ? dst1 + d : dst1;
|
||||
for (int y = 0; y < 8; y += 4) {
|
||||
__m128i r0 =
|
||||
_mm_maskload_epi32((const int*)(src_row + y * s), col_mask);
|
||||
__m128i r1 =
|
||||
_mm_maskload_epi32((const int*)(src_row + (y + 1) * s), col_mask);
|
||||
__m128i r2 =
|
||||
_mm_maskload_epi32((const int*)(src_row + (y + 2) * s), col_mask);
|
||||
__m128i r3 =
|
||||
_mm_maskload_epi32((const int*)(src_row + (y + 3) * s), col_mask);
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
|
||||
_mm_storeu_si128((__m128i*)(dst2 + y * 4), _mm_unpacklo_epi64(t2, t3));
|
||||
_mm_storeu_si128((__m128i*)(dst1 + y * 4), _mm_unpackhi_epi64(t0, t1));
|
||||
_mm_storeu_si128((__m128i*)(dst0 + y * 4), _mm_unpacklo_epi64(t0, t1));
|
||||
}
|
||||
}
|
||||
src += 8 * s;
|
||||
dst += 32;
|
||||
height -= 8;
|
||||
}
|
||||
while (height > 0) {
|
||||
const int bh = (height >= 4) ? 4 : height;
|
||||
const uint8_t* src_row = src;
|
||||
uint8_t* dst_col = dst;
|
||||
int w = width;
|
||||
while (w >= 4) {
|
||||
TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, bh);
|
||||
src_row += 16;
|
||||
dst_col += 4 * d;
|
||||
w -= 4;
|
||||
}
|
||||
if (w > 0) {
|
||||
const __m128i col_mask =
|
||||
_mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (w * 8)) - 1));
|
||||
const __m128i row_mask =
|
||||
_mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (bh * 8)) - 1));
|
||||
uint8_t* dst0 = dst_col;
|
||||
uint8_t* dst1 = (w > 1) ? dst0 + d : dst0;
|
||||
uint8_t* dst2 = (w > 2) ? dst1 + d : dst1;
|
||||
__m128i r0 = _mm_maskload_epi32((const int*)(src_row), col_mask);
|
||||
__m128i r1 = (bh > 1)
|
||||
? _mm_maskload_epi32((const int*)(src_row + s), col_mask)
|
||||
: r0;
|
||||
__m128i r2 =
|
||||
(bh > 2) ? _mm_maskload_epi32((const int*)(src_row + s * 2), col_mask)
|
||||
: r1;
|
||||
__m128i r3 =
|
||||
(bh > 3) ? _mm_maskload_epi32((const int*)(src_row + s * 3), col_mask)
|
||||
: r2;
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
|
||||
_mm_maskstore_epi32((int*)dst2, row_mask, _mm_unpacklo_epi64(t2, t3));
|
||||
_mm_maskstore_epi32((int*)dst1, row_mask, _mm_unpackhi_epi64(t0, t1));
|
||||
_mm_maskstore_epi32((int*)dst0, row_mask, _mm_unpacklo_epi64(t0, t1));
|
||||
}
|
||||
src += bh * s;
|
||||
dst += bh * 4;
|
||||
height -= bh;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSENX4_32_AVX512BW)
|
||||
// Transpose 32 bit values (ARGB) in Nx4 tiles with 4 blocks of 4x4 at a time
|
||||
// in zmm0..zmm3 and masked tail for arbitrary N.
|
||||
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void TransposeNx4_32_AVX512BW(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
void TransposeWx16_32_AVX512BW(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (width >= 16) {
|
||||
while (width >= 4) {
|
||||
__m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src)));
|
||||
__m512i r1 =
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s)));
|
||||
@ -680,6 +796,7 @@ void TransposeNx4_32_AVX512BW(const uint8_t* src,
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2)));
|
||||
__m512i r3 =
|
||||
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3)));
|
||||
|
||||
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 4)),
|
||||
1);
|
||||
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 5)),
|
||||
@ -688,6 +805,7 @@ void TransposeNx4_32_AVX512BW(const uint8_t* src,
|
||||
1);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 7)),
|
||||
1);
|
||||
|
||||
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)),
|
||||
2);
|
||||
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)),
|
||||
@ -696,6 +814,7 @@ void TransposeNx4_32_AVX512BW(const uint8_t* src,
|
||||
2);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)),
|
||||
2);
|
||||
|
||||
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 12)),
|
||||
3);
|
||||
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 13)),
|
||||
@ -704,215 +823,74 @@ void TransposeNx4_32_AVX512BW(const uint8_t* src,
|
||||
3);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 15)),
|
||||
3);
|
||||
|
||||
__m512i t0 = _mm512_unpacklo_epi32(r0, r1);
|
||||
__m512i t1 = _mm512_unpacklo_epi32(r2, r3);
|
||||
__m512i t2 = _mm512_unpackhi_epi32(r0, r1);
|
||||
__m512i t3 = _mm512_unpackhi_epi32(r2, r3);
|
||||
|
||||
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(t0, t1));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(t0, t1));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(t2, t3));
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(t2, t3));
|
||||
src += s * 16;
|
||||
dst += 64;
|
||||
width -= 16;
|
||||
}
|
||||
while (width >= 4) {
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
|
||||
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
|
||||
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
|
||||
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
|
||||
_mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3));
|
||||
src += s * 4;
|
||||
dst += 16;
|
||||
src += 16;
|
||||
dst += d * 4;
|
||||
width -= 4;
|
||||
}
|
||||
if (width > 0) {
|
||||
const __mmask8 mask = (1u << width) - 1;
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
|
||||
__m128i r1 = (width > 1) ? _mm_loadu_si128((const __m128i*)(src + s)) : r0;
|
||||
__m128i r2 =
|
||||
(width > 2) ? _mm_loadu_si128((const __m128i*)(src + s * 2)) : r1;
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r2);
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r2);
|
||||
_mm_mask_storeu_epi32(dst, mask, _mm_unpacklo_epi64(t0, t1));
|
||||
_mm_mask_storeu_epi32(dst + d, mask, _mm_unpackhi_epi64(t0, t1));
|
||||
_mm_mask_storeu_epi32(dst + d * 2, mask, _mm_unpacklo_epi64(t2, t3));
|
||||
_mm_mask_storeu_epi32(dst + d * 3, mask, _mm_unpackhi_epi64(t2, t3));
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSENX4_32_AVX512BW)
|
||||
const __mmask8 col_mask = (1u << width) - 1;
|
||||
__m512i r0 = _mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src));
|
||||
__m512i r1 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src + s));
|
||||
__m512i r2 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src + s * 2));
|
||||
__m512i r3 =
|
||||
_mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src + s * 3));
|
||||
|
||||
#if defined(HAS_TRANSPOSEWXH_32_AVX512BW)
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void TransposeWxH_32_AVX512BW(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (height > 0) {
|
||||
const int bh = (height >= 16) ? 16 : height;
|
||||
const __mmask16 row_mask = (1u << bh) - 1;
|
||||
const uint8_t* src_row = src;
|
||||
uint8_t* dst_col = dst;
|
||||
int w = width;
|
||||
while (w > 0) {
|
||||
const int bw = (w >= 16) ? 16 : w;
|
||||
const __mmask16 col_mask = (1u << bw) - 1;
|
||||
__m512i z = _mm512_setzero_si512();
|
||||
__m512i r0 = _mm512_maskz_loadu_epi32(col_mask, src_row);
|
||||
__m512i r1 =
|
||||
(bh > 1) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s) : z;
|
||||
__m512i r2 =
|
||||
(bh > 2) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 2) : z;
|
||||
__m512i r3 =
|
||||
(bh > 3) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 3) : z;
|
||||
__m512i r4 =
|
||||
(bh > 4) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 4) : z;
|
||||
__m512i r5 =
|
||||
(bh > 5) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 5) : z;
|
||||
__m512i r6 =
|
||||
(bh > 6) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 6) : z;
|
||||
__m512i r7 =
|
||||
(bh > 7) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 7) : z;
|
||||
__m512i r8 =
|
||||
(bh > 8) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 8) : z;
|
||||
__m512i r9 =
|
||||
(bh > 9) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 9) : z;
|
||||
__m512i r10 =
|
||||
(bh > 10) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 10) : z;
|
||||
__m512i r11 =
|
||||
(bh > 11) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 11) : z;
|
||||
__m512i r12 =
|
||||
(bh > 12) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 12) : z;
|
||||
__m512i r13 =
|
||||
(bh > 13) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 13) : z;
|
||||
__m512i r14 =
|
||||
(bh > 14) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 14) : z;
|
||||
__m512i r15 =
|
||||
(bh > 15) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 15) : z;
|
||||
r0 =
|
||||
_mm512_inserti32x4(r0, _mm_maskz_loadu_epi32(col_mask, src + s * 4), 1);
|
||||
r1 =
|
||||
_mm512_inserti32x4(r1, _mm_maskz_loadu_epi32(col_mask, src + s * 5), 1);
|
||||
r2 =
|
||||
_mm512_inserti32x4(r2, _mm_maskz_loadu_epi32(col_mask, src + s * 6), 1);
|
||||
r3 =
|
||||
_mm512_inserti32x4(r3, _mm_maskz_loadu_epi32(col_mask, src + s * 7), 1);
|
||||
|
||||
__m512i t0 = _mm512_unpacklo_epi32(r0, r1);
|
||||
__m512i t1 = _mm512_unpackhi_epi32(r0, r1);
|
||||
__m512i t2 = _mm512_unpacklo_epi32(r2, r3);
|
||||
__m512i t3 = _mm512_unpackhi_epi32(r2, r3);
|
||||
__m512i t4 = _mm512_unpacklo_epi32(r4, r5);
|
||||
__m512i t5 = _mm512_unpackhi_epi32(r4, r5);
|
||||
__m512i t6 = _mm512_unpacklo_epi32(r6, r7);
|
||||
__m512i t7 = _mm512_unpackhi_epi32(r6, r7);
|
||||
__m512i t8 = _mm512_unpacklo_epi32(r8, r9);
|
||||
__m512i t9 = _mm512_unpackhi_epi32(r8, r9);
|
||||
__m512i t10 = _mm512_unpacklo_epi32(r10, r11);
|
||||
__m512i t11 = _mm512_unpackhi_epi32(r10, r11);
|
||||
__m512i t12 = _mm512_unpacklo_epi32(r12, r13);
|
||||
__m512i t13 = _mm512_unpackhi_epi32(r12, r13);
|
||||
__m512i t14 = _mm512_unpacklo_epi32(r14, r15);
|
||||
__m512i t15 = _mm512_unpackhi_epi32(r14, r15);
|
||||
r0 =
|
||||
_mm512_inserti32x4(r0, _mm_maskz_loadu_epi32(col_mask, src + s * 8), 2);
|
||||
r1 =
|
||||
_mm512_inserti32x4(r1, _mm_maskz_loadu_epi32(col_mask, src + s * 9), 2);
|
||||
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi32(col_mask, src + s * 10),
|
||||
2);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi32(col_mask, src + s * 11),
|
||||
2);
|
||||
|
||||
r0 = _mm512_unpacklo_epi64(t0, t2);
|
||||
r1 = _mm512_unpackhi_epi64(t0, t2);
|
||||
r2 = _mm512_unpacklo_epi64(t1, t3);
|
||||
r3 = _mm512_unpackhi_epi64(t1, t3);
|
||||
r4 = _mm512_unpacklo_epi64(t4, t6);
|
||||
r5 = _mm512_unpackhi_epi64(t4, t6);
|
||||
r6 = _mm512_unpacklo_epi64(t5, t7);
|
||||
r7 = _mm512_unpackhi_epi64(t5, t7);
|
||||
r8 = _mm512_unpacklo_epi64(t8, t10);
|
||||
r9 = _mm512_unpackhi_epi64(t8, t10);
|
||||
r10 = _mm512_unpacklo_epi64(t9, t11);
|
||||
r11 = _mm512_unpackhi_epi64(t9, t11);
|
||||
r12 = _mm512_unpacklo_epi64(t12, t14);
|
||||
r13 = _mm512_unpackhi_epi64(t12, t14);
|
||||
r14 = _mm512_unpacklo_epi64(t13, t15);
|
||||
r15 = _mm512_unpackhi_epi64(t13, t15);
|
||||
r0 = _mm512_inserti32x4(r0, _mm_maskz_loadu_epi32(col_mask, src + s * 12),
|
||||
3);
|
||||
r1 = _mm512_inserti32x4(r1, _mm_maskz_loadu_epi32(col_mask, src + s * 13),
|
||||
3);
|
||||
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi32(col_mask, src + s * 14),
|
||||
3);
|
||||
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi32(col_mask, src + s * 15),
|
||||
3);
|
||||
|
||||
t0 = _mm512_shuffle_i32x4(r0, r4, 0x88);
|
||||
t1 = _mm512_shuffle_i32x4(r1, r5, 0x88);
|
||||
t2 = _mm512_shuffle_i32x4(r2, r6, 0x88);
|
||||
t3 = _mm512_shuffle_i32x4(r3, r7, 0x88);
|
||||
t4 = _mm512_shuffle_i32x4(r0, r4, 0xdd);
|
||||
t5 = _mm512_shuffle_i32x4(r1, r5, 0xdd);
|
||||
t6 = _mm512_shuffle_i32x4(r2, r6, 0xdd);
|
||||
t7 = _mm512_shuffle_i32x4(r3, r7, 0xdd);
|
||||
t8 = _mm512_shuffle_i32x4(r8, r12, 0x88);
|
||||
t9 = _mm512_shuffle_i32x4(r9, r13, 0x88);
|
||||
t10 = _mm512_shuffle_i32x4(r10, r14, 0x88);
|
||||
t11 = _mm512_shuffle_i32x4(r11, r15, 0x88);
|
||||
t12 = _mm512_shuffle_i32x4(r8, r12, 0xdd);
|
||||
t13 = _mm512_shuffle_i32x4(r9, r13, 0xdd);
|
||||
t14 = _mm512_shuffle_i32x4(r10, r14, 0xdd);
|
||||
t15 = _mm512_shuffle_i32x4(r11, r15, 0xdd);
|
||||
__m512i t0 = _mm512_unpacklo_epi32(r0, r1);
|
||||
__m512i t1 = _mm512_unpacklo_epi32(r2, r3);
|
||||
__m512i t2 = _mm512_unpackhi_epi32(r0, r1);
|
||||
__m512i t3 = _mm512_unpackhi_epi32(r2, r3);
|
||||
|
||||
_mm512_mask_storeu_epi32(dst_col, row_mask,
|
||||
_mm512_shuffle_i32x4(t0, t8, 0x88));
|
||||
if (bw > 1)
|
||||
_mm512_mask_storeu_epi32(dst_col + d, row_mask,
|
||||
_mm512_shuffle_i32x4(t1, t9, 0x88));
|
||||
if (bw > 2)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 2, row_mask,
|
||||
_mm512_shuffle_i32x4(t2, t10, 0x88));
|
||||
if (bw > 3)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 3, row_mask,
|
||||
_mm512_shuffle_i32x4(t3, t11, 0x88));
|
||||
if (bw > 4)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 4, row_mask,
|
||||
_mm512_shuffle_i32x4(t4, t12, 0x88));
|
||||
if (bw > 5)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 5, row_mask,
|
||||
_mm512_shuffle_i32x4(t5, t13, 0x88));
|
||||
if (bw > 6)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 6, row_mask,
|
||||
_mm512_shuffle_i32x4(t6, t14, 0x88));
|
||||
if (bw > 7)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 7, row_mask,
|
||||
_mm512_shuffle_i32x4(t7, t15, 0x88));
|
||||
if (bw > 8)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 8, row_mask,
|
||||
_mm512_shuffle_i32x4(t0, t8, 0xdd));
|
||||
if (bw > 9)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 9, row_mask,
|
||||
_mm512_shuffle_i32x4(t1, t9, 0xdd));
|
||||
if (bw > 10)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 10, row_mask,
|
||||
_mm512_shuffle_i32x4(t2, t10, 0xdd));
|
||||
if (bw > 11)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 11, row_mask,
|
||||
_mm512_shuffle_i32x4(t3, t11, 0xdd));
|
||||
if (bw > 12)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 12, row_mask,
|
||||
_mm512_shuffle_i32x4(t4, t12, 0xdd));
|
||||
if (bw > 13)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 13, row_mask,
|
||||
_mm512_shuffle_i32x4(t5, t13, 0xdd));
|
||||
if (bw > 14)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 14, row_mask,
|
||||
_mm512_shuffle_i32x4(t6, t14, 0xdd));
|
||||
if (bw > 15)
|
||||
_mm512_mask_storeu_epi32(dst_col + d * 15, row_mask,
|
||||
_mm512_shuffle_i32x4(t7, t15, 0xdd));
|
||||
src_row += 64;
|
||||
dst_col += 16 * d;
|
||||
w -= 16;
|
||||
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(t0, t1));
|
||||
if (width > 1) {
|
||||
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(t0, t1));
|
||||
}
|
||||
if (width > 2) {
|
||||
_mm512_storeu_si512((__m512i*)(dst + d * 2),
|
||||
_mm512_unpacklo_epi64(t2, t3));
|
||||
}
|
||||
src += 16 * s;
|
||||
dst += 64;
|
||||
height -= 16;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWXH_32_AVX512BW)
|
||||
#endif // defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
|
||||
@ -2088,9 +2088,15 @@ ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 31)
|
||||
#ifdef HAS_MIRRORROW_LASX
|
||||
ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 63)
|
||||
#endif
|
||||
#ifdef HAS_MIRRORUVROW_AVX512BW
|
||||
ANY11M(MirrorUVRow_Any_AVX512BW, MirrorUVRow_AVX512BW, 2, 31)
|
||||
#endif
|
||||
#ifdef HAS_MIRRORUVROW_AVX2
|
||||
ANY11M(MirrorUVRow_Any_AVX2, MirrorUVRow_AVX2, 2, 15)
|
||||
#endif
|
||||
#ifdef HAS_MIRRORUVROW_SSSE3
|
||||
ANY11M(MirrorUVRow_Any_SSSE3, MirrorUVRow_SSSE3, 2, 7)
|
||||
#endif
|
||||
#ifdef HAS_MIRRORUVROW_NEON
|
||||
ANY11M(MirrorUVRow_Any_NEON, MirrorUVRow_NEON, 2, 31)
|
||||
#endif
|
||||
|
||||
@ -5357,6 +5357,31 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
|
||||
}
|
||||
#endif // HAS_MIRRORUVROW_AVX2
|
||||
|
||||
#ifdef HAS_MIRRORUVROW_AVX512BW
|
||||
static const uint16_t kMirrorUVPermute_AVX512[32] = {
|
||||
31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u,
|
||||
20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u,
|
||||
9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
|
||||
void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
|
||||
ptrdiff_t temp_width = (ptrdiff_t)(width);
|
||||
asm volatile("vmovdqu16 %3,%%zmm5 \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vpermw -0x40(%0,%2,2),%%zmm5,%%zmm0 \n"
|
||||
"vmovdqu16 %%zmm0,(%1) \n"
|
||||
"lea 0x40(%1),%1 \n"
|
||||
"sub $0x20,%2 \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
: "+r"(src_uv), // %0
|
||||
"+r"(dst_uv), // %1
|
||||
"+r"(temp_width) // %2
|
||||
: "m"(kMirrorUVPermute_AVX512) // %3
|
||||
: "memory", "cc", "xmm0", "xmm5");
|
||||
}
|
||||
#endif // HAS_MIRRORUVROW_AVX512BW
|
||||
|
||||
#ifdef HAS_RGB24MIRRORROW_SSSE3
|
||||
|
||||
// Shuffle first 5 pixels to last 5 mirrored. first byte zero
|
||||
|
||||
@ -2628,6 +2628,53 @@ void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
|
||||
}
|
||||
#endif // HAS_ARGBMIRRORROW_RVV
|
||||
|
||||
#ifdef HAS_MIRRORUVROW_RVV
|
||||
void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
|
||||
int vl;
|
||||
ptrdiff_t byte_vl;
|
||||
src_uv += (ptrdiff_t)width * 2;
|
||||
asm volatile(
|
||||
"vsetvli %[vl], %[w], e16, m8, ta, ma \n"
|
||||
"slli %[byte_vl], %[vl], 1 \n"
|
||||
"vid.v v16 \n"
|
||||
"addi %[vl], %[vl], -1 \n"
|
||||
"vrsub.vx v16, v16, %[vl] \n"
|
||||
"addi %[vl], %[vl], 1 \n"
|
||||
"sub %[w], %[w], %[vl] \n"
|
||||
"bltz %[w], 2f \n"
|
||||
"1: \n"
|
||||
"sub %[src], %[src], %[byte_vl] \n"
|
||||
"vle16.v v8, (%[src]) \n"
|
||||
"vrgather.vv v0, v8, v16 \n"
|
||||
"vse16.v v0, (%[dst]) \n"
|
||||
"add %[dst], %[dst], %[byte_vl] \n"
|
||||
"sub %[w], %[w], %[vl] \n"
|
||||
"bgez %[w], 1b \n"
|
||||
"2: \n"
|
||||
"add %[w], %[w], %[vl] \n"
|
||||
"beqz %[w], 3f \n"
|
||||
"vsetvli %[vl], %[w], e16, m8, ta, ma \n"
|
||||
"slli %[byte_vl], %[vl], 1 \n"
|
||||
"vid.v v16 \n"
|
||||
"addi %[vl], %[vl], -1 \n"
|
||||
"vrsub.vx v16, v16, %[vl] \n"
|
||||
"sub %[src], %[src], %[byte_vl] \n"
|
||||
"vle16.v v8, (%[src]) \n"
|
||||
"vrgather.vv v0, v8, v16 \n"
|
||||
"vse16.v v0, (%[dst]) \n"
|
||||
"3: \n"
|
||||
: [src] "+r"(src_uv), // %[src]
|
||||
[dst] "+r"(dst_uv), // %[dst]
|
||||
[w] "+r"(width), // %[w]
|
||||
[vl] "=&r"(vl), // %[vl]
|
||||
[byte_vl] "=&r"(byte_vl) // %[byte_vl]
|
||||
:
|
||||
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
|
||||
"v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
|
||||
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
|
||||
}
|
||||
#endif // HAS_MIRRORUVROW_RVV
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -915,6 +915,25 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
|
||||
}
|
||||
#endif // HAS_MIRRORUVROW_AVX2
|
||||
|
||||
#ifdef HAS_MIRRORUVROW_AVX512BW
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
|
||||
const __m512i zmm_idx = _mm512_set_epi16(
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
|
||||
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
|
||||
src_uv += width * 2;
|
||||
while (width > 0) {
|
||||
src_uv -= 64;
|
||||
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_uv);
|
||||
zmm0 = _mm512_permutexvar_epi16(zmm_idx, zmm0);
|
||||
_mm512_storeu_si512((__m512i*)dst_uv, zmm0);
|
||||
dst_uv += 64;
|
||||
width -= 32;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // HAS_MIRRORUVROW_AVX512BW
|
||||
|
||||
#ifdef HAS_SWAPUVROW_AVX2
|
||||
LIBYUV_TARGET_AVX2
|
||||
void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
|
||||
#include "../unit_test/unit_test.h"
|
||||
#include "libyuv/cpu_id.h"
|
||||
#include "libyuv/planar_functions.h"
|
||||
#include "libyuv/rotate_argb.h"
|
||||
|
||||
namespace libyuv {
|
||||
@ -263,15 +264,15 @@ static void TestRotatePlane_16(int src_width,
|
||||
int src_plane_size = src_stride * abs(src_height);
|
||||
align_buffer_page_end_16(src, src_plane_size);
|
||||
for (int i = 0; i < src_plane_size; ++i) {
|
||||
src[i] = fastrand() & 0xff;
|
||||
src[i] = fastrand() & 0xffff;
|
||||
}
|
||||
|
||||
int dst_stride = dst_width;
|
||||
int dst_plane_size = dst_stride * dst_height;
|
||||
align_buffer_page_end_16(dst_c, dst_plane_size);
|
||||
align_buffer_page_end_16(dst_opt, dst_plane_size);
|
||||
memset(dst_c, 2, dst_plane_size);
|
||||
memset(dst_opt, 3, dst_plane_size);
|
||||
memset(dst_c, 2, dst_plane_size * 2);
|
||||
memset(dst_opt, 3, dst_plane_size * 2);
|
||||
|
||||
MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
|
||||
RotatePlane_16(src, src_stride, dst_c, dst_stride, src_width, src_height,
|
||||
@ -345,4 +346,96 @@ TEST_F(LibYUVRotateTest, RotatePlane270_16_Odd) {
|
||||
benchmark_cpu_info_);
|
||||
}
|
||||
|
||||
static void TestRotatePlane16_SplitMergeUV(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
libyuv::RotationMode mode,
|
||||
int benchmark_cpu_info) {
|
||||
if (src_width < 1) {
|
||||
src_width = 1;
|
||||
}
|
||||
if (src_height < 1) {
|
||||
src_height = 1;
|
||||
}
|
||||
if (dst_width < 1) {
|
||||
dst_width = 1;
|
||||
}
|
||||
if (dst_height < 1) {
|
||||
dst_height = 1;
|
||||
}
|
||||
const int src_pixels = src_width * src_height;
|
||||
const int dst_pixels = dst_width * dst_height;
|
||||
align_buffer_page_end(src_16, src_pixels * 2);
|
||||
align_buffer_page_end(dst_16, dst_pixels * 2);
|
||||
align_buffer_page_end(ref_16, dst_pixels * 2);
|
||||
align_buffer_page_end(src_lo, src_pixels);
|
||||
align_buffer_page_end(src_hi, src_pixels);
|
||||
align_buffer_page_end(dst_lo, dst_pixels);
|
||||
align_buffer_page_end(dst_hi, dst_pixels);
|
||||
for (int i = 0; i < src_pixels * 2; ++i) {
|
||||
src_16[i] = fastrand() & 0xff;
|
||||
}
|
||||
memset(dst_16, 2, dst_pixels * 2);
|
||||
memset(ref_16, 3, dst_pixels * 2);
|
||||
|
||||
MaskCpuFlags(benchmark_cpu_info);
|
||||
SplitUVPlane(src_16, src_width * 2, src_lo, src_width, src_hi, src_width,
|
||||
src_width, src_height);
|
||||
RotatePlane(src_lo, src_width, dst_lo, dst_width, src_width, src_height,
|
||||
mode);
|
||||
RotatePlane(src_hi, src_width, dst_hi, dst_width, src_width, src_height,
|
||||
mode);
|
||||
MergeUVPlane(dst_lo, dst_width, dst_hi, dst_width, ref_16, dst_width * 2,
|
||||
dst_width, dst_height);
|
||||
|
||||
I410Rotate(reinterpret_cast<const uint16_t*>(src_16), src_width,
|
||||
reinterpret_cast<const uint16_t*>(src_16), src_width,
|
||||
reinterpret_cast<const uint16_t*>(src_16), src_width,
|
||||
reinterpret_cast<uint16_t*>(dst_16), dst_width,
|
||||
reinterpret_cast<uint16_t*>(dst_16), dst_width,
|
||||
reinterpret_cast<uint16_t*>(dst_16), dst_width, src_width,
|
||||
src_height, mode);
|
||||
|
||||
EXPECT_EQ(0, memcmp(ref_16, dst_16, dst_pixels * 2));
|
||||
|
||||
free_aligned_buffer_page_end(dst_hi);
|
||||
free_aligned_buffer_page_end(dst_lo);
|
||||
free_aligned_buffer_page_end(src_hi);
|
||||
free_aligned_buffer_page_end(src_lo);
|
||||
free_aligned_buffer_page_end(ref_16);
|
||||
free_aligned_buffer_page_end(dst_16);
|
||||
free_aligned_buffer_page_end(src_16);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVRotateTest, RotatePlane16_SplitMergeUV_Opt) {
|
||||
TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
|
||||
benchmark_width_, benchmark_height_, kRotate0,
|
||||
benchmark_cpu_info_);
|
||||
TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
|
||||
benchmark_height_, benchmark_width_, kRotate90,
|
||||
benchmark_cpu_info_);
|
||||
TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
|
||||
benchmark_width_, benchmark_height_,
|
||||
kRotate180, benchmark_cpu_info_);
|
||||
TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
|
||||
benchmark_height_, benchmark_width_,
|
||||
kRotate270, benchmark_cpu_info_);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVRotateTest, RotatePlane16_SplitMergeUV_Odd) {
|
||||
TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
|
||||
benchmark_width_ + 3, benchmark_height_ + 5,
|
||||
kRotate0, benchmark_cpu_info_);
|
||||
TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
|
||||
benchmark_height_ + 5, benchmark_width_ + 3,
|
||||
kRotate90, benchmark_cpu_info_);
|
||||
TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
|
||||
benchmark_width_ + 3, benchmark_height_ + 5,
|
||||
kRotate180, benchmark_cpu_info_);
|
||||
TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
|
||||
benchmark_height_ + 5, benchmark_width_ + 3,
|
||||
kRotate270, benchmark_cpu_info_);
|
||||
}
|
||||
|
||||
} // namespace libyuv
|
||||
|
||||
@ -864,7 +864,7 @@ TEST_F(LibYUVRotateTest, I410Rotate270_Opt) {
|
||||
|
||||
#if defined(ENABLE_ROW_TESTS)
|
||||
|
||||
TEST_F(LibYUVRotateTest, Transpose4x4_Test) {
|
||||
TEST_F(LibYUVRotateTest, TransposeWx4_32_Test) {
|
||||
// dst width and height
|
||||
const int width = 4;
|
||||
const int height = 4;
|
||||
@ -880,26 +880,26 @@ TEST_F(LibYUVRotateTest, Transpose4x4_Test) {
|
||||
memset(dst_pixels_c, 1, width * height * 4);
|
||||
memset(dst_pixels_opt, 2, width * height * 4);
|
||||
|
||||
Transpose4x4_32_C((const uint8_t*)src_pixels, height * 4,
|
||||
TransposeWx4_32_C((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_c, width * 4, width);
|
||||
|
||||
const int benchmark_iterations =
|
||||
(benchmark_iterations_ * benchmark_width_ * benchmark_height_ + 15) /
|
||||
(4 * 4);
|
||||
for (int i = 0; i < benchmark_iterations; ++i) {
|
||||
#if defined(HAS_TRANSPOSE4X4_32_NEON)
|
||||
#if defined(HAS_TRANSPOSEWX4_32_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
Transpose4x4_32_NEON((const uint8_t*)src_pixels, height * 4,
|
||||
TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
} else
|
||||
#elif defined(HAS_TRANSPOSE4X4_32_SSE2)
|
||||
#elif defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
Transpose4x4_32_SSE2((const uint8_t*)src_pixels, height * 4,
|
||||
TransposeWx4_32_SSE2((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
Transpose4x4_32_C((const uint8_t*)src_pixels, height * 4,
|
||||
TransposeWx4_32_C((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
}
|
||||
}
|
||||
@ -912,7 +912,7 @@ TEST_F(LibYUVRotateTest, Transpose4x4_Test) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LibYUVRotateTest, Transpose4x4_Opt) {
|
||||
TEST_F(LibYUVRotateTest, TransposeWx4_32_Opt) {
|
||||
// dst width and height
|
||||
const int width = ((benchmark_width_ * benchmark_height_ + 3) / 4 + 3) & ~3;
|
||||
const int height = 4;
|
||||
@ -924,26 +924,23 @@ TEST_F(LibYUVRotateTest, Transpose4x4_Opt) {
|
||||
memset(dst_pixels_c, 1, width * height * 4);
|
||||
memset(dst_pixels_opt, 2, width * height * 4);
|
||||
|
||||
Transpose4x4_32_C((const uint8_t*)src_pixels, height * 4,
|
||||
TransposeWx4_32_C((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_c, width * 4, width);
|
||||
|
||||
for (int i = 0; i < benchmark_iterations_; ++i) {
|
||||
#if defined(HAS_TRANSPOSE4X4_32_NEON)
|
||||
#if defined(HAS_TRANSPOSEWX4_32_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
Transpose4x4_32_NEON((const uint8_t*)src_pixels, height * 4,
|
||||
TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
} else
|
||||
#elif defined(HAS_TRANSPOSE4X4_32_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
Transpose4x4_32_AVX2((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
} else if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
Transpose4x4_32_SSE2((const uint8_t*)src_pixels, height * 4,
|
||||
#elif defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
TransposeWx4_32_SSE2((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
Transpose4x4_32_C((const uint8_t*)src_pixels, height * 4,
|
||||
TransposeWx4_32_C((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user