coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-29 02:17:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Optimize RotatePlane_16

Browse Source 
Bug: None
Change-Id: I91ba6cac22185b5d294fcb98344fdb8a23cc94ff
This commit is contained in:
Frank Barchard 2026-10-08 18:44:37 -07:00
parent eeb06a6451
commit 6f583da63a
20 changed files with 1635 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
include/libyuv/rotate_row.h
View File

@ -31,6 +31,8 @@ extern "C" {
defined(LIBYUV_ENABLE_ROWWIN))))
#define HAS_TRANSPOSEWX8_SSSE3
#define HAS_TRANSPOSEUVWX8_SSE2
#define HAS_TRANSPOSEWX8_16_SSE2
#define HAS_TRANSPOSEWX16_16_AVX2
#define HAS_TRANSPOSE4X4_32_SSE2
#define HAS_TRANSPOSENX4_32_AVX2
#define HAS_TRANSPOSEWXH_32_AVX2
@ -39,6 +41,7 @@ extern "C" {
#if defined(__x86_64__) || defined(_M_X64)
#define HAS_TRANSPOSEWX16_AVX512BW
#define HAS_TRANSPOSEUVWX16_AVX512BW
#define HAS_TRANSPOSEWX32_16_AVX512BW
#define HAS_TRANSPOSENX4_32_AVX512BW
#define HAS_TRANSPOSEWXH_32_AVX512BW
#endif
@ -50,6 +53,8 @@ extern "C" {
!defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_TRANSPOSEWX8_SSSE3
#define HAS_TRANSPOSEUVWX8_SSE2
#define HAS_TRANSPOSEWX8_16_SSE2
#define HAS_TRANSPOSEWX16_16_AVX2
#define HAS_TRANSPOSE4X4_32_SSE2
#define HAS_TRANSPOSENX4_32_AVX2
#define HAS_TRANSPOSEWXH_32_AVX2
@ -62,6 +67,7 @@ extern "C" {
defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_TRANSPOSEWX16_AVX512BW
#define HAS_TRANSPOSEUVWX16_AVX512BW
#define HAS_TRANSPOSEWX32_16_AVX512BW
#define HAS_TRANSPOSENX4_32_AVX512BW
#define HAS_TRANSPOSEWXH_32_AVX512BW
#endif
@ -74,11 +80,13 @@ extern "C" {
#define HAS_TRANSPOSEWX8_NEON
#endif
#define HAS_TRANSPOSEUVWX8_NEON
#define HAS_TRANSPOSEWX8_16_NEON
#define HAS_TRANSPOSENX4_32_NEON
#define HAS_TRANSPOSEWXH_32_NEON
#endif
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#define HAS_TRANSPOSEWX8_16_SVE2
#define HAS_TRANSPOSENX4_32_SVE2
#endif
@ -86,20 +94,24 @@ extern "C" {
defined(__aarch64__)
#define HAS_TRANSPOSEWXH_SME
#define HAS_TRANSPOSEUVWXH_SME
#define HAS_TRANSPOSEWXH_16_SME
#define HAS_TRANSPOSEWXH_32_SME
#endif
#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
#define HAS_TRANSPOSEWX16_LSX
#define HAS_TRANSPOSEUVWX16_LSX
#define HAS_TRANSPOSEWX8_16_LSX
#define HAS_TRANSPOSE4X4_32_LSX
#endif
#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
#define HAS_TRANSPOSEWX16_16_LASX
#define HAS_TRANSPOSE4X4_32_LASX
#endif
#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
#define HAS_TRANSPOSEWXH_16_RVV
#define HAS_TRANSPOSENX4_32_RVV
#define HAS_TRANSPOSEWXH_32_RVV
#endif
@ -109,6 +121,7 @@ extern "C" {
#define HAS_TRANSPOSEUVWX8_WASMSIMD
#define HAS_TRANSPOSEWX16_WASMSIMD
#define HAS_TRANSPOSEUVWX16_WASMSIMD
#define HAS_TRANSPOSEWX8_16_WASMSIMD
#define HAS_TRANSPOSE4X4_32_WASMSIMD
#endif
@ -352,6 +365,93 @@ void TransposeWx1_16_C(const uint16_t* src,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx8_16_SSE2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx16_16_AVX2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx32_16_AVX512BW(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx8_16_NEON(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx8_16_SVE2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWxH_16_SME(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width,
int height);
void TransposeWxH_16_RVV(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width,
int height);
void TransposeWx8_16_LSX(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx16_16_LASX(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx8_16_WASMSIMD(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx8_16_Any_SSE2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx16_16_Any_AVX2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx32_16_Any_AVX512BW(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx8_16_Any_NEON(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx8_16_Any_LSX(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx16_16_Any_LASX(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWx8_16_Any_WASMSIMD(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWxH_32_C(const uint8_t* src,
int src_stride,

11
include/libyuv/row.h
View File

@ -412,6 +412,9 @@ extern "C" {
#define HAS_INTERPOLATEROW_16_AVX2
#define HAS_MULTIPLYROW_16_AVX2
#define HAS_SWAPUVROW_AVX2
#if defined(__x86_64__) || defined(_M_X64)
#define HAS_MIRRORUVROW_AVX512BW
#endif
#endif
// The following are available for AVX512 clang x64 platforms:
@ -452,6 +455,7 @@ extern "C" {
#define HAS_J400TOARGBROW_AVX512BW
#define HAS_MERGEUVROW_AVX512BW
#define HAS_MIRRORROW_AVX512BW
#define HAS_MIRRORUVROW_AVX512BW
#define HAS_MIRRORSPLITUVROW_AVX512BW
#define HAS_SPLITUVROW_AVX512BW
#define HAS_RGBTOUVMATRIXROW_AVX512BW
@ -930,6 +934,7 @@ extern "C" {
#define HAS_MERGERGBROW_RVV
#define HAS_MERGEUVROW_RVV
#define HAS_MERGEXRGBROW_RVV
#define HAS_MIRRORUVROW_RVV
#define HAS_NV12TOARGBROW_RVV
#define HAS_NV12TORGB24ROW_RVV
#define HAS_NV21TOARGBROW_RVV
@ -2749,13 +2754,19 @@ void MirrorRow_Any_SSE2(const uint8_t* src, uint8_t* dst, int width);
void MirrorRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void MirrorRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void MirrorRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width);
void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width);
void MirrorUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_uv, int width);
void MirrorUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_uv, int width);
void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width);
void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width);
void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width);
void MirrorUVRow_C(const uint8_t* src_uv, uint8_t* dst_uv, int width);
void MirrorUVRow_Any_AVX512BW(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void MirrorUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void MirrorUVRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void MirrorUVRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void MirrorUVRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void MirrorUVRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);

14
source/planar_functions.cc
View File

@ -2785,6 +2785,7 @@ void MirrorUVPlane(const uint8_t* src_uv,
#endif
#if defined(HAS_MIRRORUVROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
MirrorUVRow = MirrorUVRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
MirrorUVRow = MirrorUVRow_SSSE3;
}
@ -2798,6 +2799,14 @@ void MirrorUVPlane(const uint8_t* src_uv,
}
}
#endif
#if defined(HAS_MIRRORUVROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
MirrorUVRow = MirrorUVRow_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
MirrorUVRow = MirrorUVRow_AVX512BW;
}
}
#endif
#if defined(HAS_MIRRORUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
MirrorUVRow = MirrorUVRow_Any_LSX;
@ -2814,6 +2823,11 @@ void MirrorUVPlane(const uint8_t* src_uv,
}
}
#endif
#if defined(HAS_MIRRORUVROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
MirrorUVRow = MirrorUVRow_RVV;
}
#endif
#if defined(HAS_MIRRORUVROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
MirrorUVRow = MirrorUVRow_Any_WASMSIMD;

145
source/rotate.cc
View File

@ -618,11 +618,124 @@ static void TransposePlane_16(const uint16_t* src,
int width,
int height) {
int i = height;
#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV)
void (*TransposeWxH_16)(const uint16_t* src, int src_stride, uint16_t* dst,
int dst_stride, int width, int height) = NULL;
#endif
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
void (*TransposeWx32_16)(const uint16_t* src, int src_stride, uint16_t* dst,
int dst_stride, int width) = NULL;
#endif
#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX)
void (*TransposeWx16_16)(const uint16_t* src, int src_stride, uint16_t* dst,
int dst_stride, int width) = NULL;
#endif
void (*TransposeWx8_16)(const uint16_t* src, int src_stride, uint16_t* dst,
int dst_stride, int width) = TransposeWx8_16_C;
#if defined(HAS_TRANSPOSEWX8_16_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
TransposeWx8_16 = TransposeWx8_16_Any_NEON;
if (IS_ALIGNED(width, 8)) {
TransposeWx8_16 = TransposeWx8_16_NEON;
}
}
#endif
#if defined(HAS_TRANSPOSEWX8_16_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
TransposeWx8_16 = TransposeWx8_16_SVE2;
}
#endif
#if defined(HAS_TRANSPOSEWXH_16_SME)
if (TestCpuFlag(kCpuHasSME)) {
TransposeWxH_16 = TransposeWxH_16_SME;
}
#endif
#if defined(HAS_TRANSPOSEWXH_16_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
TransposeWxH_16 = TransposeWxH_16_RVV;
}
#endif
#if defined(HAS_TRANSPOSEWX8_16_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
TransposeWx8_16 = TransposeWx8_16_Any_SSE2;
if (IS_ALIGNED(width, 8)) {
TransposeWx8_16 = TransposeWx8_16_SSE2;
}
}
#endif
#if defined(HAS_TRANSPOSEWX16_16_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
TransposeWx16_16 = TransposeWx16_16_Any_AVX2;
if (IS_ALIGNED(width, 8)) {
TransposeWx16_16 = TransposeWx16_16_AVX2;
}
}
#endif
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
TransposeWx32_16 = TransposeWx32_16_Any_AVX512BW;
if (IS_ALIGNED(width, 8)) {
TransposeWx32_16 = TransposeWx32_16_AVX512BW;
}
}
#endif
#if defined(HAS_TRANSPOSEWX8_16_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
TransposeWx8_16 = TransposeWx8_16_Any_LSX;
if (IS_ALIGNED(width, 8)) {
TransposeWx8_16 = TransposeWx8_16_LSX;
}
}
#endif
#if defined(HAS_TRANSPOSEWX16_16_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
TransposeWx16_16 = TransposeWx16_16_Any_LASX;
if (IS_ALIGNED(width, 8)) {
TransposeWx16_16 = TransposeWx16_16_LASX;
}
}
#endif
#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
TransposeWx8_16 = TransposeWx8_16_Any_WASMSIMD;
if (IS_ALIGNED(width, 8)) {
TransposeWx8_16 = TransposeWx8_16_WASMSIMD;
}
}
#endif
#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV)
if (TransposeWxH_16) {
TransposeWxH_16(src, src_stride, dst, dst_stride, width, height);
return;
}
#endif
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
if (TransposeWx32_16) {
while (i >= 32) {
TransposeWx32_16(src, src_stride, dst, dst_stride, width);
src += 32 * (ptrdiff_t)src_stride;
dst += 32;
i -= 32;
}
}
#endif
#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX)
if (TransposeWx16_16) {
while (i >= 16) {
TransposeWx16_16(src, src_stride, dst, dst_stride, width);
src += 16 * (ptrdiff_t)src_stride;
dst += 16;
i -= 16;
}
}
#endif
// Work across the source in 8x8 tiles
while (i >= 8) {
TransposeWx8_16_C(src, src_stride, dst, dst_stride, width);
src += 8 * src_stride; // Go down 8 rows.
dst += 8; // Move over 8 columns.
TransposeWx8_16(src, src_stride, dst, dst_stride, width);
src += 8 * (ptrdiff_t)src_stride; // Go down 8 rows.
dst += 8; // Move over 8 columns.
i -= 8;
}
@ -665,29 +778,9 @@ static void RotatePlane180_16(const uint16_t* src,
int dst_stride,
int width,
int height) {
const uint16_t* src_bot = src + (ptrdiff_t)src_stride * (height - 1);
uint16_t* dst_bot = dst + (ptrdiff_t)dst_stride * (height - 1);
int half_height = (height + 1) >> 1;
int y;
// Swap top and bottom row and mirror the content. Uses a temporary row.
align_buffer_64(row, width * 2);
uint16_t* row_tmp = (uint16_t*)row;
assert(row);
if (!row)
return;
// Odd height will harmlessly mirror the middle row twice.
for (y = 0; y < half_height; ++y) {
CopyRow_16_C(src, row_tmp, width); // Copy top row into buffer
MirrorRow_16_C(src_bot, dst, width); // Mirror bottom row into top row
MirrorRow_16_C(row_tmp, dst_bot, width); // Mirror buffer into bottom row
src += src_stride;
dst += dst_stride;
src_bot -= src_stride;
dst_bot -= dst_stride;
}
free_aligned_buffer_64(row);
// Rotate by 180 is a vertical flip (negative height) and horizontal mirror.
MirrorUVPlane((const uint8_t*)src, src_stride * 2, (uint8_t*)dst,
dst_stride * 2, width, -height);
}
LIBYUV_API

36
source/rotate_any.cc
View File

@ -122,6 +122,42 @@ TANY32(Transpose4x4_32_Any_WASMSIMD,
#endif
#undef TANY32
#define TANY16(NAMEANY, TPOS_SIMD, MASK, HEIGHT) \
void NAMEANY(const uint16_t* src, int src_stride, uint16_t* dst, \
int dst_stride, int width) { \
int r = width & MASK; \
int n = width - r; \
if (n > 0) { \
TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
} \
ptrdiff_t np = n; \
TransposeWxH_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \
r, HEIGHT); \
}
#ifdef HAS_TRANSPOSEWX8_16_NEON
TANY16(TransposeWx8_16_Any_NEON, TransposeWx8_16_NEON, 7, 8)
#endif
#ifdef HAS_TRANSPOSEWX8_16_SSE2
TANY16(TransposeWx8_16_Any_SSE2, TransposeWx8_16_SSE2, 7, 8)
#endif
#ifdef HAS_TRANSPOSEWX16_16_AVX2
TANY16(TransposeWx16_16_Any_AVX2, TransposeWx16_16_AVX2, 7, 16)
#endif
#ifdef HAS_TRANSPOSEWX32_16_AVX512BW
TANY16(TransposeWx32_16_Any_AVX512BW, TransposeWx32_16_AVX512BW, 7, 32)
#endif
#ifdef HAS_TRANSPOSEWX8_16_LSX
TANY16(TransposeWx8_16_Any_LSX, TransposeWx8_16_LSX, 7, 8)
#endif
#ifdef HAS_TRANSPOSEWX16_16_LASX
TANY16(TransposeWx16_16_Any_LASX, TransposeWx16_16_LASX, 7, 16)
#endif
#ifdef HAS_TRANSPOSEWX8_16_WASMSIMD
TANY16(TransposeWx8_16_Any_WASMSIMD, TransposeWx8_16_WASMSIMD, 7, 8)
#endif
#undef TANY16
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

324
source/rotate_gcc.cc
View File

@ -664,6 +664,330 @@ void TransposeUVWx8_SSE2(const uint8_t* src,
}
#endif // defined(HAS_TRANSPOSEUVWX8_SSE2)
#if defined(HAS_TRANSPOSEWX8_16_SSE2)
void TransposeWx8_16_SSE2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint8_t* src_b = (const uint8_t*)src;
uint8_t* dst_b = (uint8_t*)dst;
const uint8_t* temp;
asm volatile(
"1: \n"
"movdqu (%[src]),%%xmm0 \n"
"movdqu (%[src],%[s]),%%xmm1 \n"
"lea (%[src],%[s],2),%[temp] \n"
"movdqu (%[temp]),%%xmm2 \n"
"movdqu (%[temp],%[s]),%%xmm3 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"movdqu (%[temp]),%%xmm4 \n"
"movdqu (%[temp],%[s]),%%xmm5 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"movdqu (%[temp]),%%xmm6 \n"
"movdqu (%[temp],%[s]),%%xmm7 \n"
"add $0x10,%[src] \n"
// Stage 1: 16-bit unpack -> xmm8..xmm15
"movdqa %%xmm0,%%xmm8 \n"
"punpcklwd %%xmm1,%%xmm8 \n"
"movdqa %%xmm0,%%xmm9 \n"
"punpckhwd %%xmm1,%%xmm9 \n"
"movdqa %%xmm2,%%xmm10 \n"
"punpcklwd %%xmm3,%%xmm10 \n"
"movdqa %%xmm2,%%xmm11 \n"
"punpckhwd %%xmm3,%%xmm11 \n"
"movdqa %%xmm4,%%xmm12 \n"
"punpcklwd %%xmm5,%%xmm12 \n"
"movdqa %%xmm4,%%xmm13 \n"
"punpckhwd %%xmm5,%%xmm13 \n"
"movdqa %%xmm6,%%xmm14 \n"
"punpcklwd %%xmm7,%%xmm14 \n"
"movdqa %%xmm6,%%xmm15 \n"
"punpckhwd %%xmm7,%%xmm15 \n"
// Stage 2: 32-bit unpack -> xmm0..xmm7
"movdqa %%xmm8,%%xmm0 \n"
"punpckldq %%xmm10,%%xmm0 \n"
"movdqa %%xmm8,%%xmm1 \n"
"punpckhdq %%xmm10,%%xmm1 \n"
"movdqa %%xmm9,%%xmm2 \n"
"punpckldq %%xmm11,%%xmm2 \n"
"movdqa %%xmm9,%%xmm3 \n"
"punpckhdq %%xmm11,%%xmm3 \n"
"movdqa %%xmm12,%%xmm4 \n"
"punpckldq %%xmm14,%%xmm4 \n"
"movdqa %%xmm12,%%xmm5 \n"
"punpckhdq %%xmm14,%%xmm5 \n"
"movdqa %%xmm13,%%xmm6 \n"
"punpckldq %%xmm15,%%xmm6 \n"
"movdqa %%xmm13,%%xmm7 \n"
"punpckhdq %%xmm15,%%xmm7 \n"
// Stage 3: 64-bit unpack -> xmm8..xmm15
"movdqa %%xmm0,%%xmm8 \n"
"punpcklqdq %%xmm4,%%xmm8 \n"
"movdqa %%xmm0,%%xmm9 \n"
"punpckhqdq %%xmm4,%%xmm9 \n"
"movdqa %%xmm1,%%xmm10 \n"
"punpcklqdq %%xmm5,%%xmm10 \n"
"movdqa %%xmm1,%%xmm11 \n"
"punpckhqdq %%xmm5,%%xmm11 \n"
"movdqa %%xmm2,%%xmm12 \n"
"punpcklqdq %%xmm6,%%xmm12 \n"
"movdqa %%xmm2,%%xmm13 \n"
"punpckhqdq %%xmm6,%%xmm13 \n"
"movdqa %%xmm3,%%xmm14 \n"
"punpcklqdq %%xmm7,%%xmm14 \n"
"movdqa %%xmm3,%%xmm15 \n"
"punpckhqdq %%xmm7,%%xmm15 \n"
"movdqu %%xmm8,(%[dst]) \n"
"movdqu %%xmm9,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"movdqu %%xmm10,(%[dst]) \n"
"movdqu %%xmm11,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"movdqu %%xmm12,(%[dst]) \n"
"movdqu %%xmm13,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"movdqu %%xmm14,(%[dst]) \n"
"movdqu %%xmm15,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"sub $0x8,%[w] \n"
"jg 1b \n"
: [src] "+r"(src_b), // %[src]
[dst] "+r"(dst_b), // %[dst]
[w] "+r"(width), // %[w]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
"xmm15");
}
#endif // defined(HAS_TRANSPOSEWX8_16_SSE2)
#if defined(HAS_TRANSPOSEWX16_16_AVX2)
void TransposeWx16_16_AVX2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint8_t* src_b = (const uint8_t*)src;
uint8_t* dst_b = (uint8_t*)dst;
const uint8_t* temp;
asm volatile(
"1: \n"
"vmovdqu (%[src]),%%xmm0 \n"
"vmovdqu (%[src],%[s]),%%xmm1 \n"
"lea (%[src],%[s],2),%[temp] \n"
"vmovdqu (%[temp]),%%xmm2 \n"
"vmovdqu (%[temp],%[s]),%%xmm3 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu (%[temp]),%%xmm4 \n"
"vmovdqu (%[temp],%[s]),%%xmm5 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu (%[temp]),%%xmm6 \n"
"vmovdqu (%[temp],%[s]),%%xmm7 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n"
"vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n"
"vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti128 $1,(%[temp]),%%ymm4,%%ymm4 \n"
"vinserti128 $1,(%[temp],%[s]),%%ymm5,%%ymm5 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti128 $1,(%[temp]),%%ymm6,%%ymm6 \n"
"vinserti128 $1,(%[temp],%[s]),%%ymm7,%%ymm7 \n"
"add $0x10,%[src] \n"
// Stage 1: 16-bit unpack -> ymm8..ymm15
"vpunpcklwd %%ymm1,%%ymm0,%%ymm8 \n"
"vpunpckhwd %%ymm1,%%ymm0,%%ymm9 \n"
"vpunpcklwd %%ymm3,%%ymm2,%%ymm10 \n"
"vpunpckhwd %%ymm3,%%ymm2,%%ymm11 \n"
"vpunpcklwd %%ymm5,%%ymm4,%%ymm12 \n"
"vpunpckhwd %%ymm5,%%ymm4,%%ymm13 \n"
"vpunpcklwd %%ymm7,%%ymm6,%%ymm14 \n"
"vpunpckhwd %%ymm7,%%ymm6,%%ymm15 \n"
// Stage 2: 32-bit unpack -> ymm0..ymm7
"vpunpckldq %%ymm10,%%ymm8,%%ymm0 \n"
"vpunpckhdq %%ymm10,%%ymm8,%%ymm1 \n"
"vpunpckldq %%ymm11,%%ymm9,%%ymm2 \n"
"vpunpckhdq %%ymm11,%%ymm9,%%ymm3 \n"
"vpunpckldq %%ymm14,%%ymm12,%%ymm4 \n"
"vpunpckhdq %%ymm14,%%ymm12,%%ymm5 \n"
"vpunpckldq %%ymm15,%%ymm13,%%ymm6 \n"
"vpunpckhdq %%ymm15,%%ymm13,%%ymm7 \n"
// Stage 3: 64-bit unpack -> ymm8..ymm15
"vpunpcklqdq %%ymm4,%%ymm0,%%ymm8 \n"
"vpunpckhqdq %%ymm4,%%ymm0,%%ymm9 \n"
"vpunpcklqdq %%ymm5,%%ymm1,%%ymm10 \n"
"vpunpckhqdq %%ymm5,%%ymm1,%%ymm11 \n"
"vpunpcklqdq %%ymm6,%%ymm2,%%ymm12 \n"
"vpunpckhqdq %%ymm6,%%ymm2,%%ymm13 \n"
"vpunpcklqdq %%ymm7,%%ymm3,%%ymm14 \n"
"vpunpckhqdq %%ymm7,%%ymm3,%%ymm15 \n"
"vmovdqu %%ymm8,(%[dst]) \n"
"vmovdqu %%ymm9,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"vmovdqu %%ymm10,(%[dst]) \n"
"vmovdqu %%ymm11,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"vmovdqu %%ymm12,(%[dst]) \n"
"vmovdqu %%ymm13,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"vmovdqu %%ymm14,(%[dst]) \n"
"vmovdqu %%ymm15,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"sub $0x8,%[w] \n"
"jg 1b \n"
"vzeroupper \n"
: [src] "+r"(src_b), // %[src]
[dst] "+r"(dst_b), // %[dst]
[w] "+r"(width), // %[w]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
"xmm15");
}
#endif // defined(HAS_TRANSPOSEWX16_16_AVX2)
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
void TransposeWx32_16_AVX512BW(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint8_t* src_b = (const uint8_t*)src;
uint8_t* dst_b = (uint8_t*)dst;
const uint8_t* temp;
asm volatile(
"1: \n"
"vmovdqu (%[src]),%%xmm0 \n"
"vmovdqu (%[src],%[s]),%%xmm1 \n"
"lea (%[src],%[s],2),%[temp] \n"
"vmovdqu (%[temp]),%%xmm2 \n"
"vmovdqu (%[temp],%[s]),%%xmm3 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu (%[temp]),%%xmm4 \n"
"vmovdqu (%[temp],%[s]),%%xmm5 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu (%[temp]),%%xmm6 \n"
"vmovdqu (%[temp],%[s]),%%xmm7 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n"
"vinserti32x4 $1,(%[temp],%[s]),%%zmm1,%%zmm1 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n"
"vinserti32x4 $1,(%[temp],%[s]),%%zmm3,%%zmm3 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $1,(%[temp]),%%zmm4,%%zmm4 \n"
"vinserti32x4 $1,(%[temp],%[s]),%%zmm5,%%zmm5 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $1,(%[temp]),%%zmm6,%%zmm6 \n"
"vinserti32x4 $1,(%[temp],%[s]),%%zmm7,%%zmm7 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n"
"vinserti32x4 $2,(%[temp],%[s]),%%zmm1,%%zmm1 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n"
"vinserti32x4 $2,(%[temp],%[s]),%%zmm3,%%zmm3 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $2,(%[temp]),%%zmm4,%%zmm4 \n"
"vinserti32x4 $2,(%[temp],%[s]),%%zmm5,%%zmm5 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $2,(%[temp]),%%zmm6,%%zmm6 \n"
"vinserti32x4 $2,(%[temp],%[s]),%%zmm7,%%zmm7 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n"
"vinserti32x4 $3,(%[temp],%[s]),%%zmm1,%%zmm1 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n"
"vinserti32x4 $3,(%[temp],%[s]),%%zmm3,%%zmm3 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $3,(%[temp]),%%zmm4,%%zmm4 \n"
"vinserti32x4 $3,(%[temp],%[s]),%%zmm5,%%zmm5 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vinserti32x4 $3,(%[temp]),%%zmm6,%%zmm6 \n"
"vinserti32x4 $3,(%[temp],%[s]),%%zmm7,%%zmm7 \n"
"add $0x10,%[src] \n"
// Stage 1: 16-bit unpack -> zmm8..zmm15
"vpunpcklwd %%zmm1,%%zmm0,%%zmm8 \n"
"vpunpckhwd %%zmm1,%%zmm0,%%zmm9 \n"
"vpunpcklwd %%zmm3,%%zmm2,%%zmm10 \n"
"vpunpckhwd %%zmm3,%%zmm2,%%zmm11 \n"
"vpunpcklwd %%zmm5,%%zmm4,%%zmm12 \n"
"vpunpckhwd %%zmm5,%%zmm4,%%zmm13 \n"
"vpunpcklwd %%zmm7,%%zmm6,%%zmm14 \n"
"vpunpckhwd %%zmm7,%%zmm6,%%zmm15 \n"
// Stage 2: 32-bit unpack -> zmm0..zmm7
"vpunpckldq %%zmm10,%%zmm8,%%zmm0 \n"
"vpunpckhdq %%zmm10,%%zmm8,%%zmm1 \n"
"vpunpckldq %%zmm11,%%zmm9,%%zmm2 \n"
"vpunpckhdq %%zmm11,%%zmm9,%%zmm3 \n"
"vpunpckldq %%zmm14,%%zmm12,%%zmm4 \n"
"vpunpckhdq %%zmm14,%%zmm12,%%zmm5 \n"
"vpunpckldq %%zmm15,%%zmm13,%%zmm6 \n"
"vpunpckhdq %%zmm15,%%zmm13,%%zmm7 \n"
// Stage 3: 64-bit unpack -> zmm8..zmm15
"vpunpcklqdq %%zmm4,%%zmm0,%%zmm8 \n"
"vpunpckhqdq %%zmm4,%%zmm0,%%zmm9 \n"
"vpunpcklqdq %%zmm5,%%zmm1,%%zmm10 \n"
"vpunpckhqdq %%zmm5,%%zmm1,%%zmm11 \n"
"vpunpcklqdq %%zmm6,%%zmm2,%%zmm12 \n"
"vpunpckhqdq %%zmm6,%%zmm2,%%zmm13 \n"
"vpunpcklqdq %%zmm7,%%zmm3,%%zmm14 \n"
"vpunpckhqdq %%zmm7,%%zmm3,%%zmm15 \n"
"vmovdqu16 %%zmm8,(%[dst]) \n"
"vmovdqu16 %%zmm9,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"vmovdqu16 %%zmm10,(%[dst]) \n"
"vmovdqu16 %%zmm11,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"vmovdqu16 %%zmm12,(%[dst]) \n"
"vmovdqu16 %%zmm13,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"vmovdqu16 %%zmm14,(%[dst]) \n"
"vmovdqu16 %%zmm15,(%[dst],%[d]) \n"
"lea (%[dst],%[d],2),%[dst] \n"
"sub $0x8,%[w] \n"
"jg 1b \n"
"vzeroupper \n"
: [src] "+r"(src_b), // %[src]
[dst] "+r"(dst_b), // %[dst]
[w] "+r"(width), // %[w]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
"xmm15");
}
#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW)
#if defined(HAS_TRANSPOSE4X4_32_SSE2)
// 4 values, little endian view
// a b c d

115
source/rotate_lasx.cc
View File

@ -95,6 +95,121 @@ void Transpose4x4_32_LASX(const uint8_t* src,
"$xr7");
}
void TransposeWx16_16_LASX(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint8_t* src_b = (const uint8_t*)src;
uint8_t* dst_b = (uint8_t*)dst;
const uint8_t* t;
asm volatile(
"srli.w %[w], %[w], 3 \n"
"beqz %[w], 2f \n"
"1: \n"
"move %[t], %[src] \n"
"vld $vr0, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr1, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr2, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr3, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr4, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr5, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr6, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr7, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr8, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr9, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr10, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr11, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr12, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr13, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr14, %[t], 0 \n"
"add.d %[t], %[t], %[s] \n"
"vld $vr15, %[t], 0 \n"
"addi.d %[src], %[src], 16 \n"
"xvpermi.q $xr0, $xr8, 0x02 \n"
"xvpermi.q $xr1, $xr9, 0x02 \n"
"xvpermi.q $xr2, $xr10, 0x02 \n"
"xvpermi.q $xr3, $xr11, 0x02 \n"
"xvpermi.q $xr4, $xr12, 0x02 \n"
"xvpermi.q $xr5, $xr13, 0x02 \n"
"xvpermi.q $xr6, $xr14, 0x02 \n"
"xvpermi.q $xr7, $xr15, 0x02 \n"
"xvilvl.h $xr8, $xr1, $xr0 \n"
"xvilvh.h $xr9, $xr1, $xr0 \n"
"xvilvl.h $xr10, $xr3, $xr2 \n"
"xvilvh.h $xr11, $xr3, $xr2 \n"
"xvilvl.h $xr12, $xr5, $xr4 \n"
"xvilvh.h $xr13, $xr5, $xr4 \n"
"xvilvl.h $xr14, $xr7, $xr6 \n"
"xvilvh.h $xr15, $xr7, $xr6 \n"
"xvilvl.w $xr0, $xr10, $xr8 \n"
"xvilvh.w $xr1, $xr10, $xr8 \n"
"xvilvl.w $xr2, $xr11, $xr9 \n"
"xvilvh.w $xr3, $xr11, $xr9 \n"
"xvilvl.w $xr4, $xr14, $xr12 \n"
"xvilvh.w $xr5, $xr14, $xr12 \n"
"xvilvl.w $xr6, $xr15, $xr13 \n"
"xvilvh.w $xr7, $xr15, $xr13 \n"
"xvilvl.d $xr8, $xr4, $xr0 \n"
"xvilvh.d $xr9, $xr4, $xr0 \n"
"xvilvl.d $xr10, $xr5, $xr1 \n"
"xvilvh.d $xr11, $xr5, $xr1 \n"
"xvilvl.d $xr12, $xr6, $xr2 \n"
"xvilvh.d $xr13, $xr6, $xr2 \n"
"xvilvl.d $xr14, $xr7, $xr3 \n"
"xvilvh.d $xr15, $xr7, $xr3 \n"
"xvst $xr8, %[dst], 0 \n"
"add.d %[dst], %[dst], %[d] \n"
"xvst $xr9, %[dst], 0 \n"
"add.d %[dst], %[dst], %[d] \n"
"xvst $xr10, %[dst], 0 \n"
"add.d %[dst], %[dst], %[d] \n"
"xvst $xr11, %[dst], 0 \n"
"add.d %[dst], %[dst], %[d] \n"
"xvst $xr12, %[dst], 0 \n"
"add.d %[dst], %[dst], %[d] \n"
"xvst $xr13, %[dst], 0 \n"
"add.d %[dst], %[dst], %[d] \n"
"xvst $xr14, %[dst], 0 \n"
"add.d %[dst], %[dst], %[d] \n"
"xvst $xr15, %[dst], 0 \n"
"add.d %[dst], %[dst], %[d] \n"
"addi.w %[w], %[w], -1 \n"
"bnez %[w], 1b \n"
"2: \n"
: [src] "+r"(src_b), // %[src]
[dst] "+r"(dst_b), // %[dst]
[w] "+r"(width), // %[w]
[t] "=&r"(t) // %[t]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
"$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
"$xr15");
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

51
source/rotate_lsx.cc
View File

@ -242,6 +242,57 @@ void Transpose4x4_32_LSX(const uint8_t* src,
}
}
void TransposeWx8_16_LSX(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint8_t* src_b = (const uint8_t*)src;
uint8_t* dst_b = (uint8_t*)dst;
while (width > 0) {
__m128i r0 = __lsx_vld(src_b, 0);
__m128i r1 = __lsx_vldx(src_b, s);
__m128i r2 = __lsx_vldx(src_b, s * 2);
__m128i r3 = __lsx_vldx(src_b, s * 3);
__m128i r4 = __lsx_vldx(src_b, s * 4);
__m128i r5 = __lsx_vldx(src_b, s * 5);
__m128i r6 = __lsx_vldx(src_b, s * 6);
__m128i r7 = __lsx_vldx(src_b, s * 7);
__m128i t0 = __lsx_vilvl_h(r1, r0);
__m128i t1 = __lsx_vilvh_h(r1, r0);
__m128i t2 = __lsx_vilvl_h(r3, r2);
__m128i t3 = __lsx_vilvh_h(r3, r2);
__m128i t4 = __lsx_vilvl_h(r5, r4);
__m128i t5 = __lsx_vilvh_h(r5, r4);
__m128i t6 = __lsx_vilvl_h(r7, r6);
__m128i t7 = __lsx_vilvh_h(r7, r6);
__m128i u0 = __lsx_vilvl_w(t2, t0);
__m128i u1 = __lsx_vilvh_w(t2, t0);
__m128i u2 = __lsx_vilvl_w(t3, t1);
__m128i u3 = __lsx_vilvh_w(t3, t1);
__m128i u4 = __lsx_vilvl_w(t6, t4);
__m128i u5 = __lsx_vilvh_w(t6, t4);
__m128i u6 = __lsx_vilvl_w(t7, t5);
__m128i u7 = __lsx_vilvh_w(t7, t5);
__lsx_vst(__lsx_vilvl_d(u4, u0), dst_b, 0);
__lsx_vstx(__lsx_vilvh_d(u4, u0), dst_b, d);
__lsx_vstx(__lsx_vilvl_d(u5, u1), dst_b, d * 2);
__lsx_vstx(__lsx_vilvh_d(u5, u1), dst_b, d * 3);
__lsx_vstx(__lsx_vilvl_d(u6, u2), dst_b, d * 4);
__lsx_vstx(__lsx_vilvh_d(u6, u2), dst_b, d * 5);
__lsx_vstx(__lsx_vilvl_d(u7, u3), dst_b, d * 6);
__lsx_vstx(__lsx_vilvh_d(u7, u3), dst_b, d * 7);
src_b += 16;
dst_b += d * 8;
width -= 8;
}
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

58
source/rotate_neon.cc
View File

@ -419,6 +419,64 @@ void TransposeWxH_32_NEON(const uint8_t* src,
}
}
void TransposeWx8_16_NEON(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint8_t* temp;
asm volatile(
"1: \n"
"mov %[temp], %[src] \n"
"vld1.8 {q0}, [%[temp]], %[s] \n"
"vld1.8 {q1}, [%[temp]], %[s] \n"
"vld1.8 {q2}, [%[temp]], %[s] \n"
"vld1.8 {q3}, [%[temp]], %[s] \n"
"vld1.8 {q8}, [%[temp]], %[s] \n"
"vld1.8 {q9}, [%[temp]], %[s] \n"
"vld1.8 {q10}, [%[temp]], %[s] \n"
"vld1.8 {q11}, [%[temp]] \n"
"add %[src], %[src], #16 \n"
// Stage 1: 16-bit transpose
"vtrn.16 q0, q1 \n"
"vtrn.16 q2, q3 \n"
"vtrn.16 q8, q9 \n"
"vtrn.16 q10, q11 \n"
// Stage 2: 32-bit transpose
"vtrn.32 q0, q2 \n"
"vtrn.32 q1, q3 \n"
"vtrn.32 q8, q10 \n"
"vtrn.32 q9, q11 \n"
// Stage 3: 64-bit swap
"vswp d1, d16 \n"
"vswp d3, d18 \n"
"vswp d5, d20 \n"
"vswp d7, d22 \n"
"subs %[w], %[w], #8 \n"
"vst1.8 {q0}, [%[dst]], %[d] \n"
"vst1.8 {q1}, [%[dst]], %[d] \n"
"vst1.8 {q2}, [%[dst]], %[d] \n"
"vst1.8 {q3}, [%[dst]], %[d] \n"
"vst1.8 {q8}, [%[dst]], %[d] \n"
"vst1.8 {q9}, [%[dst]], %[d] \n"
"vst1.8 {q10}, [%[dst]], %[d] \n"
"vst1.8 {q11}, [%[dst]], %[d] \n"
"bgt 1b \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[w] "+r"(width), // %[w]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
}
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus

71
source/rotate_neon64.cc
View File

@ -474,6 +474,77 @@ void TransposeWxH_32_NEON(const uint8_t* src,
}
}
void TransposeWx8_16_NEON(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint16_t* temp;
asm volatile(
"1: \n"
"mov %[temp], %[src] \n"
"ld1 {v0.8h}, [%[temp]], %[s] \n"
"ld1 {v1.8h}, [%[temp]], %[s] \n"
"ld1 {v2.8h}, [%[temp]], %[s] \n"
"ld1 {v3.8h}, [%[temp]], %[s] \n"
"ld1 {v4.8h}, [%[temp]], %[s] \n"
"ld1 {v5.8h}, [%[temp]], %[s] \n"
"ld1 {v6.8h}, [%[temp]], %[s] \n"
"ld1 {v7.8h}, [%[temp]] \n"
"add %[src], %[src], #16 \n"
// Stage 1: 16-bit transpose
"trn1 v16.8h, v0.8h, v1.8h \n"
"trn2 v17.8h, v0.8h, v1.8h \n"
"trn1 v18.8h, v2.8h, v3.8h \n"
"trn2 v19.8h, v2.8h, v3.8h \n"
"trn1 v20.8h, v4.8h, v5.8h \n"
"trn2 v21.8h, v4.8h, v5.8h \n"
"trn1 v22.8h, v6.8h, v7.8h \n"
"trn2 v23.8h, v6.8h, v7.8h \n"
// Stage 2: 32-bit transpose
"trn1 v0.4s, v16.4s, v18.4s \n"
"trn1 v1.4s, v17.4s, v19.4s \n"
"trn2 v2.4s, v16.4s, v18.4s \n"
"trn2 v3.4s, v17.4s, v19.4s \n"
"trn1 v4.4s, v20.4s, v22.4s \n"
"trn1 v5.4s, v21.4s, v23.4s \n"
"trn2 v6.4s, v20.4s, v22.4s \n"
"trn2 v7.4s, v21.4s, v23.4s \n"
// Stage 3: 64-bit transpose
"trn1 v16.2d, v0.2d, v4.2d \n"
"trn1 v17.2d, v1.2d, v5.2d \n"
"trn1 v18.2d, v2.2d, v6.2d \n"
"trn1 v19.2d, v3.2d, v7.2d \n"
"trn2 v20.2d, v0.2d, v4.2d \n"
"trn2 v21.2d, v1.2d, v5.2d \n"
"trn2 v22.2d, v2.2d, v6.2d \n"
"trn2 v23.2d, v3.2d, v7.2d \n"
"subs %w[w], %w[w], #8 \n"
"st1 {v16.8h}, [%[dst]], %[d] \n"
"st1 {v17.8h}, [%[dst]], %[d] \n"
"st1 {v18.8h}, [%[dst]], %[d] \n"
"st1 {v19.8h}, [%[dst]], %[d] \n"
"st1 {v20.8h}, [%[dst]], %[d] \n"
"st1 {v21.8h}, [%[dst]], %[d] \n"
"st1 {v22.8h}, [%[dst]], %[d] \n"
"st1 {v23.8h}, [%[dst]], %[d] \n"
"b.gt 1b \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[w] "+r"(width), // %[w]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus

124
source/rotate_rvv.cc
View File

@ -143,6 +143,130 @@ void TransposeWxH_32_RVV(const uint8_t* src,
}
#endif // HAS_TRANSPOSEWXH_32_RVV
#ifdef HAS_TRANSPOSEWXH_16_RVV
void TransposeWxH_16_RVV(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width,
int height) {
const ptrdiff_t s_stride = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d_stride = (ptrdiff_t)dst_stride * 2;
while (width >= 4) {
const uint16_t* s0 = src;
const uint16_t* s1 = src + 1;
const uint16_t* s2 = src + 2;
const uint16_t* s3 = src + 3;
uint8_t* d0 = (uint8_t*)dst;
uint8_t* d1 = d0 + d_stride;
uint8_t* d2 = d1 + d_stride;
uint8_t* d3 = d2 + d_stride;
int h = height;
int vl;
ptrdiff_t src_step;
ptrdiff_t dst_step;
asm volatile(
"vsetvli %[vl], %[h], e16, m8, ta, ma \n"
"slli %[d_step], %[vl], 1 \n"
"mul %[s_step], %[vl], %[s_stride] \n"
"sub %[h], %[h], %[vl] \n"
"bltz %[h], 2f \n"
"1: \n"
"vlse16.v v0, (%[s0]), %[s_stride] \n"
"vlse16.v v8, (%[s1]), %[s_stride] \n"
"vlse16.v v16, (%[s2]), %[s_stride] \n"
"vlse16.v v24, (%[s3]), %[s_stride] \n"
"add %[s0], %[s0], %[s_step] \n"
"add %[s1], %[s1], %[s_step] \n"
"add %[s2], %[s2], %[s_step] \n"
"add %[s3], %[s3], %[s_step] \n"
"vse16.v v0, (%[d0]) \n"
"vse16.v v8, (%[d1]) \n"
"vse16.v v16, (%[d2]) \n"
"vse16.v v24, (%[d3]) \n"
"add %[d0], %[d0], %[d_step] \n"
"add %[d1], %[d1], %[d_step] \n"
"add %[d2], %[d2], %[d_step] \n"
"add %[d3], %[d3], %[d_step] \n"
"sub %[h], %[h], %[vl] \n"
"bgez %[h], 1b \n"
"2: \n"
"add %[h], %[h], %[vl] \n"
"beqz %[h], 3f \n"
"vsetvli %[vl], %[h], e16, m8, ta, ma \n"
"vlse16.v v0, (%[s0]), %[s_stride] \n"
"vlse16.v v8, (%[s1]), %[s_stride] \n"
"vlse16.v v16, (%[s2]), %[s_stride] \n"
"vlse16.v v24, (%[s3]), %[s_stride] \n"
"vse16.v v0, (%[d0]) \n"
"vse16.v v8, (%[d1]) \n"
"vse16.v v16, (%[d2]) \n"
"vse16.v v24, (%[d3]) \n"
"3: \n"
: [h] "+r"(h), // %[h]
[s0] "+r"(s0), // %[s0]
[s1] "+r"(s1), // %[s1]
[s2] "+r"(s2), // %[s2]
[s3] "+r"(s3), // %[s3]
[d0] "+r"(d0), // %[d0]
[d1] "+r"(d1), // %[d1]
[d2] "+r"(d2), // %[d2]
[d3] "+r"(d3), // %[d3]
[vl] "=&r"(vl), // %[vl]
[s_step] "=&r"(src_step), // %[s_step]
[d_step] "=&r"(dst_step) // %[d_step]
: [s_stride] "r"(s_stride) // %[s_stride]
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
"v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
"v26", "v27", "v28", "v29", "v30", "v31");
src += 4;
dst += 4 * (ptrdiff_t)dst_stride;
width -= 4;
}
while (width > 0) {
const uint16_t* s_col = src;
uint16_t* d_row = dst;
int h = height;
int vl;
ptrdiff_t src_step;
ptrdiff_t dst_step;
asm volatile(
"vsetvli %[vl], %[h], e16, m8, ta, ma \n"
"slli %[d_step], %[vl], 1 \n"
"mul %[s_step], %[vl], %[s_stride] \n"
"sub %[h], %[h], %[vl] \n"
"bltz %[h], 2f \n"
"1: \n"
"vlse16.v v8, (%[s]), %[s_stride] \n"
"add %[s], %[s], %[s_step] \n"
"vse16.v v8, (%[d]) \n"
"add %[d], %[d], %[d_step] \n"
"sub %[h], %[h], %[vl] \n"
"bgez %[h], 1b \n"
"2: \n"
"add %[h], %[h], %[vl] \n"
"beqz %[h], 3f \n"
"vsetvli %[vl], %[h], e16, m8, ta, ma \n"
"vlse16.v v8, (%[s]), %[s_stride] \n"
"vse16.v v8, (%[d]) \n"
"3: \n"
: [h] "+r"(h), // %[h]
[s] "+r"(s_col), // %[s]
[d] "+r"(d_row), // %[d]
[vl] "=&r"(vl), // %[vl]
[s_step] "=&r"(src_step), // %[s_step]
[d_step] "=&r"(dst_step) // %[d_step]
: [s_stride] "r"(s_stride) // %[s_stride]
: RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12",
"v13", "v14", "v15");
src += 1;
dst += (ptrdiff_t)dst_stride;
width -= 1;
}
}
#endif // HAS_TRANSPOSEWXH_16_RVV
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

67
source/rotate_sme.cc
View File

@ -237,6 +237,73 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_32_SME(
} while (height > 0);
}
__arm_locally_streaming __arm_new("za") void TransposeWxH_16_SME(
const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width,
int height) {
int vl;
asm("cnth %x0" : "=r"(vl));
do {
const uint16_t* src2 = src;
uint16_t* dst2 = dst;
// Process up to VL 16-bit rows per iteration of the outer loop.
int block_height = height > vl ? vl : height;
int width2 = width;
do {
const uint16_t* src3 = src2;
// Process up to VL 16-bit columns per iteration of the inner loop.
int block_width = width2 > vl ? vl : width2;
asm volatile(
"mov w12, #0 \n"
// Create a predicate to handle loading partial rows.
"whilelt p0.h, wzr, %w[block_width] \n"
// Load H <= VL rows into ZA0.H.
"1: \n"
"ld1h {za0h.h[w12, 0]}, p0/z, [%[src3]] \n"
"add %[src3], %[src3], %[src_stride] \n"
"add w12, w12, #1 \n"
"cmp w12, %w[block_height] \n"
"b.ne 1b \n"
// Create a predicate to handle storing partial columns.
"whilelt p0.h, wzr, %w[block_height] \n"
"mov w12, #0 \n"
// Store W <= VL columns from ZA0.H.
"2: \n"
"st1h {za0v.h[w12, 0]}, p0, [%[dst2]] \n"
"add %[dst2], %[dst2], %[dst_stride] \n"
"add w12, w12, #1 \n"
"cmp w12, %w[block_width] \n"
"b.ne 2b \n"
: [src3] "+r"(src3), // %[src3]
[dst2] "+r"(dst2) // %[dst2]
: [src_stride] "r"((ptrdiff_t)src_stride * 2), // %[src_stride]
[dst_stride] "r"((ptrdiff_t)dst_stride * 2), // %[dst_stride]
[block_width] "r"(block_width), // %[block_width]
[block_height] "r"(block_height) // %[block_height]
: "memory", "cc", "w12", "p0", "za");
src2 += vl;
width2 -= vl;
} while (width2 > 0);
src += (ptrdiff_t)vl * src_stride;
dst += vl;
height -= vl;
} while (height > 0);
}
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
// defined(__aarch64__)

162
source/rotate_sve.cc
View File

@ -109,6 +109,168 @@ void TransposeNx4_32_SVE2(const uint8_t* src,
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
}
void TransposeWx8_16_SVE2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = (ptrdiff_t)src_stride * 2;
const ptrdiff_t d = (ptrdiff_t)dst_stride * 2;
const uint16_t* temp;
asm volatile(
"ptrue p0.h, vl8 \n"
"subs %w[w], %w[w], #8 \n"
"b.lt 2f \n"
"1: \n"
"mov %[temp], %[src] \n"
"ld1h {z0.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z1.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z2.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z3.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z4.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z5.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z6.h}, p0/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z7.h}, p0/z, [%[temp]] \n"
"add %[src], %[src], #16 \n"
"trn1 z16.h, z0.h, z1.h \n"
"trn2 z17.h, z0.h, z1.h \n"
"trn1 z18.h, z2.h, z3.h \n"
"trn2 z19.h, z2.h, z3.h \n"
"trn1 z20.h, z4.h, z5.h \n"
"trn2 z21.h, z4.h, z5.h \n"
"trn1 z22.h, z6.h, z7.h \n"
"trn2 z23.h, z6.h, z7.h \n"
"trn1 z0.s, z16.s, z18.s \n"
"trn1 z1.s, z17.s, z19.s \n"
"trn2 z2.s, z16.s, z18.s \n"
"trn2 z3.s, z17.s, z19.s \n"
"trn1 z4.s, z20.s, z22.s \n"
"trn1 z5.s, z21.s, z23.s \n"
"trn2 z6.s, z20.s, z22.s \n"
"trn2 z7.s, z21.s, z23.s \n"
"trn1 z16.d, z0.d, z4.d \n"
"trn1 z17.d, z1.d, z5.d \n"
"trn1 z18.d, z2.d, z6.d \n"
"trn1 z19.d, z3.d, z7.d \n"
"trn2 z20.d, z0.d, z4.d \n"
"trn2 z21.d, z1.d, z5.d \n"
"trn2 z22.d, z2.d, z6.d \n"
"trn2 z23.d, z3.d, z7.d \n"
"st1h {z16.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z17.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z18.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z19.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z20.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z21.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z22.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z23.h}, p0, [%[dst]] \n"
"add %[dst], %[dst], %[d] \n"
"subs %w[w], %w[w], #8 \n"
"b.ge 1b \n"
"2: \n"
"adds %w[w], %w[w], #8 \n"
"b.eq 3f \n"
"whilelt p1.h, wzr, %w[w] \n"
"mov %[temp], %[src] \n"
"ld1h {z0.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z1.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z2.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z3.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z4.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z5.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z6.h}, p1/z, [%[temp]] \n"
"add %[temp], %[temp], %[s] \n"
"ld1h {z7.h}, p1/z, [%[temp]] \n"
"trn1 z16.h, z0.h, z1.h \n"
"trn2 z17.h, z0.h, z1.h \n"
"trn1 z18.h, z2.h, z3.h \n"
"trn2 z19.h, z2.h, z3.h \n"
"trn1 z20.h, z4.h, z5.h \n"
"trn2 z21.h, z4.h, z5.h \n"
"trn1 z22.h, z6.h, z7.h \n"
"trn2 z23.h, z6.h, z7.h \n"
"trn1 z0.s, z16.s, z18.s \n"
"trn1 z1.s, z17.s, z19.s \n"
"trn2 z2.s, z16.s, z18.s \n"
"trn2 z3.s, z17.s, z19.s \n"
"trn1 z4.s, z20.s, z22.s \n"
"trn1 z5.s, z21.s, z23.s \n"
"trn2 z6.s, z20.s, z22.s \n"
"trn2 z7.s, z21.s, z23.s \n"
"trn1 z16.d, z0.d, z4.d \n"
"trn1 z17.d, z1.d, z5.d \n"
"trn1 z18.d, z2.d, z6.d \n"
"trn1 z19.d, z3.d, z7.d \n"
"trn2 z20.d, z0.d, z4.d \n"
"trn2 z21.d, z1.d, z5.d \n"
"trn2 z22.d, z2.d, z6.d \n"
"trn2 z23.d, z3.d, z7.d \n"
"st1h {z16.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z17.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z18.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z19.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z20.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z21.h}, p0, [%[dst]] \n"
"subs %w[w], %w[w], #1 \n"
"b.eq 3f \n"
"add %[dst], %[dst], %[d] \n"
"st1h {z22.h}, p0, [%[dst]] \n"
"3: \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[w] "+r"(width), // %[w]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "p0", "p1", "z0", "z1", "z2", "z3", "z4", "z5", "z6",
"z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23");
}
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#ifdef __cplusplus

51
source/rotate_wasm.cc
View File

@ -260,6 +260,57 @@ void Transpose4x4_32_WASMSIMD(const uint8_t* src,
}
#endif // HAS_TRANSPOSE4X4_32_WASMSIMD
#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD)
void TransposeWx8_16_WASMSIMD(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (width > 0) {
v128_t r0 = wasm_v128_load(src);
v128_t r1 = wasm_v128_load(src + s);
v128_t r2 = wasm_v128_load(src + s * 2);
v128_t r3 = wasm_v128_load(src + s * 3);
v128_t r4 = wasm_v128_load(src + s * 4);
v128_t r5 = wasm_v128_load(src + s * 5);
v128_t r6 = wasm_v128_load(src + s * 6);
v128_t r7 = wasm_v128_load(src + s * 7);
v128_t t0 = ZIPLO16(r0, r1);
v128_t t1 = ZIPHI16(r0, r1);
v128_t t2 = ZIPLO16(r2, r3);
v128_t t3 = ZIPHI16(r2, r3);
v128_t t4 = ZIPLO16(r4, r5);
v128_t t5 = ZIPHI16(r4, r5);
v128_t t6 = ZIPLO16(r6, r7);
v128_t t7 = ZIPHI16(r6, r7);
v128_t u0 = ZIPLO32(t0, t2);
v128_t u1 = ZIPHI32(t0, t2);
v128_t u2 = ZIPLO32(t1, t3);
v128_t u3 = ZIPHI32(t1, t3);
v128_t u4 = ZIPLO32(t4, t6);
v128_t u5 = ZIPHI32(t4, t6);
v128_t u6 = ZIPLO32(t5, t7);
v128_t u7 = ZIPHI32(t5, t7);
wasm_v128_store(dst, ZIPLO64(u0, u4));
wasm_v128_store(dst + d, ZIPHI64(u0, u4));
wasm_v128_store(dst + d * 2, ZIPLO64(u1, u5));
wasm_v128_store(dst + d * 3, ZIPHI64(u1, u5));
wasm_v128_store(dst + d * 4, ZIPLO64(u2, u6));
wasm_v128_store(dst + d * 5, ZIPHI64(u2, u6));
wasm_v128_store(dst + d * 6, ZIPLO64(u3, u7));
wasm_v128_store(dst + d * 7, ZIPHI64(u3, u7));
src += 8;
dst += d * 8;
width -= 8;
}
}
#endif // HAS_TRANSPOSEWX8_16_WASMSIMD
#undef ZIPLO8
#undef ZIPHI8
#undef ZIPLO16

232
source/rotate_win.cc
View File

@ -417,6 +417,238 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src,
}
#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
#if defined(HAS_TRANSPOSEWX8_16_SSE2)
LIBYUV_TARGET_SSE2
void TransposeWx8_16_SSE2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (width > 0) {
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
__m128i r4 = _mm_loadu_si128((const __m128i*)(src + s * 4));
__m128i r5 = _mm_loadu_si128((const __m128i*)(src + s * 5));
__m128i r6 = _mm_loadu_si128((const __m128i*)(src + s * 6));
__m128i r7 = _mm_loadu_si128((const __m128i*)(src + s * 7));
__m128i t0 = _mm_unpacklo_epi16(r0, r1);
__m128i t1 = _mm_unpackhi_epi16(r0, r1);
__m128i t2 = _mm_unpacklo_epi16(r2, r3);
__m128i t3 = _mm_unpackhi_epi16(r2, r3);
__m128i t4 = _mm_unpacklo_epi16(r4, r5);
__m128i t5 = _mm_unpackhi_epi16(r4, r5);
__m128i t6 = _mm_unpacklo_epi16(r6, r7);
__m128i t7 = _mm_unpackhi_epi16(r6, r7);
__m128i u0 = _mm_unpacklo_epi32(t0, t2);
__m128i u1 = _mm_unpackhi_epi32(t0, t2);
__m128i u2 = _mm_unpacklo_epi32(t1, t3);
__m128i u3 = _mm_unpackhi_epi32(t1, t3);
__m128i u4 = _mm_unpacklo_epi32(t4, t6);
__m128i u5 = _mm_unpackhi_epi32(t4, t6);
__m128i u6 = _mm_unpacklo_epi32(t5, t7);
__m128i u7 = _mm_unpackhi_epi32(t5, t7);
_mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(u0, u4));
_mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(u0, u4));
_mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(u1, u5));
_mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(u1, u5));
_mm_storeu_si128((__m128i*)(dst + d * 4), _mm_unpacklo_epi64(u2, u6));
_mm_storeu_si128((__m128i*)(dst + d * 5), _mm_unpackhi_epi64(u2, u6));
_mm_storeu_si128((__m128i*)(dst + d * 6), _mm_unpacklo_epi64(u3, u7));
_mm_storeu_si128((__m128i*)(dst + d * 7), _mm_unpackhi_epi64(u3, u7));
src += 8;
dst += d * 8;
width -= 8;
}
}
#endif // defined(HAS_TRANSPOSEWX8_16_SSE2)
#if defined(HAS_TRANSPOSEWX16_16_AVX2)
LIBYUV_TARGET_AVX2
void TransposeWx16_16_AVX2(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (width > 0) {
__m256i r0 = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
_mm_loadu_si128((const __m128i*)(src + s * 8)), 1);
__m256i r1 = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
_mm_loadu_si128((const __m128i*)(src + s * 9)), 1);
__m256i r2 = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
_mm_loadu_si128((const __m128i*)(src + s * 10)), 1);
__m256i r3 = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
_mm_loadu_si128((const __m128i*)(src + s * 11)), 1);
__m256i r4 = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 4))),
_mm_loadu_si128((const __m128i*)(src + s * 12)), 1);
__m256i r5 = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 5))),
_mm_loadu_si128((const __m128i*)(src + s * 13)), 1);
__m256i r6 = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 6))),
_mm_loadu_si128((const __m128i*)(src + s * 14)), 1);
__m256i r7 = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 7))),
_mm_loadu_si128((const __m128i*)(src + s * 15)), 1);
__m256i t0 = _mm256_unpacklo_epi16(r0, r1);
__m256i t1 = _mm256_unpackhi_epi16(r0, r1);
__m256i t2 = _mm256_unpacklo_epi16(r2, r3);
__m256i t3 = _mm256_unpackhi_epi16(r2, r3);
__m256i t4 = _mm256_unpacklo_epi16(r4, r5);
__m256i t5 = _mm256_unpackhi_epi16(r4, r5);
__m256i t6 = _mm256_unpacklo_epi16(r6, r7);
__m256i t7 = _mm256_unpackhi_epi16(r6, r7);
__m256i u0 = _mm256_unpacklo_epi32(t0, t2);
__m256i u1 = _mm256_unpackhi_epi32(t0, t2);
__m256i u2 = _mm256_unpacklo_epi32(t1, t3);
__m256i u3 = _mm256_unpackhi_epi32(t1, t3);
__m256i u4 = _mm256_unpacklo_epi32(t4, t6);
__m256i u5 = _mm256_unpackhi_epi32(t4, t6);
__m256i u6 = _mm256_unpacklo_epi32(t5, t7);
__m256i u7 = _mm256_unpackhi_epi32(t5, t7);
_mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(u0, u4));
_mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(u0, u4));
_mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(u1, u5));
_mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(u1, u5));
_mm256_storeu_si256((__m256i*)(dst + d * 4), _mm256_unpacklo_epi64(u2, u6));
_mm256_storeu_si256((__m256i*)(dst + d * 5), _mm256_unpackhi_epi64(u2, u6));
_mm256_storeu_si256((__m256i*)(dst + d * 6), _mm256_unpacklo_epi64(u3, u7));
_mm256_storeu_si256((__m256i*)(dst + d * 7), _mm256_unpackhi_epi64(u3, u7));
src += 8;
dst += d * 8;
width -= 8;
}
_mm256_zeroupper();
}
#endif // defined(HAS_TRANSPOSEWX16_16_AVX2)
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
LIBYUV_TARGET_AVX512BW
void TransposeWx32_16_AVX512BW(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (width > 0) {
__m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src)));
__m512i r1 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s)));
__m512i r2 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2)));
__m512i r3 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3)));
__m512i r4 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 4)));
__m512i r5 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 5)));
__m512i r6 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 6)));
__m512i r7 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 7)));
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)),
1);
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)),
1);
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)),
1);
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)),
1);
r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 12)),
1);
r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 13)),
1);
r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 14)),
1);
r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 15)),
1);
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 16)),
2);
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 17)),
2);
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 18)),
2);
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 19)),
2);
r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 20)),
2);
r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 21)),
2);
r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 22)),
2);
r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 23)),
2);
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 24)),
3);
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 25)),
3);
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 26)),
3);
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 27)),
3);
r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 28)),
3);
r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 29)),
3);
r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 30)),
3);
r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 31)),
3);
__m512i t0 = _mm512_unpacklo_epi16(r0, r1);
__m512i t1 = _mm512_unpackhi_epi16(r0, r1);
__m512i t2 = _mm512_unpacklo_epi16(r2, r3);
__m512i t3 = _mm512_unpackhi_epi16(r2, r3);
__m512i t4 = _mm512_unpacklo_epi16(r4, r5);
__m512i t5 = _mm512_unpackhi_epi16(r4, r5);
__m512i t6 = _mm512_unpacklo_epi16(r6, r7);
__m512i t7 = _mm512_unpackhi_epi16(r6, r7);
__m512i u0 = _mm512_unpacklo_epi32(t0, t2);
__m512i u1 = _mm512_unpackhi_epi32(t0, t2);
__m512i u2 = _mm512_unpacklo_epi32(t1, t3);
__m512i u3 = _mm512_unpackhi_epi32(t1, t3);
__m512i u4 = _mm512_unpacklo_epi32(t4, t6);
__m512i u5 = _mm512_unpackhi_epi32(t4, t6);
__m512i u6 = _mm512_unpacklo_epi32(t5, t7);
__m512i u7 = _mm512_unpackhi_epi32(t5, t7);
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(u0, u4));
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(u0, u4));
_mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(u1, u5));
_mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(u1, u5));
_mm512_storeu_si512((__m512i*)(dst + d * 4), _mm512_unpacklo_epi64(u2, u6));
_mm512_storeu_si512((__m512i*)(dst + d * 5), _mm512_unpackhi_epi64(u2, u6));
_mm512_storeu_si512((__m512i*)(dst + d * 6), _mm512_unpacklo_epi64(u3, u7));
_mm512_storeu_si512((__m512i*)(dst + d * 7), _mm512_unpackhi_epi64(u3, u7));
src += 8;
dst += d * 8;
width -= 8;
}
_mm256_zeroupper();
}
#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW)
#if defined(HAS_TRANSPOSE4X4_32_SSE2)
// Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row.
LIBYUV_TARGET_SSE2

6
source/row_any.cc
View File

@ -2088,9 +2088,15 @@ ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 31)
#ifdef HAS_MIRRORROW_LASX
ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 63)
#endif
#ifdef HAS_MIRRORUVROW_AVX512BW
ANY11M(MirrorUVRow_Any_AVX512BW, MirrorUVRow_AVX512BW, 2, 31)
#endif
#ifdef HAS_MIRRORUVROW_AVX2
ANY11M(MirrorUVRow_Any_AVX2, MirrorUVRow_AVX2, 2, 15)
#endif
#ifdef HAS_MIRRORUVROW_SSSE3
ANY11M(MirrorUVRow_Any_SSSE3, MirrorUVRow_SSSE3, 2, 7)
#endif
#ifdef HAS_MIRRORUVROW_NEON
ANY11M(MirrorUVRow_Any_NEON, MirrorUVRow_NEON, 2, 31)
#endif

25
source/row_gcc.cc
View File

@ -5357,6 +5357,31 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
}
#endif // HAS_MIRRORUVROW_AVX2
#ifdef HAS_MIRRORUVROW_AVX512BW
static const uint16_t kMirrorUVPermute_AVX512[32] = {
31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u,
20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u,
9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
ptrdiff_t temp_width = (ptrdiff_t)(width);
asm volatile("vmovdqu16 %3,%%zmm5 \n"
LABELALIGN
"1: \n"
"vpermw -0x40(%0,%2,2),%%zmm5,%%zmm0 \n"
"vmovdqu16 %%zmm0,(%1) \n"
"lea 0x40(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_uv), // %0
"+r"(dst_uv), // %1
"+r"(temp_width) // %2
: "m"(kMirrorUVPermute_AVX512) // %3
: "memory", "cc", "xmm0", "xmm5");
}
#endif // HAS_MIRRORUVROW_AVX512BW
#ifdef HAS_RGB24MIRRORROW_SSSE3
// Shuffle first 5 pixels to last 5 mirrored. first byte zero

47
source/row_rvv.cc
View File

@ -2628,6 +2628,53 @@ void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
}
#endif // HAS_ARGBMIRRORROW_RVV
#ifdef HAS_MIRRORUVROW_RVV
void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
int vl;
ptrdiff_t byte_vl;
src_uv += (ptrdiff_t)width * 2;
asm volatile(
"vsetvli %[vl], %[w], e16, m8, ta, ma \n"
"slli %[byte_vl], %[vl], 1 \n"
"vid.v v16 \n"
"addi %[vl], %[vl], -1 \n"
"vrsub.vx v16, v16, %[vl] \n"
"addi %[vl], %[vl], 1 \n"
"sub %[w], %[w], %[vl] \n"
"bltz %[w], 2f \n"
"1: \n"
"sub %[src], %[src], %[byte_vl] \n"
"vle16.v v8, (%[src]) \n"
"vrgather.vv v0, v8, v16 \n"
"vse16.v v0, (%[dst]) \n"
"add %[dst], %[dst], %[byte_vl] \n"
"sub %[w], %[w], %[vl] \n"
"bgez %[w], 1b \n"
"2: \n"
"add %[w], %[w], %[vl] \n"
"beqz %[w], 3f \n"
"vsetvli %[vl], %[w], e16, m8, ta, ma \n"
"slli %[byte_vl], %[vl], 1 \n"
"vid.v v16 \n"
"addi %[vl], %[vl], -1 \n"
"vrsub.vx v16, v16, %[vl] \n"
"sub %[src], %[src], %[byte_vl] \n"
"vle16.v v8, (%[src]) \n"
"vrgather.vv v0, v8, v16 \n"
"vse16.v v0, (%[dst]) \n"
"3: \n"
: [src] "+r"(src_uv), // %[src]
[dst] "+r"(dst_uv), // %[dst]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl), // %[vl]
[byte_vl] "=&r"(byte_vl) // %[byte_vl]
:
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
"v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
#endif // HAS_MIRRORUVROW_RVV
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

19
source/row_win.cc
View File

@ -915,6 +915,25 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
}
#endif // HAS_MIRRORUVROW_AVX2
#ifdef HAS_MIRRORUVROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
const __m512i zmm_idx = _mm512_set_epi16(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
src_uv += width * 2;
while (width > 0) {
src_uv -= 64;
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_uv);
zmm0 = _mm512_permutexvar_epi16(zmm_idx, zmm0);
_mm512_storeu_si512((__m512i*)dst_uv, zmm0);
dst_uv += 64;
width -= 32;
}
_mm256_zeroupper();
}
#endif // HAS_MIRRORUVROW_AVX512BW
#ifdef HAS_SWAPUVROW_AVX2
LIBYUV_TARGET_AVX2
void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) {

6
unit_test/rotate_argb_test.cc
View File

@ -263,15 +263,15 @@ static void TestRotatePlane_16(int src_width,
int src_plane_size = src_stride * abs(src_height);
align_buffer_page_end_16(src, src_plane_size);
for (int i = 0; i < src_plane_size; ++i) {
src[i] = fastrand() & 0xff;
src[i] = fastrand() & 0xffff;
}
int dst_stride = dst_width;
int dst_plane_size = dst_stride * dst_height;
align_buffer_page_end_16(dst_c, dst_plane_size);
align_buffer_page_end_16(dst_opt, dst_plane_size);
memset(dst_c, 2, dst_plane_size);
memset(dst_opt, 3, dst_plane_size);
memset(dst_c, 2, dst_plane_size * 2);
memset(dst_opt, 3, dst_plane_size * 2);
MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
RotatePlane_16(src, src_stride, dst_c, dst_stride, src_width, src_height,