coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Rotate for AVX512

Browse Source 
- TransposeWx16_AVX512BW and TransposeUVWx16_AVX512BW
- tail masking instead of 'Any' to odd source width

I420Rotate90_Opt (LIBYUV_WIDTH=1279 LIBYUV_HEIGHT=719 LIBYUV_REPEAT=1000):
CPU                     Before     After     Speedup
Intel Emerald Rapids    249 ms     182 ms      1.37x
Intel Ice Lake          311 ms     234 ms      1.33x
Intel Granite Rapids    246 ms     186 ms      1.32x
Intel Sapphire Rapids   274 ms     226 ms      1.21x

NV12Rotate90_Opt (LIBYUV_WIDTH=1279 LIBYUV_HEIGHT=719 LIBYUV_REPEAT=1000):
CPU                     Before     After     Speedup
Intel Granite Rapids    263 ms     181 ms      1.45x
Intel Emerald Rapids    257 ms     180 ms      1.43x
Intel Ice Lake          311 ms     233 ms      1.33x
Intel Sapphire Rapids   282 ms     222 ms      1.27x

Bug: None
Change-Id: Ibe0df79275099dc06bf4218484a24732cd7fb9b5
TAG=agy
CONV=bea53b37-b2bf-46fa-9e44-a708b438fabc
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8497726
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard 2026-10-01 02:38:01 -07:00
parent eee4597984
commit 98fa761478
4 changed files with 404 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
include/libyuv/rotate_row.h
View File

@ -41,10 +41,16 @@ extern "C" {
// The following are available for 64 bit GCC: // The following are available for 64 bit GCC:
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ #if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
!defined(LIBYUV_ENABLE_ROWWIN) !defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_TRANSPOSEWX8_FAST_SSSE3
#define HAS_TRANSPOSEUVWX8_SSE2 #define HAS_TRANSPOSEUVWX8_SSE2
#endif #endif
// The following are available for AVX512 clang x64 platforms:
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_TRANSPOSEWX16_AVX512BW
#define HAS_TRANSPOSEUVWX16_AVX512BW
#endif
#if !defined(LIBYUV_DISABLE_NEON) && \ #if !defined(LIBYUV_DISABLE_NEON) && \
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
#if defined(__aarch64__) #if defined(__aarch64__)
@ -105,11 +111,11 @@ void TransposeWx8_SSSE3(const uint8_t* src,
uint8_t* dst, uint8_t* dst,
int dst_stride, int dst_stride,
int width); int width);
void TransposeWx8_Fast_SSSE3(const uint8_t* src, void TransposeWx16_AVX512BW(const uint8_t* src,
int src_stride, int src_stride,
uint8_t* dst, uint8_t* dst,
int dst_stride, int dst_stride,
int width); int width);
void TransposeWx16_LSX(const uint8_t* src, void TransposeWx16_LSX(const uint8_t* src,
int src_stride, int src_stride,
uint8_t* dst, uint8_t* dst,
@ -131,11 +137,6 @@ void TransposeWx8_Any_SSSE3(const uint8_t* src,
uint8_t* dst, uint8_t* dst,
int dst_stride, int dst_stride,
int width); int width);
void TransposeWx8_Fast_Any_SSSE3(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
void TransposeWx16_Any_LSX(const uint8_t* src, void TransposeWx16_Any_LSX(const uint8_t* src,
int src_stride, int src_stride,
uint8_t* dst, uint8_t* dst,
@ -172,6 +173,13 @@ void TransposeUVWx8_SSE2(const uint8_t* src,
uint8_t* dst_b, uint8_t* dst_b,
int dst_stride_b, int dst_stride_b,
int width); int width);
void TransposeUVWx16_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width);
void TransposeUVWx8_NEON(const uint8_t* src, void TransposeUVWx8_NEON(const uint8_t* src,
int src_stride, int src_stride,
uint8_t* dst_a, uint8_t* dst_a,

78
source/rotate.cc
View File

@ -36,10 +36,14 @@ void TransposePlane(const uint8_t* src,
void (*TransposeWxH)(const uint8_t* src, int src_stride, uint8_t* dst, void (*TransposeWxH)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width, int height) = NULL; int dst_stride, int width, int height) = NULL;
#endif #endif
#if defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) #if defined(HAS_TRANSPOSEWX16_AVX512BW)
void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = NULL;
#elif defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON)
void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst, void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = TransposeWx16_C; int dst_stride, int width) = TransposeWx16_C;
#else #endif
#if !defined(HAS_TRANSPOSEWX16_LSX) && !defined(HAS_TRANSPOSEWX16_NEON)
void (*TransposeWx8)(const uint8_t* src, int src_stride, uint8_t* dst, void (*TransposeWx8)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = TransposeWx8_C; int dst_stride, int width) = TransposeWx8_C;
#endif #endif
@ -68,17 +72,20 @@ void TransposePlane(const uint8_t* src,
#if defined(HAS_TRANSPOSEWX8_SSSE3) #if defined(HAS_TRANSPOSEWX8_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) { if (TestCpuFlag(kCpuHasSSSE3)) {
TransposeWx8 = TransposeWx8_Any_SSSE3; TransposeWx8 = TransposeWx8_Any_SSSE3;
#if defined(__x86_64__)
if (IS_ALIGNED(width, 16)) {
TransposeWx8 = TransposeWx8_SSSE3;
}
#else
if (IS_ALIGNED(width, 8)) { if (IS_ALIGNED(width, 8)) {
TransposeWx8 = TransposeWx8_SSSE3; TransposeWx8 = TransposeWx8_SSSE3;
} }
#endif
} }
#endif #endif
#if defined(HAS_TRANSPOSEWX8_FAST_SSSE3) #if defined(HAS_TRANSPOSEWX16_AVX512BW)
if (TestCpuFlag(kCpuHasSSSE3)) { if (TestCpuFlag(kCpuHasAVX512BW)) {
TransposeWx8 = TransposeWx8_Fast_Any_SSSE3; TransposeWx16 = TransposeWx16_AVX512BW;
if (IS_ALIGNED(width, 16)) {
TransposeWx8 = TransposeWx8_Fast_SSSE3;
}
} }
#endif #endif
#if defined(HAS_TRANSPOSEWX16_LSX) #if defined(HAS_TRANSPOSEWX16_LSX)
@ -96,15 +103,19 @@ void TransposePlane(const uint8_t* src,
return; return;
} }
#endif #endif
#if defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) #if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEWX16_LSX) || \
defined(HAS_TRANSPOSEWX16_NEON)
// Work across the source in 16x16 tiles // Work across the source in 16x16 tiles
while (i >= 16) { if (TransposeWx16) {
TransposeWx16(src, src_stride, dst, dst_stride, width); while (i >= 16) {
src += 16 * src_stride; // Go down 16 rows. TransposeWx16(src, src_stride, dst, dst_stride, width);
dst += 16; // Move over 16 columns. src += 16 * src_stride; // Go down 16 rows.
i -= 16; dst += 16; // Move over 16 columns.
i -= 16;
}
} }
#else #endif
#if !defined(HAS_TRANSPOSEWX16_LSX) && !defined(HAS_TRANSPOSEWX16_NEON)
// Work across the source in 8x8 tiles // Work across the source in 8x8 tiles
while (i >= 8) { while (i >= 8) {
TransposeWx8(src, src_stride, dst, dst_stride, width); TransposeWx8(src, src_stride, dst, dst_stride, width);
@ -276,11 +287,16 @@ void SplitTransposeUV(const uint8_t* src,
int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
int width, int height) = TransposeUVWxH_C; int width, int height) = TransposeUVWxH_C;
#endif #endif
#if defined(HAS_TRANSPOSEUVWX16_LSX) #if defined(HAS_TRANSPOSEUVWX16_AVX512BW)
void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a,
int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
int width) = NULL;
#elif defined(HAS_TRANSPOSEUVWX16_LSX)
void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a, void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a,
int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
int width) = TransposeUVWx16_C; int width) = TransposeUVWx16_C;
#else #endif
#if !defined(HAS_TRANSPOSEUVWX16_LSX)
void (*TransposeUVWx8)(const uint8_t* src, int src_stride, uint8_t* dst_a, void (*TransposeUVWx8)(const uint8_t* src, int src_stride, uint8_t* dst_a,
int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
int width) = TransposeUVWx8_C; int width) = TransposeUVWx8_C;
@ -315,6 +331,11 @@ void SplitTransposeUV(const uint8_t* src,
} }
} }
#endif #endif
#if defined(HAS_TRANSPOSEUVWX16_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
TransposeUVWx16 = TransposeUVWx16_AVX512BW;
}
#endif
#if defined(HAS_TRANSPOSEUVWXH_SME) #if defined(HAS_TRANSPOSEUVWXH_SME)
if (TestCpuFlag(kCpuHasSME)) { if (TestCpuFlag(kCpuHasSME)) {
@ -323,17 +344,20 @@ void SplitTransposeUV(const uint8_t* src,
return; return;
} }
#endif #endif
#if defined(HAS_TRANSPOSEUVWX16_LSX) #if defined(HAS_TRANSPOSEUVWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_LSX)
// Work through the source in 8x8 tiles. // Work through the source in 16x16 tiles.
while (i >= 16) { if (TransposeUVWx16) {
TransposeUVWx16(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, while (i >= 16) {
width); TransposeUVWx16(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
src += 16 * src_stride; // Go down 16 rows. width);
dst_a += 16; // Move over 8 columns. src += 16 * src_stride; // Go down 16 rows.
dst_b += 16; // Move over 8 columns. dst_a += 16; // Move over 16 columns.
i -= 16; dst_b += 16; // Move over 16 columns.
i -= 16;
}
} }
#else #endif
#if !defined(HAS_TRANSPOSEUVWX16_LSX)
// Work through the source in 8x8 tiles. // Work through the source in 8x8 tiles.
while (i >= 8) { while (i >= 8) {
TransposeUVWx8(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, TransposeUVWx8(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,

5
source/rotate_any.cc
View File

@ -35,10 +35,11 @@ TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7)
TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15) TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15)
#endif #endif
#ifdef HAS_TRANSPOSEWX8_SSSE3 #ifdef HAS_TRANSPOSEWX8_SSSE3
#if defined(__x86_64__)
TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15)
#else
TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 7) TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 7)
#endif #endif
#ifdef HAS_TRANSPOSEWX8_FAST_SSSE3
TANY(TransposeWx8_Fast_Any_SSSE3, TransposeWx8_Fast_SSSE3, TransposeWx8_C, 15)
#endif #endif
#ifdef HAS_TRANSPOSEWX16_LSX #ifdef HAS_TRANSPOSEWX16_LSX
TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, TransposeWx16_C, 15) TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, TransposeWx16_C, 15)

423
source/rotate_gcc.cc
View File

@ -21,103 +21,14 @@ extern "C" {
(defined(__x86_64__) || defined(__i386__)) && \ (defined(__x86_64__) || defined(__i386__)) && \
!defined(LIBYUV_ENABLE_ROWWIN) !defined(LIBYUV_ENABLE_ROWWIN)
// Transpose 8x8. 32 or 64 bit, but not NaCL for 64 bit.
#if defined(HAS_TRANSPOSEWX8_SSSE3) #if defined(HAS_TRANSPOSEWX8_SSSE3)
#if defined(__x86_64__)
// Transpose 16x8. 64 bit.
void TransposeWx8_SSSE3(const uint8_t* src, void TransposeWx8_SSSE3(const uint8_t* src,
int src_stride, int src_stride,
uint8_t* dst, uint8_t* dst,
int dst_stride, int dst_stride,
int width) { int width) {
asm volatile(
// Read in the data from the source pointer.
// First round of bit swap.
LABELALIGN
"1: \n"
"movq (%0),%%xmm0 \n"
"movq (%0,%3),%%xmm1 \n"
"lea (%0,%3,2),%0 \n"
"punpcklbw %%xmm1,%%xmm0 \n"
"movq (%0),%%xmm2 \n"
"movdqa %%xmm0,%%xmm1 \n"
"palignr $0x8,%%xmm1,%%xmm1 \n"
"movq (%0,%3),%%xmm3 \n"
"lea (%0,%3,2),%0 \n"
"punpcklbw %%xmm3,%%xmm2 \n"
"movdqa %%xmm2,%%xmm3 \n"
"movq (%0),%%xmm4 \n"
"palignr $0x8,%%xmm3,%%xmm3 \n"
"movq (%0,%3),%%xmm5 \n"
"lea (%0,%3,2),%0 \n"
"punpcklbw %%xmm5,%%xmm4 \n"
"movdqa %%xmm4,%%xmm5 \n"
"movq (%0),%%xmm6 \n"
"palignr $0x8,%%xmm5,%%xmm5 \n"
"movq (%0,%3),%%xmm7 \n"
"lea (%0,%3,2),%0 \n"
"punpcklbw %%xmm7,%%xmm6 \n"
"neg %3 \n"
"movdqa %%xmm6,%%xmm7 \n"
"lea 0x8(%0,%3,8),%0 \n"
"palignr $0x8,%%xmm7,%%xmm7 \n"
"neg %3 \n"
// Second round of bit swap.
"punpcklwd %%xmm2,%%xmm0 \n"
"punpcklwd %%xmm3,%%xmm1 \n"
"movdqa %%xmm0,%%xmm2 \n"
"movdqa %%xmm1,%%xmm3 \n"
"palignr $0x8,%%xmm2,%%xmm2 \n"
"palignr $0x8,%%xmm3,%%xmm3 \n"
"punpcklwd %%xmm6,%%xmm4 \n"
"punpcklwd %%xmm7,%%xmm5 \n"
"movdqa %%xmm4,%%xmm6 \n"
"movdqa %%xmm5,%%xmm7 \n"
"palignr $0x8,%%xmm6,%%xmm6 \n"
"palignr $0x8,%%xmm7,%%xmm7 \n"
// Third round of bit swap.
// Write to the destination pointer.
"punpckldq %%xmm4,%%xmm0 \n"
"movq %%xmm0,(%1) \n"
"movdqa %%xmm0,%%xmm4 \n"
"palignr $0x8,%%xmm4,%%xmm4 \n"
"movq %%xmm4,(%1,%4) \n"
"lea (%1,%4,2),%1 \n"
"punpckldq %%xmm6,%%xmm2 \n"
"movdqa %%xmm2,%%xmm6 \n"
"movq %%xmm2,(%1) \n"
"palignr $0x8,%%xmm6,%%xmm6 \n"
"punpckldq %%xmm5,%%xmm1 \n"
"movq %%xmm6,(%1,%4) \n"
"lea (%1,%4,2),%1 \n"
"movdqa %%xmm1,%%xmm5 \n"
"movq %%xmm1,(%1) \n"
"palignr $0x8,%%xmm5,%%xmm5 \n"
"movq %%xmm5,(%1,%4) \n"
"lea (%1,%4,2),%1 \n"
"punpckldq %%xmm7,%%xmm3 \n"
"movq %%xmm3,(%1) \n"
"movdqa %%xmm3,%%xmm7 \n"
"palignr $0x8,%%xmm7,%%xmm7 \n"
"sub $0x8,%2 \n"
"movq %%xmm7,(%1,%4) \n"
"lea (%1,%4,2),%1 \n"
"jg 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "r"((ptrdiff_t)(src_stride)), // %3
"r"((ptrdiff_t)(dst_stride)) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
#endif // defined(HAS_TRANSPOSEWX8_SSSE3)
// Transpose 16x8. 64 bit
#if defined(HAS_TRANSPOSEWX8_FAST_SSSE3)
void TransposeWx8_Fast_SSSE3(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
asm volatile( asm volatile(
// Read in the data from the source pointer. // Read in the data from the source pointer.
// First round of bit swap. // First round of bit swap.
@ -252,7 +163,335 @@ void TransposeWx8_Fast_SSSE3(const uint8_t* src,
"xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
"xmm15"); "xmm15");
} }
#endif // defined(HAS_TRANSPOSEWX8_FAST_SSSE3) #else // defined(__x86_64__)
// Transpose 8x8. 32 bit.
void TransposeWx8_SSSE3(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
asm volatile(
// Read in the data from the source pointer.
// First round of bit swap.
LABELALIGN
"1: \n"
"movq (%0),%%xmm0 \n"
"movq (%0,%3),%%xmm1 \n"
"lea (%0,%3,2),%0 \n"
"punpcklbw %%xmm1,%%xmm0 \n"
"movq (%0),%%xmm2 \n"
"movdqa %%xmm0,%%xmm1 \n"
"palignr $0x8,%%xmm1,%%xmm1 \n"
"movq (%0,%3),%%xmm3 \n"
"lea (%0,%3,2),%0 \n"
"punpcklbw %%xmm3,%%xmm2 \n"
"movdqa %%xmm2,%%xmm3 \n"
"movq (%0),%%xmm4 \n"
"palignr $0x8,%%xmm3,%%xmm3 \n"
"movq (%0,%3),%%xmm5 \n"
"lea (%0,%3,2),%0 \n"
"punpcklbw %%xmm5,%%xmm4 \n"
"movdqa %%xmm4,%%xmm5 \n"
"movq (%0),%%xmm6 \n"
"palignr $0x8,%%xmm5,%%xmm5 \n"
"movq (%0,%3),%%xmm7 \n"
"lea (%0,%3,2),%0 \n"
"punpcklbw %%xmm7,%%xmm6 \n"
"neg %3 \n"
"movdqa %%xmm6,%%xmm7 \n"
"lea 0x8(%0,%3,8),%0 \n"
"palignr $0x8,%%xmm7,%%xmm7 \n"
"neg %3 \n"
// Second round of bit swap.
"punpcklwd %%xmm2,%%xmm0 \n"
"punpcklwd %%xmm3,%%xmm1 \n"
"movdqa %%xmm0,%%xmm2 \n"
"movdqa %%xmm1,%%xmm3 \n"
"palignr $0x8,%%xmm2,%%xmm2 \n"
"palignr $0x8,%%xmm3,%%xmm3 \n"
"punpcklwd %%xmm6,%%xmm4 \n"
"punpcklwd %%xmm7,%%xmm5 \n"
"movdqa %%xmm4,%%xmm6 \n"
"movdqa %%xmm5,%%xmm7 \n"
"palignr $0x8,%%xmm6,%%xmm6 \n"
"palignr $0x8,%%xmm7,%%xmm7 \n"
// Third round of bit swap.
// Write to the destination pointer.
"punpckldq %%xmm4,%%xmm0 \n"
"movq %%xmm0,(%1) \n"
"movdqa %%xmm0,%%xmm4 \n"
"palignr $0x8,%%xmm4,%%xmm4 \n"
"movq %%xmm4,(%1,%4) \n"
"lea (%1,%4,2),%1 \n"
"punpckldq %%xmm6,%%xmm2 \n"
"movdqa %%xmm2,%%xmm6 \n"
"movq %%xmm2,(%1) \n"
"palignr $0x8,%%xmm6,%%xmm6 \n"
"punpckldq %%xmm5,%%xmm1 \n"
"movq %%xmm6,(%1,%4) \n"
"lea (%1,%4,2),%1 \n"
"movdqa %%xmm1,%%xmm5 \n"
"movq %%xmm1,(%1) \n"
"palignr $0x8,%%xmm5,%%xmm5 \n"
"movq %%xmm5,(%1,%4) \n"
"lea (%1,%4,2),%1 \n"
"punpckldq %%xmm7,%%xmm3 \n"
"movq %%xmm3,(%1) \n"
"movdqa %%xmm3,%%xmm7 \n"
"palignr $0x8,%%xmm7,%%xmm7 \n"
"sub $0x8,%2 \n"
"movq %%xmm7,(%1,%4) \n"
"lea (%1,%4,2),%1 \n"
"jg 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "r"((ptrdiff_t)(src_stride)), // %3
"r"((ptrdiff_t)(dst_stride)) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
#endif // defined(__x86_64__)
#endif // defined(HAS_TRANSPOSEWX8_SSSE3)
#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW)
// Dword permutes to gather 16 rows of a column from 4 lanes.
static const uint32_t kPermdTranspose_AVX512BW[32] = {
0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30,
1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31};
// Transpose 16x16 bytes held in 4 lanes of 4 zmm registers.
// Source row r is in lane r / 4 of zmm(r % 4). Outputs zmm0..zmm3 are 16
// destination rows, lane n of zmm0/zmm2 for dst_a and lane n of zmm1/zmm3
// for dst_b.
#define TRANSPOSE16X16_AVX512BW \
"vpunpcklbw %%zmm1,%%zmm0,%%zmm4 \n" \
"vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n" \
"vpunpcklbw %%zmm3,%%zmm2,%%zmm0 \n" \
"vpunpckhbw %%zmm3,%%zmm2,%%zmm1 \n" \
"vpunpcklwd %%zmm0,%%zmm4,%%zmm2 \n" \
"vpunpckhwd %%zmm0,%%zmm4,%%zmm3 \n" \
"vpunpcklwd %%zmm1,%%zmm5,%%zmm4 \n" \
"vpunpckhwd %%zmm1,%%zmm5,%%zmm5 \n" \
"vmovdqa64 %%zmm2,%%zmm0 \n" \
"vpermt2d %%zmm4,%%zmm6,%%zmm0 \n" \
"vmovdqa64 %%zmm2,%%zmm1 \n" \
"vpermt2d %%zmm4,%%zmm7,%%zmm1 \n" \
"vmovdqa64 %%zmm3,%%zmm2 \n" \
"vpermt2d %%zmm5,%%zmm6,%%zmm2 \n" \
"vpermt2d %%zmm5,%%zmm7,%%zmm3 \n"
// Transpose 16x16 bytes using 4 lanes of AVX512BW with tail masking.
// Can be used for 8-bit planar transpose (dst_a = even rows, dst_b = odd rows)
// or 16-bit UV split transpose (dst_a = U plane, dst_b = V plane).
// TODO(fbarchard): Port to rotate_win.cc using intrinsics.
// TODO(fbarchard): Use for ARGB (32 bit) and 16 bit channel transposes.
static void TransposeWx16_Byte_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int byte_width) {
uintptr_t temp;
asm volatile(
"vmovdqu32 (%[shuf]),%%zmm6 \n"
"vmovdqu32 0x40(%[shuf]),%%zmm7 \n"
"sub $0x10,%[width] \n"
"jl 2f \n"
// Main loop: 16 source rows x 16 bytes.
LABELALIGN
"1: \n"
"vmovdqu (%[src]),%%xmm0 \n"
"vmovdqu (%[src],%[src_stride]),%%xmm1 \n"
"lea (%[src],%[src_stride],2),%[temp] \n"
"lea 0x10(%[src]),%[src] \n"
"vmovdqu (%[temp]),%%xmm2 \n"
"vmovdqu (%[temp],%[src_stride]),%%xmm3 \n"
"lea (%[temp],%[src_stride],2),%[temp] \n"
"vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n"
"vinserti32x4 $1,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n"
"lea (%[temp],%[src_stride],2),%[temp] \n"
"vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n"
"vinserti32x4 $1,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n"
"lea (%[temp],%[src_stride],2),%[temp] \n"
"vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n"
"vinserti32x4 $2,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n"
"lea (%[temp],%[src_stride],2),%[temp] \n"
"vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n"
"vinserti32x4 $2,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n"
"lea (%[temp],%[src_stride],2),%[temp] \n"
"vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n"
"vinserti32x4 $3,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n"
"lea (%[temp],%[src_stride],2),%[temp] \n"
"vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n"
"vinserti32x4 $3,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n"
TRANSPOSE16X16_AVX512BW
"vmovdqu %%xmm0,(%[dst_a]) \n"
"vmovdqu %%xmm1,(%[dst_b]) \n"
"vextracti32x4 $1,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
"vextracti32x4 $1,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
"vmovdqu %%xmm2,(%[dst_a]) \n"
"vmovdqu %%xmm3,(%[dst_b]) \n"
"vextracti32x4 $1,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
"vextracti32x4 $1,%%zmm3,(%[dst_b],%[dst_stride_b]) \n"
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
"vextracti32x4 $2,%%zmm0,(%[dst_a]) \n"
"vextracti32x4 $2,%%zmm1,(%[dst_b]) \n"
"vextracti32x4 $3,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
"vextracti32x4 $3,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
"vextracti32x4 $2,%%zmm2,(%[dst_a]) \n"
"vextracti32x4 $2,%%zmm3,(%[dst_b]) \n"
"vextracti32x4 $3,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
"vextracti32x4 $3,%%zmm3,(%[dst_b],%[dst_stride_b]) \n"
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
"sub $0x10,%[width] \n"
"jge 1b \n"
// Remainder: 1 to 15 bytes, masked loads.
"2: \n"
"add $0x10,%[width] \n"
"je 99f \n"
"mov $-1,%k[temp] \n"
"bzhi %k[width],%k[temp],%k[temp] \n"
"kmovw %k[temp],%%k1 \n"
"vmovdqu8 (%[src]),%%xmm0%{%%k1%}%{z%} \n"
"vmovdqu8 (%[src],%[src_stride]),%%xmm1%{%%k1%}%{z%} \n"
"lea (%[src],%[src_stride],2),%[src] \n"
"vmovdqu8 (%[src]),%%xmm2%{%%k1%}%{z%} \n"
"vmovdqu8 (%[src],%[src_stride]),%%xmm3%{%%k1%}%{z%} \n"
"lea (%[src],%[src_stride],2),%[src] \n"
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm4,%%zmm0,%%zmm0 \n"
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm4,%%zmm1,%%zmm1 \n"
"lea (%[src],%[src_stride],2),%[src] \n"
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm4,%%zmm2,%%zmm2 \n"
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm4,%%zmm3,%%zmm3 \n"
"lea (%[src],%[src_stride],2),%[src] \n"
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm4,%%zmm0,%%zmm0 \n"
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm4,%%zmm1,%%zmm1 \n"
"lea (%[src],%[src_stride],2),%[src] \n"
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm4,%%zmm2,%%zmm2 \n"
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm4,%%zmm3,%%zmm3 \n"
"lea (%[src],%[src_stride],2),%[src] \n"
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm4,%%zmm0,%%zmm0 \n"
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm4,%%zmm1,%%zmm1 \n"
"lea (%[src],%[src_stride],2),%[src] \n"
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm4,%%zmm2,%%zmm2 \n"
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm4,%%zmm3,%%zmm3 \n"
TRANSPOSE16X16_AVX512BW
// Store 1 destination row per remaining byte.
"vmovdqu %%xmm0,(%[dst_a]) \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vmovdqu %%xmm1,(%[dst_b]) \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $1,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $1,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vmovdqu %%xmm2,(%[dst_a]) \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vmovdqu %%xmm3,(%[dst_b]) \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $1,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $1,%%zmm3,(%[dst_b],%[dst_stride_b]) \n"
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $2,%%zmm0,(%[dst_a]) \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $2,%%zmm1,(%[dst_b]) \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $3,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $3,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $2,%%zmm2,(%[dst_a]) \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $2,%%zmm3,(%[dst_b]) \n"
"sub $0x1,%[width] \n"
"je 99f \n"
"vextracti32x4 $3,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
"99: \n"
"vzeroupper \n"
: [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b),
[width] "+r"(byte_width), [temp] "=&r"(temp)
: [src_stride] "r"((ptrdiff_t)src_stride),
[dst_stride_a] "r"((ptrdiff_t)dst_stride_a),
[dst_stride_b] "r"((ptrdiff_t)dst_stride_b),
[shuf] "r"(kPermdTranspose_AVX512BW)
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7");
}
#undef TRANSPOSE16X16_AVX512BW
#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) ||
// defined(HAS_TRANSPOSEUVWX16_AVX512BW)
#if defined(HAS_TRANSPOSEWX16_AVX512BW)
void TransposeWx16_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
TransposeWx16_Byte_AVX512BW(src, src_stride, dst, dst_stride * 2,
dst + (width > 1 ? dst_stride : 0),
dst_stride * 2, width);
}
#endif // defined(HAS_TRANSPOSEWX16_AVX512BW)
#if defined(HAS_TRANSPOSEUVWX16_AVX512BW)
void TransposeUVWx16_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width) {
TransposeWx16_Byte_AVX512BW(src, src_stride, dst_a, dst_stride_a, dst_b,
dst_stride_b, width * 2);
}
#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
// Transpose UV 8x8. 64 bit. // Transpose UV 8x8. 64 bit.
#if defined(HAS_TRANSPOSEUVWX8_SSE2) #if defined(HAS_TRANSPOSEUVWX8_SSE2)