diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index b40a8effc..2950ee3fb 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -41,10 +41,16 @@ extern "C" { // The following are available for 64 bit GCC: #if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ !defined(LIBYUV_ENABLE_ROWWIN) -#define HAS_TRANSPOSEWX8_FAST_SSSE3 #define HAS_TRANSPOSEUVWX8_SSE2 #endif +// The following are available for AVX512 clang x64 platforms: +#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ + defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) +#define HAS_TRANSPOSEWX16_AVX512BW +#define HAS_TRANSPOSEUVWX16_AVX512BW +#endif + #if !defined(LIBYUV_DISABLE_NEON) && \ (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) #if defined(__aarch64__) @@ -105,11 +111,11 @@ void TransposeWx8_SSSE3(const uint8_t* src, uint8_t* dst, int dst_stride, int width); -void TransposeWx8_Fast_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); +void TransposeWx16_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx16_LSX(const uint8_t* src, int src_stride, uint8_t* dst, @@ -131,11 +137,6 @@ void TransposeWx8_Any_SSSE3(const uint8_t* src, uint8_t* dst, int dst_stride, int width); -void TransposeWx8_Fast_Any_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); void TransposeWx16_Any_LSX(const uint8_t* src, int src_stride, uint8_t* dst, @@ -172,6 +173,13 @@ void TransposeUVWx8_SSE2(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); +void TransposeUVWx16_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); void TransposeUVWx8_NEON(const uint8_t* src, int src_stride, uint8_t* dst_a, diff --git a/source/rotate.cc b/source/rotate.cc index 93e92145f..2cff13945 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -36,10 +36,14 @@ void TransposePlane(const uint8_t* src, void (*TransposeWxH)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width, int height) = NULL; #endif -#if defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) +#if defined(HAS_TRANSPOSEWX16_AVX512BW) + void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst, + int dst_stride, int width) = NULL; +#elif defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) = TransposeWx16_C; -#else +#endif +#if !defined(HAS_TRANSPOSEWX16_LSX) && !defined(HAS_TRANSPOSEWX16_NEON) void (*TransposeWx8)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) = TransposeWx8_C; #endif @@ -68,17 +72,20 @@ void TransposePlane(const uint8_t* src, #if defined(HAS_TRANSPOSEWX8_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { TransposeWx8 = TransposeWx8_Any_SSSE3; +#if defined(__x86_64__) + if (IS_ALIGNED(width, 16)) { + TransposeWx8 = TransposeWx8_SSSE3; + } +#else if (IS_ALIGNED(width, 8)) { TransposeWx8 = TransposeWx8_SSSE3; } +#endif } #endif -#if defined(HAS_TRANSPOSEWX8_FAST_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - TransposeWx8 = TransposeWx8_Fast_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - TransposeWx8 = TransposeWx8_Fast_SSSE3; - } +#if defined(HAS_TRANSPOSEWX16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + TransposeWx16 = TransposeWx16_AVX512BW; } #endif #if defined(HAS_TRANSPOSEWX16_LSX) @@ -96,15 +103,19 @@ void TransposePlane(const uint8_t* src, return; } #endif -#if defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) +#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEWX16_LSX) || \ + defined(HAS_TRANSPOSEWX16_NEON) // Work across the source in 16x16 tiles - while (i >= 16) { - TransposeWx16(src, src_stride, dst, dst_stride, width); - src += 16 * src_stride; // Go down 16 rows. - dst += 16; // Move over 16 columns. - i -= 16; + if (TransposeWx16) { + while (i >= 16) { + TransposeWx16(src, src_stride, dst, dst_stride, width); + src += 16 * src_stride; // Go down 16 rows. + dst += 16; // Move over 16 columns. + i -= 16; + } } -#else +#endif +#if !defined(HAS_TRANSPOSEWX16_LSX) && !defined(HAS_TRANSPOSEWX16_NEON) // Work across the source in 8x8 tiles while (i >= 8) { TransposeWx8(src, src_stride, dst, dst_stride, width); @@ -276,11 +287,16 @@ void SplitTransposeUV(const uint8_t* src, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width, int height) = TransposeUVWxH_C; #endif -#if defined(HAS_TRANSPOSEUVWX16_LSX) +#if defined(HAS_TRANSPOSEUVWX16_AVX512BW) + void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a, + int dst_stride_a, uint8_t* dst_b, int dst_stride_b, + int width) = NULL; +#elif defined(HAS_TRANSPOSEUVWX16_LSX) void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width) = TransposeUVWx16_C; -#else +#endif +#if !defined(HAS_TRANSPOSEUVWX16_LSX) void (*TransposeUVWx8)(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width) = TransposeUVWx8_C; @@ -315,6 +331,11 @@ void SplitTransposeUV(const uint8_t* src, } } #endif +#if defined(HAS_TRANSPOSEUVWX16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + TransposeUVWx16 = TransposeUVWx16_AVX512BW; + } +#endif #if defined(HAS_TRANSPOSEUVWXH_SME) if (TestCpuFlag(kCpuHasSME)) { @@ -323,17 +344,20 @@ void SplitTransposeUV(const uint8_t* src, return; } #endif -#if defined(HAS_TRANSPOSEUVWX16_LSX) - // Work through the source in 8x8 tiles. - while (i >= 16) { - TransposeUVWx16(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, - width); - src += 16 * src_stride; // Go down 16 rows. - dst_a += 16; // Move over 8 columns. - dst_b += 16; // Move over 8 columns. - i -= 16; +#if defined(HAS_TRANSPOSEUVWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_LSX) + // Work through the source in 16x16 tiles. + if (TransposeUVWx16) { + while (i >= 16) { + TransposeUVWx16(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, + width); + src += 16 * src_stride; // Go down 16 rows. + dst_a += 16; // Move over 16 columns. + dst_b += 16; // Move over 16 columns. + i -= 16; + } } -#else +#endif +#if !defined(HAS_TRANSPOSEUVWX16_LSX) // Work through the source in 8x8 tiles. while (i >= 8) { TransposeUVWx8(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, diff --git a/source/rotate_any.cc b/source/rotate_any.cc index 06e7461de..07c2443c8 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -35,10 +35,11 @@ TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7) TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15) #endif #ifdef HAS_TRANSPOSEWX8_SSSE3 +#if defined(__x86_64__) +TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15) +#else TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 7) #endif -#ifdef HAS_TRANSPOSEWX8_FAST_SSSE3 -TANY(TransposeWx8_Fast_Any_SSSE3, TransposeWx8_Fast_SSSE3, TransposeWx8_C, 15) #endif #ifdef HAS_TRANSPOSEWX16_LSX TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, TransposeWx16_C, 15) diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index 9847ecd48..e11eb2c47 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -21,103 +21,14 @@ extern "C" { (defined(__x86_64__) || defined(__i386__)) && \ !defined(LIBYUV_ENABLE_ROWWIN) -// Transpose 8x8. 32 or 64 bit, but not NaCL for 64 bit. #if defined(HAS_TRANSPOSEWX8_SSSE3) +#if defined(__x86_64__) +// Transpose 16x8. 64 bit. void TransposeWx8_SSSE3(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) { - asm volatile( - // Read in the data from the source pointer. - // First round of bit swap. - LABELALIGN - "1: \n" - "movq (%0),%%xmm0 \n" - "movq (%0,%3),%%xmm1 \n" - "lea (%0,%3,2),%0 \n" - "punpcklbw %%xmm1,%%xmm0 \n" - "movq (%0),%%xmm2 \n" - "movdqa %%xmm0,%%xmm1 \n" - "palignr $0x8,%%xmm1,%%xmm1 \n" - "movq (%0,%3),%%xmm3 \n" - "lea (%0,%3,2),%0 \n" - "punpcklbw %%xmm3,%%xmm2 \n" - "movdqa %%xmm2,%%xmm3 \n" - "movq (%0),%%xmm4 \n" - "palignr $0x8,%%xmm3,%%xmm3 \n" - "movq (%0,%3),%%xmm5 \n" - "lea (%0,%3,2),%0 \n" - "punpcklbw %%xmm5,%%xmm4 \n" - "movdqa %%xmm4,%%xmm5 \n" - "movq (%0),%%xmm6 \n" - "palignr $0x8,%%xmm5,%%xmm5 \n" - "movq (%0,%3),%%xmm7 \n" - "lea (%0,%3,2),%0 \n" - "punpcklbw %%xmm7,%%xmm6 \n" - "neg %3 \n" - "movdqa %%xmm6,%%xmm7 \n" - "lea 0x8(%0,%3,8),%0 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "neg %3 \n" - // Second round of bit swap. - "punpcklwd %%xmm2,%%xmm0 \n" - "punpcklwd %%xmm3,%%xmm1 \n" - "movdqa %%xmm0,%%xmm2 \n" - "movdqa %%xmm1,%%xmm3 \n" - "palignr $0x8,%%xmm2,%%xmm2 \n" - "palignr $0x8,%%xmm3,%%xmm3 \n" - "punpcklwd %%xmm6,%%xmm4 \n" - "punpcklwd %%xmm7,%%xmm5 \n" - "movdqa %%xmm4,%%xmm6 \n" - "movdqa %%xmm5,%%xmm7 \n" - "palignr $0x8,%%xmm6,%%xmm6 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - // Third round of bit swap. - // Write to the destination pointer. - "punpckldq %%xmm4,%%xmm0 \n" - "movq %%xmm0,(%1) \n" - "movdqa %%xmm0,%%xmm4 \n" - "palignr $0x8,%%xmm4,%%xmm4 \n" - "movq %%xmm4,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm6,%%xmm2 \n" - "movdqa %%xmm2,%%xmm6 \n" - "movq %%xmm2,(%1) \n" - "palignr $0x8,%%xmm6,%%xmm6 \n" - "punpckldq %%xmm5,%%xmm1 \n" - "movq %%xmm6,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "movdqa %%xmm1,%%xmm5 \n" - "movq %%xmm1,(%1) \n" - "palignr $0x8,%%xmm5,%%xmm5 \n" - "movq %%xmm5,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm7,%%xmm3 \n" - "movq %%xmm3,(%1) \n" - "movdqa %%xmm3,%%xmm7 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "sub $0x8,%2 \n" - "movq %%xmm7,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "r"((ptrdiff_t)(src_stride)), // %3 - "r"((ptrdiff_t)(dst_stride)) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); -} -#endif // defined(HAS_TRANSPOSEWX8_SSSE3) - -// Transpose 16x8. 64 bit -#if defined(HAS_TRANSPOSEWX8_FAST_SSSE3) -void TransposeWx8_Fast_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { asm volatile( // Read in the data from the source pointer. // First round of bit swap. @@ -252,7 +163,335 @@ void TransposeWx8_Fast_SSSE3(const uint8_t* src, "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); } -#endif // defined(HAS_TRANSPOSEWX8_FAST_SSSE3) +#else // defined(__x86_64__) +// Transpose 8x8. 32 bit. +void TransposeWx8_SSSE3(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + asm volatile( + // Read in the data from the source pointer. + // First round of bit swap. + LABELALIGN + "1: \n" + "movq (%0),%%xmm0 \n" + "movq (%0,%3),%%xmm1 \n" + "lea (%0,%3,2),%0 \n" + "punpcklbw %%xmm1,%%xmm0 \n" + "movq (%0),%%xmm2 \n" + "movdqa %%xmm0,%%xmm1 \n" + "palignr $0x8,%%xmm1,%%xmm1 \n" + "movq (%0,%3),%%xmm3 \n" + "lea (%0,%3,2),%0 \n" + "punpcklbw %%xmm3,%%xmm2 \n" + "movdqa %%xmm2,%%xmm3 \n" + "movq (%0),%%xmm4 \n" + "palignr $0x8,%%xmm3,%%xmm3 \n" + "movq (%0,%3),%%xmm5 \n" + "lea (%0,%3,2),%0 \n" + "punpcklbw %%xmm5,%%xmm4 \n" + "movdqa %%xmm4,%%xmm5 \n" + "movq (%0),%%xmm6 \n" + "palignr $0x8,%%xmm5,%%xmm5 \n" + "movq (%0,%3),%%xmm7 \n" + "lea (%0,%3,2),%0 \n" + "punpcklbw %%xmm7,%%xmm6 \n" + "neg %3 \n" + "movdqa %%xmm6,%%xmm7 \n" + "lea 0x8(%0,%3,8),%0 \n" + "palignr $0x8,%%xmm7,%%xmm7 \n" + "neg %3 \n" + // Second round of bit swap. + "punpcklwd %%xmm2,%%xmm0 \n" + "punpcklwd %%xmm3,%%xmm1 \n" + "movdqa %%xmm0,%%xmm2 \n" + "movdqa %%xmm1,%%xmm3 \n" + "palignr $0x8,%%xmm2,%%xmm2 \n" + "palignr $0x8,%%xmm3,%%xmm3 \n" + "punpcklwd %%xmm6,%%xmm4 \n" + "punpcklwd %%xmm7,%%xmm5 \n" + "movdqa %%xmm4,%%xmm6 \n" + "movdqa %%xmm5,%%xmm7 \n" + "palignr $0x8,%%xmm6,%%xmm6 \n" + "palignr $0x8,%%xmm7,%%xmm7 \n" + // Third round of bit swap. + // Write to the destination pointer. + "punpckldq %%xmm4,%%xmm0 \n" + "movq %%xmm0,(%1) \n" + "movdqa %%xmm0,%%xmm4 \n" + "palignr $0x8,%%xmm4,%%xmm4 \n" + "movq %%xmm4,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "punpckldq %%xmm6,%%xmm2 \n" + "movdqa %%xmm2,%%xmm6 \n" + "movq %%xmm2,(%1) \n" + "palignr $0x8,%%xmm6,%%xmm6 \n" + "punpckldq %%xmm5,%%xmm1 \n" + "movq %%xmm6,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "movdqa %%xmm1,%%xmm5 \n" + "movq %%xmm1,(%1) \n" + "palignr $0x8,%%xmm5,%%xmm5 \n" + "movq %%xmm5,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "punpckldq %%xmm7,%%xmm3 \n" + "movq %%xmm3,(%1) \n" + "movdqa %%xmm3,%%xmm7 \n" + "palignr $0x8,%%xmm7,%%xmm7 \n" + "sub $0x8,%2 \n" + "movq %%xmm7,(%1,%4) \n" + "lea (%1,%4,2),%1 \n" + "jg 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "r"((ptrdiff_t)(src_stride)), // %3 + "r"((ptrdiff_t)(dst_stride)) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // defined(__x86_64__) +#endif // defined(HAS_TRANSPOSEWX8_SSSE3) + +#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW) +// Dword permutes to gather 16 rows of a column from 4 lanes. +static const uint32_t kPermdTranspose_AVX512BW[32] = { + 0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30, + 1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31}; + +// Transpose 16x16 bytes held in 4 lanes of 4 zmm registers. +// Source row r is in lane r / 4 of zmm(r % 4). Outputs zmm0..zmm3 are 16 +// destination rows, lane n of zmm0/zmm2 for dst_a and lane n of zmm1/zmm3 +// for dst_b. +#define TRANSPOSE16X16_AVX512BW \ + "vpunpcklbw %%zmm1,%%zmm0,%%zmm4 \n" \ + "vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n" \ + "vpunpcklbw %%zmm3,%%zmm2,%%zmm0 \n" \ + "vpunpckhbw %%zmm3,%%zmm2,%%zmm1 \n" \ + "vpunpcklwd %%zmm0,%%zmm4,%%zmm2 \n" \ + "vpunpckhwd %%zmm0,%%zmm4,%%zmm3 \n" \ + "vpunpcklwd %%zmm1,%%zmm5,%%zmm4 \n" \ + "vpunpckhwd %%zmm1,%%zmm5,%%zmm5 \n" \ + "vmovdqa64 %%zmm2,%%zmm0 \n" \ + "vpermt2d %%zmm4,%%zmm6,%%zmm0 \n" \ + "vmovdqa64 %%zmm2,%%zmm1 \n" \ + "vpermt2d %%zmm4,%%zmm7,%%zmm1 \n" \ + "vmovdqa64 %%zmm3,%%zmm2 \n" \ + "vpermt2d %%zmm5,%%zmm6,%%zmm2 \n" \ + "vpermt2d %%zmm5,%%zmm7,%%zmm3 \n" + +// Transpose 16x16 bytes using 4 lanes of AVX512BW with tail masking. +// Can be used for 8-bit planar transpose (dst_a = even rows, dst_b = odd rows) +// or 16-bit UV split transpose (dst_a = U plane, dst_b = V plane). +// TODO(fbarchard): Port to rotate_win.cc using intrinsics. +// TODO(fbarchard): Use for ARGB (32 bit) and 16 bit channel transposes. +static void TransposeWx16_Byte_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int byte_width) { + uintptr_t temp; + asm volatile( + "vmovdqu32 (%[shuf]),%%zmm6 \n" + "vmovdqu32 0x40(%[shuf]),%%zmm7 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + // Main loop: 16 source rows x 16 bytes. + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[src_stride]),%%xmm1 \n" + "lea (%[src],%[src_stride],2),%[temp] \n" + "lea 0x10(%[src]),%[src] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[src_stride]),%%xmm3 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $1,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $1,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $2,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $2,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $3,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $3,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n" + + TRANSPOSE16X16_AVX512BW + + "vmovdqu %%xmm0,(%[dst_a]) \n" + "vmovdqu %%xmm1,(%[dst_b]) \n" + "vextracti32x4 $1,%%zmm0,(%[dst_a],%[dst_stride_a]) \n" + "vextracti32x4 $1,%%zmm1,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "vmovdqu %%xmm2,(%[dst_a]) \n" + "vmovdqu %%xmm3,(%[dst_b]) \n" + "vextracti32x4 $1,%%zmm2,(%[dst_a],%[dst_stride_a]) \n" + "vextracti32x4 $1,%%zmm3,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "vextracti32x4 $2,%%zmm0,(%[dst_a]) \n" + "vextracti32x4 $2,%%zmm1,(%[dst_b]) \n" + "vextracti32x4 $3,%%zmm0,(%[dst_a],%[dst_stride_a]) \n" + "vextracti32x4 $3,%%zmm1,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "vextracti32x4 $2,%%zmm2,(%[dst_a]) \n" + "vextracti32x4 $2,%%zmm3,(%[dst_b]) \n" + "vextracti32x4 $3,%%zmm2,(%[dst_a],%[dst_stride_a]) \n" + "vextracti32x4 $3,%%zmm3,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + // Remainder: 1 to 15 bytes, masked loads. + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k[width],%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu8 (%[src]),%%xmm0%{%%k1%}%{z%} \n" + "vmovdqu8 (%[src],%[src_stride]),%%xmm1%{%%k1%}%{z%} \n" + "lea (%[src],%[src_stride],2),%[src] \n" + "vmovdqu8 (%[src]),%%xmm2%{%%k1%}%{z%} \n" + "vmovdqu8 (%[src],%[src_stride]),%%xmm3%{%%k1%}%{z%} \n" + "lea (%[src],%[src_stride],2),%[src] \n" + "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm4,%%zmm0,%%zmm0 \n" + "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm4,%%zmm1,%%zmm1 \n" + "lea (%[src],%[src_stride],2),%[src] \n" + "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm4,%%zmm2,%%zmm2 \n" + "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm4,%%zmm3,%%zmm3 \n" + "lea (%[src],%[src_stride],2),%[src] \n" + "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm4,%%zmm0,%%zmm0 \n" + "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm4,%%zmm1,%%zmm1 \n" + "lea (%[src],%[src_stride],2),%[src] \n" + "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm4,%%zmm2,%%zmm2 \n" + "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm4,%%zmm3,%%zmm3 \n" + "lea (%[src],%[src_stride],2),%[src] \n" + "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm4,%%zmm0,%%zmm0 \n" + "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm4,%%zmm1,%%zmm1 \n" + "lea (%[src],%[src_stride],2),%[src] \n" + "vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm4,%%zmm2,%%zmm2 \n" + "vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm4,%%zmm3,%%zmm3 \n" + + TRANSPOSE16X16_AVX512BW + + // Store 1 destination row per remaining byte. + "vmovdqu %%xmm0,(%[dst_a]) \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vmovdqu %%xmm1,(%[dst_b]) \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $1,%%zmm0,(%[dst_a],%[dst_stride_a]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $1,%%zmm1,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vmovdqu %%xmm2,(%[dst_a]) \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vmovdqu %%xmm3,(%[dst_b]) \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $1,%%zmm2,(%[dst_a],%[dst_stride_a]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $1,%%zmm3,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $2,%%zmm0,(%[dst_a]) \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $2,%%zmm1,(%[dst_b]) \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $3,%%zmm0,(%[dst_a],%[dst_stride_a]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $3,%%zmm1,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $2,%%zmm2,(%[dst_a]) \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $2,%%zmm3,(%[dst_b]) \n" + "sub $0x1,%[width] \n" + "je 99f \n" + "vextracti32x4 $3,%%zmm2,(%[dst_a],%[dst_stride_a]) \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b), + [width] "+r"(byte_width), [temp] "=&r"(temp) + : [src_stride] "r"((ptrdiff_t)src_stride), + [dst_stride_a] "r"((ptrdiff_t)dst_stride_a), + [dst_stride_b] "r"((ptrdiff_t)dst_stride_b), + [shuf] "r"(kPermdTranspose_AVX512BW) + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#undef TRANSPOSE16X16_AVX512BW +#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) || + // defined(HAS_TRANSPOSEUVWX16_AVX512BW) + +#if defined(HAS_TRANSPOSEWX16_AVX512BW) +void TransposeWx16_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx16_Byte_AVX512BW(src, src_stride, dst, dst_stride * 2, + dst + (width > 1 ? dst_stride : 0), + dst_stride * 2, width); +} +#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) + +#if defined(HAS_TRANSPOSEUVWX16_AVX512BW) +void TransposeUVWx16_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeWx16_Byte_AVX512BW(src, src_stride, dst_a, dst_stride_a, dst_b, + dst_stride_b, width * 2); +} +#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW) // Transpose UV 8x8. 64 bit. #if defined(HAS_TRANSPOSEUVWX8_SSE2)