diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index 2a79f2dbd..af3288fa7 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -22,9 +22,9 @@ extern "C" { #endif // The following are available for Visual C and GCC: -#if !defined(LIBYUV_DISABLE_X86) && \ - ((defined(__x86_64__) && !defined(LIBYUV_ENABLE_ROWWIN)) || \ - defined(__i386__) || defined(_M_IX86)) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || ((defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN))) #define HAS_HASHDJB2_SSE41 #define HAS_SUMSQUAREERROR_SSE2 #define HAS_HAMMINGDISTANCE_SSE42 diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index b1b4f2a7b..aa376579b 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -21,27 +21,34 @@ namespace libyuv { extern "C" { #endif -// The following are available for Visual C 32 bit: -// TODO - port to clangcl on rotate_win -#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER) && \ - !defined(__clang__) +// The following are available for 32 bit x86, Visual C 64 bit, and for clang +// with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These +// are intrinsics in rotate_win.cc. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__i386__) || defined(_M_IX86) || \ + ((defined(__x86_64__) || defined(_M_X64)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)))) #define HAS_TRANSPOSEWX8_SSSE3 #define HAS_TRANSPOSEUVWX8_SSE2 -#endif - -// The following are available for GCC 32 or 64 bit: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__i386__) || defined(__x86_64__)) && \ - !defined(LIBYUV_ENABLE_ROWWIN) -#define HAS_TRANSPOSEWX8_SSSE3 #define HAS_TRANSPOSE4X4_32_SSE2 #define HAS_TRANSPOSE4X4_32_AVX2 +#define HAS_TRANSPOSEWX16_AVX2 +#define HAS_TRANSPOSEUVWX16_AVX2 +#if defined(__x86_64__) || defined(_M_X64) +#define HAS_TRANSPOSEWX16_AVX512BW +#define HAS_TRANSPOSEUVWX16_AVX512BW +#endif #endif -// The following are available for 64 bit GCC: +// The following are available for 64 bit GCC and clang, as inline assembly in +// rotate_gcc.cc: #if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ !defined(LIBYUV_ENABLE_ROWWIN) +#define HAS_TRANSPOSEWX8_SSSE3 #define HAS_TRANSPOSEUVWX8_SSE2 +#define HAS_TRANSPOSE4X4_32_SSE2 +#define HAS_TRANSPOSE4X4_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 #endif diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index d806867e3..5f10e592a 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -23,10 +23,9 @@ extern "C" { #endif // The following are available on all x86 platforms: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(_M_IX86) || \ - (defined(__x86_64__) && !defined(LIBYUV_ENABLE_ROWWIN)) || \ - defined(__i386__)) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || ((defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN))) #define HAS_FIXEDDIV1_X86 #define HAS_FIXEDDIV_X86 #define HAS_SCALEADDROW_SSE2 diff --git a/source/rotate.cc b/source/rotate.cc index aa71b0b19..95767b4ce 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -72,15 +72,9 @@ void TransposePlane(const uint8_t* src, #if defined(HAS_TRANSPOSEWX8_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { TransposeWx8 = TransposeWx8_Any_SSSE3; -#if defined(__x86_64__) if (IS_ALIGNED(width, 16)) { TransposeWx8 = TransposeWx8_SSSE3; } -#else - if (IS_ALIGNED(width, 8)) { - TransposeWx8 = TransposeWx8_SSSE3; - } -#endif } #endif #if defined(HAS_TRANSPOSEWX16_AVX2) diff --git a/source/rotate_any.cc b/source/rotate_any.cc index 00af1dd0b..52dd8ac2b 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -35,11 +35,7 @@ TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7) TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15) #endif #ifdef HAS_TRANSPOSEWX8_SSSE3 -#if defined(__x86_64__) TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15) -#else -TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 7) -#endif #endif #ifdef HAS_TRANSPOSEWX16_AVX2 TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15) diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index 7d0e21942..64181160a 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -16,13 +16,11 @@ namespace libyuv { extern "C" { #endif -// This module is for GCC x86 and x64. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(__i386__)) && \ +// This module is for GCC and clang x64. 32 bit x86 uses rotate_win.cc. +#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ !defined(LIBYUV_ENABLE_ROWWIN) #if defined(HAS_TRANSPOSEWX8_SSSE3) -#if defined(__x86_64__) // Transpose 16x8. 64 bit. void TransposeWx8_SSSE3(const uint8_t* src, int src_stride, @@ -163,95 +161,6 @@ void TransposeWx8_SSSE3(const uint8_t* src, "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); } -#else // defined(__x86_64__) -// Transpose 8x8. 32 bit. -void TransposeWx8_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - asm volatile( - // Read in the data from the source pointer. - // First round of bit swap. - LABELALIGN - "1: \n" - "movq (%0),%%xmm0 \n" - "movq (%0,%3),%%xmm1 \n" - "lea (%0,%3,2),%0 \n" - "punpcklbw %%xmm1,%%xmm0 \n" - "movq (%0),%%xmm2 \n" - "movdqa %%xmm0,%%xmm1 \n" - "palignr $0x8,%%xmm1,%%xmm1 \n" - "movq (%0,%3),%%xmm3 \n" - "lea (%0,%3,2),%0 \n" - "punpcklbw %%xmm3,%%xmm2 \n" - "movdqa %%xmm2,%%xmm3 \n" - "movq (%0),%%xmm4 \n" - "palignr $0x8,%%xmm3,%%xmm3 \n" - "movq (%0,%3),%%xmm5 \n" - "lea (%0,%3,2),%0 \n" - "punpcklbw %%xmm5,%%xmm4 \n" - "movdqa %%xmm4,%%xmm5 \n" - "movq (%0),%%xmm6 \n" - "palignr $0x8,%%xmm5,%%xmm5 \n" - "movq (%0,%3),%%xmm7 \n" - "lea (%0,%3,2),%0 \n" - "punpcklbw %%xmm7,%%xmm6 \n" - "neg %3 \n" - "movdqa %%xmm6,%%xmm7 \n" - "lea 0x8(%0,%3,8),%0 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "neg %3 \n" - // Second round of bit swap. - "punpcklwd %%xmm2,%%xmm0 \n" - "punpcklwd %%xmm3,%%xmm1 \n" - "movdqa %%xmm0,%%xmm2 \n" - "movdqa %%xmm1,%%xmm3 \n" - "palignr $0x8,%%xmm2,%%xmm2 \n" - "palignr $0x8,%%xmm3,%%xmm3 \n" - "punpcklwd %%xmm6,%%xmm4 \n" - "punpcklwd %%xmm7,%%xmm5 \n" - "movdqa %%xmm4,%%xmm6 \n" - "movdqa %%xmm5,%%xmm7 \n" - "palignr $0x8,%%xmm6,%%xmm6 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - // Third round of bit swap. - // Write to the destination pointer. - "punpckldq %%xmm4,%%xmm0 \n" - "movq %%xmm0,(%1) \n" - "movdqa %%xmm0,%%xmm4 \n" - "palignr $0x8,%%xmm4,%%xmm4 \n" - "movq %%xmm4,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm6,%%xmm2 \n" - "movdqa %%xmm2,%%xmm6 \n" - "movq %%xmm2,(%1) \n" - "palignr $0x8,%%xmm6,%%xmm6 \n" - "punpckldq %%xmm5,%%xmm1 \n" - "movq %%xmm6,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "movdqa %%xmm1,%%xmm5 \n" - "movq %%xmm1,(%1) \n" - "palignr $0x8,%%xmm5,%%xmm5 \n" - "movq %%xmm5,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm7,%%xmm3 \n" - "movq %%xmm3,(%1) \n" - "movdqa %%xmm3,%%xmm7 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "sub $0x8,%2 \n" - "movq %%xmm7,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "r"((ptrdiff_t)(src_stride)), // %3 - "r"((ptrdiff_t)(dst_stride)) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); -} -#endif // defined(__x86_64__) #endif // defined(HAS_TRANSPOSEWX8_SSSE3) #if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW) @@ -877,7 +786,8 @@ void Transpose4x4_32_AVX2(const uint8_t* src, } #endif // defined(HAS_TRANSPOSE4X4_32_AVX2) -#endif // defined(__x86_64__) || defined(__i386__) +#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && + // !defined(LIBYUV_ENABLE_ROWWIN) #ifdef __cplusplus } // extern "C" diff --git a/source/rotate_win.cc b/source/rotate_win.cc index 5b40f62a0..12eb63862 100644 --- a/source/rotate_win.cc +++ b/source/rotate_win.cc @@ -11,243 +11,485 @@ #include "libyuv/rotate_row.h" #include "libyuv/row.h" +// This module is for 32 bit x86 with any compiler, Visual C 64 bit, and for +// clang with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__i386__) || defined(_M_IX86) || \ + ((defined(__x86_64__) || defined(_M_X64)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)))) + +#include +#include // For AVX2 and AVX512 intrinsics +#if defined(_MSC_VER) && !defined(__clang__) +#include +#endif + #ifdef __cplusplus namespace libyuv { extern "C" { #endif -// This module is for 32 bit Visual C x86 -#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) && \ - (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN)) +// Target attributes allow gcc and clang to build each function for its +// instruction set regardless of the baseline, which may lack SSE2 for 32 bit. +#if defined(__clang__) || defined(__GNUC__) +#define LIBYUV_TARGET_SSE2 __attribute__((target("sse2"))) +#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) +#define LIBYUV_TARGET_AVX512BW \ + __attribute__((target("avx512bw,avx512vl,avx512f"))) +#else +#define LIBYUV_TARGET_SSE2 +#define LIBYUV_TARGET_AVX2 +#define LIBYUV_TARGET_AVX512BW +#endif // defined(__clang__) || defined(__GNUC__) -__declspec(naked) void TransposeWx8_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - __asm { - push edi - push esi - push ebp - mov eax, [esp + 12 + 4] // src - mov edi, [esp + 12 + 8] // src_stride - mov edx, [esp + 12 + 12] // dst - mov esi, [esp + 12 + 16] // dst_stride - mov ecx, [esp + 12 + 20] // width +#if defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2) +// Transpose 8 rows x 16 bytes with SSE2. Each output holds 2 columns x 8 rows. +// The low 8 bytes (even column) go to dst_a and the high 8 bytes (odd column) +// go to dst_b. Width is a multiple of 16 bytes. +LIBYUV_TARGET_SSE2 +static void TransposeWx8_Byte_SSE2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int byte_width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t sa = dst_stride_a; + const ptrdiff_t sb = dst_stride_b; + while (byte_width > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); + __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); + __m128i r4 = _mm_loadu_si128((const __m128i*)(src + s * 4)); + __m128i r5 = _mm_loadu_si128((const __m128i*)(src + s * 5)); + __m128i r6 = _mm_loadu_si128((const __m128i*)(src + s * 6)); + __m128i r7 = _mm_loadu_si128((const __m128i*)(src + s * 7)); - // Read in the data from the source pointer. - // First round of bit swap. - align 4 - convertloop: - movq xmm0, qword ptr [eax] - lea ebp, [eax + 8] - movq xmm1, qword ptr [eax + edi] - lea eax, [eax + 2 * edi] - punpcklbw xmm0, xmm1 - movq xmm2, qword ptr [eax] - movdqa xmm1, xmm0 - palignr xmm1, xmm1, 8 - movq xmm3, qword ptr [eax + edi] - lea eax, [eax + 2 * edi] - punpcklbw xmm2, xmm3 - movdqa xmm3, xmm2 - movq xmm4, qword ptr [eax] - palignr xmm3, xmm3, 8 - movq xmm5, qword ptr [eax + edi] - punpcklbw xmm4, xmm5 - lea eax, [eax + 2 * edi] - movdqa xmm5, xmm4 - movq xmm6, qword ptr [eax] - palignr xmm5, xmm5, 8 - movq xmm7, qword ptr [eax + edi] - punpcklbw xmm6, xmm7 - mov eax, ebp - movdqa xmm7, xmm6 - palignr xmm7, xmm7, 8 - // Second round of bit swap. - punpcklwd xmm0, xmm2 - punpcklwd xmm1, xmm3 - movdqa xmm2, xmm0 - movdqa xmm3, xmm1 - palignr xmm2, xmm2, 8 - palignr xmm3, xmm3, 8 - punpcklwd xmm4, xmm6 - punpcklwd xmm5, xmm7 - movdqa xmm6, xmm4 - movdqa xmm7, xmm5 - palignr xmm6, xmm6, 8 - palignr xmm7, xmm7, 8 - // Third round of bit swap. - // Write to the destination pointer. - punpckldq xmm0, xmm4 - movq qword ptr [edx], xmm0 - movdqa xmm4, xmm0 - palignr xmm4, xmm4, 8 - movq qword ptr [edx + esi], xmm4 - lea edx, [edx + 2 * esi] - punpckldq xmm2, xmm6 - movdqa xmm6, xmm2 - palignr xmm6, xmm6, 8 - movq qword ptr [edx], xmm2 - punpckldq xmm1, xmm5 - movq qword ptr [edx + esi], xmm6 - lea edx, [edx + 2 * esi] - movdqa xmm5, xmm1 - movq qword ptr [edx], xmm1 - palignr xmm5, xmm5, 8 - punpckldq xmm3, xmm7 - movq qword ptr [edx + esi], xmm5 - lea edx, [edx + 2 * esi] - movq qword ptr [edx], xmm3 - movdqa xmm7, xmm3 - palignr xmm7, xmm7, 8 - sub ecx, 8 - movq qword ptr [edx + esi], xmm7 - lea edx, [edx + 2 * esi] - jg convertloop + // Bytes of row pairs. a0 is columns 0..7 of rows 0, 1. + __m128i a0 = _mm_unpacklo_epi8(r0, r1); + __m128i a1 = _mm_unpackhi_epi8(r0, r1); + __m128i a2 = _mm_unpacklo_epi8(r2, r3); + __m128i a3 = _mm_unpackhi_epi8(r2, r3); + __m128i a4 = _mm_unpacklo_epi8(r4, r5); + __m128i a5 = _mm_unpackhi_epi8(r4, r5); + __m128i a6 = _mm_unpacklo_epi8(r6, r7); + __m128i a7 = _mm_unpackhi_epi8(r6, r7); - pop ebp - pop esi - pop edi - ret + // Words. b0 is columns 0..3 of rows 0..3. + __m128i b0 = _mm_unpacklo_epi16(a0, a2); + __m128i b1 = _mm_unpackhi_epi16(a0, a2); + __m128i b2 = _mm_unpacklo_epi16(a1, a3); + __m128i b3 = _mm_unpackhi_epi16(a1, a3); + __m128i b4 = _mm_unpacklo_epi16(a4, a6); + __m128i b5 = _mm_unpackhi_epi16(a4, a6); + __m128i b6 = _mm_unpacklo_epi16(a5, a7); + __m128i b7 = _mm_unpackhi_epi16(a5, a7); + + // Dwords. c0 is columns 0, 1 of rows 0..7. + __m128i c0 = _mm_unpacklo_epi32(b0, b4); + __m128i c1 = _mm_unpackhi_epi32(b0, b4); + __m128i c2 = _mm_unpacklo_epi32(b1, b5); + __m128i c3 = _mm_unpackhi_epi32(b1, b5); + __m128i c4 = _mm_unpacklo_epi32(b2, b6); + __m128i c5 = _mm_unpackhi_epi32(b2, b6); + __m128i c6 = _mm_unpacklo_epi32(b3, b7); + __m128i c7 = _mm_unpackhi_epi32(b3, b7); + + _mm_storel_epi64((__m128i*)(dst_a), c0); + _mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(c0)); + _mm_storel_epi64((__m128i*)(dst_a + sa), c1); + _mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(c1)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 2), c2); + _mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(c2)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 3), c3); + _mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(c3)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 4), c4); + _mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(c4)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 5), c5); + _mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(c5)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 6), c6); + _mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(c6)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 7), c7); + _mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(c7)); + + src += 16; + dst_a += sa * 8; + dst_b += sb * 8; + byte_width -= 16; } } +#endif // defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2) -__declspec(naked) void TransposeUVWx8_SSE2(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int w) { - __asm { - push ebx - push esi - push edi - push ebp - mov eax, [esp + 16 + 4] // src - mov edi, [esp + 16 + 8] // src_stride - mov edx, [esp + 16 + 12] // dst_a - mov esi, [esp + 16 + 16] // dst_stride_a - mov ebx, [esp + 16 + 20] // dst_b - mov ebp, [esp + 16 + 24] // dst_stride_b - mov ecx, esp - sub esp, 4 + 16 - and esp, ~15 - mov [esp + 16], ecx - mov ecx, [ecx + 16 + 28] // w +#if defined(HAS_TRANSPOSEWX8_SSSE3) +// Transpose 16x8. Width is a multiple of 16. +LIBYUV_TARGET_SSE2 +void TransposeWx8_SSSE3(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx8_Byte_SSE2(src, src_stride, dst, dst_stride * 2, dst + dst_stride, + dst_stride * 2, width); +} +#endif // defined(HAS_TRANSPOSEWX8_SSSE3) - align 4 - // Read in the data from the source pointer. - // First round of bit swap. - convertloop: - movdqu xmm0, [eax] - movdqu xmm1, [eax + edi] - lea eax, [eax + 2 * edi] - movdqa xmm7, xmm0 // use xmm7 as temp register. - punpcklbw xmm0, xmm1 - punpckhbw xmm7, xmm1 - movdqa xmm1, xmm7 - movdqu xmm2, [eax] - movdqu xmm3, [eax + edi] - lea eax, [eax + 2 * edi] - movdqa xmm7, xmm2 - punpcklbw xmm2, xmm3 - punpckhbw xmm7, xmm3 - movdqa xmm3, xmm7 - movdqu xmm4, [eax] - movdqu xmm5, [eax + edi] - lea eax, [eax + 2 * edi] - movdqa xmm7, xmm4 - punpcklbw xmm4, xmm5 - punpckhbw xmm7, xmm5 - movdqa xmm5, xmm7 - movdqu xmm6, [eax] - movdqu xmm7, [eax + edi] - lea eax, [eax + 2 * edi] - movdqu [esp], xmm5 // backup xmm5 - neg edi - movdqa xmm5, xmm6 // use xmm5 as temp register. - punpcklbw xmm6, xmm7 - punpckhbw xmm5, xmm7 - movdqa xmm7, xmm5 - lea eax, [eax + 8 * edi + 16] - neg edi - // Second round of bit swap. - movdqa xmm5, xmm0 - punpcklwd xmm0, xmm2 - punpckhwd xmm5, xmm2 - movdqa xmm2, xmm5 - movdqa xmm5, xmm1 - punpcklwd xmm1, xmm3 - punpckhwd xmm5, xmm3 - movdqa xmm3, xmm5 - movdqa xmm5, xmm4 - punpcklwd xmm4, xmm6 - punpckhwd xmm5, xmm6 - movdqa xmm6, xmm5 - movdqu xmm5, [esp] // restore xmm5 - movdqu [esp], xmm6 // backup xmm6 - movdqa xmm6, xmm5 // use xmm6 as temp register. - punpcklwd xmm5, xmm7 - punpckhwd xmm6, xmm7 - movdqa xmm7, xmm6 +#if defined(HAS_TRANSPOSEUVWX8_SSE2) +// Transpose UV 8x8. Width is a multiple of 8. +LIBYUV_TARGET_SSE2 +void TransposeUVWx8_SSE2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeWx8_Byte_SSE2(src, src_stride, dst_a, dst_stride_a, dst_b, + dst_stride_b, width * 2); +} +#endif // defined(HAS_TRANSPOSEUVWX8_SSE2) - // Third round of bit swap. - // Write to the destination pointer. - movdqa xmm6, xmm0 - punpckldq xmm0, xmm4 - punpckhdq xmm6, xmm4 - movdqa xmm4, xmm6 - movdqu xmm6, [esp] // restore xmm6 - movlpd qword ptr [edx], xmm0 - movhpd qword ptr [ebx], xmm0 - movlpd qword ptr [edx + esi], xmm4 - lea edx, [edx + 2 * esi] - movhpd qword ptr [ebx + ebp], xmm4 - lea ebx, [ebx + 2 * ebp] - movdqa xmm0, xmm2 // use xmm0 as the temp register. - punpckldq xmm2, xmm6 - movlpd qword ptr [edx], xmm2 - movhpd qword ptr [ebx], xmm2 - punpckhdq xmm0, xmm6 - movlpd qword ptr [edx + esi], xmm0 - lea edx, [edx + 2 * esi] - movhpd qword ptr [ebx + ebp], xmm0 - lea ebx, [ebx + 2 * ebp] - movdqa xmm0, xmm1 // use xmm0 as the temp register. - punpckldq xmm1, xmm5 - movlpd qword ptr [edx], xmm1 - movhpd qword ptr [ebx], xmm1 - punpckhdq xmm0, xmm5 - movlpd qword ptr [edx + esi], xmm0 - lea edx, [edx + 2 * esi] - movhpd qword ptr [ebx + ebp], xmm0 - lea ebx, [ebx + 2 * ebp] - movdqa xmm0, xmm3 // use xmm0 as the temp register. - punpckldq xmm3, xmm7 - movlpd qword ptr [edx], xmm3 - movhpd qword ptr [ebx], xmm3 - punpckhdq xmm0, xmm7 - sub ecx, 8 - movlpd qword ptr [edx + esi], xmm0 - lea edx, [edx + 2 * esi] - movhpd qword ptr [ebx + ebp], xmm0 - lea ebx, [ebx + 2 * ebp] - jg convertloop +#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2) +// Load row r into lane 0 and row r + 8 into lane 1. +#define LOAD2ROWS_AVX2(p, s8) \ + _mm256_inserti128_si256( \ + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(p))), \ + _mm_loadu_si128((const __m128i*)((p) + (s8))), 1) - mov esp, [esp + 16] - pop ebp - pop edi - pop esi - pop ebx - ret +// Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8 +// of register r % 8. 3 unpack stages leave each lane with 2 columns x 8 rows, +// and vpermq joins the rows 0..7 and 8..15 halves. Lane 0 of each output is an +// even column for dst_a and lane 1 an odd column for dst_b. +// Width is a multiple of 16 bytes. +LIBYUV_TARGET_AVX2 +static void TransposeWx16_Byte_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int byte_width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t sa = dst_stride_a; + const ptrdiff_t sb = dst_stride_b; + while (byte_width > 0) { + __m256i r0 = LOAD2ROWS_AVX2(src, s * 8); + __m256i r1 = LOAD2ROWS_AVX2(src + s, s * 8); + __m256i r2 = LOAD2ROWS_AVX2(src + s * 2, s * 8); + __m256i r3 = LOAD2ROWS_AVX2(src + s * 3, s * 8); + __m256i r4 = LOAD2ROWS_AVX2(src + s * 4, s * 8); + __m256i r5 = LOAD2ROWS_AVX2(src + s * 5, s * 8); + __m256i r6 = LOAD2ROWS_AVX2(src + s * 6, s * 8); + __m256i r7 = LOAD2ROWS_AVX2(src + s * 7, s * 8); + + __m256i a0 = _mm256_unpacklo_epi8(r0, r1); + __m256i a1 = _mm256_unpackhi_epi8(r0, r1); + __m256i a2 = _mm256_unpacklo_epi8(r2, r3); + __m256i a3 = _mm256_unpackhi_epi8(r2, r3); + __m256i a4 = _mm256_unpacklo_epi8(r4, r5); + __m256i a5 = _mm256_unpackhi_epi8(r4, r5); + __m256i a6 = _mm256_unpacklo_epi8(r6, r7); + __m256i a7 = _mm256_unpackhi_epi8(r6, r7); + + __m256i b0 = _mm256_unpacklo_epi16(a0, a2); + __m256i b1 = _mm256_unpackhi_epi16(a0, a2); + __m256i b2 = _mm256_unpacklo_epi16(a1, a3); + __m256i b3 = _mm256_unpackhi_epi16(a1, a3); + __m256i b4 = _mm256_unpacklo_epi16(a4, a6); + __m256i b5 = _mm256_unpackhi_epi16(a4, a6); + __m256i b6 = _mm256_unpacklo_epi16(a5, a7); + __m256i b7 = _mm256_unpackhi_epi16(a5, a7); + + __m256i c0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b0, b4), 0xd8); + __m256i c1 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b0, b4), 0xd8); + __m256i c2 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b1, b5), 0xd8); + __m256i c3 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b1, b5), 0xd8); + __m256i c4 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b2, b6), 0xd8); + __m256i c5 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b2, b6), 0xd8); + __m256i c6 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b3, b7), 0xd8); + __m256i c7 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b3, b7), 0xd8); + + _mm_storeu_si128((__m128i*)(dst_a), _mm256_castsi256_si128(c0)); + _mm_storeu_si128((__m128i*)(dst_b), _mm256_extracti128_si256(c0, 1)); + _mm_storeu_si128((__m128i*)(dst_a + sa), _mm256_castsi256_si128(c1)); + _mm_storeu_si128((__m128i*)(dst_b + sb), _mm256_extracti128_si256(c1, 1)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 2), _mm256_castsi256_si128(c2)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 2), + _mm256_extracti128_si256(c2, 1)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 3), _mm256_castsi256_si128(c3)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 3), + _mm256_extracti128_si256(c3, 1)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 4), _mm256_castsi256_si128(c4)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 4), + _mm256_extracti128_si256(c4, 1)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 5), _mm256_castsi256_si128(c5)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 5), + _mm256_extracti128_si256(c5, 1)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 6), _mm256_castsi256_si128(c6)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 6), + _mm256_extracti128_si256(c6, 1)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 7), _mm256_castsi256_si128(c7)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 7), + _mm256_extracti128_si256(c7, 1)); + + src += 16; + dst_a += sa * 8; + dst_b += sb * 8; + byte_width -= 16; } + _mm256_zeroupper(); +} +#undef LOAD2ROWS_AVX2 +#endif // defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2) + +#if defined(HAS_TRANSPOSEWX16_AVX2) +LIBYUV_TARGET_AVX2 +void TransposeWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx16_Byte_AVX2(src, src_stride, dst, dst_stride * 2, + dst + dst_stride, dst_stride * 2, width); +} +#endif // defined(HAS_TRANSPOSEWX16_AVX2) + +#if defined(HAS_TRANSPOSEUVWX16_AVX2) +LIBYUV_TARGET_AVX2 +void TransposeUVWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeWx16_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b, + dst_stride_b, width * 2); +} +#endif // defined(HAS_TRANSPOSEUVWX16_AVX2) + +#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW) +// Dword permutes to gather 16 rows of a column from 4 lanes. +static const uint32_t kPermdTranspose_AVX512BW[32] = { + 0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30, + 1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31}; + +// Load 16 masked bytes from rows p, p + s4, p + s4 * 2, p + s4 * 3 into +// lanes 0..3. +#define LOAD4ROWS_AVX512BW(p, s4, k) \ + _mm512_inserti32x4( \ + _mm512_inserti32x4( \ + _mm512_inserti32x4( \ + _mm512_castsi128_si512(_mm_maskz_loadu_epi8(k, (p))), \ + _mm_maskz_loadu_epi8(k, (p) + (s4)), 1), \ + _mm_maskz_loadu_epi8(k, (p) + (s4) * 2), 2), \ + _mm_maskz_loadu_epi8(k, (p) + (s4) * 3), 3) + +// Store 16 destination rows. Lane n of o0/o2 is for dst_a and lane n of +// o1/o3 is for dst_b. +LIBYUV_TARGET_AVX512BW +static void Store16x16_AVX512BW(__m512i o0, + __m512i o1, + __m512i o2, + __m512i o3, + uint8_t* dst_a, + ptrdiff_t sa, + uint8_t* dst_b, + ptrdiff_t sb) { + _mm_storeu_si128((__m128i*)(dst_a), _mm512_castsi512_si128(o0)); + _mm_storeu_si128((__m128i*)(dst_b), _mm512_castsi512_si128(o1)); + _mm_storeu_si128((__m128i*)(dst_a + sa), _mm512_extracti32x4_epi32(o0, 1)); + _mm_storeu_si128((__m128i*)(dst_b + sb), _mm512_extracti32x4_epi32(o1, 1)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 2), _mm512_castsi512_si128(o2)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 2), _mm512_castsi512_si128(o3)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 3), + _mm512_extracti32x4_epi32(o2, 1)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 3), + _mm512_extracti32x4_epi32(o3, 1)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 4), + _mm512_extracti32x4_epi32(o0, 2)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 4), + _mm512_extracti32x4_epi32(o1, 2)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 5), + _mm512_extracti32x4_epi32(o0, 3)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 5), + _mm512_extracti32x4_epi32(o1, 3)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 6), + _mm512_extracti32x4_epi32(o2, 2)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 6), + _mm512_extracti32x4_epi32(o3, 2)); + _mm_storeu_si128((__m128i*)(dst_a + sa * 7), + _mm512_extracti32x4_epi32(o2, 3)); + _mm_storeu_si128((__m128i*)(dst_b + sb * 7), + _mm512_extracti32x4_epi32(o3, 3)); } -#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) +// Transpose 16x16 bytes using 4 lanes of AVX512BW with tail masking. +// Source row r is in lane r / 4 of register r % 4. +// Can be used for 8-bit planar transpose (dst_a = even rows, dst_b = odd rows) +// or 16-bit UV split transpose (dst_a = U plane, dst_b = V plane). +// TODO(fbarchard): Consider removing in favor of TransposeWx16_Byte_AVX2, +// which is within 10% of AVX512BW on most CPUs. +LIBYUV_TARGET_AVX512BW +static void TransposeWx16_Byte_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int byte_width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t sa = dst_stride_a; + const ptrdiff_t sb = dst_stride_b; + const __m512i perm0 = _mm512_loadu_si512(kPermdTranspose_AVX512BW); + const __m512i perm1 = _mm512_loadu_si512(kPermdTranspose_AVX512BW + 16); + while (byte_width > 0) { + const int n = byte_width < 16 ? byte_width : 16; + const __mmask16 k = (__mmask16)((1u << n) - 1); + __m512i z0 = LOAD4ROWS_AVX512BW(src, s * 4, k); + __m512i z1 = LOAD4ROWS_AVX512BW(src + s, s * 4, k); + __m512i z2 = LOAD4ROWS_AVX512BW(src + s * 2, s * 4, k); + __m512i z3 = LOAD4ROWS_AVX512BW(src + s * 3, s * 4, k); + + __m512i a0 = _mm512_unpacklo_epi8(z0, z1); + __m512i a1 = _mm512_unpackhi_epi8(z0, z1); + __m512i a2 = _mm512_unpacklo_epi8(z2, z3); + __m512i a3 = _mm512_unpackhi_epi8(z2, z3); + __m512i b0 = _mm512_unpacklo_epi16(a0, a2); + __m512i b1 = _mm512_unpackhi_epi16(a0, a2); + __m512i b2 = _mm512_unpacklo_epi16(a1, a3); + __m512i b3 = _mm512_unpackhi_epi16(a1, a3); + __m512i o0 = _mm512_permutex2var_epi32(b0, perm0, b2); + __m512i o1 = _mm512_permutex2var_epi32(b0, perm1, b2); + __m512i o2 = _mm512_permutex2var_epi32(b1, perm0, b3); + __m512i o3 = _mm512_permutex2var_epi32(b1, perm1, b3); + + if (n == 16) { + Store16x16_AVX512BW(o0, o1, o2, o3, dst_a, sa, dst_b, sb); + dst_a += sa * 8; + dst_b += sb * 8; + } else { + // Remainder: 1 to 15 bytes. Store 1 destination row per byte. + SIMD_ALIGNED(uint8_t temp[16 * 16]); + int i; + Store16x16_AVX512BW(o0, o1, o2, o3, temp, 32, temp + 16, 32); + for (i = 0; i < n; ++i) { + uint8_t* dst = (i & 1) ? dst_b + sb * (i >> 1) : dst_a + sa * (i >> 1); + _mm_storeu_si128((__m128i*)dst, + _mm_load_si128((const __m128i*)(temp + i * 16))); + } + } + src += 16; + byte_width -= 16; + } + _mm256_zeroupper(); +} +#undef LOAD4ROWS_AVX512BW +#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) || + // defined(HAS_TRANSPOSEUVWX16_AVX512BW) + +#if defined(HAS_TRANSPOSEWX16_AVX512BW) +LIBYUV_TARGET_AVX512BW +void TransposeWx16_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx16_Byte_AVX512BW(src, src_stride, dst, dst_stride * 2, + dst + (width > 1 ? dst_stride : 0), + dst_stride * 2, width); +} +#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) + +#if defined(HAS_TRANSPOSEUVWX16_AVX512BW) +LIBYUV_TARGET_AVX512BW +void TransposeUVWx16_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeWx16_Byte_AVX512BW(src, src_stride, dst_a, dst_stride_a, dst_b, + dst_stride_b, width * 2); +} +#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW) + +#if defined(HAS_TRANSPOSE4X4_32_SSE2) +// Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row. +LIBYUV_TARGET_SSE2 +void Transpose4x4_32_SSE2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); // a b c d + __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); // e f g h + __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); // i j k l + __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); // m n o p + __m128i t0 = _mm_unpacklo_epi32(r0, r1); // a e b f + __m128i t1 = _mm_unpacklo_epi32(r2, r3); // i m j n + __m128i t2 = _mm_unpackhi_epi32(r0, r1); // c g d h + __m128i t3 = _mm_unpackhi_epi32(r2, r3); // k o l p + _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); // a e i m + _mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3)); + _mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3)); + src += s * 4; + dst += 16; + width -= 4; + } +} +#endif // defined(HAS_TRANSPOSE4X4_32_SSE2) + +#if defined(HAS_TRANSPOSE4X4_32_AVX2) +// Transpose 32 bit values (ARGB). Read a column of 8 pixels, write a row. +LIBYUV_TARGET_AVX2 +void Transpose4x4_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), + _mm_loadu_si128((const __m128i*)(src + s * 4)), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), + _mm_loadu_si128((const __m128i*)(src + s * 5)), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), + _mm_loadu_si128((const __m128i*)(src + s * 6)), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), + _mm_loadu_si128((const __m128i*)(src + s * 7)), 1); + __m256i t0 = _mm256_unpacklo_epi32(r0, r1); // a e b f + __m256i t1 = _mm256_unpacklo_epi32(r2, r3); // i m j n + __m256i t2 = _mm256_unpackhi_epi32(r0, r1); // c g d h + __m256i t3 = _mm256_unpackhi_epi32(r2, r3); // k o l p + _mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1)); + _mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1)); + _mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3)); + _mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3)); + src += s * 8; + dst += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSE4X4_32_AVX2) #ifdef __cplusplus } // extern "C" } // namespace libyuv #endif + +#endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) && + // (Visual C or LIBYUV_ENABLE_ROWWIN)