mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
Rotate intrinsics for Visual C, ROWWIN and 32 bit x86
- Port rotate_gcc.cc to intrinsics in rotate_win.cc: TransposeWx8_SSSE3,
TransposeUVWx8_SSE2, TransposeWx16_AVX2, TransposeUVWx16_AVX2,
Transpose4x4_32_SSE2, Transpose4x4_32_AVX2, and for 64 bit,
TransposeWx16_AVX512BW and TransposeUVWx16_AVX512BW.
- Remove the 32 bit Visual C inline assembly.
- Enable rotate_win.cc for Visual C 32/64 bit, LIBYUV_ENABLE_ROWWIN,
which MemorySanitizer also enables, and all 32 bit x86 compilers.
Visual C 64 bit previously used C.
- rotate_gcc.cc is now 64 bit only. 32 bit gcc and clang only had an
8x8 TransposeWx8_SSSE3 and Transpose4x4_32, because the UV and AVX2
inline assembly uses xmm8..xmm15, so 32 bit NV12 rotate used C.
Remove the 32 bit TransposeWx8_SSSE3 inline assembly.
- Add a target("sse2") attribute to the SSE2 intrinsics so gcc and
clang build them when the baseline lacks SSE2, such as
-march=pentium3.
- Disable the generic x86 scale_row.h and compare_row.h HAS_ macros for
32 bit LIBYUV_ENABLE_ROWWIN, as is done for 64 bit. scale_gcc.cc and
compare_gcc.cc are disabled for ROWWIN and scale_win.cc and
compare_win.cc require _MSC_VER, so out/rowwin_x86 failed to link with
undefined symbols such as ScaleARGBCols_SSE2 and HashDjb2_SSE41.
- TransposeWx8_SSSE3 and TransposeUVWx8_SSE2 share a 16x8 helper.
- TransposeWx8_SSSE3 requires a multiple of 16 on all platforms.
- Byte transposes use a shared helper with dst_a / dst_b outputs so
2, 3 and 4 channel variants can reuse the same shuffles.
Functions available:
Function rotate_gcc rotate_win rotate_win
64 bit 64 bit 32 bit
TransposeWx8_SSSE3 asm intrinsics intrinsics
TransposeUVWx8_SSE2 asm intrinsics intrinsics
TransposeWx16_AVX2 asm intrinsics intrinsics
TransposeUVWx16_AVX2 asm intrinsics intrinsics
Transpose4x4_32_SSE2 asm intrinsics intrinsics
Transpose4x4_32_AVX2 asm intrinsics intrinsics
TransposeWx16_AVX512BW asm intrinsics -
TransposeUVWx16_AVX512BW asm intrinsics -
32 bit x86 Linux clang on AMD Milan (Zen 3), ms
(LIBYUV_WIDTH=1280 LIBYUV_HEIGHT=720 LIBYUV_REPEAT=2000):
C Before After SSE2 After AVX2
I420Rotate90_Opt 944 321 322 220
NV12Rotate90_Opt 1412 935 315 232
Testing:
for w in 1 2 3 7 8 9 15 16 17 23 24 31 32 33 64 65 1279 1280; do
for h in 1 7 8 15 16 17 24 31 32 33 719; do
LIBYUV_WIDTH=$w LIBYUV_HEIGHT=$h out/rowwin/libyuv_unittest \
--gtest_filter='*Rotate*:*Transpose*'
done
done
Repeated with LIBYUV_DISABLE_AVX2=1, LIBYUV_DISABLE_AVX512BW=1, with
out/Release and out/Release_x86, under sde64 -skx for AVX512BW, and
with out/msan (is_msan = true) and out/rowwin_x86
(target_cpu = "x86", libyuv_enable_rowwin = true). Built out/android_x86
(target_os = "android", target_cpu = "x86"). Compiled rotate_win.cc
with g++ and clang++ -m32 -march=pentium3.
Bug: None
Change-Id: Ic70c4f0b7eb46b1b8522a1eba851dac5f506ec3e
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8499394
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard
parent
20b0a91bc1
commit
bad060c3a5
@ -22,9 +22,9 @@ extern "C" {
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
// The following are available for Visual C and GCC:
|
// The following are available for Visual C and GCC:
|
||||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||||
((defined(__x86_64__) && !defined(LIBYUV_ENABLE_ROWWIN)) || \
|
(defined(_M_IX86) || ((defined(__x86_64__) || defined(__i386__)) && \
|
||||||
defined(__i386__) || defined(_M_IX86))
|
!defined(LIBYUV_ENABLE_ROWWIN)))
|
||||||
#define HAS_HASHDJB2_SSE41
|
#define HAS_HASHDJB2_SSE41
|
||||||
#define HAS_SUMSQUAREERROR_SSE2
|
#define HAS_SUMSQUAREERROR_SSE2
|
||||||
#define HAS_HAMMINGDISTANCE_SSE42
|
#define HAS_HAMMINGDISTANCE_SSE42
|
||||||
|
|||||||
@ -21,27 +21,34 @@ namespace libyuv {
|
|||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// The following are available for Visual C 32 bit:
|
// The following are available for 32 bit x86, Visual C 64 bit, and for clang
|
||||||
// TODO - port to clangcl on rotate_win
|
// with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These
|
||||||
#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER) && \
|
// are intrinsics in rotate_win.cc.
|
||||||
!defined(__clang__)
|
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||||
|
(defined(__i386__) || defined(_M_IX86) || \
|
||||||
|
((defined(__x86_64__) || defined(_M_X64)) && \
|
||||||
|
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||||
|
defined(LIBYUV_ENABLE_ROWWIN))))
|
||||||
#define HAS_TRANSPOSEWX8_SSSE3
|
#define HAS_TRANSPOSEWX8_SSSE3
|
||||||
#define HAS_TRANSPOSEUVWX8_SSE2
|
#define HAS_TRANSPOSEUVWX8_SSE2
|
||||||
#endif
|
|
||||||
|
|
||||||
// The following are available for GCC 32 or 64 bit:
|
|
||||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
|
||||||
(defined(__i386__) || defined(__x86_64__)) && \
|
|
||||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
|
||||||
#define HAS_TRANSPOSEWX8_SSSE3
|
|
||||||
#define HAS_TRANSPOSE4X4_32_SSE2
|
#define HAS_TRANSPOSE4X4_32_SSE2
|
||||||
#define HAS_TRANSPOSE4X4_32_AVX2
|
#define HAS_TRANSPOSE4X4_32_AVX2
|
||||||
|
#define HAS_TRANSPOSEWX16_AVX2
|
||||||
|
#define HAS_TRANSPOSEUVWX16_AVX2
|
||||||
|
#if defined(__x86_64__) || defined(_M_X64)
|
||||||
|
#define HAS_TRANSPOSEWX16_AVX512BW
|
||||||
|
#define HAS_TRANSPOSEUVWX16_AVX512BW
|
||||||
|
#endif
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// The following are available for 64 bit GCC:
|
// The following are available for 64 bit GCC and clang, as inline assembly in
|
||||||
|
// rotate_gcc.cc:
|
||||||
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
||||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||||
|
#define HAS_TRANSPOSEWX8_SSSE3
|
||||||
#define HAS_TRANSPOSEUVWX8_SSE2
|
#define HAS_TRANSPOSEUVWX8_SSE2
|
||||||
|
#define HAS_TRANSPOSE4X4_32_SSE2
|
||||||
|
#define HAS_TRANSPOSE4X4_32_AVX2
|
||||||
#define HAS_TRANSPOSEWX16_AVX2
|
#define HAS_TRANSPOSEWX16_AVX2
|
||||||
#define HAS_TRANSPOSEUVWX16_AVX2
|
#define HAS_TRANSPOSEUVWX16_AVX2
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@ -23,10 +23,9 @@ extern "C" {
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
// The following are available on all x86 platforms:
|
// The following are available on all x86 platforms:
|
||||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||||
(defined(_M_IX86) || \
|
(defined(_M_IX86) || ((defined(__x86_64__) || defined(__i386__)) && \
|
||||||
(defined(__x86_64__) && !defined(LIBYUV_ENABLE_ROWWIN)) || \
|
!defined(LIBYUV_ENABLE_ROWWIN)))
|
||||||
defined(__i386__))
|
|
||||||
#define HAS_FIXEDDIV1_X86
|
#define HAS_FIXEDDIV1_X86
|
||||||
#define HAS_FIXEDDIV_X86
|
#define HAS_FIXEDDIV_X86
|
||||||
#define HAS_SCALEADDROW_SSE2
|
#define HAS_SCALEADDROW_SSE2
|
||||||
|
|||||||
@ -72,15 +72,9 @@ void TransposePlane(const uint8_t* src,
|
|||||||
#if defined(HAS_TRANSPOSEWX8_SSSE3)
|
#if defined(HAS_TRANSPOSEWX8_SSSE3)
|
||||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||||
TransposeWx8 = TransposeWx8_Any_SSSE3;
|
TransposeWx8 = TransposeWx8_Any_SSSE3;
|
||||||
#if defined(__x86_64__)
|
|
||||||
if (IS_ALIGNED(width, 16)) {
|
if (IS_ALIGNED(width, 16)) {
|
||||||
TransposeWx8 = TransposeWx8_SSSE3;
|
TransposeWx8 = TransposeWx8_SSSE3;
|
||||||
}
|
}
|
||||||
#else
|
|
||||||
if (IS_ALIGNED(width, 8)) {
|
|
||||||
TransposeWx8 = TransposeWx8_SSSE3;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
#if defined(HAS_TRANSPOSEWX16_AVX2)
|
#if defined(HAS_TRANSPOSEWX16_AVX2)
|
||||||
|
|||||||
@ -35,11 +35,7 @@ TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7)
|
|||||||
TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15)
|
TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15)
|
||||||
#endif
|
#endif
|
||||||
#ifdef HAS_TRANSPOSEWX8_SSSE3
|
#ifdef HAS_TRANSPOSEWX8_SSSE3
|
||||||
#if defined(__x86_64__)
|
|
||||||
TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15)
|
TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15)
|
||||||
#else
|
|
||||||
TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 7)
|
|
||||||
#endif
|
|
||||||
#endif
|
#endif
|
||||||
#ifdef HAS_TRANSPOSEWX16_AVX2
|
#ifdef HAS_TRANSPOSEWX16_AVX2
|
||||||
TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15)
|
TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15)
|
||||||
|
|||||||
@ -16,13 +16,11 @@ namespace libyuv {
|
|||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// This module is for GCC x86 and x64.
|
// This module is for GCC and clang x64. 32 bit x86 uses rotate_win.cc.
|
||||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
||||||
(defined(__x86_64__) || defined(__i386__)) && \
|
|
||||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||||
|
|
||||||
#if defined(HAS_TRANSPOSEWX8_SSSE3)
|
#if defined(HAS_TRANSPOSEWX8_SSSE3)
|
||||||
#if defined(__x86_64__)
|
|
||||||
// Transpose 16x8. 64 bit.
|
// Transpose 16x8. 64 bit.
|
||||||
void TransposeWx8_SSSE3(const uint8_t* src,
|
void TransposeWx8_SSSE3(const uint8_t* src,
|
||||||
int src_stride,
|
int src_stride,
|
||||||
@ -163,95 +161,6 @@ void TransposeWx8_SSSE3(const uint8_t* src,
|
|||||||
"xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
|
"xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
|
||||||
"xmm15");
|
"xmm15");
|
||||||
}
|
}
|
||||||
#else // defined(__x86_64__)
|
|
||||||
// Transpose 8x8. 32 bit.
|
|
||||||
void TransposeWx8_SSSE3(const uint8_t* src,
|
|
||||||
int src_stride,
|
|
||||||
uint8_t* dst,
|
|
||||||
int dst_stride,
|
|
||||||
int width) {
|
|
||||||
asm volatile(
|
|
||||||
// Read in the data from the source pointer.
|
|
||||||
// First round of bit swap.
|
|
||||||
LABELALIGN
|
|
||||||
"1: \n"
|
|
||||||
"movq (%0),%%xmm0 \n"
|
|
||||||
"movq (%0,%3),%%xmm1 \n"
|
|
||||||
"lea (%0,%3,2),%0 \n"
|
|
||||||
"punpcklbw %%xmm1,%%xmm0 \n"
|
|
||||||
"movq (%0),%%xmm2 \n"
|
|
||||||
"movdqa %%xmm0,%%xmm1 \n"
|
|
||||||
"palignr $0x8,%%xmm1,%%xmm1 \n"
|
|
||||||
"movq (%0,%3),%%xmm3 \n"
|
|
||||||
"lea (%0,%3,2),%0 \n"
|
|
||||||
"punpcklbw %%xmm3,%%xmm2 \n"
|
|
||||||
"movdqa %%xmm2,%%xmm3 \n"
|
|
||||||
"movq (%0),%%xmm4 \n"
|
|
||||||
"palignr $0x8,%%xmm3,%%xmm3 \n"
|
|
||||||
"movq (%0,%3),%%xmm5 \n"
|
|
||||||
"lea (%0,%3,2),%0 \n"
|
|
||||||
"punpcklbw %%xmm5,%%xmm4 \n"
|
|
||||||
"movdqa %%xmm4,%%xmm5 \n"
|
|
||||||
"movq (%0),%%xmm6 \n"
|
|
||||||
"palignr $0x8,%%xmm5,%%xmm5 \n"
|
|
||||||
"movq (%0,%3),%%xmm7 \n"
|
|
||||||
"lea (%0,%3,2),%0 \n"
|
|
||||||
"punpcklbw %%xmm7,%%xmm6 \n"
|
|
||||||
"neg %3 \n"
|
|
||||||
"movdqa %%xmm6,%%xmm7 \n"
|
|
||||||
"lea 0x8(%0,%3,8),%0 \n"
|
|
||||||
"palignr $0x8,%%xmm7,%%xmm7 \n"
|
|
||||||
"neg %3 \n"
|
|
||||||
// Second round of bit swap.
|
|
||||||
"punpcklwd %%xmm2,%%xmm0 \n"
|
|
||||||
"punpcklwd %%xmm3,%%xmm1 \n"
|
|
||||||
"movdqa %%xmm0,%%xmm2 \n"
|
|
||||||
"movdqa %%xmm1,%%xmm3 \n"
|
|
||||||
"palignr $0x8,%%xmm2,%%xmm2 \n"
|
|
||||||
"palignr $0x8,%%xmm3,%%xmm3 \n"
|
|
||||||
"punpcklwd %%xmm6,%%xmm4 \n"
|
|
||||||
"punpcklwd %%xmm7,%%xmm5 \n"
|
|
||||||
"movdqa %%xmm4,%%xmm6 \n"
|
|
||||||
"movdqa %%xmm5,%%xmm7 \n"
|
|
||||||
"palignr $0x8,%%xmm6,%%xmm6 \n"
|
|
||||||
"palignr $0x8,%%xmm7,%%xmm7 \n"
|
|
||||||
// Third round of bit swap.
|
|
||||||
// Write to the destination pointer.
|
|
||||||
"punpckldq %%xmm4,%%xmm0 \n"
|
|
||||||
"movq %%xmm0,(%1) \n"
|
|
||||||
"movdqa %%xmm0,%%xmm4 \n"
|
|
||||||
"palignr $0x8,%%xmm4,%%xmm4 \n"
|
|
||||||
"movq %%xmm4,(%1,%4) \n"
|
|
||||||
"lea (%1,%4,2),%1 \n"
|
|
||||||
"punpckldq %%xmm6,%%xmm2 \n"
|
|
||||||
"movdqa %%xmm2,%%xmm6 \n"
|
|
||||||
"movq %%xmm2,(%1) \n"
|
|
||||||
"palignr $0x8,%%xmm6,%%xmm6 \n"
|
|
||||||
"punpckldq %%xmm5,%%xmm1 \n"
|
|
||||||
"movq %%xmm6,(%1,%4) \n"
|
|
||||||
"lea (%1,%4,2),%1 \n"
|
|
||||||
"movdqa %%xmm1,%%xmm5 \n"
|
|
||||||
"movq %%xmm1,(%1) \n"
|
|
||||||
"palignr $0x8,%%xmm5,%%xmm5 \n"
|
|
||||||
"movq %%xmm5,(%1,%4) \n"
|
|
||||||
"lea (%1,%4,2),%1 \n"
|
|
||||||
"punpckldq %%xmm7,%%xmm3 \n"
|
|
||||||
"movq %%xmm3,(%1) \n"
|
|
||||||
"movdqa %%xmm3,%%xmm7 \n"
|
|
||||||
"palignr $0x8,%%xmm7,%%xmm7 \n"
|
|
||||||
"sub $0x8,%2 \n"
|
|
||||||
"movq %%xmm7,(%1,%4) \n"
|
|
||||||
"lea (%1,%4,2),%1 \n"
|
|
||||||
"jg 1b \n"
|
|
||||||
: "+r"(src), // %0
|
|
||||||
"+r"(dst), // %1
|
|
||||||
"+r"(width) // %2
|
|
||||||
: "r"((ptrdiff_t)(src_stride)), // %3
|
|
||||||
"r"((ptrdiff_t)(dst_stride)) // %4
|
|
||||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
|
||||||
"xmm7");
|
|
||||||
}
|
|
||||||
#endif // defined(__x86_64__)
|
|
||||||
#endif // defined(HAS_TRANSPOSEWX8_SSSE3)
|
#endif // defined(HAS_TRANSPOSEWX8_SSSE3)
|
||||||
|
|
||||||
#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
||||||
@ -877,7 +786,8 @@ void Transpose4x4_32_AVX2(const uint8_t* src,
|
|||||||
}
|
}
|
||||||
#endif // defined(HAS_TRANSPOSE4X4_32_AVX2)
|
#endif // defined(HAS_TRANSPOSE4X4_32_AVX2)
|
||||||
|
|
||||||
#endif // defined(__x86_64__) || defined(__i386__)
|
#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) &&
|
||||||
|
// !defined(LIBYUV_ENABLE_ROWWIN)
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
} // extern "C"
|
} // extern "C"
|
||||||
|
|||||||
@ -11,243 +11,485 @@
|
|||||||
#include "libyuv/rotate_row.h"
|
#include "libyuv/rotate_row.h"
|
||||||
#include "libyuv/row.h"
|
#include "libyuv/row.h"
|
||||||
|
|
||||||
|
// This module is for 32 bit x86 with any compiler, Visual C 64 bit, and for
|
||||||
|
// clang with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
|
||||||
|
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||||
|
(defined(__i386__) || defined(_M_IX86) || \
|
||||||
|
((defined(__x86_64__) || defined(_M_X64)) && \
|
||||||
|
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||||
|
defined(LIBYUV_ENABLE_ROWWIN))))
|
||||||
|
|
||||||
|
#include <emmintrin.h>
|
||||||
|
#include <immintrin.h> // For AVX2 and AVX512 intrinsics
|
||||||
|
#if defined(_MSC_VER) && !defined(__clang__)
|
||||||
|
#include <intrin.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
namespace libyuv {
|
namespace libyuv {
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// This module is for 32 bit Visual C x86
|
// Target attributes allow gcc and clang to build each function for its
|
||||||
#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) && \
|
// instruction set regardless of the baseline, which may lack SSE2 for 32 bit.
|
||||||
(!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN))
|
#if defined(__clang__) || defined(__GNUC__)
|
||||||
|
#define LIBYUV_TARGET_SSE2 __attribute__((target("sse2")))
|
||||||
|
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
|
||||||
|
#define LIBYUV_TARGET_AVX512BW \
|
||||||
|
__attribute__((target("avx512bw,avx512vl,avx512f")))
|
||||||
|
#else
|
||||||
|
#define LIBYUV_TARGET_SSE2
|
||||||
|
#define LIBYUV_TARGET_AVX2
|
||||||
|
#define LIBYUV_TARGET_AVX512BW
|
||||||
|
#endif // defined(__clang__) || defined(__GNUC__)
|
||||||
|
|
||||||
__declspec(naked) void TransposeWx8_SSSE3(const uint8_t* src,
|
#if defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2)
|
||||||
int src_stride,
|
// Transpose 8 rows x 16 bytes with SSE2. Each output holds 2 columns x 8 rows.
|
||||||
uint8_t* dst,
|
// The low 8 bytes (even column) go to dst_a and the high 8 bytes (odd column)
|
||||||
int dst_stride,
|
// go to dst_b. Width is a multiple of 16 bytes.
|
||||||
int width) {
|
LIBYUV_TARGET_SSE2
|
||||||
__asm {
|
static void TransposeWx8_Byte_SSE2(const uint8_t* src,
|
||||||
push edi
|
int src_stride,
|
||||||
push esi
|
uint8_t* dst_a,
|
||||||
push ebp
|
int dst_stride_a,
|
||||||
mov eax, [esp + 12 + 4] // src
|
uint8_t* dst_b,
|
||||||
mov edi, [esp + 12 + 8] // src_stride
|
int dst_stride_b,
|
||||||
mov edx, [esp + 12 + 12] // dst
|
int byte_width) {
|
||||||
mov esi, [esp + 12 + 16] // dst_stride
|
const ptrdiff_t s = src_stride;
|
||||||
mov ecx, [esp + 12 + 20] // width
|
const ptrdiff_t sa = dst_stride_a;
|
||||||
|
const ptrdiff_t sb = dst_stride_b;
|
||||||
|
while (byte_width > 0) {
|
||||||
|
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
|
||||||
|
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
|
||||||
|
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
|
||||||
|
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
|
||||||
|
__m128i r4 = _mm_loadu_si128((const __m128i*)(src + s * 4));
|
||||||
|
__m128i r5 = _mm_loadu_si128((const __m128i*)(src + s * 5));
|
||||||
|
__m128i r6 = _mm_loadu_si128((const __m128i*)(src + s * 6));
|
||||||
|
__m128i r7 = _mm_loadu_si128((const __m128i*)(src + s * 7));
|
||||||
|
|
||||||
// Read in the data from the source pointer.
|
// Bytes of row pairs. a0 is columns 0..7 of rows 0, 1.
|
||||||
// First round of bit swap.
|
__m128i a0 = _mm_unpacklo_epi8(r0, r1);
|
||||||
align 4
|
__m128i a1 = _mm_unpackhi_epi8(r0, r1);
|
||||||
convertloop:
|
__m128i a2 = _mm_unpacklo_epi8(r2, r3);
|
||||||
movq xmm0, qword ptr [eax]
|
__m128i a3 = _mm_unpackhi_epi8(r2, r3);
|
||||||
lea ebp, [eax + 8]
|
__m128i a4 = _mm_unpacklo_epi8(r4, r5);
|
||||||
movq xmm1, qword ptr [eax + edi]
|
__m128i a5 = _mm_unpackhi_epi8(r4, r5);
|
||||||
lea eax, [eax + 2 * edi]
|
__m128i a6 = _mm_unpacklo_epi8(r6, r7);
|
||||||
punpcklbw xmm0, xmm1
|
__m128i a7 = _mm_unpackhi_epi8(r6, r7);
|
||||||
movq xmm2, qword ptr [eax]
|
|
||||||
movdqa xmm1, xmm0
|
|
||||||
palignr xmm1, xmm1, 8
|
|
||||||
movq xmm3, qword ptr [eax + edi]
|
|
||||||
lea eax, [eax + 2 * edi]
|
|
||||||
punpcklbw xmm2, xmm3
|
|
||||||
movdqa xmm3, xmm2
|
|
||||||
movq xmm4, qword ptr [eax]
|
|
||||||
palignr xmm3, xmm3, 8
|
|
||||||
movq xmm5, qword ptr [eax + edi]
|
|
||||||
punpcklbw xmm4, xmm5
|
|
||||||
lea eax, [eax + 2 * edi]
|
|
||||||
movdqa xmm5, xmm4
|
|
||||||
movq xmm6, qword ptr [eax]
|
|
||||||
palignr xmm5, xmm5, 8
|
|
||||||
movq xmm7, qword ptr [eax + edi]
|
|
||||||
punpcklbw xmm6, xmm7
|
|
||||||
mov eax, ebp
|
|
||||||
movdqa xmm7, xmm6
|
|
||||||
palignr xmm7, xmm7, 8
|
|
||||||
// Second round of bit swap.
|
|
||||||
punpcklwd xmm0, xmm2
|
|
||||||
punpcklwd xmm1, xmm3
|
|
||||||
movdqa xmm2, xmm0
|
|
||||||
movdqa xmm3, xmm1
|
|
||||||
palignr xmm2, xmm2, 8
|
|
||||||
palignr xmm3, xmm3, 8
|
|
||||||
punpcklwd xmm4, xmm6
|
|
||||||
punpcklwd xmm5, xmm7
|
|
||||||
movdqa xmm6, xmm4
|
|
||||||
movdqa xmm7, xmm5
|
|
||||||
palignr xmm6, xmm6, 8
|
|
||||||
palignr xmm7, xmm7, 8
|
|
||||||
// Third round of bit swap.
|
|
||||||
// Write to the destination pointer.
|
|
||||||
punpckldq xmm0, xmm4
|
|
||||||
movq qword ptr [edx], xmm0
|
|
||||||
movdqa xmm4, xmm0
|
|
||||||
palignr xmm4, xmm4, 8
|
|
||||||
movq qword ptr [edx + esi], xmm4
|
|
||||||
lea edx, [edx + 2 * esi]
|
|
||||||
punpckldq xmm2, xmm6
|
|
||||||
movdqa xmm6, xmm2
|
|
||||||
palignr xmm6, xmm6, 8
|
|
||||||
movq qword ptr [edx], xmm2
|
|
||||||
punpckldq xmm1, xmm5
|
|
||||||
movq qword ptr [edx + esi], xmm6
|
|
||||||
lea edx, [edx + 2 * esi]
|
|
||||||
movdqa xmm5, xmm1
|
|
||||||
movq qword ptr [edx], xmm1
|
|
||||||
palignr xmm5, xmm5, 8
|
|
||||||
punpckldq xmm3, xmm7
|
|
||||||
movq qword ptr [edx + esi], xmm5
|
|
||||||
lea edx, [edx + 2 * esi]
|
|
||||||
movq qword ptr [edx], xmm3
|
|
||||||
movdqa xmm7, xmm3
|
|
||||||
palignr xmm7, xmm7, 8
|
|
||||||
sub ecx, 8
|
|
||||||
movq qword ptr [edx + esi], xmm7
|
|
||||||
lea edx, [edx + 2 * esi]
|
|
||||||
jg convertloop
|
|
||||||
|
|
||||||
pop ebp
|
// Words. b0 is columns 0..3 of rows 0..3.
|
||||||
pop esi
|
__m128i b0 = _mm_unpacklo_epi16(a0, a2);
|
||||||
pop edi
|
__m128i b1 = _mm_unpackhi_epi16(a0, a2);
|
||||||
ret
|
__m128i b2 = _mm_unpacklo_epi16(a1, a3);
|
||||||
|
__m128i b3 = _mm_unpackhi_epi16(a1, a3);
|
||||||
|
__m128i b4 = _mm_unpacklo_epi16(a4, a6);
|
||||||
|
__m128i b5 = _mm_unpackhi_epi16(a4, a6);
|
||||||
|
__m128i b6 = _mm_unpacklo_epi16(a5, a7);
|
||||||
|
__m128i b7 = _mm_unpackhi_epi16(a5, a7);
|
||||||
|
|
||||||
|
// Dwords. c0 is columns 0, 1 of rows 0..7.
|
||||||
|
__m128i c0 = _mm_unpacklo_epi32(b0, b4);
|
||||||
|
__m128i c1 = _mm_unpackhi_epi32(b0, b4);
|
||||||
|
__m128i c2 = _mm_unpacklo_epi32(b1, b5);
|
||||||
|
__m128i c3 = _mm_unpackhi_epi32(b1, b5);
|
||||||
|
__m128i c4 = _mm_unpacklo_epi32(b2, b6);
|
||||||
|
__m128i c5 = _mm_unpackhi_epi32(b2, b6);
|
||||||
|
__m128i c6 = _mm_unpacklo_epi32(b3, b7);
|
||||||
|
__m128i c7 = _mm_unpackhi_epi32(b3, b7);
|
||||||
|
|
||||||
|
_mm_storel_epi64((__m128i*)(dst_a), c0);
|
||||||
|
_mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(c0));
|
||||||
|
_mm_storel_epi64((__m128i*)(dst_a + sa), c1);
|
||||||
|
_mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(c1));
|
||||||
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 2), c2);
|
||||||
|
_mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(c2));
|
||||||
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 3), c3);
|
||||||
|
_mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(c3));
|
||||||
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 4), c4);
|
||||||
|
_mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(c4));
|
||||||
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 5), c5);
|
||||||
|
_mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(c5));
|
||||||
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 6), c6);
|
||||||
|
_mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(c6));
|
||||||
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 7), c7);
|
||||||
|
_mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(c7));
|
||||||
|
|
||||||
|
src += 16;
|
||||||
|
dst_a += sa * 8;
|
||||||
|
dst_b += sb * 8;
|
||||||
|
byte_width -= 16;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#endif // defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2)
|
||||||
|
|
||||||
__declspec(naked) void TransposeUVWx8_SSE2(const uint8_t* src,
|
#if defined(HAS_TRANSPOSEWX8_SSSE3)
|
||||||
int src_stride,
|
// Transpose 16x8. Width is a multiple of 16.
|
||||||
uint8_t* dst_a,
|
LIBYUV_TARGET_SSE2
|
||||||
int dst_stride_a,
|
void TransposeWx8_SSSE3(const uint8_t* src,
|
||||||
uint8_t* dst_b,
|
int src_stride,
|
||||||
int dst_stride_b,
|
uint8_t* dst,
|
||||||
int w) {
|
int dst_stride,
|
||||||
__asm {
|
int width) {
|
||||||
push ebx
|
TransposeWx8_Byte_SSE2(src, src_stride, dst, dst_stride * 2, dst + dst_stride,
|
||||||
push esi
|
dst_stride * 2, width);
|
||||||
push edi
|
}
|
||||||
push ebp
|
#endif // defined(HAS_TRANSPOSEWX8_SSSE3)
|
||||||
mov eax, [esp + 16 + 4] // src
|
|
||||||
mov edi, [esp + 16 + 8] // src_stride
|
|
||||||
mov edx, [esp + 16 + 12] // dst_a
|
|
||||||
mov esi, [esp + 16 + 16] // dst_stride_a
|
|
||||||
mov ebx, [esp + 16 + 20] // dst_b
|
|
||||||
mov ebp, [esp + 16 + 24] // dst_stride_b
|
|
||||||
mov ecx, esp
|
|
||||||
sub esp, 4 + 16
|
|
||||||
and esp, ~15
|
|
||||||
mov [esp + 16], ecx
|
|
||||||
mov ecx, [ecx + 16 + 28] // w
|
|
||||||
|
|
||||||
align 4
|
#if defined(HAS_TRANSPOSEUVWX8_SSE2)
|
||||||
// Read in the data from the source pointer.
|
// Transpose UV 8x8. Width is a multiple of 8.
|
||||||
// First round of bit swap.
|
LIBYUV_TARGET_SSE2
|
||||||
convertloop:
|
void TransposeUVWx8_SSE2(const uint8_t* src,
|
||||||
movdqu xmm0, [eax]
|
int src_stride,
|
||||||
movdqu xmm1, [eax + edi]
|
uint8_t* dst_a,
|
||||||
lea eax, [eax + 2 * edi]
|
int dst_stride_a,
|
||||||
movdqa xmm7, xmm0 // use xmm7 as temp register.
|
uint8_t* dst_b,
|
||||||
punpcklbw xmm0, xmm1
|
int dst_stride_b,
|
||||||
punpckhbw xmm7, xmm1
|
int width) {
|
||||||
movdqa xmm1, xmm7
|
TransposeWx8_Byte_SSE2(src, src_stride, dst_a, dst_stride_a, dst_b,
|
||||||
movdqu xmm2, [eax]
|
dst_stride_b, width * 2);
|
||||||
movdqu xmm3, [eax + edi]
|
}
|
||||||
lea eax, [eax + 2 * edi]
|
#endif // defined(HAS_TRANSPOSEUVWX8_SSE2)
|
||||||
movdqa xmm7, xmm2
|
|
||||||
punpcklbw xmm2, xmm3
|
|
||||||
punpckhbw xmm7, xmm3
|
|
||||||
movdqa xmm3, xmm7
|
|
||||||
movdqu xmm4, [eax]
|
|
||||||
movdqu xmm5, [eax + edi]
|
|
||||||
lea eax, [eax + 2 * edi]
|
|
||||||
movdqa xmm7, xmm4
|
|
||||||
punpcklbw xmm4, xmm5
|
|
||||||
punpckhbw xmm7, xmm5
|
|
||||||
movdqa xmm5, xmm7
|
|
||||||
movdqu xmm6, [eax]
|
|
||||||
movdqu xmm7, [eax + edi]
|
|
||||||
lea eax, [eax + 2 * edi]
|
|
||||||
movdqu [esp], xmm5 // backup xmm5
|
|
||||||
neg edi
|
|
||||||
movdqa xmm5, xmm6 // use xmm5 as temp register.
|
|
||||||
punpcklbw xmm6, xmm7
|
|
||||||
punpckhbw xmm5, xmm7
|
|
||||||
movdqa xmm7, xmm5
|
|
||||||
lea eax, [eax + 8 * edi + 16]
|
|
||||||
neg edi
|
|
||||||
// Second round of bit swap.
|
|
||||||
movdqa xmm5, xmm0
|
|
||||||
punpcklwd xmm0, xmm2
|
|
||||||
punpckhwd xmm5, xmm2
|
|
||||||
movdqa xmm2, xmm5
|
|
||||||
movdqa xmm5, xmm1
|
|
||||||
punpcklwd xmm1, xmm3
|
|
||||||
punpckhwd xmm5, xmm3
|
|
||||||
movdqa xmm3, xmm5
|
|
||||||
movdqa xmm5, xmm4
|
|
||||||
punpcklwd xmm4, xmm6
|
|
||||||
punpckhwd xmm5, xmm6
|
|
||||||
movdqa xmm6, xmm5
|
|
||||||
movdqu xmm5, [esp] // restore xmm5
|
|
||||||
movdqu [esp], xmm6 // backup xmm6
|
|
||||||
movdqa xmm6, xmm5 // use xmm6 as temp register.
|
|
||||||
punpcklwd xmm5, xmm7
|
|
||||||
punpckhwd xmm6, xmm7
|
|
||||||
movdqa xmm7, xmm6
|
|
||||||
|
|
||||||
// Third round of bit swap.
|
#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
|
||||||
// Write to the destination pointer.
|
// Load row r into lane 0 and row r + 8 into lane 1.
|
||||||
movdqa xmm6, xmm0
|
#define LOAD2ROWS_AVX2(p, s8) \
|
||||||
punpckldq xmm0, xmm4
|
_mm256_inserti128_si256( \
|
||||||
punpckhdq xmm6, xmm4
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(p))), \
|
||||||
movdqa xmm4, xmm6
|
_mm_loadu_si128((const __m128i*)((p) + (s8))), 1)
|
||||||
movdqu xmm6, [esp] // restore xmm6
|
|
||||||
movlpd qword ptr [edx], xmm0
|
|
||||||
movhpd qword ptr [ebx], xmm0
|
|
||||||
movlpd qword ptr [edx + esi], xmm4
|
|
||||||
lea edx, [edx + 2 * esi]
|
|
||||||
movhpd qword ptr [ebx + ebp], xmm4
|
|
||||||
lea ebx, [ebx + 2 * ebp]
|
|
||||||
movdqa xmm0, xmm2 // use xmm0 as the temp register.
|
|
||||||
punpckldq xmm2, xmm6
|
|
||||||
movlpd qword ptr [edx], xmm2
|
|
||||||
movhpd qword ptr [ebx], xmm2
|
|
||||||
punpckhdq xmm0, xmm6
|
|
||||||
movlpd qword ptr [edx + esi], xmm0
|
|
||||||
lea edx, [edx + 2 * esi]
|
|
||||||
movhpd qword ptr [ebx + ebp], xmm0
|
|
||||||
lea ebx, [ebx + 2 * ebp]
|
|
||||||
movdqa xmm0, xmm1 // use xmm0 as the temp register.
|
|
||||||
punpckldq xmm1, xmm5
|
|
||||||
movlpd qword ptr [edx], xmm1
|
|
||||||
movhpd qword ptr [ebx], xmm1
|
|
||||||
punpckhdq xmm0, xmm5
|
|
||||||
movlpd qword ptr [edx + esi], xmm0
|
|
||||||
lea edx, [edx + 2 * esi]
|
|
||||||
movhpd qword ptr [ebx + ebp], xmm0
|
|
||||||
lea ebx, [ebx + 2 * ebp]
|
|
||||||
movdqa xmm0, xmm3 // use xmm0 as the temp register.
|
|
||||||
punpckldq xmm3, xmm7
|
|
||||||
movlpd qword ptr [edx], xmm3
|
|
||||||
movhpd qword ptr [ebx], xmm3
|
|
||||||
punpckhdq xmm0, xmm7
|
|
||||||
sub ecx, 8
|
|
||||||
movlpd qword ptr [edx + esi], xmm0
|
|
||||||
lea edx, [edx + 2 * esi]
|
|
||||||
movhpd qword ptr [ebx + ebp], xmm0
|
|
||||||
lea ebx, [ebx + 2 * ebp]
|
|
||||||
jg convertloop
|
|
||||||
|
|
||||||
mov esp, [esp + 16]
|
// Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8
|
||||||
pop ebp
|
// of register r % 8. 3 unpack stages leave each lane with 2 columns x 8 rows,
|
||||||
pop edi
|
// and vpermq joins the rows 0..7 and 8..15 halves. Lane 0 of each output is an
|
||||||
pop esi
|
// even column for dst_a and lane 1 an odd column for dst_b.
|
||||||
pop ebx
|
// Width is a multiple of 16 bytes.
|
||||||
ret
|
LIBYUV_TARGET_AVX2
|
||||||
|
static void TransposeWx16_Byte_AVX2(const uint8_t* src,
|
||||||
|
int src_stride,
|
||||||
|
uint8_t* dst_a,
|
||||||
|
int dst_stride_a,
|
||||||
|
uint8_t* dst_b,
|
||||||
|
int dst_stride_b,
|
||||||
|
int byte_width) {
|
||||||
|
const ptrdiff_t s = src_stride;
|
||||||
|
const ptrdiff_t sa = dst_stride_a;
|
||||||
|
const ptrdiff_t sb = dst_stride_b;
|
||||||
|
while (byte_width > 0) {
|
||||||
|
__m256i r0 = LOAD2ROWS_AVX2(src, s * 8);
|
||||||
|
__m256i r1 = LOAD2ROWS_AVX2(src + s, s * 8);
|
||||||
|
__m256i r2 = LOAD2ROWS_AVX2(src + s * 2, s * 8);
|
||||||
|
__m256i r3 = LOAD2ROWS_AVX2(src + s * 3, s * 8);
|
||||||
|
__m256i r4 = LOAD2ROWS_AVX2(src + s * 4, s * 8);
|
||||||
|
__m256i r5 = LOAD2ROWS_AVX2(src + s * 5, s * 8);
|
||||||
|
__m256i r6 = LOAD2ROWS_AVX2(src + s * 6, s * 8);
|
||||||
|
__m256i r7 = LOAD2ROWS_AVX2(src + s * 7, s * 8);
|
||||||
|
|
||||||
|
__m256i a0 = _mm256_unpacklo_epi8(r0, r1);
|
||||||
|
__m256i a1 = _mm256_unpackhi_epi8(r0, r1);
|
||||||
|
__m256i a2 = _mm256_unpacklo_epi8(r2, r3);
|
||||||
|
__m256i a3 = _mm256_unpackhi_epi8(r2, r3);
|
||||||
|
__m256i a4 = _mm256_unpacklo_epi8(r4, r5);
|
||||||
|
__m256i a5 = _mm256_unpackhi_epi8(r4, r5);
|
||||||
|
__m256i a6 = _mm256_unpacklo_epi8(r6, r7);
|
||||||
|
__m256i a7 = _mm256_unpackhi_epi8(r6, r7);
|
||||||
|
|
||||||
|
__m256i b0 = _mm256_unpacklo_epi16(a0, a2);
|
||||||
|
__m256i b1 = _mm256_unpackhi_epi16(a0, a2);
|
||||||
|
__m256i b2 = _mm256_unpacklo_epi16(a1, a3);
|
||||||
|
__m256i b3 = _mm256_unpackhi_epi16(a1, a3);
|
||||||
|
__m256i b4 = _mm256_unpacklo_epi16(a4, a6);
|
||||||
|
__m256i b5 = _mm256_unpackhi_epi16(a4, a6);
|
||||||
|
__m256i b6 = _mm256_unpacklo_epi16(a5, a7);
|
||||||
|
__m256i b7 = _mm256_unpackhi_epi16(a5, a7);
|
||||||
|
|
||||||
|
__m256i c0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b0, b4), 0xd8);
|
||||||
|
__m256i c1 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b0, b4), 0xd8);
|
||||||
|
__m256i c2 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b1, b5), 0xd8);
|
||||||
|
__m256i c3 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b1, b5), 0xd8);
|
||||||
|
__m256i c4 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b2, b6), 0xd8);
|
||||||
|
__m256i c5 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b2, b6), 0xd8);
|
||||||
|
__m256i c6 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b3, b7), 0xd8);
|
||||||
|
__m256i c7 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b3, b7), 0xd8);
|
||||||
|
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a), _mm256_castsi256_si128(c0));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b), _mm256_extracti128_si256(c0, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa), _mm256_castsi256_si128(c1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb), _mm256_extracti128_si256(c1, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 2), _mm256_castsi256_si128(c2));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 2),
|
||||||
|
_mm256_extracti128_si256(c2, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 3), _mm256_castsi256_si128(c3));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 3),
|
||||||
|
_mm256_extracti128_si256(c3, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 4), _mm256_castsi256_si128(c4));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 4),
|
||||||
|
_mm256_extracti128_si256(c4, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 5), _mm256_castsi256_si128(c5));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 5),
|
||||||
|
_mm256_extracti128_si256(c5, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 6), _mm256_castsi256_si128(c6));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 6),
|
||||||
|
_mm256_extracti128_si256(c6, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 7), _mm256_castsi256_si128(c7));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 7),
|
||||||
|
_mm256_extracti128_si256(c7, 1));
|
||||||
|
|
||||||
|
src += 16;
|
||||||
|
dst_a += sa * 8;
|
||||||
|
dst_b += sb * 8;
|
||||||
|
byte_width -= 16;
|
||||||
}
|
}
|
||||||
|
_mm256_zeroupper();
|
||||||
|
}
|
||||||
|
#undef LOAD2ROWS_AVX2
|
||||||
|
#endif // defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
|
||||||
|
|
||||||
|
#if defined(HAS_TRANSPOSEWX16_AVX2)
|
||||||
|
LIBYUV_TARGET_AVX2
|
||||||
|
void TransposeWx16_AVX2(const uint8_t* src,
|
||||||
|
int src_stride,
|
||||||
|
uint8_t* dst,
|
||||||
|
int dst_stride,
|
||||||
|
int width) {
|
||||||
|
TransposeWx16_Byte_AVX2(src, src_stride, dst, dst_stride * 2,
|
||||||
|
dst + dst_stride, dst_stride * 2, width);
|
||||||
|
}
|
||||||
|
#endif // defined(HAS_TRANSPOSEWX16_AVX2)
|
||||||
|
|
||||||
|
#if defined(HAS_TRANSPOSEUVWX16_AVX2)
|
||||||
|
LIBYUV_TARGET_AVX2
|
||||||
|
void TransposeUVWx16_AVX2(const uint8_t* src,
|
||||||
|
int src_stride,
|
||||||
|
uint8_t* dst_a,
|
||||||
|
int dst_stride_a,
|
||||||
|
uint8_t* dst_b,
|
||||||
|
int dst_stride_b,
|
||||||
|
int width) {
|
||||||
|
TransposeWx16_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b,
|
||||||
|
dst_stride_b, width * 2);
|
||||||
|
}
|
||||||
|
#endif // defined(HAS_TRANSPOSEUVWX16_AVX2)
|
||||||
|
|
||||||
|
#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
||||||
|
// Dword permutes to gather 16 rows of a column from 4 lanes.
|
||||||
|
static const uint32_t kPermdTranspose_AVX512BW[32] = {
|
||||||
|
0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30,
|
||||||
|
1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31};
|
||||||
|
|
||||||
|
// Load 16 masked bytes from rows p, p + s4, p + s4 * 2, p + s4 * 3 into
|
||||||
|
// lanes 0..3.
|
||||||
|
#define LOAD4ROWS_AVX512BW(p, s4, k) \
|
||||||
|
_mm512_inserti32x4( \
|
||||||
|
_mm512_inserti32x4( \
|
||||||
|
_mm512_inserti32x4( \
|
||||||
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi8(k, (p))), \
|
||||||
|
_mm_maskz_loadu_epi8(k, (p) + (s4)), 1), \
|
||||||
|
_mm_maskz_loadu_epi8(k, (p) + (s4) * 2), 2), \
|
||||||
|
_mm_maskz_loadu_epi8(k, (p) + (s4) * 3), 3)
|
||||||
|
|
||||||
|
// Store 16 destination rows. Lane n of o0/o2 is for dst_a and lane n of
|
||||||
|
// o1/o3 is for dst_b.
|
||||||
|
LIBYUV_TARGET_AVX512BW
|
||||||
|
static void Store16x16_AVX512BW(__m512i o0,
|
||||||
|
__m512i o1,
|
||||||
|
__m512i o2,
|
||||||
|
__m512i o3,
|
||||||
|
uint8_t* dst_a,
|
||||||
|
ptrdiff_t sa,
|
||||||
|
uint8_t* dst_b,
|
||||||
|
ptrdiff_t sb) {
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a), _mm512_castsi512_si128(o0));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b), _mm512_castsi512_si128(o1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa), _mm512_extracti32x4_epi32(o0, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb), _mm512_extracti32x4_epi32(o1, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 2), _mm512_castsi512_si128(o2));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 2), _mm512_castsi512_si128(o3));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 3),
|
||||||
|
_mm512_extracti32x4_epi32(o2, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 3),
|
||||||
|
_mm512_extracti32x4_epi32(o3, 1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 4),
|
||||||
|
_mm512_extracti32x4_epi32(o0, 2));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 4),
|
||||||
|
_mm512_extracti32x4_epi32(o1, 2));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 5),
|
||||||
|
_mm512_extracti32x4_epi32(o0, 3));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 5),
|
||||||
|
_mm512_extracti32x4_epi32(o1, 3));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 6),
|
||||||
|
_mm512_extracti32x4_epi32(o2, 2));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 6),
|
||||||
|
_mm512_extracti32x4_epi32(o3, 2));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 7),
|
||||||
|
_mm512_extracti32x4_epi32(o2, 3));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 7),
|
||||||
|
_mm512_extracti32x4_epi32(o3, 3));
|
||||||
}
|
}
|
||||||
|
|
||||||
#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86)
|
// Transpose 16x16 bytes using 4 lanes of AVX512BW with tail masking.
|
||||||
|
// Source row r is in lane r / 4 of register r % 4.
|
||||||
|
// Can be used for 8-bit planar transpose (dst_a = even rows, dst_b = odd rows)
|
||||||
|
// or 16-bit UV split transpose (dst_a = U plane, dst_b = V plane).
|
||||||
|
// TODO(fbarchard): Consider removing in favor of TransposeWx16_Byte_AVX2,
|
||||||
|
// which is within 10% of AVX512BW on most CPUs.
|
||||||
|
LIBYUV_TARGET_AVX512BW
|
||||||
|
static void TransposeWx16_Byte_AVX512BW(const uint8_t* src,
|
||||||
|
int src_stride,
|
||||||
|
uint8_t* dst_a,
|
||||||
|
int dst_stride_a,
|
||||||
|
uint8_t* dst_b,
|
||||||
|
int dst_stride_b,
|
||||||
|
int byte_width) {
|
||||||
|
const ptrdiff_t s = src_stride;
|
||||||
|
const ptrdiff_t sa = dst_stride_a;
|
||||||
|
const ptrdiff_t sb = dst_stride_b;
|
||||||
|
const __m512i perm0 = _mm512_loadu_si512(kPermdTranspose_AVX512BW);
|
||||||
|
const __m512i perm1 = _mm512_loadu_si512(kPermdTranspose_AVX512BW + 16);
|
||||||
|
while (byte_width > 0) {
|
||||||
|
const int n = byte_width < 16 ? byte_width : 16;
|
||||||
|
const __mmask16 k = (__mmask16)((1u << n) - 1);
|
||||||
|
__m512i z0 = LOAD4ROWS_AVX512BW(src, s * 4, k);
|
||||||
|
__m512i z1 = LOAD4ROWS_AVX512BW(src + s, s * 4, k);
|
||||||
|
__m512i z2 = LOAD4ROWS_AVX512BW(src + s * 2, s * 4, k);
|
||||||
|
__m512i z3 = LOAD4ROWS_AVX512BW(src + s * 3, s * 4, k);
|
||||||
|
|
||||||
|
__m512i a0 = _mm512_unpacklo_epi8(z0, z1);
|
||||||
|
__m512i a1 = _mm512_unpackhi_epi8(z0, z1);
|
||||||
|
__m512i a2 = _mm512_unpacklo_epi8(z2, z3);
|
||||||
|
__m512i a3 = _mm512_unpackhi_epi8(z2, z3);
|
||||||
|
__m512i b0 = _mm512_unpacklo_epi16(a0, a2);
|
||||||
|
__m512i b1 = _mm512_unpackhi_epi16(a0, a2);
|
||||||
|
__m512i b2 = _mm512_unpacklo_epi16(a1, a3);
|
||||||
|
__m512i b3 = _mm512_unpackhi_epi16(a1, a3);
|
||||||
|
__m512i o0 = _mm512_permutex2var_epi32(b0, perm0, b2);
|
||||||
|
__m512i o1 = _mm512_permutex2var_epi32(b0, perm1, b2);
|
||||||
|
__m512i o2 = _mm512_permutex2var_epi32(b1, perm0, b3);
|
||||||
|
__m512i o3 = _mm512_permutex2var_epi32(b1, perm1, b3);
|
||||||
|
|
||||||
|
if (n == 16) {
|
||||||
|
Store16x16_AVX512BW(o0, o1, o2, o3, dst_a, sa, dst_b, sb);
|
||||||
|
dst_a += sa * 8;
|
||||||
|
dst_b += sb * 8;
|
||||||
|
} else {
|
||||||
|
// Remainder: 1 to 15 bytes. Store 1 destination row per byte.
|
||||||
|
SIMD_ALIGNED(uint8_t temp[16 * 16]);
|
||||||
|
int i;
|
||||||
|
Store16x16_AVX512BW(o0, o1, o2, o3, temp, 32, temp + 16, 32);
|
||||||
|
for (i = 0; i < n; ++i) {
|
||||||
|
uint8_t* dst = (i & 1) ? dst_b + sb * (i >> 1) : dst_a + sa * (i >> 1);
|
||||||
|
_mm_storeu_si128((__m128i*)dst,
|
||||||
|
_mm_load_si128((const __m128i*)(temp + i * 16)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
src += 16;
|
||||||
|
byte_width -= 16;
|
||||||
|
}
|
||||||
|
_mm256_zeroupper();
|
||||||
|
}
|
||||||
|
#undef LOAD4ROWS_AVX512BW
|
||||||
|
#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) ||
|
||||||
|
// defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
||||||
|
|
||||||
|
#if defined(HAS_TRANSPOSEWX16_AVX512BW)
|
||||||
|
LIBYUV_TARGET_AVX512BW
|
||||||
|
void TransposeWx16_AVX512BW(const uint8_t* src,
|
||||||
|
int src_stride,
|
||||||
|
uint8_t* dst,
|
||||||
|
int dst_stride,
|
||||||
|
int width) {
|
||||||
|
TransposeWx16_Byte_AVX512BW(src, src_stride, dst, dst_stride * 2,
|
||||||
|
dst + (width > 1 ? dst_stride : 0),
|
||||||
|
dst_stride * 2, width);
|
||||||
|
}
|
||||||
|
#endif // defined(HAS_TRANSPOSEWX16_AVX512BW)
|
||||||
|
|
||||||
|
#if defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
||||||
|
LIBYUV_TARGET_AVX512BW
|
||||||
|
void TransposeUVWx16_AVX512BW(const uint8_t* src,
|
||||||
|
int src_stride,
|
||||||
|
uint8_t* dst_a,
|
||||||
|
int dst_stride_a,
|
||||||
|
uint8_t* dst_b,
|
||||||
|
int dst_stride_b,
|
||||||
|
int width) {
|
||||||
|
TransposeWx16_Byte_AVX512BW(src, src_stride, dst_a, dst_stride_a, dst_b,
|
||||||
|
dst_stride_b, width * 2);
|
||||||
|
}
|
||||||
|
#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
||||||
|
|
||||||
|
#if defined(HAS_TRANSPOSE4X4_32_SSE2)
|
||||||
|
// Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row.
|
||||||
|
LIBYUV_TARGET_SSE2
|
||||||
|
void Transpose4x4_32_SSE2(const uint8_t* src,
|
||||||
|
int src_stride,
|
||||||
|
uint8_t* dst,
|
||||||
|
int dst_stride,
|
||||||
|
int width) {
|
||||||
|
const ptrdiff_t s = src_stride;
|
||||||
|
const ptrdiff_t d = dst_stride;
|
||||||
|
while (width > 0) {
|
||||||
|
__m128i r0 = _mm_loadu_si128((const __m128i*)(src)); // a b c d
|
||||||
|
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); // e f g h
|
||||||
|
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); // i j k l
|
||||||
|
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); // m n o p
|
||||||
|
__m128i t0 = _mm_unpacklo_epi32(r0, r1); // a e b f
|
||||||
|
__m128i t1 = _mm_unpacklo_epi32(r2, r3); // i m j n
|
||||||
|
__m128i t2 = _mm_unpackhi_epi32(r0, r1); // c g d h
|
||||||
|
__m128i t3 = _mm_unpackhi_epi32(r2, r3); // k o l p
|
||||||
|
_mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); // a e i m
|
||||||
|
_mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3));
|
||||||
|
_mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3));
|
||||||
|
src += s * 4;
|
||||||
|
dst += 16;
|
||||||
|
width -= 4;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif // defined(HAS_TRANSPOSE4X4_32_SSE2)
|
||||||
|
|
||||||
|
#if defined(HAS_TRANSPOSE4X4_32_AVX2)
|
||||||
|
// Transpose 32 bit values (ARGB). Read a column of 8 pixels, write a row.
|
||||||
|
LIBYUV_TARGET_AVX2
|
||||||
|
void Transpose4x4_32_AVX2(const uint8_t* src,
|
||||||
|
int src_stride,
|
||||||
|
uint8_t* dst,
|
||||||
|
int dst_stride,
|
||||||
|
int width) {
|
||||||
|
const ptrdiff_t s = src_stride;
|
||||||
|
const ptrdiff_t d = dst_stride;
|
||||||
|
while (width > 0) {
|
||||||
|
__m256i r0 = _mm256_inserti128_si256(
|
||||||
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
|
||||||
|
_mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
|
||||||
|
__m256i r1 = _mm256_inserti128_si256(
|
||||||
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
|
||||||
|
_mm_loadu_si128((const __m128i*)(src + s * 5)), 1);
|
||||||
|
__m256i r2 = _mm256_inserti128_si256(
|
||||||
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
|
||||||
|
_mm_loadu_si128((const __m128i*)(src + s * 6)), 1);
|
||||||
|
__m256i r3 = _mm256_inserti128_si256(
|
||||||
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
|
||||||
|
_mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
|
||||||
|
__m256i t0 = _mm256_unpacklo_epi32(r0, r1); // a e b f
|
||||||
|
__m256i t1 = _mm256_unpacklo_epi32(r2, r3); // i m j n
|
||||||
|
__m256i t2 = _mm256_unpackhi_epi32(r0, r1); // c g d h
|
||||||
|
__m256i t3 = _mm256_unpackhi_epi32(r2, r3); // k o l p
|
||||||
|
_mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1));
|
||||||
|
_mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1));
|
||||||
|
_mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3));
|
||||||
|
_mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3));
|
||||||
|
src += s * 8;
|
||||||
|
dst += 32;
|
||||||
|
width -= 8;
|
||||||
|
}
|
||||||
|
_mm256_zeroupper();
|
||||||
|
}
|
||||||
|
#endif // defined(HAS_TRANSPOSE4X4_32_AVX2)
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
} // extern "C"
|
} // extern "C"
|
||||||
} // namespace libyuv
|
} // namespace libyuv
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) &&
|
||||||
|
// (Visual C or LIBYUV_ENABLE_ROWWIN)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user