mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
eeb06a6451
Optimize 32-bit ARGBRotate (0, 90, 180, 270) across Intel x86/x64
architectures (SSE2, AVX2, AVX512BW), with matching kernels for ARM
(NEON, SVE2, SME), WASM, LoongArch (LSX, LASX), and RISC-V (RVV):
- Replace column-by-column ScaleARGBRowDownEven gather loops in
ARGBTranspose (90/270) with full-plane horizontal tile transposes:
- AVX512BW (TransposeWxH_32_AVX512BW / TransposeNx4_32_AVX512BW):
Transpose 16x16 blocks across zmm0..zmm31 using 4-stage
vpunpckldq/hdq, vpunpcklqdq/hqdq, and vshufi32x4 butterflies with
k-masked right and bottom tails, eliminating Any wrappers and scalar
C edge fallbacks.
- AVX2 (TransposeWxH_32_AVX2 / TransposeNx4_32_AVX2):
Transpose 8x8 blocks across ymm0..ymm15 using vpunpck* and
vperm2i128 with vpmaskmovd masked tails.
- SSE2 (Transpose4x4_32_SSE2 / Transpose4x4_32_Any_SSE2):
Transpose 4x4 blocks in xmm registers.
- Implement ARGBRotate180 directly via ARGBMirror with negative height,
avoiding intermediate row buffer allocations and extra copies.
- Implement ARGBRotate0 via ARGBCopy.
Speedup vs Before (1275x1650, US Letter 150 DPI, 1000 iterations):
CPU | 0 deg | 90 deg | 180 deg | 270 deg
---------------------|---------|---------|---------|--------
Intel Ice Lake | 1.02x | 1.34x | 1.18x | 1.33x
Intel Skylake-X | 1.02x | 1.20x | 1.07x | 1.13x
Intel Emerald Rapids | 1.01x | 1.07x | 1.00x | 1.07x
Intel AVX-512 vs C | 1.00x | 2.67x | 2.41x | 2.63x
R=rrwinterton@gmail.com
Bug: b/42280915
Test: LIBYUV_WIDTH=1275 LIBYUV_HEIGHT=1650 LIBYUV_REPEAT=1000 ./out/Release/libyuv_unittest --gtest_filter=*ARGBRotate*
Change-Id: Ib308a225c17e46eab083a9da4033598fcf0c303c
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8538986
Reviewed-by: richard winterton <rrwinterton@gmail.com>
1587 lines
77 KiB
C++
1587 lines
77 KiB
C++
/*
|
|
* Copyright 2015 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/rotate_row.h"
|
|
#include "libyuv/row.h"
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
// Note: every register written by an inline asm block (vector, general
|
|
// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
|
|
// listed in its clobber list, even when the ABI treats it as caller-saved.
|
|
// With LTO/LTCG the compiler may keep values live in any register across the
|
|
// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
|
|
// are callee-saved. List "memory" first, then "cc", then registers.
|
|
|
|
// This module is for GCC and clang x64. 32 bit x86 uses rotate_win.cc.
|
|
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
|
!defined(LIBYUV_ENABLE_ROWWIN)
|
|
|
|
#if defined(HAS_TRANSPOSEWX8_SSSE3)
|
|
// Transpose 16x8. 64 bit.
|
|
void TransposeWx8_SSSE3(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
asm volatile(
|
|
// Read in the data from the source pointer.
|
|
// First round of bit swap.
|
|
LABELALIGN
|
|
"1: \n"
|
|
"movdqu (%0),%%xmm0 \n"
|
|
"movdqu (%0,%3),%%xmm1 \n"
|
|
"lea (%0,%3,2),%0 \n"
|
|
"movdqa %%xmm0,%%xmm8 \n"
|
|
"punpcklbw %%xmm1,%%xmm0 \n"
|
|
"punpckhbw %%xmm1,%%xmm8 \n"
|
|
"movdqu (%0),%%xmm2 \n"
|
|
"movdqa %%xmm0,%%xmm1 \n"
|
|
"movdqa %%xmm8,%%xmm9 \n"
|
|
"palignr $0x8,%%xmm1,%%xmm1 \n"
|
|
"palignr $0x8,%%xmm9,%%xmm9 \n"
|
|
"movdqu (%0,%3),%%xmm3 \n"
|
|
"lea (%0,%3,2),%0 \n"
|
|
"movdqa %%xmm2,%%xmm10 \n"
|
|
"punpcklbw %%xmm3,%%xmm2 \n"
|
|
"punpckhbw %%xmm3,%%xmm10 \n"
|
|
"movdqa %%xmm2,%%xmm3 \n"
|
|
"movdqa %%xmm10,%%xmm11 \n"
|
|
"movdqu (%0),%%xmm4 \n"
|
|
"palignr $0x8,%%xmm3,%%xmm3 \n"
|
|
"palignr $0x8,%%xmm11,%%xmm11 \n"
|
|
"movdqu (%0,%3),%%xmm5 \n"
|
|
"lea (%0,%3,2),%0 \n"
|
|
"movdqa %%xmm4,%%xmm12 \n"
|
|
"punpcklbw %%xmm5,%%xmm4 \n"
|
|
"punpckhbw %%xmm5,%%xmm12 \n"
|
|
"movdqa %%xmm4,%%xmm5 \n"
|
|
"movdqa %%xmm12,%%xmm13 \n"
|
|
"movdqu (%0),%%xmm6 \n"
|
|
"palignr $0x8,%%xmm5,%%xmm5 \n"
|
|
"palignr $0x8,%%xmm13,%%xmm13 \n"
|
|
"movdqu (%0,%3),%%xmm7 \n"
|
|
"lea (%0,%3,2),%0 \n"
|
|
"movdqa %%xmm6,%%xmm14 \n"
|
|
"punpcklbw %%xmm7,%%xmm6 \n"
|
|
"punpckhbw %%xmm7,%%xmm14 \n"
|
|
"neg %3 \n"
|
|
"movdqa %%xmm6,%%xmm7 \n"
|
|
"movdqa %%xmm14,%%xmm15 \n"
|
|
"lea 0x10(%0,%3,8),%0 \n"
|
|
"palignr $0x8,%%xmm7,%%xmm7 \n"
|
|
"palignr $0x8,%%xmm15,%%xmm15 \n"
|
|
"neg %3 \n"
|
|
// Second round of bit swap.
|
|
"punpcklwd %%xmm2,%%xmm0 \n"
|
|
"punpcklwd %%xmm3,%%xmm1 \n"
|
|
"movdqa %%xmm0,%%xmm2 \n"
|
|
"movdqa %%xmm1,%%xmm3 \n"
|
|
"palignr $0x8,%%xmm2,%%xmm2 \n"
|
|
"palignr $0x8,%%xmm3,%%xmm3 \n"
|
|
"punpcklwd %%xmm6,%%xmm4 \n"
|
|
"punpcklwd %%xmm7,%%xmm5 \n"
|
|
"movdqa %%xmm4,%%xmm6 \n"
|
|
"movdqa %%xmm5,%%xmm7 \n"
|
|
"palignr $0x8,%%xmm6,%%xmm6 \n"
|
|
"palignr $0x8,%%xmm7,%%xmm7 \n"
|
|
"punpcklwd %%xmm10,%%xmm8 \n"
|
|
"punpcklwd %%xmm11,%%xmm9 \n"
|
|
"movdqa %%xmm8,%%xmm10 \n"
|
|
"movdqa %%xmm9,%%xmm11 \n"
|
|
"palignr $0x8,%%xmm10,%%xmm10 \n"
|
|
"palignr $0x8,%%xmm11,%%xmm11 \n"
|
|
"punpcklwd %%xmm14,%%xmm12 \n"
|
|
"punpcklwd %%xmm15,%%xmm13 \n"
|
|
"movdqa %%xmm12,%%xmm14 \n"
|
|
"movdqa %%xmm13,%%xmm15 \n"
|
|
"palignr $0x8,%%xmm14,%%xmm14 \n"
|
|
"palignr $0x8,%%xmm15,%%xmm15 \n"
|
|
// Third round of bit swap.
|
|
// Write to the destination pointer.
|
|
"punpckldq %%xmm4,%%xmm0 \n"
|
|
"movq %%xmm0,(%1) \n"
|
|
"movdqa %%xmm0,%%xmm4 \n"
|
|
"palignr $0x8,%%xmm4,%%xmm4 \n"
|
|
"movq %%xmm4,(%1,%4) \n"
|
|
"lea (%1,%4,2),%1 \n"
|
|
"punpckldq %%xmm6,%%xmm2 \n"
|
|
"movdqa %%xmm2,%%xmm6 \n"
|
|
"movq %%xmm2,(%1) \n"
|
|
"palignr $0x8,%%xmm6,%%xmm6 \n"
|
|
"punpckldq %%xmm5,%%xmm1 \n"
|
|
"movq %%xmm6,(%1,%4) \n"
|
|
"lea (%1,%4,2),%1 \n"
|
|
"movdqa %%xmm1,%%xmm5 \n"
|
|
"movq %%xmm1,(%1) \n"
|
|
"palignr $0x8,%%xmm5,%%xmm5 \n"
|
|
"movq %%xmm5,(%1,%4) \n"
|
|
"lea (%1,%4,2),%1 \n"
|
|
"punpckldq %%xmm7,%%xmm3 \n"
|
|
"movq %%xmm3,(%1) \n"
|
|
"movdqa %%xmm3,%%xmm7 \n"
|
|
"palignr $0x8,%%xmm7,%%xmm7 \n"
|
|
"movq %%xmm7,(%1,%4) \n"
|
|
"lea (%1,%4,2),%1 \n"
|
|
"punpckldq %%xmm12,%%xmm8 \n"
|
|
"movq %%xmm8,(%1) \n"
|
|
"movdqa %%xmm8,%%xmm12 \n"
|
|
"palignr $0x8,%%xmm12,%%xmm12 \n"
|
|
"movq %%xmm12,(%1,%4) \n"
|
|
"lea (%1,%4,2),%1 \n"
|
|
"punpckldq %%xmm14,%%xmm10 \n"
|
|
"movdqa %%xmm10,%%xmm14 \n"
|
|
"movq %%xmm10,(%1) \n"
|
|
"palignr $0x8,%%xmm14,%%xmm14 \n"
|
|
"punpckldq %%xmm13,%%xmm9 \n"
|
|
"movq %%xmm14,(%1,%4) \n"
|
|
"lea (%1,%4,2),%1 \n"
|
|
"movdqa %%xmm9,%%xmm13 \n"
|
|
"movq %%xmm9,(%1) \n"
|
|
"palignr $0x8,%%xmm13,%%xmm13 \n"
|
|
"movq %%xmm13,(%1,%4) \n"
|
|
"lea (%1,%4,2),%1 \n"
|
|
"punpckldq %%xmm15,%%xmm11 \n"
|
|
"movq %%xmm11,(%1) \n"
|
|
"movdqa %%xmm11,%%xmm15 \n"
|
|
"palignr $0x8,%%xmm15,%%xmm15 \n"
|
|
"sub $0x10,%2 \n"
|
|
"movq %%xmm15,(%1,%4) \n"
|
|
"lea (%1,%4,2),%1 \n"
|
|
"jg 1b \n"
|
|
: "+r"(src), // %0
|
|
"+r"(dst), // %1
|
|
"+r"(width) // %2
|
|
: "r"((ptrdiff_t)(src_stride)), // %3
|
|
"r"((ptrdiff_t)(dst_stride)) // %4
|
|
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
|
"xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
|
|
"xmm15");
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX8_SSSE3)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
|
// Dword permutes to gather 16 rows of a column from 4 lanes.
|
|
static const uint32_t kPermdTranspose_AVX512BW[32] = {
|
|
0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30,
|
|
1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31};
|
|
|
|
// Transpose 16x16 bytes held in 4 lanes of 4 zmm registers.
|
|
// Source row r is in lane r / 4 of zmm(r % 4). Outputs zmm0..zmm3 are 16
|
|
// destination rows, lane n of zmm0/zmm2 for dst_a and lane n of zmm1/zmm3
|
|
// for dst_b.
|
|
#define TRANSPOSE16X16_AVX512BW \
|
|
"vpunpcklbw %%zmm1,%%zmm0,%%zmm4 \n" \
|
|
"vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n" \
|
|
"vpunpcklbw %%zmm3,%%zmm2,%%zmm0 \n" \
|
|
"vpunpckhbw %%zmm3,%%zmm2,%%zmm1 \n" \
|
|
"vpunpcklwd %%zmm0,%%zmm4,%%zmm2 \n" \
|
|
"vpunpckhwd %%zmm0,%%zmm4,%%zmm3 \n" \
|
|
"vpunpcklwd %%zmm1,%%zmm5,%%zmm4 \n" \
|
|
"vpunpckhwd %%zmm1,%%zmm5,%%zmm5 \n" \
|
|
"vmovdqa64 %%zmm2,%%zmm0 \n" \
|
|
"vpermt2d %%zmm4,%%zmm6,%%zmm0 \n" \
|
|
"vmovdqa64 %%zmm2,%%zmm1 \n" \
|
|
"vpermt2d %%zmm4,%%zmm7,%%zmm1 \n" \
|
|
"vmovdqa64 %%zmm3,%%zmm2 \n" \
|
|
"vpermt2d %%zmm5,%%zmm6,%%zmm2 \n" \
|
|
"vpermt2d %%zmm5,%%zmm7,%%zmm3 \n"
|
|
|
|
// Transpose 16x16 bytes using 4 lanes of AVX512BW with tail masking.
|
|
// Can be used for 8-bit planar transpose (dst_a = even rows, dst_b = odd rows)
|
|
// or 16-bit UV split transpose (dst_a = U plane, dst_b = V plane).
|
|
// TODO(fbarchard): Port to rotate_win.cc using intrinsics.
|
|
// TODO(fbarchard): Use for ARGB (32 bit) and 16 bit channel transposes.
|
|
// TODO(fbarchard): Consider removing in favor of TransposeWx16_Byte_AVX2,
|
|
// which is within 10% of AVX512BW on most CPUs.
|
|
static void TransposeWx16_Byte_AVX512BW(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int byte_width) {
|
|
uintptr_t temp;
|
|
asm volatile(
|
|
"vmovdqu32 (%[shuf]),%%zmm6 \n"
|
|
"vmovdqu32 0x40(%[shuf]),%%zmm7 \n"
|
|
"sub $0x10,%[width] \n"
|
|
"jl 2f \n"
|
|
|
|
// Main loop: 16 source rows x 16 bytes.
|
|
LABELALIGN
|
|
"1: \n"
|
|
"vmovdqu (%[src]),%%xmm0 \n"
|
|
"vmovdqu (%[src],%[src_stride]),%%xmm1 \n"
|
|
"lea (%[src],%[src_stride],2),%[temp] \n"
|
|
"lea 0x10(%[src]),%[src] \n"
|
|
"vmovdqu (%[temp]),%%xmm2 \n"
|
|
"vmovdqu (%[temp],%[src_stride]),%%xmm3 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n"
|
|
"vinserti32x4 $1,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n"
|
|
"vinserti32x4 $1,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n"
|
|
"vinserti32x4 $2,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n"
|
|
"vinserti32x4 $2,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n"
|
|
"vinserti32x4 $3,(%[temp],%[src_stride]),%%zmm1,%%zmm1 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n"
|
|
"vinserti32x4 $3,(%[temp],%[src_stride]),%%zmm3,%%zmm3 \n"
|
|
|
|
TRANSPOSE16X16_AVX512BW
|
|
|
|
"vmovdqu %%xmm0,(%[dst_a]) \n"
|
|
"vmovdqu %%xmm1,(%[dst_b]) \n"
|
|
"vextracti32x4 $1,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
|
|
"vextracti32x4 $1,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"vmovdqu %%xmm2,(%[dst_a]) \n"
|
|
"vmovdqu %%xmm3,(%[dst_b]) \n"
|
|
"vextracti32x4 $1,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
|
|
"vextracti32x4 $1,%%zmm3,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"vextracti32x4 $2,%%zmm0,(%[dst_a]) \n"
|
|
"vextracti32x4 $2,%%zmm1,(%[dst_b]) \n"
|
|
"vextracti32x4 $3,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
|
|
"vextracti32x4 $3,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"vextracti32x4 $2,%%zmm2,(%[dst_a]) \n"
|
|
"vextracti32x4 $2,%%zmm3,(%[dst_b]) \n"
|
|
"vextracti32x4 $3,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
|
|
"vextracti32x4 $3,%%zmm3,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"sub $0x10,%[width] \n"
|
|
"jge 1b \n"
|
|
|
|
// Remainder: 1 to 15 bytes, masked loads.
|
|
"2: \n"
|
|
"add $0x10,%[width] \n"
|
|
"je 99f \n"
|
|
"mov $-1,%k[temp] \n"
|
|
"bzhi %k[width],%k[temp],%k[temp] \n"
|
|
"kmovw %k[temp],%%k1 \n"
|
|
"vmovdqu8 (%[src]),%%xmm0%{%%k1%}%{z%} \n"
|
|
"vmovdqu8 (%[src],%[src_stride]),%%xmm1%{%%k1%}%{z%} \n"
|
|
"lea (%[src],%[src_stride],2),%[src] \n"
|
|
"vmovdqu8 (%[src]),%%xmm2%{%%k1%}%{z%} \n"
|
|
"vmovdqu8 (%[src],%[src_stride]),%%xmm3%{%%k1%}%{z%} \n"
|
|
"lea (%[src],%[src_stride],2),%[src] \n"
|
|
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $1,%%xmm4,%%zmm0,%%zmm0 \n"
|
|
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $1,%%xmm4,%%zmm1,%%zmm1 \n"
|
|
"lea (%[src],%[src_stride],2),%[src] \n"
|
|
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $1,%%xmm4,%%zmm2,%%zmm2 \n"
|
|
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $1,%%xmm4,%%zmm3,%%zmm3 \n"
|
|
"lea (%[src],%[src_stride],2),%[src] \n"
|
|
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $2,%%xmm4,%%zmm0,%%zmm0 \n"
|
|
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $2,%%xmm4,%%zmm1,%%zmm1 \n"
|
|
"lea (%[src],%[src_stride],2),%[src] \n"
|
|
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $2,%%xmm4,%%zmm2,%%zmm2 \n"
|
|
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $2,%%xmm4,%%zmm3,%%zmm3 \n"
|
|
"lea (%[src],%[src_stride],2),%[src] \n"
|
|
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $3,%%xmm4,%%zmm0,%%zmm0 \n"
|
|
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $3,%%xmm4,%%zmm1,%%zmm1 \n"
|
|
"lea (%[src],%[src_stride],2),%[src] \n"
|
|
"vmovdqu8 (%[src]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $3,%%xmm4,%%zmm2,%%zmm2 \n"
|
|
"vmovdqu8 (%[src],%[src_stride]),%%xmm4%{%%k1%}%{z%} \n"
|
|
"vinserti32x4 $3,%%xmm4,%%zmm3,%%zmm3 \n"
|
|
|
|
TRANSPOSE16X16_AVX512BW
|
|
|
|
// Store 1 destination row per remaining byte.
|
|
"vmovdqu %%xmm0,(%[dst_a]) \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vmovdqu %%xmm1,(%[dst_b]) \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $1,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $1,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vmovdqu %%xmm2,(%[dst_a]) \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vmovdqu %%xmm3,(%[dst_b]) \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $1,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $1,%%zmm3,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $2,%%zmm0,(%[dst_a]) \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $2,%%zmm1,(%[dst_b]) \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $3,%%zmm0,(%[dst_a],%[dst_stride_a]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $3,%%zmm1,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $2,%%zmm2,(%[dst_a]) \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $2,%%zmm3,(%[dst_b]) \n"
|
|
"sub $0x1,%[width] \n"
|
|
"je 99f \n"
|
|
"vextracti32x4 $3,%%zmm2,(%[dst_a],%[dst_stride_a]) \n"
|
|
|
|
"99: \n"
|
|
"vzeroupper \n"
|
|
: [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b),
|
|
[width] "+r"(byte_width), [temp] "=&r"(temp)
|
|
: [src_stride] "r"((ptrdiff_t)src_stride),
|
|
[dst_stride_a] "r"((ptrdiff_t)dst_stride_a),
|
|
[dst_stride_b] "r"((ptrdiff_t)dst_stride_b),
|
|
[shuf] "r"(kPermdTranspose_AVX512BW)
|
|
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
|
|
"xmm6", "xmm7");
|
|
}
|
|
#undef TRANSPOSE16X16_AVX512BW
|
|
#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) ||
|
|
// defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_AVX512BW)
|
|
void TransposeWx16_AVX512BW(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
TransposeWx16_Byte_AVX512BW(src, src_stride, dst, dst_stride * 2,
|
|
dst + (width > 1 ? dst_stride : 0),
|
|
dst_stride * 2, width);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX16_AVX512BW)
|
|
|
|
#if defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
|
void TransposeUVWx16_AVX512BW(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int width) {
|
|
TransposeWx16_Byte_AVX512BW(src, src_stride, dst_a, dst_stride_a, dst_b,
|
|
dst_stride_b, width * 2);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
|
|
// Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8
|
|
// of ymm(r % 8). 3 unpack stages leave each lane with 2 columns x 8 rows, and
|
|
// vpermq joins the rows 0..7 and 8..15 halves. Lane 0 of each output is an
|
|
// even column for dst_a and lane 1 an odd column for dst_b.
|
|
// Width is a multiple of 16 bytes.
|
|
static void TransposeWx16_Byte_AVX2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int byte_width) {
|
|
uintptr_t temp;
|
|
asm volatile(
|
|
"1: \n"
|
|
"vmovdqu (%[src]),%%xmm0 \n"
|
|
"vmovdqu (%[src],%[src_stride]),%%xmm1 \n"
|
|
"lea (%[src],%[src_stride],2),%[temp] \n"
|
|
"lea 0x10(%[src]),%[src] \n"
|
|
"vmovdqu (%[temp]),%%xmm2 \n"
|
|
"vmovdqu (%[temp],%[src_stride]),%%xmm3 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vmovdqu (%[temp]),%%xmm4 \n"
|
|
"vmovdqu (%[temp],%[src_stride]),%%xmm5 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vmovdqu (%[temp]),%%xmm6 \n"
|
|
"vmovdqu (%[temp],%[src_stride]),%%xmm7 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n"
|
|
"vinserti128 $1,(%[temp],%[src_stride]),%%ymm1,%%ymm1 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n"
|
|
"vinserti128 $1,(%[temp],%[src_stride]),%%ymm3,%%ymm3 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti128 $1,(%[temp]),%%ymm4,%%ymm4 \n"
|
|
"vinserti128 $1,(%[temp],%[src_stride]),%%ymm5,%%ymm5 \n"
|
|
"lea (%[temp],%[src_stride],2),%[temp] \n"
|
|
"vinserti128 $1,(%[temp]),%%ymm6,%%ymm6 \n"
|
|
"vinserti128 $1,(%[temp],%[src_stride]),%%ymm7,%%ymm7 \n"
|
|
|
|
"vpunpcklbw %%ymm1,%%ymm0,%%ymm8 \n"
|
|
"vpunpckhbw %%ymm1,%%ymm0,%%ymm9 \n"
|
|
"vpunpcklbw %%ymm3,%%ymm2,%%ymm10 \n"
|
|
"vpunpckhbw %%ymm3,%%ymm2,%%ymm11 \n"
|
|
"vpunpcklbw %%ymm5,%%ymm4,%%ymm12 \n"
|
|
"vpunpckhbw %%ymm5,%%ymm4,%%ymm13 \n"
|
|
"vpunpcklbw %%ymm7,%%ymm6,%%ymm14 \n"
|
|
"vpunpckhbw %%ymm7,%%ymm6,%%ymm15 \n"
|
|
|
|
"vpunpcklwd %%ymm10,%%ymm8,%%ymm0 \n"
|
|
"vpunpckhwd %%ymm10,%%ymm8,%%ymm1 \n"
|
|
"vpunpcklwd %%ymm11,%%ymm9,%%ymm2 \n"
|
|
"vpunpckhwd %%ymm11,%%ymm9,%%ymm3 \n"
|
|
"vpunpcklwd %%ymm14,%%ymm12,%%ymm4 \n"
|
|
"vpunpckhwd %%ymm14,%%ymm12,%%ymm5 \n"
|
|
"vpunpcklwd %%ymm15,%%ymm13,%%ymm6 \n"
|
|
"vpunpckhwd %%ymm15,%%ymm13,%%ymm7 \n"
|
|
|
|
"vpunpckldq %%ymm4,%%ymm0,%%ymm8 \n"
|
|
"vpunpckhdq %%ymm4,%%ymm0,%%ymm9 \n"
|
|
"vpunpckldq %%ymm5,%%ymm1,%%ymm10 \n"
|
|
"vpunpckhdq %%ymm5,%%ymm1,%%ymm11 \n"
|
|
"vpunpckldq %%ymm6,%%ymm2,%%ymm12 \n"
|
|
"vpunpckhdq %%ymm6,%%ymm2,%%ymm13 \n"
|
|
"vpunpckldq %%ymm7,%%ymm3,%%ymm14 \n"
|
|
"vpunpckhdq %%ymm7,%%ymm3,%%ymm15 \n"
|
|
|
|
"vpermq $0xd8,%%ymm8,%%ymm8 \n"
|
|
"vpermq $0xd8,%%ymm9,%%ymm9 \n"
|
|
"vpermq $0xd8,%%ymm10,%%ymm10 \n"
|
|
"vpermq $0xd8,%%ymm11,%%ymm11 \n"
|
|
"vpermq $0xd8,%%ymm12,%%ymm12 \n"
|
|
"vpermq $0xd8,%%ymm13,%%ymm13 \n"
|
|
"vpermq $0xd8,%%ymm14,%%ymm14 \n"
|
|
"vpermq $0xd8,%%ymm15,%%ymm15 \n"
|
|
|
|
"vmovdqu %%xmm8,(%[dst_a]) \n"
|
|
"vextracti128 $1,%%ymm8,(%[dst_b]) \n"
|
|
"vmovdqu %%xmm9,(%[dst_a],%[dst_stride_a]) \n"
|
|
"vextracti128 $1,%%ymm9,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"vmovdqu %%xmm10,(%[dst_a]) \n"
|
|
"vextracti128 $1,%%ymm10,(%[dst_b]) \n"
|
|
"vmovdqu %%xmm11,(%[dst_a],%[dst_stride_a]) \n"
|
|
"vextracti128 $1,%%ymm11,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"vmovdqu %%xmm12,(%[dst_a]) \n"
|
|
"vextracti128 $1,%%ymm12,(%[dst_b]) \n"
|
|
"vmovdqu %%xmm13,(%[dst_a],%[dst_stride_a]) \n"
|
|
"vextracti128 $1,%%ymm13,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"vmovdqu %%xmm14,(%[dst_a]) \n"
|
|
"vextracti128 $1,%%ymm14,(%[dst_b]) \n"
|
|
"vmovdqu %%xmm15,(%[dst_a],%[dst_stride_a]) \n"
|
|
"vextracti128 $1,%%ymm15,(%[dst_b],%[dst_stride_b]) \n"
|
|
"lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n"
|
|
"lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n"
|
|
"sub $0x10,%[width] \n"
|
|
"jg 1b \n"
|
|
"vzeroupper \n"
|
|
: [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b),
|
|
[width] "+r"(byte_width), [temp] "=&r"(temp)
|
|
: [src_stride] "r"((ptrdiff_t)src_stride),
|
|
[dst_stride_a] "r"((ptrdiff_t)dst_stride_a),
|
|
[dst_stride_b] "r"((ptrdiff_t)dst_stride_b)
|
|
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
|
"xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
|
|
"xmm15");
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_AVX2)
|
|
void TransposeWx16_AVX2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
TransposeWx16_Byte_AVX2(src, src_stride, dst, dst_stride * 2,
|
|
dst + dst_stride, dst_stride * 2, width);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX16_AVX2)
|
|
|
|
#if defined(HAS_TRANSPOSEUVWX16_AVX2)
|
|
void TransposeUVWx16_AVX2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int width) {
|
|
TransposeWx16_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b,
|
|
dst_stride_b, width * 2);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEUVWX16_AVX2)
|
|
|
|
// Transpose UV 8x8. 64 bit.
|
|
#if defined(HAS_TRANSPOSEUVWX8_SSE2)
|
|
void TransposeUVWx8_SSE2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int width) {
|
|
asm volatile(
|
|
// Read in the data from the source pointer.
|
|
// First round of bit swap.
|
|
LABELALIGN
|
|
"1: \n"
|
|
"movdqu (%0),%%xmm0 \n"
|
|
"movdqu (%0,%4),%%xmm1 \n"
|
|
"lea (%0,%4,2),%0 \n"
|
|
"movdqa %%xmm0,%%xmm8 \n"
|
|
"punpcklbw %%xmm1,%%xmm0 \n"
|
|
"punpckhbw %%xmm1,%%xmm8 \n"
|
|
"movdqa %%xmm8,%%xmm1 \n"
|
|
"movdqu (%0),%%xmm2 \n"
|
|
"movdqu (%0,%4),%%xmm3 \n"
|
|
"lea (%0,%4,2),%0 \n"
|
|
"movdqa %%xmm2,%%xmm8 \n"
|
|
"punpcklbw %%xmm3,%%xmm2 \n"
|
|
"punpckhbw %%xmm3,%%xmm8 \n"
|
|
"movdqa %%xmm8,%%xmm3 \n"
|
|
"movdqu (%0),%%xmm4 \n"
|
|
"movdqu (%0,%4),%%xmm5 \n"
|
|
"lea (%0,%4,2),%0 \n"
|
|
"movdqa %%xmm4,%%xmm8 \n"
|
|
"punpcklbw %%xmm5,%%xmm4 \n"
|
|
"punpckhbw %%xmm5,%%xmm8 \n"
|
|
"movdqa %%xmm8,%%xmm5 \n"
|
|
"movdqu (%0),%%xmm6 \n"
|
|
"movdqu (%0,%4),%%xmm7 \n"
|
|
"lea (%0,%4,2),%0 \n"
|
|
"movdqa %%xmm6,%%xmm8 \n"
|
|
"punpcklbw %%xmm7,%%xmm6 \n"
|
|
"neg %4 \n"
|
|
"lea 0x10(%0,%4,8),%0 \n"
|
|
"punpckhbw %%xmm7,%%xmm8 \n"
|
|
"movdqa %%xmm8,%%xmm7 \n"
|
|
"neg %4 \n"
|
|
// Second round of bit swap.
|
|
"movdqa %%xmm0,%%xmm8 \n"
|
|
"movdqa %%xmm1,%%xmm9 \n"
|
|
"punpckhwd %%xmm2,%%xmm8 \n"
|
|
"punpckhwd %%xmm3,%%xmm9 \n"
|
|
"punpcklwd %%xmm2,%%xmm0 \n"
|
|
"punpcklwd %%xmm3,%%xmm1 \n"
|
|
"movdqa %%xmm8,%%xmm2 \n"
|
|
"movdqa %%xmm9,%%xmm3 \n"
|
|
"movdqa %%xmm4,%%xmm8 \n"
|
|
"movdqa %%xmm5,%%xmm9 \n"
|
|
"punpckhwd %%xmm6,%%xmm8 \n"
|
|
"punpckhwd %%xmm7,%%xmm9 \n"
|
|
"punpcklwd %%xmm6,%%xmm4 \n"
|
|
"punpcklwd %%xmm7,%%xmm5 \n"
|
|
"movdqa %%xmm8,%%xmm6 \n"
|
|
"movdqa %%xmm9,%%xmm7 \n"
|
|
// Third round of bit swap.
|
|
// Write to the destination pointer.
|
|
"movdqa %%xmm0,%%xmm8 \n"
|
|
"punpckldq %%xmm4,%%xmm0 \n"
|
|
"movlpd %%xmm0,(%1) \n" // Write back U channel
|
|
"movhpd %%xmm0,(%2) \n" // Write back V channel
|
|
"punpckhdq %%xmm4,%%xmm8 \n"
|
|
"movlpd %%xmm8,(%1,%5) \n"
|
|
"lea (%1,%5,2),%1 \n"
|
|
"movhpd %%xmm8,(%2,%6) \n"
|
|
"lea (%2,%6,2),%2 \n"
|
|
"movdqa %%xmm2,%%xmm8 \n"
|
|
"punpckldq %%xmm6,%%xmm2 \n"
|
|
"movlpd %%xmm2,(%1) \n"
|
|
"movhpd %%xmm2,(%2) \n"
|
|
"punpckhdq %%xmm6,%%xmm8 \n"
|
|
"movlpd %%xmm8,(%1,%5) \n"
|
|
"lea (%1,%5,2),%1 \n"
|
|
"movhpd %%xmm8,(%2,%6) \n"
|
|
"lea (%2,%6,2),%2 \n"
|
|
"movdqa %%xmm1,%%xmm8 \n"
|
|
"punpckldq %%xmm5,%%xmm1 \n"
|
|
"movlpd %%xmm1,(%1) \n"
|
|
"movhpd %%xmm1,(%2) \n"
|
|
"punpckhdq %%xmm5,%%xmm8 \n"
|
|
"movlpd %%xmm8,(%1,%5) \n"
|
|
"lea (%1,%5,2),%1 \n"
|
|
"movhpd %%xmm8,(%2,%6) \n"
|
|
"lea (%2,%6,2),%2 \n"
|
|
"movdqa %%xmm3,%%xmm8 \n"
|
|
"punpckldq %%xmm7,%%xmm3 \n"
|
|
"movlpd %%xmm3,(%1) \n"
|
|
"movhpd %%xmm3,(%2) \n"
|
|
"punpckhdq %%xmm7,%%xmm8 \n"
|
|
"sub $0x8,%3 \n"
|
|
"movlpd %%xmm8,(%1,%5) \n"
|
|
"lea (%1,%5,2),%1 \n"
|
|
"movhpd %%xmm8,(%2,%6) \n"
|
|
"lea (%2,%6,2),%2 \n"
|
|
"jg 1b \n"
|
|
: "+r"(src), // %0
|
|
"+r"(dst_a), // %1
|
|
"+r"(dst_b), // %2
|
|
"+r"(width) // %3
|
|
: "r"((ptrdiff_t)(src_stride)), // %4
|
|
"r"((ptrdiff_t)(dst_stride_a)), // %5
|
|
"r"((ptrdiff_t)(dst_stride_b)) // %6
|
|
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
|
"xmm7", "xmm8", "xmm9");
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEUVWX8_SSE2)
|
|
|
|
#if defined(HAS_TRANSPOSE4X4_32_SSE2)
|
|
// 4 values, little endian view
|
|
// a b c d
|
|
// e f g h
|
|
// i j k l
|
|
// m n o p
|
|
|
|
// transpose 2x2
|
|
// a e b f from row 0, 1
|
|
// i m j n from row 2, 3
|
|
// c g d h from row 0, 1
|
|
// k o l p from row 2, 3
|
|
|
|
// transpose 4x4
|
|
// a e i m from row 0, 1
|
|
// b f j n from row 0, 1
|
|
// c g k o from row 2, 3
|
|
// d h l p from row 2, 3
|
|
|
|
// Transpose 32 bit values (ARGB)
|
|
void Transpose4x4_32_SSE2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
asm volatile(
|
|
// Main loop transpose 4x4. Read a column, write a row.
|
|
"1: \n"
|
|
"movdqu (%0),%%xmm0 \n" // a b c d
|
|
"movdqu (%0,%3),%%xmm1 \n" // e f g h
|
|
"lea (%0,%3,2),%0 \n" // src += stride * 2
|
|
"movdqu (%0),%%xmm2 \n" // i j k l
|
|
"movdqu (%0,%3),%%xmm3 \n" // m n o p
|
|
"lea (%0,%3,2),%0 \n" // src += stride * 2
|
|
|
|
// Transpose 2x2
|
|
"movdqa %%xmm0,%%xmm4 \n"
|
|
"movdqa %%xmm2,%%xmm5 \n"
|
|
"movdqa %%xmm0,%%xmm6 \n"
|
|
"movdqa %%xmm2,%%xmm7 \n"
|
|
"punpckldq %%xmm1,%%xmm4 \n" // a e b f from row 0, 1
|
|
"punpckldq %%xmm3,%%xmm5 \n" // i m j n from row 2, 3
|
|
"punpckhdq %%xmm1,%%xmm6 \n" // c g d h from row 0, 1
|
|
"punpckhdq %%xmm3,%%xmm7 \n" // k o l p from row 2, 3
|
|
|
|
// Transpose 4x4
|
|
"movdqa %%xmm4,%%xmm0 \n"
|
|
"movdqa %%xmm4,%%xmm1 \n"
|
|
"movdqa %%xmm6,%%xmm2 \n"
|
|
"movdqa %%xmm6,%%xmm3 \n"
|
|
"punpcklqdq %%xmm5,%%xmm0 \n" // a e i m from row 0, 1
|
|
"punpckhqdq %%xmm5,%%xmm1 \n" // b f j n from row 0, 1
|
|
"punpcklqdq %%xmm7,%%xmm2 \n" // c g k o from row 2, 3
|
|
"punpckhqdq %%xmm7,%%xmm3 \n" // d h l p from row 2, 3
|
|
|
|
"movdqu %%xmm0,(%1) \n"
|
|
"lea 16(%1,%4),%1 \n" // dst += stride + 16
|
|
"movdqu %%xmm1,-16(%1) \n"
|
|
"movdqu %%xmm2,-16(%1,%4) \n"
|
|
"movdqu %%xmm3,-16(%1,%4,2) \n"
|
|
"sub %4,%1 \n"
|
|
"sub $0x4,%2 \n"
|
|
"jg 1b \n"
|
|
: "+r"(src), // %0
|
|
"+r"(dst), // %1
|
|
"+rm"(width) // %2
|
|
: "r"((ptrdiff_t)(src_stride)), // %3
|
|
"r"((ptrdiff_t)(dst_stride)) // %4
|
|
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
|
"xmm7");
|
|
}
|
|
#endif // defined(HAS_TRANSPOSE4X4_32_SSE2)
|
|
|
|
#if defined(HAS_TRANSPOSENX4_32_AVX2)
|
|
// Transpose 32 bit values (ARGB) in Nx4 tiles with 2 blocks of 4x4 at a time
|
|
// in ymm0..ymm3 and vpmaskmovd tail for arbitrary N.
|
|
void TransposeNx4_32_AVX2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t d = dst_stride;
|
|
uint8_t* dst1 = dst + d;
|
|
uint8_t* dst2 = dst1 + d;
|
|
uint8_t* dst3 = dst2 + d;
|
|
asm volatile(
|
|
"sub $0x8,%[w] \n"
|
|
"jl 2f \n"
|
|
|
|
// Main loop: 2 blocks of 4x4 (8 source rows -> 8 dest columns / 32 B).
|
|
"1: \n"
|
|
"vmovdqu (%[src]),%%xmm0 \n" // a b c d
|
|
"vmovdqu (%[src],%[s]),%%xmm1 \n" // e f g h
|
|
"lea (%[src],%[s],2),%[src] \n" // src += stride * 2
|
|
"vmovdqu (%[src]),%%xmm2 \n" // i j k l
|
|
"vmovdqu (%[src],%[s]),%%xmm3 \n" // m n o p
|
|
"lea (%[src],%[s],2),%[src] \n" // src += stride * 2
|
|
|
|
"vinserti128 $1,(%[src]),%%ymm0,%%ymm0 \n"
|
|
"vinserti128 $1,(%[src],%[s]),%%ymm1,%%ymm1 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vinserti128 $1,(%[src]),%%ymm2,%%ymm2 \n"
|
|
"vinserti128 $1,(%[src],%[s]),%%ymm3,%%ymm3 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
|
|
// Transpose 2x2 across both 128-bit lanes
|
|
"vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n"
|
|
"vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n"
|
|
"vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n"
|
|
"vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n"
|
|
|
|
// Transpose 4x4 across both 128-bit lanes
|
|
"vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n"
|
|
"vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n"
|
|
"vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n"
|
|
"vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n"
|
|
|
|
"vmovdqu %%ymm0,(%[dst0]) \n"
|
|
"vmovdqu %%ymm1,(%[dst1]) \n"
|
|
"vmovdqu %%ymm2,(%[dst2]) \n"
|
|
"vmovdqu %%ymm3,(%[dst3]) \n"
|
|
"add $0x20,%[dst0] \n"
|
|
"add $0x20,%[dst1] \n"
|
|
"add $0x20,%[dst2] \n"
|
|
"add $0x20,%[dst3] \n"
|
|
"sub $0x8,%[w] \n"
|
|
"jge 1b \n"
|
|
|
|
"2: \n"
|
|
"add $0x8,%[w] \n"
|
|
"jz 5f \n"
|
|
|
|
// 4x4 block for 4..7 remainder elements
|
|
"cmp $0x4,%[w] \n"
|
|
"jl 3f \n"
|
|
"vmovdqu (%[src]),%%xmm0 \n"
|
|
"vmovdqu (%[src],%[s]),%%xmm1 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vmovdqu (%[src]),%%xmm2 \n"
|
|
"vmovdqu (%[src],%[s]),%%xmm3 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
|
|
"vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
|
|
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
|
|
"vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n"
|
|
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
|
|
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
|
|
"vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n"
|
|
"vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n"
|
|
"vmovdqu %%xmm0,(%[dst0]) \n"
|
|
"vmovdqu %%xmm1,(%[dst1]) \n"
|
|
"vmovdqu %%xmm2,(%[dst2]) \n"
|
|
"vmovdqu %%xmm3,(%[dst3]) \n"
|
|
"add $0x10,%[dst0] \n"
|
|
"add $0x10,%[dst1] \n"
|
|
"add $0x10,%[dst2] \n"
|
|
"add $0x10,%[dst3] \n"
|
|
"sub $0x4,%[w] \n"
|
|
"jz 5f \n"
|
|
|
|
// Masked tail for 1..3 remainder elements using vpmaskmovd
|
|
"3: \n"
|
|
"vmovdqu (%[src]),%%xmm0 \n"
|
|
"vmovdqa %%xmm0,%%xmm1 \n"
|
|
"vmovdqa %%xmm0,%%xmm2 \n"
|
|
"mov $0x000000ff,%%eax \n"
|
|
"cmp $2,%[w] \n"
|
|
"jl 4f \n"
|
|
"vmovdqu (%[src],%[s]),%%xmm1 \n"
|
|
"mov $0x0000ffff,%%eax \n"
|
|
"je 4f \n"
|
|
"vmovdqu (%[src],%[s],2),%%xmm2 \n"
|
|
"mov $0x00ffffff,%%eax \n"
|
|
"4: \n"
|
|
"vmovd %%eax,%%xmm7 \n"
|
|
"vpmovsxbd %%xmm7,%%xmm7 \n"
|
|
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
|
|
"vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n"
|
|
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
|
|
"vpunpckhdq %%xmm2,%%xmm2,%%xmm3 \n"
|
|
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
|
|
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
|
|
"vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n"
|
|
"vpunpckhqdq %%xmm3,%%xmm6,%%xmm3 \n"
|
|
"vpmaskmovd %%xmm0,%%xmm7,(%[dst0]) \n"
|
|
"vpmaskmovd %%xmm1,%%xmm7,(%[dst1]) \n"
|
|
"vpmaskmovd %%xmm2,%%xmm7,(%[dst2]) \n"
|
|
"vpmaskmovd %%xmm3,%%xmm7,(%[dst3]) \n"
|
|
"5: \n"
|
|
"vzeroupper \n"
|
|
: [src] "+r"(src), // %[src]
|
|
[dst0] "+r"(dst), // %[dst0]
|
|
[dst1] "+r"(dst1), // %[dst1]
|
|
[dst2] "+r"(dst2), // %[dst2]
|
|
[dst3] "+r"(dst3), // %[dst3]
|
|
[w] "+r"(width) // %[w]
|
|
: [s] "r"(s) // %[s]
|
|
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
|
|
"xmm6", "xmm7");
|
|
}
|
|
#endif // defined(HAS_TRANSPOSENX4_32_AVX2)
|
|
|
|
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
|
|
// Transpose 32 bit plane of any width x height using AVX2 vpmaskmovd.
|
|
void TransposeWxH_32_AVX2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width,
|
|
int height) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t d = dst_stride;
|
|
while (height >= 8) {
|
|
const uint8_t* src_row = src;
|
|
uint8_t* dst_col = dst;
|
|
int w = width;
|
|
while (w >= 8) {
|
|
asm volatile(
|
|
"vmovdqu (%[s0]),%%ymm0 \n"
|
|
"vmovdqu (%[s0],%[s]),%%ymm1 \n"
|
|
"lea (%[s0],%[s],2),%%rax \n"
|
|
"vmovdqu (%%rax),%%ymm2 \n"
|
|
"vmovdqu (%%rax,%[s]),%%ymm3 \n"
|
|
"lea (%%rax,%[s],2),%%rax \n"
|
|
"vmovdqu (%%rax),%%ymm4 \n"
|
|
"vmovdqu (%%rax,%[s]),%%ymm5 \n"
|
|
"lea (%%rax,%[s],2),%%rax \n"
|
|
"vmovdqu (%%rax),%%ymm6 \n"
|
|
"vmovdqu (%%rax,%[s]),%%ymm7 \n"
|
|
|
|
"vpunpckldq %%ymm1,%%ymm0,%%ymm8 \n"
|
|
"vpunpckhdq %%ymm1,%%ymm0,%%ymm9 \n"
|
|
"vpunpckldq %%ymm3,%%ymm2,%%ymm10 \n"
|
|
"vpunpckhdq %%ymm3,%%ymm2,%%ymm11 \n"
|
|
"vpunpckldq %%ymm5,%%ymm4,%%ymm12 \n"
|
|
"vpunpckhdq %%ymm5,%%ymm4,%%ymm13 \n"
|
|
"vpunpckldq %%ymm7,%%ymm6,%%ymm14 \n"
|
|
"vpunpckhdq %%ymm7,%%ymm6,%%ymm15 \n"
|
|
|
|
"vpunpcklqdq %%ymm10,%%ymm8,%%ymm0 \n"
|
|
"vpunpckhqdq %%ymm10,%%ymm8,%%ymm1 \n"
|
|
"vpunpcklqdq %%ymm11,%%ymm9,%%ymm2 \n"
|
|
"vpunpckhqdq %%ymm11,%%ymm9,%%ymm3 \n"
|
|
"vpunpcklqdq %%ymm14,%%ymm12,%%ymm4 \n"
|
|
"vpunpckhqdq %%ymm14,%%ymm12,%%ymm5 \n"
|
|
"vpunpcklqdq %%ymm15,%%ymm13,%%ymm6 \n"
|
|
"vpunpckhqdq %%ymm15,%%ymm13,%%ymm7 \n"
|
|
|
|
"vperm2i128 $0x20,%%ymm4,%%ymm0,%%ymm8 \n"
|
|
"vperm2i128 $0x20,%%ymm5,%%ymm1,%%ymm9 \n"
|
|
"vperm2i128 $0x20,%%ymm6,%%ymm2,%%ymm10 \n"
|
|
"vperm2i128 $0x20,%%ymm7,%%ymm3,%%ymm11 \n"
|
|
"vperm2i128 $0x31,%%ymm4,%%ymm0,%%ymm12 \n"
|
|
"vperm2i128 $0x31,%%ymm5,%%ymm1,%%ymm13 \n"
|
|
"vperm2i128 $0x31,%%ymm6,%%ymm2,%%ymm14 \n"
|
|
"vperm2i128 $0x31,%%ymm7,%%ymm3,%%ymm15 \n"
|
|
|
|
"vmovdqu %%ymm8,(%[d0]) \n"
|
|
"vmovdqu %%ymm9,(%[d0],%[d]) \n"
|
|
"lea (%[d0],%[d],2),%%rax \n"
|
|
"vmovdqu %%ymm10,(%%rax) \n"
|
|
"vmovdqu %%ymm11,(%%rax,%[d]) \n"
|
|
"lea (%%rax,%[d],2),%%rax \n"
|
|
"vmovdqu %%ymm12,(%%rax) \n"
|
|
"vmovdqu %%ymm13,(%%rax,%[d]) \n"
|
|
"lea (%%rax,%[d],2),%%rax \n"
|
|
"vmovdqu %%ymm14,(%%rax) \n"
|
|
"vmovdqu %%ymm15,(%%rax,%[d]) \n"
|
|
:
|
|
: [s0] "r"(src_row), // %[s0]
|
|
[d0] "r"(dst_col), // %[d0]
|
|
[s] "r"(s), // %[s]
|
|
[d] "r"(d) // %[d]
|
|
: "memory", "cc", "rax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
|
|
"xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
|
|
"xmm13", "xmm14", "xmm15");
|
|
src_row += 32;
|
|
dst_col += 8 * d;
|
|
w -= 8;
|
|
}
|
|
if (w >= 4) {
|
|
TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, 8);
|
|
src_row += 16;
|
|
dst_col += 4 * d;
|
|
w -= 4;
|
|
}
|
|
if (w > 0) {
|
|
TransposeWxH_32_C(src_row, src_stride, dst_col, dst_stride, w, 8);
|
|
}
|
|
src += 8 * s;
|
|
dst += 32;
|
|
height -= 8;
|
|
}
|
|
asm volatile("vzeroupper \n");
|
|
if (height > 0) {
|
|
while (width >= 4) {
|
|
TransposeNx4_32_AVX2(src, src_stride, dst, dst_stride, height);
|
|
src += 16;
|
|
dst += 4 * d;
|
|
width -= 4;
|
|
}
|
|
if (width > 0) {
|
|
const uint8_t* src_rem = src;
|
|
uint8_t* dst0 = dst;
|
|
uint8_t* dst1 = (width > 1) ? dst0 + d : dst0;
|
|
uint8_t* dst2 = (width > 2) ? dst1 + d : dst1;
|
|
int col_mask_bits = (1 << (width * 8)) - 1;
|
|
int h_rem = height;
|
|
asm volatile(
|
|
"vmovd %[col_mask],%%xmm7 \n"
|
|
"vpmovsxbd %%xmm7,%%xmm7 \n"
|
|
"sub $0x4,%[h] \n"
|
|
"jl 2f \n"
|
|
"1: \n"
|
|
"vpmaskmovd (%[src]),%%xmm7,%%xmm0 \n"
|
|
"vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm1 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vpmaskmovd (%[src]),%%xmm7,%%xmm2 \n"
|
|
"vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm3 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
|
|
"vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
|
|
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
|
|
"vpunpckhdq %%xmm3,%%xmm2,%%xmm3 \n"
|
|
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
|
|
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
|
|
"vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n"
|
|
"vmovdqu %%xmm2,(%[dst2]) \n"
|
|
"vmovdqu %%xmm1,(%[dst1]) \n"
|
|
"vmovdqu %%xmm0,(%[dst0]) \n"
|
|
"add $0x10,%[dst2] \n"
|
|
"add $0x10,%[dst1] \n"
|
|
"add $0x10,%[dst0] \n"
|
|
"sub $0x4,%[h] \n"
|
|
"jge 1b \n"
|
|
"2: \n"
|
|
"add $0x4,%[h] \n"
|
|
"jz 4f \n"
|
|
"vpmaskmovd (%[src]),%%xmm7,%%xmm0 \n"
|
|
"vmovdqa %%xmm0,%%xmm1 \n"
|
|
"vmovdqa %%xmm0,%%xmm2 \n"
|
|
"mov $0x000000ff,%%eax \n"
|
|
"cmp $2,%[h] \n"
|
|
"jl 3f \n"
|
|
"vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm1 \n"
|
|
"mov $0x0000ffff,%%eax \n"
|
|
"je 3f \n"
|
|
"vpmaskmovd (%[src],%[s],2),%%xmm7,%%xmm2 \n"
|
|
"mov $0x00ffffff,%%eax \n"
|
|
"3: \n"
|
|
"vmovd %%eax,%%xmm7 \n"
|
|
"vpmovsxbd %%xmm7,%%xmm7 \n"
|
|
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
|
|
"vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n"
|
|
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
|
|
"vpunpckhdq %%xmm2,%%xmm2,%%xmm3 \n"
|
|
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
|
|
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
|
|
"vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n"
|
|
"vpmaskmovd %%xmm2,%%xmm7,(%[dst2]) \n"
|
|
"vpmaskmovd %%xmm1,%%xmm7,(%[dst1]) \n"
|
|
"vpmaskmovd %%xmm0,%%xmm7,(%[dst0]) \n"
|
|
"4: \n"
|
|
"vzeroupper \n"
|
|
: [src] "+r"(src_rem), // %[src]
|
|
[dst0] "+r"(dst0), // %[dst0]
|
|
[dst1] "+r"(dst1), // %[dst1]
|
|
[dst2] "+r"(dst2), // %[dst2]
|
|
[h] "+r"(h_rem) // %[h]
|
|
: [s] "r"(s), // %[s]
|
|
[col_mask] "r"(col_mask_bits) // %[col_mask]
|
|
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
|
|
"xmm5", "xmm6", "xmm7");
|
|
}
|
|
}
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
|
|
|
|
#if defined(HAS_TRANSPOSENX4_32_AVX512BW)
|
|
// Transpose 32 bit values (ARGB) in Nx4 tiles with 4 blocks of 4x4 at a time
|
|
// in zmm0..zmm3 and masked tail for arbitrary N.
|
|
void TransposeNx4_32_AVX512BW(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t d = dst_stride;
|
|
uint8_t* dst1 = dst + d;
|
|
uint8_t* dst2 = dst1 + d;
|
|
uint8_t* dst3 = dst2 + d;
|
|
asm volatile(
|
|
"sub $0x10,%[w] \n"
|
|
"jl 2f \n"
|
|
|
|
// Main loop: 4 blocks of 4x4 (16 source rows -> 16 dest columns / 64 B).
|
|
"1: \n"
|
|
"vmovdqu (%[src]),%%xmm0 \n"
|
|
"vmovdqu (%[src],%[s]),%%xmm1 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vmovdqu (%[src]),%%xmm2 \n"
|
|
"vmovdqu (%[src],%[s]),%%xmm3 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
|
|
"vinserti32x4 $1,(%[src]),%%zmm0,%%zmm0 \n"
|
|
"vinserti32x4 $1,(%[src],%[s]),%%zmm1,%%zmm1 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vinserti32x4 $1,(%[src]),%%zmm2,%%zmm2 \n"
|
|
"vinserti32x4 $1,(%[src],%[s]),%%zmm3,%%zmm3 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
|
|
"vinserti32x4 $2,(%[src]),%%zmm0,%%zmm0 \n"
|
|
"vinserti32x4 $2,(%[src],%[s]),%%zmm1,%%zmm1 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vinserti32x4 $2,(%[src]),%%zmm2,%%zmm2 \n"
|
|
"vinserti32x4 $2,(%[src],%[s]),%%zmm3,%%zmm3 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
|
|
"vinserti32x4 $3,(%[src]),%%zmm0,%%zmm0 \n"
|
|
"vinserti32x4 $3,(%[src],%[s]),%%zmm1,%%zmm1 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vinserti32x4 $3,(%[src]),%%zmm2,%%zmm2 \n"
|
|
"vinserti32x4 $3,(%[src],%[s]),%%zmm3,%%zmm3 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
|
|
// Transpose 2x2 across all 4 128-bit lanes
|
|
"vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n"
|
|
"vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n"
|
|
"vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n"
|
|
"vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n"
|
|
|
|
// Transpose 4x4 across all 4 128-bit lanes
|
|
"vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n"
|
|
"vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n"
|
|
"vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n"
|
|
"vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n"
|
|
|
|
"vmovdqu32 %%zmm0,(%[dst0]) \n"
|
|
"vmovdqu32 %%zmm1,(%[dst1]) \n"
|
|
"vmovdqu32 %%zmm2,(%[dst2]) \n"
|
|
"vmovdqu32 %%zmm3,(%[dst3]) \n"
|
|
"add $0x40,%[dst0] \n"
|
|
"add $0x40,%[dst1] \n"
|
|
"add $0x40,%[dst2] \n"
|
|
"add $0x40,%[dst3] \n"
|
|
"sub $0x10,%[w] \n"
|
|
"jge 1b \n"
|
|
|
|
"2: \n"
|
|
"add $0x10,%[w] \n"
|
|
"jz 5f \n"
|
|
|
|
// 4x4 loop for 4..15 remainder elements
|
|
"sub $0x4,%[w] \n"
|
|
"jl 4f \n"
|
|
"3: \n"
|
|
"vmovdqu (%[src]),%%xmm0 \n"
|
|
"vmovdqu (%[src],%[s]),%%xmm1 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vmovdqu (%[src]),%%xmm2 \n"
|
|
"vmovdqu (%[src],%[s]),%%xmm3 \n"
|
|
"lea (%[src],%[s],2),%[src] \n"
|
|
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
|
|
"vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
|
|
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
|
|
"vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n"
|
|
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
|
|
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
|
|
"vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n"
|
|
"vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n"
|
|
"vmovdqu %%xmm0,(%[dst0]) \n"
|
|
"vmovdqu %%xmm1,(%[dst1]) \n"
|
|
"vmovdqu %%xmm2,(%[dst2]) \n"
|
|
"vmovdqu %%xmm3,(%[dst3]) \n"
|
|
"add $0x10,%[dst0] \n"
|
|
"add $0x10,%[dst1] \n"
|
|
"add $0x10,%[dst2] \n"
|
|
"add $0x10,%[dst3] \n"
|
|
"sub $0x4,%[w] \n"
|
|
"jge 3b \n"
|
|
|
|
"4: \n"
|
|
"add $0x4,%[w] \n"
|
|
"jz 5f \n"
|
|
|
|
// Masked tail for 1..3 remainder elements (in-bounds row loads + k1 mask)
|
|
"mov $1,%%eax \n"
|
|
"shlx %[w],%%eax,%%eax \n"
|
|
"dec %%eax \n"
|
|
"kmovw %%eax,%%k1 \n"
|
|
"vmovdqu (%[src]),%%xmm0 \n"
|
|
"vmovdqa %%xmm0,%%xmm1 \n"
|
|
"vmovdqa %%xmm0,%%xmm2 \n"
|
|
"cmp $1,%[w] \n"
|
|
"jle 6f \n"
|
|
"vmovdqu (%[src],%[s]),%%xmm1 \n"
|
|
"cmp $2,%[w] \n"
|
|
"jle 6f \n"
|
|
"vmovdqu (%[src],%[s],2),%%xmm2 \n"
|
|
"6: \n"
|
|
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
|
|
"vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n"
|
|
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
|
|
"vpunpckhdq %%xmm2,%%xmm2,%%xmm7 \n"
|
|
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
|
|
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
|
|
"vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n"
|
|
"vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n"
|
|
"vmovdqu32 %%xmm0,(%[dst0])%{%%k1%} \n"
|
|
"vmovdqu32 %%xmm1,(%[dst1])%{%%k1%} \n"
|
|
"vmovdqu32 %%xmm2,(%[dst2])%{%%k1%} \n"
|
|
"vmovdqu32 %%xmm3,(%[dst3])%{%%k1%} \n"
|
|
"5: \n"
|
|
"vzeroupper \n"
|
|
: [src] "+r"(src), // %[src]
|
|
[dst0] "+r"(dst), // %[dst0]
|
|
[dst1] "+r"(dst1), // %[dst1]
|
|
[dst2] "+r"(dst2), // %[dst2]
|
|
[dst3] "+r"(dst3), // %[dst3]
|
|
[w] "+r"(width) // %[w]
|
|
: [s] "r"(s) // %[s]
|
|
: "memory", "cc", "eax", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
|
|
"xmm5", "xmm6", "xmm7");
|
|
}
|
|
#endif // defined(HAS_TRANSPOSENX4_32_AVX512BW)
|
|
|
|
#if defined(HAS_TRANSPOSEWXH_32_AVX512BW)
|
|
// Transpose 32 bit plane of any width x height using 16x16 ZMM horizontal tiles
|
|
// in zmm0..zmm31 with k-masked right and bottom tails.
|
|
void TransposeWxH_32_AVX512BW(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width,
|
|
int height) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t d = dst_stride;
|
|
while (height > 0) {
|
|
const int bh = (height >= 16) ? 16 : height;
|
|
const uint8_t* src_row = src;
|
|
uint8_t* dst_col = dst;
|
|
int w = width;
|
|
if (bh == 16) {
|
|
while (w >= 16) {
|
|
const uint8_t* s_ptr = src_row;
|
|
uint8_t* d_ptr = dst_col;
|
|
asm volatile(
|
|
"vmovdqu32 (%[s_ptr]),%%zmm0 \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm1 \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm2 \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm3 \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm4 \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm5 \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm6 \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm7 \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm8 \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm9 \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm10 \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm11 \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm12 \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm13 \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm14 \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm15 \n"
|
|
|
|
// Stage 1: 32-bit unpack
|
|
"vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n"
|
|
"vpunpckhdq %%zmm1,%%zmm0,%%zmm17 \n"
|
|
"vpunpckldq %%zmm3,%%zmm2,%%zmm18 \n"
|
|
"vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n"
|
|
"vpunpckldq %%zmm5,%%zmm4,%%zmm20 \n"
|
|
"vpunpckhdq %%zmm5,%%zmm4,%%zmm21 \n"
|
|
"vpunpckldq %%zmm7,%%zmm6,%%zmm22 \n"
|
|
"vpunpckhdq %%zmm7,%%zmm6,%%zmm23 \n"
|
|
"vpunpckldq %%zmm9,%%zmm8,%%zmm24 \n"
|
|
"vpunpckhdq %%zmm9,%%zmm8,%%zmm25 \n"
|
|
"vpunpckldq %%zmm11,%%zmm10,%%zmm26 \n"
|
|
"vpunpckhdq %%zmm11,%%zmm10,%%zmm27 \n"
|
|
"vpunpckldq %%zmm13,%%zmm12,%%zmm28 \n"
|
|
"vpunpckhdq %%zmm13,%%zmm12,%%zmm29 \n"
|
|
"vpunpckldq %%zmm15,%%zmm14,%%zmm30 \n"
|
|
"vpunpckhdq %%zmm15,%%zmm14,%%zmm31 \n"
|
|
|
|
// Stage 2: 64-bit unpack
|
|
"vpunpcklqdq %%zmm18,%%zmm16,%%zmm0 \n"
|
|
"vpunpckhqdq %%zmm18,%%zmm16,%%zmm1 \n"
|
|
"vpunpcklqdq %%zmm19,%%zmm17,%%zmm2 \n"
|
|
"vpunpckhqdq %%zmm19,%%zmm17,%%zmm3 \n"
|
|
"vpunpcklqdq %%zmm22,%%zmm20,%%zmm4 \n"
|
|
"vpunpckhqdq %%zmm22,%%zmm20,%%zmm5 \n"
|
|
"vpunpcklqdq %%zmm23,%%zmm21,%%zmm6 \n"
|
|
"vpunpckhqdq %%zmm23,%%zmm21,%%zmm7 \n"
|
|
"vpunpcklqdq %%zmm26,%%zmm24,%%zmm8 \n"
|
|
"vpunpckhqdq %%zmm26,%%zmm24,%%zmm9 \n"
|
|
"vpunpcklqdq %%zmm27,%%zmm25,%%zmm10 \n"
|
|
"vpunpckhqdq %%zmm27,%%zmm25,%%zmm11 \n"
|
|
"vpunpcklqdq %%zmm30,%%zmm28,%%zmm12 \n"
|
|
"vpunpckhqdq %%zmm30,%%zmm28,%%zmm13 \n"
|
|
"vpunpcklqdq %%zmm31,%%zmm29,%%zmm14 \n"
|
|
"vpunpckhqdq %%zmm31,%%zmm29,%%zmm15 \n"
|
|
|
|
// Stage 3: 128-bit lane shuffle
|
|
"vshufi32x4 $0x88,%%zmm4,%%zmm0,%%zmm16 \n"
|
|
"vshufi32x4 $0x88,%%zmm5,%%zmm1,%%zmm17 \n"
|
|
"vshufi32x4 $0x88,%%zmm6,%%zmm2,%%zmm18 \n"
|
|
"vshufi32x4 $0x88,%%zmm7,%%zmm3,%%zmm19 \n"
|
|
"vshufi32x4 $0xdd,%%zmm4,%%zmm0,%%zmm20 \n"
|
|
"vshufi32x4 $0xdd,%%zmm5,%%zmm1,%%zmm21 \n"
|
|
"vshufi32x4 $0xdd,%%zmm6,%%zmm2,%%zmm22 \n"
|
|
"vshufi32x4 $0xdd,%%zmm7,%%zmm3,%%zmm23 \n"
|
|
"vshufi32x4 $0x88,%%zmm12,%%zmm8,%%zmm24 \n"
|
|
"vshufi32x4 $0x88,%%zmm13,%%zmm9,%%zmm25 \n"
|
|
"vshufi32x4 $0x88,%%zmm14,%%zmm10,%%zmm26 \n"
|
|
"vshufi32x4 $0x88,%%zmm15,%%zmm11,%%zmm27 \n"
|
|
"vshufi32x4 $0xdd,%%zmm12,%%zmm8,%%zmm28 \n"
|
|
"vshufi32x4 $0xdd,%%zmm13,%%zmm9,%%zmm29 \n"
|
|
"vshufi32x4 $0xdd,%%zmm14,%%zmm10,%%zmm30 \n"
|
|
"vshufi32x4 $0xdd,%%zmm15,%%zmm11,%%zmm31 \n"
|
|
|
|
// Stage 4: 256-bit half shuffle -> columns 0..15 in zmm0..zmm15
|
|
"vshufi32x4 $0x88,%%zmm24,%%zmm16,%%zmm0 \n"
|
|
"vshufi32x4 $0x88,%%zmm25,%%zmm17,%%zmm1 \n"
|
|
"vshufi32x4 $0x88,%%zmm26,%%zmm18,%%zmm2 \n"
|
|
"vshufi32x4 $0x88,%%zmm27,%%zmm19,%%zmm3 \n"
|
|
"vshufi32x4 $0x88,%%zmm28,%%zmm20,%%zmm4 \n"
|
|
"vshufi32x4 $0x88,%%zmm29,%%zmm21,%%zmm5 \n"
|
|
"vshufi32x4 $0x88,%%zmm30,%%zmm22,%%zmm6 \n"
|
|
"vshufi32x4 $0x88,%%zmm31,%%zmm23,%%zmm7 \n"
|
|
"vshufi32x4 $0xdd,%%zmm24,%%zmm16,%%zmm8 \n"
|
|
"vshufi32x4 $0xdd,%%zmm25,%%zmm17,%%zmm9 \n"
|
|
"vshufi32x4 $0xdd,%%zmm26,%%zmm18,%%zmm10 \n"
|
|
"vshufi32x4 $0xdd,%%zmm27,%%zmm19,%%zmm11 \n"
|
|
"vshufi32x4 $0xdd,%%zmm28,%%zmm20,%%zmm12 \n"
|
|
"vshufi32x4 $0xdd,%%zmm29,%%zmm21,%%zmm13 \n"
|
|
"vshufi32x4 $0xdd,%%zmm30,%%zmm22,%%zmm14 \n"
|
|
"vshufi32x4 $0xdd,%%zmm31,%%zmm23,%%zmm15 \n"
|
|
|
|
"vmovdqu32 %%zmm0,(%[d_ptr]) \n"
|
|
"vmovdqu32 %%zmm1,(%[d_ptr],%[d]) \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm2,(%[d_ptr]) \n"
|
|
"vmovdqu32 %%zmm3,(%[d_ptr],%[d]) \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm4,(%[d_ptr]) \n"
|
|
"vmovdqu32 %%zmm5,(%[d_ptr],%[d]) \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm6,(%[d_ptr]) \n"
|
|
"vmovdqu32 %%zmm7,(%[d_ptr],%[d]) \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm8,(%[d_ptr]) \n"
|
|
"vmovdqu32 %%zmm9,(%[d_ptr],%[d]) \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm10,(%[d_ptr]) \n"
|
|
"vmovdqu32 %%zmm11,(%[d_ptr],%[d]) \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm12,(%[d_ptr]) \n"
|
|
"vmovdqu32 %%zmm13,(%[d_ptr],%[d]) \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm14,(%[d_ptr]) \n"
|
|
"vmovdqu32 %%zmm15,(%[d_ptr],%[d]) \n"
|
|
: [s_ptr] "+r"(s_ptr), // %[s_ptr]
|
|
[d_ptr] "+r"(d_ptr) // %[d_ptr]
|
|
: [s] "r"(s), // %[s]
|
|
[d] "r"(d) // %[d]
|
|
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
|
|
"xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
|
|
"xmm13", "xmm14", "xmm15", "xmm16", "xmm17", "xmm18", "xmm19",
|
|
"xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26",
|
|
"xmm27", "xmm28", "xmm29", "xmm30", "xmm31");
|
|
src_row += 64;
|
|
dst_col += 16 * d;
|
|
w -= 16;
|
|
}
|
|
}
|
|
const uint32_t row_mask = (1u << bh) - 1;
|
|
while (w > 0) {
|
|
const int bw = (w >= 16) ? 16 : w;
|
|
const uint32_t col_mask = (1u << bw) - 1;
|
|
const uint8_t* s_ptr = src_row;
|
|
uint8_t* d_ptr = dst_col;
|
|
asm volatile(
|
|
"kmovw %[col_mask],%%k1 \n"
|
|
"kmovw %[row_mask],%%k2 \n"
|
|
"vpxord %%zmm0,%%zmm0,%%zmm0 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm1 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm2 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm3 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm4 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm5 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm6 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm7 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm8 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm9 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm10 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm11 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm12 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm13 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm14 \n"
|
|
"vmovdqa64 %%zmm0,%%zmm15 \n"
|
|
|
|
"vmovdqu32 (%[s_ptr]),%%zmm0%{%%k1%}%{z%} \n"
|
|
"cmp $1,%[bh] \n"
|
|
"jle 1f \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm1%{%%k1%}%{z%} \n"
|
|
"cmp $2,%[bh] \n"
|
|
"jle 1f \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm2%{%%k1%}%{z%} \n"
|
|
"cmp $3,%[bh] \n"
|
|
"jle 1f \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm3%{%%k1%}%{z%} \n"
|
|
"cmp $4,%[bh] \n"
|
|
"jle 1f \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm4%{%%k1%}%{z%} \n"
|
|
"cmp $5,%[bh] \n"
|
|
"jle 1f \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm5%{%%k1%}%{z%} \n"
|
|
"cmp $6,%[bh] \n"
|
|
"jle 1f \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm6%{%%k1%}%{z%} \n"
|
|
"cmp $7,%[bh] \n"
|
|
"jle 1f \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm7%{%%k1%}%{z%} \n"
|
|
"cmp $8,%[bh] \n"
|
|
"jle 1f \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm8%{%%k1%}%{z%} \n"
|
|
"cmp $9,%[bh] \n"
|
|
"jle 1f \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm9%{%%k1%}%{z%} \n"
|
|
"cmp $10,%[bh] \n"
|
|
"jle 1f \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm10%{%%k1%}%{z%} \n"
|
|
"cmp $11,%[bh] \n"
|
|
"jle 1f \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm11%{%%k1%}%{z%} \n"
|
|
"cmp $12,%[bh] \n"
|
|
"jle 1f \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm12%{%%k1%}%{z%} \n"
|
|
"cmp $13,%[bh] \n"
|
|
"jle 1f \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm13%{%%k1%}%{z%} \n"
|
|
"cmp $14,%[bh] \n"
|
|
"jle 1f \n"
|
|
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
|
|
"vmovdqu32 (%[s_ptr]),%%zmm14%{%%k1%}%{z%} \n"
|
|
"cmp $15,%[bh] \n"
|
|
"jle 1f \n"
|
|
"vmovdqu32 (%[s_ptr],%[s]),%%zmm15%{%%k1%}%{z%} \n"
|
|
"1: \n"
|
|
|
|
// Stage 1: 32-bit unpack
|
|
"vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n"
|
|
"vpunpckhdq %%zmm1,%%zmm0,%%zmm17 \n"
|
|
"vpunpckldq %%zmm3,%%zmm2,%%zmm18 \n"
|
|
"vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n"
|
|
"vpunpckldq %%zmm5,%%zmm4,%%zmm20 \n"
|
|
"vpunpckhdq %%zmm5,%%zmm4,%%zmm21 \n"
|
|
"vpunpckldq %%zmm7,%%zmm6,%%zmm22 \n"
|
|
"vpunpckhdq %%zmm7,%%zmm6,%%zmm23 \n"
|
|
"vpunpckldq %%zmm9,%%zmm8,%%zmm24 \n"
|
|
"vpunpckhdq %%zmm9,%%zmm8,%%zmm25 \n"
|
|
"vpunpckldq %%zmm11,%%zmm10,%%zmm26 \n"
|
|
"vpunpckhdq %%zmm11,%%zmm10,%%zmm27 \n"
|
|
"vpunpckldq %%zmm13,%%zmm12,%%zmm28 \n"
|
|
"vpunpckhdq %%zmm13,%%zmm12,%%zmm29 \n"
|
|
"vpunpckldq %%zmm15,%%zmm14,%%zmm30 \n"
|
|
"vpunpckhdq %%zmm15,%%zmm14,%%zmm31 \n"
|
|
|
|
// Stage 2: 64-bit unpack
|
|
"vpunpcklqdq %%zmm18,%%zmm16,%%zmm0 \n"
|
|
"vpunpckhqdq %%zmm18,%%zmm16,%%zmm1 \n"
|
|
"vpunpcklqdq %%zmm19,%%zmm17,%%zmm2 \n"
|
|
"vpunpckhqdq %%zmm19,%%zmm17,%%zmm3 \n"
|
|
"vpunpcklqdq %%zmm22,%%zmm20,%%zmm4 \n"
|
|
"vpunpckhqdq %%zmm22,%%zmm20,%%zmm5 \n"
|
|
"vpunpcklqdq %%zmm23,%%zmm21,%%zmm6 \n"
|
|
"vpunpckhqdq %%zmm23,%%zmm21,%%zmm7 \n"
|
|
"vpunpcklqdq %%zmm26,%%zmm24,%%zmm8 \n"
|
|
"vpunpckhqdq %%zmm26,%%zmm24,%%zmm9 \n"
|
|
"vpunpcklqdq %%zmm27,%%zmm25,%%zmm10 \n"
|
|
"vpunpckhqdq %%zmm27,%%zmm25,%%zmm11 \n"
|
|
"vpunpcklqdq %%zmm30,%%zmm28,%%zmm12 \n"
|
|
"vpunpckhqdq %%zmm30,%%zmm28,%%zmm13 \n"
|
|
"vpunpcklqdq %%zmm31,%%zmm29,%%zmm14 \n"
|
|
"vpunpckhqdq %%zmm31,%%zmm29,%%zmm15 \n"
|
|
|
|
// Stage 3: 128-bit lane shuffle
|
|
"vshufi32x4 $0x88,%%zmm4,%%zmm0,%%zmm16 \n"
|
|
"vshufi32x4 $0x88,%%zmm5,%%zmm1,%%zmm17 \n"
|
|
"vshufi32x4 $0x88,%%zmm6,%%zmm2,%%zmm18 \n"
|
|
"vshufi32x4 $0x88,%%zmm7,%%zmm3,%%zmm19 \n"
|
|
"vshufi32x4 $0xdd,%%zmm4,%%zmm0,%%zmm20 \n"
|
|
"vshufi32x4 $0xdd,%%zmm5,%%zmm1,%%zmm21 \n"
|
|
"vshufi32x4 $0xdd,%%zmm6,%%zmm2,%%zmm22 \n"
|
|
"vshufi32x4 $0xdd,%%zmm7,%%zmm3,%%zmm23 \n"
|
|
"vshufi32x4 $0x88,%%zmm12,%%zmm8,%%zmm24 \n"
|
|
"vshufi32x4 $0x88,%%zmm13,%%zmm9,%%zmm25 \n"
|
|
"vshufi32x4 $0x88,%%zmm14,%%zmm10,%%zmm26 \n"
|
|
"vshufi32x4 $0x88,%%zmm15,%%zmm11,%%zmm27 \n"
|
|
"vshufi32x4 $0xdd,%%zmm12,%%zmm8,%%zmm28 \n"
|
|
"vshufi32x4 $0xdd,%%zmm13,%%zmm9,%%zmm29 \n"
|
|
"vshufi32x4 $0xdd,%%zmm14,%%zmm10,%%zmm30 \n"
|
|
"vshufi32x4 $0xdd,%%zmm15,%%zmm11,%%zmm31 \n"
|
|
|
|
// Stage 4: 256-bit half shuffle -> columns 0..15 in zmm0..zmm15
|
|
"vshufi32x4 $0x88,%%zmm24,%%zmm16,%%zmm0 \n"
|
|
"vshufi32x4 $0x88,%%zmm25,%%zmm17,%%zmm1 \n"
|
|
"vshufi32x4 $0x88,%%zmm26,%%zmm18,%%zmm2 \n"
|
|
"vshufi32x4 $0x88,%%zmm27,%%zmm19,%%zmm3 \n"
|
|
"vshufi32x4 $0x88,%%zmm28,%%zmm20,%%zmm4 \n"
|
|
"vshufi32x4 $0x88,%%zmm29,%%zmm21,%%zmm5 \n"
|
|
"vshufi32x4 $0x88,%%zmm30,%%zmm22,%%zmm6 \n"
|
|
"vshufi32x4 $0x88,%%zmm31,%%zmm23,%%zmm7 \n"
|
|
"vshufi32x4 $0xdd,%%zmm24,%%zmm16,%%zmm8 \n"
|
|
"vshufi32x4 $0xdd,%%zmm25,%%zmm17,%%zmm9 \n"
|
|
"vshufi32x4 $0xdd,%%zmm26,%%zmm18,%%zmm10 \n"
|
|
"vshufi32x4 $0xdd,%%zmm27,%%zmm19,%%zmm11 \n"
|
|
"vshufi32x4 $0xdd,%%zmm28,%%zmm20,%%zmm12 \n"
|
|
"vshufi32x4 $0xdd,%%zmm29,%%zmm21,%%zmm13 \n"
|
|
"vshufi32x4 $0xdd,%%zmm30,%%zmm22,%%zmm14 \n"
|
|
"vshufi32x4 $0xdd,%%zmm31,%%zmm23,%%zmm15 \n"
|
|
|
|
"vmovdqu32 %%zmm0,(%[d_ptr])%{%%k2%} \n"
|
|
"cmp $1,%[bw] \n"
|
|
"jle 2f \n"
|
|
"vmovdqu32 %%zmm1,(%[d_ptr],%[d])%{%%k2%} \n"
|
|
"cmp $2,%[bw] \n"
|
|
"jle 2f \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm2,(%[d_ptr])%{%%k2%} \n"
|
|
"cmp $3,%[bw] \n"
|
|
"jle 2f \n"
|
|
"vmovdqu32 %%zmm3,(%[d_ptr],%[d])%{%%k2%} \n"
|
|
"cmp $4,%[bw] \n"
|
|
"jle 2f \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm4,(%[d_ptr])%{%%k2%} \n"
|
|
"cmp $5,%[bw] \n"
|
|
"jle 2f \n"
|
|
"vmovdqu32 %%zmm5,(%[d_ptr],%[d])%{%%k2%} \n"
|
|
"cmp $6,%[bw] \n"
|
|
"jle 2f \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm6,(%[d_ptr])%{%%k2%} \n"
|
|
"cmp $7,%[bw] \n"
|
|
"jle 2f \n"
|
|
"vmovdqu32 %%zmm7,(%[d_ptr],%[d])%{%%k2%} \n"
|
|
"cmp $8,%[bw] \n"
|
|
"jle 2f \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm8,(%[d_ptr])%{%%k2%} \n"
|
|
"cmp $9,%[bw] \n"
|
|
"jle 2f \n"
|
|
"vmovdqu32 %%zmm9,(%[d_ptr],%[d])%{%%k2%} \n"
|
|
"cmp $10,%[bw] \n"
|
|
"jle 2f \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm10,(%[d_ptr])%{%%k2%} \n"
|
|
"cmp $11,%[bw] \n"
|
|
"jle 2f \n"
|
|
"vmovdqu32 %%zmm11,(%[d_ptr],%[d])%{%%k2%} \n"
|
|
"cmp $12,%[bw] \n"
|
|
"jle 2f \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm12,(%[d_ptr])%{%%k2%} \n"
|
|
"cmp $13,%[bw] \n"
|
|
"jle 2f \n"
|
|
"vmovdqu32 %%zmm13,(%[d_ptr],%[d])%{%%k2%} \n"
|
|
"cmp $14,%[bw] \n"
|
|
"jle 2f \n"
|
|
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
|
|
"vmovdqu32 %%zmm14,(%[d_ptr])%{%%k2%} \n"
|
|
"cmp $15,%[bw] \n"
|
|
"jle 2f \n"
|
|
"vmovdqu32 %%zmm15,(%[d_ptr],%[d])%{%%k2%} \n"
|
|
"2: \n"
|
|
: [s_ptr] "+r"(s_ptr), // %[s_ptr]
|
|
[d_ptr] "+r"(d_ptr) // %[d_ptr]
|
|
: [s] "r"(s), // %[s]
|
|
[d] "r"(d), // %[d]
|
|
[col_mask] "r"(col_mask), // %[col_mask]
|
|
[row_mask] "r"(row_mask), // %[row_mask]
|
|
[bh] "r"(bh), // %[bh]
|
|
[bw] "r"(bw) // %[bw]
|
|
: "memory", "cc", "k1", "k2", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
|
|
"xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
|
|
"xmm13", "xmm14", "xmm15", "xmm16", "xmm17", "xmm18", "xmm19",
|
|
"xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26",
|
|
"xmm27", "xmm28", "xmm29", "xmm30", "xmm31");
|
|
src_row += 64;
|
|
dst_col += 16 * d;
|
|
w -= 16;
|
|
}
|
|
src += 16 * s;
|
|
dst += 64;
|
|
height -= 16;
|
|
}
|
|
asm volatile("vzeroupper \n");
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWXH_32_AVX512BW)
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) &&
|
|
// !defined(LIBYUV_ENABLE_ROWWIN)
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|