mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
16af162cf9
Implement SIMD 16-bit plane rotation (TransposePlane_16, RotatePlane90_16, RotatePlane180_16, RotatePlane270_16) used by I010Rotate, I210Rotate, and I410Rotate, keeping tile sizes compact with full-register writes and zero cross-lane shuffles: - AVX2: 8x16 tile in ymm0..ymm15 with _Any tail (low-end x86 SIMD) - AVX-512BW: 8x32 tile in zmm0..zmm15 with k1/k2 right/bottom masks - NEON (ARM64/ARM32): 8x16 tile in v0..v23 / q0..q11 with _Any tail - SME: 8xVL tile using ZA16 horizontal/vertical array slices - RVV: 8xVL tile using widening unpacks with hardware vl tail - WASM SIMD: 8x8 tile in v128 with _Any tail Also update 32-bit ARGBRotate x86 transposes to compact vertical in-lane tiles with full-register writes and zero cross-lane shuffles: - TransposeWxH_32_AVX2: 4x8 tile in ymm0..ymm7 with vpmaskmovd tails - TransposeWxH_32_AVX512BW: 4x16 tile in zmm0..zmm7 with k1/k2 tails Add RotatePlane16_SplitMergeUV_Opt and RotatePlane16_SplitMergeUV_Odd unit tests comparing 16-bit rotation against SplitUVPlane + RotatePlane + MergeUVPlane across 0/90/180/270 degrees. Performance on 1275x1650 (1000 iterations, before -> after): CPU 90 degree 180 degree Coral RVV (Verilator) 7306 -> 1415c (5.16x) 2274 -> 719c (3.16x) AMD Rome (Zen 2) 821 -> 215ms (3.82x) 259 -> 181ms (1.44x) AMD Genoa (Zen 4) 483 -> 129ms (3.73x) 149 -> 111ms (1.34x) AMD Turin (Zen 5) 321 -> 101ms (3.18x) 109 -> 78ms (1.40x) AMD Milan (Zen 3) 505 -> 189ms (2.67x) 199 -> 118ms (1.69x) Intel Skylake-X 1076 -> 447ms (2.41x) 353 -> 317ms (1.11x) WASM SIMD (d8) 415 -> 203ms (2.04x) 370 -> 106ms (3.49x) Intel Emerald Rapids 735 -> 422ms (1.74x) 238 -> 233ms (1.02x) Intel Haswell 1120 -> 648ms (1.73x) 438 -> 393ms (1.11x) Intel Ice Lake 774 -> 474ms (1.63x) 279 -> 290ms (0.96x) ARM Neoverse-N1 864 -> 544ms (1.59x) 404 -> 266ms (1.52x) Ampere Siryn 862 -> 576ms (1.50x) 568 -> 329ms (1.72x) Google Axion (Maple) 554 -> 379ms (1.46x) 199 -> 142ms (1.41x) Intel Sapphire Rapids 1126 -> 772ms (1.46x) 301 -> 288ms (1.04x) Intel Granite Rapids 730 -> 565ms (1.29x) 280 -> 280ms (1.00x) NVIDIA Grace 462 -> 365ms (1.26x) 192 -> 126ms (1.52x) ARM Cortex-A53 28290 -> 24386ms (1.16x) 7178 -> 5075ms (1.41x) Google Axion 2 (Tamar) 523 -> 453ms (1.15x) 217 -> 112ms (1.93x) Test: libyuv_unittest --gtest_filter=*Rotate* Bug: b/42280942 Change-Id: I91ba6cac22185b5d294fcb98344fdb8a23cc94ff Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8545294 Reviewed-by: richard winterton <rrwinterton@gmail.com> Commit-Queue: Frank Barchard <fbarchard@google.com>
902 lines
40 KiB
C++
902 lines
40 KiB
C++
/*
|
|
* Copyright 2013 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/rotate_row.h"
|
|
#include "libyuv/row.h"
|
|
|
|
// This module is for 32 bit x86 with any compiler, Visual C 64 bit, and for
|
|
// clang with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
|
|
#if !defined(LIBYUV_DISABLE_X86) && \
|
|
(defined(__i386__) || defined(_M_IX86) || \
|
|
((defined(__x86_64__) || defined(_M_X64)) && \
|
|
((defined(_MSC_VER) && !defined(__clang__)) || \
|
|
defined(LIBYUV_ENABLE_ROWWIN))))
|
|
|
|
#include <emmintrin.h>
|
|
#include <immintrin.h> // For AVX2 and AVX512 intrinsics
|
|
#if defined(_MSC_VER) && !defined(__clang__)
|
|
#include <intrin.h>
|
|
#endif
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
// Target attributes allow gcc and clang to build each function for its
|
|
// instruction set regardless of the baseline, which may lack SSE2 for 32 bit.
|
|
#if defined(__clang__) || defined(__GNUC__)
|
|
#define LIBYUV_TARGET_SSE2 __attribute__((target("sse2")))
|
|
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
|
|
#define LIBYUV_TARGET_AVX512BW \
|
|
__attribute__((target("avx512bw,avx512vl,avx512f")))
|
|
#else
|
|
#define LIBYUV_TARGET_SSE2
|
|
#define LIBYUV_TARGET_AVX2
|
|
#define LIBYUV_TARGET_AVX512BW
|
|
#endif // defined(__clang__) || defined(__GNUC__)
|
|
|
|
#if defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2)
|
|
// Transpose 8 rows x 16 bytes with SSE2. Each output holds 2 columns x 8 rows.
|
|
// The low 8 bytes (even column) go to dst_a and the high 8 bytes (odd column)
|
|
// go to dst_b. Width is a multiple of 16 bytes.
|
|
LIBYUV_TARGET_SSE2
|
|
static void TransposeWx8_Byte_SSE2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int byte_width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t sa = dst_stride_a;
|
|
const ptrdiff_t sb = dst_stride_b;
|
|
while (byte_width > 0) {
|
|
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
|
|
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
|
|
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
|
|
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
|
|
__m128i r4 = _mm_loadu_si128((const __m128i*)(src + s * 4));
|
|
__m128i r5 = _mm_loadu_si128((const __m128i*)(src + s * 5));
|
|
__m128i r6 = _mm_loadu_si128((const __m128i*)(src + s * 6));
|
|
__m128i r7 = _mm_loadu_si128((const __m128i*)(src + s * 7));
|
|
|
|
// Bytes of row pairs. a0 is columns 0..7 of rows 0, 1.
|
|
__m128i a0 = _mm_unpacklo_epi8(r0, r1);
|
|
__m128i a1 = _mm_unpackhi_epi8(r0, r1);
|
|
__m128i a2 = _mm_unpacklo_epi8(r2, r3);
|
|
__m128i a3 = _mm_unpackhi_epi8(r2, r3);
|
|
__m128i a4 = _mm_unpacklo_epi8(r4, r5);
|
|
__m128i a5 = _mm_unpackhi_epi8(r4, r5);
|
|
__m128i a6 = _mm_unpacklo_epi8(r6, r7);
|
|
__m128i a7 = _mm_unpackhi_epi8(r6, r7);
|
|
|
|
// Words. b0 is columns 0..3 of rows 0..3.
|
|
__m128i b0 = _mm_unpacklo_epi16(a0, a2);
|
|
__m128i b1 = _mm_unpackhi_epi16(a0, a2);
|
|
__m128i b2 = _mm_unpacklo_epi16(a1, a3);
|
|
__m128i b3 = _mm_unpackhi_epi16(a1, a3);
|
|
__m128i b4 = _mm_unpacklo_epi16(a4, a6);
|
|
__m128i b5 = _mm_unpackhi_epi16(a4, a6);
|
|
__m128i b6 = _mm_unpacklo_epi16(a5, a7);
|
|
__m128i b7 = _mm_unpackhi_epi16(a5, a7);
|
|
|
|
// Dwords. c0 is columns 0, 1 of rows 0..7.
|
|
__m128i c0 = _mm_unpacklo_epi32(b0, b4);
|
|
__m128i c1 = _mm_unpackhi_epi32(b0, b4);
|
|
__m128i c2 = _mm_unpacklo_epi32(b1, b5);
|
|
__m128i c3 = _mm_unpackhi_epi32(b1, b5);
|
|
__m128i c4 = _mm_unpacklo_epi32(b2, b6);
|
|
__m128i c5 = _mm_unpackhi_epi32(b2, b6);
|
|
__m128i c6 = _mm_unpacklo_epi32(b3, b7);
|
|
__m128i c7 = _mm_unpackhi_epi32(b3, b7);
|
|
|
|
_mm_storel_epi64((__m128i*)(dst_a), c0);
|
|
_mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(c0));
|
|
_mm_storel_epi64((__m128i*)(dst_a + sa), c1);
|
|
_mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(c1));
|
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 2), c2);
|
|
_mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(c2));
|
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 3), c3);
|
|
_mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(c3));
|
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 4), c4);
|
|
_mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(c4));
|
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 5), c5);
|
|
_mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(c5));
|
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 6), c6);
|
|
_mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(c6));
|
|
_mm_storel_epi64((__m128i*)(dst_a + sa * 7), c7);
|
|
_mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(c7));
|
|
|
|
src += 16;
|
|
dst_a += sa * 8;
|
|
dst_b += sb * 8;
|
|
byte_width -= 16;
|
|
}
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2)
|
|
|
|
#if defined(HAS_TRANSPOSEWX8_SSSE3)
|
|
// Transpose 16x8. Width is a multiple of 16.
|
|
LIBYUV_TARGET_SSE2
|
|
void TransposeWx8_SSSE3(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
TransposeWx8_Byte_SSE2(src, src_stride, dst, dst_stride * 2, dst + dst_stride,
|
|
dst_stride * 2, width);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX8_SSSE3)
|
|
|
|
#if defined(HAS_TRANSPOSEUVWX8_SSE2)
|
|
// Transpose UV 8x8. Width is a multiple of 8.
|
|
LIBYUV_TARGET_SSE2
|
|
void TransposeUVWx8_SSE2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int width) {
|
|
TransposeWx8_Byte_SSE2(src, src_stride, dst_a, dst_stride_a, dst_b,
|
|
dst_stride_b, width * 2);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEUVWX8_SSE2)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
|
|
// Load row r into lane 0 and row r + 8 into lane 1.
|
|
#define LOAD2ROWS_AVX2(p, s8) \
|
|
_mm256_inserti128_si256( \
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(p))), \
|
|
_mm_loadu_si128((const __m128i*)((p) + (s8))), 1)
|
|
|
|
// Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8
|
|
// of register r % 8. 3 unpack stages leave each lane with 2 columns x 8 rows,
|
|
// and vpermq joins the rows 0..7 and 8..15 halves. Lane 0 of each output is an
|
|
// even column for dst_a and lane 1 an odd column for dst_b.
|
|
// Width is a multiple of 16 bytes.
|
|
LIBYUV_TARGET_AVX2
|
|
static void TransposeWx16_Byte_AVX2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int byte_width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t sa = dst_stride_a;
|
|
const ptrdiff_t sb = dst_stride_b;
|
|
while (byte_width > 0) {
|
|
__m256i r0 = LOAD2ROWS_AVX2(src, s * 8);
|
|
__m256i r1 = LOAD2ROWS_AVX2(src + s, s * 8);
|
|
__m256i r2 = LOAD2ROWS_AVX2(src + s * 2, s * 8);
|
|
__m256i r3 = LOAD2ROWS_AVX2(src + s * 3, s * 8);
|
|
__m256i r4 = LOAD2ROWS_AVX2(src + s * 4, s * 8);
|
|
__m256i r5 = LOAD2ROWS_AVX2(src + s * 5, s * 8);
|
|
__m256i r6 = LOAD2ROWS_AVX2(src + s * 6, s * 8);
|
|
__m256i r7 = LOAD2ROWS_AVX2(src + s * 7, s * 8);
|
|
|
|
__m256i a0 = _mm256_unpacklo_epi8(r0, r1);
|
|
__m256i a1 = _mm256_unpackhi_epi8(r0, r1);
|
|
__m256i a2 = _mm256_unpacklo_epi8(r2, r3);
|
|
__m256i a3 = _mm256_unpackhi_epi8(r2, r3);
|
|
__m256i a4 = _mm256_unpacklo_epi8(r4, r5);
|
|
__m256i a5 = _mm256_unpackhi_epi8(r4, r5);
|
|
__m256i a6 = _mm256_unpacklo_epi8(r6, r7);
|
|
__m256i a7 = _mm256_unpackhi_epi8(r6, r7);
|
|
|
|
__m256i b0 = _mm256_unpacklo_epi16(a0, a2);
|
|
__m256i b1 = _mm256_unpackhi_epi16(a0, a2);
|
|
__m256i b2 = _mm256_unpacklo_epi16(a1, a3);
|
|
__m256i b3 = _mm256_unpackhi_epi16(a1, a3);
|
|
__m256i b4 = _mm256_unpacklo_epi16(a4, a6);
|
|
__m256i b5 = _mm256_unpackhi_epi16(a4, a6);
|
|
__m256i b6 = _mm256_unpacklo_epi16(a5, a7);
|
|
__m256i b7 = _mm256_unpackhi_epi16(a5, a7);
|
|
|
|
__m256i c0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b0, b4), 0xd8);
|
|
__m256i c1 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b0, b4), 0xd8);
|
|
__m256i c2 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b1, b5), 0xd8);
|
|
__m256i c3 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b1, b5), 0xd8);
|
|
__m256i c4 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b2, b6), 0xd8);
|
|
__m256i c5 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b2, b6), 0xd8);
|
|
__m256i c6 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(b3, b7), 0xd8);
|
|
__m256i c7 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(b3, b7), 0xd8);
|
|
|
|
_mm_storeu_si128((__m128i*)(dst_a), _mm256_castsi256_si128(c0));
|
|
_mm_storeu_si128((__m128i*)(dst_b), _mm256_extracti128_si256(c0, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa), _mm256_castsi256_si128(c1));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb), _mm256_extracti128_si256(c1, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 2), _mm256_castsi256_si128(c2));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 2),
|
|
_mm256_extracti128_si256(c2, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 3), _mm256_castsi256_si128(c3));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 3),
|
|
_mm256_extracti128_si256(c3, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 4), _mm256_castsi256_si128(c4));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 4),
|
|
_mm256_extracti128_si256(c4, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 5), _mm256_castsi256_si128(c5));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 5),
|
|
_mm256_extracti128_si256(c5, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 6), _mm256_castsi256_si128(c6));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 6),
|
|
_mm256_extracti128_si256(c6, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 7), _mm256_castsi256_si128(c7));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 7),
|
|
_mm256_extracti128_si256(c7, 1));
|
|
|
|
src += 16;
|
|
dst_a += sa * 8;
|
|
dst_b += sb * 8;
|
|
byte_width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#undef LOAD2ROWS_AVX2
|
|
#endif // defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void TransposeWx16_AVX2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
TransposeWx16_Byte_AVX2(src, src_stride, dst, dst_stride * 2,
|
|
dst + dst_stride, dst_stride * 2, width);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX16_AVX2)
|
|
|
|
#if defined(HAS_TRANSPOSEUVWX16_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void TransposeUVWx16_AVX2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int width) {
|
|
TransposeWx16_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b,
|
|
dst_stride_b, width * 2);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEUVWX16_AVX2)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
|
// Dword permutes to gather 16 rows of a column from 4 lanes.
|
|
static const uint32_t kPermdTranspose_AVX512BW[32] = {
|
|
0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30,
|
|
1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31};
|
|
|
|
// Load 16 masked bytes from rows p, p + s4, p + s4 * 2, p + s4 * 3 into
|
|
// lanes 0..3.
|
|
#define LOAD4ROWS_AVX512BW(p, s4, k) \
|
|
_mm512_inserti32x4( \
|
|
_mm512_inserti32x4( \
|
|
_mm512_inserti32x4( \
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi8(k, (p))), \
|
|
_mm_maskz_loadu_epi8(k, (p) + (s4)), 1), \
|
|
_mm_maskz_loadu_epi8(k, (p) + (s4) * 2), 2), \
|
|
_mm_maskz_loadu_epi8(k, (p) + (s4) * 3), 3)
|
|
|
|
// Store 16 destination rows. Lane n of o0/o2 is for dst_a and lane n of
|
|
// o1/o3 is for dst_b.
|
|
LIBYUV_TARGET_AVX512BW
|
|
static void Store16x16_AVX512BW(__m512i o0,
|
|
__m512i o1,
|
|
__m512i o2,
|
|
__m512i o3,
|
|
uint8_t* dst_a,
|
|
ptrdiff_t sa,
|
|
uint8_t* dst_b,
|
|
ptrdiff_t sb) {
|
|
_mm_storeu_si128((__m128i*)(dst_a), _mm512_castsi512_si128(o0));
|
|
_mm_storeu_si128((__m128i*)(dst_b), _mm512_castsi512_si128(o1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa), _mm512_extracti32x4_epi32(o0, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb), _mm512_extracti32x4_epi32(o1, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 2), _mm512_castsi512_si128(o2));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 2), _mm512_castsi512_si128(o3));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 3),
|
|
_mm512_extracti32x4_epi32(o2, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 3),
|
|
_mm512_extracti32x4_epi32(o3, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 4),
|
|
_mm512_extracti32x4_epi32(o0, 2));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 4),
|
|
_mm512_extracti32x4_epi32(o1, 2));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 5),
|
|
_mm512_extracti32x4_epi32(o0, 3));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 5),
|
|
_mm512_extracti32x4_epi32(o1, 3));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 6),
|
|
_mm512_extracti32x4_epi32(o2, 2));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 6),
|
|
_mm512_extracti32x4_epi32(o3, 2));
|
|
_mm_storeu_si128((__m128i*)(dst_a + sa * 7),
|
|
_mm512_extracti32x4_epi32(o2, 3));
|
|
_mm_storeu_si128((__m128i*)(dst_b + sb * 7),
|
|
_mm512_extracti32x4_epi32(o3, 3));
|
|
}
|
|
|
|
// Transpose 16x16 bytes using 4 lanes of AVX512BW with tail masking.
|
|
// Source row r is in lane r / 4 of register r % 4.
|
|
// Can be used for 8-bit planar transpose (dst_a = even rows, dst_b = odd rows)
|
|
// or 16-bit UV split transpose (dst_a = U plane, dst_b = V plane).
|
|
// TODO(fbarchard): Consider removing in favor of TransposeWx16_Byte_AVX2,
|
|
// which is within 10% of AVX512BW on most CPUs.
|
|
LIBYUV_TARGET_AVX512BW
|
|
static void TransposeWx16_Byte_AVX512BW(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int byte_width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t sa = dst_stride_a;
|
|
const ptrdiff_t sb = dst_stride_b;
|
|
const __m512i perm0 = _mm512_loadu_si512(kPermdTranspose_AVX512BW);
|
|
const __m512i perm1 = _mm512_loadu_si512(kPermdTranspose_AVX512BW + 16);
|
|
while (byte_width > 0) {
|
|
const int n = byte_width < 16 ? byte_width : 16;
|
|
const __mmask16 k = (__mmask16)((1u << n) - 1);
|
|
__m512i z0 = LOAD4ROWS_AVX512BW(src, s * 4, k);
|
|
__m512i z1 = LOAD4ROWS_AVX512BW(src + s, s * 4, k);
|
|
__m512i z2 = LOAD4ROWS_AVX512BW(src + s * 2, s * 4, k);
|
|
__m512i z3 = LOAD4ROWS_AVX512BW(src + s * 3, s * 4, k);
|
|
|
|
__m512i a0 = _mm512_unpacklo_epi8(z0, z1);
|
|
__m512i a1 = _mm512_unpackhi_epi8(z0, z1);
|
|
__m512i a2 = _mm512_unpacklo_epi8(z2, z3);
|
|
__m512i a3 = _mm512_unpackhi_epi8(z2, z3);
|
|
__m512i b0 = _mm512_unpacklo_epi16(a0, a2);
|
|
__m512i b1 = _mm512_unpackhi_epi16(a0, a2);
|
|
__m512i b2 = _mm512_unpacklo_epi16(a1, a3);
|
|
__m512i b3 = _mm512_unpackhi_epi16(a1, a3);
|
|
__m512i o0 = _mm512_permutex2var_epi32(b0, perm0, b2);
|
|
__m512i o1 = _mm512_permutex2var_epi32(b0, perm1, b2);
|
|
__m512i o2 = _mm512_permutex2var_epi32(b1, perm0, b3);
|
|
__m512i o3 = _mm512_permutex2var_epi32(b1, perm1, b3);
|
|
|
|
if (n == 16) {
|
|
Store16x16_AVX512BW(o0, o1, o2, o3, dst_a, sa, dst_b, sb);
|
|
dst_a += sa * 8;
|
|
dst_b += sb * 8;
|
|
} else {
|
|
// Remainder: 1 to 15 bytes. Store 1 destination row per byte.
|
|
SIMD_ALIGNED(uint8_t temp[16 * 16]);
|
|
int i;
|
|
Store16x16_AVX512BW(o0, o1, o2, o3, temp, 32, temp + 16, 32);
|
|
for (i = 0; i < n; ++i) {
|
|
uint8_t* dst = (i & 1) ? dst_b + sb * (i >> 1) : dst_a + sa * (i >> 1);
|
|
_mm_storeu_si128((__m128i*)dst,
|
|
_mm_load_si128((const __m128i*)(temp + i * 16)));
|
|
}
|
|
}
|
|
src += 16;
|
|
byte_width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#undef LOAD4ROWS_AVX512BW
|
|
#endif // defined(HAS_TRANSPOSEWX16_AVX512BW) ||
|
|
// defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_AVX512BW)
|
|
LIBYUV_TARGET_AVX512BW
|
|
void TransposeWx16_AVX512BW(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
TransposeWx16_Byte_AVX512BW(src, src_stride, dst, dst_stride * 2,
|
|
dst + (width > 1 ? dst_stride : 0),
|
|
dst_stride * 2, width);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX16_AVX512BW)
|
|
|
|
#if defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
|
LIBYUV_TARGET_AVX512BW
|
|
void TransposeUVWx16_AVX512BW(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst_a,
|
|
int dst_stride_a,
|
|
uint8_t* dst_b,
|
|
int dst_stride_b,
|
|
int width) {
|
|
TransposeWx16_Byte_AVX512BW(src, src_stride, dst_a, dst_stride_a, dst_b,
|
|
dst_stride_b, width * 2);
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_16_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void TransposeWx16_16_AVX2(const uint16_t* src,
|
|
int src_stride,
|
|
uint16_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t d = dst_stride;
|
|
while (width > 0) {
|
|
__m256i r0 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 8)), 1);
|
|
__m256i r1 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 9)), 1);
|
|
__m256i r2 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 10)), 1);
|
|
__m256i r3 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 11)), 1);
|
|
__m256i r4 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 4))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 12)), 1);
|
|
__m256i r5 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 5))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 13)), 1);
|
|
__m256i r6 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 6))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 14)), 1);
|
|
__m256i r7 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 7))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 15)), 1);
|
|
|
|
__m256i t0 = _mm256_unpacklo_epi16(r0, r1);
|
|
__m256i t1 = _mm256_unpackhi_epi16(r0, r1);
|
|
__m256i t2 = _mm256_unpacklo_epi16(r2, r3);
|
|
__m256i t3 = _mm256_unpackhi_epi16(r2, r3);
|
|
__m256i t4 = _mm256_unpacklo_epi16(r4, r5);
|
|
__m256i t5 = _mm256_unpackhi_epi16(r4, r5);
|
|
__m256i t6 = _mm256_unpacklo_epi16(r6, r7);
|
|
__m256i t7 = _mm256_unpackhi_epi16(r6, r7);
|
|
|
|
__m256i u0 = _mm256_unpacklo_epi32(t0, t2);
|
|
__m256i u1 = _mm256_unpackhi_epi32(t0, t2);
|
|
__m256i u2 = _mm256_unpacklo_epi32(t1, t3);
|
|
__m256i u3 = _mm256_unpackhi_epi32(t1, t3);
|
|
__m256i u4 = _mm256_unpacklo_epi32(t4, t6);
|
|
__m256i u5 = _mm256_unpackhi_epi32(t4, t6);
|
|
__m256i u6 = _mm256_unpacklo_epi32(t5, t7);
|
|
__m256i u7 = _mm256_unpackhi_epi32(t5, t7);
|
|
|
|
_mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(u0, u4));
|
|
_mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(u0, u4));
|
|
_mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(u1, u5));
|
|
_mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(u1, u5));
|
|
_mm256_storeu_si256((__m256i*)(dst + d * 4), _mm256_unpacklo_epi64(u2, u6));
|
|
_mm256_storeu_si256((__m256i*)(dst + d * 5), _mm256_unpackhi_epi64(u2, u6));
|
|
_mm256_storeu_si256((__m256i*)(dst + d * 6), _mm256_unpacklo_epi64(u3, u7));
|
|
_mm256_storeu_si256((__m256i*)(dst + d * 7), _mm256_unpackhi_epi64(u3, u7));
|
|
src += 8;
|
|
dst += d * 8;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX16_16_AVX2)
|
|
|
|
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
|
LIBYUV_TARGET_AVX512BW
|
|
void TransposeWx32_16_AVX512BW(const uint16_t* src,
|
|
int src_stride,
|
|
uint16_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t d = dst_stride;
|
|
while (width >= 8) {
|
|
__m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src)));
|
|
__m512i r1 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s)));
|
|
__m512i r2 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2)));
|
|
__m512i r3 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3)));
|
|
__m512i r4 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 4)));
|
|
__m512i r5 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 5)));
|
|
__m512i r6 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 6)));
|
|
__m512i r7 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 7)));
|
|
|
|
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)),
|
|
1);
|
|
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)),
|
|
1);
|
|
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)),
|
|
1);
|
|
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)),
|
|
1);
|
|
r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 12)),
|
|
1);
|
|
r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 13)),
|
|
1);
|
|
r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 14)),
|
|
1);
|
|
r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 15)),
|
|
1);
|
|
|
|
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 16)),
|
|
2);
|
|
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 17)),
|
|
2);
|
|
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 18)),
|
|
2);
|
|
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 19)),
|
|
2);
|
|
r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 20)),
|
|
2);
|
|
r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 21)),
|
|
2);
|
|
r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 22)),
|
|
2);
|
|
r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 23)),
|
|
2);
|
|
|
|
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 24)),
|
|
3);
|
|
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 25)),
|
|
3);
|
|
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 26)),
|
|
3);
|
|
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 27)),
|
|
3);
|
|
r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 28)),
|
|
3);
|
|
r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 29)),
|
|
3);
|
|
r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 30)),
|
|
3);
|
|
r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 31)),
|
|
3);
|
|
|
|
__m512i t0 = _mm512_unpacklo_epi16(r0, r1);
|
|
__m512i t1 = _mm512_unpackhi_epi16(r0, r1);
|
|
__m512i t2 = _mm512_unpacklo_epi16(r2, r3);
|
|
__m512i t3 = _mm512_unpackhi_epi16(r2, r3);
|
|
__m512i t4 = _mm512_unpacklo_epi16(r4, r5);
|
|
__m512i t5 = _mm512_unpackhi_epi16(r4, r5);
|
|
__m512i t6 = _mm512_unpacklo_epi16(r6, r7);
|
|
__m512i t7 = _mm512_unpackhi_epi16(r6, r7);
|
|
|
|
__m512i u0 = _mm512_unpacklo_epi32(t0, t2);
|
|
__m512i u1 = _mm512_unpackhi_epi32(t0, t2);
|
|
__m512i u2 = _mm512_unpacklo_epi32(t1, t3);
|
|
__m512i u3 = _mm512_unpackhi_epi32(t1, t3);
|
|
__m512i u4 = _mm512_unpacklo_epi32(t4, t6);
|
|
__m512i u5 = _mm512_unpackhi_epi32(t4, t6);
|
|
__m512i u6 = _mm512_unpacklo_epi32(t5, t7);
|
|
__m512i u7 = _mm512_unpackhi_epi32(t5, t7);
|
|
|
|
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(u0, u4));
|
|
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(u0, u4));
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(u1, u5));
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(u1, u5));
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 4), _mm512_unpacklo_epi64(u2, u6));
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 5), _mm512_unpackhi_epi64(u2, u6));
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 6), _mm512_unpacklo_epi64(u3, u7));
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 7), _mm512_unpackhi_epi64(u3, u7));
|
|
src += 8;
|
|
dst += d * 8;
|
|
width -= 8;
|
|
}
|
|
if (width > 0) {
|
|
const __mmask8 col_mask = (1u << width) - 1;
|
|
__m512i r0 = _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src));
|
|
__m512i r1 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s));
|
|
__m512i r2 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 2));
|
|
__m512i r3 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 3));
|
|
__m512i r4 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 4));
|
|
__m512i r5 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 5));
|
|
__m512i r6 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 6));
|
|
__m512i r7 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src + s * 7));
|
|
|
|
r0 =
|
|
_mm512_inserti32x4(r0, _mm_maskz_loadu_epi16(col_mask, src + s * 8), 1);
|
|
r1 =
|
|
_mm512_inserti32x4(r1, _mm_maskz_loadu_epi16(col_mask, src + s * 9), 1);
|
|
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi16(col_mask, src + s * 10),
|
|
1);
|
|
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi16(col_mask, src + s * 11),
|
|
1);
|
|
r4 = _mm512_inserti32x4(r4, _mm_maskz_loadu_epi16(col_mask, src + s * 12),
|
|
1);
|
|
r5 = _mm512_inserti32x4(r5, _mm_maskz_loadu_epi16(col_mask, src + s * 13),
|
|
1);
|
|
r6 = _mm512_inserti32x4(r6, _mm_maskz_loadu_epi16(col_mask, src + s * 14),
|
|
1);
|
|
r7 = _mm512_inserti32x4(r7, _mm_maskz_loadu_epi16(col_mask, src + s * 15),
|
|
1);
|
|
|
|
r0 = _mm512_inserti32x4(r0, _mm_maskz_loadu_epi16(col_mask, src + s * 16),
|
|
2);
|
|
r1 = _mm512_inserti32x4(r1, _mm_maskz_loadu_epi16(col_mask, src + s * 17),
|
|
2);
|
|
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi16(col_mask, src + s * 18),
|
|
2);
|
|
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi16(col_mask, src + s * 19),
|
|
2);
|
|
r4 = _mm512_inserti32x4(r4, _mm_maskz_loadu_epi16(col_mask, src + s * 20),
|
|
2);
|
|
r5 = _mm512_inserti32x4(r5, _mm_maskz_loadu_epi16(col_mask, src + s * 21),
|
|
2);
|
|
r6 = _mm512_inserti32x4(r6, _mm_maskz_loadu_epi16(col_mask, src + s * 22),
|
|
2);
|
|
r7 = _mm512_inserti32x4(r7, _mm_maskz_loadu_epi16(col_mask, src + s * 23),
|
|
2);
|
|
|
|
r0 = _mm512_inserti32x4(r0, _mm_maskz_loadu_epi16(col_mask, src + s * 24),
|
|
3);
|
|
r1 = _mm512_inserti32x4(r1, _mm_maskz_loadu_epi16(col_mask, src + s * 25),
|
|
3);
|
|
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi16(col_mask, src + s * 26),
|
|
3);
|
|
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi16(col_mask, src + s * 27),
|
|
3);
|
|
r4 = _mm512_inserti32x4(r4, _mm_maskz_loadu_epi16(col_mask, src + s * 28),
|
|
3);
|
|
r5 = _mm512_inserti32x4(r5, _mm_maskz_loadu_epi16(col_mask, src + s * 29),
|
|
3);
|
|
r6 = _mm512_inserti32x4(r6, _mm_maskz_loadu_epi16(col_mask, src + s * 30),
|
|
3);
|
|
r7 = _mm512_inserti32x4(r7, _mm_maskz_loadu_epi16(col_mask, src + s * 31),
|
|
3);
|
|
|
|
__m512i t0 = _mm512_unpacklo_epi16(r0, r1);
|
|
__m512i t1 = _mm512_unpackhi_epi16(r0, r1);
|
|
__m512i t2 = _mm512_unpacklo_epi16(r2, r3);
|
|
__m512i t3 = _mm512_unpackhi_epi16(r2, r3);
|
|
__m512i t4 = _mm512_unpacklo_epi16(r4, r5);
|
|
__m512i t5 = _mm512_unpackhi_epi16(r4, r5);
|
|
__m512i t6 = _mm512_unpacklo_epi16(r6, r7);
|
|
__m512i t7 = _mm512_unpackhi_epi16(r6, r7);
|
|
|
|
__m512i u0 = _mm512_unpacklo_epi32(t0, t2);
|
|
__m512i u1 = _mm512_unpackhi_epi32(t0, t2);
|
|
__m512i u2 = _mm512_unpacklo_epi32(t1, t3);
|
|
__m512i u3 = _mm512_unpackhi_epi32(t1, t3);
|
|
__m512i u4 = _mm512_unpacklo_epi32(t4, t6);
|
|
__m512i u5 = _mm512_unpackhi_epi32(t4, t6);
|
|
__m512i u6 = _mm512_unpacklo_epi32(t5, t7);
|
|
__m512i u7 = _mm512_unpackhi_epi32(t5, t7);
|
|
|
|
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(u0, u4));
|
|
if (width > 1) {
|
|
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(u0, u4));
|
|
}
|
|
if (width > 2) {
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 2),
|
|
_mm512_unpacklo_epi64(u1, u5));
|
|
}
|
|
if (width > 3) {
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 3),
|
|
_mm512_unpackhi_epi64(u1, u5));
|
|
}
|
|
if (width > 4) {
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 4),
|
|
_mm512_unpacklo_epi64(u2, u6));
|
|
}
|
|
if (width > 5) {
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 5),
|
|
_mm512_unpackhi_epi64(u2, u6));
|
|
}
|
|
if (width > 6) {
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 6),
|
|
_mm512_unpacklo_epi64(u3, u7));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
|
|
|
#if defined(HAS_TRANSPOSEWX4_32_SSE2)
|
|
// Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row.
|
|
LIBYUV_TARGET_SSE2
|
|
void TransposeWx4_32_SSE2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t d = dst_stride;
|
|
while (width > 0) {
|
|
__m128i r0 = _mm_loadu_si128((const __m128i*)(src)); // a b c d
|
|
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); // e f g h
|
|
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); // i j k l
|
|
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); // m n o p
|
|
__m128i t0 = _mm_unpacklo_epi32(r0, r1); // a e b f
|
|
__m128i t1 = _mm_unpacklo_epi32(r2, r3); // i m j n
|
|
__m128i t2 = _mm_unpackhi_epi32(r0, r1); // c g d h
|
|
__m128i t3 = _mm_unpackhi_epi32(r2, r3); // k o l p
|
|
_mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); // a e i m
|
|
_mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1));
|
|
_mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3));
|
|
_mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3));
|
|
src += s * 4;
|
|
dst += 16;
|
|
width -= 4;
|
|
}
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX4_32_SSE2)
|
|
|
|
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void TransposeWx8_32_AVX2(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t d = dst_stride;
|
|
while (width > 0) {
|
|
__m256i r0 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
|
|
__m256i r1 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 5)), 1);
|
|
__m256i r2 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 6)), 1);
|
|
__m256i r3 = _mm256_inserti128_si256(
|
|
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
|
|
_mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
|
|
|
|
__m256i t0 = _mm256_unpacklo_epi32(r0, r1);
|
|
__m256i t1 = _mm256_unpacklo_epi32(r2, r3);
|
|
__m256i t2 = _mm256_unpackhi_epi32(r0, r1);
|
|
__m256i t3 = _mm256_unpackhi_epi32(r2, r3);
|
|
|
|
_mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1));
|
|
_mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1));
|
|
_mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3));
|
|
_mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3));
|
|
src += 16;
|
|
dst += d * 4;
|
|
width -= 4;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX8_32_AVX2)
|
|
|
|
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
|
LIBYUV_TARGET_AVX512BW
|
|
void TransposeWx16_32_AVX512BW(const uint8_t* src,
|
|
int src_stride,
|
|
uint8_t* dst,
|
|
int dst_stride,
|
|
int width) {
|
|
const ptrdiff_t s = src_stride;
|
|
const ptrdiff_t d = dst_stride;
|
|
while (width >= 4) {
|
|
__m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src)));
|
|
__m512i r1 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s)));
|
|
__m512i r2 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2)));
|
|
__m512i r3 =
|
|
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3)));
|
|
|
|
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 4)),
|
|
1);
|
|
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 5)),
|
|
1);
|
|
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 6)),
|
|
1);
|
|
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 7)),
|
|
1);
|
|
|
|
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)),
|
|
2);
|
|
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)),
|
|
2);
|
|
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)),
|
|
2);
|
|
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)),
|
|
2);
|
|
|
|
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 12)),
|
|
3);
|
|
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 13)),
|
|
3);
|
|
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 14)),
|
|
3);
|
|
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 15)),
|
|
3);
|
|
|
|
__m512i t0 = _mm512_unpacklo_epi32(r0, r1);
|
|
__m512i t1 = _mm512_unpacklo_epi32(r2, r3);
|
|
__m512i t2 = _mm512_unpackhi_epi32(r0, r1);
|
|
__m512i t3 = _mm512_unpackhi_epi32(r2, r3);
|
|
|
|
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(t0, t1));
|
|
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(t0, t1));
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(t2, t3));
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(t2, t3));
|
|
src += 16;
|
|
dst += d * 4;
|
|
width -= 4;
|
|
}
|
|
if (width > 0) {
|
|
const __mmask8 col_mask = (1u << width) - 1;
|
|
__m512i r0 = _mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src));
|
|
__m512i r1 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src + s));
|
|
__m512i r2 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src + s * 2));
|
|
__m512i r3 =
|
|
_mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src + s * 3));
|
|
|
|
r0 =
|
|
_mm512_inserti32x4(r0, _mm_maskz_loadu_epi32(col_mask, src + s * 4), 1);
|
|
r1 =
|
|
_mm512_inserti32x4(r1, _mm_maskz_loadu_epi32(col_mask, src + s * 5), 1);
|
|
r2 =
|
|
_mm512_inserti32x4(r2, _mm_maskz_loadu_epi32(col_mask, src + s * 6), 1);
|
|
r3 =
|
|
_mm512_inserti32x4(r3, _mm_maskz_loadu_epi32(col_mask, src + s * 7), 1);
|
|
|
|
r0 =
|
|
_mm512_inserti32x4(r0, _mm_maskz_loadu_epi32(col_mask, src + s * 8), 2);
|
|
r1 =
|
|
_mm512_inserti32x4(r1, _mm_maskz_loadu_epi32(col_mask, src + s * 9), 2);
|
|
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi32(col_mask, src + s * 10),
|
|
2);
|
|
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi32(col_mask, src + s * 11),
|
|
2);
|
|
|
|
r0 = _mm512_inserti32x4(r0, _mm_maskz_loadu_epi32(col_mask, src + s * 12),
|
|
3);
|
|
r1 = _mm512_inserti32x4(r1, _mm_maskz_loadu_epi32(col_mask, src + s * 13),
|
|
3);
|
|
r2 = _mm512_inserti32x4(r2, _mm_maskz_loadu_epi32(col_mask, src + s * 14),
|
|
3);
|
|
r3 = _mm512_inserti32x4(r3, _mm_maskz_loadu_epi32(col_mask, src + s * 15),
|
|
3);
|
|
|
|
__m512i t0 = _mm512_unpacklo_epi32(r0, r1);
|
|
__m512i t1 = _mm512_unpacklo_epi32(r2, r3);
|
|
__m512i t2 = _mm512_unpackhi_epi32(r0, r1);
|
|
__m512i t3 = _mm512_unpackhi_epi32(r2, r3);
|
|
|
|
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(t0, t1));
|
|
if (width > 1) {
|
|
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(t0, t1));
|
|
}
|
|
if (width > 2) {
|
|
_mm512_storeu_si512((__m512i*)(dst + d * 2),
|
|
_mm512_unpacklo_epi64(t2, t3));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) &&
|
|
// (Visual C or LIBYUV_ENABLE_ROWWIN)
|