mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
e58eeea369
Add SwapUVRow_AVX2 implementation in row_win.cc for MSVC and enable HAS_SWAPUVROW_AVX2 under USE_ROW_WIN. Share kShuffleUVToVU shuffle table between SSSE3 and AVX2 in row_gcc.cc. Make NV12ToNV21 the primary implementation and alias NV21ToNV12 to it. Add NV12ToNV21 test in convert_test.cc and benchmark dispatch for SSSE3 and AVX2 in planar_test.cc. CPU Before After Speedup ---------------------------------------------------------------- Intel Granite Rapids 282 ms 125 ms 2.26x Intel Emerald Rapids 264 ms 108 ms 2.44x Intel Sapphire Rapids 276 ms 122 ms 2.26x Intel Ice Lake 310 ms 122 ms 2.54x Intel Skylake-X 573 ms 157 ms 3.65x Intel Haswell 620 ms 162 ms 3.83x AMD Zen 4 (Genoa) 266 ms 52 ms 5.12x AMD Zen 6 (Venice) 229 ms 43 ms 5.33x ARM Neoverse-N1 140 ms 71 ms 1.97x NVIDIA Grace (Neoverse-V2) 62 ms 69 ms 0.90x R=wtc@google.com Test: libyuv_unittest --gtest_filter=*SwapUV*:*NV12ToNV21*:*NV21ToNV12* Bug: 406251841 Change-Id: I9bed5522aedadea13cdf69c27e3cbe633e31dd5b TAG=agy CONV=bb50ac62-0c55-4969-8fff-c71ab3d9a79f Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8356968 Reviewed-by: richard winterton <rrwinterton@gmail.com> Commit-Queue: Frank Barchard <fbarchard@google.com>
2427 lines
92 KiB
C++
2427 lines
92 KiB
C++
/*
|
|
* Copyright 2011 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/convert_from_argb.h" // For ArgbConstants
|
|
#include "libyuv/row.h"
|
|
|
|
// This module is for Visual C 32/64 bit
|
|
#if !defined(LIBYUV_DISABLE_X86) && \
|
|
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
|
defined(_M_X86)) && \
|
|
((defined(_MSC_VER) && !defined(__clang__)) || \
|
|
defined(LIBYUV_ENABLE_ROWWIN))
|
|
|
|
#include <emmintrin.h>
|
|
#include <immintrin.h> // For AVX2 intrinsics
|
|
#include <tmmintrin.h> // For _mm_maddubs_epi16
|
|
#if defined(_MSC_VER) && !defined(__clang__)
|
|
#include <intrin.h>
|
|
#endif
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
#if defined(__clang__) || defined(__GNUC__)
|
|
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
|
|
#define LIBYUV_TARGET_AVX512BW \
|
|
__attribute__((target("avx512bw,avx512vl,avx512f")))
|
|
#define LIBYUV_TARGET_AVX512VBMI \
|
|
__attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f")))
|
|
#else
|
|
#define LIBYUV_TARGET_AVX2
|
|
#define LIBYUV_TARGET_AVX512BW
|
|
#define LIBYUV_TARGET_AVX512VBMI
|
|
#endif // defined(__clang__) || defined(__GNUC__)
|
|
|
|
// Read 8 UV from 444
|
|
#define READYUV444 \
|
|
xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \
|
|
xmm1 = _mm_loadl_epi64((__m128i*)(u_buf + offset)); \
|
|
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
|
|
u_buf += 8; \
|
|
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
|
|
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
|
|
y_buf += 8;
|
|
|
|
// Read 8 UV from 444, With 8 Alpha.
|
|
#define READYUVA444 \
|
|
xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \
|
|
xmm1 = _mm_loadl_epi64((__m128i*)(u_buf + offset)); \
|
|
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
|
|
u_buf += 8; \
|
|
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
|
|
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
|
|
y_buf += 8; \
|
|
xmm5 = _mm_loadl_epi64((__m128i*)a_buf); \
|
|
a_buf += 8;
|
|
|
|
// Read 4 UV from 422, upsample to 8 UV.
|
|
#define READYUV422 \
|
|
xmm3 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
|
|
xmm1 = _mm_cvtsi32_si128(*(uint32_t*)(u_buf + offset)); \
|
|
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
|
|
xmm3 = _mm_unpacklo_epi16(xmm3, xmm3); \
|
|
u_buf += 4; \
|
|
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
|
|
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
|
|
y_buf += 8;
|
|
|
|
// Read 4 UV from 422, upsample to 8 UV. With 8 Alpha.
|
|
#define READYUVA422 \
|
|
xmm3 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
|
|
xmm1 = _mm_cvtsi32_si128(*(uint32_t*)(u_buf + offset)); \
|
|
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
|
|
xmm3 = _mm_unpacklo_epi16(xmm3, xmm3); \
|
|
u_buf += 4; \
|
|
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
|
|
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
|
|
y_buf += 8; \
|
|
xmm5 = _mm_loadl_epi64((__m128i*)a_buf); \
|
|
a_buf += 8;
|
|
|
|
// Convert 8 pixels: 8 UV and 8 Y.
|
|
#define YUVTORGB(yuvconstants) \
|
|
xmm3 = _mm_sub_epi8(xmm3, _mm_set1_epi8((char)0x80)); \
|
|
xmm4 = _mm_mulhi_epu16(xmm4, *(__m128i*)yuvconstants->kYToRgb); \
|
|
xmm4 = _mm_add_epi16(xmm4, *(__m128i*)yuvconstants->kYBiasToRgb); \
|
|
xmm0 = _mm_maddubs_epi16(*(__m128i*)yuvconstants->kUVToB, xmm3); \
|
|
xmm1 = _mm_maddubs_epi16(*(__m128i*)yuvconstants->kUVToG, xmm3); \
|
|
xmm2 = _mm_maddubs_epi16(*(__m128i*)yuvconstants->kUVToR, xmm3); \
|
|
xmm0 = _mm_adds_epi16(xmm4, xmm0); \
|
|
xmm1 = _mm_subs_epi16(xmm4, xmm1); \
|
|
xmm2 = _mm_adds_epi16(xmm4, xmm2); \
|
|
xmm0 = _mm_srai_epi16(xmm0, 6); \
|
|
xmm1 = _mm_srai_epi16(xmm1, 6); \
|
|
xmm2 = _mm_srai_epi16(xmm2, 6); \
|
|
xmm0 = _mm_packus_epi16(xmm0, xmm0); \
|
|
xmm1 = _mm_packus_epi16(xmm1, xmm1); \
|
|
xmm2 = _mm_packus_epi16(xmm2, xmm2);
|
|
|
|
// Store 8 ARGB values.
|
|
#define STOREARGB \
|
|
xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); \
|
|
xmm2 = _mm_unpacklo_epi8(xmm2, xmm5); \
|
|
xmm1 = _mm_loadu_si128(&xmm0); \
|
|
xmm0 = _mm_unpacklo_epi16(xmm0, xmm2); \
|
|
xmm1 = _mm_unpackhi_epi16(xmm1, xmm2); \
|
|
_mm_storeu_si128((__m128i*)dst_argb, xmm0); \
|
|
_mm_storeu_si128((__m128i*)(dst_argb + 16), xmm1); \
|
|
dst_argb += 32;
|
|
|
|
// Store 8 AR30 values.
|
|
#define STOREAR30 \
|
|
xmm1 = _mm_and_si128(xmm1, xmm7); \
|
|
xmm2 = _mm_and_si128(xmm2, xmm7); \
|
|
xmm0 = _mm_srli_epi16(xmm0, 4); \
|
|
xmm3 = _mm_slli_epi16(xmm1, 6); \
|
|
xmm0 = _mm_or_si128(xmm0, xmm3); \
|
|
xmm1 = _mm_srli_epi16(xmm1, 10); \
|
|
xmm2 = _mm_or_si128(xmm2, xmm6); \
|
|
xmm2 = _mm_or_si128(xmm2, xmm1); \
|
|
xmm1 = _mm_unpackhi_epi16(xmm0, xmm2); \
|
|
xmm0 = _mm_unpacklo_epi16(xmm0, xmm2); \
|
|
_mm_storeu_si128((__m128i*)dst_ar30, xmm0); \
|
|
_mm_storeu_si128((__m128i*)(dst_ar30 + 16), xmm1); \
|
|
dst_ar30 += 32;
|
|
|
|
// Convert 32 ARGB pixels (128 bytes) to 32 UV444 values.
|
|
#if defined(HAS_ARGBTOYMATRIXROW_AVX2) || defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m256i ymm_u =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
__m256i ymm_v =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
__m256i ymm5 = _mm256_set1_epi16((short)0x8000);
|
|
__m256i perm_mask = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
|
|
__m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src_argb + 64));
|
|
__m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src_argb + 96));
|
|
src_argb += 128;
|
|
|
|
__m256i ymm0_u = _mm256_maddubs_epi16(ymm0, ymm_u);
|
|
__m256i ymm1_u = _mm256_maddubs_epi16(ymm1, ymm_u);
|
|
__m256i ymm2_u = _mm256_maddubs_epi16(ymm2, ymm_u);
|
|
__m256i ymm3_u = _mm256_maddubs_epi16(ymm3, ymm_u);
|
|
|
|
__m256i ymm0_v = _mm256_maddubs_epi16(ymm0, ymm_v);
|
|
__m256i ymm1_v = _mm256_maddubs_epi16(ymm1, ymm_v);
|
|
__m256i ymm2_v = _mm256_maddubs_epi16(ymm2, ymm_v);
|
|
__m256i ymm3_v = _mm256_maddubs_epi16(ymm3, ymm_v);
|
|
|
|
ymm0_u = _mm256_hadd_epi16(ymm0_u, ymm1_u);
|
|
ymm2_u = _mm256_hadd_epi16(ymm2_u, ymm3_u);
|
|
|
|
ymm0_v = _mm256_hadd_epi16(ymm0_v, ymm1_v);
|
|
ymm2_v = _mm256_hadd_epi16(ymm2_v, ymm3_v);
|
|
|
|
ymm0_u = _mm256_sub_epi16(ymm5, ymm0_u);
|
|
ymm2_u = _mm256_sub_epi16(ymm5, ymm2_u);
|
|
|
|
ymm0_v = _mm256_sub_epi16(ymm5, ymm0_v);
|
|
ymm2_v = _mm256_sub_epi16(ymm5, ymm2_v);
|
|
|
|
ymm0_u = _mm256_srli_epi16(ymm0_u, 8);
|
|
ymm2_u = _mm256_srli_epi16(ymm2_u, 8);
|
|
|
|
ymm0_v = _mm256_srli_epi16(ymm0_v, 8);
|
|
ymm2_v = _mm256_srli_epi16(ymm2_v, 8);
|
|
|
|
ymm0_u = _mm256_packus_epi16(ymm0_u, ymm2_u);
|
|
ymm0_u = _mm256_permutevar8x32_epi32(ymm0_u, perm_mask);
|
|
|
|
ymm0_v = _mm256_packus_epi16(ymm0_v, ymm2_v);
|
|
ymm0_v = _mm256_permutevar8x32_epi32(ymm0_v, perm_mask);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_u, ymm0_u);
|
|
_mm256_storeu_si256((__m256i*)dst_v, ymm0_v);
|
|
dst_u += 32;
|
|
dst_v += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_ARGBTOYMATRIXROW_AVX2) ||
|
|
// defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
|
|
|
|
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_y,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m256i ymm5 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm4 =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToY));
|
|
__m256i ymm7 =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kAddY));
|
|
__m256i ymm6 = _mm256_maddubs_epi16(ymm4, ymm5);
|
|
ymm6 = _mm256_hadd_epi16(ymm6, ymm6);
|
|
ymm7 = _mm256_sub_epi16(ymm7, ymm6);
|
|
__m256i perm_mask = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
|
|
__m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src_argb + 64));
|
|
__m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src_argb + 96));
|
|
src_argb += 128;
|
|
|
|
ymm0 = _mm256_sub_epi8(ymm0, ymm5);
|
|
ymm1 = _mm256_sub_epi8(ymm1, ymm5);
|
|
ymm2 = _mm256_sub_epi8(ymm2, ymm5);
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm5);
|
|
|
|
ymm0 = _mm256_maddubs_epi16(ymm4, ymm0);
|
|
ymm1 = _mm256_maddubs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_maddubs_epi16(ymm4, ymm2);
|
|
ymm3 = _mm256_maddubs_epi16(ymm4, ymm3);
|
|
|
|
ymm0 = _mm256_hadd_epi16(ymm0, ymm1);
|
|
ymm2 = _mm256_hadd_epi16(ymm2, ymm3);
|
|
|
|
ymm0 = _mm256_add_epi16(ymm0, ymm7);
|
|
ymm2 = _mm256_add_epi16(ymm2, ymm7);
|
|
|
|
ymm0 = _mm256_srli_epi16(ymm0, 8);
|
|
ymm2 = _mm256_srli_epi16(ymm2, 8);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm2);
|
|
ymm0 = _mm256_permutevar8x32_epi32(ymm0, perm_mask);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_y, ymm0);
|
|
dst_y += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_ARGBTOYMATRIXROW_AVX2)
|
|
|
|
#ifdef HAS_RAWTOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
|
|
__m256i ymm_alpha = _mm256_set1_epi32(0xff000000);
|
|
__m128i shuf_low =
|
|
_mm_set_epi8(-1, 9, 10, 11, -1, 6, 7, 8, -1, 3, 4, 5, -1, 0, 1, 2);
|
|
__m128i shuf_high =
|
|
_mm_set_epi8(-1, 13, 14, 15, -1, 10, 11, 12, -1, 7, 8, 9, -1, 4, 5, 6);
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(shuf_low);
|
|
__m256i ymm_shuf2 = _mm256_broadcastsi128_si256(shuf_high);
|
|
|
|
while (width > 0) {
|
|
__m128i xmm0 = _mm_loadu_si128((const __m128i*)src_raw);
|
|
__m256i ymm0 = _mm256_castsi128_si256(xmm0);
|
|
ymm0 = _mm256_inserti128_si256(
|
|
ymm0, _mm_loadu_si128((const __m128i*)(src_raw + 12)), 1);
|
|
|
|
__m128i xmm1 = _mm_loadu_si128((const __m128i*)(src_raw + 24));
|
|
__m256i ymm1 = _mm256_castsi128_si256(xmm1);
|
|
ymm1 = _mm256_inserti128_si256(
|
|
ymm1, _mm_loadu_si128((const __m128i*)(src_raw + 36)), 1);
|
|
|
|
__m128i xmm2 = _mm_loadu_si128((const __m128i*)(src_raw + 48));
|
|
__m256i ymm2 = _mm256_castsi128_si256(xmm2);
|
|
ymm2 = _mm256_inserti128_si256(
|
|
ymm2, _mm_loadu_si128((const __m128i*)(src_raw + 60)), 1);
|
|
|
|
__m128i xmm3 = _mm_loadu_si128((const __m128i*)(src_raw + 68));
|
|
__m256i ymm3 = _mm256_castsi128_si256(xmm3);
|
|
ymm3 = _mm256_inserti128_si256(
|
|
ymm3, _mm_loadu_si128((const __m128i*)(src_raw + 80)), 1);
|
|
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
|
|
ymm2 = _mm256_shuffle_epi8(ymm2, ymm_shuf);
|
|
ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf2);
|
|
|
|
ymm0 = _mm256_or_si256(ymm0, ymm_alpha);
|
|
ymm1 = _mm256_or_si256(ymm1, ymm_alpha);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm_alpha);
|
|
ymm3 = _mm256_or_si256(ymm3, ymm_alpha);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 64), ymm2);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 96), ymm3);
|
|
|
|
src_raw += 96;
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTOARGBROW_AVX2
|
|
|
|
#if defined(HAS_RAWTOARGBROW_AVX512BW) || \
|
|
defined(HAS_RGB24TOARGBROW_AVX512BW) || \
|
|
defined(HAS_RAWTORGB24ROW_AVX512BW) || \
|
|
defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
|
static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2,
|
|
3, 4, 4, 5};
|
|
static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8,
|
|
9, 10, 10, 11};
|
|
static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6,
|
|
7, 8, 8, 9};
|
|
static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4,
|
|
5, 6, 6, 7};
|
|
#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
|
|
// defined(HAS_RGB24TOARGBROW_AVX512BW) ||
|
|
// defined(HAS_RAWTORGB24ROW_AVX512BW) ||
|
|
// defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
|
|
|
#if defined(HAS_RGB24TOARGBROW_AVX512BW) || \
|
|
defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
|
static const uint8_t kShuffleMaskRGB24ToARGB_AVX512BW[32] = {
|
|
0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u,
|
|
4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, 128u};
|
|
#endif // defined(HAS_RGB24TOARGBROW_AVX512BW) ||
|
|
// defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
|
|
|
#if defined(HAS_RAWTOARGBROW_AVX512BW)
|
|
static const uint8_t kShuffleMaskRAWToARGB_AVX512BW[32] = {
|
|
2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u,
|
|
6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u};
|
|
#endif // defined(HAS_RAWTOARGBROW_AVX512BW)
|
|
|
|
#if defined(HAS_RAWTORGB24ROW_AVX2) || defined(HAS_RAWTORGB24ROW_AVX512BW)
|
|
static const uint8_t kShuffleMaskRAWToRGB24_AVX2[32] = {
|
|
2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 6u, 11u, 10u, 9u, 128u, 128u, 128u, 128u,
|
|
6u, 5u, 4u, 9u, 8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u, 128u, 128u, 128u, 128u};
|
|
#endif // defined(HAS_RAWTORGB24ROW_AVX2) ||
|
|
// defined(HAS_RAWTORGB24ROW_AVX512BW)
|
|
|
|
#if defined(HAS_RAWTORGB24ROW_AVX2)
|
|
static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 0, 0};
|
|
static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 0, 0, 0, 0};
|
|
static const uint32_t kPermd2_AVX2[8] = {0, 0, 0, 0, 0, 1, 2, 4};
|
|
static const uint32_t kPermd3_AVX2[8] = {5, 6, 0, 0, 0, 0, 0, 0};
|
|
static const uint32_t kPermd4_AVX2[8] = {0, 0, 0, 1, 2, 4, 5, 6};
|
|
#endif // defined(HAS_RAWTORGB24ROW_AVX2)
|
|
|
|
#if defined(HAS_RAWTORGB24ROW_AVX512BW)
|
|
static const uint32_t kPermdRGB24_0_AVX512BW[16] = {
|
|
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20};
|
|
static const uint32_t kPermdRGB24_1_AVX512BW[16] = {
|
|
5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25};
|
|
static const uint32_t kPermdRGB24_2_AVX512BW[16] = {
|
|
10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30};
|
|
#endif // defined(HAS_RAWTORGB24ROW_AVX512BW)
|
|
#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
|
|
static const uint8_t kPermRAWToRGB24_0[64] = {
|
|
0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a, 0x09,
|
|
0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17, 0x16, 0x15,
|
|
0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e, 0x23, 0x22, 0x21,
|
|
0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b, 0x2a, 0x2f, 0x2e, 0x2d,
|
|
0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38, 0x37, 0x36, 0x3b, 0x3a, 0x39,
|
|
0x3e, 0x3d, 0x3c, 0x41};
|
|
static const uint8_t kPermRAWToRGB24_1[64] = {
|
|
0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08, 0x0d,
|
|
0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15, 0x14, 0x19,
|
|
0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22, 0x21, 0x20, 0x25,
|
|
0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29, 0x2e, 0x2d, 0x2c, 0x31,
|
|
0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36, 0x35, 0x3a, 0x39, 0x38, 0x3d,
|
|
0x3c, 0x3b, 0x00, 0x3f};
|
|
static const uint8_t kPermRAWToRGB24_2[64] = {
|
|
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b,
|
|
0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
|
|
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23,
|
|
0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
|
|
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b,
|
|
0x3c, 0x3d, 0x40, 0x3f};
|
|
static const uint8_t kPermRAWToRGB24_3[64] = {
|
|
0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c, 0x4b,
|
|
0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53, 0x58, 0x57,
|
|
0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60, 0x5f, 0x64, 0x63,
|
|
0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d, 0x6c, 0x6b, 0x70, 0x6f,
|
|
0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74, 0x79, 0x78, 0x77, 0x7c, 0x7b,
|
|
0x7a, 0x7f, 0x7e, 0x7d};
|
|
#endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI)
|
|
|
|
#if defined(HAS_RAWTOARGBROW_AVX512BW) || defined(HAS_RGB24TOARGBROW_AVX512BW)
|
|
LIBYUV_TARGET_AVX512BW
|
|
static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
|
|
uint8_t* dst_argb,
|
|
const uint32_t* shuffler,
|
|
int width) {
|
|
__m512i zmm_alpha = _mm512_set1_epi32(0xff000000);
|
|
__m512i zmm_q0 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
|
|
__m512i zmm_q1 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
|
|
__m512i zmm_q2 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
|
|
__m512i zmm_q3 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
|
|
__m512i zmm_shuf =
|
|
_mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)shuffler));
|
|
|
|
while (width > 0) {
|
|
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
|
|
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
|
|
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
|
|
|
|
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
|
|
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
|
|
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
|
|
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_or_si512(zmm0, zmm_alpha);
|
|
zmm1 = _mm512_or_si512(zmm1, zmm_alpha);
|
|
zmm2 = _mm512_or_si512(zmm2, zmm_alpha);
|
|
zmm3 = _mm512_or_si512(zmm3, zmm_alpha);
|
|
|
|
_mm512_storeu_si512(dst_argb, zmm0);
|
|
_mm512_storeu_si512(dst_argb + 64, zmm1);
|
|
_mm512_storeu_si512(dst_argb + 128, zmm2);
|
|
_mm512_storeu_si512(dst_argb + 192, zmm3);
|
|
|
|
src_raw += 192;
|
|
dst_argb += 256;
|
|
width -= 64;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
|
|
// defined(HAS_RGB24TOARGBROW_AVX512BW)
|
|
|
|
#ifdef HAS_RAWTOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
RGBToARGBRow_AVX512BW(
|
|
src_raw, dst_argb,
|
|
(const uint32_t*)kShuffleMaskRAWToARGB_AVX512BW, width);
|
|
}
|
|
#endif // HAS_RAWTOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_RGB24TOARGBROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
RGBToARGBRow_AVX512BW(
|
|
src_rgb24, dst_argb,
|
|
(const uint32_t*)kShuffleMaskRGB24ToARGB_AVX512BW, width);
|
|
}
|
|
#endif // HAS_RGB24TOARGBROW_AVX512BW
|
|
|
|
#ifdef HAS_RAWTORGB24ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RAWToRGB24Row_AVX2(const uint8_t* src_raw,
|
|
uint8_t* dst_rgb24,
|
|
int width) {
|
|
__m256i ymm_shuf =
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskRAWToRGB24_AVX2);
|
|
__m256i ymm_p0 = _mm256_loadu_si256((const __m256i*)kPermd0_AVX2);
|
|
__m256i ymm_p1 = _mm256_loadu_si256((const __m256i*)kPermd1_AVX2);
|
|
__m256i ymm_p2 = _mm256_loadu_si256((const __m256i*)kPermd2_AVX2);
|
|
__m256i ymm_p3 = _mm256_loadu_si256((const __m256i*)kPermd3_AVX2);
|
|
__m256i ymm_p4 = _mm256_loadu_si256((const __m256i*)kPermd4_AVX2);
|
|
|
|
while (width > 0) {
|
|
__m256i raw0 = _mm256_loadu_si256((const __m256i*)src_raw);
|
|
__m256i raw1 = _mm256_loadu_si256((const __m256i*)(src_raw + 32));
|
|
__m256i raw2 = _mm256_loadu_si256((const __m256i*)(src_raw + 64));
|
|
|
|
__m256i b0 = _mm256_permute4x64_epi64(raw0, 0x94);
|
|
__m256i m01 = _mm256_permute2x128_si256(raw0, raw1, 0x21);
|
|
__m256i b1 = _mm256_permute4x64_epi64(m01, 0xe9);
|
|
|
|
__m256i m12 = _mm256_permute2x128_si256(raw1, raw2, 0x21);
|
|
__m256i b2 = _mm256_permute4x64_epi64(m12, 0x94);
|
|
__m256i b3 = _mm256_permute4x64_epi64(raw2, 0xe9);
|
|
|
|
b0 = _mm256_shuffle_epi8(b0, ymm_shuf);
|
|
b1 = _mm256_shuffle_epi8(b1, ymm_shuf);
|
|
b2 = _mm256_shuffle_epi8(b2, ymm_shuf);
|
|
b3 = _mm256_shuffle_epi8(b3, ymm_shuf);
|
|
|
|
// Dst0 = blend(b0_dwords, b1_q0)
|
|
__m256i d0 = _mm256_permutevar8x32_epi32(b0, ymm_p0);
|
|
__m256i b1_q0 = _mm256_permute4x64_epi64(b1, 0x00);
|
|
__m256i dst0 = _mm256_blend_epi32(d0, b1_q0, 0xc0);
|
|
|
|
// Dst1 = blend(b1_dwords, b2_dwords)
|
|
__m256i d1 = _mm256_permutevar8x32_epi32(b1, ymm_p1);
|
|
__m256i d2 = _mm256_permutevar8x32_epi32(b2, ymm_p2);
|
|
__m256i dst1 = _mm256_blend_epi32(d1, d2, 0xf0);
|
|
|
|
// Dst2 = blend(b2_dwords, b3_dwords)
|
|
__m256i d2_c45 = _mm256_permutevar8x32_epi32(b2, ymm_p3);
|
|
__m256i d3 = _mm256_permutevar8x32_epi32(b3, ymm_p4);
|
|
__m256i dst2 = _mm256_blend_epi32(d2_c45, d3, 0xfc);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_rgb24, dst0);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 32), dst1);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), dst2);
|
|
|
|
src_raw += 96;
|
|
dst_rgb24 += 96;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTORGB24ROW_AVX2
|
|
|
|
#ifdef HAS_RAWTORGB24ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw,
|
|
uint8_t* dst_rgb24,
|
|
int width) {
|
|
__m512i zmm_q0 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
|
|
__m512i zmm_q1 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
|
|
__m512i zmm_q2 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
|
|
__m512i zmm_q3 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
|
|
__m512i zmm_shuf = _mm512_broadcast_i64x4(
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskRAWToRGB24_AVX2));
|
|
__m512i zmm_d0 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_0_AVX512BW);
|
|
__m512i zmm_d1 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_1_AVX512BW);
|
|
__m512i zmm_d2 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_2_AVX512BW);
|
|
|
|
while (width > 0) {
|
|
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
|
|
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
|
|
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
|
|
|
|
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
|
|
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
|
|
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
|
|
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
__m512i dst0 = _mm512_permutex2var_epi32(zmm0, zmm_d0, zmm1);
|
|
__m512i dst1 = _mm512_permutex2var_epi32(zmm1, zmm_d1, zmm2);
|
|
__m512i dst2 = _mm512_permutex2var_epi32(zmm2, zmm_d2, zmm3);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), dst1);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), dst2);
|
|
|
|
src_raw += 192;
|
|
dst_rgb24 += 192;
|
|
width -= 64;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTORGB24ROW_AVX512BW
|
|
|
|
#ifdef HAS_RAWTORGB24ROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw,
|
|
uint8_t* dst_rgb24,
|
|
int width) {
|
|
__m512i zmm_p0 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_0);
|
|
__m512i zmm_p1 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_1);
|
|
__m512i zmm_p2 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_2);
|
|
__m512i zmm_p3 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_3);
|
|
|
|
while (width > 0) {
|
|
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
|
|
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
|
|
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
|
|
|
|
__m512i tmp1 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p1, zmm_raw0);
|
|
__m512i dst0 = _mm512_permutex2var_epi8(zmm_raw0, zmm_p0, zmm_raw1);
|
|
__m512i dst2 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p3, zmm_raw2);
|
|
__m512i dst1 = _mm512_permutex2var_epi8(tmp1, zmm_p2, zmm_raw2);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), dst1);
|
|
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), dst2);
|
|
|
|
src_raw += 192;
|
|
dst_rgb24 += 192;
|
|
width -= 64;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RAWTORGB24ROW_AVX512VBMI
|
|
|
|
#ifdef HAS_RGBTOYMATRIXROW_AVX512BW
|
|
static const uint32_t kPermdARGBToY_AVX512BW[16] = {
|
|
0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15};
|
|
|
|
LIBYUV_TARGET_AVX512BW
|
|
void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
|
|
uint8_t* dst_y,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m512i zmm_q0 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
|
|
__m512i zmm_q1 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
|
|
__m512i zmm_q2 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
|
|
__m512i zmm_q3 =
|
|
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
|
|
__m512i zmm_shuf = _mm512_broadcast_i64x4(
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskRGB24ToARGB_AVX512BW));
|
|
__m512i zmm_perm_y =
|
|
_mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
|
|
|
|
__m512i zmm_80 = _mm512_set1_epi8((char)0x80);
|
|
__m512i zmm_one = _mm512_set1_epi16(1);
|
|
__m512i zmm_coeff = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)c->kRGBToY));
|
|
__m512i zmm_add = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)c->kAddY));
|
|
|
|
// Compute bias adjusted for 0x80 subtraction:
|
|
// bias = kAddY - pmaddwd(pmaddubsw(kRGBToY, 0x80), 1)
|
|
__m512i zmm_bias = _mm512_maddubs_epi16(zmm_coeff, zmm_80);
|
|
zmm_bias = _mm512_madd_epi16(zmm_bias, zmm_one);
|
|
zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias);
|
|
zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias);
|
|
|
|
while (width > 0) {
|
|
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_rgb);
|
|
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_rgb + 64));
|
|
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_rgb + 128));
|
|
|
|
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
|
|
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
|
|
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
|
|
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
|
|
|
|
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
|
|
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
|
|
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
|
|
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
|
|
|
|
zmm0 = _mm512_xor_si512(zmm0, zmm_80);
|
|
zmm1 = _mm512_xor_si512(zmm1, zmm_80);
|
|
zmm2 = _mm512_xor_si512(zmm2, zmm_80);
|
|
zmm3 = _mm512_xor_si512(zmm3, zmm_80);
|
|
|
|
zmm0 = _mm512_maddubs_epi16(zmm_coeff, zmm0);
|
|
zmm1 = _mm512_maddubs_epi16(zmm_coeff, zmm1);
|
|
zmm2 = _mm512_maddubs_epi16(zmm_coeff, zmm2);
|
|
zmm3 = _mm512_maddubs_epi16(zmm_coeff, zmm3);
|
|
|
|
zmm0 = _mm512_madd_epi16(zmm0, zmm_one);
|
|
zmm1 = _mm512_madd_epi16(zmm1, zmm_one);
|
|
zmm0 = _mm512_packs_epi32(zmm0, zmm1);
|
|
|
|
zmm2 = _mm512_madd_epi16(zmm2, zmm_one);
|
|
zmm3 = _mm512_madd_epi16(zmm3, zmm_one);
|
|
zmm2 = _mm512_packs_epi32(zmm2, zmm3);
|
|
|
|
zmm0 = _mm512_add_epi16(zmm0, zmm_add);
|
|
zmm2 = _mm512_add_epi16(zmm2, zmm_add);
|
|
|
|
zmm0 = _mm512_srli_epi16(zmm0, 8);
|
|
zmm2 = _mm512_srli_epi16(zmm2, 8);
|
|
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
|
|
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_y, zmm0);
|
|
|
|
src_rgb += 192;
|
|
dst_y += 64;
|
|
width -= 64;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RGBTOYMATRIXROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBTOUVMATRIXROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
|
|
int src_stride_argb,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width,
|
|
const struct ArgbConstants* c) {
|
|
__m256i ymm_u = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToU));
|
|
__m256i ymm_v = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToV));
|
|
__m256i ymm_0101 = _mm256_set1_epi16(0x0101);
|
|
__m256i ymm_shuf =
|
|
_mm256_setr_epi8(0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15, 0,
|
|
4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15);
|
|
__m256i ymm_8000 = _mm256_set1_epi16((short)0x8000);
|
|
__m256i ymm_zero = _mm256_setzero_si256();
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
|
|
__m256i ymm2 =
|
|
_mm256_loadu_si256((const __m256i*)(src_argb + src_stride_argb));
|
|
__m256i ymm3 =
|
|
_mm256_loadu_si256((const __m256i*)(src_argb + src_stride_argb + 32));
|
|
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
|
|
ymm2 = _mm256_shuffle_epi8(ymm2, ymm_shuf);
|
|
ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf);
|
|
|
|
ymm0 = _mm256_maddubs_epi16(ymm0, ymm_0101);
|
|
ymm1 = _mm256_maddubs_epi16(ymm1, ymm_0101);
|
|
ymm2 = _mm256_maddubs_epi16(ymm2, ymm_0101);
|
|
ymm3 = _mm256_maddubs_epi16(ymm3, ymm_0101);
|
|
|
|
ymm0 = _mm256_add_epi16(ymm0, ymm2);
|
|
ymm1 = _mm256_add_epi16(ymm1, ymm3);
|
|
|
|
ymm0 = _mm256_srli_epi16(ymm0, 1);
|
|
ymm1 = _mm256_srli_epi16(ymm1, 1);
|
|
ymm0 = _mm256_avg_epu16(ymm0, ymm_zero);
|
|
ymm1 = _mm256_avg_epu16(ymm1, ymm_zero);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
|
|
ymm1 = _mm256_maddubs_epi16(ymm0, ymm_v);
|
|
ymm0 = _mm256_maddubs_epi16(ymm0, ymm_u);
|
|
|
|
ymm0 = _mm256_hadd_epi16(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm0 = _mm256_sub_epi16(ymm_8000, ymm0);
|
|
ymm0 = _mm256_srli_epi16(ymm0, 8);
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
|
|
__m128i xmm_u = _mm256_castsi256_si128(ymm0);
|
|
__m128i xmm_v = _mm256_extracti128_si256(ymm0, 1);
|
|
|
|
_mm_storel_epi64((__m128i*)dst_u, xmm_u);
|
|
_mm_storel_epi64((__m128i*)dst_v, xmm_v);
|
|
|
|
src_argb += 64;
|
|
dst_u += 8;
|
|
dst_v += 8;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOUVMATRIXROW_AVX2
|
|
|
|
#ifdef HAS_CONVERT8TO16ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void Convert8To16Row_AVX2(const uint8_t* src_y,
|
|
uint16_t* dst_y,
|
|
int bits,
|
|
int width) {
|
|
const int shift = 16 - bits;
|
|
__m128i shift128 = _mm_cvtsi32_si128(shift);
|
|
do {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
__m256i row0 = _mm256_unpacklo_epi8(ymm0, ymm0);
|
|
__m256i row1 = _mm256_unpackhi_epi8(ymm0, ymm0);
|
|
row0 = _mm256_srl_epi16(row0, shift128);
|
|
row1 = _mm256_srl_epi16(row1, shift128);
|
|
_mm256_storeu_si256((__m256i*)dst_y, row0);
|
|
_mm256_storeu_si256((__m256i*)(dst_y + 16), row1);
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_CONVERT8TO16ROW_AVX2
|
|
|
|
#ifdef HAS_MULTIPLYROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MultiplyRow_16_AVX2(const uint16_t* src_y,
|
|
uint16_t* dst_y,
|
|
int scale,
|
|
int width) {
|
|
__m256i scale256 = _mm256_set1_epi16((short)scale);
|
|
do {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_y + 16));
|
|
ymm0 = _mm256_mullo_epi16(ymm0, scale256);
|
|
ymm1 = _mm256_mullo_epi16(ymm1, scale256);
|
|
_mm256_storeu_si256((__m256i*)dst_y, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_y + 16), ymm1);
|
|
src_y += 32;
|
|
dst_y += 32;
|
|
width -= 32;
|
|
} while (width > 0);
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MULTIPLYROW_16_AVX2
|
|
|
|
#ifdef HAS_MERGEUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MergeUVRow_AVX2(const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_uv,
|
|
int width) {
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_u));
|
|
__m256i ymm1 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_v));
|
|
|
|
ymm1 = _mm256_slli_epi16(ymm1, 8);
|
|
ymm0 = _mm256_or_si256(ymm0, ymm1);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_uv, ymm0);
|
|
|
|
src_u += 16;
|
|
src_v += 16;
|
|
dst_uv += 32;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MERGEUVROW_AVX2
|
|
|
|
#ifdef HAS_MIRRORROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_setr_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
|
|
src += width;
|
|
while (width > 0) {
|
|
src -= 32;
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src);
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0x4e);
|
|
_mm256_storeu_si256((__m256i*)dst, ymm0);
|
|
dst += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORROW_AVX2
|
|
|
|
#ifdef HAS_MIRRORUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1));
|
|
src_uv += width * 2;
|
|
while (width > 0) {
|
|
src_uv -= 32;
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0x4e);
|
|
_mm256_storeu_si256((__m256i*)dst_uv, ymm0);
|
|
dst_uv += 32;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORUVROW_AVX2
|
|
|
|
#ifdef HAS_SWAPUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14));
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32));
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
|
|
_mm256_storeu_si256((__m256i*)dst_vu, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_vu + 32), ymm1);
|
|
src_uv += 64;
|
|
dst_vu += 64;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SWAPUVROW_AVX2
|
|
|
|
#ifdef HAS_MIRRORSPLITUVROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void MirrorSplitUVRow_AVX2(const uint8_t* src_uv,
|
|
uint8_t* dst_u,
|
|
uint8_t* dst_v,
|
|
int width) {
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1));
|
|
src_uv += width * 2;
|
|
while (width > 0) {
|
|
src_uv -= 32;
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_uv);
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0x72);
|
|
_mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(ymm0));
|
|
_mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(ymm0, 1));
|
|
dst_u += 16;
|
|
dst_v += 16;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_MIRRORSPLITUVROW_AVX2
|
|
|
|
#ifdef HAS_RGB24MIRRORROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
|
|
uint8_t* dst_rgb24,
|
|
int width) {
|
|
__m256i shuf0 =
|
|
_mm256_setr_epi8(-1, 12, 13, 14, 9, 10, 11, 6, 7, 8, 3, 4, 5, 0, 1, 2, -1,
|
|
12, 13, 14, 9, 10, 11, 6, 7, 8, 3, 4, 5, 0, 1, 2);
|
|
__m128i shuf1 =
|
|
_mm_setr_epi8(13, 14, 15, 10, 11, 12, 7, 8, 9, 4, 5, 6, 1, 2, 3, -1);
|
|
|
|
src_rgb24 += width * 3 - 96;
|
|
while (width > 0) {
|
|
__m128i v0_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 0));
|
|
__m128i v0_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 15));
|
|
__m256i v0 =
|
|
_mm256_inserti128_si256(_mm256_castsi128_si256(v0_lo), v0_hi, 1);
|
|
|
|
__m128i v1_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 30));
|
|
__m128i v1_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 45));
|
|
__m256i v1 =
|
|
_mm256_inserti128_si256(_mm256_castsi128_si256(v1_lo), v1_hi, 1);
|
|
|
|
__m128i v2_lo = _mm_loadu_si128((const __m128i*)(src_rgb24 + 60));
|
|
__m128i v2_hi = _mm_loadu_si128((const __m128i*)(src_rgb24 + 75));
|
|
__m256i v2 =
|
|
_mm256_inserti128_si256(_mm256_castsi128_si256(v2_lo), v2_hi, 1);
|
|
|
|
__m128i v3 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 80));
|
|
|
|
v0 = _mm256_shuffle_epi8(v0, shuf0);
|
|
v1 = _mm256_shuffle_epi8(v1, shuf0);
|
|
v2 = _mm256_shuffle_epi8(v2, shuf0);
|
|
v3 = _mm_shuffle_epi8(v3, shuf1);
|
|
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 80), _mm256_castsi256_si128(v0));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 65),
|
|
_mm256_extracti128_si256(v0, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 50), _mm256_castsi256_si128(v1));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 35),
|
|
_mm256_extracti128_si256(v1, 1));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 20), _mm256_castsi256_si128(v2));
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 5),
|
|
_mm256_extracti128_si256(v2, 1));
|
|
_mm_storel_epi64((__m128i*)(dst_rgb24 + 0), v3);
|
|
|
|
src_rgb24 -= 96;
|
|
dst_rgb24 += 96;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RGB24MIRRORROW_AVX2
|
|
|
|
#ifdef HAS_INTERPOLATEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void InterpolateRow_AVX2(uint8_t* dst_ptr,
|
|
const uint8_t* src_ptr,
|
|
ptrdiff_t src_stride,
|
|
int width,
|
|
int source_y_fraction) {
|
|
int y1 = source_y_fraction;
|
|
int y0 = 256 - y1;
|
|
const uint8_t* src_ptr1 = src_ptr + src_stride;
|
|
__m256i ymm_y = _mm256_set1_epi16((y1 << 8) | y0);
|
|
__m256i ymm_8080 = _mm256_set1_epi16(0x8080);
|
|
int i;
|
|
|
|
if (y1 == 0) {
|
|
for (i = 0; i < width; i += 32) {
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_loadu_si256((const __m256i*)(src_ptr + i)));
|
|
}
|
|
} else if (y1 == 128) {
|
|
for (i = 0; i < width; i += 32) {
|
|
__m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i), _mm256_avg_epu8(row0, row1));
|
|
}
|
|
} else {
|
|
for (i = 0; i < width; i += 32) {
|
|
__m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
|
|
__m256i low = _mm256_unpacklo_epi8(row0, row1);
|
|
__m256i high = _mm256_unpackhi_epi8(row0, row1);
|
|
low = _mm256_sub_epi8(low, ymm_8080);
|
|
high = _mm256_sub_epi8(high, ymm_8080);
|
|
low = _mm256_maddubs_epi16(ymm_y, low);
|
|
high = _mm256_maddubs_epi16(ymm_y, high);
|
|
low = _mm256_add_epi16(low, ymm_8080);
|
|
high = _mm256_add_epi16(high, ymm_8080);
|
|
low = _mm256_srli_epi16(low, 8);
|
|
high = _mm256_srli_epi16(high, 8);
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_packus_epi16(low, high));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_INTERPOLATEROW_AVX2
|
|
|
|
#ifdef HAS_INTERPOLATEROW_16_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void InterpolateRow_16_AVX2(uint16_t* dst_ptr,
|
|
const uint16_t* src_ptr,
|
|
ptrdiff_t src_stride,
|
|
int width,
|
|
int source_y_fraction) {
|
|
int y1 = source_y_fraction;
|
|
int y0 = 256 - y1;
|
|
const uint16_t* src_ptr1 = src_ptr + src_stride;
|
|
__m256i ymm_y = _mm256_set1_epi32((y1 << 16) | y0);
|
|
__m256i ymm_8000 = _mm256_set1_epi16((short)0x8000);
|
|
__m256i ymm_round = _mm256_set1_epi32(8388736); // 0x800000 + 128
|
|
int i;
|
|
|
|
if (y1 == 0) {
|
|
for (i = 0; i < width; i += 16) {
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_loadu_si256((const __m256i*)(src_ptr + i)));
|
|
}
|
|
} else if (y1 == 128) {
|
|
for (i = 0; i < width; i += 16) {
|
|
__m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_avg_epu16(row0, row1));
|
|
}
|
|
} else {
|
|
for (i = 0; i < width; i += 16) {
|
|
__m256i row0 = _mm256_loadu_si256((const __m256i*)(src_ptr + i));
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_ptr1 + i));
|
|
__m256i row0l = _mm256_unpacklo_epi16(row0, row1);
|
|
__m256i row0h = _mm256_unpackhi_epi16(row0, row1);
|
|
row0l = _mm256_sub_epi16(row0l, ymm_8000);
|
|
row0h = _mm256_sub_epi16(row0h, ymm_8000);
|
|
__m256i resl = _mm256_madd_epi16(row0l, ymm_y);
|
|
__m256i resh = _mm256_madd_epi16(row0h, ymm_y);
|
|
resl = _mm256_add_epi32(resl, ymm_round);
|
|
resh = _mm256_add_epi32(resh, ymm_round);
|
|
resl = _mm256_srai_epi32(resl, 8);
|
|
resh = _mm256_srai_epi32(resh, 8);
|
|
_mm256_storeu_si256((__m256i*)(dst_ptr + i),
|
|
_mm256_packus_epi32(resl, resh));
|
|
}
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_INTERPOLATEROW_16_AVX2
|
|
|
|
#ifdef HAS_ARGBMIRRORROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
|
__m256i ymm_shuf = _mm256_setr_epi32(7, 6, 5, 4, 3, 2, 1, 0);
|
|
src += width * 4;
|
|
while (width > 0) {
|
|
src -= 32;
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src);
|
|
ymm0 = _mm256_permutevar8x32_epi32(ymm0, ymm_shuf);
|
|
_mm256_storeu_si256((__m256i*)dst, ymm0);
|
|
dst += 32;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBMIRRORROW_AVX2
|
|
|
|
#ifdef HAS_J400TOARGBROW_AVX2
|
|
static const uint8_t kShuffleMaskJ400ToARGB_0[32] = {
|
|
0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u, 3u, 128u,
|
|
4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u, 7u, 7u, 7u, 128u};
|
|
static const uint8_t kShuffleMaskJ400ToARGB_1[32] = {
|
|
8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u, 10u,
|
|
128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, 13u, 13u,
|
|
13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u};
|
|
|
|
LIBYUV_TARGET_AVX2
|
|
void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
|
|
__m256i ymm_mask0 =
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_0);
|
|
__m256i ymm_mask1 =
|
|
_mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_1);
|
|
__m256i ymm_alpha = _mm256_set1_epi32((int)0xff000000u);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)src_y));
|
|
|
|
__m256i ymm1 = _mm256_shuffle_epi8(ymm0, ymm_mask0);
|
|
__m256i ymm2 = _mm256_shuffle_epi8(ymm0, ymm_mask1);
|
|
|
|
ymm1 = _mm256_or_si256(ymm1, ymm_alpha);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm_alpha);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm1);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm2);
|
|
|
|
src_y += 16;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_J400TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_RGB24TOARGBROW_AVX2
|
|
static const uint8_t kShuffleMaskRGB24ToARGB[2][16] = {
|
|
{0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u},
|
|
{4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u,
|
|
128u}};
|
|
#endif // HAS_RGB24TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_RGB565TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m256i ymm_scale_rb = _mm256_set1_epi32(0x01080108);
|
|
__m256i ymm_scale_g = _mm256_set1_epi32(0x20802080);
|
|
__m256i ymm_mask_b = _mm256_set1_epi16((short)0xf800);
|
|
__m256i ymm_mask_g = _mm256_set1_epi16(0x07e0);
|
|
__m256i ymm_mask_a = _mm256_set1_epi16((short)0xff00);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_rgb565);
|
|
__m256i ymm1 = ymm0;
|
|
__m256i ymm2 = ymm0;
|
|
|
|
ymm1 = _mm256_and_si256(ymm1, ymm_mask_b);
|
|
ymm2 = _mm256_slli_epi16(ymm2, 11);
|
|
ymm1 = _mm256_mulhi_epu16(ymm1, ymm_scale_rb);
|
|
ymm2 = _mm256_mulhi_epu16(ymm2, ymm_scale_rb);
|
|
ymm1 = _mm256_slli_epi16(ymm1, 8);
|
|
ymm1 = _mm256_or_si256(ymm1, ymm2); // RB
|
|
|
|
ymm0 = _mm256_and_si256(ymm0, ymm_mask_g);
|
|
ymm0 = _mm256_mulhi_epu16(ymm0, ymm_scale_g);
|
|
ymm0 = _mm256_or_si256(ymm0, ymm_mask_a); // GA
|
|
|
|
ymm2 = _mm256_unpacklo_epi8(ymm1, ymm0);
|
|
ymm1 = _mm256_unpackhi_epi8(ymm1, ymm0);
|
|
|
|
ymm0 = _mm256_permute2x128_si256(ymm2, ymm1, 0x20);
|
|
ymm1 = _mm256_permute2x128_si256(ymm2, ymm1, 0x31);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
|
|
src_rgb565 += 32;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RGB565TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_ARGB1555TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGB1555ToARGBRow_AVX2(const uint8_t* src_argb1555,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m256i ymm_scale_rb = _mm256_set1_epi32(0x01080108);
|
|
__m256i ymm_scale_g = _mm256_set1_epi32(0x42004200);
|
|
__m256i ymm_mask_b = _mm256_set1_epi16((short)0xf800);
|
|
__m256i ymm_mask_g = _mm256_set1_epi16(0x03e0);
|
|
__m256i ymm_mask_a = _mm256_set1_epi16((short)0xff00);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb1555);
|
|
__m256i ymm1 = ymm0;
|
|
__m256i ymm2 = ymm0;
|
|
|
|
ymm1 = _mm256_slli_epi16(ymm1, 1);
|
|
ymm2 = _mm256_slli_epi16(ymm2, 11);
|
|
ymm1 = _mm256_and_si256(ymm1, ymm_mask_b);
|
|
ymm2 = _mm256_mulhi_epu16(ymm2, ymm_scale_rb);
|
|
ymm1 = _mm256_mulhi_epu16(ymm1, ymm_scale_rb);
|
|
ymm1 = _mm256_slli_epi16(ymm1, 8);
|
|
ymm1 = _mm256_or_si256(ymm1, ymm2); // RB
|
|
|
|
ymm2 = ymm0;
|
|
ymm0 = _mm256_and_si256(ymm0, ymm_mask_g);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 8);
|
|
ymm0 = _mm256_mulhi_epu16(ymm0, ymm_scale_g);
|
|
ymm2 = _mm256_and_si256(ymm2, ymm_mask_a);
|
|
ymm0 = _mm256_or_si256(ymm0, ymm2); // GA
|
|
|
|
ymm2 = _mm256_unpacklo_epi8(ymm1, ymm0);
|
|
ymm1 = _mm256_unpackhi_epi8(ymm1, ymm0);
|
|
|
|
ymm0 = _mm256_permute2x128_si256(ymm2, ymm1, 0x20);
|
|
ymm1 = _mm256_permute2x128_si256(ymm2, ymm1, 0x31);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
|
|
src_argb1555 += 32;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGB1555TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_ARGB4444TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGB4444ToARGBRow_AVX2(const uint8_t* src_argb4444,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m256i ymm_mask = _mm256_set1_epi32(0x0f0f0f0f);
|
|
__m256i ymm_mask2 = _mm256_slli_epi32(ymm_mask, 4);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb4444);
|
|
__m256i ymm2 = ymm0;
|
|
|
|
ymm0 = _mm256_and_si256(ymm0, ymm_mask);
|
|
ymm2 = _mm256_and_si256(ymm2, ymm_mask2);
|
|
|
|
__m256i ymm1 = ymm0;
|
|
__m256i ymm3 = ymm2;
|
|
|
|
ymm1 = _mm256_slli_epi16(ymm1, 4);
|
|
ymm3 = _mm256_srli_epi16(ymm3, 4);
|
|
|
|
ymm0 = _mm256_or_si256(ymm0, ymm1);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm3);
|
|
|
|
ymm1 = ymm0;
|
|
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm2);
|
|
ymm1 = _mm256_unpackhi_epi8(ymm1, ymm2);
|
|
|
|
ymm2 = _mm256_permute2x128_si256(ymm0, ymm1, 0x20);
|
|
ymm1 = _mm256_permute2x128_si256(ymm0, ymm1, 0x31);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm2);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
|
|
src_argb4444 += 32;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGB4444TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTORGB565DITHERROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToRGB565DitherRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
uint32_t dither4,
|
|
int width) {
|
|
__m128i dither_shuf =
|
|
_mm_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3);
|
|
__m128i d128 = _mm_shuffle_epi8(_mm_cvtsi32_si128((int)dither4), dither_shuf);
|
|
__m256i ymm_dither = _mm256_broadcastsi128_si256(d128);
|
|
__m256i mask_b = _mm256_set1_epi32(0x0000001f);
|
|
__m256i mask_g = _mm256_set1_epi32(0x000007e0);
|
|
__m256i mask_r = _mm256_set1_epi32(0x0000f800);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
ymm0 = _mm256_adds_epu8(ymm0, ymm_dither);
|
|
__m256i ymm2 = _mm256_srli_epi32(ymm0, 5);
|
|
__m256i ymm1 = _mm256_srli_epi32(ymm0, 3);
|
|
ymm0 = _mm256_srli_epi32(ymm0, 8);
|
|
ymm2 = _mm256_and_si256(ymm2, mask_g);
|
|
ymm1 = _mm256_and_si256(ymm1, mask_b);
|
|
ymm0 = _mm256_and_si256(ymm0, mask_r);
|
|
ymm1 = _mm256_or_si256(ymm2, ymm1);
|
|
ymm0 = _mm256_or_si256(ymm1, ymm0);
|
|
ymm0 = _mm256_packus_epi32(ymm0, ymm0);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_rgb, _mm256_castsi256_si128(ymm0));
|
|
src_argb += 32;
|
|
dst_rgb += 16;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTORGB565DITHERROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTORGB565ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToRGB565Row_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_rgb,
|
|
int width) {
|
|
__m256i mask_b = _mm256_set1_epi32(0x0000001f);
|
|
__m256i mask_g = _mm256_set1_epi32(0x000007e0);
|
|
__m256i mask_r = _mm256_set1_epi32(0x0000f800);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm2 = _mm256_srli_epi32(ymm0, 5);
|
|
__m256i ymm1 = _mm256_srli_epi32(ymm0, 3);
|
|
ymm0 = _mm256_srli_epi32(ymm0, 8);
|
|
ymm2 = _mm256_and_si256(ymm2, mask_g);
|
|
ymm1 = _mm256_and_si256(ymm1, mask_b);
|
|
ymm0 = _mm256_and_si256(ymm0, mask_r);
|
|
ymm1 = _mm256_or_si256(ymm2, ymm1);
|
|
ymm0 = _mm256_or_si256(ymm1, ymm0);
|
|
ymm0 = _mm256_packus_epi32(ymm0, ymm0);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_rgb, _mm256_castsi256_si128(ymm0));
|
|
src_argb += 32;
|
|
dst_rgb += 16;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTORGB565ROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTOARGB1555ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToARGB1555Row_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb1555,
|
|
int width) {
|
|
__m256i mask_b = _mm256_set1_epi32(0x0000001f);
|
|
__m256i mask_g = _mm256_set1_epi32(0x000003e0);
|
|
__m256i mask_r = _mm256_set1_epi32(0x00007c00);
|
|
__m256i mask_a = _mm256_set1_epi32(0x00008000);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm2 = _mm256_srli_epi32(ymm0, 6);
|
|
__m256i ymm1 = _mm256_srli_epi32(ymm0, 3);
|
|
__m256i ymm7 = _mm256_srli_epi32(ymm0, 16);
|
|
ymm0 = _mm256_srli_epi32(ymm0, 9);
|
|
ymm2 = _mm256_and_si256(ymm2, mask_g);
|
|
ymm1 = _mm256_and_si256(ymm1, mask_b);
|
|
ymm7 = _mm256_and_si256(ymm7, mask_a);
|
|
ymm0 = _mm256_and_si256(ymm0, mask_r);
|
|
ymm1 = _mm256_or_si256(ymm2, ymm1);
|
|
ymm0 = _mm256_or_si256(ymm7, ymm0);
|
|
ymm0 = _mm256_or_si256(ymm1, ymm0);
|
|
ymm0 = _mm256_packus_epi32(ymm0, ymm0);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_argb1555, _mm256_castsi256_si128(ymm0));
|
|
src_argb += 32;
|
|
dst_argb1555 += 16;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOARGB1555ROW_AVX2
|
|
|
|
#ifdef HAS_ARGBTOARGB4444ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBToARGB4444Row_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb4444,
|
|
int width) {
|
|
__m256i mask_b = _mm256_set1_epi32(0x0000000f);
|
|
__m256i mask_g = _mm256_set1_epi32(0x000000f0);
|
|
__m256i mask_r = _mm256_set1_epi32(0x00000f00);
|
|
__m256i mask_a = _mm256_set1_epi32(0x0000f000);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i ymm2 = _mm256_srli_epi32(ymm0, 8);
|
|
__m256i ymm1 = _mm256_srli_epi32(ymm0, 4);
|
|
__m256i ymm7 = _mm256_srli_epi32(ymm0, 16);
|
|
ymm0 = _mm256_srli_epi32(ymm0, 12);
|
|
ymm2 = _mm256_and_si256(ymm2, mask_g);
|
|
ymm1 = _mm256_and_si256(ymm1, mask_b);
|
|
ymm7 = _mm256_and_si256(ymm7, mask_a);
|
|
ymm0 = _mm256_and_si256(ymm0, mask_r);
|
|
ymm1 = _mm256_or_si256(ymm2, ymm1);
|
|
ymm0 = _mm256_or_si256(ymm7, ymm0);
|
|
ymm0 = _mm256_or_si256(ymm1, ymm0);
|
|
ymm0 = _mm256_packus_epi32(ymm0, ymm0);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
_mm_storeu_si128((__m128i*)dst_argb4444, _mm256_castsi256_si128(ymm0));
|
|
src_argb += 32;
|
|
dst_argb4444 += 16;
|
|
width -= 8;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBTOARGB4444ROW_AVX2
|
|
|
|
#ifdef HAS_RGB24TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m256i ymm_alpha = _mm256_set1_epi32(0xff000000);
|
|
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[0]));
|
|
__m256i ymm_shuf2 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[1]));
|
|
|
|
while (width > 0) {
|
|
__m128i xmm0 = _mm_loadu_si128((const __m128i*)src_rgb24);
|
|
__m256i ymm0 = _mm256_castsi128_si256(xmm0);
|
|
ymm0 = _mm256_inserti128_si256(
|
|
ymm0, _mm_loadu_si128((const __m128i*)(src_rgb24 + 12)), 1);
|
|
|
|
__m128i xmm1 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 24));
|
|
__m256i ymm1 = _mm256_castsi128_si256(xmm1);
|
|
ymm1 = _mm256_inserti128_si256(
|
|
ymm1, _mm_loadu_si128((const __m128i*)(src_rgb24 + 36)), 1);
|
|
|
|
__m128i xmm2 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 48));
|
|
__m256i ymm2 = _mm256_castsi128_si256(xmm2);
|
|
ymm2 = _mm256_inserti128_si256(
|
|
ymm2, _mm_loadu_si128((const __m128i*)(src_rgb24 + 60)), 1);
|
|
|
|
__m128i xmm3 = _mm_loadu_si128((const __m128i*)(src_rgb24 + 68));
|
|
__m256i ymm3 = _mm256_castsi128_si256(xmm3);
|
|
ymm3 = _mm256_inserti128_si256(
|
|
ymm3, _mm_loadu_si128((const __m128i*)(src_rgb24 + 80)), 1);
|
|
|
|
ymm0 = _mm256_shuffle_epi8(ymm0, ymm_shuf);
|
|
ymm1 = _mm256_shuffle_epi8(ymm1, ymm_shuf);
|
|
ymm2 = _mm256_shuffle_epi8(ymm2, ymm_shuf);
|
|
ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf2);
|
|
|
|
ymm0 = _mm256_or_si256(ymm0, ymm_alpha);
|
|
ymm1 = _mm256_or_si256(ymm1, ymm_alpha);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm_alpha);
|
|
ymm3 = _mm256_or_si256(ymm3, ymm_alpha);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, ymm0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), ymm1);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 64), ymm2);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 96), ymm3);
|
|
|
|
src_rgb24 += 96;
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_RGB24TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_ARGBSHUFFLEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void ARGBShuffleRow_AVX2(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
const uint8_t* shuffler,
|
|
int width) {
|
|
__m256i control =
|
|
_mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuffler));
|
|
while (width >= 16) {
|
|
__m256i row = _mm256_loadu_si256((const __m256i*)src_argb);
|
|
__m256i row1 = _mm256_loadu_si256((const __m256i*)(src_argb + 32));
|
|
row = _mm256_shuffle_epi8(row, control);
|
|
row1 = _mm256_shuffle_epi8(row1, control);
|
|
_mm256_storeu_si256((__m256i*)dst_argb, row);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), row1);
|
|
src_argb += 64;
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_ARGBSHUFFLEROW_AVX2
|
|
|
|
#ifdef HAS_ARGBSHUFFLEROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
|
|
uint8_t* dst_argb,
|
|
const uint8_t* shuffler,
|
|
int width) {
|
|
__m512i control =
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuffler));
|
|
while (width >= 32) {
|
|
__m512i row = _mm512_loadu_si512((const __m512i*)src_argb);
|
|
__m512i row1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64));
|
|
row = _mm512_shuffle_epi8(row, control);
|
|
row1 = _mm512_shuffle_epi8(row1, control);
|
|
_mm512_storeu_si512((__m512i*)dst_argb, row);
|
|
_mm512_storeu_si512((__m512i*)(dst_argb + 64), row1);
|
|
src_argb += 128;
|
|
dst_argb += 128;
|
|
width -= 32;
|
|
}
|
|
}
|
|
#endif // HAS_ARGBSHUFFLEROW_AVX512BW
|
|
|
|
#ifdef HAS_I422TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void I422ToARGBRow_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 16) {
|
|
__m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
|
|
src_u += 8;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
|
|
__m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
|
|
|
|
ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_srai_epi16(ymm0, 6);
|
|
ymm1 = _mm256_srai_epi16(ymm1, 6);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 6);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
|
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
|
|
|
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
|
|
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
|
|
|
|
__m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
|
|
__m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, argb0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
|
|
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_I422TOAR30ROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void I422ToAR30Row_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb = _mm256_sub_epi16(
|
|
_mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb),
|
|
_mm256_set1_epi16(24));
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_3ff0 = _mm256_set1_epi16((short)0x3ff0);
|
|
__m256i ymm_c000 = _mm256_set1_epi16((short)0xc000);
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 16) {
|
|
__m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
|
|
src_u += 8;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
|
|
__m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
|
|
|
|
ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_min_epi16(ymm0, ymm_3ff0);
|
|
ymm1 = _mm256_min_epi16(ymm1, ymm_3ff0);
|
|
ymm2 = _mm256_min_epi16(ymm2, ymm_3ff0);
|
|
ymm0 = _mm256_max_epi16(ymm0, ymm_u_zero);
|
|
ymm1 = _mm256_max_epi16(ymm1, ymm_u_zero);
|
|
ymm2 = _mm256_max_epi16(ymm2, ymm_u_zero);
|
|
|
|
__m256i ymm_g_masked = _mm256_and_si256(ymm1, ymm_3ff0);
|
|
__m256i ymm_r_masked = _mm256_and_si256(ymm2, ymm_3ff0);
|
|
ymm0 = _mm256_srli_epi16(ymm0, 4);
|
|
ymm3 = _mm256_slli_epi16(ymm_g_masked, 6);
|
|
ymm0 = _mm256_or_si256(ymm0, ymm3);
|
|
ymm1 = _mm256_srli_epi16(ymm1, 10);
|
|
ymm2 = _mm256_or_si256(ymm_r_masked, ymm_c000);
|
|
ymm2 = _mm256_or_si256(ymm2, ymm1);
|
|
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
|
|
|
|
__m256i ar30_lo = _mm256_unpacklo_epi16(ymm0, ymm2);
|
|
__m256i ar30_hi = _mm256_unpackhi_epi16(ymm0, ymm2);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_ar30, ar30_lo);
|
|
_mm256_storeu_si256((__m256i*)(dst_ar30 + 32), ar30_hi);
|
|
|
|
dst_ar30 += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TOAR30ROW_AVX2
|
|
|
|
#ifdef HAS_NV12TOARGBROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void NV12ToARGBRow_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_uv,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
while (width >= 16) {
|
|
__m128i xmm_uv = _mm_loadu_si128((const __m128i*)src_uv);
|
|
src_uv += 16;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_uv, 0);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
__m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_srai_epi16(ymm0, 6);
|
|
ymm1 = _mm256_srai_epi16(ymm1, 6);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 6);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
|
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
|
|
|
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
|
|
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
|
|
|
|
__m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
|
|
__m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, argb0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
|
|
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV12TOARGBROW_AVX2
|
|
|
|
#ifdef HAS_NV21TOARGBROW_AVX2
|
|
static const uint8_t kShuffleNV21_AVX2[32] = {
|
|
1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u,
|
|
1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u};
|
|
|
|
LIBYUV_TARGET_AVX2
|
|
void NV21ToARGBRow_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_vu,
|
|
uint8_t* dst_argb,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
|
|
__m256i ymm_shuf_nv21 = _mm256_loadu_si256((const __m256i*)kShuffleNV21_AVX2);
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
while (width >= 16) {
|
|
__m128i xmm_vu = _mm_loadu_si128((const __m128i*)src_vu);
|
|
src_vu += 16;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_vu, 0);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf_nv21);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
__m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_srai_epi16(ymm0, 6);
|
|
ymm1 = _mm256_srai_epi16(ymm1, 6);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 6);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
|
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
|
|
|
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
|
|
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
|
|
ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
|
|
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
|
|
|
|
__m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
|
|
__m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
|
|
|
|
_mm256_storeu_si256((__m256i*)dst_argb, argb0);
|
|
_mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
|
|
|
|
dst_argb += 64;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_NV21TOARGBROW_AVX2
|
|
|
|
#if defined(HAS_I422TORGB24ROW_AVX2) || defined(HAS_I422TORGB24ROW_AVX512BW)
|
|
static const uint8_t kShuffleMaskARGBToRGB24[2][16] = {
|
|
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
|
|
128u},
|
|
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u,
|
|
14u}};
|
|
|
|
LIBYUV_TARGET_AVX2
|
|
void I422ToRGB24Row_AVX2(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
// Constants
|
|
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
|
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
|
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
|
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
|
__m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
|
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
|
|
|
__m256i ymm_shuf0 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1]));
|
|
__m256i ymm_shuf1 = _mm256_broadcastsi128_si256(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0]));
|
|
__m256i ymm_u_zero = _mm256_setzero_si256();
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 16) {
|
|
// READYUV422_AVX2
|
|
__m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
|
|
src_u += 8;
|
|
|
|
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
|
|
__m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
|
|
|
|
ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
|
|
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
|
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
|
|
|
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
|
src_y += 16;
|
|
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
|
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
|
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
|
|
|
// YUVTORGB_AVX2
|
|
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
|
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
|
|
|
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
|
ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
|
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
|
|
|
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
|
|
|
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
|
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
|
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
|
|
|
ymm0 = _mm256_srai_epi16(ymm0, 6);
|
|
ymm1 = _mm256_srai_epi16(ymm1, 6);
|
|
ymm2 = _mm256_srai_epi16(ymm2, 6);
|
|
|
|
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
|
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
|
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
|
|
|
// STORERGB24_AVX2
|
|
__m256i ymm0_packed = _mm256_unpacklo_epi8(ymm0, ymm1);
|
|
__m256i ymm2_packed = _mm256_unpacklo_epi8(ymm2, ymm2);
|
|
__m256i ymm1_packed = ymm0_packed;
|
|
|
|
ymm0_packed = _mm256_unpacklo_epi16(ymm0_packed, ymm2_packed);
|
|
ymm1_packed = _mm256_unpackhi_epi16(ymm1_packed, ymm2_packed);
|
|
|
|
ymm0_packed = _mm256_shuffle_epi8(ymm0_packed, ymm_shuf0);
|
|
ymm1_packed = _mm256_shuffle_epi8(ymm1_packed, ymm_shuf1);
|
|
|
|
ymm1_packed = _mm256_alignr_epi8(ymm1_packed, ymm0_packed, 0xc);
|
|
|
|
__m128i xmm0_store = _mm256_castsi256_si128(ymm0_packed);
|
|
__m128i xmm1_store = _mm256_castsi256_si128(ymm1_packed);
|
|
__m128i xmm2_store = _mm256_extractf128_si256(ymm0_packed, 1);
|
|
__m128i xmm3_store = _mm256_extractf128_si256(ymm1_packed, 1);
|
|
|
|
xmm0_store = _mm_unpacklo_epi64(xmm0_store, xmm1_store);
|
|
xmm2_store = _mm_alignr_epi8(xmm2_store, xmm1_store, 8);
|
|
|
|
_mm_storeu_si128((__m128i*)dst_rgb24, xmm0_store);
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 16), xmm2_store);
|
|
_mm_storeu_si128((__m128i*)(dst_rgb24 + 32), xmm3_store);
|
|
|
|
dst_rgb24 += 48;
|
|
width -= 16;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // defined(HAS_I422TORGB24ROW_AVX2) ||
|
|
// defined(HAS_I422TORGB24ROW_AVX512BW)
|
|
|
|
#ifdef HAS_I422TORGB24ROW_AVX512VBMI
|
|
LIBYUV_TARGET_AVX512VBMI
|
|
void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
// Masks
|
|
static const uint8_t kMaskBG[64] = {
|
|
0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, 0x45,
|
|
0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, 0x13, 0x53,
|
|
0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, 0x60, 0x21, 0x61,
|
|
0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, 0x26, 0x66, 0x27, 0x67,
|
|
0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, 0x73, 0x34, 0x74, 0x35, 0x75,
|
|
0x36, 0x76, 0x37, 0x77};
|
|
static const uint8_t kMaskDST0[64] = {
|
|
0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, 0x43,
|
|
0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, 0x0f, 0x47,
|
|
0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, 0x16, 0x17, 0x53,
|
|
0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, 0x56, 0x1e, 0x1f, 0x57,
|
|
0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, 0x25, 0x62, 0x26, 0x27, 0x63,
|
|
0x28, 0x29, 0x64, 0x2a};
|
|
static const uint8_t kMaskDST1[64] = {
|
|
0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, 0x32,
|
|
0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, 0x74, 0x3a,
|
|
0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00};
|
|
|
|
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
|
|
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
|
|
|
|
// Constants
|
|
__m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
__m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
__m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
__m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
__m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
|
__m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
|
|
__m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG);
|
|
__m512i zmm_mask_DST0 = _mm512_loadu_si512((const __m512i*)kMaskDST0);
|
|
__m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1);
|
|
__m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
|
|
__m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 32) {
|
|
// READYUV422_AVX512BW
|
|
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
|
|
src_u += 16;
|
|
|
|
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
|
|
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
|
|
|
|
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
|
|
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
|
|
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
|
|
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
|
|
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
|
|
|
|
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
|
|
src_y += 32;
|
|
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
|
|
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
|
|
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
|
|
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
|
|
|
// YUVTORGB_AVX512BW
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
|
|
|
|
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
|
|
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
|
|
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
|
|
|
|
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
|
|
|
|
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
|
|
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
|
|
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
|
|
|
|
zmm0 = _mm512_srai_epi16(zmm0, 6);
|
|
zmm1 = _mm512_srai_epi16(zmm1, 6);
|
|
zmm2 = _mm512_srai_epi16(zmm2, 6);
|
|
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
|
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
|
|
|
// STORERGB24_AVX512VBMI
|
|
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1));
|
|
|
|
dst_rgb24 += 96;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TORGB24ROW_AVX512VBMI
|
|
|
|
#ifdef HAS_I422TORGB24ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_rgb24,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12};
|
|
static const uint64_t kStitchRGB24_1[8] = {13, 6, 14, 15, 0, 0, 0, 0};
|
|
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
|
|
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
|
|
|
|
// Constants
|
|
__m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
__m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
__m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
__m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
__m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
|
__m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
|
|
__m512i zmm_shuf0 = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1]));
|
|
__m512i zmm_shuf1 = _mm512_broadcast_i32x4(
|
|
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0]));
|
|
__m512i zmm_stitch0 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_0);
|
|
__m512i zmm_stitch1 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_1);
|
|
__m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
|
|
__m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 32) {
|
|
// READYUV422_AVX512BW
|
|
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
|
|
src_u += 16;
|
|
|
|
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
|
|
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
|
|
|
|
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
|
|
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
|
|
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
|
|
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
|
|
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
|
|
|
|
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
|
|
src_y += 32;
|
|
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
|
|
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
|
|
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
|
|
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
|
|
|
// YUVTORGB_AVX512BW
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
|
|
|
|
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
|
|
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
|
|
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
|
|
|
|
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
|
|
|
|
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
|
|
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
|
|
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
|
|
|
|
zmm0 = _mm512_srai_epi16(zmm0, 6);
|
|
zmm1 = _mm512_srai_epi16(zmm1, 6);
|
|
zmm2 = _mm512_srai_epi16(zmm2, 6);
|
|
|
|
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
|
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
|
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
|
|
|
// STORERGB24_AVX512BW
|
|
__m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1);
|
|
__m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2);
|
|
__m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr);
|
|
__m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr);
|
|
zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0);
|
|
zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1);
|
|
zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12);
|
|
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
|
|
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1));
|
|
|
|
dst_rgb24 += 96;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TORGB24ROW_AVX512BW
|
|
|
|
#ifdef HAS_I422TOAR30ROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
|
|
const uint8_t* src_u,
|
|
const uint8_t* src_v,
|
|
uint8_t* dst_ar30,
|
|
const struct YuvConstants* yuvconstants,
|
|
int width) {
|
|
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
|
|
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
|
|
static const uint16_t kPermAR30_0[32] = {
|
|
0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
|
|
8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47};
|
|
static const uint16_t kPermAR30_1[32] = {
|
|
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55,
|
|
24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63};
|
|
|
|
// Constants
|
|
__m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
|
__m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
|
__m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
|
__m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
|
__m512i zmm_kYBiasToRgb = _mm512_sub_epi16(
|
|
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)),
|
|
_mm512_set1_epi16(24));
|
|
__m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
|
__m512i zmm_3ff0 = _mm512_set1_epi16((short)0x3ff0);
|
|
__m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00);
|
|
__m512i zmm_c000 = _mm512_set1_epi16((short)0xc000);
|
|
__m512i zmm_zero = _mm512_setzero_si512();
|
|
|
|
__m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0);
|
|
__m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1);
|
|
__m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
|
|
__m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
|
|
|
|
ptrdiff_t offset = src_v - src_u;
|
|
|
|
while (width >= 32) {
|
|
// READYUV422_AVX512BW
|
|
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
|
|
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
|
|
src_u += 16;
|
|
|
|
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
|
|
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
|
|
|
|
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
|
|
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
|
|
|
|
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
|
|
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
|
|
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
|
|
|
|
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
|
|
src_y += 32;
|
|
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
|
|
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
|
|
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
|
|
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
|
|
|
// YUVTORGB_AVX512BW
|
|
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
|
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
|
|
|
|
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
|
|
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
|
|
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
|
|
|
|
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
|
|
|
|
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
|
|
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
|
|
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
|
|
|
|
// Clamp to [0, 0x3ff0] (1023 * 16)
|
|
zmm0 = _mm512_min_epi16(zmm0, zmm_3ff0);
|
|
zmm1 = _mm512_min_epi16(zmm1, zmm_3ff0);
|
|
zmm2 = _mm512_min_epi16(zmm2, zmm_3ff0);
|
|
zmm0 = _mm512_max_epi16(zmm0, zmm_zero);
|
|
zmm1 = _mm512_max_epi16(zmm1, zmm_zero);
|
|
zmm2 = _mm512_max_epi16(zmm2, zmm_zero);
|
|
|
|
// STOREAR30_AVX512BW:
|
|
zmm0 = _mm512_srli_epi16(zmm0, 4);
|
|
zmm3 = _mm512_srli_epi16(zmm1, 10);
|
|
zmm1 = _mm512_slli_epi16(zmm1, 6);
|
|
zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm1, zmm_fc00, 0xf8);
|
|
zmm2 = _mm512_ternarylogic_epi32(zmm2, zmm3, zmm_3ff0, 0xe4);
|
|
zmm2 = _mm512_or_si512(zmm2, zmm_c000);
|
|
|
|
__m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2);
|
|
__m512i zmm_dst1 = _mm512_permutex2var_epi16(zmm0, zmm_perm1, zmm2);
|
|
|
|
_mm512_storeu_si512((__m512i*)dst_ar30, zmm_dst0);
|
|
_mm512_storeu_si512((__m512i*)(dst_ar30 + 64), zmm_dst1);
|
|
|
|
dst_ar30 += 128;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_I422TOAR30ROW_AVX512BW
|
|
|
|
#ifdef HAS_ARGBBLENDROW_SSSE3
|
|
static const uint8_t kShuffleAlpha[16] = {
|
|
3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80,
|
|
11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80};
|
|
|
|
void ARGBBlendRow_SSSE3(const uint8_t* src_argb,
|
|
const uint8_t* src_argb1,
|
|
uint8_t* dst_argb,
|
|
int width) {
|
|
__m128i shuff = _mm_loadu_si128((const __m128i*)kShuffleAlpha);
|
|
__m128i xmm_1 = _mm_set1_epi16(1);
|
|
__m128i mask_lo = _mm_set1_epi16(0x00ff);
|
|
__m128i mask_hi = _mm_set1_epi16((short)0xff00);
|
|
__m128i mask_alpha = _mm_set1_epi32((int)0xff000000);
|
|
|
|
while (width >= 4) {
|
|
__m128i xmm_s0 = _mm_loadu_si128((const __m128i*)src_argb);
|
|
__m128i xmm_s1 = _mm_loadu_si128((const __m128i*)src_argb1);
|
|
src_argb += 16;
|
|
src_argb1 += 16;
|
|
|
|
__m128i xmm_a = _mm_xor_si128(xmm_s0, mask_alpha);
|
|
xmm_a = _mm_shuffle_epi8(xmm_a, shuff);
|
|
xmm_a = _mm_add_epi16(xmm_a, xmm_1);
|
|
|
|
__m128i xmm_s1_lo = _mm_and_si128(xmm_s1, mask_lo);
|
|
__m128i xmm_s1_hi = _mm_srli_epi16(xmm_s1, 8);
|
|
|
|
xmm_s1_lo = _mm_mullo_epi16(xmm_s1_lo, xmm_a);
|
|
xmm_s1_hi = _mm_mullo_epi16(xmm_s1_hi, xmm_a);
|
|
|
|
xmm_s1_lo = _mm_srli_epi16(xmm_s1_lo, 8);
|
|
xmm_s1_hi = _mm_and_si128(xmm_s1_hi, mask_hi);
|
|
|
|
__m128i res = _mm_or_si128(xmm_s0, mask_alpha);
|
|
res = _mm_adds_epu8(res, xmm_s1_lo);
|
|
res = _mm_adds_epu8(res, xmm_s1_hi);
|
|
|
|
_mm_storeu_si128((__m128i*)dst_argb, res);
|
|
dst_argb += 16;
|
|
width -= 4;
|
|
}
|
|
|
|
while (width > 0) {
|
|
uint32_t s0 = *(const uint32_t*)src_argb;
|
|
uint32_t s1 = *(const uint32_t*)src_argb1;
|
|
src_argb += 4;
|
|
src_argb1 += 4;
|
|
|
|
uint32_t a = 256 - (s0 >> 24);
|
|
uint32_t b = (((s1 & 0x000000ff) * a) >> 8) + (s0 & 0x000000ff);
|
|
uint32_t g = (((s1 & 0x0000ff00) * a) >> 8) + (s0 & 0x0000ff00);
|
|
uint32_t r = (((s1 & 0x00ff0000) >> 8) * a) + (s0 & 0x00ff0000);
|
|
|
|
b = (b > 255) ? 255 : b;
|
|
g = (g > 0xff00) ? 0xff00 : (g & 0xff00);
|
|
r = (r > 0x00ff0000) ? 0x00ff0000 : (r & 0x00ff0000);
|
|
|
|
*(uint32_t*)dst_argb = 0xff000000 | r | g | b;
|
|
dst_argb += 4;
|
|
--width;
|
|
}
|
|
}
|
|
#endif // HAS_ARGBBLENDROW_SSSE3
|
|
|
|
#ifdef HAS_BLENDPLANEROW_SSSE3
|
|
void BlendPlaneRow_SSSE3(const uint8_t* src0,
|
|
const uint8_t* src1,
|
|
const uint8_t* alpha,
|
|
uint8_t* dst,
|
|
int width) {
|
|
__m128i xmm_ff00 = _mm_set1_epi16((short)0xff00);
|
|
__m128i xmm_80 = _mm_set1_epi8((char)0x80);
|
|
__m128i xmm_round = _mm_set1_epi16((short)0x807f);
|
|
|
|
while (width > 0) {
|
|
__m128i xmm_a = _mm_loadl_epi64((const __m128i*)alpha);
|
|
__m128i xmm_s0 = _mm_loadl_epi64((const __m128i*)src0);
|
|
__m128i xmm_s1 = _mm_loadl_epi64((const __m128i*)src1);
|
|
alpha += 8;
|
|
src0 += 8;
|
|
src1 += 8;
|
|
|
|
xmm_a = _mm_unpacklo_epi8(xmm_a, xmm_a);
|
|
xmm_a = _mm_xor_si128(xmm_a, xmm_ff00);
|
|
|
|
__m128i xmm_s = _mm_unpacklo_epi8(xmm_s0, xmm_s1);
|
|
xmm_s = _mm_sub_epi8(xmm_s, xmm_80);
|
|
|
|
__m128i xmm_p = _mm_maddubs_epi16(xmm_a, xmm_s);
|
|
xmm_p = _mm_add_epi16(xmm_p, xmm_round);
|
|
xmm_p = _mm_srli_epi16(xmm_p, 8);
|
|
|
|
__m128i xmm_d = _mm_packus_epi16(xmm_p, xmm_p);
|
|
_mm_storel_epi64((__m128i*)dst, xmm_d);
|
|
dst += 8;
|
|
width -= 8;
|
|
}
|
|
}
|
|
#endif // HAS_BLENDPLANEROW_SSSE3
|
|
|
|
#ifdef HAS_BLENDPLANEROW_AVX2
|
|
LIBYUV_TARGET_AVX2
|
|
void BlendPlaneRow_AVX2(const uint8_t* src0,
|
|
const uint8_t* src1,
|
|
const uint8_t* alpha,
|
|
uint8_t* dst,
|
|
int width) {
|
|
__m256i ymm_ff00 = _mm256_set1_epi16((short)0xff00);
|
|
__m256i ymm_80 = _mm256_set1_epi8((char)0x80);
|
|
__m256i ymm_round = _mm256_set1_epi16((short)0x807f);
|
|
|
|
while (width > 0) {
|
|
__m256i ymm_a = _mm256_loadu_si256((const __m256i*)alpha);
|
|
__m256i ymm_s0 = _mm256_loadu_si256((const __m256i*)src0);
|
|
__m256i ymm_s1 = _mm256_loadu_si256((const __m256i*)src1);
|
|
alpha += 32;
|
|
src0 += 32;
|
|
src1 += 32;
|
|
|
|
__m256i ymm_a_lo = _mm256_unpacklo_epi8(ymm_a, ymm_a);
|
|
__m256i ymm_a_hi = _mm256_unpackhi_epi8(ymm_a, ymm_a);
|
|
ymm_a_lo = _mm256_xor_si256(ymm_a_lo, ymm_ff00);
|
|
ymm_a_hi = _mm256_xor_si256(ymm_a_hi, ymm_ff00);
|
|
|
|
__m256i ymm_s_lo = _mm256_unpacklo_epi8(ymm_s0, ymm_s1);
|
|
__m256i ymm_s_hi = _mm256_unpackhi_epi8(ymm_s0, ymm_s1);
|
|
ymm_s_lo = _mm256_sub_epi8(ymm_s_lo, ymm_80);
|
|
ymm_s_hi = _mm256_sub_epi8(ymm_s_hi, ymm_80);
|
|
|
|
__m256i ymm_p_lo = _mm256_maddubs_epi16(ymm_a_lo, ymm_s_lo);
|
|
__m256i ymm_p_hi = _mm256_maddubs_epi16(ymm_a_hi, ymm_s_hi);
|
|
|
|
ymm_p_lo = _mm256_add_epi16(ymm_p_lo, ymm_round);
|
|
ymm_p_hi = _mm256_add_epi16(ymm_p_hi, ymm_round);
|
|
|
|
ymm_p_lo = _mm256_srli_epi16(ymm_p_lo, 8);
|
|
ymm_p_hi = _mm256_srli_epi16(ymm_p_hi, 8);
|
|
|
|
__m256i ymm_d = _mm256_packus_epi16(ymm_p_lo, ymm_p_hi);
|
|
_mm256_storeu_si256((__m256i*)dst, ymm_d);
|
|
dst += 32;
|
|
width -= 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_BLENDPLANEROW_AVX2
|
|
|
|
#ifdef HAS_BLENDPLANEROW_AVX512BW
|
|
LIBYUV_TARGET_AVX512BW
|
|
void BlendPlaneRow_AVX512BW(const uint8_t* src0,
|
|
const uint8_t* src1,
|
|
const uint8_t* alpha,
|
|
uint8_t* dst,
|
|
int width) {
|
|
__m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00);
|
|
__m512i zmm_80 = _mm512_set1_epi8((char)0x80);
|
|
__m512i zmm_round = _mm512_set1_epi16((short)0x807f);
|
|
|
|
while (width > 0) {
|
|
__m512i zmm_a = _mm512_loadu_si512((const __m512i*)alpha);
|
|
__m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src0);
|
|
__m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)src1);
|
|
alpha += 64;
|
|
src0 += 64;
|
|
src1 += 64;
|
|
|
|
__m512i zmm_a_lo = _mm512_unpacklo_epi8(zmm_a, zmm_a);
|
|
__m512i zmm_a_hi = _mm512_unpackhi_epi8(zmm_a, zmm_a);
|
|
zmm_a_lo = _mm512_xor_si512(zmm_a_lo, zmm_ff00);
|
|
zmm_a_hi = _mm512_xor_si512(zmm_a_hi, zmm_ff00);
|
|
|
|
__m512i zmm_s_lo = _mm512_unpacklo_epi8(zmm_s0, zmm_s1);
|
|
__m512i zmm_s_hi = _mm512_unpackhi_epi8(zmm_s0, zmm_s1);
|
|
zmm_s_lo = _mm512_sub_epi8(zmm_s_lo, zmm_80);
|
|
zmm_s_hi = _mm512_sub_epi8(zmm_s_hi, zmm_80);
|
|
|
|
__m512i zmm_p_lo = _mm512_maddubs_epi16(zmm_a_lo, zmm_s_lo);
|
|
__m512i zmm_p_hi = _mm512_maddubs_epi16(zmm_a_hi, zmm_s_hi);
|
|
|
|
zmm_p_lo = _mm512_add_epi16(zmm_p_lo, zmm_round);
|
|
zmm_p_hi = _mm512_add_epi16(zmm_p_hi, zmm_round);
|
|
|
|
zmm_p_lo = _mm512_srli_epi16(zmm_p_lo, 8);
|
|
zmm_p_hi = _mm512_srli_epi16(zmm_p_hi, 8);
|
|
|
|
__m512i zmm_d = _mm512_packus_epi16(zmm_p_lo, zmm_p_hi);
|
|
_mm512_storeu_si512((__m512i*)dst, zmm_d);
|
|
dst += 64;
|
|
width -= 64;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_BLENDPLANEROW_AVX512BW
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) ||
|
|
// defined(__i386__) || defined(_M_X64) || defined(_M_X86)) &&
|
|
// ((defined(_MSC_VER) && !defined(__clang__)) ||
|
|
// defined(LIBYUV_ENABLE_ROWWIN))
|