mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-07-30 16:26:19 +08:00
I420ToRAW and I420ToRGB24 1 pass AVX2 and AVX512VBMI
- Implemented `I422ToRGB24Row_AVX2` and `I422ToRGB24Row_AVX512VBMI` in: - `row_gcc.cc`: Inline assembly for GCC/Clang. - `row_win.cc`: C++ intrinsics for MSVC (also verified with Clang). Reduced width alignment requirement: changed from 32-pixel to 16-pixel alignment in `convert_argb.cc` and `row_any.cc`. This allows the AVX2 path to be used for more common video resolutions. ``` I420ToRAW vs Rust on Icelake Xeon Size I420ToRAW yuv420_to_rgb iterations ------- --------- --------------- ---------- 640x480 57 us 77.377 µs/iter 3000 1280x720 170 us 221.671 µs/iter 1000 1920x1080 396 us 494.324 µs/iter 500 3840x2160 2040 us 2357 µs/iter 200 ``` Bug: 42280902 Change-Id: I07c0505c95410ea16a6218c858844791a11ef073
This commit is contained in:
Frank Barchard
parent
45258a40a0
commit
609be0a5f9
@ -53,12 +53,13 @@ and compiler compatibility.
|
||||
2. **Feature Macros**: Use the `HAS_` macros in `include/libyuv/row.h` to
|
||||
enable or disable specific AVX512 versions.
|
||||
|
||||
## Changelist (CL) & Commit Guidelines
|
||||
## Changelist (CL) Format Guidelines
|
||||
|
||||
When generating descriptions, follow the Chromium/Google standard format. Wrap
|
||||
When adding new code, remove trailing spaces.
|
||||
When generating descriptions, follow the Chromium standard format. Wrap
|
||||
commit message text at 72 characters
|
||||
|
||||
### Format Example:
|
||||
### Changelist (CL) Description Example
|
||||
|
||||
\[libyuv] Optimized ARGBToRGB24 for AVX2
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Name: libyuv
|
||||
URL: https://chromium.googlesource.com/libyuv/libyuv/
|
||||
Version: 1949
|
||||
Version: 1950
|
||||
Revision: DEPS
|
||||
License: BSD-3-Clause
|
||||
License File: LICENSE
|
||||
|
||||
@ -349,6 +349,7 @@ extern "C" {
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
#define HAS_RAWTOARGBROW_AVX2
|
||||
#define HAS_I422TORGB24ROW_AVX2
|
||||
#define HAS_RGB24TOARGBROW_AVX2
|
||||
#define HAS_RGB565TOARGBROW_AVX2
|
||||
#define HAS_ARGB1555TOARGBROW_AVX2
|
||||
@ -358,6 +359,7 @@ extern "C" {
|
||||
#define HAS_RAWTOARGBROW_AVX512BW
|
||||
#define HAS_RGB24TOARGBROW_AVX512BW
|
||||
#define HAS_ARGBSHUFFLEROW_AVX512BW
|
||||
#define HAS_I422TORGB24ROW_AVX512VBMI
|
||||
#endif
|
||||
#define HAS_ARGBTOYROW_AVX2
|
||||
#define HAS_ARGBTOYMATRIXROW_AVX2
|
||||
@ -399,7 +401,6 @@ extern "C" {
|
||||
#define HAS_ARGBTOYROW_AVX512BW
|
||||
#define HAS_ARGBTOYMATRIXROW_AVX512BW
|
||||
#define HAS_I422TORGB24ROW_AVX512VBMI
|
||||
#define HAS_I422TORGB24ROW_AVX512BW
|
||||
#define HAS_ARGBTOUVJ444ROW_AVX512BW
|
||||
#define HAS_ARGBTOUVROW_AVX512BW
|
||||
#define HAS_ARGBTOUVJROW_AVX512BW
|
||||
@ -5155,12 +5156,6 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width);
|
||||
void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width);
|
||||
void I422ToARGBRow_Any_AVX2(const uint8_t* y_buf,
|
||||
const uint8_t* u_buf,
|
||||
const uint8_t* v_buf,
|
||||
|
||||
@ -11,6 +11,6 @@
|
||||
#ifndef INCLUDE_LIBYUV_VERSION_H_
|
||||
#define INCLUDE_LIBYUV_VERSION_H_
|
||||
|
||||
#define LIBYUV_VERSION 1949
|
||||
#define LIBYUV_VERSION 1950
|
||||
|
||||
#endif // INCLUDE_LIBYUV_VERSION_H_
|
||||
|
||||
@ -5551,19 +5551,11 @@ int I420ToRGB24Matrix(const uint8_t* src_y,
|
||||
#if defined(HAS_I422TORGB24ROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI;
|
||||
@ -5772,19 +5764,11 @@ int I422ToRGB24Matrix(const uint8_t* src_y,
|
||||
#if defined(HAS_I422TORGB24ROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI;
|
||||
|
||||
@ -385,14 +385,11 @@ ANY31C(I444ToARGBRow_Any_SSSE3, I444ToARGBRow_SSSE3, 0, 0, 4, 7)
|
||||
ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 15)
|
||||
#endif
|
||||
#ifdef HAS_I422TORGB24ROW_AVX2
|
||||
ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 31)
|
||||
ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15)
|
||||
#endif
|
||||
#ifdef HAS_I422TORGB24ROW_AVX512VBMI
|
||||
ANY31C(I422ToRGB24Row_Any_AVX512VBMI, I422ToRGB24Row_AVX512VBMI, 1, 0, 3, 31)
|
||||
#endif
|
||||
#ifdef HAS_I422TORGB24ROW_AVX512BW
|
||||
ANY31C(I422ToRGB24Row_Any_AVX512BW, I422ToRGB24Row_AVX512BW, 1, 0, 3, 31)
|
||||
#endif
|
||||
#ifdef HAS_I422TOARGBROW_AVX2
|
||||
ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15)
|
||||
#endif
|
||||
|
||||
@ -97,7 +97,9 @@ static __inline uint32_t Clamp10(int32_t val) {
|
||||
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
|
||||
defined(_M_IX86) || defined(__arm__) || defined(_M_ARM) || \
|
||||
(defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
#define WRITEWORD(p, v) *(uint32_t*)(p) = v
|
||||
static inline void WRITEWORD(uint8_t* p, uint32_t v) {
|
||||
memcpy(p, &v, 4);
|
||||
}
|
||||
#else
|
||||
static inline void WRITEWORD(uint8_t* p, uint32_t v) {
|
||||
p[0] = (uint8_t)(v & 255);
|
||||
@ -4276,71 +4278,7 @@ void I422ToARGB4444Row_AVX2(const uint8_t* src_y,
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_I422TOARGBROW_AVX2) && defined(HAS_ARGBTORGB24ROW_AVX2)
|
||||
void I422ToRGB24Row_AVX2(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
// Row buffer for intermediate ARGB pixels.
|
||||
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
|
||||
while (width > 0) {
|
||||
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
|
||||
I422ToARGBRow_AVX2(src_y, src_u, src_v, row, yuvconstants, twidth);
|
||||
ARGBToRGB24Row_AVX2(row, dst_rgb24, twidth);
|
||||
src_y += twidth;
|
||||
src_u += twidth / 2;
|
||||
src_v += twidth / 2;
|
||||
dst_rgb24 += twidth * 3;
|
||||
width -= twidth;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTORGB24ROW_AVX512VBMI)
|
||||
void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
// Row buffer for intermediate ARGB pixels.
|
||||
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
|
||||
while (width > 0) {
|
||||
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
|
||||
I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
|
||||
ARGBToRGB24Row_AVX512VBMI(row, dst_rgb24, twidth);
|
||||
src_y += twidth;
|
||||
src_u += twidth / 2;
|
||||
src_v += twidth / 2;
|
||||
dst_rgb24 += twidth * 3;
|
||||
width -= twidth;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTORGB24ROW_AVX2)
|
||||
void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
// Row buffer for intermediate ARGB pixels.
|
||||
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
|
||||
while (width > 0) {
|
||||
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
|
||||
I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
|
||||
ARGBToRGB24Row_AVX2(row, dst_rgb24, twidth);
|
||||
src_y += twidth;
|
||||
src_u += twidth / 2;
|
||||
src_v += twidth / 2;
|
||||
dst_rgb24 += twidth * 3;
|
||||
width -= twidth;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_I444TOARGBROW_AVX2) && defined(HAS_ARGBTORGB24ROW_AVX2)
|
||||
void I444ToRGB24Row_AVX2(const uint8_t* src_y,
|
||||
|
||||
@ -84,16 +84,17 @@ static const uvec8 kShuffleMaskRAWToRGB24_2 = {
|
||||
128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
|
||||
|
||||
// Shuffle table for converting ARGB to RGB24.
|
||||
static const uvec8 kShuffleMaskARGBToRGB24 = {
|
||||
0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u};
|
||||
// and for I422ToRGB24. First 8 + next 4
|
||||
static const uvec8 kShuffleMaskARGBToRGB24[2] = {
|
||||
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
|
||||
128u},
|
||||
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u,
|
||||
14u}};
|
||||
|
||||
// Shuffle table for converting ARGB to RAW.
|
||||
static const uvec8 kShuffleMaskARGBToRAW = {
|
||||
2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, 128u};
|
||||
|
||||
// Shuffle table for converting ARGBToRGB24 for I422ToRGB24. First 8 + next 4
|
||||
static const uvec8 kShuffleMaskARGBToRGB24_0 = {
|
||||
0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u};
|
||||
|
||||
// YUY2 shuf 16 Y to 32 Y.
|
||||
static const vec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6,
|
||||
@ -119,7 +120,7 @@ static const lvec8 kShuffleNV21 = {
|
||||
#endif // HAS_RGB24TOARGBROW_SSSE3
|
||||
|
||||
#if defined(HAS_J400TOARGBROW_AVX2) || defined(HAS_J400TOARGBROW_AVX512BW)
|
||||
alignas(64) static const uint8_t kShuffleMaskJ400ToARGB[64] = {
|
||||
static const uint8_t kShuffleMaskJ400ToARGB[64] = {
|
||||
0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u,
|
||||
3u, 128u, 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u,
|
||||
7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u,
|
||||
@ -132,8 +133,8 @@ void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
|
||||
asm volatile(
|
||||
"vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n"
|
||||
"vpslld $0x18,%%ymm7,%%ymm7 \n"
|
||||
"vmovdqa (%3),%%ymm5 \n"
|
||||
"vmovdqa 0x20(%3),%%ymm6 \n"
|
||||
"vmovdqu (%3),%%ymm5 \n"
|
||||
"vmovdqu 0x20(%3),%%ymm6 \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
@ -164,12 +165,12 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y,
|
||||
asm volatile(
|
||||
"vpternlogd $0xff,%%zmm7,%%zmm7,%%zmm7 \n" // 0xffffffff
|
||||
"vpslld $0x18,%%zmm7,%%zmm7 \n" // 0xff000000
|
||||
"vmovdqa64 %3,%%zmm5 \n"
|
||||
"vmovdqu64 %3,%%zmm5 \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vbroadcasti32x4 (%0),%%zmm0 \n"
|
||||
"vbroadcasti32x4 0x10(%0),%%zmm1 \n"
|
||||
"vbroadcasti32x4 0x10(%0),%%zmm1 \n"
|
||||
"vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
|
||||
"vpshufb %%zmm5,%%zmm1,%%zmm1 \n"
|
||||
"vpord %%zmm7,%%zmm0,%%zmm0 \n"
|
||||
@ -664,10 +665,10 @@ void ARGBToRGB24Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
|
||||
"lea 0x30(%1),%1 \n"
|
||||
"sub $0x10,%2 \n"
|
||||
"jg 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
: "m"(kShuffleMaskARGBToRGB24) // %3
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
: "m"(kShuffleMaskARGBToRGB24[0]) // %3
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
|
||||
"xmm6");
|
||||
}
|
||||
@ -749,11 +750,11 @@ void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
||||
"sub $0x20,%2 \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
: "m"(kShuffleMaskARGBToRGB24), // %3
|
||||
"m"(kPermdRGB24_AVX) // %4
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
: "m"(kShuffleMaskARGBToRGB24[0]), // %3
|
||||
"m"(kPermdRGB24_AVX) // %4
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||
"xmm7");
|
||||
}
|
||||
@ -1720,10 +1721,10 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
|
||||
int width,
|
||||
const struct ArgbConstants* c) {
|
||||
asm volatile(
|
||||
"vbroadcasti64x4 0x20(%4),%%zmm3 \n" // kRGBToU
|
||||
"vbroadcasti64x4 0x40(%4),%%zmm4 \n" // kRGBToV
|
||||
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // -1
|
||||
"vpsllw $15,%%zmm16,%%zmm5 \n" // 0x8000
|
||||
"vbroadcasti64x4 0x20(%4),%%zmm3 \n" // kRGBToU
|
||||
"vbroadcasti64x4 0x40(%4),%%zmm4 \n" // kRGBToV
|
||||
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // -1
|
||||
"vpsllw $15,%%zmm16,%%zmm5 \n" // 0x8000
|
||||
"vmovups %5,%%zmm7 \n"
|
||||
"sub %1,%2 \n"
|
||||
|
||||
@ -2180,12 +2181,12 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
|
||||
int width,
|
||||
const struct ArgbConstants* c) {
|
||||
asm volatile(
|
||||
"vbroadcasti64x4 0x20(%5),%%zmm4 \n" // RGBToU
|
||||
"vbroadcasti64x4 0x40(%5),%%zmm5 \n" // RGBToV
|
||||
"vbroadcasti64x4 0x20(%5),%%zmm4 \n" // RGBToU
|
||||
"vbroadcasti64x4 0x40(%5),%%zmm5 \n" // RGBToV
|
||||
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
|
||||
"vpabsb %%zmm16,%%zmm6 \n" // 0x0101
|
||||
"vpsllw $15,%%zmm16,%%zmm17 \n" // 0x8000
|
||||
"vbroadcasti64x4 %6,%%zmm7 \n" // kShuffleAARRGGBB
|
||||
"vpabsb %%zmm16,%%zmm6 \n" // 0x0101
|
||||
"vpsllw $15,%%zmm16,%%zmm17 \n" // 0x8000
|
||||
"vbroadcasti64x4 %6,%%zmm7 \n" // kShuffleAARRGGBB
|
||||
"vmovups %7,%%zmm18 \n" // kPermdARGBToY_AVX512BW
|
||||
"vmovups %8,%%zmm19 \n" // kPermdARGBToUV_AVX512BW
|
||||
"sub %1,%2 \n"
|
||||
@ -2697,9 +2698,13 @@ void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
// Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is
|
||||
// accessed via offset in assembly.
|
||||
const uvec8* dummy = &kShuffleMaskARGBToRGB24[1];
|
||||
(void)dummy;
|
||||
asm volatile (
|
||||
YUVTORGB_SETUP(yuvconstants)
|
||||
"movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm5 \n"
|
||||
"movdqa 16+%[kShuffleMaskARGBToRGB24],%%xmm5 \n"
|
||||
"movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n"
|
||||
"sub %[u_buf],%[v_buf] \n"
|
||||
|
||||
@ -2720,8 +2725,7 @@ void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf,
|
||||
[width]"+rm"(width) // %[width]
|
||||
#endif
|
||||
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
|
||||
[kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24_0),
|
||||
[kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24)
|
||||
[kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24[0])
|
||||
: "memory", "cc", YUVTORGB_REGS
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
|
||||
);
|
||||
@ -2733,9 +2737,13 @@ void OMITFP I444ToRGB24Row_SSSE3(const uint8_t* y_buf,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
// Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is
|
||||
// accessed via offset in assembly.
|
||||
const uvec8* dummy = &kShuffleMaskARGBToRGB24[1];
|
||||
(void)dummy;
|
||||
asm volatile (
|
||||
YUVTORGB_SETUP(yuvconstants)
|
||||
"movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm5 \n"
|
||||
"movdqa 16+%[kShuffleMaskARGBToRGB24],%%xmm5 \n"
|
||||
"movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n"
|
||||
"sub %[u_buf],%[v_buf] \n"
|
||||
|
||||
@ -2756,8 +2764,7 @@ void OMITFP I444ToRGB24Row_SSSE3(const uint8_t* y_buf,
|
||||
[width]"+rm"(width) // %[width]
|
||||
#endif
|
||||
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
|
||||
[kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24_0),
|
||||
[kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24)
|
||||
[kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24[0])
|
||||
: "memory", "cc", YUVTORGB_REGS
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
|
||||
);
|
||||
@ -3775,6 +3782,62 @@ void OMITFP I422ToARGBRow_AVX2(const uint8_t* y_buf,
|
||||
}
|
||||
#endif // HAS_I422TOARGBROW_AVX2
|
||||
|
||||
#if defined(HAS_I422TORGB24ROW_AVX2)
|
||||
// 16 pixels
|
||||
// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 RGB24 (48 bytes).
|
||||
void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf,
|
||||
const uint8_t* u_buf,
|
||||
const uint8_t* v_buf,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
// Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is
|
||||
// accessed via offset in assembly.
|
||||
const uvec8* dummy = &kShuffleMaskARGBToRGB24[1];
|
||||
(void)dummy;
|
||||
asm volatile (
|
||||
YUVTORGB_SETUP_AVX2(yuvconstants)
|
||||
"vbroadcasti128 16+%[kShuffleMaskARGBToRGB24],%%ymm5 \n"
|
||||
"vbroadcasti128 %[kShuffleMaskARGBToRGB24],%%ymm6 \n"
|
||||
"sub %[u_buf],%[v_buf] \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
READYUV422_AVX2
|
||||
YUVTORGB_AVX2(yuvconstants)
|
||||
"vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n"
|
||||
"vmovdqa %%ymm0,%%ymm1 \n"
|
||||
"vpunpcklwd %%ymm2,%%ymm0,%%ymm0 \n"
|
||||
"vpunpckhwd %%ymm2,%%ymm1,%%ymm1 \n"
|
||||
"vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
|
||||
"vpshufb %%ymm6,%%ymm1,%%ymm1 \n"
|
||||
"vpalignr $0xc,%%ymm0,%%ymm1,%%ymm1 \n"
|
||||
"vextracti128 $1,%%ymm0,%%xmm2 \n"
|
||||
"vextracti128 $1,%%ymm1,%%xmm3 \n"
|
||||
"vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vpalignr $8,%%xmm1,%%xmm2,%%xmm2 \n"
|
||||
"vmovdqu %%xmm0,(%[dst_rgb24]) \n"
|
||||
"vmovdqu %%xmm2,0x10(%[dst_rgb24]) \n"
|
||||
"vmovdqu %%xmm3,0x20(%[dst_rgb24]) \n"
|
||||
"lea 0x30(%[dst_rgb24]),%[dst_rgb24] \n"
|
||||
"sub $0x10,%[width] \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
: [y_buf]"+r"(y_buf), // %[y_buf]
|
||||
[u_buf]"+r"(u_buf), // %[u_buf]
|
||||
[v_buf]"+r"(v_buf), // %[v_buf]
|
||||
[dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
|
||||
[width]"+rm"(width) // %[width]
|
||||
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
|
||||
[kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24[0])
|
||||
: "memory", "cc", YUVTORGB_REGS_AVX2
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
|
||||
);
|
||||
}
|
||||
#endif // HAS_I422TORGB24ROW_AVX2
|
||||
|
||||
|
||||
#if defined(HAS_I422TOARGBROW_AVX512BW)
|
||||
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
|
||||
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
|
||||
@ -3817,6 +3880,82 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
|
||||
);
|
||||
}
|
||||
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
|
||||
static const uint8_t kMaskBG[64] = {
|
||||
0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, 0x45,
|
||||
0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, 0x13, 0x53,
|
||||
0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, 0x60, 0x21, 0x61,
|
||||
0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, 0x26, 0x66, 0x27, 0x67,
|
||||
0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, 0x73, 0x34, 0x74, 0x35, 0x75,
|
||||
0x36, 0x76, 0x37, 0x77};
|
||||
static const uint8_t kMaskDST0[64] = {
|
||||
0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, 0x43,
|
||||
0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, 0x0f, 0x47,
|
||||
0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, 0x16, 0x17, 0x53,
|
||||
0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, 0x56, 0x1e, 0x1f, 0x57,
|
||||
0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, 0x25, 0x62, 0x26, 0x27, 0x63,
|
||||
0x28, 0x29, 0x64, 0x2a};
|
||||
static const uint8_t kMaskDST1[64] = {
|
||||
0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, 0x32,
|
||||
0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, 0x74, 0x3a,
|
||||
0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00};
|
||||
|
||||
|
||||
// 32 pixels
|
||||
// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96 bytes).
|
||||
void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
|
||||
const uint8_t* u_buf,
|
||||
const uint8_t* v_buf,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
asm volatile (
|
||||
YUVTORGB_SETUP_AVX512BW(yuvconstants)
|
||||
"vmovdqu32 %[kMaskBG],%%zmm20 \n"
|
||||
"vmovdqu32 %[kMaskDST0],%%zmm21 \n"
|
||||
"vmovdqu32 %[kMaskDST1],%%zmm22 \n"
|
||||
"sub %[u_buf],%[v_buf] \n"
|
||||
"vpcmpeqb %%xmm5,%%xmm5,%%xmm5 \n"
|
||||
"vpbroadcastq %%xmm5,%%zmm5 \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
READYUV422_AVX512BW
|
||||
YUVTORGB_AVX512BW(yuvconstants)
|
||||
"vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" // zmm0 = BG
|
||||
"vmovdqa64 %%zmm0,%%zmm3 \n" // zmm3 = BG copy
|
||||
"vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" // zmm3 = dst0
|
||||
"vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" // zmm0 = dst1
|
||||
"vmovdqu8 %%zmm3,(%[dst_rgb24]) \n"
|
||||
"vmovdqu8 %%ymm0,0x40(%[dst_rgb24]) \n"
|
||||
"lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n"
|
||||
"sub $0x20,%[width] \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
: [y_buf]"+r"(y_buf), // %[y_buf]
|
||||
[u_buf]"+r"(u_buf), // %[u_buf]
|
||||
[v_buf]"+r"(v_buf), // %[v_buf]
|
||||
[dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
|
||||
[width]"+rm"(width) // %[width]
|
||||
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
|
||||
[quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
|
||||
[dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm]
|
||||
[unperm]"r"(kUnpermuteAVX512), // %[unperm]
|
||||
[kMaskBG]"m"(kMaskBG), // %[kMaskBG]
|
||||
[kMaskDST0]"m"(kMaskDST0), // %[kMaskDST0]
|
||||
[kMaskDST1]"m"(kMaskDST1) // %[kMaskDST1]
|
||||
: "memory", "cc", YUVTORGB_REGS_AVX512BW
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
|
||||
"xmm20", "xmm21", "xmm22",
|
||||
"xmm20", "xmm21", "xmm22"
|
||||
);
|
||||
}
|
||||
#endif // HAS_I422TORGB24ROW_AVX512VBMI
|
||||
|
||||
#endif // HAS_I422TOARGBROW_AVX512BW
|
||||
|
||||
#if defined(HAS_I422TOAR30ROW_AVX2)
|
||||
@ -4630,7 +4769,7 @@ void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
|
||||
"+r"(dst), // %1
|
||||
"+r"(temp_width) // %2
|
||||
: "m"(kShuffleMirror) // %3
|
||||
: "memory", "cc", "zmm0", "zmm5");
|
||||
: "memory", "cc", "xmm0", "xmm5");
|
||||
}
|
||||
#endif // HAS_MIRRORROW_AVX512BW
|
||||
|
||||
@ -4696,7 +4835,7 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src,
|
||||
"+r"(temp_width) // %3
|
||||
: "m"(kShuffleMirrorSplitUV), // %4
|
||||
"m"(kMirrorSplitUVPermute) // %5
|
||||
: "memory", "cc", "zmm0", "zmm1", "zmm2", "zmm3");
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
|
||||
}
|
||||
#endif // HAS_MIRRORSPLITUVROW_AVX512BW
|
||||
|
||||
@ -5001,7 +5140,7 @@ void SplitUVRow_AVX512BW(const uint8_t* src_uv,
|
||||
"+r"(dst_v), // %2
|
||||
"+r"(width) // %3
|
||||
: "m"(kSplitUVPermute) // %4
|
||||
: "memory", "cc", "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5");
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||
}
|
||||
#endif // HAS_SPLITUVROW_AVX512BW
|
||||
|
||||
|
||||
@ -108,9 +108,12 @@ extern "C" {
|
||||
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
|
||||
#define LIBYUV_TARGET_AVX512BW \
|
||||
__attribute__((target("avx512bw,avx512vl,avx512f")))
|
||||
#define LIBYUV_TARGET_AVX512VBMI \
|
||||
__attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f")))
|
||||
#else
|
||||
#define LIBYUV_TARGET_AVX2
|
||||
#define LIBYUV_TARGET_AVX512BW
|
||||
#define LIBYUV_TARGET_AVX512VBMI
|
||||
#endif
|
||||
|
||||
// Convert 32 ARGB pixels (128 bytes) to 32 UV444 values.
|
||||
@ -695,10 +698,10 @@ void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
|
||||
#endif
|
||||
|
||||
#ifdef HAS_J400TOARGBROW_AVX2
|
||||
alignas(32) static const uint8_t kShuffleMaskJ400ToARGB_0[32] = {
|
||||
static const uint8_t kShuffleMaskJ400ToARGB_0[32] = {
|
||||
0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u, 3u, 128u,
|
||||
4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u, 7u, 7u, 7u, 128u};
|
||||
alignas(32) static const uint8_t kShuffleMaskJ400ToARGB_1[32] = {
|
||||
static const uint8_t kShuffleMaskJ400ToARGB_1[32] = {
|
||||
8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u, 10u,
|
||||
128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, 13u, 13u,
|
||||
13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u};
|
||||
@ -706,9 +709,9 @@ alignas(32) static const uint8_t kShuffleMaskJ400ToARGB_1[32] = {
|
||||
LIBYUV_TARGET_AVX2
|
||||
void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
|
||||
__m256i ymm_mask0 =
|
||||
_mm256_load_si256((const __m256i*)kShuffleMaskJ400ToARGB_0);
|
||||
_mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_0);
|
||||
__m256i ymm_mask1 =
|
||||
_mm256_load_si256((const __m256i*)kShuffleMaskJ400ToARGB_1);
|
||||
_mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_1);
|
||||
__m256i ymm_alpha = _mm256_set1_epi32((int)0xff000000u);
|
||||
|
||||
while (width > 0) {
|
||||
@ -732,7 +735,7 @@ void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
|
||||
#endif // HAS_J400TOARGBROW_AVX2
|
||||
|
||||
#ifdef HAS_RGB24TOARGBROW_AVX2
|
||||
alignas(16) static const uint8_t kShuffleMaskRGB24ToARGB[2][16] = {
|
||||
static const uint8_t kShuffleMaskRGB24ToARGB[2][16] = {
|
||||
{0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u},
|
||||
{4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u,
|
||||
128u}};
|
||||
@ -879,9 +882,9 @@ void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
|
||||
int width) {
|
||||
__m256i ymm_alpha = _mm256_set1_epi32(0xff000000);
|
||||
__m256i ymm_shuf = _mm256_broadcastsi128_si256(
|
||||
_mm_load_si128((const __m128i*)kShuffleMaskRGB24ToARGB[0]));
|
||||
_mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[0]));
|
||||
__m256i ymm_shuf2 = _mm256_broadcastsi128_si256(
|
||||
_mm_load_si128((const __m128i*)kShuffleMaskRGB24ToARGB[1]));
|
||||
_mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[1]));
|
||||
|
||||
while (width > 0) {
|
||||
__m128i xmm0 = _mm_loadu_si128((const __m128i*)src_rgb24);
|
||||
@ -973,6 +976,217 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef HAS_I422TORGB24ROW_AVX2
|
||||
static const uint8_t kShuffleMaskARGBToRGB24[2][16] = {
|
||||
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
|
||||
128u},
|
||||
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u,
|
||||
14u}};
|
||||
|
||||
LIBYUV_TARGET_AVX2
|
||||
void I422ToRGB24Row_AVX2(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
// Constants
|
||||
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
|
||||
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
|
||||
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
|
||||
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
|
||||
__m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
|
||||
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
|
||||
|
||||
__m256i ymm_shuf0 = _mm256_broadcastsi128_si256(
|
||||
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1]));
|
||||
__m256i ymm_shuf1 = _mm256_broadcastsi128_si256(
|
||||
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0]));
|
||||
__m256i ymm_u_zero = _mm256_setzero_si256();
|
||||
|
||||
ptrdiff_t offset = src_v - src_u;
|
||||
|
||||
while (width >= 16) {
|
||||
// READYUV422_AVX2
|
||||
__m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
|
||||
__m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
|
||||
src_u += 8;
|
||||
|
||||
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
|
||||
__m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
|
||||
|
||||
ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
|
||||
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
|
||||
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
|
||||
|
||||
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
|
||||
src_y += 16;
|
||||
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
|
||||
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
|
||||
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
|
||||
|
||||
// YUVTORGB_AVX2
|
||||
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
|
||||
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
|
||||
|
||||
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
|
||||
ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
|
||||
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
|
||||
|
||||
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
|
||||
|
||||
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
|
||||
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
|
||||
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
|
||||
|
||||
ymm0 = _mm256_srai_epi16(ymm0, 6);
|
||||
ymm1 = _mm256_srai_epi16(ymm1, 6);
|
||||
ymm2 = _mm256_srai_epi16(ymm2, 6);
|
||||
|
||||
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
|
||||
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
|
||||
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
|
||||
|
||||
// STORERGB24_AVX2
|
||||
__m256i ymm0_packed = _mm256_unpacklo_epi8(ymm0, ymm1);
|
||||
__m256i ymm2_packed = _mm256_unpacklo_epi8(ymm2, ymm2);
|
||||
__m256i ymm1_packed = ymm0_packed;
|
||||
|
||||
ymm0_packed = _mm256_unpacklo_epi16(ymm0_packed, ymm2_packed);
|
||||
ymm1_packed = _mm256_unpackhi_epi16(ymm1_packed, ymm2_packed);
|
||||
|
||||
ymm0_packed = _mm256_shuffle_epi8(ymm0_packed, ymm_shuf0);
|
||||
ymm1_packed = _mm256_shuffle_epi8(ymm1_packed, ymm_shuf1);
|
||||
|
||||
ymm1_packed = _mm256_alignr_epi8(ymm1_packed, ymm0_packed, 0xc);
|
||||
|
||||
__m128i xmm0_store = _mm256_castsi256_si128(ymm0_packed);
|
||||
__m128i xmm1_store = _mm256_castsi256_si128(ymm1_packed);
|
||||
__m128i xmm2_store = _mm256_extractf128_si256(ymm0_packed, 1);
|
||||
__m128i xmm3_store = _mm256_extractf128_si256(ymm1_packed, 1);
|
||||
|
||||
_mm_storel_epi64((__m128i*)dst_rgb24, xmm0_store);
|
||||
_mm_storeu_si128((__m128i*)(dst_rgb24 + 8), xmm1_store);
|
||||
_mm_storel_epi64((__m128i*)(dst_rgb24 + 24), xmm2_store);
|
||||
_mm_storeu_si128((__m128i*)(dst_rgb24 + 32), xmm3_store);
|
||||
|
||||
dst_rgb24 += 48;
|
||||
width -= 16;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAS_I422TORGB24ROW_AVX512VBMI
|
||||
LIBYUV_TARGET_AVX512VBMI
|
||||
void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
// Masks
|
||||
static const uint8_t kMaskBG[64] = {
|
||||
0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, 0x45,
|
||||
0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, 0x13, 0x53,
|
||||
0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, 0x60, 0x21, 0x61,
|
||||
0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, 0x26, 0x66, 0x27, 0x67,
|
||||
0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, 0x73, 0x34, 0x74, 0x35, 0x75,
|
||||
0x36, 0x76, 0x37, 0x77};
|
||||
static const uint8_t kMaskDST0[64] = {
|
||||
0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, 0x43,
|
||||
0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, 0x0f, 0x47,
|
||||
0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, 0x16, 0x17, 0x53,
|
||||
0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, 0x56, 0x1e, 0x1f, 0x57,
|
||||
0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, 0x25, 0x62, 0x26, 0x27, 0x63,
|
||||
0x28, 0x29, 0x64, 0x2a};
|
||||
static const uint8_t kMaskDST1[64] = {
|
||||
0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, 0x32,
|
||||
0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, 0x74, 0x3a,
|
||||
0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00};
|
||||
|
||||
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
|
||||
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
|
||||
|
||||
// Constants
|
||||
__m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
||||
__m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
||||
__m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
||||
__m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
||||
__m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
||||
__m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
||||
|
||||
__m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG);
|
||||
__m512i zmm_mask_DST0 = _mm512_loadu_si512((const __m512i*)kMaskDST0);
|
||||
__m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1);
|
||||
__m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
|
||||
__m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
|
||||
|
||||
ptrdiff_t offset = src_v - src_u;
|
||||
|
||||
while (width >= 32) {
|
||||
// READYUV422_AVX512BW
|
||||
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
|
||||
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
|
||||
src_u += 16;
|
||||
|
||||
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
|
||||
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
|
||||
|
||||
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
|
||||
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
|
||||
|
||||
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
|
||||
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
|
||||
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
|
||||
|
||||
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
|
||||
src_y += 32;
|
||||
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
|
||||
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
|
||||
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
|
||||
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
||||
|
||||
// YUVTORGB_AVX512BW
|
||||
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
||||
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
|
||||
|
||||
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
|
||||
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
|
||||
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
|
||||
|
||||
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
|
||||
|
||||
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
|
||||
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
|
||||
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
|
||||
|
||||
zmm0 = _mm512_srai_epi16(zmm0, 6);
|
||||
zmm1 = _mm512_srai_epi16(zmm1, 6);
|
||||
zmm2 = _mm512_srai_epi16(zmm2, 6);
|
||||
|
||||
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
||||
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
||||
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
||||
|
||||
// STORERGB24_AVX512VBMI
|
||||
__m512i zmm_BG = _mm512_permi2var_epi8(zmm0, zmm_mask_BG, zmm1);
|
||||
__m512i zmm_dst0 = _mm512_permi2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
|
||||
__m512i zmm_dst1 = _mm512_permi2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
|
||||
|
||||
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
|
||||
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1));
|
||||
|
||||
dst_rgb24 += 96;
|
||||
width -= 32;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user