coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-07-30 16:26:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

I420ToRAW and I420ToRGB24 1 pass AVX2 and AVX512VBMI

Browse Source 
- Implemented `I422ToRGB24Row_AVX2` and `I422ToRGB24Row_AVX512VBMI` in:
 - `row_gcc.cc`: Inline assembly for GCC/Clang.
 - `row_win.cc`: C++ intrinsics for MSVC (also verified with Clang).

Reduced width alignment requirement: changed from 32-pixel to 16-pixel
alignment in `convert_argb.cc` and `row_any.cc`. This allows the AVX2
path to be used for more common video resolutions.

```
I420ToRAW vs Rust on Icelake Xeon
Size        I420ToRAW     yuv420_to_rgb  iterations
-------     ---------   ---------------  ----------
640x480         57 us    77.377 µs/iter     3000
1280x720       170 us   221.671 µs/iter     1000
1920x1080      396 us   494.324 µs/iter      500
3840x2160     2040 us      2357 µs/iter      200
```

Bug: 42280902
Change-Id: I07c0505c95410ea16a6218c858844791a11ef073
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/7908323
Reviewed-by: Wan-Teh Chang <wtc@google.com>
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard 2026-07-20 16:30:40 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent 45258a40a0
commit 86a4d94d7a
9 changed files with 411 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
GEMINI.md
View File

@ -53,12 +53,13 @@ and compiler compatibility.
2. **Feature Macros**: Use the `HAS_` macros in `include/libyuv/row.h` to 2. **Feature Macros**: Use the `HAS_` macros in `include/libyuv/row.h` to
enable or disable specific AVX512 versions. enable or disable specific AVX512 versions.
## Changelist (CL) & Commit Guidelines ## Changelist (CL) Format Guidelines
When generating descriptions, follow the Chromium/Google standard format. Wrap When adding new code, remove trailing spaces.
When generating descriptions, follow the Chromium standard format. Wrap
commit message text at 72 characters commit message text at 72 characters
### Format Example: ### Changelist (CL) Description Example
\[libyuv] Optimized ARGBToRGB24 for AVX2 \[libyuv] Optimized ARGBToRGB24 for AVX2

2
README.chromium
View File

@ -1,6 +1,6 @@
Name: libyuv Name: libyuv
URL: https://chromium.googlesource.com/libyuv/libyuv/ URL: https://chromium.googlesource.com/libyuv/libyuv/
Version: 1949 Version: 1950
Revision: DEPS Revision: DEPS
License: BSD-3-Clause License: BSD-3-Clause
License File: LICENSE License File: LICENSE

9
include/libyuv/row.h
View File

@ -349,6 +349,7 @@ extern "C" {
((defined(_MSC_VER) && !defined(__clang__)) || \ ((defined(_MSC_VER) && !defined(__clang__)) || \
defined(LIBYUV_ENABLE_ROWWIN)) defined(LIBYUV_ENABLE_ROWWIN))
#define HAS_RAWTOARGBROW_AVX2 #define HAS_RAWTOARGBROW_AVX2
#define HAS_I422TORGB24ROW_AVX2
#define HAS_RGB24TOARGBROW_AVX2 #define HAS_RGB24TOARGBROW_AVX2
#define HAS_RGB565TOARGBROW_AVX2 #define HAS_RGB565TOARGBROW_AVX2
#define HAS_ARGB1555TOARGBROW_AVX2 #define HAS_ARGB1555TOARGBROW_AVX2
@ -358,6 +359,7 @@ extern "C" {
#define HAS_RAWTOARGBROW_AVX512BW #define HAS_RAWTOARGBROW_AVX512BW
#define HAS_RGB24TOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW
#define HAS_ARGBSHUFFLEROW_AVX512BW #define HAS_ARGBSHUFFLEROW_AVX512BW
#define HAS_I422TORGB24ROW_AVX512VBMI
#endif #endif
#define HAS_ARGBTOYROW_AVX2 #define HAS_ARGBTOYROW_AVX2
#define HAS_ARGBTOYMATRIXROW_AVX2 #define HAS_ARGBTOYMATRIXROW_AVX2
@ -399,7 +401,6 @@ extern "C" {
#define HAS_ARGBTOYROW_AVX512BW #define HAS_ARGBTOYROW_AVX512BW
#define HAS_ARGBTOYMATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW
#define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_I422TORGB24ROW_AVX512VBMI
#define HAS_I422TORGB24ROW_AVX512BW
#define HAS_ARGBTOUVJ444ROW_AVX512BW #define HAS_ARGBTOUVJ444ROW_AVX512BW
#define HAS_ARGBTOUVROW_AVX512BW #define HAS_ARGBTOUVROW_AVX512BW
#define HAS_ARGBTOUVJROW_AVX512BW #define HAS_ARGBTOUVJROW_AVX512BW
@ -5155,12 +5156,6 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
uint8_t* dst_rgb24, uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants, const struct YuvConstants* yuvconstants,
int width); int width);
void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
void I422ToARGBRow_Any_AVX2(const uint8_t* y_buf, void I422ToARGBRow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* u_buf, const uint8_t* u_buf,
const uint8_t* v_buf, const uint8_t* v_buf,

2
include/libyuv/version.h
View File

@ -11,6 +11,6 @@
#ifndef INCLUDE_LIBYUV_VERSION_H_ #ifndef INCLUDE_LIBYUV_VERSION_H_
#define INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_
#define LIBYUV_VERSION 1949 #define LIBYUV_VERSION 1950
#endif // INCLUDE_LIBYUV_VERSION_H_ #endif // INCLUDE_LIBYUV_VERSION_H_

20
source/convert_argb.cc
View File

@ -5551,19 +5551,11 @@ int I420ToRGB24Matrix(const uint8_t* src_y,
#if defined(HAS_I422TORGB24ROW_AVX2) #if defined(HAS_I422TORGB24ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) { if (TestCpuFlag(kCpuHasAVX2)) {
I422ToRGB24Row = I422ToRGB24Row_Any_AVX2; I422ToRGB24Row = I422ToRGB24Row_Any_AVX2;
if (IS_ALIGNED(width, 32)) { if (IS_ALIGNED(width, 16)) {
I422ToRGB24Row = I422ToRGB24Row_AVX2; I422ToRGB24Row = I422ToRGB24Row_AVX2;
} }
} }
#endif #endif
#if defined(HAS_I422TORGB24ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
}
}
#endif
#if defined(HAS_I422TORGB24ROW_AVX512VBMI) #if defined(HAS_I422TORGB24ROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI)) { if (TestCpuFlag(kCpuHasAVX512VBMI)) {
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI; I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI;
@ -5772,19 +5764,11 @@ int I422ToRGB24Matrix(const uint8_t* src_y,
#if defined(HAS_I422TORGB24ROW_AVX2) #if defined(HAS_I422TORGB24ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) { if (TestCpuFlag(kCpuHasAVX2)) {
I422ToRGB24Row = I422ToRGB24Row_Any_AVX2; I422ToRGB24Row = I422ToRGB24Row_Any_AVX2;
if (IS_ALIGNED(width, 32)) { if (IS_ALIGNED(width, 16)) {
I422ToRGB24Row = I422ToRGB24Row_AVX2; I422ToRGB24Row = I422ToRGB24Row_AVX2;
} }
} }
#endif #endif
#if defined(HAS_I422TORGB24ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
}
}
#endif
#if defined(HAS_I422TORGB24ROW_AVX512VBMI) #if defined(HAS_I422TORGB24ROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI)) { if (TestCpuFlag(kCpuHasAVX512VBMI)) {
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI; I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI;

5
source/row_any.cc
View File

@ -385,14 +385,11 @@ ANY31C(I444ToARGBRow_Any_SSSE3, I444ToARGBRow_SSSE3, 0, 0, 4, 7)
ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 15) ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 15)
#endif #endif
#ifdef HAS_I422TORGB24ROW_AVX2 #ifdef HAS_I422TORGB24ROW_AVX2
ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 31) ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15)
#endif #endif
#ifdef HAS_I422TORGB24ROW_AVX512VBMI #ifdef HAS_I422TORGB24ROW_AVX512VBMI
ANY31C(I422ToRGB24Row_Any_AVX512VBMI, I422ToRGB24Row_AVX512VBMI, 1, 0, 3, 31) ANY31C(I422ToRGB24Row_Any_AVX512VBMI, I422ToRGB24Row_AVX512VBMI, 1, 0, 3, 31)
#endif #endif
#ifdef HAS_I422TORGB24ROW_AVX512BW
ANY31C(I422ToRGB24Row_Any_AVX512BW, I422ToRGB24Row_AVX512BW, 1, 0, 3, 31)
#endif
#ifdef HAS_I422TOARGBROW_AVX2 #ifdef HAS_I422TOARGBROW_AVX2
ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15) ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15)
#endif #endif

68
source/row_common.cc
View File

@ -97,7 +97,9 @@ static __inline uint32_t Clamp10(int32_t val) {
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
defined(_M_IX86) || defined(__arm__) || defined(_M_ARM) || \ defined(_M_IX86) || defined(__arm__) || defined(_M_ARM) || \
(defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define WRITEWORD(p, v) *(uint32_t*)(p) = v static inline void WRITEWORD(uint8_t* p, uint32_t v) {
memcpy(p, &v, 4);
}
#else #else
static inline void WRITEWORD(uint8_t* p, uint32_t v) { static inline void WRITEWORD(uint8_t* p, uint32_t v) {
p[0] = (uint8_t)(v & 255); p[0] = (uint8_t)(v & 255);
@ -4276,71 +4278,7 @@ void I422ToARGB4444Row_AVX2(const uint8_t* src_y,
} }
#endif #endif
#if defined(HAS_I422TOARGBROW_AVX2) && defined(HAS_ARGBTORGB24ROW_AVX2)
void I422ToRGB24Row_AVX2(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
// Row buffer for intermediate ARGB pixels.
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
I422ToARGBRow_AVX2(src_y, src_u, src_v, row, yuvconstants, twidth);
ARGBToRGB24Row_AVX2(row, dst_rgb24, twidth);
src_y += twidth;
src_u += twidth / 2;
src_v += twidth / 2;
dst_rgb24 += twidth * 3;
width -= twidth;
}
}
#endif
#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTORGB24ROW_AVX512VBMI)
void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
// Row buffer for intermediate ARGB pixels.
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
ARGBToRGB24Row_AVX512VBMI(row, dst_rgb24, twidth);
src_y += twidth;
src_u += twidth / 2;
src_v += twidth / 2;
dst_rgb24 += twidth * 3;
width -= twidth;
}
}
#endif
#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTORGB24ROW_AVX2)
void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
// Row buffer for intermediate ARGB pixels.
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
ARGBToRGB24Row_AVX2(row, dst_rgb24, twidth);
src_y += twidth;
src_u += twidth / 2;
src_v += twidth / 2;
dst_rgb24 += twidth * 3;
width -= twidth;
}
}
#endif
#if defined(HAS_I444TOARGBROW_AVX2) && defined(HAS_ARGBTORGB24ROW_AVX2) #if defined(HAS_I444TOARGBROW_AVX2) && defined(HAS_ARGBTORGB24ROW_AVX2)
void I444ToRGB24Row_AVX2(const uint8_t* src_y, void I444ToRGB24Row_AVX2(const uint8_t* src_y,

179
source/row_gcc.cc
View File

@ -84,16 +84,17 @@ static const uvec8 kShuffleMaskRAWToRGB24_2 = {
128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u}; 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
// Shuffle table for converting ARGB to RGB24. // Shuffle table for converting ARGB to RGB24.
static const uvec8 kShuffleMaskARGBToRGB24 = { // and for I422ToRGB24. First 8 + next 4
0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u}; static const uvec8 kShuffleMaskARGBToRGB24[2] = {
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
128u},
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u,
14u}};
// Shuffle table for converting ARGB to RAW. // Shuffle table for converting ARGB to RAW.
static const uvec8 kShuffleMaskARGBToRAW = { static const uvec8 kShuffleMaskARGBToRAW = {
2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, 128u}; 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, 128u};
// Shuffle table for converting ARGBToRGB24 for I422ToRGB24. First 8 + next 4
static const uvec8 kShuffleMaskARGBToRGB24_0 = {
0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u};
// YUY2 shuf 16 Y to 32 Y. // YUY2 shuf 16 Y to 32 Y.
static const vec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6, static const vec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6,
@ -119,7 +120,7 @@ static const lvec8 kShuffleNV21 = {
#endif // HAS_RGB24TOARGBROW_SSSE3 #endif // HAS_RGB24TOARGBROW_SSSE3
#if defined(HAS_J400TOARGBROW_AVX2) || defined(HAS_J400TOARGBROW_AVX512BW) #if defined(HAS_J400TOARGBROW_AVX2) || defined(HAS_J400TOARGBROW_AVX512BW)
alignas(64) static const uint8_t kShuffleMaskJ400ToARGB[64] = { static const uint8_t kShuffleMaskJ400ToARGB[64] = {
0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u, 0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u,
3u, 128u, 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u, 3u, 128u, 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u,
7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u, 7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u,
@ -132,8 +133,8 @@ void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
asm volatile( asm volatile(
"vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n" "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n"
"vpslld $0x18,%%ymm7,%%ymm7 \n" "vpslld $0x18,%%ymm7,%%ymm7 \n"
"vmovdqa (%3),%%ymm5 \n" "vmovdqu (%3),%%ymm5 \n"
"vmovdqa 0x20(%3),%%ymm6 \n" "vmovdqu 0x20(%3),%%ymm6 \n"
LABELALIGN LABELALIGN
"1: \n" "1: \n"
@ -164,7 +165,7 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y,
asm volatile( asm volatile(
"vpternlogd $0xff,%%zmm7,%%zmm7,%%zmm7 \n" // 0xffffffff "vpternlogd $0xff,%%zmm7,%%zmm7,%%zmm7 \n" // 0xffffffff
"vpslld $0x18,%%zmm7,%%zmm7 \n" // 0xff000000 "vpslld $0x18,%%zmm7,%%zmm7 \n" // 0xff000000
"vmovdqa64 %3,%%zmm5 \n" "vmovdqu64 %3,%%zmm5 \n"
LABELALIGN LABELALIGN
"1: \n" "1: \n"
@ -667,7 +668,7 @@ void ARGBToRGB24Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
: "+r"(src), // %0 : "+r"(src), // %0
"+r"(dst), // %1 "+r"(dst), // %1
"+r"(width) // %2 "+r"(width) // %2
: "m"(kShuffleMaskARGBToRGB24) // %3 : "m"(kShuffleMaskARGBToRGB24[0]) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6"); "xmm6");
} }
@ -752,7 +753,7 @@ void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
: "+r"(src), // %0 : "+r"(src), // %0
"+r"(dst), // %1 "+r"(dst), // %1
"+r"(width) // %2 "+r"(width) // %2
: "m"(kShuffleMaskARGBToRGB24), // %3 : "m"(kShuffleMaskARGBToRGB24[0]), // %3
"m"(kPermdRGB24_AVX) // %4 "m"(kPermdRGB24_AVX) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7"); "xmm7");
@ -2697,9 +2698,13 @@ void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf,
uint8_t* dst_rgb24, uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants, const struct YuvConstants* yuvconstants,
int width) { int width) {
// Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is
// accessed via offset in assembly.
const uvec8* dummy = &kShuffleMaskARGBToRGB24[1];
(void)dummy;
asm volatile ( asm volatile (
YUVTORGB_SETUP(yuvconstants) YUVTORGB_SETUP(yuvconstants)
"movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm5 \n" "movdqa 16+%[kShuffleMaskARGBToRGB24],%%xmm5 \n"
"movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n" "movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n"
"sub %[u_buf],%[v_buf] \n" "sub %[u_buf],%[v_buf] \n"
@ -2720,8 +2725,7 @@ void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf,
[width]"+rm"(width) // %[width] [width]"+rm"(width) // %[width]
#endif #endif
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
[kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24_0), [kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24[0])
[kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24)
: "memory", "cc", YUVTORGB_REGS : "memory", "cc", YUVTORGB_REGS
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
); );
@ -2733,9 +2737,13 @@ void OMITFP I444ToRGB24Row_SSSE3(const uint8_t* y_buf,
uint8_t* dst_rgb24, uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants, const struct YuvConstants* yuvconstants,
int width) { int width) {
// Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is
// accessed via offset in assembly.
const uvec8* dummy = &kShuffleMaskARGBToRGB24[1];
(void)dummy;
asm volatile ( asm volatile (
YUVTORGB_SETUP(yuvconstants) YUVTORGB_SETUP(yuvconstants)
"movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm5 \n" "movdqa 16+%[kShuffleMaskARGBToRGB24],%%xmm5 \n"
"movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n" "movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n"
"sub %[u_buf],%[v_buf] \n" "sub %[u_buf],%[v_buf] \n"
@ -2756,8 +2764,7 @@ void OMITFP I444ToRGB24Row_SSSE3(const uint8_t* y_buf,
[width]"+rm"(width) // %[width] [width]"+rm"(width) // %[width]
#endif #endif
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
[kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24_0), [kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24[0])
[kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24)
: "memory", "cc", YUVTORGB_REGS : "memory", "cc", YUVTORGB_REGS
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
); );
@ -3775,6 +3782,62 @@ void OMITFP I422ToARGBRow_AVX2(const uint8_t* y_buf,
} }
#endif // HAS_I422TOARGBROW_AVX2 #endif // HAS_I422TOARGBROW_AVX2
#if defined(HAS_I422TORGB24ROW_AVX2)
// 16 pixels
// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 RGB24 (48 bytes).
void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
// Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is
// accessed via offset in assembly.
const uvec8* dummy = &kShuffleMaskARGBToRGB24[1];
(void)dummy;
asm volatile (
YUVTORGB_SETUP_AVX2(yuvconstants)
"vbroadcasti128 16+%[kShuffleMaskARGBToRGB24],%%ymm5 \n"
"vbroadcasti128 %[kShuffleMaskARGBToRGB24],%%ymm6 \n"
"sub %[u_buf],%[v_buf] \n"
LABELALIGN
"1: \n"
READYUV422_AVX2
YUVTORGB_AVX2(yuvconstants)
"vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n"
"vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n"
"vmovdqa %%ymm0,%%ymm1 \n"
"vpunpcklwd %%ymm2,%%ymm0,%%ymm0 \n"
"vpunpckhwd %%ymm2,%%ymm1,%%ymm1 \n"
"vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
"vpshufb %%ymm6,%%ymm1,%%ymm1 \n"
"vpalignr $0xc,%%ymm0,%%ymm1,%%ymm1 \n"
"vextracti128 $1,%%ymm0,%%xmm2 \n"
"vextracti128 $1,%%ymm1,%%xmm3 \n"
"vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n"
"vpalignr $8,%%xmm1,%%xmm2,%%xmm2 \n"
"vmovdqu %%xmm0,(%[dst_rgb24]) \n"
"vmovdqu %%xmm2,0x10(%[dst_rgb24]) \n"
"vmovdqu %%xmm3,0x20(%[dst_rgb24]) \n"
"lea 0x30(%[dst_rgb24]),%[dst_rgb24] \n"
"sub $0x10,%[width] \n"
"jg 1b \n"
"vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
[dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
[width]"+rm"(width) // %[width]
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
[kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24[0])
: "memory", "cc", YUVTORGB_REGS_AVX2
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
);
}
#endif // HAS_I422TORGB24ROW_AVX2
#if defined(HAS_I422TOARGBROW_AVX512BW) #if defined(HAS_I422TOARGBROW_AVX512BW)
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2}; static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
@ -3817,6 +3880,82 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
); );
} }
#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
static const uint8_t kMaskBG[64] = {
0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, 0x45,
0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, 0x13, 0x53,
0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, 0x60, 0x21, 0x61,
0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, 0x26, 0x66, 0x27, 0x67,
0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, 0x73, 0x34, 0x74, 0x35, 0x75,
0x36, 0x76, 0x37, 0x77};
static const uint8_t kMaskDST0[64] = {
0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, 0x43,
0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, 0x0f, 0x47,
0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, 0x16, 0x17, 0x53,
0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, 0x56, 0x1e, 0x1f, 0x57,
0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, 0x25, 0x62, 0x26, 0x27, 0x63,
0x28, 0x29, 0x64, 0x2a};
static const uint8_t kMaskDST1[64] = {
0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, 0x32,
0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, 0x74, 0x3a,
0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00};
// 32 pixels
// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96 bytes).
void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
asm volatile (
YUVTORGB_SETUP_AVX512BW(yuvconstants)
"vmovdqu32 %[kMaskBG],%%zmm20 \n"
"vmovdqu32 %[kMaskDST0],%%zmm21 \n"
"vmovdqu32 %[kMaskDST1],%%zmm22 \n"
"sub %[u_buf],%[v_buf] \n"
"vpcmpeqb %%xmm5,%%xmm5,%%xmm5 \n"
"vpbroadcastq %%xmm5,%%zmm5 \n"
LABELALIGN
"1: \n"
READYUV422_AVX512BW
YUVTORGB_AVX512BW(yuvconstants)
"vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" // zmm0 = BG
"vmovdqa64 %%zmm0,%%zmm3 \n" // zmm3 = BG copy
"vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" // zmm3 = dst0
"vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" // zmm0 = dst1
"vmovdqu8 %%zmm3,(%[dst_rgb24]) \n"
"vmovdqu8 %%ymm0,0x40(%[dst_rgb24]) \n"
"lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n"
"sub $0x20,%[width] \n"
"jg 1b \n"
"vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
[dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
[width]"+rm"(width) // %[width]
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
[quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
[dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm]
[unperm]"r"(kUnpermuteAVX512), // %[unperm]
[kMaskBG]"m"(kMaskBG), // %[kMaskBG]
[kMaskDST0]"m"(kMaskDST0), // %[kMaskDST0]
[kMaskDST1]"m"(kMaskDST1) // %[kMaskDST1]
: "memory", "cc", YUVTORGB_REGS_AVX512BW
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm20", "xmm21", "xmm22",
"xmm20", "xmm21", "xmm22"
);
}
#endif // HAS_I422TORGB24ROW_AVX512VBMI
#endif // HAS_I422TOARGBROW_AVX512BW #endif // HAS_I422TOARGBROW_AVX512BW
#if defined(HAS_I422TOAR30ROW_AVX2) #if defined(HAS_I422TOAR30ROW_AVX2)
@ -4630,7 +4769,7 @@ void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
"+r"(dst), // %1 "+r"(dst), // %1
"+r"(temp_width) // %2 "+r"(temp_width) // %2
: "m"(kShuffleMirror) // %3 : "m"(kShuffleMirror) // %3
: "memory", "cc", "zmm0", "zmm5"); : "memory", "cc", "xmm0", "xmm5");
} }
#endif // HAS_MIRRORROW_AVX512BW #endif // HAS_MIRRORROW_AVX512BW
@ -4696,7 +4835,7 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src,
"+r"(temp_width) // %3 "+r"(temp_width) // %3
: "m"(kShuffleMirrorSplitUV), // %4 : "m"(kShuffleMirrorSplitUV), // %4
"m"(kMirrorSplitUVPermute) // %5 "m"(kMirrorSplitUVPermute) // %5
: "memory", "cc", "zmm0", "zmm1", "zmm2", "zmm3"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
} }
#endif // HAS_MIRRORSPLITUVROW_AVX512BW #endif // HAS_MIRRORSPLITUVROW_AVX512BW
@ -5001,7 +5140,7 @@ void SplitUVRow_AVX512BW(const uint8_t* src_uv,
"+r"(dst_v), // %2 "+r"(dst_v), // %2
"+r"(width) // %3 "+r"(width) // %3
: "m"(kSplitUVPermute) // %4 : "m"(kSplitUVPermute) // %4
: "memory", "cc", "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_SPLITUVROW_AVX512BW #endif // HAS_SPLITUVROW_AVX512BW

228
source/row_win.cc
View File

@ -108,9 +108,12 @@ extern "C" {
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
#define LIBYUV_TARGET_AVX512BW \ #define LIBYUV_TARGET_AVX512BW \
__attribute__((target("avx512bw,avx512vl,avx512f"))) __attribute__((target("avx512bw,avx512vl,avx512f")))
#define LIBYUV_TARGET_AVX512VBMI \
__attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f")))
#else #else
#define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX2
#define LIBYUV_TARGET_AVX512BW #define LIBYUV_TARGET_AVX512BW
#define LIBYUV_TARGET_AVX512VBMI
#endif #endif
// Convert 32 ARGB pixels (128 bytes) to 32 UV444 values. // Convert 32 ARGB pixels (128 bytes) to 32 UV444 values.
@ -695,10 +698,10 @@ void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
#endif #endif
#ifdef HAS_J400TOARGBROW_AVX2 #ifdef HAS_J400TOARGBROW_AVX2
alignas(32) static const uint8_t kShuffleMaskJ400ToARGB_0[32] = { static const uint8_t kShuffleMaskJ400ToARGB_0[32] = {
0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u, 3u, 128u, 0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, 3u, 3u, 128u,
4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u, 7u, 7u, 7u, 128u}; 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, 6u, 128u, 7u, 7u, 7u, 128u};
alignas(32) static const uint8_t kShuffleMaskJ400ToARGB_1[32] = { static const uint8_t kShuffleMaskJ400ToARGB_1[32] = {
8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u, 10u, 8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u, 10u,
128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, 13u, 13u, 128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, 13u, 13u,
13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u}; 13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u};
@ -706,9 +709,9 @@ alignas(32) static const uint8_t kShuffleMaskJ400ToARGB_1[32] = {
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) { void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
__m256i ymm_mask0 = __m256i ymm_mask0 =
_mm256_load_si256((const __m256i*)kShuffleMaskJ400ToARGB_0); _mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_0);
__m256i ymm_mask1 = __m256i ymm_mask1 =
_mm256_load_si256((const __m256i*)kShuffleMaskJ400ToARGB_1); _mm256_loadu_si256((const __m256i*)kShuffleMaskJ400ToARGB_1);
__m256i ymm_alpha = _mm256_set1_epi32((int)0xff000000u); __m256i ymm_alpha = _mm256_set1_epi32((int)0xff000000u);
while (width > 0) { while (width > 0) {
@ -732,7 +735,7 @@ void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
#endif // HAS_J400TOARGBROW_AVX2 #endif // HAS_J400TOARGBROW_AVX2
#ifdef HAS_RGB24TOARGBROW_AVX2 #ifdef HAS_RGB24TOARGBROW_AVX2
alignas(16) static const uint8_t kShuffleMaskRGB24ToARGB[2][16] = { static const uint8_t kShuffleMaskRGB24ToARGB[2][16] = {
{0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u}, {0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u},
{4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, {4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u,
128u}}; 128u}};
@ -879,9 +882,9 @@ void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
int width) { int width) {
__m256i ymm_alpha = _mm256_set1_epi32(0xff000000); __m256i ymm_alpha = _mm256_set1_epi32(0xff000000);
__m256i ymm_shuf = _mm256_broadcastsi128_si256( __m256i ymm_shuf = _mm256_broadcastsi128_si256(
_mm_load_si128((const __m128i*)kShuffleMaskRGB24ToARGB[0])); _mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[0]));
__m256i ymm_shuf2 = _mm256_broadcastsi128_si256( __m256i ymm_shuf2 = _mm256_broadcastsi128_si256(
_mm_load_si128((const __m128i*)kShuffleMaskRGB24ToARGB[1])); _mm_loadu_si128((const __m128i*)kShuffleMaskRGB24ToARGB[1]));
while (width > 0) { while (width > 0) {
__m128i xmm0 = _mm_loadu_si128((const __m128i*)src_rgb24); __m128i xmm0 = _mm_loadu_si128((const __m128i*)src_rgb24);
@ -973,6 +976,217 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
#endif #endif
#ifdef HAS_I422TORGB24ROW_AVX2
static const uint8_t kShuffleMaskARGBToRGB24[2][16] = {
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
128u},
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u,
14u}};
LIBYUV_TARGET_AVX2
void I422ToRGB24Row_AVX2(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
// Constants
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
__m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
__m256i ymm_shuf0 = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1]));
__m256i ymm_shuf1 = _mm256_broadcastsi128_si256(
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0]));
__m256i ymm_u_zero = _mm256_setzero_si256();
ptrdiff_t offset = src_v - src_u;
while (width >= 16) {
// READYUV422_AVX2
__m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
__m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
src_u += 8;
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
__m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
src_y += 16;
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
// YUVTORGB_AVX2
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
ymm0 = _mm256_srai_epi16(ymm0, 6);
ymm1 = _mm256_srai_epi16(ymm1, 6);
ymm2 = _mm256_srai_epi16(ymm2, 6);
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
// STORERGB24_AVX2
__m256i ymm0_packed = _mm256_unpacklo_epi8(ymm0, ymm1);
__m256i ymm2_packed = _mm256_unpacklo_epi8(ymm2, ymm2);
__m256i ymm1_packed = ymm0_packed;
ymm0_packed = _mm256_unpacklo_epi16(ymm0_packed, ymm2_packed);
ymm1_packed = _mm256_unpackhi_epi16(ymm1_packed, ymm2_packed);
ymm0_packed = _mm256_shuffle_epi8(ymm0_packed, ymm_shuf0);
ymm1_packed = _mm256_shuffle_epi8(ymm1_packed, ymm_shuf1);
ymm1_packed = _mm256_alignr_epi8(ymm1_packed, ymm0_packed, 0xc);
__m128i xmm0_store = _mm256_castsi256_si128(ymm0_packed);
__m128i xmm1_store = _mm256_castsi256_si128(ymm1_packed);
__m128i xmm2_store = _mm256_extractf128_si256(ymm0_packed, 1);
__m128i xmm3_store = _mm256_extractf128_si256(ymm1_packed, 1);
_mm_storel_epi64((__m128i*)dst_rgb24, xmm0_store);
_mm_storeu_si128((__m128i*)(dst_rgb24 + 8), xmm1_store);
_mm_storel_epi64((__m128i*)(dst_rgb24 + 24), xmm2_store);
_mm_storeu_si128((__m128i*)(dst_rgb24 + 32), xmm3_store);
dst_rgb24 += 48;
width -= 16;
}
_mm256_zeroupper();
}
#endif
#ifdef HAS_I422TORGB24ROW_AVX512VBMI
LIBYUV_TARGET_AVX512VBMI
void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
// Masks
static const uint8_t kMaskBG[64] = {
0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, 0x45,
0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, 0x13, 0x53,
0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, 0x60, 0x21, 0x61,
0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, 0x26, 0x66, 0x27, 0x67,
0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, 0x73, 0x34, 0x74, 0x35, 0x75,
0x36, 0x76, 0x37, 0x77};
static const uint8_t kMaskDST0[64] = {
0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, 0x43,
0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, 0x0f, 0x47,
0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, 0x16, 0x17, 0x53,
0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, 0x56, 0x1e, 0x1f, 0x57,
0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, 0x25, 0x62, 0x26, 0x27, 0x63,
0x28, 0x29, 0x64, 0x2a};
static const uint8_t kMaskDST1[64] = {
0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, 0x32,
0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, 0x74, 0x3a,
0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00};
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
// Constants
__m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
__m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
__m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
__m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
__m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
__m512i zmm_128 = _mm512_set1_epi8((char)0x80);
__m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG);
__m512i zmm_mask_DST0 = _mm512_loadu_si512((const __m512i*)kMaskDST0);
__m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1);
__m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
__m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
ptrdiff_t offset = src_v - src_u;
while (width >= 32) {
// READYUV422_AVX512BW
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
src_u += 16;
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
src_y += 32;
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
// YUVTORGB_AVX512BW
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
zmm0 = _mm512_srai_epi16(zmm0, 6);
zmm1 = _mm512_srai_epi16(zmm1, 6);
zmm2 = _mm512_srai_epi16(zmm2, 6);
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
// STORERGB24_AVX512VBMI
__m512i zmm_BG = _mm512_permi2var_epi8(zmm0, zmm_mask_BG, zmm1);
__m512i zmm_dst0 = _mm512_permi2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
__m512i zmm_dst1 = _mm512_permi2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1));
dst_rgb24 += 96;
width -= 32;
}
_mm256_zeroupper();
}
#endif
#ifdef __cplusplus #ifdef __cplusplus
} // extern "C" } // extern "C"
} // namespace libyuv } // namespace libyuv