coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

RAWToRGB24 avx2/avx512 use permq/shufb and avx512vbmi

Browse Source 
CPU / Arch                 Before (Kernel) After (Kernel)        Delta
----------------------------------------------------------------------
Haswell (x86_64)           263 ms (SSSE3)  263 ms (AVX2)          0.0%
Milan (Zen 3)              115 ms (SSSE3)  138 ms (AVX2)        -20.0%
SapphRap (x86_64)          173 ms (SSSE3)  178 ms (AVX512VBMI)   -2.9%
GNR (Granite R)            191 ms (SSSE3)  189 ms (AVX512VBMI)   +1.0%
Turin (Zen 5)               69 ms (SSSE3)   55 ms (AVX512VBMI)  +20.3%
Icelake (x86_64)           204 ms (SSSE3)  186 ms (AVX512VBMI)   +8.8%
Cascade Lake               237 ms (SSSE3)  229 ms (AVX512BW)     +3.4%
Neoverse (aarch64)*        103 ms (SVE2)   103 ms (SVE2)          0.0%

Bug: None
Change-Id: Ib218632e16b3308a214e9ffcbc745d6c41116c6e
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8280536
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard 2026-08-25 14:28:31 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent a0fa632939
commit 3793561ddc
8 changed files with 707 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.chromium
View File

@ -1,6 +1,6 @@
Name: libyuv Name: libyuv
URL: https://chromium.googlesource.com/libyuv/libyuv/ URL: https://chromium.googlesource.com/libyuv/libyuv/
Version: 1964 Version: 1965
Revision: DEPS Revision: DEPS
License: BSD-3-Clause License: BSD-3-Clause
License File: LICENSE License File: LICENSE

23
include/libyuv/row.h
View File

@ -313,6 +313,7 @@ extern "C" {
#define HAS_I410TOARGBROW_AVX2 #define HAS_I410TOARGBROW_AVX2
#if defined(__x86_64__) #if defined(__x86_64__)
#define HAS_I422TOAR30ROW_AVX2 #define HAS_I422TOAR30ROW_AVX2
#define HAS_RAWTORGB24ROW_AVX2
#endif #endif
#define HAS_I422TOUYVYROW_AVX2 #define HAS_I422TOUYVYROW_AVX2
#define HAS_I422TOYUY2ROW_AVX2 #define HAS_I422TOYUY2ROW_AVX2
@ -369,10 +370,13 @@ extern "C" {
#define HAS_NV12TORGB565ROW_AVX2 #define HAS_NV12TORGB565ROW_AVX2
#if defined(__x86_64__) || defined(_M_X64) #if defined(__x86_64__) || defined(_M_X64)
#define HAS_I422TOAR30ROW_AVX2 #define HAS_I422TOAR30ROW_AVX2
#define HAS_RAWTORGB24ROW_AVX2
#define HAS_RAWTOARGBROW_AVX512BW #define HAS_RAWTOARGBROW_AVX512BW
#define HAS_RGB24TOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW
#define HAS_RAWTORGB24ROW_AVX512BW
#define HAS_ARGBSHUFFLEROW_AVX512BW #define HAS_ARGBSHUFFLEROW_AVX512BW
#define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_I422TORGB24ROW_AVX512VBMI
#define HAS_RAWTORGB24ROW_AVX512VBMI
#define HAS_I422TORGB24ROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512BW
#define HAS_I422TOAR30ROW_AVX512BW #define HAS_I422TOAR30ROW_AVX512BW
#define HAS_RGBTOYMATRIXROW_AVX512BW #define HAS_RGBTOYMATRIXROW_AVX512BW
@ -408,6 +412,8 @@ extern "C" {
#define HAS_COPYROW_AVX512BW #define HAS_COPYROW_AVX512BW
#define HAS_RAWTOARGBROW_AVX512BW #define HAS_RAWTOARGBROW_AVX512BW
#define HAS_RGB24TOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW
#define HAS_RAWTORGB24ROW_AVX512BW
#define HAS_RAWTORGB24ROW_AVX512VBMI
#define HAS_ARGBTORGB24ROW_AVX512VBMI #define HAS_ARGBTORGB24ROW_AVX512VBMI
#define HAS_CONVERT16TO8ROW_AVX512BW #define HAS_CONVERT16TO8ROW_AVX512BW
#define HAS_HALFROW_16TO8_AVX512BW #define HAS_HALFROW_16TO8_AVX512BW
@ -425,6 +431,7 @@ extern "C" {
#define HAS_RGBTOUV444MATRIXROW_AVX512BW #define HAS_RGBTOUV444MATRIXROW_AVX512BW
#define HAS_ARGBTOYMATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW
#define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_I422TORGB24ROW_AVX512VBMI
#define HAS_RAWTORGB24ROW_AVX512VBMI
#define HAS_I422TORGB24ROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512BW
#define HAS_ARGBTOUVMATRIXROW_AVX512BW #define HAS_ARGBTOUVMATRIXROW_AVX512BW
#define HAS_J400TOARGBROW_AVX512BW #define HAS_J400TOARGBROW_AVX512BW
@ -3949,6 +3956,13 @@ void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
int width); int width);
void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width);
void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width);
void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565, void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565,
uint8_t* dst_argb, uint8_t* dst_argb,
@ -4054,6 +4068,15 @@ void RAWToRGBARow_Any_SSSE3(const uint8_t* src_ptr,
void RAWToRGB24Row_Any_SSSE3(const uint8_t* src_ptr, void RAWToRGB24Row_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr, uint8_t* dst_ptr,
int width); int width);
void RAWToRGB24Row_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RAWToRGB24Row_Any_AVX512BW(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RAWToRGB24Row_Any_AVX512VBMI(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RGB565ToARGBRow_Any_AVX2(const uint8_t* src_ptr, void RGB565ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr, uint8_t* dst_ptr,

2
include/libyuv/version.h
View File

@ -11,6 +11,6 @@
#ifndef INCLUDE_LIBYUV_VERSION_H_ #ifndef INCLUDE_LIBYUV_VERSION_H_
#define INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_
#define LIBYUV_VERSION 1964 #define LIBYUV_VERSION 1965
#endif // INCLUDE_LIBYUV_VERSION_H_ #endif // INCLUDE_LIBYUV_VERSION_H_

24
source/planar_functions.cc
View File

@ -3509,6 +3509,30 @@ int RAWToRGB24(const uint8_t* src_raw,
} }
} }
#endif #endif
#if defined(HAS_RAWTORGB24ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
RAWToRGB24Row = RAWToRGB24Row_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
RAWToRGB24Row = RAWToRGB24Row_AVX2;
}
}
#endif
#if defined(HAS_RAWTORGB24ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
RAWToRGB24Row = RAWToRGB24Row_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
RAWToRGB24Row = RAWToRGB24Row_AVX512BW;
}
}
#endif
#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
RAWToRGB24Row = RAWToRGB24Row_Any_AVX512VBMI;
if (IS_ALIGNED(width, 64)) {
RAWToRGB24Row = RAWToRGB24Row_AVX512VBMI;
}
}
#endif
#if defined(HAS_RAWTORGB24ROW_NEON) #if defined(HAS_RAWTORGB24ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) { if (TestCpuFlag(kCpuHasNEON)) {
RAWToRGB24Row = RAWToRGB24Row_Any_NEON; RAWToRGB24Row = RAWToRGB24Row_Any_NEON;

9
source/row_any.cc
View File

@ -1071,6 +1071,15 @@ ANY11(RAWToRGBARow_Any_SSSE3, RAWToRGBARow_SSSE3, 0, 3, 4, 15)
#if defined(HAS_RAWTORGB24ROW_SSSE3) #if defined(HAS_RAWTORGB24ROW_SSSE3)
ANY11(RAWToRGB24Row_Any_SSSE3, RAWToRGB24Row_SSSE3, 0, 3, 3, 7) ANY11(RAWToRGB24Row_Any_SSSE3, RAWToRGB24Row_SSSE3, 0, 3, 3, 7)
#endif #endif
#if defined(HAS_RAWTORGB24ROW_AVX2)
ANY11(RAWToRGB24Row_Any_AVX2, RAWToRGB24Row_AVX2, 0, 3, 3, 31)
#endif
#if defined(HAS_RAWTORGB24ROW_AVX512BW)
ANY11(RAWToRGB24Row_Any_AVX512BW, RAWToRGB24Row_AVX512BW, 0, 3, 3, 63)
#endif
#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
ANY11(RAWToRGB24Row_Any_AVX512VBMI, RAWToRGB24Row_AVX512VBMI, 0, 3, 3, 63)
#endif
#if defined(HAS_RGB565TOARGBROW_AVX2) #if defined(HAS_RGB565TOARGBROW_AVX2)
ANY11(RGB565ToARGBRow_Any_AVX2, RGB565ToARGBRow_AVX2, 0, 2, 4, 15) ANY11(RGB565ToARGBRow_Any_AVX2, RGB565ToARGBRow_AVX2, 0, 2, 4, 15)
#endif #endif

427
source/row_gcc.cc
View File

@ -46,16 +46,23 @@ static const uvec8 kARGBToYJ = {29u, 150u, 77u, 0u, 29u, 150u, 77u, 0u,
static const uvec16 kSub128 = {0x8080u, 0x8080u, 0x8080u, 0x8080u, static const uvec16 kSub128 = {0x8080u, 0x8080u, 0x8080u, 0x8080u,
0x8080u, 0x8080u, 0x8080u, 0x8080u}; 0x8080u, 0x8080u, 0x8080u, 0x8080u};
#endif // defined(HAS_ARGBTOYMATRIXROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3) #endif // defined(HAS_ARGBTOYMATRIXROW_SSSE3) ||
// defined(HAS_I422TOARGBROW_SSSE3)
#ifdef HAS_RGB24TOARGBROW_SSSE3
#if defined(HAS_RGB24TOARGBROW_SSSE3) || defined(HAS_RGB24TOARGBROW_AVX2) || \
defined(HAS_RGB24TOARGBROW_AVX512BW) || \
defined(HAS_RGBTOYMATRIXROW_AVX512BW)
// Shuffle table for converting RGB24 to ARGB. // Shuffle table for converting RGB24 to ARGB.
static const uvec8 kShuffleMaskRGB24ToARGB[2] = { static const uvec8 kShuffleMaskRGB24ToARGB[2] = {
{0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u}, {0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u},
{4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, {4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u,
128u}}; 128u}};
#endif // defined(HAS_RGB24TOARGBROW_SSSE3) ||
// defined(HAS_RGB24TOARGBROW_AVX2) ||
// defined(HAS_RGB24TOARGBROW_AVX512BW) ||
// defined(HAS_RGBTOYMATRIXROW_AVX512BW)
#if defined(HAS_RAWTOARGBROW_SSSE3) || defined(HAS_RAWTOARGBROW_AVX2)
// Shuffle table for converting RAW to ARGB. // Shuffle table for converting RAW to ARGB.
static const uvec8 kShuffleMaskRAWToARGB = { static const uvec8 kShuffleMaskRAWToARGB = {
2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u}; 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u};
@ -63,17 +70,73 @@ static const uvec8 kShuffleMaskRAWToARGB = {
static const uvec8 kShuffleMaskRAWToARGB_0 = {6u, 5u, 4u, 128u, 9u, 8u, static const uvec8 kShuffleMaskRAWToARGB_0 = {6u, 5u, 4u, 128u, 9u, 8u,
7u, 128u, 12u, 11u, 10u, 128u, 7u, 128u, 12u, 11u, 10u, 128u,
15u, 14u, 13u, 128u}; 15u, 14u, 13u, 128u};
#endif // defined(HAS_RAWTOARGBROW_SSSE3) || defined(HAS_RAWTOARGBROW_AVX2)
#if defined(HAS_RAWTOARGBROW_AVX512BW) #if defined(HAS_RAWTOARGBROW_AVX512BW)
static const uvec8 kShuffleMaskRAWToARGB_AVX512BW[2] = { static const uvec8 kShuffleMaskRAWToARGB_AVX512BW[2] = {
{2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u}, {2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u},
{6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, {6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u,
128u}}; 128u}};
#endif #endif // defined(HAS_RAWTOARGBROW_AVX512BW)
#if defined(HAS_RAWTORGB24ROW_AVX2) || defined(HAS_RAWTORGB24ROW_AVX512BW)
static const uvec8 kShuffleMaskRAWToRGB24_AVX2[2] = {
{2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 6u, 11u, 10u, 9u, 128u, 128u, 128u, 128u},
{6u, 5u, 4u, 9u, 8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u, 128u, 128u, 128u,
128u}};
#endif // defined(HAS_RAWTORGB24ROW_AVX2) ||
// defined(HAS_RAWTORGB24ROW_AVX512BW)
#if defined(HAS_RAWTORGB24ROW_AVX2)
static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 0, 0};
static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 0, 0, 0, 0};
static const uint32_t kPermd2_AVX2[8] = {0, 0, 0, 0, 0, 1, 2, 4};
static const uint32_t kPermd3_AVX2[8] = {5, 6, 0, 0, 0, 0, 0, 0};
static const uint32_t kPermd4_AVX2[8] = {0, 0, 0, 1, 2, 4, 5, 6};
#endif // defined(HAS_RAWTORGB24ROW_AVX2)
#if defined(HAS_RAWTORGB24ROW_AVX512BW)
static const uint32_t kPermdRGB24_0_AVX512BW[16] = {
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20};
static const uint32_t kPermdRGB24_1_AVX512BW[16] = {
5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25};
static const uint32_t kPermdRGB24_2_AVX512BW[16] = {
10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30};
#endif // defined(HAS_RAWTORGB24ROW_AVX512BW)
#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
static const uint8_t kPermRAWToRGB24_0[64] = {
0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a, 0x09,
0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17, 0x16, 0x15,
0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e, 0x23, 0x22, 0x21,
0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b, 0x2a, 0x2f, 0x2e, 0x2d,
0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38, 0x37, 0x36, 0x3b, 0x3a, 0x39,
0x3e, 0x3d, 0x3c, 0x41};
static const uint8_t kPermRAWToRGB24_1[64] = {
0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08, 0x0d,
0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15, 0x14, 0x19,
0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22, 0x21, 0x20, 0x25,
0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29, 0x2e, 0x2d, 0x2c, 0x31,
0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36, 0x35, 0x3a, 0x39, 0x38, 0x3d,
0x3c, 0x3b, 0x00, 0x3f};
static const uint8_t kPermRAWToRGB24_2[64] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b,
0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23,
0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b,
0x3c, 0x3d, 0x40, 0x3f};
static const uint8_t kPermRAWToRGB24_3[64] = {
0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c, 0x4b,
0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53, 0x58, 0x57,
0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60, 0x5f, 0x64, 0x63,
0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d, 0x6c, 0x6b, 0x70, 0x6f,
0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74, 0x79, 0x78, 0x77, 0x7c, 0x7b,
0x7a, 0x7f, 0x7e, 0x7d};
#endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI)
#if defined(HAS_RAWTORGBAROW_SSSE3)
// Shuffle table for converting RAW to RGBA. // Shuffle table for converting RAW to RGBA.
static const uvec8 kShuffleMaskRAWToRGBA = { static const uvec8 kShuffleMaskRAWToRGBA = {
128u, 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u}; 128u, 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u};
#endif // defined(HAS_RAWTORGBAROW_SSSE3)
#if defined(HAS_RAWTORGB24ROW_SSSE3)
// Shuffle table for converting RAW to RGB24. First 8. // Shuffle table for converting RAW to RGB24. First 8.
static const uvec8 kShuffleMaskRAWToRGB24_0 = { static const uvec8 kShuffleMaskRAWToRGB24_0 = {
2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u,
@ -88,6 +151,7 @@ static const uvec8 kShuffleMaskRAWToRGB24_1 = {
static const uvec8 kShuffleMaskRAWToRGB24_2 = { static const uvec8 kShuffleMaskRAWToRGB24_2 = {
8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u, 8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u,
128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u}; 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
#endif // defined(HAS_RAWTORGB24ROW_SSSE3)
// Shuffle table for converting ARGB to RGB24. // Shuffle table for converting ARGB to RGB24.
// and for I422ToRGB24. First 8 + next 4 // and for I422ToRGB24. First 8 + next 4
@ -122,7 +186,6 @@ static const lvec8 kShuffleNV21 = {
1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6, 1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6, 1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
}; };
#endif // HAS_RGB24TOARGBROW_SSSE3
#if defined(HAS_J400TOARGBROW_AVX2) || defined(HAS_J400TOARGBROW_AVX512BW) #if defined(HAS_J400TOARGBROW_AVX2) || defined(HAS_J400TOARGBROW_AVX512BW)
static const uint8_t kShuffleMaskJ400ToARGB[64] = { static const uint8_t kShuffleMaskJ400ToARGB[64] = {
@ -131,7 +194,8 @@ static const uint8_t kShuffleMaskJ400ToARGB[64] = {
7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u, 7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u, 128u, 10u, 10u,
10u, 128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, 13u, 13u, 13u, 128u, 10u, 128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, 13u, 13u, 13u, 128u,
14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u}; 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u};
#endif #endif // defined(HAS_J400TOARGBROW_AVX2) ||
// defined(HAS_J400TOARGBROW_AVX512BW)
#ifdef HAS_J400TOARGBROW_AVX2 #ifdef HAS_J400TOARGBROW_AVX2
void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) { void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
@ -235,6 +299,7 @@ void RGB24ToARGBRow_SSSE3(const uint8_t* src_rgb24,
: "m"(kShuffleMaskRGB24ToARGB[0]) // %3 : "m"(kShuffleMaskRGB24ToARGB[0]) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_RGB24TOARGBROW_SSSE3
#ifdef HAS_RGB24TOARGBROW_AVX2 #ifdef HAS_RGB24TOARGBROW_AVX2
void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24, void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
@ -282,6 +347,7 @@ void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
} }
#endif // HAS_RGB24TOARGBROW_AVX2 #endif // HAS_RGB24TOARGBROW_AVX2
#ifdef HAS_RAWTOARGBROW_SSSE3
void RAWToARGBRow_SSSE3(const uint8_t* src_raw, uint8_t* dst_argb, int width) { void RAWToARGBRow_SSSE3(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
asm volatile( asm volatile(
"pcmpeqb %%xmm6,%%xmm6 \n" // 0xff000000 "pcmpeqb %%xmm6,%%xmm6 \n" // 0xff000000
@ -318,7 +384,9 @@ void RAWToARGBRow_SSSE3(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
"m"(kShuffleMaskRAWToARGB_0) // %4 "m"(kShuffleMaskRAWToARGB_0) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
} }
#endif // HAS_RAWTOARGBROW_SSSE3
#ifdef HAS_RAWTOARGBROW_AVX2
void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) { void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
asm volatile( asm volatile(
"vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" // 0xff000000 "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" // 0xff000000
@ -360,9 +428,11 @@ void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
"m"(kShuffleMaskRAWToARGB_0) // %4 "m"(kShuffleMaskRAWToARGB_0) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
} }
#endif // HAS_RAWTOARGBROW_AVX2
#if defined(HAS_RAWTOARGBROW_AVX512BW) || \ #if defined(HAS_RAWTOARGBROW_AVX512BW) || \
defined(HAS_RGB24TOARGBROW_AVX512BW) || \ defined(HAS_RGB24TOARGBROW_AVX512BW) || \
defined(HAS_RAWTORGB24ROW_AVX512BW) || \
defined(HAS_RGBTOYMATRIXROW_AVX512BW) defined(HAS_RGBTOYMATRIXROW_AVX512BW)
static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2,
3, 4, 4, 5}; 3, 4, 4, 5};
@ -372,7 +442,10 @@ static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6,
7, 8, 8, 9}; 7, 8, 8, 9};
static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4,
5, 6, 6, 7}; 5, 6, 6, 7};
#endif #endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
// defined(HAS_RGB24TOARGBROW_AVX512BW) ||
// defined(HAS_RAWTORGB24ROW_AVX512BW) ||
// defined(HAS_RGBTOYMATRIXROW_AVX512BW)
#ifdef HAS_RAWTOARGBROW_AVX512BW #ifdef HAS_RAWTOARGBROW_AVX512BW
void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
@ -444,8 +517,9 @@ void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
RGBToARGBRow_AVX512BW(src_rgb24, dst_argb, RGBToARGBRow_AVX512BW(src_rgb24, dst_argb,
(const uint32_t*)&kShuffleMaskRGB24ToARGB[0], width); (const uint32_t*)&kShuffleMaskRGB24ToARGB[0], width);
} }
#endif #endif // HAS_RAWTOARGBROW_AVX512BW
#ifdef HAS_RAWTORGBAROW_SSSE3
// Same code as RAWToARGB with different shuffler and A in low bits // Same code as RAWToARGB with different shuffler and A in low bits
void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
asm volatile( asm volatile(
@ -484,7 +558,9 @@ void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
: "m"(kShuffleMaskRAWToRGBA) // %3 : "m"(kShuffleMaskRAWToRGBA) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_RAWTORGBAROW_SSSE3
#ifdef HAS_RAWTORGB24ROW_SSSE3
void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, void RAWToRGB24Row_SSSE3(const uint8_t* src_raw,
uint8_t* dst_rgb24, uint8_t* dst_rgb24,
int width) { int width) {
@ -516,6 +592,174 @@ void RAWToRGB24Row_SSSE3(const uint8_t* src_raw,
"m"(kShuffleMaskRAWToRGB24_2) // %5 "m"(kShuffleMaskRAWToRGB24_2) // %5
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_RAWTORGB24ROW_SSSE3
#ifdef HAS_RAWTORGB24ROW_AVX2
void RAWToRGB24Row_AVX2(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width) {
asm volatile(
"vmovdqu %3,%%ymm7 \n" // shuffler (32 bytes)
"vmovdqu %4,%%ymm8 \n" // kPermd0_AVX2
"vmovdqu %5,%%ymm9 \n" // kPermd1_AVX2
"vmovdqu %6,%%ymm10 \n" // kPermd2_AVX2
"vmovdqu %7,%%ymm11 \n" // kPermd3_AVX2
"vmovdqu %8,%%ymm12 \n" // kPermd4_AVX2
LABELALIGN
"1: \n"
"vmovdqu (%0),%%ymm0 \n" // raw0
"vmovdqu 32(%0),%%ymm1 \n" // raw1
"vmovdqu 64(%0),%%ymm2 \n" // raw2
"lea 96(%0),%0 \n"
"vpermq $0x94,%%ymm0,%%ymm3 \n" // b0
"vperm2i128 $0x21,%%ymm1,%%ymm0,%%ymm4 \n" // m01
"vperm2i128 $0x21,%%ymm2,%%ymm1,%%ymm5 \n" // m12
"vpermq $0xe9,%%ymm2,%%ymm6 \n" // b3
"vpermq $0xe9,%%ymm4,%%ymm4 \n" // b1
"vpermq $0x94,%%ymm5,%%ymm5 \n" // b2
"vpshufb %%ymm7,%%ymm3,%%ymm3 \n"
"vpshufb %%ymm7,%%ymm4,%%ymm4 \n"
"vpshufb %%ymm7,%%ymm5,%%ymm5 \n"
"vpshufb %%ymm7,%%ymm6,%%ymm6 \n"
// Dst0
"vpermd %%ymm3,%%ymm8,%%ymm0 \n"
"vpermq $0x00,%%ymm4,%%ymm1 \n"
"vpermd %%ymm4,%%ymm9,%%ymm2 \n"
"vpermd %%ymm5,%%ymm10,%%ymm3 \n"
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu %%ymm0,(%1) \n"
// Dst1
"vpblendd $0xf0,%%ymm3,%%ymm2,%%ymm2 \n"
"vpermd %%ymm5,%%ymm11,%%ymm4 \n"
"vpermd %%ymm6,%%ymm12,%%ymm5 \n"
"vmovdqu %%ymm2,32(%1) \n"
// Dst2
"vpblendd $0xfc,%%ymm5,%%ymm4,%%ymm4 \n"
"vmovdqu %%ymm4,64(%1) \n"
"lea 96(%1),%1 \n"
"sub $32,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_raw), // %0
"+r"(dst_rgb24), // %1
"+r"(width) // %2
: "m"(kShuffleMaskRAWToRGB24_AVX2[0]), // %3
"m"(kPermd0_AVX2), // %4
"m"(kPermd1_AVX2), // %5
"m"(kPermd2_AVX2), // %6
"m"(kPermd3_AVX2), // %7
"m"(kPermd4_AVX2) // %8
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12");
}
#endif // HAS_RAWTORGB24ROW_AVX2
#ifdef HAS_RAWTORGB24ROW_AVX512BW
void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width) {
asm volatile(
"vmovdqu64 %3,%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW
"vmovdqu64 %4,%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW
"vmovdqu64 %5,%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW
"vmovdqu64 %6,%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW
"vbroadcasti64x4 %7,%%zmm4 \n" // shuffler (32 bytes)
"vmovdqu32 %8,%%zmm24 \n" // kPermdRGB24_0_AVX512BW
"vmovdqu32 %9,%%zmm25 \n" // kPermdRGB24_1_AVX512BW
"vmovdqu32 %10,%%zmm26 \n" // kPermdRGB24_2_AVX512BW
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0 \n"
"vmovdqu8 64(%0),%%zmm1 \n"
"vmovdqu8 128(%0),%%zmm2 \n"
"lea 192(%0),%0 \n"
"vpermq %%zmm2,%%zmm23,%%zmm3 \n"
"vmovdqa64 %%zmm22,%%zmm18 \n"
"vpermi2q %%zmm2,%%zmm1,%%zmm18 \n"
"vmovdqa64 %%zmm21,%%zmm19 \n"
"vpermi2q %%zmm1,%%zmm0,%%zmm19 \n"
"vpermq %%zmm0,%%zmm20,%%zmm0 \n"
"vpshufb %%zmm4,%%zmm0,%%zmm0 \n"
"vpshufb %%zmm4,%%zmm19,%%zmm1 \n"
"vpshufb %%zmm4,%%zmm18,%%zmm2 \n"
"vpshufb %%zmm4,%%zmm3,%%zmm3 \n"
"vmovdqa32 %%zmm24,%%zmm16 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm16 \n"
"vmovdqa32 %%zmm25,%%zmm17 \n"
"vpermi2d %%zmm2,%%zmm1,%%zmm17 \n"
"vmovdqa32 %%zmm26,%%zmm18 \n"
"vpermi2d %%zmm3,%%zmm2,%%zmm18 \n"
"vmovdqu8 %%zmm16,(%1) \n"
"vmovdqu8 %%zmm17,0x40(%1) \n"
"vmovdqu8 %%zmm18,0x80(%1) \n"
"lea 0xc0(%1),%1 \n"
"sub $0x40,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_raw), // %0
"+r"(dst_rgb24), // %1
"+r"(width) // %2
: "m"(kPermqRGB24ToARGB0_AVX512BW), // %3
"m"(kPermqRGB24ToARGB1_AVX512BW), // %4
"m"(kPermqRGB24ToARGB2_AVX512BW), // %5
"m"(kPermqRGB24ToARGB3_AVX512BW), // %6
"m"(kShuffleMaskRAWToRGB24_AVX2[0]), // %7
"m"(kPermdRGB24_0_AVX512BW), // %8
"m"(kPermdRGB24_1_AVX512BW), // %9
"m"(kPermdRGB24_2_AVX512BW) // %10
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16",
"xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23",
"xmm24", "xmm25", "xmm26");
}
#endif // HAS_RAWTORGB24ROW_AVX512BW
#ifdef HAS_RAWTORGB24ROW_AVX512VBMI
void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width) {
asm volatile(
"vmovdqu32 %[kPermRAWToRGB24_0],%%zmm20 \n"
"vmovdqu32 %[kPermRAWToRGB24_1],%%zmm21 \n"
"vmovdqu32 %[kPermRAWToRGB24_2],%%zmm22 \n"
"vmovdqu32 %[kPermRAWToRGB24_3],%%zmm23 \n"
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0 \n"
"vmovdqu8 64(%0),%%zmm1 \n"
"vmovdqu8 128(%0),%%zmm2 \n"
"lea 192(%0),%0 \n"
"vmovdqa64 %%zmm1,%%zmm3 \n"
"vpermt2b %%zmm0,%%zmm21,%%zmm1 \n"
"vpermt2b %%zmm3,%%zmm20,%%zmm0 \n"
"vpermt2b %%zmm2,%%zmm23,%%zmm3 \n"
"vpermt2b %%zmm2,%%zmm22,%%zmm1 \n"
"vmovdqu8 %%zmm0,(%1) \n"
"vmovdqu8 %%zmm1,0x40(%1) \n"
"vmovdqu8 %%zmm3,0x80(%1) \n"
"lea 0xc0(%1),%1 \n"
"sub $0x40,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_raw), // %0
"+r"(dst_rgb24), // %1
"+r"(width) // %2
: [kPermRAWToRGB24_0] "m"(kPermRAWToRGB24_0),
[kPermRAWToRGB24_1] "m"(kPermRAWToRGB24_1),
[kPermRAWToRGB24_2] "m"(kPermRAWToRGB24_2),
[kPermRAWToRGB24_3] "m"(kPermRAWToRGB24_3)
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm20", "xmm21",
"xmm22", "xmm23");
}
#endif // HAS_RAWTORGB24ROW_AVX512VBMI
#ifdef HAS_RGB565TOARGBROW_AVX2 #ifdef HAS_RGB565TOARGBROW_AVX2
void RGB565ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { void RGB565ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
@ -565,7 +809,7 @@ void RGB565ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7"); "xmm6", "xmm7");
} }
#endif #endif // HAS_RGB565TOARGBROW_AVX2
#ifdef HAS_ARGB1555TOARGBROW_AVX2 #ifdef HAS_ARGB1555TOARGBROW_AVX2
void ARGB1555ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { void ARGB1555ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
@ -616,7 +860,7 @@ void ARGB1555ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7"); "xmm6", "xmm7");
} }
#endif #endif // HAS_ARGB1555TOARGBROW_AVX2
#ifdef HAS_ARGB4444TOARGBROW_AVX2 #ifdef HAS_ARGB4444TOARGBROW_AVX2
void ARGB4444ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { void ARGB4444ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
@ -653,8 +897,9 @@ void ARGB4444ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
: :
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
} }
#endif #endif // HAS_ARGB4444TOARGBROW_AVX2
#ifdef HAS_ARGBTORGB24ROW_SSSE3
void ARGBToRGB24Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) { void ARGBToRGB24Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
asm volatile("movdqa %3,%%xmm6 \n" asm volatile("movdqa %3,%%xmm6 \n"
@ -692,7 +937,9 @@ void ARGBToRGB24Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6"); "xmm6");
} }
#endif // HAS_ARGBTORGB24ROW_SSSE3
#ifdef HAS_ARGBTORAWROW_SSSE3
void ARGBToRAWRow_SSSE3(const uint8_t* src, uint8_t* dst, int width) { void ARGBToRAWRow_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
asm volatile("movdqa %3,%%xmm6 \n" asm volatile("movdqa %3,%%xmm6 \n"
@ -730,6 +977,7 @@ void ARGBToRAWRow_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6"); "xmm6");
} }
#endif // HAS_ARGBTORAWROW_SSSE3
#ifdef HAS_ARGBTORGB24ROW_AVX2 #ifdef HAS_ARGBTORGB24ROW_AVX2
// vpermd for 12+12 to 24 // vpermd for 12+12 to 24
@ -778,7 +1026,7 @@ void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7"); "xmm7");
} }
#endif #endif // HAS_ARGBTORGB24ROW_AVX2
#ifdef HAS_ARGBTORGB24ROW_AVX512VBMI #ifdef HAS_ARGBTORGB24ROW_AVX512VBMI
// Shuffle table for converting ARGBToRGB24 // Shuffle table for converting ARGBToRGB24
@ -827,7 +1075,7 @@ void ARGBToRGB24Row_AVX512VBMI(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm16", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm16",
"xmm17"); "xmm17");
} }
#endif #endif // HAS_ARGBTORGB24ROW_AVX512VBMI
#ifdef HAS_ARGBTORAWROW_AVX2 #ifdef HAS_ARGBTORAWROW_AVX2
void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
@ -873,7 +1121,7 @@ void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7"); "xmm7");
} }
#endif #endif // HAS_ARGBTORAWROW_AVX2
#ifdef HAS_ARGBTORGB565DITHERROW_AVX2 #ifdef HAS_ARGBTORGB565DITHERROW_AVX2
void ARGBToRGB565DitherRow_AVX2(const uint8_t* src, void ARGBToRGB565DitherRow_AVX2(const uint8_t* src,
@ -1039,8 +1287,6 @@ void ARGBToARGB4444Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
} }
#endif // HAS_ARGBTOARGB4444ROW_AVX2 #endif // HAS_ARGBTOARGB4444ROW_AVX2
#endif // HAS_RGB24TOARGBROW_SSSE3
/* /*
ARGBToAR30Row: ARGBToAR30Row:
@ -1187,7 +1433,7 @@ void ARGBToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
"m"(kMulAG10) // %7 "m"(kMulAG10) // %7
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
} }
#endif #endif // HAS_ARGBTOAR30ROW_AVX2
#ifdef HAS_ABGRTOAR30ROW_AVX2 #ifdef HAS_ABGRTOAR30ROW_AVX2
void ABGRToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { void ABGRToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
@ -1224,7 +1470,7 @@ void ABGRToAR30Row_AVX2(const uint8_t* src, uint8_t* dst, int width) {
"m"(kMulAG10) // %7 "m"(kMulAG10) // %7
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
} }
#endif #endif // HAS_ABGRTOAR30ROW_AVX2
static const uvec8 kShuffleARGBToABGR = {2, 1, 0, 3, 6, 5, 4, 7, static const uvec8 kShuffleARGBToABGR = {2, 1, 0, 3, 6, 5, 4, 7,
10, 9, 8, 11, 14, 13, 12, 15}; 10, 9, 8, 11, 14, 13, 12, 15};
@ -1351,7 +1597,7 @@ void ARGBToAR64Row_AVX2(const uint8_t* src_argb,
::"memory", ::"memory",
"cc", "xmm0", "xmm1"); "cc", "xmm0", "xmm1");
} }
#endif #endif // HAS_ARGBTOAR64ROW_AVX2
#ifdef HAS_ARGBTOAB64ROW_AVX2 #ifdef HAS_ARGBTOAB64ROW_AVX2
void ARGBToAB64Row_AVX2(const uint8_t* src_argb, void ARGBToAB64Row_AVX2(const uint8_t* src_argb,
@ -1379,7 +1625,7 @@ void ARGBToAB64Row_AVX2(const uint8_t* src_argb,
"m"(kShuffleARGBToAB64Hi) // %3 "m"(kShuffleARGBToAB64Hi) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
} }
#endif #endif // HAS_ARGBTOAB64ROW_AVX2
#ifdef HAS_AR64TOARGBROW_AVX2 #ifdef HAS_AR64TOARGBROW_AVX2
void AR64ToARGBRow_AVX2(const uint16_t* src_ar64, void AR64ToARGBRow_AVX2(const uint16_t* src_ar64,
@ -1405,7 +1651,7 @@ void AR64ToARGBRow_AVX2(const uint16_t* src_ar64,
::"memory", ::"memory",
"cc", "xmm0", "xmm1"); "cc", "xmm0", "xmm1");
} }
#endif #endif // HAS_AR64TOARGBROW_AVX2
#ifdef HAS_AB64TOARGBROW_AVX2 #ifdef HAS_AB64TOARGBROW_AVX2
void AB64ToARGBRow_AVX2(const uint16_t* src_ab64, void AB64ToARGBRow_AVX2(const uint16_t* src_ab64,
@ -1432,7 +1678,7 @@ void AB64ToARGBRow_AVX2(const uint16_t* src_ab64,
: "m"(kShuffleARGBToABGR) // %3 : "m"(kShuffleARGBToABGR) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2"); : "memory", "cc", "xmm0", "xmm1", "xmm2");
} }
#endif #endif // HAS_AB64TOARGBROW_AVX2
// clang-format off // clang-format off
@ -1506,7 +1752,9 @@ void AB64ToARGBRow_AVX2(const uint16_t* src_ab64,
defined(HAS_ARGBEXTRACTALPHAROW_AVX2) defined(HAS_ARGBEXTRACTALPHAROW_AVX2)
// vpermd for vphaddw + vpackuswb vpermd. // vpermd for vphaddw + vpackuswb vpermd.
static const lvec32 kPermdARGBToY_AVX = {0, 4, 1, 5, 2, 6, 3, 7}; static const lvec32 kPermdARGBToY_AVX = {0, 4, 1, 5, 2, 6, 3, 7};
#endif #endif // defined(HAS_ARGBTOYMATRIXROW_AVX2) ||
// defined(HAS_ARGBTOUV444MATRIXROW_AVX2) ||
// defined(HAS_ARGBEXTRACTALPHAROW_AVX2)
#ifdef HAS_ARGBTOYMATRIXROW_SSSE3 #ifdef HAS_ARGBTOYMATRIXROW_SSSE3
void ARGBToYMatrixRow_SSSE3(const uint8_t* src_argb, void ARGBToYMatrixRow_SSSE3(const uint8_t* src_argb,
@ -1530,7 +1778,7 @@ void ARGBToYMatrixRow_SSSE3(const uint8_t* src_argb,
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7"); "xmm7");
} }
#endif #endif // HAS_ARGBTOYMATRIXROW_SSSE3
#ifdef HAS_ARGBTOYMATRIXROW_AVX2 #ifdef HAS_ARGBTOYMATRIXROW_AVX2
void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb, void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
@ -1556,7 +1804,7 @@ void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7"); "xmm7");
} }
#endif #endif // HAS_ARGBTOYMATRIXROW_AVX2
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \ #if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \
defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \ defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \
@ -1564,12 +1812,15 @@ void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
defined(HAS_RGBTOYMATRIXROW_AVX512BW) defined(HAS_RGBTOYMATRIXROW_AVX512BW)
static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13, static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13,
2, 6, 10, 14, 3, 7, 11, 15}; 2, 6, 10, 14, 3, 7, 11, 15};
#endif #endif // defined(HAS_ARGBTOYMATRIXROW_AVX512BW) ||
// defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) ||
// defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) ||
// defined(HAS_RGBTOYMATRIXROW_AVX512BW)
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
static const uint32_t kPermdARGBToUV_AVX512BW[16] = { static const uint32_t kPermdARGBToUV_AVX512BW[16] = {
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15}; 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15};
#endif #endif // defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
#ifdef HAS_ARGBTOYMATRIXROW_AVX512BW #ifdef HAS_ARGBTOYMATRIXROW_AVX512BW
void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
@ -1628,7 +1879,7 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm16", "xmm17", "xmm18"); "xmm16", "xmm17", "xmm18");
} }
#endif #endif // HAS_ARGBTOYMATRIXROW_AVX512BW
#ifdef HAS_RGBTOYMATRIXROW_AVX512BW #ifdef HAS_RGBTOYMATRIXROW_AVX512BW
void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
@ -1709,7 +1960,7 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
"xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22",
"xmm23", "xmm24", "xmm25"); "xmm23", "xmm24", "xmm25");
} }
#endif #endif // HAS_RGBTOYMATRIXROW_AVX512BW
#ifdef HAS_ARGBTOUV444MATRIXROW_SSSE3 #ifdef HAS_ARGBTOUV444MATRIXROW_SSSE3
void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb, void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb,
@ -2443,7 +2694,7 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
"movdqa %%xmm4,%%xmm1 \n" "movdqa %%xmm4,%%xmm1 \n"
#define YUVTORGB_REGS #define YUVTORGB_REGS
#endif #endif // defined(__x86_64__)
#define YUVTORGB(yuvconstants) \ #define YUVTORGB(yuvconstants) \
YUVTORGB16(yuvconstants) \ YUVTORGB16(yuvconstants) \
@ -2899,7 +3150,7 @@ void OMITFP I210AlphaToARGBRow_SSSE3(const uint16_t* y_buf,
: "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
"xmm4", "xmm5"); "xmm4", "xmm5");
} }
#endif #endif // HAS_I210ALPHATOARGBROW_SSSE3
#ifdef HAS_I410ALPHATOARGBROW_SSSE3 #ifdef HAS_I410ALPHATOARGBROW_SSSE3
// 10 bit YUVA to ARGB // 10 bit YUVA to ARGB
@ -2931,7 +3182,7 @@ void OMITFP I410AlphaToARGBRow_SSSE3(const uint16_t* y_buf,
: "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
"xmm4", "xmm5"); "xmm4", "xmm5");
} }
#endif #endif // HAS_I410ALPHATOARGBROW_SSSE3
// 10 bit YUV to AR30 // 10 bit YUV to AR30
void OMITFP I410ToAR30Row_SSSE3(const uint16_t* y_buf, void OMITFP I410ToAR30Row_SSSE3(const uint16_t* y_buf,
@ -3217,7 +3468,7 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
); );
} }
#endif // HAS_I422TOARGBROW_SSSE3 #endif // defined(HAS_I422TOARGBROW_SSSE3) || defined(HAS_I422TOARGBROW_AVX2)
// Read 16 UV from 444 // Read 16 UV from 444
#define READYUV444_AVX2 \ #define READYUV444_AVX2 \
@ -3542,7 +3793,7 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
"vpaddsw %%ymm4,%%ymm2,%%ymm2 \n" "vpaddsw %%ymm4,%%ymm2,%%ymm2 \n"
#define YUVTORGB_REGS_AVX2 #define YUVTORGB_REGS_AVX2
#endif #endif // defined(__x86_64__)
#define YUVTORGB_AVX2(yuvconstants) \ #define YUVTORGB_AVX2(yuvconstants) \
YUVTORGB16_AVX2(yuvconstants) \ YUVTORGB16_AVX2(yuvconstants) \
@ -3745,7 +3996,7 @@ void OMITFP I422ToARGBRow_AVX2(const uint8_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
); );
} }
#endif // HAS_I422TOARGBROW_AVX2 #endif // defined(HAS_I422TOARGBROW_AVX2)
#if defined(HAS_I422TORGB24ROW_AVX2) #if defined(HAS_I422TORGB24ROW_AVX2)
// 16 pixels // 16 pixels
@ -3786,7 +4037,7 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"
); );
} }
#endif // HAS_I422TORGB24ROW_AVX2 #endif // defined(HAS_I422TORGB24ROW_AVX2)
#if defined(HAS_I422TOARGBROW_AVX512BW) || \ #if defined(HAS_I422TOARGBROW_AVX512BW) || \
defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ defined(HAS_I422TORGB24ROW_AVX512VBMI) || \
@ -3797,7 +4048,10 @@ static const uint16_t kSplitQuadWords[32] = {
8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15}; 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
static const uint64_t kUnpermuteAVX512[8] = {0, 4, 1, 5, 2, 6, 3, 7}; static const uint64_t kUnpermuteAVX512[8] = {0, 4, 1, 5, 2, 6, 3, 7};
#endif #endif // defined(HAS_I422TOARGBROW_AVX512BW) ||
// defined(HAS_I422TORGB24ROW_AVX512VBMI) ||
// defined(HAS_I422TORGB24ROW_AVX512BW) ||
// defined(HAS_I422TOAR30ROW_AVX512BW)
#if defined(HAS_I422TOAR30ROW_AVX512BW) #if defined(HAS_I422TOAR30ROW_AVX512BW)
static const uint16_t kPermAR30_0[32] = { static const uint16_t kPermAR30_0[32] = {
@ -3806,7 +4060,7 @@ static const uint16_t kPermAR30_0[32] = {
static const uint16_t kPermAR30_1[32] = { static const uint16_t kPermAR30_1[32] = {
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55,
24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63}; 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63};
#endif #endif // defined(HAS_I422TOAR30ROW_AVX512BW)
#if defined(HAS_I422TOARGBROW_AVX512BW) #if defined(HAS_I422TOARGBROW_AVX512BW)
// 32 pixels // 32 pixels
@ -3845,7 +4099,7 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
); );
} }
#endif // HAS_I422TOARGBROW_AVX512BW #endif // defined(HAS_I422TOARGBROW_AVX512BW)
#if defined(HAS_I422TORGB24ROW_AVX512VBMI) #if defined(HAS_I422TORGB24ROW_AVX512VBMI)
static const uint8_t kMaskBG[64] = { static const uint8_t kMaskBG[64] = {
@ -3910,7 +4164,7 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
"xmm20", "xmm21", "xmm22" "xmm20", "xmm21", "xmm22"
); );
} }
#endif // HAS_I422TORGB24ROW_AVX512VBMI #endif // defined(HAS_I422TORGB24ROW_AVX512VBMI)
#if defined(HAS_I422TORGB24ROW_AVX512BW) #if defined(HAS_I422TORGB24ROW_AVX512BW)
static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12}; static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12};
@ -3959,7 +4213,7 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf,
"xmm20", "xmm21" "xmm20", "xmm21"
); );
} }
#endif // HAS_I422TORGB24ROW_AVX512BW #endif // defined(HAS_I422TORGB24ROW_AVX512BW)
#if defined(HAS_I422TOAR30ROW_AVX512BW) #if defined(HAS_I422TOAR30ROW_AVX512BW)
// 32 pixels // 32 pixels
@ -4008,7 +4262,7 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf,
"xmm19", "xmm20", "xmm21" "xmm19", "xmm20", "xmm21"
); );
} }
#endif // HAS_I422TOAR30ROW_AVX512BW #endif // defined(HAS_I422TOAR30ROW_AVX512BW)
#if defined(HAS_I422TOAR30ROW_AVX2) #if defined(HAS_I422TOAR30ROW_AVX2)
// 16 pixels // 16 pixels
@ -4047,7 +4301,7 @@ void OMITFP I422ToAR30Row_AVX2(const uint8_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
); );
} }
#endif // HAS_I422TOAR30ROW_AVX2 #endif // defined(HAS_I422TOAR30ROW_AVX2)
#if defined(HAS_I210TOARGBROW_AVX2) #if defined(HAS_I210TOARGBROW_AVX2)
// 16 pixels // 16 pixels
@ -4082,7 +4336,7 @@ void OMITFP I210ToARGBRow_AVX2(const uint16_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
); );
} }
#endif // HAS_I210TOARGBROW_AVX2 #endif // defined(HAS_I210TOARGBROW_AVX2)
#if defined(HAS_I212TOARGBROW_AVX2) #if defined(HAS_I212TOARGBROW_AVX2)
// 16 pixels // 16 pixels
@ -4117,7 +4371,7 @@ void OMITFP I212ToARGBRow_AVX2(const uint16_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
); );
} }
#endif // HAS_I212TOARGBROW_AVX2 #endif // defined(HAS_I212TOARGBROW_AVX2)
#if defined(HAS_I210TOAR30ROW_AVX2) #if defined(HAS_I210TOAR30ROW_AVX2)
// 16 pixels // 16 pixels
@ -4156,7 +4410,7 @@ void OMITFP I210ToAR30Row_AVX2(const uint16_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
); );
} }
#endif // HAS_I210TOAR30ROW_AVX2 #endif // defined(HAS_I210TOAR30ROW_AVX2)
#if defined(HAS_I212TOAR30ROW_AVX2) #if defined(HAS_I212TOAR30ROW_AVX2)
// 16 pixels // 16 pixels
@ -4195,7 +4449,7 @@ void OMITFP I212ToAR30Row_AVX2(const uint16_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
); );
} }
#endif // HAS_I212TOAR30ROW_AVX2 #endif // defined(HAS_I212TOAR30ROW_AVX2)
#if defined(HAS_I410TOARGBROW_AVX2) #if defined(HAS_I410TOARGBROW_AVX2)
// 16 pixels // 16 pixels
@ -4230,7 +4484,7 @@ void OMITFP I410ToARGBRow_AVX2(const uint16_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
); );
} }
#endif // HAS_I410TOARGBROW_AVX2 #endif // defined(HAS_I410TOARGBROW_AVX2)
#if defined(HAS_I210ALPHATOARGBROW_AVX2) #if defined(HAS_I210ALPHATOARGBROW_AVX2)
// 16 pixels // 16 pixels
@ -4265,7 +4519,7 @@ void OMITFP I210AlphaToARGBRow_AVX2(const uint16_t* y_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
"xmm3", "xmm4", "xmm5"); "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_I210TOARGBROW_AVX2 #endif // defined(HAS_I210ALPHATOARGBROW_AVX2)
#if defined(HAS_I410ALPHATOARGBROW_AVX2) #if defined(HAS_I410ALPHATOARGBROW_AVX2)
// 16 pixels // 16 pixels
@ -4300,7 +4554,7 @@ void OMITFP I410AlphaToARGBRow_AVX2(const uint16_t* y_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
"xmm3", "xmm4", "xmm5"); "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_I410TOARGBROW_AVX2 #endif // defined(HAS_I410ALPHATOARGBROW_AVX2)
#if defined(HAS_I410TOAR30ROW_AVX2) #if defined(HAS_I410TOAR30ROW_AVX2)
// 16 pixels // 16 pixels
@ -4339,7 +4593,7 @@ void OMITFP I410ToAR30Row_AVX2(const uint16_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
); );
} }
#endif // HAS_I410TOAR30ROW_AVX2 #endif // defined(HAS_I410TOAR30ROW_AVX2)
#if defined(HAS_I444ALPHATOARGBROW_AVX2) #if defined(HAS_I444ALPHATOARGBROW_AVX2)
// 16 pixels // 16 pixels
@ -4373,7 +4627,7 @@ void OMITFP I444AlphaToARGBRow_AVX2(const uint8_t* y_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
"xmm3", "xmm4", "xmm5"); "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_I444ALPHATOARGBROW_AVX2 #endif // defined(HAS_I444ALPHATOARGBROW_AVX2)
#if defined(HAS_I422ALPHATOARGBROW_AVX2) #if defined(HAS_I422ALPHATOARGBROW_AVX2)
// 16 pixels // 16 pixels
@ -4407,7 +4661,7 @@ void OMITFP I422AlphaToARGBRow_AVX2(const uint8_t* y_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
"xmm3", "xmm4", "xmm5"); "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_I422ALPHATOARGBROW_AVX2 #endif // defined(HAS_I422ALPHATOARGBROW_AVX2)
#if defined(HAS_I422TORGBAROW_AVX2) #if defined(HAS_I422TORGBAROW_AVX2)
// 16 pixels // 16 pixels
@ -4451,7 +4705,7 @@ void OMITFP I422ToRGBARow_AVX2(const uint8_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
); );
} }
#endif // HAS_I422TORGBAROW_AVX2 #endif // defined(HAS_I422TORGBAROW_AVX2)
#if defined(HAS_NV12TOARGBROW_AVX2) #if defined(HAS_NV12TOARGBROW_AVX2)
// 16 pixels. // 16 pixels.
@ -4477,7 +4731,7 @@ void OMITFP NV12ToARGBRow_AVX2(const uint8_t* y_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
"xmm2", "xmm3", "xmm4", "xmm5"); "xmm2", "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_NV12TOARGBROW_AVX2 #endif // defined(HAS_NV12TOARGBROW_AVX2)
#if defined(HAS_NV21TOARGBROW_AVX2) #if defined(HAS_NV21TOARGBROW_AVX2)
// 16 pixels. // 16 pixels.
@ -4504,7 +4758,7 @@ void OMITFP NV21ToARGBRow_AVX2(const uint8_t* y_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
"xmm2", "xmm3", "xmm4", "xmm5"); "xmm2", "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_NV21TOARGBROW_AVX2 #endif // defined(HAS_NV21TOARGBROW_AVX2)
#if defined(HAS_YUY2TOARGBROW_AVX2) #if defined(HAS_YUY2TOARGBROW_AVX2)
// 16 pixels. // 16 pixels.
@ -4531,7 +4785,7 @@ void OMITFP YUY2ToARGBRow_AVX2(const uint8_t* yuy2_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1", "xmm2", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1", "xmm2",
"xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); "xmm3", "xmm4", "xmm5", "xmm6", "xmm7");
} }
#endif // HAS_YUY2TOARGBROW_AVX2 #endif // defined(HAS_YUY2TOARGBROW_AVX2)
#if defined(HAS_UYVYTOARGBROW_AVX2) #if defined(HAS_UYVYTOARGBROW_AVX2)
// 16 pixels. // 16 pixels.
@ -4558,7 +4812,7 @@ void OMITFP UYVYToARGBRow_AVX2(const uint8_t* uyvy_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3",
"xmm4", "xmm5", "xmm6", "xmm7"); "xmm4", "xmm5", "xmm6", "xmm7");
} }
#endif // HAS_UYVYTOARGBROW_AVX2 #endif // defined(HAS_UYVYTOARGBROW_AVX2)
#if defined(HAS_P210TOARGBROW_AVX2) #if defined(HAS_P210TOARGBROW_AVX2)
// 16 pixels. // 16 pixels.
@ -4584,7 +4838,7 @@ void OMITFP P210ToARGBRow_AVX2(const uint16_t* y_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
"xmm2", "xmm3", "xmm4", "xmm5"); "xmm2", "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_P210TOARGBROW_AVX2 #endif // defined(HAS_P210TOARGBROW_AVX2)
#if defined(HAS_P410TOARGBROW_AVX2) #if defined(HAS_P410TOARGBROW_AVX2)
// 16 pixels. // 16 pixels.
@ -4610,7 +4864,7 @@ void OMITFP P410ToARGBRow_AVX2(const uint16_t* y_buf,
: "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1", : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
"xmm2", "xmm3", "xmm4", "xmm5"); "xmm2", "xmm3", "xmm4", "xmm5");
} }
#endif // HAS_P410TOARGBROW_AVX2 #endif // defined(HAS_P410TOARGBROW_AVX2)
#if defined(HAS_P210TOAR30ROW_AVX2) #if defined(HAS_P210TOAR30ROW_AVX2)
// 16 pixels // 16 pixels
@ -4646,7 +4900,7 @@ void OMITFP P210ToAR30Row_AVX2(const uint16_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
); );
} }
#endif // HAS_P210TOAR30ROW_AVX2 #endif // defined(HAS_P210TOAR30ROW_AVX2)
#if defined(HAS_P410TOAR30ROW_AVX2) #if defined(HAS_P410TOAR30ROW_AVX2)
// 16 pixels // 16 pixels
@ -4682,7 +4936,7 @@ void OMITFP P410ToAR30Row_AVX2(const uint16_t* y_buf,
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
); );
} }
#endif // HAS_P410TOAR30ROW_AVX2 #endif // defined(HAS_P410TOAR30ROW_AVX2)
#ifdef HAS_I400TOARGBROW_SSE2 #ifdef HAS_I400TOARGBROW_SSE2
void I400ToARGBRow_SSE2(const uint8_t* y_buf, void I400ToARGBRow_SSE2(const uint8_t* y_buf,
@ -4846,7 +5100,8 @@ void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
// Shuffle table for reversing the bytes of UV channels. // Shuffle table for reversing the bytes of UV channels.
static const uvec8 kShuffleMirrorSplitUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u, static const uvec8 kShuffleMirrorSplitUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u,
15u, 13u, 11u, 9u, 7u, 5u, 3u, 1u}; 15u, 13u, 11u, 9u, 7u, 5u, 3u, 1u};
#endif #endif // defined(HAS_MIRRORSPLITUVROW_AVX2) ||
// defined(HAS_MIRRORSPLITUVROW_AVX512BW)
#ifdef HAS_MIRRORSPLITUVROW_AVX512BW #ifdef HAS_MIRRORSPLITUVROW_AVX512BW
static const uint64_t kMirrorSplitUVPermute[8] = {6, 4, 2, 0, 7, 5, 3, 1}; static const uint64_t kMirrorSplitUVPermute[8] = {6, 4, 2, 0, 7, 5, 3, 1};
@ -5269,7 +5524,7 @@ void DetileRow_16_SSE2(const uint16_t* src,
: "r"(src_tile_stride) // %3 : "r"(src_tile_stride) // %3
: "cc", "memory", "xmm0", "xmm1"); : "cc", "memory", "xmm0", "xmm1");
} }
#endif // HAS_DETILEROW_SSE2 #endif // HAS_DETILEROW_16_SSE2
#ifdef HAS_DETILEROW_16_AVX #ifdef HAS_DETILEROW_16_AVX
void DetileRow_16_AVX(const uint16_t* src, void DetileRow_16_AVX(const uint16_t* src,
@ -5291,7 +5546,7 @@ void DetileRow_16_AVX(const uint16_t* src,
: "r"(src_tile_stride) // %3 : "r"(src_tile_stride) // %3
: "cc", "memory", "xmm0"); : "cc", "memory", "xmm0");
} }
#endif // HAS_DETILEROW_AVX #endif // HAS_DETILEROW_16_AVX
#ifdef HAS_DETILETOYUY2_SSE2 #ifdef HAS_DETILETOYUY2_SSE2
// Read 16 Y, 8 UV, and write 8 YUYV. // Read 16 Y, 8 UV, and write 8 YUYV.
@ -5324,7 +5579,7 @@ void DetileToYUY2_SSE2(const uint8_t* src_y,
: "cc", "memory", "xmm0", "xmm1", "xmm2" // Clobber list : "cc", "memory", "xmm0", "xmm1", "xmm2" // Clobber list
); );
} }
#endif #endif // HAS_DETILETOYUY2_SSE2
#ifdef HAS_DETILESPLITUVROW_SSSE3 #ifdef HAS_DETILESPLITUVROW_SSSE3
// TODO(greenjustin): Look into generating these constants instead of loading // TODO(greenjustin): Look into generating these constants instead of loading
@ -5481,7 +5736,7 @@ void MergeUVRow_16_AVX2(const uint16_t* src_u,
"r"(32 - depth) // %5 "r"(32 - depth) // %5
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
} }
#endif // HAS_MERGEUVROW_AVX2 #endif // HAS_MERGEUVROW_16_AVX2
#ifdef HAS_SPLITUVROW_16_AVX2 #ifdef HAS_SPLITUVROW_16_AVX2
const uvec8 kSplitUVShuffle16 = {0, 1, 4, 5, 8, 9, 12, 13, const uvec8 kSplitUVShuffle16 = {0, 1, 4, 5, 8, 9, 12, 13,
@ -5599,7 +5854,7 @@ void DivideRow_16_AVX2(const uint16_t* src_y,
: :
: "memory", "cc", "xmm0", "xmm1", "xmm3"); : "memory", "cc", "xmm0", "xmm1", "xmm3");
} }
#endif // HAS_MULTIPLYROW_16_AVX2 #endif // HAS_DIVIDEROW_16_AVX2
// Use scale to convert lsb formats to msb, depending how many bits there are: // Use scale to convert lsb formats to msb, depending how many bits there are:
// 32768 = 9 bits // 32768 = 9 bits
@ -6341,7 +6596,7 @@ void MergeARGBRow_SSE2(const uint8_t* src_r,
: :
: "memory", "cc", "xmm0", "xmm1", "xmm2"); : "memory", "cc", "xmm0", "xmm1", "xmm2");
} }
#endif #endif // HAS_MERGEARGBROW_SSE2
#ifdef HAS_MERGEXRGBROW_SSE2 #ifdef HAS_MERGEXRGBROW_SSE2
void MergeXRGBRow_SSE2(const uint8_t* src_r, void MergeXRGBRow_SSE2(const uint8_t* src_r,
@ -6378,7 +6633,7 @@ void MergeXRGBRow_SSE2(const uint8_t* src_r,
: :
: "memory", "cc", "xmm0", "xmm1", "xmm2"); : "memory", "cc", "xmm0", "xmm1", "xmm2");
} }
#endif // HAS_MERGEARGBROW_SSE2 #endif // HAS_MERGEXRGBROW_SSE2
#ifdef HAS_MERGEARGBROW_AVX2 #ifdef HAS_MERGEARGBROW_AVX2
void MergeARGBRow_AVX2(const uint8_t* src_r, void MergeARGBRow_AVX2(const uint8_t* src_r,
@ -6424,7 +6679,7 @@ void MergeARGBRow_AVX2(const uint8_t* src_r,
: :
: "memory", "cc", "xmm0", "xmm1", "xmm2"); : "memory", "cc", "xmm0", "xmm1", "xmm2");
} }
#endif #endif // HAS_MERGEARGBROW_AVX2
#ifdef HAS_MERGEXRGBROW_AVX2 #ifdef HAS_MERGEXRGBROW_AVX2
void MergeXRGBRow_AVX2(const uint8_t* src_r, void MergeXRGBRow_AVX2(const uint8_t* src_r,
@ -6465,7 +6720,7 @@ void MergeXRGBRow_AVX2(const uint8_t* src_r,
::"memory", ::"memory",
"cc", "xmm0", "xmm1", "xmm2"); "cc", "xmm0", "xmm1", "xmm2");
} }
#endif // HAS_MERGEARGBROW_AVX2 #endif // HAS_MERGEXRGBROW_AVX2
#ifdef HAS_SPLITARGBROW_SSE2 #ifdef HAS_SPLITARGBROW_SSE2
void SplitARGBRow_SSE2(const uint8_t* src_argb, void SplitARGBRow_SSE2(const uint8_t* src_argb,
@ -6517,7 +6772,7 @@ void SplitARGBRow_SSE2(const uint8_t* src_argb,
: :
: "memory", "cc", "xmm0", "xmm1", "xmm2"); : "memory", "cc", "xmm0", "xmm1", "xmm2");
} }
#endif #endif // HAS_SPLITARGBROW_SSE2
#ifdef HAS_SPLITXRGBROW_SSE2 #ifdef HAS_SPLITXRGBROW_SSE2
void SplitXRGBRow_SSE2(const uint8_t* src_argb, void SplitXRGBRow_SSE2(const uint8_t* src_argb,
@ -6563,7 +6818,7 @@ void SplitXRGBRow_SSE2(const uint8_t* src_argb,
: :
: "memory", "cc", "xmm0", "xmm1", "xmm2"); : "memory", "cc", "xmm0", "xmm1", "xmm2");
} }
#endif #endif // HAS_SPLITXRGBROW_SSE2
static const uvec8 kShuffleMaskARGBSplit = {0, 4, 8, 12, 1, 5, 9, 13, static const uvec8 kShuffleMaskARGBSplit = {0, 4, 8, 12, 1, 5, 9, 13,
2, 6, 10, 14, 3, 7, 11, 15}; 2, 6, 10, 14, 3, 7, 11, 15};
@ -6612,7 +6867,7 @@ void SplitARGBRow_SSSE3(const uint8_t* src_argb,
: "m"(kShuffleMaskARGBSplit) // %6 : "m"(kShuffleMaskARGBSplit) // %6
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
} }
#endif #endif // HAS_SPLITARGBROW_SSSE3
#ifdef HAS_SPLITXRGBROW_SSSE3 #ifdef HAS_SPLITXRGBROW_SSSE3
void SplitXRGBRow_SSSE3(const uint8_t* src_argb, void SplitXRGBRow_SSSE3(const uint8_t* src_argb,
@ -6651,7 +6906,7 @@ void SplitXRGBRow_SSSE3(const uint8_t* src_argb,
: "m"(kShuffleMaskARGBSplit) // %5 : "m"(kShuffleMaskARGBSplit) // %5
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
} }
#endif #endif // HAS_SPLITXRGBROW_SSSE3
#ifdef HAS_SPLITARGBROW_AVX2 #ifdef HAS_SPLITARGBROW_AVX2
static const ulvec32 kShuffleMaskARGBPermute = {0, 4, 1, 5, 2, 6, 3, 7}; static const ulvec32 kShuffleMaskARGBPermute = {0, 4, 1, 5, 2, 6, 3, 7};
@ -6704,7 +6959,7 @@ void SplitARGBRow_AVX2(const uint8_t* src_argb,
"m"(kShuffleMaskARGBPermute) // %7 "m"(kShuffleMaskARGBPermute) // %7
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
} }
#endif #endif // HAS_SPLITARGBROW_AVX2
#ifdef HAS_SPLITXRGBROW_AVX2 #ifdef HAS_SPLITXRGBROW_AVX2
void SplitXRGBRow_AVX2(const uint8_t* src_argb, void SplitXRGBRow_AVX2(const uint8_t* src_argb,
@ -6749,7 +7004,7 @@ void SplitXRGBRow_AVX2(const uint8_t* src_argb,
"m"(kShuffleMaskARGBPermute) // %6 "m"(kShuffleMaskARGBPermute) // %6
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
} }
#endif #endif // HAS_SPLITXRGBROW_AVX2
#ifdef HAS_MERGEXR30ROW_AVX2 #ifdef HAS_MERGEXR30ROW_AVX2
void MergeXR30Row_AVX2(const uint16_t* src_r, void MergeXR30Row_AVX2(const uint16_t* src_r,
@ -6812,7 +7067,7 @@ void MergeXR30Row_AVX2(const uint16_t* src_r,
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6"); "xmm6");
} }
#endif #endif // HAS_MERGEXR30ROW_AVX2
#ifdef HAS_MERGEAR64ROW_AVX2 #ifdef HAS_MERGEAR64ROW_AVX2
static const lvec32 MergeAR64Permute = {0, 4, 2, 6, 1, 5, 3, 7}; static const lvec32 MergeAR64Permute = {0, 4, 2, 6, 1, 5, 3, 7};
@ -6885,7 +7140,7 @@ void MergeAR64Row_AVX2(const uint16_t* src_r,
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7"); "xmm7");
} }
#endif #endif // HAS_MERGEAR64ROW_AVX2
#ifdef HAS_MERGEXR64ROW_AVX2 #ifdef HAS_MERGEXR64ROW_AVX2
void MergeXR64Row_AVX2(const uint16_t* src_r, void MergeXR64Row_AVX2(const uint16_t* src_r,
@ -6947,7 +7202,7 @@ void MergeXR64Row_AVX2(const uint16_t* src_r,
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7"); "xmm7");
} }
#endif #endif // HAS_MERGEXR64ROW_AVX2
#ifdef HAS_MERGEARGB16TO8ROW_AVX2 #ifdef HAS_MERGEARGB16TO8ROW_AVX2
static const uvec8 MergeARGB16To8Shuffle = {0, 8, 1, 9, 2, 10, 3, 11, static const uvec8 MergeARGB16To8Shuffle = {0, 8, 1, 9, 2, 10, 3, 11,
@ -7006,7 +7261,7 @@ void MergeARGB16To8Row_AVX2(const uint16_t* src_r,
"m"(MergeARGB16To8Shuffle) // %7 "m"(MergeARGB16To8Shuffle) // %7
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
} }
#endif #endif // HAS_MERGEARGB16TO8ROW_AVX2
#ifdef HAS_MERGEXRGB16TO8ROW_AVX2 #ifdef HAS_MERGEXRGB16TO8ROW_AVX2
void MergeXRGB16To8Row_AVX2(const uint16_t* src_r, void MergeXRGB16To8Row_AVX2(const uint16_t* src_r,
@ -7056,7 +7311,7 @@ void MergeXRGB16To8Row_AVX2(const uint16_t* src_r,
"m"(MergeARGB16To8Shuffle) // %6 "m"(MergeARGB16To8Shuffle) // %6
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
} }
#endif #endif // HAS_MERGEXRGB16TO8ROW_AVX2
#ifdef HAS_COPYROW_SSE2 #ifdef HAS_COPYROW_SSE2
void CopyRow_SSE2(const uint8_t* src, uint8_t* dst, int width) { void CopyRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
@ -7138,7 +7393,7 @@ void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
: :
: "memory", "cc", "xmm0", "xmm1"); : "memory", "cc", "xmm0", "xmm1");
} }
#endif // HAS_COPYROW_AVX512 #endif // HAS_COPYROW_AVX512BW
#ifdef HAS_COPYROW_ERMS #ifdef HAS_COPYROW_ERMS
// Multiple of 1. // Multiple of 1.
@ -10339,7 +10594,9 @@ void Convert8To8Row_AVX2(const uint8_t* src_y,
} }
#endif // HAS_CONVERT16TO8ROW_AVX2 #endif // HAS_CONVERT16TO8ROW_AVX2
#endif // defined(__x86_64__) || defined(__i386__) #endif // !defined(LIBYUV_DISABLE_X86) &&
// (defined(__x86_64__) || defined(__i386__)) &&
// !defined(LIBYUV_ENABLE_ROWWIN)
#ifdef __cplusplus #ifdef __cplusplus
} // extern "C" } // extern "C"

357
source/row_win.cc
View File

@ -27,6 +27,40 @@ namespace libyuv {
extern "C" { extern "C" {
#endif #endif
#if defined(__clang__) || defined(__GNUC__)
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
#define LIBYUV_TARGET_AVX512BW \
__attribute__((target("avx512bw,avx512vl,avx512f")))
#define LIBYUV_TARGET_AVX512VBMI \
__attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f")))
#else
#define LIBYUV_TARGET_AVX2
#define LIBYUV_TARGET_AVX512BW
#define LIBYUV_TARGET_AVX512VBMI
#endif // defined(__clang__) || defined(__GNUC__)
// Read 8 UV from 444
#define READYUV444 \
xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \
xmm1 = _mm_loadl_epi64((__m128i*)(u_buf + offset)); \
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
u_buf += 8; \
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
y_buf += 8;
// Read 8 UV from 444, With 8 Alpha.
#define READYUVA444 \
xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \
xmm1 = _mm_loadl_epi64((__m128i*)(u_buf + offset)); \
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
u_buf += 8; \
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
y_buf += 8; \
xmm5 = _mm_loadl_epi64((__m128i*)a_buf); \
a_buf += 8;
// Read 4 UV from 422, upsample to 8 UV. // Read 4 UV from 422, upsample to 8 UV.
#define READYUV422 \ #define READYUV422 \
xmm3 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \ xmm3 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
@ -38,6 +72,19 @@ extern "C" {
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \ xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
y_buf += 8; y_buf += 8;
// Read 4 UV from 422, upsample to 8 UV. With 8 Alpha.
#define READYUVA422 \
xmm3 = _mm_cvtsi32_si128(*(uint32_t*)u_buf); \
xmm1 = _mm_cvtsi32_si128(*(uint32_t*)(u_buf + offset)); \
xmm3 = _mm_unpacklo_epi8(xmm3, xmm1); \
xmm3 = _mm_unpacklo_epi16(xmm3, xmm3); \
u_buf += 4; \
xmm4 = _mm_loadl_epi64((__m128i*)y_buf); \
xmm4 = _mm_unpacklo_epi8(xmm4, xmm4); \
y_buf += 8; \
xmm5 = _mm_loadl_epi64((__m128i*)a_buf); \
a_buf += 8;
// Convert 8 pixels: 8 UV and 8 Y. // Convert 8 pixels: 8 UV and 8 Y.
#define YUVTORGB(yuvconstants) \ #define YUVTORGB(yuvconstants) \
xmm3 = _mm_sub_epi8(xmm3, _mm_set1_epi8((char)0x80)); \ xmm3 = _mm_sub_epi8(xmm3, _mm_set1_epi8((char)0x80)); \
@ -56,6 +103,17 @@ extern "C" {
xmm1 = _mm_packus_epi16(xmm1, xmm1); \ xmm1 = _mm_packus_epi16(xmm1, xmm1); \
xmm2 = _mm_packus_epi16(xmm2, xmm2); xmm2 = _mm_packus_epi16(xmm2, xmm2);
// Store 8 ARGB values.
#define STOREARGB \
xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); \
xmm2 = _mm_unpacklo_epi8(xmm2, xmm5); \
xmm1 = _mm_loadu_si128(&xmm0); \
xmm0 = _mm_unpacklo_epi16(xmm0, xmm2); \
xmm1 = _mm_unpackhi_epi16(xmm1, xmm2); \
_mm_storeu_si128((__m128i*)dst_argb, xmm0); \
_mm_storeu_si128((__m128i*)(dst_argb + 16), xmm1); \
dst_argb += 32;
// Store 8 AR30 values. // Store 8 AR30 values.
#define STOREAR30 \ #define STOREAR30 \
xmm1 = _mm_and_si128(xmm1, xmm7); \ xmm1 = _mm_and_si128(xmm1, xmm7); \
@ -72,20 +130,6 @@ extern "C" {
_mm_storeu_si128((__m128i*)(dst_ar30 + 16), xmm1); \ _mm_storeu_si128((__m128i*)(dst_ar30 + 16), xmm1); \
dst_ar30 += 32; dst_ar30 += 32;
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
#if defined(__clang__) || defined(__GNUC__)
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
#define LIBYUV_TARGET_AVX512BW \
__attribute__((target("avx512bw,avx512vl,avx512f")))
#define LIBYUV_TARGET_AVX512VBMI \
__attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f")))
#else
#define LIBYUV_TARGET_AVX2
#define LIBYUV_TARGET_AVX512BW
#define LIBYUV_TARGET_AVX512VBMI
#endif
// Convert 32 ARGB pixels (128 bytes) to 32 UV444 values. // Convert 32 ARGB pixels (128 bytes) to 32 UV444 values.
#if defined(HAS_ARGBTOYMATRIXROW_AVX2) || defined(HAS_ARGBTOUV444MATRIXROW_AVX2) #if defined(HAS_ARGBTOYMATRIXROW_AVX2) || defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -150,7 +194,10 @@ void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // defined(HAS_ARGBTOYMATRIXROW_AVX2) ||
// defined(HAS_ARGBTOUV444MATRIXROW_AVX2)
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb, void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
uint8_t* dst_y, uint8_t* dst_y,
@ -201,6 +248,7 @@ void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif // defined(HAS_ARGBTOYMATRIXROW_AVX2)
#ifdef HAS_RAWTOARGBROW_AVX2 #ifdef HAS_RAWTOARGBROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -255,10 +303,11 @@ void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_RAWTOARGBROW_AVX2
#if defined(HAS_RAWTOARGBROW_AVX512BW) || \ #if defined(HAS_RAWTOARGBROW_AVX512BW) || \
defined(HAS_RGB24TOARGBROW_AVX512BW) || \ defined(HAS_RGB24TOARGBROW_AVX512BW) || \
defined(HAS_RAWTORGB24ROW_AVX512BW) || \
defined(HAS_RGBTOYMATRIXROW_AVX512BW) defined(HAS_RGBTOYMATRIXROW_AVX512BW)
static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2,
3, 4, 4, 5}; 3, 4, 4, 5};
@ -268,18 +317,80 @@ static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6,
7, 8, 8, 9}; 7, 8, 8, 9};
static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4,
5, 6, 6, 7}; 5, 6, 6, 7};
#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
// defined(HAS_RGB24TOARGBROW_AVX512BW) ||
// defined(HAS_RAWTORGB24ROW_AVX512BW) ||
// defined(HAS_RGBTOYMATRIXROW_AVX512BW)
#if defined(HAS_RGB24TOARGBROW_AVX512BW) || \
defined(HAS_RGBTOYMATRIXROW_AVX512BW)
static const uint8_t kShuffleMaskRGB24ToARGB_AVX512BW[32] = { static const uint8_t kShuffleMaskRGB24ToARGB_AVX512BW[32] = {
0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u, 0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u,
4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, 128u}; 4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, 128u};
#endif #endif // defined(HAS_RGB24TOARGBROW_AVX512BW) ||
// defined(HAS_RGBTOYMATRIXROW_AVX512BW)
#if defined(HAS_RAWTOARGBROW_AVX512BW) #if defined(HAS_RAWTOARGBROW_AVX512BW)
static const uint8_t kShuffleMaskRAWToARGB_AVX512BW[32] = { static const uint8_t kShuffleMaskRAWToARGB_AVX512BW[32] = {
2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u, 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u,
6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u}; 6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u};
#endif #endif // defined(HAS_RAWTOARGBROW_AVX512BW)
#ifdef HAS_RAWTOARGBROW_AVX512BW #if defined(HAS_RAWTORGB24ROW_AVX2) || defined(HAS_RAWTORGB24ROW_AVX512BW)
static const uint8_t kShuffleMaskRAWToRGB24_AVX2[32] = {
2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 6u, 11u, 10u, 9u, 128u, 128u, 128u, 128u,
6u, 5u, 4u, 9u, 8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u, 128u, 128u, 128u, 128u};
#endif // defined(HAS_RAWTORGB24ROW_AVX2) ||
// defined(HAS_RAWTORGB24ROW_AVX512BW)
#if defined(HAS_RAWTORGB24ROW_AVX2)
static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 0, 0};
static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 0, 0, 0, 0};
static const uint32_t kPermd2_AVX2[8] = {0, 0, 0, 0, 0, 1, 2, 4};
static const uint32_t kPermd3_AVX2[8] = {5, 6, 0, 0, 0, 0, 0, 0};
static const uint32_t kPermd4_AVX2[8] = {0, 0, 0, 1, 2, 4, 5, 6};
#endif // defined(HAS_RAWTORGB24ROW_AVX2)
#if defined(HAS_RAWTORGB24ROW_AVX512BW)
static const uint32_t kPermdRGB24_0_AVX512BW[16] = {
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20};
static const uint32_t kPermdRGB24_1_AVX512BW[16] = {
5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25};
static const uint32_t kPermdRGB24_2_AVX512BW[16] = {
10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30};
#endif // defined(HAS_RAWTORGB24ROW_AVX512BW)
#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
static const uint8_t kPermRAWToRGB24_0[64] = {
0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a, 0x09,
0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17, 0x16, 0x15,
0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e, 0x23, 0x22, 0x21,
0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b, 0x2a, 0x2f, 0x2e, 0x2d,
0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38, 0x37, 0x36, 0x3b, 0x3a, 0x39,
0x3e, 0x3d, 0x3c, 0x41};
static const uint8_t kPermRAWToRGB24_1[64] = {
0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08, 0x0d,
0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15, 0x14, 0x19,
0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22, 0x21, 0x20, 0x25,
0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29, 0x2e, 0x2d, 0x2c, 0x31,
0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36, 0x35, 0x3a, 0x39, 0x38, 0x3d,
0x3c, 0x3b, 0x00, 0x3f};
static const uint8_t kPermRAWToRGB24_2[64] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b,
0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23,
0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b,
0x3c, 0x3d, 0x40, 0x3f};
static const uint8_t kPermRAWToRGB24_3[64] = {
0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c, 0x4b,
0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53, 0x58, 0x57,
0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60, 0x5f, 0x64, 0x63,
0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d, 0x6c, 0x6b, 0x70, 0x6f,
0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74, 0x79, 0x78, 0x77, 0x7c, 0x7b,
0x7a, 0x7f, 0x7e, 0x7d};
#endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI)
#if defined(HAS_RAWTOARGBROW_AVX512BW) || defined(HAS_RGB24TOARGBROW_AVX512BW)
LIBYUV_TARGET_AVX512BW LIBYUV_TARGET_AVX512BW
static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_argb, uint8_t* dst_argb,
@ -328,7 +439,10 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
// defined(HAS_RGB24TOARGBROW_AVX512BW)
#ifdef HAS_RAWTOARGBROW_AVX512BW
LIBYUV_TARGET_AVX512BW LIBYUV_TARGET_AVX512BW
void RAWToARGBRow_AVX512BW(const uint8_t* src_raw, void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_argb, uint8_t* dst_argb,
@ -337,7 +451,9 @@ void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
src_raw, dst_argb, src_raw, dst_argb,
(const uint32_t*)kShuffleMaskRAWToARGB_AVX512BW, width); (const uint32_t*)kShuffleMaskRAWToARGB_AVX512BW, width);
} }
#endif // HAS_RAWTOARGBROW_AVX512BW
#ifdef HAS_RGB24TOARGBROW_AVX512BW
LIBYUV_TARGET_AVX512BW LIBYUV_TARGET_AVX512BW
void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
uint8_t* dst_argb, uint8_t* dst_argb,
@ -346,6 +462,147 @@ void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
src_rgb24, dst_argb, src_rgb24, dst_argb,
(const uint32_t*)kShuffleMaskRGB24ToARGB_AVX512BW, width); (const uint32_t*)kShuffleMaskRGB24ToARGB_AVX512BW, width);
} }
#endif // HAS_RGB24TOARGBROW_AVX512BW
#ifdef HAS_RAWTORGB24ROW_AVX2
LIBYUV_TARGET_AVX2
void RAWToRGB24Row_AVX2(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width) {
__m256i ymm_shuf =
_mm256_loadu_si256((const __m256i*)kShuffleMaskRAWToRGB24_AVX2);
__m256i ymm_p0 = _mm256_loadu_si256((const __m256i*)kPermd0_AVX2);
__m256i ymm_p1 = _mm256_loadu_si256((const __m256i*)kPermd1_AVX2);
__m256i ymm_p2 = _mm256_loadu_si256((const __m256i*)kPermd2_AVX2);
__m256i ymm_p3 = _mm256_loadu_si256((const __m256i*)kPermd3_AVX2);
__m256i ymm_p4 = _mm256_loadu_si256((const __m256i*)kPermd4_AVX2);
while (width > 0) {
__m256i raw0 = _mm256_loadu_si256((const __m256i*)src_raw);
__m256i raw1 = _mm256_loadu_si256((const __m256i*)(src_raw + 32));
__m256i raw2 = _mm256_loadu_si256((const __m256i*)(src_raw + 64));
__m256i b0 = _mm256_permute4x64_epi64(raw0, 0x94);
__m256i m01 = _mm256_permute2x128_si256(raw0, raw1, 0x21);
__m256i b1 = _mm256_permute4x64_epi64(m01, 0xe9);
__m256i m12 = _mm256_permute2x128_si256(raw1, raw2, 0x21);
__m256i b2 = _mm256_permute4x64_epi64(m12, 0x94);
__m256i b3 = _mm256_permute4x64_epi64(raw2, 0xe9);
b0 = _mm256_shuffle_epi8(b0, ymm_shuf);
b1 = _mm256_shuffle_epi8(b1, ymm_shuf);
b2 = _mm256_shuffle_epi8(b2, ymm_shuf);
b3 = _mm256_shuffle_epi8(b3, ymm_shuf);
// Dst0 = blend(b0_dwords, b1_q0)
__m256i d0 = _mm256_permutevar8x32_epi32(b0, ymm_p0);
__m256i b1_q0 = _mm256_permute4x64_epi64(b1, 0x00);
__m256i dst0 = _mm256_blend_epi32(d0, b1_q0, 0xc0);
// Dst1 = blend(b1_dwords, b2_dwords)
__m256i d1 = _mm256_permutevar8x32_epi32(b1, ymm_p1);
__m256i d2 = _mm256_permutevar8x32_epi32(b2, ymm_p2);
__m256i dst1 = _mm256_blend_epi32(d1, d2, 0xf0);
// Dst2 = blend(b2_dwords, b3_dwords)
__m256i d2_c45 = _mm256_permutevar8x32_epi32(b2, ymm_p3);
__m256i d3 = _mm256_permutevar8x32_epi32(b3, ymm_p4);
__m256i dst2 = _mm256_blend_epi32(d2_c45, d3, 0xfc);
_mm256_storeu_si256((__m256i*)dst_rgb24, dst0);
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 32), dst1);
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), dst2);
src_raw += 96;
dst_rgb24 += 96;
width -= 32;
}
_mm256_zeroupper();
}
#endif // HAS_RAWTORGB24ROW_AVX2
#ifdef HAS_RAWTORGB24ROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width) {
__m512i zmm_q0 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
__m512i zmm_q1 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
__m512i zmm_q2 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
__m512i zmm_q3 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
__m512i zmm_shuf = _mm512_broadcast_i64x4(
_mm256_loadu_si256((const __m256i*)kShuffleMaskRAWToRGB24_AVX2));
__m512i zmm_d0 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_0_AVX512BW);
__m512i zmm_d1 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_1_AVX512BW);
__m512i zmm_d2 = _mm512_loadu_si512((const __m512i*)kPermdRGB24_2_AVX512BW);
while (width > 0) {
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
__m512i dst0 = _mm512_permutex2var_epi32(zmm0, zmm_d0, zmm1);
__m512i dst1 = _mm512_permutex2var_epi32(zmm1, zmm_d1, zmm2);
__m512i dst2 = _mm512_permutex2var_epi32(zmm2, zmm_d2, zmm3);
_mm512_storeu_si512((__m512i*)dst_rgb24, dst0);
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), dst1);
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), dst2);
src_raw += 192;
dst_rgb24 += 192;
width -= 64;
}
_mm256_zeroupper();
}
#endif // HAS_RAWTORGB24ROW_AVX512BW
#ifdef HAS_RAWTORGB24ROW_AVX512VBMI
LIBYUV_TARGET_AVX512VBMI
void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width) {
__m512i zmm_p0 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_0);
__m512i zmm_p1 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_1);
__m512i zmm_p2 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_2);
__m512i zmm_p3 = _mm512_loadu_si512((const __m512i*)kPermRAWToRGB24_3);
while (width > 0) {
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
__m512i tmp1 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p1, zmm_raw0);
__m512i dst0 = _mm512_permutex2var_epi8(zmm_raw0, zmm_p0, zmm_raw1);
__m512i dst2 = _mm512_permutex2var_epi8(zmm_raw1, zmm_p3, zmm_raw2);
__m512i dst1 = _mm512_permutex2var_epi8(tmp1, zmm_p2, zmm_raw2);
_mm512_storeu_si512((__m512i*)dst_rgb24, dst0);
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 64), dst1);
_mm512_storeu_si512((__m512i*)(dst_rgb24 + 128), dst2);
src_raw += 192;
dst_rgb24 += 192;
width -= 64;
}
_mm256_zeroupper();
}
#endif // HAS_RAWTORGB24ROW_AVX512VBMI
#ifdef HAS_RGBTOYMATRIXROW_AVX512BW #ifdef HAS_RGBTOYMATRIXROW_AVX512BW
static const uint32_t kPermdARGBToY_AVX512BW[16] = { static const uint32_t kPermdARGBToY_AVX512BW[16] = {
@ -433,8 +690,7 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_RGBTOYMATRIXROW_AVX512BW
#endif
#ifdef HAS_ARGBTOUVMATRIXROW_AVX2 #ifdef HAS_ARGBTOUVMATRIXROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -504,7 +760,7 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_ARGBTOUVMATRIXROW_AVX2
#ifdef HAS_MERGEUVROW_AVX2 #ifdef HAS_MERGEUVROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -528,7 +784,7 @@ void MergeUVRow_AVX2(const uint8_t* src_u,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_MERGEUVROW_AVX2
#ifdef HAS_MIRRORROW_AVX2 #ifdef HAS_MIRRORROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -547,7 +803,7 @@ void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_MIRRORROW_AVX2
#ifdef HAS_MIRRORUVROW_AVX2 #ifdef HAS_MIRRORUVROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -566,7 +822,7 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_MIRRORUVROW_AVX2
#ifdef HAS_MIRRORSPLITUVROW_AVX2 #ifdef HAS_MIRRORSPLITUVROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -590,7 +846,7 @@ void MirrorSplitUVRow_AVX2(const uint8_t* src_uv,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_MIRRORSPLITUVROW_AVX2
#ifdef HAS_RGB24MIRRORROW_AVX2 #ifdef HAS_RGB24MIRRORROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -644,7 +900,7 @@ void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_RGB24MIRRORROW_AVX2
#ifdef HAS_INTERPOLATEROW_AVX2 #ifdef HAS_INTERPOLATEROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -691,7 +947,7 @@ void InterpolateRow_AVX2(uint8_t* dst_ptr,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_INTERPOLATEROW_AVX2
#ifdef HAS_INTERPOLATEROW_16_AVX2 #ifdef HAS_INTERPOLATEROW_16_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -740,7 +996,7 @@ void InterpolateRow_16_AVX2(uint16_t* dst_ptr,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_INTERPOLATEROW_16_AVX2
#ifdef HAS_ARGBMIRRORROW_AVX2 #ifdef HAS_ARGBMIRRORROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -757,7 +1013,7 @@ void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_ARGBMIRRORROW_AVX2
#ifdef HAS_J400TOARGBROW_AVX2 #ifdef HAS_J400TOARGBROW_AVX2
static const uint8_t kShuffleMaskJ400ToARGB_0[32] = { static const uint8_t kShuffleMaskJ400ToARGB_0[32] = {
@ -802,7 +1058,7 @@ static const uint8_t kShuffleMaskRGB24ToARGB[2][16] = {
{0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u}, {0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u},
{4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, {4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u,
128u}}; 128u}};
#endif #endif // HAS_RGB24TOARGBROW_AVX2
#ifdef HAS_RGB565TOARGBROW_AVX2 #ifdef HAS_RGB565TOARGBROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -846,7 +1102,7 @@ void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_RGB565TOARGBROW_AVX2
#ifdef HAS_ARGB1555TOARGBROW_AVX2 #ifdef HAS_ARGB1555TOARGBROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -894,7 +1150,7 @@ void ARGB1555ToARGBRow_AVX2(const uint8_t* src_argb1555,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_ARGB1555TOARGBROW_AVX2
#ifdef HAS_ARGB4444TOARGBROW_AVX2 #ifdef HAS_ARGB4444TOARGBROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -936,7 +1192,7 @@ void ARGB4444ToARGBRow_AVX2(const uint8_t* src_argb4444,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_ARGB4444TOARGBROW_AVX2
#ifdef HAS_ARGBTORGB565DITHERROW_AVX2 #ifdef HAS_ARGBTORGB565DITHERROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -972,7 +1228,7 @@ void ARGBToRGB565DitherRow_AVX2(const uint8_t* src_argb,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_ARGBTORGB565DITHERROW_AVX2
#ifdef HAS_ARGBTORGB565ROW_AVX2 #ifdef HAS_ARGBTORGB565ROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -1002,7 +1258,7 @@ void ARGBToRGB565Row_AVX2(const uint8_t* src_argb,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_ARGBTORGB565ROW_AVX2
#ifdef HAS_ARGBTOARGB1555ROW_AVX2 #ifdef HAS_ARGBTOARGB1555ROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -1036,7 +1292,7 @@ void ARGBToARGB1555Row_AVX2(const uint8_t* src_argb,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_ARGBTOARGB1555ROW_AVX2
#ifdef HAS_ARGBTOARGB4444ROW_AVX2 #ifdef HAS_ARGBTOARGB4444ROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -1070,7 +1326,7 @@ void ARGBToARGB4444Row_AVX2(const uint8_t* src_argb,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_ARGBTOARGB4444ROW_AVX2
#ifdef HAS_RGB24TOARGBROW_AVX2 #ifdef HAS_RGB24TOARGBROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -1125,7 +1381,7 @@ void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_RGB24TOARGBROW_AVX2
#ifdef HAS_ARGBSHUFFLEROW_AVX2 #ifdef HAS_ARGBSHUFFLEROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -1148,7 +1404,7 @@ void ARGBShuffleRow_AVX2(const uint8_t* src_argb,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_ARGBSHUFFLEROW_AVX2
#ifdef HAS_ARGBSHUFFLEROW_AVX512BW #ifdef HAS_ARGBSHUFFLEROW_AVX512BW
LIBYUV_TARGET_AVX512BW LIBYUV_TARGET_AVX512BW
@ -1170,9 +1426,7 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
width -= 32; width -= 32;
} }
} }
#endif #endif // HAS_ARGBSHUFFLEROW_AVX512BW
#endif
#ifdef HAS_I422TOARGBROW_AVX2 #ifdef HAS_I422TOARGBROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -1248,7 +1502,7 @@ void I422ToARGBRow_AVX2(const uint8_t* src_y,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_I422TOARGBROW_AVX2
#ifdef HAS_I422TOAR30ROW_AVX2 #ifdef HAS_I422TOAR30ROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -1337,7 +1591,7 @@ void I422ToAR30Row_AVX2(const uint8_t* src_y,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_I422TOAR30ROW_AVX2
#ifdef HAS_NV12TOARGBROW_AVX2 #ifdef HAS_NV12TOARGBROW_AVX2
LIBYUV_TARGET_AVX2 LIBYUV_TARGET_AVX2
@ -1406,7 +1660,7 @@ void NV12ToARGBRow_AVX2(const uint8_t* src_y,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_NV12TOARGBROW_AVX2
#ifdef HAS_NV21TOARGBROW_AVX2 #ifdef HAS_NV21TOARGBROW_AVX2
static const uint8_t kShuffleNV21_AVX2[32] = { static const uint8_t kShuffleNV21_AVX2[32] = {
@ -1481,7 +1735,7 @@ void NV21ToARGBRow_AVX2(const uint8_t* src_y,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_NV21TOARGBROW_AVX2
#if defined(HAS_I422TORGB24ROW_AVX2) || defined(HAS_I422TORGB24ROW_AVX512BW) #if defined(HAS_I422TORGB24ROW_AVX2) || defined(HAS_I422TORGB24ROW_AVX512BW)
static const uint8_t kShuffleMaskARGBToRGB24[2][16] = { static const uint8_t kShuffleMaskARGBToRGB24[2][16] = {
@ -1584,7 +1838,8 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // defined(HAS_I422TORGB24ROW_AVX2) ||
// defined(HAS_I422TORGB24ROW_AVX512BW)
#ifdef HAS_I422TORGB24ROW_AVX512VBMI #ifdef HAS_I422TORGB24ROW_AVX512VBMI
LIBYUV_TARGET_AVX512VBMI LIBYUV_TARGET_AVX512VBMI
@ -1694,7 +1949,7 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_I422TORGB24ROW_AVX512VBMI
#ifdef HAS_I422TORGB24ROW_AVX512BW #ifdef HAS_I422TORGB24ROW_AVX512BW
LIBYUV_TARGET_AVX512BW LIBYUV_TARGET_AVX512BW
@ -1793,7 +2048,7 @@ void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_I422TORGB24ROW_AVX512BW
#ifdef HAS_I422TOAR30ROW_AVX512BW #ifdef HAS_I422TOAR30ROW_AVX512BW
LIBYUV_TARGET_AVX512BW LIBYUV_TARGET_AVX512BW
@ -1901,7 +2156,7 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
} }
_mm256_zeroupper(); _mm256_zeroupper();
} }
#endif #endif // HAS_I422TOAR30ROW_AVX512BW
#ifdef __cplusplus #ifdef __cplusplus
} // extern "C" } // extern "C"

1
unit_test/convert_argb_test.cc
View File

@ -643,6 +643,7 @@ TESTATOB(J400, uint8_t, 1, 1, 1, J400, uint8_t, 1, 1, 1)
TESTATOB(RAW, uint8_t, 3, 3, 1, ARGB, uint8_t, 4, 4, 1) TESTATOB(RAW, uint8_t, 3, 3, 1, ARGB, uint8_t, 4, 4, 1)
TESTATOB(RAW, uint8_t, 3, 3, 1, RGBA, uint8_t, 4, 4, 1) TESTATOB(RAW, uint8_t, 3, 3, 1, RGBA, uint8_t, 4, 4, 1)
TESTATOB(RAW, uint8_t, 3, 3, 1, RGB24, uint8_t, 3, 3, 1) TESTATOB(RAW, uint8_t, 3, 3, 1, RGB24, uint8_t, 3, 3, 1)
TESTATOB(RGB24, uint8_t, 3, 3, 1, RAW, uint8_t, 3, 3, 1)
TESTATOB(RGB24, uint8_t, 3, 3, 1, ARGB, uint8_t, 4, 4, 1) TESTATOB(RGB24, uint8_t, 3, 3, 1, ARGB, uint8_t, 4, 4, 1)
TESTATOB(RGB24, uint8_t, 3, 3, 1, J400, uint8_t, 1, 1, 1) TESTATOB(RGB24, uint8_t, 3, 3, 1, J400, uint8_t, 1, 1, 1)
TESTATOB(RGB24, uint8_t, 3, 3, 1, RGB24Mirror, uint8_t, 3, 3, 1) TESTATOB(RGB24, uint8_t, 3, 3, 1, RGB24Mirror, uint8_t, 3, 3, 1)