mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
I422ToRGB24 avx512bw tail and row coalesce
I422ToRGB24Matrix (299x299, repeat=10000) ==================================================================== CPU Before After Improvement -------------------------------------------------------------------- Intel Skylake-X 275 ms 213 ms -22.5% (1.29x) Intel Ice Lake 225 ms 177 ms -21.3% (1.27x) Intel Sapphire Rapids 181 ms 144 ms -20.4% (1.26x) Intel Emerald Rapids 167 ms 136 ms -18.6% (1.23x) Intel Granite Rapids 170 ms 140 ms -17.6% (1.21x) AMD Zen 4 (Genoa) 192 ms 136 ms -29.2% (1.41x) AMD Zen 5 (Turin) 102 ms 68 ms -33.3% (1.50x) AMD Zen 6 (Venice) 102 ms 60 ms -41.2% (1.70x) ==================================================================== I422ToRGB24Matrix (298x299, repeat=10000) ==================================================================== CPU Before After Improvement -------------------------------------------------------------------- Intel Skylake-X 271 ms 184 ms -32.1% (1.47x) Intel Ice Lake 235 ms 152 ms -35.3% (1.55x) Intel Sapphire Rapids 199 ms 128 ms -35.7% (1.55x) Intel Emerald Rapids 170 ms 121 ms -28.8% (1.40x) Intel Granite Rapids 177 ms 125 ms -29.4% (1.42x) AMD Zen 4 (Genoa) 185 ms 119 ms -35.7% (1.55x) AMD Zen 5 (Turin) 102 ms 59 ms -42.2% (1.73x) AMD Zen 6 (Venice) 100 ms 51 ms -49.0% (1.96x) ==================================================================== Bug: None Change-Id: Id75e3d1df6d26bc28a252672816a7c438cb664a4 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8452502 Reviewed-by: richard winterton <rrwinterton@gmail.com> Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard
libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent
41a6e684a6
commit
2124173f34
@ -375,6 +375,7 @@ extern "C" {
|
||||
#define HAS_RGB24TOARGBROW_AVX512BW
|
||||
#define HAS_RAWTORGB24ROW_AVX512BW
|
||||
#define HAS_ARGBSHUFFLEROW_AVX512BW
|
||||
#define HAS_I422TORGB24ROW_AVX512VBMI
|
||||
#define HAS_RAWTORGB24ROW_AVX512VBMI
|
||||
#define HAS_I422TORGB24ROW_AVX512BW
|
||||
#define HAS_I422TOAR30ROW_AVX512BW
|
||||
@ -417,7 +418,6 @@ extern "C" {
|
||||
#define HAS_RAWTORGB24ROW_AVX512BW
|
||||
#define HAS_RAWTORGB24ROW_AVX512VBMI
|
||||
#define HAS_ARGBTORGB24ROW_AVX512VBMI
|
||||
#define HAS_I422TORGB24ROW_AVX512VBMI
|
||||
#define HAS_CONVERT16TO8ROW_AVX512BW
|
||||
#define HAS_CONVERT8TO16ROW_AVX512BW
|
||||
#define HAS_HALFROW_16TO8_AVX512BW
|
||||
@ -436,6 +436,7 @@ extern "C" {
|
||||
#define HAS_BLENDPLANEROW_AVX512BW
|
||||
#define HAS_RGBTOUV444MATRIXROW_AVX512BW
|
||||
#define HAS_ARGBTOYMATRIXROW_AVX512BW
|
||||
#define HAS_I422TORGB24ROW_AVX512VBMI
|
||||
#define HAS_RAWTORGB24ROW_AVX512VBMI
|
||||
#define HAS_I422TORGB24ROW_AVX512BW
|
||||
#define HAS_ARGBTOUVMATRIXROW_AVX512BW
|
||||
@ -5274,12 +5275,6 @@ void NV12ToRGB565Row_Any_AVX2(const uint8_t* y_buf,
|
||||
uint8_t* dst_ptr,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width);
|
||||
void I422ToRGB24Row_Any_AVX512BW(const uint8_t* y_buf,
|
||||
const uint8_t* u_buf,
|
||||
const uint8_t* v_buf,
|
||||
uint8_t* dst_ptr,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width);
|
||||
|
||||
void I400ToARGBRow_C(const uint8_t* src_y,
|
||||
uint8_t* rgb_buf,
|
||||
|
||||
@ -1717,6 +1717,14 @@ int I210ToARGBMatrix(const uint16_t* src_y,
|
||||
dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
|
||||
dst_stride_argb = -dst_stride_argb;
|
||||
}
|
||||
// Coalesce rows.
|
||||
if (src_stride_y == width && src_stride_u * 2 == width &&
|
||||
src_stride_v * 2 == width && dst_stride_argb == width * 4 &&
|
||||
(ptrdiff_t)width * height <= INT_MAX) {
|
||||
width *= height;
|
||||
height = 1;
|
||||
src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_I210TOARGBROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I210ToARGBRow = I210ToARGBRow_Any_SSSE3;
|
||||
@ -1897,6 +1905,13 @@ int I410ToARGBMatrix(const uint16_t* src_y,
|
||||
dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
|
||||
dst_stride_argb = -dst_stride_argb;
|
||||
}
|
||||
// Coalesce rows.
|
||||
if (src_stride_y == width && src_stride_u == width && src_stride_v == width &&
|
||||
dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
|
||||
width *= height;
|
||||
height = 1;
|
||||
src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_I410TOARGBROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I410ToARGBRow = I410ToARGBRow_Any_SSSE3;
|
||||
@ -2541,6 +2556,15 @@ int I422AlphaToARGBMatrix(const uint8_t* src_y,
|
||||
dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
|
||||
dst_stride_argb = -dst_stride_argb;
|
||||
}
|
||||
// Coalesce rows.
|
||||
if (src_stride_y == width && src_stride_u * 2 == width &&
|
||||
src_stride_v * 2 == width && src_stride_a == width &&
|
||||
dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
|
||||
width *= height;
|
||||
height = 1;
|
||||
src_stride_y = src_stride_u = src_stride_v = src_stride_a =
|
||||
dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_I422ALPHATOARGBROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I422AlphaToARGBRow = I422AlphaToARGBRow_Any_SSSE3;
|
||||
@ -2692,6 +2716,15 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y,
|
||||
dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
|
||||
dst_stride_argb = -dst_stride_argb;
|
||||
}
|
||||
// Coalesce rows.
|
||||
if (src_stride_y == width && src_stride_u == width && src_stride_v == width &&
|
||||
src_stride_a == width && dst_stride_argb == width * 4 &&
|
||||
(ptrdiff_t)width * height <= INT_MAX) {
|
||||
width *= height;
|
||||
height = 1;
|
||||
src_stride_y = src_stride_u = src_stride_v = src_stride_a =
|
||||
dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_I444ALPHATOARGBROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_SSSE3;
|
||||
@ -5320,6 +5353,14 @@ int I422ToRGBAMatrix(const uint8_t* src_y,
|
||||
dst_rgba = dst_rgba + (ptrdiff_t)(height - 1) * dst_stride_rgba;
|
||||
dst_stride_rgba = -dst_stride_rgba;
|
||||
}
|
||||
// Coalesce rows.
|
||||
if (src_stride_y == width && src_stride_u * 2 == width &&
|
||||
src_stride_v * 2 == width && dst_stride_rgba == width * 4 &&
|
||||
(ptrdiff_t)width * height <= INT_MAX) {
|
||||
width *= height;
|
||||
height = 1;
|
||||
src_stride_y = src_stride_u = src_stride_v = dst_stride_rgba = 0;
|
||||
}
|
||||
#if defined(HAS_I422TORGBAROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I422ToRGBARow = I422ToRGBARow_Any_SSSE3;
|
||||
@ -5695,10 +5736,7 @@ int I420ToRGB24Matrix(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
|
||||
}
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
|
||||
@ -5895,6 +5933,14 @@ int I422ToRGB24Matrix(const uint8_t* src_y,
|
||||
dst_rgb24 = dst_rgb24 + (ptrdiff_t)(height - 1) * dst_stride_rgb24;
|
||||
dst_stride_rgb24 = -dst_stride_rgb24;
|
||||
}
|
||||
// Coalesce rows.
|
||||
if (src_stride_y == width && src_stride_u * 2 == width &&
|
||||
src_stride_v * 2 == width && dst_stride_rgb24 == width * 3 &&
|
||||
(ptrdiff_t)width * height <= INT_MAX) {
|
||||
width *= height;
|
||||
height = 1;
|
||||
src_stride_y = src_stride_u = src_stride_v = dst_stride_rgb24 = 0;
|
||||
}
|
||||
#if defined(HAS_I422TORGB24ROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3;
|
||||
@ -5913,10 +5959,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
|
||||
}
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
|
||||
@ -6364,6 +6407,14 @@ int I422ToRGB565Matrix(const uint8_t* src_y,
|
||||
dst_rgb565 = dst_rgb565 + (ptrdiff_t)(height - 1) * dst_stride_rgb565;
|
||||
dst_stride_rgb565 = -dst_stride_rgb565;
|
||||
}
|
||||
// Coalesce rows.
|
||||
if (src_stride_y == width && src_stride_u * 2 == width &&
|
||||
src_stride_v * 2 == width && dst_stride_rgb565 == width * 2 &&
|
||||
(ptrdiff_t)width * height <= INT_MAX) {
|
||||
width *= height;
|
||||
height = 1;
|
||||
src_stride_y = src_stride_u = src_stride_v = dst_stride_rgb565 = 0;
|
||||
}
|
||||
#if defined(HAS_I422TORGB565ROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I422ToRGB565Row = I422ToRGB565Row_Any_SSSE3;
|
||||
|
||||
@ -420,9 +420,6 @@ ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 15)
|
||||
#ifdef HAS_I422TORGB24ROW_AVX2
|
||||
ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15)
|
||||
#endif
|
||||
#ifdef HAS_I422TORGB24ROW_AVX512BW
|
||||
ANY31C(I422ToRGB24Row_Any_AVX512BW, I422ToRGB24Row_AVX512BW, 1, 0, 3, 31)
|
||||
#endif
|
||||
#ifdef HAS_I422TOARGBROW_AVX2
|
||||
ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15)
|
||||
#endif
|
||||
|
||||
@ -3506,6 +3506,27 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
|
||||
"vpord %%zmm3,%%zmm4,%%zmm4 \n" \
|
||||
"vmovdqa64 %%zmm1,%%zmm3 \n"
|
||||
|
||||
#define READYUV422_MASK_AVX512BW \
|
||||
"mov $-1,%[temp] \n" \
|
||||
"bzhi %q[width],%[temp],%[temp] \n" \
|
||||
"kmovd %k[temp],%%k1 \n" \
|
||||
"vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \
|
||||
"mov $-1,%[temp] \n" \
|
||||
"lea 1(%q[width]),%k[y_buf] \n" \
|
||||
"shr $1,%k[y_buf] \n" \
|
||||
"bzhi %q[y_buf],%[temp],%[y_buf] \n" \
|
||||
"kmovw %k[y_buf],%%k2 \n" \
|
||||
"vmovdqu8 (%[u_buf]),%%xmm3%{%%k2%}%{z%} \n" \
|
||||
"vmovdqu8 0x00(%[u_buf],%[v_buf],1),%%xmm1%{%%k2%}%{z%} \n" \
|
||||
"vpunpcklbw %%xmm1,%%xmm3,%%xmm2 \n" \
|
||||
"vpunpckhbw %%xmm1,%%xmm3,%%xmm3 \n" \
|
||||
"vinserti64x2 $1,%%xmm3,%%ymm2,%%ymm3 \n" \
|
||||
"vmovdqa64 %%zmm16,%%zmm1 \n" \
|
||||
"vpermi2w %%zmm3,%%zmm3,%%zmm1 \n" \
|
||||
"vpsllw $8,%%zmm4,%%zmm3 \n" \
|
||||
"vpord %%zmm3,%%zmm4,%%zmm4 \n" \
|
||||
"vmovdqa64 %%zmm1,%%zmm3 \n"
|
||||
|
||||
// Read 8 UV from 210, upsample to 16 UV
|
||||
// TODO(fbarchard): Consider vpshufb to replace pack/unpack
|
||||
// TODO(fbarchard): Consider vunpcklpd to combine the 2 registers into 1.
|
||||
@ -3854,6 +3875,23 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
|
||||
"vmovdqu8 %%ymm0,0x40(%[dst_rgb24]) \n" \
|
||||
"lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n"
|
||||
|
||||
// Store masked RGB24 values with VBMI.
|
||||
#define STORERGB24_MASK_AVX512VBMI \
|
||||
"lea (%q[width],%q[width],2),%k[width] \n" \
|
||||
"bzhi %q[width],%[temp],%[y_buf] \n" \
|
||||
"kmovq %[y_buf],%%k3 \n" \
|
||||
"xor %k[y_buf],%k[y_buf] \n" \
|
||||
"sub $0x40,%k[width] \n" \
|
||||
"cmovg %[temp],%[y_buf] \n" \
|
||||
"bzhi %q[width],%[y_buf],%[y_buf] \n" \
|
||||
"kmovd %k[y_buf],%%k4 \n" \
|
||||
"vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" \
|
||||
"vmovdqa64 %%zmm0,%%zmm3 \n" \
|
||||
"vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" \
|
||||
"vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" \
|
||||
"vmovdqu8 %%zmm3,(%[dst_rgb24])%{%%k3%} \n" \
|
||||
"vmovdqu8 %%ymm0,0x40(%[dst_rgb24])%{%%k4%} \n"
|
||||
|
||||
// Store 32 RGB24 values with AVX512BW.
|
||||
#define STORERGB24_AVX512BW \
|
||||
"vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \
|
||||
@ -3872,6 +3910,31 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
|
||||
"vmovdqu8 %%ymm4,0x40(%[dst_rgb24]) \n" \
|
||||
"lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n"
|
||||
|
||||
// Store masked RGB24 values with AVX512BW.
|
||||
#define STORERGB24_MASK_AVX512BW \
|
||||
"lea (%q[width],%q[width],2),%k[width] \n" \
|
||||
"bzhi %q[width],%[temp],%[y_buf] \n" \
|
||||
"kmovq %[y_buf],%%k3 \n" \
|
||||
"xor %k[y_buf],%k[y_buf] \n" \
|
||||
"sub $0x40,%k[width] \n" \
|
||||
"cmovg %[temp],%[y_buf] \n" \
|
||||
"bzhi %q[width],%[y_buf],%[y_buf] \n" \
|
||||
"kmovd %k[y_buf],%%k4 \n" \
|
||||
"vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \
|
||||
"vpunpcklbw %%zmm2,%%zmm2,%%zmm2 \n" \
|
||||
"vmovdqa64 %%zmm0,%%zmm1 \n" \
|
||||
"vpunpcklwd %%zmm2,%%zmm0,%%zmm0 \n" \
|
||||
"vpunpckhwd %%zmm2,%%zmm1,%%zmm1 \n" \
|
||||
"vpshufb %%zmm5,%%zmm0,%%zmm0 \n" \
|
||||
"vpshufb %%zmm6,%%zmm1,%%zmm1 \n" \
|
||||
"vpalignr $0xc,%%zmm0,%%zmm1,%%zmm1 \n" \
|
||||
"vmovdqa64 %%zmm20,%%zmm3 \n" \
|
||||
"vpermi2q %%zmm1,%%zmm0,%%zmm3 \n" \
|
||||
"vmovdqa64 %%zmm21,%%zmm4 \n" \
|
||||
"vpermi2q %%zmm1,%%zmm0,%%zmm4 \n" \
|
||||
"vmovdqu8 %%zmm3,(%[dst_rgb24])%{%%k3%} \n" \
|
||||
"vmovdqu8 %%ymm4,0x40(%[dst_rgb24])%{%%k4%} \n"
|
||||
|
||||
// Store 32 AR30 values.
|
||||
#define STOREAR30_AVX512BW \
|
||||
"vpminsw %%zmm7,%%zmm0,%%zmm0 \n" \
|
||||
@ -4142,42 +4205,9 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
|
||||
"add $0x20,%[width] \n"
|
||||
"je 99f \n"
|
||||
|
||||
// Calculate a mask for the final iteration to deal with the tail.
|
||||
"mov $-1,%[temp] \n"
|
||||
"bzhi %q[width],%[temp],%[temp] \n"
|
||||
"kmovd %k[temp],%%k1 \n"
|
||||
"vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%}\n"
|
||||
"mov $-1,%[temp] \n"
|
||||
"lea 1(%q[width]),%k[y_buf] \n"
|
||||
"shr $1,%k[y_buf] \n"
|
||||
"bzhi %q[y_buf],%[temp],%[y_buf] \n"
|
||||
"kmovw %k[y_buf],%%k2 \n"
|
||||
"vmovdqu8 (%[u_buf]),%%xmm3%{%%k2%}%{z%}\n"
|
||||
"vmovdqu8 0x00(%[u_buf],%[v_buf],1),%%xmm1%{%%k2%}%{z%}\n"
|
||||
"lea (%q[width],%q[width],2),%k[width]\n"
|
||||
"bzhi %q[width],%[temp],%[y_buf] \n"
|
||||
"kmovq %[y_buf],%%k3 \n"
|
||||
"xor %k[y_buf],%k[y_buf] \n"
|
||||
"sub $0x40,%k[width] \n"
|
||||
"cmovg %[temp],%[y_buf] \n"
|
||||
"bzhi %q[width],%[y_buf],%[y_buf] \n"
|
||||
"kmovd %k[y_buf],%%k4 \n"
|
||||
|
||||
"vpunpcklbw %%xmm1,%%xmm3,%%xmm2 \n"
|
||||
"vpunpckhbw %%xmm1,%%xmm3,%%xmm3 \n"
|
||||
"vinserti64x2 $1,%%xmm3,%%ymm2,%%ymm3 \n"
|
||||
"vmovdqa64 %%zmm16,%%zmm1 \n"
|
||||
"vpermi2w %%zmm3,%%zmm3,%%zmm1 \n"
|
||||
"vpsllw $8,%%zmm4,%%zmm3 \n"
|
||||
"vpord %%zmm3,%%zmm4,%%zmm4 \n"
|
||||
"vmovdqa64 %%zmm1,%%zmm3 \n"
|
||||
READYUV422_MASK_AVX512BW
|
||||
YUVTORGB_AVX512BW(yuvconstants)
|
||||
"vpermt2b %%zmm1,%%zmm20,%%zmm0 \n"
|
||||
"vmovdqa64 %%zmm0,%%zmm3 \n"
|
||||
"vpermt2b %%zmm2,%%zmm21,%%zmm3 \n"
|
||||
"vpermt2b %%zmm2,%%zmm22,%%zmm0 \n"
|
||||
"vmovdqu8 %%zmm3,(%[dst_rgb24])%{%%k3%} \n"
|
||||
"vmovdqu8 %%ymm0,0x40(%[dst_rgb24])%{%%k4%}\n"
|
||||
STORERGB24_MASK_AVX512VBMI
|
||||
|
||||
"99: \n"
|
||||
"vzeroupper \n"
|
||||
@ -4215,6 +4245,7 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
uintptr_t temp;
|
||||
asm volatile (
|
||||
YUVTORGB_SETUP_AVX512BW(yuvconstants)
|
||||
"vbroadcasti32x4 %[kShuffleMaskARGBToRGB24_1],%%zmm5 \n"
|
||||
@ -4222,6 +4253,8 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf,
|
||||
"vmovdqu64 %[kStitchRGB24_0],%%zmm20 \n"
|
||||
"vmovdqu64 %[kStitchRGB24_1],%%zmm21 \n"
|
||||
"sub %[u_buf],%[v_buf] \n"
|
||||
"sub $0x20,%[width] \n"
|
||||
"jl 2f \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
@ -4229,13 +4262,24 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf,
|
||||
YUVTORGB_AVX512BW(yuvconstants)
|
||||
STORERGB24_AVX512BW
|
||||
"sub $0x20,%[width] \n"
|
||||
"jg 1b \n"
|
||||
"jge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"add $0x20,%[width] \n"
|
||||
"je 99f \n"
|
||||
|
||||
READYUV422_MASK_AVX512BW
|
||||
YUVTORGB_AVX512BW(yuvconstants)
|
||||
STORERGB24_MASK_AVX512BW
|
||||
|
||||
"99: \n"
|
||||
"vzeroupper \n"
|
||||
: [y_buf]"+r"(y_buf), // %[y_buf]
|
||||
[u_buf]"+r"(u_buf), // %[u_buf]
|
||||
[v_buf]"+r"(v_buf), // %[v_buf]
|
||||
[dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
|
||||
[width]"+rm"(width) // %[width]
|
||||
[width]"+r"(width), // %[width]
|
||||
[temp]"=&r"(temp) // %[temp]
|
||||
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
|
||||
[quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
|
||||
[dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm]
|
||||
@ -4246,7 +4290,8 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf,
|
||||
[kStitchRGB24_1]"m"(kStitchRGB24_1) // %[kStitchRGB24_1]
|
||||
: "memory", "cc", YUVTORGB_REGS_AVX512BW
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||
"xmm20", "xmm21"
|
||||
"xmm20", "xmm21",
|
||||
"k1", "k2", "k3", "k4"
|
||||
);
|
||||
}
|
||||
#endif // defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
|
||||
@ -1907,6 +1907,119 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y,
|
||||
#endif // defined(HAS_I422TORGB24ROW_AVX2) ||
|
||||
// defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \
|
||||
defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
#define READYUV422_AVX512BW \
|
||||
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u); \
|
||||
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset)); \
|
||||
src_u += 16; \
|
||||
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \
|
||||
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \
|
||||
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \
|
||||
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \
|
||||
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \
|
||||
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \
|
||||
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \
|
||||
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); \
|
||||
src_y += 32; \
|
||||
__m512i zmm4 = _mm512_castsi256_si512(ymm_y); \
|
||||
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \
|
||||
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \
|
||||
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
||||
|
||||
#define READYUV422_MASK_AVX512BW \
|
||||
__mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); \
|
||||
__mmask32 mask_y = (__mmask32)((1u << width) - 1u); \
|
||||
__m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u); \
|
||||
__m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); \
|
||||
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \
|
||||
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \
|
||||
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \
|
||||
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \
|
||||
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \
|
||||
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \
|
||||
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \
|
||||
__m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); \
|
||||
__m512i zmm4 = _mm512_castsi256_si512(ymm_y); \
|
||||
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \
|
||||
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \
|
||||
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
||||
|
||||
#define YUVTORGB_AVX512BW \
|
||||
zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \
|
||||
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); \
|
||||
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); \
|
||||
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); \
|
||||
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); \
|
||||
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); \
|
||||
zmm0 = _mm512_adds_epi16(zmm0, zmm4); \
|
||||
zmm1 = _mm512_subs_epi16(zmm4, zmm1); \
|
||||
zmm2 = _mm512_adds_epi16(zmm2, zmm4); \
|
||||
zmm0 = _mm512_srai_epi16(zmm0, 6); \
|
||||
zmm1 = _mm512_srai_epi16(zmm1, 6); \
|
||||
zmm2 = _mm512_srai_epi16(zmm2, 6); \
|
||||
zmm0 = _mm512_packus_epi16(zmm0, zmm0); \
|
||||
zmm1 = _mm512_packus_epi16(zmm1, zmm1); \
|
||||
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
||||
|
||||
#define STORERGB24_AVX512VBMI \
|
||||
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \
|
||||
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \
|
||||
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \
|
||||
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \
|
||||
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \
|
||||
_mm512_castsi512_si256(zmm_dst1)); \
|
||||
dst_rgb24 += 96;
|
||||
|
||||
#define STORERGB24_MASK_AVX512VBMI \
|
||||
int dst_bytes = width * 3; \
|
||||
__mmask64 mask_dst0 = \
|
||||
dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \
|
||||
__mmask32 mask_dst1 = \
|
||||
dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \
|
||||
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \
|
||||
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \
|
||||
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \
|
||||
_mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \
|
||||
_mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \
|
||||
_mm512_castsi512_si256(zmm_dst1));
|
||||
|
||||
#define STORERGB24_AVX512BW \
|
||||
__m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \
|
||||
__m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \
|
||||
__m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \
|
||||
__m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \
|
||||
zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \
|
||||
zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \
|
||||
zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \
|
||||
__m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \
|
||||
__m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \
|
||||
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \
|
||||
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \
|
||||
_mm512_castsi512_si256(zmm_dst1)); \
|
||||
dst_rgb24 += 96;
|
||||
|
||||
#define STORERGB24_MASK_AVX512BW \
|
||||
int dst_bytes = width * 3; \
|
||||
__mmask64 mask_dst0 = \
|
||||
dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \
|
||||
__mmask32 mask_dst1 = \
|
||||
dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \
|
||||
__m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \
|
||||
__m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \
|
||||
__m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \
|
||||
__m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \
|
||||
zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \
|
||||
zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \
|
||||
zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \
|
||||
__m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \
|
||||
__m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \
|
||||
_mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \
|
||||
_mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \
|
||||
_mm512_castsi512_si256(zmm_dst1));
|
||||
#endif // defined(HAS_I422TORGB24ROW_AVX512VBMI) ||
|
||||
// defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
|
||||
#ifdef HAS_I422TORGB24ROW_AVX512VBMI
|
||||
LIBYUV_TARGET_AVX512VBMI
|
||||
void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
|
||||
@ -1958,61 +2071,16 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
|
||||
ptrdiff_t offset = src_v - src_u;
|
||||
|
||||
while (width >= 32) {
|
||||
// READYUV422_AVX512BW
|
||||
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
|
||||
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
|
||||
src_u += 16;
|
||||
|
||||
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
|
||||
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
|
||||
|
||||
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
|
||||
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
|
||||
|
||||
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
|
||||
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
|
||||
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
|
||||
|
||||
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
|
||||
src_y += 32;
|
||||
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
|
||||
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
|
||||
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
|
||||
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
||||
|
||||
// YUVTORGB_AVX512BW
|
||||
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
||||
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
|
||||
|
||||
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
|
||||
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
|
||||
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
|
||||
|
||||
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
|
||||
|
||||
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
|
||||
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
|
||||
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
|
||||
|
||||
zmm0 = _mm512_srai_epi16(zmm0, 6);
|
||||
zmm1 = _mm512_srai_epi16(zmm1, 6);
|
||||
zmm2 = _mm512_srai_epi16(zmm2, 6);
|
||||
|
||||
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
||||
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
||||
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
||||
|
||||
// STORERGB24_AVX512VBMI
|
||||
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
|
||||
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
|
||||
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
|
||||
|
||||
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
|
||||
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1));
|
||||
|
||||
dst_rgb24 += 96;
|
||||
READYUV422_AVX512BW
|
||||
YUVTORGB_AVX512BW
|
||||
STORERGB24_AVX512VBMI
|
||||
width -= 32;
|
||||
}
|
||||
if (width > 0) {
|
||||
READYUV422_MASK_AVX512BW
|
||||
YUVTORGB_AVX512BW
|
||||
STORERGB24_MASK_AVX512VBMI
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // HAS_I422TORGB24ROW_AVX512VBMI
|
||||
@ -2050,72 +2118,31 @@ void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
|
||||
ptrdiff_t offset = src_v - src_u;
|
||||
|
||||
while (width >= 32) {
|
||||
// READYUV422_AVX512BW
|
||||
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
|
||||
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
|
||||
src_u += 16;
|
||||
|
||||
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
|
||||
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
|
||||
|
||||
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
|
||||
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
|
||||
|
||||
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
|
||||
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
|
||||
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
|
||||
|
||||
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
|
||||
src_y += 32;
|
||||
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
|
||||
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
|
||||
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
|
||||
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
||||
|
||||
// YUVTORGB_AVX512BW
|
||||
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
||||
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
|
||||
|
||||
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
|
||||
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
|
||||
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
|
||||
|
||||
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
|
||||
|
||||
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
|
||||
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
|
||||
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
|
||||
|
||||
zmm0 = _mm512_srai_epi16(zmm0, 6);
|
||||
zmm1 = _mm512_srai_epi16(zmm1, 6);
|
||||
zmm2 = _mm512_srai_epi16(zmm2, 6);
|
||||
|
||||
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
||||
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
||||
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
||||
|
||||
// STORERGB24_AVX512BW
|
||||
__m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1);
|
||||
__m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2);
|
||||
__m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr);
|
||||
__m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr);
|
||||
zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0);
|
||||
zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1);
|
||||
zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12);
|
||||
|
||||
__m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi);
|
||||
__m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi);
|
||||
|
||||
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
|
||||
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1));
|
||||
|
||||
dst_rgb24 += 96;
|
||||
READYUV422_AVX512BW
|
||||
YUVTORGB_AVX512BW
|
||||
STORERGB24_AVX512BW
|
||||
width -= 32;
|
||||
}
|
||||
if (width > 0) {
|
||||
READYUV422_MASK_AVX512BW
|
||||
YUVTORGB_AVX512BW
|
||||
STORERGB24_MASK_AVX512BW
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // HAS_I422TORGB24ROW_AVX512BW
|
||||
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \
|
||||
defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
#undef READYUV422_AVX512BW
|
||||
#undef READYUV422_MASK_AVX512BW
|
||||
#undef YUVTORGB_AVX512BW
|
||||
#undef STORERGB24_AVX512VBMI
|
||||
#undef STORERGB24_MASK_AVX512VBMI
|
||||
#undef STORERGB24_AVX512BW
|
||||
#undef STORERGB24_MASK_AVX512BW
|
||||
#endif
|
||||
|
||||
#ifdef HAS_I422TOAR30ROW_AVX512BW
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
|
||||
|
||||
@ -331,8 +331,8 @@ TESTPLANARTOB(V422, 2, 1, ARGB, 4, 4, 1)
|
||||
TESTPLANARTOB(V422, 2, 1, ABGR, 4, 4, 1)
|
||||
TESTPLANARTOB(I422, 2, 1, BGRA, 4, 4, 1)
|
||||
TESTPLANARTOB(I422, 2, 1, RGBA, 4, 4, 1)
|
||||
TESTPLANARTOB(I422, 1, 1, RGB24, 3, 3, 1)
|
||||
TESTPLANARTOB(I422, 1, 1, RAW, 3, 3, 1)
|
||||
TESTPLANARTOB(I422, 2, 1, RGB24, 3, 3, 1)
|
||||
TESTPLANARTOB(I422, 2, 1, RAW, 3, 3, 1)
|
||||
TESTPLANARTOB(I444, 1, 1, ARGB, 4, 4, 1)
|
||||
TESTPLANARTOB(I444, 1, 1, ABGR, 4, 4, 1)
|
||||
TESTPLANARTOB(I444, 1, 1, RGB24, 3, 3, 1)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user