diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 12d31fede..4c8b005d7 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -375,6 +375,7 @@ extern "C" { #define HAS_RGB24TOARGBROW_AVX512BW #define HAS_RAWTORGB24ROW_AVX512BW #define HAS_ARGBSHUFFLEROW_AVX512BW +#define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_RAWTORGB24ROW_AVX512VBMI #define HAS_I422TORGB24ROW_AVX512BW #define HAS_I422TOAR30ROW_AVX512BW @@ -417,7 +418,6 @@ extern "C" { #define HAS_RAWTORGB24ROW_AVX512BW #define HAS_RAWTORGB24ROW_AVX512VBMI #define HAS_ARGBTORGB24ROW_AVX512VBMI -#define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_CONVERT16TO8ROW_AVX512BW #define HAS_CONVERT8TO16ROW_AVX512BW #define HAS_HALFROW_16TO8_AVX512BW @@ -436,6 +436,7 @@ extern "C" { #define HAS_BLENDPLANEROW_AVX512BW #define HAS_RGBTOUV444MATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW +#define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_RAWTORGB24ROW_AVX512VBMI #define HAS_I422TORGB24ROW_AVX512BW #define HAS_ARGBTOUVMATRIXROW_AVX512BW @@ -5274,12 +5275,6 @@ void NV12ToRGB565Row_Any_AVX2(const uint8_t* y_buf, uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, int width); -void I422ToRGB24Row_Any_AVX512BW(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - const struct YuvConstants* yuvconstants, - int width); void I400ToARGBRow_C(const uint8_t* src_y, uint8_t* rgb_buf, diff --git a/source/convert_argb.cc b/source/convert_argb.cc index d31c78ff7..172b1f6c2 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -1717,6 +1717,14 @@ int I210ToARGBMatrix(const uint16_t* src_y, dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb; dst_stride_argb = -dst_stride_argb; } + // Coalesce rows. + if (src_stride_y == width && src_stride_u * 2 == width && + src_stride_v * 2 == width && dst_stride_argb == width * 4 && + (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0; + } #if defined(HAS_I210TOARGBROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I210ToARGBRow = I210ToARGBRow_Any_SSSE3; @@ -1897,6 +1905,13 @@ int I410ToARGBMatrix(const uint16_t* src_y, dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb; dst_stride_argb = -dst_stride_argb; } + // Coalesce rows. + if (src_stride_y == width && src_stride_u == width && src_stride_v == width && + dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_argb = 0; + } #if defined(HAS_I410TOARGBROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I410ToARGBRow = I410ToARGBRow_Any_SSSE3; @@ -2541,6 +2556,15 @@ int I422AlphaToARGBMatrix(const uint8_t* src_y, dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb; dst_stride_argb = -dst_stride_argb; } + // Coalesce rows. + if (src_stride_y == width && src_stride_u * 2 == width && + src_stride_v * 2 == width && src_stride_a == width && + dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = src_stride_a = + dst_stride_argb = 0; + } #if defined(HAS_I422ALPHATOARGBROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I422AlphaToARGBRow = I422AlphaToARGBRow_Any_SSSE3; @@ -2692,6 +2716,15 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y, dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb; dst_stride_argb = -dst_stride_argb; } + // Coalesce rows. + if (src_stride_y == width && src_stride_u == width && src_stride_v == width && + src_stride_a == width && dst_stride_argb == width * 4 && + (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = src_stride_a = + dst_stride_argb = 0; + } #if defined(HAS_I444ALPHATOARGBROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I444AlphaToARGBRow = I444AlphaToARGBRow_Any_SSSE3; @@ -5320,6 +5353,14 @@ int I422ToRGBAMatrix(const uint8_t* src_y, dst_rgba = dst_rgba + (ptrdiff_t)(height - 1) * dst_stride_rgba; dst_stride_rgba = -dst_stride_rgba; } + // Coalesce rows. + if (src_stride_y == width && src_stride_u * 2 == width && + src_stride_v * 2 == width && dst_stride_rgba == width * 4 && + (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_rgba = 0; + } #if defined(HAS_I422TORGBAROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I422ToRGBARow = I422ToRGBARow_Any_SSSE3; @@ -5695,10 +5736,7 @@ int I420ToRGB24Matrix(const uint8_t* src_y, #endif #if defined(HAS_I422TORGB24ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I422ToRGB24Row = I422ToRGB24Row_AVX512BW; - } + I422ToRGB24Row = I422ToRGB24Row_AVX512BW; } #endif #if defined(HAS_I422TORGB24ROW_AVX512VBMI) @@ -5895,6 +5933,14 @@ int I422ToRGB24Matrix(const uint8_t* src_y, dst_rgb24 = dst_rgb24 + (ptrdiff_t)(height - 1) * dst_stride_rgb24; dst_stride_rgb24 = -dst_stride_rgb24; } + // Coalesce rows. + if (src_stride_y == width && src_stride_u * 2 == width && + src_stride_v * 2 == width && dst_stride_rgb24 == width * 3 && + (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_rgb24 = 0; + } #if defined(HAS_I422TORGB24ROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3; @@ -5913,10 +5959,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y, #endif #if defined(HAS_I422TORGB24ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I422ToRGB24Row = I422ToRGB24Row_AVX512BW; - } + I422ToRGB24Row = I422ToRGB24Row_AVX512BW; } #endif #if defined(HAS_I422TORGB24ROW_AVX512VBMI) @@ -6364,6 +6407,14 @@ int I422ToRGB565Matrix(const uint8_t* src_y, dst_rgb565 = dst_rgb565 + (ptrdiff_t)(height - 1) * dst_stride_rgb565; dst_stride_rgb565 = -dst_stride_rgb565; } + // Coalesce rows. + if (src_stride_y == width && src_stride_u * 2 == width && + src_stride_v * 2 == width && dst_stride_rgb565 == width * 2 && + (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_y = src_stride_u = src_stride_v = dst_stride_rgb565 = 0; + } #if defined(HAS_I422TORGB565ROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { I422ToRGB565Row = I422ToRGB565Row_Any_SSSE3; diff --git a/source/row_any.cc b/source/row_any.cc index 14ab2f2d4..c3320a5e1 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -420,9 +420,6 @@ ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 15) #ifdef HAS_I422TORGB24ROW_AVX2 ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15) #endif -#ifdef HAS_I422TORGB24ROW_AVX512BW -ANY31C(I422ToRGB24Row_Any_AVX512BW, I422ToRGB24Row_AVX512BW, 1, 0, 3, 31) -#endif #ifdef HAS_I422TOARGBROW_AVX2 ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15) #endif diff --git a/source/row_gcc.cc b/source/row_gcc.cc index c02f124f6..4b19e8353 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -3506,6 +3506,27 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpord %%zmm3,%%zmm4,%%zmm4 \n" \ "vmovdqa64 %%zmm1,%%zmm3 \n" +#define READYUV422_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "mov $-1,%[temp] \n" \ + "lea 1(%q[width]),%k[y_buf] \n" \ + "shr $1,%k[y_buf] \n" \ + "bzhi %q[y_buf],%[temp],%[y_buf] \n" \ + "kmovw %k[y_buf],%%k2 \n" \ + "vmovdqu8 (%[u_buf]),%%xmm3%{%%k2%}%{z%} \n" \ + "vmovdqu8 0x00(%[u_buf],%[v_buf],1),%%xmm1%{%%k2%}%{z%} \n" \ + "vpunpcklbw %%xmm1,%%xmm3,%%xmm2 \n" \ + "vpunpckhbw %%xmm1,%%xmm3,%%xmm3 \n" \ + "vinserti64x2 $1,%%xmm3,%%ymm2,%%ymm3 \n" \ + "vmovdqa64 %%zmm16,%%zmm1 \n" \ + "vpermi2w %%zmm3,%%zmm3,%%zmm1 \n" \ + "vpsllw $8,%%zmm4,%%zmm3 \n" \ + "vpord %%zmm3,%%zmm4,%%zmm4 \n" \ + "vmovdqa64 %%zmm1,%%zmm3 \n" + // Read 8 UV from 210, upsample to 16 UV // TODO(fbarchard): Consider vpshufb to replace pack/unpack // TODO(fbarchard): Consider vunpcklpd to combine the 2 registers into 1. @@ -3854,6 +3875,23 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vmovdqu8 %%ymm0,0x40(%[dst_rgb24]) \n" \ "lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n" +// Store masked RGB24 values with VBMI. +#define STORERGB24_MASK_AVX512VBMI \ + "lea (%q[width],%q[width],2),%k[width] \n" \ + "bzhi %q[width],%[temp],%[y_buf] \n" \ + "kmovq %[y_buf],%%k3 \n" \ + "xor %k[y_buf],%k[y_buf] \n" \ + "sub $0x40,%k[width] \n" \ + "cmovg %[temp],%[y_buf] \n" \ + "bzhi %q[width],%[y_buf],%[y_buf] \n" \ + "kmovd %k[y_buf],%%k4 \n" \ + "vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" \ + "vmovdqa64 %%zmm0,%%zmm3 \n" \ + "vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" \ + "vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" \ + "vmovdqu8 %%zmm3,(%[dst_rgb24])%{%%k3%} \n" \ + "vmovdqu8 %%ymm0,0x40(%[dst_rgb24])%{%%k4%} \n" + // Store 32 RGB24 values with AVX512BW. #define STORERGB24_AVX512BW \ "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \ @@ -3872,6 +3910,31 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vmovdqu8 %%ymm4,0x40(%[dst_rgb24]) \n" \ "lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n" +// Store masked RGB24 values with AVX512BW. +#define STORERGB24_MASK_AVX512BW \ + "lea (%q[width],%q[width],2),%k[width] \n" \ + "bzhi %q[width],%[temp],%[y_buf] \n" \ + "kmovq %[y_buf],%%k3 \n" \ + "xor %k[y_buf],%k[y_buf] \n" \ + "sub $0x40,%k[width] \n" \ + "cmovg %[temp],%[y_buf] \n" \ + "bzhi %q[width],%[y_buf],%[y_buf] \n" \ + "kmovd %k[y_buf],%%k4 \n" \ + "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \ + "vpunpcklbw %%zmm2,%%zmm2,%%zmm2 \n" \ + "vmovdqa64 %%zmm0,%%zmm1 \n" \ + "vpunpcklwd %%zmm2,%%zmm0,%%zmm0 \n" \ + "vpunpckhwd %%zmm2,%%zmm1,%%zmm1 \n" \ + "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" \ + "vpshufb %%zmm6,%%zmm1,%%zmm1 \n" \ + "vpalignr $0xc,%%zmm0,%%zmm1,%%zmm1 \n" \ + "vmovdqa64 %%zmm20,%%zmm3 \n" \ + "vpermi2q %%zmm1,%%zmm0,%%zmm3 \n" \ + "vmovdqa64 %%zmm21,%%zmm4 \n" \ + "vpermi2q %%zmm1,%%zmm0,%%zmm4 \n" \ + "vmovdqu8 %%zmm3,(%[dst_rgb24])%{%%k3%} \n" \ + "vmovdqu8 %%ymm4,0x40(%[dst_rgb24])%{%%k4%} \n" + // Store 32 AR30 values. #define STOREAR30_AVX512BW \ "vpminsw %%zmm7,%%zmm0,%%zmm0 \n" \ @@ -4142,42 +4205,9 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, "add $0x20,%[width] \n" "je 99f \n" - // Calculate a mask for the final iteration to deal with the tail. - "mov $-1,%[temp] \n" - "bzhi %q[width],%[temp],%[temp] \n" - "kmovd %k[temp],%%k1 \n" - "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%}\n" - "mov $-1,%[temp] \n" - "lea 1(%q[width]),%k[y_buf] \n" - "shr $1,%k[y_buf] \n" - "bzhi %q[y_buf],%[temp],%[y_buf] \n" - "kmovw %k[y_buf],%%k2 \n" - "vmovdqu8 (%[u_buf]),%%xmm3%{%%k2%}%{z%}\n" - "vmovdqu8 0x00(%[u_buf],%[v_buf],1),%%xmm1%{%%k2%}%{z%}\n" - "lea (%q[width],%q[width],2),%k[width]\n" - "bzhi %q[width],%[temp],%[y_buf] \n" - "kmovq %[y_buf],%%k3 \n" - "xor %k[y_buf],%k[y_buf] \n" - "sub $0x40,%k[width] \n" - "cmovg %[temp],%[y_buf] \n" - "bzhi %q[width],%[y_buf],%[y_buf] \n" - "kmovd %k[y_buf],%%k4 \n" - - "vpunpcklbw %%xmm1,%%xmm3,%%xmm2 \n" - "vpunpckhbw %%xmm1,%%xmm3,%%xmm3 \n" - "vinserti64x2 $1,%%xmm3,%%ymm2,%%ymm3 \n" - "vmovdqa64 %%zmm16,%%zmm1 \n" - "vpermi2w %%zmm3,%%zmm3,%%zmm1 \n" - "vpsllw $8,%%zmm4,%%zmm3 \n" - "vpord %%zmm3,%%zmm4,%%zmm4 \n" - "vmovdqa64 %%zmm1,%%zmm3 \n" + READYUV422_MASK_AVX512BW YUVTORGB_AVX512BW(yuvconstants) - "vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" - "vmovdqa64 %%zmm0,%%zmm3 \n" - "vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" - "vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" - "vmovdqu8 %%zmm3,(%[dst_rgb24])%{%%k3%} \n" - "vmovdqu8 %%ymm0,0x40(%[dst_rgb24])%{%%k4%}\n" + STORERGB24_MASK_AVX512VBMI "99: \n" "vzeroupper \n" @@ -4215,6 +4245,7 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { + uintptr_t temp; asm volatile ( YUVTORGB_SETUP_AVX512BW(yuvconstants) "vbroadcasti32x4 %[kShuffleMaskARGBToRGB24_1],%%zmm5 \n" @@ -4222,6 +4253,8 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf, "vmovdqu64 %[kStitchRGB24_0],%%zmm20 \n" "vmovdqu64 %[kStitchRGB24_1],%%zmm21 \n" "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" @@ -4229,13 +4262,24 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf, YUVTORGB_AVX512BW(yuvconstants) STORERGB24_AVX512BW "sub $0x20,%[width] \n" - "jg 1b \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV422_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_MASK_AVX512BW + + "99: \n" "vzeroupper \n" : [y_buf]"+r"(y_buf), // %[y_buf] [u_buf]"+r"(u_buf), // %[u_buf] [v_buf]"+r"(v_buf), // %[v_buf] [dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24] - [width]"+rm"(width) // %[width] + [width]"+r"(width), // %[width] + [temp]"=&r"(temp) // %[temp] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm] [dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm] @@ -4246,7 +4290,8 @@ void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf, [kStitchRGB24_1]"m"(kStitchRGB24_1) // %[kStitchRGB24_1] : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm20", "xmm21" + "xmm20", "xmm21", + "k1", "k2", "k3", "k4" ); } #endif // defined(HAS_I422TORGB24ROW_AVX512BW) diff --git a/source/row_win.cc b/source/row_win.cc index 6831137d5..71bfc78aa 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -1907,6 +1907,119 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y, #endif // defined(HAS_I422TORGB24ROW_AVX2) || // defined(HAS_I422TORGB24ROW_AVX512BW) +#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I422TORGB24ROW_AVX512BW) +#define READYUV422_AVX512BW \ + __m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u); \ + __m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset)); \ + src_u += 16; \ + __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \ + __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \ + zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \ + zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \ + __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \ + zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \ + zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \ + __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); \ + src_y += 32; \ + __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \ + zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \ + zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \ + zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); + +#define READYUV422_MASK_AVX512BW \ + __mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); \ + __mmask32 mask_y = (__mmask32)((1u << width) - 1u); \ + __m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u); \ + __m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); \ + __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \ + __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \ + zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \ + zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \ + __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \ + zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \ + zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \ + __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); \ + __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \ + zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \ + zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \ + zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); + +#define YUVTORGB_AVX512BW \ + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \ + zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); \ + __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); \ + __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); \ + __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); \ + zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); \ + zmm0 = _mm512_adds_epi16(zmm0, zmm4); \ + zmm1 = _mm512_subs_epi16(zmm4, zmm1); \ + zmm2 = _mm512_adds_epi16(zmm2, zmm4); \ + zmm0 = _mm512_srai_epi16(zmm0, 6); \ + zmm1 = _mm512_srai_epi16(zmm1, 6); \ + zmm2 = _mm512_srai_epi16(zmm2, 6); \ + zmm0 = _mm512_packus_epi16(zmm0, zmm0); \ + zmm1 = _mm512_packus_epi16(zmm1, zmm1); \ + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + +#define STORERGB24_AVX512VBMI \ + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \ + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \ + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \ + _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \ + _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \ + _mm512_castsi512_si256(zmm_dst1)); \ + dst_rgb24 += 96; + +#define STORERGB24_MASK_AVX512VBMI \ + int dst_bytes = width * 3; \ + __mmask64 mask_dst0 = \ + dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \ + __mmask32 mask_dst1 = \ + dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \ + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \ + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \ + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \ + _mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \ + _mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \ + _mm512_castsi512_si256(zmm_dst1)); + +#define STORERGB24_AVX512BW \ + __m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \ + __m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \ + __m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \ + __m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \ + zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \ + zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \ + zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \ + __m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \ + __m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \ + _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \ + _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \ + _mm512_castsi512_si256(zmm_dst1)); \ + dst_rgb24 += 96; + +#define STORERGB24_MASK_AVX512BW \ + int dst_bytes = width * 3; \ + __mmask64 mask_dst0 = \ + dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \ + __mmask32 mask_dst1 = \ + dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \ + __m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \ + __m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \ + __m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \ + __m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \ + zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \ + zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \ + zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \ + __m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \ + __m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \ + _mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \ + _mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \ + _mm512_castsi512_si256(zmm_dst1)); +#endif // defined(HAS_I422TORGB24ROW_AVX512VBMI) || + // defined(HAS_I422TORGB24ROW_AVX512BW) + #ifdef HAS_I422TORGB24ROW_AVX512VBMI LIBYUV_TARGET_AVX512VBMI void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y, @@ -1958,61 +2071,16 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y, ptrdiff_t offset = src_v - src_u; while (width >= 32) { - // READYUV422_AVX512BW - __m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u); - __m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset)); - src_u += 16; - - __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); - __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); - - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); - - __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); - zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); - zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); - - __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); - src_y += 32; - __m512i zmm4 = _mm512_castsi256_si512(ymm_y); - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); - zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); - zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); - - // YUVTORGB_AVX512BW - zmm3 = _mm512_sub_epi8(zmm3, zmm_128); - zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); - - __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); - __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); - __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); - - zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); - - zmm0 = _mm512_adds_epi16(zmm0, zmm4); - zmm1 = _mm512_subs_epi16(zmm4, zmm1); - zmm2 = _mm512_adds_epi16(zmm2, zmm4); - - zmm0 = _mm512_srai_epi16(zmm0, 6); - zmm1 = _mm512_srai_epi16(zmm1, 6); - zmm2 = _mm512_srai_epi16(zmm2, 6); - - zmm0 = _mm512_packus_epi16(zmm0, zmm0); - zmm1 = _mm512_packus_epi16(zmm1, zmm1); - zmm2 = _mm512_packus_epi16(zmm2, zmm2); - - // STORERGB24_AVX512VBMI - __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); - __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); - __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); - - _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); - _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1)); - - dst_rgb24 += 96; + READYUV422_AVX512BW + YUVTORGB_AVX512BW + STORERGB24_AVX512VBMI width -= 32; } + if (width > 0) { + READYUV422_MASK_AVX512BW + YUVTORGB_AVX512BW + STORERGB24_MASK_AVX512VBMI + } _mm256_zeroupper(); } #endif // HAS_I422TORGB24ROW_AVX512VBMI @@ -2050,72 +2118,31 @@ void I422ToRGB24Row_AVX512BW(const uint8_t* src_y, ptrdiff_t offset = src_v - src_u; while (width >= 32) { - // READYUV422_AVX512BW - __m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u); - __m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset)); - src_u += 16; - - __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); - __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); - - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); - - __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); - zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); - zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); - - __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); - src_y += 32; - __m512i zmm4 = _mm512_castsi256_si512(ymm_y); - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); - zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); - zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); - - // YUVTORGB_AVX512BW - zmm3 = _mm512_sub_epi8(zmm3, zmm_128); - zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); - - __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); - __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); - __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); - - zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); - - zmm0 = _mm512_adds_epi16(zmm0, zmm4); - zmm1 = _mm512_subs_epi16(zmm4, zmm1); - zmm2 = _mm512_adds_epi16(zmm2, zmm4); - - zmm0 = _mm512_srai_epi16(zmm0, 6); - zmm1 = _mm512_srai_epi16(zmm1, 6); - zmm2 = _mm512_srai_epi16(zmm2, 6); - - zmm0 = _mm512_packus_epi16(zmm0, zmm0); - zmm1 = _mm512_packus_epi16(zmm1, zmm1); - zmm2 = _mm512_packus_epi16(zmm2, zmm2); - - // STORERGB24_AVX512BW - __m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); - __m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); - __m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); - __m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); - zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); - zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); - zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); - - __m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); - __m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); - - _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); - _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1)); - - dst_rgb24 += 96; + READYUV422_AVX512BW + YUVTORGB_AVX512BW + STORERGB24_AVX512BW width -= 32; } + if (width > 0) { + READYUV422_MASK_AVX512BW + YUVTORGB_AVX512BW + STORERGB24_MASK_AVX512BW + } _mm256_zeroupper(); } #endif // HAS_I422TORGB24ROW_AVX512BW +#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I422TORGB24ROW_AVX512BW) +#undef READYUV422_AVX512BW +#undef READYUV422_MASK_AVX512BW +#undef YUVTORGB_AVX512BW +#undef STORERGB24_AVX512VBMI +#undef STORERGB24_MASK_AVX512VBMI +#undef STORERGB24_AVX512BW +#undef STORERGB24_MASK_AVX512BW +#endif + #ifdef HAS_I422TOAR30ROW_AVX512BW LIBYUV_TARGET_AVX512BW void I422ToAR30Row_AVX512BW(const uint8_t* src_y, diff --git a/unit_test/convert_argb_test.cc b/unit_test/convert_argb_test.cc index bdc71a697..dfffb77c8 100644 --- a/unit_test/convert_argb_test.cc +++ b/unit_test/convert_argb_test.cc @@ -331,8 +331,8 @@ TESTPLANARTOB(V422, 2, 1, ARGB, 4, 4, 1) TESTPLANARTOB(V422, 2, 1, ABGR, 4, 4, 1) TESTPLANARTOB(I422, 2, 1, BGRA, 4, 4, 1) TESTPLANARTOB(I422, 2, 1, RGBA, 4, 4, 1) -TESTPLANARTOB(I422, 1, 1, RGB24, 3, 3, 1) -TESTPLANARTOB(I422, 1, 1, RAW, 3, 3, 1) +TESTPLANARTOB(I422, 2, 1, RGB24, 3, 3, 1) +TESTPLANARTOB(I422, 2, 1, RAW, 3, 3, 1) TESTPLANARTOB(I444, 1, 1, ARGB, 4, 4, 1) TESTPLANARTOB(I444, 1, 1, ABGR, 4, 4, 1) TESTPLANARTOB(I444, 1, 1, RGB24, 3, 3, 1)