diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 805b76c47..12d31fede 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -375,7 +375,6 @@ extern "C" { #define HAS_RGB24TOARGBROW_AVX512BW #define HAS_RAWTORGB24ROW_AVX512BW #define HAS_ARGBSHUFFLEROW_AVX512BW -#define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_RAWTORGB24ROW_AVX512VBMI #define HAS_I422TORGB24ROW_AVX512BW #define HAS_I422TOAR30ROW_AVX512BW @@ -418,6 +417,7 @@ extern "C" { #define HAS_RAWTORGB24ROW_AVX512BW #define HAS_RAWTORGB24ROW_AVX512VBMI #define HAS_ARGBTORGB24ROW_AVX512VBMI +#define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_CONVERT16TO8ROW_AVX512BW #define HAS_CONVERT8TO16ROW_AVX512BW #define HAS_HALFROW_16TO8_AVX512BW @@ -436,7 +436,6 @@ extern "C" { #define HAS_BLENDPLANEROW_AVX512BW #define HAS_RGBTOUV444MATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW -#define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_RAWTORGB24ROW_AVX512VBMI #define HAS_I422TORGB24ROW_AVX512BW #define HAS_ARGBTOUVMATRIXROW_AVX512BW @@ -5275,12 +5274,6 @@ void NV12ToRGB565Row_Any_AVX2(const uint8_t* y_buf, uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, int width); -void I422ToRGB24Row_Any_AVX512VBMI(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - const struct YuvConstants* yuvconstants, - int width); void I422ToRGB24Row_Any_AVX512BW(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, diff --git a/source/convert_argb.cc b/source/convert_argb.cc index 238aea2b2..d31c78ff7 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -5703,10 +5703,7 @@ int I420ToRGB24Matrix(const uint8_t* src_y, #endif #if defined(HAS_I422TORGB24ROW_AVX512VBMI) if (TestCpuFlag(kCpuHasAVX512VBMI)) { - I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI; - if (IS_ALIGNED(width, 32)) { - I422ToRGB24Row = I422ToRGB24Row_AVX512VBMI; - } + I422ToRGB24Row = I422ToRGB24Row_AVX512VBMI; } #endif #if defined(HAS_I422TORGB24ROW_NEON) @@ -5924,10 +5921,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y, #endif #if defined(HAS_I422TORGB24ROW_AVX512VBMI) if (TestCpuFlag(kCpuHasAVX512VBMI)) { - I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI; - if (IS_ALIGNED(width, 32)) { - I422ToRGB24Row = I422ToRGB24Row_AVX512VBMI; - } + I422ToRGB24Row = I422ToRGB24Row_AVX512VBMI; } #endif #if defined(HAS_I422TORGB24ROW_NEON) diff --git a/source/row_any.cc b/source/row_any.cc index b53b829e9..14ab2f2d4 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -420,9 +420,6 @@ ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 15) #ifdef HAS_I422TORGB24ROW_AVX2 ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15) #endif -#ifdef HAS_I422TORGB24ROW_AVX512VBMI -ANY31C(I422ToRGB24Row_Any_AVX512VBMI, I422ToRGB24Row_AVX512VBMI, 1, 0, 3, 31) -#endif #ifdef HAS_I422TORGB24ROW_AVX512BW ANY31C(I422ToRGB24Row_Any_AVX512BW, I422ToRGB24Row_AVX512BW, 1, 0, 3, 31) #endif diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 23b317aea..c02f124f6 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -4120,12 +4120,15 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { + uintptr_t temp; asm volatile ( YUVTORGB_SETUP_AVX512BW(yuvconstants) "vmovdqu32 %[kMaskBG],%%zmm20 \n" "vmovdqu32 %[kMaskDST0],%%zmm21 \n" "vmovdqu32 %[kMaskDST1],%%zmm22 \n" "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" @@ -4133,13 +4136,57 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, YUVTORGB_AVX512BW(yuvconstants) STORERGB24_AVX512VBMI "sub $0x20,%[width] \n" - "jg 1b \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + // Calculate a mask for the final iteration to deal with the tail. + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%}\n" + "mov $-1,%[temp] \n" + "lea 1(%q[width]),%k[y_buf] \n" + "shr $1,%k[y_buf] \n" + "bzhi %q[y_buf],%[temp],%[y_buf] \n" + "kmovw %k[y_buf],%%k2 \n" + "vmovdqu8 (%[u_buf]),%%xmm3%{%%k2%}%{z%}\n" + "vmovdqu8 0x00(%[u_buf],%[v_buf],1),%%xmm1%{%%k2%}%{z%}\n" + "lea (%q[width],%q[width],2),%k[width]\n" + "bzhi %q[width],%[temp],%[y_buf] \n" + "kmovq %[y_buf],%%k3 \n" + "xor %k[y_buf],%k[y_buf] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[y_buf] \n" + "bzhi %q[width],%[y_buf],%[y_buf] \n" + "kmovd %k[y_buf],%%k4 \n" + + "vpunpcklbw %%xmm1,%%xmm3,%%xmm2 \n" + "vpunpckhbw %%xmm1,%%xmm3,%%xmm3 \n" + "vinserti64x2 $1,%%xmm3,%%ymm2,%%ymm3 \n" + "vmovdqa64 %%zmm16,%%zmm1 \n" + "vpermi2w %%zmm3,%%zmm3,%%zmm1 \n" + "vpsllw $8,%%zmm4,%%zmm3 \n" + "vpord %%zmm3,%%zmm4,%%zmm4 \n" + "vmovdqa64 %%zmm1,%%zmm3 \n" + YUVTORGB_AVX512BW(yuvconstants) + "vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" + "vmovdqa64 %%zmm0,%%zmm3 \n" + "vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" + "vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" + "vmovdqu8 %%zmm3,(%[dst_rgb24])%{%%k3%} \n" + "vmovdqu8 %%ymm0,0x40(%[dst_rgb24])%{%%k4%}\n" + + "99: \n" "vzeroupper \n" : [y_buf]"+r"(y_buf), // %[y_buf] [u_buf]"+r"(u_buf), // %[u_buf] [v_buf]"+r"(v_buf), // %[v_buf] [dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24] - [width]"+rm"(width) // %[width] + [width]"+r"(width), // %[width] + [temp]"=&r"(temp) // %[temp] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm] [dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm] @@ -4149,7 +4196,8 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, [kMaskDST1]"m"(kMaskDST1) // %[kMaskDST1] : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm20", "xmm21", "xmm22" + "xmm20", "xmm21", "xmm22", + "k1", "k2", "k3", "k4" ); } #endif // defined(HAS_I422TORGB24ROW_AVX512VBMI)