diff --git a/include/libyuv/row.h b/include/libyuv/row.h index dd9a6e108..bf01f1c59 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -2059,12 +2059,6 @@ void RGBToUVMatrixRow_AVX512BW(const uint8_t* src_rgb, uint8_t* dst_v, int width, const struct ArgbConstants* c); -void RGBToUVMatrixRow_Any_AVX512BW(const uint8_t* src_rgb, - int src_stride_rgb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c); void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, int src_stride_argb, uint8_t* dst_u, @@ -2418,12 +2412,6 @@ void ARGBToUVMatrixRow_Any_AVX2(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c); -void ARGBToUVMatrixRow_Any_AVX512BW(const uint8_t* src_argb, - int src_stride_argb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c); void ARGBToUV444MatrixRow_Any_SSSE3(const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, @@ -3933,7 +3921,6 @@ void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int width); void CopyRow_C(const uint8_t* src, uint8_t* dst, int count); void CopyRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_AVX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); -void CopyRow_Any_AVX512BW(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4061,10 +4048,6 @@ void ARGBShuffleRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, const uint8_t* param, int width); -void ARGBShuffleRow_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_ptr, - const uint8_t* param, - int width); void ARGBShuffleRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, const uint8_t* param, @@ -4192,15 +4175,9 @@ void RAWToARGBRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void RGB24ToARGBRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); -void RAWToARGBRow_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, uint8_t* dst_argb, int width); -void RGB24ToARGBRow_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); void RAWToRGBARow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -5042,18 +5019,6 @@ void I422ToARGBRow_Any_AVX2(const uint8_t* y_buf, uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, int width); -void I422ToARGBRow_Any_AVX512BW(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - const struct YuvConstants* yuvconstants, - int width); -void I422ToAR30Row_Any_AVX512BW(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - const struct YuvConstants* yuvconstants, - int width); void I422ToRGBARow_Any_AVX2(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -5508,11 +5473,6 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0, const uint8_t* alpha, uint8_t* dst, int width); -void BlendPlaneRow_Any_AVX512BW(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - int width); void BlendPlaneRow_NEON(const uint8_t* src0, const uint8_t* src1, const uint8_t* alpha, diff --git a/source/convert.cc b/source/convert.cc index cc93155b1..075b3e869 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -2926,10 +2926,7 @@ int ARGBToI420Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; - } + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) @@ -3161,10 +3158,7 @@ int ARGBToI420Alpha(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; - } + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) @@ -3416,10 +3410,7 @@ int RGB24ToI420(const uint8_t* src_rgb24, #endif #if defined(HAS_RGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; - } + RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUVMATRIXROW_NEON) @@ -3564,10 +3555,7 @@ int RGB24ToJ420(const uint8_t* src_rgb24, #endif #if defined(HAS_RGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; - } + RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUVMATRIXROW_NEON) @@ -3717,10 +3705,7 @@ int RAWToI420(const uint8_t* src_raw, #endif #if defined(HAS_RGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; - } + RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUVMATRIXROW_NEON) @@ -3855,10 +3840,7 @@ int RAWToJ420(const uint8_t* src_raw, #endif #if defined(HAS_RGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; - } + RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUVMATRIXROW_NEON) diff --git a/source/convert_argb.cc b/source/convert_argb.cc index a47659625..a889889c0 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -99,10 +99,7 @@ int I420ToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I422TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == (kCpuHasAVX512BW | kCpuHasAVX512VL)) { - I422ToARGBRow = I422ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I422ToARGBRow = I422ToARGBRow_AVX512BW; - } + I422ToARGBRow = I422ToARGBRow_AVX512BW; } #endif #if defined(HAS_I422TOARGBROW_NEON) @@ -365,10 +362,7 @@ int I422ToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I422TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == (kCpuHasAVX512BW | kCpuHasAVX512VL)) { - I422ToARGBRow = I422ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I422ToARGBRow = I422ToARGBRow_AVX512BW; - } + I422ToARGBRow = I422ToARGBRow_AVX512BW; } #endif #if defined(HAS_I422TOARGBROW_NEON) @@ -3874,10 +3868,7 @@ int RGB24ToARGB(const uint8_t* src_rgb24, #endif #if defined(HAS_RGB24TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGB24ToARGBRow = RGB24ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGB24ToARGBRow = RGB24ToARGBRow_AVX512BW; - } + RGB24ToARGBRow = RGB24ToARGBRow_AVX512BW; } #endif #if defined(HAS_RGB24TOARGBROW_NEON) @@ -3975,10 +3966,7 @@ int RAWToARGB(const uint8_t* src_raw, #endif #if defined(HAS_RAWTOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RAWToARGBRow = RAWToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RAWToARGBRow = RAWToARGBRow_AVX512BW; - } + RAWToARGBRow = RAWToARGBRow_AVX512BW; } #endif #if defined(HAS_RAWTOARGBROW_NEON) @@ -6689,10 +6677,7 @@ int I420ToRGB565Dither(const uint8_t* src_y, #if defined(HAS_I422TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == (kCpuHasAVX512BW | kCpuHasAVX512VL)) { - I422ToARGBRow = I422ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I422ToARGBRow = I422ToARGBRow_AVX512BW; - } + I422ToARGBRow = I422ToARGBRow_AVX512BW; } #endif #if defined(HAS_I422TOARGBROW_NEON) @@ -6865,10 +6850,7 @@ int I420ToAR30Matrix(const uint8_t* src_y, #if defined(HAS_I422TOAR30ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == (kCpuHasAVX512BW | kCpuHasAVX512VL)) { - I422ToAR30Row = I422ToAR30Row_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I422ToAR30Row = I422ToAR30Row_AVX512BW; - } + I422ToAR30Row = I422ToAR30Row_AVX512BW; } #endif #if defined(HAS_I422TOAR30ROW_NEON) diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index ae25009ae..09a07dc81 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -380,10 +380,7 @@ int ARGBToI422Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; - } + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) @@ -856,10 +853,7 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; - } + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) @@ -1660,10 +1654,7 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; - } + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) @@ -1829,10 +1820,7 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; - } + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) @@ -2095,10 +2083,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; - } + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) @@ -2496,10 +2481,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb, #endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; - } + ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_ARGBTOUVMATRIXROW_RVV) @@ -4206,10 +4188,7 @@ static int RAWToNVMatrix(const uint8_t* src_raw, #endif #if defined(HAS_RGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; - } + RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; } #endif #if defined(HAS_RGBTOUVMATRIXROW_NEON) diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 9f1a10062..0590febb9 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -67,7 +67,7 @@ void CopyPlane(const uint8_t* src_y, #endif #if defined(HAS_COPYROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - CopyRow = IS_ALIGNED(width, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW; + CopyRow = CopyRow_AVX512BW; } #endif #if defined(HAS_COPYROW_ERMS) @@ -3219,10 +3219,7 @@ int BlendPlane(const uint8_t* src_y0, #endif #if defined(HAS_BLENDPLANEROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - BlendPlaneRow = BlendPlaneRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - BlendPlaneRow = BlendPlaneRow_AVX512BW; - } + BlendPlaneRow = BlendPlaneRow_AVX512BW; } #endif #if defined(HAS_BLENDPLANEROW_NEON) @@ -3326,10 +3323,7 @@ int I420Blend(const uint8_t* src_y0, #endif #if defined(HAS_BLENDPLANEROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - BlendPlaneRow = BlendPlaneRow_Any_AVX512BW; - if (IS_ALIGNED(halfwidth, 64)) { - BlendPlaneRow = BlendPlaneRow_AVX512BW; - } + BlendPlaneRow = BlendPlaneRow_AVX512BW; } #endif #if defined(HAS_BLENDPLANEROW_NEON) @@ -5008,10 +5002,7 @@ int ARGBShuffle(const uint8_t* src_argb, #endif #if defined(HAS_ARGBSHUFFLEROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBShuffleRow = ARGBShuffleRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - ARGBShuffleRow = ARGBShuffleRow_AVX512BW; - } + ARGBShuffleRow = ARGBShuffleRow_AVX512BW; } #endif #if defined(HAS_ARGBSHUFFLEROW_NEON) @@ -5103,10 +5094,7 @@ int AR64Shuffle(const uint16_t* src_ar64, #endif #if defined(HAS_ARGBSHUFFLEROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - AR64ShuffleRow = ARGBShuffleRow_Any_AVX512BW; - if (IS_ALIGNED(width, 16)) { - AR64ShuffleRow = ARGBShuffleRow_AVX512BW; - } + AR64ShuffleRow = ARGBShuffleRow_AVX512BW; } #endif #if defined(HAS_ARGBSHUFFLEROW_NEON) diff --git a/source/rotate.cc b/source/rotate.cc index ac7f099a3..c155a0d90 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -258,7 +258,7 @@ void RotatePlane180(const uint8_t* src, #endif #if defined(HAS_COPYROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - CopyRow = IS_ALIGNED(width, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW; + CopyRow = CopyRow_AVX512BW; } #endif #if defined(HAS_COPYROW_ERMS) @@ -499,7 +499,7 @@ void SplitRotateUV180(const uint8_t* src, } #endif #if defined(HAS_MIRRORSPLITUVROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW) && IS_ALIGNED(width, 32)) { + if (TestCpuFlag(kCpuHasAVX512BW)) { MirrorSplitUVRow = MirrorSplitUVRow_AVX512BW; } #endif diff --git a/source/row_any.cc b/source/row_any.cc index c12ad5e10..dbd724915 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -340,9 +340,6 @@ ANY31(I422ToUYVYRow_Any_LASX, I422ToUYVYRow_LASX, 1, 1, 4, 31) #ifdef HAS_BLENDPLANEROW_AVX2 ANY31(BlendPlaneRow_Any_AVX2, BlendPlaneRow_AVX2, 0, 0, 1, 31) #endif -#ifdef HAS_BLENDPLANEROW_AVX512BW -ANY31(BlendPlaneRow_Any_AVX512BW, BlendPlaneRow_AVX512BW, 0, 0, 1, 63) -#endif #ifdef HAS_BLENDPLANEROW_SSSE3 ANY31(BlendPlaneRow_Any_SSSE3, BlendPlaneRow_SSSE3, 0, 0, 1, 7) #endif @@ -409,9 +406,6 @@ ANY31C(I422ToAR30Row_Any_SSSE3, I422ToAR30Row_SSSE3, 1, 0, 4, 7) #ifdef HAS_I422TOAR30ROW_AVX2 ANY31C(I422ToAR30Row_Any_AVX2, I422ToAR30Row_AVX2, 1, 0, 4, 15) #endif -#ifdef HAS_I422TOAR30ROW_AVX512BW -ANY31C(I422ToAR30Row_Any_AVX512BW, I422ToAR30Row_AVX512BW, 1, 0, 4, 31) -#endif #ifdef HAS_I444TOARGBROW_SSSE3 ANY31C(I444ToARGBRow_Any_SSSE3, I444ToARGBRow_SSSE3, 0, 0, 4, 7) #endif @@ -424,9 +418,6 @@ ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15) #ifdef HAS_I422TOARGBROW_AVX2 ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15) #endif -#ifdef HAS_I422TOARGBROW_AVX512BW -ANY31C(I422ToARGBRow_Any_AVX512BW, I422ToARGBRow_AVX512BW, 1, 0, 4, 31) -#endif #ifdef HAS_I422TORGBAROW_AVX2 ANY31C(I422ToRGBARow_Any_AVX2, I422ToRGBARow_AVX2, 1, 0, 4, 15) #endif @@ -1034,9 +1025,6 @@ ANY21PT(MergeUVRow_16_Any_NEON, MergeUVRow_16_NEON, uint16_t, 2, 7) memcpy(dst_ptr + np * BPP, vout, r * BPP); \ } -#ifdef HAS_COPYROW_AVX512BW -ANY11(CopyRow_Any_AVX512BW, CopyRow_AVX512BW, 0, 1, 1, 127) -#endif #ifdef HAS_COPYROW_AVX ANY11(CopyRow_Any_AVX, CopyRow_AVX, 0, 1, 1, 63) #endif @@ -1108,12 +1096,6 @@ ANY11(RAWToARGBRow_Any_AVX2, RAWToARGBRow_AVX2, 0, 3, 4, 31) #if defined(HAS_RGB24TOARGBROW_AVX2) ANY11(RGB24ToARGBRow_Any_AVX2, RGB24ToARGBRow_AVX2, 0, 3, 4, 31) #endif -#if defined(HAS_RAWTOARGBROW_AVX512BW) -ANY11(RAWToARGBRow_Any_AVX512BW, RAWToARGBRow_AVX512BW, 0, 3, 4, 63) -#endif -#if defined(HAS_RGB24TOARGBROW_AVX512BW) -ANY11(RGB24ToARGBRow_Any_AVX512BW, RGB24ToARGBRow_AVX512BW, 0, 3, 4, 63) -#endif #if defined(HAS_RAWTORGBAROW_SSSE3) ANY11(RAWToRGBARow_Any_SSSE3, RAWToRGBARow_SSSE3, 0, 3, 4, 15) #endif @@ -1503,14 +1485,6 @@ ANY11P(ARGBShuffleRow_Any_SSSE3, ARGBShuffleRow_SSSE3, const uint8_t*, 4, 4, 7) #ifdef HAS_ARGBSHUFFLEROW_AVX2 ANY11P(ARGBShuffleRow_Any_AVX2, ARGBShuffleRow_AVX2, const uint8_t*, 4, 4, 15) #endif -#ifdef HAS_ARGBSHUFFLEROW_AVX512BW -ANY11P(ARGBShuffleRow_Any_AVX512BW, - ARGBShuffleRow_AVX512BW, - const uint8_t*, - 4, - 4, - 31) -#endif #ifdef HAS_ARGBSHUFFLEROW_NEON ANY11P(ARGBShuffleRow_Any_NEON, ARGBShuffleRow_NEON, const uint8_t*, 4, 4, 3) #endif @@ -2465,12 +2439,6 @@ ANY12MS(ARGB1555ToUVMatrixRow_Any_AVX2, ARGB1555ToUVMatrixRow_AVX2, 0, 2, 31) ANY12MS(ARGB4444ToUVMatrixRow_Any_AVX2, ARGB4444ToUVMatrixRow_AVX2, 0, 2, 31) #endif #endif -#ifdef HAS_ARGBTOUVMATRIXROW_AVX512BW -ANY12MS(ARGBToUVMatrixRow_Any_AVX512BW, ARGBToUVMatrixRow_AVX512BW, 0, 4, 63) -#endif -#ifdef HAS_RGBTOUVMATRIXROW_AVX512BW -ANY12MS(RGBToUVMatrixRow_Any_AVX512BW, RGBToUVMatrixRow_AVX512BW, 0, 3, 63) -#endif #ifdef HAS_ARGBTOUVMATRIXROW_SSSE3 ANY12MS(ARGBToUVMatrixRow_Any_SSSE3, ARGBToUVMatrixRow_SSSE3, 0, 4, 7) #endif diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 52ec255e4..cce7a9120 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -458,21 +458,25 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, uint8_t* dst_argb, const uint32_t* shuffler, int width) { + uintptr_t temp; + uintptr_t mask; asm volatile( - "vmovdqu64 %4,%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW - "vmovdqu64 %5,%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW - "vmovdqu64 %6,%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW - "vmovdqu64 %7,%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW - "vbroadcasti64x4 %3,%%zmm4 \n" // shuffler (32 bytes) + "vmovdqu64 %[q0],%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW + "vmovdqu64 %[q1],%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW + "vmovdqu64 %[q2],%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW + "vmovdqu64 %[q3],%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW + "vbroadcasti64x4 (%[shuffler]),%%zmm4 \n" // shuffler (32 bytes) "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // 0xffffffff "vpslld $0x18,%%zmm16,%%zmm16 \n" // 0xff000000 + "sub $0x40,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" - "vmovdqu8 (%0),%%zmm0 \n" - "vmovdqu8 64(%0),%%zmm1 \n" - "vmovdqu8 128(%0),%%zmm2 \n" - "lea 192(%0),%0 \n" + "vmovdqu8 (%[src_raw]),%%zmm0 \n" + "vmovdqu8 64(%[src_raw]),%%zmm1 \n" + "vmovdqu8 128(%[src_raw]),%%zmm2 \n" + "lea 192(%[src_raw]),%[src_raw] \n" "vpermq %%zmm2,%%zmm23,%%zmm3 \n" "vmovdqa64 %%zmm22,%%zmm18 \n" "vpermi2q %%zmm2,%%zmm1,%%zmm18 \n" @@ -489,24 +493,71 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, "vpord %%zmm16,%%zmm1,%%zmm1 \n" "vpord %%zmm16,%%zmm2,%%zmm2 \n" "vpord %%zmm16,%%zmm3,%%zmm3 \n" - "vmovdqu32 %%zmm0,(%1) \n" - "vmovdqu32 %%zmm1,0x40(%1) \n" - "vmovdqu32 %%zmm2,0x80(%1) \n" - "vmovdqu32 %%zmm3,0xc0(%1) \n" - "lea 0x100(%1),%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" + "vmovdqu32 %%zmm0,(%[dst_argb]) \n" + "vmovdqu32 %%zmm1,0x40(%[dst_argb]) \n" + "vmovdqu32 %%zmm2,0x80(%[dst_argb]) \n" + "vmovdqu32 %%zmm3,0xc0(%[dst_argb]) \n" + "lea 0x100(%[dst_argb]),%[dst_argb]\n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[mask] \n" + "kmovq %[mask],%%k1 \n" + "kshiftrq $16,%%k1,%%k2 \n" + "kshiftrq $32,%%k1,%%k3 \n" + "kshiftrq $48,%%k1,%%k4 \n" + "lea (%q[width],%q[width],2),%k[width] \n" + "bzhi %q[width],%[temp],%[mask] \n" + "kmovq %[mask],%%k5 \n" + "vmovdqu8 (%[src_raw]),%%zmm0%{%%k5%}%{z%} \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k5 \n" + "vmovdqu8 64(%[src_raw]),%%zmm1%{%%k5%}%{z%} \n" + "xor %k[mask],%k[mask] \n" + "sub $0x40,%k[width] \n" + "cmovg %[temp],%[mask] \n" + "bzhi %q[width],%[mask],%[mask] \n" + "kmovq %[mask],%%k5 \n" + "vmovdqu8 128(%[src_raw]),%%zmm2%{%%k5%}%{z%} \n" + "vpermq %%zmm2,%%zmm23,%%zmm3 \n" + "vpermi2q %%zmm2,%%zmm1,%%zmm22 \n" + "vpermi2q %%zmm1,%%zmm0,%%zmm21 \n" + "vpermq %%zmm0,%%zmm20,%%zmm0 \n" + "vpshufb %%zmm4,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm4,%%zmm21,%%zmm1 \n" + "vpshufb %%zmm4,%%zmm22,%%zmm2 \n" + "vpshufb %%zmm4,%%zmm3,%%zmm3 \n" + "vpord %%zmm16,%%zmm0,%%zmm0 \n" + "vpord %%zmm16,%%zmm1,%%zmm1 \n" + "vpord %%zmm16,%%zmm2,%%zmm2 \n" + "vpord %%zmm16,%%zmm3,%%zmm3 \n" + "vmovdqu32 %%zmm0,(%[dst_argb])%{%%k1%} \n" + "vmovdqu32 %%zmm1,0x40(%[dst_argb])%{%%k2%} \n" + "vmovdqu32 %%zmm2,0x80(%[dst_argb])%{%%k3%} \n" + "vmovdqu32 %%zmm3,0xc0(%[dst_argb])%{%%k4%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "m"(*shuffler), // %3 - "m"(kPermqRGB24ToARGB0_AVX512BW), // %4 - "m"(kPermqRGB24ToARGB1_AVX512BW), // %5 - "m"(kPermqRGB24ToARGB2_AVX512BW), // %6 - "m"(kPermqRGB24ToARGB3_AVX512BW) // %7 + : [src_raw] "+r"(src_raw), // %[src_raw] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width), // %[width] + [temp] "=&r"(temp), // %[temp] + [mask] "=&r"(mask) // %[mask] + : [shuffler] "r"(shuffler), // %[shuffler] + [q0] "m"(kPermqRGB24ToARGB0_AVX512BW), // %[q0] + [q1] "m"(kPermqRGB24ToARGB1_AVX512BW), // %[q1] + [q2] "m"(kPermqRGB24ToARGB2_AVX512BW), // %[q2] + [q3] "m"(kPermqRGB24ToARGB3_AVX512BW) // %[q3] : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", - "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23"); + "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1", "k2", "k3", + "k4", "k5"); } void RAWToARGBRow_AVX512BW(const uint8_t* src_raw, @@ -2410,23 +2461,26 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c) { + uintptr_t temp; asm volatile( - "vbroadcasti32x4 0x20(%5),%%zmm4 \n" // RGBToU - "vbroadcasti32x4 0x40(%5),%%zmm5 \n" // RGBToV + "vbroadcasti32x4 0x20(%[c]),%%zmm4 \n" // RGBToU + "vbroadcasti32x4 0x40(%[c]),%%zmm5 \n" // RGBToV "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" "vpabsb %%zmm16,%%zmm20 \n" // 0x0101 "vpsllw $15,%%zmm16,%%zmm17 \n" // 0x8000 - "vbroadcasti32x4 %6,%%zmm21 \n" // kShuffleAARRGGBB - "vmovups %7,%%zmm18 \n" // kPermdARGBToY_AVX512BW - "vmovups %8,%%zmm19 \n" // kPermdARGBToUV_AVX512BW + "vbroadcasti32x4 %[shuf],%%zmm21 \n" // kShuffleAARRGGBB + "vmovups %[permy],%%zmm18 \n" // kPermdARGBToY_AVX512BW + "vmovups %[permuv],%%zmm19 \n" // kPermdARGBToUV_AVX512BW "sub %1,%2 \n" + "subl $0x20,%3 \n" + "jl 2f \n" LABELALIGN "1: \n" "vmovups (%0),%%zmm0 \n" // Read 32x2 ARGB Pixels "vmovups 0x40(%0),%%zmm1 \n" - "vmovups 0x00(%0,%4,1),%%zmm2 \n" - "vmovups 0x40(%0,%4,1),%%zmm3 \n" + "vmovups 0x00(%0,%[stride],1),%%zmm2 \n" + "vmovups 0x40(%0,%[stride],1),%%zmm3 \n" "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" // aarrggbb "vpshufb %%zmm21,%%zmm1,%%zmm1 \n" "vpshufb %%zmm21,%%zmm2,%%zmm2 \n" @@ -2463,23 +2517,76 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, "lea 0x80(%0),%0 \n" "lea 0x10(%1),%1 \n" "subl $0x20,%3 \n" - "jg 1b \n" + "jge 1b \n" + + "2: \n" + "addl $0x20,%3 \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q3,%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "kshiftrd $16,%%k1,%%k2 \n" + "knotw %%k1,%%k3 \n" + "knotw %%k2,%%k4 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu32 0x00(%0,%[stride],1),%%zmm2%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%0,%[stride],1),%%zmm3%{%%k2%}%{z%} \n" + "vpshufd $0xa0,%%zmm0,%%zmm0%{%%k3%} \n" + "vpshufd $0xa0,%%zmm1,%%zmm1%{%%k4%} \n" + "vpshufd $0xa0,%%zmm2,%%zmm2%{%%k3%} \n" + "vpshufd $0xa0,%%zmm3,%%zmm3%{%%k4%} \n" + "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm21,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm21,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm21,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" + "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm0,%%zmm2,%%zmm0 \n" + "vpaddw %%zmm1,%%zmm3,%%zmm1 \n" + "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" + "vpsrlw $1,%%zmm0,%%zmm0 \n" + "vpsrlw $1,%%zmm1,%%zmm1 \n" + "vpavgw %%zmm2,%%zmm0,%%zmm0 \n" + "vpavgw %%zmm2,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm19,%%zmm0 \n" + + "vpmaddubsw %%zmm4,%%zmm0,%%zmm1 \n" + "vpmaddubsw %%zmm5,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" + "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" + "vpackssdw %%zmm0,%%zmm1,%%zmm0 \n" + "vpaddw %%zmm17,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm0,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm18,%%zmm0 \n" + + "addl $1,%3 \n" + "shrl $1,%3 \n" + "mov $-1,%[temp] \n" + "bzhi %q3,%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu8 %%xmm0,(%1)%{%%k1%} \n" + "vextracti32x4 $0x1,%%zmm0,%%xmm0 \n" + "vmovdqu8 %%xmm0,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src_argb), // %0 - "+r"(dst_u), // %1 - "+r"(dst_v), // %2 -#if defined(__i386__) - "+m"(width) // %3 -#else - "+rm"(width) // %3 -#endif - : "r"((ptrdiff_t)(src_stride_argb)), // %4 - "r"(c), // %5 - "m"(kShuffleAARRGGBB), // %6 - "m"(kPermdARGBToY_AVX512BW), // %7 - "m"(kPermdARGBToUV_AVX512BW) // %8 + : "+r"(src_argb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : [stride] "r"((ptrdiff_t)(src_stride_argb)), // %[stride] + [c] "r"(c), // %[c] + [shuf] "m"(kShuffleAARRGGBB), // %[shuf] + [permy] "m"(kPermdARGBToY_AVX512BW), // %[permy] + [permuv] "m"(kPermdARGBToUV_AVX512BW) // %[permuv] : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", - "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4"); } #endif // HAS_ARGBTOUVMATRIXROW_AVX512BW @@ -3899,6 +4006,18 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vmovups %%zmm0,0x40(%[dst_argb]) \n" \ "lea 0x80(%[dst_argb]), %[dst_argb] \n" +// Store masked ARGB values with AVX512BW. +#define STOREARGB_MASK_AVX512BW \ + "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \ + "vpermq %%zmm0,%%zmm18,%%zmm0 \n" \ + "vpunpcklbw %%zmm5,%%zmm2,%%zmm2 \n" \ + "vpermq %%zmm2,%%zmm18,%%zmm2 \n" \ + "vpunpcklwd %%zmm2,%%zmm0,%%zmm1 \n" \ + "vpunpckhwd %%zmm2,%%zmm0,%%zmm0 \n" \ + "kshiftrd $16,%%k1,%%k2 \n" \ + "vmovdqu32 %%zmm1,(%[dst_argb])%{%%k1%} \n" \ + "vmovdqu32 %%zmm0,0x40(%[dst_argb])%{%%k2%} \n" + // Store 16 RGB24 values. #define STORERGB24_AVX2 \ "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \ @@ -4009,6 +4128,27 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vmovdqu32 %%zmm3,0x40(%[dst_ar30]) \n" \ "lea 0x80(%[dst_ar30]), %[dst_ar30] \n" +// Store masked AR30 values with AVX512BW. +#define STOREAR30_MASK_AVX512BW \ + "vpminsw %%zmm7,%%zmm0,%%zmm0 \n" \ + "vpminsw %%zmm7,%%zmm1,%%zmm1 \n" \ + "vpminsw %%zmm7,%%zmm2,%%zmm2 \n" \ + "vpmaxsw %%zmm6,%%zmm0,%%zmm0 \n" \ + "vpmaxsw %%zmm6,%%zmm1,%%zmm1 \n" \ + "vpmaxsw %%zmm6,%%zmm2,%%zmm2 \n" \ + "vpsrlw $4,%%zmm0,%%zmm0 \n" \ + "vpsrlw $10,%%zmm1,%%zmm3 \n" \ + "vpsllw $6,%%zmm1,%%zmm1 \n" \ + "vpternlogd $0xf8,%%zmm19,%%zmm1,%%zmm0 \n" \ + "vpternlogd $0xe4,%%zmm7,%%zmm3,%%zmm2 \n" \ + "vpord %%zmm5,%%zmm2,%%zmm2 \n" \ + "vmovdqa64 %%zmm0,%%zmm3 \n" \ + "vpermt2w %%zmm2,%%zmm20,%%zmm0 \n" \ + "vpermt2w %%zmm2,%%zmm21,%%zmm3 \n" \ + "kshiftrd $16,%%k1,%%k2 \n" \ + "vmovdqu32 %%zmm0,(%[dst_ar30])%{%%k1%} \n" \ + "vmovdqu32 %%zmm3,0x40(%[dst_ar30])%{%%k2%} \n" + // Store 16 AR30 values. #define STOREAR30_AVX2 \ "vpminsw %%ymm7,%%ymm0,%%ymm0 \n" \ @@ -4177,10 +4317,13 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { + uintptr_t temp; asm volatile ( YUVTORGB_SETUP_AVX512BW(yuvconstants) "sub %[u_buf],%[v_buf] \n" "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" @@ -4188,20 +4331,31 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf, YUVTORGB_AVX512BW(yuvconstants) STOREARGB_AVX512BW "sub $0x20,%[width] \n" - "jg 1b \n" + "jge 1b \n" + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV422_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" "vzeroupper \n" : [y_buf]"+r"(y_buf), // %[y_buf] [u_buf]"+r"(u_buf), // %[u_buf] [v_buf]"+r"(v_buf), // %[v_buf] [dst_argb]"+r"(dst_argb), // %[dst_argb] - [width]"+rm"(width) // %[width] + [width]"+r"(width), // %[width] + [temp]"=&r"(temp) // %[temp] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm] [dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm] [unperm]"r"(kUnpermuteAVX512) // %[unperm] : "memory", "cc", YUVTORGB_REGS_AVX512BW - "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "k1", "k2" ); } #endif // defined(HAS_I422TOARGBROW_AVX512BW) @@ -4361,6 +4515,7 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width) { + uintptr_t temp; asm volatile ( YUVTORGB_SETUP_AR30_AVX512BW(yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -4372,6 +4527,8 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf, "vpxord %%zmm6,%%zmm6,%%zmm6 \n" // 0 for min "vmovdqu16 (%[permar30_0]),%%zmm20 \n" "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" LABELALIGN "1: \n" @@ -4379,14 +4536,24 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf, YUVTORGB16_AVX512BW(yuvconstants) STOREAR30_AVX512BW "sub $0x20,%[width] \n" - "jg 1b \n" + "jge 1b \n" + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV422_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" "vzeroupper \n" : [y_buf]"+r"(y_buf), // %[y_buf] [u_buf]"+r"(u_buf), // %[u_buf] [v_buf]"+r"(v_buf), // %[v_buf] [dst_ar30]"+r"(dst_ar30), // %[dst_ar30] - [width]"+rm"(width) // %[width] + [width]"+r"(width), // %[width] + [temp]"=&r"(temp) // %[temp] : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm] [dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm] @@ -4395,7 +4562,7 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf, [permar30_1]"r"(kPermAR30_1) // %[permar30_1] : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", - "xmm19", "xmm20", "xmm21" + "xmm19", "xmm20", "xmm21", "k1", "k2" ); } #endif // defined(HAS_I422TOAR30ROW_AVX512BW) @@ -5239,6 +5406,14 @@ static const uvec8 kShuffleMirrorSplitUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u, #endif // defined(HAS_MIRRORSPLITUVROW_AVX2) || // defined(HAS_MIRRORSPLITUVROW_AVX512BW) +#if defined(HAS_MIRRORSPLITUVROW_AVX512BW) || defined(HAS_MIRRORUVROW_AVX512BW) +static const uint16_t kMirrorUVPermute_AVX512[32] = { + 31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u, + 20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u, + 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u}; +#endif // defined(HAS_MIRRORSPLITUVROW_AVX512BW) || + // defined(HAS_MIRRORUVROW_AVX512BW) + #ifdef HAS_MIRRORSPLITUVROW_AVX512BW static const uint64_t kMirrorSplitUVPermute[8] = {6, 4, 2, 0, 7, 5, 3, 1}; @@ -5247,16 +5422,17 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src, uint8_t* dst_v, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); + uintptr_t temp; asm volatile( - "vbroadcasti32x4 %4,%%zmm1 \n" - "lea -0x40(%0,%3,2),%0 \n" + "vbroadcasti32x4 %[shuf],%%zmm1 \n" "sub %1,%2 \n" - "vmovdqu64 %5,%%zmm3 \n" + "vmovdqu64 %[perm],%%zmm3 \n" + "sub $0x20,%3 \n" + "jl 2f \n" LABELALIGN "1: \n" - "vmovdqu8 (%0),%%zmm0 \n" - "lea -0x40(%0),%0 \n" + "vmovdqu8 (%0,%3,2),%%zmm0 \n" "vpshufb %%zmm1,%%zmm0,%%zmm0 \n" "vpermq %%zmm0,%%zmm3,%%zmm0 \n" "vextracti64x4 $0x1,%%zmm0,%%ymm2 \n" @@ -5264,15 +5440,36 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src, "vmovdqu %%ymm2,0x00(%1,%2,1) \n" "lea 0x20(%1),%1 \n" "sub $0x20,%3 \n" - "jg 1b \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q3,%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "mov $0x20,%k[temp] \n" + "sub %k3,%k[temp] \n" + "vpbroadcastw %k[temp],%%zmm2 \n" + "vmovdqu16 %[mirror_uv],%%zmm4 \n" + "vpsubw %%zmm2,%%zmm4,%%zmm2 \n" + "vpermw %%zmm0,%%zmm2,%%zmm0 \n" + "vpmovwb %%zmm0,(%1)%{%%k1%} \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpmovwb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src), // %0 - "+r"(dst_u), // %1 - "+r"(dst_v), // %2 - "+r"(temp_width) // %3 - : "m"(kShuffleMirrorSplitUV), // %4 - "m"(kMirrorSplitUVPermute) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + : "+r"(src), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(temp_width), // %3 + [temp] "=&r"(temp) // %[temp] + : [shuf] "m"(kShuffleMirrorSplitUV), // %[shuf] + [perm] "m"(kMirrorSplitUVPermute), // %[perm] + [mirror_uv] "m"(kMirrorUVPermute_AVX512) // %[mirror_uv] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "k1"); } #endif // HAS_MIRRORSPLITUVROW_AVX512BW @@ -5358,10 +5555,6 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { #endif // HAS_MIRRORUVROW_AVX2 #ifdef HAS_MIRRORUVROW_AVX512BW -static const uint16_t kMirrorUVPermute_AVX512[32] = { - 31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u, - 20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u, - 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u}; void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); asm volatile("vmovdqu16 %3,%%zmm5 \n" @@ -7773,7 +7966,12 @@ void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) { #ifdef HAS_COPYROW_AVX512BW void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + uintptr_t temp; asm volatile( + "sub $0x80,%2 \n" + "jl 2f \n" + + LABELALIGN "1: \n" "vmovups (%0),%%zmm0 \n" "vmovups 0x40(%0),%%zmm1 \n" @@ -7782,13 +7980,35 @@ void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { "vmovups %%zmm1,0x40(%1) \n" "lea 0x80(%1),%1 \n" "sub $0x80,%2 \n" - "jg 1b \n" + "jge 1b \n" + + "2: \n" + "add $0x80,%2 \n" + "je 99f \n" + "cmp $0x40,%2 \n" + "jl 3f \n" + "vmovups (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vmovups %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "je 99f \n" + + "3: \n" + "mov $-1,%[temp] \n" + "bzhi %q2,%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "vmovdqu8 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu8 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] : - : "memory", "cc", "xmm0", "xmm1"); + : "memory", "cc", "xmm0", "xmm1", "k1"); } #endif // HAS_COPYROW_AVX512BW @@ -8812,17 +9032,20 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0, const uint8_t* alpha, uint8_t* dst, int width) { + uintptr_t temp; asm volatile( "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n" "vpsllw $0x8,%%zmm17,%%zmm16 \n" // 0xff00 "vpabsb %%zmm17,%%zmm17 \n" // 0x0101 "vpsllw $7,%%zmm17,%%zmm17 \n" // 0x8080 - "mov $0x807f807f,%%eax \n" - "vmovd %%eax,%%xmm18 \n" + "mov $0x807f807f,%k[temp] \n" + "vmovd %k[temp],%%xmm18 \n" "vbroadcastss %%xmm18,%%zmm18 \n" // 0x807f "sub %2,%0 \n" "sub %2,%1 \n" "sub %2,%3 \n" + "sub $64,%4 \n" + "jl 2f \n" // 64 pixel loop. LABELALIGN @@ -8848,16 +9071,45 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0, "vmovdqu8 %%zmm0,(%3,%2,1) \n" "add $64,%2 \n" "sub $64,%4 \n" - "jg 1b \n" + "jge 1b \n" + + "2: \n" + "add $64,%4 \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q4,%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "vmovdqu8 (%2),%%zmm0%{%%k1%}%{z%} \n" + "vpunpckhbw %%zmm0,%%zmm0,%%zmm3 \n" + "vpunpcklbw %%zmm0,%%zmm0,%%zmm0 \n" + "vpxorq %%zmm16,%%zmm3,%%zmm3 \n" + "vpxorq %%zmm16,%%zmm0,%%zmm0 \n" + "vmovdqu8 (%0,%2,1),%%zmm1%{%%k1%}%{z%} \n" + "vmovdqu8 (%1,%2,1),%%zmm2%{%%k1%}%{z%} \n" + "vpunpckhbw %%zmm2,%%zmm1,%%zmm4 \n" + "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n" + "vpsubb %%zmm17,%%zmm4,%%zmm4 \n" + "vpsubb %%zmm17,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm4,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm18,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm3,%%zmm3 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm3,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%3,%2,1)%{%%k1%} \n" + + "99: \n" "vzeroupper \n" - : "+r"(src0), // %0 - "+r"(src1), // %1 - "+r"(alpha), // %2 - "+r"(dst), // %3 - "+rm"(width) // %4 + : "+r"(src0), // %0 + "+r"(src1), // %1 + "+r"(alpha), // %2 + "+r"(dst), // %3 + "+r"(width), // %4 + [temp] "=&r"(temp) // %[temp] ::"memory", - "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", "xmm17", - "xmm18"); + "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", "xmm17", "xmm18", + "k1"); } #endif // HAS_BLENDPLANEROW_AVX512BW @@ -10624,26 +10876,47 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, const uint8_t* shuffler, int width) { - asm volatile("vbroadcasti32x4 (%3),%%zmm5 \n" + uintptr_t temp; + asm volatile( + "vbroadcasti32x4 (%[shuffler]),%%zmm5 \n" + "sub $0x20,%2 \n" + "jl 2f \n" - LABELALIGN - "1: \n" - "vmovdqu8 (%0),%%zmm0 \n" - "vmovdqu8 0x40(%0),%%zmm1 \n" - "lea 0x80(%0),%0 \n" - "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" - "vpshufb %%zmm5,%%zmm1,%%zmm1 \n" - "vmovdqu8 %%zmm0,(%1) \n" - "vmovdqu8 %%zmm1,0x40(%1) \n" - "lea 0x80(%1),%1 \n" - "sub $0x20,%2 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_argb), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "r"(shuffler) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm5"); + LABELALIGN + "1: \n" + "vmovdqu8 (%0),%%zmm0 \n" + "vmovdqu8 0x40(%0),%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm5,%%zmm1,%%zmm1 \n" + "vmovdqu8 %%zmm0,(%1) \n" + "vmovdqu8 %%zmm1,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x20,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q2,%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "kshiftrd $16,%%k1,%%k2 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n" + "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm5,%%zmm1,%%zmm1 \n" + "vmovdqu32 %%zmm0,(%1)%{%%k1%} \n" + "vmovdqu32 %%zmm1,0x40(%1)%{%%k2%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : [shuffler] "r"(shuffler) // %[shuffler] + : "memory", "cc", "xmm0", "xmm1", "xmm5", "k1", "k2"); } #endif // HAS_ARGBSHUFFLEROW_AVX512BW diff --git a/source/row_win.cc b/source/row_win.cc index 24d4d888f..90cfb4086 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -428,7 +428,7 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, __m512i zmm_shuf = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)shuffler)); - while (width > 0) { + while (width >= 64) { __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw); __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128)); @@ -457,6 +457,42 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, dst_argb += 256; width -= 64; } + if (width > 0) { + uint64_t dst_mask = (1ULL << width) - 1ULL; + int src_bytes = width * 3; + __mmask64 src_mask0 = + (src_bytes >= 64) ? ~0ULL : ((1ULL << src_bytes) - 1ULL); + __mmask64 src_mask1 = + (src_bytes >= 128) + ? ~0ULL + : ((src_bytes > 64) ? ((1ULL << (src_bytes - 64)) - 1ULL) : 0ULL); + __mmask64 src_mask2 = + (src_bytes > 128) ? ((1ULL << (src_bytes - 128)) - 1ULL) : 0ULL; + + __m512i zmm_raw0 = _mm512_maskz_loadu_epi8(src_mask0, src_raw); + __m512i zmm_raw1 = _mm512_maskz_loadu_epi8(src_mask1, src_raw + 64); + __m512i zmm_raw2 = _mm512_maskz_loadu_epi8(src_mask2, src_raw + 128); + + __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); + __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); + __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_or_si512(zmm0, zmm_alpha); + zmm1 = _mm512_or_si512(zmm1, zmm_alpha); + zmm2 = _mm512_or_si512(zmm2, zmm_alpha); + zmm3 = _mm512_or_si512(zmm3, zmm_alpha); + + _mm512_mask_storeu_epi32(dst_argb, (__mmask16)dst_mask, zmm0); + _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(dst_mask >> 16), zmm1); + _mm512_mask_storeu_epi32(dst_argb + 128, (__mmask16)(dst_mask >> 32), zmm2); + _mm512_mask_storeu_epi32(dst_argb + 192, (__mmask16)(dst_mask >> 48), zmm3); + } _mm256_zeroupper(); } #endif // defined(HAS_RAWTOARGBROW_AVX512BW) || @@ -1630,6 +1666,18 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb, dst_argb += 128; width -= 32; } + if (width > 0) { + uint32_t mask = (1u << width) - 1u; + __mmask16 mask0 = (__mmask16)mask; + __mmask16 mask1 = (__mmask16)(mask >> 16); + __m512i row = _mm512_maskz_loadu_epi32(mask0, src_argb); + __m512i row1 = _mm512_maskz_loadu_epi32(mask1, src_argb + 64); + row = _mm512_shuffle_epi8(row, control); + row1 = _mm512_shuffle_epi8(row1, control); + _mm512_mask_storeu_epi32(dst_argb, mask0, row); + _mm512_mask_storeu_epi32(dst_argb + 64, mask1, row1); + } + _mm256_zeroupper(); } #endif // HAS_ARGBSHUFFLEROW_AVX512BW @@ -2401,6 +2449,62 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, dst_ar30 += 128; width -= 32; } + if (width > 0) { + __mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __mmask32 mask_y = (__mmask32)((1u << width) - 1u); + __m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u); + __m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); + + __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); + __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); + + zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); + zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); + + __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); + zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); + zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); + + __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); + __m512i zmm4 = _mm512_castsi256_si512(ymm_y); + zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); + zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); + zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); + + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); + + __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); + __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); + __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); + + zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); + + zmm0 = _mm512_adds_epi16(zmm0, zmm4); + zmm1 = _mm512_subs_epi16(zmm4, zmm1); + zmm2 = _mm512_adds_epi16(zmm2, zmm4); + + zmm0 = _mm512_min_epi16(zmm0, zmm_3ff0); + zmm1 = _mm512_min_epi16(zmm1, zmm_3ff0); + zmm2 = _mm512_min_epi16(zmm2, zmm_3ff0); + zmm0 = _mm512_max_epi16(zmm0, zmm_zero); + zmm1 = _mm512_max_epi16(zmm1, zmm_zero); + zmm2 = _mm512_max_epi16(zmm2, zmm_zero); + + zmm0 = _mm512_srli_epi16(zmm0, 4); + zmm3 = _mm512_srli_epi16(zmm1, 10); + zmm1 = _mm512_slli_epi16(zmm1, 6); + zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm1, zmm_fc00, 0xf8); + zmm2 = _mm512_ternarylogic_epi32(zmm2, zmm3, zmm_3ff0, 0xe4); + zmm2 = _mm512_or_si512(zmm2, zmm_c000); + + __m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi16(zmm0, zmm_perm1, zmm2); + + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)mask_y, zmm_dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(mask_y >> 16), + zmm_dst1); + } _mm256_zeroupper(); } #endif // HAS_I422TOAR30ROW_AVX512BW @@ -2564,7 +2668,7 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0, __m512i zmm_80 = _mm512_set1_epi8((char)0x80); __m512i zmm_round = _mm512_set1_epi16((short)0x807f); - while (width > 0) { + while (width >= 64) { __m512i zmm_a = _mm512_loadu_si512((const __m512i*)alpha); __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src0); __m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)src1); @@ -2596,6 +2700,34 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0, dst += 64; width -= 64; } + if (width > 0) { + __mmask64 mask = (1ULL << width) - 1ULL; + __m512i zmm_a = _mm512_maskz_loadu_epi8(mask, alpha); + __m512i zmm_s0 = _mm512_maskz_loadu_epi8(mask, src0); + __m512i zmm_s1 = _mm512_maskz_loadu_epi8(mask, src1); + + __m512i zmm_a_lo = _mm512_unpacklo_epi8(zmm_a, zmm_a); + __m512i zmm_a_hi = _mm512_unpackhi_epi8(zmm_a, zmm_a); + zmm_a_lo = _mm512_xor_si512(zmm_a_lo, zmm_ff00); + zmm_a_hi = _mm512_xor_si512(zmm_a_hi, zmm_ff00); + + __m512i zmm_s_lo = _mm512_unpacklo_epi8(zmm_s0, zmm_s1); + __m512i zmm_s_hi = _mm512_unpackhi_epi8(zmm_s0, zmm_s1); + zmm_s_lo = _mm512_sub_epi8(zmm_s_lo, zmm_80); + zmm_s_hi = _mm512_sub_epi8(zmm_s_hi, zmm_80); + + __m512i zmm_p_lo = _mm512_maddubs_epi16(zmm_a_lo, zmm_s_lo); + __m512i zmm_p_hi = _mm512_maddubs_epi16(zmm_a_hi, zmm_s_hi); + + zmm_p_lo = _mm512_add_epi16(zmm_p_lo, zmm_round); + zmm_p_hi = _mm512_add_epi16(zmm_p_hi, zmm_round); + + zmm_p_lo = _mm512_srli_epi16(zmm_p_lo, 8); + zmm_p_hi = _mm512_srli_epi16(zmm_p_hi, 8); + + __m512i zmm_d = _mm512_packus_epi16(zmm_p_lo, zmm_p_hi); + _mm512_mask_storeu_epi8(dst, mask, zmm_d); + } _mm256_zeroupper(); } #endif // HAS_BLENDPLANEROW_AVX512BW