diff --git a/include/libyuv/row.h b/include/libyuv/row.h index d3531b2de..54ba69504 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -380,6 +380,7 @@ extern "C" { #define HAS_I422TORGB24ROW_AVX512BW #define HAS_I422TOAR30ROW_AVX512BW #define HAS_RGBTOYMATRIXROW_AVX512BW +#define HAS_BLENDPLANEROW_AVX512BW #endif #define HAS_ARGBTOYMATRIXROW_AVX2 #define HAS_RGBTOYMATRIXROW_AVX2 @@ -428,6 +429,7 @@ extern "C" { #define HAS_I422TOAR30ROW_AVX512BW #define HAS_ARGBSHUFFLEROW_AVX512BW #define HAS_ARGBTOUV444MATRIXROW_AVX512BW +#define HAS_BLENDPLANEROW_AVX512BW #define HAS_RGBTOUV444MATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512VBMI @@ -558,6 +560,7 @@ extern "C" { #define HAS_ARGBADDROW_NEON #define HAS_ARGBATTENUATEROW_NEON #define HAS_ARGBBLENDROW_NEON +#define HAS_BLENDPLANEROW_NEON #define HAS_ARGBCOLORMATRIXROW_NEON #define HAS_ARGBGRAYROW_NEON #define HAS_ARGBMIRRORROW_NEON @@ -5335,6 +5338,10 @@ void ARGBBlendRow_LSX(const uint8_t* src_argb0, const uint8_t* src_argb1, uint8_t* dst_argb, int width); +void ARGBBlendRow_Any_LSX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); void ARGBBlendRow_RVV(const uint8_t* src_argb0, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -5365,6 +5372,26 @@ void BlendPlaneRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* v_buf, uint8_t* dst_ptr, int width); +void BlendPlaneRow_AVX512BW(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width); +void BlendPlaneRow_Any_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + int width); +void BlendPlaneRow_NEON(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width); +void BlendPlaneRow_Any_NEON(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + int width); void BlendPlaneRow_RVV(const uint8_t* src0, const uint8_t* src1, const uint8_t* alpha, diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 2b8a7f894..39b6c150e 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -2996,7 +2996,10 @@ int ARGBBlend(const uint8_t* src_argb0, #endif #if defined(HAS_ARGBBLENDROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { - ARGBBlendRow = ARGBBlendRow_LSX; + ARGBBlendRow = ARGBBlendRow_Any_LSX; + if (IS_ALIGNED(width, 8)) { + ARGBBlendRow = ARGBBlendRow_LSX; + } } #endif #if defined(HAS_ARGBBLENDROW_RVV) @@ -3065,6 +3068,22 @@ int BlendPlane(const uint8_t* src_y0, } } #endif +#if defined(HAS_BLENDPLANEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + BlendPlaneRow = BlendPlaneRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + BlendPlaneRow = BlendPlaneRow_AVX512BW; + } + } +#endif +#if defined(HAS_BLENDPLANEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + BlendPlaneRow = BlendPlaneRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + BlendPlaneRow = BlendPlaneRow_NEON; + } + } +#endif #if defined(HAS_BLENDPLANEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { BlendPlaneRow = BlendPlaneRow_RVV; @@ -3148,6 +3167,22 @@ int I420Blend(const uint8_t* src_y0, } } #endif +#if defined(HAS_BLENDPLANEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + BlendPlaneRow = BlendPlaneRow_Any_AVX512BW; + if (IS_ALIGNED(halfwidth, 64)) { + BlendPlaneRow = BlendPlaneRow_AVX512BW; + } + } +#endif +#if defined(HAS_BLENDPLANEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + BlendPlaneRow = BlendPlaneRow_Any_NEON; + if (IS_ALIGNED(halfwidth, 16)) { + BlendPlaneRow = BlendPlaneRow_NEON; + } + } +#endif #if defined(HAS_BLENDPLANEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { BlendPlaneRow = BlendPlaneRow_RVV; diff --git a/source/row_any.cc b/source/row_any.cc index 9ecd5dc74..ef04e9a8d 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -339,9 +339,15 @@ ANY31(I422ToUYVYRow_Any_LASX, I422ToUYVYRow_LASX, 1, 1, 4, 31) #ifdef HAS_BLENDPLANEROW_AVX2 ANY31(BlendPlaneRow_Any_AVX2, BlendPlaneRow_AVX2, 0, 0, 1, 31) #endif +#ifdef HAS_BLENDPLANEROW_AVX512BW +ANY31(BlendPlaneRow_Any_AVX512BW, BlendPlaneRow_AVX512BW, 0, 0, 1, 63) +#endif #ifdef HAS_BLENDPLANEROW_SSSE3 ANY31(BlendPlaneRow_Any_SSSE3, BlendPlaneRow_SSSE3, 0, 0, 1, 7) #endif +#ifdef HAS_BLENDPLANEROW_NEON +ANY31(BlendPlaneRow_Any_NEON, BlendPlaneRow_NEON, 0, 0, 1, 15) +#endif #undef ANY31 // Note that odd width replication includes 444 due to implementation @@ -704,6 +710,9 @@ ANY21(ARGBAddRow_Any_NEON, ARGBAddRow_NEON, 0, 4, 4, 4, 7) #ifdef HAS_ARGBSUBTRACTROW_NEON ANY21(ARGBSubtractRow_Any_NEON, ARGBSubtractRow_NEON, 0, 4, 4, 4, 7) #endif +#ifdef HAS_ARGBBLENDROW_LSX +ANY21(ARGBBlendRow_Any_LSX, ARGBBlendRow_LSX, 0, 4, 4, 4, 7) +#endif #ifdef HAS_ARGBMULTIPLYROW_LSX ANY21(ARGBMultiplyRow_Any_LSX, ARGBMultiplyRow_LSX, 0, 4, 4, 4, 3) #endif diff --git a/source/row_gcc.cc b/source/row_gcc.cc index bda52fe9d..8b90d894c 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -8292,6 +8292,66 @@ void BlendPlaneRow_AVX2(const uint8_t* src0, } #endif // HAS_BLENDPLANEROW_AVX2 +#ifdef HAS_BLENDPLANEROW_AVX512BW +// Blend 64 pixels at a time. +// unsigned version of math +// =((A2*C2)+(B2*(255-C2))+255)/256 +// signed version of math +// =(((A2-128)*C2)+((B2-128)*(255-C2))+32768+127)/256 +void BlendPlaneRow_AVX512BW(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width) { + asm volatile( + "vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n" + "vpsllw $0x8,%%zmm17,%%zmm16 \n" // 0xff00 + "vpabsb %%zmm17,%%zmm17 \n" // 0x0101 + "vpsllw $7,%%zmm17,%%zmm17 \n" // 0x8080 + "mov $0x807f807f,%%eax \n" + "vmovd %%eax,%%xmm18 \n" + "vbroadcastss %%xmm18,%%zmm18 \n" // 0x807f + "sub %2,%0 \n" + "sub %2,%1 \n" + "sub %2,%3 \n" + + // 64 pixel loop. + LABELALIGN + "1: \n" + "vmovdqu8 (%2),%%zmm0 \n" + "vpunpckhbw %%zmm0,%%zmm0,%%zmm3 \n" + "vpunpcklbw %%zmm0,%%zmm0,%%zmm0 \n" + "vpxorq %%zmm16,%%zmm3,%%zmm3 \n" + "vpxorq %%zmm16,%%zmm0,%%zmm0 \n" + "vmovdqu8 (%0,%2,1),%%zmm1 \n" + "vmovdqu8 (%1,%2,1),%%zmm2 \n" + "vpunpckhbw %%zmm2,%%zmm1,%%zmm4 \n" + "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n" + "vpsubb %%zmm17,%%zmm4,%%zmm4 \n" + "vpsubb %%zmm17,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm4,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm18,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm18,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm3,%%zmm3 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm3,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%3,%2,1) \n" + "add $64,%2 \n" + "sub $64,%4 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src0), // %0 + "+r"(src1), // %1 + "+r"(alpha), // %2 + "+r"(dst), // %3 + "+rm"(width) // %4 + ::"memory", + "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", "xmm17", + "xmm18"); +} +#endif // HAS_BLENDPLANEROW_AVX512BW + #ifdef HAS_ARGBATTENUATEROW_SSSE3 // Shuffle table duplicating alpha. static const vec8 kAttenuateShuffle = {6, -128, 6, -128, 6, -128, diff --git a/source/row_neon.cc b/source/row_neon.cc index 06de40e80..3486c7d0a 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -3102,6 +3102,68 @@ void ARGBSubtractRow_NEON(const uint8_t* src_argb, : "cc", "memory", "q0", "q1", "q2", "q3"); } +// Blend 32 pixels at a time. +// dst = (((a) * src0) + ((255 - a) * src1) + 255) >> 8 +void BlendPlaneRow_NEON(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width) { + asm volatile( + "vmov.u16 q15, #255 \n" + "subs %4, %4, #32 \n" + "blt 19f \n" + "1: \n" + "vld1.8 {q0, q1}, [%0]! \n" // load 32 src0 + "vld1.8 {q2, q3}, [%1]! \n" // load 32 src1 + "vld1.8 {q8, q9}, [%2]! \n" // load 32 alpha + "subs %4, %4, #32 \n" // 32 processed per loop + "vmvn.8 q10, q8 \n" // 255 - alpha0 + "vmvn.8 q11, q9 \n" // 255 - alpha1 + "vmull.u8 q4, d0, d16 \n" // low src0 * alpha + "pld [%0, #448] \n" + "vmull.u8 q5, d1, d17 \n" // high src0 * alpha + "pld [%1, #448] \n" + "vmull.u8 q6, d2, d18 \n" + "pld [%2, #448] \n" + "vmull.u8 q7, d3, d19 \n" + "vmlal.u8 q4, d4, d20 \n" // low + src1 * (255 - alpha) + "vmlal.u8 q5, d5, d21 \n" // high + src1 * (255 - alpha) + "vmlal.u8 q6, d6, d22 \n" + "vmlal.u8 q7, d7, d23 \n" + "vaddhn.u16 d0, q4, q15 \n" // (low + 255) >> 8 + "vaddhn.u16 d1, q5, q15 \n" // (high + 255) >> 8 + "vaddhn.u16 d2, q6, q15 \n" + "vaddhn.u16 d3, q7, q15 \n" + "vst1.8 {q0, q1}, [%3]! \n" // store 32 dst + "bge 1b \n" + "19: \n" + "adds %4, %4, #32 \n" + "ble 99f \n" + + // 16 pixel tail + "vld1.8 {q0}, [%0]! \n" // load 16 src0 + "vld1.8 {q1}, [%1]! \n" // load 16 src1 + "vld1.8 {q2}, [%2]! \n" // load 16 alpha + "vmvn.8 q12, q2 \n" // 255 - alpha + "vmull.u8 q4, d0, d4 \n" // low src0 * alpha + "vmull.u8 q5, d1, d5 \n" // high src0 * alpha + "vmlal.u8 q4, d2, d24 \n" // low + src1 * (255 - alpha) + "vmlal.u8 q5, d3, d25 \n" // high + src1 * (255 - alpha) + "vaddhn.u16 d0, q4, q15 \n" // + 255 >> 8 + "vaddhn.u16 d1, q5, q15 \n" // + 255 >> 8 + "vst1.8 {q0}, [%3]! \n" // store 16 dst + "99: \n" + : "+r"(src0), // %0 + "+r"(src1), // %1 + "+r"(alpha), // %2 + "+r"(dst), // %3 + "+r"(width) // %4 + : + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11", "q12", "q15"); +} + // Adds Sobel X and Sobel Y and stores Sobel into ARGB. // A = 255 // R = Sobel diff --git a/source/row_neon64.cc b/source/row_neon64.cc index 0fe996169..ff92df3fb 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -4276,6 +4276,64 @@ void ARGBSubtractRow_NEON(const uint8_t* src_argb, : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); } +// Blend 32 pixels at a time. +// dst = (((a) * src0) + ((255 - a) * src1) + 255) >> 8 +void BlendPlaneRow_NEON(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width) { + asm volatile( + "movi v15.8h, #255 \n" + "subs %w4, %w4, #32 \n" + "blt 19f \n" + "1: \n" + "ld1 {v0.16b, v1.16b}, [%0], #32 \n" // load 32 src0 + "ld1 {v2.16b, v3.16b}, [%1], #32 \n" // load 32 src1 + "ld1 {v4.16b, v5.16b}, [%2], #32 \n" // load 32 alpha + "subs %w4, %w4, #32 \n" // 32 processed per loop + "mvn v6.16b, v4.16b \n" // 255 - alpha0 + "mvn v7.16b, v5.16b \n" // 255 - alpha1 + "umull v8.8h, v0.8b, v4.8b \n" // low src0 * alpha + "prfm pldl1keep, [%0, 448] \n" + "umull2 v9.8h, v0.16b, v4.16b \n" // high src0 * alpha + "prfm pldl1keep, [%1, 448] \n" + "umull v10.8h, v1.8b, v5.8b \n" + "prfm pldl1keep, [%2, 448] \n" + "umull2 v11.8h, v1.16b, v5.16b \n" + "umlal v8.8h, v2.8b, v6.8b \n" // low + src1 * (255 - alpha) + "umlal2 v9.8h, v2.16b, v6.16b \n" // high + src1 * (255 - alpha) + "umlal v10.8h, v3.8b, v7.8b \n" + "umlal2 v11.8h, v3.16b, v7.16b \n" + "addhn v0.8b, v8.8h, v15.8h \n" // (low + 255) >> 8 + "addhn v1.8b, v9.8h, v15.8h \n" // (high + 255) >> 8 + "addhn v2.8b, v10.8h, v15.8h \n" + "addhn v3.8b, v11.8h, v15.8h \n" + "st1 {v0.8b, v1.8b, v2.8b, v3.8b}, [%3], #32 \n" // store 32 dst + "b.ge 1b \n" + "19: \n" + "adds %w4, %w4, #32 \n" + "b.le 99f \n" + + // 16 pixel tail + "ld1 {v0.16b}, [%0], #16 \n" // load 16 src0 + "ld1 {v1.16b}, [%1], #16 \n" // load 16 src1 + "ld1 {v2.16b}, [%2], #16 \n" // load 16 alpha + "mvn v3.16b, v2.16b \n" // 255 - alpha + "umull v4.8h, v0.8b, v2.8b \n" // low src0 * alpha + "umull2 v5.8h, v0.16b, v2.16b \n" // high src0 * alpha + "umlal v4.8h, v1.8b, v3.8b \n" // low + src1 * (255 - alpha) + "umlal2 v5.8h, v1.16b, v3.16b \n" // high + src1 * (255 - alpha) + "addhn v0.8b, v4.8h, v15.8h \n" // (low + 255) >> 8 + "addhn v1.8b, v5.8h, v15.8h \n" // (high + 255) >> 8 + "st1 {v0.8b, v1.8b}, [%3], #16 \n" // store 16 dst + "99: \n" + : "+r"(src0), "+r"(src1), "+r"(alpha), "+r"(dst), "+r"(width) + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v15"); +} + // Adds Sobel X and Sobel Y and stores Sobel into ARGB. // A = 255 // R = Sobel diff --git a/source/row_win.cc b/source/row_win.cc index 79f7dc703..125bb7ccb 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -2158,6 +2158,201 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, } #endif // HAS_I422TOAR30ROW_AVX512BW +#ifdef HAS_ARGBBLENDROW_SSSE3 +static const uint8_t kShuffleAlpha[16] = { + 3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, + 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80}; + +void ARGBBlendRow_SSSE3(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + __m128i shuff = _mm_loadu_si128((const __m128i*)kShuffleAlpha); + __m128i xmm_1 = _mm_set1_epi16(1); + __m128i mask_lo = _mm_set1_epi16(0x00ff); + __m128i mask_hi = _mm_set1_epi16((short)0xff00); + __m128i mask_alpha = _mm_set1_epi32((int)0xff000000); + + while (width >= 4) { + __m128i xmm_s0 = _mm_loadu_si128((const __m128i*)src_argb); + __m128i xmm_s1 = _mm_loadu_si128((const __m128i*)src_argb1); + src_argb += 16; + src_argb1 += 16; + + __m128i xmm_a = _mm_xor_si128(xmm_s0, mask_alpha); + xmm_a = _mm_shuffle_epi8(xmm_a, shuff); + xmm_a = _mm_add_epi16(xmm_a, xmm_1); + + __m128i xmm_s1_lo = _mm_and_si128(xmm_s1, mask_lo); + __m128i xmm_s1_hi = _mm_srli_epi16(xmm_s1, 8); + + xmm_s1_lo = _mm_mullo_epi16(xmm_s1_lo, xmm_a); + xmm_s1_hi = _mm_mullo_epi16(xmm_s1_hi, xmm_a); + + xmm_s1_lo = _mm_srli_epi16(xmm_s1_lo, 8); + xmm_s1_hi = _mm_and_si128(xmm_s1_hi, mask_hi); + + __m128i res = _mm_or_si128(xmm_s0, mask_alpha); + res = _mm_adds_epu8(res, xmm_s1_lo); + res = _mm_adds_epu8(res, xmm_s1_hi); + + _mm_storeu_si128((__m128i*)dst_argb, res); + dst_argb += 16; + width -= 4; + } + + while (width > 0) { + uint32_t s0 = *(const uint32_t*)src_argb; + uint32_t s1 = *(const uint32_t*)src_argb1; + src_argb += 4; + src_argb1 += 4; + + uint32_t a = 256 - (s0 >> 24); + uint32_t b = (((s1 & 0x000000ff) * a) >> 8) + (s0 & 0x000000ff); + uint32_t g = (((s1 & 0x0000ff00) * a) >> 8) + (s0 & 0x0000ff00); + uint32_t r = (((s1 & 0x00ff0000) >> 8) * a) + (s0 & 0x00ff0000); + + b = (b > 255) ? 255 : b; + g = (g > 0xff00) ? 0xff00 : (g & 0xff00); + r = (r > 0x00ff0000) ? 0x00ff0000 : (r & 0x00ff0000); + + *(uint32_t*)dst_argb = 0xff000000 | r | g | b; + dst_argb += 4; + --width; + } +} +#endif // HAS_ARGBBLENDROW_SSSE3 + +#ifdef HAS_BLENDPLANEROW_SSSE3 +void BlendPlaneRow_SSSE3(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width) { + __m128i xmm_ff00 = _mm_set1_epi16((short)0xff00); + __m128i xmm_80 = _mm_set1_epi8((char)0x80); + __m128i xmm_round = _mm_set1_epi16((short)0x807f); + + while (width > 0) { + __m128i xmm_a = _mm_loadl_epi64((const __m128i*)alpha); + __m128i xmm_s0 = _mm_loadl_epi64((const __m128i*)src0); + __m128i xmm_s1 = _mm_loadl_epi64((const __m128i*)src1); + alpha += 8; + src0 += 8; + src1 += 8; + + xmm_a = _mm_unpacklo_epi8(xmm_a, xmm_a); + xmm_a = _mm_xor_si128(xmm_a, xmm_ff00); + + __m128i xmm_s = _mm_unpacklo_epi8(xmm_s0, xmm_s1); + xmm_s = _mm_sub_epi8(xmm_s, xmm_80); + + __m128i xmm_p = _mm_maddubs_epi16(xmm_a, xmm_s); + xmm_p = _mm_add_epi16(xmm_p, xmm_round); + xmm_p = _mm_srli_epi16(xmm_p, 8); + + __m128i xmm_d = _mm_packus_epi16(xmm_p, xmm_p); + _mm_storel_epi64((__m128i*)dst, xmm_d); + dst += 8; + width -= 8; + } +} +#endif // HAS_BLENDPLANEROW_SSSE3 + +#ifdef HAS_BLENDPLANEROW_AVX2 +LIBYUV_TARGET_AVX2 +void BlendPlaneRow_AVX2(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width) { + __m256i ymm_ff00 = _mm256_set1_epi16((short)0xff00); + __m256i ymm_80 = _mm256_set1_epi8((char)0x80); + __m256i ymm_round = _mm256_set1_epi16((short)0x807f); + + while (width > 0) { + __m256i ymm_a = _mm256_loadu_si256((const __m256i*)alpha); + __m256i ymm_s0 = _mm256_loadu_si256((const __m256i*)src0); + __m256i ymm_s1 = _mm256_loadu_si256((const __m256i*)src1); + alpha += 32; + src0 += 32; + src1 += 32; + + __m256i ymm_a_lo = _mm256_unpacklo_epi8(ymm_a, ymm_a); + __m256i ymm_a_hi = _mm256_unpackhi_epi8(ymm_a, ymm_a); + ymm_a_lo = _mm256_xor_si256(ymm_a_lo, ymm_ff00); + ymm_a_hi = _mm256_xor_si256(ymm_a_hi, ymm_ff00); + + __m256i ymm_s_lo = _mm256_unpacklo_epi8(ymm_s0, ymm_s1); + __m256i ymm_s_hi = _mm256_unpackhi_epi8(ymm_s0, ymm_s1); + ymm_s_lo = _mm256_sub_epi8(ymm_s_lo, ymm_80); + ymm_s_hi = _mm256_sub_epi8(ymm_s_hi, ymm_80); + + __m256i ymm_p_lo = _mm256_maddubs_epi16(ymm_a_lo, ymm_s_lo); + __m256i ymm_p_hi = _mm256_maddubs_epi16(ymm_a_hi, ymm_s_hi); + + ymm_p_lo = _mm256_add_epi16(ymm_p_lo, ymm_round); + ymm_p_hi = _mm256_add_epi16(ymm_p_hi, ymm_round); + + ymm_p_lo = _mm256_srli_epi16(ymm_p_lo, 8); + ymm_p_hi = _mm256_srli_epi16(ymm_p_hi, 8); + + __m256i ymm_d = _mm256_packus_epi16(ymm_p_lo, ymm_p_hi); + _mm256_storeu_si256((__m256i*)dst, ymm_d); + dst += 32; + width -= 32; + } + _mm256_zeroupper(); +} +#endif // HAS_BLENDPLANEROW_AVX2 + +#ifdef HAS_BLENDPLANEROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void BlendPlaneRow_AVX512BW(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width) { + __m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00); + __m512i zmm_80 = _mm512_set1_epi8((char)0x80); + __m512i zmm_round = _mm512_set1_epi16((short)0x807f); + + while (width > 0) { + __m512i zmm_a = _mm512_loadu_si512((const __m512i*)alpha); + __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src0); + __m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)src1); + alpha += 64; + src0 += 64; + src1 += 64; + + __m512i zmm_a_lo = _mm512_unpacklo_epi8(zmm_a, zmm_a); + __m512i zmm_a_hi = _mm512_unpackhi_epi8(zmm_a, zmm_a); + zmm_a_lo = _mm512_xor_si512(zmm_a_lo, zmm_ff00); + zmm_a_hi = _mm512_xor_si512(zmm_a_hi, zmm_ff00); + + __m512i zmm_s_lo = _mm512_unpacklo_epi8(zmm_s0, zmm_s1); + __m512i zmm_s_hi = _mm512_unpackhi_epi8(zmm_s0, zmm_s1); + zmm_s_lo = _mm512_sub_epi8(zmm_s_lo, zmm_80); + zmm_s_hi = _mm512_sub_epi8(zmm_s_hi, zmm_80); + + __m512i zmm_p_lo = _mm512_maddubs_epi16(zmm_a_lo, zmm_s_lo); + __m512i zmm_p_hi = _mm512_maddubs_epi16(zmm_a_hi, zmm_s_hi); + + zmm_p_lo = _mm512_add_epi16(zmm_p_lo, zmm_round); + zmm_p_hi = _mm512_add_epi16(zmm_p_hi, zmm_round); + + zmm_p_lo = _mm512_srli_epi16(zmm_p_lo, 8); + zmm_p_hi = _mm512_srli_epi16(zmm_p_hi, 8); + + __m512i zmm_d = _mm512_packus_epi16(zmm_p_lo, zmm_p_hi); + _mm512_storeu_si512((__m512i*)dst, zmm_d); + dst += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif // HAS_BLENDPLANEROW_AVX512BW + #ifdef __cplusplus } // extern "C" } // namespace libyuv