diff --git a/include/libyuv/planar_functions.h b/include/libyuv/planar_functions.h index b93b729df..6306cd75e 100644 --- a/include/libyuv/planar_functions.h +++ b/include/libyuv/planar_functions.h @@ -855,6 +855,18 @@ int ARGBBlend(const uint8_t* src_argb0, int width, int height); +// Alpha Blend unattenuated ARGB images and store to destination. +// Source and destination are not pre-multiplied by alpha. +LIBYUV_API +int ARGBAlphaBlend(const uint8_t* src_argb0, + int src_stride_argb0, + const uint8_t* src_argb1, + int src_stride_argb1, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height); + // Alpha Blend plane and store to destination. // Source is not pre-multiplied by alpha. LIBYUV_API @@ -929,6 +941,17 @@ int ARGBSubtract(const uint8_t* src_argb0, int width, int height); +// Bitwise OR ARGB image with a specified ARGB value (e.g. 0xff000000 to force +// opaque alpha). +LIBYUV_API +int ARGBOr(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height, + uint32_t value); + // Convert I422 to YUY2. LIBYUV_API int I422ToYUY2(const uint8_t* src_y, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index e189d9d81..c7c67bd0f 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -78,6 +78,7 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_SSE2 #define HAS_ARGBAFFINEROW_SSE2 +#define HAS_ARGBALPHABLENDROW_SSSE3 #define HAS_ARGBBLENDROW_SSSE3 #define HAS_ARGBCOLORMATRIXROW_SSSE3 #define HAS_ARGBCOLORTABLEROW_X86 @@ -87,6 +88,7 @@ extern "C" { #define HAS_ARGBLUMACOLORTABLEROW_SSSE3 #define HAS_ARGBMIRRORROW_SSE2 #define HAS_ARGBMULTIPLYROW_SSE2 +#define HAS_ARGBORROW_SSE2 #define HAS_ARGBPOLYNOMIALROW_SSE2 #define HAS_ARGBQUANTIZEROW_SSE2 #define HAS_ARGBSEPIAROW_SSSE3 @@ -180,7 +182,9 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_AVX2 +#define HAS_ARGBALPHABLENDROW_AVX2 #define HAS_ARGBMULTIPLYROW_AVX2 +#define HAS_ARGBORROW_AVX2 #define HAS_ARGBSUBTRACTROW_AVX2 #define HAS_BLENDPLANEROW_AVX2 #endif @@ -409,7 +413,13 @@ extern "C" { #define HAS_I400TOARGBROW_AVX512BW #define HAS_ARGBGRAYROW_AVX512BW #define HAS_ARGBBLENDROW_AVX512BW +#define HAS_ARGBALPHABLENDROW_AVX512BW +#define HAS_ARGBORROW_AVX512BW #endif +#define HAS_ARGBALPHABLENDROW_SSSE3 +#define HAS_ARGBORROW_SSE2 +#define HAS_ARGBALPHABLENDROW_AVX2 +#define HAS_ARGBORROW_AVX2 #define HAS_ARGBTOYMATRIXROW_AVX2 #define HAS_RGBTOYMATRIXROW_AVX2 #define HAS_RGB565TOYMATRIXROW_AVX2 @@ -508,6 +518,8 @@ extern "C" { #define HAS_I400TOARGBROW_AVX512BW #define HAS_ARGBGRAYROW_AVX512BW #define HAS_ARGBBLENDROW_AVX512BW +#define HAS_ARGBALPHABLENDROW_AVX512BW +#define HAS_ARGBORROW_AVX512BW #endif // The following are available on Neon platforms: @@ -625,6 +637,7 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_NEON +#define HAS_ARGBALPHABLENDROW_NEON #define HAS_ARGBATTENUATEROW_NEON #define HAS_ARGBBLENDROW_NEON #define HAS_BLENDPLANEROW_NEON @@ -632,6 +645,7 @@ extern "C" { #define HAS_ARGBGRAYROW_NEON #define HAS_ARGBMIRRORROW_NEON #define HAS_ARGBMULTIPLYROW_NEON +#define HAS_ARGBORROW_NEON #define HAS_ARGBQUANTIZEROW_NEON #define HAS_ARGBSEPIAROW_NEON #define HAS_ARGBSHADEROW_NEON @@ -683,6 +697,8 @@ extern "C" { // The following are available on AArch64 SVE platforms: #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #define HAS_ARGB1555TOARGBROW_SVE2 +#define HAS_ARGBALPHABLENDROW_SVE2 +#define HAS_ARGBORROW_SVE2 #define HAS_ARGBTORAWROW_SVE2 #define HAS_ARGBTORGB24ROW_SVE2 #define HAS_ARGBTORGB565DITHERROW_SVE2 @@ -748,7 +764,9 @@ extern "C" { // The following are available on AArch64 SME platforms: #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ defined(__aarch64__) +#define HAS_ARGBALPHABLENDROW_SME #define HAS_ARGBMULTIPLYROW_SME +#define HAS_ARGBORROW_SME #define HAS_ARGBTOUVMATRIXROW_SME #define HAS_ARGBTOYMATRIXROW_SME #define HAS_CONVERT16TO8ROW_SME @@ -807,11 +825,13 @@ extern "C" { #define HAS_ARGB1555TOYROW_LSX #define HAS_ARGB4444TOARGBROW_LSX #define HAS_ARGBADDROW_LSX +#define HAS_ARGBALPHABLENDROW_LSX #define HAS_ARGBATTENUATEROW_LSX #define HAS_ARGBBLENDROW_LSX #define HAS_ARGBCOLORMATRIXROW_LSX #define HAS_ARGBEXTRACTALPHAROW_LSX #define HAS_ARGBGRAYROW_LSX +#define HAS_ARGBORROW_LSX #define HAS_ARGBSEPIAROW_LSX #define HAS_ARGBSHADEROW_LSX #define HAS_ARGBSHUFFLEROW_LSX @@ -881,6 +901,7 @@ extern "C" { #define HAS_ARGB1555TOYROW_LASX #define HAS_ARGB4444TOARGBROW_LASX #define HAS_ARGBADDROW_LASX +#define HAS_ARGBALPHABLENDROW_LASX #define HAS_ARGBATTENUATEROW_LASX #define HAS_ARGBBLENDROW_LASX #define HAS_ARGBCOLORMATRIXROW_LASX @@ -888,6 +909,7 @@ extern "C" { #define HAS_ARGBGRAYROW_LASX #define HAS_ARGBMIRRORROW_LASX #define HAS_ARGBMULTIPLYROW_LASX +#define HAS_ARGBORROW_LASX #define HAS_ARGBQUANTIZEROW_LASX #define HAS_ARGBSEPIAROW_LASX #define HAS_ARGBSETROW_LASX @@ -977,8 +999,10 @@ extern "C" { #define HAS_SWAPUVROW_RVV #define HAS_AB64TOARGBROW_RVV #define HAS_AR64TOAB64ROW_RVV +#define HAS_ARGBALPHABLENDROW_RVV #define HAS_ARGBATTENUATEROW_RVV #define HAS_ARGBBLENDROW_RVV +#define HAS_ARGBORROW_RVV #define HAS_ARGBTOAB64ROW_RVV #define HAS_ARGBTOABGRROW_RVV #define HAS_ARGBTOBGRAROW_RVV @@ -1018,10 +1042,12 @@ extern "C" { #define HAS_ARGB4444TOARGBROW_WASMSIMD #define HAS_ARGB4444TOUVMATRIXROW_WASMSIMD #define HAS_ARGB4444TOYMATRIXROW_WASMSIMD +#define HAS_ARGBALPHABLENDROW_WASMSIMD #define HAS_ARGBBLENDROW_WASMSIMD #define HAS_ARGBCOPYYTOALPHAROW_WASMSIMD #define HAS_ARGBEXTRACTALPHAROW_WASMSIMD #define HAS_ARGBMIRRORROW_WASMSIMD +#define HAS_ARGBORROW_WASMSIMD #define HAS_ARGBSHUFFLEROW_WASMSIMD #define HAS_ARGBTOARGB1555ROW_WASMSIMD #define HAS_ARGBTOARGB4444ROW_WASMSIMD @@ -1067,6 +1093,7 @@ extern "C" { #define HAS_AR64TOAB64ROW_HVX #define HAS_AR64TOARGBROW_HVX #define HAS_ARGBADDROW_HVX +#define HAS_ARGBALPHABLENDROW_HVX #define HAS_ARGBATTENUATEROW_HVX #define HAS_ARGBBLENDROW_HVX #define HAS_ARGBCOPYALPHAROW_HVX @@ -1074,6 +1101,7 @@ extern "C" { #define HAS_ARGBEXTRACTALPHAROW_HVX #define HAS_ARGBMIRRORROW_HVX #define HAS_ARGBMULTIPLYROW_HVX +#define HAS_ARGBORROW_HVX #define HAS_ARGBSETROW_HVX #define HAS_ARGBSUBTRACTROW_HVX #define HAS_ARGBTOAB64ROW_HVX @@ -5933,6 +5961,154 @@ void ARGBSubtractRow_Any_LASX(const uint8_t* y_buf, uint8_t* dst_ptr, int width); +// ARGB bitwise OR with 32-bit value. +void ARGBOrRow_C(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_SSE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_SSE2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_SVE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_SME(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_LASX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_RVV(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_HVX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); + +// Unattenuated ARGB alpha blend. +void ARGBAlphaBlendRow_C(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_SSSE3(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_AVX2(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_NEON(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_NEON(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_SVE2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_SME(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_LSX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_LSX(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_LASX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_RVV(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_WASMSIMD(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_HVX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); + void ARGBToRGB24Row_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 19559b64f..7faf57db8 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -2360,6 +2360,160 @@ static inline float SsimWindowRow_SVE_SC(const uint32_t* row0, return total; } +static inline void ARGBOrRow_SVE_SC(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) STREAMING_COMPATIBLE { + uint32_t vl; + asm volatile( + "cntw %x[vl] \n" + "dup z2.s, %w[value] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + "ptrue p0.s \n" + "1: \n" + "ld1w {z0.s}, p0/z, [%[src]] \n" + "ld1w {z1.s}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" + "orr z0.d, z0.d, z2.d \n" + "orr z1.d, z1.d, z2.d \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "st1w {z0.s}, p0, [%[dst]] \n" + "st1w {z1.s}, p0, [%[dst], #1, mul vl] \n" + "incb %[dst], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[width] \n" + "whilelt p1.s, %w[vl], %w[width] \n" + "ld1w {z0.s}, p0/z, [%[src]] \n" + "ld1w {z1.s}, p1/z, [%[src], #1, mul vl] \n" + "orr z0.d, z0.d, z2.d \n" + "orr z1.d, z1.d, z2.d \n" + "st1w {z0.s}, p0, [%[dst]] \n" + "st1w {z1.s}, p1, [%[dst], #1, mul vl] \n" + + "99: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [value] "r"(value) // %[value] + : "memory", "cc", "z0", "z1", "z2", "p0", "p1"); +} + +#define ARGBALPHABLEND_SVE_2X(pred0, pred1) \ + "ld1w {z0.s}, " #pred0 "/z, [%[src0]] \n" \ + "ld1w {z1.s}, " #pred1 "/z, [%[src0], #1, mul vl] \n" \ + "ld1w {z2.s}, " #pred0 "/z, [%[src1]] \n" \ + "ld1w {z3.s}, " #pred1 "/z, [%[src1], #1, mul vl] \n" \ + "incb %[src0], all, mul #2 \n" \ + "incb %[src1], all, mul #2 \n" \ + "lsr z4.s, z0.s, #24 \n" \ + "lsr z5.s, z1.s, #24 \n" \ + "lsr z6.s, z2.s, #24 \n" \ + "lsr z7.s, z3.s, #24 \n" \ + "mov z16.d, z24.d \n" \ + "mov z17.d, z24.d \n" \ + "umlalb z16.h, z6.b, z4.b \n" \ + "umlalb z17.h, z7.b, z5.b \n" \ + "usra z16.h, z16.h, #8 \n" \ + "usra z17.h, z17.h, #8 \n" \ + "lsr z16.h, z16.h, #8 \n" \ + "lsr z17.h, z17.h, #8 \n" \ + "sub z16.s, z4.s, z16.s \n" \ + "sub z17.s, z5.s, z17.s \n" \ + "add z16.s, z6.s, z16.s \n" \ + "add z17.s, z7.s, z17.s \n" \ + "lsl z18.s, z4.s, #8 \n" \ + "lsl z19.s, z5.s, #8 \n" \ + "sub z18.s, z18.s, z4.s \n" \ + "sub z19.s, z19.s, z5.s \n" \ + "sub z6.s, z6.s, #1 \n" \ + "sub z7.s, z7.s, #1 \n" \ + "asr z6.s, z6.s, #31 \n" \ + "asr z7.s, z7.s, #31 \n" \ + "udiv z18.s, p0/m, z18.s, z16.s \n" \ + "udiv z19.s, p0/m, z19.s, z17.s \n" \ + "mul z18.s, z18.s, z25.s \n" \ + "mul z19.s, z19.s, z25.s \n" \ + "orr z18.d, z18.d, z6.d \n" \ + "orr z19.d, z19.d, z7.d \n" \ + "not z20.b, p0/m, z18.b \n" \ + "not z21.b, p0/m, z19.b \n" \ + "mov z4.d, z24.d \n" \ + "mov z5.d, z24.d \n" \ + "mov z6.d, z24.d \n" \ + "mov z7.d, z24.d \n" \ + "umlalb z4.h, z2.b, z20.b \n" \ + "umlalt z5.h, z2.b, z20.b \n" \ + "umlalb z6.h, z3.b, z21.b \n" \ + "umlalt z7.h, z3.b, z21.b \n" \ + "umlalb z4.h, z0.b, z18.b \n" \ + "umlalt z5.h, z0.b, z18.b \n" \ + "umlalb z6.h, z1.b, z19.b \n" \ + "umlalt z7.h, z1.b, z19.b \n" \ + "usra z4.h, z4.h, #8 \n" \ + "usra z5.h, z5.h, #8 \n" \ + "usra z6.h, z6.h, #8 \n" \ + "usra z7.h, z7.h, #8 \n" \ + "shrnb z0.b, z4.h, #8 \n" \ + "shrnt z0.b, z5.h, #8 \n" \ + "shrnb z1.b, z6.h, #8 \n" \ + "shrnt z1.b, z7.h, #8 \n" \ + "and z0.d, z0.d, z26.d \n" \ + "and z1.d, z1.d, z26.d \n" \ + "lsl z16.s, z16.s, #24 \n" \ + "lsl z17.s, z17.s, #24 \n" \ + "orr z0.d, z0.d, z16.d \n" \ + "orr z1.d, z1.d, z17.d \n" \ + "st1w {z0.s}, " #pred0 ", [%[dst]] \n" \ + "st1w {z1.s}, " #pred1 ", [%[dst], #1, mul vl] \n" \ + "incb %[dst], all, mul #2 \n" + +static inline void ARGBAlphaBlendRow_SVE_SC(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) STREAMING_COMPATIBLE { + uint32_t vl; + asm volatile( + "cntw %x[vl] \n" + "ptrue p0.b \n" + "mov z24.h, #1 \n" + "mov z25.s, #0x01010101 \n" + "mov z26.s, #0x00ffffff \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + "1: \n" + ARGBALPHABLEND_SVE_2X(p0, p0) + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + "whilelt p1.s, wzr, %w[width] \n" + "whilelt p2.s, %w[vl], %w[width] \n" + ARGBALPHABLEND_SVE_2X(p1, p2) + + "99: \n" + : [src0] "+r"(src_argb0), // %[src0] + [src1] "+r"(src_argb1), // %[src1] + [dst] "+r"(dst_argb), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", + "z17", "z18", "z19", "z20", "z21", "z24", "z25", "z26", "p0", "p1", + "p2"); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/planar_functions.cc b/source/planar_functions.cc index ecf6fc8a9..4901686ad 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -3468,6 +3468,118 @@ int ARGBBlend(const uint8_t* src_argb0, return 0; } +// Alpha Blend 2 unattenuated ARGB images and store to destination. +LIBYUV_API +int ARGBAlphaBlend(const uint8_t* src_argb0, + int src_stride_argb0, + const uint8_t* src_argb1, + int src_stride_argb1, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height) { + int y; + void (*ARGBAlphaBlendRow)(const uint8_t* src_argb, const uint8_t* src_argb1, + uint8_t* dst_argb, int width) = ARGBAlphaBlendRow_C; + if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0 || + height == INT_MIN) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_argb0 == width * 4 && src_stride_argb1 == width * 4 && + dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; + } +#if defined(HAS_ARGBALPHABLENDROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_SSSE3; + if (IS_ALIGNED(width, 4)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_AVX512BW; + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_NEON; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_SVE2; + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_SME; + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_LSX; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_LASX; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_RVV; + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_HVX; + } +#endif + for (y = 0; y < height; ++y) { + ARGBAlphaBlendRow(src_argb0, src_argb1, dst_argb, width); + src_argb0 += src_stride_argb0; + src_argb1 += src_stride_argb1; + dst_argb += dst_stride_argb; + } + return 0; +} + // Alpha Blend plane and store to destination. LIBYUV_API int BlendPlane(const uint8_t* src_y0, @@ -4030,6 +4142,116 @@ int ARGBSubtract(const uint8_t* src_argb0, return 0; } +// Bitwise OR ARGB image with a specified ARGB value. +LIBYUV_API +int ARGBOr(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height, + uint32_t value) { + int y; + void (*ARGBOrRow)(const uint8_t* src_argb, uint8_t* dst_argb, uint32_t value, + int width) = ARGBOrRow_C; + if (!src_argb || !dst_argb || width <= 0 || height == 0 || + height == INT_MIN) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 && + (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBORROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBOrRow = ARGBOrRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBOrRow = ARGBOrRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBORROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBOrRow = ARGBOrRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBOrRow = ARGBOrRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBORROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBOrRow = ARGBOrRow_AVX512BW; + } +#endif +#if defined(HAS_ARGBORROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBOrRow = ARGBOrRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBOrRow = ARGBOrRow_NEON; + } + } +#endif +#if defined(HAS_ARGBORROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBOrRow = ARGBOrRow_SVE2; + } +#endif +#if defined(HAS_ARGBORROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBOrRow = ARGBOrRow_SME; + } +#endif +#if defined(HAS_ARGBORROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBOrRow = ARGBOrRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBOrRow = ARGBOrRow_LSX; + } + } +#endif +#if defined(HAS_ARGBORROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBOrRow = ARGBOrRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBOrRow = ARGBOrRow_LASX; + } + } +#endif +#if defined(HAS_ARGBORROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBOrRow = ARGBOrRow_RVV; + } +#endif +#if defined(HAS_ARGBORROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBOrRow = ARGBOrRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBOrRow = ARGBOrRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBORROW_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + ARGBOrRow = ARGBOrRow_HVX; + } +#endif + for (y = 0; y < height; ++y) { + ARGBOrRow(src_argb, dst_argb, value, width); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + // Convert RAW to RGB24. LIBYUV_API int RAWToRGB24(const uint8_t* src_raw, diff --git a/source/row_any.cc b/source/row_any.cc index f2f41ea68..57806d43c 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -751,6 +751,24 @@ ANY21(ARGBSubtractRow_Any_LSX, ARGBSubtractRow_LSX, 0, 4, 4, 4, 3) #ifdef HAS_ARGBSUBTRACTROW_LASX ANY21(ARGBSubtractRow_Any_LASX, ARGBSubtractRow_LASX, 0, 4, 4, 4, 7) #endif +#ifdef HAS_ARGBALPHABLENDROW_SSSE3 +ANY21(ARGBAlphaBlendRow_Any_SSSE3, ARGBAlphaBlendRow_SSSE3, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBALPHABLENDROW_AVX2 +ANY21(ARGBAlphaBlendRow_Any_AVX2, ARGBAlphaBlendRow_AVX2, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBALPHABLENDROW_NEON +ANY21(ARGBAlphaBlendRow_Any_NEON, ARGBAlphaBlendRow_NEON, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBALPHABLENDROW_LSX +ANY21(ARGBAlphaBlendRow_Any_LSX, ARGBAlphaBlendRow_LSX, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBALPHABLENDROW_LASX +ANY21(ARGBAlphaBlendRow_Any_LASX, ARGBAlphaBlendRow_LASX, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBALPHABLENDROW_WASMSIMD +ANY21(ARGBAlphaBlendRow_Any_WASMSIMD, ARGBAlphaBlendRow_WASMSIMD, 0, 4, 4, 4, 3) +#endif #ifdef HAS_SOBELROW_SSE2 ANY21(SobelRow_Any_SSE2, SobelRow_SSE2, 0, 1, 1, 4, 15) #endif @@ -1542,6 +1560,24 @@ ANY11P(ARGBShuffleRow_Any_WASMSIMD, 4, 3) #endif +#ifdef HAS_ARGBORROW_SSE2 +ANY11P(ARGBOrRow_Any_SSE2, ARGBOrRow_SSE2, const uint32_t, 4, 4, 3) +#endif +#ifdef HAS_ARGBORROW_AVX2 +ANY11P(ARGBOrRow_Any_AVX2, ARGBOrRow_AVX2, const uint32_t, 4, 4, 7) +#endif +#ifdef HAS_ARGBORROW_NEON +ANY11P(ARGBOrRow_Any_NEON, ARGBOrRow_NEON, const uint32_t, 4, 4, 7) +#endif +#ifdef HAS_ARGBORROW_LSX +ANY11P(ARGBOrRow_Any_LSX, ARGBOrRow_LSX, const uint32_t, 4, 4, 3) +#endif +#ifdef HAS_ARGBORROW_LASX +ANY11P(ARGBOrRow_Any_LASX, ARGBOrRow_LASX, const uint32_t, 4, 4, 7) +#endif +#ifdef HAS_ARGBORROW_WASMSIMD +ANY11P(ARGBOrRow_Any_WASMSIMD, ARGBOrRow_WASMSIMD, const uint32_t, 4, 4, 3) +#endif #undef ANY11P #undef ANY11P diff --git a/source/row_common.cc b/source/row_common.cc index 8fcec481a..1bb42ed9b 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -3473,6 +3473,67 @@ void ARGBBlendRow_C(const uint8_t* src_argb, } #undef BLEND +// Blend unattenuated src_argb0 over src_argb1 and store to dst_argb. +void ARGBAlphaBlendRow_C(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + int x; + for (x = 0; x < width; ++x) { + int sa = src_argb0[3]; + int da = src_argb1[3]; + if (da == 0) { + dst_argb[0] = src_argb0[0]; + dst_argb[1] = src_argb0[1]; + dst_argb[2] = src_argb0[2]; + dst_argb[3] = STATIC_CAST(uint8_t, sa); + } else if (sa == 0) { + dst_argb[0] = src_argb1[0]; + dst_argb[1] = src_argb1[1]; + dst_argb[2] = src_argb1[2]; + dst_argb[3] = STATIC_CAST(uint8_t, da); + } else { + int dst_alpha = da + sa - da * sa / 255; + int alpha_ratio = sa * 255 / dst_alpha; + dst_argb[0] = STATIC_CAST( + uint8_t, + (src_argb1[0] * (255 - alpha_ratio) + src_argb0[0] * alpha_ratio) / + 255); + dst_argb[1] = STATIC_CAST( + uint8_t, + (src_argb1[1] * (255 - alpha_ratio) + src_argb0[1] * alpha_ratio) / + 255); + dst_argb[2] = STATIC_CAST( + uint8_t, + (src_argb1[2] * (255 - alpha_ratio) + src_argb0[2] * alpha_ratio) / + 255); + dst_argb[3] = STATIC_CAST(uint8_t, dst_alpha); + } + src_argb0 += 4; + src_argb1 += 4; + dst_argb += 4; + } +} + +void ARGBOrRow_C(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + const uint8_t v0 = value & 0xff; + const uint8_t v1 = (value >> 8) & 0xff; + const uint8_t v2 = (value >> 16) & 0xff; + const uint8_t v3 = (value >> 24) & 0xff; + int i; + for (i = 0; i < width; ++i) { + dst_argb[0] = src_argb[0] | v0; + dst_argb[1] = src_argb[1] | v1; + dst_argb[2] = src_argb[2] | v2; + dst_argb[3] = src_argb[3] | v3; + src_argb += 4; + dst_argb += 4; + } +} + #define UBLEND(f, b, a) (((a) * f) + ((255 - a) * b) + 255) >> 8 void BlendPlaneRow_C(const uint8_t* src0, const uint8_t* src1, diff --git a/source/row_gcc.cc b/source/row_gcc.cc index d2a345e30..ee9f648ee 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -13470,6 +13470,424 @@ void OMITFP ARGBBlendRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBBLENDROW_AVX512BW +#ifdef HAS_ARGBORROW_SSE2 +void ARGBOrRow_SSE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "movd %[value],%%xmm1 \n" + "pshufd $0x00,%%xmm1,%%xmm1 \n" + + LABELALIGN + "1: \n" + "movdqu (%[src]),%%xmm0 \n" + "lea 0x10(%[src]),%[src] \n" + "por %%xmm1,%%xmm0 \n" + "movdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "sub $0x4,%[width] \n" + "jg 1b \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width) + : [value] "r"(value) + : "memory", "cc", "xmm0", "xmm1"); +} +#endif // HAS_ARGBORROW_SSE2 + +#ifdef HAS_ARGBORROW_AVX2 +void ARGBOrRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "vmovd %[value],%%xmm1 \n" + "vpbroadcastd %%xmm1,%%ymm1 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width) + : [value] "r"(value) + : "memory", "cc", "xmm0", "xmm1"); +} +#endif // HAS_ARGBORROW_AVX2 + +#ifdef HAS_ARGBORROW_AVX512BW +void OMITFP ARGBOrRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + uintptr_t temp; + asm volatile( + "vpbroadcastd %[value],%%zmm1 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width), + [temp] "=&r"(temp) + : [value] "r"(value) + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_ARGBORROW_AVX512BW + +#ifdef HAS_ARGBALPHABLENDROW_SSSE3 +void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "pcmpeqb %%xmm5,%%xmm5 \n" + "pabsb %%xmm5,%%xmm4 \n" // 0x01010101... + "psllw $7,%%xmm4 \n" + "movdqa %%xmm4,%%xmm6 \n" // xmm6 = 0x8080... + "psrlw $7,%%xmm4 \n" // xmm4 = 0x0101... + "psubw %%xmm4,%%xmm6 \n" // 0x7f7f... + "psubw %%xmm5,%%xmm6 \n" // 0x7f80... + "psubw %%xmm5,%%xmm6 \n" // xmm6 = 0x7f81... + + LABELALIGN + "1: \n" + "movdqu (%[src0]),%%xmm0 \n" + "movdqu (%[src1]),%%xmm1 \n" + "lea 0x10(%[src0]),%[src0] \n" + "lea 0x10(%[src1]),%[src1] \n" + + "movdqa %%xmm0,%%xmm2 \n" + "movdqa %%xmm1,%%xmm3 \n" + "psrld $24,%%xmm2 \n" // sa + "psrld $24,%%xmm3 \n" // da + + "pcmpeqb %%xmm5,%%xmm5 \n" // -1 + "movdqa %%xmm2,%%xmm7 \n" + "pmullw %%xmm3,%%xmm7 \n" // da * sa + "psubw %%xmm5,%%xmm7 \n" // da * sa + 1 + "pmulhuw %%xmm4,%%xmm7 \n" // (da * sa) / 255 + + "paddd %%xmm3,%%xmm5 \n" // da - 1 + "psrld $24,%%xmm5 \n" // t = (da==0) ? 255 : 0 + + "paddw %%xmm2,%%xmm3 \n" // da + sa + "psubw %%xmm7,%%xmm3 \n" // da2 + + "por %%xmm5,%%xmm2 \n" // sa_eff + "movdqa %%xmm3,%%xmm7 \n" + "por %%xmm5,%%xmm7 \n" // den_eff + + "movdqa %%xmm2,%%xmm5 \n" + "pslld $8,%%xmm2 \n" + "psubd %%xmm5,%%xmm2 \n" // sa_eff * 255 + + "cvtdq2ps %%xmm2,%%xmm2 \n" + "cvtdq2ps %%xmm7,%%xmm7 \n" + "divps %%xmm7,%%xmm2 \n" + "cvttps2dq %%xmm2,%%xmm2 \n" // ratio + + "pcmpeqb %%xmm5,%%xmm5 \n" + "psllw $8,%%xmm5 \n" // 0xff00 + + "pmullw %%xmm4,%%xmm2 \n" + "pshufhw $0xa0,%%xmm2,%%xmm2 \n" + "pshuflw $0xa0,%%xmm2,%%xmm2 \n" + "pxor %%xmm5,%%xmm2 \n" + "movdqa %%xmm2,%%xmm7 \n" + "punpcklwd %%xmm2,%%xmm2 \n" + "punpckhwd %%xmm7,%%xmm7 \n" + + "movdqa %%xmm4,%%xmm5 \n" + "psllw $7,%%xmm5 \n" // 0x8080 + "psubb %%xmm5,%%xmm0 \n" + "psubb %%xmm5,%%xmm1 \n" + "movdqa %%xmm0,%%xmm5 \n" + "punpcklbw %%xmm1,%%xmm0 \n" + "punpckhbw %%xmm1,%%xmm5 \n" + + "pmaddubsw %%xmm0,%%xmm2 \n" + "pmaddubsw %%xmm5,%%xmm7 \n" + "paddw %%xmm6,%%xmm2 \n" + "paddw %%xmm6,%%xmm7 \n" + "pmulhuw %%xmm4,%%xmm2 \n" + "pmulhuw %%xmm4,%%xmm7 \n" + "packuswb %%xmm7,%%xmm2 \n" + + "pslld $8,%%xmm2 \n" + "psrld $8,%%xmm2 \n" + "pslld $24,%%xmm3 \n" + "por %%xmm3,%%xmm2 \n" + + "movdqu %%xmm2,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "sub $0x4,%[width] \n" + "jg 1b \n" + : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [width] "+rm"(width) + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_ARGBALPHABLENDROW_SSSE3 + +#ifdef HAS_ARGBALPHABLENDROW_AVX2 +void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpabsb %%ymm5,%%ymm4 \n" // 0x0101... + "vpsllw $7,%%ymm4,%%ymm6 \n" // 0x8080... + "vpsubw %%ymm4,%%ymm6,%%ymm6 \n" // 0x7f7f... + "vpsubw %%ymm5,%%ymm6,%%ymm6 \n" // 0x7f80... + "vpsubw %%ymm5,%%ymm6,%%ymm6 \n" // 0x7f81... + + LABELALIGN + "1: \n" + "vmovdqu (%[src0]),%%ymm0 \n" + "vmovdqu (%[src1]),%%ymm1 \n" + "lea 0x20(%[src0]),%[src0] \n" + "lea 0x20(%[src1]),%[src1] \n" + + "vpsrld $24,%%ymm0,%%ymm2 \n" // sa + "vpsrld $24,%%ymm1,%%ymm3 \n" // da + + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // -1 + "vpmullw %%ymm3,%%ymm2,%%ymm7 \n" // da * sa + "vpsubw %%ymm5,%%ymm7,%%ymm7 \n" // da * sa + 1 + "vpmulhuw %%ymm4,%%ymm7,%%ymm7 \n" // (da * sa) / 255 + + "vpaddd %%ymm3,%%ymm5,%%ymm5 \n" // da - 1 + "vpsrld $24,%%ymm5,%%ymm5 \n" // t = (da==0) ? 255 : 0 + + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" // da + sa + "vpsubw %%ymm7,%%ymm3,%%ymm3 \n" // da2 + + "vpor %%ymm5,%%ymm2,%%ymm2 \n" // sa_eff + "vpor %%ymm5,%%ymm3,%%ymm7 \n" // den_eff + + "vpslld $8,%%ymm2,%%ymm5 \n" + "vpsubd %%ymm2,%%ymm5,%%ymm2 \n" // sa_eff * 255 + + "vcvtdq2ps %%ymm2,%%ymm2 \n" + "vcvtdq2ps %%ymm7,%%ymm7 \n" + "vdivps %%ymm7,%%ymm2,%%ymm2 \n" + "vcvttps2dq %%ymm2,%%ymm2 \n" // ratio + + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsllw $8,%%ymm5,%%ymm5 \n" // 0xff00 + + "vpmullw %%ymm4,%%ymm2,%%ymm2 \n" + "vpshufhw $0xa0,%%ymm2,%%ymm2 \n" + "vpshuflw $0xa0,%%ymm2,%%ymm2 \n" + "vpxor %%ymm5,%%ymm2,%%ymm2 \n" + "vpunpckhwd %%ymm2,%%ymm2,%%ymm7 \n" + "vpunpcklwd %%ymm2,%%ymm2,%%ymm2 \n" + + "vpsllw $7,%%ymm4,%%ymm5 \n" // 0x8080 + "vpsubb %%ymm5,%%ymm0,%%ymm0 \n" + "vpsubb %%ymm5,%%ymm1,%%ymm1 \n" + "vpunpckhbw %%ymm1,%%ymm0,%%ymm5 \n" + "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" + + "vpmaddubsw %%ymm0,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm5,%%ymm7,%%ymm7 \n" + "vpaddw %%ymm6,%%ymm2,%%ymm2 \n" + "vpaddw %%ymm6,%%ymm7,%%ymm7 \n" + "vpmulhuw %%ymm4,%%ymm2,%%ymm2 \n" + "vpmulhuw %%ymm4,%%ymm7,%%ymm7 \n" + "vpackuswb %%ymm7,%%ymm2,%%ymm2 \n" + + "vpslld $8,%%ymm2,%%ymm2 \n" + "vpsrld $8,%%ymm2,%%ymm2 \n" + "vpslld $24,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vmovdqu %%ymm2,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [width] "+rm"(width) + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_ARGBALPHABLENDROW_AVX2 + +#ifdef HAS_ARGBALPHABLENDROW_AVX512BW +void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + uintptr_t temp; + asm volatile( + "vpternlogd $0xff,%%zmm20,%%zmm20,%%zmm20 \n" // -1 + "vpabsb %%zmm20,%%zmm16 \n" // zmm16 = 0x0101... + "vpsllw $7,%%zmm16,%%zmm21 \n" // zmm21 = 0x8080... + "vpsubw %%zmm16,%%zmm21,%%zmm18 \n" // 0x7f7f... + "vpsubw %%zmm20,%%zmm18,%%zmm18 \n" // 0x7f80... + "vpsubw %%zmm20,%%zmm18,%%zmm18 \n" // zmm18 = 0x7f81... + "vpsllw $8,%%zmm20,%%zmm17 \n" // zmm17 = 0xff00... + "vpsrld $24,%%zmm20,%%zmm19 \n" // zmm19 = 255 in dwords + + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src0]),%%zmm0 \n" + "vmovdqu32 (%[src1]),%%zmm1 \n" + "lea 0x40(%[src0]),%[src0] \n" + "lea 0x40(%[src1]),%[src1] \n" + + "vpsrld $24,%%zmm0,%%zmm2 \n" // sa + "vpsrld $24,%%zmm1,%%zmm3 \n" // da + + "vpmullw %%zmm3,%%zmm2,%%zmm7 \n" + "vpsubw %%zmm20,%%zmm7,%%zmm7 \n" // da * sa + 1 + "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n" + + "vpaddd %%zmm3,%%zmm20,%%zmm5 \n" // da - 1 + "vpsrld $24,%%zmm5,%%zmm5 \n" // t + + "vpaddw %%zmm2,%%zmm3,%%zmm3 \n" + "vpsubw %%zmm7,%%zmm3,%%zmm3 \n" // da2 + + "vpord %%zmm5,%%zmm2,%%zmm2 \n" // sa_eff + "vpord %%zmm5,%%zmm3,%%zmm7 \n" // den_eff + "vpmulld %%zmm19,%%zmm2,%%zmm2 \n" // sa_eff * 255 + + "vcvtdq2ps %%zmm2,%%zmm2 \n" + "vcvtdq2ps %%zmm7,%%zmm7 \n" + "vdivps %%zmm7,%%zmm2,%%zmm2 \n" + "vcvttps2dq %%zmm2,%%zmm2 \n" + + "vpmullw %%zmm16,%%zmm2,%%zmm2 \n" + "vpshufhw $0xa0,%%zmm2,%%zmm2 \n" + "vpshuflw $0xa0,%%zmm2,%%zmm2 \n" + "vpxord %%zmm17,%%zmm2,%%zmm2 \n" + "vpunpckhwd %%zmm2,%%zmm2,%%zmm7 \n" + "vpunpcklwd %%zmm2,%%zmm2,%%zmm2 \n" + + "vpsubb %%zmm21,%%zmm0,%%zmm0 \n" + "vpsubb %%zmm21,%%zmm1,%%zmm1 \n" + "vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n" + "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" + + "vpmaddubsw %%zmm0,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm5,%%zmm7,%%zmm7 \n" + "vpaddw %%zmm18,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm7,%%zmm7 \n" + "vpmulhuw %%zmm16,%%zmm2,%%zmm2 \n" + "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n" + "vpackuswb %%zmm7,%%zmm2,%%zmm2 \n" + + "vpslld $8,%%zmm2,%%zmm2 \n" + "vpsrld $8,%%zmm2,%%zmm2 \n" + "vpslld $24,%%zmm3,%%zmm3 \n" + "vpord %%zmm3,%%zmm2,%%zmm2 \n" + + "vmovdqu32 %%zmm2,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src0]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 (%[src1]),%%zmm1%{%%k1%}%{z%} \n" + + "vpsrld $24,%%zmm0,%%zmm2 \n" + "vpsrld $24,%%zmm1,%%zmm3 \n" + + "vpmullw %%zmm3,%%zmm2,%%zmm7 \n" + "vpsubw %%zmm20,%%zmm7,%%zmm7 \n" + "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n" + + "vpaddd %%zmm3,%%zmm20,%%zmm5 \n" + "vpsrld $24,%%zmm5,%%zmm5 \n" + + "vpaddw %%zmm2,%%zmm3,%%zmm3 \n" + "vpsubw %%zmm7,%%zmm3,%%zmm3 \n" + + "vpord %%zmm5,%%zmm2,%%zmm2 \n" + "vpord %%zmm5,%%zmm3,%%zmm7 \n" + "vpmulld %%zmm19,%%zmm2,%%zmm2 \n" + + "vcvtdq2ps %%zmm2,%%zmm2 \n" + "vcvtdq2ps %%zmm7,%%zmm7 \n" + "vdivps %%zmm7,%%zmm2,%%zmm2 \n" + "vcvttps2dq %%zmm2,%%zmm2 \n" + + "vpmullw %%zmm16,%%zmm2,%%zmm2 \n" + "vpshufhw $0xa0,%%zmm2,%%zmm2 \n" + "vpshuflw $0xa0,%%zmm2,%%zmm2 \n" + "vpxord %%zmm17,%%zmm2,%%zmm2 \n" + "vpunpckhwd %%zmm2,%%zmm2,%%zmm7 \n" + "vpunpcklwd %%zmm2,%%zmm2,%%zmm2 \n" + + "vpsubb %%zmm21,%%zmm0,%%zmm0 \n" + "vpsubb %%zmm21,%%zmm1,%%zmm1 \n" + "vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n" + "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" + + "vpmaddubsw %%zmm0,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm5,%%zmm7,%%zmm7 \n" + "vpaddw %%zmm18,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm7,%%zmm7 \n" + "vpmulhuw %%zmm16,%%zmm2,%%zmm2 \n" + "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n" + "vpackuswb %%zmm7,%%zmm2,%%zmm2 \n" + + "vpslld $8,%%zmm2,%%zmm2 \n" + "vpsrld $8,%%zmm2,%%zmm2 \n" + "vpslld $24,%%zmm3,%%zmm3 \n" + "vpord %%zmm3,%%zmm2,%%zmm2 \n" + + "vmovdqu32 %%zmm2,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [width] "+r"(width), [temp] "=&r"(temp) + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1"); +} +#endif // HAS_ARGBALPHABLENDROW_AVX512BW + #endif // !defined(LIBYUV_DISABLE_X86) && // (defined(__x86_64__) || defined(__i386__)) && // !defined(LIBYUV_ENABLE_ROWWIN) diff --git a/source/row_hvx.cc b/source/row_hvx.cc index de6c4e4f0..1a8be0689 100644 --- a/source/row_hvx.cc +++ b/source/row_hvx.cc @@ -112,6 +112,25 @@ void ARGBSetRow_HVX(uint8_t* dst_argb, uint32_t v32, int width) { } } +void ARGBOrRow_HVX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + HVX_Vector v = Q6_V_vsplat_R(value); + int bytes = width * 4; + while (bytes >= 128) { + HVX_Vector s = vloadu(src_argb); + vstoreu(dst_argb, Q6_V_vor_VV(s, v)); + src_argb += 128; + dst_argb += 128; + bytes -= 128; + } + if (bytes > 0) { + HVX_Vector s = vloadu_partial_zero(src_argb, bytes); + vstoreu_partial(dst_argb, Q6_V_vor_VV(s, v), bytes); + } +} + void MirrorRow_HVX(const uint8_t* src, uint8_t* dst, int width) { HVX_Vector ctrl = Q6_Vb_vsplat_R(0x7F); src += width; @@ -639,6 +658,73 @@ void ARGBBlendRow_HVX(const uint8_t* src_argb, } } +void ARGBAlphaBlendRow_HVX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + HVX_Vector vzero = Q6_V_vzero(); + HVX_Vector v0001 = Q6_Vh_vsplat_R(1); + HVX_Vector v00ff = Q6_Vh_vsplat_R(255); + HVX_Vector vffff = Q6_Vh_vsplat_R(-1); + HVX_Vector v_rgb_mask = Q6_V_vsplat_R(0x00FFFFFFu); + int bytes = width * 4; + while (bytes > 0) { + int b_cnt = bytes >= 128 ? 128 : bytes; + HVX_Vector fg = vloadu_partial_zero(src_argb0, b_cnt); + HVX_Vector bg = vloadu_partial_zero(src_argb1, b_cnt); + + // Broadcast sa and da to all 4 byte lanes of each 32-bit pixel. + HVX_Vector t_fg = Q6_Vb_vshuffo_VbVb(fg, fg); + HVX_Vector sa_vec = Q6_Vh_vshuffo_VhVh(t_fg, t_fg); + HVX_Vector t_bg = Q6_Vb_vshuffo_VbVb(bg, bg); + HVX_Vector da_vec = Q6_Vh_vshuffo_VhVh(t_bg, t_bg); + + // Both uint16_t halfwords of each pixel hold sa and da (0..255). + HVX_Vector sa16 = Q6_V_vand_VV(sa_vec, v00ff); + HVX_Vector da16 = Q6_V_vand_VV(da_vec, v00ff); + + // x = da * sa in both uint16_t halfwords of each pixel. + HVX_Vector x = Q6_V_lo_W(Q6_Wuh_vmpy_VubVub(da_vec, sa_vec)); + // prod_div255 = (x + 1 + (x >> 8)) >> 8. + HVX_Vector prod_div255 = Q6_Vuh_vlsr_VuhR( + Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(x, v0001), Q6_Vuh_vlsr_VuhR(x, 8)), 8); + // out_a = da + sa - (da * sa) / 255. + HVX_Vector out_a16 = + Q6_Vh_vadd_VhVh(da16, Q6_Vh_vsub_VhVh(sa16, prod_div255)); + + // 8-step restoring division: ratio = (da == 0) ? 255 : (sa * 255) / out_a. + HVX_Vector rem = Q6_Vh_vsub_VhVh(Q6_Vh_vasl_VhR(sa16, 8), sa16); + HVX_VectorPred q_da_zero = Q6_Q_vcmp_eq_VhVh(da16, vzero); + rem = Q6_V_vmux_QVV(q_da_zero, vffff, rem); + HVX_Vector ratio = vzero; + + for (int bit = 7; bit >= 0; --bit) { + HVX_Vector d_sh = Q6_Vh_vasl_VhR(out_a16, bit); + HVX_VectorPred q_gt = Q6_Q_vcmp_gt_VuhVuh(d_sh, rem); + rem = Q6_V_vmux_QVV(q_gt, rem, Q6_Vh_vsub_VhVh(rem, d_sh)); + ratio = Q6_V_vmux_QVV(q_gt, ratio, + Q6_V_vor_VV(ratio, Q6_Vb_vsplat_R(1 << bit))); + } + + // Exact RGB lerp: (bg * (255 - ratio) + fg * ratio) / 255. + HVX_Vector inv_ratio = Q6_V_vnot_V(ratio); + HVX_VectorPair p_rgb = Q6_Wuh_vmpy_VubVub(fg, ratio); + p_rgb = Q6_Wuh_vmpyacc_WuhVubVub(p_rgb, bg, inv_ratio); + HVX_Vector lo = Q6_V_lo_W(p_rgb); + HVX_Vector hi = Q6_V_hi_W(p_rgb); + lo = Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(lo, v0001), Q6_Vuh_vlsr_VuhR(lo, 8)); + hi = Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(hi, v0001), Q6_Vuh_vlsr_VuhR(hi, 8)); + HVX_Vector out = Q6_Vb_vshuffo_VbVb(hi, lo); + out = + Q6_V_vor_VV(Q6_V_vand_VV(out, v_rgb_mask), Q6_Vw_vasl_VwR(out_a16, 24)); + vstoreu_partial(dst_argb, out, b_cnt); + src_argb0 += 128; + src_argb1 += 128; + dst_argb += 128; + bytes -= 128; + } +} + void ARGBAttenuateRow_HVX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { diff --git a/source/row_lasx.cc b/source/row_lasx.cc index 99042acd2..ea466412f 100644 --- a/source/row_lasx.cc +++ b/source/row_lasx.cc @@ -1965,6 +1965,89 @@ void ARGBBlendRow_LASX(const uint8_t* src_argb0, "$xr7", "$xr8"); } +void ARGBOrRow_LASX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "xvreplgr2vr.w $xr1, %[val]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvor.v $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(value) + : "memory", "$xr0", "$xr1"); +} + +void ARGBAlphaBlendRow_LASX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + static const uint8_t kShufRatio[16] = {0, 0, 0, 0, 4, 4, 4, 4, + 8, 8, 8, 8, 12, 12, 12, 12}; + asm volatile( + "vld $vr8, %[shuf], 0\n\t" + "xvpermi.q $xr8, $xr8, 0x00\n\t" + "xvldi $xr9, 1\n\t" + "xvsrli.h $xr10, $xr9, 8\n\t" + "xvldi $xr11, 0xff\n\t" + "xvsrli.w $xr11, $xr11, 8\n\t" + "xvldi $xr12, 0\n\t" + "1:\n\t" + "xvld $xr0, %[s0], 0\n\t" + "xvld $xr1, %[s1], 0\n\t" + "xvsrli.w $xr2, $xr0, 24\n\t" + "xvsrli.w $xr3, $xr1, 24\n\t" + "xvmul.h $xr4, $xr3, $xr2\n\t" + "xvadd.h $xr4, $xr4, $xr10\n\t" + "xvmuh.hu $xr4, $xr4, $xr9\n\t" + "xvsub.w $xr4, $xr2, $xr4\n\t" + "xvadd.w $xr4, $xr3, $xr4\n\t" + "xvseq.w $xr5, $xr3, $xr12\n\t" + "xvsrli.w $xr5, $xr5, 24\n\t" + "xvor.v $xr6, $xr2, $xr5\n\t" + "xvor.v $xr7, $xr4, $xr5\n\t" + "xvslli.w $xr5, $xr6, 8\n\t" + "xvsub.w $xr5, $xr5, $xr6\n\t" + "xvffint.s.wu $xr5, $xr5\n\t" + "xvffint.s.wu $xr7, $xr7\n\t" + "xvfdiv.s $xr5, $xr5, $xr7\n\t" + "xvftintrz.wu.s $xr5, $xr5\n\t" + "xvshuf.b $xr5, $xr5, $xr5, $xr8\n\t" + "xvnor.v $xr6, $xr5, $xr5\n\t" + "xvmulwev.h.bu $xr2, $xr1, $xr6\n\t" + "xvmulwod.h.bu $xr3, $xr1, $xr6\n\t" + "xvmulwev.h.bu $xr7, $xr0, $xr5\n\t" + "xvmulwod.h.bu $xr0, $xr0, $xr5\n\t" + "xvadd.h $xr2, $xr2, $xr7\n\t" + "xvadd.h $xr3, $xr3, $xr0\n\t" + "xvadd.h $xr2, $xr2, $xr10\n\t" + "xvadd.h $xr3, $xr3, $xr10\n\t" + "xvmuh.hu $xr2, $xr2, $xr9\n\t" + "xvmuh.hu $xr3, $xr3, $xr9\n\t" + "xvslli.h $xr3, $xr3, 8\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvand.v $xr2, $xr2, $xr11\n\t" + "xvslli.w $xr4, $xr4, 24\n\t" + "xvor.v $xr2, $xr2, $xr4\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [shuf] "r"(kShufRatio) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12"); +} + void ARGBShadeRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width, diff --git a/source/row_lsx.cc b/source/row_lsx.cc index 8bd10f519..cdfe51ace 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -1683,6 +1683,88 @@ void ARGBBlendRow_LSX(const uint8_t* src_argb0, "$vr7", "$vr8"); } +void ARGBOrRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "vreplgr2vr.w $vr1, %[val]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(value) + : "memory", "$vr0", "$vr1"); +} + +void ARGBAlphaBlendRow_LSX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + static const uint8_t kShufRatio[16] = {0, 0, 0, 0, 4, 4, 4, 4, + 8, 8, 8, 8, 12, 12, 12, 12}; + asm volatile( + "vld $vr8, %[shuf], 0\n\t" + "vldi $vr9, 1\n\t" + "vsrli.h $vr10, $vr9, 8\n\t" + "vldi $vr11, 0xff\n\t" + "vsrli.w $vr11, $vr11, 8\n\t" + "vldi $vr12, 0\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vsrli.w $vr2, $vr0, 24\n\t" + "vsrli.w $vr3, $vr1, 24\n\t" + "vmul.h $vr4, $vr3, $vr2\n\t" + "vadd.h $vr4, $vr4, $vr10\n\t" + "vmuh.hu $vr4, $vr4, $vr9\n\t" + "vsub.w $vr4, $vr2, $vr4\n\t" + "vadd.w $vr4, $vr3, $vr4\n\t" + "vseq.w $vr5, $vr3, $vr12\n\t" + "vsrli.w $vr5, $vr5, 24\n\t" + "vor.v $vr6, $vr2, $vr5\n\t" + "vor.v $vr7, $vr4, $vr5\n\t" + "vslli.w $vr5, $vr6, 8\n\t" + "vsub.w $vr5, $vr5, $vr6\n\t" + "vffint.s.wu $vr5, $vr5\n\t" + "vffint.s.wu $vr7, $vr7\n\t" + "vfdiv.s $vr5, $vr5, $vr7\n\t" + "vftintrz.wu.s $vr5, $vr5\n\t" + "vshuf.b $vr5, $vr5, $vr5, $vr8\n\t" + "vnor.v $vr6, $vr5, $vr5\n\t" + "vmulwev.h.bu $vr2, $vr1, $vr6\n\t" + "vmulwod.h.bu $vr3, $vr1, $vr6\n\t" + "vmulwev.h.bu $vr7, $vr0, $vr5\n\t" + "vmulwod.h.bu $vr0, $vr0, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr7\n\t" + "vadd.h $vr3, $vr3, $vr0\n\t" + "vadd.h $vr2, $vr2, $vr10\n\t" + "vadd.h $vr3, $vr3, $vr10\n\t" + "vmuh.hu $vr2, $vr2, $vr9\n\t" + "vmuh.hu $vr3, $vr3, $vr9\n\t" + "vslli.h $vr3, $vr3, 8\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vand.v $vr2, $vr2, $vr11\n\t" + "vslli.w $vr4, $vr4, 24\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [shuf] "r"(kShufRatio) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12"); +} + void ARGBShadeRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width, diff --git a/source/row_neon.cc b/source/row_neon.cc index 8f921c1e4..ea072ee9b 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -2822,6 +2822,103 @@ void ARGBBlendRow_NEON(const uint8_t* src_argb, : "cc", "memory", "q0", "q1", "q2", "q3", "q10", "q11", "q12"); } +void ARGBOrRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "vdup.32 q2, %3 \n" + "1: \n" + "vld1.8 {q0, q1}, [%0]! \n" + "subs %2, %2, #8 \n" + "vorr q0, q0, q2 \n" + "vorr q1, q1, q2 \n" + "vst1.8 {q0, q1}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(value) // %3 + : "memory", "cc", "q0", "q1", "q2"); +} + +void ARGBAlphaBlendRow_NEON(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "vmov.u8 d28, #255 \n" + "vmov.i16 q15, #1 \n" + "1: \n" + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" + "vld4.8 {d4, d5, d6, d7}, [%1]! \n" + "subs %3, %3, #8 \n" + "vmov q8, q15 \n" + "vmlal.u8 q8, d7, d3 \n" + "vsra.u16 q8, q8, #8 \n" + "vshrn.i16 d16, q8, #8 \n" + "vsub.i8 d16, d3, d16 \n" + "vadd.i8 d7, d7, d16 \n" + "vceq.i8 d17, d7, #0 \n" + "vorr d18, d3, d17 \n" + "vorr d19, d7, d17 \n" + "vmull.u8 q8, d18, d28 \n" + "vmovl.u8 q9, d19 \n" + "vmovl.u16 q10, d16 \n" + "vmovl.u16 q11, d17 \n" + "vmovl.u16 q12, d18 \n" + "vmovl.u16 q13, d19 \n" + "vcvt.f32.u32 q10, q10 \n" + "vcvt.f32.u32 q11, q11 \n" + "vcvt.f32.u32 q12, q12 \n" + "vcvt.f32.u32 q13, q13 \n" + "vrecpe.f32 q8, q12 \n" + "vrecpe.f32 q9, q13 \n" + "vrecps.f32 q4, q12, q8 \n" + "vrecps.f32 q5, q13, q9 \n" + "vmul.f32 q8, q8, q4 \n" + "vmul.f32 q9, q9, q5 \n" + "vrecps.f32 q4, q12, q8 \n" + "vrecps.f32 q5, q13, q9 \n" + "vmul.f32 q8, q8, q4 \n" + "vmul.f32 q9, q9, q5 \n" + "vsra.u32 q8, q8, #29 \n" + "vsra.u32 q9, q9, #29 \n" + "vmul.f32 q10, q10, q8 \n" + "vmul.f32 q11, q11, q9 \n" + "vcvt.u32.f32 q10, q10 \n" + "vcvt.u32.f32 q11, q11 \n" + "vmovn.i32 d16, q10 \n" + "vmovn.i32 d17, q11 \n" + "vmovn.i16 d16, q8 \n" + "vsub.i8 d17, d28, d16 \n" + "vmov q9, q15 \n" + "vmov q10, q15 \n" + "vmov q11, q15 \n" + "vmlal.u8 q9, d4, d17 \n" + "vmlal.u8 q10, d5, d17 \n" + "vmlal.u8 q11, d6, d17 \n" + "vmlal.u8 q9, d0, d16 \n" + "vmlal.u8 q10, d1, d16 \n" + "vmlal.u8 q11, d2, d16 \n" + "vsra.u16 q9, q9, #8 \n" + "vsra.u16 q10, q10, #8 \n" + "vsra.u16 q11, q11, #8 \n" + "vshrn.i16 d0, q9, #8 \n" + "vshrn.i16 d1, q10, #8 \n" + "vshrn.i16 d2, q11, #8 \n" + "vmov d3, d7 \n" + "vst4.8 {d0, d1, d2, d3}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9", "q10", + "q11", "q12", "q13", "q14", "q15"); +} + // Attenuate 8 pixels at a time. void ARGBAttenuateRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, diff --git a/source/row_neon64.cc b/source/row_neon64.cc index f4e0315c6..f23ddcc8f 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -3866,6 +3866,91 @@ void ARGBBlendRow_NEON(const uint8_t* src_argb, "v17", "v18"); } +void ARGBOrRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "dup v2.4s, %w3 \n" + "1: \n" + "ldp q0, q1, [%0], #32 \n" + "subs %w2, %w2, #8 \n" + "orr v0.16b, v0.16b, v2.16b \n" + "orr v1.16b, v1.16b, v2.16b \n" + "stp q0, q1, [%1], #32 \n" + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(value) // %3 + : "memory", "cc", "v0", "v1", "v2"); +} + +void ARGBAlphaBlendRow_NEON(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "movi v28.8b, #255 \n" + "movi v29.8h, #1 \n" + "1: \n" + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" + "subs %w3, %w3, #8 \n" + "mov v16.16b, v29.16b \n" + "umlal v16.8h, v7.8b, v3.8b \n" + "usra v16.8h, v16.8h, #8 \n" + "shrn v16.8b, v16.8h, #8 \n" + "cmeq v17.8b, v7.8b, #0 \n" + "sub v16.8b, v3.8b, v16.8b \n" + "add v7.8b, v7.8b, v16.8b \n" + "orr v18.8b, v3.8b, v17.8b \n" + "orr v19.8b, v7.8b, v17.8b \n" + "umull v16.8h, v18.8b, v28.8b \n" + "uxtl v17.8h, v19.8b \n" + "uxtl v20.4s, v16.4h \n" + "uxtl2 v21.4s, v16.8h \n" + "uxtl v22.4s, v17.4h \n" + "uxtl2 v23.4s, v17.8h \n" + "ucvtf v20.4s, v20.4s \n" + "ucvtf v21.4s, v21.4s \n" + "ucvtf v22.4s, v22.4s \n" + "ucvtf v23.4s, v23.4s \n" + "fdiv v20.4s, v20.4s, v22.4s \n" + "fdiv v21.4s, v21.4s, v23.4s \n" + "fcvtzu v20.4s, v20.4s \n" + "fcvtzu v21.4s, v21.4s \n" + "xtn v16.4h, v20.4s \n" + "xtn2 v16.8h, v21.4s \n" + "xtn v16.8b, v16.8h \n" + "sub v17.8b, v28.8b, v16.8b \n" + "mov v20.16b, v29.16b \n" + "mov v21.16b, v29.16b \n" + "mov v22.16b, v29.16b \n" + "umlal v20.8h, v4.8b, v17.8b \n" + "umlal v21.8h, v5.8b, v17.8b \n" + "umlal v22.8h, v6.8b, v17.8b \n" + "umlal v20.8h, v0.8b, v16.8b \n" + "umlal v21.8h, v1.8b, v16.8b \n" + "umlal v22.8h, v2.8b, v16.8b \n" + "usra v20.8h, v20.8h, #8 \n" + "usra v21.8h, v21.8h, #8 \n" + "usra v22.8h, v22.8h, #8 \n" + "shrn v0.8b, v20.8h, #8 \n" + "shrn v1.8b, v21.8h, #8 \n" + "shrn v2.8b, v22.8h, #8 \n" + "mov v3.8b, v7.8b \n" + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v28", "v29"); +} + // Attenuate 8 pixels at a time. void ARGBAttenuateRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 6b7173675..2f899ca3c 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -2245,6 +2245,99 @@ void ARGBBlendRow_RVV(const uint8_t* src_argb, } #endif +#ifdef HAS_ARGBORROW_RVV +void ARGBOrRow_RVV(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + size_t vl; + int w = width * 4; + asm volatile( + "vsetvli %[vl], zero, e32, m8, ta, ma \n" + "vmv.v.x v16, %[val] \n" + "1: \n" + "vsetvli %[vl], %[w], e8, m8, ta, ma \n" + "vle8.v v8, (%[src]) \n" + "vor.vv v8, v8, v16 \n" + "vse8.v v8, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src], %[src], %[vl] \n" + "add %[dst], %[dst], %[vl] \n" + "bgtz %[w], 1b \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(w), // %[w] + [vl] "=&r"(vl) // %[vl] + : [val] "r"(value) // %[val] + : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} +#endif + +#ifdef HAS_ARGBALPHABLENDROW_RVV +void ARGBAlphaBlendRow_RVV(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v16, (%[src_argb]) \n" + "vlseg4e8.v v24, (%[src_argb1]) \n" + "vwmulu.vv v4, v30, v22 \n" + "vwmulu.vx v8, v22, %[k255] \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vi v4, v4, 1 \n" + "vmulhu.vx v4, v4, %[k0101] \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v0, v4, 0 \n" + "vsub.vv v0, v22, v0 \n" + "vadd.vv v30, v30, v0 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vzext.vf2 v12, v30 \n" + "vdivu.vv v8, v8, v12 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v0, v8, 0 \n" + "vrsub.vx v2, v0, %[k255] \n" + "vwmulu.vv v4, v24, v2 \n" + "vwmulu.vv v8, v26, v2 \n" + "vwmulu.vv v12, v28, v2 \n" + "vwmaccu.vv v4, v0, v16 \n" + "vwmaccu.vv v8, v0, v18 \n" + "vwmaccu.vv v12, v0, v20 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vi v4, v4, 1 \n" + "vadd.vi v8, v8, 1 \n" + "vadd.vi v12, v12, 1 \n" + "vmulhu.vx v4, v4, %[k0101] \n" + "vmulhu.vx v8, v8, %[k0101] \n" + "vmulhu.vx v12, v12, %[k0101] \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v24, v4, 0 \n" + "vnsrl.wi v26, v8, 0 \n" + "vnsrl.wi v28, v12, 0 \n" + "vsseg4e8.v v24, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl] \n" + "add %[src_argb1], %[src_argb1], %[vl]\n" + "add %[dst_argb], %[dst_argb], %[vl] \n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [src_argb1] "+r"(src_argb1), // %[src_argb1] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [k0101] "r"(0x0101), // %[k0101] + [k255] "r"(255) // %[k255] + : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30", "v31"); +} +#endif + #ifdef HAS_BLENDPLANEROW_RVV void BlendPlaneRow_RVV(const uint8_t* src0, const uint8_t* src1, diff --git a/source/row_sme.cc b/source/row_sme.cc index 5ec51382c..21abac922 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -1142,6 +1142,24 @@ __arm_locally_streaming float SsimWindowRow_SME(const uint32_t* row0, } #endif // HAS_SSIMWINDOWROW_SME +#ifdef HAS_ARGBORROW_SME +__arm_locally_streaming void ARGBOrRow_SME(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + ARGBOrRow_SVE_SC(src_argb, dst_argb, value, width); +} +#endif // HAS_ARGBORROW_SME + +#ifdef HAS_ARGBALPHABLENDROW_SME +__arm_locally_streaming void ARGBAlphaBlendRow_SME(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + ARGBAlphaBlendRow_SVE_SC(src_argb0, src_argb1, dst_argb, width); +} +#endif // HAS_ARGBALPHABLENDROW_SME + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/row_sve.cc b/source/row_sve.cc index 2cec59d30..28c3b1a16 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1751,6 +1751,24 @@ float SsimWindowRow_SVE2(const uint32_t* row0, } #endif // HAS_SSIMWINDOWROW_SVE2 +#ifdef HAS_ARGBORROW_SVE2 +void ARGBOrRow_SVE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + ARGBOrRow_SVE_SC(src_argb, dst_argb, value, width); +} +#endif // HAS_ARGBORROW_SVE2 + +#ifdef HAS_ARGBALPHABLENDROW_SVE2 +void ARGBAlphaBlendRow_SVE2(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + ARGBAlphaBlendRow_SVE_SC(src_argb0, src_argb1, dst_argb, width); +} +#endif // HAS_ARGBALPHABLENDROW_SVE2 + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/row_wasm.cc b/source/row_wasm.cc index a68774edd..df9e3917a 100644 --- a/source/row_wasm.cc +++ b/source/row_wasm.cc @@ -1970,6 +1970,73 @@ void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBCOPYYTOALPHAROW_WASMSIMD +#ifdef HAS_ARGBORROW_WASMSIMD +void ARGBOrRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + v128_t val = wasm_i32x4_splat((int32_t)value); + do { + asm("" : "+r"(src_argb), "+r"(dst_argb)); + v128_t s = wasm_v128_load(src_argb); + wasm_v128_store(dst_argb, wasm_v128_or(s, val)); + src_argb += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBORROW_WASMSIMD + +#ifdef HAS_ARGBALPHABLENDROW_WASMSIMD +void ARGBAlphaBlendRow_WASMSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + v128_t c1 = SplatConst16(1); + v128_t c00ffffff = SplatConst32(0x00ffffff); + do { + asm("" : "+r"(src_argb0), "+r"(src_argb1), "+r"(dst_argb)); + v128_t s = wasm_v128_load(src_argb0); + v128_t d = wasm_v128_load(src_argb1); + src_argb0 += 16; + src_argb1 += 16; + v128_t sa = wasm_u32x4_shr(s, 24); + v128_t da = wasm_u32x4_shr(d, 24); + v128_t prod16 = wasm_i16x8_add(wasm_i16x8_mul(da, sa), c1); + v128_t da_sa = + wasm_u16x8_shr(wasm_i16x8_add(prod16, wasm_u16x8_shr(prod16, 8)), 8); + v128_t da2 = wasm_i32x4_add(da, wasm_i32x4_sub(sa, da_sa)); + v128_t t = wasm_u32x4_shr(wasm_i32x4_eq(da, wasm_i32x4_const_splat(0)), 24); + v128_t sa_eff = wasm_v128_or(sa, t); + v128_t den_eff = wasm_v128_or(da2, t); + v128_t num = wasm_i32x4_sub(wasm_i32x4_shl(sa_eff, 8), sa_eff); + v128_t ratio = wasm_u32x4_trunc_sat_f32x4(wasm_f32x4_div( + wasm_f32x4_convert_u32x4(num), wasm_f32x4_convert_u32x4(den_eff))); + v128_t r8 = wasm_i8x16_shuffle(ratio, ratio, 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, + 8, 8, 12, 12, 12, 12); + v128_t inv_r8 = wasm_v128_not(r8); + v128_t acc_lo = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(d, inv_r8), + wasm_u16x8_extmul_low_u8x16(s, r8)), + c1); + v128_t acc_hi = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(d, inv_r8), + wasm_u16x8_extmul_high_u8x16(s, r8)), + c1); + acc_lo = + wasm_u16x8_shr(wasm_i16x8_add(acc_lo, wasm_u16x8_shr(acc_lo, 8)), 8); + acc_hi = + wasm_u16x8_shr(wasm_i16x8_add(acc_hi, wasm_u16x8_shr(acc_hi, 8)), 8); + v128_t rgb = wasm_u8x16_narrow_i16x8(acc_lo, acc_hi); + v128_t out = wasm_v128_or(wasm_v128_and(rgb, c00ffffff), + wasm_i32x4_shl(da2, 24)); + wasm_v128_store(dst_argb, out); + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBALPHABLENDROW_WASMSIMD + #undef LIBYUV_WASM_SWIZZLE #ifdef __cplusplus diff --git a/source/row_win.cc b/source/row_win.cc index e00419249..88c6af8fc 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -4371,6 +4371,267 @@ void ARGBBlendRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBBLENDROW_AVX512BW +#ifdef HAS_ARGBORROW_SSE2 +void ARGBOrRow_SSE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + const __m128i val = _mm_set1_epi32((int)value); + while (width >= 4) { + __m128i s = _mm_loadu_si128((const __m128i*)src_argb); + src_argb += 16; + _mm_storeu_si128((__m128i*)dst_argb, _mm_or_si128(s, val)); + dst_argb += 16; + width -= 4; + } +} +#endif // HAS_ARGBORROW_SSE2 + +#ifdef HAS_ARGBORROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBOrRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + const __m256i val = _mm256_set1_epi32((int)value); + while (width >= 8) { + __m256i s = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_or_si256(s, val)); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBORROW_AVX2 + +#ifdef HAS_ARGBORROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBOrRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + const __m512i val = _mm512_set1_epi32((int)value); + while (width >= 16) { + __m512i s = _mm512_loadu_si512((const __m512i*)src_argb); + src_argb += 64; + _mm512_storeu_si512((__m512i*)dst_argb, _mm512_or_si512(s, val)); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s = _mm512_maskz_loadu_epi32(k, src_argb); + _mm512_mask_storeu_epi32(dst_argb, k, _mm512_or_si512(s, val)); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBORROW_AVX512BW + +#ifdef HAS_ARGBALPHABLENDROW_SSSE3 +LIBYUV_TARGET_SSSE3 +void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + const __m128i zero = _mm_setzero_si128(); + const __m128i k0001 = _mm_set1_epi16(0x0001); + const __m128i k0101 = _mm_set1_epi16(0x0101); + const __m128i k00ff = _mm_set1_epi16(0x00ff); + const __m128i k8080 = _mm_set1_epi8((char)0x80); + const __m128i k7f81 = _mm_set1_epi16(0x7f81); + const __m128i k00ffffff = _mm_set1_epi32(0x00ffffff); + while (width >= 4) { + __m128i s = _mm_loadu_si128((const __m128i*)src_argb0); + __m128i d = _mm_loadu_si128((const __m128i*)src_argb1); + src_argb0 += 16; + src_argb1 += 16; + __m128i sa = _mm_srli_epi32(s, 24); + __m128i da = _mm_srli_epi32(d, 24); + __m128i da_sa = _mm_mulhi_epu16( + _mm_add_epi16(_mm_mullo_epi16(da, sa), k0001), k0101); + __m128i da2 = _mm_add_epi32(da, _mm_sub_epi32(sa, da_sa)); + __m128i t = _mm_srli_epi32(_mm_cmpeq_epi32(da, zero), 24); + __m128i sa_eff = _mm_or_si128(sa, t); + __m128i den_eff = _mm_or_si128(da2, t); + __m128i num = _mm_sub_epi32(_mm_slli_epi32(sa_eff, 8), sa_eff); + __m128i ratio = _mm_cvttps_epi32( + _mm_div_ps(_mm_cvtepi32_ps(num), _mm_cvtepi32_ps(den_eff))); + __m128i r_rep = _mm_shufflelo_epi16(_mm_shufflehi_epi16(ratio, 0xa0), 0xa0); + __m128i w = + _mm_or_si128(r_rep, _mm_slli_epi16(_mm_sub_epi16(k00ff, r_rep), 8)); + __m128i w_lo = _mm_unpacklo_epi16(w, w); + __m128i w_hi = _mm_unpackhi_epi16(w, w); + __m128i sb = _mm_sub_epi8(s, k8080); + __m128i db = _mm_sub_epi8(d, k8080); + __m128i lo = _mm_mulhi_epu16( + _mm_add_epi16(_mm_maddubs_epi16(w_lo, _mm_unpacklo_epi8(sb, db)), + k7f81), + k0101); + __m128i hi = _mm_mulhi_epu16( + _mm_add_epi16(_mm_maddubs_epi16(w_hi, _mm_unpackhi_epi8(sb, db)), + k7f81), + k0101); + __m128i rgb = _mm_packus_epi16(lo, hi); + __m128i out = + _mm_or_si128(_mm_and_si128(rgb, k00ffffff), _mm_slli_epi32(da2, 24)); + _mm_storeu_si128((__m128i*)dst_argb, out); + dst_argb += 16; + width -= 4; + } +} +#endif // HAS_ARGBALPHABLENDROW_SSSE3 + +#ifdef HAS_ARGBALPHABLENDROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + const __m256i zero = _mm256_setzero_si256(); + const __m256i k0001 = _mm256_set1_epi16(0x0001); + const __m256i k0101 = _mm256_set1_epi16(0x0101); + const __m256i k00ff = _mm256_set1_epi16(0x00ff); + const __m256i k8080 = _mm256_set1_epi8((char)0x80); + const __m256i k7f81 = _mm256_set1_epi16(0x7f81); + const __m256i k00ffffff = _mm256_set1_epi32(0x00ffffff); + while (width >= 8) { + __m256i s = _mm256_loadu_si256((const __m256i*)src_argb0); + __m256i d = _mm256_loadu_si256((const __m256i*)src_argb1); + src_argb0 += 32; + src_argb1 += 32; + __m256i sa = _mm256_srli_epi32(s, 24); + __m256i da = _mm256_srli_epi32(d, 24); + __m256i da_sa = _mm256_mulhi_epu16( + _mm256_add_epi16(_mm256_mullo_epi16(da, sa), k0001), k0101); + __m256i da2 = _mm256_add_epi32(da, _mm256_sub_epi32(sa, da_sa)); + __m256i t = _mm256_srli_epi32(_mm256_cmpeq_epi32(da, zero), 24); + __m256i sa_eff = _mm256_or_si256(sa, t); + __m256i den_eff = _mm256_or_si256(da2, t); + __m256i num = _mm256_sub_epi32(_mm256_slli_epi32(sa_eff, 8), sa_eff); + __m256i ratio = _mm256_cvttps_epi32( + _mm256_div_ps(_mm256_cvtepi32_ps(num), _mm256_cvtepi32_ps(den_eff))); + __m256i r_rep = + _mm256_shufflelo_epi16(_mm256_shufflehi_epi16(ratio, 0xa0), 0xa0); + __m256i w = _mm256_or_si256( + r_rep, _mm256_slli_epi16(_mm256_sub_epi16(k00ff, r_rep), 8)); + __m256i w_lo = _mm256_unpacklo_epi16(w, w); + __m256i w_hi = _mm256_unpackhi_epi16(w, w); + __m256i sb = _mm256_sub_epi8(s, k8080); + __m256i db = _mm256_sub_epi8(d, k8080); + __m256i lo = _mm256_mulhi_epu16( + _mm256_add_epi16( + _mm256_maddubs_epi16(w_lo, _mm256_unpacklo_epi8(sb, db)), k7f81), + k0101); + __m256i hi = _mm256_mulhi_epu16( + _mm256_add_epi16( + _mm256_maddubs_epi16(w_hi, _mm256_unpackhi_epi8(sb, db)), k7f81), + k0101); + __m256i rgb = _mm256_packus_epi16(lo, hi); + __m256i out = _mm256_or_si256(_mm256_and_si256(rgb, k00ffffff), + _mm256_slli_epi32(da2, 24)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBALPHABLENDROW_AVX2 + +#ifdef HAS_ARGBALPHABLENDROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + const __m512i zero = _mm512_setzero_si512(); + const __m512i k0001 = _mm512_set1_epi16(0x0001); + const __m512i k0101 = _mm512_set1_epi16(0x0101); + const __m512i k00ff = _mm512_set1_epi16(0x00ff); + const __m512i k8080 = _mm512_set1_epi8((char)0x80); + const __m512i k7f81 = _mm512_set1_epi16(0x7f81); + const __m512i k00ffffff = _mm512_set1_epi32(0x00ffffff); + while (width >= 16) { + __m512i s = _mm512_loadu_si512((const __m512i*)src_argb0); + __m512i d = _mm512_loadu_si512((const __m512i*)src_argb1); + src_argb0 += 64; + src_argb1 += 64; + __m512i sa = _mm512_srli_epi32(s, 24); + __m512i da = _mm512_srli_epi32(d, 24); + __m512i da_sa = _mm512_mulhi_epu16( + _mm512_add_epi16(_mm512_mullo_epi16(da, sa), k0001), k0101); + __m512i da2 = _mm512_add_epi32(da, _mm512_sub_epi32(sa, da_sa)); + __mmask16 k_da0 = _mm512_cmpeq_epi32_mask(da, zero); + __m512i t = _mm512_maskz_set1_epi32(k_da0, 255); + __m512i sa_eff = _mm512_or_si512(sa, t); + __m512i den_eff = _mm512_or_si512(da2, t); + __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff); + __m512i ratio = _mm512_cvttps_epi32( + _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff))); + __m512i r_rep = + _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0); + __m512i w = _mm512_or_si512( + r_rep, _mm512_slli_epi16(_mm512_sub_epi16(k00ff, r_rep), 8)); + __m512i w_lo = _mm512_unpacklo_epi16(w, w); + __m512i w_hi = _mm512_unpackhi_epi16(w, w); + __m512i sb = _mm512_sub_epi8(s, k8080); + __m512i db = _mm512_sub_epi8(d, k8080); + __m512i lo = _mm512_mulhi_epu16( + _mm512_add_epi16( + _mm512_maddubs_epi16(w_lo, _mm512_unpacklo_epi8(sb, db)), k7f81), + k0101); + __m512i hi = _mm512_mulhi_epu16( + _mm512_add_epi16( + _mm512_maddubs_epi16(w_hi, _mm512_unpackhi_epi8(sb, db)), k7f81), + k0101); + __m512i rgb = _mm512_packus_epi16(lo, hi); + __m512i out = _mm512_or_si512(_mm512_and_si512(rgb, k00ffffff), + _mm512_slli_epi32(da2, 24)); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s = _mm512_maskz_loadu_epi32(k, src_argb0); + __m512i d = _mm512_maskz_loadu_epi32(k, src_argb1); + __m512i sa = _mm512_srli_epi32(s, 24); + __m512i da = _mm512_srli_epi32(d, 24); + __m512i da_sa = _mm512_mulhi_epu16( + _mm512_add_epi16(_mm512_mullo_epi16(da, sa), k0001), k0101); + __m512i da2 = _mm512_add_epi32(da, _mm512_sub_epi32(sa, da_sa)); + __mmask16 k_da0 = _mm512_cmpeq_epi32_mask(da, zero); + __m512i t = _mm512_maskz_set1_epi32(k_da0, 255); + __m512i sa_eff = _mm512_or_si512(sa, t); + __m512i den_eff = _mm512_or_si512(da2, t); + __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff); + __m512i ratio = _mm512_cvttps_epi32( + _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff))); + __m512i r_rep = + _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0); + __m512i w = _mm512_or_si512( + r_rep, _mm512_slli_epi16(_mm512_sub_epi16(k00ff, r_rep), 8)); + __m512i w_lo = _mm512_unpacklo_epi16(w, w); + __m512i w_hi = _mm512_unpackhi_epi16(w, w); + __m512i sb = _mm512_sub_epi8(s, k8080); + __m512i db = _mm512_sub_epi8(d, k8080); + __m512i lo = _mm512_mulhi_epu16( + _mm512_add_epi16( + _mm512_maddubs_epi16(w_lo, _mm512_unpacklo_epi8(sb, db)), k7f81), + k0101); + __m512i hi = _mm512_mulhi_epu16( + _mm512_add_epi16( + _mm512_maddubs_epi16(w_hi, _mm512_unpackhi_epi8(sb, db)), k7f81), + k0101); + __m512i rgb = _mm512_packus_epi16(lo, hi); + __m512i out = _mm512_or_si512(_mm512_and_si512(rgb, k00ffffff), + _mm512_slli_epi32(da2, 24)); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBALPHABLENDROW_AVX512BW + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index e891ba346..d12be099b 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -5095,4 +5095,182 @@ TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32Column) { #endif // defined(ENABLE_ROW_TESTS) && defined(__aarch64__) +static int TestARGBOr(int width, + int height, + int benchmark_iterations, + int disable_cpu_flags, + int benchmark_cpu_info, + int invert, + int off, + uint32_t value) { + if (width < 1) { + width = 1; + } + const int kBpp = 4; + const int kStride = width * kBpp; + align_buffer_page_end(src_argb, kStride * abs(height) + off); + align_buffer_page_end(dst_argb_c, kStride * abs(height)); + align_buffer_page_end(dst_argb_opt, kStride * abs(height)); + MemRandomize(src_argb, kStride * abs(height) + off); + memset(dst_argb_c, 0, kStride * abs(height)); + memset(dst_argb_opt, 1, kStride * abs(height)); + + MaskCpuFlags(disable_cpu_flags); + ARGBOr(src_argb + off, kStride, dst_argb_c, kStride, width, invert * height, + value); + MaskCpuFlags(benchmark_cpu_info); + for (int i = 0; i < benchmark_iterations; ++i) { + ARGBOr(src_argb + off, kStride, dst_argb_opt, kStride, width, + invert * height, value); + } + int max_diff = 0; + for (int i = 0; i < kStride * abs(height); ++i) { + int abs_diff = abs(static_cast(dst_argb_c[i]) - + static_cast(dst_argb_opt[i])); + if (abs_diff > max_diff) { + max_diff = abs_diff; + } + } + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(dst_argb_c); + free_aligned_buffer_page_end(dst_argb_opt); + return max_diff; +} + +TEST_F(LibYUVPlanarTest, ARGBOr_Any) { + int max_diff = TestARGBOr(benchmark_width_ + 1, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 0, 0xff000000u); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBOr_Unaligned) { + int max_diff = TestARGBOr(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 1, 0xff000000u); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBOr_Invert) { + int max_diff = TestARGBOr(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, -1, 0, 0xff000000u); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBOr_Opt) { + int max_diff = TestARGBOr(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 0, 0xff000000u); + EXPECT_EQ(0, max_diff); +} + +static int TestARGBAlphaBlend(int width, + int height, + int benchmark_iterations, + int disable_cpu_flags, + int benchmark_cpu_info, + int invert, + int off) { + if (width < 1) { + width = 1; + } + const int kBpp = 4; + const int kStride = width * kBpp; + align_buffer_page_end(src_argb0, kStride * abs(height) + off); + align_buffer_page_end(src_argb1, kStride * abs(height) + off); + align_buffer_page_end(dst_argb_c, kStride * abs(height)); + align_buffer_page_end(dst_argb_opt, kStride * abs(height)); + MemRandomize(src_argb0, kStride * abs(height) + off); + MemRandomize(src_argb1, kStride * abs(height) + off); + memset(dst_argb_c, 0, kStride * abs(height)); + memset(dst_argb_opt, 1, kStride * abs(height)); + + MaskCpuFlags(disable_cpu_flags); + ARGBAlphaBlend(src_argb0 + off, kStride, src_argb1 + off, kStride, dst_argb_c, + kStride, width, invert * height); + MaskCpuFlags(benchmark_cpu_info); + for (int i = 0; i < benchmark_iterations; ++i) { + ARGBAlphaBlend(src_argb0 + off, kStride, src_argb1 + off, kStride, + dst_argb_opt, kStride, width, invert * height); + } + int max_diff = 0; + for (int i = 0; i < kStride * abs(height); ++i) { + int abs_diff = abs(static_cast(dst_argb_c[i]) - + static_cast(dst_argb_opt[i])); + if (abs_diff > max_diff) { + max_diff = abs_diff; + } + } + free_aligned_buffer_page_end(src_argb0); + free_aligned_buffer_page_end(src_argb1); + free_aligned_buffer_page_end(dst_argb_c); + free_aligned_buffer_page_end(dst_argb_opt); + return max_diff; +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Any) { + int max_diff = TestARGBAlphaBlend(benchmark_width_ + 1, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 0); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Unaligned) { + int max_diff = TestARGBAlphaBlend(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 1); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Invert) { + int max_diff = TestARGBAlphaBlend(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, -1, 0); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Opt) { + int max_diff = TestARGBAlphaBlend(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 0); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_AllAlphas) { + const int kWidth = 256; + const int kHeight = 256; + const int kStride = kWidth * 4; + align_buffer_page_end(src_argb0, kStride * kHeight); + align_buffer_page_end(src_argb1, kStride * kHeight); + align_buffer_page_end(dst_argb_c, kStride * kHeight); + align_buffer_page_end(dst_argb_opt, kStride * kHeight); + for (int da = 0; da < 256; ++da) { + for (int sa = 0; sa < 256; ++sa) { + int idx = (da * 256 + sa) * 4; + src_argb0[idx + 0] = (sa * 7 + da * 3 + 13) & 255; + src_argb0[idx + 1] = (sa * 13 + da * 5 + 29) & 255; + src_argb0[idx + 2] = (sa * 29 + da * 11 + 47) & 255; + src_argb0[idx + 3] = sa; + src_argb1[idx + 0] = (da * 17 + sa * 19 + 53) & 255; + src_argb1[idx + 1] = (da * 23 + sa * 31 + 71) & 255; + src_argb1[idx + 2] = (da * 37 + sa * 41 + 89) & 255; + src_argb1[idx + 3] = da; + } + } + MaskCpuFlags(disable_cpu_flags_); + ARGBAlphaBlend(src_argb0, kStride, src_argb1, kStride, dst_argb_c, kStride, + kWidth, kHeight); + MaskCpuFlags(benchmark_cpu_info_); + ARGBAlphaBlend(src_argb0, kStride, src_argb1, kStride, dst_argb_opt, kStride, + kWidth, kHeight); + for (int i = 0; i < kStride * kHeight; ++i) { + ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]) << "Mismatch at byte " << i; + } + free_aligned_buffer_page_end(src_argb0); + free_aligned_buffer_page_end(src_argb1); + free_aligned_buffer_page_end(dst_argb_c); + free_aligned_buffer_page_end(dst_argb_opt); +} + } // namespace libyuv