From 38e21134f7363a02b0c348c3fe3ec7e78b3fba9f Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Sat, 24 Oct 2026 04:43:06 -0700 Subject: [PATCH] [libyuv] Add ARGBOr and ARGBAlphaBlend with multi-platform SIMD Add two new planar/row compositing functions for 32-bit ARGB/BGRA scanlines: - ARGBOr / ARGBOrRow_*: Applies a bitwise OR with a 32-bit ARGB mask to every pixel (e.g., 0xFF000000 to force opaque alpha for BGRx to BGRA conversion). Implemented for C, SSE2, AVX2, AVX-512BW, NEON (32/64-bit), SVE2, SME, RVV, WASMSIMD, LSX, LASX, HVX, and Windows intrinsics (row_win.cc). - ARGBAlphaBlend / ARGBAlphaBlendRow_*: Implements straight-alpha (unattenuated) Porter-Duff source-over blending matching PDFium's CompositeRowBgra2Bgra: dst_a = back_a + src_a - (back_a * src_a) / 255 ratio = (src_a * 255) / dst_a dst_c = (back_c * (255 - ratio) + src_c * ratio) / 255 Implemented with bit-exact division by 255 across all 65,536 (back_a, src_a) pairs for C, SSSE3, AVX2, AVX-512BW, NEON (32/64-bit), SVE2, SME, RVV, WASMSIMD, LSX, LASX, HVX, and Windows intrinsics (row_win.cc). ARGBOr Performance (2048x64 BGRx->BGRA scanlines, us per call): uArch Before Highway libyuv vs Hwy ------------------------ ------ ------- ------- ------ Qualcomm Hexagon v81 HVX 259.63 260.97 15.78 16.54x SiFive X280 (RVV) 1107.57 72.49 50.70 1.43x SiFive X390 (RVV) 1108.82 70.42 52.26 1.35x ARGBAlphaBlend Performance (2048x64 BGRA scanlines, us per call): uArch Before Highway libyuv vs Hwy ------------------------ ------ ------- ------- ------ Intel Lion Cove (AVX2) 429.79 141.97 41.89 3.39x Intel Cougar Cove (AVX2) 516.83 181.98 51.33 3.55x Intel Golden Cove (AVX2) 560.31 206.28 67.08 3.08x AMD Zen 3 (AVX2) 1025.73 251.09 98.99 2.54x Wasm SIMD128 (V8) 1014.14 964.83 385.80 2.50x Apple M6 Super (NEON) 221.90 118.42 46.35 2.55x Apple M5 Super (NEON) 241.90 130.07 49.60 2.62x ARM C1-Ultra (SME2) 384.63 112.52 63.62 1.77x Qualcomm Oryon V3-L (SME) 388.13 154.87 87.65 1.77x Qualcomm Oryon-L (NEON) 318.28 149.36 57.23 2.61x ARM Cortex-X4 (SVE2) 675.47 213.56 102.53 2.08x ARM Cortex-A720 (NEON) 795.85 586.55 235.81 2.49x ARM Cortex-A520 (NEON) 2360.95 2122.99 787.65 2.70x ARM Cortex-A35 (ARMv7) 4907.31 5895.04 1474.51 4.00x SiFive X280 (RVV) 4145.06 1096.42 596.82 1.84x SiFive X390 (RVV) 4144.57 1084.93 597.62 1.82x Qualcomm Hexagon v81 HVX 4912.03 5786.25 952.19 6.08x Coral V4 RVV (1024 px) 50276 c 48992 c 4299 c 11.40x R=wtc@google.com Bug: none Change-Id: If661239826a033e89d4b5c1adc0bfdf048a7a726 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8647895 Reviewed-by: richard winterton --- include/libyuv/planar_functions.h | 23 ++ include/libyuv/row.h | 176 +++++++++++++ include/libyuv/row_sve.h | 154 +++++++++++ source/planar_functions.cc | 222 ++++++++++++++++ source/row_any.cc | 36 +++ source/row_common.cc | 61 +++++ source/row_gcc.cc | 418 ++++++++++++++++++++++++++++++ source/row_hvx.cc | 86 ++++++ source/row_lasx.cc | 83 ++++++ source/row_lsx.cc | 82 ++++++ source/row_neon.cc | 97 +++++++ source/row_neon64.cc | 85 ++++++ source/row_rvv.cc | 93 +++++++ source/row_sme.cc | 18 ++ source/row_sve.cc | 18 ++ source/row_wasm.cc | 67 +++++ source/row_win.cc | 261 +++++++++++++++++++ unit_test/planar_test.cc | 178 +++++++++++++ 18 files changed, 2158 insertions(+) diff --git a/include/libyuv/planar_functions.h b/include/libyuv/planar_functions.h index b93b729df..6306cd75e 100644 --- a/include/libyuv/planar_functions.h +++ b/include/libyuv/planar_functions.h @@ -855,6 +855,18 @@ int ARGBBlend(const uint8_t* src_argb0, int width, int height); +// Alpha Blend unattenuated ARGB images and store to destination. +// Source and destination are not pre-multiplied by alpha. +LIBYUV_API +int ARGBAlphaBlend(const uint8_t* src_argb0, + int src_stride_argb0, + const uint8_t* src_argb1, + int src_stride_argb1, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height); + // Alpha Blend plane and store to destination. // Source is not pre-multiplied by alpha. LIBYUV_API @@ -929,6 +941,17 @@ int ARGBSubtract(const uint8_t* src_argb0, int width, int height); +// Bitwise OR ARGB image with a specified ARGB value (e.g. 0xff000000 to force +// opaque alpha). +LIBYUV_API +int ARGBOr(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height, + uint32_t value); + // Convert I422 to YUY2. LIBYUV_API int I422ToYUY2(const uint8_t* src_y, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index e189d9d81..c7c67bd0f 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -78,6 +78,7 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_SSE2 #define HAS_ARGBAFFINEROW_SSE2 +#define HAS_ARGBALPHABLENDROW_SSSE3 #define HAS_ARGBBLENDROW_SSSE3 #define HAS_ARGBCOLORMATRIXROW_SSSE3 #define HAS_ARGBCOLORTABLEROW_X86 @@ -87,6 +88,7 @@ extern "C" { #define HAS_ARGBLUMACOLORTABLEROW_SSSE3 #define HAS_ARGBMIRRORROW_SSE2 #define HAS_ARGBMULTIPLYROW_SSE2 +#define HAS_ARGBORROW_SSE2 #define HAS_ARGBPOLYNOMIALROW_SSE2 #define HAS_ARGBQUANTIZEROW_SSE2 #define HAS_ARGBSEPIAROW_SSSE3 @@ -180,7 +182,9 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_AVX2 +#define HAS_ARGBALPHABLENDROW_AVX2 #define HAS_ARGBMULTIPLYROW_AVX2 +#define HAS_ARGBORROW_AVX2 #define HAS_ARGBSUBTRACTROW_AVX2 #define HAS_BLENDPLANEROW_AVX2 #endif @@ -409,7 +413,13 @@ extern "C" { #define HAS_I400TOARGBROW_AVX512BW #define HAS_ARGBGRAYROW_AVX512BW #define HAS_ARGBBLENDROW_AVX512BW +#define HAS_ARGBALPHABLENDROW_AVX512BW +#define HAS_ARGBORROW_AVX512BW #endif +#define HAS_ARGBALPHABLENDROW_SSSE3 +#define HAS_ARGBORROW_SSE2 +#define HAS_ARGBALPHABLENDROW_AVX2 +#define HAS_ARGBORROW_AVX2 #define HAS_ARGBTOYMATRIXROW_AVX2 #define HAS_RGBTOYMATRIXROW_AVX2 #define HAS_RGB565TOYMATRIXROW_AVX2 @@ -508,6 +518,8 @@ extern "C" { #define HAS_I400TOARGBROW_AVX512BW #define HAS_ARGBGRAYROW_AVX512BW #define HAS_ARGBBLENDROW_AVX512BW +#define HAS_ARGBALPHABLENDROW_AVX512BW +#define HAS_ARGBORROW_AVX512BW #endif // The following are available on Neon platforms: @@ -625,6 +637,7 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_NEON +#define HAS_ARGBALPHABLENDROW_NEON #define HAS_ARGBATTENUATEROW_NEON #define HAS_ARGBBLENDROW_NEON #define HAS_BLENDPLANEROW_NEON @@ -632,6 +645,7 @@ extern "C" { #define HAS_ARGBGRAYROW_NEON #define HAS_ARGBMIRRORROW_NEON #define HAS_ARGBMULTIPLYROW_NEON +#define HAS_ARGBORROW_NEON #define HAS_ARGBQUANTIZEROW_NEON #define HAS_ARGBSEPIAROW_NEON #define HAS_ARGBSHADEROW_NEON @@ -683,6 +697,8 @@ extern "C" { // The following are available on AArch64 SVE platforms: #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #define HAS_ARGB1555TOARGBROW_SVE2 +#define HAS_ARGBALPHABLENDROW_SVE2 +#define HAS_ARGBORROW_SVE2 #define HAS_ARGBTORAWROW_SVE2 #define HAS_ARGBTORGB24ROW_SVE2 #define HAS_ARGBTORGB565DITHERROW_SVE2 @@ -748,7 +764,9 @@ extern "C" { // The following are available on AArch64 SME platforms: #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ defined(__aarch64__) +#define HAS_ARGBALPHABLENDROW_SME #define HAS_ARGBMULTIPLYROW_SME +#define HAS_ARGBORROW_SME #define HAS_ARGBTOUVMATRIXROW_SME #define HAS_ARGBTOYMATRIXROW_SME #define HAS_CONVERT16TO8ROW_SME @@ -807,11 +825,13 @@ extern "C" { #define HAS_ARGB1555TOYROW_LSX #define HAS_ARGB4444TOARGBROW_LSX #define HAS_ARGBADDROW_LSX +#define HAS_ARGBALPHABLENDROW_LSX #define HAS_ARGBATTENUATEROW_LSX #define HAS_ARGBBLENDROW_LSX #define HAS_ARGBCOLORMATRIXROW_LSX #define HAS_ARGBEXTRACTALPHAROW_LSX #define HAS_ARGBGRAYROW_LSX +#define HAS_ARGBORROW_LSX #define HAS_ARGBSEPIAROW_LSX #define HAS_ARGBSHADEROW_LSX #define HAS_ARGBSHUFFLEROW_LSX @@ -881,6 +901,7 @@ extern "C" { #define HAS_ARGB1555TOYROW_LASX #define HAS_ARGB4444TOARGBROW_LASX #define HAS_ARGBADDROW_LASX +#define HAS_ARGBALPHABLENDROW_LASX #define HAS_ARGBATTENUATEROW_LASX #define HAS_ARGBBLENDROW_LASX #define HAS_ARGBCOLORMATRIXROW_LASX @@ -888,6 +909,7 @@ extern "C" { #define HAS_ARGBGRAYROW_LASX #define HAS_ARGBMIRRORROW_LASX #define HAS_ARGBMULTIPLYROW_LASX +#define HAS_ARGBORROW_LASX #define HAS_ARGBQUANTIZEROW_LASX #define HAS_ARGBSEPIAROW_LASX #define HAS_ARGBSETROW_LASX @@ -977,8 +999,10 @@ extern "C" { #define HAS_SWAPUVROW_RVV #define HAS_AB64TOARGBROW_RVV #define HAS_AR64TOAB64ROW_RVV +#define HAS_ARGBALPHABLENDROW_RVV #define HAS_ARGBATTENUATEROW_RVV #define HAS_ARGBBLENDROW_RVV +#define HAS_ARGBORROW_RVV #define HAS_ARGBTOAB64ROW_RVV #define HAS_ARGBTOABGRROW_RVV #define HAS_ARGBTOBGRAROW_RVV @@ -1018,10 +1042,12 @@ extern "C" { #define HAS_ARGB4444TOARGBROW_WASMSIMD #define HAS_ARGB4444TOUVMATRIXROW_WASMSIMD #define HAS_ARGB4444TOYMATRIXROW_WASMSIMD +#define HAS_ARGBALPHABLENDROW_WASMSIMD #define HAS_ARGBBLENDROW_WASMSIMD #define HAS_ARGBCOPYYTOALPHAROW_WASMSIMD #define HAS_ARGBEXTRACTALPHAROW_WASMSIMD #define HAS_ARGBMIRRORROW_WASMSIMD +#define HAS_ARGBORROW_WASMSIMD #define HAS_ARGBSHUFFLEROW_WASMSIMD #define HAS_ARGBTOARGB1555ROW_WASMSIMD #define HAS_ARGBTOARGB4444ROW_WASMSIMD @@ -1067,6 +1093,7 @@ extern "C" { #define HAS_AR64TOAB64ROW_HVX #define HAS_AR64TOARGBROW_HVX #define HAS_ARGBADDROW_HVX +#define HAS_ARGBALPHABLENDROW_HVX #define HAS_ARGBATTENUATEROW_HVX #define HAS_ARGBBLENDROW_HVX #define HAS_ARGBCOPYALPHAROW_HVX @@ -1074,6 +1101,7 @@ extern "C" { #define HAS_ARGBEXTRACTALPHAROW_HVX #define HAS_ARGBMIRRORROW_HVX #define HAS_ARGBMULTIPLYROW_HVX +#define HAS_ARGBORROW_HVX #define HAS_ARGBSETROW_HVX #define HAS_ARGBSUBTRACTROW_HVX #define HAS_ARGBTOAB64ROW_HVX @@ -5933,6 +5961,154 @@ void ARGBSubtractRow_Any_LASX(const uint8_t* y_buf, uint8_t* dst_ptr, int width); +// ARGB bitwise OR with 32-bit value. +void ARGBOrRow_C(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_SSE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_SSE2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_SVE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_SME(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_LASX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_RVV(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); +void ARGBOrRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + uint32_t value, + int width); +void ARGBOrRow_HVX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width); + +// Unattenuated ARGB alpha blend. +void ARGBAlphaBlendRow_C(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_SSSE3(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_AVX2(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_NEON(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_NEON(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_SVE2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_SME(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_LSX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_LSX(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_LASX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_RVV(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_WASMSIMD(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAlphaBlendRow_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void ARGBAlphaBlendRow_HVX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); + void ARGBToRGB24Row_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 19559b64f..7faf57db8 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -2360,6 +2360,160 @@ static inline float SsimWindowRow_SVE_SC(const uint32_t* row0, return total; } +static inline void ARGBOrRow_SVE_SC(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) STREAMING_COMPATIBLE { + uint32_t vl; + asm volatile( + "cntw %x[vl] \n" + "dup z2.s, %w[value] \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + "ptrue p0.s \n" + "1: \n" + "ld1w {z0.s}, p0/z, [%[src]] \n" + "ld1w {z1.s}, p0/z, [%[src], #1, mul vl] \n" + "incb %[src], all, mul #2 \n" + "orr z0.d, z0.d, z2.d \n" + "orr z1.d, z1.d, z2.d \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "st1w {z0.s}, p0, [%[dst]] \n" + "st1w {z1.s}, p0, [%[dst], #1, mul vl] \n" + "incb %[dst], all, mul #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[width] \n" + "whilelt p1.s, %w[vl], %w[width] \n" + "ld1w {z0.s}, p0/z, [%[src]] \n" + "ld1w {z1.s}, p1/z, [%[src], #1, mul vl] \n" + "orr z0.d, z0.d, z2.d \n" + "orr z1.d, z1.d, z2.d \n" + "st1w {z0.s}, p0, [%[dst]] \n" + "st1w {z1.s}, p1, [%[dst], #1, mul vl] \n" + + "99: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [value] "r"(value) // %[value] + : "memory", "cc", "z0", "z1", "z2", "p0", "p1"); +} + +#define ARGBALPHABLEND_SVE_2X(pred0, pred1) \ + "ld1w {z0.s}, " #pred0 "/z, [%[src0]] \n" \ + "ld1w {z1.s}, " #pred1 "/z, [%[src0], #1, mul vl] \n" \ + "ld1w {z2.s}, " #pred0 "/z, [%[src1]] \n" \ + "ld1w {z3.s}, " #pred1 "/z, [%[src1], #1, mul vl] \n" \ + "incb %[src0], all, mul #2 \n" \ + "incb %[src1], all, mul #2 \n" \ + "lsr z4.s, z0.s, #24 \n" \ + "lsr z5.s, z1.s, #24 \n" \ + "lsr z6.s, z2.s, #24 \n" \ + "lsr z7.s, z3.s, #24 \n" \ + "mov z16.d, z24.d \n" \ + "mov z17.d, z24.d \n" \ + "umlalb z16.h, z6.b, z4.b \n" \ + "umlalb z17.h, z7.b, z5.b \n" \ + "usra z16.h, z16.h, #8 \n" \ + "usra z17.h, z17.h, #8 \n" \ + "lsr z16.h, z16.h, #8 \n" \ + "lsr z17.h, z17.h, #8 \n" \ + "sub z16.s, z4.s, z16.s \n" \ + "sub z17.s, z5.s, z17.s \n" \ + "add z16.s, z6.s, z16.s \n" \ + "add z17.s, z7.s, z17.s \n" \ + "lsl z18.s, z4.s, #8 \n" \ + "lsl z19.s, z5.s, #8 \n" \ + "sub z18.s, z18.s, z4.s \n" \ + "sub z19.s, z19.s, z5.s \n" \ + "sub z6.s, z6.s, #1 \n" \ + "sub z7.s, z7.s, #1 \n" \ + "asr z6.s, z6.s, #31 \n" \ + "asr z7.s, z7.s, #31 \n" \ + "udiv z18.s, p0/m, z18.s, z16.s \n" \ + "udiv z19.s, p0/m, z19.s, z17.s \n" \ + "mul z18.s, z18.s, z25.s \n" \ + "mul z19.s, z19.s, z25.s \n" \ + "orr z18.d, z18.d, z6.d \n" \ + "orr z19.d, z19.d, z7.d \n" \ + "not z20.b, p0/m, z18.b \n" \ + "not z21.b, p0/m, z19.b \n" \ + "mov z4.d, z24.d \n" \ + "mov z5.d, z24.d \n" \ + "mov z6.d, z24.d \n" \ + "mov z7.d, z24.d \n" \ + "umlalb z4.h, z2.b, z20.b \n" \ + "umlalt z5.h, z2.b, z20.b \n" \ + "umlalb z6.h, z3.b, z21.b \n" \ + "umlalt z7.h, z3.b, z21.b \n" \ + "umlalb z4.h, z0.b, z18.b \n" \ + "umlalt z5.h, z0.b, z18.b \n" \ + "umlalb z6.h, z1.b, z19.b \n" \ + "umlalt z7.h, z1.b, z19.b \n" \ + "usra z4.h, z4.h, #8 \n" \ + "usra z5.h, z5.h, #8 \n" \ + "usra z6.h, z6.h, #8 \n" \ + "usra z7.h, z7.h, #8 \n" \ + "shrnb z0.b, z4.h, #8 \n" \ + "shrnt z0.b, z5.h, #8 \n" \ + "shrnb z1.b, z6.h, #8 \n" \ + "shrnt z1.b, z7.h, #8 \n" \ + "and z0.d, z0.d, z26.d \n" \ + "and z1.d, z1.d, z26.d \n" \ + "lsl z16.s, z16.s, #24 \n" \ + "lsl z17.s, z17.s, #24 \n" \ + "orr z0.d, z0.d, z16.d \n" \ + "orr z1.d, z1.d, z17.d \n" \ + "st1w {z0.s}, " #pred0 ", [%[dst]] \n" \ + "st1w {z1.s}, " #pred1 ", [%[dst], #1, mul vl] \n" \ + "incb %[dst], all, mul #2 \n" + +static inline void ARGBAlphaBlendRow_SVE_SC(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) STREAMING_COMPATIBLE { + uint32_t vl; + asm volatile( + "cntw %x[vl] \n" + "ptrue p0.b \n" + "mov z24.h, #1 \n" + "mov z25.s, #0x01010101 \n" + "mov z26.s, #0x00ffffff \n" + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.lt 2f \n" + + "1: \n" + ARGBALPHABLEND_SVE_2X(p0, p0) + "subs %w[width], %w[width], %w[vl], lsl #1 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #1 \n" + "b.eq 99f \n" + + "whilelt p1.s, wzr, %w[width] \n" + "whilelt p2.s, %w[vl], %w[width] \n" + ARGBALPHABLEND_SVE_2X(p1, p2) + + "99: \n" + : [src0] "+r"(src_argb0), // %[src0] + [src1] "+r"(src_argb1), // %[src1] + [dst] "+r"(dst_argb), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", + "z17", "z18", "z19", "z20", "z21", "z24", "z25", "z26", "p0", "p1", + "p2"); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/planar_functions.cc b/source/planar_functions.cc index ecf6fc8a9..4901686ad 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -3468,6 +3468,118 @@ int ARGBBlend(const uint8_t* src_argb0, return 0; } +// Alpha Blend 2 unattenuated ARGB images and store to destination. +LIBYUV_API +int ARGBAlphaBlend(const uint8_t* src_argb0, + int src_stride_argb0, + const uint8_t* src_argb1, + int src_stride_argb1, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height) { + int y; + void (*ARGBAlphaBlendRow)(const uint8_t* src_argb, const uint8_t* src_argb1, + uint8_t* dst_argb, int width) = ARGBAlphaBlendRow_C; + if (!src_argb0 || !src_argb1 || !dst_argb || width <= 0 || height == 0 || + height == INT_MIN) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb; + dst_stride_argb = -dst_stride_argb; + } + // Coalesce rows. + if (src_stride_argb0 == width * 4 && src_stride_argb1 == width * 4 && + dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; + } +#if defined(HAS_ARGBALPHABLENDROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_SSSE3; + if (IS_ALIGNED(width, 4)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_SSSE3; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_AVX512BW; + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_NEON; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_SVE2; + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_SME; + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_LSX; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_LASX; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_RVV; + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBALPHABLENDROW_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + ARGBAlphaBlendRow = ARGBAlphaBlendRow_HVX; + } +#endif + for (y = 0; y < height; ++y) { + ARGBAlphaBlendRow(src_argb0, src_argb1, dst_argb, width); + src_argb0 += src_stride_argb0; + src_argb1 += src_stride_argb1; + dst_argb += dst_stride_argb; + } + return 0; +} + // Alpha Blend plane and store to destination. LIBYUV_API int BlendPlane(const uint8_t* src_y0, @@ -4030,6 +4142,116 @@ int ARGBSubtract(const uint8_t* src_argb0, return 0; } +// Bitwise OR ARGB image with a specified ARGB value. +LIBYUV_API +int ARGBOr(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height, + uint32_t value) { + int y; + void (*ARGBOrRow)(const uint8_t* src_argb, uint8_t* dst_argb, uint32_t value, + int width) = ARGBOrRow_C; + if (!src_argb || !dst_argb || width <= 0 || height == 0 || + height == INT_MIN) { + return -1; + } + // Negative height means invert the image. + if (height < 0) { + height = -height; + src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb; + src_stride_argb = -src_stride_argb; + } + // Coalesce rows. + if (src_stride_argb == width * 4 && dst_stride_argb == width * 4 && + (ptrdiff_t)width * height <= INT_MAX) { + width *= height; + height = 1; + src_stride_argb = dst_stride_argb = 0; + } +#if defined(HAS_ARGBORROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ARGBOrRow = ARGBOrRow_Any_SSE2; + if (IS_ALIGNED(width, 4)) { + ARGBOrRow = ARGBOrRow_SSE2; + } + } +#endif +#if defined(HAS_ARGBORROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBOrRow = ARGBOrRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBOrRow = ARGBOrRow_AVX2; + } + } +#endif +#if defined(HAS_ARGBORROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBOrRow = ARGBOrRow_AVX512BW; + } +#endif +#if defined(HAS_ARGBORROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBOrRow = ARGBOrRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBOrRow = ARGBOrRow_NEON; + } + } +#endif +#if defined(HAS_ARGBORROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBOrRow = ARGBOrRow_SVE2; + } +#endif +#if defined(HAS_ARGBORROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBOrRow = ARGBOrRow_SME; + } +#endif +#if defined(HAS_ARGBORROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBOrRow = ARGBOrRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBOrRow = ARGBOrRow_LSX; + } + } +#endif +#if defined(HAS_ARGBORROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBOrRow = ARGBOrRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBOrRow = ARGBOrRow_LASX; + } + } +#endif +#if defined(HAS_ARGBORROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBOrRow = ARGBOrRow_RVV; + } +#endif +#if defined(HAS_ARGBORROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBOrRow = ARGBOrRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBOrRow = ARGBOrRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBORROW_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + ARGBOrRow = ARGBOrRow_HVX; + } +#endif + for (y = 0; y < height; ++y) { + ARGBOrRow(src_argb, dst_argb, value, width); + src_argb += src_stride_argb; + dst_argb += dst_stride_argb; + } + return 0; +} + // Convert RAW to RGB24. LIBYUV_API int RAWToRGB24(const uint8_t* src_raw, diff --git a/source/row_any.cc b/source/row_any.cc index f2f41ea68..57806d43c 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -751,6 +751,24 @@ ANY21(ARGBSubtractRow_Any_LSX, ARGBSubtractRow_LSX, 0, 4, 4, 4, 3) #ifdef HAS_ARGBSUBTRACTROW_LASX ANY21(ARGBSubtractRow_Any_LASX, ARGBSubtractRow_LASX, 0, 4, 4, 4, 7) #endif +#ifdef HAS_ARGBALPHABLENDROW_SSSE3 +ANY21(ARGBAlphaBlendRow_Any_SSSE3, ARGBAlphaBlendRow_SSSE3, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBALPHABLENDROW_AVX2 +ANY21(ARGBAlphaBlendRow_Any_AVX2, ARGBAlphaBlendRow_AVX2, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBALPHABLENDROW_NEON +ANY21(ARGBAlphaBlendRow_Any_NEON, ARGBAlphaBlendRow_NEON, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBALPHABLENDROW_LSX +ANY21(ARGBAlphaBlendRow_Any_LSX, ARGBAlphaBlendRow_LSX, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBALPHABLENDROW_LASX +ANY21(ARGBAlphaBlendRow_Any_LASX, ARGBAlphaBlendRow_LASX, 0, 4, 4, 4, 7) +#endif +#ifdef HAS_ARGBALPHABLENDROW_WASMSIMD +ANY21(ARGBAlphaBlendRow_Any_WASMSIMD, ARGBAlphaBlendRow_WASMSIMD, 0, 4, 4, 4, 3) +#endif #ifdef HAS_SOBELROW_SSE2 ANY21(SobelRow_Any_SSE2, SobelRow_SSE2, 0, 1, 1, 4, 15) #endif @@ -1542,6 +1560,24 @@ ANY11P(ARGBShuffleRow_Any_WASMSIMD, 4, 3) #endif +#ifdef HAS_ARGBORROW_SSE2 +ANY11P(ARGBOrRow_Any_SSE2, ARGBOrRow_SSE2, const uint32_t, 4, 4, 3) +#endif +#ifdef HAS_ARGBORROW_AVX2 +ANY11P(ARGBOrRow_Any_AVX2, ARGBOrRow_AVX2, const uint32_t, 4, 4, 7) +#endif +#ifdef HAS_ARGBORROW_NEON +ANY11P(ARGBOrRow_Any_NEON, ARGBOrRow_NEON, const uint32_t, 4, 4, 7) +#endif +#ifdef HAS_ARGBORROW_LSX +ANY11P(ARGBOrRow_Any_LSX, ARGBOrRow_LSX, const uint32_t, 4, 4, 3) +#endif +#ifdef HAS_ARGBORROW_LASX +ANY11P(ARGBOrRow_Any_LASX, ARGBOrRow_LASX, const uint32_t, 4, 4, 7) +#endif +#ifdef HAS_ARGBORROW_WASMSIMD +ANY11P(ARGBOrRow_Any_WASMSIMD, ARGBOrRow_WASMSIMD, const uint32_t, 4, 4, 3) +#endif #undef ANY11P #undef ANY11P diff --git a/source/row_common.cc b/source/row_common.cc index 8fcec481a..1bb42ed9b 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -3473,6 +3473,67 @@ void ARGBBlendRow_C(const uint8_t* src_argb, } #undef BLEND +// Blend unattenuated src_argb0 over src_argb1 and store to dst_argb. +void ARGBAlphaBlendRow_C(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + int x; + for (x = 0; x < width; ++x) { + int sa = src_argb0[3]; + int da = src_argb1[3]; + if (da == 0) { + dst_argb[0] = src_argb0[0]; + dst_argb[1] = src_argb0[1]; + dst_argb[2] = src_argb0[2]; + dst_argb[3] = STATIC_CAST(uint8_t, sa); + } else if (sa == 0) { + dst_argb[0] = src_argb1[0]; + dst_argb[1] = src_argb1[1]; + dst_argb[2] = src_argb1[2]; + dst_argb[3] = STATIC_CAST(uint8_t, da); + } else { + int dst_alpha = da + sa - da * sa / 255; + int alpha_ratio = sa * 255 / dst_alpha; + dst_argb[0] = STATIC_CAST( + uint8_t, + (src_argb1[0] * (255 - alpha_ratio) + src_argb0[0] * alpha_ratio) / + 255); + dst_argb[1] = STATIC_CAST( + uint8_t, + (src_argb1[1] * (255 - alpha_ratio) + src_argb0[1] * alpha_ratio) / + 255); + dst_argb[2] = STATIC_CAST( + uint8_t, + (src_argb1[2] * (255 - alpha_ratio) + src_argb0[2] * alpha_ratio) / + 255); + dst_argb[3] = STATIC_CAST(uint8_t, dst_alpha); + } + src_argb0 += 4; + src_argb1 += 4; + dst_argb += 4; + } +} + +void ARGBOrRow_C(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + const uint8_t v0 = value & 0xff; + const uint8_t v1 = (value >> 8) & 0xff; + const uint8_t v2 = (value >> 16) & 0xff; + const uint8_t v3 = (value >> 24) & 0xff; + int i; + for (i = 0; i < width; ++i) { + dst_argb[0] = src_argb[0] | v0; + dst_argb[1] = src_argb[1] | v1; + dst_argb[2] = src_argb[2] | v2; + dst_argb[3] = src_argb[3] | v3; + src_argb += 4; + dst_argb += 4; + } +} + #define UBLEND(f, b, a) (((a) * f) + ((255 - a) * b) + 255) >> 8 void BlendPlaneRow_C(const uint8_t* src0, const uint8_t* src1, diff --git a/source/row_gcc.cc b/source/row_gcc.cc index d2a345e30..ee9f648ee 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -13470,6 +13470,424 @@ void OMITFP ARGBBlendRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBBLENDROW_AVX512BW +#ifdef HAS_ARGBORROW_SSE2 +void ARGBOrRow_SSE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "movd %[value],%%xmm1 \n" + "pshufd $0x00,%%xmm1,%%xmm1 \n" + + LABELALIGN + "1: \n" + "movdqu (%[src]),%%xmm0 \n" + "lea 0x10(%[src]),%[src] \n" + "por %%xmm1,%%xmm0 \n" + "movdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "sub $0x4,%[width] \n" + "jg 1b \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width) + : [value] "r"(value) + : "memory", "cc", "xmm0", "xmm1"); +} +#endif // HAS_ARGBORROW_SSE2 + +#ifdef HAS_ARGBORROW_AVX2 +void ARGBOrRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "vmovd %[value],%%xmm1 \n" + "vpbroadcastd %%xmm1,%%ymm1 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width) + : [value] "r"(value) + : "memory", "cc", "xmm0", "xmm1"); +} +#endif // HAS_ARGBORROW_AVX2 + +#ifdef HAS_ARGBORROW_AVX512BW +void OMITFP ARGBOrRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + uintptr_t temp; + asm volatile( + "vpbroadcastd %[value],%%zmm1 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width), + [temp] "=&r"(temp) + : [value] "r"(value) + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_ARGBORROW_AVX512BW + +#ifdef HAS_ARGBALPHABLENDROW_SSSE3 +void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "pcmpeqb %%xmm5,%%xmm5 \n" + "pabsb %%xmm5,%%xmm4 \n" // 0x01010101... + "psllw $7,%%xmm4 \n" + "movdqa %%xmm4,%%xmm6 \n" // xmm6 = 0x8080... + "psrlw $7,%%xmm4 \n" // xmm4 = 0x0101... + "psubw %%xmm4,%%xmm6 \n" // 0x7f7f... + "psubw %%xmm5,%%xmm6 \n" // 0x7f80... + "psubw %%xmm5,%%xmm6 \n" // xmm6 = 0x7f81... + + LABELALIGN + "1: \n" + "movdqu (%[src0]),%%xmm0 \n" + "movdqu (%[src1]),%%xmm1 \n" + "lea 0x10(%[src0]),%[src0] \n" + "lea 0x10(%[src1]),%[src1] \n" + + "movdqa %%xmm0,%%xmm2 \n" + "movdqa %%xmm1,%%xmm3 \n" + "psrld $24,%%xmm2 \n" // sa + "psrld $24,%%xmm3 \n" // da + + "pcmpeqb %%xmm5,%%xmm5 \n" // -1 + "movdqa %%xmm2,%%xmm7 \n" + "pmullw %%xmm3,%%xmm7 \n" // da * sa + "psubw %%xmm5,%%xmm7 \n" // da * sa + 1 + "pmulhuw %%xmm4,%%xmm7 \n" // (da * sa) / 255 + + "paddd %%xmm3,%%xmm5 \n" // da - 1 + "psrld $24,%%xmm5 \n" // t = (da==0) ? 255 : 0 + + "paddw %%xmm2,%%xmm3 \n" // da + sa + "psubw %%xmm7,%%xmm3 \n" // da2 + + "por %%xmm5,%%xmm2 \n" // sa_eff + "movdqa %%xmm3,%%xmm7 \n" + "por %%xmm5,%%xmm7 \n" // den_eff + + "movdqa %%xmm2,%%xmm5 \n" + "pslld $8,%%xmm2 \n" + "psubd %%xmm5,%%xmm2 \n" // sa_eff * 255 + + "cvtdq2ps %%xmm2,%%xmm2 \n" + "cvtdq2ps %%xmm7,%%xmm7 \n" + "divps %%xmm7,%%xmm2 \n" + "cvttps2dq %%xmm2,%%xmm2 \n" // ratio + + "pcmpeqb %%xmm5,%%xmm5 \n" + "psllw $8,%%xmm5 \n" // 0xff00 + + "pmullw %%xmm4,%%xmm2 \n" + "pshufhw $0xa0,%%xmm2,%%xmm2 \n" + "pshuflw $0xa0,%%xmm2,%%xmm2 \n" + "pxor %%xmm5,%%xmm2 \n" + "movdqa %%xmm2,%%xmm7 \n" + "punpcklwd %%xmm2,%%xmm2 \n" + "punpckhwd %%xmm7,%%xmm7 \n" + + "movdqa %%xmm4,%%xmm5 \n" + "psllw $7,%%xmm5 \n" // 0x8080 + "psubb %%xmm5,%%xmm0 \n" + "psubb %%xmm5,%%xmm1 \n" + "movdqa %%xmm0,%%xmm5 \n" + "punpcklbw %%xmm1,%%xmm0 \n" + "punpckhbw %%xmm1,%%xmm5 \n" + + "pmaddubsw %%xmm0,%%xmm2 \n" + "pmaddubsw %%xmm5,%%xmm7 \n" + "paddw %%xmm6,%%xmm2 \n" + "paddw %%xmm6,%%xmm7 \n" + "pmulhuw %%xmm4,%%xmm2 \n" + "pmulhuw %%xmm4,%%xmm7 \n" + "packuswb %%xmm7,%%xmm2 \n" + + "pslld $8,%%xmm2 \n" + "psrld $8,%%xmm2 \n" + "pslld $24,%%xmm3 \n" + "por %%xmm3,%%xmm2 \n" + + "movdqu %%xmm2,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "sub $0x4,%[width] \n" + "jg 1b \n" + : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [width] "+rm"(width) + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_ARGBALPHABLENDROW_SSSE3 + +#ifdef HAS_ARGBALPHABLENDROW_AVX2 +void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpabsb %%ymm5,%%ymm4 \n" // 0x0101... + "vpsllw $7,%%ymm4,%%ymm6 \n" // 0x8080... + "vpsubw %%ymm4,%%ymm6,%%ymm6 \n" // 0x7f7f... + "vpsubw %%ymm5,%%ymm6,%%ymm6 \n" // 0x7f80... + "vpsubw %%ymm5,%%ymm6,%%ymm6 \n" // 0x7f81... + + LABELALIGN + "1: \n" + "vmovdqu (%[src0]),%%ymm0 \n" + "vmovdqu (%[src1]),%%ymm1 \n" + "lea 0x20(%[src0]),%[src0] \n" + "lea 0x20(%[src1]),%[src1] \n" + + "vpsrld $24,%%ymm0,%%ymm2 \n" // sa + "vpsrld $24,%%ymm1,%%ymm3 \n" // da + + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // -1 + "vpmullw %%ymm3,%%ymm2,%%ymm7 \n" // da * sa + "vpsubw %%ymm5,%%ymm7,%%ymm7 \n" // da * sa + 1 + "vpmulhuw %%ymm4,%%ymm7,%%ymm7 \n" // (da * sa) / 255 + + "vpaddd %%ymm3,%%ymm5,%%ymm5 \n" // da - 1 + "vpsrld $24,%%ymm5,%%ymm5 \n" // t = (da==0) ? 255 : 0 + + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" // da + sa + "vpsubw %%ymm7,%%ymm3,%%ymm3 \n" // da2 + + "vpor %%ymm5,%%ymm2,%%ymm2 \n" // sa_eff + "vpor %%ymm5,%%ymm3,%%ymm7 \n" // den_eff + + "vpslld $8,%%ymm2,%%ymm5 \n" + "vpsubd %%ymm2,%%ymm5,%%ymm2 \n" // sa_eff * 255 + + "vcvtdq2ps %%ymm2,%%ymm2 \n" + "vcvtdq2ps %%ymm7,%%ymm7 \n" + "vdivps %%ymm7,%%ymm2,%%ymm2 \n" + "vcvttps2dq %%ymm2,%%ymm2 \n" // ratio + + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsllw $8,%%ymm5,%%ymm5 \n" // 0xff00 + + "vpmullw %%ymm4,%%ymm2,%%ymm2 \n" + "vpshufhw $0xa0,%%ymm2,%%ymm2 \n" + "vpshuflw $0xa0,%%ymm2,%%ymm2 \n" + "vpxor %%ymm5,%%ymm2,%%ymm2 \n" + "vpunpckhwd %%ymm2,%%ymm2,%%ymm7 \n" + "vpunpcklwd %%ymm2,%%ymm2,%%ymm2 \n" + + "vpsllw $7,%%ymm4,%%ymm5 \n" // 0x8080 + "vpsubb %%ymm5,%%ymm0,%%ymm0 \n" + "vpsubb %%ymm5,%%ymm1,%%ymm1 \n" + "vpunpckhbw %%ymm1,%%ymm0,%%ymm5 \n" + "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" + + "vpmaddubsw %%ymm0,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm5,%%ymm7,%%ymm7 \n" + "vpaddw %%ymm6,%%ymm2,%%ymm2 \n" + "vpaddw %%ymm6,%%ymm7,%%ymm7 \n" + "vpmulhuw %%ymm4,%%ymm2,%%ymm2 \n" + "vpmulhuw %%ymm4,%%ymm7,%%ymm7 \n" + "vpackuswb %%ymm7,%%ymm2,%%ymm2 \n" + + "vpslld $8,%%ymm2,%%ymm2 \n" + "vpsrld $8,%%ymm2,%%ymm2 \n" + "vpslld $24,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vmovdqu %%ymm2,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [width] "+rm"(width) + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_ARGBALPHABLENDROW_AVX2 + +#ifdef HAS_ARGBALPHABLENDROW_AVX512BW +void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + uintptr_t temp; + asm volatile( + "vpternlogd $0xff,%%zmm20,%%zmm20,%%zmm20 \n" // -1 + "vpabsb %%zmm20,%%zmm16 \n" // zmm16 = 0x0101... + "vpsllw $7,%%zmm16,%%zmm21 \n" // zmm21 = 0x8080... + "vpsubw %%zmm16,%%zmm21,%%zmm18 \n" // 0x7f7f... + "vpsubw %%zmm20,%%zmm18,%%zmm18 \n" // 0x7f80... + "vpsubw %%zmm20,%%zmm18,%%zmm18 \n" // zmm18 = 0x7f81... + "vpsllw $8,%%zmm20,%%zmm17 \n" // zmm17 = 0xff00... + "vpsrld $24,%%zmm20,%%zmm19 \n" // zmm19 = 255 in dwords + + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src0]),%%zmm0 \n" + "vmovdqu32 (%[src1]),%%zmm1 \n" + "lea 0x40(%[src0]),%[src0] \n" + "lea 0x40(%[src1]),%[src1] \n" + + "vpsrld $24,%%zmm0,%%zmm2 \n" // sa + "vpsrld $24,%%zmm1,%%zmm3 \n" // da + + "vpmullw %%zmm3,%%zmm2,%%zmm7 \n" + "vpsubw %%zmm20,%%zmm7,%%zmm7 \n" // da * sa + 1 + "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n" + + "vpaddd %%zmm3,%%zmm20,%%zmm5 \n" // da - 1 + "vpsrld $24,%%zmm5,%%zmm5 \n" // t + + "vpaddw %%zmm2,%%zmm3,%%zmm3 \n" + "vpsubw %%zmm7,%%zmm3,%%zmm3 \n" // da2 + + "vpord %%zmm5,%%zmm2,%%zmm2 \n" // sa_eff + "vpord %%zmm5,%%zmm3,%%zmm7 \n" // den_eff + "vpmulld %%zmm19,%%zmm2,%%zmm2 \n" // sa_eff * 255 + + "vcvtdq2ps %%zmm2,%%zmm2 \n" + "vcvtdq2ps %%zmm7,%%zmm7 \n" + "vdivps %%zmm7,%%zmm2,%%zmm2 \n" + "vcvttps2dq %%zmm2,%%zmm2 \n" + + "vpmullw %%zmm16,%%zmm2,%%zmm2 \n" + "vpshufhw $0xa0,%%zmm2,%%zmm2 \n" + "vpshuflw $0xa0,%%zmm2,%%zmm2 \n" + "vpxord %%zmm17,%%zmm2,%%zmm2 \n" + "vpunpckhwd %%zmm2,%%zmm2,%%zmm7 \n" + "vpunpcklwd %%zmm2,%%zmm2,%%zmm2 \n" + + "vpsubb %%zmm21,%%zmm0,%%zmm0 \n" + "vpsubb %%zmm21,%%zmm1,%%zmm1 \n" + "vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n" + "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" + + "vpmaddubsw %%zmm0,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm5,%%zmm7,%%zmm7 \n" + "vpaddw %%zmm18,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm7,%%zmm7 \n" + "vpmulhuw %%zmm16,%%zmm2,%%zmm2 \n" + "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n" + "vpackuswb %%zmm7,%%zmm2,%%zmm2 \n" + + "vpslld $8,%%zmm2,%%zmm2 \n" + "vpsrld $8,%%zmm2,%%zmm2 \n" + "vpslld $24,%%zmm3,%%zmm3 \n" + "vpord %%zmm3,%%zmm2,%%zmm2 \n" + + "vmovdqu32 %%zmm2,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src0]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 (%[src1]),%%zmm1%{%%k1%}%{z%} \n" + + "vpsrld $24,%%zmm0,%%zmm2 \n" + "vpsrld $24,%%zmm1,%%zmm3 \n" + + "vpmullw %%zmm3,%%zmm2,%%zmm7 \n" + "vpsubw %%zmm20,%%zmm7,%%zmm7 \n" + "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n" + + "vpaddd %%zmm3,%%zmm20,%%zmm5 \n" + "vpsrld $24,%%zmm5,%%zmm5 \n" + + "vpaddw %%zmm2,%%zmm3,%%zmm3 \n" + "vpsubw %%zmm7,%%zmm3,%%zmm3 \n" + + "vpord %%zmm5,%%zmm2,%%zmm2 \n" + "vpord %%zmm5,%%zmm3,%%zmm7 \n" + "vpmulld %%zmm19,%%zmm2,%%zmm2 \n" + + "vcvtdq2ps %%zmm2,%%zmm2 \n" + "vcvtdq2ps %%zmm7,%%zmm7 \n" + "vdivps %%zmm7,%%zmm2,%%zmm2 \n" + "vcvttps2dq %%zmm2,%%zmm2 \n" + + "vpmullw %%zmm16,%%zmm2,%%zmm2 \n" + "vpshufhw $0xa0,%%zmm2,%%zmm2 \n" + "vpshuflw $0xa0,%%zmm2,%%zmm2 \n" + "vpxord %%zmm17,%%zmm2,%%zmm2 \n" + "vpunpckhwd %%zmm2,%%zmm2,%%zmm7 \n" + "vpunpcklwd %%zmm2,%%zmm2,%%zmm2 \n" + + "vpsubb %%zmm21,%%zmm0,%%zmm0 \n" + "vpsubb %%zmm21,%%zmm1,%%zmm1 \n" + "vpunpckhbw %%zmm1,%%zmm0,%%zmm5 \n" + "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" + + "vpmaddubsw %%zmm0,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm5,%%zmm7,%%zmm7 \n" + "vpaddw %%zmm18,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm18,%%zmm7,%%zmm7 \n" + "vpmulhuw %%zmm16,%%zmm2,%%zmm2 \n" + "vpmulhuw %%zmm16,%%zmm7,%%zmm7 \n" + "vpackuswb %%zmm7,%%zmm2,%%zmm2 \n" + + "vpslld $8,%%zmm2,%%zmm2 \n" + "vpsrld $8,%%zmm2,%%zmm2 \n" + "vpslld $24,%%zmm3,%%zmm3 \n" + "vpord %%zmm3,%%zmm2,%%zmm2 \n" + + "vmovdqu32 %%zmm2,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src0] "+r"(src_argb), [src1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [width] "+r"(width), [temp] "=&r"(temp) + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1"); +} +#endif // HAS_ARGBALPHABLENDROW_AVX512BW + #endif // !defined(LIBYUV_DISABLE_X86) && // (defined(__x86_64__) || defined(__i386__)) && // !defined(LIBYUV_ENABLE_ROWWIN) diff --git a/source/row_hvx.cc b/source/row_hvx.cc index de6c4e4f0..1a8be0689 100644 --- a/source/row_hvx.cc +++ b/source/row_hvx.cc @@ -112,6 +112,25 @@ void ARGBSetRow_HVX(uint8_t* dst_argb, uint32_t v32, int width) { } } +void ARGBOrRow_HVX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + HVX_Vector v = Q6_V_vsplat_R(value); + int bytes = width * 4; + while (bytes >= 128) { + HVX_Vector s = vloadu(src_argb); + vstoreu(dst_argb, Q6_V_vor_VV(s, v)); + src_argb += 128; + dst_argb += 128; + bytes -= 128; + } + if (bytes > 0) { + HVX_Vector s = vloadu_partial_zero(src_argb, bytes); + vstoreu_partial(dst_argb, Q6_V_vor_VV(s, v), bytes); + } +} + void MirrorRow_HVX(const uint8_t* src, uint8_t* dst, int width) { HVX_Vector ctrl = Q6_Vb_vsplat_R(0x7F); src += width; @@ -639,6 +658,73 @@ void ARGBBlendRow_HVX(const uint8_t* src_argb, } } +void ARGBAlphaBlendRow_HVX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + HVX_Vector vzero = Q6_V_vzero(); + HVX_Vector v0001 = Q6_Vh_vsplat_R(1); + HVX_Vector v00ff = Q6_Vh_vsplat_R(255); + HVX_Vector vffff = Q6_Vh_vsplat_R(-1); + HVX_Vector v_rgb_mask = Q6_V_vsplat_R(0x00FFFFFFu); + int bytes = width * 4; + while (bytes > 0) { + int b_cnt = bytes >= 128 ? 128 : bytes; + HVX_Vector fg = vloadu_partial_zero(src_argb0, b_cnt); + HVX_Vector bg = vloadu_partial_zero(src_argb1, b_cnt); + + // Broadcast sa and da to all 4 byte lanes of each 32-bit pixel. + HVX_Vector t_fg = Q6_Vb_vshuffo_VbVb(fg, fg); + HVX_Vector sa_vec = Q6_Vh_vshuffo_VhVh(t_fg, t_fg); + HVX_Vector t_bg = Q6_Vb_vshuffo_VbVb(bg, bg); + HVX_Vector da_vec = Q6_Vh_vshuffo_VhVh(t_bg, t_bg); + + // Both uint16_t halfwords of each pixel hold sa and da (0..255). + HVX_Vector sa16 = Q6_V_vand_VV(sa_vec, v00ff); + HVX_Vector da16 = Q6_V_vand_VV(da_vec, v00ff); + + // x = da * sa in both uint16_t halfwords of each pixel. + HVX_Vector x = Q6_V_lo_W(Q6_Wuh_vmpy_VubVub(da_vec, sa_vec)); + // prod_div255 = (x + 1 + (x >> 8)) >> 8. + HVX_Vector prod_div255 = Q6_Vuh_vlsr_VuhR( + Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(x, v0001), Q6_Vuh_vlsr_VuhR(x, 8)), 8); + // out_a = da + sa - (da * sa) / 255. + HVX_Vector out_a16 = + Q6_Vh_vadd_VhVh(da16, Q6_Vh_vsub_VhVh(sa16, prod_div255)); + + // 8-step restoring division: ratio = (da == 0) ? 255 : (sa * 255) / out_a. + HVX_Vector rem = Q6_Vh_vsub_VhVh(Q6_Vh_vasl_VhR(sa16, 8), sa16); + HVX_VectorPred q_da_zero = Q6_Q_vcmp_eq_VhVh(da16, vzero); + rem = Q6_V_vmux_QVV(q_da_zero, vffff, rem); + HVX_Vector ratio = vzero; + + for (int bit = 7; bit >= 0; --bit) { + HVX_Vector d_sh = Q6_Vh_vasl_VhR(out_a16, bit); + HVX_VectorPred q_gt = Q6_Q_vcmp_gt_VuhVuh(d_sh, rem); + rem = Q6_V_vmux_QVV(q_gt, rem, Q6_Vh_vsub_VhVh(rem, d_sh)); + ratio = Q6_V_vmux_QVV(q_gt, ratio, + Q6_V_vor_VV(ratio, Q6_Vb_vsplat_R(1 << bit))); + } + + // Exact RGB lerp: (bg * (255 - ratio) + fg * ratio) / 255. + HVX_Vector inv_ratio = Q6_V_vnot_V(ratio); + HVX_VectorPair p_rgb = Q6_Wuh_vmpy_VubVub(fg, ratio); + p_rgb = Q6_Wuh_vmpyacc_WuhVubVub(p_rgb, bg, inv_ratio); + HVX_Vector lo = Q6_V_lo_W(p_rgb); + HVX_Vector hi = Q6_V_hi_W(p_rgb); + lo = Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(lo, v0001), Q6_Vuh_vlsr_VuhR(lo, 8)); + hi = Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(hi, v0001), Q6_Vuh_vlsr_VuhR(hi, 8)); + HVX_Vector out = Q6_Vb_vshuffo_VbVb(hi, lo); + out = + Q6_V_vor_VV(Q6_V_vand_VV(out, v_rgb_mask), Q6_Vw_vasl_VwR(out_a16, 24)); + vstoreu_partial(dst_argb, out, b_cnt); + src_argb0 += 128; + src_argb1 += 128; + dst_argb += 128; + bytes -= 128; + } +} + void ARGBAttenuateRow_HVX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { diff --git a/source/row_lasx.cc b/source/row_lasx.cc index 99042acd2..ea466412f 100644 --- a/source/row_lasx.cc +++ b/source/row_lasx.cc @@ -1965,6 +1965,89 @@ void ARGBBlendRow_LASX(const uint8_t* src_argb0, "$xr7", "$xr8"); } +void ARGBOrRow_LASX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "xvreplgr2vr.w $xr1, %[val]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvor.v $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(value) + : "memory", "$xr0", "$xr1"); +} + +void ARGBAlphaBlendRow_LASX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + static const uint8_t kShufRatio[16] = {0, 0, 0, 0, 4, 4, 4, 4, + 8, 8, 8, 8, 12, 12, 12, 12}; + asm volatile( + "vld $vr8, %[shuf], 0\n\t" + "xvpermi.q $xr8, $xr8, 0x00\n\t" + "xvldi $xr9, 1\n\t" + "xvsrli.h $xr10, $xr9, 8\n\t" + "xvldi $xr11, 0xff\n\t" + "xvsrli.w $xr11, $xr11, 8\n\t" + "xvldi $xr12, 0\n\t" + "1:\n\t" + "xvld $xr0, %[s0], 0\n\t" + "xvld $xr1, %[s1], 0\n\t" + "xvsrli.w $xr2, $xr0, 24\n\t" + "xvsrli.w $xr3, $xr1, 24\n\t" + "xvmul.h $xr4, $xr3, $xr2\n\t" + "xvadd.h $xr4, $xr4, $xr10\n\t" + "xvmuh.hu $xr4, $xr4, $xr9\n\t" + "xvsub.w $xr4, $xr2, $xr4\n\t" + "xvadd.w $xr4, $xr3, $xr4\n\t" + "xvseq.w $xr5, $xr3, $xr12\n\t" + "xvsrli.w $xr5, $xr5, 24\n\t" + "xvor.v $xr6, $xr2, $xr5\n\t" + "xvor.v $xr7, $xr4, $xr5\n\t" + "xvslli.w $xr5, $xr6, 8\n\t" + "xvsub.w $xr5, $xr5, $xr6\n\t" + "xvffint.s.wu $xr5, $xr5\n\t" + "xvffint.s.wu $xr7, $xr7\n\t" + "xvfdiv.s $xr5, $xr5, $xr7\n\t" + "xvftintrz.wu.s $xr5, $xr5\n\t" + "xvshuf.b $xr5, $xr5, $xr5, $xr8\n\t" + "xvnor.v $xr6, $xr5, $xr5\n\t" + "xvmulwev.h.bu $xr2, $xr1, $xr6\n\t" + "xvmulwod.h.bu $xr3, $xr1, $xr6\n\t" + "xvmulwev.h.bu $xr7, $xr0, $xr5\n\t" + "xvmulwod.h.bu $xr0, $xr0, $xr5\n\t" + "xvadd.h $xr2, $xr2, $xr7\n\t" + "xvadd.h $xr3, $xr3, $xr0\n\t" + "xvadd.h $xr2, $xr2, $xr10\n\t" + "xvadd.h $xr3, $xr3, $xr10\n\t" + "xvmuh.hu $xr2, $xr2, $xr9\n\t" + "xvmuh.hu $xr3, $xr3, $xr9\n\t" + "xvslli.h $xr3, $xr3, 8\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvand.v $xr2, $xr2, $xr11\n\t" + "xvslli.w $xr4, $xr4, 24\n\t" + "xvor.v $xr2, $xr2, $xr4\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [shuf] "r"(kShufRatio) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12"); +} + void ARGBShadeRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width, diff --git a/source/row_lsx.cc b/source/row_lsx.cc index 8bd10f519..cdfe51ace 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -1683,6 +1683,88 @@ void ARGBBlendRow_LSX(const uint8_t* src_argb0, "$vr7", "$vr8"); } +void ARGBOrRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "vreplgr2vr.w $vr1, %[val]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(value) + : "memory", "$vr0", "$vr1"); +} + +void ARGBAlphaBlendRow_LSX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + static const uint8_t kShufRatio[16] = {0, 0, 0, 0, 4, 4, 4, 4, + 8, 8, 8, 8, 12, 12, 12, 12}; + asm volatile( + "vld $vr8, %[shuf], 0\n\t" + "vldi $vr9, 1\n\t" + "vsrli.h $vr10, $vr9, 8\n\t" + "vldi $vr11, 0xff\n\t" + "vsrli.w $vr11, $vr11, 8\n\t" + "vldi $vr12, 0\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vsrli.w $vr2, $vr0, 24\n\t" + "vsrli.w $vr3, $vr1, 24\n\t" + "vmul.h $vr4, $vr3, $vr2\n\t" + "vadd.h $vr4, $vr4, $vr10\n\t" + "vmuh.hu $vr4, $vr4, $vr9\n\t" + "vsub.w $vr4, $vr2, $vr4\n\t" + "vadd.w $vr4, $vr3, $vr4\n\t" + "vseq.w $vr5, $vr3, $vr12\n\t" + "vsrli.w $vr5, $vr5, 24\n\t" + "vor.v $vr6, $vr2, $vr5\n\t" + "vor.v $vr7, $vr4, $vr5\n\t" + "vslli.w $vr5, $vr6, 8\n\t" + "vsub.w $vr5, $vr5, $vr6\n\t" + "vffint.s.wu $vr5, $vr5\n\t" + "vffint.s.wu $vr7, $vr7\n\t" + "vfdiv.s $vr5, $vr5, $vr7\n\t" + "vftintrz.wu.s $vr5, $vr5\n\t" + "vshuf.b $vr5, $vr5, $vr5, $vr8\n\t" + "vnor.v $vr6, $vr5, $vr5\n\t" + "vmulwev.h.bu $vr2, $vr1, $vr6\n\t" + "vmulwod.h.bu $vr3, $vr1, $vr6\n\t" + "vmulwev.h.bu $vr7, $vr0, $vr5\n\t" + "vmulwod.h.bu $vr0, $vr0, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr7\n\t" + "vadd.h $vr3, $vr3, $vr0\n\t" + "vadd.h $vr2, $vr2, $vr10\n\t" + "vadd.h $vr3, $vr3, $vr10\n\t" + "vmuh.hu $vr2, $vr2, $vr9\n\t" + "vmuh.hu $vr3, $vr3, $vr9\n\t" + "vslli.h $vr3, $vr3, 8\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vand.v $vr2, $vr2, $vr11\n\t" + "vslli.w $vr4, $vr4, 24\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [shuf] "r"(kShufRatio) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12"); +} + void ARGBShadeRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width, diff --git a/source/row_neon.cc b/source/row_neon.cc index 8f921c1e4..ea072ee9b 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -2822,6 +2822,103 @@ void ARGBBlendRow_NEON(const uint8_t* src_argb, : "cc", "memory", "q0", "q1", "q2", "q3", "q10", "q11", "q12"); } +void ARGBOrRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "vdup.32 q2, %3 \n" + "1: \n" + "vld1.8 {q0, q1}, [%0]! \n" + "subs %2, %2, #8 \n" + "vorr q0, q0, q2 \n" + "vorr q1, q1, q2 \n" + "vst1.8 {q0, q1}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(value) // %3 + : "memory", "cc", "q0", "q1", "q2"); +} + +void ARGBAlphaBlendRow_NEON(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "vmov.u8 d28, #255 \n" + "vmov.i16 q15, #1 \n" + "1: \n" + "vld4.8 {d0, d1, d2, d3}, [%0]! \n" + "vld4.8 {d4, d5, d6, d7}, [%1]! \n" + "subs %3, %3, #8 \n" + "vmov q8, q15 \n" + "vmlal.u8 q8, d7, d3 \n" + "vsra.u16 q8, q8, #8 \n" + "vshrn.i16 d16, q8, #8 \n" + "vsub.i8 d16, d3, d16 \n" + "vadd.i8 d7, d7, d16 \n" + "vceq.i8 d17, d7, #0 \n" + "vorr d18, d3, d17 \n" + "vorr d19, d7, d17 \n" + "vmull.u8 q8, d18, d28 \n" + "vmovl.u8 q9, d19 \n" + "vmovl.u16 q10, d16 \n" + "vmovl.u16 q11, d17 \n" + "vmovl.u16 q12, d18 \n" + "vmovl.u16 q13, d19 \n" + "vcvt.f32.u32 q10, q10 \n" + "vcvt.f32.u32 q11, q11 \n" + "vcvt.f32.u32 q12, q12 \n" + "vcvt.f32.u32 q13, q13 \n" + "vrecpe.f32 q8, q12 \n" + "vrecpe.f32 q9, q13 \n" + "vrecps.f32 q4, q12, q8 \n" + "vrecps.f32 q5, q13, q9 \n" + "vmul.f32 q8, q8, q4 \n" + "vmul.f32 q9, q9, q5 \n" + "vrecps.f32 q4, q12, q8 \n" + "vrecps.f32 q5, q13, q9 \n" + "vmul.f32 q8, q8, q4 \n" + "vmul.f32 q9, q9, q5 \n" + "vsra.u32 q8, q8, #29 \n" + "vsra.u32 q9, q9, #29 \n" + "vmul.f32 q10, q10, q8 \n" + "vmul.f32 q11, q11, q9 \n" + "vcvt.u32.f32 q10, q10 \n" + "vcvt.u32.f32 q11, q11 \n" + "vmovn.i32 d16, q10 \n" + "vmovn.i32 d17, q11 \n" + "vmovn.i16 d16, q8 \n" + "vsub.i8 d17, d28, d16 \n" + "vmov q9, q15 \n" + "vmov q10, q15 \n" + "vmov q11, q15 \n" + "vmlal.u8 q9, d4, d17 \n" + "vmlal.u8 q10, d5, d17 \n" + "vmlal.u8 q11, d6, d17 \n" + "vmlal.u8 q9, d0, d16 \n" + "vmlal.u8 q10, d1, d16 \n" + "vmlal.u8 q11, d2, d16 \n" + "vsra.u16 q9, q9, #8 \n" + "vsra.u16 q10, q10, #8 \n" + "vsra.u16 q11, q11, #8 \n" + "vshrn.i16 d0, q9, #8 \n" + "vshrn.i16 d1, q10, #8 \n" + "vshrn.i16 d2, q11, #8 \n" + "vmov d3, d7 \n" + "vst4.8 {d0, d1, d2, d3}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9", "q10", + "q11", "q12", "q13", "q14", "q15"); +} + // Attenuate 8 pixels at a time. void ARGBAttenuateRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, diff --git a/source/row_neon64.cc b/source/row_neon64.cc index f4e0315c6..f23ddcc8f 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -3866,6 +3866,91 @@ void ARGBBlendRow_NEON(const uint8_t* src_argb, "v17", "v18"); } +void ARGBOrRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "dup v2.4s, %w3 \n" + "1: \n" + "ldp q0, q1, [%0], #32 \n" + "subs %w2, %w2, #8 \n" + "orr v0.16b, v0.16b, v2.16b \n" + "orr v1.16b, v1.16b, v2.16b \n" + "stp q0, q1, [%1], #32 \n" + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(value) // %3 + : "memory", "cc", "v0", "v1", "v2"); +} + +void ARGBAlphaBlendRow_NEON(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "movi v28.8b, #255 \n" + "movi v29.8h, #1 \n" + "1: \n" + "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" + "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" + "subs %w3, %w3, #8 \n" + "mov v16.16b, v29.16b \n" + "umlal v16.8h, v7.8b, v3.8b \n" + "usra v16.8h, v16.8h, #8 \n" + "shrn v16.8b, v16.8h, #8 \n" + "cmeq v17.8b, v7.8b, #0 \n" + "sub v16.8b, v3.8b, v16.8b \n" + "add v7.8b, v7.8b, v16.8b \n" + "orr v18.8b, v3.8b, v17.8b \n" + "orr v19.8b, v7.8b, v17.8b \n" + "umull v16.8h, v18.8b, v28.8b \n" + "uxtl v17.8h, v19.8b \n" + "uxtl v20.4s, v16.4h \n" + "uxtl2 v21.4s, v16.8h \n" + "uxtl v22.4s, v17.4h \n" + "uxtl2 v23.4s, v17.8h \n" + "ucvtf v20.4s, v20.4s \n" + "ucvtf v21.4s, v21.4s \n" + "ucvtf v22.4s, v22.4s \n" + "ucvtf v23.4s, v23.4s \n" + "fdiv v20.4s, v20.4s, v22.4s \n" + "fdiv v21.4s, v21.4s, v23.4s \n" + "fcvtzu v20.4s, v20.4s \n" + "fcvtzu v21.4s, v21.4s \n" + "xtn v16.4h, v20.4s \n" + "xtn2 v16.8h, v21.4s \n" + "xtn v16.8b, v16.8h \n" + "sub v17.8b, v28.8b, v16.8b \n" + "mov v20.16b, v29.16b \n" + "mov v21.16b, v29.16b \n" + "mov v22.16b, v29.16b \n" + "umlal v20.8h, v4.8b, v17.8b \n" + "umlal v21.8h, v5.8b, v17.8b \n" + "umlal v22.8h, v6.8b, v17.8b \n" + "umlal v20.8h, v0.8b, v16.8b \n" + "umlal v21.8h, v1.8b, v16.8b \n" + "umlal v22.8h, v2.8b, v16.8b \n" + "usra v20.8h, v20.8h, #8 \n" + "usra v21.8h, v21.8h, #8 \n" + "usra v22.8h, v22.8h, #8 \n" + "shrn v0.8b, v20.8h, #8 \n" + "shrn v1.8b, v21.8h, #8 \n" + "shrn v2.8b, v22.8h, #8 \n" + "mov v3.8b, v7.8b \n" + "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v28", "v29"); +} + // Attenuate 8 pixels at a time. void ARGBAttenuateRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 6b7173675..2f899ca3c 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -2245,6 +2245,99 @@ void ARGBBlendRow_RVV(const uint8_t* src_argb, } #endif +#ifdef HAS_ARGBORROW_RVV +void ARGBOrRow_RVV(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + size_t vl; + int w = width * 4; + asm volatile( + "vsetvli %[vl], zero, e32, m8, ta, ma \n" + "vmv.v.x v16, %[val] \n" + "1: \n" + "vsetvli %[vl], %[w], e8, m8, ta, ma \n" + "vle8.v v8, (%[src]) \n" + "vor.vv v8, v8, v16 \n" + "vse8.v v8, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src], %[src], %[vl] \n" + "add %[dst], %[dst], %[vl] \n" + "bgtz %[w], 1b \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(w), // %[w] + [vl] "=&r"(vl) // %[vl] + : [val] "r"(value) // %[val] + : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} +#endif + +#ifdef HAS_ARGBALPHABLENDROW_RVV +void ARGBAlphaBlendRow_RVV(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v16, (%[src_argb]) \n" + "vlseg4e8.v v24, (%[src_argb1]) \n" + "vwmulu.vv v4, v30, v22 \n" + "vwmulu.vx v8, v22, %[k255] \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vi v4, v4, 1 \n" + "vmulhu.vx v4, v4, %[k0101] \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v0, v4, 0 \n" + "vsub.vv v0, v22, v0 \n" + "vadd.vv v30, v30, v0 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vzext.vf2 v12, v30 \n" + "vdivu.vv v8, v8, v12 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v0, v8, 0 \n" + "vrsub.vx v2, v0, %[k255] \n" + "vwmulu.vv v4, v24, v2 \n" + "vwmulu.vv v8, v26, v2 \n" + "vwmulu.vv v12, v28, v2 \n" + "vwmaccu.vv v4, v0, v16 \n" + "vwmaccu.vv v8, v0, v18 \n" + "vwmaccu.vv v12, v0, v20 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vi v4, v4, 1 \n" + "vadd.vi v8, v8, 1 \n" + "vadd.vi v12, v12, 1 \n" + "vmulhu.vx v4, v4, %[k0101] \n" + "vmulhu.vx v8, v8, %[k0101] \n" + "vmulhu.vx v12, v12, %[k0101] \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v24, v4, 0 \n" + "vnsrl.wi v26, v8, 0 \n" + "vnsrl.wi v28, v12, 0 \n" + "vsseg4e8.v v24, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl] \n" + "add %[src_argb1], %[src_argb1], %[vl]\n" + "add %[dst_argb], %[dst_argb], %[vl] \n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [src_argb1] "+r"(src_argb1), // %[src_argb1] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [k0101] "r"(0x0101), // %[k0101] + [k255] "r"(255) // %[k255] + : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30", "v31"); +} +#endif + #ifdef HAS_BLENDPLANEROW_RVV void BlendPlaneRow_RVV(const uint8_t* src0, const uint8_t* src1, diff --git a/source/row_sme.cc b/source/row_sme.cc index 5ec51382c..21abac922 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -1142,6 +1142,24 @@ __arm_locally_streaming float SsimWindowRow_SME(const uint32_t* row0, } #endif // HAS_SSIMWINDOWROW_SME +#ifdef HAS_ARGBORROW_SME +__arm_locally_streaming void ARGBOrRow_SME(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + ARGBOrRow_SVE_SC(src_argb, dst_argb, value, width); +} +#endif // HAS_ARGBORROW_SME + +#ifdef HAS_ARGBALPHABLENDROW_SME +__arm_locally_streaming void ARGBAlphaBlendRow_SME(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + ARGBAlphaBlendRow_SVE_SC(src_argb0, src_argb1, dst_argb, width); +} +#endif // HAS_ARGBALPHABLENDROW_SME + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/row_sve.cc b/source/row_sve.cc index 2cec59d30..28c3b1a16 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1751,6 +1751,24 @@ float SsimWindowRow_SVE2(const uint32_t* row0, } #endif // HAS_SSIMWINDOWROW_SVE2 +#ifdef HAS_ARGBORROW_SVE2 +void ARGBOrRow_SVE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + ARGBOrRow_SVE_SC(src_argb, dst_argb, value, width); +} +#endif // HAS_ARGBORROW_SVE2 + +#ifdef HAS_ARGBALPHABLENDROW_SVE2 +void ARGBAlphaBlendRow_SVE2(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + ARGBAlphaBlendRow_SVE_SC(src_argb0, src_argb1, dst_argb, width); +} +#endif // HAS_ARGBALPHABLENDROW_SVE2 + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/row_wasm.cc b/source/row_wasm.cc index a68774edd..df9e3917a 100644 --- a/source/row_wasm.cc +++ b/source/row_wasm.cc @@ -1970,6 +1970,73 @@ void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBCOPYYTOALPHAROW_WASMSIMD +#ifdef HAS_ARGBORROW_WASMSIMD +void ARGBOrRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + v128_t val = wasm_i32x4_splat((int32_t)value); + do { + asm("" : "+r"(src_argb), "+r"(dst_argb)); + v128_t s = wasm_v128_load(src_argb); + wasm_v128_store(dst_argb, wasm_v128_or(s, val)); + src_argb += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBORROW_WASMSIMD + +#ifdef HAS_ARGBALPHABLENDROW_WASMSIMD +void ARGBAlphaBlendRow_WASMSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + v128_t c1 = SplatConst16(1); + v128_t c00ffffff = SplatConst32(0x00ffffff); + do { + asm("" : "+r"(src_argb0), "+r"(src_argb1), "+r"(dst_argb)); + v128_t s = wasm_v128_load(src_argb0); + v128_t d = wasm_v128_load(src_argb1); + src_argb0 += 16; + src_argb1 += 16; + v128_t sa = wasm_u32x4_shr(s, 24); + v128_t da = wasm_u32x4_shr(d, 24); + v128_t prod16 = wasm_i16x8_add(wasm_i16x8_mul(da, sa), c1); + v128_t da_sa = + wasm_u16x8_shr(wasm_i16x8_add(prod16, wasm_u16x8_shr(prod16, 8)), 8); + v128_t da2 = wasm_i32x4_add(da, wasm_i32x4_sub(sa, da_sa)); + v128_t t = wasm_u32x4_shr(wasm_i32x4_eq(da, wasm_i32x4_const_splat(0)), 24); + v128_t sa_eff = wasm_v128_or(sa, t); + v128_t den_eff = wasm_v128_or(da2, t); + v128_t num = wasm_i32x4_sub(wasm_i32x4_shl(sa_eff, 8), sa_eff); + v128_t ratio = wasm_u32x4_trunc_sat_f32x4(wasm_f32x4_div( + wasm_f32x4_convert_u32x4(num), wasm_f32x4_convert_u32x4(den_eff))); + v128_t r8 = wasm_i8x16_shuffle(ratio, ratio, 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, + 8, 8, 12, 12, 12, 12); + v128_t inv_r8 = wasm_v128_not(r8); + v128_t acc_lo = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(d, inv_r8), + wasm_u16x8_extmul_low_u8x16(s, r8)), + c1); + v128_t acc_hi = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(d, inv_r8), + wasm_u16x8_extmul_high_u8x16(s, r8)), + c1); + acc_lo = + wasm_u16x8_shr(wasm_i16x8_add(acc_lo, wasm_u16x8_shr(acc_lo, 8)), 8); + acc_hi = + wasm_u16x8_shr(wasm_i16x8_add(acc_hi, wasm_u16x8_shr(acc_hi, 8)), 8); + v128_t rgb = wasm_u8x16_narrow_i16x8(acc_lo, acc_hi); + v128_t out = wasm_v128_or(wasm_v128_and(rgb, c00ffffff), + wasm_i32x4_shl(da2, 24)); + wasm_v128_store(dst_argb, out); + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBALPHABLENDROW_WASMSIMD + #undef LIBYUV_WASM_SWIZZLE #ifdef __cplusplus diff --git a/source/row_win.cc b/source/row_win.cc index e00419249..88c6af8fc 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -4371,6 +4371,267 @@ void ARGBBlendRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBBLENDROW_AVX512BW +#ifdef HAS_ARGBORROW_SSE2 +void ARGBOrRow_SSE2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + const __m128i val = _mm_set1_epi32((int)value); + while (width >= 4) { + __m128i s = _mm_loadu_si128((const __m128i*)src_argb); + src_argb += 16; + _mm_storeu_si128((__m128i*)dst_argb, _mm_or_si128(s, val)); + dst_argb += 16; + width -= 4; + } +} +#endif // HAS_ARGBORROW_SSE2 + +#ifdef HAS_ARGBORROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBOrRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + const __m256i val = _mm256_set1_epi32((int)value); + while (width >= 8) { + __m256i s = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_or_si256(s, val)); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBORROW_AVX2 + +#ifdef HAS_ARGBORROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBOrRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + const __m512i val = _mm512_set1_epi32((int)value); + while (width >= 16) { + __m512i s = _mm512_loadu_si512((const __m512i*)src_argb); + src_argb += 64; + _mm512_storeu_si512((__m512i*)dst_argb, _mm512_or_si512(s, val)); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s = _mm512_maskz_loadu_epi32(k, src_argb); + _mm512_mask_storeu_epi32(dst_argb, k, _mm512_or_si512(s, val)); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBORROW_AVX512BW + +#ifdef HAS_ARGBALPHABLENDROW_SSSE3 +LIBYUV_TARGET_SSSE3 +void ARGBAlphaBlendRow_SSSE3(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + const __m128i zero = _mm_setzero_si128(); + const __m128i k0001 = _mm_set1_epi16(0x0001); + const __m128i k0101 = _mm_set1_epi16(0x0101); + const __m128i k00ff = _mm_set1_epi16(0x00ff); + const __m128i k8080 = _mm_set1_epi8((char)0x80); + const __m128i k7f81 = _mm_set1_epi16(0x7f81); + const __m128i k00ffffff = _mm_set1_epi32(0x00ffffff); + while (width >= 4) { + __m128i s = _mm_loadu_si128((const __m128i*)src_argb0); + __m128i d = _mm_loadu_si128((const __m128i*)src_argb1); + src_argb0 += 16; + src_argb1 += 16; + __m128i sa = _mm_srli_epi32(s, 24); + __m128i da = _mm_srli_epi32(d, 24); + __m128i da_sa = _mm_mulhi_epu16( + _mm_add_epi16(_mm_mullo_epi16(da, sa), k0001), k0101); + __m128i da2 = _mm_add_epi32(da, _mm_sub_epi32(sa, da_sa)); + __m128i t = _mm_srli_epi32(_mm_cmpeq_epi32(da, zero), 24); + __m128i sa_eff = _mm_or_si128(sa, t); + __m128i den_eff = _mm_or_si128(da2, t); + __m128i num = _mm_sub_epi32(_mm_slli_epi32(sa_eff, 8), sa_eff); + __m128i ratio = _mm_cvttps_epi32( + _mm_div_ps(_mm_cvtepi32_ps(num), _mm_cvtepi32_ps(den_eff))); + __m128i r_rep = _mm_shufflelo_epi16(_mm_shufflehi_epi16(ratio, 0xa0), 0xa0); + __m128i w = + _mm_or_si128(r_rep, _mm_slli_epi16(_mm_sub_epi16(k00ff, r_rep), 8)); + __m128i w_lo = _mm_unpacklo_epi16(w, w); + __m128i w_hi = _mm_unpackhi_epi16(w, w); + __m128i sb = _mm_sub_epi8(s, k8080); + __m128i db = _mm_sub_epi8(d, k8080); + __m128i lo = _mm_mulhi_epu16( + _mm_add_epi16(_mm_maddubs_epi16(w_lo, _mm_unpacklo_epi8(sb, db)), + k7f81), + k0101); + __m128i hi = _mm_mulhi_epu16( + _mm_add_epi16(_mm_maddubs_epi16(w_hi, _mm_unpackhi_epi8(sb, db)), + k7f81), + k0101); + __m128i rgb = _mm_packus_epi16(lo, hi); + __m128i out = + _mm_or_si128(_mm_and_si128(rgb, k00ffffff), _mm_slli_epi32(da2, 24)); + _mm_storeu_si128((__m128i*)dst_argb, out); + dst_argb += 16; + width -= 4; + } +} +#endif // HAS_ARGBALPHABLENDROW_SSSE3 + +#ifdef HAS_ARGBALPHABLENDROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBAlphaBlendRow_AVX2(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + const __m256i zero = _mm256_setzero_si256(); + const __m256i k0001 = _mm256_set1_epi16(0x0001); + const __m256i k0101 = _mm256_set1_epi16(0x0101); + const __m256i k00ff = _mm256_set1_epi16(0x00ff); + const __m256i k8080 = _mm256_set1_epi8((char)0x80); + const __m256i k7f81 = _mm256_set1_epi16(0x7f81); + const __m256i k00ffffff = _mm256_set1_epi32(0x00ffffff); + while (width >= 8) { + __m256i s = _mm256_loadu_si256((const __m256i*)src_argb0); + __m256i d = _mm256_loadu_si256((const __m256i*)src_argb1); + src_argb0 += 32; + src_argb1 += 32; + __m256i sa = _mm256_srli_epi32(s, 24); + __m256i da = _mm256_srli_epi32(d, 24); + __m256i da_sa = _mm256_mulhi_epu16( + _mm256_add_epi16(_mm256_mullo_epi16(da, sa), k0001), k0101); + __m256i da2 = _mm256_add_epi32(da, _mm256_sub_epi32(sa, da_sa)); + __m256i t = _mm256_srli_epi32(_mm256_cmpeq_epi32(da, zero), 24); + __m256i sa_eff = _mm256_or_si256(sa, t); + __m256i den_eff = _mm256_or_si256(da2, t); + __m256i num = _mm256_sub_epi32(_mm256_slli_epi32(sa_eff, 8), sa_eff); + __m256i ratio = _mm256_cvttps_epi32( + _mm256_div_ps(_mm256_cvtepi32_ps(num), _mm256_cvtepi32_ps(den_eff))); + __m256i r_rep = + _mm256_shufflelo_epi16(_mm256_shufflehi_epi16(ratio, 0xa0), 0xa0); + __m256i w = _mm256_or_si256( + r_rep, _mm256_slli_epi16(_mm256_sub_epi16(k00ff, r_rep), 8)); + __m256i w_lo = _mm256_unpacklo_epi16(w, w); + __m256i w_hi = _mm256_unpackhi_epi16(w, w); + __m256i sb = _mm256_sub_epi8(s, k8080); + __m256i db = _mm256_sub_epi8(d, k8080); + __m256i lo = _mm256_mulhi_epu16( + _mm256_add_epi16( + _mm256_maddubs_epi16(w_lo, _mm256_unpacklo_epi8(sb, db)), k7f81), + k0101); + __m256i hi = _mm256_mulhi_epu16( + _mm256_add_epi16( + _mm256_maddubs_epi16(w_hi, _mm256_unpackhi_epi8(sb, db)), k7f81), + k0101); + __m256i rgb = _mm256_packus_epi16(lo, hi); + __m256i out = _mm256_or_si256(_mm256_and_si256(rgb, k00ffffff), + _mm256_slli_epi32(da2, 24)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBALPHABLENDROW_AVX2 + +#ifdef HAS_ARGBALPHABLENDROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + const __m512i zero = _mm512_setzero_si512(); + const __m512i k0001 = _mm512_set1_epi16(0x0001); + const __m512i k0101 = _mm512_set1_epi16(0x0101); + const __m512i k00ff = _mm512_set1_epi16(0x00ff); + const __m512i k8080 = _mm512_set1_epi8((char)0x80); + const __m512i k7f81 = _mm512_set1_epi16(0x7f81); + const __m512i k00ffffff = _mm512_set1_epi32(0x00ffffff); + while (width >= 16) { + __m512i s = _mm512_loadu_si512((const __m512i*)src_argb0); + __m512i d = _mm512_loadu_si512((const __m512i*)src_argb1); + src_argb0 += 64; + src_argb1 += 64; + __m512i sa = _mm512_srli_epi32(s, 24); + __m512i da = _mm512_srli_epi32(d, 24); + __m512i da_sa = _mm512_mulhi_epu16( + _mm512_add_epi16(_mm512_mullo_epi16(da, sa), k0001), k0101); + __m512i da2 = _mm512_add_epi32(da, _mm512_sub_epi32(sa, da_sa)); + __mmask16 k_da0 = _mm512_cmpeq_epi32_mask(da, zero); + __m512i t = _mm512_maskz_set1_epi32(k_da0, 255); + __m512i sa_eff = _mm512_or_si512(sa, t); + __m512i den_eff = _mm512_or_si512(da2, t); + __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff); + __m512i ratio = _mm512_cvttps_epi32( + _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff))); + __m512i r_rep = + _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0); + __m512i w = _mm512_or_si512( + r_rep, _mm512_slli_epi16(_mm512_sub_epi16(k00ff, r_rep), 8)); + __m512i w_lo = _mm512_unpacklo_epi16(w, w); + __m512i w_hi = _mm512_unpackhi_epi16(w, w); + __m512i sb = _mm512_sub_epi8(s, k8080); + __m512i db = _mm512_sub_epi8(d, k8080); + __m512i lo = _mm512_mulhi_epu16( + _mm512_add_epi16( + _mm512_maddubs_epi16(w_lo, _mm512_unpacklo_epi8(sb, db)), k7f81), + k0101); + __m512i hi = _mm512_mulhi_epu16( + _mm512_add_epi16( + _mm512_maddubs_epi16(w_hi, _mm512_unpackhi_epi8(sb, db)), k7f81), + k0101); + __m512i rgb = _mm512_packus_epi16(lo, hi); + __m512i out = _mm512_or_si512(_mm512_and_si512(rgb, k00ffffff), + _mm512_slli_epi32(da2, 24)); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s = _mm512_maskz_loadu_epi32(k, src_argb0); + __m512i d = _mm512_maskz_loadu_epi32(k, src_argb1); + __m512i sa = _mm512_srli_epi32(s, 24); + __m512i da = _mm512_srli_epi32(d, 24); + __m512i da_sa = _mm512_mulhi_epu16( + _mm512_add_epi16(_mm512_mullo_epi16(da, sa), k0001), k0101); + __m512i da2 = _mm512_add_epi32(da, _mm512_sub_epi32(sa, da_sa)); + __mmask16 k_da0 = _mm512_cmpeq_epi32_mask(da, zero); + __m512i t = _mm512_maskz_set1_epi32(k_da0, 255); + __m512i sa_eff = _mm512_or_si512(sa, t); + __m512i den_eff = _mm512_or_si512(da2, t); + __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff); + __m512i ratio = _mm512_cvttps_epi32( + _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff))); + __m512i r_rep = + _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0); + __m512i w = _mm512_or_si512( + r_rep, _mm512_slli_epi16(_mm512_sub_epi16(k00ff, r_rep), 8)); + __m512i w_lo = _mm512_unpacklo_epi16(w, w); + __m512i w_hi = _mm512_unpackhi_epi16(w, w); + __m512i sb = _mm512_sub_epi8(s, k8080); + __m512i db = _mm512_sub_epi8(d, k8080); + __m512i lo = _mm512_mulhi_epu16( + _mm512_add_epi16( + _mm512_maddubs_epi16(w_lo, _mm512_unpacklo_epi8(sb, db)), k7f81), + k0101); + __m512i hi = _mm512_mulhi_epu16( + _mm512_add_epi16( + _mm512_maddubs_epi16(w_hi, _mm512_unpackhi_epi8(sb, db)), k7f81), + k0101); + __m512i rgb = _mm512_packus_epi16(lo, hi); + __m512i out = _mm512_or_si512(_mm512_and_si512(rgb, k00ffffff), + _mm512_slli_epi32(da2, 24)); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBALPHABLENDROW_AVX512BW + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index e891ba346..d12be099b 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -5095,4 +5095,182 @@ TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32Column) { #endif // defined(ENABLE_ROW_TESTS) && defined(__aarch64__) +static int TestARGBOr(int width, + int height, + int benchmark_iterations, + int disable_cpu_flags, + int benchmark_cpu_info, + int invert, + int off, + uint32_t value) { + if (width < 1) { + width = 1; + } + const int kBpp = 4; + const int kStride = width * kBpp; + align_buffer_page_end(src_argb, kStride * abs(height) + off); + align_buffer_page_end(dst_argb_c, kStride * abs(height)); + align_buffer_page_end(dst_argb_opt, kStride * abs(height)); + MemRandomize(src_argb, kStride * abs(height) + off); + memset(dst_argb_c, 0, kStride * abs(height)); + memset(dst_argb_opt, 1, kStride * abs(height)); + + MaskCpuFlags(disable_cpu_flags); + ARGBOr(src_argb + off, kStride, dst_argb_c, kStride, width, invert * height, + value); + MaskCpuFlags(benchmark_cpu_info); + for (int i = 0; i < benchmark_iterations; ++i) { + ARGBOr(src_argb + off, kStride, dst_argb_opt, kStride, width, + invert * height, value); + } + int max_diff = 0; + for (int i = 0; i < kStride * abs(height); ++i) { + int abs_diff = abs(static_cast(dst_argb_c[i]) - + static_cast(dst_argb_opt[i])); + if (abs_diff > max_diff) { + max_diff = abs_diff; + } + } + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(dst_argb_c); + free_aligned_buffer_page_end(dst_argb_opt); + return max_diff; +} + +TEST_F(LibYUVPlanarTest, ARGBOr_Any) { + int max_diff = TestARGBOr(benchmark_width_ + 1, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 0, 0xff000000u); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBOr_Unaligned) { + int max_diff = TestARGBOr(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 1, 0xff000000u); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBOr_Invert) { + int max_diff = TestARGBOr(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, -1, 0, 0xff000000u); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBOr_Opt) { + int max_diff = TestARGBOr(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 0, 0xff000000u); + EXPECT_EQ(0, max_diff); +} + +static int TestARGBAlphaBlend(int width, + int height, + int benchmark_iterations, + int disable_cpu_flags, + int benchmark_cpu_info, + int invert, + int off) { + if (width < 1) { + width = 1; + } + const int kBpp = 4; + const int kStride = width * kBpp; + align_buffer_page_end(src_argb0, kStride * abs(height) + off); + align_buffer_page_end(src_argb1, kStride * abs(height) + off); + align_buffer_page_end(dst_argb_c, kStride * abs(height)); + align_buffer_page_end(dst_argb_opt, kStride * abs(height)); + MemRandomize(src_argb0, kStride * abs(height) + off); + MemRandomize(src_argb1, kStride * abs(height) + off); + memset(dst_argb_c, 0, kStride * abs(height)); + memset(dst_argb_opt, 1, kStride * abs(height)); + + MaskCpuFlags(disable_cpu_flags); + ARGBAlphaBlend(src_argb0 + off, kStride, src_argb1 + off, kStride, dst_argb_c, + kStride, width, invert * height); + MaskCpuFlags(benchmark_cpu_info); + for (int i = 0; i < benchmark_iterations; ++i) { + ARGBAlphaBlend(src_argb0 + off, kStride, src_argb1 + off, kStride, + dst_argb_opt, kStride, width, invert * height); + } + int max_diff = 0; + for (int i = 0; i < kStride * abs(height); ++i) { + int abs_diff = abs(static_cast(dst_argb_c[i]) - + static_cast(dst_argb_opt[i])); + if (abs_diff > max_diff) { + max_diff = abs_diff; + } + } + free_aligned_buffer_page_end(src_argb0); + free_aligned_buffer_page_end(src_argb1); + free_aligned_buffer_page_end(dst_argb_c); + free_aligned_buffer_page_end(dst_argb_opt); + return max_diff; +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Any) { + int max_diff = TestARGBAlphaBlend(benchmark_width_ + 1, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 0); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Unaligned) { + int max_diff = TestARGBAlphaBlend(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 1); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Invert) { + int max_diff = TestARGBAlphaBlend(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, -1, 0); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_Opt) { + int max_diff = TestARGBAlphaBlend(benchmark_width_, benchmark_height_, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_, +1, 0); + EXPECT_EQ(0, max_diff); +} + +TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_AllAlphas) { + const int kWidth = 256; + const int kHeight = 256; + const int kStride = kWidth * 4; + align_buffer_page_end(src_argb0, kStride * kHeight); + align_buffer_page_end(src_argb1, kStride * kHeight); + align_buffer_page_end(dst_argb_c, kStride * kHeight); + align_buffer_page_end(dst_argb_opt, kStride * kHeight); + for (int da = 0; da < 256; ++da) { + for (int sa = 0; sa < 256; ++sa) { + int idx = (da * 256 + sa) * 4; + src_argb0[idx + 0] = (sa * 7 + da * 3 + 13) & 255; + src_argb0[idx + 1] = (sa * 13 + da * 5 + 29) & 255; + src_argb0[idx + 2] = (sa * 29 + da * 11 + 47) & 255; + src_argb0[idx + 3] = sa; + src_argb1[idx + 0] = (da * 17 + sa * 19 + 53) & 255; + src_argb1[idx + 1] = (da * 23 + sa * 31 + 71) & 255; + src_argb1[idx + 2] = (da * 37 + sa * 41 + 89) & 255; + src_argb1[idx + 3] = da; + } + } + MaskCpuFlags(disable_cpu_flags_); + ARGBAlphaBlend(src_argb0, kStride, src_argb1, kStride, dst_argb_c, kStride, + kWidth, kHeight); + MaskCpuFlags(benchmark_cpu_info_); + ARGBAlphaBlend(src_argb0, kStride, src_argb1, kStride, dst_argb_opt, kStride, + kWidth, kHeight); + for (int i = 0; i < kStride * kHeight; ++i) { + ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]) << "Mismatch at byte " << i; + } + free_aligned_buffer_page_end(src_argb0); + free_aligned_buffer_page_end(src_argb1); + free_aligned_buffer_page_end(dst_argb_c); + free_aligned_buffer_page_end(dst_argb_opt); +} + } // namespace libyuv