From 715a84778eb0ff95944dd5b6ef2a724ca648eefb Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Fri, 18 Sep 2026 16:44:52 -0700 Subject: [PATCH] ARGBMultiplyRow with rounding for AVX2 and RVV - AVX2 uses vpmulhrsw to compute ((128 * a * b) + 16384) >> 15. - RVV uses e8, m2 (widening to e16, m4) with vwmulu.vv, vwaddu.wx 128, and vnsrl.wi by 8. CPU C Intrinsics Inline ASM Speedup (Inline) ------------------------------------------------------------------------ SiFive X390 gen 2 (est.) 2150 ms 29 ms 27 ms 79.63x (1.07x) SpacemiT K3 (est.) 980 ms 41 ms 38 ms 25.79x (1.08x) T-Head C908 (est.) 2060 ms 100 ms 94 ms 21.91x (1.06x) Google Coral (est.) 2680 ms 94 ms 88 ms 30.45x (1.07x) Test: libyuv_unittest --gtest_filter=*ARGBMultiply* Bug: None Change-Id: Iff65794623a6f29e5cb6bda7bd3084c8a5f676d1 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8422278 Reviewed-by: richard winterton Commit-Queue: Frank Barchard --- include/libyuv/row.h | 17 ++++++------ source/planar_functions.cc | 5 ++++ source/row_gcc.cc | 34 +++++++++++++++--------- source/row_lasx.cc | 13 +++------ source/row_lsx.cc | 13 +++------ source/row_rvv.cc | 54 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 98 insertions(+), 38 deletions(-) diff --git a/include/libyuv/row.h b/include/libyuv/row.h index c7079fdb3..170d0b7b8 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -86,8 +86,7 @@ extern "C" { #define HAS_ARGBGRAYROW_SSSE3 #define HAS_ARGBLUMACOLORTABLEROW_SSSE3 #define HAS_ARGBMIRRORROW_SSE2 -// TODO: Re-enable once rounding behaviour is fixed. -// #define HAS_ARGBMULTIPLYROW_SSE2 +#define HAS_ARGBMULTIPLYROW_SSE2 #define HAS_ARGBPOLYNOMIALROW_SSE2 #define HAS_ARGBQUANTIZEROW_SSE2 #define HAS_ARGBSEPIAROW_SSSE3 @@ -187,8 +186,7 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_AVX2 -// TODO: Re-enable once rounding behaviour is fixed. -// #define HAS_ARGBMULTIPLYROW_AVX2 +#define HAS_ARGBMULTIPLYROW_AVX2 #define HAS_ARGBSUBTRACTROW_AVX2 #define HAS_BLENDPLANEROW_AVX2 #endif @@ -759,8 +757,7 @@ extern "C" { #define HAS_ARGBTORGB565ROW_LSX #define HAS_ARGBTORGB565DITHERROW_LSX #define HAS_ARGBMIRRORROW_LSX -// TODO: Re-enable once rounding behaviour is fixed. -// #define HAS_ARGBMULTIPLYROW_LSX +#define HAS_ARGBMULTIPLYROW_LSX #define HAS_I400TOARGBROW_LSX #define HAS_I444TOARGBROW_LSX #define HAS_INTERPOLATEROW_LSX @@ -818,8 +815,7 @@ extern "C" { #define HAS_ARGBATTENUATEROW_LASX #define HAS_ARGBGRAYROW_LASX #define HAS_ARGBMIRRORROW_LASX -// TODO: Re-enable once rounding behaviour is fixed. -// #define HAS_ARGBMULTIPLYROW_LASX +#define HAS_ARGBMULTIPLYROW_LASX #define HAS_ARGBSEPIAROW_LASX #define HAS_ARGBSHADEROW_LASX #define HAS_ARGBSHUFFLEROW_LASX @@ -879,6 +875,7 @@ extern "C" { #define HAS_AR64TOARGBROW_RVV #define HAS_ARGBCOPYYTOALPHAROW_RVV #define HAS_ARGBEXTRACTALPHAROW_RVV +#define HAS_ARGBMULTIPLYROW_RVV #define HAS_ARGBTOAR64ROW_RVV #define HAS_ARGBTOUV444MATRIXROW_RVV #define HAS_ARGBTOUVMATRIXROW_RVV @@ -5494,6 +5491,10 @@ void ARGBMultiplyRow_Any_LASX(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); +void ARGBMultiplyRow_RVV(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); // ARGB add images. void ARGBAddRow_C(const uint8_t* src_argb, diff --git a/source/planar_functions.cc b/source/planar_functions.cc index a64bb6a99..61872ffa9 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -3370,6 +3370,11 @@ int ARGBMultiply(const uint8_t* src_argb0, } } #endif +#if defined(HAS_ARGBMULTIPLYROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBMultiplyRow = ARGBMultiplyRow_RVV; + } +#endif // Multiply plane for (y = 0; y < height; ++y) { diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 90d6786df..23b317aea 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -8898,6 +8898,9 @@ void ARGBMultiplyRow_SSE2(const uint8_t* src_argb, uint8_t* dst_argb, int width) { asm volatile("pxor %%xmm5,%%xmm5 \n" + "pcmpeqw %%xmm4,%%xmm4 \n" + "psrlw $0xf,%%xmm4 \n" + "psllw $0x7,%%xmm4 \n" // 4 pixel loop. LABELALIGN @@ -8906,14 +8909,18 @@ void ARGBMultiplyRow_SSE2(const uint8_t* src_argb, "lea 0x10(%0),%0 \n" "movdqu (%1),%%xmm2 \n" "lea 0x10(%1),%1 \n" - "movdqu %%xmm0,%%xmm1 \n" - "movdqu %%xmm2,%%xmm3 \n" - "punpcklbw %%xmm0,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm1 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm3 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "punpckhbw %%xmm5,%%xmm1 \n" "punpcklbw %%xmm5,%%xmm2 \n" "punpckhbw %%xmm5,%%xmm3 \n" - "pmulhuw %%xmm2,%%xmm0 \n" - "pmulhuw %%xmm3,%%xmm1 \n" + "pmullw %%xmm2,%%xmm0 \n" + "pmullw %%xmm3,%%xmm1 \n" + "paddw %%xmm4,%%xmm0 \n" + "paddw %%xmm4,%%xmm1 \n" + "psrlw $0x8,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" "packuswb %%xmm1,%%xmm0 \n" "movdqu %%xmm0,(%2) \n" "lea 0x10(%2),%2 \n" @@ -8924,7 +8931,8 @@ void ARGBMultiplyRow_SSE2(const uint8_t* src_argb, "+r"(dst_argb), // %2 "+r"(width) // %3 : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", + "xmm5"); } #endif // HAS_ARGBMULTIPLYROW_SSE2 @@ -8936,19 +8944,21 @@ void ARGBMultiplyRow_AVX2(const uint8_t* src_argb, int width) { asm volatile("vpxor %%ymm5,%%ymm5,%%ymm5 \n" - // 4 pixel loop. + // 8 pixel loop. LABELALIGN "1: \n" "vmovdqu (%0),%%ymm1 \n" "lea 0x20(%0),%0 \n" "vmovdqu (%1),%%ymm3 \n" "lea 0x20(%1),%1 \n" - "vpunpcklbw %%ymm1,%%ymm1,%%ymm0 \n" - "vpunpckhbw %%ymm1,%%ymm1,%%ymm1 \n" + "vpunpcklbw %%ymm1,%%ymm5,%%ymm0 \n" + "vpunpckhbw %%ymm1,%%ymm5,%%ymm1 \n" + "vpsrlw $0x1,%%ymm0,%%ymm0 \n" + "vpsrlw $0x1,%%ymm1,%%ymm1 \n" "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n" "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n" - "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" - "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n" + "vpmulhrsw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmulhrsw %%ymm3,%%ymm1,%%ymm1 \n" "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" "vmovdqu %%ymm0,(%2) \n" "lea 0x20(%2),%2 \n" diff --git a/source/row_lasx.cc b/source/row_lasx.cc index 7a09e1406..b3865918a 100644 --- a/source/row_lasx.cc +++ b/source/row_lasx.cc @@ -964,19 +964,14 @@ void ARGBMultiplyRow_LASX(const uint8_t* src_argb0, int width) { int x; int len = width / 8; - __m256i zero = __lasx_xvldi(0); + __m256i half = __lasx_xvldi(0x480); __m256i src0, src1, dst0, dst1; - __m256i tmp0, tmp1, tmp2, tmp3; for (x = 0; x < len; x++) { DUP2_ARG2(__lasx_xvld, src_argb0, 0, src_argb1, 0, src0, src1); - tmp0 = __lasx_xvilvl_b(src0, src0); - tmp1 = __lasx_xvilvh_b(src0, src0); - tmp2 = __lasx_xvilvl_b(zero, src1); - tmp3 = __lasx_xvilvh_b(zero, src1); - dst0 = __lasx_xvmuh_hu(tmp0, tmp2); - dst1 = __lasx_xvmuh_hu(tmp1, tmp3); - dst0 = __lasx_xvpickev_b(dst1, dst0); + dst0 = __lasx_xvmaddwev_h_bu(half, src0, src1); + dst1 = __lasx_xvmaddwod_h_bu(half, src0, src1); + dst0 = __lasx_xvpackod_b(dst1, dst0); __lasx_xvst(dst0, dst_argb, 0); src_argb0 += 32; src_argb1 += 32; diff --git a/source/row_lsx.cc b/source/row_lsx.cc index c4547e5c9..d1549967d 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -930,19 +930,14 @@ void ARGBMultiplyRow_LSX(const uint8_t* src_argb0, int width) { int x; int len = width / 4; - __m128i zero = __lsx_vldi(0); + __m128i half = __lsx_vldi(0x480); __m128i src0, src1, dst0, dst1; - __m128i tmp0, tmp1, tmp2, tmp3; for (x = 0; x < len; x++) { DUP2_ARG2(__lsx_vld, src_argb0, 0, src_argb1, 0, src0, src1); - tmp0 = __lsx_vilvl_b(src0, src0); - tmp1 = __lsx_vilvh_b(src0, src0); - tmp2 = __lsx_vilvl_b(zero, src1); - tmp3 = __lsx_vilvh_b(zero, src1); - dst0 = __lsx_vmuh_hu(tmp0, tmp2); - dst1 = __lsx_vmuh_hu(tmp1, tmp3); - dst0 = __lsx_vpickev_b(dst1, dst0); + dst0 = __lsx_vmaddwev_h_bu(half, src0, src1); + dst1 = __lsx_vmaddwod_h_bu(half, src0, src1); + dst0 = __lsx_vpackod_b(dst1, dst0); __lsx_vst(dst0, dst_argb, 0); src_argb0 += 16; src_argb1 += 16; diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 78fa46c35..d098690c8 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -1886,6 +1886,60 @@ void ARGBAttenuateRow_RVV(const uint8_t* src_argb, } #endif +#ifdef HAS_ARGBMULTIPLYROW_RVV +void ARGBMultiplyRow_RVV(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + assert(width != 0); + size_t w = (size_t)width * 4; +#ifdef RVV_ASM +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wvla" + size_t vl; + asm( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle8.v v8, (%[src0]) \n" + "vle8.v v10, (%[src1]) \n" + "vwmulu.vv v12, v8, v10 \n" + "vwaddu.wx v12, v12, %[k128] \n" + "vnsrl.wi v8, v12, 8 \n" + "vse8.v v8, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src0], %[src0], %[vl] \n" + "add %[src1], %[src1], %[vl] \n" + "add %[dst], %[dst], %[vl] \n" + "bgtz %[w], 1b \n" + : [src0] "+r"(src_argb), // %[src0] + [src1] "+r"(src_argb1), // %[src1] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(w), // %[w] + [vl] "=&r"(vl), // %[vl] + "=m"(*(uint8_t (*)[w])dst_argb) + : [k128] "r"(128), // %[k128] + "m"(*(const uint8_t (*)[w])src_argb), + "m"(*(const uint8_t (*)[w])src_argb1) + : "vl", "vtype", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"); +#pragma GCC diagnostic pop +#else + do { + size_t vl = __riscv_vsetvl_e8m2(w); + vuint8m2_t v_src0 = __riscv_vle8_v_u8m2(src_argb, vl); + vuint8m2_t v_src1 = __riscv_vle8_v_u8m2(src_argb1, vl); + vuint16m4_t v_mul = __riscv_vwmulu_vv_u16m4(v_src0, v_src1, vl); + v_mul = __riscv_vwaddu_wx_u16m4(v_mul, 128, vl); + vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_mul, 8, vl); + __riscv_vse8_v_u8m2(dst_argb, v_dst, vl); + w -= vl; + src_argb += vl; + src_argb1 += vl; + dst_argb += vl; + } while (w > 0); +#endif +} +#endif + #ifdef HAS_ARGBEXTRACTALPHAROW_RVV void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb, uint8_t* dst_a,