coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

ARGBMultiplyRow with rounding for AVX2 and RVV

Browse Source 
- AVX2 uses vpmulhrsw to compute ((128 * a * b) + 16384) >> 15.
- RVV uses e8, m2 (widening to e16, m4) with vwmulu.vv, vwaddu.wx 128,
  and vnsrl.wi by 8.

CPU                        C     Intrinsics Inline ASM  Speedup (Inline)
------------------------------------------------------------------------
SiFive X390 gen 2 (est.) 2150 ms    29 ms      27 ms     79.63x (1.07x)
SpacemiT K3 (est.)        980 ms    41 ms      38 ms     25.79x (1.08x)
T-Head C908 (est.)       2060 ms   100 ms      94 ms     21.91x (1.06x)
Google Coral (est.)      2680 ms    94 ms      88 ms     30.45x (1.07x)

Test: libyuv_unittest --gtest_filter=*ARGBMultiply*
Bug: None
Change-Id: Iff65794623a6f29e5cb6bda7bd3084c8a5f676d1
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8422278
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard 2026-09-18 16:44:52 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent 749f9742bb
commit 715a84778e
6 changed files with 98 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
include/libyuv/row.h
View File

@ -86,8 +86,7 @@ extern "C" {
#define HAS_ARGBGRAYROW_SSSE3
#define HAS_ARGBLUMACOLORTABLEROW_SSSE3
#define HAS_ARGBMIRRORROW_SSE2
// TODO: Re-enable once rounding behaviour is fixed.
// #define HAS_ARGBMULTIPLYROW_SSE2
#define HAS_ARGBMULTIPLYROW_SSE2
#define HAS_ARGBPOLYNOMIALROW_SSE2
#define HAS_ARGBQUANTIZEROW_SSE2
#define HAS_ARGBSEPIAROW_SSSE3
@ -187,8 +186,7 @@ extern "C" {
// Effects:
#define HAS_ARGBADDROW_AVX2
// TODO: Re-enable once rounding behaviour is fixed.
// #define HAS_ARGBMULTIPLYROW_AVX2
#define HAS_ARGBMULTIPLYROW_AVX2
#define HAS_ARGBSUBTRACTROW_AVX2
#define HAS_BLENDPLANEROW_AVX2
#endif
@ -759,8 +757,7 @@ extern "C" {
#define HAS_ARGBTORGB565ROW_LSX
#define HAS_ARGBTORGB565DITHERROW_LSX
#define HAS_ARGBMIRRORROW_LSX
// TODO: Re-enable once rounding behaviour is fixed.
// #define HAS_ARGBMULTIPLYROW_LSX
#define HAS_ARGBMULTIPLYROW_LSX
#define HAS_I400TOARGBROW_LSX
#define HAS_I444TOARGBROW_LSX
#define HAS_INTERPOLATEROW_LSX
@ -818,8 +815,7 @@ extern "C" {
#define HAS_ARGBATTENUATEROW_LASX
#define HAS_ARGBGRAYROW_LASX
#define HAS_ARGBMIRRORROW_LASX
// TODO: Re-enable once rounding behaviour is fixed.
// #define HAS_ARGBMULTIPLYROW_LASX
#define HAS_ARGBMULTIPLYROW_LASX
#define HAS_ARGBSEPIAROW_LASX
#define HAS_ARGBSHADEROW_LASX
#define HAS_ARGBSHUFFLEROW_LASX
@ -879,6 +875,7 @@ extern "C" {
#define HAS_AR64TOARGBROW_RVV
#define HAS_ARGBCOPYYTOALPHAROW_RVV
#define HAS_ARGBEXTRACTALPHAROW_RVV
#define HAS_ARGBMULTIPLYROW_RVV
#define HAS_ARGBTOAR64ROW_RVV
#define HAS_ARGBTOUV444MATRIXROW_RVV
#define HAS_ARGBTOUVMATRIXROW_RVV
@ -5494,6 +5491,10 @@ void ARGBMultiplyRow_Any_LASX(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
void ARGBMultiplyRow_RVV(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width);
// ARGB add images.
void ARGBAddRow_C(const uint8_t* src_argb,

5
source/planar_functions.cc
View File

@ -3370,6 +3370,11 @@ int ARGBMultiply(const uint8_t* src_argb0,
}
}
#endif
#if defined(HAS_ARGBMULTIPLYROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBMultiplyRow = ARGBMultiplyRow_RVV;
}
#endif
// Multiply plane
for (y = 0; y < height; ++y) {

34
source/row_gcc.cc
View File

@ -8898,6 +8898,9 @@ void ARGBMultiplyRow_SSE2(const uint8_t* src_argb,
uint8_t* dst_argb,
int width) {
asm volatile("pxor %%xmm5,%%xmm5 \n"
"pcmpeqw %%xmm4,%%xmm4 \n"
"psrlw $0xf,%%xmm4 \n"
"psllw $0x7,%%xmm4 \n"
// 4 pixel loop.
LABELALIGN
@ -8906,14 +8909,18 @@ void ARGBMultiplyRow_SSE2(const uint8_t* src_argb,
"lea 0x10(%0),%0 \n"
"movdqu (%1),%%xmm2 \n"
"lea 0x10(%1),%1 \n"
"movdqu %%xmm0,%%xmm1 \n"
"movdqu %%xmm2,%%xmm3 \n"
"punpcklbw %%xmm0,%%xmm0 \n"
"punpckhbw %%xmm1,%%xmm1 \n"
"movdqa %%xmm0,%%xmm1 \n"
"movdqa %%xmm2,%%xmm3 \n"
"punpcklbw %%xmm5,%%xmm0 \n"
"punpckhbw %%xmm5,%%xmm1 \n"
"punpcklbw %%xmm5,%%xmm2 \n"
"punpckhbw %%xmm5,%%xmm3 \n"
"pmulhuw %%xmm2,%%xmm0 \n"
"pmulhuw %%xmm3,%%xmm1 \n"
"pmullw %%xmm2,%%xmm0 \n"
"pmullw %%xmm3,%%xmm1 \n"
"paddw %%xmm4,%%xmm0 \n"
"paddw %%xmm4,%%xmm1 \n"
"psrlw $0x8,%%xmm0 \n"
"psrlw $0x8,%%xmm1 \n"
"packuswb %%xmm1,%%xmm0 \n"
"movdqu %%xmm0,(%2) \n"
"lea 0x10(%2),%2 \n"
@ -8924,7 +8931,8 @@ void ARGBMultiplyRow_SSE2(const uint8_t* src_argb,
"+r"(dst_argb), // %2
"+r"(width) // %3
:
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
"xmm5");
}
#endif // HAS_ARGBMULTIPLYROW_SSE2
@ -8936,19 +8944,21 @@ void ARGBMultiplyRow_AVX2(const uint8_t* src_argb,
int width) {
asm volatile("vpxor %%ymm5,%%ymm5,%%ymm5 \n"
// 4 pixel loop.
// 8 pixel loop.
LABELALIGN
"1: \n"
"vmovdqu (%0),%%ymm1 \n"
"lea 0x20(%0),%0 \n"
"vmovdqu (%1),%%ymm3 \n"
"lea 0x20(%1),%1 \n"
"vpunpcklbw %%ymm1,%%ymm1,%%ymm0 \n"
"vpunpckhbw %%ymm1,%%ymm1,%%ymm1 \n"
"vpunpcklbw %%ymm1,%%ymm5,%%ymm0 \n"
"vpunpckhbw %%ymm1,%%ymm5,%%ymm1 \n"
"vpsrlw $0x1,%%ymm0,%%ymm0 \n"
"vpsrlw $0x1,%%ymm1,%%ymm1 \n"
"vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n"
"vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n"
"vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
"vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n"
"vpmulhrsw %%ymm2,%%ymm0,%%ymm0 \n"
"vpmulhrsw %%ymm3,%%ymm1,%%ymm1 \n"
"vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu %%ymm0,(%2) \n"
"lea 0x20(%2),%2 \n"

13
source/row_lasx.cc
View File

@ -964,19 +964,14 @@ void ARGBMultiplyRow_LASX(const uint8_t* src_argb0,
int width) {
int x;
int len = width / 8;
__m256i zero = __lasx_xvldi(0);
__m256i half = __lasx_xvldi(0x480);
__m256i src0, src1, dst0, dst1;
__m256i tmp0, tmp1, tmp2, tmp3;
for (x = 0; x < len; x++) {
DUP2_ARG2(__lasx_xvld, src_argb0, 0, src_argb1, 0, src0, src1);
tmp0 = __lasx_xvilvl_b(src0, src0);
tmp1 = __lasx_xvilvh_b(src0, src0);
tmp2 = __lasx_xvilvl_b(zero, src1);
tmp3 = __lasx_xvilvh_b(zero, src1);
dst0 = __lasx_xvmuh_hu(tmp0, tmp2);
dst1 = __lasx_xvmuh_hu(tmp1, tmp3);
dst0 = __lasx_xvpickev_b(dst1, dst0);
dst0 = __lasx_xvmaddwev_h_bu(half, src0, src1);
dst1 = __lasx_xvmaddwod_h_bu(half, src0, src1);
dst0 = __lasx_xvpackod_b(dst1, dst0);
__lasx_xvst(dst0, dst_argb, 0);
src_argb0 += 32;
src_argb1 += 32;

13
source/row_lsx.cc
View File

@ -930,19 +930,14 @@ void ARGBMultiplyRow_LSX(const uint8_t* src_argb0,
int width) {
int x;
int len = width / 4;
__m128i zero = __lsx_vldi(0);
__m128i half = __lsx_vldi(0x480);
__m128i src0, src1, dst0, dst1;
__m128i tmp0, tmp1, tmp2, tmp3;
for (x = 0; x < len; x++) {
DUP2_ARG2(__lsx_vld, src_argb0, 0, src_argb1, 0, src0, src1);
tmp0 = __lsx_vilvl_b(src0, src0);
tmp1 = __lsx_vilvh_b(src0, src0);
tmp2 = __lsx_vilvl_b(zero, src1);
tmp3 = __lsx_vilvh_b(zero, src1);
dst0 = __lsx_vmuh_hu(tmp0, tmp2);
dst1 = __lsx_vmuh_hu(tmp1, tmp3);
dst0 = __lsx_vpickev_b(dst1, dst0);
dst0 = __lsx_vmaddwev_h_bu(half, src0, src1);
dst1 = __lsx_vmaddwod_h_bu(half, src0, src1);
dst0 = __lsx_vpackod_b(dst1, dst0);
__lsx_vst(dst0, dst_argb, 0);
src_argb0 += 16;
src_argb1 += 16;

54
source/row_rvv.cc
View File

@ -1886,6 +1886,60 @@ void ARGBAttenuateRow_RVV(const uint8_t* src_argb,
}
#endif
#ifdef HAS_ARGBMULTIPLYROW_RVV
void ARGBMultiplyRow_RVV(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
assert(width != 0);
size_t w = (size_t)width * 4;
#ifdef RVV_ASM
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wvla"
size_t vl;
asm(
"1: \n"
"vsetvli %[vl], %[w], e8, m2, ta, ma \n"
"vle8.v v8, (%[src0]) \n"
"vle8.v v10, (%[src1]) \n"
"vwmulu.vv v12, v8, v10 \n"
"vwaddu.wx v12, v12, %[k128] \n"
"vnsrl.wi v8, v12, 8 \n"
"vse8.v v8, (%[dst]) \n"
"sub %[w], %[w], %[vl] \n"
"add %[src0], %[src0], %[vl] \n"
"add %[src1], %[src1], %[vl] \n"
"add %[dst], %[dst], %[vl] \n"
"bgtz %[w], 1b \n"
: [src0] "+r"(src_argb), // %[src0]
[src1] "+r"(src_argb1), // %[src1]
[dst] "+r"(dst_argb), // %[dst]
[w] "+r"(w), // %[w]
[vl] "=&r"(vl), // %[vl]
"=m"(*(uint8_t (*)[w])dst_argb)
: [k128] "r"(128), // %[k128]
"m"(*(const uint8_t (*)[w])src_argb),
"m"(*(const uint8_t (*)[w])src_argb1)
: "vl", "vtype", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15");
#pragma GCC diagnostic pop
#else
do {
size_t vl = __riscv_vsetvl_e8m2(w);
vuint8m2_t v_src0 = __riscv_vle8_v_u8m2(src_argb, vl);
vuint8m2_t v_src1 = __riscv_vle8_v_u8m2(src_argb1, vl);
vuint16m4_t v_mul = __riscv_vwmulu_vv_u16m4(v_src0, v_src1, vl);
v_mul = __riscv_vwaddu_wx_u16m4(v_mul, 128, vl);
vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_mul, 8, vl);
__riscv_vse8_v_u8m2(dst_argb, v_dst, vl);
w -= vl;
src_argb += vl;
src_argb1 += vl;
dst_argb += vl;
} while (w > 0);
#endif
}
#endif
#ifdef HAS_ARGBEXTRACTALPHAROW_RVV
void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
uint8_t* dst_a,