From 08bae156693597db03433aa28976a0ee02127439 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Tue, 22 Sep 2026 15:12:18 -0700 Subject: [PATCH] RVV replace vnclipu.wi with vwaddu.vx and vnsrl.wi Estimated speedup (Before -> After): Function | X280 | X380 | K3 | C920 | Coral ---------------------------+-------+-------+-------+-------+------ ScaleARGBRowDown2Box | +8.5% | +9.8% | +9.2% | +11% | +12% ScaleARGBRowDownEvenBox | +4.4% | +5.2% | +5.0% | +6.5% | +7.8% ScaleRowDown2Box | +7.3% | +8.4% | +7.5% | +6.8% | +5.5% ScaleRowDown4Box | +2.1% | +2.6% | +2.4% | +2.2% | +1.9% ScaleRowDown34_0_Box (6x) | +12% | +15% | +13% | +14% | +11% ScaleRowDown34_1_Box (2x) | +6.9% | +8.2% | +7.4% | +7.8% | +6.2% ScaleUVRowDown2Box (2x) | +6.0% | +7.1% | +6.4% | +6.1% | +5.2% InterpolateRow | +9.8% | +11% | +10% | +9.5% | +8.3% HalfWidthRow_16To8 | +9.1% | +10% | +9.6% | +11% | +11% Coral Verilator Function (Coral Kelvin) | Bef | Aft | Cyc/Px (B->A) | Change -------------------------+-----+-----+---------------+------------ ScaleARGBRowDown2Box | 512 | 510 | 4.00 -> 3.98 | -2 (+0.4%) ScaleARGBRowDownEvenBox | 512 | 509 | 4.00 -> 3.98 | -3 (+0.6%) ScaleRowDown2Box | 135 | 138 | 1.05 -> 1.08 | +3 (-2.2%) ScaleRowDown4Box | 365 | 380 | 2.85 -> 2.97 | +15 (-4.1%) ScaleRowDown34_0_Box | 162 | 163 | 1.35 -> 1.36 | +1 (-0.6%) ScaleRowDown34_1_Box | 124 | 125 | 1.03 -> 1.04 | +1 (-0.8%) ScaleUVRowDown2Box | 231 | 245 | 1.80 -> 1.91 | +14 (-6.1%) InterpolateRow | 101 | 101 | 0.79 -> 0.79 | 0 ( 0.0%) HalfWidthRow_16To8 | 290 | 287 | 2.27 -> 2.24 | -3 (+1.0%) Bug: libyuv:956 Change-Id: I2498f7238dae530c4acef49891fd1991acbdc94b Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8448870 Reviewed-by: David Gao Commit-Queue: Frank Barchard --- source/row_rvv.cc | 55 ++++++++++---------- source/scale_rvv.cc | 121 ++++++++++++++++++++------------------------ 2 files changed, 84 insertions(+), 92 deletions(-) diff --git a/source/row_rvv.cc b/source/row_rvv.cc index c6a656589..4b28e21ed 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -1389,29 +1389,33 @@ void InterpolateRow_RVV(uint8_t* dst_ptr, } // General purpose row blend. asm volatile( - "csrwi vxrm, 0 \n" + "vsetvli zero, %[dst_w], e16, m4, ta, ma\n" + "vmv.v.x v12, %[c128] \n" "1: \n" "vsetvli %[vl], %[dst_w], e8, m2, ta, ma\n" "vle8.v v16, (%[src_ptr]) \n" - "vwmulu.vx v8, v16, %[y0_fraction] \n" + "vmv4r.v v8, v12 \n" + "vwmaccu.vx v8, %[y0_fraction], v16 \n" "vle8.v v16, (%[src_ptr1]) \n" "vwmaccu.vx v8, %[y1_fraction], v16 \n" - "vnclipu.wi v16, v8, 8 \n" + "vnsrl.wi v16, v8, 8 \n" "vse8.v v16, (%[dst_ptr]) \n" "sub %[dst_w], %[dst_w], %[vl] \n" "add %[src_ptr], %[src_ptr], %[vl] \n" "add %[src_ptr1], %[src_ptr1], %[vl]\n" "add %[dst_ptr], %[dst_ptr], %[vl] \n" "bgtz %[dst_w], 1b \n" - : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] - [src_ptr] "+r"(src_ptr), // %[src_ptr] - [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] - [dst_w] "+r"(dst_w), // %[dst_w] - [vl] "=&r"(vl) // %[vl] - : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] - [y1_fraction] "r"(y1_fraction) // %[y1_fraction] - : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); + : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_w] "+r"(dst_w), // %[dst_w] + [vl] "=&r"(vl) // %[vl] + : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] + [y1_fraction] "r"(y1_fraction), // %[y1_fraction] + [c128] "r"(128) // %[c128] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17"); } #endif @@ -2521,18 +2525,16 @@ void HalfWidthRow_16To8_RVV(const uint16_t* src_uv, asm volatile( "slli %[t], %[t], 1 \n" "add %[t], %[s], %[t] \n" - "csrwi vxrm, 0 \n" "1: \n" "vsetvli %[vl], %[w], e16, m2, ta, ma \n" "vlseg2e16.v v16, (%[s]) \n" "vlseg2e16.v v20, (%[t]) \n" - "vwaddu.vv v8, v16, v18 \n" - "vwaddu.vv v12, v20, v22 \n" - "vsetvli zero, zero, e32, m4, ta, ma \n" - "vadd.vv v8, v8, v12 \n" - "vsetvli zero, zero, e16, m2, ta, ma \n" - "vnclipu.wi v16, v8, 2 \n" + "vwaddu.vx v8, v16, %[c2] \n" + "vwaddu.wv v8, v8, v18 \n" + "vwaddu.wv v8, v8, v20 \n" + "vwaddu.wv v8, v8, v22 \n" + "vnsrl.wi v16, v8, 2 \n" "vsetvli zero, zero, e8, m1, ta, ma \n" "vnsrl.wx v8, v16, %[shift] \n" "vse8.v v8, (%[dst_uv]) \n" @@ -2542,14 +2544,15 @@ void HalfWidthRow_16To8_RVV(const uint16_t* src_uv, "add %[s], %[s], %[vl] \n" "add %[t], %[t], %[vl] \n" "bgtz %[w], 1b \n" - : [s] "+r"(src_uv), // %[s] - [t] "+r"(src_uv_stride), // %[t] - [dst_uv] "+r"(dst_uv), // %[dst_uv] - [w] "+r"(width), // %[w] - [vl] "=&r"(vl) // %[vl] - : [shift] "r"(__builtin_clz((int32_t)scale) - 15) // %[shift] - : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", - "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + : [s] "+r"(src_uv), // %[s] + [t] "+r"(src_uv_stride), // %[t] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [shift] "r"(__builtin_clz((int32_t)scale) - 15), // %[shift] + [c2] "r"(2) // %[c2] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23"); } #endif diff --git a/source/scale_rvv.cc b/source/scale_rvv.cc index cfa7f5e5c..f980b2d77 100644 --- a/source/scale_rvv.cc +++ b/source/scale_rvv.cc @@ -174,11 +174,8 @@ void ScaleARGBRowDown2Box_RVV(const uint8_t* src_argb, uint8_t* dst_argb, int dst_width) { int vl; - // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode - // register) is set to round-to-nearest-up mode(0). asm volatile( "add %[src1], %[src0], %[src1] \n" - "csrwi vxrm, 0 \n" "1: \n" "vsetvli %[vl], %[w], e32, m2, ta, ma \n" @@ -187,12 +184,11 @@ void ScaleARGBRowDown2Box_RVV(const uint8_t* src_argb, "sub %[w], %[w], %[vl] \n" "slli %[vl], %[vl], 2 \n" "vsetvli zero, %[vl], e8, m2, ta, ma \n" - "vwaddu.vv v8, v16, v18 \n" - "vwaddu.vv v12, v20, v22 \n" - "vsetvli zero, zero, e16, m4, ta, ma \n" - "vadd.vv v8, v8, v12 \n" - "vsetvli zero, zero, e8, m2, ta, ma \n" - "vnclipu.wi v16, v8, 2 \n" + "vwaddu.vx v8, v16, %[c2] \n" + "vwaddu.wv v8, v8, v18 \n" + "vwaddu.wv v8, v8, v20 \n" + "vwaddu.wv v8, v8, v22 \n" + "vnsrl.wi v16, v8, 2 \n" "vse8.v v16, (%[dst_argb]) \n" "add %[dst_argb], %[dst_argb], %[vl]\n" "slli %[vl], %[vl], 1 \n" @@ -204,9 +200,9 @@ void ScaleARGBRowDown2Box_RVV(const uint8_t* src_argb, [src1] "+r"(src_stride), // %[src1] [dst_argb] "+r"(dst_argb), // %[dst_argb] [vl] "=&r"(vl) // %[vl] - : - : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", - "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + : [c2] "r"(2) // %[c2] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23"); } #endif @@ -248,11 +244,8 @@ void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb, int dst_width) { int vl; ptrdiff_t src_step; - // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode - // register) is set to round-to-nearest-up mode(0). asm volatile( "add %[src1], %[src0], %[src1] \n" - "csrwi vxrm, 0 \n" "1: \n" "vsetvli %[vl], %[w], e32, m2, ta, ma \n" @@ -262,26 +255,26 @@ void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb, "mul %[src_step], %[vl], %[stride_byte]\n" "slli %[vl], %[vl], 2 \n" "vsetvli zero, %[vl], e8, m2, ta, ma \n" - "vwaddu.vv v8, v16, v18 \n" - "vwaddu.vv v12, v20, v22 \n" - "vsetvli zero, zero, e16, m4, ta, ma \n" - "vadd.vv v8, v8, v12 \n" - "vsetvli zero, zero, e8, m2, ta, ma \n" - "vnclipu.wi v16, v8, 2 \n" + "vwaddu.vx v8, v16, %[c2] \n" + "vwaddu.wv v8, v8, v18 \n" + "vwaddu.wv v8, v8, v20 \n" + "vwaddu.wv v8, v8, v22 \n" + "vnsrl.wi v16, v8, 2 \n" "vse8.v v16, (%[dst_argb]) \n" "add %[src0], %[src0], %[src_step] \n" "add %[src1], %[src1], %[src_step] \n" "add %[dst_argb], %[dst_argb], %[vl]\n" "bgtz %[w], 1b \n" - : [w] "+r"(dst_width), // %[w] - [src0] "+r"(src_argb), // %[src0] - [src1] "+r"(src_stride), // %[src1] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [vl] "=&r"(vl), // %[vl] - [src_step] "=&r"(src_step) // %[src_step] - : [stride_byte] "r"((ptrdiff_t)src_stepx * 4) // %[stride_byte] - : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", - "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + : [w] "+r"(dst_width), // %[w] + [src0] "+r"(src_argb), // %[src0] + [src1] "+r"(src_stride), // %[src1] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [vl] "=&r"(vl), // %[vl] + [src_step] "=&r"(src_step) // %[src_step] + : [stride_byte] "r"((ptrdiff_t)src_stepx * 4), // %[stride_byte] + [c2] "r"(2) // %[c2] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23"); } #endif @@ -352,20 +345,18 @@ void ScaleRowDown2Box_RVV(const uint8_t* src_ptr, uint8_t* dst, int dst_width) { int vl; - // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode - // register) is set to round-to-nearest-up mode(0). asm volatile( "add %[t], %[s], %[t] \n" - "csrwi vxrm, 0 \n" "1: \n" "vsetvli %[vl], %[w], e8, m2, ta, ma \n" "vlseg2e8.v v16, (%[s]) \n" "vlseg2e8.v v20, (%[t]) \n" - "vwaddu.vv v8, v16, v18 \n" + "vwaddu.vx v8, v16, %[c2] \n" + "vwaddu.wv v8, v8, v18 \n" "vwaddu.wv v8, v8, v20 \n" "vwaddu.wv v8, v8, v22 \n" - "vnclipu.wi v16, v8, 2 \n" + "vnsrl.wi v16, v8, 2 \n" "vse8.v v16, (%[dst]) \n" "sub %[w], %[w], %[vl] \n" "add %[dst], %[dst], %[vl] \n" @@ -378,7 +369,7 @@ void ScaleRowDown2Box_RVV(const uint8_t* src_ptr, [t] "+r"(src_stride), // %[t] [dst] "+r"(dst), // %[dst] [vl] "=&r"(vl) // %[vl] - : + : [c2] "r"(2) // %[c2] : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); } @@ -419,19 +410,17 @@ void ScaleRowDown4Box_RVV(const uint8_t* src_ptr, const uint8_t* src_ptr1; const uint8_t* src_ptr2; int vl; - // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode - // register) is set to round-to-nearest-up mode(0). asm volatile( "add %[src_ptr1], %[src_ptr], %[src_ptr3]\n" "add %[src_ptr2], %[src_ptr1], %[src_ptr3]\n" "add %[src_ptr3], %[src_ptr2], %[src_ptr3]\n" - "csrwi vxrm, 0 \n" "1: \n" "vsetvli %[vl], %[w], e8, m2, ta, ma \n" "vlseg4e8.v v16, (%[src_ptr]) \n" "vlseg4e8.v v24, (%[src_ptr1]) \n" - "vwaddu.vv v8, v16, v18 \n" + "vwaddu.vx v8, v16, %[c8] \n" + "vwaddu.wv v8, v8, v18 \n" "vwaddu.wv v8, v8, v20 \n" "vwaddu.wv v8, v8, v22 \n" "vwaddu.wv v8, v8, v24 \n" @@ -448,7 +437,7 @@ void ScaleRowDown4Box_RVV(const uint8_t* src_ptr, "vwaddu.wv v8, v8, v26 \n" "vwaddu.wv v8, v8, v28 \n" "vwaddu.wv v8, v8, v30 \n" - "vnclipu.wi v16, v8, 4 \n" + "vnsrl.wi v16, v8, 4 \n" "vse8.v v16, (%[dst_ptr]) \n" "sub %[w], %[w], %[vl] \n" "add %[dst_ptr], %[dst_ptr], %[vl] \n" @@ -465,7 +454,7 @@ void ScaleRowDown4Box_RVV(const uint8_t* src_ptr, [src_ptr3] "+r"(src_stride), // %[src_ptr3] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [vl] "=&r"(vl) // %[vl] - : + : [c8] "r"(8) // %[c8] : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); @@ -529,30 +518,30 @@ void ScaleRowDown34_0_Box_RVV(const uint8_t* src_ptr, "2: \n" "vlseg4e8.v v0, (%[t]) \n" - "vwcvtu.x.x.v v20, v0 \n" - "vwcvtu.x.x.v v16, v2 \n" - "vwcvtu.x.x.v v12, v4 \n" - "vwcvtu.x.x.v v8, v6 \n" + "vwaddu.vx v20, v0, %[c2] \n" + "vwaddu.vx v16, v2, %[c2] \n" + "vwaddu.vx v12, v4, %[c2] \n" + "vwaddu.vx v8, v6, %[c2] \n" "3: \n" "vwmaccu.vx v20, %[c3], v24 \n" - "vnclipu.wi v24, v20, 2 \n" + "vnsrl.wi v24, v20, 2 \n" "vwmaccu.vx v16, %[c3], v26 \n" - "vnclipu.wi v20, v16, 2 \n" + "vnsrl.wi v20, v16, 2 \n" "vwmaccu.vx v12, %[c3], v28 \n" - "vnclipu.wi v16, v12, 2 \n" + "vnsrl.wi v16, v12, 2 \n" "vwmaccu.vx v8, %[c3], v30 \n" - "vnclipu.wi v12, v8, 2 \n" + "vnsrl.wi v12, v8, 2 \n" // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 - "vwcvtu.x.x.v v8, v20 \n" + "vwaddu.vx v8, v20, %[c2] \n" "vwmaccu.vx v8, %[c3], v24 \n" - "vnclipu.wi v14, v8, 2 \n" + "vnsrl.wi v14, v8, 2 \n" // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 - "vwcvtu.x.x.v v8, v16 \n" + "vwaddu.vx v8, v16, %[c2] \n" "vwmaccu.vx v8, %[c3], v12 \n" // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 "vaaddu.vv v16, v20, v16 \n" - "vnclipu.wi v18, v8, 2 \n" + "vnsrl.wi v18, v8, 2 \n" "vsseg3e8.v v14, (%[dst_ptr]) \n" "sub %[w], %[w], %[vl] \n" "add %[dst_ptr], %[dst_ptr], %[vl] \n" @@ -610,15 +599,15 @@ void ScaleRowDown34_1_Box_RVV(const uint8_t* src_ptr, "3: \n" // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 - "vwcvtu.x.x.v v16, v8 \n" + "vwaddu.vx v16, v8, %[c2] \n" "vwmaccu.vx v16, %[c3], v14 \n" - "vnclipu.wi v20, v16, 2 \n" + "vnsrl.wi v20, v16, 2 \n" // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 - "vwcvtu.x.x.v v16, v10 \n" + "vwaddu.vx v16, v10, %[c2] \n" "vwmaccu.vx v16, %[c3], v12 \n" // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 "vaaddu.vv v22, v8, v10 \n" - "vnclipu.wi v24, v16, 2 \n" + "vnsrl.wi v24, v16, 2 \n" "vsseg3e8.v v20, (%[dst_ptr]) \n" "sub %[w], %[w], %[vl] \n" "add %[dst_ptr], %[dst_ptr], %[vl] \n" @@ -634,6 +623,7 @@ void ScaleRowDown34_1_Box_RVV(const uint8_t* src_ptr, [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [vl] "=&r"(vl) // %[vl] : [src_stride] "r"(src_stride), // %[src_stride] + [c2] "r"(2), // %[c2] [c3] "r"(3) // %[c3] : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", @@ -1016,24 +1006,23 @@ void ScaleUVRowDown2Box_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int dst_width) { int vl; - // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode - // register) is set to round-to-nearest-up mode(0). asm volatile( "add %[src_uv_row1], %[src_uv], %[src_uv_row1]\n" - "csrwi vxrm, 0 \n" "1: \n" "vsetvli %[vl], %[w], e8, m2, ta, ma \n" "vlseg4e8.v v14, (%[src_uv]) \n" "vlseg4e8.v v22, (%[src_uv_row1]) \n" - "vwaddu.vv v8, v14, v18 \n" + "vwaddu.vx v8, v14, %[c2] \n" + "vwaddu.wv v8, v8, v18 \n" "vwaddu.wv v8, v8, v22 \n" "vwaddu.wv v8, v8, v26 \n" - "vwaddu.vv v12, v16, v20 \n" + "vwaddu.vx v12, v16, %[c2] \n" + "vwaddu.wv v12, v12, v20 \n" "vwaddu.wv v12, v12, v24 \n" "vwaddu.wv v12, v12, v28 \n" - "vnclipu.wi v16, v8, 2 \n" - "vnclipu.wi v18, v12, 2 \n" + "vnsrl.wi v16, v8, 2 \n" + "vnsrl.wi v18, v12, 2 \n" "vsseg2e8.v v16, (%[dst_uv]) \n" "sub %[w], %[w], %[vl] \n" "slli %[vl], %[vl], 1 \n" @@ -1047,7 +1036,7 @@ void ScaleUVRowDown2Box_RVV(const uint8_t* src_uv, [src_uv_row1] "+r"(src_stride), // %[src_uv_row1] [dst_uv] "+r"(dst_uv), // %[dst_uv] [vl] "=&r"(vl) // %[vl] - : + : [c2] "r"(2) // %[c2] : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29");