diff --git a/README.chromium b/README.chromium index 056328302..35479dae2 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1971 +Version: 1972 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 170d0b7b8..805b76c47 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -627,14 +627,17 @@ extern "C" { #define HAS_ARGBTOYMATRIXROW_SVE2 #define HAS_AYUVTOUVROW_SVE2 #define HAS_AYUVTOVUROW_SVE2 -// #define HAS_CONVERT16TO8ROW_SVE2 // Disabled: NEON version is faster for 128 bit vectors. +// Disabled: NEON version is faster for 128 bit vectors. +// #define HAS_CONVERT16TO8ROW_SVE2 #define HAS_CONVERT8TO8ROW_SVE2 #define HAS_COPYROW_SVE2 #define HAS_DIVIDEROW_16_SVE2 #define HAS_HALFFLOATROW_SVE2 #define HAS_HALFMERGEUVROW_SVE2 -// #define HAS_HALFROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors. -// #define HAS_HALFWIDTHROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors. +// Disabled: NEON version is faster for 128 bit vectors. +// #define HAS_HALFROW_16TO8_SVE2 +// Disabled: NEON version is faster for 128 bit vectors. +// #define HAS_HALFWIDTHROW_16TO8_SVE2 #define HAS_I210ALPHATOARGBROW_SVE2 #define HAS_I210TOAR30ROW_SVE2 #define HAS_I210TOARGBROW_SVE2 @@ -857,20 +860,6 @@ extern "C" { #define HAS_YUY2TOYROW_LASX #endif -#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 100000 -// Since v1.0, vcreate intrinsic is introduced -#define LIBYUV_RVV_HAS_VCREATE -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 12000 -// Since v0.12, TUPLE_TYPE is introduced for segment load and store. -#define LIBYUV_RVV_HAS_TUPLE_TYPE -// Since v0.12, VXRM(fixed-point rounding mode) is included in arguments of -// fixed-point intrinsics. -#define LIBYUV_RVV_HAS_VXRM_ARG -#endif -#endif - #if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) #define HAS_AR64TOARGBROW_RVV #define HAS_ARGBCOPYYTOALPHAROW_RVV @@ -896,20 +885,6 @@ extern "C" { #define HAS_SPLITUVROW_RVV #define HAS_SPLITXRGBROW_RVV #define HAS_SWAPUVROW_RVV - -// The following are available for RVV v0.11 and RVV v1.0 -// TODO(fbarchard): Port to RVV v0.12 (tuple) -// missing support for vcreate_v: -// __riscv_vcreate_v_u16m2x2 -// __riscv_vcreate_v_u16m2x4 -// __riscv_vcreate_v_u16m4x2 -// __riscv_vcreate_v_u8m1x3 -// __riscv_vcreate_v_u8m1x4 -// __riscv_vcreate_v_u8m2x2 -// __riscv_vcreate_v_u8m2x3 -// __riscv_vcreate_v_u8m2x4 -// __riscv_vcreate_v_u8m4x2 -#if defined(LIBYUV_RVV_HAS_VCREATE) #define HAS_AB64TOARGBROW_RVV #define HAS_AR64TOAB64ROW_RVV #define HAS_ARGBATTENUATEROW_RVV @@ -944,7 +919,6 @@ extern "C" { #define HAS_RGB24TOARGBROW_RVV #define HAS_RGBATOARGBROW_RVV #endif -#endif #if defined(_MSC_VER) && !defined(__CLR_VER) && !defined(__clang__) #if defined(VISUALC_HAS_AVX2) diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 24dd9ccd5..f18cca24d 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -151,19 +151,9 @@ extern "C" { #define HAS_SCALEROWDOWN34_LSX #endif -// The following are available on RVV with 64 bit elements -// TODO: Update compiler to support 64 bit -#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) && \ - defined(__riscv_zve64x) +#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) #define HAS_SCALEUVROWDOWN4_RVV #define HAS_SCALEARGBROWDOWN2_RVV -#endif - -#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) && \ - defined(__riscv_v_intrinsic) -// The following are available on RVV v0.11 and RVV v1.0 -// TODO: Port to RVV v0.12 -#if __riscv_v_intrinsic == 11000 || __riscv_v_intrinsic >= 100000 #define HAS_SCALEROWDOWN34_0_BOX_RVV #define HAS_SCALEROWDOWN34_1_BOX_RVV #define HAS_SCALEROWDOWN38_2_BOX_RVV @@ -190,13 +180,8 @@ extern "C" { #define HAS_SCALEUVROWDOWN2_RVV #define HAS_SCALEUVROWDOWN2BOX_RVV #define HAS_SCALEUVROWDOWN2LINEAR_RVV -#endif - -// The following are available on RVV v0.11 -#if __riscv_v_intrinsic == 11000 #define HAS_SCALEARGBFILTERCOLS_RVV #endif -#endif // Scale ARGB vertically with bilinear interpolation. void ScalePlaneVertical(int src_height, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index 1aad36e2d..f369637ea 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1971 +#define LIBYUV_VERSION 1972 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/cpu_id.cc b/source/cpu_id.cc index bebe4796c..0b6f047bb 100644 --- a/source/cpu_id.cc +++ b/source/cpu_id.cc @@ -304,7 +304,7 @@ LIBYUV_API SAFEBUFFERS int RiscvCpuCaps(const char* cpuinfo_name) { int flag = 0; FILE* f = fopen(cpuinfo_name, "re"); if (!f) { -#if defined(__riscv_vector) +#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv) && __riscv_xlen == 64 // Assume RVV if /proc/cpuinfo is unavailable. // This will occur for Chrome sandbox for Pepper or Render process. return kCpuHasRVV; @@ -364,7 +364,7 @@ LIBYUV_API SAFEBUFFERS int RiscvCpuCaps(const char* cpuinfo_name) { } } } -#if defined(__riscv_vector) +#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv) && __riscv_xlen == 64 // Assume RVV if /proc/cpuinfo is from x86 host running QEMU. else if ((memcmp(cpuinfo_line, "vendor_id\t: GenuineIntel", 24) == 0) || (memcmp(cpuinfo_line, "vendor_id\t: AuthenticAMD", 24) == 0)) { diff --git a/source/row_rvv.cc b/source/row_rvv.cc index d098690c8..c6a656589 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -20,242 +20,211 @@ // This module is for RVV (RISC-V Vector extension) #if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) #include -#include #ifdef __cplusplus namespace libyuv { extern "C" { #endif -#ifdef LIBYUV_RVV_HAS_VXRM_ARG -// Fill YUV -> RGB conversion constants into vectors -#define YUVTORGB_SETUP(yuvconst, ub, vr, ug, vg, yg, bb, bg, br) \ - { \ - ub = yuvconst->kUVCoeff[0]; \ - vr = yuvconst->kUVCoeff[1]; \ - ug = yuvconst->kUVCoeff[2]; \ - vg = yuvconst->kUVCoeff[3]; \ - yg = yuvconst->kRGBCoeffBias[0]; \ - bb = yuvconst->kRGBCoeffBias[1] + 32; \ - bg = yuvconst->kRGBCoeffBias[2] - 32; \ - br = yuvconst->kRGBCoeffBias[3] + 32; \ - } -// Fill YUV -> AR30 conversion constants into vectors -#define YUVTORGB_SETUP_AR30(yuvconst, ub, vr, ug, vg, yg, bb, bg, br) \ - { \ - ub = yuvconst->kUVCoeff[0]; \ - vr = yuvconst->kUVCoeff[1]; \ - ug = yuvconst->kUVCoeff[2]; \ - vg = yuvconst->kUVCoeff[3]; \ - yg = yuvconst->kRGBCoeffBias[0]; \ - bb = yuvconst->kRGBCoeffBias[1] + 24; \ - bg = yuvconst->kRGBCoeffBias[2] - 24; \ - br = yuvconst->kRGBCoeffBias[3] + 24; \ - } -#else -// Fill YUV -> RGB conversion constants into vectors +// Fill YUV -> RGB conversion constants into registers // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). -#define YUVTORGB_SETUP(yuvconst, ub, vr, ug, vg, yg, bb, bg, br) \ - { \ - asm volatile("csrwi vxrm, 0"); \ - ub = yuvconst->kUVCoeff[0]; \ - vr = yuvconst->kUVCoeff[1]; \ - ug = yuvconst->kUVCoeff[2]; \ - vg = yuvconst->kUVCoeff[3]; \ - yg = yuvconst->kRGBCoeffBias[0]; \ - bb = yuvconst->kRGBCoeffBias[1] + 32; \ - bg = yuvconst->kRGBCoeffBias[2] - 32; \ - br = yuvconst->kRGBCoeffBias[3] + 32; \ - } -// Fill YUV -> AR30 conversion constants into vectors -#define YUVTORGB_SETUP_AR30(yuvconst, ub, vr, ug, vg, yg, bb, bg, br) \ - { \ - asm volatile("csrwi vxrm, 0"); \ - ub = yuvconst->kUVCoeff[0]; \ - vr = yuvconst->kUVCoeff[1]; \ - ug = yuvconst->kUVCoeff[2]; \ - vg = yuvconst->kUVCoeff[3]; \ - yg = yuvconst->kRGBCoeffBias[0]; \ - bb = yuvconst->kRGBCoeffBias[1] + 24; \ - bg = yuvconst->kRGBCoeffBias[2] - 24; \ - br = yuvconst->kRGBCoeffBias[3] + 24; \ - } -#endif +#define YUVTORGB_SETUP \ + "csrwi vxrm, 0 \n" \ + "lbu %[ub], 0(%[yuvconst]) \n" \ + "lbu %[vr], 1(%[yuvconst]) \n" \ + "lbu %[ug], 2(%[yuvconst]) \n" \ + "lbu %[vg], 3(%[yuvconst]) \n" \ + "lhu %[yg], 16(%[yuvconst])\n" \ + "lh %[bb], 18(%[yuvconst])\n" \ + "lh %[bg], 20(%[yuvconst])\n" \ + "lh %[br], 22(%[yuvconst])\n" \ + "addi %[bb], %[bb], 32 \n" \ + "addi %[bg], %[bg], -32 \n" \ + "addi %[br], %[br], 32 \n" \ + "li %[k0101], 0x0101 \n" + +// Fill YUV -> AR30 conversion constants into registers +#define YUVTORGB_SETUP_AR30 \ + "csrwi vxrm, 0 \n" \ + "lbu %[ub], 0(%[yuvconst]) \n" \ + "lbu %[vr], 1(%[yuvconst]) \n" \ + "lbu %[ug], 2(%[yuvconst]) \n" \ + "lbu %[vg], 3(%[yuvconst]) \n" \ + "lhu %[yg], 16(%[yuvconst])\n" \ + "lh %[bb], 18(%[yuvconst])\n" \ + "lh %[bg], 20(%[yuvconst])\n" \ + "lh %[br], 22(%[yuvconst])\n" \ + "addi %[bb], %[bb], 24 \n" \ + "addi %[bg], %[bg], -24 \n" \ + "addi %[br], %[br], 24 \n" \ + "li %[k0101], 0x0101 \n" + // Read [2*VLEN/8] Y, [VLEN/8] U and [VLEN/8] V from 422 -#define READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16) \ - { \ - vuint8m1_t v_tmp0, v_tmp1; \ - vuint8m2_t v_y; \ - vuint16m2_t v_u_16, v_v_16; \ - vl = __riscv_vsetvl_e8m1((w + 1) / 2); \ - v_tmp0 = __riscv_vle8_v_u8m1(src_u, vl); \ - v_u_16 = __riscv_vwaddu_vx_u16m2(v_tmp0, 0, vl); \ - v_tmp1 = __riscv_vle8_v_u8m1(src_v, vl); \ - v_v_16 = __riscv_vwaddu_vx_u16m2(v_tmp1, 0, vl); \ - v_v_16 = __riscv_vmul_vx_u16m2(v_v_16, 0x0101, vl); \ - v_u_16 = __riscv_vmul_vx_u16m2(v_u_16, 0x0101, vl); \ - v_v = __riscv_vreinterpret_v_u16m2_u8m2(v_v_16); \ - v_u = __riscv_vreinterpret_v_u16m2_u8m2(v_u_16); \ - vl = __riscv_vsetvl_e8m2(w); \ - v_y = __riscv_vle8_v_u8m2(src_y, vl); \ - v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl); \ - } +#define READYUV422 \ + "addi %[vl], %[w], 1 \n" \ + "srli %[vl], %[vl], 1 \n" \ + "vsetvli %[vl], %[vl], e8, m1, ta, ma \n" \ + "vle8.v v0, (%[src_u]) \n" \ + "vle8.v v1, (%[src_v]) \n" \ + "vwcvtu.x.x.v v2, v0 \n" \ + "vwcvtu.x.x.v v4, v1 \n" \ + "vsetvli zero, zero, e16, m2, ta, ma \n" \ + "vmul.vx v2, v2, %[k0101] \n" \ + "vmul.vx v4, v4, %[k0101] \n" \ + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" \ + "vle8.v v0, (%[src_y]) \n" \ + "vwcvtu.x.x.v v8, v0 \n" // Read [2*VLEN/8] Y, [2*VLEN/8] U, and [2*VLEN/8] V from 444 -#define READYUV444(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16) \ - { \ - vuint8m2_t v_y; \ - vl = __riscv_vsetvl_e8m2(w); \ - v_y = __riscv_vle8_v_u8m2(src_y, vl); \ - v_u = __riscv_vle8_v_u8m2(src_u, vl); \ - v_v = __riscv_vle8_v_u8m2(src_v, vl); \ - v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl); \ - } - -// Convert from YUV to fixed point RGB -#define YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, \ - v_b_16, v_r_16) \ - { \ - vuint16m4_t v_tmp0, v_tmp1, v_tmp2, v_tmp3, v_tmp4; \ - vuint32m8_t v_tmp5; \ - v_tmp0 = __riscv_vwmulu_vx_u16m4(v_u, ug, vl); \ - v_y_16 = __riscv_vmul_vx_u16m4(v_y_16, 0x0101, vl); \ - v_tmp0 = __riscv_vwmaccu_vx_u16m4(v_tmp0, vg, v_v, vl); \ - v_tmp1 = __riscv_vwmulu_vx_u16m4(v_u, ub, vl); \ - v_tmp5 = __riscv_vwmulu_vx_u32m8(v_y_16, yg, vl); \ - v_tmp2 = __riscv_vnsrl_wx_u16m4(v_tmp5, 16, vl); \ - v_tmp3 = __riscv_vadd_vx_u16m4(v_tmp2, bg, vl); \ - v_tmp4 = __riscv_vadd_vv_u16m4(v_tmp2, v_tmp1, vl); \ - v_tmp2 = __riscv_vwmaccu_vx_u16m4(v_tmp2, vr, v_v, vl); \ - v_g_16 = __riscv_vssubu_vv_u16m4(v_tmp3, v_tmp0, vl); \ - v_b_16 = __riscv_vssubu_vx_u16m4(v_tmp4, bb, vl); \ - v_r_16 = __riscv_vssubu_vx_u16m4(v_tmp2, br, vl); \ - } - -#ifdef LIBYUV_RVV_HAS_VXRM_ARG -// Convert from fixed point RGB To 8 bit RGB -#define RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r) \ - { \ - v_g = __riscv_vnclipu_wx_u8m2(v_g_16, 6, __RISCV_VXRM_RNU, vl); \ - v_b = __riscv_vnclipu_wx_u8m2(v_b_16, 6, __RISCV_VXRM_RNU, vl); \ - v_r = __riscv_vnclipu_wx_u8m2(v_r_16, 6, __RISCV_VXRM_RNU, vl); \ - } -#else -// Convert from fixed point RGB To 8 bit RGB -#define RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r) \ - { \ - v_g = __riscv_vnclipu_wx_u8m2(v_g_16, 6, vl); \ - v_b = __riscv_vnclipu_wx_u8m2(v_b_16, 6, vl); \ - v_r = __riscv_vnclipu_wx_u8m2(v_r_16, 6, vl); \ - } -#endif +#define READYUV444 \ + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" \ + "vle8.v v0, (%[src_y]) \n" \ + "vle8.v v2, (%[src_u]) \n" \ + "vle8.v v4, (%[src_v]) \n" \ + "vwcvtu.x.x.v v8, v0 \n" // Read [2*VLEN/8] Y from src_y; Read [VLEN/8] U and [VLEN/8] V from src_uv -#define READNV12(vl, w, src_y, src_uv, v_u, v_v, v_y_16) \ - { \ - vuint8m1x2_t v_tmp; \ - vuint8m1_t v_tmp0, v_tmp1; \ - vuint8m2_t v_y; \ - vuint16m2_t v_u_16, v_v_16; \ - vl = __riscv_vsetvl_e8m1((w + 1) / 2); \ - v_tmp = __riscv_vlseg2e8_v_u8m1x2(src_uv, vl); \ - v_tmp0 = __riscv_vget_v_u8m1x2_u8m1(v_tmp, 0); \ - v_tmp1 = __riscv_vget_v_u8m1x2_u8m1(v_tmp, 1); \ - v_u_16 = __riscv_vwaddu_vx_u16m2(v_tmp0, 0, vl); \ - v_v_16 = __riscv_vwaddu_vx_u16m2(v_tmp1, 0, vl); \ - v_v_16 = __riscv_vmul_vx_u16m2(v_v_16, 0x0101, vl); \ - v_u_16 = __riscv_vmul_vx_u16m2(v_u_16, 0x0101, vl); \ - v_v = __riscv_vreinterpret_v_u16m2_u8m2(v_v_16); \ - v_u = __riscv_vreinterpret_v_u16m2_u8m2(v_u_16); \ - vl = __riscv_vsetvl_e8m2(w); \ - v_y = __riscv_vle8_v_u8m2(src_y, vl); \ - v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl); \ - } +#define READNV12 \ + "addi %[vl], %[w], 1 \n" \ + "srli %[vl], %[vl], 1 \n" \ + "vsetvli %[vl], %[vl], e8, m1, ta, ma \n" \ + "vlseg2e8.v v0, (%[src_uv]) \n" \ + "vwcvtu.x.x.v v2, v0 \n" \ + "vwcvtu.x.x.v v4, v1 \n" \ + "vsetvli zero, zero, e16, m2, ta, ma \n" \ + "vmul.vx v2, v2, %[k0101] \n" \ + "vmul.vx v4, v4, %[k0101] \n" \ + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" \ + "vle8.v v0, (%[src_y]) \n" \ + "vwcvtu.x.x.v v8, v0 \n" // Read 2*[VLEN/8] Y from src_y; Read [VLEN/8] U and [VLEN/8] V from src_vu -#define READNV21(vl, w, src_y, src_vu, v_u, v_v, v_y_16) \ - { \ - vuint8m1x2_t v_tmp; \ - vuint8m1_t v_tmp0, v_tmp1; \ - vuint8m2_t v_y; \ - vuint16m2_t v_u_16, v_v_16; \ - vl = __riscv_vsetvl_e8m1((w + 1) / 2); \ - v_tmp = __riscv_vlseg2e8_v_u8m1x2(src_vu, vl); \ - v_tmp0 = __riscv_vget_v_u8m1x2_u8m1(v_tmp, 0); \ - v_tmp1 = __riscv_vget_v_u8m1x2_u8m1(v_tmp, 1); \ - v_u_16 = __riscv_vwaddu_vx_u16m2(v_tmp1, 0, vl); \ - v_v_16 = __riscv_vwaddu_vx_u16m2(v_tmp0, 0, vl); \ - v_v_16 = __riscv_vmul_vx_u16m2(v_v_16, 0x0101, vl); \ - v_u_16 = __riscv_vmul_vx_u16m2(v_u_16, 0x0101, vl); \ - v_v = __riscv_vreinterpret_v_u16m2_u8m2(v_v_16); \ - v_u = __riscv_vreinterpret_v_u16m2_u8m2(v_u_16); \ - vl = __riscv_vsetvl_e8m2(w); \ - v_y = __riscv_vle8_v_u8m2(src_y, vl); \ - v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl); \ - } +#define READNV21 \ + "addi %[vl], %[w], 1 \n" \ + "srli %[vl], %[vl], 1 \n" \ + "vsetvli %[vl], %[vl], e8, m1, ta, ma \n" \ + "vlseg2e8.v v0, (%[src_vu]) \n" \ + "vwcvtu.x.x.v v4, v0 \n" \ + "vwcvtu.x.x.v v2, v1 \n" \ + "vsetvli zero, zero, e16, m2, ta, ma \n" \ + "vmul.vx v2, v2, %[k0101] \n" \ + "vmul.vx v4, v4, %[k0101] \n" \ + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" \ + "vle8.v v0, (%[src_y]) \n" \ + "vwcvtu.x.x.v v8, v0 \n" + +// Convert from YUV to fixed point RGB +#define YUVTORGB \ + "vwmulu.vx v12, v2, %[ug] \n" \ + "vwmaccu.vx v12, %[vg], v4 \n" \ + "vwmulu.vx v0, v2, %[ub] \n" \ + "vsetvli zero, zero, e16, m4, ta, ma \n" \ + "vmul.vx v8, v8, %[k0101] \n" \ + "vmulhu.vx v8, v8, %[yg] \n" \ + "vadd.vx v16, v8, %[bg] \n" \ + "vadd.vv v0, v8, v0 \n" \ + "vsetvli zero, zero, e8, m2, ta, ma \n" \ + "vwmaccu.vx v8, %[vr], v4 \n" \ + "vsetvli zero, zero, e16, m4, ta, ma \n" \ + "vssubu.vv v12, v16, v12 \n" \ + "vssubu.vx v0, v0, %[bb] \n" \ + "vssubu.vx v8, v8, %[br] \n" + +// Convert from fixed point RGB To 8 bit RGB +#define RGBTORGB8 \ + "vsetvli zero, zero, e8, m2, ta, ma \n" \ + "vnclipu.wi v26, v12, 6 \n" \ + "vnclipu.wi v24, v0, 6 \n" \ + "vnclipu.wi v28, v8, 6 \n" + +#define YUVTORGB_REGS \ + "v0", "v1", "v2", "v3", "v4", "v5", "v8", "v9", "v10", "v11", "v12", "v13", \ + "v14", "v15", "v16", "v17", "v18", "v19", "v24", "v25", "v26", "v27", \ + "v28", "v29" #ifdef HAS_ARGBTOAR64ROW_RVV void ARGBToAR64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ar64, int width) { - size_t avl = (size_t)4 * width; - do { - vuint16m8_t v_ar64; - vuint8m4_t v_argb; - size_t vl = __riscv_vsetvl_e8m4(avl); - v_argb = __riscv_vle8_v_u8m4(src_argb, vl); - v_ar64 = __riscv_vwaddu_vx_u16m8(v_argb, 0, vl); - v_ar64 = __riscv_vmul_vx_u16m8(v_ar64, 0x0101, vl); - __riscv_vse16_v_u16m8(dst_ar64, v_ar64, vl); - avl -= vl; - src_argb += vl; - dst_ar64 += vl; - } while (avl > 0); + int vl; + asm volatile( + "slli %[w], %[w], 2 \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle8.v v16, (%[src_argb]) \n" + "vwcvtu.x.x.v v8, v16 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vmul.vx v8, v8, %[k0101] \n" + "vse16.v v8, (%[dst_ar64]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_ar64], %[dst_ar64], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_ar64] "+r"(dst_ar64), // %[dst_ar64] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [k0101] "r"(0x0101) + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } #endif #ifdef HAS_ARGBTOAB64ROW_RVV void ARGBToAB64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ab64, int width) { - size_t avl = (size_t)width; - do { - vuint16m2x4_t v_dst_ab64; - vuint16m2_t v_b_16, v_g_16, v_r_16, v_a_16; - size_t vl = __riscv_vsetvl_e8m1(avl); - vuint8m1x4_t v_src_argb = __riscv_vlseg4e8_v_u8m1x4(src_argb, vl); - vuint8m1_t v_b = __riscv_vget_v_u8m1x4_u8m1(v_src_argb, 0); - vuint8m1_t v_g = __riscv_vget_v_u8m1x4_u8m1(v_src_argb, 1); - vuint8m1_t v_r = __riscv_vget_v_u8m1x4_u8m1(v_src_argb, 2); - vuint8m1_t v_a = __riscv_vget_v_u8m1x4_u8m1(v_src_argb, 3); - v_b_16 = __riscv_vwaddu_vx_u16m2(v_b, 0, vl); - v_g_16 = __riscv_vwaddu_vx_u16m2(v_g, 0, vl); - v_r_16 = __riscv_vwaddu_vx_u16m2(v_r, 0, vl); - v_a_16 = __riscv_vwaddu_vx_u16m2(v_a, 0, vl); - v_b_16 = __riscv_vmul_vx_u16m2(v_b_16, 0x0101, vl); - v_g_16 = __riscv_vmul_vx_u16m2(v_g_16, 0x0101, vl); - v_r_16 = __riscv_vmul_vx_u16m2(v_r_16, 0x0101, vl); - v_a_16 = __riscv_vmul_vx_u16m2(v_a_16, 0x0101, vl); - v_dst_ab64 = __riscv_vcreate_v_u16m2x4(v_r_16, v_g_16, v_b_16, v_a_16); - __riscv_vsseg4e16_v_u16m2x4(dst_ab64, v_dst_ab64, vl); - avl -= vl; - src_argb += 4 * vl; - dst_ab64 += 4 * vl; - } while (avl > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m1, ta, ma \n" + "vlseg4e8.v v4, (%[src_argb]) \n" + "vwcvtu.x.x.v v12, v4 \n" + "vwcvtu.x.x.v v10, v5 \n" + "vwcvtu.x.x.v v8, v6 \n" + "vwcvtu.x.x.v v14, v7 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vmul.vx v8, v8, %[k0101] \n" + "vmul.vx v10, v10, %[k0101] \n" + "vmul.vx v12, v12, %[k0101] \n" + "vmul.vx v14, v14, %[k0101] \n" + "vsseg4e16.v v8, (%[dst_ab64]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_ab64], %[dst_ab64], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_ab64] "+r"(dst_ab64), // %[dst_ab64] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [k0101] "r"(0x0101) + : "vl", "vtype", "memory", "v4", "v5", "v6", "v7", "v8", "v9", "v10", + "v11", "v12", "v13", "v14", "v15"); } #endif #ifdef HAS_AR64TOARGBROW_RVV void AR64ToARGBRow_RVV(const uint16_t* src_ar64, uint8_t* dst_argb, int width) { - size_t avl = (size_t)4 * width; - do { - vuint16m8_t v_ar64; - vuint8m4_t v_argb; - size_t vl = __riscv_vsetvl_e16m8(avl); - v_ar64 = __riscv_vle16_v_u16m8(src_ar64, vl); - v_argb = __riscv_vnsrl_wx_u8m4(v_ar64, 8, vl); - __riscv_vse8_v_u8m4(dst_argb, v_argb, vl); - avl -= vl; - src_ar64 += vl; - dst_argb += vl; - } while (avl > 0); + int vl; + asm volatile( + "slli %[w], %[w], 2 \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle16.v v8, (%[src_ar64]) \n" + "vnsrl.wi v16, v8, 8 \n" + "vse8.v v16, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[src_ar64], %[src_ar64], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_ar64] "+r"(src_ar64), // %[src_ar64] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } #endif @@ -263,119 +232,177 @@ void AR64ToARGBRow_RVV(const uint16_t* src_ar64, uint8_t* dst_argb, int width) { void AR64ToAB64Row_RVV(const uint16_t* src_ar64, uint16_t* dst_ab64, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e16m2(w); - vuint16m2x4_t v_argb16 = __riscv_vlseg4e16_v_u16m2x4(src_ar64, vl); - vuint16m2_t v_b = __riscv_vget_v_u16m2x4_u16m2(v_argb16, 0); - vuint16m2_t v_g = __riscv_vget_v_u16m2x4_u16m2(v_argb16, 1); - vuint16m2_t v_r = __riscv_vget_v_u16m2x4_u16m2(v_argb16, 2); - vuint16m2_t v_a = __riscv_vget_v_u16m2x4_u16m2(v_argb16, 3); - vuint16m2x4_t v_dst_abgr = __riscv_vcreate_v_u16m2x4(v_r, v_g, v_b, v_a); - __riscv_vsseg4e16_v_u16m2x4(dst_ab64, v_dst_abgr, vl); - w -= vl; - src_ar64 += vl * 4; - dst_ab64 += vl * 4; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e16, m2, ta, ma \n" + "vlseg4e16.v v8, (%[src_ar64]) \n" + "vmv2r.v v16, v12 \n" + "vmv2r.v v18, v10 \n" + "vmv2r.v v20, v8 \n" + "vmv2r.v v22, v14 \n" + "vsseg4e16.v v16, (%[dst_ab64]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 3 \n" + "add %[src_ar64], %[src_ar64], %[vl]\n" + "add %[dst_ab64], %[dst_ab64], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_ar64] "+r"(src_ar64), // %[src_ar64] + [dst_ab64] "+r"(dst_ab64), // %[dst_ab64] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); } #endif #ifdef HAS_AB64TOARGBROW_RVV void AB64ToARGBRow_RVV(const uint16_t* src_ab64, uint8_t* dst_argb, int width) { - size_t avl = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e16m2(avl); - vuint16m2x4_t v_abgr16 = __riscv_vlseg4e16_v_u16m2x4(src_ab64, vl); - vuint16m2_t v_r_16 = __riscv_vget_v_u16m2x4_u16m2(v_abgr16, 0); - vuint16m2_t v_g_16 = __riscv_vget_v_u16m2x4_u16m2(v_abgr16, 1); - vuint16m2_t v_b_16 = __riscv_vget_v_u16m2x4_u16m2(v_abgr16, 2); - vuint16m2_t v_a_16 = __riscv_vget_v_u16m2x4_u16m2(v_abgr16, 3); - vuint8m1_t v_b = __riscv_vnsrl_wx_u8m1(v_b_16, 8, vl); - vuint8m1_t v_g = __riscv_vnsrl_wx_u8m1(v_g_16, 8, vl); - vuint8m1_t v_r = __riscv_vnsrl_wx_u8m1(v_r_16, 8, vl); - vuint8m1_t v_a = __riscv_vnsrl_wx_u8m1(v_a_16, 8, vl); - vuint8m1x4_t v_dst_argb = __riscv_vcreate_v_u8m1x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m1x4(dst_argb, v_dst_argb, vl); - avl -= vl; - src_ab64 += 4 * vl; - dst_argb += 4 * vl; - } while (avl > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m1, ta, ma \n" + "vlseg4e16.v v8, (%[src_ab64]) \n" + "vnsrl.wi v4, v12, 8 \n" + "vnsrl.wi v5, v10, 8 \n" + "vnsrl.wi v6, v8, 8 \n" + "vnsrl.wi v7, v14, 8 \n" + "vsseg4e8.v v4, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[src_ab64], %[src_ab64], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_ab64] "+r"(src_ab64), // %[src_ab64] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v4", "v5", "v6", "v7", "v8", "v9", "v10", + "v11", "v12", "v13", "v14", "v15"); } #endif #ifdef HAS_RAWTOARGBROW_RVV void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - vuint8m2x3_t v_bgr = __riscv_vlseg3e8_v_u8m2x3(src_raw, vl); - vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 1); - vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 2); - vuint8m2x4_t v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_raw += vl * 3; - dst_argb += vl * 4; - vl = __riscv_vsetvl_e8m2(w); - } while (w > 0); + size_t vl, tmp; + asm volatile( + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vmv.v.i v14, -1 \n" + + "1: \n" + "vlseg3e8.v v16, (%[src_raw]) \n" + "vmv2r.v v8, v20 \n" + "vmv2r.v v10, v18 \n" + "vmv2r.v v12, v16 \n" + "vsseg4e8.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[tmp], %[vl], 1 \n" + "add %[src_raw], %[src_raw], %[vl] \n" + "add %[src_raw], %[src_raw], %[tmp]\n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "bgtz %[w], 1b \n" + : [src_raw] "+r"(src_raw), // %[src_raw] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21"); } #endif #ifdef HAS_RAWTORGBAROW_RVV void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - vuint8m2x3_t v_bgr = __riscv_vlseg3e8_v_u8m2x3(src_raw, vl); - vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 1); - vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 2); - vuint8m2x4_t v_dst_rgba = __riscv_vcreate_v_u8m2x4(v_a, v_b, v_g, v_r); - __riscv_vsseg4e8_v_u8m2x4(dst_rgba, v_dst_rgba, vl); - w -= vl; - src_raw += vl * 3; - dst_rgba += vl * 4; - vl = __riscv_vsetvl_e8m2(w); - } while (w > 0); + size_t vl, tmp; + asm volatile( + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vmv.v.i v8, -1 \n" + + "1: \n" + "vlseg3e8.v v16, (%[src_raw]) \n" + "vmv2r.v v10, v20 \n" + "vmv2r.v v12, v18 \n" + "vmv2r.v v14, v16 \n" + "vsseg4e8.v v8, (%[dst_rgba]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[tmp], %[vl], 1 \n" + "add %[src_raw], %[src_raw], %[vl] \n" + "add %[src_raw], %[src_raw], %[tmp]\n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_rgba], %[dst_rgba], %[vl]\n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "bgtz %[w], 1b \n" + : [src_raw] "+r"(src_raw), // %[src_raw] + [dst_rgba] "+r"(dst_rgba), // %[dst_rgba] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21"); } #endif #ifdef HAS_RAWTORGB24ROW_RVV void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x3_t v_bgr = __riscv_vlseg3e8_v_u8m2x3(src_raw, vl); - vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 1); - vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 2); - vuint8m2x3_t v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r); - __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl); - w -= vl; - src_raw += vl * 3; - dst_rgb24 += vl * 3; - } while (w > 0); + size_t vl, tmp; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg3e8.v v16, (%[src_raw]) \n" + "vmv2r.v v8, v20 \n" + "vmv2r.v v10, v18 \n" + "vmv2r.v v12, v16 \n" + "vsseg3e8.v v8, (%[dst_rgb24]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[tmp], %[vl], 1 \n" + "add %[tmp], %[tmp], %[vl] \n" + "add %[src_raw], %[src_raw], %[tmp]\n" + "add %[dst_rgb24], %[dst_rgb24], %[tmp]\n" + "bgtz %[w], 1b \n" + : [src_raw] "+r"(src_raw), // %[src_raw] + [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v16", + "v17", "v18", "v19", "v20", "v21"); } #endif #ifdef HAS_ARGBTORAWROW_RVV void ARGBToRAWRow_RVV(const uint8_t* src_argb, uint8_t* dst_raw, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); - vuint8m2x3_t v_dst_bgr = __riscv_vcreate_v_u8m2x3(v_r, v_g, v_b); - __riscv_vsseg3e8_v_u8m2x3(dst_raw, v_dst_bgr, vl); - w -= vl; - src_argb += vl * 4; - dst_raw += vl * 3; - } while (w > 0); + size_t vl, tmp; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v8, (%[src_argb]) \n" + "vmv2r.v v16, v12 \n" + "vmv2r.v v18, v10 \n" + "vmv2r.v v20, v8 \n" + "vsseg3e8.v v16, (%[dst_raw]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[tmp], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[tmp]\n" + "slli %[tmp], %[vl], 1 \n" + "add %[dst_raw], %[dst_raw], %[vl] \n" + "add %[dst_raw], %[dst_raw], %[tmp]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_raw] "+r"(dst_raw), // %[dst_raw] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21"); } #endif @@ -383,95 +410,137 @@ void ARGBToRAWRow_RVV(const uint8_t* src_argb, uint8_t* dst_raw, int width) { void ARGBToRGB24Row_RVV(const uint8_t* src_argb, uint8_t* dst_rgb24, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); - vuint8m2x3_t v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r); - __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl); - w -= vl; - src_argb += vl * 4; - dst_rgb24 += vl * 3; - } while (w > 0); + size_t vl, tmp; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v8, (%[src_argb]) \n" + "vsseg3e8.v v8, (%[dst_rgb24]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[tmp], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[tmp]\n" + "slli %[tmp], %[vl], 1 \n" + "add %[dst_rgb24], %[dst_rgb24], %[vl]\n" + "add %[dst_rgb24], %[dst_rgb24], %[tmp]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif #ifdef HAS_ARGBTOABGRROW_RVV void ARGBToABGRRow_RVV(const uint8_t* src_argb, uint8_t* dst_abgr, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); - vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3); - vuint8m2x4_t v_dst_abgr = __riscv_vcreate_v_u8m2x4(v_r, v_g, v_b, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_abgr, v_dst_abgr, vl); - w -= vl; - src_argb += vl * 4; - dst_abgr += vl * 4; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v8, (%[src_argb]) \n" + "vmv2r.v v16, v12 \n" + "vmv2r.v v18, v10 \n" + "vmv2r.v v20, v8 \n" + "vmv2r.v v22, v14 \n" + "vsseg4e8.v v16, (%[dst_abgr]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "add %[dst_abgr], %[dst_abgr], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_abgr] "+r"(dst_abgr), // %[dst_abgr] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); } #endif #ifdef HAS_ARGBTOBGRAROW_RVV void ARGBToBGRARow_RVV(const uint8_t* src_argb, uint8_t* dst_bgra, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); - vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3); - vuint8m2x4_t v_dst_bgra = __riscv_vcreate_v_u8m2x4(v_a, v_r, v_g, v_b); - __riscv_vsseg4e8_v_u8m2x4(dst_bgra, v_dst_bgra, vl); - w -= vl; - src_argb += vl * 4; - dst_bgra += vl * 4; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e32, m4, ta, ma \n" + "vle32.v v8, (%[src_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "vsrl.vi v12, v8, 16 \n" + "vsll.vi v8, v8, 16 \n" + "vor.vv v8, v8, v12 \n" + "slli %[vl], %[vl], 1 \n" + "vsetvli zero, %[vl], e16, m4, ta, ma \n" + "vsrl.vi v12, v8, 8 \n" + "vsll.vi v8, v8, 8 \n" + "vor.vv v8, v8, v12 \n" + "vse16.v v8, (%[dst_bgra]) \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "add %[dst_bgra], %[dst_bgra], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_bgra] "+r"(dst_bgra), // %[dst_bgra] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif #ifdef HAS_ARGBTORGBAROW_RVV void ARGBToRGBARow_RVV(const uint8_t* src_argb, uint8_t* dst_rgba, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); - vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3); - vuint8m2x4_t v_dst_rgba = __riscv_vcreate_v_u8m2x4(v_a, v_b, v_g, v_r); - __riscv_vsseg4e8_v_u8m2x4(dst_rgba, v_dst_rgba, vl); - w -= vl; - src_argb += vl * 4; - dst_rgba += vl * 4; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e32, m4, ta, ma \n" + "vle32.v v8, (%[src_argb]) \n" + "vsrl.vi v12, v8, 24 \n" + "vsll.vi v8, v8, 8 \n" + "vor.vv v8, v8, v12 \n" + "vse32.v v8, (%[dst_rgba]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "add %[dst_rgba], %[dst_rgba], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_rgba] "+r"(dst_rgba), // %[dst_rgba] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif #ifdef HAS_RGBATOARGBROW_RVV void RGBAToARGBRow_RVV(const uint8_t* src_rgba, uint8_t* dst_argb, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_rgba = __riscv_vlseg4e8_v_u8m2x4(src_rgba, vl); - vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 0); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 1); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 2); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 3); - vuint8m2x4_t v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_rgba += vl * 4; - dst_argb += vl * 4; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e32, m4, ta, ma \n" + "vle32.v v8, (%[src_rgba]) \n" + "vsrl.vi v12, v8, 8 \n" + "vsll.vi v8, v8, 24 \n" + "vor.vv v8, v8, v12 \n" + "vse32.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_rgba], %[src_rgba], %[vl]\n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_rgba] "+r"(src_rgba), // %[src_rgba] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif @@ -479,21 +548,30 @@ void RGBAToARGBRow_RVV(const uint8_t* src_rgba, uint8_t* dst_argb, int width) { void RGB24ToARGBRow_RVV(const uint8_t* src_rgb24, uint8_t* dst_argb, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - vuint8m2x3_t v_src_rgb = __riscv_vlseg3e8_v_u8m2x3(src_rgb24, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 2); - vuint8m2x4_t v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_rgb24 += vl * 3; - dst_argb += vl * 4; - vl = __riscv_vsetvl_e8m2(w); - } while (w > 0); + size_t vl, tmp; + asm volatile( + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vmv.v.i v14, -1 \n" + + "1: \n" + "vlseg3e8.v v8, (%[src_rgb24]) \n" + "vsseg4e8.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[tmp], %[vl], 1 \n" + "add %[src_rgb24], %[src_rgb24], %[vl]\n" + "add %[src_rgb24], %[src_rgb24], %[tmp]\n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "bgtz %[w], 1b \n" + : [src_rgb24] "+r"(src_rgb24), // %[src_rgb24] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif @@ -504,29 +582,44 @@ void I444ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r, v_a; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - vuint8m2x4_t v_dst_argb; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - READYUV444(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_y += vl; - src_u += vl; - src_v += vl; - dst_argb += vl * 4; - } while (w > 0); + size_t vl; + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + "vsetvli zero, %[w], e8, m2, ta, ma \n" + "vmv.v.i v30, -1 \n" + + "1: \n" // + READYUV444 // + "sub %[w], %[w], %[vl] \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg4e8.v v24, (%[dst_argb]) \n" + "add %[src_y], %[src_y], %[vl] \n" + "add %[src_u], %[src_u], %[vl] \n" + "add %[src_v], %[src_v], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS, "v30", "v31"); } #endif @@ -539,29 +632,44 @@ void I444AlphaToARGBRow_RVV(const uint8_t* src_y, const struct YuvConstants* yuvconstants, int width) { size_t vl; - size_t w = (size_t)width; - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r, v_a; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - do { - vuint8m2x4_t v_dst_argb; - READYUV444(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16); - v_a = __riscv_vle8_v_u8m2(src_a, vl); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_y += vl; - src_a += vl; - src_u += vl; - src_v += vl; - dst_argb += vl * 4; - } while (w > 0); + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + + "1: \n" // + READYUV444 // + "vle8.v v30, (%[src_a]) \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg4e8.v v24, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_y], %[src_y], %[vl] \n" + "add %[src_u], %[src_u], %[vl] \n" + "add %[src_v], %[src_v], %[vl] \n" + "add %[src_a], %[src_a], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [src_a] "+r"(src_a), // %[src_a] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS, "v30", "v31"); } #endif @@ -573,27 +681,42 @@ void I444ToRGB24Row_RVV(const uint8_t* src_y, const struct YuvConstants* yuvconstants, int width) { size_t vl; - size_t w = (size_t)width; - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - do { - vuint8m2x3_t v_dst_rgb; - READYUV444(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r); - __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl); - w -= vl; - src_y += vl; - src_u += vl; - src_v += vl; - dst_rgb24 += vl * 3; - } while (w > 0); + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + + "1: \n" // + READYUV444 // + "sub %[w], %[w], %[vl] \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg3e8.v v24, (%[dst_rgb24]) \n" + "add %[src_y], %[src_y], %[vl] \n" + "add %[src_u], %[src_u], %[vl] \n" + "add %[src_v], %[src_v], %[vl] \n" + "add %[dst_rgb24], %[dst_rgb24], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_rgb24], %[dst_rgb24], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS); } #endif @@ -604,29 +727,46 @@ void I422ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r, v_a; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - vuint8m2x4_t v_dst_argb; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_y += vl; - src_u += vl / 2; - src_v += vl / 2; - dst_argb += vl * 4; - } while (w > 0); + size_t vl; + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + "vsetvli zero, %[w], e8, m2, ta, ma \n" + "vmv.v.i v30, -1 \n" + + "1: \n" // + READYUV422 // + "sub %[w], %[w], %[vl] \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg4e8.v v24, (%[dst_argb]) \n" + "add %[src_y], %[src_y], %[vl] \n" + "srli %[k0101], %[vl], 1 \n" + "add %[src_u], %[src_u], %[k0101] \n" + "add %[src_v], %[src_v], %[k0101] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "li %[k0101], 0x0101 \n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS, "v30", "v31"); } #endif @@ -639,29 +779,46 @@ void I422AlphaToARGBRow_RVV(const uint8_t* src_y, const struct YuvConstants* yuvconstants, int width) { size_t vl; - size_t w = (size_t)width; - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r, v_a; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - do { - vuint8m2x4_t v_dst_argb; - READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16); - v_a = __riscv_vle8_v_u8m2(src_a, vl); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_y += vl; - src_a += vl; - src_u += vl / 2; - src_v += vl / 2; - dst_argb += vl * 4; - } while (w > 0); + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + + "1: \n" // + READYUV422 // + "vle8.v v30, (%[src_a]) \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg4e8.v v24, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_y], %[src_y], %[vl] \n" + "add %[src_a], %[src_a], %[vl] \n" + "srli %[k0101], %[vl], 1 \n" + "add %[src_u], %[src_u], %[k0101] \n" + "add %[src_v], %[src_v], %[k0101] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "li %[k0101], 0x0101 \n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [src_a] "+r"(src_a), // %[src_a] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS, "v30", "v31"); } #endif @@ -672,29 +829,45 @@ void I422ToRGBARow_RVV(const uint8_t* src_y, uint8_t* dst_rgba, const struct YuvConstants* yuvconstants, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r, v_a; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - vuint8m2x4_t v_dst_rgba; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_rgba = __riscv_vcreate_v_u8m2x4(v_a, v_b, v_g, v_r); - __riscv_vsseg4e8_v_u8m2x4(dst_rgba, v_dst_rgba, vl); - w -= vl; - src_y += vl; - src_u += vl / 2; - src_v += vl / 2; - dst_rgba += vl * 4; - } while (w > 0); + size_t vl; + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + + "1: \n" // + READYUV422 // + "sub %[w], %[w], %[vl] \n" // + YUVTORGB // + RGBTORGB8 // + "vmv.v.i v22, -1 \n" + "vsseg4e8.v v22, (%[dst_rgba]) \n" + "add %[src_y], %[src_y], %[vl] \n" + "srli %[k0101], %[vl], 1 \n" + "add %[src_u], %[src_u], %[k0101] \n" + "add %[src_v], %[src_v], %[k0101] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_rgba], %[dst_rgba], %[vl]\n" + "li %[k0101], 0x0101 \n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_rgba] "+r"(dst_rgba), // %[dst_rgba] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS, "v22", "v23"); } #endif @@ -706,27 +879,44 @@ void I422ToRGB24Row_RVV(const uint8_t* src_y, const struct YuvConstants* yuvconstants, int width) { size_t vl; - size_t w = (size_t)width; - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - vuint8m2x3_t v_dst_rgb; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - do { - READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r); - __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl); - w -= vl; - src_y += vl; - src_u += vl / 2; - src_v += vl / 2; - dst_rgb24 += vl * 3; - } while (w > 0); + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + + "1: \n" // + READYUV422 // + "sub %[w], %[w], %[vl] \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg3e8.v v24, (%[dst_rgb24]) \n" + "add %[src_y], %[src_y], %[vl] \n" + "srli %[k0101], %[vl], 1 \n" + "add %[src_u], %[src_u], %[k0101] \n" + "add %[src_v], %[src_v], %[k0101] \n" + "add %[dst_rgb24], %[dst_rgb24], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_rgb24], %[dst_rgb24], %[vl]\n" + "li %[k0101], 0x0101 \n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS); } #endif @@ -738,149 +928,225 @@ void I422ToAR30Row_RVV(const uint8_t* src_y, const struct YuvConstants* yuvconstants, int width) { size_t vl; - size_t w = (size_t)width; - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - vuint16m4_t v_b_10, v_g_10, v_r_10, v_ra_16; - vuint32m8_t v_ar30, v_ra_32; - YUVTORGB_SETUP_AR30(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - do { - READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - v_b_10 = - __riscv_vminu_vx_u16m4(__riscv_vsrl_vx_u16m4(v_b_16, 4, vl), 1023, vl); - v_g_10 = - __riscv_vminu_vx_u16m4(__riscv_vsrl_vx_u16m4(v_g_16, 4, vl), 1023, vl); - v_r_10 = - __riscv_vminu_vx_u16m4(__riscv_vsrl_vx_u16m4(v_r_16, 4, vl), 1023, vl); - v_ar30 = __riscv_vwaddu_vx_u32m8(v_b_10, 0, vl); - v_ar30 = __riscv_vwmaccu_vx_u32m8(v_ar30, 1024, v_g_10, vl); - v_ra_16 = __riscv_vor_vx_u16m4(v_r_10, 0x0c00, vl); - v_ra_32 = __riscv_vwaddu_vx_u32m8(v_ra_16, 0, vl); - v_ra_32 = __riscv_vsll_vx_u32m8(v_ra_32, 20, vl); - v_ar30 = __riscv_vor_vv_u32m8(v_ar30, v_ra_32, vl); - __riscv_vse32_v_u32m8((uint32_t*)dst_ar30, v_ar30, vl); - w -= vl; - src_y += vl; - src_u += vl / 2; - src_v += vl / 2; - dst_ar30 += vl * 4; - } while (w > 0); + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP_AR30 // + + "1: \n" + "addi %[vl], %[w], 1 \n" + "srli %[vl], %[vl], 1 \n" + "vsetvli %[vl], %[vl], e8, mf2, ta, ma \n" + "vle8.v v0, (%[src_u]) \n" + "vle8.v v1, (%[src_v]) \n" + "vwcvtu.x.x.v v2, v0 \n" + "vwcvtu.x.x.v v4, v1 \n" + "vsetvli zero, zero, e16, m1, ta, ma \n" + "vmul.vx v2, v2, %[k0101] \n" + "vmul.vx v4, v4, %[k0101] \n" + "vsetvli %[vl], %[w], e8, m1, ta, ma \n" + "vle8.v v0, (%[src_y]) \n" + "vwcvtu.x.x.v v8, v0 \n" + "vwmulu.vx v12, v2, %[ug] \n" + "vwmaccu.vx v12, %[vg], v4 \n" + "vwmulu.vx v0, v2, %[ub] \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vmul.vx v8, v8, %[k0101] \n" + "vmulhu.vx v8, v8, %[yg] \n" + "vadd.vx v16, v8, %[bg] \n" + "vadd.vv v0, v8, v0 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vwmaccu.vx v8, %[vr], v4 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vssubu.vv v12, v16, v12 \n" + "vssubu.vx v0, v0, %[bb] \n" + "vssubu.vx v8, v8, %[br] \n" + "vsrl.vi v0, v0, 4 \n" + "vsrl.vi v12, v12, 4 \n" + "vsrl.vi v8, v8, 4 \n" + "li %[k0101], 1023 \n" + "vminu.vx v0, v0, %[k0101] \n" + "vminu.vx v12, v12, %[k0101] \n" + "vminu.vx v8, v8, %[k0101] \n" + "vwcvtu.x.x.v v16, v0 \n" + "li %[k0101], 1024 \n" + "vwmaccu.vx v16, %[k0101], v12 \n" + "li %[k0101], 0x0c00 \n" + "vor.vx v8, v8, %[k0101] \n" + "vwcvtu.x.x.v v20, v8 \n" + "vsetvli zero, zero, e32, m4, ta, ma \n" + "vsll.vi v20, v20, 20 \n" + "vor.vv v16, v16, v20 \n" + "vse32.v v16, (%[dst_ar30]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_y], %[src_y], %[vl] \n" + "srli %[k0101], %[vl], 1 \n" + "add %[src_u], %[src_u], %[k0101] \n" + "add %[src_v], %[src_v], %[k0101] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_ar30], %[dst_ar30], %[vl]\n" + "li %[k0101], 0x0101 \n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_ar30] "+r"(dst_ar30), // %[dst_ar30] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v4", "v8", "v9", "v12", + "v13", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); } #endif #ifdef HAS_I400TOARGBROW_RVV -#if defined(LIBYUV_RVV_HAS_VXRM_ARG) void I400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); + size_t vl; const bool is_yb_positive = (yuvconstants->kRGBCoeffBias[4] >= 0); - vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl); - vuint16m4_t v_yg = __riscv_vmv_v_x_u16m4(yuvconstants->kRGBCoeffBias[0], vl); - vuint8m2x4_t v_dst_argb; - vuint16m4_t v_yb; + uint16_t yg = yuvconstants->kRGBCoeffBias[0]; + uint16_t yb = is_yb_positive + ? (uint16_t)(yuvconstants->kRGBCoeffBias[4] - 32) + : (uint16_t)(-yuvconstants->kRGBCoeffBias[4] + 32); + size_t k0101; if (is_yb_positive) { - v_yb = __riscv_vmv_v_x_u16m4(yuvconstants->kRGBCoeffBias[4] - 32, vl); + asm volatile( + "csrwi vxrm, 0 \n" + "li %[k0101], 0x0101 \n" + "vsetvli zero, %[w], e16, m4, ta, ma \n" + "vmv.v.x v20, %[yg] \n" + "vmv.v.x v24, %[yb] \n" + "vsetvli zero, %[w], e8, m2, ta, ma \n" + "vmv.v.i v14, -1 \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle8.v v0, (%[src_y]) \n" + "vwcvtu.x.x.v v16, v0 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vmul.vx v16, v16, %[k0101] \n" + "vmulhu.vv v16, v16, v20 \n" + "vsaddu.vv v16, v16, v24 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnclipu.wi v8, v16, 6 \n" + "vmv2r.v v10, v8 \n" + "vmv2r.v v12, v8 \n" + "vsseg4e8.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_y], %[src_y], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [k0101] "=&r"(k0101) // %[k0101] + : [yg] "r"(yg), // %[yg] + [yb] "r"(yb) // %[yb] + : "vl", "vtype", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", + "v23", "v24", "v25", "v26", "v27"); } else { - v_yb = __riscv_vmv_v_x_u16m4(-yuvconstants->kRGBCoeffBias[4] + 32, vl); + asm volatile( + "csrwi vxrm, 0 \n" + "li %[k0101], 0x0101 \n" + "vsetvli zero, %[w], e16, m4, ta, ma \n" + "vmv.v.x v20, %[yg] \n" + "vmv.v.x v24, %[yb] \n" + "vsetvli zero, %[w], e8, m2, ta, ma \n" + "vmv.v.i v14, -1 \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle8.v v0, (%[src_y]) \n" + "vwcvtu.x.x.v v16, v0 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vmul.vx v16, v16, %[k0101] \n" + "vmulhu.vv v16, v16, v20 \n" + "vssubu.vv v16, v16, v24 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnclipu.wi v8, v16, 6 \n" + "vmv2r.v v10, v8 \n" + "vmv2r.v v12, v8 \n" + "vsseg4e8.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_y], %[src_y], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [k0101] "=&r"(k0101) // %[k0101] + : [yg] "r"(yg), // %[yg] + [yb] "r"(yb) // %[yb] + : "vl", "vtype", "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", + "v23", "v24", "v25", "v26", "v27"); } - do { - vuint8m2_t v_y, v_out; - vuint16m4_t v_y_16, v_tmp0, v_tmp1, v_tmp2; - vl = __riscv_vsetvl_e8m2(w); - v_y = __riscv_vle8_v_u8m2(src_y, vl); - v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl); - v_tmp0 = __riscv_vmul_vx_u16m4(v_y_16, 0x0101, vl); // 257 * v_y - v_tmp1 = __riscv_vmulhu_vv_u16m4(v_tmp0, v_yg, vl); - if (is_yb_positive) { - v_tmp2 = __riscv_vsaddu_vv_u16m4(v_tmp1, v_yb, vl); - } else { - v_tmp2 = __riscv_vssubu_vv_u16m4(v_tmp1, v_yb, vl); - } - v_out = __riscv_vnclipu_wx_u8m2(v_tmp2, 6, __RISCV_VXRM_RNU, vl); - v_dst_argb = __riscv_vcreate_v_u8m2x4(v_out, v_out, v_out, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_y += vl; - dst_argb += vl * 4; - } while (w > 0); } -#else -void I400ToARGBRow_RVV(const uint8_t* src_y, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - const bool is_yb_positive = (yuvconstants->kRGBCoeffBias[4] >= 0); - vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl); - vuint16m4_t v_yb; - vuint16m4_t v_yg = __riscv_vmv_v_x_u16m4(yuvconstants->kRGBCoeffBias[0], vl); - // To match behavior on other platforms, vxrm (fixed-point rounding mode - // register) sets to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - if (is_yb_positive) { - v_yb = __riscv_vmv_v_x_u16m4(yuvconstants->kRGBCoeffBias[4] - 32, vl); - } else { - v_yb = __riscv_vmv_v_x_u16m4(-yuvconstants->kRGBCoeffBias[4] + 32, vl); - } - do { - vuint8m2_t v_y, v_out; - vuint16m4_t v_y_16, v_tmp0, v_tmp1, v_tmp2; - vl = __riscv_vsetvl_e8m2(w); - v_y = __riscv_vle8_v_u8m2(src_y, vl); - v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl); - v_tmp0 = __riscv_vmul_vx_u16m4(v_y_16, 0x0101, vl); // 257 * v_y - v_tmp1 = __riscv_vmulhu_vv_u16m4(v_tmp0, v_yg, vl); - if (is_yb_positive) { - v_tmp2 = __riscv_vsaddu_vv_u16m4(v_tmp1, v_yb, vl); - } else { - v_tmp2 = __riscv_vssubu_vv_u16m4(v_tmp1, v_yb, vl); - } - v_out = __riscv_vnclipu_wx_u8m2(v_tmp2, 6, vl); - __riscv_vsseg4e8_v_u8m2(dst_argb, v_out, v_out, v_out, v_a, vl); - w -= vl; - src_y += vl; - dst_argb += vl * 4; - } while (w > 0); -} -#endif #endif #ifdef HAS_J400TOARGBROW_RVV void J400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - vuint8m2_t v_y = __riscv_vle8_v_u8m2(src_y, vl); - vuint8m2x4_t v_dst_argb = __riscv_vcreate_v_u8m2x4(v_y, v_y, v_y, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_y += vl; - dst_argb += vl * 4; - vl = __riscv_vsetvl_e8m2(w); - } while (w > 0); + size_t vl; + asm volatile( + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vmv.v.i v14, -1 \n" + + "1: \n" + "vle8.v v8, (%[src_y]) \n" + "vmv2r.v v10, v8 \n" + "vmv2r.v v12, v8 \n" + "vsseg4e8.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_y], %[src_y], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif #ifdef HAS_COPYROW_RVV void CopyRow_RVV(const uint8_t* src, uint8_t* dst, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m8(w); - vuint8m8_t v_data = __riscv_vle8_v_u8m8(src, vl); - __riscv_vse8_v_u8m8(dst, v_data, vl); - w -= vl; - src += vl; - dst += vl; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m4, ta, ma \n" + "vle8.v v8, (%[src]) \n" + "vse8.v v8, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src], %[src], %[vl] \n" + "add %[dst], %[dst], %[vl] \n" + "bgtz %[w], 1b \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11"); } #endif @@ -890,28 +1156,42 @@ void NV12ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r, v_a; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - vuint8m2x4_t v_dst_argb; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - READNV12(vl, w, src_y, src_uv, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_y += vl; - src_uv += vl; - dst_argb += vl * 4; - } while (w > 0); + size_t vl; + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + "vsetvli zero, %[w], e8, m2, ta, ma \n" + "vmv.v.i v30, -1 \n" + + "1: \n" // + READNV12 // + "sub %[w], %[w], %[vl] \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg4e8.v v24, (%[dst_argb]) \n" + "add %[src_y], %[src_y], %[vl] \n" + "add %[src_uv], %[src_uv], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS, "v30", "v31"); } #endif @@ -921,27 +1201,41 @@ void NV12ToRGB24Row_RVV(const uint8_t* src_y, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r; - vuint8m2x3_t v_dst_rgb; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - do { - READNV12(vl, w, src_y, src_uv, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r); - __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl); - w -= vl; - src_y += vl; - src_uv += vl; - dst_rgb24 += vl * 3; - } while (w > 0); + size_t vl; + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + + "1: \n" // + READNV12 // + "sub %[w], %[w], %[vl] \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg3e8.v v24, (%[dst_rgb24]) \n" + "add %[src_y], %[src_y], %[vl] \n" + "add %[src_uv], %[src_uv], %[vl] \n" + "add %[dst_rgb24], %[dst_rgb24], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_rgb24], %[dst_rgb24], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS); } #endif @@ -951,28 +1245,42 @@ void NV21ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r, v_a; - vuint8m2x4_t v_dst_argb; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - READNV21(vl, w, src_y, src_vu, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_y += vl; - src_vu += vl; - dst_argb += vl * 4; - } while (w > 0); + size_t vl; + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + "vsetvli zero, %[w], e8, m2, ta, ma \n" + "vmv.v.i v30, -1 \n" + + "1: \n" // + READNV21 // + "sub %[w], %[w], %[vl] \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg4e8.v v24, (%[dst_argb]) \n" + "add %[src_y], %[src_y], %[vl] \n" + "add %[src_vu], %[src_vu], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_vu] "+r"(src_vu), // %[src_vu] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS, "v30", "v31"); } #endif @@ -982,33 +1290,46 @@ void NV21ToRGB24Row_RVV(const uint8_t* src_y, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - uint8_t ub, vr, ug, vg; - int16_t yg, bb, bg, br; - vuint8m2_t v_u, v_v; - vuint8m2_t v_b, v_g, v_r; - vuint8m2x3_t v_dst_rgb; - vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16; - YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br); - do { - READNV21(vl, w, src_y, src_vu, v_u, v_v, v_y_16); - YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16, - v_b_16, v_r_16); - RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r); - v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r); - __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl); - w -= vl; - src_y += vl; - src_vu += vl; - dst_rgb24 += vl * 3; - } while (w > 0); + size_t vl; + size_t ub, vr, ug, vg; + size_t yg, bb, bg, br; + size_t k0101; + asm volatile( // + YUVTORGB_SETUP // + + "1: \n" // + READNV21 // + "sub %[w], %[w], %[vl] \n" // + YUVTORGB // + RGBTORGB8 // + "vsseg3e8.v v24, (%[dst_rgb24]) \n" + "add %[src_y], %[src_y], %[vl] \n" + "add %[src_vu], %[src_vu], %[vl] \n" + "add %[dst_rgb24], %[dst_rgb24], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_rgb24], %[dst_rgb24], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_vu] "+r"(src_vu), // %[src_vu] + [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [ub] "=&r"(ub), // %[ub] + [vr] "=&r"(vr), // %[vr] + [ug] "=&r"(ug), // %[ug] + [vg] "=&r"(vg), // %[vg] + [yg] "=&r"(yg), // %[yg] + [bb] "=&r"(bb), // %[bb] + [bg] "=&r"(bg), // %[bg] + [br] "=&r"(br), // %[br] + [k0101] "=&r"(k0101) // %[k0101] + : [yuvconst] "r"(yuvconstants) // %[yuvconst] + : "vl", "vtype", "memory", YUVTORGB_REGS); } #endif // Bilinear filter [VLEN/8]x2 -> [VLEN/8]x1 #ifdef HAS_INTERPOLATEROW_RVV -#ifdef LIBYUV_RVV_HAS_VXRM_ARG void InterpolateRow_RVV(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -1018,108 +1339,80 @@ void InterpolateRow_RVV(uint8_t* dst_ptr, int y0_fraction = 256 - y1_fraction; const uint8_t* src_ptr1 = src_ptr + src_stride; size_t dst_w = (size_t)dst_width; + size_t vl; assert(source_y_fraction >= 0); assert(source_y_fraction < 256); // Blend 100 / 0 - Copy row unchanged. if (y1_fraction == 0) { - do { - size_t vl = __riscv_vsetvl_e8m8(dst_w); - __riscv_vse8_v_u8m8(dst_ptr, __riscv_vle8_v_u8m8(src_ptr, vl), vl); - dst_w -= vl; - src_ptr += vl; - dst_ptr += vl; - } while (dst_w > 0); + asm volatile( + "1: \n" + "vsetvli %[vl], %[dst_w], e8, m4, ta, ma\n" + "vle8.v v8, (%[src_ptr]) \n" + "vse8.v v8, (%[dst_ptr]) \n" + "sub %[dst_w], %[dst_w], %[vl] \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "bgtz %[dst_w], 1b \n" + : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst_w] "+r"(dst_w), // %[dst_w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11"); return; } // Blend 50 / 50. if (y1_fraction == 128) { - do { - size_t vl = __riscv_vsetvl_e8m8(dst_w); - vuint8m8_t row0 = __riscv_vle8_v_u8m8(src_ptr, vl); - vuint8m8_t row1 = __riscv_vle8_v_u8m8(src_ptr1, vl); - vuint8m8_t row_out = - __riscv_vaaddu_vv_u8m8(row0, row1, __RISCV_VXRM_RNU, vl); - __riscv_vse8_v_u8m8(dst_ptr, row_out, vl); - dst_w -= vl; - src_ptr += vl; - src_ptr1 += vl; - dst_ptr += vl; - } while (dst_w > 0); + asm volatile( + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[dst_w], e8, m4, ta, ma\n" + "vle8.v v8, (%[src_ptr]) \n" + "vle8.v v12, (%[src_ptr1]) \n" + "vaaddu.vv v8, v8, v12 \n" + "vse8.v v8, (%[dst_ptr]) \n" + "sub %[dst_w], %[dst_w], %[vl] \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "add %[src_ptr1], %[src_ptr1], %[vl]\n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "bgtz %[dst_w], 1b \n" + : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_w] "+r"(dst_w), // %[dst_w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", + "v14", "v15"); return; } // General purpose row blend. - do { - size_t vl = __riscv_vsetvl_e8m4(dst_w); - vuint8m4_t row0 = __riscv_vle8_v_u8m4(src_ptr, vl); - vuint16m8_t acc = __riscv_vwmulu_vx_u16m8(row0, y0_fraction, vl); - vuint8m4_t row1 = __riscv_vle8_v_u8m4(src_ptr1, vl); - acc = __riscv_vwmaccu_vx_u16m8(acc, y1_fraction, row1, vl); - __riscv_vse8_v_u8m4( - dst_ptr, __riscv_vnclipu_wx_u8m4(acc, 8, __RISCV_VXRM_RNU, vl), vl); - dst_w -= vl; - src_ptr += vl; - src_ptr1 += vl; - dst_ptr += vl; - } while (dst_w > 0); + asm volatile( + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[dst_w], e8, m2, ta, ma\n" + "vle8.v v16, (%[src_ptr]) \n" + "vwmulu.vx v8, v16, %[y0_fraction] \n" + "vle8.v v16, (%[src_ptr1]) \n" + "vwmaccu.vx v8, %[y1_fraction], v16 \n" + "vnclipu.wi v16, v8, 8 \n" + "vse8.v v16, (%[dst_ptr]) \n" + "sub %[dst_w], %[dst_w], %[vl] \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "add %[src_ptr1], %[src_ptr1], %[vl]\n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "bgtz %[dst_w], 1b \n" + : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_w] "+r"(dst_w), // %[dst_w] + [vl] "=&r"(vl) // %[vl] + : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] + [y1_fraction] "r"(y1_fraction) // %[y1_fraction] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } -#else -void InterpolateRow_RVV(uint8_t* dst_ptr, - const uint8_t* src_ptr, - ptrdiff_t src_stride, - int dst_width, - int source_y_fraction) { - int y1_fraction = source_y_fraction; - int y0_fraction = 256 - y1_fraction; - const uint8_t* src_ptr1 = src_ptr + src_stride; - size_t dst_w = (size_t)dst_width; - assert(source_y_fraction >= 0); - assert(source_y_fraction < 256); - // Blend 100 / 0 - Copy row unchanged. - if (y1_fraction == 0) { - do { - size_t vl = __riscv_vsetvl_e8m8(dst_w); - __riscv_vse8_v_u8m8(dst_ptr, __riscv_vle8_v_u8m8(src_ptr, vl), vl); - dst_w -= vl; - src_ptr += vl; - dst_ptr += vl; - } while (dst_w > 0); - return; - } - // To match behavior on other platforms, vxrm (fixed-point rounding mode - // register) is set to round-to-nearest-up(0). - asm volatile("csrwi vxrm, 0"); - // Blend 50 / 50. - if (y1_fraction == 128) { - do { - size_t vl = __riscv_vsetvl_e8m8(dst_w); - vuint8m8_t row0 = __riscv_vle8_v_u8m8(src_ptr, vl); - vuint8m8_t row1 = __riscv_vle8_v_u8m8(src_ptr1, vl); - // Use round-to-nearest-up mode for averaging add - vuint8m8_t row_out = __riscv_vaaddu_vv_u8m8(row0, row1, vl); - __riscv_vse8_v_u8m8(dst_ptr, row_out, vl); - dst_w -= vl; - src_ptr += vl; - src_ptr1 += vl; - dst_ptr += vl; - } while (dst_w > 0); - return; - } - // General purpose row blend. - do { - size_t vl = __riscv_vsetvl_e8m4(dst_w); - vuint8m4_t row0 = __riscv_vle8_v_u8m4(src_ptr, vl); - vuint16m8_t acc = __riscv_vwmulu_vx_u16m8(row0, y0_fraction, vl); - vuint8m4_t row1 = __riscv_vle8_v_u8m4(src_ptr1, vl); - acc = __riscv_vwmaccu_vx_u16m8(acc, y1_fraction, row1, vl); - // Use round-to-nearest-up mode for vnclip - __riscv_vse8_v_u8m4(dst_ptr, __riscv_vnclipu_wx_u8m4(acc, 8, vl), vl); - dst_w -= vl; - src_ptr += vl; - src_ptr1 += vl; - dst_ptr += vl; - } while (dst_w > 0); -} -#endif #endif #ifdef HAS_SPLITRGBROW_RVV @@ -1128,22 +1421,31 @@ void SplitRGBRow_RVV(const uint8_t* src_rgb, uint8_t* dst_g, uint8_t* dst_b, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x3_t v_src = __riscv_vlseg3e8_v_u8m2x3(src_rgb, vl); - vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_src, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_src, 1); - vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_src, 2); - __riscv_vse8_v_u8m2(dst_r, v_r, vl); - __riscv_vse8_v_u8m2(dst_g, v_g, vl); - __riscv_vse8_v_u8m2(dst_b, v_b, vl); - w -= vl; - dst_r += vl; - dst_g += vl; - dst_b += vl; - src_rgb += vl * 3; - } while (w > 0); + size_t vl, tmp; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg3e8.v v8, (%[src_rgb]) \n" + "vse8.v v8, (%[dst_r]) \n" + "vse8.v v10, (%[dst_g]) \n" + "vse8.v v12, (%[dst_b]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_r], %[dst_r], %[vl] \n" + "add %[dst_g], %[dst_g], %[vl] \n" + "add %[dst_b], %[dst_b], %[vl] \n" + "slli %[tmp], %[vl], 1 \n" + "add %[src_rgb], %[src_rgb], %[vl] \n" + "add %[src_rgb], %[src_rgb], %[tmp]\n" + "bgtz %[w], 1b \n" + : [src_rgb] "+r"(src_rgb), // %[src_rgb] + [dst_r] "+r"(dst_r), // %[dst_r] + [dst_g] "+r"(dst_g), // %[dst_g] + [dst_b] "+r"(dst_b), // %[dst_b] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13"); } #endif @@ -1153,20 +1455,31 @@ void MergeRGBRow_RVV(const uint8_t* src_r, const uint8_t* src_b, uint8_t* dst_rgb, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_r = __riscv_vle8_v_u8m2(src_r, vl); - vuint8m2_t v_g = __riscv_vle8_v_u8m2(src_g, vl); - vuint8m2_t v_b = __riscv_vle8_v_u8m2(src_b, vl); - vuint8m2x3_t v_dst = __riscv_vcreate_v_u8m2x3(v_r, v_g, v_b); - __riscv_vsseg3e8_v_u8m2x3(dst_rgb, v_dst, vl); - w -= vl; - src_r += vl; - src_g += vl; - src_b += vl; - dst_rgb += vl * 3; - } while (w > 0); + size_t vl, tmp; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle8.v v8, (%[src_r]) \n" + "vle8.v v10, (%[src_g]) \n" + "vle8.v v12, (%[src_b]) \n" + "vsseg3e8.v v8, (%[dst_rgb]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_r], %[src_r], %[vl] \n" + "add %[src_g], %[src_g], %[vl] \n" + "add %[src_b], %[src_b], %[vl] \n" + "slli %[tmp], %[vl], 1 \n" + "add %[dst_rgb], %[dst_rgb], %[vl] \n" + "add %[dst_rgb], %[dst_rgb], %[tmp]\n" + "bgtz %[w], 1b \n" + : [src_r] "+r"(src_r), // %[src_r] + [src_g] "+r"(src_g), // %[src_g] + [src_b] "+r"(src_b), // %[src_b] + [dst_rgb] "+r"(dst_rgb), // %[dst_rgb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13"); } #endif @@ -1177,25 +1490,33 @@ void SplitARGBRow_RVV(const uint8_t* src_argb, uint8_t* dst_b, uint8_t* dst_a, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src, 2); - vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src, 3); - __riscv_vse8_v_u8m2(dst_a, v_a, vl); - __riscv_vse8_v_u8m2(dst_r, v_r, vl); - __riscv_vse8_v_u8m2(dst_g, v_g, vl); - __riscv_vse8_v_u8m2(dst_b, v_b, vl); - w -= vl; - dst_a += vl; - dst_r += vl; - dst_g += vl; - dst_b += vl; - src_argb += vl * 4; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v8, (%[src_argb]) \n" + "vse8.v v14, (%[dst_a]) \n" + "vse8.v v12, (%[dst_r]) \n" + "vse8.v v10, (%[dst_g]) \n" + "vse8.v v8, (%[dst_b]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_a], %[dst_a], %[vl] \n" + "add %[dst_r], %[dst_r], %[vl] \n" + "add %[dst_g], %[dst_g], %[vl] \n" + "add %[dst_b], %[dst_b], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_r] "+r"(dst_r), // %[dst_r] + [dst_g] "+r"(dst_g), // %[dst_g] + [dst_b] "+r"(dst_b), // %[dst_b] + [dst_a] "+r"(dst_a), // %[dst_a] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif @@ -1206,22 +1527,33 @@ void MergeARGBRow_RVV(const uint8_t* src_r, const uint8_t* src_a, uint8_t* dst_argb, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_r = __riscv_vle8_v_u8m2(src_r, vl); - vuint8m2_t v_g = __riscv_vle8_v_u8m2(src_g, vl); - vuint8m2_t v_b = __riscv_vle8_v_u8m2(src_b, vl); - vuint8m2_t v_a = __riscv_vle8_v_u8m2(src_a, vl); - vuint8m2x4_t v_dst = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst, vl); - w -= vl; - src_r += vl; - src_g += vl; - src_b += vl; - src_a += vl; - dst_argb += vl * 4; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle8.v v12, (%[src_r]) \n" + "vle8.v v10, (%[src_g]) \n" + "vle8.v v8, (%[src_b]) \n" + "vle8.v v14, (%[src_a]) \n" + "vsseg4e8.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_r], %[src_r], %[vl] \n" + "add %[src_g], %[src_g], %[vl] \n" + "add %[src_b], %[src_b], %[vl] \n" + "add %[src_a], %[src_a], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_r] "+r"(src_r), // %[src_r] + [src_g] "+r"(src_g), // %[src_g] + [src_b] "+r"(src_b), // %[src_b] + [src_a] "+r"(src_a), // %[src_a] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif @@ -1231,22 +1563,30 @@ void SplitXRGBRow_RVV(const uint8_t* src_argb, uint8_t* dst_g, uint8_t* dst_b, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src, 2); - __riscv_vse8_v_u8m2(dst_r, v_r, vl); - __riscv_vse8_v_u8m2(dst_g, v_g, vl); - __riscv_vse8_v_u8m2(dst_b, v_b, vl); - w -= vl; - dst_r += vl; - dst_g += vl; - dst_b += vl; - src_argb += vl * 4; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v8, (%[src_argb]) \n" + "vse8.v v12, (%[dst_r]) \n" + "vse8.v v10, (%[dst_g]) \n" + "vse8.v v8, (%[dst_b]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_r], %[dst_r], %[vl] \n" + "add %[dst_g], %[dst_g], %[vl] \n" + "add %[dst_b], %[dst_b], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_r] "+r"(dst_r), // %[dst_r] + [dst_g] "+r"(dst_g), // %[dst_g] + [dst_b] "+r"(dst_b), // %[dst_b] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif @@ -1256,22 +1596,33 @@ void MergeXRGBRow_RVV(const uint8_t* src_r, const uint8_t* src_b, uint8_t* dst_argb, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl); - do { - vuint8m2_t v_r = __riscv_vle8_v_u8m2(src_r, vl); - vuint8m2_t v_g = __riscv_vle8_v_u8m2(src_g, vl); - vuint8m2_t v_b = __riscv_vle8_v_u8m2(src_b, vl); - vuint8m2x4_t v_dst = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst, vl); - w -= vl; - src_r += vl; - src_g += vl; - src_b += vl; - dst_argb += vl * 4; - vl = __riscv_vsetvl_e8m2(w); - } while (w > 0); + size_t vl; + asm volatile( + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vmv.v.i v14, -1 \n" + + "1: \n" + "vle8.v v12, (%[src_r]) \n" + "vle8.v v10, (%[src_g]) \n" + "vle8.v v8, (%[src_b]) \n" + "vsseg4e8.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_r], %[src_r], %[vl] \n" + "add %[src_g], %[src_g], %[vl] \n" + "add %[src_b], %[src_b], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "bgtz %[w], 1b \n" + : [src_r] "+r"(src_r), // %[src_r] + [src_g] "+r"(src_g), // %[src_g] + [src_b] "+r"(src_b), // %[src_b] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif @@ -1280,19 +1631,27 @@ void SplitUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m4(w); - vuint8m4x2_t v_src = __riscv_vlseg2e8_v_u8m4x2(src_uv, vl); - vuint8m4_t v_u = __riscv_vget_v_u8m4x2_u8m4(v_src, 0); - vuint8m4_t v_v = __riscv_vget_v_u8m4x2_u8m4(v_src, 1); - __riscv_vse8_v_u8m4(dst_u, v_u, vl); - __riscv_vse8_v_u8m4(dst_v, v_v, vl); - w -= vl; - dst_u += vl; - dst_v += vl; - src_uv += 2 * vl; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m4, ta, ma \n" + "vlseg2e8.v v8, (%[src_uv]) \n" + "vse8.v v8, (%[dst_u]) \n" + "vse8.v v12, (%[dst_v]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_u], %[dst_u], %[vl] \n" + "add %[dst_v], %[dst_v], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_uv], %[src_uv], %[vl] \n" + "bgtz %[w], 1b \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif @@ -1301,32 +1660,37 @@ void MergeUVRow_RVV(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m4(w); - vuint8m4_t v_u = __riscv_vle8_v_u8m4(src_u, vl); - vuint8m4_t v_v = __riscv_vle8_v_u8m4(src_v, vl); - vuint8m4x2_t v_dst = __riscv_vcreate_v_u8m4x2(v_u, v_v); - __riscv_vsseg2e8_v_u8m4x2(dst_uv, v_dst, vl); - w -= vl; - src_u += vl; - src_v += vl; - dst_uv += 2 * vl; - } while (w > 0); + size_t vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m4, ta, ma \n" + "vle8.v v8, (%[src_u]) \n" + "vle8.v v12, (%[src_v]) \n" + "vsseg2e8.v v8, (%[dst_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_u], %[src_u], %[vl] \n" + "add %[src_v], %[src_v], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "bgtz %[w], 1b \n" + : [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif #ifdef HAS_SWAPUVROW_RVV -// TODO(fbarchard): RVV_ASM uses vle16.v/vse16.v which requires hardware -// support for unaligned 16-bit vector loads and stores. void SwapUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_vu, int width) { assert(width != 0); -#ifdef RVV_ASM #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wvla" size_t vl; - asm( - "1: \n" + asm("1: \n" "vsetvli %[vl], %[w], e16, m4, ta, ma \n" "vle16.v v8, (%[src_uv]) \n" "vsrl.vi v12, v8, 8 \n" @@ -1342,35 +1706,10 @@ void SwapUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_vu, int width) { [dst_vu] "+r"(dst_vu), // %[dst_vu] [w] "+r"(width), // %[w] [vl] "=&r"(vl), // %[vl] - "=m"(*(uint8_t (*)[width * 2])dst_vu) - : "m"(*(const uint8_t (*)[width * 2])src_uv) + "=m"(*(uint8_t (*)[width * 2]) dst_vu) + : "m"(*(const uint8_t (*)[width * 2]) src_uv) : "vl", "vtype", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"); #pragma GCC diagnostic pop -#else - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m4(w); -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) - vuint8m4x2_t v_src = __riscv_vlseg2e8_v_u8m4x2(src_uv, vl); - vuint8m4_t v_u = __riscv_vget_v_u8m4x2_u8m4(v_src, 0); - vuint8m4_t v_v = __riscv_vget_v_u8m4x2_u8m4(v_src, 1); -#if defined(LIBYUV_RVV_HAS_VCREATE) - vuint8m4x2_t v_dst = __riscv_vcreate_v_u8m4x2(v_v, v_u); -#else - vuint8m4x2_t v_dst = __riscv_vset_v_u8m4_u8m4x2(v_src, 0, v_v); - v_dst = __riscv_vset_v_u8m4_u8m4x2(v_dst, 1, v_u); -#endif - __riscv_vsseg2e8_v_u8m4x2(dst_vu, v_dst, vl); -#else - vuint8m4_t v_u, v_v; - __riscv_vlseg2e8_v_u8m4(&v_u, &v_v, src_uv, vl); - __riscv_vsseg2e8_v_u8m4(dst_vu, v_v, v_u, vl); -#endif - w -= vl; - src_uv += 2 * vl; - dst_vu += 2 * vl; - } while (w > 0); -#endif } #endif @@ -1381,35 +1720,47 @@ void ARGBToYMatrixRow_RVV(const uint8_t* src_argb, int width, const struct ArgbConstants* c) { assert(width != 0); - size_t w = (size_t)width; - vuint8m2_t v_y0, v_y1, v_y2, v_y3; - vuint16m4_t v_addy; - size_t vl = __riscv_vsetvl_e8m2(w); - v_y0 = __riscv_vmv_v_x_u8m2(c->kRGBToY[0], vl); - v_y1 = __riscv_vmv_v_x_u8m2(c->kRGBToY[1], vl); - v_y2 = __riscv_vmv_v_x_u8m2(c->kRGBToY[2], vl); - v_y3 = __riscv_vmv_v_x_u8m2(c->kRGBToY[3], vl); - v_addy = __riscv_vmv_v_x_u16m4(c->kAddY[0], vl); - do { - vuint8m2_t v_y; - vuint16m4_t v_y_u16; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b0 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); - vuint8m2_t v_b1 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); - vuint8m2_t v_b2 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); - vuint8m2_t v_b3 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3); - v_y_u16 = __riscv_vwmulu_vv_u16m4(v_b0, v_y0, vl); - v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_y1, v_b1, vl); - v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_y2, v_b2, vl); - v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_y3, v_b3, vl); - v_y_u16 = __riscv_vadd_vv_u16m4(v_y_u16, v_addy, vl); - v_y = __riscv_vnsrl_wx_u8m2(v_y_u16, 8, vl); - __riscv_vse8_v_u8m2(dst_y, v_y, vl); - w -= vl; - src_argb += 4 * vl; - dst_y += vl; - } while (w > 0); + size_t vl, tmp; + asm volatile( + "lhu %[tmp], 48(%[c]) \n" + "vsetvli zero, %[w], e16, m4, ta, ma \n" + "vmv.v.x v8, %[tmp] \n" + "lbu %[vl], 0(%[c]) \n" + "lbu %[tmp], 1(%[c]) \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vmv.v.x v12, %[vl] \n" + "vmv.v.x v14, %[tmp] \n" + "lbu %[vl], 2(%[c]) \n" + "lbu %[tmp], 3(%[c]) \n" + "vmv.v.x v16, %[vl] \n" + "vmv.v.x v18, %[tmp] \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v24, (%[src_argb]) \n" + "vwmulu.vv v20, v24, v12 \n" + "vwmaccu.vv v20, v14, v26 \n" + "vwmaccu.vv v20, v16, v28 \n" + "vwmaccu.vv v20, v18, v30 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vv v20, v20, v8 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v24, v20, 8 \n" + "vse8.v v24, (%[dst_y]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_y], %[dst_y], %[vl] \n" + "slli %[tmp], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[tmp]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_y] "+r"(dst_y), // %[dst_y] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : [c] "r"(c) // %[c] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29", "v30", "v31"); } #endif @@ -1419,32 +1770,45 @@ void RGBToYMatrixRow_RVV(const uint8_t* src_rgb, int width, const struct ArgbConstants* c) { assert(width != 0); - size_t w = (size_t)width; - vuint8m2_t v_by, v_gy, v_ry; // vectors are to store RGBToY constant - vuint16m4_t v_addy; // vector is to store kAddY - size_t vl = __riscv_vsetvl_e8m2(w); - v_by = __riscv_vmv_v_x_u8m2(c->kRGBToY[0], vl); - v_gy = __riscv_vmv_v_x_u8m2(c->kRGBToY[1], vl); - v_ry = __riscv_vmv_v_x_u8m2(c->kRGBToY[2], vl); - v_addy = __riscv_vmv_v_x_u16m4(c->kAddY[0], vl); - do { - vuint8m2_t v_y; - vuint16m4_t v_y_u16; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x3_t v_src_rgb = __riscv_vlseg3e8_v_u8m2x3(src_rgb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 2); - v_y_u16 = __riscv_vwmulu_vv_u16m4(v_r, v_ry, vl); - v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_gy, v_g, vl); - v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_by, v_b, vl); - v_y_u16 = __riscv_vadd_vv_u16m4(v_y_u16, v_addy, vl); - v_y = __riscv_vnsrl_wx_u8m2(v_y_u16, 8, vl); - __riscv_vse8_v_u8m2(dst_y, v_y, vl); - w -= vl; - src_rgb += 3 * vl; - dst_y += vl; - } while (w > 0); + size_t vl, tmp; + asm volatile( + "lhu %[tmp], 48(%[c]) \n" + "vsetvli zero, %[w], e16, m4, ta, ma \n" + "vmv.v.x v8, %[tmp] \n" + "lbu %[vl], 0(%[c]) \n" + "lbu %[tmp], 1(%[c]) \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vmv.v.x v12, %[vl] \n" + "vmv.v.x v14, %[tmp] \n" + "lbu %[vl], 2(%[c]) \n" + "vmv.v.x v16, %[vl] \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg3e8.v v24, (%[src_rgb]) \n" + "vwmulu.vv v20, v28, v16 \n" + "vwmaccu.vv v20, v14, v26 \n" + "vwmaccu.vv v20, v12, v24 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vv v20, v20, v8 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v18, v20, 8 \n" + "vse8.v v18, (%[dst_y]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_y], %[dst_y], %[vl] \n" + "slli %[tmp], %[vl], 1 \n" + "add %[src_rgb], %[src_rgb], %[vl] \n" + "add %[src_rgb], %[src_rgb], %[tmp]\n" + "bgtz %[w], 1b \n" + : [src_rgb] "+r"(src_rgb), // %[src_rgb] + [dst_y] "+r"(dst_y), // %[dst_y] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : [c] "r"(c) // %[c] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29"); } #endif @@ -1455,51 +1819,67 @@ void ARGBToUV444MatrixRow_RVV(const uint8_t* src_argb, int width, const struct ArgbConstants* c) { assert(width != 0); - size_t w = (size_t)width; - vint8m2_t v_u0, v_u1, v_u2, v_u3; - vint8m2_t v_v0, v_v1, v_v2, v_v3; - vint16m4_t v_adduv; - size_t vl = __riscv_vsetvl_e8m2(w); - v_u0 = __riscv_vmv_v_x_i8m2(c->kRGBToU[0], vl); - v_u1 = __riscv_vmv_v_x_i8m2(c->kRGBToU[1], vl); - v_u2 = __riscv_vmv_v_x_i8m2(c->kRGBToU[2], vl); - v_u3 = __riscv_vmv_v_x_i8m2(c->kRGBToU[3], vl); - v_v0 = __riscv_vmv_v_x_i8m2(c->kRGBToV[0], vl); - v_v1 = __riscv_vmv_v_x_i8m2(c->kRGBToV[1], vl); - v_v2 = __riscv_vmv_v_x_i8m2(c->kRGBToV[2], vl); - v_v3 = __riscv_vmv_v_x_i8m2(c->kRGBToV[3], vl); - v_adduv = __riscv_vmv_v_x_i16m4((int16_t)c->kAddUV[0], vl); - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b0 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); - vuint8m2_t v_b1 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); - vuint8m2_t v_b2 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); - vuint8m2_t v_b3 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3); + size_t vl, tmp; + asm volatile( + "lh %[tmp], 64(%[c]) \n" + "vsetvli zero, %[w], e16, m4, ta, ma \n" + "vmv.v.x v16, %[tmp] \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "lb %[vl], 16(%[c]) \n" + "lb %[tmp], 17(%[c]) \n" + "vmv.v.x v0, %[vl] \n" + "vmv.v.x v2, %[tmp] \n" + "lb %[vl], 18(%[c]) \n" + "lb %[tmp], 19(%[c]) \n" + "vmv.v.x v4, %[vl] \n" + "vmv.v.x v6, %[tmp] \n" + "lb %[vl], 32(%[c]) \n" + "lb %[tmp], 33(%[c]) \n" + "vmv.v.x v8, %[vl] \n" + "vmv.v.x v10, %[tmp] \n" + "lb %[vl], 34(%[c]) \n" + "lb %[tmp], 35(%[c]) \n" + "vmv.v.x v12, %[vl] \n" + "vmv.v.x v14, %[tmp] \n" - vint16m4_t v_u_i16 = __riscv_vwmulsu_vv_i16m4(v_u0, v_b0, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_u1, v_b1, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_u2, v_b2, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_u3, v_b3, vl); - v_u_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_u_i16, vl); - vuint8m2_t v_u = __riscv_vnsrl_wx_u8m2( - __riscv_vreinterpret_v_i16m4_u16m4(v_u_i16), 8, vl); - __riscv_vse8_v_u8m2(dst_u, v_u, vl); - - vint16m4_t v_v_i16 = __riscv_vwmulsu_vv_i16m4(v_v0, v_b0, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_v1, v_b1, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_v2, v_b2, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_v3, v_b3, vl); - v_v_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_v_i16, vl); - vuint8m2_t v_v = __riscv_vnsrl_wx_u8m2( - __riscv_vreinterpret_v_i16m4_u16m4(v_v_i16), 8, vl); - __riscv_vse8_v_u8m2(dst_v, v_v, vl); - - w -= vl; - src_argb += 4 * vl; - dst_u += vl; - dst_v += vl; - } while (w > 0); + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v24, (%[src_argb]) \n" + "vwmulsu.vv v20, v0, v24 \n" + "vwmaccsu.vv v20, v2, v26 \n" + "vwmaccsu.vv v20, v4, v28 \n" + "vwmaccsu.vv v20, v6, v30 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vsub.vv v20, v16, v20 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v20, v20, 8 \n" + "vse8.v v20, (%[dst_u]) \n" + "vwmulsu.vv v20, v8, v24 \n" + "vwmaccsu.vv v20, v10, v26 \n" + "vwmaccsu.vv v20, v12, v28 \n" + "vwmaccsu.vv v20, v14, v30 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vsub.vv v20, v16, v20 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v20, v20, 8 \n" + "vse8.v v20, (%[dst_v]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_u], %[dst_u], %[vl] \n" + "add %[dst_v], %[dst_v], %[vl] \n" + "slli %[tmp], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[tmp]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : [c] "r"(c) // %[c] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30", "v31"); } #endif @@ -1516,90 +1896,98 @@ void ARGBToUVMatrixRow_RVV(const uint8_t* src_argb, int w = width >> 1; if (w > 0) { size_t w_len = (size_t)w; - vint8m1_t v_u0, v_u1, v_u2, v_u3; - vint8m1_t v_v0, v_v1, v_v2, v_v3; - vint16m2_t v_adduv; - size_t vl = __riscv_vsetvl_e8m1(w_len); - v_u0 = __riscv_vmv_v_x_i8m1(c->kRGBToU[0], vl); - v_u1 = __riscv_vmv_v_x_i8m1(c->kRGBToU[1], vl); - v_u2 = __riscv_vmv_v_x_i8m1(c->kRGBToU[2], vl); - v_u3 = __riscv_vmv_v_x_i8m1(c->kRGBToU[3], vl); - v_v0 = __riscv_vmv_v_x_i8m1(c->kRGBToV[0], vl); - v_v1 = __riscv_vmv_v_x_i8m1(c->kRGBToV[1], vl); - v_v2 = __riscv_vmv_v_x_i8m1(c->kRGBToV[2], vl); - v_v3 = __riscv_vmv_v_x_i8m1(c->kRGBToV[3], vl); - v_adduv = __riscv_vmv_v_x_i16m2((int16_t)c->kAddUV[0], vl); - do { - size_t vl = __riscv_vsetvl_e8m1(w_len); - vuint8m1x8_t v_src0 = __riscv_vlseg8e8_v_u8m1x8(src_argb_0, vl); - vuint8m1x8_t v_src1 = __riscv_vlseg8e8_v_u8m1x8(src_argb_1, vl); + size_t vl, tmp; + asm volatile( + "lh %[tmp], 64(%[c]) \n" + "vsetvli zero, %[w_len], e16, m2, ta, ma\n" + "vmv.v.x v8, %[tmp] \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "lb %[vl], 16(%[c]) \n" + "lb %[tmp], 17(%[c]) \n" + "vmv.v.x v10, %[vl] \n" + "vmv.v.x v11, %[tmp] \n" + "lb %[vl], 18(%[c]) \n" + "lb %[tmp], 19(%[c]) \n" + "vmv.v.x v12, %[vl] \n" + "vmv.v.x v13, %[tmp] \n" + "lb %[vl], 32(%[c]) \n" + "lb %[tmp], 33(%[c]) \n" + "vmv.v.x v14, %[vl] \n" + "vmv.v.x v15, %[tmp] \n" + "lb %[vl], 34(%[c]) \n" + "lb %[tmp], 35(%[c]) \n" + "vmv.v.x v16, %[vl] \n" + "vmv.v.x v17, %[tmp] \n" - vuint8m1_t b0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 0); - vuint8m1_t g0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 1); - vuint8m1_t r0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 2); - vuint8m1_t a0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 3); - vuint8m1_t b1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 4); - vuint8m1_t g1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 5); - vuint8m1_t r1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 6); - vuint8m1_t a1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 7); - - vuint8m1_t b0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 0); - vuint8m1_t g0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 1); - vuint8m1_t r0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 2); - vuint8m1_t a0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 3); - vuint8m1_t b1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 4); - vuint8m1_t g1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 5); - vuint8m1_t r1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 6); - vuint8m1_t a1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 7); - - vuint16m2_t sum_b = __riscv_vwaddu_vv_u16m2(b0, b1, vl); - sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b0_1, vl); - sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b1_1, vl); - sum_b = __riscv_vadd_vx_u16m2(sum_b, 2, vl); - vuint8m1_t b = __riscv_vnsrl_wx_u8m1(sum_b, 2, vl); - - vuint16m2_t sum_g = __riscv_vwaddu_vv_u16m2(g0, g1, vl); - sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g0_1, vl); - sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g1_1, vl); - sum_g = __riscv_vadd_vx_u16m2(sum_g, 2, vl); - vuint8m1_t g = __riscv_vnsrl_wx_u8m1(sum_g, 2, vl); - - vuint16m2_t sum_r = __riscv_vwaddu_vv_u16m2(r0, r1, vl); - sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r0_1, vl); - sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r1_1, vl); - sum_r = __riscv_vadd_vx_u16m2(sum_r, 2, vl); - vuint8m1_t r = __riscv_vnsrl_wx_u8m1(sum_r, 2, vl); - - vuint16m2_t sum_a = __riscv_vwaddu_vv_u16m2(a0, a1, vl); - sum_a = __riscv_vwaddu_wv_u16m2(sum_a, a0_1, vl); - sum_a = __riscv_vwaddu_wv_u16m2(sum_a, a1_1, vl); - sum_a = __riscv_vadd_vx_u16m2(sum_a, 2, vl); - vuint8m1_t a = __riscv_vnsrl_wx_u8m1(sum_a, 2, vl); - - vint16m2_t v_u_i16 = __riscv_vwmulsu_vv_i16m2(v_u0, b, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_u1, g, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_u2, r, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_u3, a, vl); - v_u_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_u_i16, vl); - vuint8m1_t v_u = __riscv_vnsrl_wx_u8m1( - __riscv_vreinterpret_v_i16m2_u16m2(v_u_i16), 8, vl); - __riscv_vse8_v_u8m1(dst_u, v_u, vl); - - vint16m2_t v_v_i16 = __riscv_vwmulsu_vv_i16m2(v_v0, b, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_v1, g, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_v2, r, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_v3, a, vl); - v_v_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_v_i16, vl); - vuint8m1_t v_v = __riscv_vnsrl_wx_u8m1( - __riscv_vreinterpret_v_i16m2_u16m2(v_v_i16), 8, vl); - __riscv_vse8_v_u8m1(dst_v, v_v, vl); - - w_len -= vl; - src_argb_0 += 8 * vl; - src_argb_1 += 8 * vl; - dst_u += vl; - dst_v += vl; - } while (w_len > 0); + "1: \n" + "vsetvli %[vl], %[w_len], e8, m1, ta, ma\n" + "vlseg8e8.v v20, (%[src_argb_0]) \n" + "vlseg8e8.v v0, (%[src_argb_1]) \n" + "vwaddu.vv v18, v20, v24 \n" + "vwaddu.wv v18, v18, v0 \n" + "vwaddu.wv v18, v18, v4 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vi v18, v18, 2 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v20, v18, 2 \n" + "vwaddu.vv v18, v21, v25 \n" + "vwaddu.wv v18, v18, v1 \n" + "vwaddu.wv v18, v18, v5 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vi v18, v18, 2 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v21, v18, 2 \n" + "vwaddu.vv v18, v22, v26 \n" + "vwaddu.wv v18, v18, v2 \n" + "vwaddu.wv v18, v18, v6 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vi v18, v18, 2 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v22, v18, 2 \n" + "vwaddu.vv v18, v23, v27 \n" + "vwaddu.wv v18, v18, v3 \n" + "vwaddu.wv v18, v18, v7 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vi v18, v18, 2 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v23, v18, 2 \n" + "vwmulsu.vv v18, v10, v20 \n" + "vwmaccsu.vv v18, v11, v21 \n" + "vwmaccsu.vv v18, v12, v22 \n" + "vwmaccsu.vv v18, v13, v23 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vsub.vv v18, v8, v18 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v24, v18, 8 \n" + "vse8.v v24, (%[dst_u]) \n" + "vwmulsu.vv v18, v14, v20 \n" + "vwmaccsu.vv v18, v15, v21 \n" + "vwmaccsu.vv v18, v16, v22 \n" + "vwmaccsu.vv v18, v17, v23 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vsub.vv v18, v8, v18 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v20, v18, 8 \n" + "vse8.v v20, (%[dst_v]) \n" + "sub %[w_len], %[w_len], %[vl] \n" + "add %[dst_u], %[dst_u], %[vl] \n" + "add %[dst_v], %[dst_v], %[vl] \n" + "slli %[tmp], %[vl], 3 \n" + "add %[src_argb_0], %[src_argb_0], %[tmp]\n" + "add %[src_argb_1], %[src_argb_1], %[tmp]\n" + "bgtz %[w_len], 1b \n" + : [src_argb_0] "+r"(src_argb_0), // %[src_argb_0] + [src_argb_1] "+r"(src_argb_1), // %[src_argb_1] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [w_len] "+r"(w_len), // %[w_len] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : [c] "r"(c) // %[c] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", + "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", + "v27"); } if (width & 1) { uint8_t b = (src_argb_0[0] + src_argb_1[0] + 1) >> 1; @@ -1623,46 +2011,61 @@ void RGBToUV444MatrixRow_RVV(const uint8_t* src_rgb, int width, const struct ArgbConstants* c) { assert(width != 0); - size_t w = (size_t)width; - vint8m2_t v_bu, v_gu, v_ru; - vint8m2_t v_bv, v_gv, v_rv; - vint16m4_t v_adduv; - size_t vl = __riscv_vsetvl_e8m2(w); - v_bu = __riscv_vmv_v_x_i8m2(c->kRGBToU[0], vl); - v_gu = __riscv_vmv_v_x_i8m2(c->kRGBToU[1], vl); - v_ru = __riscv_vmv_v_x_i8m2(c->kRGBToU[2], vl); - v_bv = __riscv_vmv_v_x_i8m2(c->kRGBToV[0], vl); - v_gv = __riscv_vmv_v_x_i8m2(c->kRGBToV[1], vl); - v_rv = __riscv_vmv_v_x_i8m2(c->kRGBToV[2], vl); - v_adduv = __riscv_vmv_v_x_i16m4((int16_t)c->kAddUV[0], vl); - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x3_t v_src_rgb = __riscv_vlseg3e8_v_u8m2x3(src_rgb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 2); + size_t vl, tmp; + asm volatile( + "lh %[tmp], 64(%[c]) \n" + "vsetvli zero, %[w], e16, m4, ta, ma \n" + "vmv.v.x v16, %[tmp] \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "lb %[vl], 16(%[c]) \n" + "lb %[tmp], 17(%[c]) \n" + "vmv.v.x v0, %[vl] \n" + "vmv.v.x v2, %[tmp] \n" + "lb %[vl], 18(%[c]) \n" + "vmv.v.x v4, %[vl] \n" + "lb %[vl], 32(%[c]) \n" + "lb %[tmp], 33(%[c]) \n" + "vmv.v.x v8, %[vl] \n" + "vmv.v.x v10, %[tmp] \n" + "lb %[vl], 34(%[c]) \n" + "vmv.v.x v12, %[vl] \n" - vint16m4_t v_u_i16 = __riscv_vwmulsu_vv_i16m4(v_ru, v_r, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_gu, v_g, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_bu, v_b, vl); - v_u_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_u_i16, vl); - vuint8m2_t v_u = __riscv_vnsrl_wx_u8m2( - __riscv_vreinterpret_v_i16m4_u16m4(v_u_i16), 8, vl); - __riscv_vse8_v_u8m2(dst_u, v_u, vl); - - vint16m4_t v_v_i16 = __riscv_vwmulsu_vv_i16m4(v_rv, v_r, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_gv, v_g, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_bv, v_b, vl); - v_v_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_v_i16, vl); - vuint8m2_t v_v = __riscv_vnsrl_wx_u8m2( - __riscv_vreinterpret_v_i16m4_u16m4(v_v_i16), 8, vl); - __riscv_vse8_v_u8m2(dst_v, v_v, vl); - - w -= vl; - src_rgb += 3 * vl; - dst_u += vl; - dst_v += vl; - } while (w > 0); + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg3e8.v v24, (%[src_rgb]) \n" + "vwmulsu.vv v20, v4, v28 \n" + "vwmaccsu.vv v20, v2, v26 \n" + "vwmaccsu.vv v20, v0, v24 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vsub.vv v20, v16, v20 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v20, v20, 8 \n" + "vse8.v v20, (%[dst_u]) \n" + "vwmulsu.vv v20, v12, v28 \n" + "vwmaccsu.vv v20, v10, v26 \n" + "vwmaccsu.vv v20, v8, v24 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vsub.vv v20, v16, v20 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v20, v20, 8 \n" + "vse8.v v20, (%[dst_v]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_u], %[dst_u], %[vl] \n" + "add %[dst_v], %[dst_v], %[vl] \n" + "slli %[tmp], %[vl], 1 \n" + "add %[src_rgb], %[src_rgb], %[vl] \n" + "add %[src_rgb], %[src_rgb], %[tmp]\n" + "bgtz %[w], 1b \n" + : [src_rgb] "+r"(src_rgb), // %[src_rgb] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : [c] "r"(c) // %[c] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v8", "v9", + "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29"); } #endif @@ -1679,86 +2082,96 @@ void RGBToUVMatrixRow_RVV(const uint8_t* src_rgb, int w = width >> 1; if (w > 0) { size_t w_len = (size_t)w; - vint8m1_t v_bu, v_gu, v_ru; - vint8m1_t v_bv, v_gv, v_rv; - vint16m2_t v_adduv; - size_t vl = __riscv_vsetvl_e8m1(w_len); - v_bu = __riscv_vmv_v_x_i8m1(c->kRGBToU[0], vl); - v_gu = __riscv_vmv_v_x_i8m1(c->kRGBToU[1], vl); - v_ru = __riscv_vmv_v_x_i8m1(c->kRGBToU[2], vl); - v_bv = __riscv_vmv_v_x_i8m1(c->kRGBToV[0], vl); - v_gv = __riscv_vmv_v_x_i8m1(c->kRGBToV[1], vl); - v_rv = __riscv_vmv_v_x_i8m1(c->kRGBToV[2], vl); - v_adduv = __riscv_vmv_v_x_i16m2((int16_t)c->kAddUV[0], vl); - do { - size_t vl = __riscv_vsetvl_e8m1(w_len); - vuint8m1x6_t v_src0 = __riscv_vlseg6e8_v_u8m1x6(src_rgb_0, vl); - vuint8m1x6_t v_src1 = __riscv_vlseg6e8_v_u8m1x6(src_rgb_1, vl); + size_t vl, tmp; + asm volatile( + "lh %[tmp], 64(%[c]) \n" + "vsetvli zero, %[w_len], e16, m2, ta, ma\n" + "vmv.v.x v8, %[tmp] \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "lb %[vl], 16(%[c]) \n" + "lb %[tmp], 17(%[c]) \n" + "vmv.v.x v10, %[vl] \n" + "vmv.v.x v11, %[tmp] \n" + "lb %[vl], 18(%[c]) \n" + "vmv.v.x v12, %[vl] \n" + "lb %[vl], 32(%[c]) \n" + "lb %[tmp], 33(%[c]) \n" + "vmv.v.x v14, %[vl] \n" + "vmv.v.x v15, %[tmp] \n" + "lb %[vl], 34(%[c]) \n" + "vmv.v.x v16, %[vl] \n" - vuint8m1_t b0 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 0); - vuint8m1_t g0 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 1); - vuint8m1_t r0 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 2); - vuint8m1_t b1 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 3); - vuint8m1_t g1 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 4); - vuint8m1_t r1 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 5); - - vuint8m1_t b0_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 0); - vuint8m1_t g0_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 1); - vuint8m1_t r0_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 2); - vuint8m1_t b1_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 3); - vuint8m1_t g1_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 4); - vuint8m1_t r1_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 5); - - vuint16m2_t sum_b = __riscv_vwaddu_vv_u16m2(b0, b1, vl); - sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b0_1, vl); - sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b1_1, vl); - sum_b = __riscv_vadd_vx_u16m2(sum_b, 2, vl); - vuint8m1_t b = __riscv_vnsrl_wx_u8m1(sum_b, 2, vl); - - vuint16m2_t sum_g = __riscv_vwaddu_vv_u16m2(g0, g1, vl); - sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g0_1, vl); - sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g1_1, vl); - sum_g = __riscv_vadd_vx_u16m2(sum_g, 2, vl); - vuint8m1_t g = __riscv_vnsrl_wx_u8m1(sum_g, 2, vl); - - vuint16m2_t sum_r = __riscv_vwaddu_vv_u16m2(r0, r1, vl); - sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r0_1, vl); - sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r1_1, vl); - sum_r = __riscv_vadd_vx_u16m2(sum_r, 2, vl); - vuint8m1_t r = __riscv_vnsrl_wx_u8m1(sum_r, 2, vl); - - vint16m2_t v_u_i16 = __riscv_vwmulsu_vv_i16m2(v_ru, r, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_gu, g, vl); - v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_bu, b, vl); - v_u_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_u_i16, vl); - vuint8m1_t v_u = __riscv_vnsrl_wx_u8m1( - __riscv_vreinterpret_v_i16m2_u16m2(v_u_i16), 8, vl); - __riscv_vse8_v_u8m1(dst_u, v_u, vl); - - vint16m2_t v_v_i16 = __riscv_vwmulsu_vv_i16m2(v_rv, r, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_gv, g, vl); - v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_bv, b, vl); - v_v_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_v_i16, vl); - vuint8m1_t v_v = __riscv_vnsrl_wx_u8m1( - __riscv_vreinterpret_v_i16m2_u16m2(v_v_i16), 8, vl); - __riscv_vse8_v_u8m1(dst_v, v_v, vl); - - w_len -= vl; - src_rgb_0 += 6 * vl; - src_rgb_1 += 6 * vl; - dst_u += vl; - dst_v += vl; - } while (w_len > 0); + "1: \n" + "vsetvli %[vl], %[w_len], e8, m1, ta, ma\n" + "vlseg6e8.v v20, (%[src_rgb_0]) \n" + "vlseg6e8.v v0, (%[src_rgb_1]) \n" + "vwaddu.vv v18, v20, v23 \n" + "vwaddu.wv v18, v18, v0 \n" + "vwaddu.wv v18, v18, v3 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vi v18, v18, 2 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v20, v18, 2 \n" + "vwaddu.vv v18, v21, v24 \n" + "vwaddu.wv v18, v18, v1 \n" + "vwaddu.wv v18, v18, v4 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vi v18, v18, 2 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v21, v18, 2 \n" + "vwaddu.vv v18, v22, v25 \n" + "vwaddu.wv v18, v18, v2 \n" + "vwaddu.wv v18, v18, v5 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vi v18, v18, 2 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v22, v18, 2 \n" + "vwmulsu.vv v18, v12, v22 \n" + "vwmaccsu.vv v18, v11, v21 \n" + "vwmaccsu.vv v18, v10, v20 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vsub.vv v18, v8, v18 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v24, v18, 8 \n" + "vse8.v v24, (%[dst_u]) \n" + "vwmulsu.vv v18, v16, v22 \n" + "vwmaccsu.vv v18, v15, v21 \n" + "vwmaccsu.vv v18, v14, v20 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vsub.vv v18, v8, v18 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v20, v18, 8 \n" + "vse8.v v20, (%[dst_v]) \n" + "sub %[w_len], %[w_len], %[vl] \n" + "add %[dst_u], %[dst_u], %[vl] \n" + "add %[dst_v], %[dst_v], %[vl] \n" + "slli %[tmp], %[vl], 1 \n" + "add %[tmp], %[tmp], %[vl] \n" + "slli %[tmp], %[tmp], 1 \n" + "add %[src_rgb_0], %[src_rgb_0], %[tmp]\n" + "add %[src_rgb_1], %[src_rgb_1], %[tmp]\n" + "bgtz %[w_len], 1b \n" + : [src_rgb_0] "+r"(src_rgb_0), // %[src_rgb_0] + [src_rgb_1] "+r"(src_rgb_1), // %[src_rgb_1] + [dst_u] "+r"(dst_u), // %[dst_u] + [dst_v] "+r"(dst_v), // %[dst_v] + [w_len] "+r"(w_len), // %[w_len] + [vl] "=&r"(vl), // %[vl] + [tmp] "=&r"(tmp) // %[tmp] + : [c] "r"(c) // %[c] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v8", + "v9", "v10", "v11", "v12", "v14", "v15", "v16", "v18", "v19", "v20", + "v21", "v22", "v23", "v24", "v25"); } if (width & 1) { uint8_t b = (src_rgb_0[0] + src_rgb_1[0] + 1) >> 1; uint8_t g = (src_rgb_0[1] + src_rgb_1[1] + 1) >> 1; uint8_t r = (src_rgb_0[2] + src_rgb_1[2] + 1) >> 1; - dst_u[0] = (c->kAddUV[0] - (c->kRGBToU[0] * b + c->kRGBToU[1] * g + - c->kRGBToU[2] * r)) >> + dst_u[0] = (c->kAddUV[0] - + (c->kRGBToU[0] * b + c->kRGBToU[1] * g + c->kRGBToU[2] * r)) >> 8; - dst_v[0] = (c->kAddUV[0] - (c->kRGBToV[0] * b + c->kRGBToV[1] * g + - c->kRGBToV[2] * r)) >> + dst_v[0] = (c->kAddUV[0] - + (c->kRGBToV[0] * b + c->kRGBToV[1] * g + c->kRGBToV[2] * r)) >> 8; } } @@ -1772,47 +2185,40 @@ void ARGBBlendRow_RVV(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, int width) { - size_t w = (size_t)width; - size_t vl = __riscv_vsetvlmax_e8m2(); - // clamp255((((256 - a) * b) >> 8) + f) - // = b * (256 - a) / 256 + f - // = b - (b * a / 256) + f - vuint8m2_t v_255 = __riscv_vmv_v_x_u8m2(255, vl); - do { - vuint8m2_t v_tmp_b, v_tmp_g, v_tmp_r; - vuint8m2_t v_dst_b, v_dst_g, v_dst_r; - vuint8m2x4_t v_dst_argb; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src0_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_src0_b = __riscv_vget_v_u8m2x4_u8m2(v_src0_argb, 0); - vuint8m2_t v_src0_g = __riscv_vget_v_u8m2x4_u8m2(v_src0_argb, 1); - vuint8m2_t v_src0_r = __riscv_vget_v_u8m2x4_u8m2(v_src0_argb, 2); - vuint8m2_t v_src0_a = __riscv_vget_v_u8m2x4_u8m2(v_src0_argb, 3); - vuint8m2x4_t v_src1_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb1, vl); - vuint8m2_t v_src1_b = __riscv_vget_v_u8m2x4_u8m2(v_src1_argb, 0); - vuint8m2_t v_src1_g = __riscv_vget_v_u8m2x4_u8m2(v_src1_argb, 1); - vuint8m2_t v_src1_r = __riscv_vget_v_u8m2x4_u8m2(v_src1_argb, 2); + size_t vl; + asm volatile( + "vsetvli %[vl], zero, e8, m2, ta, ma \n" + "vmv.v.i v14, -1 \n" - v_tmp_b = __riscv_vmulhu_vv_u8m2(v_src1_b, v_src0_a, vl); - v_tmp_g = __riscv_vmulhu_vv_u8m2(v_src1_g, v_src0_a, vl); - v_tmp_r = __riscv_vmulhu_vv_u8m2(v_src1_r, v_src0_a, vl); - - v_dst_b = __riscv_vsub_vv_u8m2(v_src1_b, v_tmp_b, vl); - v_dst_g = __riscv_vsub_vv_u8m2(v_src1_g, v_tmp_g, vl); - v_dst_r = __riscv_vsub_vv_u8m2(v_src1_r, v_tmp_r, vl); - - v_dst_b = __riscv_vsaddu_vv_u8m2(v_dst_b, v_src0_b, vl); - v_dst_g = __riscv_vsaddu_vv_u8m2(v_dst_g, v_src0_g, vl); - v_dst_r = __riscv_vsaddu_vv_u8m2(v_dst_r, v_src0_r, vl); - - v_dst_argb = __riscv_vcreate_v_u8m2x4(v_dst_b, v_dst_g, v_dst_r, v_255); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - - w -= vl; - src_argb += 4 * vl; - src_argb1 += 4 * vl; - dst_argb += 4 * vl; - } while (w > 0); + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v16, (%[src_argb]) \n" + "vlseg4e8.v v24, (%[src_argb1]) \n" + "vmulhu.vv v8, v24, v22 \n" + "vmulhu.vv v10, v26, v22 \n" + "vmulhu.vv v12, v28, v22 \n" + "vsub.vv v8, v24, v8 \n" + "vsub.vv v10, v26, v10 \n" + "vsub.vv v12, v28, v12 \n" + "vsaddu.vv v8, v8, v16 \n" + "vsaddu.vv v10, v10, v18 \n" + "vsaddu.vv v12, v12, v20 \n" + "vsseg4e8.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "add %[src_argb1], %[src_argb1], %[vl]\n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [src_argb1] "+r"(src_argb1), // %[src_argb1] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29", "v30", "v31"); } #endif @@ -1822,30 +2228,36 @@ void BlendPlaneRow_RVV(const uint8_t* src0, const uint8_t* alpha, uint8_t* dst, int width) { - size_t w = (size_t)width; - do { - vuint16m8_t v_dst_u16; - vuint8m4_t v_dst; - size_t vl = __riscv_vsetvl_e8m4(w); - vuint8m4_t v_src0 = __riscv_vle8_v_u8m4(src0, vl); - vuint8m4_t v_src1 = __riscv_vle8_v_u8m4(src1, vl); - vuint8m4_t v_alpha = __riscv_vle8_v_u8m4(alpha, vl); - vuint8m4_t v_255_minus_alpha = __riscv_vrsub_vx_u8m4(v_alpha, 255u, vl); - - // (a * foreground) + (1-a) * background - v_dst_u16 = __riscv_vwmulu_vv_u16m8(v_alpha, v_src0, vl); - v_dst_u16 = - __riscv_vwmaccu_vv_u16m8(v_dst_u16, v_255_minus_alpha, v_src1, vl); - v_dst_u16 = __riscv_vadd_vx_u16m8(v_dst_u16, 255u, vl); - v_dst = __riscv_vnsrl_wx_u8m4(v_dst_u16, 8, vl); - - __riscv_vse8_v_u8m4(dst, v_dst, vl); - w -= vl; - src0 += vl; - src1 += vl; - alpha += vl; - dst += vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle8.v v16, (%[src0]) \n" + "vle8.v v18, (%[src1]) \n" + "vle8.v v20, (%[alpha]) \n" + "vrsub.vx v22, v20, %[k255] \n" + "vwmulu.vv v8, v20, v16 \n" + "vwmaccu.vv v8, v22, v18 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vx v8, v8, %[k255] \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v16, v8, 8 \n" + "vse8.v v16, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src0], %[src0], %[vl] \n" + "add %[src1], %[src1], %[vl] \n" + "add %[alpha], %[alpha], %[vl] \n" + "add %[dst], %[dst], %[vl] \n" + "bgtz %[w], 1b \n" + : [src0] "+r"(src0), // %[src0] + [src1] "+r"(src1), // %[src1] + [alpha] "+r"(alpha), // %[alpha] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [k255] "r"(255) + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23"); } #endif @@ -1854,35 +2266,36 @@ void BlendPlaneRow_RVV(const uint8_t* src0, void ARGBAttenuateRow_RVV(const uint8_t* src_argb, uint8_t* dst_argb, int width) { - size_t w = (size_t)width; - do { - vuint16m4_t v_ba_16, v_ga_16, v_ra_16; - vuint8m2x4_t v_dst_argb; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); - vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3); - // f * a - v_ba_16 = __riscv_vwmulu_vv_u16m4(v_b, v_a, vl); - v_ga_16 = __riscv_vwmulu_vv_u16m4(v_g, v_a, vl); - v_ra_16 = __riscv_vwmulu_vv_u16m4(v_r, v_a, vl); - // f * a + 255 - v_ba_16 = __riscv_vadd_vx_u16m4(v_ba_16, 255u, vl); - v_ga_16 = __riscv_vadd_vx_u16m4(v_ga_16, 255u, vl); - v_ra_16 = __riscv_vadd_vx_u16m4(v_ra_16, 255u, vl); - // (f * a + 255) >> 8 - v_b = __riscv_vnsrl_wx_u8m2(v_ba_16, 8, vl); - v_g = __riscv_vnsrl_wx_u8m2(v_ga_16, 8, vl); - v_r = __riscv_vnsrl_wx_u8m2(v_ra_16, 8, vl); - - v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a); - __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl); - w -= vl; - src_argb += vl * 4; - dst_argb += vl * 4; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v8, (%[src_argb]) \n" + "vwmulu.vv v16, v8, v14 \n" + "vwmulu.vv v20, v10, v14 \n" + "vwmulu.vv v24, v12, v14 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vx v16, v16, %[k255] \n" + "vadd.vx v20, v20, %[k255] \n" + "vadd.vx v24, v24, %[k255] \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v8, v16, 8 \n" + "vnsrl.wi v10, v20, 8 \n" + "vnsrl.wi v12, v24, 8 \n" + "vsseg4e8.v v8, (%[dst_argb]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [k255] "r"(255) + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27"); } #endif @@ -1893,7 +2306,6 @@ void ARGBMultiplyRow_RVV(const uint8_t* src_argb, int width) { assert(width != 0); size_t w = (size_t)width * 4; -#ifdef RVV_ASM #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wvla" size_t vl; @@ -1922,21 +2334,6 @@ void ARGBMultiplyRow_RVV(const uint8_t* src_argb, "m"(*(const uint8_t (*)[w])src_argb1) : "vl", "vtype", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"); #pragma GCC diagnostic pop -#else - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_src0 = __riscv_vle8_v_u8m2(src_argb, vl); - vuint8m2_t v_src1 = __riscv_vle8_v_u8m2(src_argb1, vl); - vuint16m4_t v_mul = __riscv_vwmulu_vv_u16m4(v_src0, v_src1, vl); - v_mul = __riscv_vwaddu_wx_u16m4(v_mul, 128, vl); - vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_mul, 8, vl); - __riscv_vse8_v_u8m2(dst_argb, v_dst, vl); - w -= vl; - src_argb += vl; - src_argb1 += vl; - dst_argb += vl; - } while (w > 0); -#endif } #endif @@ -1944,32 +2341,50 @@ void ARGBMultiplyRow_RVV(const uint8_t* src_argb, void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb, uint8_t* dst_a, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3); - __riscv_vse8_v_u8m2(dst_a, v_a, vl); - w -= vl; - src_argb += vl * 4; - dst_a += vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e16, m2, ta, ma \n" + "vle32.v v8, (%[src_argb]) \n" + "vnsrl.wi v16, v8, 16 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v8, v16, 8 \n" + "vse8.v v8, (%[dst_a]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_a], %[dst_a], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_argb], %[src_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [src_argb] "+r"(src_argb), // %[src_argb] + [dst_a] "+r"(dst_a), // %[dst_a] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } #endif #ifdef HAS_ARGBCOPYYTOALPHAROW_RVV void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width) { - size_t w = (size_t)width; - const ptrdiff_t dst_stride = 4; - dst += 3; - do { - size_t vl = __riscv_vsetvl_e8m8(w); - vuint8m8_t v_a = __riscv_vle8_v_u8m8(src, vl); - __riscv_vsse8_v_u8m8(dst, dst_stride, v_a, vl); - w -= vl; - src += vl; - dst += vl * dst_stride; - } while (w > 0); + int vl; + asm volatile( + "addi %[dst], %[dst], 3 \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m4, ta, ma \n" + "vle8.v v8, (%[src]) \n" + "vsse8.v v8, (%[dst]), %[dst_stride] \n" + "sub %[w], %[w], %[vl] \n" + "add %[src], %[src], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst], %[dst], %[vl] \n" + "bgtz %[w], 1b \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [dst_stride] "r"(4) // %[dst_stride] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11"); } #endif @@ -1978,17 +2393,24 @@ void Convert16To8Row_RVV(const uint16_t* src_y, uint8_t* dst_y, int scale, int width) { - size_t w = (size_t)width; - const int shift = __builtin_clz((int32_t)scale) - 15; - do { - size_t vl = __riscv_vsetvl_e16m4(w); - vuint16m4_t v_src = __riscv_vle16_v_u16m4(src_y, vl); - vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_src, shift, vl); - __riscv_vse8_v_u8m2(dst_y, v_dst, vl); - w -= vl; - src_y += vl; - dst_y += vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle16.v v8, (%[src_y]) \n" + "vnsrl.wx v16, v8, %[shift] \n" + "vse8.v v16, (%[dst_y]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_y], %[dst_y], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_y], %[src_y], %[vl] \n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [dst_y] "+r"(dst_y), // %[dst_y] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [shift] "r"(__builtin_clz((int32_t)scale) - 15) // %[shift] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } #endif @@ -2002,19 +2424,27 @@ void Convert8To16Row_RVV(const uint8_t* src_y, uint16_t* dst_y, int bits, int width) { - size_t w = (size_t)width; - const int shift = 16 - bits; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_src = __riscv_vle8_v_u8m2(src_y, vl); - vuint16m4_t v_dst = __riscv_vwaddu_vx_u16m4(v_src, 0, vl); - v_dst = __riscv_vmul_vx_u16m4(v_dst, 0x0101, vl); - v_dst = __riscv_vsrl_vx_u16m4(v_dst, shift, vl); - __riscv_vse16_v_u16m4(dst_y, v_dst, vl); - w -= vl; - src_y += vl; - dst_y += vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle8.v v16, (%[src_y]) \n" + "vwcvtu.x.x.v v8, v16 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vmul.vx v8, v8, %[k0101] \n" + "vsrl.vx v8, v8, %[shift] \n" + "vse16.v v8, (%[dst_y]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_y], %[src_y], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_y], %[dst_y], %[vl] \n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [dst_y] "+r"(dst_y), // %[dst_y] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [shift] "r"(16 - bits), [k0101] "r"(0x0101) + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } #endif @@ -2023,16 +2453,24 @@ void MultiplyRow_16_RVV(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { - size_t w = (size_t)width; - do { - size_t vl = __riscv_vsetvl_e16m8(w); - vuint16m8_t v_src = __riscv_vle16_v_u16m8(src_y, vl); - vuint16m8_t v_dst = __riscv_vmul_vx_u16m8(v_src, (uint16_t)scale, vl); - __riscv_vse16_v_u16m8(dst_y, v_dst, vl); - w -= vl; - src_y += vl; - dst_y += vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e16, m4, ta, ma \n" + "vle16.v v8, (%[src_y]) \n" + "vmul.vx v8, v8, %[scale] \n" + "vse16.v v8, (%[dst_y]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_y], %[src_y], %[vl] \n" + "add %[dst_y], %[dst_y], %[vl] \n" + "bgtz %[w], 1b \n" + : [src_y] "+r"(src_y), // %[src_y] + [dst_y] "+r"(dst_y), // %[dst_y] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [scale] "r"(scale) // %[scale] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11"); } #endif @@ -2042,93 +2480,77 @@ void HalfRow_16To8_RVV(const uint16_t* src_uv, uint8_t* dst_uv, int scale, int width) { - size_t w = (size_t)width; - const uint16_t* src_uv1 = src_uv + src_uv_stride; - const int shift = __builtin_clz((int32_t)scale) - 15; -#ifndef LIBYUV_RVV_HAS_VXRM_ARG - asm volatile("csrwi vxrm, 0"); -#endif - do { - size_t vl = __riscv_vsetvl_e16m4(w); - vuint16m4_t v_src0 = __riscv_vle16_v_u16m4(src_uv, vl); - vuint16m4_t v_src1 = __riscv_vle16_v_u16m4(src_uv1, vl); -#ifdef LIBYUV_RVV_HAS_VXRM_ARG - vuint16m4_t v_avg = - __riscv_vaaddu_vv_u16m4(v_src0, v_src1, __RISCV_VXRM_RNU, vl); -#else - vuint16m4_t v_avg = __riscv_vaaddu_vv_u16m4(v_src0, v_src1, vl); -#endif - vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_avg, shift, vl); - __riscv_vse8_v_u8m2(dst_uv, v_dst, vl); - w -= vl; - src_uv += vl; - src_uv1 += vl; - dst_uv += vl; - } while (w > 0); + int vl; + asm volatile( + "slli %[src_uv1], %[src_uv1], 1 \n" + "add %[src_uv1], %[src_uv], %[src_uv1]\n" + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[w], e16, m4, ta, ma \n" + "vle16.v v8, (%[src_uv]) \n" + "vle16.v v12, (%[src_uv1]) \n" + "vaaddu.vv v8, v8, v12 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wx v16, v8, %[shift] \n" + "vse8.v v16, (%[dst_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_uv], %[src_uv], %[vl] \n" + "add %[src_uv1], %[src_uv1], %[vl] \n" + "bgtz %[w], 1b \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [src_uv1] "+r"(src_uv_stride), // %[src_uv1] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [shift] "r"(__builtin_clz((int32_t)scale) - 15) // %[shift] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17"); } #endif #ifdef HAS_HALFWIDTHROW_16TO8_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void HalfWidthRow_16To8_RVV(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, int scale, int width) { - const uint16_t* s = src_uv; - const uint16_t* t = src_uv + src_uv_stride; - size_t w = (size_t)width; - const int shift = __builtin_clz((int32_t)scale) - 15; - do { - size_t vl = __riscv_vsetvl_e16m4(w); - vuint16m4x2_t v_s = __riscv_vlseg2e16_v_u16m4x2(s, vl); - vuint16m4x2_t v_t = __riscv_vlseg2e16_v_u16m4x2(t, vl); - vuint16m4_t v_s0 = __riscv_vget_v_u16m4x2_u16m4(v_s, 0); - vuint16m4_t v_s1 = __riscv_vget_v_u16m4x2_u16m4(v_s, 1); - vuint16m4_t v_t0 = __riscv_vget_v_u16m4x2_u16m4(v_t, 0); - vuint16m4_t v_t1 = __riscv_vget_v_u16m4x2_u16m4(v_t, 1); - vuint32m8_t v_s01 = __riscv_vwaddu_vv_u32m8(v_s0, v_s1, vl); - vuint32m8_t v_t01 = __riscv_vwaddu_vv_u32m8(v_t0, v_t1, vl); - vuint32m8_t v_st01 = __riscv_vadd_vv_u32m8(v_s01, v_t01, vl); - vuint16m4_t v_avg = - __riscv_vnclipu_wx_u16m4(v_st01, 2, __RISCV_VXRM_RNU, vl); - vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_avg, shift, vl); - __riscv_vse8_v_u8m2(dst_uv, v_dst, vl); - w -= vl; - s += 2 * vl; - t += 2 * vl; - dst_uv += vl; - } while (w > 0); + int vl; + asm volatile( + "slli %[t], %[t], 1 \n" + "add %[t], %[s], %[t] \n" + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[w], e16, m2, ta, ma \n" + "vlseg2e16.v v16, (%[s]) \n" + "vlseg2e16.v v20, (%[t]) \n" + "vwaddu.vv v8, v16, v18 \n" + "vwaddu.vv v12, v20, v22 \n" + "vsetvli zero, zero, e32, m4, ta, ma \n" + "vadd.vv v8, v8, v12 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vnclipu.wi v16, v8, 2 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wx v8, v16, %[shift] \n" + "vse8.v v8, (%[dst_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[s], %[s], %[vl] \n" + "add %[t], %[t], %[vl] \n" + "bgtz %[w], 1b \n" + : [s] "+r"(src_uv), // %[s] + [t] "+r"(src_uv_stride), // %[t] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [shift] "r"(__builtin_clz((int32_t)scale) - 15) // %[shift] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); } -#else -void HalfWidthRow_16To8_RVV(const uint16_t* src_uv, - ptrdiff_t src_uv_stride, - uint8_t* dst_uv, - int scale, - int width) { - const uint16_t* s = src_uv; - const uint16_t* t = src_uv + src_uv_stride; - size_t w = (size_t)width; - const int shift = __builtin_clz((int32_t)scale) - 15; - asm volatile("csrwi vxrm, 0"); - do { - size_t vl = __riscv_vsetvl_e16m4(w); - vuint16m4_t v_s0, v_s1, v_t0, v_t1; - __riscv_vlseg2e16_v_u16m4(&v_s0, &v_s1, s, vl); - __riscv_vlseg2e16_v_u16m4(&v_t0, &v_t1, t, vl); - vuint32m8_t v_s01 = __riscv_vwaddu_vv_u32m8(v_s0, v_s1, vl); - vuint32m8_t v_t01 = __riscv_vwaddu_vv_u32m8(v_t0, v_t1, vl); - vuint32m8_t v_st01 = __riscv_vadd_vv_u32m8(v_s01, v_t01, vl); - vuint16m4_t v_avg = __riscv_vnclipu_wx_u16m4(v_st01, 2, vl); - vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_avg, shift, vl); - __riscv_vse8_v_u8m2(dst_uv, v_dst, vl); - w -= vl; - s += 2 * vl; - t += 2 * vl; - dst_uv += vl; - } while (w > 0); -} -#endif #endif #ifdef __cplusplus diff --git a/source/scale_rvv.cc b/source/scale_rvv.cc index a8b314af6..cfa7f5e5c 100644 --- a/source/scale_rvv.cc +++ b/source/scale_rvv.cc @@ -21,7 +21,6 @@ // This module is for RVV (RISC-V Vector extension) #if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) #include -#include #ifdef __cplusplus namespace libyuv { extern "C" { @@ -34,232 +33,182 @@ void ScaleARGBFilterCols_RVV(uint8_t* dst_argb, int x, int dx) { assert(x >= 0); + int vl, vl_dx; + asm volatile( + "vsetvli %[vl], %[w], e32, m2, ta, ma \n" + "vmv.v.x v0, %[x] \n" + "vid.v v2 \n" + "vmacc.vx v0, %[dx], v2 \n" - size_t vl = __riscv_vsetvl_e32m4(dst_width); - vuint32m4_t vx = __riscv_vmv_v_x_u32m4(x, vl); - vx = __riscv_vmacc_vx_u32m4(vx, dx, __riscv_vid_v_u32m4(vl), vl); - do { - vuint32m4_t v0_argb, v1_argb; - vuint32m4_t v_xf0_u32, v_xf1_u32; - vuint8m4_t v0_argb_u8, v1_argb_u8, v_xf0_u8, v_xf1_u8; - vuint16m8_t _v0_argb_u16, v_row_u16; - // idx is x >> 16 - vuint32m4_t v_xi_bindex = __riscv_vsrl_vx_u32m4(vx, 14, vl); - v_xi_bindex = __riscv_vand_vx_u32m4(v_xi_bindex, ~3u, vl); - // Read Packed ARGB w/ byte index. - __riscv_vluxseg2ei32_v_u32m4(&v0_argb, &v1_argb, (const uint32_t*)src_argb, - v_xi_bindex, vl); - // xf = (x >> 9) & 0x7f; - v_xf0_u32 = __riscv_vsrl_vx_u32m4(vx, 9, vl); - v_xf0_u32 = __riscv_vand_vx_u32m4(v_xf0_u32, 0x7f, vl); - vx = __riscv_vadd_vx_u32m4(vx, vl * dx, vl); - // duplicate v_xf0_u32[i] from {0,0,0,f[i]} to {f[i],f[i],f[i],f[i]} - v_xf0_u32 = __riscv_vmul_vx_u32m4(v_xf0_u32, 0x01010101, vl); - // TODO(fbarchard): Replace 0x7f ^ f with 128-f. bug=607. - v_xf1_u32 = __riscv_vxor_vx_u32m4(v_xf0_u32, 0x7f7f7f7f, vl); - - v0_argb_u8 = __riscv_vreinterpret_v_u32m4_u8m4(v0_argb); - v1_argb_u8 = __riscv_vreinterpret_v_u32m4_u8m4(v1_argb); - v_xf0_u8 = __riscv_vreinterpret_v_u32m4_u8m4(v_xf0_u32); - v_xf1_u8 = __riscv_vreinterpret_v_u32m4_u8m4(v_xf1_u32); - // ((a) * (0x7f ^ f) + (b)*f) >> 7 - _v0_argb_u16 = __riscv_vwmulu_vv_u16m8(v0_argb_u8, v_xf1_u8, 4 * vl); - v_row_u16 = - __riscv_vwmaccu_vv_u16m8(_v0_argb_u16, v1_argb_u8, v_xf0_u8, 4 * vl); - - __riscv_vse8_v_u8m4(dst_argb, __riscv_vnsrl_wx_u8m4(v_row_u16, 7, 4 * vl), - 4 * vl); - dst_width -= vl; - dst_argb += 4 * vl; - vl = __riscv_vsetvl_e32m4(dst_width); - } while (dst_width > 0); + "1: \n" + // idx is x >> 16, byte index is (x >> 14) & ~3u + "vsrl.vi v2, v0, 14 \n" + "vand.vi v2, v2, -4 \n" + // Read Packed ARGB w/ byte index. + "vluxseg2ei32.v v4, (%[src_argb]), v2 \n" + // xf = (x >> 9) & 0x7f; + "vsrl.vi v8, v0, 9 \n" + "vand.vx v8, v8, %[c_7f] \n" + "mul %[vl_dx], %[vl], %[dx] \n" + "vadd.vx v0, v0, %[vl_dx] \n" + // duplicate v_xf0_u32[i] from {0,0,0,f[i]} to {f[i],f[i],f[i],f[i]} + "vmul.vx v8, v8, %[c_01010101] \n" + // TODO(fbarchard): Replace 0x7f ^ f with 128-f. bug=607. + "vxor.vx v10, v8, %[c_7f7f7f7f] \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "vsetvli zero, %[vl], e8, m2, ta, ma \n" + // ((a) * (0x7f ^ f) + (b)*f) >> 7 + "vwmulu.vv v12, v4, v10 \n" + "vwmaccu.vv v12, v6, v8 \n" + "vnsrl.wi v4, v12, 7 \n" + "vse8.v v4, (%[dst_argb]) \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "vsetvli %[vl], %[w], e32, m2, ta, ma \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [vl] "=&r"(vl), // %[vl] + [vl_dx] "=&r"(vl_dx) // %[vl_dx] + : [src_argb] "r"(src_argb), // %[src_argb] + [x] "r"(x), // %[x] + [dx] "r"(dx), // %[dx] + [c_7f] "r"(0x7f), // %[c_7f] + [c_01010101] "r"(0x01010101u), // %[c_01010101] + [c_7f7f7f7f] "r"(0x7f7f7f7fu) // %[c_7f7f7f7f] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"); } #endif #ifdef HAS_SCALEADDROW_RVV void ScaleAddRow_RVV(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) { - size_t w = (size_t)src_width; - do { - size_t vl = __riscv_vsetvl_e8m4(w); - vuint8m4_t v_src = __riscv_vle8_v_u8m4(src_ptr, vl); - vuint16m8_t v_dst = __riscv_vle16_v_u16m8(dst_ptr, vl); - // Use widening multiply-add instead of widening + add - v_dst = __riscv_vwmaccu_vx_u16m8(v_dst, 1, v_src, vl); - __riscv_vse16_v_u16m8(dst_ptr, v_dst, vl); - w -= vl; - src_ptr += vl; - dst_ptr += vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle16.v v8, (%[dst_ptr]) \n" + "vle8.v v16, (%[src_ptr]) \n" + // Use widening multiply-add instead of widening + add + "vwmaccu.vx v8, %[one], v16 \n" + "vse16.v v8, (%[dst_ptr]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(src_width), // %[w] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [vl] "=&r"(vl) // %[vl] + : [one] "r"(1) // %[one] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } #endif #ifdef HAS_SCALEARGBROWDOWN2_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleARGBRowDown2_RVV(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { (void)src_stride; - size_t w = (size_t)dst_width; - const uint32_t* src = (const uint32_t*)(src_argb); - uint32_t* dst = (uint32_t*)(dst_argb); - do { - size_t vl = __riscv_vsetvl_e32m4(w); - vuint32m4x2_t v_src = __riscv_vlseg2e32_v_u32m4x2(src, vl); - vuint32m4_t v_dst = __riscv_vget_v_u32m4x2_u32m4(v_src, 1); - __riscv_vse32_v_u32m4(dst, v_dst, vl); - w -= vl; - src += vl * 2; - dst += vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e32, m4, ta, ma \n" + "vlseg2e32.v v8, (%[src]) \n" + "vse32.v v12, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst], %[dst], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src], %[src], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } -#else -void ScaleARGBRowDown2_RVV(const uint8_t* src_argb, - ptrdiff_t src_stride, - uint8_t* dst_argb, - int dst_width) { - (void)src_stride; - size_t w = (size_t)dst_width; - const uint32_t* src = (const uint32_t*)(src_argb); - uint32_t* dst = (uint32_t*)(dst_argb); - do { - vuint32m4_t v_even, v_odd; - size_t vl = __riscv_vsetvl_e32m4(w); - __riscv_vlseg2e32_v_u32m4(&v_even, &v_odd, src, vl); - __riscv_vse32_v_u32m4(dst, v_odd, vl); - w -= vl; - src += vl * 2; - dst += vl; - } while (w > 0); -} -#endif #endif #ifdef HAS_SCALEARGBROWDOWN2LINEAR_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleARGBRowDown2Linear_RVV(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { (void)src_stride; - size_t w = (size_t)dst_width; - const uint32_t* src = (const uint32_t*)(src_argb); - do { - size_t vl = __riscv_vsetvl_e32m4(w); - vuint32m4x2_t v_src = __riscv_vlseg2e32_v_u32m4x2(src, vl); - vuint32m4_t v_even_32 = __riscv_vget_v_u32m4x2_u32m4(v_src, 0); - vuint32m4_t v_odd_32 = __riscv_vget_v_u32m4x2_u32m4(v_src, 1); - vuint8m4_t v_even = __riscv_vreinterpret_v_u32m4_u8m4(v_even_32); - vuint8m4_t v_odd = __riscv_vreinterpret_v_u32m4_u8m4(v_odd_32); - vuint8m4_t v_dst = - __riscv_vaaddu_vv_u8m4(v_even, v_odd, __RISCV_VXRM_RNU, vl * 4); - __riscv_vse8_v_u8m4(dst_argb, v_dst, vl * 4); - w -= vl; - src += vl * 2; - dst_argb += vl * 4; - } while (w > 0); -} -#else -void ScaleARGBRowDown2Linear_RVV(const uint8_t* src_argb, - ptrdiff_t src_stride, - uint8_t* dst_argb, - int dst_width) { - (void)src_stride; - size_t w = (size_t)dst_width; - const uint32_t* src = (const uint32_t*)(src_argb); + int vl; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - vuint8m4_t v_odd, v_even, v_dst; - vuint32m4_t v_odd_32, v_even_32; - size_t vl = __riscv_vsetvl_e32m4(w); - __riscv_vlseg2e32_v_u32m4(&v_even_32, &v_odd_32, src, vl); - v_even = __riscv_vreinterpret_v_u32m4_u8m4(v_even_32); - v_odd = __riscv_vreinterpret_v_u32m4_u8m4(v_odd_32); - // Use round-to-nearest-up mode for averaging add - v_dst = __riscv_vaaddu_vv_u8m4(v_even, v_odd, vl * 4); - __riscv_vse8_v_u8m4(dst_argb, v_dst, vl * 4); - w -= vl; - src += vl * 2; - dst_argb += vl * 4; - } while (w > 0); + asm volatile( + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[w], e32, m4, ta, ma \n" + "vlseg2e32.v v8, (%[src]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "vsetvli zero, %[vl], e8, m4, ta, ma \n" + "vaaddu.vv v8, v8, v12 \n" + "vse8.v v8, (%[dst_argb]) \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[src], %[src], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src] "+r"(src_argb), // %[src] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif -#endif #ifdef HAS_SCALEARGBROWDOWN2BOX_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleARGBRowDown2Box_RVV(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { - size_t w = (size_t)dst_width; - const uint32_t* src0 = (const uint32_t*)(src_argb); - const uint32_t* src1 = (const uint32_t*)(src_argb + src_stride); - do { - size_t vl = __riscv_vsetvl_e32m4(w); - vuint32m4x2_t v_src0 = __riscv_vlseg2e32_v_u32m4x2(src0, vl); - vuint32m4x2_t v_src1 = __riscv_vlseg2e32_v_u32m4x2(src1, vl); - vuint32m4_t v_row0_even_32 = __riscv_vget_v_u32m4x2_u32m4(v_src0, 0); - vuint32m4_t v_row0_odd_32 = __riscv_vget_v_u32m4x2_u32m4(v_src0, 1); - vuint32m4_t v_row1_even_32 = __riscv_vget_v_u32m4x2_u32m4(v_src1, 0); - vuint32m4_t v_row1_odd_32 = __riscv_vget_v_u32m4x2_u32m4(v_src1, 1); - vuint8m4_t v_row0_even = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_even_32); - vuint8m4_t v_row0_odd = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_odd_32); - vuint8m4_t v_row1_even = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_even_32); - vuint8m4_t v_row1_odd = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_odd_32); - vuint16m8_t v_row0_sum = - __riscv_vwaddu_vv_u16m8(v_row0_even, v_row0_odd, vl * 4); - vuint16m8_t v_row1_sum = - __riscv_vwaddu_vv_u16m8(v_row1_even, v_row1_odd, vl * 4); - vuint16m8_t v_dst_16 = - __riscv_vadd_vv_u16m8(v_row0_sum, v_row1_sum, vl * 4); - vuint8m4_t v_dst = - __riscv_vnclipu_wx_u8m4(v_dst_16, 2, __RISCV_VXRM_RNU, vl * 4); - __riscv_vse8_v_u8m4(dst_argb, v_dst, vl * 4); - w -= vl; - src0 += vl * 2; - src1 += vl * 2; - dst_argb += vl * 4; - } while (w > 0); -} -#else -void ScaleARGBRowDown2Box_RVV(const uint8_t* src_argb, - ptrdiff_t src_stride, - uint8_t* dst_argb, - int dst_width) { - size_t w = (size_t)dst_width; - const uint32_t* src0 = (const uint32_t*)(src_argb); - const uint32_t* src1 = (const uint32_t*)(src_argb + src_stride); + int vl; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - vuint8m4_t v_row0_odd, v_row0_even, v_row1_odd, v_row1_even, v_dst; - vuint16m8_t v_row0_sum, v_row1_sum, v_dst_16; - vuint32m4_t v_row0_odd_32, v_row0_even_32, v_row1_odd_32, v_row1_even_32; - size_t vl = __riscv_vsetvl_e32m4(w); - __riscv_vlseg2e32_v_u32m4(&v_row0_even_32, &v_row0_odd_32, src0, vl); - __riscv_vlseg2e32_v_u32m4(&v_row1_even_32, &v_row1_odd_32, src1, vl); - v_row0_even = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_even_32); - v_row0_odd = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_odd_32); - v_row1_even = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_even_32); - v_row1_odd = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_odd_32); - v_row0_sum = __riscv_vwaddu_vv_u16m8(v_row0_even, v_row0_odd, vl * 4); - v_row1_sum = __riscv_vwaddu_vv_u16m8(v_row1_even, v_row1_odd, vl * 4); - v_dst_16 = __riscv_vadd_vv_u16m8(v_row0_sum, v_row1_sum, vl * 4); - // Use round-to-nearest-up mode for vnclip - v_dst = __riscv_vnclipu_wx_u8m4(v_dst_16, 2, vl * 4); - __riscv_vse8_v_u8m4(dst_argb, v_dst, vl * 4); - w -= vl; - src0 += vl * 2; - src1 += vl * 2; - dst_argb += vl * 4; - } while (w > 0); + asm volatile( + "add %[src1], %[src0], %[src1] \n" + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[w], e32, m2, ta, ma \n" + "vlseg2e32.v v16, (%[src0]) \n" + "vlseg2e32.v v20, (%[src1]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "vsetvli zero, %[vl], e8, m2, ta, ma \n" + "vwaddu.vv v8, v16, v18 \n" + "vwaddu.vv v12, v20, v22 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vv v8, v8, v12 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnclipu.wi v16, v8, 2 \n" + "vse8.v v16, (%[dst_argb]) \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[src0], %[src0], %[vl] \n" + "add %[src1], %[src1], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src0] "+r"(src_argb), // %[src0] + [src1] "+r"(src_stride), // %[src1] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); } #endif -#endif #ifdef HAS_SCALEARGBROWDOWNEVEN_RVV void ScaleARGBRowDownEven_RVV(const uint8_t* src_argb, @@ -267,1035 +216,617 @@ void ScaleARGBRowDownEven_RVV(const uint8_t* src_argb, int src_stepx, uint8_t* dst_argb, int dst_width) { - size_t w = (size_t)dst_width; - const uint32_t* src = (const uint32_t*)(src_argb); - uint32_t* dst = (uint32_t*)(dst_argb); - const int stride_byte = src_stepx * 4; - do { - size_t vl = __riscv_vsetvl_e32m8(w); - vuint32m8_t v_row = __riscv_vlse32_v_u32m8(src, stride_byte, vl); - __riscv_vse32_v_u32m8(dst, v_row, vl); - w -= vl; - src += vl * src_stepx; - dst += vl; - } while (w > 0); + (void)src_stride; + int vl; + ptrdiff_t src_step; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e32, m4, ta, ma \n" + "vlse32.v v8, (%[src]), %[stride_byte] \n" + "vse32.v v8, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "mul %[src_step], %[vl], %[stride_byte]\n" + "add %[src], %[src], %[src_step] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst], %[dst], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [vl] "=&r"(vl), // %[vl] + [src_step] "=&r"(src_step) // %[src_step] + : [stride_byte] "r"((ptrdiff_t)src_stepx * 4) // %[stride_byte] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11"); } #endif #ifdef HAS_SCALEARGBROWDOWNEVENBOX_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, int dst_width) { - size_t w = (size_t)dst_width; - const uint32_t* src0 = (const uint32_t*)(src_argb); - const uint32_t* src1 = (const uint32_t*)(src_argb + src_stride); - const int stride_byte = src_stepx * 4; - do { - size_t vl = __riscv_vsetvl_e32m4(w); - vuint32m4x2_t v_src0 = __riscv_vlsseg2e32_v_u32m4x2(src0, stride_byte, vl); - vuint32m4x2_t v_src1 = __riscv_vlsseg2e32_v_u32m4x2(src1, stride_byte, vl); - vuint32m4_t v_row0_low_32 = __riscv_vget_v_u32m4x2_u32m4(v_src0, 0); - vuint32m4_t v_row0_high_32 = __riscv_vget_v_u32m4x2_u32m4(v_src0, 1); - vuint32m4_t v_row1_low_32 = __riscv_vget_v_u32m4x2_u32m4(v_src1, 0); - vuint32m4_t v_row1_high_32 = __riscv_vget_v_u32m4x2_u32m4(v_src1, 1); - vuint8m4_t v_row0_low = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_low_32); - vuint8m4_t v_row0_high = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_high_32); - vuint8m4_t v_row1_low = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_low_32); - vuint8m4_t v_row1_high = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_high_32); - vuint16m8_t v_row0_sum = - __riscv_vwaddu_vv_u16m8(v_row0_low, v_row0_high, vl * 4); - vuint16m8_t v_row1_sum = - __riscv_vwaddu_vv_u16m8(v_row1_low, v_row1_high, vl * 4); - vuint16m8_t v_sum = __riscv_vadd_vv_u16m8(v_row0_sum, v_row1_sum, vl * 4); - vuint8m4_t v_dst = - __riscv_vnclipu_wx_u8m4(v_sum, 2, __RISCV_VXRM_RNU, vl * 4); - __riscv_vse8_v_u8m4(dst_argb, v_dst, vl * 4); - w -= vl; - src0 += vl * src_stepx; - src1 += vl * src_stepx; - dst_argb += vl * 4; - } while (w > 0); -} -#else -void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb, - ptrdiff_t src_stride, - int src_stepx, - uint8_t* dst_argb, - int dst_width) { - size_t w = (size_t)dst_width; - const uint32_t* src0 = (const uint32_t*)(src_argb); - const uint32_t* src1 = (const uint32_t*)(src_argb + src_stride); - const int stride_byte = src_stepx * 4; + int vl; + ptrdiff_t src_step; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - vuint8m4_t v_row0_low, v_row0_high, v_row1_low, v_row1_high, v_dst; - vuint16m8_t v_row0_sum, v_row1_sum, v_sum; - vuint32m4_t v_row0_low_32, v_row0_high_32, v_row1_low_32, v_row1_high_32; - size_t vl = __riscv_vsetvl_e32m4(w); - __riscv_vlsseg2e32_v_u32m4(&v_row0_low_32, &v_row0_high_32, src0, - stride_byte, vl); - __riscv_vlsseg2e32_v_u32m4(&v_row1_low_32, &v_row1_high_32, src1, - stride_byte, vl); - v_row0_low = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_low_32); - v_row0_high = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_high_32); - v_row1_low = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_low_32); - v_row1_high = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_high_32); - v_row0_sum = __riscv_vwaddu_vv_u16m8(v_row0_low, v_row0_high, vl * 4); - v_row1_sum = __riscv_vwaddu_vv_u16m8(v_row1_low, v_row1_high, vl * 4); - v_sum = __riscv_vadd_vv_u16m8(v_row0_sum, v_row1_sum, vl * 4); - // Use round-to-nearest-up mode for vnclip - v_dst = __riscv_vnclipu_wx_u8m4(v_sum, 2, vl * 4); - __riscv_vse8_v_u8m4(dst_argb, v_dst, vl * 4); - w -= vl; - src0 += vl * src_stepx; - src1 += vl * src_stepx; - dst_argb += vl * 4; - } while (w > 0); + asm volatile( + "add %[src1], %[src0], %[src1] \n" + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[w], e32, m2, ta, ma \n" + "vlsseg2e32.v v16, (%[src0]), %[stride_byte] \n" + "vlsseg2e32.v v20, (%[src1]), %[stride_byte] \n" + "sub %[w], %[w], %[vl] \n" + "mul %[src_step], %[vl], %[stride_byte]\n" + "slli %[vl], %[vl], 2 \n" + "vsetvli zero, %[vl], e8, m2, ta, ma \n" + "vwaddu.vv v8, v16, v18 \n" + "vwaddu.vv v12, v20, v22 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vadd.vv v8, v8, v12 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnclipu.wi v16, v8, 2 \n" + "vse8.v v16, (%[dst_argb]) \n" + "add %[src0], %[src0], %[src_step] \n" + "add %[src1], %[src1], %[src_step] \n" + "add %[dst_argb], %[dst_argb], %[vl]\n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src0] "+r"(src_argb), // %[src0] + [src1] "+r"(src_stride), // %[src1] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [vl] "=&r"(vl), // %[vl] + [src_step] "=&r"(src_step) // %[src_step] + : [stride_byte] "r"((ptrdiff_t)src_stepx * 4) // %[stride_byte] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); } #endif -#endif #ifdef HAS_SCALEROWDOWN2_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleRowDown2_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - size_t w = (size_t)dst_width; (void)src_stride; - do { - size_t vl = __riscv_vsetvl_e8m4(w); - vuint8m4x2_t v_src = __riscv_vlseg2e8_v_u8m4x2(src_ptr, vl); - vuint8m4_t v_dst = __riscv_vget_v_u8m4x2_u8m4(v_src, 1); - __riscv_vse8_v_u8m4(dst, v_dst, vl); - w -= vl; - src_ptr += 2 * vl; - dst += vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle16.v v8, (%[src_ptr]) \n" + "vnsrl.wi v16, v8, 8 \n" + "vse8.v v16, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst], %[dst], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst] "+r"(dst), // %[dst] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } -#else -void ScaleRowDown2_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width) { - size_t w = (size_t)dst_width; - (void)src_stride; - do { - vuint8m4_t v_s0, v_s1; - size_t vl = __riscv_vsetvl_e8m4(w); - __riscv_vlseg2e8_v_u8m4(&v_s0, &v_s1, src_ptr, vl); - __riscv_vse8_v_u8m4(dst, v_s1, vl); - w -= vl; - src_ptr += 2 * vl; - dst += vl; - } while (w > 0); -} -#endif #endif #ifdef HAS_SCALEROWDOWN2LINEAR_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleRowDown2Linear_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - size_t w = (size_t)dst_width; - (void)src_stride; - do { - size_t vl = __riscv_vsetvl_e8m4(w); - vuint8m4x2_t v_src = __riscv_vlseg2e8_v_u8m4x2(src_ptr, vl); - vuint8m4_t v_s0 = __riscv_vget_v_u8m4x2_u8m4(v_src, 0); - vuint8m4_t v_s1 = __riscv_vget_v_u8m4x2_u8m4(v_src, 1); - vuint8m4_t v_dst = __riscv_vaaddu_vv_u8m4(v_s0, v_s1, __RISCV_VXRM_RNU, vl); - __riscv_vse8_v_u8m4(dst, v_dst, vl); - w -= vl; - src_ptr += 2 * vl; - dst += vl; - } while (w > 0); -} -#else -void ScaleRowDown2Linear_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width) { - size_t w = (size_t)dst_width; (void)src_stride; + int vl; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - vuint8m4_t v_s0, v_s1, v_dst; - size_t vl = __riscv_vsetvl_e8m4(w); - __riscv_vlseg2e8_v_u8m4(&v_s0, &v_s1, src_ptr, vl); - // Use round-to-nearest-up mode for averaging add - v_dst = __riscv_vaaddu_vv_u8m4(v_s0, v_s1, vl); - __riscv_vse8_v_u8m4(dst, v_dst, vl); - w -= vl; - src_ptr += 2 * vl; - dst += vl; - } while (w > 0); + asm volatile( + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vle16.v v8, (%[src_ptr]) \n" + "vnsrl.wi v16, v8, 0 \n" + "vnsrl.wi v18, v8, 8 \n" + "vaaddu.vv v16, v16, v18 \n" + "vse8.v v16, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst], %[dst], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst] "+r"(dst), // %[dst] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18", + "v19"); } #endif -#endif #ifdef HAS_SCALEROWDOWN2BOX_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleRowDown2Box_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - const uint8_t* s = src_ptr; - const uint8_t* t = src_ptr + src_stride; - size_t w = (size_t)dst_width; - do { - size_t vl = __riscv_vsetvl_e8m4(w); - vuint8m4x2_t v_s = __riscv_vlseg2e8_v_u8m4x2(s, vl); - vuint8m4x2_t v_t = __riscv_vlseg2e8_v_u8m4x2(t, vl); - vuint8m4_t v_s0 = __riscv_vget_v_u8m4x2_u8m4(v_s, 0); - vuint8m4_t v_s1 = __riscv_vget_v_u8m4x2_u8m4(v_s, 1); - vuint8m4_t v_t0 = __riscv_vget_v_u8m4x2_u8m4(v_t, 0); - vuint8m4_t v_t1 = __riscv_vget_v_u8m4x2_u8m4(v_t, 1); - vuint16m8_t v_s01 = __riscv_vwaddu_vv_u16m8(v_s0, v_s1, vl); - vuint16m8_t v_t01 = __riscv_vwaddu_vv_u16m8(v_t0, v_t1, vl); - vuint16m8_t v_st01 = __riscv_vadd_vv_u16m8(v_s01, v_t01, vl); - // Use round-to-nearest-up mode for vnclip - vuint8m4_t v_dst = __riscv_vnclipu_wx_u8m4(v_st01, 2, __RISCV_VXRM_RNU, vl); - __riscv_vse8_v_u8m4(dst, v_dst, vl); - w -= vl; - s += 2 * vl; - t += 2 * vl; - dst += vl; - } while (w > 0); -} -#else -void ScaleRowDown2Box_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width) { - const uint8_t* s = src_ptr; - const uint8_t* t = src_ptr + src_stride; - size_t w = (size_t)dst_width; + int vl; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - size_t vl = __riscv_vsetvl_e8m4(w); - vuint8m4_t v_s0, v_s1, v_t0, v_t1; - vuint16m8_t v_s01, v_t01, v_st01; - vuint8m4_t v_dst; - __riscv_vlseg2e8_v_u8m4(&v_s0, &v_s1, s, vl); - __riscv_vlseg2e8_v_u8m4(&v_t0, &v_t1, t, vl); - v_s01 = __riscv_vwaddu_vv_u16m8(v_s0, v_s1, vl); - v_t01 = __riscv_vwaddu_vv_u16m8(v_t0, v_t1, vl); - v_st01 = __riscv_vadd_vv_u16m8(v_s01, v_t01, vl); - // Use round-to-nearest-up mode for vnclip - v_dst = __riscv_vnclipu_wx_u8m4(v_st01, 2, vl); - __riscv_vse8_v_u8m4(dst, v_dst, vl); - w -= vl; - s += 2 * vl; - t += 2 * vl; - dst += vl; - } while (w > 0); + asm volatile( + "add %[t], %[s], %[t] \n" + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg2e8.v v16, (%[s]) \n" + "vlseg2e8.v v20, (%[t]) \n" + "vwaddu.vv v8, v16, v18 \n" + "vwaddu.wv v8, v8, v20 \n" + "vwaddu.wv v8, v8, v22 \n" + "vnclipu.wi v16, v8, 2 \n" + "vse8.v v16, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst], %[dst], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[s], %[s], %[vl] \n" + "add %[t], %[t], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [s] "+r"(src_ptr), // %[s] + [t] "+r"(src_stride), // %[t] + [dst] "+r"(dst), // %[dst] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23"); } #endif -#endif #ifdef HAS_SCALEROWDOWN4_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleRowDown4_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - size_t w = (size_t)dst_width; (void)src_stride; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(src_ptr, vl); - vuint8m2_t v_s2 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2); - __riscv_vse8_v_u8m2(dst_ptr, v_s2, vl); - w -= vl; - src_ptr += (4 * vl); - dst_ptr += vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v8, (%[src_ptr]) \n" + "vse8.v v12, (%[dst_ptr]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } -#else -void ScaleRowDown4_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - size_t w = (size_t)dst_width; - (void)src_stride; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_s0, v_s1, v_s2, v_s3; - __riscv_vlseg4e8_v_u8m2(&v_s0, &v_s1, &v_s2, &v_s3, src_ptr, vl); - __riscv_vse8_v_u8m2(dst_ptr, v_s2, vl); - w -= vl; - src_ptr += (4 * vl); - dst_ptr += vl; - } while (w > 0); -} -#endif #endif #ifdef HAS_SCALEROWDOWN4BOX_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleRowDown4Box_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - const uint8_t* src_ptr1 = src_ptr + src_stride; - const uint8_t* src_ptr2 = src_ptr + src_stride * 2; - const uint8_t* src_ptr3 = src_ptr + src_stride * 3; - size_t w = (size_t)dst_width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(src_ptr, vl); - vuint8m2_t v_s0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 0); - vuint8m2_t v_s1 = __riscv_vget_v_u8m2x4_u8m2(v_s, 1); - vuint8m2_t v_s2 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2); - vuint8m2_t v_s3 = __riscv_vget_v_u8m2x4_u8m2(v_s, 3); - vuint16m4_t v_s01 = __riscv_vwaddu_vv_u16m4(v_s0, v_s1, vl); - vuint8m2x4_t v_t = __riscv_vlseg4e8_v_u8m2x4(src_ptr1, vl); - vuint8m2_t v_t0 = __riscv_vget_v_u8m2x4_u8m2(v_t, 0); - vuint8m2_t v_t1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 1); - vuint8m2_t v_t2 = __riscv_vget_v_u8m2x4_u8m2(v_t, 2); - vuint8m2_t v_t3 = __riscv_vget_v_u8m2x4_u8m2(v_t, 3); - vuint16m4_t v_t01 = __riscv_vwaddu_vv_u16m4(v_t0, v_t1, vl); - vuint8m2x4_t v_u = __riscv_vlseg4e8_v_u8m2x4(src_ptr2, vl); - vuint8m2_t v_u0 = __riscv_vget_v_u8m2x4_u8m2(v_u, 0); - vuint8m2_t v_u1 = __riscv_vget_v_u8m2x4_u8m2(v_u, 1); - vuint8m2_t v_u2 = __riscv_vget_v_u8m2x4_u8m2(v_u, 2); - vuint8m2_t v_u3 = __riscv_vget_v_u8m2x4_u8m2(v_u, 3); - vuint16m4_t v_u01 = __riscv_vwaddu_vv_u16m4(v_u0, v_u1, vl); - vuint16m4_t v_u23 = __riscv_vwaddu_vv_u16m4(v_u2, v_u3, vl); - vuint16m4_t v_s23 = __riscv_vwaddu_vv_u16m4(v_s2, v_s3, vl); - vuint16m4_t v_t23 = __riscv_vwaddu_vv_u16m4(v_t2, v_t3, vl); - vuint16m4_t v_st01 = __riscv_vadd_vv_u16m4(v_s01, v_t01, vl); - vuint16m4_t v_st23 = __riscv_vadd_vv_u16m4(v_s23, v_t23, vl); - vuint8m2x4_t v_v = __riscv_vlseg4e8_v_u8m2x4(src_ptr3, vl); - vuint8m2_t v_v0 = __riscv_vget_v_u8m2x4_u8m2(v_v, 0); - vuint8m2_t v_v1 = __riscv_vget_v_u8m2x4_u8m2(v_v, 1); - vuint8m2_t v_v2 = __riscv_vget_v_u8m2x4_u8m2(v_v, 2); - vuint8m2_t v_v3 = __riscv_vget_v_u8m2x4_u8m2(v_v, 3); - - vuint16m4_t v_v01 = __riscv_vwaddu_vv_u16m4(v_v0, v_v1, vl); - vuint16m4_t v_v23 = __riscv_vwaddu_vv_u16m4(v_v2, v_v3, vl); - - vuint16m4_t v_uv01 = __riscv_vadd_vv_u16m4(v_u01, v_v01, vl); - vuint16m4_t v_uv23 = __riscv_vadd_vv_u16m4(v_u23, v_v23, vl); - - vuint16m4_t v_st0123 = __riscv_vadd_vv_u16m4(v_st01, v_st23, vl); - vuint16m4_t v_uv0123 = __riscv_vadd_vv_u16m4(v_uv01, v_uv23, vl); - vuint16m4_t v_stuv0123 = __riscv_vadd_vv_u16m4(v_st0123, v_uv0123, vl); - vuint8m2_t v_dst = - __riscv_vnclipu_wx_u8m2(v_stuv0123, 4, __RISCV_VXRM_RNU, vl); - __riscv_vse8_v_u8m2(dst_ptr, v_dst, vl); - w -= vl; - src_ptr += 4 * vl; - src_ptr1 += 4 * vl; - src_ptr2 += 4 * vl; - src_ptr3 += 4 * vl; - dst_ptr += vl; - } while (w > 0); -} -#else -void ScaleRowDown4Box_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - const uint8_t* src_ptr1 = src_ptr + src_stride; - const uint8_t* src_ptr2 = src_ptr + src_stride * 2; - const uint8_t* src_ptr3 = src_ptr + src_stride * 3; - size_t w = (size_t)dst_width; + const uint8_t* src_ptr1; + const uint8_t* src_ptr2; + int vl; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - vuint8m2_t v_s0, v_s1, v_s2, v_s3; - vuint8m2_t v_t0, v_t1, v_t2, v_t3; - vuint8m2_t v_u0, v_u1, v_u2, v_u3; - vuint8m2_t v_v0, v_v1, v_v2, v_v3; - vuint16m4_t v_s01, v_s23, v_t01, v_t23; - vuint16m4_t v_u01, v_u23, v_v01, v_v23; - vuint16m4_t v_st01, v_st23, v_uv01, v_uv23; - vuint16m4_t v_st0123, v_uv0123, v_stuv0123; - vuint8m2_t v_dst; - size_t vl = __riscv_vsetvl_e8m2(w); + asm volatile( + "add %[src_ptr1], %[src_ptr], %[src_ptr3]\n" + "add %[src_ptr2], %[src_ptr1], %[src_ptr3]\n" + "add %[src_ptr3], %[src_ptr2], %[src_ptr3]\n" + "csrwi vxrm, 0 \n" - __riscv_vlseg4e8_v_u8m2(&v_s0, &v_s1, &v_s2, &v_s3, src_ptr, vl); - v_s01 = __riscv_vwaddu_vv_u16m4(v_s0, v_s1, vl); - - __riscv_vlseg4e8_v_u8m2(&v_t0, &v_t1, &v_t2, &v_t3, src_ptr1, vl); - v_t01 = __riscv_vwaddu_vv_u16m4(v_t0, v_t1, vl); - - __riscv_vlseg4e8_v_u8m2(&v_u0, &v_u1, &v_u2, &v_u3, src_ptr2, vl); - v_u01 = __riscv_vwaddu_vv_u16m4(v_u0, v_u1, vl); - v_u23 = __riscv_vwaddu_vv_u16m4(v_u2, v_u3, vl); - - v_s23 = __riscv_vwaddu_vv_u16m4(v_s2, v_s3, vl); - v_t23 = __riscv_vwaddu_vv_u16m4(v_t2, v_t3, vl); - v_st01 = __riscv_vadd_vv_u16m4(v_s01, v_t01, vl); - v_st23 = __riscv_vadd_vv_u16m4(v_s23, v_t23, vl); - - __riscv_vlseg4e8_v_u8m2(&v_v0, &v_v1, &v_v2, &v_v3, src_ptr3, vl); - - v_v01 = __riscv_vwaddu_vv_u16m4(v_v0, v_v1, vl); - v_v23 = __riscv_vwaddu_vv_u16m4(v_v2, v_v3, vl); - - v_uv01 = __riscv_vadd_vv_u16m4(v_u01, v_v01, vl); - v_uv23 = __riscv_vadd_vv_u16m4(v_u23, v_v23, vl); - - v_st0123 = __riscv_vadd_vv_u16m4(v_st01, v_st23, vl); - v_uv0123 = __riscv_vadd_vv_u16m4(v_uv01, v_uv23, vl); - v_stuv0123 = __riscv_vadd_vv_u16m4(v_st0123, v_uv0123, vl); - // Use round-to-nearest-up mode for vnclip - v_dst = __riscv_vnclipu_wx_u8m2(v_stuv0123, 4, vl); - __riscv_vse8_v_u8m2(dst_ptr, v_dst, vl); - w -= vl; - src_ptr += 4 * vl; - src_ptr1 += 4 * vl; - src_ptr2 += 4 * vl; - src_ptr3 += 4 * vl; - dst_ptr += vl; - } while (w > 0); + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v16, (%[src_ptr]) \n" + "vlseg4e8.v v24, (%[src_ptr1]) \n" + "vwaddu.vv v8, v16, v18 \n" + "vwaddu.wv v8, v8, v20 \n" + "vwaddu.wv v8, v8, v22 \n" + "vwaddu.wv v8, v8, v24 \n" + "vwaddu.wv v8, v8, v26 \n" + "vwaddu.wv v8, v8, v28 \n" + "vwaddu.wv v8, v8, v30 \n" + "vlseg4e8.v v16, (%[src_ptr2]) \n" + "vlseg4e8.v v24, (%[src_ptr3]) \n" + "vwaddu.wv v8, v8, v16 \n" + "vwaddu.wv v8, v8, v18 \n" + "vwaddu.wv v8, v8, v20 \n" + "vwaddu.wv v8, v8, v22 \n" + "vwaddu.wv v8, v8, v24 \n" + "vwaddu.wv v8, v8, v26 \n" + "vwaddu.wv v8, v8, v28 \n" + "vwaddu.wv v8, v8, v30 \n" + "vnclipu.wi v16, v8, 4 \n" + "vse8.v v16, (%[dst_ptr]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "add %[src_ptr1], %[src_ptr1], %[vl]\n" + "add %[src_ptr2], %[src_ptr2], %[vl]\n" + "add %[src_ptr3], %[src_ptr3], %[vl]\n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "=&r"(src_ptr1), // %[src_ptr1] + [src_ptr2] "=&r"(src_ptr2), // %[src_ptr2] + [src_ptr3] "+r"(src_stride), // %[src_ptr3] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", + "v29", "v30", "v31"); } #endif -#endif #ifdef HAS_SCALEROWDOWN34_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleRowDown34_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - size_t w = (size_t)dst_width / 3u; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src = __riscv_vlseg4e8_v_u8m2x4(src_ptr, vl); - vuint8m2_t v_0 = __riscv_vget_v_u8m2x4_u8m2(v_src, 0); - vuint8m2_t v_1 = __riscv_vget_v_u8m2x4_u8m2(v_src, 1); - vuint8m2_t v_3 = __riscv_vget_v_u8m2x4_u8m2(v_src, 3); - vuint8m2x3_t v_dst = __riscv_vcreate_v_u8m2x3(v_0, v_1, v_3); - __riscv_vsseg3e8_v_u8m2x3(dst_ptr, v_dst, vl); - w -= vl; - src_ptr += 4 * vl; - dst_ptr += 3 * vl; - } while (w > 0); + (void)src_stride; + dst_width /= 3; + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v8, (%[src_ptr]) \n" + "vmv2r.v v12, v14 \n" + "vsseg3e8.v v8, (%[dst_ptr]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } -#else -void ScaleRowDown34_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - size_t w = (size_t)dst_width / 3u; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2_t v_s0, v_s1, v_s2, v_s3; - __riscv_vlseg4e8_v_u8m2(&v_s0, &v_s1, &v_s2, &v_s3, src_ptr, vl); - __riscv_vsseg3e8_v_u8m2(dst_ptr, v_s0, v_s1, v_s3, vl); - w -= vl; - src_ptr += 4 * vl; - dst_ptr += 3 * vl; - } while (w > 0); -} -#endif #endif #ifdef HAS_SCALEROWDOWN34_0_BOX_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleRowDown34_0_Box_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - size_t w = (size_t)dst_width / 3u; - const uint8_t* s = src_ptr; - const uint8_t* t = src_ptr + src_stride; - do { - vuint16m4_t v_t0_u16, v_t1_u16, v_t2_u16, v_t3_u16; - vuint8m2_t v_u0, v_u1, v_u2, v_u3; - vuint16m4_t v_u1_u16; - vuint8m2_t v_a0, v_a1, v_a2; - vuint8m2x3_t v_dst; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(s, vl); - vuint8m2_t v_s0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 0); - vuint8m2_t v_s1 = __riscv_vget_v_u8m2x4_u8m2(v_s, 1); - vuint8m2_t v_s2 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2); - vuint8m2_t v_s3 = __riscv_vget_v_u8m2x4_u8m2(v_s, 3); - - if (src_stride == 0) { - v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_s0, 2, vl); - v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_s1, 2, vl); - v_t2_u16 = __riscv_vwaddu_vx_u16m4(v_s2, 2, vl); - v_t3_u16 = __riscv_vwaddu_vx_u16m4(v_s3, 2, vl); - } else { - vuint8m2x4_t v_t = __riscv_vlseg4e8_v_u8m2x4(t, vl); - vuint8m2_t v_t0 = __riscv_vget_v_u8m2x4_u8m2(v_t, 0); - vuint8m2_t v_t1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 1); - vuint8m2_t v_t2 = __riscv_vget_v_u8m2x4_u8m2(v_t, 2); - vuint8m2_t v_t3 = __riscv_vget_v_u8m2x4_u8m2(v_t, 3); - v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_t0, 0, vl); - v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_t1, 0, vl); - v_t2_u16 = __riscv_vwaddu_vx_u16m4(v_t2, 0, vl); - v_t3_u16 = __riscv_vwaddu_vx_u16m4(v_t3, 0, vl); - t += 4 * vl; - } - - v_t0_u16 = __riscv_vwmaccu_vx_u16m4(v_t0_u16, 3, v_s0, vl); - v_t1_u16 = __riscv_vwmaccu_vx_u16m4(v_t1_u16, 3, v_s1, vl); - v_t2_u16 = __riscv_vwmaccu_vx_u16m4(v_t2_u16, 3, v_s2, vl); - v_t3_u16 = __riscv_vwmaccu_vx_u16m4(v_t3_u16, 3, v_s3, vl); - - v_u0 = __riscv_vnclipu_wx_u8m2(v_t0_u16, 2, __RISCV_VXRM_RNU, vl); - v_u1 = __riscv_vnclipu_wx_u8m2(v_t1_u16, 2, __RISCV_VXRM_RNU, vl); - v_u2 = __riscv_vnclipu_wx_u8m2(v_t2_u16, 2, __RISCV_VXRM_RNU, vl); - v_u3 = __riscv_vnclipu_wx_u8m2(v_t3_u16, 2, __RISCV_VXRM_RNU, vl); - // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 - v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_u1, 0, vl); - v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_u0, vl); - v_a0 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, __RISCV_VXRM_RNU, vl); - // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 - v_a1 = __riscv_vaaddu_vv_u8m2(v_u1, v_u2, __RISCV_VXRM_RNU, vl); - // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 - v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_u2, 0, vl); - v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_u3, vl); - v_a2 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, __RISCV_VXRM_RNU, vl); - - v_dst = __riscv_vcreate_v_u8m2x3(v_a0, v_a1, v_a2); - __riscv_vsseg3e8_v_u8m2x3(dst_ptr, v_dst, vl); - - w -= vl; - s += 4 * vl; - dst_ptr += 3 * vl; - } while (w > 0); -} -#else -void ScaleRowDown34_0_Box_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - size_t w = (size_t)dst_width / 3u; - const uint8_t* s = src_ptr; - const uint8_t* t = src_ptr + src_stride; + dst_width /= 3; + const uint8_t* t; + int vl; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - vuint8m2_t v_s0, v_s1, v_s2, v_s3; - vuint16m4_t v_t0_u16, v_t1_u16, v_t2_u16, v_t3_u16; - vuint8m2_t v_u0, v_u1, v_u2, v_u3; - vuint16m4_t v_u1_u16; - vuint8m2_t v_a0, v_a1, v_a2; - size_t vl = __riscv_vsetvl_e8m2(w); - __riscv_vlseg4e8_v_u8m2(&v_s0, &v_s1, &v_s2, &v_s3, s, vl); + asm volatile( + "add %[t], %[s], %[src_stride] \n" + "csrwi vxrm, 0 \n" - if (src_stride == 0) { - v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_s0, 2, vl); - v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_s1, 2, vl); - v_t2_u16 = __riscv_vwaddu_vx_u16m4(v_s2, 2, vl); - v_t3_u16 = __riscv_vwaddu_vx_u16m4(v_s3, 2, vl); - } else { - vuint8m2_t v_t0, v_t1, v_t2, v_t3; - __riscv_vlseg4e8_v_u8m2(&v_t0, &v_t1, &v_t2, &v_t3, t, vl); - v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_t0, 0, vl); - v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_t1, 0, vl); - v_t2_u16 = __riscv_vwaddu_vx_u16m4(v_t2, 0, vl); - v_t3_u16 = __riscv_vwaddu_vx_u16m4(v_t3, 0, vl); - t += 4 * vl; - } + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v24, (%[s]) \n" + "bnez %[src_stride], 2f \n" + "vwaddu.vx v20, v24, %[c2] \n" + "vwaddu.vx v16, v26, %[c2] \n" + "vwaddu.vx v12, v28, %[c2] \n" + "vwaddu.vx v8, v30, %[c2] \n" + "j 3f \n" - v_t0_u16 = __riscv_vwmaccu_vx_u16m4(v_t0_u16, 3, v_s0, vl); - v_t1_u16 = __riscv_vwmaccu_vx_u16m4(v_t1_u16, 3, v_s1, vl); - v_t2_u16 = __riscv_vwmaccu_vx_u16m4(v_t2_u16, 3, v_s2, vl); - v_t3_u16 = __riscv_vwmaccu_vx_u16m4(v_t3_u16, 3, v_s3, vl); + "2: \n" + "vlseg4e8.v v0, (%[t]) \n" + "vwcvtu.x.x.v v20, v0 \n" + "vwcvtu.x.x.v v16, v2 \n" + "vwcvtu.x.x.v v12, v4 \n" + "vwcvtu.x.x.v v8, v6 \n" - // Use round-to-nearest-up mode for vnclip & averaging add - v_u0 = __riscv_vnclipu_wx_u8m2(v_t0_u16, 2, vl); - v_u1 = __riscv_vnclipu_wx_u8m2(v_t1_u16, 2, vl); - v_u2 = __riscv_vnclipu_wx_u8m2(v_t2_u16, 2, vl); - v_u3 = __riscv_vnclipu_wx_u8m2(v_t3_u16, 2, vl); - - // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 - v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_u1, 0, vl); - v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_u0, vl); - v_a0 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, vl); - - // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 - v_a1 = __riscv_vaaddu_vv_u8m2(v_u1, v_u2, vl); - - // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 - v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_u2, 0, vl); - v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_u3, vl); - v_a2 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, vl); - - __riscv_vsseg3e8_v_u8m2(dst_ptr, v_a0, v_a1, v_a2, vl); - - w -= vl; - s += 4 * vl; - dst_ptr += 3 * vl; - } while (w > 0); + "3: \n" + "vwmaccu.vx v20, %[c3], v24 \n" + "vnclipu.wi v24, v20, 2 \n" + "vwmaccu.vx v16, %[c3], v26 \n" + "vnclipu.wi v20, v16, 2 \n" + "vwmaccu.vx v12, %[c3], v28 \n" + "vnclipu.wi v16, v12, 2 \n" + "vwmaccu.vx v8, %[c3], v30 \n" + "vnclipu.wi v12, v8, 2 \n" + // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 + "vwcvtu.x.x.v v8, v20 \n" + "vwmaccu.vx v8, %[c3], v24 \n" + "vnclipu.wi v14, v8, 2 \n" + // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 + "vwcvtu.x.x.v v8, v16 \n" + "vwmaccu.vx v8, %[c3], v12 \n" + // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 + "vaaddu.vv v16, v20, v16 \n" + "vnclipu.wi v18, v8, 2 \n" + "vsseg3e8.v v14, (%[dst_ptr]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[s], %[s], %[vl] \n" + "add %[t], %[t], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [s] "+r"(src_ptr), // %[s] + [t] "=&r"(t), // %[t] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [vl] "=&r"(vl) // %[vl] + : [src_stride] "r"(src_stride), // %[src_stride] + [c2] "r"(2), // %[c2] + [c3] "r"(3) // %[c3] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30", "v31"); } #endif -#endif #ifdef HAS_SCALEROWDOWN34_1_BOX_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleRowDown34_1_Box_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - size_t w = (size_t)dst_width / 3u; - const uint8_t* s = src_ptr; - const uint8_t* t = src_ptr + src_stride; - do { - vuint8m2_t v_ave0, v_ave1, v_ave2, v_ave3; - vuint16m4_t v_u1_u16; - vuint8m2_t v_a0, v_a1, v_a2; - vuint8m2x3_t v_dst; - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(s, vl); - vuint8m2_t v_s0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 0); - vuint8m2_t v_s1 = __riscv_vget_v_u8m2x4_u8m2(v_s, 1); - vuint8m2_t v_s2 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2); - vuint8m2_t v_s3 = __riscv_vget_v_u8m2x4_u8m2(v_s, 3); - - // Use round-to-nearest-up mode for vnclip & averaging add - if (src_stride == 0) { - v_ave0 = __riscv_vaaddu_vv_u8m2(v_s0, v_s0, __RISCV_VXRM_RNU, vl); - v_ave1 = __riscv_vaaddu_vv_u8m2(v_s1, v_s1, __RISCV_VXRM_RNU, vl); - v_ave2 = __riscv_vaaddu_vv_u8m2(v_s2, v_s2, __RISCV_VXRM_RNU, vl); - v_ave3 = __riscv_vaaddu_vv_u8m2(v_s3, v_s3, __RISCV_VXRM_RNU, vl); - } else { - vuint8m2x4_t v_t = __riscv_vlseg4e8_v_u8m2x4(t, vl); - vuint8m2_t v_t0 = __riscv_vget_v_u8m2x4_u8m2(v_t, 0); - vuint8m2_t v_t1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 1); - vuint8m2_t v_t2 = __riscv_vget_v_u8m2x4_u8m2(v_t, 2); - vuint8m2_t v_t3 = __riscv_vget_v_u8m2x4_u8m2(v_t, 3); - v_ave0 = __riscv_vaaddu_vv_u8m2(v_s0, v_t0, __RISCV_VXRM_RNU, vl); - v_ave1 = __riscv_vaaddu_vv_u8m2(v_s1, v_t1, __RISCV_VXRM_RNU, vl); - v_ave2 = __riscv_vaaddu_vv_u8m2(v_s2, v_t2, __RISCV_VXRM_RNU, vl); - v_ave3 = __riscv_vaaddu_vv_u8m2(v_s3, v_t3, __RISCV_VXRM_RNU, vl); - t += 4 * vl; - } - // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 - v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_ave1, 0, vl); - v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_ave0, vl); - v_a0 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, __RISCV_VXRM_RNU, vl); - - // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 - v_a1 = __riscv_vaaddu_vv_u8m2(v_ave1, v_ave2, __RISCV_VXRM_RNU, vl); - - // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 - v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_ave2, 0, vl); - v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_ave3, vl); - v_a2 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, __RISCV_VXRM_RNU, vl); - - v_dst = __riscv_vcreate_v_u8m2x3(v_a0, v_a1, v_a2); - __riscv_vsseg3e8_v_u8m2x3(dst_ptr, v_dst, vl); - - w -= vl; - s += 4 * vl; - dst_ptr += 3 * vl; - } while (w > 0); -} -#else -void ScaleRowDown34_1_Box_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - size_t w = (size_t)dst_width / 3u; - const uint8_t* s = src_ptr; - const uint8_t* t = src_ptr + src_stride; + dst_width /= 3; + const uint8_t* t; + int vl; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - vuint8m2_t v_s0, v_s1, v_s2, v_s3; - vuint8m2_t v_ave0, v_ave1, v_ave2, v_ave3; - vuint16m4_t v_u1_u16; - vuint8m2_t v_a0, v_a1, v_a2; - size_t vl = __riscv_vsetvl_e8m2(w); - __riscv_vlseg4e8_v_u8m2(&v_s0, &v_s1, &v_s2, &v_s3, s, vl); + asm volatile( + "add %[t], %[s], %[src_stride] \n" + "csrwi vxrm, 0 \n" - // Use round-to-nearest-up mode for vnclip & averaging add - if (src_stride == 0) { - v_ave0 = __riscv_vaaddu_vv_u8m2(v_s0, v_s0, vl); - v_ave1 = __riscv_vaaddu_vv_u8m2(v_s1, v_s1, vl); - v_ave2 = __riscv_vaaddu_vv_u8m2(v_s2, v_s2, vl); - v_ave3 = __riscv_vaaddu_vv_u8m2(v_s3, v_s3, vl); - } else { - vuint8m2_t v_t0, v_t1, v_t2, v_t3; - __riscv_vlseg4e8_v_u8m2(&v_t0, &v_t1, &v_t2, &v_t3, t, vl); - v_ave0 = __riscv_vaaddu_vv_u8m2(v_s0, v_t0, vl); - v_ave1 = __riscv_vaaddu_vv_u8m2(v_s1, v_t1, vl); - v_ave2 = __riscv_vaaddu_vv_u8m2(v_s2, v_t2, vl); - v_ave3 = __riscv_vaaddu_vv_u8m2(v_s3, v_t3, vl); - t += 4 * vl; - } - // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 - v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_ave1, 0, vl); - v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_ave0, vl); - v_a0 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, vl); + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v14, (%[s]) \n" + "bnez %[src_stride], 2f \n" + "vaaddu.vv v14, v14, v14 \n" + "vaaddu.vv v8, v16, v16 \n" + "vaaddu.vv v10, v18, v18 \n" + "vaaddu.vv v12, v20, v20 \n" + "j 3f \n" - // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 - v_a1 = __riscv_vaaddu_vv_u8m2(v_ave1, v_ave2, vl); + "2: \n" + "vlseg4e8.v v22, (%[t]) \n" + "vaaddu.vv v14, v14, v22 \n" + "vaaddu.vv v8, v16, v24 \n" + "vaaddu.vv v10, v18, v26 \n" + "vaaddu.vv v12, v20, v28 \n" - // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 - v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_ave2, 0, vl); - v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_ave3, vl); - v_a2 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, vl); - - __riscv_vsseg3e8_v_u8m2(dst_ptr, v_a0, v_a1, v_a2, vl); - - w -= vl; - s += 4 * vl; - dst_ptr += 3 * vl; - } while (w > 0); + "3: \n" + // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 + "vwcvtu.x.x.v v16, v8 \n" + "vwmaccu.vx v16, %[c3], v14 \n" + "vnclipu.wi v20, v16, 2 \n" + // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 + "vwcvtu.x.x.v v16, v10 \n" + "vwmaccu.vx v16, %[c3], v12 \n" + // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 + "vaaddu.vv v22, v8, v10 \n" + "vnclipu.wi v24, v16, 2 \n" + "vsseg3e8.v v20, (%[dst_ptr]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[s], %[s], %[vl] \n" + "add %[t], %[t], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [s] "+r"(src_ptr), // %[s] + [t] "=&r"(t), // %[t] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [vl] "=&r"(vl) // %[vl] + : [src_stride] "r"(src_stride), // %[src_stride] + [c3] "r"(3) // %[c3] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29"); } #endif -#endif #ifdef HAS_SCALEROWDOWN38_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleRowDown38_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - size_t w = (size_t)dst_width / 3u; (void)src_stride; assert(dst_width % 3 == 0); - do { - size_t vl = __riscv_vsetvl_e8m1(w); - vuint8m1x8_t v_src = __riscv_vlseg8e8_v_u8m1x8(src_ptr, vl); - vuint8m1_t v_s0 = __riscv_vget_v_u8m1x8_u8m1(v_src, 0); - vuint8m1_t v_s3 = __riscv_vget_v_u8m1x8_u8m1(v_src, 3); - vuint8m1_t v_s6 = __riscv_vget_v_u8m1x8_u8m1(v_src, 6); - vuint8m1x3_t v_dst = __riscv_vcreate_v_u8m1x3(v_s0, v_s3, v_s6); - __riscv_vsseg3e8_v_u8m1x3(dst_ptr, v_dst, vl); - w -= vl; - src_ptr += 8 * vl; - dst_ptr += 3 * vl; - } while (w > 0); + dst_width /= 3; + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, m1, ta, ma \n" + "vlseg8e8.v v8, (%[src_ptr]) \n" + "vmv1r.v v9, v11 \n" + "vmv1r.v v10, v14 \n" + "vsseg3e8.v v8, (%[dst_ptr]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } -#else -void ScaleRowDown38_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - size_t w = (size_t)dst_width / 3u; - (void)src_stride; - assert(dst_width % 3 == 0); - do { - vuint8m1_t v_s0, v_s1, v_s2, v_s3, v_s4, v_s5, v_s6, v_s7; - size_t vl = __riscv_vsetvl_e8m1(w); - __riscv_vlseg8e8_v_u8m1(&v_s0, &v_s1, &v_s2, &v_s3, &v_s4, &v_s5, &v_s6, - &v_s7, src_ptr, vl); - __riscv_vsseg3e8_v_u8m1(dst_ptr, v_s0, v_s3, v_s6, vl); - w -= vl; - src_ptr += 8 * vl; - dst_ptr += 3 * vl; - } while (w > 0); -} -#endif #endif #ifdef HAS_SCALEROWDOWN38_2_BOX_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleRowDown38_2_Box_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - size_t w = (size_t)dst_width / 3u; - const uint16_t coeff_a = (65536u / 6u); - const uint16_t coeff_b = (65536u / 4u); assert((dst_width % 3 == 0) && (dst_width > 0)); - do { - vuint16m2_t v_e, v_f, v_g; - vuint8m1_t v_dst_e, v_dst_f, v_dst_g; - vuint8m1x3_t v_dst; - size_t vl = __riscv_vsetvl_e8m1(w); - // s: e00, e10, e20, f00, f10, f20, g00, g10 - vuint8m1x8_t v_s = __riscv_vlseg8e8_v_u8m1x8(src_ptr, vl); - vuint8m1_t v_s0 = __riscv_vget_v_u8m1x8_u8m1(v_s, 0); - vuint8m1_t v_s1 = __riscv_vget_v_u8m1x8_u8m1(v_s, 1); - vuint8m1_t v_s2 = __riscv_vget_v_u8m1x8_u8m1(v_s, 2); - vuint8m1_t v_s3 = __riscv_vget_v_u8m1x8_u8m1(v_s, 3); - vuint8m1_t v_s4 = __riscv_vget_v_u8m1x8_u8m1(v_s, 4); - vuint8m1_t v_s5 = __riscv_vget_v_u8m1x8_u8m1(v_s, 5); - vuint8m1_t v_s6 = __riscv_vget_v_u8m1x8_u8m1(v_s, 6); - vuint8m1_t v_s7 = __riscv_vget_v_u8m1x8_u8m1(v_s, 7); - // t: e01, e11, e21, f01, f11, f21, g01, g11 - vuint8m1x8_t v_t = __riscv_vlseg8e8_v_u8m1x8(src_ptr + src_stride, vl); - vuint8m1_t v_t0 = __riscv_vget_v_u8m1x8_u8m1(v_t, 0); - vuint8m1_t v_t1 = __riscv_vget_v_u8m1x8_u8m1(v_t, 1); - vuint8m1_t v_t2 = __riscv_vget_v_u8m1x8_u8m1(v_t, 2); - vuint8m1_t v_t3 = __riscv_vget_v_u8m1x8_u8m1(v_t, 3); - vuint8m1_t v_t4 = __riscv_vget_v_u8m1x8_u8m1(v_t, 4); - vuint8m1_t v_t5 = __riscv_vget_v_u8m1x8_u8m1(v_t, 5); - vuint8m1_t v_t6 = __riscv_vget_v_u8m1x8_u8m1(v_t, 6); - vuint8m1_t v_t7 = __riscv_vget_v_u8m1x8_u8m1(v_t, 7); - // Calculate sum of [e00, e21] to v_e - // Calculate sum of [f00, f21] to v_f - // Calculate sum of [g00, g11] to v_g - vuint16m2_t v_e0 = __riscv_vwaddu_vv_u16m2(v_s0, v_t0, vl); - vuint16m2_t v_e1 = __riscv_vwaddu_vv_u16m2(v_s1, v_t1, vl); - vuint16m2_t v_e2 = __riscv_vwaddu_vv_u16m2(v_s2, v_t2, vl); - vuint16m2_t v_f0 = __riscv_vwaddu_vv_u16m2(v_s3, v_t3, vl); - vuint16m2_t v_f1 = __riscv_vwaddu_vv_u16m2(v_s4, v_t4, vl); - vuint16m2_t v_f2 = __riscv_vwaddu_vv_u16m2(v_s5, v_t5, vl); - vuint16m2_t v_g0 = __riscv_vwaddu_vv_u16m2(v_s6, v_t6, vl); - vuint16m2_t v_g1 = __riscv_vwaddu_vv_u16m2(v_s7, v_t7, vl); + dst_width /= 3; + int vl; + asm volatile( + "add %[src_ptr1], %[src_ptr], %[src_ptr1]\n" - v_e0 = __riscv_vadd_vv_u16m2(v_e0, v_e1, vl); - v_f0 = __riscv_vadd_vv_u16m2(v_f0, v_f1, vl); - v_e = __riscv_vadd_vv_u16m2(v_e0, v_e2, vl); - v_f = __riscv_vadd_vv_u16m2(v_f0, v_f2, vl); - v_g = __riscv_vadd_vv_u16m2(v_g0, v_g1, vl); - - // Average in 16-bit fixed-point - v_e = __riscv_vmulhu_vx_u16m2(v_e, coeff_a, vl); - v_f = __riscv_vmulhu_vx_u16m2(v_f, coeff_a, vl); - v_g = __riscv_vmulhu_vx_u16m2(v_g, coeff_b, vl); - v_dst_e = __riscv_vnsrl_wx_u8m1(v_e, 0, vl); - v_dst_f = __riscv_vnsrl_wx_u8m1(v_f, 0, vl); - v_dst_g = __riscv_vnsrl_wx_u8m1(v_g, 0, vl); - - v_dst = __riscv_vcreate_v_u8m1x3(v_dst_e, v_dst_f, v_dst_g); - __riscv_vsseg3e8_v_u8m1x3(dst_ptr, v_dst, vl); - w -= vl; - src_ptr += 8 * vl; - dst_ptr += 3 * vl; - } while (w > 0); + "1: \n" + "vsetvli %[vl], %[w], e8, m1, ta, ma \n" + // s: e00, e10, e20, f00, f10, f20, g00, g10 + "vlseg8e8.v v14, (%[src_ptr]) \n" + // t: e01, e11, e21, f01, f11, f21, g01, g11 + "vlseg8e8.v v22, (%[src_ptr1]) \n" + "vwaddu.vv v8, v14, v22 \n" + "vwaddu.vv v10, v15, v23 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vv v8, v8, v10 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vwaddu.vv v10, v17, v25 \n" + "vwaddu.vv v12, v18, v26 \n" + "vwaddu.vv v14, v16, v24 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vv v10, v10, v12 \n" + "vadd.vv v8, v8, v14 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vwaddu.vv v12, v19, v27 \n" + "vwaddu.vv v14, v20, v28 \n" + "vwaddu.vv v16, v21, v29 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vv v10, v10, v12 \n" + "vadd.vv v12, v14, v16 \n" + // Average in 16-bit fixed-point + "vmulhu.vx v8, v8, %[coeff_a] \n" + "vmulhu.vx v10, v10, %[coeff_a] \n" + "vmulhu.vx v12, v12, %[coeff_b] \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v14, v8, 0 \n" + "vnsrl.wi v15, v10, 0 \n" + "vnsrl.wi v16, v12, 0 \n" + "vsseg3e8.v v14, (%[dst_ptr]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "add %[src_ptr1], %[src_ptr1], %[vl]\n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_stride), // %[src_ptr1] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [vl] "=&r"(vl) // %[vl] + : [coeff_a] "r"(65536u / 6u), // %[coeff_a] + [coeff_b] "r"(65536u / 4u) // %[coeff_b] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29"); } -#else -void ScaleRowDown38_2_Box_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - size_t w = (size_t)dst_width / 3u; - const uint16_t coeff_a = (65536u / 6u); - const uint16_t coeff_b = (65536u / 4u); - assert((dst_width % 3 == 0) && (dst_width > 0)); - do { - vuint8m1_t v_s0, v_s1, v_s2, v_s3, v_s4, v_s5, v_s6, v_s7; - vuint8m1_t v_t0, v_t1, v_t2, v_t3, v_t4, v_t5, v_t6, v_t7; - vuint16m2_t v_e0, v_e1, v_e2, v_e; - vuint16m2_t v_f0, v_f1, v_f2, v_f; - vuint16m2_t v_g0, v_g1, v_g; - vuint8m1_t v_dst_e, v_dst_f, v_dst_g; - size_t vl = __riscv_vsetvl_e8m1(w); - // s: e00, e10, e20, f00, f10, f20, g00, g10 - // t: e01, e11, e21, f01, f11, f21, g01, g11 - __riscv_vlseg8e8_v_u8m1(&v_s0, &v_s1, &v_s2, &v_s3, &v_s4, &v_s5, &v_s6, - &v_s7, src_ptr, vl); - __riscv_vlseg8e8_v_u8m1(&v_t0, &v_t1, &v_t2, &v_t3, &v_t4, &v_t5, &v_t6, - &v_t7, src_ptr + src_stride, vl); - // Calculate sum of [e00, e21] to v_e - // Calculate sum of [f00, f21] to v_f - // Calculate sum of [g00, g11] to v_g - v_e0 = __riscv_vwaddu_vv_u16m2(v_s0, v_t0, vl); - v_e1 = __riscv_vwaddu_vv_u16m2(v_s1, v_t1, vl); - v_e2 = __riscv_vwaddu_vv_u16m2(v_s2, v_t2, vl); - v_f0 = __riscv_vwaddu_vv_u16m2(v_s3, v_t3, vl); - v_f1 = __riscv_vwaddu_vv_u16m2(v_s4, v_t4, vl); - v_f2 = __riscv_vwaddu_vv_u16m2(v_s5, v_t5, vl); - v_g0 = __riscv_vwaddu_vv_u16m2(v_s6, v_t6, vl); - v_g1 = __riscv_vwaddu_vv_u16m2(v_s7, v_t7, vl); - - v_e0 = __riscv_vadd_vv_u16m2(v_e0, v_e1, vl); - v_f0 = __riscv_vadd_vv_u16m2(v_f0, v_f1, vl); - v_e = __riscv_vadd_vv_u16m2(v_e0, v_e2, vl); - v_f = __riscv_vadd_vv_u16m2(v_f0, v_f2, vl); - v_g = __riscv_vadd_vv_u16m2(v_g0, v_g1, vl); - - // Average in 16-bit fixed-point - v_e = __riscv_vmulhu_vx_u16m2(v_e, coeff_a, vl); - v_f = __riscv_vmulhu_vx_u16m2(v_f, coeff_a, vl); - v_g = __riscv_vmulhu_vx_u16m2(v_g, coeff_b, vl); - - v_dst_e = __riscv_vnsrl_wx_u8m1(v_e, 0, vl); - v_dst_f = __riscv_vnsrl_wx_u8m1(v_f, 0, vl); - v_dst_g = __riscv_vnsrl_wx_u8m1(v_g, 0, vl); - - __riscv_vsseg3e8_v_u8m1(dst_ptr, v_dst_e, v_dst_f, v_dst_g, vl); - w -= vl; - src_ptr += 8 * vl; - dst_ptr += 3 * vl; - } while (w > 0); -} -#endif #endif #ifdef HAS_SCALEROWDOWN38_3_BOX_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleRowDown38_3_Box_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - size_t w = (size_t)dst_width / 3u; - const uint16_t coeff_a = (65536u / 9u); - const uint16_t coeff_b = (65536u / 6u); assert((dst_width % 3 == 0) && (dst_width > 0)); - do { - vuint16m2_t v_e0, v_e1, v_e2, v_e3, v_e4, v_e; - vuint16m2_t v_f0, v_f1, v_f2, v_f3, v_f4, v_f; - vuint16m2_t v_g0, v_g1, v_g2, v_g; - vuint8m1_t v_dst_e, v_dst_f, v_dst_g; - vuint8m1x3_t v_dst; - size_t vl = __riscv_vsetvl_e8m1(w); - // s: e00, e10, e20, f00, f10, f20, g00, g10 - vuint8m1x8_t v_s = __riscv_vlseg8e8_v_u8m1x8(src_ptr, vl); - vuint8m1_t v_s0 = __riscv_vget_v_u8m1x8_u8m1(v_s, 0); - vuint8m1_t v_s1 = __riscv_vget_v_u8m1x8_u8m1(v_s, 1); - vuint8m1_t v_s2 = __riscv_vget_v_u8m1x8_u8m1(v_s, 2); - vuint8m1_t v_s3 = __riscv_vget_v_u8m1x8_u8m1(v_s, 3); - vuint8m1_t v_s4 = __riscv_vget_v_u8m1x8_u8m1(v_s, 4); - vuint8m1_t v_s5 = __riscv_vget_v_u8m1x8_u8m1(v_s, 5); - vuint8m1_t v_s6 = __riscv_vget_v_u8m1x8_u8m1(v_s, 6); - vuint8m1_t v_s7 = __riscv_vget_v_u8m1x8_u8m1(v_s, 7); - // t: e01, e11, e21, f01, f11, f21, g01, g11 - vuint8m1x8_t v_t = __riscv_vlseg8e8_v_u8m1x8(src_ptr + src_stride, vl); - vuint8m1_t v_t0 = __riscv_vget_v_u8m1x8_u8m1(v_t, 0); - vuint8m1_t v_t1 = __riscv_vget_v_u8m1x8_u8m1(v_t, 1); - vuint8m1_t v_t2 = __riscv_vget_v_u8m1x8_u8m1(v_t, 2); - vuint8m1_t v_t3 = __riscv_vget_v_u8m1x8_u8m1(v_t, 3); - vuint8m1_t v_t4 = __riscv_vget_v_u8m1x8_u8m1(v_t, 4); - vuint8m1_t v_t5 = __riscv_vget_v_u8m1x8_u8m1(v_t, 5); - vuint8m1_t v_t6 = __riscv_vget_v_u8m1x8_u8m1(v_t, 6); - vuint8m1_t v_t7 = __riscv_vget_v_u8m1x8_u8m1(v_t, 7); - // u: e02, e12, e22, f02, f12, f22, g02, g12 - vuint8m1x8_t v_u = __riscv_vlseg8e8_v_u8m1x8(src_ptr + 2 * src_stride, vl); - vuint8m1_t v_u0 = __riscv_vget_v_u8m1x8_u8m1(v_u, 0); - vuint8m1_t v_u1 = __riscv_vget_v_u8m1x8_u8m1(v_u, 1); - vuint8m1_t v_u2 = __riscv_vget_v_u8m1x8_u8m1(v_u, 2); - vuint8m1_t v_u3 = __riscv_vget_v_u8m1x8_u8m1(v_u, 3); - vuint8m1_t v_u4 = __riscv_vget_v_u8m1x8_u8m1(v_u, 4); - vuint8m1_t v_u5 = __riscv_vget_v_u8m1x8_u8m1(v_u, 5); - vuint8m1_t v_u6 = __riscv_vget_v_u8m1x8_u8m1(v_u, 6); - vuint8m1_t v_u7 = __riscv_vget_v_u8m1x8_u8m1(v_u, 7); - // Calculate sum of [e00, e22] - v_e0 = __riscv_vwaddu_vv_u16m2(v_s0, v_t0, vl); - v_e1 = __riscv_vwaddu_vv_u16m2(v_s1, v_t1, vl); - v_e2 = __riscv_vwaddu_vv_u16m2(v_s2, v_t2, vl); - v_e3 = __riscv_vwaddu_vv_u16m2(v_u0, v_u1, vl); - v_e4 = __riscv_vwaddu_vx_u16m2(v_u2, 0, vl); + dst_width /= 3; + const uint8_t* src_ptr1; + int vl; + asm volatile( + "add %[src_ptr1], %[src_ptr], %[src_ptr2]\n" + "add %[src_ptr2], %[src_ptr1], %[src_ptr2]\n" - v_e0 = __riscv_vadd_vv_u16m2(v_e0, v_e1, vl); - v_e2 = __riscv_vadd_vv_u16m2(v_e2, v_e3, vl); - v_e0 = __riscv_vadd_vv_u16m2(v_e0, v_e4, vl); - v_e = __riscv_vadd_vv_u16m2(v_e0, v_e2, vl); - // Calculate sum of [f00, f22] - v_f0 = __riscv_vwaddu_vv_u16m2(v_s3, v_t3, vl); - v_f1 = __riscv_vwaddu_vv_u16m2(v_s4, v_t4, vl); - v_f2 = __riscv_vwaddu_vv_u16m2(v_s5, v_t5, vl); - v_f3 = __riscv_vwaddu_vv_u16m2(v_u3, v_u4, vl); - v_f4 = __riscv_vwaddu_vx_u16m2(v_u5, 0, vl); - - v_f0 = __riscv_vadd_vv_u16m2(v_f0, v_f1, vl); - v_f2 = __riscv_vadd_vv_u16m2(v_f2, v_f3, vl); - v_f0 = __riscv_vadd_vv_u16m2(v_f0, v_f4, vl); - v_f = __riscv_vadd_vv_u16m2(v_f0, v_f2, vl); - // Calculate sum of [g00, g12] - v_g0 = __riscv_vwaddu_vv_u16m2(v_s6, v_t6, vl); - v_g1 = __riscv_vwaddu_vv_u16m2(v_s7, v_t7, vl); - v_g2 = __riscv_vwaddu_vv_u16m2(v_u6, v_u7, vl); - - v_g = __riscv_vadd_vv_u16m2(v_g0, v_g1, vl); - v_g = __riscv_vadd_vv_u16m2(v_g, v_g2, vl); - - // Average in 16-bit fixed-point - v_e = __riscv_vmulhu_vx_u16m2(v_e, coeff_a, vl); - v_f = __riscv_vmulhu_vx_u16m2(v_f, coeff_a, vl); - v_g = __riscv_vmulhu_vx_u16m2(v_g, coeff_b, vl); - v_dst_e = __riscv_vnsrl_wx_u8m1(v_e, 0, vl); - v_dst_f = __riscv_vnsrl_wx_u8m1(v_f, 0, vl); - v_dst_g = __riscv_vnsrl_wx_u8m1(v_g, 0, vl); - - v_dst = __riscv_vcreate_v_u8m1x3(v_dst_e, v_dst_f, v_dst_g); - __riscv_vsseg3e8_v_u8m1x3(dst_ptr, v_dst, vl); - w -= vl; - src_ptr += 8 * vl; - dst_ptr += 3 * vl; - } while (w > 0); + "1: \n" + "vsetvli %[vl], %[w], e8, m1, ta, ma \n" + // s: e00, e10, e20, f00, f10, f20, g00, g10 + "vlseg8e8.v v8, (%[src_ptr]) \n" + // t: e01, e11, e21, f01, f11, f21, g01, g11 + "vlseg8e8.v v19, (%[src_ptr1]) \n" + // u: e02, e12, e22, f02, f12, f22, g02, g12 + "vlseg8e8.v v0, (%[src_ptr2]) \n" + // Calculate sum of [e00, e22] + "vwaddu.vv v16, v8, v19 \n" + "vwaddu.vv v18, v9, v20 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vv v8, v16, v18 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vwaddu.vv v16, v10, v21 \n" + "vwaddu.vv v18, v0, v1 \n" + "vwcvtu.x.x.v v20, v2 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vv v16, v16, v18 \n" + "vadd.vv v8, v8, v20 \n" + "vadd.vv v8, v8, v16 \n" + // Calculate sum of [f00, f22] + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vwaddu.vv v16, v11, v22 \n" + "vwaddu.vv v10, v12, v23 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vv v10, v16, v10 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vwaddu.vv v16, v13, v24 \n" + "vwaddu.vv v12, v3, v4 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vv v12, v16, v12 \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vwcvtu.x.x.v v16, v5 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vv v10, v10, v16 \n" + "vadd.vv v10, v10, v12 \n" + // Calculate sum of [g00, g12] + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vwaddu.vv v12, v14, v25 \n" + "vwaddu.vv v16, v15, v26 \n" + "vwaddu.vv v14, v6, v7 \n" + "vsetvli zero, zero, e16, m2, ta, ma \n" + "vadd.vv v12, v12, v16 \n" + "vadd.vv v12, v12, v14 \n" + // Average in 16-bit fixed-point + "vmulhu.vx v8, v8, %[coeff_a] \n" + "vmulhu.vx v10, v10, %[coeff_a] \n" + "vmulhu.vx v12, v12, %[coeff_b] \n" + "vsetvli zero, zero, e8, m1, ta, ma \n" + "vnsrl.wi v14, v8, 0 \n" + "vnsrl.wi v15, v10, 0 \n" + "vnsrl.wi v16, v12, 0 \n" + "vsseg3e8.v v14, (%[dst_ptr]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_ptr], %[dst_ptr], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "add %[src_ptr1], %[src_ptr1], %[vl]\n" + "add %[src_ptr2], %[src_ptr2], %[vl]\n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "=&r"(src_ptr1), // %[src_ptr1] + [src_ptr2] "+r"(src_stride), // %[src_ptr2] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [vl] "=&r"(vl) // %[vl] + : [coeff_a] "r"(65536u / 9u), // %[coeff_a] + [coeff_b] "r"(65536u / 6u) // %[coeff_b] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26"); } -#else -void ScaleRowDown38_3_Box_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - size_t w = (size_t)dst_width / 3u; - const uint16_t coeff_a = (65536u / 9u); - const uint16_t coeff_b = (65536u / 6u); - assert((dst_width % 3 == 0) && (dst_width > 0)); - do { - vuint8m1_t v_s0, v_s1, v_s2, v_s3, v_s4, v_s5, v_s6, v_s7; - vuint8m1_t v_t0, v_t1, v_t2, v_t3, v_t4, v_t5, v_t6, v_t7; - vuint8m1_t v_u0, v_u1, v_u2, v_u3, v_u4, v_u5, v_u6, v_u7; - vuint16m2_t v_e0, v_e1, v_e2, v_e3, v_e4, v_e; - vuint16m2_t v_f0, v_f1, v_f2, v_f3, v_f4, v_f; - vuint16m2_t v_g0, v_g1, v_g2, v_g; - vuint8m1_t v_dst_e, v_dst_f, v_dst_g; - size_t vl = __riscv_vsetvl_e8m1(w); - // s: e00, e10, e20, f00, f10, f20, g00, g10 - // t: e01, e11, e21, f01, f11, f21, g01, g11 - // u: e02, e12, e22, f02, f12, f22, g02, g12 - __riscv_vlseg8e8_v_u8m1(&v_s0, &v_s1, &v_s2, &v_s3, &v_s4, &v_s5, &v_s6, - &v_s7, src_ptr, vl); - __riscv_vlseg8e8_v_u8m1(&v_t0, &v_t1, &v_t2, &v_t3, &v_t4, &v_t5, &v_t6, - &v_t7, src_ptr + src_stride, vl); - __riscv_vlseg8e8_v_u8m1(&v_u0, &v_u1, &v_u2, &v_u3, &v_u4, &v_u5, &v_u6, - &v_u7, src_ptr + 2 * src_stride, vl); - // Calculate sum of [e00, e22] - v_e0 = __riscv_vwaddu_vv_u16m2(v_s0, v_t0, vl); - v_e1 = __riscv_vwaddu_vv_u16m2(v_s1, v_t1, vl); - v_e2 = __riscv_vwaddu_vv_u16m2(v_s2, v_t2, vl); - v_e3 = __riscv_vwaddu_vv_u16m2(v_u0, v_u1, vl); - v_e4 = __riscv_vwaddu_vx_u16m2(v_u2, 0, vl); - - v_e0 = __riscv_vadd_vv_u16m2(v_e0, v_e1, vl); - v_e2 = __riscv_vadd_vv_u16m2(v_e2, v_e3, vl); - v_e0 = __riscv_vadd_vv_u16m2(v_e0, v_e4, vl); - v_e = __riscv_vadd_vv_u16m2(v_e0, v_e2, vl); - // Calculate sum of [f00, f22] - v_f0 = __riscv_vwaddu_vv_u16m2(v_s3, v_t3, vl); - v_f1 = __riscv_vwaddu_vv_u16m2(v_s4, v_t4, vl); - v_f2 = __riscv_vwaddu_vv_u16m2(v_s5, v_t5, vl); - v_f3 = __riscv_vwaddu_vv_u16m2(v_u3, v_u4, vl); - v_f4 = __riscv_vwaddu_vx_u16m2(v_u5, 0, vl); - - v_f0 = __riscv_vadd_vv_u16m2(v_f0, v_f1, vl); - v_f2 = __riscv_vadd_vv_u16m2(v_f2, v_f3, vl); - v_f0 = __riscv_vadd_vv_u16m2(v_f0, v_f4, vl); - v_f = __riscv_vadd_vv_u16m2(v_f0, v_f2, vl); - // Calculate sum of [g00, g12] - v_g0 = __riscv_vwaddu_vv_u16m2(v_s6, v_t6, vl); - v_g1 = __riscv_vwaddu_vv_u16m2(v_s7, v_t7, vl); - v_g2 = __riscv_vwaddu_vv_u16m2(v_u6, v_u7, vl); - - v_g = __riscv_vadd_vv_u16m2(v_g0, v_g1, vl); - v_g = __riscv_vadd_vv_u16m2(v_g, v_g2, vl); - - // Average in 16-bit fixed-point - v_e = __riscv_vmulhu_vx_u16m2(v_e, coeff_a, vl); - v_f = __riscv_vmulhu_vx_u16m2(v_f, coeff_a, vl); - v_g = __riscv_vmulhu_vx_u16m2(v_g, coeff_b, vl); - - v_dst_e = __riscv_vnsrl_wx_u8m1(v_e, 0, vl); - v_dst_f = __riscv_vnsrl_wx_u8m1(v_f, 0, vl); - v_dst_g = __riscv_vnsrl_wx_u8m1(v_g, 0, vl); - __riscv_vsseg3e8_v_u8m1(dst_ptr, v_dst_e, v_dst_f, v_dst_g, vl); - w -= vl; - src_ptr += 8 * vl; - dst_ptr += 3 * vl; - } while (w > 0); -} -#endif #endif // ScaleUVRowUp2_(Bi)linear_RVV function is equal to other platforms' @@ -1303,401 +834,225 @@ void ScaleRowDown38_3_Box_RVV(const uint8_t* src_ptr, // platforms only implement non-edge part of image and process edge with scalar. #ifdef HAS_SCALEROWUP2_LINEAR_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleRowUp2_Linear_RVV(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width) { - size_t work_width = (size_t)dst_width - 1u; - size_t src_width = work_width >> 1u; - const uint8_t* work_src_ptr = src_ptr; - uint8_t* work_dst_ptr = dst_ptr + 1; - size_t vl = __riscv_vsetvlmax_e8m4(); - vuint8m4_t v_3 = __riscv_vmv_v_x_u8m4(3, vl); + int src_width = (dst_width - 1) >> 1; dst_ptr[0] = src_ptr[0]; - while (src_width > 0) { - vuint8m4_t v_src0, v_src1, v_dst_odd, v_dst_even; - vuint16m8_t v_src0_u16, v_src1_u16; - vuint8m4x2_t v_dst; - size_t vl = __riscv_vsetvl_e8m4(src_width); - v_src0 = __riscv_vle8_v_u8m4(work_src_ptr, vl); - v_src1 = __riscv_vle8_v_u8m4(work_src_ptr + 1, vl); - - v_src0_u16 = __riscv_vwaddu_vx_u16m8(v_src0, 2, vl); - v_src1_u16 = __riscv_vwaddu_vx_u16m8(v_src1, 2, vl); - v_src0_u16 = __riscv_vwmaccu_vv_u16m8(v_src0_u16, v_3, v_src1, vl); - v_src1_u16 = __riscv_vwmaccu_vv_u16m8(v_src1_u16, v_3, v_src0, vl); - - v_dst_odd = __riscv_vnsrl_wx_u8m4(v_src0_u16, 2, vl); - v_dst_even = __riscv_vnsrl_wx_u8m4(v_src1_u16, 2, vl); - - v_dst = __riscv_vcreate_v_u8m4x2(v_dst_even, v_dst_odd); - __riscv_vsseg2e8_v_u8m4x2(work_dst_ptr, v_dst, vl); - - src_width -= vl; - work_src_ptr += vl; - work_dst_ptr += 2 * vl; + if (src_width > 0) { + int vl; + const uint8_t* work_src_next; + uint8_t* work_dst_ptr = dst_ptr + 1; + asm volatile( + "1: \n" + "vsetvli %[vl], %[src_width], e8, m2, ta, ma\n" + "vle8.v v12, (%[src_ptr]) \n" + "addi %[work_src_next], %[src_ptr], 1\n" + "vle8.v v24, (%[work_src_next]) \n" + "vwaddu.vx v16, v12, %[c2] \n" + "vwmaccu.vx v16, %[c3], v24 \n" + "vnsrl.wi v26, v16, 2 \n" + "vwaddu.vx v16, v24, %[c2] \n" + "vwmaccu.vx v16, %[c3], v12 \n" + "vnsrl.wi v24, v16, 2 \n" + "vsseg2e8.v v24, (%[work_dst_ptr]) \n" + "sub %[src_width], %[src_width], %[vl]\n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[work_dst_ptr], %[work_dst_ptr], %[vl]\n" + "bgtz %[src_width], 1b \n" + : [src_width] "+r"(src_width), // %[src_width] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [work_dst_ptr] "+r"(work_dst_ptr), // %[work_dst_ptr] + [vl] "=&r"(vl), // %[vl] + [work_src_next] "=&r"(work_src_next) // %[work_src_next] + : [c2] "r"(2), // %[c2] + [c3] "r"(3) // %[c3] + : "vl", "vtype", "memory", "v12", "v13", "v16", "v17", "v18", "v19", + "v24", "v25", "v26", "v27"); } - dst_ptr[dst_width - 1] = src_ptr[(dst_width - 1) / 2]; + dst_ptr[dst_width - 1] = src_ptr[0]; } -#else -void ScaleRowUp2_Linear_RVV(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int dst_width) { - size_t work_width = (size_t)dst_width - 1u; - size_t src_width = work_width >> 1u; - const uint8_t* work_src_ptr = src_ptr; - uint8_t* work_dst_ptr = dst_ptr + 1; - size_t vl = __riscv_vsetvlmax_e8m4(); - vuint8m4_t v_3 = __riscv_vmv_v_x_u8m4(3, vl); - dst_ptr[0] = src_ptr[0]; - while (src_width > 0) { - vuint8m4_t v_src0, v_src1, v_dst_odd, v_dst_even; - vuint16m8_t v_src0_u16, v_src1_u16; - size_t vl = __riscv_vsetvl_e8m4(src_width); - v_src0 = __riscv_vle8_v_u8m4(work_src_ptr, vl); - v_src1 = __riscv_vle8_v_u8m4(work_src_ptr + 1, vl); - - v_src0_u16 = __riscv_vwaddu_vx_u16m8(v_src0, 2, vl); - v_src1_u16 = __riscv_vwaddu_vx_u16m8(v_src1, 2, vl); - v_src0_u16 = __riscv_vwmaccu_vv_u16m8(v_src0_u16, v_3, v_src1, vl); - v_src1_u16 = __riscv_vwmaccu_vv_u16m8(v_src1_u16, v_3, v_src0, vl); - - v_dst_odd = __riscv_vnsrl_wx_u8m4(v_src0_u16, 2, vl); - v_dst_even = __riscv_vnsrl_wx_u8m4(v_src1_u16, 2, vl); - - __riscv_vsseg2e8_v_u8m4(work_dst_ptr, v_dst_even, v_dst_odd, vl); - - src_width -= vl; - work_src_ptr += vl; - work_dst_ptr += 2 * vl; - } - dst_ptr[dst_width - 1] = src_ptr[(dst_width - 1) / 2]; -} -#endif #endif #ifdef HAS_SCALEROWUP2_BILINEAR_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleRowUp2_Bilinear_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) { - size_t work_width = ((size_t)dst_width - 1u) & ~1u; - size_t src_width = work_width >> 1u; - const uint8_t* work_s = src_ptr; - const uint8_t* work_t = src_ptr + src_stride; - const uint8_t* s = work_s; - const uint8_t* t = work_t; - uint8_t* d = dst_ptr; + int src_width = (dst_width - 1) >> 1; + const uint8_t* t = src_ptr + src_stride; uint8_t* e = dst_ptr + dst_stride; - uint8_t* work_d = d + 1; - uint8_t* work_e = e + 1; - size_t vl = __riscv_vsetvlmax_e16m4(); - vuint16m4_t v_3_u16 = __riscv_vmv_v_x_u16m4(3, vl); - vuint8m2_t v_3_u8 = __riscv_vmv_v_x_u8m2(3, vl); - d[0] = (3 * s[0] + t[0] + 2) >> 2; - e[0] = (s[0] + 3 * t[0] + 2) >> 2; - while (src_width > 0) { - vuint8m2_t v_s0, v_s1, v_t0, v_t1; - vuint16m4_t v_s0_u16, v_s1_u16, v_t0_u16, v_t1_u16; - vuint16m4_t v_t0_u16_, v_t1_u16_; - vuint8m2_t v_dst0_even, v_dst0_odd, v_dst1_even, v_dst1_odd; - vuint8m2x2_t v_dst0, v_dst1; - size_t vl = __riscv_vsetvl_e8m2(src_width); - v_s0 = __riscv_vle8_v_u8m2(work_s, vl); - v_s1 = __riscv_vle8_v_u8m2(work_s + 1, vl); - - v_s0_u16 = __riscv_vwaddu_vx_u16m4(v_s0, 2, vl); - v_s1_u16 = __riscv_vwaddu_vx_u16m4(v_s1, 2, vl); - v_s0_u16 = __riscv_vwmaccu_vv_u16m4(v_s0_u16, v_3_u8, v_s1, vl); - v_s1_u16 = __riscv_vwmaccu_vv_u16m4(v_s1_u16, v_3_u8, v_s0, vl); - - v_t0 = __riscv_vle8_v_u8m2(work_t, vl); - v_t1 = __riscv_vle8_v_u8m2(work_t + 1, vl); - - v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_t0, 2, vl); - v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_t1, 2, vl); - v_t0_u16 = __riscv_vwmaccu_vv_u16m4(v_t0_u16, v_3_u8, v_t1, vl); - v_t1_u16 = __riscv_vwmaccu_vv_u16m4(v_t1_u16, v_3_u8, v_t0, vl); - - v_t0_u16_ = __riscv_vmv_v_v_u16m4(v_t0_u16, vl); - v_t1_u16_ = __riscv_vmv_v_v_u16m4(v_t1_u16, vl); - - v_t0_u16 = __riscv_vmacc_vv_u16m4(v_t0_u16, v_3_u16, v_s0_u16, vl); - v_t1_u16 = __riscv_vmacc_vv_u16m4(v_t1_u16, v_3_u16, v_s1_u16, vl); - v_s0_u16 = __riscv_vmacc_vv_u16m4(v_s0_u16, v_3_u16, v_t0_u16_, vl); - v_s1_u16 = __riscv_vmacc_vv_u16m4(v_s1_u16, v_3_u16, v_t1_u16_, vl); - - v_dst0_odd = __riscv_vnsrl_wx_u8m2(v_t0_u16, 4, vl); - v_dst0_even = __riscv_vnsrl_wx_u8m2(v_t1_u16, 4, vl); - v_dst1_odd = __riscv_vnsrl_wx_u8m2(v_s0_u16, 4, vl); - v_dst1_even = __riscv_vnsrl_wx_u8m2(v_s1_u16, 4, vl); - - v_dst0 = __riscv_vcreate_v_u8m2x2(v_dst0_even, v_dst0_odd); - __riscv_vsseg2e8_v_u8m2x2(work_d, v_dst0, vl); - v_dst1 = __riscv_vcreate_v_u8m2x2(v_dst1_even, v_dst1_odd); - __riscv_vsseg2e8_v_u8m2x2(work_e, v_dst1, vl); - src_width -= vl; - work_s += vl; - work_t += vl; - work_d += 2 * vl; - work_e += 2 * vl; + dst_ptr[0] = (3 * src_ptr[0] + t[0] + 2) >> 2; + e[0] = (src_ptr[0] + 3 * t[0] + 2) >> 2; + if (src_width > 0) { + int vl; + const uint8_t* tmp_ptr; + uint8_t* work_d = dst_ptr + 1; + uint8_t* work_e = e + 1; + asm volatile( + "1: \n" + "vsetvli %[vl], %[src_width], e8, m2, ta, ma\n" + "vle8.v v0, (%[work_s]) \n" + "addi %[tmp_ptr], %[work_s], 1 \n" + "vle8.v v2, (%[tmp_ptr]) \n" + "vwaddu.vx v8, v0, %[c2] \n" + "vwaddu.vx v12, v2, %[c2] \n" + "vwmaccu.vx v8, %[c3], v2 \n" + "vwmaccu.vx v12, %[c3], v0 \n" + "vle8.v v4, (%[work_t]) \n" + "addi %[tmp_ptr], %[work_t], 1 \n" + "vle8.v v6, (%[tmp_ptr]) \n" + "vwaddu.vx v16, v4, %[c2] \n" + "vwaddu.vx v20, v6, %[c2] \n" + "vwmaccu.vx v16, %[c3], v6 \n" + "vwmaccu.vx v20, %[c3], v4 \n" + "vmv4r.v v24, v16 \n" + "vmv4r.v v28, v20 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vmacc.vx v16, %[c3], v8 \n" + "vmacc.vx v20, %[c3], v12 \n" + "vmacc.vx v8, %[c3], v24 \n" + "vmacc.vx v12, %[c3], v28 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + "vnsrl.wi v0, v20, 4 \n" + "vnsrl.wi v2, v16, 4 \n" + "vsseg2e8.v v0, (%[work_d]) \n" + "vnsrl.wi v4, v12, 4 \n" + "vnsrl.wi v6, v8, 4 \n" + "vsseg2e8.v v4, (%[work_e]) \n" + "sub %[src_width], %[src_width], %[vl]\n" + "add %[work_s], %[work_s], %[vl] \n" + "add %[work_t], %[work_t], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[work_d], %[work_d], %[vl] \n" + "add %[work_e], %[work_e], %[vl] \n" + "bgtz %[src_width], 1b \n" + : [src_width] "+r"(src_width), // %[src_width] + [work_s] "+r"(src_ptr), // %[work_s] + [work_t] "+r"(t), // %[work_t] + [work_d] "+r"(work_d), // %[work_d] + [work_e] "+r"(work_e), // %[work_e] + [vl] "=&r"(vl), // %[vl] + [tmp_ptr] "=&r"(tmp_ptr) // %[tmp_ptr] + : [c2] "r"(2), // %[c2] + [c3] "r"(3) // %[c3] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", + "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", + "v27", "v28", "v29", "v30", "v31"); } - d[dst_width - 1] = - (3 * s[(dst_width - 1) / 2] + t[(dst_width - 1) / 2] + 2) >> 2; - e[dst_width - 1] = - (s[(dst_width - 1) / 2] + 3 * t[(dst_width - 1) / 2] + 2) >> 2; + dst_ptr[dst_width - 1] = (3 * src_ptr[0] + t[0] + 2) >> 2; + e[dst_width - 1] = (src_ptr[0] + 3 * t[0] + 2) >> 2; } -#else -void ScaleRowUp2_Bilinear_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - ptrdiff_t dst_stride, - int dst_width) { - size_t work_width = ((size_t)dst_width - 1u) & ~1u; - size_t src_width = work_width >> 1u; - const uint8_t* work_s = src_ptr; - const uint8_t* work_t = src_ptr + src_stride; - const uint8_t* s = work_s; - const uint8_t* t = work_t; - uint8_t* d = dst_ptr; - uint8_t* e = dst_ptr + dst_stride; - uint8_t* work_d = d + 1; - uint8_t* work_e = e + 1; - size_t vl = __riscv_vsetvlmax_e16m4(); - vuint16m4_t v_3_u16 = __riscv_vmv_v_x_u16m4(3, vl); - vuint8m2_t v_3_u8 = __riscv_vmv_v_x_u8m2(3, vl); - d[0] = (3 * s[0] + t[0] + 2) >> 2; - e[0] = (s[0] + 3 * t[0] + 2) >> 2; - while (src_width > 0) { - vuint8m2_t v_s0, v_s1, v_t0, v_t1; - vuint16m4_t v_s0_u16, v_s1_u16, v_t0_u16, v_t1_u16; - vuint16m4_t v_t0_u16_, v_t1_u16_; - vuint8m2_t v_dst0_even, v_dst0_odd, v_dst1_even, v_dst1_odd; - size_t vl = __riscv_vsetvl_e8m2(src_width); - v_s0 = __riscv_vle8_v_u8m2(work_s, vl); - v_s1 = __riscv_vle8_v_u8m2(work_s + 1, vl); - - v_s0_u16 = __riscv_vwaddu_vx_u16m4(v_s0, 2, vl); - v_s1_u16 = __riscv_vwaddu_vx_u16m4(v_s1, 2, vl); - v_s0_u16 = __riscv_vwmaccu_vv_u16m4(v_s0_u16, v_3_u8, v_s1, vl); - v_s1_u16 = __riscv_vwmaccu_vv_u16m4(v_s1_u16, v_3_u8, v_s0, vl); - - v_t0 = __riscv_vle8_v_u8m2(work_t, vl); - v_t1 = __riscv_vle8_v_u8m2(work_t + 1, vl); - - v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_t0, 2, vl); - v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_t1, 2, vl); - v_t0_u16 = __riscv_vwmaccu_vv_u16m4(v_t0_u16, v_3_u8, v_t1, vl); - v_t1_u16 = __riscv_vwmaccu_vv_u16m4(v_t1_u16, v_3_u8, v_t0, vl); - - v_t0_u16_ = __riscv_vmv_v_v_u16m4(v_t0_u16, vl); - v_t1_u16_ = __riscv_vmv_v_v_u16m4(v_t1_u16, vl); - - v_t0_u16 = __riscv_vmacc_vv_u16m4(v_t0_u16, v_3_u16, v_s0_u16, vl); - v_t1_u16 = __riscv_vmacc_vv_u16m4(v_t1_u16, v_3_u16, v_s1_u16, vl); - v_s0_u16 = __riscv_vmacc_vv_u16m4(v_s0_u16, v_3_u16, v_t0_u16_, vl); - v_s1_u16 = __riscv_vmacc_vv_u16m4(v_s1_u16, v_3_u16, v_t1_u16_, vl); - - v_dst0_odd = __riscv_vnsrl_wx_u8m2(v_t0_u16, 4, vl); - v_dst0_even = __riscv_vnsrl_wx_u8m2(v_t1_u16, 4, vl); - v_dst1_odd = __riscv_vnsrl_wx_u8m2(v_s0_u16, 4, vl); - v_dst1_even = __riscv_vnsrl_wx_u8m2(v_s1_u16, 4, vl); - - __riscv_vsseg2e8_v_u8m2(work_d, v_dst0_even, v_dst0_odd, vl); - __riscv_vsseg2e8_v_u8m2(work_e, v_dst1_even, v_dst1_odd, vl); - - src_width -= vl; - work_s += vl; - work_t += vl; - work_d += 2 * vl; - work_e += 2 * vl; - } - d[dst_width - 1] = - (3 * s[(dst_width - 1) / 2] + t[(dst_width - 1) / 2] + 2) >> 2; - e[dst_width - 1] = - (s[(dst_width - 1) / 2] + 3 * t[(dst_width - 1) / 2] + 2) >> 2; -} -#endif #endif #ifdef HAS_SCALEUVROWDOWN2_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleUVRowDown2_RVV(const uint8_t* src_uv, ptrdiff_t src_stride, uint8_t* dst_uv, int dst_width) { - size_t w = (size_t)dst_width; (void)src_stride; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src = __riscv_vlseg4e8_v_u8m2x4(src_uv, vl); - vuint8m2_t v_u1 = __riscv_vget_v_u8m2x4_u8m2(v_src, 2); - vuint8m2_t v_v1 = __riscv_vget_v_u8m2x4_u8m2(v_src, 3); - vuint8m2x2_t v_dst = __riscv_vcreate_v_u8m2x2(v_u1, v_v1); - __riscv_vsseg2e8_v_u8m2x2(dst_uv, v_dst, vl); - w -= vl; - src_uv += 4 * vl; - dst_uv += 2 * vl; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e16, m2, ta, ma \n" + "vle32.v v8, (%[src_uv]) \n" + "vnsrl.wi v16, v8, 16 \n" + "vse16.v v16, (%[dst_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_uv], %[src_uv], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } -#else -void ScaleUVRowDown2_RVV(const uint8_t* src_uv, - ptrdiff_t src_stride, - uint8_t* dst_uv, - int dst_width) { - size_t w = (size_t)dst_width; - (void)src_stride; - do { - vuint8m2_t v_u0, v_v0, v_u1, v_v1; - size_t vl = __riscv_vsetvl_e8m2(w); - __riscv_vlseg4e8_v_u8m2(&v_u0, &v_v0, &v_u1, &v_v1, src_uv, vl); - __riscv_vsseg2e8_v_u8m2(dst_uv, v_u1, v_v1, vl); - w -= vl; - src_uv += 4 * vl; - dst_uv += 2 * vl; - } while (w > 0); -} -#endif #endif #ifdef HAS_SCALEUVROWDOWN2LINEAR_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleUVRowDown2Linear_RVV(const uint8_t* src_uv, ptrdiff_t src_stride, uint8_t* dst_uv, int dst_width) { - size_t w = (size_t)dst_width; - (void)src_stride; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_src = __riscv_vlseg4e8_v_u8m2x4(src_uv, vl); - vuint8m2_t v_u0 = __riscv_vget_v_u8m2x4_u8m2(v_src, 0); - vuint8m2_t v_v0 = __riscv_vget_v_u8m2x4_u8m2(v_src, 1); - vuint8m2_t v_u1 = __riscv_vget_v_u8m2x4_u8m2(v_src, 2); - vuint8m2_t v_v1 = __riscv_vget_v_u8m2x4_u8m2(v_src, 3); - vuint8m2_t v_u_avg = - __riscv_vaaddu_vv_u8m2(v_u0, v_u1, __RISCV_VXRM_RNU, vl); - vuint8m2_t v_v_avg = - __riscv_vaaddu_vv_u8m2(v_v0, v_v1, __RISCV_VXRM_RNU, vl); - vuint8m2x2_t v_dst = __riscv_vcreate_v_u8m2x2(v_u_avg, v_v_avg); - __riscv_vsseg2e8_v_u8m2x2(dst_uv, v_dst, vl); - w -= vl; - src_uv += 4 * vl; - dst_uv += 2 * vl; - } while (w > 0); -} -#else -void ScaleUVRowDown2Linear_RVV(const uint8_t* src_uv, - ptrdiff_t src_stride, - uint8_t* dst_uv, - int dst_width) { - size_t w = (size_t)dst_width; (void)src_stride; + int vl; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - vuint8m2_t v_u0, v_v0, v_u1, v_v1; - size_t vl = __riscv_vsetvl_e8m2(w); - __riscv_vlseg4e8_v_u8m2(&v_u0, &v_v0, &v_u1, &v_v1, src_uv, vl); - vuint8m2_t v_u_avg = __riscv_vaaddu_vv_u8m2(v_u0, v_u1, vl); - vuint8m2_t v_v_avg = __riscv_vaaddu_vv_u8m2(v_v0, v_v1, vl); - __riscv_vsseg2e8_v_u8m2(dst_uv, v_u_avg, v_v_avg, vl); - w -= vl; - src_uv += 4 * vl; - dst_uv += 2 * vl; - } while (w > 0); + asm volatile( + "csrwi vxrm, 0 \n" + + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v8, (%[src_uv]) \n" + "vaaddu.vv v8, v8, v12 \n" + "vaaddu.vv v10, v10, v14 \n" + "vsseg2e8.v v8, (%[dst_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_uv], %[src_uv], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); } #endif -#endif #ifdef HAS_SCALEUVROWDOWN2BOX_RVV -#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) void ScaleUVRowDown2Box_RVV(const uint8_t* src_uv, ptrdiff_t src_stride, uint8_t* dst_uv, int dst_width) { - const uint8_t* src_uv_row1 = src_uv + src_stride; - size_t w = (size_t)dst_width; - do { - size_t vl = __riscv_vsetvl_e8m2(w); - vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(src_uv, vl); - vuint8m2_t v_u0_row0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 0); - vuint8m2_t v_v0_row0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 1); - vuint8m2_t v_u1_row0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2); - vuint8m2_t v_v1_row0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 3); - vuint8m2x4_t v_t = __riscv_vlseg4e8_v_u8m2x4(src_uv_row1, vl); - vuint8m2_t v_u0_row1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 0); - vuint8m2_t v_v0_row1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 1); - vuint8m2_t v_u1_row1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 2); - vuint8m2_t v_v1_row1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 3); - - vuint16m4_t v_u0u1_row0 = __riscv_vwaddu_vv_u16m4(v_u0_row0, v_u1_row0, vl); - vuint16m4_t v_u0u1_row1 = __riscv_vwaddu_vv_u16m4(v_u0_row1, v_u1_row1, vl); - vuint16m4_t v_v0v1_row0 = __riscv_vwaddu_vv_u16m4(v_v0_row0, v_v1_row0, vl); - vuint16m4_t v_v0v1_row1 = __riscv_vwaddu_vv_u16m4(v_v0_row1, v_v1_row1, vl); - vuint16m4_t v_sum0 = __riscv_vadd_vv_u16m4(v_u0u1_row0, v_u0u1_row1, vl); - vuint16m4_t v_sum1 = __riscv_vadd_vv_u16m4(v_v0v1_row0, v_v0v1_row1, vl); - vuint8m2_t v_dst_u = - __riscv_vnclipu_wx_u8m2(v_sum0, 2, __RISCV_VXRM_RNU, vl); - vuint8m2_t v_dst_v = - __riscv_vnclipu_wx_u8m2(v_sum1, 2, __RISCV_VXRM_RNU, vl); - - vuint8m2x2_t v_dst_uv = __riscv_vcreate_v_u8m2x2(v_dst_u, v_dst_v); - __riscv_vsseg2e8_v_u8m2x2(dst_uv, v_dst_uv, vl); - - dst_uv += 2 * vl; - src_uv += 4 * vl; - w -= vl; - src_uv_row1 += 4 * vl; - } while (w > 0); -} -#else -void ScaleUVRowDown2Box_RVV(const uint8_t* src_uv, - ptrdiff_t src_stride, - uint8_t* dst_uv, - int dst_width) { - const uint8_t* src_uv_row1 = src_uv + src_stride; - size_t w = (size_t)dst_width; + int vl; // NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode // register) is set to round-to-nearest-up mode(0). - asm volatile("csrwi vxrm, 0"); - do { - vuint8m2_t v_u0_row0, v_v0_row0, v_u1_row0, v_v1_row0; - vuint8m2_t v_u0_row1, v_v0_row1, v_u1_row1, v_v1_row1; - vuint16m4_t v_u0u1_row0, v_u0u1_row1, v_v0v1_row0, v_v0v1_row1; - vuint16m4_t v_sum0, v_sum1; - vuint8m2_t v_dst_u, v_dst_v; - size_t vl = __riscv_vsetvl_e8m2(w); + asm volatile( + "add %[src_uv_row1], %[src_uv], %[src_uv_row1]\n" + "csrwi vxrm, 0 \n" - __riscv_vlseg4e8_v_u8m2(&v_u0_row0, &v_v0_row0, &v_u1_row0, &v_v1_row0, - src_uv, vl); - __riscv_vlseg4e8_v_u8m2(&v_u0_row1, &v_v0_row1, &v_u1_row1, &v_v1_row1, - src_uv_row1, vl); - - v_u0u1_row0 = __riscv_vwaddu_vv_u16m4(v_u0_row0, v_u1_row0, vl); - v_u0u1_row1 = __riscv_vwaddu_vv_u16m4(v_u0_row1, v_u1_row1, vl); - v_v0v1_row0 = __riscv_vwaddu_vv_u16m4(v_v0_row0, v_v1_row0, vl); - v_v0v1_row1 = __riscv_vwaddu_vv_u16m4(v_v0_row1, v_v1_row1, vl); - - v_sum0 = __riscv_vadd_vv_u16m4(v_u0u1_row0, v_u0u1_row1, vl); - v_sum1 = __riscv_vadd_vv_u16m4(v_v0v1_row0, v_v0v1_row1, vl); - // Use round-to-nearest-up mode for vnclip - v_dst_u = __riscv_vnclipu_wx_u8m2(v_sum0, 2, vl); - v_dst_v = __riscv_vnclipu_wx_u8m2(v_sum1, 2, vl); - - __riscv_vsseg2e8_v_u8m2(dst_uv, v_dst_u, v_dst_v, vl); - - dst_uv += 2 * vl; - src_uv += 4 * vl; - w -= vl; - src_uv_row1 += 4 * vl; - } while (w > 0); + "1: \n" + "vsetvli %[vl], %[w], e8, m2, ta, ma \n" + "vlseg4e8.v v14, (%[src_uv]) \n" + "vlseg4e8.v v22, (%[src_uv_row1]) \n" + "vwaddu.vv v8, v14, v18 \n" + "vwaddu.wv v8, v8, v22 \n" + "vwaddu.wv v8, v8, v26 \n" + "vwaddu.vv v12, v16, v20 \n" + "vwaddu.wv v12, v12, v24 \n" + "vwaddu.wv v12, v12, v28 \n" + "vnclipu.wi v16, v8, 2 \n" + "vnclipu.wi v18, v12, 2 \n" + "vsseg2e8.v v16, (%[dst_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[src_uv], %[src_uv], %[vl] \n" + "add %[src_uv_row1], %[src_uv_row1], %[vl]\n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_uv] "+r"(src_uv), // %[src_uv] + [src_uv_row1] "+r"(src_stride), // %[src_uv_row1] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29"); } #endif -#endif #ifdef HAS_SCALEUVROWDOWN4_RVV void ScaleUVRowDown4_RVV(const uint8_t* src_uv, @@ -1705,66 +1060,60 @@ void ScaleUVRowDown4_RVV(const uint8_t* src_uv, int src_stepx, uint8_t* dst_uv, int dst_width) { - // Overflow will never happen here, since sizeof(size_t)/sizeof(int)=2. - // dst_width = src_width / 4 and src_width is also int. - size_t w = (size_t)dst_width * 8; (void)src_stride; (void)src_stepx; - do { - size_t vl = __riscv_vsetvl_e8m8(w); - vuint8m8_t v_row = __riscv_vle8_v_u8m8(src_uv, vl); - vuint64m8_t v_row_64 = __riscv_vreinterpret_v_u8m8_u64m8(v_row); - // Narrowing without clipping - vuint32m4_t v_tmp = __riscv_vncvt_x_x_w_u32m4(v_row_64, vl / 8); - vuint16m2_t v_dst_16 = __riscv_vncvt_x_x_w_u16m2(v_tmp, vl / 8); - vuint8m2_t v_dst = __riscv_vreinterpret_v_u16m2_u8m2(v_dst_16); - __riscv_vse8_v_u8m2(dst_uv, v_dst, vl / 4); - w -= vl; - src_uv += vl; - dst_uv += vl / 4; - } while (w > 0); + int vl; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e32, m2, ta, ma \n" + "vle64.v v8, (%[src_uv]) \n" + "vnsrl.wi v16, v8, 0 \n" + "vsetvli zero, zero, e16, m1, ta, ma \n" + "vnsrl.wi v8, v16, 0 \n" + "vse16.v v8, (%[dst_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[src_uv], %[src_uv], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [vl] "=&r"(vl) // %[vl] + : + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v16", "v17"); } #endif #ifdef HAS_SCALEUVROWDOWNEVEN_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleUVRowDownEven_RVV(const uint8_t* src_uv, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_uv, int dst_width) { - size_t w = (size_t)dst_width; - const ptrdiff_t stride_byte = (ptrdiff_t)src_stepx * 2; (void)src_stride; - do { - size_t vl = __riscv_vsetvl_e8m4(w); - vuint8m4x2_t v_row = __riscv_vlsseg2e8_v_u8m4x2(src_uv, stride_byte, vl); - __riscv_vsseg2e8_v_u8m4x2(dst_uv, v_row, vl); - w -= vl; - src_uv += vl * stride_byte; - dst_uv += vl * 2; - } while (w > 0); + int vl; + ptrdiff_t src_step; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e16, m4, ta, ma \n" + "vlse16.v v8, (%[src_uv]), %[stride_byte]\n" + "vse16.v v8, (%[dst_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "mul %[src_step], %[vl], %[stride_byte]\n" + "add %[src_uv], %[src_uv], %[src_step]\n" + "slli %[vl], %[vl], 1 \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(dst_width), // %[w] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [vl] "=&r"(vl), // %[vl] + [src_step] "=&r"(src_step) // %[src_step] + : [stride_byte] "r"((ptrdiff_t)src_stepx * 2) // %[stride_byte] + : "vl", "vtype", "memory", "v8", "v9", "v10", "v11"); } -#else -void ScaleUVRowDownEven_RVV(const uint8_t* src_uv, - ptrdiff_t src_stride, - int src_stepx, - uint8_t* dst_uv, - int dst_width) { - size_t w = (size_t)dst_width; - const ptrdiff_t stride_byte = (ptrdiff_t)src_stepx * 2; - (void)src_stride; - do { - vuint8m4_t v_u, v_v; - size_t vl = __riscv_vsetvl_e8m4(w); - __riscv_vlsseg2e8_v_u8m4(&v_u, &v_v, src_uv, stride_byte, vl); - __riscv_vsseg2e8_v_u8m4(dst_uv, v_u, v_v, vl); - w -= vl; - src_uv += vl * stride_byte; - dst_uv += vl * 2; - } while (w > 0); -} -#endif #endif // ScaleUVRowUp2_(Bi)linear_RVV function is equal to other platforms' @@ -1773,308 +1122,177 @@ void ScaleUVRowDownEven_RVV(const uint8_t* src_uv, // scalar. #ifdef HAS_SCALEUVROWUP2_LINEAR_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleUVRowUp2_Linear_RVV(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width) { - size_t work_width = ((size_t)dst_width - 1u) & ~1u; - size_t src_pairs = work_width >> 1u; - uint8_t* work_dst_ptr = dst_ptr + 2; - const uint8_t* work_src_ptr = src_ptr; - size_t vlmax = __riscv_vsetvlmax_e8m2(); - vuint8m2_t v_3_u8 = __riscv_vmv_v_x_u8m2(3, vlmax); + int src_pairs = (dst_width - 1) >> 1; dst_ptr[0] = src_ptr[0]; dst_ptr[1] = src_ptr[1]; - while (src_pairs > 0) { - size_t vl = __riscv_vsetvl_e8m2(src_pairs); - vuint8m2x2_t v_uv0 = __riscv_vlseg2e8_v_u8m2x2(work_src_ptr, vl); - vuint8m2x2_t v_uv1 = __riscv_vlseg2e8_v_u8m2x2(work_src_ptr + 2, vl); - vuint8m2_t u0 = __riscv_vget_v_u8m2x2_u8m2(v_uv0, 0); - vuint8m2_t v0 = __riscv_vget_v_u8m2x2_u8m2(v_uv0, 1); - vuint8m2_t u1 = __riscv_vget_v_u8m2x2_u8m2(v_uv1, 0); - vuint8m2_t v1 = __riscv_vget_v_u8m2x2_u8m2(v_uv1, 1); - - vuint16m4_t u0_16 = __riscv_vwaddu_vx_u16m4(u0, 2, vl); - vuint16m4_t u1_16 = __riscv_vwaddu_vx_u16m4(u1, 2, vl); - vuint16m4_t u_even_16 = __riscv_vwmaccu_vv_u16m4(u1_16, v_3_u8, u0, vl); - vuint16m4_t u_odd_16 = __riscv_vwmaccu_vv_u16m4(u0_16, v_3_u8, u1, vl); - vuint8m2_t u_even = __riscv_vnsrl_wx_u8m2(u_even_16, 2, vl); - vuint8m2_t u_odd = __riscv_vnsrl_wx_u8m2(u_odd_16, 2, vl); - - vuint16m4_t v0_16 = __riscv_vwaddu_vx_u16m4(v0, 2, vl); - vuint16m4_t v1_16 = __riscv_vwaddu_vx_u16m4(v1, 2, vl); - vuint16m4_t v_even_16 = __riscv_vwmaccu_vv_u16m4(v1_16, v_3_u8, v0, vl); - vuint16m4_t v_odd_16 = __riscv_vwmaccu_vv_u16m4(v0_16, v_3_u8, v1, vl); - vuint8m2_t v_even = __riscv_vnsrl_wx_u8m2(v_even_16, 2, vl); - vuint8m2_t v_odd = __riscv_vnsrl_wx_u8m2(v_odd_16, 2, vl); - - vuint8m2x4_t v_dst = __riscv_vcreate_v_u8m2x4(u_even, v_even, u_odd, v_odd); - __riscv_vsseg4e8_v_u8m2x4(work_dst_ptr, v_dst, vl); - - src_pairs -= vl; - work_src_ptr += 2 * vl; - work_dst_ptr += 4 * vl; + if (src_pairs > 0) { + int vl; + const uint8_t* work_src_next; + uint8_t* work_dst_ptr = dst_ptr + 2; + asm volatile( + "1: \n" + "vsetvli %[vl], %[src_pairs], e8, m2, ta, ma\n" + "addi %[work_src_next], %[src_ptr], 2\n" + "vlseg2e8.v v16, (%[work_src_next]) \n" + "vlseg2e8.v v20, (%[src_ptr]) \n" + "vwaddu.vx v12, v16, %[c2] \n" + "vwmaccu.vx v12, %[c3], v20 \n" + "vnsrl.wi v24, v12, 2 \n" + "vwaddu.vx v12, v20, %[c2] \n" + "vwmaccu.vx v12, %[c3], v16 \n" + "vnsrl.wi v28, v12, 2 \n" + "vwaddu.vx v12, v18, %[c2] \n" + "vwmaccu.vx v12, %[c3], v22 \n" + "vnsrl.wi v26, v12, 2 \n" + "vwaddu.vx v12, v22, %[c2] \n" + "vwmaccu.vx v12, %[c3], v18 \n" + "vnsrl.wi v30, v12, 2 \n" + "vsseg4e8.v v24, (%[work_dst_ptr]) \n" + "sub %[src_pairs], %[src_pairs], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[src_ptr], %[src_ptr], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[work_dst_ptr], %[work_dst_ptr], %[vl]\n" + "bgtz %[src_pairs], 1b \n" + : [src_pairs] "+r"(src_pairs), // %[src_pairs] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [work_dst_ptr] "+r"(work_dst_ptr), // %[work_dst_ptr] + [vl] "=&r"(vl), // %[vl] + [work_src_next] "=&r"(work_src_next) // %[work_src_next] + : [c2] "r"(2), // %[c2] + [c3] "r"(3) // %[c3] + : "vl", "vtype", "memory", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30", "v31"); } - dst_ptr[2 * dst_width - 2] = src_ptr[((dst_width + 1) & ~1) - 2]; - dst_ptr[2 * dst_width - 1] = src_ptr[((dst_width + 1) & ~1) - 1]; + dst_ptr[2 * dst_width - 2] = src_ptr[0]; + dst_ptr[2 * dst_width - 1] = src_ptr[1]; } -#else -void ScaleUVRowUp2_Linear_RVV(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int dst_width) { - size_t work_width = ((size_t)dst_width - 1u) & ~1u; - size_t src_pairs = work_width >> 1u; - uint8_t* work_dst_ptr = dst_ptr + 2; - const uint8_t* work_src_ptr = src_ptr; - size_t vlmax = __riscv_vsetvlmax_e8m2(); - vuint8m2_t v_3_u8 = __riscv_vmv_v_x_u8m2(3, vlmax); - dst_ptr[0] = src_ptr[0]; - dst_ptr[1] = src_ptr[1]; - while (src_pairs > 0) { - vuint8m2_t u0, v0, u1, v1; - size_t vl = __riscv_vsetvl_e8m2(src_pairs); - __riscv_vlseg2e8_v_u8m2(&u0, &v0, work_src_ptr, vl); - __riscv_vlseg2e8_v_u8m2(&u1, &v1, work_src_ptr + 2, vl); - - vuint16m4_t u0_16 = __riscv_vwaddu_vx_u16m4(u0, 2, vl); - vuint16m4_t u1_16 = __riscv_vwaddu_vx_u16m4(u1, 2, vl); - vuint16m4_t u_even_16 = __riscv_vwmaccu_vv_u16m4(u1_16, v_3_u8, u0, vl); - vuint16m4_t u_odd_16 = __riscv_vwmaccu_vv_u16m4(u0_16, v_3_u8, u1, vl); - vuint8m2_t u_even = __riscv_vnsrl_wx_u8m2(u_even_16, 2, vl); - vuint8m2_t u_odd = __riscv_vnsrl_wx_u8m2(u_odd_16, 2, vl); - - vuint16m4_t v0_16 = __riscv_vwaddu_vx_u16m4(v0, 2, vl); - vuint16m4_t v1_16 = __riscv_vwaddu_vx_u16m4(v1, 2, vl); - vuint16m4_t v_even_16 = __riscv_vwmaccu_vv_u16m4(v1_16, v_3_u8, v0, vl); - vuint16m4_t v_odd_16 = __riscv_vwmaccu_vv_u16m4(v0_16, v_3_u8, v1, vl); - vuint8m2_t v_even = __riscv_vnsrl_wx_u8m2(v_even_16, 2, vl); - vuint8m2_t v_odd = __riscv_vnsrl_wx_u8m2(v_odd_16, 2, vl); - - __riscv_vsseg4e8_v_u8m2(work_dst_ptr, u_even, v_even, u_odd, v_odd, vl); - - src_pairs -= vl; - work_src_ptr += 2 * vl; - work_dst_ptr += 4 * vl; - } - dst_ptr[2 * dst_width - 2] = src_ptr[((dst_width + 1) & ~1) - 2]; - dst_ptr[2 * dst_width - 1] = src_ptr[((dst_width + 1) & ~1) - 1]; -} -#endif #endif #ifdef HAS_SCALEUVROWUP2_BILINEAR_RVV -#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) { - size_t work_width = ((size_t)dst_width - 1u) & ~1u; - size_t src_pairs = work_width >> 1u; - const uint8_t* work_s = src_ptr; - const uint8_t* work_t = src_ptr + src_stride; - const uint8_t* s = work_s; - const uint8_t* t = work_t; - uint8_t* d = dst_ptr; + int src_pairs = (dst_width - 1) >> 1; + const uint8_t* t = src_ptr + src_stride; uint8_t* e = dst_ptr + dst_stride; - uint8_t* work_d = d + 2; - uint8_t* work_e = e + 2; - size_t vlmax = __riscv_vsetvlmax_e16m4(); - vuint16m4_t v_3_u16 = __riscv_vmv_v_x_u16m4(3, vlmax); - vuint8m2_t v_3_u8 = __riscv_vmv_v_x_u8m2(3, vlmax); - d[0] = (3 * s[0] + t[0] + 2) >> 2; - e[0] = (s[0] + 3 * t[0] + 2) >> 2; - d[1] = (3 * s[1] + t[1] + 2) >> 2; - e[1] = (s[1] + 3 * t[1] + 2) >> 2; - while (src_pairs > 0) { - size_t vl = __riscv_vsetvl_e8m2(src_pairs); - vuint8m2x2_t v_s0 = __riscv_vlseg2e8_v_u8m2x2(work_s, vl); - vuint8m2x2_t v_s1 = __riscv_vlseg2e8_v_u8m2x2(work_s + 2, vl); - vuint8m2_t s_u0 = __riscv_vget_v_u8m2x2_u8m2(v_s0, 0); - vuint8m2_t s_v0 = __riscv_vget_v_u8m2x2_u8m2(v_s0, 1); - vuint8m2_t s_u1 = __riscv_vget_v_u8m2x2_u8m2(v_s1, 0); - vuint8m2_t s_v1 = __riscv_vget_v_u8m2x2_u8m2(v_s1, 1); - - vuint8m2x2_t v_t0 = __riscv_vlseg2e8_v_u8m2x2(work_t, vl); - vuint8m2x2_t v_t1 = __riscv_vlseg2e8_v_u8m2x2(work_t + 2, vl); - vuint8m2_t t_u0 = __riscv_vget_v_u8m2x2_u8m2(v_t0, 0); - vuint8m2_t t_v0 = __riscv_vget_v_u8m2x2_u8m2(v_t0, 1); - vuint8m2_t t_u1 = __riscv_vget_v_u8m2x2_u8m2(v_t1, 0); - vuint8m2_t t_v1 = __riscv_vget_v_u8m2x2_u8m2(v_t1, 1); - - vuint16m4_t u_s0_16 = __riscv_vwaddu_vx_u16m4(s_u0, 2, vl); - vuint16m4_t u_s1_16 = __riscv_vwaddu_vx_u16m4(s_u1, 2, vl); - u_s0_16 = __riscv_vwmaccu_vv_u16m4(u_s0_16, v_3_u8, s_u1, vl); - u_s1_16 = __riscv_vwmaccu_vv_u16m4(u_s1_16, v_3_u8, s_u0, vl); - - vuint16m4_t u_t0_16 = __riscv_vwaddu_vx_u16m4(t_u0, 2, vl); - vuint16m4_t u_t1_16 = __riscv_vwaddu_vx_u16m4(t_u1, 2, vl); - u_t0_16 = __riscv_vwmaccu_vv_u16m4(u_t0_16, v_3_u8, t_u1, vl); - u_t1_16 = __riscv_vwmaccu_vv_u16m4(u_t1_16, v_3_u8, t_u0, vl); - - vuint16m4_t u_t0_16_ = __riscv_vmv_v_v_u16m4(u_t0_16, vl); - vuint16m4_t u_t1_16_ = __riscv_vmv_v_v_u16m4(u_t1_16, vl); - - u_t0_16 = __riscv_vmacc_vv_u16m4(u_t0_16, v_3_u16, u_s0_16, vl); - u_t1_16 = __riscv_vmacc_vv_u16m4(u_t1_16, v_3_u16, u_s1_16, vl); - u_s0_16 = __riscv_vmacc_vv_u16m4(u_s0_16, v_3_u16, u_t0_16_, vl); - u_s1_16 = __riscv_vmacc_vv_u16m4(u_s1_16, v_3_u16, u_t1_16_, vl); - - vuint8m2_t u_d_odd = __riscv_vnsrl_wx_u8m2(u_t0_16, 4, vl); - vuint8m2_t u_d_even = __riscv_vnsrl_wx_u8m2(u_t1_16, 4, vl); - vuint8m2_t u_e_odd = __riscv_vnsrl_wx_u8m2(u_s0_16, 4, vl); - vuint8m2_t u_e_even = __riscv_vnsrl_wx_u8m2(u_s1_16, 4, vl); - - vuint16m4_t v_s0_16 = __riscv_vwaddu_vx_u16m4(s_v0, 2, vl); - vuint16m4_t v_s1_16 = __riscv_vwaddu_vx_u16m4(s_v1, 2, vl); - v_s0_16 = __riscv_vwmaccu_vv_u16m4(v_s0_16, v_3_u8, s_v1, vl); - v_s1_16 = __riscv_vwmaccu_vv_u16m4(v_s1_16, v_3_u8, s_v0, vl); - - vuint16m4_t v_t0_16 = __riscv_vwaddu_vx_u16m4(t_v0, 2, vl); - vuint16m4_t v_t1_16 = __riscv_vwaddu_vx_u16m4(t_v1, 2, vl); - v_t0_16 = __riscv_vwmaccu_vv_u16m4(v_t0_16, v_3_u8, t_v1, vl); - v_t1_16 = __riscv_vwmaccu_vv_u16m4(v_t1_16, v_3_u8, t_v0, vl); - - vuint16m4_t v_t0_16_ = __riscv_vmv_v_v_u16m4(v_t0_16, vl); - vuint16m4_t v_t1_16_ = __riscv_vmv_v_v_u16m4(v_t1_16, vl); - - v_t0_16 = __riscv_vmacc_vv_u16m4(v_t0_16, v_3_u16, v_s0_16, vl); - v_t1_16 = __riscv_vmacc_vv_u16m4(v_t1_16, v_3_u16, v_s1_16, vl); - v_s0_16 = __riscv_vmacc_vv_u16m4(v_s0_16, v_3_u16, v_t0_16_, vl); - v_s1_16 = __riscv_vmacc_vv_u16m4(v_s1_16, v_3_u16, v_t1_16_, vl); - - vuint8m2_t v_d_odd = __riscv_vnsrl_wx_u8m2(v_t0_16, 4, vl); - vuint8m2_t v_d_even = __riscv_vnsrl_wx_u8m2(v_t1_16, 4, vl); - vuint8m2_t v_e_odd = __riscv_vnsrl_wx_u8m2(v_s0_16, 4, vl); - vuint8m2_t v_e_even = __riscv_vnsrl_wx_u8m2(v_s1_16, 4, vl); - - vuint8m2x4_t v_dst0 = - __riscv_vcreate_v_u8m2x4(u_d_even, v_d_even, u_d_odd, v_d_odd); - __riscv_vsseg4e8_v_u8m2x4(work_d, v_dst0, vl); - vuint8m2x4_t v_dst1 = - __riscv_vcreate_v_u8m2x4(u_e_even, v_e_even, u_e_odd, v_e_odd); - __riscv_vsseg4e8_v_u8m2x4(work_e, v_dst1, vl); - - src_pairs -= vl; - work_s += 2 * vl; - work_t += 2 * vl; - work_d += 4 * vl; - work_e += 4 * vl; + dst_ptr[0] = (3 * src_ptr[0] + t[0] + 2) >> 2; + e[0] = (src_ptr[0] + 3 * t[0] + 2) >> 2; + dst_ptr[1] = (3 * src_ptr[1] + t[1] + 2) >> 2; + e[1] = (src_ptr[1] + 3 * t[1] + 2) >> 2; + if (src_pairs > 0) { + int vl; + const uint8_t* tmp_ptr; + uint8_t* work_d = dst_ptr + 2; + uint8_t* work_e = e + 2; + asm volatile( + "1: \n" + "vsetvli %[vl], %[src_pairs], e8, m2, ta, ma\n" + // Load s_u0 (v0), s_v0 (v2) and s_u1 (v4), s_v1 (v6) + "vlseg2e8.v v0, (%[work_s]) \n" + "addi %[tmp_ptr], %[work_s], 2 \n" + "vlseg2e8.v v4, (%[tmp_ptr]) \n" + // Load t_u0 (v8), t_v0 (v10) and t_u1 (v12), t_v1 (v14) + "vlseg2e8.v v8, (%[work_t]) \n" + "addi %[tmp_ptr], %[work_t], 2 \n" + "vlseg2e8.v v12, (%[tmp_ptr]) \n" + // Compute U channel horizontal interpolation into v16..v31 + "vwaddu.vx v16, v0, %[c2] \n" + "vwaddu.vx v20, v4, %[c2] \n" + "vwmaccu.vx v16, %[c3], v4 \n" + "vwmaccu.vx v20, %[c3], v0 \n" + "vwaddu.vx v24, v8, %[c2] \n" + "vwaddu.vx v28, v12, %[c2] \n" + "vwmaccu.vx v24, %[c3], v12 \n" + "vwmaccu.vx v28, %[c3], v8 \n" + // Move t_v0 (v10) -> v0, t_v1 (v14) -> v4 so v8..v15 are free + "vmv2r.v v0, v10 \n" + "vmv2r.v v4, v14 \n" + "vmv4r.v v8, v24 \n" + "vmv4r.v v12, v28 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vmacc.vx v24, %[c3], v16 \n" + "vmacc.vx v28, %[c3], v20 \n" + "vmacc.vx v16, %[c3], v8 \n" + "vmacc.vx v20, %[c3], v12 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + // Move s_v0 (v2) -> v8, s_v1 (v6) -> v10, t_v0 (v0) -> v12, t_v1 (v4) + // -> v14 + "vmv2r.v v8, v2 \n" + "vmv2r.v v10, v6 \n" + "vmv2r.v v12, v0 \n" + "vmv2r.v v14, v4 \n" + // Narrow U results into v0 (u_d_even), v2 (u_d_odd), v4 (u_e_even), v6 + // (u_e_odd) + "vnsrl.wi v0, v28, 4 \n" + "vnsrl.wi v2, v24, 4 \n" + "vnsrl.wi v4, v20, 4 \n" + "vnsrl.wi v6, v16, 4 \n" + // Compute V channel horizontal interpolation from v8, v10, v12, v14 + // into v16..v31 + "vwaddu.vx v16, v8, %[c2] \n" + "vwaddu.vx v20, v10, %[c2] \n" + "vwmaccu.vx v16, %[c3], v10 \n" + "vwmaccu.vx v20, %[c3], v8 \n" + "vwaddu.vx v24, v12, %[c2] \n" + "vwaddu.vx v28, v14, %[c2] \n" + "vwmaccu.vx v24, %[c3], v14 \n" + "vwmaccu.vx v28, %[c3], v12 \n" + "vmv4r.v v8, v24 \n" + "vmv4r.v v12, v28 \n" + "vsetvli zero, zero, e16, m4, ta, ma \n" + "vmacc.vx v24, %[c3], v16 \n" + "vmacc.vx v28, %[c3], v20 \n" + "vmacc.vx v16, %[c3], v8 \n" + "vmacc.vx v20, %[c3], v12 \n" + "vsetvli zero, zero, e8, m2, ta, ma \n" + // Assemble and store row 0: (u_d_even, v_d_even, u_d_odd, v_d_odd) in + // v8..v15 + "vmv2r.v v8, v0 \n" + "vnsrl.wi v10, v28, 4 \n" + "vmv2r.v v12, v2 \n" + "vnsrl.wi v14, v24, 4 \n" + "vsseg4e8.v v8, (%[work_d]) \n" + // Assemble and store row 1: (u_e_even, v_e_even, u_e_odd, v_e_odd) in + // v8..v15 + "vmv2r.v v8, v4 \n" + "vnsrl.wi v10, v20, 4 \n" + "vmv2r.v v12, v6 \n" + "vnsrl.wi v14, v16, 4 \n" + "vsseg4e8.v v8, (%[work_e]) \n" + "sub %[src_pairs], %[src_pairs], %[vl]\n" + "slli %[vl], %[vl], 1 \n" + "add %[work_s], %[work_s], %[vl] \n" + "add %[work_t], %[work_t], %[vl] \n" + "slli %[vl], %[vl], 1 \n" + "add %[work_d], %[work_d], %[vl] \n" + "add %[work_e], %[work_e], %[vl] \n" + "bgtz %[src_pairs], 1b \n" + : [src_pairs] "+r"(src_pairs), // %[src_pairs] + [work_s] "+r"(src_ptr), // %[work_s] + [work_t] "+r"(t), // %[work_t] + [work_d] "+r"(work_d), // %[work_d] + [work_e] "+r"(work_e), // %[work_e] + [vl] "=&r"(vl), // %[vl] + [tmp_ptr] "=&r"(tmp_ptr) // %[tmp_ptr] + : [c2] "r"(2), // %[c2] + [c3] "r"(3) // %[c3] + : "vl", "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", + "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", + "v27", "v28", "v29", "v30", "v31"); } - d[2 * dst_width - 2] = - (3 * s[((dst_width + 1) & ~1) - 2] + t[((dst_width + 1) & ~1) - 2] + 2) >> - 2; - e[2 * dst_width - 2] = - (s[((dst_width + 1) & ~1) - 2] + 3 * t[((dst_width + 1) & ~1) - 2] + 2) >> - 2; - d[2 * dst_width - 1] = - (3 * s[((dst_width + 1) & ~1) - 1] + t[((dst_width + 1) & ~1) - 1] + 2) >> - 2; - e[2 * dst_width - 1] = - (s[((dst_width + 1) & ~1) - 1] + 3 * t[((dst_width + 1) & ~1) - 1] + 2) >> - 2; + dst_ptr[2 * dst_width - 2] = (3 * src_ptr[0] + t[0] + 2) >> 2; + e[2 * dst_width - 2] = (src_ptr[0] + 3 * t[0] + 2) >> 2; + dst_ptr[2 * dst_width - 1] = (3 * src_ptr[1] + t[1] + 2) >> 2; + e[2 * dst_width - 1] = (src_ptr[1] + 3 * t[1] + 2) >> 2; } -#else -void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - ptrdiff_t dst_stride, - int dst_width) { - size_t work_width = ((size_t)dst_width - 1u) & ~1u; - size_t src_pairs = work_width >> 1u; - const uint8_t* work_s = src_ptr; - const uint8_t* work_t = src_ptr + src_stride; - const uint8_t* s = work_s; - const uint8_t* t = work_t; - uint8_t* d = dst_ptr; - uint8_t* e = dst_ptr + dst_stride; - uint8_t* work_d = d + 2; - uint8_t* work_e = e + 2; - size_t vlmax = __riscv_vsetvlmax_e16m4(); - vuint16m4_t v_3_u16 = __riscv_vmv_v_x_u16m4(3, vlmax); - vuint8m2_t v_3_u8 = __riscv_vmv_v_x_u8m2(3, vlmax); - d[0] = (3 * s[0] + t[0] + 2) >> 2; - e[0] = (s[0] + 3 * t[0] + 2) >> 2; - d[1] = (3 * s[1] + t[1] + 2) >> 2; - e[1] = (s[1] + 3 * t[1] + 2) >> 2; - while (src_pairs > 0) { - vuint8m2_t s_u0, s_v0, s_u1, s_v1; - vuint8m2_t t_u0, t_v0, t_u1, t_v1; - size_t vl = __riscv_vsetvl_e8m2(src_pairs); - __riscv_vlseg2e8_v_u8m2(&s_u0, &s_v0, work_s, vl); - __riscv_vlseg2e8_v_u8m2(&s_u1, &s_v1, work_s + 2, vl); - __riscv_vlseg2e8_v_u8m2(&t_u0, &t_v0, work_t, vl); - __riscv_vlseg2e8_v_u8m2(&t_u1, &t_v1, work_t + 2, vl); - - vuint16m4_t u_s0_16 = __riscv_vwaddu_vx_u16m4(s_u0, 2, vl); - vuint16m4_t u_s1_16 = __riscv_vwaddu_vx_u16m4(s_u1, 2, vl); - u_s0_16 = __riscv_vwmaccu_vv_u16m4(u_s0_16, v_3_u8, s_u1, vl); - u_s1_16 = __riscv_vwmaccu_vv_u16m4(u_s1_16, v_3_u8, s_u0, vl); - - vuint16m4_t u_t0_16 = __riscv_vwaddu_vx_u16m4(t_u0, 2, vl); - vuint16m4_t u_t1_16 = __riscv_vwaddu_vx_u16m4(t_u1, 2, vl); - u_t0_16 = __riscv_vwmaccu_vv_u16m4(u_t0_16, v_3_u8, t_u1, vl); - u_t1_16 = __riscv_vwmaccu_vv_u16m4(u_t1_16, v_3_u8, t_u0, vl); - - vuint16m4_t u_t0_16_ = __riscv_vmv_v_v_u16m4(u_t0_16, vl); - vuint16m4_t u_t1_16_ = __riscv_vmv_v_v_u16m4(u_t1_16, vl); - - u_t0_16 = __riscv_vmacc_vv_u16m4(u_t0_16, v_3_u16, u_s0_16, vl); - u_t1_16 = __riscv_vmacc_vv_u16m4(u_t1_16, v_3_u16, u_s1_16, vl); - u_s0_16 = __riscv_vmacc_vv_u16m4(u_s0_16, v_3_u16, u_t0_16_, vl); - u_s1_16 = __riscv_vmacc_vv_u16m4(u_s1_16, v_3_u16, u_t1_16_, vl); - - vuint8m2_t u_d_odd = __riscv_vnsrl_wx_u8m2(u_t0_16, 4, vl); - vuint8m2_t u_d_even = __riscv_vnsrl_wx_u8m2(u_t1_16, 4, vl); - vuint8m2_t u_e_odd = __riscv_vnsrl_wx_u8m2(u_s0_16, 4, vl); - vuint8m2_t u_e_even = __riscv_vnsrl_wx_u8m2(u_s1_16, 4, vl); - - vuint16m4_t v_s0_16 = __riscv_vwaddu_vx_u16m4(s_v0, 2, vl); - vuint16m4_t v_s1_16 = __riscv_vwaddu_vx_u16m4(s_v1, 2, vl); - v_s0_16 = __riscv_vwmaccu_vv_u16m4(v_s0_16, v_3_u8, s_v1, vl); - v_s1_16 = __riscv_vwmaccu_vv_u16m4(v_s1_16, v_3_u8, s_v0, vl); - - vuint16m4_t v_t0_16 = __riscv_vwaddu_vx_u16m4(t_v0, 2, vl); - vuint16m4_t v_t1_16 = __riscv_vwaddu_vx_u16m4(t_v1, 2, vl); - v_t0_16 = __riscv_vwmaccu_vv_u16m4(v_t0_16, v_3_u8, t_v1, vl); - v_t1_16 = __riscv_vwmaccu_vv_u16m4(v_t1_16, v_3_u8, t_v0, vl); - - vuint16m4_t v_t0_16_ = __riscv_vmv_v_v_u16m4(v_t0_16, vl); - vuint16m4_t v_t1_16_ = __riscv_vmv_v_v_u16m4(v_t1_16, vl); - - v_t0_16 = __riscv_vmacc_vv_u16m4(v_t0_16, v_3_u16, v_s0_16, vl); - v_t1_16 = __riscv_vmacc_vv_u16m4(v_t1_16, v_3_u16, v_s1_16, vl); - v_s0_16 = __riscv_vmacc_vv_u16m4(v_s0_16, v_3_u16, v_t0_16_, vl); - v_s1_16 = __riscv_vmacc_vv_u16m4(v_s1_16, v_3_u16, v_t1_16_, vl); - - vuint8m2_t v_d_odd = __riscv_vnsrl_wx_u8m2(v_t0_16, 4, vl); - vuint8m2_t v_d_even = __riscv_vnsrl_wx_u8m2(v_t1_16, 4, vl); - vuint8m2_t v_e_odd = __riscv_vnsrl_wx_u8m2(v_s0_16, 4, vl); - vuint8m2_t v_e_even = __riscv_vnsrl_wx_u8m2(v_s1_16, 4, vl); - - __riscv_vsseg4e8_v_u8m2(work_d, u_d_even, v_d_even, u_d_odd, v_d_odd, vl); - __riscv_vsseg4e8_v_u8m2(work_e, u_e_even, v_e_even, u_e_odd, v_e_odd, vl); - - src_pairs -= vl; - work_s += 2 * vl; - work_t += 2 * vl; - work_d += 4 * vl; - work_e += 4 * vl; - } - d[2 * dst_width - 2] = - (3 * s[((dst_width + 1) & ~1) - 2] + t[((dst_width + 1) & ~1) - 2] + 2) >> - 2; - e[2 * dst_width - 2] = - (s[((dst_width + 1) & ~1) - 2] + 3 * t[((dst_width + 1) & ~1) - 2] + 2) >> - 2; - d[2 * dst_width - 1] = - (3 * s[((dst_width + 1) & ~1) - 1] + t[((dst_width + 1) & ~1) - 1] + 2) >> - 2; - e[2 * dst_width - 1] = - (s[((dst_width + 1) & ~1) - 1] + 3 * t[((dst_width + 1) & ~1) - 1] + 2) >> - 2; -} -#endif #endif #ifdef __cplusplus