diff --git a/include/libyuv/row.h b/include/libyuv/row.h index dbafb0e17..202b0b197 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -563,6 +563,7 @@ extern "C" { #define HAS_NV12TOARGBROW_SVE2 #define HAS_NV21TOARGBROW_SVE2 #define HAS_P210TOARGBROW_SVE2 +#define HAS_P410TOARGBROW_SVE2 #define HAS_RAWTOARGBROW_SVE2 #define HAS_RAWTORGB24ROW_SVE2 #define HAS_RAWTORGBAROW_SVE2 @@ -5387,6 +5388,11 @@ void P410ToARGBRow_NEON(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void P410ToARGBRow_SVE2(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void P210ToAR30Row_NEON(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_ar30, diff --git a/source/convert_argb.cc b/source/convert_argb.cc index 22b7056a7..33f654595 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -8056,6 +8056,11 @@ static int P010ToARGBMatrixBilinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOARGBROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + P410ToARGBRow = P410ToARGBRow_SVE2; + } +#endif #ifdef HAS_SCALEUVROWUP2_BILINEAR_16_SSE41 if (TestCpuFlag(kCpuHasSSE41)) { @@ -8157,6 +8162,11 @@ static int P210ToARGBMatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOARGBROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + P410ToARGBRow = P410ToARGBRow_SVE2; + } +#endif #ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41 if (TestCpuFlag(kCpuHasSSE41)) { diff --git a/source/row_sve.cc b/source/row_sve.cc index 13564a713..649d6b2a4 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -87,6 +87,18 @@ extern "C" { "prfm pldl1keep, [%[src_uv], 256] \n" \ "tbl z1.b, {z1.b}, z22.b \n" +// We need different predicates for the UV components since we are reading +// 32-bit (pairs of UV) elements rather than 16-bit Y elements. +#define READP410_SVE \ + "ld1h {z0.h}, p1/z, [%[src_y]] \n" \ + "ld1w {z1.s}, p2/z, [%[src_uv]] \n" \ + "ld1w {z2.s}, p3/z, [%[src_uv], #1, mul vl] \n" \ + "incb %[src_y] \n" \ + "incb %[src_uv], all, mul #2 \n" \ + "prfm pldl1keep, [%[src_y], 448] \n" \ + "prfm pldl1keep, [%[src_uv], 256] \n" \ + "uzp2 z1.b, z1.b, z2.b \n" + #define READYUY2_SVE \ "ld1w {z0.s}, p2/z, [%[src_yuy2]] \n" /* YUYV */ \ "incb %[src_yuy2] \n" \ @@ -1854,6 +1866,57 @@ void P210ToARGBRow_SVE2(const uint16_t* src_y, : "cc", "memory", YUVTORGB_SVE_REGS); } +void P410ToARGBRow_SVE2(const uint16_t* src_y, + const uint16_t* src_uv, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uint64_t vl; + asm("cnth %0" : "=r"(vl)); + int width_last_y = width & (vl - 1); + width_last_y = width_last_y == 0 ? vl : width_last_y; + asm volatile( + "ptrue p0.b \n" YUVTORGB_SVE_SETUP + "dup z19.b, #255 \n" // A + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + // Run bulk of computation with an all-true predicate to avoid predicate + // generation overhead. + "ptrue p1.h \n" + "ptrue p2.s \n" + "ptrue p3.s \n" + "1: \n" // + READP410_SVE NVTORGB_SVE RGBTOARGB8_SVE + "subs %w[width], %w[width], %w[vl] \n" + "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + // Calculate a predicate for the final iteration to deal with the tail. + "whilelt p1.h, wzr, %w[width_last_y] \n" + "whilelt p2.s, wzr, %w[width_last_y] \n" + "cntw %x[vl] \n" + "whilelt p3.s, %w[vl], %w[width_last_y] \n" // + READP410_SVE NVTORGB_SVE RGBTOARGB8_SVE + "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + + "99: \n" + : [src_y] "+r"(src_y), // %[src_y] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] + [width_last_y] "r"(width_last_y) // %[width_last_y] + : "cc", "memory", YUVTORGB_SVE_REGS); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus