From 5dc477899b17a7fe5be99fbd794ae1854293b6f1 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Sat, 8 Aug 2026 11:52:15 -0700 Subject: [PATCH] Harden I21xToI420 against oversize heights I21xToI420 (reached via I210ToI420 / I212ToI420): fix integer overflows in ScalePlaneVertical_16To8() by switching to HalfRow_16To8. Add a unit test covering the I21x vertical-scale path. Co-authored-by: Timothy Nikkel Test: libyuv_unittest --gtest_filter=*.I21xToI420NoHeightOverflow AMD Zen4 Was LibYUVConvertTest.I210ToI420_Opt (81 ms) 30.32% Convert16To8Row_AVX2 29.81% Convert16To8Row_AVX512BW 25.72% InterpolateRow_16To8_AVX2 Now LibYUVConvertTest.I210ToI420_Opt (56 ms) 41.32% Convert16To8Row_AVX512BW 40.13% HalfRow_16To8_AVX512BW Vivo X300 (ARM Lunix C1 Ultra) Was LibYUVConvertTest.I210ToI420_Opt (89 ms) 38.95% InterpolateRow_16To8_NEON 37.25% Convert16To8Row_NEON Now LibYUVConvertTest.I210ToI420_Opt (85 ms) 47.26% Convert16To8Row_SVE2 30.39% HalfRow_16To8_SVE2 Bug: chromium:531172025 Change-Id: I23e1215030c40b8577d58617098a8aeec27dd99a Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8221883 Reviewed-by: Wan-Teh Chang Commit-Queue: Frank Barchard --- README.chromium | 2 +- include/libyuv/row.h | 118 +++++++++++++++++---------- include/libyuv/scale_row.h | 14 ---- include/libyuv/version.h | 2 +- source/convert.cc | 158 ++++++++++++++++++++++++++++++++----- source/convert_from.cc | 24 ++++-- source/row_any.cc | 69 ++++++++-------- source/row_common.cc | 71 ++--------------- source/row_gcc.cc | 116 ++++++++++++++++++++++++++- source/row_neon.cc | 29 +++++++ source/row_neon64.cc | 110 +++++++------------------- source/row_rvv.cc | 50 ++++++++++++ source/row_sme.cc | 101 ++---------------------- source/row_sve.cc | 92 +++++++++++++++++++++ source/scale_common.cc | 69 ---------------- unit_test/convert_test.cc | 81 ++++++++++++++++--- 16 files changed, 663 insertions(+), 443 deletions(-) diff --git a/README.chromium b/README.chromium index 9aeafeb91..ad40a378f 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1954 +Version: 1955 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/row.h b/include/libyuv/row.h index d1cc9d257..4647844c4 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -205,6 +205,7 @@ extern "C" { #define HAS_DETILESPLITUVROW_SSSE3 #define HAS_DETILETOYUY2_SSE2 #define HAS_HALFMERGEUVROW_SSSE3 +#define HAS_HALFROW_16TO8_SSSE3 #define HAS_I210TOAR30ROW_SSSE3 #define HAS_I210TOARGBROW_SSSE3 #define HAS_I212TOAR30ROW_SSSE3 @@ -287,6 +288,7 @@ extern "C" { #define HAS_DETILEROW_16_AVX #define HAS_DIVIDEROW_16_AVX2 #define HAS_HALFMERGEUVROW_AVX2 +#define HAS_HALFROW_16TO8_AVX2 #define HAS_I210TOAR30ROW_AVX2 #define HAS_I210TOARGBROW_AVX2 #define HAS_I212TOAR30ROW_AVX2 @@ -297,7 +299,6 @@ extern "C" { #define HAS_I422TOAR30ROW_AVX2 #define HAS_I422TOUYVYROW_AVX2 #define HAS_I422TOYUY2ROW_AVX2 -#define HAS_INTERPOLATEROW_16TO8_AVX2 #define HAS_MERGEAR64ROW_AVX2 #define HAS_MERGEARGB16TO8ROW_AVX2 #define HAS_MERGEARGBROW_AVX2 @@ -381,6 +382,7 @@ extern "C" { #define HAS_RGB24TOARGBROW_AVX512BW #define HAS_ARGBTORGB24ROW_AVX512VBMI #define HAS_CONVERT16TO8ROW_AVX512BW +#define HAS_HALFROW_16TO8_AVX512BW #endif // The following are available for AVX512 clang x64 platforms: @@ -451,6 +453,7 @@ extern "C" { #define HAS_DIVIDEROW_16_NEON #define HAS_HALFFLOATROW_NEON #define HAS_HALFMERGEUVROW_NEON +#define HAS_HALFROW_16TO8_NEON #define HAS_I400TOARGBROW_NEON #define HAS_I422ALPHATOARGBROW_NEON #define HAS_I422TOARGB1555ROW_NEON @@ -578,11 +581,13 @@ extern "C" { #define HAS_ARGBTOYMATRIXROW_SVE2 #define HAS_AYUVTOUVROW_SVE2 #define HAS_AYUVTOVUROW_SVE2 +#define HAS_CONVERT16TO8ROW_SVE2 #define HAS_CONVERT8TO8ROW_SVE2 #define HAS_COPYROW_SVE2 #define HAS_DIVIDEROW_16_SVE2 #define HAS_HALFFLOATROW_SVE2 #define HAS_HALFMERGEUVROW_SVE2 +#define HAS_HALFROW_16TO8_SVE2 #define HAS_I210ALPHATOARGBROW_SVE2 #define HAS_I210TOAR30ROW_SVE2 #define HAS_I210TOARGBROW_SVE2 @@ -636,6 +641,7 @@ extern "C" { #define HAS_CONVERT8TO16ROW_SME #define HAS_CONVERT8TO8ROW_SME #define HAS_COPYROW_SME +#define HAS_HALFROW_16TO8_SME #define HAS_I210ALPHATOARGBROW_SME #define HAS_I210TOAR30ROW_SME #define HAS_I210TOARGBROW_SME @@ -657,7 +663,6 @@ extern "C" { #define HAS_I444TOARGBROW_SME #define HAS_I444TORGB24ROW_SME #define HAS_INTERPOLATEROW_16_SME -#define HAS_INTERPOLATEROW_16TO8_SME #define HAS_INTERPOLATEROW_SME #define HAS_MERGEUVROW_16_SME #define HAS_MERGEUVROW_SME @@ -677,7 +682,6 @@ extern "C" { #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #define HAS_GAUSSCOL_F32_NEON #define HAS_GAUSSROW_F32_NEON -#define HAS_INTERPOLATEROW_16TO8_NEON #define HAS_SCALESUMSAMPLES_NEON #endif @@ -828,7 +832,9 @@ extern "C" { #define HAS_ARGBTOUV444MATRIXROW_RVV #define HAS_ARGBTOUVMATRIXROW_RVV #define HAS_ARGBTOYMATRIXROW_RVV +#define HAS_CONVERT16TO8ROW_RVV #define HAS_COPYROW_RVV +#define HAS_HALFROW_16TO8_RVV #define HAS_INTERPOLATEROW_RVV #define HAS_RGBTOUV444MATRIXROW_RVV #define HAS_RGBTOUVMATRIXROW_RVV @@ -3587,6 +3593,75 @@ void Convert16To8Row_SME(const uint16_t* src_y, uint8_t* dst_y, int scale, int width); +void Convert16To8Row_SVE2(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width); +void Convert16To8Row_RVV(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width); + +void HalfRow_16To8_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_SSSE3(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_Any_SSSE3(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); +void HalfRow_16To8_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); +void HalfRow_16To8_Any_AVX512BW(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); +void HalfRow_16To8_NEON(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_Any_NEON(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); +void HalfRow_16To8_SVE2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_SME(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_RVV(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); void Convert8To8Row_C(const uint8_t* src_y, uint8_t* dst_y, @@ -6379,43 +6454,6 @@ void InterpolateRow_16_SME(uint16_t* dst_ptr, int width, int source_y_fraction); -void InterpolateRow_16To8_C(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width, - int source_y_fraction); -void InterpolateRow_16To8_NEON(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width, - int source_y_fraction); -void InterpolateRow_16To8_Any_NEON(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width, - int source_y_fraction); -void InterpolateRow_16To8_SME(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width, - int source_y_fraction); -void InterpolateRow_16To8_AVX2(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width, - int source_y_fraction); -void InterpolateRow_16To8_Any_AVX2(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width, - int source_y_fraction); - // Sobel images. void SobelXRow_C(const uint8_t* src_y0, const uint8_t* src_y1, diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 16f626439..90b949a67 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -223,20 +223,6 @@ void ScalePlaneVertical_16(int src_height, int wpp, enum FilterMode filtering); -void ScalePlaneVertical_16To8(int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint16_t* src_argb, - uint8_t* dst_argb, - int x, - int y, - int dy, - int wpp, - int scale, - enum FilterMode filtering); - void ScalePlaneDown2_16To8(int src_width, int src_height, int dst_width, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index dc58a3d69..b913fe481 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1954 +#define LIBYUV_VERSION 1955 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/convert.cc b/source/convert.cc index 36657c633..6a916bbb1 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -232,16 +232,19 @@ static int I41xToI420(const uint16_t* src_y, int depth) { const int scale = 1 << (24 - depth); - if (width <= 0 || height == 0 || height == INT_MIN) { + if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 || + height == 0 || height == INT_MIN) { return -1; } // Negative height means invert the image. if (height < 0) { height = -height; - src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + if (src_y) { + src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u; src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v; - src_stride_y = -src_stride_y; src_stride_u = -src_stride_u; src_stride_v = -src_stride_v; } @@ -250,8 +253,10 @@ static int I41xToI420(const uint16_t* src_y, const int uv_width = SUBSAMPLE(width, 1, 1); const int uv_height = SUBSAMPLE(height, 1, 1); - Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width, - height); + if (dst_y) { + Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width, + height); + } ScalePlaneDown2_16To8(width, height, uv_width, uv_height, src_stride_u, dst_stride_u, src_u, dst_u, scale, kFilterBilinear); ScalePlaneDown2_16To8(width, height, uv_width, uv_height, src_stride_v, @@ -275,35 +280,148 @@ static int I21xToI420(const uint16_t* src_y, int width, int height, int depth) { + int y; const int scale = 1 << (24 - depth); + const int uv_width = SUBSAMPLE(width, 1, 1); + void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale, + int width) = Convert16To8Row_C; + void (*HalfRow_16To8)(const uint16_t* src_uv, ptrdiff_t src_uv_stride, + uint8_t* dst_uv, int scale, int width) = + HalfRow_16To8_C; - if (width <= 0 || height == 0 || height == INT_MIN) { + if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 || + height == 0 || height == INT_MIN) { return -1; } // Negative height means invert the image. if (height < 0) { height = -height; - src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + if (src_y) { + src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u; src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v; - src_stride_y = -src_stride_y; src_stride_u = -src_stride_u; src_stride_v = -src_stride_v; } - { - const int uv_width = SUBSAMPLE(width, 1, 1); - const int uv_height = SUBSAMPLE(height, 1, 1); - const int dy = FixedDiv(height, uv_height); +#if defined(HAS_CONVERT16TO8ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + Convert16To8Row = Convert16To8Row_Any_NEON; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_NEON; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + Convert16To8Row = Convert16To8Row_SVE2; + } +#endif +#if defined(HAS_CONVERT16TO8ROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + Convert16To8Row = Convert16To8Row_SME; + } +#endif +#if defined(HAS_CONVERT16TO8ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + Convert16To8Row = Convert16To8Row_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_SSSE3; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + Convert16To8Row = Convert16To8Row_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + Convert16To8Row = Convert16To8Row_AVX2; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + Convert16To8Row = Convert16To8Row_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + Convert16To8Row = Convert16To8Row_AVX512BW; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + Convert16To8Row = Convert16To8Row_RVV; + } +#endif - Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width, - height); - ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_u, - dst_stride_u, src_u, dst_u, 0, 32768, dy, - /*bpp=*/1, scale, kFilterBilinear); - ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_v, - dst_stride_v, src_v, dst_v, 0, 32768, dy, - /*bpp=*/1, scale, kFilterBilinear); +#if defined(HAS_HALFROW_16TO8_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + HalfRow_16To8 = HalfRow_16To8_Any_NEON; + if (IS_ALIGNED(uv_width, 16)) { + HalfRow_16To8 = HalfRow_16To8_NEON; + } + } +#endif +#if defined(HAS_HALFROW_16TO8_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + HalfRow_16To8 = HalfRow_16To8_SVE2; + } +#endif +#if defined(HAS_HALFROW_16TO8_SME) + if (TestCpuFlag(kCpuHasSME)) { + HalfRow_16To8 = HalfRow_16To8_SME; + } +#endif +#if defined(HAS_HALFROW_16TO8_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + HalfRow_16To8 = HalfRow_16To8_Any_SSSE3; + if (IS_ALIGNED(uv_width, 16)) { + HalfRow_16To8 = HalfRow_16To8_SSSE3; + } + } +#endif +#if defined(HAS_HALFROW_16TO8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + HalfRow_16To8 = HalfRow_16To8_Any_AVX2; + if (IS_ALIGNED(uv_width, 32)) { + HalfRow_16To8 = HalfRow_16To8_AVX2; + } + } +#endif +#if defined(HAS_HALFROW_16TO8_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + HalfRow_16To8 = HalfRow_16To8_Any_AVX512BW; + if (IS_ALIGNED(uv_width, 64)) { + HalfRow_16To8 = HalfRow_16To8_AVX512BW; + } + } +#endif +#if defined(HAS_HALFROW_16TO8_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + HalfRow_16To8 = HalfRow_16To8_RVV; + } +#endif + + for (y = 0; y < height - 1; y += 2) { + HalfRow_16To8(src_u, src_stride_u, dst_u, scale, uv_width); + HalfRow_16To8(src_v, src_stride_v, dst_v, scale, uv_width); + if (dst_y) { + Convert16To8Row(src_y, dst_y, scale, width); + Convert16To8Row(src_y + src_stride_y, dst_y + dst_stride_y, scale, width); + src_y += src_stride_y * 2; + dst_y += dst_stride_y * 2; + } + src_u += src_stride_u * 2; + src_v += src_stride_v * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + HalfRow_16To8(src_u, 0, dst_u, scale, uv_width); + HalfRow_16To8(src_v, 0, dst_v, scale, uv_width); + if (dst_y) { + Convert16To8Row(src_y, dst_y, scale, width); + } } return 0; } diff --git a/source/convert_from.cc b/source/convert_from.cc index 40ca02190..1331699c9 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -96,17 +96,21 @@ int I420ToI010(const uint8_t* src_y, if (height < 0) { height = -height; halfheight = (height + 1) >> 1; - src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + if (src_y) { + src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u; src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v; - src_stride_y = -src_stride_y; src_stride_u = -src_stride_u; src_stride_v = -src_stride_v; } // Convert Y plane. - Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 1024, width, - height); + if (dst_y) { + Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 1024, width, + height); + } // Convert UV planes. Convert8To16Plane(src_u, src_stride_u, dst_u, dst_stride_u, 1024, halfwidth, halfheight); @@ -141,17 +145,21 @@ int I420ToI012(const uint8_t* src_y, if (height < 0) { height = -height; halfheight = (height + 1) >> 1; - src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + if (src_y) { + src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; + src_stride_y = -src_stride_y; + } src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u; src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v; - src_stride_y = -src_stride_y; src_stride_u = -src_stride_u; src_stride_v = -src_stride_v; } // Convert Y plane. - Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 4096, width, - height); + if (dst_y) { + Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 4096, width, + height); + } // Convert UV planes. Convert8To16Plane(src_u, src_stride_u, dst_u, dst_stride_u, 4096, halfwidth, halfheight); diff --git a/source/row_any.cc b/source/row_any.cc index 4c19d8e15..bb53194b7 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1689,48 +1689,43 @@ ANY11I(InterpolateRow_16_Any_AVX2, #endif #undef ANY11I -// Any 1 to 1 interpolate with scale param -#define ANY11IS(NAMEANY, ANY_SIMD, TD, TS, SBPP, BPP, MASK) \ - void NAMEANY(TD* dst_ptr, const TS* src_ptr, ptrdiff_t src_stride, \ - int scale, int width, int source_y_fraction) { \ - SIMD_ALIGNED(TS vin[64 * 2]); \ - SIMD_ALIGNED(TD vout[64]); \ - memset(vin, 0, sizeof(vin)); /* for msan */ \ - int r = width & MASK; \ - int n = width & ~MASK; \ - if (n > 0) { \ - ANY_SIMD(dst_ptr, src_ptr, src_stride, scale, n, source_y_fraction); \ - } \ - ptrdiff_t np = n; \ - memcpy(vin, src_ptr + np * SBPP, r * SBPP * sizeof(TS)); \ - if (source_y_fraction) { \ - memcpy(vin + 64, src_ptr + src_stride + np * SBPP, \ - r * SBPP * sizeof(TS)); \ - } \ - ANY_SIMD(vout, vin, 64, scale, MASK + 1, source_y_fraction); \ - memcpy(dst_ptr + np * BPP, vout, r * BPP * sizeof(TD)); \ +// Any 1 to 1 half row with scale param +#define ANY11HS(NAMEANY, ANY_SIMD, TS, TD, MASK) \ + void NAMEANY(const TS* src_ptr, ptrdiff_t src_stride, TD* dst_ptr, \ + int scale, int width) { \ + SIMD_ALIGNED(TS vin[64 * 2]); \ + static_assert((MASK + 1) * sizeof(TS) <= sizeof(vin) / 2, \ + "vin buffer too small"); \ + SIMD_ALIGNED(TD vout[64]); \ + static_assert((MASK + 1) * sizeof(TD) <= sizeof(vout), \ + "vout buffer too small"); \ + memset(vin, 0, sizeof(vin)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr, src_stride, dst_ptr, scale, n); \ + } \ + ptrdiff_t np = n; \ + memcpy(vin, src_ptr + np, r * sizeof(TS)); \ + memcpy(vin + 64, src_ptr + src_stride + np, r * sizeof(TS)); \ + ANY_SIMD(vin, 64, vout, scale, MASK + 1); \ + memcpy(dst_ptr + np, vout, r * sizeof(TD)); \ } -#ifdef HAS_INTERPOLATEROW_16TO8_NEON -ANY11IS(InterpolateRow_16To8_Any_NEON, - InterpolateRow_16To8_NEON, - uint8_t, - uint16_t, - 1, - 1, - 7) +#ifdef HAS_HALFROW_16TO8_SSSE3 +ANY11HS(HalfRow_16To8_Any_SSSE3, HalfRow_16To8_SSSE3, uint16_t, uint8_t, 15) #endif -#ifdef HAS_INTERPOLATEROW_16TO8_AVX2 -ANY11IS(InterpolateRow_16To8_Any_AVX2, - InterpolateRow_16To8_AVX2, - uint8_t, - uint16_t, - 1, - 1, - 31) +#ifdef HAS_HALFROW_16TO8_AVX2 +ANY11HS(HalfRow_16To8_Any_AVX2, HalfRow_16To8_AVX2, uint16_t, uint8_t, 31) +#endif +#ifdef HAS_HALFROW_16TO8_AVX512BW +ANY11HS(HalfRow_16To8_Any_AVX512BW, HalfRow_16To8_AVX512BW, uint16_t, uint8_t, 63) +#endif +#ifdef HAS_HALFROW_16TO8_NEON +ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15) #endif -#undef ANY11IS +#undef ANY11HS // Any 1 to 1 mirror. #define ANY11M(NAMEANY, ANY_SIMD, BPP, MASK) \ diff --git a/source/row_common.cc b/source/row_common.cc index c547556a8..d1fc258ff 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -3664,11 +3664,11 @@ static void HalfRow_16_C(const uint16_t* src_uv, } } -static void HalfRow_16To8_C(const uint16_t* src_uv, - ptrdiff_t src_uv_stride, - uint8_t* dst_uv, - int scale, - int width) { +void HalfRow_16To8_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { int x; for (x = 0; x < width; ++x) { dst_uv[x] = STATIC_CAST( @@ -3739,47 +3739,6 @@ void InterpolateRow_16_C(uint16_t* dst_ptr, } } -// C version 2x2 16 bit-> 2x1 8 bit. -// Use scale to convert lsb formats to msb, depending how many bits there are: -// 32768 = 9 bits -// 16384 = 10 bits -// 4096 = 12 bits -// 256 = 16 bits -// TODO(fbarchard): change scale to bits - -void InterpolateRow_16To8_C(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width, - int source_y_fraction) { - int y1_fraction = source_y_fraction; - int y0_fraction = 256 - y1_fraction; - const uint16_t* src_ptr1 = src_ptr + src_stride; - int x; - assert(source_y_fraction >= 0); - assert(source_y_fraction < 256); - - if (source_y_fraction == 0) { - Convert16To8Row_C(src_ptr, dst_ptr, scale, width); - return; - } - if (source_y_fraction == 128) { - HalfRow_16To8_C(src_ptr, src_stride, dst_ptr, scale, width); - return; - } - for (x = 0; x < width; ++x) { - dst_ptr[0] = STATIC_CAST( - uint8_t, - C16TO8( - (src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction + 128) >> 8, - scale)); - src_ptr += 1; - src_ptr1 += 1; - dst_ptr += 1; - } -} - // Use first 4 shuffler values to reorder ARGB channels. void ARGBShuffleRow_C(const uint8_t* src_argb, uint8_t* dst_argb, @@ -4291,26 +4250,6 @@ void NV12ToRGB565Row_AVX2(const uint8_t* src_y, } #endif -#ifdef HAS_INTERPOLATEROW_16TO8_AVX2 -void InterpolateRow_16To8_AVX2(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width, - int source_y_fraction) { - // Row buffer for intermediate 16 bit pixels. - SIMD_ALIGNED(uint16_t row[MAXTWIDTH]); - while (width > 0) { - int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; - InterpolateRow_16_AVX2(row, src_ptr, src_stride, twidth, source_y_fraction); - Convert16To8Row_AVX2(row, dst_ptr, scale, twidth); - src_ptr += twidth; - dst_ptr += twidth; - width -= twidth; - } -} -#endif // HAS_INTERPOLATEROW_16TO8_AVX2 - float ScaleSumSamples_C(const float* src, float* dst, float scale, int width) { float fsum = 0.f; int i; diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 4318cbf93..02aad5d69 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -241,6 +241,7 @@ void RGB24ToARGBRow_SSSE3(const uint8_t* src_rgb24, void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24, uint8_t* dst_argb, int width) { + // TODO(fbarchard): Fix ASAN issue with kShuffleMaskRGB24ToARGB[1] // Reference to prevent discarding of kShuffleMaskRGB24ToARGB[1] which is // accessed via offset in assembly. const uvec8* dummy = &kShuffleMaskRGB24ToARGB[1]; @@ -2458,6 +2459,7 @@ void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { + // TODO(fbarchard): Fix ASAN issue with kShuffleMaskARGBToRGB24[1] // Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is // accessed via offset in assembly. const uvec8* dummy = &kShuffleMaskARGBToRGB24[1]; @@ -2497,6 +2499,7 @@ void OMITFP I444ToRGB24Row_SSSE3(const uint8_t* y_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { + // TODO(fbarchard): Fix ASAN issue with kShuffleMaskARGBToRGB24[1] // Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is // accessed via offset in assembly. const uvec8* dummy = &kShuffleMaskARGBToRGB24[1]; @@ -3551,6 +3554,7 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { + // TODO(fbarchard): Fix ASAN issue with kShuffleMaskARGBToRGB24[1] // Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is // accessed via offset in assembly. const uvec8* dummy = &kShuffleMaskARGBToRGB24[1]; @@ -5414,7 +5418,117 @@ void Convert16To8Row_AVX512BW(const uint16_t* src_y, : "r"(scale) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2"); } -#endif // HAS_CONVERT16TO8ROW_AVX2 +#endif // HAS_CONVERT16TO8ROW_AVX512BW + +#ifdef HAS_HALFROW_16TO8_SSSE3 +void HalfRow_16To8_SSSE3(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + asm volatile( + "movd %4,%%xmm2 \n" + "punpcklwd %%xmm2,%%xmm2 \n" + "pshufd $0x0,%%xmm2,%%xmm2 \n" + + // 16 pixels per loop. + LABELALIGN + "1: \n" + "movdqu (%0),%%xmm0 \n" + "movdqu 0x10(%0),%%xmm1 \n" + "movdqu (%0,%3,2),%%xmm3 \n" + "movdqu 0x10(%0,%3,2),%%xmm4 \n" + "add $0x20,%0 \n" + "pavgw %%xmm3,%%xmm0 \n" + "pavgw %%xmm4,%%xmm1 \n" + "pmulhuw %%xmm2,%%xmm0 \n" + "pmulhuw %%xmm2,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0,(%1) \n" + "add $0x10,%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(width) // %2 + : "r"(src_uv_stride), // %3 + "r"(scale) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); +} +#endif // HAS_HALFROW_16TO8_SSSE3 + +#ifdef HAS_HALFROW_16TO8_AVX2 +void HalfRow_16To8_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + asm volatile( + "vmovd %4,%%xmm2 \n" + "vpbroadcastw %%xmm2,%%ymm2 \n" + + // 32 pixels per loop. + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vpavgw (%0,%3,2),%%ymm0,%%ymm0 \n" + "vpavgw 0x20(%0,%3,2),%%ymm1,%%ymm1 \n" + "add $0x40,%0 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "add $0x20,%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(width) // %2 + : "r"(src_uv_stride), // %3 + "r"(scale) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2"); +} +#endif // HAS_HALFROW_16TO8_AVX2 + +#ifdef HAS_HALFROW_16TO8_AVX512BW +void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + const int shift = 16 - __builtin_ctz((uint32_t)scale); + asm volatile( + "vpbroadcastw %4,%%zmm2 \n" + + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vmovdqu64 (%0),%%zmm0 \n" + "vmovdqu64 0x40(%0),%%zmm1 \n" + "vpavgw (%1),%%zmm0,%%zmm0 \n" + "vpavgw 0x40(%1),%%zmm1,%%zmm1 \n" + "add $0x80,%0 \n" + "add $0x80,%1 \n" + "vpsrlvw %%zmm2,%%zmm0,%%zmm0 \n" + "vpsrlvw %%zmm2,%%zmm1,%%zmm1 \n" + "vpmovuswb %%zmm0,(%2) \n" + "vpmovuswb %%zmm1,0x20(%2) \n" + "add $0x40,%2 \n" + "sub $0x40,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(src_uv1), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 + : "r"(shift) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2"); +} +#endif // HAS_HALFROW_16TO8_AVX512BW // Use scale to convert to lsb formats depending how many bits there are: // 512 = 9 bits diff --git a/source/row_neon.cc b/source/row_neon.cc index 50ba7dbde..70da2c442 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -3681,6 +3681,35 @@ void Convert16To8Row_NEON(const uint16_t* src_y, : "cc", "memory", "q0", "q1", "q2"); } +void HalfRow_16To8_NEON(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + const int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr + asm volatile( + "vdup.16 q4, %4 \n" + "1: \n" + "vld1.16 {q0, q1}, [%0]! \n" + "vld1.16 {q2, q3}, [%1]! \n" + "subs %3, %3, #16 \n" // 16 src pixels per loop + "vrhadd.u16 q0, q0, q2 \n" + "vrhadd.u16 q1, q1, q3 \n" + "vshl.u16 q0, q0, q4 \n" // shr = q4 is negative + "vshl.u16 q1, q1, q4 \n" + "vqmovn.u16 d0, q0 \n" + "vqmovn.u16 d1, q1 \n" + "vst1.8 {q0}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_uv), // %0 + "+r"(src_uv1), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 + : "r"(shift) // %4 + : "cc", "memory", "q0", "q1", "q2", "q3", "q4"); +} + // Use scale to convert J420 to I420 // scale parameter is 8.8 fixed point but limited to 0 to 255 // Function is based on DivideRow, but adds a bias diff --git a/source/row_neon64.cc b/source/row_neon64.cc index 782a6653f..ca879dd1b 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -3728,86 +3728,6 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5"); } -// Bilinear filter 8x2 -> 8x1 -// Use scale to convert lsb formats to msb, depending how many bits there are: -// 32768 = 9 bits -// 16384 = 10 bits -// 4096 = 12 bits -// 256 = 16 bits -void InterpolateRow_16To8_NEON(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int dst_width, - int source_y_fraction) { - const int y1_fraction = source_y_fraction; - const int y0_fraction = 256 - y1_fraction; - const uint16_t* src_ptr1 = src_ptr + src_stride; - const int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr - - asm volatile( - "dup v6.8h, %w6 \n" - "cmp %w4, #0 \n" - "b.eq 100f \n" - "cmp %w4, #128 \n" - "b.eq 50f \n" - - "dup v5.8h, %w4 \n" - "dup v4.8h, %w5 \n" - // General purpose row blend. - "1: \n" - "ld1 {v0.8h}, [%1], #16 \n" - "ld1 {v1.8h}, [%2], #16 \n" - "subs %w3, %w3, #8 \n" - "umull v2.4s, v0.4h, v4.4h \n" - "prfm pldl1keep, [%1, 448] \n" - "umull2 v3.4s, v0.8h, v4.8h \n" - "prfm pldl1keep, [%2, 448] \n" - "umlal v2.4s, v1.4h, v5.4h \n" - "umlal2 v3.4s, v1.8h, v5.8h \n" - "rshrn v0.4h, v2.4s, #8 \n" - "rshrn2 v0.8h, v3.4s, #8 \n" - "ushl v0.8h, v0.8h, v6.8h \n" - "uqxtn v0.8b, v0.8h \n" - "st1 {v0.8b}, [%0], #8 \n" - "b.gt 1b \n" - "b 99f \n" - - // Blend 50 / 50. - "50: \n" - "ld1 {v0.8h}, [%1], #16 \n" - "ld1 {v1.8h}, [%2], #16 \n" - "subs %w3, %w3, #8 \n" - "prfm pldl1keep, [%1, 448] \n" - "urhadd v0.8h, v0.8h, v1.8h \n" - "prfm pldl1keep, [%2, 448] \n" - "ushl v0.8h, v0.8h, v6.8h \n" - "uqxtn v0.8b, v0.8h \n" - "st1 {v0.8b}, [%0], #8 \n" - "b.gt 50b \n" - "b 99f \n" - - // Blend 100 / 0 - Copy row unchanged. - "100: \n" - "ldr q0, [%1], #16 \n" - "ushl v0.8h, v0.8h, v2.8h \n" // shr = v2 is negative - "prfm pldl1keep, [%1, 448] \n" - "uqxtn v0.8b, v0.8h \n" - "subs %w3, %w3, #8 \n" // 8 src pixels per loop - "str d0, [%0], #8 \n" // store 8 pixels - "b.gt 100b \n" - - "99: \n" - : "+r"(dst_ptr), // %0 - "+r"(src_ptr), // %1 - "+r"(src_ptr1), // %2 - "+r"(dst_width) // %3 - : "r"(y1_fraction), // %4 - "r"(y0_fraction), // %5 - "r"(shift) // %6 - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6"); -} - // dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr void ARGBBlendRow_NEON(const uint8_t* src_argb, const uint8_t* src_argb1, @@ -5169,6 +5089,36 @@ void Convert16To8Row_NEON(const uint16_t* src_y, : "cc", "memory", "v0", "v1", "v2"); } +void HalfRow_16To8_NEON(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + const int shift = 23 - __builtin_clz((int32_t)scale); + asm volatile( + "dup v4.8h, %w4 \n" + "1: \n" + "ldp q0, q1, [%0], #32 \n" + "ldp q2, q3, [%1], #32 \n" + "subs %w3, %w3, #16 \n" + "urhadd v0.8h, v0.8h, v2.8h \n" + "urhadd v1.8h, v1.8h, v3.8h \n" + "prfm pldl1keep, [%0, 448] \n" + "prfm pldl1keep, [%1, 448] \n" + "uqshl v0.8h, v0.8h, v4.8h \n" + "uqshl v1.8h, v1.8h, v4.8h \n" + "uzp2 v0.16b, v0.16b, v1.16b \n" + "str q0, [%2], #16 \n" + "b.gt 1b \n" + : "+r"(src_uv), // %0 + "+r"(src_uv1), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 + : "r"(shift) // %4 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4"); +} + // Use scale to convert J420 to I420 // scale parameter is 8.8 fixed point but limited to 0 to 255 // Function is based on DivideRow, but adds a bias diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 7ef2c64ed..17ba7bc83 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -1794,6 +1794,56 @@ void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width) { } #endif +#ifdef HAS_CONVERT16TO8ROW_RVV +void Convert16To8Row_RVV(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width) { + size_t w = (size_t)width; + const int shift = __builtin_clz((int32_t)scale) - 15; + do { + size_t vl = __riscv_vsetvl_e16m4(w); + vuint16m4_t v_src = __riscv_vle16_v_u16m4(src_y, vl); + vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_src, shift, vl); + __riscv_vse8_v_u8m2(dst_y, v_dst, vl); + w -= vl; + src_y += vl; + dst_y += vl; + } while (w > 0); +} +#endif + +#ifdef HAS_HALFROW_16TO8_RVV +void HalfRow_16To8_RVV(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + size_t w = (size_t)width; + const uint16_t* src_uv1 = src_uv + src_uv_stride; + const int shift = __builtin_clz((int32_t)scale) - 15; +#ifndef LIBYUV_RVV_HAS_VXRM_ARG + asm volatile("csrwi vxrm, 0"); +#endif + do { + size_t vl = __riscv_vsetvl_e16m4(w); + vuint16m4_t v_src0 = __riscv_vle16_v_u16m4(src_uv, vl); + vuint16m4_t v_src1 = __riscv_vle16_v_u16m4(src_uv1, vl); +#ifdef LIBYUV_RVV_HAS_VXRM_ARG + vuint16m4_t v_avg = __riscv_vaaddu_vv_u16m4_vxrm(v_src0, v_src1, 0, vl); +#else + vuint16m4_t v_avg = __riscv_vaaddu_vv_u16m4(v_src0, v_src1, vl); +#endif + vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_avg, shift, vl); + __riscv_vse8_v_u8m2(dst_uv, v_dst, vl); + w -= vl; + src_uv += vl; + src_uv1 += vl; + dst_uv += vl; + } while (w > 0); +} +#endif + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_sme.cc b/source/row_sme.cc index 9ac3fb7f4..7a18898d0 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -907,11 +907,11 @@ __arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr, : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0"); } -__arm_locally_streaming static void HalfRow_16To8_SME(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width) { +__arm_locally_streaming void HalfRow_16To8_SME(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width) { const uint16_t* src_ptr1 = src_ptr + src_stride; // 15 - clz(scale), + 8 to shift result into the high half of the lane to @@ -965,97 +965,6 @@ __arm_locally_streaming static void HalfRow_16To8_SME(uint8_t* dst_ptr, : "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0"); } -// Use scale to convert lsb formats to msb, depending how many bits there are: -// 32768 = 9 bits -// 16384 = 10 bits -// 4096 = 12 bits -// 256 = 16 bits -// TODO(fbarchard): change scale to bits -__arm_locally_streaming void InterpolateRow_16To8_SME(uint8_t* dst_ptr, - const uint16_t* src_ptr, - ptrdiff_t src_stride, - int scale, - int width, - int source_y_fraction) { - const int y1_fraction = source_y_fraction; - const int y0_fraction = 256 - y1_fraction; - const uint16_t* src_ptr1 = src_ptr + src_stride; - - // y0_fraction == 0 is never called here. - if (y0_fraction == 128) { - HalfRow_16To8_SME(dst_ptr, src_ptr, src_stride, scale, width); - return; - } - if (y0_fraction == 256) { - Convert16To8Row_SME(src_ptr, dst_ptr, scale, width); - return; - } - - // 15 - clz(scale), + 8 to shift result into the high half of the lane to - // saturate, then we can just use UZP2 to narrow rather than a pair of - // saturating narrow instructions. - const int shift = 23 - __builtin_clz((int32_t)scale); - - int vl; - asm volatile( - "cnth %x[vl] \n" - "dup z31.h, %w[shift] \n" - "dup z0.h, %w[y0_fraction] \n" - "dup z1.h, %w[y1_fraction] \n" - "subs %w[width], %w[width], %w[vl] \n" - "b.lt 2f \n" - - // Run bulk of computation with an all-true predicate to avoid predicate - // generation overhead. - "ptrue p0.h \n" - "1: \n" - "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" - "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" - "incb %[src_ptr] \n" - "incb %[src_ptr1] \n" - "umullb z4.s, z2.h, z0.h \n" - "umullt z2.s, z2.h, z0.h \n" - "subs %w[width], %w[width], %w[vl] \n" - "umlalb z4.s, z3.h, z1.h \n" - "umlalt z2.s, z3.h, z1.h \n" - "rshrnb z3.h, z4.s, #8 \n" - "rshrnt z3.h, z2.s, #8 \n" - "uqshl z3.h, p0/m, z3.h, z31.h \n" - "shrnb z3.b, z3.h, #8 \n" - "st1b {z3.h}, p0, [%[dst_ptr]] \n" - "inch %[dst_ptr] \n" - "b.ge 1b \n" - - "2: \n" - "adds %w[width], %w[width], %w[vl] \n" - "b.eq 99f \n" - - // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p0.h, wzr, %w[width] \n" - "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" - "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" - "umullb z4.s, z2.h, z0.h \n" - "umullt z2.s, z2.h, z0.h \n" - "umlalb z4.s, z3.h, z1.h \n" - "umlalt z2.s, z3.h, z1.h \n" - "rshrnb z3.h, z4.s, #8 \n" - "rshrnt z3.h, z2.s, #8 \n" - "uqshl z3.h, p0/m, z3.h, z31.h \n" - "shrnb z3.b, z3.h, #8 \n" - "st1b {z3.h}, p0, [%[dst_ptr]] \n" - - "99: \n" - : [src_ptr] "+r"(src_ptr), // %[src_ptr] - [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] - [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] - : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] - [y1_fraction] "r"(y1_fraction), // %[y1_fraction] - [shift] "r"(shift) // %[shift] - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z31", "p0"); -} - __arm_locally_streaming void Convert8To8Row_SME(const uint8_t* src_y, uint8_t* dst_y, int scale, diff --git a/source/row_sve.cc b/source/row_sve.cc index 749c0b7d5..4e465f7c4 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1474,6 +1474,98 @@ void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1"); } +void Convert16To8Row_SVE2(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width) { + const int shift = 23 - __builtin_clz((int32_t)scale); + int vl; + asm volatile( + "cnth %x[vl] \n" + "dup z31.h, %w[shift] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.h \n" + "1: \n" + "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" + "incb %[src_ptr] \n" + "subs %w[width], %w[width], %w[vl] \n" + "uqshl z2.h, p0/m, z2.h, z31.h \n" + "shrnb z2.b, z2.h, #8 \n" + "st1b {z2.h}, p0, [%[dst_ptr]] \n" + "inch %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.h, wzr, %w[width] \n" + "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" + "uqshl z2.h, p0/m, z2.h, z31.h \n" + "shrnb z2.b, z2.h, #8 \n" + "st1b {z2.h}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_y), // %[src_ptr] + [dst_ptr] "+r"(dst_y), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [shift] "r"(shift) // %[shift] + : "cc", "memory", "z2", "z31", "p0"); +} + +void HalfRow_16To8_SVE2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + const int shift = 23 - __builtin_clz((int32_t)scale); + int vl; + asm volatile( + "cnth %x[vl] \n" + "dup z31.h, %w[shift] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.h \n" + "1: \n" + "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" + "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" + "incb %[src_ptr] \n" + "incb %[src_ptr1] \n" + "urhadd z2.h, p0/m, z2.h, z3.h \n" + "subs %w[width], %w[width], %w[vl] \n" + "uqshl z2.h, p0/m, z2.h, z31.h \n" + "shrnb z2.b, z2.h, #8 \n" + "st1b {z2.h}, p0, [%[dst_ptr]] \n" + "inch %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.h, wzr, %w[width] \n" + "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" + "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" + "urhadd z2.h, p0/m, z2.h, z3.h \n" + "uqshl z2.h, p0/m, z2.h, z31.h \n" + "shrnb z2.b, z2.h, #8 \n" + "st1b {z2.h}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_uv), // %[src_ptr] + [src_ptr1] "+r"(src_uv1), // %[src_ptr1] + [dst_ptr] "+r"(dst_uv), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [shift] "r"(shift) // %[shift] + : "cc", "memory", "z2", "z3", "z31", "p0"); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/scale_common.cc b/source/scale_common.cc index 1c9b005d0..bf9aa3e27 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1759,75 +1759,6 @@ void ScalePlaneVertical_16(int src_height, } } -// Use scale to convert lsb formats to msb, depending how many bits there are: -// 32768 = 9 bits -// 16384 = 10 bits -// 4096 = 12 bits -// 256 = 16 bits -// TODO(fbarchard): change scale to bits -void ScalePlaneVertical_16To8(int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint16_t* src_argb, - uint8_t* dst_argb, - int x, - int y, - int dy, - int wpp, /* words per pixel. normally 1 */ - int scale, - enum FilterMode filtering) { - // TODO(fbarchard): Allow higher wpp. - int dst_width_words = dst_width * wpp; - // TODO(https://crbug.com/libyuv/931): Add NEON 32 bit and AVX2 versions. - void (*InterpolateRow_16To8)(uint8_t* dst_argb, const uint16_t* src_argb, - ptrdiff_t src_stride, int scale, int dst_width, - int source_y_fraction) = InterpolateRow_16To8_C; - const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0; - int j; - assert(wpp >= 1 && wpp <= 2); - assert(src_height != 0); - assert(dst_width > 0); - assert(dst_height > 0); - src_argb += (x >> 16) * wpp; - -#if defined(HAS_INTERPOLATEROW_16TO8_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - InterpolateRow_16To8 = InterpolateRow_16To8_Any_NEON; - if (IS_ALIGNED(dst_width, 8)) { - InterpolateRow_16To8 = InterpolateRow_16To8_NEON; - } - } -#endif -#if defined(HAS_INTERPOLATEROW_16TO8_SME) - if (TestCpuFlag(kCpuHasSME)) { - InterpolateRow_16To8 = InterpolateRow_16To8_SME; - } -#endif -#if defined(HAS_INTERPOLATEROW_16TO8_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - InterpolateRow_16To8 = InterpolateRow_16To8_Any_AVX2; - if (IS_ALIGNED(dst_width, 32)) { - InterpolateRow_16To8 = InterpolateRow_16To8_AVX2; - } - } -#endif - for (j = 0; j < dst_height; ++j) { - int yi; - int yf; - if (y > max_y) { - y = max_y; - } - yi = y >> 16; - yf = filtering ? ((y >> 8) & 255) : 0; - InterpolateRow_16To8(dst_argb, src_argb + yi * (ptrdiff_t)src_stride, - src_stride, scale, dst_width_words, yf); - dst_argb += dst_stride; - y += dy; - } -} - // Simplify the filtering based on scale factors. enum FilterMode ScaleFilterReduce(int src_width, int src_height, diff --git a/unit_test/convert_test.cc b/unit_test/convert_test.cc index ba1102e50..9d490aa7e 100644 --- a/unit_test/convert_test.cc +++ b/unit_test/convert_test.cc @@ -59,7 +59,7 @@ namespace libyuv { #define TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \ - DST_SUBSAMP_X, DST_SUBSAMP_Y, W1280, N, NEG, OFF, \ + DST_SUBSAMP_X, DST_SUBSAMP_Y, W1280, N, NEG, OFF, DOY, \ SRC_DEPTH) \ TEST_F(LibYUVConvertTest, SRC_FMT_PLANAR##To##FMT_PLANAR##N) { \ static_assert(SRC_BPC == 1 || SRC_BPC == 2, "SRC BPC unsupported"); \ @@ -113,7 +113,7 @@ namespace libyuv { MaskCpuFlags(disable_cpu_flags_); \ SRC_FMT_PLANAR##To##FMT_PLANAR( \ src_y_p, kWidth, src_u_p, kSrcHalfWidth, src_v_p, kSrcHalfWidth, \ - reinterpret_cast(dst_y_c), kWidth, \ + DOY ? reinterpret_cast(dst_y_c) : NULL, kWidth, \ reinterpret_cast(dst_u_c), kDstHalfWidth, \ reinterpret_cast(dst_v_c), kDstHalfWidth, kWidth, \ NEG kHeight); \ @@ -121,13 +121,15 @@ namespace libyuv { for (int i = 0; i < benchmark_iterations_; ++i) { \ SRC_FMT_PLANAR##To##FMT_PLANAR( \ src_y_p, kWidth, src_u_p, kSrcHalfWidth, src_v_p, kSrcHalfWidth, \ - reinterpret_cast(dst_y_opt), kWidth, \ + DOY ? reinterpret_cast(dst_y_opt) : NULL, kWidth, \ reinterpret_cast(dst_u_opt), kDstHalfWidth, \ reinterpret_cast(dst_v_opt), kDstHalfWidth, kWidth, \ NEG kHeight); \ } \ - for (int i = 0; i < kHeight * kWidth * DST_BPC; ++i) { \ - ASSERT_EQ(dst_y_c[i], dst_y_opt[i]); \ + if (DOY) { \ + for (int i = 0; i < kHeight * kWidth * DST_BPC; ++i) { \ + ASSERT_EQ(dst_y_c[i], dst_y_opt[i]); \ + } \ } \ for (int i = 0; i < kDstHalfWidth * kDstHalfHeight * DST_BPC; ++i) { \ ASSERT_EQ(dst_u_c[i], dst_u_opt[i]); \ @@ -150,23 +152,29 @@ namespace libyuv { DST_SUBSAMP_X, DST_SUBSAMP_Y, SRC_DEPTH) \ TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \ - benchmark_width_ + 1, _Any, +, 0, SRC_DEPTH) \ + benchmark_width_ + 1, _Any, +, 0, 1, SRC_DEPTH) \ TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \ - benchmark_width_, _Unaligned, +, 2, SRC_DEPTH) \ + benchmark_width_, _Unaligned, +, 2, 1, SRC_DEPTH) \ TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \ - benchmark_width_, _Invert, -, 0, SRC_DEPTH) \ + benchmark_width_, _Invert, -, 0, 1, SRC_DEPTH) \ TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \ - benchmark_width_, _Opt, +, 0, SRC_DEPTH) + benchmark_width_, _Opt, +, 0, 1, SRC_DEPTH) \ + TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \ + FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \ + benchmark_width_, _NullY, +, 0, 0, SRC_DEPTH) #else #define TESTPLANARTOP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \ SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \ DST_SUBSAMP_X, DST_SUBSAMP_Y, SRC_DEPTH) \ TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \ FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \ - benchmark_width_, _Opt, +, 0, SRC_DEPTH) + benchmark_width_, _NullY, +, 0, 0, SRC_DEPTH) \ + TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \ + FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \ + benchmark_width_, _Opt, +, 0, 1, SRC_DEPTH) #endif TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I420, uint8_t, 1, 2, 2, 8) @@ -2789,6 +2797,59 @@ TEST_F(LibYUVConvertTest, ABGRToI420_Check) { benchmark_cpu_info_); } +// I210ToI420/I212ToI420 funnel through I21xToI420, which scales chroma +// vertically. Verify oversize heights do not overflow a signed int. +TEST_F(LibYUVConvertTest, I21xToI420NoHeightOverflow) { + int width = 4; + int height = 40000; + int half_width = (width + 1) / 2; + int half_height = (height + 1) / 2; + // I21x: 16-bit 4:2:2 + align_buffer_page_end_16(src_y, width * height); + align_buffer_page_end_16(src_u, half_width * height); + align_buffer_page_end_16(src_v, half_width * height); + // I420: 8-bit 4:2:0 + align_buffer_page_end(dst_y, width * height); + align_buffer_page_end(dst_u, half_width * half_height); + align_buffer_page_end(dst_v, half_width * half_height); + + memset(src_y, 0, width * height * sizeof(uint16_t)); + memset(src_u, 0, half_width * height * sizeof(uint16_t)); + memset(src_v, 0, half_width * height * sizeof(uint16_t)); + + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32767)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32768)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32769)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32767)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32768)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32769)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32767)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32768)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32769)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32767)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32768)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32769)); + + free_aligned_buffer_page_end_16(src_y); + free_aligned_buffer_page_end_16(src_u); + free_aligned_buffer_page_end_16(src_v); + free_aligned_buffer_page_end(dst_y); + free_aligned_buffer_page_end(dst_u); + free_aligned_buffer_page_end(dst_v); +} + #endif // !defined(LEAN_TESTS) } // namespace libyuv