diff --git a/README.chromium b/README.chromium index c5d59b89b..9642ba633 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1956 +Version: 1957 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 4647844c4..25c146d63 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -206,6 +206,7 @@ extern "C" { #define HAS_DETILETOYUY2_SSE2 #define HAS_HALFMERGEUVROW_SSSE3 #define HAS_HALFROW_16TO8_SSSE3 +#define HAS_HALFWIDTHROW_16TO8_SSSE3 #define HAS_I210TOAR30ROW_SSSE3 #define HAS_I210TOARGBROW_SSSE3 #define HAS_I212TOAR30ROW_SSSE3 @@ -289,6 +290,7 @@ extern "C" { #define HAS_DIVIDEROW_16_AVX2 #define HAS_HALFMERGEUVROW_AVX2 #define HAS_HALFROW_16TO8_AVX2 +#define HAS_HALFWIDTHROW_16TO8_AVX2 #define HAS_I210TOAR30ROW_AVX2 #define HAS_I210TOARGBROW_AVX2 #define HAS_I212TOAR30ROW_AVX2 @@ -383,6 +385,7 @@ extern "C" { #define HAS_ARGBTORGB24ROW_AVX512VBMI #define HAS_CONVERT16TO8ROW_AVX512BW #define HAS_HALFROW_16TO8_AVX512BW +#define HAS_HALFWIDTHROW_16TO8_AVX512BW #endif // The following are available for AVX512 clang x64 platforms: @@ -454,6 +457,7 @@ extern "C" { #define HAS_HALFFLOATROW_NEON #define HAS_HALFMERGEUVROW_NEON #define HAS_HALFROW_16TO8_NEON +#define HAS_HALFWIDTHROW_16TO8_NEON #define HAS_I400TOARGBROW_NEON #define HAS_I422ALPHATOARGBROW_NEON #define HAS_I422TOARGB1555ROW_NEON @@ -581,13 +585,14 @@ extern "C" { #define HAS_ARGBTOYMATRIXROW_SVE2 #define HAS_AYUVTOUVROW_SVE2 #define HAS_AYUVTOVUROW_SVE2 -#define HAS_CONVERT16TO8ROW_SVE2 +// #define HAS_CONVERT16TO8ROW_SVE2 // Disabled: NEON version is faster for 128 bit vectors. #define HAS_CONVERT8TO8ROW_SVE2 #define HAS_COPYROW_SVE2 #define HAS_DIVIDEROW_16_SVE2 #define HAS_HALFFLOATROW_SVE2 #define HAS_HALFMERGEUVROW_SVE2 -#define HAS_HALFROW_16TO8_SVE2 +// #define HAS_HALFROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors. +// #define HAS_HALFWIDTHROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors. #define HAS_I210ALPHATOARGBROW_SVE2 #define HAS_I210TOAR30ROW_SVE2 #define HAS_I210TOARGBROW_SVE2 @@ -642,6 +647,7 @@ extern "C" { #define HAS_CONVERT8TO8ROW_SME #define HAS_COPYROW_SME #define HAS_HALFROW_16TO8_SME +#define HAS_HALFWIDTHROW_16TO8_SME #define HAS_I210ALPHATOARGBROW_SME #define HAS_I210TOAR30ROW_SME #define HAS_I210TOARGBROW_SME @@ -835,6 +841,7 @@ extern "C" { #define HAS_CONVERT16TO8ROW_RVV #define HAS_COPYROW_RVV #define HAS_HALFROW_16TO8_RVV +#define HAS_HALFWIDTHROW_16TO8_RVV #define HAS_INTERPOLATEROW_RVV #define HAS_RGBTOUV444MATRIXROW_RVV #define HAS_RGBTOUVMATRIXROW_RVV @@ -3663,6 +3670,72 @@ void HalfRow_16To8_RVV(const uint16_t* src_uv, int scale, int width); +void HalfWidthRow_16To8_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_Odd_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_Any_SSSE3(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); +void HalfWidthRow_16To8_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); +void HalfWidthRow_16To8_Any_AVX512BW(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); +void HalfWidthRow_16To8_NEON(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_Any_NEON(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); +void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_SME(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_RVV(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); + void Convert8To8Row_C(const uint8_t* src_y, uint8_t* dst_y, int scale, diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 90b949a67..99b450379 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -223,16 +223,6 @@ void ScalePlaneVertical_16(int src_height, int wpp, enum FilterMode filtering); -void ScalePlaneDown2_16To8(int src_width, - int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint16_t* src_ptr, - uint8_t* dst_ptr, - int scale, - enum FilterMode filtering); // Simplify the filtering based on scale factors. enum FilterMode ScaleFilterReduce(int src_width, @@ -274,16 +264,6 @@ void ScaleRowDown2_16_C(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst, int dst_width); -void ScaleRowDown2_16To8_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale); -void ScaleRowDown2_16To8_Odd_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale); void ScaleRowDown2Linear_C(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, @@ -292,16 +272,6 @@ void ScaleRowDown2Linear_16_C(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst, int dst_width); -void ScaleRowDown2Linear_16To8_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale); -void ScaleRowDown2Linear_16To8_Odd_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale); void ScaleRowDown2Box_C(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, @@ -314,16 +284,6 @@ void ScaleRowDown2Box_16_C(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst, int dst_width); -void ScaleRowDown2Box_16To8_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale); -void ScaleRowDown2Box_16To8_Odd_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale); void ScaleRowDown4_C(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index 13d655c5c..91ef0c164 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1956 +#define LIBYUV_VERSION 1957 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/convert.cc b/source/convert.cc index 6a916bbb1..b3d6cd201 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -230,7 +230,14 @@ static int I41xToI420(const uint16_t* src_y, int width, int height, int depth) { + int y; const int scale = 1 << (24 - depth); + const int uv_width = SUBSAMPLE(width, 1, 1); + void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale, + int width) = Convert16To8Row_C; + void (*HalfWidthRow_16To8)(const uint16_t* src_uv, ptrdiff_t src_uv_stride, + uint8_t* dst_uv, int scale, int width) = + (width & 1) ? HalfWidthRow_16To8_Odd_C : HalfWidthRow_16To8_C; if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -249,18 +256,124 @@ static int I41xToI420(const uint16_t* src_y, src_stride_v = -src_stride_v; } - { - const int uv_width = SUBSAMPLE(width, 1, 1); - const int uv_height = SUBSAMPLE(height, 1, 1); - - if (dst_y) { - Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width, - height); +#if defined(HAS_CONVERT16TO8ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + Convert16To8Row = Convert16To8Row_Any_NEON; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_NEON; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + Convert16To8Row = Convert16To8Row_SVE2; + } +#endif +#if defined(HAS_CONVERT16TO8ROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + Convert16To8Row = Convert16To8Row_SME; + } +#endif +#if defined(HAS_CONVERT16TO8ROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + Convert16To8Row = Convert16To8Row_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_SSSE3; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + Convert16To8Row = Convert16To8Row_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + Convert16To8Row = Convert16To8Row_AVX2; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + Convert16To8Row = Convert16To8Row_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + Convert16To8Row = Convert16To8Row_AVX512BW; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + Convert16To8Row = Convert16To8Row_RVV; + } +#endif + + if (IS_ALIGNED(width, 2)) { +#if defined(HAS_HALFWIDTHROW_16TO8_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_NEON; + if (IS_ALIGNED(uv_width, 8)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_NEON; + } + } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_SVE2; + } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_SME) + if (TestCpuFlag(kCpuHasSME)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_SME; + } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_SSSE3; + if (IS_ALIGNED(uv_width, 8)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_SSSE3; + } + } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_AVX2; + if (IS_ALIGNED(uv_width, 16)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX2; + } + } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_AVX512BW; + if (IS_ALIGNED(uv_width, 32)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX512BW; + } + } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_RVV; + } +#endif + } + + for (y = 0; y < height - 1; y += 2) { + HalfWidthRow_16To8(src_u, src_stride_u, dst_u, scale, uv_width); + HalfWidthRow_16To8(src_v, src_stride_v, dst_v, scale, uv_width); + if (dst_y) { + Convert16To8Row(src_y, dst_y, scale, width); + Convert16To8Row(src_y + src_stride_y, dst_y + dst_stride_y, scale, width); + src_y += src_stride_y * 2; + dst_y += dst_stride_y * 2; + } + src_u += src_stride_u * 2; + src_v += src_stride_v * 2; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + if (height & 1) { + HalfWidthRow_16To8(src_u, 0, dst_u, scale, uv_width); + HalfWidthRow_16To8(src_v, 0, dst_v, scale, uv_width); + if (dst_y) { + Convert16To8Row(src_y, dst_y, scale, width); } - ScalePlaneDown2_16To8(width, height, uv_width, uv_height, src_stride_u, - dst_stride_u, src_u, dst_u, scale, kFilterBilinear); - ScalePlaneDown2_16To8(width, height, uv_width, uv_height, src_stride_v, - dst_stride_v, src_v, dst_v, scale, kFilterBilinear); } return 0; } diff --git a/source/row_any.cc b/source/row_any.cc index 566f584e5..6efdbc4a9 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1732,6 +1732,44 @@ ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15) #undef ANY11HS +// Any 2 to 1 half-width row with scale param +#define ANY11HWS(NAMEANY, ANY_SIMD, TS, TD, MASK) \ + void NAMEANY(const TS* src_ptr, ptrdiff_t src_stride, TD* dst_ptr, \ + int scale, int width) { \ + SIMD_ALIGNED(TS vin[128 * 2]); \ + static_assert((MASK + 1) * 2 * sizeof(TS) <= sizeof(vin), \ + "vin buffer too small"); \ + SIMD_ALIGNED(TD vout[64]); \ + static_assert((MASK + 1) * sizeof(TD) <= sizeof(vout), \ + "vout buffer too small"); \ + memset(vin, 0, sizeof(vin)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr, src_stride, dst_ptr, scale, n); \ + } \ + ptrdiff_t np = n; \ + memcpy(vin, src_ptr + np * 2, r * 2 * sizeof(TS)); \ + memcpy(vin + 128, src_ptr + src_stride + np * 2, r * 2 * sizeof(TS)); \ + ANY_SIMD(vin, 128, vout, scale, MASK + 1); \ + memcpy(dst_ptr + np, vout, r * sizeof(TD)); \ + } + +#ifdef HAS_HALFWIDTHROW_16TO8_SSSE3 +ANY11HWS(HalfWidthRow_16To8_Any_SSSE3, HalfWidthRow_16To8_SSSE3, uint16_t, uint8_t, 7) +#endif +#ifdef HAS_HALFWIDTHROW_16TO8_AVX2 +ANY11HWS(HalfWidthRow_16To8_Any_AVX2, HalfWidthRow_16To8_AVX2, uint16_t, uint8_t, 15) +#endif +#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW +ANY11HWS(HalfWidthRow_16To8_Any_AVX512BW, HalfWidthRow_16To8_AVX512BW, uint16_t, uint8_t, 31) +#endif +#ifdef HAS_HALFWIDTHROW_16TO8_NEON +ANY11HWS(HalfWidthRow_16To8_Any_NEON, HalfWidthRow_16To8_NEON, uint16_t, uint8_t, 7) +#endif + +#undef ANY11HWS + // Any 1 to 1 mirror. #define ANY11M(NAMEANY, ANY_SIMD, BPP, MASK) \ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \ diff --git a/source/row_common.cc b/source/row_common.cc index d1fc258ff..df43aecae 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -3677,6 +3677,63 @@ void HalfRow_16To8_C(const uint16_t* src_uv, } } +void HalfWidthRow_16To8_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* s = src_uv; + const uint16_t* t = src_uv + src_uv_stride; + int x; + for (x = 0; x < width - 1; x += 2) { + dst_uv[0] = STATIC_CAST( + uint8_t, + C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); + dst_uv[1] = STATIC_CAST( + uint8_t, + C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale)); + dst_uv += 2; + s += 4; + t += 4; + } + if (width & 1) { + dst_uv[0] = STATIC_CAST( + uint8_t, + C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); + } +} + +void HalfWidthRow_16To8_Odd_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* s = src_uv; + const uint16_t* t = src_uv + src_uv_stride; + int x; + width -= 1; + for (x = 0; x < width - 1; x += 2) { + dst_uv[0] = STATIC_CAST( + uint8_t, + C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); + dst_uv[1] = STATIC_CAST( + uint8_t, + C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale)); + dst_uv += 2; + s += 4; + t += 4; + } + if (width & 1) { + dst_uv[0] = STATIC_CAST( + uint8_t, + C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); + dst_uv += 1; + s += 2; + t += 2; + } + dst_uv[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + t[0] + 1) >> 1, scale)); +} + // C version 2x2 -> 2x1. void InterpolateRow_C(uint8_t* dst_ptr, const uint8_t* src_ptr, diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 197b705c0..46bdd02e8 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -5438,6 +5438,65 @@ void HalfRow_16To8_SSSE3(const uint16_t* src_uv, } #endif // HAS_HALFROW_16TO8_SSSE3 +#ifdef HAS_HALFWIDTHROW_16TO8_SSSE3 +static const uvec8 kShuffleMaskHalfWidth16To8 = { + 0u, 1u, 4u, 5u, 8u, 9u, 12u, 13u, + 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u}; + +void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + asm volatile( + "movd %4,%%xmm2 \n" + "punpcklwd %%xmm2,%%xmm2 \n" + "pshufd $0x0,%%xmm2,%%xmm2 \n" + "pcmpeqw %%xmm5,%%xmm5 \n" + "psrlw $15,%%xmm5 \n" + "mov $2,%%eax \n" + "movd %%eax,%%xmm6 \n" + "pshufd $0x0,%%xmm6,%%xmm6 \n" + "movdqa %5,%%xmm7 \n" + + // 8 pixels per loop. + LABELALIGN + "1: \n" + "movdqu (%0),%%xmm0 \n" + "movdqu 0x10(%0),%%xmm1 \n" + "movdqu (%0,%3,2),%%xmm3 \n" + "movdqu 0x10(%0,%3,2),%%xmm4 \n" + "add $0x20,%0 \n" + "pmaddwd %%xmm5,%%xmm0 \n" + "pmaddwd %%xmm5,%%xmm1 \n" + "pmaddwd %%xmm5,%%xmm3 \n" + "pmaddwd %%xmm5,%%xmm4 \n" + "paddd %%xmm3,%%xmm0 \n" + "paddd %%xmm4,%%xmm1 \n" + "paddd %%xmm6,%%xmm0 \n" + "paddd %%xmm6,%%xmm1 \n" + "psrld $2,%%xmm0 \n" + "psrld $2,%%xmm1 \n" + "pshufb %%xmm7,%%xmm0 \n" + "pshufb %%xmm7,%%xmm1 \n" + "punpcklqdq %%xmm1,%%xmm0 \n" + "pmulhuw %%xmm2,%%xmm0 \n" + "packuswb %%xmm0,%%xmm0 \n" + "movq %%xmm0,(%1) \n" + "add $0x08,%1 \n" + "sub $0x08,%2 \n" + "jg 1b \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(width) // %2 + : "r"(src_uv_stride), // %3 + "r"(scale), // %4 + "m"(kShuffleMaskHalfWidth16To8) // %5 + : "eax", "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#endif // HAS_HALFWIDTHROW_16TO8_SSSE3 + #ifdef HAS_HALFROW_16TO8_AVX2 void HalfRow_16To8_AVX2(const uint16_t* src_uv, ptrdiff_t src_uv_stride, @@ -5474,6 +5533,59 @@ void HalfRow_16To8_AVX2(const uint16_t* src_uv, } #endif // HAS_HALFROW_16TO8_AVX2 +#ifdef HAS_HALFWIDTHROW_16TO8_AVX2 +void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + asm volatile( + "vmovd %4,%%xmm2 \n" + "vpbroadcastw %%xmm2,%%ymm2 \n" + "vpcmpeqw %%ymm5,%%ymm5,%%ymm5 \n" + "vpsrlw $15,%%ymm5,%%ymm5 \n" + "mov $2,%%eax \n" + "vmovd %%eax,%%xmm6 \n" + "vpbroadcastd %%xmm6,%%ymm6 \n" + + // 16 pixels per loop. + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vmovdqu (%0,%3,2),%%ymm3 \n" + "vmovdqu 0x20(%0,%3,2),%%ymm4 \n" + "add $0x40,%0 \n" + "vpmaddwd %%ymm5,%%ymm0,%%ymm0 \n" + "vpmaddwd %%ymm5,%%ymm1,%%ymm1 \n" + "vpmaddwd %%ymm5,%%ymm3,%%ymm3 \n" + "vpmaddwd %%ymm5,%%ymm4,%%ymm4 \n" + "vpaddd %%ymm3,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm4,%%ymm1,%%ymm1 \n" + "vpaddd %%ymm6,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm6,%%ymm1,%%ymm1 \n" + "vpsrld $2,%%ymm0,%%ymm0 \n" + "vpsrld $2,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%1) \n" + "add $0x10,%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(width) // %2 + : "r"(src_uv_stride), // %3 + "r"(scale) // %4 + : "eax", "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6"); +} +#endif // HAS_HALFWIDTHROW_16TO8_AVX2 + #ifdef HAS_HALFROW_16TO8_AVX512BW void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, ptrdiff_t src_uv_stride, @@ -5511,6 +5623,60 @@ void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, } #endif // HAS_HALFROW_16TO8_AVX512BW +#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW +void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + asm volatile( + "vpbroadcastw %4,%%zmm2 \n" + "mov $1,%%eax \n" + "vmovd %%eax,%%xmm5 \n" + "vpbroadcastw %%xmm5,%%zmm5 \n" + "mov $2,%%eax \n" + "vmovd %%eax,%%xmm6 \n" + "vpbroadcastd %%xmm6,%%zmm6 \n" + + // 32 pixels per loop. + LABELALIGN + "1: \n" + "vmovdqu64 (%0),%%zmm0 \n" + "vmovdqu64 0x40(%0),%%zmm1 \n" + "vmovdqu64 (%1),%%zmm3 \n" + "vmovdqu64 0x40(%1),%%zmm4 \n" + "add $0x80,%0 \n" + "add $0x80,%1 \n" + "vpmaddwd %%zmm5,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm5,%%zmm1,%%zmm1 \n" + "vpmaddwd %%zmm5,%%zmm3,%%zmm3 \n" + "vpmaddwd %%zmm5,%%zmm4,%%zmm4 \n" + "vpaddd %%zmm3,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm4,%%zmm1,%%zmm1 \n" + "vpaddd %%zmm6,%%zmm0,%%zmm0 \n" + "vpaddd %%zmm6,%%zmm1,%%zmm1 \n" + "vpsrld $2,%%zmm0,%%zmm0 \n" + "vpsrld $2,%%zmm1,%%zmm1 \n" + "vpmovusdw %%zmm0,%%ymm0 \n" + "vpmovusdw %%zmm1,%%ymm1 \n" + "vinserti64x4 $1,%%ymm1,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmovuswb %%zmm0,(%2) \n" + "add $0x20,%2 \n" + "sub $0x20,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(src_uv1), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 + : "r"(scale) // %4 + : "eax", "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6"); +} +#endif // HAS_HALFWIDTHROW_16TO8_AVX512BW + // Use scale to convert to lsb formats depending how many bits there are: // 512 = 9 bits // 1024 = 10 bits diff --git a/source/row_neon.cc b/source/row_neon.cc index 70da2c442..160c7b2b4 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -3710,6 +3710,37 @@ void HalfRow_16To8_NEON(const uint16_t* src_uv, : "cc", "memory", "q0", "q1", "q2", "q3", "q4"); } +void HalfWidthRow_16To8_NEON(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + const int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr + asm volatile( + "vdup.16 q4, %4 \n" + "1: \n" + "vld1.16 {q0, q1}, [%0]! \n" + "vld1.16 {q2, q3}, [%1]! \n" + "subs %3, %3, #8 \n" // 8 dst pixels per loop + "vpaddl.u16 q0, q0 \n" + "vpaddl.u16 q1, q1 \n" + "vpadal.u16 q0, q2 \n" + "vpadal.u16 q1, q3 \n" + "vrshrn.u32 d0, q0, #2 \n" + "vrshrn.u32 d1, q1, #2 \n" + "vshl.u16 q0, q0, q4 \n" // shr = q4 is negative + "vqmovn.u16 d0, q0 \n" + "vst1.8 {d0}, [%2]! \n" + "bgt 1b \n" + : "+r"(src_uv), // %0 + "+r"(src_uv1), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 + : "r"(shift) // %4 + : "cc", "memory", "q0", "q1", "q2", "q3", "q4"); +} + // Use scale to convert J420 to I420 // scale parameter is 8.8 fixed point but limited to 0 to 255 // Function is based on DivideRow, but adds a bias diff --git a/source/row_neon64.cc b/source/row_neon64.cc index 6becaec87..ead3e6cdd 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -5135,6 +5135,39 @@ void HalfRow_16To8_NEON(const uint16_t* src_uv, : "cc", "memory", "v0", "v1", "v2", "v3", "v4"); } +void HalfWidthRow_16To8_NEON(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + const int shift = 23 - __builtin_clz((int32_t)scale); + asm volatile( + "dup v4.8h, %w4 \n" + "1: \n" + "ldp q0, q1, [%0], #32 \n" + "ldp q2, q3, [%1], #32 \n" + "subs %w3, %w3, #8 \n" + "uaddlp v0.4s, v0.8h \n" + "uaddlp v1.4s, v1.8h \n" + "prfm pldl1keep, [%0, 448] \n" + "uadalp v0.4s, v2.8h \n" + "uadalp v1.4s, v3.8h \n" + "prfm pldl1keep, [%1, 448] \n" + "rshrn v0.4h, v0.4s, #2 \n" + "rshrn2 v0.8h, v1.4s, #2 \n" + "uqshl v0.8h, v0.8h, v4.8h \n" + "uzp2 v0.16b, v0.16b, v0.16b \n" + "str d0, [%2], #8 \n" + "b.gt 1b \n" + : "+r"(src_uv), // %0 + "+r"(src_uv1), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 + : "r"(shift) // %4 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4"); +} + // Use scale to convert J420 to I420 // scale parameter is 8.8 fixed point but limited to 0 to 255 // Function is based on DivideRow, but adds a bias diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 17ba7bc83..90b921e44 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -1830,7 +1830,8 @@ void HalfRow_16To8_RVV(const uint16_t* src_uv, vuint16m4_t v_src0 = __riscv_vle16_v_u16m4(src_uv, vl); vuint16m4_t v_src1 = __riscv_vle16_v_u16m4(src_uv1, vl); #ifdef LIBYUV_RVV_HAS_VXRM_ARG - vuint16m4_t v_avg = __riscv_vaaddu_vv_u16m4_vxrm(v_src0, v_src1, 0, vl); + vuint16m4_t v_avg = + __riscv_vaaddu_vv_u16m4(v_src0, v_src1, __RISCV_VXRM_RNU, vl); #else vuint16m4_t v_avg = __riscv_vaaddu_vv_u16m4(v_src0, v_src1, vl); #endif @@ -1844,6 +1845,69 @@ void HalfRow_16To8_RVV(const uint16_t* src_uv, } #endif +#ifdef HAS_HALFWIDTHROW_16TO8_RVV +#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG) +void HalfWidthRow_16To8_RVV(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* s = src_uv; + const uint16_t* t = src_uv + src_uv_stride; + size_t w = (size_t)width; + const int shift = __builtin_clz((int32_t)scale) - 15; + do { + size_t vl = __riscv_vsetvl_e16m4(w); + vuint16m4x2_t v_s = __riscv_vlseg2e16_v_u16m4x2(s, vl); + vuint16m4x2_t v_t = __riscv_vlseg2e16_v_u16m4x2(t, vl); + vuint16m4_t v_s0 = __riscv_vget_v_u16m4x2_u16m4(v_s, 0); + vuint16m4_t v_s1 = __riscv_vget_v_u16m4x2_u16m4(v_s, 1); + vuint16m4_t v_t0 = __riscv_vget_v_u16m4x2_u16m4(v_t, 0); + vuint16m4_t v_t1 = __riscv_vget_v_u16m4x2_u16m4(v_t, 1); + vuint32m8_t v_s01 = __riscv_vwaddu_vv_u32m8(v_s0, v_s1, vl); + vuint32m8_t v_t01 = __riscv_vwaddu_vv_u32m8(v_t0, v_t1, vl); + vuint32m8_t v_st01 = __riscv_vadd_vv_u32m8(v_s01, v_t01, vl); + vuint16m4_t v_avg = + __riscv_vnclipu_wx_u16m4(v_st01, 2, __RISCV_VXRM_RNU, vl); + vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_avg, shift, vl); + __riscv_vse8_v_u8m2(dst_uv, v_dst, vl); + w -= vl; + s += 2 * vl; + t += 2 * vl; + dst_uv += vl; + } while (w > 0); +} +#else +void HalfWidthRow_16To8_RVV(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* s = src_uv; + const uint16_t* t = src_uv + src_uv_stride; + size_t w = (size_t)width; + const int shift = __builtin_clz((int32_t)scale) - 15; + asm volatile("csrwi vxrm, 0"); + do { + size_t vl = __riscv_vsetvl_e16m4(w); + vuint16m4_t v_s0, v_s1, v_t0, v_t1; + __riscv_vlseg2e16_v_u16m4(&v_s0, &v_s1, s, vl); + __riscv_vlseg2e16_v_u16m4(&v_t0, &v_t1, t, vl); + vuint32m8_t v_s01 = __riscv_vwaddu_vv_u32m8(v_s0, v_s1, vl); + vuint32m8_t v_t01 = __riscv_vwaddu_vv_u32m8(v_t0, v_t1, vl); + vuint32m8_t v_st01 = __riscv_vadd_vv_u32m8(v_s01, v_t01, vl); + vuint16m4_t v_avg = __riscv_vnclipu_wx_u16m4(v_st01, 2, vl); + vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_avg, shift, vl); + __riscv_vse8_v_u8m2(dst_uv, v_dst, vl); + w -= vl; + s += 2 * vl; + t += 2 * vl; + dst_uv += vl; + } while (w > 0); +} +#endif +#endif + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_sme.cc b/source/row_sme.cc index 7a18898d0..fe2d0a117 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -965,6 +965,71 @@ __arm_locally_streaming void HalfRow_16To8_SME(const uint16_t* src_ptr, : "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0"); } +__arm_locally_streaming void HalfWidthRow_16To8_SME(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width) { + const uint16_t* src_ptr1 = src_ptr + src_stride; + const int shift = 23 - __builtin_clz((int32_t)scale); + int vl; + asm volatile( + "cnth %x[vl] \n" + "dup z31.h, %w[shift] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.h \n" + "1: \n" + "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n" + "ld2h {z2.h, z3.h}, p0/z, [%[src_ptr1]] \n" + "incb %[src_ptr], all, mul #2 \n" + "incb %[src_ptr1], all, mul #2 \n" + "uaddlb z4.s, z0.h, z1.h \n" + "uaddlt z5.s, z0.h, z1.h \n" + "uaddlb z6.s, z2.h, z3.h \n" + "uaddlt z7.s, z2.h, z3.h \n" + "add z4.s, z4.s, z6.s \n" + "add z5.s, z5.s, z7.s \n" + "rshrnb z0.h, z4.s, #2 \n" + "rshrnt z0.h, z5.s, #2 \n" + "subs %w[width], %w[width], %w[vl] \n" + "uqshl z0.h, p0/m, z0.h, z31.h \n" + "shrnb z0.b, z0.h, #8 \n" + "st1b {z0.h}, p0, [%[dst_ptr]] \n" + "inch %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.h, wzr, %w[width] \n" + "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n" + "ld2h {z2.h, z3.h}, p0/z, [%[src_ptr1]] \n" + "uaddlb z4.s, z0.h, z1.h \n" + "uaddlt z5.s, z0.h, z1.h \n" + "uaddlb z6.s, z2.h, z3.h \n" + "uaddlt z7.s, z2.h, z3.h \n" + "add z4.s, z4.s, z6.s \n" + "add z5.s, z5.s, z7.s \n" + "rshrnb z0.h, z4.s, #2 \n" + "rshrnt z0.h, z5.s, #2 \n" + "uqshl z0.h, p0/m, z0.h, z31.h \n" + "shrnb z0.b, z0.h, #8 \n" + "st1b {z0.h}, p0, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] + : [shift] "r"(shift) // %[shift] + : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31", + "p0"); +} + __arm_locally_streaming void Convert8To8Row_SME(const uint8_t* src_y, uint8_t* dst_y, int scale, diff --git a/source/row_sve.cc b/source/row_sve.cc index 1ce86940d..7c389ecc0 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1530,6 +1530,8 @@ void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, : "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1"); } +#ifdef HAS_CONVERT16TO8ROW_SVE2 +// Disabled: NEON version is faster for 128 bit vectors. void Convert16To8Row_SVE2(const uint16_t* src_y, uint8_t* dst_y, int scale, @@ -1571,7 +1573,10 @@ void Convert16To8Row_SVE2(const uint16_t* src_y, : [shift] "r"(shift) // %[shift] : "cc", "memory", "z2", "z31", "p0"); } +#endif // HAS_CONVERT16TO8ROW_SVE2 +#ifdef HAS_HALFROW_16TO8_SVE2 +// Disabled: NEON version is faster for 128 bit vectors. void HalfRow_16To8_SVE2(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, @@ -1621,6 +1626,92 @@ void HalfRow_16To8_SVE2(const uint16_t* src_uv, : [shift] "r"(shift) // %[shift] : "cc", "memory", "z2", "z3", "z31", "p0"); } +#endif // HAS_HALFROW_16TO8_SVE2 + +#ifdef HAS_HALFWIDTHROW_16TO8_SVE2 +// Disabled: NEON version is faster for 128 bit vectors. +void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + const int shift = 23 - __builtin_clz((int32_t)scale); + int vl; + int vl_input; + asm volatile( + "cnth %x[vl] \n" + "dup z31.h, %w[shift] \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.h \n" + "1: \n" + "ld1h {z0.h}, p0/z, [%[src_ptr]] \n" + "ld1h {z1.h}, p0/z, [%[src_ptr], #1, mul vl] \n" + "ld1h {z2.h}, p0/z, [%[src_ptr1]] \n" + "ld1h {z3.h}, p0/z, [%[src_ptr1], #1, mul vl] \n" + "incb %[src_ptr], all, mul #4 \n" + "incb %[src_ptr1], all, mul #4 \n" + "uzp1 z4.h, z0.h, z1.h \n" + "uzp2 z5.h, z0.h, z1.h \n" + "uzp1 z6.h, z2.h, z3.h \n" + "uzp2 z7.h, z2.h, z3.h \n" + "uaddlb z0.s, z4.h, z5.h \n" + "uaddlt z1.s, z4.h, z5.h \n" + "uaddlb z2.s, z6.h, z7.h \n" + "uaddlt z3.s, z6.h, z7.h \n" + "add z0.s, z0.s, z2.s \n" + "add z1.s, z1.s, z3.s \n" + "rshrnb z0.h, z0.s, #2 \n" + "rshrnt z0.h, z1.s, #2 \n" + "subs %w[width], %w[width], %w[vl] \n" + "uqshl z0.h, p0/m, z0.h, z31.h \n" + "shrnb z0.b, z0.h, #8 \n" + "st1b {z0.h}, p0, [%[dst_ptr]] \n" + "inch %[dst_ptr] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "lsl %w[vl_input], %w[width], #1 \n" + "whilelt p0.h, wzr, %w[vl_input] \n" + "whilelt p1.h, %w[vl], %w[vl_input] \n" + "whilelt p2.h, wzr, %w[width] \n" + "ld1h {z0.h}, p0/z, [%[src_ptr]] \n" + "ld1h {z1.h}, p1/z, [%[src_ptr], #1, mul vl] \n" + "ld1h {z2.h}, p0/z, [%[src_ptr1]] \n" + "ld1h {z3.h}, p1/z, [%[src_ptr1], #1, mul vl] \n" + "uzp1 z4.h, z0.h, z1.h \n" + "uzp2 z5.h, z0.h, z1.h \n" + "uzp1 z6.h, z2.h, z3.h \n" + "uzp2 z7.h, z2.h, z3.h \n" + "uaddlb z0.s, z4.h, z5.h \n" + "uaddlt z1.s, z4.h, z5.h \n" + "uaddlb z2.s, z6.h, z7.h \n" + "uaddlt z3.s, z6.h, z7.h \n" + "add z0.s, z0.s, z2.s \n" + "add z1.s, z1.s, z3.s \n" + "rshrnb z0.h, z0.s, #2 \n" + "rshrnt z0.h, z1.s, #2 \n" + "uqshl z0.h, p2/m, z0.h, z31.h \n" + "shrnb z0.b, z0.h, #8 \n" + "st1b {z0.h}, p2, [%[dst_ptr]] \n" + + "99: \n" + : [src_ptr] "+r"(src_uv), // %[src_ptr] + [src_ptr1] "+r"(src_uv1), // %[src_ptr1] + [dst_ptr] "+r"(dst_uv), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [vl_input] "=&r"(vl_input) // %[vl_input] + : [shift] "r"(shift) // %[shift] + : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31", + "p0", "p1", "p2"); +} +#endif // HAS_HALFWIDTHROW_16TO8_SVE2 #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) diff --git a/source/scale.cc b/source/scale.cc index 09be3c626..ade50df2c 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -207,50 +207,6 @@ static void ScalePlaneDown2_16(int src_width, } } -void ScalePlaneDown2_16To8(int src_width, - int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint16_t* src_ptr, - uint8_t* dst_ptr, - int scale, - enum FilterMode filtering) { - int y; - void (*ScaleRowDown2)(const uint16_t* src_ptr, ptrdiff_t src_stride, - uint8_t* dst_ptr, int dst_width, int scale) = - (src_width & 1) - ? (filtering == kFilterNone - ? ScaleRowDown2_16To8_Odd_C - : (filtering == kFilterLinear ? ScaleRowDown2Linear_16To8_Odd_C - : ScaleRowDown2Box_16To8_Odd_C)) - : (filtering == kFilterNone - ? ScaleRowDown2_16To8_C - : (filtering == kFilterLinear ? ScaleRowDown2Linear_16To8_C - : ScaleRowDown2Box_16To8_C)); - ptrdiff_t row_stride = (ptrdiff_t)src_stride * 2; - (void)dst_height; - if (!filtering) { - src_ptr += src_stride; // Point to odd rows. - src_stride = 0; - } - - if (filtering == kFilterLinear) { - src_stride = 0; - } - for (y = 0; y < src_height / 2; ++y) { - ScaleRowDown2(src_ptr, src_stride, dst_ptr, dst_width, scale); - src_ptr += row_stride; - dst_ptr += dst_stride; - } - if (src_height & 1) { - if (!filtering) { - src_ptr -= src_stride; // Point to last row. - } - ScaleRowDown2(src_ptr, 0, dst_ptr, dst_width, scale); - } -} // Scale plane, 1/4 // This is an optimized version for scaling down a plane to 1/4 of diff --git a/source/scale_common.cc b/source/scale_common.cc index bf9aa3e27..ce8843c3b 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -29,19 +29,6 @@ extern "C" { #define STATIC_CAST(type, expr) (type)(expr) #endif -// TODO(fbarchard): make clamp255 preserve negative values. -static __inline int32_t clamp255(int32_t v) { - return (-(v >= 255) | v) & 255; -} - -// Use scale to convert lsb formats to msb, depending how many bits there are: -// 32768 = 9 bits -// 16384 = 10 bits -// 4096 = 12 bits -// 256 = 16 bits -// TODO(fbarchard): change scale to bits -#define C16TO8(v, scale) clamp255(((v) * (scale)) >> 16) - static __inline int Abs(int v) { return v >= 0 ? v : -v; } @@ -81,49 +68,6 @@ void ScaleRowDown2_16_C(const uint16_t* src_ptr, } } -void ScaleRowDown2_16To8_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale) { - int x; - (void)src_stride; - assert(scale >= 256); - assert(scale <= 32768); - for (x = 0; x < dst_width - 1; x += 2) { - dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[1], scale)); - dst[1] = STATIC_CAST(uint8_t, C16TO8(src_ptr[3], scale)); - dst += 2; - src_ptr += 4; - } - if (dst_width & 1) { - dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[1], scale)); - } -} - -void ScaleRowDown2_16To8_Odd_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale) { - int x; - (void)src_stride; - assert(scale >= 256); - assert(scale <= 32768); - dst_width -= 1; - for (x = 0; x < dst_width - 1; x += 2) { - dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[1], scale)); - dst[1] = STATIC_CAST(uint8_t, C16TO8(src_ptr[3], scale)); - dst += 2; - src_ptr += 4; - } - if (dst_width & 1) { - dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[1], scale)); - dst += 1; - src_ptr += 2; - } - dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[0], scale)); -} void ScaleRowDown2Linear_C(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -161,51 +105,6 @@ void ScaleRowDown2Linear_16_C(const uint16_t* src_ptr, } } -void ScaleRowDown2Linear_16To8_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale) { - const uint16_t* s = src_ptr; - int x; - (void)src_stride; - assert(scale >= 256); - assert(scale <= 32768); - for (x = 0; x < dst_width - 1; x += 2) { - dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + s[1] + 1) >> 1, scale)); - dst[1] = STATIC_CAST(uint8_t, C16TO8((s[2] + s[3] + 1) >> 1, scale)); - dst += 2; - s += 4; - } - if (dst_width & 1) { - dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + s[1] + 1) >> 1, scale)); - } -} - -void ScaleRowDown2Linear_16To8_Odd_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale) { - const uint16_t* s = src_ptr; - int x; - (void)src_stride; - assert(scale >= 256); - assert(scale <= 32768); - dst_width -= 1; - for (x = 0; x < dst_width - 1; x += 2) { - dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + s[1] + 1) >> 1, scale)); - dst[1] = STATIC_CAST(uint8_t, C16TO8((s[2] + s[3] + 1) >> 1, scale)); - dst += 2; - s += 4; - } - if (dst_width & 1) { - dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + s[1] + 1) >> 1, scale)); - dst += 1; - s += 2; - } - dst[0] = STATIC_CAST(uint8_t, C16TO8(s[0], scale)); -} void ScaleRowDown2Box_C(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -269,60 +168,6 @@ void ScaleRowDown2Box_16_C(const uint16_t* src_ptr, } } -void ScaleRowDown2Box_16To8_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale) { - const uint16_t* s = src_ptr; - const uint16_t* t = src_ptr + src_stride; - int x; - assert(scale >= 256); - assert(scale <= 32768); - for (x = 0; x < dst_width - 1; x += 2) { - dst[0] = STATIC_CAST(uint8_t, - C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); - dst[1] = STATIC_CAST(uint8_t, - C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale)); - dst += 2; - s += 4; - t += 4; - } - if (dst_width & 1) { - dst[0] = STATIC_CAST(uint8_t, - C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); - } -} - -void ScaleRowDown2Box_16To8_Odd_C(const uint16_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst, - int dst_width, - int scale) { - const uint16_t* s = src_ptr; - const uint16_t* t = src_ptr + src_stride; - int x; - assert(scale >= 256); - assert(scale <= 32768); - dst_width -= 1; - for (x = 0; x < dst_width - 1; x += 2) { - dst[0] = STATIC_CAST(uint8_t, - C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); - dst[1] = STATIC_CAST(uint8_t, - C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale)); - dst += 2; - s += 4; - t += 4; - } - if (dst_width & 1) { - dst[0] = STATIC_CAST(uint8_t, - C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); - dst += 1; - s += 2; - t += 2; - } - dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + t[0] + 1) >> 1, scale)); -} void ScaleRowDown4_C(const uint8_t* src_ptr, ptrdiff_t src_stride, diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 86e7f740c..1ee5ec0f6 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -3973,6 +3973,154 @@ TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) { } #endif // HAS_CONVERT16TO8ROW_AVX2 +#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) || defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) || defined(HAS_HALFWIDTHROW_16TO8_NEON) +TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Opt) { + const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63; + align_buffer_page_end(src_pixels_y, kPixels * 4); + align_buffer_page_end(dst_pixels_y_opt, kPixels); + align_buffer_page_end(dst_pixels_y_c, kPixels); + + MemRandomize(src_pixels_y, kPixels * 4); + for (int i = 0; i < kPixels * 2; ++i) { + reinterpret_cast(src_pixels_y)[i] &= 1023; + } + + memset(dst_pixels_y_opt, 0, kPixels); + memset(dst_pixels_y_c, 1, kPixels); + + HalfWidthRow_16To8_C(reinterpret_cast(src_pixels_y), kPixels, + dst_pixels_y_c, 16384, kPixels); + +#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) + int has_ssse3 = TestCpuFlag(kCpuHasSSSE3); +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_NEON) + int has_neon = TestCpuFlag(kCpuHasNEON); +#endif + + for (int i = 0; i < benchmark_iterations_; ++i) { +#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) + if (has_avx512) { + HalfWidthRow_16To8_AVX512BW( + reinterpret_cast(src_pixels_y), kPixels, + dst_pixels_y_opt, 16384, kPixels); + } else +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) + if (has_avx2) { + HalfWidthRow_16To8_AVX2(reinterpret_cast(src_pixels_y), + kPixels, dst_pixels_y_opt, 16384, kPixels); + } else +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) + if (has_ssse3) { + HalfWidthRow_16To8_SSSE3(reinterpret_cast(src_pixels_y), + kPixels, dst_pixels_y_opt, 16384, kPixels); + } else +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_NEON) + if (has_neon) { + HalfWidthRow_16To8_NEON(reinterpret_cast(src_pixels_y), + kPixels, dst_pixels_y_opt, 16384, kPixels); + } else +#endif + { + HalfWidthRow_16To8_C(reinterpret_cast(src_pixels_y), + kPixels, dst_pixels_y_opt, 16384, kPixels); + } + } + + for (int i = 0; i < kPixels; ++i) { + ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]); + } + + free_aligned_buffer_page_end(src_pixels_y); + free_aligned_buffer_page_end(dst_pixels_y_opt); + free_aligned_buffer_page_end(dst_pixels_y_c); +} + +TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) { + const int kMaxPixels = 256; + align_buffer_page_end(src_pixels_y, kMaxPixels * 4); + align_buffer_page_end(dst_pixels_y_opt, kMaxPixels); + align_buffer_page_end(dst_pixels_y_c, kMaxPixels); + + MemRandomize(src_pixels_y, kMaxPixels * 4); + for (int i = 0; i < kMaxPixels * 2; ++i) { + reinterpret_cast(src_pixels_y)[i] &= 1023; + } + +#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) + int has_ssse3 = TestCpuFlag(kCpuHasSSSE3); +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_NEON) + int has_neon = TestCpuFlag(kCpuHasNEON); +#endif + + for (int width = 1; width <= 129; ++width) { + memset(dst_pixels_y_opt, 0, kMaxPixels); + memset(dst_pixels_y_c, 1, kMaxPixels); + + HalfWidthRow_16To8_C(reinterpret_cast(src_pixels_y), + kMaxPixels, dst_pixels_y_c, 16384, width); + +#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) + if (has_avx512) { + HalfWidthRow_16To8_Any_AVX512BW( + reinterpret_cast(src_pixels_y), kMaxPixels, + dst_pixels_y_opt, 16384, width); + } else +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) + if (has_avx2) { + HalfWidthRow_16To8_Any_AVX2( + reinterpret_cast(src_pixels_y), kMaxPixels, + dst_pixels_y_opt, 16384, width); + } else +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) + if (has_ssse3) { + HalfWidthRow_16To8_Any_SSSE3( + reinterpret_cast(src_pixels_y), kMaxPixels, + dst_pixels_y_opt, 16384, width); + } else +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_NEON) + if (has_neon) { + HalfWidthRow_16To8_Any_NEON( + reinterpret_cast(src_pixels_y), kMaxPixels, + dst_pixels_y_opt, 16384, width); + } else +#endif + { + HalfWidthRow_16To8_C(reinterpret_cast(src_pixels_y), + kMaxPixels, dst_pixels_y_opt, 16384, width); + } + + for (int i = 0; i < width; ++i) { + ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]); + } + } + + free_aligned_buffer_page_end(src_pixels_y); + free_aligned_buffer_page_end(dst_pixels_y_opt); + free_aligned_buffer_page_end(dst_pixels_y_c); +} +#endif + #ifdef HAS_UYVYTOYROW_NEON TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) { // NEON does multiple of 16, so round count up