From 67f79132074deb67c3b104fddaadad396d4c8d39 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Fri, 16 Oct 2026 16:51:20 -0700 Subject: [PATCH] Multi-platform ScaleUVAddCols3 for ScaleUVBox at 3x R=rrwinterton@gmail.com Change-Id: Ib9e14f3efcb097842c3d0e9fc9b473b6e2f81871 --- include/libyuv/scale_row.h | 37 +++++ source/scale_any.cc | 19 +++ source/scale_common.cc | 16 ++ source/scale_gcc.cc | 310 +++++++++++++++++++++++++++++++++++++ source/scale_neon.cc | 37 +++++ source/scale_neon64.cc | 32 ++++ source/scale_rvv.cc | 36 +++++ source/scale_sme.cc | 65 ++++++++ source/scale_uv.cc | 50 +++++- unit_test/scale_uv_test.cc | 4 + 10 files changed, 605 insertions(+), 1 deletion(-) diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 5a4681163..9177ed234 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -68,6 +68,7 @@ extern "C" { #if !defined(LIBYUV_DISABLE_X86) && \ (defined(__x86_64__) || defined(__i386__)) && \ (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) +#define HAS_SCALEUVADDCOLS3_AVX2 #define HAS_SCALEUVROWDOWN2BOX_AVX2 #define HAS_SCALEROWUP2_LINEAR_AVX2 #define HAS_SCALEROWUP2_BILINEAR_AVX2 @@ -98,6 +99,7 @@ extern "C" { !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_SCALEADDROW_AVX512BW #define HAS_SCALEADDCOLS_AVX512BW +#define HAS_SCALEUVADDCOLS3_AVX512VBMI #define HAS_SCALEROWDOWN2_AVX512BW #endif @@ -114,6 +116,7 @@ extern "C" { #define HAS_SCALEROWDOWN34_NEON #define HAS_SCALEROWDOWN38_NEON #define HAS_SCALEROWDOWN4_NEON +#define HAS_SCALEUVADDCOLS3_NEON #define HAS_SCALEUVROWDOWN2_NEON #define HAS_SCALEUVROWDOWN2LINEAR_NEON #define HAS_SCALEUVROWDOWN2BOX_NEON @@ -141,6 +144,7 @@ extern "C" { #define HAS_SCALEARGBROWDOWN2_SME #define HAS_SCALEROWDOWN2_16_SME #define HAS_SCALEROWDOWN2_SME +#define HAS_SCALEUVADDCOLS3_SME #define HAS_SCALEUVROWDOWN2BOX_SME #define HAS_SCALEUVROWDOWN2LINEAR_SME #define HAS_SCALEUVROWDOWN2_SME @@ -192,6 +196,7 @@ extern "C" { #define HAS_SCALEROWDOWN2LINEAR_RVV #define HAS_SCALEROWDOWN4_RVV #define HAS_SCALEROWDOWN4BOX_RVV +#define HAS_SCALEUVADDCOLS3_RVV #define HAS_SCALEUVROWDOWN2_RVV #define HAS_SCALEUVROWDOWN2BOX_RVV #define HAS_SCALEUVROWDOWN2LINEAR_RVV @@ -569,6 +574,10 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv, int dst_width, int x32, int dx); +void ScaleUVAddCols3_C(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); // Specialized scalers for x86. void ScaleRowDown2_SSSE3(const uint8_t* src_ptr, @@ -1321,6 +1330,34 @@ void ScaleUVRowUp2_Bilinear_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, ptrdiff_t dst_stride, int dst_width); +void ScaleUVAddCols3_AVX2(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_Any_AVX2(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_AVX512VBMI(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_NEON(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_Any_NEON(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_SME(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_RVV(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); // ScaleRowDown2Box also used by planar functions // NEON/SME downscalers with interpolation. diff --git a/source/scale_any.cc b/source/scale_any.cc index 8d17e62c7..68e20fea4 100644 --- a/source/scale_any.cc +++ b/source/scale_any.cc @@ -1131,6 +1131,25 @@ SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_NEON, #undef SBU2BLANY +// Scale UV columns by 3 box filter. +#define SUVAC3ANY(NAMEANY, SIMD, C, MASK) \ + void NAMEANY(int dst_width, int scaleval, const uint16_t* src_uv, \ + uint8_t* dst_uv) { \ + int r = dst_width & MASK; \ + int n = dst_width & ~MASK; \ + if (n > 0) { \ + SIMD(n, scaleval, src_uv, dst_uv); \ + } \ + C(r, scaleval, src_uv + (ptrdiff_t)n * 6, dst_uv + (ptrdiff_t)n * 2); \ + } +#ifdef HAS_SCALEUVADDCOLS3_AVX2 +SUVAC3ANY(ScaleUVAddCols3_Any_AVX2, ScaleUVAddCols3_AVX2, ScaleUVAddCols3_C, 7) +#endif +#ifdef HAS_SCALEUVADDCOLS3_NEON +SUVAC3ANY(ScaleUVAddCols3_Any_NEON, ScaleUVAddCols3_NEON, ScaleUVAddCols3_C, 7) +#endif +#undef SUVAC3ANY + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale_common.cc b/source/scale_common.cc index 1dc311bd7..3643db85c 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1366,6 +1366,22 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv, } #undef BLENDER1 +// Scale UV columns by 3 box filter. +void ScaleUVAddCols3_C(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + int i; + for (i = 0; i < dst_width; ++i) { + uint32_t u = src_uv[0] + src_uv[2] + src_uv[4]; + uint32_t v = src_uv[1] + src_uv[3] + src_uv[5]; + src_uv += 6; + dst_uv[0] = (uint8_t)((u * scaleval) >> 16); + dst_uv[1] = (uint8_t)((v * scaleval) >> 16); + dst_uv += 2; + } +} + // Scale plane vertically with bilinear interpolation. void ScalePlaneVertical(int src_height, int dst_width, diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index 835665ee0..c92a482e4 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -3708,6 +3708,316 @@ void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr, } #endif +#ifdef HAS_SCALEUVADDCOLS3_AVX2 +static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5}; +static const ulvec32 kPermUV3_1 = {3, 4, 5, 6, 7, 0, 1, 2}; +static const ulvec32 kPermUV3_2 = {6, 7, 0, 1, 2, 3, 4, 5}; + +// Scale UV columns by 3 box filter. +// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per block. +void ScaleUVAddCols3_AVX2(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { +#if defined(__x86_64__) + asm volatile( + "vmovd %k[scaleval],%%xmm7 \n" + "vpbroadcastw %%xmm7,%%ymm7 \n" + "vmovdqa %[kPerm0],%%ymm4 \n" + "vmovdqa %[kPerm1],%%ymm5 \n" + "vmovdqa %[kPerm2],%%ymm6 \n" + "sub $0x10,%[dst_width] \n" + "jl 2f \n" + + // Process 16 output UV pixels (192 input bytes -> 32 output bytes). + LABELALIGN + "1: \n" + "vmovdqu (%[src_uv]),%%ymm0 \n" + "vmovdqu 0x20(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm0,%%ymm4,%%ymm0 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n" + "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n" + "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu 0x40(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n" + "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermd %%ymm2,%%ymm5,%%ymm2 \n" + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddw %%ymm0,%%ymm3,%%ymm8 \n" + "vpmulhuw %%ymm7,%%ymm8,%%ymm8 \n" + + "vmovdqu 0x60(%[src_uv]),%%ymm0 \n" + "vmovdqu 0x80(%[src_uv]),%%ymm1 \n" + "lea 0xc0(%[src_uv]),%[src_uv] \n" + "vpermd %%ymm0,%%ymm4,%%ymm0 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n" + "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n" + "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu -0x20(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n" + "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermd %%ymm2,%%ymm5,%%ymm2 \n" + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddw %%ymm0,%%ymm3,%%ymm3 \n" + "vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n" + + "vpackuswb %%ymm3,%%ymm8,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst_uv]) \n" + "lea 0x20(%[dst_uv]),%[dst_uv] \n" + "sub $0x10,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[dst_width] \n" + "jle 99f \n" + + // 8-pixel remainder. + "vmovdqu (%[src_uv]),%%ymm0 \n" + "vmovdqu 0x20(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm0,%%ymm4,%%ymm0 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n" + "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n" + "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu 0x40(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n" + "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermd %%ymm2,%%ymm5,%%ymm2 \n" + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddw %%ymm0,%%ymm3,%%ymm3 \n" + "vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n" + "vextracti128 $1,%%ymm3,%%xmm2 \n" + "vpackuswb %%xmm2,%%xmm3,%%xmm3 \n" + "vmovdqu %%xmm3,(%[dst_uv]) \n" + + "99: \n" + "vzeroupper \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "r"(scaleval), // %[scaleval] + [kPerm0] "m"(kPermUV3_0), // %[kPerm0] + [kPerm1] "m"(kPermUV3_1), // %[kPerm1] + [kPerm2] "m"(kPermUV3_2) // %[kPerm2] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8"); +#else // defined(__x86_64__) + asm volatile( + "vmovd %[scaleval],%%xmm7 \n" + "vpbroadcastw %%xmm7,%%ymm7 \n" + "vmovdqa %[kPerm0],%%ymm4 \n" + "vmovdqa %[kPerm1],%%ymm5 \n" + "vmovdqa %[kPerm2],%%ymm6 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src_uv]),%%ymm0 \n" + "vmovdqu 0x20(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm0,%%ymm4,%%ymm0 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n" + "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n" + "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu 0x40(%[src_uv]),%%ymm1 \n" + "lea 0x60(%[src_uv]),%[src_uv] \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n" + "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermd %%ymm2,%%ymm5,%%ymm2 \n" + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddw %%ymm0,%%ymm3,%%ymm3 \n" + "vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n" + "vextracti128 $1,%%ymm3,%%xmm2 \n" + "vpackuswb %%xmm2,%%xmm3,%%xmm3 \n" + "vmovdqu %%xmm3,(%[dst_uv]) \n" + "lea 0x10(%[dst_uv]),%[dst_uv] \n" + "sub $0x8,%[dst_width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "rm"(scaleval), // %[scaleval] + [kPerm0] "m"(kPermUV3_0), // %[kPerm0] + [kPerm1] "m"(kPermUV3_1), // %[kPerm1] + [kPerm2] "m"(kPermUV3_2) // %[kPerm2] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +#endif // defined(__x86_64__) +} +#endif // HAS_SCALEUVADDCOLS3_AVX2 + +#ifdef HAS_SCALEUVADDCOLS3_AVX512VBMI +static const uint8_t kPerm512_Col0[16] = {0, 3, 6, 9, 12, 15, 18, 21, + 24, 27, 30, 1, 4, 7, 10, 13}; +static const uint8_t kPerm512_Col1[16] = {1, 4, 7, 10, 13, 16, 19, 22, + 25, 28, 31, 2, 5, 8, 11, 14}; +static const uint8_t kPerm512_Col2[16] = {2, 5, 8, 11, 14, 17, 20, 23, + 26, 29, 0, 3, 6, 9, 12, 15}; +static const uint8_t kPackEvenBytes[64] = { + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, + 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, + 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, + 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, + 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126}; + +// Scale UV columns by 3 box filter using AVX512VBMI. +// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop +// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels). +void ScaleUVAddCols3_AVX512VBMI(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + intptr_t temp, mask; + asm volatile( + "vpbroadcastw %k[scaleval],%%zmm20 \n" + "vpmovzxbd %[kCol0],%%zmm16 \n" + "vpmovzxbd %[kCol1],%%zmm17 \n" + "vpmovzxbd %[kCol2],%%zmm18 \n" + "vmovdqu8 %[kPack],%%zmm19 \n" + "mov $0xf800,%k[temp] \n" + "kmovw %k[temp],%%k5 \n" + "mov $0xfc00,%k[temp] \n" + "kmovw %k[temp],%%k6 \n" + + "sub $0x20,%[dst_width] \n" + "jl 2f \n" + + // Main loop: 32 output UV pixels (64 output bytes) per iteration. + LABELALIGN + "1: \n" + "vmovdqu32 (%[src_uv]),%%zmm0 \n" + "vmovdqu32 0x40(%[src_uv]),%%zmm1 \n" + "vmovdqu32 0x80(%[src_uv]),%%zmm2 \n" + "vmovdqa32 %%zmm16,%%zmm3 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm3 \n" + "vmovdqa32 %%zmm17,%%zmm4 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm4 \n" + "vmovdqa32 %%zmm18,%%zmm5 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm5 \n" + "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n" + "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n" + "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n" + "vpaddw %%zmm4,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm5,%%zmm3,%%zmm3 \n" + "vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n" + + "vmovdqu32 0xc0(%[src_uv]),%%zmm6 \n" + "vmovdqu32 0x100(%[src_uv]),%%zmm1 \n" + "vmovdqu32 0x140(%[src_uv]),%%zmm2 \n" + "lea 0x180(%[src_uv]),%[src_uv] \n" + "vmovdqa32 %%zmm16,%%zmm3 \n" + "vpermi2d %%zmm1,%%zmm6,%%zmm3 \n" + "vmovdqa32 %%zmm17,%%zmm4 \n" + "vpermi2d %%zmm1,%%zmm6,%%zmm4 \n" + "vmovdqa32 %%zmm18,%%zmm5 \n" + "vpermi2d %%zmm1,%%zmm6,%%zmm5 \n" + "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n" + "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n" + "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n" + "vpaddw %%zmm4,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm5,%%zmm3,%%zmm3 \n" + "vpmulhuw %%zmm20,%%zmm3,%%zmm3 \n" + + "vmovdqa64 %%zmm19,%%zmm2 \n" + "vpermi2b %%zmm3,%%zmm0,%%zmm2 \n" + "vmovdqu8 %%zmm2,(%[dst_uv]) \n" + "lea 0x40(%[dst_uv]),%[dst_uv] \n" + "sub $0x20,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[dst_width] \n" + "jle 99f \n" + "cmp $0x10,%[dst_width] \n" + "jl 3f \n" + + // 16-pixel half-block (32 output bytes). + "vmovdqu32 (%[src_uv]),%%zmm0 \n" + "vmovdqu32 0x40(%[src_uv]),%%zmm1 \n" + "vmovdqu32 0x80(%[src_uv]),%%zmm2 \n" + "lea 0xc0(%[src_uv]),%[src_uv] \n" + "vmovdqa32 %%zmm16,%%zmm3 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm3 \n" + "vmovdqa32 %%zmm17,%%zmm4 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm4 \n" + "vmovdqa32 %%zmm18,%%zmm5 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm5 \n" + "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n" + "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n" + "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n" + "vpaddw %%zmm4,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm5,%%zmm3,%%zmm3 \n" + "vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n" + "vpmovwb %%zmm0,(%[dst_uv]) \n" + "lea 0x20(%[dst_uv]),%[dst_uv] \n" + "sub $0x10,%[dst_width] \n" + "jle 99f \n" + + // Tail: 1..15 output UV pixels (3..45 input dwords, 2..30 output bytes). + "3: \n" + "lea (%q[dst_width],%q[dst_width],2),%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovw %k[mask],%%k1 \n" + "shr $16,%q[mask] \n" + "kmovw %k[mask],%%k2 \n" + "shr $16,%q[mask] \n" + "kmovw %k[mask],%%k3 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask] \n" + "kmovw %k[mask],%%k4 \n" + + "vmovdqu32 (%[src_uv]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src_uv]),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu32 0x80(%[src_uv]),%%zmm2%{%%k3%}%{z%} \n" + "vmovdqa32 %%zmm16,%%zmm3 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm3 \n" + "vmovdqa32 %%zmm17,%%zmm4 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm4 \n" + "vmovdqa32 %%zmm18,%%zmm5 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm5 \n" + "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n" + "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n" + "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n" + "vpaddw %%zmm4,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm5,%%zmm3,%%zmm3 \n" + "vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n" + "vpmovwb %%zmm0,%%ymm0 \n" + "vmovdqu16 %%ymm0,(%[dst_uv])%{%%k4%} \n" + + "99: \n" + "vzeroupper \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width), // %[dst_width] + [temp] "=&r"(temp), // %[temp] + [mask] "=&r"(mask) // %[mask] + : [scaleval] "r"(scaleval), // %[scaleval] + [kCol0] "m"(kPerm512_Col0), // %[kCol0] + [kCol1] "m"(kPerm512_Col1), // %[kCol1] + [kCol2] "m"(kPerm512_Col2), // %[kCol2] + [kPack] "m"(kPackEvenBytes) // %[kPack] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "k1", "k2", "k3", "k4", + "k5", "k6"); +} +#endif // HAS_SCALEUVADDCOLS3_AVX512VBMI + #endif // defined(__x86_64__) || defined(__i386__) #ifdef __cplusplus diff --git a/source/scale_neon.cc b/source/scale_neon.cc index 4f9e03b8a..2d5be0af3 100644 --- a/source/scale_neon.cc +++ b/source/scale_neon.cc @@ -1448,6 +1448,43 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr, : "memory", "cc", "d0"); } +// Scale UV columns by 3 box filter. +// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop. +void ScaleUVAddCols3_NEON(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + asm volatile( + "vdup.16 d16, %[scaleval] \n" + "1: \n" + "vld3.32 {d0, d2, d4}, [%[src_uv]]! \n" + "vld3.32 {d1, d3, d5}, [%[src_uv]]! \n" + "vadd.u16 q0, q0, q1 \n" + "vadd.u16 q0, q0, q2 \n" + "vld3.32 {d2, d4, d6}, [%[src_uv]]! \n" + "vld3.32 {d3, d5, d7}, [%[src_uv]]! \n" + "vadd.u16 q1, q1, q2 \n" + "vadd.u16 q1, q1, q3 \n" + "vmull.u16 q2, d0, d16 \n" + "vmull.u16 q3, d1, d16 \n" + "vmull.u16 q9, d2, d16 \n" + "vmull.u16 q10, d3, d16 \n" + "vshrn.i32 d0, q2, #16 \n" + "vshrn.i32 d1, q3, #16 \n" + "vshrn.i32 d2, q9, #16 \n" + "vshrn.i32 d3, q10, #16 \n" + "vmovn.i16 d0, q0 \n" + "vmovn.i16 d1, q1 \n" + "subs %[dst_width], %[dst_width], #8 \n" + "vst1.8 {q0}, [%[dst_uv]]! \n" + "bgt 1b \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "r"(scaleval) // %[scaleval] + : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10"); +} + #endif // defined(__ARM_NEON__) && !defined(__aarch64__) #ifdef __cplusplus diff --git a/source/scale_neon64.cc b/source/scale_neon64.cc index ae4768f22..3514bb079 100644 --- a/source/scale_neon64.cc +++ b/source/scale_neon64.cc @@ -1551,6 +1551,38 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr, : "memory", "cc", "v0", "v1", "v2", "v3"); } +// Scale UV columns by 3 box filter. +// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop. +void ScaleUVAddCols3_NEON(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + asm volatile( + "dup v6.8h, %w[scaleval] \n" + "1: \n" + "ld3 {v0.4s, v1.4s, v2.4s}, [%[src_uv]], #48 \n" + "ld3 {v3.4s, v4.4s, v5.4s}, [%[src_uv]], #48 \n" + "subs %w[dst_width], %w[dst_width], #8 \n" + "add v0.8h, v0.8h, v1.8h \n" + "add v3.8h, v3.8h, v4.8h \n" + "add v0.8h, v0.8h, v2.8h \n" + "add v3.8h, v3.8h, v5.8h \n" + "umull v1.4s, v0.4h, v6.4h \n" + "umull2 v2.4s, v0.8h, v6.8h \n" + "umull v4.4s, v3.4h, v6.4h \n" + "umull2 v5.4s, v3.8h, v6.8h \n" + "uzp2 v0.8h, v1.8h, v2.8h \n" + "uzp2 v3.8h, v4.8h, v5.8h \n" + "uzp1 v0.16b, v0.16b, v3.16b \n" + "st1 {v0.16b}, [%[dst_uv]], #16 \n" + "b.gt 1b \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "r"(scaleval) // %[scaleval] + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/scale_rvv.cc b/source/scale_rvv.cc index 99c945a31..238a9a953 100644 --- a/source/scale_rvv.cc +++ b/source/scale_rvv.cc @@ -1304,6 +1304,42 @@ void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr, } #endif +#ifdef HAS_SCALEUVADDCOLS3_RVV +void ScaleUVAddCols3_RVV(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + size_t vl, vl2; + size_t w = (size_t)dst_width; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e32, m2, ta, ma \n" + "vlseg3e32.v v8, (%[src_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl2], %[vl], 2 \n" + "add %[src_uv], %[src_uv], %[vl2] \n" + "slli %[vl2], %[vl2], 1 \n" + "add %[src_uv], %[src_uv], %[vl2] \n" + "slli %[vl], %[vl], 1 \n" + "vsetvli zero, %[vl], e16, m2, ta, ma \n" + "vadd.vv v8, v8, v10 \n" + "vadd.vv v8, v8, v12 \n" + "vmulhu.vx v8, v8, %[scaleval] \n" + "vsetvli zero, %[vl], e8, m1, ta, ma \n" + "vnsrl.wi v4, v8, 0 \n" + "vse8.v v4, (%[dst_uv]) \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(w), // %[w] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [vl] "=&r"(vl), // %[vl] + [vl2] "=&r"(vl2) // %[vl2] + : [scaleval] "r"(scaleval) // %[scaleval] + : RVV_VL_CLOBBER "memory", "v4", "v8", "v9", "v10", "v11", "v12", "v13"); +} +#endif + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale_sme.cc b/source/scale_sme.cc index 70e3e2d72..1dc5c17ba 100644 --- a/source/scale_sme.cc +++ b/source/scale_sme.cc @@ -554,6 +554,71 @@ __arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb, "p1"); } +// Scale UV columns by 3 box filter using Streaming SVE. +__arm_locally_streaming void ScaleUVAddCols3_SME(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + // Streaming-SVE only, no use of ZA tile. + int vl; + int vl_w; + asm volatile( + "cnth %x[vl] \n" + "cntw %x[vl_w] \n" + "dup z6.h, %w[scaleval] \n" + "ptrue p3.h \n" + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.s \n" + "ptrue p1.h \n" + "1: \n" + "ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n" + "incb %[src_uv], all, mul #3 \n" + "ld3w {z3.s, z4.s, z5.s}, p0/z, [%[src_uv]] \n" + "incb %[src_uv], all, mul #3 \n" + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "add z0.h, z0.h, z1.h \n" + "add z3.h, z3.h, z4.h \n" + "add z0.h, z0.h, z2.h \n" + "add z3.h, z3.h, z5.h \n" + "umulh z0.h, p3/m, z0.h, z6.h \n" + "umulh z3.h, p3/m, z3.h, z6.h \n" + "uzp1 z0.b, z0.b, z3.b \n" + "st1h {z0.h}, p1, [%[dst_uv]] \n" + "incb %[dst_uv] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[dst_width], %w[dst_width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[dst_width] \n" + "whilelt p2.s, %w[vl_w], %w[dst_width] \n" + "whilelt p1.h, wzr, %w[dst_width] \n" + "ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n" + "incb %[src_uv], all, mul #3 \n" + "ld3w {z3.s, z4.s, z5.s}, p2/z, [%[src_uv]] \n" + "add z0.h, z0.h, z1.h \n" + "add z3.h, z3.h, z4.h \n" + "add z0.h, z0.h, z2.h \n" + "add z3.h, z3.h, z5.h \n" + "umulh z0.h, p3/m, z0.h, z6.h \n" + "umulh z3.h, p3/m, z3.h, z6.h \n" + "uzp1 z0.b, z0.b, z3.b \n" + "st1h {z0.h}, p1, [%[dst_uv]] \n" + + "99: \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width), // %[dst_width] + [vl] "=&r"(vl), // %[vl] + [vl_w] "=&r"(vl_w) // %[vl_w] + : [scaleval] "r"(scaleval) // %[scaleval] + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "p0", "p1", + "p2", "p3"); +} + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 251bea8cd..28f506fd0 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -332,10 +332,46 @@ static int ScaleUVBox(int src_width, void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx, const uint16_t* src_uv, uint8_t* dst_uv) = (dx & 0xffff) ? ScaleUVAddCols2_C : ScaleUVAddCols1_C; + void (*ScaleUVAddCols3)(int dst_width, int scaleval, + const uint16_t* src_uv, uint8_t* dst_uv) = NULL; // Summing a row is position wise, so an interleaved UV row can be fed to // the plane accumulator as src_width * 2 samples. void (*ScaleAddRow)(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) = ScaleAddRow_C; + if (dx == 0x30000) { + ScaleUVAddCols3 = ScaleUVAddCols3_C; +#if defined(HAS_SCALEUVADDCOLS3_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleUVAddCols3 = ScaleUVAddCols3_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVAddCols3 = ScaleUVAddCols3_AVX2; + } + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + ScaleUVAddCols3 = ScaleUVAddCols3_AVX512VBMI; + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleUVAddCols3 = ScaleUVAddCols3_Any_NEON; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVAddCols3 = ScaleUVAddCols3_NEON; + } + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_SME) + if (TestCpuFlag(kCpuHasSME)) { + ScaleUVAddCols3 = ScaleUVAddCols3_SME; + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ScaleUVAddCols3 = ScaleUVAddCols3_RVV; + } +#endif + } #if defined(HAS_SCALEADDROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { ScaleAddRow = ScaleAddRow_Any_SSE2; @@ -352,6 +388,11 @@ static int ScaleUVBox(int src_width, } } #endif +#if defined(HAS_SCALEADDROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ScaleAddRow = ScaleAddRow_AVX512BW; + } +#endif #if defined(HAS_SCALEADDROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ScaleAddRow = ScaleAddRow_Any_NEON; @@ -389,7 +430,14 @@ static int ScaleUVBox(int src_width, ScaleAddRow(src, (uint16_t*)(row16), row_samples); src += src_stride; } - ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16), dst_uv); + if (ScaleUVAddCols3 && boxheight <= 85) { + ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight), + (const uint16_t*)row16 + (ptrdiff_t)(x >> 16) * 2, + dst_uv); + } else { + ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16), + dst_uv); + } dst_uv += dst_stride; } } diff --git a/unit_test/scale_uv_test.cc b/unit_test/scale_uv_test.cc index 65a7c662f..5a8d160a9 100644 --- a/unit_test/scale_uv_test.cc +++ b/unit_test/scale_uv_test.cc @@ -331,6 +331,10 @@ TEST_BOXPLANE(By3_75, 960, 540, 256, 144, 0) /* 1080p to 512x288 */ TEST_BOXPLANE(By6, 640, 360, 106, 60, 0) /* 720p to 213x120 */ TEST_BOXPLANE(By8, 640, 360, 80, 45, 0) /* 720p to 160x90 */ TEST_BOXPLANE(OddTail, 33, 17, 7, 5, 0) /* not a multiple of 8 wide */ +TEST_BOXPLANE(By3OddTail7, 21, 15, 7, 5, 0) /* 3x, dst_width = 7 */ +TEST_BOXPLANE(By3OddTail23, 69, 15, 23, 5, 0) /* 3x, dst_width = 23 */ +TEST_BOXPLANE(By3OddTail47, 141, 15, 47, 5, 0) /* 3x, dst_width = 47 */ +TEST_BOXPLANE(By3OddTail79, 237, 15, 79, 5, 0) /* 3x, dst_width = 79 */ TEST_BOXPLANE(Tiny, 80, 60, 3, 2, 0) /* dst below one SIMD group */ #undef TEST_BOXPLANE