From 378bc91324587eedaadde88c81a5d05fada8a581 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Fri, 16 Oct 2026 16:51:17 -0700 Subject: [PATCH] [libyuv] Multi-platform ScaleUVAddCols3 for ScaleUVBox at 3x Based on Philipp Hancke's AVX2 column kernel for ScaleUVBox at 3x (https://crrev.com/c/8427347). Add ScaleUVAddCols3 for exact 3x horizontal downscaling (dx == 0x30000 and boxheight <= 85) in ScaleUVBox, plus ScaleAddRow_AVX512BW and ScaleAddRow_WASMSIMD dispatch, and port 3x/box scaling kernels in scale_win.cc to 32/64-bit C++ intrinsics matching scale_gcc.cc: - ScaleUVAddCols3_C (portable C) - ScaleUVAddCols3_AVX2 and ScaleUVAddCols3_Any_AVX2 (scale_gcc.cc and scale_win.cc) - ScaleUVAddCols3_AVX512VBMI (scale_gcc.cc and scale_win.cc) - ScaleAddRow_SSE2, ScaleAddRow_AVX2, ScaleAddRow_AVX512BW, ScaleAddCols1_AVX512BW, ScaleAddCols2_AVX512BW, ScaleRowDown38_SSSE3, ScaleRowDown38_2_Box_SSSE3, and ScaleRowDown38_3_Box_SSSE3 in scale_win.cc for 32/64-bit Visual C and LIBYUV_ENABLE_ROWWIN - ScaleUVAddCols3_NEON (32-bit ARM and 64-bit AArch64) and ScaleUVAddCols3_Any_NEON - ScaleUVAddCols3_SME (ARMv9 Streaming SVE with whilelt predicate tail) - ScaleUVAddCols3_RVV (RISC-V Vector vlseg3e32.v inline asm) - ScaleUVAddCols3_LSX and ScaleUVAddCols3_Any_LSX (LoongArch LSX) - ScaleAddRow_WASMSIMD, ScaleAddRow_Any_WASMSIMD, ScaleUVAddCols3_WASMSIMD, and ScaleUVAddCols3_Any_WASMSIMD - ScaleUVAddCols3_HVX (Hexagon HVX vrdelta + vmpy_VuhRuh) Co-authored-by: Philipp Hancke Bug: libyuv:564129034 R=rrwinterton@gmail.com Change-Id: Ib9e14f3efcb097842c3d0e9fc9b473b6e2f81871 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8596436 Reviewed-by: richard winterton --- include/libyuv/scale_row.h | 98 +++- source/scale.cc | 8 + source/scale_any.cc | 34 ++ source/scale_common.cc | 16 + source/scale_gcc.cc | 310 +++++++++++ source/scale_hvx.cc | 95 ++++ source/scale_lsx.cc | 61 +++ source/scale_neon.cc | 37 ++ source/scale_neon64.cc | 32 ++ source/scale_rvv.cc | 36 ++ source/scale_sme.cc | 65 +++ source/scale_uv.cc | 79 ++- source/scale_wasm.cc | 68 +++ source/scale_win.cc | 1019 +++++++++++++++++++++++++++--------- unit_test/scale_uv_test.cc | 4 + 15 files changed, 1715 insertions(+), 247 deletions(-) diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index a67f93d84..49ed592f3 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -28,7 +28,6 @@ extern "C" { !defined(LIBYUV_ENABLE_ROWWIN))) #define HAS_FIXEDDIV1_X86 #define HAS_FIXEDDIV_X86 -#define HAS_SCALEADDROW_SSE2 #define HAS_SCALEARGBCOLS_SSE2 #define HAS_SCALEARGBCOLSUP2_SSE2 #define HAS_SCALEARGBFILTERCOLS_SSSE3 @@ -38,7 +37,6 @@ extern "C" { #define HAS_SCALEFILTERCOLS_SSSE3 #define HAS_SCALEROWDOWN2_SSSE3 #define HAS_SCALEROWDOWN34_SSSE3 -#define HAS_SCALEROWDOWN38_SSSE3 #define HAS_SCALEROWDOWN4_SSSE3 #endif @@ -47,6 +45,8 @@ extern "C" { #if !defined(LIBYUV_DISABLE_X86) && \ (defined(__x86_64__) || defined(__i386__)) && \ !defined(LIBYUV_ENABLE_ROWWIN) +#define HAS_SCALEADDROW_SSE2 +#define HAS_SCALEROWDOWN38_SSSE3 #define HAS_SCALEUVROWDOWN2BOX_SSSE3 #define HAS_SCALEROWUP2_LINEAR_SSE2 #define HAS_SCALEROWUP2_LINEAR_SSSE3 @@ -65,9 +65,12 @@ extern "C" { // The following are available for gcc/clang x86 platforms, but // require clang 3.4 or gcc 4.7. // TODO(fbarchard): Port to Visual C -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(__i386__)) && \ - (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) +#define HAS_SCALEADDROW_AVX2 +#define HAS_SCALEUVADDCOLS3_AVX2 #define HAS_SCALEUVROWDOWN2BOX_AVX2 #define HAS_SCALEROWUP2_LINEAR_AVX2 #define HAS_SCALEROWUP2_BILINEAR_AVX2 @@ -87,7 +90,6 @@ extern "C" { #if !defined(LIBYUV_DISABLE_X86) && \ (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \ defined(GCC_HAS_AVX2)) -#define HAS_SCALEADDROW_AVX2 #define HAS_SCALEROWDOWN2_AVX2 #define HAS_SCALEROWDOWN4_AVX2 #endif @@ -98,9 +100,28 @@ extern "C" { !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_SCALEADDROW_AVX512BW #define HAS_SCALEADDCOLS_AVX512BW +#define HAS_SCALEUVADDCOLS3_AVX512VBMI #define HAS_SCALEROWDOWN2_AVX512BW #endif +// The following are available for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) +#define HAS_SCALEADDROW_SSE2 +#define HAS_SCALEROWDOWN38_SSSE3 +#define HAS_SCALEADDROW_AVX2 +#define HAS_SCALEUVADDCOLS3_AVX2 +#if defined(__x86_64__) || defined(_M_X64) +#define HAS_SCALEADDROW_AVX512BW +#define HAS_SCALEADDCOLS_AVX512BW +#define HAS_SCALEUVADDCOLS3_AVX512VBMI +#endif +#endif + // The following are available on Neon platforms: #if !defined(LIBYUV_DISABLE_NEON) && \ (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) @@ -114,6 +135,7 @@ extern "C" { #define HAS_SCALEROWDOWN34_NEON #define HAS_SCALEROWDOWN38_NEON #define HAS_SCALEROWDOWN4_NEON +#define HAS_SCALEUVADDCOLS3_NEON #define HAS_SCALEUVROWDOWN2_NEON #define HAS_SCALEUVROWDOWN2LINEAR_NEON #define HAS_SCALEUVROWDOWN2BOX_NEON @@ -141,6 +163,7 @@ extern "C" { #define HAS_SCALEARGBROWDOWN2_SME #define HAS_SCALEROWDOWN2_16_SME #define HAS_SCALEROWDOWN2_SME +#define HAS_SCALEUVADDCOLS3_SME #define HAS_SCALEUVROWDOWN2BOX_SME #define HAS_SCALEUVROWDOWN2LINEAR_SME #define HAS_SCALEUVROWDOWN2_SME @@ -157,6 +180,7 @@ extern "C" { #define HAS_SCALEARGBCOLS_LSX #define HAS_SCALEARGBFILTERCOLS_LSX #define HAS_SCALEROWDOWN34_LSX +#define HAS_SCALEUVADDCOLS3_LSX #endif #if !defined(LIBYUV_DISABLE_RVV) && \ @@ -192,6 +216,7 @@ extern "C" { #define HAS_SCALEROWDOWN2LINEAR_RVV #define HAS_SCALEROWDOWN4_RVV #define HAS_SCALEROWDOWN4BOX_RVV +#define HAS_SCALEUVADDCOLS3_RVV #define HAS_SCALEUVROWDOWN2_RVV #define HAS_SCALEUVROWDOWN2BOX_RVV #define HAS_SCALEUVROWDOWN2LINEAR_RVV @@ -200,6 +225,7 @@ extern "C" { // The following are available for WebAssembly SIMD128: #if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#define HAS_SCALEADDROW_WASMSIMD #define HAS_SCALEARGBCOLS_WASMSIMD #define HAS_SCALEARGBCOLSUP2_WASMSIMD #define HAS_SCALEARGBFILTERCOLS_WASMSIMD @@ -210,6 +236,7 @@ extern "C" { #define HAS_SCALEROWDOWN34_WASMSIMD #define HAS_SCALEROWDOWN38_WASMSIMD #define HAS_SCALEROWDOWN4_WASMSIMD +#define HAS_SCALEUVADDCOLS3_WASMSIMD #endif // The following are available for Hexagon HVX: @@ -219,6 +246,7 @@ extern "C" { #define HAS_SCALEROWDOWN2_16_HVX #define HAS_SCALEROWDOWN2_HVX #define HAS_SCALEROWDOWN4_HVX +#define HAS_SCALEUVADDCOLS3_HVX #define HAS_SCALEUVROWDOWN2_HVX #define HAS_SCALEUVROWDOWN2BOX_HVX #define HAS_SCALEUVROWDOWN2LINEAR_HVX @@ -581,6 +609,10 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv, int dst_width, int x32, int dx); +void ScaleUVAddCols3_C(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); // Specialized scalers for x86. void ScaleRowDown2_SSSE3(const uint8_t* src_ptr, @@ -1333,6 +1365,54 @@ void ScaleUVRowUp2_Bilinear_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, ptrdiff_t dst_stride, int dst_width); +void ScaleUVAddCols3_AVX2(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_Any_AVX2(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_AVX512VBMI(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_NEON(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_Any_NEON(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_SME(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_RVV(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_LSX(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_Any_LSX(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_WASMSIMD(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_Any_WASMSIMD(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_HVX(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); // ScaleRowDown2Box also used by planar functions // NEON/SME downscalers with interpolation. @@ -1815,6 +1895,12 @@ void ScaleRowDown38_2_Box_Any_WASMSIMD(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); +void ScaleAddRow_WASMSIMD(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width); +void ScaleAddRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width); void ScaleFilterCols_WASMSIMD(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, diff --git a/source/scale.cc b/source/scale.cc index 633d13483..8650bffaa 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -1034,6 +1034,14 @@ static int ScalePlaneBox(int src_width, ScaleAddRow = ScaleAddRow_HVX; } #endif +#if defined(HAS_SCALEADDROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASM)) { + ScaleAddRow = ScaleAddRow_Any_WASMSIMD; + if (IS_ALIGNED(src_width, 16)) { + ScaleAddRow = ScaleAddRow_WASMSIMD; + } + } +#endif uint32_t uy = y; for (j = 0; j < dst_height; ++j) { diff --git a/source/scale_any.cc b/source/scale_any.cc index 3062ee474..86da894cd 100644 --- a/source/scale_any.cc +++ b/source/scale_any.cc @@ -622,6 +622,9 @@ SAROW(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, 1, 2, 15) #ifdef HAS_SCALEADDROW_LSX SAROW(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, 1, 2, 15) #endif +#ifdef HAS_SCALEADDROW_WASMSIMD +SAROW(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, 1, 2, 15) +#endif #undef SAANY #else @@ -648,6 +651,9 @@ SAANY(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, ScaleAddRow_C, 15) #ifdef HAS_SCALEADDROW_LSX SAANY(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, ScaleAddRow_C, 15) #endif +#ifdef HAS_SCALEADDROW_WASMSIMD +SAANY(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, ScaleAddRow_C, 15) +#endif #undef SAANY #endif // SASIMDONLY @@ -1131,6 +1137,34 @@ SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_NEON, #undef SBU2BLANY +// Scale UV columns by 3 box filter. +#define SUVAC3ANY(NAMEANY, SIMD, C, MASK) \ + void NAMEANY(int dst_width, int scaleval, const uint16_t* src_uv, \ + uint8_t* dst_uv) { \ + int r = dst_width & MASK; \ + int n = dst_width & ~MASK; \ + if (n > 0) { \ + SIMD(n, scaleval, src_uv, dst_uv); \ + } \ + C(r, scaleval, src_uv + (ptrdiff_t)n * 6, dst_uv + (ptrdiff_t)n * 2); \ + } +#ifdef HAS_SCALEUVADDCOLS3_AVX2 +SUVAC3ANY(ScaleUVAddCols3_Any_AVX2, ScaleUVAddCols3_AVX2, ScaleUVAddCols3_C, 7) +#endif +#ifdef HAS_SCALEUVADDCOLS3_NEON +SUVAC3ANY(ScaleUVAddCols3_Any_NEON, ScaleUVAddCols3_NEON, ScaleUVAddCols3_C, 7) +#endif +#ifdef HAS_SCALEUVADDCOLS3_LSX +SUVAC3ANY(ScaleUVAddCols3_Any_LSX, ScaleUVAddCols3_LSX, ScaleUVAddCols3_C, 7) +#endif +#ifdef HAS_SCALEUVADDCOLS3_WASMSIMD +SUVAC3ANY(ScaleUVAddCols3_Any_WASMSIMD, + ScaleUVAddCols3_WASMSIMD, + ScaleUVAddCols3_C, + 7) +#endif +#undef SUVAC3ANY + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale_common.cc b/source/scale_common.cc index b20da19b2..0f1e7c845 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1366,6 +1366,22 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv, } #undef BLENDER1 +// Scale UV columns by 3 box filter. +void ScaleUVAddCols3_C(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + int i; + for (i = 0; i < dst_width; ++i) { + uint32_t u = src_uv[0] + src_uv[2] + src_uv[4]; + uint32_t v = src_uv[1] + src_uv[3] + src_uv[5]; + src_uv += 6; + dst_uv[0] = (uint8_t)((u * scaleval) >> 16); + dst_uv[1] = (uint8_t)((v * scaleval) >> 16); + dst_uv += 2; + } +} + // Scale plane vertically with bilinear interpolation. void ScalePlaneVertical(int src_height, int dst_width, diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index 835665ee0..c92a482e4 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -3708,6 +3708,316 @@ void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr, } #endif +#ifdef HAS_SCALEUVADDCOLS3_AVX2 +static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5}; +static const ulvec32 kPermUV3_1 = {3, 4, 5, 6, 7, 0, 1, 2}; +static const ulvec32 kPermUV3_2 = {6, 7, 0, 1, 2, 3, 4, 5}; + +// Scale UV columns by 3 box filter. +// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per block. +void ScaleUVAddCols3_AVX2(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { +#if defined(__x86_64__) + asm volatile( + "vmovd %k[scaleval],%%xmm7 \n" + "vpbroadcastw %%xmm7,%%ymm7 \n" + "vmovdqa %[kPerm0],%%ymm4 \n" + "vmovdqa %[kPerm1],%%ymm5 \n" + "vmovdqa %[kPerm2],%%ymm6 \n" + "sub $0x10,%[dst_width] \n" + "jl 2f \n" + + // Process 16 output UV pixels (192 input bytes -> 32 output bytes). + LABELALIGN + "1: \n" + "vmovdqu (%[src_uv]),%%ymm0 \n" + "vmovdqu 0x20(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm0,%%ymm4,%%ymm0 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n" + "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n" + "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu 0x40(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n" + "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermd %%ymm2,%%ymm5,%%ymm2 \n" + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddw %%ymm0,%%ymm3,%%ymm8 \n" + "vpmulhuw %%ymm7,%%ymm8,%%ymm8 \n" + + "vmovdqu 0x60(%[src_uv]),%%ymm0 \n" + "vmovdqu 0x80(%[src_uv]),%%ymm1 \n" + "lea 0xc0(%[src_uv]),%[src_uv] \n" + "vpermd %%ymm0,%%ymm4,%%ymm0 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n" + "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n" + "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu -0x20(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n" + "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermd %%ymm2,%%ymm5,%%ymm2 \n" + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddw %%ymm0,%%ymm3,%%ymm3 \n" + "vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n" + + "vpackuswb %%ymm3,%%ymm8,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst_uv]) \n" + "lea 0x20(%[dst_uv]),%[dst_uv] \n" + "sub $0x10,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[dst_width] \n" + "jle 99f \n" + + // 8-pixel remainder. + "vmovdqu (%[src_uv]),%%ymm0 \n" + "vmovdqu 0x20(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm0,%%ymm4,%%ymm0 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n" + "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n" + "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu 0x40(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n" + "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermd %%ymm2,%%ymm5,%%ymm2 \n" + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddw %%ymm0,%%ymm3,%%ymm3 \n" + "vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n" + "vextracti128 $1,%%ymm3,%%xmm2 \n" + "vpackuswb %%xmm2,%%xmm3,%%xmm3 \n" + "vmovdqu %%xmm3,(%[dst_uv]) \n" + + "99: \n" + "vzeroupper \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "r"(scaleval), // %[scaleval] + [kPerm0] "m"(kPermUV3_0), // %[kPerm0] + [kPerm1] "m"(kPermUV3_1), // %[kPerm1] + [kPerm2] "m"(kPermUV3_2) // %[kPerm2] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8"); +#else // defined(__x86_64__) + asm volatile( + "vmovd %[scaleval],%%xmm7 \n" + "vpbroadcastw %%xmm7,%%ymm7 \n" + "vmovdqa %[kPerm0],%%ymm4 \n" + "vmovdqa %[kPerm1],%%ymm5 \n" + "vmovdqa %[kPerm2],%%ymm6 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src_uv]),%%ymm0 \n" + "vmovdqu 0x20(%[src_uv]),%%ymm1 \n" + "vpermd %%ymm0,%%ymm4,%%ymm0 \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n" + "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n" + "vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu 0x40(%[src_uv]),%%ymm1 \n" + "lea 0x60(%[src_uv]),%[src_uv] \n" + "vpermd %%ymm1,%%ymm4,%%ymm1 \n" + "vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n" + "vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n" + "vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermd %%ymm2,%%ymm5,%%ymm2 \n" + "vpermd %%ymm0,%%ymm6,%%ymm0 \n" + "vpaddw %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddw %%ymm0,%%ymm3,%%ymm3 \n" + "vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n" + "vextracti128 $1,%%ymm3,%%xmm2 \n" + "vpackuswb %%xmm2,%%xmm3,%%xmm3 \n" + "vmovdqu %%xmm3,(%[dst_uv]) \n" + "lea 0x10(%[dst_uv]),%[dst_uv] \n" + "sub $0x8,%[dst_width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "rm"(scaleval), // %[scaleval] + [kPerm0] "m"(kPermUV3_0), // %[kPerm0] + [kPerm1] "m"(kPermUV3_1), // %[kPerm1] + [kPerm2] "m"(kPermUV3_2) // %[kPerm2] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +#endif // defined(__x86_64__) +} +#endif // HAS_SCALEUVADDCOLS3_AVX2 + +#ifdef HAS_SCALEUVADDCOLS3_AVX512VBMI +static const uint8_t kPerm512_Col0[16] = {0, 3, 6, 9, 12, 15, 18, 21, + 24, 27, 30, 1, 4, 7, 10, 13}; +static const uint8_t kPerm512_Col1[16] = {1, 4, 7, 10, 13, 16, 19, 22, + 25, 28, 31, 2, 5, 8, 11, 14}; +static const uint8_t kPerm512_Col2[16] = {2, 5, 8, 11, 14, 17, 20, 23, + 26, 29, 0, 3, 6, 9, 12, 15}; +static const uint8_t kPackEvenBytes[64] = { + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, + 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, + 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, + 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, + 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126}; + +// Scale UV columns by 3 box filter using AVX512VBMI. +// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop +// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels). +void ScaleUVAddCols3_AVX512VBMI(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + intptr_t temp, mask; + asm volatile( + "vpbroadcastw %k[scaleval],%%zmm20 \n" + "vpmovzxbd %[kCol0],%%zmm16 \n" + "vpmovzxbd %[kCol1],%%zmm17 \n" + "vpmovzxbd %[kCol2],%%zmm18 \n" + "vmovdqu8 %[kPack],%%zmm19 \n" + "mov $0xf800,%k[temp] \n" + "kmovw %k[temp],%%k5 \n" + "mov $0xfc00,%k[temp] \n" + "kmovw %k[temp],%%k6 \n" + + "sub $0x20,%[dst_width] \n" + "jl 2f \n" + + // Main loop: 32 output UV pixels (64 output bytes) per iteration. + LABELALIGN + "1: \n" + "vmovdqu32 (%[src_uv]),%%zmm0 \n" + "vmovdqu32 0x40(%[src_uv]),%%zmm1 \n" + "vmovdqu32 0x80(%[src_uv]),%%zmm2 \n" + "vmovdqa32 %%zmm16,%%zmm3 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm3 \n" + "vmovdqa32 %%zmm17,%%zmm4 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm4 \n" + "vmovdqa32 %%zmm18,%%zmm5 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm5 \n" + "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n" + "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n" + "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n" + "vpaddw %%zmm4,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm5,%%zmm3,%%zmm3 \n" + "vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n" + + "vmovdqu32 0xc0(%[src_uv]),%%zmm6 \n" + "vmovdqu32 0x100(%[src_uv]),%%zmm1 \n" + "vmovdqu32 0x140(%[src_uv]),%%zmm2 \n" + "lea 0x180(%[src_uv]),%[src_uv] \n" + "vmovdqa32 %%zmm16,%%zmm3 \n" + "vpermi2d %%zmm1,%%zmm6,%%zmm3 \n" + "vmovdqa32 %%zmm17,%%zmm4 \n" + "vpermi2d %%zmm1,%%zmm6,%%zmm4 \n" + "vmovdqa32 %%zmm18,%%zmm5 \n" + "vpermi2d %%zmm1,%%zmm6,%%zmm5 \n" + "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n" + "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n" + "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n" + "vpaddw %%zmm4,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm5,%%zmm3,%%zmm3 \n" + "vpmulhuw %%zmm20,%%zmm3,%%zmm3 \n" + + "vmovdqa64 %%zmm19,%%zmm2 \n" + "vpermi2b %%zmm3,%%zmm0,%%zmm2 \n" + "vmovdqu8 %%zmm2,(%[dst_uv]) \n" + "lea 0x40(%[dst_uv]),%[dst_uv] \n" + "sub $0x20,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[dst_width] \n" + "jle 99f \n" + "cmp $0x10,%[dst_width] \n" + "jl 3f \n" + + // 16-pixel half-block (32 output bytes). + "vmovdqu32 (%[src_uv]),%%zmm0 \n" + "vmovdqu32 0x40(%[src_uv]),%%zmm1 \n" + "vmovdqu32 0x80(%[src_uv]),%%zmm2 \n" + "lea 0xc0(%[src_uv]),%[src_uv] \n" + "vmovdqa32 %%zmm16,%%zmm3 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm3 \n" + "vmovdqa32 %%zmm17,%%zmm4 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm4 \n" + "vmovdqa32 %%zmm18,%%zmm5 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm5 \n" + "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n" + "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n" + "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n" + "vpaddw %%zmm4,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm5,%%zmm3,%%zmm3 \n" + "vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n" + "vpmovwb %%zmm0,(%[dst_uv]) \n" + "lea 0x20(%[dst_uv]),%[dst_uv] \n" + "sub $0x10,%[dst_width] \n" + "jle 99f \n" + + // Tail: 1..15 output UV pixels (3..45 input dwords, 2..30 output bytes). + "3: \n" + "lea (%q[dst_width],%q[dst_width],2),%q[temp] \n" + "mov $-1,%q[mask] \n" + "bzhi %q[temp],%q[mask],%q[mask] \n" + "kmovw %k[mask],%%k1 \n" + "shr $16,%q[mask] \n" + "kmovw %k[mask],%%k2 \n" + "shr $16,%q[mask] \n" + "kmovw %k[mask],%%k3 \n" + "mov $-1,%q[mask] \n" + "bzhi %q[dst_width],%q[mask],%q[mask] \n" + "kmovw %k[mask],%%k4 \n" + + "vmovdqu32 (%[src_uv]),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x40(%[src_uv]),%%zmm1%{%%k2%}%{z%} \n" + "vmovdqu32 0x80(%[src_uv]),%%zmm2%{%%k3%}%{z%} \n" + "vmovdqa32 %%zmm16,%%zmm3 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm3 \n" + "vmovdqa32 %%zmm17,%%zmm4 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm4 \n" + "vmovdqa32 %%zmm18,%%zmm5 \n" + "vpermi2d %%zmm1,%%zmm0,%%zmm5 \n" + "vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n" + "vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n" + "vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n" + "vpaddw %%zmm4,%%zmm3,%%zmm3 \n" + "vpaddw %%zmm5,%%zmm3,%%zmm3 \n" + "vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n" + "vpmovwb %%zmm0,%%ymm0 \n" + "vmovdqu16 %%ymm0,(%[dst_uv])%{%%k4%} \n" + + "99: \n" + "vzeroupper \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width), // %[dst_width] + [temp] "=&r"(temp), // %[temp] + [mask] "=&r"(mask) // %[mask] + : [scaleval] "r"(scaleval), // %[scaleval] + [kCol0] "m"(kPerm512_Col0), // %[kCol0] + [kCol1] "m"(kPerm512_Col1), // %[kCol1] + [kCol2] "m"(kPerm512_Col2), // %[kCol2] + [kPack] "m"(kPackEvenBytes) // %[kPack] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "k1", "k2", "k3", "k4", + "k5", "k6"); +} +#endif // HAS_SCALEUVADDCOLS3_AVX512VBMI + #endif // defined(__x86_64__) || defined(__i386__) #ifdef __cplusplus diff --git a/source/scale_hvx.cc b/source/scale_hvx.cc index e4e748f22..0aab64a6a 100644 --- a/source/scale_hvx.cc +++ b/source/scale_hvx.cc @@ -92,6 +92,101 @@ void ScaleAddRow_HVX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) { } } +// vrdelta control vectors to compact 3 vectors of 3-column UV sums into 1 +// vector of 32 UV pairs (64 uint16_t): +// kUVAddCols3P0 routes dwords 0, 3, ..., 30 -> dwords 0..10 (bytes 0..43) +static const uint8_t kUVAddCols3P0[128] __attribute__((aligned(128))) = { + 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x08, 0x08, 0x08, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; +// kUVAddCols3P1 routes dwords 1, 4, ..., 31 -> dwords 11..21 (bytes 44..87) +static const uint8_t kUVAddCols3P1[128] __attribute__((aligned(128))) = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x10, 0x10, 0x10, 0x10, + 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, + 0x10, 0x10, 0x10, 0x10, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; +// kUVAddCols3P2 routes dwords 2, 5, ..., 29 -> dwords 22..31 (bytes 88..127) +static const uint8_t kUVAddCols3P2[128] __attribute__((aligned(128))) = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x08, 0x08, 0x08, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08, +}; + +void ScaleUVAddCols3_HVX(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + const HVX_Vector p0 = vloada(kUVAddCols3P0); + const HVX_Vector p1 = vloada(kUVAddCols3P1); + const HVX_Vector p2 = vloada(kUVAddCols3P2); + const HVX_VectorPred q44 = Q6_Q_vsetq_R(44); + const HVX_VectorPred q88 = Q6_Q_vsetq_R(88); + const uint32_t scale_splat = + ((uint32_t)(uint16_t)scaleval << 16) | (uint16_t)scaleval; + while (dst_width > 0) { + int w = dst_width >= 32 ? 32 : dst_width; + int src_bytes = w * 12; + HVX_Vector v0 = + vloadu_partial_zero(src_uv, src_bytes > 128 ? 128 : src_bytes); + HVX_Vector v1 = + src_bytes > 128 + ? vloadu_partial_zero((const uint8_t*)src_uv + 128, + src_bytes > 256 ? 128 : src_bytes - 128) + : Q6_V_vzero(); + HVX_Vector v2 = + src_bytes > 256 + ? vloadu_partial_zero((const uint8_t*)src_uv + 256, src_bytes - 256) + : Q6_V_vzero(); + + HVX_Vector s0 = + Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v0, Q6_V_valign_VVR(v1, v0, 4)), + Q6_V_valign_VVR(v1, v0, 8)); + HVX_Vector s1 = + Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v1, Q6_V_valign_VVR(v2, v1, 4)), + Q6_V_valign_VVR(v2, v1, 8)); + HVX_Vector s2 = + Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v2, Q6_V_valign_VVR(v2, v2, 4)), + Q6_V_valign_VVR(v2, v2, 8)); + + HVX_Vector sum = Q6_V_vmux_QVV( + q44, Q6_V_vrdelta_VV(s0, p0), + Q6_V_vmux_QVV(q88, Q6_V_vrdelta_VV(s1, p1), Q6_V_vrdelta_VV(s2, p2))); + + HVX_VectorPair prod = Q6_Wuw_vmpy_VuhRuh(sum, scale_splat); + HVX_Vector scaled = Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(prod), Q6_V_lo_W(prod)); + HVX_Vector out = Q6_Vub_vpack_VhVh_sat(scaled, scaled); + vstoreu_partial(dst_uv, out, w * 2); + src_uv += 192; + dst_uv += 64; + dst_width -= 32; + } +} + void ScaleRowDown2_HVX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, diff --git a/source/scale_lsx.cc b/source/scale_lsx.cc index e44596781..cdb333ee4 100644 --- a/source/scale_lsx.cc +++ b/source/scale_lsx.cc @@ -581,6 +581,67 @@ void ScaleAddRow_LSX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) { : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); } +static const uint8_t kScaleUVAddCols3Shuf[] = { + 0, 1, 2, 3, 12, 13, 14, 15, 24, 25, 26, 27, 4, 5, 6, 7, + 4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31, 8, 9, 10, 11, + 8, 9, 10, 11, 20, 21, 22, 23, 0, 1, 2, 3, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 20, 21, 22, 23, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 24, 25, 26, 27, + 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31, +}; + +void ScaleUVAddCols3_LSX(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + asm volatile( + "vld $vr6, %[shuf], 0 \n" + "vld $vr7, %[shuf], 16 \n" + "vld $vr8, %[shuf], 32 \n" + "vld $vr9, %[shuf], 48 \n" + "vld $vr10, %[shuf], 64 \n" + "vld $vr11, %[shuf], 80 \n" + "vreplgr2vr.h $vr12, %[scaleval] \n" + "1: \n" + "vld $vr0, %[src_uv], 0 \n" + "vld $vr1, %[src_uv], 16 \n" + "vld $vr2, %[src_uv], 32 \n" + "vshuf.b $vr3, $vr1, $vr0, $vr6 \n" + "vshuf.b $vr4, $vr1, $vr0, $vr7 \n" + "vshuf.b $vr5, $vr1, $vr0, $vr8 \n" + "vshuf.b $vr3, $vr2, $vr3, $vr9 \n" + "vshuf.b $vr4, $vr2, $vr4, $vr10 \n" + "vshuf.b $vr5, $vr2, $vr5, $vr11 \n" + "vadd.h $vr3, $vr3, $vr4 \n" + "vadd.h $vr3, $vr3, $vr5 \n" + "vmuh.hu $vr3, $vr3, $vr12 \n" + "vld $vr0, %[src_uv], 48 \n" + "vld $vr1, %[src_uv], 64 \n" + "vld $vr2, %[src_uv], 80 \n" + "addi.d %[src_uv], %[src_uv], 96 \n" + "vshuf.b $vr13, $vr1, $vr0, $vr6 \n" + "vshuf.b $vr4, $vr1, $vr0, $vr7 \n" + "vshuf.b $vr5, $vr1, $vr0, $vr8 \n" + "vshuf.b $vr13, $vr2, $vr13, $vr9 \n" + "vshuf.b $vr4, $vr2, $vr4, $vr10 \n" + "vshuf.b $vr5, $vr2, $vr5, $vr11 \n" + "vadd.h $vr13, $vr13, $vr4 \n" + "vadd.h $vr13, $vr13, $vr5 \n" + "vmuh.hu $vr13, $vr13, $vr12 \n" + "vpickev.b $vr0, $vr13, $vr3 \n" + "vst $vr0, %[dst_uv], 0 \n" + "addi.d %[dst_uv], %[dst_uv], 16 \n" + "addi.w %[dst_width], %[dst_width], -8 \n" + "bgtz %[dst_width], 1b \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "r"(scaleval), // %[scaleval] + [shuf] "r"(kScaleUVAddCols3Shuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13"); +} + void ScaleFilterCols_LSX(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, diff --git a/source/scale_neon.cc b/source/scale_neon.cc index 4f9e03b8a..2d5be0af3 100644 --- a/source/scale_neon.cc +++ b/source/scale_neon.cc @@ -1448,6 +1448,43 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr, : "memory", "cc", "d0"); } +// Scale UV columns by 3 box filter. +// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop. +void ScaleUVAddCols3_NEON(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + asm volatile( + "vdup.16 d16, %[scaleval] \n" + "1: \n" + "vld3.32 {d0, d2, d4}, [%[src_uv]]! \n" + "vld3.32 {d1, d3, d5}, [%[src_uv]]! \n" + "vadd.u16 q0, q0, q1 \n" + "vadd.u16 q0, q0, q2 \n" + "vld3.32 {d2, d4, d6}, [%[src_uv]]! \n" + "vld3.32 {d3, d5, d7}, [%[src_uv]]! \n" + "vadd.u16 q1, q1, q2 \n" + "vadd.u16 q1, q1, q3 \n" + "vmull.u16 q2, d0, d16 \n" + "vmull.u16 q3, d1, d16 \n" + "vmull.u16 q9, d2, d16 \n" + "vmull.u16 q10, d3, d16 \n" + "vshrn.i32 d0, q2, #16 \n" + "vshrn.i32 d1, q3, #16 \n" + "vshrn.i32 d2, q9, #16 \n" + "vshrn.i32 d3, q10, #16 \n" + "vmovn.i16 d0, q0 \n" + "vmovn.i16 d1, q1 \n" + "subs %[dst_width], %[dst_width], #8 \n" + "vst1.8 {q0}, [%[dst_uv]]! \n" + "bgt 1b \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "r"(scaleval) // %[scaleval] + : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10"); +} + #endif // defined(__ARM_NEON__) && !defined(__aarch64__) #ifdef __cplusplus diff --git a/source/scale_neon64.cc b/source/scale_neon64.cc index ae4768f22..3514bb079 100644 --- a/source/scale_neon64.cc +++ b/source/scale_neon64.cc @@ -1551,6 +1551,38 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr, : "memory", "cc", "v0", "v1", "v2", "v3"); } +// Scale UV columns by 3 box filter. +// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop. +void ScaleUVAddCols3_NEON(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + asm volatile( + "dup v6.8h, %w[scaleval] \n" + "1: \n" + "ld3 {v0.4s, v1.4s, v2.4s}, [%[src_uv]], #48 \n" + "ld3 {v3.4s, v4.4s, v5.4s}, [%[src_uv]], #48 \n" + "subs %w[dst_width], %w[dst_width], #8 \n" + "add v0.8h, v0.8h, v1.8h \n" + "add v3.8h, v3.8h, v4.8h \n" + "add v0.8h, v0.8h, v2.8h \n" + "add v3.8h, v3.8h, v5.8h \n" + "umull v1.4s, v0.4h, v6.4h \n" + "umull2 v2.4s, v0.8h, v6.8h \n" + "umull v4.4s, v3.4h, v6.4h \n" + "umull2 v5.4s, v3.8h, v6.8h \n" + "uzp2 v0.8h, v1.8h, v2.8h \n" + "uzp2 v3.8h, v4.8h, v5.8h \n" + "uzp1 v0.16b, v0.16b, v3.16b \n" + "st1 {v0.16b}, [%[dst_uv]], #16 \n" + "b.gt 1b \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "r"(scaleval) // %[scaleval] + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/scale_rvv.cc b/source/scale_rvv.cc index 99c945a31..238a9a953 100644 --- a/source/scale_rvv.cc +++ b/source/scale_rvv.cc @@ -1304,6 +1304,42 @@ void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr, } #endif +#ifdef HAS_SCALEUVADDCOLS3_RVV +void ScaleUVAddCols3_RVV(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + size_t vl, vl2; + size_t w = (size_t)dst_width; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e32, m2, ta, ma \n" + "vlseg3e32.v v8, (%[src_uv]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl2], %[vl], 2 \n" + "add %[src_uv], %[src_uv], %[vl2] \n" + "slli %[vl2], %[vl2], 1 \n" + "add %[src_uv], %[src_uv], %[vl2] \n" + "slli %[vl], %[vl], 1 \n" + "vsetvli zero, %[vl], e16, m2, ta, ma \n" + "vadd.vv v8, v8, v10 \n" + "vadd.vv v8, v8, v12 \n" + "vmulhu.vx v8, v8, %[scaleval] \n" + "vsetvli zero, %[vl], e8, m1, ta, ma \n" + "vnsrl.wi v4, v8, 0 \n" + "vse8.v v4, (%[dst_uv]) \n" + "add %[dst_uv], %[dst_uv], %[vl] \n" + "bgtz %[w], 1b \n" + : [w] "+r"(w), // %[w] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [vl] "=&r"(vl), // %[vl] + [vl2] "=&r"(vl2) // %[vl2] + : [scaleval] "r"(scaleval) // %[scaleval] + : RVV_VL_CLOBBER "memory", "v4", "v8", "v9", "v10", "v11", "v12", "v13"); +} +#endif + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale_sme.cc b/source/scale_sme.cc index 70e3e2d72..1dc5c17ba 100644 --- a/source/scale_sme.cc +++ b/source/scale_sme.cc @@ -554,6 +554,71 @@ __arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb, "p1"); } +// Scale UV columns by 3 box filter using Streaming SVE. +__arm_locally_streaming void ScaleUVAddCols3_SME(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + // Streaming-SVE only, no use of ZA tile. + int vl; + int vl_w; + asm volatile( + "cnth %x[vl] \n" + "cntw %x[vl_w] \n" + "dup z6.h, %w[scaleval] \n" + "ptrue p3.h \n" + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.s \n" + "ptrue p1.h \n" + "1: \n" + "ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n" + "incb %[src_uv], all, mul #3 \n" + "ld3w {z3.s, z4.s, z5.s}, p0/z, [%[src_uv]] \n" + "incb %[src_uv], all, mul #3 \n" + "subs %w[dst_width], %w[dst_width], %w[vl] \n" + "add z0.h, z0.h, z1.h \n" + "add z3.h, z3.h, z4.h \n" + "add z0.h, z0.h, z2.h \n" + "add z3.h, z3.h, z5.h \n" + "umulh z0.h, p3/m, z0.h, z6.h \n" + "umulh z3.h, p3/m, z3.h, z6.h \n" + "uzp1 z0.b, z0.b, z3.b \n" + "st1h {z0.h}, p1, [%[dst_uv]] \n" + "incb %[dst_uv] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[dst_width], %w[dst_width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[dst_width] \n" + "whilelt p2.s, %w[vl_w], %w[dst_width] \n" + "whilelt p1.h, wzr, %w[dst_width] \n" + "ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n" + "incb %[src_uv], all, mul #3 \n" + "ld3w {z3.s, z4.s, z5.s}, p2/z, [%[src_uv]] \n" + "add z0.h, z0.h, z1.h \n" + "add z3.h, z3.h, z4.h \n" + "add z0.h, z0.h, z2.h \n" + "add z3.h, z3.h, z5.h \n" + "umulh z0.h, p3/m, z0.h, z6.h \n" + "umulh z3.h, p3/m, z3.h, z6.h \n" + "uzp1 z0.b, z0.b, z3.b \n" + "st1h {z0.h}, p1, [%[dst_uv]] \n" + + "99: \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width), // %[dst_width] + [vl] "=&r"(vl), // %[vl] + [vl_w] "=&r"(vl_w) // %[vl_w] + : [scaleval] "r"(scaleval) // %[scaleval] + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "p0", "p1", + "p2", "p3"); +} + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 51467dee9..1e21a4948 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -346,10 +346,67 @@ static int ScaleUVBox(int src_width, void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx, const uint16_t* src_uv, uint8_t* dst_uv) = (dx & 0xffff) ? ScaleUVAddCols2_C : ScaleUVAddCols1_C; + void (*ScaleUVAddCols3)(int dst_width, int scaleval, + const uint16_t* src_uv, uint8_t* dst_uv) = NULL; // Summing a row is position wise, so an interleaved UV row can be fed to // the plane accumulator as src_width * 2 samples. void (*ScaleAddRow)(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) = ScaleAddRow_C; + if (dx == 0x30000) { + ScaleUVAddCols3 = ScaleUVAddCols3_C; +#if defined(HAS_SCALEUVADDCOLS3_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleUVAddCols3 = ScaleUVAddCols3_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVAddCols3 = ScaleUVAddCols3_AVX2; + } + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + ScaleUVAddCols3 = ScaleUVAddCols3_AVX512VBMI; + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleUVAddCols3 = ScaleUVAddCols3_Any_NEON; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVAddCols3 = ScaleUVAddCols3_NEON; + } + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_SME) + if (TestCpuFlag(kCpuHasSME)) { + ScaleUVAddCols3 = ScaleUVAddCols3_SME; + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ScaleUVAddCols3 = ScaleUVAddCols3_RVV; + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ScaleUVAddCols3 = ScaleUVAddCols3_Any_LSX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVAddCols3 = ScaleUVAddCols3_LSX; + } + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_WASMSIMD) + if (TestCpuFlag(kCpuHasWASM)) { + ScaleUVAddCols3 = ScaleUVAddCols3_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVAddCols3 = ScaleUVAddCols3_WASMSIMD; + } + } +#endif +#if defined(HAS_SCALEUVADDCOLS3_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + ScaleUVAddCols3 = ScaleUVAddCols3_HVX; + } +#endif + } #if defined(HAS_SCALEADDROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { ScaleAddRow = ScaleAddRow_Any_SSE2; @@ -366,6 +423,11 @@ static int ScaleUVBox(int src_width, } } #endif +#if defined(HAS_SCALEADDROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ScaleAddRow = ScaleAddRow_AVX512BW; + } +#endif #if defined(HAS_SCALEADDROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ScaleAddRow = ScaleAddRow_Any_NEON; @@ -392,6 +454,14 @@ static int ScaleUVBox(int src_width, ScaleAddRow = ScaleAddRow_HVX; } #endif +#if defined(HAS_SCALEADDROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASM)) { + ScaleAddRow = ScaleAddRow_Any_WASMSIMD; + if (IS_ALIGNED(row_samples, 16)) { + ScaleAddRow = ScaleAddRow_WASMSIMD; + } + } +#endif uint32_t uy = y; for (j = 0; j < dst_height; ++j) { @@ -408,7 +478,14 @@ static int ScaleUVBox(int src_width, ScaleAddRow(src, (uint16_t*)(row16), row_samples); src += src_stride; } - ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16), dst_uv); + if (ScaleUVAddCols3 && boxheight <= 85) { + ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight), + (const uint16_t*)row16 + (ptrdiff_t)(x >> 16) * 2, + dst_uv); + } else { + ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16), + dst_uv); + } dst_uv += dst_stride; } } diff --git a/source/scale_wasm.cc b/source/scale_wasm.cc index 661f923eb..62e613ea3 100644 --- a/source/scale_wasm.cc +++ b/source/scale_wasm.cc @@ -637,6 +637,74 @@ void ScaleARGBColsUp2_WASMSIMD(uint8_t* dst_argb, } #endif // HAS_SCALEARGBCOLSUP2_WASMSIMD +#ifdef HAS_SCALEADDROW_WASMSIMD +// Reads 16 bytes and accumulates to 16 shorts at a time. +void ScaleAddRow_WASMSIMD(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width) { + while (src_width > 0) { + v128_t s = wasm_v128_load(src_ptr); + v128_t d0 = wasm_v128_load(dst_ptr); + v128_t d1 = wasm_v128_load(dst_ptr + 8); + d0 = wasm_i16x8_add(d0, wasm_u16x8_extend_low_u8x16(s)); + d1 = wasm_i16x8_add(d1, wasm_u16x8_extend_high_u8x16(s)); + wasm_v128_store(dst_ptr, d0); + wasm_v128_store(dst_ptr + 8, d1); + src_ptr += 16; + dst_ptr += 16; + src_width -= 16; + } +} +#endif // HAS_SCALEADDROW_WASMSIMD + +#ifdef HAS_SCALEUVADDCOLS3_WASMSIMD +// Reads 24 UV pixels (48 uint16_t) and writes 8 UV pixels (16 bytes). +void ScaleUVAddCols3_WASMSIMD(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + const v128_t scale = wasm_i16x8_splat((int16_t)scaleval); + while (dst_width > 0) { + v128_t s0 = wasm_v128_load(src_uv); + v128_t s1 = wasm_v128_load(src_uv + 8); + v128_t s2 = wasm_v128_load(src_uv + 16); + v128_t s3 = wasm_v128_load(src_uv + 24); + v128_t s4 = wasm_v128_load(src_uv + 32); + v128_t s5 = wasm_v128_load(src_uv + 40); + src_uv += 48; + + // Group 0..3 of 3 UV pairs from s0, s1, s2 (dwords 0..11) + v128_t a0 = wasm_i32x4_shuffle(s0, s1, 0, 3, 6, 1); + v128_t b0 = wasm_i32x4_shuffle(s0, s1, 1, 4, 7, 2); + v128_t c0 = wasm_i32x4_shuffle(s0, s1, 2, 5, 0, 3); + a0 = wasm_i32x4_shuffle(a0, s2, 0, 1, 2, 5); + b0 = wasm_i32x4_shuffle(b0, s2, 0, 1, 2, 6); + c0 = wasm_i32x4_shuffle(c0, s2, 0, 1, 4, 7); + v128_t sum0 = wasm_i16x8_add(wasm_i16x8_add(a0, b0), c0); + + // Group 4..7 of 3 UV pairs from s3, s4, s5 (dwords 0..11) + v128_t a1 = wasm_i32x4_shuffle(s3, s4, 0, 3, 6, 1); + v128_t b1 = wasm_i32x4_shuffle(s3, s4, 1, 4, 7, 2); + v128_t c1 = wasm_i32x4_shuffle(s3, s4, 2, 5, 0, 3); + a1 = wasm_i32x4_shuffle(a1, s5, 0, 1, 2, 5); + b1 = wasm_i32x4_shuffle(b1, s5, 0, 1, 2, 6); + c1 = wasm_i32x4_shuffle(c1, s5, 0, 1, 4, 7); + v128_t sum1 = wasm_i16x8_add(wasm_i16x8_add(a1, b1), c1); + + // Multiply unsigned 16-bit sums by scaleval and shift right by 16. + v128_t lo0 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum0, scale), 16); + v128_t hi0 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum0, scale), 16); + v128_t lo1 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum1, scale), 16); + v128_t hi1 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum1, scale), 16); + v128_t p0 = wasm_i16x8_narrow_i32x4(lo0, hi0); + v128_t p1 = wasm_i16x8_narrow_i32x4(lo1, hi1); + wasm_v128_store(dst_uv, wasm_u8x16_narrow_i16x8(p0, p1)); + dst_uv += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVADDCOLS3_WASMSIMD + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale_win.cc b/source/scale_win.cc index 4b7fd3590..a85cb0a4f 100644 --- a/source/scale_win.cc +++ b/source/scale_win.cc @@ -11,6 +11,19 @@ #include "libyuv/row.h" #include "libyuv/scale_row.h" +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) +#include +#include +#include +#if defined(_MSC_VER) && !defined(__clang__) +#include +#endif +#endif + #ifdef __cplusplus namespace libyuv { extern "C" { @@ -55,40 +68,6 @@ static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3}; // Coefficients for source bytes 21 to 31 static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2}; -static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128}; - -static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0, 3, - 6, 8, 11, 14, 128, 128, 128, 128}; - -// Arrange words 0,3,6 into 0,1,2 -static const uvec8 kShufAc = {0, 1, 6, 7, 12, 13, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128}; - -// Arrange words 0,3,6 into 3,4,5 -static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0, 1, - 6, 7, 12, 13, 128, 128, 128, 128}; - -// Scaling values for boxes of 3x3 and 2x3 -static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9, - 65536 / 9, 65536 / 6, 0, 0}; - -// Arrange first value for pixels 0,1,2,3,4,5 -static const uvec8 kShufAb0 = {0, 128, 3, 128, 6, 128, 8, 128, - 11, 128, 14, 128, 128, 128, 128, 128}; - -// Arrange second value for pixels 0,1,2,3,4,5 -static const uvec8 kShufAb1 = {1, 128, 4, 128, 7, 128, 9, 128, - 12, 128, 15, 128, 128, 128, 128, 128}; - -// Arrange third value for pixels 0,1,2,3,4,5 -static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128, - 13, 128, 128, 128, 128, 128, 128, 128}; - -// Scaling values for boxes of 3x2 and 2x2 -static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3, - 65536 / 3, 65536 / 2, 0, 0}; - // Reads 32 pixels, throws half away and writes 16 pixels. __declspec(naked) void ScaleRowDown2_SSSE3(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -664,212 +643,6 @@ __declspec(naked) void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, } } -// 3/8 point sampler - -// Scale 32 pixels to 12 -__declspec(naked) void ScaleRowDown38_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_ptr - // src_stride ignored - mov edx, [esp + 12] // dst_ptr - mov ecx, [esp + 16] // dst_width - movdqa xmm4, xmmword ptr kShuf38a - movdqa xmm5, xmmword ptr kShuf38b - - xloop: - movdqu xmm0, [eax] // 16 pixels -> 0,1,2,3,4,5 - movdqu xmm1, [eax + 16] // 16 pixels -> 6,7,8,9,10,11 - lea eax, [eax + 32] - pshufb xmm0, xmm4 - pshufb xmm1, xmm5 - paddusb xmm0, xmm1 - - movq qword ptr [edx], xmm0 // write 12 pixels - movhlps xmm1, xmm0 - movd [edx + 8], xmm1 - lea edx, [edx + 12] - sub ecx, 12 - jg xloop - - ret - } -} - -// Scale 16x3 pixels to 6x1 with interpolation -__declspec(naked) void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_ptr - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_ptr - mov ecx, [esp + 4 + 16] // dst_width - movdqa xmm2, xmmword ptr kShufAc - movdqa xmm3, xmmword ptr kShufAc3 - movdqa xmm4, xmmword ptr kScaleAc33 - pxor xmm5, xmm5 - - xloop: - movdqu xmm0, [eax] // sum up 3 rows into xmm0/1 - movdqu xmm6, [eax + esi] - movhlps xmm1, xmm0 - movhlps xmm7, xmm6 - punpcklbw xmm0, xmm5 - punpcklbw xmm1, xmm5 - punpcklbw xmm6, xmm5 - punpcklbw xmm7, xmm5 - paddusw xmm0, xmm6 - paddusw xmm1, xmm7 - movdqu xmm6, [eax + esi * 2] - lea eax, [eax + 16] - movhlps xmm7, xmm6 - punpcklbw xmm6, xmm5 - punpcklbw xmm7, xmm5 - paddusw xmm0, xmm6 - paddusw xmm1, xmm7 - - movdqa xmm6, xmm0 // 8 pixels -> 0,1,2 of xmm6 - psrldq xmm0, 2 - paddusw xmm6, xmm0 - psrldq xmm0, 2 - paddusw xmm6, xmm0 - pshufb xmm6, xmm2 - - movdqa xmm7, xmm1 // 8 pixels -> 3,4,5 of xmm6 - psrldq xmm1, 2 - paddusw xmm7, xmm1 - psrldq xmm1, 2 - paddusw xmm7, xmm1 - pshufb xmm7, xmm3 - paddusw xmm6, xmm7 - - pmulhuw xmm6, xmm4 // divide by 9,9,6, 9,9,6 - packuswb xmm6, xmm6 - - movd [edx], xmm6 // write 6 pixels - psrlq xmm6, 16 - movd [edx + 2], xmm6 - lea edx, [edx + 6] - sub ecx, 6 - jg xloop - - pop esi - ret - } -} - -// Scale 16x2 pixels to 6x1 with interpolation -__declspec(naked) void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_ptr - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_ptr - mov ecx, [esp + 4 + 16] // dst_width - movdqa xmm2, xmmword ptr kShufAb0 - movdqa xmm3, xmmword ptr kShufAb1 - movdqa xmm4, xmmword ptr kShufAb2 - movdqa xmm5, xmmword ptr kScaleAb2 - - xloop: - movdqu xmm0, [eax] // average 2 rows into xmm0 - movdqu xmm1, [eax + esi] - lea eax, [eax + 16] - pavgb xmm0, xmm1 - - movdqa xmm1, xmm0 // 16 pixels -> 0,1,2,3,4,5 of xmm1 - pshufb xmm1, xmm2 - movdqa xmm6, xmm0 - pshufb xmm6, xmm3 - paddusw xmm1, xmm6 - pshufb xmm0, xmm4 - paddusw xmm1, xmm0 - - pmulhuw xmm1, xmm5 // divide by 3,3,2, 3,3,2 - packuswb xmm1, xmm1 - - movd [edx], xmm1 // write 6 pixels - psrlq xmm1, 16 - movd [edx + 2], xmm1 - lea edx, [edx + 6] - sub ecx, 6 - jg xloop - - pop esi - ret - } -} - -// Reads 16 bytes and accumulates to 16 shorts at a time. -__declspec(naked) void ScaleAddRow_SSE2(const uint8_t* src_ptr, - uint16_t* dst_ptr, - int src_width) { - __asm { - mov eax, [esp + 4] // src_ptr - mov edx, [esp + 8] // dst_ptr - mov ecx, [esp + 12] // src_width - pxor xmm5, xmm5 - - // sum rows - xloop: - movdqu xmm3, [eax] // read 16 bytes - lea eax, [eax + 16] - movdqu xmm0, [edx] // read 16 words from destination - movdqu xmm1, [edx + 16] - movdqa xmm2, xmm3 - punpcklbw xmm2, xmm5 - punpckhbw xmm3, xmm5 - paddusw xmm0, xmm2 // sum 16 words - paddusw xmm1, xmm3 - movdqu [edx], xmm0 // write 16 words to destination - movdqu [edx + 16], xmm1 - lea edx, [edx + 32] - sub ecx, 16 - jg xloop - ret - } -} - -#ifdef HAS_SCALEADDROW_AVX2 -// Reads 32 bytes and accumulates to 32 shorts at a time. -__declspec(naked) void ScaleAddRow_AVX2(const uint8_t* src_ptr, - uint16_t* dst_ptr, - int src_width) { - __asm { - mov eax, [esp + 4] // src_ptr - mov edx, [esp + 8] // dst_ptr - mov ecx, [esp + 12] // src_width - vpxor ymm5, ymm5, ymm5 - - // sum rows - xloop: - vmovdqu ymm3, [eax] // read 32 bytes - lea eax, [eax + 32] - vpermq ymm3, ymm3, 0xd8 // unmutate for vpunpck - vpunpcklbw ymm2, ymm3, ymm5 - vpunpckhbw ymm3, ymm3, ymm5 - vpaddusw ymm0, ymm2, [edx] // sum 16 words - vpaddusw ymm1, ymm3, [edx + 32] - vmovdqu [edx], ymm0 // write 32 words to destination - vmovdqu [edx + 32], ymm1 - lea edx, [edx + 64] - sub ecx, 32 - jg xloop - - vzeroupper - ret - } -} -#endif // HAS_SCALEADDROW_AVX2 - static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040}; @@ -1382,6 +1155,772 @@ __declspec(naked) int FixedDiv1_X86(int num, int div) { } #endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) +// The following are available for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) + +#if defined(__clang__) || defined(__GNUC__) +#define LIBYUV_TARGET_SSSE3 __attribute__((target("ssse3"))) +#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) +#define LIBYUV_TARGET_AVX512BW \ + __attribute__((target("avx512bw,avx512vl,avx512f"))) +#define LIBYUV_TARGET_AVX512VBMI \ + __attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f"))) +#else +#define LIBYUV_TARGET_SSSE3 +#define LIBYUV_TARGET_AVX2 +#define LIBYUV_TARGET_AVX512BW +#define LIBYUV_TARGET_AVX512VBMI +#endif // defined(__clang__) || defined(__GNUC__) + +#ifdef HAS_SCALEROWDOWN38_SSSE3 +static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128}; + +static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0, 3, + 6, 8, 11, 14, 128, 128, 128, 128}; + +// Arrange words 0,3,6 into 0,1,2 +static const uvec8 kShufAc = {0, 1, 6, 7, 12, 13, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128}; + +// Arrange words 0,3,6 into 3,4,5 +static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0, 1, + 6, 7, 12, 13, 128, 128, 128, 128}; + +// Scaling values for boxes of 3x3 and 2x3 +static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9, + 65536 / 9, 65536 / 6, 0, 0}; + +// Arrange first value for pixels 0,1,2,3,4,5 +static const uvec8 kShufAb0 = {0, 128, 3, 128, 6, 128, 8, 128, + 11, 128, 14, 128, 128, 128, 128, 128}; + +// Arrange second value for pixels 0,1,2,3,4,5 +static const uvec8 kShufAb1 = {1, 128, 4, 128, 7, 128, 9, 128, + 12, 128, 15, 128, 128, 128, 128, 128}; + +// Arrange third value for pixels 0,1,2,3,4,5 +static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128, + 13, 128, 128, 128, 128, 128, 128, 128}; + +// Scaling values for boxes of 3x2 and 2x2 +static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3, + 65536 / 3, 65536 / 2, 0, 0}; + +// Scale 32 pixels to 12 +LIBYUV_TARGET_SSSE3 +void ScaleRowDown38_SSSE3(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m128i shuf38a = _mm_loadu_si128((const __m128i*)&kShuf38a); + const __m128i shuf38b = _mm_loadu_si128((const __m128i*)&kShuf38b); + while (dst_width > 0) { + __m128i s0 = _mm_loadu_si128((const __m128i*)src_ptr); + __m128i s1 = _mm_loadu_si128((const __m128i*)(src_ptr + 16)); + src_ptr += 32; + s0 = _mm_shuffle_epi8(s0, shuf38a); + s1 = _mm_shuffle_epi8(s1, shuf38b); + s0 = _mm_adds_epu8(s0, s1); + _mm_storel_epi64((__m128i*)dst_ptr, s0); + *(uint32_t*)(dst_ptr + 8) = + (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(s0, 8)); + dst_ptr += 12; + dst_width -= 12; + } +} + +// Scale 16x3 pixels to 6x1 with interpolation +LIBYUV_TARGET_SSSE3 +void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m128i shuf_ac = _mm_loadu_si128((const __m128i*)&kShufAc); + const __m128i shuf_ac3 = _mm_loadu_si128((const __m128i*)&kShufAc3); + const __m128i scale_ac33 = _mm_loadu_si128((const __m128i*)&kScaleAc33); + const __m128i zero = _mm_setzero_si128(); + while (dst_width > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)src_ptr); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src_ptr + src_stride)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src_ptr + src_stride * 2)); + src_ptr += 16; + __m128i s0 = _mm_adds_epu16(_mm_adds_epu16(_mm_unpacklo_epi8(r0, zero), + _mm_unpacklo_epi8(r1, zero)), + _mm_unpacklo_epi8(r2, zero)); + __m128i s1 = _mm_adds_epu16(_mm_adds_epu16(_mm_unpackhi_epi8(r0, zero), + _mm_unpackhi_epi8(r1, zero)), + _mm_unpackhi_epi8(r2, zero)); + __m128i t0 = _mm_adds_epu16(_mm_adds_epu16(s0, _mm_srli_si128(s0, 2)), + _mm_srli_si128(s0, 4)); + __m128i t1 = _mm_adds_epu16(_mm_adds_epu16(s1, _mm_srli_si128(s1, 2)), + _mm_srli_si128(s1, 4)); + t0 = _mm_shuffle_epi8(t0, shuf_ac); + t1 = _mm_shuffle_epi8(t1, shuf_ac3); + t0 = _mm_adds_epu16(t0, t1); + t0 = _mm_mulhi_epu16(t0, scale_ac33); + t0 = _mm_packus_epi16(t0, t0); + *(uint32_t*)dst_ptr = (uint32_t)_mm_cvtsi128_si32(t0); + *(uint16_t*)(dst_ptr + 4) = + (uint16_t)_mm_cvtsi128_si32(_mm_srli_si128(t0, 4)); + dst_ptr += 6; + dst_width -= 6; + } +} + +// Scale 16x2 pixels to 6x1 with interpolation +LIBYUV_TARGET_SSSE3 +void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m128i shuf_ab0 = _mm_loadu_si128((const __m128i*)&kShufAb0); + const __m128i shuf_ab1 = _mm_loadu_si128((const __m128i*)&kShufAb1); + const __m128i shuf_ab2 = _mm_loadu_si128((const __m128i*)&kShufAb2); + const __m128i scale_ab2 = _mm_loadu_si128((const __m128i*)&kScaleAb2); + while (dst_width > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)src_ptr); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src_ptr + src_stride)); + src_ptr += 16; + r0 = _mm_avg_epu8(r0, r1); + __m128i t0 = _mm_shuffle_epi8(r0, shuf_ab0); + __m128i t1 = _mm_shuffle_epi8(r0, shuf_ab1); + __m128i t2 = _mm_shuffle_epi8(r0, shuf_ab2); + t0 = _mm_adds_epu16(_mm_adds_epu16(t0, t1), t2); + t0 = _mm_mulhi_epu16(t0, scale_ab2); + t0 = _mm_packus_epi16(t0, t0); + *(uint32_t*)dst_ptr = (uint32_t)_mm_cvtsi128_si32(t0); + *(uint16_t*)(dst_ptr + 4) = + (uint16_t)_mm_cvtsi128_si32(_mm_srli_si128(t0, 4)); + dst_ptr += 6; + dst_width -= 6; + } +} +#endif // HAS_SCALEROWDOWN38_SSSE3 + +#ifdef HAS_SCALEADDROW_SSE2 +// Reads 16 bytes and accumulates to 16 shorts at a time. +void ScaleAddRow_SSE2(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width) { + const __m128i zero = _mm_setzero_si128(); + while (src_width > 0) { + __m128i s = _mm_loadu_si128((const __m128i*)src_ptr); + src_ptr += 16; + __m128i d0 = _mm_loadu_si128((const __m128i*)dst_ptr); + __m128i d1 = _mm_loadu_si128((const __m128i*)(dst_ptr + 8)); + d0 = _mm_adds_epu16(d0, _mm_unpacklo_epi8(s, zero)); + d1 = _mm_adds_epu16(d1, _mm_unpackhi_epi8(s, zero)); + _mm_storeu_si128((__m128i*)dst_ptr, d0); + _mm_storeu_si128((__m128i*)(dst_ptr + 8), d1); + dst_ptr += 16; + src_width -= 16; + } +} +#endif // HAS_SCALEADDROW_SSE2 + +#ifdef HAS_SCALEADDROW_AVX2 +// Reads 32 bytes and accumulates to 32 shorts at a time. +LIBYUV_TARGET_AVX2 +void ScaleAddRow_AVX2(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width) { + while (src_width > 0) { + __m256i s0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_ptr)); + __m256i s1 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_ptr + 16))); + src_ptr += 32; + __m256i d0 = _mm256_loadu_si256((const __m256i*)dst_ptr); + __m256i d1 = _mm256_loadu_si256((const __m256i*)(dst_ptr + 16)); + d0 = _mm256_adds_epu16(d0, s0); + d1 = _mm256_adds_epu16(d1, s1); + _mm256_storeu_si256((__m256i*)dst_ptr, d0); + _mm256_storeu_si256((__m256i*)(dst_ptr + 16), d1); + dst_ptr += 32; + src_width -= 32; + } +} +#endif // HAS_SCALEADDROW_AVX2 + +#ifdef HAS_SCALEADDROW_AVX512BW +// Reads 64 bytes and accumulates to 64 shorts at a time, with masked tail. +LIBYUV_TARGET_AVX512BW +void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width) { + while (src_width >= 64) { + __m512i s0 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_ptr)); + __m512i s1 = _mm512_cvtepu8_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32))); + src_ptr += 64; + __m512i d0 = _mm512_loadu_si512((const __m512i*)dst_ptr); + __m512i d1 = _mm512_loadu_si512((const __m512i*)(dst_ptr + 32)); + d0 = _mm512_adds_epu16(d0, s0); + d1 = _mm512_adds_epu16(d1, s1); + _mm512_storeu_si512((__m512i*)dst_ptr, d0); + _mm512_storeu_si512((__m512i*)(dst_ptr + 32), d1); + dst_ptr += 64; + src_width -= 64; + } + if (src_width >= 32) { + __m512i s0 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_ptr)); + src_ptr += 32; + __m512i d0 = _mm512_loadu_si512((const __m512i*)dst_ptr); + d0 = _mm512_adds_epu16(d0, s0); + _mm512_storeu_si512((__m512i*)dst_ptr, d0); + dst_ptr += 32; + src_width -= 32; + } + if (src_width > 0) { + __mmask32 k = (__mmask32)((1ULL << src_width) - 1); + __m512i s0 = _mm512_maskz_cvtepu8_epi16( + k, _mm256_maskz_loadu_epi8(k, (const void*)src_ptr)); + __m512i d0 = _mm512_maskz_loadu_epi16(k, (const void*)dst_ptr); + d0 = _mm512_maskz_adds_epu16(k, d0, s0); + _mm512_mask_storeu_epi16((void*)dst_ptr, k, d0); + } +} +#endif // HAS_SCALEADDROW_AVX512BW + +#ifdef HAS_SCALEADDCOLS_AVX512BW +static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15}; + +LIBYUV_TARGET_AVX512BW +void ScaleAddCols1_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr) { + int boxwidth = dx >> 16; + if (boxwidth < 1) { + boxwidth = 1; + } + int scaleval = 65536 / (boxwidth * boxheight); + src_ptr += (x >> 16); + const __m512i mask_ffff = _mm512_set1_epi32(0x0000ffff); + const __m512i vscale = _mm512_set1_epi32(scaleval); + if (boxwidth <= 4) { + __m512i seq = + _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kSequence0_15)); + seq = _mm512_or_si512(seq, _mm512_slli_epi32(seq, 16)); + __m512i vbw = _mm512_set1_epi16((int16_t)boxwidth); + __m512i base = _mm512_mullo_epi16(seq, vbw); + __m512i c01 = _mm512_set1_epi32(0x00010000); + __m512i c23 = _mm512_set1_epi32(0x00030002); + __m512i idx01 = _mm512_add_epi16(base, c01); + __m512i idx23 = _mm512_add_epi16(base, c23); + __mmask32 k2 = _mm512_cmpgt_epu16_mask(vbw, c01); + __mmask32 k3 = _mm512_cmpgt_epu16_mask(vbw, c23); + int span = boxwidth * 16; + uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1); + __mmask32 k1 = (__mmask32)m64; + __mmask32 k4 = (__mmask32)(m64 >> 32); + while (dst_width >= 16) { + __m512i s0 = _mm512_maskz_loadu_epi16(k1, src_ptr); + __m512i s1 = _mm512_maskz_loadu_epi16(k4, src_ptr + 32); + src_ptr += span; + __m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1); + __m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1); + __m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff), + _mm512_srli_epi32(p01, 16)); + __m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff), + _mm512_srli_epi32(p23, 16)); + __m512i sum = _mm512_add_epi32(sum01, sum23); + sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16); + _mm_storeu_si128((__m128i*)dst_ptr, _mm512_cvtepi32_epi8(sum)); + dst_ptr += 16; + dst_width -= 16; + } + if (dst_width > 0) { + int rem_span = dst_width * boxwidth; + uint64_t rm64 = (rem_span >= 64) ? ~0ULL : ((1ULL << rem_span) - 1); + __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)rm64, src_ptr); + __m512i s1 = + _mm512_maskz_loadu_epi16((__mmask32)(rm64 >> 32), src_ptr + 32); + __m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1); + __m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1); + __m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff), + _mm512_srli_epi32(p01, 16)); + __m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff), + _mm512_srli_epi32(p23, 16)); + __m512i sum = _mm512_add_epi32(sum01, sum23); + sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16); + __mmask16 km = (__mmask16)((1U << dst_width) - 1); + _mm512_mask_cvtepi32_storeu_epi8(dst_ptr, km, sum); + } + } else if (boxwidth <= 8) { + __m512i seq = + _mm512_cvtepu8_epi64(_mm_loadu_si128((const __m128i*)kSequence0_15)); + seq = _mm512_or_si512(seq, _mm512_slli_epi64(seq, 16)); + seq = _mm512_or_si512(seq, _mm512_slli_epi64(seq, 32)); + __m512i vbw = _mm512_set1_epi16((int16_t)boxwidth); + __m512i base = _mm512_mullo_epi16(seq, vbw); + __m512i c03 = _mm512_set1_epi64(0x0003000200010000ULL); + __m512i c47 = _mm512_set1_epi64(0x0007000600050004ULL); + __m512i idx03 = _mm512_add_epi16(base, c03); + __m512i idx47 = _mm512_add_epi16(base, c47); + __mmask32 k3 = _mm512_cmpgt_epu16_mask(vbw, c47); + int span = boxwidth * 8; + uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1); + __mmask32 k1 = (__mmask32)m64; + __mmask32 k4 = (__mmask32)(m64 >> 32); + while (dst_width >= 8) { + __m512i s0 = _mm512_maskz_loadu_epi16(k1, src_ptr); + __m512i s1 = _mm512_maskz_loadu_epi16(k4, src_ptr + 32); + src_ptr += span; + __m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1); + __m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1); + __m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff), + _mm512_srli_epi32(p03, 16)); + __m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff), + _mm512_srli_epi32(p47, 16)); + __m512i sum = _mm512_add_epi32(sum03, sum47); + sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32)); + sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16); + _mm_storel_epi64((__m128i*)dst_ptr, _mm512_cvtepi64_epi8(sum)); + dst_ptr += 8; + dst_width -= 8; + } + if (dst_width > 0) { + int rem_span = dst_width * boxwidth; + uint64_t rm64 = (rem_span >= 64) ? ~0ULL : ((1ULL << rem_span) - 1); + __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)rm64, src_ptr); + __m512i s1 = + _mm512_maskz_loadu_epi16((__mmask32)(rm64 >> 32), src_ptr + 32); + __m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1); + __m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1); + __m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff), + _mm512_srli_epi32(p03, 16)); + __m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff), + _mm512_srli_epi32(p47, 16)); + __m512i sum = _mm512_add_epi32(sum03, sum47); + sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32)); + sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vscale), 16); + __mmask8 km = (__mmask8)((1U << dst_width) - 1); + _mm512_mask_cvtepi64_storeu_epi8(dst_ptr, km, sum); + } + } else { + int rem = boxwidth & 31; + __mmask32 k1 = (__mmask32)((1ULL << rem) - 1); + while (dst_width > 0) { + __m512i acc = _mm512_setzero_si512(); + const uint16_t* p = src_ptr; + src_ptr += boxwidth; + int n = boxwidth; + while (n >= 32) { + __m512i s = _mm512_loadu_si512((const __m512i*)p); + p += 32; + acc = _mm512_add_epi32(acc, + _mm512_add_epi32(_mm512_and_si512(s, mask_ffff), + _mm512_srli_epi32(s, 16))); + n -= 32; + } + if (n > 0) { + __m512i s = _mm512_maskz_loadu_epi16(k1, p); + acc = _mm512_add_epi32(acc, + _mm512_add_epi32(_mm512_and_si512(s, mask_ffff), + _mm512_srli_epi32(s, 16))); + } + uint32_t total = (uint32_t)_mm512_reduce_add_epi32(acc); + *dst_ptr++ = (uint8_t)((total * (uint32_t)scaleval) >> 16); + --dst_width; + } + } +} + +LIBYUV_TARGET_AVX512BW +void ScaleAddCols2_AVX512BW(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_ptr, + uint8_t* dst_ptr) { + int minboxwidth = dx >> 16; + int scale0 = 65536 / ((minboxwidth < 1 ? 1 : minboxwidth) * boxheight); + int scale1 = + 65536 / (((minboxwidth + 1) < 1 ? 1 : (minboxwidth + 1)) * boxheight); + const __m512i mask_ffff = _mm512_set1_epi32(0x0000ffff); + const __m512i vscale0 = _mm512_set1_epi32(scale0); + const __m512i vscale1 = _mm512_set1_epi32(scale1); + if (minboxwidth >= 1 && minboxwidth <= 3) { + __m512i seq = + _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kSequence0_15)); + __m512i vdx = _mm512_set1_epi32(dx); + __m512i step0 = _mm512_mullo_epi32(seq, vdx); + __m512i step1 = _mm512_add_epi32(step0, vdx); + __m512i c01 = _mm512_set1_epi32(0x00010000); + __m512i c23 = _mm512_set1_epi32(0x00030002); + __m512i vminbw = _mm512_set1_epi32(minboxwidth); + while (dst_width >= 16) { + int base_ix = x >> 16; + __m512i xfrac = _mm512_set1_epi32(x & 0xffff); + x += dx * 16; + int span = (x >> 16) - base_ix; + uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1); + __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix); + __m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32), + src_ptr + base_ix + 32); + __m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16); + __m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16); + __m512i bw = _mm512_sub_epi32(ix1, ix0); + __mmask16 k4 = _mm512_cmpgt_epi32_mask(bw, vminbw); + __m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi32(ix0, 16)); + __m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi32(bw, 16)); + __m512i idx01 = _mm512_add_epi16(ix0_16, c01); + __m512i idx23 = _mm512_add_epi16(ix0_16, c23); + __mmask32 k2 = _mm512_cmpgt_epu16_mask(bw_16, c01); + __mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c23); + __m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1); + __m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1); + __m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff), + _mm512_srli_epi32(p01, 16)); + __m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff), + _mm512_srli_epi32(p23, 16)); + __m512i sum = _mm512_add_epi32(sum01, sum23); + __m512i vsc = _mm512_mask_blend_epi32(k4, vscale0, vscale1); + sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16); + _mm_storeu_si128((__m128i*)dst_ptr, _mm512_cvtepi32_epi8(sum)); + dst_ptr += 16; + dst_width -= 16; + } + if (dst_width > 0) { + int base_ix = x >> 16; + __m512i xfrac = _mm512_set1_epi32(x & 0xffff); + int end_x = x + dx * dst_width; + int span = (end_x >> 16) - base_ix; + uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1); + __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix); + __m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32), + src_ptr + base_ix + 32); + __m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16); + __m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16); + __m512i bw = _mm512_sub_epi32(ix1, ix0); + __mmask16 k4 = _mm512_cmpgt_epi32_mask(bw, vminbw); + __m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi32(ix0, 16)); + __m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi32(bw, 16)); + __m512i idx01 = _mm512_add_epi16(ix0_16, c01); + __m512i idx23 = _mm512_add_epi16(ix0_16, c23); + __mmask32 k2 = _mm512_cmpgt_epu16_mask(bw_16, c01); + __mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c23); + __m512i p01 = _mm512_maskz_permutex2var_epi16(k2, s0, idx01, s1); + __m512i p23 = _mm512_maskz_permutex2var_epi16(k3, s0, idx23, s1); + __m512i sum01 = _mm512_add_epi32(_mm512_and_si512(p01, mask_ffff), + _mm512_srli_epi32(p01, 16)); + __m512i sum23 = _mm512_add_epi32(_mm512_and_si512(p23, mask_ffff), + _mm512_srli_epi32(p23, 16)); + __m512i sum = _mm512_add_epi32(sum01, sum23); + __m512i vsc = _mm512_mask_blend_epi32(k4, vscale0, vscale1); + sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16); + __mmask16 km = (__mmask16)((1U << dst_width) - 1); + _mm512_mask_cvtepi32_storeu_epi8(dst_ptr, km, sum); + } + } else if (minboxwidth >= 4 && minboxwidth <= 7) { + __m512i seq = + _mm512_cvtepu8_epi64(_mm_loadu_si128((const __m128i*)kSequence0_15)); + __m512i vdx = _mm512_set1_epi64(dx); + __m512i step0 = _mm512_mullo_epi32(seq, vdx); + __m512i step1 = _mm512_add_epi64(step0, vdx); + __m512i c03 = _mm512_set1_epi64(0x0003000200010000ULL); + __m512i c47 = _mm512_set1_epi64(0x0007000600050004ULL); + __m512i vminbw = _mm512_set1_epi64(minboxwidth); + while (dst_width >= 8) { + int base_ix = x >> 16; + __m512i xfrac = _mm512_set1_epi64(x & 0xffff); + x += dx * 8; + int span = (x >> 16) - base_ix; + uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1); + __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix); + __m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32), + src_ptr + base_ix + 32); + __m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16); + __m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16); + __m512i bw = _mm512_sub_epi32(ix1, ix0); + __mmask8 k4 = _mm512_cmpgt_epi64_mask(bw, vminbw); + __m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi64(ix0, 16)); + ix0_16 = _mm512_or_si512(ix0_16, _mm512_slli_epi64(ix0_16, 32)); + __m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi64(bw, 16)); + bw_16 = _mm512_or_si512(bw_16, _mm512_slli_epi64(bw_16, 32)); + __m512i idx03 = _mm512_add_epi16(ix0_16, c03); + __m512i idx47 = _mm512_add_epi16(ix0_16, c47); + __mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c47); + __m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1); + __m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1); + __m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff), + _mm512_srli_epi32(p03, 16)); + __m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff), + _mm512_srli_epi32(p47, 16)); + __m512i sum = _mm512_add_epi32(sum03, sum47); + sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32)); + __m512i vsc = _mm512_mask_blend_epi64(k4, vscale0, vscale1); + sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16); + _mm_storel_epi64((__m128i*)dst_ptr, _mm512_cvtepi64_epi8(sum)); + dst_ptr += 8; + dst_width -= 8; + } + if (dst_width > 0) { + int base_ix = x >> 16; + __m512i xfrac = _mm512_set1_epi64(x & 0xffff); + int end_x = x + dx * dst_width; + int span = (end_x >> 16) - base_ix; + uint64_t m64 = (span >= 64) ? ~0ULL : ((1ULL << span) - 1); + __m512i s0 = _mm512_maskz_loadu_epi16((__mmask32)m64, src_ptr + base_ix); + __m512i s1 = _mm512_maskz_loadu_epi16((__mmask32)(m64 >> 32), + src_ptr + base_ix + 32); + __m512i ix0 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step0), 16); + __m512i ix1 = _mm512_srli_epi32(_mm512_add_epi32(xfrac, step1), 16); + __m512i bw = _mm512_sub_epi32(ix1, ix0); + __mmask8 k4 = _mm512_cmpgt_epi64_mask(bw, vminbw); + __m512i ix0_16 = _mm512_or_si512(ix0, _mm512_slli_epi64(ix0, 16)); + ix0_16 = _mm512_or_si512(ix0_16, _mm512_slli_epi64(ix0_16, 32)); + __m512i bw_16 = _mm512_or_si512(bw, _mm512_slli_epi64(bw, 16)); + bw_16 = _mm512_or_si512(bw_16, _mm512_slli_epi64(bw_16, 32)); + __m512i idx03 = _mm512_add_epi16(ix0_16, c03); + __m512i idx47 = _mm512_add_epi16(ix0_16, c47); + __mmask32 k3 = _mm512_cmpgt_epu16_mask(bw_16, c47); + __m512i p03 = _mm512_permutex2var_epi16(s0, idx03, s1); + __m512i p47 = _mm512_maskz_permutex2var_epi16(k3, s0, idx47, s1); + __m512i sum03 = _mm512_add_epi32(_mm512_and_si512(p03, mask_ffff), + _mm512_srli_epi32(p03, 16)); + __m512i sum47 = _mm512_add_epi32(_mm512_and_si512(p47, mask_ffff), + _mm512_srli_epi32(p47, 16)); + __m512i sum = _mm512_add_epi32(sum03, sum47); + sum = _mm512_add_epi32(sum, _mm512_srli_epi64(sum, 32)); + __m512i vsc = _mm512_mask_blend_epi64(k4, vscale0, vscale1); + sum = _mm512_srli_epi32(_mm512_mullo_epi32(sum, vsc), 16); + __mmask8 km = (__mmask8)((1U << dst_width) - 1); + _mm512_mask_cvtepi64_storeu_epi8(dst_ptr, km, sum); + } + } else { + while (dst_width > 0) { + int ix = x >> 16; + x += dx; + int bw = (x >> 16) - ix; + if (bw < 1) { + bw = 1; + } + int sc = (bw > minboxwidth) ? scale1 : scale0; + __m512i acc = _mm512_setzero_si512(); + const uint16_t* p = src_ptr + ix; + int n = bw; + while (n >= 32) { + __m512i s = _mm512_loadu_si512((const __m512i*)p); + p += 32; + acc = _mm512_add_epi32(acc, + _mm512_add_epi32(_mm512_and_si512(s, mask_ffff), + _mm512_srli_epi32(s, 16))); + n -= 32; + } + if (n > 0) { + __mmask32 k1 = (__mmask32)((1ULL << n) - 1); + __m512i s = _mm512_maskz_loadu_epi16(k1, p); + acc = _mm512_add_epi32(acc, + _mm512_add_epi32(_mm512_and_si512(s, mask_ffff), + _mm512_srli_epi32(s, 16))); + } + uint32_t total = (uint32_t)_mm512_reduce_add_epi32(acc); + *dst_ptr++ = (uint8_t)((total * (uint32_t)sc) >> 16); + --dst_width; + } + } +} +#endif // HAS_SCALEADDCOLS_AVX512BW + +#ifdef HAS_SCALEUVADDCOLS3_AVX2 +static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5}; +static const ulvec32 kPermUV3_1 = {3, 4, 5, 6, 7, 0, 1, 2}; +static const ulvec32 kPermUV3_2 = {6, 7, 0, 1, 2, 3, 4, 5}; + +// Scale UV columns by 3 box filter. +// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per block. +LIBYUV_TARGET_AVX2 +void ScaleUVAddCols3_AVX2(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + const __m256i vscale = _mm256_set1_epi16((int16_t)scaleval); + const __m256i perm0 = _mm256_loadu_si256((const __m256i*)&kPermUV3_0); + const __m256i perm1 = _mm256_loadu_si256((const __m256i*)&kPermUV3_1); + const __m256i perm2 = _mm256_loadu_si256((const __m256i*)&kPermUV3_2); + while (dst_width >= 16) { + __m256i a0 = _mm256_permutevar8x32_epi32( + _mm256_loadu_si256((const __m256i*)src_uv), perm0); + __m256i a1 = _mm256_permutevar8x32_epi32( + _mm256_loadu_si256((const __m256i*)(src_uv + 16)), perm0); + __m256i a2 = _mm256_permutevar8x32_epi32( + _mm256_loadu_si256((const __m256i*)(src_uv + 32)), perm0); + __m256i c0 = _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x38), a2, 0xc0); + __m256i c1 = _mm256_permutevar8x32_epi32( + _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0xc0), a2, 0x07), perm1); + __m256i c2 = _mm256_permutevar8x32_epi32( + _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x07), a2, 0x38), perm2); + __m256i sum0 = _mm256_mulhi_epu16( + _mm256_add_epi16(_mm256_add_epi16(c0, c1), c2), vscale); + + __m256i b0 = _mm256_permutevar8x32_epi32( + _mm256_loadu_si256((const __m256i*)(src_uv + 48)), perm0); + __m256i b1 = _mm256_permutevar8x32_epi32( + _mm256_loadu_si256((const __m256i*)(src_uv + 64)), perm0); + __m256i b2 = _mm256_permutevar8x32_epi32( + _mm256_loadu_si256((const __m256i*)(src_uv + 80)), perm0); + src_uv += 96; + __m256i d0 = _mm256_blend_epi32(_mm256_blend_epi32(b0, b1, 0x38), b2, 0xc0); + __m256i d1 = _mm256_permutevar8x32_epi32( + _mm256_blend_epi32(_mm256_blend_epi32(b0, b1, 0xc0), b2, 0x07), perm1); + __m256i d2 = _mm256_permutevar8x32_epi32( + _mm256_blend_epi32(_mm256_blend_epi32(b0, b1, 0x07), b2, 0x38), perm2); + __m256i sum1 = _mm256_mulhi_epu16( + _mm256_add_epi16(_mm256_add_epi16(d0, d1), d2), vscale); + + __m256i packed = + _mm256_permute4x64_epi64(_mm256_packus_epi16(sum0, sum1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_uv, packed); + dst_uv += 32; + dst_width -= 16; + } + while (dst_width > 0) { + __m256i a0 = _mm256_permutevar8x32_epi32( + _mm256_loadu_si256((const __m256i*)src_uv), perm0); + __m256i a1 = _mm256_permutevar8x32_epi32( + _mm256_loadu_si256((const __m256i*)(src_uv + 16)), perm0); + __m256i a2 = _mm256_permutevar8x32_epi32( + _mm256_loadu_si256((const __m256i*)(src_uv + 32)), perm0); + src_uv += 48; + __m256i c0 = _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x38), a2, 0xc0); + __m256i c1 = _mm256_permutevar8x32_epi32( + _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0xc0), a2, 0x07), perm1); + __m256i c2 = _mm256_permutevar8x32_epi32( + _mm256_blend_epi32(_mm256_blend_epi32(a0, a1, 0x07), a2, 0x38), perm2); + __m256i sum = _mm256_mulhi_epu16( + _mm256_add_epi16(_mm256_add_epi16(c0, c1), c2), vscale); + __m128i lo = _mm256_castsi256_si128(sum); + __m128i hi = _mm256_extracti128_si256(sum, 1); + _mm_storeu_si128((__m128i*)dst_uv, _mm_packus_epi16(lo, hi)); + dst_uv += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVADDCOLS3_AVX2 + +#ifdef HAS_SCALEUVADDCOLS3_AVX512VBMI +static const uint8_t kPerm512_Col0[16] = {0, 3, 6, 9, 12, 15, 18, 21, + 24, 27, 30, 1, 4, 7, 10, 13}; +static const uint8_t kPerm512_Col1[16] = {1, 4, 7, 10, 13, 16, 19, 22, + 25, 28, 31, 2, 5, 8, 11, 14}; +static const uint8_t kPerm512_Col2[16] = {2, 5, 8, 11, 14, 17, 20, 23, + 26, 29, 0, 3, 6, 9, 12, 15}; +static const uint8_t kPackEvenBytes[64] = { + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, + 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, + 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, + 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, + 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126}; + +// Scale UV columns by 3 box filter using AVX512VBMI. +// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop +// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels). +LIBYUV_TARGET_AVX512VBMI +void ScaleUVAddCols3_AVX512VBMI(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + const __m512i vscale = _mm512_set1_epi16((int16_t)scaleval); + const __m512i col0 = + _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kPerm512_Col0)); + const __m512i col1 = + _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kPerm512_Col1)); + const __m512i col2 = + _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*)kPerm512_Col2)); + const __m512i pack_even = _mm512_loadu_si512((const __m512i*)kPackEvenBytes); + const __mmask16 k5 = 0xf800; + const __mmask16 k6 = 0xfc00; + + while (dst_width >= 32) { + __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv); + __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); + __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64)); + __m512i a0 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2); + __m512i a1 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2); + __m512i a2 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2); + __m512i sum0 = _mm512_mulhi_epu16( + _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale); + + __m512i s3 = _mm512_loadu_si512((const __m512i*)(src_uv + 96)); + __m512i s4 = _mm512_loadu_si512((const __m512i*)(src_uv + 128)); + __m512i s5 = _mm512_loadu_si512((const __m512i*)(src_uv + 160)); + src_uv += 192; + __m512i b0 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s3, col0, s4), k5, col0, s5); + __m512i b1 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s3, col1, s4), k5, col1, s5); + __m512i b2 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s3, col2, s4), k6, col2, s5); + __m512i sum1 = _mm512_mulhi_epu16( + _mm512_add_epi16(_mm512_add_epi16(b0, b1), b2), vscale); + + __m512i out = _mm512_permutex2var_epi8(sum0, pack_even, sum1); + _mm512_storeu_si512((__m512i*)dst_uv, out); + dst_uv += 64; + dst_width -= 32; + } + if (dst_width >= 16) { + __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv); + __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); + __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64)); + src_uv += 96; + __m512i a0 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2); + __m512i a1 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2); + __m512i a2 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2); + __m512i sum0 = _mm512_mulhi_epu16( + _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale); + _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(sum0)); + dst_uv += 32; + dst_width -= 16; + } + if (dst_width > 0) { + uint64_t m64 = (1ULL << (dst_width * 3)) - 1; + __mmask16 k1 = (__mmask16)m64; + __mmask16 k2 = (__mmask16)(m64 >> 16); + __mmask16 k3 = (__mmask16)(m64 >> 32); + __mmask16 k4 = (__mmask16)((1U << dst_width) - 1); + __m512i s0 = _mm512_maskz_loadu_epi32(k1, src_uv); + __m512i s1 = _mm512_maskz_loadu_epi32(k2, src_uv + 32); + __m512i s2 = _mm512_maskz_loadu_epi32(k3, src_uv + 64); + __m512i a0 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2); + __m512i a1 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2); + __m512i a2 = _mm512_mask_permutexvar_epi32( + _mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2); + __m512i sum0 = _mm512_mulhi_epu16( + _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale); + _mm256_mask_storeu_epi16(dst_uv, k4, _mm512_cvtepi16_epi8(sum0)); + } +} +#endif // HAS_SCALEUVADDCOLS3_AVX512VBMI + +#undef LIBYUV_TARGET_SSSE3 +#undef LIBYUV_TARGET_AVX2 +#undef LIBYUV_TARGET_AVX512BW +#undef LIBYUV_TARGET_AVX512VBMI + +#endif // !defined(LIBYUV_DISABLE_X86) && ... + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/unit_test/scale_uv_test.cc b/unit_test/scale_uv_test.cc index 65a7c662f..5a8d160a9 100644 --- a/unit_test/scale_uv_test.cc +++ b/unit_test/scale_uv_test.cc @@ -331,6 +331,10 @@ TEST_BOXPLANE(By3_75, 960, 540, 256, 144, 0) /* 1080p to 512x288 */ TEST_BOXPLANE(By6, 640, 360, 106, 60, 0) /* 720p to 213x120 */ TEST_BOXPLANE(By8, 640, 360, 80, 45, 0) /* 720p to 160x90 */ TEST_BOXPLANE(OddTail, 33, 17, 7, 5, 0) /* not a multiple of 8 wide */ +TEST_BOXPLANE(By3OddTail7, 21, 15, 7, 5, 0) /* 3x, dst_width = 7 */ +TEST_BOXPLANE(By3OddTail23, 69, 15, 23, 5, 0) /* 3x, dst_width = 23 */ +TEST_BOXPLANE(By3OddTail47, 141, 15, 47, 5, 0) /* 3x, dst_width = 47 */ +TEST_BOXPLANE(By3OddTail79, 237, 15, 79, 5, 0) /* 3x, dst_width = 79 */ TEST_BOXPLANE(Tiny, 80, 60, 3, 2, 0) /* dst below one SIMD group */ #undef TEST_BOXPLANE