coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

[libyuv] Multi-platform ScaleUVAddCols3 for ScaleUVBox at 3x

Browse Source 
Based on Philipp Hancke's AVX2 column kernel for ScaleUVBox at 3x
(https://crrev.com/c/8427347).

Add ScaleUVAddCols3 for exact 3x horizontal downscaling (dx == 0x30000
and boxheight <= 85) in ScaleUVBox, plus ScaleAddRow_AVX512BW and
ScaleAddRow_WASMSIMD dispatch, and port 3x/box scaling kernels in
scale_win.cc to 32/64-bit C++ intrinsics matching scale_gcc.cc:
- ScaleUVAddCols3_C (portable C)
- ScaleUVAddCols3_AVX2 and ScaleUVAddCols3_Any_AVX2 (scale_gcc.cc and
  scale_win.cc)
- ScaleUVAddCols3_AVX512VBMI (scale_gcc.cc and scale_win.cc)
- ScaleAddRow_SSE2, ScaleAddRow_AVX2, ScaleAddRow_AVX512BW,
  ScaleAddCols1_AVX512BW, ScaleAddCols2_AVX512BW, ScaleRowDown38_SSSE3,
  ScaleRowDown38_2_Box_SSSE3, and ScaleRowDown38_3_Box_SSSE3 in
  scale_win.cc for 32/64-bit Visual C and LIBYUV_ENABLE_ROWWIN
- ScaleUVAddCols3_NEON (32-bit ARM and 64-bit AArch64) and
  ScaleUVAddCols3_Any_NEON
- ScaleUVAddCols3_SME (ARMv9 Streaming SVE with whilelt predicate tail)
- ScaleUVAddCols3_RVV (RISC-V Vector vlseg3e32.v inline asm)
- ScaleUVAddCols3_LSX and ScaleUVAddCols3_Any_LSX (LoongArch LSX)
- ScaleAddRow_WASMSIMD, ScaleAddRow_Any_WASMSIMD,
  ScaleUVAddCols3_WASMSIMD, and ScaleUVAddCols3_Any_WASMSIMD
- ScaleUVAddCols3_HVX (Hexagon HVX vrdelta + vmpy_VuhRuh)

Co-authored-by: Philipp Hancke <philipp.hancke@googlemail.com>
Bug: libyuv:564129034
R=rrwinterton@gmail.com
Change-Id: Ib9e14f3efcb097842c3d0e9fc9b473b6e2f81871
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8596436
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard 2026-10-16 16:51:17 -07:00
parent 700f153cb8
commit 378bc91324
15 changed files with 1715 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
include/libyuv/scale_row.h
View File

@ -28,7 +28,6 @@ extern "C" {
!defined(LIBYUV_ENABLE_ROWWIN)))
#define HAS_FIXEDDIV1_X86
#define HAS_FIXEDDIV_X86
#define HAS_SCALEADDROW_SSE2
#define HAS_SCALEARGBCOLS_SSE2
#define HAS_SCALEARGBCOLSUP2_SSE2
#define HAS_SCALEARGBFILTERCOLS_SSSE3
@ -38,7 +37,6 @@ extern "C" {
#define HAS_SCALEFILTERCOLS_SSSE3
#define HAS_SCALEROWDOWN2_SSSE3
#define HAS_SCALEROWDOWN34_SSSE3
#define HAS_SCALEROWDOWN38_SSSE3
#define HAS_SCALEROWDOWN4_SSSE3
#endif
@ -47,6 +45,8 @@ extern "C" {
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__)) && \
!defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_SCALEADDROW_SSE2
#define HAS_SCALEROWDOWN38_SSSE3
#define HAS_SCALEUVROWDOWN2BOX_SSSE3
#define HAS_SCALEROWUP2_LINEAR_SSE2
#define HAS_SCALEROWUP2_LINEAR_SSSE3
@ -65,9 +65,12 @@ extern "C" {
// The following are available for gcc/clang x86 platforms, but
// require clang 3.4 or gcc 4.7.
// TODO(fbarchard): Port to Visual C
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__)) && \
(defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2))
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__)) && \
(defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \
!defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_SCALEADDROW_AVX2
#define HAS_SCALEUVADDCOLS3_AVX2
#define HAS_SCALEUVROWDOWN2BOX_AVX2
#define HAS_SCALEROWUP2_LINEAR_AVX2
#define HAS_SCALEROWUP2_BILINEAR_AVX2
@ -87,7 +90,6 @@ extern "C" {
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
defined(GCC_HAS_AVX2))
#define HAS_SCALEADDROW_AVX2
#define HAS_SCALEROWDOWN2_AVX2
#define HAS_SCALEROWDOWN4_AVX2
#endif
@ -98,9 +100,28 @@ extern "C" {
!defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_SCALEADDROW_AVX512BW
#define HAS_SCALEADDCOLS_AVX512BW
#define HAS_SCALEUVADDCOLS3_AVX512VBMI
#define HAS_SCALEROWDOWN2_AVX512BW
#endif
// The following are available for Visual C 32/64 bit, and for clang with
// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer:
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
defined(_M_IX86)) && \
((defined(_MSC_VER) && !defined(__clang__)) || \
defined(LIBYUV_ENABLE_ROWWIN))
#define HAS_SCALEADDROW_SSE2
#define HAS_SCALEROWDOWN38_SSSE3
#define HAS_SCALEADDROW_AVX2
#define HAS_SCALEUVADDCOLS3_AVX2
#if defined(__x86_64__) || defined(_M_X64)
#define HAS_SCALEADDROW_AVX512BW
#define HAS_SCALEADDCOLS_AVX512BW
#define HAS_SCALEUVADDCOLS3_AVX512VBMI
#endif
#endif
// The following are available on Neon platforms:
#if !defined(LIBYUV_DISABLE_NEON) && \
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
@ -114,6 +135,7 @@ extern "C" {
#define HAS_SCALEROWDOWN34_NEON
#define HAS_SCALEROWDOWN38_NEON
#define HAS_SCALEROWDOWN4_NEON
#define HAS_SCALEUVADDCOLS3_NEON
#define HAS_SCALEUVROWDOWN2_NEON
#define HAS_SCALEUVROWDOWN2LINEAR_NEON
#define HAS_SCALEUVROWDOWN2BOX_NEON
@ -141,6 +163,7 @@ extern "C" {
#define HAS_SCALEARGBROWDOWN2_SME
#define HAS_SCALEROWDOWN2_16_SME
#define HAS_SCALEROWDOWN2_SME
#define HAS_SCALEUVADDCOLS3_SME
#define HAS_SCALEUVROWDOWN2BOX_SME
#define HAS_SCALEUVROWDOWN2LINEAR_SME
#define HAS_SCALEUVROWDOWN2_SME
@ -157,6 +180,7 @@ extern "C" {
#define HAS_SCALEARGBCOLS_LSX
#define HAS_SCALEARGBFILTERCOLS_LSX
#define HAS_SCALEROWDOWN34_LSX
#define HAS_SCALEUVADDCOLS3_LSX
#endif
#if !defined(LIBYUV_DISABLE_RVV) && \
@ -192,6 +216,7 @@ extern "C" {
#define HAS_SCALEROWDOWN2LINEAR_RVV
#define HAS_SCALEROWDOWN4_RVV
#define HAS_SCALEROWDOWN4BOX_RVV
#define HAS_SCALEUVADDCOLS3_RVV
#define HAS_SCALEUVROWDOWN2_RVV
#define HAS_SCALEUVROWDOWN2BOX_RVV
#define HAS_SCALEUVROWDOWN2LINEAR_RVV
@ -200,6 +225,7 @@ extern "C" {
// The following are available for WebAssembly SIMD128:
#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
#define HAS_SCALEADDROW_WASMSIMD
#define HAS_SCALEARGBCOLS_WASMSIMD
#define HAS_SCALEARGBCOLSUP2_WASMSIMD
#define HAS_SCALEARGBFILTERCOLS_WASMSIMD
@ -210,6 +236,7 @@ extern "C" {
#define HAS_SCALEROWDOWN34_WASMSIMD
#define HAS_SCALEROWDOWN38_WASMSIMD
#define HAS_SCALEROWDOWN4_WASMSIMD
#define HAS_SCALEUVADDCOLS3_WASMSIMD
#endif
// The following are available for Hexagon HVX:
@ -219,6 +246,7 @@ extern "C" {
#define HAS_SCALEROWDOWN2_16_HVX
#define HAS_SCALEROWDOWN2_HVX
#define HAS_SCALEROWDOWN4_HVX
#define HAS_SCALEUVADDCOLS3_HVX
#define HAS_SCALEUVROWDOWN2_HVX
#define HAS_SCALEUVROWDOWN2BOX_HVX
#define HAS_SCALEUVROWDOWN2LINEAR_HVX
@ -581,6 +609,10 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv,
int dst_width,
int x32,
int dx);
void ScaleUVAddCols3_C(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
// Specialized scalers for x86.
void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
@ -1333,6 +1365,54 @@ void ScaleUVRowUp2_Bilinear_16_Any_NEON(const uint16_t* src_ptr,
uint16_t* dst_ptr,
ptrdiff_t dst_stride,
int dst_width);
void ScaleUVAddCols3_AVX2(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_Any_AVX2(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_AVX512VBMI(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_NEON(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_Any_NEON(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_SME(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_RVV(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_LSX(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_Any_LSX(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_WASMSIMD(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_Any_WASMSIMD(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_HVX(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
// ScaleRowDown2Box also used by planar functions
// NEON/SME downscalers with interpolation.
@ -1815,6 +1895,12 @@ void ScaleRowDown38_2_Box_Any_WASMSIMD(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width);
void ScaleAddRow_WASMSIMD(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);
void ScaleAddRow_Any_WASMSIMD(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);
void ScaleFilterCols_WASMSIMD(uint8_t* dst_ptr,
const uint8_t* src_ptr,
int dst_width,

8
source/scale.cc
View File

@ -1034,6 +1034,14 @@ static int ScalePlaneBox(int src_width,
ScaleAddRow = ScaleAddRow_HVX;
}
#endif
#if defined(HAS_SCALEADDROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASM)) {
ScaleAddRow = ScaleAddRow_Any_WASMSIMD;
if (IS_ALIGNED(src_width, 16)) {
ScaleAddRow = ScaleAddRow_WASMSIMD;
}
}
#endif
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {

34
source/scale_any.cc
View File

@ -622,6 +622,9 @@ SAROW(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, 1, 2, 15)
#ifdef HAS_SCALEADDROW_LSX
SAROW(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, 1, 2, 15)
#endif
#ifdef HAS_SCALEADDROW_WASMSIMD
SAROW(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, 1, 2, 15)
#endif
#undef SAANY
#else
@ -648,6 +651,9 @@ SAANY(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, ScaleAddRow_C, 15)
#ifdef HAS_SCALEADDROW_LSX
SAANY(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, ScaleAddRow_C, 15)
#endif
#ifdef HAS_SCALEADDROW_WASMSIMD
SAANY(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, ScaleAddRow_C, 15)
#endif
#undef SAANY
#endif // SASIMDONLY
@ -1131,6 +1137,34 @@ SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_NEON,
#undef SBU2BLANY
// Scale UV columns by 3 box filter.
#define SUVAC3ANY(NAMEANY, SIMD, C, MASK) \
void NAMEANY(int dst_width, int scaleval, const uint16_t* src_uv, \
uint8_t* dst_uv) { \
int r = dst_width & MASK; \
int n = dst_width & ~MASK; \
if (n > 0) { \
SIMD(n, scaleval, src_uv, dst_uv); \
} \
C(r, scaleval, src_uv + (ptrdiff_t)n * 6, dst_uv + (ptrdiff_t)n * 2); \
}
#ifdef HAS_SCALEUVADDCOLS3_AVX2
SUVAC3ANY(ScaleUVAddCols3_Any_AVX2, ScaleUVAddCols3_AVX2, ScaleUVAddCols3_C, 7)
#endif
#ifdef HAS_SCALEUVADDCOLS3_NEON
SUVAC3ANY(ScaleUVAddCols3_Any_NEON, ScaleUVAddCols3_NEON, ScaleUVAddCols3_C, 7)
#endif
#ifdef HAS_SCALEUVADDCOLS3_LSX
SUVAC3ANY(ScaleUVAddCols3_Any_LSX, ScaleUVAddCols3_LSX, ScaleUVAddCols3_C, 7)
#endif
#ifdef HAS_SCALEUVADDCOLS3_WASMSIMD
SUVAC3ANY(ScaleUVAddCols3_Any_WASMSIMD,
ScaleUVAddCols3_WASMSIMD,
ScaleUVAddCols3_C,
7)
#endif
#undef SUVAC3ANY
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

16
source/scale_common.cc
View File

@ -1366,6 +1366,22 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv,
}
#undef BLENDER1
// Scale UV columns by 3 box filter.
void ScaleUVAddCols3_C(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
int i;
for (i = 0; i < dst_width; ++i) {
uint32_t u = src_uv[0] + src_uv[2] + src_uv[4];
uint32_t v = src_uv[1] + src_uv[3] + src_uv[5];
src_uv += 6;
dst_uv[0] = (uint8_t)((u * scaleval) >> 16);
dst_uv[1] = (uint8_t)((v * scaleval) >> 16);
dst_uv += 2;
}
}
// Scale plane vertically with bilinear interpolation.
void ScalePlaneVertical(int src_height,
int dst_width,

310
source/scale_gcc.cc
View File

@ -3708,6 +3708,316 @@ void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
}
#endif
#ifdef HAS_SCALEUVADDCOLS3_AVX2
static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5};
static const ulvec32 kPermUV3_1 = {3, 4, 5, 6, 7, 0, 1, 2};
static const ulvec32 kPermUV3_2 = {6, 7, 0, 1, 2, 3, 4, 5};
// Scale UV columns by 3 box filter.
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per block.
void ScaleUVAddCols3_AVX2(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
#if defined(__x86_64__)
asm volatile(
"vmovd %k[scaleval],%%xmm7 \n"
"vpbroadcastw %%xmm7,%%ymm7 \n"
"vmovdqa %[kPerm0],%%ymm4 \n"
"vmovdqa %[kPerm1],%%ymm5 \n"
"vmovdqa %[kPerm2],%%ymm6 \n"
"sub $0x10,%[dst_width] \n"
"jl 2f \n"
// Process 16 output UV pixels (192 input bytes -> 32 output bytes).
LABELALIGN
"1: \n"
"vmovdqu (%[src_uv]),%%ymm0 \n"
"vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
"vpaddw %%ymm0,%%ymm3,%%ymm8 \n"
"vpmulhuw %%ymm7,%%ymm8,%%ymm8 \n"
"vmovdqu 0x60(%[src_uv]),%%ymm0 \n"
"vmovdqu 0x80(%[src_uv]),%%ymm1 \n"
"lea 0xc0(%[src_uv]),%[src_uv] \n"
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu -0x20(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
"vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
"vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
"vpackuswb %%ymm3,%%ymm8,%%ymm0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vmovdqu %%ymm0,(%[dst_uv]) \n"
"lea 0x20(%[dst_uv]),%[dst_uv] \n"
"sub $0x10,%[dst_width] \n"
"jge 1b \n"
"2: \n"
"add $0x10,%[dst_width] \n"
"jle 99f \n"
// 8-pixel remainder.
"vmovdqu (%[src_uv]),%%ymm0 \n"
"vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
"vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
"vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
"vextracti128 $1,%%ymm3,%%xmm2 \n"
"vpackuswb %%xmm2,%%xmm3,%%xmm3 \n"
"vmovdqu %%xmm3,(%[dst_uv]) \n"
"99: \n"
"vzeroupper \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width) // %[dst_width]
: [scaleval] "r"(scaleval), // %[scaleval]
[kPerm0] "m"(kPermUV3_0), // %[kPerm0]
[kPerm1] "m"(kPermUV3_1), // %[kPerm1]
[kPerm2] "m"(kPermUV3_2) // %[kPerm2]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "xmm8");
#else // defined(__x86_64__)
asm volatile(
"vmovd %[scaleval],%%xmm7 \n"
"vpbroadcastw %%xmm7,%%ymm7 \n"
"vmovdqa %[kPerm0],%%ymm4 \n"
"vmovdqa %[kPerm1],%%ymm5 \n"
"vmovdqa %[kPerm2],%%ymm6 \n"
LABELALIGN
"1: \n"
"vmovdqu (%[src_uv]),%%ymm0 \n"
"vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
"lea 0x60(%[src_uv]),%[src_uv] \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
"vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
"vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
"vextracti128 $1,%%ymm3,%%xmm2 \n"
"vpackuswb %%xmm2,%%xmm3,%%xmm3 \n"
"vmovdqu %%xmm3,(%[dst_uv]) \n"
"lea 0x10(%[dst_uv]),%[dst_uv] \n"
"sub $0x8,%[dst_width] \n"
"jg 1b \n"
"vzeroupper \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width) // %[dst_width]
: [scaleval] "rm"(scaleval), // %[scaleval]
[kPerm0] "m"(kPermUV3_0), // %[kPerm0]
[kPerm1] "m"(kPermUV3_1), // %[kPerm1]
[kPerm2] "m"(kPermUV3_2) // %[kPerm2]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
#endif // defined(__x86_64__)
}
#endif // HAS_SCALEUVADDCOLS3_AVX2
#ifdef HAS_SCALEUVADDCOLS3_AVX512VBMI
static const uint8_t kPerm512_Col0[16] = {0, 3, 6, 9, 12, 15, 18, 21,
24, 27, 30, 1, 4, 7, 10, 13};
static const uint8_t kPerm512_Col1[16] = {1, 4, 7, 10, 13, 16, 19, 22,
25, 28, 31, 2, 5, 8, 11, 14};
static const uint8_t kPerm512_Col2[16] = {2, 5, 8, 11, 14, 17, 20, 23,
26, 29, 0, 3, 6, 9, 12, 15};
static const uint8_t kPackEvenBytes[64] = {
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24,
26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50,
52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76,
78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102,
104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126};
// Scale UV columns by 3 box filter using AVX512VBMI.
// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop
// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels).
void ScaleUVAddCols3_AVX512VBMI(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
intptr_t temp, mask;
asm volatile(
"vpbroadcastw %k[scaleval],%%zmm20 \n"
"vpmovzxbd %[kCol0],%%zmm16 \n"
"vpmovzxbd %[kCol1],%%zmm17 \n"
"vpmovzxbd %[kCol2],%%zmm18 \n"
"vmovdqu8 %[kPack],%%zmm19 \n"
"mov $0xf800,%k[temp] \n"
"kmovw %k[temp],%%k5 \n"
"mov $0xfc00,%k[temp] \n"
"kmovw %k[temp],%%k6 \n"
"sub $0x20,%[dst_width] \n"
"jl 2f \n"
// Main loop: 32 output UV pixels (64 output bytes) per iteration.
LABELALIGN
"1: \n"
"vmovdqu32 (%[src_uv]),%%zmm0 \n"
"vmovdqu32 0x40(%[src_uv]),%%zmm1 \n"
"vmovdqu32 0x80(%[src_uv]),%%zmm2 \n"
"vmovdqa32 %%zmm16,%%zmm3 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
"vmovdqa32 %%zmm17,%%zmm4 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
"vmovdqa32 %%zmm18,%%zmm5 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
"vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
"vmovdqu32 0xc0(%[src_uv]),%%zmm6 \n"
"vmovdqu32 0x100(%[src_uv]),%%zmm1 \n"
"vmovdqu32 0x140(%[src_uv]),%%zmm2 \n"
"lea 0x180(%[src_uv]),%[src_uv] \n"
"vmovdqa32 %%zmm16,%%zmm3 \n"
"vpermi2d %%zmm1,%%zmm6,%%zmm3 \n"
"vmovdqa32 %%zmm17,%%zmm4 \n"
"vpermi2d %%zmm1,%%zmm6,%%zmm4 \n"
"vmovdqa32 %%zmm18,%%zmm5 \n"
"vpermi2d %%zmm1,%%zmm6,%%zmm5 \n"
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
"vpmulhuw %%zmm20,%%zmm3,%%zmm3 \n"
"vmovdqa64 %%zmm19,%%zmm2 \n"
"vpermi2b %%zmm3,%%zmm0,%%zmm2 \n"
"vmovdqu8 %%zmm2,(%[dst_uv]) \n"
"lea 0x40(%[dst_uv]),%[dst_uv] \n"
"sub $0x20,%[dst_width] \n"
"jge 1b \n"
"2: \n"
"add $0x20,%[dst_width] \n"
"jle 99f \n"
"cmp $0x10,%[dst_width] \n"
"jl 3f \n"
// 16-pixel half-block (32 output bytes).
"vmovdqu32 (%[src_uv]),%%zmm0 \n"
"vmovdqu32 0x40(%[src_uv]),%%zmm1 \n"
"vmovdqu32 0x80(%[src_uv]),%%zmm2 \n"
"lea 0xc0(%[src_uv]),%[src_uv] \n"
"vmovdqa32 %%zmm16,%%zmm3 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
"vmovdqa32 %%zmm17,%%zmm4 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
"vmovdqa32 %%zmm18,%%zmm5 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
"vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
"vpmovwb %%zmm0,(%[dst_uv]) \n"
"lea 0x20(%[dst_uv]),%[dst_uv] \n"
"sub $0x10,%[dst_width] \n"
"jle 99f \n"
// Tail: 1..15 output UV pixels (3..45 input dwords, 2..30 output bytes).
"3: \n"
"lea (%q[dst_width],%q[dst_width],2),%q[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovw %k[mask],%%k1 \n"
"shr $16,%q[mask] \n"
"kmovw %k[mask],%%k2 \n"
"shr $16,%q[mask] \n"
"kmovw %k[mask],%%k3 \n"
"mov $-1,%q[mask] \n"
"bzhi %q[dst_width],%q[mask],%q[mask] \n"
"kmovw %k[mask],%%k4 \n"
"vmovdqu32 (%[src_uv]),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu32 0x40(%[src_uv]),%%zmm1%{%%k2%}%{z%} \n"
"vmovdqu32 0x80(%[src_uv]),%%zmm2%{%%k3%}%{z%} \n"
"vmovdqa32 %%zmm16,%%zmm3 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
"vmovdqa32 %%zmm17,%%zmm4 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
"vmovdqa32 %%zmm18,%%zmm5 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
"vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
"vpmovwb %%zmm0,%%ymm0 \n"
"vmovdqu16 %%ymm0,(%[dst_uv])%{%%k4%} \n"
"99: \n"
"vzeroupper \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width), // %[dst_width]
[temp] "=&r"(temp), // %[temp]
[mask] "=&r"(mask) // %[mask]
: [scaleval] "r"(scaleval), // %[scaleval]
[kCol0] "m"(kPerm512_Col0), // %[kCol0]
[kCol1] "m"(kPerm512_Col1), // %[kCol1]
[kCol2] "m"(kPerm512_Col2), // %[kCol2]
[kPack] "m"(kPackEvenBytes) // %[kPack]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "k1", "k2", "k3", "k4",
"k5", "k6");
}
#endif // HAS_SCALEUVADDCOLS3_AVX512VBMI
#endif // defined(__x86_64__) || defined(__i386__)
#ifdef __cplusplus

95
source/scale_hvx.cc
View File

@ -92,6 +92,101 @@ void ScaleAddRow_HVX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
}
}
// vrdelta control vectors to compact 3 vectors of 3-column UV sums into 1
// vector of 32 UV pairs (64 uint16_t):
// kUVAddCols3P0 routes dwords 0, 3, ..., 30 -> dwords 0..10 (bytes 0..43)
static const uint8_t kUVAddCols3P0[128] __attribute__((aligned(128))) = {
0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x08, 0x08, 0x08, 0x08,
0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08,
0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
};
// kUVAddCols3P1 routes dwords 1, 4, ..., 31 -> dwords 11..21 (bytes 44..87)
static const uint8_t kUVAddCols3P1[128] __attribute__((aligned(128))) = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x10, 0x10, 0x10, 0x10,
0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08,
0x10, 0x10, 0x10, 0x10, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
};
// kUVAddCols3P2 routes dwords 2, 5, ..., 29 -> dwords 22..31 (bytes 88..127)
static const uint8_t kUVAddCols3P2[128] __attribute__((aligned(128))) = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
0x08, 0x08, 0x08, 0x08, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08,
};
void ScaleUVAddCols3_HVX(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
const HVX_Vector p0 = vloada(kUVAddCols3P0);
const HVX_Vector p1 = vloada(kUVAddCols3P1);
const HVX_Vector p2 = vloada(kUVAddCols3P2);
const HVX_VectorPred q44 = Q6_Q_vsetq_R(44);
const HVX_VectorPred q88 = Q6_Q_vsetq_R(88);
const uint32_t scale_splat =
((uint32_t)(uint16_t)scaleval << 16) | (uint16_t)scaleval;
while (dst_width > 0) {
int w = dst_width >= 32 ? 32 : dst_width;
int src_bytes = w * 12;
HVX_Vector v0 =
vloadu_partial_zero(src_uv, src_bytes > 128 ? 128 : src_bytes);
HVX_Vector v1 =
src_bytes > 128
? vloadu_partial_zero((const uint8_t*)src_uv + 128,
src_bytes > 256 ? 128 : src_bytes - 128)
: Q6_V_vzero();
HVX_Vector v2 =
src_bytes > 256
? vloadu_partial_zero((const uint8_t*)src_uv + 256, src_bytes - 256)
: Q6_V_vzero();
HVX_Vector s0 =
Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v0, Q6_V_valign_VVR(v1, v0, 4)),
Q6_V_valign_VVR(v1, v0, 8));
HVX_Vector s1 =
Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v1, Q6_V_valign_VVR(v2, v1, 4)),
Q6_V_valign_VVR(v2, v1, 8));
HVX_Vector s2 =
Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v2, Q6_V_valign_VVR(v2, v2, 4)),
Q6_V_valign_VVR(v2, v2, 8));
HVX_Vector sum = Q6_V_vmux_QVV(
q44, Q6_V_vrdelta_VV(s0, p0),
Q6_V_vmux_QVV(q88, Q6_V_vrdelta_VV(s1, p1), Q6_V_vrdelta_VV(s2, p2)));
HVX_VectorPair prod = Q6_Wuw_vmpy_VuhRuh(sum, scale_splat);
HVX_Vector scaled = Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(prod), Q6_V_lo_W(prod));
HVX_Vector out = Q6_Vub_vpack_VhVh_sat(scaled, scaled);
vstoreu_partial(dst_uv, out, w * 2);
src_uv += 192;
dst_uv += 64;
dst_width -= 32;
}
}
void ScaleRowDown2_HVX(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,

61
source/scale_lsx.cc
View File

@ -581,6 +581,67 @@ void ScaleAddRow_LSX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5");
}
static const uint8_t kScaleUVAddCols3Shuf[] = {
0, 1, 2, 3, 12, 13, 14, 15, 24, 25, 26, 27, 4, 5, 6, 7,
4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31, 8, 9, 10, 11,
8, 9, 10, 11, 20, 21, 22, 23, 0, 1, 2, 3, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 20, 21, 22, 23,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 24, 25, 26, 27,
0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31,
};
void ScaleUVAddCols3_LSX(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
asm volatile(
"vld $vr6, %[shuf], 0 \n"
"vld $vr7, %[shuf], 16 \n"
"vld $vr8, %[shuf], 32 \n"
"vld $vr9, %[shuf], 48 \n"
"vld $vr10, %[shuf], 64 \n"
"vld $vr11, %[shuf], 80 \n"
"vreplgr2vr.h $vr12, %[scaleval] \n"
"1: \n"
"vld $vr0, %[src_uv], 0 \n"
"vld $vr1, %[src_uv], 16 \n"
"vld $vr2, %[src_uv], 32 \n"
"vshuf.b $vr3, $vr1, $vr0, $vr6 \n"
"vshuf.b $vr4, $vr1, $vr0, $vr7 \n"
"vshuf.b $vr5, $vr1, $vr0, $vr8 \n"
"vshuf.b $vr3, $vr2, $vr3, $vr9 \n"
"vshuf.b $vr4, $vr2, $vr4, $vr10 \n"
"vshuf.b $vr5, $vr2, $vr5, $vr11 \n"
"vadd.h $vr3, $vr3, $vr4 \n"
"vadd.h $vr3, $vr3, $vr5 \n"
"vmuh.hu $vr3, $vr3, $vr12 \n"
"vld $vr0, %[src_uv], 48 \n"
"vld $vr1, %[src_uv], 64 \n"
"vld $vr2, %[src_uv], 80 \n"
"addi.d %[src_uv], %[src_uv], 96 \n"
"vshuf.b $vr13, $vr1, $vr0, $vr6 \n"
"vshuf.b $vr4, $vr1, $vr0, $vr7 \n"
"vshuf.b $vr5, $vr1, $vr0, $vr8 \n"
"vshuf.b $vr13, $vr2, $vr13, $vr9 \n"
"vshuf.b $vr4, $vr2, $vr4, $vr10 \n"
"vshuf.b $vr5, $vr2, $vr5, $vr11 \n"
"vadd.h $vr13, $vr13, $vr4 \n"
"vadd.h $vr13, $vr13, $vr5 \n"
"vmuh.hu $vr13, $vr13, $vr12 \n"
"vpickev.b $vr0, $vr13, $vr3 \n"
"vst $vr0, %[dst_uv], 0 \n"
"addi.d %[dst_uv], %[dst_uv], 16 \n"
"addi.w %[dst_width], %[dst_width], -8 \n"
"bgtz %[dst_width], 1b \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width) // %[dst_width]
: [scaleval] "r"(scaleval), // %[scaleval]
[shuf] "r"(kScaleUVAddCols3Shuf)
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
"$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13");
}
void ScaleFilterCols_LSX(uint8_t* dst_ptr,
const uint8_t* src_ptr,
int dst_width,

37
source/scale_neon.cc
View File

@ -1448,6 +1448,43 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
: "memory", "cc", "d0");
}
// Scale UV columns by 3 box filter.
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop.
void ScaleUVAddCols3_NEON(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
asm volatile(
"vdup.16 d16, %[scaleval] \n"
"1: \n"
"vld3.32 {d0, d2, d4}, [%[src_uv]]! \n"
"vld3.32 {d1, d3, d5}, [%[src_uv]]! \n"
"vadd.u16 q0, q0, q1 \n"
"vadd.u16 q0, q0, q2 \n"
"vld3.32 {d2, d4, d6}, [%[src_uv]]! \n"
"vld3.32 {d3, d5, d7}, [%[src_uv]]! \n"
"vadd.u16 q1, q1, q2 \n"
"vadd.u16 q1, q1, q3 \n"
"vmull.u16 q2, d0, d16 \n"
"vmull.u16 q3, d1, d16 \n"
"vmull.u16 q9, d2, d16 \n"
"vmull.u16 q10, d3, d16 \n"
"vshrn.i32 d0, q2, #16 \n"
"vshrn.i32 d1, q3, #16 \n"
"vshrn.i32 d2, q9, #16 \n"
"vshrn.i32 d3, q10, #16 \n"
"vmovn.i16 d0, q0 \n"
"vmovn.i16 d1, q1 \n"
"subs %[dst_width], %[dst_width], #8 \n"
"vst1.8 {q0}, [%[dst_uv]]! \n"
"bgt 1b \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width) // %[dst_width]
: [scaleval] "r"(scaleval) // %[scaleval]
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10");
}
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus

32
source/scale_neon64.cc
View File

@ -1551,6 +1551,38 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
: "memory", "cc", "v0", "v1", "v2", "v3");
}
// Scale UV columns by 3 box filter.
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop.
void ScaleUVAddCols3_NEON(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
asm volatile(
"dup v6.8h, %w[scaleval] \n"
"1: \n"
"ld3 {v0.4s, v1.4s, v2.4s}, [%[src_uv]], #48 \n"
"ld3 {v3.4s, v4.4s, v5.4s}, [%[src_uv]], #48 \n"
"subs %w[dst_width], %w[dst_width], #8 \n"
"add v0.8h, v0.8h, v1.8h \n"
"add v3.8h, v3.8h, v4.8h \n"
"add v0.8h, v0.8h, v2.8h \n"
"add v3.8h, v3.8h, v5.8h \n"
"umull v1.4s, v0.4h, v6.4h \n"
"umull2 v2.4s, v0.8h, v6.8h \n"
"umull v4.4s, v3.4h, v6.4h \n"
"umull2 v5.4s, v3.8h, v6.8h \n"
"uzp2 v0.8h, v1.8h, v2.8h \n"
"uzp2 v3.8h, v4.8h, v5.8h \n"
"uzp1 v0.16b, v0.16b, v3.16b \n"
"st1 {v0.16b}, [%[dst_uv]], #16 \n"
"b.gt 1b \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width) // %[dst_width]
: [scaleval] "r"(scaleval) // %[scaleval]
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
}
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus

36
source/scale_rvv.cc
View File

@ -1304,6 +1304,42 @@ void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
}
#endif
#ifdef HAS_SCALEUVADDCOLS3_RVV
void ScaleUVAddCols3_RVV(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
size_t vl, vl2;
size_t w = (size_t)dst_width;
asm volatile(
"1: \n"
"vsetvli %[vl], %[w], e32, m2, ta, ma \n"
"vlseg3e32.v v8, (%[src_uv]) \n"
"sub %[w], %[w], %[vl] \n"
"slli %[vl2], %[vl], 2 \n"
"add %[src_uv], %[src_uv], %[vl2] \n"
"slli %[vl2], %[vl2], 1 \n"
"add %[src_uv], %[src_uv], %[vl2] \n"
"slli %[vl], %[vl], 1 \n"
"vsetvli zero, %[vl], e16, m2, ta, ma \n"
"vadd.vv v8, v8, v10 \n"
"vadd.vv v8, v8, v12 \n"
"vmulhu.vx v8, v8, %[scaleval] \n"
"vsetvli zero, %[vl], e8, m1, ta, ma \n"
"vnsrl.wi v4, v8, 0 \n"
"vse8.v v4, (%[dst_uv]) \n"
"add %[dst_uv], %[dst_uv], %[vl] \n"
"bgtz %[w], 1b \n"
: [w] "+r"(w), // %[w]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[vl] "=&r"(vl), // %[vl]
[vl2] "=&r"(vl2) // %[vl2]
: [scaleval] "r"(scaleval) // %[scaleval]
: RVV_VL_CLOBBER "memory", "v4", "v8", "v9", "v10", "v11", "v12", "v13");
}
#endif
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

65
source/scale_sme.cc
View File

@ -554,6 +554,71 @@ __arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb,
"p1");
}
// Scale UV columns by 3 box filter using Streaming SVE.
__arm_locally_streaming void ScaleUVAddCols3_SME(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
// Streaming-SVE only, no use of ZA tile.
int vl;
int vl_w;
asm volatile(
"cnth %x[vl] \n"
"cntw %x[vl_w] \n"
"dup z6.h, %w[scaleval] \n"
"ptrue p3.h \n"
"subs %w[dst_width], %w[dst_width], %w[vl] \n"
"b.lt 2f \n"
"ptrue p0.s \n"
"ptrue p1.h \n"
"1: \n"
"ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n"
"incb %[src_uv], all, mul #3 \n"
"ld3w {z3.s, z4.s, z5.s}, p0/z, [%[src_uv]] \n"
"incb %[src_uv], all, mul #3 \n"
"subs %w[dst_width], %w[dst_width], %w[vl] \n"
"add z0.h, z0.h, z1.h \n"
"add z3.h, z3.h, z4.h \n"
"add z0.h, z0.h, z2.h \n"
"add z3.h, z3.h, z5.h \n"
"umulh z0.h, p3/m, z0.h, z6.h \n"
"umulh z3.h, p3/m, z3.h, z6.h \n"
"uzp1 z0.b, z0.b, z3.b \n"
"st1h {z0.h}, p1, [%[dst_uv]] \n"
"incb %[dst_uv] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[dst_width], %w[dst_width], %w[vl] \n"
"b.eq 99f \n"
"whilelt p0.s, wzr, %w[dst_width] \n"
"whilelt p2.s, %w[vl_w], %w[dst_width] \n"
"whilelt p1.h, wzr, %w[dst_width] \n"
"ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n"
"incb %[src_uv], all, mul #3 \n"
"ld3w {z3.s, z4.s, z5.s}, p2/z, [%[src_uv]] \n"
"add z0.h, z0.h, z1.h \n"
"add z3.h, z3.h, z4.h \n"
"add z0.h, z0.h, z2.h \n"
"add z3.h, z3.h, z5.h \n"
"umulh z0.h, p3/m, z0.h, z6.h \n"
"umulh z3.h, p3/m, z3.h, z6.h \n"
"uzp1 z0.b, z0.b, z3.b \n"
"st1h {z0.h}, p1, [%[dst_uv]] \n"
"99: \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=&r"(vl), // %[vl]
[vl_w] "=&r"(vl_w) // %[vl_w]
: [scaleval] "r"(scaleval) // %[scaleval]
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "p0", "p1",
"p2", "p3");
}
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
// defined(__aarch64__)

79
source/scale_uv.cc
View File

@ -346,10 +346,67 @@ static int ScaleUVBox(int src_width,
void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx,
const uint16_t* src_uv, uint8_t* dst_uv) =
(dx & 0xffff) ? ScaleUVAddCols2_C : ScaleUVAddCols1_C;
void (*ScaleUVAddCols3)(int dst_width, int scaleval,
const uint16_t* src_uv, uint8_t* dst_uv) = NULL;
// Summing a row is position wise, so an interleaved UV row can be fed to
// the plane accumulator as src_width * 2 samples.
void (*ScaleAddRow)(const uint8_t* src_ptr, uint16_t* dst_ptr,
int src_width) = ScaleAddRow_C;
if (dx == 0x30000) {
ScaleUVAddCols3 = ScaleUVAddCols3_C;
#if defined(HAS_SCALEUVADDCOLS3_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_AVX2;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_AVX2;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
ScaleUVAddCols3 = ScaleUVAddCols3_AVX512VBMI;
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_NEON;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_SME)
if (TestCpuFlag(kCpuHasSME)) {
ScaleUVAddCols3 = ScaleUVAddCols3_SME;
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleUVAddCols3 = ScaleUVAddCols3_RVV;
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_LSX;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_LSX;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_WASMSIMD)
if (TestCpuFlag(kCpuHasWASM)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_WASMSIMD;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_WASMSIMD;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_HVX)
if (TestCpuFlag(kCpuHasHVX)) {
ScaleUVAddCols3 = ScaleUVAddCols3_HVX;
}
#endif
}
#if defined(HAS_SCALEADDROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ScaleAddRow = ScaleAddRow_Any_SSE2;
@ -366,6 +423,11 @@ static int ScaleUVBox(int src_width,
}
}
#endif
#if defined(HAS_SCALEADDROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ScaleAddRow = ScaleAddRow_AVX512BW;
}
#endif
#if defined(HAS_SCALEADDROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleAddRow = ScaleAddRow_Any_NEON;
@ -392,6 +454,14 @@ static int ScaleUVBox(int src_width,
ScaleAddRow = ScaleAddRow_HVX;
}
#endif
#if defined(HAS_SCALEADDROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASM)) {
ScaleAddRow = ScaleAddRow_Any_WASMSIMD;
if (IS_ALIGNED(row_samples, 16)) {
ScaleAddRow = ScaleAddRow_WASMSIMD;
}
}
#endif
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
@ -408,7 +478,14 @@ static int ScaleUVBox(int src_width,
ScaleAddRow(src, (uint16_t*)(row16), row_samples);
src += src_stride;
}
ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16), dst_uv);
if (ScaleUVAddCols3 && boxheight <= 85) {
ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight),
(const uint16_t*)row16 + (ptrdiff_t)(x >> 16) * 2,
dst_uv);
} else {
ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16),
dst_uv);
}
dst_uv += dst_stride;
}
}

68
source/scale_wasm.cc
View File

@ -637,6 +637,74 @@ void ScaleARGBColsUp2_WASMSIMD(uint8_t* dst_argb,
}
#endif // HAS_SCALEARGBCOLSUP2_WASMSIMD
#ifdef HAS_SCALEADDROW_WASMSIMD
// Reads 16 bytes and accumulates to 16 shorts at a time.
void ScaleAddRow_WASMSIMD(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width) {
while (src_width > 0) {
v128_t s = wasm_v128_load(src_ptr);
v128_t d0 = wasm_v128_load(dst_ptr);
v128_t d1 = wasm_v128_load(dst_ptr + 8);
d0 = wasm_i16x8_add(d0, wasm_u16x8_extend_low_u8x16(s));
d1 = wasm_i16x8_add(d1, wasm_u16x8_extend_high_u8x16(s));
wasm_v128_store(dst_ptr, d0);
wasm_v128_store(dst_ptr + 8, d1);
src_ptr += 16;
dst_ptr += 16;
src_width -= 16;
}
}
#endif // HAS_SCALEADDROW_WASMSIMD
#ifdef HAS_SCALEUVADDCOLS3_WASMSIMD
// Reads 24 UV pixels (48 uint16_t) and writes 8 UV pixels (16 bytes).
void ScaleUVAddCols3_WASMSIMD(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
const v128_t scale = wasm_i16x8_splat((int16_t)scaleval);
while (dst_width > 0) {
v128_t s0 = wasm_v128_load(src_uv);
v128_t s1 = wasm_v128_load(src_uv + 8);
v128_t s2 = wasm_v128_load(src_uv + 16);
v128_t s3 = wasm_v128_load(src_uv + 24);
v128_t s4 = wasm_v128_load(src_uv + 32);
v128_t s5 = wasm_v128_load(src_uv + 40);
src_uv += 48;
// Group 0..3 of 3 UV pairs from s0, s1, s2 (dwords 0..11)
v128_t a0 = wasm_i32x4_shuffle(s0, s1, 0, 3, 6, 1);
v128_t b0 = wasm_i32x4_shuffle(s0, s1, 1, 4, 7, 2);
v128_t c0 = wasm_i32x4_shuffle(s0, s1, 2, 5, 0, 3);
a0 = wasm_i32x4_shuffle(a0, s2, 0, 1, 2, 5);
b0 = wasm_i32x4_shuffle(b0, s2, 0, 1, 2, 6);
c0 = wasm_i32x4_shuffle(c0, s2, 0, 1, 4, 7);
v128_t sum0 = wasm_i16x8_add(wasm_i16x8_add(a0, b0), c0);
// Group 4..7 of 3 UV pairs from s3, s4, s5 (dwords 0..11)
v128_t a1 = wasm_i32x4_shuffle(s3, s4, 0, 3, 6, 1);
v128_t b1 = wasm_i32x4_shuffle(s3, s4, 1, 4, 7, 2);
v128_t c1 = wasm_i32x4_shuffle(s3, s4, 2, 5, 0, 3);
a1 = wasm_i32x4_shuffle(a1, s5, 0, 1, 2, 5);
b1 = wasm_i32x4_shuffle(b1, s5, 0, 1, 2, 6);
c1 = wasm_i32x4_shuffle(c1, s5, 0, 1, 4, 7);
v128_t sum1 = wasm_i16x8_add(wasm_i16x8_add(a1, b1), c1);
// Multiply unsigned 16-bit sums by scaleval and shift right by 16.
v128_t lo0 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum0, scale), 16);
v128_t hi0 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum0, scale), 16);
v128_t lo1 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum1, scale), 16);
v128_t hi1 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum1, scale), 16);
v128_t p0 = wasm_i16x8_narrow_i32x4(lo0, hi0);
v128_t p1 = wasm_i16x8_narrow_i32x4(lo1, hi1);
wasm_v128_store(dst_uv, wasm_u8x16_narrow_i16x8(p0, p1));
dst_uv += 16;
dst_width -= 8;
}
}
#endif // HAS_SCALEUVADDCOLS3_WASMSIMD
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

1019
source/scale_win.cc
View File

File diff suppressed because it is too large Load Diff

4
unit_test/scale_uv_test.cc
View File

@ -331,6 +331,10 @@ TEST_BOXPLANE(By3_75, 960, 540, 256, 144, 0) /* 1080p to 512x288 */
TEST_BOXPLANE(By6, 640, 360, 106, 60, 0) /* 720p to 213x120 */
TEST_BOXPLANE(By8, 640, 360, 80, 45, 0) /* 720p to 160x90 */
TEST_BOXPLANE(OddTail, 33, 17, 7, 5, 0) /* not a multiple of 8 wide */
TEST_BOXPLANE(By3OddTail7, 21, 15, 7, 5, 0) /* 3x, dst_width = 7 */
TEST_BOXPLANE(By3OddTail23, 69, 15, 23, 5, 0) /* 3x, dst_width = 23 */
TEST_BOXPLANE(By3OddTail47, 141, 15, 47, 5, 0) /* 3x, dst_width = 47 */
TEST_BOXPLANE(By3OddTail79, 237, 15, 79, 5, 0) /* 3x, dst_width = 79 */
TEST_BOXPLANE(Tiny, 80, 60, 3, 2, 0) /* dst below one SIMD group */
#undef TEST_BOXPLANE