coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Multi-platform ScaleUVAddCols3 for ScaleUVBox at 3x

Browse Source 
R=rrwinterton@gmail.com

Change-Id: Ib9e14f3efcb097842c3d0e9fc9b473b6e2f81871
This commit is contained in:
Frank Barchard 2026-10-16 21:59:21 -07:00
parent 2404f9730e
commit da1e3b3393
10 changed files with 605 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
include/libyuv/scale_row.h
View File

@ -68,6 +68,7 @@ extern "C" {
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__)) && \
(defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2))
#define HAS_SCALEUVADDCOLS3_AVX2
#define HAS_SCALEUVROWDOWN2BOX_AVX2
#define HAS_SCALEROWUP2_LINEAR_AVX2
#define HAS_SCALEROWUP2_BILINEAR_AVX2
@ -98,6 +99,7 @@ extern "C" {
!defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_SCALEADDROW_AVX512BW
#define HAS_SCALEADDCOLS_AVX512BW
#define HAS_SCALEUVADDCOLS3_AVX512VBMI
#define HAS_SCALEROWDOWN2_AVX512BW
#endif
@ -114,6 +116,7 @@ extern "C" {
#define HAS_SCALEROWDOWN34_NEON
#define HAS_SCALEROWDOWN38_NEON
#define HAS_SCALEROWDOWN4_NEON
#define HAS_SCALEUVADDCOLS3_NEON
#define HAS_SCALEUVROWDOWN2_NEON
#define HAS_SCALEUVROWDOWN2LINEAR_NEON
#define HAS_SCALEUVROWDOWN2BOX_NEON
@ -141,6 +144,7 @@ extern "C" {
#define HAS_SCALEARGBROWDOWN2_SME
#define HAS_SCALEROWDOWN2_16_SME
#define HAS_SCALEROWDOWN2_SME
#define HAS_SCALEUVADDCOLS3_SME
#define HAS_SCALEUVROWDOWN2BOX_SME
#define HAS_SCALEUVROWDOWN2LINEAR_SME
#define HAS_SCALEUVROWDOWN2_SME
@ -192,6 +196,7 @@ extern "C" {
#define HAS_SCALEROWDOWN2LINEAR_RVV
#define HAS_SCALEROWDOWN4_RVV
#define HAS_SCALEROWDOWN4BOX_RVV
#define HAS_SCALEUVADDCOLS3_RVV
#define HAS_SCALEUVROWDOWN2_RVV
#define HAS_SCALEUVROWDOWN2BOX_RVV
#define HAS_SCALEUVROWDOWN2LINEAR_RVV
@ -569,6 +574,10 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv,
int dst_width,
int x32,
int dx);
void ScaleUVAddCols3_C(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
// Specialized scalers for x86.
void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
@ -1321,6 +1330,34 @@ void ScaleUVRowUp2_Bilinear_16_Any_NEON(const uint16_t* src_ptr,
uint16_t* dst_ptr,
ptrdiff_t dst_stride,
int dst_width);
void ScaleUVAddCols3_AVX2(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_Any_AVX2(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_AVX512VBMI(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_NEON(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_Any_NEON(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_SME(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleUVAddCols3_RVV(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
// ScaleRowDown2Box also used by planar functions
// NEON/SME downscalers with interpolation.

19
source/scale_any.cc
View File

@ -1131,6 +1131,25 @@ SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_NEON,
#undef SBU2BLANY
// Scale UV columns by 3 box filter.
#define SUVAC3ANY(NAMEANY, SIMD, C, MASK) \
void NAMEANY(int dst_width, int scaleval, const uint16_t* src_uv, \
uint8_t* dst_uv) { \
int r = dst_width & MASK; \
int n = dst_width & ~MASK; \
if (n > 0) { \
SIMD(n, scaleval, src_uv, dst_uv); \
} \
C(r, scaleval, src_uv + (ptrdiff_t)n * 6, dst_uv + (ptrdiff_t)n * 2); \
}
#ifdef HAS_SCALEUVADDCOLS3_AVX2
SUVAC3ANY(ScaleUVAddCols3_Any_AVX2, ScaleUVAddCols3_AVX2, ScaleUVAddCols3_C, 7)
#endif
#ifdef HAS_SCALEUVADDCOLS3_NEON
SUVAC3ANY(ScaleUVAddCols3_Any_NEON, ScaleUVAddCols3_NEON, ScaleUVAddCols3_C, 7)
#endif
#undef SUVAC3ANY
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

16
source/scale_common.cc
View File

@ -1366,6 +1366,22 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv,
}
#undef BLENDER1
// Scale UV columns by 3 box filter.
void ScaleUVAddCols3_C(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
int i;
for (i = 0; i < dst_width; ++i) {
uint32_t u = src_uv[0] + src_uv[2] + src_uv[4];
uint32_t v = src_uv[1] + src_uv[3] + src_uv[5];
src_uv += 6;
dst_uv[0] = (uint8_t)((u * scaleval) >> 16);
dst_uv[1] = (uint8_t)((v * scaleval) >> 16);
dst_uv += 2;
}
}
// Scale plane vertically with bilinear interpolation.
void ScalePlaneVertical(int src_height,
int dst_width,

310
source/scale_gcc.cc
View File

@ -3708,6 +3708,316 @@ void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
}
#endif
#ifdef HAS_SCALEUVADDCOLS3_AVX2
static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5};
static const ulvec32 kPermUV3_1 = {3, 4, 5, 6, 7, 0, 1, 2};
static const ulvec32 kPermUV3_2 = {6, 7, 0, 1, 2, 3, 4, 5};
// Scale UV columns by 3 box filter.
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per block.
void ScaleUVAddCols3_AVX2(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
#if defined(__x86_64__)
asm volatile(
"vmovd %k[scaleval],%%xmm7 \n"
"vpbroadcastw %%xmm7,%%ymm7 \n"
"vmovdqa %[kPerm0],%%ymm4 \n"
"vmovdqa %[kPerm1],%%ymm5 \n"
"vmovdqa %[kPerm2],%%ymm6 \n"
"sub $0x10,%[dst_width] \n"
"jl 2f \n"
// Process 16 output UV pixels (192 input bytes -> 32 output bytes).
LABELALIGN
"1: \n"
"vmovdqu (%[src_uv]),%%ymm0 \n"
"vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
"vpaddw %%ymm0,%%ymm3,%%ymm8 \n"
"vpmulhuw %%ymm7,%%ymm8,%%ymm8 \n"
"vmovdqu 0x60(%[src_uv]),%%ymm0 \n"
"vmovdqu 0x80(%[src_uv]),%%ymm1 \n"
"lea 0xc0(%[src_uv]),%[src_uv] \n"
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu -0x20(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
"vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
"vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
"vpackuswb %%ymm3,%%ymm8,%%ymm0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vmovdqu %%ymm0,(%[dst_uv]) \n"
"lea 0x20(%[dst_uv]),%[dst_uv] \n"
"sub $0x10,%[dst_width] \n"
"jge 1b \n"
"2: \n"
"add $0x10,%[dst_width] \n"
"jle 99f \n"
// 8-pixel remainder.
"vmovdqu (%[src_uv]),%%ymm0 \n"
"vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
"vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
"vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
"vextracti128 $1,%%ymm3,%%xmm2 \n"
"vpackuswb %%xmm2,%%xmm3,%%xmm3 \n"
"vmovdqu %%xmm3,(%[dst_uv]) \n"
"99: \n"
"vzeroupper \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width) // %[dst_width]
: [scaleval] "r"(scaleval), // %[scaleval]
[kPerm0] "m"(kPermUV3_0), // %[kPerm0]
[kPerm1] "m"(kPermUV3_1), // %[kPerm1]
[kPerm2] "m"(kPermUV3_2) // %[kPerm2]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "xmm8");
#else // defined(__x86_64__)
asm volatile(
"vmovd %[scaleval],%%xmm7 \n"
"vpbroadcastw %%xmm7,%%ymm7 \n"
"vmovdqa %[kPerm0],%%ymm4 \n"
"vmovdqa %[kPerm1],%%ymm5 \n"
"vmovdqa %[kPerm2],%%ymm6 \n"
LABELALIGN
"1: \n"
"vmovdqu (%[src_uv]),%%ymm0 \n"
"vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
"vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
"lea 0x60(%[src_uv]),%[src_uv] \n"
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
"vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
"vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
"vextracti128 $1,%%ymm3,%%xmm2 \n"
"vpackuswb %%xmm2,%%xmm3,%%xmm3 \n"
"vmovdqu %%xmm3,(%[dst_uv]) \n"
"lea 0x10(%[dst_uv]),%[dst_uv] \n"
"sub $0x8,%[dst_width] \n"
"jg 1b \n"
"vzeroupper \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width) // %[dst_width]
: [scaleval] "rm"(scaleval), // %[scaleval]
[kPerm0] "m"(kPermUV3_0), // %[kPerm0]
[kPerm1] "m"(kPermUV3_1), // %[kPerm1]
[kPerm2] "m"(kPermUV3_2) // %[kPerm2]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
#endif // defined(__x86_64__)
}
#endif // HAS_SCALEUVADDCOLS3_AVX2
#ifdef HAS_SCALEUVADDCOLS3_AVX512VBMI
static const uint8_t kPerm512_Col0[16] = {0, 3, 6, 9, 12, 15, 18, 21,
24, 27, 30, 1, 4, 7, 10, 13};
static const uint8_t kPerm512_Col1[16] = {1, 4, 7, 10, 13, 16, 19, 22,
25, 28, 31, 2, 5, 8, 11, 14};
static const uint8_t kPerm512_Col2[16] = {2, 5, 8, 11, 14, 17, 20, 23,
26, 29, 0, 3, 6, 9, 12, 15};
static const uint8_t kPackEvenBytes[64] = {
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24,
26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50,
52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76,
78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102,
104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126};
// Scale UV columns by 3 box filter using AVX512VBMI.
// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop
// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels).
void ScaleUVAddCols3_AVX512VBMI(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
intptr_t temp, mask;
asm volatile(
"vpbroadcastw %k[scaleval],%%zmm20 \n"
"vpmovzxbd %[kCol0],%%zmm16 \n"
"vpmovzxbd %[kCol1],%%zmm17 \n"
"vpmovzxbd %[kCol2],%%zmm18 \n"
"vmovdqu8 %[kPack],%%zmm19 \n"
"mov $0xf800,%k[temp] \n"
"kmovw %k[temp],%%k5 \n"
"mov $0xfc00,%k[temp] \n"
"kmovw %k[temp],%%k6 \n"
"sub $0x20,%[dst_width] \n"
"jl 2f \n"
// Main loop: 32 output UV pixels (64 output bytes) per iteration.
LABELALIGN
"1: \n"
"vmovdqu32 (%[src_uv]),%%zmm0 \n"
"vmovdqu32 0x40(%[src_uv]),%%zmm1 \n"
"vmovdqu32 0x80(%[src_uv]),%%zmm2 \n"
"vmovdqa32 %%zmm16,%%zmm3 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
"vmovdqa32 %%zmm17,%%zmm4 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
"vmovdqa32 %%zmm18,%%zmm5 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
"vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
"vmovdqu32 0xc0(%[src_uv]),%%zmm6 \n"
"vmovdqu32 0x100(%[src_uv]),%%zmm1 \n"
"vmovdqu32 0x140(%[src_uv]),%%zmm2 \n"
"lea 0x180(%[src_uv]),%[src_uv] \n"
"vmovdqa32 %%zmm16,%%zmm3 \n"
"vpermi2d %%zmm1,%%zmm6,%%zmm3 \n"
"vmovdqa32 %%zmm17,%%zmm4 \n"
"vpermi2d %%zmm1,%%zmm6,%%zmm4 \n"
"vmovdqa32 %%zmm18,%%zmm5 \n"
"vpermi2d %%zmm1,%%zmm6,%%zmm5 \n"
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
"vpmulhuw %%zmm20,%%zmm3,%%zmm3 \n"
"vmovdqa64 %%zmm19,%%zmm2 \n"
"vpermi2b %%zmm3,%%zmm0,%%zmm2 \n"
"vmovdqu8 %%zmm2,(%[dst_uv]) \n"
"lea 0x40(%[dst_uv]),%[dst_uv] \n"
"sub $0x20,%[dst_width] \n"
"jge 1b \n"
"2: \n"
"add $0x20,%[dst_width] \n"
"jle 99f \n"
"cmp $0x10,%[dst_width] \n"
"jl 3f \n"
// 16-pixel half-block (32 output bytes).
"vmovdqu32 (%[src_uv]),%%zmm0 \n"
"vmovdqu32 0x40(%[src_uv]),%%zmm1 \n"
"vmovdqu32 0x80(%[src_uv]),%%zmm2 \n"
"lea 0xc0(%[src_uv]),%[src_uv] \n"
"vmovdqa32 %%zmm16,%%zmm3 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
"vmovdqa32 %%zmm17,%%zmm4 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
"vmovdqa32 %%zmm18,%%zmm5 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
"vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
"vpmovwb %%zmm0,(%[dst_uv]) \n"
"lea 0x20(%[dst_uv]),%[dst_uv] \n"
"sub $0x10,%[dst_width] \n"
"jle 99f \n"
// Tail: 1..15 output UV pixels (3..45 input dwords, 2..30 output bytes).
"3: \n"
"lea (%q[dst_width],%q[dst_width],2),%q[temp] \n"
"mov $-1,%q[mask] \n"
"bzhi %q[temp],%q[mask],%q[mask] \n"
"kmovw %k[mask],%%k1 \n"
"shr $16,%q[mask] \n"
"kmovw %k[mask],%%k2 \n"
"shr $16,%q[mask] \n"
"kmovw %k[mask],%%k3 \n"
"mov $-1,%q[mask] \n"
"bzhi %q[dst_width],%q[mask],%q[mask] \n"
"kmovw %k[mask],%%k4 \n"
"vmovdqu32 (%[src_uv]),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu32 0x40(%[src_uv]),%%zmm1%{%%k2%}%{z%} \n"
"vmovdqu32 0x80(%[src_uv]),%%zmm2%{%%k3%}%{z%} \n"
"vmovdqa32 %%zmm16,%%zmm3 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
"vmovdqa32 %%zmm17,%%zmm4 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
"vmovdqa32 %%zmm18,%%zmm5 \n"
"vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
"vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
"vpmovwb %%zmm0,%%ymm0 \n"
"vmovdqu16 %%ymm0,(%[dst_uv])%{%%k4%} \n"
"99: \n"
"vzeroupper \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width), // %[dst_width]
[temp] "=&r"(temp), // %[temp]
[mask] "=&r"(mask) // %[mask]
: [scaleval] "r"(scaleval), // %[scaleval]
[kCol0] "m"(kPerm512_Col0), // %[kCol0]
[kCol1] "m"(kPerm512_Col1), // %[kCol1]
[kCol2] "m"(kPerm512_Col2), // %[kCol2]
[kPack] "m"(kPackEvenBytes) // %[kPack]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "k1", "k2", "k3", "k4",
"k5", "k6");
}
#endif // HAS_SCALEUVADDCOLS3_AVX512VBMI
#endif // defined(__x86_64__) || defined(__i386__)
#ifdef __cplusplus

37
source/scale_neon.cc
View File

@ -1448,6 +1448,43 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
: "memory", "cc", "d0");
}
// Scale UV columns by 3 box filter.
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop.
void ScaleUVAddCols3_NEON(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
asm volatile(
"vdup.16 d16, %[scaleval] \n"
"1: \n"
"vld3.32 {d0, d2, d4}, [%[src_uv]]! \n"
"vld3.32 {d1, d3, d5}, [%[src_uv]]! \n"
"vadd.u16 q0, q0, q1 \n"
"vadd.u16 q0, q0, q2 \n"
"vld3.32 {d2, d4, d6}, [%[src_uv]]! \n"
"vld3.32 {d3, d5, d7}, [%[src_uv]]! \n"
"vadd.u16 q1, q1, q2 \n"
"vadd.u16 q1, q1, q3 \n"
"vmull.u16 q2, d0, d16 \n"
"vmull.u16 q3, d1, d16 \n"
"vmull.u16 q9, d2, d16 \n"
"vmull.u16 q10, d3, d16 \n"
"vshrn.i32 d0, q2, #16 \n"
"vshrn.i32 d1, q3, #16 \n"
"vshrn.i32 d2, q9, #16 \n"
"vshrn.i32 d3, q10, #16 \n"
"vmovn.i16 d0, q0 \n"
"vmovn.i16 d1, q1 \n"
"subs %[dst_width], %[dst_width], #8 \n"
"vst1.8 {q0}, [%[dst_uv]]! \n"
"bgt 1b \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width) // %[dst_width]
: [scaleval] "r"(scaleval) // %[scaleval]
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10");
}
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus

32
source/scale_neon64.cc
View File

@ -1551,6 +1551,38 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
: "memory", "cc", "v0", "v1", "v2", "v3");
}
// Scale UV columns by 3 box filter.
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop.
void ScaleUVAddCols3_NEON(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
asm volatile(
"dup v6.8h, %w[scaleval] \n"
"1: \n"
"ld3 {v0.4s, v1.4s, v2.4s}, [%[src_uv]], #48 \n"
"ld3 {v3.4s, v4.4s, v5.4s}, [%[src_uv]], #48 \n"
"subs %w[dst_width], %w[dst_width], #8 \n"
"add v0.8h, v0.8h, v1.8h \n"
"add v3.8h, v3.8h, v4.8h \n"
"add v0.8h, v0.8h, v2.8h \n"
"add v3.8h, v3.8h, v5.8h \n"
"umull v1.4s, v0.4h, v6.4h \n"
"umull2 v2.4s, v0.8h, v6.8h \n"
"umull v4.4s, v3.4h, v6.4h \n"
"umull2 v5.4s, v3.8h, v6.8h \n"
"uzp2 v0.8h, v1.8h, v2.8h \n"
"uzp2 v3.8h, v4.8h, v5.8h \n"
"uzp1 v0.16b, v0.16b, v3.16b \n"
"st1 {v0.16b}, [%[dst_uv]], #16 \n"
"b.gt 1b \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width) // %[dst_width]
: [scaleval] "r"(scaleval) // %[scaleval]
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
}
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus

36
source/scale_rvv.cc
View File

@ -1304,6 +1304,42 @@ void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
}
#endif
#ifdef HAS_SCALEUVADDCOLS3_RVV
void ScaleUVAddCols3_RVV(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
size_t vl, vl2;
size_t w = (size_t)dst_width;
asm volatile(
"1: \n"
"vsetvli %[vl], %[w], e32, m2, ta, ma \n"
"vlseg3e32.v v8, (%[src_uv]) \n"
"sub %[w], %[w], %[vl] \n"
"slli %[vl2], %[vl], 2 \n"
"add %[src_uv], %[src_uv], %[vl2] \n"
"slli %[vl2], %[vl2], 1 \n"
"add %[src_uv], %[src_uv], %[vl2] \n"
"slli %[vl], %[vl], 1 \n"
"vsetvli zero, %[vl], e16, m2, ta, ma \n"
"vadd.vv v8, v8, v10 \n"
"vadd.vv v8, v8, v12 \n"
"vmulhu.vx v8, v8, %[scaleval] \n"
"vsetvli zero, %[vl], e8, m1, ta, ma \n"
"vnsrl.wi v4, v8, 0 \n"
"vse8.v v4, (%[dst_uv]) \n"
"add %[dst_uv], %[dst_uv], %[vl] \n"
"bgtz %[w], 1b \n"
: [w] "+r"(w), // %[w]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[vl] "=&r"(vl), // %[vl]
[vl2] "=&r"(vl2) // %[vl2]
: [scaleval] "r"(scaleval) // %[scaleval]
: RVV_VL_CLOBBER "memory", "v4", "v8", "v9", "v10", "v11", "v12", "v13");
}
#endif
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

65
source/scale_sme.cc
View File

@ -554,6 +554,71 @@ __arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb,
"p1");
}
// Scale UV columns by 3 box filter using Streaming SVE.
__arm_locally_streaming void ScaleUVAddCols3_SME(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
// Streaming-SVE only, no use of ZA tile.
int vl;
int vl_w;
asm volatile(
"cnth %x[vl] \n"
"cntw %x[vl_w] \n"
"dup z6.h, %w[scaleval] \n"
"ptrue p3.h \n"
"subs %w[dst_width], %w[dst_width], %w[vl] \n"
"b.lt 2f \n"
"ptrue p0.s \n"
"ptrue p1.h \n"
"1: \n"
"ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n"
"incb %[src_uv], all, mul #3 \n"
"ld3w {z3.s, z4.s, z5.s}, p0/z, [%[src_uv]] \n"
"incb %[src_uv], all, mul #3 \n"
"subs %w[dst_width], %w[dst_width], %w[vl] \n"
"add z0.h, z0.h, z1.h \n"
"add z3.h, z3.h, z4.h \n"
"add z0.h, z0.h, z2.h \n"
"add z3.h, z3.h, z5.h \n"
"umulh z0.h, p3/m, z0.h, z6.h \n"
"umulh z3.h, p3/m, z3.h, z6.h \n"
"uzp1 z0.b, z0.b, z3.b \n"
"st1h {z0.h}, p1, [%[dst_uv]] \n"
"incb %[dst_uv] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[dst_width], %w[dst_width], %w[vl] \n"
"b.eq 99f \n"
"whilelt p0.s, wzr, %w[dst_width] \n"
"whilelt p2.s, %w[vl_w], %w[dst_width] \n"
"whilelt p1.h, wzr, %w[dst_width] \n"
"ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n"
"incb %[src_uv], all, mul #3 \n"
"ld3w {z3.s, z4.s, z5.s}, p2/z, [%[src_uv]] \n"
"add z0.h, z0.h, z1.h \n"
"add z3.h, z3.h, z4.h \n"
"add z0.h, z0.h, z2.h \n"
"add z3.h, z3.h, z5.h \n"
"umulh z0.h, p3/m, z0.h, z6.h \n"
"umulh z3.h, p3/m, z3.h, z6.h \n"
"uzp1 z0.b, z0.b, z3.b \n"
"st1h {z0.h}, p1, [%[dst_uv]] \n"
"99: \n"
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=&r"(vl), // %[vl]
[vl_w] "=&r"(vl_w) // %[vl_w]
: [scaleval] "r"(scaleval) // %[scaleval]
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "p0", "p1",
"p2", "p3");
}
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
// defined(__aarch64__)

50
source/scale_uv.cc
View File

@ -332,10 +332,46 @@ static int ScaleUVBox(int src_width,
void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx,
const uint16_t* src_uv, uint8_t* dst_uv) =
(dx & 0xffff) ? ScaleUVAddCols2_C : ScaleUVAddCols1_C;
void (*ScaleUVAddCols3)(int dst_width, int scaleval,
const uint16_t* src_uv, uint8_t* dst_uv) = NULL;
// Summing a row is position wise, so an interleaved UV row can be fed to
// the plane accumulator as src_width * 2 samples.
void (*ScaleAddRow)(const uint8_t* src_ptr, uint16_t* dst_ptr,
int src_width) = ScaleAddRow_C;
if (dx == 0x30000) {
ScaleUVAddCols3 = ScaleUVAddCols3_C;
#if defined(HAS_SCALEUVADDCOLS3_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_AVX2;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_AVX2;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
ScaleUVAddCols3 = ScaleUVAddCols3_AVX512VBMI;
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_NEON;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_SME)
if (TestCpuFlag(kCpuHasSME)) {
ScaleUVAddCols3 = ScaleUVAddCols3_SME;
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleUVAddCols3 = ScaleUVAddCols3_RVV;
}
#endif
}
#if defined(HAS_SCALEADDROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ScaleAddRow = ScaleAddRow_Any_SSE2;
@ -352,6 +388,11 @@ static int ScaleUVBox(int src_width,
}
}
#endif
#if defined(HAS_SCALEADDROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ScaleAddRow = ScaleAddRow_AVX512BW;
}
#endif
#if defined(HAS_SCALEADDROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleAddRow = ScaleAddRow_Any_NEON;
@ -389,7 +430,14 @@ static int ScaleUVBox(int src_width,
ScaleAddRow(src, (uint16_t*)(row16), row_samples);
src += src_stride;
}
ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16), dst_uv);
if (ScaleUVAddCols3 && boxheight <= 85) {
ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight),
(const uint16_t*)row16 + (ptrdiff_t)(x >> 16) * 2,
dst_uv);
} else {
ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16),
dst_uv);
}
dst_uv += dst_stride;
}
}

4
unit_test/scale_uv_test.cc
View File

@ -331,6 +331,10 @@ TEST_BOXPLANE(By3_75, 960, 540, 256, 144, 0) /* 1080p to 512x288 */
TEST_BOXPLANE(By6, 640, 360, 106, 60, 0) /* 720p to 213x120 */
TEST_BOXPLANE(By8, 640, 360, 80, 45, 0) /* 720p to 160x90 */
TEST_BOXPLANE(OddTail, 33, 17, 7, 5, 0) /* not a multiple of 8 wide */
TEST_BOXPLANE(By3OddTail7, 21, 15, 7, 5, 0) /* 3x, dst_width = 7 */
TEST_BOXPLANE(By3OddTail23, 69, 15, 23, 5, 0) /* 3x, dst_width = 23 */
TEST_BOXPLANE(By3OddTail47, 141, 15, 47, 5, 0) /* 3x, dst_width = 47 */
TEST_BOXPLANE(By3OddTail79, 237, 15, 79, 5, 0) /* 3x, dst_width = 79 */
TEST_BOXPLANE(Tiny, 80, 60, 3, 2, 0) /* dst below one SIMD group */
#undef TEST_BOXPLANE