mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
[libyuv] Multi-platform ScaleUVAddCols3 for ScaleUVBox at 3x
Based on Philipp Hancke's AVX2 column kernel for ScaleUVBox at 3x (https://crrev.com/c/8427347). Add ScaleUVAddCols3 for exact 3x horizontal downscaling (dx == 0x30000 and boxheight <= 85) in ScaleUVBox, plus ScaleAddRow_AVX512BW and ScaleAddRow_WASMSIMD dispatch, and port 3x/box scaling kernels in scale_win.cc to 32/64-bit C++ intrinsics matching scale_gcc.cc: - ScaleUVAddCols3_C (portable C) - ScaleUVAddCols3_AVX2 and ScaleUVAddCols3_Any_AVX2 (scale_gcc.cc and scale_win.cc) - ScaleUVAddCols3_AVX512VBMI (scale_gcc.cc and scale_win.cc) - ScaleAddRow_SSE2, ScaleAddRow_AVX2, ScaleAddRow_AVX512BW, ScaleAddCols1_AVX512BW, ScaleAddCols2_AVX512BW, ScaleRowDown38_SSSE3, ScaleRowDown38_2_Box_SSSE3, and ScaleRowDown38_3_Box_SSSE3 in scale_win.cc for 32/64-bit Visual C and LIBYUV_ENABLE_ROWWIN - ScaleUVAddCols3_NEON (32-bit ARM and 64-bit AArch64) and ScaleUVAddCols3_Any_NEON - ScaleUVAddCols3_SME (ARMv9 Streaming SVE with whilelt predicate tail) - ScaleUVAddCols3_RVV (RISC-V Vector vlseg3e32.v inline asm) - ScaleUVAddCols3_LSX and ScaleUVAddCols3_Any_LSX (LoongArch LSX) - ScaleAddRow_WASMSIMD, ScaleAddRow_Any_WASMSIMD, ScaleUVAddCols3_WASMSIMD, and ScaleUVAddCols3_Any_WASMSIMD - ScaleUVAddCols3_HVX (Hexagon HVX vrdelta + vmpy_VuhRuh) Co-authored-by: Philipp Hancke <philipp.hancke@googlemail.com> Bug: libyuv:564129034 R=rrwinterton@gmail.com Change-Id: Ib9e14f3efcb097842c3d0e9fc9b473b6e2f81871 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8596436 Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard
parent
700f153cb8
commit
378bc91324
@ -28,7 +28,6 @@ extern "C" {
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)))
|
||||
#define HAS_FIXEDDIV1_X86
|
||||
#define HAS_FIXEDDIV_X86
|
||||
#define HAS_SCALEADDROW_SSE2
|
||||
#define HAS_SCALEARGBCOLS_SSE2
|
||||
#define HAS_SCALEARGBCOLSUP2_SSE2
|
||||
#define HAS_SCALEARGBFILTERCOLS_SSSE3
|
||||
@ -38,7 +37,6 @@ extern "C" {
|
||||
#define HAS_SCALEFILTERCOLS_SSSE3
|
||||
#define HAS_SCALEROWDOWN2_SSSE3
|
||||
#define HAS_SCALEROWDOWN34_SSSE3
|
||||
#define HAS_SCALEROWDOWN38_SSSE3
|
||||
#define HAS_SCALEROWDOWN4_SSSE3
|
||||
#endif
|
||||
|
||||
@ -47,6 +45,8 @@ extern "C" {
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||
#define HAS_SCALEADDROW_SSE2
|
||||
#define HAS_SCALEROWDOWN38_SSSE3
|
||||
#define HAS_SCALEUVROWDOWN2BOX_SSSE3
|
||||
#define HAS_SCALEROWUP2_LINEAR_SSE2
|
||||
#define HAS_SCALEROWUP2_LINEAR_SSSE3
|
||||
@ -65,9 +65,12 @@ extern "C" {
|
||||
// The following are available for gcc/clang x86 platforms, but
|
||||
// require clang 3.4 or gcc 4.7.
|
||||
// TODO(fbarchard): Port to Visual C
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
(defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2))
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
(defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||
#define HAS_SCALEADDROW_AVX2
|
||||
#define HAS_SCALEUVADDCOLS3_AVX2
|
||||
#define HAS_SCALEUVROWDOWN2BOX_AVX2
|
||||
#define HAS_SCALEROWUP2_LINEAR_AVX2
|
||||
#define HAS_SCALEROWUP2_BILINEAR_AVX2
|
||||
@ -87,7 +90,6 @@ extern "C" {
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
|
||||
defined(GCC_HAS_AVX2))
|
||||
#define HAS_SCALEADDROW_AVX2
|
||||
#define HAS_SCALEROWDOWN2_AVX2
|
||||
#define HAS_SCALEROWDOWN4_AVX2
|
||||
#endif
|
||||
@ -98,9 +100,28 @@ extern "C" {
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||
#define HAS_SCALEADDROW_AVX512BW
|
||||
#define HAS_SCALEADDCOLS_AVX512BW
|
||||
#define HAS_SCALEUVADDCOLS3_AVX512VBMI
|
||||
#define HAS_SCALEROWDOWN2_AVX512BW
|
||||
#endif
|
||||
|
||||
// The following are available for Visual C 32/64 bit, and for clang with
|
||||
// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
||||
defined(_M_IX86)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
#define HAS_SCALEADDROW_SSE2
|
||||
#define HAS_SCALEROWDOWN38_SSSE3
|
||||
#define HAS_SCALEADDROW_AVX2
|
||||
#define HAS_SCALEUVADDCOLS3_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define HAS_SCALEADDROW_AVX512BW
|
||||
#define HAS_SCALEADDCOLS_AVX512BW
|
||||
#define HAS_SCALEUVADDCOLS3_AVX512VBMI
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// The following are available on Neon platforms:
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && \
|
||||
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
|
||||
@ -114,6 +135,7 @@ extern "C" {
|
||||
#define HAS_SCALEROWDOWN34_NEON
|
||||
#define HAS_SCALEROWDOWN38_NEON
|
||||
#define HAS_SCALEROWDOWN4_NEON
|
||||
#define HAS_SCALEUVADDCOLS3_NEON
|
||||
#define HAS_SCALEUVROWDOWN2_NEON
|
||||
#define HAS_SCALEUVROWDOWN2LINEAR_NEON
|
||||
#define HAS_SCALEUVROWDOWN2BOX_NEON
|
||||
@ -141,6 +163,7 @@ extern "C" {
|
||||
#define HAS_SCALEARGBROWDOWN2_SME
|
||||
#define HAS_SCALEROWDOWN2_16_SME
|
||||
#define HAS_SCALEROWDOWN2_SME
|
||||
#define HAS_SCALEUVADDCOLS3_SME
|
||||
#define HAS_SCALEUVROWDOWN2BOX_SME
|
||||
#define HAS_SCALEUVROWDOWN2LINEAR_SME
|
||||
#define HAS_SCALEUVROWDOWN2_SME
|
||||
@ -157,6 +180,7 @@ extern "C" {
|
||||
#define HAS_SCALEARGBCOLS_LSX
|
||||
#define HAS_SCALEARGBFILTERCOLS_LSX
|
||||
#define HAS_SCALEROWDOWN34_LSX
|
||||
#define HAS_SCALEUVADDCOLS3_LSX
|
||||
#endif
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_RVV) && \
|
||||
@ -192,6 +216,7 @@ extern "C" {
|
||||
#define HAS_SCALEROWDOWN2LINEAR_RVV
|
||||
#define HAS_SCALEROWDOWN4_RVV
|
||||
#define HAS_SCALEROWDOWN4BOX_RVV
|
||||
#define HAS_SCALEUVADDCOLS3_RVV
|
||||
#define HAS_SCALEUVROWDOWN2_RVV
|
||||
#define HAS_SCALEUVROWDOWN2BOX_RVV
|
||||
#define HAS_SCALEUVROWDOWN2LINEAR_RVV
|
||||
@ -200,6 +225,7 @@ extern "C" {
|
||||
|
||||
// The following are available for WebAssembly SIMD128:
|
||||
#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
|
||||
#define HAS_SCALEADDROW_WASMSIMD
|
||||
#define HAS_SCALEARGBCOLS_WASMSIMD
|
||||
#define HAS_SCALEARGBCOLSUP2_WASMSIMD
|
||||
#define HAS_SCALEARGBFILTERCOLS_WASMSIMD
|
||||
@ -210,6 +236,7 @@ extern "C" {
|
||||
#define HAS_SCALEROWDOWN34_WASMSIMD
|
||||
#define HAS_SCALEROWDOWN38_WASMSIMD
|
||||
#define HAS_SCALEROWDOWN4_WASMSIMD
|
||||
#define HAS_SCALEUVADDCOLS3_WASMSIMD
|
||||
#endif
|
||||
|
||||
// The following are available for Hexagon HVX:
|
||||
@ -219,6 +246,7 @@ extern "C" {
|
||||
#define HAS_SCALEROWDOWN2_16_HVX
|
||||
#define HAS_SCALEROWDOWN2_HVX
|
||||
#define HAS_SCALEROWDOWN4_HVX
|
||||
#define HAS_SCALEUVADDCOLS3_HVX
|
||||
#define HAS_SCALEUVROWDOWN2_HVX
|
||||
#define HAS_SCALEUVROWDOWN2BOX_HVX
|
||||
#define HAS_SCALEUVROWDOWN2LINEAR_HVX
|
||||
@ -581,6 +609,10 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv,
|
||||
int dst_width,
|
||||
int x32,
|
||||
int dx);
|
||||
void ScaleUVAddCols3_C(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
|
||||
// Specialized scalers for x86.
|
||||
void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
|
||||
@ -1333,6 +1365,54 @@ void ScaleUVRowUp2_Bilinear_16_Any_NEON(const uint16_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
ptrdiff_t dst_stride,
|
||||
int dst_width);
|
||||
void ScaleUVAddCols3_AVX2(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_Any_AVX2(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_AVX512VBMI(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_NEON(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_Any_NEON(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_SME(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_RVV(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_LSX(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_Any_LSX(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_WASMSIMD(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_Any_WASMSIMD(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_HVX(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
|
||||
// ScaleRowDown2Box also used by planar functions
|
||||
// NEON/SME downscalers with interpolation.
|
||||
@ -1815,6 +1895,12 @@ void ScaleRowDown38_2_Box_Any_WASMSIMD(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleAddRow_WASMSIMD(const uint8_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
int src_width);
|
||||
void ScaleAddRow_Any_WASMSIMD(const uint8_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
int src_width);
|
||||
void ScaleFilterCols_WASMSIMD(uint8_t* dst_ptr,
|
||||
const uint8_t* src_ptr,
|
||||
int dst_width,
|
||||
|
||||
@ -1034,6 +1034,14 @@ static int ScalePlaneBox(int src_width,
|
||||
ScaleAddRow = ScaleAddRow_HVX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEADDROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASM)) {
|
||||
ScaleAddRow = ScaleAddRow_Any_WASMSIMD;
|
||||
if (IS_ALIGNED(src_width, 16)) {
|
||||
ScaleAddRow = ScaleAddRow_WASMSIMD;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
uint32_t uy = y;
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
|
||||
@ -622,6 +622,9 @@ SAROW(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, 1, 2, 15)
|
||||
#ifdef HAS_SCALEADDROW_LSX
|
||||
SAROW(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, 1, 2, 15)
|
||||
#endif
|
||||
#ifdef HAS_SCALEADDROW_WASMSIMD
|
||||
SAROW(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, 1, 2, 15)
|
||||
#endif
|
||||
#undef SAANY
|
||||
|
||||
#else
|
||||
@ -648,6 +651,9 @@ SAANY(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, ScaleAddRow_C, 15)
|
||||
#ifdef HAS_SCALEADDROW_LSX
|
||||
SAANY(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, ScaleAddRow_C, 15)
|
||||
#endif
|
||||
#ifdef HAS_SCALEADDROW_WASMSIMD
|
||||
SAANY(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, ScaleAddRow_C, 15)
|
||||
#endif
|
||||
#undef SAANY
|
||||
|
||||
#endif // SASIMDONLY
|
||||
@ -1131,6 +1137,34 @@ SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_NEON,
|
||||
|
||||
#undef SBU2BLANY
|
||||
|
||||
// Scale UV columns by 3 box filter.
|
||||
#define SUVAC3ANY(NAMEANY, SIMD, C, MASK) \
|
||||
void NAMEANY(int dst_width, int scaleval, const uint16_t* src_uv, \
|
||||
uint8_t* dst_uv) { \
|
||||
int r = dst_width & MASK; \
|
||||
int n = dst_width & ~MASK; \
|
||||
if (n > 0) { \
|
||||
SIMD(n, scaleval, src_uv, dst_uv); \
|
||||
} \
|
||||
C(r, scaleval, src_uv + (ptrdiff_t)n * 6, dst_uv + (ptrdiff_t)n * 2); \
|
||||
}
|
||||
#ifdef HAS_SCALEUVADDCOLS3_AVX2
|
||||
SUVAC3ANY(ScaleUVAddCols3_Any_AVX2, ScaleUVAddCols3_AVX2, ScaleUVAddCols3_C, 7)
|
||||
#endif
|
||||
#ifdef HAS_SCALEUVADDCOLS3_NEON
|
||||
SUVAC3ANY(ScaleUVAddCols3_Any_NEON, ScaleUVAddCols3_NEON, ScaleUVAddCols3_C, 7)
|
||||
#endif
|
||||
#ifdef HAS_SCALEUVADDCOLS3_LSX
|
||||
SUVAC3ANY(ScaleUVAddCols3_Any_LSX, ScaleUVAddCols3_LSX, ScaleUVAddCols3_C, 7)
|
||||
#endif
|
||||
#ifdef HAS_SCALEUVADDCOLS3_WASMSIMD
|
||||
SUVAC3ANY(ScaleUVAddCols3_Any_WASMSIMD,
|
||||
ScaleUVAddCols3_WASMSIMD,
|
||||
ScaleUVAddCols3_C,
|
||||
7)
|
||||
#endif
|
||||
#undef SUVAC3ANY
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -1366,6 +1366,22 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv,
|
||||
}
|
||||
#undef BLENDER1
|
||||
|
||||
// Scale UV columns by 3 box filter.
|
||||
void ScaleUVAddCols3_C(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
int i;
|
||||
for (i = 0; i < dst_width; ++i) {
|
||||
uint32_t u = src_uv[0] + src_uv[2] + src_uv[4];
|
||||
uint32_t v = src_uv[1] + src_uv[3] + src_uv[5];
|
||||
src_uv += 6;
|
||||
dst_uv[0] = (uint8_t)((u * scaleval) >> 16);
|
||||
dst_uv[1] = (uint8_t)((v * scaleval) >> 16);
|
||||
dst_uv += 2;
|
||||
}
|
||||
}
|
||||
|
||||
// Scale plane vertically with bilinear interpolation.
|
||||
void ScalePlaneVertical(int src_height,
|
||||
int dst_width,
|
||||
|
||||
@ -3708,6 +3708,316 @@ void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr,
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAS_SCALEUVADDCOLS3_AVX2
|
||||
static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5};
|
||||
static const ulvec32 kPermUV3_1 = {3, 4, 5, 6, 7, 0, 1, 2};
|
||||
static const ulvec32 kPermUV3_2 = {6, 7, 0, 1, 2, 3, 4, 5};
|
||||
|
||||
// Scale UV columns by 3 box filter.
|
||||
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per block.
|
||||
void ScaleUVAddCols3_AVX2(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
#if defined(__x86_64__)
|
||||
asm volatile(
|
||||
"vmovd %k[scaleval],%%xmm7 \n"
|
||||
"vpbroadcastw %%xmm7,%%ymm7 \n"
|
||||
"vmovdqa %[kPerm0],%%ymm4 \n"
|
||||
"vmovdqa %[kPerm1],%%ymm5 \n"
|
||||
"vmovdqa %[kPerm2],%%ymm6 \n"
|
||||
"sub $0x10,%[dst_width] \n"
|
||||
"jl 2f \n"
|
||||
|
||||
// Process 16 output UV pixels (192 input bytes -> 32 output bytes).
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vmovdqu (%[src_uv]),%%ymm0 \n"
|
||||
"vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
|
||||
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
|
||||
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
|
||||
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
|
||||
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
|
||||
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
|
||||
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
|
||||
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
|
||||
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
|
||||
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
|
||||
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
|
||||
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
|
||||
"vpaddw %%ymm0,%%ymm3,%%ymm8 \n"
|
||||
"vpmulhuw %%ymm7,%%ymm8,%%ymm8 \n"
|
||||
|
||||
"vmovdqu 0x60(%[src_uv]),%%ymm0 \n"
|
||||
"vmovdqu 0x80(%[src_uv]),%%ymm1 \n"
|
||||
"lea 0xc0(%[src_uv]),%[src_uv] \n"
|
||||
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
|
||||
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
|
||||
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
|
||||
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
|
||||
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vmovdqu -0x20(%[src_uv]),%%ymm1 \n"
|
||||
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
|
||||
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
|
||||
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
|
||||
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
|
||||
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
|
||||
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
|
||||
"vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
|
||||
"vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
|
||||
|
||||
"vpackuswb %%ymm3,%%ymm8,%%ymm0 \n"
|
||||
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
|
||||
"vmovdqu %%ymm0,(%[dst_uv]) \n"
|
||||
"lea 0x20(%[dst_uv]),%[dst_uv] \n"
|
||||
"sub $0x10,%[dst_width] \n"
|
||||
"jge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"add $0x10,%[dst_width] \n"
|
||||
"jle 99f \n"
|
||||
|
||||
// 8-pixel remainder.
|
||||
"vmovdqu (%[src_uv]),%%ymm0 \n"
|
||||
"vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
|
||||
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
|
||||
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
|
||||
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
|
||||
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
|
||||
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
|
||||
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
|
||||
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
|
||||
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
|
||||
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
|
||||
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
|
||||
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
|
||||
"vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
|
||||
"vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
|
||||
"vextracti128 $1,%%ymm3,%%xmm2 \n"
|
||||
"vpackuswb %%xmm2,%%xmm3,%%xmm3 \n"
|
||||
"vmovdqu %%xmm3,(%[dst_uv]) \n"
|
||||
|
||||
"99: \n"
|
||||
"vzeroupper \n"
|
||||
: [src_uv] "+r"(src_uv), // %[src_uv]
|
||||
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
||||
[dst_width] "+r"(dst_width) // %[dst_width]
|
||||
: [scaleval] "r"(scaleval), // %[scaleval]
|
||||
[kPerm0] "m"(kPermUV3_0), // %[kPerm0]
|
||||
[kPerm1] "m"(kPermUV3_1), // %[kPerm1]
|
||||
[kPerm2] "m"(kPermUV3_2) // %[kPerm2]
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||
"xmm7", "xmm8");
|
||||
#else // defined(__x86_64__)
|
||||
asm volatile(
|
||||
"vmovd %[scaleval],%%xmm7 \n"
|
||||
"vpbroadcastw %%xmm7,%%ymm7 \n"
|
||||
"vmovdqa %[kPerm0],%%ymm4 \n"
|
||||
"vmovdqa %[kPerm1],%%ymm5 \n"
|
||||
"vmovdqa %[kPerm2],%%ymm6 \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vmovdqu (%[src_uv]),%%ymm0 \n"
|
||||
"vmovdqu 0x20(%[src_uv]),%%ymm1 \n"
|
||||
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
|
||||
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
|
||||
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm3 \n"
|
||||
"vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm2 \n"
|
||||
"vpblendd $0x07,%%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vmovdqu 0x40(%[src_uv]),%%ymm1 \n"
|
||||
"lea 0x60(%[src_uv]),%[src_uv] \n"
|
||||
"vpermd %%ymm1,%%ymm4,%%ymm1 \n"
|
||||
"vpblendd $0xc0,%%ymm1,%%ymm3,%%ymm3 \n"
|
||||
"vpblendd $0x07,%%ymm1,%%ymm2,%%ymm2 \n"
|
||||
"vpblendd $0x38,%%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vpermd %%ymm2,%%ymm5,%%ymm2 \n"
|
||||
"vpermd %%ymm0,%%ymm6,%%ymm0 \n"
|
||||
"vpaddw %%ymm2,%%ymm3,%%ymm3 \n"
|
||||
"vpaddw %%ymm0,%%ymm3,%%ymm3 \n"
|
||||
"vpmulhuw %%ymm7,%%ymm3,%%ymm3 \n"
|
||||
"vextracti128 $1,%%ymm3,%%xmm2 \n"
|
||||
"vpackuswb %%xmm2,%%xmm3,%%xmm3 \n"
|
||||
"vmovdqu %%xmm3,(%[dst_uv]) \n"
|
||||
"lea 0x10(%[dst_uv]),%[dst_uv] \n"
|
||||
"sub $0x8,%[dst_width] \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
: [src_uv] "+r"(src_uv), // %[src_uv]
|
||||
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
||||
[dst_width] "+r"(dst_width) // %[dst_width]
|
||||
: [scaleval] "rm"(scaleval), // %[scaleval]
|
||||
[kPerm0] "m"(kPermUV3_0), // %[kPerm0]
|
||||
[kPerm1] "m"(kPermUV3_1), // %[kPerm1]
|
||||
[kPerm2] "m"(kPermUV3_2) // %[kPerm2]
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||
"xmm7");
|
||||
#endif // defined(__x86_64__)
|
||||
}
|
||||
#endif // HAS_SCALEUVADDCOLS3_AVX2
|
||||
|
||||
#ifdef HAS_SCALEUVADDCOLS3_AVX512VBMI
|
||||
static const uint8_t kPerm512_Col0[16] = {0, 3, 6, 9, 12, 15, 18, 21,
|
||||
24, 27, 30, 1, 4, 7, 10, 13};
|
||||
static const uint8_t kPerm512_Col1[16] = {1, 4, 7, 10, 13, 16, 19, 22,
|
||||
25, 28, 31, 2, 5, 8, 11, 14};
|
||||
static const uint8_t kPerm512_Col2[16] = {2, 5, 8, 11, 14, 17, 20, 23,
|
||||
26, 29, 0, 3, 6, 9, 12, 15};
|
||||
static const uint8_t kPackEvenBytes[64] = {
|
||||
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24,
|
||||
26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50,
|
||||
52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76,
|
||||
78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102,
|
||||
104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126};
|
||||
|
||||
// Scale UV columns by 3 box filter using AVX512VBMI.
|
||||
// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop
|
||||
// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels).
|
||||
void ScaleUVAddCols3_AVX512VBMI(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
intptr_t temp, mask;
|
||||
asm volatile(
|
||||
"vpbroadcastw %k[scaleval],%%zmm20 \n"
|
||||
"vpmovzxbd %[kCol0],%%zmm16 \n"
|
||||
"vpmovzxbd %[kCol1],%%zmm17 \n"
|
||||
"vpmovzxbd %[kCol2],%%zmm18 \n"
|
||||
"vmovdqu8 %[kPack],%%zmm19 \n"
|
||||
"mov $0xf800,%k[temp] \n"
|
||||
"kmovw %k[temp],%%k5 \n"
|
||||
"mov $0xfc00,%k[temp] \n"
|
||||
"kmovw %k[temp],%%k6 \n"
|
||||
|
||||
"sub $0x20,%[dst_width] \n"
|
||||
"jl 2f \n"
|
||||
|
||||
// Main loop: 32 output UV pixels (64 output bytes) per iteration.
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vmovdqu32 (%[src_uv]),%%zmm0 \n"
|
||||
"vmovdqu32 0x40(%[src_uv]),%%zmm1 \n"
|
||||
"vmovdqu32 0x80(%[src_uv]),%%zmm2 \n"
|
||||
"vmovdqa32 %%zmm16,%%zmm3 \n"
|
||||
"vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
|
||||
"vmovdqa32 %%zmm17,%%zmm4 \n"
|
||||
"vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
|
||||
"vmovdqa32 %%zmm18,%%zmm5 \n"
|
||||
"vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
|
||||
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
|
||||
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
|
||||
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
|
||||
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
|
||||
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
|
||||
"vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
|
||||
|
||||
"vmovdqu32 0xc0(%[src_uv]),%%zmm6 \n"
|
||||
"vmovdqu32 0x100(%[src_uv]),%%zmm1 \n"
|
||||
"vmovdqu32 0x140(%[src_uv]),%%zmm2 \n"
|
||||
"lea 0x180(%[src_uv]),%[src_uv] \n"
|
||||
"vmovdqa32 %%zmm16,%%zmm3 \n"
|
||||
"vpermi2d %%zmm1,%%zmm6,%%zmm3 \n"
|
||||
"vmovdqa32 %%zmm17,%%zmm4 \n"
|
||||
"vpermi2d %%zmm1,%%zmm6,%%zmm4 \n"
|
||||
"vmovdqa32 %%zmm18,%%zmm5 \n"
|
||||
"vpermi2d %%zmm1,%%zmm6,%%zmm5 \n"
|
||||
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
|
||||
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
|
||||
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
|
||||
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
|
||||
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
|
||||
"vpmulhuw %%zmm20,%%zmm3,%%zmm3 \n"
|
||||
|
||||
"vmovdqa64 %%zmm19,%%zmm2 \n"
|
||||
"vpermi2b %%zmm3,%%zmm0,%%zmm2 \n"
|
||||
"vmovdqu8 %%zmm2,(%[dst_uv]) \n"
|
||||
"lea 0x40(%[dst_uv]),%[dst_uv] \n"
|
||||
"sub $0x20,%[dst_width] \n"
|
||||
"jge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"add $0x20,%[dst_width] \n"
|
||||
"jle 99f \n"
|
||||
"cmp $0x10,%[dst_width] \n"
|
||||
"jl 3f \n"
|
||||
|
||||
// 16-pixel half-block (32 output bytes).
|
||||
"vmovdqu32 (%[src_uv]),%%zmm0 \n"
|
||||
"vmovdqu32 0x40(%[src_uv]),%%zmm1 \n"
|
||||
"vmovdqu32 0x80(%[src_uv]),%%zmm2 \n"
|
||||
"lea 0xc0(%[src_uv]),%[src_uv] \n"
|
||||
"vmovdqa32 %%zmm16,%%zmm3 \n"
|
||||
"vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
|
||||
"vmovdqa32 %%zmm17,%%zmm4 \n"
|
||||
"vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
|
||||
"vmovdqa32 %%zmm18,%%zmm5 \n"
|
||||
"vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
|
||||
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
|
||||
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
|
||||
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
|
||||
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
|
||||
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
|
||||
"vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
|
||||
"vpmovwb %%zmm0,(%[dst_uv]) \n"
|
||||
"lea 0x20(%[dst_uv]),%[dst_uv] \n"
|
||||
"sub $0x10,%[dst_width] \n"
|
||||
"jle 99f \n"
|
||||
|
||||
// Tail: 1..15 output UV pixels (3..45 input dwords, 2..30 output bytes).
|
||||
"3: \n"
|
||||
"lea (%q[dst_width],%q[dst_width],2),%q[temp] \n"
|
||||
"mov $-1,%q[mask] \n"
|
||||
"bzhi %q[temp],%q[mask],%q[mask] \n"
|
||||
"kmovw %k[mask],%%k1 \n"
|
||||
"shr $16,%q[mask] \n"
|
||||
"kmovw %k[mask],%%k2 \n"
|
||||
"shr $16,%q[mask] \n"
|
||||
"kmovw %k[mask],%%k3 \n"
|
||||
"mov $-1,%q[mask] \n"
|
||||
"bzhi %q[dst_width],%q[mask],%q[mask] \n"
|
||||
"kmovw %k[mask],%%k4 \n"
|
||||
|
||||
"vmovdqu32 (%[src_uv]),%%zmm0%{%%k1%}%{z%} \n"
|
||||
"vmovdqu32 0x40(%[src_uv]),%%zmm1%{%%k2%}%{z%} \n"
|
||||
"vmovdqu32 0x80(%[src_uv]),%%zmm2%{%%k3%}%{z%} \n"
|
||||
"vmovdqa32 %%zmm16,%%zmm3 \n"
|
||||
"vpermi2d %%zmm1,%%zmm0,%%zmm3 \n"
|
||||
"vmovdqa32 %%zmm17,%%zmm4 \n"
|
||||
"vpermi2d %%zmm1,%%zmm0,%%zmm4 \n"
|
||||
"vmovdqa32 %%zmm18,%%zmm5 \n"
|
||||
"vpermi2d %%zmm1,%%zmm0,%%zmm5 \n"
|
||||
"vpermd %%zmm2,%%zmm16,%%zmm3%{%%k5%} \n"
|
||||
"vpermd %%zmm2,%%zmm17,%%zmm4%{%%k5%} \n"
|
||||
"vpermd %%zmm2,%%zmm18,%%zmm5%{%%k6%} \n"
|
||||
"vpaddw %%zmm4,%%zmm3,%%zmm3 \n"
|
||||
"vpaddw %%zmm5,%%zmm3,%%zmm3 \n"
|
||||
"vpmulhuw %%zmm20,%%zmm3,%%zmm0 \n"
|
||||
"vpmovwb %%zmm0,%%ymm0 \n"
|
||||
"vmovdqu16 %%ymm0,(%[dst_uv])%{%%k4%} \n"
|
||||
|
||||
"99: \n"
|
||||
"vzeroupper \n"
|
||||
: [src_uv] "+r"(src_uv), // %[src_uv]
|
||||
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
||||
[dst_width] "+r"(dst_width), // %[dst_width]
|
||||
[temp] "=&r"(temp), // %[temp]
|
||||
[mask] "=&r"(mask) // %[mask]
|
||||
: [scaleval] "r"(scaleval), // %[scaleval]
|
||||
[kCol0] "m"(kPerm512_Col0), // %[kCol0]
|
||||
[kCol1] "m"(kPerm512_Col1), // %[kCol1]
|
||||
[kCol2] "m"(kPerm512_Col2), // %[kCol2]
|
||||
[kPack] "m"(kPackEvenBytes) // %[kPack]
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||
"xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "k1", "k2", "k3", "k4",
|
||||
"k5", "k6");
|
||||
}
|
||||
#endif // HAS_SCALEUVADDCOLS3_AVX512VBMI
|
||||
|
||||
#endif // defined(__x86_64__) || defined(__i386__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -92,6 +92,101 @@ void ScaleAddRow_HVX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
|
||||
}
|
||||
}
|
||||
|
||||
// vrdelta control vectors to compact 3 vectors of 3-column UV sums into 1
|
||||
// vector of 32 UV pairs (64 uint16_t):
|
||||
// kUVAddCols3P0 routes dwords 0, 3, ..., 30 -> dwords 0..10 (bytes 0..43)
|
||||
static const uint8_t kUVAddCols3P0[128] __attribute__((aligned(128))) = {
|
||||
0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10,
|
||||
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x08, 0x08, 0x08, 0x08,
|
||||
0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08,
|
||||
0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
};
|
||||
// kUVAddCols3P1 routes dwords 1, 4, ..., 31 -> dwords 11..21 (bytes 44..87)
|
||||
static const uint8_t kUVAddCols3P1[128] __attribute__((aligned(128))) = {
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20,
|
||||
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x10, 0x10, 0x10, 0x10,
|
||||
0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08,
|
||||
0x10, 0x10, 0x10, 0x10, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
|
||||
0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
};
|
||||
// kUVAddCols3P2 routes dwords 2, 5, ..., 29 -> dwords 22..31 (bytes 88..127)
|
||||
static const uint8_t kUVAddCols3P2[128] __attribute__((aligned(128))) = {
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
|
||||
0x08, 0x08, 0x08, 0x08, 0x10, 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
|
||||
0x08, 0x08, 0x08, 0x08, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
|
||||
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
|
||||
0x10, 0x10, 0x10, 0x10, 0x08, 0x08, 0x08, 0x08,
|
||||
};
|
||||
|
||||
void ScaleUVAddCols3_HVX(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
const HVX_Vector p0 = vloada(kUVAddCols3P0);
|
||||
const HVX_Vector p1 = vloada(kUVAddCols3P1);
|
||||
const HVX_Vector p2 = vloada(kUVAddCols3P2);
|
||||
const HVX_VectorPred q44 = Q6_Q_vsetq_R(44);
|
||||
const HVX_VectorPred q88 = Q6_Q_vsetq_R(88);
|
||||
const uint32_t scale_splat =
|
||||
((uint32_t)(uint16_t)scaleval << 16) | (uint16_t)scaleval;
|
||||
while (dst_width > 0) {
|
||||
int w = dst_width >= 32 ? 32 : dst_width;
|
||||
int src_bytes = w * 12;
|
||||
HVX_Vector v0 =
|
||||
vloadu_partial_zero(src_uv, src_bytes > 128 ? 128 : src_bytes);
|
||||
HVX_Vector v1 =
|
||||
src_bytes > 128
|
||||
? vloadu_partial_zero((const uint8_t*)src_uv + 128,
|
||||
src_bytes > 256 ? 128 : src_bytes - 128)
|
||||
: Q6_V_vzero();
|
||||
HVX_Vector v2 =
|
||||
src_bytes > 256
|
||||
? vloadu_partial_zero((const uint8_t*)src_uv + 256, src_bytes - 256)
|
||||
: Q6_V_vzero();
|
||||
|
||||
HVX_Vector s0 =
|
||||
Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v0, Q6_V_valign_VVR(v1, v0, 4)),
|
||||
Q6_V_valign_VVR(v1, v0, 8));
|
||||
HVX_Vector s1 =
|
||||
Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v1, Q6_V_valign_VVR(v2, v1, 4)),
|
||||
Q6_V_valign_VVR(v2, v1, 8));
|
||||
HVX_Vector s2 =
|
||||
Q6_Vh_vadd_VhVh(Q6_Vh_vadd_VhVh(v2, Q6_V_valign_VVR(v2, v2, 4)),
|
||||
Q6_V_valign_VVR(v2, v2, 8));
|
||||
|
||||
HVX_Vector sum = Q6_V_vmux_QVV(
|
||||
q44, Q6_V_vrdelta_VV(s0, p0),
|
||||
Q6_V_vmux_QVV(q88, Q6_V_vrdelta_VV(s1, p1), Q6_V_vrdelta_VV(s2, p2)));
|
||||
|
||||
HVX_VectorPair prod = Q6_Wuw_vmpy_VuhRuh(sum, scale_splat);
|
||||
HVX_Vector scaled = Q6_Vh_vshuffo_VhVh(Q6_V_hi_W(prod), Q6_V_lo_W(prod));
|
||||
HVX_Vector out = Q6_Vub_vpack_VhVh_sat(scaled, scaled);
|
||||
vstoreu_partial(dst_uv, out, w * 2);
|
||||
src_uv += 192;
|
||||
dst_uv += 64;
|
||||
dst_width -= 32;
|
||||
}
|
||||
}
|
||||
|
||||
void ScaleRowDown2_HVX(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst,
|
||||
|
||||
@ -581,6 +581,67 @@ void ScaleAddRow_LSX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
|
||||
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5");
|
||||
}
|
||||
|
||||
static const uint8_t kScaleUVAddCols3Shuf[] = {
|
||||
0, 1, 2, 3, 12, 13, 14, 15, 24, 25, 26, 27, 4, 5, 6, 7,
|
||||
4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31, 8, 9, 10, 11,
|
||||
8, 9, 10, 11, 20, 21, 22, 23, 0, 1, 2, 3, 12, 13, 14, 15,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 20, 21, 22, 23,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 24, 25, 26, 27,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31,
|
||||
};
|
||||
|
||||
void ScaleUVAddCols3_LSX(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
asm volatile(
|
||||
"vld $vr6, %[shuf], 0 \n"
|
||||
"vld $vr7, %[shuf], 16 \n"
|
||||
"vld $vr8, %[shuf], 32 \n"
|
||||
"vld $vr9, %[shuf], 48 \n"
|
||||
"vld $vr10, %[shuf], 64 \n"
|
||||
"vld $vr11, %[shuf], 80 \n"
|
||||
"vreplgr2vr.h $vr12, %[scaleval] \n"
|
||||
"1: \n"
|
||||
"vld $vr0, %[src_uv], 0 \n"
|
||||
"vld $vr1, %[src_uv], 16 \n"
|
||||
"vld $vr2, %[src_uv], 32 \n"
|
||||
"vshuf.b $vr3, $vr1, $vr0, $vr6 \n"
|
||||
"vshuf.b $vr4, $vr1, $vr0, $vr7 \n"
|
||||
"vshuf.b $vr5, $vr1, $vr0, $vr8 \n"
|
||||
"vshuf.b $vr3, $vr2, $vr3, $vr9 \n"
|
||||
"vshuf.b $vr4, $vr2, $vr4, $vr10 \n"
|
||||
"vshuf.b $vr5, $vr2, $vr5, $vr11 \n"
|
||||
"vadd.h $vr3, $vr3, $vr4 \n"
|
||||
"vadd.h $vr3, $vr3, $vr5 \n"
|
||||
"vmuh.hu $vr3, $vr3, $vr12 \n"
|
||||
"vld $vr0, %[src_uv], 48 \n"
|
||||
"vld $vr1, %[src_uv], 64 \n"
|
||||
"vld $vr2, %[src_uv], 80 \n"
|
||||
"addi.d %[src_uv], %[src_uv], 96 \n"
|
||||
"vshuf.b $vr13, $vr1, $vr0, $vr6 \n"
|
||||
"vshuf.b $vr4, $vr1, $vr0, $vr7 \n"
|
||||
"vshuf.b $vr5, $vr1, $vr0, $vr8 \n"
|
||||
"vshuf.b $vr13, $vr2, $vr13, $vr9 \n"
|
||||
"vshuf.b $vr4, $vr2, $vr4, $vr10 \n"
|
||||
"vshuf.b $vr5, $vr2, $vr5, $vr11 \n"
|
||||
"vadd.h $vr13, $vr13, $vr4 \n"
|
||||
"vadd.h $vr13, $vr13, $vr5 \n"
|
||||
"vmuh.hu $vr13, $vr13, $vr12 \n"
|
||||
"vpickev.b $vr0, $vr13, $vr3 \n"
|
||||
"vst $vr0, %[dst_uv], 0 \n"
|
||||
"addi.d %[dst_uv], %[dst_uv], 16 \n"
|
||||
"addi.w %[dst_width], %[dst_width], -8 \n"
|
||||
"bgtz %[dst_width], 1b \n"
|
||||
: [src_uv] "+r"(src_uv), // %[src_uv]
|
||||
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
||||
[dst_width] "+r"(dst_width) // %[dst_width]
|
||||
: [scaleval] "r"(scaleval), // %[scaleval]
|
||||
[shuf] "r"(kScaleUVAddCols3Shuf)
|
||||
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
|
||||
"$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13");
|
||||
}
|
||||
|
||||
void ScaleFilterCols_LSX(uint8_t* dst_ptr,
|
||||
const uint8_t* src_ptr,
|
||||
int dst_width,
|
||||
|
||||
@ -1448,6 +1448,43 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
|
||||
: "memory", "cc", "d0");
|
||||
}
|
||||
|
||||
// Scale UV columns by 3 box filter.
|
||||
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop.
|
||||
void ScaleUVAddCols3_NEON(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
asm volatile(
|
||||
"vdup.16 d16, %[scaleval] \n"
|
||||
"1: \n"
|
||||
"vld3.32 {d0, d2, d4}, [%[src_uv]]! \n"
|
||||
"vld3.32 {d1, d3, d5}, [%[src_uv]]! \n"
|
||||
"vadd.u16 q0, q0, q1 \n"
|
||||
"vadd.u16 q0, q0, q2 \n"
|
||||
"vld3.32 {d2, d4, d6}, [%[src_uv]]! \n"
|
||||
"vld3.32 {d3, d5, d7}, [%[src_uv]]! \n"
|
||||
"vadd.u16 q1, q1, q2 \n"
|
||||
"vadd.u16 q1, q1, q3 \n"
|
||||
"vmull.u16 q2, d0, d16 \n"
|
||||
"vmull.u16 q3, d1, d16 \n"
|
||||
"vmull.u16 q9, d2, d16 \n"
|
||||
"vmull.u16 q10, d3, d16 \n"
|
||||
"vshrn.i32 d0, q2, #16 \n"
|
||||
"vshrn.i32 d1, q3, #16 \n"
|
||||
"vshrn.i32 d2, q9, #16 \n"
|
||||
"vshrn.i32 d3, q10, #16 \n"
|
||||
"vmovn.i16 d0, q0 \n"
|
||||
"vmovn.i16 d1, q1 \n"
|
||||
"subs %[dst_width], %[dst_width], #8 \n"
|
||||
"vst1.8 {q0}, [%[dst_uv]]! \n"
|
||||
"bgt 1b \n"
|
||||
: [src_uv] "+r"(src_uv), // %[src_uv]
|
||||
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
||||
[dst_width] "+r"(dst_width) // %[dst_width]
|
||||
: [scaleval] "r"(scaleval) // %[scaleval]
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10");
|
||||
}
|
||||
|
||||
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -1551,6 +1551,38 @@ void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3");
|
||||
}
|
||||
|
||||
// Scale UV columns by 3 box filter.
|
||||
// Reads 96 bytes (24 UV pixels) and writes 16 bytes (8 UV pixels) per loop.
|
||||
void ScaleUVAddCols3_NEON(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
asm volatile(
|
||||
"dup v6.8h, %w[scaleval] \n"
|
||||
"1: \n"
|
||||
"ld3 {v0.4s, v1.4s, v2.4s}, [%[src_uv]], #48 \n"
|
||||
"ld3 {v3.4s, v4.4s, v5.4s}, [%[src_uv]], #48 \n"
|
||||
"subs %w[dst_width], %w[dst_width], #8 \n"
|
||||
"add v0.8h, v0.8h, v1.8h \n"
|
||||
"add v3.8h, v3.8h, v4.8h \n"
|
||||
"add v0.8h, v0.8h, v2.8h \n"
|
||||
"add v3.8h, v3.8h, v5.8h \n"
|
||||
"umull v1.4s, v0.4h, v6.4h \n"
|
||||
"umull2 v2.4s, v0.8h, v6.8h \n"
|
||||
"umull v4.4s, v3.4h, v6.4h \n"
|
||||
"umull2 v5.4s, v3.8h, v6.8h \n"
|
||||
"uzp2 v0.8h, v1.8h, v2.8h \n"
|
||||
"uzp2 v3.8h, v4.8h, v5.8h \n"
|
||||
"uzp1 v0.16b, v0.16b, v3.16b \n"
|
||||
"st1 {v0.16b}, [%[dst_uv]], #16 \n"
|
||||
"b.gt 1b \n"
|
||||
: [src_uv] "+r"(src_uv), // %[src_uv]
|
||||
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
||||
[dst_width] "+r"(dst_width) // %[dst_width]
|
||||
: [scaleval] "r"(scaleval) // %[scaleval]
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -1304,6 +1304,42 @@ void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAS_SCALEUVADDCOLS3_RVV
|
||||
void ScaleUVAddCols3_RVV(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
size_t vl, vl2;
|
||||
size_t w = (size_t)dst_width;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"vsetvli %[vl], %[w], e32, m2, ta, ma \n"
|
||||
"vlseg3e32.v v8, (%[src_uv]) \n"
|
||||
"sub %[w], %[w], %[vl] \n"
|
||||
"slli %[vl2], %[vl], 2 \n"
|
||||
"add %[src_uv], %[src_uv], %[vl2] \n"
|
||||
"slli %[vl2], %[vl2], 1 \n"
|
||||
"add %[src_uv], %[src_uv], %[vl2] \n"
|
||||
"slli %[vl], %[vl], 1 \n"
|
||||
"vsetvli zero, %[vl], e16, m2, ta, ma \n"
|
||||
"vadd.vv v8, v8, v10 \n"
|
||||
"vadd.vv v8, v8, v12 \n"
|
||||
"vmulhu.vx v8, v8, %[scaleval] \n"
|
||||
"vsetvli zero, %[vl], e8, m1, ta, ma \n"
|
||||
"vnsrl.wi v4, v8, 0 \n"
|
||||
"vse8.v v4, (%[dst_uv]) \n"
|
||||
"add %[dst_uv], %[dst_uv], %[vl] \n"
|
||||
"bgtz %[w], 1b \n"
|
||||
: [w] "+r"(w), // %[w]
|
||||
[src_uv] "+r"(src_uv), // %[src_uv]
|
||||
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
||||
[vl] "=&r"(vl), // %[vl]
|
||||
[vl2] "=&r"(vl2) // %[vl2]
|
||||
: [scaleval] "r"(scaleval) // %[scaleval]
|
||||
: RVV_VL_CLOBBER "memory", "v4", "v8", "v9", "v10", "v11", "v12", "v13");
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -554,6 +554,71 @@ __arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb,
|
||||
"p1");
|
||||
}
|
||||
|
||||
// Scale UV columns by 3 box filter using Streaming SVE.
|
||||
__arm_locally_streaming void ScaleUVAddCols3_SME(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
// Streaming-SVE only, no use of ZA tile.
|
||||
int vl;
|
||||
int vl_w;
|
||||
asm volatile(
|
||||
"cnth %x[vl] \n"
|
||||
"cntw %x[vl_w] \n"
|
||||
"dup z6.h, %w[scaleval] \n"
|
||||
"ptrue p3.h \n"
|
||||
"subs %w[dst_width], %w[dst_width], %w[vl] \n"
|
||||
"b.lt 2f \n"
|
||||
|
||||
"ptrue p0.s \n"
|
||||
"ptrue p1.h \n"
|
||||
"1: \n"
|
||||
"ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n"
|
||||
"incb %[src_uv], all, mul #3 \n"
|
||||
"ld3w {z3.s, z4.s, z5.s}, p0/z, [%[src_uv]] \n"
|
||||
"incb %[src_uv], all, mul #3 \n"
|
||||
"subs %w[dst_width], %w[dst_width], %w[vl] \n"
|
||||
"add z0.h, z0.h, z1.h \n"
|
||||
"add z3.h, z3.h, z4.h \n"
|
||||
"add z0.h, z0.h, z2.h \n"
|
||||
"add z3.h, z3.h, z5.h \n"
|
||||
"umulh z0.h, p3/m, z0.h, z6.h \n"
|
||||
"umulh z3.h, p3/m, z3.h, z6.h \n"
|
||||
"uzp1 z0.b, z0.b, z3.b \n"
|
||||
"st1h {z0.h}, p1, [%[dst_uv]] \n"
|
||||
"incb %[dst_uv] \n"
|
||||
"b.ge 1b \n"
|
||||
|
||||
"2: \n"
|
||||
"adds %w[dst_width], %w[dst_width], %w[vl] \n"
|
||||
"b.eq 99f \n"
|
||||
|
||||
"whilelt p0.s, wzr, %w[dst_width] \n"
|
||||
"whilelt p2.s, %w[vl_w], %w[dst_width] \n"
|
||||
"whilelt p1.h, wzr, %w[dst_width] \n"
|
||||
"ld3w {z0.s, z1.s, z2.s}, p0/z, [%[src_uv]] \n"
|
||||
"incb %[src_uv], all, mul #3 \n"
|
||||
"ld3w {z3.s, z4.s, z5.s}, p2/z, [%[src_uv]] \n"
|
||||
"add z0.h, z0.h, z1.h \n"
|
||||
"add z3.h, z3.h, z4.h \n"
|
||||
"add z0.h, z0.h, z2.h \n"
|
||||
"add z3.h, z3.h, z5.h \n"
|
||||
"umulh z0.h, p3/m, z0.h, z6.h \n"
|
||||
"umulh z3.h, p3/m, z3.h, z6.h \n"
|
||||
"uzp1 z0.b, z0.b, z3.b \n"
|
||||
"st1h {z0.h}, p1, [%[dst_uv]] \n"
|
||||
|
||||
"99: \n"
|
||||
: [src_uv] "+r"(src_uv), // %[src_uv]
|
||||
[dst_uv] "+r"(dst_uv), // %[dst_uv]
|
||||
[dst_width] "+r"(dst_width), // %[dst_width]
|
||||
[vl] "=&r"(vl), // %[vl]
|
||||
[vl_w] "=&r"(vl_w) // %[vl_w]
|
||||
: [scaleval] "r"(scaleval) // %[scaleval]
|
||||
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "p0", "p1",
|
||||
"p2", "p3");
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
|
||||
// defined(__aarch64__)
|
||||
|
||||
|
||||
@ -346,10 +346,67 @@ static int ScaleUVBox(int src_width,
|
||||
void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx,
|
||||
const uint16_t* src_uv, uint8_t* dst_uv) =
|
||||
(dx & 0xffff) ? ScaleUVAddCols2_C : ScaleUVAddCols1_C;
|
||||
void (*ScaleUVAddCols3)(int dst_width, int scaleval,
|
||||
const uint16_t* src_uv, uint8_t* dst_uv) = NULL;
|
||||
// Summing a row is position wise, so an interleaved UV row can be fed to
|
||||
// the plane accumulator as src_width * 2 samples.
|
||||
void (*ScaleAddRow)(const uint8_t* src_ptr, uint16_t* dst_ptr,
|
||||
int src_width) = ScaleAddRow_C;
|
||||
if (dx == 0x30000) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_C;
|
||||
#if defined(HAS_SCALEUVADDCOLS3_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_Any_AVX2;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVADDCOLS3_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_AVX512VBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVADDCOLS3_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_Any_NEON;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVADDCOLS3_SME)
|
||||
if (TestCpuFlag(kCpuHasSME)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVADDCOLS3_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_RVV;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVADDCOLS3_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_Any_LSX;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_LSX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVADDCOLS3_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASM)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_Any_WASMSIMD;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_WASMSIMD;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVADDCOLS3_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
ScaleUVAddCols3 = ScaleUVAddCols3_HVX;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#if defined(HAS_SCALEADDROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
ScaleAddRow = ScaleAddRow_Any_SSE2;
|
||||
@ -366,6 +423,11 @@ static int ScaleUVBox(int src_width,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEADDROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ScaleAddRow = ScaleAddRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEADDROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ScaleAddRow = ScaleAddRow_Any_NEON;
|
||||
@ -392,6 +454,14 @@ static int ScaleUVBox(int src_width,
|
||||
ScaleAddRow = ScaleAddRow_HVX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEADDROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASM)) {
|
||||
ScaleAddRow = ScaleAddRow_Any_WASMSIMD;
|
||||
if (IS_ALIGNED(row_samples, 16)) {
|
||||
ScaleAddRow = ScaleAddRow_WASMSIMD;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
uint32_t uy = y;
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
@ -408,7 +478,14 @@ static int ScaleUVBox(int src_width,
|
||||
ScaleAddRow(src, (uint16_t*)(row16), row_samples);
|
||||
src += src_stride;
|
||||
}
|
||||
ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16), dst_uv);
|
||||
if (ScaleUVAddCols3 && boxheight <= 85) {
|
||||
ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight),
|
||||
(const uint16_t*)row16 + (ptrdiff_t)(x >> 16) * 2,
|
||||
dst_uv);
|
||||
} else {
|
||||
ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16),
|
||||
dst_uv);
|
||||
}
|
||||
dst_uv += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
@ -637,6 +637,74 @@ void ScaleARGBColsUp2_WASMSIMD(uint8_t* dst_argb,
|
||||
}
|
||||
#endif // HAS_SCALEARGBCOLSUP2_WASMSIMD
|
||||
|
||||
#ifdef HAS_SCALEADDROW_WASMSIMD
|
||||
// Reads 16 bytes and accumulates to 16 shorts at a time.
|
||||
void ScaleAddRow_WASMSIMD(const uint8_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
int src_width) {
|
||||
while (src_width > 0) {
|
||||
v128_t s = wasm_v128_load(src_ptr);
|
||||
v128_t d0 = wasm_v128_load(dst_ptr);
|
||||
v128_t d1 = wasm_v128_load(dst_ptr + 8);
|
||||
d0 = wasm_i16x8_add(d0, wasm_u16x8_extend_low_u8x16(s));
|
||||
d1 = wasm_i16x8_add(d1, wasm_u16x8_extend_high_u8x16(s));
|
||||
wasm_v128_store(dst_ptr, d0);
|
||||
wasm_v128_store(dst_ptr + 8, d1);
|
||||
src_ptr += 16;
|
||||
dst_ptr += 16;
|
||||
src_width -= 16;
|
||||
}
|
||||
}
|
||||
#endif // HAS_SCALEADDROW_WASMSIMD
|
||||
|
||||
#ifdef HAS_SCALEUVADDCOLS3_WASMSIMD
|
||||
// Reads 24 UV pixels (48 uint16_t) and writes 8 UV pixels (16 bytes).
|
||||
void ScaleUVAddCols3_WASMSIMD(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv) {
|
||||
const v128_t scale = wasm_i16x8_splat((int16_t)scaleval);
|
||||
while (dst_width > 0) {
|
||||
v128_t s0 = wasm_v128_load(src_uv);
|
||||
v128_t s1 = wasm_v128_load(src_uv + 8);
|
||||
v128_t s2 = wasm_v128_load(src_uv + 16);
|
||||
v128_t s3 = wasm_v128_load(src_uv + 24);
|
||||
v128_t s4 = wasm_v128_load(src_uv + 32);
|
||||
v128_t s5 = wasm_v128_load(src_uv + 40);
|
||||
src_uv += 48;
|
||||
|
||||
// Group 0..3 of 3 UV pairs from s0, s1, s2 (dwords 0..11)
|
||||
v128_t a0 = wasm_i32x4_shuffle(s0, s1, 0, 3, 6, 1);
|
||||
v128_t b0 = wasm_i32x4_shuffle(s0, s1, 1, 4, 7, 2);
|
||||
v128_t c0 = wasm_i32x4_shuffle(s0, s1, 2, 5, 0, 3);
|
||||
a0 = wasm_i32x4_shuffle(a0, s2, 0, 1, 2, 5);
|
||||
b0 = wasm_i32x4_shuffle(b0, s2, 0, 1, 2, 6);
|
||||
c0 = wasm_i32x4_shuffle(c0, s2, 0, 1, 4, 7);
|
||||
v128_t sum0 = wasm_i16x8_add(wasm_i16x8_add(a0, b0), c0);
|
||||
|
||||
// Group 4..7 of 3 UV pairs from s3, s4, s5 (dwords 0..11)
|
||||
v128_t a1 = wasm_i32x4_shuffle(s3, s4, 0, 3, 6, 1);
|
||||
v128_t b1 = wasm_i32x4_shuffle(s3, s4, 1, 4, 7, 2);
|
||||
v128_t c1 = wasm_i32x4_shuffle(s3, s4, 2, 5, 0, 3);
|
||||
a1 = wasm_i32x4_shuffle(a1, s5, 0, 1, 2, 5);
|
||||
b1 = wasm_i32x4_shuffle(b1, s5, 0, 1, 2, 6);
|
||||
c1 = wasm_i32x4_shuffle(c1, s5, 0, 1, 4, 7);
|
||||
v128_t sum1 = wasm_i16x8_add(wasm_i16x8_add(a1, b1), c1);
|
||||
|
||||
// Multiply unsigned 16-bit sums by scaleval and shift right by 16.
|
||||
v128_t lo0 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum0, scale), 16);
|
||||
v128_t hi0 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum0, scale), 16);
|
||||
v128_t lo1 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum1, scale), 16);
|
||||
v128_t hi1 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum1, scale), 16);
|
||||
v128_t p0 = wasm_i16x8_narrow_i32x4(lo0, hi0);
|
||||
v128_t p1 = wasm_i16x8_narrow_i32x4(lo1, hi1);
|
||||
wasm_v128_store(dst_uv, wasm_u8x16_narrow_i16x8(p0, p1));
|
||||
dst_uv += 16;
|
||||
dst_width -= 8;
|
||||
}
|
||||
}
|
||||
#endif // HAS_SCALEUVADDCOLS3_WASMSIMD
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
1019
source/scale_win.cc
1019
source/scale_win.cc
File diff suppressed because it is too large
Load Diff
@ -331,6 +331,10 @@ TEST_BOXPLANE(By3_75, 960, 540, 256, 144, 0) /* 1080p to 512x288 */
|
||||
TEST_BOXPLANE(By6, 640, 360, 106, 60, 0) /* 720p to 213x120 */
|
||||
TEST_BOXPLANE(By8, 640, 360, 80, 45, 0) /* 720p to 160x90 */
|
||||
TEST_BOXPLANE(OddTail, 33, 17, 7, 5, 0) /* not a multiple of 8 wide */
|
||||
TEST_BOXPLANE(By3OddTail7, 21, 15, 7, 5, 0) /* 3x, dst_width = 7 */
|
||||
TEST_BOXPLANE(By3OddTail23, 69, 15, 23, 5, 0) /* 3x, dst_width = 23 */
|
||||
TEST_BOXPLANE(By3OddTail47, 141, 15, 47, 5, 0) /* 3x, dst_width = 47 */
|
||||
TEST_BOXPLANE(By3OddTail79, 237, 15, 79, 5, 0) /* 3x, dst_width = 79 */
|
||||
TEST_BOXPLANE(Tiny, 80, 60, 3, 2, 0) /* dst below one SIMD group */
|
||||
#undef TEST_BOXPLANE
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user