mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
Handle SsimWindowRow remainder in SIMD for AVX512BW, SVE2, SME, RVV
Because each 8x8 SSIM window spans two 4x4 tiles horizontally, num_windows = (width - 5) >> 2, so aligned widths that are multiples of 64 (e.g. 64, 128, 640, 1280, 1920) always have remainder 15 (num_windows & 15 == 15). - Update SsimWindowRow_AVX512BW (compare_gcc.cc and compare_win.cc) to handle any remainder (1..15 windows) via k-masking (load_mask for the (rem + 1) * 4 tile words and win_mask for the rem window scores) and set win_simd_size = 1. - Add SsimWindowRow_SVE2, SsimWindowRow_SME, and SsimWindowRow_RVV with win_simd_size = 1 using predicated/vl-controlled loops so no C code (Ssim4x4Row_C or SsimWindowRow_C) is executed on AVX-512BW, SVE2, SME, or RVV CPUs for any image dimensions. - Remove LIBYUV_TARGET_AVX512BW __attribute__((target(...))) from compare_gcc.cc and GCC_HAS_AVX512 from cpu_support.h, using %%zmm0 directly in inline assembly. SSIM Performance (Before vs After): uArch / Target Before After Speedup -------------------------- --------- --------- ------- Coral RVV Verilator (64x16) 5302 cyc 2219 cyc 2.39x Coral RVV WindowRow (15w) 1854 cyc 308 cyc 6.02x Apple M6 SME (64x36 10kx) 16.41 ms 4.69 ms 3.50x Apple M6 SME (720p 1000x) 350.76 ms 66.09 ms 5.31x R=rrwinterton@gmail.com Bug: libyuv:352521931 Change-Id: Ic7d55184c60611af4a57d113cc4ee3d211debadf Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8621918 Commit-Queue: Frank Barchard <fbarchard@google.com> Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard
libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent
64ddbbcdde
commit
47ba8e2160
@ -64,7 +64,7 @@ extern "C" {
|
|||||||
#define HAS_SSIM4X4ROW_AVX2
|
#define HAS_SSIM4X4ROW_AVX2
|
||||||
#define HAS_SSIMWINDOWROW_AVX2
|
#define HAS_SSIMWINDOWROW_AVX2
|
||||||
#endif
|
#endif
|
||||||
#if defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512)
|
#if defined(CLANG_HAS_AVX512)
|
||||||
#define HAS_HAMMINGDISTANCE_AVX512BW
|
#define HAS_HAMMINGDISTANCE_AVX512BW
|
||||||
#define HAS_SUMSQUAREERROR_AVX512BW
|
#define HAS_SUMSQUAREERROR_AVX512BW
|
||||||
#define HAS_HASHDJB2_AVX512BW
|
#define HAS_HASHDJB2_AVX512BW
|
||||||
@ -93,18 +93,23 @@ extern "C" {
|
|||||||
// The following are available for AArch64 SVE:
|
// The following are available for AArch64 SVE:
|
||||||
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
||||||
#define HAS_SSIM4X4ROW_SVE2
|
#define HAS_SSIM4X4ROW_SVE2
|
||||||
|
#define HAS_SSIMWINDOWROW_SVE2
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// The following are available for AArch64 SME:
|
// The following are available for AArch64 SME:
|
||||||
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
|
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
|
||||||
defined(__aarch64__)
|
defined(__aarch64__)
|
||||||
#define HAS_SSIM4X4ROW_SME
|
#define HAS_SSIM4X4ROW_SME
|
||||||
|
#define HAS_SSIMWINDOWROW_SME
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// The following are available for RISC-V Vector (RVV):
|
// The following are available for RISC-V Vector (RVV):
|
||||||
#if !defined(LIBYUV_DISABLE_RVV) && \
|
#if !defined(LIBYUV_DISABLE_RVV) && \
|
||||||
(defined(__riscv_vector) || defined(LIBYUV_RVV))
|
(defined(__riscv_vector) || defined(LIBYUV_RVV))
|
||||||
#define HAS_SSIM4X4ROW_RVV
|
#define HAS_SSIM4X4ROW_RVV
|
||||||
|
#if defined(__riscv_f) || defined(__riscv_zve32f)
|
||||||
|
#define HAS_SSIMWINDOWROW_RVV
|
||||||
|
#endif
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// The following are available for LoongArch LSX:
|
// The following are available for LoongArch LSX:
|
||||||
@ -283,6 +288,15 @@ float SsimWindowRow_AVX2(const uint32_t* row0,
|
|||||||
float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
||||||
const uint32_t* row1,
|
const uint32_t* row1,
|
||||||
int num_windows);
|
int num_windows);
|
||||||
|
float SsimWindowRow_SVE2(const uint32_t* row0,
|
||||||
|
const uint32_t* row1,
|
||||||
|
int num_windows);
|
||||||
|
float SsimWindowRow_SME(const uint32_t* row0,
|
||||||
|
const uint32_t* row1,
|
||||||
|
int num_windows);
|
||||||
|
float SsimWindowRow_RVV(const uint32_t* row0,
|
||||||
|
const uint32_t* row1,
|
||||||
|
int num_windows);
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
} // extern "C"
|
} // extern "C"
|
||||||
|
|||||||
@ -70,15 +70,6 @@ extern "C" {
|
|||||||
#endif // clang >= 3.4
|
#endif // clang >= 3.4
|
||||||
#endif // __clang__
|
#endif // __clang__
|
||||||
|
|
||||||
// GCC >= 6.0.0 required for AVX512.
|
|
||||||
#if defined(__GNUC__) && !defined(__clang__) && \
|
|
||||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
|
||||||
(defined(__x86_64__) || defined(__i386__))
|
|
||||||
#if (__GNUC__ >= 6)
|
|
||||||
#define GCC_HAS_AVX512 1
|
|
||||||
#endif // GNUC >= 6
|
|
||||||
#endif // __GNUC__
|
|
||||||
|
|
||||||
// clang >= 6.0.0 required for AVX512.
|
// clang >= 6.0.0 required for AVX512.
|
||||||
#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
|
#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||||
(defined(__x86_64__) || defined(__i386__))
|
(defined(__x86_64__) || defined(__i386__))
|
||||||
|
|||||||
@ -2285,6 +2285,81 @@ static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a,
|
|||||||
"z17", "z18", "z19", "z20", "p0");
|
"z17", "z18", "z19", "z20", "p0");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline float SsimWindowRow_SVE_SC(const uint32_t* row0,
|
||||||
|
const uint32_t* row1,
|
||||||
|
int num_windows) STREAMING_COMPATIBLE {
|
||||||
|
uint64_t vl;
|
||||||
|
const uint32_t* row0_next = row0 + 4;
|
||||||
|
const uint32_t* row1_next = row1 + 4;
|
||||||
|
float total;
|
||||||
|
asm volatile(
|
||||||
|
"cntw %x[vl] \n"
|
||||||
|
"ptrue p1.s \n"
|
||||||
|
"mov z24.s, #0 \n"
|
||||||
|
"dup z25.s, %w[c1] \n"
|
||||||
|
"dup z26.s, %w[c2] \n"
|
||||||
|
|
||||||
|
"1: \n"
|
||||||
|
"whilelt p0.s, wzr, %w[num_windows] \n"
|
||||||
|
"ld4w {z0.s, z1.s, z2.s, z3.s}, p0/z, [%[row0]] \n"
|
||||||
|
"ld4w {z4.s, z5.s, z6.s, z7.s}, p0/z, [%[row0_next]] \n"
|
||||||
|
"ld4w {z16.s, z17.s, z18.s, z19.s}, p0/z, [%[row1]] \n"
|
||||||
|
"ld4w {z20.s, z21.s, z22.s, z23.s}, p0/z, [%[row1_next]] \n"
|
||||||
|
"incb %[row0], all, mul #4 \n"
|
||||||
|
"incb %[row0_next], all, mul #4 \n"
|
||||||
|
"incb %[row1], all, mul #4 \n"
|
||||||
|
"incb %[row1_next], all, mul #4 \n"
|
||||||
|
"add z0.s, z0.s, z4.s \n"
|
||||||
|
"add z1.s, z1.s, z5.s \n"
|
||||||
|
"add z2.s, z2.s, z6.s \n"
|
||||||
|
"add z3.s, z3.s, z7.s \n"
|
||||||
|
"add z16.s, z16.s, z20.s \n"
|
||||||
|
"add z17.s, z17.s, z21.s \n"
|
||||||
|
"add z18.s, z18.s, z22.s \n"
|
||||||
|
"add z19.s, z19.s, z23.s \n"
|
||||||
|
"add z0.s, z0.s, z16.s \n"
|
||||||
|
"add z1.s, z1.s, z17.s \n"
|
||||||
|
"add z2.s, z2.s, z18.s \n"
|
||||||
|
"add z3.s, z3.s, z19.s \n"
|
||||||
|
"mul z4.s, z0.s, z1.s \n"
|
||||||
|
"lsl z4.s, z4.s, #1 \n"
|
||||||
|
"mul z5.s, z0.s, z0.s \n"
|
||||||
|
"mla z5.s, p1/m, z1.s, z1.s \n"
|
||||||
|
"lsl z3.s, z3.s, #7 \n"
|
||||||
|
"sub z3.s, z3.s, z4.s \n"
|
||||||
|
"add z4.s, z4.s, z25.s \n"
|
||||||
|
"add z3.s, z3.s, z26.s \n"
|
||||||
|
"lsl z2.s, z2.s, #6 \n"
|
||||||
|
"sub z2.s, z2.s, z5.s \n"
|
||||||
|
"add z5.s, z5.s, z25.s \n"
|
||||||
|
"add z2.s, z2.s, z26.s \n"
|
||||||
|
"scvtf z4.s, p1/m, z4.s \n"
|
||||||
|
"scvtf z3.s, p1/m, z3.s \n"
|
||||||
|
"scvtf z5.s, p1/m, z5.s \n"
|
||||||
|
"scvtf z2.s, p1/m, z2.s \n"
|
||||||
|
"fmul z4.s, z4.s, z3.s \n"
|
||||||
|
"fmul z5.s, z5.s, z2.s \n"
|
||||||
|
"fdiv z4.s, p1/m, z4.s, z5.s \n"
|
||||||
|
"fadd z24.s, p0/m, z24.s, z4.s \n"
|
||||||
|
"subs %w[num_windows], %w[num_windows], %w[vl] \n"
|
||||||
|
"b.gt 1b \n"
|
||||||
|
|
||||||
|
"faddv %s[total], p1, z24.s \n"
|
||||||
|
: [row0] "+r"(row0), // %[row0]
|
||||||
|
[row0_next] "+r"(row0_next), // %[row0_next]
|
||||||
|
[row1] "+r"(row1), // %[row1]
|
||||||
|
[row1_next] "+r"(row1_next), // %[row1_next]
|
||||||
|
[num_windows] "+r"(num_windows), // %[num_windows]
|
||||||
|
[vl] "=&r"(vl), // %[vl]
|
||||||
|
[total] "=w"(total) // %[total]
|
||||||
|
: [c1] "r"(26634), // %[c1]
|
||||||
|
[c2] "r"(239708) // %[c2]
|
||||||
|
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16",
|
||||||
|
"z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26",
|
||||||
|
"p0", "p1");
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
|
|||||||
@ -473,7 +473,7 @@ double CalcFrameSsim(const uint8_t* src_a,
|
|||||||
#if defined(HAS_SSIMWINDOWROW_AVX512BW)
|
#if defined(HAS_SSIMWINDOWROW_AVX512BW)
|
||||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||||
SsimWindowRow = SsimWindowRow_AVX512BW;
|
SsimWindowRow = SsimWindowRow_AVX512BW;
|
||||||
win_simd_size = 16;
|
win_simd_size = 1;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
#if defined(HAS_SSIM4X4ROW_NEON)
|
#if defined(HAS_SSIM4X4ROW_NEON)
|
||||||
@ -494,18 +494,36 @@ double CalcFrameSsim(const uint8_t* src_a,
|
|||||||
simd_size = 4;
|
simd_size = 4;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
#if defined(HAS_SSIMWINDOWROW_SVE2)
|
||||||
|
if (TestCpuFlag(kCpuHasSVE2)) {
|
||||||
|
SsimWindowRow = SsimWindowRow_SVE2;
|
||||||
|
win_simd_size = 1;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
#if defined(HAS_SSIM4X4ROW_SME)
|
#if defined(HAS_SSIM4X4ROW_SME)
|
||||||
if (TestCpuFlag(kCpuHasSME)) {
|
if (TestCpuFlag(kCpuHasSME)) {
|
||||||
Ssim4x4Row = Ssim4x4Row_SME;
|
Ssim4x4Row = Ssim4x4Row_SME;
|
||||||
simd_size = 4;
|
simd_size = 4;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
#if defined(HAS_SSIMWINDOWROW_SME)
|
||||||
|
if (TestCpuFlag(kCpuHasSME)) {
|
||||||
|
SsimWindowRow = SsimWindowRow_SME;
|
||||||
|
win_simd_size = 1;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
#if defined(HAS_SSIM4X4ROW_RVV)
|
#if defined(HAS_SSIM4X4ROW_RVV)
|
||||||
if (TestCpuFlag(kCpuHasRVV)) {
|
if (TestCpuFlag(kCpuHasRVV)) {
|
||||||
Ssim4x4Row = Ssim4x4Row_RVV;
|
Ssim4x4Row = Ssim4x4Row_RVV;
|
||||||
simd_size = 4;
|
simd_size = 4;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
#if defined(HAS_SSIMWINDOWROW_RVV)
|
||||||
|
if (TestCpuFlag(kCpuHasRVV)) {
|
||||||
|
SsimWindowRow = SsimWindowRow_RVV;
|
||||||
|
win_simd_size = 1;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
#if defined(HAS_SSIM4X4ROW_LSX)
|
#if defined(HAS_SSIM4X4ROW_LSX)
|
||||||
if (TestCpuFlag(kCpuHasLSX)) {
|
if (TestCpuFlag(kCpuHasLSX)) {
|
||||||
Ssim4x4Row = Ssim4x4Row_LSX;
|
Ssim4x4Row = Ssim4x4Row_LSX;
|
||||||
|
|||||||
@ -28,13 +28,6 @@ extern "C" {
|
|||||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||||
(!defined(_MSC_VER) || defined(__clang__))
|
(!defined(_MSC_VER) || defined(__clang__))
|
||||||
|
|
||||||
#if defined(__clang__) || defined(__GNUC__)
|
|
||||||
#define LIBYUV_TARGET_AVX512BW \
|
|
||||||
__attribute__((target("avx512bw,avx512vl,avx512f")))
|
|
||||||
#else
|
|
||||||
#define LIBYUV_TARGET_AVX512BW
|
|
||||||
#endif
|
|
||||||
|
|
||||||
// "memory" clobber prevents the reads from being removed
|
// "memory" clobber prevents the reads from being removed
|
||||||
|
|
||||||
uint32_t HammingDistance_SSE42(const uint8_t* src_a,
|
uint32_t HammingDistance_SSE42(const uint8_t* src_a,
|
||||||
@ -385,7 +378,6 @@ static uint64_t LowBytesMask64(int n) {
|
|||||||
#ifdef HAS_HAMMINGDISTANCE_AVX512BW
|
#ifdef HAS_HAMMINGDISTANCE_AVX512BW
|
||||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||||
// loads that read zeros, so any count is supported.
|
// loads that read zeros, so any count is supported.
|
||||||
LIBYUV_TARGET_AVX512BW
|
|
||||||
uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
|
uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
|
||||||
const uint8_t* src_b,
|
const uint8_t* src_b,
|
||||||
int count) {
|
int count) {
|
||||||
@ -454,7 +446,6 @@ uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
|
|||||||
#ifdef HAS_SUMSQUAREERROR_AVX512BW
|
#ifdef HAS_SUMSQUAREERROR_AVX512BW
|
||||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||||
// loads that read zeros, so any count is supported.
|
// loads that read zeros, so any count is supported.
|
||||||
LIBYUV_TARGET_AVX512BW
|
|
||||||
uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
|
uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
|
||||||
const uint8_t* src_b,
|
const uint8_t* src_b,
|
||||||
int count) {
|
int count) {
|
||||||
@ -672,7 +663,6 @@ static uint32_t Pow33(int n) {
|
|||||||
// Process 64 bytes per loop. The remainder of n = 1 to 63 bytes is loaded
|
// Process 64 bytes per loop. The remainder of n = 1 to 63 bytes is loaded
|
||||||
// into the high n bytes with a masked load ending at src + count, so the
|
// into the high n bytes with a masked load ending at src + count, so the
|
||||||
// same multipliers apply, and hash is multiplied by 33 ^ n.
|
// same multipliers apply, and hash is multiplied by 33 ^ n.
|
||||||
LIBYUV_TARGET_AVX512BW
|
|
||||||
uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) {
|
uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) {
|
||||||
const int n = count & 63;
|
const int n = count & 63;
|
||||||
const uint64_t mask = n ? ~0ULL << (64 - n) : 0ULL;
|
const uint64_t mask = n ? ~0ULL << (64 - n) : 0ULL;
|
||||||
@ -1012,7 +1002,6 @@ void Ssim4x4Row_AVX2(const uint8_t* src_a,
|
|||||||
|
|
||||||
// Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60
|
// Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60
|
||||||
// pixels (1 to 15 tiles) with masked loads and stores.
|
// pixels (1 to 15 tiles) with masked loads and stores.
|
||||||
LIBYUV_TARGET_AVX512BW
|
|
||||||
void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
|
void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
|
||||||
int stride_a,
|
int stride_a,
|
||||||
const uint8_t* src_b,
|
const uint8_t* src_b,
|
||||||
@ -1262,15 +1251,52 @@ float SsimWindowRow_AVX2(const uint32_t* row0,
|
|||||||
#endif // HAS_SSIMWINDOWROW_AVX2
|
#endif // HAS_SSIMWINDOWROW_AVX2
|
||||||
|
|
||||||
#ifdef HAS_SSIMWINDOWROW_AVX512BW
|
#ifdef HAS_SSIMWINDOWROW_AVX512BW
|
||||||
// num_windows is a multiple of 16.
|
#define SSIM_WINDOW_AVX512BW \
|
||||||
LIBYUV_TARGET_AVX512BW
|
"vpunpckldq %%zmm2,%%zmm1,%%zmm5 \n" \
|
||||||
|
"vpunpckhdq %%zmm2,%%zmm1,%%zmm1 \n" \
|
||||||
|
"vpunpckldq %%zmm4,%%zmm3,%%zmm2 \n" \
|
||||||
|
"vpunpckhdq %%zmm4,%%zmm3,%%zmm3 \n" \
|
||||||
|
"vpunpcklqdq %%zmm2,%%zmm5,%%zmm4 \n" \
|
||||||
|
"vpunpckhqdq %%zmm2,%%zmm5,%%zmm2 \n" \
|
||||||
|
"vpunpcklqdq %%zmm3,%%zmm1,%%zmm5 \n" \
|
||||||
|
"vpunpckhqdq %%zmm3,%%zmm1,%%zmm3 \n" \
|
||||||
|
"vpmaddwd %%zmm2,%%zmm4,%%zmm1 \n" \
|
||||||
|
"vpmaddwd %%zmm4,%%zmm4,%%zmm4 \n" \
|
||||||
|
"vpmaddwd %%zmm2,%%zmm2,%%zmm2 \n" \
|
||||||
|
"vpaddd %%zmm2,%%zmm4,%%zmm4 \n" \
|
||||||
|
"vpslld $0x1,%%zmm1,%%zmm1 \n" \
|
||||||
|
"vpslld $0x7,%%zmm3,%%zmm3 \n" \
|
||||||
|
"vpsubd %%zmm1,%%zmm3,%%zmm3 \n" \
|
||||||
|
"vpslld $0x6,%%zmm5,%%zmm5 \n" \
|
||||||
|
"vpsubd %%zmm4,%%zmm5,%%zmm5 \n" \
|
||||||
|
"vpaddd %[c1]%{1to16%},%%zmm1,%%zmm1 \n" \
|
||||||
|
"vpaddd %[c2]%{1to16%},%%zmm3,%%zmm3 \n" \
|
||||||
|
"vpaddd %[c1]%{1to16%},%%zmm4,%%zmm4 \n" \
|
||||||
|
"vpaddd %[c2]%{1to16%},%%zmm5,%%zmm5 \n" \
|
||||||
|
"vcvtdq2ps %%zmm1,%%zmm1 \n" \
|
||||||
|
"vcvtdq2ps %%zmm3,%%zmm3 \n" \
|
||||||
|
"vcvtdq2ps %%zmm4,%%zmm4 \n" \
|
||||||
|
"vcvtdq2ps %%zmm5,%%zmm5 \n" \
|
||||||
|
"vmulps %%zmm3,%%zmm1,%%zmm1 \n" \
|
||||||
|
"vmulps %%zmm5,%%zmm4,%%zmm4 \n" \
|
||||||
|
"vdivps %%zmm4,%%zmm1,%%zmm1 \n"
|
||||||
|
|
||||||
|
// Process 16 8x8 windows per loop and the remainder of 1 to 15 windows with
|
||||||
|
// masked loads and masked accumulation.
|
||||||
float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
||||||
const uint32_t* row1,
|
const uint32_t* row1,
|
||||||
int num_windows) {
|
int num_windows) {
|
||||||
|
const int rem = num_windows & 15;
|
||||||
|
const uint64_t load_mask = rem ? (~0ULL >> (60 - rem * 4)) : 0ULL;
|
||||||
|
const uint16_t win_mask =
|
||||||
|
(uint16_t)(((0x01110U >> (16 - (rem & 3) * 4)) << (rem >> 2)) +
|
||||||
|
(0x1111U << (rem >> 2)) - 0x1111U);
|
||||||
float total;
|
float total;
|
||||||
asm volatile(
|
asm volatile(
|
||||||
"sub %[row0],%[row1] \n"
|
"sub %[row0],%[row1] \n"
|
||||||
"vxorps %g[total],%g[total],%g[total] \n"
|
"vxorps %%xmm0,%%xmm0,%%xmm0 \n"
|
||||||
|
"sub $0x10,%[num_windows] \n"
|
||||||
|
"jl 2f \n"
|
||||||
|
|
||||||
LABELALIGN
|
LABELALIGN
|
||||||
"1: \n"
|
"1: \n"
|
||||||
@ -1291,54 +1317,57 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
|||||||
"vmovdqu32 0xd0(%[row0]),%%zmm5 \n"
|
"vmovdqu32 0xd0(%[row0]),%%zmm5 \n"
|
||||||
"vpaddd 0xd0(%[row0],%[row1]),%%zmm5,%%zmm5 \n"
|
"vpaddd 0xd0(%[row0],%[row1]),%%zmm5,%%zmm5 \n"
|
||||||
"add $0x100,%[row0] \n"
|
"add $0x100,%[row0] \n"
|
||||||
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n"
|
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n" //
|
||||||
"vpunpckldq %%zmm2,%%zmm1,%%zmm5 \n"
|
SSIM_WINDOW_AVX512BW
|
||||||
"vpunpckhdq %%zmm2,%%zmm1,%%zmm1 \n"
|
"vaddps %%zmm1,%%zmm0,%%zmm0 \n"
|
||||||
"vpunpckldq %%zmm4,%%zmm3,%%zmm2 \n"
|
|
||||||
"vpunpckhdq %%zmm4,%%zmm3,%%zmm3 \n"
|
|
||||||
"vpunpcklqdq %%zmm2,%%zmm5,%%zmm4 \n"
|
|
||||||
"vpunpckhqdq %%zmm2,%%zmm5,%%zmm2 \n"
|
|
||||||
"vpunpcklqdq %%zmm3,%%zmm1,%%zmm5 \n"
|
|
||||||
"vpunpckhqdq %%zmm3,%%zmm1,%%zmm3 \n"
|
|
||||||
"vpmaddwd %%zmm2,%%zmm4,%%zmm1 \n"
|
|
||||||
"vpmaddwd %%zmm4,%%zmm4,%%zmm4 \n"
|
|
||||||
"vpmaddwd %%zmm2,%%zmm2,%%zmm2 \n"
|
|
||||||
"vpaddd %%zmm2,%%zmm4,%%zmm4 \n"
|
|
||||||
"vpslld $0x1,%%zmm1,%%zmm1 \n"
|
|
||||||
"vpslld $0x7,%%zmm3,%%zmm3 \n"
|
|
||||||
"vpsubd %%zmm1,%%zmm3,%%zmm3 \n"
|
|
||||||
"vpslld $0x6,%%zmm5,%%zmm5 \n"
|
|
||||||
"vpsubd %%zmm4,%%zmm5,%%zmm5 \n"
|
|
||||||
"vpaddd %[c1]%{1to16%},%%zmm1,%%zmm1 \n"
|
|
||||||
"vpaddd %[c2]%{1to16%},%%zmm3,%%zmm3 \n"
|
|
||||||
"vpaddd %[c1]%{1to16%},%%zmm4,%%zmm4 \n"
|
|
||||||
"vpaddd %[c2]%{1to16%},%%zmm5,%%zmm5 \n"
|
|
||||||
"vcvtdq2ps %%zmm1,%%zmm1 \n"
|
|
||||||
"vcvtdq2ps %%zmm3,%%zmm3 \n"
|
|
||||||
"vcvtdq2ps %%zmm4,%%zmm4 \n"
|
|
||||||
"vcvtdq2ps %%zmm5,%%zmm5 \n"
|
|
||||||
"vmulps %%zmm3,%%zmm1,%%zmm1 \n"
|
|
||||||
"vmulps %%zmm5,%%zmm4,%%zmm4 \n"
|
|
||||||
"vdivps %%zmm4,%%zmm1,%%zmm1 \n"
|
|
||||||
"vaddps %%zmm1,%g[total],%g[total] \n"
|
|
||||||
"sub $0x10,%[num_windows] \n"
|
"sub $0x10,%[num_windows] \n"
|
||||||
"jg 1b \n"
|
"jge 1b \n"
|
||||||
|
|
||||||
"vextractf32x8 $0x1,%g[total],%%ymm1 \n"
|
"2: \n"
|
||||||
"vaddps %%ymm1,%t[total],%t[total] \n"
|
"add $0x10,%[num_windows] \n"
|
||||||
"vextractf128 $0x1,%t[total],%%xmm1 \n"
|
"je 3f \n"
|
||||||
"vaddps %%xmm1,%x[total],%x[total] \n"
|
"kmovq %[load_mask],%%k1 \n"
|
||||||
"vmovhlps %x[total],%%xmm1,%%xmm1 \n"
|
"vmovdqu32 0x00(%[row0]),%%zmm1%{%%k1%}%{z%} \n"
|
||||||
"vaddps %%xmm1,%x[total],%x[total] \n"
|
"vpaddd 0x00(%[row0],%[row1]),%%zmm1,%%zmm1%{%%k1%}%{z%} \n"
|
||||||
"vshufps $0x1,%x[total],%x[total],%%xmm1 \n"
|
"kshiftrq $0x10,%%k1,%%k1 \n"
|
||||||
"vaddss %%xmm1,%x[total],%x[total] \n"
|
"vmovdqu32 0x40(%[row0]),%%zmm2%{%%k1%}%{z%} \n"
|
||||||
|
"vpaddd 0x40(%[row0],%[row1]),%%zmm2,%%zmm2%{%%k1%}%{z%} \n"
|
||||||
|
"kshiftrq $0x10,%%k1,%%k1 \n"
|
||||||
|
"vmovdqu32 0x80(%[row0]),%%zmm3%{%%k1%}%{z%} \n"
|
||||||
|
"vpaddd 0x80(%[row0],%[row1]),%%zmm3,%%zmm3%{%%k1%}%{z%} \n"
|
||||||
|
"kshiftrq $0x10,%%k1,%%k1 \n"
|
||||||
|
"vmovdqu32 0xc0(%[row0]),%%zmm4%{%%k1%}%{z%} \n"
|
||||||
|
"vpaddd 0xc0(%[row0],%[row1]),%%zmm4,%%zmm4%{%%k1%}%{z%} \n"
|
||||||
|
"valignd $0x4,%%zmm1,%%zmm2,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm5,%%zmm1,%%zmm1 \n"
|
||||||
|
"valignd $0x4,%%zmm2,%%zmm3,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm5,%%zmm2,%%zmm2 \n"
|
||||||
|
"valignd $0x4,%%zmm3,%%zmm4,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm5,%%zmm3,%%zmm3 \n"
|
||||||
|
"valignd $0x4,%%zmm4,%%zmm4,%%zmm5 \n"
|
||||||
|
"vpaddd %%zmm5,%%zmm4,%%zmm4 \n" //
|
||||||
|
SSIM_WINDOW_AVX512BW
|
||||||
|
"kmovw %[win_mask],%%k1 \n"
|
||||||
|
"vaddps %%zmm1,%%zmm0,%%zmm0%{%%k1%} \n"
|
||||||
|
|
||||||
|
"3: \n"
|
||||||
|
"vextractf32x8 $0x1,%%zmm0,%%ymm1 \n"
|
||||||
|
"vaddps %%ymm1,%%ymm0,%%ymm0 \n"
|
||||||
|
"vextractf128 $0x1,%%ymm0,%%xmm1 \n"
|
||||||
|
"vaddps %%xmm1,%%xmm0,%%xmm0 \n"
|
||||||
|
"vmovhlps %%xmm0,%%xmm1,%%xmm1 \n"
|
||||||
|
"vaddps %%xmm1,%%xmm0,%%xmm0 \n"
|
||||||
|
"vshufps $0x1,%%xmm0,%%xmm0,%%xmm1 \n"
|
||||||
|
"vaddss %%xmm1,%%xmm0,%[total] \n"
|
||||||
"vzeroupper \n"
|
"vzeroupper \n"
|
||||||
: [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows),
|
: [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows),
|
||||||
[total] "=&x"(total)
|
[total] "=x"(total)
|
||||||
: [c1] "m"(kSsimC1), [c2] "m"(kSsimC2)
|
: [c1] "m"(kSsimC1), [c2] "m"(kSsimC2), [load_mask] "m"(load_mask),
|
||||||
: "memory", "cc", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
[win_mask] "m"(win_mask)
|
||||||
|
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||||
return total;
|
return total;
|
||||||
}
|
}
|
||||||
|
#undef SSIM_WINDOW_AVX512BW
|
||||||
#endif // HAS_SSIMWINDOWROW_AVX512BW
|
#endif // HAS_SSIMWINDOWROW_AVX512BW
|
||||||
|
|
||||||
#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) &&
|
#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) &&
|
||||||
|
|||||||
@ -812,7 +812,6 @@ float SsimWindowRow_AVX2(const uint32_t* row0,
|
|||||||
#endif // HAS_SSIMWINDOWROW_AVX2
|
#endif // HAS_SSIMWINDOWROW_AVX2
|
||||||
|
|
||||||
#if defined(HAS_SSIMWINDOWROW_AVX512BW)
|
#if defined(HAS_SSIMWINDOWROW_AVX512BW)
|
||||||
// num_windows is a multiple of 16.
|
|
||||||
LIBYUV_TARGET_AVX512BW
|
LIBYUV_TARGET_AVX512BW
|
||||||
float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
||||||
const uint32_t* row1,
|
const uint32_t* row1,
|
||||||
@ -820,8 +819,29 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
|||||||
const __m512i vc1 = _mm512_set1_epi32(26634);
|
const __m512i vc1 = _mm512_set1_epi32(26634);
|
||||||
const __m512i vc2 = _mm512_set1_epi32(239708);
|
const __m512i vc2 = _mm512_set1_epi32(239708);
|
||||||
__m512 total = _mm512_setzero_ps();
|
__m512 total = _mm512_setzero_ps();
|
||||||
int w;
|
#define SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3) \
|
||||||
for (w = 0; w < num_windows; w += 16) {
|
__m512i u0 = _mm512_unpacklo_epi32(w0, w1); \
|
||||||
|
__m512i u1 = _mm512_unpackhi_epi32(w0, w1); \
|
||||||
|
__m512i u2 = _mm512_unpacklo_epi32(w2, w3); \
|
||||||
|
__m512i u3 = _mm512_unpackhi_epi32(w2, w3); \
|
||||||
|
__m512i sum_a = _mm512_unpacklo_epi64(u0, u2); \
|
||||||
|
__m512i sum_b = _mm512_unpackhi_epi64(u0, u2); \
|
||||||
|
__m512i sum_sq = _mm512_unpacklo_epi64(u1, u3); \
|
||||||
|
__m512i sum_axb = _mm512_unpackhi_epi64(u1, u3); \
|
||||||
|
__m512i ab2 = _mm512_slli_epi32(_mm512_madd_epi16(sum_a, sum_b), 1); \
|
||||||
|
__m512i a2b2 = _mm512_add_epi32(_mm512_madd_epi16(sum_a, sum_a), \
|
||||||
|
_mm512_madd_epi16(sum_b, sum_b)); \
|
||||||
|
__m512i n1 = _mm512_add_epi32(ab2, vc1); \
|
||||||
|
__m512i n2 = _mm512_add_epi32( \
|
||||||
|
_mm512_sub_epi32(_mm512_slli_epi32(sum_axb, 7), ab2), vc2); \
|
||||||
|
__m512i d1 = _mm512_add_epi32(a2b2, vc1); \
|
||||||
|
__m512i d2 = _mm512_add_epi32( \
|
||||||
|
_mm512_sub_epi32(_mm512_slli_epi32(sum_sq, 6), a2b2), vc2); \
|
||||||
|
__m512 num = _mm512_mul_ps(_mm512_cvtepi32_ps(n1), _mm512_cvtepi32_ps(n2)); \
|
||||||
|
__m512 den = _mm512_mul_ps(_mm512_cvtepi32_ps(d1), _mm512_cvtepi32_ps(d2)); \
|
||||||
|
__m512 ssim = _mm512_div_ps(num, den)
|
||||||
|
|
||||||
|
while (num_windows >= 16) {
|
||||||
__m512i t0 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 0),
|
__m512i t0 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 0),
|
||||||
_mm512_loadu_si512(row1 + 0));
|
_mm512_loadu_si512(row1 + 0));
|
||||||
__m512i t1 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 16),
|
__m512i t1 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 16),
|
||||||
@ -834,34 +854,40 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
|||||||
_mm512_loadu_si512(row1 + 52));
|
_mm512_loadu_si512(row1 + 52));
|
||||||
row0 += 64;
|
row0 += 64;
|
||||||
row1 += 64;
|
row1 += 64;
|
||||||
|
num_windows -= 16;
|
||||||
__m512i w0 = _mm512_add_epi32(t0, _mm512_alignr_epi32(t1, t0, 4));
|
__m512i w0 = _mm512_add_epi32(t0, _mm512_alignr_epi32(t1, t0, 4));
|
||||||
__m512i w1 = _mm512_add_epi32(t1, _mm512_alignr_epi32(t2, t1, 4));
|
__m512i w1 = _mm512_add_epi32(t1, _mm512_alignr_epi32(t2, t1, 4));
|
||||||
__m512i w2 = _mm512_add_epi32(t2, _mm512_alignr_epi32(t3, t2, 4));
|
__m512i w2 = _mm512_add_epi32(t2, _mm512_alignr_epi32(t3, t2, 4));
|
||||||
__m512i w3 = _mm512_add_epi32(t3, t4);
|
__m512i w3 = _mm512_add_epi32(t3, t4);
|
||||||
|
SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3);
|
||||||
__m512i u0 = _mm512_unpacklo_epi32(w0, w1);
|
total = _mm512_add_ps(total, ssim);
|
||||||
__m512i u1 = _mm512_unpackhi_epi32(w0, w1);
|
|
||||||
__m512i u2 = _mm512_unpacklo_epi32(w2, w3);
|
|
||||||
__m512i u3 = _mm512_unpackhi_epi32(w2, w3);
|
|
||||||
__m512i sum_a = _mm512_unpacklo_epi64(u0, u2);
|
|
||||||
__m512i sum_b = _mm512_unpackhi_epi64(u0, u2);
|
|
||||||
__m512i sum_sq = _mm512_unpacklo_epi64(u1, u3);
|
|
||||||
__m512i sum_axb = _mm512_unpackhi_epi64(u1, u3);
|
|
||||||
|
|
||||||
__m512i ab2 = _mm512_slli_epi32(_mm512_madd_epi16(sum_a, sum_b), 1);
|
|
||||||
__m512i a2b2 = _mm512_add_epi32(_mm512_madd_epi16(sum_a, sum_a),
|
|
||||||
_mm512_madd_epi16(sum_b, sum_b));
|
|
||||||
__m512i n1 = _mm512_add_epi32(ab2, vc1);
|
|
||||||
__m512i n2 = _mm512_add_epi32(
|
|
||||||
_mm512_sub_epi32(_mm512_slli_epi32(sum_axb, 7), ab2), vc2);
|
|
||||||
__m512i d1 = _mm512_add_epi32(a2b2, vc1);
|
|
||||||
__m512i d2 = _mm512_add_epi32(
|
|
||||||
_mm512_sub_epi32(_mm512_slli_epi32(sum_sq, 6), a2b2), vc2);
|
|
||||||
|
|
||||||
__m512 num = _mm512_mul_ps(_mm512_cvtepi32_ps(n1), _mm512_cvtepi32_ps(n2));
|
|
||||||
__m512 den = _mm512_mul_ps(_mm512_cvtepi32_ps(d1), _mm512_cvtepi32_ps(d2));
|
|
||||||
total = _mm512_add_ps(total, _mm512_div_ps(num, den));
|
|
||||||
}
|
}
|
||||||
|
if (num_windows > 0) {
|
||||||
|
const uint64_t load_mask = ~0ULL >> (60 - num_windows * 4);
|
||||||
|
const __mmask16 win_mask =
|
||||||
|
(__mmask16)(((0x01110U >> (16 - (num_windows & 3) * 4))
|
||||||
|
<< (num_windows >> 2)) +
|
||||||
|
(0x1111U << (num_windows >> 2)) - 0x1111U);
|
||||||
|
const __mmask16 m0 = (__mmask16)load_mask;
|
||||||
|
const __mmask16 m1 = (__mmask16)(load_mask >> 16);
|
||||||
|
const __mmask16 m2 = (__mmask16)(load_mask >> 32);
|
||||||
|
const __mmask16 m3 = (__mmask16)(load_mask >> 48);
|
||||||
|
__m512i t0 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m0, row0 + 0),
|
||||||
|
_mm512_maskz_loadu_epi32(m0, row1 + 0));
|
||||||
|
__m512i t1 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m1, row0 + 16),
|
||||||
|
_mm512_maskz_loadu_epi32(m1, row1 + 16));
|
||||||
|
__m512i t2 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m2, row0 + 32),
|
||||||
|
_mm512_maskz_loadu_epi32(m2, row1 + 32));
|
||||||
|
__m512i t3 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m3, row0 + 48),
|
||||||
|
_mm512_maskz_loadu_epi32(m3, row1 + 48));
|
||||||
|
__m512i w0 = _mm512_add_epi32(t0, _mm512_alignr_epi32(t1, t0, 4));
|
||||||
|
__m512i w1 = _mm512_add_epi32(t1, _mm512_alignr_epi32(t2, t1, 4));
|
||||||
|
__m512i w2 = _mm512_add_epi32(t2, _mm512_alignr_epi32(t3, t2, 4));
|
||||||
|
__m512i w3 = _mm512_add_epi32(t3, _mm512_alignr_epi32(t3, t3, 4));
|
||||||
|
SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3);
|
||||||
|
total = _mm512_mask_add_ps(total, win_mask, total, ssim);
|
||||||
|
}
|
||||||
|
#undef SSIM_WINDOW_AVX512_WIN
|
||||||
return _mm512_reduce_add_ps(total);
|
return _mm512_reduce_add_ps(total);
|
||||||
}
|
}
|
||||||
#endif // HAS_SSIMWINDOWROW_AVX512BW
|
#endif // HAS_SSIMWINDOWROW_AVX512BW
|
||||||
|
|||||||
@ -2851,6 +2851,85 @@ void Ssim4x4Row_RVV(const uint8_t* src_a,
|
|||||||
#undef SSIM4X4_ACCUM_RVV
|
#undef SSIM4X4_ACCUM_RVV
|
||||||
#endif // HAS_SSIM4X4ROW_RVV
|
#endif // HAS_SSIM4X4ROW_RVV
|
||||||
|
|
||||||
|
#ifdef HAS_SSIMWINDOWROW_RVV
|
||||||
|
float SsimWindowRow_RVV(const uint32_t* row0,
|
||||||
|
const uint32_t* row1,
|
||||||
|
int num_windows) {
|
||||||
|
size_t vl;
|
||||||
|
size_t byte_vl;
|
||||||
|
const uint32_t* row0_next = row0 + 4;
|
||||||
|
const uint32_t* row1_next = row1 + 4;
|
||||||
|
float total;
|
||||||
|
asm volatile(
|
||||||
|
"vsetvli %[vl], zero, e32, m2, ta, ma \n"
|
||||||
|
"vmv.v.i v0, 0 \n"
|
||||||
|
"vmv.s.x v4, zero \n"
|
||||||
|
"1: \n"
|
||||||
|
"vsetvli %[vl], %[w], e32, m2, tu, ma \n"
|
||||||
|
"vlseg4e32.v v8, (%[row0]) \n"
|
||||||
|
"vlseg4e32.v v16, (%[row0_next]) \n"
|
||||||
|
"vadd.vv v8, v8, v16 \n"
|
||||||
|
"vadd.vv v10, v10, v18 \n"
|
||||||
|
"vadd.vv v12, v12, v20 \n"
|
||||||
|
"vadd.vv v14, v14, v22 \n"
|
||||||
|
"vlseg4e32.v v16, (%[row1]) \n"
|
||||||
|
"vlseg4e32.v v24, (%[row1_next]) \n"
|
||||||
|
"vadd.vv v16, v16, v24 \n"
|
||||||
|
"vadd.vv v18, v18, v26 \n"
|
||||||
|
"vadd.vv v20, v20, v28 \n"
|
||||||
|
"vadd.vv v22, v22, v30 \n"
|
||||||
|
"vadd.vv v8, v8, v16 \n"
|
||||||
|
"vadd.vv v10, v10, v18 \n"
|
||||||
|
"vadd.vv v12, v12, v20 \n"
|
||||||
|
"vadd.vv v14, v14, v22 \n"
|
||||||
|
"vmul.vv v16, v8, v10 \n"
|
||||||
|
"vsll.vi v16, v16, 1 \n"
|
||||||
|
"vmul.vv v18, v8, v8 \n"
|
||||||
|
"vmacc.vv v18, v10, v10 \n"
|
||||||
|
"vsll.vi v14, v14, 7 \n"
|
||||||
|
"vsub.vv v14, v14, v16 \n"
|
||||||
|
"vadd.vx v16, v16, %[c1] \n"
|
||||||
|
"vadd.vx v14, v14, %[c2] \n"
|
||||||
|
"vsll.vi v12, v12, 6 \n"
|
||||||
|
"vsub.vv v12, v12, v18 \n"
|
||||||
|
"vadd.vx v18, v18, %[c1] \n"
|
||||||
|
"vadd.vx v12, v12, %[c2] \n"
|
||||||
|
"vfcvt.f.x.v v16, v16 \n"
|
||||||
|
"vfcvt.f.x.v v14, v14 \n"
|
||||||
|
"vfcvt.f.x.v v18, v18 \n"
|
||||||
|
"vfcvt.f.x.v v12, v12 \n"
|
||||||
|
"vfmul.vv v16, v16, v14 \n"
|
||||||
|
"vfmul.vv v18, v18, v12 \n"
|
||||||
|
"vfdiv.vv v8, v16, v18 \n"
|
||||||
|
"vfadd.vv v0, v0, v8 \n"
|
||||||
|
"sub %[w], %[w], %[vl] \n"
|
||||||
|
"slli %[byte_vl], %[vl], 4 \n"
|
||||||
|
"add %[row0], %[row0], %[byte_vl] \n"
|
||||||
|
"add %[row0_next], %[row0_next], %[byte_vl]\n"
|
||||||
|
"add %[row1], %[row1], %[byte_vl] \n"
|
||||||
|
"add %[row1_next], %[row1_next], %[byte_vl]\n"
|
||||||
|
"bgtz %[w], 1b \n"
|
||||||
|
"vsetvli %[vl], zero, e32, m2, ta, ma \n"
|
||||||
|
"vfredusum.vs v4, v0, v4 \n"
|
||||||
|
"vfmv.f.s %[total], v4 \n"
|
||||||
|
: [row0] "+r"(row0), // %[row0]
|
||||||
|
[row0_next] "+r"(row0_next), // %[row0_next]
|
||||||
|
[row1] "+r"(row1), // %[row1]
|
||||||
|
[row1_next] "+r"(row1_next), // %[row1_next]
|
||||||
|
[w] "+r"(num_windows), // %[w]
|
||||||
|
[vl] "=&r"(vl), // %[vl]
|
||||||
|
[byte_vl] "=&r"(byte_vl), // %[byte_vl]
|
||||||
|
[total] "=f"(total) // %[total]
|
||||||
|
: [c1] "r"(26634), // %[c1]
|
||||||
|
[c2] "r"(239708) // %[c2]
|
||||||
|
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v4", "v8", "v9", "v10",
|
||||||
|
"v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20",
|
||||||
|
"v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
|
||||||
|
"v31");
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
#endif // HAS_SSIMWINDOWROW_RVV
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
} // extern "C"
|
} // extern "C"
|
||||||
} // namespace libyuv
|
} // namespace libyuv
|
||||||
|
|||||||
@ -1134,6 +1134,14 @@ __arm_locally_streaming void Ssim4x4Row_SME(const uint8_t* src_a,
|
|||||||
}
|
}
|
||||||
#endif // HAS_SSIM4X4ROW_SME
|
#endif // HAS_SSIM4X4ROW_SME
|
||||||
|
|
||||||
|
#ifdef HAS_SSIMWINDOWROW_SME
|
||||||
|
__arm_locally_streaming float SsimWindowRow_SME(const uint32_t* row0,
|
||||||
|
const uint32_t* row1,
|
||||||
|
int num_windows) {
|
||||||
|
return SsimWindowRow_SVE_SC(row0, row1, num_windows);
|
||||||
|
}
|
||||||
|
#endif // HAS_SSIMWINDOWROW_SME
|
||||||
|
|
||||||
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
|
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
|
||||||
// defined(__aarch64__)
|
// defined(__aarch64__)
|
||||||
|
|
||||||
|
|||||||
@ -1743,6 +1743,14 @@ void Ssim4x4Row_SVE2(const uint8_t* src_a,
|
|||||||
}
|
}
|
||||||
#endif // HAS_SSIM4X4ROW_SVE2
|
#endif // HAS_SSIM4X4ROW_SVE2
|
||||||
|
|
||||||
|
#ifdef HAS_SSIMWINDOWROW_SVE2
|
||||||
|
float SsimWindowRow_SVE2(const uint32_t* row0,
|
||||||
|
const uint32_t* row1,
|
||||||
|
int num_windows) {
|
||||||
|
return SsimWindowRow_SVE_SC(row0, row1, num_windows);
|
||||||
|
}
|
||||||
|
#endif // HAS_SSIMWINDOWROW_SVE2
|
||||||
|
|
||||||
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user