From 47ba8e2160b71a7cf95c0720cd8944ab771e533e Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Tue, 20 Oct 2026 16:15:50 -0700 Subject: [PATCH] Handle SsimWindowRow remainder in SIMD for AVX512BW, SVE2, SME, RVV Because each 8x8 SSIM window spans two 4x4 tiles horizontally, num_windows = (width - 5) >> 2, so aligned widths that are multiples of 64 (e.g. 64, 128, 640, 1280, 1920) always have remainder 15 (num_windows & 15 == 15). - Update SsimWindowRow_AVX512BW (compare_gcc.cc and compare_win.cc) to handle any remainder (1..15 windows) via k-masking (load_mask for the (rem + 1) * 4 tile words and win_mask for the rem window scores) and set win_simd_size = 1. - Add SsimWindowRow_SVE2, SsimWindowRow_SME, and SsimWindowRow_RVV with win_simd_size = 1 using predicated/vl-controlled loops so no C code (Ssim4x4Row_C or SsimWindowRow_C) is executed on AVX-512BW, SVE2, SME, or RVV CPUs for any image dimensions. - Remove LIBYUV_TARGET_AVX512BW __attribute__((target(...))) from compare_gcc.cc and GCC_HAS_AVX512 from cpu_support.h, using %%zmm0 directly in inline assembly. SSIM Performance (Before vs After): uArch / Target Before After Speedup -------------------------- --------- --------- ------- Coral RVV Verilator (64x16) 5302 cyc 2219 cyc 2.39x Coral RVV WindowRow (15w) 1854 cyc 308 cyc 6.02x Apple M6 SME (64x36 10kx) 16.41 ms 4.69 ms 3.50x Apple M6 SME (720p 1000x) 350.76 ms 66.09 ms 5.31x R=rrwinterton@gmail.com Bug: libyuv:352521931 Change-Id: Ic7d55184c60611af4a57d113cc4ee3d211debadf Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8621918 Commit-Queue: Frank Barchard Reviewed-by: richard winterton --- include/libyuv/compare_row.h | 16 +++- include/libyuv/cpu_support.h | 9 --- include/libyuv/row_sve.h | 75 +++++++++++++++++++ source/compare.cc | 20 ++++- source/compare_gcc.cc | 141 +++++++++++++++++++++-------------- source/compare_win.cc | 78 ++++++++++++------- source/row_rvv.cc | 79 ++++++++++++++++++++ source/row_sme.cc | 8 ++ source/row_sve.cc | 8 ++ 9 files changed, 341 insertions(+), 93 deletions(-) diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index 22336f9a1..301a103e9 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -64,7 +64,7 @@ extern "C" { #define HAS_SSIM4X4ROW_AVX2 #define HAS_SSIMWINDOWROW_AVX2 #endif -#if defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512) +#if defined(CLANG_HAS_AVX512) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW #define HAS_HASHDJB2_AVX512BW @@ -93,18 +93,23 @@ extern "C" { // The following are available for AArch64 SVE: #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #define HAS_SSIM4X4ROW_SVE2 +#define HAS_SSIMWINDOWROW_SVE2 #endif // The following are available for AArch64 SME: #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ defined(__aarch64__) #define HAS_SSIM4X4ROW_SME +#define HAS_SSIMWINDOWROW_SME #endif // The following are available for RISC-V Vector (RVV): #if !defined(LIBYUV_DISABLE_RVV) && \ (defined(__riscv_vector) || defined(LIBYUV_RVV)) #define HAS_SSIM4X4ROW_RVV +#if defined(__riscv_f) || defined(__riscv_zve32f) +#define HAS_SSIMWINDOWROW_RVV +#endif #endif // The following are available for LoongArch LSX: @@ -283,6 +288,15 @@ float SsimWindowRow_AVX2(const uint32_t* row0, float SsimWindowRow_AVX512BW(const uint32_t* row0, const uint32_t* row1, int num_windows); +float SsimWindowRow_SVE2(const uint32_t* row0, + const uint32_t* row1, + int num_windows); +float SsimWindowRow_SME(const uint32_t* row0, + const uint32_t* row1, + int num_windows); +float SsimWindowRow_RVV(const uint32_t* row0, + const uint32_t* row1, + int num_windows); #ifdef __cplusplus } // extern "C" diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index 1dd9d71a4..adb50c42d 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -70,15 +70,6 @@ extern "C" { #endif // clang >= 3.4 #endif // __clang__ -// GCC >= 6.0.0 required for AVX512. -#if defined(__GNUC__) && !defined(__clang__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) && \ - (defined(__x86_64__) || defined(__i386__)) -#if (__GNUC__ >= 6) -#define GCC_HAS_AVX512 1 -#endif // GNUC >= 6 -#endif // __GNUC__ - // clang >= 6.0.0 required for AVX512. #if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \ (defined(__x86_64__) || defined(__i386__)) diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 97616b9a3..19559b64f 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -2285,6 +2285,81 @@ static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a, "z17", "z18", "z19", "z20", "p0"); } +static inline float SsimWindowRow_SVE_SC(const uint32_t* row0, + const uint32_t* row1, + int num_windows) STREAMING_COMPATIBLE { + uint64_t vl; + const uint32_t* row0_next = row0 + 4; + const uint32_t* row1_next = row1 + 4; + float total; + asm volatile( + "cntw %x[vl] \n" + "ptrue p1.s \n" + "mov z24.s, #0 \n" + "dup z25.s, %w[c1] \n" + "dup z26.s, %w[c2] \n" + + "1: \n" + "whilelt p0.s, wzr, %w[num_windows] \n" + "ld4w {z0.s, z1.s, z2.s, z3.s}, p0/z, [%[row0]] \n" + "ld4w {z4.s, z5.s, z6.s, z7.s}, p0/z, [%[row0_next]] \n" + "ld4w {z16.s, z17.s, z18.s, z19.s}, p0/z, [%[row1]] \n" + "ld4w {z20.s, z21.s, z22.s, z23.s}, p0/z, [%[row1_next]] \n" + "incb %[row0], all, mul #4 \n" + "incb %[row0_next], all, mul #4 \n" + "incb %[row1], all, mul #4 \n" + "incb %[row1_next], all, mul #4 \n" + "add z0.s, z0.s, z4.s \n" + "add z1.s, z1.s, z5.s \n" + "add z2.s, z2.s, z6.s \n" + "add z3.s, z3.s, z7.s \n" + "add z16.s, z16.s, z20.s \n" + "add z17.s, z17.s, z21.s \n" + "add z18.s, z18.s, z22.s \n" + "add z19.s, z19.s, z23.s \n" + "add z0.s, z0.s, z16.s \n" + "add z1.s, z1.s, z17.s \n" + "add z2.s, z2.s, z18.s \n" + "add z3.s, z3.s, z19.s \n" + "mul z4.s, z0.s, z1.s \n" + "lsl z4.s, z4.s, #1 \n" + "mul z5.s, z0.s, z0.s \n" + "mla z5.s, p1/m, z1.s, z1.s \n" + "lsl z3.s, z3.s, #7 \n" + "sub z3.s, z3.s, z4.s \n" + "add z4.s, z4.s, z25.s \n" + "add z3.s, z3.s, z26.s \n" + "lsl z2.s, z2.s, #6 \n" + "sub z2.s, z2.s, z5.s \n" + "add z5.s, z5.s, z25.s \n" + "add z2.s, z2.s, z26.s \n" + "scvtf z4.s, p1/m, z4.s \n" + "scvtf z3.s, p1/m, z3.s \n" + "scvtf z5.s, p1/m, z5.s \n" + "scvtf z2.s, p1/m, z2.s \n" + "fmul z4.s, z4.s, z3.s \n" + "fmul z5.s, z5.s, z2.s \n" + "fdiv z4.s, p1/m, z4.s, z5.s \n" + "fadd z24.s, p0/m, z24.s, z4.s \n" + "subs %w[num_windows], %w[num_windows], %w[vl] \n" + "b.gt 1b \n" + + "faddv %s[total], p1, z24.s \n" + : [row0] "+r"(row0), // %[row0] + [row0_next] "+r"(row0_next), // %[row0_next] + [row1] "+r"(row1), // %[row1] + [row1_next] "+r"(row1_next), // %[row1_next] + [num_windows] "+r"(num_windows), // %[num_windows] + [vl] "=&r"(vl), // %[vl] + [total] "=w"(total) // %[total] + : [c1] "r"(26634), // %[c1] + [c2] "r"(239708) // %[c2] + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", + "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", + "p0", "p1"); + return total; +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/compare.cc b/source/compare.cc index 0783cdf52..352cedb1b 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -473,7 +473,7 @@ double CalcFrameSsim(const uint8_t* src_a, #if defined(HAS_SSIMWINDOWROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { SsimWindowRow = SsimWindowRow_AVX512BW; - win_simd_size = 16; + win_simd_size = 1; } #endif #if defined(HAS_SSIM4X4ROW_NEON) @@ -494,18 +494,36 @@ double CalcFrameSsim(const uint8_t* src_a, simd_size = 4; } #endif +#if defined(HAS_SSIMWINDOWROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + SsimWindowRow = SsimWindowRow_SVE2; + win_simd_size = 1; + } +#endif #if defined(HAS_SSIM4X4ROW_SME) if (TestCpuFlag(kCpuHasSME)) { Ssim4x4Row = Ssim4x4Row_SME; simd_size = 4; } #endif +#if defined(HAS_SSIMWINDOWROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + SsimWindowRow = SsimWindowRow_SME; + win_simd_size = 1; + } +#endif #if defined(HAS_SSIM4X4ROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { Ssim4x4Row = Ssim4x4Row_RVV; simd_size = 4; } #endif +#if defined(HAS_SSIMWINDOWROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + SsimWindowRow = SsimWindowRow_RVV; + win_simd_size = 1; + } +#endif #if defined(HAS_SSIM4X4ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { Ssim4x4Row = Ssim4x4Row_LSX; diff --git a/source/compare_gcc.cc b/source/compare_gcc.cc index fcbb87de4..123d92fea 100644 --- a/source/compare_gcc.cc +++ b/source/compare_gcc.cc @@ -28,13 +28,6 @@ extern "C" { !defined(LIBYUV_ENABLE_ROWWIN) && \ (!defined(_MSC_VER) || defined(__clang__)) -#if defined(__clang__) || defined(__GNUC__) -#define LIBYUV_TARGET_AVX512BW \ - __attribute__((target("avx512bw,avx512vl,avx512f"))) -#else -#define LIBYUV_TARGET_AVX512BW -#endif - // "memory" clobber prevents the reads from being removed uint32_t HammingDistance_SSE42(const uint8_t* src_a, @@ -385,7 +378,6 @@ static uint64_t LowBytesMask64(int n) { #ifdef HAS_HAMMINGDISTANCE_AVX512BW // Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked // loads that read zeros, so any count is supported. -LIBYUV_TARGET_AVX512BW uint32_t HammingDistance_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -454,7 +446,6 @@ uint32_t HammingDistance_AVX512BW(const uint8_t* src_a, #ifdef HAS_SUMSQUAREERROR_AVX512BW // Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked // loads that read zeros, so any count is supported. -LIBYUV_TARGET_AVX512BW uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -672,7 +663,6 @@ static uint32_t Pow33(int n) { // Process 64 bytes per loop. The remainder of n = 1 to 63 bytes is loaded // into the high n bytes with a masked load ending at src + count, so the // same multipliers apply, and hash is multiplied by 33 ^ n. -LIBYUV_TARGET_AVX512BW uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) { const int n = count & 63; const uint64_t mask = n ? ~0ULL << (64 - n) : 0ULL; @@ -1012,7 +1002,6 @@ void Ssim4x4Row_AVX2(const uint8_t* src_a, // Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60 // pixels (1 to 15 tiles) with masked loads and stores. -LIBYUV_TARGET_AVX512BW void Ssim4x4Row_AVX512BW(const uint8_t* src_a, int stride_a, const uint8_t* src_b, @@ -1262,15 +1251,52 @@ float SsimWindowRow_AVX2(const uint32_t* row0, #endif // HAS_SSIMWINDOWROW_AVX2 #ifdef HAS_SSIMWINDOWROW_AVX512BW -// num_windows is a multiple of 16. -LIBYUV_TARGET_AVX512BW +#define SSIM_WINDOW_AVX512BW \ + "vpunpckldq %%zmm2,%%zmm1,%%zmm5 \n" \ + "vpunpckhdq %%zmm2,%%zmm1,%%zmm1 \n" \ + "vpunpckldq %%zmm4,%%zmm3,%%zmm2 \n" \ + "vpunpckhdq %%zmm4,%%zmm3,%%zmm3 \n" \ + "vpunpcklqdq %%zmm2,%%zmm5,%%zmm4 \n" \ + "vpunpckhqdq %%zmm2,%%zmm5,%%zmm2 \n" \ + "vpunpcklqdq %%zmm3,%%zmm1,%%zmm5 \n" \ + "vpunpckhqdq %%zmm3,%%zmm1,%%zmm3 \n" \ + "vpmaddwd %%zmm2,%%zmm4,%%zmm1 \n" \ + "vpmaddwd %%zmm4,%%zmm4,%%zmm4 \n" \ + "vpmaddwd %%zmm2,%%zmm2,%%zmm2 \n" \ + "vpaddd %%zmm2,%%zmm4,%%zmm4 \n" \ + "vpslld $0x1,%%zmm1,%%zmm1 \n" \ + "vpslld $0x7,%%zmm3,%%zmm3 \n" \ + "vpsubd %%zmm1,%%zmm3,%%zmm3 \n" \ + "vpslld $0x6,%%zmm5,%%zmm5 \n" \ + "vpsubd %%zmm4,%%zmm5,%%zmm5 \n" \ + "vpaddd %[c1]%{1to16%},%%zmm1,%%zmm1 \n" \ + "vpaddd %[c2]%{1to16%},%%zmm3,%%zmm3 \n" \ + "vpaddd %[c1]%{1to16%},%%zmm4,%%zmm4 \n" \ + "vpaddd %[c2]%{1to16%},%%zmm5,%%zmm5 \n" \ + "vcvtdq2ps %%zmm1,%%zmm1 \n" \ + "vcvtdq2ps %%zmm3,%%zmm3 \n" \ + "vcvtdq2ps %%zmm4,%%zmm4 \n" \ + "vcvtdq2ps %%zmm5,%%zmm5 \n" \ + "vmulps %%zmm3,%%zmm1,%%zmm1 \n" \ + "vmulps %%zmm5,%%zmm4,%%zmm4 \n" \ + "vdivps %%zmm4,%%zmm1,%%zmm1 \n" + +// Process 16 8x8 windows per loop and the remainder of 1 to 15 windows with +// masked loads and masked accumulation. float SsimWindowRow_AVX512BW(const uint32_t* row0, const uint32_t* row1, int num_windows) { + const int rem = num_windows & 15; + const uint64_t load_mask = rem ? (~0ULL >> (60 - rem * 4)) : 0ULL; + const uint16_t win_mask = + (uint16_t)(((0x01110U >> (16 - (rem & 3) * 4)) << (rem >> 2)) + + (0x1111U << (rem >> 2)) - 0x1111U); float total; asm volatile( "sub %[row0],%[row1] \n" - "vxorps %g[total],%g[total],%g[total] \n" + "vxorps %%xmm0,%%xmm0,%%xmm0 \n" + "sub $0x10,%[num_windows] \n" + "jl 2f \n" LABELALIGN "1: \n" @@ -1291,54 +1317,57 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, "vmovdqu32 0xd0(%[row0]),%%zmm5 \n" "vpaddd 0xd0(%[row0],%[row1]),%%zmm5,%%zmm5 \n" "add $0x100,%[row0] \n" - "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" - "vpunpckldq %%zmm2,%%zmm1,%%zmm5 \n" - "vpunpckhdq %%zmm2,%%zmm1,%%zmm1 \n" - "vpunpckldq %%zmm4,%%zmm3,%%zmm2 \n" - "vpunpckhdq %%zmm4,%%zmm3,%%zmm3 \n" - "vpunpcklqdq %%zmm2,%%zmm5,%%zmm4 \n" - "vpunpckhqdq %%zmm2,%%zmm5,%%zmm2 \n" - "vpunpcklqdq %%zmm3,%%zmm1,%%zmm5 \n" - "vpunpckhqdq %%zmm3,%%zmm1,%%zmm3 \n" - "vpmaddwd %%zmm2,%%zmm4,%%zmm1 \n" - "vpmaddwd %%zmm4,%%zmm4,%%zmm4 \n" - "vpmaddwd %%zmm2,%%zmm2,%%zmm2 \n" - "vpaddd %%zmm2,%%zmm4,%%zmm4 \n" - "vpslld $0x1,%%zmm1,%%zmm1 \n" - "vpslld $0x7,%%zmm3,%%zmm3 \n" - "vpsubd %%zmm1,%%zmm3,%%zmm3 \n" - "vpslld $0x6,%%zmm5,%%zmm5 \n" - "vpsubd %%zmm4,%%zmm5,%%zmm5 \n" - "vpaddd %[c1]%{1to16%},%%zmm1,%%zmm1 \n" - "vpaddd %[c2]%{1to16%},%%zmm3,%%zmm3 \n" - "vpaddd %[c1]%{1to16%},%%zmm4,%%zmm4 \n" - "vpaddd %[c2]%{1to16%},%%zmm5,%%zmm5 \n" - "vcvtdq2ps %%zmm1,%%zmm1 \n" - "vcvtdq2ps %%zmm3,%%zmm3 \n" - "vcvtdq2ps %%zmm4,%%zmm4 \n" - "vcvtdq2ps %%zmm5,%%zmm5 \n" - "vmulps %%zmm3,%%zmm1,%%zmm1 \n" - "vmulps %%zmm5,%%zmm4,%%zmm4 \n" - "vdivps %%zmm4,%%zmm1,%%zmm1 \n" - "vaddps %%zmm1,%g[total],%g[total] \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" // + SSIM_WINDOW_AVX512BW + "vaddps %%zmm1,%%zmm0,%%zmm0 \n" "sub $0x10,%[num_windows] \n" - "jg 1b \n" + "jge 1b \n" - "vextractf32x8 $0x1,%g[total],%%ymm1 \n" - "vaddps %%ymm1,%t[total],%t[total] \n" - "vextractf128 $0x1,%t[total],%%xmm1 \n" - "vaddps %%xmm1,%x[total],%x[total] \n" - "vmovhlps %x[total],%%xmm1,%%xmm1 \n" - "vaddps %%xmm1,%x[total],%x[total] \n" - "vshufps $0x1,%x[total],%x[total],%%xmm1 \n" - "vaddss %%xmm1,%x[total],%x[total] \n" + "2: \n" + "add $0x10,%[num_windows] \n" + "je 3f \n" + "kmovq %[load_mask],%%k1 \n" + "vmovdqu32 0x00(%[row0]),%%zmm1%{%%k1%}%{z%} \n" + "vpaddd 0x00(%[row0],%[row1]),%%zmm1,%%zmm1%{%%k1%}%{z%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 0x40(%[row0]),%%zmm2%{%%k1%}%{z%} \n" + "vpaddd 0x40(%[row0],%[row1]),%%zmm2,%%zmm2%{%%k1%}%{z%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 0x80(%[row0]),%%zmm3%{%%k1%}%{z%} \n" + "vpaddd 0x80(%[row0],%[row1]),%%zmm3,%%zmm3%{%%k1%}%{z%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 0xc0(%[row0]),%%zmm4%{%%k1%}%{z%} \n" + "vpaddd 0xc0(%[row0],%[row1]),%%zmm4,%%zmm4%{%%k1%}%{z%} \n" + "valignd $0x4,%%zmm1,%%zmm2,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm1,%%zmm1 \n" + "valignd $0x4,%%zmm2,%%zmm3,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm2,%%zmm2 \n" + "valignd $0x4,%%zmm3,%%zmm4,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm3,%%zmm3 \n" + "valignd $0x4,%%zmm4,%%zmm4,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" // + SSIM_WINDOW_AVX512BW + "kmovw %[win_mask],%%k1 \n" + "vaddps %%zmm1,%%zmm0,%%zmm0%{%%k1%} \n" + + "3: \n" + "vextractf32x8 $0x1,%%zmm0,%%ymm1 \n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" + "vextractf128 $0x1,%%ymm0,%%xmm1 \n" + "vaddps %%xmm1,%%xmm0,%%xmm0 \n" + "vmovhlps %%xmm0,%%xmm1,%%xmm1 \n" + "vaddps %%xmm1,%%xmm0,%%xmm0 \n" + "vshufps $0x1,%%xmm0,%%xmm0,%%xmm1 \n" + "vaddss %%xmm1,%%xmm0,%[total] \n" "vzeroupper \n" : [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows), - [total] "=&x"(total) - : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2) - : "memory", "cc", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + [total] "=x"(total) + : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2), [load_mask] "m"(load_mask), + [win_mask] "m"(win_mask) + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); return total; } +#undef SSIM_WINDOW_AVX512BW #endif // HAS_SSIMWINDOWROW_AVX512BW #endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && diff --git a/source/compare_win.cc b/source/compare_win.cc index c6a811901..d1166ad46 100644 --- a/source/compare_win.cc +++ b/source/compare_win.cc @@ -812,7 +812,6 @@ float SsimWindowRow_AVX2(const uint32_t* row0, #endif // HAS_SSIMWINDOWROW_AVX2 #if defined(HAS_SSIMWINDOWROW_AVX512BW) -// num_windows is a multiple of 16. LIBYUV_TARGET_AVX512BW float SsimWindowRow_AVX512BW(const uint32_t* row0, const uint32_t* row1, @@ -820,8 +819,29 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, const __m512i vc1 = _mm512_set1_epi32(26634); const __m512i vc2 = _mm512_set1_epi32(239708); __m512 total = _mm512_setzero_ps(); - int w; - for (w = 0; w < num_windows; w += 16) { +#define SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3) \ + __m512i u0 = _mm512_unpacklo_epi32(w0, w1); \ + __m512i u1 = _mm512_unpackhi_epi32(w0, w1); \ + __m512i u2 = _mm512_unpacklo_epi32(w2, w3); \ + __m512i u3 = _mm512_unpackhi_epi32(w2, w3); \ + __m512i sum_a = _mm512_unpacklo_epi64(u0, u2); \ + __m512i sum_b = _mm512_unpackhi_epi64(u0, u2); \ + __m512i sum_sq = _mm512_unpacklo_epi64(u1, u3); \ + __m512i sum_axb = _mm512_unpackhi_epi64(u1, u3); \ + __m512i ab2 = _mm512_slli_epi32(_mm512_madd_epi16(sum_a, sum_b), 1); \ + __m512i a2b2 = _mm512_add_epi32(_mm512_madd_epi16(sum_a, sum_a), \ + _mm512_madd_epi16(sum_b, sum_b)); \ + __m512i n1 = _mm512_add_epi32(ab2, vc1); \ + __m512i n2 = _mm512_add_epi32( \ + _mm512_sub_epi32(_mm512_slli_epi32(sum_axb, 7), ab2), vc2); \ + __m512i d1 = _mm512_add_epi32(a2b2, vc1); \ + __m512i d2 = _mm512_add_epi32( \ + _mm512_sub_epi32(_mm512_slli_epi32(sum_sq, 6), a2b2), vc2); \ + __m512 num = _mm512_mul_ps(_mm512_cvtepi32_ps(n1), _mm512_cvtepi32_ps(n2)); \ + __m512 den = _mm512_mul_ps(_mm512_cvtepi32_ps(d1), _mm512_cvtepi32_ps(d2)); \ + __m512 ssim = _mm512_div_ps(num, den) + + while (num_windows >= 16) { __m512i t0 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 0), _mm512_loadu_si512(row1 + 0)); __m512i t1 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 16), @@ -834,34 +854,40 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, _mm512_loadu_si512(row1 + 52)); row0 += 64; row1 += 64; + num_windows -= 16; __m512i w0 = _mm512_add_epi32(t0, _mm512_alignr_epi32(t1, t0, 4)); __m512i w1 = _mm512_add_epi32(t1, _mm512_alignr_epi32(t2, t1, 4)); __m512i w2 = _mm512_add_epi32(t2, _mm512_alignr_epi32(t3, t2, 4)); __m512i w3 = _mm512_add_epi32(t3, t4); - - __m512i u0 = _mm512_unpacklo_epi32(w0, w1); - __m512i u1 = _mm512_unpackhi_epi32(w0, w1); - __m512i u2 = _mm512_unpacklo_epi32(w2, w3); - __m512i u3 = _mm512_unpackhi_epi32(w2, w3); - __m512i sum_a = _mm512_unpacklo_epi64(u0, u2); - __m512i sum_b = _mm512_unpackhi_epi64(u0, u2); - __m512i sum_sq = _mm512_unpacklo_epi64(u1, u3); - __m512i sum_axb = _mm512_unpackhi_epi64(u1, u3); - - __m512i ab2 = _mm512_slli_epi32(_mm512_madd_epi16(sum_a, sum_b), 1); - __m512i a2b2 = _mm512_add_epi32(_mm512_madd_epi16(sum_a, sum_a), - _mm512_madd_epi16(sum_b, sum_b)); - __m512i n1 = _mm512_add_epi32(ab2, vc1); - __m512i n2 = _mm512_add_epi32( - _mm512_sub_epi32(_mm512_slli_epi32(sum_axb, 7), ab2), vc2); - __m512i d1 = _mm512_add_epi32(a2b2, vc1); - __m512i d2 = _mm512_add_epi32( - _mm512_sub_epi32(_mm512_slli_epi32(sum_sq, 6), a2b2), vc2); - - __m512 num = _mm512_mul_ps(_mm512_cvtepi32_ps(n1), _mm512_cvtepi32_ps(n2)); - __m512 den = _mm512_mul_ps(_mm512_cvtepi32_ps(d1), _mm512_cvtepi32_ps(d2)); - total = _mm512_add_ps(total, _mm512_div_ps(num, den)); + SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3); + total = _mm512_add_ps(total, ssim); } + if (num_windows > 0) { + const uint64_t load_mask = ~0ULL >> (60 - num_windows * 4); + const __mmask16 win_mask = + (__mmask16)(((0x01110U >> (16 - (num_windows & 3) * 4)) + << (num_windows >> 2)) + + (0x1111U << (num_windows >> 2)) - 0x1111U); + const __mmask16 m0 = (__mmask16)load_mask; + const __mmask16 m1 = (__mmask16)(load_mask >> 16); + const __mmask16 m2 = (__mmask16)(load_mask >> 32); + const __mmask16 m3 = (__mmask16)(load_mask >> 48); + __m512i t0 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m0, row0 + 0), + _mm512_maskz_loadu_epi32(m0, row1 + 0)); + __m512i t1 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m1, row0 + 16), + _mm512_maskz_loadu_epi32(m1, row1 + 16)); + __m512i t2 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m2, row0 + 32), + _mm512_maskz_loadu_epi32(m2, row1 + 32)); + __m512i t3 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m3, row0 + 48), + _mm512_maskz_loadu_epi32(m3, row1 + 48)); + __m512i w0 = _mm512_add_epi32(t0, _mm512_alignr_epi32(t1, t0, 4)); + __m512i w1 = _mm512_add_epi32(t1, _mm512_alignr_epi32(t2, t1, 4)); + __m512i w2 = _mm512_add_epi32(t2, _mm512_alignr_epi32(t3, t2, 4)); + __m512i w3 = _mm512_add_epi32(t3, _mm512_alignr_epi32(t3, t3, 4)); + SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3); + total = _mm512_mask_add_ps(total, win_mask, total, ssim); + } +#undef SSIM_WINDOW_AVX512_WIN return _mm512_reduce_add_ps(total); } #endif // HAS_SSIMWINDOWROW_AVX512BW diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 363cbec9b..6b7173675 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -2851,6 +2851,85 @@ void Ssim4x4Row_RVV(const uint8_t* src_a, #undef SSIM4X4_ACCUM_RVV #endif // HAS_SSIM4X4ROW_RVV +#ifdef HAS_SSIMWINDOWROW_RVV +float SsimWindowRow_RVV(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + size_t vl; + size_t byte_vl; + const uint32_t* row0_next = row0 + 4; + const uint32_t* row1_next = row1 + 4; + float total; + asm volatile( + "vsetvli %[vl], zero, e32, m2, ta, ma \n" + "vmv.v.i v0, 0 \n" + "vmv.s.x v4, zero \n" + "1: \n" + "vsetvli %[vl], %[w], e32, m2, tu, ma \n" + "vlseg4e32.v v8, (%[row0]) \n" + "vlseg4e32.v v16, (%[row0_next]) \n" + "vadd.vv v8, v8, v16 \n" + "vadd.vv v10, v10, v18 \n" + "vadd.vv v12, v12, v20 \n" + "vadd.vv v14, v14, v22 \n" + "vlseg4e32.v v16, (%[row1]) \n" + "vlseg4e32.v v24, (%[row1_next]) \n" + "vadd.vv v16, v16, v24 \n" + "vadd.vv v18, v18, v26 \n" + "vadd.vv v20, v20, v28 \n" + "vadd.vv v22, v22, v30 \n" + "vadd.vv v8, v8, v16 \n" + "vadd.vv v10, v10, v18 \n" + "vadd.vv v12, v12, v20 \n" + "vadd.vv v14, v14, v22 \n" + "vmul.vv v16, v8, v10 \n" + "vsll.vi v16, v16, 1 \n" + "vmul.vv v18, v8, v8 \n" + "vmacc.vv v18, v10, v10 \n" + "vsll.vi v14, v14, 7 \n" + "vsub.vv v14, v14, v16 \n" + "vadd.vx v16, v16, %[c1] \n" + "vadd.vx v14, v14, %[c2] \n" + "vsll.vi v12, v12, 6 \n" + "vsub.vv v12, v12, v18 \n" + "vadd.vx v18, v18, %[c1] \n" + "vadd.vx v12, v12, %[c2] \n" + "vfcvt.f.x.v v16, v16 \n" + "vfcvt.f.x.v v14, v14 \n" + "vfcvt.f.x.v v18, v18 \n" + "vfcvt.f.x.v v12, v12 \n" + "vfmul.vv v16, v16, v14 \n" + "vfmul.vv v18, v18, v12 \n" + "vfdiv.vv v8, v16, v18 \n" + "vfadd.vv v0, v0, v8 \n" + "sub %[w], %[w], %[vl] \n" + "slli %[byte_vl], %[vl], 4 \n" + "add %[row0], %[row0], %[byte_vl] \n" + "add %[row0_next], %[row0_next], %[byte_vl]\n" + "add %[row1], %[row1], %[byte_vl] \n" + "add %[row1_next], %[row1_next], %[byte_vl]\n" + "bgtz %[w], 1b \n" + "vsetvli %[vl], zero, e32, m2, ta, ma \n" + "vfredusum.vs v4, v0, v4 \n" + "vfmv.f.s %[total], v4 \n" + : [row0] "+r"(row0), // %[row0] + [row0_next] "+r"(row0_next), // %[row0_next] + [row1] "+r"(row1), // %[row1] + [row1_next] "+r"(row1_next), // %[row1_next] + [w] "+r"(num_windows), // %[w] + [vl] "=&r"(vl), // %[vl] + [byte_vl] "=&r"(byte_vl), // %[byte_vl] + [total] "=f"(total) // %[total] + : [c1] "r"(26634), // %[c1] + [c2] "r"(239708) // %[c2] + : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v4", "v8", "v9", "v10", + "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31"); + return total; +} +#endif // HAS_SSIMWINDOWROW_RVV + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_sme.cc b/source/row_sme.cc index 4d9beacb6..5ec51382c 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -1134,6 +1134,14 @@ __arm_locally_streaming void Ssim4x4Row_SME(const uint8_t* src_a, } #endif // HAS_SSIM4X4ROW_SME +#ifdef HAS_SSIMWINDOWROW_SME +__arm_locally_streaming float SsimWindowRow_SME(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + return SsimWindowRow_SVE_SC(row0, row1, num_windows); +} +#endif // HAS_SSIMWINDOWROW_SME + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/row_sve.cc b/source/row_sve.cc index 716428096..2cec59d30 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1743,6 +1743,14 @@ void Ssim4x4Row_SVE2(const uint8_t* src_a, } #endif // HAS_SSIM4X4ROW_SVE2 +#ifdef HAS_SSIMWINDOWROW_SVE2 +float SsimWindowRow_SVE2(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + return SsimWindowRow_SVE_SC(row0, row1, num_windows); +} +#endif // HAS_SSIMWINDOWROW_SVE2 + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus