diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index 22336f9a1..301a103e9 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -64,7 +64,7 @@ extern "C" { #define HAS_SSIM4X4ROW_AVX2 #define HAS_SSIMWINDOWROW_AVX2 #endif -#if defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512) +#if defined(CLANG_HAS_AVX512) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW #define HAS_HASHDJB2_AVX512BW @@ -93,18 +93,23 @@ extern "C" { // The following are available for AArch64 SVE: #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #define HAS_SSIM4X4ROW_SVE2 +#define HAS_SSIMWINDOWROW_SVE2 #endif // The following are available for AArch64 SME: #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ defined(__aarch64__) #define HAS_SSIM4X4ROW_SME +#define HAS_SSIMWINDOWROW_SME #endif // The following are available for RISC-V Vector (RVV): #if !defined(LIBYUV_DISABLE_RVV) && \ (defined(__riscv_vector) || defined(LIBYUV_RVV)) #define HAS_SSIM4X4ROW_RVV +#if defined(__riscv_f) || defined(__riscv_zve32f) +#define HAS_SSIMWINDOWROW_RVV +#endif #endif // The following are available for LoongArch LSX: @@ -283,6 +288,15 @@ float SsimWindowRow_AVX2(const uint32_t* row0, float SsimWindowRow_AVX512BW(const uint32_t* row0, const uint32_t* row1, int num_windows); +float SsimWindowRow_SVE2(const uint32_t* row0, + const uint32_t* row1, + int num_windows); +float SsimWindowRow_SME(const uint32_t* row0, + const uint32_t* row1, + int num_windows); +float SsimWindowRow_RVV(const uint32_t* row0, + const uint32_t* row1, + int num_windows); #ifdef __cplusplus } // extern "C" diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index 1dd9d71a4..adb50c42d 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -70,15 +70,6 @@ extern "C" { #endif // clang >= 3.4 #endif // __clang__ -// GCC >= 6.0.0 required for AVX512. -#if defined(__GNUC__) && !defined(__clang__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) && \ - (defined(__x86_64__) || defined(__i386__)) -#if (__GNUC__ >= 6) -#define GCC_HAS_AVX512 1 -#endif // GNUC >= 6 -#endif // __GNUC__ - // clang >= 6.0.0 required for AVX512. #if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \ (defined(__x86_64__) || defined(__i386__)) diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 97616b9a3..19559b64f 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -2285,6 +2285,81 @@ static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a, "z17", "z18", "z19", "z20", "p0"); } +static inline float SsimWindowRow_SVE_SC(const uint32_t* row0, + const uint32_t* row1, + int num_windows) STREAMING_COMPATIBLE { + uint64_t vl; + const uint32_t* row0_next = row0 + 4; + const uint32_t* row1_next = row1 + 4; + float total; + asm volatile( + "cntw %x[vl] \n" + "ptrue p1.s \n" + "mov z24.s, #0 \n" + "dup z25.s, %w[c1] \n" + "dup z26.s, %w[c2] \n" + + "1: \n" + "whilelt p0.s, wzr, %w[num_windows] \n" + "ld4w {z0.s, z1.s, z2.s, z3.s}, p0/z, [%[row0]] \n" + "ld4w {z4.s, z5.s, z6.s, z7.s}, p0/z, [%[row0_next]] \n" + "ld4w {z16.s, z17.s, z18.s, z19.s}, p0/z, [%[row1]] \n" + "ld4w {z20.s, z21.s, z22.s, z23.s}, p0/z, [%[row1_next]] \n" + "incb %[row0], all, mul #4 \n" + "incb %[row0_next], all, mul #4 \n" + "incb %[row1], all, mul #4 \n" + "incb %[row1_next], all, mul #4 \n" + "add z0.s, z0.s, z4.s \n" + "add z1.s, z1.s, z5.s \n" + "add z2.s, z2.s, z6.s \n" + "add z3.s, z3.s, z7.s \n" + "add z16.s, z16.s, z20.s \n" + "add z17.s, z17.s, z21.s \n" + "add z18.s, z18.s, z22.s \n" + "add z19.s, z19.s, z23.s \n" + "add z0.s, z0.s, z16.s \n" + "add z1.s, z1.s, z17.s \n" + "add z2.s, z2.s, z18.s \n" + "add z3.s, z3.s, z19.s \n" + "mul z4.s, z0.s, z1.s \n" + "lsl z4.s, z4.s, #1 \n" + "mul z5.s, z0.s, z0.s \n" + "mla z5.s, p1/m, z1.s, z1.s \n" + "lsl z3.s, z3.s, #7 \n" + "sub z3.s, z3.s, z4.s \n" + "add z4.s, z4.s, z25.s \n" + "add z3.s, z3.s, z26.s \n" + "lsl z2.s, z2.s, #6 \n" + "sub z2.s, z2.s, z5.s \n" + "add z5.s, z5.s, z25.s \n" + "add z2.s, z2.s, z26.s \n" + "scvtf z4.s, p1/m, z4.s \n" + "scvtf z3.s, p1/m, z3.s \n" + "scvtf z5.s, p1/m, z5.s \n" + "scvtf z2.s, p1/m, z2.s \n" + "fmul z4.s, z4.s, z3.s \n" + "fmul z5.s, z5.s, z2.s \n" + "fdiv z4.s, p1/m, z4.s, z5.s \n" + "fadd z24.s, p0/m, z24.s, z4.s \n" + "subs %w[num_windows], %w[num_windows], %w[vl] \n" + "b.gt 1b \n" + + "faddv %s[total], p1, z24.s \n" + : [row0] "+r"(row0), // %[row0] + [row0_next] "+r"(row0_next), // %[row0_next] + [row1] "+r"(row1), // %[row1] + [row1_next] "+r"(row1_next), // %[row1_next] + [num_windows] "+r"(num_windows), // %[num_windows] + [vl] "=&r"(vl), // %[vl] + [total] "=w"(total) // %[total] + : [c1] "r"(26634), // %[c1] + [c2] "r"(239708) // %[c2] + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", + "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", + "p0", "p1"); + return total; +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/compare.cc b/source/compare.cc index 0783cdf52..352cedb1b 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -473,7 +473,7 @@ double CalcFrameSsim(const uint8_t* src_a, #if defined(HAS_SSIMWINDOWROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { SsimWindowRow = SsimWindowRow_AVX512BW; - win_simd_size = 16; + win_simd_size = 1; } #endif #if defined(HAS_SSIM4X4ROW_NEON) @@ -494,18 +494,36 @@ double CalcFrameSsim(const uint8_t* src_a, simd_size = 4; } #endif +#if defined(HAS_SSIMWINDOWROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + SsimWindowRow = SsimWindowRow_SVE2; + win_simd_size = 1; + } +#endif #if defined(HAS_SSIM4X4ROW_SME) if (TestCpuFlag(kCpuHasSME)) { Ssim4x4Row = Ssim4x4Row_SME; simd_size = 4; } #endif +#if defined(HAS_SSIMWINDOWROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + SsimWindowRow = SsimWindowRow_SME; + win_simd_size = 1; + } +#endif #if defined(HAS_SSIM4X4ROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { Ssim4x4Row = Ssim4x4Row_RVV; simd_size = 4; } #endif +#if defined(HAS_SSIMWINDOWROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + SsimWindowRow = SsimWindowRow_RVV; + win_simd_size = 1; + } +#endif #if defined(HAS_SSIM4X4ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { Ssim4x4Row = Ssim4x4Row_LSX; diff --git a/source/compare_gcc.cc b/source/compare_gcc.cc index fcbb87de4..123d92fea 100644 --- a/source/compare_gcc.cc +++ b/source/compare_gcc.cc @@ -28,13 +28,6 @@ extern "C" { !defined(LIBYUV_ENABLE_ROWWIN) && \ (!defined(_MSC_VER) || defined(__clang__)) -#if defined(__clang__) || defined(__GNUC__) -#define LIBYUV_TARGET_AVX512BW \ - __attribute__((target("avx512bw,avx512vl,avx512f"))) -#else -#define LIBYUV_TARGET_AVX512BW -#endif - // "memory" clobber prevents the reads from being removed uint32_t HammingDistance_SSE42(const uint8_t* src_a, @@ -385,7 +378,6 @@ static uint64_t LowBytesMask64(int n) { #ifdef HAS_HAMMINGDISTANCE_AVX512BW // Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked // loads that read zeros, so any count is supported. -LIBYUV_TARGET_AVX512BW uint32_t HammingDistance_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -454,7 +446,6 @@ uint32_t HammingDistance_AVX512BW(const uint8_t* src_a, #ifdef HAS_SUMSQUAREERROR_AVX512BW // Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked // loads that read zeros, so any count is supported. -LIBYUV_TARGET_AVX512BW uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -672,7 +663,6 @@ static uint32_t Pow33(int n) { // Process 64 bytes per loop. The remainder of n = 1 to 63 bytes is loaded // into the high n bytes with a masked load ending at src + count, so the // same multipliers apply, and hash is multiplied by 33 ^ n. -LIBYUV_TARGET_AVX512BW uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) { const int n = count & 63; const uint64_t mask = n ? ~0ULL << (64 - n) : 0ULL; @@ -1012,7 +1002,6 @@ void Ssim4x4Row_AVX2(const uint8_t* src_a, // Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60 // pixels (1 to 15 tiles) with masked loads and stores. -LIBYUV_TARGET_AVX512BW void Ssim4x4Row_AVX512BW(const uint8_t* src_a, int stride_a, const uint8_t* src_b, @@ -1262,15 +1251,52 @@ float SsimWindowRow_AVX2(const uint32_t* row0, #endif // HAS_SSIMWINDOWROW_AVX2 #ifdef HAS_SSIMWINDOWROW_AVX512BW -// num_windows is a multiple of 16. -LIBYUV_TARGET_AVX512BW +#define SSIM_WINDOW_AVX512BW \ + "vpunpckldq %%zmm2,%%zmm1,%%zmm5 \n" \ + "vpunpckhdq %%zmm2,%%zmm1,%%zmm1 \n" \ + "vpunpckldq %%zmm4,%%zmm3,%%zmm2 \n" \ + "vpunpckhdq %%zmm4,%%zmm3,%%zmm3 \n" \ + "vpunpcklqdq %%zmm2,%%zmm5,%%zmm4 \n" \ + "vpunpckhqdq %%zmm2,%%zmm5,%%zmm2 \n" \ + "vpunpcklqdq %%zmm3,%%zmm1,%%zmm5 \n" \ + "vpunpckhqdq %%zmm3,%%zmm1,%%zmm3 \n" \ + "vpmaddwd %%zmm2,%%zmm4,%%zmm1 \n" \ + "vpmaddwd %%zmm4,%%zmm4,%%zmm4 \n" \ + "vpmaddwd %%zmm2,%%zmm2,%%zmm2 \n" \ + "vpaddd %%zmm2,%%zmm4,%%zmm4 \n" \ + "vpslld $0x1,%%zmm1,%%zmm1 \n" \ + "vpslld $0x7,%%zmm3,%%zmm3 \n" \ + "vpsubd %%zmm1,%%zmm3,%%zmm3 \n" \ + "vpslld $0x6,%%zmm5,%%zmm5 \n" \ + "vpsubd %%zmm4,%%zmm5,%%zmm5 \n" \ + "vpaddd %[c1]%{1to16%},%%zmm1,%%zmm1 \n" \ + "vpaddd %[c2]%{1to16%},%%zmm3,%%zmm3 \n" \ + "vpaddd %[c1]%{1to16%},%%zmm4,%%zmm4 \n" \ + "vpaddd %[c2]%{1to16%},%%zmm5,%%zmm5 \n" \ + "vcvtdq2ps %%zmm1,%%zmm1 \n" \ + "vcvtdq2ps %%zmm3,%%zmm3 \n" \ + "vcvtdq2ps %%zmm4,%%zmm4 \n" \ + "vcvtdq2ps %%zmm5,%%zmm5 \n" \ + "vmulps %%zmm3,%%zmm1,%%zmm1 \n" \ + "vmulps %%zmm5,%%zmm4,%%zmm4 \n" \ + "vdivps %%zmm4,%%zmm1,%%zmm1 \n" + +// Process 16 8x8 windows per loop and the remainder of 1 to 15 windows with +// masked loads and masked accumulation. float SsimWindowRow_AVX512BW(const uint32_t* row0, const uint32_t* row1, int num_windows) { + const int rem = num_windows & 15; + const uint64_t load_mask = rem ? (~0ULL >> (60 - rem * 4)) : 0ULL; + const uint16_t win_mask = + (uint16_t)(((0x01110U >> (16 - (rem & 3) * 4)) << (rem >> 2)) + + (0x1111U << (rem >> 2)) - 0x1111U); float total; asm volatile( "sub %[row0],%[row1] \n" - "vxorps %g[total],%g[total],%g[total] \n" + "vxorps %%xmm0,%%xmm0,%%xmm0 \n" + "sub $0x10,%[num_windows] \n" + "jl 2f \n" LABELALIGN "1: \n" @@ -1291,54 +1317,57 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, "vmovdqu32 0xd0(%[row0]),%%zmm5 \n" "vpaddd 0xd0(%[row0],%[row1]),%%zmm5,%%zmm5 \n" "add $0x100,%[row0] \n" - "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" - "vpunpckldq %%zmm2,%%zmm1,%%zmm5 \n" - "vpunpckhdq %%zmm2,%%zmm1,%%zmm1 \n" - "vpunpckldq %%zmm4,%%zmm3,%%zmm2 \n" - "vpunpckhdq %%zmm4,%%zmm3,%%zmm3 \n" - "vpunpcklqdq %%zmm2,%%zmm5,%%zmm4 \n" - "vpunpckhqdq %%zmm2,%%zmm5,%%zmm2 \n" - "vpunpcklqdq %%zmm3,%%zmm1,%%zmm5 \n" - "vpunpckhqdq %%zmm3,%%zmm1,%%zmm3 \n" - "vpmaddwd %%zmm2,%%zmm4,%%zmm1 \n" - "vpmaddwd %%zmm4,%%zmm4,%%zmm4 \n" - "vpmaddwd %%zmm2,%%zmm2,%%zmm2 \n" - "vpaddd %%zmm2,%%zmm4,%%zmm4 \n" - "vpslld $0x1,%%zmm1,%%zmm1 \n" - "vpslld $0x7,%%zmm3,%%zmm3 \n" - "vpsubd %%zmm1,%%zmm3,%%zmm3 \n" - "vpslld $0x6,%%zmm5,%%zmm5 \n" - "vpsubd %%zmm4,%%zmm5,%%zmm5 \n" - "vpaddd %[c1]%{1to16%},%%zmm1,%%zmm1 \n" - "vpaddd %[c2]%{1to16%},%%zmm3,%%zmm3 \n" - "vpaddd %[c1]%{1to16%},%%zmm4,%%zmm4 \n" - "vpaddd %[c2]%{1to16%},%%zmm5,%%zmm5 \n" - "vcvtdq2ps %%zmm1,%%zmm1 \n" - "vcvtdq2ps %%zmm3,%%zmm3 \n" - "vcvtdq2ps %%zmm4,%%zmm4 \n" - "vcvtdq2ps %%zmm5,%%zmm5 \n" - "vmulps %%zmm3,%%zmm1,%%zmm1 \n" - "vmulps %%zmm5,%%zmm4,%%zmm4 \n" - "vdivps %%zmm4,%%zmm1,%%zmm1 \n" - "vaddps %%zmm1,%g[total],%g[total] \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" // + SSIM_WINDOW_AVX512BW + "vaddps %%zmm1,%%zmm0,%%zmm0 \n" "sub $0x10,%[num_windows] \n" - "jg 1b \n" + "jge 1b \n" - "vextractf32x8 $0x1,%g[total],%%ymm1 \n" - "vaddps %%ymm1,%t[total],%t[total] \n" - "vextractf128 $0x1,%t[total],%%xmm1 \n" - "vaddps %%xmm1,%x[total],%x[total] \n" - "vmovhlps %x[total],%%xmm1,%%xmm1 \n" - "vaddps %%xmm1,%x[total],%x[total] \n" - "vshufps $0x1,%x[total],%x[total],%%xmm1 \n" - "vaddss %%xmm1,%x[total],%x[total] \n" + "2: \n" + "add $0x10,%[num_windows] \n" + "je 3f \n" + "kmovq %[load_mask],%%k1 \n" + "vmovdqu32 0x00(%[row0]),%%zmm1%{%%k1%}%{z%} \n" + "vpaddd 0x00(%[row0],%[row1]),%%zmm1,%%zmm1%{%%k1%}%{z%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 0x40(%[row0]),%%zmm2%{%%k1%}%{z%} \n" + "vpaddd 0x40(%[row0],%[row1]),%%zmm2,%%zmm2%{%%k1%}%{z%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 0x80(%[row0]),%%zmm3%{%%k1%}%{z%} \n" + "vpaddd 0x80(%[row0],%[row1]),%%zmm3,%%zmm3%{%%k1%}%{z%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 0xc0(%[row0]),%%zmm4%{%%k1%}%{z%} \n" + "vpaddd 0xc0(%[row0],%[row1]),%%zmm4,%%zmm4%{%%k1%}%{z%} \n" + "valignd $0x4,%%zmm1,%%zmm2,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm1,%%zmm1 \n" + "valignd $0x4,%%zmm2,%%zmm3,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm2,%%zmm2 \n" + "valignd $0x4,%%zmm3,%%zmm4,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm3,%%zmm3 \n" + "valignd $0x4,%%zmm4,%%zmm4,%%zmm5 \n" + "vpaddd %%zmm5,%%zmm4,%%zmm4 \n" // + SSIM_WINDOW_AVX512BW + "kmovw %[win_mask],%%k1 \n" + "vaddps %%zmm1,%%zmm0,%%zmm0%{%%k1%} \n" + + "3: \n" + "vextractf32x8 $0x1,%%zmm0,%%ymm1 \n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" + "vextractf128 $0x1,%%ymm0,%%xmm1 \n" + "vaddps %%xmm1,%%xmm0,%%xmm0 \n" + "vmovhlps %%xmm0,%%xmm1,%%xmm1 \n" + "vaddps %%xmm1,%%xmm0,%%xmm0 \n" + "vshufps $0x1,%%xmm0,%%xmm0,%%xmm1 \n" + "vaddss %%xmm1,%%xmm0,%[total] \n" "vzeroupper \n" : [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows), - [total] "=&x"(total) - : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2) - : "memory", "cc", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + [total] "=x"(total) + : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2), [load_mask] "m"(load_mask), + [win_mask] "m"(win_mask) + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); return total; } +#undef SSIM_WINDOW_AVX512BW #endif // HAS_SSIMWINDOWROW_AVX512BW #endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && diff --git a/source/compare_win.cc b/source/compare_win.cc index c6a811901..d1166ad46 100644 --- a/source/compare_win.cc +++ b/source/compare_win.cc @@ -812,7 +812,6 @@ float SsimWindowRow_AVX2(const uint32_t* row0, #endif // HAS_SSIMWINDOWROW_AVX2 #if defined(HAS_SSIMWINDOWROW_AVX512BW) -// num_windows is a multiple of 16. LIBYUV_TARGET_AVX512BW float SsimWindowRow_AVX512BW(const uint32_t* row0, const uint32_t* row1, @@ -820,8 +819,29 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, const __m512i vc1 = _mm512_set1_epi32(26634); const __m512i vc2 = _mm512_set1_epi32(239708); __m512 total = _mm512_setzero_ps(); - int w; - for (w = 0; w < num_windows; w += 16) { +#define SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3) \ + __m512i u0 = _mm512_unpacklo_epi32(w0, w1); \ + __m512i u1 = _mm512_unpackhi_epi32(w0, w1); \ + __m512i u2 = _mm512_unpacklo_epi32(w2, w3); \ + __m512i u3 = _mm512_unpackhi_epi32(w2, w3); \ + __m512i sum_a = _mm512_unpacklo_epi64(u0, u2); \ + __m512i sum_b = _mm512_unpackhi_epi64(u0, u2); \ + __m512i sum_sq = _mm512_unpacklo_epi64(u1, u3); \ + __m512i sum_axb = _mm512_unpackhi_epi64(u1, u3); \ + __m512i ab2 = _mm512_slli_epi32(_mm512_madd_epi16(sum_a, sum_b), 1); \ + __m512i a2b2 = _mm512_add_epi32(_mm512_madd_epi16(sum_a, sum_a), \ + _mm512_madd_epi16(sum_b, sum_b)); \ + __m512i n1 = _mm512_add_epi32(ab2, vc1); \ + __m512i n2 = _mm512_add_epi32( \ + _mm512_sub_epi32(_mm512_slli_epi32(sum_axb, 7), ab2), vc2); \ + __m512i d1 = _mm512_add_epi32(a2b2, vc1); \ + __m512i d2 = _mm512_add_epi32( \ + _mm512_sub_epi32(_mm512_slli_epi32(sum_sq, 6), a2b2), vc2); \ + __m512 num = _mm512_mul_ps(_mm512_cvtepi32_ps(n1), _mm512_cvtepi32_ps(n2)); \ + __m512 den = _mm512_mul_ps(_mm512_cvtepi32_ps(d1), _mm512_cvtepi32_ps(d2)); \ + __m512 ssim = _mm512_div_ps(num, den) + + while (num_windows >= 16) { __m512i t0 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 0), _mm512_loadu_si512(row1 + 0)); __m512i t1 = _mm512_add_epi32(_mm512_loadu_si512(row0 + 16), @@ -834,34 +854,40 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, _mm512_loadu_si512(row1 + 52)); row0 += 64; row1 += 64; + num_windows -= 16; __m512i w0 = _mm512_add_epi32(t0, _mm512_alignr_epi32(t1, t0, 4)); __m512i w1 = _mm512_add_epi32(t1, _mm512_alignr_epi32(t2, t1, 4)); __m512i w2 = _mm512_add_epi32(t2, _mm512_alignr_epi32(t3, t2, 4)); __m512i w3 = _mm512_add_epi32(t3, t4); - - __m512i u0 = _mm512_unpacklo_epi32(w0, w1); - __m512i u1 = _mm512_unpackhi_epi32(w0, w1); - __m512i u2 = _mm512_unpacklo_epi32(w2, w3); - __m512i u3 = _mm512_unpackhi_epi32(w2, w3); - __m512i sum_a = _mm512_unpacklo_epi64(u0, u2); - __m512i sum_b = _mm512_unpackhi_epi64(u0, u2); - __m512i sum_sq = _mm512_unpacklo_epi64(u1, u3); - __m512i sum_axb = _mm512_unpackhi_epi64(u1, u3); - - __m512i ab2 = _mm512_slli_epi32(_mm512_madd_epi16(sum_a, sum_b), 1); - __m512i a2b2 = _mm512_add_epi32(_mm512_madd_epi16(sum_a, sum_a), - _mm512_madd_epi16(sum_b, sum_b)); - __m512i n1 = _mm512_add_epi32(ab2, vc1); - __m512i n2 = _mm512_add_epi32( - _mm512_sub_epi32(_mm512_slli_epi32(sum_axb, 7), ab2), vc2); - __m512i d1 = _mm512_add_epi32(a2b2, vc1); - __m512i d2 = _mm512_add_epi32( - _mm512_sub_epi32(_mm512_slli_epi32(sum_sq, 6), a2b2), vc2); - - __m512 num = _mm512_mul_ps(_mm512_cvtepi32_ps(n1), _mm512_cvtepi32_ps(n2)); - __m512 den = _mm512_mul_ps(_mm512_cvtepi32_ps(d1), _mm512_cvtepi32_ps(d2)); - total = _mm512_add_ps(total, _mm512_div_ps(num, den)); + SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3); + total = _mm512_add_ps(total, ssim); } + if (num_windows > 0) { + const uint64_t load_mask = ~0ULL >> (60 - num_windows * 4); + const __mmask16 win_mask = + (__mmask16)(((0x01110U >> (16 - (num_windows & 3) * 4)) + << (num_windows >> 2)) + + (0x1111U << (num_windows >> 2)) - 0x1111U); + const __mmask16 m0 = (__mmask16)load_mask; + const __mmask16 m1 = (__mmask16)(load_mask >> 16); + const __mmask16 m2 = (__mmask16)(load_mask >> 32); + const __mmask16 m3 = (__mmask16)(load_mask >> 48); + __m512i t0 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m0, row0 + 0), + _mm512_maskz_loadu_epi32(m0, row1 + 0)); + __m512i t1 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m1, row0 + 16), + _mm512_maskz_loadu_epi32(m1, row1 + 16)); + __m512i t2 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m2, row0 + 32), + _mm512_maskz_loadu_epi32(m2, row1 + 32)); + __m512i t3 = _mm512_add_epi32(_mm512_maskz_loadu_epi32(m3, row0 + 48), + _mm512_maskz_loadu_epi32(m3, row1 + 48)); + __m512i w0 = _mm512_add_epi32(t0, _mm512_alignr_epi32(t1, t0, 4)); + __m512i w1 = _mm512_add_epi32(t1, _mm512_alignr_epi32(t2, t1, 4)); + __m512i w2 = _mm512_add_epi32(t2, _mm512_alignr_epi32(t3, t2, 4)); + __m512i w3 = _mm512_add_epi32(t3, _mm512_alignr_epi32(t3, t3, 4)); + SSIM_WINDOW_AVX512_WIN(w0, w1, w2, w3); + total = _mm512_mask_add_ps(total, win_mask, total, ssim); + } +#undef SSIM_WINDOW_AVX512_WIN return _mm512_reduce_add_ps(total); } #endif // HAS_SSIMWINDOWROW_AVX512BW diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 363cbec9b..6b7173675 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -2851,6 +2851,85 @@ void Ssim4x4Row_RVV(const uint8_t* src_a, #undef SSIM4X4_ACCUM_RVV #endif // HAS_SSIM4X4ROW_RVV +#ifdef HAS_SSIMWINDOWROW_RVV +float SsimWindowRow_RVV(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + size_t vl; + size_t byte_vl; + const uint32_t* row0_next = row0 + 4; + const uint32_t* row1_next = row1 + 4; + float total; + asm volatile( + "vsetvli %[vl], zero, e32, m2, ta, ma \n" + "vmv.v.i v0, 0 \n" + "vmv.s.x v4, zero \n" + "1: \n" + "vsetvli %[vl], %[w], e32, m2, tu, ma \n" + "vlseg4e32.v v8, (%[row0]) \n" + "vlseg4e32.v v16, (%[row0_next]) \n" + "vadd.vv v8, v8, v16 \n" + "vadd.vv v10, v10, v18 \n" + "vadd.vv v12, v12, v20 \n" + "vadd.vv v14, v14, v22 \n" + "vlseg4e32.v v16, (%[row1]) \n" + "vlseg4e32.v v24, (%[row1_next]) \n" + "vadd.vv v16, v16, v24 \n" + "vadd.vv v18, v18, v26 \n" + "vadd.vv v20, v20, v28 \n" + "vadd.vv v22, v22, v30 \n" + "vadd.vv v8, v8, v16 \n" + "vadd.vv v10, v10, v18 \n" + "vadd.vv v12, v12, v20 \n" + "vadd.vv v14, v14, v22 \n" + "vmul.vv v16, v8, v10 \n" + "vsll.vi v16, v16, 1 \n" + "vmul.vv v18, v8, v8 \n" + "vmacc.vv v18, v10, v10 \n" + "vsll.vi v14, v14, 7 \n" + "vsub.vv v14, v14, v16 \n" + "vadd.vx v16, v16, %[c1] \n" + "vadd.vx v14, v14, %[c2] \n" + "vsll.vi v12, v12, 6 \n" + "vsub.vv v12, v12, v18 \n" + "vadd.vx v18, v18, %[c1] \n" + "vadd.vx v12, v12, %[c2] \n" + "vfcvt.f.x.v v16, v16 \n" + "vfcvt.f.x.v v14, v14 \n" + "vfcvt.f.x.v v18, v18 \n" + "vfcvt.f.x.v v12, v12 \n" + "vfmul.vv v16, v16, v14 \n" + "vfmul.vv v18, v18, v12 \n" + "vfdiv.vv v8, v16, v18 \n" + "vfadd.vv v0, v0, v8 \n" + "sub %[w], %[w], %[vl] \n" + "slli %[byte_vl], %[vl], 4 \n" + "add %[row0], %[row0], %[byte_vl] \n" + "add %[row0_next], %[row0_next], %[byte_vl]\n" + "add %[row1], %[row1], %[byte_vl] \n" + "add %[row1_next], %[row1_next], %[byte_vl]\n" + "bgtz %[w], 1b \n" + "vsetvli %[vl], zero, e32, m2, ta, ma \n" + "vfredusum.vs v4, v0, v4 \n" + "vfmv.f.s %[total], v4 \n" + : [row0] "+r"(row0), // %[row0] + [row0_next] "+r"(row0_next), // %[row0_next] + [row1] "+r"(row1), // %[row1] + [row1_next] "+r"(row1_next), // %[row1_next] + [w] "+r"(num_windows), // %[w] + [vl] "=&r"(vl), // %[vl] + [byte_vl] "=&r"(byte_vl), // %[byte_vl] + [total] "=f"(total) // %[total] + : [c1] "r"(26634), // %[c1] + [c2] "r"(239708) // %[c2] + : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v4", "v8", "v9", "v10", + "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31"); + return total; +} +#endif // HAS_SSIMWINDOWROW_RVV + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_sme.cc b/source/row_sme.cc index 4d9beacb6..5ec51382c 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -1134,6 +1134,14 @@ __arm_locally_streaming void Ssim4x4Row_SME(const uint8_t* src_a, } #endif // HAS_SSIM4X4ROW_SME +#ifdef HAS_SSIMWINDOWROW_SME +__arm_locally_streaming float SsimWindowRow_SME(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + return SsimWindowRow_SVE_SC(row0, row1, num_windows); +} +#endif // HAS_SSIMWINDOWROW_SME + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/row_sve.cc b/source/row_sve.cc index 716428096..2cec59d30 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1743,6 +1743,14 @@ void Ssim4x4Row_SVE2(const uint8_t* src_a, } #endif // HAS_SSIM4X4ROW_SVE2 +#ifdef HAS_SSIMWINDOWROW_SVE2 +float SsimWindowRow_SVE2(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + return SsimWindowRow_SVE_SC(row0, row1, num_windows); +} +#endif // HAS_SSIMWINDOWROW_SVE2 + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus