From 147a28c4aac51d0c384f433fad9059f951999af0 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Thu, 15 Oct 2026 15:33:21 -0700 Subject: [PATCH] [libyuv] Optimize CalcFrameSsim with 2-row 4x4 tile buffer and SIMD Refactor CalcFrameSsim to decompose overlapping 8x8 SSIM windows (stepped by 4 pixels horizontally and vertically) into non-overlapping 4x4 pixel tiles across a 2-row rotating buffer (row0, row1), reducing memory reads and pixel accumulation work by 4x: - Ssim4x4Row_* computes 4 uint32_t stats {sum_a, sum_b, sum_sq_ab, sum_axb} per 4x4 pixel tile across a 4-row strip (also sized to fit future 10-bit/12-bit formats like I010 without uint32_t overflow). - SsimWindowRow_C combines top and bottom 4x4 tile strips across adjacent horizontal pairs and returns the exact double SSIM sum, producing bit-identical results to Ssim8x8_C. - CalcFrameSsim rotates row0 and row1 (with thread-local buffers and continuity tracking under _OPENMP) and falls back to Ssim8x8_C only if buffer allocation fails. - Implement Ssim4x4Row across all SIMD architectures: - x86/x64: Ssim4x4Row_SSE2, Ssim4x4Row_AVX2, and Ssim4x4Row_AVX512BW (with k-masked tail handling for 4..60 pixels) in both compare_gcc.cc and compare_win.cc. - ARM: Ssim4x4Row_NEON (compare_neon.cc, compare_neon64.cc), Ssim4x4Row_NEON_DotProd (compare_neon64.cc), Ssim4x4Row_SVE2 (row_sve.h, row_sve.cc), and Ssim4x4Row_SME (row_sme.cc). - RISC-V: Ssim4x4Row_RVV (row_rvv.cc). - LoongArch: Ssim4x4Row_LSX (row_lsx.cc) and Ssim4x4Row_LASX (row_lasx.cc). - WebAssembly: Ssim4x4Row_WASMSIMD (compare_wasm.cc). - Enable BenchmarkSsim_Opt and add BenchmarkSsim_Unaligned and TestSsimAnyWidth in unit_test/compare_test.cc. BenchmarkSsim_Opt (1280x720, single core, us/frame): uArch Before (C) After (us) Improvement -------------------------------- ---------- ----------- ----------- Intel Lion Cove (AVX2) 356.33 119.09 2.99x AMD Zen 6 (AVX512BW) 455.86 124.67 3.66x AMD Zen 5 (AVX512BW) 518.91 128.29 4.04x Intel Emerald Rapids (AVX512BW) 601.58 180.41 3.33x Intel Cougar Cove (AVX2) 608.11 183.34 3.32x Intel Granite Rapids (AVX512BW) 596.57 187.04 3.19x Intel Golden Cove (AVX2) 534.58 187.93 2.84x Intel Sapphire Rapids (AVX512BW) 634.36 189.92 3.34x AMD Zen 4 (AVX512BW) 713.06 223.65 3.19x Intel Skymont (SSE2) 445.99 250.52 1.78x Intel Ice Lake (AVX512BW) 845.16 267.23 3.16x AMD Zen 3 (AVX2) 741.02 267.38 2.77x AMD Zen 2 (AVX2) 907.04 301.02 3.01x Intel Skylake-X (AVX512BW) 1097.86 301.63 3.64x Intel Darkmont (SSE2) 772.18 383.39 2.01x Intel Gracemont (SSE2) 961.80 421.83 2.28x Intel Haswell (AVX2) 1159.74 479.14 2.42x Intel Ivy Bridge (SSE2) 1296.99 597.32 2.17x Apple M6 Super (SME) 404.66 90.09 4.49x Apple M6 Perf (DotProd) 444.72 104.59 4.25x Qualcomm Oryon-L (DotProd) 490.76 108.79 4.51x Qualcomm Oryon V3-L (SVE2) 663.13 150.75 4.40x ARM Cortex-X4 (DotProd) 620.21 177.29 3.50x ARM Neoverse-V2 (SVE2) 682.93 205.18 3.33x ARM Cortex-X3 (DotProd) 737.38 226.24 3.26x ARM C1-Ultra (DotProd) 596.71 229.10 2.60x ARM Cortex-X2 (DotProd) 820.94 247.94 3.31x Qualcomm Oryon-M (DotProd) 1146.09 257.89 4.44x Qualcomm Oryon V3-M (SME) 1068.78 276.23 3.87x ARM Cortex-A720 (DotProd) 993.59 292.62 3.40x ARM Cortex-X1 (DotProd) 912.19 326.44 2.79x ARM Cortex-A715 (DotProd) 1135.67 340.12 3.34x ARM Cortex-A710 (DotProd) 1222.04 342.51 3.57x ARM Neoverse-N1 (DotProd) 1406.72 376.46 3.74x ARM Cortex-A725 (SVE2) 1104.13 376.86 2.93x ARM C1-Pro (SVE2) 1073.69 413.97 2.59x ARM Cortex-A76 (DotProd) 1491.02 423.58 3.52x ARM Cortex-A78 (DotProd) 1503.73 485.00 3.10x AmpereOne (DotProd) 1518.61 518.40 2.93x Apple M6 Eff (DotProd) 4051.10 572.93 7.07x ARM Cortex-A75 (NEON) 2055.04 812.09 2.53x ARM Cortex-A520 (DotProd) 3307.98 963.86 3.43x ARM Cortex-A73 (NEON) 3120.09 1180.17 2.64x Qualcomm Kryo Perf (NEON) 4921.16 1459.77 3.37x ARM Cortex-A510 (DotProd) 4645.10 1641.06 2.83x ARM Cortex-A55 (DotProd) 4571.02 1672.09 2.73x Qualcomm Kryo Eff (NEON) 6000.96 1796.17 3.34x ARM Cortex-A55r0 (NEON) 10486.61 4358.61 2.41x ARM Cortex-A53 (NEON) 10121.34 5700.99 1.78x R=rrwinterton@gmail.com Bug: None Test: out/Release/libyuv_unittest --gtest_filter=*Ssim* Change-Id: Ibb95ae74424ecdda073ab7ab10ab1cf610b121b6 --- include/libyuv/compare_row.h | 112 +++++++++++ include/libyuv/row_sve.h | 87 ++++++++ source/compare.cc | 147 ++++++++++++-- source/compare_common.cc | 104 ++++++++++ source/compare_gcc.cc | 376 ++++++++++++++++++++++++++++++++++- source/compare_neon.cc | 62 ++++++ source/compare_neon64.cc | 129 ++++++++++++ source/compare_wasm.cc | 112 +++++++++++ source/compare_win.cc | 255 ++++++++++++++++++++++++ source/row_lasx.cc | 128 ++++++++++++ source/row_lsx.cc | 114 +++++++++++ source/row_rvv.cc | 112 +++++++++++ source/row_sme.cc | 12 ++ source/row_sve.cc | 12 ++ unit_test/compare_test.cc | 53 ++++- 15 files changed, 1800 insertions(+), 15 deletions(-) diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index 3c7722baf..6a66f0785 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -32,12 +32,15 @@ extern "C" { #define HAS_HAMMINGDISTANCE_SSE42 #define HAS_SUMSQUAREERROR_SSE2 #define HAS_HASHDJB2_SSE41 +#define HAS_SSIM4X4ROW_SSE2 #define HAS_HAMMINGDISTANCE_AVX2 #define HAS_SUMSQUAREERROR_AVX2 #define HAS_HASHDJB2_AVX2 +#define HAS_SSIM4X4ROW_AVX2 #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW #define HAS_HASHDJB2_AVX512BW +#define HAS_SSIM4X4ROW_AVX512BW #endif // The following are available for GCC and clang 32/64 bit, as inline assembly @@ -49,15 +52,18 @@ extern "C" { #define HAS_HAMMINGDISTANCE_SSSE3 #define HAS_SUMSQUAREERROR_SSE2 #define HAS_HASHDJB2_SSE41 +#define HAS_SSIM4X4ROW_SSE2 #if defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2) #define HAS_HAMMINGDISTANCE_AVX2 #define HAS_SUMSQUAREERROR_AVX2 #define HAS_HASHDJB2_AVX2 +#define HAS_SSIM4X4ROW_AVX2 #endif #if defined(CLANG_HAS_AVX512) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW #define HAS_HASHDJB2_AVX512BW +#define HAS_SSIM4X4ROW_AVX512BW #endif #endif @@ -66,6 +72,7 @@ extern "C" { (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) #define HAS_HAMMINGDISTANCE_NEON #define HAS_SUMSQUAREERROR_NEON +#define HAS_SSIM4X4ROW_NEON #endif // The following are available for AArch64 Neon: @@ -74,6 +81,34 @@ extern "C" { #define HAS_HAMMINGDISTANCE_NEON_DOTPROD #define HAS_SUMSQUAREERROR_NEON_DOTPROD +#define HAS_SSIM4X4ROW_NEON_DOTPROD +#endif + +// The following are available for AArch64 SVE: +#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) +#define HAS_SSIM4X4ROW_SVE2 +#endif + +// The following are available for AArch64 SME: +#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ + defined(__aarch64__) +#define HAS_SSIM4X4ROW_SME +#endif + +// The following are available for RISC-V Vector (RVV): +#if !defined(LIBYUV_DISABLE_RVV) && \ + (defined(__riscv_vector) || defined(LIBYUV_RVV)) +#define HAS_SSIM4X4ROW_RVV +#endif + +// The following are available for LoongArch LSX: +#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) +#define HAS_SSIM4X4ROW_LSX +#endif + +// The following are available for LoongArch LASX: +#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#define HAS_SSIM4X4ROW_LASX #endif // The following are available for WebAssembly SIMD128: @@ -81,6 +116,7 @@ extern "C" { #define HAS_HAMMINGDISTANCE_WASMSIMD #define HAS_SUMSQUAREERROR_WASMSIMD #define HAS_HASHDJB2_WASMSIMD +#define HAS_SSIM4X4ROW_WASMSIMD #endif uint32_t HammingDistance_C(const uint8_t* src_a, @@ -136,6 +172,82 @@ uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed); uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed); uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed); +void Ssim4x4Row_C(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_SSE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_AVX2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_AVX512BW(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_NEON(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_NEON_DotProd(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_SVE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_SME(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_RVV(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_LSX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_LASX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_WASMSIMD(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +double SsimWindowRow_C(const uint32_t* row0, + const uint32_t* row1, + int num_windows); + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 467ec199b..7e0ca8776 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -2224,6 +2224,93 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, "z20", "z24", "z25", "p0", "p1", "p2", "p3", "p4", "p5"); } +#define SSIM4X4_SVE \ + "add %[tmp_a], %[src_a], %[stride_a] \n" \ + "add %[tmp_b], %[src_b], %[stride_b] \n" \ + "ld1w {z0.s}, p0/z, [%[src_a]] \n" \ + "ld1w {z1.s}, p0/z, [%[tmp_a]] \n" \ + "add %[tmp_a], %[tmp_a], %[stride_a] \n" \ + "ld1w {z4.s}, p0/z, [%[src_b]] \n" \ + "ld1w {z5.s}, p0/z, [%[tmp_b]] \n" \ + "add %[tmp_b], %[tmp_b], %[stride_b] \n" \ + "ld1w {z2.s}, p0/z, [%[tmp_a]] \n" \ + "add %[tmp_a], %[tmp_a], %[stride_a] \n" \ + "ld1w {z3.s}, p0/z, [%[tmp_a]] \n" \ + "ld1w {z6.s}, p0/z, [%[tmp_b]] \n" \ + "add %[tmp_b], %[tmp_b], %[stride_b] \n" \ + "ld1w {z7.s}, p0/z, [%[tmp_b]] \n" \ + "incb %[src_a] \n" \ + "incb %[src_b] \n" \ + "mov z16.s, #0 \n" \ + "mov z17.s, #0 \n" \ + "mov z18.s, #0 \n" \ + "mov z19.s, #0 \n" \ + "udot z16.s, z0.b, z20.b \n" \ + "udot z17.s, z4.b, z20.b \n" \ + "udot z18.s, z0.b, z0.b \n" \ + "udot z19.s, z0.b, z4.b \n" \ + "udot z16.s, z1.b, z20.b \n" \ + "udot z17.s, z5.b, z20.b \n" \ + "udot z18.s, z1.b, z1.b \n" \ + "udot z19.s, z1.b, z5.b \n" \ + "udot z16.s, z2.b, z20.b \n" \ + "udot z17.s, z6.b, z20.b \n" \ + "udot z18.s, z2.b, z2.b \n" \ + "udot z19.s, z2.b, z6.b \n" \ + "udot z16.s, z3.b, z20.b \n" \ + "udot z17.s, z7.b, z20.b \n" \ + "udot z18.s, z3.b, z3.b \n" \ + "udot z19.s, z3.b, z7.b \n" \ + "udot z18.s, z4.b, z4.b \n" \ + "udot z18.s, z5.b, z5.b \n" \ + "udot z18.s, z6.b, z6.b \n" \ + "udot z18.s, z7.b, z7.b \n" \ + "subs %w[width], %w[width], %w[vl] \n" \ + "st4w {z16.s, z17.s, z18.s, z19.s}, p0, [%[dst]]\n" \ + "incb %[dst], all, mul #4 \n" + +static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) STREAMING_COMPATIBLE { + uint64_t vl; + const uint8_t* tmp_a; + const uint8_t* tmp_b; + width >>= 2; + asm volatile( + "cntw %x[vl] \n" + "mov z20.b, #1 \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.s \n" + "1: \n" // + SSIM4X4_SVE + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[width] \n" // + SSIM4X4_SVE + + "99: \n" + : [src_a] "+r"(src_a), // %[src_a] + [src_b] "+r"(src_b), // %[src_b] + [dst] "+r"(dst), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [tmp_a] "=&r"(tmp_a), // %[tmp_a] + [tmp_b] "=&r"(tmp_b) // %[tmp_b] + : [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a] + [stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b] + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", + "z17", "z18", "z19", "z20", "p0"); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/compare.cc b/source/compare.cc index 9c3a5ed16..5828bd15f 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -427,20 +427,143 @@ double CalcFrameSsim(const uint8_t* src_a, int height) { int samples = 0; double ssim_total = 0; - double (*Ssim8x8)(const uint8_t* src_a, int stride_a, const uint8_t* src_b, - int stride_b) = Ssim8x8_C; + if (width <= 8 || height <= 8) { + return ssim_total / samples; + } - // sample point start with each 4x4 location - int i; - for (i = 0; i < height - 8; i += 4) { - int j; - for (j = 0; j < width - 8; j += 4) { - ssim_total += Ssim8x8(src_a + j, stride_a, src_b + j, stride_b); - samples++; + int simd_size = 4; + void (*Ssim4x4Row)(const uint8_t* src_a, int stride_a, const uint8_t* src_b, + int stride_b, uint32_t* dst, int width) = Ssim4x4Row_C; +#if defined(HAS_SSIM4X4ROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + Ssim4x4Row = Ssim4x4Row_SSE2; + simd_size = 16; + } +#endif +#if defined(HAS_SSIM4X4ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + Ssim4x4Row = Ssim4x4Row_AVX2; + simd_size = 32; + } +#endif +#if defined(HAS_SSIM4X4ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + Ssim4x4Row = Ssim4x4Row_AVX512BW; + simd_size = 4; + } +#endif +#if defined(HAS_SSIM4X4ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + Ssim4x4Row = Ssim4x4Row_NEON; + simd_size = 16; + } +#endif +#if defined(HAS_SSIM4X4ROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + Ssim4x4Row = Ssim4x4Row_NEON_DotProd; + simd_size = 16; + } +#endif +#if defined(HAS_SSIM4X4ROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + Ssim4x4Row = Ssim4x4Row_SVE2; + simd_size = 4; + } +#endif +#if defined(HAS_SSIM4X4ROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + Ssim4x4Row = Ssim4x4Row_SME; + simd_size = 4; + } +#endif +#if defined(HAS_SSIM4X4ROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + Ssim4x4Row = Ssim4x4Row_RVV; + simd_size = 4; + } +#endif +#if defined(HAS_SSIM4X4ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Ssim4x4Row = Ssim4x4Row_LSX; + simd_size = 16; + } +#endif +#if defined(HAS_SSIM4X4ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Ssim4x4Row = Ssim4x4Row_LASX; + simd_size = 32; + } +#endif +#if defined(HAS_SSIM4X4ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Ssim4x4Row = Ssim4x4Row_WASMSIMD; + simd_size = 16; + } +#endif + + const int num_windows = (width - 8 + 3) >> 2; + const int num_tiles = num_windows + 1; + const int tile_width = num_tiles * 4; + const int main_width = tile_width & ~(simd_size - 1); + const int rem_width = tile_width & (simd_size - 1); + +#ifdef _OPENMP +#pragma omp parallel reduction(+ : ssim_total, samples) +#endif + { + align_buffer_64(row_buf, (size_t)tile_width * 2 * sizeof(uint32_t)); + if (row_buf) { + uint32_t* row0 = (uint32_t*)row_buf; + uint32_t* row1 = row0 + tile_width; + int prev_i = -8; + int i; +#ifdef _OPENMP +#pragma omp for schedule(static) +#endif + for (i = 0; i < height - 8; i += 4) { + const uint8_t* a = src_a + (ptrdiff_t)i * stride_a; + const uint8_t* b = src_b + (ptrdiff_t)i * stride_b; + if (prev_i + 4 != i) { + if (main_width > 0) { + Ssim4x4Row(a, stride_a, b, stride_b, row0, main_width); + } + if (rem_width > 0) { + Ssim4x4Row_C(a + main_width, stride_a, b + main_width, stride_b, + row0 + main_width, rem_width); + } + } + a += (ptrdiff_t)stride_a * 4; + b += (ptrdiff_t)stride_b * 4; + if (main_width > 0) { + Ssim4x4Row(a, stride_a, b, stride_b, row1, main_width); + } + if (rem_width > 0) { + Ssim4x4Row_C(a + main_width, stride_a, b + main_width, stride_b, + row1 + main_width, rem_width); + } + ssim_total += SsimWindowRow_C(row0, row1, num_windows); + samples += num_windows; + uint32_t* tmp = row0; + row0 = row1; + row1 = tmp; + prev_i = i; + } + free_aligned_buffer_64(row_buf); + } else { + int i; +#ifdef _OPENMP +#pragma omp for schedule(static) +#endif + for (i = 0; i < height - 8; i += 4) { + const uint8_t* a = src_a + (ptrdiff_t)i * stride_a; + const uint8_t* b = src_b + (ptrdiff_t)i * stride_b; + int j; + for (j = 0; j < width - 8; j += 4) { + ssim_total += Ssim8x8_C(a + j, stride_a, b + j, stride_b); + samples++; + } + } } - - src_a += stride_a * 4; - src_b += stride_b * 4; } ssim_total /= samples; diff --git a/source/compare_common.cc b/source/compare_common.cc index c7c4fe879..f375a5839 100644 --- a/source/compare_common.cc +++ b/source/compare_common.cc @@ -10,6 +10,8 @@ #include "libyuv/compare_row.h" +#include + #ifdef __cplusplus namespace libyuv { extern "C" { @@ -66,6 +68,108 @@ uint32_t HashDjb2_C(const uint8_t* src, int count, uint32_t seed) { return hash; } +// Compute 4x4 tile statistics (sum_a, sum_b, sum_sq_a + sum_sq_b, sum_axb) +// across 4 rows for width pixels (width is a multiple of 4). +void Ssim4x4Row_C(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + int x; + for (x = 0; x < width; x += 4) { + uint32_t sum_a = a0[0] + a0[1] + a0[2] + a0[3] + a1[0] + a1[1] + a1[2] + + a1[3] + a2[0] + a2[1] + a2[2] + a2[3] + a3[0] + a3[1] + + a3[2] + a3[3]; + uint32_t sum_b = b0[0] + b0[1] + b0[2] + b0[3] + b1[0] + b1[1] + b1[2] + + b1[3] + b2[0] + b2[1] + b2[2] + b2[3] + b3[0] + b3[1] + + b3[2] + b3[3]; + uint32_t sum_sq_ab = + a0[0] * a0[0] + a0[1] * a0[1] + a0[2] * a0[2] + a0[3] * a0[3] + + a1[0] * a1[0] + a1[1] * a1[1] + a1[2] * a1[2] + a1[3] * a1[3] + + a2[0] * a2[0] + a2[1] * a2[1] + a2[2] * a2[2] + a2[3] * a2[3] + + a3[0] * a3[0] + a3[1] * a3[1] + a3[2] * a3[2] + a3[3] * a3[3] + + b0[0] * b0[0] + b0[1] * b0[1] + b0[2] * b0[2] + b0[3] * b0[3] + + b1[0] * b1[0] + b1[1] * b1[1] + b1[2] * b1[2] + b1[3] * b1[3] + + b2[0] * b2[0] + b2[1] * b2[1] + b2[2] * b2[2] + b2[3] * b2[3] + + b3[0] * b3[0] + b3[1] * b3[1] + b3[2] * b3[2] + b3[3] * b3[3]; + uint32_t sum_axb = + a0[0] * b0[0] + a0[1] * b0[1] + a0[2] * b0[2] + a0[3] * b0[3] + + a1[0] * b1[0] + a1[1] * b1[1] + a1[2] * b1[2] + a1[3] * b1[3] + + a2[0] * b2[0] + a2[1] * b2[1] + a2[2] * b2[2] + a2[3] * b2[3] + + a3[0] * b3[0] + a3[1] * b3[1] + a3[2] * b3[2] + a3[3] * b3[3]; + dst[0] = sum_a; + dst[1] = sum_b; + dst[2] = sum_sq_ab; + dst[3] = sum_axb; + a0 += 4; + a1 += 4; + a2 += 4; + a3 += 4; + b0 += 4; + b1 += 4; + b2 += 4; + b3 += 4; + dst += 4; + } +} + +// Combine two consecutive 4x4 tile rows (row0 and row1) into 8x8 windows +// stepped on a 4-pixel horizontal grid and return the sum of SSIM values. +double SsimWindowRow_C(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + const int64_t count = 64; + // scale the constants by number of pixels + const int64_t cc1 = 26634; // (64^2*(.01*255)^2) + const int64_t cc2 = 239708; // (64^2*(.03*255)^2) + const int64_t c1 = (cc1 * count * count) >> 12; + const int64_t c2 = (cc2 * count * count) >> 12; + + double ssim_total = 0.0; + uint32_t prev_a = row0[0] + row1[0]; + uint32_t prev_b = row0[1] + row1[1]; + uint32_t prev_sq = row0[2] + row1[2]; + uint32_t prev_axb = row0[3] + row1[3]; + int w; + for (w = 0; w < num_windows; ++w) { + row0 += 4; + row1 += 4; + const uint32_t next_a = row0[0] + row1[0]; + const uint32_t next_b = row0[1] + row1[1]; + const uint32_t next_sq = row0[2] + row1[2]; + const uint32_t next_axb = row0[3] + row1[3]; + const int64_t sum_a = prev_a + next_a; + const int64_t sum_b = prev_b + next_b; + const int64_t sum_sq_ab = prev_sq + next_sq; + const int64_t sum_axb = prev_axb + next_axb; + prev_a = next_a; + prev_b = next_b; + prev_sq = next_sq; + prev_axb = next_axb; + + const int64_t sum_a_x_sum_b = sum_a * sum_b; + const int64_t ssim_n = (2 * sum_a_x_sum_b + c1) * + (2 * count * sum_axb - 2 * sum_a_x_sum_b + c2); + const int64_t sum_a_sq = sum_a * sum_a; + const int64_t sum_b_sq = sum_b * sum_b; + const int64_t ssim_d = (sum_a_sq + sum_b_sq + c1) * + (count * sum_sq_ab - sum_a_sq - sum_b_sq + c2); + ssim_total += ssim_n * 1.0 / ssim_d; + } + return ssim_total; +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/compare_gcc.cc b/source/compare_gcc.cc index 723d2aef4..2053e78c8 100644 --- a/source/compare_gcc.cc +++ b/source/compare_gcc.cc @@ -407,8 +407,9 @@ uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) { "xmm7"); return hash; } -#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \ - defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW) +#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \ + defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW) || \ + defined(HAS_SSIM4X4ROW_AVX512BW) // Mask of the low n bytes, for n = 0 to 63. static uint64_t LowBytesMask64(int n) { return n ? ~0ULL >> (64 - n) : 0ULL; @@ -749,6 +750,377 @@ uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) { #undef HASH64_AVX512BW #endif // HAS_HASHDJB2_AVX512BW +#ifdef HAS_SSIM4X4ROW_SSE2 +#define SSIM_ROW0_SSE2(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "movdqa %%xmm0,%%xmm2 \n" \ + "psrlw $0x8,%%xmm0 \n" \ + "psllw $0x8,%%xmm2 \n" \ + "psrlw $0x8,%%xmm2 \n" \ + "movdqa %%xmm0,%%xmm4 \n" \ + "paddw %%xmm2,%%xmm4 \n" \ + "movdqa %%xmm0,%%xmm6 \n" \ + "pmaddwd %%xmm6,%%xmm6 \n" \ + "movdqa %%xmm2,%%xmm3 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm3,%%xmm6 \n" \ + "movdqa %%xmm1,%%xmm3 \n" \ + "psrlw $0x8,%%xmm1 \n" \ + "psllw $0x8,%%xmm3 \n" \ + "psrlw $0x8,%%xmm3 \n" \ + "movdqa %%xmm1,%%xmm5 \n" \ + "paddw %%xmm3,%%xmm5 \n" \ + "pmaddwd %%xmm1,%%xmm0 \n" \ + "pmaddwd %%xmm3,%%xmm2 \n" \ + "movdqa %%xmm0,%%xmm7 \n" \ + "paddd %%xmm2,%%xmm7 \n" \ + "pmaddwd %%xmm1,%%xmm1 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm1,%%xmm6 \n" \ + "paddd %%xmm3,%%xmm6 \n" + +#define SSIM_ROW_SSE2(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "movdqa %%xmm0,%%xmm2 \n" \ + "psrlw $0x8,%%xmm0 \n" \ + "psllw $0x8,%%xmm2 \n" \ + "psrlw $0x8,%%xmm2 \n" \ + "paddw %%xmm0,%%xmm4 \n" \ + "paddw %%xmm2,%%xmm4 \n" \ + "movdqa %%xmm0,%%xmm3 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm3,%%xmm6 \n" \ + "movdqa %%xmm2,%%xmm3 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm3,%%xmm6 \n" \ + "movdqa %%xmm1,%%xmm3 \n" \ + "psrlw $0x8,%%xmm1 \n" \ + "psllw $0x8,%%xmm3 \n" \ + "psrlw $0x8,%%xmm3 \n" \ + "paddw %%xmm1,%%xmm5 \n" \ + "paddw %%xmm3,%%xmm5 \n" \ + "pmaddwd %%xmm1,%%xmm0 \n" \ + "pmaddwd %%xmm3,%%xmm2 \n" \ + "paddd %%xmm0,%%xmm7 \n" \ + "paddd %%xmm2,%%xmm7 \n" \ + "pmaddwd %%xmm1,%%xmm1 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm1,%%xmm6 \n" \ + "paddd %%xmm3,%%xmm6 \n" + +void Ssim4x4Row_SSE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + asm volatile( + LABELALIGN "1: \n" // + SSIM_ROW0_SSE2("movdqu (%[src_a]),%%xmm0\n", + "movdqu (%[src_b]),%%xmm1\n") // + "add %[stride_a],%[src_a] \n" + "add %[stride_b],%[src_b] \n" // + SSIM_ROW_SSE2("movdqu (%[src_a]),%%xmm0\n", + "movdqu (%[src_b]),%%xmm1\n") // + SSIM_ROW_SSE2("movdqu (%[src_a],%[stride_a]),%%xmm0\n", + "movdqu (%[src_b],%[stride_b]),%%xmm1\n") // + SSIM_ROW_SSE2("movdqu (%[src_a],%[stride_a],2),%%xmm0\n", + "movdqu (%[src_b],%[stride_b],2),%%xmm1\n") // + "sub %[stride_a],%[src_a] \n" + "sub %[stride_b],%[src_b] \n" + "add $0x10,%[src_a] \n" + "add $0x10,%[src_b] \n" + "pcmpeqd %%xmm0,%%xmm0 \n" + "psrlw $0xf,%%xmm0 \n" + "pmaddwd %%xmm0,%%xmm4 \n" + "pmaddwd %%xmm0,%%xmm5 \n" + "movdqa %%xmm4,%%xmm0 \n" + "punpckldq %%xmm5,%%xmm0 \n" + "punpckhdq %%xmm5,%%xmm4 \n" + "movdqa %%xmm6,%%xmm1 \n" + "punpckldq %%xmm7,%%xmm1 \n" + "punpckhdq %%xmm7,%%xmm6 \n" + "movdqa %%xmm0,%%xmm2 \n" + "punpcklqdq %%xmm1,%%xmm2 \n" + "punpckhqdq %%xmm1,%%xmm0 \n" + "movdqa %%xmm4,%%xmm3 \n" + "punpcklqdq %%xmm6,%%xmm3 \n" + "punpckhqdq %%xmm6,%%xmm4 \n" + "movdqu %%xmm2,0x00(%[dst]) \n" + "movdqu %%xmm0,0x10(%[dst]) \n" + "movdqu %%xmm3,0x20(%[dst]) \n" + "movdqu %%xmm4,0x30(%[dst]) \n" + "add $0x40,%[dst] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), + [width] "+r"(width) + : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#undef SSIM_ROW0_SSE2 +#undef SSIM_ROW_SSE2 +#endif // HAS_SSIM4X4ROW_SSE2 + +#ifdef HAS_SSIM4X4ROW_AVX2 +#define SSIM_ROW0_AVX2(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "vpsrlw $0x8,%%ymm0,%%ymm2 \n" \ + "vpsllw $0x8,%%ymm0,%%ymm0 \n" \ + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" \ + "vpaddw %%ymm0,%%ymm2,%%ymm4 \n" \ + "vpmaddwd %%ymm2,%%ymm2,%%ymm6 \n" \ + "vpmaddwd %%ymm0,%%ymm0,%%ymm3 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpsrlw $0x8,%%ymm1,%%ymm3 \n" \ + "vpsllw $0x8,%%ymm1,%%ymm1 \n" \ + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" \ + "vpaddw %%ymm1,%%ymm3,%%ymm5 \n" \ + "vpmaddwd %%ymm3,%%ymm2,%%ymm7 \n" \ + "vpmaddwd %%ymm1,%%ymm0,%%ymm0 \n" \ + "vpaddd %%ymm0,%%ymm7,%%ymm7 \n" \ + "vpmaddwd %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpmaddwd %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpaddd %%ymm1,%%ymm6,%%ymm6 \n" + +#define SSIM_ROW_AVX2(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "vpsrlw $0x8,%%ymm0,%%ymm2 \n" \ + "vpsllw $0x8,%%ymm0,%%ymm0 \n" \ + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" \ + "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" \ + "vpaddw %%ymm0,%%ymm4,%%ymm4 \n" \ + "vpmaddwd %%ymm2,%%ymm2,%%ymm3 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpmaddwd %%ymm0,%%ymm0,%%ymm3 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpsrlw $0x8,%%ymm1,%%ymm3 \n" \ + "vpsllw $0x8,%%ymm1,%%ymm1 \n" \ + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" \ + "vpaddw %%ymm3,%%ymm5,%%ymm5 \n" \ + "vpaddw %%ymm1,%%ymm5,%%ymm5 \n" \ + "vpmaddwd %%ymm3,%%ymm2,%%ymm2 \n" \ + "vpmaddwd %%ymm1,%%ymm0,%%ymm0 \n" \ + "vpaddd %%ymm2,%%ymm7,%%ymm7 \n" \ + "vpaddd %%ymm0,%%ymm7,%%ymm7 \n" \ + "vpmaddwd %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpmaddwd %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpaddd %%ymm1,%%ymm6,%%ymm6 \n" + +void Ssim4x4Row_AVX2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + asm volatile( + LABELALIGN "1: \n" // + SSIM_ROW0_AVX2("vmovdqu (%[src_a]),%%ymm0\n", + "vmovdqu (%[src_b]),%%ymm1\n") // + "add %[stride_a],%[src_a] \n" + "add %[stride_b],%[src_b] \n" // + SSIM_ROW_AVX2("vmovdqu (%[src_a]),%%ymm0\n", + "vmovdqu (%[src_b]),%%ymm1\n") // + SSIM_ROW_AVX2("vmovdqu (%[src_a],%[stride_a]),%%ymm0\n", + "vmovdqu (%[src_b],%[stride_b]),%%ymm1\n") // + SSIM_ROW_AVX2("vmovdqu (%[src_a],%[stride_a],2),%%ymm0\n", + "vmovdqu (%[src_b],%[stride_b],2),%%ymm1\n") // + "sub %[stride_a],%[src_a] \n" + "sub %[stride_b],%[src_b] \n" + "add $0x20,%[src_a] \n" + "add $0x20,%[src_b] \n" + "vpcmpeqd %%ymm0,%%ymm0,%%ymm0 \n" + "vpsrlw $0xf,%%ymm0,%%ymm0 \n" + "vpmaddwd %%ymm0,%%ymm4,%%ymm4 \n" + "vpmaddwd %%ymm0,%%ymm5,%%ymm5 \n" + "vpunpckldq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpckldq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhdq %%ymm7,%%ymm6,%%ymm3 \n" + "vpunpcklqdq %%ymm2,%%ymm0,%%ymm4 \n" + "vpunpckhqdq %%ymm2,%%ymm0,%%ymm5 \n" + "vpunpcklqdq %%ymm3,%%ymm1,%%ymm6 \n" + "vpunpckhqdq %%ymm3,%%ymm1,%%ymm7 \n" + "vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm0 \n" + "vperm2i128 $0x20,%%ymm7,%%ymm6,%%ymm1 \n" + "vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm2 \n" + "vperm2i128 $0x31,%%ymm7,%%ymm6,%%ymm3 \n" + "vmovdqu %%ymm0,0x00(%[dst]) \n" + "vmovdqu %%ymm1,0x20(%[dst]) \n" + "vmovdqu %%ymm2,0x40(%[dst]) \n" + "vmovdqu %%ymm3,0x60(%[dst]) \n" + "add $0x80,%[dst] \n" + "sub $0x20,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), + [width] "+r"(width) + : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#undef SSIM_ROW0_AVX2 +#undef SSIM_ROW_AVX2 +#endif // HAS_SSIM4X4ROW_AVX2 + +#ifdef HAS_SSIM4X4ROW_AVX512BW +#define SSIM_ROW0_AVX512BW(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "vpsrlw $0x8,%%zmm0,%%zmm2 \n" \ + "vpsllw $0x8,%%zmm0,%%zmm0 \n" \ + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" \ + "vpaddw %%zmm0,%%zmm2,%%zmm4 \n" \ + "vpmaddwd %%zmm2,%%zmm2,%%zmm6 \n" \ + "vpmaddwd %%zmm0,%%zmm0,%%zmm3 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpsrlw $0x8,%%zmm1,%%zmm3 \n" \ + "vpsllw $0x8,%%zmm1,%%zmm1 \n" \ + "vpsrlw $0x8,%%zmm1,%%zmm1 \n" \ + "vpaddw %%zmm1,%%zmm3,%%zmm5 \n" \ + "vpmaddwd %%zmm3,%%zmm2,%%zmm7 \n" \ + "vpmaddwd %%zmm1,%%zmm0,%%zmm0 \n" \ + "vpaddd %%zmm0,%%zmm7,%%zmm7 \n" \ + "vpmaddwd %%zmm3,%%zmm3,%%zmm3 \n" \ + "vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpaddd %%zmm1,%%zmm6,%%zmm6 \n" + +#define SSIM_ROW_AVX512BW(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "vpsrlw $0x8,%%zmm0,%%zmm2 \n" \ + "vpsllw $0x8,%%zmm0,%%zmm0 \n" \ + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" \ + "vpaddw %%zmm2,%%zmm4,%%zmm4 \n" \ + "vpaddw %%zmm0,%%zmm4,%%zmm4 \n" \ + "vpmaddwd %%zmm2,%%zmm2,%%zmm3 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpmaddwd %%zmm0,%%zmm0,%%zmm3 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpsrlw $0x8,%%zmm1,%%zmm3 \n" \ + "vpsllw $0x8,%%zmm1,%%zmm1 \n" \ + "vpsrlw $0x8,%%zmm1,%%zmm1 \n" \ + "vpaddw %%zmm3,%%zmm5,%%zmm5 \n" \ + "vpaddw %%zmm1,%%zmm5,%%zmm5 \n" \ + "vpmaddwd %%zmm3,%%zmm2,%%zmm2 \n" \ + "vpmaddwd %%zmm1,%%zmm0,%%zmm0 \n" \ + "vpaddd %%zmm2,%%zmm7,%%zmm7 \n" \ + "vpaddd %%zmm0,%%zmm7,%%zmm7 \n" \ + "vpmaddwd %%zmm3,%%zmm3,%%zmm3 \n" \ + "vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpaddd %%zmm1,%%zmm6,%%zmm6 \n" + +#define SSIM_TRANSPOSE_AVX512BW \ + "vpternlogd $0xff,%%zmm0,%%zmm0,%%zmm0 \n" \ + "vpsrlw $0xf,%%zmm0,%%zmm0 \n" \ + "vpmaddwd %%zmm0,%%zmm4,%%zmm4 \n" \ + "vpmaddwd %%zmm0,%%zmm5,%%zmm5 \n" \ + "vpunpckldq %%zmm5,%%zmm4,%%zmm0 \n" \ + "vpunpckhdq %%zmm5,%%zmm4,%%zmm1 \n" \ + "vpunpckldq %%zmm7,%%zmm6,%%zmm2 \n" \ + "vpunpckhdq %%zmm7,%%zmm6,%%zmm3 \n" \ + "vpunpcklqdq %%zmm2,%%zmm0,%%zmm4 \n" \ + "vpunpckhqdq %%zmm2,%%zmm0,%%zmm5 \n" \ + "vpunpcklqdq %%zmm3,%%zmm1,%%zmm6 \n" \ + "vpunpckhqdq %%zmm3,%%zmm1,%%zmm7 \n" \ + "vshufi32x4 $0x44,%%zmm5,%%zmm4,%%zmm0 \n" \ + "vshufi32x4 $0x44,%%zmm7,%%zmm6,%%zmm1 \n" \ + "vshufi32x4 $0xee,%%zmm5,%%zmm4,%%zmm2 \n" \ + "vshufi32x4 $0xee,%%zmm7,%%zmm6,%%zmm3 \n" \ + "vshufi32x4 $0x88,%%zmm1,%%zmm0,%%zmm4 \n" \ + "vshufi32x4 $0xdd,%%zmm1,%%zmm0,%%zmm5 \n" \ + "vshufi32x4 $0x88,%%zmm3,%%zmm2,%%zmm6 \n" \ + "vshufi32x4 $0xdd,%%zmm3,%%zmm2,%%zmm7 \n" + +// Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60 +// pixels (1 to 15 tiles) with masked loads and stores. +void Ssim4x4Row_AVX512BW(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const int rem = width & 60; + const uint16_t load_mask = rem ? (uint16_t)((1U << (rem >> 2)) - 1U) : 0U; + const uint64_t store_mask = LowBytesMask64(rem); + asm volatile( + "sub $0x40,%[width] \n" + "jl 2f \n" + + LABELALIGN "1: \n" // + SSIM_ROW0_AVX512BW("vmovdqu8 (%[src_a]),%%zmm0\n", + "vmovdqu8 (%[src_b]),%%zmm1\n") // + "add %[stride_a],%[src_a] \n" + "add %[stride_b],%[src_b] \n" // + SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a]),%%zmm0\n", + "vmovdqu8 (%[src_b]),%%zmm1\n") // + SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a],%[stride_a]),%%zmm0\n", + "vmovdqu8 (%[src_b],%[stride_b]),%%zmm1\n") // + SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a],%[stride_a],2),%%zmm0\n", + "vmovdqu8 (%[src_b],%[stride_b],2),%%zmm1\n") // + "sub %[stride_a],%[src_a] \n" + "sub %[stride_b],%[src_b] \n" + "add $0x40,%[src_a] \n" + "add $0x40,%[src_b] \n" // + SSIM_TRANSPOSE_AVX512BW + "vmovdqu32 %%zmm4,0x00(%[dst]) \n" + "vmovdqu32 %%zmm5,0x40(%[dst]) \n" + "vmovdqu32 %%zmm6,0x80(%[dst]) \n" + "vmovdqu32 %%zmm7,0xc0(%[dst]) \n" + "add $0x100,%[dst] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "je 3f \n" + "kmovw %[load_mask],%%k1 \n" // + SSIM_ROW0_AVX512BW("vmovdqu32 (%[src_a]),%%zmm0%{%%k1%}%{z%}\n", + "vmovdqu32 (%[src_b]),%%zmm1%{%%k1%}%{z%}\n") // + "add %[stride_a],%[src_a] \n" + "add %[stride_b],%[src_b] \n" // + SSIM_ROW_AVX512BW("vmovdqu32 (%[src_a]),%%zmm0%{%%k1%}%{z%}\n", + "vmovdqu32 (%[src_b]),%%zmm1%{%%k1%}%{z%}\n") // + SSIM_ROW_AVX512BW( + "vmovdqu32 (%[src_a],%[stride_a]),%%zmm0%{%%k1%}%{z%}\n", + "vmovdqu32 (%[src_b],%[stride_b]),%%zmm1%{%%k1%}%{z%}\n") // + SSIM_ROW_AVX512BW( + "vmovdqu32 (%[src_a],%[stride_a],2),%%zmm0%{%%k1%}%{z%}\n", + "vmovdqu32 (%[src_b],%[stride_b],2),%%zmm1%{%%k1%}%{z%}\n") // + SSIM_TRANSPOSE_AVX512BW + "kmovq %[store_mask],%%k1 \n" + "vmovdqu32 %%zmm4,0x00(%[dst])%{%%k1%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 %%zmm5,0x40(%[dst])%{%%k1%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 %%zmm6,0x80(%[dst])%{%%k1%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 %%zmm7,0xc0(%[dst])%{%%k1%} \n" + + "3: \n" + "vzeroupper \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), + [width] "+r"(width) + : [stride_a] "r"((ptrdiff_t)stride_a), + [stride_b] "r"((ptrdiff_t)stride_b), [load_mask] "m"(load_mask), + [store_mask] "m"(store_mask) + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#undef SSIM_ROW0_AVX512BW +#undef SSIM_ROW_AVX512BW +#undef SSIM_TRANSPOSE_AVX512BW +#endif // HAS_SSIM4X4ROW_AVX512BW + #endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) && // !defined(LIBYUV_ENABLE_ROWWIN) diff --git a/source/compare_neon.cc b/source/compare_neon.cc index b3a4b12f4..5c7924b7e 100644 --- a/source/compare_neon.cc +++ b/source/compare_neon.cc @@ -93,6 +93,68 @@ uint32_t SumSquareError_NEON(const uint8_t* src_a, return sse; } +#define SSIM_ROW_NEON32(LOAD_A, LOAD_B, PADD_U8, PADD_U16) \ + LOAD_A \ + LOAD_B \ + PADD_U8 " q8, q0 \n" PADD_U8 \ + " q9, q1 \n" \ + "vmull.u8 q2, d0, d0 \n" \ + "vmull.u8 q3, d1, d1 \n" PADD_U16 \ + " q10, q2 \n" PADD_U16 \ + " q11, q3 \n" \ + "vmull.u8 q2, d2, d2 \n" \ + "vmull.u8 q3, d3, d3 \n" \ + "vpadal.u16 q10, q2 \n" \ + "vpadal.u16 q11, q3 \n" \ + "vmull.u8 q2, d0, d2 \n" \ + "vmull.u8 q3, d1, d3 \n" PADD_U16 \ + " q12, q2 \n" PADD_U16 \ + " q13, q3 \n" + +void Ssim4x4Row_NEON(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + asm volatile( + "1: \n" // + SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n", + "vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpaddl.u8", + "vpaddl.u16") // + SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n", + "vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpadal.u8", + "vpadal.u16") // + SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n", + "vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpadal.u8", + "vpadal.u16") // + SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]]\n", + "vld1.8 {d2, d3}, [%[src_b]]\n", "vpadal.u8", + "vpadal.u16") // + "sub %[src_a], %[src_a], %[stride_a], lsl #1 \n" + "sub %[src_a], %[src_a], %[stride_a] \n" + "add %[src_a], %[src_a], #16 \n" + "sub %[src_b], %[src_b], %[stride_b], lsl #1 \n" + "sub %[src_b], %[src_b], %[stride_b] \n" + "add %[src_b], %[src_b], #16 \n" + "vpaddl.u16 q8, q8 \n" + "vpaddl.u16 q9, q9 \n" + "vpadd.u32 d20, d20, d21 \n" + "vpadd.u32 d21, d22, d23 \n" + "vpadd.u32 d22, d24, d25 \n" + "vpadd.u32 d23, d26, d27 \n" + "subs %[width], %[width], #16 \n" + "vst4.32 {d16, d18, d20, d22}, [%[dst]]! \n" + "vst4.32 {d17, d19, d21, d23}, [%[dst]]! \n" + "bgt 1b \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), + [width] "+r"(width) + : [stride_a] "r"(stride_a), [stride_b] "r"(stride_b) + : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12", + "q13"); +} +#undef SSIM_ROW_NEON32 + #endif // defined(__ARM_NEON__) && !defined(__aarch64__) #ifdef __cplusplus diff --git a/source/compare_neon64.cc b/source/compare_neon64.cc index 89dba1ba0..9ffc4328e 100644 --- a/source/compare_neon64.cc +++ b/source/compare_neon64.cc @@ -220,6 +220,135 @@ uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a, return sse; } +#define SSIM_ROW_NEON64(VA, VB, PADD_U8, PADD_U16) \ + PADD_U8 " v4.8h, " #VA ".16b \n" PADD_U8 \ + " v5.8h, " #VB \ + ".16b \n" \ + "umull v6.8h, " #VA ".8b, " #VA \ + ".8b \n" \ + "umull2 v7.8h, " #VA ".16b, " #VA ".16b \n" PADD_U16 \ + " v20.4s, v6.8h \n" PADD_U16 \ + " v21.4s, v7.8h \n" \ + "umull v6.8h, " #VB ".8b, " #VB \ + ".8b \n" \ + "umull2 v7.8h, " #VB ".16b, " #VB \ + ".16b \n" \ + "uadalp v20.4s, v6.8h \n" \ + "uadalp v21.4s, v7.8h \n" \ + "umull v6.8h, " #VA ".8b, " #VB \ + ".8b \n" \ + "umull2 v7.8h, " #VA ".16b, " #VB ".16b \n" PADD_U16 \ + " v22.4s, v6.8h \n" PADD_U16 \ + " v23.4s, v7.8h \n" + +void Ssim4x4Row_NEON(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + asm volatile( + "1: \n" + "ldr q0, [%[a0]], #16 \n" + "ldr q1, [%[b0]], #16 \n" + "ldr q2, [%[a1]], #16 \n" + "ldr q3, [%[b1]], #16 \n" // + SSIM_ROW_NEON64(v0, v1, "uaddlp", "uaddlp") // + SSIM_ROW_NEON64(v2, v3, "uadalp", "uadalp") // + "ldr q0, [%[a2]], #16 \n" + "ldr q1, [%[b2]], #16 \n" + "ldr q2, [%[a3]], #16 \n" + "ldr q3, [%[b3]], #16 \n" // + SSIM_ROW_NEON64(v0, v1, "uadalp", "uadalp") // + SSIM_ROW_NEON64(v2, v3, "uadalp", "uadalp") // + "uaddlp v16.4s, v4.8h \n" + "uaddlp v17.4s, v5.8h \n" + "addp v18.4s, v20.4s, v21.4s \n" + "addp v19.4s, v22.4s, v23.4s \n" + "subs %w[width], %w[width], #16 \n" + "st4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[dst]], #64 \n" + "b.gt 1b \n" + : [a0] "+r"(a0), [a1] "+r"(a1), [a2] "+r"(a2), [a3] "+r"(a3), + [b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3), + [dst] "+r"(dst), [width] "+r"(width) + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} +#undef SSIM_ROW_NEON64 + +void Ssim4x4Row_NEON_DotProd(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + asm volatile( + "movi v6.16b, #1 \n" + "1: \n" + "movi v16.4s, #0 \n" + "movi v17.4s, #0 \n" + "movi v18.4s, #0 \n" + "movi v19.4s, #0 \n" + "ldr q0, [%[a0]], #16 \n" + "ldr q1, [%[b0]], #16 \n" + "ldr q2, [%[a1]], #16 \n" + "ldr q3, [%[b1]], #16 \n" + "udot v16.4s, v0.16b, v6.16b \n" + "udot v17.4s, v1.16b, v6.16b \n" + "udot v18.4s, v0.16b, v0.16b \n" + "udot v18.4s, v1.16b, v1.16b \n" + "udot v19.4s, v0.16b, v1.16b \n" + "udot v16.4s, v2.16b, v6.16b \n" + "udot v17.4s, v3.16b, v6.16b \n" + "udot v18.4s, v2.16b, v2.16b \n" + "udot v18.4s, v3.16b, v3.16b \n" + "udot v19.4s, v2.16b, v3.16b \n" + "ldr q0, [%[a2]], #16 \n" + "ldr q1, [%[b2]], #16 \n" + "ldr q2, [%[a3]], #16 \n" + "ldr q3, [%[b3]], #16 \n" + "udot v16.4s, v0.16b, v6.16b \n" + "udot v17.4s, v1.16b, v6.16b \n" + "udot v18.4s, v0.16b, v0.16b \n" + "udot v18.4s, v1.16b, v1.16b \n" + "udot v19.4s, v0.16b, v1.16b \n" + "udot v16.4s, v2.16b, v6.16b \n" + "udot v17.4s, v3.16b, v6.16b \n" + "udot v18.4s, v2.16b, v2.16b \n" + "udot v18.4s, v3.16b, v3.16b \n" + "udot v19.4s, v2.16b, v3.16b \n" + "subs %w[width], %w[width], #16 \n" + "st4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[dst]], #64 \n" + "b.gt 1b \n" + : [a0] "+r"(a0), [a1] "+r"(a1), [a2] "+r"(a2), [a3] "+r"(a3), + [b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3), + [dst] "+r"(dst), [width] "+r"(width) + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v6", "v16", "v17", "v18", + "v19"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/compare_wasm.cc b/source/compare_wasm.cc index 6400e1ae9..ecb76b09c 100644 --- a/source/compare_wasm.cc +++ b/source/compare_wasm.cc @@ -129,6 +129,118 @@ uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed) { } #endif // HAS_HASHDJB2_WASMSIMD +#if defined(HAS_SSIM4X4ROW_WASMSIMD) +void Ssim4x4Row_WASMSIMD(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const uint8_t* src_a1 = src_a + stride_a; + const uint8_t* src_a2 = src_a1 + stride_a; + const uint8_t* src_a3 = src_a2 + stride_a; + const uint8_t* src_b1 = src_b + stride_b; + const uint8_t* src_b2 = src_b1 + stride_b; + const uint8_t* src_b3 = src_b2 + stride_b; + const v128_t mask = wasm_i16x8_splat(0x00ff); + for (; width > 0; width -= 16) { + v128_t a0 = wasm_v128_load(src_a); + v128_t a1 = wasm_v128_load(src_a1); + v128_t a2 = wasm_v128_load(src_a2); + v128_t a3 = wasm_v128_load(src_a3); + v128_t b0 = wasm_v128_load(src_b); + v128_t b1 = wasm_v128_load(src_b1); + v128_t b2 = wasm_v128_load(src_b2); + v128_t b3 = wasm_v128_load(src_b3); + + v128_t a0_ev = wasm_v128_and(a0, mask); + v128_t a0_od = wasm_u16x8_shr(a0, 8); + v128_t a1_ev = wasm_v128_and(a1, mask); + v128_t a1_od = wasm_u16x8_shr(a1, 8); + v128_t a2_ev = wasm_v128_and(a2, mask); + v128_t a2_od = wasm_u16x8_shr(a2, 8); + v128_t a3_ev = wasm_v128_and(a3, mask); + v128_t a3_od = wasm_u16x8_shr(a3, 8); + + v128_t b0_ev = wasm_v128_and(b0, mask); + v128_t b0_od = wasm_u16x8_shr(b0, 8); + v128_t b1_ev = wasm_v128_and(b1, mask); + v128_t b1_od = wasm_u16x8_shr(b1, 8); + v128_t b2_ev = wasm_v128_and(b2, mask); + v128_t b2_od = wasm_u16x8_shr(b2, 8); + v128_t b3_ev = wasm_v128_and(b3, mask); + v128_t b3_od = wasm_u16x8_shr(b3, 8); + + v128_t sum_a_16 = + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(a0_ev, a0_od), + wasm_i16x8_add(a1_ev, a1_od)), + wasm_i16x8_add(wasm_i16x8_add(a2_ev, a2_od), + wasm_i16x8_add(a3_ev, a3_od))); + v128_t sum_b_16 = + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(b0_ev, b0_od), + wasm_i16x8_add(b1_ev, b1_od)), + wasm_i16x8_add(wasm_i16x8_add(b2_ev, b2_od), + wasm_i16x8_add(b3_ev, b3_od))); + + v128_t sum_a = wasm_u32x4_extadd_pairwise_u16x8(sum_a_16); + v128_t sum_b = wasm_u32x4_extadd_pairwise_u16x8(sum_b_16); + + v128_t sum_axb = wasm_i32x4_add( + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, b0_ev), + wasm_i32x4_dot_i16x8(a0_od, b0_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, b1_ev), + wasm_i32x4_dot_i16x8(a1_od, b1_od))), + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, b2_ev), + wasm_i32x4_dot_i16x8(a2_od, b2_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, b3_ev), + wasm_i32x4_dot_i16x8(a3_od, b3_od)))); + + v128_t sq0 = + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, a0_ev), + wasm_i32x4_dot_i16x8(a0_od, a0_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(b0_ev, b0_ev), + wasm_i32x4_dot_i16x8(b0_od, b0_od))); + v128_t sq1 = + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, a1_ev), + wasm_i32x4_dot_i16x8(a1_od, a1_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(b1_ev, b1_ev), + wasm_i32x4_dot_i16x8(b1_od, b1_od))); + v128_t sq2 = + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, a2_ev), + wasm_i32x4_dot_i16x8(a2_od, a2_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(b2_ev, b2_ev), + wasm_i32x4_dot_i16x8(b2_od, b2_od))); + v128_t sq3 = + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, a3_ev), + wasm_i32x4_dot_i16x8(a3_od, a3_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(b3_ev, b3_ev), + wasm_i32x4_dot_i16x8(b3_od, b3_od))); + v128_t sum_sq_ab = + wasm_i32x4_add(wasm_i32x4_add(sq0, sq1), wasm_i32x4_add(sq2, sq3)); + + v128_t t0 = wasm_i32x4_shuffle(sum_a, sum_b, 0, 4, 1, 5); + v128_t t1 = wasm_i32x4_shuffle(sum_a, sum_b, 2, 6, 3, 7); + v128_t t2 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 0, 4, 1, 5); + v128_t t3 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 2, 6, 3, 7); + + wasm_v128_store(dst, wasm_i64x2_shuffle(t0, t2, 0, 2)); + wasm_v128_store(dst + 4, wasm_i64x2_shuffle(t0, t2, 1, 3)); + wasm_v128_store(dst + 8, wasm_i64x2_shuffle(t1, t3, 0, 2)); + wasm_v128_store(dst + 12, wasm_i64x2_shuffle(t1, t3, 1, 3)); + + src_a += 16; + src_a1 += 16; + src_a2 += 16; + src_a3 += 16; + src_b += 16; + src_b1 += 16; + src_b2 += 16; + src_b3 += 16; + dst += 16; + } +} +#endif // HAS_SSIM4X4ROW_WASMSIMD + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/compare_win.cc b/source/compare_win.cc index 9b6fd0ee5..469fa674d 100644 --- a/source/compare_win.cc +++ b/source/compare_win.cc @@ -422,6 +422,261 @@ uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) { } #endif // HAS_HASHDJB2_AVX512BW +#if defined(HAS_SSIM4X4ROW_SSE2) +// Width is a multiple of 16. +LIBYUV_TARGET_SSE2 +void Ssim4x4Row_SSE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + const __m128i kMaskFF = _mm_set1_epi16(0x00ff); + const __m128i kOnes16 = _mm_set1_epi16(1); + int x; + for (x = 0; x < width; x += 16) { + __m128i acc_a = _mm_setzero_si128(); + __m128i acc_b = _mm_setzero_si128(); + __m128i acc_sq = _mm_setzero_si128(); + __m128i acc_axb = _mm_setzero_si128(); +#define SSIM_ROW_SSE2_WIN(ar, br) \ + do { \ + __m128i va = _mm_loadu_si128((const __m128i*)((ar) + x)); \ + __m128i vb = _mm_loadu_si128((const __m128i*)((br) + x)); \ + __m128i a_ev = _mm_and_si128(va, kMaskFF); \ + __m128i a_od = _mm_srli_epi16(va, 8); \ + __m128i b_ev = _mm_and_si128(vb, kMaskFF); \ + __m128i b_od = _mm_srli_epi16(vb, 8); \ + acc_a = _mm_add_epi16(acc_a, _mm_add_epi16(a_ev, a_od)); \ + acc_b = _mm_add_epi16(acc_b, _mm_add_epi16(b_ev, b_od)); \ + acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(a_ev, a_ev), \ + _mm_madd_epi16(a_od, a_od))); \ + acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(b_ev, b_ev), \ + _mm_madd_epi16(b_od, b_od))); \ + acc_axb = \ + _mm_add_epi32(acc_axb, _mm_add_epi32(_mm_madd_epi16(a_ev, b_ev), \ + _mm_madd_epi16(a_od, b_od))); \ + } while (0) + SSIM_ROW_SSE2_WIN(a0, b0); + SSIM_ROW_SSE2_WIN(a1, b1); + SSIM_ROW_SSE2_WIN(a2, b2); + SSIM_ROW_SSE2_WIN(a3, b3); +#undef SSIM_ROW_SSE2_WIN + acc_a = _mm_madd_epi16(acc_a, kOnes16); + acc_b = _mm_madd_epi16(acc_b, kOnes16); + { + __m128i t0 = _mm_unpacklo_epi32(acc_a, acc_b); + __m128i t1 = _mm_unpackhi_epi32(acc_a, acc_b); + __m128i t2 = _mm_unpacklo_epi32(acc_sq, acc_axb); + __m128i t3 = _mm_unpackhi_epi32(acc_sq, acc_axb); + _mm_storeu_si128((__m128i*)(dst + 0), _mm_unpacklo_epi64(t0, t2)); + _mm_storeu_si128((__m128i*)(dst + 4), _mm_unpackhi_epi64(t0, t2)); + _mm_storeu_si128((__m128i*)(dst + 8), _mm_unpacklo_epi64(t1, t3)); + _mm_storeu_si128((__m128i*)(dst + 12), _mm_unpackhi_epi64(t1, t3)); + } + dst += 16; + } +} +#endif // HAS_SSIM4X4ROW_SSE2 + +#if defined(HAS_SSIM4X4ROW_AVX2) +// Width is a multiple of 32. +LIBYUV_TARGET_AVX2 +void Ssim4x4Row_AVX2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + const __m256i kMaskFF = _mm256_set1_epi16(0x00ff); + const __m256i kOnes16 = _mm256_set1_epi16(1); + int x; + for (x = 0; x < width; x += 32) { + __m256i acc_a = _mm256_setzero_si256(); + __m256i acc_b = _mm256_setzero_si256(); + __m256i acc_sq = _mm256_setzero_si256(); + __m256i acc_axb = _mm256_setzero_si256(); +#define SSIM_ROW_AVX2_WIN(ar, br) \ + do { \ + __m256i va = _mm256_loadu_si256((const __m256i*)((ar) + x)); \ + __m256i vb = _mm256_loadu_si256((const __m256i*)((br) + x)); \ + __m256i a_ev = _mm256_and_si256(va, kMaskFF); \ + __m256i a_od = _mm256_srli_epi16(va, 8); \ + __m256i b_ev = _mm256_and_si256(vb, kMaskFF); \ + __m256i b_od = _mm256_srli_epi16(vb, 8); \ + acc_a = _mm256_add_epi16(acc_a, _mm256_add_epi16(a_ev, a_od)); \ + acc_b = _mm256_add_epi16(acc_b, _mm256_add_epi16(b_ev, b_od)); \ + acc_sq = _mm256_add_epi32( \ + acc_sq, _mm256_add_epi32(_mm256_madd_epi16(a_ev, a_ev), \ + _mm256_madd_epi16(a_od, a_od))); \ + acc_sq = _mm256_add_epi32( \ + acc_sq, _mm256_add_epi32(_mm256_madd_epi16(b_ev, b_ev), \ + _mm256_madd_epi16(b_od, b_od))); \ + acc_axb = _mm256_add_epi32( \ + acc_axb, _mm256_add_epi32(_mm256_madd_epi16(a_ev, b_ev), \ + _mm256_madd_epi16(a_od, b_od))); \ + } while (0) + SSIM_ROW_AVX2_WIN(a0, b0); + SSIM_ROW_AVX2_WIN(a1, b1); + SSIM_ROW_AVX2_WIN(a2, b2); + SSIM_ROW_AVX2_WIN(a3, b3); +#undef SSIM_ROW_AVX2_WIN + acc_a = _mm256_madd_epi16(acc_a, kOnes16); + acc_b = _mm256_madd_epi16(acc_b, kOnes16); + { + __m256i t0 = _mm256_unpacklo_epi32(acc_a, acc_b); + __m256i t1 = _mm256_unpackhi_epi32(acc_a, acc_b); + __m256i t2 = _mm256_unpacklo_epi32(acc_sq, acc_axb); + __m256i t3 = _mm256_unpackhi_epi32(acc_sq, acc_axb); + __m256i u0 = _mm256_unpacklo_epi64(t0, t2); + __m256i u1 = _mm256_unpackhi_epi64(t0, t2); + __m256i u2 = _mm256_unpacklo_epi64(t1, t3); + __m256i u3 = _mm256_unpackhi_epi64(t1, t3); + _mm256_storeu_si256((__m256i*)(dst + 0), + _mm256_permute2x128_si256(u0, u1, 0x20)); + _mm256_storeu_si256((__m256i*)(dst + 8), + _mm256_permute2x128_si256(u2, u3, 0x20)); + _mm256_storeu_si256((__m256i*)(dst + 16), + _mm256_permute2x128_si256(u0, u1, 0x31)); + _mm256_storeu_si256((__m256i*)(dst + 24), + _mm256_permute2x128_si256(u2, u3, 0x31)); + } + dst += 32; + } + _mm256_zeroupper(); +} +#endif // HAS_SSIM4X4ROW_AVX2 + +#if defined(HAS_SSIM4X4ROW_AVX512BW) +// Width is a multiple of 4. +LIBYUV_TARGET_AVX512BW +void Ssim4x4Row_AVX512BW(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + const __m512i kMaskFF = _mm512_set1_epi16(0x00ff); + const __m512i kOnes16 = _mm512_set1_epi16(1); +#define SSIM_ROW_AVX512_WIN(va, vb) \ + do { \ + __m512i a_ev = _mm512_and_si512(va, kMaskFF); \ + __m512i a_od = _mm512_srli_epi16(va, 8); \ + __m512i b_ev = _mm512_and_si512(vb, kMaskFF); \ + __m512i b_od = _mm512_srli_epi16(vb, 8); \ + acc_a = _mm512_add_epi16(acc_a, _mm512_add_epi16(a_ev, a_od)); \ + acc_b = _mm512_add_epi16(acc_b, _mm512_add_epi16(b_ev, b_od)); \ + acc_sq = _mm512_add_epi32( \ + acc_sq, _mm512_add_epi32(_mm512_madd_epi16(a_ev, a_ev), \ + _mm512_madd_epi16(a_od, a_od))); \ + acc_sq = _mm512_add_epi32( \ + acc_sq, _mm512_add_epi32(_mm512_madd_epi16(b_ev, b_ev), \ + _mm512_madd_epi16(b_od, b_od))); \ + acc_axb = _mm512_add_epi32( \ + acc_axb, _mm512_add_epi32(_mm512_madd_epi16(a_ev, b_ev), \ + _mm512_madd_epi16(a_od, b_od))); \ + } while (0) +#define SSIM_TRANSPOSE_AVX512_WIN() \ + acc_a = _mm512_madd_epi16(acc_a, kOnes16); \ + acc_b = _mm512_madd_epi16(acc_b, kOnes16); \ + __m512i t0 = _mm512_unpacklo_epi32(acc_a, acc_b); \ + __m512i t1 = _mm512_unpackhi_epi32(acc_a, acc_b); \ + __m512i t2 = _mm512_unpacklo_epi32(acc_sq, acc_axb); \ + __m512i t3 = _mm512_unpackhi_epi32(acc_sq, acc_axb); \ + __m512i u0 = _mm512_unpacklo_epi64(t0, t2); \ + __m512i u1 = _mm512_unpackhi_epi64(t0, t2); \ + __m512i u2 = _mm512_unpacklo_epi64(t1, t3); \ + __m512i u3 = _mm512_unpackhi_epi64(t1, t3); \ + __m512i v0 = _mm512_shuffle_i32x4(u0, u1, 0x44); \ + __m512i v1 = _mm512_shuffle_i32x4(u2, u3, 0x44); \ + __m512i v2 = _mm512_shuffle_i32x4(u0, u1, 0xee); \ + __m512i v3 = _mm512_shuffle_i32x4(u2, u3, 0xee); \ + __m512i out0 = _mm512_shuffle_i32x4(v0, v1, 0x88); \ + __m512i out1 = _mm512_shuffle_i32x4(v0, v1, 0xdd); \ + __m512i out2 = _mm512_shuffle_i32x4(v2, v3, 0x88); \ + __m512i out3 = _mm512_shuffle_i32x4(v2, v3, 0xdd) + + while (width >= 64) { + __m512i acc_a = _mm512_setzero_si512(); + __m512i acc_b = _mm512_setzero_si512(); + __m512i acc_sq = _mm512_setzero_si512(); + __m512i acc_axb = _mm512_setzero_si512(); + SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a0), _mm512_loadu_si512(b0)); + SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a1), _mm512_loadu_si512(b1)); + SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a2), _mm512_loadu_si512(b2)); + SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a3), _mm512_loadu_si512(b3)); + SSIM_TRANSPOSE_AVX512_WIN(); + _mm512_storeu_si512(dst + 0, out0); + _mm512_storeu_si512(dst + 16, out1); + _mm512_storeu_si512(dst + 32, out2); + _mm512_storeu_si512(dst + 48, out3); + a0 += 64; + a1 += 64; + a2 += 64; + a3 += 64; + b0 += 64; + b1 += 64; + b2 += 64; + b3 += 64; + dst += 64; + width -= 64; + } + if (width > 0) { + const __mmask16 load_mask = (__mmask16)((1U << (width >> 2)) - 1U); + const uint64_t store_mask = LowBytesMask64(width); + __m512i acc_a = _mm512_setzero_si512(); + __m512i acc_b = _mm512_setzero_si512(); + __m512i acc_sq = _mm512_setzero_si512(); + __m512i acc_axb = _mm512_setzero_si512(); + SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a0), + _mm512_maskz_loadu_epi32(load_mask, b0)); + SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a1), + _mm512_maskz_loadu_epi32(load_mask, b1)); + SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a2), + _mm512_maskz_loadu_epi32(load_mask, b2)); + SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a3), + _mm512_maskz_loadu_epi32(load_mask, b3)); + SSIM_TRANSPOSE_AVX512_WIN(); + _mm512_mask_storeu_epi32(dst + 0, (__mmask16)store_mask, out0); + _mm512_mask_storeu_epi32(dst + 16, (__mmask16)(store_mask >> 16), out1); + _mm512_mask_storeu_epi32(dst + 32, (__mmask16)(store_mask >> 32), out2); + _mm512_mask_storeu_epi32(dst + 48, (__mmask16)(store_mask >> 48), out3); + } +#undef SSIM_ROW_AVX512_WIN +#undef SSIM_TRANSPOSE_AVX512_WIN +} +#endif // HAS_SSIM4X4ROW_AVX512BW + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_lasx.cc b/source/row_lasx.cc index 5a43f638a..be7124f23 100644 --- a/source/row_lasx.cc +++ b/source/row_lasx.cc @@ -10,6 +10,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "libyuv/compare_row.h" #include "libyuv/row.h" // Note: every register written by an inline asm block (vector, general @@ -2045,6 +2046,133 @@ void RGBToYMatrixRow_LASX(const uint8_t* src_rgba, "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr16", "$xr17"); } + +#ifdef HAS_SSIM4X4ROW_LASX +void Ssim4x4Row_LASX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + int x; + int len = width / 32; + const uint8_t* src_a1 = src_a + stride_a; + const uint8_t* src_a2 = src_a1 + stride_a; + const uint8_t* src_a3 = src_a2 + stride_a; + const uint8_t* src_b1 = src_b + stride_b; + const uint8_t* src_b2 = src_b1 + stride_b; + const uint8_t* src_b3 = src_b2 + stride_b; + + for (x = 0; x < len; x++) { + __m256i a0 = __lasx_xvld(src_a, 0); + __m256i a1 = __lasx_xvld(src_a1, 0); + __m256i a2 = __lasx_xvld(src_a2, 0); + __m256i a3 = __lasx_xvld(src_a3, 0); + __m256i b0 = __lasx_xvld(src_b, 0); + __m256i b1 = __lasx_xvld(src_b1, 0); + __m256i b2 = __lasx_xvld(src_b2, 0); + __m256i b3 = __lasx_xvld(src_b3, 0); + + __m256i sum_a_16 = + __lasx_xvadd_h(__lasx_xvadd_h(__lasx_xvhaddw_hu_bu(a0, a0), + __lasx_xvhaddw_hu_bu(a1, a1)), + __lasx_xvadd_h(__lasx_xvhaddw_hu_bu(a2, a2), + __lasx_xvhaddw_hu_bu(a3, a3))); + __m256i sum_b_16 = + __lasx_xvadd_h(__lasx_xvadd_h(__lasx_xvhaddw_hu_bu(b0, b0), + __lasx_xvhaddw_hu_bu(b1, b1)), + __lasx_xvadd_h(__lasx_xvhaddw_hu_bu(b2, b2), + __lasx_xvhaddw_hu_bu(b3, b3))); + __m256i sum_a = __lasx_xvhaddw_wu_hu(sum_a_16, sum_a_16); + __m256i sum_b = __lasx_xvhaddw_wu_hu(sum_b_16, sum_b_16); + + __m256i ab0_ev = __lasx_xvmulwev_h_bu(a0, b0); + __m256i ab0_od = __lasx_xvmulwod_h_bu(a0, b0); + __m256i ab1_ev = __lasx_xvmulwev_h_bu(a1, b1); + __m256i ab1_od = __lasx_xvmulwod_h_bu(a1, b1); + __m256i ab2_ev = __lasx_xvmulwev_h_bu(a2, b2); + __m256i ab2_od = __lasx_xvmulwod_h_bu(a2, b2); + __m256i ab3_ev = __lasx_xvmulwev_h_bu(a3, b3); + __m256i ab3_od = __lasx_xvmulwod_h_bu(a3, b3); + __m256i sum_axb = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab0_ev, ab0_ev), + __lasx_xvhaddw_wu_hu(ab0_od, ab0_od)), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab1_ev, ab1_ev), + __lasx_xvhaddw_wu_hu(ab1_od, ab1_od))), + __lasx_xvadd_w(__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab2_ev, ab2_ev), + __lasx_xvhaddw_wu_hu(ab2_od, ab2_od)), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab3_ev, ab3_ev), + __lasx_xvhaddw_wu_hu(ab3_od, ab3_od)))); + + __m256i sq0 = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a0, a0), + __lasx_xvmulwev_h_bu(a0, a0)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a0, a0), + __lasx_xvmulwod_h_bu(a0, a0))), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b0, b0), + __lasx_xvmulwev_h_bu(b0, b0)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b0, b0), + __lasx_xvmulwod_h_bu(b0, b0)))); + __m256i sq1 = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a1, a1), + __lasx_xvmulwev_h_bu(a1, a1)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a1, a1), + __lasx_xvmulwod_h_bu(a1, a1))), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b1, b1), + __lasx_xvmulwev_h_bu(b1, b1)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b1, b1), + __lasx_xvmulwod_h_bu(b1, b1)))); + __m256i sq2 = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a2, a2), + __lasx_xvmulwev_h_bu(a2, a2)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a2, a2), + __lasx_xvmulwod_h_bu(a2, a2))), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b2, b2), + __lasx_xvmulwev_h_bu(b2, b2)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b2, b2), + __lasx_xvmulwod_h_bu(b2, b2)))); + __m256i sq3 = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a3, a3), + __lasx_xvmulwev_h_bu(a3, a3)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a3, a3), + __lasx_xvmulwod_h_bu(a3, a3))), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b3, b3), + __lasx_xvmulwev_h_bu(b3, b3)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b3, b3), + __lasx_xvmulwod_h_bu(b3, b3)))); + __m256i sum_sq_ab = + __lasx_xvadd_w(__lasx_xvadd_w(sq0, sq1), __lasx_xvadd_w(sq2, sq3)); + + __m256i t0 = __lasx_xvilvl_w(sum_b, sum_a); + __m256i t1 = __lasx_xvilvh_w(sum_b, sum_a); + __m256i t2 = __lasx_xvilvl_w(sum_axb, sum_sq_ab); + __m256i t3 = __lasx_xvilvh_w(sum_axb, sum_sq_ab); + __m256i p0 = __lasx_xvilvl_d(t2, t0); + __m256i p1 = __lasx_xvilvh_d(t2, t0); + __m256i p2 = __lasx_xvilvl_d(t3, t1); + __m256i p3 = __lasx_xvilvh_d(t3, t1); + __m256i out0 = __lasx_xvpermi_q(p1, p0, 0x20); + __m256i out1 = __lasx_xvpermi_q(p3, p2, 0x20); + __m256i out2 = __lasx_xvpermi_q(p1, p0, 0x31); + __m256i out3 = __lasx_xvpermi_q(p3, p2, 0x31); + __lasx_xvst(out0, dst, 0); + __lasx_xvst(out1, dst, 32); + __lasx_xvst(out2, dst, 64); + __lasx_xvst(out3, dst, 96); + + src_a += 32; + src_a1 += 32; + src_a2 += 32; + src_a3 += 32; + src_b += 32; + src_b1 += 32; + src_b2 += 32; + src_b3 += 32; + dst += 32; + } +} +#endif // HAS_SSIM4X4ROW_LASX + // undef for unified sources build #undef ALPHA_VAL #undef YUVTORGB_SETUP diff --git a/source/row_lsx.cc b/source/row_lsx.cc index b4fdaaf69..647d949d4 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -10,6 +10,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "libyuv/compare_row.h" #include "libyuv/row.h" // Note: every register written by an inline asm block (vector, general @@ -2715,6 +2716,119 @@ void RGBToYMatrixRow_LSX(const uint8_t* src_rgba, "$vr16", "$vr17"); } +#ifdef HAS_SSIM4X4ROW_LSX +void Ssim4x4Row_LSX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + int x; + int len = width / 16; + const uint8_t* src_a1 = src_a + stride_a; + const uint8_t* src_a2 = src_a1 + stride_a; + const uint8_t* src_a3 = src_a2 + stride_a; + const uint8_t* src_b1 = src_b + stride_b; + const uint8_t* src_b2 = src_b1 + stride_b; + const uint8_t* src_b3 = src_b2 + stride_b; + + for (x = 0; x < len; x++) { + __m128i a0 = __lsx_vld(src_a, 0); + __m128i a1 = __lsx_vld(src_a1, 0); + __m128i a2 = __lsx_vld(src_a2, 0); + __m128i a3 = __lsx_vld(src_a3, 0); + __m128i b0 = __lsx_vld(src_b, 0); + __m128i b1 = __lsx_vld(src_b1, 0); + __m128i b2 = __lsx_vld(src_b2, 0); + __m128i b3 = __lsx_vld(src_b3, 0); + + __m128i sum_a_16 = __lsx_vadd_h( + __lsx_vadd_h(__lsx_vhaddw_hu_bu(a0, a0), __lsx_vhaddw_hu_bu(a1, a1)), + __lsx_vadd_h(__lsx_vhaddw_hu_bu(a2, a2), __lsx_vhaddw_hu_bu(a3, a3))); + __m128i sum_b_16 = __lsx_vadd_h( + __lsx_vadd_h(__lsx_vhaddw_hu_bu(b0, b0), __lsx_vhaddw_hu_bu(b1, b1)), + __lsx_vadd_h(__lsx_vhaddw_hu_bu(b2, b2), __lsx_vhaddw_hu_bu(b3, b3))); + __m128i sum_a = __lsx_vhaddw_wu_hu(sum_a_16, sum_a_16); + __m128i sum_b = __lsx_vhaddw_wu_hu(sum_b_16, sum_b_16); + + __m128i ab0_ev = __lsx_vmulwev_h_bu(a0, b0); + __m128i ab0_od = __lsx_vmulwod_h_bu(a0, b0); + __m128i ab1_ev = __lsx_vmulwev_h_bu(a1, b1); + __m128i ab1_od = __lsx_vmulwod_h_bu(a1, b1); + __m128i ab2_ev = __lsx_vmulwev_h_bu(a2, b2); + __m128i ab2_od = __lsx_vmulwod_h_bu(a2, b2); + __m128i ab3_ev = __lsx_vmulwev_h_bu(a3, b3); + __m128i ab3_od = __lsx_vmulwod_h_bu(a3, b3); + __m128i sum_axb = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vadd_w(__lsx_vhaddw_wu_hu(ab0_ev, ab0_ev), + __lsx_vhaddw_wu_hu(ab0_od, ab0_od)), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(ab1_ev, ab1_ev), + __lsx_vhaddw_wu_hu(ab1_od, ab1_od))), + __lsx_vadd_w(__lsx_vadd_w(__lsx_vhaddw_wu_hu(ab2_ev, ab2_ev), + __lsx_vhaddw_wu_hu(ab2_od, ab2_od)), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(ab3_ev, ab3_ev), + __lsx_vhaddw_wu_hu(ab3_od, ab3_od)))); + + __m128i sq0 = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a0, a0), + __lsx_vmulwev_h_bu(a0, a0)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a0, a0), + __lsx_vmulwod_h_bu(a0, a0))), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b0, b0), + __lsx_vmulwev_h_bu(b0, b0)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b0, b0), + __lsx_vmulwod_h_bu(b0, b0)))); + __m128i sq1 = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a1, a1), + __lsx_vmulwev_h_bu(a1, a1)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a1, a1), + __lsx_vmulwod_h_bu(a1, a1))), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b1, b1), + __lsx_vmulwev_h_bu(b1, b1)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b1, b1), + __lsx_vmulwod_h_bu(b1, b1)))); + __m128i sq2 = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a2, a2), + __lsx_vmulwev_h_bu(a2, a2)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a2, a2), + __lsx_vmulwod_h_bu(a2, a2))), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b2, b2), + __lsx_vmulwev_h_bu(b2, b2)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b2, b2), + __lsx_vmulwod_h_bu(b2, b2)))); + __m128i sq3 = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a3, a3), + __lsx_vmulwev_h_bu(a3, a3)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a3, a3), + __lsx_vmulwod_h_bu(a3, a3))), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b3, b3), + __lsx_vmulwev_h_bu(b3, b3)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b3, b3), + __lsx_vmulwod_h_bu(b3, b3)))); + __m128i sum_sq_ab = + __lsx_vadd_w(__lsx_vadd_w(sq0, sq1), __lsx_vadd_w(sq2, sq3)); + + __m128i out0, out1, out2, out3; + LSX_TRANSPOSE4x4_W(sum_a, sum_b, sum_sq_ab, sum_axb, out0, out1, out2, + out3); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + __lsx_vst(out2, dst, 32); + __lsx_vst(out3, dst, 48); + + src_a += 16; + src_a1 += 16; + src_a2 += 16; + src_a3 += 16; + src_b += 16; + src_b1 += 16; + src_b2 += 16; + src_b3 += 16; + dst += 16; + } +} +#endif // HAS_SSIM4X4ROW_LSX + // undef for unified sources build #undef YUVTORGB_SETUP #undef READYUV422_D diff --git a/source/row_rvv.cc b/source/row_rvv.cc index ca87e4603..d64729845 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -15,6 +15,7 @@ * Contributed by Bruce Lai */ +#include "libyuv/compare_row.h" #include "libyuv/row.h" // Note: every register written by an inline asm block (vector, general @@ -2669,6 +2670,117 @@ void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) { } #endif // HAS_MIRRORUVROW_RVV +#ifdef HAS_SSIM4X4ROW_RVV +#define SSIM4X4_LOAD_RVV(ptr_a, ptr_b) \ + "vlseg4e8.v v0, (" #ptr_a \ + ") \n" \ + "vlseg4e8.v v4, (" #ptr_b \ + ") \n" \ + "vwaddu.vv v10, v0, v1 \n" \ + "vwaddu.vv v11, v2, v3 \n" \ + "vwaddu.vv v12, v4, v5 \n" \ + "vwaddu.vv v13, v6, v7 \n" \ + "vwmulu.vv v14, v0, v4 \n" \ + "vwmulu.vv v15, v1, v5 \n" \ + "vwmulu.vv v24, v2, v6 \n" \ + "vwmulu.vv v25, v3, v7 \n" \ + "vwmulu.vv v26, v0, v0 \n" \ + "vwmulu.vv v27, v1, v1 \n" \ + "vwmulu.vv v28, v2, v2 \n" \ + "vwmulu.vv v29, v3, v3 \n" \ + "vwmulu.vv v0, v4, v4 \n" \ + "vwmulu.vv v1, v5, v5 \n" \ + "vwmulu.vv v2, v6, v6 \n" \ + "vwmulu.vv v3, v7, v7 \n" + +#define SSIM4X4_ACCUM_RVV(ptr_a, ptr_b) \ + "vsetvli zero, zero, e8, mf2, ta, ma \n" \ + SSIM4X4_LOAD_RVV(ptr_a, ptr_b) \ + "vsetvli zero, zero, e16, m1, ta, ma \n" \ + "vadd.vv v10, v10, v11 \n" \ + "vadd.vv v12, v12, v13 \n" \ + "vadd.vv v8, v8, v10 \n" \ + "vadd.vv v9, v9, v12 \n" \ + "vwaddu.wv v22, v22, v14 \n" \ + "vwaddu.wv v22, v22, v15 \n" \ + "vwaddu.wv v22, v22, v24 \n" \ + "vwaddu.wv v22, v22, v25 \n" \ + "vwaddu.wv v20, v20, v26 \n" \ + "vwaddu.wv v20, v20, v27 \n" \ + "vwaddu.wv v20, v20, v28 \n" \ + "vwaddu.wv v20, v20, v29 \n" \ + "vwaddu.wv v20, v20, v0 \n" \ + "vwaddu.wv v20, v20, v1 \n" \ + "vwaddu.wv v20, v20, v2 \n" \ + "vwaddu.wv v20, v20, v3 \n" + +void Ssim4x4Row_RVV(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + size_t vl; + size_t byte_vl; + const uint8_t* tmp_a; + const uint8_t* tmp_b; + width >>= 2; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, mf2, ta, ma \n" + "add %[tmp_a], %[src_a], %[stride_a]\n" + "add %[tmp_b], %[src_b], %[stride_b]\n" + SSIM4X4_LOAD_RVV(%[src_a], %[src_b]) + "vsetvli zero, zero, e16, m1, ta, ma \n" + "vadd.vv v8, v10, v11 \n" + "vadd.vv v9, v12, v13 \n" + "vwaddu.vv v22, v14, v15 \n" + "vwaddu.wv v22, v22, v24 \n" + "vwaddu.wv v22, v22, v25 \n" + "vwaddu.vv v20, v26, v27 \n" + "vwaddu.wv v20, v20, v28 \n" + "vwaddu.wv v20, v20, v29 \n" + "vwaddu.wv v20, v20, v0 \n" + "vwaddu.wv v20, v20, v1 \n" + "vwaddu.wv v20, v20, v2 \n" + "vwaddu.wv v20, v20, v3 \n" + SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b]) + "add %[tmp_a], %[tmp_a], %[stride_a]\n" + "add %[tmp_b], %[tmp_b], %[stride_b]\n" + SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b]) + "add %[tmp_a], %[tmp_a], %[stride_a]\n" + "add %[tmp_b], %[tmp_b], %[stride_b]\n" + SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b]) + "vwcvtu.x.x.v v16, v8 \n" + "vwcvtu.x.x.v v18, v9 \n" + "vsetvli zero, zero, e32, m2, ta, ma \n" + "vsseg4e32.v v16, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[byte_vl], %[vl], 2 \n" + "add %[src_a], %[src_a], %[byte_vl]\n" + "add %[src_b], %[src_b], %[byte_vl]\n" + "slli %[byte_vl], %[vl], 4 \n" + "add %[dst], %[dst], %[byte_vl] \n" + "bgtz %[w], 1b \n" + : [src_a] "+r"(src_a), // %[src_a] + [src_b] "+r"(src_b), // %[src_b] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [byte_vl] "=&r"(byte_vl), // %[byte_vl] + [tmp_a] "=&r"(tmp_a), // %[tmp_a] + [tmp_b] "=&r"(tmp_b) // %[tmp_b] + : [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a] + [stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b] + : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", + "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", + "v27", "v28", "v29"); +} +#undef SSIM4X4_LOAD_RVV +#undef SSIM4X4_ACCUM_RVV +#endif // HAS_SSIM4X4ROW_RVV + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_sme.cc b/source/row_sme.cc index dac14ac34..4d9beacb6 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "libyuv/compare_row.h" #include "libyuv/row.h" #include "libyuv/row_sve.h" @@ -1122,6 +1123,17 @@ __arm_locally_streaming void ARGBToYMatrixRow_SME( ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c); } +#ifdef HAS_SSIM4X4ROW_SME +__arm_locally_streaming void Ssim4x4Row_SME(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width); +} +#endif // HAS_SSIM4X4ROW_SME + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/row_sve.cc b/source/row_sve.cc index efa1ac705..28ddc94fb 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -9,6 +9,7 @@ */ #include "libyuv/row_sve.h" +#include "libyuv/compare_row.h" #include "libyuv/row.h" #ifdef __cplusplus @@ -1724,6 +1725,17 @@ void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv, } #endif // HAS_HALFWIDTHROW_16TO8_SVE2 +#ifdef HAS_SSIM4X4ROW_SVE2 +void Ssim4x4Row_SVE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width); +} +#endif // HAS_SSIM4X4ROW_SVE2 + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/unit_test/compare_test.cc b/unit_test/compare_test.cc index 2456db281..c6d19fb37 100644 --- a/unit_test/compare_test.cc +++ b/unit_test/compare_test.cc @@ -654,7 +654,7 @@ TEST_F(LibYUVCompareTest, Psnr) { free_aligned_buffer_page_end(src_b); } -TEST_F(LibYUVCompareTest, DISABLED_BenchmarkSsim_Opt) { +TEST_F(LibYUVCompareTest, BenchmarkSsim_Opt) { align_buffer_page_end(src_a, benchmark_width_ * benchmark_height_); align_buffer_page_end(src_b, benchmark_width_ * benchmark_height_); for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) { @@ -679,6 +679,57 @@ TEST_F(LibYUVCompareTest, DISABLED_BenchmarkSsim_Opt) { free_aligned_buffer_page_end(src_b); } +TEST_F(LibYUVCompareTest, BenchmarkSsim_Unaligned) { + align_buffer_page_end(src_a, benchmark_width_ * benchmark_height_ + 1); + align_buffer_page_end(src_b, benchmark_width_ * benchmark_height_); + for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) { + src_a[i + 1] = i; + src_b[i] = i; + } + + MaskCpuFlags(benchmark_cpu_info_); + + double opt_time = get_time(); + for (int i = 0; i < benchmark_iterations_; ++i) { + CalcFrameSsim(src_a + 1, benchmark_width_, src_b, benchmark_width_, + benchmark_width_, benchmark_height_); + } + + opt_time = (get_time() - opt_time) / benchmark_iterations_; + printf("BenchmarkSsim_Opt - %8.2f us opt\n", opt_time * 1e6); + + ASSERT_EQ(0, 0); + + free_aligned_buffer_page_end(src_a); + free_aligned_buffer_page_end(src_b); +} + +TEST_F(LibYUVCompareTest, TestSsimAnyWidth) { + const int kMaxWidth = 140; + const int kHeight = 16; + const int kPlaneSize = kMaxWidth * kHeight + 1; + align_buffer_page_end(src_a, kPlaneSize); + align_buffer_page_end(src_b, kPlaneSize); + MemRandomize(src_a, kPlaneSize); + MemRandomize(src_b, kPlaneSize); + + for (int width = 9; width <= kMaxWidth; ++width) { + for (int offset = 0; offset < 2; ++offset) { + const int size = width * kHeight; + const uint8_t* a = src_a + kPlaneSize - size - offset; + const uint8_t* b = src_b + kPlaneSize - size - offset; + MaskCpuFlags(disable_cpu_flags_); + double c_ssim = CalcFrameSsim(a, width, b, width, width, kHeight); + MaskCpuFlags(benchmark_cpu_info_); + double opt_ssim = CalcFrameSsim(a, width, b, width, width, kHeight); + EXPECT_EQ(c_ssim, opt_ssim) << "width " << width << " offset " << offset; + } + } + + free_aligned_buffer_page_end(src_a); + free_aligned_buffer_page_end(src_b); +} + TEST_F(LibYUVCompareTest, Ssim) { const int kSrcWidth = benchmark_width_; const int kSrcHeight = benchmark_height_;