diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index 3c7722baf..6a66f0785 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -32,12 +32,15 @@ extern "C" { #define HAS_HAMMINGDISTANCE_SSE42 #define HAS_SUMSQUAREERROR_SSE2 #define HAS_HASHDJB2_SSE41 +#define HAS_SSIM4X4ROW_SSE2 #define HAS_HAMMINGDISTANCE_AVX2 #define HAS_SUMSQUAREERROR_AVX2 #define HAS_HASHDJB2_AVX2 +#define HAS_SSIM4X4ROW_AVX2 #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW #define HAS_HASHDJB2_AVX512BW +#define HAS_SSIM4X4ROW_AVX512BW #endif // The following are available for GCC and clang 32/64 bit, as inline assembly @@ -49,15 +52,18 @@ extern "C" { #define HAS_HAMMINGDISTANCE_SSSE3 #define HAS_SUMSQUAREERROR_SSE2 #define HAS_HASHDJB2_SSE41 +#define HAS_SSIM4X4ROW_SSE2 #if defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2) #define HAS_HAMMINGDISTANCE_AVX2 #define HAS_SUMSQUAREERROR_AVX2 #define HAS_HASHDJB2_AVX2 +#define HAS_SSIM4X4ROW_AVX2 #endif #if defined(CLANG_HAS_AVX512) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW #define HAS_HASHDJB2_AVX512BW +#define HAS_SSIM4X4ROW_AVX512BW #endif #endif @@ -66,6 +72,7 @@ extern "C" { (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) #define HAS_HAMMINGDISTANCE_NEON #define HAS_SUMSQUAREERROR_NEON +#define HAS_SSIM4X4ROW_NEON #endif // The following are available for AArch64 Neon: @@ -74,6 +81,34 @@ extern "C" { #define HAS_HAMMINGDISTANCE_NEON_DOTPROD #define HAS_SUMSQUAREERROR_NEON_DOTPROD +#define HAS_SSIM4X4ROW_NEON_DOTPROD +#endif + +// The following are available for AArch64 SVE: +#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) +#define HAS_SSIM4X4ROW_SVE2 +#endif + +// The following are available for AArch64 SME: +#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ + defined(__aarch64__) +#define HAS_SSIM4X4ROW_SME +#endif + +// The following are available for RISC-V Vector (RVV): +#if !defined(LIBYUV_DISABLE_RVV) && \ + (defined(__riscv_vector) || defined(LIBYUV_RVV)) +#define HAS_SSIM4X4ROW_RVV +#endif + +// The following are available for LoongArch LSX: +#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) +#define HAS_SSIM4X4ROW_LSX +#endif + +// The following are available for LoongArch LASX: +#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#define HAS_SSIM4X4ROW_LASX #endif // The following are available for WebAssembly SIMD128: @@ -81,6 +116,7 @@ extern "C" { #define HAS_HAMMINGDISTANCE_WASMSIMD #define HAS_SUMSQUAREERROR_WASMSIMD #define HAS_HASHDJB2_WASMSIMD +#define HAS_SSIM4X4ROW_WASMSIMD #endif uint32_t HammingDistance_C(const uint8_t* src_a, @@ -136,6 +172,82 @@ uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed); uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed); uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed); +void Ssim4x4Row_C(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_SSE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_AVX2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_AVX512BW(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_NEON(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_NEON_DotProd(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_SVE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_SME(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_RVV(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_LSX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_LASX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +void Ssim4x4Row_WASMSIMD(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width); +double SsimWindowRow_C(const uint32_t* row0, + const uint32_t* row1, + int num_windows); + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 467ec199b..7e0ca8776 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -2224,6 +2224,93 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, "z20", "z24", "z25", "p0", "p1", "p2", "p3", "p4", "p5"); } +#define SSIM4X4_SVE \ + "add %[tmp_a], %[src_a], %[stride_a] \n" \ + "add %[tmp_b], %[src_b], %[stride_b] \n" \ + "ld1w {z0.s}, p0/z, [%[src_a]] \n" \ + "ld1w {z1.s}, p0/z, [%[tmp_a]] \n" \ + "add %[tmp_a], %[tmp_a], %[stride_a] \n" \ + "ld1w {z4.s}, p0/z, [%[src_b]] \n" \ + "ld1w {z5.s}, p0/z, [%[tmp_b]] \n" \ + "add %[tmp_b], %[tmp_b], %[stride_b] \n" \ + "ld1w {z2.s}, p0/z, [%[tmp_a]] \n" \ + "add %[tmp_a], %[tmp_a], %[stride_a] \n" \ + "ld1w {z3.s}, p0/z, [%[tmp_a]] \n" \ + "ld1w {z6.s}, p0/z, [%[tmp_b]] \n" \ + "add %[tmp_b], %[tmp_b], %[stride_b] \n" \ + "ld1w {z7.s}, p0/z, [%[tmp_b]] \n" \ + "incb %[src_a] \n" \ + "incb %[src_b] \n" \ + "mov z16.s, #0 \n" \ + "mov z17.s, #0 \n" \ + "mov z18.s, #0 \n" \ + "mov z19.s, #0 \n" \ + "udot z16.s, z0.b, z20.b \n" \ + "udot z17.s, z4.b, z20.b \n" \ + "udot z18.s, z0.b, z0.b \n" \ + "udot z19.s, z0.b, z4.b \n" \ + "udot z16.s, z1.b, z20.b \n" \ + "udot z17.s, z5.b, z20.b \n" \ + "udot z18.s, z1.b, z1.b \n" \ + "udot z19.s, z1.b, z5.b \n" \ + "udot z16.s, z2.b, z20.b \n" \ + "udot z17.s, z6.b, z20.b \n" \ + "udot z18.s, z2.b, z2.b \n" \ + "udot z19.s, z2.b, z6.b \n" \ + "udot z16.s, z3.b, z20.b \n" \ + "udot z17.s, z7.b, z20.b \n" \ + "udot z18.s, z3.b, z3.b \n" \ + "udot z19.s, z3.b, z7.b \n" \ + "udot z18.s, z4.b, z4.b \n" \ + "udot z18.s, z5.b, z5.b \n" \ + "udot z18.s, z6.b, z6.b \n" \ + "udot z18.s, z7.b, z7.b \n" \ + "subs %w[width], %w[width], %w[vl] \n" \ + "st4w {z16.s, z17.s, z18.s, z19.s}, p0, [%[dst]]\n" \ + "incb %[dst], all, mul #4 \n" + +static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) STREAMING_COMPATIBLE { + uint64_t vl; + const uint8_t* tmp_a; + const uint8_t* tmp_b; + width >>= 2; + asm volatile( + "cntw %x[vl] \n" + "mov z20.b, #1 \n" + "subs %w[width], %w[width], %w[vl] \n" + "b.lt 2f \n" + + "ptrue p0.s \n" + "1: \n" // + SSIM4X4_SVE + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" + + "whilelt p0.s, wzr, %w[width] \n" // + SSIM4X4_SVE + + "99: \n" + : [src_a] "+r"(src_a), // %[src_a] + [src_b] "+r"(src_b), // %[src_b] + [dst] "+r"(dst), // %[dst] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [tmp_a] "=&r"(tmp_a), // %[tmp_a] + [tmp_b] "=&r"(tmp_b) // %[tmp_b] + : [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a] + [stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b] + : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", + "z17", "z18", "z19", "z20", "p0"); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/compare.cc b/source/compare.cc index 9c3a5ed16..5828bd15f 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -427,20 +427,143 @@ double CalcFrameSsim(const uint8_t* src_a, int height) { int samples = 0; double ssim_total = 0; - double (*Ssim8x8)(const uint8_t* src_a, int stride_a, const uint8_t* src_b, - int stride_b) = Ssim8x8_C; + if (width <= 8 || height <= 8) { + return ssim_total / samples; + } - // sample point start with each 4x4 location - int i; - for (i = 0; i < height - 8; i += 4) { - int j; - for (j = 0; j < width - 8; j += 4) { - ssim_total += Ssim8x8(src_a + j, stride_a, src_b + j, stride_b); - samples++; + int simd_size = 4; + void (*Ssim4x4Row)(const uint8_t* src_a, int stride_a, const uint8_t* src_b, + int stride_b, uint32_t* dst, int width) = Ssim4x4Row_C; +#if defined(HAS_SSIM4X4ROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + Ssim4x4Row = Ssim4x4Row_SSE2; + simd_size = 16; + } +#endif +#if defined(HAS_SSIM4X4ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + Ssim4x4Row = Ssim4x4Row_AVX2; + simd_size = 32; + } +#endif +#if defined(HAS_SSIM4X4ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + Ssim4x4Row = Ssim4x4Row_AVX512BW; + simd_size = 4; + } +#endif +#if defined(HAS_SSIM4X4ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + Ssim4x4Row = Ssim4x4Row_NEON; + simd_size = 16; + } +#endif +#if defined(HAS_SSIM4X4ROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + Ssim4x4Row = Ssim4x4Row_NEON_DotProd; + simd_size = 16; + } +#endif +#if defined(HAS_SSIM4X4ROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + Ssim4x4Row = Ssim4x4Row_SVE2; + simd_size = 4; + } +#endif +#if defined(HAS_SSIM4X4ROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + Ssim4x4Row = Ssim4x4Row_SME; + simd_size = 4; + } +#endif +#if defined(HAS_SSIM4X4ROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + Ssim4x4Row = Ssim4x4Row_RVV; + simd_size = 4; + } +#endif +#if defined(HAS_SSIM4X4ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Ssim4x4Row = Ssim4x4Row_LSX; + simd_size = 16; + } +#endif +#if defined(HAS_SSIM4X4ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Ssim4x4Row = Ssim4x4Row_LASX; + simd_size = 32; + } +#endif +#if defined(HAS_SSIM4X4ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Ssim4x4Row = Ssim4x4Row_WASMSIMD; + simd_size = 16; + } +#endif + + const int num_windows = (width - 8 + 3) >> 2; + const int num_tiles = num_windows + 1; + const int tile_width = num_tiles * 4; + const int main_width = tile_width & ~(simd_size - 1); + const int rem_width = tile_width & (simd_size - 1); + +#ifdef _OPENMP +#pragma omp parallel reduction(+ : ssim_total, samples) +#endif + { + align_buffer_64(row_buf, (size_t)tile_width * 2 * sizeof(uint32_t)); + if (row_buf) { + uint32_t* row0 = (uint32_t*)row_buf; + uint32_t* row1 = row0 + tile_width; + int prev_i = -8; + int i; +#ifdef _OPENMP +#pragma omp for schedule(static) +#endif + for (i = 0; i < height - 8; i += 4) { + const uint8_t* a = src_a + (ptrdiff_t)i * stride_a; + const uint8_t* b = src_b + (ptrdiff_t)i * stride_b; + if (prev_i + 4 != i) { + if (main_width > 0) { + Ssim4x4Row(a, stride_a, b, stride_b, row0, main_width); + } + if (rem_width > 0) { + Ssim4x4Row_C(a + main_width, stride_a, b + main_width, stride_b, + row0 + main_width, rem_width); + } + } + a += (ptrdiff_t)stride_a * 4; + b += (ptrdiff_t)stride_b * 4; + if (main_width > 0) { + Ssim4x4Row(a, stride_a, b, stride_b, row1, main_width); + } + if (rem_width > 0) { + Ssim4x4Row_C(a + main_width, stride_a, b + main_width, stride_b, + row1 + main_width, rem_width); + } + ssim_total += SsimWindowRow_C(row0, row1, num_windows); + samples += num_windows; + uint32_t* tmp = row0; + row0 = row1; + row1 = tmp; + prev_i = i; + } + free_aligned_buffer_64(row_buf); + } else { + int i; +#ifdef _OPENMP +#pragma omp for schedule(static) +#endif + for (i = 0; i < height - 8; i += 4) { + const uint8_t* a = src_a + (ptrdiff_t)i * stride_a; + const uint8_t* b = src_b + (ptrdiff_t)i * stride_b; + int j; + for (j = 0; j < width - 8; j += 4) { + ssim_total += Ssim8x8_C(a + j, stride_a, b + j, stride_b); + samples++; + } + } } - - src_a += stride_a * 4; - src_b += stride_b * 4; } ssim_total /= samples; diff --git a/source/compare_common.cc b/source/compare_common.cc index c7c4fe879..f375a5839 100644 --- a/source/compare_common.cc +++ b/source/compare_common.cc @@ -10,6 +10,8 @@ #include "libyuv/compare_row.h" +#include + #ifdef __cplusplus namespace libyuv { extern "C" { @@ -66,6 +68,108 @@ uint32_t HashDjb2_C(const uint8_t* src, int count, uint32_t seed) { return hash; } +// Compute 4x4 tile statistics (sum_a, sum_b, sum_sq_a + sum_sq_b, sum_axb) +// across 4 rows for width pixels (width is a multiple of 4). +void Ssim4x4Row_C(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + int x; + for (x = 0; x < width; x += 4) { + uint32_t sum_a = a0[0] + a0[1] + a0[2] + a0[3] + a1[0] + a1[1] + a1[2] + + a1[3] + a2[0] + a2[1] + a2[2] + a2[3] + a3[0] + a3[1] + + a3[2] + a3[3]; + uint32_t sum_b = b0[0] + b0[1] + b0[2] + b0[3] + b1[0] + b1[1] + b1[2] + + b1[3] + b2[0] + b2[1] + b2[2] + b2[3] + b3[0] + b3[1] + + b3[2] + b3[3]; + uint32_t sum_sq_ab = + a0[0] * a0[0] + a0[1] * a0[1] + a0[2] * a0[2] + a0[3] * a0[3] + + a1[0] * a1[0] + a1[1] * a1[1] + a1[2] * a1[2] + a1[3] * a1[3] + + a2[0] * a2[0] + a2[1] * a2[1] + a2[2] * a2[2] + a2[3] * a2[3] + + a3[0] * a3[0] + a3[1] * a3[1] + a3[2] * a3[2] + a3[3] * a3[3] + + b0[0] * b0[0] + b0[1] * b0[1] + b0[2] * b0[2] + b0[3] * b0[3] + + b1[0] * b1[0] + b1[1] * b1[1] + b1[2] * b1[2] + b1[3] * b1[3] + + b2[0] * b2[0] + b2[1] * b2[1] + b2[2] * b2[2] + b2[3] * b2[3] + + b3[0] * b3[0] + b3[1] * b3[1] + b3[2] * b3[2] + b3[3] * b3[3]; + uint32_t sum_axb = + a0[0] * b0[0] + a0[1] * b0[1] + a0[2] * b0[2] + a0[3] * b0[3] + + a1[0] * b1[0] + a1[1] * b1[1] + a1[2] * b1[2] + a1[3] * b1[3] + + a2[0] * b2[0] + a2[1] * b2[1] + a2[2] * b2[2] + a2[3] * b2[3] + + a3[0] * b3[0] + a3[1] * b3[1] + a3[2] * b3[2] + a3[3] * b3[3]; + dst[0] = sum_a; + dst[1] = sum_b; + dst[2] = sum_sq_ab; + dst[3] = sum_axb; + a0 += 4; + a1 += 4; + a2 += 4; + a3 += 4; + b0 += 4; + b1 += 4; + b2 += 4; + b3 += 4; + dst += 4; + } +} + +// Combine two consecutive 4x4 tile rows (row0 and row1) into 8x8 windows +// stepped on a 4-pixel horizontal grid and return the sum of SSIM values. +double SsimWindowRow_C(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + const int64_t count = 64; + // scale the constants by number of pixels + const int64_t cc1 = 26634; // (64^2*(.01*255)^2) + const int64_t cc2 = 239708; // (64^2*(.03*255)^2) + const int64_t c1 = (cc1 * count * count) >> 12; + const int64_t c2 = (cc2 * count * count) >> 12; + + double ssim_total = 0.0; + uint32_t prev_a = row0[0] + row1[0]; + uint32_t prev_b = row0[1] + row1[1]; + uint32_t prev_sq = row0[2] + row1[2]; + uint32_t prev_axb = row0[3] + row1[3]; + int w; + for (w = 0; w < num_windows; ++w) { + row0 += 4; + row1 += 4; + const uint32_t next_a = row0[0] + row1[0]; + const uint32_t next_b = row0[1] + row1[1]; + const uint32_t next_sq = row0[2] + row1[2]; + const uint32_t next_axb = row0[3] + row1[3]; + const int64_t sum_a = prev_a + next_a; + const int64_t sum_b = prev_b + next_b; + const int64_t sum_sq_ab = prev_sq + next_sq; + const int64_t sum_axb = prev_axb + next_axb; + prev_a = next_a; + prev_b = next_b; + prev_sq = next_sq; + prev_axb = next_axb; + + const int64_t sum_a_x_sum_b = sum_a * sum_b; + const int64_t ssim_n = (2 * sum_a_x_sum_b + c1) * + (2 * count * sum_axb - 2 * sum_a_x_sum_b + c2); + const int64_t sum_a_sq = sum_a * sum_a; + const int64_t sum_b_sq = sum_b * sum_b; + const int64_t ssim_d = (sum_a_sq + sum_b_sq + c1) * + (count * sum_sq_ab - sum_a_sq - sum_b_sq + c2); + ssim_total += ssim_n * 1.0 / ssim_d; + } + return ssim_total; +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/compare_gcc.cc b/source/compare_gcc.cc index 723d2aef4..2053e78c8 100644 --- a/source/compare_gcc.cc +++ b/source/compare_gcc.cc @@ -407,8 +407,9 @@ uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) { "xmm7"); return hash; } -#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \ - defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW) +#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \ + defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW) || \ + defined(HAS_SSIM4X4ROW_AVX512BW) // Mask of the low n bytes, for n = 0 to 63. static uint64_t LowBytesMask64(int n) { return n ? ~0ULL >> (64 - n) : 0ULL; @@ -749,6 +750,377 @@ uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) { #undef HASH64_AVX512BW #endif // HAS_HASHDJB2_AVX512BW +#ifdef HAS_SSIM4X4ROW_SSE2 +#define SSIM_ROW0_SSE2(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "movdqa %%xmm0,%%xmm2 \n" \ + "psrlw $0x8,%%xmm0 \n" \ + "psllw $0x8,%%xmm2 \n" \ + "psrlw $0x8,%%xmm2 \n" \ + "movdqa %%xmm0,%%xmm4 \n" \ + "paddw %%xmm2,%%xmm4 \n" \ + "movdqa %%xmm0,%%xmm6 \n" \ + "pmaddwd %%xmm6,%%xmm6 \n" \ + "movdqa %%xmm2,%%xmm3 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm3,%%xmm6 \n" \ + "movdqa %%xmm1,%%xmm3 \n" \ + "psrlw $0x8,%%xmm1 \n" \ + "psllw $0x8,%%xmm3 \n" \ + "psrlw $0x8,%%xmm3 \n" \ + "movdqa %%xmm1,%%xmm5 \n" \ + "paddw %%xmm3,%%xmm5 \n" \ + "pmaddwd %%xmm1,%%xmm0 \n" \ + "pmaddwd %%xmm3,%%xmm2 \n" \ + "movdqa %%xmm0,%%xmm7 \n" \ + "paddd %%xmm2,%%xmm7 \n" \ + "pmaddwd %%xmm1,%%xmm1 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm1,%%xmm6 \n" \ + "paddd %%xmm3,%%xmm6 \n" + +#define SSIM_ROW_SSE2(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "movdqa %%xmm0,%%xmm2 \n" \ + "psrlw $0x8,%%xmm0 \n" \ + "psllw $0x8,%%xmm2 \n" \ + "psrlw $0x8,%%xmm2 \n" \ + "paddw %%xmm0,%%xmm4 \n" \ + "paddw %%xmm2,%%xmm4 \n" \ + "movdqa %%xmm0,%%xmm3 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm3,%%xmm6 \n" \ + "movdqa %%xmm2,%%xmm3 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm3,%%xmm6 \n" \ + "movdqa %%xmm1,%%xmm3 \n" \ + "psrlw $0x8,%%xmm1 \n" \ + "psllw $0x8,%%xmm3 \n" \ + "psrlw $0x8,%%xmm3 \n" \ + "paddw %%xmm1,%%xmm5 \n" \ + "paddw %%xmm3,%%xmm5 \n" \ + "pmaddwd %%xmm1,%%xmm0 \n" \ + "pmaddwd %%xmm3,%%xmm2 \n" \ + "paddd %%xmm0,%%xmm7 \n" \ + "paddd %%xmm2,%%xmm7 \n" \ + "pmaddwd %%xmm1,%%xmm1 \n" \ + "pmaddwd %%xmm3,%%xmm3 \n" \ + "paddd %%xmm1,%%xmm6 \n" \ + "paddd %%xmm3,%%xmm6 \n" + +void Ssim4x4Row_SSE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + asm volatile( + LABELALIGN "1: \n" // + SSIM_ROW0_SSE2("movdqu (%[src_a]),%%xmm0\n", + "movdqu (%[src_b]),%%xmm1\n") // + "add %[stride_a],%[src_a] \n" + "add %[stride_b],%[src_b] \n" // + SSIM_ROW_SSE2("movdqu (%[src_a]),%%xmm0\n", + "movdqu (%[src_b]),%%xmm1\n") // + SSIM_ROW_SSE2("movdqu (%[src_a],%[stride_a]),%%xmm0\n", + "movdqu (%[src_b],%[stride_b]),%%xmm1\n") // + SSIM_ROW_SSE2("movdqu (%[src_a],%[stride_a],2),%%xmm0\n", + "movdqu (%[src_b],%[stride_b],2),%%xmm1\n") // + "sub %[stride_a],%[src_a] \n" + "sub %[stride_b],%[src_b] \n" + "add $0x10,%[src_a] \n" + "add $0x10,%[src_b] \n" + "pcmpeqd %%xmm0,%%xmm0 \n" + "psrlw $0xf,%%xmm0 \n" + "pmaddwd %%xmm0,%%xmm4 \n" + "pmaddwd %%xmm0,%%xmm5 \n" + "movdqa %%xmm4,%%xmm0 \n" + "punpckldq %%xmm5,%%xmm0 \n" + "punpckhdq %%xmm5,%%xmm4 \n" + "movdqa %%xmm6,%%xmm1 \n" + "punpckldq %%xmm7,%%xmm1 \n" + "punpckhdq %%xmm7,%%xmm6 \n" + "movdqa %%xmm0,%%xmm2 \n" + "punpcklqdq %%xmm1,%%xmm2 \n" + "punpckhqdq %%xmm1,%%xmm0 \n" + "movdqa %%xmm4,%%xmm3 \n" + "punpcklqdq %%xmm6,%%xmm3 \n" + "punpckhqdq %%xmm6,%%xmm4 \n" + "movdqu %%xmm2,0x00(%[dst]) \n" + "movdqu %%xmm0,0x10(%[dst]) \n" + "movdqu %%xmm3,0x20(%[dst]) \n" + "movdqu %%xmm4,0x30(%[dst]) \n" + "add $0x40,%[dst] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), + [width] "+r"(width) + : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#undef SSIM_ROW0_SSE2 +#undef SSIM_ROW_SSE2 +#endif // HAS_SSIM4X4ROW_SSE2 + +#ifdef HAS_SSIM4X4ROW_AVX2 +#define SSIM_ROW0_AVX2(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "vpsrlw $0x8,%%ymm0,%%ymm2 \n" \ + "vpsllw $0x8,%%ymm0,%%ymm0 \n" \ + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" \ + "vpaddw %%ymm0,%%ymm2,%%ymm4 \n" \ + "vpmaddwd %%ymm2,%%ymm2,%%ymm6 \n" \ + "vpmaddwd %%ymm0,%%ymm0,%%ymm3 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpsrlw $0x8,%%ymm1,%%ymm3 \n" \ + "vpsllw $0x8,%%ymm1,%%ymm1 \n" \ + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" \ + "vpaddw %%ymm1,%%ymm3,%%ymm5 \n" \ + "vpmaddwd %%ymm3,%%ymm2,%%ymm7 \n" \ + "vpmaddwd %%ymm1,%%ymm0,%%ymm0 \n" \ + "vpaddd %%ymm0,%%ymm7,%%ymm7 \n" \ + "vpmaddwd %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpmaddwd %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpaddd %%ymm1,%%ymm6,%%ymm6 \n" + +#define SSIM_ROW_AVX2(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "vpsrlw $0x8,%%ymm0,%%ymm2 \n" \ + "vpsllw $0x8,%%ymm0,%%ymm0 \n" \ + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" \ + "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" \ + "vpaddw %%ymm0,%%ymm4,%%ymm4 \n" \ + "vpmaddwd %%ymm2,%%ymm2,%%ymm3 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpmaddwd %%ymm0,%%ymm0,%%ymm3 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpsrlw $0x8,%%ymm1,%%ymm3 \n" \ + "vpsllw $0x8,%%ymm1,%%ymm1 \n" \ + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" \ + "vpaddw %%ymm3,%%ymm5,%%ymm5 \n" \ + "vpaddw %%ymm1,%%ymm5,%%ymm5 \n" \ + "vpmaddwd %%ymm3,%%ymm2,%%ymm2 \n" \ + "vpmaddwd %%ymm1,%%ymm0,%%ymm0 \n" \ + "vpaddd %%ymm2,%%ymm7,%%ymm7 \n" \ + "vpaddd %%ymm0,%%ymm7,%%ymm7 \n" \ + "vpmaddwd %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpmaddwd %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \ + "vpaddd %%ymm1,%%ymm6,%%ymm6 \n" + +void Ssim4x4Row_AVX2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + asm volatile( + LABELALIGN "1: \n" // + SSIM_ROW0_AVX2("vmovdqu (%[src_a]),%%ymm0\n", + "vmovdqu (%[src_b]),%%ymm1\n") // + "add %[stride_a],%[src_a] \n" + "add %[stride_b],%[src_b] \n" // + SSIM_ROW_AVX2("vmovdqu (%[src_a]),%%ymm0\n", + "vmovdqu (%[src_b]),%%ymm1\n") // + SSIM_ROW_AVX2("vmovdqu (%[src_a],%[stride_a]),%%ymm0\n", + "vmovdqu (%[src_b],%[stride_b]),%%ymm1\n") // + SSIM_ROW_AVX2("vmovdqu (%[src_a],%[stride_a],2),%%ymm0\n", + "vmovdqu (%[src_b],%[stride_b],2),%%ymm1\n") // + "sub %[stride_a],%[src_a] \n" + "sub %[stride_b],%[src_b] \n" + "add $0x20,%[src_a] \n" + "add $0x20,%[src_b] \n" + "vpcmpeqd %%ymm0,%%ymm0,%%ymm0 \n" + "vpsrlw $0xf,%%ymm0,%%ymm0 \n" + "vpmaddwd %%ymm0,%%ymm4,%%ymm4 \n" + "vpmaddwd %%ymm0,%%ymm5,%%ymm5 \n" + "vpunpckldq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpckldq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhdq %%ymm7,%%ymm6,%%ymm3 \n" + "vpunpcklqdq %%ymm2,%%ymm0,%%ymm4 \n" + "vpunpckhqdq %%ymm2,%%ymm0,%%ymm5 \n" + "vpunpcklqdq %%ymm3,%%ymm1,%%ymm6 \n" + "vpunpckhqdq %%ymm3,%%ymm1,%%ymm7 \n" + "vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm0 \n" + "vperm2i128 $0x20,%%ymm7,%%ymm6,%%ymm1 \n" + "vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm2 \n" + "vperm2i128 $0x31,%%ymm7,%%ymm6,%%ymm3 \n" + "vmovdqu %%ymm0,0x00(%[dst]) \n" + "vmovdqu %%ymm1,0x20(%[dst]) \n" + "vmovdqu %%ymm2,0x40(%[dst]) \n" + "vmovdqu %%ymm3,0x60(%[dst]) \n" + "add $0x80,%[dst] \n" + "sub $0x20,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), + [width] "+r"(width) + : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#undef SSIM_ROW0_AVX2 +#undef SSIM_ROW_AVX2 +#endif // HAS_SSIM4X4ROW_AVX2 + +#ifdef HAS_SSIM4X4ROW_AVX512BW +#define SSIM_ROW0_AVX512BW(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "vpsrlw $0x8,%%zmm0,%%zmm2 \n" \ + "vpsllw $0x8,%%zmm0,%%zmm0 \n" \ + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" \ + "vpaddw %%zmm0,%%zmm2,%%zmm4 \n" \ + "vpmaddwd %%zmm2,%%zmm2,%%zmm6 \n" \ + "vpmaddwd %%zmm0,%%zmm0,%%zmm3 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpsrlw $0x8,%%zmm1,%%zmm3 \n" \ + "vpsllw $0x8,%%zmm1,%%zmm1 \n" \ + "vpsrlw $0x8,%%zmm1,%%zmm1 \n" \ + "vpaddw %%zmm1,%%zmm3,%%zmm5 \n" \ + "vpmaddwd %%zmm3,%%zmm2,%%zmm7 \n" \ + "vpmaddwd %%zmm1,%%zmm0,%%zmm0 \n" \ + "vpaddd %%zmm0,%%zmm7,%%zmm7 \n" \ + "vpmaddwd %%zmm3,%%zmm3,%%zmm3 \n" \ + "vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpaddd %%zmm1,%%zmm6,%%zmm6 \n" + +#define SSIM_ROW_AVX512BW(LOAD_A, LOAD_B) \ + LOAD_A \ + LOAD_B \ + "vpsrlw $0x8,%%zmm0,%%zmm2 \n" \ + "vpsllw $0x8,%%zmm0,%%zmm0 \n" \ + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" \ + "vpaddw %%zmm2,%%zmm4,%%zmm4 \n" \ + "vpaddw %%zmm0,%%zmm4,%%zmm4 \n" \ + "vpmaddwd %%zmm2,%%zmm2,%%zmm3 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpmaddwd %%zmm0,%%zmm0,%%zmm3 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpsrlw $0x8,%%zmm1,%%zmm3 \n" \ + "vpsllw $0x8,%%zmm1,%%zmm1 \n" \ + "vpsrlw $0x8,%%zmm1,%%zmm1 \n" \ + "vpaddw %%zmm3,%%zmm5,%%zmm5 \n" \ + "vpaddw %%zmm1,%%zmm5,%%zmm5 \n" \ + "vpmaddwd %%zmm3,%%zmm2,%%zmm2 \n" \ + "vpmaddwd %%zmm1,%%zmm0,%%zmm0 \n" \ + "vpaddd %%zmm2,%%zmm7,%%zmm7 \n" \ + "vpaddd %%zmm0,%%zmm7,%%zmm7 \n" \ + "vpmaddwd %%zmm3,%%zmm3,%%zmm3 \n" \ + "vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n" \ + "vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \ + "vpaddd %%zmm1,%%zmm6,%%zmm6 \n" + +#define SSIM_TRANSPOSE_AVX512BW \ + "vpternlogd $0xff,%%zmm0,%%zmm0,%%zmm0 \n" \ + "vpsrlw $0xf,%%zmm0,%%zmm0 \n" \ + "vpmaddwd %%zmm0,%%zmm4,%%zmm4 \n" \ + "vpmaddwd %%zmm0,%%zmm5,%%zmm5 \n" \ + "vpunpckldq %%zmm5,%%zmm4,%%zmm0 \n" \ + "vpunpckhdq %%zmm5,%%zmm4,%%zmm1 \n" \ + "vpunpckldq %%zmm7,%%zmm6,%%zmm2 \n" \ + "vpunpckhdq %%zmm7,%%zmm6,%%zmm3 \n" \ + "vpunpcklqdq %%zmm2,%%zmm0,%%zmm4 \n" \ + "vpunpckhqdq %%zmm2,%%zmm0,%%zmm5 \n" \ + "vpunpcklqdq %%zmm3,%%zmm1,%%zmm6 \n" \ + "vpunpckhqdq %%zmm3,%%zmm1,%%zmm7 \n" \ + "vshufi32x4 $0x44,%%zmm5,%%zmm4,%%zmm0 \n" \ + "vshufi32x4 $0x44,%%zmm7,%%zmm6,%%zmm1 \n" \ + "vshufi32x4 $0xee,%%zmm5,%%zmm4,%%zmm2 \n" \ + "vshufi32x4 $0xee,%%zmm7,%%zmm6,%%zmm3 \n" \ + "vshufi32x4 $0x88,%%zmm1,%%zmm0,%%zmm4 \n" \ + "vshufi32x4 $0xdd,%%zmm1,%%zmm0,%%zmm5 \n" \ + "vshufi32x4 $0x88,%%zmm3,%%zmm2,%%zmm6 \n" \ + "vshufi32x4 $0xdd,%%zmm3,%%zmm2,%%zmm7 \n" + +// Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60 +// pixels (1 to 15 tiles) with masked loads and stores. +void Ssim4x4Row_AVX512BW(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const int rem = width & 60; + const uint16_t load_mask = rem ? (uint16_t)((1U << (rem >> 2)) - 1U) : 0U; + const uint64_t store_mask = LowBytesMask64(rem); + asm volatile( + "sub $0x40,%[width] \n" + "jl 2f \n" + + LABELALIGN "1: \n" // + SSIM_ROW0_AVX512BW("vmovdqu8 (%[src_a]),%%zmm0\n", + "vmovdqu8 (%[src_b]),%%zmm1\n") // + "add %[stride_a],%[src_a] \n" + "add %[stride_b],%[src_b] \n" // + SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a]),%%zmm0\n", + "vmovdqu8 (%[src_b]),%%zmm1\n") // + SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a],%[stride_a]),%%zmm0\n", + "vmovdqu8 (%[src_b],%[stride_b]),%%zmm1\n") // + SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a],%[stride_a],2),%%zmm0\n", + "vmovdqu8 (%[src_b],%[stride_b],2),%%zmm1\n") // + "sub %[stride_a],%[src_a] \n" + "sub %[stride_b],%[src_b] \n" + "add $0x40,%[src_a] \n" + "add $0x40,%[src_b] \n" // + SSIM_TRANSPOSE_AVX512BW + "vmovdqu32 %%zmm4,0x00(%[dst]) \n" + "vmovdqu32 %%zmm5,0x40(%[dst]) \n" + "vmovdqu32 %%zmm6,0x80(%[dst]) \n" + "vmovdqu32 %%zmm7,0xc0(%[dst]) \n" + "add $0x100,%[dst] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "je 3f \n" + "kmovw %[load_mask],%%k1 \n" // + SSIM_ROW0_AVX512BW("vmovdqu32 (%[src_a]),%%zmm0%{%%k1%}%{z%}\n", + "vmovdqu32 (%[src_b]),%%zmm1%{%%k1%}%{z%}\n") // + "add %[stride_a],%[src_a] \n" + "add %[stride_b],%[src_b] \n" // + SSIM_ROW_AVX512BW("vmovdqu32 (%[src_a]),%%zmm0%{%%k1%}%{z%}\n", + "vmovdqu32 (%[src_b]),%%zmm1%{%%k1%}%{z%}\n") // + SSIM_ROW_AVX512BW( + "vmovdqu32 (%[src_a],%[stride_a]),%%zmm0%{%%k1%}%{z%}\n", + "vmovdqu32 (%[src_b],%[stride_b]),%%zmm1%{%%k1%}%{z%}\n") // + SSIM_ROW_AVX512BW( + "vmovdqu32 (%[src_a],%[stride_a],2),%%zmm0%{%%k1%}%{z%}\n", + "vmovdqu32 (%[src_b],%[stride_b],2),%%zmm1%{%%k1%}%{z%}\n") // + SSIM_TRANSPOSE_AVX512BW + "kmovq %[store_mask],%%k1 \n" + "vmovdqu32 %%zmm4,0x00(%[dst])%{%%k1%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 %%zmm5,0x40(%[dst])%{%%k1%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 %%zmm6,0x80(%[dst])%{%%k1%} \n" + "kshiftrq $0x10,%%k1,%%k1 \n" + "vmovdqu32 %%zmm7,0xc0(%[dst])%{%%k1%} \n" + + "3: \n" + "vzeroupper \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), + [width] "+r"(width) + : [stride_a] "r"((ptrdiff_t)stride_a), + [stride_b] "r"((ptrdiff_t)stride_b), [load_mask] "m"(load_mask), + [store_mask] "m"(store_mask) + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#undef SSIM_ROW0_AVX512BW +#undef SSIM_ROW_AVX512BW +#undef SSIM_TRANSPOSE_AVX512BW +#endif // HAS_SSIM4X4ROW_AVX512BW + #endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) && // !defined(LIBYUV_ENABLE_ROWWIN) diff --git a/source/compare_neon.cc b/source/compare_neon.cc index b3a4b12f4..5c7924b7e 100644 --- a/source/compare_neon.cc +++ b/source/compare_neon.cc @@ -93,6 +93,68 @@ uint32_t SumSquareError_NEON(const uint8_t* src_a, return sse; } +#define SSIM_ROW_NEON32(LOAD_A, LOAD_B, PADD_U8, PADD_U16) \ + LOAD_A \ + LOAD_B \ + PADD_U8 " q8, q0 \n" PADD_U8 \ + " q9, q1 \n" \ + "vmull.u8 q2, d0, d0 \n" \ + "vmull.u8 q3, d1, d1 \n" PADD_U16 \ + " q10, q2 \n" PADD_U16 \ + " q11, q3 \n" \ + "vmull.u8 q2, d2, d2 \n" \ + "vmull.u8 q3, d3, d3 \n" \ + "vpadal.u16 q10, q2 \n" \ + "vpadal.u16 q11, q3 \n" \ + "vmull.u8 q2, d0, d2 \n" \ + "vmull.u8 q3, d1, d3 \n" PADD_U16 \ + " q12, q2 \n" PADD_U16 \ + " q13, q3 \n" + +void Ssim4x4Row_NEON(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + asm volatile( + "1: \n" // + SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n", + "vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpaddl.u8", + "vpaddl.u16") // + SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n", + "vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpadal.u8", + "vpadal.u16") // + SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n", + "vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpadal.u8", + "vpadal.u16") // + SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]]\n", + "vld1.8 {d2, d3}, [%[src_b]]\n", "vpadal.u8", + "vpadal.u16") // + "sub %[src_a], %[src_a], %[stride_a], lsl #1 \n" + "sub %[src_a], %[src_a], %[stride_a] \n" + "add %[src_a], %[src_a], #16 \n" + "sub %[src_b], %[src_b], %[stride_b], lsl #1 \n" + "sub %[src_b], %[src_b], %[stride_b] \n" + "add %[src_b], %[src_b], #16 \n" + "vpaddl.u16 q8, q8 \n" + "vpaddl.u16 q9, q9 \n" + "vpadd.u32 d20, d20, d21 \n" + "vpadd.u32 d21, d22, d23 \n" + "vpadd.u32 d22, d24, d25 \n" + "vpadd.u32 d23, d26, d27 \n" + "subs %[width], %[width], #16 \n" + "vst4.32 {d16, d18, d20, d22}, [%[dst]]! \n" + "vst4.32 {d17, d19, d21, d23}, [%[dst]]! \n" + "bgt 1b \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), + [width] "+r"(width) + : [stride_a] "r"(stride_a), [stride_b] "r"(stride_b) + : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12", + "q13"); +} +#undef SSIM_ROW_NEON32 + #endif // defined(__ARM_NEON__) && !defined(__aarch64__) #ifdef __cplusplus diff --git a/source/compare_neon64.cc b/source/compare_neon64.cc index 89dba1ba0..9ffc4328e 100644 --- a/source/compare_neon64.cc +++ b/source/compare_neon64.cc @@ -220,6 +220,135 @@ uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a, return sse; } +#define SSIM_ROW_NEON64(VA, VB, PADD_U8, PADD_U16) \ + PADD_U8 " v4.8h, " #VA ".16b \n" PADD_U8 \ + " v5.8h, " #VB \ + ".16b \n" \ + "umull v6.8h, " #VA ".8b, " #VA \ + ".8b \n" \ + "umull2 v7.8h, " #VA ".16b, " #VA ".16b \n" PADD_U16 \ + " v20.4s, v6.8h \n" PADD_U16 \ + " v21.4s, v7.8h \n" \ + "umull v6.8h, " #VB ".8b, " #VB \ + ".8b \n" \ + "umull2 v7.8h, " #VB ".16b, " #VB \ + ".16b \n" \ + "uadalp v20.4s, v6.8h \n" \ + "uadalp v21.4s, v7.8h \n" \ + "umull v6.8h, " #VA ".8b, " #VB \ + ".8b \n" \ + "umull2 v7.8h, " #VA ".16b, " #VB ".16b \n" PADD_U16 \ + " v22.4s, v6.8h \n" PADD_U16 \ + " v23.4s, v7.8h \n" + +void Ssim4x4Row_NEON(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + asm volatile( + "1: \n" + "ldr q0, [%[a0]], #16 \n" + "ldr q1, [%[b0]], #16 \n" + "ldr q2, [%[a1]], #16 \n" + "ldr q3, [%[b1]], #16 \n" // + SSIM_ROW_NEON64(v0, v1, "uaddlp", "uaddlp") // + SSIM_ROW_NEON64(v2, v3, "uadalp", "uadalp") // + "ldr q0, [%[a2]], #16 \n" + "ldr q1, [%[b2]], #16 \n" + "ldr q2, [%[a3]], #16 \n" + "ldr q3, [%[b3]], #16 \n" // + SSIM_ROW_NEON64(v0, v1, "uadalp", "uadalp") // + SSIM_ROW_NEON64(v2, v3, "uadalp", "uadalp") // + "uaddlp v16.4s, v4.8h \n" + "uaddlp v17.4s, v5.8h \n" + "addp v18.4s, v20.4s, v21.4s \n" + "addp v19.4s, v22.4s, v23.4s \n" + "subs %w[width], %w[width], #16 \n" + "st4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[dst]], #64 \n" + "b.gt 1b \n" + : [a0] "+r"(a0), [a1] "+r"(a1), [a2] "+r"(a2), [a3] "+r"(a3), + [b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3), + [dst] "+r"(dst), [width] "+r"(width) + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} +#undef SSIM_ROW_NEON64 + +void Ssim4x4Row_NEON_DotProd(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + asm volatile( + "movi v6.16b, #1 \n" + "1: \n" + "movi v16.4s, #0 \n" + "movi v17.4s, #0 \n" + "movi v18.4s, #0 \n" + "movi v19.4s, #0 \n" + "ldr q0, [%[a0]], #16 \n" + "ldr q1, [%[b0]], #16 \n" + "ldr q2, [%[a1]], #16 \n" + "ldr q3, [%[b1]], #16 \n" + "udot v16.4s, v0.16b, v6.16b \n" + "udot v17.4s, v1.16b, v6.16b \n" + "udot v18.4s, v0.16b, v0.16b \n" + "udot v18.4s, v1.16b, v1.16b \n" + "udot v19.4s, v0.16b, v1.16b \n" + "udot v16.4s, v2.16b, v6.16b \n" + "udot v17.4s, v3.16b, v6.16b \n" + "udot v18.4s, v2.16b, v2.16b \n" + "udot v18.4s, v3.16b, v3.16b \n" + "udot v19.4s, v2.16b, v3.16b \n" + "ldr q0, [%[a2]], #16 \n" + "ldr q1, [%[b2]], #16 \n" + "ldr q2, [%[a3]], #16 \n" + "ldr q3, [%[b3]], #16 \n" + "udot v16.4s, v0.16b, v6.16b \n" + "udot v17.4s, v1.16b, v6.16b \n" + "udot v18.4s, v0.16b, v0.16b \n" + "udot v18.4s, v1.16b, v1.16b \n" + "udot v19.4s, v0.16b, v1.16b \n" + "udot v16.4s, v2.16b, v6.16b \n" + "udot v17.4s, v3.16b, v6.16b \n" + "udot v18.4s, v2.16b, v2.16b \n" + "udot v18.4s, v3.16b, v3.16b \n" + "udot v19.4s, v2.16b, v3.16b \n" + "subs %w[width], %w[width], #16 \n" + "st4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[dst]], #64 \n" + "b.gt 1b \n" + : [a0] "+r"(a0), [a1] "+r"(a1), [a2] "+r"(a2), [a3] "+r"(a3), + [b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3), + [dst] "+r"(dst), [width] "+r"(width) + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v6", "v16", "v17", "v18", + "v19"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/compare_wasm.cc b/source/compare_wasm.cc index 6400e1ae9..ecb76b09c 100644 --- a/source/compare_wasm.cc +++ b/source/compare_wasm.cc @@ -129,6 +129,118 @@ uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed) { } #endif // HAS_HASHDJB2_WASMSIMD +#if defined(HAS_SSIM4X4ROW_WASMSIMD) +void Ssim4x4Row_WASMSIMD(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const uint8_t* src_a1 = src_a + stride_a; + const uint8_t* src_a2 = src_a1 + stride_a; + const uint8_t* src_a3 = src_a2 + stride_a; + const uint8_t* src_b1 = src_b + stride_b; + const uint8_t* src_b2 = src_b1 + stride_b; + const uint8_t* src_b3 = src_b2 + stride_b; + const v128_t mask = wasm_i16x8_splat(0x00ff); + for (; width > 0; width -= 16) { + v128_t a0 = wasm_v128_load(src_a); + v128_t a1 = wasm_v128_load(src_a1); + v128_t a2 = wasm_v128_load(src_a2); + v128_t a3 = wasm_v128_load(src_a3); + v128_t b0 = wasm_v128_load(src_b); + v128_t b1 = wasm_v128_load(src_b1); + v128_t b2 = wasm_v128_load(src_b2); + v128_t b3 = wasm_v128_load(src_b3); + + v128_t a0_ev = wasm_v128_and(a0, mask); + v128_t a0_od = wasm_u16x8_shr(a0, 8); + v128_t a1_ev = wasm_v128_and(a1, mask); + v128_t a1_od = wasm_u16x8_shr(a1, 8); + v128_t a2_ev = wasm_v128_and(a2, mask); + v128_t a2_od = wasm_u16x8_shr(a2, 8); + v128_t a3_ev = wasm_v128_and(a3, mask); + v128_t a3_od = wasm_u16x8_shr(a3, 8); + + v128_t b0_ev = wasm_v128_and(b0, mask); + v128_t b0_od = wasm_u16x8_shr(b0, 8); + v128_t b1_ev = wasm_v128_and(b1, mask); + v128_t b1_od = wasm_u16x8_shr(b1, 8); + v128_t b2_ev = wasm_v128_and(b2, mask); + v128_t b2_od = wasm_u16x8_shr(b2, 8); + v128_t b3_ev = wasm_v128_and(b3, mask); + v128_t b3_od = wasm_u16x8_shr(b3, 8); + + v128_t sum_a_16 = + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(a0_ev, a0_od), + wasm_i16x8_add(a1_ev, a1_od)), + wasm_i16x8_add(wasm_i16x8_add(a2_ev, a2_od), + wasm_i16x8_add(a3_ev, a3_od))); + v128_t sum_b_16 = + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(b0_ev, b0_od), + wasm_i16x8_add(b1_ev, b1_od)), + wasm_i16x8_add(wasm_i16x8_add(b2_ev, b2_od), + wasm_i16x8_add(b3_ev, b3_od))); + + v128_t sum_a = wasm_u32x4_extadd_pairwise_u16x8(sum_a_16); + v128_t sum_b = wasm_u32x4_extadd_pairwise_u16x8(sum_b_16); + + v128_t sum_axb = wasm_i32x4_add( + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, b0_ev), + wasm_i32x4_dot_i16x8(a0_od, b0_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, b1_ev), + wasm_i32x4_dot_i16x8(a1_od, b1_od))), + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, b2_ev), + wasm_i32x4_dot_i16x8(a2_od, b2_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, b3_ev), + wasm_i32x4_dot_i16x8(a3_od, b3_od)))); + + v128_t sq0 = + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, a0_ev), + wasm_i32x4_dot_i16x8(a0_od, a0_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(b0_ev, b0_ev), + wasm_i32x4_dot_i16x8(b0_od, b0_od))); + v128_t sq1 = + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, a1_ev), + wasm_i32x4_dot_i16x8(a1_od, a1_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(b1_ev, b1_ev), + wasm_i32x4_dot_i16x8(b1_od, b1_od))); + v128_t sq2 = + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, a2_ev), + wasm_i32x4_dot_i16x8(a2_od, a2_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(b2_ev, b2_ev), + wasm_i32x4_dot_i16x8(b2_od, b2_od))); + v128_t sq3 = + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, a3_ev), + wasm_i32x4_dot_i16x8(a3_od, a3_od)), + wasm_i32x4_add(wasm_i32x4_dot_i16x8(b3_ev, b3_ev), + wasm_i32x4_dot_i16x8(b3_od, b3_od))); + v128_t sum_sq_ab = + wasm_i32x4_add(wasm_i32x4_add(sq0, sq1), wasm_i32x4_add(sq2, sq3)); + + v128_t t0 = wasm_i32x4_shuffle(sum_a, sum_b, 0, 4, 1, 5); + v128_t t1 = wasm_i32x4_shuffle(sum_a, sum_b, 2, 6, 3, 7); + v128_t t2 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 0, 4, 1, 5); + v128_t t3 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 2, 6, 3, 7); + + wasm_v128_store(dst, wasm_i64x2_shuffle(t0, t2, 0, 2)); + wasm_v128_store(dst + 4, wasm_i64x2_shuffle(t0, t2, 1, 3)); + wasm_v128_store(dst + 8, wasm_i64x2_shuffle(t1, t3, 0, 2)); + wasm_v128_store(dst + 12, wasm_i64x2_shuffle(t1, t3, 1, 3)); + + src_a += 16; + src_a1 += 16; + src_a2 += 16; + src_a3 += 16; + src_b += 16; + src_b1 += 16; + src_b2 += 16; + src_b3 += 16; + dst += 16; + } +} +#endif // HAS_SSIM4X4ROW_WASMSIMD + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/compare_win.cc b/source/compare_win.cc index 9b6fd0ee5..469fa674d 100644 --- a/source/compare_win.cc +++ b/source/compare_win.cc @@ -422,6 +422,261 @@ uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) { } #endif // HAS_HASHDJB2_AVX512BW +#if defined(HAS_SSIM4X4ROW_SSE2) +// Width is a multiple of 16. +LIBYUV_TARGET_SSE2 +void Ssim4x4Row_SSE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + const __m128i kMaskFF = _mm_set1_epi16(0x00ff); + const __m128i kOnes16 = _mm_set1_epi16(1); + int x; + for (x = 0; x < width; x += 16) { + __m128i acc_a = _mm_setzero_si128(); + __m128i acc_b = _mm_setzero_si128(); + __m128i acc_sq = _mm_setzero_si128(); + __m128i acc_axb = _mm_setzero_si128(); +#define SSIM_ROW_SSE2_WIN(ar, br) \ + do { \ + __m128i va = _mm_loadu_si128((const __m128i*)((ar) + x)); \ + __m128i vb = _mm_loadu_si128((const __m128i*)((br) + x)); \ + __m128i a_ev = _mm_and_si128(va, kMaskFF); \ + __m128i a_od = _mm_srli_epi16(va, 8); \ + __m128i b_ev = _mm_and_si128(vb, kMaskFF); \ + __m128i b_od = _mm_srli_epi16(vb, 8); \ + acc_a = _mm_add_epi16(acc_a, _mm_add_epi16(a_ev, a_od)); \ + acc_b = _mm_add_epi16(acc_b, _mm_add_epi16(b_ev, b_od)); \ + acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(a_ev, a_ev), \ + _mm_madd_epi16(a_od, a_od))); \ + acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(b_ev, b_ev), \ + _mm_madd_epi16(b_od, b_od))); \ + acc_axb = \ + _mm_add_epi32(acc_axb, _mm_add_epi32(_mm_madd_epi16(a_ev, b_ev), \ + _mm_madd_epi16(a_od, b_od))); \ + } while (0) + SSIM_ROW_SSE2_WIN(a0, b0); + SSIM_ROW_SSE2_WIN(a1, b1); + SSIM_ROW_SSE2_WIN(a2, b2); + SSIM_ROW_SSE2_WIN(a3, b3); +#undef SSIM_ROW_SSE2_WIN + acc_a = _mm_madd_epi16(acc_a, kOnes16); + acc_b = _mm_madd_epi16(acc_b, kOnes16); + { + __m128i t0 = _mm_unpacklo_epi32(acc_a, acc_b); + __m128i t1 = _mm_unpackhi_epi32(acc_a, acc_b); + __m128i t2 = _mm_unpacklo_epi32(acc_sq, acc_axb); + __m128i t3 = _mm_unpackhi_epi32(acc_sq, acc_axb); + _mm_storeu_si128((__m128i*)(dst + 0), _mm_unpacklo_epi64(t0, t2)); + _mm_storeu_si128((__m128i*)(dst + 4), _mm_unpackhi_epi64(t0, t2)); + _mm_storeu_si128((__m128i*)(dst + 8), _mm_unpacklo_epi64(t1, t3)); + _mm_storeu_si128((__m128i*)(dst + 12), _mm_unpackhi_epi64(t1, t3)); + } + dst += 16; + } +} +#endif // HAS_SSIM4X4ROW_SSE2 + +#if defined(HAS_SSIM4X4ROW_AVX2) +// Width is a multiple of 32. +LIBYUV_TARGET_AVX2 +void Ssim4x4Row_AVX2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + const __m256i kMaskFF = _mm256_set1_epi16(0x00ff); + const __m256i kOnes16 = _mm256_set1_epi16(1); + int x; + for (x = 0; x < width; x += 32) { + __m256i acc_a = _mm256_setzero_si256(); + __m256i acc_b = _mm256_setzero_si256(); + __m256i acc_sq = _mm256_setzero_si256(); + __m256i acc_axb = _mm256_setzero_si256(); +#define SSIM_ROW_AVX2_WIN(ar, br) \ + do { \ + __m256i va = _mm256_loadu_si256((const __m256i*)((ar) + x)); \ + __m256i vb = _mm256_loadu_si256((const __m256i*)((br) + x)); \ + __m256i a_ev = _mm256_and_si256(va, kMaskFF); \ + __m256i a_od = _mm256_srli_epi16(va, 8); \ + __m256i b_ev = _mm256_and_si256(vb, kMaskFF); \ + __m256i b_od = _mm256_srli_epi16(vb, 8); \ + acc_a = _mm256_add_epi16(acc_a, _mm256_add_epi16(a_ev, a_od)); \ + acc_b = _mm256_add_epi16(acc_b, _mm256_add_epi16(b_ev, b_od)); \ + acc_sq = _mm256_add_epi32( \ + acc_sq, _mm256_add_epi32(_mm256_madd_epi16(a_ev, a_ev), \ + _mm256_madd_epi16(a_od, a_od))); \ + acc_sq = _mm256_add_epi32( \ + acc_sq, _mm256_add_epi32(_mm256_madd_epi16(b_ev, b_ev), \ + _mm256_madd_epi16(b_od, b_od))); \ + acc_axb = _mm256_add_epi32( \ + acc_axb, _mm256_add_epi32(_mm256_madd_epi16(a_ev, b_ev), \ + _mm256_madd_epi16(a_od, b_od))); \ + } while (0) + SSIM_ROW_AVX2_WIN(a0, b0); + SSIM_ROW_AVX2_WIN(a1, b1); + SSIM_ROW_AVX2_WIN(a2, b2); + SSIM_ROW_AVX2_WIN(a3, b3); +#undef SSIM_ROW_AVX2_WIN + acc_a = _mm256_madd_epi16(acc_a, kOnes16); + acc_b = _mm256_madd_epi16(acc_b, kOnes16); + { + __m256i t0 = _mm256_unpacklo_epi32(acc_a, acc_b); + __m256i t1 = _mm256_unpackhi_epi32(acc_a, acc_b); + __m256i t2 = _mm256_unpacklo_epi32(acc_sq, acc_axb); + __m256i t3 = _mm256_unpackhi_epi32(acc_sq, acc_axb); + __m256i u0 = _mm256_unpacklo_epi64(t0, t2); + __m256i u1 = _mm256_unpackhi_epi64(t0, t2); + __m256i u2 = _mm256_unpacklo_epi64(t1, t3); + __m256i u3 = _mm256_unpackhi_epi64(t1, t3); + _mm256_storeu_si256((__m256i*)(dst + 0), + _mm256_permute2x128_si256(u0, u1, 0x20)); + _mm256_storeu_si256((__m256i*)(dst + 8), + _mm256_permute2x128_si256(u2, u3, 0x20)); + _mm256_storeu_si256((__m256i*)(dst + 16), + _mm256_permute2x128_si256(u0, u1, 0x31)); + _mm256_storeu_si256((__m256i*)(dst + 24), + _mm256_permute2x128_si256(u2, u3, 0x31)); + } + dst += 32; + } + _mm256_zeroupper(); +} +#endif // HAS_SSIM4X4ROW_AVX2 + +#if defined(HAS_SSIM4X4ROW_AVX512BW) +// Width is a multiple of 4. +LIBYUV_TARGET_AVX512BW +void Ssim4x4Row_AVX512BW(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const ptrdiff_t sa = stride_a; + const ptrdiff_t sb = stride_b; + const uint8_t* a0 = src_a; + const uint8_t* a1 = a0 + sa; + const uint8_t* a2 = a1 + sa; + const uint8_t* a3 = a2 + sa; + const uint8_t* b0 = src_b; + const uint8_t* b1 = b0 + sb; + const uint8_t* b2 = b1 + sb; + const uint8_t* b3 = b2 + sb; + const __m512i kMaskFF = _mm512_set1_epi16(0x00ff); + const __m512i kOnes16 = _mm512_set1_epi16(1); +#define SSIM_ROW_AVX512_WIN(va, vb) \ + do { \ + __m512i a_ev = _mm512_and_si512(va, kMaskFF); \ + __m512i a_od = _mm512_srli_epi16(va, 8); \ + __m512i b_ev = _mm512_and_si512(vb, kMaskFF); \ + __m512i b_od = _mm512_srli_epi16(vb, 8); \ + acc_a = _mm512_add_epi16(acc_a, _mm512_add_epi16(a_ev, a_od)); \ + acc_b = _mm512_add_epi16(acc_b, _mm512_add_epi16(b_ev, b_od)); \ + acc_sq = _mm512_add_epi32( \ + acc_sq, _mm512_add_epi32(_mm512_madd_epi16(a_ev, a_ev), \ + _mm512_madd_epi16(a_od, a_od))); \ + acc_sq = _mm512_add_epi32( \ + acc_sq, _mm512_add_epi32(_mm512_madd_epi16(b_ev, b_ev), \ + _mm512_madd_epi16(b_od, b_od))); \ + acc_axb = _mm512_add_epi32( \ + acc_axb, _mm512_add_epi32(_mm512_madd_epi16(a_ev, b_ev), \ + _mm512_madd_epi16(a_od, b_od))); \ + } while (0) +#define SSIM_TRANSPOSE_AVX512_WIN() \ + acc_a = _mm512_madd_epi16(acc_a, kOnes16); \ + acc_b = _mm512_madd_epi16(acc_b, kOnes16); \ + __m512i t0 = _mm512_unpacklo_epi32(acc_a, acc_b); \ + __m512i t1 = _mm512_unpackhi_epi32(acc_a, acc_b); \ + __m512i t2 = _mm512_unpacklo_epi32(acc_sq, acc_axb); \ + __m512i t3 = _mm512_unpackhi_epi32(acc_sq, acc_axb); \ + __m512i u0 = _mm512_unpacklo_epi64(t0, t2); \ + __m512i u1 = _mm512_unpackhi_epi64(t0, t2); \ + __m512i u2 = _mm512_unpacklo_epi64(t1, t3); \ + __m512i u3 = _mm512_unpackhi_epi64(t1, t3); \ + __m512i v0 = _mm512_shuffle_i32x4(u0, u1, 0x44); \ + __m512i v1 = _mm512_shuffle_i32x4(u2, u3, 0x44); \ + __m512i v2 = _mm512_shuffle_i32x4(u0, u1, 0xee); \ + __m512i v3 = _mm512_shuffle_i32x4(u2, u3, 0xee); \ + __m512i out0 = _mm512_shuffle_i32x4(v0, v1, 0x88); \ + __m512i out1 = _mm512_shuffle_i32x4(v0, v1, 0xdd); \ + __m512i out2 = _mm512_shuffle_i32x4(v2, v3, 0x88); \ + __m512i out3 = _mm512_shuffle_i32x4(v2, v3, 0xdd) + + while (width >= 64) { + __m512i acc_a = _mm512_setzero_si512(); + __m512i acc_b = _mm512_setzero_si512(); + __m512i acc_sq = _mm512_setzero_si512(); + __m512i acc_axb = _mm512_setzero_si512(); + SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a0), _mm512_loadu_si512(b0)); + SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a1), _mm512_loadu_si512(b1)); + SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a2), _mm512_loadu_si512(b2)); + SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a3), _mm512_loadu_si512(b3)); + SSIM_TRANSPOSE_AVX512_WIN(); + _mm512_storeu_si512(dst + 0, out0); + _mm512_storeu_si512(dst + 16, out1); + _mm512_storeu_si512(dst + 32, out2); + _mm512_storeu_si512(dst + 48, out3); + a0 += 64; + a1 += 64; + a2 += 64; + a3 += 64; + b0 += 64; + b1 += 64; + b2 += 64; + b3 += 64; + dst += 64; + width -= 64; + } + if (width > 0) { + const __mmask16 load_mask = (__mmask16)((1U << (width >> 2)) - 1U); + const uint64_t store_mask = LowBytesMask64(width); + __m512i acc_a = _mm512_setzero_si512(); + __m512i acc_b = _mm512_setzero_si512(); + __m512i acc_sq = _mm512_setzero_si512(); + __m512i acc_axb = _mm512_setzero_si512(); + SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a0), + _mm512_maskz_loadu_epi32(load_mask, b0)); + SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a1), + _mm512_maskz_loadu_epi32(load_mask, b1)); + SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a2), + _mm512_maskz_loadu_epi32(load_mask, b2)); + SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a3), + _mm512_maskz_loadu_epi32(load_mask, b3)); + SSIM_TRANSPOSE_AVX512_WIN(); + _mm512_mask_storeu_epi32(dst + 0, (__mmask16)store_mask, out0); + _mm512_mask_storeu_epi32(dst + 16, (__mmask16)(store_mask >> 16), out1); + _mm512_mask_storeu_epi32(dst + 32, (__mmask16)(store_mask >> 32), out2); + _mm512_mask_storeu_epi32(dst + 48, (__mmask16)(store_mask >> 48), out3); + } +#undef SSIM_ROW_AVX512_WIN +#undef SSIM_TRANSPOSE_AVX512_WIN +} +#endif // HAS_SSIM4X4ROW_AVX512BW + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_lasx.cc b/source/row_lasx.cc index 5a43f638a..be7124f23 100644 --- a/source/row_lasx.cc +++ b/source/row_lasx.cc @@ -10,6 +10,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "libyuv/compare_row.h" #include "libyuv/row.h" // Note: every register written by an inline asm block (vector, general @@ -2045,6 +2046,133 @@ void RGBToYMatrixRow_LASX(const uint8_t* src_rgba, "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr16", "$xr17"); } + +#ifdef HAS_SSIM4X4ROW_LASX +void Ssim4x4Row_LASX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + int x; + int len = width / 32; + const uint8_t* src_a1 = src_a + stride_a; + const uint8_t* src_a2 = src_a1 + stride_a; + const uint8_t* src_a3 = src_a2 + stride_a; + const uint8_t* src_b1 = src_b + stride_b; + const uint8_t* src_b2 = src_b1 + stride_b; + const uint8_t* src_b3 = src_b2 + stride_b; + + for (x = 0; x < len; x++) { + __m256i a0 = __lasx_xvld(src_a, 0); + __m256i a1 = __lasx_xvld(src_a1, 0); + __m256i a2 = __lasx_xvld(src_a2, 0); + __m256i a3 = __lasx_xvld(src_a3, 0); + __m256i b0 = __lasx_xvld(src_b, 0); + __m256i b1 = __lasx_xvld(src_b1, 0); + __m256i b2 = __lasx_xvld(src_b2, 0); + __m256i b3 = __lasx_xvld(src_b3, 0); + + __m256i sum_a_16 = + __lasx_xvadd_h(__lasx_xvadd_h(__lasx_xvhaddw_hu_bu(a0, a0), + __lasx_xvhaddw_hu_bu(a1, a1)), + __lasx_xvadd_h(__lasx_xvhaddw_hu_bu(a2, a2), + __lasx_xvhaddw_hu_bu(a3, a3))); + __m256i sum_b_16 = + __lasx_xvadd_h(__lasx_xvadd_h(__lasx_xvhaddw_hu_bu(b0, b0), + __lasx_xvhaddw_hu_bu(b1, b1)), + __lasx_xvadd_h(__lasx_xvhaddw_hu_bu(b2, b2), + __lasx_xvhaddw_hu_bu(b3, b3))); + __m256i sum_a = __lasx_xvhaddw_wu_hu(sum_a_16, sum_a_16); + __m256i sum_b = __lasx_xvhaddw_wu_hu(sum_b_16, sum_b_16); + + __m256i ab0_ev = __lasx_xvmulwev_h_bu(a0, b0); + __m256i ab0_od = __lasx_xvmulwod_h_bu(a0, b0); + __m256i ab1_ev = __lasx_xvmulwev_h_bu(a1, b1); + __m256i ab1_od = __lasx_xvmulwod_h_bu(a1, b1); + __m256i ab2_ev = __lasx_xvmulwev_h_bu(a2, b2); + __m256i ab2_od = __lasx_xvmulwod_h_bu(a2, b2); + __m256i ab3_ev = __lasx_xvmulwev_h_bu(a3, b3); + __m256i ab3_od = __lasx_xvmulwod_h_bu(a3, b3); + __m256i sum_axb = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab0_ev, ab0_ev), + __lasx_xvhaddw_wu_hu(ab0_od, ab0_od)), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab1_ev, ab1_ev), + __lasx_xvhaddw_wu_hu(ab1_od, ab1_od))), + __lasx_xvadd_w(__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab2_ev, ab2_ev), + __lasx_xvhaddw_wu_hu(ab2_od, ab2_od)), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab3_ev, ab3_ev), + __lasx_xvhaddw_wu_hu(ab3_od, ab3_od)))); + + __m256i sq0 = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a0, a0), + __lasx_xvmulwev_h_bu(a0, a0)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a0, a0), + __lasx_xvmulwod_h_bu(a0, a0))), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b0, b0), + __lasx_xvmulwev_h_bu(b0, b0)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b0, b0), + __lasx_xvmulwod_h_bu(b0, b0)))); + __m256i sq1 = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a1, a1), + __lasx_xvmulwev_h_bu(a1, a1)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a1, a1), + __lasx_xvmulwod_h_bu(a1, a1))), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b1, b1), + __lasx_xvmulwev_h_bu(b1, b1)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b1, b1), + __lasx_xvmulwod_h_bu(b1, b1)))); + __m256i sq2 = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a2, a2), + __lasx_xvmulwev_h_bu(a2, a2)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a2, a2), + __lasx_xvmulwod_h_bu(a2, a2))), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b2, b2), + __lasx_xvmulwev_h_bu(b2, b2)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b2, b2), + __lasx_xvmulwod_h_bu(b2, b2)))); + __m256i sq3 = __lasx_xvadd_w( + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a3, a3), + __lasx_xvmulwev_h_bu(a3, a3)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a3, a3), + __lasx_xvmulwod_h_bu(a3, a3))), + __lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b3, b3), + __lasx_xvmulwev_h_bu(b3, b3)), + __lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b3, b3), + __lasx_xvmulwod_h_bu(b3, b3)))); + __m256i sum_sq_ab = + __lasx_xvadd_w(__lasx_xvadd_w(sq0, sq1), __lasx_xvadd_w(sq2, sq3)); + + __m256i t0 = __lasx_xvilvl_w(sum_b, sum_a); + __m256i t1 = __lasx_xvilvh_w(sum_b, sum_a); + __m256i t2 = __lasx_xvilvl_w(sum_axb, sum_sq_ab); + __m256i t3 = __lasx_xvilvh_w(sum_axb, sum_sq_ab); + __m256i p0 = __lasx_xvilvl_d(t2, t0); + __m256i p1 = __lasx_xvilvh_d(t2, t0); + __m256i p2 = __lasx_xvilvl_d(t3, t1); + __m256i p3 = __lasx_xvilvh_d(t3, t1); + __m256i out0 = __lasx_xvpermi_q(p1, p0, 0x20); + __m256i out1 = __lasx_xvpermi_q(p3, p2, 0x20); + __m256i out2 = __lasx_xvpermi_q(p1, p0, 0x31); + __m256i out3 = __lasx_xvpermi_q(p3, p2, 0x31); + __lasx_xvst(out0, dst, 0); + __lasx_xvst(out1, dst, 32); + __lasx_xvst(out2, dst, 64); + __lasx_xvst(out3, dst, 96); + + src_a += 32; + src_a1 += 32; + src_a2 += 32; + src_a3 += 32; + src_b += 32; + src_b1 += 32; + src_b2 += 32; + src_b3 += 32; + dst += 32; + } +} +#endif // HAS_SSIM4X4ROW_LASX + // undef for unified sources build #undef ALPHA_VAL #undef YUVTORGB_SETUP diff --git a/source/row_lsx.cc b/source/row_lsx.cc index b4fdaaf69..647d949d4 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -10,6 +10,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "libyuv/compare_row.h" #include "libyuv/row.h" // Note: every register written by an inline asm block (vector, general @@ -2715,6 +2716,119 @@ void RGBToYMatrixRow_LSX(const uint8_t* src_rgba, "$vr16", "$vr17"); } +#ifdef HAS_SSIM4X4ROW_LSX +void Ssim4x4Row_LSX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + int x; + int len = width / 16; + const uint8_t* src_a1 = src_a + stride_a; + const uint8_t* src_a2 = src_a1 + stride_a; + const uint8_t* src_a3 = src_a2 + stride_a; + const uint8_t* src_b1 = src_b + stride_b; + const uint8_t* src_b2 = src_b1 + stride_b; + const uint8_t* src_b3 = src_b2 + stride_b; + + for (x = 0; x < len; x++) { + __m128i a0 = __lsx_vld(src_a, 0); + __m128i a1 = __lsx_vld(src_a1, 0); + __m128i a2 = __lsx_vld(src_a2, 0); + __m128i a3 = __lsx_vld(src_a3, 0); + __m128i b0 = __lsx_vld(src_b, 0); + __m128i b1 = __lsx_vld(src_b1, 0); + __m128i b2 = __lsx_vld(src_b2, 0); + __m128i b3 = __lsx_vld(src_b3, 0); + + __m128i sum_a_16 = __lsx_vadd_h( + __lsx_vadd_h(__lsx_vhaddw_hu_bu(a0, a0), __lsx_vhaddw_hu_bu(a1, a1)), + __lsx_vadd_h(__lsx_vhaddw_hu_bu(a2, a2), __lsx_vhaddw_hu_bu(a3, a3))); + __m128i sum_b_16 = __lsx_vadd_h( + __lsx_vadd_h(__lsx_vhaddw_hu_bu(b0, b0), __lsx_vhaddw_hu_bu(b1, b1)), + __lsx_vadd_h(__lsx_vhaddw_hu_bu(b2, b2), __lsx_vhaddw_hu_bu(b3, b3))); + __m128i sum_a = __lsx_vhaddw_wu_hu(sum_a_16, sum_a_16); + __m128i sum_b = __lsx_vhaddw_wu_hu(sum_b_16, sum_b_16); + + __m128i ab0_ev = __lsx_vmulwev_h_bu(a0, b0); + __m128i ab0_od = __lsx_vmulwod_h_bu(a0, b0); + __m128i ab1_ev = __lsx_vmulwev_h_bu(a1, b1); + __m128i ab1_od = __lsx_vmulwod_h_bu(a1, b1); + __m128i ab2_ev = __lsx_vmulwev_h_bu(a2, b2); + __m128i ab2_od = __lsx_vmulwod_h_bu(a2, b2); + __m128i ab3_ev = __lsx_vmulwev_h_bu(a3, b3); + __m128i ab3_od = __lsx_vmulwod_h_bu(a3, b3); + __m128i sum_axb = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vadd_w(__lsx_vhaddw_wu_hu(ab0_ev, ab0_ev), + __lsx_vhaddw_wu_hu(ab0_od, ab0_od)), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(ab1_ev, ab1_ev), + __lsx_vhaddw_wu_hu(ab1_od, ab1_od))), + __lsx_vadd_w(__lsx_vadd_w(__lsx_vhaddw_wu_hu(ab2_ev, ab2_ev), + __lsx_vhaddw_wu_hu(ab2_od, ab2_od)), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(ab3_ev, ab3_ev), + __lsx_vhaddw_wu_hu(ab3_od, ab3_od)))); + + __m128i sq0 = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a0, a0), + __lsx_vmulwev_h_bu(a0, a0)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a0, a0), + __lsx_vmulwod_h_bu(a0, a0))), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b0, b0), + __lsx_vmulwev_h_bu(b0, b0)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b0, b0), + __lsx_vmulwod_h_bu(b0, b0)))); + __m128i sq1 = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a1, a1), + __lsx_vmulwev_h_bu(a1, a1)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a1, a1), + __lsx_vmulwod_h_bu(a1, a1))), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b1, b1), + __lsx_vmulwev_h_bu(b1, b1)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b1, b1), + __lsx_vmulwod_h_bu(b1, b1)))); + __m128i sq2 = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a2, a2), + __lsx_vmulwev_h_bu(a2, a2)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a2, a2), + __lsx_vmulwod_h_bu(a2, a2))), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b2, b2), + __lsx_vmulwev_h_bu(b2, b2)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b2, b2), + __lsx_vmulwod_h_bu(b2, b2)))); + __m128i sq3 = __lsx_vadd_w( + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a3, a3), + __lsx_vmulwev_h_bu(a3, a3)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a3, a3), + __lsx_vmulwod_h_bu(a3, a3))), + __lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b3, b3), + __lsx_vmulwev_h_bu(b3, b3)), + __lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b3, b3), + __lsx_vmulwod_h_bu(b3, b3)))); + __m128i sum_sq_ab = + __lsx_vadd_w(__lsx_vadd_w(sq0, sq1), __lsx_vadd_w(sq2, sq3)); + + __m128i out0, out1, out2, out3; + LSX_TRANSPOSE4x4_W(sum_a, sum_b, sum_sq_ab, sum_axb, out0, out1, out2, + out3); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + __lsx_vst(out2, dst, 32); + __lsx_vst(out3, dst, 48); + + src_a += 16; + src_a1 += 16; + src_a2 += 16; + src_a3 += 16; + src_b += 16; + src_b1 += 16; + src_b2 += 16; + src_b3 += 16; + dst += 16; + } +} +#endif // HAS_SSIM4X4ROW_LSX + // undef for unified sources build #undef YUVTORGB_SETUP #undef READYUV422_D diff --git a/source/row_rvv.cc b/source/row_rvv.cc index ca87e4603..d64729845 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -15,6 +15,7 @@ * Contributed by Bruce Lai */ +#include "libyuv/compare_row.h" #include "libyuv/row.h" // Note: every register written by an inline asm block (vector, general @@ -2669,6 +2670,117 @@ void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) { } #endif // HAS_MIRRORUVROW_RVV +#ifdef HAS_SSIM4X4ROW_RVV +#define SSIM4X4_LOAD_RVV(ptr_a, ptr_b) \ + "vlseg4e8.v v0, (" #ptr_a \ + ") \n" \ + "vlseg4e8.v v4, (" #ptr_b \ + ") \n" \ + "vwaddu.vv v10, v0, v1 \n" \ + "vwaddu.vv v11, v2, v3 \n" \ + "vwaddu.vv v12, v4, v5 \n" \ + "vwaddu.vv v13, v6, v7 \n" \ + "vwmulu.vv v14, v0, v4 \n" \ + "vwmulu.vv v15, v1, v5 \n" \ + "vwmulu.vv v24, v2, v6 \n" \ + "vwmulu.vv v25, v3, v7 \n" \ + "vwmulu.vv v26, v0, v0 \n" \ + "vwmulu.vv v27, v1, v1 \n" \ + "vwmulu.vv v28, v2, v2 \n" \ + "vwmulu.vv v29, v3, v3 \n" \ + "vwmulu.vv v0, v4, v4 \n" \ + "vwmulu.vv v1, v5, v5 \n" \ + "vwmulu.vv v2, v6, v6 \n" \ + "vwmulu.vv v3, v7, v7 \n" + +#define SSIM4X4_ACCUM_RVV(ptr_a, ptr_b) \ + "vsetvli zero, zero, e8, mf2, ta, ma \n" \ + SSIM4X4_LOAD_RVV(ptr_a, ptr_b) \ + "vsetvli zero, zero, e16, m1, ta, ma \n" \ + "vadd.vv v10, v10, v11 \n" \ + "vadd.vv v12, v12, v13 \n" \ + "vadd.vv v8, v8, v10 \n" \ + "vadd.vv v9, v9, v12 \n" \ + "vwaddu.wv v22, v22, v14 \n" \ + "vwaddu.wv v22, v22, v15 \n" \ + "vwaddu.wv v22, v22, v24 \n" \ + "vwaddu.wv v22, v22, v25 \n" \ + "vwaddu.wv v20, v20, v26 \n" \ + "vwaddu.wv v20, v20, v27 \n" \ + "vwaddu.wv v20, v20, v28 \n" \ + "vwaddu.wv v20, v20, v29 \n" \ + "vwaddu.wv v20, v20, v0 \n" \ + "vwaddu.wv v20, v20, v1 \n" \ + "vwaddu.wv v20, v20, v2 \n" \ + "vwaddu.wv v20, v20, v3 \n" + +void Ssim4x4Row_RVV(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + size_t vl; + size_t byte_vl; + const uint8_t* tmp_a; + const uint8_t* tmp_b; + width >>= 2; + asm volatile( + "1: \n" + "vsetvli %[vl], %[w], e8, mf2, ta, ma \n" + "add %[tmp_a], %[src_a], %[stride_a]\n" + "add %[tmp_b], %[src_b], %[stride_b]\n" + SSIM4X4_LOAD_RVV(%[src_a], %[src_b]) + "vsetvli zero, zero, e16, m1, ta, ma \n" + "vadd.vv v8, v10, v11 \n" + "vadd.vv v9, v12, v13 \n" + "vwaddu.vv v22, v14, v15 \n" + "vwaddu.wv v22, v22, v24 \n" + "vwaddu.wv v22, v22, v25 \n" + "vwaddu.vv v20, v26, v27 \n" + "vwaddu.wv v20, v20, v28 \n" + "vwaddu.wv v20, v20, v29 \n" + "vwaddu.wv v20, v20, v0 \n" + "vwaddu.wv v20, v20, v1 \n" + "vwaddu.wv v20, v20, v2 \n" + "vwaddu.wv v20, v20, v3 \n" + SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b]) + "add %[tmp_a], %[tmp_a], %[stride_a]\n" + "add %[tmp_b], %[tmp_b], %[stride_b]\n" + SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b]) + "add %[tmp_a], %[tmp_a], %[stride_a]\n" + "add %[tmp_b], %[tmp_b], %[stride_b]\n" + SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b]) + "vwcvtu.x.x.v v16, v8 \n" + "vwcvtu.x.x.v v18, v9 \n" + "vsetvli zero, zero, e32, m2, ta, ma \n" + "vsseg4e32.v v16, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[byte_vl], %[vl], 2 \n" + "add %[src_a], %[src_a], %[byte_vl]\n" + "add %[src_b], %[src_b], %[byte_vl]\n" + "slli %[byte_vl], %[vl], 4 \n" + "add %[dst], %[dst], %[byte_vl] \n" + "bgtz %[w], 1b \n" + : [src_a] "+r"(src_a), // %[src_a] + [src_b] "+r"(src_b), // %[src_b] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [byte_vl] "=&r"(byte_vl), // %[byte_vl] + [tmp_a] "=&r"(tmp_a), // %[tmp_a] + [tmp_b] "=&r"(tmp_b) // %[tmp_b] + : [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a] + [stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b] + : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", + "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", + "v27", "v28", "v29"); +} +#undef SSIM4X4_LOAD_RVV +#undef SSIM4X4_ACCUM_RVV +#endif // HAS_SSIM4X4ROW_RVV + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_sme.cc b/source/row_sme.cc index dac14ac34..4d9beacb6 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "libyuv/compare_row.h" #include "libyuv/row.h" #include "libyuv/row_sve.h" @@ -1122,6 +1123,17 @@ __arm_locally_streaming void ARGBToYMatrixRow_SME( ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c); } +#ifdef HAS_SSIM4X4ROW_SME +__arm_locally_streaming void Ssim4x4Row_SME(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width); +} +#endif // HAS_SSIM4X4ROW_SME + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/row_sve.cc b/source/row_sve.cc index efa1ac705..28ddc94fb 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -9,6 +9,7 @@ */ #include "libyuv/row_sve.h" +#include "libyuv/compare_row.h" #include "libyuv/row.h" #ifdef __cplusplus @@ -1724,6 +1725,17 @@ void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv, } #endif // HAS_HALFWIDTHROW_16TO8_SVE2 +#ifdef HAS_SSIM4X4ROW_SVE2 +void Ssim4x4Row_SVE2(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width); +} +#endif // HAS_SSIM4X4ROW_SVE2 + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/unit_test/compare_test.cc b/unit_test/compare_test.cc index 2456db281..c6d19fb37 100644 --- a/unit_test/compare_test.cc +++ b/unit_test/compare_test.cc @@ -654,7 +654,7 @@ TEST_F(LibYUVCompareTest, Psnr) { free_aligned_buffer_page_end(src_b); } -TEST_F(LibYUVCompareTest, DISABLED_BenchmarkSsim_Opt) { +TEST_F(LibYUVCompareTest, BenchmarkSsim_Opt) { align_buffer_page_end(src_a, benchmark_width_ * benchmark_height_); align_buffer_page_end(src_b, benchmark_width_ * benchmark_height_); for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) { @@ -679,6 +679,57 @@ TEST_F(LibYUVCompareTest, DISABLED_BenchmarkSsim_Opt) { free_aligned_buffer_page_end(src_b); } +TEST_F(LibYUVCompareTest, BenchmarkSsim_Unaligned) { + align_buffer_page_end(src_a, benchmark_width_ * benchmark_height_ + 1); + align_buffer_page_end(src_b, benchmark_width_ * benchmark_height_); + for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) { + src_a[i + 1] = i; + src_b[i] = i; + } + + MaskCpuFlags(benchmark_cpu_info_); + + double opt_time = get_time(); + for (int i = 0; i < benchmark_iterations_; ++i) { + CalcFrameSsim(src_a + 1, benchmark_width_, src_b, benchmark_width_, + benchmark_width_, benchmark_height_); + } + + opt_time = (get_time() - opt_time) / benchmark_iterations_; + printf("BenchmarkSsim_Opt - %8.2f us opt\n", opt_time * 1e6); + + ASSERT_EQ(0, 0); + + free_aligned_buffer_page_end(src_a); + free_aligned_buffer_page_end(src_b); +} + +TEST_F(LibYUVCompareTest, TestSsimAnyWidth) { + const int kMaxWidth = 140; + const int kHeight = 16; + const int kPlaneSize = kMaxWidth * kHeight + 1; + align_buffer_page_end(src_a, kPlaneSize); + align_buffer_page_end(src_b, kPlaneSize); + MemRandomize(src_a, kPlaneSize); + MemRandomize(src_b, kPlaneSize); + + for (int width = 9; width <= kMaxWidth; ++width) { + for (int offset = 0; offset < 2; ++offset) { + const int size = width * kHeight; + const uint8_t* a = src_a + kPlaneSize - size - offset; + const uint8_t* b = src_b + kPlaneSize - size - offset; + MaskCpuFlags(disable_cpu_flags_); + double c_ssim = CalcFrameSsim(a, width, b, width, width, kHeight); + MaskCpuFlags(benchmark_cpu_info_); + double opt_ssim = CalcFrameSsim(a, width, b, width, width, kHeight); + EXPECT_EQ(c_ssim, opt_ssim) << "width " << width << " offset " << offset; + } + } + + free_aligned_buffer_page_end(src_a); + free_aligned_buffer_page_end(src_b); +} + TEST_F(LibYUVCompareTest, Ssim) { const int kSrcWidth = benchmark_width_; const int kSrcHeight = benchmark_height_;