coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

[libyuv] Optimize CalcFrameSsim with 2-row 4x4 tile buffer and SIMD

Browse Source 
Refactor CalcFrameSsim to decompose overlapping 8x8 SSIM windows
(stepped by 4 pixels horizontally and vertically) into non-overlapping
4x4 pixel tiles across a 2-row rotating buffer (row0, row1), reducing
memory reads and pixel accumulation work by 4x:
- Ssim4x4Row_* computes 4 uint32_t statistics {sum_a, sum_b, sum_sq_ab,
  sum_axb} per 4x4 pixel tile across a 4-row strip (also sized to fit
  future 10-bit/12-bit formats like I010 without uint32_t overflow).
- SsimWindowRow_C combines top and bottom 4x4 tile strips across
  adjacent horizontal pairs and returns the exact double SSIM sum,
  producing bit-identical results to Ssim8x8_C.
- CalcFrameSsim rotates row0 and row1 (with thread-local buffers and
  continuity tracking under _OPENMP) and falls back to Ssim8x8_C only if
  buffer allocation fails.
- Implement Ssim4x4Row across all SIMD architectures:
  - x86/x64: Ssim4x4Row_SSE2, Ssim4x4Row_AVX2, and Ssim4x4Row_AVX512BW
    (with k-masked tail handling for 4..60 pixels) in both
    compare_gcc.cc and compare_win.cc.
  - ARM: Ssim4x4Row_NEON (compare_neon.cc, compare_neon64.cc),
    Ssim4x4Row_NEON_DotProd (compare_neon64.cc), Ssim4x4Row_SVE2
    (row_sve.h, row_sve.cc), and Ssim4x4Row_SME (row_sme.cc).
  - RISC-V: Ssim4x4Row_RVV (row_rvv.cc).
  - LoongArch: Ssim4x4Row_LSX (row_lsx.cc) and Ssim4x4Row_LASX
    (row_lasx.cc).
  - WebAssembly: Ssim4x4Row_WASMSIMD (compare_wasm.cc).
- Enable BenchmarkSsim_Opt and add BenchmarkSsim_Unaligned and
  TestSsimAnyWidth in unit_test/compare_test.cc.

R=rrwinterton@gmail.com

Bug: None
Test: out/Release/libyuv_unittest --gtest_filter=*Ssim*
Change-Id: Ibb95ae74424ecdda073ab7ab10ab1cf610b121b6
This commit is contained in:
Frank Barchard 2026-10-15 15:33:21 -07:00
parent 0b9e5c922d
commit dcbc5f9f2b
15 changed files with 1800 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
include/libyuv/compare_row.h
View File

@ -32,12 +32,15 @@ extern "C" {
#define HAS_HAMMINGDISTANCE_SSE42
#define HAS_SUMSQUAREERROR_SSE2
#define HAS_HASHDJB2_SSE41
#define HAS_SSIM4X4ROW_SSE2
#define HAS_HAMMINGDISTANCE_AVX2
#define HAS_SUMSQUAREERROR_AVX2
#define HAS_HASHDJB2_AVX2
#define HAS_SSIM4X4ROW_AVX2
#define HAS_HAMMINGDISTANCE_AVX512BW
#define HAS_SUMSQUAREERROR_AVX512BW
#define HAS_HASHDJB2_AVX512BW
#define HAS_SSIM4X4ROW_AVX512BW
#endif
// The following are available for GCC and clang 32/64 bit, as inline assembly
@ -49,15 +52,18 @@ extern "C" {
#define HAS_HAMMINGDISTANCE_SSSE3
#define HAS_SUMSQUAREERROR_SSE2
#define HAS_HASHDJB2_SSE41
#define HAS_SSIM4X4ROW_SSE2
#if defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)
#define HAS_HAMMINGDISTANCE_AVX2
#define HAS_SUMSQUAREERROR_AVX2
#define HAS_HASHDJB2_AVX2
#define HAS_SSIM4X4ROW_AVX2
#endif
#if defined(CLANG_HAS_AVX512)
#define HAS_HAMMINGDISTANCE_AVX512BW
#define HAS_SUMSQUAREERROR_AVX512BW
#define HAS_HASHDJB2_AVX512BW
#define HAS_SSIM4X4ROW_AVX512BW
#endif
#endif
@ -66,6 +72,7 @@ extern "C" {
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
#define HAS_HAMMINGDISTANCE_NEON
#define HAS_SUMSQUAREERROR_NEON
#define HAS_SSIM4X4ROW_NEON
#endif
// The following are available for AArch64 Neon:
@ -74,6 +81,34 @@ extern "C" {
#define HAS_HAMMINGDISTANCE_NEON_DOTPROD
#define HAS_SUMSQUAREERROR_NEON_DOTPROD
#define HAS_SSIM4X4ROW_NEON_DOTPROD
#endif
// The following are available for AArch64 SVE:
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#define HAS_SSIM4X4ROW_SVE2
#endif
// The following are available for AArch64 SME:
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
defined(__aarch64__)
#define HAS_SSIM4X4ROW_SME
#endif
// The following are available for RISC-V Vector (RVV):
#if !defined(LIBYUV_DISABLE_RVV) && \
(defined(__riscv_vector) || defined(LIBYUV_RVV))
#define HAS_SSIM4X4ROW_RVV
#endif
// The following are available for LoongArch LSX:
#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
#define HAS_SSIM4X4ROW_LSX
#endif
// The following are available for LoongArch LASX:
#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
#define HAS_SSIM4X4ROW_LASX
#endif
// The following are available for WebAssembly SIMD128:
@ -81,6 +116,7 @@ extern "C" {
#define HAS_HAMMINGDISTANCE_WASMSIMD
#define HAS_SUMSQUAREERROR_WASMSIMD
#define HAS_HASHDJB2_WASMSIMD
#define HAS_SSIM4X4ROW_WASMSIMD
#endif
uint32_t HammingDistance_C(const uint8_t* src_a,
@ -136,6 +172,82 @@ uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed);
uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed);
uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed);
void Ssim4x4Row_C(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_SSE2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_AVX2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_NEON(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_NEON_DotProd(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_SVE2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_SME(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_RVV(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_LSX(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_LASX(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
void Ssim4x4Row_WASMSIMD(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width);
double SsimWindowRow_C(const uint32_t* row0,
const uint32_t* row1,
int num_windows);
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

87
include/libyuv/row_sve.h
View File

@ -2224,6 +2224,93 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb,
"z20", "z24", "z25", "p0", "p1", "p2", "p3", "p4", "p5");
}
#define SSIM4X4_SVE \
"add %[tmp_a], %[src_a], %[stride_a] \n" \
"add %[tmp_b], %[src_b], %[stride_b] \n" \
"ld1w {z0.s}, p0/z, [%[src_a]] \n" \
"ld1w {z1.s}, p0/z, [%[tmp_a]] \n" \
"add %[tmp_a], %[tmp_a], %[stride_a] \n" \
"ld1w {z4.s}, p0/z, [%[src_b]] \n" \
"ld1w {z5.s}, p0/z, [%[tmp_b]] \n" \
"add %[tmp_b], %[tmp_b], %[stride_b] \n" \
"ld1w {z2.s}, p0/z, [%[tmp_a]] \n" \
"add %[tmp_a], %[tmp_a], %[stride_a] \n" \
"ld1w {z3.s}, p0/z, [%[tmp_a]] \n" \
"ld1w {z6.s}, p0/z, [%[tmp_b]] \n" \
"add %[tmp_b], %[tmp_b], %[stride_b] \n" \
"ld1w {z7.s}, p0/z, [%[tmp_b]] \n" \
"incb %[src_a] \n" \
"incb %[src_b] \n" \
"mov z16.s, #0 \n" \
"mov z17.s, #0 \n" \
"mov z18.s, #0 \n" \
"mov z19.s, #0 \n" \
"udot z16.s, z0.b, z20.b \n" \
"udot z17.s, z4.b, z20.b \n" \
"udot z18.s, z0.b, z0.b \n" \
"udot z19.s, z0.b, z4.b \n" \
"udot z16.s, z1.b, z20.b \n" \
"udot z17.s, z5.b, z20.b \n" \
"udot z18.s, z1.b, z1.b \n" \
"udot z19.s, z1.b, z5.b \n" \
"udot z16.s, z2.b, z20.b \n" \
"udot z17.s, z6.b, z20.b \n" \
"udot z18.s, z2.b, z2.b \n" \
"udot z19.s, z2.b, z6.b \n" \
"udot z16.s, z3.b, z20.b \n" \
"udot z17.s, z7.b, z20.b \n" \
"udot z18.s, z3.b, z3.b \n" \
"udot z19.s, z3.b, z7.b \n" \
"udot z18.s, z4.b, z4.b \n" \
"udot z18.s, z5.b, z5.b \n" \
"udot z18.s, z6.b, z6.b \n" \
"udot z18.s, z7.b, z7.b \n" \
"subs %w[width], %w[width], %w[vl] \n" \
"st4w {z16.s, z17.s, z18.s, z19.s}, p0, [%[dst]]\n" \
"incb %[dst], all, mul #4 \n"
static inline void Ssim4x4Row_SVE_SC(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
const uint8_t* tmp_a;
const uint8_t* tmp_b;
width >>= 2;
asm volatile(
"cntw %x[vl] \n"
"mov z20.b, #1 \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
"ptrue p0.s \n"
"1: \n" //
SSIM4X4_SVE
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
"whilelt p0.s, wzr, %w[width] \n" //
SSIM4X4_SVE
"99: \n"
: [src_a] "+r"(src_a), // %[src_a]
[src_b] "+r"(src_b), // %[src_b]
[dst] "+r"(dst), // %[dst]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl), // %[vl]
[tmp_a] "=&r"(tmp_a), // %[tmp_a]
[tmp_b] "=&r"(tmp_b) // %[tmp_b]
: [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a]
[stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b]
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16",
"z17", "z18", "z19", "z20", "p0");
}
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#ifdef __cplusplus

147
source/compare.cc
View File

@ -427,20 +427,143 @@ double CalcFrameSsim(const uint8_t* src_a,
int height) {
int samples = 0;
double ssim_total = 0;
double (*Ssim8x8)(const uint8_t* src_a, int stride_a, const uint8_t* src_b,
int stride_b) = Ssim8x8_C;
if (width <= 8 || height <= 8) {
return ssim_total / samples;
}
// sample point start with each 4x4 location
int i;
for (i = 0; i < height - 8; i += 4) {
int j;
for (j = 0; j < width - 8; j += 4) {
ssim_total += Ssim8x8(src_a + j, stride_a, src_b + j, stride_b);
samples++;
int simd_size = 4;
void (*Ssim4x4Row)(const uint8_t* src_a, int stride_a, const uint8_t* src_b,
int stride_b, uint32_t* dst, int width) = Ssim4x4Row_C;
#if defined(HAS_SSIM4X4ROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
Ssim4x4Row = Ssim4x4Row_SSE2;
simd_size = 16;
}
#endif
#if defined(HAS_SSIM4X4ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
Ssim4x4Row = Ssim4x4Row_AVX2;
simd_size = 32;
}
#endif
#if defined(HAS_SSIM4X4ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
Ssim4x4Row = Ssim4x4Row_AVX512BW;
simd_size = 4;
}
#endif
#if defined(HAS_SSIM4X4ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
Ssim4x4Row = Ssim4x4Row_NEON;
simd_size = 16;
}
#endif
#if defined(HAS_SSIM4X4ROW_NEON_DOTPROD)
if (TestCpuFlag(kCpuHasNeonDotProd)) {
Ssim4x4Row = Ssim4x4Row_NEON_DotProd;
simd_size = 16;
}
#endif
#if defined(HAS_SSIM4X4ROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
Ssim4x4Row = Ssim4x4Row_SVE2;
simd_size = 4;
}
#endif
#if defined(HAS_SSIM4X4ROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
Ssim4x4Row = Ssim4x4Row_SME;
simd_size = 4;
}
#endif
#if defined(HAS_SSIM4X4ROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
Ssim4x4Row = Ssim4x4Row_RVV;
simd_size = 4;
}
#endif
#if defined(HAS_SSIM4X4ROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
Ssim4x4Row = Ssim4x4Row_LSX;
simd_size = 16;
}
#endif
#if defined(HAS_SSIM4X4ROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
Ssim4x4Row = Ssim4x4Row_LASX;
simd_size = 32;
}
#endif
#if defined(HAS_SSIM4X4ROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
Ssim4x4Row = Ssim4x4Row_WASMSIMD;
simd_size = 16;
}
#endif
const int num_windows = (width - 8 + 3) >> 2;
const int num_tiles = num_windows + 1;
const int tile_width = num_tiles * 4;
const int main_width = tile_width & ~(simd_size - 1);
const int rem_width = tile_width & (simd_size - 1);
#ifdef _OPENMP
#pragma omp parallel reduction(+ : ssim_total, samples)
#endif
{
align_buffer_64(row_buf, (size_t)tile_width * 2 * sizeof(uint32_t));
if (row_buf) {
uint32_t* row0 = (uint32_t*)row_buf;
uint32_t* row1 = row0 + tile_width;
int prev_i = -8;
int i;
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for (i = 0; i < height - 8; i += 4) {
const uint8_t* a = src_a + (ptrdiff_t)i * stride_a;
const uint8_t* b = src_b + (ptrdiff_t)i * stride_b;
if (prev_i + 4 != i) {
if (main_width > 0) {
Ssim4x4Row(a, stride_a, b, stride_b, row0, main_width);
}
if (rem_width > 0) {
Ssim4x4Row_C(a + main_width, stride_a, b + main_width, stride_b,
row0 + main_width, rem_width);
}
}
a += (ptrdiff_t)stride_a * 4;
b += (ptrdiff_t)stride_b * 4;
if (main_width > 0) {
Ssim4x4Row(a, stride_a, b, stride_b, row1, main_width);
}
if (rem_width > 0) {
Ssim4x4Row_C(a + main_width, stride_a, b + main_width, stride_b,
row1 + main_width, rem_width);
}
ssim_total += SsimWindowRow_C(row0, row1, num_windows);
samples += num_windows;
uint32_t* tmp = row0;
row0 = row1;
row1 = tmp;
prev_i = i;
}
free_aligned_buffer_64(row_buf);
} else {
int i;
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for (i = 0; i < height - 8; i += 4) {
const uint8_t* a = src_a + (ptrdiff_t)i * stride_a;
const uint8_t* b = src_b + (ptrdiff_t)i * stride_b;
int j;
for (j = 0; j < width - 8; j += 4) {
ssim_total += Ssim8x8_C(a + j, stride_a, b + j, stride_b);
samples++;
}
}
}
src_a += stride_a * 4;
src_b += stride_b * 4;
}
ssim_total /= samples;

104
source/compare_common.cc
View File

@ -10,6 +10,8 @@
#include "libyuv/compare_row.h"
#include <float.h>
#ifdef __cplusplus
namespace libyuv {
extern "C" {
@ -66,6 +68,108 @@ uint32_t HashDjb2_C(const uint8_t* src, int count, uint32_t seed) {
return hash;
}
// Compute 4x4 tile statistics (sum_a, sum_b, sum_sq_a + sum_sq_b, sum_axb)
// across 4 rows for width pixels (width is a multiple of 4).
void Ssim4x4Row_C(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const ptrdiff_t sa = stride_a;
const ptrdiff_t sb = stride_b;
const uint8_t* a0 = src_a;
const uint8_t* a1 = a0 + sa;
const uint8_t* a2 = a1 + sa;
const uint8_t* a3 = a2 + sa;
const uint8_t* b0 = src_b;
const uint8_t* b1 = b0 + sb;
const uint8_t* b2 = b1 + sb;
const uint8_t* b3 = b2 + sb;
int x;
for (x = 0; x < width; x += 4) {
uint32_t sum_a = a0[0] + a0[1] + a0[2] + a0[3] + a1[0] + a1[1] + a1[2] +
a1[3] + a2[0] + a2[1] + a2[2] + a2[3] + a3[0] + a3[1] +
a3[2] + a3[3];
uint32_t sum_b = b0[0] + b0[1] + b0[2] + b0[3] + b1[0] + b1[1] + b1[2] +
b1[3] + b2[0] + b2[1] + b2[2] + b2[3] + b3[0] + b3[1] +
b3[2] + b3[3];
uint32_t sum_sq_ab =
a0[0] * a0[0] + a0[1] * a0[1] + a0[2] * a0[2] + a0[3] * a0[3] +
a1[0] * a1[0] + a1[1] * a1[1] + a1[2] * a1[2] + a1[3] * a1[3] +
a2[0] * a2[0] + a2[1] * a2[1] + a2[2] * a2[2] + a2[3] * a2[3] +
a3[0] * a3[0] + a3[1] * a3[1] + a3[2] * a3[2] + a3[3] * a3[3] +
b0[0] * b0[0] + b0[1] * b0[1] + b0[2] * b0[2] + b0[3] * b0[3] +
b1[0] * b1[0] + b1[1] * b1[1] + b1[2] * b1[2] + b1[3] * b1[3] +
b2[0] * b2[0] + b2[1] * b2[1] + b2[2] * b2[2] + b2[3] * b2[3] +
b3[0] * b3[0] + b3[1] * b3[1] + b3[2] * b3[2] + b3[3] * b3[3];
uint32_t sum_axb =
a0[0] * b0[0] + a0[1] * b0[1] + a0[2] * b0[2] + a0[3] * b0[3] +
a1[0] * b1[0] + a1[1] * b1[1] + a1[2] * b1[2] + a1[3] * b1[3] +
a2[0] * b2[0] + a2[1] * b2[1] + a2[2] * b2[2] + a2[3] * b2[3] +
a3[0] * b3[0] + a3[1] * b3[1] + a3[2] * b3[2] + a3[3] * b3[3];
dst[0] = sum_a;
dst[1] = sum_b;
dst[2] = sum_sq_ab;
dst[3] = sum_axb;
a0 += 4;
a1 += 4;
a2 += 4;
a3 += 4;
b0 += 4;
b1 += 4;
b2 += 4;
b3 += 4;
dst += 4;
}
}
// Combine two consecutive 4x4 tile rows (row0 and row1) into 8x8 windows
// stepped on a 4-pixel horizontal grid and return the sum of SSIM values.
double SsimWindowRow_C(const uint32_t* row0,
const uint32_t* row1,
int num_windows) {
const int64_t count = 64;
// scale the constants by number of pixels
const int64_t cc1 = 26634; // (64^2*(.01*255)^2)
const int64_t cc2 = 239708; // (64^2*(.03*255)^2)
const int64_t c1 = (cc1 * count * count) >> 12;
const int64_t c2 = (cc2 * count * count) >> 12;
double ssim_total = 0.0;
uint32_t prev_a = row0[0] + row1[0];
uint32_t prev_b = row0[1] + row1[1];
uint32_t prev_sq = row0[2] + row1[2];
uint32_t prev_axb = row0[3] + row1[3];
int w;
for (w = 0; w < num_windows; ++w) {
row0 += 4;
row1 += 4;
const uint32_t next_a = row0[0] + row1[0];
const uint32_t next_b = row0[1] + row1[1];
const uint32_t next_sq = row0[2] + row1[2];
const uint32_t next_axb = row0[3] + row1[3];
const int64_t sum_a = prev_a + next_a;
const int64_t sum_b = prev_b + next_b;
const int64_t sum_sq_ab = prev_sq + next_sq;
const int64_t sum_axb = prev_axb + next_axb;
prev_a = next_a;
prev_b = next_b;
prev_sq = next_sq;
prev_axb = next_axb;
const int64_t sum_a_x_sum_b = sum_a * sum_b;
const int64_t ssim_n = (2 * sum_a_x_sum_b + c1) *
(2 * count * sum_axb - 2 * sum_a_x_sum_b + c2);
const int64_t sum_a_sq = sum_a * sum_a;
const int64_t sum_b_sq = sum_b * sum_b;
const int64_t ssim_d = (sum_a_sq + sum_b_sq + c1) *
(count * sum_sq_ab - sum_a_sq - sum_b_sq + c2);
ssim_total += ssim_n * 1.0 / ssim_d;
}
return ssim_total;
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

376
source/compare_gcc.cc
View File

@ -407,8 +407,9 @@ uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) {
"xmm7");
return hash;
}
#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW)
#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW) || \
defined(HAS_SSIM4X4ROW_AVX512BW)
// Mask of the low n bytes, for n = 0 to 63.
static uint64_t LowBytesMask64(int n) {
return n ? ~0ULL >> (64 - n) : 0ULL;
@ -749,6 +750,377 @@ uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) {
#undef HASH64_AVX512BW
#endif // HAS_HASHDJB2_AVX512BW
#ifdef HAS_SSIM4X4ROW_SSE2
#define SSIM_ROW0_SSE2(LOAD_A, LOAD_B) \
LOAD_A \
LOAD_B \
"movdqa %%xmm0,%%xmm2 \n" \
"psrlw $0x8,%%xmm0 \n" \
"psllw $0x8,%%xmm2 \n" \
"psrlw $0x8,%%xmm2 \n" \
"movdqa %%xmm0,%%xmm4 \n" \
"paddw %%xmm2,%%xmm4 \n" \
"movdqa %%xmm0,%%xmm6 \n" \
"pmaddwd %%xmm6,%%xmm6 \n" \
"movdqa %%xmm2,%%xmm3 \n" \
"pmaddwd %%xmm3,%%xmm3 \n" \
"paddd %%xmm3,%%xmm6 \n" \
"movdqa %%xmm1,%%xmm3 \n" \
"psrlw $0x8,%%xmm1 \n" \
"psllw $0x8,%%xmm3 \n" \
"psrlw $0x8,%%xmm3 \n" \
"movdqa %%xmm1,%%xmm5 \n" \
"paddw %%xmm3,%%xmm5 \n" \
"pmaddwd %%xmm1,%%xmm0 \n" \
"pmaddwd %%xmm3,%%xmm2 \n" \
"movdqa %%xmm0,%%xmm7 \n" \
"paddd %%xmm2,%%xmm7 \n" \
"pmaddwd %%xmm1,%%xmm1 \n" \
"pmaddwd %%xmm3,%%xmm3 \n" \
"paddd %%xmm1,%%xmm6 \n" \
"paddd %%xmm3,%%xmm6 \n"
#define SSIM_ROW_SSE2(LOAD_A, LOAD_B) \
LOAD_A \
LOAD_B \
"movdqa %%xmm0,%%xmm2 \n" \
"psrlw $0x8,%%xmm0 \n" \
"psllw $0x8,%%xmm2 \n" \
"psrlw $0x8,%%xmm2 \n" \
"paddw %%xmm0,%%xmm4 \n" \
"paddw %%xmm2,%%xmm4 \n" \
"movdqa %%xmm0,%%xmm3 \n" \
"pmaddwd %%xmm3,%%xmm3 \n" \
"paddd %%xmm3,%%xmm6 \n" \
"movdqa %%xmm2,%%xmm3 \n" \
"pmaddwd %%xmm3,%%xmm3 \n" \
"paddd %%xmm3,%%xmm6 \n" \
"movdqa %%xmm1,%%xmm3 \n" \
"psrlw $0x8,%%xmm1 \n" \
"psllw $0x8,%%xmm3 \n" \
"psrlw $0x8,%%xmm3 \n" \
"paddw %%xmm1,%%xmm5 \n" \
"paddw %%xmm3,%%xmm5 \n" \
"pmaddwd %%xmm1,%%xmm0 \n" \
"pmaddwd %%xmm3,%%xmm2 \n" \
"paddd %%xmm0,%%xmm7 \n" \
"paddd %%xmm2,%%xmm7 \n" \
"pmaddwd %%xmm1,%%xmm1 \n" \
"pmaddwd %%xmm3,%%xmm3 \n" \
"paddd %%xmm1,%%xmm6 \n" \
"paddd %%xmm3,%%xmm6 \n"
void Ssim4x4Row_SSE2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
asm volatile(
LABELALIGN "1: \n" //
SSIM_ROW0_SSE2("movdqu (%[src_a]),%%xmm0\n",
"movdqu (%[src_b]),%%xmm1\n") //
"add %[stride_a],%[src_a] \n"
"add %[stride_b],%[src_b] \n" //
SSIM_ROW_SSE2("movdqu (%[src_a]),%%xmm0\n",
"movdqu (%[src_b]),%%xmm1\n") //
SSIM_ROW_SSE2("movdqu (%[src_a],%[stride_a]),%%xmm0\n",
"movdqu (%[src_b],%[stride_b]),%%xmm1\n") //
SSIM_ROW_SSE2("movdqu (%[src_a],%[stride_a],2),%%xmm0\n",
"movdqu (%[src_b],%[stride_b],2),%%xmm1\n") //
"sub %[stride_a],%[src_a] \n"
"sub %[stride_b],%[src_b] \n"
"add $0x10,%[src_a] \n"
"add $0x10,%[src_b] \n"
"pcmpeqd %%xmm0,%%xmm0 \n"
"psrlw $0xf,%%xmm0 \n"
"pmaddwd %%xmm0,%%xmm4 \n"
"pmaddwd %%xmm0,%%xmm5 \n"
"movdqa %%xmm4,%%xmm0 \n"
"punpckldq %%xmm5,%%xmm0 \n"
"punpckhdq %%xmm5,%%xmm4 \n"
"movdqa %%xmm6,%%xmm1 \n"
"punpckldq %%xmm7,%%xmm1 \n"
"punpckhdq %%xmm7,%%xmm6 \n"
"movdqa %%xmm0,%%xmm2 \n"
"punpcklqdq %%xmm1,%%xmm2 \n"
"punpckhqdq %%xmm1,%%xmm0 \n"
"movdqa %%xmm4,%%xmm3 \n"
"punpcklqdq %%xmm6,%%xmm3 \n"
"punpckhqdq %%xmm6,%%xmm4 \n"
"movdqu %%xmm2,0x00(%[dst]) \n"
"movdqu %%xmm0,0x10(%[dst]) \n"
"movdqu %%xmm3,0x20(%[dst]) \n"
"movdqu %%xmm4,0x30(%[dst]) \n"
"add $0x40,%[dst] \n"
"sub $0x10,%[width] \n"
"jg 1b \n"
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
[width] "+r"(width)
: [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b)
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
#undef SSIM_ROW0_SSE2
#undef SSIM_ROW_SSE2
#endif // HAS_SSIM4X4ROW_SSE2
#ifdef HAS_SSIM4X4ROW_AVX2
#define SSIM_ROW0_AVX2(LOAD_A, LOAD_B) \
LOAD_A \
LOAD_B \
"vpsrlw $0x8,%%ymm0,%%ymm2 \n" \
"vpsllw $0x8,%%ymm0,%%ymm0 \n" \
"vpsrlw $0x8,%%ymm0,%%ymm0 \n" \
"vpaddw %%ymm0,%%ymm2,%%ymm4 \n" \
"vpmaddwd %%ymm2,%%ymm2,%%ymm6 \n" \
"vpmaddwd %%ymm0,%%ymm0,%%ymm3 \n" \
"vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
"vpsrlw $0x8,%%ymm1,%%ymm3 \n" \
"vpsllw $0x8,%%ymm1,%%ymm1 \n" \
"vpsrlw $0x8,%%ymm1,%%ymm1 \n" \
"vpaddw %%ymm1,%%ymm3,%%ymm5 \n" \
"vpmaddwd %%ymm3,%%ymm2,%%ymm7 \n" \
"vpmaddwd %%ymm1,%%ymm0,%%ymm0 \n" \
"vpaddd %%ymm0,%%ymm7,%%ymm7 \n" \
"vpmaddwd %%ymm3,%%ymm3,%%ymm3 \n" \
"vpmaddwd %%ymm1,%%ymm1,%%ymm1 \n" \
"vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
"vpaddd %%ymm1,%%ymm6,%%ymm6 \n"
#define SSIM_ROW_AVX2(LOAD_A, LOAD_B) \
LOAD_A \
LOAD_B \
"vpsrlw $0x8,%%ymm0,%%ymm2 \n" \
"vpsllw $0x8,%%ymm0,%%ymm0 \n" \
"vpsrlw $0x8,%%ymm0,%%ymm0 \n" \
"vpaddw %%ymm2,%%ymm4,%%ymm4 \n" \
"vpaddw %%ymm0,%%ymm4,%%ymm4 \n" \
"vpmaddwd %%ymm2,%%ymm2,%%ymm3 \n" \
"vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
"vpmaddwd %%ymm0,%%ymm0,%%ymm3 \n" \
"vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
"vpsrlw $0x8,%%ymm1,%%ymm3 \n" \
"vpsllw $0x8,%%ymm1,%%ymm1 \n" \
"vpsrlw $0x8,%%ymm1,%%ymm1 \n" \
"vpaddw %%ymm3,%%ymm5,%%ymm5 \n" \
"vpaddw %%ymm1,%%ymm5,%%ymm5 \n" \
"vpmaddwd %%ymm3,%%ymm2,%%ymm2 \n" \
"vpmaddwd %%ymm1,%%ymm0,%%ymm0 \n" \
"vpaddd %%ymm2,%%ymm7,%%ymm7 \n" \
"vpaddd %%ymm0,%%ymm7,%%ymm7 \n" \
"vpmaddwd %%ymm3,%%ymm3,%%ymm3 \n" \
"vpmaddwd %%ymm1,%%ymm1,%%ymm1 \n" \
"vpaddd %%ymm3,%%ymm6,%%ymm6 \n" \
"vpaddd %%ymm1,%%ymm6,%%ymm6 \n"
void Ssim4x4Row_AVX2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
asm volatile(
LABELALIGN "1: \n" //
SSIM_ROW0_AVX2("vmovdqu (%[src_a]),%%ymm0\n",
"vmovdqu (%[src_b]),%%ymm1\n") //
"add %[stride_a],%[src_a] \n"
"add %[stride_b],%[src_b] \n" //
SSIM_ROW_AVX2("vmovdqu (%[src_a]),%%ymm0\n",
"vmovdqu (%[src_b]),%%ymm1\n") //
SSIM_ROW_AVX2("vmovdqu (%[src_a],%[stride_a]),%%ymm0\n",
"vmovdqu (%[src_b],%[stride_b]),%%ymm1\n") //
SSIM_ROW_AVX2("vmovdqu (%[src_a],%[stride_a],2),%%ymm0\n",
"vmovdqu (%[src_b],%[stride_b],2),%%ymm1\n") //
"sub %[stride_a],%[src_a] \n"
"sub %[stride_b],%[src_b] \n"
"add $0x20,%[src_a] \n"
"add $0x20,%[src_b] \n"
"vpcmpeqd %%ymm0,%%ymm0,%%ymm0 \n"
"vpsrlw $0xf,%%ymm0,%%ymm0 \n"
"vpmaddwd %%ymm0,%%ymm4,%%ymm4 \n"
"vpmaddwd %%ymm0,%%ymm5,%%ymm5 \n"
"vpunpckldq %%ymm5,%%ymm4,%%ymm0 \n"
"vpunpckhdq %%ymm5,%%ymm4,%%ymm1 \n"
"vpunpckldq %%ymm7,%%ymm6,%%ymm2 \n"
"vpunpckhdq %%ymm7,%%ymm6,%%ymm3 \n"
"vpunpcklqdq %%ymm2,%%ymm0,%%ymm4 \n"
"vpunpckhqdq %%ymm2,%%ymm0,%%ymm5 \n"
"vpunpcklqdq %%ymm3,%%ymm1,%%ymm6 \n"
"vpunpckhqdq %%ymm3,%%ymm1,%%ymm7 \n"
"vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm0 \n"
"vperm2i128 $0x20,%%ymm7,%%ymm6,%%ymm1 \n"
"vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm2 \n"
"vperm2i128 $0x31,%%ymm7,%%ymm6,%%ymm3 \n"
"vmovdqu %%ymm0,0x00(%[dst]) \n"
"vmovdqu %%ymm1,0x20(%[dst]) \n"
"vmovdqu %%ymm2,0x40(%[dst]) \n"
"vmovdqu %%ymm3,0x60(%[dst]) \n"
"add $0x80,%[dst] \n"
"sub $0x20,%[width] \n"
"jg 1b \n"
"vzeroupper \n"
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
[width] "+r"(width)
: [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b)
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
#undef SSIM_ROW0_AVX2
#undef SSIM_ROW_AVX2
#endif // HAS_SSIM4X4ROW_AVX2
#ifdef HAS_SSIM4X4ROW_AVX512BW
#define SSIM_ROW0_AVX512BW(LOAD_A, LOAD_B) \
LOAD_A \
LOAD_B \
"vpsrlw $0x8,%%zmm0,%%zmm2 \n" \
"vpsllw $0x8,%%zmm0,%%zmm0 \n" \
"vpsrlw $0x8,%%zmm0,%%zmm0 \n" \
"vpaddw %%zmm0,%%zmm2,%%zmm4 \n" \
"vpmaddwd %%zmm2,%%zmm2,%%zmm6 \n" \
"vpmaddwd %%zmm0,%%zmm0,%%zmm3 \n" \
"vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
"vpsrlw $0x8,%%zmm1,%%zmm3 \n" \
"vpsllw $0x8,%%zmm1,%%zmm1 \n" \
"vpsrlw $0x8,%%zmm1,%%zmm1 \n" \
"vpaddw %%zmm1,%%zmm3,%%zmm5 \n" \
"vpmaddwd %%zmm3,%%zmm2,%%zmm7 \n" \
"vpmaddwd %%zmm1,%%zmm0,%%zmm0 \n" \
"vpaddd %%zmm0,%%zmm7,%%zmm7 \n" \
"vpmaddwd %%zmm3,%%zmm3,%%zmm3 \n" \
"vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n" \
"vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
"vpaddd %%zmm1,%%zmm6,%%zmm6 \n"
#define SSIM_ROW_AVX512BW(LOAD_A, LOAD_B) \
LOAD_A \
LOAD_B \
"vpsrlw $0x8,%%zmm0,%%zmm2 \n" \
"vpsllw $0x8,%%zmm0,%%zmm0 \n" \
"vpsrlw $0x8,%%zmm0,%%zmm0 \n" \
"vpaddw %%zmm2,%%zmm4,%%zmm4 \n" \
"vpaddw %%zmm0,%%zmm4,%%zmm4 \n" \
"vpmaddwd %%zmm2,%%zmm2,%%zmm3 \n" \
"vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
"vpmaddwd %%zmm0,%%zmm0,%%zmm3 \n" \
"vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
"vpsrlw $0x8,%%zmm1,%%zmm3 \n" \
"vpsllw $0x8,%%zmm1,%%zmm1 \n" \
"vpsrlw $0x8,%%zmm1,%%zmm1 \n" \
"vpaddw %%zmm3,%%zmm5,%%zmm5 \n" \
"vpaddw %%zmm1,%%zmm5,%%zmm5 \n" \
"vpmaddwd %%zmm3,%%zmm2,%%zmm2 \n" \
"vpmaddwd %%zmm1,%%zmm0,%%zmm0 \n" \
"vpaddd %%zmm2,%%zmm7,%%zmm7 \n" \
"vpaddd %%zmm0,%%zmm7,%%zmm7 \n" \
"vpmaddwd %%zmm3,%%zmm3,%%zmm3 \n" \
"vpmaddwd %%zmm1,%%zmm1,%%zmm1 \n" \
"vpaddd %%zmm3,%%zmm6,%%zmm6 \n" \
"vpaddd %%zmm1,%%zmm6,%%zmm6 \n"
#define SSIM_TRANSPOSE_AVX512BW \
"vpternlogd $0xff,%%zmm0,%%zmm0,%%zmm0 \n" \
"vpsrlw $0xf,%%zmm0,%%zmm0 \n" \
"vpmaddwd %%zmm0,%%zmm4,%%zmm4 \n" \
"vpmaddwd %%zmm0,%%zmm5,%%zmm5 \n" \
"vpunpckldq %%zmm5,%%zmm4,%%zmm0 \n" \
"vpunpckhdq %%zmm5,%%zmm4,%%zmm1 \n" \
"vpunpckldq %%zmm7,%%zmm6,%%zmm2 \n" \
"vpunpckhdq %%zmm7,%%zmm6,%%zmm3 \n" \
"vpunpcklqdq %%zmm2,%%zmm0,%%zmm4 \n" \
"vpunpckhqdq %%zmm2,%%zmm0,%%zmm5 \n" \
"vpunpcklqdq %%zmm3,%%zmm1,%%zmm6 \n" \
"vpunpckhqdq %%zmm3,%%zmm1,%%zmm7 \n" \
"vshufi32x4 $0x44,%%zmm5,%%zmm4,%%zmm0 \n" \
"vshufi32x4 $0x44,%%zmm7,%%zmm6,%%zmm1 \n" \
"vshufi32x4 $0xee,%%zmm5,%%zmm4,%%zmm2 \n" \
"vshufi32x4 $0xee,%%zmm7,%%zmm6,%%zmm3 \n" \
"vshufi32x4 $0x88,%%zmm1,%%zmm0,%%zmm4 \n" \
"vshufi32x4 $0xdd,%%zmm1,%%zmm0,%%zmm5 \n" \
"vshufi32x4 $0x88,%%zmm3,%%zmm2,%%zmm6 \n" \
"vshufi32x4 $0xdd,%%zmm3,%%zmm2,%%zmm7 \n"
// Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60
// pixels (1 to 15 tiles) with masked loads and stores.
void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const int rem = width & 60;
const uint16_t load_mask = rem ? (uint16_t)((1U << (rem >> 2)) - 1U) : 0U;
const uint64_t store_mask = LowBytesMask64(rem);
asm volatile(
"sub $0x40,%[width] \n"
"jl 2f \n"
LABELALIGN "1: \n" //
SSIM_ROW0_AVX512BW("vmovdqu8 (%[src_a]),%%zmm0\n",
"vmovdqu8 (%[src_b]),%%zmm1\n") //
"add %[stride_a],%[src_a] \n"
"add %[stride_b],%[src_b] \n" //
SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a]),%%zmm0\n",
"vmovdqu8 (%[src_b]),%%zmm1\n") //
SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a],%[stride_a]),%%zmm0\n",
"vmovdqu8 (%[src_b],%[stride_b]),%%zmm1\n") //
SSIM_ROW_AVX512BW("vmovdqu8 (%[src_a],%[stride_a],2),%%zmm0\n",
"vmovdqu8 (%[src_b],%[stride_b],2),%%zmm1\n") //
"sub %[stride_a],%[src_a] \n"
"sub %[stride_b],%[src_b] \n"
"add $0x40,%[src_a] \n"
"add $0x40,%[src_b] \n" //
SSIM_TRANSPOSE_AVX512BW
"vmovdqu32 %%zmm4,0x00(%[dst]) \n"
"vmovdqu32 %%zmm5,0x40(%[dst]) \n"
"vmovdqu32 %%zmm6,0x80(%[dst]) \n"
"vmovdqu32 %%zmm7,0xc0(%[dst]) \n"
"add $0x100,%[dst] \n"
"sub $0x40,%[width] \n"
"jge 1b \n"
"2: \n"
"add $0x40,%[width] \n"
"je 3f \n"
"kmovw %[load_mask],%%k1 \n" //
SSIM_ROW0_AVX512BW("vmovdqu32 (%[src_a]),%%zmm0%{%%k1%}%{z%}\n",
"vmovdqu32 (%[src_b]),%%zmm1%{%%k1%}%{z%}\n") //
"add %[stride_a],%[src_a] \n"
"add %[stride_b],%[src_b] \n" //
SSIM_ROW_AVX512BW("vmovdqu32 (%[src_a]),%%zmm0%{%%k1%}%{z%}\n",
"vmovdqu32 (%[src_b]),%%zmm1%{%%k1%}%{z%}\n") //
SSIM_ROW_AVX512BW(
"vmovdqu32 (%[src_a],%[stride_a]),%%zmm0%{%%k1%}%{z%}\n",
"vmovdqu32 (%[src_b],%[stride_b]),%%zmm1%{%%k1%}%{z%}\n") //
SSIM_ROW_AVX512BW(
"vmovdqu32 (%[src_a],%[stride_a],2),%%zmm0%{%%k1%}%{z%}\n",
"vmovdqu32 (%[src_b],%[stride_b],2),%%zmm1%{%%k1%}%{z%}\n") //
SSIM_TRANSPOSE_AVX512BW
"kmovq %[store_mask],%%k1 \n"
"vmovdqu32 %%zmm4,0x00(%[dst])%{%%k1%} \n"
"kshiftrq $0x10,%%k1,%%k1 \n"
"vmovdqu32 %%zmm5,0x40(%[dst])%{%%k1%} \n"
"kshiftrq $0x10,%%k1,%%k1 \n"
"vmovdqu32 %%zmm6,0x80(%[dst])%{%%k1%} \n"
"kshiftrq $0x10,%%k1,%%k1 \n"
"vmovdqu32 %%zmm7,0xc0(%[dst])%{%%k1%} \n"
"3: \n"
"vzeroupper \n"
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
[width] "+r"(width)
: [stride_a] "r"((ptrdiff_t)stride_a),
[stride_b] "r"((ptrdiff_t)stride_b), [load_mask] "m"(load_mask),
[store_mask] "m"(store_mask)
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7");
}
#undef SSIM_ROW0_AVX512BW
#undef SSIM_ROW_AVX512BW
#undef SSIM_TRANSPOSE_AVX512BW
#endif // HAS_SSIM4X4ROW_AVX512BW
#endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) &&
// !defined(LIBYUV_ENABLE_ROWWIN)

62
source/compare_neon.cc
View File

@ -93,6 +93,68 @@ uint32_t SumSquareError_NEON(const uint8_t* src_a,
return sse;
}
#define SSIM_ROW_NEON32(LOAD_A, LOAD_B, PADD_U8, PADD_U16) \
LOAD_A \
LOAD_B \
PADD_U8 " q8, q0 \n" PADD_U8 \
" q9, q1 \n" \
"vmull.u8 q2, d0, d0 \n" \
"vmull.u8 q3, d1, d1 \n" PADD_U16 \
" q10, q2 \n" PADD_U16 \
" q11, q3 \n" \
"vmull.u8 q2, d2, d2 \n" \
"vmull.u8 q3, d3, d3 \n" \
"vpadal.u16 q10, q2 \n" \
"vpadal.u16 q11, q3 \n" \
"vmull.u8 q2, d0, d2 \n" \
"vmull.u8 q3, d1, d3 \n" PADD_U16 \
" q12, q2 \n" PADD_U16 \
" q13, q3 \n"
void Ssim4x4Row_NEON(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
asm volatile(
"1: \n" //
SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n",
"vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpaddl.u8",
"vpaddl.u16") //
SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n",
"vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpadal.u8",
"vpadal.u16") //
SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]], %[stride_a]\n",
"vld1.8 {d2, d3}, [%[src_b]], %[stride_b]\n", "vpadal.u8",
"vpadal.u16") //
SSIM_ROW_NEON32("vld1.8 {d0, d1}, [%[src_a]]\n",
"vld1.8 {d2, d3}, [%[src_b]]\n", "vpadal.u8",
"vpadal.u16") //
"sub %[src_a], %[src_a], %[stride_a], lsl #1 \n"
"sub %[src_a], %[src_a], %[stride_a] \n"
"add %[src_a], %[src_a], #16 \n"
"sub %[src_b], %[src_b], %[stride_b], lsl #1 \n"
"sub %[src_b], %[src_b], %[stride_b] \n"
"add %[src_b], %[src_b], #16 \n"
"vpaddl.u16 q8, q8 \n"
"vpaddl.u16 q9, q9 \n"
"vpadd.u32 d20, d20, d21 \n"
"vpadd.u32 d21, d22, d23 \n"
"vpadd.u32 d22, d24, d25 \n"
"vpadd.u32 d23, d26, d27 \n"
"subs %[width], %[width], #16 \n"
"vst4.32 {d16, d18, d20, d22}, [%[dst]]! \n"
"vst4.32 {d17, d19, d21, d23}, [%[dst]]! \n"
"bgt 1b \n"
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
[width] "+r"(width)
: [stride_a] "r"(stride_a), [stride_b] "r"(stride_b)
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12",
"q13");
}
#undef SSIM_ROW_NEON32
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus

129
source/compare_neon64.cc
View File

@ -220,6 +220,135 @@ uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a,
return sse;
}
#define SSIM_ROW_NEON64(VA, VB, PADD_U8, PADD_U16) \
PADD_U8 " v4.8h, " #VA ".16b \n" PADD_U8 \
" v5.8h, " #VB \
".16b \n" \
"umull v6.8h, " #VA ".8b, " #VA \
".8b \n" \
"umull2 v7.8h, " #VA ".16b, " #VA ".16b \n" PADD_U16 \
" v20.4s, v6.8h \n" PADD_U16 \
" v21.4s, v7.8h \n" \
"umull v6.8h, " #VB ".8b, " #VB \
".8b \n" \
"umull2 v7.8h, " #VB ".16b, " #VB \
".16b \n" \
"uadalp v20.4s, v6.8h \n" \
"uadalp v21.4s, v7.8h \n" \
"umull v6.8h, " #VA ".8b, " #VB \
".8b \n" \
"umull2 v7.8h, " #VA ".16b, " #VB ".16b \n" PADD_U16 \
" v22.4s, v6.8h \n" PADD_U16 \
" v23.4s, v7.8h \n"
void Ssim4x4Row_NEON(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const ptrdiff_t sa = stride_a;
const ptrdiff_t sb = stride_b;
const uint8_t* a0 = src_a;
const uint8_t* a1 = a0 + sa;
const uint8_t* a2 = a1 + sa;
const uint8_t* a3 = a2 + sa;
const uint8_t* b0 = src_b;
const uint8_t* b1 = b0 + sb;
const uint8_t* b2 = b1 + sb;
const uint8_t* b3 = b2 + sb;
asm volatile(
"1: \n"
"ldr q0, [%[a0]], #16 \n"
"ldr q1, [%[b0]], #16 \n"
"ldr q2, [%[a1]], #16 \n"
"ldr q3, [%[b1]], #16 \n" //
SSIM_ROW_NEON64(v0, v1, "uaddlp", "uaddlp") //
SSIM_ROW_NEON64(v2, v3, "uadalp", "uadalp") //
"ldr q0, [%[a2]], #16 \n"
"ldr q1, [%[b2]], #16 \n"
"ldr q2, [%[a3]], #16 \n"
"ldr q3, [%[b3]], #16 \n" //
SSIM_ROW_NEON64(v0, v1, "uadalp", "uadalp") //
SSIM_ROW_NEON64(v2, v3, "uadalp", "uadalp") //
"uaddlp v16.4s, v4.8h \n"
"uaddlp v17.4s, v5.8h \n"
"addp v18.4s, v20.4s, v21.4s \n"
"addp v19.4s, v22.4s, v23.4s \n"
"subs %w[width], %w[width], #16 \n"
"st4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[dst]], #64 \n"
"b.gt 1b \n"
: [a0] "+r"(a0), [a1] "+r"(a1), [a2] "+r"(a2), [a3] "+r"(a3),
[b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3),
[dst] "+r"(dst), [width] "+r"(width)
:
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
#undef SSIM_ROW_NEON64
void Ssim4x4Row_NEON_DotProd(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const ptrdiff_t sa = stride_a;
const ptrdiff_t sb = stride_b;
const uint8_t* a0 = src_a;
const uint8_t* a1 = a0 + sa;
const uint8_t* a2 = a1 + sa;
const uint8_t* a3 = a2 + sa;
const uint8_t* b0 = src_b;
const uint8_t* b1 = b0 + sb;
const uint8_t* b2 = b1 + sb;
const uint8_t* b3 = b2 + sb;
asm volatile(
"movi v6.16b, #1 \n"
"1: \n"
"movi v16.4s, #0 \n"
"movi v17.4s, #0 \n"
"movi v18.4s, #0 \n"
"movi v19.4s, #0 \n"
"ldr q0, [%[a0]], #16 \n"
"ldr q1, [%[b0]], #16 \n"
"ldr q2, [%[a1]], #16 \n"
"ldr q3, [%[b1]], #16 \n"
"udot v16.4s, v0.16b, v6.16b \n"
"udot v17.4s, v1.16b, v6.16b \n"
"udot v18.4s, v0.16b, v0.16b \n"
"udot v18.4s, v1.16b, v1.16b \n"
"udot v19.4s, v0.16b, v1.16b \n"
"udot v16.4s, v2.16b, v6.16b \n"
"udot v17.4s, v3.16b, v6.16b \n"
"udot v18.4s, v2.16b, v2.16b \n"
"udot v18.4s, v3.16b, v3.16b \n"
"udot v19.4s, v2.16b, v3.16b \n"
"ldr q0, [%[a2]], #16 \n"
"ldr q1, [%[b2]], #16 \n"
"ldr q2, [%[a3]], #16 \n"
"ldr q3, [%[b3]], #16 \n"
"udot v16.4s, v0.16b, v6.16b \n"
"udot v17.4s, v1.16b, v6.16b \n"
"udot v18.4s, v0.16b, v0.16b \n"
"udot v18.4s, v1.16b, v1.16b \n"
"udot v19.4s, v0.16b, v1.16b \n"
"udot v16.4s, v2.16b, v6.16b \n"
"udot v17.4s, v3.16b, v6.16b \n"
"udot v18.4s, v2.16b, v2.16b \n"
"udot v18.4s, v3.16b, v3.16b \n"
"udot v19.4s, v2.16b, v3.16b \n"
"subs %w[width], %w[width], #16 \n"
"st4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[dst]], #64 \n"
"b.gt 1b \n"
: [a0] "+r"(a0), [a1] "+r"(a1), [a2] "+r"(a2), [a3] "+r"(a3),
[b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3),
[dst] "+r"(dst), [width] "+r"(width)
:
: "memory", "cc", "v0", "v1", "v2", "v3", "v6", "v16", "v17", "v18",
"v19");
}
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus

112
source/compare_wasm.cc
View File

@ -129,6 +129,118 @@ uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed) {
}
#endif // HAS_HASHDJB2_WASMSIMD
#if defined(HAS_SSIM4X4ROW_WASMSIMD)
void Ssim4x4Row_WASMSIMD(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const uint8_t* src_a1 = src_a + stride_a;
const uint8_t* src_a2 = src_a1 + stride_a;
const uint8_t* src_a3 = src_a2 + stride_a;
const uint8_t* src_b1 = src_b + stride_b;
const uint8_t* src_b2 = src_b1 + stride_b;
const uint8_t* src_b3 = src_b2 + stride_b;
const v128_t mask = wasm_i16x8_splat(0x00ff);
for (; width > 0; width -= 16) {
v128_t a0 = wasm_v128_load(src_a);
v128_t a1 = wasm_v128_load(src_a1);
v128_t a2 = wasm_v128_load(src_a2);
v128_t a3 = wasm_v128_load(src_a3);
v128_t b0 = wasm_v128_load(src_b);
v128_t b1 = wasm_v128_load(src_b1);
v128_t b2 = wasm_v128_load(src_b2);
v128_t b3 = wasm_v128_load(src_b3);
v128_t a0_ev = wasm_v128_and(a0, mask);
v128_t a0_od = wasm_u16x8_shr(a0, 8);
v128_t a1_ev = wasm_v128_and(a1, mask);
v128_t a1_od = wasm_u16x8_shr(a1, 8);
v128_t a2_ev = wasm_v128_and(a2, mask);
v128_t a2_od = wasm_u16x8_shr(a2, 8);
v128_t a3_ev = wasm_v128_and(a3, mask);
v128_t a3_od = wasm_u16x8_shr(a3, 8);
v128_t b0_ev = wasm_v128_and(b0, mask);
v128_t b0_od = wasm_u16x8_shr(b0, 8);
v128_t b1_ev = wasm_v128_and(b1, mask);
v128_t b1_od = wasm_u16x8_shr(b1, 8);
v128_t b2_ev = wasm_v128_and(b2, mask);
v128_t b2_od = wasm_u16x8_shr(b2, 8);
v128_t b3_ev = wasm_v128_and(b3, mask);
v128_t b3_od = wasm_u16x8_shr(b3, 8);
v128_t sum_a_16 =
wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(a0_ev, a0_od),
wasm_i16x8_add(a1_ev, a1_od)),
wasm_i16x8_add(wasm_i16x8_add(a2_ev, a2_od),
wasm_i16x8_add(a3_ev, a3_od)));
v128_t sum_b_16 =
wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(b0_ev, b0_od),
wasm_i16x8_add(b1_ev, b1_od)),
wasm_i16x8_add(wasm_i16x8_add(b2_ev, b2_od),
wasm_i16x8_add(b3_ev, b3_od)));
v128_t sum_a = wasm_u32x4_extadd_pairwise_u16x8(sum_a_16);
v128_t sum_b = wasm_u32x4_extadd_pairwise_u16x8(sum_b_16);
v128_t sum_axb = wasm_i32x4_add(
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, b0_ev),
wasm_i32x4_dot_i16x8(a0_od, b0_od)),
wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, b1_ev),
wasm_i32x4_dot_i16x8(a1_od, b1_od))),
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, b2_ev),
wasm_i32x4_dot_i16x8(a2_od, b2_od)),
wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, b3_ev),
wasm_i32x4_dot_i16x8(a3_od, b3_od))));
v128_t sq0 =
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, a0_ev),
wasm_i32x4_dot_i16x8(a0_od, a0_od)),
wasm_i32x4_add(wasm_i32x4_dot_i16x8(b0_ev, b0_ev),
wasm_i32x4_dot_i16x8(b0_od, b0_od)));
v128_t sq1 =
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, a1_ev),
wasm_i32x4_dot_i16x8(a1_od, a1_od)),
wasm_i32x4_add(wasm_i32x4_dot_i16x8(b1_ev, b1_ev),
wasm_i32x4_dot_i16x8(b1_od, b1_od)));
v128_t sq2 =
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, a2_ev),
wasm_i32x4_dot_i16x8(a2_od, a2_od)),
wasm_i32x4_add(wasm_i32x4_dot_i16x8(b2_ev, b2_ev),
wasm_i32x4_dot_i16x8(b2_od, b2_od)));
v128_t sq3 =
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, a3_ev),
wasm_i32x4_dot_i16x8(a3_od, a3_od)),
wasm_i32x4_add(wasm_i32x4_dot_i16x8(b3_ev, b3_ev),
wasm_i32x4_dot_i16x8(b3_od, b3_od)));
v128_t sum_sq_ab =
wasm_i32x4_add(wasm_i32x4_add(sq0, sq1), wasm_i32x4_add(sq2, sq3));
v128_t t0 = wasm_i32x4_shuffle(sum_a, sum_b, 0, 4, 1, 5);
v128_t t1 = wasm_i32x4_shuffle(sum_a, sum_b, 2, 6, 3, 7);
v128_t t2 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 0, 4, 1, 5);
v128_t t3 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 2, 6, 3, 7);
wasm_v128_store(dst, wasm_i64x2_shuffle(t0, t2, 0, 2));
wasm_v128_store(dst + 4, wasm_i64x2_shuffle(t0, t2, 1, 3));
wasm_v128_store(dst + 8, wasm_i64x2_shuffle(t1, t3, 0, 2));
wasm_v128_store(dst + 12, wasm_i64x2_shuffle(t1, t3, 1, 3));
src_a += 16;
src_a1 += 16;
src_a2 += 16;
src_a3 += 16;
src_b += 16;
src_b1 += 16;
src_b2 += 16;
src_b3 += 16;
dst += 16;
}
}
#endif // HAS_SSIM4X4ROW_WASMSIMD
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

255
source/compare_win.cc
View File

@ -422,6 +422,261 @@ uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) {
}
#endif // HAS_HASHDJB2_AVX512BW
#if defined(HAS_SSIM4X4ROW_SSE2)
// Width is a multiple of 16.
LIBYUV_TARGET_SSE2
void Ssim4x4Row_SSE2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const ptrdiff_t sa = stride_a;
const ptrdiff_t sb = stride_b;
const uint8_t* a0 = src_a;
const uint8_t* a1 = a0 + sa;
const uint8_t* a2 = a1 + sa;
const uint8_t* a3 = a2 + sa;
const uint8_t* b0 = src_b;
const uint8_t* b1 = b0 + sb;
const uint8_t* b2 = b1 + sb;
const uint8_t* b3 = b2 + sb;
const __m128i kMaskFF = _mm_set1_epi16(0x00ff);
const __m128i kOnes16 = _mm_set1_epi16(1);
int x;
for (x = 0; x < width; x += 16) {
__m128i acc_a = _mm_setzero_si128();
__m128i acc_b = _mm_setzero_si128();
__m128i acc_sq = _mm_setzero_si128();
__m128i acc_axb = _mm_setzero_si128();
#define SSIM_ROW_SSE2_WIN(ar, br) \
do { \
__m128i va = _mm_loadu_si128((const __m128i*)((ar) + x)); \
__m128i vb = _mm_loadu_si128((const __m128i*)((br) + x)); \
__m128i a_ev = _mm_and_si128(va, kMaskFF); \
__m128i a_od = _mm_srli_epi16(va, 8); \
__m128i b_ev = _mm_and_si128(vb, kMaskFF); \
__m128i b_od = _mm_srli_epi16(vb, 8); \
acc_a = _mm_add_epi16(acc_a, _mm_add_epi16(a_ev, a_od)); \
acc_b = _mm_add_epi16(acc_b, _mm_add_epi16(b_ev, b_od)); \
acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(a_ev, a_ev), \
_mm_madd_epi16(a_od, a_od))); \
acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(b_ev, b_ev), \
_mm_madd_epi16(b_od, b_od))); \
acc_axb = \
_mm_add_epi32(acc_axb, _mm_add_epi32(_mm_madd_epi16(a_ev, b_ev), \
_mm_madd_epi16(a_od, b_od))); \
} while (0)
SSIM_ROW_SSE2_WIN(a0, b0);
SSIM_ROW_SSE2_WIN(a1, b1);
SSIM_ROW_SSE2_WIN(a2, b2);
SSIM_ROW_SSE2_WIN(a3, b3);
#undef SSIM_ROW_SSE2_WIN
acc_a = _mm_madd_epi16(acc_a, kOnes16);
acc_b = _mm_madd_epi16(acc_b, kOnes16);
{
__m128i t0 = _mm_unpacklo_epi32(acc_a, acc_b);
__m128i t1 = _mm_unpackhi_epi32(acc_a, acc_b);
__m128i t2 = _mm_unpacklo_epi32(acc_sq, acc_axb);
__m128i t3 = _mm_unpackhi_epi32(acc_sq, acc_axb);
_mm_storeu_si128((__m128i*)(dst + 0), _mm_unpacklo_epi64(t0, t2));
_mm_storeu_si128((__m128i*)(dst + 4), _mm_unpackhi_epi64(t0, t2));
_mm_storeu_si128((__m128i*)(dst + 8), _mm_unpacklo_epi64(t1, t3));
_mm_storeu_si128((__m128i*)(dst + 12), _mm_unpackhi_epi64(t1, t3));
}
dst += 16;
}
}
#endif // HAS_SSIM4X4ROW_SSE2
#if defined(HAS_SSIM4X4ROW_AVX2)
// Width is a multiple of 32.
LIBYUV_TARGET_AVX2
void Ssim4x4Row_AVX2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const ptrdiff_t sa = stride_a;
const ptrdiff_t sb = stride_b;
const uint8_t* a0 = src_a;
const uint8_t* a1 = a0 + sa;
const uint8_t* a2 = a1 + sa;
const uint8_t* a3 = a2 + sa;
const uint8_t* b0 = src_b;
const uint8_t* b1 = b0 + sb;
const uint8_t* b2 = b1 + sb;
const uint8_t* b3 = b2 + sb;
const __m256i kMaskFF = _mm256_set1_epi16(0x00ff);
const __m256i kOnes16 = _mm256_set1_epi16(1);
int x;
for (x = 0; x < width; x += 32) {
__m256i acc_a = _mm256_setzero_si256();
__m256i acc_b = _mm256_setzero_si256();
__m256i acc_sq = _mm256_setzero_si256();
__m256i acc_axb = _mm256_setzero_si256();
#define SSIM_ROW_AVX2_WIN(ar, br) \
do { \
__m256i va = _mm256_loadu_si256((const __m256i*)((ar) + x)); \
__m256i vb = _mm256_loadu_si256((const __m256i*)((br) + x)); \
__m256i a_ev = _mm256_and_si256(va, kMaskFF); \
__m256i a_od = _mm256_srli_epi16(va, 8); \
__m256i b_ev = _mm256_and_si256(vb, kMaskFF); \
__m256i b_od = _mm256_srli_epi16(vb, 8); \
acc_a = _mm256_add_epi16(acc_a, _mm256_add_epi16(a_ev, a_od)); \
acc_b = _mm256_add_epi16(acc_b, _mm256_add_epi16(b_ev, b_od)); \
acc_sq = _mm256_add_epi32( \
acc_sq, _mm256_add_epi32(_mm256_madd_epi16(a_ev, a_ev), \
_mm256_madd_epi16(a_od, a_od))); \
acc_sq = _mm256_add_epi32( \
acc_sq, _mm256_add_epi32(_mm256_madd_epi16(b_ev, b_ev), \
_mm256_madd_epi16(b_od, b_od))); \
acc_axb = _mm256_add_epi32( \
acc_axb, _mm256_add_epi32(_mm256_madd_epi16(a_ev, b_ev), \
_mm256_madd_epi16(a_od, b_od))); \
} while (0)
SSIM_ROW_AVX2_WIN(a0, b0);
SSIM_ROW_AVX2_WIN(a1, b1);
SSIM_ROW_AVX2_WIN(a2, b2);
SSIM_ROW_AVX2_WIN(a3, b3);
#undef SSIM_ROW_AVX2_WIN
acc_a = _mm256_madd_epi16(acc_a, kOnes16);
acc_b = _mm256_madd_epi16(acc_b, kOnes16);
{
__m256i t0 = _mm256_unpacklo_epi32(acc_a, acc_b);
__m256i t1 = _mm256_unpackhi_epi32(acc_a, acc_b);
__m256i t2 = _mm256_unpacklo_epi32(acc_sq, acc_axb);
__m256i t3 = _mm256_unpackhi_epi32(acc_sq, acc_axb);
__m256i u0 = _mm256_unpacklo_epi64(t0, t2);
__m256i u1 = _mm256_unpackhi_epi64(t0, t2);
__m256i u2 = _mm256_unpacklo_epi64(t1, t3);
__m256i u3 = _mm256_unpackhi_epi64(t1, t3);
_mm256_storeu_si256((__m256i*)(dst + 0),
_mm256_permute2x128_si256(u0, u1, 0x20));
_mm256_storeu_si256((__m256i*)(dst + 8),
_mm256_permute2x128_si256(u2, u3, 0x20));
_mm256_storeu_si256((__m256i*)(dst + 16),
_mm256_permute2x128_si256(u0, u1, 0x31));
_mm256_storeu_si256((__m256i*)(dst + 24),
_mm256_permute2x128_si256(u2, u3, 0x31));
}
dst += 32;
}
_mm256_zeroupper();
}
#endif // HAS_SSIM4X4ROW_AVX2
#if defined(HAS_SSIM4X4ROW_AVX512BW)
// Width is a multiple of 4.
LIBYUV_TARGET_AVX512BW
void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const ptrdiff_t sa = stride_a;
const ptrdiff_t sb = stride_b;
const uint8_t* a0 = src_a;
const uint8_t* a1 = a0 + sa;
const uint8_t* a2 = a1 + sa;
const uint8_t* a3 = a2 + sa;
const uint8_t* b0 = src_b;
const uint8_t* b1 = b0 + sb;
const uint8_t* b2 = b1 + sb;
const uint8_t* b3 = b2 + sb;
const __m512i kMaskFF = _mm512_set1_epi16(0x00ff);
const __m512i kOnes16 = _mm512_set1_epi16(1);
#define SSIM_ROW_AVX512_WIN(va, vb) \
do { \
__m512i a_ev = _mm512_and_si512(va, kMaskFF); \
__m512i a_od = _mm512_srli_epi16(va, 8); \
__m512i b_ev = _mm512_and_si512(vb, kMaskFF); \
__m512i b_od = _mm512_srli_epi16(vb, 8); \
acc_a = _mm512_add_epi16(acc_a, _mm512_add_epi16(a_ev, a_od)); \
acc_b = _mm512_add_epi16(acc_b, _mm512_add_epi16(b_ev, b_od)); \
acc_sq = _mm512_add_epi32( \
acc_sq, _mm512_add_epi32(_mm512_madd_epi16(a_ev, a_ev), \
_mm512_madd_epi16(a_od, a_od))); \
acc_sq = _mm512_add_epi32( \
acc_sq, _mm512_add_epi32(_mm512_madd_epi16(b_ev, b_ev), \
_mm512_madd_epi16(b_od, b_od))); \
acc_axb = _mm512_add_epi32( \
acc_axb, _mm512_add_epi32(_mm512_madd_epi16(a_ev, b_ev), \
_mm512_madd_epi16(a_od, b_od))); \
} while (0)
#define SSIM_TRANSPOSE_AVX512_WIN() \
acc_a = _mm512_madd_epi16(acc_a, kOnes16); \
acc_b = _mm512_madd_epi16(acc_b, kOnes16); \
__m512i t0 = _mm512_unpacklo_epi32(acc_a, acc_b); \
__m512i t1 = _mm512_unpackhi_epi32(acc_a, acc_b); \
__m512i t2 = _mm512_unpacklo_epi32(acc_sq, acc_axb); \
__m512i t3 = _mm512_unpackhi_epi32(acc_sq, acc_axb); \
__m512i u0 = _mm512_unpacklo_epi64(t0, t2); \
__m512i u1 = _mm512_unpackhi_epi64(t0, t2); \
__m512i u2 = _mm512_unpacklo_epi64(t1, t3); \
__m512i u3 = _mm512_unpackhi_epi64(t1, t3); \
__m512i v0 = _mm512_shuffle_i32x4(u0, u1, 0x44); \
__m512i v1 = _mm512_shuffle_i32x4(u2, u3, 0x44); \
__m512i v2 = _mm512_shuffle_i32x4(u0, u1, 0xee); \
__m512i v3 = _mm512_shuffle_i32x4(u2, u3, 0xee); \
__m512i out0 = _mm512_shuffle_i32x4(v0, v1, 0x88); \
__m512i out1 = _mm512_shuffle_i32x4(v0, v1, 0xdd); \
__m512i out2 = _mm512_shuffle_i32x4(v2, v3, 0x88); \
__m512i out3 = _mm512_shuffle_i32x4(v2, v3, 0xdd)
while (width >= 64) {
__m512i acc_a = _mm512_setzero_si512();
__m512i acc_b = _mm512_setzero_si512();
__m512i acc_sq = _mm512_setzero_si512();
__m512i acc_axb = _mm512_setzero_si512();
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a0), _mm512_loadu_si512(b0));
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a1), _mm512_loadu_si512(b1));
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a2), _mm512_loadu_si512(b2));
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a3), _mm512_loadu_si512(b3));
SSIM_TRANSPOSE_AVX512_WIN();
_mm512_storeu_si512(dst + 0, out0);
_mm512_storeu_si512(dst + 16, out1);
_mm512_storeu_si512(dst + 32, out2);
_mm512_storeu_si512(dst + 48, out3);
a0 += 64;
a1 += 64;
a2 += 64;
a3 += 64;
b0 += 64;
b1 += 64;
b2 += 64;
b3 += 64;
dst += 64;
width -= 64;
}
if (width > 0) {
const __mmask16 load_mask = (__mmask16)((1U << (width >> 2)) - 1U);
const uint64_t store_mask = LowBytesMask64(width);
__m512i acc_a = _mm512_setzero_si512();
__m512i acc_b = _mm512_setzero_si512();
__m512i acc_sq = _mm512_setzero_si512();
__m512i acc_axb = _mm512_setzero_si512();
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a0),
_mm512_maskz_loadu_epi32(load_mask, b0));
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a1),
_mm512_maskz_loadu_epi32(load_mask, b1));
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a2),
_mm512_maskz_loadu_epi32(load_mask, b2));
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a3),
_mm512_maskz_loadu_epi32(load_mask, b3));
SSIM_TRANSPOSE_AVX512_WIN();
_mm512_mask_storeu_epi32(dst + 0, (__mmask16)store_mask, out0);
_mm512_mask_storeu_epi32(dst + 16, (__mmask16)(store_mask >> 16), out1);
_mm512_mask_storeu_epi32(dst + 32, (__mmask16)(store_mask >> 32), out2);
_mm512_mask_storeu_epi32(dst + 48, (__mmask16)(store_mask >> 48), out3);
}
#undef SSIM_ROW_AVX512_WIN
#undef SSIM_TRANSPOSE_AVX512_WIN
}
#endif // HAS_SSIM4X4ROW_AVX512BW
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

128
source/row_lasx.cc
View File

@ -10,6 +10,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
// Note: every register written by an inline asm block (vector, general
@ -2045,6 +2046,133 @@ void RGBToYMatrixRow_LASX(const uint8_t* src_rgba,
"$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
"$xr15", "$xr16", "$xr17");
}
#ifdef HAS_SSIM4X4ROW_LASX
void Ssim4x4Row_LASX(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
int x;
int len = width / 32;
const uint8_t* src_a1 = src_a + stride_a;
const uint8_t* src_a2 = src_a1 + stride_a;
const uint8_t* src_a3 = src_a2 + stride_a;
const uint8_t* src_b1 = src_b + stride_b;
const uint8_t* src_b2 = src_b1 + stride_b;
const uint8_t* src_b3 = src_b2 + stride_b;
for (x = 0; x < len; x++) {
__m256i a0 = __lasx_xvld(src_a, 0);
__m256i a1 = __lasx_xvld(src_a1, 0);
__m256i a2 = __lasx_xvld(src_a2, 0);
__m256i a3 = __lasx_xvld(src_a3, 0);
__m256i b0 = __lasx_xvld(src_b, 0);
__m256i b1 = __lasx_xvld(src_b1, 0);
__m256i b2 = __lasx_xvld(src_b2, 0);
__m256i b3 = __lasx_xvld(src_b3, 0);
__m256i sum_a_16 =
__lasx_xvadd_h(__lasx_xvadd_h(__lasx_xvhaddw_hu_bu(a0, a0),
__lasx_xvhaddw_hu_bu(a1, a1)),
__lasx_xvadd_h(__lasx_xvhaddw_hu_bu(a2, a2),
__lasx_xvhaddw_hu_bu(a3, a3)));
__m256i sum_b_16 =
__lasx_xvadd_h(__lasx_xvadd_h(__lasx_xvhaddw_hu_bu(b0, b0),
__lasx_xvhaddw_hu_bu(b1, b1)),
__lasx_xvadd_h(__lasx_xvhaddw_hu_bu(b2, b2),
__lasx_xvhaddw_hu_bu(b3, b3)));
__m256i sum_a = __lasx_xvhaddw_wu_hu(sum_a_16, sum_a_16);
__m256i sum_b = __lasx_xvhaddw_wu_hu(sum_b_16, sum_b_16);
__m256i ab0_ev = __lasx_xvmulwev_h_bu(a0, b0);
__m256i ab0_od = __lasx_xvmulwod_h_bu(a0, b0);
__m256i ab1_ev = __lasx_xvmulwev_h_bu(a1, b1);
__m256i ab1_od = __lasx_xvmulwod_h_bu(a1, b1);
__m256i ab2_ev = __lasx_xvmulwev_h_bu(a2, b2);
__m256i ab2_od = __lasx_xvmulwod_h_bu(a2, b2);
__m256i ab3_ev = __lasx_xvmulwev_h_bu(a3, b3);
__m256i ab3_od = __lasx_xvmulwod_h_bu(a3, b3);
__m256i sum_axb = __lasx_xvadd_w(
__lasx_xvadd_w(__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab0_ev, ab0_ev),
__lasx_xvhaddw_wu_hu(ab0_od, ab0_od)),
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab1_ev, ab1_ev),
__lasx_xvhaddw_wu_hu(ab1_od, ab1_od))),
__lasx_xvadd_w(__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab2_ev, ab2_ev),
__lasx_xvhaddw_wu_hu(ab2_od, ab2_od)),
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(ab3_ev, ab3_ev),
__lasx_xvhaddw_wu_hu(ab3_od, ab3_od))));
__m256i sq0 = __lasx_xvadd_w(
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a0, a0),
__lasx_xvmulwev_h_bu(a0, a0)),
__lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a0, a0),
__lasx_xvmulwod_h_bu(a0, a0))),
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b0, b0),
__lasx_xvmulwev_h_bu(b0, b0)),
__lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b0, b0),
__lasx_xvmulwod_h_bu(b0, b0))));
__m256i sq1 = __lasx_xvadd_w(
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a1, a1),
__lasx_xvmulwev_h_bu(a1, a1)),
__lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a1, a1),
__lasx_xvmulwod_h_bu(a1, a1))),
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b1, b1),
__lasx_xvmulwev_h_bu(b1, b1)),
__lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b1, b1),
__lasx_xvmulwod_h_bu(b1, b1))));
__m256i sq2 = __lasx_xvadd_w(
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a2, a2),
__lasx_xvmulwev_h_bu(a2, a2)),
__lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a2, a2),
__lasx_xvmulwod_h_bu(a2, a2))),
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b2, b2),
__lasx_xvmulwev_h_bu(b2, b2)),
__lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b2, b2),
__lasx_xvmulwod_h_bu(b2, b2))));
__m256i sq3 = __lasx_xvadd_w(
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(a3, a3),
__lasx_xvmulwev_h_bu(a3, a3)),
__lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(a3, a3),
__lasx_xvmulwod_h_bu(a3, a3))),
__lasx_xvadd_w(__lasx_xvhaddw_wu_hu(__lasx_xvmulwev_h_bu(b3, b3),
__lasx_xvmulwev_h_bu(b3, b3)),
__lasx_xvhaddw_wu_hu(__lasx_xvmulwod_h_bu(b3, b3),
__lasx_xvmulwod_h_bu(b3, b3))));
__m256i sum_sq_ab =
__lasx_xvadd_w(__lasx_xvadd_w(sq0, sq1), __lasx_xvadd_w(sq2, sq3));
__m256i t0 = __lasx_xvilvl_w(sum_b, sum_a);
__m256i t1 = __lasx_xvilvh_w(sum_b, sum_a);
__m256i t2 = __lasx_xvilvl_w(sum_axb, sum_sq_ab);
__m256i t3 = __lasx_xvilvh_w(sum_axb, sum_sq_ab);
__m256i p0 = __lasx_xvilvl_d(t2, t0);
__m256i p1 = __lasx_xvilvh_d(t2, t0);
__m256i p2 = __lasx_xvilvl_d(t3, t1);
__m256i p3 = __lasx_xvilvh_d(t3, t1);
__m256i out0 = __lasx_xvpermi_q(p1, p0, 0x20);
__m256i out1 = __lasx_xvpermi_q(p3, p2, 0x20);
__m256i out2 = __lasx_xvpermi_q(p1, p0, 0x31);
__m256i out3 = __lasx_xvpermi_q(p3, p2, 0x31);
__lasx_xvst(out0, dst, 0);
__lasx_xvst(out1, dst, 32);
__lasx_xvst(out2, dst, 64);
__lasx_xvst(out3, dst, 96);
src_a += 32;
src_a1 += 32;
src_a2 += 32;
src_a3 += 32;
src_b += 32;
src_b1 += 32;
src_b2 += 32;
src_b3 += 32;
dst += 32;
}
}
#endif // HAS_SSIM4X4ROW_LASX
// undef for unified sources build
#undef ALPHA_VAL
#undef YUVTORGB_SETUP

114
source/row_lsx.cc
View File

@ -10,6 +10,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
// Note: every register written by an inline asm block (vector, general
@ -2715,6 +2716,119 @@ void RGBToYMatrixRow_LSX(const uint8_t* src_rgba,
"$vr16", "$vr17");
}
#ifdef HAS_SSIM4X4ROW_LSX
void Ssim4x4Row_LSX(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
int x;
int len = width / 16;
const uint8_t* src_a1 = src_a + stride_a;
const uint8_t* src_a2 = src_a1 + stride_a;
const uint8_t* src_a3 = src_a2 + stride_a;
const uint8_t* src_b1 = src_b + stride_b;
const uint8_t* src_b2 = src_b1 + stride_b;
const uint8_t* src_b3 = src_b2 + stride_b;
for (x = 0; x < len; x++) {
__m128i a0 = __lsx_vld(src_a, 0);
__m128i a1 = __lsx_vld(src_a1, 0);
__m128i a2 = __lsx_vld(src_a2, 0);
__m128i a3 = __lsx_vld(src_a3, 0);
__m128i b0 = __lsx_vld(src_b, 0);
__m128i b1 = __lsx_vld(src_b1, 0);
__m128i b2 = __lsx_vld(src_b2, 0);
__m128i b3 = __lsx_vld(src_b3, 0);
__m128i sum_a_16 = __lsx_vadd_h(
__lsx_vadd_h(__lsx_vhaddw_hu_bu(a0, a0), __lsx_vhaddw_hu_bu(a1, a1)),
__lsx_vadd_h(__lsx_vhaddw_hu_bu(a2, a2), __lsx_vhaddw_hu_bu(a3, a3)));
__m128i sum_b_16 = __lsx_vadd_h(
__lsx_vadd_h(__lsx_vhaddw_hu_bu(b0, b0), __lsx_vhaddw_hu_bu(b1, b1)),
__lsx_vadd_h(__lsx_vhaddw_hu_bu(b2, b2), __lsx_vhaddw_hu_bu(b3, b3)));
__m128i sum_a = __lsx_vhaddw_wu_hu(sum_a_16, sum_a_16);
__m128i sum_b = __lsx_vhaddw_wu_hu(sum_b_16, sum_b_16);
__m128i ab0_ev = __lsx_vmulwev_h_bu(a0, b0);
__m128i ab0_od = __lsx_vmulwod_h_bu(a0, b0);
__m128i ab1_ev = __lsx_vmulwev_h_bu(a1, b1);
__m128i ab1_od = __lsx_vmulwod_h_bu(a1, b1);
__m128i ab2_ev = __lsx_vmulwev_h_bu(a2, b2);
__m128i ab2_od = __lsx_vmulwod_h_bu(a2, b2);
__m128i ab3_ev = __lsx_vmulwev_h_bu(a3, b3);
__m128i ab3_od = __lsx_vmulwod_h_bu(a3, b3);
__m128i sum_axb = __lsx_vadd_w(
__lsx_vadd_w(__lsx_vadd_w(__lsx_vhaddw_wu_hu(ab0_ev, ab0_ev),
__lsx_vhaddw_wu_hu(ab0_od, ab0_od)),
__lsx_vadd_w(__lsx_vhaddw_wu_hu(ab1_ev, ab1_ev),
__lsx_vhaddw_wu_hu(ab1_od, ab1_od))),
__lsx_vadd_w(__lsx_vadd_w(__lsx_vhaddw_wu_hu(ab2_ev, ab2_ev),
__lsx_vhaddw_wu_hu(ab2_od, ab2_od)),
__lsx_vadd_w(__lsx_vhaddw_wu_hu(ab3_ev, ab3_ev),
__lsx_vhaddw_wu_hu(ab3_od, ab3_od))));
__m128i sq0 = __lsx_vadd_w(
__lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a0, a0),
__lsx_vmulwev_h_bu(a0, a0)),
__lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a0, a0),
__lsx_vmulwod_h_bu(a0, a0))),
__lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b0, b0),
__lsx_vmulwev_h_bu(b0, b0)),
__lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b0, b0),
__lsx_vmulwod_h_bu(b0, b0))));
__m128i sq1 = __lsx_vadd_w(
__lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a1, a1),
__lsx_vmulwev_h_bu(a1, a1)),
__lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a1, a1),
__lsx_vmulwod_h_bu(a1, a1))),
__lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b1, b1),
__lsx_vmulwev_h_bu(b1, b1)),
__lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b1, b1),
__lsx_vmulwod_h_bu(b1, b1))));
__m128i sq2 = __lsx_vadd_w(
__lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a2, a2),
__lsx_vmulwev_h_bu(a2, a2)),
__lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a2, a2),
__lsx_vmulwod_h_bu(a2, a2))),
__lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b2, b2),
__lsx_vmulwev_h_bu(b2, b2)),
__lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b2, b2),
__lsx_vmulwod_h_bu(b2, b2))));
__m128i sq3 = __lsx_vadd_w(
__lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(a3, a3),
__lsx_vmulwev_h_bu(a3, a3)),
__lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(a3, a3),
__lsx_vmulwod_h_bu(a3, a3))),
__lsx_vadd_w(__lsx_vhaddw_wu_hu(__lsx_vmulwev_h_bu(b3, b3),
__lsx_vmulwev_h_bu(b3, b3)),
__lsx_vhaddw_wu_hu(__lsx_vmulwod_h_bu(b3, b3),
__lsx_vmulwod_h_bu(b3, b3))));
__m128i sum_sq_ab =
__lsx_vadd_w(__lsx_vadd_w(sq0, sq1), __lsx_vadd_w(sq2, sq3));
__m128i out0, out1, out2, out3;
LSX_TRANSPOSE4x4_W(sum_a, sum_b, sum_sq_ab, sum_axb, out0, out1, out2,
out3);
__lsx_vst(out0, dst, 0);
__lsx_vst(out1, dst, 16);
__lsx_vst(out2, dst, 32);
__lsx_vst(out3, dst, 48);
src_a += 16;
src_a1 += 16;
src_a2 += 16;
src_a3 += 16;
src_b += 16;
src_b1 += 16;
src_b2 += 16;
src_b3 += 16;
dst += 16;
}
}
#endif // HAS_SSIM4X4ROW_LSX
// undef for unified sources build
#undef YUVTORGB_SETUP
#undef READYUV422_D

112
source/row_rvv.cc
View File

@ -15,6 +15,7 @@
* Contributed by Bruce Lai <bruce.lai@sifive.com>
*/
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
// Note: every register written by an inline asm block (vector, general
@ -2669,6 +2670,117 @@ void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
}
#endif // HAS_MIRRORUVROW_RVV
#ifdef HAS_SSIM4X4ROW_RVV
#define SSIM4X4_LOAD_RVV(ptr_a, ptr_b) \
"vlseg4e8.v v0, (" #ptr_a \
") \n" \
"vlseg4e8.v v4, (" #ptr_b \
") \n" \
"vwaddu.vv v10, v0, v1 \n" \
"vwaddu.vv v11, v2, v3 \n" \
"vwaddu.vv v12, v4, v5 \n" \
"vwaddu.vv v13, v6, v7 \n" \
"vwmulu.vv v14, v0, v4 \n" \
"vwmulu.vv v15, v1, v5 \n" \
"vwmulu.vv v24, v2, v6 \n" \
"vwmulu.vv v25, v3, v7 \n" \
"vwmulu.vv v26, v0, v0 \n" \
"vwmulu.vv v27, v1, v1 \n" \
"vwmulu.vv v28, v2, v2 \n" \
"vwmulu.vv v29, v3, v3 \n" \
"vwmulu.vv v0, v4, v4 \n" \
"vwmulu.vv v1, v5, v5 \n" \
"vwmulu.vv v2, v6, v6 \n" \
"vwmulu.vv v3, v7, v7 \n"
#define SSIM4X4_ACCUM_RVV(ptr_a, ptr_b) \
"vsetvli zero, zero, e8, mf2, ta, ma \n" \
SSIM4X4_LOAD_RVV(ptr_a, ptr_b) \
"vsetvli zero, zero, e16, m1, ta, ma \n" \
"vadd.vv v10, v10, v11 \n" \
"vadd.vv v12, v12, v13 \n" \
"vadd.vv v8, v8, v10 \n" \
"vadd.vv v9, v9, v12 \n" \
"vwaddu.wv v22, v22, v14 \n" \
"vwaddu.wv v22, v22, v15 \n" \
"vwaddu.wv v22, v22, v24 \n" \
"vwaddu.wv v22, v22, v25 \n" \
"vwaddu.wv v20, v20, v26 \n" \
"vwaddu.wv v20, v20, v27 \n" \
"vwaddu.wv v20, v20, v28 \n" \
"vwaddu.wv v20, v20, v29 \n" \
"vwaddu.wv v20, v20, v0 \n" \
"vwaddu.wv v20, v20, v1 \n" \
"vwaddu.wv v20, v20, v2 \n" \
"vwaddu.wv v20, v20, v3 \n"
void Ssim4x4Row_RVV(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
size_t vl;
size_t byte_vl;
const uint8_t* tmp_a;
const uint8_t* tmp_b;
width >>= 2;
asm volatile(
"1: \n"
"vsetvli %[vl], %[w], e8, mf2, ta, ma \n"
"add %[tmp_a], %[src_a], %[stride_a]\n"
"add %[tmp_b], %[src_b], %[stride_b]\n"
SSIM4X4_LOAD_RVV(%[src_a], %[src_b])
"vsetvli zero, zero, e16, m1, ta, ma \n"
"vadd.vv v8, v10, v11 \n"
"vadd.vv v9, v12, v13 \n"
"vwaddu.vv v22, v14, v15 \n"
"vwaddu.wv v22, v22, v24 \n"
"vwaddu.wv v22, v22, v25 \n"
"vwaddu.vv v20, v26, v27 \n"
"vwaddu.wv v20, v20, v28 \n"
"vwaddu.wv v20, v20, v29 \n"
"vwaddu.wv v20, v20, v0 \n"
"vwaddu.wv v20, v20, v1 \n"
"vwaddu.wv v20, v20, v2 \n"
"vwaddu.wv v20, v20, v3 \n"
SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b])
"add %[tmp_a], %[tmp_a], %[stride_a]\n"
"add %[tmp_b], %[tmp_b], %[stride_b]\n"
SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b])
"add %[tmp_a], %[tmp_a], %[stride_a]\n"
"add %[tmp_b], %[tmp_b], %[stride_b]\n"
SSIM4X4_ACCUM_RVV(%[tmp_a], %[tmp_b])
"vwcvtu.x.x.v v16, v8 \n"
"vwcvtu.x.x.v v18, v9 \n"
"vsetvli zero, zero, e32, m2, ta, ma \n"
"vsseg4e32.v v16, (%[dst]) \n"
"sub %[w], %[w], %[vl] \n"
"slli %[byte_vl], %[vl], 2 \n"
"add %[src_a], %[src_a], %[byte_vl]\n"
"add %[src_b], %[src_b], %[byte_vl]\n"
"slli %[byte_vl], %[vl], 4 \n"
"add %[dst], %[dst], %[byte_vl] \n"
"bgtz %[w], 1b \n"
: [src_a] "+r"(src_a), // %[src_a]
[src_b] "+r"(src_b), // %[src_b]
[dst] "+r"(dst), // %[dst]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl), // %[vl]
[byte_vl] "=&r"(byte_vl), // %[byte_vl]
[tmp_a] "=&r"(tmp_a), // %[tmp_a]
[tmp_b] "=&r"(tmp_b) // %[tmp_b]
: [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a]
[stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b]
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
"v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29");
}
#undef SSIM4X4_LOAD_RVV
#undef SSIM4X4_ACCUM_RVV
#endif // HAS_SSIM4X4ROW_RVV
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

12
source/row_sme.cc
View File

@ -8,6 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
#include "libyuv/row_sve.h"
@ -1122,6 +1123,17 @@ __arm_locally_streaming void ARGBToYMatrixRow_SME(
ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c);
}
#ifdef HAS_SSIM4X4ROW_SME
__arm_locally_streaming void Ssim4x4Row_SME(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width);
}
#endif // HAS_SSIM4X4ROW_SME
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
// defined(__aarch64__)

12
source/row_sve.cc
View File

@ -9,6 +9,7 @@
*/
#include "libyuv/row_sve.h"
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
#ifdef __cplusplus
@ -1724,6 +1725,17 @@ void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv,
}
#endif // HAS_HALFWIDTHROW_16TO8_SVE2
#ifdef HAS_SSIM4X4ROW_SVE2
void Ssim4x4Row_SVE2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
Ssim4x4Row_SVE_SC(src_a, stride_a, src_b, stride_b, dst, width);
}
#endif // HAS_SSIM4X4ROW_SVE2
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#ifdef __cplusplus

53
unit_test/compare_test.cc
View File

@ -654,7 +654,7 @@ TEST_F(LibYUVCompareTest, Psnr) {
free_aligned_buffer_page_end(src_b);
}
TEST_F(LibYUVCompareTest, DISABLED_BenchmarkSsim_Opt) {
TEST_F(LibYUVCompareTest, BenchmarkSsim_Opt) {
align_buffer_page_end(src_a, benchmark_width_ * benchmark_height_);
align_buffer_page_end(src_b, benchmark_width_ * benchmark_height_);
for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
@ -679,6 +679,57 @@ TEST_F(LibYUVCompareTest, DISABLED_BenchmarkSsim_Opt) {
free_aligned_buffer_page_end(src_b);
}
TEST_F(LibYUVCompareTest, BenchmarkSsim_Unaligned) {
align_buffer_page_end(src_a, benchmark_width_ * benchmark_height_ + 1);
align_buffer_page_end(src_b, benchmark_width_ * benchmark_height_);
for (int i = 0; i < benchmark_width_ * benchmark_height_; ++i) {
src_a[i + 1] = i;
src_b[i] = i;
}
MaskCpuFlags(benchmark_cpu_info_);
double opt_time = get_time();
for (int i = 0; i < benchmark_iterations_; ++i) {
CalcFrameSsim(src_a + 1, benchmark_width_, src_b, benchmark_width_,
benchmark_width_, benchmark_height_);
}
opt_time = (get_time() - opt_time) / benchmark_iterations_;
printf("BenchmarkSsim_Opt - %8.2f us opt\n", opt_time * 1e6);
ASSERT_EQ(0, 0);
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
}
TEST_F(LibYUVCompareTest, TestSsimAnyWidth) {
const int kMaxWidth = 140;
const int kHeight = 16;
const int kPlaneSize = kMaxWidth * kHeight + 1;
align_buffer_page_end(src_a, kPlaneSize);
align_buffer_page_end(src_b, kPlaneSize);
MemRandomize(src_a, kPlaneSize);
MemRandomize(src_b, kPlaneSize);
for (int width = 9; width <= kMaxWidth; ++width) {
for (int offset = 0; offset < 2; ++offset) {
const int size = width * kHeight;
const uint8_t* a = src_a + kPlaneSize - size - offset;
const uint8_t* b = src_b + kPlaneSize - size - offset;
MaskCpuFlags(disable_cpu_flags_);
double c_ssim = CalcFrameSsim(a, width, b, width, width, kHeight);
MaskCpuFlags(benchmark_cpu_info_);
double opt_ssim = CalcFrameSsim(a, width, b, width, width, kHeight);
EXPECT_EQ(c_ssim, opt_ssim) << "width " << width << " offset " << offset;
}
}
free_aligned_buffer_page_end(src_a);
free_aligned_buffer_page_end(src_b);
}
TEST_F(LibYUVCompareTest, Ssim) {
const int kSrcWidth = benchmark_width_;
const int kSrcHeight = benchmark_height_;