mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-29 02:17:13 +08:00
dcbc5f9f2b
Refactor CalcFrameSsim to decompose overlapping 8x8 SSIM windows
(stepped by 4 pixels horizontally and vertically) into non-overlapping
4x4 pixel tiles across a 2-row rotating buffer (row0, row1), reducing
memory reads and pixel accumulation work by 4x:
- Ssim4x4Row_* computes 4 uint32_t statistics {sum_a, sum_b, sum_sq_ab,
sum_axb} per 4x4 pixel tile across a 4-row strip (also sized to fit
future 10-bit/12-bit formats like I010 without uint32_t overflow).
- SsimWindowRow_C combines top and bottom 4x4 tile strips across
adjacent horizontal pairs and returns the exact double SSIM sum,
producing bit-identical results to Ssim8x8_C.
- CalcFrameSsim rotates row0 and row1 (with thread-local buffers and
continuity tracking under _OPENMP) and falls back to Ssim8x8_C only if
buffer allocation fails.
- Implement Ssim4x4Row across all SIMD architectures:
- x86/x64: Ssim4x4Row_SSE2, Ssim4x4Row_AVX2, and Ssim4x4Row_AVX512BW
(with k-masked tail handling for 4..60 pixels) in both
compare_gcc.cc and compare_win.cc.
- ARM: Ssim4x4Row_NEON (compare_neon.cc, compare_neon64.cc),
Ssim4x4Row_NEON_DotProd (compare_neon64.cc), Ssim4x4Row_SVE2
(row_sve.h, row_sve.cc), and Ssim4x4Row_SME (row_sme.cc).
- RISC-V: Ssim4x4Row_RVV (row_rvv.cc).
- LoongArch: Ssim4x4Row_LSX (row_lsx.cc) and Ssim4x4Row_LASX
(row_lasx.cc).
- WebAssembly: Ssim4x4Row_WASMSIMD (compare_wasm.cc).
- Enable BenchmarkSsim_Opt and add BenchmarkSsim_Unaligned and
TestSsimAnyWidth in unit_test/compare_test.cc.
R=rrwinterton@gmail.com
Bug: None
Test: out/Release/libyuv_unittest --gtest_filter=*Ssim*
Change-Id: Ibb95ae74424ecdda073ab7ab10ab1cf610b121b6
250 lines
10 KiB
C++
250 lines
10 KiB
C++
/*
|
|
* Copyright 2026 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/compare_row.h"
|
|
#include "libyuv/row.h"
|
|
|
|
#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
|
|
#include <wasm_simd128.h>
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
// Add the 4 32 bit lanes.
|
|
static inline uint32_t SumLanes_WASMSIMD(v128_t sum) {
|
|
sum = wasm_i32x4_add(sum, wasm_i64x2_shuffle(sum, sum, 1, 0));
|
|
sum = wasm_i32x4_add(sum, wasm_i32x4_shuffle(sum, sum, 1, 0, 3, 2));
|
|
return wasm_u32x4_extract_lane(sum, 0);
|
|
}
|
|
|
|
#if defined(HAS_HAMMINGDISTANCE_WASMSIMD)
|
|
// Bit counts of 64 bytes are added as bytes, then pairs of bytes are added
|
|
// to 16 bit sums, which hold up to 64 per 64 bytes, so count can be up to
|
|
// 65535. Count is a multiple of 64.
|
|
uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a,
|
|
const uint8_t* src_b,
|
|
int count) {
|
|
v128_t sum = wasm_i16x8_splat(0);
|
|
while (count > 0) {
|
|
v128_t c = wasm_i8x16_splat(0);
|
|
int i;
|
|
for (i = 0; i < 64; i += 16) {
|
|
c = wasm_i8x16_add(
|
|
c, wasm_i8x16_popcnt(wasm_v128_xor(wasm_v128_load(src_a + i),
|
|
wasm_v128_load(src_b + i))));
|
|
}
|
|
sum = wasm_i16x8_add(sum, wasm_u16x8_extadd_pairwise_u8x16(c));
|
|
src_a += 64;
|
|
src_b += 64;
|
|
count -= 64;
|
|
}
|
|
return SumLanes_WASMSIMD(wasm_u32x4_extadd_pairwise_u16x8(sum));
|
|
}
|
|
#endif // HAS_HAMMINGDISTANCE_WASMSIMD
|
|
|
|
#if defined(HAS_SUMSQUAREERROR_WASMSIMD)
|
|
// Differences are widened to signed 16 bits and squared with i32x4.dot.
|
|
// Count is a multiple of 16.
|
|
uint32_t SumSquareError_WASMSIMD(const uint8_t* src_a,
|
|
const uint8_t* src_b,
|
|
int count) {
|
|
v128_t sum = wasm_i32x4_splat(0);
|
|
while (count > 0) {
|
|
v128_t a = wasm_v128_load(src_a);
|
|
v128_t b = wasm_v128_load(src_b);
|
|
v128_t lo = wasm_i16x8_sub(wasm_u16x8_extend_low_u8x16(a),
|
|
wasm_u16x8_extend_low_u8x16(b));
|
|
v128_t hi = wasm_i16x8_sub(wasm_u16x8_extend_high_u8x16(a),
|
|
wasm_u16x8_extend_high_u8x16(b));
|
|
sum = wasm_i32x4_add(sum, wasm_i32x4_add(wasm_i32x4_dot_i16x8(lo, lo),
|
|
wasm_i32x4_dot_i16x8(hi, hi)));
|
|
src_a += 16;
|
|
src_b += 16;
|
|
count -= 16;
|
|
}
|
|
return SumLanes_WASMSIMD(sum);
|
|
}
|
|
#endif // HAS_SUMSQUAREERROR_WASMSIMD
|
|
|
|
#if defined(HAS_HASHDJB2_WASMSIMD)
|
|
static const uint32_t kHashMul_WASMSIMD[24] = {
|
|
// 33 ^ (30 - 2 * i) for byte pair i of 32 bytes.
|
|
0xccc4cfc1, 0xd61beb81, 0x13791741, 0xac185301, // 33 ^ 30 .. 24
|
|
0xc8359ec1, 0x510cfa81, 0xb0da6641, 0x92d9e201, // 33 ^ 22 .. 16
|
|
0xa3476dc1, 0x4f5f0981, 0x855cb541, 0x747c7101, // 33 ^ 14 .. 8
|
|
0x4cfa3cc1, 0x00121881, 0x00000441, 0x00000001, // 33 ^ 6 .. 0
|
|
0x1137c401, 0x1137c401, 0x1137c401, 0x1137c401, // 33 ^ 32
|
|
0x00010021, 0x00010021, 0x00010021, 0x00010021, // 16 bit 33, 1
|
|
};
|
|
|
|
// Sum of 16 bytes times 33 ^ n, as 4 lanes. Byte pairs are combined as
|
|
// src[2 * i] * 33 + src[2 * i + 1] with i32x4.dot, then multiplied by
|
|
// 33 ^ n for the pair.
|
|
static inline v128_t HashMul16_WASMSIMD(const uint8_t* src,
|
|
const v128_t* m,
|
|
v128_t k33) {
|
|
v128_t s = wasm_v128_load(src);
|
|
v128_t lo = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_low_u8x16(s), k33);
|
|
v128_t hi = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_high_u8x16(s), k33);
|
|
return wasm_i32x4_add(wasm_i32x4_mul(lo, m[0]), wasm_i32x4_mul(hi, m[1]));
|
|
}
|
|
|
|
// HashDjb2 of n bytes is hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)).
|
|
// Products are summed per lane, multiplying the sums by 33 ^ 32 for each 32
|
|
// bytes, so one horizontal add is done at the end.
|
|
// Count is a multiple of 16.
|
|
uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed) {
|
|
const v128_t* m = (const v128_t*)kHashMul_WASMSIMD;
|
|
v128_t sum = wasm_i32x4_make(0, 0, 0, (int32_t)seed);
|
|
// Load constants into Wasm locals outside the loop, instead of v128.const
|
|
// in the loop.
|
|
asm("" : "+r"(m));
|
|
const v128_t k33_32 = m[4];
|
|
const v128_t k33 = m[5];
|
|
if (count & 16) {
|
|
// Lane 3 of m[1] is 33 ^ 16, and lanes 0..2 of sum are 0.
|
|
sum = wasm_i32x4_add(wasm_i32x4_mul(sum, m[1]),
|
|
HashMul16_WASMSIMD(src, m + 2, k33));
|
|
src += 16;
|
|
count -= 16;
|
|
}
|
|
while (count > 0) {
|
|
sum = wasm_i32x4_add(
|
|
wasm_i32x4_mul(sum, k33_32),
|
|
wasm_i32x4_add(HashMul16_WASMSIMD(src, m, k33),
|
|
HashMul16_WASMSIMD(src + 16, m + 2, k33)));
|
|
src += 32;
|
|
count -= 32;
|
|
}
|
|
return SumLanes_WASMSIMD(sum);
|
|
}
|
|
#endif // HAS_HASHDJB2_WASMSIMD
|
|
|
|
#if defined(HAS_SSIM4X4ROW_WASMSIMD)
|
|
void Ssim4x4Row_WASMSIMD(const uint8_t* src_a,
|
|
int stride_a,
|
|
const uint8_t* src_b,
|
|
int stride_b,
|
|
uint32_t* dst,
|
|
int width) {
|
|
const uint8_t* src_a1 = src_a + stride_a;
|
|
const uint8_t* src_a2 = src_a1 + stride_a;
|
|
const uint8_t* src_a3 = src_a2 + stride_a;
|
|
const uint8_t* src_b1 = src_b + stride_b;
|
|
const uint8_t* src_b2 = src_b1 + stride_b;
|
|
const uint8_t* src_b3 = src_b2 + stride_b;
|
|
const v128_t mask = wasm_i16x8_splat(0x00ff);
|
|
for (; width > 0; width -= 16) {
|
|
v128_t a0 = wasm_v128_load(src_a);
|
|
v128_t a1 = wasm_v128_load(src_a1);
|
|
v128_t a2 = wasm_v128_load(src_a2);
|
|
v128_t a3 = wasm_v128_load(src_a3);
|
|
v128_t b0 = wasm_v128_load(src_b);
|
|
v128_t b1 = wasm_v128_load(src_b1);
|
|
v128_t b2 = wasm_v128_load(src_b2);
|
|
v128_t b3 = wasm_v128_load(src_b3);
|
|
|
|
v128_t a0_ev = wasm_v128_and(a0, mask);
|
|
v128_t a0_od = wasm_u16x8_shr(a0, 8);
|
|
v128_t a1_ev = wasm_v128_and(a1, mask);
|
|
v128_t a1_od = wasm_u16x8_shr(a1, 8);
|
|
v128_t a2_ev = wasm_v128_and(a2, mask);
|
|
v128_t a2_od = wasm_u16x8_shr(a2, 8);
|
|
v128_t a3_ev = wasm_v128_and(a3, mask);
|
|
v128_t a3_od = wasm_u16x8_shr(a3, 8);
|
|
|
|
v128_t b0_ev = wasm_v128_and(b0, mask);
|
|
v128_t b0_od = wasm_u16x8_shr(b0, 8);
|
|
v128_t b1_ev = wasm_v128_and(b1, mask);
|
|
v128_t b1_od = wasm_u16x8_shr(b1, 8);
|
|
v128_t b2_ev = wasm_v128_and(b2, mask);
|
|
v128_t b2_od = wasm_u16x8_shr(b2, 8);
|
|
v128_t b3_ev = wasm_v128_and(b3, mask);
|
|
v128_t b3_od = wasm_u16x8_shr(b3, 8);
|
|
|
|
v128_t sum_a_16 =
|
|
wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(a0_ev, a0_od),
|
|
wasm_i16x8_add(a1_ev, a1_od)),
|
|
wasm_i16x8_add(wasm_i16x8_add(a2_ev, a2_od),
|
|
wasm_i16x8_add(a3_ev, a3_od)));
|
|
v128_t sum_b_16 =
|
|
wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_add(b0_ev, b0_od),
|
|
wasm_i16x8_add(b1_ev, b1_od)),
|
|
wasm_i16x8_add(wasm_i16x8_add(b2_ev, b2_od),
|
|
wasm_i16x8_add(b3_ev, b3_od)));
|
|
|
|
v128_t sum_a = wasm_u32x4_extadd_pairwise_u16x8(sum_a_16);
|
|
v128_t sum_b = wasm_u32x4_extadd_pairwise_u16x8(sum_b_16);
|
|
|
|
v128_t sum_axb = wasm_i32x4_add(
|
|
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, b0_ev),
|
|
wasm_i32x4_dot_i16x8(a0_od, b0_od)),
|
|
wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, b1_ev),
|
|
wasm_i32x4_dot_i16x8(a1_od, b1_od))),
|
|
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, b2_ev),
|
|
wasm_i32x4_dot_i16x8(a2_od, b2_od)),
|
|
wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, b3_ev),
|
|
wasm_i32x4_dot_i16x8(a3_od, b3_od))));
|
|
|
|
v128_t sq0 =
|
|
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a0_ev, a0_ev),
|
|
wasm_i32x4_dot_i16x8(a0_od, a0_od)),
|
|
wasm_i32x4_add(wasm_i32x4_dot_i16x8(b0_ev, b0_ev),
|
|
wasm_i32x4_dot_i16x8(b0_od, b0_od)));
|
|
v128_t sq1 =
|
|
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a1_ev, a1_ev),
|
|
wasm_i32x4_dot_i16x8(a1_od, a1_od)),
|
|
wasm_i32x4_add(wasm_i32x4_dot_i16x8(b1_ev, b1_ev),
|
|
wasm_i32x4_dot_i16x8(b1_od, b1_od)));
|
|
v128_t sq2 =
|
|
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a2_ev, a2_ev),
|
|
wasm_i32x4_dot_i16x8(a2_od, a2_od)),
|
|
wasm_i32x4_add(wasm_i32x4_dot_i16x8(b2_ev, b2_ev),
|
|
wasm_i32x4_dot_i16x8(b2_od, b2_od)));
|
|
v128_t sq3 =
|
|
wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_dot_i16x8(a3_ev, a3_ev),
|
|
wasm_i32x4_dot_i16x8(a3_od, a3_od)),
|
|
wasm_i32x4_add(wasm_i32x4_dot_i16x8(b3_ev, b3_ev),
|
|
wasm_i32x4_dot_i16x8(b3_od, b3_od)));
|
|
v128_t sum_sq_ab =
|
|
wasm_i32x4_add(wasm_i32x4_add(sq0, sq1), wasm_i32x4_add(sq2, sq3));
|
|
|
|
v128_t t0 = wasm_i32x4_shuffle(sum_a, sum_b, 0, 4, 1, 5);
|
|
v128_t t1 = wasm_i32x4_shuffle(sum_a, sum_b, 2, 6, 3, 7);
|
|
v128_t t2 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 0, 4, 1, 5);
|
|
v128_t t3 = wasm_i32x4_shuffle(sum_sq_ab, sum_axb, 2, 6, 3, 7);
|
|
|
|
wasm_v128_store(dst, wasm_i64x2_shuffle(t0, t2, 0, 2));
|
|
wasm_v128_store(dst + 4, wasm_i64x2_shuffle(t0, t2, 1, 3));
|
|
wasm_v128_store(dst + 8, wasm_i64x2_shuffle(t1, t3, 0, 2));
|
|
wasm_v128_store(dst + 12, wasm_i64x2_shuffle(t1, t3, 1, 3));
|
|
|
|
src_a += 16;
|
|
src_a1 += 16;
|
|
src_a2 += 16;
|
|
src_a3 += 16;
|
|
src_b += 16;
|
|
src_b1 += 16;
|
|
src_b2 += 16;
|
|
src_b3 += 16;
|
|
dst += 16;
|
|
}
|
|
}
|
|
#endif // HAS_SSIM4X4ROW_WASMSIMD
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
|