coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity
libyuv/source/compare_win.cc
Frank Barchard dcbc5f9f2b [libyuv] Optimize CalcFrameSsim with 2-row 4x4 tile buffer and SIMD 
Refactor CalcFrameSsim to decompose overlapping 8x8 SSIM windows
(stepped by 4 pixels horizontally and vertically) into non-overlapping
4x4 pixel tiles across a 2-row rotating buffer (row0, row1), reducing
memory reads and pixel accumulation work by 4x:
- Ssim4x4Row_* computes 4 uint32_t statistics {sum_a, sum_b, sum_sq_ab,
  sum_axb} per 4x4 pixel tile across a 4-row strip (also sized to fit
  future 10-bit/12-bit formats like I010 without uint32_t overflow).
- SsimWindowRow_C combines top and bottom 4x4 tile strips across
  adjacent horizontal pairs and returns the exact double SSIM sum,
  producing bit-identical results to Ssim8x8_C.
- CalcFrameSsim rotates row0 and row1 (with thread-local buffers and
  continuity tracking under _OPENMP) and falls back to Ssim8x8_C only if
  buffer allocation fails.
- Implement Ssim4x4Row across all SIMD architectures:
  - x86/x64: Ssim4x4Row_SSE2, Ssim4x4Row_AVX2, and Ssim4x4Row_AVX512BW
    (with k-masked tail handling for 4..60 pixels) in both
    compare_gcc.cc and compare_win.cc.
  - ARM: Ssim4x4Row_NEON (compare_neon.cc, compare_neon64.cc),
    Ssim4x4Row_NEON_DotProd (compare_neon64.cc), Ssim4x4Row_SVE2
    (row_sve.h, row_sve.cc), and Ssim4x4Row_SME (row_sme.cc).
  - RISC-V: Ssim4x4Row_RVV (row_rvv.cc).
  - LoongArch: Ssim4x4Row_LSX (row_lsx.cc) and Ssim4x4Row_LASX
    (row_lasx.cc).
  - WebAssembly: Ssim4x4Row_WASMSIMD (compare_wasm.cc).
- Enable BenchmarkSsim_Opt and add BenchmarkSsim_Unaligned and
  TestSsimAnyWidth in unit_test/compare_test.cc.

R=rrwinterton@gmail.com

Bug: None
Test: out/Release/libyuv_unittest --gtest_filter=*Ssim*
Change-Id: Ibb95ae74424ecdda073ab7ab10ab1cf610b121b6
2026-10-15 15:33:21 -07:00

687 lines
29 KiB
C++
Raw Blame History

/*
* Copyright 2012 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
// This module is for Visual C 32/64 bit, and for clang with
// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
defined(_M_IX86)) && \
((defined(_MSC_VER) && !defined(__clang__)) || \
defined(LIBYUV_ENABLE_ROWWIN))
#include <immintrin.h> // For SSE2 to AVX512 intrinsics
#include <string.h> // For memcpy
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
#endif
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
// Target attributes allow gcc and clang to build each function for its
// instruction set regardless of the baseline.
#if defined(__clang__) || defined(__GNUC__)
#define LIBYUV_TARGET_SSE2 __attribute__((target("sse2")))
#define LIBYUV_TARGET_SSE41 __attribute__((target("sse4.1")))
#define LIBYUV_TARGET_SSE42 __attribute__((target("sse4.2,popcnt")))
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
#define LIBYUV_TARGET_AVX512BW __attribute__((target("avx512bw,avx512f")))
#else
#define LIBYUV_TARGET_SSE2
#define LIBYUV_TARGET_SSE41
#define LIBYUV_TARGET_SSE42
#define LIBYUV_TARGET_AVX2
#define LIBYUV_TARGET_AVX512BW
#endif // defined(__clang__) || defined(__GNUC__)
#if defined(HAS_HAMMINGDISTANCE_SSE42)
// Count is a multiple of 8.
LIBYUV_TARGET_SSE42
uint32_t HammingDistance_SSE42(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
uint32_t diff = 0u;
int i;
for (i = 0; i < count; i += 8) {
uint64_t a;
uint64_t b;
memcpy(&a, src_a + i, 8);
memcpy(&b, src_b + i, 8);
#if defined(__x86_64__) || defined(_M_X64)
diff += (uint32_t)_mm_popcnt_u64(a ^ b);
#else
diff += _mm_popcnt_u32((uint32_t)(a ^ b)) +
_mm_popcnt_u32((uint32_t)((a ^ b) >> 32));
#endif
}
return diff;
}
#endif // HAS_HAMMINGDISTANCE_SSE42
#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW)
// Mask of the low n bytes, for n = 0 to 64.
static __mmask64 LowBytesMask64(int n) {
return (__mmask64)(n >= 64 ? ~0ULL : (1ULL << n) - 1);
}
#endif
#if defined(HAS_HAMMINGDISTANCE_AVX2)
// Count is a multiple of 64.
LIBYUV_TARGET_AVX2
uint32_t HammingDistance_AVX2(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
const __m256i kNibbleMask = _mm256_set1_epi8(15);
const __m256i kBitCount = _mm256_broadcastsi128_si256(
_mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4));
const __m256i zero = _mm256_setzero_si256();
__m256i sum = _mm256_setzero_si256();
__m128i sum128;
int i;
for (i = 0; i < count; i += 64) {
__m256i x0 =
_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(src_a + i)),
_mm256_loadu_si256((const __m256i*)(src_b + i)));
__m256i x1 =
_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(src_a + i + 32)),
_mm256_loadu_si256((const __m256i*)(src_b + i + 32)));
__m256i c0 = _mm256_add_epi8(
_mm256_shuffle_epi8(kBitCount, _mm256_and_si256(x0, kNibbleMask)),
_mm256_shuffle_epi8(
kBitCount,
_mm256_and_si256(_mm256_srli_epi16(x0, 4), kNibbleMask)));
__m256i c1 = _mm256_add_epi8(
_mm256_shuffle_epi8(kBitCount, _mm256_and_si256(x1, kNibbleMask)),
_mm256_shuffle_epi8(
kBitCount,
_mm256_and_si256(_mm256_srli_epi16(x1, 4), kNibbleMask)));
sum = _mm256_add_epi64(sum, _mm256_sad_epu8(_mm256_add_epi8(c0, c1), zero));
}
sum128 = _mm_add_epi64(_mm256_castsi256_si128(sum),
_mm256_extracti128_si256(sum, 1));
sum128 = _mm_add_epi64(sum128, _mm_unpackhi_epi64(sum128, sum128));
_mm256_zeroupper();
return (uint32_t)_mm_cvtsi128_si32(sum128);
}
#endif // HAS_HAMMINGDISTANCE_AVX2
#if defined(HAS_HAMMINGDISTANCE_AVX512BW)
// Bit count of 64 bytes, summed to 8 qwords.
LIBYUV_TARGET_AVX512BW
static __m512i BitCount64_AVX512BW(__m512i x) {
const __m512i kNibbleMask = _mm512_set1_epi8(15);
const __m512i kBitCount = _mm512_broadcast_i32x4(
_mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4));
__m512i lo = _mm512_shuffle_epi8(kBitCount, _mm512_and_si512(x, kNibbleMask));
__m512i hi = _mm512_shuffle_epi8(
kBitCount, _mm512_and_si512(_mm512_srli_epi16(x, 4), kNibbleMask));
return _mm512_sad_epu8(_mm512_add_epi8(lo, hi), _mm512_setzero_si512());
}
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
// loads that read zeros, so any count is supported.
LIBYUV_TARGET_AVX512BW
uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
__m512i sum = _mm512_setzero_si512();
while (count >= 64) {
sum = _mm512_add_epi64(
sum, BitCount64_AVX512BW(_mm512_xor_si512(_mm512_loadu_si512(src_a),
_mm512_loadu_si512(src_b))));
src_a += 64;
src_b += 64;
count -= 64;
}
if (count > 0) {
const __mmask64 k = LowBytesMask64(count);
sum = _mm512_add_epi64(sum, BitCount64_AVX512BW(_mm512_xor_si512(
_mm512_maskz_loadu_epi8(k, src_a),
_mm512_maskz_loadu_epi8(k, src_b))));
}
return (uint32_t)_mm512_reduce_add_epi64(sum);
}
#endif // HAS_HAMMINGDISTANCE_AVX512BW
#if defined(HAS_SUMSQUAREERROR_SSE2)
// Count is a multiple of 16.
LIBYUV_TARGET_SSE2
uint32_t SumSquareError_SSE2(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
const __m128i zero = _mm_setzero_si128();
__m128i sum = _mm_setzero_si128();
int i;
for (i = 0; i < count; i += 16) {
__m128i a = _mm_loadu_si128((const __m128i*)(src_a + i));
__m128i b = _mm_loadu_si128((const __m128i*)(src_b + i));
__m128i d = _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a));
__m128i lo = _mm_unpacklo_epi8(d, zero);
__m128i hi = _mm_unpackhi_epi8(d, zero);
sum = _mm_add_epi32(sum, _mm_madd_epi16(lo, lo));
sum = _mm_add_epi32(sum, _mm_madd_epi16(hi, hi));
}
sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0xee));
sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x01));
return (uint32_t)_mm_cvtsi128_si32(sum);
}
#endif // HAS_SUMSQUAREERROR_SSE2
#if defined(HAS_SUMSQUAREERROR_AVX2)
// Count is a multiple of 32.
LIBYUV_TARGET_AVX2
uint32_t SumSquareError_AVX2(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
const __m256i zero = _mm256_setzero_si256();
__m256i sum = _mm256_setzero_si256();
__m128i sum128;
int i;
for (i = 0; i < count; i += 32) {
__m256i a = _mm256_loadu_si256((const __m256i*)(src_a + i));
__m256i b = _mm256_loadu_si256((const __m256i*)(src_b + i));
__m256i d = _mm256_or_si256(_mm256_subs_epu8(a, b), _mm256_subs_epu8(b, a));
__m256i lo = _mm256_unpacklo_epi8(d, zero);
__m256i hi = _mm256_unpackhi_epi8(d, zero);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(lo, lo));
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(hi, hi));
}
sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum),
_mm256_extracti128_si256(sum, 1));
sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, 0xee));
sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, 0x01));
_mm256_zeroupper();
return (uint32_t)_mm_cvtsi128_si32(sum128);
}
#endif // HAS_SUMSQUAREERROR_AVX2
#if defined(HAS_SUMSQUAREERROR_AVX512BW)
// Sum of squared differences of 64 bytes, as 16 dwords.
LIBYUV_TARGET_AVX512BW
static __m512i SumSquare64_AVX512BW(__m512i a, __m512i b) {
const __m512i zero = _mm512_setzero_si512();
__m512i d = _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
__m512i lo = _mm512_unpacklo_epi8(d, zero);
__m512i hi = _mm512_unpackhi_epi8(d, zero);
return _mm512_add_epi32(_mm512_madd_epi16(lo, lo), _mm512_madd_epi16(hi, hi));
}
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
// loads that read zeros, so any count is supported.
LIBYUV_TARGET_AVX512BW
uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
__m512i sum = _mm512_setzero_si512();
while (count >= 64) {
sum =
_mm512_add_epi32(sum, SumSquare64_AVX512BW(_mm512_loadu_si512(src_a),
_mm512_loadu_si512(src_b)));
src_a += 64;
src_b += 64;
count -= 64;
}
if (count > 0) {
const __mmask64 k = LowBytesMask64(count);
sum = _mm512_add_epi32(
sum, SumSquare64_AVX512BW(_mm512_maskz_loadu_epi8(k, src_a),
_mm512_maskz_loadu_epi8(k, src_b)));
}
return (uint32_t)_mm512_reduce_add_epi32(sum);
}
#endif // HAS_SUMSQUAREERROR_AVX512BW
#if defined(HAS_HASHDJB2_SSE41)
// 33 ^ (15 - i) for byte i of 16.
static const uint32_t kHashMul16[16] = {
0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, // 33 ^ 15 .. 12
0x30f35d61, 0x855cb541, 0x040a9121, 0x747c7101, // 33 ^ 11 .. 8
0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881, // 33 ^ 7 .. 4
0x00008c61, 0x00000441, 0x00000021, 0x00000001, // 33 ^ 3 .. 0
};
static const uint32_t kHash16x33 = 0x92d9e201; // 33 ^ 16
// Count is a multiple of 16.
LIBYUV_TARGET_SSE41
uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) {
const __m128i zero = _mm_setzero_si128();
const __m128i mul0 = _mm_loadu_si128((const __m128i*)(kHashMul16));
const __m128i mul1 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 4));
const __m128i mul2 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 8));
const __m128i mul3 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 12));
int i;
for (i = 0; i < count; i += 16) {
__m128i s = _mm_loadu_si128((const __m128i*)(src + i));
__m128i lo = _mm_unpacklo_epi8(s, zero);
__m128i hi = _mm_unpackhi_epi8(s, zero);
__m128i sum = _mm_add_epi32(
_mm_add_epi32(_mm_mullo_epi32(_mm_unpacklo_epi16(lo, zero), mul0),
_mm_mullo_epi32(_mm_unpackhi_epi16(lo, zero), mul1)),
_mm_add_epi32(_mm_mullo_epi32(_mm_unpacklo_epi16(hi, zero), mul2),
_mm_mullo_epi32(_mm_unpackhi_epi16(hi, zero), mul3)));
sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x0e));
sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x01));
seed = seed * kHash16x33 + (uint32_t)_mm_cvtsi128_si32(sum);
}
return seed;
}
#endif // HAS_HASHDJB2_SSE41
#if defined(HAS_HASHDJB2_AVX2) || defined(HAS_HASHDJB2_AVX512BW)
// HashDjb2 is hash = hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)).
// The 32 bit multipliers 33 ^ (63 - i) are split into 16 bit halves so that
// vpmaddwd can be used instead of vpmulld, which is 2 uops and 10 cycles.
// m = hi * 65536 + lo, with lo treated as signed and hi adjusted by the carry,
// so src * m = src * lo + ((src * hi) << 16) modulo 2 ^ 32.
// The first 64 are lo and the next 64 are hi.
static const uint16_t kHashMulLoHi[128] = {
// lo for 33 ^ 63 .. 33 ^ 0
0x0be1, 0x93c1, 0x9fa1, 0xaf81, 0x4361, 0xdb41, 0xf721, 0x1701, //
0xbae1, 0x62c1, 0x8ea1, 0xbe81, 0x7261, 0x2a41, 0x6621, 0xa601, //
0x69e1, 0x31c1, 0x7da1, 0xcd81, 0xa161, 0x7941, 0xd521, 0x3501, //
0x18e1, 0x00c1, 0x6ca1, 0xdc81, 0xd061, 0xc841, 0x4421, 0xc401, //
0xc7e1, 0xcfc1, 0x5ba1, 0xeb81, 0xff61, 0x1741, 0xb321, 0x5301, //
0x76e1, 0x9ec1, 0x4aa1, 0xfa81, 0x2e61, 0x6641, 0x2221, 0xe201, //
0x25e1, 0x6dc1, 0x39a1, 0x0981, 0x5d61, 0xb541, 0x9121, 0x7101, //
0xd4e1, 0x3cc1, 0x28a1, 0x1881, 0x8c61, 0x0441, 0x0021, 0x0001, //
// hi for 33 ^ 63 .. 33 ^ 0
0xd8be, 0xef4c, 0x6458, 0x6022, 0x40f9, 0x193e, 0x6d5f, 0x31dc, //
0x0945, 0xc238, 0x1567, 0xa38f, 0x3380, 0x301b, 0x664e, 0x031a, //
0xd18c, 0xb8c6, 0x79f6, 0xae5d, 0x0549, 0xc219, 0x7a3f, 0x1338, //
0x0095, 0xc1f5, 0xa107, 0xaf8c, 0x0552, 0x3e39, 0x3830, 0x1138, //
0x655f, 0xccc5, 0x9999, 0xd61c, 0x829c, 0x1379, 0x2f23, 0xac18, //
0xcee9, 0xc836, 0x72ac, 0x510d, 0xcc27, 0xb0da, 0xee16, 0x92da, //
0x0c35, 0xa347, 0x3b40, 0x4f5f, 0x30f3, 0x855d, 0x040b, 0x747c, //
0xec42, 0x4cfa, 0x0255, 0x0012, 0x0001, 0x0000, 0x0000, 0x0000, //
};
#endif
#if defined(HAS_HASHDJB2_AVX2)
// lo + (hi << 16), then add 8 dwords.
LIBYUV_TARGET_AVX2
static uint32_t HashSum_AVX2(__m256i lo, __m256i hi) {
__m256i s = _mm256_add_epi32(lo, _mm256_slli_epi32(hi, 16));
__m128i s128 =
_mm_add_epi32(_mm256_castsi256_si128(s), _mm256_extracti128_si256(s, 1));
s128 = _mm_add_epi32(s128, _mm_shuffle_epi32(s128, 0x0e));
s128 = _mm_add_epi32(s128, _mm_shuffle_epi32(s128, 0x01));
return (uint32_t)_mm_cvtsi128_si32(s128);
}
// Count is a multiple of 32. Process 64 bytes per loop with one horizontal
// add, then 32 bytes if needed.
LIBYUV_TARGET_AVX2
uint32_t HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed) {
const __m256i* m = (const __m256i*)kHashMulLoHi;
while (count >= 64) {
__m256i w0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src));
__m256i w1 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16)));
__m256i w2 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 32)));
__m256i w3 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 48)));
__m256i lo = _mm256_add_epi32(
_mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 0)),
_mm256_madd_epi16(w1, _mm256_loadu_si256(m + 1))),
_mm256_add_epi32(_mm256_madd_epi16(w2, _mm256_loadu_si256(m + 2)),
_mm256_madd_epi16(w3, _mm256_loadu_si256(m + 3))));
__m256i hi = _mm256_add_epi32(
_mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 4)),
_mm256_madd_epi16(w1, _mm256_loadu_si256(m + 5))),
_mm256_add_epi32(_mm256_madd_epi16(w2, _mm256_loadu_si256(m + 6)),
_mm256_madd_epi16(w3, _mm256_loadu_si256(m + 7))));
seed = seed * 0xf07f8801u + HashSum_AVX2(lo, hi); // 33 ^ 64
src += 64;
count -= 64;
}
if (count > 0) {
__m256i w0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src));
__m256i w1 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16)));
__m256i lo =
_mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 2)),
_mm256_madd_epi16(w1, _mm256_loadu_si256(m + 3)));
__m256i hi =
_mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 6)),
_mm256_madd_epi16(w1, _mm256_loadu_si256(m + 7)));
seed = seed * 0x1137c401u + HashSum_AVX2(lo, hi); // 33 ^ 32
}
_mm256_zeroupper();
return seed;
}
#endif // HAS_HASHDJB2_AVX2
#if defined(HAS_HASHDJB2_AVX512BW)
static uint32_t Pow33(int n) {
uint32_t r = 1u;
uint32_t b = 33u;
while (n) {
if (n & 1) {
r *= b;
}
b *= b;
n >>= 1;
}
return r;
}
// Sum of 64 bytes, zero extended to words in w0 and w1, times 33 ^ (63 - i).
LIBYUV_TARGET_AVX512BW
static uint32_t HashSum64_AVX512BW(__m512i w0, __m512i w1) {
const __m512i* m = (const __m512i*)kHashMulLoHi;
__m512i lo =
_mm512_add_epi32(_mm512_madd_epi16(w0, _mm512_loadu_si512(m)),
_mm512_madd_epi16(w1, _mm512_loadu_si512(m + 1)));
__m512i hi =
_mm512_add_epi32(_mm512_madd_epi16(w0, _mm512_loadu_si512(m + 2)),
_mm512_madd_epi16(w1, _mm512_loadu_si512(m + 3)));
return (uint32_t)_mm512_reduce_add_epi32(
_mm512_add_epi32(lo, _mm512_slli_epi32(hi, 16)));
}
// hash = hash * 33 ^ 64 + sum(src[i] * 33 ^ (63 - i)) for each 64 bytes.
// The remainder of n = 1 to 63 bytes is loaded into the high n bytes with a
// masked load ending at src + count, so the same multipliers apply, and
// hash is multiplied by 33 ^ n.
LIBYUV_TARGET_AVX512BW
uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) {
int n;
while (count >= 64) {
seed = seed * 0xf07f8801u + // 33 ^ 64
HashSum64_AVX512BW(
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src)),
_mm512_cvtepu8_epi16(
_mm256_loadu_si256((const __m256i*)(src + 32))));
src += 64;
count -= 64;
}
n = count;
if (n) {
const __mmask64 k = (__mmask64)~LowBytesMask64(64 - n);
__m512i s = _mm512_maskz_loadu_epi8(k, src + n - 64);
seed = seed * Pow33(n) +
HashSum64_AVX512BW(
_mm512_cvtepu8_epi16(_mm512_castsi512_si256(s)),
_mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(s, 1)));
}
return seed;
}
#endif // HAS_HASHDJB2_AVX512BW
#if defined(HAS_SSIM4X4ROW_SSE2)
// Width is a multiple of 16.
LIBYUV_TARGET_SSE2
void Ssim4x4Row_SSE2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const ptrdiff_t sa = stride_a;
const ptrdiff_t sb = stride_b;
const uint8_t* a0 = src_a;
const uint8_t* a1 = a0 + sa;
const uint8_t* a2 = a1 + sa;
const uint8_t* a3 = a2 + sa;
const uint8_t* b0 = src_b;
const uint8_t* b1 = b0 + sb;
const uint8_t* b2 = b1 + sb;
const uint8_t* b3 = b2 + sb;
const __m128i kMaskFF = _mm_set1_epi16(0x00ff);
const __m128i kOnes16 = _mm_set1_epi16(1);
int x;
for (x = 0; x < width; x += 16) {
__m128i acc_a = _mm_setzero_si128();
__m128i acc_b = _mm_setzero_si128();
__m128i acc_sq = _mm_setzero_si128();
__m128i acc_axb = _mm_setzero_si128();
#define SSIM_ROW_SSE2_WIN(ar, br) \
do { \
__m128i va = _mm_loadu_si128((const __m128i*)((ar) + x)); \
__m128i vb = _mm_loadu_si128((const __m128i*)((br) + x)); \
__m128i a_ev = _mm_and_si128(va, kMaskFF); \
__m128i a_od = _mm_srli_epi16(va, 8); \
__m128i b_ev = _mm_and_si128(vb, kMaskFF); \
__m128i b_od = _mm_srli_epi16(vb, 8); \
acc_a = _mm_add_epi16(acc_a, _mm_add_epi16(a_ev, a_od)); \
acc_b = _mm_add_epi16(acc_b, _mm_add_epi16(b_ev, b_od)); \
acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(a_ev, a_ev), \
_mm_madd_epi16(a_od, a_od))); \
acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(b_ev, b_ev), \
_mm_madd_epi16(b_od, b_od))); \
acc_axb = \
_mm_add_epi32(acc_axb, _mm_add_epi32(_mm_madd_epi16(a_ev, b_ev), \
_mm_madd_epi16(a_od, b_od))); \
} while (0)
SSIM_ROW_SSE2_WIN(a0, b0);
SSIM_ROW_SSE2_WIN(a1, b1);
SSIM_ROW_SSE2_WIN(a2, b2);
SSIM_ROW_SSE2_WIN(a3, b3);
#undef SSIM_ROW_SSE2_WIN
acc_a = _mm_madd_epi16(acc_a, kOnes16);
acc_b = _mm_madd_epi16(acc_b, kOnes16);
{
__m128i t0 = _mm_unpacklo_epi32(acc_a, acc_b);
__m128i t1 = _mm_unpackhi_epi32(acc_a, acc_b);
__m128i t2 = _mm_unpacklo_epi32(acc_sq, acc_axb);
__m128i t3 = _mm_unpackhi_epi32(acc_sq, acc_axb);
_mm_storeu_si128((__m128i*)(dst + 0), _mm_unpacklo_epi64(t0, t2));
_mm_storeu_si128((__m128i*)(dst + 4), _mm_unpackhi_epi64(t0, t2));
_mm_storeu_si128((__m128i*)(dst + 8), _mm_unpacklo_epi64(t1, t3));
_mm_storeu_si128((__m128i*)(dst + 12), _mm_unpackhi_epi64(t1, t3));
}
dst += 16;
}
}
#endif // HAS_SSIM4X4ROW_SSE2
#if defined(HAS_SSIM4X4ROW_AVX2)
// Width is a multiple of 32.
LIBYUV_TARGET_AVX2
void Ssim4x4Row_AVX2(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const ptrdiff_t sa = stride_a;
const ptrdiff_t sb = stride_b;
const uint8_t* a0 = src_a;
const uint8_t* a1 = a0 + sa;
const uint8_t* a2 = a1 + sa;
const uint8_t* a3 = a2 + sa;
const uint8_t* b0 = src_b;
const uint8_t* b1 = b0 + sb;
const uint8_t* b2 = b1 + sb;
const uint8_t* b3 = b2 + sb;
const __m256i kMaskFF = _mm256_set1_epi16(0x00ff);
const __m256i kOnes16 = _mm256_set1_epi16(1);
int x;
for (x = 0; x < width; x += 32) {
__m256i acc_a = _mm256_setzero_si256();
__m256i acc_b = _mm256_setzero_si256();
__m256i acc_sq = _mm256_setzero_si256();
__m256i acc_axb = _mm256_setzero_si256();
#define SSIM_ROW_AVX2_WIN(ar, br) \
do { \
__m256i va = _mm256_loadu_si256((const __m256i*)((ar) + x)); \
__m256i vb = _mm256_loadu_si256((const __m256i*)((br) + x)); \
__m256i a_ev = _mm256_and_si256(va, kMaskFF); \
__m256i a_od = _mm256_srli_epi16(va, 8); \
__m256i b_ev = _mm256_and_si256(vb, kMaskFF); \
__m256i b_od = _mm256_srli_epi16(vb, 8); \
acc_a = _mm256_add_epi16(acc_a, _mm256_add_epi16(a_ev, a_od)); \
acc_b = _mm256_add_epi16(acc_b, _mm256_add_epi16(b_ev, b_od)); \
acc_sq = _mm256_add_epi32( \
acc_sq, _mm256_add_epi32(_mm256_madd_epi16(a_ev, a_ev), \
_mm256_madd_epi16(a_od, a_od))); \
acc_sq = _mm256_add_epi32( \
acc_sq, _mm256_add_epi32(_mm256_madd_epi16(b_ev, b_ev), \
_mm256_madd_epi16(b_od, b_od))); \
acc_axb = _mm256_add_epi32( \
acc_axb, _mm256_add_epi32(_mm256_madd_epi16(a_ev, b_ev), \
_mm256_madd_epi16(a_od, b_od))); \
} while (0)
SSIM_ROW_AVX2_WIN(a0, b0);
SSIM_ROW_AVX2_WIN(a1, b1);
SSIM_ROW_AVX2_WIN(a2, b2);
SSIM_ROW_AVX2_WIN(a3, b3);
#undef SSIM_ROW_AVX2_WIN
acc_a = _mm256_madd_epi16(acc_a, kOnes16);
acc_b = _mm256_madd_epi16(acc_b, kOnes16);
{
__m256i t0 = _mm256_unpacklo_epi32(acc_a, acc_b);
__m256i t1 = _mm256_unpackhi_epi32(acc_a, acc_b);
__m256i t2 = _mm256_unpacklo_epi32(acc_sq, acc_axb);
__m256i t3 = _mm256_unpackhi_epi32(acc_sq, acc_axb);
__m256i u0 = _mm256_unpacklo_epi64(t0, t2);
__m256i u1 = _mm256_unpackhi_epi64(t0, t2);
__m256i u2 = _mm256_unpacklo_epi64(t1, t3);
__m256i u3 = _mm256_unpackhi_epi64(t1, t3);
_mm256_storeu_si256((__m256i*)(dst + 0),
_mm256_permute2x128_si256(u0, u1, 0x20));
_mm256_storeu_si256((__m256i*)(dst + 8),
_mm256_permute2x128_si256(u2, u3, 0x20));
_mm256_storeu_si256((__m256i*)(dst + 16),
_mm256_permute2x128_si256(u0, u1, 0x31));
_mm256_storeu_si256((__m256i*)(dst + 24),
_mm256_permute2x128_si256(u2, u3, 0x31));
}
dst += 32;
}
_mm256_zeroupper();
}
#endif // HAS_SSIM4X4ROW_AVX2
#if defined(HAS_SSIM4X4ROW_AVX512BW)
// Width is a multiple of 4.
LIBYUV_TARGET_AVX512BW
void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
int stride_a,
const uint8_t* src_b,
int stride_b,
uint32_t* dst,
int width) {
const ptrdiff_t sa = stride_a;
const ptrdiff_t sb = stride_b;
const uint8_t* a0 = src_a;
const uint8_t* a1 = a0 + sa;
const uint8_t* a2 = a1 + sa;
const uint8_t* a3 = a2 + sa;
const uint8_t* b0 = src_b;
const uint8_t* b1 = b0 + sb;
const uint8_t* b2 = b1 + sb;
const uint8_t* b3 = b2 + sb;
const __m512i kMaskFF = _mm512_set1_epi16(0x00ff);
const __m512i kOnes16 = _mm512_set1_epi16(1);
#define SSIM_ROW_AVX512_WIN(va, vb) \
do { \
__m512i a_ev = _mm512_and_si512(va, kMaskFF); \
__m512i a_od = _mm512_srli_epi16(va, 8); \
__m512i b_ev = _mm512_and_si512(vb, kMaskFF); \
__m512i b_od = _mm512_srli_epi16(vb, 8); \
acc_a = _mm512_add_epi16(acc_a, _mm512_add_epi16(a_ev, a_od)); \
acc_b = _mm512_add_epi16(acc_b, _mm512_add_epi16(b_ev, b_od)); \
acc_sq = _mm512_add_epi32( \
acc_sq, _mm512_add_epi32(_mm512_madd_epi16(a_ev, a_ev), \
_mm512_madd_epi16(a_od, a_od))); \
acc_sq = _mm512_add_epi32( \
acc_sq, _mm512_add_epi32(_mm512_madd_epi16(b_ev, b_ev), \
_mm512_madd_epi16(b_od, b_od))); \
acc_axb = _mm512_add_epi32( \
acc_axb, _mm512_add_epi32(_mm512_madd_epi16(a_ev, b_ev), \
_mm512_madd_epi16(a_od, b_od))); \
} while (0)
#define SSIM_TRANSPOSE_AVX512_WIN() \
acc_a = _mm512_madd_epi16(acc_a, kOnes16); \
acc_b = _mm512_madd_epi16(acc_b, kOnes16); \
__m512i t0 = _mm512_unpacklo_epi32(acc_a, acc_b); \
__m512i t1 = _mm512_unpackhi_epi32(acc_a, acc_b); \
__m512i t2 = _mm512_unpacklo_epi32(acc_sq, acc_axb); \
__m512i t3 = _mm512_unpackhi_epi32(acc_sq, acc_axb); \
__m512i u0 = _mm512_unpacklo_epi64(t0, t2); \
__m512i u1 = _mm512_unpackhi_epi64(t0, t2); \
__m512i u2 = _mm512_unpacklo_epi64(t1, t3); \
__m512i u3 = _mm512_unpackhi_epi64(t1, t3); \
__m512i v0 = _mm512_shuffle_i32x4(u0, u1, 0x44); \
__m512i v1 = _mm512_shuffle_i32x4(u2, u3, 0x44); \
__m512i v2 = _mm512_shuffle_i32x4(u0, u1, 0xee); \
__m512i v3 = _mm512_shuffle_i32x4(u2, u3, 0xee); \
__m512i out0 = _mm512_shuffle_i32x4(v0, v1, 0x88); \
__m512i out1 = _mm512_shuffle_i32x4(v0, v1, 0xdd); \
__m512i out2 = _mm512_shuffle_i32x4(v2, v3, 0x88); \
__m512i out3 = _mm512_shuffle_i32x4(v2, v3, 0xdd)
while (width >= 64) {
__m512i acc_a = _mm512_setzero_si512();
__m512i acc_b = _mm512_setzero_si512();
__m512i acc_sq = _mm512_setzero_si512();
__m512i acc_axb = _mm512_setzero_si512();
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a0), _mm512_loadu_si512(b0));
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a1), _mm512_loadu_si512(b1));
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a2), _mm512_loadu_si512(b2));
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a3), _mm512_loadu_si512(b3));
SSIM_TRANSPOSE_AVX512_WIN();
_mm512_storeu_si512(dst + 0, out0);
_mm512_storeu_si512(dst + 16, out1);
_mm512_storeu_si512(dst + 32, out2);
_mm512_storeu_si512(dst + 48, out3);
a0 += 64;
a1 += 64;
a2 += 64;
a3 += 64;
b0 += 64;
b1 += 64;
b2 += 64;
b3 += 64;
dst += 64;
width -= 64;
}
if (width > 0) {
const __mmask16 load_mask = (__mmask16)((1U << (width >> 2)) - 1U);
const uint64_t store_mask = LowBytesMask64(width);
__m512i acc_a = _mm512_setzero_si512();
__m512i acc_b = _mm512_setzero_si512();
__m512i acc_sq = _mm512_setzero_si512();
__m512i acc_axb = _mm512_setzero_si512();
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a0),
_mm512_maskz_loadu_epi32(load_mask, b0));
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a1),
_mm512_maskz_loadu_epi32(load_mask, b1));
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a2),
_mm512_maskz_loadu_epi32(load_mask, b2));
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a3),
_mm512_maskz_loadu_epi32(load_mask, b3));
SSIM_TRANSPOSE_AVX512_WIN();
_mm512_mask_storeu_epi32(dst + 0, (__mmask16)store_mask, out0);
_mm512_mask_storeu_epi32(dst + 16, (__mmask16)(store_mask >> 16), out1);
_mm512_mask_storeu_epi32(dst + 32, (__mmask16)(store_mask >> 32), out2);
_mm512_mask_storeu_epi32(dst + 48, (__mmask16)(store_mask >> 48), out3);
}
#undef SSIM_ROW_AVX512_WIN
#undef SSIM_TRANSPOSE_AVX512_WIN
}
#endif // HAS_SSIM4X4ROW_AVX512BW
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
#endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) &&
// (Visual C or LIBYUV_ENABLE_ROWWIN)
Reference in New Issue View Git Blame Copy Permalink