mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
ec29aca502
Refactor CalcFrameSsim to decompose overlapping 8x8 SSIM windows
(stepped by 4 pixels horizontally and vertically) into non-overlapping
4x4 pixel tiles across a 2-row rotating buffer (row0, row1), reducing
memory reads and pixel accumulation work by 4x:
- Ssim4x4Row_* computes 4 uint32_t stats {sum_a, sum_b, sum_sq_ab,
sum_axb} per 4x4 pixel tile across a 4-row strip (also sized to fit
future 10-bit/12-bit formats like I010 without uint32_t overflow).
- SsimWindowRow_C combines top and bottom 4x4 tile strips across
adjacent horizontal pairs and returns the exact double SSIM sum,
producing bit-identical results to Ssim8x8_C.
- CalcFrameSsim rotates row0 and row1 (with thread-local buffers and
continuity tracking under _OPENMP) and falls back to Ssim8x8_C only
if buffer allocation fails.
- Implement Ssim4x4Row across all SIMD architectures:
- x86/x64: Ssim4x4Row_SSE2, Ssim4x4Row_AVX2, and Ssim4x4Row_AVX512BW
(with k-masked tail handling for 4..60 pixels) in both
compare_gcc.cc and compare_win.cc.
- ARM: Ssim4x4Row_NEON (compare_neon.cc, compare_neon64.cc),
Ssim4x4Row_NEON_DotProd (compare_neon64.cc), Ssim4x4Row_SVE2
(row_sve.h, row_sve.cc), and Ssim4x4Row_SME (row_sme.cc).
- RISC-V: Ssim4x4Row_RVV (row_rvv.cc).
- LoongArch: Ssim4x4Row_LSX (row_lsx.cc) and Ssim4x4Row_LASX
(row_lasx.cc).
- WebAssembly: Ssim4x4Row_WASMSIMD (compare_wasm.cc).
- Enable BenchmarkSsim_Opt and add BenchmarkSsim_Unaligned and
TestSsimAnyWidth in unit_test/compare_test.cc.
BenchmarkSsim_Opt (1280x720, single core, us/frame):
uArch Before (C) After (us) Improvement
-------------------------------- ---------- ----------- -----------
Intel Lion Cove (AVX2) 356.33 119.09 2.99x
AMD Zen 6 (AVX512BW) 455.86 124.67 3.66x
AMD Zen 5 (AVX512BW) 518.91 128.29 4.04x
Intel Emerald Rapids (AVX512BW) 601.58 180.41 3.33x
Intel Cougar Cove (AVX2) 608.11 183.34 3.32x
Intel Granite Rapids (AVX512BW) 596.57 187.04 3.19x
Intel Golden Cove (AVX2) 534.58 187.93 2.84x
Intel Sapphire Rapids (AVX512BW) 634.36 189.92 3.34x
AMD Zen 4 (AVX512BW) 713.06 223.65 3.19x
Intel Skymont (SSE2) 445.99 250.52 1.78x
Intel Ice Lake (AVX512BW) 845.16 267.23 3.16x
AMD Zen 3 (AVX2) 741.02 267.38 2.77x
AMD Zen 2 (AVX2) 907.04 301.02 3.01x
Intel Skylake-X (AVX512BW) 1097.86 301.63 3.64x
Intel Darkmont (SSE2) 772.18 383.39 2.01x
Intel Gracemont (SSE2) 961.80 421.83 2.28x
Intel Haswell (AVX2) 1159.74 479.14 2.42x
Intel Ivy Bridge (SSE2) 1296.99 597.32 2.17x
Apple M6 Super (SME) 404.66 90.09 4.49x
Apple M6 Perf (DotProd) 444.72 104.59 4.25x
Qualcomm Oryon-L (DotProd) 490.76 108.79 4.51x
Qualcomm Oryon V3-L (SVE2) 663.13 150.75 4.40x
ARM Cortex-X4 (DotProd) 620.21 177.29 3.50x
ARM Neoverse-V2 (SVE2) 682.93 205.18 3.33x
ARM Cortex-X3 (DotProd) 737.38 226.24 3.26x
ARM C1-Ultra (DotProd) 596.71 229.10 2.60x
ARM Cortex-X2 (DotProd) 820.94 247.94 3.31x
Qualcomm Oryon-M (DotProd) 1146.09 257.89 4.44x
Qualcomm Oryon V3-M (SME) 1068.78 276.23 3.87x
ARM Cortex-A720 (DotProd) 993.59 292.62 3.40x
ARM Cortex-X1 (DotProd) 912.19 326.44 2.79x
ARM Cortex-A715 (DotProd) 1135.67 340.12 3.34x
ARM Cortex-A710 (DotProd) 1222.04 342.51 3.57x
ARM Neoverse-N1 (DotProd) 1406.72 376.46 3.74x
ARM Cortex-A725 (SVE2) 1104.13 376.86 2.93x
ARM C1-Pro (SVE2) 1073.69 413.97 2.59x
ARM Cortex-A76 (DotProd) 1491.02 423.58 3.52x
ARM Cortex-A78 (DotProd) 1503.73 485.00 3.10x
AmpereOne (DotProd) 1518.61 518.40 2.93x
Apple M6 Eff (DotProd) 4051.10 572.93 7.07x
ARM Cortex-A75 (NEON) 2055.04 812.09 2.53x
ARM Cortex-A520 (DotProd) 3307.98 963.86 3.43x
ARM Cortex-A73 (NEON) 3120.09 1180.17 2.64x
Qualcomm Kryo Perf (NEON) 4921.16 1459.77 3.37x
ARM Cortex-A510 (DotProd) 4645.10 1641.06 2.83x
ARM Cortex-A55 (DotProd) 4571.02 1672.09 2.73x
Qualcomm Kryo Eff (NEON) 6000.96 1796.17 3.34x
ARM Cortex-A55r0 (NEON) 10486.61 4358.61 2.41x
ARM Cortex-A53 (NEON) 10121.34 5700.99 1.78x
R=rrwinterton@gmail.com
Bug: None
Test: out/Release/libyuv_unittest --gtest_filter=*Ssim*
Change-Id: Ibb95ae74424ecdda073ab7ab10ab1cf610b121b6
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8593040
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
687 lines
29 KiB
C++
687 lines
29 KiB
C++
/*
|
|
* Copyright 2012 The LibYuv Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "libyuv/compare_row.h"
|
|
#include "libyuv/row.h"
|
|
|
|
// This module is for Visual C 32/64 bit, and for clang with
|
|
// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
|
|
#if !defined(LIBYUV_DISABLE_X86) && \
|
|
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
|
defined(_M_IX86)) && \
|
|
((defined(_MSC_VER) && !defined(__clang__)) || \
|
|
defined(LIBYUV_ENABLE_ROWWIN))
|
|
|
|
#include <immintrin.h> // For SSE2 to AVX512 intrinsics
|
|
#include <string.h> // For memcpy
|
|
#if defined(_MSC_VER) && !defined(__clang__)
|
|
#include <intrin.h>
|
|
#endif
|
|
|
|
#ifdef __cplusplus
|
|
namespace libyuv {
|
|
extern "C" {
|
|
#endif
|
|
|
|
// Target attributes allow gcc and clang to build each function for its
|
|
// instruction set regardless of the baseline.
|
|
#if defined(__clang__) || defined(__GNUC__)
|
|
#define LIBYUV_TARGET_SSE2 __attribute__((target("sse2")))
|
|
#define LIBYUV_TARGET_SSE41 __attribute__((target("sse4.1")))
|
|
#define LIBYUV_TARGET_SSE42 __attribute__((target("sse4.2,popcnt")))
|
|
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
|
|
#define LIBYUV_TARGET_AVX512BW __attribute__((target("avx512bw,avx512f")))
|
|
#else
|
|
#define LIBYUV_TARGET_SSE2
|
|
#define LIBYUV_TARGET_SSE41
|
|
#define LIBYUV_TARGET_SSE42
|
|
#define LIBYUV_TARGET_AVX2
|
|
#define LIBYUV_TARGET_AVX512BW
|
|
#endif // defined(__clang__) || defined(__GNUC__)
|
|
|
|
#if defined(HAS_HAMMINGDISTANCE_SSE42)
|
|
// Count is a multiple of 8.
|
|
LIBYUV_TARGET_SSE42
|
|
uint32_t HammingDistance_SSE42(const uint8_t* src_a,
|
|
const uint8_t* src_b,
|
|
int count) {
|
|
uint32_t diff = 0u;
|
|
int i;
|
|
for (i = 0; i < count; i += 8) {
|
|
uint64_t a;
|
|
uint64_t b;
|
|
memcpy(&a, src_a + i, 8);
|
|
memcpy(&b, src_b + i, 8);
|
|
#if defined(__x86_64__) || defined(_M_X64)
|
|
diff += (uint32_t)_mm_popcnt_u64(a ^ b);
|
|
#else
|
|
diff += _mm_popcnt_u32((uint32_t)(a ^ b)) +
|
|
_mm_popcnt_u32((uint32_t)((a ^ b) >> 32));
|
|
#endif
|
|
}
|
|
return diff;
|
|
}
|
|
#endif // HAS_HAMMINGDISTANCE_SSE42
|
|
|
|
#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
|
|
defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW)
|
|
// Mask of the low n bytes, for n = 0 to 64.
|
|
static __mmask64 LowBytesMask64(int n) {
|
|
return (__mmask64)(n >= 64 ? ~0ULL : (1ULL << n) - 1);
|
|
}
|
|
#endif
|
|
|
|
#if defined(HAS_HAMMINGDISTANCE_AVX2)
|
|
// Count is a multiple of 64.
|
|
LIBYUV_TARGET_AVX2
|
|
uint32_t HammingDistance_AVX2(const uint8_t* src_a,
|
|
const uint8_t* src_b,
|
|
int count) {
|
|
const __m256i kNibbleMask = _mm256_set1_epi8(15);
|
|
const __m256i kBitCount = _mm256_broadcastsi128_si256(
|
|
_mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4));
|
|
const __m256i zero = _mm256_setzero_si256();
|
|
__m256i sum = _mm256_setzero_si256();
|
|
__m128i sum128;
|
|
int i;
|
|
for (i = 0; i < count; i += 64) {
|
|
__m256i x0 =
|
|
_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(src_a + i)),
|
|
_mm256_loadu_si256((const __m256i*)(src_b + i)));
|
|
__m256i x1 =
|
|
_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(src_a + i + 32)),
|
|
_mm256_loadu_si256((const __m256i*)(src_b + i + 32)));
|
|
__m256i c0 = _mm256_add_epi8(
|
|
_mm256_shuffle_epi8(kBitCount, _mm256_and_si256(x0, kNibbleMask)),
|
|
_mm256_shuffle_epi8(
|
|
kBitCount,
|
|
_mm256_and_si256(_mm256_srli_epi16(x0, 4), kNibbleMask)));
|
|
__m256i c1 = _mm256_add_epi8(
|
|
_mm256_shuffle_epi8(kBitCount, _mm256_and_si256(x1, kNibbleMask)),
|
|
_mm256_shuffle_epi8(
|
|
kBitCount,
|
|
_mm256_and_si256(_mm256_srli_epi16(x1, 4), kNibbleMask)));
|
|
sum = _mm256_add_epi64(sum, _mm256_sad_epu8(_mm256_add_epi8(c0, c1), zero));
|
|
}
|
|
sum128 = _mm_add_epi64(_mm256_castsi256_si128(sum),
|
|
_mm256_extracti128_si256(sum, 1));
|
|
sum128 = _mm_add_epi64(sum128, _mm_unpackhi_epi64(sum128, sum128));
|
|
_mm256_zeroupper();
|
|
return (uint32_t)_mm_cvtsi128_si32(sum128);
|
|
}
|
|
#endif // HAS_HAMMINGDISTANCE_AVX2
|
|
|
|
#if defined(HAS_HAMMINGDISTANCE_AVX512BW)
|
|
// Bit count of 64 bytes, summed to 8 qwords.
|
|
LIBYUV_TARGET_AVX512BW
|
|
static __m512i BitCount64_AVX512BW(__m512i x) {
|
|
const __m512i kNibbleMask = _mm512_set1_epi8(15);
|
|
const __m512i kBitCount = _mm512_broadcast_i32x4(
|
|
_mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4));
|
|
__m512i lo = _mm512_shuffle_epi8(kBitCount, _mm512_and_si512(x, kNibbleMask));
|
|
__m512i hi = _mm512_shuffle_epi8(
|
|
kBitCount, _mm512_and_si512(_mm512_srli_epi16(x, 4), kNibbleMask));
|
|
return _mm512_sad_epu8(_mm512_add_epi8(lo, hi), _mm512_setzero_si512());
|
|
}
|
|
|
|
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
|
// loads that read zeros, so any count is supported.
|
|
LIBYUV_TARGET_AVX512BW
|
|
uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
|
|
const uint8_t* src_b,
|
|
int count) {
|
|
__m512i sum = _mm512_setzero_si512();
|
|
while (count >= 64) {
|
|
sum = _mm512_add_epi64(
|
|
sum, BitCount64_AVX512BW(_mm512_xor_si512(_mm512_loadu_si512(src_a),
|
|
_mm512_loadu_si512(src_b))));
|
|
src_a += 64;
|
|
src_b += 64;
|
|
count -= 64;
|
|
}
|
|
if (count > 0) {
|
|
const __mmask64 k = LowBytesMask64(count);
|
|
sum = _mm512_add_epi64(sum, BitCount64_AVX512BW(_mm512_xor_si512(
|
|
_mm512_maskz_loadu_epi8(k, src_a),
|
|
_mm512_maskz_loadu_epi8(k, src_b))));
|
|
}
|
|
return (uint32_t)_mm512_reduce_add_epi64(sum);
|
|
}
|
|
#endif // HAS_HAMMINGDISTANCE_AVX512BW
|
|
|
|
#if defined(HAS_SUMSQUAREERROR_SSE2)
|
|
// Count is a multiple of 16.
|
|
LIBYUV_TARGET_SSE2
|
|
uint32_t SumSquareError_SSE2(const uint8_t* src_a,
|
|
const uint8_t* src_b,
|
|
int count) {
|
|
const __m128i zero = _mm_setzero_si128();
|
|
__m128i sum = _mm_setzero_si128();
|
|
int i;
|
|
for (i = 0; i < count; i += 16) {
|
|
__m128i a = _mm_loadu_si128((const __m128i*)(src_a + i));
|
|
__m128i b = _mm_loadu_si128((const __m128i*)(src_b + i));
|
|
__m128i d = _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a));
|
|
__m128i lo = _mm_unpacklo_epi8(d, zero);
|
|
__m128i hi = _mm_unpackhi_epi8(d, zero);
|
|
sum = _mm_add_epi32(sum, _mm_madd_epi16(lo, lo));
|
|
sum = _mm_add_epi32(sum, _mm_madd_epi16(hi, hi));
|
|
}
|
|
sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0xee));
|
|
sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x01));
|
|
return (uint32_t)_mm_cvtsi128_si32(sum);
|
|
}
|
|
#endif // HAS_SUMSQUAREERROR_SSE2
|
|
|
|
#if defined(HAS_SUMSQUAREERROR_AVX2)
|
|
// Count is a multiple of 32.
|
|
LIBYUV_TARGET_AVX2
|
|
uint32_t SumSquareError_AVX2(const uint8_t* src_a,
|
|
const uint8_t* src_b,
|
|
int count) {
|
|
const __m256i zero = _mm256_setzero_si256();
|
|
__m256i sum = _mm256_setzero_si256();
|
|
__m128i sum128;
|
|
int i;
|
|
for (i = 0; i < count; i += 32) {
|
|
__m256i a = _mm256_loadu_si256((const __m256i*)(src_a + i));
|
|
__m256i b = _mm256_loadu_si256((const __m256i*)(src_b + i));
|
|
__m256i d = _mm256_or_si256(_mm256_subs_epu8(a, b), _mm256_subs_epu8(b, a));
|
|
__m256i lo = _mm256_unpacklo_epi8(d, zero);
|
|
__m256i hi = _mm256_unpackhi_epi8(d, zero);
|
|
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(lo, lo));
|
|
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(hi, hi));
|
|
}
|
|
sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum),
|
|
_mm256_extracti128_si256(sum, 1));
|
|
sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, 0xee));
|
|
sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, 0x01));
|
|
_mm256_zeroupper();
|
|
return (uint32_t)_mm_cvtsi128_si32(sum128);
|
|
}
|
|
#endif // HAS_SUMSQUAREERROR_AVX2
|
|
|
|
#if defined(HAS_SUMSQUAREERROR_AVX512BW)
|
|
// Sum of squared differences of 64 bytes, as 16 dwords.
|
|
LIBYUV_TARGET_AVX512BW
|
|
static __m512i SumSquare64_AVX512BW(__m512i a, __m512i b) {
|
|
const __m512i zero = _mm512_setzero_si512();
|
|
__m512i d = _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
|
|
__m512i lo = _mm512_unpacklo_epi8(d, zero);
|
|
__m512i hi = _mm512_unpackhi_epi8(d, zero);
|
|
return _mm512_add_epi32(_mm512_madd_epi16(lo, lo), _mm512_madd_epi16(hi, hi));
|
|
}
|
|
|
|
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
|
// loads that read zeros, so any count is supported.
|
|
LIBYUV_TARGET_AVX512BW
|
|
uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
|
|
const uint8_t* src_b,
|
|
int count) {
|
|
__m512i sum = _mm512_setzero_si512();
|
|
while (count >= 64) {
|
|
sum =
|
|
_mm512_add_epi32(sum, SumSquare64_AVX512BW(_mm512_loadu_si512(src_a),
|
|
_mm512_loadu_si512(src_b)));
|
|
src_a += 64;
|
|
src_b += 64;
|
|
count -= 64;
|
|
}
|
|
if (count > 0) {
|
|
const __mmask64 k = LowBytesMask64(count);
|
|
sum = _mm512_add_epi32(
|
|
sum, SumSquare64_AVX512BW(_mm512_maskz_loadu_epi8(k, src_a),
|
|
_mm512_maskz_loadu_epi8(k, src_b)));
|
|
}
|
|
return (uint32_t)_mm512_reduce_add_epi32(sum);
|
|
}
|
|
#endif // HAS_SUMSQUAREERROR_AVX512BW
|
|
|
|
#if defined(HAS_HASHDJB2_SSE41)
|
|
// 33 ^ (15 - i) for byte i of 16.
|
|
static const uint32_t kHashMul16[16] = {
|
|
0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, // 33 ^ 15 .. 12
|
|
0x30f35d61, 0x855cb541, 0x040a9121, 0x747c7101, // 33 ^ 11 .. 8
|
|
0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881, // 33 ^ 7 .. 4
|
|
0x00008c61, 0x00000441, 0x00000021, 0x00000001, // 33 ^ 3 .. 0
|
|
};
|
|
static const uint32_t kHash16x33 = 0x92d9e201; // 33 ^ 16
|
|
|
|
// Count is a multiple of 16.
|
|
LIBYUV_TARGET_SSE41
|
|
uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) {
|
|
const __m128i zero = _mm_setzero_si128();
|
|
const __m128i mul0 = _mm_loadu_si128((const __m128i*)(kHashMul16));
|
|
const __m128i mul1 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 4));
|
|
const __m128i mul2 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 8));
|
|
const __m128i mul3 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 12));
|
|
int i;
|
|
for (i = 0; i < count; i += 16) {
|
|
__m128i s = _mm_loadu_si128((const __m128i*)(src + i));
|
|
__m128i lo = _mm_unpacklo_epi8(s, zero);
|
|
__m128i hi = _mm_unpackhi_epi8(s, zero);
|
|
__m128i sum = _mm_add_epi32(
|
|
_mm_add_epi32(_mm_mullo_epi32(_mm_unpacklo_epi16(lo, zero), mul0),
|
|
_mm_mullo_epi32(_mm_unpackhi_epi16(lo, zero), mul1)),
|
|
_mm_add_epi32(_mm_mullo_epi32(_mm_unpacklo_epi16(hi, zero), mul2),
|
|
_mm_mullo_epi32(_mm_unpackhi_epi16(hi, zero), mul3)));
|
|
sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x0e));
|
|
sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x01));
|
|
seed = seed * kHash16x33 + (uint32_t)_mm_cvtsi128_si32(sum);
|
|
}
|
|
return seed;
|
|
}
|
|
#endif // HAS_HASHDJB2_SSE41
|
|
|
|
#if defined(HAS_HASHDJB2_AVX2) || defined(HAS_HASHDJB2_AVX512BW)
|
|
// HashDjb2 is hash = hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)).
|
|
// The 32 bit multipliers 33 ^ (63 - i) are split into 16 bit halves so that
|
|
// vpmaddwd can be used instead of vpmulld, which is 2 uops and 10 cycles.
|
|
// m = hi * 65536 + lo, with lo treated as signed and hi adjusted by the carry,
|
|
// so src * m = src * lo + ((src * hi) << 16) modulo 2 ^ 32.
|
|
// The first 64 are lo and the next 64 are hi.
|
|
static const uint16_t kHashMulLoHi[128] = {
|
|
// lo for 33 ^ 63 .. 33 ^ 0
|
|
0x0be1, 0x93c1, 0x9fa1, 0xaf81, 0x4361, 0xdb41, 0xf721, 0x1701, //
|
|
0xbae1, 0x62c1, 0x8ea1, 0xbe81, 0x7261, 0x2a41, 0x6621, 0xa601, //
|
|
0x69e1, 0x31c1, 0x7da1, 0xcd81, 0xa161, 0x7941, 0xd521, 0x3501, //
|
|
0x18e1, 0x00c1, 0x6ca1, 0xdc81, 0xd061, 0xc841, 0x4421, 0xc401, //
|
|
0xc7e1, 0xcfc1, 0x5ba1, 0xeb81, 0xff61, 0x1741, 0xb321, 0x5301, //
|
|
0x76e1, 0x9ec1, 0x4aa1, 0xfa81, 0x2e61, 0x6641, 0x2221, 0xe201, //
|
|
0x25e1, 0x6dc1, 0x39a1, 0x0981, 0x5d61, 0xb541, 0x9121, 0x7101, //
|
|
0xd4e1, 0x3cc1, 0x28a1, 0x1881, 0x8c61, 0x0441, 0x0021, 0x0001, //
|
|
// hi for 33 ^ 63 .. 33 ^ 0
|
|
0xd8be, 0xef4c, 0x6458, 0x6022, 0x40f9, 0x193e, 0x6d5f, 0x31dc, //
|
|
0x0945, 0xc238, 0x1567, 0xa38f, 0x3380, 0x301b, 0x664e, 0x031a, //
|
|
0xd18c, 0xb8c6, 0x79f6, 0xae5d, 0x0549, 0xc219, 0x7a3f, 0x1338, //
|
|
0x0095, 0xc1f5, 0xa107, 0xaf8c, 0x0552, 0x3e39, 0x3830, 0x1138, //
|
|
0x655f, 0xccc5, 0x9999, 0xd61c, 0x829c, 0x1379, 0x2f23, 0xac18, //
|
|
0xcee9, 0xc836, 0x72ac, 0x510d, 0xcc27, 0xb0da, 0xee16, 0x92da, //
|
|
0x0c35, 0xa347, 0x3b40, 0x4f5f, 0x30f3, 0x855d, 0x040b, 0x747c, //
|
|
0xec42, 0x4cfa, 0x0255, 0x0012, 0x0001, 0x0000, 0x0000, 0x0000, //
|
|
};
|
|
#endif
|
|
|
|
#if defined(HAS_HASHDJB2_AVX2)
|
|
// lo + (hi << 16), then add 8 dwords.
|
|
LIBYUV_TARGET_AVX2
|
|
static uint32_t HashSum_AVX2(__m256i lo, __m256i hi) {
|
|
__m256i s = _mm256_add_epi32(lo, _mm256_slli_epi32(hi, 16));
|
|
__m128i s128 =
|
|
_mm_add_epi32(_mm256_castsi256_si128(s), _mm256_extracti128_si256(s, 1));
|
|
s128 = _mm_add_epi32(s128, _mm_shuffle_epi32(s128, 0x0e));
|
|
s128 = _mm_add_epi32(s128, _mm_shuffle_epi32(s128, 0x01));
|
|
return (uint32_t)_mm_cvtsi128_si32(s128);
|
|
}
|
|
|
|
// Count is a multiple of 32. Process 64 bytes per loop with one horizontal
|
|
// add, then 32 bytes if needed.
|
|
LIBYUV_TARGET_AVX2
|
|
uint32_t HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed) {
|
|
const __m256i* m = (const __m256i*)kHashMulLoHi;
|
|
while (count >= 64) {
|
|
__m256i w0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src));
|
|
__m256i w1 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16)));
|
|
__m256i w2 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 32)));
|
|
__m256i w3 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 48)));
|
|
__m256i lo = _mm256_add_epi32(
|
|
_mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 0)),
|
|
_mm256_madd_epi16(w1, _mm256_loadu_si256(m + 1))),
|
|
_mm256_add_epi32(_mm256_madd_epi16(w2, _mm256_loadu_si256(m + 2)),
|
|
_mm256_madd_epi16(w3, _mm256_loadu_si256(m + 3))));
|
|
__m256i hi = _mm256_add_epi32(
|
|
_mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 4)),
|
|
_mm256_madd_epi16(w1, _mm256_loadu_si256(m + 5))),
|
|
_mm256_add_epi32(_mm256_madd_epi16(w2, _mm256_loadu_si256(m + 6)),
|
|
_mm256_madd_epi16(w3, _mm256_loadu_si256(m + 7))));
|
|
seed = seed * 0xf07f8801u + HashSum_AVX2(lo, hi); // 33 ^ 64
|
|
src += 64;
|
|
count -= 64;
|
|
}
|
|
if (count > 0) {
|
|
__m256i w0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src));
|
|
__m256i w1 =
|
|
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16)));
|
|
__m256i lo =
|
|
_mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 2)),
|
|
_mm256_madd_epi16(w1, _mm256_loadu_si256(m + 3)));
|
|
__m256i hi =
|
|
_mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 6)),
|
|
_mm256_madd_epi16(w1, _mm256_loadu_si256(m + 7)));
|
|
seed = seed * 0x1137c401u + HashSum_AVX2(lo, hi); // 33 ^ 32
|
|
}
|
|
_mm256_zeroupper();
|
|
return seed;
|
|
}
|
|
#endif // HAS_HASHDJB2_AVX2
|
|
|
|
#if defined(HAS_HASHDJB2_AVX512BW)
|
|
static uint32_t Pow33(int n) {
|
|
uint32_t r = 1u;
|
|
uint32_t b = 33u;
|
|
while (n) {
|
|
if (n & 1) {
|
|
r *= b;
|
|
}
|
|
b *= b;
|
|
n >>= 1;
|
|
}
|
|
return r;
|
|
}
|
|
|
|
// Sum of 64 bytes, zero extended to words in w0 and w1, times 33 ^ (63 - i).
|
|
LIBYUV_TARGET_AVX512BW
|
|
static uint32_t HashSum64_AVX512BW(__m512i w0, __m512i w1) {
|
|
const __m512i* m = (const __m512i*)kHashMulLoHi;
|
|
__m512i lo =
|
|
_mm512_add_epi32(_mm512_madd_epi16(w0, _mm512_loadu_si512(m)),
|
|
_mm512_madd_epi16(w1, _mm512_loadu_si512(m + 1)));
|
|
__m512i hi =
|
|
_mm512_add_epi32(_mm512_madd_epi16(w0, _mm512_loadu_si512(m + 2)),
|
|
_mm512_madd_epi16(w1, _mm512_loadu_si512(m + 3)));
|
|
return (uint32_t)_mm512_reduce_add_epi32(
|
|
_mm512_add_epi32(lo, _mm512_slli_epi32(hi, 16)));
|
|
}
|
|
|
|
// hash = hash * 33 ^ 64 + sum(src[i] * 33 ^ (63 - i)) for each 64 bytes.
|
|
// The remainder of n = 1 to 63 bytes is loaded into the high n bytes with a
|
|
// masked load ending at src + count, so the same multipliers apply, and
|
|
// hash is multiplied by 33 ^ n.
|
|
LIBYUV_TARGET_AVX512BW
|
|
uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) {
|
|
int n;
|
|
while (count >= 64) {
|
|
seed = seed * 0xf07f8801u + // 33 ^ 64
|
|
HashSum64_AVX512BW(
|
|
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src)),
|
|
_mm512_cvtepu8_epi16(
|
|
_mm256_loadu_si256((const __m256i*)(src + 32))));
|
|
src += 64;
|
|
count -= 64;
|
|
}
|
|
n = count;
|
|
if (n) {
|
|
const __mmask64 k = (__mmask64)~LowBytesMask64(64 - n);
|
|
__m512i s = _mm512_maskz_loadu_epi8(k, src + n - 64);
|
|
seed = seed * Pow33(n) +
|
|
HashSum64_AVX512BW(
|
|
_mm512_cvtepu8_epi16(_mm512_castsi512_si256(s)),
|
|
_mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(s, 1)));
|
|
}
|
|
return seed;
|
|
}
|
|
#endif // HAS_HASHDJB2_AVX512BW
|
|
|
|
#if defined(HAS_SSIM4X4ROW_SSE2)
|
|
// Width is a multiple of 16.
|
|
LIBYUV_TARGET_SSE2
|
|
void Ssim4x4Row_SSE2(const uint8_t* src_a,
|
|
int stride_a,
|
|
const uint8_t* src_b,
|
|
int stride_b,
|
|
uint32_t* dst,
|
|
int width) {
|
|
const ptrdiff_t sa = stride_a;
|
|
const ptrdiff_t sb = stride_b;
|
|
const uint8_t* a0 = src_a;
|
|
const uint8_t* a1 = a0 + sa;
|
|
const uint8_t* a2 = a1 + sa;
|
|
const uint8_t* a3 = a2 + sa;
|
|
const uint8_t* b0 = src_b;
|
|
const uint8_t* b1 = b0 + sb;
|
|
const uint8_t* b2 = b1 + sb;
|
|
const uint8_t* b3 = b2 + sb;
|
|
const __m128i kMaskFF = _mm_set1_epi16(0x00ff);
|
|
const __m128i kOnes16 = _mm_set1_epi16(1);
|
|
int x;
|
|
for (x = 0; x < width; x += 16) {
|
|
__m128i acc_a = _mm_setzero_si128();
|
|
__m128i acc_b = _mm_setzero_si128();
|
|
__m128i acc_sq = _mm_setzero_si128();
|
|
__m128i acc_axb = _mm_setzero_si128();
|
|
#define SSIM_ROW_SSE2_WIN(ar, br) \
|
|
do { \
|
|
__m128i va = _mm_loadu_si128((const __m128i*)((ar) + x)); \
|
|
__m128i vb = _mm_loadu_si128((const __m128i*)((br) + x)); \
|
|
__m128i a_ev = _mm_and_si128(va, kMaskFF); \
|
|
__m128i a_od = _mm_srli_epi16(va, 8); \
|
|
__m128i b_ev = _mm_and_si128(vb, kMaskFF); \
|
|
__m128i b_od = _mm_srli_epi16(vb, 8); \
|
|
acc_a = _mm_add_epi16(acc_a, _mm_add_epi16(a_ev, a_od)); \
|
|
acc_b = _mm_add_epi16(acc_b, _mm_add_epi16(b_ev, b_od)); \
|
|
acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(a_ev, a_ev), \
|
|
_mm_madd_epi16(a_od, a_od))); \
|
|
acc_sq = _mm_add_epi32(acc_sq, _mm_add_epi32(_mm_madd_epi16(b_ev, b_ev), \
|
|
_mm_madd_epi16(b_od, b_od))); \
|
|
acc_axb = \
|
|
_mm_add_epi32(acc_axb, _mm_add_epi32(_mm_madd_epi16(a_ev, b_ev), \
|
|
_mm_madd_epi16(a_od, b_od))); \
|
|
} while (0)
|
|
SSIM_ROW_SSE2_WIN(a0, b0);
|
|
SSIM_ROW_SSE2_WIN(a1, b1);
|
|
SSIM_ROW_SSE2_WIN(a2, b2);
|
|
SSIM_ROW_SSE2_WIN(a3, b3);
|
|
#undef SSIM_ROW_SSE2_WIN
|
|
acc_a = _mm_madd_epi16(acc_a, kOnes16);
|
|
acc_b = _mm_madd_epi16(acc_b, kOnes16);
|
|
{
|
|
__m128i t0 = _mm_unpacklo_epi32(acc_a, acc_b);
|
|
__m128i t1 = _mm_unpackhi_epi32(acc_a, acc_b);
|
|
__m128i t2 = _mm_unpacklo_epi32(acc_sq, acc_axb);
|
|
__m128i t3 = _mm_unpackhi_epi32(acc_sq, acc_axb);
|
|
_mm_storeu_si128((__m128i*)(dst + 0), _mm_unpacklo_epi64(t0, t2));
|
|
_mm_storeu_si128((__m128i*)(dst + 4), _mm_unpackhi_epi64(t0, t2));
|
|
_mm_storeu_si128((__m128i*)(dst + 8), _mm_unpacklo_epi64(t1, t3));
|
|
_mm_storeu_si128((__m128i*)(dst + 12), _mm_unpackhi_epi64(t1, t3));
|
|
}
|
|
dst += 16;
|
|
}
|
|
}
|
|
#endif // HAS_SSIM4X4ROW_SSE2
|
|
|
|
#if defined(HAS_SSIM4X4ROW_AVX2)
|
|
// Width is a multiple of 32.
|
|
LIBYUV_TARGET_AVX2
|
|
void Ssim4x4Row_AVX2(const uint8_t* src_a,
|
|
int stride_a,
|
|
const uint8_t* src_b,
|
|
int stride_b,
|
|
uint32_t* dst,
|
|
int width) {
|
|
const ptrdiff_t sa = stride_a;
|
|
const ptrdiff_t sb = stride_b;
|
|
const uint8_t* a0 = src_a;
|
|
const uint8_t* a1 = a0 + sa;
|
|
const uint8_t* a2 = a1 + sa;
|
|
const uint8_t* a3 = a2 + sa;
|
|
const uint8_t* b0 = src_b;
|
|
const uint8_t* b1 = b0 + sb;
|
|
const uint8_t* b2 = b1 + sb;
|
|
const uint8_t* b3 = b2 + sb;
|
|
const __m256i kMaskFF = _mm256_set1_epi16(0x00ff);
|
|
const __m256i kOnes16 = _mm256_set1_epi16(1);
|
|
int x;
|
|
for (x = 0; x < width; x += 32) {
|
|
__m256i acc_a = _mm256_setzero_si256();
|
|
__m256i acc_b = _mm256_setzero_si256();
|
|
__m256i acc_sq = _mm256_setzero_si256();
|
|
__m256i acc_axb = _mm256_setzero_si256();
|
|
#define SSIM_ROW_AVX2_WIN(ar, br) \
|
|
do { \
|
|
__m256i va = _mm256_loadu_si256((const __m256i*)((ar) + x)); \
|
|
__m256i vb = _mm256_loadu_si256((const __m256i*)((br) + x)); \
|
|
__m256i a_ev = _mm256_and_si256(va, kMaskFF); \
|
|
__m256i a_od = _mm256_srli_epi16(va, 8); \
|
|
__m256i b_ev = _mm256_and_si256(vb, kMaskFF); \
|
|
__m256i b_od = _mm256_srli_epi16(vb, 8); \
|
|
acc_a = _mm256_add_epi16(acc_a, _mm256_add_epi16(a_ev, a_od)); \
|
|
acc_b = _mm256_add_epi16(acc_b, _mm256_add_epi16(b_ev, b_od)); \
|
|
acc_sq = _mm256_add_epi32( \
|
|
acc_sq, _mm256_add_epi32(_mm256_madd_epi16(a_ev, a_ev), \
|
|
_mm256_madd_epi16(a_od, a_od))); \
|
|
acc_sq = _mm256_add_epi32( \
|
|
acc_sq, _mm256_add_epi32(_mm256_madd_epi16(b_ev, b_ev), \
|
|
_mm256_madd_epi16(b_od, b_od))); \
|
|
acc_axb = _mm256_add_epi32( \
|
|
acc_axb, _mm256_add_epi32(_mm256_madd_epi16(a_ev, b_ev), \
|
|
_mm256_madd_epi16(a_od, b_od))); \
|
|
} while (0)
|
|
SSIM_ROW_AVX2_WIN(a0, b0);
|
|
SSIM_ROW_AVX2_WIN(a1, b1);
|
|
SSIM_ROW_AVX2_WIN(a2, b2);
|
|
SSIM_ROW_AVX2_WIN(a3, b3);
|
|
#undef SSIM_ROW_AVX2_WIN
|
|
acc_a = _mm256_madd_epi16(acc_a, kOnes16);
|
|
acc_b = _mm256_madd_epi16(acc_b, kOnes16);
|
|
{
|
|
__m256i t0 = _mm256_unpacklo_epi32(acc_a, acc_b);
|
|
__m256i t1 = _mm256_unpackhi_epi32(acc_a, acc_b);
|
|
__m256i t2 = _mm256_unpacklo_epi32(acc_sq, acc_axb);
|
|
__m256i t3 = _mm256_unpackhi_epi32(acc_sq, acc_axb);
|
|
__m256i u0 = _mm256_unpacklo_epi64(t0, t2);
|
|
__m256i u1 = _mm256_unpackhi_epi64(t0, t2);
|
|
__m256i u2 = _mm256_unpacklo_epi64(t1, t3);
|
|
__m256i u3 = _mm256_unpackhi_epi64(t1, t3);
|
|
_mm256_storeu_si256((__m256i*)(dst + 0),
|
|
_mm256_permute2x128_si256(u0, u1, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst + 8),
|
|
_mm256_permute2x128_si256(u2, u3, 0x20));
|
|
_mm256_storeu_si256((__m256i*)(dst + 16),
|
|
_mm256_permute2x128_si256(u0, u1, 0x31));
|
|
_mm256_storeu_si256((__m256i*)(dst + 24),
|
|
_mm256_permute2x128_si256(u2, u3, 0x31));
|
|
}
|
|
dst += 32;
|
|
}
|
|
_mm256_zeroupper();
|
|
}
|
|
#endif // HAS_SSIM4X4ROW_AVX2
|
|
|
|
#if defined(HAS_SSIM4X4ROW_AVX512BW)
|
|
// Width is a multiple of 4.
|
|
LIBYUV_TARGET_AVX512BW
|
|
void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
|
|
int stride_a,
|
|
const uint8_t* src_b,
|
|
int stride_b,
|
|
uint32_t* dst,
|
|
int width) {
|
|
const ptrdiff_t sa = stride_a;
|
|
const ptrdiff_t sb = stride_b;
|
|
const uint8_t* a0 = src_a;
|
|
const uint8_t* a1 = a0 + sa;
|
|
const uint8_t* a2 = a1 + sa;
|
|
const uint8_t* a3 = a2 + sa;
|
|
const uint8_t* b0 = src_b;
|
|
const uint8_t* b1 = b0 + sb;
|
|
const uint8_t* b2 = b1 + sb;
|
|
const uint8_t* b3 = b2 + sb;
|
|
const __m512i kMaskFF = _mm512_set1_epi16(0x00ff);
|
|
const __m512i kOnes16 = _mm512_set1_epi16(1);
|
|
#define SSIM_ROW_AVX512_WIN(va, vb) \
|
|
do { \
|
|
__m512i a_ev = _mm512_and_si512(va, kMaskFF); \
|
|
__m512i a_od = _mm512_srli_epi16(va, 8); \
|
|
__m512i b_ev = _mm512_and_si512(vb, kMaskFF); \
|
|
__m512i b_od = _mm512_srli_epi16(vb, 8); \
|
|
acc_a = _mm512_add_epi16(acc_a, _mm512_add_epi16(a_ev, a_od)); \
|
|
acc_b = _mm512_add_epi16(acc_b, _mm512_add_epi16(b_ev, b_od)); \
|
|
acc_sq = _mm512_add_epi32( \
|
|
acc_sq, _mm512_add_epi32(_mm512_madd_epi16(a_ev, a_ev), \
|
|
_mm512_madd_epi16(a_od, a_od))); \
|
|
acc_sq = _mm512_add_epi32( \
|
|
acc_sq, _mm512_add_epi32(_mm512_madd_epi16(b_ev, b_ev), \
|
|
_mm512_madd_epi16(b_od, b_od))); \
|
|
acc_axb = _mm512_add_epi32( \
|
|
acc_axb, _mm512_add_epi32(_mm512_madd_epi16(a_ev, b_ev), \
|
|
_mm512_madd_epi16(a_od, b_od))); \
|
|
} while (0)
|
|
#define SSIM_TRANSPOSE_AVX512_WIN() \
|
|
acc_a = _mm512_madd_epi16(acc_a, kOnes16); \
|
|
acc_b = _mm512_madd_epi16(acc_b, kOnes16); \
|
|
__m512i t0 = _mm512_unpacklo_epi32(acc_a, acc_b); \
|
|
__m512i t1 = _mm512_unpackhi_epi32(acc_a, acc_b); \
|
|
__m512i t2 = _mm512_unpacklo_epi32(acc_sq, acc_axb); \
|
|
__m512i t3 = _mm512_unpackhi_epi32(acc_sq, acc_axb); \
|
|
__m512i u0 = _mm512_unpacklo_epi64(t0, t2); \
|
|
__m512i u1 = _mm512_unpackhi_epi64(t0, t2); \
|
|
__m512i u2 = _mm512_unpacklo_epi64(t1, t3); \
|
|
__m512i u3 = _mm512_unpackhi_epi64(t1, t3); \
|
|
__m512i v0 = _mm512_shuffle_i32x4(u0, u1, 0x44); \
|
|
__m512i v1 = _mm512_shuffle_i32x4(u2, u3, 0x44); \
|
|
__m512i v2 = _mm512_shuffle_i32x4(u0, u1, 0xee); \
|
|
__m512i v3 = _mm512_shuffle_i32x4(u2, u3, 0xee); \
|
|
__m512i out0 = _mm512_shuffle_i32x4(v0, v1, 0x88); \
|
|
__m512i out1 = _mm512_shuffle_i32x4(v0, v1, 0xdd); \
|
|
__m512i out2 = _mm512_shuffle_i32x4(v2, v3, 0x88); \
|
|
__m512i out3 = _mm512_shuffle_i32x4(v2, v3, 0xdd)
|
|
|
|
while (width >= 64) {
|
|
__m512i acc_a = _mm512_setzero_si512();
|
|
__m512i acc_b = _mm512_setzero_si512();
|
|
__m512i acc_sq = _mm512_setzero_si512();
|
|
__m512i acc_axb = _mm512_setzero_si512();
|
|
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a0), _mm512_loadu_si512(b0));
|
|
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a1), _mm512_loadu_si512(b1));
|
|
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a2), _mm512_loadu_si512(b2));
|
|
SSIM_ROW_AVX512_WIN(_mm512_loadu_si512(a3), _mm512_loadu_si512(b3));
|
|
SSIM_TRANSPOSE_AVX512_WIN();
|
|
_mm512_storeu_si512(dst + 0, out0);
|
|
_mm512_storeu_si512(dst + 16, out1);
|
|
_mm512_storeu_si512(dst + 32, out2);
|
|
_mm512_storeu_si512(dst + 48, out3);
|
|
a0 += 64;
|
|
a1 += 64;
|
|
a2 += 64;
|
|
a3 += 64;
|
|
b0 += 64;
|
|
b1 += 64;
|
|
b2 += 64;
|
|
b3 += 64;
|
|
dst += 64;
|
|
width -= 64;
|
|
}
|
|
if (width > 0) {
|
|
const __mmask16 load_mask = (__mmask16)((1U << (width >> 2)) - 1U);
|
|
const uint64_t store_mask = LowBytesMask64(width);
|
|
__m512i acc_a = _mm512_setzero_si512();
|
|
__m512i acc_b = _mm512_setzero_si512();
|
|
__m512i acc_sq = _mm512_setzero_si512();
|
|
__m512i acc_axb = _mm512_setzero_si512();
|
|
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a0),
|
|
_mm512_maskz_loadu_epi32(load_mask, b0));
|
|
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a1),
|
|
_mm512_maskz_loadu_epi32(load_mask, b1));
|
|
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a2),
|
|
_mm512_maskz_loadu_epi32(load_mask, b2));
|
|
SSIM_ROW_AVX512_WIN(_mm512_maskz_loadu_epi32(load_mask, a3),
|
|
_mm512_maskz_loadu_epi32(load_mask, b3));
|
|
SSIM_TRANSPOSE_AVX512_WIN();
|
|
_mm512_mask_storeu_epi32(dst + 0, (__mmask16)store_mask, out0);
|
|
_mm512_mask_storeu_epi32(dst + 16, (__mmask16)(store_mask >> 16), out1);
|
|
_mm512_mask_storeu_epi32(dst + 32, (__mmask16)(store_mask >> 32), out2);
|
|
_mm512_mask_storeu_epi32(dst + 48, (__mmask16)(store_mask >> 48), out3);
|
|
}
|
|
#undef SSIM_ROW_AVX512_WIN
|
|
#undef SSIM_TRANSPOSE_AVX512_WIN
|
|
}
|
|
#endif // HAS_SSIM4X4ROW_AVX512BW
|
|
|
|
#ifdef __cplusplus
|
|
} // extern "C"
|
|
} // namespace libyuv
|
|
#endif
|
|
|
|
#endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) &&
|
|
// (Visual C or LIBYUV_ENABLE_ROWWIN)
|