mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-11-09 14:49:57 +08:00
[libyuv] Add 100% SIMD coverage across all compare functions
Implement SIMD kernels across all 6 compare row functions
(ARGBDetectRow, HammingDistance, HashDjb2, Ssim4x4Row, SsimWindowRow,
and SumSquareError) so that ARM (32-bit and 64-bit NEON), x86 (AVX2 and
AVX-512), RISC-V (RVV), Hexagon (HVX), WebAssembly (WASMSIMD and
RelaxedSIMD), and LoongArch (LSX and LASX) each reach 6 / 6 (100%)
coverage:
* HammingDistance:
- Add HammingDistance_AVX512VNNI in compare_gcc.cc and compare_win.cc
using vpshufb nibble popcount lookup tables accumulated into 32-bit
counters via vpdpbusd with k1 masked tail loads (1.39x faster than
HammingDistance_AVX512BW).
- Update HammingDistance_WASMSIMD in compare_wasm.cc to accumulate
per-byte popcounts into 32-bit counters using
wasm_i32x4_relaxed_dot_i8x16_i7x16_add when __wasm_relaxed_simd__ is
enabled.
- Add HammingDistance_RVV, HammingDistance_LSX, and
HammingDistance_LASX.
* SumSquareError:
- Add SumSquareError_RVV (vwsubu.vv + vwmacc.vv), SumSquareError_LSX,
and SumSquareError_LASX (xvabsd.bu + xvmulwev/wod.h.bu).
* HashDjb2:
- Add 32-bit ARM HashDjb2_NEON in compare_neon.cc, HashDjb2_RVV (with
arbitrary tail length support), HashDjb2_LSX, and HashDjb2_LASX.
* SsimWindowRow:
- Add SsimWindowRow_NEON (32-bit and 64-bit), SsimWindowRow_WASMSIMD,
SsimWindowRow_HVX, SsimWindowRow_LSX, and SsimWindowRow_LASX.
* ARGBDetectRow:
- Add ARGBDetectRow_NEON (32-bit and 64-bit), ARGBDetectRow_RVV,
ARGBDetectRow_WASMSIMD, ARGBDetectRow_HVX, ARGBDetectRow_LSX, and
ARGBDetectRow_LASX, and add 1..7 pixel SIMD tail handling to
ARGBDetectRow_AVX2 in compare_win.cc.
* Remainder & Test Coverage:
- Handle tail counts/widths in HashDjb2, ComputeHammingDistance,
ComputeSumSquareError, and CalcFrameSsim via padded SIMD buffers
when a SIMD kernel is active, using zero-numerator padding in
SsimWindowRow_Rem to avoid float cancellation.
- Add TestARGBDetectAnyWidth, TestCompareAnyCount, and
TestSsimAnyWidth in compare_test.cc.
Compare SIMD ISA per *_C:
------------------------------------------------------------------------
C Function arm x86 riscv hexagon wasm loong
------------------------------------------------------------------------
ARGBDetectRow_C NEON AVX2 RVV HVX SIMD LASX
HammingDistance_C DOTPROD AVX512VNNI RVV HVX RELAXED LASX
HashDjb2_C NEON AVX512BW RVV HVX SIMD LASX
Ssim4x4Row_C SME AVX512BW RVV HVX SIMD LASX
SsimWindowRow_C SME AVX512BW RVV HVX SIMD LASX
SumSquareError_C DOTPROD AVX512VNNI RVV HVX SIMD LASX
------------------------------------------------------------------------
SIMD (6 compare) 6 100% 6 100% 6 100% 6 100% 6 100% 6 100%
------------------------------------------------------------------------
Test: libyuv_unittest --gtest_filter=*Compare*
Bug: none
Change-Id: I72a026ca5d9b3da373e32c01396dddba8c494089
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8692272
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard
parent
59fb9e18a8
commit
bb310b4e7e
@ -42,6 +42,7 @@ extern "C" {
|
||||
#define HAS_ARGBDETECTROW_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define HAS_HAMMINGDISTANCE_AVX512BW
|
||||
#define HAS_HAMMINGDISTANCE_AVX512VNNI
|
||||
#define HAS_SUMSQUAREERROR_AVX512BW
|
||||
#define HAS_SUMSQUAREERROR_AVX512VNNI
|
||||
#define HAS_HASHDJB2_AVX512BW
|
||||
@ -74,6 +75,7 @@ extern "C" {
|
||||
#endif
|
||||
#if defined(__x86_64__) && defined(CLANG_HAS_AVX512)
|
||||
#define HAS_HAMMINGDISTANCE_AVX512BW
|
||||
#define HAS_HAMMINGDISTANCE_AVX512VNNI
|
||||
#define HAS_SUMSQUAREERROR_AVX512BW
|
||||
#define HAS_SUMSQUAREERROR_AVX512VNNI
|
||||
#define HAS_HASHDJB2_AVX512BW
|
||||
@ -87,13 +89,14 @@ extern "C" {
|
||||
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
|
||||
#define HAS_HAMMINGDISTANCE_NEON
|
||||
#define HAS_SUMSQUAREERROR_NEON
|
||||
#define HAS_HASHDJB2_NEON
|
||||
#define HAS_SSIM4X4ROW_NEON
|
||||
#define HAS_SSIMWINDOWROW_NEON
|
||||
#define HAS_ARGBDETECTROW_NEON
|
||||
#endif
|
||||
|
||||
// The following are available for AArch64 Neon:
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
#define HAS_HASHDJB2_NEON
|
||||
|
||||
#define HAS_HAMMINGDISTANCE_NEON_DOTPROD
|
||||
#define HAS_SUMSQUAREERROR_NEON_DOTPROD
|
||||
#define HAS_SSIM4X4ROW_NEON_DOTPROD
|
||||
@ -115,20 +118,34 @@ extern "C" {
|
||||
// The following are available for RISC-V Vector (RVV):
|
||||
#if !defined(LIBYUV_DISABLE_RVV) && \
|
||||
(defined(__riscv_vector) || defined(LIBYUV_RVV))
|
||||
#define HAS_HAMMINGDISTANCE_RVV
|
||||
#define HAS_SUMSQUAREERROR_RVV
|
||||
#define HAS_HASHDJB2_RVV
|
||||
#define HAS_SSIM4X4ROW_RVV
|
||||
#if defined(__riscv_f) || defined(__riscv_zve32f)
|
||||
#define HAS_SSIMWINDOWROW_RVV
|
||||
#endif
|
||||
#define HAS_ARGBDETECTROW_RVV
|
||||
#endif
|
||||
|
||||
// The following are available for LoongArch LSX:
|
||||
#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
|
||||
#define HAS_HAMMINGDISTANCE_LSX
|
||||
#define HAS_SUMSQUAREERROR_LSX
|
||||
#define HAS_HASHDJB2_LSX
|
||||
#define HAS_SSIM4X4ROW_LSX
|
||||
#define HAS_SSIMWINDOWROW_LSX
|
||||
#define HAS_ARGBDETECTROW_LSX
|
||||
#endif
|
||||
|
||||
// The following are available for LoongArch LASX:
|
||||
#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
|
||||
#define HAS_HAMMINGDISTANCE_LASX
|
||||
#define HAS_SUMSQUAREERROR_LASX
|
||||
#define HAS_HASHDJB2_LASX
|
||||
#define HAS_SSIM4X4ROW_LASX
|
||||
#define HAS_SSIMWINDOWROW_LASX
|
||||
#define HAS_ARGBDETECTROW_LASX
|
||||
#endif
|
||||
|
||||
// The following are available for WebAssembly SIMD128:
|
||||
@ -137,6 +154,8 @@ extern "C" {
|
||||
#define HAS_SUMSQUAREERROR_WASMSIMD
|
||||
#define HAS_HASHDJB2_WASMSIMD
|
||||
#define HAS_SSIM4X4ROW_WASMSIMD
|
||||
#define HAS_SSIMWINDOWROW_WASMSIMD
|
||||
#define HAS_ARGBDETECTROW_WASMSIMD
|
||||
#endif
|
||||
|
||||
// The following are available for Hexagon HVX:
|
||||
@ -145,6 +164,8 @@ extern "C" {
|
||||
#define HAS_SUMSQUAREERROR_HVX
|
||||
#define HAS_HASHDJB2_HVX
|
||||
#define HAS_SSIM4X4ROW_HVX
|
||||
#define HAS_SSIMWINDOWROW_HVX
|
||||
#define HAS_ARGBDETECTROW_HVX
|
||||
#endif
|
||||
|
||||
uint32_t HammingDistance_C(const uint8_t* src_a,
|
||||
@ -162,12 +183,24 @@ uint32_t HammingDistance_AVX2(const uint8_t* src_a,
|
||||
uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t HammingDistance_AVX512VNNI(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t HammingDistance_NEON(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t HammingDistance_NEON_DotProd(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t HammingDistance_RVV(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t HammingDistance_LSX(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t HammingDistance_LASX(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
@ -195,6 +228,15 @@ uint32_t SumSquareError_NEON(const uint8_t* src_a,
|
||||
uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t SumSquareError_RVV(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t SumSquareError_LSX(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t SumSquareError_LASX(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t SumSquareError_WASMSIMD(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
@ -207,6 +249,9 @@ uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed);
|
||||
uint32_t HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed);
|
||||
uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed);
|
||||
uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed);
|
||||
uint32_t HashDjb2_RVV(const uint8_t* src, int count, uint32_t seed);
|
||||
uint32_t HashDjb2_LSX(const uint8_t* src, int count, uint32_t seed);
|
||||
uint32_t HashDjb2_LASX(const uint8_t* src, int count, uint32_t seed);
|
||||
uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed);
|
||||
uint32_t HashDjb2_HVX(const uint8_t* src, int count, uint32_t seed);
|
||||
|
||||
@ -300,6 +345,9 @@ float SsimWindowRow_AVX2(const uint32_t* row0,
|
||||
float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows);
|
||||
float SsimWindowRow_NEON(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows);
|
||||
float SsimWindowRow_SVE2(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows);
|
||||
@ -309,9 +357,27 @@ float SsimWindowRow_SME(const uint32_t* row0,
|
||||
float SsimWindowRow_RVV(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows);
|
||||
float SsimWindowRow_LSX(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows);
|
||||
float SsimWindowRow_LASX(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows);
|
||||
float SsimWindowRow_WASMSIMD(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows);
|
||||
float SsimWindowRow_HVX(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows);
|
||||
|
||||
uint32_t ARGBDetectRow_C(const uint8_t* argb, int width);
|
||||
uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width);
|
||||
uint32_t ARGBDetectRow_NEON(const uint8_t* argb, int width);
|
||||
uint32_t ARGBDetectRow_RVV(const uint8_t* argb, int width);
|
||||
uint32_t ARGBDetectRow_LSX(const uint8_t* argb, int width);
|
||||
uint32_t ARGBDetectRow_LASX(const uint8_t* argb, int width);
|
||||
uint32_t ARGBDetectRow_WASMSIMD(const uint8_t* argb, int width);
|
||||
uint32_t ARGBDetectRow_HVX(const uint8_t* argb, int width);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
#include <float.h>
|
||||
#include <limits.h>
|
||||
#include <math.h>
|
||||
#include <string.h>
|
||||
#ifdef _OPENMP
|
||||
#include <omp.h>
|
||||
#endif
|
||||
@ -57,6 +58,22 @@ uint32_t HashDjb2(const uint8_t* src, uint64_t count, uint32_t seed) {
|
||||
simd_size = 1; // Any count.
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HASHDJB2_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
HashDjb2_SSE = HashDjb2_RVV;
|
||||
simd_size = 1; // Any count.
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HASHDJB2_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
HashDjb2_SSE = HashDjb2_LSX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HASHDJB2_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
HashDjb2_SSE = HashDjb2_LASX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HASHDJB2_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
HashDjb2_SSE = HashDjb2_WASMSIMD;
|
||||
@ -82,7 +99,19 @@ uint32_t HashDjb2(const uint8_t* src, uint64_t count, uint32_t seed) {
|
||||
}
|
||||
remainder = (int)count & (simd_size - 1);
|
||||
if (remainder) {
|
||||
seed = HashDjb2_C(src, remainder, seed);
|
||||
if (HashDjb2_SSE == HashDjb2_C) {
|
||||
seed = HashDjb2_C(src, remainder, seed);
|
||||
} else {
|
||||
SIMD_ALIGNED(uint8_t buf[32]);
|
||||
uint32_t pow33 = 1;
|
||||
int i;
|
||||
memset(buf, 0, sizeof(buf));
|
||||
for (i = 0; i < remainder; ++i) {
|
||||
pow33 *= 33;
|
||||
}
|
||||
memcpy(buf + simd_size - remainder, src, remainder);
|
||||
seed = seed * pow33 + HashDjb2_SSE(buf, simd_size, 0);
|
||||
}
|
||||
}
|
||||
return seed;
|
||||
}
|
||||
@ -111,6 +140,36 @@ uint32_t ARGBDetect(const uint8_t* argb,
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBDetectRow = ARGBDetectRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBDETECTROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBDetectRow = ARGBDetectRow_NEON;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBDETECTROW_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
ARGBDetectRow = ARGBDetectRow_RVV;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBDETECTROW_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
ARGBDetectRow = ARGBDetectRow_LSX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBDETECTROW_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
ARGBDetectRow = ARGBDetectRow_LASX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBDETECTROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
ARGBDetectRow = ARGBDetectRow_WASMSIMD;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBDETECTROW_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
ARGBDetectRow = ARGBDetectRow_HVX;
|
||||
}
|
||||
#endif
|
||||
for (h = 0; h < height && fourcc == 0; ++h) {
|
||||
fourcc = ARGBDetectRow(argb, width);
|
||||
@ -164,6 +223,28 @@ uint64_t ComputeHammingDistance(const uint8_t* src_a,
|
||||
simd_size = 1; // Any count.
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HAMMINGDISTANCE_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
HammingDistance = HammingDistance_AVX512VNNI;
|
||||
simd_size = 1; // Any count.
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HAMMINGDISTANCE_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
HammingDistance = HammingDistance_RVV;
|
||||
simd_size = 1; // Any count.
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HAMMINGDISTANCE_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
HammingDistance = HammingDistance_LSX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HAMMINGDISTANCE_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
HammingDistance = HammingDistance_LASX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HAMMINGDISTANCE_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
HammingDistance = HammingDistance_WASMSIMD;
|
||||
@ -192,7 +273,17 @@ uint64_t ComputeHammingDistance(const uint8_t* src_a,
|
||||
}
|
||||
remainder = count & (simd_size - 1);
|
||||
if (remainder) {
|
||||
diff += HammingDistance_C(src_a, src_b, remainder);
|
||||
if (HammingDistance == HammingDistance_C) {
|
||||
diff += HammingDistance_C(src_a, src_b, remainder);
|
||||
} else {
|
||||
SIMD_ALIGNED(uint8_t buf_a[64]);
|
||||
SIMD_ALIGNED(uint8_t buf_b[64]);
|
||||
memset(buf_a, 0, sizeof(buf_a));
|
||||
memset(buf_b, 0, sizeof(buf_b));
|
||||
memcpy(buf_a, src_a, remainder);
|
||||
memcpy(buf_b, src_b, remainder);
|
||||
diff += HammingDistance(buf_a, buf_b, simd_size);
|
||||
}
|
||||
}
|
||||
return diff;
|
||||
}
|
||||
@ -246,6 +337,22 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a,
|
||||
simd_size = 1; // Any count.
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SUMSQUAREERROR_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
SumSquareError = SumSquareError_RVV;
|
||||
simd_size = 1; // Any count.
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SUMSQUAREERROR_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
SumSquareError = SumSquareError_LSX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SUMSQUAREERROR_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
SumSquareError = SumSquareError_LASX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SUMSQUAREERROR_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
SumSquareError = SumSquareError_WASMSIMD;
|
||||
@ -273,7 +380,17 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a,
|
||||
}
|
||||
remainder = count & (simd_size - 1);
|
||||
if (remainder) {
|
||||
sse += SumSquareError_C(src_a, src_b, remainder);
|
||||
if (SumSquareError == SumSquareError_C) {
|
||||
sse += SumSquareError_C(src_a, src_b, remainder);
|
||||
} else {
|
||||
SIMD_ALIGNED(uint8_t buf_a[32]);
|
||||
SIMD_ALIGNED(uint8_t buf_b[32]);
|
||||
memset(buf_a, 0, sizeof(buf_a));
|
||||
memset(buf_b, 0, sizeof(buf_b));
|
||||
memcpy(buf_a, src_a, remainder);
|
||||
memcpy(buf_b, src_b, remainder);
|
||||
sse += SumSquareError(buf_a, buf_b, simd_size);
|
||||
}
|
||||
}
|
||||
return sse;
|
||||
}
|
||||
@ -400,6 +517,78 @@ static float Ssim8x8(const uint8_t* src_a,
|
||||
}
|
||||
}
|
||||
|
||||
static void Ssim4x4Row_Rem(void (*Ssim4x4Row)(const uint8_t* src_a,
|
||||
int stride_a,
|
||||
const uint8_t* src_b,
|
||||
int stride_b,
|
||||
uint32_t* dst,
|
||||
int width),
|
||||
int simd_size,
|
||||
const uint8_t* src_a,
|
||||
int stride_a,
|
||||
const uint8_t* src_b,
|
||||
int stride_b,
|
||||
uint32_t* dst,
|
||||
int rem_width) {
|
||||
if (Ssim4x4Row == Ssim4x4Row_C) {
|
||||
Ssim4x4Row_C(src_a, stride_a, src_b, stride_b, dst, rem_width);
|
||||
return;
|
||||
}
|
||||
SIMD_ALIGNED(uint8_t buf_a[32 * 4]);
|
||||
SIMD_ALIGNED(uint8_t buf_b[32 * 4]);
|
||||
SIMD_ALIGNED(uint32_t buf_dst[32]);
|
||||
memset(buf_a, 0, sizeof(buf_a));
|
||||
memset(buf_b, 0, sizeof(buf_b));
|
||||
int r;
|
||||
for (r = 0; r < 4; ++r) {
|
||||
memcpy(buf_a + r * 32, src_a + (ptrdiff_t)r * stride_a, rem_width);
|
||||
memcpy(buf_b + r * 32, src_b + (ptrdiff_t)r * stride_b, rem_width);
|
||||
}
|
||||
Ssim4x4Row(buf_a, 32, buf_b, 32, buf_dst, simd_size);
|
||||
memcpy(dst, buf_dst, (size_t)rem_width * sizeof(uint32_t));
|
||||
}
|
||||
|
||||
static float SsimWindowRow_Rem(float (*SsimWindowRow)(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows),
|
||||
int win_simd_size,
|
||||
const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int rem_windows) {
|
||||
if (SsimWindowRow == SsimWindowRow_C) {
|
||||
return SsimWindowRow_C(row0, row1, rem_windows);
|
||||
}
|
||||
SIMD_ALIGNED(uint32_t win0[9 * 4]);
|
||||
SIMD_ALIGNED(uint32_t win1[9 * 4]);
|
||||
memset(win0, 0, sizeof(win0));
|
||||
memset(win1, 0, sizeof(win1));
|
||||
// Set odd tiles (1, 3, 5, 7) in win1 so each unused window k >= 1 has
|
||||
// sum_a = 4439, sum_b = -3, sum_sq = 307887, giving n1 = 2*4439*(-3) + c1 = 0
|
||||
// and SSIM = 0.0f (avoiding float precision loss from adding 1.0f per lane).
|
||||
int k;
|
||||
for (k = 1; k <= win_simd_size; k += 2) {
|
||||
win1[k * 4 + 0] = 4439u;
|
||||
win1[k * 4 + 1] = (uint32_t)-3;
|
||||
win1[k * 4 + 2] = 307887u;
|
||||
}
|
||||
float total = 0.0f;
|
||||
int w;
|
||||
for (w = 0; w < rem_windows; ++w) {
|
||||
const uint32_t* r0 = row0 + w * 4;
|
||||
const uint32_t* r1 = row1 + w * 4;
|
||||
win0[0] = r0[0] + r0[4];
|
||||
win0[1] = r0[1] + r0[5];
|
||||
win0[2] = r0[2] + r0[6];
|
||||
win0[3] = r0[3] + r0[7];
|
||||
win1[0] = r1[0] + r1[4] - 4439u;
|
||||
win1[1] = r1[1] + r1[5] - (uint32_t)-3;
|
||||
win1[2] = r1[2] + r1[6] - 307887u;
|
||||
win1[3] = r1[3] + r1[7];
|
||||
total += SsimWindowRow(win0, win1, win_simd_size);
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
// We are using a 8x8 moving window with starting location of each 8x8 window
|
||||
// on the 4x4 pixel grid. Such arrangement allows the windows to overlap
|
||||
// block boundaries to penalize blocking artifacts.
|
||||
@ -472,6 +661,12 @@ double CalcFrameSsim(const uint8_t* src_a,
|
||||
simd_size = 16;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SSIMWINDOWROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
SsimWindowRow = SsimWindowRow_NEON;
|
||||
win_simd_size = 4;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SSIM4X4ROW_SVE2)
|
||||
if (TestCpuFlag(kCpuHasSVE2)) {
|
||||
Ssim4x4Row = Ssim4x4Row_SVE2;
|
||||
@ -514,24 +709,48 @@ double CalcFrameSsim(const uint8_t* src_a,
|
||||
simd_size = 16;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SSIMWINDOWROW_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
SsimWindowRow = SsimWindowRow_LSX;
|
||||
win_simd_size = 1;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SSIM4X4ROW_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
Ssim4x4Row = Ssim4x4Row_LASX;
|
||||
simd_size = 32;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SSIMWINDOWROW_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
SsimWindowRow = SsimWindowRow_LASX;
|
||||
win_simd_size = 1;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SSIM4X4ROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
Ssim4x4Row = Ssim4x4Row_WASMSIMD;
|
||||
simd_size = 16;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SSIMWINDOWROW_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
SsimWindowRow = SsimWindowRow_WASMSIMD;
|
||||
win_simd_size = 4;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SSIM4X4ROW_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
Ssim4x4Row = Ssim4x4Row_HVX;
|
||||
simd_size = 4;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SSIMWINDOWROW_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
SsimWindowRow = SsimWindowRow_HVX;
|
||||
win_simd_size = 1;
|
||||
}
|
||||
#endif
|
||||
|
||||
const int num_windows = (width - 8 + 3) >> 2;
|
||||
const int num_tiles = num_windows + 1;
|
||||
@ -562,8 +781,9 @@ double CalcFrameSsim(const uint8_t* src_a,
|
||||
Ssim4x4Row(a, stride_a, b, stride_b, row0, main_width);
|
||||
}
|
||||
if (rem_width > 0) {
|
||||
Ssim4x4Row_C(a + main_width, stride_a, b + main_width, stride_b,
|
||||
row0 + main_width, rem_width);
|
||||
Ssim4x4Row_Rem(Ssim4x4Row, simd_size, a + main_width, stride_a,
|
||||
b + main_width, stride_b, row0 + main_width,
|
||||
rem_width);
|
||||
}
|
||||
}
|
||||
a += (ptrdiff_t)stride_a * 4;
|
||||
@ -572,15 +792,17 @@ double CalcFrameSsim(const uint8_t* src_a,
|
||||
Ssim4x4Row(a, stride_a, b, stride_b, row1, main_width);
|
||||
}
|
||||
if (rem_width > 0) {
|
||||
Ssim4x4Row_C(a + main_width, stride_a, b + main_width, stride_b,
|
||||
row1 + main_width, rem_width);
|
||||
Ssim4x4Row_Rem(Ssim4x4Row, simd_size, a + main_width, stride_a,
|
||||
b + main_width, stride_b, row1 + main_width,
|
||||
rem_width);
|
||||
}
|
||||
if (main_windows > 0) {
|
||||
ssim_total += SsimWindowRow(row0, row1, main_windows);
|
||||
}
|
||||
if (rem_windows > 0) {
|
||||
ssim_total += SsimWindowRow_C(row0 + main_windows * 4,
|
||||
row1 + main_windows * 4, rem_windows);
|
||||
ssim_total += SsimWindowRow_Rem(SsimWindowRow, win_simd_size,
|
||||
row0 + main_windows * 4,
|
||||
row1 + main_windows * 4, rem_windows);
|
||||
}
|
||||
samples += num_windows;
|
||||
uint32_t* tmp = row0;
|
||||
|
||||
@ -375,9 +375,11 @@ uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) {
|
||||
"xmm7");
|
||||
return hash;
|
||||
}
|
||||
#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
|
||||
defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW) || \
|
||||
defined(HAS_SSIM4X4ROW_AVX512BW)
|
||||
#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
|
||||
defined(HAS_HAMMINGDISTANCE_AVX512VNNI) || \
|
||||
defined(HAS_SUMSQUAREERROR_AVX512BW) || \
|
||||
defined(HAS_SUMSQUAREERROR_AVX512VNNI) || \
|
||||
defined(HAS_HASHDJB2_AVX512BW) || defined(HAS_SSIM4X4ROW_AVX512BW)
|
||||
// Mask of the low n bytes, for n = 0 to 63.
|
||||
static uint64_t LowBytesMask64(int n) {
|
||||
return n ? ~0ULL >> (64 - n) : 0ULL;
|
||||
@ -447,11 +449,81 @@ uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
|
||||
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
|
||||
[diff] "=r"(diff)
|
||||
: [nibble] "m"(kNibbleMask), [bitcount] "m"(kBitCount), [mask] "m"(mask)
|
||||
: "cc", "memory", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||
return diff;
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_AVX512BW
|
||||
|
||||
#ifdef HAS_HAMMINGDISTANCE_AVX512VNNI
|
||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||
// loads that read zeros, accumulating byte popcounts with vpdpbusd.
|
||||
uint32_t HammingDistance_AVX512VNNI(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
uint32_t diff;
|
||||
const uint64_t mask = LowBytesMask64(count & 63);
|
||||
asm volatile(
|
||||
"vbroadcasti32x4 %[nibble],%%zmm2 \n"
|
||||
"vbroadcasti32x4 %[bitcount],%%zmm3 \n"
|
||||
"vpxord %%zmm0,%%zmm0,%%zmm0 \n"
|
||||
"vpternlogd $0xff,%%zmm1,%%zmm1,%%zmm1 \n"
|
||||
"vpsrlw $0xf,%%zmm1,%%zmm1 \n"
|
||||
"vpshufb %%zmm0,%%zmm1,%%zmm1 \n"
|
||||
"kmovq %[mask],%%k1 \n"
|
||||
"sub %[src_a],%[src_b] \n"
|
||||
"sub $0x40,%[count] \n"
|
||||
"jl 2f \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vmovdqu8 (%[src_a]),%%zmm4 \n"
|
||||
"vpxorq (%[src_a],%[src_b]),%%zmm4,%%zmm4 \n"
|
||||
"add $0x40,%[src_a] \n"
|
||||
"vpandq %%zmm2,%%zmm4,%%zmm5 \n"
|
||||
"vpsrlw $0x4,%%zmm4,%%zmm4 \n"
|
||||
"vpandq %%zmm2,%%zmm4,%%zmm4 \n"
|
||||
"vpshufb %%zmm5,%%zmm3,%%zmm5 \n"
|
||||
"vpshufb %%zmm4,%%zmm3,%%zmm4 \n"
|
||||
"vpaddb %%zmm5,%%zmm4,%%zmm4 \n"
|
||||
"vpdpbusd %%zmm4,%%zmm1,%%zmm0 \n"
|
||||
"sub $0x40,%[count] \n"
|
||||
"jge 1b \n"
|
||||
|
||||
// Remainder: 1 to 63 bytes, masked loads.
|
||||
"2: \n"
|
||||
"add $0x40,%[count] \n"
|
||||
"je 3f \n"
|
||||
"vmovdqu8 (%[src_a]),%%zmm4%{%%k1%}%{z%} \n"
|
||||
"vmovdqu8 (%[src_a],%[src_b]),%%zmm5%{%%k1%}%{z%} \n"
|
||||
"vpxorq %%zmm5,%%zmm4,%%zmm4 \n"
|
||||
"vpandq %%zmm2,%%zmm4,%%zmm5 \n"
|
||||
"vpsrlw $0x4,%%zmm4,%%zmm4 \n"
|
||||
"vpandq %%zmm2,%%zmm4,%%zmm4 \n"
|
||||
"vpshufb %%zmm5,%%zmm3,%%zmm5 \n"
|
||||
"vpshufb %%zmm4,%%zmm3,%%zmm4 \n"
|
||||
"vpaddb %%zmm5,%%zmm4,%%zmm4 \n"
|
||||
"vpdpbusd %%zmm4,%%zmm1,%%zmm0 \n"
|
||||
|
||||
// Add 16 dwords.
|
||||
"3: \n"
|
||||
"vextracti64x4 $1,%%zmm0,%%ymm1 \n"
|
||||
"vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vextracti128 $1,%%ymm0,%%xmm1 \n"
|
||||
"vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vpshufd $0xee,%%xmm0,%%xmm1 \n"
|
||||
"vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vpshufd $0x1,%%xmm0,%%xmm1 \n"
|
||||
"vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vmovd %%xmm0,%[diff] \n"
|
||||
"vzeroupper \n"
|
||||
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
|
||||
[diff] "=r"(diff)
|
||||
: [nibble] "m"(kNibbleMask), [bitcount] "m"(kBitCount), [mask] "m"(mask)
|
||||
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||
return diff;
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_AVX512VNNI
|
||||
|
||||
#ifdef HAS_SUMSQUAREERROR_AVX512BW
|
||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||
// loads that read zeros, so any count is supported.
|
||||
@ -1454,8 +1526,7 @@ uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) {
|
||||
asm volatile(
|
||||
"vpcmpeqd %%ymm1,%%ymm1,%%ymm1 \n"
|
||||
"sub $0x8,%[width] \n"
|
||||
"jl 2f \n"
|
||||
LABELALIGN
|
||||
"jl 2f \n" LABELALIGN
|
||||
"1: \n"
|
||||
"vpcmpeqb (%[argb]),%%ymm1,%%ymm0 \n"
|
||||
"vpmovmskb %%ymm0,%[diff] \n"
|
||||
@ -1467,8 +1538,7 @@ uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) {
|
||||
"jge 1b \n"
|
||||
"2: \n"
|
||||
"add $0x8,%[width] \n"
|
||||
"jle 99f \n"
|
||||
LABELALIGN
|
||||
"jle 99f \n" LABELALIGN
|
||||
"3: \n"
|
||||
"vmovd (%[argb]),%%xmm0 \n"
|
||||
"add $0x4,%[argb] \n"
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
*/
|
||||
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__)
|
||||
|
||||
@ -282,6 +283,96 @@ void Ssim4x4Row_HVX(const uint8_t* src_a,
|
||||
#undef SSIM_4X4_HVX
|
||||
#endif // defined(HAS_SSIM4X4ROW_HVX)
|
||||
|
||||
#if defined(HAS_SSIMWINDOWROW_HVX)
|
||||
float SsimWindowRow_HVX(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows) {
|
||||
const HVX_Vector vc1 = Q6_V_vsplat_R(26634);
|
||||
const HVX_Vector vc2 = Q6_V_vsplat_R(239708);
|
||||
float total = 0.0f;
|
||||
while (num_windows > 0) {
|
||||
int count = num_windows < 8 ? num_windows : 8;
|
||||
HVX_Vector t0, t1;
|
||||
if (num_windows >= 8) {
|
||||
t0 = Q6_Vw_vadd_VwVw(vloadu(row0), vloadu(row1));
|
||||
t1 = Q6_Vw_vadd_VwVw(vloadu(row0 + 4), vloadu(row1 + 4));
|
||||
} else {
|
||||
int bytes0 = count * 16;
|
||||
int bytes1 = (count + 1) * 16;
|
||||
t0 = Q6_Vw_vadd_VwVw(vloadu_partial_zero(row0, bytes0),
|
||||
vloadu_partial_zero(row1, bytes0));
|
||||
t1 = Q6_Vw_vadd_VwVw(vloadu_partial_zero(row0 + 4, bytes1 - 16),
|
||||
vloadu_partial_zero(row1 + 4, bytes1 - 16));
|
||||
}
|
||||
HVX_Vector w = Q6_Vw_vadd_VwVw(t0, t1);
|
||||
HVX_Vector w_b = Q6_V_vror_VR(w, 4);
|
||||
HVX_Vector w_sq = Q6_V_vror_VR(w, 8);
|
||||
HVX_Vector w_axb = Q6_V_vror_VR(w, 12);
|
||||
HVX_Vector ab2 = Q6_Vw_vasl_VwR(Q6_Vw_vmpyie_VwVuh(w, w_b), 1);
|
||||
HVX_Vector a2b2 =
|
||||
Q6_Vw_vmpyieacc_VwVwVuh(Q6_Vw_vmpyie_VwVuh(w, w), w_b, w_b);
|
||||
HVX_Vector vn1 = Q6_Vw_vadd_VwVw(ab2, vc1);
|
||||
HVX_Vector vn2 =
|
||||
Q6_Vw_vadd_VwVw(Q6_Vw_vsub_VwVw(Q6_Vw_vasl_VwR(w_axb, 7), ab2), vc2);
|
||||
HVX_Vector vd1 = Q6_Vw_vadd_VwVw(a2b2, vc1);
|
||||
HVX_Vector vd2 =
|
||||
Q6_Vw_vadd_VwVw(Q6_Vw_vsub_VwVw(Q6_Vw_vasl_VwR(w_sq, 6), a2b2), vc2);
|
||||
int32_t n1[32] __attribute__((aligned(128)));
|
||||
int32_t n2[32] __attribute__((aligned(128)));
|
||||
int32_t d1[32] __attribute__((aligned(128)));
|
||||
int32_t d2[32] __attribute__((aligned(128)));
|
||||
vstoreu(n1, vn1);
|
||||
vstoreu(n2, vn2);
|
||||
vstoreu(d1, vd1);
|
||||
vstoreu(d2, vd2);
|
||||
int i;
|
||||
for (i = 0; i < count; ++i) {
|
||||
float num = (float)n1[i * 4] * (float)n2[i * 4];
|
||||
float den = (float)d1[i * 4] * (float)d2[i * 4];
|
||||
total += num / den;
|
||||
}
|
||||
row0 += 32;
|
||||
row1 += 32;
|
||||
num_windows -= 8;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
#endif // defined(HAS_SSIMWINDOWROW_HVX)
|
||||
|
||||
#if defined(HAS_ARGBDETECTROW_HVX)
|
||||
uint32_t ARGBDetectRow_HVX(const uint8_t* argb, int width) {
|
||||
const HVX_Vector mask_rgb = Q6_V_vsplat_R(0x00ffff00);
|
||||
const HVX_Vector ones = Q6_Vb_vsplat_R(-1);
|
||||
const HVX_Vector zero = Q6_V_vzero();
|
||||
while (width > 0) {
|
||||
HVX_Vector v;
|
||||
if (width >= 32) {
|
||||
v = vloadu(argb);
|
||||
} else {
|
||||
int bytes = width * 4;
|
||||
v = Q6_V_vmux_QVV(Q6_Q_vsetq_R(bytes), vloadu_partial_zero(argb, bytes),
|
||||
ones);
|
||||
}
|
||||
v = Q6_V_vor_VV(v, mask_rgb);
|
||||
HVX_VectorPred eq = Q6_Q_vcmp_eq_VbVb(v, ones);
|
||||
HVX_Vector diff_v = Q6_V_vmux_QVV(eq, zero, ones);
|
||||
if (horizontal_sum_u32(Q6_Vuw_vrmpy_VubRub(diff_v, 0x01010101)) != 0) {
|
||||
uint32_t diff[32] __attribute__((aligned(128)));
|
||||
vstoreu(diff, diff_v);
|
||||
int i;
|
||||
for (i = 0; i < 32; ++i) {
|
||||
if (diff[i] != 0) {
|
||||
return (diff[i] & 0xffu) ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
}
|
||||
}
|
||||
argb += 128;
|
||||
width -= 32;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // defined(HAS_ARGBDETECTROW_HVX)
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/row.h"
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
namespace libyuv {
|
||||
@ -155,6 +156,176 @@ void Ssim4x4Row_NEON(const uint8_t* src_a,
|
||||
}
|
||||
#undef SSIM_ROW_NEON32
|
||||
|
||||
static const uvec32 kDjb2Multiplicands[] = {
|
||||
{0x0c3525e1, // 33^15
|
||||
0xa3476dc1, // 33^14
|
||||
0x3b4039a1, // 33^13
|
||||
0x4f5f0981}, // 33^12
|
||||
{0x30f35d61, // 33^11
|
||||
0x855cb541, // 33^10
|
||||
0x040a9121, // 33^9
|
||||
0x747c7101}, // 33^8
|
||||
{0xec41d4e1, // 33^7
|
||||
0x4cfa3cc1, // 33^6
|
||||
0x025528a1, // 33^5
|
||||
0x00121881}, // 33^4
|
||||
{0x00008c61, // 33^3
|
||||
0x00000441, // 33^2
|
||||
0x00000021, // 33^1
|
||||
0x00000001}, // 33^0
|
||||
};
|
||||
|
||||
uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed) {
|
||||
uint32_t hash = seed;
|
||||
const uint32_t c16 = 0x92d9e201; // 33^16
|
||||
const void* muls0 = &kDjb2Multiplicands[0];
|
||||
const void* muls1 = &kDjb2Multiplicands[2];
|
||||
uint32_t tmp;
|
||||
asm volatile(
|
||||
"vld1.32 {d16, d17, d18, d19}, [%[muls0]] \n"
|
||||
"vld1.32 {d20, d21, d22, d23}, [%[muls1]] \n"
|
||||
"1: \n"
|
||||
"vld1.8 {d0, d1}, [%[src]]! \n"
|
||||
"subs %[count], %[count], #16 \n"
|
||||
"vmovl.u8 q1, d0 \n"
|
||||
"vmovl.u8 q2, d1 \n"
|
||||
"vmovl.u16 q0, d2 \n"
|
||||
"vmovl.u16 q1, d3 \n"
|
||||
"vmovl.u16 q3, d5 \n"
|
||||
"vmovl.u16 q2, d4 \n"
|
||||
"vmul.i32 q0, q0, q8 \n"
|
||||
"vmla.i32 q0, q1, q9 \n"
|
||||
"vmla.i32 q0, q2, q10 \n"
|
||||
"vmla.i32 q0, q3, q11 \n"
|
||||
"vpadd.i32 d0, d0, d1 \n"
|
||||
"vpadd.i32 d0, d0, d0 \n"
|
||||
"vmov.32 %[tmp], d0[0] \n"
|
||||
"mla %[hash], %[hash], %[c16], %[tmp] \n"
|
||||
"bgt 1b \n"
|
||||
:
|
||||
[src] "+r"(src), [hash] "+r"(hash), [count] "+r"(count), [tmp] "=&r"(tmp)
|
||||
: [muls0] "r"(muls0), [muls1] "r"(muls1), [c16] "r"(c16)
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
|
||||
return hash;
|
||||
}
|
||||
|
||||
float SsimWindowRow_NEON(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows) {
|
||||
const uint32_t* row0_next = row0 + 4;
|
||||
const uint32_t* row1_next = row1 + 4;
|
||||
float total;
|
||||
asm volatile(
|
||||
"vmov.i32 q12, #0 \n"
|
||||
"vdup.32 q13, %[c1] \n"
|
||||
"vdup.32 q14, %[c2] \n"
|
||||
"1: \n"
|
||||
"vld4.32 {d0, d2, d4, d6}, [%[row0]]! \n"
|
||||
"vld4.32 {d1, d3, d5, d7}, [%[row0]]! \n"
|
||||
"vld4.32 {d16, d18, d20, d22}, [%[row0_next]]! \n"
|
||||
"vld4.32 {d17, d19, d21, d23}, [%[row0_next]]! \n"
|
||||
"vadd.i32 q0, q0, q8 \n"
|
||||
"vadd.i32 q1, q1, q9 \n"
|
||||
"vadd.i32 q2, q2, q10 \n"
|
||||
"vadd.i32 q3, q3, q11 \n"
|
||||
"vld4.32 {d16, d18, d20, d22}, [%[row1]]! \n"
|
||||
"vld4.32 {d17, d19, d21, d23}, [%[row1]]! \n"
|
||||
"vadd.i32 q0, q0, q8 \n"
|
||||
"vadd.i32 q1, q1, q9 \n"
|
||||
"vadd.i32 q2, q2, q10 \n"
|
||||
"vadd.i32 q3, q3, q11 \n"
|
||||
"vld4.32 {d16, d18, d20, d22}, [%[row1_next]]! \n"
|
||||
"vld4.32 {d17, d19, d21, d23}, [%[row1_next]]! \n"
|
||||
"vadd.i32 q0, q0, q8 \n"
|
||||
"vadd.i32 q1, q1, q9 \n"
|
||||
"vadd.i32 q2, q2, q10 \n"
|
||||
"vadd.i32 q3, q3, q11 \n"
|
||||
"vmul.i32 q8, q0, q1 \n"
|
||||
"vshl.i32 q8, q8, #1 \n"
|
||||
"vmul.i32 q9, q0, q0 \n"
|
||||
"vmla.i32 q9, q1, q1 \n"
|
||||
"vshl.i32 q3, q3, #7 \n"
|
||||
"vsub.i32 q3, q3, q8 \n"
|
||||
"vadd.i32 q8, q8, q13 \n"
|
||||
"vadd.i32 q3, q3, q14 \n"
|
||||
"vshl.i32 q2, q2, #6 \n"
|
||||
"vsub.i32 q2, q2, q9 \n"
|
||||
"vadd.i32 q9, q9, q13 \n"
|
||||
"vadd.i32 q2, q2, q14 \n"
|
||||
"vcvt.f32.s32 q8, q8 \n"
|
||||
"vcvt.f32.s32 q3, q3 \n"
|
||||
"vcvt.f32.s32 q9, q9 \n"
|
||||
"vcvt.f32.s32 q2, q2 \n"
|
||||
"vmul.f32 q3, q8, q3 \n"
|
||||
"vmul.f32 q2, q9, q2 \n"
|
||||
"vdiv.f32 s12, s12, s8 \n"
|
||||
"vdiv.f32 s13, s13, s9 \n"
|
||||
"vdiv.f32 s14, s14, s10 \n"
|
||||
"vdiv.f32 s15, s15, s11 \n"
|
||||
"subs %[num_windows], %[num_windows], #4 \n"
|
||||
"vadd.f32 q12, q12, q3 \n"
|
||||
"bgt 1b \n"
|
||||
"vpadd.f32 d0, d24, d25 \n"
|
||||
"vpadd.f32 d0, d0, d0 \n"
|
||||
"vmov.f32 %[total], s0 \n"
|
||||
: [row0] "+r"(row0), [row0_next] "+r"(row0_next), [row1] "+r"(row1),
|
||||
[row1_next] "+r"(row1_next), [num_windows] "+r"(num_windows),
|
||||
[total] "=r"(total)
|
||||
: [c1] "r"(26634), [c2] "r"(239708)
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12",
|
||||
"q13", "q14");
|
||||
return total;
|
||||
}
|
||||
|
||||
uint32_t ARGBDetectRow_NEON(const uint8_t* argb, int width) {
|
||||
uint32_t diff_lo = 0;
|
||||
uint32_t diff_hi = 0;
|
||||
uint32_t tmp;
|
||||
asm volatile(
|
||||
"vmov.i32 q2, #0x00ff0000 \n"
|
||||
"vorr.i32 q2, #0x0000ff00 \n"
|
||||
"vmov.i8 q3, #0xff \n"
|
||||
"subs %[width], %[width], #4 \n"
|
||||
"blt 2f \n"
|
||||
"1: \n"
|
||||
"vld1.8 {q0}, [%[argb]]! \n"
|
||||
"vorr q0, q0, q2 \n"
|
||||
"vceq.i8 q0, q0, q3 \n"
|
||||
"vmvn q0, q0 \n"
|
||||
"vshrn.i16 d0, q0, #4 \n"
|
||||
"vmov %[diff_lo], %[diff_hi], d0 \n"
|
||||
"orrs %[tmp], %[diff_lo], %[diff_hi] \n"
|
||||
"bne 99f \n"
|
||||
"subs %[width], %[width], #4 \n"
|
||||
"bge 1b \n"
|
||||
"2: \n"
|
||||
"adds %[width], %[width], #4 \n"
|
||||
"ble 99f \n"
|
||||
"3: \n"
|
||||
"vld1.32 {d0[0]}, [%[argb]]! \n"
|
||||
"vorr d0, d0, d4 \n"
|
||||
"vceq.i8 d0, d0, d6 \n"
|
||||
"vmvn d0, d0 \n"
|
||||
"vshrn.i16 d0, q0, #4 \n"
|
||||
"vmov.u16 %[diff_lo], d0[0] \n"
|
||||
"cmp %[diff_lo], #0 \n"
|
||||
"bne 99f \n"
|
||||
"subs %[width], %[width], #1 \n"
|
||||
"bgt 3b \n"
|
||||
"99: \n"
|
||||
: [argb] "+r"(argb), [width] "+r"(width), [diff_lo] "+r"(diff_lo),
|
||||
[diff_hi] "+r"(diff_hi), [tmp] "=&r"(tmp)
|
||||
:
|
||||
: "memory", "cc", "q0", "q2", "q3");
|
||||
if (diff_lo) {
|
||||
return (__builtin_ctz(diff_lo) & 12) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
if (diff_hi) {
|
||||
return (__builtin_ctz(diff_hi) & 12) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/row.h"
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
namespace libyuv {
|
||||
@ -349,6 +350,107 @@ void Ssim4x4Row_NEON_DotProd(const uint8_t* src_a,
|
||||
"v19");
|
||||
}
|
||||
|
||||
float SsimWindowRow_NEON(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows) {
|
||||
const uint32_t* row0_next = row0 + 4;
|
||||
const uint32_t* row1_next = row1 + 4;
|
||||
float total;
|
||||
asm volatile(
|
||||
"movi v24.4s, #0 \n"
|
||||
"dup v25.4s, %w[c1] \n"
|
||||
"dup v26.4s, %w[c2] \n"
|
||||
"1: \n"
|
||||
"ld4 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[row0]], #64 \n"
|
||||
"ld4 {v4.4s, v5.4s, v6.4s, v7.4s}, [%[row0_next]], #64 \n"
|
||||
"ld4 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[row1]], #64 \n"
|
||||
"ld4 {v20.4s, v21.4s, v22.4s, v23.4s}, [%[row1_next]], #64 \n"
|
||||
"add v0.4s, v0.4s, v4.4s \n"
|
||||
"add v1.4s, v1.4s, v5.4s \n"
|
||||
"add v2.4s, v2.4s, v6.4s \n"
|
||||
"add v3.4s, v3.4s, v7.4s \n"
|
||||
"add v16.4s, v16.4s, v20.4s \n"
|
||||
"add v17.4s, v17.4s, v21.4s \n"
|
||||
"add v18.4s, v18.4s, v22.4s \n"
|
||||
"add v19.4s, v19.4s, v23.4s \n"
|
||||
"add v0.4s, v0.4s, v16.4s \n"
|
||||
"add v1.4s, v1.4s, v17.4s \n"
|
||||
"add v2.4s, v2.4s, v18.4s \n"
|
||||
"add v3.4s, v3.4s, v19.4s \n"
|
||||
"mul v4.4s, v0.4s, v1.4s \n"
|
||||
"shl v4.4s, v4.4s, #1 \n"
|
||||
"mul v5.4s, v0.4s, v0.4s \n"
|
||||
"mla v5.4s, v1.4s, v1.4s \n"
|
||||
"shl v3.4s, v3.4s, #7 \n"
|
||||
"sub v3.4s, v3.4s, v4.4s \n"
|
||||
"add v4.4s, v4.4s, v25.4s \n"
|
||||
"add v3.4s, v3.4s, v26.4s \n"
|
||||
"shl v2.4s, v2.4s, #6 \n"
|
||||
"sub v2.4s, v2.4s, v5.4s \n"
|
||||
"add v5.4s, v5.4s, v25.4s \n"
|
||||
"add v2.4s, v2.4s, v26.4s \n"
|
||||
"scvtf v4.4s, v4.4s \n"
|
||||
"scvtf v3.4s, v3.4s \n"
|
||||
"scvtf v5.4s, v5.4s \n"
|
||||
"scvtf v2.4s, v2.4s \n"
|
||||
"fmul v4.4s, v4.4s, v3.4s \n"
|
||||
"fmul v5.4s, v5.4s, v2.4s \n"
|
||||
"fdiv v4.4s, v4.4s, v5.4s \n"
|
||||
"subs %w[num_windows], %w[num_windows], #4 \n"
|
||||
"fadd v24.4s, v24.4s, v4.4s \n"
|
||||
"b.gt 1b \n"
|
||||
"faddp v24.4s, v24.4s, v24.4s \n"
|
||||
"faddp %s[total], v24.2s \n"
|
||||
: [row0] "+r"(row0), [row0_next] "+r"(row0_next), [row1] "+r"(row1),
|
||||
[row1_next] "+r"(row1_next), [num_windows] "+r"(num_windows),
|
||||
[total] "=w"(total)
|
||||
: [c1] "r"(26634), [c2] "r"(239708)
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
|
||||
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26");
|
||||
return total;
|
||||
}
|
||||
|
||||
uint32_t ARGBDetectRow_NEON(const uint8_t* argb, int width) {
|
||||
uint64_t diff = 0;
|
||||
asm volatile(
|
||||
"movi v2.2d, #0x00ffff0000ffff00 \n"
|
||||
"movi v3.16b, #0xff \n"
|
||||
"subs %w[width], %w[width], #4 \n"
|
||||
"b.lt 2f \n"
|
||||
"1: \n"
|
||||
"ldr q0, [%[argb]], #16 \n"
|
||||
"orr v0.16b, v0.16b, v2.16b \n"
|
||||
"cmeq v0.16b, v0.16b, v3.16b \n"
|
||||
"mvn v0.16b, v0.16b \n"
|
||||
"shrn v0.8b, v0.8h, #4 \n"
|
||||
"fmov %x[diff], d0 \n"
|
||||
"cbnz %x[diff], 99f \n"
|
||||
"subs %w[width], %w[width], #4 \n"
|
||||
"b.ge 1b \n"
|
||||
"2: \n"
|
||||
"adds %w[width], %w[width], #4 \n"
|
||||
"b.le 99f \n"
|
||||
"3: \n"
|
||||
"ldr s0, [%[argb]], #4 \n"
|
||||
"orr v0.8b, v0.8b, v2.8b \n"
|
||||
"cmeq v0.8b, v0.8b, v3.8b \n"
|
||||
"mvn v0.8b, v0.8b \n"
|
||||
"shrn v0.8b, v0.8h, #4 \n"
|
||||
"fmov %x[diff], d0 \n"
|
||||
"and %x[diff], %x[diff], #0xffff \n"
|
||||
"cbnz %x[diff], 99f \n"
|
||||
"subs %w[width], %w[width], #1 \n"
|
||||
"b.gt 3b \n"
|
||||
"99: \n"
|
||||
: [argb] "+r"(argb), [width] "+r"(width), [diff] "+r"(diff)
|
||||
:
|
||||
: "memory", "cc", "v0", "v2", "v3");
|
||||
if (diff) {
|
||||
return (__builtin_ctzll(diff) & 12) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/row.h"
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
|
||||
#include <wasm_simd128.h>
|
||||
@ -27,27 +28,37 @@ static inline uint32_t SumLanes_WASMSIMD(v128_t sum) {
|
||||
}
|
||||
|
||||
#if defined(HAS_HAMMINGDISTANCE_WASMSIMD)
|
||||
// Bit counts of 64 bytes are added as bytes, then pairs of bytes are added
|
||||
// to 16 bit sums, which hold up to 64 per 64 bytes, so count can be up to
|
||||
// 65535. Count is a multiple of 64.
|
||||
// Bit counts of 64 bytes are added as bytes, then accumulated into 32 bit
|
||||
// lanes using relaxed i8x16 dot product when available or pairwise widening
|
||||
// adds. Count is a multiple of 64.
|
||||
uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
v128_t sum = wasm_i16x8_splat(0);
|
||||
v128_t sum = wasm_i32x4_splat(0);
|
||||
#ifdef __wasm_relaxed_simd__
|
||||
const v128_t ones = wasm_i8x16_splat(1);
|
||||
#endif
|
||||
while (count > 0) {
|
||||
v128_t c = wasm_i8x16_splat(0);
|
||||
int i;
|
||||
for (i = 0; i < 64; i += 16) {
|
||||
c = wasm_i8x16_add(
|
||||
c, wasm_i8x16_popcnt(wasm_v128_xor(wasm_v128_load(src_a + i),
|
||||
wasm_v128_load(src_b + i))));
|
||||
}
|
||||
sum = wasm_i16x8_add(sum, wasm_u16x8_extadd_pairwise_u8x16(c));
|
||||
v128_t c0 = wasm_i8x16_popcnt(
|
||||
wasm_v128_xor(wasm_v128_load(src_a), wasm_v128_load(src_b)));
|
||||
v128_t c1 = wasm_i8x16_popcnt(
|
||||
wasm_v128_xor(wasm_v128_load(src_a + 16), wasm_v128_load(src_b + 16)));
|
||||
v128_t c2 = wasm_i8x16_popcnt(
|
||||
wasm_v128_xor(wasm_v128_load(src_a + 32), wasm_v128_load(src_b + 32)));
|
||||
v128_t c3 = wasm_i8x16_popcnt(
|
||||
wasm_v128_xor(wasm_v128_load(src_a + 48), wasm_v128_load(src_b + 48)));
|
||||
v128_t c = wasm_i8x16_add(wasm_i8x16_add(c0, c1), wasm_i8x16_add(c2, c3));
|
||||
#ifdef __wasm_relaxed_simd__
|
||||
sum = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(c, ones, sum);
|
||||
#else
|
||||
sum = wasm_i32x4_add(sum, wasm_u32x4_extadd_pairwise_u16x8(
|
||||
wasm_u16x8_extadd_pairwise_u8x16(c)));
|
||||
#endif
|
||||
src_a += 64;
|
||||
src_b += 64;
|
||||
count -= 64;
|
||||
}
|
||||
return SumLanes_WASMSIMD(wasm_u32x4_extadd_pairwise_u16x8(sum));
|
||||
return SumLanes_WASMSIMD(sum);
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_WASMSIMD
|
||||
|
||||
@ -241,6 +252,91 @@ void Ssim4x4Row_WASMSIMD(const uint8_t* src_a,
|
||||
}
|
||||
#endif // HAS_SSIM4X4ROW_WASMSIMD
|
||||
|
||||
#if defined(HAS_SSIMWINDOWROW_WASMSIMD)
|
||||
float SsimWindowRow_WASMSIMD(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows) {
|
||||
const v128_t vc1 = wasm_i32x4_splat(26634);
|
||||
const v128_t vc2 = wasm_i32x4_splat(239708);
|
||||
v128_t total = wasm_f32x4_splat(0.0f);
|
||||
v128_t t0 = wasm_i32x4_add(wasm_v128_load(row0), wasm_v128_load(row1));
|
||||
int w;
|
||||
for (w = 0; w < num_windows; w += 4) {
|
||||
v128_t t1 =
|
||||
wasm_i32x4_add(wasm_v128_load(row0 + 4), wasm_v128_load(row1 + 4));
|
||||
v128_t t2 =
|
||||
wasm_i32x4_add(wasm_v128_load(row0 + 8), wasm_v128_load(row1 + 8));
|
||||
v128_t t3 =
|
||||
wasm_i32x4_add(wasm_v128_load(row0 + 12), wasm_v128_load(row1 + 12));
|
||||
v128_t t4 =
|
||||
wasm_i32x4_add(wasm_v128_load(row0 + 16), wasm_v128_load(row1 + 16));
|
||||
row0 += 16;
|
||||
row1 += 16;
|
||||
v128_t w0 = wasm_i32x4_add(t0, t1);
|
||||
v128_t w1 = wasm_i32x4_add(t1, t2);
|
||||
v128_t w2 = wasm_i32x4_add(t2, t3);
|
||||
v128_t w3 = wasm_i32x4_add(t3, t4);
|
||||
t0 = t4;
|
||||
|
||||
v128_t u0 = wasm_i32x4_shuffle(w0, w1, 0, 4, 1, 5);
|
||||
v128_t u1 = wasm_i32x4_shuffle(w0, w1, 2, 6, 3, 7);
|
||||
v128_t u2 = wasm_i32x4_shuffle(w2, w3, 0, 4, 1, 5);
|
||||
v128_t u3 = wasm_i32x4_shuffle(w2, w3, 2, 6, 3, 7);
|
||||
v128_t sum_a = wasm_i64x2_shuffle(u0, u2, 0, 2);
|
||||
v128_t sum_b = wasm_i64x2_shuffle(u0, u2, 1, 3);
|
||||
v128_t sum_sq = wasm_i64x2_shuffle(u1, u3, 0, 2);
|
||||
v128_t sum_axb = wasm_i64x2_shuffle(u1, u3, 1, 3);
|
||||
|
||||
v128_t ab2 = wasm_i32x4_shl(wasm_i32x4_dot_i16x8(sum_a, sum_b), 1);
|
||||
v128_t a2b2 = wasm_i32x4_add(wasm_i32x4_dot_i16x8(sum_a, sum_a),
|
||||
wasm_i32x4_dot_i16x8(sum_b, sum_b));
|
||||
v128_t n1 = wasm_i32x4_add(ab2, vc1);
|
||||
v128_t n2 =
|
||||
wasm_i32x4_add(wasm_i32x4_sub(wasm_i32x4_shl(sum_axb, 7), ab2), vc2);
|
||||
v128_t d1 = wasm_i32x4_add(a2b2, vc1);
|
||||
v128_t d2 =
|
||||
wasm_i32x4_add(wasm_i32x4_sub(wasm_i32x4_shl(sum_sq, 6), a2b2), vc2);
|
||||
|
||||
v128_t num = wasm_f32x4_mul(wasm_f32x4_convert_i32x4(n1),
|
||||
wasm_f32x4_convert_i32x4(n2));
|
||||
v128_t den = wasm_f32x4_mul(wasm_f32x4_convert_i32x4(d1),
|
||||
wasm_f32x4_convert_i32x4(d2));
|
||||
total = wasm_f32x4_add(total, wasm_f32x4_div(num, den));
|
||||
}
|
||||
total = wasm_f32x4_add(total, wasm_i64x2_shuffle(total, total, 1, 0));
|
||||
total = wasm_f32x4_add(total, wasm_i32x4_shuffle(total, total, 1, 0, 3, 2));
|
||||
return wasm_f32x4_extract_lane(total, 0);
|
||||
}
|
||||
#endif // HAS_SSIMWINDOWROW_WASMSIMD
|
||||
|
||||
#if defined(HAS_ARGBDETECTROW_WASMSIMD)
|
||||
uint32_t ARGBDetectRow_WASMSIMD(const uint8_t* argb, int width) {
|
||||
const v128_t mask_rgb = wasm_i32x4_splat(0x00ffff00);
|
||||
const v128_t ones = wasm_i8x16_splat(-1);
|
||||
while (width >= 4) {
|
||||
v128_t v = wasm_v128_or(wasm_v128_load(argb), mask_rgb);
|
||||
uint32_t diff =
|
||||
(~(uint32_t)wasm_i8x16_bitmask(wasm_i8x16_eq(v, ones))) & 0xffffu;
|
||||
if (diff) {
|
||||
return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
argb += 16;
|
||||
width -= 4;
|
||||
}
|
||||
while (width > 0) {
|
||||
v128_t v = wasm_v128_or(wasm_v128_load32_zero(argb), mask_rgb);
|
||||
uint32_t diff =
|
||||
(~(uint32_t)wasm_i8x16_bitmask(wasm_i8x16_eq(v, ones))) & 0x9u;
|
||||
if (diff) {
|
||||
return (diff & 1u) ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
argb += 4;
|
||||
width -= 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // HAS_ARGBDETECTROW_WASMSIMD
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -75,8 +75,11 @@ uint32_t HammingDistance_SSE42(const uint8_t* src_a,
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_SSE42
|
||||
|
||||
#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
|
||||
defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW)
|
||||
#if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \
|
||||
defined(HAS_HAMMINGDISTANCE_AVX512VNNI) || \
|
||||
defined(HAS_SUMSQUAREERROR_AVX512BW) || \
|
||||
defined(HAS_SUMSQUAREERROR_AVX512VNNI) || \
|
||||
defined(HAS_HASHDJB2_AVX512BW) || defined(HAS_SSIM4X4ROW_AVX512BW)
|
||||
// Mask of the low n bytes, for n = 0 to 64.
|
||||
static __mmask64 LowBytesMask64(int n) {
|
||||
return (__mmask64)(n >= 64 ? ~0ULL : (1ULL << n) - 1);
|
||||
@ -161,6 +164,46 @@ uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_AVX512BW
|
||||
|
||||
#if defined(HAS_HAMMINGDISTANCE_AVX512VNNI)
|
||||
// Bit count of 64 bytes accumulated into 16 dwords with vpdpbusd.
|
||||
LIBYUV_TARGET_AVX512VNNI
|
||||
static __m512i BitCount64_AVX512VNNI(__m512i x, __m512i sum) {
|
||||
const __m512i kNibbleMask = _mm512_set1_epi8(15);
|
||||
const __m512i kOnes = _mm512_set1_epi8(1);
|
||||
const __m512i kBitCount = _mm512_broadcast_i32x4(
|
||||
_mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4));
|
||||
__m512i lo = _mm512_shuffle_epi8(kBitCount, _mm512_and_si512(x, kNibbleMask));
|
||||
__m512i hi = _mm512_shuffle_epi8(
|
||||
kBitCount, _mm512_and_si512(_mm512_srli_epi16(x, 4), kNibbleMask));
|
||||
return _mm512_dpbusd_epi32(sum, kOnes, _mm512_add_epi8(lo, hi));
|
||||
}
|
||||
|
||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||
// loads that read zeros, so any count is supported.
|
||||
LIBYUV_TARGET_AVX512VNNI
|
||||
uint32_t HammingDistance_AVX512VNNI(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
__m512i sum = _mm512_setzero_si512();
|
||||
while (count >= 64) {
|
||||
sum = BitCount64_AVX512VNNI(
|
||||
_mm512_xor_si512(_mm512_loadu_si512(src_a), _mm512_loadu_si512(src_b)),
|
||||
sum);
|
||||
src_a += 64;
|
||||
src_b += 64;
|
||||
count -= 64;
|
||||
}
|
||||
if (count > 0) {
|
||||
const __mmask64 k = LowBytesMask64(count);
|
||||
sum = BitCount64_AVX512VNNI(
|
||||
_mm512_xor_si512(_mm512_maskz_loadu_epi8(k, src_a),
|
||||
_mm512_maskz_loadu_epi8(k, src_b)),
|
||||
sum);
|
||||
}
|
||||
return (uint32_t)_mm512_reduce_add_epi32(sum);
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_AVX512VNNI
|
||||
|
||||
#if defined(HAS_SUMSQUAREERROR_SSE2)
|
||||
// Count is a multiple of 16.
|
||||
LIBYUV_TARGET_SSE2
|
||||
@ -262,8 +305,7 @@ uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a,
|
||||
while (count >= 64) {
|
||||
__m512i a = _mm512_loadu_si512(src_a);
|
||||
__m512i b = _mm512_loadu_si512(src_b);
|
||||
__m512i d =
|
||||
_mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
|
||||
__m512i d = _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
|
||||
__m512i lo = _mm512_unpacklo_epi8(d, zero);
|
||||
__m512i hi = _mm512_unpackhi_epi8(d, zero);
|
||||
sum0 = _mm512_dpwssd_epi32(sum0, lo, lo);
|
||||
@ -276,8 +318,7 @@ uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a,
|
||||
const __mmask64 k = LowBytesMask64(count);
|
||||
__m512i a = _mm512_maskz_loadu_epi8(k, src_a);
|
||||
__m512i b = _mm512_maskz_loadu_epi8(k, src_b);
|
||||
__m512i d =
|
||||
_mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
|
||||
__m512i d = _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
|
||||
__m512i lo = _mm512_unpacklo_epi8(d, zero);
|
||||
__m512i hi = _mm512_unpackhi_epi8(d, zero);
|
||||
sum0 = _mm512_dpwssd_epi32(sum0, lo, lo);
|
||||
@ -961,7 +1002,18 @@ uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) {
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
if (width > 0) {
|
||||
return ARGBDetectRow_C(argb, width);
|
||||
const __m128i mask_rgb = _mm_set1_epi32(0x00ffff00);
|
||||
const __m128i ones = _mm_set1_epi8(-1);
|
||||
do {
|
||||
__m128i v = _mm_or_si128(_mm_loadu_si32(argb), mask_rgb);
|
||||
uint32_t diff =
|
||||
(~(uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, ones))) & 0x9u;
|
||||
if (diff) {
|
||||
return (diff & 1u) ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
argb += 4;
|
||||
width -= 1;
|
||||
} while (width > 0);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/row.h"
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
// Note: every register written by an inline asm block (vector, general
|
||||
// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
|
||||
@ -3289,14 +3290,266 @@ void Ssim4x4Row_LASX(const uint8_t* src_a,
|
||||
"addi.d %[dst], %[dst], 128\n\t"
|
||||
"addi.d %[cnt], %[cnt], -32\n\t"
|
||||
"bnez %[cnt], 1b\n\t"
|
||||
: [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst),
|
||||
[cnt] "+r"(width), [pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r)
|
||||
: [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst), [cnt] "+r"(width),
|
||||
[pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r)
|
||||
: [sta] "r"((ptrdiff_t)stride_a), [stb] "r"((ptrdiff_t)stride_b)
|
||||
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
|
||||
"$xr7", "$xr8", "$xr9", "$xr10", "$xr11");
|
||||
}
|
||||
#endif // HAS_SSIM4X4ROW_LASX
|
||||
|
||||
#ifdef HAS_SSIMWINDOWROW_LASX
|
||||
float SsimWindowRow_LASX(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows) {
|
||||
const uint32_t* row0_next = row0 + 4;
|
||||
const uint32_t* row1_next = row1 + 4;
|
||||
float total;
|
||||
uint32_t c1 = 26634;
|
||||
uint32_t c2 = 239708;
|
||||
uint32_t c0 = 0;
|
||||
asm volatile(
|
||||
"xvreplgr2vr.w $xr6, %[c0]\n\t"
|
||||
"xvreplgr2vr.w $xr4, %[c1]\n\t"
|
||||
"xvreplgr2vr.w $xr5, %[c2]\n\t"
|
||||
"addi.d %[cnt], %[cnt], -2\n\t"
|
||||
"bltz %[cnt], 2f\n\t"
|
||||
"1:\n\t"
|
||||
"xvld $xr0, %[r0], 0\n\t"
|
||||
"xvld $xr1, %[r0n], 0\n\t"
|
||||
"xvld $xr2, %[r1], 0\n\t"
|
||||
"xvld $xr3, %[r1n], 0\n\t"
|
||||
"xvadd.w $xr0, $xr0, $xr1\n\t"
|
||||
"xvadd.w $xr2, $xr2, $xr3\n\t"
|
||||
"xvadd.w $xr0, $xr0, $xr2\n\t"
|
||||
"xvshuf4i.w $xr1, $xr0, 0x00\n\t"
|
||||
"xvshuf4i.w $xr2, $xr0, 0x55\n\t"
|
||||
"xvshuf4i.w $xr3, $xr0, 0xaa\n\t"
|
||||
"xvshuf4i.w $xr0, $xr0, 0xff\n\t"
|
||||
"xvmul.w $xr7, $xr1, $xr2\n\t"
|
||||
"xvslli.w $xr7, $xr7, 1\n\t"
|
||||
"xvmul.w $xr1, $xr1, $xr1\n\t"
|
||||
"xvmadd.w $xr1, $xr2, $xr2\n\t"
|
||||
"xvslli.w $xr0, $xr0, 7\n\t"
|
||||
"xvsub.w $xr0, $xr0, $xr7\n\t"
|
||||
"xvadd.w $xr7, $xr7, $xr4\n\t"
|
||||
"xvadd.w $xr0, $xr0, $xr5\n\t"
|
||||
"xvslli.w $xr3, $xr3, 6\n\t"
|
||||
"xvsub.w $xr3, $xr3, $xr1\n\t"
|
||||
"xvadd.w $xr1, $xr1, $xr4\n\t"
|
||||
"xvadd.w $xr3, $xr3, $xr5\n\t"
|
||||
"xvffint.s.w $xr7, $xr7\n\t"
|
||||
"xvffint.s.w $xr0, $xr0\n\t"
|
||||
"xvffint.s.w $xr1, $xr1\n\t"
|
||||
"xvffint.s.w $xr3, $xr3\n\t"
|
||||
"xvfmul.s $xr7, $xr7, $xr0\n\t"
|
||||
"xvfmul.s $xr1, $xr1, $xr3\n\t"
|
||||
"xvfdiv.s $xr7, $xr7, $xr1\n\t"
|
||||
"xvfadd.s $xr6, $xr6, $xr7\n\t"
|
||||
"addi.d %[r0], %[r0], 32\n\t"
|
||||
"addi.d %[r0n], %[r0n], 32\n\t"
|
||||
"addi.d %[r1], %[r1], 32\n\t"
|
||||
"addi.d %[r1n], %[r1n], 32\n\t"
|
||||
"addi.d %[cnt], %[cnt], -2\n\t"
|
||||
"bgez %[cnt], 1b\n\t"
|
||||
"xvpermi.q $xr0, $xr6, 0x01\n\t"
|
||||
"vfadd.s $vr6, $vr6, $vr0\n\t"
|
||||
"2:\n\t"
|
||||
"addi.d %[cnt], %[cnt], 2\n\t"
|
||||
"beqz %[cnt], 3f\n\t"
|
||||
"vld $vr0, %[r0], 0\n\t"
|
||||
"vld $vr1, %[r0n], 0\n\t"
|
||||
"vld $vr2, %[r1], 0\n\t"
|
||||
"vld $vr3, %[r1n], 0\n\t"
|
||||
"vadd.w $vr0, $vr0, $vr1\n\t"
|
||||
"vadd.w $vr2, $vr2, $vr3\n\t"
|
||||
"vadd.w $vr0, $vr0, $vr2\n\t"
|
||||
"vshuf4i.w $vr1, $vr0, 0x00\n\t"
|
||||
"vshuf4i.w $vr2, $vr0, 0x55\n\t"
|
||||
"vshuf4i.w $vr3, $vr0, 0xaa\n\t"
|
||||
"vshuf4i.w $vr0, $vr0, 0xff\n\t"
|
||||
"vmul.w $vr7, $vr1, $vr2\n\t"
|
||||
"vslli.w $vr7, $vr7, 1\n\t"
|
||||
"vmul.w $vr1, $vr1, $vr1\n\t"
|
||||
"vmadd.w $vr1, $vr2, $vr2\n\t"
|
||||
"vslli.w $vr0, $vr0, 7\n\t"
|
||||
"vsub.w $vr0, $vr0, $vr7\n\t"
|
||||
"vadd.w $vr7, $vr7, $vr4\n\t"
|
||||
"vadd.w $vr0, $vr0, $vr5\n\t"
|
||||
"vslli.w $vr3, $vr3, 6\n\t"
|
||||
"vsub.w $vr3, $vr3, $vr1\n\t"
|
||||
"vadd.w $vr1, $vr1, $vr4\n\t"
|
||||
"vadd.w $vr3, $vr3, $vr5\n\t"
|
||||
"vffint.s.w $vr7, $vr7\n\t"
|
||||
"vffint.s.w $vr0, $vr0\n\t"
|
||||
"vffint.s.w $vr1, $vr1\n\t"
|
||||
"vffint.s.w $vr3, $vr3\n\t"
|
||||
"vfmul.s $vr7, $vr7, $vr0\n\t"
|
||||
"vfmul.s $vr1, $vr1, $vr3\n\t"
|
||||
"vfdiv.s $vr7, $vr7, $vr1\n\t"
|
||||
"vfadd.s $vr6, $vr6, $vr7\n\t"
|
||||
"3:\n\t"
|
||||
"vstelm.w $vr6, %[tot], 0, 0\n\t"
|
||||
: [r0] "+r"(row0), [r0n] "+r"(row0_next), [r1] "+r"(row1),
|
||||
[r1n] "+r"(row1_next), [cnt] "+r"(num_windows)
|
||||
: [c0] "r"(c0), [c1] "r"(c1), [c2] "r"(c2), [tot] "r"(&total)
|
||||
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
|
||||
"$xr7");
|
||||
return total;
|
||||
}
|
||||
#endif // HAS_SSIMWINDOWROW_LASX
|
||||
|
||||
#ifdef HAS_ARGBDETECTROW_LASX
|
||||
uint32_t ARGBDetectRow_LASX(const uint8_t* argb, int width) {
|
||||
uint32_t diff = 0;
|
||||
uint32_t tmp;
|
||||
uint32_t rgb_mask = 0x00ffff00u;
|
||||
asm volatile(
|
||||
"xvreplgr2vr.w $xr2, %[rgb_mask]\n\t"
|
||||
"xvldi $xr3, -1\n\t"
|
||||
"addi.d %[w], %[w], -8\n\t"
|
||||
"bltz %[w], 2f\n\t"
|
||||
"1:\n\t"
|
||||
"xvld $xr0, %[argb], 0\n\t"
|
||||
"addi.d %[argb], %[argb], 32\n\t"
|
||||
"xvor.v $xr0, $xr0, $xr2\n\t"
|
||||
"xvseq.b $xr0, $xr0, $xr3\n\t"
|
||||
"xvmskltz.b $xr0, $xr0\n\t"
|
||||
"xvpickve2gr.wu %[diff], $xr0, 0\n\t"
|
||||
"xvpickve2gr.wu %[tmp], $xr0, 4\n\t"
|
||||
"bstrins.w %[diff], %[tmp], 31, 16\n\t"
|
||||
"nor %[diff], %[diff], $zero\n\t"
|
||||
"bnez %[diff], 4f\n\t"
|
||||
"addi.d %[w], %[w], -8\n\t"
|
||||
"bgez %[w], 1b\n\t"
|
||||
"2:\n\t"
|
||||
"addi.d %[w], %[w], 8\n\t"
|
||||
"beqz %[w], 4f\n\t"
|
||||
"3:\n\t"
|
||||
"vldrepl.w $vr0, %[argb], 0\n\t"
|
||||
"addi.d %[argb], %[argb], 4\n\t"
|
||||
"vor.v $vr0, $vr0, $vr2\n\t"
|
||||
"vseq.b $vr0, $vr0, $vr3\n\t"
|
||||
"vmskltz.b $vr0, $vr0\n\t"
|
||||
"vpickve2gr.hu %[diff], $vr0, 0\n\t"
|
||||
"nor %[diff], %[diff], $zero\n\t"
|
||||
"bstrpick.w %[diff], %[diff], 15, 0\n\t"
|
||||
"bnez %[diff], 4f\n\t"
|
||||
"addi.d %[w], %[w], -1\n\t"
|
||||
"bnez %[w], 3b\n\t"
|
||||
"4:\n\t"
|
||||
: [argb] "+r"(argb), [w] "+r"(width), [diff] "+r"(diff), [tmp] "=&r"(tmp)
|
||||
: [rgb_mask] "r"(rgb_mask)
|
||||
: "memory", "$xr0", "$xr2", "$xr3");
|
||||
if (diff) {
|
||||
return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // HAS_ARGBDETECTROW_LASX
|
||||
|
||||
#ifdef HAS_HAMMINGDISTANCE_LASX
|
||||
uint32_t HammingDistance_LASX(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
uint32_t diff;
|
||||
asm volatile(
|
||||
"xvxor.v $xr4, $xr4, $xr4\n\t"
|
||||
"1:\n\t"
|
||||
"xvld $xr0, %[sa], 0\n\t"
|
||||
"xvld $xr1, %[sb], 0\n\t"
|
||||
"xvxor.v $xr0, $xr0, $xr1\n\t"
|
||||
"xvpcnt.b $xr0, $xr0\n\t"
|
||||
"xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t"
|
||||
"xvhaddw.wu.hu $xr0, $xr0, $xr0\n\t"
|
||||
"xvadd.w $xr4, $xr4, $xr0\n\t"
|
||||
"addi.d %[sa], %[sa], 32\n\t"
|
||||
"addi.d %[sb], %[sb], 32\n\t"
|
||||
"addi.d %[cnt], %[cnt], -32\n\t"
|
||||
"bnez %[cnt], 1b\n\t"
|
||||
"xvhaddw.du.wu $xr4, $xr4, $xr4\n\t"
|
||||
"xvhaddw.qu.du $xr4, $xr4, $xr4\n\t"
|
||||
"xvpermi.q $xr0, $xr4, 0x01\n\t"
|
||||
"vadd.w $vr4, $vr4, $vr0\n\t"
|
||||
"vpickve2gr.wu %[diff], $vr4, 0\n\t"
|
||||
: [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [diff] "=r"(diff)
|
||||
:
|
||||
: "memory", "$xr0", "$xr1", "$xr4");
|
||||
return diff;
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_LASX
|
||||
|
||||
#ifdef HAS_SUMSQUAREERROR_LASX
|
||||
uint32_t SumSquareError_LASX(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
uint32_t sse;
|
||||
asm volatile(
|
||||
"xvxor.v $xr4, $xr4, $xr4\n\t"
|
||||
"1:\n\t"
|
||||
"xvld $xr0, %[sa], 0\n\t"
|
||||
"xvld $xr1, %[sb], 0\n\t"
|
||||
"xvabsd.bu $xr0, $xr0, $xr1\n\t"
|
||||
"xvmulwev.h.bu $xr2, $xr0, $xr0\n\t"
|
||||
"xvmulwod.h.bu $xr3, $xr0, $xr0\n\t"
|
||||
"xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t"
|
||||
"xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t"
|
||||
"xvadd.w $xr4, $xr4, $xr2\n\t"
|
||||
"xvadd.w $xr4, $xr4, $xr3\n\t"
|
||||
"addi.d %[sa], %[sa], 32\n\t"
|
||||
"addi.d %[sb], %[sb], 32\n\t"
|
||||
"addi.d %[cnt], %[cnt], -32\n\t"
|
||||
"bnez %[cnt], 1b\n\t"
|
||||
"xvhaddw.du.wu $xr4, $xr4, $xr4\n\t"
|
||||
"xvhaddw.qu.du $xr4, $xr4, $xr4\n\t"
|
||||
"xvpermi.q $xr0, $xr4, 0x01\n\t"
|
||||
"vadd.w $vr4, $vr4, $vr0\n\t"
|
||||
"vpickve2gr.wu %[sse], $vr4, 0\n\t"
|
||||
: [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [sse] "=r"(sse)
|
||||
:
|
||||
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4");
|
||||
return sse;
|
||||
}
|
||||
#endif // HAS_SUMSQUAREERROR_LASX
|
||||
|
||||
#ifdef HAS_HASHDJB2_LASX
|
||||
static const uint32_t kDjb2Muls_LASX[16] = {
|
||||
0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, 0x30f35d61, 0x855cb541,
|
||||
0x040a9121, 0x747c7101, 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881,
|
||||
0x00008c61, 0x00000441, 0x00000021, 0x00000001,
|
||||
};
|
||||
|
||||
uint32_t HashDjb2_LASX(const uint8_t* src, int count, uint32_t seed) {
|
||||
uint32_t hash = seed;
|
||||
uint32_t tmp;
|
||||
uint32_t c16 = 0x92d9e201u;
|
||||
asm volatile(
|
||||
"xvld $xr4, %[muls], 0\n\t"
|
||||
"xvld $xr5, %[muls], 32\n\t"
|
||||
"1:\n\t"
|
||||
"vld $vr0, %[src], 0\n\t"
|
||||
"vbsrl.v $vr1, $vr0, 8\n\t"
|
||||
"vext2xv.wu.bu $xr0, $xr0\n\t"
|
||||
"vext2xv.wu.bu $xr1, $xr1\n\t"
|
||||
"xvmul.w $xr0, $xr0, $xr4\n\t"
|
||||
"xvmadd.w $xr0, $xr1, $xr5\n\t"
|
||||
"xvhaddw.du.wu $xr0, $xr0, $xr0\n\t"
|
||||
"xvhaddw.qu.du $xr0, $xr0, $xr0\n\t"
|
||||
"xvpermi.q $xr1, $xr0, 0x01\n\t"
|
||||
"vadd.w $vr0, $vr0, $vr1\n\t"
|
||||
"vpickve2gr.wu %[tmp], $vr0, 0\n\t"
|
||||
"mul.w %[hash], %[hash], %[c16]\n\t"
|
||||
"add.w %[hash], %[hash], %[tmp]\n\t"
|
||||
"addi.d %[src], %[src], 16\n\t"
|
||||
"addi.d %[cnt], %[cnt], -16\n\t"
|
||||
"bnez %[cnt], 1b\n\t"
|
||||
: [src] "+r"(src), [cnt] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp)
|
||||
: [muls] "r"(kDjb2Muls_LASX), [c16] "r"(c16)
|
||||
: "memory", "$xr0", "$xr1", "$xr4", "$xr5");
|
||||
return hash;
|
||||
}
|
||||
#endif // HAS_HASHDJB2_LASX
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/row.h"
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
// Note: every register written by an inline asm block (vector, general
|
||||
// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
|
||||
@ -2882,14 +2883,227 @@ void Ssim4x4Row_LSX(const uint8_t* src_a,
|
||||
"addi.d %[dst], %[dst], 64\n\t"
|
||||
"addi.d %[cnt], %[cnt], -16\n\t"
|
||||
"bnez %[cnt], 1b\n\t"
|
||||
: [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst),
|
||||
[cnt] "+r"(width), [pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r)
|
||||
: [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst), [cnt] "+r"(width),
|
||||
[pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r)
|
||||
: [sta] "r"((ptrdiff_t)stride_a), [stb] "r"((ptrdiff_t)stride_b)
|
||||
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
|
||||
"$vr7", "$vr8", "$vr9", "$vr10", "$vr11");
|
||||
}
|
||||
#endif // HAS_SSIM4X4ROW_LSX
|
||||
|
||||
#ifdef HAS_SSIMWINDOWROW_LSX
|
||||
float SsimWindowRow_LSX(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows) {
|
||||
const uint32_t* row0_next = row0 + 4;
|
||||
const uint32_t* row1_next = row1 + 4;
|
||||
float total;
|
||||
uint32_t c1 = 26634;
|
||||
uint32_t c2 = 239708;
|
||||
uint32_t c0 = 0;
|
||||
asm volatile(
|
||||
"vreplgr2vr.w $vr6, %[c0]\n\t"
|
||||
"vreplgr2vr.w $vr4, %[c1]\n\t"
|
||||
"vreplgr2vr.w $vr5, %[c2]\n\t"
|
||||
"1:\n\t"
|
||||
"vld $vr0, %[r0], 0\n\t"
|
||||
"vld $vr1, %[r0n], 0\n\t"
|
||||
"vld $vr2, %[r1], 0\n\t"
|
||||
"vld $vr3, %[r1n], 0\n\t"
|
||||
"vadd.w $vr0, $vr0, $vr1\n\t"
|
||||
"vadd.w $vr2, $vr2, $vr3\n\t"
|
||||
"vadd.w $vr0, $vr0, $vr2\n\t"
|
||||
"vshuf4i.w $vr1, $vr0, 0x00\n\t"
|
||||
"vshuf4i.w $vr2, $vr0, 0x55\n\t"
|
||||
"vshuf4i.w $vr3, $vr0, 0xaa\n\t"
|
||||
"vshuf4i.w $vr0, $vr0, 0xff\n\t"
|
||||
"vmul.w $vr7, $vr1, $vr2\n\t"
|
||||
"vslli.w $vr7, $vr7, 1\n\t"
|
||||
"vmul.w $vr1, $vr1, $vr1\n\t"
|
||||
"vmadd.w $vr1, $vr2, $vr2\n\t"
|
||||
"vslli.w $vr0, $vr0, 7\n\t"
|
||||
"vsub.w $vr0, $vr0, $vr7\n\t"
|
||||
"vadd.w $vr7, $vr7, $vr4\n\t"
|
||||
"vadd.w $vr0, $vr0, $vr5\n\t"
|
||||
"vslli.w $vr3, $vr3, 6\n\t"
|
||||
"vsub.w $vr3, $vr3, $vr1\n\t"
|
||||
"vadd.w $vr1, $vr1, $vr4\n\t"
|
||||
"vadd.w $vr3, $vr3, $vr5\n\t"
|
||||
"vffint.s.w $vr7, $vr7\n\t"
|
||||
"vffint.s.w $vr0, $vr0\n\t"
|
||||
"vffint.s.w $vr1, $vr1\n\t"
|
||||
"vffint.s.w $vr3, $vr3\n\t"
|
||||
"vfmul.s $vr7, $vr7, $vr0\n\t"
|
||||
"vfmul.s $vr1, $vr1, $vr3\n\t"
|
||||
"vfdiv.s $vr7, $vr7, $vr1\n\t"
|
||||
"vfadd.s $vr6, $vr6, $vr7\n\t"
|
||||
"addi.d %[r0], %[r0], 16\n\t"
|
||||
"addi.d %[r0n], %[r0n], 16\n\t"
|
||||
"addi.d %[r1], %[r1], 16\n\t"
|
||||
"addi.d %[r1n], %[r1n], 16\n\t"
|
||||
"addi.d %[cnt], %[cnt], -1\n\t"
|
||||
"bnez %[cnt], 1b\n\t"
|
||||
"vstelm.w $vr6, %[tot], 0, 0\n\t"
|
||||
: [r0] "+r"(row0), [r0n] "+r"(row0_next), [r1] "+r"(row1),
|
||||
[r1n] "+r"(row1_next), [cnt] "+r"(num_windows)
|
||||
: [c0] "r"(c0), [c1] "r"(c1), [c2] "r"(c2), [tot] "r"(&total)
|
||||
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
|
||||
"$vr7");
|
||||
return total;
|
||||
}
|
||||
#endif // HAS_SSIMWINDOWROW_LSX
|
||||
|
||||
#ifdef HAS_ARGBDETECTROW_LSX
|
||||
uint32_t ARGBDetectRow_LSX(const uint8_t* argb, int width) {
|
||||
uint32_t diff = 0;
|
||||
uint32_t rgb_mask = 0x00ffff00u;
|
||||
asm volatile(
|
||||
"vreplgr2vr.w $vr2, %[rgb_mask]\n\t"
|
||||
"vldi $vr3, -1\n\t"
|
||||
"addi.d %[w], %[w], -4\n\t"
|
||||
"bltz %[w], 2f\n\t"
|
||||
"1:\n\t"
|
||||
"vld $vr0, %[argb], 0\n\t"
|
||||
"addi.d %[argb], %[argb], 16\n\t"
|
||||
"vor.v $vr0, $vr0, $vr2\n\t"
|
||||
"vseq.b $vr0, $vr0, $vr3\n\t"
|
||||
"vmskltz.b $vr0, $vr0\n\t"
|
||||
"vpickve2gr.hu %[diff], $vr0, 0\n\t"
|
||||
"nor %[diff], %[diff], $zero\n\t"
|
||||
"bstrpick.w %[diff], %[diff], 15, 0\n\t"
|
||||
"bnez %[diff], 4f\n\t"
|
||||
"addi.d %[w], %[w], -4\n\t"
|
||||
"bgez %[w], 1b\n\t"
|
||||
"2:\n\t"
|
||||
"addi.d %[w], %[w], 4\n\t"
|
||||
"beqz %[w], 4f\n\t"
|
||||
"3:\n\t"
|
||||
"vldrepl.w $vr0, %[argb], 0\n\t"
|
||||
"addi.d %[argb], %[argb], 4\n\t"
|
||||
"vor.v $vr0, $vr0, $vr2\n\t"
|
||||
"vseq.b $vr0, $vr0, $vr3\n\t"
|
||||
"vmskltz.b $vr0, $vr0\n\t"
|
||||
"vpickve2gr.hu %[diff], $vr0, 0\n\t"
|
||||
"nor %[diff], %[diff], $zero\n\t"
|
||||
"bstrpick.w %[diff], %[diff], 15, 0\n\t"
|
||||
"bnez %[diff], 4f\n\t"
|
||||
"addi.d %[w], %[w], -1\n\t"
|
||||
"bnez %[w], 3b\n\t"
|
||||
"4:\n\t"
|
||||
: [argb] "+r"(argb), [w] "+r"(width), [diff] "+r"(diff)
|
||||
: [rgb_mask] "r"(rgb_mask)
|
||||
: "memory", "$vr0", "$vr2", "$vr3");
|
||||
if (diff) {
|
||||
return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // HAS_ARGBDETECTROW_LSX
|
||||
|
||||
#ifdef HAS_HAMMINGDISTANCE_LSX
|
||||
uint32_t HammingDistance_LSX(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
uint32_t diff;
|
||||
asm volatile(
|
||||
"vxor.v $vr4, $vr4, $vr4\n\t"
|
||||
"1:\n\t"
|
||||
"vld $vr0, %[sa], 0\n\t"
|
||||
"vld $vr1, %[sb], 0\n\t"
|
||||
"vxor.v $vr0, $vr0, $vr1\n\t"
|
||||
"vpcnt.b $vr0, $vr0\n\t"
|
||||
"vhaddw.hu.bu $vr0, $vr0, $vr0\n\t"
|
||||
"vhaddw.wu.hu $vr0, $vr0, $vr0\n\t"
|
||||
"vadd.w $vr4, $vr4, $vr0\n\t"
|
||||
"addi.d %[sa], %[sa], 16\n\t"
|
||||
"addi.d %[sb], %[sb], 16\n\t"
|
||||
"addi.d %[cnt], %[cnt], -16\n\t"
|
||||
"bnez %[cnt], 1b\n\t"
|
||||
"vhaddw.du.wu $vr4, $vr4, $vr4\n\t"
|
||||
"vhaddw.qu.du $vr4, $vr4, $vr4\n\t"
|
||||
"vpickve2gr.wu %[diff], $vr4, 0\n\t"
|
||||
: [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [diff] "=r"(diff)
|
||||
:
|
||||
: "memory", "$vr0", "$vr1", "$vr4");
|
||||
return diff;
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_LSX
|
||||
|
||||
#ifdef HAS_SUMSQUAREERROR_LSX
|
||||
uint32_t SumSquareError_LSX(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
uint32_t sse;
|
||||
asm volatile(
|
||||
"vxor.v $vr4, $vr4, $vr4\n\t"
|
||||
"1:\n\t"
|
||||
"vld $vr0, %[sa], 0\n\t"
|
||||
"vld $vr1, %[sb], 0\n\t"
|
||||
"vabsd.bu $vr0, $vr0, $vr1\n\t"
|
||||
"vmulwev.h.bu $vr2, $vr0, $vr0\n\t"
|
||||
"vmulwod.h.bu $vr3, $vr0, $vr0\n\t"
|
||||
"vhaddw.wu.hu $vr2, $vr2, $vr2\n\t"
|
||||
"vhaddw.wu.hu $vr3, $vr3, $vr3\n\t"
|
||||
"vadd.w $vr4, $vr4, $vr2\n\t"
|
||||
"vadd.w $vr4, $vr4, $vr3\n\t"
|
||||
"addi.d %[sa], %[sa], 16\n\t"
|
||||
"addi.d %[sb], %[sb], 16\n\t"
|
||||
"addi.d %[cnt], %[cnt], -16\n\t"
|
||||
"bnez %[cnt], 1b\n\t"
|
||||
"vhaddw.du.wu $vr4, $vr4, $vr4\n\t"
|
||||
"vhaddw.qu.du $vr4, $vr4, $vr4\n\t"
|
||||
"vpickve2gr.wu %[sse], $vr4, 0\n\t"
|
||||
: [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [sse] "=r"(sse)
|
||||
:
|
||||
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
|
||||
return sse;
|
||||
}
|
||||
#endif // HAS_SUMSQUAREERROR_LSX
|
||||
|
||||
#ifdef HAS_HASHDJB2_LSX
|
||||
static const uint32_t kDjb2Muls_LSX[16] = {
|
||||
0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, 0x30f35d61, 0x855cb541,
|
||||
0x040a9121, 0x747c7101, 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881,
|
||||
0x00008c61, 0x00000441, 0x00000021, 0x00000001,
|
||||
};
|
||||
|
||||
uint32_t HashDjb2_LSX(const uint8_t* src, int count, uint32_t seed) {
|
||||
uint32_t hash = seed;
|
||||
uint32_t tmp;
|
||||
uint32_t c16 = 0x92d9e201u;
|
||||
asm volatile(
|
||||
"vld $vr4, %[muls], 0\n\t"
|
||||
"vld $vr5, %[muls], 16\n\t"
|
||||
"vld $vr6, %[muls], 32\n\t"
|
||||
"vld $vr7, %[muls], 48\n\t"
|
||||
"1:\n\t"
|
||||
"vld $vr0, %[src], 0\n\t"
|
||||
"vsllwil.hu.bu $vr1, $vr0, 0\n\t"
|
||||
"vexth.hu.bu $vr2, $vr0\n\t"
|
||||
"vsllwil.wu.hu $vr0, $vr1, 0\n\t"
|
||||
"vexth.wu.hu $vr1, $vr1\n\t"
|
||||
"vsllwil.wu.hu $vr3, $vr2, 0\n\t"
|
||||
"vexth.wu.hu $vr2, $vr2\n\t"
|
||||
"vmul.w $vr0, $vr0, $vr4\n\t"
|
||||
"vmadd.w $vr0, $vr1, $vr5\n\t"
|
||||
"vmadd.w $vr0, $vr3, $vr6\n\t"
|
||||
"vmadd.w $vr0, $vr2, $vr7\n\t"
|
||||
"vhaddw.du.wu $vr0, $vr0, $vr0\n\t"
|
||||
"vhaddw.qu.du $vr0, $vr0, $vr0\n\t"
|
||||
"vpickve2gr.wu %[tmp], $vr0, 0\n\t"
|
||||
"mul.w %[hash], %[hash], %[c16]\n\t"
|
||||
"add.w %[hash], %[hash], %[tmp]\n\t"
|
||||
"addi.d %[src], %[src], 16\n\t"
|
||||
"addi.d %[cnt], %[cnt], -16\n\t"
|
||||
"bnez %[cnt], 1b\n\t"
|
||||
: [src] "+r"(src), [cnt] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp)
|
||||
: [muls] "r"(kDjb2Muls_LSX), [c16] "r"(c16)
|
||||
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
|
||||
"$vr7");
|
||||
return hash;
|
||||
}
|
||||
#endif // HAS_HASHDJB2_LSX
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/row.h"
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
// Note: every register written by an inline asm block (vector, general
|
||||
// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
|
||||
@ -3015,7 +3016,7 @@ float SsimWindowRow_RVV(const uint32_t* row0,
|
||||
[total] "=f"(total) // %[total]
|
||||
: [c1] "r"(26634), // %[c1]
|
||||
[c2] "r"(239708) // %[c2]
|
||||
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v4", "v8", "v9", "v10",
|
||||
: "memory", RVV_VL_CLOBBER "vtype", "v0", "v1", "v4", "v8", "v9", "v10",
|
||||
"v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20",
|
||||
"v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
|
||||
"v31");
|
||||
@ -3023,6 +3024,173 @@ float SsimWindowRow_RVV(const uint32_t* row0,
|
||||
}
|
||||
#endif // HAS_SSIMWINDOWROW_RVV
|
||||
|
||||
#ifdef HAS_ARGBDETECTROW_RVV
|
||||
uint32_t ARGBDetectRow_RVV(const uint8_t* argb, int width) {
|
||||
size_t vl;
|
||||
size_t byte_vl;
|
||||
long idx = -1;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"vsetvli %[vl], %[w], e32, m8, ta, ma \n"
|
||||
"slli %[byte_vl], %[vl], 2 \n"
|
||||
"vsetvli zero, %[byte_vl], e8, m8, ta, ma \n"
|
||||
"vle8.v v8, (%[argb]) \n"
|
||||
"vsetvli zero, %[vl], e32, m8, ta, ma \n"
|
||||
"vor.vx v8, v8, %[mask_rgb] \n"
|
||||
"sub %[w], %[w], %[vl] \n"
|
||||
"add %[argb], %[argb], %[byte_vl] \n"
|
||||
"vsetvli zero, %[byte_vl], e8, m8, ta, ma \n"
|
||||
"vmsne.vi v0, v8, -1 \n"
|
||||
"vfirst.m %[idx], v0 \n"
|
||||
"bgez %[idx], 2f \n"
|
||||
"bgtz %[w], 1b \n"
|
||||
"2: \n"
|
||||
: [argb] "+r"(argb), [w] "+r"(width), [vl] "=&r"(vl),
|
||||
[byte_vl] "=&r"(byte_vl), [idx] "+r"(idx)
|
||||
: [mask_rgb] "r"(0x00ffff00u)
|
||||
: "memory", RVV_VL_CLOBBER "vtype", "v0", "v8", "v9", "v10", "v11", "v12",
|
||||
"v13", "v14", "v15");
|
||||
if (idx >= 0) {
|
||||
return (idx & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // HAS_ARGBDETECTROW_RVV
|
||||
|
||||
#ifdef HAS_SUMSQUAREERROR_RVV
|
||||
uint32_t SumSquareError_RVV(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
size_t vl;
|
||||
uint32_t sse;
|
||||
asm volatile(
|
||||
"vsetvli %[vl], zero, e32, m8, ta, ma \n"
|
||||
"vmv.v.i v24, 0 \n"
|
||||
"vmv.s.x v0, zero \n"
|
||||
"1: \n"
|
||||
"vsetvli %[vl], %[count], e8, m2, tu, ma \n"
|
||||
"vle8.v v8, (%[src_a]) \n"
|
||||
"vle8.v v10, (%[src_b]) \n"
|
||||
"vwsubu.vv v12, v8, v10 \n"
|
||||
"sub %[count], %[count], %[vl] \n"
|
||||
"add %[src_a], %[src_a], %[vl] \n"
|
||||
"add %[src_b], %[src_b], %[vl] \n"
|
||||
"vsetvli zero, %[vl], e16, m4, tu, ma \n"
|
||||
"vwmacc.vv v24, v12, v12 \n"
|
||||
"bgtz %[count], 1b \n"
|
||||
"vsetvli %[vl], zero, e32, m8, ta, ma \n"
|
||||
"vredsum.vs v0, v24, v0 \n"
|
||||
"vmv.x.s %[sse], v0 \n"
|
||||
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
|
||||
[vl] "=&r"(vl), [sse] "=r"(sse)
|
||||
:
|
||||
: "memory", RVV_VL_CLOBBER "vtype", "v0", "v8", "v9", "v10", "v11", "v12",
|
||||
"v13", "v14", "v15", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
|
||||
"v31");
|
||||
return sse;
|
||||
}
|
||||
#endif // HAS_SUMSQUAREERROR_RVV
|
||||
|
||||
#ifdef HAS_HAMMINGDISTANCE_RVV
|
||||
static const uint8_t kNibblePopcnt_RVV[16] = {0, 1, 1, 2, 1, 2, 2, 3,
|
||||
1, 2, 2, 3, 2, 3, 3, 4};
|
||||
|
||||
uint32_t HammingDistance_RVV(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
size_t vl;
|
||||
uint32_t diff;
|
||||
asm volatile(
|
||||
"vsetivli zero, 16, e8, m1, ta, ma \n"
|
||||
"vle8.v v4, (%[lut]) \n"
|
||||
"vsetvli %[vl], zero, e32, m4, ta, ma \n"
|
||||
"vmv.v.i v24, 0 \n"
|
||||
"vsetvli %[vl], zero, e32, m1, ta, ma \n"
|
||||
"vmv.s.x v0, zero \n"
|
||||
"1: \n"
|
||||
"vsetvli %[vl], %[count], e8, m1, tu, ma \n"
|
||||
"vle8.v v8, (%[src_a]) \n"
|
||||
"vle8.v v9, (%[src_b]) \n"
|
||||
"vxor.vv v8, v8, v9 \n"
|
||||
"vand.vi v10, v8, 15 \n"
|
||||
"vsrl.vi v8, v8, 4 \n"
|
||||
"vrgather.vv v11, v4, v10 \n"
|
||||
"vrgather.vv v12, v4, v8 \n"
|
||||
"vwaddu.vv v16, v11, v12 \n"
|
||||
"sub %[count], %[count], %[vl] \n"
|
||||
"add %[src_a], %[src_a], %[vl] \n"
|
||||
"add %[src_b], %[src_b], %[vl] \n"
|
||||
"vsetvli zero, %[vl], e16, m2, tu, ma \n"
|
||||
"vwaddu.wv v24, v24, v16 \n"
|
||||
"bgtz %[count], 1b \n"
|
||||
"vsetvli %[vl], zero, e32, m4, ta, ma \n"
|
||||
"vredsum.vs v0, v24, v0 \n"
|
||||
"vsetvli %[vl], zero, e32, m1, ta, ma \n"
|
||||
"vmv.x.s %[diff], v0 \n"
|
||||
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
|
||||
[vl] "=&r"(vl), [diff] "=r"(diff)
|
||||
: [lut] "r"(kNibblePopcnt_RVV)
|
||||
: "memory", RVV_VL_CLOBBER "vtype", "v0", "v4", "v8", "v9", "v10", "v11",
|
||||
"v12", "v16", "v17", "v24", "v25", "v26", "v27");
|
||||
return diff;
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_RVV
|
||||
|
||||
#ifdef HAS_HASHDJB2_RVV
|
||||
static const uint32_t kDjb2Muls_RVV[17] = {
|
||||
0x92d9e201, // 33 ^ 16
|
||||
0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, 0x30f35d61, 0x855cb541,
|
||||
0x040a9121, 0x747c7101, 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881,
|
||||
0x00008c61, 0x00000441, 0x00000021, 0x00000001,
|
||||
};
|
||||
|
||||
uint32_t HashDjb2_RVV(const uint8_t* src, int count, uint32_t seed) {
|
||||
uint32_t hash = seed;
|
||||
uint32_t tmp;
|
||||
const uint32_t c16 = 0x92d9e201u;
|
||||
const int rem = count & 15;
|
||||
const uint32_t* tail_muls = &kDjb2Muls_RVV[17 - rem];
|
||||
const uint32_t pow_n = kDjb2Muls_RVV[16 - rem];
|
||||
asm volatile(
|
||||
"vsetivli zero, 16, e32, m4, ta, ma \n"
|
||||
"vmv.s.x v4, zero \n"
|
||||
"addi %[count], %[count], -16 \n"
|
||||
"bltz %[count], 2f \n"
|
||||
"vle32.v v16, (%[muls16]) \n"
|
||||
"1: \n"
|
||||
"vle8.v v8, (%[src]) \n"
|
||||
"vzext.vf4 v12, v8 \n"
|
||||
"vmul.vv v12, v12, v16 \n"
|
||||
"vredsum.vs v0, v12, v4 \n"
|
||||
"vmv.x.s %[tmp], v0 \n"
|
||||
"mul %[hash], %[hash], %[c16] \n"
|
||||
"add %[hash], %[hash], %[tmp] \n"
|
||||
"addi %[src], %[src], 16 \n"
|
||||
"addi %[count], %[count], -16 \n"
|
||||
"bgez %[count], 1b \n"
|
||||
"2: \n"
|
||||
"addi %[count], %[count], 16 \n"
|
||||
"beqz %[count], 3f \n"
|
||||
"vsetvli zero, %[count], e32, m4, ta, ma \n"
|
||||
"vle32.v v16, (%[tail_muls]) \n"
|
||||
"vle8.v v8, (%[src]) \n"
|
||||
"vzext.vf4 v12, v8 \n"
|
||||
"vmul.vv v12, v12, v16 \n"
|
||||
"vredsum.vs v0, v12, v4 \n"
|
||||
"vmv.x.s %[tmp], v0 \n"
|
||||
"mul %[hash], %[hash], %[pow_n] \n"
|
||||
"add %[hash], %[hash], %[tmp] \n"
|
||||
"3: \n"
|
||||
:
|
||||
[src] "+r"(src), [count] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp)
|
||||
: [muls16] "r"(kDjb2Muls_RVV + 1), [c16] "r"(c16),
|
||||
[tail_muls] "r"(tail_muls), [pow_n] "r"(pow_n)
|
||||
: "memory", RVV_VL_CLOBBER "vtype", "v0", "v4", "v8", "v12", "v13", "v14",
|
||||
"v15", "v16", "v17", "v18", "v19");
|
||||
return hash;
|
||||
}
|
||||
#endif // HAS_HASHDJB2_RVV
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -287,32 +287,66 @@ TEST_F(LibYUVCompareTest, BenchmarkHammingDistance_Opt) {
|
||||
benchmark_iterations_ *
|
||||
((benchmark_width_ * benchmark_height_ + kMaxWidth - 1) / kMaxWidth);
|
||||
for (int i = 0; i < count; ++i) {
|
||||
#if defined(HAS_HAMMINGDISTANCE_NEON)
|
||||
h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
if (has_avx2) {
|
||||
h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
|
||||
#if defined(HAS_HAMMINGDISTANCE_NEON_DOTPROD)
|
||||
if (TestCpuFlag(kCpuHasNeonDotProd)) {
|
||||
h1 = HammingDistance_NEON_DotProd(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
int has_sse42 = TestCpuFlag(kCpuHasSSE42);
|
||||
if (has_sse42) {
|
||||
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
|
||||
if (has_ssse3) {
|
||||
h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
}
|
||||
h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_NEON)
|
||||
h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
h1 = HammingDistance_AVX512VNNI(src_a, src_b, kMaxWidth);
|
||||
} else if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
h1 = HammingDistance_AVX512BW(src_a, src_b, kMaxWidth);
|
||||
} else if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSE42)
|
||||
} else if (TestCpuFlag(kCpuHasSSE42)) {
|
||||
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSSE3)
|
||||
} else if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
} else {
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSE42)
|
||||
} else if (TestCpuFlag(kCpuHasSSE42)) {
|
||||
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSSE3)
|
||||
} else if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
} else {
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_SSE42)
|
||||
int has_sse42 = TestCpuFlag(kCpuHasSSE42);
|
||||
if (has_sse42) {
|
||||
if (TestCpuFlag(kCpuHasSSE42)) {
|
||||
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_RVV)
|
||||
h1 = HammingDistance_RVV(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
h1 = HammingDistance_LASX(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_LSX)
|
||||
h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_WASMSIMD)
|
||||
h1 = HammingDistance_WASMSIMD(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_HVX)
|
||||
h1 = HammingDistance_HVX(src_a, src_b, kMaxWidth);
|
||||
#else
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
@ -408,32 +442,66 @@ TEST_F(LibYUVCompareTest, TestHammingDistance_Opt) {
|
||||
ASSERT_EQ(kMaxWidth * 8ULL, h0);
|
||||
|
||||
for (int i = 0; i < benchmark_iterations_; ++i) {
|
||||
#if defined(HAS_HAMMINGDISTANCE_NEON)
|
||||
h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
if (has_avx2) {
|
||||
h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
|
||||
#if defined(HAS_HAMMINGDISTANCE_NEON_DOTPROD)
|
||||
if (TestCpuFlag(kCpuHasNeonDotProd)) {
|
||||
h1 = HammingDistance_NEON_DotProd(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
int has_sse42 = TestCpuFlag(kCpuHasSSE42);
|
||||
if (has_sse42) {
|
||||
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
|
||||
if (has_ssse3) {
|
||||
h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
}
|
||||
h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_NEON)
|
||||
h1 = HammingDistance_NEON(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
h1 = HammingDistance_AVX512VNNI(src_a, src_b, kMaxWidth);
|
||||
} else if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
h1 = HammingDistance_AVX512BW(src_a, src_b, kMaxWidth);
|
||||
} else if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSE42)
|
||||
} else if (TestCpuFlag(kCpuHasSSE42)) {
|
||||
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSSE3)
|
||||
} else if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
} else {
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
h1 = HammingDistance_AVX2(src_a, src_b, kMaxWidth);
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSE42)
|
||||
} else if (TestCpuFlag(kCpuHasSSE42)) {
|
||||
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSSE3)
|
||||
} else if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
h1 = HammingDistance_SSSE3(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
} else {
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_SSE42)
|
||||
int has_sse42 = TestCpuFlag(kCpuHasSSE42);
|
||||
if (has_sse42) {
|
||||
if (TestCpuFlag(kCpuHasSSE42)) {
|
||||
h1 = HammingDistance_SSE42(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_RVV)
|
||||
h1 = HammingDistance_RVV(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
h1 = HammingDistance_LASX(src_a, src_b, kMaxWidth);
|
||||
} else {
|
||||
h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth);
|
||||
}
|
||||
#elif defined(HAS_HAMMINGDISTANCE_LSX)
|
||||
h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_WASMSIMD)
|
||||
h1 = HammingDistance_WASMSIMD(src_a, src_b, kMaxWidth);
|
||||
#elif defined(HAS_HAMMINGDISTANCE_HVX)
|
||||
h1 = HammingDistance_HVX(src_a, src_b, kMaxWidth);
|
||||
#else
|
||||
h1 = HammingDistance_C(src_a, src_b, kMaxWidth);
|
||||
#endif
|
||||
@ -560,6 +628,14 @@ TEST_F(LibYUVCompareTest, TestCompareAnyCount) {
|
||||
MemRandomize(src_a, kMaxCount + 1);
|
||||
MemRandomize(src_b, kMaxCount + 1);
|
||||
|
||||
const int kCpuMasks[] = {
|
||||
benchmark_cpu_info_,
|
||||
benchmark_cpu_info_ & ~kCpuHasAVX512VNNI,
|
||||
benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VNNI),
|
||||
benchmark_cpu_info_ &
|
||||
~(kCpuHasAVX2 | kCpuHasAVX512BW | kCpuHasAVX512VNNI),
|
||||
};
|
||||
|
||||
for (int count = 0; count <= kMaxCount; ++count) {
|
||||
for (int offset = 0; offset < 2; ++offset) {
|
||||
const uint8_t* a = src_a + kMaxCount + 1 - count - offset;
|
||||
@ -568,13 +644,19 @@ TEST_F(LibYUVCompareTest, TestCompareAnyCount) {
|
||||
uint32_t c_hash = HashDjb2(a, count, 5381);
|
||||
uint64_t c_diff = ComputeHammingDistance(a, b, count);
|
||||
uint64_t c_err = ComputeSumSquareError(a, b, count);
|
||||
for (size_t m = 0; m < sizeof(kCpuMasks) / sizeof(kCpuMasks[0]); ++m) {
|
||||
MaskCpuFlags(kCpuMasks[m]);
|
||||
uint32_t opt_hash = HashDjb2(a, count, 5381);
|
||||
uint64_t opt_diff = ComputeHammingDistance(a, b, count);
|
||||
uint64_t opt_err = ComputeSumSquareError(a, b, count);
|
||||
EXPECT_EQ(c_hash, opt_hash)
|
||||
<< "count " << count << " offset " << offset << " mask " << m;
|
||||
EXPECT_EQ(c_diff, opt_diff)
|
||||
<< "count " << count << " offset " << offset << " mask " << m;
|
||||
EXPECT_EQ(c_err, opt_err)
|
||||
<< "count " << count << " offset " << offset << " mask " << m;
|
||||
}
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
uint32_t opt_hash = HashDjb2(a, count, 5381);
|
||||
uint64_t opt_diff = ComputeHammingDistance(a, b, count);
|
||||
uint64_t opt_err = ComputeSumSquareError(a, b, count);
|
||||
EXPECT_EQ(c_hash, opt_hash) << "count " << count << " offset " << offset;
|
||||
EXPECT_EQ(c_diff, opt_diff) << "count " << count << " offset " << offset;
|
||||
EXPECT_EQ(c_err, opt_err) << "count " << count << " offset " << offset;
|
||||
}
|
||||
}
|
||||
|
||||
@ -772,6 +854,13 @@ TEST_F(LibYUVCompareTest, TestSsimAnyWidth) {
|
||||
src_b[i] = (i * 151 + 43) & 0xff;
|
||||
}
|
||||
|
||||
const int kCpuMasks[] = {
|
||||
benchmark_cpu_info_,
|
||||
benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VNNI),
|
||||
benchmark_cpu_info_ &
|
||||
~(kCpuHasAVX2 | kCpuHasAVX512BW | kCpuHasAVX512VNNI),
|
||||
};
|
||||
|
||||
for (int width = 9; width <= kMaxWidth; ++width) {
|
||||
for (int offset = 0; offset < 2; ++offset) {
|
||||
const int size = width * kHeight;
|
||||
@ -779,10 +868,13 @@ TEST_F(LibYUVCompareTest, TestSsimAnyWidth) {
|
||||
const uint8_t* b = src_b + kPlaneSize - size - offset;
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
double c_ssim = CalcFrameSsim(a, width, b, width, width, kHeight);
|
||||
for (size_t m = 0; m < sizeof(kCpuMasks) / sizeof(kCpuMasks[0]); ++m) {
|
||||
MaskCpuFlags(kCpuMasks[m]);
|
||||
double opt_ssim = CalcFrameSsim(a, width, b, width, width, kHeight);
|
||||
EXPECT_NEAR(c_ssim, opt_ssim, 1e-5)
|
||||
<< "width " << width << " offset " << offset << " mask " << m;
|
||||
}
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
double opt_ssim = CalcFrameSsim(a, width, b, width, width, kHeight);
|
||||
EXPECT_NEAR(c_ssim, opt_ssim, 1e-5)
|
||||
<< "width " << width << " offset " << offset;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user