/* * Copyright 2012 The LibYuv Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "libyuv/compare_row.h" #include "libyuv/row.h" // This module is for Visual C 32/64 bit, and for clang with // LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. #if !defined(LIBYUV_DISABLE_X86) && \ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ defined(_M_IX86)) && \ ((defined(_MSC_VER) && !defined(__clang__)) || \ defined(LIBYUV_ENABLE_ROWWIN)) #include // For SSE2 to AVX512 intrinsics #include // For memcpy #if defined(_MSC_VER) && !defined(__clang__) #include #endif #ifdef __cplusplus namespace libyuv { extern "C" { #endif // Target attributes allow gcc and clang to build each function for its // instruction set regardless of the baseline. #if defined(__clang__) || defined(__GNUC__) #define LIBYUV_TARGET_SSE2 __attribute__((target("sse2"))) #define LIBYUV_TARGET_SSE41 __attribute__((target("sse4.1"))) #define LIBYUV_TARGET_SSE42 __attribute__((target("sse4.2,popcnt"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW __attribute__((target("avx512bw,avx512f"))) #else #define LIBYUV_TARGET_SSE2 #define LIBYUV_TARGET_SSE41 #define LIBYUV_TARGET_SSE42 #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW #endif // defined(__clang__) || defined(__GNUC__) #if defined(HAS_HAMMINGDISTANCE_SSE42) // Count is a multiple of 8. LIBYUV_TARGET_SSE42 uint32_t HammingDistance_SSE42(const uint8_t* src_a, const uint8_t* src_b, int count) { uint32_t diff = 0u; int i; for (i = 0; i < count; i += 8) { uint64_t a; uint64_t b; memcpy(&a, src_a + i, 8); memcpy(&b, src_b + i, 8); #if defined(__x86_64__) || defined(_M_X64) diff += (uint32_t)_mm_popcnt_u64(a ^ b); #else diff += _mm_popcnt_u32((uint32_t)(a ^ b)) + _mm_popcnt_u32((uint32_t)((a ^ b) >> 32)); #endif } return diff; } #endif // HAS_HAMMINGDISTANCE_SSE42 #if defined(HAS_HAMMINGDISTANCE_AVX512BW) || \ defined(HAS_SUMSQUAREERROR_AVX512BW) || defined(HAS_HASHDJB2_AVX512BW) // Mask of the low n bytes, for n = 0 to 64. static __mmask64 LowBytesMask64(int n) { return (__mmask64)(n >= 64 ? ~0ULL : (1ULL << n) - 1); } #endif #if defined(HAS_HAMMINGDISTANCE_AVX2) // Count is a multiple of 64. LIBYUV_TARGET_AVX2 uint32_t HammingDistance_AVX2(const uint8_t* src_a, const uint8_t* src_b, int count) { const __m256i kNibbleMask = _mm256_set1_epi8(15); const __m256i kBitCount = _mm256_broadcastsi128_si256( _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4)); const __m256i zero = _mm256_setzero_si256(); __m256i sum = _mm256_setzero_si256(); __m128i sum128; int i; for (i = 0; i < count; i += 64) { __m256i x0 = _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(src_a + i)), _mm256_loadu_si256((const __m256i*)(src_b + i))); __m256i x1 = _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(src_a + i + 32)), _mm256_loadu_si256((const __m256i*)(src_b + i + 32))); __m256i c0 = _mm256_add_epi8( _mm256_shuffle_epi8(kBitCount, _mm256_and_si256(x0, kNibbleMask)), _mm256_shuffle_epi8( kBitCount, _mm256_and_si256(_mm256_srli_epi16(x0, 4), kNibbleMask))); __m256i c1 = _mm256_add_epi8( _mm256_shuffle_epi8(kBitCount, _mm256_and_si256(x1, kNibbleMask)), _mm256_shuffle_epi8( kBitCount, _mm256_and_si256(_mm256_srli_epi16(x1, 4), kNibbleMask))); sum = _mm256_add_epi64(sum, _mm256_sad_epu8(_mm256_add_epi8(c0, c1), zero)); } sum128 = _mm_add_epi64(_mm256_castsi256_si128(sum), _mm256_extracti128_si256(sum, 1)); sum128 = _mm_add_epi64(sum128, _mm_unpackhi_epi64(sum128, sum128)); _mm256_zeroupper(); return (uint32_t)_mm_cvtsi128_si32(sum128); } #endif // HAS_HAMMINGDISTANCE_AVX2 #if defined(HAS_HAMMINGDISTANCE_AVX512BW) // Bit count of 64 bytes, summed to 8 qwords. LIBYUV_TARGET_AVX512BW static __m512i BitCount64_AVX512BW(__m512i x) { const __m512i kNibbleMask = _mm512_set1_epi8(15); const __m512i kBitCount = _mm512_broadcast_i32x4( _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4)); __m512i lo = _mm512_shuffle_epi8(kBitCount, _mm512_and_si512(x, kNibbleMask)); __m512i hi = _mm512_shuffle_epi8( kBitCount, _mm512_and_si512(_mm512_srli_epi16(x, 4), kNibbleMask)); return _mm512_sad_epu8(_mm512_add_epi8(lo, hi), _mm512_setzero_si512()); } // Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked // loads that read zeros, so any count is supported. LIBYUV_TARGET_AVX512BW uint32_t HammingDistance_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count) { __m512i sum = _mm512_setzero_si512(); while (count >= 64) { sum = _mm512_add_epi64( sum, BitCount64_AVX512BW(_mm512_xor_si512(_mm512_loadu_si512(src_a), _mm512_loadu_si512(src_b)))); src_a += 64; src_b += 64; count -= 64; } if (count > 0) { const __mmask64 k = LowBytesMask64(count); sum = _mm512_add_epi64(sum, BitCount64_AVX512BW(_mm512_xor_si512( _mm512_maskz_loadu_epi8(k, src_a), _mm512_maskz_loadu_epi8(k, src_b)))); } return (uint32_t)_mm512_reduce_add_epi64(sum); } #endif // HAS_HAMMINGDISTANCE_AVX512BW #if defined(HAS_SUMSQUAREERROR_SSE2) // Count is a multiple of 16. LIBYUV_TARGET_SSE2 uint32_t SumSquareError_SSE2(const uint8_t* src_a, const uint8_t* src_b, int count) { const __m128i zero = _mm_setzero_si128(); __m128i sum = _mm_setzero_si128(); int i; for (i = 0; i < count; i += 16) { __m128i a = _mm_loadu_si128((const __m128i*)(src_a + i)); __m128i b = _mm_loadu_si128((const __m128i*)(src_b + i)); __m128i d = _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a)); __m128i lo = _mm_unpacklo_epi8(d, zero); __m128i hi = _mm_unpackhi_epi8(d, zero); sum = _mm_add_epi32(sum, _mm_madd_epi16(lo, lo)); sum = _mm_add_epi32(sum, _mm_madd_epi16(hi, hi)); } sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0xee)); sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x01)); return (uint32_t)_mm_cvtsi128_si32(sum); } #endif // HAS_SUMSQUAREERROR_SSE2 #if defined(HAS_SUMSQUAREERROR_AVX2) // Count is a multiple of 32. LIBYUV_TARGET_AVX2 uint32_t SumSquareError_AVX2(const uint8_t* src_a, const uint8_t* src_b, int count) { const __m256i zero = _mm256_setzero_si256(); __m256i sum = _mm256_setzero_si256(); __m128i sum128; int i; for (i = 0; i < count; i += 32) { __m256i a = _mm256_loadu_si256((const __m256i*)(src_a + i)); __m256i b = _mm256_loadu_si256((const __m256i*)(src_b + i)); __m256i d = _mm256_or_si256(_mm256_subs_epu8(a, b), _mm256_subs_epu8(b, a)); __m256i lo = _mm256_unpacklo_epi8(d, zero); __m256i hi = _mm256_unpackhi_epi8(d, zero); sum = _mm256_add_epi32(sum, _mm256_madd_epi16(lo, lo)); sum = _mm256_add_epi32(sum, _mm256_madd_epi16(hi, hi)); } sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum), _mm256_extracti128_si256(sum, 1)); sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, 0xee)); sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, 0x01)); _mm256_zeroupper(); return (uint32_t)_mm_cvtsi128_si32(sum128); } #endif // HAS_SUMSQUAREERROR_AVX2 #if defined(HAS_SUMSQUAREERROR_AVX512BW) // Sum of squared differences of 64 bytes, as 16 dwords. LIBYUV_TARGET_AVX512BW static __m512i SumSquare64_AVX512BW(__m512i a, __m512i b) { const __m512i zero = _mm512_setzero_si512(); __m512i d = _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a)); __m512i lo = _mm512_unpacklo_epi8(d, zero); __m512i hi = _mm512_unpackhi_epi8(d, zero); return _mm512_add_epi32(_mm512_madd_epi16(lo, lo), _mm512_madd_epi16(hi, hi)); } // Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked // loads that read zeros, so any count is supported. LIBYUV_TARGET_AVX512BW uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count) { __m512i sum = _mm512_setzero_si512(); while (count >= 64) { sum = _mm512_add_epi32(sum, SumSquare64_AVX512BW(_mm512_loadu_si512(src_a), _mm512_loadu_si512(src_b))); src_a += 64; src_b += 64; count -= 64; } if (count > 0) { const __mmask64 k = LowBytesMask64(count); sum = _mm512_add_epi32( sum, SumSquare64_AVX512BW(_mm512_maskz_loadu_epi8(k, src_a), _mm512_maskz_loadu_epi8(k, src_b))); } return (uint32_t)_mm512_reduce_add_epi32(sum); } #endif // HAS_SUMSQUAREERROR_AVX512BW #if defined(HAS_HASHDJB2_SSE41) // 33 ^ (15 - i) for byte i of 16. static const uint32_t kHashMul16[16] = { 0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, // 33 ^ 15 .. 12 0x30f35d61, 0x855cb541, 0x040a9121, 0x747c7101, // 33 ^ 11 .. 8 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881, // 33 ^ 7 .. 4 0x00008c61, 0x00000441, 0x00000021, 0x00000001, // 33 ^ 3 .. 0 }; static const uint32_t kHash16x33 = 0x92d9e201; // 33 ^ 16 // Count is a multiple of 16. LIBYUV_TARGET_SSE41 uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed) { const __m128i zero = _mm_setzero_si128(); const __m128i mul0 = _mm_loadu_si128((const __m128i*)(kHashMul16)); const __m128i mul1 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 4)); const __m128i mul2 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 8)); const __m128i mul3 = _mm_loadu_si128((const __m128i*)(kHashMul16 + 12)); int i; for (i = 0; i < count; i += 16) { __m128i s = _mm_loadu_si128((const __m128i*)(src + i)); __m128i lo = _mm_unpacklo_epi8(s, zero); __m128i hi = _mm_unpackhi_epi8(s, zero); __m128i sum = _mm_add_epi32( _mm_add_epi32(_mm_mullo_epi32(_mm_unpacklo_epi16(lo, zero), mul0), _mm_mullo_epi32(_mm_unpackhi_epi16(lo, zero), mul1)), _mm_add_epi32(_mm_mullo_epi32(_mm_unpacklo_epi16(hi, zero), mul2), _mm_mullo_epi32(_mm_unpackhi_epi16(hi, zero), mul3))); sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x0e)); sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x01)); seed = seed * kHash16x33 + (uint32_t)_mm_cvtsi128_si32(sum); } return seed; } #endif // HAS_HASHDJB2_SSE41 #if defined(HAS_HASHDJB2_AVX2) || defined(HAS_HASHDJB2_AVX512BW) // HashDjb2 is hash = hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)). // The 32 bit multipliers 33 ^ (63 - i) are split into 16 bit halves so that // vpmaddwd can be used instead of vpmulld, which is 2 uops and 10 cycles. // m = hi * 65536 + lo, with lo treated as signed and hi adjusted by the carry, // so src * m = src * lo + ((src * hi) << 16) modulo 2 ^ 32. // The first 64 are lo and the next 64 are hi. static const uint16_t kHashMulLoHi[128] = { // lo for 33 ^ 63 .. 33 ^ 0 0x0be1, 0x93c1, 0x9fa1, 0xaf81, 0x4361, 0xdb41, 0xf721, 0x1701, // 0xbae1, 0x62c1, 0x8ea1, 0xbe81, 0x7261, 0x2a41, 0x6621, 0xa601, // 0x69e1, 0x31c1, 0x7da1, 0xcd81, 0xa161, 0x7941, 0xd521, 0x3501, // 0x18e1, 0x00c1, 0x6ca1, 0xdc81, 0xd061, 0xc841, 0x4421, 0xc401, // 0xc7e1, 0xcfc1, 0x5ba1, 0xeb81, 0xff61, 0x1741, 0xb321, 0x5301, // 0x76e1, 0x9ec1, 0x4aa1, 0xfa81, 0x2e61, 0x6641, 0x2221, 0xe201, // 0x25e1, 0x6dc1, 0x39a1, 0x0981, 0x5d61, 0xb541, 0x9121, 0x7101, // 0xd4e1, 0x3cc1, 0x28a1, 0x1881, 0x8c61, 0x0441, 0x0021, 0x0001, // // hi for 33 ^ 63 .. 33 ^ 0 0xd8be, 0xef4c, 0x6458, 0x6022, 0x40f9, 0x193e, 0x6d5f, 0x31dc, // 0x0945, 0xc238, 0x1567, 0xa38f, 0x3380, 0x301b, 0x664e, 0x031a, // 0xd18c, 0xb8c6, 0x79f6, 0xae5d, 0x0549, 0xc219, 0x7a3f, 0x1338, // 0x0095, 0xc1f5, 0xa107, 0xaf8c, 0x0552, 0x3e39, 0x3830, 0x1138, // 0x655f, 0xccc5, 0x9999, 0xd61c, 0x829c, 0x1379, 0x2f23, 0xac18, // 0xcee9, 0xc836, 0x72ac, 0x510d, 0xcc27, 0xb0da, 0xee16, 0x92da, // 0x0c35, 0xa347, 0x3b40, 0x4f5f, 0x30f3, 0x855d, 0x040b, 0x747c, // 0xec42, 0x4cfa, 0x0255, 0x0012, 0x0001, 0x0000, 0x0000, 0x0000, // }; #endif #if defined(HAS_HASHDJB2_AVX2) // lo + (hi << 16), then add 8 dwords. LIBYUV_TARGET_AVX2 static uint32_t HashSum_AVX2(__m256i lo, __m256i hi) { __m256i s = _mm256_add_epi32(lo, _mm256_slli_epi32(hi, 16)); __m128i s128 = _mm_add_epi32(_mm256_castsi256_si128(s), _mm256_extracti128_si256(s, 1)); s128 = _mm_add_epi32(s128, _mm_shuffle_epi32(s128, 0x0e)); s128 = _mm_add_epi32(s128, _mm_shuffle_epi32(s128, 0x01)); return (uint32_t)_mm_cvtsi128_si32(s128); } // Count is a multiple of 32. Process 64 bytes per loop with one horizontal // add, then 32 bytes if needed. LIBYUV_TARGET_AVX2 uint32_t HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed) { const __m256i* m = (const __m256i*)kHashMulLoHi; while (count >= 64) { __m256i w0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src)); __m256i w1 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16))); __m256i w2 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 32))); __m256i w3 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 48))); __m256i lo = _mm256_add_epi32( _mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 0)), _mm256_madd_epi16(w1, _mm256_loadu_si256(m + 1))), _mm256_add_epi32(_mm256_madd_epi16(w2, _mm256_loadu_si256(m + 2)), _mm256_madd_epi16(w3, _mm256_loadu_si256(m + 3)))); __m256i hi = _mm256_add_epi32( _mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 4)), _mm256_madd_epi16(w1, _mm256_loadu_si256(m + 5))), _mm256_add_epi32(_mm256_madd_epi16(w2, _mm256_loadu_si256(m + 6)), _mm256_madd_epi16(w3, _mm256_loadu_si256(m + 7)))); seed = seed * 0xf07f8801u + HashSum_AVX2(lo, hi); // 33 ^ 64 src += 64; count -= 64; } if (count > 0) { __m256i w0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src)); __m256i w1 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16))); __m256i lo = _mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 2)), _mm256_madd_epi16(w1, _mm256_loadu_si256(m + 3))); __m256i hi = _mm256_add_epi32(_mm256_madd_epi16(w0, _mm256_loadu_si256(m + 6)), _mm256_madd_epi16(w1, _mm256_loadu_si256(m + 7))); seed = seed * 0x1137c401u + HashSum_AVX2(lo, hi); // 33 ^ 32 } _mm256_zeroupper(); return seed; } #endif // HAS_HASHDJB2_AVX2 #if defined(HAS_HASHDJB2_AVX512BW) static uint32_t Pow33(int n) { uint32_t r = 1u; uint32_t b = 33u; while (n) { if (n & 1) { r *= b; } b *= b; n >>= 1; } return r; } // Sum of 64 bytes, zero extended to words in w0 and w1, times 33 ^ (63 - i). LIBYUV_TARGET_AVX512BW static uint32_t HashSum64_AVX512BW(__m512i w0, __m512i w1) { const __m512i* m = (const __m512i*)kHashMulLoHi; __m512i lo = _mm512_add_epi32(_mm512_madd_epi16(w0, _mm512_loadu_si512(m)), _mm512_madd_epi16(w1, _mm512_loadu_si512(m + 1))); __m512i hi = _mm512_add_epi32(_mm512_madd_epi16(w0, _mm512_loadu_si512(m + 2)), _mm512_madd_epi16(w1, _mm512_loadu_si512(m + 3))); return (uint32_t)_mm512_reduce_add_epi32( _mm512_add_epi32(lo, _mm512_slli_epi32(hi, 16))); } // hash = hash * 33 ^ 64 + sum(src[i] * 33 ^ (63 - i)) for each 64 bytes. // The remainder of n = 1 to 63 bytes is loaded into the high n bytes with a // masked load ending at src + count, so the same multipliers apply, and // hash is multiplied by 33 ^ n. LIBYUV_TARGET_AVX512BW uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) { int n; while (count >= 64) { seed = seed * 0xf07f8801u + // 33 ^ 64 HashSum64_AVX512BW( _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src)), _mm512_cvtepu8_epi16( _mm256_loadu_si256((const __m256i*)(src + 32)))); src += 64; count -= 64; } n = count; if (n) { const __mmask64 k = (__mmask64)~LowBytesMask64(64 - n); __m512i s = _mm512_maskz_loadu_epi8(k, src + n - 64); seed = seed * Pow33(n) + HashSum64_AVX512BW( _mm512_cvtepu8_epi16(_mm512_castsi512_si256(s)), _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(s, 1))); } return seed; } #endif // HAS_HASHDJB2_AVX512BW #ifdef __cplusplus } // extern "C" } // namespace libyuv #endif #endif // !defined(LIBYUV_DISABLE_X86) && (x86 or x64) && // (Visual C or LIBYUV_ENABLE_ROWWIN)