From 59fb9e18a848a25b0ad4686e4328edd747363289 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Thu, 29 Oct 2026 17:18:55 -0700 Subject: [PATCH] [libyuv] Fix MSan AVX-512 intrinsics and standalone dynamic linker Clang MemorySanitizer builds define LIBYUV_ENABLE_ROWWIN and compile row_win.cc and scale_win.cc instead of inline assembly. Replace three families of AVX-512 intrinsics that LLVM MSan does not model bitwise or lane-by-lane: - _mm512_ternarylogic_epi32 (@llvm.x86.avx512.pternlog.d.512) in ARGBGrayRow_AVX512VNNI, ARGBToUV444MatrixRow_AVX512VNNIVBMI, and I422ToAR30Row_AVX512BW, which raises false use-of-uninitialized-value errors whenever any bit of any operand is uninitialized (such as the unused alpha channel of an RGBX buffer). Use _mm512_and_si512, _mm512_andnot_si512, _mm512_or_si512, _mm512_xor_si512, and _mm512_shuffle_i64x2, which LLVM MSan models bit-by-bit. - _mm512_permutexvar_epi{8,16,32,64} and _mm512_mask_permutexvar_epi{8,32} (@llvm.x86.avx512.permvar.*) across row_win.cc and ScaleUVAddCols3_AVX512VBMI in scale_win.cc, which OR input shadows without permuting them and poison initialized output lanes whenever any lane of the input vector is uninitialized. Route through PermutexvarEpi* and MaskPermutexvarEpi* helpers that use portable lane-by-lane C loops under LIBYUV_MSAN and the native AVX-512 intrinsics otherwise. - _mm512_cvttps_epi32 (@llvm.x86.avx512.mask.cvttps2dq.512) in ARGBAlphaBlendRow_AVX512BW. Route through CvttpsEpi32, which uses a portable lane-by-lane C loop under LIBYUV_MSAN and _mm512_cvttps_epi32 otherwise. In standalone MSan builds (!build_with_chromium && prebuilt_instrumented_libraries_available && is_msan), override the relative ELF PT_INTERP from //third_party/instrumented_libs with the absolute path to the instrumented ld-linux-x86-64.so.2 so test and utility binaries launch cleanly from any working directory (fixing the LUCI Linux MSan test runner FileNotFoundError). Add MSan regression tests with poisoned RGBX alpha channels, poisoned tail padding, and poisoned destination buffers in planar_test.cc, and document the AVX-512 MSan intrinsic guidelines in GEMINI.md. Test: out/msan/libyuv_unittest Bug: libyuv:388428508 Change-Id: I6a002393db2d2e629cd7a3cf2bf4d98ee9daf79d Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8686036 Reviewed-by: richard winterton --- BUILD.gn | 46 +++++ GEMINI.md | 53 +++-- source/row_win.cc | 414 +++++++++++++++++++++++---------------- source/scale_win.cc | 74 ++++--- unit_test/planar_test.cc | 310 +++++++++++++++++++++++++++++ 5 files changed, 694 insertions(+), 203 deletions(-) diff --git a/BUILD.gn b/BUILD.gn index c055dc9df..3c37e0591 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -7,6 +7,7 @@ # be found in the AUTHORS file in the root of the source tree. import("//build/config/features.gni") +import("//build/config/sanitizers/sanitizers.gni") import("//testing/test.gni") import("libyuv.gni") @@ -383,6 +384,23 @@ if (libyuv_include_tests) { defines = [ "GTEST_RELATIVE_PATH" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + # Override the relative PT_INTERP from //third_party/instrumented_libs so + # standalone test binaries can be launched from any working directory + # (such as the builder root on LUCI Linux MSan). + config("libyuv_msan_ldflags") { + if (msan_track_origins == 0) { + _msan_archive = "msan-no-origins-noble-lib" + } else { + _msan_archive = "msan-chained-origins-noble-lib" + } + _ld_linux = rebase_path( + "//third_party/instrumented_libs/binaries/$_msan_archive/lib/ld-linux-x86-64.so.2") + ldflags = [ "-Wl,--dynamic-linker=$_ld_linux" ] + } + } + test("libyuv_unittest") { testonly = true @@ -423,6 +441,10 @@ if (libyuv_include_tests) { } configs += [ ":libyuv_unittest_warnings_config" ] + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } public_deps = [ "//testing/gtest" ] public_configs = [ ":libyuv_unittest_config" ] @@ -466,6 +488,10 @@ if (libyuv_include_tests) { if (is_linux || is_chromeos) { cflags = [ "-fexceptions" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("yuvconvert") { @@ -477,6 +503,10 @@ if (libyuv_include_tests) { if (is_linux || is_chromeos) { cflags = [ "-fexceptions" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("yuvconstants") { @@ -488,6 +518,10 @@ if (libyuv_include_tests) { if (is_linux || is_chromeos) { cflags = [ "-fexceptions" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("psnr") { @@ -502,6 +536,10 @@ if (libyuv_include_tests) { if (!is_ios && !libyuv_disable_jpeg) { defines = [ "HAVE_JPEG" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("i444tonv12_eg") { @@ -510,6 +548,10 @@ if (libyuv_include_tests) { "util/i444tonv12_eg.cc", ] deps = [ ":libyuv" ] + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("cpuid") { @@ -518,6 +560,10 @@ if (libyuv_include_tests) { "util/cpuid.c", ] deps = [ ":libyuv" ] + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } } diff --git a/GEMINI.md b/GEMINI.md index 923077777..d9cfd977a 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -101,10 +101,11 @@ from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing `"ymmN"` or `"zmmN"`, including for AVX2 and AVX512 code. Clang for Windows must preserve xmm6-xmm15 and does not save them when they are declared as ymm or zmm. -4. **MSan Intrinsics (`row_win.cc` / `scale_win.cc`)**: Clang builds with - MemorySanitizer (`-fsanitize=memory`) define `LIBYUV_ENABLE_ROWWIN` and - compile `row_win.cc` and `scale_win.cc` instead of inline assembly. Avoid - AVX-512 intrinsics that LLVM's MSan pass does not instrument: +4. **MSan Intrinsics (`*_win.cc`)**: Clang builds with MemorySanitizer + (`-fsanitize=memory`) define `LIBYUV_ENABLE_ROWWIN` and compile `*_win.cc` + (`row_win.cc`, `scale_win.cc`, `compare_win.cc`, `rotate_win.cc`) instead of + inline assembly (`*_gcc.cc`). When porting `*_gcc.cc` to `*_win.cc`, never + use AVX-512 intrinsics that LLVM's MSan pass does not model: * **Down-converting masked stores**: Never use `_mm512_mask_cvtepi32_storeu_epi8`, `_mm512_mask_cvtepi64_storeu_epi8`, `_mm512_mask_cvtepi16_storeu_epi8`, or @@ -115,14 +116,42 @@ from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing store (e.g. `_mm_mask_storeu_epi8(dst, mask, _mm512_cvtepi32_epi8(v))`), which lowers to `@llvm.masked.store` for MSan and still compiles to a single `vpmovdb` instruction at `-O2`. - * **Ternary logic (`_mm512_ternarylogic_epi32` / `epi64`) on partially - uninitialized vectors**: LLVM MSan does not model `vpternlog` truth - tables bitwise and flags `use-of-uninitialized-value` if any bit of any - of the three inputs is uninitialized, even if masked off by `imm8`. For - partial channel updates (e.g. writing alpha into `dst` in - `ARGBCopyYToAlphaRow_AVX512BW`), use byte-masked stores - (`_mm512_mask_storeu_epi8`) instead of reading `dst` and blending with - `_mm512_ternarylogic_epi32`. + * **Ternary logic (`_mm*_ternarylogic_epi32` / `epi64`)**: Never use + `_mm512_ternarylogic_epi32`/`epi64` (or `_mm256_`/`_mm_` variants) + anywhere in `*_win.cc` when porting `vpternlogd`/`vpternlogq` from + `*_gcc.cc`. They lower to `@llvm.x86.avx512.pternlog.*`, which LLVM MSan + does not model bitwise—`handleUnknownIntrinsic` traps immediately with + `use-of-uninitialized-value` if *any* bit of any of the three inputs is + uninitialized (such as the unused alpha byte of an `RGBX` buffer or tail + padding), even if masked off by `imm8` or merely copied to memory. + * For boolean blends and inversions (e.g. `ARGBGrayRow_AVX512VNNI`, + `I422ToAR30Row_AVX512BW`, `ARGBToUV444MatrixRow_AVX512VNNIVBMI`), + use standard bitwise intrinsics (`_mm512_and_si512`, + `_mm512_andnot_si512`, `_mm512_or_si512`, `_mm512_xor_si512`), which + LLVM MSan models bit-by-bit and Clang `-O2` folds back into + `vpternlog` in non-MSan builds. + * For write-only partial channel updates into `dst` (e.g. writing + alpha in `ARGBCopyYToAlphaRow_AVX512BW`), use byte-masked stores + (`_mm512_mask_storeu_epi8`) instead of reading `dst` and blending. + * **Single-source variable permutes (`_mm512_permutexvar_epi*` / + `_mm512_mask_permutexvar_epi*`)**: Never use + `_mm512_permutexvar_epi{8,16,32,64}` or + `_mm512_mask_permutexvar_epi{8,16,32,64}` directly. They lower to + `@llvm.x86.avx512.permvar.*`, which LLVM MSan does not model + lane-by-lane—instead it ORs the input shadows without permuting them, + poisoning initialized output lanes whenever *any* lane of the input + vector is uninitialized (e.g. `_mm512_castsi128_si512`, + `_mm512_castsi256_si512`, unused `RGBX` alpha bytes, or uninitialized + tail lanes). Use the `PermutexvarEpi*` and `MaskPermutexvarEpi*` + helpers in `row_win.cc` / `scale_win.cc`, which use portable + lane-by-lane C loops under `LIBYUV_MSAN` and the native intrinsics + otherwise. + * **512-bit float-to-int truncation (`_mm512_cvttps_epi32`)**: Never use + `_mm512_cvttps_epi32` directly; it lowers to + `@llvm.x86.avx512.mask.cvttps2dq.512`, which LLVM MSan does not model + lane-by-lane. Use the `CvttpsEpi32` helper in `row_win.cc`, which uses + a portable lane-by-lane C loop under `LIBYUV_MSAN` and + `_mm512_cvttps_epi32` otherwise. ### AArch64 diff --git a/source/row_win.cc b/source/row_win.cc index a986d79ef..69873fafb 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -53,6 +53,118 @@ extern "C" { #define LIBYUV_TARGET_AVX512VNNIVBMI #endif // defined(__clang__) || defined(__GNUC__) +#if defined(__x86_64__) || defined(_M_X64) +// Under MemorySanitizer, use portable lane-by-lane C loops for single-source +// variable permutes and 512-bit float-to-int truncation so shadow propagation +// relies only on standard C memory/array semantics rather than target-specific +// LLVM intrinsic modeling. In non-MSan builds, call the native AVX-512 +// intrinsics directly. +LIBYUV_TARGET_AVX512VBMI +static inline __m512i PermutexvarEpi8(__m512i idx, __m512i v) { +#if defined(LIBYUV_MSAN) + uint8_t idx_arr[64]; + uint8_t v_arr[64]; + uint8_t out_arr[64]; + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 64; ++i) { + out_arr[i] = v_arr[idx_arr[i] & 63]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_permutexvar_epi8(idx, v); +#endif +} + +LIBYUV_TARGET_AVX512VBMI +static inline __m512i MaskPermutexvarEpi8(__m512i src, + __mmask64 k, + __m512i idx, + __m512i v) { +#if defined(LIBYUV_MSAN) + uint8_t src_arr[64]; + uint8_t idx_arr[64]; + uint8_t v_arr[64]; + uint8_t out_arr[64]; + _mm512_storeu_si512((__m512i*)src_arr, src); + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 64; ++i) { + out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 63] : src_arr[i]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_mask_permutexvar_epi8(src, k, idx, v); +#endif +} + +LIBYUV_TARGET_AVX512BW +static inline __m512i PermutexvarEpi16(__m512i idx, __m512i v) { +#if defined(LIBYUV_MSAN) + uint16_t idx_arr[32]; + uint16_t v_arr[32]; + uint16_t out_arr[32]; + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 32; ++i) { + out_arr[i] = v_arr[idx_arr[i] & 31]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_permutexvar_epi16(idx, v); +#endif +} + +LIBYUV_TARGET_AVX512BW +static inline __m512i PermutexvarEpi32(__m512i idx, __m512i v) { +#if defined(LIBYUV_MSAN) + uint32_t idx_arr[16]; + uint32_t v_arr[16]; + uint32_t out_arr[16]; + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 16; ++i) { + out_arr[i] = v_arr[idx_arr[i] & 15]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_permutexvar_epi32(idx, v); +#endif +} + +LIBYUV_TARGET_AVX512BW +static inline __m512i PermutexvarEpi64(__m512i idx, __m512i v) { +#if defined(LIBYUV_MSAN) + uint64_t idx_arr[8]; + uint64_t v_arr[8]; + uint64_t out_arr[8]; + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 8; ++i) { + out_arr[i] = v_arr[idx_arr[i] & 7]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_permutexvar_epi64(idx, v); +#endif +} + +LIBYUV_TARGET_AVX512BW +static inline __m512i CvttpsEpi32(__m512 v) { +#if defined(LIBYUV_MSAN) + float v_arr[16]; + int32_t out_arr[16]; + _mm512_storeu_ps(v_arr, v); + for (int i = 0; i < 16; ++i) { + out_arr[i] = (int32_t)v_arr[i]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_cvttps_epi32(v); +#endif +} +#endif // defined(__x86_64__) || defined(_M_X64) + // Read 8 UV from 444 #define READYUV444 \ xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \ @@ -444,10 +556,10 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128)); - __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); - __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); @@ -484,10 +596,10 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, __m512i zmm_raw1 = _mm512_maskz_loadu_epi8(src_mask1, src_raw + 64); __m512i zmm_raw2 = _mm512_maskz_loadu_epi8(src_mask2, src_raw + 128); - __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); - __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); @@ -635,10 +747,10 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128)); - __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); - __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); @@ -742,10 +854,10 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_rgb + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_rgb + 128)); - __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); - __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); @@ -777,7 +889,7 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, zmm2 = _mm512_srli_epi16(zmm2, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm2); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm512_storeu_si512((__m512i*)dst_y, zmm0); @@ -830,7 +942,7 @@ void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm2 = _mm512_srli_epi16(zmm2, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm2); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm512_storeu_si512((__m512i*)dst_y, zmm0); @@ -947,7 +1059,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm0 = _mm512_srli_epi16(zmm0, 8); zmm2 = _mm512_srli_epi16(zmm2, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm2); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm512_storeu_si512((__m512i*)dst_u, zmm0); zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_v); @@ -962,7 +1074,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm0 = _mm512_srli_epi16(zmm0, 8); zmm2 = _mm512_srli_epi16(zmm2, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm2); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm512_storeu_si512((__m512i*)dst_v, zmm0); src_argb += 256; @@ -978,7 +1090,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, // Requires AVX512VNNI and AVX512VBMI. // U = (kAddUV - dot(kRGBToU, argb)) >> 8. The accumulators start at // ~kAddUV so they hold ~(kAddUV - dot) and U is the inverse of byte 1 of each -// int32. The inverse is folded into the ternary logic that merges the halves. +// int32. LIBYUV_TARGET_AVX512VNNIVBMI void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, uint8_t* dst_u, @@ -992,12 +1104,10 @@ void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); __m512i zmm_v = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + const __m512i zmm_neg1 = _mm512_set1_epi32(-1); __m512i zmm_add = _mm512_xor_si512( _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)), - _mm512_set1_epi32(-1)); // ~kAddUV - // Select low 256 bits from the first operand, high 256 bits from the second. - __m512i zmm_sel = - _mm512_inserti64x4(_mm512_setzero_si512(), _mm256_set1_epi32(-1), 1); + zmm_neg1); // ~kAddUV while (width > 0) { __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb); @@ -1014,12 +1124,15 @@ void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, __m512i zmm6 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_v); __m512i zmm7 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_v); - zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // ~U 0..31 - zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // ~U 32..63 - zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); // ~V 0..31 - zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); // ~V 32..63 - zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm2, zmm_sel, 0x27); // U - zmm4 = _mm512_ternarylogic_epi32(zmm4, zmm6, zmm_sel, 0x27); // V + zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // ~U 0..31 + zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // ~U 32..63 + zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); // ~V 0..31 + zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); // ~V 32..63 + // Avoid _mm512_ternarylogic_epi32 (unmodeled by LLVM MSan; see GEMINI.md). + zmm0 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm0, zmm2, 0x44), + zmm_neg1); // U + zmm4 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm4, zmm6, 0x44), + zmm_neg1); // V _mm512_storeu_si512((__m512i*)dst_u, zmm0); _mm512_storeu_si512((__m512i*)dst_v, zmm4); @@ -1086,7 +1199,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); zmm0 = _mm512_packus_epi16(zmm0, zmm1); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0); zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u); zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v); @@ -1094,7 +1207,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm0 = _mm512_sub_epi16(zmm_8000, zmm0); zmm0 = _mm512_srli_epi16(zmm0, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm0); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0)); _mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1)); @@ -1140,7 +1253,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); zmm0 = _mm512_packus_epi16(zmm0, zmm1); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0); zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u); zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v); @@ -1148,7 +1261,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm0 = _mm512_sub_epi16(zmm_8000, zmm0); zmm0 = _mm512_srli_epi16(zmm0, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm0); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); int halfwidth = (width + 1) >> 1; __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u); @@ -1216,7 +1329,7 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); zmm0 = _mm512_packus_epi16(zmm0, zmm1); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0); zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V @@ -1268,7 +1381,7 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); zmm0 = _mm512_packus_epi16(zmm0, zmm1); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0); zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V @@ -1490,7 +1603,7 @@ void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { v = _mm512_or_si512(_mm512_srli_epi16(v, 8), _mm512_slli_epi16(v, 8)); __m512i idx = _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)((64 - width) >> 1))); - v = _mm512_permutexvar_epi16(idx, v); + v = PermutexvarEpi16(idx, v); if (width & 1) { __m512i v_hi = _mm512_alignr_epi64(v, v, 2); v = _mm512_alignr_epi8(v_hi, v, 1); @@ -1518,7 +1631,7 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv, s -= 64; __m512i v = _mm512_loadu_si512((const __m512i*)s); v = _mm512_shuffle_epi8(v, zmm_shuf); - v = _mm512_permutexvar_epi64(zmm_perm, v); + v = PermutexvarEpi64(zmm_perm, v); _mm256_storeu_si256((__m256i*)dst_u, _mm512_castsi512_si256(v)); _mm256_storeu_si256((__m256i*)dst_v, _mm512_extracti64x4_epi64(v, 1)); dst_u += 32; @@ -1530,7 +1643,7 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv, __m512i idx = _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)(32 - width))); __m512i v = _mm512_maskz_loadu_epi16(k, src_uv); - v = _mm512_permutexvar_epi16(idx, v); + v = PermutexvarEpi16(idx, v); _mm256_mask_storeu_epi8(dst_u, k, _mm512_cvtepi16_epi8(v)); _mm256_mask_storeu_epi8(dst_v, k, _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8))); @@ -1549,7 +1662,7 @@ void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { while (width >= 32) { s -= 64; __m512i zmm0 = _mm512_loadu_si512((const __m512i*)s); - zmm0 = _mm512_permutexvar_epi16(zmm_idx, zmm0); + zmm0 = PermutexvarEpi16(zmm_idx, zmm0); _mm512_storeu_si512((__m512i*)dst_uv, zmm0); dst_uv += 64; width -= 32; @@ -1559,7 +1672,7 @@ void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { __m512i idx = _mm512_sub_epi16(zmm_idx, _mm512_set1_epi16((short)(32 - width))); __m512i zmm0 = _mm512_maskz_loadu_epi16(k, src_uv); - _mm512_mask_storeu_epi16(dst_uv, k, _mm512_permutexvar_epi16(idx, zmm0)); + _mm512_mask_storeu_epi16(dst_uv, k, PermutexvarEpi16(idx, zmm0)); } _mm256_zeroupper(); } @@ -2707,15 +2820,15 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y, src_u += 16; \ __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \ __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \ - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \ - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \ + zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \ + zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \ __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \ zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \ zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \ __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); \ src_y += 32; \ __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \ - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \ + zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \ zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); @@ -2726,14 +2839,14 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y, __m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); \ __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \ __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \ - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \ - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \ + zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \ + zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \ __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \ zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \ zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \ __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); \ __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \ - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \ + zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \ zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); @@ -2996,8 +3109,8 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); + zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); + zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); @@ -3006,7 +3119,7 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); src_y += 32; __m512i zmm4 = _mm512_castsi256_si512(ymm_y); - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); + zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); @@ -3036,8 +3149,9 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, zmm0 = _mm512_srli_epi16(zmm0, 4); zmm3 = _mm512_srli_epi16(zmm1, 10); zmm1 = _mm512_slli_epi16(zmm1, 6); - zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm1, zmm_fc00, 0xf8); - zmm2 = _mm512_ternarylogic_epi32(zmm2, zmm3, zmm_3ff0, 0xe4); + zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00)); + zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0), + _mm512_andnot_si512(zmm_3ff0, zmm3)); zmm2 = _mm512_or_si512(zmm2, zmm_c000); __m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2); @@ -3058,8 +3172,8 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); + zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); + zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); @@ -3067,7 +3181,7 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); __m512i zmm4 = _mm512_castsi256_si512(ymm_y); - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); + zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); @@ -3094,8 +3208,9 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, zmm0 = _mm512_srli_epi16(zmm0, 4); zmm3 = _mm512_srli_epi16(zmm1, 10); zmm1 = _mm512_slli_epi16(zmm1, 6); - zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm1, zmm_fc00, 0xf8); - zmm2 = _mm512_ternarylogic_epi32(zmm2, zmm3, zmm_3ff0, 0xe4); + zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00)); + zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0), + _mm512_andnot_si512(zmm_3ff0, zmm3)); zmm2 = _mm512_or_si512(zmm2, zmm_c000); __m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2); @@ -3420,7 +3535,7 @@ void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { s -= 64; _mm512_storeu_si512( (__m512i*)dst, - _mm512_permutexvar_epi32(rev, _mm512_loadu_si512((const __m512i*)s))); + PermutexvarEpi32(rev, _mm512_loadu_si512((const __m512i*)s))); dst += 64; width -= 16; } @@ -3429,7 +3544,7 @@ void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { __mmask16 mask = (__mmask16)((1u << width) - 1u); __m512i idx = _mm512_sub_epi32(rev, _mm512_set1_epi32(16 - width)); __m512i v = _mm512_maskz_loadu_epi32(mask, src); - _mm512_mask_storeu_epi32(dst, mask, _mm512_permutexvar_epi32(idx, v)); + _mm512_mask_storeu_epi32(dst, mask, PermutexvarEpi32(idx, v)); } _mm256_zeroupper(); } @@ -3504,12 +3619,12 @@ void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, __m512i t0 = _mm512_sub_epi8(idx0, _mm512_set1_epi8((char)(off - 64))); __m512i t1 = _mm512_sub_epi8(idx1, _mm512_set1_epi8((char)off)); __m512i t2 = _mm512_sub_epi8(idx2, _mm512_set1_epi8((char)off)); - __m512i d0 = _mm512_mask_permutexvar_epi8( - _mm512_permutex2var_epi8(a, t0, b), _mm512_movepi8_mask(t0), t0, c); - __m512i d1 = _mm512_mask_permutexvar_epi8( - _mm512_permutex2var_epi8(a, t1, b), _mm512_movepi8_mask(t1), t1, c); - __m512i d2 = _mm512_mask_permutexvar_epi8( - _mm512_permutex2var_epi8(a, t2, b), _mm512_movepi8_mask(t2), t2, c); + __m512i d0 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t0, b), + _mm512_movepi8_mask(t0), t0, c); + __m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t1, b), + _mm512_movepi8_mask(t1), t1, c); + __m512i d2 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t2, b), + _mm512_movepi8_mask(t2), t2, c); _mm512_mask_storeu_epi8(dst_rgb24, m0, d0); _mm512_mask_storeu_epi8(dst_rgb24 + 64, m1, d1); _mm512_mask_storeu_epi8(dst_rgb24 + 128, m2, d2); @@ -4083,8 +4198,7 @@ void MergeRGBRow_AVX512VBMI(const uint8_t* src_r, src_g += 64; src_b += 64; __m512i d0 = _mm512_permutex2var_epi8(rg0, m_rgb0, br); - __m512i d1 = - _mm512_mask_permutexvar_epi8(_mm512_permutex2var_epi8(rg0, m_rgb1, br), + __m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(rg0, m_rgb1, br), 0xb6db6db600000000ull, m_rgb1, gb1); __m512i d2 = _mm512_permutex2var_epi8(br, m_rgb2, gb1); _mm512_storeu_si512((__m512i*)dst_rgb, d0); @@ -4150,7 +4264,7 @@ void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, __m512i s1 = _mm512_castsi256_si512( _mm256_loadu_si256((const __m256i*)(src_raw + 64))); src_raw += 96; - __m512i d0 = _mm512_or_si512(_mm512_permutexvar_epi8(p0, s0), alpha); + __m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha); __m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha); _mm512_storeu_si512((__m512i*)dst_rgba, d0); _mm512_storeu_si512((__m512i*)(dst_rgba + 64), d1); @@ -4164,7 +4278,7 @@ void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, __m512i s0 = _mm512_maskz_loadu_epi8(ks0, src_raw); __m512i s1 = _mm512_castsi256_si512(_mm256_maskz_loadu_epi8(ks1, src_raw + 64)); - __m512i d0 = _mm512_or_si512(_mm512_permutexvar_epi8(p0, s0), alpha); + __m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha); __m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha); uint32_t m32 = (1u << width) - 1u; _mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, d0); @@ -4208,7 +4322,7 @@ void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) { while (width >= 32) { src -= 32; __m512i v = _mm512_loadu_si512((const __m512i*)src); - _mm512_storeu_si512((__m512i*)dst, _mm512_permutexvar_epi16(rev, v)); + _mm512_storeu_si512((__m512i*)dst, PermutexvarEpi16(rev, v)); dst += 32; width -= 32; } @@ -4216,7 +4330,7 @@ void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) { __mmask32 k = (__mmask32)((1u << width) - 1u); __m512i idx = _mm512_sub_epi16(rev, _mm512_set1_epi16((short)(32 - width))); __m512i v = _mm512_maskz_loadu_epi16(k, src - width); - _mm512_mask_storeu_epi16(dst, k, _mm512_permutexvar_epi16(idx, v)); + _mm512_mask_storeu_epi16(dst, k, PermutexvarEpi16(idx, v)); } _mm256_zeroupper(); } @@ -4402,7 +4516,7 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win); const __m512i zmm5 = _mm512_set1_epi32(-1); while (width >= 32) { - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf))); __m512i zmm4 = @@ -4425,10 +4539,8 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra)); _mm512_storeu_si512((__m512i*)(dst_argb + 64), _mm512_unpackhi_epi16(bg, ra)); @@ -4438,7 +4550,7 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, if (width > 0) { uint32_t m32 = (1u << width) - 1u; __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf))); __m512i zmm4 = @@ -4459,10 +4571,8 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, _mm512_unpacklo_epi16(bg, ra)); _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), @@ -4496,7 +4606,7 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win); const __m512i zmm5 = _mm512_set1_epi32(-1); while (width >= 32) { - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_or_si256( _mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8), @@ -4521,10 +4631,8 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra)); _mm512_storeu_si512((__m512i*)(dst_argb + 64), _mm512_unpackhi_epi16(bg, ra)); @@ -4534,7 +4642,7 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, if (width > 0) { uint32_t m32 = (1u << width) - 1u; __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_or_si256( _mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8), @@ -4557,10 +4665,8 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, _mm512_unpacklo_epi16(bg, ra)); _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), @@ -4621,7 +4727,7 @@ void NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, const __m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1_NV); while (width >= 32) { - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf))); __m512i zmm4 = @@ -4653,7 +4759,7 @@ void NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, if (width > 0) { uint32_t m32 = (1u << width) - 1u; __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf))); __m512i zmm4 = @@ -4711,7 +4817,7 @@ void NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, const __m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1_NV); while (width >= 32) { - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_or_si256( _mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8), @@ -4745,7 +4851,7 @@ void NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, if (width > 0) { uint32_t m32 = (1u << width) - 1u; __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_or_si256( _mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8), @@ -4811,8 +4917,7 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, __m256i yuv = _mm256_inserti128_si256( _mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_castsi256_si512(yuv)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); __m512i zmm5 = _mm512_packus_epi16( @@ -4836,10 +4941,8 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra)); _mm512_storeu_si512((__m512i*)(dst_argb + 64), _mm512_unpackhi_epi16(bg, ra)); @@ -4854,8 +4957,7 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, __m256i yuv = _mm256_inserti128_si256( _mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_castsi256_si512(yuv)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); __m512i zmm5 = _mm512_packus_epi16( @@ -4877,10 +4979,8 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, _mm512_unpacklo_epi16(bg, ra)); _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), @@ -4918,8 +5018,7 @@ void I400ToARGBRow_AVX512BW(const uint8_t* y_buf, y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8)); y16 = _mm512_srai_epi16( _mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6); - __m512i g = - _mm512_permutexvar_epi64(zmm_lin, _mm512_packus_epi16(y16, y16)); + __m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16)); __m256i g256 = _mm512_castsi512_si256(g); __m512i d0 = _mm512_or_si512( _mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)), @@ -4941,8 +5040,7 @@ void I400ToARGBRow_AVX512BW(const uint8_t* y_buf, y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8)); y16 = _mm512_srai_epi16( _mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6); - __m512i g = - _mm512_permutexvar_epi64(zmm_lin, _mm512_packus_epi16(y16, y16)); + __m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16)); __m256i g256 = _mm512_castsi512_si256(g); __m512i d0 = _mm512_or_si512( _mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)), @@ -5111,8 +5209,8 @@ void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, src_argb += 64; __m512i y32 = _mm512_srli_epi32( _mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8); - __m512i out = _mm512_ternarylogic_epi32(_mm512_shuffle_epi8(y32, c_shuf), s, - c_alpha, 0xf8); + __m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf), + _mm512_and_si512(s, c_alpha)); _mm512_storeu_si512((__m512i*)dst_argb, out); dst_argb += 64; width -= 16; @@ -5122,8 +5220,8 @@ void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, __m512i s = _mm512_maskz_loadu_epi32(k, src_argb); __m512i y32 = _mm512_srli_epi32( _mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8); - __m512i out = _mm512_ternarylogic_epi32(_mm512_shuffle_epi8(y32, c_shuf), s, - c_alpha, 0xf8); + __m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf), + _mm512_and_si512(s, c_alpha)); _mm512_mask_storeu_epi32(dst_argb, k, out); } _mm256_zeroupper(); @@ -5377,7 +5475,7 @@ void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0, __m512i sa_eff = _mm512_or_si512(sa, t); __m512i den_eff = _mm512_or_si512(da2, t); __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff); - __m512i ratio = _mm512_cvttps_epi32( + __m512i ratio = CvttpsEpi32( _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff))); __m512i r_rep = _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0); @@ -5416,7 +5514,7 @@ void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0, __m512i sa_eff = _mm512_or_si512(sa, t); __m512i den_eff = _mm512_or_si512(da2, t); __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff); - __m512i ratio = _mm512_cvttps_epi32( + __m512i ratio = CvttpsEpi32( _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff))); __m512i r_rep = _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0); @@ -6671,25 +6769,25 @@ static const uint8_t kShuffleUYVYUV_Win[16] = {0, 2, 0, 2, 4, 6, 4, 6, #define STOREARGB_WIN_AVX512BW(dst_ptr) \ do { \ - __m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \ - _mm512_unpacklo_epi8(zmm0, zmm1)); \ - __m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \ - _mm512_unpacklo_epi8(zmm2, zmm5)); \ + __m512i bg = \ + PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \ + __m512i ra = \ + PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \ _mm512_storeu_si512((__m512i*)(dst_ptr), _mm512_unpacklo_epi16(bg, ra)); \ _mm512_storeu_si512((__m512i*)((dst_ptr) + 64), \ _mm512_unpackhi_epi16(bg, ra)); \ } while (0) -#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \ - do { \ - __m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \ - _mm512_unpacklo_epi8(zmm0, zmm1)); \ - __m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \ - _mm512_unpacklo_epi8(zmm2, zmm5)); \ - _mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \ - _mm512_unpacklo_epi16(bg, ra)); \ - _mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \ - _mm512_unpackhi_epi16(bg, ra)); \ +#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \ + do { \ + __m512i bg = \ + PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \ + __m512i ra = \ + PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \ + _mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \ + _mm512_unpacklo_epi16(bg, ra)); \ + _mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \ + _mm512_unpackhi_epi16(bg, ra)); \ } while (0) #define PACKAR30_WIN_AVX512BW(dst0, dst1) \ @@ -6728,8 +6826,7 @@ void I422ToARGBRow_AVX512BW(const uint8_t* y_buf, __m256i uv256 = _mm256_inserti128_si256( _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); y_buf += 32; @@ -6747,8 +6844,7 @@ void I422ToARGBRow_AVX512BW(const uint8_t* y_buf, __m256i uv256 = _mm256_inserti128_si256( _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); @@ -6781,17 +6877,14 @@ void I422ToRGBARow_AVX512BW(const uint8_t* y_buf, __m256i uv256 = _mm256_inserti128_si256( _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); y_buf += 32; zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); YUVTORGB8_WIN_AVX512BW; - __m512i ab = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); - __m512i gr = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); + __m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); + __m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); _mm512_storeu_si512((__m512i*)dst_rgba, _mm512_unpacklo_epi16(ab, gr)); _mm512_storeu_si512((__m512i*)(dst_rgba + 64), _mm512_unpackhi_epi16(ab, gr)); @@ -6806,16 +6899,13 @@ void I422ToRGBARow_AVX512BW(const uint8_t* y_buf, __m256i uv256 = _mm256_inserti128_si256( _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); YUVTORGB8_WIN_AVX512BW; - __m512i ab = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); - __m512i gr = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); + __m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); + __m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); _mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, _mm512_unpacklo_epi16(ab, gr)); _mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16), @@ -7110,8 +7200,7 @@ void I210ToARGBRow_AVX512BW(const uint16_t* y_buf, y_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); YUVTORGB8_WIN_AVX512BW; @@ -7127,8 +7216,7 @@ void I210ToARGBRow_AVX512BW(const uint16_t* y_buf, __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); YUVTORGB8_WIN_AVX512BW; @@ -7166,8 +7254,7 @@ void I210ToAR30Row_AVX512BW(const uint16_t* y_buf, y_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); YUVTORGB16_WIN_AVX512BW; @@ -7185,8 +7272,7 @@ void I210ToAR30Row_AVX512BW(const uint16_t* y_buf, __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); YUVTORGB16_WIN_AVX512BW; @@ -7226,8 +7312,7 @@ void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, a_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); __m512i zmm5 = _mm512_packus_epi16(a16, a16); @@ -7246,8 +7331,7 @@ void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, _mm512_srli_epi16(_mm512_maskz_loadu_epi16((__mmask32)m32, a_buf), 2); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); __m512i zmm5 = _mm512_packus_epi16(a16, a16); @@ -7283,8 +7367,7 @@ void I212ToARGBRow_AVX512BW(const uint16_t* y_buf, y_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); YUVTORGB8_WIN_AVX512BW; @@ -7300,8 +7383,7 @@ void I212ToARGBRow_AVX512BW(const uint16_t* y_buf, __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); YUVTORGB8_WIN_AVX512BW; @@ -7339,8 +7421,7 @@ void I212ToAR30Row_AVX512BW(const uint16_t* y_buf, y_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); YUVTORGB16_WIN_AVX512BW; @@ -7358,8 +7439,7 @@ void I212ToAR30Row_AVX512BW(const uint16_t* y_buf, __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); YUVTORGB16_WIN_AVX512BW; @@ -8071,7 +8151,7 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8); zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); _mm512_storeu_si512((__m512i*)dst_y, zmm0); src_argb += 256; @@ -8116,7 +8196,7 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, 8); _mm512_storeu_si512( (__m512i*)dst_u, - _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(u01, u23))); + PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(u01, u23))); __m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one); __m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one); @@ -8130,7 +8210,7 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, 8); _mm512_storeu_si512( (__m512i*)dst_v, - _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(v01, v23))); + PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(v01, v23))); src_argb += 256; dst_u += 64; @@ -8192,13 +8272,13 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero); zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); + zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one); zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one); zmm0 = _mm512_packs_epi32(zmm1, zmm2); zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); _mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0)); _mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1)); @@ -8241,13 +8321,13 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero); zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); + zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one); zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one); zmm0 = _mm512_packs_epi32(zmm1, zmm2); zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); int halfwidth = (width + 1) >> 1; __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u); diff --git a/source/scale_win.cc b/source/scale_win.cc index d63877083..93921c794 100644 --- a/source/scale_win.cc +++ b/source/scale_win.cc @@ -1544,6 +1544,32 @@ static const uint8_t kPackEvenBytes[64] = { 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126}; +// Under MemorySanitizer, use a portable lane-by-lane C loop so shadow +// propagation relies only on standard C memory/array semantics rather than +// target-specific LLVM intrinsic modeling. In non-MSan builds, call the native +// AVX-512 intrinsic directly. +LIBYUV_TARGET_AVX512VBMI +static inline __m512i MaskPermutexvarEpi32(__m512i src, + __mmask16 k, + __m512i idx, + __m512i v) { +#if defined(LIBYUV_MSAN) + uint32_t src_arr[16]; + uint32_t idx_arr[16]; + uint32_t v_arr[16]; + uint32_t out_arr[16]; + _mm512_storeu_si512((__m512i*)src_arr, src); + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 16; ++i) { + out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 15] : src_arr[i]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_mask_permutexvar_epi32(src, k, idx, v); +#endif +} + // Scale UV columns by 3 box filter using AVX512VBMI. // Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop // iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels). @@ -1567,12 +1593,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv); __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64)); - __m512i a0 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2); - __m512i a1 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2); - __m512i a2 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2); + __m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1), + k5, col0, s2); + __m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1), + k5, col1, s2); + __m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1), + k6, col2, s2); __m512i sum0 = _mm512_mulhi_epu16( _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale); @@ -1580,12 +1606,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, __m512i s4 = _mm512_loadu_si512((const __m512i*)(src_uv + 128)); __m512i s5 = _mm512_loadu_si512((const __m512i*)(src_uv + 160)); src_uv += 192; - __m512i b0 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s3, col0, s4), k5, col0, s5); - __m512i b1 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s3, col1, s4), k5, col1, s5); - __m512i b2 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s3, col2, s4), k6, col2, s5); + __m512i b0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col0, s4), + k5, col0, s5); + __m512i b1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col1, s4), + k5, col1, s5); + __m512i b2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col2, s4), + k6, col2, s5); __m512i sum1 = _mm512_mulhi_epu16( _mm512_add_epi16(_mm512_add_epi16(b0, b1), b2), vscale); @@ -1599,12 +1625,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64)); src_uv += 96; - __m512i a0 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2); - __m512i a1 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2); - __m512i a2 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2); + __m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1), + k5, col0, s2); + __m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1), + k5, col1, s2); + __m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1), + k6, col2, s2); __m512i sum0 = _mm512_mulhi_epu16( _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale); _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(sum0)); @@ -1620,12 +1646,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, __m512i s0 = _mm512_maskz_loadu_epi32(k1, src_uv); __m512i s1 = _mm512_maskz_loadu_epi32(k2, src_uv + 32); __m512i s2 = _mm512_maskz_loadu_epi32(k3, src_uv + 64); - __m512i a0 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2); - __m512i a1 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2); - __m512i a2 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2); + __m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1), + k5, col0, s2); + __m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1), + k5, col1, s2); + __m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1), + k6, col2, s2); __m512i sum0 = _mm512_mulhi_epu16( _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale); _mm256_mask_storeu_epi16(dst_uv, k4, _mm512_cvtepi16_epi8(sum0)); diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 5f7cf116e..cd4d98efd 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -22,6 +22,7 @@ #include "libyuv/planar_functions.h" #include "libyuv/rotate.h" #include "libyuv/scale.h" +#include "libyuv/scale_uv.h" #if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \ !defined(ENABLE_ROW_TESTS) @@ -38,11 +39,16 @@ #if __has_feature(memory_sanitizer) #include #define MSAN_POISON(ptr, size) __msan_allocated_memory(ptr, size) +#define MSAN_CHECK_INITIALIZED(ptr, size) \ + __msan_check_mem_is_initialized(ptr, size) #endif #endif #ifndef MSAN_POISON #define MSAN_POISON(ptr, size) #endif +#ifndef MSAN_CHECK_INITIALIZED +#define MSAN_CHECK_INITIALIZED(ptr, size) +#endif #define EXPECTED_UNATTENUATE_DIFF 0 @@ -501,6 +507,36 @@ TEST_F(LibYUVPlanarTest, TestARGBGrayTo) { ASSERT_EQ(i, orig_pixels[i][2]); ASSERT_EQ(i, orig_pixels[i][3]); } + + // Test RGBX input with uninitialized alpha channel under MSan. + SIMD_ALIGNED(uint8_t gray_pixels_c[37][4]); + for (int i = 0; i < 37; ++i) { + orig_pixels[i][0] = static_cast(i * 7); + orig_pixels[i][1] = static_cast(i * 13); + orig_pixels[i][2] = static_cast(i * 23); + orig_pixels[i][3] = 0u; + } + MaskCpuFlags(disable_cpu_flags_); + ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels_c[0][0], 0, 37, 1); + MaskCpuFlags(benchmark_cpu_info_); + for (int i = 0; i < 37; ++i) { + MSAN_POISON(&orig_pixels[i][3], 1); + } + MSAN_POISON(gray_pixels, sizeof(gray_pixels)); + ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels[0][0], 0, 37, 1); + for (int i = 0; i < 37; ++i) { + MSAN_CHECK_INITIALIZED(&gray_pixels[i][0], 3); + ASSERT_EQ(gray_pixels_c[i][0], gray_pixels[i][0]); + ASSERT_EQ(gray_pixels_c[i][1], gray_pixels[i][1]); + ASSERT_EQ(gray_pixels_c[i][2], gray_pixels[i][2]); + } + ARGBGray(&orig_pixels[0][0], 0, 0, 0, 37, 1); + for (int i = 0; i < 37; ++i) { + MSAN_CHECK_INITIALIZED(&orig_pixels[i][0], 3); + ASSERT_EQ(gray_pixels_c[i][0], orig_pixels[i][0]); + ASSERT_EQ(gray_pixels_c[i][1], orig_pixels[i][1]); + ASSERT_EQ(gray_pixels_c[i][2], orig_pixels[i][2]); + } } TEST_F(LibYUVPlanarTest, TestARGBSepia) { @@ -5874,4 +5910,278 @@ TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_AllAlphas) { free_aligned_buffer_page_end(dst_argb_opt); } +TEST_F(LibYUVPlanarTest, MSanUninitializedInputsAndTails) { + static const int kWidths[] = {1, 7, 15, 16, 17, 31, 32, 33, 63, 64, 65, 99}; + const int kMaxW = 128; + const int kMaxSrcUV = kMaxW * 3 * 2 + 128; + align_buffer_page_end(src0, kMaxW * 4 + 128); + align_buffer_page_end(src1, kMaxW * 4 + 128); + align_buffer_page_end(src2, kMaxW * 4 + 128); + align_buffer_page_end(dst_c0, kMaxW * 4 + 128); + align_buffer_page_end(dst_c1, kMaxW * 4 + 128); + align_buffer_page_end(dst_c2, kMaxW * 4 + 128); + align_buffer_page_end(dst_opt0, kMaxW * 4 + 128); + align_buffer_page_end(dst_opt1, kMaxW * 4 + 128); + align_buffer_page_end(dst_opt2, kMaxW * 4 + 128); + align_buffer_page_end(uv_src, kMaxSrcUV * 3); + + for (size_t wi = 0; wi < sizeof(kWidths) / sizeof(kWidths[0]); ++wi) { + const int w = kWidths[wi]; + const int uv_w = (w + 1) / 2; + + // 1. ARGBToI444 and ARGBToI420 with poisoned RGBX alpha and poisoned tail. + for (int i = 0; i < w; ++i) { + src0[i * 4 + 0] = static_cast((i * 11 + 3) & 255); + src0[i * 4 + 1] = static_cast((i * 17 + 7) & 255); + src0[i * 4 + 2] = static_cast((i * 29 + 13) & 255); + src0[i * 4 + 3] = 0u; + } + MaskCpuFlags(disable_cpu_flags_); + ARGBToI444(src0, w * 4, dst_c0, w, dst_c1, w, dst_c2, w, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + for (int i = 0; i < w; ++i) { + MSAN_POISON(&src0[i * 4 + 3], 1); + } + MSAN_POISON(src0 + w * 4, 128); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MSAN_POISON(dst_opt2, kMaxW * 4); + ARGBToI444(src0, w * 4, dst_opt0, w, dst_opt1, w, dst_opt2, w, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w); + MSAN_CHECK_INITIALIZED(dst_opt1, w); + MSAN_CHECK_INITIALIZED(dst_opt2, w); + for (int i = 0; i < w; ++i) { + EXPECT_EQ(dst_c0[i], dst_opt0[i]) << "ARGBToI444 Y w=" << w; + EXPECT_EQ(dst_c1[i], dst_opt1[i]) << "ARGBToI444 U w=" << w; + EXPECT_EQ(dst_c2[i], dst_opt2[i]) << "ARGBToI444 V w=" << w; + } + + for (int i = 0; i < w; ++i) { + src0[i * 4 + 3] = 0u; + } + MaskCpuFlags(disable_cpu_flags_); + ARGBToI420(src0, w * 4, dst_c0, w, dst_c1, uv_w, dst_c2, uv_w, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + for (int i = 0; i < w; ++i) { + MSAN_POISON(&src0[i * 4 + 3], 1); + } + MSAN_POISON(src0 + w * 4, 128); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MSAN_POISON(dst_opt2, kMaxW * 4); + ARGBToI420(src0, w * 4, dst_opt0, w, dst_opt1, uv_w, dst_opt2, uv_w, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w); + MSAN_CHECK_INITIALIZED(dst_opt1, uv_w); + MSAN_CHECK_INITIALIZED(dst_opt2, uv_w); + for (int i = 0; i < w; ++i) { + EXPECT_EQ(dst_c0[i], dst_opt0[i]) << "ARGBToI420 Y w=" << w; + } + for (int i = 0; i < uv_w; ++i) { + EXPECT_EQ(dst_c1[i], dst_opt1[i]) << "ARGBToI420 U w=" << w; + EXPECT_EQ(dst_c2[i], dst_opt2[i]) << "ARGBToI420 V w=" << w; + } + + // 2. ARGBMirror with poisoned alpha on even pixels (verifying per-lane + // shadow permutation) and poisoned tail. + for (int i = 0; i < w; ++i) { + src0[i * 4 + 0] = static_cast((i * 5 + 1) & 255); + src0[i * 4 + 1] = static_cast((i * 9 + 2) & 255); + src0[i * 4 + 2] = static_cast((i * 15 + 3) & 255); + src0[i * 4 + 3] = static_cast((i * 21 + 4) & 255); + } + MaskCpuFlags(disable_cpu_flags_); + ARGBMirror(src0, w * 4, dst_c0, w * 4, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + for (int i = 0; i < w; i += 2) { + MSAN_POISON(&src0[i * 4 + 3], 1); + } + MSAN_POISON(src0 + w * 4, 128); + MSAN_POISON(dst_opt0, kMaxW * 4); + ARGBMirror(src0, w * 4, dst_opt0, w * 4, w, 1); + for (int i = 0; i < w; ++i) { + const int src_i = w - 1 - i; + MSAN_CHECK_INITIALIZED(&dst_opt0[i * 4], 3); + EXPECT_EQ(dst_c0[i * 4 + 0], dst_opt0[i * 4 + 0]); + EXPECT_EQ(dst_c0[i * 4 + 1], dst_opt0[i * 4 + 1]); + EXPECT_EQ(dst_c0[i * 4 + 2], dst_opt0[i * 4 + 2]); + if (src_i & 1) { + MSAN_CHECK_INITIALIZED(&dst_opt0[i * 4 + 3], 1); + EXPECT_EQ(dst_c0[i * 4 + 3], dst_opt0[i * 4 + 3]); + } + } + + // 3. RGB24Mirror, RAWToRGBA, RAWToRGB24, MergeRGBPlane with poisoned tails. + for (int i = 0; i < w * 3; ++i) { + src0[i] = static_cast((i * 19 + 5) & 255); + } + MSAN_POISON(src0 + w * 3, 128); + + MaskCpuFlags(disable_cpu_flags_); + RGB24Mirror(src0, w * 3, dst_c0, w * 3, w, 1); + RAWToRGBA(src0, w * 3, dst_c1, w * 4, w, 1); + RAWToRGB24(src0, w * 3, dst_c2, w * 3, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MSAN_POISON(dst_opt2, kMaxW * 4); + RGB24Mirror(src0, w * 3, dst_opt0, w * 3, w, 1); + RAWToRGBA(src0, w * 3, dst_opt1, w * 4, w, 1); + RAWToRGB24(src0, w * 3, dst_opt2, w * 3, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 3); + MSAN_CHECK_INITIALIZED(dst_opt1, w * 4); + MSAN_CHECK_INITIALIZED(dst_opt2, w * 3); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 3)); + EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 4)); + EXPECT_EQ(0, memcmp(dst_c2, dst_opt2, w * 3)); + + for (int i = 0; i < w; ++i) { + src0[i] = static_cast((i * 3 + 11) & 255); + src1[i] = static_cast((i * 7 + 23) & 255); + src2[i] = static_cast((i * 13 + 37) & 255); + } + MSAN_POISON(src0 + w, 128); + MSAN_POISON(src1 + w, 128); + MSAN_POISON(src2 + w, 128); + MaskCpuFlags(disable_cpu_flags_); + MergeRGBPlane(src0, w, src1, w, src2, w, dst_c0, w * 3, w, 1); + MirrorPlane(src0, w, dst_c1, w, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MergeRGBPlane(src0, w, src1, w, src2, w, dst_opt0, w * 3, w, 1); + MirrorPlane(src0, w, dst_opt1, w, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 3); + MSAN_CHECK_INITIALIZED(dst_opt1, w); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 3)); + EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w)); + + // 4. MirrorUVPlane and MirrorPlane_16 with poisoned tails. + for (int i = 0; i < w * 2; ++i) { + src0[i] = static_cast((i * 23 + 9) & 255); + } + MSAN_POISON(src0 + w * 2, 128); + MaskCpuFlags(disable_cpu_flags_); + MirrorUVPlane(src0, w * 2, dst_c0, w * 2, w, 1); + MirrorPlane_16(reinterpret_cast(src0), w, + reinterpret_cast(dst_c1), w, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MirrorUVPlane(src0, w * 2, dst_opt0, w * 2, w, 1); + MirrorPlane_16(reinterpret_cast(src0), w, + reinterpret_cast(dst_opt1), w, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 2); + MSAN_CHECK_INITIALIZED(dst_opt1, w * 2); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 2)); + EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 2)); + + // 5. I420ToAR30, NV12ToARGB, NV12ToRGB24, I444ToARGB, I210ToARGB. + for (int i = 0; i < w; ++i) { + src0[i] = static_cast((i * 5 + 16) & 255); + src1[i] = static_cast((i * 9 + 64) & 255); + src2[i] = static_cast((i * 11 + 128) & 255); + } + MSAN_POISON(src0 + w, 128); + MSAN_POISON(src1 + uv_w, 128); + MSAN_POISON(src2 + uv_w, 128); + MaskCpuFlags(disable_cpu_flags_); + I420ToAR30(src0, w, src1, uv_w, src2, uv_w, dst_c0, w * 4, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + I420ToAR30(src0, w, src1, uv_w, src2, uv_w, dst_opt0, w * 4, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 4); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4)); + + for (int i = 0; i < uv_w * 2; ++i) { + src1[i] = static_cast((i * 13 + 80) & 255); + } + MSAN_POISON(src1 + uv_w * 2, 128); + MaskCpuFlags(disable_cpu_flags_); + NV12ToARGB(src0, w, src1, uv_w * 2, dst_c0, w * 4, w, 1); + NV12ToRGB24(src0, w, src1, uv_w * 2, dst_c1, w * 3, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + NV12ToARGB(src0, w, src1, uv_w * 2, dst_opt0, w * 4, w, 1); + NV12ToRGB24(src0, w, src1, uv_w * 2, dst_opt1, w * 3, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 4); + MSAN_CHECK_INITIALIZED(dst_opt1, w * 3); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4)); + EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 3)); + + for (int i = 0; i < w; ++i) { + reinterpret_cast(src0)[i] = + static_cast((i * 17 + 64) & 1023); + } + for (int i = 0; i < uv_w; ++i) { + reinterpret_cast(src1)[i] = + static_cast((i * 23 + 256) & 1023); + reinterpret_cast(src2)[i] = + static_cast((i * 31 + 512) & 1023); + } + MSAN_POISON(src0 + w * 2, 128); + MSAN_POISON(src1 + uv_w * 2, 128); + MSAN_POISON(src2 + uv_w * 2, 128); + MaskCpuFlags(disable_cpu_flags_); + I210ToARGB(reinterpret_cast(src0), w, + reinterpret_cast(src1), uv_w, + reinterpret_cast(src2), uv_w, dst_c0, w * 4, w, + 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + I210ToARGB(reinterpret_cast(src0), w, + reinterpret_cast(src1), uv_w, + reinterpret_cast(src2), uv_w, dst_opt0, w * 4, + w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 4); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4)); + + // 6. ARGBAlphaBlend with poisoned tails and poisoned destination. + for (int i = 0; i < w * 4; ++i) { + src0[i] = static_cast((i * 29 + 3) & 255); + src1[i] = static_cast((i * 37 + 11) & 255); + } + MSAN_POISON(src0 + w * 4, 128); + MSAN_POISON(src1 + w * 4, 128); + MaskCpuFlags(disable_cpu_flags_); + ARGBAlphaBlend(src0, w * 4, src1, w * 4, dst_c0, w * 4, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + ARGBAlphaBlend(src0, w * 4, src1, w * 4, dst_opt0, w * 4, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 4); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4)); + + // 7. UVScale 3x box downscale (hits ScaleUVAddCols3_AVX512VBMI). + const int src_uv_w = w * 3; + const int src_uv_stride = src_uv_w * 2 + 64; + MSAN_POISON(uv_src, kMaxSrcUV * 3); + for (int row = 0; row < 3; ++row) { + for (int i = 0; i < src_uv_w * 2; ++i) { + uv_src[row * src_uv_stride + i] = + static_cast((row * 53 + i * 17 + 7) & 255); + } + } + MaskCpuFlags(disable_cpu_flags_); + UVScale(uv_src, src_uv_stride, src_uv_w, 3, dst_c0, w * 2, w, 1, + kFilterBox); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + UVScale(uv_src, src_uv_stride, src_uv_w, 3, dst_opt0, w * 2, w, 1, + kFilterBox); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 2); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 2)); + } + + free_aligned_buffer_page_end(src0); + free_aligned_buffer_page_end(src1); + free_aligned_buffer_page_end(src2); + free_aligned_buffer_page_end(dst_c0); + free_aligned_buffer_page_end(dst_c1); + free_aligned_buffer_page_end(dst_c2); + free_aligned_buffer_page_end(dst_opt0); + free_aligned_buffer_page_end(dst_opt1); + free_aligned_buffer_page_end(dst_opt2); + free_aligned_buffer_page_end(uv_src); +} + } // namespace libyuv