diff --git a/BUILD.gn b/BUILD.gn index c055dc9df..3c37e0591 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -7,6 +7,7 @@ # be found in the AUTHORS file in the root of the source tree. import("//build/config/features.gni") +import("//build/config/sanitizers/sanitizers.gni") import("//testing/test.gni") import("libyuv.gni") @@ -383,6 +384,23 @@ if (libyuv_include_tests) { defines = [ "GTEST_RELATIVE_PATH" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + # Override the relative PT_INTERP from //third_party/instrumented_libs so + # standalone test binaries can be launched from any working directory + # (such as the builder root on LUCI Linux MSan). + config("libyuv_msan_ldflags") { + if (msan_track_origins == 0) { + _msan_archive = "msan-no-origins-noble-lib" + } else { + _msan_archive = "msan-chained-origins-noble-lib" + } + _ld_linux = rebase_path( + "//third_party/instrumented_libs/binaries/$_msan_archive/lib/ld-linux-x86-64.so.2") + ldflags = [ "-Wl,--dynamic-linker=$_ld_linux" ] + } + } + test("libyuv_unittest") { testonly = true @@ -423,6 +441,10 @@ if (libyuv_include_tests) { } configs += [ ":libyuv_unittest_warnings_config" ] + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } public_deps = [ "//testing/gtest" ] public_configs = [ ":libyuv_unittest_config" ] @@ -466,6 +488,10 @@ if (libyuv_include_tests) { if (is_linux || is_chromeos) { cflags = [ "-fexceptions" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("yuvconvert") { @@ -477,6 +503,10 @@ if (libyuv_include_tests) { if (is_linux || is_chromeos) { cflags = [ "-fexceptions" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("yuvconstants") { @@ -488,6 +518,10 @@ if (libyuv_include_tests) { if (is_linux || is_chromeos) { cflags = [ "-fexceptions" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("psnr") { @@ -502,6 +536,10 @@ if (libyuv_include_tests) { if (!is_ios && !libyuv_disable_jpeg) { defines = [ "HAVE_JPEG" ] } + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("i444tonv12_eg") { @@ -510,6 +548,10 @@ if (libyuv_include_tests) { "util/i444tonv12_eg.cc", ] deps = [ ":libyuv" ] + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } executable("cpuid") { @@ -518,6 +560,10 @@ if (libyuv_include_tests) { "util/cpuid.c", ] deps = [ ":libyuv" ] + if (!build_with_chromium && prebuilt_instrumented_libraries_available && + is_msan) { + configs += [ ":libyuv_msan_ldflags" ] + } } } diff --git a/GEMINI.md b/GEMINI.md index 923077777..d9cfd977a 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -101,10 +101,11 @@ from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing `"ymmN"` or `"zmmN"`, including for AVX2 and AVX512 code. Clang for Windows must preserve xmm6-xmm15 and does not save them when they are declared as ymm or zmm. -4. **MSan Intrinsics (`row_win.cc` / `scale_win.cc`)**: Clang builds with - MemorySanitizer (`-fsanitize=memory`) define `LIBYUV_ENABLE_ROWWIN` and - compile `row_win.cc` and `scale_win.cc` instead of inline assembly. Avoid - AVX-512 intrinsics that LLVM's MSan pass does not instrument: +4. **MSan Intrinsics (`*_win.cc`)**: Clang builds with MemorySanitizer + (`-fsanitize=memory`) define `LIBYUV_ENABLE_ROWWIN` and compile `*_win.cc` + (`row_win.cc`, `scale_win.cc`, `compare_win.cc`, `rotate_win.cc`) instead of + inline assembly (`*_gcc.cc`). When porting `*_gcc.cc` to `*_win.cc`, never + use AVX-512 intrinsics that LLVM's MSan pass does not model: * **Down-converting masked stores**: Never use `_mm512_mask_cvtepi32_storeu_epi8`, `_mm512_mask_cvtepi64_storeu_epi8`, `_mm512_mask_cvtepi16_storeu_epi8`, or @@ -115,14 +116,42 @@ from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing store (e.g. `_mm_mask_storeu_epi8(dst, mask, _mm512_cvtepi32_epi8(v))`), which lowers to `@llvm.masked.store` for MSan and still compiles to a single `vpmovdb` instruction at `-O2`. - * **Ternary logic (`_mm512_ternarylogic_epi32` / `epi64`) on partially - uninitialized vectors**: LLVM MSan does not model `vpternlog` truth - tables bitwise and flags `use-of-uninitialized-value` if any bit of any - of the three inputs is uninitialized, even if masked off by `imm8`. For - partial channel updates (e.g. writing alpha into `dst` in - `ARGBCopyYToAlphaRow_AVX512BW`), use byte-masked stores - (`_mm512_mask_storeu_epi8`) instead of reading `dst` and blending with - `_mm512_ternarylogic_epi32`. + * **Ternary logic (`_mm*_ternarylogic_epi32` / `epi64`)**: Never use + `_mm512_ternarylogic_epi32`/`epi64` (or `_mm256_`/`_mm_` variants) + anywhere in `*_win.cc` when porting `vpternlogd`/`vpternlogq` from + `*_gcc.cc`. They lower to `@llvm.x86.avx512.pternlog.*`, which LLVM MSan + does not model bitwise—`handleUnknownIntrinsic` traps immediately with + `use-of-uninitialized-value` if *any* bit of any of the three inputs is + uninitialized (such as the unused alpha byte of an `RGBX` buffer or tail + padding), even if masked off by `imm8` or merely copied to memory. + * For boolean blends and inversions (e.g. `ARGBGrayRow_AVX512VNNI`, + `I422ToAR30Row_AVX512BW`, `ARGBToUV444MatrixRow_AVX512VNNIVBMI`), + use standard bitwise intrinsics (`_mm512_and_si512`, + `_mm512_andnot_si512`, `_mm512_or_si512`, `_mm512_xor_si512`), which + LLVM MSan models bit-by-bit and Clang `-O2` folds back into + `vpternlog` in non-MSan builds. + * For write-only partial channel updates into `dst` (e.g. writing + alpha in `ARGBCopyYToAlphaRow_AVX512BW`), use byte-masked stores + (`_mm512_mask_storeu_epi8`) instead of reading `dst` and blending. + * **Single-source variable permutes (`_mm512_permutexvar_epi*` / + `_mm512_mask_permutexvar_epi*`)**: Never use + `_mm512_permutexvar_epi{8,16,32,64}` or + `_mm512_mask_permutexvar_epi{8,16,32,64}` directly. They lower to + `@llvm.x86.avx512.permvar.*`, which LLVM MSan does not model + lane-by-lane—instead it ORs the input shadows without permuting them, + poisoning initialized output lanes whenever *any* lane of the input + vector is uninitialized (e.g. `_mm512_castsi128_si512`, + `_mm512_castsi256_si512`, unused `RGBX` alpha bytes, or uninitialized + tail lanes). Use the `PermutexvarEpi*` and `MaskPermutexvarEpi*` + helpers in `row_win.cc` / `scale_win.cc`, which use portable + lane-by-lane C loops under `LIBYUV_MSAN` and the native intrinsics + otherwise. + * **512-bit float-to-int truncation (`_mm512_cvttps_epi32`)**: Never use + `_mm512_cvttps_epi32` directly; it lowers to + `@llvm.x86.avx512.mask.cvttps2dq.512`, which LLVM MSan does not model + lane-by-lane. Use the `CvttpsEpi32` helper in `row_win.cc`, which uses + a portable lane-by-lane C loop under `LIBYUV_MSAN` and + `_mm512_cvttps_epi32` otherwise. ### AArch64 diff --git a/source/row_win.cc b/source/row_win.cc index a986d79ef..69873fafb 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -53,6 +53,118 @@ extern "C" { #define LIBYUV_TARGET_AVX512VNNIVBMI #endif // defined(__clang__) || defined(__GNUC__) +#if defined(__x86_64__) || defined(_M_X64) +// Under MemorySanitizer, use portable lane-by-lane C loops for single-source +// variable permutes and 512-bit float-to-int truncation so shadow propagation +// relies only on standard C memory/array semantics rather than target-specific +// LLVM intrinsic modeling. In non-MSan builds, call the native AVX-512 +// intrinsics directly. +LIBYUV_TARGET_AVX512VBMI +static inline __m512i PermutexvarEpi8(__m512i idx, __m512i v) { +#if defined(LIBYUV_MSAN) + uint8_t idx_arr[64]; + uint8_t v_arr[64]; + uint8_t out_arr[64]; + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 64; ++i) { + out_arr[i] = v_arr[idx_arr[i] & 63]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_permutexvar_epi8(idx, v); +#endif +} + +LIBYUV_TARGET_AVX512VBMI +static inline __m512i MaskPermutexvarEpi8(__m512i src, + __mmask64 k, + __m512i idx, + __m512i v) { +#if defined(LIBYUV_MSAN) + uint8_t src_arr[64]; + uint8_t idx_arr[64]; + uint8_t v_arr[64]; + uint8_t out_arr[64]; + _mm512_storeu_si512((__m512i*)src_arr, src); + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 64; ++i) { + out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 63] : src_arr[i]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_mask_permutexvar_epi8(src, k, idx, v); +#endif +} + +LIBYUV_TARGET_AVX512BW +static inline __m512i PermutexvarEpi16(__m512i idx, __m512i v) { +#if defined(LIBYUV_MSAN) + uint16_t idx_arr[32]; + uint16_t v_arr[32]; + uint16_t out_arr[32]; + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 32; ++i) { + out_arr[i] = v_arr[idx_arr[i] & 31]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_permutexvar_epi16(idx, v); +#endif +} + +LIBYUV_TARGET_AVX512BW +static inline __m512i PermutexvarEpi32(__m512i idx, __m512i v) { +#if defined(LIBYUV_MSAN) + uint32_t idx_arr[16]; + uint32_t v_arr[16]; + uint32_t out_arr[16]; + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 16; ++i) { + out_arr[i] = v_arr[idx_arr[i] & 15]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_permutexvar_epi32(idx, v); +#endif +} + +LIBYUV_TARGET_AVX512BW +static inline __m512i PermutexvarEpi64(__m512i idx, __m512i v) { +#if defined(LIBYUV_MSAN) + uint64_t idx_arr[8]; + uint64_t v_arr[8]; + uint64_t out_arr[8]; + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 8; ++i) { + out_arr[i] = v_arr[idx_arr[i] & 7]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_permutexvar_epi64(idx, v); +#endif +} + +LIBYUV_TARGET_AVX512BW +static inline __m512i CvttpsEpi32(__m512 v) { +#if defined(LIBYUV_MSAN) + float v_arr[16]; + int32_t out_arr[16]; + _mm512_storeu_ps(v_arr, v); + for (int i = 0; i < 16; ++i) { + out_arr[i] = (int32_t)v_arr[i]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_cvttps_epi32(v); +#endif +} +#endif // defined(__x86_64__) || defined(_M_X64) + // Read 8 UV from 444 #define READYUV444 \ xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \ @@ -444,10 +556,10 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128)); - __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); - __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); @@ -484,10 +596,10 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, __m512i zmm_raw1 = _mm512_maskz_loadu_epi8(src_mask1, src_raw + 64); __m512i zmm_raw2 = _mm512_maskz_loadu_epi8(src_mask2, src_raw + 128); - __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); - __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); @@ -635,10 +747,10 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128)); - __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); - __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); @@ -742,10 +854,10 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_rgb + 64)); __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_rgb + 128)); - __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0); __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); - __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + __m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2); zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); @@ -777,7 +889,7 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, zmm2 = _mm512_srli_epi16(zmm2, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm2); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm512_storeu_si512((__m512i*)dst_y, zmm0); @@ -830,7 +942,7 @@ void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm2 = _mm512_srli_epi16(zmm2, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm2); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm512_storeu_si512((__m512i*)dst_y, zmm0); @@ -947,7 +1059,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm0 = _mm512_srli_epi16(zmm0, 8); zmm2 = _mm512_srli_epi16(zmm2, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm2); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm512_storeu_si512((__m512i*)dst_u, zmm0); zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_v); @@ -962,7 +1074,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm0 = _mm512_srli_epi16(zmm0, 8); zmm2 = _mm512_srli_epi16(zmm2, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm2); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm512_storeu_si512((__m512i*)dst_v, zmm0); src_argb += 256; @@ -978,7 +1090,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb, // Requires AVX512VNNI and AVX512VBMI. // U = (kAddUV - dot(kRGBToU, argb)) >> 8. The accumulators start at // ~kAddUV so they hold ~(kAddUV - dot) and U is the inverse of byte 1 of each -// int32. The inverse is folded into the ternary logic that merges the halves. +// int32. LIBYUV_TARGET_AVX512VNNIVBMI void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, uint8_t* dst_u, @@ -992,12 +1104,10 @@ void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); __m512i zmm_v = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + const __m512i zmm_neg1 = _mm512_set1_epi32(-1); __m512i zmm_add = _mm512_xor_si512( _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)), - _mm512_set1_epi32(-1)); // ~kAddUV - // Select low 256 bits from the first operand, high 256 bits from the second. - __m512i zmm_sel = - _mm512_inserti64x4(_mm512_setzero_si512(), _mm256_set1_epi32(-1), 1); + zmm_neg1); // ~kAddUV while (width > 0) { __m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb); @@ -1014,12 +1124,15 @@ void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, __m512i zmm6 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_v); __m512i zmm7 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_v); - zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // ~U 0..31 - zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // ~U 32..63 - zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); // ~V 0..31 - zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); // ~V 32..63 - zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm2, zmm_sel, 0x27); // U - zmm4 = _mm512_ternarylogic_epi32(zmm4, zmm6, zmm_sel, 0x27); // V + zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // ~U 0..31 + zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // ~U 32..63 + zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); // ~V 0..31 + zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); // ~V 32..63 + // Avoid _mm512_ternarylogic_epi32 (unmodeled by LLVM MSan; see GEMINI.md). + zmm0 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm0, zmm2, 0x44), + zmm_neg1); // U + zmm4 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm4, zmm6, 0x44), + zmm_neg1); // V _mm512_storeu_si512((__m512i*)dst_u, zmm0); _mm512_storeu_si512((__m512i*)dst_v, zmm4); @@ -1086,7 +1199,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); zmm0 = _mm512_packus_epi16(zmm0, zmm1); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0); zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u); zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v); @@ -1094,7 +1207,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm0 = _mm512_sub_epi16(zmm_8000, zmm0); zmm0 = _mm512_srli_epi16(zmm0, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm0); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); _mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0)); _mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1)); @@ -1140,7 +1253,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); zmm0 = _mm512_packus_epi16(zmm0, zmm1); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0); zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u); zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v); @@ -1148,7 +1261,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb, zmm0 = _mm512_sub_epi16(zmm_8000, zmm0); zmm0 = _mm512_srli_epi16(zmm0, 8); zmm0 = _mm512_packus_epi16(zmm0, zmm0); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0); int halfwidth = (width + 1) >> 1; __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u); @@ -1216,7 +1329,7 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); zmm0 = _mm512_packus_epi16(zmm0, zmm1); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0); zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V @@ -1268,7 +1381,7 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, zmm1 = _mm512_avg_epu16(zmm1, zmm_zero); zmm0 = _mm512_packus_epi16(zmm0, zmm1); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0); + zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0); zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V @@ -1490,7 +1603,7 @@ void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { v = _mm512_or_si512(_mm512_srli_epi16(v, 8), _mm512_slli_epi16(v, 8)); __m512i idx = _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)((64 - width) >> 1))); - v = _mm512_permutexvar_epi16(idx, v); + v = PermutexvarEpi16(idx, v); if (width & 1) { __m512i v_hi = _mm512_alignr_epi64(v, v, 2); v = _mm512_alignr_epi8(v_hi, v, 1); @@ -1518,7 +1631,7 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv, s -= 64; __m512i v = _mm512_loadu_si512((const __m512i*)s); v = _mm512_shuffle_epi8(v, zmm_shuf); - v = _mm512_permutexvar_epi64(zmm_perm, v); + v = PermutexvarEpi64(zmm_perm, v); _mm256_storeu_si256((__m256i*)dst_u, _mm512_castsi512_si256(v)); _mm256_storeu_si256((__m256i*)dst_v, _mm512_extracti64x4_epi64(v, 1)); dst_u += 32; @@ -1530,7 +1643,7 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv, __m512i idx = _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)(32 - width))); __m512i v = _mm512_maskz_loadu_epi16(k, src_uv); - v = _mm512_permutexvar_epi16(idx, v); + v = PermutexvarEpi16(idx, v); _mm256_mask_storeu_epi8(dst_u, k, _mm512_cvtepi16_epi8(v)); _mm256_mask_storeu_epi8(dst_v, k, _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8))); @@ -1549,7 +1662,7 @@ void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { while (width >= 32) { s -= 64; __m512i zmm0 = _mm512_loadu_si512((const __m512i*)s); - zmm0 = _mm512_permutexvar_epi16(zmm_idx, zmm0); + zmm0 = PermutexvarEpi16(zmm_idx, zmm0); _mm512_storeu_si512((__m512i*)dst_uv, zmm0); dst_uv += 64; width -= 32; @@ -1559,7 +1672,7 @@ void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { __m512i idx = _mm512_sub_epi16(zmm_idx, _mm512_set1_epi16((short)(32 - width))); __m512i zmm0 = _mm512_maskz_loadu_epi16(k, src_uv); - _mm512_mask_storeu_epi16(dst_uv, k, _mm512_permutexvar_epi16(idx, zmm0)); + _mm512_mask_storeu_epi16(dst_uv, k, PermutexvarEpi16(idx, zmm0)); } _mm256_zeroupper(); } @@ -2707,15 +2820,15 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y, src_u += 16; \ __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \ __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \ - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \ - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \ + zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \ + zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \ __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \ zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \ zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \ __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); \ src_y += 32; \ __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \ - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \ + zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \ zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); @@ -2726,14 +2839,14 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y, __m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); \ __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \ __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \ - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \ - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \ + zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \ + zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \ __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \ zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \ zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \ __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); \ __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \ - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \ + zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \ zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); @@ -2996,8 +3109,8 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); + zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); + zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); @@ -3006,7 +3119,7 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); src_y += 32; __m512i zmm4 = _mm512_castsi256_si512(ymm_y); - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); + zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); @@ -3036,8 +3149,9 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, zmm0 = _mm512_srli_epi16(zmm0, 4); zmm3 = _mm512_srli_epi16(zmm1, 10); zmm1 = _mm512_slli_epi16(zmm1, 6); - zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm1, zmm_fc00, 0xf8); - zmm2 = _mm512_ternarylogic_epi32(zmm2, zmm3, zmm_3ff0, 0xe4); + zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00)); + zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0), + _mm512_andnot_si512(zmm_3ff0, zmm3)); zmm2 = _mm512_or_si512(zmm2, zmm_c000); __m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2); @@ -3058,8 +3172,8 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); + zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); + zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); @@ -3067,7 +3181,7 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); __m512i zmm4 = _mm512_castsi256_si512(ymm_y); - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); + zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); @@ -3094,8 +3208,9 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, zmm0 = _mm512_srli_epi16(zmm0, 4); zmm3 = _mm512_srli_epi16(zmm1, 10); zmm1 = _mm512_slli_epi16(zmm1, 6); - zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm1, zmm_fc00, 0xf8); - zmm2 = _mm512_ternarylogic_epi32(zmm2, zmm3, zmm_3ff0, 0xe4); + zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00)); + zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0), + _mm512_andnot_si512(zmm_3ff0, zmm3)); zmm2 = _mm512_or_si512(zmm2, zmm_c000); __m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2); @@ -3420,7 +3535,7 @@ void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { s -= 64; _mm512_storeu_si512( (__m512i*)dst, - _mm512_permutexvar_epi32(rev, _mm512_loadu_si512((const __m512i*)s))); + PermutexvarEpi32(rev, _mm512_loadu_si512((const __m512i*)s))); dst += 64; width -= 16; } @@ -3429,7 +3544,7 @@ void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { __mmask16 mask = (__mmask16)((1u << width) - 1u); __m512i idx = _mm512_sub_epi32(rev, _mm512_set1_epi32(16 - width)); __m512i v = _mm512_maskz_loadu_epi32(mask, src); - _mm512_mask_storeu_epi32(dst, mask, _mm512_permutexvar_epi32(idx, v)); + _mm512_mask_storeu_epi32(dst, mask, PermutexvarEpi32(idx, v)); } _mm256_zeroupper(); } @@ -3504,12 +3619,12 @@ void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, __m512i t0 = _mm512_sub_epi8(idx0, _mm512_set1_epi8((char)(off - 64))); __m512i t1 = _mm512_sub_epi8(idx1, _mm512_set1_epi8((char)off)); __m512i t2 = _mm512_sub_epi8(idx2, _mm512_set1_epi8((char)off)); - __m512i d0 = _mm512_mask_permutexvar_epi8( - _mm512_permutex2var_epi8(a, t0, b), _mm512_movepi8_mask(t0), t0, c); - __m512i d1 = _mm512_mask_permutexvar_epi8( - _mm512_permutex2var_epi8(a, t1, b), _mm512_movepi8_mask(t1), t1, c); - __m512i d2 = _mm512_mask_permutexvar_epi8( - _mm512_permutex2var_epi8(a, t2, b), _mm512_movepi8_mask(t2), t2, c); + __m512i d0 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t0, b), + _mm512_movepi8_mask(t0), t0, c); + __m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t1, b), + _mm512_movepi8_mask(t1), t1, c); + __m512i d2 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t2, b), + _mm512_movepi8_mask(t2), t2, c); _mm512_mask_storeu_epi8(dst_rgb24, m0, d0); _mm512_mask_storeu_epi8(dst_rgb24 + 64, m1, d1); _mm512_mask_storeu_epi8(dst_rgb24 + 128, m2, d2); @@ -4083,8 +4198,7 @@ void MergeRGBRow_AVX512VBMI(const uint8_t* src_r, src_g += 64; src_b += 64; __m512i d0 = _mm512_permutex2var_epi8(rg0, m_rgb0, br); - __m512i d1 = - _mm512_mask_permutexvar_epi8(_mm512_permutex2var_epi8(rg0, m_rgb1, br), + __m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(rg0, m_rgb1, br), 0xb6db6db600000000ull, m_rgb1, gb1); __m512i d2 = _mm512_permutex2var_epi8(br, m_rgb2, gb1); _mm512_storeu_si512((__m512i*)dst_rgb, d0); @@ -4150,7 +4264,7 @@ void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, __m512i s1 = _mm512_castsi256_si512( _mm256_loadu_si256((const __m256i*)(src_raw + 64))); src_raw += 96; - __m512i d0 = _mm512_or_si512(_mm512_permutexvar_epi8(p0, s0), alpha); + __m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha); __m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha); _mm512_storeu_si512((__m512i*)dst_rgba, d0); _mm512_storeu_si512((__m512i*)(dst_rgba + 64), d1); @@ -4164,7 +4278,7 @@ void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw, __m512i s0 = _mm512_maskz_loadu_epi8(ks0, src_raw); __m512i s1 = _mm512_castsi256_si512(_mm256_maskz_loadu_epi8(ks1, src_raw + 64)); - __m512i d0 = _mm512_or_si512(_mm512_permutexvar_epi8(p0, s0), alpha); + __m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha); __m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha); uint32_t m32 = (1u << width) - 1u; _mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, d0); @@ -4208,7 +4322,7 @@ void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) { while (width >= 32) { src -= 32; __m512i v = _mm512_loadu_si512((const __m512i*)src); - _mm512_storeu_si512((__m512i*)dst, _mm512_permutexvar_epi16(rev, v)); + _mm512_storeu_si512((__m512i*)dst, PermutexvarEpi16(rev, v)); dst += 32; width -= 32; } @@ -4216,7 +4330,7 @@ void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) { __mmask32 k = (__mmask32)((1u << width) - 1u); __m512i idx = _mm512_sub_epi16(rev, _mm512_set1_epi16((short)(32 - width))); __m512i v = _mm512_maskz_loadu_epi16(k, src - width); - _mm512_mask_storeu_epi16(dst, k, _mm512_permutexvar_epi16(idx, v)); + _mm512_mask_storeu_epi16(dst, k, PermutexvarEpi16(idx, v)); } _mm256_zeroupper(); } @@ -4402,7 +4516,7 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win); const __m512i zmm5 = _mm512_set1_epi32(-1); while (width >= 32) { - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf))); __m512i zmm4 = @@ -4425,10 +4539,8 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra)); _mm512_storeu_si512((__m512i*)(dst_argb + 64), _mm512_unpackhi_epi16(bg, ra)); @@ -4438,7 +4550,7 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, if (width > 0) { uint32_t m32 = (1u << width) - 1u; __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf))); __m512i zmm4 = @@ -4459,10 +4571,8 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, _mm512_unpacklo_epi16(bg, ra)); _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), @@ -4496,7 +4606,7 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win); const __m512i zmm5 = _mm512_set1_epi32(-1); while (width >= 32) { - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_or_si256( _mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8), @@ -4521,10 +4631,8 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra)); _mm512_storeu_si512((__m512i*)(dst_argb + 64), _mm512_unpackhi_epi16(bg, ra)); @@ -4534,7 +4642,7 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, if (width > 0) { uint32_t m32 = (1u << width) - 1u; __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_or_si256( _mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8), @@ -4557,10 +4665,8 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, _mm512_unpacklo_epi16(bg, ra)); _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), @@ -4621,7 +4727,7 @@ void NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, const __m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1_NV); while (width >= 32) { - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf))); __m512i zmm4 = @@ -4653,7 +4759,7 @@ void NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, if (width > 0) { uint32_t m32 = (1u << width) - 1u; __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf))); __m512i zmm4 = @@ -4711,7 +4817,7 @@ void NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, const __m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1_NV); while (width >= 32) { - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_or_si256( _mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8), @@ -4745,7 +4851,7 @@ void NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, if (width > 0) { uint32_t m32 = (1u << width) - 1u; __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); - __m512i zmm3 = _mm512_permutexvar_epi16( + __m512i zmm3 = PermutexvarEpi16( zmm_quad, _mm512_castsi256_si512(_mm256_or_si256( _mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8), @@ -4811,8 +4917,7 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, __m256i yuv = _mm256_inserti128_si256( _mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_castsi256_si512(yuv)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); __m512i zmm5 = _mm512_packus_epi16( @@ -4836,10 +4941,8 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra)); _mm512_storeu_si512((__m512i*)(dst_argb + 64), _mm512_unpackhi_epi16(bg, ra)); @@ -4854,8 +4957,7 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, __m256i yuv = _mm256_inserti128_si256( _mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_castsi256_si512(yuv)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); __m512i zmm5 = _mm512_packus_epi16( @@ -4877,10 +4979,8 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, _mm512_srai_epi16( _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6), zmm5); - __m512i bg = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); - __m512i ra = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); + __m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); + __m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); _mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32, _mm512_unpacklo_epi16(bg, ra)); _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16), @@ -4918,8 +5018,7 @@ void I400ToARGBRow_AVX512BW(const uint8_t* y_buf, y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8)); y16 = _mm512_srai_epi16( _mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6); - __m512i g = - _mm512_permutexvar_epi64(zmm_lin, _mm512_packus_epi16(y16, y16)); + __m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16)); __m256i g256 = _mm512_castsi512_si256(g); __m512i d0 = _mm512_or_si512( _mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)), @@ -4941,8 +5040,7 @@ void I400ToARGBRow_AVX512BW(const uint8_t* y_buf, y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8)); y16 = _mm512_srai_epi16( _mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6); - __m512i g = - _mm512_permutexvar_epi64(zmm_lin, _mm512_packus_epi16(y16, y16)); + __m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16)); __m256i g256 = _mm512_castsi512_si256(g); __m512i d0 = _mm512_or_si512( _mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)), @@ -5111,8 +5209,8 @@ void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, src_argb += 64; __m512i y32 = _mm512_srli_epi32( _mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8); - __m512i out = _mm512_ternarylogic_epi32(_mm512_shuffle_epi8(y32, c_shuf), s, - c_alpha, 0xf8); + __m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf), + _mm512_and_si512(s, c_alpha)); _mm512_storeu_si512((__m512i*)dst_argb, out); dst_argb += 64; width -= 16; @@ -5122,8 +5220,8 @@ void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb, __m512i s = _mm512_maskz_loadu_epi32(k, src_argb); __m512i y32 = _mm512_srli_epi32( _mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8); - __m512i out = _mm512_ternarylogic_epi32(_mm512_shuffle_epi8(y32, c_shuf), s, - c_alpha, 0xf8); + __m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf), + _mm512_and_si512(s, c_alpha)); _mm512_mask_storeu_epi32(dst_argb, k, out); } _mm256_zeroupper(); @@ -5377,7 +5475,7 @@ void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0, __m512i sa_eff = _mm512_or_si512(sa, t); __m512i den_eff = _mm512_or_si512(da2, t); __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff); - __m512i ratio = _mm512_cvttps_epi32( + __m512i ratio = CvttpsEpi32( _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff))); __m512i r_rep = _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0); @@ -5416,7 +5514,7 @@ void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0, __m512i sa_eff = _mm512_or_si512(sa, t); __m512i den_eff = _mm512_or_si512(da2, t); __m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff); - __m512i ratio = _mm512_cvttps_epi32( + __m512i ratio = CvttpsEpi32( _mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff))); __m512i r_rep = _mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0); @@ -6671,25 +6769,25 @@ static const uint8_t kShuffleUYVYUV_Win[16] = {0, 2, 0, 2, 4, 6, 4, 6, #define STOREARGB_WIN_AVX512BW(dst_ptr) \ do { \ - __m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \ - _mm512_unpacklo_epi8(zmm0, zmm1)); \ - __m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \ - _mm512_unpacklo_epi8(zmm2, zmm5)); \ + __m512i bg = \ + PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \ + __m512i ra = \ + PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \ _mm512_storeu_si512((__m512i*)(dst_ptr), _mm512_unpacklo_epi16(bg, ra)); \ _mm512_storeu_si512((__m512i*)((dst_ptr) + 64), \ _mm512_unpackhi_epi16(bg, ra)); \ } while (0) -#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \ - do { \ - __m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \ - _mm512_unpacklo_epi8(zmm0, zmm1)); \ - __m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \ - _mm512_unpacklo_epi8(zmm2, zmm5)); \ - _mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \ - _mm512_unpacklo_epi16(bg, ra)); \ - _mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \ - _mm512_unpackhi_epi16(bg, ra)); \ +#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \ + do { \ + __m512i bg = \ + PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \ + __m512i ra = \ + PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \ + _mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \ + _mm512_unpacklo_epi16(bg, ra)); \ + _mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \ + _mm512_unpackhi_epi16(bg, ra)); \ } while (0) #define PACKAR30_WIN_AVX512BW(dst0, dst1) \ @@ -6728,8 +6826,7 @@ void I422ToARGBRow_AVX512BW(const uint8_t* y_buf, __m256i uv256 = _mm256_inserti128_si256( _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); y_buf += 32; @@ -6747,8 +6844,7 @@ void I422ToARGBRow_AVX512BW(const uint8_t* y_buf, __m256i uv256 = _mm256_inserti128_si256( _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); @@ -6781,17 +6877,14 @@ void I422ToRGBARow_AVX512BW(const uint8_t* y_buf, __m256i uv256 = _mm256_inserti128_si256( _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); y_buf += 32; zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); YUVTORGB8_WIN_AVX512BW; - __m512i ab = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); - __m512i gr = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); + __m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); + __m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); _mm512_storeu_si512((__m512i*)dst_rgba, _mm512_unpacklo_epi16(ab, gr)); _mm512_storeu_si512((__m512i*)(dst_rgba + 64), _mm512_unpackhi_epi16(ab, gr)); @@ -6806,16 +6899,13 @@ void I422ToRGBARow_AVX512BW(const uint8_t* y_buf, __m256i uv256 = _mm256_inserti128_si256( _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), _mm_unpackhi_epi8(xu, xv), 1); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); YUVTORGB8_WIN_AVX512BW; - __m512i ab = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); - __m512i gr = - _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); + __m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); + __m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); _mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, _mm512_unpacklo_epi16(ab, gr)); _mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16), @@ -7110,8 +7200,7 @@ void I210ToARGBRow_AVX512BW(const uint16_t* y_buf, y_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); YUVTORGB8_WIN_AVX512BW; @@ -7127,8 +7216,7 @@ void I210ToARGBRow_AVX512BW(const uint16_t* y_buf, __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); YUVTORGB8_WIN_AVX512BW; @@ -7166,8 +7254,7 @@ void I210ToAR30Row_AVX512BW(const uint16_t* y_buf, y_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); YUVTORGB16_WIN_AVX512BW; @@ -7185,8 +7272,7 @@ void I210ToAR30Row_AVX512BW(const uint16_t* y_buf, __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); YUVTORGB16_WIN_AVX512BW; @@ -7226,8 +7312,7 @@ void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, a_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); __m512i zmm5 = _mm512_packus_epi16(a16, a16); @@ -7246,8 +7331,7 @@ void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, _mm512_srli_epi16(_mm512_maskz_loadu_epi16((__mmask32)m32, a_buf), 2); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); __m512i zmm5 = _mm512_packus_epi16(a16, a16); @@ -7283,8 +7367,7 @@ void I212ToARGBRow_AVX512BW(const uint16_t* y_buf, y_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); YUVTORGB8_WIN_AVX512BW; @@ -7300,8 +7383,7 @@ void I212ToARGBRow_AVX512BW(const uint16_t* y_buf, __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); YUVTORGB8_WIN_AVX512BW; @@ -7339,8 +7421,7 @@ void I212ToAR30Row_AVX512BW(const uint16_t* y_buf, y_buf += 32; __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); YUVTORGB16_WIN_AVX512BW; @@ -7358,8 +7439,7 @@ void I212ToAR30Row_AVX512BW(const uint16_t* y_buf, __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), _mm256_packus_epi16(v_s, v_s)); - __m512i zmm3 = - _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256)); __m512i zmm4 = _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); YUVTORGB16_WIN_AVX512BW; @@ -8071,7 +8151,7 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8); zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); _mm512_storeu_si512((__m512i*)dst_y, zmm0); src_argb += 256; @@ -8116,7 +8196,7 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, 8); _mm512_storeu_si512( (__m512i*)dst_u, - _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(u01, u23))); + PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(u01, u23))); __m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one); __m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one); @@ -8130,7 +8210,7 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, 8); _mm512_storeu_si512( (__m512i*)dst_v, - _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(v01, v23))); + PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(v01, v23))); src_argb += 256; dst_u += 64; @@ -8192,13 +8272,13 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero); zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); + zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one); zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one); zmm0 = _mm512_packs_epi32(zmm1, zmm2); zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); _mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0)); _mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1)); @@ -8241,13 +8321,13 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero); zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); + zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one); zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one); zmm0 = _mm512_packs_epi32(zmm1, zmm2); zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8); - zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); + zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); int halfwidth = (width + 1) >> 1; __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u); diff --git a/source/scale_win.cc b/source/scale_win.cc index d63877083..93921c794 100644 --- a/source/scale_win.cc +++ b/source/scale_win.cc @@ -1544,6 +1544,32 @@ static const uint8_t kPackEvenBytes[64] = { 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126}; +// Under MemorySanitizer, use a portable lane-by-lane C loop so shadow +// propagation relies only on standard C memory/array semantics rather than +// target-specific LLVM intrinsic modeling. In non-MSan builds, call the native +// AVX-512 intrinsic directly. +LIBYUV_TARGET_AVX512VBMI +static inline __m512i MaskPermutexvarEpi32(__m512i src, + __mmask16 k, + __m512i idx, + __m512i v) { +#if defined(LIBYUV_MSAN) + uint32_t src_arr[16]; + uint32_t idx_arr[16]; + uint32_t v_arr[16]; + uint32_t out_arr[16]; + _mm512_storeu_si512((__m512i*)src_arr, src); + _mm512_storeu_si512((__m512i*)idx_arr, idx); + _mm512_storeu_si512((__m512i*)v_arr, v); + for (int i = 0; i < 16; ++i) { + out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 15] : src_arr[i]; + } + return _mm512_loadu_si512((const __m512i*)out_arr); +#else + return _mm512_mask_permutexvar_epi32(src, k, idx, v); +#endif +} + // Scale UV columns by 3 box filter using AVX512VBMI. // Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop // iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels). @@ -1567,12 +1593,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv); __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64)); - __m512i a0 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2); - __m512i a1 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2); - __m512i a2 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2); + __m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1), + k5, col0, s2); + __m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1), + k5, col1, s2); + __m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1), + k6, col2, s2); __m512i sum0 = _mm512_mulhi_epu16( _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale); @@ -1580,12 +1606,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, __m512i s4 = _mm512_loadu_si512((const __m512i*)(src_uv + 128)); __m512i s5 = _mm512_loadu_si512((const __m512i*)(src_uv + 160)); src_uv += 192; - __m512i b0 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s3, col0, s4), k5, col0, s5); - __m512i b1 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s3, col1, s4), k5, col1, s5); - __m512i b2 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s3, col2, s4), k6, col2, s5); + __m512i b0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col0, s4), + k5, col0, s5); + __m512i b1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col1, s4), + k5, col1, s5); + __m512i b2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col2, s4), + k6, col2, s5); __m512i sum1 = _mm512_mulhi_epu16( _mm512_add_epi16(_mm512_add_epi16(b0, b1), b2), vscale); @@ -1599,12 +1625,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64)); src_uv += 96; - __m512i a0 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2); - __m512i a1 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2); - __m512i a2 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2); + __m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1), + k5, col0, s2); + __m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1), + k5, col1, s2); + __m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1), + k6, col2, s2); __m512i sum0 = _mm512_mulhi_epu16( _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale); _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(sum0)); @@ -1620,12 +1646,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, __m512i s0 = _mm512_maskz_loadu_epi32(k1, src_uv); __m512i s1 = _mm512_maskz_loadu_epi32(k2, src_uv + 32); __m512i s2 = _mm512_maskz_loadu_epi32(k3, src_uv + 64); - __m512i a0 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2); - __m512i a1 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2); - __m512i a2 = _mm512_mask_permutexvar_epi32( - _mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2); + __m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1), + k5, col0, s2); + __m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1), + k5, col1, s2); + __m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1), + k6, col2, s2); __m512i sum0 = _mm512_mulhi_epu16( _mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale); _mm256_mask_storeu_epi16(dst_uv, k4, _mm512_cvtepi16_epi8(sum0)); diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 5f7cf116e..cd4d98efd 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -22,6 +22,7 @@ #include "libyuv/planar_functions.h" #include "libyuv/rotate.h" #include "libyuv/scale.h" +#include "libyuv/scale_uv.h" #if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \ !defined(ENABLE_ROW_TESTS) @@ -38,11 +39,16 @@ #if __has_feature(memory_sanitizer) #include #define MSAN_POISON(ptr, size) __msan_allocated_memory(ptr, size) +#define MSAN_CHECK_INITIALIZED(ptr, size) \ + __msan_check_mem_is_initialized(ptr, size) #endif #endif #ifndef MSAN_POISON #define MSAN_POISON(ptr, size) #endif +#ifndef MSAN_CHECK_INITIALIZED +#define MSAN_CHECK_INITIALIZED(ptr, size) +#endif #define EXPECTED_UNATTENUATE_DIFF 0 @@ -501,6 +507,36 @@ TEST_F(LibYUVPlanarTest, TestARGBGrayTo) { ASSERT_EQ(i, orig_pixels[i][2]); ASSERT_EQ(i, orig_pixels[i][3]); } + + // Test RGBX input with uninitialized alpha channel under MSan. + SIMD_ALIGNED(uint8_t gray_pixels_c[37][4]); + for (int i = 0; i < 37; ++i) { + orig_pixels[i][0] = static_cast(i * 7); + orig_pixels[i][1] = static_cast(i * 13); + orig_pixels[i][2] = static_cast(i * 23); + orig_pixels[i][3] = 0u; + } + MaskCpuFlags(disable_cpu_flags_); + ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels_c[0][0], 0, 37, 1); + MaskCpuFlags(benchmark_cpu_info_); + for (int i = 0; i < 37; ++i) { + MSAN_POISON(&orig_pixels[i][3], 1); + } + MSAN_POISON(gray_pixels, sizeof(gray_pixels)); + ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels[0][0], 0, 37, 1); + for (int i = 0; i < 37; ++i) { + MSAN_CHECK_INITIALIZED(&gray_pixels[i][0], 3); + ASSERT_EQ(gray_pixels_c[i][0], gray_pixels[i][0]); + ASSERT_EQ(gray_pixels_c[i][1], gray_pixels[i][1]); + ASSERT_EQ(gray_pixels_c[i][2], gray_pixels[i][2]); + } + ARGBGray(&orig_pixels[0][0], 0, 0, 0, 37, 1); + for (int i = 0; i < 37; ++i) { + MSAN_CHECK_INITIALIZED(&orig_pixels[i][0], 3); + ASSERT_EQ(gray_pixels_c[i][0], orig_pixels[i][0]); + ASSERT_EQ(gray_pixels_c[i][1], orig_pixels[i][1]); + ASSERT_EQ(gray_pixels_c[i][2], orig_pixels[i][2]); + } } TEST_F(LibYUVPlanarTest, TestARGBSepia) { @@ -5874,4 +5910,278 @@ TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_AllAlphas) { free_aligned_buffer_page_end(dst_argb_opt); } +TEST_F(LibYUVPlanarTest, MSanUninitializedInputsAndTails) { + static const int kWidths[] = {1, 7, 15, 16, 17, 31, 32, 33, 63, 64, 65, 99}; + const int kMaxW = 128; + const int kMaxSrcUV = kMaxW * 3 * 2 + 128; + align_buffer_page_end(src0, kMaxW * 4 + 128); + align_buffer_page_end(src1, kMaxW * 4 + 128); + align_buffer_page_end(src2, kMaxW * 4 + 128); + align_buffer_page_end(dst_c0, kMaxW * 4 + 128); + align_buffer_page_end(dst_c1, kMaxW * 4 + 128); + align_buffer_page_end(dst_c2, kMaxW * 4 + 128); + align_buffer_page_end(dst_opt0, kMaxW * 4 + 128); + align_buffer_page_end(dst_opt1, kMaxW * 4 + 128); + align_buffer_page_end(dst_opt2, kMaxW * 4 + 128); + align_buffer_page_end(uv_src, kMaxSrcUV * 3); + + for (size_t wi = 0; wi < sizeof(kWidths) / sizeof(kWidths[0]); ++wi) { + const int w = kWidths[wi]; + const int uv_w = (w + 1) / 2; + + // 1. ARGBToI444 and ARGBToI420 with poisoned RGBX alpha and poisoned tail. + for (int i = 0; i < w; ++i) { + src0[i * 4 + 0] = static_cast((i * 11 + 3) & 255); + src0[i * 4 + 1] = static_cast((i * 17 + 7) & 255); + src0[i * 4 + 2] = static_cast((i * 29 + 13) & 255); + src0[i * 4 + 3] = 0u; + } + MaskCpuFlags(disable_cpu_flags_); + ARGBToI444(src0, w * 4, dst_c0, w, dst_c1, w, dst_c2, w, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + for (int i = 0; i < w; ++i) { + MSAN_POISON(&src0[i * 4 + 3], 1); + } + MSAN_POISON(src0 + w * 4, 128); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MSAN_POISON(dst_opt2, kMaxW * 4); + ARGBToI444(src0, w * 4, dst_opt0, w, dst_opt1, w, dst_opt2, w, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w); + MSAN_CHECK_INITIALIZED(dst_opt1, w); + MSAN_CHECK_INITIALIZED(dst_opt2, w); + for (int i = 0; i < w; ++i) { + EXPECT_EQ(dst_c0[i], dst_opt0[i]) << "ARGBToI444 Y w=" << w; + EXPECT_EQ(dst_c1[i], dst_opt1[i]) << "ARGBToI444 U w=" << w; + EXPECT_EQ(dst_c2[i], dst_opt2[i]) << "ARGBToI444 V w=" << w; + } + + for (int i = 0; i < w; ++i) { + src0[i * 4 + 3] = 0u; + } + MaskCpuFlags(disable_cpu_flags_); + ARGBToI420(src0, w * 4, dst_c0, w, dst_c1, uv_w, dst_c2, uv_w, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + for (int i = 0; i < w; ++i) { + MSAN_POISON(&src0[i * 4 + 3], 1); + } + MSAN_POISON(src0 + w * 4, 128); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MSAN_POISON(dst_opt2, kMaxW * 4); + ARGBToI420(src0, w * 4, dst_opt0, w, dst_opt1, uv_w, dst_opt2, uv_w, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w); + MSAN_CHECK_INITIALIZED(dst_opt1, uv_w); + MSAN_CHECK_INITIALIZED(dst_opt2, uv_w); + for (int i = 0; i < w; ++i) { + EXPECT_EQ(dst_c0[i], dst_opt0[i]) << "ARGBToI420 Y w=" << w; + } + for (int i = 0; i < uv_w; ++i) { + EXPECT_EQ(dst_c1[i], dst_opt1[i]) << "ARGBToI420 U w=" << w; + EXPECT_EQ(dst_c2[i], dst_opt2[i]) << "ARGBToI420 V w=" << w; + } + + // 2. ARGBMirror with poisoned alpha on even pixels (verifying per-lane + // shadow permutation) and poisoned tail. + for (int i = 0; i < w; ++i) { + src0[i * 4 + 0] = static_cast((i * 5 + 1) & 255); + src0[i * 4 + 1] = static_cast((i * 9 + 2) & 255); + src0[i * 4 + 2] = static_cast((i * 15 + 3) & 255); + src0[i * 4 + 3] = static_cast((i * 21 + 4) & 255); + } + MaskCpuFlags(disable_cpu_flags_); + ARGBMirror(src0, w * 4, dst_c0, w * 4, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + for (int i = 0; i < w; i += 2) { + MSAN_POISON(&src0[i * 4 + 3], 1); + } + MSAN_POISON(src0 + w * 4, 128); + MSAN_POISON(dst_opt0, kMaxW * 4); + ARGBMirror(src0, w * 4, dst_opt0, w * 4, w, 1); + for (int i = 0; i < w; ++i) { + const int src_i = w - 1 - i; + MSAN_CHECK_INITIALIZED(&dst_opt0[i * 4], 3); + EXPECT_EQ(dst_c0[i * 4 + 0], dst_opt0[i * 4 + 0]); + EXPECT_EQ(dst_c0[i * 4 + 1], dst_opt0[i * 4 + 1]); + EXPECT_EQ(dst_c0[i * 4 + 2], dst_opt0[i * 4 + 2]); + if (src_i & 1) { + MSAN_CHECK_INITIALIZED(&dst_opt0[i * 4 + 3], 1); + EXPECT_EQ(dst_c0[i * 4 + 3], dst_opt0[i * 4 + 3]); + } + } + + // 3. RGB24Mirror, RAWToRGBA, RAWToRGB24, MergeRGBPlane with poisoned tails. + for (int i = 0; i < w * 3; ++i) { + src0[i] = static_cast((i * 19 + 5) & 255); + } + MSAN_POISON(src0 + w * 3, 128); + + MaskCpuFlags(disable_cpu_flags_); + RGB24Mirror(src0, w * 3, dst_c0, w * 3, w, 1); + RAWToRGBA(src0, w * 3, dst_c1, w * 4, w, 1); + RAWToRGB24(src0, w * 3, dst_c2, w * 3, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MSAN_POISON(dst_opt2, kMaxW * 4); + RGB24Mirror(src0, w * 3, dst_opt0, w * 3, w, 1); + RAWToRGBA(src0, w * 3, dst_opt1, w * 4, w, 1); + RAWToRGB24(src0, w * 3, dst_opt2, w * 3, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 3); + MSAN_CHECK_INITIALIZED(dst_opt1, w * 4); + MSAN_CHECK_INITIALIZED(dst_opt2, w * 3); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 3)); + EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 4)); + EXPECT_EQ(0, memcmp(dst_c2, dst_opt2, w * 3)); + + for (int i = 0; i < w; ++i) { + src0[i] = static_cast((i * 3 + 11) & 255); + src1[i] = static_cast((i * 7 + 23) & 255); + src2[i] = static_cast((i * 13 + 37) & 255); + } + MSAN_POISON(src0 + w, 128); + MSAN_POISON(src1 + w, 128); + MSAN_POISON(src2 + w, 128); + MaskCpuFlags(disable_cpu_flags_); + MergeRGBPlane(src0, w, src1, w, src2, w, dst_c0, w * 3, w, 1); + MirrorPlane(src0, w, dst_c1, w, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MergeRGBPlane(src0, w, src1, w, src2, w, dst_opt0, w * 3, w, 1); + MirrorPlane(src0, w, dst_opt1, w, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 3); + MSAN_CHECK_INITIALIZED(dst_opt1, w); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 3)); + EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w)); + + // 4. MirrorUVPlane and MirrorPlane_16 with poisoned tails. + for (int i = 0; i < w * 2; ++i) { + src0[i] = static_cast((i * 23 + 9) & 255); + } + MSAN_POISON(src0 + w * 2, 128); + MaskCpuFlags(disable_cpu_flags_); + MirrorUVPlane(src0, w * 2, dst_c0, w * 2, w, 1); + MirrorPlane_16(reinterpret_cast(src0), w, + reinterpret_cast(dst_c1), w, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + MirrorUVPlane(src0, w * 2, dst_opt0, w * 2, w, 1); + MirrorPlane_16(reinterpret_cast(src0), w, + reinterpret_cast(dst_opt1), w, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 2); + MSAN_CHECK_INITIALIZED(dst_opt1, w * 2); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 2)); + EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 2)); + + // 5. I420ToAR30, NV12ToARGB, NV12ToRGB24, I444ToARGB, I210ToARGB. + for (int i = 0; i < w; ++i) { + src0[i] = static_cast((i * 5 + 16) & 255); + src1[i] = static_cast((i * 9 + 64) & 255); + src2[i] = static_cast((i * 11 + 128) & 255); + } + MSAN_POISON(src0 + w, 128); + MSAN_POISON(src1 + uv_w, 128); + MSAN_POISON(src2 + uv_w, 128); + MaskCpuFlags(disable_cpu_flags_); + I420ToAR30(src0, w, src1, uv_w, src2, uv_w, dst_c0, w * 4, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + I420ToAR30(src0, w, src1, uv_w, src2, uv_w, dst_opt0, w * 4, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 4); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4)); + + for (int i = 0; i < uv_w * 2; ++i) { + src1[i] = static_cast((i * 13 + 80) & 255); + } + MSAN_POISON(src1 + uv_w * 2, 128); + MaskCpuFlags(disable_cpu_flags_); + NV12ToARGB(src0, w, src1, uv_w * 2, dst_c0, w * 4, w, 1); + NV12ToRGB24(src0, w, src1, uv_w * 2, dst_c1, w * 3, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + MSAN_POISON(dst_opt1, kMaxW * 4); + NV12ToARGB(src0, w, src1, uv_w * 2, dst_opt0, w * 4, w, 1); + NV12ToRGB24(src0, w, src1, uv_w * 2, dst_opt1, w * 3, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 4); + MSAN_CHECK_INITIALIZED(dst_opt1, w * 3); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4)); + EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 3)); + + for (int i = 0; i < w; ++i) { + reinterpret_cast(src0)[i] = + static_cast((i * 17 + 64) & 1023); + } + for (int i = 0; i < uv_w; ++i) { + reinterpret_cast(src1)[i] = + static_cast((i * 23 + 256) & 1023); + reinterpret_cast(src2)[i] = + static_cast((i * 31 + 512) & 1023); + } + MSAN_POISON(src0 + w * 2, 128); + MSAN_POISON(src1 + uv_w * 2, 128); + MSAN_POISON(src2 + uv_w * 2, 128); + MaskCpuFlags(disable_cpu_flags_); + I210ToARGB(reinterpret_cast(src0), w, + reinterpret_cast(src1), uv_w, + reinterpret_cast(src2), uv_w, dst_c0, w * 4, w, + 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + I210ToARGB(reinterpret_cast(src0), w, + reinterpret_cast(src1), uv_w, + reinterpret_cast(src2), uv_w, dst_opt0, w * 4, + w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 4); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4)); + + // 6. ARGBAlphaBlend with poisoned tails and poisoned destination. + for (int i = 0; i < w * 4; ++i) { + src0[i] = static_cast((i * 29 + 3) & 255); + src1[i] = static_cast((i * 37 + 11) & 255); + } + MSAN_POISON(src0 + w * 4, 128); + MSAN_POISON(src1 + w * 4, 128); + MaskCpuFlags(disable_cpu_flags_); + ARGBAlphaBlend(src0, w * 4, src1, w * 4, dst_c0, w * 4, w, 1); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + ARGBAlphaBlend(src0, w * 4, src1, w * 4, dst_opt0, w * 4, w, 1); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 4); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4)); + + // 7. UVScale 3x box downscale (hits ScaleUVAddCols3_AVX512VBMI). + const int src_uv_w = w * 3; + const int src_uv_stride = src_uv_w * 2 + 64; + MSAN_POISON(uv_src, kMaxSrcUV * 3); + for (int row = 0; row < 3; ++row) { + for (int i = 0; i < src_uv_w * 2; ++i) { + uv_src[row * src_uv_stride + i] = + static_cast((row * 53 + i * 17 + 7) & 255); + } + } + MaskCpuFlags(disable_cpu_flags_); + UVScale(uv_src, src_uv_stride, src_uv_w, 3, dst_c0, w * 2, w, 1, + kFilterBox); + MaskCpuFlags(benchmark_cpu_info_); + MSAN_POISON(dst_opt0, kMaxW * 4); + UVScale(uv_src, src_uv_stride, src_uv_w, 3, dst_opt0, w * 2, w, 1, + kFilterBox); + MSAN_CHECK_INITIALIZED(dst_opt0, w * 2); + EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 2)); + } + + free_aligned_buffer_page_end(src0); + free_aligned_buffer_page_end(src1); + free_aligned_buffer_page_end(src2); + free_aligned_buffer_page_end(dst_c0); + free_aligned_buffer_page_end(dst_c1); + free_aligned_buffer_page_end(dst_c2); + free_aligned_buffer_page_end(dst_opt0); + free_aligned_buffer_page_end(dst_opt1); + free_aligned_buffer_page_end(dst_opt2); + free_aligned_buffer_page_end(uv_src); +} + } // namespace libyuv