coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-11-09 14:49:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

[libyuv] Fix MSan AVX-512 intrinsics and standalone dynamic linker

Browse Source 
Clang MemorySanitizer builds define LIBYUV_ENABLE_ROWWIN and compile
row_win.cc and scale_win.cc instead of inline assembly. Replace three
families of AVX-512 intrinsics that LLVM MSan does not model bitwise or
lane-by-lane:

- _mm512_ternarylogic_epi32 (@llvm.x86.avx512.pternlog.d.512) in
  ARGBGrayRow_AVX512VNNI, ARGBToUV444MatrixRow_AVX512VNNIVBMI, and
  I422ToAR30Row_AVX512BW, which raises false use-of-uninitialized-value
  errors whenever any bit of any operand is uninitialized (such as the
  unused alpha channel of an RGBX buffer). Use _mm512_and_si512,
  _mm512_andnot_si512, _mm512_or_si512, _mm512_xor_si512, and
  _mm512_shuffle_i64x2, which LLVM MSan models bit-by-bit.
- _mm512_permutexvar_epi{8,16,32,64} and
  _mm512_mask_permutexvar_epi{8,32} (@llvm.x86.avx512.permvar.*) across
  row_win.cc and ScaleUVAddCols3_AVX512VBMI in scale_win.cc, which OR
  input shadows without permuting them and poison initialized output
  lanes whenever any lane of the input vector is uninitialized. Route
  through PermutexvarEpi* and MaskPermutexvarEpi* helpers that use
  portable lane-by-lane C loops under LIBYUV_MSAN and the native
  AVX-512 intrinsics otherwise.
- _mm512_cvttps_epi32 (@llvm.x86.avx512.mask.cvttps2dq.512) in
  ARGBAlphaBlendRow_AVX512BW. Route through CvttpsEpi32, which uses a
  portable lane-by-lane C loop under LIBYUV_MSAN and _mm512_cvttps_epi32
  otherwise.

In standalone MSan builds (!build_with_chromium &&
prebuilt_instrumented_libraries_available && is_msan), override the
relative ELF PT_INTERP from //third_party/instrumented_libs with the
absolute path to the instrumented ld-linux-x86-64.so.2 so test and
utility binaries launch cleanly from any working directory (fixing the
LUCI Linux MSan test runner FileNotFoundError).

Add MSan regression tests with poisoned RGBX alpha channels, poisoned
tail padding, and poisoned destination buffers in planar_test.cc, and
document the AVX-512 MSan intrinsic guidelines in GEMINI.md.

Test: out/msan/libyuv_unittest
Bug: libyuv:388428508
Change-Id: I6a002393db2d2e629cd7a3cf2bf4d98ee9daf79d
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8686036
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard 2026-10-29 17:18:55 -07:00
parent 0b8dd23180
commit 59fb9e18a8
5 changed files with 694 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
BUILD.gn
View File

@ -7,6 +7,7 @@
# be found in the AUTHORS file in the root of the source tree.
import("//build/config/features.gni")
import("//build/config/sanitizers/sanitizers.gni")
import("//testing/test.gni")
import("libyuv.gni")
@ -383,6 +384,23 @@ if (libyuv_include_tests) {
defines = [ "GTEST_RELATIVE_PATH" ]
}
if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
is_msan) {
# Override the relative PT_INTERP from //third_party/instrumented_libs so
# standalone test binaries can be launched from any working directory
# (such as the builder root on LUCI Linux MSan).
config("libyuv_msan_ldflags") {
if (msan_track_origins == 0) {
_msan_archive = "msan-no-origins-noble-lib"
} else {
_msan_archive = "msan-chained-origins-noble-lib"
}
_ld_linux = rebase_path(
"//third_party/instrumented_libs/binaries/$_msan_archive/lib/ld-linux-x86-64.so.2")
ldflags = [ "-Wl,--dynamic-linker=$_ld_linux" ]
}
}
test("libyuv_unittest") {
testonly = true
@ -423,6 +441,10 @@ if (libyuv_include_tests) {
}
configs += [ ":libyuv_unittest_warnings_config" ]
if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
is_msan) {
configs += [ ":libyuv_msan_ldflags" ]
}
public_deps = [ "//testing/gtest" ]
public_configs = [ ":libyuv_unittest_config" ]
@ -466,6 +488,10 @@ if (libyuv_include_tests) {
if (is_linux || is_chromeos) {
cflags = [ "-fexceptions" ]
}
if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
is_msan) {
configs += [ ":libyuv_msan_ldflags" ]
}
}
executable("yuvconvert") {
@ -477,6 +503,10 @@ if (libyuv_include_tests) {
if (is_linux || is_chromeos) {
cflags = [ "-fexceptions" ]
}
if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
is_msan) {
configs += [ ":libyuv_msan_ldflags" ]
}
}
executable("yuvconstants") {
@ -488,6 +518,10 @@ if (libyuv_include_tests) {
if (is_linux || is_chromeos) {
cflags = [ "-fexceptions" ]
}
if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
is_msan) {
configs += [ ":libyuv_msan_ldflags" ]
}
}
executable("psnr") {
@ -502,6 +536,10 @@ if (libyuv_include_tests) {
if (!is_ios && !libyuv_disable_jpeg) {
defines = [ "HAVE_JPEG" ]
}
if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
is_msan) {
configs += [ ":libyuv_msan_ldflags" ]
}
}
executable("i444tonv12_eg") {
@ -510,6 +548,10 @@ if (libyuv_include_tests) {
"util/i444tonv12_eg.cc",
]
deps = [ ":libyuv" ]
if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
is_msan) {
configs += [ ":libyuv_msan_ldflags" ]
}
}
executable("cpuid") {
@ -518,6 +560,10 @@ if (libyuv_include_tests) {
"util/cpuid.c",
]
deps = [ ":libyuv" ]
if (!build_with_chromium && prebuilt_instrumented_libraries_available &&
is_msan) {
configs += [ ":libyuv_msan_ldflags" ]
}
}
}

53
GEMINI.md
View File

@ -101,10 +101,11 @@ from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing
`"ymmN"` or `"zmmN"`, including for AVX2 and AVX512 code. Clang for
Windows must preserve xmm6-xmm15 and does not save them when they are
declared as ymm or zmm.
4. **MSan Intrinsics (`row_win.cc` / `scale_win.cc`)**: Clang builds with
MemorySanitizer (`-fsanitize=memory`) define `LIBYUV_ENABLE_ROWWIN` and
compile `row_win.cc` and `scale_win.cc` instead of inline assembly. Avoid
AVX-512 intrinsics that LLVM's MSan pass does not instrument:
4. **MSan Intrinsics (`*_win.cc`)**: Clang builds with MemorySanitizer
(`-fsanitize=memory`) define `LIBYUV_ENABLE_ROWWIN` and compile `*_win.cc`
(`row_win.cc`, `scale_win.cc`, `compare_win.cc`, `rotate_win.cc`) instead of
inline assembly (`*_gcc.cc`). When porting `*_gcc.cc` to `*_win.cc`, never
use AVX-512 intrinsics that LLVM's MSan pass does not model:
* **Down-converting masked stores**: Never use
`_mm512_mask_cvtepi32_storeu_epi8`, `_mm512_mask_cvtepi64_storeu_epi8`,
`_mm512_mask_cvtepi16_storeu_epi8`, or
@ -115,14 +116,42 @@ from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing
store (e.g. `_mm_mask_storeu_epi8(dst, mask, _mm512_cvtepi32_epi8(v))`),
which lowers to `@llvm.masked.store` for MSan and still compiles to a
single `vpmovdb` instruction at `-O2`.
* **Ternary logic (`_mm512_ternarylogic_epi32` / `epi64`) on partially
uninitialized vectors**: LLVM MSan does not model `vpternlog` truth
tables bitwise and flags `use-of-uninitialized-value` if any bit of any
of the three inputs is uninitialized, even if masked off by `imm8`. For
partial channel updates (e.g. writing alpha into `dst` in
`ARGBCopyYToAlphaRow_AVX512BW`), use byte-masked stores
(`_mm512_mask_storeu_epi8`) instead of reading `dst` and blending with
`_mm512_ternarylogic_epi32`.
* **Ternary logic (`_mm*_ternarylogic_epi32` / `epi64`)**: Never use
`_mm512_ternarylogic_epi32`/`epi64` (or `_mm256_`/`_mm_` variants)
anywhere in `*_win.cc` when porting `vpternlogd`/`vpternlogq` from
`*_gcc.cc`. They lower to `@llvm.x86.avx512.pternlog.*`, which LLVM MSan
does not model bitwise—`handleUnknownIntrinsic` traps immediately with
`use-of-uninitialized-value` if *any* bit of any of the three inputs is
uninitialized (such as the unused alpha byte of an `RGBX` buffer or tail
padding), even if masked off by `imm8` or merely copied to memory.
* For boolean blends and inversions (e.g. `ARGBGrayRow_AVX512VNNI`,
`I422ToAR30Row_AVX512BW`, `ARGBToUV444MatrixRow_AVX512VNNIVBMI`),
use standard bitwise intrinsics (`_mm512_and_si512`,
`_mm512_andnot_si512`, `_mm512_or_si512`, `_mm512_xor_si512`), which
LLVM MSan models bit-by-bit and Clang `-O2` folds back into
`vpternlog` in non-MSan builds.
* For write-only partial channel updates into `dst` (e.g. writing
alpha in `ARGBCopyYToAlphaRow_AVX512BW`), use byte-masked stores
(`_mm512_mask_storeu_epi8`) instead of reading `dst` and blending.
* **Single-source variable permutes (`_mm512_permutexvar_epi*` /
`_mm512_mask_permutexvar_epi*`)**: Never use
`_mm512_permutexvar_epi{8,16,32,64}` or
`_mm512_mask_permutexvar_epi{8,16,32,64}` directly. They lower to
`@llvm.x86.avx512.permvar.*`, which LLVM MSan does not model
lane-by-lane—instead it ORs the input shadows without permuting them,
poisoning initialized output lanes whenever *any* lane of the input
vector is uninitialized (e.g. `_mm512_castsi128_si512`,
`_mm512_castsi256_si512`, unused `RGBX` alpha bytes, or uninitialized
tail lanes). Use the `PermutexvarEpi*` and `MaskPermutexvarEpi*`
helpers in `row_win.cc` / `scale_win.cc`, which use portable
lane-by-lane C loops under `LIBYUV_MSAN` and the native intrinsics
otherwise.
* **512-bit float-to-int truncation (`_mm512_cvttps_epi32`)**: Never use
`_mm512_cvttps_epi32` directly; it lowers to
`@llvm.x86.avx512.mask.cvttps2dq.512`, which LLVM MSan does not model
lane-by-lane. Use the `CvttpsEpi32` helper in `row_win.cc`, which uses
a portable lane-by-lane C loop under `LIBYUV_MSAN` and
`_mm512_cvttps_epi32` otherwise.
### AArch64

414
source/row_win.cc
View File

@ -53,6 +53,118 @@ extern "C" {
#define LIBYUV_TARGET_AVX512VNNIVBMI
#endif // defined(__clang__) || defined(__GNUC__)
#if defined(__x86_64__) || defined(_M_X64)
// Under MemorySanitizer, use portable lane-by-lane C loops for single-source
// variable permutes and 512-bit float-to-int truncation so shadow propagation
// relies only on standard C memory/array semantics rather than target-specific
// LLVM intrinsic modeling. In non-MSan builds, call the native AVX-512
// intrinsics directly.
LIBYUV_TARGET_AVX512VBMI
static inline __m512i PermutexvarEpi8(__m512i idx, __m512i v) {
#if defined(LIBYUV_MSAN)
uint8_t idx_arr[64];
uint8_t v_arr[64];
uint8_t out_arr[64];
_mm512_storeu_si512((__m512i*)idx_arr, idx);
_mm512_storeu_si512((__m512i*)v_arr, v);
for (int i = 0; i < 64; ++i) {
out_arr[i] = v_arr[idx_arr[i] & 63];
}
return _mm512_loadu_si512((const __m512i*)out_arr);
#else
return _mm512_permutexvar_epi8(idx, v);
#endif
}
LIBYUV_TARGET_AVX512VBMI
static inline __m512i MaskPermutexvarEpi8(__m512i src,
__mmask64 k,
__m512i idx,
__m512i v) {
#if defined(LIBYUV_MSAN)
uint8_t src_arr[64];
uint8_t idx_arr[64];
uint8_t v_arr[64];
uint8_t out_arr[64];
_mm512_storeu_si512((__m512i*)src_arr, src);
_mm512_storeu_si512((__m512i*)idx_arr, idx);
_mm512_storeu_si512((__m512i*)v_arr, v);
for (int i = 0; i < 64; ++i) {
out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 63] : src_arr[i];
}
return _mm512_loadu_si512((const __m512i*)out_arr);
#else
return _mm512_mask_permutexvar_epi8(src, k, idx, v);
#endif
}
LIBYUV_TARGET_AVX512BW
static inline __m512i PermutexvarEpi16(__m512i idx, __m512i v) {
#if defined(LIBYUV_MSAN)
uint16_t idx_arr[32];
uint16_t v_arr[32];
uint16_t out_arr[32];
_mm512_storeu_si512((__m512i*)idx_arr, idx);
_mm512_storeu_si512((__m512i*)v_arr, v);
for (int i = 0; i < 32; ++i) {
out_arr[i] = v_arr[idx_arr[i] & 31];
}
return _mm512_loadu_si512((const __m512i*)out_arr);
#else
return _mm512_permutexvar_epi16(idx, v);
#endif
}
LIBYUV_TARGET_AVX512BW
static inline __m512i PermutexvarEpi32(__m512i idx, __m512i v) {
#if defined(LIBYUV_MSAN)
uint32_t idx_arr[16];
uint32_t v_arr[16];
uint32_t out_arr[16];
_mm512_storeu_si512((__m512i*)idx_arr, idx);
_mm512_storeu_si512((__m512i*)v_arr, v);
for (int i = 0; i < 16; ++i) {
out_arr[i] = v_arr[idx_arr[i] & 15];
}
return _mm512_loadu_si512((const __m512i*)out_arr);
#else
return _mm512_permutexvar_epi32(idx, v);
#endif
}
LIBYUV_TARGET_AVX512BW
static inline __m512i PermutexvarEpi64(__m512i idx, __m512i v) {
#if defined(LIBYUV_MSAN)
uint64_t idx_arr[8];
uint64_t v_arr[8];
uint64_t out_arr[8];
_mm512_storeu_si512((__m512i*)idx_arr, idx);
_mm512_storeu_si512((__m512i*)v_arr, v);
for (int i = 0; i < 8; ++i) {
out_arr[i] = v_arr[idx_arr[i] & 7];
}
return _mm512_loadu_si512((const __m512i*)out_arr);
#else
return _mm512_permutexvar_epi64(idx, v);
#endif
}
LIBYUV_TARGET_AVX512BW
static inline __m512i CvttpsEpi32(__m512 v) {
#if defined(LIBYUV_MSAN)
float v_arr[16];
int32_t out_arr[16];
_mm512_storeu_ps(v_arr, v);
for (int i = 0; i < 16; ++i) {
out_arr[i] = (int32_t)v_arr[i];
}
return _mm512_loadu_si512((const __m512i*)out_arr);
#else
return _mm512_cvttps_epi32(v);
#endif
}
#endif // defined(__x86_64__) || defined(_M_X64)
// Read 8 UV from 444
#define READYUV444 \
xmm3 = _mm_loadl_epi64((__m128i*)u_buf); \
@ -444,10 +556,10 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
@ -484,10 +596,10 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
__m512i zmm_raw1 = _mm512_maskz_loadu_epi8(src_mask1, src_raw + 64);
__m512i zmm_raw2 = _mm512_maskz_loadu_epi8(src_mask2, src_raw + 128);
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
@ -635,10 +747,10 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw,
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
@ -742,10 +854,10 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_rgb + 64));
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_rgb + 128));
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
__m512i zmm0 = PermutexvarEpi64(zmm_q0, zmm_raw0);
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
__m512i zmm3 = PermutexvarEpi64(zmm_q3, zmm_raw2);
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
@ -777,7 +889,7 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
zmm2 = _mm512_srli_epi16(zmm2, 8);
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
_mm512_storeu_si512((__m512i*)dst_y, zmm0);
@ -830,7 +942,7 @@ void ARGBToYMatrixRow_AVX512VNNI(const uint8_t* src_argb,
zmm2 = _mm512_srli_epi16(zmm2, 8);
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
_mm512_storeu_si512((__m512i*)dst_y, zmm0);
@ -947,7 +1059,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb,
zmm0 = _mm512_srli_epi16(zmm0, 8);
zmm2 = _mm512_srli_epi16(zmm2, 8);
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
_mm512_storeu_si512((__m512i*)dst_u, zmm0);
zmm0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), zmm_s0, zmm_v);
@ -962,7 +1074,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb,
zmm0 = _mm512_srli_epi16(zmm0, 8);
zmm2 = _mm512_srli_epi16(zmm2, 8);
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
_mm512_storeu_si512((__m512i*)dst_v, zmm0);
src_argb += 256;
@ -978,7 +1090,7 @@ void ARGBToUV444MatrixRow_AVX512VNNI(const uint8_t* src_argb,
// Requires AVX512VNNI and AVX512VBMI.
// U = (kAddUV - dot(kRGBToU, argb)) >> 8. The accumulators start at
// ~kAddUV so they hold ~(kAddUV - dot) and U is the inverse of byte 1 of each
// int32. The inverse is folded into the ternary logic that merges the halves.
// int32.
LIBYUV_TARGET_AVX512VNNIVBMI
void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
uint8_t* dst_u,
@ -992,12 +1104,10 @@ void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU));
__m512i zmm_v =
_mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV));
const __m512i zmm_neg1 = _mm512_set1_epi32(-1);
__m512i zmm_add = _mm512_xor_si512(
_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)c->kAddUV)),
_mm512_set1_epi32(-1)); // ~kAddUV
// Select low 256 bits from the first operand, high 256 bits from the second.
__m512i zmm_sel =
_mm512_inserti64x4(_mm512_setzero_si512(), _mm256_set1_epi32(-1), 1);
zmm_neg1); // ~kAddUV
while (width > 0) {
__m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src_argb);
@ -1014,12 +1124,15 @@ void ARGBToUV444MatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
__m512i zmm6 = _mm512_dpbusd_epi32(zmm_add, zmm_s2, zmm_v);
__m512i zmm7 = _mm512_dpbusd_epi32(zmm_add, zmm_s3, zmm_v);
zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // ~U 0..31
zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // ~U 32..63
zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); // ~V 0..31
zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); // ~V 32..63
zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm2, zmm_sel, 0x27); // U
zmm4 = _mm512_ternarylogic_epi32(zmm4, zmm6, zmm_sel, 0x27); // V
zmm0 = _mm512_permutex2var_epi8(zmm0, zmm_perm, zmm1); // ~U 0..31
zmm2 = _mm512_permutex2var_epi8(zmm2, zmm_perm, zmm3); // ~U 32..63
zmm4 = _mm512_permutex2var_epi8(zmm4, zmm_perm, zmm5); // ~V 0..31
zmm6 = _mm512_permutex2var_epi8(zmm6, zmm_perm, zmm7); // ~V 32..63
// Avoid _mm512_ternarylogic_epi32 (unmodeled by LLVM MSan; see GEMINI.md).
zmm0 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm0, zmm2, 0x44),
zmm_neg1); // U
zmm4 = _mm512_xor_si512(_mm512_shuffle_i64x2(zmm4, zmm6, 0x44),
zmm_neg1); // V
_mm512_storeu_si512((__m512i*)dst_u, zmm0);
_mm512_storeu_si512((__m512i*)dst_v, zmm4);
@ -1086,7 +1199,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb,
zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
zmm0 = _mm512_packus_epi16(zmm0, zmm1);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u);
zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v);
@ -1094,7 +1207,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb,
zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
zmm0 = _mm512_srli_epi16(zmm0, 8);
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
_mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0));
_mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1));
@ -1140,7 +1253,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb,
zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
zmm0 = _mm512_packus_epi16(zmm0, zmm1);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
zmm1 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_u);
zmm2 = _mm512_dpbusd_epi32(zmm_zero, zmm0, zmm_v);
@ -1148,7 +1261,7 @@ void ARGBToUVMatrixRow_AVX512VNNI(const uint8_t* src_argb,
zmm0 = _mm512_sub_epi16(zmm_8000, zmm0);
zmm0 = _mm512_srli_epi16(zmm0, 8);
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_y, zmm0);
int halfwidth = (width + 1) >> 1;
__mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u);
@ -1216,7 +1329,7 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
zmm0 = _mm512_packus_epi16(zmm0, zmm1);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U
zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V
@ -1268,7 +1381,7 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb,
zmm1 = _mm512_avg_epu16(zmm1, zmm_zero);
zmm0 = _mm512_packus_epi16(zmm0, zmm1);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, zmm0);
zmm0 = PermutexvarEpi32(zmm_perm_uv, zmm0);
zmm1 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_u); // ~U
zmm2 = _mm512_dpbusd_epi32(zmm_add, zmm0, zmm_v); // ~V
@ -1490,7 +1603,7 @@ void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
v = _mm512_or_si512(_mm512_srli_epi16(v, 8), _mm512_slli_epi16(v, 8));
__m512i idx =
_mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)((64 - width) >> 1)));
v = _mm512_permutexvar_epi16(idx, v);
v = PermutexvarEpi16(idx, v);
if (width & 1) {
__m512i v_hi = _mm512_alignr_epi64(v, v, 2);
v = _mm512_alignr_epi8(v_hi, v, 1);
@ -1518,7 +1631,7 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv,
s -= 64;
__m512i v = _mm512_loadu_si512((const __m512i*)s);
v = _mm512_shuffle_epi8(v, zmm_shuf);
v = _mm512_permutexvar_epi64(zmm_perm, v);
v = PermutexvarEpi64(zmm_perm, v);
_mm256_storeu_si256((__m256i*)dst_u, _mm512_castsi512_si256(v));
_mm256_storeu_si256((__m256i*)dst_v, _mm512_extracti64x4_epi64(v, 1));
dst_u += 32;
@ -1530,7 +1643,7 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv,
__m512i idx =
_mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)(32 - width)));
__m512i v = _mm512_maskz_loadu_epi16(k, src_uv);
v = _mm512_permutexvar_epi16(idx, v);
v = PermutexvarEpi16(idx, v);
_mm256_mask_storeu_epi8(dst_u, k, _mm512_cvtepi16_epi8(v));
_mm256_mask_storeu_epi8(dst_v, k,
_mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8)));
@ -1549,7 +1662,7 @@ void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
while (width >= 32) {
s -= 64;
__m512i zmm0 = _mm512_loadu_si512((const __m512i*)s);
zmm0 = _mm512_permutexvar_epi16(zmm_idx, zmm0);
zmm0 = PermutexvarEpi16(zmm_idx, zmm0);
_mm512_storeu_si512((__m512i*)dst_uv, zmm0);
dst_uv += 64;
width -= 32;
@ -1559,7 +1672,7 @@ void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
__m512i idx =
_mm512_sub_epi16(zmm_idx, _mm512_set1_epi16((short)(32 - width)));
__m512i zmm0 = _mm512_maskz_loadu_epi16(k, src_uv);
_mm512_mask_storeu_epi16(dst_uv, k, _mm512_permutexvar_epi16(idx, zmm0));
_mm512_mask_storeu_epi16(dst_uv, k, PermutexvarEpi16(idx, zmm0));
}
_mm256_zeroupper();
}
@ -2707,15 +2820,15 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y,
src_u += 16; \
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \
zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \
zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); \
src_y += 32; \
__m512i zmm4 = _mm512_castsi256_si512(ymm_y); \
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \
zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
@ -2726,14 +2839,14 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y,
__m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); \
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \
zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val); \
zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val); \
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \
__m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); \
__m512i zmm4 = _mm512_castsi256_si512(ymm_y); \
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \
zmm4 = PermutexvarEpi64(zmm_split_y, zmm4); \
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
@ -2996,8 +3109,8 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val);
zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val);
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
@ -3006,7 +3119,7 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
src_y += 32;
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
zmm4 = PermutexvarEpi64(zmm_split_y, zmm4);
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
@ -3036,8 +3149,9 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
zmm0 = _mm512_srli_epi16(zmm0, 4);
zmm3 = _mm512_srli_epi16(zmm1, 10);
zmm1 = _mm512_slli_epi16(zmm1, 6);
zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm1, zmm_fc00, 0xf8);
zmm2 = _mm512_ternarylogic_epi32(zmm2, zmm3, zmm_3ff0, 0xe4);
zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00));
zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0),
_mm512_andnot_si512(zmm_3ff0, zmm3));
zmm2 = _mm512_or_si512(zmm2, zmm_c000);
__m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2);
@ -3058,8 +3172,8 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
zmm_u_val = PermutexvarEpi64(zmm_split, zmm_u_val);
zmm_v_val = PermutexvarEpi64(zmm_split, zmm_v_val);
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
@ -3067,7 +3181,7 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
__m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y);
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
zmm4 = PermutexvarEpi64(zmm_split_y, zmm4);
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
@ -3094,8 +3208,9 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
zmm0 = _mm512_srli_epi16(zmm0, 4);
zmm3 = _mm512_srli_epi16(zmm1, 10);
zmm1 = _mm512_slli_epi16(zmm1, 6);
zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm1, zmm_fc00, 0xf8);
zmm2 = _mm512_ternarylogic_epi32(zmm2, zmm3, zmm_3ff0, 0xe4);
zmm0 = _mm512_or_si512(zmm0, _mm512_and_si512(zmm1, zmm_fc00));
zmm2 = _mm512_or_si512(_mm512_and_si512(zmm2, zmm_3ff0),
_mm512_andnot_si512(zmm_3ff0, zmm3));
zmm2 = _mm512_or_si512(zmm2, zmm_c000);
__m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2);
@ -3420,7 +3535,7 @@ void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
s -= 64;
_mm512_storeu_si512(
(__m512i*)dst,
_mm512_permutexvar_epi32(rev, _mm512_loadu_si512((const __m512i*)s)));
PermutexvarEpi32(rev, _mm512_loadu_si512((const __m512i*)s)));
dst += 64;
width -= 16;
}
@ -3429,7 +3544,7 @@ void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
__mmask16 mask = (__mmask16)((1u << width) - 1u);
__m512i idx = _mm512_sub_epi32(rev, _mm512_set1_epi32(16 - width));
__m512i v = _mm512_maskz_loadu_epi32(mask, src);
_mm512_mask_storeu_epi32(dst, mask, _mm512_permutexvar_epi32(idx, v));
_mm512_mask_storeu_epi32(dst, mask, PermutexvarEpi32(idx, v));
}
_mm256_zeroupper();
}
@ -3504,12 +3619,12 @@ void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24,
__m512i t0 = _mm512_sub_epi8(idx0, _mm512_set1_epi8((char)(off - 64)));
__m512i t1 = _mm512_sub_epi8(idx1, _mm512_set1_epi8((char)off));
__m512i t2 = _mm512_sub_epi8(idx2, _mm512_set1_epi8((char)off));
__m512i d0 = _mm512_mask_permutexvar_epi8(
_mm512_permutex2var_epi8(a, t0, b), _mm512_movepi8_mask(t0), t0, c);
__m512i d1 = _mm512_mask_permutexvar_epi8(
_mm512_permutex2var_epi8(a, t1, b), _mm512_movepi8_mask(t1), t1, c);
__m512i d2 = _mm512_mask_permutexvar_epi8(
_mm512_permutex2var_epi8(a, t2, b), _mm512_movepi8_mask(t2), t2, c);
__m512i d0 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t0, b),
_mm512_movepi8_mask(t0), t0, c);
__m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t1, b),
_mm512_movepi8_mask(t1), t1, c);
__m512i d2 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(a, t2, b),
_mm512_movepi8_mask(t2), t2, c);
_mm512_mask_storeu_epi8(dst_rgb24, m0, d0);
_mm512_mask_storeu_epi8(dst_rgb24 + 64, m1, d1);
_mm512_mask_storeu_epi8(dst_rgb24 + 128, m2, d2);
@ -4083,8 +4198,7 @@ void MergeRGBRow_AVX512VBMI(const uint8_t* src_r,
src_g += 64;
src_b += 64;
__m512i d0 = _mm512_permutex2var_epi8(rg0, m_rgb0, br);
__m512i d1 =
_mm512_mask_permutexvar_epi8(_mm512_permutex2var_epi8(rg0, m_rgb1, br),
__m512i d1 = MaskPermutexvarEpi8(_mm512_permutex2var_epi8(rg0, m_rgb1, br),
0xb6db6db600000000ull, m_rgb1, gb1);
__m512i d2 = _mm512_permutex2var_epi8(br, m_rgb2, gb1);
_mm512_storeu_si512((__m512i*)dst_rgb, d0);
@ -4150,7 +4264,7 @@ void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw,
__m512i s1 = _mm512_castsi256_si512(
_mm256_loadu_si256((const __m256i*)(src_raw + 64)));
src_raw += 96;
__m512i d0 = _mm512_or_si512(_mm512_permutexvar_epi8(p0, s0), alpha);
__m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha);
__m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha);
_mm512_storeu_si512((__m512i*)dst_rgba, d0);
_mm512_storeu_si512((__m512i*)(dst_rgba + 64), d1);
@ -4164,7 +4278,7 @@ void RAWToRGBARow_AVX512VBMI(const uint8_t* src_raw,
__m512i s0 = _mm512_maskz_loadu_epi8(ks0, src_raw);
__m512i s1 =
_mm512_castsi256_si512(_mm256_maskz_loadu_epi8(ks1, src_raw + 64));
__m512i d0 = _mm512_or_si512(_mm512_permutexvar_epi8(p0, s0), alpha);
__m512i d0 = _mm512_or_si512(PermutexvarEpi8(p0, s0), alpha);
__m512i d1 = _mm512_or_si512(_mm512_permutex2var_epi8(s0, p1, s1), alpha);
uint32_t m32 = (1u << width) - 1u;
_mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, d0);
@ -4208,7 +4322,7 @@ void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) {
while (width >= 32) {
src -= 32;
__m512i v = _mm512_loadu_si512((const __m512i*)src);
_mm512_storeu_si512((__m512i*)dst, _mm512_permutexvar_epi16(rev, v));
_mm512_storeu_si512((__m512i*)dst, PermutexvarEpi16(rev, v));
dst += 32;
width -= 32;
}
@ -4216,7 +4330,7 @@ void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width) {
__mmask32 k = (__mmask32)((1u << width) - 1u);
__m512i idx = _mm512_sub_epi16(rev, _mm512_set1_epi16((short)(32 - width)));
__m512i v = _mm512_maskz_loadu_epi16(k, src - width);
_mm512_mask_storeu_epi16(dst, k, _mm512_permutexvar_epi16(idx, v));
_mm512_mask_storeu_epi16(dst, k, PermutexvarEpi16(idx, v));
}
_mm256_zeroupper();
}
@ -4402,7 +4516,7 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf,
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win);
const __m512i zmm5 = _mm512_set1_epi32(-1);
while (width >= 32) {
__m512i zmm3 = _mm512_permutexvar_epi16(
__m512i zmm3 = PermutexvarEpi16(
zmm_quad,
_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf)));
__m512i zmm4 =
@ -4425,10 +4539,8 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf,
_mm512_srai_epi16(
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
zmm5);
__m512i bg =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
_mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra));
_mm512_storeu_si512((__m512i*)(dst_argb + 64),
_mm512_unpackhi_epi16(bg, ra));
@ -4438,7 +4550,7 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf,
if (width > 0) {
uint32_t m32 = (1u << width) - 1u;
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
__m512i zmm3 = _mm512_permutexvar_epi16(
__m512i zmm3 = PermutexvarEpi16(
zmm_quad,
_mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf)));
__m512i zmm4 =
@ -4459,10 +4571,8 @@ void NV12ToARGBRow_AVX512BW(const uint8_t* y_buf,
_mm512_srai_epi16(
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
zmm5);
__m512i bg =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32,
_mm512_unpacklo_epi16(bg, ra));
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16),
@ -4496,7 +4606,7 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf,
_mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_Win);
const __m512i zmm5 = _mm512_set1_epi32(-1);
while (width >= 32) {
__m512i zmm3 = _mm512_permutexvar_epi16(
__m512i zmm3 = PermutexvarEpi16(
zmm_quad,
_mm512_castsi256_si512(_mm256_or_si256(
_mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8),
@ -4521,10 +4631,8 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf,
_mm512_srai_epi16(
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
zmm5);
__m512i bg =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
_mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra));
_mm512_storeu_si512((__m512i*)(dst_argb + 64),
_mm512_unpackhi_epi16(bg, ra));
@ -4534,7 +4642,7 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf,
if (width > 0) {
uint32_t m32 = (1u << width) - 1u;
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
__m512i zmm3 = _mm512_permutexvar_epi16(
__m512i zmm3 = PermutexvarEpi16(
zmm_quad,
_mm512_castsi256_si512(_mm256_or_si256(
_mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8),
@ -4557,10 +4665,8 @@ void NV21ToARGBRow_AVX512BW(const uint8_t* y_buf,
_mm512_srai_epi16(
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
zmm5);
__m512i bg =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32,
_mm512_unpacklo_epi16(bg, ra));
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16),
@ -4621,7 +4727,7 @@ void NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
const __m512i zmm_mask_DST1 =
_mm512_loadu_si512((const __m512i*)kMaskDST1_NV);
while (width >= 32) {
__m512i zmm3 = _mm512_permutexvar_epi16(
__m512i zmm3 = PermutexvarEpi16(
zmm_quad,
_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i*)uv_buf)));
__m512i zmm4 =
@ -4653,7 +4759,7 @@ void NV12ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
if (width > 0) {
uint32_t m32 = (1u << width) - 1u;
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
__m512i zmm3 = _mm512_permutexvar_epi16(
__m512i zmm3 = PermutexvarEpi16(
zmm_quad,
_mm512_castsi256_si512(_mm256_maskz_loadu_epi16(k_uv, uv_buf)));
__m512i zmm4 =
@ -4711,7 +4817,7 @@ void NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
const __m512i zmm_mask_DST1 =
_mm512_loadu_si512((const __m512i*)kMaskDST1_NV);
while (width >= 32) {
__m512i zmm3 = _mm512_permutexvar_epi16(
__m512i zmm3 = PermutexvarEpi16(
zmm_quad,
_mm512_castsi256_si512(_mm256_or_si256(
_mm256_slli_epi16(_mm256_loadu_si256((const __m256i*)vu_buf), 8),
@ -4745,7 +4851,7 @@ void NV21ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
if (width > 0) {
uint32_t m32 = (1u << width) - 1u;
__mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
__m512i zmm3 = _mm512_permutexvar_epi16(
__m512i zmm3 = PermutexvarEpi16(
zmm_quad,
_mm512_castsi256_si512(_mm256_or_si256(
_mm256_slli_epi16(_mm256_maskz_loadu_epi16(k_uv, vu_buf), 8),
@ -4811,8 +4917,7 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
__m256i yuv = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)),
_mm_unpackhi_epi8(xu, xv), 1);
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_castsi256_si512(yuv));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv));
__m512i zmm4 =
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
__m512i zmm5 = _mm512_packus_epi16(
@ -4836,10 +4941,8 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
_mm512_srai_epi16(
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
zmm5);
__m512i bg =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
_mm512_storeu_si512((__m512i*)dst_argb, _mm512_unpacklo_epi16(bg, ra));
_mm512_storeu_si512((__m512i*)(dst_argb + 64),
_mm512_unpackhi_epi16(bg, ra));
@ -4854,8 +4957,7 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
__m256i yuv = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_unpacklo_epi8(xu, xv)),
_mm_unpackhi_epi8(xu, xv), 1);
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_castsi256_si512(yuv));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_castsi256_si512(yuv));
__m512i zmm4 =
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
__m512i zmm5 = _mm512_packus_epi16(
@ -4877,10 +4979,8 @@ void I422AlphaToARGBRow_AVX512BW(const uint8_t* y_buf,
_mm512_srai_epi16(
_mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4), 6),
zmm5);
__m512i bg =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
__m512i bg = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1));
__m512i ra = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5));
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)m32,
_mm512_unpacklo_epi16(bg, ra));
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(m32 >> 16),
@ -4918,8 +5018,7 @@ void I400ToARGBRow_AVX512BW(const uint8_t* y_buf,
y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8));
y16 = _mm512_srai_epi16(
_mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6);
__m512i g =
_mm512_permutexvar_epi64(zmm_lin, _mm512_packus_epi16(y16, y16));
__m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16));
__m256i g256 = _mm512_castsi512_si256(g);
__m512i d0 = _mm512_or_si512(
_mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)),
@ -4941,8 +5040,7 @@ void I400ToARGBRow_AVX512BW(const uint8_t* y_buf,
y16 = _mm512_or_si512(y16, _mm512_slli_epi16(y16, 8));
y16 = _mm512_srai_epi16(
_mm512_adds_epi16(_mm512_mulhi_epu16(y16, zmm_yg), zmm_ygb), 6);
__m512i g =
_mm512_permutexvar_epi64(zmm_lin, _mm512_packus_epi16(y16, y16));
__m512i g = PermutexvarEpi64(zmm_lin, _mm512_packus_epi16(y16, y16));
__m256i g256 = _mm512_castsi512_si256(g);
__m512i d0 = _mm512_or_si512(
_mm512_shuffle_epi8(_mm512_cvtepu8_epi32(_mm256_castsi256_si128(g256)),
@ -5111,8 +5209,8 @@ void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb,
src_argb += 64;
__m512i y32 = _mm512_srli_epi32(
_mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8);
__m512i out = _mm512_ternarylogic_epi32(_mm512_shuffle_epi8(y32, c_shuf), s,
c_alpha, 0xf8);
__m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf),
_mm512_and_si512(s, c_alpha));
_mm512_storeu_si512((__m512i*)dst_argb, out);
dst_argb += 64;
width -= 16;
@ -5122,8 +5220,8 @@ void ARGBGrayRow_AVX512VNNI(const uint8_t* src_argb,
__m512i s = _mm512_maskz_loadu_epi32(k, src_argb);
__m512i y32 = _mm512_srli_epi32(
_mm512_dpbusd_epi32(c_bias, c_coeff, _mm512_xor_si512(s, c_sub128)), 8);
__m512i out = _mm512_ternarylogic_epi32(_mm512_shuffle_epi8(y32, c_shuf), s,
c_alpha, 0xf8);
__m512i out = _mm512_or_si512(_mm512_shuffle_epi8(y32, c_shuf),
_mm512_and_si512(s, c_alpha));
_mm512_mask_storeu_epi32(dst_argb, k, out);
}
_mm256_zeroupper();
@ -5377,7 +5475,7 @@ void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0,
__m512i sa_eff = _mm512_or_si512(sa, t);
__m512i den_eff = _mm512_or_si512(da2, t);
__m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff);
__m512i ratio = _mm512_cvttps_epi32(
__m512i ratio = CvttpsEpi32(
_mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff)));
__m512i r_rep =
_mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0);
@ -5416,7 +5514,7 @@ void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0,
__m512i sa_eff = _mm512_or_si512(sa, t);
__m512i den_eff = _mm512_or_si512(da2, t);
__m512i num = _mm512_sub_epi32(_mm512_slli_epi32(sa_eff, 8), sa_eff);
__m512i ratio = _mm512_cvttps_epi32(
__m512i ratio = CvttpsEpi32(
_mm512_div_ps(_mm512_cvtepi32_ps(num), _mm512_cvtepi32_ps(den_eff)));
__m512i r_rep =
_mm512_shufflelo_epi16(_mm512_shufflehi_epi16(ratio, 0xa0), 0xa0);
@ -6671,25 +6769,25 @@ static const uint8_t kShuffleUYVYUV_Win[16] = {0, 2, 0, 2, 4, 6, 4, 6,
#define STOREARGB_WIN_AVX512BW(dst_ptr) \
do { \
__m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \
_mm512_unpacklo_epi8(zmm0, zmm1)); \
__m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \
_mm512_unpacklo_epi8(zmm2, zmm5)); \
__m512i bg = \
PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \
__m512i ra = \
PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \
_mm512_storeu_si512((__m512i*)(dst_ptr), _mm512_unpacklo_epi16(bg, ra)); \
_mm512_storeu_si512((__m512i*)((dst_ptr) + 64), \
_mm512_unpackhi_epi16(bg, ra)); \
} while (0)
#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \
do { \
__m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \
_mm512_unpacklo_epi8(zmm0, zmm1)); \
__m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \
_mm512_unpacklo_epi8(zmm2, zmm5)); \
_mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \
_mm512_unpacklo_epi16(bg, ra)); \
_mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \
_mm512_unpackhi_epi16(bg, ra)); \
#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \
do { \
__m512i bg = \
PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm0, zmm1)); \
__m512i ra = \
PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm2, zmm5)); \
_mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \
_mm512_unpacklo_epi16(bg, ra)); \
_mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \
_mm512_unpackhi_epi16(bg, ra)); \
} while (0)
#define PACKAR30_WIN_AVX512BW(dst0, dst1) \
@ -6728,8 +6826,7 @@ void I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
__m256i uv256 = _mm256_inserti128_si256(
_mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
_mm_unpackhi_epi8(xu, xv), 1);
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
y_buf += 32;
@ -6747,8 +6844,7 @@ void I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
__m256i uv256 = _mm256_inserti128_si256(
_mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
_mm_unpackhi_epi8(xu, xv), 1);
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
@ -6781,17 +6877,14 @@ void I422ToRGBARow_AVX512BW(const uint8_t* y_buf,
__m256i uv256 = _mm256_inserti128_si256(
_mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
_mm_unpackhi_epi8(xu, xv), 1);
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf));
y_buf += 32;
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
YUVTORGB8_WIN_AVX512BW;
__m512i ab =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0));
__m512i gr =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2));
__m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0));
__m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2));
_mm512_storeu_si512((__m512i*)dst_rgba, _mm512_unpacklo_epi16(ab, gr));
_mm512_storeu_si512((__m512i*)(dst_rgba + 64),
_mm512_unpackhi_epi16(ab, gr));
@ -6806,16 +6899,13 @@ void I422ToRGBARow_AVX512BW(const uint8_t* y_buf,
__m256i uv256 = _mm256_inserti128_si256(
_mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)),
_mm_unpackhi_epi8(xu, xv), 1);
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf));
zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8));
YUVTORGB8_WIN_AVX512BW;
__m512i ab =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0));
__m512i gr =
_mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2));
__m512i ab = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0));
__m512i gr = PermutexvarEpi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2));
_mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32,
_mm512_unpacklo_epi16(ab, gr));
_mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16),
@ -7110,8 +7200,7 @@ void I210ToARGBRow_AVX512BW(const uint16_t* y_buf,
y_buf += 32;
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
YUVTORGB8_WIN_AVX512BW;
@ -7127,8 +7216,7 @@ void I210ToARGBRow_AVX512BW(const uint16_t* y_buf,
__m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
YUVTORGB8_WIN_AVX512BW;
@ -7166,8 +7254,7 @@ void I210ToAR30Row_AVX512BW(const uint16_t* y_buf,
y_buf += 32;
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
YUVTORGB16_WIN_AVX512BW;
@ -7185,8 +7272,7 @@ void I210ToAR30Row_AVX512BW(const uint16_t* y_buf,
__m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
YUVTORGB16_WIN_AVX512BW;
@ -7226,8 +7312,7 @@ void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
a_buf += 32;
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
__m512i zmm5 = _mm512_packus_epi16(a16, a16);
@ -7246,8 +7331,7 @@ void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf,
_mm512_srli_epi16(_mm512_maskz_loadu_epi16((__mmask32)m32, a_buf), 2);
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4));
__m512i zmm5 = _mm512_packus_epi16(a16, a16);
@ -7283,8 +7367,7 @@ void I212ToARGBRow_AVX512BW(const uint16_t* y_buf,
y_buf += 32;
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
YUVTORGB8_WIN_AVX512BW;
@ -7300,8 +7383,7 @@ void I212ToARGBRow_AVX512BW(const uint16_t* y_buf,
__m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
YUVTORGB8_WIN_AVX512BW;
@ -7339,8 +7421,7 @@ void I212ToAR30Row_AVX512BW(const uint16_t* y_buf,
y_buf += 32;
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
YUVTORGB16_WIN_AVX512BW;
@ -7358,8 +7439,7 @@ void I212ToAR30Row_AVX512BW(const uint16_t* y_buf,
__m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf);
__m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s),
_mm256_packus_epi16(v_s, v_s));
__m512i zmm3 =
_mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm3 = PermutexvarEpi16(zmm_quad, _mm512_zextsi256_si512(uv256));
__m512i zmm4 =
_mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8));
YUVTORGB16_WIN_AVX512BW;
@ -8071,7 +8151,7 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8);
zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2));
_mm512_storeu_si512((__m512i*)dst_y, zmm0);
src_argb += 256;
@ -8116,7 +8196,7 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
8);
_mm512_storeu_si512(
(__m512i*)dst_u,
_mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(u01, u23)));
PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(u01, u23)));
__m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one);
__m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one);
@ -8130,7 +8210,7 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
8);
_mm512_storeu_si512(
(__m512i*)dst_v,
_mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(v01, v23)));
PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(v01, v23)));
src_argb += 256;
dst_u += 64;
@ -8192,13 +8272,13 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero);
zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1));
zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1));
zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one);
zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one);
zmm0 = _mm512_packs_epi32(zmm1, zmm2);
zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0));
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0));
_mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0));
_mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1));
@ -8241,13 +8321,13 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero);
zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1));
zmm0 = PermutexvarEpi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1));
zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one);
zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one);
zmm0 = _mm512_packs_epi32(zmm1, zmm2);
zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0));
zmm0 = PermutexvarEpi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0));
int halfwidth = (width + 1) >> 1;
__mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u);

74
source/scale_win.cc
View File

@ -1544,6 +1544,32 @@ static const uint8_t kPackEvenBytes[64] = {
78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102,
104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126};
// Under MemorySanitizer, use a portable lane-by-lane C loop so shadow
// propagation relies only on standard C memory/array semantics rather than
// target-specific LLVM intrinsic modeling. In non-MSan builds, call the native
// AVX-512 intrinsic directly.
LIBYUV_TARGET_AVX512VBMI
static inline __m512i MaskPermutexvarEpi32(__m512i src,
__mmask16 k,
__m512i idx,
__m512i v) {
#if defined(LIBYUV_MSAN)
uint32_t src_arr[16];
uint32_t idx_arr[16];
uint32_t v_arr[16];
uint32_t out_arr[16];
_mm512_storeu_si512((__m512i*)src_arr, src);
_mm512_storeu_si512((__m512i*)idx_arr, idx);
_mm512_storeu_si512((__m512i*)v_arr, v);
for (int i = 0; i < 16; ++i) {
out_arr[i] = ((k >> i) & 1) ? v_arr[idx_arr[i] & 15] : src_arr[i];
}
return _mm512_loadu_si512((const __m512i*)out_arr);
#else
return _mm512_mask_permutexvar_epi32(src, k, idx, v);
#endif
}
// Scale UV columns by 3 box filter using AVX512VBMI.
// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop
// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels).
@ -1567,12 +1593,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width,
__m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv);
__m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
__m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64));
__m512i a0 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2);
__m512i a1 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2);
__m512i a2 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2);
__m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1),
k5, col0, s2);
__m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1),
k5, col1, s2);
__m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1),
k6, col2, s2);
__m512i sum0 = _mm512_mulhi_epu16(
_mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale);
@ -1580,12 +1606,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width,
__m512i s4 = _mm512_loadu_si512((const __m512i*)(src_uv + 128));
__m512i s5 = _mm512_loadu_si512((const __m512i*)(src_uv + 160));
src_uv += 192;
__m512i b0 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s3, col0, s4), k5, col0, s5);
__m512i b1 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s3, col1, s4), k5, col1, s5);
__m512i b2 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s3, col2, s4), k6, col2, s5);
__m512i b0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col0, s4),
k5, col0, s5);
__m512i b1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col1, s4),
k5, col1, s5);
__m512i b2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s3, col2, s4),
k6, col2, s5);
__m512i sum1 = _mm512_mulhi_epu16(
_mm512_add_epi16(_mm512_add_epi16(b0, b1), b2), vscale);
@ -1599,12 +1625,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width,
__m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32));
__m512i s2 = _mm512_loadu_si512((const __m512i*)(src_uv + 64));
src_uv += 96;
__m512i a0 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2);
__m512i a1 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2);
__m512i a2 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2);
__m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1),
k5, col0, s2);
__m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1),
k5, col1, s2);
__m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1),
k6, col2, s2);
__m512i sum0 = _mm512_mulhi_epu16(
_mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale);
_mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(sum0));
@ -1620,12 +1646,12 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width,
__m512i s0 = _mm512_maskz_loadu_epi32(k1, src_uv);
__m512i s1 = _mm512_maskz_loadu_epi32(k2, src_uv + 32);
__m512i s2 = _mm512_maskz_loadu_epi32(k3, src_uv + 64);
__m512i a0 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s0, col0, s1), k5, col0, s2);
__m512i a1 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s0, col1, s1), k5, col1, s2);
__m512i a2 = _mm512_mask_permutexvar_epi32(
_mm512_permutex2var_epi32(s0, col2, s1), k6, col2, s2);
__m512i a0 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col0, s1),
k5, col0, s2);
__m512i a1 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col1, s1),
k5, col1, s2);
__m512i a2 = MaskPermutexvarEpi32(_mm512_permutex2var_epi32(s0, col2, s1),
k6, col2, s2);
__m512i sum0 = _mm512_mulhi_epu16(
_mm512_add_epi16(_mm512_add_epi16(a0, a1), a2), vscale);
_mm256_mask_storeu_epi16(dst_uv, k4, _mm512_cvtepi16_epi8(sum0));

310
unit_test/planar_test.cc
View File

@ -22,6 +22,7 @@
#include "libyuv/planar_functions.h"
#include "libyuv/rotate.h"
#include "libyuv/scale.h"
#include "libyuv/scale_uv.h"
#if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \
!defined(ENABLE_ROW_TESTS)
@ -38,11 +39,16 @@
#if __has_feature(memory_sanitizer)
#include <sanitizer/msan_interface.h>
#define MSAN_POISON(ptr, size) __msan_allocated_memory(ptr, size)
#define MSAN_CHECK_INITIALIZED(ptr, size) \
__msan_check_mem_is_initialized(ptr, size)
#endif
#endif
#ifndef MSAN_POISON
#define MSAN_POISON(ptr, size)
#endif
#ifndef MSAN_CHECK_INITIALIZED
#define MSAN_CHECK_INITIALIZED(ptr, size)
#endif
#define EXPECTED_UNATTENUATE_DIFF 0
@ -501,6 +507,36 @@ TEST_F(LibYUVPlanarTest, TestARGBGrayTo) {
ASSERT_EQ(i, orig_pixels[i][2]);
ASSERT_EQ(i, orig_pixels[i][3]);
}
// Test RGBX input with uninitialized alpha channel under MSan.
SIMD_ALIGNED(uint8_t gray_pixels_c[37][4]);
for (int i = 0; i < 37; ++i) {
orig_pixels[i][0] = static_cast<uint8_t>(i * 7);
orig_pixels[i][1] = static_cast<uint8_t>(i * 13);
orig_pixels[i][2] = static_cast<uint8_t>(i * 23);
orig_pixels[i][3] = 0u;
}
MaskCpuFlags(disable_cpu_flags_);
ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels_c[0][0], 0, 37, 1);
MaskCpuFlags(benchmark_cpu_info_);
for (int i = 0; i < 37; ++i) {
MSAN_POISON(&orig_pixels[i][3], 1);
}
MSAN_POISON(gray_pixels, sizeof(gray_pixels));
ARGBGrayTo(&orig_pixels[0][0], 0, &gray_pixels[0][0], 0, 37, 1);
for (int i = 0; i < 37; ++i) {
MSAN_CHECK_INITIALIZED(&gray_pixels[i][0], 3);
ASSERT_EQ(gray_pixels_c[i][0], gray_pixels[i][0]);
ASSERT_EQ(gray_pixels_c[i][1], gray_pixels[i][1]);
ASSERT_EQ(gray_pixels_c[i][2], gray_pixels[i][2]);
}
ARGBGray(&orig_pixels[0][0], 0, 0, 0, 37, 1);
for (int i = 0; i < 37; ++i) {
MSAN_CHECK_INITIALIZED(&orig_pixels[i][0], 3);
ASSERT_EQ(gray_pixels_c[i][0], orig_pixels[i][0]);
ASSERT_EQ(gray_pixels_c[i][1], orig_pixels[i][1]);
ASSERT_EQ(gray_pixels_c[i][2], orig_pixels[i][2]);
}
}
TEST_F(LibYUVPlanarTest, TestARGBSepia) {
@ -5874,4 +5910,278 @@ TEST_F(LibYUVPlanarTest, ARGBAlphaBlend_AllAlphas) {
free_aligned_buffer_page_end(dst_argb_opt);
}
TEST_F(LibYUVPlanarTest, MSanUninitializedInputsAndTails) {
static const int kWidths[] = {1, 7, 15, 16, 17, 31, 32, 33, 63, 64, 65, 99};
const int kMaxW = 128;
const int kMaxSrcUV = kMaxW * 3 * 2 + 128;
align_buffer_page_end(src0, kMaxW * 4 + 128);
align_buffer_page_end(src1, kMaxW * 4 + 128);
align_buffer_page_end(src2, kMaxW * 4 + 128);
align_buffer_page_end(dst_c0, kMaxW * 4 + 128);
align_buffer_page_end(dst_c1, kMaxW * 4 + 128);
align_buffer_page_end(dst_c2, kMaxW * 4 + 128);
align_buffer_page_end(dst_opt0, kMaxW * 4 + 128);
align_buffer_page_end(dst_opt1, kMaxW * 4 + 128);
align_buffer_page_end(dst_opt2, kMaxW * 4 + 128);
align_buffer_page_end(uv_src, kMaxSrcUV * 3);
for (size_t wi = 0; wi < sizeof(kWidths) / sizeof(kWidths[0]); ++wi) {
const int w = kWidths[wi];
const int uv_w = (w + 1) / 2;
// 1. ARGBToI444 and ARGBToI420 with poisoned RGBX alpha and poisoned tail.
for (int i = 0; i < w; ++i) {
src0[i * 4 + 0] = static_cast<uint8_t>((i * 11 + 3) & 255);
src0[i * 4 + 1] = static_cast<uint8_t>((i * 17 + 7) & 255);
src0[i * 4 + 2] = static_cast<uint8_t>((i * 29 + 13) & 255);
src0[i * 4 + 3] = 0u;
}
MaskCpuFlags(disable_cpu_flags_);
ARGBToI444(src0, w * 4, dst_c0, w, dst_c1, w, dst_c2, w, w, 1);
MaskCpuFlags(benchmark_cpu_info_);
for (int i = 0; i < w; ++i) {
MSAN_POISON(&src0[i * 4 + 3], 1);
}
MSAN_POISON(src0 + w * 4, 128);
MSAN_POISON(dst_opt0, kMaxW * 4);
MSAN_POISON(dst_opt1, kMaxW * 4);
MSAN_POISON(dst_opt2, kMaxW * 4);
ARGBToI444(src0, w * 4, dst_opt0, w, dst_opt1, w, dst_opt2, w, w, 1);
MSAN_CHECK_INITIALIZED(dst_opt0, w);
MSAN_CHECK_INITIALIZED(dst_opt1, w);
MSAN_CHECK_INITIALIZED(dst_opt2, w);
for (int i = 0; i < w; ++i) {
EXPECT_EQ(dst_c0[i], dst_opt0[i]) << "ARGBToI444 Y w=" << w;
EXPECT_EQ(dst_c1[i], dst_opt1[i]) << "ARGBToI444 U w=" << w;
EXPECT_EQ(dst_c2[i], dst_opt2[i]) << "ARGBToI444 V w=" << w;
}
for (int i = 0; i < w; ++i) {
src0[i * 4 + 3] = 0u;
}
MaskCpuFlags(disable_cpu_flags_);
ARGBToI420(src0, w * 4, dst_c0, w, dst_c1, uv_w, dst_c2, uv_w, w, 1);
MaskCpuFlags(benchmark_cpu_info_);
for (int i = 0; i < w; ++i) {
MSAN_POISON(&src0[i * 4 + 3], 1);
}
MSAN_POISON(src0 + w * 4, 128);
MSAN_POISON(dst_opt0, kMaxW * 4);
MSAN_POISON(dst_opt1, kMaxW * 4);
MSAN_POISON(dst_opt2, kMaxW * 4);
ARGBToI420(src0, w * 4, dst_opt0, w, dst_opt1, uv_w, dst_opt2, uv_w, w, 1);
MSAN_CHECK_INITIALIZED(dst_opt0, w);
MSAN_CHECK_INITIALIZED(dst_opt1, uv_w);
MSAN_CHECK_INITIALIZED(dst_opt2, uv_w);
for (int i = 0; i < w; ++i) {
EXPECT_EQ(dst_c0[i], dst_opt0[i]) << "ARGBToI420 Y w=" << w;
}
for (int i = 0; i < uv_w; ++i) {
EXPECT_EQ(dst_c1[i], dst_opt1[i]) << "ARGBToI420 U w=" << w;
EXPECT_EQ(dst_c2[i], dst_opt2[i]) << "ARGBToI420 V w=" << w;
}
// 2. ARGBMirror with poisoned alpha on even pixels (verifying per-lane
// shadow permutation) and poisoned tail.
for (int i = 0; i < w; ++i) {
src0[i * 4 + 0] = static_cast<uint8_t>((i * 5 + 1) & 255);
src0[i * 4 + 1] = static_cast<uint8_t>((i * 9 + 2) & 255);
src0[i * 4 + 2] = static_cast<uint8_t>((i * 15 + 3) & 255);
src0[i * 4 + 3] = static_cast<uint8_t>((i * 21 + 4) & 255);
}
MaskCpuFlags(disable_cpu_flags_);
ARGBMirror(src0, w * 4, dst_c0, w * 4, w, 1);
MaskCpuFlags(benchmark_cpu_info_);
for (int i = 0; i < w; i += 2) {
MSAN_POISON(&src0[i * 4 + 3], 1);
}
MSAN_POISON(src0 + w * 4, 128);
MSAN_POISON(dst_opt0, kMaxW * 4);
ARGBMirror(src0, w * 4, dst_opt0, w * 4, w, 1);
for (int i = 0; i < w; ++i) {
const int src_i = w - 1 - i;
MSAN_CHECK_INITIALIZED(&dst_opt0[i * 4], 3);
EXPECT_EQ(dst_c0[i * 4 + 0], dst_opt0[i * 4 + 0]);
EXPECT_EQ(dst_c0[i * 4 + 1], dst_opt0[i * 4 + 1]);
EXPECT_EQ(dst_c0[i * 4 + 2], dst_opt0[i * 4 + 2]);
if (src_i & 1) {
MSAN_CHECK_INITIALIZED(&dst_opt0[i * 4 + 3], 1);
EXPECT_EQ(dst_c0[i * 4 + 3], dst_opt0[i * 4 + 3]);
}
}
// 3. RGB24Mirror, RAWToRGBA, RAWToRGB24, MergeRGBPlane with poisoned tails.
for (int i = 0; i < w * 3; ++i) {
src0[i] = static_cast<uint8_t>((i * 19 + 5) & 255);
}
MSAN_POISON(src0 + w * 3, 128);
MaskCpuFlags(disable_cpu_flags_);
RGB24Mirror(src0, w * 3, dst_c0, w * 3, w, 1);
RAWToRGBA(src0, w * 3, dst_c1, w * 4, w, 1);
RAWToRGB24(src0, w * 3, dst_c2, w * 3, w, 1);
MaskCpuFlags(benchmark_cpu_info_);
MSAN_POISON(dst_opt0, kMaxW * 4);
MSAN_POISON(dst_opt1, kMaxW * 4);
MSAN_POISON(dst_opt2, kMaxW * 4);
RGB24Mirror(src0, w * 3, dst_opt0, w * 3, w, 1);
RAWToRGBA(src0, w * 3, dst_opt1, w * 4, w, 1);
RAWToRGB24(src0, w * 3, dst_opt2, w * 3, w, 1);
MSAN_CHECK_INITIALIZED(dst_opt0, w * 3);
MSAN_CHECK_INITIALIZED(dst_opt1, w * 4);
MSAN_CHECK_INITIALIZED(dst_opt2, w * 3);
EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 3));
EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 4));
EXPECT_EQ(0, memcmp(dst_c2, dst_opt2, w * 3));
for (int i = 0; i < w; ++i) {
src0[i] = static_cast<uint8_t>((i * 3 + 11) & 255);
src1[i] = static_cast<uint8_t>((i * 7 + 23) & 255);
src2[i] = static_cast<uint8_t>((i * 13 + 37) & 255);
}
MSAN_POISON(src0 + w, 128);
MSAN_POISON(src1 + w, 128);
MSAN_POISON(src2 + w, 128);
MaskCpuFlags(disable_cpu_flags_);
MergeRGBPlane(src0, w, src1, w, src2, w, dst_c0, w * 3, w, 1);
MirrorPlane(src0, w, dst_c1, w, w, 1);
MaskCpuFlags(benchmark_cpu_info_);
MSAN_POISON(dst_opt0, kMaxW * 4);
MSAN_POISON(dst_opt1, kMaxW * 4);
MergeRGBPlane(src0, w, src1, w, src2, w, dst_opt0, w * 3, w, 1);
MirrorPlane(src0, w, dst_opt1, w, w, 1);
MSAN_CHECK_INITIALIZED(dst_opt0, w * 3);
MSAN_CHECK_INITIALIZED(dst_opt1, w);
EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 3));
EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w));
// 4. MirrorUVPlane and MirrorPlane_16 with poisoned tails.
for (int i = 0; i < w * 2; ++i) {
src0[i] = static_cast<uint8_t>((i * 23 + 9) & 255);
}
MSAN_POISON(src0 + w * 2, 128);
MaskCpuFlags(disable_cpu_flags_);
MirrorUVPlane(src0, w * 2, dst_c0, w * 2, w, 1);
MirrorPlane_16(reinterpret_cast<const uint16_t*>(src0), w,
reinterpret_cast<uint16_t*>(dst_c1), w, w, 1);
MaskCpuFlags(benchmark_cpu_info_);
MSAN_POISON(dst_opt0, kMaxW * 4);
MSAN_POISON(dst_opt1, kMaxW * 4);
MirrorUVPlane(src0, w * 2, dst_opt0, w * 2, w, 1);
MirrorPlane_16(reinterpret_cast<const uint16_t*>(src0), w,
reinterpret_cast<uint16_t*>(dst_opt1), w, w, 1);
MSAN_CHECK_INITIALIZED(dst_opt0, w * 2);
MSAN_CHECK_INITIALIZED(dst_opt1, w * 2);
EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 2));
EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 2));
// 5. I420ToAR30, NV12ToARGB, NV12ToRGB24, I444ToARGB, I210ToARGB.
for (int i = 0; i < w; ++i) {
src0[i] = static_cast<uint8_t>((i * 5 + 16) & 255);
src1[i] = static_cast<uint8_t>((i * 9 + 64) & 255);
src2[i] = static_cast<uint8_t>((i * 11 + 128) & 255);
}
MSAN_POISON(src0 + w, 128);
MSAN_POISON(src1 + uv_w, 128);
MSAN_POISON(src2 + uv_w, 128);
MaskCpuFlags(disable_cpu_flags_);
I420ToAR30(src0, w, src1, uv_w, src2, uv_w, dst_c0, w * 4, w, 1);
MaskCpuFlags(benchmark_cpu_info_);
MSAN_POISON(dst_opt0, kMaxW * 4);
I420ToAR30(src0, w, src1, uv_w, src2, uv_w, dst_opt0, w * 4, w, 1);
MSAN_CHECK_INITIALIZED(dst_opt0, w * 4);
EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4));
for (int i = 0; i < uv_w * 2; ++i) {
src1[i] = static_cast<uint8_t>((i * 13 + 80) & 255);
}
MSAN_POISON(src1 + uv_w * 2, 128);
MaskCpuFlags(disable_cpu_flags_);
NV12ToARGB(src0, w, src1, uv_w * 2, dst_c0, w * 4, w, 1);
NV12ToRGB24(src0, w, src1, uv_w * 2, dst_c1, w * 3, w, 1);
MaskCpuFlags(benchmark_cpu_info_);
MSAN_POISON(dst_opt0, kMaxW * 4);
MSAN_POISON(dst_opt1, kMaxW * 4);
NV12ToARGB(src0, w, src1, uv_w * 2, dst_opt0, w * 4, w, 1);
NV12ToRGB24(src0, w, src1, uv_w * 2, dst_opt1, w * 3, w, 1);
MSAN_CHECK_INITIALIZED(dst_opt0, w * 4);
MSAN_CHECK_INITIALIZED(dst_opt1, w * 3);
EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4));
EXPECT_EQ(0, memcmp(dst_c1, dst_opt1, w * 3));
for (int i = 0; i < w; ++i) {
reinterpret_cast<uint16_t*>(src0)[i] =
static_cast<uint16_t>((i * 17 + 64) & 1023);
}
for (int i = 0; i < uv_w; ++i) {
reinterpret_cast<uint16_t*>(src1)[i] =
static_cast<uint16_t>((i * 23 + 256) & 1023);
reinterpret_cast<uint16_t*>(src2)[i] =
static_cast<uint16_t>((i * 31 + 512) & 1023);
}
MSAN_POISON(src0 + w * 2, 128);
MSAN_POISON(src1 + uv_w * 2, 128);
MSAN_POISON(src2 + uv_w * 2, 128);
MaskCpuFlags(disable_cpu_flags_);
I210ToARGB(reinterpret_cast<const uint16_t*>(src0), w,
reinterpret_cast<const uint16_t*>(src1), uv_w,
reinterpret_cast<const uint16_t*>(src2), uv_w, dst_c0, w * 4, w,
1);
MaskCpuFlags(benchmark_cpu_info_);
MSAN_POISON(dst_opt0, kMaxW * 4);
I210ToARGB(reinterpret_cast<const uint16_t*>(src0), w,
reinterpret_cast<const uint16_t*>(src1), uv_w,
reinterpret_cast<const uint16_t*>(src2), uv_w, dst_opt0, w * 4,
w, 1);
MSAN_CHECK_INITIALIZED(dst_opt0, w * 4);
EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4));
// 6. ARGBAlphaBlend with poisoned tails and poisoned destination.
for (int i = 0; i < w * 4; ++i) {
src0[i] = static_cast<uint8_t>((i * 29 + 3) & 255);
src1[i] = static_cast<uint8_t>((i * 37 + 11) & 255);
}
MSAN_POISON(src0 + w * 4, 128);
MSAN_POISON(src1 + w * 4, 128);
MaskCpuFlags(disable_cpu_flags_);
ARGBAlphaBlend(src0, w * 4, src1, w * 4, dst_c0, w * 4, w, 1);
MaskCpuFlags(benchmark_cpu_info_);
MSAN_POISON(dst_opt0, kMaxW * 4);
ARGBAlphaBlend(src0, w * 4, src1, w * 4, dst_opt0, w * 4, w, 1);
MSAN_CHECK_INITIALIZED(dst_opt0, w * 4);
EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 4));
// 7. UVScale 3x box downscale (hits ScaleUVAddCols3_AVX512VBMI).
const int src_uv_w = w * 3;
const int src_uv_stride = src_uv_w * 2 + 64;
MSAN_POISON(uv_src, kMaxSrcUV * 3);
for (int row = 0; row < 3; ++row) {
for (int i = 0; i < src_uv_w * 2; ++i) {
uv_src[row * src_uv_stride + i] =
static_cast<uint8_t>((row * 53 + i * 17 + 7) & 255);
}
}
MaskCpuFlags(disable_cpu_flags_);
UVScale(uv_src, src_uv_stride, src_uv_w, 3, dst_c0, w * 2, w, 1,
kFilterBox);
MaskCpuFlags(benchmark_cpu_info_);
MSAN_POISON(dst_opt0, kMaxW * 4);
UVScale(uv_src, src_uv_stride, src_uv_w, 3, dst_opt0, w * 2, w, 1,
kFilterBox);
MSAN_CHECK_INITIALIZED(dst_opt0, w * 2);
EXPECT_EQ(0, memcmp(dst_c0, dst_opt0, w * 2));
}
free_aligned_buffer_page_end(src0);
free_aligned_buffer_page_end(src1);
free_aligned_buffer_page_end(src2);
free_aligned_buffer_page_end(dst_c0);
free_aligned_buffer_page_end(dst_c1);
free_aligned_buffer_page_end(dst_c2);
free_aligned_buffer_page_end(dst_opt0);
free_aligned_buffer_page_end(dst_opt1);
free_aligned_buffer_page_end(dst_opt2);
free_aligned_buffer_page_end(uv_src);
}
} // namespace libyuv