Fix unit test buffer sizes and skip large allocation on MSan

In P010ToARGBMatrixFilterOverflow, P210ToARGBMatrixFilterOverflow,
P010ToAR30MatrixFilterOverflow, and P210ToAR30MatrixFilterOverflow,
the tests set src_stride_uv to width (1039) and allocated
width * src_uv_height elements. For odd widths, the interleaved UV
row requires (width + 1) & ~1 elements (1040). Using width for the
stride and buffer size caused the second row to read past the allocated
buffer and access uninitialized memory under MSan.
Fix this by computing src_stride_uv = (width + 1) & ~1 and sizing the
allocated buffer to src_stride_uv * src_uv_height.

In TestI400LargeSize, the test attempts large allocations totaling
over 10 GB (~2 GB for source and ~8.5 GB for destination). Under
MemorySanitizer (MSan), large allocations can exceed shadow memory
virtual address limits or assert. Skip this test under MSan by checking
__SANITIZE_MEMORY__ and __has_feature(memory_sanitizer).

Test: libyuv_unittest --gtest_filter=*FilterOverflow*:*I400Large*
Bug: 549112862
Change-Id: I679b7c67d93822e83f066b1895924589fc7242e7
TAG=agy
CONV=dd635180-175b-4e35-b7f0-2e38105d3506
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8269585
Commit-Queue: Frank Barchard <fbarchard@google.com>
Reviewed-by: Wan-Teh Chang <wtc@google.com>
This commit is contained in:
Frank Barchard 2026-08-18 19:48:01 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent 50bfd8fc08
commit fd64b1b5ab

View File

@ -2742,12 +2742,14 @@ TEST_F(LibYUVConvertTest, P010ToARGBMatrixFilterOverflow) {
int width = 1024 + 15; // 1039
int height = 4;
int dst_stride_argb = width * 4;
int src_stride_uv = (width + 1) & ~1;
int src_uv_height = (height + 1) / 2;
align_buffer_page_end_16(src_y, width * height);
align_buffer_page_end_16(src_uv, width * ((height + 1) / 2));
align_buffer_page_end_16(src_uv, src_stride_uv * src_uv_height);
align_buffer_page_end(dst_argb, dst_stride_argb * height);
memset(src_y, 0x02, width * height * sizeof(uint16_t));
memset(src_uv, 0x02, width * ((height + 1) / 2) * sizeof(uint16_t));
EXPECT_EQ(0, P010ToARGBMatrixFilter(src_y, width, src_uv, width, dst_argb,
memset(src_uv, 0x02, src_stride_uv * src_uv_height * sizeof(uint16_t));
EXPECT_EQ(0, P010ToARGBMatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb,
dst_stride_argb, &kYuvI601Constants,
width, height, kFilterBilinear));
free_aligned_buffer_page_end_16(src_y);
@ -2761,12 +2763,13 @@ TEST_F(LibYUVConvertTest, P210ToARGBMatrixFilterOverflow) {
int width = 1024 + 15; // 1039
int height = 4;
int dst_stride_argb = width * 4;
int src_stride_uv = (width + 1) & ~1;
align_buffer_page_end_16(src_y, width * height);
align_buffer_page_end_16(src_uv, width * height);
align_buffer_page_end_16(src_uv, src_stride_uv * height);
align_buffer_page_end(dst_argb, dst_stride_argb * height);
memset(src_y, 0x02, width * height * sizeof(uint16_t));
memset(src_uv, 0x02, width * height * sizeof(uint16_t));
EXPECT_EQ(0, P210ToARGBMatrixFilter(src_y, width, src_uv, width, dst_argb,
memset(src_uv, 0x02, src_stride_uv * height * sizeof(uint16_t));
EXPECT_EQ(0, P210ToARGBMatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb,
dst_stride_argb, &kYuvI601Constants,
width, height, kFilterBilinear));
free_aligned_buffer_page_end_16(src_y);
@ -2780,12 +2783,14 @@ TEST_F(LibYUVConvertTest, P010ToAR30MatrixFilterOverflow) {
int width = 1024 + 15; // 1039
int height = 4;
int dst_stride_argb = width * 4;
int src_stride_uv = (width + 1) & ~1;
int src_uv_height = (height + 1) / 2;
align_buffer_page_end_16(src_y, width * height);
align_buffer_page_end_16(src_uv, width * ((height + 1) / 2));
align_buffer_page_end_16(src_uv, src_stride_uv * src_uv_height);
align_buffer_page_end(dst_argb, dst_stride_argb * height);
memset(src_y, 0x02, width * height * sizeof(uint16_t));
memset(src_uv, 0x02, width * ((height + 1) / 2) * sizeof(uint16_t));
EXPECT_EQ(0, P010ToAR30MatrixFilter(src_y, width, src_uv, width, dst_argb,
memset(src_uv, 0x02, src_stride_uv * src_uv_height * sizeof(uint16_t));
EXPECT_EQ(0, P010ToAR30MatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb,
dst_stride_argb, &kYuvI601Constants,
width, height, kFilterBilinear));
free_aligned_buffer_page_end_16(src_y);
@ -2799,12 +2804,13 @@ TEST_F(LibYUVConvertTest, P210ToAR30MatrixFilterOverflow) {
int width = 1024 + 15; // 1039
int height = 4;
int dst_stride_ar30 = width * 4;
int src_stride_uv = (width + 1) & ~1;
align_buffer_page_end_16(src_y, width * height);
align_buffer_page_end_16(src_uv, width * height);
align_buffer_page_end_16(src_uv, src_stride_uv * height);
align_buffer_page_end(dst_ar30, dst_stride_ar30 * height);
memset(src_y, 0x02, width * height * sizeof(uint16_t));
memset(src_uv, 0x02, width * height * sizeof(uint16_t));
EXPECT_EQ(0, P210ToAR30MatrixFilter(src_y, width, src_uv, width, dst_ar30,
memset(src_uv, 0x02, src_stride_uv * height * sizeof(uint16_t));
EXPECT_EQ(0, P210ToAR30MatrixFilter(src_y, width, src_uv, src_stride_uv, dst_ar30,
dst_stride_ar30, &kYuvI601Constants,
width, height, kFilterBilinear));
free_aligned_buffer_page_end_16(src_y);
@ -2874,6 +2880,14 @@ TEST_F(LibYUVConvertTest, TestI400LargeSize) {
#if defined(__aarch64__)
// Infer malloc can accept a large size for cpu with dot product (a76/a55)
int has_large_malloc = TestCpuFlag(kCpuHasNeonDotProd);
#elif defined(__SANITIZE_MEMORY__)
int has_large_malloc = 0;
#elif defined(__has_feature)
#if __has_feature(memory_sanitizer)
int has_large_malloc = 0;
#else
int has_large_malloc = 1;
#endif
#else
int has_large_malloc = 1;
#endif