From 407f21fe2985bc4a7a0c746e4e411bfe3a16af46 Mon Sep 17 00:00:00 2001 From: Wan-Teh Chang Date: Mon, 27 Jul 2026 12:37:24 -0700 Subject: [PATCH] Harden I21xToI420 against oversize heights I21xToI420 (reached via I210ToI420 / I212ToI420): fix integer overflows in ScalePlaneVertical_16To8(). Make the same changes to the related ScalePlaneVertical() and ScalePlaneVertical_16() functions. Add a unit test covering the I21x vertical-scale path. Co-authored-by: Timothy Nikkel Test: libyuv_unittest --gtest_filter=*.I21xToI420NoHeightOverflow Bug: chromium:531172025 Change-Id: Ieb8baad2576f4ef5ec71d771a6738a0e5cc7d21c --- include/libyuv/scale_row.h | 74 ++++++++++---------- source/convert.cc | 19 +++-- source/scale.cc | 12 ++-- source/scale_argb.cc | 6 +- source/scale_common.cc | 138 +++++++++++++++++++++---------------- source/scale_uv.cc | 6 +- unit_test/convert_test.cc | 54 +++++++++++++++ 7 files changed, 196 insertions(+), 113 deletions(-) diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 16f626439..63da57d32 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -197,45 +197,45 @@ extern "C" { #endif // Scale ARGB vertically with bilinear interpolation. -void ScalePlaneVertical(int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint8_t* src_argb, - uint8_t* dst_argb, - int x, - int y, - int dy, - int bpp, - enum FilterMode filtering); +int ScalePlaneVertical(int src_height, + int dst_width, + int dst_height, + int src_stride, + int dst_stride, + const uint8_t* src_argb, + uint8_t* dst_argb, + int x, + int y, + int dy, + int bpp, + enum FilterMode filtering); -void ScalePlaneVertical_16(int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint16_t* src_argb, - uint16_t* dst_argb, - int x, - int y, - int dy, - int wpp, - enum FilterMode filtering); +int ScalePlaneVertical_16(int src_height, + int dst_width, + int dst_height, + int src_stride, + int dst_stride, + const uint16_t* src_argb, + uint16_t* dst_argb, + int x, + int y, + int dy, + int wpp, + enum FilterMode filtering); -void ScalePlaneVertical_16To8(int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint16_t* src_argb, - uint8_t* dst_argb, - int x, - int y, - int dy, - int wpp, - int scale, - enum FilterMode filtering); +int ScalePlaneVertical_16To8(int src_height, + int dst_width, + int dst_height, + int src_stride, + int dst_stride, + const uint16_t* src_argb, + uint8_t* dst_argb, + int x, + int y, + int dy, + int wpp, + int scale, + enum FilterMode filtering); void ScalePlaneDown2_16To8(int src_width, int src_height, diff --git a/source/convert.cc b/source/convert.cc index f341c7b85..a492f08fa 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -295,15 +295,22 @@ static int I21xToI420(const uint16_t* src_y, const int uv_width = SUBSAMPLE(width, 1, 1); const int uv_height = SUBSAMPLE(height, 1, 1); const int dy = FixedDiv(height, uv_height); + int r; Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width, height); - ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_u, - dst_stride_u, src_u, dst_u, 0, 32768, dy, - /*bpp=*/1, scale, kFilterBilinear); - ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_v, - dst_stride_v, src_v, dst_v, 0, 32768, dy, - /*bpp=*/1, scale, kFilterBilinear); + r = ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_u, + dst_stride_u, src_u, dst_u, 0, 32768, dy, + /*bpp=*/1, scale, kFilterBilinear); + if (r != 0) { + return r; + } + r = ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_v, + dst_stride_v, src_v, dst_v, 0, 32768, dy, + /*bpp=*/1, scale, kFilterBilinear); + if (r != 0) { + return r; + } } return 0; } diff --git a/source/scale.cc b/source/scale.cc index 4b7b2d3bc..b06a8abb9 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -1985,9 +1985,9 @@ int ScalePlane(const uint8_t* src, dy = FixedDiv1(src_height, dst_height); } // Arbitrary scale vertically, but unscaled horizontally. - ScalePlaneVertical(src_height, dst_width, dst_height, src_stride, - dst_stride, src, dst, 0, y, dy, /*bpp=*/1, filtering); - return 0; + return ScalePlaneVertical(src_height, dst_width, dst_height, src_stride, + dst_stride, src, dst, 0, y, dy, /*bpp=*/1, + filtering); } if (dst_width <= Abs(src_width) && dst_height <= src_height) { // Scale down. @@ -2096,9 +2096,9 @@ int ScalePlane_16(const uint16_t* src, dy = FixedDiv1(src_height, dst_height); } // Arbitrary scale vertically, but unscaled horizontally. - ScalePlaneVertical_16(src_height, dst_width, dst_height, src_stride, - dst_stride, src, dst, 0, y, dy, /*bpp=*/1, filtering); - return 0; + return ScalePlaneVertical_16(src_height, dst_width, dst_height, src_stride, + dst_stride, src, dst, 0, y, dy, /*bpp=*/1, + filtering); } if (dst_width <= Abs(src_width) && dst_height <= src_height) { // Scale down. diff --git a/source/scale_argb.cc b/source/scale_argb.cc index 8758d79c2..50bb6efbb 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -746,9 +746,9 @@ static int ScaleARGB(const uint8_t* src, } if (dx == 0x10000 && (x & 0xffff) == 0) { // Arbitrary scale vertically, but unscaled horizontally. - ScalePlaneVertical(src_height, clip_width, clip_height, src_stride, - dst_stride, src, dst, x, y, dy, /*bpp=*/4, filtering); - return 0; + return ScalePlaneVertical(src_height, clip_width, clip_height, src_stride, + dst_stride, src, dst, x, y, dy, /*bpp=*/4, + filtering); } if (filtering && dy < 65536) { return ScaleARGBBilinearUp(src_width, src_height, clip_width, clip_height, diff --git a/source/scale_common.cc b/source/scale_common.cc index e2447119b..13a261ec5 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -11,6 +11,7 @@ #include "libyuv/scale.h" #include +#include #include #include "libyuv/cpu_id.h" @@ -1612,24 +1613,29 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv, #undef BLENDER // Scale plane vertically with bilinear interpolation. -void ScalePlaneVertical(int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint8_t* src_argb, - uint8_t* dst_argb, - int x, - int y, - int dy, - int bpp, // bytes per pixel. 4 for ARGB. - enum FilterMode filtering) { +int ScalePlaneVertical(int src_height, + int dst_width, + int dst_height, + int src_stride, + int dst_stride, + const uint8_t* src_argb, + uint8_t* dst_argb, + int x, + int y, + int dy, + int bpp, // bytes per pixel. 4 for ARGB. + enum FilterMode filtering) { // TODO(fbarchard): Allow higher bpp. - int dst_width_bytes = dst_width * bpp; + int64_t dst_width_bytes64 = (int64_t)dst_width * bpp; + if (dst_width_bytes64 > INT_MAX) { + return -1; + } + int dst_width_bytes = (int)dst_width_bytes64; void (*InterpolateRow)(uint8_t* dst_argb, const uint8_t* src_argb, ptrdiff_t src_stride, int dst_width, int source_y_fraction) = InterpolateRow_C; - const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0; + const int64_t max_y = + (src_height > 1) ? (((int64_t)src_height - 1) << 16) - 1 : 0; int j; assert(bpp >= 1 && bpp <= 4); assert(src_height != 0); @@ -1671,39 +1677,46 @@ void ScalePlaneVertical(int src_height, } #endif + int64_t y64 = y; for (j = 0; j < dst_height; ++j) { int yi; int yf; - if (y > max_y) { - y = max_y; + if (y64 > max_y) { + y64 = max_y; } - yi = y >> 16; - yf = filtering ? ((y >> 8) & 255) : 0; + yi = (int)(y64 >> 16); + yf = filtering ? (int)((y64 >> 8) & 255) : 0; InterpolateRow(dst_argb, src_argb + yi * (ptrdiff_t)src_stride, src_stride, dst_width_bytes, yf); dst_argb += dst_stride; - y += dy; + y64 += dy; } + return 0; } -void ScalePlaneVertical_16(int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint16_t* src_argb, - uint16_t* dst_argb, - int x, - int y, - int dy, - int wpp, /* words per pixel. normally 1 */ - enum FilterMode filtering) { +int ScalePlaneVertical_16(int src_height, + int dst_width, + int dst_height, + int src_stride, + int dst_stride, + const uint16_t* src_argb, + uint16_t* dst_argb, + int x, + int y, + int dy, + int wpp, /* words per pixel. normally 1 */ + enum FilterMode filtering) { // TODO(fbarchard): Allow higher wpp. - int dst_width_words = dst_width * wpp; + int64_t dst_width_words64 = (int64_t)dst_width * wpp; + if (dst_width_words64 > INT_MAX) { + return -1; + } + int dst_width_words = (int)dst_width_words64; void (*InterpolateRow)(uint16_t* dst_argb, const uint16_t* src_argb, ptrdiff_t src_stride, int dst_width, int source_y_fraction) = InterpolateRow_16_C; - const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0; + const int64_t max_y = + (src_height > 1) ? (((int64_t)src_height - 1) << 16) - 1 : 0; int j; assert(wpp >= 1 && wpp <= 2); assert(src_height != 0); @@ -1739,19 +1752,21 @@ void ScalePlaneVertical_16(int src_height, InterpolateRow = InterpolateRow_16_SME; } #endif + int64_t y64 = y; for (j = 0; j < dst_height; ++j) { int yi; int yf; - if (y > max_y) { - y = max_y; + if (y64 > max_y) { + y64 = max_y; } - yi = y >> 16; - yf = filtering ? ((y >> 8) & 255) : 0; + yi = (int)(y64 >> 16); + yf = filtering ? (int)((y64 >> 8) & 255) : 0; InterpolateRow(dst_argb, src_argb + yi * (ptrdiff_t)src_stride, src_stride, dst_width_words, yf); dst_argb += dst_stride; - y += dy; + y64 += dy; } + return 0; } // Use scale to convert lsb formats to msb, depending how many bits there are: @@ -1760,26 +1775,31 @@ void ScalePlaneVertical_16(int src_height, // 4096 = 12 bits // 256 = 16 bits // TODO(fbarchard): change scale to bits -void ScalePlaneVertical_16To8(int src_height, - int dst_width, - int dst_height, - int src_stride, - int dst_stride, - const uint16_t* src_argb, - uint8_t* dst_argb, - int x, - int y, - int dy, - int wpp, /* words per pixel. normally 1 */ - int scale, - enum FilterMode filtering) { +int ScalePlaneVertical_16To8(int src_height, + int dst_width, + int dst_height, + int src_stride, + int dst_stride, + const uint16_t* src_argb, + uint8_t* dst_argb, + int x, + int y, + int dy, + int wpp, /* words per pixel. normally 1 */ + int scale, + enum FilterMode filtering) { // TODO(fbarchard): Allow higher wpp. - int dst_width_words = dst_width * wpp; + int64_t dst_width_words64 = (int64_t)dst_width * wpp; + if (dst_width_words64 > INT_MAX) { + return -1; + } + int dst_width_words = (int)dst_width_words64; // TODO(https://crbug.com/libyuv/931): Add NEON 32 bit and AVX2 versions. void (*InterpolateRow_16To8)(uint8_t* dst_argb, const uint16_t* src_argb, ptrdiff_t src_stride, int scale, int dst_width, int source_y_fraction) = InterpolateRow_16To8_C; - const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0; + const int64_t max_y = + (src_height > 1) ? (((int64_t)src_height - 1) << 16) - 1 : 0; int j; assert(wpp >= 1 && wpp <= 2); assert(src_height != 0); @@ -1808,19 +1828,21 @@ void ScalePlaneVertical_16To8(int src_height, } } #endif + int64_t y64 = y; for (j = 0; j < dst_height; ++j) { int yi; int yf; - if (y > max_y) { - y = max_y; + if (y64 > max_y) { + y64 = max_y; } - yi = y >> 16; - yf = filtering ? ((y >> 8) & 255) : 0; + yi = (int)(y64 >> 16); + yf = filtering ? (int)((y64 >> 8) & 255) : 0; InterpolateRow_16To8(dst_argb, src_argb + yi * (ptrdiff_t)src_stride, src_stride, scale, dst_width_words, yf); dst_argb += dst_stride; - y += dy; + y64 += dy; } + return 0; } // Simplify the filtering based on scale factors. diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 43a464732..5e5be5016 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -1015,9 +1015,9 @@ static int ScaleUV(const uint8_t* src, // HAS_SCALEPLANEVERTICAL if (dx == 0x10000 && (x & 0xffff) == 0) { // Arbitrary scale vertically, but unscaled horizontally. - ScalePlaneVertical(src_height, clip_width, clip_height, src_stride, - dst_stride, src, dst, x, y, dy, /*bpp=*/2, filtering); - return 0; + return ScalePlaneVertical(src_height, clip_width, clip_height, src_stride, + dst_stride, src, dst, x, y, dy, /*bpp=*/2, + filtering); } if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) { ScaleUVLinearUp2(src_width, src_height, clip_width, clip_height, src_stride, diff --git a/unit_test/convert_test.cc b/unit_test/convert_test.cc index 12398fc08..a6a30f4dc 100644 --- a/unit_test/convert_test.cc +++ b/unit_test/convert_test.cc @@ -2593,6 +2593,60 @@ TEST_F(LibYUVConvertTest, ABGRToI420_Check) { benchmark_cpu_info_); } +// I210ToI420/I212ToI420 funnel through I21xToI420, which scales chroma +// vertically with ScalePlaneVertical_16To8. Verify oversize heights do not +// overflow a signed int. +TEST_F(LibYUVConvertTest, I21xToI420NoHeightOverflow) { + int width = 4; + int height = 40000; + int half_width = (width + 1) / 2; + int half_height = (height + 1) / 2; + // I21x: 16-bit 4:2:2 + align_buffer_page_end_16(src_y, width * height); + align_buffer_page_end_16(src_u, half_width * height); + align_buffer_page_end_16(src_v, half_width * height); + // I420: 8-bit 4:2:0 + align_buffer_page_end(dst_y, width * height); + align_buffer_page_end(dst_u, half_width * half_height); + align_buffer_page_end(dst_v, half_width * half_height); + + memset(src_y, 0, width * height * sizeof(uint16_t)); + memset(src_u, 0, half_width * height * sizeof(uint16_t)); + memset(src_v, 0, half_width * height * sizeof(uint16_t)); + + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32767)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32768)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32769)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32767)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32768)); + EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32769)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32767)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32768)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, 32769)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32767)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32768)); + EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2, + dst_v, 2, 4, -32769)); + + free_aligned_buffer_page_end_16(src_y); + free_aligned_buffer_page_end_16(src_u); + free_aligned_buffer_page_end_16(src_v); + free_aligned_buffer_page_end(dst_y); + free_aligned_buffer_page_end(dst_u); + free_aligned_buffer_page_end(dst_v); +} + #endif // !defined(LEAN_TESTS) } // namespace libyuv