From e5b20f969fd4cdcd071ffc44ead660463e860c41 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Wed, 14 Oct 2026 23:01:28 -0700 Subject: [PATCH] Avoid signed int overflow in scale functions using uint32_t Use uint32_t for the 16.16 fixed-point row coordinate (uy) and max_y in the vertical loops of ScalePlane*, ScaleUV*, and the Up2 linear helpers. With src_height up to 32768, (src_height << 16) and y += dy overflow signed int (e.g. for dst_height == 2 the post-read step reaches 5/4 * (src_height << 16), which exceeds INT_MAX for any src_height >= 26215, and for kFilterNone the final increment after the last row reaches (src_height << 16) + dy / 2). uint32_t cannot overflow for these ranges (uy <= 0xFFFF0000) and stays in a single 32-bit register on 32-bit targets (x86, Arm, WASM), unlike int64_t. Undo the interim +/-32767 height cap from https://crrev.com/c/8524280 so ScalePlane, ScalePlane_16, UVScale, and UVScale_16 accept |src_height| <= 32768 again. ScaleUVBilinearUp2 and ScaleUVBilinearUp2_16 take ptrdiff_t strides so 2 * dst_stride does not overflow int for images over 2 GB. ARGBScale, ARGBScaleClip, and UVScale_16 reject |src_width| > 32768 and |src_height| > 32768, and ScalePlane, ScalePlane_16, and ScaleSlope avoid FixedDiv(32768, 1) overflow when dst_width == 1 or dst_height == 1. Bug: chromium:531172025, chromium:546841026, libyuv:573815252 Test: libyuv_unittest --gtest_filter=*ScalePlaneOversizeHeight*:*ScalePlaneBoxMaxSrcHeight*:*ScaleLargeImageOver2GB*:*ARGBScale* Change-Id: Ica2e05d7dfe94f81c6631c587f1ff46c2ad44735 --- source/scale.cc | 191 ++++++++++++++++++---------------- source/scale_argb.cc | 9 +- source/scale_common.cc | 6 +- source/scale_uv.cc | 93 +++++++++-------- unit_test/scale_argb_test.cc | 42 ++++++++ unit_test/scale_plane_test.cc | 65 ++++++++++++ 6 files changed, 265 insertions(+), 141 deletions(-) diff --git a/source/scale.cc b/source/scale.cc index 98d417aa4..f10ead6cb 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -944,7 +944,7 @@ static int ScalePlaneBox(int src_width, int y = 0; int dx = 0; int dy = 0; - const int max_y = (src_height << 16); + const uint32_t max_y = (uint32_t)src_height << 16; ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterBox, &x, &y, &dx, &dy); src_width = Abs(src_width); @@ -1009,15 +1009,16 @@ static int ScalePlaneBox(int src_width, } #endif + uint32_t uy = y; for (j = 0; j < dst_height; ++j) { int boxheight; - int iy = y >> 16; + int iy = uy >> 16; const uint8_t* src = src_ptr + iy * src_stride; - y += dy; - if (y > max_y) { - y = max_y; + uy += dy; + if (uy > max_y) { + uy = max_y; } - boxheight = MIN1((y >> 16) - iy); + boxheight = MIN1((int)(uy >> 16) - iy); memset(row16, 0, src_width * 2); for (k = 0; k < boxheight; ++k) { ScaleAddRow(src, (uint16_t*)(row16), src_width); @@ -1045,7 +1046,7 @@ static int ScalePlaneBox_16(int src_width, int y = 0; int dx = 0; int dy = 0; - const int max_y = (src_height << 16); + const uint32_t max_y = (uint32_t)src_height << 16; ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterBox, &x, &y, &dx, &dy); src_width = Abs(src_width); @@ -1066,15 +1067,16 @@ static int ScalePlaneBox_16(int src_width, } #endif + uint32_t uy = y; for (j = 0; j < dst_height; ++j) { int boxheight; - int iy = y >> 16; + int iy = uy >> 16; const uint16_t* src = src_ptr + iy * src_stride; - y += dy; - if (y > max_y) { - y = max_y; + uy += dy; + if (uy > max_y) { + uy = max_y; } - boxheight = MIN1((y >> 16) - iy); + boxheight = MIN1((int)(uy >> 16) - iy); memset(row32, 0, src_width * 4); for (k = 0; k < boxheight; ++k) { ScaleAddRow(src, (uint32_t*)(row32), src_width); @@ -1109,7 +1111,7 @@ static int ScalePlaneBilinearDown(int src_width, if (!row) return 1; - const int max_y = (src_height - 1) << 16; + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; int j; void (*ScaleFilterCols)(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, int x, int dx) = @@ -1203,25 +1205,23 @@ static int ScalePlaneBilinearDown(int src_width, } } #endif - if (y > max_y) { - y = max_y; - } + uint32_t uy = y; for (j = 0; j < dst_height; ++j) { - int yi = y >> 16; + if (uy > max_y) { + uy = max_y; + } + int yi = uy >> 16; const uint8_t* src = src_ptr + yi * src_stride; if (filtering == kFilterLinear) { ScaleFilterCols(dst_ptr, src, dst_width, x, dx); } else { - int yf = (y >> 8) & 255; + int yf = (uy >> 8) & 255; InterpolateRow(row, src, src_stride, src_width, yf); ScaleFilterCols(dst_ptr, row, dst_width, x, dx); } dst_ptr += dst_stride; - y += dy; - if (y > max_y) { - y = max_y; - } + uy += dy; } free_aligned_buffer_64(row); return 0; @@ -1247,7 +1247,7 @@ static int ScalePlaneBilinearDown_16(int src_width, if (!row) return 1; - const int max_y = (src_height - 1) << 16; + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; int j; void (*ScaleFilterCols)(uint16_t* dst_ptr, const uint16_t* src_ptr, int dst_width, int x, int dx) = @@ -1307,25 +1307,23 @@ static int ScalePlaneBilinearDown_16(int src_width, ScaleFilterCols = ScaleFilterCols_16_SSSE3; } #endif - if (y > max_y) { - y = max_y; - } + uint32_t uy = y; for (j = 0; j < dst_height; ++j) { - int yi = y >> 16; + if (uy > max_y) { + uy = max_y; + } + int yi = uy >> 16; const uint16_t* src = src_ptr + yi * src_stride; if (filtering == kFilterLinear) { ScaleFilterCols(dst_ptr, src, dst_width, x, dx); } else { - int yf = (y >> 8) & 255; + int yf = (uy >> 8) & 255; InterpolateRow((uint16_t*)row, src, src_stride, src_width, yf); ScaleFilterCols(dst_ptr, (uint16_t*)row, dst_width, x, dx); } dst_ptr += dst_stride; - y += dy; - if (y > max_y) { - y = max_y; - } + uy += dy; } free_aligned_buffer_64(row); return 0; @@ -1352,7 +1350,7 @@ static int ScalePlaneBilinearUp(int src_width, int y = 0; int dx = 0; int dy = 0; - const int max_y = (src_height - 1) << 16; + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, int dst_width, int source_y_fraction) = InterpolateRow_C; @@ -1450,11 +1448,12 @@ static int ScalePlaneBilinearUp(int src_width, #endif } - if (y > max_y) { - y = max_y; + uint32_t uy = y; + if (uy > max_y) { + uy = max_y; } { - int yi = y >> 16; + int yi = uy >> 16; const uint8_t* src = src_ptr + yi * src_stride; // Allocate 2 row buffers. @@ -1483,31 +1482,31 @@ static int ScalePlaneBilinearUp(int src_width, // 1. Scale the next source row into the older buffer (rowptr). // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;) // so rowptr points to yi and (rowptr + rowstride) points to yi + 1. - // 3. Advance src by 1 row if row yi + 2 exists ((y + 65536) < max_y), + // 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y), // otherwise clamp src at (src_height - 1) to avoid reading out of // bounds. for (j = 0; j < dst_height; ++j) { - if (y > max_y) { - y = max_y; + if (uy > max_y) { + uy = max_y; } - yi = y >> 16; + yi = uy >> 16; if (yi != lasty) { ScaleFilterCols(rowptr, src, dst_width, x, dx); rowptr += rowstride; rowstride = -rowstride; lasty = yi; - if ((y + 65536) < max_y) { + if ((uy + 65536) < max_y) { src += src_stride; } } if (filtering == kFilterLinear) { InterpolateRow(dst_ptr, rowptr, 0, dst_width, 0); } else { - int yf = (y >> 8) & 255; + int yf = (uy >> 8) & 255; InterpolateRow(dst_ptr, rowptr, rowstride, dst_width, yf); } dst_ptr += dst_stride; - y += dy; + uy += dy; } free_aligned_buffer_64(row); } @@ -1530,8 +1529,6 @@ static void ScalePlaneUp2_Linear(int src_width, void (*ScaleRowUp)(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width) = ScaleRowUp2_Linear_Any_C; int i; - int y; - int dy; (void)src_width; // This function can only scale up by 2 times horizontally. @@ -1570,12 +1567,13 @@ static void ScalePlaneUp2_Linear(int src_width, ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr, dst_width); } else { - dy = FixedDiv(src_height - 1, dst_height - 1); - y = (1 << 15) - 1; + uint32_t uy = (1 << 15) - 1; + int dy = FixedDiv(src_height - 1, dst_height - 1); for (i = 0; i < dst_height; ++i) { - ScaleRowUp(src_ptr + (y >> 16) * src_stride, dst_ptr, dst_width); + int yi = uy >> 16; + ScaleRowUp(src_ptr + yi * src_stride, dst_ptr, dst_width); dst_ptr += dst_stride; - y += dy; + uy += dy; } } } @@ -1661,8 +1659,6 @@ static void ScalePlaneUp2_12_Linear(int src_width, void (*ScaleRowUp)(const uint16_t* src_ptr, uint16_t* dst_ptr, int dst_width) = ScaleRowUp2_Linear_16_Any_C; int i; - int y; - int dy; (void)src_width; // This function can only scale up by 2 times horizontally. @@ -1690,12 +1686,13 @@ static void ScalePlaneUp2_12_Linear(int src_width, ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr, dst_width); } else { - dy = FixedDiv(src_height - 1, dst_height - 1); - y = (1 << 15) - 1; + uint32_t uy = (1 << 15) - 1; + int dy = FixedDiv(src_height - 1, dst_height - 1); for (i = 0; i < dst_height; ++i) { - ScaleRowUp(src_ptr + (y >> 16) * src_stride, dst_ptr, dst_width); + int yi = uy >> 16; + ScaleRowUp(src_ptr + yi * src_stride, dst_ptr, dst_width); dst_ptr += dst_stride; - y += dy; + uy += dy; } } } @@ -1764,8 +1761,6 @@ static void ScalePlaneUp2_16_Linear(int src_width, void (*ScaleRowUp)(const uint16_t* src_ptr, uint16_t* dst_ptr, int dst_width) = ScaleRowUp2_Linear_16_Any_C; int i; - int y; - int dy; (void)src_width; // This function can only scale up by 2 times horizontally. @@ -1793,12 +1788,13 @@ static void ScalePlaneUp2_16_Linear(int src_width, ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr, dst_width); } else { - dy = FixedDiv(src_height - 1, dst_height - 1); - y = (1 << 15) - 1; + uint32_t uy = (1 << 15) - 1; + int dy = FixedDiv(src_height - 1, dst_height - 1); for (i = 0; i < dst_height; ++i) { - ScaleRowUp(src_ptr + (y >> 16) * src_stride, dst_ptr, dst_width); + int yi = uy >> 16; + ScaleRowUp(src_ptr + yi * src_stride, dst_ptr, dst_width); dst_ptr += dst_stride; - y += dy; + uy += dy; } } } @@ -1871,7 +1867,7 @@ static int ScalePlaneBilinearUp_16(int src_width, int y = 0; int dx = 0; int dy = 0; - const int max_y = (src_height - 1) << 16; + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; void (*InterpolateRow)(uint16_t* dst_ptr, const uint16_t* src_ptr, ptrdiff_t src_stride, int dst_width, int source_y_fraction) = InterpolateRow_16_C; @@ -1943,11 +1939,12 @@ static int ScalePlaneBilinearUp_16(int src_width, #endif } - if (y > max_y) { - y = max_y; + uint32_t uy = y; + if (uy > max_y) { + uy = max_y; } { - int yi = y >> 16; + int yi = uy >> 16; const uint16_t* src = src_ptr + yi * src_stride; // Allocate 2 row buffers. @@ -1975,31 +1972,31 @@ static int ScalePlaneBilinearUp_16(int src_width, // 1. Scale the next source row into the older buffer (rowptr). // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;) // so rowptr points to yi and (rowptr + rowstride) points to yi + 1. - // 3. Advance src by 1 row if row yi + 2 exists ((y + 65536) < max_y), + // 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y), // otherwise clamp src at (src_height - 1) to avoid reading out of // bounds. for (j = 0; j < dst_height; ++j) { - if (y > max_y) { - y = max_y; + if (uy > max_y) { + uy = max_y; } - yi = y >> 16; + yi = uy >> 16; if (yi != lasty) { ScaleFilterCols(rowptr, src, dst_width, x, dx); rowptr += rowstride; rowstride = -rowstride; lasty = yi; - if ((y + 65536) < max_y) { + if ((uy + 65536) < max_y) { src += src_stride; } } if (filtering == kFilterLinear) { InterpolateRow(dst_ptr, rowptr, 0, dst_width, 0); } else { - int yf = (y >> 8) & 255; + int yf = (uy >> 8) & 255; InterpolateRow(dst_ptr, rowptr, rowstride, dst_width, yf); } dst_ptr += dst_stride; - y += dy; + uy += dy; } free_aligned_buffer_64(row); } @@ -2040,10 +2037,12 @@ static void ScalePlaneSimple(int src_width, #endif } + uint32_t uy = y; for (i = 0; i < dst_height; ++i) { - ScaleCols(dst_ptr, src_ptr + (y >> 16) * src_stride, dst_width, x, dx); + int yi = uy >> 16; + ScaleCols(dst_ptr, src_ptr + yi * src_stride, dst_width, x, dx); dst_ptr += dst_stride; - y += dy; + uy += dy; } } @@ -2076,10 +2075,12 @@ static void ScalePlaneSimple_16(int src_width, #endif } + uint32_t uy = y; for (i = 0; i < dst_height; ++i) { - ScaleCols(dst_ptr, src_ptr + (y >> 16) * src_stride, dst_width, x, dx); + int yi = uy >> 16; + ScaleCols(dst_ptr, src_ptr + yi * src_stride, dst_width, x, dx); dst_ptr += dst_stride; - y += dy; + uy += dy; } } @@ -2095,12 +2096,11 @@ int ScalePlane(const uint8_t* src, int dst_width, int dst_height, enum FilterMode filtering) { - // Reject width larger than 32768 and height larger than 32767 or smaller than - // -32767. This prevents FixedDiv signed integer overflows that can lead to - // division by zero/overflow crashes (SIGFPE on x86) or incorrect step - // calculations. + // Reject dimensions larger than 32768 (or smaller than -32768 for height). + // This prevents FixedDiv signed integer overflows that can lead to division + // by zero/overflow crashes (SIGFPE on x86) or incorrect step calculations. if (!src || src_width <= 0 || src_height == 0 || src_width > 32768 || - src_height < -32767 || src_height > 32767 || !dst || dst_width <= 0 || + src_height < -32768 || src_height > 32768 || !dst || dst_width <= 0 || dst_height <= 0) { return -1; } @@ -2124,8 +2124,14 @@ int ScalePlane(const uint8_t* src, if (dst_width == src_width && filtering != kFilterBox) { int dy = 0; int y = 0; - // When scaling down, use the center 2 rows to filter. - if (dst_height <= src_height) { + // Check for 1 pixel and avoid FixedDiv overflow. + // TODO(b/573815252): Fix center row calculation when dst_height == 1 and + // src_height == 32768. + if (dst_height == 1 && src_height >= 32768) { + dy = FixedDiv(src_height, src_height); + y = CENTERSTART(dy, -32768); + } else if (dst_height <= src_height) { + // When scaling down, use the center 2 rows to filter. dy = FixedDiv(src_height, dst_height); y = CENTERSTART(dy, -32768); // Subtract 0.5 (32768) to center filter. } else if (src_height > 1 && dst_height > 1) { @@ -2211,12 +2217,11 @@ int ScalePlane_16(const uint16_t* src, int dst_width, int dst_height, enum FilterMode filtering) { - // Reject width larger than 32768 and height larger than 32767 or smaller than - // -32767. This prevents FixedDiv signed integer overflows that can lead to - // division by zero/overflow crashes (SIGFPE on x86) or incorrect step - // calculations. + // Reject dimensions larger than 32768 (or smaller than -32768 for height). + // This prevents FixedDiv signed integer overflows that can lead to division + // by zero/overflow crashes (SIGFPE on x86) or incorrect step calculations. if (!src || src_width <= 0 || src_height == 0 || src_width > 32768 || - src_height < -32767 || src_height > 32767 || !dst || dst_width <= 0 || + src_height < -32768 || src_height > 32768 || !dst || dst_width <= 0 || dst_height <= 0) { return -1; } @@ -2240,8 +2245,14 @@ int ScalePlane_16(const uint16_t* src, if (dst_width == src_width && filtering != kFilterBox) { int dy = 0; int y = 0; - // When scaling down, use the center 2 rows to filter. - if (dst_height <= src_height) { + // Check for 1 pixel and avoid FixedDiv overflow. + // TODO(b/573815252): Fix center row calculation when dst_height == 1 and + // src_height == 32768. + if (dst_height == 1 && src_height >= 32768) { + dy = FixedDiv(src_height, src_height); + y = CENTERSTART(dy, -32768); + } else if (dst_height <= src_height) { + // When scaling down, use the center 2 rows to filter. dy = FixedDiv(src_height, dst_height); y = CENTERSTART(dy, -32768); // Subtract 0.5 (32768) to center filter. } else if (src_height > 1 && dst_height > 1) { diff --git a/source/scale_argb.cc b/source/scale_argb.cc index a1650dc72..714124746 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -906,7 +906,8 @@ int ARGBScaleClip(const uint8_t* src_argb, int clip_width, int clip_height, enum FilterMode filtering) { - if (!src_argb || src_width == 0 || src_height == 0 || src_height == INT_MIN || + if (!src_argb || src_width == 0 || src_width > 32768 || src_width < -32768 || + src_height == 0 || src_height > 32768 || src_height < -32768 || !dst_argb || dst_width <= 0 || dst_height <= 0 || clip_x < 0 || clip_y < 0 || clip_width > 32768 || clip_height > 32768 || (clip_x + clip_width) > dst_width || @@ -929,9 +930,9 @@ int ARGBScale(const uint8_t* src_argb, int dst_width, int dst_height, enum FilterMode filtering) { - if (!src_argb || src_width == 0 || src_height == 0 || src_height == INT_MIN || - src_width > 32768 || src_height > 32768 || !dst_argb || dst_width <= 0 || - dst_height <= 0) { + if (!src_argb || src_width == 0 || src_width > 32768 || src_width < -32768 || + src_height == 0 || src_height > 32768 || src_height < -32768 || + !dst_argb || dst_width <= 0 || dst_height <= 0) { return -1; } return ScaleARGB(src_argb, src_stride_argb, src_width, src_height, dst_argb, diff --git a/source/scale_common.cc b/source/scale_common.cc index f6633116f..1dc311bd7 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1640,8 +1640,10 @@ void ScaleSlope(int src_width, assert(dst_width > 0); assert(dst_height > 0); // Check for 1 pixel and avoid FixedDiv overflow. - if (dst_width == 1 && src_width >= 32768) { - dst_width = src_width; + // TODO(b/573815252): Fix center coordinate calculation when dst == 1 and + // src == 32768. + if (dst_width == 1 && Abs(src_width) >= 32768) { + dst_width = Abs(src_width); } if (dst_height == 1 && src_height >= 32768) { dst_height = src_height; diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 90dc8b2ce..251bea8cd 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -320,7 +320,7 @@ static int ScaleUVBox(int src_width, int y, int dy) { int j, k; - const int max_y = (src_height << 16); + const uint32_t max_y = (uint32_t)src_height << 16; // Sized on src_width to keep ScaleAddRow aligned. const int row_samples = src_width * 2; { @@ -374,15 +374,16 @@ static int ScaleUVBox(int src_width, } #endif + uint32_t uy = y; for (j = 0; j < dst_height; ++j) { int boxheight; - int iy = y >> 16; + int iy = uy >> 16; const uint8_t* src = src_uv + iy * src_stride; - y += dy; - if (y > max_y) { - y = max_y; + uy += dy; + if (uy > max_y) { + uy = max_y; } - boxheight = MIN1((y >> 16) - iy); + boxheight = MIN1((int)(uy >> 16) - iy); memset(row16, 0, row_samples * 2); for (k = 0; k < boxheight; ++k) { ScaleAddRow(src, (uint16_t*)(row16), row_samples); @@ -581,27 +582,28 @@ static int ScaleUVBilinearDown(int src_width, // TODO(fbarchard): Consider not allocating row buffer for kFilterLinear. // Allocate a row of UV. { - const int max_y = (src_height - 1) << 16; + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; + uint32_t uy = y; align_buffer_64(row, clip_src_width * 2); if (!row) return 1; - if (y > max_y) { - y = max_y; + if (uy > max_y) { + uy = max_y; } for (j = 0; j < dst_height; ++j) { - int yi = y >> 16; + int yi = uy >> 16; const uint8_t* src = src_uv + yi * src_stride; if (filtering == kFilterLinear) { ScaleUVFilterCols(dst_uv, src, dst_width, x, dx); } else { - int yf = (y >> 8) & 255; + int yf = (uy >> 8) & 255; InterpolateRow(row, src, src_stride, clip_src_width, yf); ScaleUVFilterCols(dst_uv, row, dst_width, x, dx); } dst_uv += dst_stride; - y += dy; - if (y > max_y) { - y = max_y; + uy += dy; + if (uy > max_y) { + uy = max_y; } } free_aligned_buffer_64(row); @@ -638,7 +640,8 @@ static int ScaleUVBilinearUp(int src_width, void (*ScaleUVFilterCols)(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width, int x, int dx) = filtering ? ScaleUVFilterCols_C : ScaleUVCols_C; - const int max_y = (src_height - 1) << 16; + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; + uint32_t uy = y; #if defined(HAS_INTERPOLATEROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_Any_AVX2; @@ -729,12 +732,12 @@ static int ScaleUVBilinearUp(int src_width, #endif } - if (y > max_y) { - y = max_y; + if (uy > max_y) { + uy = max_y; } { - int yi = y >> 16; + int yi = uy >> 16; const uint8_t* src = src_uv + yi * src_stride; // Allocate 2 rows of UV. @@ -763,31 +766,31 @@ static int ScaleUVBilinearUp(int src_width, // 1. Scale the next source row into the older buffer (rowptr). // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;) // so rowptr points to yi and (rowptr + rowstride) points to yi + 1. - // 3. Advance src by 1 row if row yi + 2 exists ((y + 65536) < max_y), + // 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y), // otherwise clamp src at (src_height - 1) to avoid reading out of // bounds. for (j = 0; j < dst_height; ++j) { - if (y > max_y) { - y = max_y; + if (uy > max_y) { + uy = max_y; } - yi = y >> 16; + yi = uy >> 16; if (yi != lasty) { ScaleUVFilterCols(rowptr, src, dst_width, x, dx); rowptr += rowstride; rowstride = -rowstride; lasty = yi; - if ((y + 65536) < max_y) { + if ((uy + 65536) < max_y) { src += src_stride; } } if (filtering == kFilterLinear) { InterpolateRow(dst_uv, rowptr, 0, dst_width * 2, 0); } else { - int yf = (y >> 8) & 255; + int yf = (uy >> 8) & 255; InterpolateRow(dst_uv, rowptr, rowstride, dst_width * 2, yf); } dst_uv += dst_stride; - y += dy; + uy += dy; } free_aligned_buffer_64(row); } @@ -811,8 +814,6 @@ static void ScaleUVLinearUp2(int src_width, void (*ScaleRowUp)(const uint8_t* src_uv, uint8_t* dst_uv, int dst_width) = ScaleUVRowUp2_Linear_Any_C; int i; - int y; - int dy; // This function can only scale up by 2 times horizontally. (void)src_width; @@ -845,12 +846,13 @@ static void ScaleUVLinearUp2(int src_width, if (dst_height == 1) { ScaleRowUp(src_uv + ((src_height - 1) / 2) * src_stride, dst_uv, dst_width); } else { - dy = FixedDiv(src_height - 1, dst_height - 1); - y = (1 << 15) - 1; + uint32_t uy = (1 << 15) - 1; + int dy = FixedDiv(src_height - 1, dst_height - 1); for (i = 0; i < dst_height; ++i) { - ScaleRowUp(src_uv + (y >> 16) * src_stride, dst_uv, dst_width); + int yi = uy >> 16; + ScaleRowUp(src_uv + yi * src_stride, dst_uv, dst_width); dst_uv += dst_stride; - y += dy; + uy += dy; } } } @@ -863,8 +865,8 @@ static void ScaleUVBilinearUp2(int src_width, int src_height, int dst_width, int dst_height, - int src_stride, - int dst_stride, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, const uint8_t* src_ptr, uint8_t* dst_ptr) { void (*Scale2RowUp)(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -931,8 +933,6 @@ static void ScaleUVLinearUp2_16(int src_width, void (*ScaleRowUp)(const uint16_t* src_uv, uint16_t* dst_uv, int dst_width) = ScaleUVRowUp2_Linear_16_Any_C; int i; - int y; - int dy; // This function can only scale up by 2 times horizontally. (void)src_width; @@ -959,12 +959,13 @@ static void ScaleUVLinearUp2_16(int src_width, if (dst_height == 1) { ScaleRowUp(src_uv + ((src_height - 1) / 2) * src_stride, dst_uv, dst_width); } else { - dy = FixedDiv(src_height - 1, dst_height - 1); - y = (1 << 15) - 1; + uint32_t uy = (1 << 15) - 1; + int dy = FixedDiv(src_height - 1, dst_height - 1); for (i = 0; i < dst_height; ++i) { - ScaleRowUp(src_uv + (y >> 16) * src_stride, dst_uv, dst_width); + int yi = uy >> 16; + ScaleRowUp(src_uv + yi * src_stride, dst_uv, dst_width); dst_uv += dst_stride; - y += dy; + uy += dy; } } } @@ -977,8 +978,8 @@ static void ScaleUVBilinearUp2_16(int src_width, int src_height, int dst_width, int dst_height, - int src_stride, - int dst_stride, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, const uint16_t* src_ptr, uint16_t* dst_ptr) { void (*Scale2RowUp)(const uint16_t* src_ptr, ptrdiff_t src_stride, @@ -1067,10 +1068,12 @@ static void ScaleUVSimple(int src_width, #endif } + uint32_t uy = y; for (j = 0; j < dst_height; ++j) { - ScaleUVCols(dst_uv, src_uv + (y >> 16) * src_stride, dst_width, x, dx); + int yi = uy >> 16; + ScaleUVCols(dst_uv, src_uv + yi * src_stride, dst_width, x, dx); dst_uv += dst_stride; - y += dy; + uy += dy; } } @@ -1267,7 +1270,7 @@ int UVScale(const uint8_t* src_uv, int dst_height, enum FilterMode filtering) { if (!src_uv || src_width <= 0 || src_height == 0 || src_width > 32768 || - src_height < -32767 || src_height > 32767 || !dst_uv || dst_width <= 0 || + src_height < -32768 || src_height > 32768 || !dst_uv || dst_width <= 0 || dst_height <= 0) { return -1; } @@ -1291,7 +1294,7 @@ int UVScale_16(const uint16_t* src_uv, int dy = 0; if (!src_uv || src_width <= 0 || src_height == 0 || src_width > 32768 || - src_height < -32767 || src_height > 32767 || !dst_uv || dst_width <= 0 || + src_height < -32768 || src_height > 32768 || !dst_uv || dst_width <= 0 || dst_height <= 0) { return -1; } diff --git a/unit_test/scale_argb_test.cc b/unit_test/scale_argb_test.cc index 2fd4d33ea..532bcb509 100644 --- a/unit_test/scale_argb_test.cc +++ b/unit_test/scale_argb_test.cc @@ -636,4 +636,46 @@ TEST_F(LibYUVScaleTest, ARGBScale_LargeHeightOverflow) { free_aligned_buffer_page_end(orig_pixels); } +TEST_F(LibYUVScaleTest, ARGBScaleDimensionOverflow) { + uint8_t src[16] = {}; + uint8_t dst[4] = {}; + EXPECT_EQ(-1, ARGBScale(src, 4, 32769, 1, dst, 4, 1, 1, kFilterNone)); + EXPECT_EQ(-1, ARGBScale(src, 4, -32769, 1, dst, 4, 1, 1, kFilterNone)); + EXPECT_EQ(-1, ARGBScale(src, 4, 1, 32769, dst, 4, 1, 1, kFilterNone)); + EXPECT_EQ(-1, ARGBScale(src, 4, 1, -32769, dst, 4, 1, 1, kFilterNone)); + EXPECT_EQ(-1, ARGBScaleClip(src, 4, 32769, 1, dst, 4, 1, 1, 0, 0, 1, 1, + kFilterNone)); + EXPECT_EQ(-1, ARGBScaleClip(src, 4, -32769, 1, dst, 4, 1, 1, 0, 0, 1, 1, + kFilterNone)); + EXPECT_EQ(-1, ARGBScaleClip(src, 4, 1, 32769, dst, 4, 1, 1, 0, 0, 1, 1, + kFilterNone)); + EXPECT_EQ(-1, ARGBScaleClip(src, 4, 1, -32769, dst, 4, 1, 1, 0, 0, 1, 1, + kFilterNone)); +} + +#if defined(__LP64__) || defined(_WIN64) +TEST_F(LibYUVScaleTest, ARGBScaleLargeImageOver2GB) { + const int kSrcWidth = 4; + const int kSrcHeight = 32768; + const int kSrcStride = 131072; // 32768 * 131072 = 4 GB (> INT_MAX) + const size_t kSrcSize = (size_t)kSrcStride * kSrcHeight; + uint8_t* src = static_cast(calloc(1, kSrcSize)); + if (!src) { + GTEST_SKIP() << "could not allocate " << kSrcSize << " bytes"; + } + uint8_t dst[8 * 4 * 4] = {}; + + for (int dst_w : {4, 2}) { + for (FilterMode f : + {kFilterNone, kFilterLinear, kFilterBilinear, kFilterBox}) { + EXPECT_EQ(0, ARGBScale(src, kSrcStride, kSrcWidth, kSrcHeight, dst, + dst_w * 4, dst_w, 2, f)); + EXPECT_EQ(0, ARGBScale(src, kSrcStride, kSrcWidth, -kSrcHeight, dst, + dst_w * 4, dst_w, 2, f)); + } + } + free(src); +} +#endif + } // namespace libyuv diff --git a/unit_test/scale_plane_test.cc b/unit_test/scale_plane_test.cc index ab521ebcf..544a0f098 100644 --- a/unit_test/scale_plane_test.cc +++ b/unit_test/scale_plane_test.cc @@ -19,6 +19,7 @@ #include "../unit_test/unit_test.h" #include "libyuv/cpu_id.h" #include "libyuv/scale.h" +#include "libyuv/scale_uv.h" #ifdef ENABLE_ROW_TESTS #include "libyuv/scale_row.h" // For ScaleRowDown2Box_Odd_C @@ -805,4 +806,68 @@ TEST_F(LibYUVScaleTest, ScalePlane_12_InvalidInputs) { EXPECT_EQ(-1, ScalePlane_12(src, 4, 4, -32769, dst, 4, 4, 4, kFilterNone)); } +TEST_F(LibYUVScaleTest, ScalePlaneOversizeHeight) { + const int kSrcWidth = 4; + const int kSrcHeight = 32768; + uint8_t* src = new uint8_t[kSrcWidth * kSrcHeight]; + memset(src, 0, kSrcWidth * kSrcHeight); + uint8_t dst[4 * 2] = {}; + + for (int dst_w : {4, 2}) { + for (int dst_h : {1, 2}) { + for (FilterMode f : + {kFilterNone, kFilterLinear, kFilterBilinear, kFilterBox}) { + EXPECT_EQ(0, ScalePlane(src, kSrcWidth, kSrcWidth, kSrcHeight, dst, + dst_w, dst_w, dst_h, f)); + EXPECT_EQ(0, ScalePlane(src, kSrcWidth, kSrcWidth, -kSrcHeight, dst, + dst_w, dst_w, dst_h, f)); + } + } + } + delete[] src; +} + +TEST_F(LibYUVScaleTest, ScalePlaneBoxMaxSrcHeight) { + const int kSrcWidth = 4; + const int kSrcHeight = 32768; + uint8_t* src = new uint8_t[kSrcWidth * kSrcHeight]; + memset(src, 0, kSrcWidth * kSrcHeight); + uint8_t dst[2 * 4] = {}; + + EXPECT_EQ(0, ScalePlane(src, kSrcWidth, kSrcWidth, kSrcHeight, dst, 2, 2, 4, + kFilterBox)); + delete[] src; +} + +#if defined(__LP64__) || defined(_WIN64) +TEST_F(LibYUVScaleTest, ScaleLargeImageOver2GB) { + const int kSrcWidth = 4; + const int kSrcHeight = 32768; + const int kSrcStride = 131072; // 32768 * 131072 = 4 GB (> INT_MAX) + const size_t kSrcSize = (size_t)kSrcStride * kSrcHeight; + uint8_t* src = static_cast(calloc(1, kSrcSize)); + if (!src) { + GTEST_SKIP() << "could not allocate " << kSrcSize << " bytes"; + } + uint8_t dst[8 * 4 * 4] = {}; + + for (FilterMode f : + {kFilterNone, kFilterLinear, kFilterBilinear, kFilterBox}) { + EXPECT_EQ( + 0, ScalePlane(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 4, 4, 2, f)); + EXPECT_EQ(0, ScalePlane(src, kSrcStride, kSrcWidth, -kSrcHeight, dst, 4, 4, + 2, f)); + EXPECT_EQ( + 0, ScalePlane(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 8, 8, 2, f)); + EXPECT_EQ(0, UVScale(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 4 * 2, 4, + 2, f)); + EXPECT_EQ(0, UVScale(src, kSrcStride, kSrcWidth, -kSrcHeight, dst, 4 * 2, 4, + 2, f)); + EXPECT_EQ(0, UVScale(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 8 * 2, 8, + 2, f)); + } + free(src); +} +#endif + } // namespace libyuv