Avoid signed int overflow in scale functions using uint32_t

Use uint32_t for 16.16 fixed-point coordinates (uy, ux, max_y) in scale
loops across ScalePlane*, ScaleUV*, ScaleARGB*, ScalePlaneVertical*, and
C column scaling helpers. When downscaling large images (e.g.
src_height >= 26215 with dst_height == 2 or src_height == 32768),
signed 32-bit y + dy or (src_height << 16) exceeds INT_MAX. Using
uint32_t prevents undefined signed integer overflow up to src_height
32768 while staying in a single 32-bit register on 32-bit and 64-bit
targets (x86, ARM, WASM, and RISC-V RVV) with zero performance overhead.

Also check |src_width| and |src_height| <= 32768 in ARGBScale,
ARGBScaleClip, and UVScale_16, and avoid FixedDiv(32768, 1) overflow in
ScalePlane, ScalePlane_16, and ScaleSlope.

Bug: chromium:531172025
Test: libyuv_unittest --gtest_filter=*ScalePlaneOversizeHeight*:*ScalePlaneBoxMaxSrcHeight*:*ARGBScale*
Change-Id: Ica2e05d7dfe94f81c6631c587f1ff46c2ad44735
This commit is contained in:
Frank Barchard 2026-10-08 00:26:24 -07:00
parent 9456b06227
commit 7fcaff2f76
6 changed files with 342 additions and 199 deletions

View File

@ -944,7 +944,7 @@ static int ScalePlaneBox(int src_width,
int y = 0;
int dx = 0;
int dy = 0;
const int max_y = (src_height << 16);
const uint32_t max_y = (uint32_t)src_height << 16;
ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterBox, &x, &y,
&dx, &dy);
src_width = Abs(src_width);
@ -1009,15 +1009,16 @@ static int ScalePlaneBox(int src_width,
}
#endif
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
int boxheight;
int iy = y >> 16;
int iy = uy >> 16;
const uint8_t* src = src_ptr + iy * src_stride;
y += dy;
if (y > max_y) {
y = max_y;
uy += dy;
if (uy > max_y) {
uy = max_y;
}
boxheight = MIN1((y >> 16) - iy);
boxheight = MIN1((int)(uy >> 16) - iy);
memset(row16, 0, src_width * 2);
for (k = 0; k < boxheight; ++k) {
ScaleAddRow(src, (uint16_t*)(row16), src_width);
@ -1045,7 +1046,7 @@ static int ScalePlaneBox_16(int src_width,
int y = 0;
int dx = 0;
int dy = 0;
const int max_y = (src_height << 16);
const uint32_t max_y = (uint32_t)src_height << 16;
ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterBox, &x, &y,
&dx, &dy);
src_width = Abs(src_width);
@ -1066,15 +1067,16 @@ static int ScalePlaneBox_16(int src_width,
}
#endif
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
int boxheight;
int iy = y >> 16;
int iy = uy >> 16;
const uint16_t* src = src_ptr + iy * src_stride;
y += dy;
if (y > max_y) {
y = max_y;
uy += dy;
if (uy > max_y) {
uy = max_y;
}
boxheight = MIN1((y >> 16) - iy);
boxheight = MIN1((int)(uy >> 16) - iy);
memset(row32, 0, src_width * 4);
for (k = 0; k < boxheight; ++k) {
ScaleAddRow(src, (uint32_t*)(row32), src_width);
@ -1109,7 +1111,7 @@ static int ScalePlaneBilinearDown(int src_width,
if (!row)
return 1;
const int max_y = (src_height - 1) << 16;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
int j;
void (*ScaleFilterCols)(uint8_t* dst_ptr, const uint8_t* src_ptr,
int dst_width, int x, int dx) =
@ -1203,25 +1205,23 @@ static int ScalePlaneBilinearDown(int src_width,
}
}
#endif
if (y > max_y) {
y = max_y;
}
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
int yi = y >> 16;
if (uy > max_y) {
uy = max_y;
}
int yi = uy >> 16;
const uint8_t* src = src_ptr + yi * src_stride;
if (filtering == kFilterLinear) {
ScaleFilterCols(dst_ptr, src, dst_width, x, dx);
} else {
int yf = (y >> 8) & 255;
int yf = (uy >> 8) & 255;
InterpolateRow(row, src, src_stride, src_width, yf);
ScaleFilterCols(dst_ptr, row, dst_width, x, dx);
}
dst_ptr += dst_stride;
y += dy;
if (y > max_y) {
y = max_y;
}
uy += dy;
}
free_aligned_buffer_64(row);
return 0;
@ -1247,7 +1247,7 @@ static int ScalePlaneBilinearDown_16(int src_width,
if (!row)
return 1;
const int max_y = (src_height - 1) << 16;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
int j;
void (*ScaleFilterCols)(uint16_t* dst_ptr, const uint16_t* src_ptr,
int dst_width, int x, int dx) =
@ -1302,25 +1302,23 @@ static int ScalePlaneBilinearDown_16(int src_width,
ScaleFilterCols = ScaleFilterCols_16_SSSE3;
}
#endif
if (y > max_y) {
y = max_y;
}
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
int yi = y >> 16;
if (uy > max_y) {
uy = max_y;
}
int yi = uy >> 16;
const uint16_t* src = src_ptr + yi * src_stride;
if (filtering == kFilterLinear) {
ScaleFilterCols(dst_ptr, src, dst_width, x, dx);
} else {
int yf = (y >> 8) & 255;
int yf = (uy >> 8) & 255;
InterpolateRow((uint16_t*)row, src, src_stride, src_width, yf);
ScaleFilterCols(dst_ptr, (uint16_t*)row, dst_width, x, dx);
}
dst_ptr += dst_stride;
y += dy;
if (y > max_y) {
y = max_y;
}
uy += dy;
}
free_aligned_buffer_64(row);
return 0;
@ -1347,7 +1345,7 @@ static int ScalePlaneBilinearUp(int src_width,
int y = 0;
int dx = 0;
int dy = 0;
const int max_y = (src_height - 1) << 16;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
@ -1445,11 +1443,12 @@ static int ScalePlaneBilinearUp(int src_width,
#endif
}
if (y > max_y) {
y = max_y;
uint32_t uy = y;
if (uy > max_y) {
uy = max_y;
}
{
int yi = y >> 16;
int yi = uy >> 16;
const uint8_t* src = src_ptr + yi * src_stride;
// Allocate 2 row buffers.
@ -1478,31 +1477,31 @@ static int ScalePlaneBilinearUp(int src_width,
// 1. Scale the next source row into the older buffer (rowptr).
// 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;)
// so rowptr points to yi and (rowptr + rowstride) points to yi + 1.
// 3. Advance src by 1 row if row yi + 2 exists ((y + 65536) < max_y),
// 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y),
// otherwise clamp src at (src_height - 1) to avoid reading out of
// bounds.
for (j = 0; j < dst_height; ++j) {
if (y > max_y) {
y = max_y;
if (uy > max_y) {
uy = max_y;
}
yi = y >> 16;
yi = uy >> 16;
if (yi != lasty) {
ScaleFilterCols(rowptr, src, dst_width, x, dx);
rowptr += rowstride;
rowstride = -rowstride;
lasty = yi;
if ((y + 65536) < max_y) {
if ((uy + 65536) < max_y) {
src += src_stride;
}
}
if (filtering == kFilterLinear) {
InterpolateRow(dst_ptr, rowptr, 0, dst_width, 0);
} else {
int yf = (y >> 8) & 255;
int yf = (uy >> 8) & 255;
InterpolateRow(dst_ptr, rowptr, rowstride, dst_width, yf);
}
dst_ptr += dst_stride;
y += dy;
uy += dy;
}
free_aligned_buffer_64(row);
}
@ -1525,7 +1524,6 @@ static void ScalePlaneUp2_Linear(int src_width,
void (*ScaleRowUp)(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width) =
ScaleRowUp2_Linear_Any_C;
int i;
int y;
int dy;
(void)src_width;
@ -1565,12 +1563,13 @@ static void ScalePlaneUp2_Linear(int src_width,
ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr,
dst_width);
} else {
uint32_t uy = (1 << 15) - 1;
dy = FixedDiv(src_height - 1, dst_height - 1);
y = (1 << 15) - 1;
for (i = 0; i < dst_height; ++i) {
ScaleRowUp(src_ptr + (y >> 16) * src_stride, dst_ptr, dst_width);
int yi = uy >> 16;
ScaleRowUp(src_ptr + yi * src_stride, dst_ptr, dst_width);
dst_ptr += dst_stride;
y += dy;
uy += dy;
}
}
}
@ -1656,7 +1655,6 @@ static void ScalePlaneUp2_12_Linear(int src_width,
void (*ScaleRowUp)(const uint16_t* src_ptr, uint16_t* dst_ptr,
int dst_width) = ScaleRowUp2_Linear_16_Any_C;
int i;
int y;
int dy;
(void)src_width;
@ -1685,12 +1683,13 @@ static void ScalePlaneUp2_12_Linear(int src_width,
ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr,
dst_width);
} else {
uint32_t uy = (1 << 15) - 1;
dy = FixedDiv(src_height - 1, dst_height - 1);
y = (1 << 15) - 1;
for (i = 0; i < dst_height; ++i) {
ScaleRowUp(src_ptr + (y >> 16) * src_stride, dst_ptr, dst_width);
int yi = uy >> 16;
ScaleRowUp(src_ptr + yi * src_stride, dst_ptr, dst_width);
dst_ptr += dst_stride;
y += dy;
uy += dy;
}
}
}
@ -1759,7 +1758,6 @@ static void ScalePlaneUp2_16_Linear(int src_width,
void (*ScaleRowUp)(const uint16_t* src_ptr, uint16_t* dst_ptr,
int dst_width) = ScaleRowUp2_Linear_16_Any_C;
int i;
int y;
int dy;
(void)src_width;
@ -1788,12 +1786,13 @@ static void ScalePlaneUp2_16_Linear(int src_width,
ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr,
dst_width);
} else {
uint32_t uy = (1 << 15) - 1;
dy = FixedDiv(src_height - 1, dst_height - 1);
y = (1 << 15) - 1;
for (i = 0; i < dst_height; ++i) {
ScaleRowUp(src_ptr + (y >> 16) * src_stride, dst_ptr, dst_width);
int yi = uy >> 16;
ScaleRowUp(src_ptr + yi * src_stride, dst_ptr, dst_width);
dst_ptr += dst_stride;
y += dy;
uy += dy;
}
}
}
@ -1866,7 +1865,7 @@ static int ScalePlaneBilinearUp_16(int src_width,
int y = 0;
int dx = 0;
int dy = 0;
const int max_y = (src_height - 1) << 16;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
void (*InterpolateRow)(uint16_t* dst_ptr, const uint16_t* src_ptr,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_16_C;
@ -1933,11 +1932,12 @@ static int ScalePlaneBilinearUp_16(int src_width,
#endif
}
if (y > max_y) {
y = max_y;
uint32_t uy = y;
if (uy > max_y) {
uy = max_y;
}
{
int yi = y >> 16;
int yi = uy >> 16;
const uint16_t* src = src_ptr + yi * src_stride;
// Allocate 2 row buffers.
@ -1965,31 +1965,31 @@ static int ScalePlaneBilinearUp_16(int src_width,
// 1. Scale the next source row into the older buffer (rowptr).
// 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;)
// so rowptr points to yi and (rowptr + rowstride) points to yi + 1.
// 3. Advance src by 1 row if row yi + 2 exists ((y + 65536) < max_y),
// 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y),
// otherwise clamp src at (src_height - 1) to avoid reading out of
// bounds.
for (j = 0; j < dst_height; ++j) {
if (y > max_y) {
y = max_y;
if (uy > max_y) {
uy = max_y;
}
yi = y >> 16;
yi = uy >> 16;
if (yi != lasty) {
ScaleFilterCols(rowptr, src, dst_width, x, dx);
rowptr += rowstride;
rowstride = -rowstride;
lasty = yi;
if ((y + 65536) < max_y) {
if ((uy + 65536) < max_y) {
src += src_stride;
}
}
if (filtering == kFilterLinear) {
InterpolateRow(dst_ptr, rowptr, 0, dst_width, 0);
} else {
int yf = (y >> 8) & 255;
int yf = (uy >> 8) & 255;
InterpolateRow(dst_ptr, rowptr, rowstride, dst_width, yf);
}
dst_ptr += dst_stride;
y += dy;
uy += dy;
}
free_aligned_buffer_64(row);
}
@ -2017,6 +2017,7 @@ static void ScalePlaneSimple(int src_width,
int y = 0;
int dx = 0;
int dy = 0;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterNone, &x, &y,
&dx, &dy);
src_width = Abs(src_width);
@ -2030,10 +2031,15 @@ static void ScalePlaneSimple(int src_width,
#endif
}
uint32_t uy = y;
for (i = 0; i < dst_height; ++i) {
ScaleCols(dst_ptr, src_ptr + (y >> 16) * src_stride, dst_width, x, dx);
if (uy > max_y) {
uy = max_y;
}
int yi = uy >> 16;
ScaleCols(dst_ptr, src_ptr + yi * src_stride, dst_width, x, dx);
dst_ptr += dst_stride;
y += dy;
uy += dy;
}
}
@ -2053,6 +2059,7 @@ static void ScalePlaneSimple_16(int src_width,
int y = 0;
int dx = 0;
int dy = 0;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
ScaleSlope(src_width, src_height, dst_width, dst_height, kFilterNone, &x, &y,
&dx, &dy);
src_width = Abs(src_width);
@ -2066,10 +2073,15 @@ static void ScalePlaneSimple_16(int src_width,
#endif
}
uint32_t uy = y;
for (i = 0; i < dst_height; ++i) {
ScaleCols(dst_ptr, src_ptr + (y >> 16) * src_stride, dst_width, x, dx);
if (uy > max_y) {
uy = max_y;
}
int yi = uy >> 16;
ScaleCols(dst_ptr, src_ptr + yi * src_stride, dst_width, x, dx);
dst_ptr += dst_stride;
y += dy;
uy += dy;
}
}
@ -2114,7 +2126,10 @@ int ScalePlane(const uint8_t* src,
int dy = 0;
int y = 0;
// When scaling down, use the center 2 rows to filter.
if (dst_height <= src_height) {
if (dst_height == 1 && src_height >= 32768) {
dy = FixedDiv(src_height, src_height);
y = CENTERSTART(dy, -32768);
} else if (dst_height <= src_height) {
dy = FixedDiv(src_height, dst_height);
y = CENTERSTART(dy, -32768); // Subtract 0.5 (32768) to center filter.
} else if (src_height > 1 && dst_height > 1) {
@ -2229,7 +2244,10 @@ int ScalePlane_16(const uint16_t* src,
int dy = 0;
int y = 0;
// When scaling down, use the center 2 rows to filter.
if (dst_height <= src_height) {
if (dst_height == 1 && src_height >= 32768) {
dy = FixedDiv(src_height, src_height);
y = CENTERSTART(dy, -32768);
} else if (dst_height <= src_height) {
dy = FixedDiv(src_height, dst_height);
y = CENTERSTART(dy, -32768); // Subtract 0.5 (32768) to center filter.
} else if (src_height > 1 && dst_height > 1) {

View File

@ -459,23 +459,23 @@ static int ScaleARGBBilinearDown(int src_width,
if (!row)
return 1;
const int64_t max_y = (int64_t)(src_height - 1) << 16;
int64_t y64 = y;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
if (y64 > max_y) {
y64 = max_y;
if (uy > max_y) {
uy = max_y;
}
int yi = (int)(y64 >> 16);
int yi = uy >> 16;
const uint8_t* src = src_argb + yi * src_stride;
if (filtering == kFilterLinear) {
ScaleARGBFilterCols(dst_argb, src, dst_width, x, dx);
} else {
int yf = (int)((y64 >> 8) & 255);
int yf = (uy >> 8) & 255;
InterpolateRow(row, src, src_stride, clip_src_width, yf);
ScaleARGBFilterCols(dst_argb, row, dst_width, x, dx);
}
dst_argb += dst_stride;
y64 += dy;
uy += dy;
}
free_aligned_buffer_64(row);
}
@ -509,7 +509,7 @@ static int ScaleARGBBilinearUp(int src_width,
void (*ScaleARGBFilterCols)(uint8_t* dst_argb, const uint8_t* src_argb,
int dst_width, int x, int dx) =
filtering ? ScaleARGBFilterCols_C : ScaleARGBCols_C;
const int64_t max_y = (int64_t)(src_height - 1) << 16;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
@ -643,13 +643,13 @@ static int ScaleARGBBilinearUp(int src_width,
#endif
}
int64_t y64 = y;
if (y64 > max_y) {
y64 = max_y;
uint32_t uy = y;
if (uy > max_y) {
uy = max_y;
}
{
int yi = (int)(y64 >> 16);
int yi = uy >> 16;
const uint8_t* src = src_argb + yi * src_stride;
// Allocate 2 rows of ARGB.
@ -678,31 +678,31 @@ static int ScaleARGBBilinearUp(int src_width,
// 1. Scale the next source row into the older buffer (rowptr).
// 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;)
// so rowptr points to yi and (rowptr + rowstride) points to yi + 1.
// 3. Advance src by 1 row if row yi + 2 exists ((y64 + 65536) < max_y),
// 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y),
// otherwise clamp src at (src_height - 1) to avoid reading out of
// bounds.
for (j = 0; j < dst_height; ++j) {
if (y64 > max_y) {
y64 = max_y;
if (uy > max_y) {
uy = max_y;
}
yi = (int)(y64 >> 16);
yi = uy >> 16;
if (yi != lasty) {
ScaleARGBFilterCols(rowptr, src, dst_width, x, dx);
rowptr += rowstride;
rowstride = -rowstride;
lasty = yi;
if ((y64 + 65536) < max_y) {
if ((uy + 65536) < max_y) {
src += src_stride;
}
}
if (filtering == kFilterLinear) {
InterpolateRow(dst_argb, rowptr, 0, dst_width * 4, 0);
} else {
int yf = (int)((y64 >> 8) & 255);
int yf = (uy >> 8) & 255;
InterpolateRow(dst_argb, rowptr, rowstride, dst_width * 4, yf);
}
dst_argb += dst_stride;
y64 += dy;
uy += dy;
}
free_aligned_buffer_64(row);
}
@ -730,7 +730,7 @@ static void ScaleARGBSimple(int src_width,
void (*ScaleARGBCols)(uint8_t* dst_argb, const uint8_t* src_argb,
int dst_width, int x, int dx) =
(src_width >= 32768) ? ScaleARGBCols64_C : ScaleARGBCols_C;
const int64_t max_y = (int64_t)(src_height - 1) << 16;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
#if defined(HAS_SCALEARGBCOLS_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && src_width < 32768) {
ScaleARGBCols = ScaleARGBCols_SSE2;
@ -774,15 +774,15 @@ static void ScaleARGBSimple(int src_width,
#endif
}
int64_t y64 = y;
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
if (y64 > max_y) {
y64 = max_y;
if (uy > max_y) {
uy = max_y;
}
ScaleARGBCols(dst_argb, src_argb + (y64 >> 16) * src_stride, dst_width, x,
dx);
int yi = uy >> 16;
ScaleARGBCols(dst_argb, src_argb + yi * src_stride, dst_width, x, dx);
dst_argb += dst_stride;
y64 += dy;
uy += dy;
}
}
@ -906,7 +906,8 @@ int ARGBScaleClip(const uint8_t* src_argb,
int clip_width,
int clip_height,
enum FilterMode filtering) {
if (!src_argb || src_width == 0 || src_height == 0 || src_height == INT_MIN ||
if (!src_argb || src_width == 0 || src_width > 32768 || src_width < -32768 ||
src_height == 0 || src_height > 32768 || src_height < -32768 ||
!dst_argb || dst_width <= 0 || dst_height <= 0 || clip_x < 0 ||
clip_y < 0 || clip_width > 32768 || clip_height > 32768 ||
(clip_x + clip_width) > dst_width ||
@ -929,9 +930,9 @@ int ARGBScale(const uint8_t* src_argb,
int dst_width,
int dst_height,
enum FilterMode filtering) {
if (!src_argb || src_width == 0 || src_height == 0 || src_height == INT_MIN ||
src_width > 32768 || src_height > 32768 || !dst_argb || dst_width <= 0 ||
dst_height <= 0) {
if (!src_argb || src_width == 0 || src_width > 32768 || src_width < -32768 ||
src_height == 0 || src_height > 32768 || src_height < -32768 ||
!dst_argb || dst_width <= 0 || dst_height <= 0) {
return -1;
}
return ScaleARGB(src_argb, src_stride_argb, src_width, src_height, dst_argb,

View File

@ -499,16 +499,17 @@ void ScaleCols_C(uint8_t* dst_ptr,
int dst_width,
int x,
int dx) {
uint32_t ux = x;
int j;
for (j = 0; j < dst_width - 1; j += 2) {
dst_ptr[0] = src_ptr[x >> 16];
x += dx;
dst_ptr[1] = src_ptr[x >> 16];
x += dx;
dst_ptr[0] = src_ptr[ux >> 16];
ux += dx;
dst_ptr[1] = src_ptr[ux >> 16];
ux += dx;
dst_ptr += 2;
}
if (dst_width & 1) {
dst_ptr[0] = src_ptr[x >> 16];
dst_ptr[0] = src_ptr[ux >> 16];
}
}
@ -517,16 +518,17 @@ void ScaleCols_16_C(uint16_t* dst_ptr,
int dst_width,
int x,
int dx) {
uint32_t ux = x;
int j;
for (j = 0; j < dst_width - 1; j += 2) {
dst_ptr[0] = src_ptr[x >> 16];
x += dx;
dst_ptr[1] = src_ptr[x >> 16];
x += dx;
dst_ptr[0] = src_ptr[ux >> 16];
ux += dx;
dst_ptr[1] = src_ptr[ux >> 16];
ux += dx;
dst_ptr += 2;
}
if (dst_width & 1) {
dst_ptr[0] = src_ptr[x >> 16];
dst_ptr[0] = src_ptr[ux >> 16];
}
}
@ -582,25 +584,26 @@ void ScaleFilterCols_C(uint8_t* dst_ptr,
int dst_width,
int x,
int dx) {
uint32_t ux = x;
int j;
for (j = 0; j < dst_width - 1; j += 2) {
int xi = x >> 16;
int xi = ux >> 16;
int a = src_ptr[xi];
int b = src_ptr[xi + 1];
dst_ptr[0] = BLENDER(a, b, x & 0xffff);
x += dx;
xi = x >> 16;
dst_ptr[0] = BLENDER(a, b, ux & 0xffff);
ux += dx;
xi = ux >> 16;
a = src_ptr[xi];
b = src_ptr[xi + 1];
dst_ptr[1] = BLENDER(a, b, x & 0xffff);
x += dx;
dst_ptr[1] = BLENDER(a, b, ux & 0xffff);
ux += dx;
dst_ptr += 2;
}
if (dst_width & 1) {
int xi = x >> 16;
int xi = ux >> 16;
int a = src_ptr[xi];
int b = src_ptr[xi + 1];
dst_ptr[0] = BLENDER(a, b, x & 0xffff);
dst_ptr[0] = BLENDER(a, b, ux & 0xffff);
}
}
@ -644,25 +647,26 @@ void ScaleFilterCols_16_C(uint16_t* dst_ptr,
int dst_width,
int x,
int dx) {
uint32_t ux = x;
int j;
for (j = 0; j < dst_width - 1; j += 2) {
int xi = x >> 16;
int xi = ux >> 16;
int a = src_ptr[xi];
int b = src_ptr[xi + 1];
dst_ptr[0] = BLENDER(a, b, x & 0xffff);
x += dx;
xi = x >> 16;
dst_ptr[0] = BLENDER(a, b, ux & 0xffff);
ux += dx;
xi = ux >> 16;
a = src_ptr[xi];
b = src_ptr[xi + 1];
dst_ptr[1] = BLENDER(a, b, x & 0xffff);
x += dx;
dst_ptr[1] = BLENDER(a, b, ux & 0xffff);
ux += dx;
dst_ptr += 2;
}
if (dst_width & 1) {
int xi = x >> 16;
int xi = ux >> 16;
int a = src_ptr[xi];
int b = src_ptr[xi + 1];
dst_ptr[0] = BLENDER(a, b, x & 0xffff);
dst_ptr[0] = BLENDER(a, b, ux & 0xffff);
}
}
@ -973,15 +977,16 @@ void ScaleARGBCols_C(uint8_t* dst_argb,
int dst_width,
int x,
int dx) {
uint32_t ux = x;
int j;
for (j = 0; j < dst_width; ++j) {
const uint8_t* src = src_argb + (x >> 16) * 4;
const uint8_t* src = src_argb + (ux >> 16) * 4;
dst_argb[0] = src[0];
dst_argb[1] = src[1];
dst_argb[2] = src[2];
dst_argb[3] = src[3];
dst_argb += 4;
x += dx;
ux += dx;
}
}
@ -1037,10 +1042,11 @@ void ScaleARGBFilterCols_C(uint8_t* dst_argb,
int dst_width,
int x,
int dx) {
uint32_t ux = x;
int j;
for (j = 0; j < dst_width; ++j) {
int xi = (x >> 16) * 4;
int xf = (x >> 9) & 0x7f;
int xi = (ux >> 16) * 4;
int xf = (ux >> 9) & 0x7f;
const uint8_t* a = src_argb + xi;
const uint8_t* b = a + 4;
dst_argb[0] = (uint8_t)BLENDER1(a[0], b[0], xf);
@ -1048,7 +1054,7 @@ void ScaleARGBFilterCols_C(uint8_t* dst_argb,
dst_argb[2] = (uint8_t)BLENDER1(a[2], b[2], xf);
dst_argb[3] = (uint8_t)BLENDER1(a[3], b[3], xf);
dst_argb += 4;
x += dx;
ux += dx;
}
}
@ -1084,7 +1090,7 @@ void ScaleUVRowDown2_C(const uint8_t* src_uv,
int x;
(void)src_stride;
for (x = 0; x < dst_width; ++x) {
dst_uv[0] = src_uv[2]; // Store the 2nd UV
dst_uv[0] = src_uv[2];
dst_uv[1] = src_uv[3];
src_uv += 4;
dst_uv += 2;
@ -1277,13 +1283,14 @@ void ScaleUVCols_C(uint8_t* dst_uv,
int dst_width,
int x,
int dx) {
uint32_t ux = x;
int j;
for (j = 0; j < dst_width; ++j) {
const uint8_t* src = src_uv + (x >> 16) * 2;
const uint8_t* src = src_uv + (ux >> 16) * 2;
dst_uv[0] = src[0];
dst_uv[1] = src[1];
dst_uv += 2;
x += dx;
ux += dx;
}
}
@ -1333,16 +1340,17 @@ void ScaleUVFilterCols_C(uint8_t* dst_uv,
int dst_width,
int x,
int dx) {
uint32_t ux = x;
int j;
for (j = 0; j < dst_width; ++j) {
int xi = (x >> 16) * 2;
int xf = (x >> 9) & 0x7f;
int xi = (ux >> 16) * 2;
int xf = (ux >> 9) & 0x7f;
const uint8_t* a = src_uv + xi;
const uint8_t* b = a + 2;
dst_uv[0] = (uint8_t)BLENDER1(a[0], b[0], xf);
dst_uv[1] = (uint8_t)BLENDER1(a[1], b[1], xf);
dst_uv += 2;
x += dx;
ux += dx;
}
}
@ -1385,9 +1393,9 @@ void ScalePlaneVertical(int src_height,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
const bool interpolate = filtering == kFilterBilinear;
int64_t max_y = 0;
uint32_t max_y = 0;
if (src_height > 1) {
max_y = ((int64_t)src_height - 1) << 16;
max_y = (uint32_t)(src_height - 1) << 16;
if (interpolate) {
--max_y;
}
@ -1396,7 +1404,7 @@ void ScalePlaneVertical(int src_height,
// stride keeps it from forming a second source row pointer past the end when
// y is clamped to the last row.
const ptrdiff_t interpolate_stride = interpolate ? src_stride : 0;
int64_t y64 = y;
uint32_t uy = y;
int j;
assert(bpp >= 1 && bpp <= 4);
assert(src_height != 0);
@ -1459,15 +1467,15 @@ void ScalePlaneVertical(int src_height,
for (j = 0; j < dst_height; ++j) {
int yi;
int yf;
if (y64 > max_y) {
y64 = max_y;
if (uy > max_y) {
uy = max_y;
}
yi = (int)(y64 >> 16);
yf = interpolate ? (int)((y64 >> 8) & 255) : 0;
yi = uy >> 16;
yf = interpolate ? (int)((uy >> 8) & 255) : 0;
InterpolateRow(dst_argb, src_argb + yi * (ptrdiff_t)src_stride,
interpolate_stride, dst_width_bytes, yf);
dst_argb += dst_stride;
y64 += dy;
uy += dy;
}
}
@ -1489,9 +1497,9 @@ void ScalePlaneVertical_16(int src_height,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_16_C;
const bool interpolate = filtering == kFilterBilinear;
int64_t max_y = 0;
uint32_t max_y = 0;
if (src_height > 1) {
max_y = ((int64_t)src_height - 1) << 16;
max_y = (uint32_t)(src_height - 1) << 16;
if (interpolate) {
--max_y;
}
@ -1500,7 +1508,7 @@ void ScalePlaneVertical_16(int src_height,
// stride keeps it from forming a second source row pointer past the end when
// y is clamped to the last row.
const ptrdiff_t interpolate_stride = interpolate ? src_stride : 0;
int64_t y64 = y;
uint32_t uy = y;
int j;
assert(wpp >= 1 && wpp <= 2);
assert(src_height != 0);
@ -1547,15 +1555,15 @@ void ScalePlaneVertical_16(int src_height,
for (j = 0; j < dst_height; ++j) {
int yi;
int yf;
if (y64 > max_y) {
y64 = max_y;
if (uy > max_y) {
uy = max_y;
}
yi = (int)(y64 >> 16);
yf = interpolate ? (int)((y64 >> 8) & 255) : 0;
yi = uy >> 16;
yf = interpolate ? (int)((uy >> 8) & 255) : 0;
InterpolateRow(dst_argb, src_argb + yi * (ptrdiff_t)src_stride,
interpolate_stride, dst_width_words, yf);
dst_argb += dst_stride;
y64 += dy;
uy += dy;
}
}
@ -1635,8 +1643,8 @@ void ScaleSlope(int src_width,
assert(dst_width > 0);
assert(dst_height > 0);
// Check for 1 pixel and avoid FixedDiv overflow.
if (dst_width == 1 && src_width >= 32768) {
dst_width = src_width;
if (dst_width == 1 && Abs(src_width) >= 32768) {
dst_width = Abs(src_width);
}
if (dst_height == 1 && src_height >= 32768) {
dst_height = src_height;
@ -1652,7 +1660,7 @@ void ScaleSlope(int src_width,
if (dst_width <= Abs(src_width)) {
*dx = FixedDiv(Abs(src_width), dst_width);
*x = CENTERSTART(*dx, -32768); // Subtract 0.5 (32768) to center filter.
} else if (src_width > 1 && dst_width > 1) {
} else if (Abs(src_width) > 1 && dst_width > 1) {
*dx = FixedDiv1(Abs(src_width), dst_width);
*x = 0;
}
@ -1668,7 +1676,7 @@ void ScaleSlope(int src_width,
if (dst_width <= Abs(src_width)) {
*dx = FixedDiv(Abs(src_width), dst_width);
*x = CENTERSTART(*dx, -32768); // Subtract 0.5 (32768) to center filter.
} else if (src_width > 1 && dst_width > 1) {
} else if (Abs(src_width) > 1 && dst_width > 1) {
*dx = FixedDiv1(Abs(src_width), dst_width);
*x = 0;
}

View File

@ -320,7 +320,7 @@ static int ScaleUVBox(int src_width,
int y,
int dy) {
int j, k;
const int max_y = (src_height << 16);
const uint32_t max_y = (uint32_t)src_height << 16;
// Sized on src_width to keep ScaleAddRow aligned.
const int row_samples = src_width * 2;
{
@ -374,15 +374,16 @@ static int ScaleUVBox(int src_width,
}
#endif
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
int boxheight;
int iy = y >> 16;
int iy = uy >> 16;
const uint8_t* src = src_uv + iy * src_stride;
y += dy;
if (y > max_y) {
y = max_y;
uy += dy;
if (uy > max_y) {
uy = max_y;
}
boxheight = MIN1((y >> 16) - iy);
boxheight = MIN1((int)(uy >> 16) - iy);
memset(row16, 0, row_samples * 2);
for (k = 0; k < boxheight; ++k) {
ScaleAddRow(src, (uint16_t*)(row16), row_samples);
@ -581,27 +582,28 @@ static int ScaleUVBilinearDown(int src_width,
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
// Allocate a row of UV.
{
const int max_y = (src_height - 1) << 16;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
uint32_t uy = y;
align_buffer_64(row, clip_src_width * 2);
if (!row)
return 1;
if (y > max_y) {
y = max_y;
if (uy > max_y) {
uy = max_y;
}
for (j = 0; j < dst_height; ++j) {
int yi = y >> 16;
int yi = uy >> 16;
const uint8_t* src = src_uv + yi * src_stride;
if (filtering == kFilterLinear) {
ScaleUVFilterCols(dst_uv, src, dst_width, x, dx);
} else {
int yf = (y >> 8) & 255;
int yf = (uy >> 8) & 255;
InterpolateRow(row, src, src_stride, clip_src_width, yf);
ScaleUVFilterCols(dst_uv, row, dst_width, x, dx);
}
dst_uv += dst_stride;
y += dy;
if (y > max_y) {
y = max_y;
uy += dy;
if (uy > max_y) {
uy = max_y;
}
}
free_aligned_buffer_64(row);
@ -638,7 +640,8 @@ static int ScaleUVBilinearUp(int src_width,
void (*ScaleUVFilterCols)(uint8_t* dst_uv, const uint8_t* src_uv,
int dst_width, int x, int dx) =
filtering ? ScaleUVFilterCols_C : ScaleUVCols_C;
const int max_y = (src_height - 1) << 16;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
uint32_t uy = y;
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
@ -729,12 +732,12 @@ static int ScaleUVBilinearUp(int src_width,
#endif
}
if (y > max_y) {
y = max_y;
if (uy > max_y) {
uy = max_y;
}
{
int yi = y >> 16;
int yi = uy >> 16;
const uint8_t* src = src_uv + yi * src_stride;
// Allocate 2 rows of UV.
@ -763,31 +766,31 @@ static int ScaleUVBilinearUp(int src_width,
// 1. Scale the next source row into the older buffer (rowptr).
// 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;)
// so rowptr points to yi and (rowptr + rowstride) points to yi + 1.
// 3. Advance src by 1 row if row yi + 2 exists ((y + 65536) < max_y),
// 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y),
// otherwise clamp src at (src_height - 1) to avoid reading out of
// bounds.
for (j = 0; j < dst_height; ++j) {
if (y > max_y) {
y = max_y;
if (uy > max_y) {
uy = max_y;
}
yi = y >> 16;
yi = uy >> 16;
if (yi != lasty) {
ScaleUVFilterCols(rowptr, src, dst_width, x, dx);
rowptr += rowstride;
rowstride = -rowstride;
lasty = yi;
if ((y + 65536) < max_y) {
if ((uy + 65536) < max_y) {
src += src_stride;
}
}
if (filtering == kFilterLinear) {
InterpolateRow(dst_uv, rowptr, 0, dst_width * 2, 0);
} else {
int yf = (y >> 8) & 255;
int yf = (uy >> 8) & 255;
InterpolateRow(dst_uv, rowptr, rowstride, dst_width * 2, yf);
}
dst_uv += dst_stride;
y += dy;
uy += dy;
}
free_aligned_buffer_64(row);
}
@ -811,7 +814,6 @@ static void ScaleUVLinearUp2(int src_width,
void (*ScaleRowUp)(const uint8_t* src_uv, uint8_t* dst_uv, int dst_width) =
ScaleUVRowUp2_Linear_Any_C;
int i;
int y;
int dy;
// This function can only scale up by 2 times horizontally.
@ -845,12 +847,13 @@ static void ScaleUVLinearUp2(int src_width,
if (dst_height == 1) {
ScaleRowUp(src_uv + ((src_height - 1) / 2) * src_stride, dst_uv, dst_width);
} else {
uint32_t uy = (1 << 15) - 1;
dy = FixedDiv(src_height - 1, dst_height - 1);
y = (1 << 15) - 1;
for (i = 0; i < dst_height; ++i) {
ScaleRowUp(src_uv + (y >> 16) * src_stride, dst_uv, dst_width);
int yi = uy >> 16;
ScaleRowUp(src_uv + yi * src_stride, dst_uv, dst_width);
dst_uv += dst_stride;
y += dy;
uy += dy;
}
}
}
@ -863,8 +866,8 @@ static void ScaleUVBilinearUp2(int src_width,
int src_height,
int dst_width,
int dst_height,
int src_stride,
int dst_stride,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_ptr,
uint8_t* dst_ptr) {
void (*Scale2RowUp)(const uint8_t* src_ptr, ptrdiff_t src_stride,
@ -931,7 +934,6 @@ static void ScaleUVLinearUp2_16(int src_width,
void (*ScaleRowUp)(const uint16_t* src_uv, uint16_t* dst_uv, int dst_width) =
ScaleUVRowUp2_Linear_16_Any_C;
int i;
int y;
int dy;
// This function can only scale up by 2 times horizontally.
@ -959,12 +961,13 @@ static void ScaleUVLinearUp2_16(int src_width,
if (dst_height == 1) {
ScaleRowUp(src_uv + ((src_height - 1) / 2) * src_stride, dst_uv, dst_width);
} else {
uint32_t uy = (1 << 15) - 1;
dy = FixedDiv(src_height - 1, dst_height - 1);
y = (1 << 15) - 1;
for (i = 0; i < dst_height; ++i) {
ScaleRowUp(src_uv + (y >> 16) * src_stride, dst_uv, dst_width);
int yi = uy >> 16;
ScaleRowUp(src_uv + yi * src_stride, dst_uv, dst_width);
dst_uv += dst_stride;
y += dy;
uy += dy;
}
}
}
@ -977,8 +980,8 @@ static void ScaleUVBilinearUp2_16(int src_width,
int src_height,
int dst_width,
int dst_height,
int src_stride,
int dst_stride,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint16_t* src_ptr,
uint16_t* dst_ptr) {
void (*Scale2RowUp)(const uint16_t* src_ptr, ptrdiff_t src_stride,
@ -1067,10 +1070,16 @@ static void ScaleUVSimple(int src_width,
#endif
}
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
ScaleUVCols(dst_uv, src_uv + (y >> 16) * src_stride, dst_width, x, dx);
if (uy > max_y) {
uy = max_y;
}
int yi = uy >> 16;
ScaleUVCols(dst_uv, src_uv + yi * src_stride, dst_width, x, dx);
dst_uv += dst_stride;
y += dy;
uy += dy;
}
}
@ -1290,7 +1299,7 @@ int UVScale_16(const uint16_t* src_uv,
enum FilterMode filtering) {
int dy = 0;
if (!src_uv || src_width <= 0 || src_height == 0 || src_height == INT_MIN ||
if (!src_uv || src_width <= 0 || src_height == 0 || src_height < -32768 ||
src_width > 32768 || src_height > 32768 || !dst_uv || dst_width <= 0 ||
dst_height <= 0) {
return -1;

View File

@ -636,4 +636,46 @@ TEST_F(LibYUVScaleTest, ARGBScale_LargeHeightOverflow) {
free_aligned_buffer_page_end(orig_pixels);
}
TEST_F(LibYUVScaleTest, ARGBScaleDimensionOverflow) {
uint8_t src[16] = {};
uint8_t dst[4] = {};
EXPECT_EQ(-1, ARGBScale(src, 4, 32769, 1, dst, 4, 1, 1, kFilterNone));
EXPECT_EQ(-1, ARGBScale(src, 4, -32769, 1, dst, 4, 1, 1, kFilterNone));
EXPECT_EQ(-1, ARGBScale(src, 4, 1, 32769, dst, 4, 1, 1, kFilterNone));
EXPECT_EQ(-1, ARGBScale(src, 4, 1, -32769, dst, 4, 1, 1, kFilterNone));
EXPECT_EQ(-1, ARGBScaleClip(src, 4, 32769, 1, dst, 4, 1, 1, 0, 0, 1, 1,
kFilterNone));
EXPECT_EQ(-1, ARGBScaleClip(src, 4, -32769, 1, dst, 4, 1, 1, 0, 0, 1, 1,
kFilterNone));
EXPECT_EQ(-1, ARGBScaleClip(src, 4, 1, 32769, dst, 4, 1, 1, 0, 0, 1, 1,
kFilterNone));
EXPECT_EQ(-1, ARGBScaleClip(src, 4, 1, -32769, dst, 4, 1, 1, 0, 0, 1, 1,
kFilterNone));
}
#if defined(__LP64__) || defined(_WIN64)
TEST_F(LibYUVScaleTest, ARGBScaleLargeImageOver2GB) {
const int kSrcWidth = 4;
const int kSrcHeight = 32768;
const int kSrcStride = 131072; // 32768 * 131072 = 4 GB (> INT_MAX)
const size_t kSrcSize = (size_t)kSrcStride * kSrcHeight;
uint8_t* src = static_cast<uint8_t*>(calloc(1, kSrcSize));
if (!src) {
GTEST_SKIP() << "could not allocate " << kSrcSize << " bytes";
}
uint8_t dst[8 * 4 * 4] = {};
for (int dst_w : {4, 2}) {
for (FilterMode f :
{kFilterNone, kFilterLinear, kFilterBilinear, kFilterBox}) {
EXPECT_EQ(0, ARGBScale(src, kSrcStride, kSrcWidth, kSrcHeight, dst,
dst_w * 4, dst_w, 2, f));
EXPECT_EQ(0, ARGBScale(src, kSrcStride, kSrcWidth, -kSrcHeight, dst,
dst_w * 4, dst_w, 2, f));
}
}
free(src);
}
#endif
} // namespace libyuv

View File

@ -19,6 +19,7 @@
#include "../unit_test/unit_test.h"
#include "libyuv/cpu_id.h"
#include "libyuv/scale.h"
#include "libyuv/scale_uv.h"
#ifdef ENABLE_ROW_TESTS
#include "libyuv/scale_row.h" // For ScaleRowDown2Box_Odd_C
@ -805,4 +806,68 @@ TEST_F(LibYUVScaleTest, ScalePlane_12_InvalidInputs) {
EXPECT_EQ(-1, ScalePlane_12(src, 4, 4, -32769, dst, 4, 4, 4, kFilterNone));
}
TEST_F(LibYUVScaleTest, ScalePlaneOversizeHeight) {
const int kSrcWidth = 4;
const int kSrcHeight = 32768;
uint8_t* src = new uint8_t[kSrcWidth * kSrcHeight];
memset(src, 0, kSrcWidth * kSrcHeight);
uint8_t dst[4 * 2] = {};
for (int dst_w : {4, 2}) {
for (int dst_h : {1, 2}) {
for (FilterMode f :
{kFilterNone, kFilterLinear, kFilterBilinear, kFilterBox}) {
EXPECT_EQ(0, ScalePlane(src, kSrcWidth, kSrcWidth, kSrcHeight, dst,
dst_w, dst_w, dst_h, f));
EXPECT_EQ(0, ScalePlane(src, kSrcWidth, kSrcWidth, -kSrcHeight, dst,
dst_w, dst_w, dst_h, f));
}
}
}
delete[] src;
}
TEST_F(LibYUVScaleTest, ScalePlaneBoxMaxSrcHeight) {
const int kSrcWidth = 4;
const int kSrcHeight = 32768;
uint8_t* src = new uint8_t[kSrcWidth * kSrcHeight];
memset(src, 0, kSrcWidth * kSrcHeight);
uint8_t dst[2 * 4] = {};
EXPECT_EQ(0, ScalePlane(src, kSrcWidth, kSrcWidth, kSrcHeight, dst, 2, 2, 4,
kFilterBox));
delete[] src;
}
#if defined(__LP64__) || defined(_WIN64)
TEST_F(LibYUVScaleTest, ScaleLargeImageOver2GB) {
const int kSrcWidth = 4;
const int kSrcHeight = 32768;
const int kSrcStride = 131072; // 32768 * 131072 = 4 GB (> INT_MAX)
const size_t kSrcSize = (size_t)kSrcStride * kSrcHeight;
uint8_t* src = static_cast<uint8_t*>(calloc(1, kSrcSize));
if (!src) {
GTEST_SKIP() << "could not allocate " << kSrcSize << " bytes";
}
uint8_t dst[8 * 4 * 4] = {};
for (FilterMode f :
{kFilterNone, kFilterLinear, kFilterBilinear, kFilterBox}) {
EXPECT_EQ(
0, ScalePlane(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 4, 4, 2, f));
EXPECT_EQ(0, ScalePlane(src, kSrcStride, kSrcWidth, -kSrcHeight, dst, 4, 4,
2, f));
EXPECT_EQ(
0, ScalePlane(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 8, 8, 2, f));
EXPECT_EQ(0, UVScale(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 4 * 2, 4,
2, f));
EXPECT_EQ(0, UVScale(src, kSrcStride, kSrcWidth, -kSrcHeight, dst, 4 * 2, 4,
2, f));
EXPECT_EQ(0, UVScale(src, kSrcStride, kSrcWidth, kSrcHeight, dst, 8 * 2, 8,
2, f));
}
free(src);
}
#endif
} // namespace libyuv