From 7633328b5faa2801f8d3297c71f303455ae1676c Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Tue, 8 Oct 2024 12:52:14 -0700 Subject: [PATCH] Make functions that malloc check for ubsan math overflow - add support for negative heights - sanity check null pointers and invalid width/height Bug: b/371615496 Change-Id: Icbefcb1ccc5cdf90e417c73440c6fad3b63ed7df Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/5917072 Reviewed-by: Wan-Teh Chang --- source/convert_to_argb.cc | 8 +++++++- source/convert_to_i420.cc | 10 ++++++---- source/scale_argb.cc | 14 ++++++++++++-- source/scale_rgb.cc | 13 +++++++++++-- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index 952457163..d0ff18a5e 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -10,6 +10,10 @@ #include "libyuv/convert_argb.h" +#include +#include +#include + #include "libyuv/cpu_id.h" #ifdef HAVE_JPEG #include "libyuv/mjpeg_decoder.h" @@ -66,7 +70,9 @@ int ConvertToARGB(const uint8_t* sample, uint8_t* rotate_buffer = NULL; int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height; - if (dst_argb == NULL || sample == NULL || src_width <= 0 || crop_width <= 0 || + if (dst_argb == NULL || sample == NULL || + src_width <= 0 || src_width > INT_MAX / 4 || + crop_width <= 0 || crop_width > INT_MAX / 4 || src_height == 0 || crop_height == 0) { return -1; } diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc index 505f7dcd2..a2bc189be 100644 --- a/source/convert_to_i420.cc +++ b/source/convert_to_i420.cc @@ -8,10 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include - #include "libyuv/convert.h" +#include +#include +#include +#include + #include "libyuv/video_common.h" #ifdef __cplusplus @@ -46,7 +49,6 @@ int ConvertToI420(const uint8_t* sample, const uint8_t* src; const uint8_t* src_uv; const int abs_src_height = (src_height < 0) ? -src_height : src_height; - // TODO(nisse): Why allow crop_height < 0? const int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height; int r = 0; LIBYUV_BOOL need_buf = @@ -63,7 +65,7 @@ int ConvertToI420(const uint8_t* sample, const int inv_crop_height = (src_height < 0) ? -abs_crop_height : abs_crop_height; - if (!dst_y || !dst_u || !dst_v || !sample || src_width <= 0 || + if (!dst_y || !dst_u || !dst_v || !sample || src_width <= 0 || src_width > INT_MAX / 4 || crop_width <= 0 || src_height == 0 || crop_height == 0) { return -1; } diff --git a/source/scale_argb.cc b/source/scale_argb.cc index e32469a44..e95aa596f 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -10,7 +10,10 @@ #include "libyuv/scale.h" +#include #include +#include +#include #include #include "libyuv/cpu_id.h" @@ -1180,7 +1183,14 @@ int YUVToARGBScaleClip(const uint8_t* src_y, int r; (void)src_fourcc; // TODO(fbarchard): implement and/or assert. (void)dst_fourcc; - const uint64_t argb_buffer_size = (uint64_t)src_width * src_height * 4; + const int abs_src_height = (src_height < 0) ? -src_height : src_height; + if (!src_y || !src_u || !src_v || !dst_argb || + src_width <= 0 || src_width > INT_MAX / 4 || src_height == 0 || + dst_width <= 0 || dst_height <= 0 || + clip_width <= 0 || clip_height <= 0) { + return -1; + } + const uint64_t argb_buffer_size = (uint64_t)src_width * abs_src_height * 4; if (argb_buffer_size > SIZE_MAX) { return -1; // Invalid size. } @@ -1191,7 +1201,7 @@ int YUVToARGBScaleClip(const uint8_t* src_y, I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, argb_buffer, src_width * 4, src_width, src_height); - r = ARGBScaleClip(argb_buffer, src_width * 4, src_width, src_height, dst_argb, + r = ARGBScaleClip(argb_buffer, src_width * 4, src_width, abs_src_height, dst_argb, dst_stride_argb, dst_width, dst_height, clip_x, clip_y, clip_width, clip_height, filtering); free(argb_buffer); diff --git a/source/scale_rgb.cc b/source/scale_rgb.cc index 8bc576c99..225fd21ec 100644 --- a/source/scale_rgb.cc +++ b/source/scale_rgb.cc @@ -10,7 +10,10 @@ #include "libyuv/scale.h" /* For FilterMode */ +#include #include +#include +#include #include #include "libyuv/convert_argb.h" @@ -38,7 +41,13 @@ int RGBScale(const uint8_t* src_rgb, int dst_height, enum FilterMode filtering) { int r; - const uint64_t src_argb_size = (uint64_t)src_width * src_height * 4; + if (!src_rgb || !dst_rgb || + src_width <= 0 || src_width > INT_MAX / 4 || src_height == 0 || + dst_width <= 0 || dst_width > INT_MAX / 4 || dst_height <= 0) { + return -1; + } + const int abs_src_height = (src_height < 0) ? -src_height : src_height; + const uint64_t src_argb_size = (uint64_t)src_width * abs_src_height * 4; const uint64_t dst_argb_size = (uint64_t)dst_width * dst_height * 4; if (src_argb_size > (UINT64_MAX - dst_argb_size)) { return -1; // Invalid size. @@ -56,7 +65,7 @@ int RGBScale(const uint8_t* src_rgb, r = RGB24ToARGB(src_rgb, src_stride_rgb, src_argb, src_width * 4, src_width, src_height); if (!r) { - r = ARGBScale(src_argb, src_width * 4, src_width, src_height, dst_argb, + r = ARGBScale(src_argb, src_width * 4, src_width, abs_src_height, dst_argb, dst_width * 4, dst_width, dst_height, filtering); if (!r) { r = ARGBToRGB24(dst_argb, dst_width * 4, dst_rgb, dst_stride_rgb,