Make functions that malloc check for ubsan math overflow

- add support for negative heights
- sanity check null pointers and invalid width/height

Bug: b/371615496
Change-Id: Icbefcb1ccc5cdf90e417c73440c6fad3b63ed7df
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/5917072
Reviewed-by: Wan-Teh Chang <wtc@google.com>

source/convert_to_argb.cc

@@ -10,6 +10,10 @@
 
 #include "libyuv/convert_argb.h"
 
+#include <limits.h>
+#include <stdint.h>
+#include <stdlib.h>
+
 #include "libyuv/cpu_id.h"
 #ifdef HAVE_JPEG
 #include "libyuv/mjpeg_decoder.h"
@@ -66,7 +70,9 @@ int ConvertToARGB(const uint8_t* sample,
   uint8_t* rotate_buffer = NULL;
   int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height;
 
-  if (dst_argb == NULL || sample == NULL || src_width <= 0 || crop_width <= 0 ||
+  if (dst_argb == NULL || sample == NULL ||
+      src_width <= 0 || src_width > INT_MAX / 4 ||
+      crop_width <= 0 || crop_width > INT_MAX / 4 ||
       src_height == 0 || crop_height == 0) {
     return -1;
   }

source/convert_to_i420.cc

@@ -8,10 +8,13 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include <stdlib.h>
-
 #include "libyuv/convert.h"
 
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
 #include "libyuv/video_common.h"
 
 #ifdef __cplusplus
@@ -46,7 +49,6 @@ int ConvertToI420(const uint8_t* sample,
   const uint8_t* src;
   const uint8_t* src_uv;
   const int abs_src_height = (src_height < 0) ? -src_height : src_height;
-  // TODO(nisse): Why allow crop_height < 0?
   const int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height;
   int r = 0;
   LIBYUV_BOOL need_buf =
@@ -63,7 +65,7 @@ int ConvertToI420(const uint8_t* sample,
   const int inv_crop_height =
       (src_height < 0) ? -abs_crop_height : abs_crop_height;
 
-  if (!dst_y || !dst_u || !dst_v || !sample || src_width <= 0 ||
+  if (!dst_y || !dst_u || !dst_v || !sample || src_width <= 0 || src_width > INT_MAX / 4 ||
       crop_width <= 0 || src_height == 0 || crop_height == 0) {
     return -1;
   }

source/scale_argb.cc

@@ -10,7 +10,10 @@
 
 #include "libyuv/scale.h"
 
+#include <limits.h>
 #include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
 #include <string.h>
 
 #include "libyuv/cpu_id.h"
@@ -1180,7 +1183,14 @@ int YUVToARGBScaleClip(const uint8_t* src_y,
   int r;
   (void)src_fourcc;  // TODO(fbarchard): implement and/or assert.
   (void)dst_fourcc;
-  const uint64_t argb_buffer_size = (uint64_t)src_width * src_height * 4;
+  const int abs_src_height = (src_height < 0) ? -src_height : src_height;
+  if (!src_y || !src_u || !src_v || !dst_argb ||
+      src_width <= 0 || src_width > INT_MAX / 4 || src_height == 0 ||
+      dst_width <= 0 || dst_height <= 0 ||
+      clip_width <= 0 || clip_height <= 0) {
+    return -1;
+  }
+  const uint64_t argb_buffer_size = (uint64_t)src_width * abs_src_height * 4;
   if (argb_buffer_size > SIZE_MAX) {
     return -1;  // Invalid size.
   }
@@ -1191,7 +1201,7 @@ int YUVToARGBScaleClip(const uint8_t* src_y,
   I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
              argb_buffer, src_width * 4, src_width, src_height);
 
-  r = ARGBScaleClip(argb_buffer, src_width * 4, src_width, src_height, dst_argb,
+  r = ARGBScaleClip(argb_buffer, src_width * 4, src_width, abs_src_height, dst_argb,
                     dst_stride_argb, dst_width, dst_height, clip_x, clip_y,
                     clip_width, clip_height, filtering);
   free(argb_buffer);

source/scale_rgb.cc

@@ -10,7 +10,10 @@
 
 #include "libyuv/scale.h" /* For FilterMode */
 
+#include <limits.h>
 #include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
 #include <string.h>
 
 #include "libyuv/convert_argb.h"
@@ -38,7 +41,13 @@ int RGBScale(const uint8_t* src_rgb,
              int dst_height,
              enum FilterMode filtering) {
   int r;
-  const uint64_t src_argb_size = (uint64_t)src_width * src_height * 4;
+  if (!src_rgb || !dst_rgb ||
+      src_width <= 0 || src_width > INT_MAX / 4 || src_height == 0 ||
+      dst_width <= 0 || dst_width > INT_MAX / 4 || dst_height <= 0) {
+    return -1;
+  }
+  const int abs_src_height = (src_height < 0) ? -src_height : src_height;
+  const uint64_t src_argb_size = (uint64_t)src_width * abs_src_height * 4;
   const uint64_t dst_argb_size = (uint64_t)dst_width * dst_height * 4;
   if (src_argb_size > (UINT64_MAX - dst_argb_size)) {
     return -1;  // Invalid size.
@@ -56,7 +65,7 @@ int RGBScale(const uint8_t* src_rgb,
   r = RGB24ToARGB(src_rgb, src_stride_rgb, src_argb, src_width * 4, src_width,
                   src_height);
   if (!r) {
-    r = ARGBScale(src_argb, src_width * 4, src_width, src_height, dst_argb,
+    r = ARGBScale(src_argb, src_width * 4, src_width, abs_src_height, dst_argb,
                   dst_width * 4, dst_width, dst_height, filtering);
     if (!r) {
       r = ARGBToRGB24(dst_argb, dst_width * 4, dst_rgb, dst_stride_rgb,