Harden I21xToI420 against oversize heights

I21xToI420 (reached via I210ToI420 / I212ToI420): fix integer overflows
in ScalePlaneVertical_16To8(). Make the same changes to the related
ScalePlaneVertical() and ScalePlaneVertical_16() functions.

Add a unit test covering the I21x vertical-scale path.

Co-authored-by: Timothy Nikkel <tnikkel@mozilla.com>
Test: libyuv_unittest --gtest_filter=*.I21xToI420NoHeightOverflow
Bug: chromium:531172025
Change-Id: Ieb8baad2576f4ef5ec71d771a6738a0e5cc7d21c

include/libyuv/scale_row.h

@@ -197,45 +197,45 @@ extern "C" {
 #endif
 
 // Scale ARGB vertically with bilinear interpolation.
-void ScalePlaneVertical(int src_height,
-                        int dst_width,
-                        int dst_height,
-                        int src_stride,
-                        int dst_stride,
-                        const uint8_t* src_argb,
-                        uint8_t* dst_argb,
-                        int x,
-                        int y,
-                        int dy,
-                        int bpp,
-                        enum FilterMode filtering);
+int ScalePlaneVertical(int src_height,
+                       int dst_width,
+                       int dst_height,
+                       int src_stride,
+                       int dst_stride,
+                       const uint8_t* src_argb,
+                       uint8_t* dst_argb,
+                       int x,
+                       int y,
+                       int dy,
+                       int bpp,
+                       enum FilterMode filtering);
 
-void ScalePlaneVertical_16(int src_height,
-                           int dst_width,
-                           int dst_height,
-                           int src_stride,
-                           int dst_stride,
-                           const uint16_t* src_argb,
-                           uint16_t* dst_argb,
-                           int x,
-                           int y,
-                           int dy,
-                           int wpp,
-                           enum FilterMode filtering);
+int ScalePlaneVertical_16(int src_height,
+                          int dst_width,
+                          int dst_height,
+                          int src_stride,
+                          int dst_stride,
+                          const uint16_t* src_argb,
+                          uint16_t* dst_argb,
+                          int x,
+                          int y,
+                          int dy,
+                          int wpp,
+                          enum FilterMode filtering);
 
-void ScalePlaneVertical_16To8(int src_height,
-                              int dst_width,
-                              int dst_height,
-                              int src_stride,
-                              int dst_stride,
-                              const uint16_t* src_argb,
-                              uint8_t* dst_argb,
-                              int x,
-                              int y,
-                              int dy,
-                              int wpp,
-                              int scale,
-                              enum FilterMode filtering);
+int ScalePlaneVertical_16To8(int src_height,
+                             int dst_width,
+                             int dst_height,
+                             int src_stride,
+                             int dst_stride,
+                             const uint16_t* src_argb,
+                             uint8_t* dst_argb,
+                             int x,
+                             int y,
+                             int dy,
+                             int wpp,
+                             int scale,
+                             enum FilterMode filtering);
 
 void ScalePlaneDown2_16To8(int src_width,
                            int src_height,

source/convert.cc

@@ -295,15 +295,22 @@ static int I21xToI420(const uint16_t* src_y,
     const int uv_width = SUBSAMPLE(width, 1, 1);
     const int uv_height = SUBSAMPLE(height, 1, 1);
     const int dy = FixedDiv(height, uv_height);
+    int r;
 
     Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
                       height);
-    ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_u,
-                             dst_stride_u, src_u, dst_u, 0, 32768, dy,
-                             /*bpp=*/1, scale, kFilterBilinear);
-    ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_v,
-                             dst_stride_v, src_v, dst_v, 0, 32768, dy,
-                             /*bpp=*/1, scale, kFilterBilinear);
+    r = ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_u,
+                                 dst_stride_u, src_u, dst_u, 0, 32768, dy,
+                                 /*bpp=*/1, scale, kFilterBilinear);
+    if (r != 0) {
+      return r;
+    }
+    r = ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_v,
+                                 dst_stride_v, src_v, dst_v, 0, 32768, dy,
+                                 /*bpp=*/1, scale, kFilterBilinear);
+    if (r != 0) {
+      return r;
+    }
   }
   return 0;
 }

source/scale.cc

@@ -1985,9 +1985,9 @@ int ScalePlane(const uint8_t* src,
       dy = FixedDiv1(src_height, dst_height);
     }
     // Arbitrary scale vertically, but unscaled horizontally.
-    ScalePlaneVertical(src_height, dst_width, dst_height, src_stride,
-                       dst_stride, src, dst, 0, y, dy, /*bpp=*/1, filtering);
-    return 0;
+    return ScalePlaneVertical(src_height, dst_width, dst_height, src_stride,
+                              dst_stride, src, dst, 0, y, dy, /*bpp=*/1,
+                              filtering);
   }
   if (dst_width <= Abs(src_width) && dst_height <= src_height) {
     // Scale down.
@@ -2096,9 +2096,9 @@ int ScalePlane_16(const uint16_t* src,
       dy = FixedDiv1(src_height, dst_height);
     }
     // Arbitrary scale vertically, but unscaled horizontally.
-    ScalePlaneVertical_16(src_height, dst_width, dst_height, src_stride,
-                          dst_stride, src, dst, 0, y, dy, /*bpp=*/1, filtering);
-    return 0;
+    return ScalePlaneVertical_16(src_height, dst_width, dst_height, src_stride,
+                                 dst_stride, src, dst, 0, y, dy, /*bpp=*/1,
+                                 filtering);
   }
   if (dst_width <= Abs(src_width) && dst_height <= src_height) {
     // Scale down.

source/scale_argb.cc

@@ -746,9 +746,9 @@ static int ScaleARGB(const uint8_t* src,
   }
   if (dx == 0x10000 && (x & 0xffff) == 0) {
     // Arbitrary scale vertically, but unscaled horizontally.
-    ScalePlaneVertical(src_height, clip_width, clip_height, src_stride,
-                       dst_stride, src, dst, x, y, dy, /*bpp=*/4, filtering);
-    return 0;
+    return ScalePlaneVertical(src_height, clip_width, clip_height, src_stride,
+                              dst_stride, src, dst, x, y, dy, /*bpp=*/4,
+                              filtering);
   }
   if (filtering && dy < 65536) {
     return ScaleARGBBilinearUp(src_width, src_height, clip_width, clip_height,

source/scale_common.cc

@@ -11,6 +11,7 @@
 #include "libyuv/scale.h"
 
 #include <assert.h>
+#include <limits.h>
 #include <string.h>
 
 #include "libyuv/cpu_id.h"
@@ -1612,24 +1613,29 @@ void ScaleUVFilterCols64_C(uint8_t* dst_uv,
 #undef BLENDER
 
 // Scale plane vertically with bilinear interpolation.
-void ScalePlaneVertical(int src_height,
-                        int dst_width,
-                        int dst_height,
-                        int src_stride,
-                        int dst_stride,
-                        const uint8_t* src_argb,
-                        uint8_t* dst_argb,
-                        int x,
-                        int y,
-                        int dy,
-                        int bpp,  // bytes per pixel. 4 for ARGB.
-                        enum FilterMode filtering) {
+int ScalePlaneVertical(int src_height,
+                       int dst_width,
+                       int dst_height,
+                       int src_stride,
+                       int dst_stride,
+                       const uint8_t* src_argb,
+                       uint8_t* dst_argb,
+                       int x,
+                       int y,
+                       int dy,
+                       int bpp,  // bytes per pixel. 4 for ARGB.
+                       enum FilterMode filtering) {
   // TODO(fbarchard): Allow higher bpp.
-  int dst_width_bytes = dst_width * bpp;
+  int64_t dst_width_bytes64 = (int64_t)dst_width * bpp;
+  if (dst_width_bytes64 > INT_MAX) {
+    return -1;
+  }
+  int dst_width_bytes = (int)dst_width_bytes64;
   void (*InterpolateRow)(uint8_t* dst_argb, const uint8_t* src_argb,
                          ptrdiff_t src_stride, int dst_width,
                          int source_y_fraction) = InterpolateRow_C;
-  const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0;
+  const int64_t max_y =
+      (src_height > 1) ? (((int64_t)src_height - 1) << 16) - 1 : 0;
   int j;
   assert(bpp >= 1 && bpp <= 4);
   assert(src_height != 0);
@@ -1671,39 +1677,46 @@ void ScalePlaneVertical(int src_height,
   }
 #endif
 
+  int64_t y64 = y;
   for (j = 0; j < dst_height; ++j) {
     int yi;
     int yf;
-    if (y > max_y) {
-      y = max_y;
+    if (y64 > max_y) {
+      y64 = max_y;
     }
-    yi = y >> 16;
-    yf = filtering ? ((y >> 8) & 255) : 0;
+    yi = (int)(y64 >> 16);
+    yf = filtering ? (int)((y64 >> 8) & 255) : 0;
     InterpolateRow(dst_argb, src_argb + yi * (ptrdiff_t)src_stride, src_stride,
                    dst_width_bytes, yf);
     dst_argb += dst_stride;
-    y += dy;
+    y64 += dy;
   }
+  return 0;
 }
 
-void ScalePlaneVertical_16(int src_height,
-                           int dst_width,
-                           int dst_height,
-                           int src_stride,
-                           int dst_stride,
-                           const uint16_t* src_argb,
-                           uint16_t* dst_argb,
-                           int x,
-                           int y,
-                           int dy,
-                           int wpp, /* words per pixel. normally 1 */
-                           enum FilterMode filtering) {
+int ScalePlaneVertical_16(int src_height,
+                          int dst_width,
+                          int dst_height,
+                          int src_stride,
+                          int dst_stride,
+                          const uint16_t* src_argb,
+                          uint16_t* dst_argb,
+                          int x,
+                          int y,
+                          int dy,
+                          int wpp, /* words per pixel. normally 1 */
+                          enum FilterMode filtering) {
   // TODO(fbarchard): Allow higher wpp.
-  int dst_width_words = dst_width * wpp;
+  int64_t dst_width_words64 = (int64_t)dst_width * wpp;
+  if (dst_width_words64 > INT_MAX) {
+    return -1;
+  }
+  int dst_width_words = (int)dst_width_words64;
   void (*InterpolateRow)(uint16_t* dst_argb, const uint16_t* src_argb,
                          ptrdiff_t src_stride, int dst_width,
                          int source_y_fraction) = InterpolateRow_16_C;
-  const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0;
+  const int64_t max_y =
+      (src_height > 1) ? (((int64_t)src_height - 1) << 16) - 1 : 0;
   int j;
   assert(wpp >= 1 && wpp <= 2);
   assert(src_height != 0);
@@ -1739,19 +1752,21 @@ void ScalePlaneVertical_16(int src_height,
     InterpolateRow = InterpolateRow_16_SME;
   }
 #endif
+  int64_t y64 = y;
   for (j = 0; j < dst_height; ++j) {
     int yi;
     int yf;
-    if (y > max_y) {
-      y = max_y;
+    if (y64 > max_y) {
+      y64 = max_y;
     }
-    yi = y >> 16;
-    yf = filtering ? ((y >> 8) & 255) : 0;
+    yi = (int)(y64 >> 16);
+    yf = filtering ? (int)((y64 >> 8) & 255) : 0;
     InterpolateRow(dst_argb, src_argb + yi * (ptrdiff_t)src_stride, src_stride,
                    dst_width_words, yf);
     dst_argb += dst_stride;
-    y += dy;
+    y64 += dy;
   }
+  return 0;
 }
 
 // Use scale to convert lsb formats to msb, depending how many bits there are:
@@ -1760,26 +1775,31 @@ void ScalePlaneVertical_16(int src_height,
 // 4096 = 12 bits
 // 256 = 16 bits
 // TODO(fbarchard): change scale to bits
-void ScalePlaneVertical_16To8(int src_height,
-                              int dst_width,
-                              int dst_height,
-                              int src_stride,
-                              int dst_stride,
-                              const uint16_t* src_argb,
-                              uint8_t* dst_argb,
-                              int x,
-                              int y,
-                              int dy,
-                              int wpp, /* words per pixel. normally 1 */
-                              int scale,
-                              enum FilterMode filtering) {
+int ScalePlaneVertical_16To8(int src_height,
+                             int dst_width,
+                             int dst_height,
+                             int src_stride,
+                             int dst_stride,
+                             const uint16_t* src_argb,
+                             uint8_t* dst_argb,
+                             int x,
+                             int y,
+                             int dy,
+                             int wpp, /* words per pixel. normally 1 */
+                             int scale,
+                             enum FilterMode filtering) {
   // TODO(fbarchard): Allow higher wpp.
-  int dst_width_words = dst_width * wpp;
+  int64_t dst_width_words64 = (int64_t)dst_width * wpp;
+  if (dst_width_words64 > INT_MAX) {
+    return -1;
+  }
+  int dst_width_words = (int)dst_width_words64;
   // TODO(https://crbug.com/libyuv/931): Add NEON 32 bit and AVX2 versions.
   void (*InterpolateRow_16To8)(uint8_t* dst_argb, const uint16_t* src_argb,
                                ptrdiff_t src_stride, int scale, int dst_width,
                                int source_y_fraction) = InterpolateRow_16To8_C;
-  const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0;
+  const int64_t max_y =
+      (src_height > 1) ? (((int64_t)src_height - 1) << 16) - 1 : 0;
   int j;
   assert(wpp >= 1 && wpp <= 2);
   assert(src_height != 0);
@@ -1808,19 +1828,21 @@ void ScalePlaneVertical_16To8(int src_height,
     }
   }
 #endif
+  int64_t y64 = y;
   for (j = 0; j < dst_height; ++j) {
     int yi;
     int yf;
-    if (y > max_y) {
-      y = max_y;
+    if (y64 > max_y) {
+      y64 = max_y;
     }
-    yi = y >> 16;
-    yf = filtering ? ((y >> 8) & 255) : 0;
+    yi = (int)(y64 >> 16);
+    yf = filtering ? (int)((y64 >> 8) & 255) : 0;
     InterpolateRow_16To8(dst_argb, src_argb + yi * (ptrdiff_t)src_stride,
                          src_stride, scale, dst_width_words, yf);
     dst_argb += dst_stride;
-    y += dy;
+    y64 += dy;
   }
+  return 0;
 }
 
 // Simplify the filtering based on scale factors.

source/scale_uv.cc

@@ -1015,9 +1015,9 @@ static int ScaleUV(const uint8_t* src,
   // HAS_SCALEPLANEVERTICAL
   if (dx == 0x10000 && (x & 0xffff) == 0) {
     // Arbitrary scale vertically, but unscaled horizontally.
-    ScalePlaneVertical(src_height, clip_width, clip_height, src_stride,
-                       dst_stride, src, dst, x, y, dy, /*bpp=*/2, filtering);
-    return 0;
+    return ScalePlaneVertical(src_height, clip_width, clip_height, src_stride,
+                              dst_stride, src, dst, x, y, dy, /*bpp=*/2,
+                              filtering);
   }
   if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) {
     ScaleUVLinearUp2(src_width, src_height, clip_width, clip_height, src_stride,

unit_test/convert_test.cc

@@ -2593,6 +2593,60 @@ TEST_F(LibYUVConvertTest, ABGRToI420_Check) {
                 benchmark_cpu_info_);
 }
 
+// I210ToI420/I212ToI420 funnel through I21xToI420, which scales chroma
+// vertically with ScalePlaneVertical_16To8. Verify oversize heights do not
+// overflow a signed int.
+TEST_F(LibYUVConvertTest, I21xToI420NoHeightOverflow) {
+  int width = 4;
+  int height = 40000;
+  int half_width = (width + 1) / 2;
+  int half_height = (height + 1) / 2;
+  // I21x: 16-bit 4:2:2
+  align_buffer_page_end_16(src_y, width * height);
+  align_buffer_page_end_16(src_u, half_width * height);
+  align_buffer_page_end_16(src_v, half_width * height);
+  // I420: 8-bit 4:2:0
+  align_buffer_page_end(dst_y, width * height);
+  align_buffer_page_end(dst_u, half_width * half_height);
+  align_buffer_page_end(dst_v, half_width * half_height);
+
+  memset(src_y, 0, width * height * sizeof(uint16_t));
+  memset(src_u, 0, half_width * height * sizeof(uint16_t));
+  memset(src_v, 0, half_width * height * sizeof(uint16_t));
+
+  EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, 32767));
+  EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, 32768));
+  EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, 32769));
+  EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, -32767));
+  EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, -32768));
+  EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, -32769));
+  EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, 32767));
+  EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, 32768));
+  EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, 32769));
+  EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, -32767));
+  EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, -32768));
+  EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
+                          dst_v, 2, 4, -32769));
+
+  free_aligned_buffer_page_end_16(src_y);
+  free_aligned_buffer_page_end_16(src_u);
+  free_aligned_buffer_page_end_16(src_v);
+  free_aligned_buffer_page_end(dst_y);
+  free_aligned_buffer_page_end(dst_u);
+  free_aligned_buffer_page_end(dst_v);
+}
+
 #endif  // !defined(LEAN_TESTS)
 
 }  // namespace libyuv