From c682abe597d8ab5de7ad107c1bf5bc95a0c6402f Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Wed, 10 Jan 2018 15:02:37 -0800 Subject: [PATCH] libyuv: fix undefined mul overflow Bug: libyuv:771 Test: build asm with ubsan and check Change-Id: I966d0bff74eef9ddfbeb93965fbff24c1472927c Reviewed-on: https://chromium-review.googlesource.com/860898 Reviewed-by: Weiyong Yao Reviewed-by: Frank Barchard --- source/compare.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/compare.cc b/source/compare.cc index 8c379b59c..fd4acaddb 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -272,7 +272,7 @@ double CalcFramePsnr(const uint8* src_a, int stride_b, int width, int height) { - const uint64 samples = width * height; + const uint64 samples = (uint64)width * (uint64)height; const uint64 sse = ComputeSumSquareErrorPlane(src_a, stride_a, src_b, stride_b, width, height); return SumSquareErrorToPsnr(sse, samples); @@ -301,7 +301,8 @@ double I420Psnr(const uint8* src_y_a, src_u_a, stride_u_a, src_u_b, stride_u_b, width_uv, height_uv); const uint64 sse_v = ComputeSumSquareErrorPlane( src_v_a, stride_v_a, src_v_b, stride_v_b, width_uv, height_uv); - const uint64 samples = width * height + 2 * (width_uv * height_uv); + const uint64 samples = (uint64)width * (uint64)height + + 2 * ((uint64)width_uv * (uint64)height_uv); const uint64 sse = sse_y + sse_u + sse_v; return SumSquareErrorToPsnr(sse, samples); }