diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index a30a66a73..f53ea8bcc 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -50,10 +50,19 @@ int ConvertToARGB(const uint8_t* sample, int crop_height, enum RotationMode rotation, uint32_t fourcc) { + if (src_height == INT_MIN || crop_height == INT_MIN) { + return -1; + } + + int abs_src_height = (src_height < 0) ? -src_height : src_height; + int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height; + if (dst_argb == NULL || sample == NULL || src_width <= 0 || src_width > INT_MAX / 4 || crop_width <= 0 || crop_width > INT_MAX / 4 || - src_height == 0 || src_height == INT_MIN || crop_height == 0 || - crop_height == INT_MIN) { + src_height == 0 || crop_height == 0 || crop_x < 0 || crop_y < 0 || + crop_width > src_width || crop_x > src_width - crop_width || + abs_crop_height > abs_src_height || + crop_y > abs_src_height - abs_crop_height) { return -1; } @@ -61,8 +70,6 @@ int ConvertToARGB(const uint8_t* sample, int aligned_src_width = (src_width + 1) & ~1; const uint8_t* src; const uint8_t* src_uv; - int abs_src_height = (src_height < 0) ? -src_height : src_height; - int inv_crop_height = (crop_height < 0) ? -crop_height : crop_height; int r = 0; // One pass rotation is available for some formats. For the rest, convert @@ -75,7 +82,7 @@ int ConvertToARGB(const uint8_t* sample, uint8_t* dest_argb = dst_argb; int dest_dst_stride_argb = dst_stride_argb; uint8_t* rotate_buffer = NULL; - int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height; + int inv_crop_height = (crop_height < 0) ? -crop_height : crop_height; if (src_height < 0) { inv_crop_height = -inv_crop_height; @@ -98,95 +105,97 @@ int ConvertToARGB(const uint8_t* sample, switch (format) { // Single plane formats case FOURCC_YUY2: - src = sample + (aligned_src_width * crop_y + crop_x) * 2; + src = sample + ((ptrdiff_t)aligned_src_width * crop_y + crop_x) * 2; r = YUY2ToARGB(src, aligned_src_width * 2, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_UYVY: - src = sample + (aligned_src_width * crop_y + crop_x) * 2; + src = sample + ((ptrdiff_t)aligned_src_width * crop_y + crop_x) * 2; r = UYVYToARGB(src, aligned_src_width * 2, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_24BG: - src = sample + (src_width * crop_y + crop_x) * 3; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 3; r = RGB24ToARGB(src, src_width * 3, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_RAW: - src = sample + (src_width * crop_y + crop_x) * 3; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 3; r = RAWToARGB(src, src_width * 3, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_ARGB: if (!need_buf && !rotation) { - src = sample + (src_width * crop_y + crop_x) * 4; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4; r = ARGBToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width, inv_crop_height); } break; case FOURCC_BGRA: - src = sample + (src_width * crop_y + crop_x) * 4; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4; r = BGRAToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_ABGR: - src = sample + (src_width * crop_y + crop_x) * 4; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4; r = ABGRToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_RGBA: - src = sample + (src_width * crop_y + crop_x) * 4; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4; r = RGBAToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_AR30: - src = sample + (src_width * crop_y + crop_x) * 4; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4; r = AR30ToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_AB30: - src = sample + (src_width * crop_y + crop_x) * 4; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4; r = AB30ToARGB(src, src_width * 4, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_RGBP: - src = sample + (src_width * crop_y + crop_x) * 2; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 2; r = RGB565ToARGB(src, src_width * 2, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_RGBO: - src = sample + (src_width * crop_y + crop_x) * 2; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 2; r = ARGB1555ToARGB(src, src_width * 2, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_R444: - src = sample + (src_width * crop_y + crop_x) * 2; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 2; r = ARGB4444ToARGB(src, src_width * 2, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_I400: - src = sample + src_width * crop_y + crop_x; + src = sample + (ptrdiff_t)src_width * crop_y + crop_x; r = I400ToARGB(src, src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_J400: - src = sample + src_width * crop_y + crop_x; + src = sample + (ptrdiff_t)src_width * crop_y + crop_x; r = J400ToARGB(src, src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; // Biplanar formats case FOURCC_NV12: - src = sample + (src_width * crop_y + crop_x); - src_uv = - sample + aligned_src_width * (abs_src_height + crop_y / 2) + crop_x; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x); + src_uv = sample + + aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) + + crop_x; r = NV12ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; case FOURCC_NV21: - src = sample + (src_width * crop_y + crop_x); - src_uv = - sample + aligned_src_width * (abs_src_height + crop_y / 2) + crop_x; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x); + src_uv = sample + + aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) + + crop_x; // Call NV12 but with u and v parameters swapped. r = NV21ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); @@ -194,21 +203,21 @@ int ConvertToARGB(const uint8_t* sample, // Triplanar formats case FOURCC_I420: case FOURCC_YV12: { - const uint8_t* src_y = sample + (src_width * crop_y + crop_x); + const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); const uint8_t* src_u; const uint8_t* src_v; int halfwidth = (src_width + 1) / 2; int halfheight = (abs_src_height + 1) / 2; if (format == FOURCC_YV12) { - src_v = sample + src_width * abs_src_height + - (halfwidth * crop_y + crop_x) / 2; - src_u = sample + src_width * abs_src_height + - halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + src_v = sample + (ptrdiff_t)src_width * abs_src_height + + ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + src_u = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; } else { - src_u = sample + src_width * abs_src_height + - (halfwidth * crop_y + crop_x) / 2; - src_v = sample + src_width * abs_src_height + - halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + src_u = sample + (ptrdiff_t)src_width * abs_src_height + + ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + src_v = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; } r = I420ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); @@ -218,11 +227,12 @@ int ConvertToARGB(const uint8_t* sample, case FOURCC_J420: { int halfwidth = (src_width + 1) / 2; int halfheight = (abs_src_height + 1) / 2; - const uint8_t* src_y = sample + (src_width * crop_y + crop_x); - const uint8_t* src_u = sample + src_width * abs_src_height + - (halfwidth * crop_y + crop_x) / 2; - const uint8_t* src_v = sample + src_width * abs_src_height + - halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); + const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + + ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + + crop_x / 2; r = J420ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; @@ -231,11 +241,12 @@ int ConvertToARGB(const uint8_t* sample, case FOURCC_H420: { int halfwidth = (src_width + 1) / 2; int halfheight = (abs_src_height + 1) / 2; - const uint8_t* src_y = sample + (src_width * crop_y + crop_x); - const uint8_t* src_u = sample + src_width * abs_src_height + - (halfwidth * crop_y + crop_x) / 2; - const uint8_t* src_v = sample + src_width * abs_src_height + - halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); + const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + + ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + + crop_x / 2; r = H420ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; @@ -244,11 +255,12 @@ int ConvertToARGB(const uint8_t* sample, case FOURCC_U420: { int halfwidth = (src_width + 1) / 2; int halfheight = (abs_src_height + 1) / 2; - const uint8_t* src_y = sample + (src_width * crop_y + crop_x); - const uint8_t* src_u = sample + src_width * abs_src_height + - (halfwidth * crop_y + crop_x) / 2; - const uint8_t* src_v = sample + src_width * abs_src_height + - halfwidth * (halfheight + crop_y / 2) + crop_x / 2; + const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); + const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + + ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + + crop_x / 2; r = U420ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; @@ -257,19 +269,19 @@ int ConvertToARGB(const uint8_t* sample, case FOURCC_I422: case FOURCC_YV16: { int halfwidth = (src_width + 1) / 2; - const uint8_t* src_y = sample + src_width * crop_y + crop_x; + const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x; const uint8_t* src_u; const uint8_t* src_v; if (format == FOURCC_YV16) { - src_v = sample + src_width * abs_src_height + halfwidth * crop_y + - crop_x / 2; - src_u = sample + src_width * abs_src_height + - halfwidth * (abs_src_height + crop_y) + crop_x / 2; + src_v = sample + (ptrdiff_t)src_width * abs_src_height + + (ptrdiff_t)halfwidth * crop_y + crop_x / 2; + src_u = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + crop_x / 2; } else { - src_u = sample + src_width * abs_src_height + halfwidth * crop_y + - crop_x / 2; - src_v = sample + src_width * abs_src_height + - halfwidth * (abs_src_height + crop_y) + crop_x / 2; + src_u = sample + (ptrdiff_t)src_width * abs_src_height + + (ptrdiff_t)halfwidth * crop_y + crop_x / 2; + src_v = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + crop_x / 2; } r = I422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); @@ -278,11 +290,12 @@ int ConvertToARGB(const uint8_t* sample, case FOURCC_J422: { int halfwidth = (src_width + 1) / 2; - const uint8_t* src_y = sample + src_width * crop_y + crop_x; - const uint8_t* src_u = - sample + src_width * abs_src_height + halfwidth * crop_y + crop_x / 2; - const uint8_t* src_v = sample + src_width * abs_src_height + - halfwidth * (abs_src_height + crop_y) + crop_x / 2; + const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x; + const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + + (ptrdiff_t)halfwidth * crop_y + crop_x / 2; + const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + + crop_x / 2; r = J422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; @@ -290,11 +303,12 @@ int ConvertToARGB(const uint8_t* sample, case FOURCC_H422: { int halfwidth = (src_width + 1) / 2; - const uint8_t* src_y = sample + src_width * crop_y + crop_x; - const uint8_t* src_u = - sample + src_width * abs_src_height + halfwidth * crop_y + crop_x / 2; - const uint8_t* src_v = sample + src_width * abs_src_height + - halfwidth * (abs_src_height + crop_y) + crop_x / 2; + const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x; + const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + + (ptrdiff_t)halfwidth * crop_y + crop_x / 2; + const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + + crop_x / 2; r = H422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; @@ -302,11 +316,12 @@ int ConvertToARGB(const uint8_t* sample, case FOURCC_U422: { int halfwidth = (src_width + 1) / 2; - const uint8_t* src_y = sample + src_width * crop_y + crop_x; - const uint8_t* src_u = - sample + src_width * abs_src_height + halfwidth * crop_y + crop_x / 2; - const uint8_t* src_v = sample + src_width * abs_src_height + - halfwidth * (abs_src_height + crop_y) + crop_x / 2; + const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x; + const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + + (ptrdiff_t)halfwidth * crop_y + crop_x / 2; + const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + + halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + + crop_x / 2; r = H422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; @@ -314,15 +329,19 @@ int ConvertToARGB(const uint8_t* sample, case FOURCC_I444: case FOURCC_YV24: { - const uint8_t* src_y = sample + src_width * crop_y + crop_x; + const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x; const uint8_t* src_u; const uint8_t* src_v; if (format == FOURCC_YV24) { - src_v = sample + src_width * (abs_src_height + crop_y) + crop_x; - src_u = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x; + src_v = + sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x; + src_u = sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) + + crop_x; } else { - src_u = sample + src_width * (abs_src_height + crop_y) + crop_x; - src_v = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x; + src_u = + sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x; + src_v = sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) + + crop_x; } r = I444ToARGB(src_y, src_width, src_u, src_width, src_v, src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); @@ -330,33 +349,39 @@ int ConvertToARGB(const uint8_t* sample, } case FOURCC_J444: { - const uint8_t* src_y = sample + src_width * crop_y + crop_x; + const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x; const uint8_t* src_u; const uint8_t* src_v; - src_u = sample + src_width * (abs_src_height + crop_y) + crop_x; - src_v = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x; + src_u = + sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x; + src_v = sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) + + crop_x; r = J444ToARGB(src_y, src_width, src_u, src_width, src_v, src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; } case FOURCC_H444: { - const uint8_t* src_y = sample + src_width * crop_y + crop_x; + const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x; const uint8_t* src_u; const uint8_t* src_v; - src_u = sample + src_width * (abs_src_height + crop_y) + crop_x; - src_v = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x; + src_u = + sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x; + src_v = sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) + + crop_x; r = H444ToARGB(src_y, src_width, src_u, src_width, src_v, src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; } case FOURCC_U444: { - const uint8_t* src_y = sample + src_width * crop_y + crop_x; + const uint8_t* src_y = sample + (ptrdiff_t)src_width * crop_y + crop_x; const uint8_t* src_u; const uint8_t* src_v; - src_u = sample + src_width * (abs_src_height + crop_y) + crop_x; - src_v = sample + src_width * (abs_src_height * 2 + crop_y) + crop_x; + src_u = + sample + src_width * ((ptrdiff_t)abs_src_height + crop_y) + crop_x; + src_v = sample + src_width * ((ptrdiff_t)abs_src_height * 2 + crop_y) + + crop_x; r = U444ToARGB(src_y, src_width, src_u, src_width, src_v, src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; @@ -379,7 +404,7 @@ int ConvertToARGB(const uint8_t* sample, } free(rotate_buffer); } else if (rotation) { - src = sample + (src_width * crop_y + crop_x) * 4; + src = sample + ((ptrdiff_t)src_width * crop_y + crop_x) * 4; r = ARGBRotate(src, src_width * 4, dst_argb, dst_stride_argb, crop_width, inv_crop_height, rotation); }