From f1c9b5898619f2ced836030c023249f5a8b189d1 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Fri, 25 Sep 2026 17:58:08 -0700 Subject: [PATCH] clang format and bump version Bug: None Change-Id: I6a482d5c538b113549b88e820473af644d6d3704 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8467772 Reviewed-by: richard winterton --- README.chromium | 2 +- include/libyuv/convert_from_argb.h | 27 +- include/libyuv/planar_functions.h | 18 +- include/libyuv/row.h | 82 +-- include/libyuv/row_sve.h | 53 +- include/libyuv/scale_row.h | 1 - include/libyuv/version.h | 2 +- source/convert.cc | 41 +- source/convert_from.cc | 4 +- source/convert_from_argb.cc | 35 +- source/convert_to_argb.cc | 12 +- source/convert_to_i420.cc | 27 +- source/mjpeg_decoder.cc | 3 +- source/planar_functions.cc | 15 +- source/row_any.cc | 35 +- source/row_common.cc | 38 +- source/row_gcc.cc | 855 ++++++++++++++--------------- source/row_lsx.cc | 8 +- source/row_neon.cc | 44 +- source/row_neon64.cc | 86 +-- source/row_rvv.cc | 21 +- source/row_sve.cc | 40 +- source/row_win.cc | 392 ++++++------- source/scale.cc | 7 +- source/scale_argb.cc | 3 +- source/scale_common.cc | 3 - source/scale_gcc.cc | 74 +-- source/scale_uv.cc | 3 +- unit_test/convert_argb_test.cc | 69 +-- unit_test/convert_test.cc | 623 +++++++++++---------- unit_test/planar_test.cc | 89 ++- unit_test/scale_argb_test.cc | 3 +- util/cpuid.c | 9 +- 33 files changed, 1360 insertions(+), 1364 deletions(-) diff --git a/README.chromium b/README.chromium index 35479dae2..7c7ac625b 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1972 +Version: 1973 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/convert_from_argb.h b/include/libyuv/convert_from_argb.h index cd27409c7..511160e2f 100644 --- a/include/libyuv/convert_from_argb.h +++ b/include/libyuv/convert_from_argb.h @@ -196,7 +196,8 @@ int ARGBToI444(const uint8_t* src_argb, int width, int height); -// ARGB to I444 with matrix. See ArgbConstants at the top of this file for usage. +// ARGB to I444 with matrix. See ArgbConstants at the top of this file for +// usage. LIBYUV_API int ARGBToI444Matrix(const uint8_t* src_argb, int src_stride_argb, @@ -260,7 +261,8 @@ int ABGRToI422(const uint8_t* src_abgr, int width, int height); -// ARGB to I422 with matrix. See ArgbConstants at the top of this file for usage. +// ARGB to I422 with matrix. See ArgbConstants at the top of this file for +// usage. LIBYUV_API int ARGBToI422Matrix(const uint8_t* src_argb, int src_stride_argb, @@ -408,7 +410,8 @@ int ARGBToNV12(const uint8_t* src_argb, int width, int height); -// ARGB to NV12 with matrix. See ArgbConstants at the top of this file for usage. +// ARGB to NV12 with matrix. See ArgbConstants at the top of this file for +// usage. LIBYUV_API int ARGBToNV12Matrix(const uint8_t* src_argb, int src_stride_argb, @@ -431,7 +434,8 @@ int ARGBToNV16(const uint8_t* src_argb, int width, int height); -// ARGB to NV16 with matrix. See ArgbConstants at the top of this file for usage. +// ARGB to NV16 with matrix. See ArgbConstants at the top of this file for +// usage. LIBYUV_API int ARGBToNV16Matrix(const uint8_t* src_argb, int src_stride_argb, @@ -454,7 +458,8 @@ int ARGBToNV24(const uint8_t* src_argb, int width, int height); -// ARGB to NV24 with matrix. See ArgbConstants at the top of this file for usage. +// ARGB to NV24 with matrix. See ArgbConstants at the top of this file for +// usage. LIBYUV_API int ARGBToNV24Matrix(const uint8_t* src_argb, int src_stride_argb, @@ -466,7 +471,8 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, int width, int height); -// ARGB to NV21 with matrix. See ArgbConstants at the top of this file for usage. +// ARGB to NV21 with matrix. See ArgbConstants at the top of this file for +// usage. LIBYUV_API int ARGBToNV21Matrix(const uint8_t* src_argb, int src_stride_argb, @@ -500,7 +506,8 @@ int ARGBToP010(const uint8_t* src_argb, int width, int height); -// ARGB to P010 with matrix. See ArgbConstants at the top of this file for usage. +// ARGB to P010 with matrix. See ArgbConstants at the top of this file for +// usage. LIBYUV_API int ARGBToP010Matrix(const uint8_t* src_argb, int src_stride_argb, @@ -523,7 +530,8 @@ int ARGBToP210(const uint8_t* src_argb, int width, int height); -// ARGB to P210 with matrix. See ArgbConstants at the top of this file for usage. +// ARGB to P210 with matrix. See ArgbConstants at the top of this file for +// usage. LIBYUV_API int ARGBToP210Matrix(const uint8_t* src_argb, int src_stride_argb, @@ -546,7 +554,8 @@ int ARGBToP410(const uint8_t* src_argb, int width, int height); -// ARGB to P410 with matrix. See ArgbConstants at the top of this file for usage. +// ARGB to P410 with matrix. See ArgbConstants at the top of this file for +// usage. LIBYUV_API int ARGBToP410Matrix(const uint8_t* src_argb, int src_stride_argb, diff --git a/include/libyuv/planar_functions.h b/include/libyuv/planar_functions.h index bb8968490..baef27306 100644 --- a/include/libyuv/planar_functions.h +++ b/include/libyuv/planar_functions.h @@ -52,15 +52,17 @@ void Convert16To8Plane(const uint16_t* src_y, int height); // Convert 8 bit Y to 10/12/16 bit. -// "bits" accepts bit depths (10, 12, 16) or legacy scale values (1024, 4096, 65536). +// "bits" accepts bit depths (10, 12, 16) or legacy scale values (1024, 4096, +// 65536). LIBYUV_API -void Convert8To16Plane(const uint8_t* src_y, - int src_stride_y, - uint16_t* dst_y, - int dst_stride_y, - int bits, // 10, 12, 16 bits (or 1024, 4096, 65536 scale) - int width, - int height); +void Convert8To16Plane( + const uint8_t* src_y, + int src_stride_y, + uint16_t* dst_y, + int dst_stride_y, + int bits, // 10, 12, 16 bits (or 1024, 4096, 65536 scale) + int width, + int height); LIBYUV_API void Convert8To8Plane(const uint8_t* src_y, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index a282b1198..dbf501f61 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -411,7 +411,7 @@ extern "C" { // The following are available for AVX512 clang x64 platforms: // TODO(fbarchard): Port to GCC and Visual C // TODO(b/42280744): re-enable HAS_ARGBTORGB24ROW_AVX512VBMI. -#if !defined(LIBYUV_DISABLE_X86) && \ +#if !defined(LIBYUV_DISABLE_X86) && \ (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \ !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_COPYROW_AVX512BW @@ -429,7 +429,7 @@ extern "C" { // The following are available for AVX512 clang x64 platforms: // TODO(fbarchard): Port to x86 -#if !defined(LIBYUV_DISABLE_X86) && \ +#if !defined(LIBYUV_DISABLE_X86) && \ (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) #define HAS_I422TOARGBROW_AVX512BW #define HAS_I422TOAR30ROW_AVX512BW @@ -2761,9 +2761,9 @@ void SplitUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_v, int width); void SplitUVRow_SVE2(const uint8_t* src_uv, - uint8_t* dst_u, - uint8_t* dst_v, - int width); + uint8_t* dst_u, + uint8_t* dst_v, + int width); void SplitUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, @@ -2926,9 +2926,9 @@ void MergeUVRow_NEON(const uint8_t* src_u, uint8_t* dst_uv, int width); void MergeUVRow_SVE2(const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_uv, - int width); + const uint8_t* src_v, + uint8_t* dst_uv, + int width); void MergeUVRow_SME(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, @@ -3980,8 +3980,8 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, - uint8_t* dst_rgb24, - int width); + uint8_t* dst_rgb24, + int width); void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_argb, @@ -5234,28 +5234,28 @@ void I422ToRGB24Row_Any_AVX2(const uint8_t* y_buf, const struct YuvConstants* yuvconstants, int width); void I422ToRGB565Row_Any_SSSE3(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - const struct YuvConstants* yuvconstants, - int width); + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGB1555Row_Any_SSSE3(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - const struct YuvConstants* yuvconstants, - int width); + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGB4444Row_Any_SSSE3(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - const struct YuvConstants* yuvconstants, - int width); + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); void NV12ToRGB565Row_Any_SSSE3(const uint8_t* y_buf, - const uint8_t* uv_buf, - uint8_t* dst_ptr, - const struct YuvConstants* yuvconstants, - int width); + const uint8_t* uv_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); void I422ToRGB565Row_Any_AVX2(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -6060,16 +6060,16 @@ void YUY2ToUVRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_v, int width); void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2, - int stride_yuy2, - uint8_t* dst_u, - uint8_t* dst_v, - int width); + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width); void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, - int stride_uyvy, - uint8_t* dst_u, - uint8_t* dst_v, - int width); + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_uv, @@ -6678,10 +6678,10 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, int dst_width, int source_y_fraction); void InterpolateRow_SVE2(uint8_t* dst_ptr, - const uint8_t* src_ptr, - ptrdiff_t src_stride, - int dst_width, - int source_y_fraction); + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int dst_width, + int source_y_fraction); void InterpolateRow_SME(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index da14fe8ff..00364e9c8 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -2150,33 +2150,38 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb, "z27", "p0", "p1", "p2", "p3", "p4", "p5"); } -#define ARGBTOYMATRIX_SVE(p1, p2, p3, p4, p5) \ - "ld1w {z0.s}, " p1 "/z, [%[src]] \n" /* load 4*vl pixels */\ - "ld1w {z1.s}, " p2 "/z, [%[src], #1, mul vl] \n" \ - "ld1w {z2.s}, " p3 "/z, [%[src], #2, mul vl] \n" \ - "ld1w {z3.s}, " p4 "/z, [%[src], #3, mul vl] \n" \ - "incb %[src], all, mul #4 \n" \ - "mov z16.b, #0 \n" \ - "mov z17.b, #0 \n" \ - "mov z18.b, #0 \n" \ - "mov z19.b, #0 \n" \ - "udot z16.s, z0.b, z24.b \n" \ - "udot z17.s, z1.b, z24.b \n" \ - "udot z18.s, z2.b, z24.b \n" \ - "udot z19.s, z3.b, z24.b \n" \ - "uzp1 z16.h, z16.h, z17.h \n" \ - "uzp1 z18.h, z18.h, z19.h \n" \ - "addhnb z16.b, z16.h, z25.h \n" \ - "addhnb z18.b, z18.h, z25.h \n" \ - "uzp1 z20.b, z16.b, z18.b \n" \ - "st1b {z20.b}, " p5 ", [%[dst_y]] \n" \ - "incb %[dst_y] \n" \ +#define ARGBTOYMATRIX_SVE(p1, p2, p3, p4, p5) \ + "ld1w {z0.s}, " p1 \ + "/z, [%[src]] \n" /* load 4*vl pixels */ \ + "ld1w {z1.s}, " p2 \ + "/z, [%[src], #1, mul vl] \n" \ + "ld1w {z2.s}, " p3 \ + "/z, [%[src], #2, mul vl] \n" \ + "ld1w {z3.s}, " p4 \ + "/z, [%[src], #3, mul vl] \n" \ + "incb %[src], all, mul #4 \n" \ + "mov z16.b, #0 \n" \ + "mov z17.b, #0 \n" \ + "mov z18.b, #0 \n" \ + "mov z19.b, #0 \n" \ + "udot z16.s, z0.b, z24.b \n" \ + "udot z17.s, z1.b, z24.b \n" \ + "udot z18.s, z2.b, z24.b \n" \ + "udot z19.s, z3.b, z24.b \n" \ + "uzp1 z16.h, z16.h, z17.h \n" \ + "uzp1 z18.h, z18.h, z19.h \n" \ + "addhnb z16.b, z16.h, z25.h \n" \ + "addhnb z18.b, z18.h, z25.h \n" \ + "uzp1 z20.b, z16.b, z18.b \n" \ + "st1b {z20.b}, " p5 \ + ", [%[dst_y]] \n" \ + "incb %[dst_y] \n" \ "subs %w[width], %w[width], %w[vl], lsl #2 \n" static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, - uint8_t* dst_y, - int width, - const struct ArgbConstants* c) + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) STREAMING_COMPATIBLE { uint64_t vl; asm("cntw %x0" : "=r"(vl)); diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index ee814d7f3..505bcac87 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -218,7 +218,6 @@ void ScalePlaneVertical_16(int src_height, int wpp, enum FilterMode filtering); - // Simplify the filtering based on scale factors. enum FilterMode ScaleFilterReduce(int src_width, int src_height, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index f369637ea..9daf405a6 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1972 +#define LIBYUV_VERSION 1973 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/convert.cc b/source/convert.cc index 0ac2f40d2..de0e62f3c 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -2091,9 +2091,9 @@ static int Px10ToI420(const uint16_t* src_y, SplitUVRow_16_C; void (*Half16To8)(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, int scale, int width) = - subsample_x ? HalfRow_16To8_C - : ((width & 1) ? HalfWidthRow_16To8_Odd_C - : HalfWidthRow_16To8_C); + subsample_x + ? HalfRow_16To8_C + : ((width & 1) ? HalfWidthRow_16To8_Odd_C : HalfWidthRow_16To8_C); #if defined(HAS_SPLITUVROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { @@ -3796,21 +3796,20 @@ int RAWToJ420(const uint8_t* src_raw, // Convert RAW to I444 with matrix. static int RAWToI444Matrix(const uint8_t* src_raw, - int src_stride_raw, - uint8_t* dst_y, - int dst_stride_y, - uint8_t* dst_u, - int dst_stride_u, - uint8_t* dst_v, - int dst_stride_v, - const struct ArgbConstants* argbconstants, - int width, - int height) { + int src_stride_raw, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + const struct ArgbConstants* argbconstants, + int width, + int height) { int y; - void (*RGBToUV444MatrixRow)(const uint8_t* src_rgb, uint8_t* dst_u, - uint8_t* dst_v, int width, - const struct ArgbConstants* c) = - RGBToUV444MatrixRow_C; + void (*RGBToUV444MatrixRow)( + const uint8_t* src_rgb, uint8_t* dst_u, uint8_t* dst_v, int width, + const struct ArgbConstants* c) = RGBToUV444MatrixRow_C; void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, const struct ArgbConstants* c) = RGBToYMatrixRow_C; @@ -3948,8 +3947,8 @@ int RAWToI444(const uint8_t* src_raw, int width, int height) { return RAWToI444Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_u, - dst_stride_u, dst_v, dst_stride_v, - &kAbgrI601Constants, width, height); + dst_stride_u, dst_v, dst_stride_v, &kAbgrI601Constants, + width, height); } // Convert RAW to J444. @@ -3965,8 +3964,8 @@ int RAWToJ444(const uint8_t* src_raw, int width, int height) { return RAWToI444Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_u, - dst_stride_u, dst_v, dst_stride_v, - &kAbgrJPEGConstants, width, height); + dst_stride_u, dst_v, dst_stride_v, &kAbgrJPEGConstants, + width, height); } // Convert RGB565 to I420. diff --git a/source/convert_from.cc b/source/convert_from.cc index 2e6e84945..6c0132b9f 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -957,8 +957,8 @@ static int I420ToPxxx(const uint8_t* src_y, dst_uv_width, width, height, dst_uv_width, abs_height); if (!r) { r = i4xx_to_pxxx(src_y, src_stride_y, tmp_u, dst_uv_width, tmp_v, - dst_uv_width, dst_y, dst_stride_y, dst_uv, - dst_stride_uv, width, height); + dst_uv_width, dst_y, dst_stride_y, dst_uv, dst_stride_uv, + width, height); } } free_aligned_buffer_64(plane_u); diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 5935b9828..2ddeca816 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -871,8 +871,8 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, } #endif - size_t row_size = (static_cast(halfwidth) + 31) & - ~static_cast(31); + size_t row_size = + (static_cast(halfwidth) + 31) & ~static_cast(31); if (row_size > SIZE_MAX / 2) { return -1; } @@ -1109,8 +1109,8 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, } #endif - size_t row_size = (static_cast(width) + 31) & - ~static_cast(31); + size_t row_size = + (static_cast(width) + 31) & ~static_cast(31); if (row_size > SIZE_MAX / 2) { return -1; } @@ -3586,7 +3586,6 @@ int RGBAToJ400(const uint8_t* src_rgba, &kRgbaJPEGConstants, width, height); } - // Convert ABGR to J420. (JPeg full range I420). LIBYUV_API int ABGRToJ420(const uint8_t* src_abgr, @@ -3767,21 +3766,20 @@ int ARGBToAB64(const uint8_t* src_argb, // Convert RAW to NV21 with Matrix. static int RAWToNVMatrix(const uint8_t* src_raw, - int src_stride_raw, - uint8_t* dst_y, - int dst_stride_y, - uint8_t* dst_uv, - int dst_stride_uv, - const struct ArgbConstants* argbconstants, - int width, - int height, - int is_nv21) { + int src_stride_raw, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + const struct ArgbConstants* argbconstants, + int width, + int height, + int is_nv21) { int y; int halfwidth = (width + 1) >> 1; void (*RGBToUVMatrixRow)(const uint8_t* src_rgb, int src_stride_rgb, uint8_t* dst_u, uint8_t* dst_v, int width, - const struct ArgbConstants* c) = - RGBToUVMatrixRow_C; + const struct ArgbConstants* c) = RGBToUVMatrixRow_C; void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, const struct ArgbConstants* c) = RGBToYMatrixRow_C; void (*MergeUVRow)(const uint8_t* src_uj, const uint8_t* src_vj, @@ -4034,9 +4032,8 @@ int RGB24ToNV12(const uint8_t* src_rgb24, int dst_stride_uv, int width, int height) { - return RAWToNVMatrix(src_rgb24, src_stride_rgb24, dst_y, dst_stride_y, - dst_uv, dst_stride_uv, &kArgbI601Constants, width, - height, 0); + return RAWToNVMatrix(src_rgb24, src_stride_rgb24, dst_y, dst_stride_y, dst_uv, + dst_stride_uv, &kArgbI601Constants, width, height, 0); } #ifdef __cplusplus diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index 1cca7a780..669154e79 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -123,9 +123,9 @@ static int NV24ToARGB(const uint8_t* src_y, int width, int height) { int y; - void (*I444ToARGBRow)( - const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, - uint8_t* rgb_buf, const struct YuvConstants* yuvconstants, int width) = + void (*I444ToARGBRow)(const uint8_t* y_buf, const uint8_t* u_buf, + const uint8_t* v_buf, uint8_t* rgb_buf, + const struct YuvConstants* yuvconstants, int width) = I444ToARGBRow_C; void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width) = SplitUVRow_C; @@ -293,8 +293,7 @@ int ConvertToARGB(const uint8_t* sample, // and then rotate the ARGB to the final destination buffer. // For in-place conversion, if destination dst_argb is same as source sample, // also enable temporary buffer. - int need_buf = - (rotation && format != FOURCC_ARGB) || dst_argb == sample; + int need_buf = (rotation && format != FOURCC_ARGB) || dst_argb == sample; uint8_t* dest_argb = dst_argb; int dest_dst_stride_argb = dst_stride_argb; uint8_t* rotate_buffer = NULL; @@ -428,8 +427,7 @@ int ConvertToARGB(const uint8_t* sample, break; case FOURCC_NV24: src = sample + ((ptrdiff_t)src_width * crop_y + crop_x); - src_uv = sample + - (ptrdiff_t)aligned_src_width * abs_src_height + + src_uv = sample + (ptrdiff_t)aligned_src_width * abs_src_height + (ptrdiff_t)aligned_src_width * 2 * crop_y + (ptrdiff_t)crop_x * 2; r = NV24ToARGB(src, src_width, src_uv, aligned_src_width * 2, dst_argb, diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc index 7fac092a0..d43b31f7b 100644 --- a/source/convert_to_i420.cc +++ b/source/convert_to_i420.cc @@ -161,14 +161,12 @@ static int NV24ToI420(const uint8_t* src_y, dst_u[halfwidth - 1] = row_uv[(halfwidth - 1) * 4 + 0]; dst_v[halfwidth - 1] = row_uv[(halfwidth - 1) * 4 + 1]; } else { - dst_u[halfwidth - 1] = - (row_uv[(halfwidth - 1) * 4 + 0] + row_uv[(halfwidth - 1) * 4 + 2] + - 1) >> - 1; - dst_v[halfwidth - 1] = - (row_uv[(halfwidth - 1) * 4 + 1] + row_uv[(halfwidth - 1) * 4 + 3] + - 1) >> - 1; + dst_u[halfwidth - 1] = (row_uv[(halfwidth - 1) * 4 + 0] + + row_uv[(halfwidth - 1) * 4 + 2] + 1) >> + 1; + dst_v[halfwidth - 1] = (row_uv[(halfwidth - 1) * 4 + 1] + + row_uv[(halfwidth - 1) * 4 + 3] + 1) >> + 1; } src_uv += (ptrdiff_t)src_stride_uv * 2; dst_u += dst_stride_u; @@ -219,10 +217,9 @@ int ConvertToI420(const uint8_t* sample, const uint8_t* src; const uint8_t* src_uv; int r = 0; - int need_buf = - (rotation && format != FOURCC_I420 && format != FOURCC_NV12 && - format != FOURCC_NV21 && format != FOURCC_YV12) || - dst_y == sample; + int need_buf = (rotation && format != FOURCC_I420 && format != FOURCC_NV12 && + format != FOURCC_NV21 && format != FOURCC_YV12) || + dst_y == sample; uint8_t* tmp_y = dst_y; uint8_t* tmp_u = dst_u; uint8_t* tmp_v = dst_v; @@ -363,8 +360,7 @@ int ConvertToI420(const uint8_t* sample, case FOURCC_NV16: src = sample + ((ptrdiff_t)src_width * crop_y + crop_x); src_uv = sample + ((ptrdiff_t)aligned_src_width * abs_src_height) + - ((ptrdiff_t)crop_y * aligned_src_width) + - ((crop_x / 2) * 2); + ((ptrdiff_t)crop_y * aligned_src_width) + ((crop_x / 2) * 2); r = NV16ToI420(src, src_width, src_uv, aligned_src_width, dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v, crop_width, inv_crop_height); @@ -372,8 +368,7 @@ int ConvertToI420(const uint8_t* sample, case FOURCC_NV24: src = sample + ((ptrdiff_t)src_width * crop_y + crop_x); src_uv = sample + ((ptrdiff_t)aligned_src_width * abs_src_height) + - ((ptrdiff_t)crop_y * aligned_src_width * 2) + - (crop_x * 2); + ((ptrdiff_t)crop_y * aligned_src_width * 2) + (crop_x * 2); r = NV24ToI420(src, src_width, src_uv, aligned_src_width * 2, dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v, crop_width, inv_crop_height); diff --git a/source/mjpeg_decoder.cc b/source/mjpeg_decoder.cc index 5388fef4e..28d18c96e 100644 --- a/source/mjpeg_decoder.cc +++ b/source/mjpeg_decoder.cc @@ -141,7 +141,8 @@ bool MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) { // next scanline. int databuf_stride = GetComponentStride(i); // Cannot overflow: - // - JPEG width is stored as an u16, so `databuf_stride` (width rounded up to + // - JPEG width is stored as an u16, so `databuf_stride` (width rounded up + // to // DCTSIZE) is at most slightly larger than UINT16_MAX. // - Sampling factor is stored in 4 bits, so scanlines_size is < 16. int databuf_size = scanlines_size * databuf_stride; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 39585e1de..09b9e9e8f 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -192,13 +192,14 @@ void Convert16To8Plane(const uint16_t* src_y, // Convert a plane of 8 bit data to 16 bit LIBYUV_API -void Convert8To16Plane(const uint8_t* src_y, - int src_stride_y, - uint16_t* dst_y, - int dst_stride_y, - int bits, // 10, 12, 16 bits (or 1024, 4096, 65536 scale) - int width, - int height) { +void Convert8To16Plane( + const uint8_t* src_y, + int src_stride_y, + uint16_t* dst_y, + int dst_stride_y, + int bits, // 10, 12, 16 bits (or 1024, 4096, 65536 scale) + int width, + int height) { int y; void (*Convert8To16Row)(const uint8_t* src_y, uint16_t* dst_y, int bits, int width) = Convert8To16Row_C; diff --git a/source/row_any.cc b/source/row_any.cc index c3320a5e1..0a2b652b5 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1826,7 +1826,11 @@ ANY11HS(HalfRow_16To8_Any_SSSE3, HalfRow_16To8_SSSE3, uint16_t, uint8_t, 15) ANY11HS(HalfRow_16To8_Any_AVX2, HalfRow_16To8_AVX2, uint16_t, uint8_t, 31) #endif #ifdef HAS_HALFROW_16TO8_AVX512BW -ANY11HS(HalfRow_16To8_Any_AVX512BW, HalfRow_16To8_AVX512BW, uint16_t, uint8_t, 63) +ANY11HS(HalfRow_16To8_Any_AVX512BW, + HalfRow_16To8_AVX512BW, + uint16_t, + uint8_t, + 63) #endif #ifdef HAS_HALFROW_16TO8_NEON ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15) @@ -1858,16 +1862,32 @@ ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15) } #ifdef HAS_HALFWIDTHROW_16TO8_SSSE3 -ANY11HWS(HalfWidthRow_16To8_Any_SSSE3, HalfWidthRow_16To8_SSSE3, uint16_t, uint8_t, 7) +ANY11HWS(HalfWidthRow_16To8_Any_SSSE3, + HalfWidthRow_16To8_SSSE3, + uint16_t, + uint8_t, + 7) #endif #ifdef HAS_HALFWIDTHROW_16TO8_AVX2 -ANY11HWS(HalfWidthRow_16To8_Any_AVX2, HalfWidthRow_16To8_AVX2, uint16_t, uint8_t, 15) +ANY11HWS(HalfWidthRow_16To8_Any_AVX2, + HalfWidthRow_16To8_AVX2, + uint16_t, + uint8_t, + 15) #endif #ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW -ANY11HWS(HalfWidthRow_16To8_Any_AVX512BW, HalfWidthRow_16To8_AVX512BW, uint16_t, uint8_t, 31) +ANY11HWS(HalfWidthRow_16To8_Any_AVX512BW, + HalfWidthRow_16To8_AVX512BW, + uint16_t, + uint8_t, + 31) #endif #ifdef HAS_HALFWIDTHROW_16TO8_NEON -ANY11HWS(HalfWidthRow_16To8_Any_NEON, HalfWidthRow_16To8_NEON, uint16_t, uint8_t, 7) +ANY11HWS(HalfWidthRow_16To8_Any_NEON, + HalfWidthRow_16To8_NEON, + uint16_t, + uint8_t, + 7) #endif #undef ANY11HWS @@ -2299,10 +2319,7 @@ ANY12M(ARGBToUV444MatrixRow_Any_NEON, ARGBToUV444MatrixRow_NEON, 4, 7) ANY12M(RGBToUV444MatrixRow_Any_NEON, RGBToUV444MatrixRow_NEON, 3, 7) #endif #ifdef HAS_ARGBTOUV444MATRIXROW_NEON_I8MM -ANY12M(ARGBToUV444MatrixRow_Any_NEON_I8MM, - ARGBToUV444MatrixRow_NEON_I8MM, - 4, - 7) +ANY12M(ARGBToUV444MatrixRow_Any_NEON_I8MM, ARGBToUV444MatrixRow_NEON_I8MM, 4, 7) #endif #define ANY11MC(NAMEANY, ANY_SIMD, BPP, MASK) \ diff --git a/source/row_common.cc b/source/row_common.cc index bd0eba368..31cbf8e4d 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -1785,12 +1785,12 @@ MAKEYUVCONSTANTS(V2020, YG, YB, UB, UG, VG, VR) #endif #if defined(__aarch64__) || defined(__arm__) || defined(__riscv) -#define LOAD_YUV_CONSTANTS_AR30 \ - int ub = yuvconstants->kUVCoeff[0]; \ - int vr = yuvconstants->kUVCoeff[1]; \ - int ug = yuvconstants->kUVCoeff[2]; \ - int vg = yuvconstants->kUVCoeff[3]; \ - int yg = yuvconstants->kRGBCoeffBias[0]; \ +#define LOAD_YUV_CONSTANTS_AR30 \ + int ub = yuvconstants->kUVCoeff[0]; \ + int vr = yuvconstants->kUVCoeff[1]; \ + int ug = yuvconstants->kUVCoeff[2]; \ + int vg = yuvconstants->kUVCoeff[3]; \ + int yg = yuvconstants->kRGBCoeffBias[0]; \ int bb = yuvconstants->kRGBCoeffBias[1] + 24; \ int bg = yuvconstants->kRGBCoeffBias[2] - 24; \ int br = yuvconstants->kRGBCoeffBias[3] + 24 @@ -3704,19 +3704,16 @@ void HalfWidthRow_16To8_C(const uint16_t* src_uv, int x; for (x = 0; x < width - 1; x += 2) { dst_uv[0] = STATIC_CAST( - uint8_t, - C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); + uint8_t, C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); dst_uv[1] = STATIC_CAST( - uint8_t, - C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale)); + uint8_t, C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale)); dst_uv += 2; s += 4; t += 4; } if (width & 1) { dst_uv[0] = STATIC_CAST( - uint8_t, - C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); + uint8_t, C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); } } @@ -3731,19 +3728,16 @@ void HalfWidthRow_16To8_Odd_C(const uint16_t* src_uv, width -= 1; for (x = 0; x < width - 1; x += 2) { dst_uv[0] = STATIC_CAST( - uint8_t, - C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); + uint8_t, C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); dst_uv[1] = STATIC_CAST( - uint8_t, - C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale)); + uint8_t, C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale)); dst_uv += 2; s += 4; t += 4; } if (width & 1) { dst_uv[0] = STATIC_CAST( - uint8_t, - C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); + uint8_t, C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale)); dst_uv += 1; s += 2; t += 2; @@ -4280,8 +4274,6 @@ void I422ToARGB4444Row_AVX2(const uint8_t* src_y, } #endif - - #if defined(HAS_I444TOARGBROW_AVX2) && defined(HAS_ARGBTORGB24ROW_AVX2) void I444ToRGB24Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, @@ -4675,8 +4667,7 @@ void RGBToUVMatrixRow_AVX512BW(const uint8_t* src_rgb, } #endif -#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) && \ - defined(HAS_RGB24TOARGBROW_SSSE3) +#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) && defined(HAS_RGB24TOARGBROW_SSSE3) void RGBToUV444MatrixRow_SSSE3(const uint8_t* src_rgb, uint8_t* dst_u, uint8_t* dst_v, @@ -4695,8 +4686,7 @@ void RGBToUV444MatrixRow_SSSE3(const uint8_t* src_rgb, } #endif -#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2) && \ - defined(HAS_RGB24TOARGBROW_AVX2) +#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2) && defined(HAS_RGB24TOARGBROW_AVX2) void RGBToUV444MatrixRow_AVX2(const uint8_t* src_rgb, uint8_t* dst_u, uint8_t* dst_v, diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 0a432ed83..695a5e215 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -50,7 +50,7 @@ static const uvec16 kSub128 = {0x8080u, 0x8080u, 0x8080u, 0x8080u, // defined(HAS_I422TOARGBROW_SSSE3) #if defined(HAS_RGB24TOARGBROW_SSSE3) || defined(HAS_RGB24TOARGBROW_AVX2) || \ - defined(HAS_RGB24TOARGBROW_AVX512BW) || \ + defined(HAS_RGB24TOARGBROW_AVX512BW) || \ defined(HAS_RGBTOYMATRIXROW_AVX512BW) // Shuffle table for converting RGB24 to ARGB. static const uvec8 kShuffleMaskRGB24ToARGB[2] = { @@ -101,33 +101,33 @@ static const uint32_t kPermdRGB24_2_AVX512BW[16] = { #endif // defined(HAS_RAWTORGB24ROW_AVX512BW) #if defined(HAS_RAWTORGB24ROW_AVX512VBMI) static const uint8_t kPermRAWToRGB24_0[64] = { - 0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a, 0x09, - 0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17, 0x16, 0x15, - 0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e, 0x23, 0x22, 0x21, - 0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b, 0x2a, 0x2f, 0x2e, 0x2d, - 0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38, 0x37, 0x36, 0x3b, 0x3a, 0x39, - 0x3e, 0x3d, 0x3c, 0x41}; + 0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a, + 0x09, 0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17, + 0x16, 0x15, 0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e, + 0x23, 0x22, 0x21, 0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b, + 0x2a, 0x2f, 0x2e, 0x2d, 0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38, + 0x37, 0x36, 0x3b, 0x3a, 0x39, 0x3e, 0x3d, 0x3c, 0x41}; static const uint8_t kPermRAWToRGB24_1[64] = { - 0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08, 0x0d, - 0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15, 0x14, 0x19, - 0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22, 0x21, 0x20, 0x25, - 0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29, 0x2e, 0x2d, 0x2c, 0x31, - 0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36, 0x35, 0x3a, 0x39, 0x38, 0x3d, - 0x3c, 0x3b, 0x00, 0x3f}; + 0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08, + 0x0d, 0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15, + 0x14, 0x19, 0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22, + 0x21, 0x20, 0x25, 0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29, + 0x2e, 0x2d, 0x2c, 0x31, 0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36, + 0x35, 0x3a, 0x39, 0x38, 0x3d, 0x3c, 0x3b, 0x00, 0x3f}; static const uint8_t kPermRAWToRGB24_2[64] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x40, 0x3f}; + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, + 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x40, 0x3f}; static const uint8_t kPermRAWToRGB24_3[64] = { - 0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c, 0x4b, - 0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53, 0x58, 0x57, - 0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60, 0x5f, 0x64, 0x63, - 0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d, 0x6c, 0x6b, 0x70, 0x6f, - 0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74, 0x79, 0x78, 0x77, 0x7c, 0x7b, - 0x7a, 0x7f, 0x7e, 0x7d}; + 0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c, + 0x4b, 0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53, + 0x58, 0x57, 0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60, + 0x5f, 0x64, 0x63, 0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d, + 0x6c, 0x6b, 0x70, 0x6f, 0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74, + 0x79, 0x78, 0x77, 0x7c, 0x7b, 0x7a, 0x7f, 0x7e, 0x7d}; #endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI) #if defined(HAS_RAWTORGBAROW_SSSE3) @@ -338,11 +338,11 @@ void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24, "sub $0x20,%2 \n" "jg 1b \n" "vzeroupper \n" - : "+r"(src_rgb24), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "m"(kShuffleMaskRGB24ToARGB[0]), // %3 - "m"(kShuffleMaskRGB24ToARGB[1]) // %4 + : "+r"(src_rgb24), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "m"(kShuffleMaskRGB24ToARGB[0]), // %3 + "m"(kShuffleMaskRGB24ToARGB[1]) // %4 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } #endif // HAS_RGB24TOARGBROW_AVX2 @@ -430,18 +430,15 @@ void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) { } #endif // HAS_RAWTOARGBROW_AVX2 -#if defined(HAS_RAWTOARGBROW_AVX512BW) || \ +#if defined(HAS_RAWTOARGBROW_AVX512BW) || \ defined(HAS_RGB24TOARGBROW_AVX512BW) || \ defined(HAS_RAWTORGB24ROW_AVX512BW) || \ defined(HAS_RGBTOYMATRIXROW_AVX512BW) -static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, - 3, 4, 4, 5}; -static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8, - 9, 10, 10, 11}; -static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6, - 7, 8, 8, 9}; -static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, - 5, 6, 6, 7}; +static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, 3, 4, 4, 5}; +static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8, + 9, 10, 10, 11}; +static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6, 7, 8, 8, 9}; +static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, 5, 6, 6, 7}; #endif // defined(HAS_RAWTOARGBROW_AVX512BW) || // defined(HAS_RGB24TOARGBROW_AVX512BW) || // defined(HAS_RAWTORGB24ROW_AVX512BW) || @@ -490,15 +487,15 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, "lea 0x100(%1),%1 \n" "sub $0x40,%2 \n" "jg 1b \n" - "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "m"(*shuffler), // %3 - "m"(kPermqRGB24ToARGB0_AVX512BW), // %4 - "m"(kPermqRGB24ToARGB1_AVX512BW), // %5 - "m"(kPermqRGB24ToARGB2_AVX512BW), // %6 - "m"(kPermqRGB24ToARGB3_AVX512BW) // %7 + "vzeroupper \n" + : "+r"(src_raw), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "m"(*shuffler), // %3 + "m"(kPermqRGB24ToARGB0_AVX512BW), // %4 + "m"(kPermqRGB24ToARGB1_AVX512BW), // %5 + "m"(kPermqRGB24ToARGB2_AVX512BW), // %6 + "m"(kPermqRGB24ToARGB3_AVX512BW) // %7 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23"); } @@ -595,9 +592,7 @@ void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, #endif // HAS_RAWTORGB24ROW_SSSE3 #ifdef HAS_RAWTORGB24ROW_AVX2 -void RAWToRGB24Row_AVX2(const uint8_t* src_raw, - uint8_t* dst_rgb24, - int width) { +void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { asm volatile( "vmovdqu %3,%%ymm7 \n" // shuffler (32 bytes) "vmovdqu %4,%%ymm8 \n" // kPermd0_AVX2 @@ -607,7 +602,7 @@ void RAWToRGB24Row_AVX2(const uint8_t* src_raw, "vmovdqu %8,%%ymm12 \n" // kPermd4_AVX2 LABELALIGN - "1: \n" + "1: \n" "vmovdqu (%0),%%ymm0 \n" // raw0 "vmovdqu 32(%0),%%ymm1 \n" // raw1 "vmovdqu 64(%0),%%ymm2 \n" // raw2 @@ -646,16 +641,16 @@ void RAWToRGB24Row_AVX2(const uint8_t* src_raw, "lea 96(%1),%1 \n" "sub $32,%2 \n" "jg 1b \n" - "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_rgb24), // %1 - "+r"(width) // %2 - : "m"(kShuffleMaskRAWToRGB24_AVX2[0]), // %3 - "m"(kPermd0_AVX2), // %4 - "m"(kPermd1_AVX2), // %5 - "m"(kPermd2_AVX2), // %6 - "m"(kPermd3_AVX2), // %7 - "m"(kPermd4_AVX2) // %8 + "vzeroupper \n" + : "+r"(src_raw), // %0 + "+r"(dst_rgb24), // %1 + "+r"(width) // %2 + : "m"(kShuffleMaskRAWToRGB24_AVX2[0]), // %3 + "m"(kPermd0_AVX2), // %4 + "m"(kPermd1_AVX2), // %5 + "m"(kPermd2_AVX2), // %6 + "m"(kPermd3_AVX2), // %7 + "m"(kPermd4_AVX2) // %8 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12"); } @@ -676,7 +671,7 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, "vmovdqu32 %10,%%zmm26 \n" // kPermdRGB24_2_AVX512BW LABELALIGN - "1: \n" + "1: \n" "vmovdqu8 (%0),%%zmm0 \n" "vmovdqu8 64(%0),%%zmm1 \n" "vmovdqu8 128(%0),%%zmm2 \n" @@ -703,21 +698,21 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, "lea 0xc0(%1),%1 \n" "sub $0x40,%2 \n" "jg 1b \n" - "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_rgb24), // %1 - "+r"(width) // %2 - : "m"(kPermqRGB24ToARGB0_AVX512BW), // %3 - "m"(kPermqRGB24ToARGB1_AVX512BW), // %4 - "m"(kPermqRGB24ToARGB2_AVX512BW), // %5 - "m"(kPermqRGB24ToARGB3_AVX512BW), // %6 - "m"(kShuffleMaskRAWToRGB24_AVX2[0]), // %7 - "m"(kPermdRGB24_0_AVX512BW), // %8 - "m"(kPermdRGB24_1_AVX512BW), // %9 - "m"(kPermdRGB24_2_AVX512BW) // %10 + "vzeroupper \n" + : "+r"(src_raw), // %0 + "+r"(dst_rgb24), // %1 + "+r"(width) // %2 + : "m"(kPermqRGB24ToARGB0_AVX512BW), // %3 + "m"(kPermqRGB24ToARGB1_AVX512BW), // %4 + "m"(kPermqRGB24ToARGB2_AVX512BW), // %5 + "m"(kPermqRGB24ToARGB3_AVX512BW), // %6 + "m"(kShuffleMaskRAWToRGB24_AVX2[0]), // %7 + "m"(kPermdRGB24_0_AVX512BW), // %8 + "m"(kPermdRGB24_1_AVX512BW), // %9 + "m"(kPermdRGB24_2_AVX512BW) // %10 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", - "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", - "xmm24", "xmm25", "xmm26"); + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", + "xmm25", "xmm26"); } #endif // HAS_RAWTORGB24ROW_AVX512BW @@ -732,7 +727,7 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, "vmovdqu32 %[kPermRAWToRGB24_3],%%zmm23 \n" LABELALIGN - "1: \n" + "1: \n" "vmovdqu8 (%0),%%zmm0 \n" "vmovdqu8 64(%0),%%zmm1 \n" "vmovdqu8 128(%0),%%zmm2 \n" @@ -748,10 +743,10 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, "lea 0xc0(%1),%1 \n" "sub $0x40,%2 \n" "jg 1b \n" - "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_rgb24), // %1 - "+r"(width) // %2 + "vzeroupper \n" + : "+r"(src_raw), // %0 + "+r"(dst_rgb24), // %1 + "+r"(width) // %2 : [kPermRAWToRGB24_0] "m"(kPermRAWToRGB24_0), [kPermRAWToRGB24_1] "m"(kPermRAWToRGB24_1), [kPermRAWToRGB24_2] "m"(kPermRAWToRGB24_2), @@ -1198,9 +1193,9 @@ void ARGBToRGB565Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { "sub $0x8,%2 \n" "jg 1b \n" "vzeroupper \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 : : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); } @@ -1238,9 +1233,9 @@ void ARGBToARGB1555Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { "sub $0x8,%2 \n" "jg 1b \n" "vzeroupper \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 : : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); @@ -1278,9 +1273,9 @@ void ARGBToARGB4444Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { "sub $0x8,%2 \n" "jg 1b \n" "vzeroupper \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 : : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); @@ -1747,7 +1742,7 @@ void AB64ToARGBRow_AVX2(const uint16_t* src_ab64, // clang-format on -#if defined(HAS_ARGBTOYMATRIXROW_AVX2) || \ +#if defined(HAS_ARGBTOYMATRIXROW_AVX2) || \ defined(HAS_ARGBTOUV444MATRIXROW_AVX2) || \ defined(HAS_ARGBEXTRACTALPHAROW_AVX2) // vpermd for vphaddw + vpackuswb vpermd. @@ -1806,12 +1801,12 @@ void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBTOYMATRIXROW_AVX2 -#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \ +#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \ defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \ - defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \ defined(HAS_RGBTOYMATRIXROW_AVX512BW) static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15}; + 2, 6, 10, 14, 3, 7, 11, 15}; #endif // defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || // defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || // defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || @@ -1838,8 +1833,7 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" "vpackssdw %%zmm18,%%zmm18,%%zmm18 \n" "vpsubw %%zmm18,%%zmm17,%%zmm17 \n" - "vmovups %4,%%zmm18 \n" - LABELALIGN + "vmovups %4,%%zmm18 \n" LABELALIGN "1: \n" "vmovups (%0),%%zmm0 \n" "vmovups 0x40(%0),%%zmm1 \n" @@ -1876,8 +1870,8 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, "+r"(width) // %2 : "r"(c), // %3 "m"(kPermdARGBToY_AVX512BW) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm16", "xmm17", "xmm18"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18"); } #endif // HAS_ARGBTOYMATRIXROW_AVX512BW @@ -1902,7 +1896,8 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" "vpackssdw %%zmm18,%%zmm18,%%zmm18 \n" "vpsubw %%zmm18,%%zmm17,%%zmm17 \n" // zmm17 = adjusted bias - "vmovups %4,%%zmm25 \n" // zmm25 = kPermdARGBToY_AVX512BW + "vmovups %4,%%zmm25 \n" // zmm25 = + // kPermdARGBToY_AVX512BW LABELALIGN "1: \n" "vmovdqu8 (%0),%%zmm0 \n" @@ -1945,20 +1940,20 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, "lea 0x40(%1),%1 \n" "sub $0x40,%2 \n" "jg 1b \n" - "vzeroupper \n" - : "+r"(src_rgb), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(c), // %3 - "m"(kPermdARGBToY_AVX512BW), // %4 - "m"(kPermqRGB24ToARGB0_AVX512BW), // %5 - "m"(kPermqRGB24ToARGB1_AVX512BW), // %6 - "m"(kPermqRGB24ToARGB2_AVX512BW), // %7 - "m"(kPermqRGB24ToARGB3_AVX512BW), // %8 - "m"(kShuffleMaskRGB24ToARGB) // %9 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", - "xmm23", "xmm24", "xmm25"); + "vzeroupper \n" + : "+r"(src_rgb), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(c), // %3 + "m"(kPermdARGBToY_AVX512BW), // %4 + "m"(kPermqRGB24ToARGB0_AVX512BW), // %5 + "m"(kPermqRGB24ToARGB1_AVX512BW), // %6 + "m"(kPermqRGB24ToARGB2_AVX512BW), // %7 + "m"(kPermqRGB24ToARGB3_AVX512BW), // %8 + "m"(kShuffleMaskRGB24ToARGB) // %9 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", + "xmm25"); } #endif // HAS_RGBTOYMATRIXROW_AVX512BW @@ -2179,8 +2174,8 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, #endif : "r"(c), // %4 "m"(kPermdARGBToY_AVX512BW) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm16", "xmm17", "xmm18"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18"); } #endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW @@ -2425,8 +2420,8 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, "m"(kShuffleAARRGGBB), // %6 "m"(kPermdARGBToY_AVX512BW), // %7 "m"(kPermdARGBToUV_AVX512BW) // %8 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16", + "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); } #endif // HAS_ARGBTOUVMATRIXROW_AVX512BW @@ -2639,11 +2634,11 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, "movdqa 96(%[yuvconstants]),%%xmm11 \n" \ "movdqa 128(%[yuvconstants]),%%xmm12 \n" -#define YUVTORGB_SETUP_AR30(yuvconstants) \ - YUVTORGB_SETUP(yuvconstants) \ - "pcmpeqb %%xmm0,%%xmm0 \n" \ - "psrlw $14,%%xmm0 \n" \ - "psllw $3,%%xmm0 \n" \ +#define YUVTORGB_SETUP_AR30(yuvconstants) \ + YUVTORGB_SETUP(yuvconstants) \ + "pcmpeqb %%xmm0,%%xmm0 \n" \ + "psrlw $14,%%xmm0 \n" \ + "psllw $3,%%xmm0 \n" \ "psubw %%xmm0,%%xmm12 \n" // Convert 8 pixels: 8 UV and 8 Y @@ -3738,11 +3733,11 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vmovups (%[dquadsplitperm]),%%zmm17 \n" \ "vmovups (%[unperm]),%%zmm18 \n" -#define YUVTORGB_SETUP_AR30_AVX2(yuvconstants) \ - YUVTORGB_SETUP_AVX2(yuvconstants) \ - "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" \ - "vpsrlw $14,%%ymm0,%%ymm0 \n" \ - "vpsllw $3,%%ymm0,%%ymm0 \n" \ +#define YUVTORGB_SETUP_AR30_AVX2(yuvconstants) \ + YUVTORGB_SETUP_AVX2(yuvconstants) \ + "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" \ + "vpsrlw $14,%%ymm0,%%ymm0 \n" \ + "vpsllw $3,%%ymm0,%%ymm0 \n" \ "vpsubw %%ymm0,%%ymm12,%%ymm12 \n" #define YUVTORGB_SETUP_AR30_AVX512BW(yuvconstants) \ @@ -3847,7 +3842,7 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "lea 0x80(%[dst_argb]), %[dst_argb] \n" // Store 16 RGB24 values. -#define STORERGB24_AVX2 \ +#define STORERGB24_AVX2 \ "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \ "vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n" \ "vmovdqa %%ymm0,%%ymm1 \n" \ @@ -4051,7 +4046,8 @@ void OMITFP I422ToARGBRow_AVX2(const uint8_t* y_buf, #if defined(HAS_I422TORGB24ROW_AVX2) // 16 pixels -// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 RGB24 (48 bytes). +// 8 UV values upsampled to 16 UV, mixed with 16 Y producing 16 RGB24 (48 +// bytes). void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -4090,12 +4086,12 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf, } #endif // defined(HAS_I422TORGB24ROW_AVX2) -#if defined(HAS_I422TOARGBROW_AVX512BW) || \ - defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ - defined(HAS_I422TORGB24ROW_AVX512BW) || \ +#if defined(HAS_I422TOARGBROW_AVX512BW) || \ + defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I422TORGB24ROW_AVX512BW) || \ defined(HAS_I422TOAR30ROW_AVX512BW) static const uint16_t kSplitQuadWords[32] = { - 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15}; static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; static const uint64_t kUnpermuteAVX512[8] = {0, 4, 1, 5, 2, 6, 3, 7}; @@ -4106,7 +4102,7 @@ static const uint64_t kUnpermuteAVX512[8] = {0, 4, 1, 5, 2, 6, 3, 7}; #if defined(HAS_I422TOAR30ROW_AVX512BW) static const uint16_t kPermAR30_0[32] = { - 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47}; static const uint16_t kPermAR30_1[32] = { 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, @@ -4154,29 +4150,30 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf, #if defined(HAS_I422TORGB24ROW_AVX512VBMI) static const uint8_t kMaskBG[64] = { - 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, 0x45, - 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, 0x13, 0x53, - 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, 0x60, 0x21, 0x61, - 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, 0x26, 0x66, 0x27, 0x67, - 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, 0x73, 0x34, 0x74, 0x35, 0x75, - 0x36, 0x76, 0x37, 0x77}; + 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, + 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, + 0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, + 0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, + 0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, + 0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77}; static const uint8_t kMaskDST0[64] = { - 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, 0x43, - 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, 0x0f, 0x47, - 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, 0x16, 0x17, 0x53, - 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, 0x56, 0x1e, 0x1f, 0x57, - 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, 0x25, 0x62, 0x26, 0x27, 0x63, - 0x28, 0x29, 0x64, 0x2a}; + 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, + 0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, + 0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, + 0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, + 0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, + 0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a}; static const uint8_t kMaskDST1[64] = { - 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, 0x32, - 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, 0x74, 0x3a, - 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}; + 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, + 0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, + 0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; // 32 pixels -// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96 bytes). +// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96 +// bytes). void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -5903,26 +5900,25 @@ void MultiplyRow_16_AVX512BW(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { - asm volatile( - "vpbroadcastw %3,%%zmm2 \n" + asm volatile("vpbroadcastw %3,%%zmm2 \n" - // 64 pixels per loop. - LABELALIGN - "1: \n" - "vpmullw (%0),%%zmm2,%%zmm0 \n" - "vpmullw 0x40(%0),%%zmm2,%%zmm1 \n" - "add $0x80,%0 \n" - "vmovdqu16 %%zmm0,(%1) \n" - "vmovdqu16 %%zmm1,0x40(%1) \n" - "add $0x80,%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_y), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(scale) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm2"); + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vpmullw (%0),%%zmm2,%%zmm0 \n" + "vpmullw 0x40(%0),%%zmm2,%%zmm1 \n" + "add $0x80,%0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "add $0x80,%1 \n" + "sub $0x40,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(scale) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm2"); } #endif // HAS_MULTIPLYROW_16_AVX512BW @@ -6144,14 +6140,14 @@ void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv, "add $0x08,%1 \n" "sub $0x08,%2 \n" "jg 1b \n" - : "+r"(src_uv), // %0 - "+r"(dst_uv), // %1 - "+r"(width) // %2 - : "r"(src_uv_stride), // %3 - "rm"(scale), // %4 + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(width) // %2 + : "r"(src_uv_stride), // %3 + "rm"(scale), // %4 "m"(kShuffleMaskHalfWidth16To8) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6", "xmm7"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); } #endif // HAS_HALFWIDTHROW_16TO8_SSSE3 @@ -6237,8 +6233,7 @@ void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, "+r"(width) // %2 : "r"(src_uv_stride), // %3 "rm"(scale) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } #endif // HAS_HALFWIDTHROW_16TO8_AVX2 @@ -6250,32 +6245,31 @@ void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, int width) { const uint16_t* src_uv1 = src_uv + src_uv_stride; const int shift = 16 - __builtin_ctz((uint32_t)scale); - asm volatile( - "vpbroadcastw %4,%%zmm2 \n" + asm volatile("vpbroadcastw %4,%%zmm2 \n" - // 64 pixels per loop. - LABELALIGN - "1: \n" - "vmovdqu64 (%0),%%zmm0 \n" - "vmovdqu64 0x40(%0),%%zmm1 \n" - "vpavgw (%1),%%zmm0,%%zmm0 \n" - "vpavgw 0x40(%1),%%zmm1,%%zmm1 \n" - "add $0x80,%0 \n" - "add $0x80,%1 \n" - "vpsrlvw %%zmm2,%%zmm0,%%zmm0 \n" - "vpsrlvw %%zmm2,%%zmm1,%%zmm1 \n" - "vpmovuswb %%zmm0,(%2) \n" - "vpmovuswb %%zmm1,0x20(%2) \n" - "add $0x40,%2 \n" - "sub $0x40,%3 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_uv), // %0 - "+r"(src_uv1), // %1 - "+r"(dst_uv), // %2 - "+r"(width) // %3 - : "rm"(shift) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2"); + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vmovdqu64 (%0),%%zmm0 \n" + "vmovdqu64 0x40(%0),%%zmm1 \n" + "vpavgw (%1),%%zmm0,%%zmm0 \n" + "vpavgw 0x40(%1),%%zmm1,%%zmm1 \n" + "add $0x80,%0 \n" + "add $0x80,%1 \n" + "vpsrlvw %%zmm2,%%zmm0,%%zmm0 \n" + "vpsrlvw %%zmm2,%%zmm1,%%zmm1 \n" + "vpmovuswb %%zmm0,(%2) \n" + "vpmovuswb %%zmm1,0x20(%2) \n" + "add $0x40,%2 \n" + "sub $0x40,%3 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(src_uv1), // %1 + "+r"(dst_uv), // %2 + "+r"(width) // %3 + : "rm"(shift) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2"); } #endif // HAS_HALFROW_16TO8_AVX512BW @@ -6325,8 +6319,7 @@ void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, "+r"(dst_uv), // %2 "+r"(width) // %3 : "rm"(scale) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } #endif // HAS_HALFWIDTHROW_16TO8_AVX512BW @@ -6340,29 +6333,28 @@ void Convert8To16Row_SSE2(const uint8_t* src_y, int bits, int width) { const int shift = 16 - bits; - asm volatile( - "movd %3,%%xmm2 \n" + asm volatile("movd %3,%%xmm2 \n" - // 16 pixels per loop. - LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqa %%xmm0,%%xmm1 \n" - "punpcklbw %%xmm0,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm1 \n" - "add $0x10,%0 \n" - "psrlw %%xmm2,%%xmm0 \n" - "psrlw %%xmm2,%%xmm1 \n" - "movdqu %%xmm0,(%1) \n" - "movdqu %%xmm1,0x10(%1) \n" - "add $0x20,%1 \n" - "sub $0x10,%2 \n" - "jg 1b \n" - : "+r"(src_y), // %0 - "+r"(dst_y), // %1 - "+r"(width) // %2 - : "r"(shift) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm2"); + // 16 pixels per loop. + LABELALIGN + "1: \n" + "movdqu (%0),%%xmm0 \n" + "movdqa %%xmm0,%%xmm1 \n" + "punpcklbw %%xmm0,%%xmm0 \n" + "punpckhbw %%xmm1,%%xmm1 \n" + "add $0x10,%0 \n" + "psrlw %%xmm2,%%xmm0 \n" + "psrlw %%xmm2,%%xmm1 \n" + "movdqu %%xmm0,(%1) \n" + "movdqu %%xmm1,0x10(%1) \n" + "add $0x20,%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(shift) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm2"); } #ifdef HAS_CONVERT8TO16ROW_AVX2 @@ -6405,24 +6397,24 @@ void Convert8To16Row_AVX512BW(const uint8_t* src_y, const int shift = 16 - bits; asm volatile( "vpbroadcastw %4,%%zmm2 \n" - "vmovd %3,%%xmm3 \n" + "vmovd %3,%%xmm3 \n" // 64 pixels per loop. LABELALIGN "1: \n" - "vpmovzxbw (%0),%%zmm0 \n" - "vpmovzxbw 0x20(%0),%%zmm1 \n" - "add $0x40,%0 \n" - "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" - "vpmullw %%zmm2,%%zmm1,%%zmm1 \n" - "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n" - "vpsrlw %%xmm3,%%zmm1,%%zmm1 \n" - "vmovdqu16 %%zmm0,(%1) \n" - "vmovdqu16 %%zmm1,0x40(%1) \n" - "add $0x80,%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" - "vzeroupper \n" + "vpmovzxbw (%0),%%zmm0 \n" + "vpmovzxbw 0x20(%0),%%zmm1 \n" + "add $0x40,%0 \n" + "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmullw %%zmm2,%%zmm1,%%zmm1 \n" + "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n" + "vpsrlw %%xmm3,%%zmm1,%%zmm1 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "add $0x80,%1 \n" + "sub $0x40,%2 \n" + "jg 1b \n" + "vzeroupper \n" : "+r"(src_y), // %0 "+r"(dst_y), // %1 "+r"(width) // %2 @@ -7206,8 +7198,7 @@ void MergeXR30Row_AVX2(const uint16_t* src_r, #else : "rm"(shift) // %5 #endif - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } #endif // HAS_MERGEXR30ROW_AVX2 @@ -8482,13 +8473,13 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0, "add $64,%2 \n" "sub $64,%4 \n" "jg 1b \n" - "vzeroupper \n" + "vzeroupper \n" : "+r"(src0), // %0 "+r"(src1), // %1 "+r"(alpha), // %2 "+r"(dst), // %3 "+rm"(width) // %4 - ::"memory", + ::"memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", "xmm17", "xmm18"); } @@ -8990,42 +8981,42 @@ void ARGBMultiplyRow_SSE2(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, int width) { - asm volatile("pxor %%xmm5,%%xmm5 \n" - "pcmpeqw %%xmm4,%%xmm4 \n" - "psrlw $0xf,%%xmm4 \n" - "psllw $0x7,%%xmm4 \n" + asm volatile( + "pxor %%xmm5,%%xmm5 \n" + "pcmpeqw %%xmm4,%%xmm4 \n" + "psrlw $0xf,%%xmm4 \n" + "psllw $0x7,%%xmm4 \n" - // 4 pixel loop. - LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "lea 0x10(%0),%0 \n" - "movdqu (%1),%%xmm2 \n" - "lea 0x10(%1),%1 \n" - "movdqa %%xmm0,%%xmm1 \n" - "movdqa %%xmm2,%%xmm3 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "punpckhbw %%xmm5,%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm2 \n" - "punpckhbw %%xmm5,%%xmm3 \n" - "pmullw %%xmm2,%%xmm0 \n" - "pmullw %%xmm3,%%xmm1 \n" - "paddw %%xmm4,%%xmm0 \n" - "paddw %%xmm4,%%xmm1 \n" - "psrlw $0x8,%%xmm0 \n" - "psrlw $0x8,%%xmm1 \n" - "packuswb %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" - "jg 1b \n" - : "+r"(src_argb), // %0 - "+r"(src_argb1), // %1 - "+r"(dst_argb), // %2 - "+r"(width) // %3 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", - "xmm5"); + // 4 pixel loop. + LABELALIGN + "1: \n" + "movdqu (%0),%%xmm0 \n" + "lea 0x10(%0),%0 \n" + "movdqu (%1),%%xmm2 \n" + "lea 0x10(%1),%1 \n" + "movdqa %%xmm0,%%xmm1 \n" + "movdqa %%xmm2,%%xmm3 \n" + "punpcklbw %%xmm5,%%xmm0 \n" + "punpckhbw %%xmm5,%%xmm1 \n" + "punpcklbw %%xmm5,%%xmm2 \n" + "punpckhbw %%xmm5,%%xmm3 \n" + "pmullw %%xmm2,%%xmm0 \n" + "pmullw %%xmm3,%%xmm1 \n" + "paddw %%xmm4,%%xmm0 \n" + "paddw %%xmm4,%%xmm1 \n" + "psrlw $0x8,%%xmm0 \n" + "psrlw $0x8,%%xmm1 \n" + "packuswb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0,(%2) \n" + "lea 0x10(%2),%2 \n" + "sub $0x4,%3 \n" + "jg 1b \n" + : "+r"(src_argb), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); } #endif // HAS_ARGBMULTIPLYROW_SSE2 @@ -9343,26 +9334,25 @@ void SobelToPlaneRow_SSE2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, int width) { - asm volatile( - "sub %0,%1 \n" + asm volatile("sub %0,%1 \n" - // 8 pixel loop. - LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x00(%0,%1,1),%%xmm1 \n" - "lea 0x10(%0),%0 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x10,%3 \n" - "jg 1b \n" - : "+r"(src_sobelx), // %0 - "+r"(src_sobely), // %1 - "+r"(dst_y), // %2 - "+r"(width) // %3 - : - : "memory", "cc", "xmm0", "xmm1"); + // 8 pixel loop. + LABELALIGN + "1: \n" + "movdqu (%0),%%xmm0 \n" + "movdqu 0x00(%0,%1,1),%%xmm1 \n" + "lea 0x10(%0),%0 \n" + "paddusb %%xmm1,%%xmm0 \n" + "movdqu %%xmm0,(%2) \n" + "lea 0x10(%2),%2 \n" + "sub $0x10,%3 \n" + "jg 1b \n" + : "+r"(src_sobelx), // %0 + "+r"(src_sobely), // %1 + "+r"(dst_y), // %2 + "+r"(width) // %3 + : + : "memory", "cc", "xmm0", "xmm1"); } #endif // HAS_SOBELTOPLANEROW_SSE2 @@ -9630,7 +9620,7 @@ void CumulativeSumToAverageRow_SSE2(const int32_t* topleft, #endif // HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 #ifdef HAS_COMPUTECUMULATIVESUMROW_AVX2 -static const uvec8 kCumSumTranspose = {0, 4, 8, 12, 1, 5, 9, 13, +static const uvec8 kCumSumTranspose = {0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15}; static const vec8 kCumSumMaddLo[2] = { {1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0}, @@ -9644,57 +9634,57 @@ void ComputeCumulativeSumRow_AVX2(const uint8_t* row, const int32_t* previous_cumsum, int width) { asm volatile( - "vpxor %%xmm0,%%xmm0,%%xmm0 \n" + "vpxor %%xmm0,%%xmm0,%%xmm0 \n" "vbroadcasti128 %4,%%ymm5 \n" - "vmovdqu %5,%%ymm6 \n" - "vmovdqu %6,%%ymm7 \n" - "vpcmpeqd %%ymm8,%%ymm8,%%ymm8 \n" - "vpsrlw $0xf,%%ymm8,%%ymm8 \n" - "sub $0x4,%3 \n" - "jl 49f \n" + "vmovdqu %5,%%ymm6 \n" + "vmovdqu %6,%%ymm7 \n" + "vpcmpeqd %%ymm8,%%ymm8,%%ymm8 \n" + "vpsrlw $0xf,%%ymm8,%%ymm8 \n" + "sub $0x4,%3 \n" + "jl 49f \n" // 4 pixel loop. LABELALIGN - "40: \n" + "40: \n" "vbroadcasti128 (%0),%%ymm2 \n" - "lea 0x10(%0),%0 \n" - "vpshufb %%ymm5,%%ymm2,%%ymm2 \n" - "vpmaddubsw %%ymm6,%%ymm2,%%ymm3 \n" - "vpmaddubsw %%ymm7,%%ymm2,%%ymm4 \n" - "vpmaddwd %%ymm8,%%ymm3,%%ymm3 \n" - "vpmaddwd %%ymm8,%%ymm4,%%ymm4 \n" - "vperm2i128 $0x11,%%ymm4,%%ymm4,%%ymm2 \n" - "vpaddd (%2),%%ymm3,%%ymm3 \n" - "vpaddd 0x20(%2),%%ymm4,%%ymm4 \n" - "lea 0x40(%2),%2 \n" - "vpaddd %%ymm0,%%ymm3,%%ymm3 \n" - "vpaddd %%ymm0,%%ymm4,%%ymm4 \n" - "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" - "vmovdqu %%ymm3,(%1) \n" - "vmovdqu %%ymm4,0x20(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x4,%3 \n" - "jge 40b \n" + "lea 0x10(%0),%0 \n" + "vpshufb %%ymm5,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm6,%%ymm2,%%ymm3 \n" + "vpmaddubsw %%ymm7,%%ymm2,%%ymm4 \n" + "vpmaddwd %%ymm8,%%ymm3,%%ymm3 \n" + "vpmaddwd %%ymm8,%%ymm4,%%ymm4 \n" + "vperm2i128 $0x11,%%ymm4,%%ymm4,%%ymm2 \n" + "vpaddd (%2),%%ymm3,%%ymm3 \n" + "vpaddd 0x20(%2),%%ymm4,%%ymm4 \n" + "lea 0x40(%2),%2 \n" + "vpaddd %%ymm0,%%ymm3,%%ymm3 \n" + "vpaddd %%ymm0,%%ymm4,%%ymm4 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm3,(%1) \n" + "vmovdqu %%ymm4,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x4,%3 \n" + "jge 40b \n" - "49: \n" - "add $0x3,%3 \n" - "jl 19f \n" + "49: \n" + "add $0x3,%3 \n" + "jl 19f \n" // 1 pixel loop. LABELALIGN - "10: \n" - "vpmovzxbd (%0),%%xmm2 \n" - "lea 0x4(%0),%0 \n" - "vpaddd %%xmm2,%%xmm0,%%xmm0 \n" - "vpaddd (%2),%%xmm0,%%xmm2 \n" - "lea 0x10(%2),%2 \n" - "vmovdqu %%xmm2,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x1,%3 \n" - "jge 10b \n" + "10: \n" + "vpmovzxbd (%0),%%xmm2 \n" + "lea 0x4(%0),%0 \n" + "vpaddd %%xmm2,%%xmm0,%%xmm0 \n" + "vpaddd (%2),%%xmm0,%%xmm2 \n" + "lea 0x10(%2),%2 \n" + "vmovdqu %%xmm2,(%1) \n" + "lea 0x10(%1),%1 \n" + "sub $0x1,%3 \n" + "jge 10b \n" - "19: \n" - "vzeroupper \n" + "19: \n" + "vzeroupper \n" : "+r"(row), // %0 "+r"(cumsum), // %1 "+r"(previous_cumsum), // %2 @@ -9715,103 +9705,103 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, uint8_t* dst, int count) { asm volatile( - "vpxor %%xmm5,%%xmm5,%%xmm5 \n" - "vcvtsi2ssl %5,%%xmm5,%%xmm5 \n" - "vrcpss %%xmm5,%%xmm5,%%xmm4 \n" - "vpshufd $0x0,%%xmm4,%%xmm4 \n" - "sub $0x4,%3 \n" - "jl 19f \n" - "vinserti128 $0x1,%%xmm4,%%ymm4,%%ymm4 \n" - "cmpl $0x80,%5 \n" - "ja 10f \n" + "vpxor %%xmm5,%%xmm5,%%xmm5 \n" + "vcvtsi2ssl %5,%%xmm5,%%xmm5 \n" + "vrcpss %%xmm5,%%xmm5,%%xmm4 \n" + "vpshufd $0x0,%%xmm4,%%xmm4 \n" + "sub $0x4,%3 \n" + "jl 19f \n" + "vinserti128 $0x1,%%xmm4,%%ymm4,%%ymm4 \n" + "cmpl $0x80,%5 \n" + "ja 10f \n" - "vpcmpeqb %%xmm6,%%xmm6,%%xmm6 \n" - "vpsrld $0x10,%%xmm6,%%xmm6 \n" - "vcvtdq2ps %%xmm6,%%xmm6 \n" - "vaddss %%xmm6,%%xmm5,%%xmm6 \n" - "vmulss %%xmm4,%%xmm6,%%xmm6 \n" - "vcvtps2dq %%xmm6,%%xmm6 \n" - "vpackssdw %%xmm6,%%xmm6,%%xmm6 \n" - "vpbroadcastw %%xmm6,%%ymm6 \n" + "vpcmpeqb %%xmm6,%%xmm6,%%xmm6 \n" + "vpsrld $0x10,%%xmm6,%%xmm6 \n" + "vcvtdq2ps %%xmm6,%%xmm6 \n" + "vaddss %%xmm6,%%xmm5,%%xmm6 \n" + "vmulss %%xmm4,%%xmm6,%%xmm6 \n" + "vcvtps2dq %%xmm6,%%xmm6 \n" + "vpackssdw %%xmm6,%%xmm6,%%xmm6 \n" + "vpbroadcastw %%xmm6,%%ymm6 \n" // 4 pixel small loop. LABELALIGN - "4: \n" - "vmovdqu (%0),%%ymm0 \n" - "vmovdqu 0x20(%0),%%ymm1 \n" - "vpsubd 0x00(%0,%4,4),%%ymm0,%%ymm0\n" - "vpsubd 0x20(%0,%4,4),%%ymm1,%%ymm1\n" - "lea 0x40(%0),%0 \n" - "vpsubd (%1),%%ymm0,%%ymm0 \n" - "vpsubd 0x20(%1),%%ymm1,%%ymm1 \n" - "vpaddd 0x00(%1,%4,4),%%ymm0,%%ymm0\n" - "vpaddd 0x20(%1,%4,4),%%ymm1,%%ymm1\n" - "lea 0x40(%1),%1 \n" - "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" - "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" - "vextracti128 $0x1,%%ymm0,%%xmm1 \n" - "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" - "vpshufd $0xd8,%%xmm0,%%xmm0 \n" - "vmovdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" - "jge 4b \n" - "vzeroupper \n" - "jmp 19f \n" + "4: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vpsubd 0x00(%0,%4,4),%%ymm0,%%ymm0\n" + "vpsubd 0x20(%0,%4,4),%%ymm1,%%ymm1\n" + "lea 0x40(%0),%0 \n" + "vpsubd (%1),%%ymm0,%%ymm0 \n" + "vpsubd 0x20(%1),%%ymm1,%%ymm1 \n" + "vpaddd 0x00(%1,%4,4),%%ymm0,%%ymm0\n" + "vpaddd 0x20(%1,%4,4),%%ymm1,%%ymm1\n" + "lea 0x40(%1),%1 \n" + "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm1 \n" + "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" + "vpshufd $0xd8,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%2) \n" + "lea 0x10(%2),%2 \n" + "sub $0x4,%3 \n" + "jge 4b \n" + "vzeroupper \n" + "jmp 19f \n" // 4 pixel loop. LABELALIGN - "10: \n" - "vmovdqu (%0),%%ymm0 \n" - "vmovdqu 0x20(%0),%%ymm1 \n" - "vpsubd 0x00(%0,%4,4),%%ymm0,%%ymm0\n" - "vpsubd 0x20(%0,%4,4),%%ymm1,%%ymm1\n" - "lea 0x40(%0),%0 \n" - "vpsubd (%1),%%ymm0,%%ymm0 \n" - "vpsubd 0x20(%1),%%ymm1,%%ymm1 \n" - "vpaddd 0x00(%1,%4,4),%%ymm0,%%ymm0\n" - "vpaddd 0x20(%1,%4,4),%%ymm1,%%ymm1\n" - "lea 0x40(%1),%1 \n" - "vcvtdq2ps %%ymm0,%%ymm0 \n" - "vcvtdq2ps %%ymm1,%%ymm1 \n" - "vmulps %%ymm4,%%ymm0,%%ymm0 \n" - "vmulps %%ymm4,%%ymm1,%%ymm1 \n" - "vcvtps2dq %%ymm0,%%ymm0 \n" - "vcvtps2dq %%ymm1,%%ymm1 \n" - "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" - "vextracti128 $0x1,%%ymm0,%%xmm1 \n" - "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" - "vpshufd $0xd8,%%xmm0,%%xmm0 \n" - "vmovdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" - "jge 10b \n" - "vzeroupper \n" + "10: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vpsubd 0x00(%0,%4,4),%%ymm0,%%ymm0\n" + "vpsubd 0x20(%0,%4,4),%%ymm1,%%ymm1\n" + "lea 0x40(%0),%0 \n" + "vpsubd (%1),%%ymm0,%%ymm0 \n" + "vpsubd 0x20(%1),%%ymm1,%%ymm1 \n" + "vpaddd 0x00(%1,%4,4),%%ymm0,%%ymm0\n" + "vpaddd 0x20(%1,%4,4),%%ymm1,%%ymm1\n" + "lea 0x40(%1),%1 \n" + "vcvtdq2ps %%ymm0,%%ymm0 \n" + "vcvtdq2ps %%ymm1,%%ymm1 \n" + "vmulps %%ymm4,%%ymm0,%%ymm0 \n" + "vmulps %%ymm4,%%ymm1,%%ymm1 \n" + "vcvtps2dq %%ymm0,%%ymm0 \n" + "vcvtps2dq %%ymm1,%%ymm1 \n" + "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm1 \n" + "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" + "vpshufd $0xd8,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%2) \n" + "lea 0x10(%2),%2 \n" + "sub $0x4,%3 \n" + "jge 10b \n" + "vzeroupper \n" - "19: \n" - "add $0x3,%3 \n" - "jl 99f \n" + "19: \n" + "add $0x3,%3 \n" + "jl 99f \n" // 1 pixel loop. LABELALIGN - "20: \n" - "vmovdqu (%0),%%xmm0 \n" - "vpsubd 0x00(%0,%4,4),%%xmm0,%%xmm0\n" - "lea 0x10(%0),%0 \n" - "vmovdqu 0x00(%1,%4,4),%%xmm1 \n" - "vpsubd (%1),%%xmm1,%%xmm1 \n" - "lea 0x10(%1),%1 \n" - "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" - "vcvtdq2ps %%xmm0,%%xmm0 \n" - "vmulps %%xmm4,%%xmm0,%%xmm0 \n" - "vcvtps2dq %%xmm0,%%xmm0 \n" - "vpackssdw %%xmm0,%%xmm0,%%xmm0 \n" - "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" - "vmovd %%xmm0,(%2) \n" - "lea 0x4(%2),%2 \n" - "sub $0x1,%3 \n" - "jge 20b \n" - "99: \n" + "20: \n" + "vmovdqu (%0),%%xmm0 \n" + "vpsubd 0x00(%0,%4,4),%%xmm0,%%xmm0\n" + "lea 0x10(%0),%0 \n" + "vmovdqu 0x00(%1,%4,4),%%xmm1 \n" + "vpsubd (%1),%%xmm1,%%xmm1 \n" + "lea 0x10(%1),%1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vcvtdq2ps %%xmm0,%%xmm0 \n" + "vmulps %%xmm4,%%xmm0,%%xmm0 \n" + "vcvtps2dq %%xmm0,%%xmm0 \n" + "vpackssdw %%xmm0,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%2) \n" + "lea 0x4(%2),%2 \n" + "sub $0x1,%3 \n" + "jge 20b \n" + "99: \n" : "+r"(topleft), // %0 "+r"(botleft), // %1 "+r"(dst), // %2 @@ -10789,7 +10779,8 @@ void NV21ToYUV24Row_AVX512(const uint8_t* src_y, "+r"(dst_yuv24), // %2 "+r"(width) // %3 : "r"(&kYUV24Perm[0]) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm16"); } #endif // HAS_NV21ToYUV24ROW_AVX512 diff --git a/source/row_lsx.cc b/source/row_lsx.cc index d1549967d..10eb88ae5 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -2645,10 +2645,10 @@ void RGBToYMatrixRow_LSX(const uint8_t* src_rgba, int width, const struct ArgbConstants* c) { static const int8_t shuff[64] = { - 0, 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18, 20, 21, 23, - 24, 26, 27, 29, 30, 0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, - 1, 0, 4, 0, 7, 0, 10, 0, 13, 0, 16, 0, 19, 0, 22, 0, - 25, 0, 28, 0, 31, 0, 2, 0, 5, 0, 8, 0, 11, 0, 14, 0, + 0, 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18, 20, 21, 23, + 24, 26, 27, 29, 30, 0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, + 1, 0, 4, 0, 7, 0, 10, 0, 13, 0, 16, 0, 19, 0, 22, 0, + 25, 0, 28, 0, 31, 0, 2, 0, 5, 0, 8, 0, 11, 0, 14, 0, }; asm volatile( "vldrepl.b $vr0, %3, 0 \n\t" // load rgbconstants diff --git a/source/row_neon.cc b/source/row_neon.cc index 1168b34b3..109a35223 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -491,12 +491,11 @@ void NV12ToRGB24Row_NEON(const uint8_t* src_y, const struct YuvConstants* yuvconstants, int width) { asm volatile( - YUVTORGB_SETUP - "1: \n" // + YUVTORGB_SETUP "1: \n" // READNV12 YUVTORGB RGBTORGB8 - "subs %[width], %[width], #8 \n" - "vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n" - "bgt 1b \n" + "subs %[width], %[width], #8 \n" + "vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n" + "bgt 1b \n" : [src_y] "+r"(src_y), // %[src_y] [src_uv] "+r"(src_uv), // %[src_uv] [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] @@ -512,12 +511,11 @@ void NV21ToRGB24Row_NEON(const uint8_t* src_y, const struct YuvConstants* yuvconstants, int width) { asm volatile( - YUVTORGB_SETUP - "1: \n" // + YUVTORGB_SETUP "1: \n" // READNV21 YUVTORGB RGBTORGB8 - "subs %[width], %[width], #8 \n" - "vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n" - "bgt 1b \n" + "subs %[width], %[width], #8 \n" + "vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n" + "bgt 1b \n" : [src_y] "+r"(src_y), // %[src_y] [src_vu] "+r"(src_vu), // %[src_vu] [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] @@ -1940,8 +1938,8 @@ void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb, : "r"(&c->kRGBToU), // %4 "r"(&c->kRGBToV), // %5 "r"(&c->kAddUV) // %6 - : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", - "q10", "q11", "q12", "q13"); + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q10", + "q11", "q12", "q13"); } // clang-format off @@ -2087,8 +2085,8 @@ void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb, "+r"(width) // %4 : "r"(&c->kRGBToU), // %5 "r"(&c->kRGBToV) // %6 - : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q8", - "q9", "q11", "q12", "q14", "q15"); + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9", "q11", + "q12", "q14", "q15"); } void RGB24ToUVRow_NEON(const uint8_t* src_rgb24, @@ -3113,7 +3111,7 @@ void BlendPlaneRow_NEON(const uint8_t* src0, "vmov.u16 q15, #255 \n" "subs %4, %4, #32 \n" "blt 19f \n" - "1: \n" + "1: \n" "vld1.8 {q0, q1}, [%0]! \n" // load 32 src0 "vld1.8 {q2, q3}, [%1]! \n" // load 32 src1 "vld1.8 {q8, q9}, [%2]! \n" // load 32 alpha @@ -3127,8 +3125,10 @@ void BlendPlaneRow_NEON(const uint8_t* src0, "vmull.u8 q6, d2, d18 \n" "pld [%2, #448] \n" "vmull.u8 q7, d3, d19 \n" - "vmlal.u8 q4, d4, d20 \n" // low + src1 * (255 - alpha) - "vmlal.u8 q5, d5, d21 \n" // high + src1 * (255 - alpha) + "vmlal.u8 q4, d4, d20 \n" // low + src1 * (255 - + // alpha) + "vmlal.u8 q5, d5, d21 \n" // high + src1 * (255 - + // alpha) "vmlal.u8 q6, d6, d22 \n" "vmlal.u8 q7, d7, d23 \n" "vaddhn.u16 d0, q4, q15 \n" // (low + 255) >> 8 @@ -3137,7 +3137,7 @@ void BlendPlaneRow_NEON(const uint8_t* src0, "vaddhn.u16 d3, q7, q15 \n" "vst1.8 {q0, q1}, [%3]! \n" // store 32 dst "bge 1b \n" - "19: \n" + "19: \n" "adds %4, %4, #32 \n" "ble 99f \n" @@ -3148,12 +3148,14 @@ void BlendPlaneRow_NEON(const uint8_t* src0, "vmvn.8 q12, q2 \n" // 255 - alpha "vmull.u8 q4, d0, d4 \n" // low src0 * alpha "vmull.u8 q5, d1, d5 \n" // high src0 * alpha - "vmlal.u8 q4, d2, d24 \n" // low + src1 * (255 - alpha) - "vmlal.u8 q5, d3, d25 \n" // high + src1 * (255 - alpha) + "vmlal.u8 q4, d2, d24 \n" // low + src1 * (255 - + // alpha) + "vmlal.u8 q5, d3, d25 \n" // high + src1 * (255 - + // alpha) "vaddhn.u16 d0, q4, q15 \n" // + 255 >> 8 "vaddhn.u16 d1, q5, q15 \n" // + 255 >> 8 "vst1.8 {q0}, [%3]! \n" // store 16 dst - "99: \n" + "99: \n" : "+r"(src0), // %0 "+r"(src1), // %1 "+r"(alpha), // %2 diff --git a/source/row_neon64.cc b/source/row_neon64.cc index 2de205133..368461e31 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -2823,7 +2823,7 @@ void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb, "dup v25.8h, v0.h[0] \n" // kAddUV "1: \n" "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // load 8 RGB - "subs %w3, %w3, #8 \n" // 8 processed per loop. + "subs %w3, %w3, #8 \n" // 8 processed per loop. "uxtl v4.8h, v0.8b \n" // B "uxtl v5.8h, v1.8b \n" // G @@ -2850,9 +2850,8 @@ void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb, "+r"(dst_v), // %2 "+r"(width) // %3 : [c] "r"(c) // %4 - : "cc", "memory", "v0", "v1", "v2", "v4", "v5", "v6", "v16", - "v17", "v18", "v19", "v20", "v21", "v22", "v24", "v25", "v26", - "v27"); + : "cc", "memory", "v0", "v1", "v2", "v4", "v5", "v6", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v24", "v25", "v26", "v27"); } void ARGBToUV444MatrixRow_NEON_I8MM(const uint8_t* src_argb, @@ -3020,7 +3019,8 @@ void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb, "dup v24.8h, v17.h[0] \n" // V0 "dup v26.8h, v17.h[1] \n" // V1 "dup v27.8h, v17.h[2] \n" // V2 - "movi v25.8h, #0x80, lsl #8 \n" // 128.0 in 16-bit (0x8000) + "movi v25.8h, #0x80, lsl #8 \n" // 128.0 in 16-bit + // (0x8000) "1: \n" "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels. @@ -3057,12 +3057,12 @@ void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb, "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. "b.gt 1b \n" - : "+r"(src_rgb), // %0 - "+r"(src_rgb_1), // %1 - "+r"(dst_u), // %2 - "+r"(dst_v), // %3 - "+r"(width) // %4 - : [c] "r"(c) // %5 + : "+r"(src_rgb), // %0 + "+r"(src_rgb_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(width) // %4 + : [c] "r"(c) // %5 : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v20", "v21", "v22", "v24", "v25", "v26", "v27"); } @@ -3624,7 +3624,7 @@ void InterpolateRow_NEON(uint8_t* dst_ptr, "dup v5.16b, %w4 \n" "dup v4.16b, %w5 \n" // General purpose row blend. - "1: \n" + "1: \n" "ld1 {v0.16b}, [%1], #16 \n" "ld1 {v1.16b}, [%2], #16 \n" "subs %w3, %w3, #16 \n" @@ -3687,7 +3687,7 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "b.lt 2f \n" // 16-element unrolled loop (32 bytes). - "1: \n" + "1: \n" "ldp q0, q1, [%1], #32 \n" "ldp q6, q7, [%2], #32 \n" "subs %w3, %w3, #16 \n" @@ -3707,7 +3707,7 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr, "stp d2, d3, [%0], #16 \n" "b.ge 1b \n" - "2: \n" + "2: \n" "adds %w3, %w3, #16 \n" "b.eq 99f \n" @@ -4283,7 +4283,7 @@ void BlendPlaneRow_NEON(const uint8_t* src0, "movi v15.8h, #255 \n" "subs %w4, %w4, #32 \n" "blt 19f \n" - "1: \n" + "1: \n" "ld1 {v0.16b, v1.16b}, [%0], #32 \n" // load 32 src0 "ld1 {v2.16b, v3.16b}, [%1], #32 \n" // load 32 src1 "ld1 {v4.16b, v5.16b}, [%2], #32 \n" // load 32 alpha @@ -4297,8 +4297,10 @@ void BlendPlaneRow_NEON(const uint8_t* src0, "umull v10.8h, v1.8b, v5.8b \n" "prfm pldl1keep, [%2, 448] \n" "umull2 v11.8h, v1.16b, v5.16b \n" - "umlal v8.8h, v2.8b, v6.8b \n" // low + src1 * (255 - alpha) - "umlal2 v9.8h, v2.16b, v6.16b \n" // high + src1 * (255 - alpha) + "umlal v8.8h, v2.8b, v6.8b \n" // low + src1 * (255 - + // alpha) + "umlal2 v9.8h, v2.16b, v6.16b \n" // high + src1 * (255 - + // alpha) "umlal v10.8h, v3.8b, v7.8b \n" "umlal2 v11.8h, v3.16b, v7.16b \n" "addhn v0.8b, v8.8h, v15.8h \n" // (low + 255) >> 8 @@ -4307,7 +4309,7 @@ void BlendPlaneRow_NEON(const uint8_t* src0, "addhn v3.8b, v11.8h, v15.8h \n" "st1 {v0.8b, v1.8b, v2.8b, v3.8b}, [%3], #32 \n" // store 32 dst "b.ge 1b \n" - "19: \n" + "19: \n" "adds %w4, %w4, #32 \n" "b.le 99f \n" @@ -4318,12 +4320,14 @@ void BlendPlaneRow_NEON(const uint8_t* src0, "mvn v3.16b, v2.16b \n" // 255 - alpha "umull v4.8h, v0.8b, v2.8b \n" // low src0 * alpha "umull2 v5.8h, v0.16b, v2.16b \n" // high src0 * alpha - "umlal v4.8h, v1.8b, v3.8b \n" // low + src1 * (255 - alpha) - "umlal2 v5.8h, v1.16b, v3.16b \n" // high + src1 * (255 - alpha) + "umlal v4.8h, v1.8b, v3.8b \n" // low + src1 * (255 - + // alpha) + "umlal2 v5.8h, v1.16b, v3.16b \n" // high + src1 * (255 - + // alpha) "addhn v0.8b, v4.8h, v15.8h \n" // (low + 255) >> 8 "addhn v1.8b, v5.8h, v15.8h \n" // (high + 255) >> 8 "st1 {v0.8b, v1.8b}, [%3], #16 \n" // store 16 dst - "99: \n" + "99: \n" : "+r"(src0), "+r"(src1), "+r"(alpha), "+r"(dst), "+r"(width) : : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", @@ -4426,7 +4430,7 @@ void ComputeCumulativeSumRow_NEON(const uint8_t* row, "b.lt 49f \n" // 4 pixel loop. - "40: \n" + "40: \n" "ldr q2, [%0], #16 \n" "ldp q16, q17, [%2] \n" "ldp q18, q19, [%2, #32] \n" @@ -4450,12 +4454,12 @@ void ComputeCumulativeSumRow_NEON(const uint8_t* row, "subs %w3, %w3, #4 \n" "b.ge 40b \n" - "49: \n" + "49: \n" "adds %w3, %w3, #3 \n" "b.lt 19f \n" // 1 pixel loop. - "10: \n" + "10: \n" "ldr s2, [%0], #4 \n" "ldr q16, [%2], #16 \n" "ushll v2.8h, v2.8b, #0 \n" @@ -4465,14 +4469,14 @@ void ComputeCumulativeSumRow_NEON(const uint8_t* row, "subs %w3, %w3, #1 \n" "b.ge 10b \n" - "19: \n" + "19: \n" : "+r"(row), // %0 "+r"(cumsum), // %1 "+r"(previous_cumsum), // %2 "+r"(width) // %3 : - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", - "v18", "v19"); + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18", + "v19"); } #endif // HAS_COMPUTECUMULATIVESUMROW_NEON @@ -4503,7 +4507,7 @@ void CumulativeSumToAverageRow_NEON(const int32_t* topleft, "dup v6.8h, %w6 \n" // 4 pixel small loop. - "4: \n" + "4: \n" "ldp q0, q1, [%0] \n" "ldp q2, q3, [%0, #32] \n" "add %0, %0, #64 \n" @@ -4539,7 +4543,7 @@ void CumulativeSumToAverageRow_NEON(const int32_t* topleft, "b 49f \n" // 4 pixel loop. - "40: \n" + "40: \n" "ldp q0, q1, [%0] \n" "ldp q2, q3, [%0, #32] \n" "add %0, %0, #64 \n" @@ -4586,12 +4590,12 @@ void CumulativeSumToAverageRow_NEON(const int32_t* topleft, "subs %w3, %w3, #4 \n" "b.ge 40b \n" - "49: \n" + "49: \n" "adds %w3, %w3, #3 \n" "b.lt 19f \n" // 1 pixel loop. - "10: \n" + "10: \n" "ldr q1, [%0, %8] \n" "ldr q0, [%0], #16 \n" "ldr q3, [%1, %8] \n" @@ -4607,16 +4611,16 @@ void CumulativeSumToAverageRow_NEON(const int32_t* topleft, "str s0, [%2], #4 \n" "subs %w3, %w3, #1 \n" "b.ge 10b \n" - "19: \n" - : "+r"(topleft), // %0 - "+r"(botleft), // %1 - "+r"(dst), // %2 - "+r"(count), // %3 - "=&r"(topright), // %4 - "=&r"(botright), // %5 - "=&r"(scale) // %6 - : "r"(area), // %7 - "r"((ptrdiff_t)width * 4) // %8 + "19: \n" + : "+r"(topleft), // %0 + "+r"(botleft), // %1 + "+r"(dst), // %2 + "+r"(count), // %3 + "=&r"(topright), // %4 + "=&r"(botright), // %5 + "=&r"(scale) // %6 + : "r"(area), // %7 + "r"((ptrdiff_t)width * 4) // %8 : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27"); } diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 4b28e21ed..f3fca7fe1 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -1406,14 +1406,14 @@ void InterpolateRow_RVV(uint8_t* dst_ptr, "add %[src_ptr1], %[src_ptr1], %[vl]\n" "add %[dst_ptr], %[dst_ptr], %[vl] \n" "bgtz %[dst_w], 1b \n" - : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] - [src_ptr] "+r"(src_ptr), // %[src_ptr] - [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] - [dst_w] "+r"(dst_w), // %[dst_w] - [vl] "=&r"(vl) // %[vl] - : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] - [y1_fraction] "r"(y1_fraction), // %[y1_fraction] - [c128] "r"(128) // %[c128] + : [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] + [src_ptr] "+r"(src_ptr), // %[src_ptr] + [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] + [dst_w] "+r"(dst_w), // %[dst_w] + [vl] "=&r"(vl) // %[vl] + : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] + [y1_fraction] "r"(y1_fraction), // %[y1_fraction] + [c128] "r"(128) // %[c128] : "vl", "vtype", "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17"); } @@ -2313,8 +2313,7 @@ void ARGBMultiplyRow_RVV(const uint8_t* src_argb, #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wvla" size_t vl; - asm( - "1: \n" + asm("1: \n" "vsetvli %[vl], %[w], e8, m2, ta, ma \n" "vle8.v v8, (%[src0]) \n" "vle8.v v10, (%[src1]) \n" @@ -2333,7 +2332,7 @@ void ARGBMultiplyRow_RVV(const uint8_t* src_argb, [w] "+r"(w), // %[w] [vl] "=&r"(vl), // %[vl] "=m"(*(uint8_t (*)[w])dst_argb) - : [k128] "r"(128), // %[k128] + : [k128] "r"(128), // %[k128] "m"(*(const uint8_t (*)[w])src_argb), "m"(*(const uint8_t (*)[w])src_argb1) : "vl", "vtype", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"); diff --git a/source/row_sve.cc b/source/row_sve.cc index f8fa227a6..2d32c7048 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -1184,13 +1184,13 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u, "st1b {z1.b}, p0, [%[dst_uv]] \n" "99: \n" - : [src_u] "+r"(src_u), // %[src_u] - [src_u1] "+r"(src_u1), // %[src_u1] - [src_v] "+r"(src_v), // %[src_v] - [src_v1] "+r"(src_v1), // %[src_v1] - [dst_uv] "+r"(dst_uv), // %[dst_uv] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl) // %[vl] + : [src_u] "+r"(src_u), // %[src_u] + [src_u1] "+r"(src_u1), // %[src_u1] + [src_v] "+r"(src_v), // %[src_v] + [src_v1] "+r"(src_v1), // %[src_v1] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl) // %[vl] : : "cc", "memory", "z1", "z2", "z3", "z4", "z5", "z6", "p0"); } @@ -1229,10 +1229,7 @@ void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) { "b.gt 3b \n" "99: \n" - : [src] "+r"(src), - [dst] "+r"(dst), - [width] "+r"(width), - [vl] "=&r"(vl), + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [vl] "=&r"(vl), [vl2] "=&r"(vl2) : : "cc", "memory", "z0", "z1", "p0"); @@ -1285,11 +1282,8 @@ void InterpolateRow_SVE2(uint8_t* dst_ptr, "st1b {z2.b}, p0, [%[dst_ptr]] \n" "99: \n" - : [src_ptr] "+r"(src_ptr), - [src_ptr1] "+r"(src_ptr1), - [dst_ptr] "+r"(dst_ptr), - [width] "+r"(width), - [vl] "=&r"(vl) + : [src_ptr] "+r"(src_ptr), [src_ptr1] "+r"(src_ptr1), + [dst_ptr] "+r"(dst_ptr), [width] "+r"(width), [vl] "=&r"(vl) : : "cc", "memory", "z2", "z3", "p0"); return; @@ -1711,13 +1705,13 @@ void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv, "st1b {z0.h}, p2, [%[dst_ptr]] \n" "99: \n" - : [src_ptr] "+r"(src_uv), // %[src_ptr] - [src_ptr1] "+r"(src_uv1), // %[src_ptr1] - [dst_ptr] "+r"(dst_uv), // %[dst_ptr] - [width] "+r"(width), // %[width] - [vl] "=&r"(vl), // %[vl] - [vl_input] "=&r"(vl_input) // %[vl_input] - : [shift] "r"(shift) // %[shift] + : [src_ptr] "+r"(src_uv), // %[src_ptr] + [src_ptr1] "+r"(src_uv1), // %[src_ptr1] + [dst_ptr] "+r"(dst_uv), // %[dst_ptr] + [width] "+r"(width), // %[width] + [vl] "=&r"(vl), // %[vl] + [vl_input] "=&r"(vl_input) // %[vl_input] + : [shift] "r"(shift) // %[shift] : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31", "p0", "p1", "p2"); } diff --git a/source/row_win.cc b/source/row_win.cc index 71bfc78aa..214767b0e 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -118,19 +118,19 @@ extern "C" { dst_argb += 32; // Store 8 AR30 values. -#define STOREAR30 \ - xmm1 = _mm_and_si128(xmm1, xmm7); \ - xmm2 = _mm_and_si128(xmm2, xmm7); \ - xmm0 = _mm_srli_epi16(xmm0, 4); \ - xmm3 = _mm_slli_epi16(xmm1, 6); \ - xmm0 = _mm_or_si128(xmm0, xmm3); \ - xmm1 = _mm_srli_epi16(xmm1, 10); \ - xmm2 = _mm_or_si128(xmm2, xmm6); \ - xmm2 = _mm_or_si128(xmm2, xmm1); \ - xmm1 = _mm_unpackhi_epi16(xmm0, xmm2); \ - xmm0 = _mm_unpacklo_epi16(xmm0, xmm2); \ - _mm_storeu_si128((__m128i*)dst_ar30, xmm0); \ - _mm_storeu_si128((__m128i*)(dst_ar30 + 16), xmm1); \ +#define STOREAR30 \ + xmm1 = _mm_and_si128(xmm1, xmm7); \ + xmm2 = _mm_and_si128(xmm2, xmm7); \ + xmm0 = _mm_srli_epi16(xmm0, 4); \ + xmm3 = _mm_slli_epi16(xmm1, 6); \ + xmm0 = _mm_or_si128(xmm0, xmm3); \ + xmm1 = _mm_srli_epi16(xmm1, 10); \ + xmm2 = _mm_or_si128(xmm2, xmm6); \ + xmm2 = _mm_or_si128(xmm2, xmm1); \ + xmm1 = _mm_unpackhi_epi16(xmm0, xmm2); \ + xmm0 = _mm_unpacklo_epi16(xmm0, xmm2); \ + _mm_storeu_si128((__m128i*)dst_ar30, xmm0); \ + _mm_storeu_si128((__m128i*)(dst_ar30 + 16), xmm1); \ dst_ar30 += 32; // Convert 32 ARGB pixels (128 bytes) to 32 UV444 values. @@ -308,18 +308,15 @@ void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) { } #endif // HAS_RAWTOARGBROW_AVX2 -#if defined(HAS_RAWTOARGBROW_AVX512BW) || \ +#if defined(HAS_RAWTOARGBROW_AVX512BW) || \ defined(HAS_RGB24TOARGBROW_AVX512BW) || \ defined(HAS_RAWTORGB24ROW_AVX512BW) || \ defined(HAS_RGBTOYMATRIXROW_AVX512BW) -static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, - 3, 4, 4, 5}; -static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8, - 9, 10, 10, 11}; -static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6, - 7, 8, 8, 9}; -static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, - 5, 6, 6, 7}; +static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, 3, 4, 4, 5}; +static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8, + 9, 10, 10, 11}; +static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6, 7, 8, 8, 9}; +static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, 5, 6, 6, 7}; #endif // defined(HAS_RAWTOARGBROW_AVX512BW) || // defined(HAS_RGB24TOARGBROW_AVX512BW) || // defined(HAS_RAWTORGB24ROW_AVX512BW) || @@ -328,21 +325,24 @@ static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, #if defined(HAS_RGB24TOARGBROW_AVX512BW) || \ defined(HAS_RGBTOYMATRIXROW_AVX512BW) static const uint8_t kShuffleMaskRGB24ToARGB_AVX512BW[32] = { - 0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u, - 4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, 128u}; + 0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, + 128u, 9u, 10u, 11u, 128u, 4u, 5u, 6u, 128u, 7u, 8u, + 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, 128u}; #endif // defined(HAS_RGB24TOARGBROW_AVX512BW) || // defined(HAS_RGBTOYMATRIXROW_AVX512BW) #if defined(HAS_RAWTOARGBROW_AVX512BW) static const uint8_t kShuffleMaskRAWToARGB_AVX512BW[32] = { - 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u, - 6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u}; + 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, + 128u, 11u, 10u, 9u, 128u, 6u, 5u, 4u, 128u, 9u, 8u, + 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u}; #endif // defined(HAS_RAWTOARGBROW_AVX512BW) #if defined(HAS_RAWTORGB24ROW_AVX2) || defined(HAS_RAWTORGB24ROW_AVX512BW) static const uint8_t kShuffleMaskRAWToRGB24_AVX2[32] = { - 2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 6u, 11u, 10u, 9u, 128u, 128u, 128u, 128u, - 6u, 5u, 4u, 9u, 8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u, 128u, 128u, 128u, 128u}; + 2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 6u, 11u, 10u, + 9u, 128u, 128u, 128u, 128u, 6u, 5u, 4u, 9u, 8u, 7u, + 12u, 11u, 10u, 15u, 14u, 13u, 128u, 128u, 128u, 128u}; #endif // defined(HAS_RAWTORGB24ROW_AVX2) || // defined(HAS_RAWTORGB24ROW_AVX512BW) @@ -364,33 +364,33 @@ static const uint32_t kPermdRGB24_2_AVX512BW[16] = { #endif // defined(HAS_RAWTORGB24ROW_AVX512BW) #if defined(HAS_RAWTORGB24ROW_AVX512VBMI) static const uint8_t kPermRAWToRGB24_0[64] = { - 0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a, 0x09, - 0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17, 0x16, 0x15, - 0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e, 0x23, 0x22, 0x21, - 0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b, 0x2a, 0x2f, 0x2e, 0x2d, - 0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38, 0x37, 0x36, 0x3b, 0x3a, 0x39, - 0x3e, 0x3d, 0x3c, 0x41}; + 0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a, + 0x09, 0x0e, 0x0d, 0x0c, 0x11, 0x10, 0x0f, 0x14, 0x13, 0x12, 0x17, + 0x16, 0x15, 0x1a, 0x19, 0x18, 0x1d, 0x1c, 0x1b, 0x20, 0x1f, 0x1e, + 0x23, 0x22, 0x21, 0x26, 0x25, 0x24, 0x29, 0x28, 0x27, 0x2c, 0x2b, + 0x2a, 0x2f, 0x2e, 0x2d, 0x32, 0x31, 0x30, 0x35, 0x34, 0x33, 0x38, + 0x37, 0x36, 0x3b, 0x3a, 0x39, 0x3e, 0x3d, 0x3c, 0x41}; static const uint8_t kPermRAWToRGB24_1[64] = { - 0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08, 0x0d, - 0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15, 0x14, 0x19, - 0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22, 0x21, 0x20, 0x25, - 0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29, 0x2e, 0x2d, 0x2c, 0x31, - 0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36, 0x35, 0x3a, 0x39, 0x38, 0x3d, - 0x3c, 0x3b, 0x00, 0x3f}; + 0x00, 0x7f, 0x04, 0x03, 0x02, 0x07, 0x06, 0x05, 0x0a, 0x09, 0x08, + 0x0d, 0x0c, 0x0b, 0x10, 0x0f, 0x0e, 0x13, 0x12, 0x11, 0x16, 0x15, + 0x14, 0x19, 0x18, 0x17, 0x1c, 0x1b, 0x1a, 0x1f, 0x1e, 0x1d, 0x22, + 0x21, 0x20, 0x25, 0x24, 0x23, 0x28, 0x27, 0x26, 0x2b, 0x2a, 0x29, + 0x2e, 0x2d, 0x2c, 0x31, 0x30, 0x2f, 0x34, 0x33, 0x32, 0x37, 0x36, + 0x35, 0x3a, 0x39, 0x38, 0x3d, 0x3c, 0x3b, 0x00, 0x3f}; static const uint8_t kPermRAWToRGB24_2[64] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, - 0x3c, 0x3d, 0x40, 0x3f}; + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, + 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x40, 0x3f}; static const uint8_t kPermRAWToRGB24_3[64] = { - 0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c, 0x4b, - 0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53, 0x58, 0x57, - 0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60, 0x5f, 0x64, 0x63, - 0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d, 0x6c, 0x6b, 0x70, 0x6f, - 0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74, 0x79, 0x78, 0x77, 0x7c, 0x7b, - 0x7a, 0x7f, 0x7e, 0x7d}; + 0x3e, 0x43, 0x42, 0x41, 0x46, 0x45, 0x44, 0x49, 0x48, 0x47, 0x4c, + 0x4b, 0x4a, 0x4f, 0x4e, 0x4d, 0x52, 0x51, 0x50, 0x55, 0x54, 0x53, + 0x58, 0x57, 0x56, 0x5b, 0x5a, 0x59, 0x5e, 0x5d, 0x5c, 0x61, 0x60, + 0x5f, 0x64, 0x63, 0x62, 0x67, 0x66, 0x65, 0x6a, 0x69, 0x68, 0x6d, + 0x6c, 0x6b, 0x70, 0x6f, 0x6e, 0x73, 0x72, 0x71, 0x76, 0x75, 0x74, + 0x79, 0x78, 0x77, 0x7c, 0x7b, 0x7a, 0x7f, 0x7e, 0x7d}; #endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI) #if defined(HAS_RAWTOARGBROW_AVX512BW) || defined(HAS_RGB24TOARGBROW_AVX512BW) @@ -450,9 +450,8 @@ LIBYUV_TARGET_AVX512BW void RAWToARGBRow_AVX512BW(const uint8_t* src_raw, uint8_t* dst_argb, int width) { - RGBToARGBRow_AVX512BW( - src_raw, dst_argb, - (const uint32_t*)kShuffleMaskRAWToARGB_AVX512BW, width); + RGBToARGBRow_AVX512BW(src_raw, dst_argb, + (const uint32_t*)kShuffleMaskRAWToARGB_AVX512BW, width); } #endif // HAS_RAWTOARGBROW_AVX512BW @@ -461,17 +460,15 @@ LIBYUV_TARGET_AVX512BW void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, uint8_t* dst_argb, int width) { - RGBToARGBRow_AVX512BW( - src_rgb24, dst_argb, - (const uint32_t*)kShuffleMaskRGB24ToARGB_AVX512BW, width); + RGBToARGBRow_AVX512BW(src_rgb24, dst_argb, + (const uint32_t*)kShuffleMaskRGB24ToARGB_AVX512BW, + width); } #endif // HAS_RGB24TOARGBROW_AVX512BW #ifdef HAS_RAWTORGB24ROW_AVX2 LIBYUV_TARGET_AVX2 -void RAWToRGB24Row_AVX2(const uint8_t* src_raw, - uint8_t* dst_rgb24, - int width) { +void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { __m256i ymm_shuf = _mm256_loadu_si256((const __m256i*)kShuffleMaskRAWToRGB24_AVX2); __m256i ymm_p0 = _mm256_loadu_si256((const __m256i*)kPermd0_AVX2); @@ -608,8 +605,8 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, #endif // HAS_RAWTORGB24ROW_AVX512VBMI #ifdef HAS_RGBTOYMATRIXROW_AVX512BW -static const uint32_t kPermdARGBToY_AVX512BW[16] = { - 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15}; +static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; LIBYUV_TARGET_AVX512BW void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, @@ -631,10 +628,10 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, __m512i zmm_80 = _mm512_set1_epi8((char)0x80); __m512i zmm_one = _mm512_set1_epi16(1); - __m512i zmm_coeff = _mm512_broadcast_i32x4( - _mm_loadu_si128((const __m128i*)c->kRGBToY)); - __m512i zmm_add = _mm512_broadcast_i32x4( - _mm_loadu_si128((const __m128i*)c->kAddY)); + __m512i zmm_coeff = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY)); + __m512i zmm_add = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddY)); // Compute bias adjusted for 0x80 subtraction: // bias = kAddY - pmaddwd(pmaddubsw(kRGBToY, 0x80), 1) @@ -703,8 +700,10 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c) { - __m256i ymm_u = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToU)); - __m256i ymm_v = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + __m256i ymm_u = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToU)); + __m256i ymm_v = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)c->kRGBToV)); __m256i ymm_0101 = _mm256_set1_epi16(0x0101); __m256i ymm_shuf = _mm256_setr_epi8(0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15, 0, @@ -1509,8 +1508,10 @@ void I422ToARGBRow_AVX2(const uint8_t* src_y, __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); - __m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); - __m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); + __m256i ymm_kYToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + __m256i ymm_kYBiasToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); __m256i ymm_128 = _mm256_set1_epi8((char)0x80); __m256i ymm_alpha = _mm256_set1_epi8((char)0xff); __m256i ymm_u_zero = _mm256_setzero_si256(); @@ -1585,7 +1586,8 @@ void I422ToAR30Row_AVX2(const uint8_t* src_y, __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); - __m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + __m256i ymm_kYToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); __m256i ymm_kYBiasToRgb = _mm256_sub_epi16( _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb), _mm256_set1_epi16(24)); @@ -1669,8 +1671,10 @@ void NV12ToARGBRow_AVX2(const uint8_t* src_y, __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); - __m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); - __m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); + __m256i ymm_kYToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + __m256i ymm_kYBiasToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); __m256i ymm_128 = _mm256_set1_epi8((char)0x80); __m256i ymm_alpha = _mm256_set1_epi8((char)0xff); __m256i ymm_u_zero = _mm256_setzero_si256(); @@ -1742,8 +1746,10 @@ void NV21ToARGBRow_AVX2(const uint8_t* src_y, __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); - __m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); - __m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); + __m256i ymm_kYToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + __m256i ymm_kYBiasToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); __m256i ymm_128 = _mm256_set1_epi8((char)0x80); __m256i ymm_alpha = _mm256_set1_epi8((char)0xff); __m256i ymm_shuf_nv21 = _mm256_loadu_si256((const __m256i*)kShuffleNV21_AVX2); @@ -1821,8 +1827,10 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y, __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); - __m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); - __m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); + __m256i ymm_kYToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + __m256i ymm_kYBiasToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); __m256i ymm_128 = _mm256_set1_epi8((char)0x80); __m256i ymm_shuf0 = _mm256_broadcastsi128_si256( @@ -1927,61 +1935,59 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y, zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); -#define READYUV422_MASK_AVX512BW \ - __mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); \ - __mmask32 mask_y = (__mmask32)((1u << width) - 1u); \ - __m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u); \ - __m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); \ - __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \ - __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \ - zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \ - zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \ - __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \ - zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \ - zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \ - __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); \ - __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \ - zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \ - zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \ +#define READYUV422_MASK_AVX512BW \ + __mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); \ + __mmask32 mask_y = (__mmask32)((1u << width) - 1u); \ + __m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u); \ + __m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset); \ + __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); \ + __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); \ + zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); \ + zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); \ + __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); \ + zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); \ + zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); \ + __m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y); \ + __m512i zmm4 = _mm512_castsi256_si512(ymm_y); \ + zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); \ + zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); \ zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); -#define YUVTORGB_AVX512BW \ - zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \ - zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); \ - __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); \ - __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); \ - __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); \ - zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); \ - zmm0 = _mm512_adds_epi16(zmm0, zmm4); \ - zmm1 = _mm512_subs_epi16(zmm4, zmm1); \ - zmm2 = _mm512_adds_epi16(zmm2, zmm4); \ - zmm0 = _mm512_srai_epi16(zmm0, 6); \ - zmm1 = _mm512_srai_epi16(zmm1, 6); \ - zmm2 = _mm512_srai_epi16(zmm2, 6); \ - zmm0 = _mm512_packus_epi16(zmm0, zmm0); \ - zmm1 = _mm512_packus_epi16(zmm1, zmm1); \ +#define YUVTORGB_AVX512BW \ + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \ + zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); \ + __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); \ + __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); \ + __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); \ + zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); \ + zmm0 = _mm512_adds_epi16(zmm0, zmm4); \ + zmm1 = _mm512_subs_epi16(zmm4, zmm1); \ + zmm2 = _mm512_adds_epi16(zmm2, zmm4); \ + zmm0 = _mm512_srai_epi16(zmm0, 6); \ + zmm1 = _mm512_srai_epi16(zmm1, 6); \ + zmm2 = _mm512_srai_epi16(zmm2, 6); \ + zmm0 = _mm512_packus_epi16(zmm0, zmm0); \ + zmm1 = _mm512_packus_epi16(zmm1, zmm1); \ zmm2 = _mm512_packus_epi16(zmm2, zmm2); -#define STORERGB24_AVX512VBMI \ - __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \ - __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \ - __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \ - _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \ - _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \ - _mm512_castsi512_si256(zmm_dst1)); \ +#define STORERGB24_AVX512VBMI \ + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \ + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \ + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \ + _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); \ + _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), \ + _mm512_castsi512_si256(zmm_dst1)); \ dst_rgb24 += 96; -#define STORERGB24_MASK_AVX512VBMI \ - int dst_bytes = width * 3; \ - __mmask64 mask_dst0 = \ - dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \ - __mmask32 mask_dst1 = \ - dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \ - __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \ - __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \ - __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \ - _mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \ - _mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \ +#define STORERGB24_MASK_AVX512VBMI \ + int dst_bytes = width * 3; \ + __mmask64 mask_dst0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \ + __mmask32 mask_dst1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \ + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); \ + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); \ + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); \ + _mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \ + _mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \ _mm512_castsi512_si256(zmm_dst1)); #define STORERGB24_AVX512BW \ @@ -1999,23 +2005,21 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y, _mm512_castsi512_si256(zmm_dst1)); \ dst_rgb24 += 96; -#define STORERGB24_MASK_AVX512BW \ - int dst_bytes = width * 3; \ - __mmask64 mask_dst0 = \ - dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \ - __mmask32 mask_dst1 = \ - dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \ - __m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \ - __m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \ - __m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \ - __m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \ - zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \ - zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \ - zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \ - __m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \ - __m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \ - _mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \ - _mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \ +#define STORERGB24_MASK_AVX512BW \ + int dst_bytes = width * 3; \ + __mmask64 mask_dst0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; \ + __mmask32 mask_dst1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; \ + __m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); \ + __m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); \ + __m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); \ + __m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); \ + zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); \ + zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); \ + zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); \ + __m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); \ + __m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); \ + _mm512_mask_storeu_epi8(dst_rgb24, mask_dst0, zmm_dst0); \ + _mm256_mask_storeu_epi8(dst_rgb24 + 64, mask_dst1, \ _mm512_castsi512_si256(zmm_dst1)); #endif // defined(HAS_I422TORGB24ROW_AVX512VBMI) || // defined(HAS_I422TORGB24ROW_AVX512BW) @@ -2030,43 +2034,49 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y, int width) { // Masks static const uint8_t kMaskBG[64] = { - 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, 0x45, - 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, 0x13, 0x53, - 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, 0x60, 0x21, 0x61, - 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, 0x26, 0x66, 0x27, 0x67, - 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, 0x73, 0x34, 0x74, 0x35, 0x75, - 0x36, 0x76, 0x37, 0x77}; + 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, + 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, + 0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, + 0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, + 0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, + 0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77}; static const uint8_t kMaskDST0[64] = { - 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, 0x43, - 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, 0x0f, 0x47, - 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, 0x16, 0x17, 0x53, - 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, 0x56, 0x1e, 0x1f, 0x57, - 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, 0x25, 0x62, 0x26, 0x27, 0x63, - 0x28, 0x29, 0x64, 0x2a}; + 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, + 0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, + 0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, + 0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, + 0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, + 0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a}; static const uint8_t kMaskDST1[64] = { - 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, 0x32, - 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, 0x74, 0x3a, - 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}; + 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, + 0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, + 0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2}; static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; // Constants - __m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); - __m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); - __m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); - __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); - __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); + __m512i zmm_kUVToB = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); + __m512i zmm_kUVToG = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); + __m512i zmm_kUVToR = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); + __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); + __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); __m512i zmm_128 = _mm512_set1_epi8((char)0x80); __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG); __m512i zmm_mask_DST0 = _mm512_loadu_si512((const __m512i*)kMaskDST0); __m512i zmm_mask_DST1 = _mm512_loadu_si512((const __m512i*)kMaskDST1); __m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords); - __m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords); + __m512i zmm_split_y = + _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords); ptrdiff_t offset = src_v - src_u; @@ -2099,11 +2109,16 @@ void I422ToRGB24Row_AVX512BW(const uint8_t* src_y, static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; // Constants - __m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); - __m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); - __m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); - __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); - __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); + __m512i zmm_kUVToB = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); + __m512i zmm_kUVToG = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); + __m512i zmm_kUVToR = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); + __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); + __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); __m512i zmm_128 = _mm512_set1_epi8((char)0x80); __m512i zmm_shuf0 = _mm512_broadcast_i32x4( @@ -2113,7 +2128,8 @@ void I422ToRGB24Row_AVX512BW(const uint8_t* src_y, __m512i zmm_stitch0 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_0); __m512i zmm_stitch1 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_1); __m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords); - __m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords); + __m512i zmm_split_y = + _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords); ptrdiff_t offset = src_v - src_u; @@ -2154,19 +2170,24 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2}; static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; static const uint16_t kPermAR30_0[32] = { - 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, - 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47}; + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, + 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47}; static const uint16_t kPermAR30_1[32] = { 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63}; // Constants - __m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); - __m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); - __m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); - __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); + __m512i zmm_kUVToB = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); + __m512i zmm_kUVToG = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); + __m512i zmm_kUVToR = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); + __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); __m512i zmm_kYBiasToRgb = _mm512_sub_epi16( - _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)), + _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)), _mm512_set1_epi16(24)); __m512i zmm_128 = _mm512_set1_epi8((char)0x80); __m512i zmm_3ff0 = _mm512_set1_epi16((short)0x3ff0); @@ -2177,7 +2198,8 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0); __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1); __m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords); - __m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords); + __m512i zmm_split_y = + _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords); ptrdiff_t offset = src_v - src_u; @@ -2248,9 +2270,9 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, #endif // HAS_I422TOAR30ROW_AVX512BW #ifdef HAS_ARGBBLENDROW_SSSE3 -static const uint8_t kShuffleAlpha[16] = { - 3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, - 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80}; +static const uint8_t kShuffleAlpha[16] = {3u, 0x80, 3u, 0x80, 7u, 0x80, + 7u, 0x80, 11u, 0x80, 11u, 0x80, + 15u, 0x80, 15u, 0x80}; void ARGBBlendRow_SSSE3(const uint8_t* src_argb, const uint8_t* src_argb1, diff --git a/source/scale.cc b/source/scale.cc index fc89a5545..aaaa52949 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -208,7 +208,6 @@ static void ScalePlaneDown2_16(int src_width, } } - // Scale plane, 1/4 // This is an optimized version for scaling down a plane to 1/4 of // its original size. @@ -1354,7 +1353,8 @@ static int ScalePlaneBilinearUp(int src_width, // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;) // so rowptr points to yi and (rowptr + rowstride) points to yi + 1. // 3. Advance src by 1 row if row yi + 2 exists ((y + 65536) < max_y), - // otherwise clamp src at (src_height - 1) to avoid reading out of bounds. + // otherwise clamp src at (src_height - 1) to avoid reading out of + // bounds. for (j = 0; j < dst_height; ++j) { if (y > max_y) { y = max_y; @@ -1832,7 +1832,8 @@ static int ScalePlaneBilinearUp_16(int src_width, // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;) // so rowptr points to yi and (rowptr + rowstride) points to yi + 1. // 3. Advance src by 1 row if row yi + 2 exists ((y + 65536) < max_y), - // otherwise clamp src at (src_height - 1) to avoid reading out of bounds. + // otherwise clamp src at (src_height - 1) to avoid reading out of + // bounds. for (j = 0; j < dst_height; ++j) { if (y > max_y) { y = max_y; diff --git a/source/scale_argb.cc b/source/scale_argb.cc index 8dda11797..b35581ca6 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -590,7 +590,8 @@ static int ScaleARGBBilinearUp(int src_width, // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;) // so rowptr points to yi and (rowptr + rowstride) points to yi + 1. // 3. Advance src by 1 row if row yi + 2 exists ((y64 + 65536) < max_y), - // otherwise clamp src at (src_height - 1) to avoid reading out of bounds. + // otherwise clamp src at (src_height - 1) to avoid reading out of + // bounds. for (j = 0; j < dst_height; ++j) { if (y64 > max_y) { y64 = max_y; diff --git a/source/scale_common.cc b/source/scale_common.cc index 25eb56361..292a852c4 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -68,7 +68,6 @@ void ScaleRowDown2_16_C(const uint16_t* src_ptr, } } - void ScaleRowDown2Linear_C(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, @@ -105,7 +104,6 @@ void ScaleRowDown2Linear_16_C(const uint16_t* src_ptr, } } - void ScaleRowDown2Box_C(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, @@ -168,7 +166,6 @@ void ScaleRowDown2Box_16_C(const uint16_t* src_ptr, } } - void ScaleRowDown4_C(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index 6c93933c8..7a1ec15d6 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -474,7 +474,7 @@ void ScaleRowDown34_SSSE3(const uint8_t* src_ptr, "movdqa %[kShuf0],%%xmm3 \n" "movdqa %[kShuf1],%%xmm4 \n" "movdqa %[kShuf2],%%xmm5 \n" - "1: \n" + "1: \n" "movdqu (%[src_ptr]),%%xmm0 \n" "movdqu 0x10(%[src_ptr]),%%xmm2 \n" "lea 0x20(%[src_ptr]),%[src_ptr] \n" @@ -489,12 +489,9 @@ void ScaleRowDown34_SSSE3(const uint8_t* src_ptr, "lea 0x18(%[dst_ptr]),%[dst_ptr] \n" "sub $0x18,%[dst_width] \n" "jg 1b \n" - : [src_ptr] "+r"(src_ptr), - [dst_ptr] "+r"(dst_ptr), + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), [dst_width] "+r"(dst_width) - : [kShuf0] "m"(kShuf0), - [kShuf1] "m"(kShuf1), - [kShuf2] "m"(kShuf2) + : [kShuf0] "m"(kShuf0), [kShuf1] "m"(kShuf1), [kShuf2] "m"(kShuf2) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); } @@ -509,7 +506,7 @@ void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr, "movdqa %[kMadd01],%%xmm5 \n" "movdqa %[kMadd11],%%xmm0 \n" "movdqa %[kRound34],%%xmm1 \n" - "1: \n" + "1: \n" "movdqu (%[src_ptr]),%%xmm6 \n" "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm7 \n" "pavgb %%xmm7,%%xmm6 \n" @@ -541,17 +538,11 @@ void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr, "lea 0x18(%[dst_ptr]),%[dst_ptr] \n" "sub $0x18,%[dst_width] \n" "jg 1b \n" - : [src_ptr] "+r"(src_ptr), - [dst_ptr] "+r"(dst_ptr), + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), [dst_width] "+r"(dst_width) - : [src_stride] "r"(src_stride), - [kShuf01] "m"(kShuf01), - [kShuf11] "m"(kShuf11), - [kShuf21] "m"(kShuf21), - [kMadd01] "m"(kMadd01), - [kMadd11] "m"(kMadd11), - [kRound34] "m"(kRound34), - [kMadd21] "m"(kMadd21) + : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), + [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), + [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } @@ -567,7 +558,7 @@ void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, "movdqa %[kMadd01],%%xmm5 \n" // kMadd01 "movdqa %[kMadd11],%%xmm0 \n" // kMadd11 "movdqa %[kRound34],%%xmm1 \n" // kRound34 - "1: \n" + "1: \n" "movdqu (%[src_ptr]),%%xmm6 \n" "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm7 \n" "pavgb %%xmm6,%%xmm7 \n" @@ -602,17 +593,11 @@ void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, "lea 0x18(%[dst_ptr]),%[dst_ptr] \n" "sub $0x18,%[dst_width] \n" "jg 1b \n" - : [src_ptr] "+r"(src_ptr), - [dst_ptr] "+r"(dst_ptr), + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), [dst_width] "+r"(dst_width) - : [src_stride] "r"(src_stride), - [kShuf01] "m"(kShuf01), - [kShuf11] "m"(kShuf11), - [kShuf21] "m"(kShuf21), - [kMadd01] "m"(kMadd01), - [kMadd11] "m"(kMadd11), - [kRound34] "m"(kRound34), - [kMadd21] "m"(kMadd21) + : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), + [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), + [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } @@ -657,7 +642,7 @@ void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, "movdqa %[kShufAb1],%%xmm3 \n" "movdqa %[kShufAb2],%%xmm4 \n" "movdqa %[kScaleAb2],%%xmm5 \n" - "1: \n" + "1: \n" "movdqu (%[src_ptr]),%%xmm0 \n" "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm1 \n" "lea 0x10(%[src_ptr]),%[src_ptr] \n" @@ -677,13 +662,10 @@ void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, "lea 0x6(%[dst_ptr]),%[dst_ptr] \n" "sub $0x6,%[dst_width] \n" "jg 1b \n" - : [src_ptr] "+r"(src_ptr), - [dst_ptr] "+r"(dst_ptr), + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), [dst_width] "+r"(dst_width) - : [src_stride] "r"(src_stride), - [kShufAb0] "m"(kShufAb0), - [kShufAb1] "m"(kShufAb1), - [kShufAb2] "m"(kShufAb2), + : [src_stride] "r"(src_stride), [kShufAb0] "m"(kShufAb0), + [kShufAb1] "m"(kShufAb1), [kShufAb2] "m"(kShufAb2), [kScaleAb2] "m"(kScaleAb2) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } @@ -697,7 +679,7 @@ void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, "movdqa %[kShufAc3],%%xmm3 \n" "movdqa %[kScaleAc33],%%xmm4 \n" "pxor %%xmm5,%%xmm5 \n" - "1: \n" + "1: \n" "movdqu (%[src_ptr]),%%xmm0 \n" "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm6 \n" "movhlps %%xmm0,%%xmm1 \n" @@ -736,13 +718,10 @@ void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, "lea 0x6(%[dst_ptr]),%[dst_ptr] \n" "sub $0x6,%[dst_width] \n" "jg 1b \n" - : [src_ptr] "+r"(src_ptr), - [dst_ptr] "+r"(dst_ptr), + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), [dst_width] "+r"(dst_width) - : [src_stride] "r"(src_stride), - [kShufAc] "m"(kShufAc), - [kShufAc3] "m"(kShufAc3), - [kScaleAc33] "m"(kScaleAc33) + : [src_stride] "r"(src_stride), [kShufAc] "m"(kShufAc), + [kShufAc3] "m"(kShufAc3), [kScaleAc33] "m"(kScaleAc33) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } @@ -2914,14 +2893,9 @@ void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb, LABELALIGN "99: \n" - : [dst_argb] "+r"(dst_argb), - [src_argb] "+r"(src_argb), - [dst_width] "+rm"(dst_width), - [x0] "=&r"(x0), - [x1] "=&r"(x1) - : [x] "rm"(x), - [dx] "rm"(dx), - [kShuffleColARGB] "m"(kShuffleColARGB), + : [dst_argb] "+r"(dst_argb), [src_argb] "+r"(src_argb), + [dst_width] "+rm"(dst_width), [x0] "=&r"(x0), [x1] "=&r"(x1) + : [x] "rm"(x), [dx] "rm"(dx), [kShuffleColARGB] "m"(kShuffleColARGB), [kShuffleFractions] "m"(kShuffleFractions) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } diff --git a/source/scale_uv.cc b/source/scale_uv.cc index c0dc4b84c..095f96e12 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -589,7 +589,8 @@ static int ScaleUVBilinearUp(int src_width, // 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;) // so rowptr points to yi and (rowptr + rowstride) points to yi + 1. // 3. Advance src by 1 row if row yi + 2 exists ((y + 65536) < max_y), - // otherwise clamp src at (src_height - 1) to avoid reading out of bounds. + // otherwise clamp src at (src_height - 1) to avoid reading out of + // bounds. for (j = 0; j < dst_height; ++j) { if (y > max_y) { y = max_y; diff --git a/unit_test/convert_argb_test.cc b/unit_test/convert_argb_test.cc index dfffb77c8..a2e1ad547 100644 --- a/unit_test/convert_argb_test.cc +++ b/unit_test/convert_argb_test.cc @@ -1749,18 +1749,16 @@ TEST_F(LibYUVConvertTest, RotateWithARGBSource) { TEST_F(LibYUVConvertTest, ConvertToARGB_##FMT) { \ const int kWidth = 64; \ const int kHeight = 48; \ - const int kUvStride = \ - (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ + const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ - const int kSampleSize = \ - (kWidth * kHeight + kUvStride * kUvHeight) * BPC; \ + const int kSampleSize = (kWidth * kHeight + kUvStride * kUvHeight) * BPC; \ align_buffer_page_end(src, kSampleSize); \ align_buffer_page_end(dst, kWidth * kHeight * 4); \ MemRandomize(src, kSampleSize); \ memset(dst, 0, kWidth * kHeight * 4); \ - EXPECT_EQ(0, ConvertToARGB(src, kSampleSize, dst, kWidth * 4, 0, 0, \ - kWidth, kHeight, kWidth, kHeight, kRotate0, \ - FOURCC_##FMT)); \ + EXPECT_EQ( \ + 0, ConvertToARGB(src, kSampleSize, dst, kWidth * 4, 0, 0, kWidth, \ + kHeight, kWidth, kHeight, kRotate0, FOURCC_##FMT)); \ free_aligned_buffer_page_end(src); \ free_aligned_buffer_page_end(dst); \ } @@ -1773,8 +1771,7 @@ TESTCONVERTTOARGBSMOKE(NV24, 1, 1, 1) TEST_F(LibYUVConvertTest, ConvertToARGB_##FMT##N) { \ const int kWidth = W1280; \ const int kHeight = benchmark_height_; \ - const int kUvStride = \ - (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ + const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ const int kYPixels = kWidth * kHeight; \ const int kUvPixels = kUvStride * kUvHeight; \ @@ -1793,9 +1790,9 @@ TESTCONVERTTOARGBSMOKE(NV24, 1, 1, 1) EXPECT_EQ(0, FMT##ToARGBMatrix(src_p, kWidth, src_p + kYPixels, kUvStride, \ ref, kWidth * 4, &kYuvI601Constants, \ kWidth, NEG kHeight)); \ - EXPECT_EQ(0, ConvertToARGB(src, 0, dst, kWidth * 4, 0, 0, kWidth, \ - NEG kHeight, kWidth, kHeight, kRotate0, \ - FOURCC_##FMT)); \ + EXPECT_EQ( \ + 0, ConvertToARGB(src, 0, dst, kWidth * 4, 0, 0, kWidth, NEG kHeight, \ + kWidth, kHeight, kRotate0, FOURCC_##FMT)); \ for (int i = 0; i < kWidth * kHeight * 4; ++i) { \ ASSERT_EQ(dst[i], ref[i]); \ } \ @@ -1805,17 +1802,17 @@ TESTCONVERTTOARGBSMOKE(NV24, 1, 1, 1) } #if defined(ENABLE_FULL_TESTS) -#define TESTCONVERTTOARGB(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ - TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_ + 1, _Any, +) \ - TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_, _Invert, -) \ - TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_, _Opt, +) +#define TESTCONVERTTOARGB(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ + TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ + benchmark_width_ + 1, _Any, +) \ + TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \ + _Invert, -) \ + TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \ + _Opt, +) #else -#define TESTCONVERTTOARGB(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ - TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_, _Opt, +) +#define TESTCONVERTTOARGB(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ + TESTCONVERTTOARGBI(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \ + _Opt, +) #endif TESTCONVERTTOARGB(P010, uint16_t, 2, 2, 2) @@ -2910,9 +2907,10 @@ TEST_F(LibYUVConvertTest, P010ToARGBMatrixFilterOverflow) { align_buffer_page_end(dst_argb, dst_stride_argb * height); memset(src_y, 0x02, width * height * sizeof(uint16_t)); memset(src_uv, 0x02, src_stride_uv * src_uv_height * sizeof(uint16_t)); - EXPECT_EQ(0, P010ToARGBMatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb, - dst_stride_argb, &kYuvI601Constants, - width, height, kFilterBilinear)); + EXPECT_EQ( + 0, P010ToARGBMatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb, + dst_stride_argb, &kYuvI601Constants, width, + height, kFilterBilinear)); free_aligned_buffer_page_end_16(src_y); free_aligned_buffer_page_end_16(src_uv); free_aligned_buffer_page_end(dst_argb); @@ -2930,9 +2928,10 @@ TEST_F(LibYUVConvertTest, P210ToARGBMatrixFilterOverflow) { align_buffer_page_end(dst_argb, dst_stride_argb * height); memset(src_y, 0x02, width * height * sizeof(uint16_t)); memset(src_uv, 0x02, src_stride_uv * height * sizeof(uint16_t)); - EXPECT_EQ(0, P210ToARGBMatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb, - dst_stride_argb, &kYuvI601Constants, - width, height, kFilterBilinear)); + EXPECT_EQ( + 0, P210ToARGBMatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb, + dst_stride_argb, &kYuvI601Constants, width, + height, kFilterBilinear)); free_aligned_buffer_page_end_16(src_y); free_aligned_buffer_page_end_16(src_uv); free_aligned_buffer_page_end(dst_argb); @@ -2951,9 +2950,10 @@ TEST_F(LibYUVConvertTest, P010ToAR30MatrixFilterOverflow) { align_buffer_page_end(dst_argb, dst_stride_argb * height); memset(src_y, 0x02, width * height * sizeof(uint16_t)); memset(src_uv, 0x02, src_stride_uv * src_uv_height * sizeof(uint16_t)); - EXPECT_EQ(0, P010ToAR30MatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb, - dst_stride_argb, &kYuvI601Constants, - width, height, kFilterBilinear)); + EXPECT_EQ( + 0, P010ToAR30MatrixFilter(src_y, width, src_uv, src_stride_uv, dst_argb, + dst_stride_argb, &kYuvI601Constants, width, + height, kFilterBilinear)); free_aligned_buffer_page_end_16(src_y); free_aligned_buffer_page_end_16(src_uv); free_aligned_buffer_page_end(dst_argb); @@ -2971,9 +2971,10 @@ TEST_F(LibYUVConvertTest, P210ToAR30MatrixFilterOverflow) { align_buffer_page_end(dst_ar30, dst_stride_ar30 * height); memset(src_y, 0x02, width * height * sizeof(uint16_t)); memset(src_uv, 0x02, src_stride_uv * height * sizeof(uint16_t)); - EXPECT_EQ(0, P210ToAR30MatrixFilter(src_y, width, src_uv, src_stride_uv, dst_ar30, - dst_stride_ar30, &kYuvI601Constants, - width, height, kFilterBilinear)); + EXPECT_EQ( + 0, P210ToAR30MatrixFilter(src_y, width, src_uv, src_stride_uv, dst_ar30, + dst_stride_ar30, &kYuvI601Constants, width, + height, kFilterBilinear)); free_aligned_buffer_page_end_16(src_y); free_aligned_buffer_page_end_16(src_uv); free_aligned_buffer_page_end(dst_ar30); diff --git a/unit_test/convert_test.cc b/unit_test/convert_test.cc index e23fa0def..337c77442 100644 --- a/unit_test/convert_test.cc +++ b/unit_test/convert_test.cc @@ -471,127 +471,125 @@ TESTPLANARTOBP(I420, uint8_t, 1, 2, 2, P410, uint16_t, 2, 1, 1, 8) TESTPLANARTOBP(I410, uint16_t, 2, 1, 1, P410, uint16_t, 2, 1, 1, 10) TESTPLANARTOBP(I412, uint16_t, 2, 1, 1, P412, uint16_t, 2, 1, 1, 12) -#define TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, W1280, N, \ - NEG) \ - TEST_F(LibYUVConvertTest, ConvertFromI420_##FMT##N) { \ - const int kWidth = W1280; \ - const int kHeight = benchmark_height_; \ - const int kHalfWidth = SUBSAMPLE(kWidth, 2); \ - const int kHalfHeight = SUBSAMPLE(kHeight, 2); \ - const int kUvStride = \ - (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ - const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ - const int kSamplePixels = kWidth * kHeight + kUvStride * kUvHeight; \ - align_buffer_page_end(src_y, kWidth * kHeight); \ - align_buffer_page_end(src_u, kHalfWidth * kHalfHeight); \ - align_buffer_page_end(src_v, kHalfWidth * kHalfHeight); \ - align_buffer_page_end(dst, kSamplePixels * BPC); \ - align_buffer_page_end(ref, kSamplePixels * BPC); \ - MemRandomize(src_y, kWidth * kHeight); \ - MemRandomize(src_u, kHalfWidth * kHalfHeight); \ - MemRandomize(src_v, kHalfWidth * kHalfHeight); \ - memset(dst, 0, kSamplePixels * BPC); \ - memset(ref, 1, kSamplePixels * BPC); \ - TYPE* dst_p = reinterpret_cast(dst); \ - TYPE* ref_p = reinterpret_cast(ref); \ - EXPECT_EQ(0, I420To##FMT(src_y, kWidth, src_u, kHalfWidth, src_v, \ - kHalfWidth, ref_p, kWidth, \ - ref_p + kWidth * kHeight, kUvStride, kWidth, \ - NEG kHeight)); \ - EXPECT_EQ(0, ConvertFromI420(src_y, kWidth, src_u, kHalfWidth, src_v, \ - kHalfWidth, dst, 0, kWidth, NEG kHeight, \ - FOURCC_##FMT)); \ - for (int i = 0; i < kSamplePixels; ++i) { \ - ASSERT_EQ(dst_p[i], ref_p[i]); \ - } \ - if (BPC == 2) { \ - EXPECT_EQ(-1, ConvertFromI420(src_y, kWidth, src_u, kHalfWidth, src_v, \ - kHalfWidth, dst, 1, kWidth, kHeight, \ - FOURCC_##FMT)); \ - } \ - free_aligned_buffer_page_end(src_y); \ - free_aligned_buffer_page_end(src_u); \ - free_aligned_buffer_page_end(src_v); \ - free_aligned_buffer_page_end(dst); \ - free_aligned_buffer_page_end(ref); \ +#define TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, W1280, N, \ + NEG) \ + TEST_F(LibYUVConvertTest, ConvertFromI420_##FMT##N) { \ + const int kWidth = W1280; \ + const int kHeight = benchmark_height_; \ + const int kHalfWidth = SUBSAMPLE(kWidth, 2); \ + const int kHalfHeight = SUBSAMPLE(kHeight, 2); \ + const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ + const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ + const int kSamplePixels = kWidth * kHeight + kUvStride * kUvHeight; \ + align_buffer_page_end(src_y, kWidth * kHeight); \ + align_buffer_page_end(src_u, kHalfWidth * kHalfHeight); \ + align_buffer_page_end(src_v, kHalfWidth * kHalfHeight); \ + align_buffer_page_end(dst, kSamplePixels * BPC); \ + align_buffer_page_end(ref, kSamplePixels * BPC); \ + MemRandomize(src_y, kWidth * kHeight); \ + MemRandomize(src_u, kHalfWidth * kHalfHeight); \ + MemRandomize(src_v, kHalfWidth * kHalfHeight); \ + memset(dst, 0, kSamplePixels * BPC); \ + memset(ref, 1, kSamplePixels * BPC); \ + TYPE* dst_p = reinterpret_cast(dst); \ + TYPE* ref_p = reinterpret_cast(ref); \ + EXPECT_EQ(0, \ + I420To##FMT(src_y, kWidth, src_u, kHalfWidth, src_v, kHalfWidth, \ + ref_p, kWidth, ref_p + kWidth * kHeight, kUvStride, \ + kWidth, NEG kHeight)); \ + EXPECT_EQ(0, ConvertFromI420(src_y, kWidth, src_u, kHalfWidth, src_v, \ + kHalfWidth, dst, 0, kWidth, NEG kHeight, \ + FOURCC_##FMT)); \ + for (int i = 0; i < kSamplePixels; ++i) { \ + ASSERT_EQ(dst_p[i], ref_p[i]); \ + } \ + if (BPC == 2) { \ + EXPECT_EQ(-1, ConvertFromI420(src_y, kWidth, src_u, kHalfWidth, src_v, \ + kHalfWidth, dst, 1, kWidth, kHeight, \ + FOURCC_##FMT)); \ + } \ + free_aligned_buffer_page_end(src_y); \ + free_aligned_buffer_page_end(src_u); \ + free_aligned_buffer_page_end(src_v); \ + free_aligned_buffer_page_end(dst); \ + free_aligned_buffer_page_end(ref); \ } -#define TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, W1280, N, \ - NEG) \ - TEST_F(LibYUVConvertTest, ConvertToI420_##FMT##N) { \ - const int kWidth = W1280; \ - const int kHeight = benchmark_height_; \ - const int kHalfWidth = SUBSAMPLE(kWidth, 2); \ - const int kHalfHeight = SUBSAMPLE(kHeight, 2); \ - const int kUvStride = \ - (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ - const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ - const int kYPixels = kWidth * kHeight; \ - const int kUvPixels = kUvStride * kUvHeight; \ - align_buffer_page_end(src, (kYPixels + kUvPixels) * BPC); \ - align_buffer_page_end(dst_y, kWidth * kHeight); \ - align_buffer_page_end(dst_u, kHalfWidth * kHalfHeight); \ - align_buffer_page_end(dst_v, kHalfWidth * kHalfHeight); \ - align_buffer_page_end(ref_y, kWidth * kHeight); \ - align_buffer_page_end(ref_u, kHalfWidth * kHalfHeight); \ - align_buffer_page_end(ref_v, kHalfWidth * kHalfHeight); \ - TYPE* src_p = reinterpret_cast(src); \ - MemRandomize(src, (kYPixels + kUvPixels) * BPC); \ - if (BPC == 2) { \ - for (int i = 0; i < kYPixels + kUvPixels; ++i) { \ - src_p[i] = static_cast(src_p[i] & 0xffc0); \ - } \ - } \ - memset(dst_y, 1, kWidth * kHeight); \ - memset(dst_u, 2, kHalfWidth * kHalfHeight); \ - memset(dst_v, 3, kHalfWidth * kHalfHeight); \ - memset(ref_y, 4, kWidth * kHeight); \ - memset(ref_u, 5, kHalfWidth * kHalfHeight); \ - memset(ref_v, 6, kHalfWidth * kHalfHeight); \ - EXPECT_EQ(0, FMT##ToI420(src_p, kWidth, src_p + kYPixels, kUvStride, \ - ref_y, kWidth, ref_u, kHalfWidth, ref_v, \ - kHalfWidth, kWidth, NEG kHeight)); \ - EXPECT_EQ(0, ConvertToI420(src, 0, dst_y, kWidth, dst_u, kHalfWidth, \ - dst_v, kHalfWidth, 0, 0, kWidth, NEG kHeight,\ - kWidth, kHeight, kRotate0, FOURCC_##FMT)); \ - for (int i = 0; i < kWidth * kHeight; ++i) { \ - ASSERT_EQ(dst_y[i], ref_y[i]); \ - } \ - for (int i = 0; i < kHalfWidth * kHalfHeight; ++i) { \ - ASSERT_EQ(dst_u[i], ref_u[i]); \ - ASSERT_EQ(dst_v[i], ref_v[i]); \ - } \ - free_aligned_buffer_page_end(src); \ - free_aligned_buffer_page_end(dst_y); \ - free_aligned_buffer_page_end(dst_u); \ - free_aligned_buffer_page_end(dst_v); \ - free_aligned_buffer_page_end(ref_y); \ - free_aligned_buffer_page_end(ref_u); \ - free_aligned_buffer_page_end(ref_v); \ +#define TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, W1280, N, \ + NEG) \ + TEST_F(LibYUVConvertTest, ConvertToI420_##FMT##N) { \ + const int kWidth = W1280; \ + const int kHeight = benchmark_height_; \ + const int kHalfWidth = SUBSAMPLE(kWidth, 2); \ + const int kHalfHeight = SUBSAMPLE(kHeight, 2); \ + const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ + const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ + const int kYPixels = kWidth * kHeight; \ + const int kUvPixels = kUvStride * kUvHeight; \ + align_buffer_page_end(src, (kYPixels + kUvPixels) * BPC); \ + align_buffer_page_end(dst_y, kWidth * kHeight); \ + align_buffer_page_end(dst_u, kHalfWidth * kHalfHeight); \ + align_buffer_page_end(dst_v, kHalfWidth * kHalfHeight); \ + align_buffer_page_end(ref_y, kWidth * kHeight); \ + align_buffer_page_end(ref_u, kHalfWidth * kHalfHeight); \ + align_buffer_page_end(ref_v, kHalfWidth * kHalfHeight); \ + TYPE* src_p = reinterpret_cast(src); \ + MemRandomize(src, (kYPixels + kUvPixels) * BPC); \ + if (BPC == 2) { \ + for (int i = 0; i < kYPixels + kUvPixels; ++i) { \ + src_p[i] = static_cast(src_p[i] & 0xffc0); \ + } \ + } \ + memset(dst_y, 1, kWidth * kHeight); \ + memset(dst_u, 2, kHalfWidth * kHalfHeight); \ + memset(dst_v, 3, kHalfWidth * kHalfHeight); \ + memset(ref_y, 4, kWidth * kHeight); \ + memset(ref_u, 5, kHalfWidth * kHalfHeight); \ + memset(ref_v, 6, kHalfWidth * kHalfHeight); \ + EXPECT_EQ(0, FMT##ToI420(src_p, kWidth, src_p + kYPixels, kUvStride, \ + ref_y, kWidth, ref_u, kHalfWidth, ref_v, \ + kHalfWidth, kWidth, NEG kHeight)); \ + EXPECT_EQ(0, ConvertToI420(src, 0, dst_y, kWidth, dst_u, kHalfWidth, \ + dst_v, kHalfWidth, 0, 0, kWidth, NEG kHeight, \ + kWidth, kHeight, kRotate0, FOURCC_##FMT)); \ + for (int i = 0; i < kWidth * kHeight; ++i) { \ + ASSERT_EQ(dst_y[i], ref_y[i]); \ + } \ + for (int i = 0; i < kHalfWidth * kHalfHeight; ++i) { \ + ASSERT_EQ(dst_u[i], ref_u[i]); \ + ASSERT_EQ(dst_v[i], ref_v[i]); \ + } \ + free_aligned_buffer_page_end(src); \ + free_aligned_buffer_page_end(dst_y); \ + free_aligned_buffer_page_end(dst_u); \ + free_aligned_buffer_page_end(dst_v); \ + free_aligned_buffer_page_end(ref_y); \ + free_aligned_buffer_page_end(ref_u); \ + free_aligned_buffer_page_end(ref_v); \ } #if defined(ENABLE_FULL_TESTS) -#define TESTCONVERTFROMI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ - TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_ + 1, _Any, +) \ - TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_, _Invert, -) \ - TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_, _Opt, +) -#define TESTCONVERTTOI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ - TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_ + 1, _Any, +) \ - TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_, _Invert, -) \ - TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_, _Opt, +) +#define TESTCONVERTFROMI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ + TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ + benchmark_width_ + 1, _Any, +) \ + TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \ + _Invert, -) \ + TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \ + _Opt, +) +#define TESTCONVERTTOI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ + TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ + benchmark_width_ + 1, _Any, +) \ + TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \ + _Invert, -) \ + TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \ + _Opt, +) #else -#define TESTCONVERTFROMI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ - TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_, _Opt, +) -#define TESTCONVERTTOI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ - TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, \ - benchmark_width_, _Opt, +) +#define TESTCONVERTFROMI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ + TESTCONVERTFROMI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \ + _Opt, +) +#define TESTCONVERTTOI420(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y) \ + TESTCONVERTTOI420I(FMT, TYPE, BPC, SUBSAMP_X, SUBSAMP_Y, benchmark_width_, \ + _Opt, +) #endif TESTCONVERTFROMI420(P010, uint16_t, 2, 2, 2) @@ -601,56 +599,53 @@ TESTCONVERTTOI420(P010, uint16_t, 2, 2, 2) TESTCONVERTTOI420(P210, uint16_t, 2, 2, 1) TESTCONVERTTOI420(P410, uint16_t, 2, 1, 1) -#define TESTCONVERTTOI420SMOKE(FMT, BPC, SUBSAMP_X, SUBSAMP_Y) \ - TEST_F(LibYUVConvertTest, ConvertToI420_##FMT) { \ - const int kWidth = 64; \ - const int kHeight = 48; \ - const int kUvStride = \ - (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ - const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ - const int kSampleSize = \ - (kWidth * kHeight + kUvStride * kUvHeight) * BPC; \ - align_buffer_page_end(src, kSampleSize); \ - align_buffer_page_end(dst_y, kWidth * kHeight); \ - align_buffer_page_end(dst_u, (kWidth / 2) * (kHeight / 2)); \ - align_buffer_page_end(dst_v, (kWidth / 2) * (kHeight / 2)); \ - MemRandomize(src, kSampleSize); \ - memset(dst_y, 0, kWidth * kHeight); \ - memset(dst_u, 0, (kWidth / 2) * (kHeight / 2)); \ - memset(dst_v, 0, (kWidth / 2) * (kHeight / 2)); \ - EXPECT_EQ(0, ConvertToI420(src, kSampleSize, dst_y, kWidth, dst_u, \ - kWidth / 2, dst_v, kWidth / 2, 0, 0, kWidth, \ - kHeight, kWidth, kHeight, kRotate0, \ - FOURCC_##FMT)); \ - free_aligned_buffer_page_end(src); \ - free_aligned_buffer_page_end(dst_y); \ - free_aligned_buffer_page_end(dst_u); \ - free_aligned_buffer_page_end(dst_v); \ +#define TESTCONVERTTOI420SMOKE(FMT, BPC, SUBSAMP_X, SUBSAMP_Y) \ + TEST_F(LibYUVConvertTest, ConvertToI420_##FMT) { \ + const int kWidth = 64; \ + const int kHeight = 48; \ + const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ + const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ + const int kSampleSize = (kWidth * kHeight + kUvStride * kUvHeight) * BPC; \ + align_buffer_page_end(src, kSampleSize); \ + align_buffer_page_end(dst_y, kWidth * kHeight); \ + align_buffer_page_end(dst_u, (kWidth / 2) * (kHeight / 2)); \ + align_buffer_page_end(dst_v, (kWidth / 2) * (kHeight / 2)); \ + MemRandomize(src, kSampleSize); \ + memset(dst_y, 0, kWidth * kHeight); \ + memset(dst_u, 0, (kWidth / 2) * (kHeight / 2)); \ + memset(dst_v, 0, (kWidth / 2) * (kHeight / 2)); \ + EXPECT_EQ( \ + 0, ConvertToI420(src, kSampleSize, dst_y, kWidth, dst_u, kWidth / 2, \ + dst_v, kWidth / 2, 0, 0, kWidth, kHeight, kWidth, \ + kHeight, kRotate0, FOURCC_##FMT)); \ + free_aligned_buffer_page_end(src); \ + free_aligned_buffer_page_end(dst_y); \ + free_aligned_buffer_page_end(dst_u); \ + free_aligned_buffer_page_end(dst_v); \ } -#define TESTCONVERTFROMI420SMOKE(FMT, BPC, SUBSAMP_X, SUBSAMP_Y) \ - TEST_F(LibYUVConvertTest, ConvertFromI420_##FMT) { \ - const int kWidth = 64; \ - const int kHeight = 48; \ - const int kUvStride = \ - (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ - const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ - const int kDstSize = (kWidth * kHeight + kUvStride * kUvHeight) * BPC; \ - align_buffer_page_end(src_y, kWidth * kHeight); \ - align_buffer_page_end(src_u, (kWidth / 2) * (kHeight / 2)); \ - align_buffer_page_end(src_v, (kWidth / 2) * (kHeight / 2)); \ - align_buffer_page_end(dst, kDstSize); \ - MemRandomize(src_y, kWidth * kHeight); \ - MemRandomize(src_u, (kWidth / 2) * (kHeight / 2)); \ - MemRandomize(src_v, (kWidth / 2) * (kHeight / 2)); \ - memset(dst, 0, kDstSize); \ - EXPECT_EQ(0, ConvertFromI420(src_y, kWidth, src_u, kWidth / 2, src_v, \ - kWidth / 2, dst, kWidth * BPC, kWidth, \ - kHeight, FOURCC_##FMT)); \ - free_aligned_buffer_page_end(src_y); \ - free_aligned_buffer_page_end(src_u); \ - free_aligned_buffer_page_end(src_v); \ - free_aligned_buffer_page_end(dst); \ +#define TESTCONVERTFROMI420SMOKE(FMT, BPC, SUBSAMP_X, SUBSAMP_Y) \ + TEST_F(LibYUVConvertTest, ConvertFromI420_##FMT) { \ + const int kWidth = 64; \ + const int kHeight = 48; \ + const int kUvStride = (SUBSAMP_X == 1) ? kWidth * 2 : ((kWidth + 1) & ~1); \ + const int kUvHeight = SUBSAMPLE(kHeight, SUBSAMP_Y); \ + const int kDstSize = (kWidth * kHeight + kUvStride * kUvHeight) * BPC; \ + align_buffer_page_end(src_y, kWidth * kHeight); \ + align_buffer_page_end(src_u, (kWidth / 2) * (kHeight / 2)); \ + align_buffer_page_end(src_v, (kWidth / 2) * (kHeight / 2)); \ + align_buffer_page_end(dst, kDstSize); \ + MemRandomize(src_y, kWidth * kHeight); \ + MemRandomize(src_u, (kWidth / 2) * (kHeight / 2)); \ + MemRandomize(src_v, (kWidth / 2) * (kHeight / 2)); \ + memset(dst, 0, kDstSize); \ + EXPECT_EQ(0, ConvertFromI420(src_y, kWidth, src_u, kWidth / 2, src_v, \ + kWidth / 2, dst, kWidth * BPC, kWidth, \ + kHeight, FOURCC_##FMT)); \ + free_aligned_buffer_page_end(src_y); \ + free_aligned_buffer_page_end(src_u); \ + free_aligned_buffer_page_end(src_v); \ + free_aligned_buffer_page_end(dst); \ } TESTCONVERTTOI420SMOKE(NV16, 1, 2, 1) @@ -1039,95 +1034,94 @@ TESTATOBP(AYUV, 1, 4, NV12, 2, 2) TESTATOBP(AYUV, 1, 4, NV21, 2, 2) // Matrix API with I601 constants matches the non-matrix wrapper. -#define TESTATOBPMATRIX(FMT_A, BPP_A, FMT_B, SUBSAMP_X, SUBSAMP_Y) \ - TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_B##Matrix) { \ - const int kWidth = 16; \ - const int kHeight = 16; \ - const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \ - const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \ - align_buffer_page_end(src_argb, kWidth * kHeight * BPP_A); \ - align_buffer_page_end(dst_y, kWidth * kHeight); \ - align_buffer_page_end(dst_uv, kStrideUV * 2 * kHeightUV); \ - align_buffer_page_end(ref_y, kWidth * kHeight); \ - align_buffer_page_end(ref_uv, kStrideUV * 2 * kHeightUV); \ - \ - MemRandomize(src_argb, kWidth * kHeight * BPP_A); \ - \ +#define TESTATOBPMATRIX(FMT_A, BPP_A, FMT_B, SUBSAMP_X, SUBSAMP_Y) \ + TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_B##Matrix) { \ + const int kWidth = 16; \ + const int kHeight = 16; \ + const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \ + const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \ + align_buffer_page_end(src_argb, kWidth * kHeight * BPP_A); \ + align_buffer_page_end(dst_y, kWidth * kHeight); \ + align_buffer_page_end(dst_uv, kStrideUV * 2 * kHeightUV); \ + align_buffer_page_end(ref_y, kWidth * kHeight); \ + align_buffer_page_end(ref_uv, kStrideUV * 2 * kHeightUV); \ + \ + MemRandomize(src_argb, kWidth * kHeight * BPP_A); \ + \ FMT_A##To##FMT_B##Matrix(src_argb, kWidth * BPP_A, dst_y, kWidth, dst_uv, \ - kStrideUV * 2, &kArgbI601Constants, kWidth, \ - kHeight); \ + kStrideUV * 2, &kArgbI601Constants, kWidth, \ + kHeight); \ FMT_A##To##FMT_B(src_argb, kWidth * BPP_A, ref_y, kWidth, ref_uv, \ - kStrideUV * 2, kWidth, kHeight); \ - for (int i = 0; i < kWidth * kHeight; ++i) { \ - ASSERT_EQ(dst_y[i], ref_y[i]); \ - } \ - for (int i = 0; i < kStrideUV * 2 * kHeightUV; ++i) { \ - ASSERT_EQ(dst_uv[i], ref_uv[i]); \ - } \ - \ - free_aligned_buffer_page_end(src_argb); \ - free_aligned_buffer_page_end(dst_y); \ - free_aligned_buffer_page_end(dst_uv); \ - free_aligned_buffer_page_end(ref_y); \ - free_aligned_buffer_page_end(ref_uv); \ + kStrideUV * 2, kWidth, kHeight); \ + for (int i = 0; i < kWidth * kHeight; ++i) { \ + ASSERT_EQ(dst_y[i], ref_y[i]); \ + } \ + for (int i = 0; i < kStrideUV * 2 * kHeightUV; ++i) { \ + ASSERT_EQ(dst_uv[i], ref_uv[i]); \ + } \ + \ + free_aligned_buffer_page_end(src_argb); \ + free_aligned_buffer_page_end(dst_y); \ + free_aligned_buffer_page_end(dst_uv); \ + free_aligned_buffer_page_end(ref_y); \ + free_aligned_buffer_page_end(ref_uv); \ } TESTATOBPMATRIX(ARGB, 4, NV12, 2, 2) TESTATOBPMATRIX(ARGB, 4, NV16, 2, 1) TESTATOBPMATRIX(ARGB, 4, NV24, 1, 1) -#define TESTATOBPI16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \ - W1280, N, NEG, OFF) \ - TEST_F(LibYUVConvertTest, FMT_A##To##FMT_PLANAR##N) { \ - const int kWidth = W1280; \ - const int kHeight = benchmark_height_; \ - const int kStride = SUBSAMPLE(kWidth, SUB_A) * BPP_A; \ - const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \ - const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \ - align_buffer_page_end(src_argb, kStride * kHeight + OFF); \ - align_buffer_page_end(dst_y_c, kWidth * kHeight * 2); \ - align_buffer_page_end(dst_uv_c, kStrideUV * 2 * kHeightUV * 2); \ - align_buffer_page_end(dst_y_opt, kWidth * kHeight * 2); \ - align_buffer_page_end(dst_uv_opt, kStrideUV * 2 * kHeightUV * 2); \ - for (int i = 0; i < kHeight; ++i) \ - for (int j = 0; j < kStride; ++j) \ - src_argb[(i * kStride) + j + OFF] = (fastrand() & 0xff); \ - memset(dst_y_c, 1, kWidth * kHeight * 2); \ - memset(dst_uv_c, 2, kStrideUV * 2 * kHeightUV * 2); \ - memset(dst_y_opt, 101, kWidth * kHeight * 2); \ - memset(dst_uv_opt, 102, kStrideUV * 2 * kHeightUV * 2); \ - MaskCpuFlags(disable_cpu_flags_); \ - FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, \ - reinterpret_cast(dst_y_c), kWidth, \ - reinterpret_cast(dst_uv_c), \ - kStrideUV * 2, kWidth, NEG kHeight); \ - MaskCpuFlags(benchmark_cpu_info_); \ - for (int i = 0; i < benchmark_iterations_; ++i) { \ - FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, \ - reinterpret_cast(dst_y_opt), kWidth, \ - reinterpret_cast(dst_uv_opt), \ - kStrideUV * 2, kWidth, NEG kHeight); \ - } \ - const uint16_t* y_c = reinterpret_cast(dst_y_c); \ - const uint16_t* y_opt = reinterpret_cast(dst_y_opt); \ - const uint16_t* uv_c = reinterpret_cast(dst_uv_c); \ - const uint16_t* uv_opt = reinterpret_cast(dst_uv_opt); \ - for (int i = 0; i < kHeight; ++i) { \ - for (int j = 0; j < kWidth; ++j) { \ - ASSERT_EQ(y_c[i * kWidth + j], y_opt[i * kWidth + j]); \ - } \ - } \ - for (int i = 0; i < kHeightUV; ++i) { \ - for (int j = 0; j < kStrideUV * 2; ++j) { \ - ASSERT_EQ(uv_c[i * kStrideUV * 2 + j], \ - uv_opt[i * kStrideUV * 2 + j]); \ - } \ - } \ - free_aligned_buffer_page_end(dst_y_c); \ - free_aligned_buffer_page_end(dst_uv_c); \ - free_aligned_buffer_page_end(dst_y_opt); \ - free_aligned_buffer_page_end(dst_uv_opt); \ - free_aligned_buffer_page_end(src_argb); \ +#define TESTATOBPI16(FMT_A, SUB_A, BPP_A, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \ + W1280, N, NEG, OFF) \ + TEST_F(LibYUVConvertTest, FMT_A##To##FMT_PLANAR##N) { \ + const int kWidth = W1280; \ + const int kHeight = benchmark_height_; \ + const int kStride = SUBSAMPLE(kWidth, SUB_A) * BPP_A; \ + const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \ + const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \ + align_buffer_page_end(src_argb, kStride * kHeight + OFF); \ + align_buffer_page_end(dst_y_c, kWidth * kHeight * 2); \ + align_buffer_page_end(dst_uv_c, kStrideUV * 2 * kHeightUV * 2); \ + align_buffer_page_end(dst_y_opt, kWidth * kHeight * 2); \ + align_buffer_page_end(dst_uv_opt, kStrideUV * 2 * kHeightUV * 2); \ + for (int i = 0; i < kHeight; ++i) \ + for (int j = 0; j < kStride; ++j) \ + src_argb[(i * kStride) + j + OFF] = (fastrand() & 0xff); \ + memset(dst_y_c, 1, kWidth * kHeight * 2); \ + memset(dst_uv_c, 2, kStrideUV * 2 * kHeightUV * 2); \ + memset(dst_y_opt, 101, kWidth * kHeight * 2); \ + memset(dst_uv_opt, 102, kStrideUV * 2 * kHeightUV * 2); \ + MaskCpuFlags(disable_cpu_flags_); \ + FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, \ + reinterpret_cast(dst_y_c), kWidth, \ + reinterpret_cast(dst_uv_c), \ + kStrideUV * 2, kWidth, NEG kHeight); \ + MaskCpuFlags(benchmark_cpu_info_); \ + for (int i = 0; i < benchmark_iterations_; ++i) { \ + FMT_A##To##FMT_PLANAR(src_argb + OFF, kStride, \ + reinterpret_cast(dst_y_opt), kWidth, \ + reinterpret_cast(dst_uv_opt), \ + kStrideUV * 2, kWidth, NEG kHeight); \ + } \ + const uint16_t* y_c = reinterpret_cast(dst_y_c); \ + const uint16_t* y_opt = reinterpret_cast(dst_y_opt); \ + const uint16_t* uv_c = reinterpret_cast(dst_uv_c); \ + const uint16_t* uv_opt = reinterpret_cast(dst_uv_opt); \ + for (int i = 0; i < kHeight; ++i) { \ + for (int j = 0; j < kWidth; ++j) { \ + ASSERT_EQ(y_c[i * kWidth + j], y_opt[i * kWidth + j]); \ + } \ + } \ + for (int i = 0; i < kHeightUV; ++i) { \ + for (int j = 0; j < kStrideUV * 2; ++j) { \ + ASSERT_EQ(uv_c[i * kStrideUV * 2 + j], uv_opt[i * kStrideUV * 2 + j]); \ + } \ + } \ + free_aligned_buffer_page_end(dst_y_c); \ + free_aligned_buffer_page_end(dst_uv_c); \ + free_aligned_buffer_page_end(dst_y_opt); \ + free_aligned_buffer_page_end(dst_uv_opt); \ + free_aligned_buffer_page_end(src_argb); \ } #if defined(ENABLE_FULL_TESTS) @@ -1150,83 +1144,82 @@ TESTATOBP16(ARGB, 1, 4, P010, 2, 2) TESTATOBP16(ARGB, 1, 4, P210, 2, 1) TESTATOBP16(ARGB, 1, 4, P410, 1, 1) -#define TESTATOBPMATRIX16(FMT_A, BPP_A, FMT_B, SUBSAMP_X, SUBSAMP_Y) \ - TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_B##Matrix) { \ - const int kWidth = 16; \ - const int kHeight = 16; \ - const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \ - const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \ - align_buffer_page_end(src_argb, kWidth * kHeight * BPP_A); \ - align_buffer_page_end(dst_y, kWidth * kHeight * 2); \ - align_buffer_page_end(dst_uv, kStrideUV * 2 * kHeightUV * 2); \ - align_buffer_page_end(ref_y, kWidth * kHeight * 2); \ - align_buffer_page_end(ref_uv, kStrideUV * 2 * kHeightUV * 2); \ - \ - MemRandomize(src_argb, kWidth * kHeight * BPP_A); \ - \ - FMT_A##To##FMT_B##Matrix( \ - src_argb, kWidth * BPP_A, reinterpret_cast(dst_y), \ - kWidth, reinterpret_cast(dst_uv), kStrideUV * 2, \ - &kArgbI601Constants, kWidth, kHeight); \ - FMT_A##To##FMT_B(src_argb, kWidth * BPP_A, \ - reinterpret_cast(ref_y), kWidth, \ - reinterpret_cast(ref_uv), kStrideUV * 2, \ - kWidth, kHeight); \ - const uint16_t* y = reinterpret_cast(dst_y); \ - const uint16_t* y_ref = reinterpret_cast(ref_y); \ - const uint16_t* uv = reinterpret_cast(dst_uv); \ - const uint16_t* uv_ref = reinterpret_cast(ref_uv); \ - for (int i = 0; i < kWidth * kHeight; ++i) { \ - ASSERT_EQ(y[i], y_ref[i]); \ - } \ - for (int i = 0; i < kStrideUV * 2 * kHeightUV; ++i) { \ - ASSERT_EQ(uv[i], uv_ref[i]); \ - } \ - \ - free_aligned_buffer_page_end(src_argb); \ - free_aligned_buffer_page_end(dst_y); \ - free_aligned_buffer_page_end(dst_uv); \ - free_aligned_buffer_page_end(ref_y); \ - free_aligned_buffer_page_end(ref_uv); \ +#define TESTATOBPMATRIX16(FMT_A, BPP_A, FMT_B, SUBSAMP_X, SUBSAMP_Y) \ + TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_B##Matrix) { \ + const int kWidth = 16; \ + const int kHeight = 16; \ + const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \ + const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \ + align_buffer_page_end(src_argb, kWidth * kHeight * BPP_A); \ + align_buffer_page_end(dst_y, kWidth * kHeight * 2); \ + align_buffer_page_end(dst_uv, kStrideUV * 2 * kHeightUV * 2); \ + align_buffer_page_end(ref_y, kWidth * kHeight * 2); \ + align_buffer_page_end(ref_uv, kStrideUV * 2 * kHeightUV * 2); \ + \ + MemRandomize(src_argb, kWidth * kHeight * BPP_A); \ + \ + FMT_A##To##FMT_B##Matrix( \ + src_argb, kWidth * BPP_A, reinterpret_cast(dst_y), kWidth, \ + reinterpret_cast(dst_uv), kStrideUV * 2, \ + &kArgbI601Constants, kWidth, kHeight); \ + FMT_A##To##FMT_B( \ + src_argb, kWidth * BPP_A, reinterpret_cast(ref_y), kWidth, \ + reinterpret_cast(ref_uv), kStrideUV * 2, kWidth, kHeight); \ + const uint16_t* y = reinterpret_cast(dst_y); \ + const uint16_t* y_ref = reinterpret_cast(ref_y); \ + const uint16_t* uv = reinterpret_cast(dst_uv); \ + const uint16_t* uv_ref = reinterpret_cast(ref_uv); \ + for (int i = 0; i < kWidth * kHeight; ++i) { \ + ASSERT_EQ(y[i], y_ref[i]); \ + } \ + for (int i = 0; i < kStrideUV * 2 * kHeightUV; ++i) { \ + ASSERT_EQ(uv[i], uv_ref[i]); \ + } \ + \ + free_aligned_buffer_page_end(src_argb); \ + free_aligned_buffer_page_end(dst_y); \ + free_aligned_buffer_page_end(dst_uv); \ + free_aligned_buffer_page_end(ref_y); \ + free_aligned_buffer_page_end(ref_uv); \ } TESTATOBPMATRIX16(ARGB, 4, P010, 2, 2) TESTATOBPMATRIX16(ARGB, 4, P210, 2, 1) TESTATOBPMATRIX16(ARGB, 4, P410, 1, 1) -#define TESTATOBP16FROM8(FMT_A, FMT_16, FMT_8, SUBSAMP_X, SUBSAMP_Y) \ - TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_16##From##FMT_8) { \ - const int kWidth = 16; \ - const int kHeight = 16; \ - const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \ - const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \ - align_buffer_page_end(src_argb, kWidth * kHeight * 4); \ - align_buffer_page_end(dst8_y, kWidth * kHeight); \ - align_buffer_page_end(dst8_uv, kStrideUV * 2 * kHeightUV); \ - align_buffer_page_end(dst16_y, kWidth * kHeight * 2); \ - align_buffer_page_end(dst16_uv, kStrideUV * 2 * kHeightUV * 2); \ - MemRandomize(src_argb, kWidth * kHeight * 4); \ - FMT_A##To##FMT_8(src_argb, kWidth * 4, dst8_y, kWidth, dst8_uv, \ - kStrideUV * 2, kWidth, kHeight); \ - FMT_A##To##FMT_16(src_argb, kWidth * 4, \ - reinterpret_cast(dst16_y), kWidth, \ - reinterpret_cast(dst16_uv), kStrideUV * 2, \ - kWidth, kHeight); \ - const uint16_t* y16 = reinterpret_cast(dst16_y); \ - const uint16_t* uv16 = reinterpret_cast(dst16_uv); \ - for (int i = 0; i < kWidth * kHeight; ++i) { \ - ASSERT_EQ(y16[i], static_cast( \ - ((dst8_y[i] * 1024u * 0x0101u) >> 16) << 6)); \ - } \ - for (int i = 0; i < kStrideUV * 2 * kHeightUV; ++i) { \ - ASSERT_EQ(uv16[i], static_cast( \ - ((dst8_uv[i] * 1024u * 0x0101u) >> 16) << 6)); \ - } \ - free_aligned_buffer_page_end(src_argb); \ - free_aligned_buffer_page_end(dst8_y); \ - free_aligned_buffer_page_end(dst8_uv); \ - free_aligned_buffer_page_end(dst16_y); \ - free_aligned_buffer_page_end(dst16_uv); \ +#define TESTATOBP16FROM8(FMT_A, FMT_16, FMT_8, SUBSAMP_X, SUBSAMP_Y) \ + TEST_F(LibYUVConvertTest, Test##FMT_A##To##FMT_16##From##FMT_8) { \ + const int kWidth = 16; \ + const int kHeight = 16; \ + const int kStrideUV = SUBSAMPLE(kWidth, SUBSAMP_X); \ + const int kHeightUV = SUBSAMPLE(kHeight, SUBSAMP_Y); \ + align_buffer_page_end(src_argb, kWidth * kHeight * 4); \ + align_buffer_page_end(dst8_y, kWidth * kHeight); \ + align_buffer_page_end(dst8_uv, kStrideUV * 2 * kHeightUV); \ + align_buffer_page_end(dst16_y, kWidth * kHeight * 2); \ + align_buffer_page_end(dst16_uv, kStrideUV * 2 * kHeightUV * 2); \ + MemRandomize(src_argb, kWidth * kHeight * 4); \ + FMT_A##To##FMT_8(src_argb, kWidth * 4, dst8_y, kWidth, dst8_uv, \ + kStrideUV * 2, kWidth, kHeight); \ + FMT_A##To##FMT_16(src_argb, kWidth * 4, \ + reinterpret_cast(dst16_y), kWidth, \ + reinterpret_cast(dst16_uv), kStrideUV * 2, \ + kWidth, kHeight); \ + const uint16_t* y16 = reinterpret_cast(dst16_y); \ + const uint16_t* uv16 = reinterpret_cast(dst16_uv); \ + for (int i = 0; i < kWidth * kHeight; ++i) { \ + ASSERT_EQ(y16[i], static_cast( \ + ((dst8_y[i] * 1024u * 0x0101u) >> 16) << 6)); \ + } \ + for (int i = 0; i < kStrideUV * 2 * kHeightUV; ++i) { \ + ASSERT_EQ(uv16[i], static_cast( \ + ((dst8_uv[i] * 1024u * 0x0101u) >> 16) << 6)); \ + } \ + free_aligned_buffer_page_end(src_argb); \ + free_aligned_buffer_page_end(dst8_y); \ + free_aligned_buffer_page_end(dst8_uv); \ + free_aligned_buffer_page_end(dst16_y); \ + free_aligned_buffer_page_end(dst16_uv); \ } TESTATOBP16FROM8(ARGB, P010, NV12, 2, 2) diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 5fd37ac27..35dd3430c 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -1760,17 +1760,15 @@ TEST_F(LibYUVPlanarTest, TestDetileSplitUVPlane_Correctness) { memset(dst_v_two_stage, 0, uv_plane_size); memset(dst_v_opt, 0, uv_plane_size); - DetileSplitUVPlane(tile_uv, tile_width, dst_u_opt, uv_width / 2, - dst_v_opt, uv_width / 2, uv_width, - benchmark_height_, 16); + DetileSplitUVPlane(tile_uv, tile_width, dst_u_opt, uv_width / 2, dst_v_opt, + uv_width / 2, uv_width, benchmark_height_, 16); // Benchmark 2 step conversion for comparison. for (j = 0; j < benchmark_iterations_; j++) { - DetilePlane(tile_uv, tile_width, detiled_uv, uv_width, - uv_width, benchmark_height_, 16); - SplitUVPlane(detiled_uv, uv_width, dst_u_two_stage, - uv_width / 2, dst_v_two_stage, - uv_width / 2, uv_width / 2, + DetilePlane(tile_uv, tile_width, detiled_uv, uv_width, uv_width, + benchmark_height_, 16); + SplitUVPlane(detiled_uv, uv_width, dst_u_two_stage, uv_width / 2, + dst_v_two_stage, uv_width / 2, uv_width / 2, benchmark_height_); } @@ -1811,17 +1809,15 @@ TEST_F(LibYUVPlanarTest, TestDetileSplitUVPlane_Benchmark) { // Disable all optimizations. MaskCpuFlags(disable_cpu_flags_); - DetileSplitUVPlane(tile_uv, tile_width, dst_u_c, uv_width / 2, - dst_v_c, uv_width / 2, uv_width, - benchmark_height_, 16); + DetileSplitUVPlane(tile_uv, tile_width, dst_u_c, uv_width / 2, dst_v_c, + uv_width / 2, uv_width, benchmark_height_, 16); // Enable optimizations. MaskCpuFlags(benchmark_cpu_info_); for (j = 0; j < benchmark_iterations_; j++) { - DetileSplitUVPlane( - tile_uv, tile_width, dst_u_opt, uv_width / 2, dst_v_opt, - uv_width / 2, uv_width, benchmark_height_, 16); + DetileSplitUVPlane(tile_uv, tile_width, dst_u_opt, uv_width / 2, dst_v_opt, + uv_width / 2, uv_width, benchmark_height_, 16); } for (i = 0; i < uv_plane_size; ++i) { @@ -3772,9 +3768,9 @@ TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) { } #endif -#if defined(HAS_MULTIPLYROW_16_AVX512BW) || defined(HAS_MULTIPLYROW_16_AVX2) || \ - defined(HAS_MULTIPLYROW_16_NEON) || defined(HAS_MULTIPLYROW_16_SME) || \ - defined(HAS_MULTIPLYROW_16_RVV) +#if defined(HAS_MULTIPLYROW_16_AVX512BW) || \ + defined(HAS_MULTIPLYROW_16_AVX2) || defined(HAS_MULTIPLYROW_16_NEON) || \ + defined(HAS_MULTIPLYROW_16_SME) || defined(HAS_MULTIPLYROW_16_RVV) TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) { // Round count up to multiple of 64 const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63; @@ -3815,28 +3811,28 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) { } else #endif #if defined(HAS_MULTIPLYROW_16_AVX2) - if (has_avx2) { + if (has_avx2) { MultiplyRow_16_AVX2(reinterpret_cast(src_pixels_y), reinterpret_cast(dst_pixels_y_opt), 64, kPixels); } else #endif #if defined(HAS_MULTIPLYROW_16_SME) - if (has_sme) { + if (has_sme) { MultiplyRow_16_SME(reinterpret_cast(src_pixels_y), reinterpret_cast(dst_pixels_y_opt), 64, kPixels); } else #endif #if defined(HAS_MULTIPLYROW_16_NEON) - if (has_neon) { + if (has_neon) { MultiplyRow_16_NEON(reinterpret_cast(src_pixels_y), reinterpret_cast(dst_pixels_y_opt), 64, kPixels); } else #endif #if defined(HAS_MULTIPLYROW_16_RVV) - if (has_rvv) { + if (has_rvv) { MultiplyRow_16_RVV(reinterpret_cast(src_pixels_y), reinterpret_cast(dst_pixels_y_opt), 64, kPixels); @@ -3858,8 +3854,8 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) { free_aligned_buffer_page_end(dst_pixels_y_c); } -#if defined(HAS_MULTIPLYROW_16_AVX512BW) || defined(HAS_MULTIPLYROW_16_AVX2) || \ - defined(HAS_MULTIPLYROW_16_NEON) +#if defined(HAS_MULTIPLYROW_16_AVX512BW) || \ + defined(HAS_MULTIPLYROW_16_AVX2) || defined(HAS_MULTIPLYROW_16_NEON) TEST_F(LibYUVPlanarTest, MultiplyRow_16_Any) { const int kMaxPixels = 256; align_buffer_page_end(src_pixels_y, kMaxPixels * 2); @@ -3889,28 +3885,29 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Any) { #if defined(HAS_MULTIPLYROW_16_AVX512BW) if (has_avx512) { - MultiplyRow_16_Any_AVX512BW(reinterpret_cast(src_pixels_y), - reinterpret_cast(dst_pixels_y_opt), - 64, width); + MultiplyRow_16_Any_AVX512BW( + reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), 64, width); } else #endif #if defined(HAS_MULTIPLYROW_16_AVX2) - if (has_avx2) { + if (has_avx2) { MultiplyRow_16_Any_AVX2(reinterpret_cast(src_pixels_y), - reinterpret_cast(dst_pixels_y_opt), - 64, width); + reinterpret_cast(dst_pixels_y_opt), 64, + width); } else #endif #if defined(HAS_MULTIPLYROW_16_NEON) - if (has_neon) { + if (has_neon) { MultiplyRow_16_Any_NEON(reinterpret_cast(src_pixels_y), - reinterpret_cast(dst_pixels_y_opt), - 64, width); + reinterpret_cast(dst_pixels_y_opt), 64, + width); } else #endif { MultiplyRow_16_C(reinterpret_cast(src_pixels_y), - reinterpret_cast(dst_pixels_y_opt), 64, width); + reinterpret_cast(dst_pixels_y_opt), 64, + width); } for (int i = 0; i < width * 2; ++i) { @@ -3922,7 +3919,8 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Any) { free_aligned_buffer_page_end(dst_pixels_y_opt); free_aligned_buffer_page_end(dst_pixels_y_c); } -#endif // HAS_MULTIPLYROW_16_AVX512BW || HAS_MULTIPLYROW_16_AVX2 || HAS_MULTIPLYROW_16_NEON +#endif // HAS_MULTIPLYROW_16_AVX512BW || HAS_MULTIPLYROW_16_AVX2 || + // HAS_MULTIPLYROW_16_NEON #endif // HAS_MULTIPLYROW_16_... TEST_F(LibYUVPlanarTest, Convert16To8Plane) { @@ -4092,8 +4090,10 @@ TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) { } #endif // HAS_CONVERT16TO8ROW_AVX2 -#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) || defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \ - defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) || defined(HAS_HALFWIDTHROW_16TO8_NEON) +#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) || \ + defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) || \ + defined(HAS_HALFWIDTHROW_16TO8_NEON) TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Opt) { const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63; align_buffer_page_end_16(src_pixels_uv, kPixels * 4); @@ -4131,19 +4131,19 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Opt) { } else #endif #if defined(HAS_HALFWIDTHROW_16TO8_AVX2) - if (has_avx2) { + if (has_avx2) { HalfWidthRow_16To8_AVX2(src_pixels_uv, kPixels * 2, dst_pixels_uv_opt, 16384, kPixels); } else #endif #if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) - if (has_ssse3) { + if (has_ssse3) { HalfWidthRow_16To8_SSSE3(src_pixels_uv, kPixels * 2, dst_pixels_uv_opt, 16384, kPixels); } else #endif #if defined(HAS_HALFWIDTHROW_16TO8_NEON) - if (has_neon) { + if (has_neon) { HalfWidthRow_16To8_NEON(src_pixels_uv, kPixels * 2, dst_pixels_uv_opt, 16384, kPixels); } else @@ -4200,19 +4200,19 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) { } else #endif #if defined(HAS_HALFWIDTHROW_16TO8_AVX2) - if (has_avx2) { + if (has_avx2) { HalfWidthRow_16To8_Any_AVX2(src_pixels_uv, kMaxPixels * 2, dst_pixels_uv_opt, 16384, width); } else #endif #if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) - if (has_ssse3) { + if (has_ssse3) { HalfWidthRow_16To8_Any_SSSE3(src_pixels_uv, kMaxPixels * 2, dst_pixels_uv_opt, 16384, width); } else #endif #if defined(HAS_HALFWIDTHROW_16TO8_NEON) - if (has_neon) { + if (has_neon) { HalfWidthRow_16To8_Any_NEON(src_pixels_uv, kMaxPixels * 2, dst_pixels_uv_opt, 16384, width); } else @@ -4276,8 +4276,7 @@ TEST_F(LibYUVPlanarTest, Convert8To16Plane) { MaskCpuFlags(disable_cpu_flags_); Convert8To16Plane(src_pixels_y, benchmark_width_, reinterpret_cast(dst_pixels_y_c), - benchmark_width_, 10, benchmark_width_, - benchmark_height_); + benchmark_width_, 10, benchmark_width_, benchmark_height_); MaskCpuFlags(benchmark_cpu_info_); for (int i = 0; i < benchmark_iterations_; ++i) { @@ -4325,7 +4324,7 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) { 10, kPixels); } else #endif - if (has_avx2) { + if (has_avx2) { Convert8To16Row_AVX2(src_pixels_y, reinterpret_cast(dst_pixels_y_opt), 10, kPixels); diff --git a/unit_test/scale_argb_test.cc b/unit_test/scale_argb_test.cc index 09b23698b..2fd4d33ea 100644 --- a/unit_test/scale_argb_test.cc +++ b/unit_test/scale_argb_test.cc @@ -604,7 +604,8 @@ TEST_F(LibYUVScaleTest, ARGBScale_LargeHeightOverflow) { align_buffer_page_end(dest_pixels, src_w * 2 * 4); memset(orig_pixels, 128, src_w * src_h * 4); - // Test both vertical-only scaling (dst_w == src_w) and 2D scaling (dst_w != src_w). + // Test both vertical-only scaling (dst_w == src_w) and 2D scaling (dst_w != + // src_w). for (int dst_w : {4, 2}) { int res = ARGBScale(orig_pixels, src_w * 4, src_w, src_h, dest_pixels, dst_w * 4, dst_w, 2, kFilterBilinear); diff --git a/util/cpuid.c b/util/cpuid.c index a789e061a..a2f6e2054 100644 --- a/util/cpuid.c +++ b/util/cpuid.c @@ -14,9 +14,9 @@ #ifdef __linux__ #include -#include -#include #include +#include +#include #endif #include "libyuv/cpu_id.h" @@ -230,7 +230,10 @@ int main(int argc, const char* argv[]) { if (sigsetjmp(vdpphps_jmpbuf, 1) == 0) { // VDPPHPS xmm0, xmm0, xmm0 - __asm__ volatile(".byte 0x62, 0xf2, 0x7c, 0x08, 0x52, 0xc0" : : : "xmm0"); + __asm__ volatile(".byte 0x62, 0xf2, 0x7c, 0x08, 0x52, 0xc0" + : + : + : "xmm0"); printf("Works!\n"); } else { printf("Crashed (SIGILL)!\n");