diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 0e167b269..c5ea28a09 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -153,13 +153,11 @@ extern "C" { #define HAS_ARGBCOPYALPHAROW_AVX2 #define HAS_ARGBCOPYYTOALPHAROW_AVX2 #define HAS_ARGBEXTRACTALPHAROW_AVX2 -#define HAS_ARGBMIRRORROW_AVX2 #define HAS_ARGBPOLYNOMIALROW_AVX2 #define HAS_ARGBSHUFFLEROW_AVX2 #define HAS_ARGBTORGB565DITHERROW_AVX2 #define HAS_COPYROW_AVX #define HAS_HALFFLOATROW_AVX2 -#define HAS_I422TOARGBROW_AVX2 #define HAS_I422TORGB24ROW_AVX2 #define HAS_I422TORGBAROW_AVX2 #define HAS_I422TORGB565ROW_AVX2 @@ -168,12 +166,8 @@ extern "C" { #define HAS_I444TOARGBROW_AVX2 #define HAS_I444TORGB24ROW_AVX2 #define HAS_J400TOARGBROW_AVX2 -#define HAS_MIRRORROW_AVX2 -#define HAS_MIRRORSPLITUVROW_AVX2 -#define HAS_NV12TOARGBROW_AVX2 #define HAS_NV12TORGB24ROW_AVX2 #define HAS_NV12TORGB565ROW_AVX2 -#define HAS_NV21TOARGBROW_AVX2 #define HAS_NV21TORGB24ROW_AVX2 #define HAS_SPLITUVROW_AVX2 #define HAS_UYVYTOARGBROW_AVX2 @@ -474,7 +468,6 @@ extern "C" { #define HAS_RGBTOUV444MATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512VBMI -#define HAS_RAWTORGB24ROW_AVX512VBMI #define HAS_I422TORGB24ROW_AVX512BW #define HAS_ARGBTOUVMATRIXROW_AVX512BW #define HAS_J400TOARGBROW_AVX512BW @@ -796,6 +789,7 @@ extern "C" { #define HAS_P210TOARGBROW_SME #define HAS_P410TOAR30ROW_SME #define HAS_P410TOARGBROW_SME +#define HAS_UYVYTOARGBROW_SME #define HAS_YUY2TOARGBROW_SME #endif diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 7e0ca8776..669de43ff 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -466,7 +466,7 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y, "ptrue p0.b \n" "dup z19.b, #255 \n" // Alpha YUVTORGB_SVE_SETUP - "cmp %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" "mov z1.h, #128 \n" // U/V "umullb z6.h, z30.b, z1.b \n" "umullb z4.h, z28.b, z1.b \n" // DB @@ -475,25 +475,28 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y, "sub z4.h, z4.h, z25.h \n" "sub z5.h, z5.h, z27.h \n" "sub z6.h, z26.h, z6.h \n" - "b.le 2f \n" + "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p1.h \n" - "sub %w[width], %w[width], %w[vl] \n" "1: \n" // READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE "subs %w[width], %w[width], %w[vl] \n" "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" - "b.gt 1b \n" - "add %w[width], %w[width], %w[vl] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "2: \n" "whilelt p1.h, wzr, %w[width] \n" // READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + + "99: \n" : [src_y] "+r"(src_y), // %[src_y] [dst_argb] "+r"(dst_argb), // %[dst_argb] [width] "+r"(width), // %[width] @@ -702,7 +705,11 @@ static inline void I422ToARGB1555Row_SVE_SC( "whilelt p1.b, wzr, %w[width] \n" // READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X RGB8TOARGB1555_SVE_FROM_TOP_2X - "st2h {z0.h, z1.h}, p1, [%[dst]] \n" + // Need to permute the data on the final iteration such that the + // predicates (.b) line up with the 16-bit element data. + "trn1 z2.b, z0.b, z1.b \n" + "trn2 z3.b, z0.b, z1.b \n" + "st2b {z2.b, z3.b}, p1, [%[dst]] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -761,7 +768,11 @@ static inline void I422ToARGB4444Row_SVE_SC( "whilelt p1.b, wzr, %w[width] \n" // READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X RGB8TOARGB4444_SVE_FROM_TOP_2X - "st2h {z0.h, z1.h}, p1, [%[dst]] \n" + // Need to permute the data on the final iteration such that the + // predicates (.b) line up with the 16-bit element data. + "trn1 z2.b, z0.b, z1.b \n" + "trn2 z3.b, z0.b, z1.b \n" + "st2b {z2.b, z3.b}, p1, [%[dst]] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -788,7 +799,7 @@ static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y, YUVTORGB_SVE_SETUP "dup z19.b, #255 \n" // Alpha "subs %w[width], %w[width], %w[vl] \n" - "b.le 2f \n" + "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. @@ -798,7 +809,7 @@ static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y, "subs %w[width], %w[width], %w[vl] \n" "st2h {z19.h, z20.h}, p1, [%[dst_rgba]] \n" "add %[dst_rgba], %[dst_rgba], %[vl], lsl #2 \n" - "b.gt 1b \n" + "b.ge 1b \n" // Calculate a predicate for the final iteration to deal with the tail. "2: \n" @@ -836,7 +847,7 @@ static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y, YUVTORGB_SVE_SETUP_AR30 "dup z23.h, %w[limit] \n" "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.le 2f \n" + "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. @@ -846,7 +857,7 @@ static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y, "1: \n" // READYUV422_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.gt 1b \n" + "b.ge 1b \n" // Calculate a predicate for the final iteration to deal with the tail. "2: \n" @@ -2120,7 +2131,7 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb, "ptrue p5.h \n" "1: \n" // ARGBTOUVMATRIX_SVE - "b.gt 1b \n" + "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl], lsl #2 \n" @@ -2197,7 +2208,7 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, "ptrue p1.s \n" "1: \n" // ARGBTOYMATRIX_SVE("p1", "p1", "p1", "p1", "p0") - "b.gt 1b \n" + "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl], lsl #2 \n" diff --git a/source/convert.cc b/source/convert.cc index 9a62bb920..246f36ff5 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -2669,7 +2669,8 @@ int AYUVToNV12(const uint8_t* src_ayuv, } #endif #if defined(HAS_AYUVTOYROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { AYUVToUVRow = AYUVToUVRow_AVX512VBMI; AYUVToYRow = AYUVToYRow_AVX512VBMI; } @@ -2755,7 +2756,8 @@ int AYUVToNV21(const uint8_t* src_ayuv, } #endif #if defined(HAS_AYUVTOYROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { AYUVToVURow = AYUVToVURow_AVX512VBMI; AYUVToYRow = AYUVToYRow_AVX512VBMI; } diff --git a/source/convert_argb.cc b/source/convert_argb.cc index 6bcc7e253..136d2bf30 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -4133,7 +4133,8 @@ int RAWToRGBA(const uint8_t* src_raw, } #endif #if defined(HAS_RAWTORGBAROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { RAWToRGBARow = RAWToRGBARow_AVX512VBMI; } #endif @@ -4990,7 +4991,8 @@ int NV12ToRGB24Matrix(const uint8_t* src_y, } #endif #if defined(HAS_NV12TORGB24ROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { NV12ToRGB24Row = NV12ToRGB24Row_AVX512VBMI; } #endif @@ -5072,7 +5074,8 @@ int NV21ToRGB24Matrix(const uint8_t* src_y, } #endif #if defined(HAS_NV21TORGB24ROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { NV21ToRGB24Row = NV21ToRGB24Row_AVX512VBMI; } #endif diff --git a/source/convert_from.cc b/source/convert_from.cc index f454c1704..094a3c162 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -1028,6 +1028,7 @@ int ConvertFromI420(const uint8_t* y, height == INT_MIN) { return -1; } + const int abs_height = Abs(height); switch (format) { // Single plane formats case FOURCC_YUY2: @@ -1097,7 +1098,7 @@ int ConvertFromI420(const uint8_t* y, break; case FOURCC_NV12: { int dst_y_stride = dst_sample_stride ? dst_sample_stride : width; - uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height; + uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height; r = I420ToNV12(y, y_stride, u, u_stride, v, v_stride, dst_sample, dst_sample_stride ? dst_sample_stride : width, dst_uv, dst_sample_stride ? dst_sample_stride : width, width, @@ -1106,7 +1107,7 @@ int ConvertFromI420(const uint8_t* y, } case FOURCC_NV21: { int dst_y_stride = dst_sample_stride ? dst_sample_stride : width; - uint8_t* dst_vu = dst_sample + (ptrdiff_t)dst_y_stride * height; + uint8_t* dst_vu = dst_sample + (ptrdiff_t)dst_y_stride * abs_height; r = I420ToNV21(y, y_stride, u, u_stride, v, v_stride, dst_sample, dst_sample_stride ? dst_sample_stride : width, dst_vu, dst_sample_stride ? dst_sample_stride : width, width, @@ -1116,13 +1117,13 @@ int ConvertFromI420(const uint8_t* y, case FOURCC_NV16: { int dst_y_stride = dst_sample_stride ? dst_sample_stride : width; int dst_uv_stride = dst_sample_stride ? dst_sample_stride : width; - uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height; + uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height; int halfwidth = (width + 1) / 2; - int halfheight = (height + 1) / 2; + int halfheight = SUBSAMPLE(height, 1, 1); CopyPlane(y, y_stride, dst_sample, dst_y_stride, width, height); MergeUVPlane(u, u_stride, v, v_stride, dst_uv, dst_uv_stride * 2, halfwidth, halfheight); - if (height > 1) { + if (abs_height > 1) { MergeUVPlane(u, u_stride, v, v_stride, dst_uv + dst_uv_stride, dst_uv_stride * 2, halfwidth, height / 2); } @@ -1132,9 +1133,9 @@ int ConvertFromI420(const uint8_t* y, case FOURCC_NV24: { int dst_y_stride = dst_sample_stride ? dst_sample_stride : width; int dst_uv_stride = dst_sample_stride ? dst_sample_stride * 2 : width * 2; - uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height; - align_buffer_64(temp_u, (size_t)width * height); - align_buffer_64(temp_v, (size_t)width * height); + uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height; + align_buffer_64(temp_u, (size_t)width * abs_height); + align_buffer_64(temp_v, (size_t)width * abs_height); if (!temp_u || !temp_v) { free_aligned_buffer_64(temp_u); free_aligned_buffer_64(temp_v); @@ -1144,7 +1145,7 @@ int ConvertFromI420(const uint8_t* y, dst_y_stride, temp_u, width, temp_v, width, width, height); if (r == 0) { MergeUVPlane(temp_u, width, temp_v, width, dst_uv, dst_uv_stride, width, - height); + abs_height); } free_aligned_buffer_64(temp_u); free_aligned_buffer_64(temp_v); @@ -1152,7 +1153,6 @@ int ConvertFromI420(const uint8_t* y, } case FOURCC_P010: case FOURCC_P210: { - const int abs_height = Abs(height); const int uv_min = ((width + 1) / 2) * 2; int dst_stride_y; int dst_stride_uv; @@ -1183,7 +1183,6 @@ int ConvertFromI420(const uint8_t* y, break; } case FOURCC_P410: { - const int abs_height = Abs(height); const int uv_min = width * 2; int dst_stride_y; int dst_stride_uv; @@ -1213,14 +1212,14 @@ int ConvertFromI420(const uint8_t* y, case FOURCC_YV12: { dst_sample_stride = dst_sample_stride ? dst_sample_stride : width; int halfstride = (dst_sample_stride + 1) / 2; - int halfheight = (height + 1) / 2; + int halfheight = (abs_height + 1) / 2; uint8_t* dst_u; uint8_t* dst_v; if (format == FOURCC_YV12) { - dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height; + dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; dst_u = dst_v + (ptrdiff_t)halfstride * halfheight; } else { - dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height; + dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; dst_v = dst_u + (ptrdiff_t)halfstride * halfheight; } r = I420Copy(y, y_stride, u, u_stride, v, v_stride, dst_sample, @@ -1235,11 +1234,11 @@ int ConvertFromI420(const uint8_t* y, uint8_t* dst_u; uint8_t* dst_v; if (format == FOURCC_YV16) { - dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height; - dst_u = dst_v + (ptrdiff_t)halfstride * height; + dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; + dst_u = dst_v + (ptrdiff_t)halfstride * abs_height; } else { - dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height; - dst_v = dst_u + (ptrdiff_t)halfstride * height; + dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; + dst_v = dst_u + (ptrdiff_t)halfstride * abs_height; } r = I420ToI422(y, y_stride, u, u_stride, v, v_stride, dst_sample, dst_sample_stride, dst_u, halfstride, dst_v, halfstride, @@ -1252,11 +1251,11 @@ int ConvertFromI420(const uint8_t* y, uint8_t* dst_u; uint8_t* dst_v; if (format == FOURCC_YV24) { - dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height; - dst_u = dst_v + (ptrdiff_t)dst_sample_stride * height; + dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; + dst_u = dst_v + (ptrdiff_t)dst_sample_stride * abs_height; } else { - dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height; - dst_v = dst_u + (ptrdiff_t)dst_sample_stride * height; + dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; + dst_v = dst_u + (ptrdiff_t)dst_sample_stride * abs_height; } r = I420ToI444(y, y_stride, u, u_stride, v, v_stride, dst_sample, dst_sample_stride, dst_u, dst_sample_stride, dst_v, diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index 36bbe6b46..113ee6e64 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -433,7 +433,7 @@ int ConvertToARGB(const uint8_t* sample, src = sample + ((ptrdiff_t)src_width * crop_y + crop_x); src_uv = sample + aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) + - crop_x; + (crop_x & ~1); r = NV12ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; @@ -441,7 +441,7 @@ int ConvertToARGB(const uint8_t* sample, src = sample + ((ptrdiff_t)src_width * crop_y + crop_x); src_uv = sample + aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) + - crop_x; + (crop_x & ~1); // Call NV12 but with u and v parameters swapped. r = NV21ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); @@ -508,12 +508,12 @@ int ConvertToARGB(const uint8_t* sample, int halfheight = (abs_src_height + 1) / 2; if (format == FOURCC_YV12) { src_v = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; src_u = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; } else { src_u = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; } @@ -527,7 +527,7 @@ int ConvertToARGB(const uint8_t* sample, int halfheight = (abs_src_height + 1) / 2; const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; @@ -541,7 +541,7 @@ int ConvertToARGB(const uint8_t* sample, int halfheight = (abs_src_height + 1) / 2; const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; @@ -555,7 +555,7 @@ int ConvertToARGB(const uint8_t* sample, int halfheight = (abs_src_height + 1) / 2; const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; @@ -620,7 +620,7 @@ int ConvertToARGB(const uint8_t* sample, const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + crop_x / 2; - r = H422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, + r = U422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; } diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 3645afb4f..742786e36 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -1595,7 +1595,8 @@ void MergeRGBPlane(const uint8_t* src_r, } #endif #if defined(HAS_MERGERGBROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { MergeRGBRow = MergeRGBRow_AVX512VBMI; } #endif @@ -1799,11 +1800,13 @@ void SplitARGBPlane(const uint8_t* src_argb, dst_r = dst_r + (ptrdiff_t)(height - 1) * dst_stride_r; dst_g = dst_g + (ptrdiff_t)(height - 1) * dst_stride_g; dst_b = dst_b + (ptrdiff_t)(height - 1) * dst_stride_b; - dst_a = dst_a + (ptrdiff_t)(height - 1) * dst_stride_a; dst_stride_r = -dst_stride_r; dst_stride_g = -dst_stride_g; dst_stride_b = -dst_stride_b; - dst_stride_a = -dst_stride_a; + if (dst_a) { + dst_a = dst_a + (ptrdiff_t)(height - 1) * dst_stride_a; + dst_stride_a = -dst_stride_a; + } } if (dst_a == NULL) { @@ -2030,7 +2033,8 @@ void MergeXR30Plane(const uint16_t* src_r, } #endif #if defined(HAS_MERGEXR30ROW_10_AVX512BW) - if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL)) { + if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { MergeXR30Row = MergeXR30Row_10_AVX512BW; } #endif diff --git a/source/rotate.cc b/source/rotate.cc index d132638f2..1fbc6b32b 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -927,6 +927,7 @@ int I422Rotate(const uint8_t* src_y, // Negative height means invert the image. if (height < 0) { height = -height; + halfheight = (height + 1) >> 1; src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u; src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v; @@ -1234,9 +1235,10 @@ int I010Rotate(const uint16_t* src_y, // Negative height means invert the image. if (height < 0) { height = -height; + halfheight = (height + 1) >> 1; src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; - src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u; - src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v; + src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u; + src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v; src_stride_y = -src_stride_y; src_stride_u = -src_stride_u; src_stride_v = -src_stride_v; @@ -1305,6 +1307,7 @@ int I210Rotate(const uint16_t* src_y, // Negative height means invert the image. if (height < 0) { height = -height; + halfheight = (height + 1) >> 1; src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u; src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v; diff --git a/source/rotate_argb.cc b/source/rotate_argb.cc index 9421dfa39..04b17268e 100644 --- a/source/rotate_argb.cc +++ b/source/rotate_argb.cc @@ -78,6 +78,9 @@ static int ARGBTranspose(const uint8_t* src_argb, #if defined(HAS_TRANSPOSEWX4_32_SVE2) if (TestCpuFlag(kCpuHasSVE2)) { TransposeWx4_32 = TransposeWx4_32_SVE2; +#if defined(HAS_TRANSPOSEWXH_32_NEON) + TransposeWxH_32 = NULL; +#endif } #endif #if defined(HAS_TRANSPOSEWXH_32_SME) diff --git a/source/row_any.cc b/source/row_any.cc index e1805ab3b..e54205c6c 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1575,7 +1575,7 @@ ANY11T(ARGBToAB64Row_Any_SSSE3, ARGBToAB64Row_SSSE3, 4, 8, uint8_t, uint16_t, 3) ANY11T(AR64ToARGBRow_Any_SSSE3, AR64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3) #endif -#ifdef HAS_ARGBTOAR64ROW_SSSE3 +#ifdef HAS_AB64TOARGBROW_SSSE3 ANY11T(AB64ToARGBRow_Any_SSSE3, AB64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3) #endif @@ -1591,7 +1591,7 @@ ANY11T(ARGBToAB64Row_Any_AVX2, ARGBToAB64Row_AVX2, 4, 8, uint8_t, uint16_t, 7) ANY11T(AR64ToARGBRow_Any_AVX2, AR64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7) #endif -#ifdef HAS_ARGBTOAR64ROW_AVX2 +#ifdef HAS_AB64TOARGBROW_AVX2 ANY11T(AB64ToARGBRow_Any_AVX2, AB64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7) #endif @@ -1607,7 +1607,7 @@ ANY11T(ARGBToAB64Row_Any_NEON, ARGBToAB64Row_NEON, 4, 8, uint8_t, uint16_t, 7) ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) #endif -#ifdef HAS_ARGBTOAR64ROW_NEON +#ifdef HAS_AB64TOARGBROW_NEON ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) #endif diff --git a/source/row_common.cc b/source/row_common.cc index 31cbf8e4d..fca045927 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -595,7 +595,7 @@ void AR64ShuffleRow_C(const uint8_t* src_ar64, int index3 = shuffler[6] / 2; // Shuffle a row of AR64. int x; - for (x = 0; x < width / 2; ++x) { + for (x = 0; x < width; ++x) { // To support in-place conversion. uint16_t b = src_ar64_16[index0]; uint16_t g = src_ar64_16[index1]; diff --git a/source/row_gcc.cc b/source/row_gcc.cc index d27099e20..2834e0ba1 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -7056,6 +7056,7 @@ void SplitRGBRow_AVX2(const uint8_t* src_rgb, "lea 0x20(%3),%3 \n" "sub $0x20,%4 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_rgb), // %0 "+r"(dst_r), // %1 "+r"(dst_g), // %2 @@ -11102,7 +11103,7 @@ void ARGBPolynomialRow_SSE2(const uint8_t* src_argb, "addps %%xmm5,%%xmm4 \n" "cvttps2dq %%xmm0,%%xmm0 \n" "cvttps2dq %%xmm4,%%xmm4 \n" - "packuswb %%xmm4,%%xmm0 \n" + "packssdw %%xmm4,%%xmm0 \n" "packuswb %%xmm0,%%xmm0 \n" "movq %%xmm0,(%1) \n" "lea 0x8(%1),%1 \n" diff --git a/source/row_sve.cc b/source/row_sve.cc index 28ddc94fb..7ecad044b 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -777,7 +777,7 @@ void DivideRow_16_SVE2(const uint16_t* src_y, "cnth %x[vl] \n" "dup z0.h, %w[scale] \n" "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.le 2f \n" + "b.lt 2f \n" // Run bulk of computation with the same predicates to avoid predicate // generation overhead. @@ -792,7 +792,7 @@ void DivideRow_16_SVE2(const uint16_t* src_y, "st1h {z1.h}, p0, [%[dst]] \n" "st1h {z2.h}, p0, [%[dst], #1, mul vl] \n" "incb %[dst], all, mul #2 \n" - "b.gt 1b \n" + "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl], lsl #1 \n" @@ -1221,7 +1221,7 @@ void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) { "st1b {z0.b}, p0, [%[dst], #0, mul vl] \n" "st1b {z1.b}, p0, [%[dst], #1, mul vl] \n" "incb %[dst], all, mul #2 \n" - "b.gt 1b \n" + "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl2] \n" diff --git a/source/scale.cc b/source/scale.cc index c1b65c295..6f8d45a91 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -917,6 +917,7 @@ static void ScaleAddCols1_16_C(int dst_width, int boxwidth = MIN1(dx >> 16); int scaleval = 65536 / (boxwidth * boxheight); int i; + x >>= 16; for (i = 0; i < dst_width; ++i) { *dst_ptr++ = SumPixels_16(boxwidth, src_ptr + x) * scaleval >> 16; x += boxwidth; @@ -1449,7 +1450,7 @@ static int ScalePlaneBilinearUp(int src_width, #endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleFilterCols = ScaleColsUp2_C; -#if defined(HAS_SCALECOLS_SSE2) +#if defined(HAS_SCALECOLSUP2_SSE2) if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { ScaleFilterCols = ScaleColsUp2_SSE2; } @@ -2038,7 +2039,7 @@ static void ScalePlaneSimple(int src_width, if (src_width * 2 == dst_width && x < 0x8000) { ScaleCols = ScaleColsUp2_C; -#if defined(HAS_SCALECOLS_SSE2) +#if defined(HAS_SCALECOLSUP2_SSE2) if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { ScaleCols = ScaleColsUp2_SSE2; } diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index 835665ee0..da6964f13 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -1275,21 +1275,6 @@ void ScaleRowUp2_Bilinear_16_SSE2(const uint16_t* src_ptr, LABELALIGN "1: \n" - "movq (%0),%%xmm0 \n" // 0011 (16b, 1u1v) - "movq 4(%0),%%xmm1 \n" // 1122 (16b, 1u1v) - "punpcklwd %%xmm7,%%xmm0 \n" // 0011 (near) (32b, 1u1v) - "punpcklwd %%xmm7,%%xmm1 \n" // 1122 (near) (32b, 1u1v) - "movdqa %%xmm0,%%xmm2 \n" - "movdqa %%xmm1,%%xmm3 \n" - "pshufd $0b01001110,%%xmm2,%%xmm2 \n" // 1100 (far) (1, lo) - "pshufd $0b01001110,%%xmm3,%%xmm3 \n" // 2211 (far) (1, hi) - "paddd %%xmm0,%%xmm2 \n" // near+far (1, lo) - "paddd %%xmm1,%%xmm3 \n" // near+far (1, hi) - "paddd %%xmm0,%%xmm0 \n" // 2*near (1, lo) - "paddd %%xmm1,%%xmm1 \n" // 2*near (1, hi) - "paddd %%xmm2,%%xmm0 \n" // 3*near+far (1, lo) - "paddd %%xmm3,%%xmm1 \n" // 3*near+far (1, hi) - "movq (%0),%%xmm0 \n" // 0123 (16b) "movq 2(%0),%%xmm1 \n" // 1234 (16b) "punpcklwd %%xmm7,%%xmm0 \n" // 0123 (32b) diff --git a/source/scale_neon64.cc b/source/scale_neon64.cc index ae4768f22..94ddb4d00 100644 --- a/source/scale_neon64.cc +++ b/source/scale_neon64.cc @@ -1013,7 +1013,7 @@ void ScaleAddRow_NEON(const uint8_t* src_ptr, } #define SCALE_FILTER_COLS_STEP_ADDR \ - "lsr %[tmp_offset], %x[x], #16 \n" \ + "asr %[tmp_offset], %x[x], #16 \n" \ "add %[tmp_ptr], %[src_ptr], %[tmp_offset] \n" \ "add %x[x], %x[x], %x[dx] \n" @@ -1027,6 +1027,8 @@ void ScaleFilterCols_NEON(uint8_t* dst_ptr, int x, int dx) { int dx_offset[4] = {0, 1, 2, 3}; + int64_t x64 = (int64_t)x; + int64_t dx64 = (int64_t)dx; int64_t tmp_offset; uint8_t* tmp_ptr; asm volatile( @@ -1080,8 +1082,8 @@ void ScaleFilterCols_NEON(uint8_t* dst_ptr, : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(dst_width), // %[width] - [x] "+r"(x), // %[x] - [dx] "+r"(dx), // %[dx] + [x] "+r"(x64), // %[x] + [dx] "+r"(dx64), // %[dx] [tmp_offset] "=&r"(tmp_offset), // %[tmp_offset] [tmp_ptr] "=&r"(tmp_ptr) // %[tmp_ptr] : [dx_offset] "r"(dx_offset) // %[dx_offset] diff --git a/source/scale_uv.cc b/source/scale_uv.cc index a75612eb7..362f0ba89 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -226,7 +226,7 @@ static int ScaleUVDown4Box(int src_width, } } #endif -#if defined(HAS_SCALEUVROWDOWN2BOX_SME) +#if defined(HAS_SCALEUVROWDOWN2_SME) if (TestCpuFlag(kCpuHasSME)) { ScaleUVRowDown2 = ScaleUVRowDown2Box_SME; }