diff --git a/include/libyuv/row.h b/include/libyuv/row.h index e8ee93d22..e189d9d81 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -153,13 +153,11 @@ extern "C" { #define HAS_ARGBCOPYALPHAROW_AVX2 #define HAS_ARGBCOPYYTOALPHAROW_AVX2 #define HAS_ARGBEXTRACTALPHAROW_AVX2 -#define HAS_ARGBMIRRORROW_AVX2 #define HAS_ARGBPOLYNOMIALROW_AVX2 #define HAS_ARGBSHUFFLEROW_AVX2 #define HAS_ARGBTORGB565DITHERROW_AVX2 #define HAS_COPYROW_AVX #define HAS_HALFFLOATROW_AVX2 -#define HAS_I422TOARGBROW_AVX2 #define HAS_I422TORGB24ROW_AVX2 #define HAS_I422TORGBAROW_AVX2 #define HAS_I422TORGB565ROW_AVX2 @@ -168,12 +166,8 @@ extern "C" { #define HAS_I444TOARGBROW_AVX2 #define HAS_I444TORGB24ROW_AVX2 #define HAS_J400TOARGBROW_AVX2 -#define HAS_MIRRORROW_AVX2 -#define HAS_MIRRORSPLITUVROW_AVX2 -#define HAS_NV12TOARGBROW_AVX2 #define HAS_NV12TORGB24ROW_AVX2 #define HAS_NV12TORGB565ROW_AVX2 -#define HAS_NV21TOARGBROW_AVX2 #define HAS_NV21TORGB24ROW_AVX2 #define HAS_SPLITUVROW_AVX2 #define HAS_UYVYTOARGBROW_AVX2 @@ -230,6 +224,7 @@ extern "C" { #define HAS_MIRRORUVROW_SSSE3 #define HAS_NV21TOYUV24ROW_SSSE3 #define HAS_P210TOAR30ROW_SSSE3 +#define HAS_RGB24MIRRORROW_SSSE3 #define HAS_P210TOARGBROW_SSSE3 #define HAS_P410TOAR30ROW_SSSE3 #define HAS_P410TOARGBROW_SSSE3 @@ -474,7 +469,6 @@ extern "C" { #define HAS_RGBTOUV444MATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512VBMI -#define HAS_RAWTORGB24ROW_AVX512VBMI #define HAS_I422TORGB24ROW_AVX512BW #define HAS_ARGBTOUVMATRIXROW_AVX512BW #define HAS_J400TOARGBROW_AVX512BW @@ -796,6 +790,7 @@ extern "C" { #define HAS_P210TOARGBROW_SME #define HAS_P410TOAR30ROW_SME #define HAS_P410TOARGBROW_SME +#define HAS_UYVYTOARGBROW_SME #define HAS_YUY2TOARGBROW_SME #endif @@ -959,6 +954,8 @@ extern "C" { #define HAS_ARGBMULTIPLYROW_RVV #define HAS_ARGBTOAR64ROW_RVV #define HAS_ARGBMIRRORROW_RVV +#define HAS_ARGBSETROW_RVV +#define HAS_ARGBSHUFFLEROW_RVV #define HAS_ARGBTOUV444MATRIXROW_RVV #define HAS_ARGBTOUVMATRIXROW_RVV #define HAS_ARGBTOYMATRIXROW_RVV @@ -2972,6 +2969,9 @@ void ARGBMirrorRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void RGB24MirrorRow_SSSE3(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width); void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width); @@ -2982,6 +2982,9 @@ void RGB24MirrorRow_NEON(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width); void RGB24MirrorRow_C(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width); +void RGB24MirrorRow_Any_SSSE3(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void RGB24MirrorRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4215,6 +4218,7 @@ void SetRow_Any_LASX(uint8_t* dst_ptr, uint8_t v32, int width); void ARGBSetRow_C(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width); +void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_Any_NEON(uint8_t* dst_ptr, uint32_t v32, int width); void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width); @@ -4242,6 +4246,10 @@ void ARGBShuffleRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, const uint8_t* shuffler, int width); +void ARGBShuffleRow_RVV(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width); void ARGBShuffleRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, const uint8_t* shuffler, diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 7e0ca8776..97616b9a3 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -466,7 +466,7 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y, "ptrue p0.b \n" "dup z19.b, #255 \n" // Alpha YUVTORGB_SVE_SETUP - "cmp %w[width], %w[vl] \n" + "subs %w[width], %w[width], %w[vl] \n" "mov z1.h, #128 \n" // U/V "umullb z6.h, z30.b, z1.b \n" "umullb z4.h, z28.b, z1.b \n" // DB @@ -475,25 +475,28 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y, "sub z4.h, z4.h, z25.h \n" "sub z5.h, z5.h, z27.h \n" "sub z6.h, z26.h, z6.h \n" - "b.le 2f \n" + "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p1.h \n" - "sub %w[width], %w[width], %w[vl] \n" "1: \n" // READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE "subs %w[width], %w[width], %w[vl] \n" "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n" - "b.gt 1b \n" - "add %w[width], %w[width], %w[vl] \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl] \n" + "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "2: \n" "whilelt p1.h, wzr, %w[width] \n" // READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" + + "99: \n" : [src_y] "+r"(src_y), // %[src_y] [dst_argb] "+r"(dst_argb), // %[dst_argb] [width] "+r"(width), // %[width] @@ -702,7 +705,11 @@ static inline void I422ToARGB1555Row_SVE_SC( "whilelt p1.b, wzr, %w[width] \n" // READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X RGB8TOARGB1555_SVE_FROM_TOP_2X - "st2h {z0.h, z1.h}, p1, [%[dst]] \n" + // Need to permute the data on the final iteration such that the + // predicates (.b) line up with the 16-bit element data. + "trn1 z2.b, z0.b, z1.b \n" + "trn2 z3.b, z0.b, z1.b \n" + "st2b {z2.b, z3.b}, p1, [%[dst]] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -761,7 +768,11 @@ static inline void I422ToARGB4444Row_SVE_SC( "whilelt p1.b, wzr, %w[width] \n" // READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X RGB8TOARGB4444_SVE_FROM_TOP_2X - "st2h {z0.h, z1.h}, p1, [%[dst]] \n" + // Need to permute the data on the final iteration such that the + // predicates (.b) line up with the 16-bit element data. + "trn1 z2.b, z0.b, z1.b \n" + "trn2 z3.b, z0.b, z1.b \n" + "st2b {z2.b, z3.b}, p1, [%[dst]] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] @@ -788,7 +799,7 @@ static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y, YUVTORGB_SVE_SETUP "dup z19.b, #255 \n" // Alpha "subs %w[width], %w[width], %w[vl] \n" - "b.le 2f \n" + "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. @@ -798,7 +809,7 @@ static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y, "subs %w[width], %w[width], %w[vl] \n" "st2h {z19.h, z20.h}, p1, [%[dst_rgba]] \n" "add %[dst_rgba], %[dst_rgba], %[vl], lsl #2 \n" - "b.gt 1b \n" + "b.ge 1b \n" // Calculate a predicate for the final iteration to deal with the tail. "2: \n" @@ -836,7 +847,7 @@ static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y, YUVTORGB_SVE_SETUP_AR30 "dup z23.h, %w[limit] \n" "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.le 2f \n" + "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. @@ -846,7 +857,7 @@ static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y, "1: \n" // READYUV422_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.gt 1b \n" + "b.ge 1b \n" // Calculate a predicate for the final iteration to deal with the tail. "2: \n" @@ -985,8 +996,6 @@ static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cntb %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); - int width_last_uv = width_last_y + (width_last_y & 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1010,21 +1019,21 @@ static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width_last_y] \n" - "whilelt p2.b, wzr, %w[width_last_uv] \n" // + "whilelt p1.b, wzr, %w[width] \n" + "add %w[width], %w[width], #1 \n" // Round up to even + "and %w[width], %w[width], #0xfffffffe \n" + "whilelt p2.b, wzr, %w[width] \n" // READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_uv] "+r"(src_uv), // %[src_uv] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y), // %[width_last_y] - [width_last_uv] "r"(width_last_uv) // %[width_last_uv] + : [src_y] "+r"(src_y), // %[src_y] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); } @@ -1035,8 +1044,6 @@ static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cntb %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); - int width_last_uv = width_last_y + (width_last_y & 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1060,21 +1067,21 @@ static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width_last_y] \n" - "whilelt p2.b, wzr, %w[width_last_uv] \n" // + "whilelt p1.b, wzr, %w[width] \n" + "add %w[width], %w[width], #1 \n" // Round up to even + "and %w[width], %w[width], #0xfffffffe \n" + "whilelt p2.b, wzr, %w[width] \n" // READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_uv] "+r"(src_vu), // %[src_vu] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y), // %[width_last_y] - [width_last_uv] "r"(width_last_uv) // %[width_last_uv] + : [src_y] "+r"(src_y), // %[src_y] + [src_uv] "+r"(src_vu), // %[src_vu] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); } @@ -1086,8 +1093,6 @@ static inline void NV12ToRGB24Row_SVE_SC( int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cntb %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); - int width_last_uv = width_last_y + (width_last_y & 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1111,21 +1116,21 @@ static inline void NV12ToRGB24Row_SVE_SC( "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width_last_y] \n" - "whilelt p2.b, wzr, %w[width_last_uv] \n" // + "whilelt p1.b, wzr, %w[width] \n" + "add %w[width], %w[width], #1 \n" // Round up to even + "and %w[width], %w[width], #0xfffffffe \n" + "whilelt p2.b, wzr, %w[width] \n" // READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_uv] "+r"(src_uv), // %[src_uv] - [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y), // %[width_last_y] - [width_last_uv] "r"(width_last_uv) // %[width_last_uv] + : [src_y] "+r"(src_y), // %[src_y] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); } @@ -1137,8 +1142,6 @@ static inline void NV21ToRGB24Row_SVE_SC( int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cntb %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); - int width_last_uv = width_last_y + (width_last_y & 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1162,21 +1165,21 @@ static inline void NV21ToRGB24Row_SVE_SC( "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.b, wzr, %w[width_last_y] \n" - "whilelt p2.b, wzr, %w[width_last_uv] \n" // + "whilelt p1.b, wzr, %w[width] \n" + "add %w[width], %w[width], #1 \n" // Round up to even + "and %w[width], %w[width], #0xfffffffe \n" + "whilelt p2.b, wzr, %w[width] \n" // READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_uv] "+r"(src_vu), // %[src_vu] - [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y), // %[width_last_y] - [width_last_uv] "r"(width_last_uv) // %[width_last_uv] + : [src_y] "+r"(src_y), // %[src_y] + [src_uv] "+r"(src_vu), // %[src_vu] + [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); } @@ -1195,8 +1198,6 @@ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2, uint32_t nv_uv_step = 0x04040404U; uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); - int width_last_uv = width_last_y + (width_last_y & 1); asm volatile( "ptrue p0.b \n" "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n" @@ -1221,8 +1222,10 @@ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" - "whilelt p2.h, wzr, %w[width_last_uv] \n" // + "whilelt p1.h, wzr, %w[width] \n" + "add %w[width], %w[width], #1 \n" // Round up to even + "and %w[width], %w[width], #0xfffffffe \n" + "whilelt p2.h, wzr, %w[width] \n" // READYUY2_SVE NVTORGB_SVE RGBTOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" @@ -1234,9 +1237,7 @@ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2, [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start] - [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step] - [width_last_y] "r"(width_last_y), // %[width_last_y] - [width_last_uv] "r"(width_last_uv) // %[width_last_uv] + [nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step] : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); } @@ -1255,8 +1256,6 @@ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy, uint32_t nv_uv_step = 0x04040404U; uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); - int width_last_uv = width_last_y + (width_last_y & 1); asm volatile( "ptrue p0.b \n" "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n" @@ -1281,9 +1280,10 @@ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "2: \n" - "whilelt p1.h, wzr, %w[width_last_y] \n" - "whilelt p2.h, wzr, %w[width_last_uv] \n" // + "whilelt p1.h, wzr, %w[width] \n" + "add %w[width], %w[width], #1 \n" // Round up to even + "and %w[width], %w[width], #0xfffffffe \n" + "whilelt p2.h, wzr, %w[width] \n" // READUYVY_SVE NVTORGB_SVE RGBTOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" @@ -1295,9 +1295,7 @@ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy, [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start] - [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step] - [width_last_y] "r"(width_last_y), // %[width_last_y] - [width_last_uv] "r"(width_last_uv) // %[width_last_uv] + [nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step] : "cc", "memory", YUVTORGB_SVE_REGS, "p2"); } @@ -1309,7 +1307,6 @@ static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1332,20 +1329,19 @@ static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" // + "whilelt p1.h, wzr, %w[width] \n" // READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_u] "+r"(src_u), // %[src_u] - [src_v] "+r"(src_v), // %[src_v] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y) // %[width_last_y] + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1359,7 +1355,6 @@ static inline void I210AlphaToARGBRow_SVE_SC( int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1385,23 +1380,22 @@ static inline void I210AlphaToARGBRow_SVE_SC( "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" // + "whilelt p1.h, wzr, %w[width] \n" // READI210_SVE "ld1h {z19.h}, p1/z, [%[src_a]] \n" // I4XXTORGB_SVE RGBATOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_u] "+r"(src_u), // %[src_u] - [src_v] "+r"(src_v), // %[src_v] - [src_a] "+r"(src_a), // %[src_a] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y) // %[width_last_y] + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [src_a] "+r"(src_a), // %[src_a] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1413,8 +1407,6 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (2 * vl - 1); - int width_last_uv = (width_last_y + 1) / 2; // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE_2X. uint16_t limit = 0x3ff0; asm volatile( @@ -1439,9 +1431,11 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_uv] \n" - "whilelt p2.h, wzr, %w[width_last_y] \n" - "whilelt p3.h, %w[vl], %w[width_last_y] \n" // + "whilelt p2.h, wzr, %w[width] \n" + "whilelt p3.h, %w[vl], %w[width] \n" + "add %w[width], %w[width], #1 \n" // (width + 1) / 2 + "lsr %w[width], %w[width], #1 \n" + "whilelt p1.h, wzr, %w[width] \n" // READI210_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X "99: \n" @@ -1453,8 +1447,6 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y, : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y), // %[width_last_y] - [width_last_uv] "r"(width_last_uv), // %[width_last_uv] [limit] "r"(limit) // %[limit] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1467,8 +1459,6 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); - int width_last_uv = width_last_y + (width_last_y & 1); uint32_t nv_uv_start = 0x03010301U; uint32_t nv_uv_step = 0x04040404U; asm volatile( @@ -1495,8 +1485,10 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" - "whilelt p2.h, wzr, %w[width_last_uv] \n" // + "whilelt p1.h, wzr, %w[width] \n" + "add %w[width], %w[width], #1 \n" // Round up to even + "and %w[width], %w[width], #0xfffffffe \n" + "whilelt p2.h, wzr, %w[width] \n" // READP210_SVE NVTORGB_SVE RGBTOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" @@ -1509,9 +1501,7 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y, [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start] - [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step] - [width_last_y] "r"(width_last_y), // %[width_last_y] - [width_last_uv] "r"(width_last_uv) // %[width_last_uv] + [nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1522,8 +1512,6 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); - int width_last_uv = width_last_y + (width_last_y & 1); uint32_t nv_uv_start = 0x03010301U; uint32_t nv_uv_step = 0x04040404U; // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE. @@ -1551,8 +1539,10 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" - "whilelt p2.h, wzr, %w[width_last_uv] \n" // + "whilelt p1.h, wzr, %w[width] \n" + "add %w[width], %w[width], #1 \n" // Round up to even + "and %w[width], %w[width], #0xfffffffe \n" + "whilelt p2.h, wzr, %w[width] \n" // READP210_SVE NVTORGB_SVE STOREAR30_SVE "99: \n" @@ -1565,8 +1555,6 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y, [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start] [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step] - [width_last_y] "r"(width_last_y), // %[width_last_y] - [width_last_uv] "r"(width_last_uv), // %[width_last_uv] [limit] "r"(limit) // %[limit] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1579,7 +1567,6 @@ static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1602,20 +1589,19 @@ static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" // + "whilelt p1.h, wzr, %w[width] \n" // READI410_SVE I4XXTORGB_SVE RGBTOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_u] "+r"(src_u), // %[src_u] - [src_v] "+r"(src_v), // %[src_v] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y) // %[width_last_y] + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1629,7 +1615,6 @@ static inline void I410AlphaToARGBRow_SVE_SC( int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1656,23 +1641,22 @@ static inline void I410AlphaToARGBRow_SVE_SC( "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" // + "whilelt p1.h, wzr, %w[width] \n" // READI410_SVE "ld1h {z19.h}, p1/z, [%[src_a]] \n" // I4XXTORGB_SVE RGBATOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_u] "+r"(src_u), // %[src_u] - [src_v] "+r"(src_v), // %[src_v] - [src_a] "+r"(src_a), // %[src_a] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y) // %[width_last_y] + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [src_a] "+r"(src_a), // %[src_a] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1684,7 +1668,6 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE. uint16_t limit = 0x3ff0; asm volatile( @@ -1707,7 +1690,7 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" // + "whilelt p1.h, wzr, %w[width] \n" // READI410_SVE I4XXTORGB_SVE STOREAR30_SVE "99: \n" @@ -1719,7 +1702,6 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y, : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y), // %[width_last_y] [limit] "r"(limit) // %[limit] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1731,7 +1713,6 @@ static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1756,22 +1737,21 @@ static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" - "whilelt p2.s, wzr, %w[width_last_y] \n" + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.s, wzr, %w[width] \n" "cntw %x[vl] \n" - "whilelt p3.s, %w[vl], %w[width_last_y] \n" // + "whilelt p3.s, %w[vl], %w[width] \n" // READP410_SVE NVTORGB_SVE RGBTOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_uv] "+r"(src_uv), // %[src_uv] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width), // %[width] - [vl] "+r"(vl) // %[vl] - : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y) // %[width_last_y] + : [src_y] "+r"(src_y), // %[src_y] + [src_uv] "+r"(src_uv), // %[src_uv] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width), // %[width] + [vl] "+r"(vl) // %[vl] + : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1782,7 +1762,6 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE. uint16_t limit = 0x3ff0; asm volatile( @@ -1808,10 +1787,10 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" - "whilelt p2.s, wzr, %w[width_last_y] \n" + "whilelt p1.h, wzr, %w[width] \n" + "whilelt p2.s, wzr, %w[width] \n" "cntw %x[vl] \n" - "whilelt p3.s, %w[vl], %w[width_last_y] \n" // + "whilelt p3.s, %w[vl], %w[width] \n" // READP410_SVE NVTORGB_SVE STOREAR30_SVE "99: \n" @@ -1822,7 +1801,6 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y, [vl] "+r"(vl) // %[vl] : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y), // %[width_last_y] [limit] "r"(limit) // %[limit] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1835,7 +1813,6 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE. uint16_t limit = 0x3ff0; asm volatile( @@ -1858,7 +1835,7 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" // + "whilelt p1.h, wzr, %w[width] \n" // READI212_SVE I4XXTORGB_SVE STOREAR30_SVE "99: \n" @@ -1870,7 +1847,6 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y, : [vl] "r"(vl), // %[vl] [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y), // %[width_last_y] [limit] "r"(limit) // %[limit] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -1883,7 +1859,6 @@ static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y, int width) STREAMING_COMPATIBLE { uint64_t vl; asm("cnth %0" : "=r"(vl)); - int width_last_y = width & (vl - 1); asm volatile( "ptrue p0.b \n" // YUVTORGB_SVE_SETUP @@ -1906,20 +1881,19 @@ static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y, "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. - "whilelt p1.h, wzr, %w[width_last_y] \n" // + "whilelt p1.h, wzr, %w[width] \n" // READI212_SVE I4XXTORGB_SVE RGBTOARGB8_SVE "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n" "99: \n" - : [src_y] "+r"(src_y), // %[src_y] - [src_u] "+r"(src_u), // %[src_u] - [src_v] "+r"(src_v), // %[src_v] - [dst_argb] "+r"(dst_argb), // %[dst_argb] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] - [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias] - [width_last_y] "r"(width_last_y) // %[width_last_y] + : [src_y] "+r"(src_y), // %[src_y] + [src_u] "+r"(src_u), // %[src_u] + [src_v] "+r"(src_v), // %[src_v] + [dst_argb] "+r"(dst_argb), // %[dst_argb] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] + [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS); } @@ -2120,7 +2094,7 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb, "ptrue p5.h \n" "1: \n" // ARGBTOUVMATRIX_SVE - "b.gt 1b \n" + "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl], lsl #2 \n" @@ -2197,7 +2171,7 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, "ptrue p1.s \n" "1: \n" // ARGBTOYMATRIX_SVE("p1", "p1", "p1", "p1", "p0") - "b.gt 1b \n" + "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl], lsl #2 \n" diff --git a/source/compare.cc b/source/compare.cc index daac8b985..2ec13abc5 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -445,8 +445,10 @@ double CalcFrameSsim(const uint8_t* src_a, int height) { int samples = 0; double ssim_total = 0; + // Return 0 rather than 0.0 / 0 (NaN) when the plane is too small for any + // sample windows, so I420Ssim on small images stays finite. if (width <= 8 || height <= 8) { - return ssim_total / samples; + return 0.0; } int simd_size = 4; diff --git a/source/convert.cc b/source/convert.cc index 297deb315..79346f245 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -1270,6 +1270,7 @@ int I422ToNV21(const uint8_t* src_y, dst_y, dst_stride_y, plane_u, halfwidth, plane_v, halfwidth, width, height); if (r != 0) { + free_aligned_buffer_64(plane_u); return r; } MergeUVPlane(plane_v, halfwidth, plane_u, halfwidth, dst_vu, dst_stride_vu, @@ -2714,7 +2715,8 @@ int AYUVToNV12(const uint8_t* src_ayuv, } #endif #if defined(HAS_AYUVTOYROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { AYUVToUVRow = AYUVToUVRow_AVX512VBMI; AYUVToYRow = AYUVToYRow_AVX512VBMI; } @@ -2800,7 +2802,8 @@ int AYUVToNV21(const uint8_t* src_ayuv, } #endif #if defined(HAS_AYUVTOYROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { AYUVToVURow = AYUVToVURow_AVX512VBMI; AYUVToYRow = AYUVToYRow_AVX512VBMI; } diff --git a/source/convert_argb.cc b/source/convert_argb.cc index 19ce4cfec..9b7dea6bf 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -4223,7 +4223,8 @@ int RAWToRGBA(const uint8_t* src_raw, } #endif #if defined(HAS_RAWTORGBAROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { RAWToRGBARow = RAWToRGBARow_AVX512VBMI; } #endif @@ -5100,7 +5101,8 @@ int NV12ToRGB24Matrix(const uint8_t* src_y, } #endif #if defined(HAS_NV12TORGB24ROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { NV12ToRGB24Row = NV12ToRGB24Row_AVX512VBMI; } #endif @@ -5182,7 +5184,8 @@ int NV21ToRGB24Matrix(const uint8_t* src_y, } #endif #if defined(HAS_NV21TORGB24ROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { NV21ToRGB24Row = NV21ToRGB24Row_AVX512VBMI; } #endif @@ -6121,7 +6124,7 @@ int I420ToRGB24Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB24ROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { I422ToRGB24Row = I422ToRGB24Row_Any_WASMSIMD; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 16)) { I422ToRGB24Row = I422ToRGB24Row_WASMSIMD; } } @@ -6352,7 +6355,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB24ROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { I422ToRGB24Row = I422ToRGB24Row_Any_WASMSIMD; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 16)) { I422ToRGB24Row = I422ToRGB24Row_WASMSIMD; } } diff --git a/source/convert_from.cc b/source/convert_from.cc index a76353269..4bca7602f 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -1038,6 +1038,7 @@ int ConvertFromI420(const uint8_t* y, height == INT_MIN) { return -1; } + const int abs_height = Abs(height); switch (format) { // Single plane formats case FOURCC_YUY2: @@ -1107,7 +1108,7 @@ int ConvertFromI420(const uint8_t* y, break; case FOURCC_NV12: { int dst_y_stride = dst_sample_stride ? dst_sample_stride : width; - uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height; + uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height; r = I420ToNV12(y, y_stride, u, u_stride, v, v_stride, dst_sample, dst_sample_stride ? dst_sample_stride : width, dst_uv, dst_sample_stride ? dst_sample_stride : width, width, @@ -1116,7 +1117,7 @@ int ConvertFromI420(const uint8_t* y, } case FOURCC_NV21: { int dst_y_stride = dst_sample_stride ? dst_sample_stride : width; - uint8_t* dst_vu = dst_sample + (ptrdiff_t)dst_y_stride * height; + uint8_t* dst_vu = dst_sample + (ptrdiff_t)dst_y_stride * abs_height; r = I420ToNV21(y, y_stride, u, u_stride, v, v_stride, dst_sample, dst_sample_stride ? dst_sample_stride : width, dst_vu, dst_sample_stride ? dst_sample_stride : width, width, @@ -1126,13 +1127,13 @@ int ConvertFromI420(const uint8_t* y, case FOURCC_NV16: { int dst_y_stride = dst_sample_stride ? dst_sample_stride : width; int dst_uv_stride = dst_sample_stride ? dst_sample_stride : width; - uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height; + uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height; int halfwidth = (width + 1) / 2; - int halfheight = (height + 1) / 2; + int halfheight = SUBSAMPLE(height, 1, 1); CopyPlane(y, y_stride, dst_sample, dst_y_stride, width, height); MergeUVPlane(u, u_stride, v, v_stride, dst_uv, dst_uv_stride * 2, halfwidth, halfheight); - if (height > 1) { + if (abs_height > 1) { MergeUVPlane(u, u_stride, v, v_stride, dst_uv + dst_uv_stride, dst_uv_stride * 2, halfwidth, height / 2); } @@ -1142,9 +1143,9 @@ int ConvertFromI420(const uint8_t* y, case FOURCC_NV24: { int dst_y_stride = dst_sample_stride ? dst_sample_stride : width; int dst_uv_stride = dst_sample_stride ? dst_sample_stride * 2 : width * 2; - uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height; - align_buffer_64(temp_u, (size_t)width * height); - align_buffer_64(temp_v, (size_t)width * height); + uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height; + align_buffer_64(temp_u, (size_t)width * abs_height); + align_buffer_64(temp_v, (size_t)width * abs_height); if (!temp_u || !temp_v) { free_aligned_buffer_64(temp_u); free_aligned_buffer_64(temp_v); @@ -1154,7 +1155,7 @@ int ConvertFromI420(const uint8_t* y, dst_y_stride, temp_u, width, temp_v, width, width, height); if (r == 0) { MergeUVPlane(temp_u, width, temp_v, width, dst_uv, dst_uv_stride, width, - height); + abs_height); } free_aligned_buffer_64(temp_u); free_aligned_buffer_64(temp_v); @@ -1162,7 +1163,6 @@ int ConvertFromI420(const uint8_t* y, } case FOURCC_P010: case FOURCC_P210: { - const int abs_height = Abs(height); const int uv_min = ((width + 1) / 2) * 2; int dst_stride_y; int dst_stride_uv; @@ -1193,7 +1193,6 @@ int ConvertFromI420(const uint8_t* y, break; } case FOURCC_P410: { - const int abs_height = Abs(height); const int uv_min = width * 2; int dst_stride_y; int dst_stride_uv; @@ -1223,14 +1222,14 @@ int ConvertFromI420(const uint8_t* y, case FOURCC_YV12: { dst_sample_stride = dst_sample_stride ? dst_sample_stride : width; int halfstride = (dst_sample_stride + 1) / 2; - int halfheight = (height + 1) / 2; + int halfheight = (abs_height + 1) / 2; uint8_t* dst_u; uint8_t* dst_v; if (format == FOURCC_YV12) { - dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height; + dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; dst_u = dst_v + (ptrdiff_t)halfstride * halfheight; } else { - dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height; + dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; dst_v = dst_u + (ptrdiff_t)halfstride * halfheight; } r = I420Copy(y, y_stride, u, u_stride, v, v_stride, dst_sample, @@ -1245,11 +1244,11 @@ int ConvertFromI420(const uint8_t* y, uint8_t* dst_u; uint8_t* dst_v; if (format == FOURCC_YV16) { - dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height; - dst_u = dst_v + (ptrdiff_t)halfstride * height; + dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; + dst_u = dst_v + (ptrdiff_t)halfstride * abs_height; } else { - dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height; - dst_v = dst_u + (ptrdiff_t)halfstride * height; + dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; + dst_v = dst_u + (ptrdiff_t)halfstride * abs_height; } r = I420ToI422(y, y_stride, u, u_stride, v, v_stride, dst_sample, dst_sample_stride, dst_u, halfstride, dst_v, halfstride, @@ -1262,11 +1261,11 @@ int ConvertFromI420(const uint8_t* y, uint8_t* dst_u; uint8_t* dst_v; if (format == FOURCC_YV24) { - dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height; - dst_u = dst_v + (ptrdiff_t)dst_sample_stride * height; + dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; + dst_u = dst_v + (ptrdiff_t)dst_sample_stride * abs_height; } else { - dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height; - dst_v = dst_u + (ptrdiff_t)dst_sample_stride * height; + dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height; + dst_v = dst_u + (ptrdiff_t)dst_sample_stride * abs_height; } r = I420ToI444(y, y_stride, u, u_stride, v, v_stride, dst_sample, dst_sample_stride, dst_u, dst_sample_stride, dst_v, diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index aee01f4dc..9c3d9d9dc 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -1630,6 +1630,14 @@ int ARGBToI400Matrix(const uint8_t* src_argb, src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb; src_stride_argb = -src_stride_argb; } +#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2; @@ -1721,6 +1729,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, dst_yuy2 = dst_yuy2 + (ptrdiff_t)(height - 1) * dst_stride_yuy2; dst_stride_yuy2 = -dst_stride_yuy2; } +#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2; @@ -1776,6 +1792,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_HVX; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; @@ -1832,6 +1856,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX; } #endif +#if defined(HAS_I422TOYUY2ROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + I422ToYUY2Row = I422ToYUY2Row_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_SSE2; + } + } +#endif #if defined(HAS_I422TOYUY2ROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { I422ToYUY2Row = I422ToYUY2Row_Any_AVX2; @@ -1897,6 +1929,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, dst_uyvy = dst_uyvy + (ptrdiff_t)(height - 1) * dst_stride_uyvy; dst_stride_uyvy = -dst_stride_uyvy; } +#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2; @@ -1952,6 +1992,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_HVX; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 8)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; @@ -2008,6 +2056,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX; } #endif +#if defined(HAS_I422TOUYVYROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + I422ToUYVYRow = I422ToUYVYRow_Any_SSE2; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_SSE2; + } + } +#endif #if defined(HAS_I422TOUYVYROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { I422ToUYVYRow = I422ToUYVYRow_Any_AVX2; diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index 66a6fb675..bc699b647 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -257,6 +257,27 @@ static int NV24ToARGB(const uint8_t* src_y, } } #endif +#if defined(HAS_SPLITUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + SplitUVRow = SplitUVRow_SVE2; + } +#endif +#if defined(HAS_SPLITUVROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + SplitUVRow = SplitUVRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + SplitUVRow = SplitUVRow_LSX; + } + } +#endif +#if defined(HAS_SPLITUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SplitUVRow = SplitUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + SplitUVRow = SplitUVRow_LASX; + } + } +#endif #if defined(HAS_SPLITUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { SplitUVRow = SplitUVRow_RVV; @@ -448,7 +469,7 @@ int ConvertToARGB(const uint8_t* sample, src = sample + ((ptrdiff_t)src_width * crop_y + crop_x); src_uv = sample + aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) + - crop_x; + (crop_x & ~1); r = NV12ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; @@ -456,7 +477,7 @@ int ConvertToARGB(const uint8_t* sample, src = sample + ((ptrdiff_t)src_width * crop_y + crop_x); src_uv = sample + aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) + - crop_x; + (crop_x & ~1); // Call NV12 but with u and v parameters swapped. r = NV21ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb, dst_stride_argb, crop_width, inv_crop_height); @@ -523,12 +544,12 @@ int ConvertToARGB(const uint8_t* sample, int halfheight = (abs_src_height + 1) / 2; if (format == FOURCC_YV12) { src_v = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; src_u = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; } else { src_u = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; } @@ -542,7 +563,7 @@ int ConvertToARGB(const uint8_t* sample, int halfheight = (abs_src_height + 1) / 2; const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; @@ -556,7 +577,7 @@ int ConvertToARGB(const uint8_t* sample, int halfheight = (abs_src_height + 1) / 2; const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; @@ -570,7 +591,7 @@ int ConvertToARGB(const uint8_t* sample, int halfheight = (abs_src_height + 1) / 2; const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x); const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height + - ((ptrdiff_t)halfwidth * crop_y + crop_x) / 2; + (ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2; const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2; @@ -635,7 +656,7 @@ int ConvertToARGB(const uint8_t* sample, const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height + halfwidth * ((ptrdiff_t)abs_src_height + crop_y) + crop_x / 2; - r = H422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, + r = U422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth, dst_argb, dst_stride_argb, crop_width, inv_crop_height); break; } diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc index 87c903d61..67f78b170 100644 --- a/source/convert_to_i420.cc +++ b/source/convert_to_i420.cc @@ -51,8 +51,13 @@ static int NV16ToI420(const uint8_t* src_y, int y; int halfwidth = (width + 1) >> 1; int halfheight = (height + 1) >> 1; + // UV rows are halfwidth * 2 (even) bytes. + int awidth = halfwidth * 2; void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width) = SplitUVRow_C; + void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) = InterpolateRow_C; if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { return -1; @@ -100,18 +105,105 @@ static int NV16ToI420(const uint8_t* src_y, } } #endif +#if defined(HAS_SPLITUVROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + SplitUVRow = SplitUVRow_SVE2; + } +#endif +#if defined(HAS_SPLITUVROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + SplitUVRow = SplitUVRow_Any_LSX; + if (IS_ALIGNED(halfwidth, 16)) { + SplitUVRow = SplitUVRow_LSX; + } + } +#endif +#if defined(HAS_SPLITUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SplitUVRow = SplitUVRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + SplitUVRow = SplitUVRow_LASX; + } + } +#endif +#if defined(HAS_SPLITUVROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + SplitUVRow = SplitUVRow_RVV; + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(awidth, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif +#if defined(HAS_INTERPOLATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_Any_NEON; + if (IS_ALIGNED(awidth, 16)) { + InterpolateRow = InterpolateRow_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + InterpolateRow = InterpolateRow_SVE2; + } +#endif +#if defined(HAS_INTERPOLATEROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + InterpolateRow = InterpolateRow_SME; + } +#endif +#if defined(HAS_INTERPOLATEROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + InterpolateRow = InterpolateRow_Any_LSX; + if (IS_ALIGNED(awidth, 16)) { + InterpolateRow = InterpolateRow_LSX; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(awidth, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + InterpolateRow = InterpolateRow_RVV; + } +#endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(awidth, 16)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } +#endif - align_buffer_64(row_uv, halfwidth * 2); + align_buffer_64(row_uv, awidth); if (!row_uv) { return 1; } for (y = 0; y < halfheight; ++y) { if (y * 2 + 1 < height) { - HalfRow_C(src_uv, src_stride_uv, row_uv, halfwidth * 2); + // Average 2 rows of UV with rounding (source_y_fraction 128). + InterpolateRow(row_uv, src_uv, src_stride_uv, awidth, 128); + SplitUVRow(row_uv, dst_u, dst_v, halfwidth); } else { - memcpy(row_uv, src_uv, (size_t)halfwidth * 2); + SplitUVRow(src_uv, dst_u, dst_v, halfwidth); } - SplitUVRow(row_uv, dst_u, dst_v, halfwidth); src_uv += (ptrdiff_t)src_stride_uv * 2; dst_u += dst_stride_u; dst_v += dst_stride_v; diff --git a/source/cpu_id.cc b/source/cpu_id.cc index dea87ad51..7bfb66de2 100644 --- a/source/cpu_id.cc +++ b/source/cpu_id.cc @@ -408,11 +408,21 @@ static SAFEBUFFERS int GetCpuFlags(void) { int cpu_info24[4] = {0, 0, 0, 0}; int cpu_info21[4] = {0, 0, 0, 0}; int cpu_amdinfo21[4] = {0, 0, 0, 0}; + int cpu_extinfo0[4] = {0, 0, 0, 0}; CpuId(0, 0, cpu_info0); CpuId(1, 0, cpu_info1); if (cpu_info0[0] >= 7) { CpuId(7, 0, cpu_info7); - CpuId(7, 1, cpu_einfo7); + // Leaf 7 EAX reports the maximum supported sub-leaf. + if (cpu_info7[0] >= 1) { + CpuId(7, 1, cpu_einfo7); + } + } + // Extended leaf 0x80000021 is only valid if the CPU reports it. Querying an + // unsupported extended leaf returns data from the highest basic leaf on + // Intel CPUs, which could spuriously set bits below. + CpuId(0x80000000, 0, cpu_extinfo0); + if ((unsigned)cpu_extinfo0[0] >= 0x80000021u) { CpuId(0x80000021, 0, cpu_amdinfo21); } if (cpu_info0[0] >= 0x21) { diff --git a/source/mjpeg_decoder.cc b/source/mjpeg_decoder.cc index 76195f2e1..4c6a22f93 100644 --- a/source/mjpeg_decoder.cc +++ b/source/mjpeg_decoder.cc @@ -127,7 +127,11 @@ bool MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) { AllocOutputBuffers(GetNumComponents()); for (int i = 0; i < num_outbufs_; ++i) { int scanlines_size = GetComponentScanlinesPerImcuRow(i); - if (scanlines_sizes_[i] != scanlines_size) { + // databuf_[i] is sized by scanlines_size * stride, so a change in either + // must trigger reallocation below. Remember the change before + // scanlines_sizes_[i] is overwritten. + bool scanlines_changed = scanlines_sizes_[i] != scanlines_size; + if (scanlines_changed) { free(scanlines_[i]); scanlines_[i] = static_cast(malloc(scanlines_size * sizeof(uint8_t*))); @@ -147,7 +151,7 @@ bool MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) { // DCTSIZE) is at most slightly larger than UINT16_MAX. // - Sampling factor is stored in 4 bits, so scanlines_size is < 16. int databuf_size = scanlines_size * databuf_stride; - if (databuf_strides_[i] != databuf_stride) { + if (scanlines_changed || databuf_strides_[i] != databuf_stride) { free(databuf_[i]); databuf_[i] = static_cast(malloc(databuf_size)); databuf_strides_[i] = databuf_stride; @@ -268,9 +272,11 @@ bool MJpegDecoder::DecodeToBuffers(uint8_t** planes, SetScanlinePointers(databuf_); int lines_left = dst_height; // Compute amount of lines to skip to implement vertical crop. - // TODO(fbarchard): Ensure skip is a multiple of maximum component - // subsample. ie 2 + // Round skip down to a multiple of the maximum vertical sampling factor so + // that every component skips a whole number of rows. The rows remaining + // below the crop are still >= dst_height. int skip = (GetHeight() - dst_height) / 2; + skip -= skip % decompress_struct_->max_v_samp_factor; if (skip > 0) { // There is no API to skip lines in the output data, so we read them // into the temp buffer. @@ -288,19 +294,23 @@ bool MJpegDecoder::DecodeToBuffers(uint8_t** planes, FinishDecode(); return false; } + // Image lines wanted from this iMCU row, clamped to the destination. + int image_lines_to_copy = GetImageScanlinesPerImcuRow() - skip; + if (image_lines_to_copy > lines_left) { + image_lines_to_copy = lines_left; + } for (int i = 0; i < num_outbufs_; ++i) { - // TODO(fbarchard): Compute skip to avoid this assert(skip % GetVertSubSampFactor(i) == 0); int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i)); int scanlines_to_copy = - GetComponentScanlinesPerImcuRow(i) - rows_to_skip; + DivideAndRoundUp(image_lines_to_copy, GetVertSubSampFactor(i)); int data_to_skip = rows_to_skip * GetComponentStride(i); CopyPlane(databuf_[i] + data_to_skip, GetComponentStride(i), planes[i], GetComponentWidth(i), GetComponentWidth(i), scanlines_to_copy); planes[i] += scanlines_to_copy * GetComponentWidth(i); } - lines_left -= (GetImageScanlinesPerImcuRow() - skip); + lines_left -= image_lines_to_copy; } } @@ -357,8 +367,12 @@ bool MJpegDecoder::DecodeToCallback(CallbackFunction fn, } SetScanlinePointers(databuf_); int lines_left = dst_height; - // TODO(fbarchard): Compute amount of lines to skip to implement vertical crop + // Compute amount of lines to skip to implement vertical crop. + // Round skip down to a multiple of the maximum vertical sampling factor so + // that every component skips a whole number of rows. The rows remaining + // below the crop are still >= dst_height. int skip = (GetHeight() - dst_height) / 2; + skip -= skip % decompress_struct_->max_v_samp_factor; if (skip > 0) { while (skip >= GetImageScanlinesPerImcuRow()) { if (!DecodeImcuRow()) { @@ -374,7 +388,6 @@ bool MJpegDecoder::DecodeToCallback(CallbackFunction fn, return false; } for (int i = 0; i < num_outbufs_; ++i) { - // TODO(fbarchard): Compute skip to avoid this assert(skip % GetVertSubSampFactor(i) == 0); int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i)); int data_to_skip = rows_to_skip * GetComponentStride(i); @@ -382,7 +395,11 @@ bool MJpegDecoder::DecodeToCallback(CallbackFunction fn, // callback. databuf_[i] += data_to_skip; } + // Image lines wanted from this iMCU row, clamped to the destination. int scanlines_to_copy = GetImageScanlinesPerImcuRow() - skip; + if (scanlines_to_copy > lines_left) { + scanlines_to_copy = lines_left; + } (*fn)(opaque, databuf_, databuf_strides_, scanlines_to_copy); // Now change them back. for (int i = 0; i < num_outbufs_; ++i) { diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 5af256382..ecf6fc8a9 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -1414,7 +1414,10 @@ void DetileSplitUVPlane(const uint8_t* src_uv, assert(tile_height > 0); assert(src_stride_uv > 0); - if (width <= 0 || height == 0 || height == INT_MIN) { + // tile_height must be a power of 2 (16 or 32); the row loop below uses it as + // a bit mask. + if (width <= 0 || height == 0 || height == INT_MIN || tile_height <= 0 || + !IS_POWEROFTWO(tile_height)) { return; } // Negative height means invert the image. @@ -1478,7 +1481,10 @@ void DetileToYUY2(const uint8_t* src_y, assert(src_stride_uv > 0); assert(tile_height > 0); - if (width <= 0 || height == 0 || height == INT_MIN || tile_height <= 0) { + // tile_height must be a power of 2 and at least 2 (16 or 32); the row loop + // uses it as a bit mask and the UV tile is tile_height / 2 rows. + if (width <= 0 || height == 0 || height == INT_MIN || tile_height < 2 || + !IS_POWEROFTWO(tile_height)) { return; } // Negative height means invert the image. @@ -1650,7 +1656,8 @@ void MergeRGBPlane(const uint8_t* src_r, } #endif #if defined(HAS_MERGERGBROW_AVX512VBMI) - if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { + if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) == + (kCpuHasAVX512VBMI | kCpuHasAVX512VL)) { MergeRGBRow = MergeRGBRow_AVX512VBMI; } #endif @@ -1864,11 +1871,13 @@ void SplitARGBPlane(const uint8_t* src_argb, dst_r = dst_r + (ptrdiff_t)(height - 1) * dst_stride_r; dst_g = dst_g + (ptrdiff_t)(height - 1) * dst_stride_g; dst_b = dst_b + (ptrdiff_t)(height - 1) * dst_stride_b; - dst_a = dst_a + (ptrdiff_t)(height - 1) * dst_stride_a; dst_stride_r = -dst_stride_r; dst_stride_g = -dst_stride_g; dst_stride_b = -dst_stride_b; - dst_stride_a = -dst_stride_a; + if (dst_a) { + dst_a = dst_a + (ptrdiff_t)(height - 1) * dst_stride_a; + dst_stride_a = -dst_stride_a; + } } if (dst_a == NULL) { @@ -2105,7 +2114,8 @@ void MergeXR30Plane(const uint16_t* src_r, } #endif #if defined(HAS_MERGEXR30ROW_10_AVX512BW) - if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL)) { + if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { MergeXR30Row = MergeXR30Row_10_AVX512BW; } #endif @@ -3334,6 +3344,14 @@ int RGB24Mirror(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGB24MIRRORROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGB24MirrorRow = RGB24MirrorRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGB24MirrorRow = RGB24MirrorRow_SSSE3; + } + } +#endif #if defined(HAS_RGB24MIRRORROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { RGB24MirrorRow = RGB24MirrorRow_Any_AVX2; @@ -4293,6 +4311,11 @@ int ARGBRect(uint8_t* dst_argb, ARGBSetRow = ARGBSetRow_HVX; } #endif +#if defined(HAS_ARGBSETROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBSetRow = ARGBSetRow_RVV; + } +#endif // Set plane for (y = 0; y < height; ++y) { @@ -5437,6 +5460,11 @@ int ARGBShuffle(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBSHUFFLEROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBShuffleRow = ARGBShuffleRow_RVV; + } +#endif for (y = 0; y < height; ++y) { ARGBShuffleRow(src_argb, dst_argb, shuffler, width); @@ -5513,6 +5541,11 @@ int AR64Shuffle(const uint16_t* src_ar64, } } #endif +#if defined(HAS_ARGBSHUFFLEROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + AR64ShuffleRow = ARGBShuffleRow_RVV; + } +#endif for (y = 0; y < height; ++y) { AR64ShuffleRow((uint8_t*)(src_ar64), (uint8_t*)(dst_ar64), shuffler, @@ -6479,6 +6512,9 @@ int UYVYToNV12(const uint8_t* src_uyvy, int height) { int y; int halfwidth = (width + 1) >> 1; + // UYVY rows are processed as awidth (even) samples. Select kernels on the + // width actually passed to them. + int awidth = halfwidth * 2; void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width) = SplitUVRow_C; void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr, @@ -6499,7 +6535,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_SPLITUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { SplitUVRow = SplitUVRow_Any_SSE2; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(awidth, 16)) { SplitUVRow = SplitUVRow_SSE2; } } @@ -6507,7 +6543,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_SPLITUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { SplitUVRow = SplitUVRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(awidth, 32)) { SplitUVRow = SplitUVRow_AVX2; } } @@ -6515,7 +6551,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_SPLITUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { SplitUVRow = SplitUVRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { + if (IS_ALIGNED(awidth, 64)) { SplitUVRow = SplitUVRow_AVX512BW; } } @@ -6523,7 +6559,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_SPLITUVROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { SplitUVRow = SplitUVRow_Any_NEON; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(awidth, 16)) { SplitUVRow = SplitUVRow_NEON; } } @@ -6536,7 +6572,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(awidth, 16)) { SplitUVRow = SplitUVRow_LSX; } } @@ -6544,7 +6580,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_SPLITUVROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { SplitUVRow = SplitUVRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(awidth, 32)) { SplitUVRow = SplitUVRow_LASX; } } @@ -6563,7 +6599,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_INTERPOLATEROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(awidth, 32)) { InterpolateRow = InterpolateRow_AVX2; } } @@ -6576,7 +6612,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(awidth, 16)) { InterpolateRow = InterpolateRow_NEON; } } @@ -6594,7 +6630,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_INTERPOLATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { InterpolateRow = InterpolateRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(awidth, 16)) { InterpolateRow = InterpolateRow_LSX; } } @@ -6602,7 +6638,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_INTERPOLATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { InterpolateRow = InterpolateRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(awidth, 32)) { InterpolateRow = InterpolateRow_LASX; } } @@ -6615,7 +6651,7 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_INTERPOLATEROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { InterpolateRow = InterpolateRow_Any_WASMSIMD; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(awidth, 16)) { InterpolateRow = InterpolateRow_WASMSIMD; } } @@ -6627,7 +6663,6 @@ int UYVYToNV12(const uint8_t* src_uyvy, #endif { - int awidth = halfwidth * 2; // row of y and 2 rows of uv align_buffer_64(rows, awidth * 3); if (!rows) diff --git a/source/rotate.cc b/source/rotate.cc index e8498c43b..59bf00824 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -959,6 +959,7 @@ int I422Rotate(const uint8_t* src_y, // Negative height means invert the image. if (height < 0) { height = -height; + halfheight = (height + 1) >> 1; src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u; src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v; @@ -1266,9 +1267,10 @@ int I010Rotate(const uint16_t* src_y, // Negative height means invert the image. if (height < 0) { height = -height; + halfheight = (height + 1) >> 1; src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; - src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u; - src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v; + src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u; + src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v; src_stride_y = -src_stride_y; src_stride_u = -src_stride_u; src_stride_v = -src_stride_v; @@ -1337,6 +1339,7 @@ int I210Rotate(const uint16_t* src_y, // Negative height means invert the image. if (height < 0) { height = -height; + halfheight = (height + 1) >> 1; src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u; src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v; diff --git a/source/rotate_argb.cc b/source/rotate_argb.cc index 0b7588a0c..00900d175 100644 --- a/source/rotate_argb.cc +++ b/source/rotate_argb.cc @@ -78,6 +78,9 @@ static int ARGBTranspose(const uint8_t* src_argb, #if defined(HAS_TRANSPOSEWX4_32_SVE2) if (TestCpuFlag(kCpuHasSVE2)) { TransposeWx4_32 = TransposeWx4_32_SVE2; +#if defined(HAS_TRANSPOSEWXH_32_NEON) + TransposeWxH_32 = NULL; +#endif } #endif #if defined(HAS_TRANSPOSEWXH_32_SME) diff --git a/source/rotate_sve.cc b/source/rotate_sve.cc index 9ee24d069..f7dc3c14d 100644 --- a/source/rotate_sve.cc +++ b/source/rotate_sve.cc @@ -53,10 +53,10 @@ void TransposeWx4_32_SVE2(const uint8_t* src, "trn2 z5.s, z0.s, z1.s \n" "trn1 z6.s, z2.s, z3.s \n" "trn2 z7.s, z2.s, z3.s \n" - "trn1 z0.d, z4.d, z6.d \n" - "trn2 z1.d, z4.d, z6.d \n" - "trn1 z2.d, z5.d, z7.d \n" - "trn2 z3.d, z5.d, z7.d \n" + "trn1 z0.d, z4.d, z6.d \n" // column 0 + "trn1 z1.d, z5.d, z7.d \n" // column 1 + "trn2 z2.d, z4.d, z6.d \n" // column 2 + "trn2 z3.d, z5.d, z7.d \n" // column 3 "subs %w[width], %w[width], #4 \n" "st1w {z0.s}, p0, [%[dst0]] \n" "st1w {z1.s}, p0, [%[dst1]] \n" @@ -86,10 +86,10 @@ void TransposeWx4_32_SVE2(const uint8_t* src, "trn2 z5.s, z0.s, z1.s \n" "trn1 z6.s, z2.s, z3.s \n" "trn2 z7.s, z2.s, z3.s \n" - "trn1 z0.d, z4.d, z6.d \n" - "trn2 z1.d, z4.d, z6.d \n" - "trn1 z2.d, z5.d, z7.d \n" - "trn2 z3.d, z5.d, z7.d \n" + "trn1 z0.d, z4.d, z6.d \n" // column 0 + "trn1 z1.d, z5.d, z7.d \n" // column 1 + "trn2 z2.d, z4.d, z6.d \n" // column 2 + "trn2 z3.d, z5.d, z7.d \n" // column 3 "whilelt p0.s, wzr, %w[width] \n" "st1w {z0.s}, p0, [%[dst0]] \n" "st1w {z1.s}, p0, [%[dst1]] \n" diff --git a/source/row_any.cc b/source/row_any.cc index e1805ab3b..f2f41ea68 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1575,7 +1575,7 @@ ANY11T(ARGBToAB64Row_Any_SSSE3, ARGBToAB64Row_SSSE3, 4, 8, uint8_t, uint16_t, 3) ANY11T(AR64ToARGBRow_Any_SSSE3, AR64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3) #endif -#ifdef HAS_ARGBTOAR64ROW_SSSE3 +#ifdef HAS_AB64TOARGBROW_SSSE3 ANY11T(AB64ToARGBRow_Any_SSSE3, AB64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3) #endif @@ -1591,7 +1591,7 @@ ANY11T(ARGBToAB64Row_Any_AVX2, ARGBToAB64Row_AVX2, 4, 8, uint8_t, uint16_t, 7) ANY11T(AR64ToARGBRow_Any_AVX2, AR64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7) #endif -#ifdef HAS_ARGBTOAR64ROW_AVX2 +#ifdef HAS_AB64TOARGBROW_AVX2 ANY11T(AB64ToARGBRow_Any_AVX2, AB64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7) #endif @@ -1607,7 +1607,7 @@ ANY11T(ARGBToAB64Row_Any_NEON, ARGBToAB64Row_NEON, 4, 8, uint8_t, uint16_t, 7) ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) #endif -#ifdef HAS_ARGBTOAR64ROW_NEON +#ifdef HAS_AB64TOARGBROW_NEON ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) #endif @@ -1836,7 +1836,7 @@ ANY11P16(HalfFloat1Row_Any_F16C, ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 15) #endif #ifdef HAS_BYTETOFLOATROW_NEON -ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 3, 7) +ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 4, 7) #endif #ifdef HAS_HALFFLOATROW_LSX ANY11P16(HalfFloatRow_Any_LSX, HalfFloatRow_LSX, uint16_t, uint16_t, 2, 2, 7) @@ -2140,6 +2140,9 @@ ANY11M(ARGBMirrorRow_Any_LASX, ARGBMirrorRow_LASX, 4, 7) #ifdef HAS_RGB24MIRRORROW_AVX2 ANY11M(RGB24MirrorRow_Any_AVX2, RGB24MirrorRow_AVX2, 3, 31) #endif +#ifdef HAS_RGB24MIRRORROW_SSSE3 +ANY11M(RGB24MirrorRow_Any_SSSE3, RGB24MirrorRow_SSSE3, 3, 15) +#endif #ifdef HAS_RGB24MIRRORROW_NEON ANY11M(RGB24MirrorRow_Any_NEON, RGB24MirrorRow_NEON, 3, 15) #endif diff --git a/source/row_common.cc b/source/row_common.cc index 31cbf8e4d..8fcec481a 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -583,6 +583,8 @@ void AR64ToAB64Row_C(const uint16_t* src_ar64, uint16_t* dst_ab64, int width) { } // TODO(fbarchard): Make shuffle compatible with SIMD versions +// width is in units of 32 bit ARGB pixels (2 per AR64 pixel) so that +// AR64Shuffle can share the ARGBShuffleRow SIMD kernels. void AR64ShuffleRow_C(const uint8_t* src_ar64, uint8_t* dst_ar64, const uint8_t* shuffler, @@ -2938,8 +2940,10 @@ void DetileToYUY2_C(const uint8_t* src_y, ptrdiff_t src_uv_tile_stride, uint8_t* dst_yuy2, int width) { - for (int x = 0; x < width - 15; x += 16) { - for (int i = 0; i < 8; i++) { + int x; + for (x = 0; x < width - 15; x += 16) { + int i; + for (i = 0; i < 8; i++) { dst_yuy2[0] = src_y[0]; dst_yuy2[1] = src_uv[0]; dst_yuy2[2] = src_y[1]; @@ -2951,6 +2955,16 @@ void DetileToYUY2_C(const uint8_t* src_y, src_y += src_y_tile_stride - 16; src_uv += src_uv_tile_stride - 16; } + // Remaining pixels of a partial 16 pixel tile. + for (x = 0; x < (width & 15); x += 2) { + dst_yuy2[0] = src_y[0]; + dst_yuy2[1] = src_uv[0]; + dst_yuy2[2] = src_y[1]; + dst_yuy2[3] = src_uv[1]; + dst_yuy2 += 4; + src_y += 2; + src_uv += 2; + } } // Unpack MT2T into tiled P010 64 pixels at a time. MT2T's bitstream is encoded @@ -3214,8 +3228,10 @@ void DivideRow_16_C(const uint16_t* src_y, int scale, int width) { int x; + // scale can be 65536 so multiply unsigned to avoid signed overflow. for (x = 0; x < width; ++x) { - dst_y[x] = (src_y[x] * scale) >> 16; + dst_y[x] = + STATIC_CAST(uint16_t, ((uint32_t)src_y[x] * (uint32_t)scale) >> 16); } } @@ -3225,7 +3241,8 @@ void DivideRow_16_C(const uint16_t* src_y, // 4096 = 12 bits // 256 = 16 bits // TODO(fbarchard): change scale to bits -#define C16TO8(v, scale) clamp255(((v) * (scale)) >> 16) +#define C16TO8(v, scale) \ + clamp255((int32_t)(((uint32_t)(v) * (uint32_t)(scale)) >> 16)) void Convert16To8Row_C(const uint16_t* src_y, uint8_t* dst_y, @@ -3943,9 +3960,15 @@ void HalfFloatRow_C(const uint16_t* src, float scale, int width) { int i; - float mult = 1.9259299444e-34f * scale; + // Multiply by scale first, then by 2^-112 to move the exponent into half + // float position. Doing the scale and bias as two multiplies keeps every + // intermediate normal; a combined (scale * 2^-112) constant is a denormal + // for scale < 2^-14 (e.g. 1/65535), which costs a microcode assist per + // multiply on Intel CPUs (50x slower). 2^-112 is a power of 2 so the + // result is identical to the single multiply when that was representable. + const float kScaleBias = 1.9259299444e-34f; // 2^-112 for (i = 0; i < width; ++i) { - float value = src[i] * mult; + float value = (src[i] * scale) * kScaleBias; dst[i] = (uint16_t)((*(const uint32_alias_t*)&value) >> 13); } } diff --git a/source/row_gcc.cc b/source/row_gcc.cc index d27099e20..d2a345e30 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -7056,6 +7056,7 @@ void SplitRGBRow_AVX2(const uint8_t* src_rgb, "lea 0x20(%3),%3 \n" "sub $0x20,%4 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_rgb), // %0 "+r"(dst_r), // %1 "+r"(dst_g), // %2 @@ -8223,7 +8224,8 @@ void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) { "movq (%0),%%xmm2 \n" "lea 0x8(%0),%0 \n" "punpcklbw %%xmm2,%%xmm2 \n" - "punpckhwd %%xmm2,%%xmm3 \n" + "movdqa %%xmm2,%%xmm3 \n" + "punpckhwd %%xmm3,%%xmm3 \n" "punpcklwd %%xmm2,%%xmm2 \n" "movdqu (%1),%%xmm4 \n" "movdqu 0x10(%1),%%xmm5 \n" @@ -11102,7 +11104,7 @@ void ARGBPolynomialRow_SSE2(const uint8_t* src_argb, "addps %%xmm5,%%xmm4 \n" "cvttps2dq %%xmm0,%%xmm0 \n" "cvttps2dq %%xmm4,%%xmm4 \n" - "packuswb %%xmm4,%%xmm0 \n" + "packssdw %%xmm4,%%xmm0 \n" "packuswb %%xmm0,%%xmm0 \n" "movq %%xmm0,(%1) \n" "lea 0x8(%1),%1 \n" @@ -11159,14 +11161,17 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, #endif // HAS_ARGBPOLYNOMIALROW_AVX2 #ifdef HAS_HALFFLOATROW_AVX2 -static float kScaleBias = 1.9259299444e-34f; +// 2^-112 moves the float exponent into half float position. Applied as a +// separate multiply after scale so no denormal constant is formed when +// scale < 2^-14 (see HalfFloatRow_C). +static const float kScaleBias = 1.9259299444e-34f; void HalfFloatRow_AVX2(const uint16_t* src, uint16_t* dst, float scale, int width) { - scale *= kScaleBias; asm volatile( "vbroadcastss %3, %%ymm4 \n" + "vbroadcastss %4, %%ymm6 \n" "vpxor %%ymm5,%%ymm5,%%ymm5 \n" "sub %0,%1 \n" @@ -11179,8 +11184,10 @@ void HalfFloatRow_AVX2(const uint16_t* src, "vpunpcklwd %%ymm5,%%ymm2,%%ymm2 \n" "vcvtdq2ps %%ymm3,%%ymm3 \n" "vcvtdq2ps %%ymm2,%%ymm2 \n" - "vmulps %%ymm3,%%ymm4,%%ymm3 \n" + "vmulps %%ymm3,%%ymm4,%%ymm3 \n" // * scale "vmulps %%ymm2,%%ymm4,%%ymm2 \n" + "vmulps %%ymm3,%%ymm6,%%ymm3 \n" // * 2^-112 + "vmulps %%ymm2,%%ymm6,%%ymm2 \n" "vpsrld $0xd,%%ymm3,%%ymm3 \n" "vpsrld $0xd,%%ymm2,%%ymm2 \n" "vpackssdw %%ymm3, %%ymm2, %%ymm2 \n" // unmutates @@ -11193,11 +11200,13 @@ void HalfFloatRow_AVX2(const uint16_t* src, "+r"(dst), // %1 "+r"(width) // %2 #if defined(__x86_64__) - : "x"(scale) // %3 + : "x"(scale), // %3 + "x"(kScaleBias) // %4 #else - : "m"(scale) // %3 + : "m"(scale), // %3 + "m"(kScaleBias) // %4 #endif - : "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5"); + : "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } #endif // HAS_HALFFLOATROW_AVX2 @@ -11716,10 +11725,10 @@ void ClampFloatToZero_SSE2(const float* src_x, float* dst_y, int width) { "1: \n" "movd (%0),%%xmm0 \n" // load float "maxss %%xmm1, %%xmm0 \n" // clamp to zero - "add 4, %0 \n" + "lea 0x4(%0),%0 \n" "movd %%xmm0, (%1) \n" // store float - "add 4, %1 \n" - "sub $0x4,%2 \n" // 1 float per loop + "lea 0x4(%1),%1 \n" + "sub $0x1,%2 \n" // 1 float per loop "jg 1b \n" : "+r"(src_x), // %0 "+r"(dst_y), // %1 diff --git a/source/row_lasx.cc b/source/row_lasx.cc index 694cf7cdd..99042acd2 100644 --- a/source/row_lasx.cc +++ b/source/row_lasx.cc @@ -2341,9 +2341,10 @@ void HalfFloatRow_LASX(const uint16_t* src, uint16_t* dst, float scale, int width) { - float mult = 1.9259299444e-34f * scale; + const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C asm volatile( - "xvldrepl.w $xr3, %[mult], 0\n\t" + "xvldrepl.w $xr3, %[scale], 0\n\t" + "xvldrepl.w $xr5, %[bias], 0\n\t" "xvldi $xr4, 0\n\t" "1:\n\t" "xvld $xr0, %[src], 0\n\t" @@ -2353,6 +2354,8 @@ void HalfFloatRow_LASX(const uint16_t* src, "xvffint.s.wu $xr2, $xr2\n\t" "xvfmul.s $xr1, $xr1, $xr3\n\t" "xvfmul.s $xr2, $xr2, $xr3\n\t" + "xvfmul.s $xr1, $xr1, $xr5\n\t" + "xvfmul.s $xr2, $xr2, $xr5\n\t" "xvsrlni.h.w $xr2, $xr1, 13\n\t" "xvst $xr2, %[dst], 0\n\t" "addi.d %[src], %[src], 32\n\t" @@ -2360,8 +2363,8 @@ void HalfFloatRow_LASX(const uint16_t* src, "addi.d %[cnt], %[cnt], -16\n\t" "bnez %[cnt], 1b\n\t" : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) - : [mult] "r"(&mult) - : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); + : [scale] "r"(&scale), [bias] "r"(&bias) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5"); } void ARGBToYMatrixRow_LASX(const uint8_t* src_argb, diff --git a/source/row_lsx.cc b/source/row_lsx.cc index 561012b07..8bd10f519 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -2022,9 +2022,10 @@ void HalfFloatRow_LSX(const uint16_t* src, uint16_t* dst, float scale, int width) { - float mult = 1.9259299444e-34f * scale; + const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C asm volatile( - "vldrepl.w $vr3, %[mult], 0\n\t" + "vldrepl.w $vr3, %[scale], 0\n\t" + "vldrepl.w $vr5, %[bias], 0\n\t" "vldi $vr4, 0\n\t" "1:\n\t" "vld $vr0, %[src], 0\n\t" @@ -2034,6 +2035,8 @@ void HalfFloatRow_LSX(const uint16_t* src, "vffint.s.wu $vr2, $vr2\n\t" "vfmul.s $vr1, $vr1, $vr3\n\t" "vfmul.s $vr2, $vr2, $vr3\n\t" + "vfmul.s $vr1, $vr1, $vr5\n\t" + "vfmul.s $vr2, $vr2, $vr5\n\t" "vsrlni.h.w $vr2, $vr1, 13\n\t" "vst $vr2, %[dst], 0\n\t" "addi.d %[src], %[src], 16\n\t" @@ -2041,8 +2044,8 @@ void HalfFloatRow_LSX(const uint16_t* src, "addi.d %[cnt], %[cnt], -8\n\t" "bnez %[cnt], 1b\n\t" : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) - : [mult] "r"(&mult) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); + : [scale] "r"(&scale), [bias] "r"(&bias) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); } void ARGBToYMatrixRow_LSX(const uint8_t* src_argb, diff --git a/source/row_neon.cc b/source/row_neon.cc index 4e4bc18a1..8f921c1e4 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -3369,20 +3369,25 @@ void HalfFloatRow_NEON(const uint16_t* src, "vcvt.f32.u32 q9, q9 \n" "vcvt.f32.u32 q10, q10 \n" "vcvt.f32.u32 q11, q11 \n" - "vmul.f32 q8, q8, %y3 \n" // adjust exponent + "vmul.f32 q8, q8, %y3 \n" // * scale "vmul.f32 q9, q9, %y3 \n" "vmul.f32 q10, q10, %y3 \n" "vmul.f32 q11, q11, %y3 \n" + "vmul.f32 q8, q8, %y4 \n" // adjust exponent + "vmul.f32 q9, q9, %y4 \n" + "vmul.f32 q10, q10, %y4 \n" + "vmul.f32 q11, q11, %y4 \n" "vqshrn.u32 d0, q8, #13 \n" // isolate halffloat "vqshrn.u32 d1, q9, #13 \n" "vqshrn.u32 d2, q10, #13 \n" "vqshrn.u32 d3, q11, #13 \n" "vst1.16 {q0, q1}, [%1]! \n" // store 16 fp16 "bgt 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "w"(scale * 1.9259299444e-34f) // %3 + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "w"(scale), // %3 + "w"(1.9259299444e-34f) // %4 2^-112, see HalfFloatRow_C : "cc", "memory", "q0", "q1", "q8", "q9", "q10", "q11"); } diff --git a/source/row_neon64.cc b/source/row_neon64.cc index a6c083bf4..f4e0315c6 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -4767,20 +4767,25 @@ void HalfFloatRow_NEON(const uint16_t* src, "scvtf v4.4s, v4.4s \n" "scvtf v3.4s, v3.4s \n" "scvtf v5.4s, v5.4s \n" - "fmul v2.4s, v2.4s, %3.s[0] \n" // adjust exponent + "fmul v2.4s, v2.4s, %3.s[0] \n" // * scale "fmul v4.4s, v4.4s, %3.s[0] \n" "fmul v3.4s, v3.4s, %3.s[0] \n" "fmul v5.4s, v5.4s, %3.s[0] \n" + "fmul v2.4s, v2.4s, %4.s[0] \n" // adjust exponent + "fmul v4.4s, v4.4s, %4.s[0] \n" + "fmul v3.4s, v3.4s, %4.s[0] \n" + "fmul v5.4s, v5.4s, %4.s[0] \n" "uqshrn v0.4h, v2.4s, #13 \n" // isolate halffloat "uqshrn v1.4h, v4.4s, #13 \n" "uqshrn2 v0.8h, v3.4s, #13 \n" "uqshrn2 v1.8h, v5.4s, #13 \n" "stp q0, q1, [%1], #32 \n" // store 16 fp16 "b.gt 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "w"(scale * 1.9259299444e-34f) // %3 + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "w"(scale), // %3 + "w"(1.9259299444e-34f) // %4 2^-112, see HalfFloatRow_C : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5"); } @@ -4836,8 +4841,8 @@ void ConvertFP16ToFP32Column_NEON(const uint16_t* src, // fp16 float* dst, int width) { asm volatile( - "cmp %w2, #8 \n" // Is there 8 rows? - "b.lo 2f \n" + "subs %w2, %w2, #8 \n" // Is there 8 rows? + "b.lt 2f \n" "1: \n" "ld1 {v0.h}[0], [%0], %3 \n" // load 8 halffloats "ld1 {v0.h}[1], [%0], %3 \n" @@ -4852,15 +4857,16 @@ void ConvertFP16ToFP32Column_NEON(const uint16_t* src, // fp16 "fcvtl v2.4s, v0.4h \n" // 4 floats "fcvtl v3.4s, v1.4h \n" // 4 more floats "stp q2, q3, [%1], #32 \n" // store 8 floats - "b.gt 1b \n" - "cmp %w2, #1 \n" // Is there 1 value? - "b.lo 3f \n" + "b.ge 1b \n" "2: \n" + "adds %w2, %w2, #8 \n" // Restore remainder. + "b.eq 3f \n" + "4: \n" "ld1 {v1.h}[0], [%0], %3 \n" // load 1 halffloats "subs %w2, %w2, #1 \n" // 1 floats per loop "fcvtl v2.4s, v1.4h \n" // 1 floats "str s2, [%1], #4 \n" // store 1 floats - "b.gt 2b \n" + "b.gt 4b \n" "3: \n" : "+r"(src), // %0 "+r"(dst), // %1 diff --git a/source/row_rvv.cc b/source/row_rvv.cc index d64729845..363cbec9b 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -2623,6 +2623,76 @@ void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBMIRRORROW_RVV +#ifdef HAS_ARGBSETROW_RVV +void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width) { + size_t vl; + asm volatile( + "vsetvli %[vl], %[w], e32, m8, ta, ma \n" + "vmv.v.x v8, %[v32] \n" + "1: \n" + "vsetvli %[vl], %[w], e32, m8, ta, ma \n" + "vse32.v v8, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "slli %[vl], %[vl], 2 \n" + "add %[dst], %[dst], %[vl] \n" + "bnez %[w], 1b \n" + : [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl) // %[vl] + : [v32] "r"(v32) // %[v32] + : RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15"); +} +#endif // HAS_ARGBSETROW_RVV + +#ifdef HAS_ARGBSHUFFLEROW_RVV +// Shuffle the 4 bytes of each ARGB pixel with a 16 byte shuffler table (only +// the first 4 entries are used, the pattern repeats every pixel), e.g. +// {0, 1, 2, 3} is a copy and {2, 1, 0, 3} swaps R and B. +// Byte index vector: index[i] = (i & ~3) + shuffler[i & 3], gathered with a +// 16 bit index so any VLEN works (e8 indices would be limited to 256 bytes). +void ARGBShuffleRow_RVV(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width) { + size_t vl; + size_t w = (size_t)width * 4; // bytes + // The 4 shuffler bytes packed little endian so a 32 bit broadcast lays them + // out as s0 s1 s2 s3 s0 s1 s2 s3 ... in memory order. + uint32_t pattern = (uint32_t)shuffler[0] | ((uint32_t)shuffler[1] << 8) | + ((uint32_t)shuffler[2] << 16) | + ((uint32_t)shuffler[3] << 24); + asm volatile( + // v16..v23 (e16, m8): index = (i & ~3) + shuffler[i & 3]. + "vsetvli %[vl], %[w], e16, m8, ta, ma \n" + "vid.v v16 \n" + "vand.vi v16, v16, -4 \n" + "srli %[vl], %[vl], 2 \n" + "vsetvli zero, %[vl], e32, m4, ta, ma \n" + "vmv.v.x v8, %[pattern] \n" // s0 s1 s2 s3 ... + "vsetvli %[vl], %[w], e16, m8, ta, ma \n" + "vzext.vf2 v24, v8 \n" // bytes -> e16 + "vadd.vv v16, v16, v24 \n" + "1: \n" + "vsetvli %[vl], %[w], e8, m4, ta, ma \n" + "vle8.v v8, (%[src]) \n" + "vrgatherei16.vv v0, v8, v16 \n" + "vse8.v v0, (%[dst]) \n" + "sub %[w], %[w], %[vl] \n" + "add %[src], %[src], %[vl] \n" + "add %[dst], %[dst], %[vl] \n" + "bnez %[w], 1b \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(w), // %[w] + [vl] "=&r"(vl) // %[vl] + : [pattern] "r"(pattern) // %[pattern] + : RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", + "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29", "v30", "v31"); +} +#endif // HAS_ARGBSHUFFLEROW_RVV + #ifdef HAS_MIRRORUVROW_RVV void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) { int vl; diff --git a/source/row_sve.cc b/source/row_sve.cc index 28ddc94fb..716428096 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -777,7 +777,7 @@ void DivideRow_16_SVE2(const uint16_t* src_y, "cnth %x[vl] \n" "dup z0.h, %w[scale] \n" "subs %w[width], %w[width], %w[vl], lsl #1 \n" - "b.le 2f \n" + "b.lt 2f \n" // Run bulk of computation with the same predicates to avoid predicate // generation overhead. @@ -792,7 +792,7 @@ void DivideRow_16_SVE2(const uint16_t* src_y, "st1h {z1.h}, p0, [%[dst]] \n" "st1h {z2.h}, p0, [%[dst], #1, mul vl] \n" "incb %[dst], all, mul #2 \n" - "b.gt 1b \n" + "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl], lsl #1 \n" @@ -827,6 +827,10 @@ void DivideRow_16_SVE2(const uint16_t* src_y, "fmul z1.s, z1.s, z4.s \n" \ "fmul z2.s, z2.s, z4.s \n" \ "fmul z3.s, z3.s, z4.s \n" \ + "fmul z0.s, z0.s, z5.s \n" \ + "fmul z1.s, z1.s, z5.s \n" \ + "fmul z2.s, z2.s, z5.s \n" \ + "fmul z3.s, z3.s, z5.s \n" \ "uqshrnb z0.h, z0.s, #13 \n" \ "uqshrnb z1.h, z1.s, #13 \n" \ "uqshrnb z2.h, z2.s, #13 \n" \ @@ -840,6 +844,7 @@ void HalfFloatRow_SVE2(const uint16_t* src, asm("cntw %x0" : "=r"(vl)); asm volatile( "mov z4.s, %s[scale] \n" + "mov z5.s, %s[bias] \n" "subs %w[width], %w[width], %w[vl], lsl #2 \n" "b.lt 2f \n" @@ -881,14 +886,16 @@ void HalfFloatRow_SVE2(const uint16_t* src, "st1h {z3.s}, p3, [%[dst], #3, mul vl] \n" "99: \n" - : [src] "+r"(src), // %[src] - [dst] "+r"(dst), // %[dst] - [width] "+r"(width) // %[width] - : [vl] "r"(vl), // %[vl] - [vl2] "r"(vl * 2), // %[vl2] - [vl3] "r"(vl * 3), // %[vl3] - [scale] "w"(scale * 1.9259299444e-34f) // %[scale] - : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2", "p3"); + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [width] "+r"(width) // %[width] + : [vl] "r"(vl), // %[vl] + [vl2] "r"(vl * 2), // %[vl2] + [vl3] "r"(vl * 3), // %[vl3] + [scale] "w"(scale), // %[scale] + [bias] "w"(1.9259299444e-34f) // %[bias] 2^-112 + : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "p0", "p1", "p2", + "p3"); } void HalfFloat1Row_SVE2(const uint16_t* src, @@ -1221,7 +1228,7 @@ void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) { "st1b {z0.b}, p0, [%[dst], #0, mul vl] \n" "st1b {z1.b}, p0, [%[dst], #1, mul vl] \n" "incb %[dst], all, mul #2 \n" - "b.gt 1b \n" + "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl2] \n" diff --git a/source/scale.cc b/source/scale.cc index 8650bffaa..a9fdf79c8 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -938,6 +938,7 @@ static void ScaleAddCols1_16_C(int dst_width, int boxwidth = MIN1(dx >> 16); int scaleval = 65536 / (boxwidth * boxheight); int i; + x >>= 16; for (i = 0; i < dst_width; ++i) { *dst_ptr++ = SumPixels_16(boxwidth, src_ptr + x) * scaleval >> 16; x += boxwidth; @@ -1498,7 +1499,7 @@ static int ScalePlaneBilinearUp(int src_width, #endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleFilterCols = ScaleColsUp2_C; -#if defined(HAS_SCALECOLS_SSE2) +#if defined(HAS_SCALECOLSUP2_SSE2) if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { ScaleFilterCols = ScaleColsUp2_SSE2; } @@ -2092,7 +2093,7 @@ static void ScalePlaneSimple(int src_width, if (src_width * 2 == dst_width && x < 0x8000) { ScaleCols = ScaleColsUp2_C; -#if defined(HAS_SCALECOLS_SSE2) +#if defined(HAS_SCALECOLSUP2_SSE2) if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { ScaleCols = ScaleColsUp2_SSE2; } diff --git a/source/scale_any.cc b/source/scale_any.cc index 86da894cd..713f3f186 100644 --- a/source/scale_any.cc +++ b/source/scale_any.cc @@ -1024,7 +1024,7 @@ SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_AVX2, SBUH2LANY(ScaleUVRowUp2_Linear_Any_NEON, ScaleUVRowUp2_Linear_NEON, ScaleUVRowUp2_Linear_C, - 15, + 7, uint8_t) #endif @@ -1032,7 +1032,7 @@ SBUH2LANY(ScaleUVRowUp2_Linear_Any_NEON, SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_NEON, ScaleUVRowUp2_Linear_16_NEON, ScaleUVRowUp2_Linear_16_C, - 15, + 7, uint16_t) #endif diff --git a/source/scale_argb.cc b/source/scale_argb.cc index 6c7423620..dd9df3612 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -482,7 +482,8 @@ static int ScaleARGBBilinearDown(int src_width, // TODO(fbarchard): Consider not allocating row buffer for kFilterLinear. // Allocate a row of ARGB. { - align_buffer_64(row, clip_src_width * 4); + // clip_src_width is already in bytes (4 per ARGB pixel). + align_buffer_64(row, clip_src_width); if (!row) return 1; diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index c92a482e4..b018af795 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -1275,21 +1275,6 @@ void ScaleRowUp2_Bilinear_16_SSE2(const uint16_t* src_ptr, LABELALIGN "1: \n" - "movq (%0),%%xmm0 \n" // 0011 (16b, 1u1v) - "movq 4(%0),%%xmm1 \n" // 1122 (16b, 1u1v) - "punpcklwd %%xmm7,%%xmm0 \n" // 0011 (near) (32b, 1u1v) - "punpcklwd %%xmm7,%%xmm1 \n" // 1122 (near) (32b, 1u1v) - "movdqa %%xmm0,%%xmm2 \n" - "movdqa %%xmm1,%%xmm3 \n" - "pshufd $0b01001110,%%xmm2,%%xmm2 \n" // 1100 (far) (1, lo) - "pshufd $0b01001110,%%xmm3,%%xmm3 \n" // 2211 (far) (1, hi) - "paddd %%xmm0,%%xmm2 \n" // near+far (1, lo) - "paddd %%xmm1,%%xmm3 \n" // near+far (1, hi) - "paddd %%xmm0,%%xmm0 \n" // 2*near (1, lo) - "paddd %%xmm1,%%xmm1 \n" // 2*near (1, hi) - "paddd %%xmm2,%%xmm0 \n" // 3*near+far (1, lo) - "paddd %%xmm3,%%xmm1 \n" // 3*near+far (1, hi) - "movq (%0),%%xmm0 \n" // 0123 (16b) "movq 2(%0),%%xmm1 \n" // 1234 (16b) "punpcklwd %%xmm7,%%xmm0 \n" // 0123 (32b) diff --git a/source/scale_neon64.cc b/source/scale_neon64.cc index 3514bb079..2bb04b6e9 100644 --- a/source/scale_neon64.cc +++ b/source/scale_neon64.cc @@ -1013,9 +1013,9 @@ void ScaleAddRow_NEON(const uint8_t* src_ptr, } #define SCALE_FILTER_COLS_STEP_ADDR \ - "lsr %[tmp_offset], %x[x], #16 \n" \ + "sbfx %[tmp_offset], %x[x], #16, #16 \n" \ "add %[tmp_ptr], %[src_ptr], %[tmp_offset] \n" \ - "add %x[x], %x[x], %x[dx] \n" + "add %w[x], %w[x], %w[dx] \n" // The Neon version mimics this formula (from scale_common.cc): // #define BLENDER(a, b, f) (uint8_t)((int)(a) + diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 1e21a4948..324822a0e 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -235,7 +235,7 @@ static int ScaleUVDown4Box(int src_width, } } #endif -#if defined(HAS_SCALEUVROWDOWN2BOX_SME) +#if defined(HAS_SCALEUVROWDOWN2_SME) if (TestCpuFlag(kCpuHasSME)) { ScaleUVRowDown2 = ScaleUVRowDown2Box_SME; } @@ -693,7 +693,8 @@ static int ScaleUVBilinearDown(int src_width, { const uint32_t max_y = (uint32_t)(src_height - 1) << 16; uint32_t uy = y; - align_buffer_64(row, clip_src_width * 2); + // clip_src_width is already in bytes (2 per UV pixel). + align_buffer_64(row, clip_src_width); if (!row) return 1; if (uy > max_y) { @@ -1450,6 +1451,14 @@ int UVScale_16(const uint16_t* src_uv, } #endif + // ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone, which has + // no 16 bit implementation other than the copy above. The Up2 kernels only + // read the pixels they need, so restore Linear or Bilinear for a 2x upscale + // so P010ToP410 and P210ToP410 work on 1 pixel wide images. + if (filtering == kFilterNone && src_width == 1 && dst_width <= 2) { + filtering = (src_height == dst_height) ? kFilterLinear : kFilterBilinear; + } + if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) { ScaleUVLinearUp2_16(src_width, src_height, dst_width, dst_height, src_stride_uv, dst_stride_uv, src_uv, dst_uv);