coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

[libyuv] Fix top 20 bugs and performance issues

Browse Source 
1. Fix multi-bit TestCpuFlag(A | B) checks in AYUVToNV12, AYUVToNV21,
   RAWToRGBA, NV12ToRGB24Matrix, NV21ToRGB24Matrix, MergeRGBPlane, and
   MergeXR30Plane to compare == (A | B) so AVX-512 VBMI/BW kernels are
   not dispatched when AVX512VBMI or AVX512VL is absent.
2. Fix 2-byte SVE2 heap buffer overflow on odd widths in
   I422ToARGB1555Row_SVE_SC and I422ToARGB4444Row_SVE_SC using
   trn1/trn2 + st2b under .b predicates.
3. Fix ARGBPolynomialRow_SSE2 to use packssdw before packuswb, and add
   missing vzeroupper to SplitRGBRow_AVX2.
4. Fix CalcFrameSsim to return 0.0 instead of 0.0 / 0 (NaN) when
   width <= 8 || height <= 8 so I420Ssim stays finite on small frames.
5. Fix negative-height vertical inversion in ConvertFromI420,
   SplitARGBPlane (when dst_a is NULL), I422Rotate, I010Rotate, and
   I210Rotate.
6. Fix horizontal flip (negative x/dx) in ScaleFilterCols_NEON (signed
   sbfx #16, #16 offset extraction) and ScaleAddCols1_16_C (shift
   x >>= 16 before indexing src_ptr).
7. Fix ConvertToARGB odd crop_x chroma alignment in NV12/NV21, src_u
   offset in I420/YV12/J420/H420/U420, and FOURCC_U422 calling
   U422ToARGB instead of H422ToARGB.
8. Remove dead 14-instruction block in ScaleRowUp2_Bilinear_16_SSE2.
9. Fix UVScale_16 for src_width == 1 (P010ToP410/P210ToP410), enable
   and fix column order in TransposeWx4_32_SVE2, fix ByteToFloatRow
   NEON Any BPP (3 -> 4), fix AB64ToARGBRow_Any_*, ScaleColsUp2_SSE2,
   ScaleUVRowDown2Box_SME, and UYVYToARGBRow_SME macro guards, fix SVE2
   loop tail branches (b.lt/b.ge), and remove duplicate HAS_* macros.
10. Fix ConvertFP16ToFP32Column_NEON loop condition and unsigned b.lo
    underflow on widths < 8 or not multiples of 8.
11. Fix UYVYToNV12 IS_ALIGNED(awidth, ...) checks and WASM SIMD
    IS_ALIGNED(width, 16) checks in I420ToRGB24Matrix /
    I422ToRGB24Matrix.
12. Wire SplitUVRow_SVE2, SplitUVRow_LSX, and SplitUVRow_LASX in
    NV24ToARGB.
13. Fix MJpegDecoder heap buffer overflows in LoadFrame (reallocate
    databuf_[i] when scanlines_size changes), DecodeToBuffers, and
    DecodeToCallback (clamp scanlines_to_copy to lines_left and align
    skip to max_v_samp_factor).
14. Replace width & (vl - 1) tail calculations with post-loop remainder
    across 18 SVE2 functions in row_sve.h for non-power-of-2 SVE VL,
    and remove duplicate label 2: in UYVYToARGBRow_SVE_SC.
15. Handle width & 15 tail pixels in DetileToYUY2_C and validate
    IS_POWEROFTWO(tile_height) in DetileSplitUVPlane and DetileToYUY2.
16. Avoid signed 32-bit overflow in DivideRow_16_C and C16TO8, fix
    ClampFloatToZero_SSE2 pointer increment (lea 0x4) and loop counter
    (sub $0x1), and initialize xmm3 in ARGBCopyYToAlphaRow_SSE2.
17. Check max leaf 7 subleaf and max extended CPUID leaf in GetCpuFlags
    before querying subleaf 7:1 and extended leaf 0x80000021.
18. Free plane_u on error in I422ToNV21, and fix 4x/2x scratch row
    over-allocation in ScaleARGBBilinearDown and ScaleUVBilinearDown.
19. Add ARGBSetRow_RVV and ARGBShuffleRow_RVV in row_rvv.cc.
20. Wire ScaleAddRow_AVX512BW in ScaleUVBox, fix NEON mask (15 -> 7) in
    ScaleUVRowUp2_Linear_Any_NEON / _16_Any_NEON, use SIMD
    InterpolateRow + SplitUVRow in NV16ToI420, wire SSSE3/SSE2 in
    ARGBToI400Matrix / ARGBToYUY2Matrix / ARGBToUYVYMatrix and
    RGB24Mirror, and split HalfFloatRow multiply into
    (src * scale) * 2^-112 to avoid 50x Intel denormal stalls.

Bug: b/574714081
Test: libyuv_unittest
Change-Id: Iec3ae5c58d52bee3ef13cf9b765ac84b733aa576
This commit is contained in:
Frank Barchard 2026-10-19 16:24:54 -07:00
parent 378bc91324
commit 1b37261a0d
30 changed files with 699 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
include/libyuv/row.h
View File

@ -153,13 +153,11 @@ extern "C" {
#define HAS_ARGBCOPYALPHAROW_AVX2
#define HAS_ARGBCOPYYTOALPHAROW_AVX2
#define HAS_ARGBEXTRACTALPHAROW_AVX2
#define HAS_ARGBMIRRORROW_AVX2
#define HAS_ARGBPOLYNOMIALROW_AVX2
#define HAS_ARGBSHUFFLEROW_AVX2
#define HAS_ARGBTORGB565DITHERROW_AVX2
#define HAS_COPYROW_AVX
#define HAS_HALFFLOATROW_AVX2
#define HAS_I422TOARGBROW_AVX2
#define HAS_I422TORGB24ROW_AVX2
#define HAS_I422TORGBAROW_AVX2
#define HAS_I422TORGB565ROW_AVX2
@ -168,12 +166,8 @@ extern "C" {
#define HAS_I444TOARGBROW_AVX2
#define HAS_I444TORGB24ROW_AVX2
#define HAS_J400TOARGBROW_AVX2
#define HAS_MIRRORROW_AVX2
#define HAS_MIRRORSPLITUVROW_AVX2
#define HAS_NV12TOARGBROW_AVX2
#define HAS_NV12TORGB24ROW_AVX2
#define HAS_NV12TORGB565ROW_AVX2
#define HAS_NV21TOARGBROW_AVX2
#define HAS_NV21TORGB24ROW_AVX2
#define HAS_SPLITUVROW_AVX2
#define HAS_UYVYTOARGBROW_AVX2
@ -230,6 +224,7 @@ extern "C" {
#define HAS_MIRRORUVROW_SSSE3
#define HAS_NV21TOYUV24ROW_SSSE3
#define HAS_P210TOAR30ROW_SSSE3
#define HAS_RGB24MIRRORROW_SSSE3
#define HAS_P210TOARGBROW_SSSE3
#define HAS_P410TOAR30ROW_SSSE3
#define HAS_P410TOARGBROW_SSSE3
@ -474,7 +469,6 @@ extern "C" {
#define HAS_RGBTOUV444MATRIXROW_AVX512BW
#define HAS_ARGBTOYMATRIXROW_AVX512BW
#define HAS_I422TORGB24ROW_AVX512VBMI
#define HAS_RAWTORGB24ROW_AVX512VBMI
#define HAS_I422TORGB24ROW_AVX512BW
#define HAS_ARGBTOUVMATRIXROW_AVX512BW
#define HAS_J400TOARGBROW_AVX512BW
@ -796,6 +790,7 @@ extern "C" {
#define HAS_P210TOARGBROW_SME
#define HAS_P410TOAR30ROW_SME
#define HAS_P410TOARGBROW_SME
#define HAS_UYVYTOARGBROW_SME
#define HAS_YUY2TOARGBROW_SME
#endif
@ -959,6 +954,8 @@ extern "C" {
#define HAS_ARGBMULTIPLYROW_RVV
#define HAS_ARGBTOAR64ROW_RVV
#define HAS_ARGBMIRRORROW_RVV
#define HAS_ARGBSETROW_RVV
#define HAS_ARGBSHUFFLEROW_RVV
#define HAS_ARGBTOUV444MATRIXROW_RVV
#define HAS_ARGBTOUVMATRIXROW_RVV
#define HAS_ARGBTOYMATRIXROW_RVV
@ -2972,6 +2969,9 @@ void ARGBMirrorRow_Any_LASX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RGB24MirrorRow_SSSE3(const uint8_t* src_rgb24,
uint8_t* dst_rgb24,
int width);
void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
uint8_t* dst_rgb24,
int width);
@ -2982,6 +2982,9 @@ void RGB24MirrorRow_NEON(const uint8_t* src_rgb24,
uint8_t* dst_rgb24,
int width);
void RGB24MirrorRow_C(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width);
void RGB24MirrorRow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RGB24MirrorRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
@ -4215,6 +4218,7 @@ void SetRow_Any_LASX(uint8_t* dst_ptr, uint8_t v32, int width);
void ARGBSetRow_C(uint8_t* dst_argb, uint32_t v32, int width);
void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width);
void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width);
void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width);
void ARGBSetRow_Any_NEON(uint8_t* dst_ptr, uint32_t v32, int width);
void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width);
void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width);
@ -4242,6 +4246,10 @@ void ARGBShuffleRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width);
void ARGBShuffleRow_RVV(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width);
void ARGBShuffleRow_LSX(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,

336
include/libyuv/row_sve.h
View File

@ -466,7 +466,7 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y,
"ptrue p0.b \n"
"dup z19.b, #255 \n" // Alpha
YUVTORGB_SVE_SETUP
"cmp %w[width], %w[vl] \n"
"subs %w[width], %w[width], %w[vl] \n"
"mov z1.h, #128 \n" // U/V
"umullb z6.h, z30.b, z1.b \n"
"umullb z4.h, z28.b, z1.b \n" // DB
@ -475,25 +475,28 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y,
"sub z4.h, z4.h, z25.h \n"
"sub z5.h, z5.h, z27.h \n"
"sub z6.h, z26.h, z6.h \n"
"b.le 2f \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p1.h \n"
"sub %w[width], %w[width], %w[vl] \n"
"1: \n" //
READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE
"subs %w[width], %w[width], %w[vl] \n"
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
"b.gt 1b \n"
"add %w[width], %w[width], %w[vl] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"2: \n"
"whilelt p1.h, wzr, %w[width] \n" //
READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width), // %[width]
@ -702,7 +705,11 @@ static inline void I422ToARGB1555Row_SVE_SC(
"whilelt p1.b, wzr, %w[width] \n" //
READYUV422_SVE_2X I422TORGB_SVE_2X
RGBTOARGB8_SVE_TOP_2X RGB8TOARGB1555_SVE_FROM_TOP_2X
"st2h {z0.h, z1.h}, p1, [%[dst]] \n"
// Need to permute the data on the final iteration such that the
// predicates (.b) line up with the 16-bit element data.
"trn1 z2.b, z0.b, z1.b \n"
"trn2 z3.b, z0.b, z1.b \n"
"st2b {z2.b, z3.b}, p1, [%[dst]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
@ -761,7 +768,11 @@ static inline void I422ToARGB4444Row_SVE_SC(
"whilelt p1.b, wzr, %w[width] \n" //
READYUV422_SVE_2X I422TORGB_SVE_2X
RGBTOARGB8_SVE_TOP_2X RGB8TOARGB4444_SVE_FROM_TOP_2X
"st2h {z0.h, z1.h}, p1, [%[dst]] \n"
// Need to permute the data on the final iteration such that the
// predicates (.b) line up with the 16-bit element data.
"trn1 z2.b, z0.b, z1.b \n"
"trn2 z3.b, z0.b, z1.b \n"
"st2b {z2.b, z3.b}, p1, [%[dst]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
@ -788,7 +799,7 @@ static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y,
YUVTORGB_SVE_SETUP
"dup z19.b, #255 \n" // Alpha
"subs %w[width], %w[width], %w[vl] \n"
"b.le 2f \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
@ -798,7 +809,7 @@ static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y,
"subs %w[width], %w[width], %w[vl] \n"
"st2h {z19.h, z20.h}, p1, [%[dst_rgba]] \n"
"add %[dst_rgba], %[dst_rgba], %[vl], lsl #2 \n"
"b.gt 1b \n"
"b.ge 1b \n"
// Calculate a predicate for the final iteration to deal with the tail.
"2: \n"
@ -836,7 +847,7 @@ static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y,
YUVTORGB_SVE_SETUP_AR30
"dup z23.h, %w[limit] \n"
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
"b.le 2f \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
@ -846,7 +857,7 @@ static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y,
"1: \n" //
READYUV422_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
"b.gt 1b \n"
"b.ge 1b \n"
// Calculate a predicate for the final iteration to deal with the tail.
"2: \n"
@ -985,8 +996,6 @@ static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cntb %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1010,21 +1019,21 @@ static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.b, wzr, %w[width_last_y] \n"
"whilelt p2.b, wzr, %w[width_last_uv] \n" //
"whilelt p1.b, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.b, wzr, %w[width] \n" //
READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
"st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1035,8 +1044,6 @@ static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cntb %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1060,21 +1067,21 @@ static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.b, wzr, %w[width_last_y] \n"
"whilelt p2.b, wzr, %w[width_last_uv] \n" //
"whilelt p1.b, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.b, wzr, %w[width] \n" //
READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
"st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_vu), // %[src_vu]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_vu), // %[src_vu]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1086,8 +1093,6 @@ static inline void NV12ToRGB24Row_SVE_SC(
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cntb %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1111,21 +1116,21 @@ static inline void NV12ToRGB24Row_SVE_SC(
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.b, wzr, %w[width_last_y] \n"
"whilelt p2.b, wzr, %w[width_last_uv] \n" //
"whilelt p1.b, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.b, wzr, %w[width] \n" //
READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
"st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1137,8 +1142,6 @@ static inline void NV21ToRGB24Row_SVE_SC(
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cntb %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1162,21 +1165,21 @@ static inline void NV21ToRGB24Row_SVE_SC(
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.b, wzr, %w[width_last_y] \n"
"whilelt p2.b, wzr, %w[width_last_uv] \n" //
"whilelt p1.b, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.b, wzr, %w[width] \n" //
READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
"st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_vu), // %[src_vu]
[dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_vu), // %[src_vu]
[dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1195,8 +1198,6 @@ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2,
uint32_t nv_uv_step = 0x04040404U;
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n"
"index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
@ -1221,8 +1222,10 @@ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.h, wzr, %w[width_last_uv] \n" //
"whilelt p1.h, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.h, wzr, %w[width] \n" //
READYUY2_SVE NVTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
@ -1234,9 +1237,7 @@ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2,
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
[nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
[nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1255,8 +1256,6 @@ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy,
uint32_t nv_uv_step = 0x04040404U;
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n"
"index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
@ -1281,9 +1280,10 @@ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"2: \n"
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.h, wzr, %w[width_last_uv] \n" //
"whilelt p1.h, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.h, wzr, %w[width] \n" //
READUYVY_SVE NVTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
@ -1295,9 +1295,7 @@ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy,
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
[nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
[nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1309,7 +1307,6 @@ static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1332,20 +1329,19 @@ static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1359,7 +1355,6 @@ static inline void I210AlphaToARGBRow_SVE_SC(
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1385,23 +1380,22 @@ static inline void I210AlphaToARGBRow_SVE_SC(
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI210_SVE
"ld1h {z19.h}, p1/z, [%[src_a]] \n" //
I4XXTORGB_SVE RGBATOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[src_a] "+r"(src_a), // %[src_a]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[src_a] "+r"(src_a), // %[src_a]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1413,8 +1407,6 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (2 * vl - 1);
int width_last_uv = (width_last_y + 1) / 2;
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE_2X.
uint16_t limit = 0x3ff0;
asm volatile(
@ -1439,9 +1431,11 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_uv] \n"
"whilelt p2.h, wzr, %w[width_last_y] \n"
"whilelt p3.h, %w[vl], %w[width_last_y] \n" //
"whilelt p2.h, wzr, %w[width] \n"
"whilelt p3.h, %w[vl], %w[width] \n"
"add %w[width], %w[width], #1 \n" // (width + 1) / 2
"lsr %w[width], %w[width], #1 \n"
"whilelt p1.h, wzr, %w[width] \n" //
READI210_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X
"99: \n"
@ -1453,8 +1447,6 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y,
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv), // %[width_last_uv]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1467,8 +1459,6 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
uint32_t nv_uv_start = 0x03010301U;
uint32_t nv_uv_step = 0x04040404U;
asm volatile(
@ -1495,8 +1485,10 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.h, wzr, %w[width_last_uv] \n" //
"whilelt p1.h, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.h, wzr, %w[width] \n" //
READP210_SVE NVTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
@ -1509,9 +1501,7 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y,
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
[nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
[nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1522,8 +1512,6 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
uint32_t nv_uv_start = 0x03010301U;
uint32_t nv_uv_step = 0x04040404U;
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
@ -1551,8 +1539,10 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.h, wzr, %w[width_last_uv] \n" //
"whilelt p1.h, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.h, wzr, %w[width] \n" //
READP210_SVE NVTORGB_SVE STOREAR30_SVE
"99: \n"
@ -1565,8 +1555,6 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y,
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
[nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv), // %[width_last_uv]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1579,7 +1567,6 @@ static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1602,20 +1589,19 @@ static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI410_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1629,7 +1615,6 @@ static inline void I410AlphaToARGBRow_SVE_SC(
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1656,23 +1641,22 @@ static inline void I410AlphaToARGBRow_SVE_SC(
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI410_SVE
"ld1h {z19.h}, p1/z, [%[src_a]] \n" //
I4XXTORGB_SVE RGBATOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[src_a] "+r"(src_a), // %[src_a]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[src_a] "+r"(src_a), // %[src_a]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1684,7 +1668,6 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
uint16_t limit = 0x3ff0;
asm volatile(
@ -1707,7 +1690,7 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI410_SVE I4XXTORGB_SVE STOREAR30_SVE
"99: \n"
@ -1719,7 +1702,6 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y,
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1731,7 +1713,6 @@ static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1756,22 +1737,21 @@ static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.s, wzr, %w[width_last_y] \n"
"whilelt p1.h, wzr, %w[width] \n"
"whilelt p2.s, wzr, %w[width] \n"
"cntw %x[vl] \n"
"whilelt p3.s, %w[vl], %w[width_last_y] \n" //
"whilelt p3.s, %w[vl], %w[width] \n" //
READP410_SVE NVTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width), // %[width]
[vl] "+r"(vl) // %[vl]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width), // %[width]
[vl] "+r"(vl) // %[vl]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1782,7 +1762,6 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
uint16_t limit = 0x3ff0;
asm volatile(
@ -1808,10 +1787,10 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.s, wzr, %w[width_last_y] \n"
"whilelt p1.h, wzr, %w[width] \n"
"whilelt p2.s, wzr, %w[width] \n"
"cntw %x[vl] \n"
"whilelt p3.s, %w[vl], %w[width_last_y] \n" //
"whilelt p3.s, %w[vl], %w[width] \n" //
READP410_SVE NVTORGB_SVE STOREAR30_SVE
"99: \n"
@ -1822,7 +1801,6 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y,
[vl] "+r"(vl) // %[vl]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1835,7 +1813,6 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
uint16_t limit = 0x3ff0;
asm volatile(
@ -1858,7 +1835,7 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI212_SVE I4XXTORGB_SVE STOREAR30_SVE
"99: \n"
@ -1870,7 +1847,6 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y,
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1883,7 +1859,6 @@ static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1906,20 +1881,19 @@ static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI212_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -2120,7 +2094,7 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb,
"ptrue p5.h \n"
"1: \n" //
ARGBTOUVMATRIX_SVE
"b.gt 1b \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl], lsl #2 \n"
@ -2197,7 +2171,7 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb,
"ptrue p1.s \n"
"1: \n" //
ARGBTOYMATRIX_SVE("p1", "p1", "p1", "p1", "p0")
"b.gt 1b \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl], lsl #2 \n"

4
source/compare.cc
View File

@ -445,8 +445,10 @@ double CalcFrameSsim(const uint8_t* src_a,
int height) {
int samples = 0;
double ssim_total = 0;
// Return 0 rather than 0.0 / 0 (NaN) when the plane is too small for any
// sample windows, so I420Ssim on small images stays finite.
if (width <= 8 || height <= 8) {
return ssim_total / samples;
return 0.0;
}
int simd_size = 4;

7
source/convert.cc
View File

@ -1270,6 +1270,7 @@ int I422ToNV21(const uint8_t* src_y,
dst_y, dst_stride_y, plane_u, halfwidth, plane_v, halfwidth,
width, height);
if (r != 0) {
free_aligned_buffer_64(plane_u);
return r;
}
MergeUVPlane(plane_v, halfwidth, plane_u, halfwidth, dst_vu, dst_stride_vu,
@ -2714,7 +2715,8 @@ int AYUVToNV12(const uint8_t* src_ayuv,
}
#endif
#if defined(HAS_AYUVTOYROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
AYUVToUVRow = AYUVToUVRow_AVX512VBMI;
AYUVToYRow = AYUVToYRow_AVX512VBMI;
}
@ -2800,7 +2802,8 @@ int AYUVToNV21(const uint8_t* src_ayuv,
}
#endif
#if defined(HAS_AYUVTOYROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
AYUVToVURow = AYUVToVURow_AVX512VBMI;
AYUVToYRow = AYUVToYRow_AVX512VBMI;
}

13
source/convert_argb.cc
View File

@ -4223,7 +4223,8 @@ int RAWToRGBA(const uint8_t* src_raw,
}
#endif
#if defined(HAS_RAWTORGBAROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
RAWToRGBARow = RAWToRGBARow_AVX512VBMI;
}
#endif
@ -5100,7 +5101,8 @@ int NV12ToRGB24Matrix(const uint8_t* src_y,
}
#endif
#if defined(HAS_NV12TORGB24ROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
NV12ToRGB24Row = NV12ToRGB24Row_AVX512VBMI;
}
#endif
@ -5182,7 +5184,8 @@ int NV21ToRGB24Matrix(const uint8_t* src_y,
}
#endif
#if defined(HAS_NV21TORGB24ROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
NV21ToRGB24Row = NV21ToRGB24Row_AVX512VBMI;
}
#endif
@ -6121,7 +6124,7 @@ int I420ToRGB24Matrix(const uint8_t* src_y,
#if defined(HAS_I422TORGB24ROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
I422ToRGB24Row = I422ToRGB24Row_Any_WASMSIMD;
if (IS_ALIGNED(width, 8)) {
if (IS_ALIGNED(width, 16)) {
I422ToRGB24Row = I422ToRGB24Row_WASMSIMD;
}
}
@ -6352,7 +6355,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y,
#if defined(HAS_I422TORGB24ROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
I422ToRGB24Row = I422ToRGB24Row_Any_WASMSIMD;
if (IS_ALIGNED(width, 8)) {
if (IS_ALIGNED(width, 16)) {
I422ToRGB24Row = I422ToRGB24Row_WASMSIMD;
}
}

43
source/convert_from.cc
View File

@ -1038,6 +1038,7 @@ int ConvertFromI420(const uint8_t* y,
height == INT_MIN) {
return -1;
}
const int abs_height = Abs(height);
switch (format) {
// Single plane formats
case FOURCC_YUY2:
@ -1107,7 +1108,7 @@ int ConvertFromI420(const uint8_t* y,
break;
case FOURCC_NV12: {
int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
r = I420ToNV12(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride ? dst_sample_stride : width, dst_uv,
dst_sample_stride ? dst_sample_stride : width, width,
@ -1116,7 +1117,7 @@ int ConvertFromI420(const uint8_t* y,
}
case FOURCC_NV21: {
int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
uint8_t* dst_vu = dst_sample + (ptrdiff_t)dst_y_stride * height;
uint8_t* dst_vu = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
r = I420ToNV21(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride ? dst_sample_stride : width, dst_vu,
dst_sample_stride ? dst_sample_stride : width, width,
@ -1126,13 +1127,13 @@ int ConvertFromI420(const uint8_t* y,
case FOURCC_NV16: {
int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
int dst_uv_stride = dst_sample_stride ? dst_sample_stride : width;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
int halfwidth = (width + 1) / 2;
int halfheight = (height + 1) / 2;
int halfheight = SUBSAMPLE(height, 1, 1);
CopyPlane(y, y_stride, dst_sample, dst_y_stride, width, height);
MergeUVPlane(u, u_stride, v, v_stride, dst_uv, dst_uv_stride * 2,
halfwidth, halfheight);
if (height > 1) {
if (abs_height > 1) {
MergeUVPlane(u, u_stride, v, v_stride, dst_uv + dst_uv_stride,
dst_uv_stride * 2, halfwidth, height / 2);
}
@ -1142,9 +1143,9 @@ int ConvertFromI420(const uint8_t* y,
case FOURCC_NV24: {
int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
int dst_uv_stride = dst_sample_stride ? dst_sample_stride * 2 : width * 2;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height;
align_buffer_64(temp_u, (size_t)width * height);
align_buffer_64(temp_v, (size_t)width * height);
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
align_buffer_64(temp_u, (size_t)width * abs_height);
align_buffer_64(temp_v, (size_t)width * abs_height);
if (!temp_u || !temp_v) {
free_aligned_buffer_64(temp_u);
free_aligned_buffer_64(temp_v);
@ -1154,7 +1155,7 @@ int ConvertFromI420(const uint8_t* y,
dst_y_stride, temp_u, width, temp_v, width, width, height);
if (r == 0) {
MergeUVPlane(temp_u, width, temp_v, width, dst_uv, dst_uv_stride, width,
height);
abs_height);
}
free_aligned_buffer_64(temp_u);
free_aligned_buffer_64(temp_v);
@ -1162,7 +1163,6 @@ int ConvertFromI420(const uint8_t* y,
}
case FOURCC_P010:
case FOURCC_P210: {
const int abs_height = Abs(height);
const int uv_min = ((width + 1) / 2) * 2;
int dst_stride_y;
int dst_stride_uv;
@ -1193,7 +1193,6 @@ int ConvertFromI420(const uint8_t* y,
break;
}
case FOURCC_P410: {
const int abs_height = Abs(height);
const int uv_min = width * 2;
int dst_stride_y;
int dst_stride_uv;
@ -1223,14 +1222,14 @@ int ConvertFromI420(const uint8_t* y,
case FOURCC_YV12: {
dst_sample_stride = dst_sample_stride ? dst_sample_stride : width;
int halfstride = (dst_sample_stride + 1) / 2;
int halfheight = (height + 1) / 2;
int halfheight = (abs_height + 1) / 2;
uint8_t* dst_u;
uint8_t* dst_v;
if (format == FOURCC_YV12) {
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_u = dst_v + (ptrdiff_t)halfstride * halfheight;
} else {
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_v = dst_u + (ptrdiff_t)halfstride * halfheight;
}
r = I420Copy(y, y_stride, u, u_stride, v, v_stride, dst_sample,
@ -1245,11 +1244,11 @@ int ConvertFromI420(const uint8_t* y,
uint8_t* dst_u;
uint8_t* dst_v;
if (format == FOURCC_YV16) {
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_u = dst_v + (ptrdiff_t)halfstride * height;
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_u = dst_v + (ptrdiff_t)halfstride * abs_height;
} else {
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_v = dst_u + (ptrdiff_t)halfstride * height;
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_v = dst_u + (ptrdiff_t)halfstride * abs_height;
}
r = I420ToI422(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride, dst_u, halfstride, dst_v, halfstride,
@ -1262,11 +1261,11 @@ int ConvertFromI420(const uint8_t* y,
uint8_t* dst_u;
uint8_t* dst_v;
if (format == FOURCC_YV24) {
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_u = dst_v + (ptrdiff_t)dst_sample_stride * height;
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_u = dst_v + (ptrdiff_t)dst_sample_stride * abs_height;
} else {
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_v = dst_u + (ptrdiff_t)dst_sample_stride * height;
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_v = dst_u + (ptrdiff_t)dst_sample_stride * abs_height;
}
r = I420ToI444(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride, dst_u, dst_sample_stride, dst_v,

56
source/convert_from_argb.cc
View File

@ -1630,6 +1630,14 @@ int ARGBToI400Matrix(const uint8_t* src_argb,
src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
@ -1721,6 +1729,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb,
dst_yuy2 = dst_yuy2 + (ptrdiff_t)(height - 1) * dst_stride_yuy2;
dst_stride_yuy2 = -dst_stride_yuy2;
}
#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
@ -1776,6 +1792,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb,
ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
@ -1832,6 +1856,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb,
ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
}
#endif
#if defined(HAS_I422TOYUY2ROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
I422ToYUY2Row = I422ToYUY2Row_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
I422ToYUY2Row = I422ToYUY2Row_SSE2;
}
}
#endif
#if defined(HAS_I422TOYUY2ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I422ToYUY2Row = I422ToYUY2Row_Any_AVX2;
@ -1897,6 +1929,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb,
dst_uyvy = dst_uyvy + (ptrdiff_t)(height - 1) * dst_stride_uyvy;
dst_stride_uyvy = -dst_stride_uyvy;
}
#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
@ -1952,6 +1992,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb,
ARGBToYMatrixRow = ARGBToYMatrixRow_HVX;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
@ -2008,6 +2056,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb,
ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX;
}
#endif
#if defined(HAS_I422TOUYVYROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
I422ToUYVYRow = I422ToUYVYRow_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
I422ToUYVYRow = I422ToUYVYRow_SSE2;
}
}
#endif
#if defined(HAS_I422TOUYVYROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I422ToUYVYRow = I422ToUYVYRow_Any_AVX2;

37
source/convert_to_argb.cc
View File

@ -257,6 +257,27 @@ static int NV24ToARGB(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_SPLITUVROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
SplitUVRow = SplitUVRow_SVE2;
}
#endif
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
SplitUVRow = SplitUVRow_LSX;
}
}
#endif
#if defined(HAS_SPLITUVROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
SplitUVRow = SplitUVRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
SplitUVRow = SplitUVRow_LASX;
}
}
#endif
#if defined(HAS_SPLITUVROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
SplitUVRow = SplitUVRow_RVV;
@ -448,7 +469,7 @@ int ConvertToARGB(const uint8_t* sample,
src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
src_uv = sample +
aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) +
crop_x;
(crop_x & ~1);
r = NV12ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb,
dst_stride_argb, crop_width, inv_crop_height);
break;
@ -456,7 +477,7 @@ int ConvertToARGB(const uint8_t* sample,
src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
src_uv = sample +
aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) +
crop_x;
(crop_x & ~1);
// Call NV12 but with u and v parameters swapped.
r = NV21ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb,
dst_stride_argb, crop_width, inv_crop_height);
@ -523,12 +544,12 @@ int ConvertToARGB(const uint8_t* sample,
int halfheight = (abs_src_height + 1) / 2;
if (format == FOURCC_YV12) {
src_v = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
src_u = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2;
} else {
src_u = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2;
}
@ -542,7 +563,7 @@ int ConvertToARGB(const uint8_t* sample,
int halfheight = (abs_src_height + 1) / 2;
const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) +
crop_x / 2;
@ -556,7 +577,7 @@ int ConvertToARGB(const uint8_t* sample,
int halfheight = (abs_src_height + 1) / 2;
const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) +
crop_x / 2;
@ -570,7 +591,7 @@ int ConvertToARGB(const uint8_t* sample,
int halfheight = (abs_src_height + 1) / 2;
const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) +
crop_x / 2;
@ -635,7 +656,7 @@ int ConvertToARGB(const uint8_t* sample,
const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)abs_src_height + crop_y) +
crop_x / 2;
r = H422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
r = U422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
dst_argb, dst_stride_argb, crop_width, inv_crop_height);
break;
}

100
source/convert_to_i420.cc
View File

@ -51,8 +51,13 @@ static int NV16ToI420(const uint8_t* src_y,
int y;
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
// UV rows are halfwidth * 2 (even) bytes.
int awidth = halfwidth * 2;
void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
int width) = SplitUVRow_C;
void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 ||
height == 0 || height == INT_MIN) {
return -1;
@ -100,18 +105,105 @@ static int NV16ToI420(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_SPLITUVROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
SplitUVRow = SplitUVRow_SVE2;
}
#endif
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;
if (IS_ALIGNED(halfwidth, 16)) {
SplitUVRow = SplitUVRow_LSX;
}
}
#endif
#if defined(HAS_SPLITUVROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
SplitUVRow = SplitUVRow_Any_LASX;
if (IS_ALIGNED(halfwidth, 32)) {
SplitUVRow = SplitUVRow_LASX;
}
}
#endif
#if defined(HAS_SPLITUVROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
SplitUVRow = SplitUVRow_RVV;
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
if (IS_ALIGNED(awidth, 32)) {
InterpolateRow = InterpolateRow_AVX2;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_NEON;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
InterpolateRow = InterpolateRow_SVE2;
}
#endif
#if defined(HAS_INTERPOLATEROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
InterpolateRow = InterpolateRow_SME;
}
#endif
#if defined(HAS_INTERPOLATEROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
InterpolateRow = InterpolateRow_Any_LSX;
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_LSX;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(awidth, 32)) {
InterpolateRow = InterpolateRow_LASX;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
InterpolateRow = InterpolateRow_RVV;
}
#endif
#if defined(HAS_INTERPOLATEROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
InterpolateRow = InterpolateRow_Any_WASMSIMD;
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
align_buffer_64(row_uv, halfwidth * 2);
align_buffer_64(row_uv, awidth);
if (!row_uv) {
return 1;
}
for (y = 0; y < halfheight; ++y) {
if (y * 2 + 1 < height) {
HalfRow_C(src_uv, src_stride_uv, row_uv, halfwidth * 2);
// Average 2 rows of UV with rounding (source_y_fraction 128).
InterpolateRow(row_uv, src_uv, src_stride_uv, awidth, 128);
SplitUVRow(row_uv, dst_u, dst_v, halfwidth);
} else {
memcpy(row_uv, src_uv, (size_t)halfwidth * 2);
SplitUVRow(src_uv, dst_u, dst_v, halfwidth);
}
SplitUVRow(row_uv, dst_u, dst_v, halfwidth);
src_uv += (ptrdiff_t)src_stride_uv * 2;
dst_u += dst_stride_u;
dst_v += dst_stride_v;

12
source/cpu_id.cc
View File

@ -408,11 +408,21 @@ static SAFEBUFFERS int GetCpuFlags(void) {
int cpu_info24[4] = {0, 0, 0, 0};
int cpu_info21[4] = {0, 0, 0, 0};
int cpu_amdinfo21[4] = {0, 0, 0, 0};
int cpu_extinfo0[4] = {0, 0, 0, 0};
CpuId(0, 0, cpu_info0);
CpuId(1, 0, cpu_info1);
if (cpu_info0[0] >= 7) {
CpuId(7, 0, cpu_info7);
CpuId(7, 1, cpu_einfo7);
// Leaf 7 EAX reports the maximum supported sub-leaf.
if (cpu_info7[0] >= 1) {
CpuId(7, 1, cpu_einfo7);
}
}
// Extended leaf 0x80000021 is only valid if the CPU reports it. Querying an
// unsupported extended leaf returns data from the highest basic leaf on
// Intel CPUs, which could spuriously set bits below.
CpuId(0x80000000, 0, cpu_extinfo0);
if ((unsigned)cpu_extinfo0[0] >= 0x80000021u) {
CpuId(0x80000021, 0, cpu_amdinfo21);
}
if (cpu_info0[0] >= 0x21) {

35
source/mjpeg_decoder.cc
View File

@ -127,7 +127,11 @@ bool MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) {
AllocOutputBuffers(GetNumComponents());
for (int i = 0; i < num_outbufs_; ++i) {
int scanlines_size = GetComponentScanlinesPerImcuRow(i);
if (scanlines_sizes_[i] != scanlines_size) {
// databuf_[i] is sized by scanlines_size * stride, so a change in either
// must trigger reallocation below. Remember the change before
// scanlines_sizes_[i] is overwritten.
bool scanlines_changed = scanlines_sizes_[i] != scanlines_size;
if (scanlines_changed) {
free(scanlines_[i]);
scanlines_[i] =
static_cast<uint8_t**>(malloc(scanlines_size * sizeof(uint8_t*)));
@ -147,7 +151,7 @@ bool MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) {
// DCTSIZE) is at most slightly larger than UINT16_MAX.
// - Sampling factor is stored in 4 bits, so scanlines_size is < 16.
int databuf_size = scanlines_size * databuf_stride;
if (databuf_strides_[i] != databuf_stride) {
if (scanlines_changed || databuf_strides_[i] != databuf_stride) {
free(databuf_[i]);
databuf_[i] = static_cast<uint8_t*>(malloc(databuf_size));
databuf_strides_[i] = databuf_stride;
@ -268,9 +272,11 @@ bool MJpegDecoder::DecodeToBuffers(uint8_t** planes,
SetScanlinePointers(databuf_);
int lines_left = dst_height;
// Compute amount of lines to skip to implement vertical crop.
// TODO(fbarchard): Ensure skip is a multiple of maximum component
// subsample. ie 2
// Round skip down to a multiple of the maximum vertical sampling factor so
// that every component skips a whole number of rows. The rows remaining
// below the crop are still >= dst_height.
int skip = (GetHeight() - dst_height) / 2;
skip -= skip % decompress_struct_->max_v_samp_factor;
if (skip > 0) {
// There is no API to skip lines in the output data, so we read them
// into the temp buffer.
@ -288,19 +294,23 @@ bool MJpegDecoder::DecodeToBuffers(uint8_t** planes,
FinishDecode();
return false;
}
// Image lines wanted from this iMCU row, clamped to the destination.
int image_lines_to_copy = GetImageScanlinesPerImcuRow() - skip;
if (image_lines_to_copy > lines_left) {
image_lines_to_copy = lines_left;
}
for (int i = 0; i < num_outbufs_; ++i) {
// TODO(fbarchard): Compute skip to avoid this
assert(skip % GetVertSubSampFactor(i) == 0);
int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i));
int scanlines_to_copy =
GetComponentScanlinesPerImcuRow(i) - rows_to_skip;
DivideAndRoundUp(image_lines_to_copy, GetVertSubSampFactor(i));
int data_to_skip = rows_to_skip * GetComponentStride(i);
CopyPlane(databuf_[i] + data_to_skip, GetComponentStride(i), planes[i],
GetComponentWidth(i), GetComponentWidth(i),
scanlines_to_copy);
planes[i] += scanlines_to_copy * GetComponentWidth(i);
}
lines_left -= (GetImageScanlinesPerImcuRow() - skip);
lines_left -= image_lines_to_copy;
}
}
@ -357,8 +367,12 @@ bool MJpegDecoder::DecodeToCallback(CallbackFunction fn,
}
SetScanlinePointers(databuf_);
int lines_left = dst_height;
// TODO(fbarchard): Compute amount of lines to skip to implement vertical crop
// Compute amount of lines to skip to implement vertical crop.
// Round skip down to a multiple of the maximum vertical sampling factor so
// that every component skips a whole number of rows. The rows remaining
// below the crop are still >= dst_height.
int skip = (GetHeight() - dst_height) / 2;
skip -= skip % decompress_struct_->max_v_samp_factor;
if (skip > 0) {
while (skip >= GetImageScanlinesPerImcuRow()) {
if (!DecodeImcuRow()) {
@ -374,7 +388,6 @@ bool MJpegDecoder::DecodeToCallback(CallbackFunction fn,
return false;
}
for (int i = 0; i < num_outbufs_; ++i) {
// TODO(fbarchard): Compute skip to avoid this
assert(skip % GetVertSubSampFactor(i) == 0);
int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i));
int data_to_skip = rows_to_skip * GetComponentStride(i);
@ -382,7 +395,11 @@ bool MJpegDecoder::DecodeToCallback(CallbackFunction fn,
// callback.
databuf_[i] += data_to_skip;
}
// Image lines wanted from this iMCU row, clamped to the destination.
int scanlines_to_copy = GetImageScanlinesPerImcuRow() - skip;
if (scanlines_to_copy > lines_left) {
scanlines_to_copy = lines_left;
}
(*fn)(opaque, databuf_, databuf_strides_, scanlines_to_copy);
// Now change them back.
for (int i = 0; i < num_outbufs_; ++i) {

71
source/planar_functions.cc
View File

@ -1414,7 +1414,10 @@ void DetileSplitUVPlane(const uint8_t* src_uv,
assert(tile_height > 0);
assert(src_stride_uv > 0);
if (width <= 0 || height == 0 || height == INT_MIN) {
// tile_height must be a power of 2 (16 or 32); the row loop below uses it as
// a bit mask.
if (width <= 0 || height == 0 || height == INT_MIN || tile_height <= 0 ||
!IS_POWEROFTWO(tile_height)) {
return;
}
// Negative height means invert the image.
@ -1478,7 +1481,10 @@ void DetileToYUY2(const uint8_t* src_y,
assert(src_stride_uv > 0);
assert(tile_height > 0);
if (width <= 0 || height == 0 || height == INT_MIN || tile_height <= 0) {
// tile_height must be a power of 2 and at least 2 (16 or 32); the row loop
// uses it as a bit mask and the UV tile is tile_height / 2 rows.
if (width <= 0 || height == 0 || height == INT_MIN || tile_height < 2 ||
!IS_POWEROFTWO(tile_height)) {
return;
}
// Negative height means invert the image.
@ -1650,7 +1656,8 @@ void MergeRGBPlane(const uint8_t* src_r,
}
#endif
#if defined(HAS_MERGERGBROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
MergeRGBRow = MergeRGBRow_AVX512VBMI;
}
#endif
@ -1864,11 +1871,13 @@ void SplitARGBPlane(const uint8_t* src_argb,
dst_r = dst_r + (ptrdiff_t)(height - 1) * dst_stride_r;
dst_g = dst_g + (ptrdiff_t)(height - 1) * dst_stride_g;
dst_b = dst_b + (ptrdiff_t)(height - 1) * dst_stride_b;
dst_a = dst_a + (ptrdiff_t)(height - 1) * dst_stride_a;
dst_stride_r = -dst_stride_r;
dst_stride_g = -dst_stride_g;
dst_stride_b = -dst_stride_b;
dst_stride_a = -dst_stride_a;
if (dst_a) {
dst_a = dst_a + (ptrdiff_t)(height - 1) * dst_stride_a;
dst_stride_a = -dst_stride_a;
}
}
if (dst_a == NULL) {
@ -2105,7 +2114,8 @@ void MergeXR30Plane(const uint16_t* src_r,
}
#endif
#if defined(HAS_MERGEXR30ROW_10_AVX512BW)
if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
MergeXR30Row = MergeXR30Row_10_AVX512BW;
}
#endif
@ -3334,6 +3344,14 @@ int RGB24Mirror(const uint8_t* src_rgb24,
}
}
#endif
#if defined(HAS_RGB24MIRRORROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
RGB24MirrorRow = RGB24MirrorRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
RGB24MirrorRow = RGB24MirrorRow_SSSE3;
}
}
#endif
#if defined(HAS_RGB24MIRRORROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
RGB24MirrorRow = RGB24MirrorRow_Any_AVX2;
@ -4293,6 +4311,11 @@ int ARGBRect(uint8_t* dst_argb,
ARGBSetRow = ARGBSetRow_HVX;
}
#endif
#if defined(HAS_ARGBSETROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBSetRow = ARGBSetRow_RVV;
}
#endif
// Set plane
for (y = 0; y < height; ++y) {
@ -5437,6 +5460,11 @@ int ARGBShuffle(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBSHUFFLEROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBShuffleRow = ARGBShuffleRow_RVV;
}
#endif
for (y = 0; y < height; ++y) {
ARGBShuffleRow(src_argb, dst_argb, shuffler, width);
@ -5513,6 +5541,11 @@ int AR64Shuffle(const uint16_t* src_ar64,
}
}
#endif
#if defined(HAS_ARGBSHUFFLEROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
AR64ShuffleRow = ARGBShuffleRow_RVV;
}
#endif
for (y = 0; y < height; ++y) {
AR64ShuffleRow((uint8_t*)(src_ar64), (uint8_t*)(dst_ar64), shuffler,
@ -6479,6 +6512,9 @@ int UYVYToNV12(const uint8_t* src_uyvy,
int height) {
int y;
int halfwidth = (width + 1) >> 1;
// UYVY rows are processed as awidth (even) samples. Select kernels on the
// width actually passed to them.
int awidth = halfwidth * 2;
void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
int width) = SplitUVRow_C;
void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
@ -6499,7 +6535,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
SplitUVRow = SplitUVRow_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
SplitUVRow = SplitUVRow_SSE2;
}
}
@ -6507,7 +6543,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
SplitUVRow = SplitUVRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
if (IS_ALIGNED(awidth, 32)) {
SplitUVRow = SplitUVRow_AVX2;
}
}
@ -6515,7 +6551,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
SplitUVRow = SplitUVRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
if (IS_ALIGNED(awidth, 64)) {
SplitUVRow = SplitUVRow_AVX512BW;
}
}
@ -6523,7 +6559,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
SplitUVRow = SplitUVRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
SplitUVRow = SplitUVRow_NEON;
}
}
@ -6536,7 +6572,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
SplitUVRow = SplitUVRow_LSX;
}
}
@ -6544,7 +6580,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
SplitUVRow = SplitUVRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
if (IS_ALIGNED(awidth, 32)) {
SplitUVRow = SplitUVRow_LASX;
}
}
@ -6563,7 +6599,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
if (IS_ALIGNED(awidth, 32)) {
InterpolateRow = InterpolateRow_AVX2;
}
}
@ -6576,7 +6612,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_NEON;
}
}
@ -6594,7 +6630,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
InterpolateRow = InterpolateRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_LSX;
}
}
@ -6602,7 +6638,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
if (IS_ALIGNED(awidth, 32)) {
InterpolateRow = InterpolateRow_LASX;
}
}
@ -6615,7 +6651,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
InterpolateRow = InterpolateRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_WASMSIMD;
}
}
@ -6627,7 +6663,6 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#endif
{
int awidth = halfwidth * 2;
// row of y and 2 rows of uv
align_buffer_64(rows, awidth * 3);
if (!rows)

7
source/rotate.cc
View File

@ -959,6 +959,7 @@ int I422Rotate(const uint8_t* src_y,
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
@ -1266,9 +1267,10 @@ int I010Rotate(const uint16_t* src_y,
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
@ -1337,6 +1339,7 @@ int I210Rotate(const uint16_t* src_y,
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;

3
source/rotate_argb.cc
View File

@ -78,6 +78,9 @@ static int ARGBTranspose(const uint8_t* src_argb,
#if defined(HAS_TRANSPOSEWX4_32_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
TransposeWx4_32 = TransposeWx4_32_SVE2;
#if defined(HAS_TRANSPOSEWXH_32_NEON)
TransposeWxH_32 = NULL;
#endif
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_SME)

16
source/rotate_sve.cc
View File

@ -53,10 +53,10 @@ void TransposeWx4_32_SVE2(const uint8_t* src,
"trn2 z5.s, z0.s, z1.s \n"
"trn1 z6.s, z2.s, z3.s \n"
"trn2 z7.s, z2.s, z3.s \n"
"trn1 z0.d, z4.d, z6.d \n"
"trn2 z1.d, z4.d, z6.d \n"
"trn1 z2.d, z5.d, z7.d \n"
"trn2 z3.d, z5.d, z7.d \n"
"trn1 z0.d, z4.d, z6.d \n" // column 0
"trn1 z1.d, z5.d, z7.d \n" // column 1
"trn2 z2.d, z4.d, z6.d \n" // column 2
"trn2 z3.d, z5.d, z7.d \n" // column 3
"subs %w[width], %w[width], #4 \n"
"st1w {z0.s}, p0, [%[dst0]] \n"
"st1w {z1.s}, p0, [%[dst1]] \n"
@ -86,10 +86,10 @@ void TransposeWx4_32_SVE2(const uint8_t* src,
"trn2 z5.s, z0.s, z1.s \n"
"trn1 z6.s, z2.s, z3.s \n"
"trn2 z7.s, z2.s, z3.s \n"
"trn1 z0.d, z4.d, z6.d \n"
"trn2 z1.d, z4.d, z6.d \n"
"trn1 z2.d, z5.d, z7.d \n"
"trn2 z3.d, z5.d, z7.d \n"
"trn1 z0.d, z4.d, z6.d \n" // column 0
"trn1 z1.d, z5.d, z7.d \n" // column 1
"trn2 z2.d, z4.d, z6.d \n" // column 2
"trn2 z3.d, z5.d, z7.d \n" // column 3
"whilelt p0.s, wzr, %w[width] \n"
"st1w {z0.s}, p0, [%[dst0]] \n"
"st1w {z1.s}, p0, [%[dst1]] \n"

11
source/row_any.cc
View File

@ -1575,7 +1575,7 @@ ANY11T(ARGBToAB64Row_Any_SSSE3, ARGBToAB64Row_SSSE3, 4, 8, uint8_t, uint16_t, 3)
ANY11T(AR64ToARGBRow_Any_SSSE3, AR64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3)
#endif
#ifdef HAS_ARGBTOAR64ROW_SSSE3
#ifdef HAS_AB64TOARGBROW_SSSE3
ANY11T(AB64ToARGBRow_Any_SSSE3, AB64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3)
#endif
@ -1591,7 +1591,7 @@ ANY11T(ARGBToAB64Row_Any_AVX2, ARGBToAB64Row_AVX2, 4, 8, uint8_t, uint16_t, 7)
ANY11T(AR64ToARGBRow_Any_AVX2, AR64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7)
#endif
#ifdef HAS_ARGBTOAR64ROW_AVX2
#ifdef HAS_AB64TOARGBROW_AVX2
ANY11T(AB64ToARGBRow_Any_AVX2, AB64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7)
#endif
@ -1607,7 +1607,7 @@ ANY11T(ARGBToAB64Row_Any_NEON, ARGBToAB64Row_NEON, 4, 8, uint8_t, uint16_t, 7)
ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
#endif
#ifdef HAS_ARGBTOAR64ROW_NEON
#ifdef HAS_AB64TOARGBROW_NEON
ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
#endif
@ -1836,7 +1836,7 @@ ANY11P16(HalfFloat1Row_Any_F16C,
ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 15)
#endif
#ifdef HAS_BYTETOFLOATROW_NEON
ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 3, 7)
ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 4, 7)
#endif
#ifdef HAS_HALFFLOATROW_LSX
ANY11P16(HalfFloatRow_Any_LSX, HalfFloatRow_LSX, uint16_t, uint16_t, 2, 2, 7)
@ -2140,6 +2140,9 @@ ANY11M(ARGBMirrorRow_Any_LASX, ARGBMirrorRow_LASX, 4, 7)
#ifdef HAS_RGB24MIRRORROW_AVX2
ANY11M(RGB24MirrorRow_Any_AVX2, RGB24MirrorRow_AVX2, 3, 31)
#endif
#ifdef HAS_RGB24MIRRORROW_SSSE3
ANY11M(RGB24MirrorRow_Any_SSSE3, RGB24MirrorRow_SSSE3, 3, 15)
#endif
#ifdef HAS_RGB24MIRRORROW_NEON
ANY11M(RGB24MirrorRow_Any_NEON, RGB24MirrorRow_NEON, 3, 15)
#endif

35
source/row_common.cc
View File

@ -583,6 +583,8 @@ void AR64ToAB64Row_C(const uint16_t* src_ar64, uint16_t* dst_ab64, int width) {
}
// TODO(fbarchard): Make shuffle compatible with SIMD versions
// width is in units of 32 bit ARGB pixels (2 per AR64 pixel) so that
// AR64Shuffle can share the ARGBShuffleRow SIMD kernels.
void AR64ShuffleRow_C(const uint8_t* src_ar64,
uint8_t* dst_ar64,
const uint8_t* shuffler,
@ -2938,8 +2940,10 @@ void DetileToYUY2_C(const uint8_t* src_y,
ptrdiff_t src_uv_tile_stride,
uint8_t* dst_yuy2,
int width) {
for (int x = 0; x < width - 15; x += 16) {
for (int i = 0; i < 8; i++) {
int x;
for (x = 0; x < width - 15; x += 16) {
int i;
for (i = 0; i < 8; i++) {
dst_yuy2[0] = src_y[0];
dst_yuy2[1] = src_uv[0];
dst_yuy2[2] = src_y[1];
@ -2951,6 +2955,16 @@ void DetileToYUY2_C(const uint8_t* src_y,
src_y += src_y_tile_stride - 16;
src_uv += src_uv_tile_stride - 16;
}
// Remaining pixels of a partial 16 pixel tile.
for (x = 0; x < (width & 15); x += 2) {
dst_yuy2[0] = src_y[0];
dst_yuy2[1] = src_uv[0];
dst_yuy2[2] = src_y[1];
dst_yuy2[3] = src_uv[1];
dst_yuy2 += 4;
src_y += 2;
src_uv += 2;
}
}
// Unpack MT2T into tiled P010 64 pixels at a time. MT2T's bitstream is encoded
@ -3214,8 +3228,10 @@ void DivideRow_16_C(const uint16_t* src_y,
int scale,
int width) {
int x;
// scale can be 65536 so multiply unsigned to avoid signed overflow.
for (x = 0; x < width; ++x) {
dst_y[x] = (src_y[x] * scale) >> 16;
dst_y[x] =
STATIC_CAST(uint16_t, ((uint32_t)src_y[x] * (uint32_t)scale) >> 16);
}
}
@ -3225,7 +3241,8 @@ void DivideRow_16_C(const uint16_t* src_y,
// 4096 = 12 bits
// 256 = 16 bits
// TODO(fbarchard): change scale to bits
#define C16TO8(v, scale) clamp255(((v) * (scale)) >> 16)
#define C16TO8(v, scale) \
clamp255((int32_t)(((uint32_t)(v) * (uint32_t)(scale)) >> 16))
void Convert16To8Row_C(const uint16_t* src_y,
uint8_t* dst_y,
@ -3943,9 +3960,15 @@ void HalfFloatRow_C(const uint16_t* src,
float scale,
int width) {
int i;
float mult = 1.9259299444e-34f * scale;
// Multiply by scale first, then by 2^-112 to move the exponent into half
// float position. Doing the scale and bias as two multiplies keeps every
// intermediate normal; a combined (scale * 2^-112) constant is a denormal
// for scale < 2^-14 (e.g. 1/65535), which costs a microcode assist per
// multiply on Intel CPUs (50x slower). 2^-112 is a power of 2 so the
// result is identical to the single multiply when that was representable.
const float kScaleBias = 1.9259299444e-34f; // 2^-112
for (i = 0; i < width; ++i) {
float value = src[i] * mult;
float value = (src[i] * scale) * kScaleBias;
dst[i] = (uint16_t)((*(const uint32_alias_t*)&value) >> 13);
}
}

31
source/row_gcc.cc
View File

@ -7056,6 +7056,7 @@ void SplitRGBRow_AVX2(const uint8_t* src_rgb,
"lea 0x20(%3),%3 \n"
"sub $0x20,%4 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_rgb), // %0
"+r"(dst_r), // %1
"+r"(dst_g), // %2
@ -8223,7 +8224,8 @@ void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
"movq (%0),%%xmm2 \n"
"lea 0x8(%0),%0 \n"
"punpcklbw %%xmm2,%%xmm2 \n"
"punpckhwd %%xmm2,%%xmm3 \n"
"movdqa %%xmm2,%%xmm3 \n"
"punpckhwd %%xmm3,%%xmm3 \n"
"punpcklwd %%xmm2,%%xmm2 \n"
"movdqu (%1),%%xmm4 \n"
"movdqu 0x10(%1),%%xmm5 \n"
@ -11102,7 +11104,7 @@ void ARGBPolynomialRow_SSE2(const uint8_t* src_argb,
"addps %%xmm5,%%xmm4 \n"
"cvttps2dq %%xmm0,%%xmm0 \n"
"cvttps2dq %%xmm4,%%xmm4 \n"
"packuswb %%xmm4,%%xmm0 \n"
"packssdw %%xmm4,%%xmm0 \n"
"packuswb %%xmm0,%%xmm0 \n"
"movq %%xmm0,(%1) \n"
"lea 0x8(%1),%1 \n"
@ -11159,14 +11161,17 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
#endif // HAS_ARGBPOLYNOMIALROW_AVX2
#ifdef HAS_HALFFLOATROW_AVX2
static float kScaleBias = 1.9259299444e-34f;
// 2^-112 moves the float exponent into half float position. Applied as a
// separate multiply after scale so no denormal constant is formed when
// scale < 2^-14 (see HalfFloatRow_C).
static const float kScaleBias = 1.9259299444e-34f;
void HalfFloatRow_AVX2(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
scale *= kScaleBias;
asm volatile(
"vbroadcastss %3, %%ymm4 \n"
"vbroadcastss %4, %%ymm6 \n"
"vpxor %%ymm5,%%ymm5,%%ymm5 \n"
"sub %0,%1 \n"
@ -11179,8 +11184,10 @@ void HalfFloatRow_AVX2(const uint16_t* src,
"vpunpcklwd %%ymm5,%%ymm2,%%ymm2 \n"
"vcvtdq2ps %%ymm3,%%ymm3 \n"
"vcvtdq2ps %%ymm2,%%ymm2 \n"
"vmulps %%ymm3,%%ymm4,%%ymm3 \n"
"vmulps %%ymm3,%%ymm4,%%ymm3 \n" // * scale
"vmulps %%ymm2,%%ymm4,%%ymm2 \n"
"vmulps %%ymm3,%%ymm6,%%ymm3 \n" // * 2^-112
"vmulps %%ymm2,%%ymm6,%%ymm2 \n"
"vpsrld $0xd,%%ymm3,%%ymm3 \n"
"vpsrld $0xd,%%ymm2,%%ymm2 \n"
"vpackssdw %%ymm3, %%ymm2, %%ymm2 \n" // unmutates
@ -11193,11 +11200,13 @@ void HalfFloatRow_AVX2(const uint16_t* src,
"+r"(dst), // %1
"+r"(width) // %2
#if defined(__x86_64__)
: "x"(scale) // %3
: "x"(scale), // %3
"x"(kScaleBias) // %4
#else
: "m"(scale) // %3
: "m"(scale), // %3
"m"(kScaleBias) // %4
#endif
: "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5");
: "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
#endif // HAS_HALFFLOATROW_AVX2
@ -11716,10 +11725,10 @@ void ClampFloatToZero_SSE2(const float* src_x, float* dst_y, int width) {
"1: \n"
"movd (%0),%%xmm0 \n" // load float
"maxss %%xmm1, %%xmm0 \n" // clamp to zero
"add 4, %0 \n"
"lea 0x4(%0),%0 \n"
"movd %%xmm0, (%1) \n" // store float
"add 4, %1 \n"
"sub $0x4,%2 \n" // 1 float per loop
"lea 0x4(%1),%1 \n"
"sub $0x1,%2 \n" // 1 float per loop
"jg 1b \n"
: "+r"(src_x), // %0
"+r"(dst_y), // %1

11
source/row_lasx.cc
View File

@ -2341,9 +2341,10 @@ void HalfFloatRow_LASX(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
float mult = 1.9259299444e-34f * scale;
const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C
asm volatile(
"xvldrepl.w $xr3, %[mult], 0\n\t"
"xvldrepl.w $xr3, %[scale], 0\n\t"
"xvldrepl.w $xr5, %[bias], 0\n\t"
"xvldi $xr4, 0\n\t"
"1:\n\t"
"xvld $xr0, %[src], 0\n\t"
@ -2353,6 +2354,8 @@ void HalfFloatRow_LASX(const uint16_t* src,
"xvffint.s.wu $xr2, $xr2\n\t"
"xvfmul.s $xr1, $xr1, $xr3\n\t"
"xvfmul.s $xr2, $xr2, $xr3\n\t"
"xvfmul.s $xr1, $xr1, $xr5\n\t"
"xvfmul.s $xr2, $xr2, $xr5\n\t"
"xvsrlni.h.w $xr2, $xr1, 13\n\t"
"xvst $xr2, %[dst], 0\n\t"
"addi.d %[src], %[src], 32\n\t"
@ -2360,8 +2363,8 @@ void HalfFloatRow_LASX(const uint16_t* src,
"addi.d %[cnt], %[cnt], -16\n\t"
"bnez %[cnt], 1b\n\t"
: [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width)
: [mult] "r"(&mult)
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4");
: [scale] "r"(&scale), [bias] "r"(&bias)
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5");
}
void ARGBToYMatrixRow_LASX(const uint8_t* src_argb,

11
source/row_lsx.cc
View File

@ -2022,9 +2022,10 @@ void HalfFloatRow_LSX(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
float mult = 1.9259299444e-34f * scale;
const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C
asm volatile(
"vldrepl.w $vr3, %[mult], 0\n\t"
"vldrepl.w $vr3, %[scale], 0\n\t"
"vldrepl.w $vr5, %[bias], 0\n\t"
"vldi $vr4, 0\n\t"
"1:\n\t"
"vld $vr0, %[src], 0\n\t"
@ -2034,6 +2035,8 @@ void HalfFloatRow_LSX(const uint16_t* src,
"vffint.s.wu $vr2, $vr2\n\t"
"vfmul.s $vr1, $vr1, $vr3\n\t"
"vfmul.s $vr2, $vr2, $vr3\n\t"
"vfmul.s $vr1, $vr1, $vr5\n\t"
"vfmul.s $vr2, $vr2, $vr5\n\t"
"vsrlni.h.w $vr2, $vr1, 13\n\t"
"vst $vr2, %[dst], 0\n\t"
"addi.d %[src], %[src], 16\n\t"
@ -2041,8 +2044,8 @@ void HalfFloatRow_LSX(const uint16_t* src,
"addi.d %[cnt], %[cnt], -8\n\t"
"bnez %[cnt], 1b\n\t"
: [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width)
: [mult] "r"(&mult)
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
: [scale] "r"(&scale), [bias] "r"(&bias)
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5");
}
void ARGBToYMatrixRow_LSX(const uint8_t* src_argb,

15
source/row_neon.cc
View File

@ -3369,20 +3369,25 @@ void HalfFloatRow_NEON(const uint16_t* src,
"vcvt.f32.u32 q9, q9 \n"
"vcvt.f32.u32 q10, q10 \n"
"vcvt.f32.u32 q11, q11 \n"
"vmul.f32 q8, q8, %y3 \n" // adjust exponent
"vmul.f32 q8, q8, %y3 \n" // * scale
"vmul.f32 q9, q9, %y3 \n"
"vmul.f32 q10, q10, %y3 \n"
"vmul.f32 q11, q11, %y3 \n"
"vmul.f32 q8, q8, %y4 \n" // adjust exponent
"vmul.f32 q9, q9, %y4 \n"
"vmul.f32 q10, q10, %y4 \n"
"vmul.f32 q11, q11, %y4 \n"
"vqshrn.u32 d0, q8, #13 \n" // isolate halffloat
"vqshrn.u32 d1, q9, #13 \n"
"vqshrn.u32 d2, q10, #13 \n"
"vqshrn.u32 d3, q11, #13 \n"
"vst1.16 {q0, q1}, [%1]! \n" // store 16 fp16
"bgt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale * 1.9259299444e-34f) // %3
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale), // %3
"w"(1.9259299444e-34f) // %4 2^-112, see HalfFloatRow_C
: "cc", "memory", "q0", "q1", "q8", "q9", "q10", "q11");
}

28
source/row_neon64.cc
View File

@ -4767,20 +4767,25 @@ void HalfFloatRow_NEON(const uint16_t* src,
"scvtf v4.4s, v4.4s \n"
"scvtf v3.4s, v3.4s \n"
"scvtf v5.4s, v5.4s \n"
"fmul v2.4s, v2.4s, %3.s[0] \n" // adjust exponent
"fmul v2.4s, v2.4s, %3.s[0] \n" // * scale
"fmul v4.4s, v4.4s, %3.s[0] \n"
"fmul v3.4s, v3.4s, %3.s[0] \n"
"fmul v5.4s, v5.4s, %3.s[0] \n"
"fmul v2.4s, v2.4s, %4.s[0] \n" // adjust exponent
"fmul v4.4s, v4.4s, %4.s[0] \n"
"fmul v3.4s, v3.4s, %4.s[0] \n"
"fmul v5.4s, v5.4s, %4.s[0] \n"
"uqshrn v0.4h, v2.4s, #13 \n" // isolate halffloat
"uqshrn v1.4h, v4.4s, #13 \n"
"uqshrn2 v0.8h, v3.4s, #13 \n"
"uqshrn2 v1.8h, v5.4s, #13 \n"
"stp q0, q1, [%1], #32 \n" // store 16 fp16
"b.gt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale * 1.9259299444e-34f) // %3
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale), // %3
"w"(1.9259299444e-34f) // %4 2^-112, see HalfFloatRow_C
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
}
@ -4836,8 +4841,8 @@ void ConvertFP16ToFP32Column_NEON(const uint16_t* src, // fp16
float* dst,
int width) {
asm volatile(
"cmp %w2, #8 \n" // Is there 8 rows?
"b.lo 2f \n"
"subs %w2, %w2, #8 \n" // Is there 8 rows?
"b.lt 2f \n"
"1: \n"
"ld1 {v0.h}[0], [%0], %3 \n" // load 8 halffloats
"ld1 {v0.h}[1], [%0], %3 \n"
@ -4852,15 +4857,16 @@ void ConvertFP16ToFP32Column_NEON(const uint16_t* src, // fp16
"fcvtl v2.4s, v0.4h \n" // 4 floats
"fcvtl v3.4s, v1.4h \n" // 4 more floats
"stp q2, q3, [%1], #32 \n" // store 8 floats
"b.gt 1b \n"
"cmp %w2, #1 \n" // Is there 1 value?
"b.lo 3f \n"
"b.ge 1b \n"
"2: \n"
"adds %w2, %w2, #8 \n" // Restore remainder.
"b.eq 3f \n"
"4: \n"
"ld1 {v1.h}[0], [%0], %3 \n" // load 1 halffloats
"subs %w2, %w2, #1 \n" // 1 floats per loop
"fcvtl v2.4s, v1.4h \n" // 1 floats
"str s2, [%1], #4 \n" // store 1 floats
"b.gt 2b \n"
"b.gt 4b \n"
"3: \n"
: "+r"(src), // %0
"+r"(dst), // %1

70
source/row_rvv.cc
View File

@ -2623,6 +2623,76 @@ void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
}
#endif // HAS_ARGBMIRRORROW_RVV
#ifdef HAS_ARGBSETROW_RVV
void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width) {
size_t vl;
asm volatile(
"vsetvli %[vl], %[w], e32, m8, ta, ma \n"
"vmv.v.x v8, %[v32] \n"
"1: \n"
"vsetvli %[vl], %[w], e32, m8, ta, ma \n"
"vse32.v v8, (%[dst]) \n"
"sub %[w], %[w], %[vl] \n"
"slli %[vl], %[vl], 2 \n"
"add %[dst], %[dst], %[vl] \n"
"bnez %[w], 1b \n"
: [dst] "+r"(dst_argb), // %[dst]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [v32] "r"(v32) // %[v32]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif // HAS_ARGBSETROW_RVV
#ifdef HAS_ARGBSHUFFLEROW_RVV
// Shuffle the 4 bytes of each ARGB pixel with a 16 byte shuffler table (only
// the first 4 entries are used, the pattern repeats every pixel), e.g.
// {0, 1, 2, 3} is a copy and {2, 1, 0, 3} swaps R and B.
// Byte index vector: index[i] = (i & ~3) + shuffler[i & 3], gathered with a
// 16 bit index so any VLEN works (e8 indices would be limited to 256 bytes).
void ARGBShuffleRow_RVV(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width) {
size_t vl;
size_t w = (size_t)width * 4; // bytes
// The 4 shuffler bytes packed little endian so a 32 bit broadcast lays them
// out as s0 s1 s2 s3 s0 s1 s2 s3 ... in memory order.
uint32_t pattern = (uint32_t)shuffler[0] | ((uint32_t)shuffler[1] << 8) |
((uint32_t)shuffler[2] << 16) |
((uint32_t)shuffler[3] << 24);
asm volatile(
// v16..v23 (e16, m8): index = (i & ~3) + shuffler[i & 3].
"vsetvli %[vl], %[w], e16, m8, ta, ma \n"
"vid.v v16 \n"
"vand.vi v16, v16, -4 \n"
"srli %[vl], %[vl], 2 \n"
"vsetvli zero, %[vl], e32, m4, ta, ma \n"
"vmv.v.x v8, %[pattern] \n" // s0 s1 s2 s3 ...
"vsetvli %[vl], %[w], e16, m8, ta, ma \n"
"vzext.vf2 v24, v8 \n" // bytes -> e16
"vadd.vv v16, v16, v24 \n"
"1: \n"
"vsetvli %[vl], %[w], e8, m4, ta, ma \n"
"vle8.v v8, (%[src]) \n"
"vrgatherei16.vv v0, v8, v16 \n"
"vse8.v v0, (%[dst]) \n"
"sub %[w], %[w], %[vl] \n"
"add %[src], %[src], %[vl] \n"
"add %[dst], %[dst], %[vl] \n"
"bnez %[w], 1b \n"
: [src] "+r"(src_argb), // %[src]
[dst] "+r"(dst_argb), // %[dst]
[w] "+r"(w), // %[w]
[vl] "=&r"(vl) // %[vl]
: [pattern] "r"(pattern) // %[pattern]
: RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10",
"v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
"v25", "v26", "v27", "v28", "v29", "v30", "v31");
}
#endif // HAS_ARGBSHUFFLEROW_RVV
#ifdef HAS_MIRRORUVROW_RVV
void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
int vl;

29
source/row_sve.cc
View File

@ -777,7 +777,7 @@ void DivideRow_16_SVE2(const uint16_t* src_y,
"cnth %x[vl] \n"
"dup z0.h, %w[scale] \n"
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
"b.le 2f \n"
"b.lt 2f \n"
// Run bulk of computation with the same predicates to avoid predicate
// generation overhead.
@ -792,7 +792,7 @@ void DivideRow_16_SVE2(const uint16_t* src_y,
"st1h {z1.h}, p0, [%[dst]] \n"
"st1h {z2.h}, p0, [%[dst], #1, mul vl] \n"
"incb %[dst], all, mul #2 \n"
"b.gt 1b \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
@ -827,6 +827,10 @@ void DivideRow_16_SVE2(const uint16_t* src_y,
"fmul z1.s, z1.s, z4.s \n" \
"fmul z2.s, z2.s, z4.s \n" \
"fmul z3.s, z3.s, z4.s \n" \
"fmul z0.s, z0.s, z5.s \n" \
"fmul z1.s, z1.s, z5.s \n" \
"fmul z2.s, z2.s, z5.s \n" \
"fmul z3.s, z3.s, z5.s \n" \
"uqshrnb z0.h, z0.s, #13 \n" \
"uqshrnb z1.h, z1.s, #13 \n" \
"uqshrnb z2.h, z2.s, #13 \n" \
@ -840,6 +844,7 @@ void HalfFloatRow_SVE2(const uint16_t* src,
asm("cntw %x0" : "=r"(vl));
asm volatile(
"mov z4.s, %s[scale] \n"
"mov z5.s, %s[bias] \n"
"subs %w[width], %w[width], %w[vl], lsl #2 \n"
"b.lt 2f \n"
@ -881,14 +886,16 @@ void HalfFloatRow_SVE2(const uint16_t* src,
"st1h {z3.s}, p3, [%[dst], #3, mul vl] \n"
"99: \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[vl2] "r"(vl * 2), // %[vl2]
[vl3] "r"(vl * 3), // %[vl3]
[scale] "w"(scale * 1.9259299444e-34f) // %[scale]
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2", "p3");
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[vl2] "r"(vl * 2), // %[vl2]
[vl3] "r"(vl * 3), // %[vl3]
[scale] "w"(scale), // %[scale]
[bias] "w"(1.9259299444e-34f) // %[bias] 2^-112
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "p0", "p1", "p2",
"p3");
}
void HalfFloat1Row_SVE2(const uint16_t* src,
@ -1221,7 +1228,7 @@ void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) {
"st1b {z0.b}, p0, [%[dst], #0, mul vl] \n"
"st1b {z1.b}, p0, [%[dst], #1, mul vl] \n"
"incb %[dst], all, mul #2 \n"
"b.gt 1b \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl2] \n"

5
source/scale.cc
View File

@ -938,6 +938,7 @@ static void ScaleAddCols1_16_C(int dst_width,
int boxwidth = MIN1(dx >> 16);
int scaleval = 65536 / (boxwidth * boxheight);
int i;
x >>= 16;
for (i = 0; i < dst_width; ++i) {
*dst_ptr++ = SumPixels_16(boxwidth, src_ptr + x) * scaleval >> 16;
x += boxwidth;
@ -1498,7 +1499,7 @@ static int ScalePlaneBilinearUp(int src_width,
#endif
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
ScaleFilterCols = ScaleColsUp2_C;
#if defined(HAS_SCALECOLS_SSE2)
#if defined(HAS_SCALECOLSUP2_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
ScaleFilterCols = ScaleColsUp2_SSE2;
}
@ -2092,7 +2093,7 @@ static void ScalePlaneSimple(int src_width,
if (src_width * 2 == dst_width && x < 0x8000) {
ScaleCols = ScaleColsUp2_C;
#if defined(HAS_SCALECOLS_SSE2)
#if defined(HAS_SCALECOLSUP2_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
ScaleCols = ScaleColsUp2_SSE2;
}

4
source/scale_any.cc
View File

@ -1024,7 +1024,7 @@ SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_AVX2,
SBUH2LANY(ScaleUVRowUp2_Linear_Any_NEON,
ScaleUVRowUp2_Linear_NEON,
ScaleUVRowUp2_Linear_C,
15,
7,
uint8_t)
#endif
@ -1032,7 +1032,7 @@ SBUH2LANY(ScaleUVRowUp2_Linear_Any_NEON,
SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_NEON,
ScaleUVRowUp2_Linear_16_NEON,
ScaleUVRowUp2_Linear_16_C,
15,
7,
uint16_t)
#endif

3
source/scale_argb.cc
View File

@ -482,7 +482,8 @@ static int ScaleARGBBilinearDown(int src_width,
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
// Allocate a row of ARGB.
{
align_buffer_64(row, clip_src_width * 4);
// clip_src_width is already in bytes (4 per ARGB pixel).
align_buffer_64(row, clip_src_width);
if (!row)
return 1;

15
source/scale_gcc.cc
View File

@ -1275,21 +1275,6 @@ void ScaleRowUp2_Bilinear_16_SSE2(const uint16_t* src_ptr,
LABELALIGN
"1: \n"
"movq (%0),%%xmm0 \n" // 0011 (16b, 1u1v)
"movq 4(%0),%%xmm1 \n" // 1122 (16b, 1u1v)
"punpcklwd %%xmm7,%%xmm0 \n" // 0011 (near) (32b, 1u1v)
"punpcklwd %%xmm7,%%xmm1 \n" // 1122 (near) (32b, 1u1v)
"movdqa %%xmm0,%%xmm2 \n"
"movdqa %%xmm1,%%xmm3 \n"
"pshufd $0b01001110,%%xmm2,%%xmm2 \n" // 1100 (far) (1, lo)
"pshufd $0b01001110,%%xmm3,%%xmm3 \n" // 2211 (far) (1, hi)
"paddd %%xmm0,%%xmm2 \n" // near+far (1, lo)
"paddd %%xmm1,%%xmm3 \n" // near+far (1, hi)
"paddd %%xmm0,%%xmm0 \n" // 2*near (1, lo)
"paddd %%xmm1,%%xmm1 \n" // 2*near (1, hi)
"paddd %%xmm2,%%xmm0 \n" // 3*near+far (1, lo)
"paddd %%xmm3,%%xmm1 \n" // 3*near+far (1, hi)
"movq (%0),%%xmm0 \n" // 0123 (16b)
"movq 2(%0),%%xmm1 \n" // 1234 (16b)
"punpcklwd %%xmm7,%%xmm0 \n" // 0123 (32b)

4
source/scale_neon64.cc
View File

@ -1013,9 +1013,9 @@ void ScaleAddRow_NEON(const uint8_t* src_ptr,
}
#define SCALE_FILTER_COLS_STEP_ADDR \
"lsr %[tmp_offset], %x[x], #16 \n" \
"sbfx %[tmp_offset], %x[x], #16, #16 \n" \
"add %[tmp_ptr], %[src_ptr], %[tmp_offset] \n" \
"add %x[x], %x[x], %x[dx] \n"
"add %w[x], %w[x], %w[dx] \n"
// The Neon version mimics this formula (from scale_common.cc):
// #define BLENDER(a, b, f) (uint8_t)((int)(a) +

13
source/scale_uv.cc
View File

@ -235,7 +235,7 @@ static int ScaleUVDown4Box(int src_width,
}
}
#endif
#if defined(HAS_SCALEUVROWDOWN2BOX_SME)
#if defined(HAS_SCALEUVROWDOWN2_SME)
if (TestCpuFlag(kCpuHasSME)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_SME;
}
@ -693,7 +693,8 @@ static int ScaleUVBilinearDown(int src_width,
{
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
uint32_t uy = y;
align_buffer_64(row, clip_src_width * 2);
// clip_src_width is already in bytes (2 per UV pixel).
align_buffer_64(row, clip_src_width);
if (!row)
return 1;
if (uy > max_y) {
@ -1450,6 +1451,14 @@ int UVScale_16(const uint16_t* src_uv,
}
#endif
// ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone, which has
// no 16 bit implementation other than the copy above. The Up2 kernels only
// read the pixels they need, so restore Linear or Bilinear for a 2x upscale
// so P010ToP410 and P210ToP410 work on 1 pixel wide images.
if (filtering == kFilterNone && src_width == 1 && dst_width <= 2) {
filtering = (src_height == dst_height) ? kFilterLinear : kFilterBilinear;
}
if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) {
ScaleUVLinearUp2_16(src_width, src_height, dst_width, dst_height,
src_stride_uv, dst_stride_uv, src_uv, dst_uv);