coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

[libyuv] Fix top 9 bugs and performance issues

Browse Source 
1. Fix TestCpuFlag(A | B) multi-bit checks in AYUVToNV12, AYUVToNV21,
   NV12ToRGB24Matrix, RAWToRGBARow, MergeRGBRow, and MergeXR30Row so
   that all required ISA flags are verified (== (A | B)) instead of
   matching when only one flag is present.
2. Fix 2-byte heap buffer overflow on odd widths in
   I422ToARGB1555Row_SVE_SC and I422ToARGB4444Row_SVE_SC by replacing
   st1h on interleaved lanes with trn1/trn2 + st2b under per-pixel byte
   predicates.
3. Fix ARGBPolynomialRow_SSE2 clamping outputs to <= 127 (packssdw
   before packuswb instead of double packuswb) and add missing
   vzeroupper to SplitRGBRow_AVX2.
4. Fix AR64ShuffleRow_C processing only half the row (loop bound width
   instead of width / 2).
5. Fix negative-height crashes and orientation errors in
   ConvertFromI420, SplitARGBPlane (when dst_a == NULL), I422Rotate,
   I010Rotate, and I210Rotate.
6. Fix signed 16.16 fixed-point coordinate handling for negative x/dx
   in ScaleFilterCols_NEON (asr + 64-bit sign-extended x64/dx64) and
   ScaleAddCols1_16_C (x >>= 16).
7. Fix ConvertToARGB odd crop_x UV alignment for NV12/NV21, src_u
   offset calculation on odd sample widths for I420/YV12/J420/H420/U420,
   and FOURCC_U422 BT.2020 matrix dispatch (U422ToARGB).
8. Remove dead 14-instruction block before loop label 1: in
   ScaleRowUp2_Bilinear_16_SSE2.
9. Fix SIMD feature macros, dead dispatches, and SVE2 loop tail
   branches:
   - Define HAS_SCALECOLSUP2_SSE2 in scale.cc for 32-bit x86.
   - Guard TransposeWx4_32_SVE2 in rotate_argb.cc with
     HAS_TRANSPOSEWX4_32_SVE2.
   - Restrict HAS_UYVYTOARGBROW_SME and HAS_SCALEUVROWDOWN2_SME to
     Clang >= 20 to match their implementations.
   - Add AB64ToARGBRow_Any_NEON/SSE2/AVX2/AVX512BW wrappers so unaligned
     widths use SIMD + C tail.
   - Remove duplicate macro definitions in include/libyuv/row.h.
   - Replace b.lt 2f / b.ge 1b with b.gt 1b in SVE2 row loops so one
     redundant tail iteration is not executed when width % vl == 0.

R=rrwinterton@gmail.com

Bug: libyuv:574714081, b/574714081
Test: out/Release/libyuv_unittest && out/asan/libyuv_unittest
Change-Id: Iec3ae5c58d52bee3ef13cf9b765ac84b733aa576
This commit is contained in:
Frank Barchard 2026-10-19 02:44:37 -07:00
parent 2404f9730e
commit 66fb736450
30 changed files with 829 additions and 870 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
include/libyuv/row.h
View File

@ -153,13 +153,11 @@ extern "C" {
#define HAS_ARGBCOPYALPHAROW_AVX2
#define HAS_ARGBCOPYYTOALPHAROW_AVX2
#define HAS_ARGBEXTRACTALPHAROW_AVX2
#define HAS_ARGBMIRRORROW_AVX2
#define HAS_ARGBPOLYNOMIALROW_AVX2
#define HAS_ARGBSHUFFLEROW_AVX2
#define HAS_ARGBTORGB565DITHERROW_AVX2
#define HAS_COPYROW_AVX
#define HAS_HALFFLOATROW_AVX2
#define HAS_I422TOARGBROW_AVX2
#define HAS_I422TORGB24ROW_AVX2
#define HAS_I422TORGBAROW_AVX2
#define HAS_I422TORGB565ROW_AVX2
@ -168,12 +166,8 @@ extern "C" {
#define HAS_I444TOARGBROW_AVX2
#define HAS_I444TORGB24ROW_AVX2
#define HAS_J400TOARGBROW_AVX2
#define HAS_MIRRORROW_AVX2
#define HAS_MIRRORSPLITUVROW_AVX2
#define HAS_NV12TOARGBROW_AVX2
#define HAS_NV12TORGB24ROW_AVX2
#define HAS_NV12TORGB565ROW_AVX2
#define HAS_NV21TOARGBROW_AVX2
#define HAS_NV21TORGB24ROW_AVX2
#define HAS_SPLITUVROW_AVX2
#define HAS_UYVYTOARGBROW_AVX2
@ -230,6 +224,7 @@ extern "C" {
#define HAS_MIRRORUVROW_SSSE3
#define HAS_NV21TOYUV24ROW_SSSE3
#define HAS_P210TOAR30ROW_SSSE3
#define HAS_RGB24MIRRORROW_SSSE3
#define HAS_P210TOARGBROW_SSSE3
#define HAS_P410TOAR30ROW_SSSE3
#define HAS_P410TOARGBROW_SSSE3
@ -474,7 +469,6 @@ extern "C" {
#define HAS_RGBTOUV444MATRIXROW_AVX512BW
#define HAS_ARGBTOYMATRIXROW_AVX512BW
#define HAS_I422TORGB24ROW_AVX512VBMI
#define HAS_RAWTORGB24ROW_AVX512VBMI
#define HAS_I422TORGB24ROW_AVX512BW
#define HAS_ARGBTOUVMATRIXROW_AVX512BW
#define HAS_J400TOARGBROW_AVX512BW
@ -796,6 +790,7 @@ extern "C" {
#define HAS_P210TOARGBROW_SME
#define HAS_P410TOAR30ROW_SME
#define HAS_P410TOARGBROW_SME
#define HAS_UYVYTOARGBROW_SME
#define HAS_YUY2TOARGBROW_SME
#endif
@ -959,6 +954,8 @@ extern "C" {
#define HAS_ARGBMULTIPLYROW_RVV
#define HAS_ARGBTOAR64ROW_RVV
#define HAS_ARGBMIRRORROW_RVV
#define HAS_ARGBSETROW_RVV
#define HAS_ARGBSHUFFLEROW_RVV
#define HAS_ARGBTOUV444MATRIXROW_RVV
#define HAS_ARGBTOUVMATRIXROW_RVV
#define HAS_ARGBTOYMATRIXROW_RVV
@ -2907,6 +2904,9 @@ void ARGBMirrorRow_Any_LASX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RGB24MirrorRow_SSSE3(const uint8_t* src_rgb24,
uint8_t* dst_rgb24,
int width);
void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
uint8_t* dst_rgb24,
int width);
@ -2917,6 +2917,9 @@ void RGB24MirrorRow_NEON(const uint8_t* src_rgb24,
uint8_t* dst_rgb24,
int width);
void RGB24MirrorRow_C(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width);
void RGB24MirrorRow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RGB24MirrorRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
@ -4150,6 +4153,7 @@ void SetRow_Any_LASX(uint8_t* dst_ptr, uint8_t v32, int width);
void ARGBSetRow_C(uint8_t* dst_argb, uint32_t v32, int width);
void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width);
void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width);
void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width);
void ARGBSetRow_Any_NEON(uint8_t* dst_ptr, uint32_t v32, int width);
void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width);
void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width);
@ -4177,6 +4181,10 @@ void ARGBShuffleRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width);
void ARGBShuffleRow_RVV(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width);
void ARGBShuffleRow_LSX(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,

336
include/libyuv/row_sve.h
View File

@ -466,7 +466,7 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y,
"ptrue p0.b \n"
"dup z19.b, #255 \n" // Alpha
YUVTORGB_SVE_SETUP
"cmp %w[width], %w[vl] \n"
"subs %w[width], %w[width], %w[vl] \n"
"mov z1.h, #128 \n" // U/V
"umullb z6.h, z30.b, z1.b \n"
"umullb z4.h, z28.b, z1.b \n" // DB
@ -475,25 +475,28 @@ static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y,
"sub z4.h, z4.h, z25.h \n"
"sub z5.h, z5.h, z27.h \n"
"sub z6.h, z26.h, z6.h \n"
"b.le 2f \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p1.h \n"
"sub %w[width], %w[width], %w[vl] \n"
"1: \n" //
READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE
"subs %w[width], %w[width], %w[vl] \n"
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
"b.gt 1b \n"
"add %w[width], %w[width], %w[vl] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"2: \n"
"whilelt p1.h, wzr, %w[width] \n" //
READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width), // %[width]
@ -702,7 +705,11 @@ static inline void I422ToARGB1555Row_SVE_SC(
"whilelt p1.b, wzr, %w[width] \n" //
READYUV422_SVE_2X I422TORGB_SVE_2X
RGBTOARGB8_SVE_TOP_2X RGB8TOARGB1555_SVE_FROM_TOP_2X
"st2h {z0.h, z1.h}, p1, [%[dst]] \n"
// Need to permute the data on the final iteration such that the
// predicates (.b) line up with the 16-bit element data.
"trn1 z2.b, z0.b, z1.b \n"
"trn2 z3.b, z0.b, z1.b \n"
"st2b {z2.b, z3.b}, p1, [%[dst]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
@ -761,7 +768,11 @@ static inline void I422ToARGB4444Row_SVE_SC(
"whilelt p1.b, wzr, %w[width] \n" //
READYUV422_SVE_2X I422TORGB_SVE_2X
RGBTOARGB8_SVE_TOP_2X RGB8TOARGB4444_SVE_FROM_TOP_2X
"st2h {z0.h, z1.h}, p1, [%[dst]] \n"
// Need to permute the data on the final iteration such that the
// predicates (.b) line up with the 16-bit element data.
"trn1 z2.b, z0.b, z1.b \n"
"trn2 z3.b, z0.b, z1.b \n"
"st2b {z2.b, z3.b}, p1, [%[dst]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
@ -788,7 +799,7 @@ static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y,
YUVTORGB_SVE_SETUP
"dup z19.b, #255 \n" // Alpha
"subs %w[width], %w[width], %w[vl] \n"
"b.le 2f \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
@ -798,7 +809,7 @@ static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y,
"subs %w[width], %w[width], %w[vl] \n"
"st2h {z19.h, z20.h}, p1, [%[dst_rgba]] \n"
"add %[dst_rgba], %[dst_rgba], %[vl], lsl #2 \n"
"b.gt 1b \n"
"b.ge 1b \n"
// Calculate a predicate for the final iteration to deal with the tail.
"2: \n"
@ -836,7 +847,7 @@ static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y,
YUVTORGB_SVE_SETUP_AR30
"dup z23.h, %w[limit] \n"
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
"b.le 2f \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
@ -846,7 +857,7 @@ static inline void I422ToAR30Row_SVE_SC(const uint8_t* src_y,
"1: \n" //
READYUV422_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
"b.gt 1b \n"
"b.ge 1b \n"
// Calculate a predicate for the final iteration to deal with the tail.
"2: \n"
@ -985,8 +996,6 @@ static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cntb %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1010,21 +1019,21 @@ static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.b, wzr, %w[width_last_y] \n"
"whilelt p2.b, wzr, %w[width_last_uv] \n" //
"whilelt p1.b, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.b, wzr, %w[width] \n" //
READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
"st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1035,8 +1044,6 @@ static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cntb %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1060,21 +1067,21 @@ static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.b, wzr, %w[width_last_y] \n"
"whilelt p2.b, wzr, %w[width_last_uv] \n" //
"whilelt p1.b, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.b, wzr, %w[width] \n" //
READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
"st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_vu), // %[src_vu]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_vu), // %[src_vu]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1086,8 +1093,6 @@ static inline void NV12ToRGB24Row_SVE_SC(
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cntb %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1111,21 +1116,21 @@ static inline void NV12ToRGB24Row_SVE_SC(
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.b, wzr, %w[width_last_y] \n"
"whilelt p2.b, wzr, %w[width_last_uv] \n" //
"whilelt p1.b, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.b, wzr, %w[width] \n" //
READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
"st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1137,8 +1142,6 @@ static inline void NV21ToRGB24Row_SVE_SC(
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cntb %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1162,21 +1165,21 @@ static inline void NV21ToRGB24Row_SVE_SC(
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.b, wzr, %w[width_last_y] \n"
"whilelt p2.b, wzr, %w[width_last_uv] \n" //
"whilelt p1.b, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.b, wzr, %w[width] \n" //
READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
"st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_vu), // %[src_vu]
[dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_vu), // %[src_vu]
[dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1195,8 +1198,6 @@ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2,
uint32_t nv_uv_step = 0x04040404U;
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n"
"index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
@ -1221,8 +1222,10 @@ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.h, wzr, %w[width_last_uv] \n" //
"whilelt p1.h, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.h, wzr, %w[width] \n" //
READYUY2_SVE NVTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
@ -1234,9 +1237,7 @@ static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2,
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
[nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
[nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1255,8 +1256,6 @@ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy,
uint32_t nv_uv_step = 0x04040404U;
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
asm volatile(
"ptrue p0.b \n"
"index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
@ -1281,9 +1280,10 @@ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"2: \n"
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.h, wzr, %w[width_last_uv] \n" //
"whilelt p1.h, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.h, wzr, %w[width] \n" //
READUYVY_SVE NVTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
@ -1295,9 +1295,7 @@ static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy,
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
[nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
[nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step]
: "cc", "memory", YUVTORGB_SVE_REGS, "p2");
}
@ -1309,7 +1307,6 @@ static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1332,20 +1329,19 @@ static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1359,7 +1355,6 @@ static inline void I210AlphaToARGBRow_SVE_SC(
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1385,23 +1380,22 @@ static inline void I210AlphaToARGBRow_SVE_SC(
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI210_SVE
"ld1h {z19.h}, p1/z, [%[src_a]] \n" //
I4XXTORGB_SVE RGBATOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[src_a] "+r"(src_a), // %[src_a]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[src_a] "+r"(src_a), // %[src_a]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1413,8 +1407,6 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (2 * vl - 1);
int width_last_uv = (width_last_y + 1) / 2;
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE_2X.
uint16_t limit = 0x3ff0;
asm volatile(
@ -1439,9 +1431,11 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_uv] \n"
"whilelt p2.h, wzr, %w[width_last_y] \n"
"whilelt p3.h, %w[vl], %w[width_last_y] \n" //
"whilelt p2.h, wzr, %w[width] \n"
"whilelt p3.h, %w[vl], %w[width] \n"
"add %w[width], %w[width], #1 \n" // (width + 1) / 2
"lsr %w[width], %w[width], #1 \n"
"whilelt p1.h, wzr, %w[width] \n" //
READI210_SVE_2X I422TORGB_SVE_2X STOREAR30_SVE_2X
"99: \n"
@ -1453,8 +1447,6 @@ static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y,
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv), // %[width_last_uv]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1467,8 +1459,6 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
uint32_t nv_uv_start = 0x03010301U;
uint32_t nv_uv_step = 0x04040404U;
asm volatile(
@ -1495,8 +1485,10 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.h, wzr, %w[width_last_uv] \n" //
"whilelt p1.h, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.h, wzr, %w[width] \n" //
READP210_SVE NVTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
@ -1509,9 +1501,7 @@ static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y,
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
[nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv) // %[width_last_uv]
[nv_uv_step] "r"(nv_uv_step) // %[nv_uv_step]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1522,8 +1512,6 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
int width_last_uv = width_last_y + (width_last_y & 1);
uint32_t nv_uv_start = 0x03010301U;
uint32_t nv_uv_step = 0x04040404U;
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
@ -1551,8 +1539,10 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.h, wzr, %w[width_last_uv] \n" //
"whilelt p1.h, wzr, %w[width] \n"
"add %w[width], %w[width], #1 \n" // Round up to even
"and %w[width], %w[width], #0xfffffffe \n"
"whilelt p2.h, wzr, %w[width] \n" //
READP210_SVE NVTORGB_SVE STOREAR30_SVE
"99: \n"
@ -1565,8 +1555,6 @@ static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y,
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
[nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[width_last_uv] "r"(width_last_uv), // %[width_last_uv]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1579,7 +1567,6 @@ static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1602,20 +1589,19 @@ static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI410_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1629,7 +1615,6 @@ static inline void I410AlphaToARGBRow_SVE_SC(
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1656,23 +1641,22 @@ static inline void I410AlphaToARGBRow_SVE_SC(
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI410_SVE
"ld1h {z19.h}, p1/z, [%[src_a]] \n" //
I4XXTORGB_SVE RGBATOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[src_a] "+r"(src_a), // %[src_a]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[src_a] "+r"(src_a), // %[src_a]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1684,7 +1668,6 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
uint16_t limit = 0x3ff0;
asm volatile(
@ -1707,7 +1690,7 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI410_SVE I4XXTORGB_SVE STOREAR30_SVE
"99: \n"
@ -1719,7 +1702,6 @@ static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y,
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1731,7 +1713,6 @@ static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1756,22 +1737,21 @@ static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.s, wzr, %w[width_last_y] \n"
"whilelt p1.h, wzr, %w[width] \n"
"whilelt p2.s, wzr, %w[width] \n"
"cntw %x[vl] \n"
"whilelt p3.s, %w[vl], %w[width_last_y] \n" //
"whilelt p3.s, %w[vl], %w[width] \n" //
READP410_SVE NVTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width), // %[width]
[vl] "+r"(vl) // %[vl]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_uv] "+r"(src_uv), // %[src_uv]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width), // %[width]
[vl] "+r"(vl) // %[vl]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1782,7 +1762,6 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
uint16_t limit = 0x3ff0;
asm volatile(
@ -1808,10 +1787,10 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n"
"whilelt p2.s, wzr, %w[width_last_y] \n"
"whilelt p1.h, wzr, %w[width] \n"
"whilelt p2.s, wzr, %w[width] \n"
"cntw %x[vl] \n"
"whilelt p3.s, %w[vl], %w[width_last_y] \n" //
"whilelt p3.s, %w[vl], %w[width] \n" //
READP410_SVE NVTORGB_SVE STOREAR30_SVE
"99: \n"
@ -1822,7 +1801,6 @@ static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y,
[vl] "+r"(vl) // %[vl]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1835,7 +1813,6 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
// The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
uint16_t limit = 0x3ff0;
asm volatile(
@ -1858,7 +1835,7 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI212_SVE I4XXTORGB_SVE STOREAR30_SVE
"99: \n"
@ -1870,7 +1847,6 @@ static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y,
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y), // %[width_last_y]
[limit] "r"(limit) // %[limit]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -1883,7 +1859,6 @@ static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y,
int width) STREAMING_COMPATIBLE {
uint64_t vl;
asm("cnth %0" : "=r"(vl));
int width_last_y = width & (vl - 1);
asm volatile(
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
@ -1906,20 +1881,19 @@ static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y,
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.h, wzr, %w[width_last_y] \n" //
"whilelt p1.h, wzr, %w[width] \n" //
READI212_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
"st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
[width_last_y] "r"(width_last_y) // %[width_last_y]
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
}
@ -2120,7 +2094,7 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb,
"ptrue p5.h \n"
"1: \n" //
ARGBTOUVMATRIX_SVE
"b.gt 1b \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl], lsl #2 \n"
@ -2197,7 +2171,7 @@ static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb,
"ptrue p1.s \n"
"1: \n" //
ARGBTOYMATRIX_SVE("p1", "p1", "p1", "p1", "p0")
"b.gt 1b \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl], lsl #2 \n"

15
source/compare.cc
View File

@ -427,8 +427,10 @@ double CalcFrameSsim(const uint8_t* src_a,
int height) {
int samples = 0;
double ssim_total = 0;
if (width <= 8 || height <= 8) {
return ssim_total / samples;
// An 8x8 window must fit. Smaller planes have no samples; return 0 rather
// than 0.0 / 0 (NaN) so I420Ssim on small images stays finite.
if (width < 8 || height < 8) {
return 0.0;
}
int simd_size = 4;
@ -501,7 +503,8 @@ double CalcFrameSsim(const uint8_t* src_a,
}
#endif
const int num_windows = (width - 8 + 3) >> 2;
// Windows start every 4 pixels; the last one starts at width - 8.
const int num_windows = ((width - 8) >> 2) + 1;
const int num_tiles = num_windows + 1;
const int tile_width = num_tiles * 4;
const int main_width = tile_width & ~(simd_size - 1);
@ -520,7 +523,7 @@ double CalcFrameSsim(const uint8_t* src_a,
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for (i = 0; i < height - 8; i += 4) {
for (i = 0; i <= height - 8; i += 4) {
const uint8_t* a = src_a + (ptrdiff_t)i * stride_a;
const uint8_t* b = src_b + (ptrdiff_t)i * stride_b;
if (prev_i + 4 != i) {
@ -554,11 +557,11 @@ double CalcFrameSsim(const uint8_t* src_a,
#ifdef _OPENMP
#pragma omp for schedule(static)
#endif
for (i = 0; i < height - 8; i += 4) {
for (i = 0; i <= height - 8; i += 4) {
const uint8_t* a = src_a + (ptrdiff_t)i * stride_a;
const uint8_t* b = src_b + (ptrdiff_t)i * stride_b;
int j;
for (j = 0; j < width - 8; j += 4) {
for (j = 0; j <= width - 8; j += 4) {
ssim_total += Ssim8x8_C(a + j, stride_a, b + j, stride_b);
samples++;
}

47
source/convert.cc
View File

@ -1245,6 +1245,7 @@ int I422ToNV21(const uint8_t* src_y,
dst_y, dst_stride_y, plane_u, halfwidth, plane_v, halfwidth,
width, height);
if (r != 0) {
free_aligned_buffer_64(plane_u);
return r;
}
MergeUVPlane(plane_v, halfwidth, plane_u, halfwidth, dst_vu, dst_stride_vu,
@ -2648,28 +2649,9 @@ int AYUVToNV12(const uint8_t* src_ayuv,
src_stride_ayuv = -src_stride_ayuv;
}
// place holders for future intel code
#if defined(HAS_AYUVTOYROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
AYUVToUVRow = AYUVToUVRow_Any_SSE2;
AYUVToYRow = AYUVToYRow_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
AYUVToUVRow = AYUVToUVRow_SSE2;
AYUVToYRow = AYUVToYRow_SSE2;
}
}
#endif
#if defined(HAS_AYUVTOYROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
AYUVToUVRow = AYUVToUVRow_Any_AVX2;
AYUVToYRow = AYUVToYRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
AYUVToUVRow = AYUVToUVRow_AVX2;
AYUVToYRow = AYUVToYRow_AVX2;
}
}
#endif
#if defined(HAS_AYUVTOYROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
AYUVToUVRow = AYUVToUVRow_AVX512VBMI;
AYUVToYRow = AYUVToYRow_AVX512VBMI;
}
@ -2734,28 +2716,9 @@ int AYUVToNV21(const uint8_t* src_ayuv,
src_stride_ayuv = -src_stride_ayuv;
}
// place holders for future intel code
#if defined(HAS_AYUVTOYROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
AYUVToVURow = AYUVToVURow_Any_SSE2;
AYUVToYRow = AYUVToYRow_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
AYUVToVURow = AYUVToVURow_SSE2;
AYUVToYRow = AYUVToYRow_SSE2;
}
}
#endif
#if defined(HAS_AYUVTOYROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
AYUVToVURow = AYUVToVURow_Any_AVX2;
AYUVToYRow = AYUVToYRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
AYUVToVURow = AYUVToVURow_AVX2;
AYUVToYRow = AYUVToYRow_AVX2;
}
}
#endif
#if defined(HAS_AYUVTOYROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
AYUVToVURow = AYUVToVURow_AVX512VBMI;
AYUVToYRow = AYUVToYRow_AVX512VBMI;
}

89
source/convert_argb.cc
View File

@ -619,14 +619,6 @@ int I444ToARGBMatrix(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_I444TOARGBROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
I444ToARGBRow = I444ToARGBRow_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I444ToARGBRow = I444ToARGBRow_AVX512BW;
}
}
#endif
#if defined(HAS_I444TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
I444ToARGBRow = I444ToARGBRow_Any_NEON;
@ -891,14 +883,6 @@ int I444ToRGB24Matrix(const uint8_t* src_y,
I444ToRGB24Row = I444ToRGB24Row_SME;
}
#endif
#if defined(HAS_I444TORGB24ROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
I444ToRGB24Row = I444ToRGB24Row_Any_LSX;
if (IS_ALIGNED(width, 16)) {
I444ToRGB24Row = I444ToRGB24Row_LSX;
}
}
#endif
#if defined(HAS_I444TORGB24ROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
I444ToRGB24Row = I444ToRGB24Row_RVV;
@ -4133,7 +4117,8 @@ int RAWToRGBA(const uint8_t* src_raw,
}
#endif
#if defined(HAS_RAWTORGBAROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
RAWToRGBARow = RAWToRGBARow_AVX512VBMI;
}
#endif
@ -4990,7 +4975,8 @@ int NV12ToRGB24Matrix(const uint8_t* src_y,
}
#endif
#if defined(HAS_NV12TORGB24ROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
NV12ToRGB24Row = NV12ToRGB24Row_AVX512VBMI;
}
#endif
@ -5072,7 +5058,8 @@ int NV21ToRGB24Matrix(const uint8_t* src_y,
}
#endif
#if defined(HAS_NV21TORGB24ROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
NV21ToRGB24Row = NV21ToRGB24Row_AVX512VBMI;
}
#endif
@ -5568,14 +5555,6 @@ int I422ToRGBAMatrix(const uint8_t* src_y,
height = 1;
src_stride_y = src_stride_u = src_stride_v = dst_stride_rgba = 0;
}
#if defined(HAS_I422TORGBAROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I422ToRGBARow = I422ToRGBARow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
I422ToRGBARow = I422ToRGBARow_SSSE3;
}
}
#endif
#if defined(HAS_I422TORGBAROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I422ToRGBARow = I422ToRGBARow_Any_AVX2;
@ -5800,14 +5779,6 @@ int I420ToRGBAMatrix(const uint8_t* src_y,
dst_rgba = dst_rgba + (ptrdiff_t)(height - 1) * dst_stride_rgba;
dst_stride_rgba = -dst_stride_rgba;
}
#if defined(HAS_I422TORGBAROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I422ToRGBARow = I422ToRGBARow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
I422ToRGBARow = I422ToRGBARow_SSSE3;
}
}
#endif
#if defined(HAS_I422TORGBAROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I422ToRGBARow = I422ToRGBARow_Any_AVX2;
@ -6866,14 +6837,6 @@ int I420ToRGB565Dither(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_ARGBTORGB565DITHERROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_SSE2;
if (IS_ALIGNED(width, 4)) {
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SSE2;
}
}
#endif
#if defined(HAS_ARGBTORGB565DITHERROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2;
@ -7152,14 +7115,6 @@ static int I420ToARGBMatrixBilinear(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_I444TOARGBROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
I444ToARGBRow = I444ToARGBRow_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I444ToARGBRow = I444ToARGBRow_AVX512BW;
}
}
#endif
#if defined(HAS_I444TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
I444ToARGBRow = I444ToARGBRow_Any_NEON;
@ -7318,14 +7273,6 @@ static int I422ToARGBMatrixLinear(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_I444TOARGBROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
I444ToARGBRow = I444ToARGBRow_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I444ToARGBRow = I444ToARGBRow_AVX512BW;
}
}
#endif
#if defined(HAS_I444TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
I444ToARGBRow = I444ToARGBRow_Any_NEON;
@ -7479,14 +7426,6 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y,
I444ToRGB24Row = I444ToRGB24Row_SME;
}
#endif
#if defined(HAS_I444TORGB24ROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
I444ToRGB24Row = I444ToRGB24Row_Any_LASX;
if (IS_ALIGNED(width, 32)) {
I444ToRGB24Row = I444ToRGB24Row_LASX;
}
}
#endif
#if defined(HAS_I444TORGB24ROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
I444ToRGB24Row = I444ToRGB24Row_RVV;
@ -8097,14 +8036,6 @@ static int I420AlphaToARGBMatrixBilinear(
I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
}
#endif
#if defined(HAS_I444ALPHATOARGBROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
I444AlphaToARGBRow = I444AlphaToARGBRow_LASX;
}
}
#endif
#if defined(HAS_I444ALPHATOARGBROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
I444AlphaToARGBRow = I444AlphaToARGBRow_RVV;
@ -8322,14 +8253,6 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y,
I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
}
#endif
#if defined(HAS_I444ALPHATOARGBROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
I444AlphaToARGBRow = I444AlphaToARGBRow_LASX;
}
}
#endif
#if defined(HAS_I444ALPHATOARGBROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
I444AlphaToARGBRow = I444AlphaToARGBRow_RVV;

43
source/convert_from.cc
View File

@ -1028,6 +1028,7 @@ int ConvertFromI420(const uint8_t* y,
height == INT_MIN) {
return -1;
}
const int abs_height = Abs(height);
switch (format) {
// Single plane formats
case FOURCC_YUY2:
@ -1097,7 +1098,7 @@ int ConvertFromI420(const uint8_t* y,
break;
case FOURCC_NV12: {
int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
r = I420ToNV12(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride ? dst_sample_stride : width, dst_uv,
dst_sample_stride ? dst_sample_stride : width, width,
@ -1106,7 +1107,7 @@ int ConvertFromI420(const uint8_t* y,
}
case FOURCC_NV21: {
int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
uint8_t* dst_vu = dst_sample + (ptrdiff_t)dst_y_stride * height;
uint8_t* dst_vu = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
r = I420ToNV21(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride ? dst_sample_stride : width, dst_vu,
dst_sample_stride ? dst_sample_stride : width, width,
@ -1116,13 +1117,13 @@ int ConvertFromI420(const uint8_t* y,
case FOURCC_NV16: {
int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
int dst_uv_stride = dst_sample_stride ? dst_sample_stride : width;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
int halfwidth = (width + 1) / 2;
int halfheight = (height + 1) / 2;
int halfheight = SUBSAMPLE(height, 1, 1);
CopyPlane(y, y_stride, dst_sample, dst_y_stride, width, height);
MergeUVPlane(u, u_stride, v, v_stride, dst_uv, dst_uv_stride * 2,
halfwidth, halfheight);
if (height > 1) {
if (abs_height > 1) {
MergeUVPlane(u, u_stride, v, v_stride, dst_uv + dst_uv_stride,
dst_uv_stride * 2, halfwidth, height / 2);
}
@ -1132,9 +1133,9 @@ int ConvertFromI420(const uint8_t* y,
case FOURCC_NV24: {
int dst_y_stride = dst_sample_stride ? dst_sample_stride : width;
int dst_uv_stride = dst_sample_stride ? dst_sample_stride * 2 : width * 2;
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * height;
align_buffer_64(temp_u, (size_t)width * height);
align_buffer_64(temp_v, (size_t)width * height);
uint8_t* dst_uv = dst_sample + (ptrdiff_t)dst_y_stride * abs_height;
align_buffer_64(temp_u, (size_t)width * abs_height);
align_buffer_64(temp_v, (size_t)width * abs_height);
if (!temp_u || !temp_v) {
free_aligned_buffer_64(temp_u);
free_aligned_buffer_64(temp_v);
@ -1144,7 +1145,7 @@ int ConvertFromI420(const uint8_t* y,
dst_y_stride, temp_u, width, temp_v, width, width, height);
if (r == 0) {
MergeUVPlane(temp_u, width, temp_v, width, dst_uv, dst_uv_stride, width,
height);
abs_height);
}
free_aligned_buffer_64(temp_u);
free_aligned_buffer_64(temp_v);
@ -1152,7 +1153,6 @@ int ConvertFromI420(const uint8_t* y,
}
case FOURCC_P010:
case FOURCC_P210: {
const int abs_height = Abs(height);
const int uv_min = ((width + 1) / 2) * 2;
int dst_stride_y;
int dst_stride_uv;
@ -1183,7 +1183,6 @@ int ConvertFromI420(const uint8_t* y,
break;
}
case FOURCC_P410: {
const int abs_height = Abs(height);
const int uv_min = width * 2;
int dst_stride_y;
int dst_stride_uv;
@ -1213,14 +1212,14 @@ int ConvertFromI420(const uint8_t* y,
case FOURCC_YV12: {
dst_sample_stride = dst_sample_stride ? dst_sample_stride : width;
int halfstride = (dst_sample_stride + 1) / 2;
int halfheight = (height + 1) / 2;
int halfheight = (abs_height + 1) / 2;
uint8_t* dst_u;
uint8_t* dst_v;
if (format == FOURCC_YV12) {
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_u = dst_v + (ptrdiff_t)halfstride * halfheight;
} else {
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_v = dst_u + (ptrdiff_t)halfstride * halfheight;
}
r = I420Copy(y, y_stride, u, u_stride, v, v_stride, dst_sample,
@ -1235,11 +1234,11 @@ int ConvertFromI420(const uint8_t* y,
uint8_t* dst_u;
uint8_t* dst_v;
if (format == FOURCC_YV16) {
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_u = dst_v + (ptrdiff_t)halfstride * height;
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_u = dst_v + (ptrdiff_t)halfstride * abs_height;
} else {
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_v = dst_u + (ptrdiff_t)halfstride * height;
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_v = dst_u + (ptrdiff_t)halfstride * abs_height;
}
r = I420ToI422(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride, dst_u, halfstride, dst_v, halfstride,
@ -1252,11 +1251,11 @@ int ConvertFromI420(const uint8_t* y,
uint8_t* dst_u;
uint8_t* dst_v;
if (format == FOURCC_YV24) {
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_u = dst_v + (ptrdiff_t)dst_sample_stride * height;
dst_v = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_u = dst_v + (ptrdiff_t)dst_sample_stride * abs_height;
} else {
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * height;
dst_v = dst_u + (ptrdiff_t)dst_sample_stride * height;
dst_u = dst_sample + (ptrdiff_t)dst_sample_stride * abs_height;
dst_v = dst_u + (ptrdiff_t)dst_sample_stride * abs_height;
}
r = I420ToI444(y, y_stride, u, u_stride, v, v_stride, dst_sample,
dst_sample_stride, dst_u, dst_sample_stride, dst_v,

56
source/convert_from_argb.cc
View File

@ -1550,6 +1550,14 @@ int ARGBToI400Matrix(const uint8_t* src_argb,
src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
@ -1636,6 +1644,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb,
dst_yuy2 = dst_yuy2 + (ptrdiff_t)(height - 1) * dst_stride_yuy2;
dst_stride_yuy2 = -dst_stride_yuy2;
}
#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
@ -1686,6 +1702,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
@ -1737,6 +1761,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_I422TOYUY2ROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
I422ToYUY2Row = I422ToYUY2Row_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
I422ToYUY2Row = I422ToYUY2Row_SSE2;
}
}
#endif
#if defined(HAS_I422TOYUY2ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I422ToYUY2Row = I422ToYUY2Row_Any_AVX2;
@ -1802,6 +1834,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb,
dst_uyvy = dst_uyvy + (ptrdiff_t)(height - 1) * dst_stride_uyvy;
dst_stride_uyvy = -dst_stride_uyvy;
}
#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
@ -1852,6 +1892,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
@ -1903,6 +1951,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_I422TOUYVYROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
I422ToUYVYRow = I422ToUYVYRow_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
I422ToUYVYRow = I422ToUYVYRow_SSE2;
}
}
#endif
#if defined(HAS_I422TOUYVYROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I422ToUYVYRow = I422ToUYVYRow_Any_AVX2;

24
source/convert_to_argb.cc
View File

@ -167,14 +167,6 @@ static int NV24ToARGB(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_I444TOARGBROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
I444ToARGBRow = I444ToARGBRow_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I444ToARGBRow = I444ToARGBRow_AVX512BW;
}
}
#endif
#if defined(HAS_I444TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
I444ToARGBRow = I444ToARGBRow_Any_NEON;
@ -433,7 +425,7 @@ int ConvertToARGB(const uint8_t* sample,
src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
src_uv = sample +
aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) +
crop_x;
(crop_x & ~1);
r = NV12ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb,
dst_stride_argb, crop_width, inv_crop_height);
break;
@ -441,7 +433,7 @@ int ConvertToARGB(const uint8_t* sample,
src = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
src_uv = sample +
aligned_src_width * ((ptrdiff_t)abs_src_height + crop_y / 2) +
crop_x;
(crop_x & ~1);
// Call NV12 but with u and v parameters swapped.
r = NV21ToARGB(src, src_width, src_uv, aligned_src_width, dst_argb,
dst_stride_argb, crop_width, inv_crop_height);
@ -508,12 +500,12 @@ int ConvertToARGB(const uint8_t* sample,
int halfheight = (abs_src_height + 1) / 2;
if (format == FOURCC_YV12) {
src_v = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
src_u = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2;
} else {
src_u = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) + crop_x / 2;
}
@ -527,7 +519,7 @@ int ConvertToARGB(const uint8_t* sample,
int halfheight = (abs_src_height + 1) / 2;
const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) +
crop_x / 2;
@ -541,7 +533,7 @@ int ConvertToARGB(const uint8_t* sample,
int halfheight = (abs_src_height + 1) / 2;
const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) +
crop_x / 2;
@ -555,7 +547,7 @@ int ConvertToARGB(const uint8_t* sample,
int halfheight = (abs_src_height + 1) / 2;
const uint8_t* src_y = sample + ((ptrdiff_t)src_width * crop_y + crop_x);
const uint8_t* src_u = sample + (ptrdiff_t)src_width * abs_src_height +
((ptrdiff_t)halfwidth * crop_y + crop_x) / 2;
(ptrdiff_t)halfwidth * (crop_y / 2) + crop_x / 2;
const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)halfheight + crop_y / 2) +
crop_x / 2;
@ -620,7 +612,7 @@ int ConvertToARGB(const uint8_t* sample,
const uint8_t* src_v = sample + (ptrdiff_t)src_width * abs_src_height +
halfwidth * ((ptrdiff_t)abs_src_height + crop_y) +
crop_x / 2;
r = H422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
r = U422ToARGB(src_y, src_width, src_u, halfwidth, src_v, halfwidth,
dst_argb, dst_stride_argb, crop_width, inv_crop_height);
break;
}

100
source/convert_to_i420.cc
View File

@ -51,8 +51,13 @@ static int NV16ToI420(const uint8_t* src_y,
int y;
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
// UV rows are halfwidth * 2 (even) bytes.
int awidth = halfwidth * 2;
void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
int width) = SplitUVRow_C;
void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 ||
height == 0 || height == INT_MIN) {
return -1;
@ -100,18 +105,105 @@ static int NV16ToI420(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_SPLITUVROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
SplitUVRow = SplitUVRow_SVE2;
}
#endif
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;
if (IS_ALIGNED(halfwidth, 16)) {
SplitUVRow = SplitUVRow_LSX;
}
}
#endif
#if defined(HAS_SPLITUVROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
SplitUVRow = SplitUVRow_Any_LASX;
if (IS_ALIGNED(halfwidth, 32)) {
SplitUVRow = SplitUVRow_LASX;
}
}
#endif
#if defined(HAS_SPLITUVROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
SplitUVRow = SplitUVRow_RVV;
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
if (IS_ALIGNED(awidth, 32)) {
InterpolateRow = InterpolateRow_AVX2;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_NEON;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
InterpolateRow = InterpolateRow_SVE2;
}
#endif
#if defined(HAS_INTERPOLATEROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
InterpolateRow = InterpolateRow_SME;
}
#endif
#if defined(HAS_INTERPOLATEROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
InterpolateRow = InterpolateRow_Any_LSX;
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_LSX;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(awidth, 32)) {
InterpolateRow = InterpolateRow_LASX;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
InterpolateRow = InterpolateRow_RVV;
}
#endif
#if defined(HAS_INTERPOLATEROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
InterpolateRow = InterpolateRow_Any_WASMSIMD;
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
align_buffer_64(row_uv, halfwidth * 2);
align_buffer_64(row_uv, awidth);
if (!row_uv) {
return 1;
}
for (y = 0; y < halfheight; ++y) {
if (y * 2 + 1 < height) {
HalfRow_C(src_uv, src_stride_uv, row_uv, halfwidth * 2);
// Average 2 rows of UV with rounding (source_y_fraction 128).
InterpolateRow(row_uv, src_uv, src_stride_uv, awidth, 128);
SplitUVRow(row_uv, dst_u, dst_v, halfwidth);
} else {
memcpy(row_uv, src_uv, (size_t)halfwidth * 2);
SplitUVRow(src_uv, dst_u, dst_v, halfwidth);
}
SplitUVRow(row_uv, dst_u, dst_v, halfwidth);
src_uv += (ptrdiff_t)src_stride_uv * 2;
dst_u += dst_stride_u;
dst_v += dst_stride_v;

12
source/cpu_id.cc
View File

@ -408,11 +408,21 @@ static SAFEBUFFERS int GetCpuFlags(void) {
int cpu_info24[4] = {0, 0, 0, 0};
int cpu_info21[4] = {0, 0, 0, 0};
int cpu_amdinfo21[4] = {0, 0, 0, 0};
int cpu_extinfo0[4] = {0, 0, 0, 0};
CpuId(0, 0, cpu_info0);
CpuId(1, 0, cpu_info1);
if (cpu_info0[0] >= 7) {
CpuId(7, 0, cpu_info7);
CpuId(7, 1, cpu_einfo7);
// Leaf 7 EAX reports the maximum supported sub-leaf.
if (cpu_info7[0] >= 1) {
CpuId(7, 1, cpu_einfo7);
}
}
// Extended leaf 0x80000021 is only valid if the CPU reports it. Querying an
// unsupported extended leaf returns data from the highest basic leaf on
// Intel CPUs, which could spuriously set bits below.
CpuId(0x80000000, 0, cpu_extinfo0);
if ((unsigned)cpu_extinfo0[0] >= 0x80000021u) {
CpuId(0x80000021, 0, cpu_amdinfo21);
}
if (cpu_info0[0] >= 0x21) {

35
source/mjpeg_decoder.cc
View File

@ -127,7 +127,11 @@ bool MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) {
AllocOutputBuffers(GetNumComponents());
for (int i = 0; i < num_outbufs_; ++i) {
int scanlines_size = GetComponentScanlinesPerImcuRow(i);
if (scanlines_sizes_[i] != scanlines_size) {
// databuf_[i] is sized by scanlines_size * stride, so a change in either
// must trigger reallocation below. Remember the change before
// scanlines_sizes_[i] is overwritten.
bool scanlines_changed = scanlines_sizes_[i] != scanlines_size;
if (scanlines_changed) {
free(scanlines_[i]);
scanlines_[i] =
static_cast<uint8_t**>(malloc(scanlines_size * sizeof(uint8_t*)));
@ -147,7 +151,7 @@ bool MJpegDecoder::LoadFrame(const uint8_t* src, size_t src_len) {
// DCTSIZE) is at most slightly larger than UINT16_MAX.
// - Sampling factor is stored in 4 bits, so scanlines_size is < 16.
int databuf_size = scanlines_size * databuf_stride;
if (databuf_strides_[i] != databuf_stride) {
if (scanlines_changed || databuf_strides_[i] != databuf_stride) {
free(databuf_[i]);
databuf_[i] = static_cast<uint8_t*>(malloc(databuf_size));
databuf_strides_[i] = databuf_stride;
@ -268,9 +272,11 @@ bool MJpegDecoder::DecodeToBuffers(uint8_t** planes,
SetScanlinePointers(databuf_);
int lines_left = dst_height;
// Compute amount of lines to skip to implement vertical crop.
// TODO(fbarchard): Ensure skip is a multiple of maximum component
// subsample. ie 2
// Round skip down to a multiple of the maximum vertical sampling factor so
// that every component skips a whole number of rows. The rows remaining
// below the crop are still >= dst_height.
int skip = (GetHeight() - dst_height) / 2;
skip -= skip % decompress_struct_->max_v_samp_factor;
if (skip > 0) {
// There is no API to skip lines in the output data, so we read them
// into the temp buffer.
@ -288,19 +294,23 @@ bool MJpegDecoder::DecodeToBuffers(uint8_t** planes,
FinishDecode();
return false;
}
// Image lines wanted from this iMCU row, clamped to the destination.
int image_lines_to_copy = GetImageScanlinesPerImcuRow() - skip;
if (image_lines_to_copy > lines_left) {
image_lines_to_copy = lines_left;
}
for (int i = 0; i < num_outbufs_; ++i) {
// TODO(fbarchard): Compute skip to avoid this
assert(skip % GetVertSubSampFactor(i) == 0);
int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i));
int scanlines_to_copy =
GetComponentScanlinesPerImcuRow(i) - rows_to_skip;
DivideAndRoundUp(image_lines_to_copy, GetVertSubSampFactor(i));
int data_to_skip = rows_to_skip * GetComponentStride(i);
CopyPlane(databuf_[i] + data_to_skip, GetComponentStride(i), planes[i],
GetComponentWidth(i), GetComponentWidth(i),
scanlines_to_copy);
planes[i] += scanlines_to_copy * GetComponentWidth(i);
}
lines_left -= (GetImageScanlinesPerImcuRow() - skip);
lines_left -= image_lines_to_copy;
}
}
@ -357,8 +367,12 @@ bool MJpegDecoder::DecodeToCallback(CallbackFunction fn,
}
SetScanlinePointers(databuf_);
int lines_left = dst_height;
// TODO(fbarchard): Compute amount of lines to skip to implement vertical crop
// Compute amount of lines to skip to implement vertical crop.
// Round skip down to a multiple of the maximum vertical sampling factor so
// that every component skips a whole number of rows. The rows remaining
// below the crop are still >= dst_height.
int skip = (GetHeight() - dst_height) / 2;
skip -= skip % decompress_struct_->max_v_samp_factor;
if (skip > 0) {
while (skip >= GetImageScanlinesPerImcuRow()) {
if (!DecodeImcuRow()) {
@ -374,7 +388,6 @@ bool MJpegDecoder::DecodeToCallback(CallbackFunction fn,
return false;
}
for (int i = 0; i < num_outbufs_; ++i) {
// TODO(fbarchard): Compute skip to avoid this
assert(skip % GetVertSubSampFactor(i) == 0);
int rows_to_skip = DivideAndRoundDown(skip, GetVertSubSampFactor(i));
int data_to_skip = rows_to_skip * GetComponentStride(i);
@ -382,7 +395,11 @@ bool MJpegDecoder::DecodeToCallback(CallbackFunction fn,
// callback.
databuf_[i] += data_to_skip;
}
// Image lines wanted from this iMCU row, clamped to the destination.
int scanlines_to_copy = GetImageScanlinesPerImcuRow() - skip;
if (scanlines_to_copy > lines_left) {
scanlines_to_copy = lines_left;
}
(*fn)(opaque, databuf_, databuf_strides_, scanlines_to_copy);
// Now change them back.
for (int i = 0; i < num_outbufs_; ++i) {

87
source/planar_functions.cc
View File

@ -1359,7 +1359,10 @@ void DetileSplitUVPlane(const uint8_t* src_uv,
assert(tile_height > 0);
assert(src_stride_uv > 0);
if (width <= 0 || height == 0 || height == INT_MIN) {
// tile_height must be a power of 2 (16 or 32); the row loop below uses it as
// a bit mask.
if (width <= 0 || height == 0 || height == INT_MIN || tile_height <= 0 ||
!IS_POWEROFTWO(tile_height)) {
return;
}
// Negative height means invert the image.
@ -1423,7 +1426,10 @@ void DetileToYUY2(const uint8_t* src_y,
assert(src_stride_uv > 0);
assert(tile_height > 0);
if (width <= 0 || height == 0 || height == INT_MIN || tile_height <= 0) {
// tile_height must be a power of 2 and at least 2 (16 or 32); the row loop
// uses it as a bit mask and the UV tile is tile_height / 2 rows.
if (width <= 0 || height == 0 || height == INT_MIN || tile_height < 2 ||
!IS_POWEROFTWO(tile_height)) {
return;
}
// Negative height means invert the image.
@ -1595,7 +1601,8 @@ void MergeRGBPlane(const uint8_t* src_r,
}
#endif
#if defined(HAS_MERGERGBROW_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
if (TestCpuFlag(kCpuHasAVX512VBMI | kCpuHasAVX512VL) ==
(kCpuHasAVX512VBMI | kCpuHasAVX512VL)) {
MergeRGBRow = MergeRGBRow_AVX512VBMI;
}
#endif
@ -1799,11 +1806,13 @@ void SplitARGBPlane(const uint8_t* src_argb,
dst_r = dst_r + (ptrdiff_t)(height - 1) * dst_stride_r;
dst_g = dst_g + (ptrdiff_t)(height - 1) * dst_stride_g;
dst_b = dst_b + (ptrdiff_t)(height - 1) * dst_stride_b;
dst_a = dst_a + (ptrdiff_t)(height - 1) * dst_stride_a;
dst_stride_r = -dst_stride_r;
dst_stride_g = -dst_stride_g;
dst_stride_b = -dst_stride_b;
dst_stride_a = -dst_stride_a;
if (dst_a) {
dst_a = dst_a + (ptrdiff_t)(height - 1) * dst_stride_a;
dst_stride_a = -dst_stride_a;
}
}
if (dst_a == NULL) {
@ -2030,7 +2039,8 @@ void MergeXR30Plane(const uint16_t* src_r,
}
#endif
#if defined(HAS_MERGEXR30ROW_10_AVX512BW)
if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
if (depth == 10 && TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
MergeXR30Row = MergeXR30Row_10_AVX512BW;
}
#endif
@ -3217,6 +3227,14 @@ int RGB24Mirror(const uint8_t* src_rgb24,
}
}
#endif
#if defined(HAS_RGB24MIRRORROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
RGB24MirrorRow = RGB24MirrorRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
RGB24MirrorRow = RGB24MirrorRow_SSSE3;
}
}
#endif
#if defined(HAS_RGB24MIRRORROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
RGB24MirrorRow = RGB24MirrorRow_Any_AVX2;
@ -4130,6 +4148,11 @@ int ARGBRect(uint8_t* dst_argb,
}
}
#endif
#if defined(HAS_ARGBSETROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBSetRow = ARGBSetRow_RVV;
}
#endif
// Set plane
for (y = 0; y < height; ++y) {
@ -5060,14 +5083,6 @@ int InterpolatePlane_16(const uint16_t* src0,
height = 1;
src_stride0 = src_stride1 = dst_stride = 0;
}
#if defined(HAS_INTERPOLATEROW_16_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
InterpolateRow_16 = InterpolateRow_16_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
InterpolateRow_16 = InterpolateRow_16_SSSE3;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_16_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow_16 = InterpolateRow_16_Any_AVX2;
@ -5094,14 +5109,6 @@ int InterpolatePlane_16(const uint16_t* src0,
InterpolateRow_16 = InterpolateRow_16_SME;
}
#endif
#if defined(HAS_INTERPOLATEROW_16_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
InterpolateRow_16 = InterpolateRow_16_Any_LSX;
if (IS_ALIGNED(width, 32)) {
InterpolateRow_16 = InterpolateRow_16_LSX;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
InterpolateRow_16 = InterpolateRow_16_Any_WASMSIMD;
@ -5259,6 +5266,11 @@ int ARGBShuffle(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBSHUFFLEROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBShuffleRow = ARGBShuffleRow_RVV;
}
#endif
for (y = 0; y < height; ++y) {
ARGBShuffleRow(src_argb, dst_argb, shuffler, width);
@ -5335,6 +5347,11 @@ int AR64Shuffle(const uint16_t* src_ar64,
}
}
#endif
#if defined(HAS_ARGBSHUFFLEROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
AR64ShuffleRow = ARGBShuffleRow_RVV;
}
#endif
for (y = 0; y < height; ++y) {
AR64ShuffleRow((uint8_t*)(src_ar64), (uint8_t*)(dst_ar64), shuffler,
@ -6276,6 +6293,9 @@ int UYVYToNV12(const uint8_t* src_uyvy,
int height) {
int y;
int halfwidth = (width + 1) >> 1;
// UYVY rows are processed as awidth (even) samples. Select kernels on the
// width actually passed to them.
int awidth = halfwidth * 2;
void (*SplitUVRow)(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v,
int width) = SplitUVRow_C;
void (*InterpolateRow)(uint8_t* dst_ptr, const uint8_t* src_ptr,
@ -6296,7 +6316,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
SplitUVRow = SplitUVRow_Any_SSE2;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
SplitUVRow = SplitUVRow_SSE2;
}
}
@ -6304,7 +6324,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
SplitUVRow = SplitUVRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
if (IS_ALIGNED(awidth, 32)) {
SplitUVRow = SplitUVRow_AVX2;
}
}
@ -6312,7 +6332,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
SplitUVRow = SplitUVRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
if (IS_ALIGNED(awidth, 64)) {
SplitUVRow = SplitUVRow_AVX512BW;
}
}
@ -6320,7 +6340,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
SplitUVRow = SplitUVRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
SplitUVRow = SplitUVRow_NEON;
}
}
@ -6333,7 +6353,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
SplitUVRow = SplitUVRow_LSX;
}
}
@ -6341,7 +6361,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_SPLITUVROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
SplitUVRow = SplitUVRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
if (IS_ALIGNED(awidth, 32)) {
SplitUVRow = SplitUVRow_LASX;
}
}
@ -6355,7 +6375,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
if (IS_ALIGNED(awidth, 32)) {
InterpolateRow = InterpolateRow_AVX2;
}
}
@ -6368,7 +6388,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_NEON;
}
}
@ -6386,7 +6406,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
InterpolateRow = InterpolateRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_LSX;
}
}
@ -6394,7 +6414,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
if (IS_ALIGNED(awidth, 32)) {
InterpolateRow = InterpolateRow_LASX;
}
}
@ -6407,14 +6427,13 @@ int UYVYToNV12(const uint8_t* src_uyvy,
#if defined(HAS_INTERPOLATEROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
InterpolateRow = InterpolateRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 16)) {
if (IS_ALIGNED(awidth, 16)) {
InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
{
int awidth = halfwidth * 2;
// row of y and 2 rows of uv
align_buffer_64(rows, awidth * 3);
if (!rows)

7
source/rotate.cc
View File

@ -927,6 +927,7 @@ int I422Rotate(const uint8_t* src_y,
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
@ -1234,9 +1235,10 @@ int I010Rotate(const uint16_t* src_y,
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
@ -1305,6 +1307,7 @@ int I210Rotate(const uint16_t* src_y,
// Negative height means invert the image.
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;

3
source/rotate_argb.cc
View File

@ -78,6 +78,9 @@ static int ARGBTranspose(const uint8_t* src_argb,
#if defined(HAS_TRANSPOSEWX4_32_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
TransposeWx4_32 = TransposeWx4_32_SVE2;
#if defined(HAS_TRANSPOSEWXH_32_NEON)
TransposeWxH_32 = NULL;
#endif
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_SME)

12
source/row_any.cc
View File

@ -385,9 +385,6 @@ ANY31(BlendPlaneRow_Any_WASMSIMD, BlendPlaneRow_WASMSIMD, 0, 0, 1, 15)
#ifdef HAS_I422TOARGBROW_SSSE3
ANY31C(I422ToARGBRow_Any_SSSE3, I422ToARGBRow_SSSE3, 1, 0, 4, 7)
#endif
#ifdef HAS_I422TORGBAROW_SSSE3
ANY31C(I422ToRGBARow_Any_SSSE3, I422ToRGBARow_SSSE3, 1, 0, 4, 7)
#endif
#ifdef HAS_I422TOARGB4444ROW_SSSE3
ANY31C(I422ToARGB4444Row_Any_SSSE3, I422ToARGB4444Row_SSSE3, 1, 0, 2, 7)
#endif
@ -1575,7 +1572,7 @@ ANY11T(ARGBToAB64Row_Any_SSSE3, ARGBToAB64Row_SSSE3, 4, 8, uint8_t, uint16_t, 3)
ANY11T(AR64ToARGBRow_Any_SSSE3, AR64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3)
#endif
#ifdef HAS_ARGBTOAR64ROW_SSSE3
#ifdef HAS_AB64TOARGBROW_SSSE3
ANY11T(AB64ToARGBRow_Any_SSSE3, AB64ToARGBRow_SSSE3, 8, 4, uint16_t, uint8_t, 3)
#endif
@ -1591,7 +1588,7 @@ ANY11T(ARGBToAB64Row_Any_AVX2, ARGBToAB64Row_AVX2, 4, 8, uint8_t, uint16_t, 7)
ANY11T(AR64ToARGBRow_Any_AVX2, AR64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7)
#endif
#ifdef HAS_ARGBTOAR64ROW_AVX2
#ifdef HAS_AB64TOARGBROW_AVX2
ANY11T(AB64ToARGBRow_Any_AVX2, AB64ToARGBRow_AVX2, 8, 4, uint16_t, uint8_t, 7)
#endif
@ -1607,7 +1604,7 @@ ANY11T(ARGBToAB64Row_Any_NEON, ARGBToAB64Row_NEON, 4, 8, uint8_t, uint16_t, 7)
ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
#endif
#ifdef HAS_ARGBTOAR64ROW_NEON
#ifdef HAS_AB64TOARGBROW_NEON
ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
#endif
@ -2140,6 +2137,9 @@ ANY11M(ARGBMirrorRow_Any_LASX, ARGBMirrorRow_LASX, 4, 7)
#ifdef HAS_RGB24MIRRORROW_AVX2
ANY11M(RGB24MirrorRow_Any_AVX2, RGB24MirrorRow_AVX2, 3, 31)
#endif
#ifdef HAS_RGB24MIRRORROW_SSSE3
ANY11M(RGB24MirrorRow_Any_SSSE3, RGB24MirrorRow_SSSE3, 3, 15)
#endif
#ifdef HAS_RGB24MIRRORROW_NEON
ANY11M(RGB24MirrorRow_Any_NEON, RGB24MirrorRow_NEON, 3, 15)
#endif

32
source/row_common.cc
View File

@ -583,6 +583,8 @@ void AR64ToAB64Row_C(const uint16_t* src_ar64, uint16_t* dst_ab64, int width) {
}
// TODO(fbarchard): Make shuffle compatible with SIMD versions
// width is in units of 32 bit ARGB pixels (2 per AR64 pixel) so that
// AR64Shuffle can share the ARGBShuffleRow SIMD kernels.
void AR64ShuffleRow_C(const uint8_t* src_ar64,
uint8_t* dst_ar64,
const uint8_t* shuffler,
@ -2938,8 +2940,10 @@ void DetileToYUY2_C(const uint8_t* src_y,
ptrdiff_t src_uv_tile_stride,
uint8_t* dst_yuy2,
int width) {
for (int x = 0; x < width - 15; x += 16) {
for (int i = 0; i < 8; i++) {
int x;
for (x = 0; x < width - 15; x += 16) {
int i;
for (i = 0; i < 8; i++) {
dst_yuy2[0] = src_y[0];
dst_yuy2[1] = src_uv[0];
dst_yuy2[2] = src_y[1];
@ -2951,6 +2955,16 @@ void DetileToYUY2_C(const uint8_t* src_y,
src_y += src_y_tile_stride - 16;
src_uv += src_uv_tile_stride - 16;
}
// Remaining pixels of a partial 16 pixel tile.
for (x = 0; x < (width & 15); x += 2) {
dst_yuy2[0] = src_y[0];
dst_yuy2[1] = src_uv[0];
dst_yuy2[2] = src_y[1];
dst_yuy2[3] = src_uv[1];
dst_yuy2 += 4;
src_y += 2;
src_uv += 2;
}
}
// Unpack MT2T into tiled P010 64 pixels at a time. MT2T's bitstream is encoded
@ -3214,8 +3228,10 @@ void DivideRow_16_C(const uint16_t* src_y,
int scale,
int width) {
int x;
// scale can be 65536 so multiply unsigned to avoid signed overflow.
for (x = 0; x < width; ++x) {
dst_y[x] = (src_y[x] * scale) >> 16;
dst_y[x] =
STATIC_CAST(uint16_t, ((uint32_t)src_y[x] * (uint32_t)scale) >> 16);
}
}
@ -3943,9 +3959,15 @@ void HalfFloatRow_C(const uint16_t* src,
float scale,
int width) {
int i;
float mult = 1.9259299444e-34f * scale;
// Multiply by scale first, then by 2^-112 to move the exponent into half
// float position. Doing the scale and bias as two multiplies keeps every
// intermediate normal; a combined (scale * 2^-112) constant is a denormal
// for scale < 2^-14 (e.g. 1/65535), which costs a microcode assist per
// multiply on Intel CPUs (50x slower). 2^-112 is a power of 2 so the
// result is identical to the single multiply when that was representable.
const float kScaleBias = 1.9259299444e-34f; // 2^-112
for (i = 0; i < width; ++i) {
float value = src[i] * mult;
float value = (src[i] * scale) * kScaleBias;
dst[i] = (uint16_t)((*(const uint32_alias_t*)&value) >> 13);
}
}

31
source/row_gcc.cc
View File

@ -7056,6 +7056,7 @@ void SplitRGBRow_AVX2(const uint8_t* src_rgb,
"lea 0x20(%3),%3 \n"
"sub $0x20,%4 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_rgb), // %0
"+r"(dst_r), // %1
"+r"(dst_g), // %2
@ -8223,7 +8224,8 @@ void ARGBCopyYToAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
"movq (%0),%%xmm2 \n"
"lea 0x8(%0),%0 \n"
"punpcklbw %%xmm2,%%xmm2 \n"
"punpckhwd %%xmm2,%%xmm3 \n"
"movdqa %%xmm2,%%xmm3 \n"
"punpckhwd %%xmm3,%%xmm3 \n"
"punpcklwd %%xmm2,%%xmm2 \n"
"movdqu (%1),%%xmm4 \n"
"movdqu 0x10(%1),%%xmm5 \n"
@ -11102,7 +11104,7 @@ void ARGBPolynomialRow_SSE2(const uint8_t* src_argb,
"addps %%xmm5,%%xmm4 \n"
"cvttps2dq %%xmm0,%%xmm0 \n"
"cvttps2dq %%xmm4,%%xmm4 \n"
"packuswb %%xmm4,%%xmm0 \n"
"packssdw %%xmm4,%%xmm0 \n"
"packuswb %%xmm0,%%xmm0 \n"
"movq %%xmm0,(%1) \n"
"lea 0x8(%1),%1 \n"
@ -11159,14 +11161,17 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
#endif // HAS_ARGBPOLYNOMIALROW_AVX2
#ifdef HAS_HALFFLOATROW_AVX2
static float kScaleBias = 1.9259299444e-34f;
// 2^-112 moves the float exponent into half float position. Applied as a
// separate multiply after scale so no denormal constant is formed when
// scale < 2^-14 (see HalfFloatRow_C).
static const float kScaleBias = 1.9259299444e-34f;
void HalfFloatRow_AVX2(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
scale *= kScaleBias;
asm volatile(
"vbroadcastss %3, %%ymm4 \n"
"vbroadcastss %4, %%ymm6 \n"
"vpxor %%ymm5,%%ymm5,%%ymm5 \n"
"sub %0,%1 \n"
@ -11179,8 +11184,10 @@ void HalfFloatRow_AVX2(const uint16_t* src,
"vpunpcklwd %%ymm5,%%ymm2,%%ymm2 \n"
"vcvtdq2ps %%ymm3,%%ymm3 \n"
"vcvtdq2ps %%ymm2,%%ymm2 \n"
"vmulps %%ymm3,%%ymm4,%%ymm3 \n"
"vmulps %%ymm3,%%ymm4,%%ymm3 \n" // * scale
"vmulps %%ymm2,%%ymm4,%%ymm2 \n"
"vmulps %%ymm3,%%ymm6,%%ymm3 \n" // * 2^-112
"vmulps %%ymm2,%%ymm6,%%ymm2 \n"
"vpsrld $0xd,%%ymm3,%%ymm3 \n"
"vpsrld $0xd,%%ymm2,%%ymm2 \n"
"vpackssdw %%ymm3, %%ymm2, %%ymm2 \n" // unmutates
@ -11193,11 +11200,13 @@ void HalfFloatRow_AVX2(const uint16_t* src,
"+r"(dst), // %1
"+r"(width) // %2
#if defined(__x86_64__)
: "x"(scale) // %3
: "x"(scale), // %3
"x"(kScaleBias) // %4
#else
: "m"(scale) // %3
: "m"(scale), // %3
"m"(kScaleBias) // %4
#endif
: "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5");
: "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
#endif // HAS_HALFFLOATROW_AVX2
@ -11716,10 +11725,10 @@ void ClampFloatToZero_SSE2(const float* src_x, float* dst_y, int width) {
"1: \n"
"movd (%0),%%xmm0 \n" // load float
"maxss %%xmm1, %%xmm0 \n" // clamp to zero
"add 4, %0 \n"
"lea 0x4(%0),%0 \n"
"movd %%xmm0, (%1) \n" // store float
"add 4, %1 \n"
"sub $0x4,%2 \n" // 1 float per loop
"lea 0x4(%1),%1 \n"
"sub $0x1,%2 \n" // 1 float per loop
"jg 1b \n"
: "+r"(src_x), // %0
"+r"(dst_y), // %1

11
source/row_lasx.cc
View File

@ -2341,9 +2341,10 @@ void HalfFloatRow_LASX(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
float mult = 1.9259299444e-34f * scale;
const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C
asm volatile(
"xvldrepl.w $xr3, %[mult], 0\n\t"
"xvldrepl.w $xr3, %[scale], 0\n\t"
"xvldrepl.w $xr5, %[bias], 0\n\t"
"xvldi $xr4, 0\n\t"
"1:\n\t"
"xvld $xr0, %[src], 0\n\t"
@ -2353,6 +2354,8 @@ void HalfFloatRow_LASX(const uint16_t* src,
"xvffint.s.wu $xr2, $xr2\n\t"
"xvfmul.s $xr1, $xr1, $xr3\n\t"
"xvfmul.s $xr2, $xr2, $xr3\n\t"
"xvfmul.s $xr1, $xr1, $xr5\n\t"
"xvfmul.s $xr2, $xr2, $xr5\n\t"
"xvsrlni.h.w $xr2, $xr1, 13\n\t"
"xvst $xr2, %[dst], 0\n\t"
"addi.d %[src], %[src], 32\n\t"
@ -2360,8 +2363,8 @@ void HalfFloatRow_LASX(const uint16_t* src,
"addi.d %[cnt], %[cnt], -16\n\t"
"bnez %[cnt], 1b\n\t"
: [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width)
: [mult] "r"(&mult)
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4");
: [scale] "r"(&scale), [bias] "r"(&bias)
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5");
}
void ARGBToYMatrixRow_LASX(const uint8_t* src_argb,

11
source/row_lsx.cc
View File

@ -2022,9 +2022,10 @@ void HalfFloatRow_LSX(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
float mult = 1.9259299444e-34f * scale;
const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C
asm volatile(
"vldrepl.w $vr3, %[mult], 0\n\t"
"vldrepl.w $vr3, %[scale], 0\n\t"
"vldrepl.w $vr5, %[bias], 0\n\t"
"vldi $vr4, 0\n\t"
"1:\n\t"
"vld $vr0, %[src], 0\n\t"
@ -2034,6 +2035,8 @@ void HalfFloatRow_LSX(const uint16_t* src,
"vffint.s.wu $vr2, $vr2\n\t"
"vfmul.s $vr1, $vr1, $vr3\n\t"
"vfmul.s $vr2, $vr2, $vr3\n\t"
"vfmul.s $vr1, $vr1, $vr5\n\t"
"vfmul.s $vr2, $vr2, $vr5\n\t"
"vsrlni.h.w $vr2, $vr1, 13\n\t"
"vst $vr2, %[dst], 0\n\t"
"addi.d %[src], %[src], 16\n\t"
@ -2041,8 +2044,8 @@ void HalfFloatRow_LSX(const uint16_t* src,
"addi.d %[cnt], %[cnt], -8\n\t"
"bnez %[cnt], 1b\n\t"
: [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width)
: [mult] "r"(&mult)
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4");
: [scale] "r"(&scale), [bias] "r"(&bias)
: "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5");
}
void ARGBToYMatrixRow_LSX(const uint8_t* src_argb,

15
source/row_neon.cc
View File

@ -3369,20 +3369,25 @@ void HalfFloatRow_NEON(const uint16_t* src,
"vcvt.f32.u32 q9, q9 \n"
"vcvt.f32.u32 q10, q10 \n"
"vcvt.f32.u32 q11, q11 \n"
"vmul.f32 q8, q8, %y3 \n" // adjust exponent
"vmul.f32 q8, q8, %y3 \n" // * scale
"vmul.f32 q9, q9, %y3 \n"
"vmul.f32 q10, q10, %y3 \n"
"vmul.f32 q11, q11, %y3 \n"
"vmul.f32 q8, q8, %y4 \n" // adjust exponent
"vmul.f32 q9, q9, %y4 \n"
"vmul.f32 q10, q10, %y4 \n"
"vmul.f32 q11, q11, %y4 \n"
"vqshrn.u32 d0, q8, #13 \n" // isolate halffloat
"vqshrn.u32 d1, q9, #13 \n"
"vqshrn.u32 d2, q10, #13 \n"
"vqshrn.u32 d3, q11, #13 \n"
"vst1.16 {q0, q1}, [%1]! \n" // store 16 fp16
"bgt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale * 1.9259299444e-34f) // %3
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale), // %3
"w"(1.9259299444e-34f) // %4 2^-112, see HalfFloatRow_C
: "cc", "memory", "q0", "q1", "q8", "q9", "q10", "q11");
}

28
source/row_neon64.cc
View File

@ -4767,20 +4767,25 @@ void HalfFloatRow_NEON(const uint16_t* src,
"scvtf v4.4s, v4.4s \n"
"scvtf v3.4s, v3.4s \n"
"scvtf v5.4s, v5.4s \n"
"fmul v2.4s, v2.4s, %3.s[0] \n" // adjust exponent
"fmul v2.4s, v2.4s, %3.s[0] \n" // * scale
"fmul v4.4s, v4.4s, %3.s[0] \n"
"fmul v3.4s, v3.4s, %3.s[0] \n"
"fmul v5.4s, v5.4s, %3.s[0] \n"
"fmul v2.4s, v2.4s, %4.s[0] \n" // adjust exponent
"fmul v4.4s, v4.4s, %4.s[0] \n"
"fmul v3.4s, v3.4s, %4.s[0] \n"
"fmul v5.4s, v5.4s, %4.s[0] \n"
"uqshrn v0.4h, v2.4s, #13 \n" // isolate halffloat
"uqshrn v1.4h, v4.4s, #13 \n"
"uqshrn2 v0.8h, v3.4s, #13 \n"
"uqshrn2 v1.8h, v5.4s, #13 \n"
"stp q0, q1, [%1], #32 \n" // store 16 fp16
"b.gt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale * 1.9259299444e-34f) // %3
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale), // %3
"w"(1.9259299444e-34f) // %4 2^-112, see HalfFloatRow_C
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
}
@ -4836,8 +4841,8 @@ void ConvertFP16ToFP32Column_NEON(const uint16_t* src, // fp16
float* dst,
int width) {
asm volatile(
"cmp %w2, #8 \n" // Is there 8 rows?
"b.lo 2f \n"
"subs %w2, %w2, #8 \n" // Is there 8 rows?
"b.lt 2f \n"
"1: \n"
"ld1 {v0.h}[0], [%0], %3 \n" // load 8 halffloats
"ld1 {v0.h}[1], [%0], %3 \n"
@ -4852,15 +4857,16 @@ void ConvertFP16ToFP32Column_NEON(const uint16_t* src, // fp16
"fcvtl v2.4s, v0.4h \n" // 4 floats
"fcvtl v3.4s, v1.4h \n" // 4 more floats
"stp q2, q3, [%1], #32 \n" // store 8 floats
"b.gt 1b \n"
"cmp %w2, #1 \n" // Is there 1 value?
"b.lo 3f \n"
"b.ge 1b \n"
"2: \n"
"adds %w2, %w2, #8 \n" // Restore remainder.
"b.eq 3f \n"
"4: \n"
"ld1 {v1.h}[0], [%0], %3 \n" // load 1 halffloats
"subs %w2, %w2, #1 \n" // 1 floats per loop
"fcvtl v2.4s, v1.4h \n" // 1 floats
"str s2, [%1], #4 \n" // store 1 floats
"b.gt 2b \n"
"b.gt 4b \n"
"3: \n"
: "+r"(src), // %0
"+r"(dst), // %1

389
source/row_rvv.cc
View File

@ -29,17 +29,12 @@
#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
#include <assert.h>
// vsetvli/vsetivli write the vl and vtype CSRs. Only "vl" is declared, and
// only for GCC; clang historically rejected it. The "vtype" register is
// intentionally not declared, and whether it should be is unresolved.
// TODO(fbarchard): consider adding "vtype" (and "vl" for clang) to the
// clobber list once confirmed with compiler maintainers.
// vsetvli/vsetivli write the vl and vtype CSRs, so every asm block that
// executes them declares both as clobbers. Both GCC and clang (17+) accept
// "vl" and "vtype" as clobber names. Blocks that write the fixed-point
// rounding mode with "csrwi vxrm" additionally declare "vxrm".
#ifndef RVV_VL_CLOBBER
#ifdef __clang__
#define RVV_VL_CLOBBER
#else
#define RVV_VL_CLOBBER "vl",
#endif
#define RVV_VL_CLOBBER "vl", "vtype",
#endif
#ifdef __cplusplus
@ -187,7 +182,7 @@ void ARGBToAR64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ar64, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [k0101] "r"(0x0101)
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v16", "v17");
}
#endif
@ -219,7 +214,7 @@ void ARGBToAB64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ab64, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [k0101] "r"(0x0101)
: RVV_VL_CLOBBER "memory", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
: "memory", RVV_VL_CLOBBER "v4", "v5", "v6", "v7", "v8", "v9", "v10",
"v11", "v12", "v13", "v14", "v15");
}
#endif
@ -245,7 +240,7 @@ void AR64ToARGBRow_RVV(const uint16_t* src_ar64, uint8_t* dst_argb, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v16", "v17");
}
#endif
@ -273,7 +268,7 @@ void AR64ToAB64Row_RVV(const uint16_t* src_ar64,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
#endif
@ -301,7 +296,7 @@ void AB64ToARGBRow_RVV(const uint16_t* src_ab64, uint8_t* dst_argb, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
: "memory", RVV_VL_CLOBBER "v4", "v5", "v6", "v7", "v8", "v9", "v10",
"v11", "v12", "v13", "v14", "v15");
}
#endif
@ -333,7 +328,7 @@ void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19", "v20", "v21");
}
#endif
@ -365,7 +360,7 @@ void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19", "v20", "v21");
}
#endif
@ -393,7 +388,7 @@ void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v16",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v16",
"v17", "v18", "v19", "v20", "v21");
}
#endif
@ -422,7 +417,7 @@ void ARGBToRAWRow_RVV(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19", "v20", "v21");
}
#endif
@ -450,7 +445,7 @@ void ARGBToRGB24Row_RVV(const uint8_t* src_argb,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -477,7 +472,7 @@ void ARGBToABGRRow_RVV(const uint8_t* src_argb, uint8_t* dst_abgr, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
#endif
@ -508,7 +503,7 @@ void ARGBToBGRARow_RVV(const uint8_t* src_argb, uint8_t* dst_bgra, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -534,7 +529,7 @@ void ARGBToRGBARow_RVV(const uint8_t* src_argb, uint8_t* dst_rgba, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -560,7 +555,7 @@ void RGBAToARGBRow_RVV(const uint8_t* src_rgba, uint8_t* dst_argb, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -591,7 +586,7 @@ void RGB24ToARGBRow_RVV(const uint8_t* src_rgb24,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -640,7 +635,7 @@ void I444ToARGBRow_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS, "v30", "v31");
}
#endif
@ -690,7 +685,7 @@ void I444AlphaToARGBRow_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS, "v30", "v31");
}
#endif
@ -737,7 +732,7 @@ void I444ToRGB24Row_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS);
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS);
}
#endif
@ -748,7 +743,7 @@ void I422ToARGBRow_RVV(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
size_t vl;
size_t vl, uv_vl;
size_t ub, vr, ug, vg;
size_t yg, bb, bg, br;
size_t k0101;
@ -764,12 +759,11 @@ void I422ToARGBRow_RVV(const uint8_t* src_y,
RGBTORGB8 //
"vsseg4e8.v v24, (%[dst_argb]) \n"
"add %[src_y], %[src_y], %[vl] \n"
"srli %[k0101], %[vl], 1 \n"
"add %[src_u], %[src_u], %[k0101] \n"
"add %[src_v], %[src_v], %[k0101] \n"
"srli %[uv_vl], %[vl], 1 \n"
"add %[src_u], %[src_u], %[uv_vl] \n"
"add %[src_v], %[src_v], %[uv_vl] \n"
"slli %[vl], %[vl], 2 \n"
"add %[dst_argb], %[dst_argb], %[vl]\n"
"li %[k0101], 0x0101 \n"
"bgtz %[w], 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
@ -777,6 +771,7 @@ void I422ToARGBRow_RVV(const uint8_t* src_y,
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl), // %[vl]
[uv_vl] "=&r"(uv_vl), // %[uv_vl]
[ub] "=&r"(ub), // %[ub]
[vr] "=&r"(vr), // %[vr]
[ug] "=&r"(ug), // %[ug]
@ -787,7 +782,7 @@ void I422ToARGBRow_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS, "v30", "v31");
}
#endif
@ -799,7 +794,7 @@ void I422AlphaToARGBRow_RVV(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
size_t vl;
size_t vl, uv_vl;
size_t ub, vr, ug, vg;
size_t yg, bb, bg, br;
size_t k0101;
@ -815,12 +810,11 @@ void I422AlphaToARGBRow_RVV(const uint8_t* src_y,
"sub %[w], %[w], %[vl] \n"
"add %[src_y], %[src_y], %[vl] \n"
"add %[src_a], %[src_a], %[vl] \n"
"srli %[k0101], %[vl], 1 \n"
"add %[src_u], %[src_u], %[k0101] \n"
"add %[src_v], %[src_v], %[k0101] \n"
"srli %[uv_vl], %[vl], 1 \n"
"add %[src_u], %[src_u], %[uv_vl] \n"
"add %[src_v], %[src_v], %[uv_vl] \n"
"slli %[vl], %[vl], 2 \n"
"add %[dst_argb], %[dst_argb], %[vl]\n"
"li %[k0101], 0x0101 \n"
"bgtz %[w], 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
@ -829,6 +823,7 @@ void I422AlphaToARGBRow_RVV(const uint8_t* src_y,
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl), // %[vl]
[uv_vl] "=&r"(uv_vl), // %[uv_vl]
[ub] "=&r"(ub), // %[ub]
[vr] "=&r"(vr), // %[vr]
[ug] "=&r"(ug), // %[ug]
@ -839,7 +834,7 @@ void I422AlphaToARGBRow_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS, "v30", "v31");
}
#endif
@ -850,7 +845,7 @@ void I422ToRGBARow_RVV(const uint8_t* src_y,
uint8_t* dst_rgba,
const struct YuvConstants* yuvconstants,
int width) {
size_t vl;
size_t vl, uv_vl;
size_t ub, vr, ug, vg;
size_t yg, bb, bg, br;
size_t k0101;
@ -865,12 +860,11 @@ void I422ToRGBARow_RVV(const uint8_t* src_y,
"vmv.v.i v22, -1 \n"
"vsseg4e8.v v22, (%[dst_rgba]) \n"
"add %[src_y], %[src_y], %[vl] \n"
"srli %[k0101], %[vl], 1 \n"
"add %[src_u], %[src_u], %[k0101] \n"
"add %[src_v], %[src_v], %[k0101] \n"
"srli %[uv_vl], %[vl], 1 \n"
"add %[src_u], %[src_u], %[uv_vl] \n"
"add %[src_v], %[src_v], %[uv_vl] \n"
"slli %[vl], %[vl], 2 \n"
"add %[dst_rgba], %[dst_rgba], %[vl]\n"
"li %[k0101], 0x0101 \n"
"bgtz %[w], 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
@ -878,6 +872,7 @@ void I422ToRGBARow_RVV(const uint8_t* src_y,
[dst_rgba] "+r"(dst_rgba), // %[dst_rgba]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl), // %[vl]
[uv_vl] "=&r"(uv_vl), // %[uv_vl]
[ub] "=&r"(ub), // %[ub]
[vr] "=&r"(vr), // %[vr]
[ug] "=&r"(ug), // %[ug]
@ -888,7 +883,7 @@ void I422ToRGBARow_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v22", "v23");
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS, "v22", "v23");
}
#endif
@ -899,7 +894,7 @@ void I422ToRGB24Row_RVV(const uint8_t* src_y,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
size_t vl;
size_t vl, uv_vl;
size_t ub, vr, ug, vg;
size_t yg, bb, bg, br;
size_t k0101;
@ -913,13 +908,12 @@ void I422ToRGB24Row_RVV(const uint8_t* src_y,
RGBTORGB8 //
"vsseg3e8.v v24, (%[dst_rgb24]) \n"
"add %[src_y], %[src_y], %[vl] \n"
"srli %[k0101], %[vl], 1 \n"
"add %[src_u], %[src_u], %[k0101] \n"
"add %[src_v], %[src_v], %[k0101] \n"
"srli %[uv_vl], %[vl], 1 \n"
"add %[src_u], %[src_u], %[uv_vl] \n"
"add %[src_v], %[src_v], %[uv_vl] \n"
"add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
"slli %[vl], %[vl], 1 \n"
"add %[dst_rgb24], %[dst_rgb24], %[vl]\n"
"li %[k0101], 0x0101 \n"
"bgtz %[w], 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
@ -927,6 +921,7 @@ void I422ToRGB24Row_RVV(const uint8_t* src_y,
[dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl), // %[vl]
[uv_vl] "=&r"(uv_vl), // %[uv_vl]
[ub] "=&r"(ub), // %[ub]
[vr] "=&r"(vr), // %[vr]
[ug] "=&r"(ug), // %[ug]
@ -937,7 +932,7 @@ void I422ToRGB24Row_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS);
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS);
}
#endif
@ -948,7 +943,7 @@ void I422ToAR30Row_RVV(const uint8_t* src_y,
uint8_t* dst_ar30,
const struct YuvConstants* yuvconstants,
int width) {
size_t vl;
size_t vl, tmp;
size_t ub, vr, ug, vg;
size_t yg, bb, bg, br;
size_t k0101;
@ -986,15 +981,15 @@ void I422ToAR30Row_RVV(const uint8_t* src_y,
"vsrl.vi v0, v0, 4 \n"
"vsrl.vi v12, v12, 4 \n"
"vsrl.vi v8, v8, 4 \n"
"li %[k0101], 1023 \n"
"vminu.vx v0, v0, %[k0101] \n"
"vminu.vx v12, v12, %[k0101] \n"
"vminu.vx v8, v8, %[k0101] \n"
"li %[tmp], 1023 \n"
"vminu.vx v0, v0, %[tmp] \n"
"vminu.vx v12, v12, %[tmp] \n"
"vminu.vx v8, v8, %[tmp] \n"
"vwcvtu.x.x.v v16, v0 \n"
"li %[k0101], 1024 \n"
"vwmaccu.vx v16, %[k0101], v12 \n"
"li %[k0101], 0x0c00 \n"
"vor.vx v8, v8, %[k0101] \n"
"li %[tmp], 1024 \n"
"vwmaccu.vx v16, %[tmp], v12 \n"
"li %[tmp], 0x0c00 \n"
"vor.vx v8, v8, %[tmp] \n"
"vwcvtu.x.x.v v20, v8 \n"
"vsetvli zero, zero, e32, m4, ta, ma \n"
"vsll.vi v20, v20, 20 \n"
@ -1002,12 +997,11 @@ void I422ToAR30Row_RVV(const uint8_t* src_y,
"vse32.v v16, (%[dst_ar30]) \n"
"sub %[w], %[w], %[vl] \n"
"add %[src_y], %[src_y], %[vl] \n"
"srli %[k0101], %[vl], 1 \n"
"add %[src_u], %[src_u], %[k0101] \n"
"add %[src_v], %[src_v], %[k0101] \n"
"srli %[tmp], %[vl], 1 \n"
"add %[src_u], %[src_u], %[tmp] \n"
"add %[src_v], %[src_v], %[tmp] \n"
"slli %[vl], %[vl], 2 \n"
"add %[dst_ar30], %[dst_ar30], %[vl]\n"
"li %[k0101], 0x0101 \n"
"bgtz %[w], 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
@ -1015,6 +1009,7 @@ void I422ToAR30Row_RVV(const uint8_t* src_y,
[dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp), // %[tmp]
[ub] "=&r"(ub), // %[ub]
[vr] "=&r"(vr), // %[vr]
[ug] "=&r"(ug), // %[ug]
@ -1025,8 +1020,8 @@ void I422ToAR30Row_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v4", "v8", "v9", "v12",
"v13", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
: "memory", RVV_VL_CLOBBER "vxrm", "v0", "v1", "v2", "v4", "v8", "v9",
"v12", "v13", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
#endif
@ -1077,9 +1072,9 @@ void I400ToARGBRow_RVV(const uint8_t* src_y,
[k0101] "=&r"(k0101) // %[k0101]
: [yg] "r"(yg), // %[yg]
[yb] "r"(yb) // %[yb]
: RVV_VL_CLOBBER "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12",
"v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27");
: "memory", RVV_VL_CLOBBER "vxrm", "v0", "v1", "v8", "v9", "v10", "v11",
"v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27");
} else {
asm volatile(
"csrwi vxrm, 0 \n"
@ -1115,9 +1110,9 @@ void I400ToARGBRow_RVV(const uint8_t* src_y,
[k0101] "=&r"(k0101) // %[k0101]
: [yg] "r"(yg), // %[yg]
[yb] "r"(yb) // %[yb]
: RVV_VL_CLOBBER "memory", "v0", "v1", "v8", "v9", "v10", "v11", "v12",
"v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27");
: "memory", RVV_VL_CLOBBER "vxrm", "v0", "v1", "v8", "v9", "v10", "v11",
"v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27");
}
}
#endif
@ -1145,7 +1140,7 @@ void J400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -1167,7 +1162,7 @@ void CopyRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11");
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11");
}
#endif
@ -1212,7 +1207,7 @@ void NV12ToARGBRow_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS, "v30", "v31");
}
#endif
@ -1256,7 +1251,7 @@ void NV12ToRGB24Row_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS);
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS);
}
#endif
@ -1301,7 +1296,7 @@ void NV21ToARGBRow_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS, "v30", "v31");
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS, "v30", "v31");
}
#endif
@ -1345,7 +1340,7 @@ void NV21ToRGB24Row_RVV(const uint8_t* src_y,
[br] "=&r"(br), // %[br]
[k0101] "=&r"(k0101) // %[k0101]
: [yuvconst] "r"(yuvconstants) // %[yuvconst]
: RVV_VL_CLOBBER "memory", YUVTORGB_REGS);
: "memory", RVV_VL_CLOBBER "vxrm", YUVTORGB_REGS);
}
#endif
@ -1379,7 +1374,7 @@ void InterpolateRow_RVV(uint8_t* dst_ptr,
[dst_w] "+r"(dst_w), // %[dst_w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11");
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11");
return;
}
// Blend 50 / 50.
@ -1404,8 +1399,8 @@ void InterpolateRow_RVV(uint8_t* dst_ptr,
[dst_w] "+r"(dst_w), // %[dst_w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13",
"v14", "v15");
: "memory", RVV_VL_CLOBBER "vxrm", "v8", "v9", "v10", "v11", "v12",
"v13", "v14", "v15");
return;
}
// General purpose row blend.
@ -1435,7 +1430,7 @@ void InterpolateRow_RVV(uint8_t* dst_ptr,
: [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
[y1_fraction] "r"(y1_fraction), // %[y1_fraction]
[c128] "r"(128) // %[c128]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17");
}
#endif
@ -1470,7 +1465,7 @@ void SplitRGBRow_RVV(const uint8_t* src_rgb,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13");
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13");
}
#endif
@ -1504,7 +1499,7 @@ void MergeRGBRow_RVV(const uint8_t* src_r,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13");
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13");
}
#endif
@ -1540,7 +1535,7 @@ void SplitARGBRow_RVV(const uint8_t* src_argb,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -1577,7 +1572,7 @@ void MergeARGBRow_RVV(const uint8_t* src_r,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -1610,7 +1605,7 @@ void SplitXRGBRow_RVV(const uint8_t* src_argb,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -1646,7 +1641,7 @@ void MergeXRGBRow_RVV(const uint8_t* src_r,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -1675,7 +1670,7 @@ void SplitUVRow_RVV(const uint8_t* src_uv,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -1704,7 +1699,7 @@ void MergeUVRow_RVV(const uint8_t* src_u,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -1731,7 +1726,7 @@ void SwapUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -1781,7 +1776,7 @@ void ARGBToYMatrixRow_RVV(const uint8_t* src_argb,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
: [c] "r"(c) // %[c]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
"v25", "v26", "v27", "v28", "v29", "v30", "v31");
}
@ -1829,7 +1824,7 @@ void RGBToYMatrixRow_RVV(const uint8_t* src_rgb,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
: [c] "r"(c) // %[c]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
"v25", "v26", "v27", "v28", "v29");
}
@ -1899,7 +1894,7 @@ void ARGBToUV444MatrixRow_RVV(const uint8_t* src_argb,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
: [c] "r"(c) // %[c]
: RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
: "memory", RVV_VL_CLOBBER "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",
"v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27",
"v28", "v29", "v30", "v31");
@ -1949,48 +1944,40 @@ void ARGBToUVMatrixRow_RVV(const uint8_t* src_argb,
"vwaddu.vv v18, v20, v24 \n"
"vwaddu.wv v18, v18, v0 \n"
"vwaddu.wv v18, v18, v4 \n"
"vwaddu.vv v28, v21, v25 \n"
"vwaddu.wv v28, v28, v1 \n"
"vwaddu.wv v28, v28, v5 \n"
"vwaddu.vv v30, v22, v26 \n"
"vwaddu.wv v30, v30, v2 \n"
"vwaddu.wv v30, v30, v6 \n"
"vwaddu.vv v4, v23, v27 \n"
"vwaddu.wv v4, v4, v3 \n"
"vwaddu.wv v4, v4, v7 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vadd.vi v18, v18, 2 \n"
"vadd.vi v28, v28, 2 \n"
"vadd.vi v30, v30, 2 \n"
"vadd.vi v4, v4, 2 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v20, v18, 2 \n"
"vwaddu.vv v18, v21, v25 \n"
"vwaddu.wv v18, v18, v1 \n"
"vwaddu.wv v18, v18, v5 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vadd.vi v18, v18, 2 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v21, v18, 2 \n"
"vwaddu.vv v18, v22, v26 \n"
"vwaddu.wv v18, v18, v2 \n"
"vwaddu.wv v18, v18, v6 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vadd.vi v18, v18, 2 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v22, v18, 2 \n"
"vwaddu.vv v18, v23, v27 \n"
"vwaddu.wv v18, v18, v3 \n"
"vwaddu.wv v18, v18, v7 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vadd.vi v18, v18, 2 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v23, v18, 2 \n"
"vnsrl.wi v21, v28, 2 \n"
"vnsrl.wi v22, v30, 2 \n"
"vnsrl.wi v23, v4, 2 \n"
"vwmulsu.vv v18, v10, v20 \n"
"vwmaccsu.vv v18, v11, v21 \n"
"vwmaccsu.vv v18, v12, v22 \n"
"vwmaccsu.vv v18, v13, v23 \n"
"vwmulsu.vv v28, v14, v20 \n"
"vwmaccsu.vv v28, v15, v21 \n"
"vwmaccsu.vv v28, v16, v22 \n"
"vwmaccsu.vv v28, v17, v23 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vsub.vv v18, v8, v18 \n"
"vsub.vv v28, v8, v28 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v24, v18, 8 \n"
"vnsrl.wi v20, v28, 8 \n"
"vse8.v v24, (%[dst_u]) \n"
"vwmulsu.vv v18, v14, v20 \n"
"vwmaccsu.vv v18, v15, v21 \n"
"vwmaccsu.vv v18, v16, v22 \n"
"vwmaccsu.vv v18, v17, v23 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vsub.vv v18, v8, v18 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v20, v18, 8 \n"
"vse8.v v20, (%[dst_v]) \n"
"sub %[w_len], %[w_len], %[vl] \n"
"add %[dst_u], %[dst_u], %[vl] \n"
@ -2007,10 +1994,10 @@ void ARGBToUVMatrixRow_RVV(const uint8_t* src_argb,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
: [c] "r"(c) // %[c]
: RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6",
: "memory", RVV_VL_CLOBBER "v0", "v1", "v2", "v3", "v4", "v5", "v6",
"v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27");
"v27", "v28", "v29", "v30", "v31");
}
if (width & 1) {
uint8_t b = (src_argb_0[0] + src_argb_1[0] + 1) >> 1;
@ -2086,7 +2073,7 @@ void RGBToUV444MatrixRow_RVV(const uint8_t* src_rgb,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
: [c] "r"(c) // %[c]
: RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v8", "v9",
: "memory", RVV_VL_CLOBBER "v0", "v1", "v2", "v3", "v4", "v5", "v8", "v9",
"v10", "v11", "v12", "v13", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29");
}
@ -2131,39 +2118,33 @@ void RGBToUVMatrixRow_RVV(const uint8_t* src_rgb,
"vwaddu.vv v18, v20, v23 \n"
"vwaddu.wv v18, v18, v0 \n"
"vwaddu.wv v18, v18, v3 \n"
"vwaddu.vv v28, v21, v24 \n"
"vwaddu.wv v28, v28, v1 \n"
"vwaddu.wv v28, v28, v4 \n"
"vwaddu.vv v30, v22, v25 \n"
"vwaddu.wv v30, v30, v2 \n"
"vwaddu.wv v30, v30, v5 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vadd.vi v18, v18, 2 \n"
"vadd.vi v28, v28, 2 \n"
"vadd.vi v30, v30, 2 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v20, v18, 2 \n"
"vwaddu.vv v18, v21, v24 \n"
"vwaddu.wv v18, v18, v1 \n"
"vwaddu.wv v18, v18, v4 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vadd.vi v18, v18, 2 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v21, v18, 2 \n"
"vwaddu.vv v18, v22, v25 \n"
"vwaddu.wv v18, v18, v2 \n"
"vwaddu.wv v18, v18, v5 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vadd.vi v18, v18, 2 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v22, v18, 2 \n"
"vnsrl.wi v21, v28, 2 \n"
"vnsrl.wi v22, v30, 2 \n"
"vwmulsu.vv v18, v12, v22 \n"
"vwmaccsu.vv v18, v11, v21 \n"
"vwmaccsu.vv v18, v10, v20 \n"
"vwmulsu.vv v28, v16, v22 \n"
"vwmaccsu.vv v28, v15, v21 \n"
"vwmaccsu.vv v28, v14, v20 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vsub.vv v18, v8, v18 \n"
"vsub.vv v28, v8, v28 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v24, v18, 8 \n"
"vnsrl.wi v20, v28, 8 \n"
"vse8.v v24, (%[dst_u]) \n"
"vwmulsu.vv v18, v16, v22 \n"
"vwmaccsu.vv v18, v15, v21 \n"
"vwmaccsu.vv v18, v14, v20 \n"
"vsetvli zero, zero, e16, m2, ta, ma \n"
"vsub.vv v18, v8, v18 \n"
"vsetvli zero, zero, e8, m1, ta, ma \n"
"vnsrl.wi v20, v18, 8 \n"
"vse8.v v20, (%[dst_v]) \n"
"sub %[w_len], %[w_len], %[vl] \n"
"add %[dst_u], %[dst_u], %[vl] \n"
@ -2182,9 +2163,9 @@ void RGBToUVMatrixRow_RVV(const uint8_t* src_rgb,
[vl] "=&r"(vl), // %[vl]
[tmp] "=&r"(tmp) // %[tmp]
: [c] "r"(c) // %[c]
: RVV_VL_CLOBBER "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v8",
: "memory", RVV_VL_CLOBBER "v0", "v1", "v2", "v3", "v4", "v5", "v8",
"v9", "v10", "v11", "v12", "v14", "v15", "v16", "v18", "v19", "v20",
"v21", "v22", "v23", "v24", "v25");
"v21", "v22", "v23", "v24", "v25", "v28", "v29", "v30", "v31");
}
if (width & 1) {
uint8_t b = (src_rgb_0[0] + src_rgb_1[0] + 1) >> 1;
@ -2239,7 +2220,7 @@ void ARGBBlendRow_RVV(const uint8_t* src_argb,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
"v25", "v26", "v27", "v28", "v29", "v30", "v31");
}
@ -2279,7 +2260,7 @@ void BlendPlaneRow_RVV(const uint8_t* src0,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [k255] "r"(255)
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v16", "v17", "v18",
"v19", "v20", "v21", "v22", "v23");
}
#endif
@ -2316,7 +2297,7 @@ void ARGBAttenuateRow_RVV(const uint8_t* src_argb,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [k255] "r"(255)
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
"v25", "v26", "v27");
}
@ -2350,7 +2331,7 @@ void ARGBMultiplyRow_RVV(const uint8_t* src_argb,
[w] "+r"(w), // %[w]
[vl] "=&r"(vl) // %[vl]
: [k128] "r"(128) // %[k128]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif
@ -2378,7 +2359,7 @@ void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
:
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v18", "v19");
}
#endif
@ -2406,7 +2387,7 @@ void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [mask] "r"(0x00ffffffu) // %[mask]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");
}
@ -2434,7 +2415,7 @@ void Convert16To8Row_RVV(const uint16_t* src_y,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [shift] "r"(__builtin_clz((int32_t)scale) - 15) // %[shift]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v16", "v17");
}
#endif
@ -2468,7 +2449,7 @@ void Convert8To16Row_RVV(const uint8_t* src_y,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [shift] "r"(16 - bits), [k0101] "r"(0x0101)
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17");
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v16", "v17");
}
#endif
@ -2494,7 +2475,7 @@ void MultiplyRow_16_RVV(const uint16_t* src_y,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [scale] "r"(scale) // %[scale]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11");
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11");
}
#endif
@ -2530,8 +2511,8 @@ void HalfRow_16To8_RVV(const uint16_t* src_uv,
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [shift] "r"(__builtin_clz((int32_t)scale) - 15) // %[shift]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15", "v16", "v17");
: "memory", RVV_VL_CLOBBER "vxrm", "v8", "v9", "v10", "v11", "v12", "v13",
"v14", "v15", "v16", "v17");
}
#endif
@ -2571,7 +2552,7 @@ void HalfWidthRow_16To8_RVV(const uint16_t* src_uv,
[vl] "=&r"(vl) // %[vl]
: [shift] "r"(__builtin_clz((int32_t)scale) - 15), // %[shift]
[c2] "r"(2) // %[c2]
: RVV_VL_CLOBBER "memory", "v8", "v9", "v10", "v11", "v16", "v17", "v18",
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v16", "v17", "v18",
"v19", "v20", "v21", "v22", "v23");
}
#endif
@ -2617,12 +2598,82 @@ void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
[vl] "=&r"(vl), // %[vl]
[byte_vl] "=&r"(byte_vl) // %[byte_vl]
:
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
"v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
: "memory", RVV_VL_CLOBBER "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",
"v18", "v19", "v20", "v21", "v22", "v23");
}
#endif // HAS_ARGBMIRRORROW_RVV
#ifdef HAS_ARGBSETROW_RVV
void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width) {
size_t vl;
asm volatile(
"vsetvli %[vl], %[w], e32, m8, ta, ma \n"
"vmv.v.x v8, %[v32] \n"
"1: \n"
"vsetvli %[vl], %[w], e32, m8, ta, ma \n"
"vse32.v v8, (%[dst]) \n"
"sub %[w], %[w], %[vl] \n"
"slli %[vl], %[vl], 2 \n"
"add %[dst], %[dst], %[vl] \n"
"bnez %[w], 1b \n"
: [dst] "+r"(dst_argb), // %[dst]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl) // %[vl]
: [v32] "r"(v32) // %[v32]
: "memory", RVV_VL_CLOBBER "v8", "v9", "v10", "v11", "v12", "v13", "v14",
"v15");
}
#endif // HAS_ARGBSETROW_RVV
#ifdef HAS_ARGBSHUFFLEROW_RVV
// Shuffle the 4 bytes of each ARGB pixel with a 16 byte shuffler table (only
// the first 4 entries are used, the pattern repeats every pixel), e.g.
// {0, 1, 2, 3} is a copy and {2, 1, 0, 3} swaps R and B.
// Byte index vector: index[i] = (i & ~3) + shuffler[i & 3], gathered with a
// 16 bit index so any VLEN works (e8 indices would be limited to 256 bytes).
void ARGBShuffleRow_RVV(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width) {
size_t vl;
size_t w = (size_t)width * 4; // bytes
// The 4 shuffler bytes packed little endian so a 32 bit broadcast lays them
// out as s0 s1 s2 s3 s0 s1 s2 s3 ... in memory order.
uint32_t pattern = (uint32_t)shuffler[0] | ((uint32_t)shuffler[1] << 8) |
((uint32_t)shuffler[2] << 16) |
((uint32_t)shuffler[3] << 24);
asm volatile(
// v16..v23 (e16, m8): index = (i & ~3) + shuffler[i & 3].
"vsetvli %[vl], %[w], e16, m8, ta, ma \n"
"vid.v v16 \n"
"vand.vi v16, v16, -4 \n"
"srli %[vl], %[vl], 2 \n"
"vsetvli zero, %[vl], e32, m4, ta, ma \n"
"vmv.v.x v8, %[pattern] \n" // s0 s1 s2 s3 ...
"vsetvli %[vl], %[w], e16, m8, ta, ma \n"
"vzext.vf2 v24, v8 \n" // bytes -> e16
"vadd.vv v16, v16, v24 \n"
"1: \n"
"vsetvli %[vl], %[w], e8, m4, ta, ma \n"
"vle8.v v8, (%[src]) \n"
"vrgatherei16.vv v0, v8, v16 \n"
"vse8.v v0, (%[dst]) \n"
"sub %[w], %[w], %[vl] \n"
"add %[src], %[src], %[vl] \n"
"add %[dst], %[dst], %[vl] \n"
"bnez %[w], 1b \n"
: [src] "+r"(src_argb), // %[src]
[dst] "+r"(dst_argb), // %[dst]
[w] "+r"(w), // %[w]
[vl] "=&r"(vl) // %[vl]
: [pattern] "r"(pattern) // %[pattern]
: "memory", RVV_VL_CLOBBER "v0", "v1", "v2", "v3", "v8", "v9", "v10",
"v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
"v25", "v26", "v27", "v28", "v29", "v30", "v31");
}
#endif // HAS_ARGBSHUFFLEROW_RVV
#ifdef HAS_MIRRORUVROW_RVV
void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
int vl;
@ -2664,9 +2715,9 @@ void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
[vl] "=&r"(vl), // %[vl]
[byte_vl] "=&r"(byte_vl) // %[byte_vl]
:
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
"v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
: "memory", RVV_VL_CLOBBER "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",
"v18", "v19", "v20", "v21", "v22", "v23");
}
#endif // HAS_MIRRORUVROW_RVV
@ -2772,7 +2823,7 @@ void Ssim4x4Row_RVV(const uint8_t* src_a,
[tmp_b] "=&r"(tmp_b) // %[tmp_b]
: [stride_a] "r"((ptrdiff_t)stride_a), // %[stride_a]
[stride_b] "r"((ptrdiff_t)stride_b) // %[stride_b]
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
: "memory", RVV_VL_CLOBBER "v0", "v1", "v2", "v3", "v4", "v5",
"v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29");

29
source/row_sve.cc
View File

@ -777,7 +777,7 @@ void DivideRow_16_SVE2(const uint16_t* src_y,
"cnth %x[vl] \n"
"dup z0.h, %w[scale] \n"
"subs %w[width], %w[width], %w[vl], lsl #1 \n"
"b.le 2f \n"
"b.lt 2f \n"
// Run bulk of computation with the same predicates to avoid predicate
// generation overhead.
@ -792,7 +792,7 @@ void DivideRow_16_SVE2(const uint16_t* src_y,
"st1h {z1.h}, p0, [%[dst]] \n"
"st1h {z2.h}, p0, [%[dst], #1, mul vl] \n"
"incb %[dst], all, mul #2 \n"
"b.gt 1b \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl], lsl #1 \n"
@ -827,6 +827,10 @@ void DivideRow_16_SVE2(const uint16_t* src_y,
"fmul z1.s, z1.s, z4.s \n" \
"fmul z2.s, z2.s, z4.s \n" \
"fmul z3.s, z3.s, z4.s \n" \
"fmul z0.s, z0.s, z5.s \n" \
"fmul z1.s, z1.s, z5.s \n" \
"fmul z2.s, z2.s, z5.s \n" \
"fmul z3.s, z3.s, z5.s \n" \
"uqshrnb z0.h, z0.s, #13 \n" \
"uqshrnb z1.h, z1.s, #13 \n" \
"uqshrnb z2.h, z2.s, #13 \n" \
@ -840,6 +844,7 @@ void HalfFloatRow_SVE2(const uint16_t* src,
asm("cntw %x0" : "=r"(vl));
asm volatile(
"mov z4.s, %s[scale] \n"
"mov z5.s, %s[bias] \n"
"subs %w[width], %w[width], %w[vl], lsl #2 \n"
"b.lt 2f \n"
@ -881,14 +886,16 @@ void HalfFloatRow_SVE2(const uint16_t* src,
"st1h {z3.s}, p3, [%[dst], #3, mul vl] \n"
"99: \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[vl2] "r"(vl * 2), // %[vl2]
[vl3] "r"(vl * 3), // %[vl3]
[scale] "w"(scale * 1.9259299444e-34f) // %[scale]
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2", "p3");
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[width] "+r"(width) // %[width]
: [vl] "r"(vl), // %[vl]
[vl2] "r"(vl * 2), // %[vl2]
[vl3] "r"(vl * 3), // %[vl3]
[scale] "w"(scale), // %[scale]
[bias] "w"(1.9259299444e-34f) // %[bias] 2^-112
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "p0", "p1", "p2",
"p3");
}
void HalfFloat1Row_SVE2(const uint16_t* src,
@ -1221,7 +1228,7 @@ void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) {
"st1b {z0.b}, p0, [%[dst], #0, mul vl] \n"
"st1b {z1.b}, p0, [%[dst], #1, mul vl] \n"
"incb %[dst], all, mul #2 \n"
"b.gt 1b \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl2] \n"

111
source/scale.cc
View File

@ -212,15 +212,6 @@ static void ScalePlaneDown2_16(int src_width,
: ScaleRowDown2Box_16_SME;
}
#endif
#if defined(HAS_SCALEROWDOWN2_16_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 16)) {
ScaleRowDown2 =
filtering == kFilterNone
? ScaleRowDown2_16_SSE2
: (filtering == kFilterLinear ? ScaleRowDown2Linear_16_SSE2
: ScaleRowDown2Box_16_SSE2);
}
#endif
if (filtering == kFilterLinear) {
src_stride = 0;
@ -339,18 +330,6 @@ static void ScalePlaneDown4_16(int src_width,
src_ptr += src_stride * 2; // Point to row 2.
src_stride = 0;
}
#if defined(HAS_SCALEROWDOWN4_16_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 8)) {
ScaleRowDown4 =
filtering ? ScaleRowDown4Box_16_NEON : ScaleRowDown4_16_NEON;
}
#endif
#if defined(HAS_SCALEROWDOWN4_16_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
ScaleRowDown4 =
filtering ? ScaleRowDown4Box_16_SSE2 : ScaleRowDown4_16_SSE2;
}
#endif
if (filtering == kFilterLinear) {
src_stride = 0;
@ -536,28 +515,6 @@ static void ScalePlaneDown34_16(int src_width,
ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_C;
ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_C;
}
#if defined(HAS_SCALEROWDOWN34_16_NEON)
if (TestCpuFlag(kCpuHasNEON) && (dst_width % 24 == 0)) {
if (!filtering) {
ScaleRowDown34_0 = ScaleRowDown34_16_NEON;
ScaleRowDown34_1 = ScaleRowDown34_16_NEON;
} else {
ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_NEON;
ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_NEON;
}
}
#endif
#if defined(HAS_SCALEROWDOWN34_16_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) {
if (!filtering) {
ScaleRowDown34_0 = ScaleRowDown34_16_SSSE3;
ScaleRowDown34_1 = ScaleRowDown34_16_SSSE3;
} else {
ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_SSSE3;
ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_SSSE3;
}
}
#endif
for (y = 0; y < dst_height - 2; y += 3) {
ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width);
@ -762,28 +719,6 @@ static void ScalePlaneDown38_16(int src_width,
ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_C;
ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_C;
}
#if defined(HAS_SCALEROWDOWN38_16_NEON)
if (TestCpuFlag(kCpuHasNEON) && (dst_width % 12 == 0)) {
if (!filtering) {
ScaleRowDown38_3 = ScaleRowDown38_16_NEON;
ScaleRowDown38_2 = ScaleRowDown38_16_NEON;
} else {
ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_NEON;
ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_NEON;
}
}
#endif
#if defined(HAS_SCALEROWDOWN38_16_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) {
if (!filtering) {
ScaleRowDown38_3 = ScaleRowDown38_16_SSSE3;
ScaleRowDown38_2 = ScaleRowDown38_16_SSSE3;
} else {
ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_SSSE3;
ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_SSSE3;
}
}
#endif
for (y = 0; y < dst_height - 2; y += 3) {
ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width);
@ -917,6 +852,7 @@ static void ScaleAddCols1_16_C(int dst_width,
int boxwidth = MIN1(dx >> 16);
int scaleval = 65536 / (boxwidth * boxheight);
int i;
x >>= 16;
for (i = 0; i < dst_width; ++i) {
*dst_ptr++ = SumPixels_16(boxwidth, src_ptr + x) * scaleval >> 16;
x += boxwidth;
@ -1061,11 +997,6 @@ static int ScalePlaneBox_16(int src_width,
void (*ScaleAddRow)(const uint16_t* src_ptr, uint32_t* dst_ptr,
int src_width) = ScaleAddRow_16_C;
#if defined(HAS_SCALEADDROW_16_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(src_width, 16)) {
ScaleAddRow = ScaleAddRow_16_SSE2;
}
#endif
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
@ -1267,14 +1198,6 @@ static int ScalePlaneBilinearDown_16(int src_width,
&dx, &dy);
src_width = Abs(src_width);
#if defined(HAS_INTERPOLATEROW_16_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
InterpolateRow = InterpolateRow_16_Any_SSSE3;
if (IS_ALIGNED(src_width, 16)) {
InterpolateRow = InterpolateRow_16_SSSE3;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_16_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_16_Any_AVX2;
@ -1310,11 +1233,6 @@ static int ScalePlaneBilinearDown_16(int src_width,
}
#endif
#if defined(HAS_SCALEFILTERCOLS_16_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleFilterCols = ScaleFilterCols_16_SSSE3;
}
#endif
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
@ -1449,7 +1367,7 @@ static int ScalePlaneBilinearUp(int src_width,
#endif
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
ScaleFilterCols = ScaleColsUp2_C;
#if defined(HAS_SCALECOLS_SSE2)
#if defined(HAS_SCALECOLSUP2_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
ScaleFilterCols = ScaleColsUp2_SSE2;
}
@ -1887,14 +1805,6 @@ static int ScalePlaneBilinearUp_16(int src_width,
assert(dy <= 65536);
src_width = Abs(src_width);
#if defined(HAS_INTERPOLATEROW_16_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
InterpolateRow = InterpolateRow_16_Any_SSSE3;
if (IS_ALIGNED(dst_width, 16)) {
InterpolateRow = InterpolateRow_16_SSSE3;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_16_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_16_Any_AVX2;
@ -1933,18 +1843,8 @@ static int ScalePlaneBilinearUp_16(int src_width,
if (filtering && src_width >= 32768) {
ScaleFilterCols = ScaleFilterCols64_16_C;
}
#if defined(HAS_SCALEFILTERCOLS_16_SSSE3)
if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleFilterCols = ScaleFilterCols_16_SSSE3;
}
#endif
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
ScaleFilterCols = ScaleColsUp2_16_C;
#if defined(HAS_SCALECOLS_16_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
ScaleFilterCols = ScaleColsUp2_16_SSE2;
}
#endif
}
uint32_t uy = y;
@ -2038,7 +1938,7 @@ static void ScalePlaneSimple(int src_width,
if (src_width * 2 == dst_width && x < 0x8000) {
ScaleCols = ScaleColsUp2_C;
#if defined(HAS_SCALECOLS_SSE2)
#if defined(HAS_SCALECOLSUP2_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
ScaleCols = ScaleColsUp2_SSE2;
}
@ -2076,11 +1976,6 @@ static void ScalePlaneSimple_16(int src_width,
if (src_width * 2 == dst_width && x < 0x8000) {
ScaleCols = ScaleColsUp2_16_C;
#if defined(HAS_SCALECOLS_16_SSE2)
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
ScaleCols = ScaleColsUp2_16_SSE2;
}
#endif
}
uint32_t uy = y;

4
source/scale_any.cc
View File

@ -1018,7 +1018,7 @@ SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_AVX2,
SBUH2LANY(ScaleUVRowUp2_Linear_Any_NEON,
ScaleUVRowUp2_Linear_NEON,
ScaleUVRowUp2_Linear_C,
15,
7,
uint8_t)
#endif
@ -1026,7 +1026,7 @@ SBUH2LANY(ScaleUVRowUp2_Linear_Any_NEON,
SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_NEON,
ScaleUVRowUp2_Linear_16_NEON,
ScaleUVRowUp2_Linear_16_C,
15,
7,
uint16_t)
#endif

3
source/scale_argb.cc
View File

@ -463,7 +463,8 @@ static int ScaleARGBBilinearDown(int src_width,
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
// Allocate a row of ARGB.
{
align_buffer_64(row, clip_src_width * 4);
// clip_src_width is already in bytes (4 per ARGB pixel).
align_buffer_64(row, clip_src_width);
if (!row)
return 1;

8
source/scale_common.cc
View File

@ -1515,14 +1515,6 @@ void ScalePlaneVertical_16(int src_height,
assert(dst_width > 0);
assert(dst_height > 0);
src_argb += (x >> 16) * wpp;
#if defined(HAS_INTERPOLATEROW_16_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
InterpolateRow = InterpolateRow_16_Any_SSSE3;
if (IS_ALIGNED(dst_width_words, 16)) {
InterpolateRow = InterpolateRow_16_SSSE3;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_16_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_16_Any_AVX2;

15
source/scale_gcc.cc
View File

@ -1275,21 +1275,6 @@ void ScaleRowUp2_Bilinear_16_SSE2(const uint16_t* src_ptr,
LABELALIGN
"1: \n"
"movq (%0),%%xmm0 \n" // 0011 (16b, 1u1v)
"movq 4(%0),%%xmm1 \n" // 1122 (16b, 1u1v)
"punpcklwd %%xmm7,%%xmm0 \n" // 0011 (near) (32b, 1u1v)
"punpcklwd %%xmm7,%%xmm1 \n" // 1122 (near) (32b, 1u1v)
"movdqa %%xmm0,%%xmm2 \n"
"movdqa %%xmm1,%%xmm3 \n"
"pshufd $0b01001110,%%xmm2,%%xmm2 \n" // 1100 (far) (1, lo)
"pshufd $0b01001110,%%xmm3,%%xmm3 \n" // 2211 (far) (1, hi)
"paddd %%xmm0,%%xmm2 \n" // near+far (1, lo)
"paddd %%xmm1,%%xmm3 \n" // near+far (1, hi)
"paddd %%xmm0,%%xmm0 \n" // 2*near (1, lo)
"paddd %%xmm1,%%xmm1 \n" // 2*near (1, hi)
"paddd %%xmm2,%%xmm0 \n" // 3*near+far (1, lo)
"paddd %%xmm3,%%xmm1 \n" // 3*near+far (1, hi)
"movq (%0),%%xmm0 \n" // 0123 (16b)
"movq 2(%0),%%xmm1 \n" // 1234 (16b)
"punpcklwd %%xmm7,%%xmm0 \n" // 0123 (32b)

8
source/scale_neon64.cc
View File

@ -1013,7 +1013,7 @@ void ScaleAddRow_NEON(const uint8_t* src_ptr,
}
#define SCALE_FILTER_COLS_STEP_ADDR \
"lsr %[tmp_offset], %x[x], #16 \n" \
"asr %[tmp_offset], %x[x], #16 \n" \
"add %[tmp_ptr], %[src_ptr], %[tmp_offset] \n" \
"add %x[x], %x[x], %x[dx] \n"
@ -1027,6 +1027,8 @@ void ScaleFilterCols_NEON(uint8_t* dst_ptr,
int x,
int dx) {
int dx_offset[4] = {0, 1, 2, 3};
int64_t x64 = (int64_t)x;
int64_t dx64 = (int64_t)dx;
int64_t tmp_offset;
uint8_t* tmp_ptr;
asm volatile(
@ -1080,8 +1082,8 @@ void ScaleFilterCols_NEON(uint8_t* dst_ptr,
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
[width] "+r"(dst_width), // %[width]
[x] "+r"(x), // %[x]
[dx] "+r"(dx), // %[dx]
[x] "+r"(x64), // %[x]
[dx] "+r"(dx64), // %[dx]
[tmp_offset] "=&r"(tmp_offset), // %[tmp_offset]
[tmp_ptr] "=&r"(tmp_ptr) // %[tmp_ptr]
: [dx_offset] "r"(dx_offset) // %[dx_offset]

116
source/scale_uv.cc
View File

@ -140,22 +140,6 @@ static void ScaleUVDown2(int src_width,
#endif
// This code is not enabled. Only box filter is available at this time.
#if defined(HAS_SCALEUVROWDOWN2_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ScaleUVRowDown2 =
filtering == kFilterNone
? ScaleUVRowDown2_Any_SSSE3
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_SSSE3
: ScaleUVRowDown2Box_Any_SSSE3);
if (IS_ALIGNED(dst_width, 2)) {
ScaleUVRowDown2 =
filtering == kFilterNone
? ScaleUVRowDown2_SSSE3
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_SSSE3
: ScaleUVRowDown2Box_SSSE3);
}
}
#endif
if (filtering == kFilterLinear) {
src_stride = 0;
@ -226,7 +210,7 @@ static int ScaleUVDown4Box(int src_width,
}
}
#endif
#if defined(HAS_SCALEUVROWDOWN2BOX_SME)
#if defined(HAS_SCALEUVROWDOWN2_SME)
if (TestCpuFlag(kCpuHasSME)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_SME;
}
@ -352,6 +336,11 @@ static int ScaleUVBox(int src_width,
}
}
#endif
#if defined(HAS_SCALEADDROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ScaleAddRow = ScaleAddRow_AVX512BW;
}
#endif
#if defined(HAS_SCALEADDROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleAddRow = ScaleAddRow_Any_NEON;
@ -427,16 +416,6 @@ static void ScaleUVDownEven(int src_width,
assert(IS_ALIGNED(src_width, 2));
assert(IS_ALIGNED(src_height, 2));
src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
#if defined(HAS_SCALEUVROWDOWNEVEN_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_SSSE3
: ScaleUVRowDownEven_Any_SSSE3;
if (IS_ALIGNED(dst_width, 4)) {
ScaleUVRowDownEven =
filtering ? ScaleUVRowDownEvenBox_SSE2 : ScaleUVRowDownEven_SSSE3;
}
}
#endif
#if defined(HAS_SCALEUVROWDOWNEVEN_NEON)
if (TestCpuFlag(kCpuHasNEON) && !filtering) {
ScaleUVRowDownEven = ScaleUVRowDownEven_Any_NEON;
@ -445,16 +424,6 @@ static void ScaleUVDownEven(int src_width,
}
}
#endif // TODO(fbarchard): Enable Box filter
#if defined(HAS_SCALEUVROWDOWNEVENBOX_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_NEON
: ScaleUVRowDownEven_Any_NEON;
if (IS_ALIGNED(dst_width, 4)) {
ScaleUVRowDownEven =
filtering ? ScaleUVRowDownEvenBox_NEON : ScaleUVRowDownEven_NEON;
}
}
#endif
#if defined(HAS_SCALEUVROWDOWNEVEN_RVV) || defined(HAS_SCALEUVROWDOWN4_RVV)
if (TestCpuFlag(kCpuHasRVV) && !filtering) {
#if defined(HAS_SCALEUVROWDOWNEVEN_RVV)
@ -573,26 +542,14 @@ static int ScaleUVBilinearDown(int src_width,
InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
#if defined(HAS_SCALEUVFILTERCOLS_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleUVFilterCols = ScaleUVFilterCols_SSSE3;
}
#endif
#if defined(HAS_SCALEUVFILTERCOLS_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON;
if (IS_ALIGNED(dst_width, 4)) {
ScaleUVFilterCols = ScaleUVFilterCols_NEON;
}
}
#endif
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
// Allocate a row of UV.
{
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
uint32_t uy = y;
align_buffer_64(row, clip_src_width * 2);
// clip_src_width is already in bytes (2 per UV pixel).
align_buffer_64(row, clip_src_width);
if (!row)
return 1;
if (uy > max_y) {
@ -713,39 +670,8 @@ static int ScaleUVBilinearUp(int src_width,
if (src_width >= 32768) {
ScaleUVFilterCols = filtering ? ScaleUVFilterCols64_C : ScaleUVCols64_C;
}
#if defined(HAS_SCALEUVFILTERCOLS_SSSE3)
if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleUVFilterCols = ScaleUVFilterCols_SSSE3;
}
#endif
#if defined(HAS_SCALEUVFILTERCOLS_NEON)
if (filtering && TestCpuFlag(kCpuHasNEON)) {
ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVFilterCols = ScaleUVFilterCols_NEON;
}
}
#endif
#if defined(HAS_SCALEUVCOLS_SSSE3)
if (!filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleUVFilterCols = ScaleUVCols_SSSE3;
}
#endif
#if defined(HAS_SCALEUVCOLS_NEON)
if (!filtering && TestCpuFlag(kCpuHasNEON)) {
ScaleUVFilterCols = ScaleUVCols_Any_NEON;
if (IS_ALIGNED(dst_width, 16)) {
ScaleUVFilterCols = ScaleUVCols_NEON;
}
}
#endif
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
ScaleUVFilterCols = ScaleUVColsUp2_C;
#if defined(HAS_SCALEUVCOLSUP2_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) {
ScaleUVFilterCols = ScaleUVColsUp2_SSSE3;
}
#endif
}
if (uy > max_y) {
@ -1062,26 +988,8 @@ static void ScaleUVSimple(int src_width,
int x, int dx) =
(src_width >= 32768) ? ScaleUVCols64_C : ScaleUVCols_C;
(void)src_height;
#if defined(HAS_SCALEUVCOLS_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleUVCols = ScaleUVCols_SSSE3;
}
#endif
#if defined(HAS_SCALEUVCOLS_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVCols = ScaleUVCols_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVCols = ScaleUVCols_NEON;
}
}
#endif
if (src_width * 2 == dst_width && x < 0x8000) {
ScaleUVCols = ScaleUVColsUp2_C;
#if defined(HAS_SCALEUVCOLSUP2_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) {
ScaleUVCols = ScaleUVColsUp2_SSSE3;
}
#endif
}
uint32_t uy = y;
@ -1344,6 +1252,14 @@ int UVScale_16(const uint16_t* src_uv,
}
#endif
// ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone, which has
// no 16 bit implementation other than the copy above. The Up2 kernels only
// read the pixels they need, so restore Linear or Bilinear for a 2x upscale
// so P010ToP410 and P210ToP410 work on 1 pixel wide images.
if (filtering == kFilterNone && src_width == 1 && dst_width <= 2) {
filtering = (src_height == dst_height) ? kFilterLinear : kFilterBilinear;
}
if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) {
ScaleUVLinearUp2_16(src_width, src_height, dst_width, dst_height,
src_stride_uv, dst_stride_uv, src_uv, dst_uv);