mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-11-09 14:49:57 +08:00
[libyuv] Fix x86 SIMD OOB, math, alignment, MSan & GCC bugs
Address x86 SIMD correctness, sanitizer, alignment, and compiler guard
bugs across row_gcc.cc, row_win.cc, row_any.cc, row_common.cc, and
caller dispatchers:
- MSan & row_win.cc: Replace _mm512_ternarylogic_epi32 in row_win.cc
(ARGBGrayRow_AVX512VNNI, ARGBToUV444MatrixRow_AVX512VNNIVBMI, and
I422ToAR30Row_AVX512BW) with bitwise intrinsics modeled bit-by-bit by
LLVM MSan, document the MSan intrinsic rules in GEMINI.md, and fix
standalone MSan ELF PT_INTERP in BUILD.gn. Add missing USE_ROW_WIN
DivideRow_16_AVX2, HalfMergeUVRow_AVX2, and Convert8To8Row_AVX2.
- Out-of-bounds & AVX-512 opmask tails: Fix RGBToUVMatrixRow_Any_SSSE3
MASK from 7 to 15 (24-byte overread), guard UnpackMT2T_C and
UnpackMT2T_AVX2 against partial < 80-byte tail blocks, add inline
k1-k7 opmask tails to all 17 remaining AVX-512 row kernels (removing
all *_Any_AVX512* wrappers), and enable opmask-tailed
NV21ToYUV24Row_AVX512VBMI.
- Math, saturation & pointer bugs: Fix MirrorRow_16_AVX512BW 64-bit
width sign extension, ARGBPolynomialRow_AVX2 vpackssdw clamping,
CumulativeSumToAverageRow_AVX2 small-area (area <= 1) handling and
vdivss precision, HalfWidthRow_16To8_{SSSE3,AVX2,AVX512BW} unsigned
pair summation for samples >= 0x8000, HalfRow_16To8_AVX512BW
non-power-of-two scale multiplication, DivideRow_16 SIMD
scale == 65536 (16-bit depth) truncation, ARGBLumaColorTableRow_AVX2
unsigned luma weight accumulation, Merge{ARGB,XRGB}16To8Row_AVX2
depth == 8 clamping, custom ArgbConstants::kAddUV loading, 16-bit YUV
unsigned shifts/ORs, and null-pointer checks in NV16ToI420/NV24ToI420.
- Alignment (#GP), ISA (#UD), UBSan & 32-bit x86: Use unaligned loads
for ArgbConstants, YuvConstants, and ARGBPolynomialRow_SSE2 float
coefficients; remove AVX2 instructions from HalfFloat{,1}Row_F16C;
fix Merge{AR,XR}64Row_AVX2 depth == 16 shift UB and ByteToFloatRow_C /
TestByteToFloat float alignment UB; and fix 32-bit -m32 -fPIC ASan
inline asm register pressure and operand suffixes.
- GCC & compiler guards: Enable GCC_HAS_AVX512 for GCC >= 9 on x86_64
with per-function target attributes, gate HAS_*_AVX2 and HAS_*_AVX512*
macros on compiler version macros, add missing #ifdef HAS_* guards in
row_gcc.cc, fix _Any / IS_ALIGNED step mismatches, wire missing
dispatchers (ARGBSobelize, SplitRotateUV180, HalfMergeUVPlane,
ARGBPolynomial), and silence GCC 15 -Werror=stringop-overflow=.
Convert SIMD ISA per *_C (* = uncalled in library):
------------------------------------------------------------------------
C Function arm x86 riscv hexagon wasm loong
------------------------------------------------------------------------
AB64ToARGBRow_C NEON AVX2 RVV HVX - -
ABGRToAR30Row_C NEON AVX512BW - - - -
AR30ToAB30Row_C NEON AVX512BW - - - -
AR30ToABGRRow_C NEON AVX512BW - - - -
AR30ToARGBRow_C NEON AVX512BW - - - -
AR64ShuffleRow_C NEON AVX512BW RVV - RELAXED LASX
AR64ToAB64Row_C NEON AVX512BW RVV HVX RELAXED LASX
AR64ToARGBRow_C NEON AVX2 RVV HVX - -
ARGB1555ToARGBRow_C SVE2 AVX512BW - - SIMD LASX
ARGB1555ToUVMatrixRow_C NEON AVX2 - - SIMD -
ARGB1555ToUVRow_C * NEON AVX2 - - - LASX
ARGB1555ToYMatrixRow_C NEON AVX2 - - SIMD -
ARGB1555ToYRow_C * NEON AVX2 - - - LASX
ARGB4444ToARGBRow_C NEON AVX2 - - SIMD LASX
ARGB4444ToUVMatrixRow_C NEON AVX2 - - SIMD -
ARGB4444ToUVRow_C * NEON AVX2 - - - -
ARGB4444ToYMatrixRow_C NEON AVX2 - - SIMD -
ARGB4444ToYRow_C * NEON AVX2 - - - -
ARGBAddRow_C NEON AVX2 - HVX - LASX
ARGBAffineRow_C * NEON AVX2 - - - -
ARGBAlphaBlendRow_C SME AVX512BW RVV HVX SIMD LASX
ARGBAttenuateRow_C NEON AVX512BW RVV HVX - LASX
ARGBBlendRow_C NEON AVX512BW RVV HVX RELAXED LASX
ARGBColorMatrixRow_C I8MM AVX512VNNI - - - LASX
ARGBColorTableRow_C NEON AVX2 - - - -
ARGBCopyAlphaRow_C NEON AVX2 - HVX - -
ARGBCopyYToAlphaRow_C NEON AVX512BW RVV HVX RELAXED -
ARGBExtractAlphaRow_C NEON AVX512BW RVV HVX SIMD LASX
ARGBGrayRow_C DOTPROD AVX512VNNI - - - LASX
ARGBLumaColorTableRow_C NEON AVX2 - - - -
ARGBMirrorRow_C NEON AVX512BW RVV HVX SIMD LASX
ARGBMultiplyRow_C SME AVX512BW RVV HVX - LASX
ARGBOrRow_C SME AVX512BW RVV HVX SIMD LASX
ARGBPolynomialRow_C NEON AVX2 - - - -
ARGBQuantizeRow_C NEON AVX2 - - - LASX
ARGBSepiaRow_C DOTPROD AVX512VNNI - - - LASX
ARGBSetRow_C NEON AVX2 RVV HVX - LASX
ARGBShadeRow_C NEON AVX2 - - - LASX
ARGBShuffleRow_C NEON AVX512BW RVV - RELAXED LASX
ARGBSubtractRow_C NEON AVX2 - HVX - LASX
ARGBToAB64Row_C NEON AVX2 RVV HVX - -
ARGBToABGRRow_C NEON AVX512BW RVV HVX RELAXED LASX
ARGBToAR30Row_C NEON AVX512BW - - - -
ARGBToAR64Row_C NEON AVX2 RVV HVX - -
ARGBToARGB1555Row_C NEON AVX2 - - SIMD LASX
ARGBToARGB4444Row_C NEON AVX2 - - SIMD LASX
ARGBToBGRARow_C NEON AVX512BW RVV HVX RELAXED LASX
ARGBToRAWRow_C SVE2 AVX512BW RVV - - LASX
ARGBToRGB24Row_C SVE2 AVX512BW RVV - - LASX
ARGBToRGB565DitherRow_C SVE2 AVX512BW - - SIMD LASX
ARGBToRGB565Row_C SVE2 AVX512BW - - SIMD LASX
ARGBToRGBARow_C NEON AVX512BW RVV HVX RELAXED LASX
ARGBToUV444MatrixRow_C I8MM AVX512VNNI RVV HVX RELAXED -
ARGBToUVMatrixRow_C SME AVX512VNNI RVV HVX SIMD -
ARGBToYMatrixRow_C SME AVX512VNNI RVV HVX SIMD LASX
ARGBUnattenuateRow_C NEON AVX2 - - - -
AYUVToUVRow_C SVE2 AVX512VBMI - - - -
AYUVToVURow_C SVE2 AVX512VBMI - - - -
AYUVToYRow_C NEON AVX512VBMI - - - -
BlendPlaneRow_C NEON AVX512BW RVV HVX SIMD -
ByteToFloatRow_C NEON AVX2 - - - -
ComputeCumulativeSumRow_C NEON AVX2 - - - -
Convert16To8Row_C SME AVX512BW RVV HVX - -
Convert8To16Row_C SME AVX512BW RVV HVX SIMD -
Convert8To8Row_C SME AVX512BW - HVX - -
CopyRow_16_C NEON AVX2 - HVX - -
CopyRow_C SME AVX512BW RVV HVX - LASX
CumulativeSumToAverageRow_C NEON AVX2 - - - -
DetileRow_16_C NEON AVX2 - - - -
DetileRow_C NEON AVX2 - - - -
DetileSplitUVRow_C NEON AVX2 - - - -
DetileToYUY2_C NEON AVX2 - - - -
DivideRow_16_C SVE2 AVX512BW - HVX - -
GaussCol_C * NEON AVX2 - - - -
GaussCol_F32_C NEON AVX2 - - - -
GaussRow_C * NEON AVX2 - - - -
GaussRow_F32_C NEON AVX2 - - - -
HalfFloatRow_C SVE2 AVX512BW - - - LASX
HalfMergeUVRow_C SVE2 AVX512BW - - - -
HalfRow_16To8_C SME AVX512BW RVV HVX - -
HalfRow_16_C - AVX2 - - - -
HalfRow_C - AVX2 - - - -
HalfWidthRow_16To8_C SME AVX512BW RVV - - -
HalfWidthRow_16To8_Odd_C - AVX2 - - - -
I210AlphaToARGBRow_C SME AVX512BW - - - -
I210ToAR30Row_C SME AVX512BW - - - -
I210ToARGBRow_C SME AVX512BW - - - -
I212ToAR30Row_C SME AVX512BW - - - -
I212ToARGBRow_C SME AVX512BW - - - -
I400ToARGBRow_C SME AVX512BW RVV HVX - LASX
I410AlphaToARGBRow_C SME AVX512BW - - - -
I410ToAR30Row_C SME AVX512BW - - - -
I410ToARGBRow_C SME AVX512BW - - - -
I422AlphaToARGBRow_C SME AVX512BW RVV HVX - LASX
I422ToAR30Row_C SME AVX512BW RVV - SIMD -
I422ToARGB1555Row_C SME AVX512BW - - SIMD LASX
I422ToARGB4444Row_C SME AVX512BW - - SIMD LASX
I422ToARGBRow_C SME AVX512BW RVV HVX SIMD LASX
I422ToRGB24Row_C SME AVX512VBMI RVV - RELAXED LASX
I422ToRGB565Row_C SME AVX512BW - - SIMD LASX
I422ToRGBARow_C SME AVX512BW RVV HVX - LASX
I422ToUYVYRow_C NEON AVX2 - - - LASX
I422ToYUY2Row_C NEON AVX2 - - - LASX
I444AlphaToARGBRow_C SME AVX512BW RVV HVX - -
I444ToARGBRow_C SME AVX512BW RVV HVX - LASX
I444ToRGB24Row_C SME AVX512VBMI RVV - - -
InterpolateRow_16_C SME AVX512BW - HVX SIMD -
InterpolateRow_C SME AVX512BW RVV HVX SIMD LASX
J400ToARGBRow_C NEON AVX512BW RVV HVX RELAXED LASX
MergeAR64Row_C NEON AVX2 - - - -
MergeARGB16To8Row_C NEON AVX2 - - - -
MergeARGBRow_C NEON AVX2 RVV HVX - -
MergeRGBRow_C NEON AVX512VBMI RVV - - -
MergeUVRow_16_C SME AVX512BW - HVX - -
MergeUVRow_C SME AVX512BW RVV HVX SIMD LASX
MergeXR30Row_C NEON AVX512BW - - - -
MergeXR64Row_C NEON AVX2 - - - -
MergeXRGB16To8Row_C NEON AVX2 - - - -
MergeXRGBRow_C NEON AVX2 RVV HVX - -
MirrorRow_16_C NEON AVX512BW RVV HVX RELAXED LASX
MirrorRow_C NEON AVX512BW - HVX RELAXED LASX
MirrorSplitUVRow_C NEON AVX512BW - HVX RELAXED LASX
MirrorUVRow_C NEON AVX512BW RVV HVX RELAXED LASX
MultiplyRow_16_C SME AVX512BW RVV HVX SIMD -
NV12ToARGBRow_C SME AVX512BW RVV HVX SIMD LASX
NV12ToRGB24Row_C SME AVX512VBMI RVV - - -
NV12ToRGB565Row_C NEON AVX2 - - SIMD LASX
NV21ToARGBRow_C SME AVX512BW RVV HVX SIMD LASX
NV21ToRGB24Row_C SME AVX512VBMI RVV - - -
NV21ToYUV24Row_C NEON AVX512VBMI - - - -
P210ToAR30Row_C SME AVX512BW - - - -
P210ToARGBRow_C SME AVX512BW - - - -
P410ToAR30Row_C SME AVX512BW - - - -
P410ToARGBRow_C SME AVX512BW - - - -
RAWToARGBRow_C SVE2 AVX512BW RVV - RELAXED LASX
RAWToRGB24Row_C SVE2 AVX512VBMI RVV - RELAXED LASX
RAWToRGBARow_C SVE2 AVX512VBMI RVV - - -
RGB24MirrorRow_C NEON AVX512VBMI - - RELAXED -
RGB24ToARGBRow_C SVE2 AVX512BW RVV - RELAXED LASX
RGB565ToARGBRow_C NEON AVX2 - - SIMD LASX
RGB565ToUVMatrixRow_C NEON AVX2 - - SIMD -
RGB565ToUVRow_C * NEON AVX2 - - - LASX
RGB565ToYMatrixRow_C NEON AVX2 - - SIMD -
RGB565ToYRow_C * NEON AVX2 - - - LASX
RGBAToARGBRow_C NEON AVX512BW RVV HVX RELAXED LASX
RGBColorTableRow_C NEON AVX2 - - - -
RGBToUV444MatrixRow_C NEON AVX512BW RVV - RELAXED -
RGBToUVMatrixRow_C NEON AVX512BW RVV - RELAXED -
RGBToYMatrixRow_C DOTPROD AVX512BW RVV - RELAXED LASX
ScaleMaxSamples_C * NEON AVX2 - - - -
ScaleSamples_C * NEON AVX2 - - - -
ScaleSumSamples_C * NEON AVX2 - - - -
SetRow_C NEON AVX2 - HVX - LASX
SobelRow_C NEON AVX2 - - - LASX
SobelToPlaneRow_C NEON AVX2 - - - LASX
SobelXRow_C NEON AVX2 - - - -
SobelXYRow_C NEON AVX2 - - - LASX
SobelYRow_C NEON AVX2 - - - -
SplitARGBRow_C NEON AVX2 RVV HVX - -
SplitRGBRow_C NEON AVX2 RVV - - -
SplitUVRow_16_C NEON AVX2 - HVX - -
SplitUVRow_C SVE2 AVX512BW RVV HVX - LASX
SplitXRGBRow_C NEON AVX2 RVV HVX - -
SwapUVRow_C NEON AVX512BW RVV HVX SIMD -
UYVYToARGBRow_C SME AVX512BW - - - LASX
UYVYToUV422Row_C NEON AVX2 - HVX - LASX
UYVYToUVRow_C SVE2 AVX512BW - HVX - LASX
UYVYToYRow_C SVE2 AVX512BW - HVX - LASX
UnpackMT2T_C NEON AVX2 - - - -
YUY2ToARGBRow_C SME AVX512BW - - - LASX
YUY2ToNVUVRow_C NEON AVX2 - - - -
YUY2ToUV422Row_C NEON AVX2 - HVX - LASX
YUY2ToUVRow_C SVE2 AVX512BW - HVX - LASX
YUY2ToYRow_C SVE2 AVX512BW - HVX - LASX
------------------------------------------------------------------------
SIMD (174 convert) 171 98% 174 100% 65 37% 64 37% 57 33% 76 44%
------------------------------------------------------------------------
Rotate SIMD ISA per *_C (* = uncalled in library):
------------------------------------------------------------------------
C Function arm x86 riscv hexagon wasm loong
------------------------------------------------------------------------
TransposeUVWx16_C - AVX512BW - - SIMD LASX
TransposeUVWx8_C NEON AVX512BW - HVX SIMD LASX
TransposeWx16_C NEON AVX512BW - - SIMD LASX
TransposeWx4_32_C * SVE2 AVX512BW RVV HVX SIMD LASX
TransposeWx8_16_C SVE2 AVX512BW RVV HVX SIMD LASX
TransposeWx8_C NEON AVX512BW - HVX SIMD LASX
TransposeWxH_32_C NEON AVX2 RVV HVX - -
------------------------------------------------------------------------
SIMD (7 rotate) 6 86% 7 100% 3 43% 5 71% 6 86% 6 86%
------------------------------------------------------------------------
Compare SIMD ISA per *_C:
------------------------------------------------------------------------
C Function arm x86 riscv hexagon wasm loong
------------------------------------------------------------------------
ARGBDetectRow_C - AVX2 - - - -
HammingDistance_C DOTPROD AVX512BW - HVX SIMD -
HashDjb2_C NEON AVX512BW - HVX SIMD -
Ssim4x4Row_C SME AVX512BW RVV HVX SIMD LASX
SsimWindowRow_C SME AVX512BW RVV - - -
SumSquareError_C DOTPROD AVX512VNNI - HVX SIMD -
------------------------------------------------------------------------
SIMD (6 compare) 5 83% 6 100% 2 33% 4 67% 4 67% 1 17%
------------------------------------------------------------------------
Scale SIMD ISA per *_C:
------------------------------------------------------------------------
C Function arm x86 riscv hexagon wasm loong
------------------------------------------------------------------------
FixedDiv1_C - X86 - - - -
FixedDiv_C - X86 - - - -
ScaleARGBCols64_C - - - - - -
ScaleARGBColsUp2_C - AVX2 - - SIMD -
ScaleARGBCols_C NEON AVX2 - - SIMD LSX
ScaleARGBFilterCols64_C - - - - - -
ScaleARGBFilterCols_C NEON AVX2 RVV - SIMD LSX
ScaleARGBRowDown2Box_C SME AVX2 RVV HVX SIMD LSX
ScaleARGBRowDown2Linear_C SME AVX2 RVV HVX SIMD LSX
ScaleARGBRowDown2_C SME AVX2 RVV HVX SIMD LSX
ScaleARGBRowDownEvenBox_C NEON AVX2 RVV - SIMD LSX
ScaleARGBRowDownEven_C NEON AVX2 - - SIMD LSX
ScaleAddCols0_C - - - - - -
ScaleAddCols1_16_C - - - - - -
ScaleAddCols1_C - AVX512BW - - - -
ScaleAddCols2_16_C - - - - - -
ScaleAddCols2_C - AVX512BW - - - -
ScaleAddRow_16_C - - - - - -
ScaleAddRow_C NEON AVX512BW RVV HVX SIMD LSX
ScaleColsUp2_16_C - - - - - -
ScaleColsUp2_C - AVX2 - - - -
ScaleCols_16_C - - - - - -
ScaleCols_C - - - - - -
ScaleFilterCols64_16_C - - - - - -
ScaleFilterCols64_C - - - - - -
ScaleFilterCols_16_C - - - - - -
ScaleFilterCols_C NEON AVX2 - - SIMD LSX
ScaleRowDown2Box_16_C SME AVX2 - HVX - -
ScaleRowDown2Box_C SME AVX512BW RVV HVX SIMD LSX
ScaleRowDown2Box_Odd_C - AVX512BW - HVX - -
ScaleRowDown2Linear_16_C SME AVX2 - HVX - -
ScaleRowDown2Linear_C SME AVX512BW RVV HVX SIMD LSX
ScaleRowDown2_16_C SME AVX2 - HVX - -
ScaleRowDown2_C SME AVX512BW RVV HVX SIMD LSX
ScaleRowDown34_0_Box_16_C - - - - - -
ScaleRowDown34_0_Box_C NEON AVX2 RVV - RELAXED LSX
ScaleRowDown34_16_C - - - - - -
ScaleRowDown34_1_Box_16_C - - - - - -
ScaleRowDown34_1_Box_C NEON AVX2 RVV - RELAXED LSX
ScaleRowDown34_C NEON AVX2 RVV - RELAXED LSX
ScaleRowDown38_16_C - - - - - -
ScaleRowDown38_2_Box_16_C - - - - - -
ScaleRowDown38_2_Box_C NEON AVX2 RVV - RELAXED LSX
ScaleRowDown38_3_Box_16_C - - - - - -
ScaleRowDown38_3_Box_C NEON AVX2 RVV - RELAXED LSX
ScaleRowDown38_C NEON AVX2 RVV - RELAXED LSX
ScaleRowDown4Box_16_C - - - - - -
ScaleRowDown4Box_C NEON AVX2 RVV HVX SIMD LSX
ScaleRowDown4_16_C - - - - - -
ScaleRowDown4_C NEON AVX2 RVV HVX SIMD LSX
ScaleRowUp2_Bilinear_16_Any_C NEON AVX2 - - - -
ScaleRowUp2_Bilinear_16_C NEON AVX2 - - - -
ScaleRowUp2_Bilinear_Any_C NEON AVX2 RVV - - -
ScaleRowUp2_Bilinear_C NEON AVX2 RVV - - -
ScaleRowUp2_Linear_16_Any_C NEON AVX2 - - - -
ScaleRowUp2_Linear_16_C NEON AVX2 - - - -
ScaleRowUp2_Linear_Any_C NEON AVX2 RVV - - -
ScaleRowUp2_Linear_C NEON AVX2 RVV - - -
ScaleUVAddCols1_16_C - - - - - -
ScaleUVAddCols1_C - - - - - -
ScaleUVAddCols2_16_C - - - - - -
ScaleUVAddCols2_C - - - - - -
ScaleUVAddCols3_16_C - - - - - -
ScaleUVAddCols3_C SME AVX512VBMI RVV HVX SIMD LSX
ScaleUVCols64_16_C - - - - - -
ScaleUVCols64_C - - - - - -
ScaleUVColsUp2_16_C - - - - - -
ScaleUVColsUp2_C - - - - - -
ScaleUVCols_16_C - - - - - -
ScaleUVCols_C - - - - - -
ScaleUVFilterCols64_16_C - - - - - -
ScaleUVFilterCols64_C - - - - - -
ScaleUVFilterCols_16_C - - - - - -
ScaleUVFilterCols_C - - - - - -
ScaleUVRowDown2Box_16_C - - - - - -
ScaleUVRowDown2Box_C SME AVX2 RVV HVX - -
ScaleUVRowDown2Linear_16_C - - - - - -
ScaleUVRowDown2Linear_C SME AVX2 RVV HVX - -
ScaleUVRowDown2_16_C - - - - - -
ScaleUVRowDown2_C SME AVX2 RVV HVX - -
ScaleUVRowDownEvenBox_16_C - - - - - -
ScaleUVRowDownEvenBox_C - - - - - -
ScaleUVRowDownEven_16_C - - - - - -
ScaleUVRowDownEven_C NEON AVX2 RVV - - -
ScaleUVRowUp2_Bilinear_16_Any_C NEON AVX2 - - - -
ScaleUVRowUp2_Bilinear_16_C NEON AVX2 - - - -
ScaleUVRowUp2_Bilinear_Any_C NEON AVX2 RVV - - -
ScaleUVRowUp2_Bilinear_C NEON AVX2 RVV - - -
ScaleUVRowUp2_Linear_16_Any_C NEON AVX2 - - - -
ScaleUVRowUp2_Linear_16_C NEON AVX2 - - - -
ScaleUVRowUp2_Linear_Any_C NEON AVX2 RVV - - -
ScaleUVRowUp2_Linear_C NEON AVX2 RVV - - -
------------------------------------------------------------------------
SIMD (92 scale) 44 48% 51 55% 30 33% 17 18% 22 24% 21 23%
------------------------------------------------------------------------
Test: libyuv_unittest (Release, row_win, x86_32, Sanitizers, GCC 15)
Bug: 388428508, 578615529, 579659306
Change-Id: Id4eaf5f7fd85c75a60984a3b1559bfdb07b21afc
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8686033
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard
parent
bb310b4e7e
commit
79ca9187e5
2
.gn
2
.gn
@ -32,8 +32,10 @@ exec_script_allowlist = build_dotfile_settings.exec_script_allowlist +
|
||||
[ "//build_overrides/build.gni" ]
|
||||
|
||||
default_args = {
|
||||
android_static_analysis = "off"
|
||||
mac_sdk_min = "10.12"
|
||||
ios_deployment_target = "12.0"
|
||||
|
||||
# Use Siso instead of Ninja.
|
||||
use_siso = true
|
||||
}
|
||||
|
||||
@ -34,13 +34,15 @@ extern "C" {
|
||||
#define HAS_HASHDJB2_SSE41
|
||||
#define HAS_SSIM4X4ROW_SSE2
|
||||
#define HAS_SSIMWINDOWROW_SSE2
|
||||
#if defined(VISUALC_HAS_AVX2)
|
||||
#define HAS_HAMMINGDISTANCE_AVX2
|
||||
#define HAS_SUMSQUAREERROR_AVX2
|
||||
#define HAS_HASHDJB2_AVX2
|
||||
#define HAS_SSIM4X4ROW_AVX2
|
||||
#define HAS_SSIMWINDOWROW_AVX2
|
||||
#define HAS_ARGBDETECTROW_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#endif
|
||||
#if defined(VISUALC_HAS_AVX512)
|
||||
#define HAS_HAMMINGDISTANCE_AVX512BW
|
||||
#define HAS_HAMMINGDISTANCE_AVX512VNNI
|
||||
#define HAS_SUMSQUAREERROR_AVX512BW
|
||||
@ -73,7 +75,8 @@ extern "C" {
|
||||
#define HAS_SSIMWINDOWROW_AVX2
|
||||
#define HAS_ARGBDETECTROW_AVX2
|
||||
#endif
|
||||
#if defined(__x86_64__) && defined(CLANG_HAS_AVX512)
|
||||
#if defined(__x86_64__) && \
|
||||
(defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512))
|
||||
#define HAS_HAMMINGDISTANCE_AVX512BW
|
||||
#define HAS_HAMMINGDISTANCE_AVX512VNNI
|
||||
#define HAS_SUMSQUAREERROR_AVX512BW
|
||||
|
||||
@ -55,13 +55,42 @@ extern "C" {
|
||||
#endif // __clang__
|
||||
|
||||
// GCC >= 4.7.0 required for AVX2.
|
||||
#if defined(__GNUC__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
#if defined(__GNUC__) && !defined(__clang__) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(defined(__x86_64__) || defined(__i386__))
|
||||
#if (__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 7))
|
||||
#define GCC_HAS_AVX2 1
|
||||
#endif // GNUC >= 4.7
|
||||
#endif // __GNUC__
|
||||
|
||||
// GCC >= 9.0.0 required for AVX512 (including VNNI and VBMI).
|
||||
#if defined(__GNUC__) && !defined(__clang__) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && defined(__x86_64__)
|
||||
#if (__GNUC__ >= 9)
|
||||
#define GCC_HAS_AVX512 1
|
||||
#endif // GNUC >= 9
|
||||
#endif // __GNUC__
|
||||
|
||||
// Function target attributes so GCC enables zmm16..zmm31 and k1..k7 clobbers
|
||||
// inside AVX512 functions without emitting AVX512 in non-AVX512 functions.
|
||||
#if defined(GCC_HAS_AVX512)
|
||||
#define LIBYUV_TARGET_AVX512BW \
|
||||
__attribute__((target("avx512bw,avx512vl,avx512f,bmi2")))
|
||||
#define LIBYUV_TARGET_AVX512VBMI \
|
||||
__attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f,bmi2")))
|
||||
#define LIBYUV_TARGET_AVX512VNNI \
|
||||
__attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f,bmi2")))
|
||||
#define LIBYUV_TARGET_AVX512VNNIVBMI \
|
||||
__attribute__(( \
|
||||
target("avx512vnni,avx512vbmi,avx512bw,avx512vl,avx512f,bmi2")))
|
||||
#elif !defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
!(defined(_MSC_VER) && !defined(__clang__))
|
||||
#define LIBYUV_TARGET_AVX512BW
|
||||
#define LIBYUV_TARGET_AVX512VBMI
|
||||
#define LIBYUV_TARGET_AVX512VNNI
|
||||
#define LIBYUV_TARGET_AVX512VNNIVBMI
|
||||
#endif
|
||||
|
||||
// clang >= 3.4.0 required for AVX2.
|
||||
#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(defined(__x86_64__) || defined(__i386__))
|
||||
@ -70,7 +99,7 @@ extern "C" {
|
||||
#endif // clang >= 3.4
|
||||
#endif // __clang__
|
||||
|
||||
// clang >= 6.0.0 required for AVX512.
|
||||
// clang >= 7.0.0 required for AVX512.
|
||||
#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(defined(__x86_64__) || defined(__i386__))
|
||||
// clang in xcode follows a different versioning scheme.
|
||||
@ -80,13 +109,25 @@ extern "C" {
|
||||
#endif // clang >= 7
|
||||
#endif // __clang__
|
||||
|
||||
// Visual C 2012 required for AVX2.
|
||||
#if (defined(_M_IX86) || defined(_M_X64)) && \
|
||||
(!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN)) && \
|
||||
defined(_MSC_VER) && _MSC_VER >= 1700
|
||||
// Visual C 2012 (or clang >= 3.4 with LIBYUV_ENABLE_ROWWIN) required for AVX2.
|
||||
#if ((defined(_M_IX86) || defined(_M_X64)) && !defined(__clang__) && \
|
||||
defined(_MSC_VER) && _MSC_VER >= 1700) || \
|
||||
(defined(LIBYUV_ENABLE_ROWWIN) && defined(__clang__) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
||||
defined(_M_IX86)) && \
|
||||
((__clang_major__ > 3) || \
|
||||
(__clang_major__ == 3 && (__clang_minor__ >= 4))))
|
||||
#define VISUALC_HAS_AVX2 1
|
||||
#endif // VisualStudio >= 2012
|
||||
|
||||
// Visual C 2019 (or clang >= 7 with LIBYUV_ENABLE_ROWWIN) required for AVX512.
|
||||
#if (defined(_M_X64) || defined(__x86_64__)) && \
|
||||
((!defined(__clang__) && defined(_MSC_VER) && _MSC_VER >= 1920) || \
|
||||
(defined(LIBYUV_ENABLE_ROWWIN) && defined(__clang__) && \
|
||||
(__clang_major__ >= 7) && !defined(__APPLE__)))
|
||||
#define VISUALC_HAS_AVX512 1
|
||||
#endif // VisualStudio >= 2019
|
||||
|
||||
#if defined(LIBYUV_ENABLE_SME)
|
||||
#undef LIBYUV_DISABLE_SME
|
||||
#endif
|
||||
|
||||
@ -29,6 +29,7 @@ extern "C" {
|
||||
defined(_M_IX86)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
#if defined(VISUALC_HAS_AVX2)
|
||||
#define HAS_TRANSPOSEWX8_AVX2
|
||||
#define HAS_TRANSPOSEUVWX8_AVX2
|
||||
#define HAS_TRANSPOSEWX8_16_AVX2
|
||||
@ -38,7 +39,8 @@ extern "C" {
|
||||
#define HAS_TRANSPOSEWXH_32_AVX2
|
||||
#define HAS_TRANSPOSEWX16_AVX2
|
||||
#define HAS_TRANSPOSEUVWX16_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#endif
|
||||
#if defined(VISUALC_HAS_AVX512)
|
||||
#define HAS_TRANSPOSEWX16_AVX512BW
|
||||
#define HAS_TRANSPOSEUVWX16_AVX512BW
|
||||
#define HAS_TRANSPOSEWX32_16_AVX512BW
|
||||
@ -48,9 +50,10 @@ extern "C" {
|
||||
|
||||
// The following are available for 32/64 bit GCC and clang, as inline assembly
|
||||
// in rotate_gcc.cc:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
(defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(!defined(_MSC_VER) || defined(__clang__))
|
||||
#define HAS_TRANSPOSEWX8_AVX2
|
||||
#define HAS_TRANSPOSEUVWX8_AVX2
|
||||
@ -63,9 +66,10 @@ extern "C" {
|
||||
#define HAS_TRANSPOSEUVWX16_AVX2
|
||||
#endif
|
||||
|
||||
// The following are available for AVX512 clang x64 platforms:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
||||
defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
// The following are available for AVX512 GCC/clang x64 platforms:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
||||
(defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(!defined(_MSC_VER) || defined(__clang__))
|
||||
#define HAS_TRANSPOSEWX16_AVX512BW
|
||||
#define HAS_TRANSPOSEUVWX16_AVX512BW
|
||||
|
||||
@ -90,9 +90,9 @@ extern "C" {
|
||||
|
||||
// The following are available on all x86 platforms, but
|
||||
// require VS2012, clang 3.4 or gcc 4.7.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
||||
defined(_M_IX86))
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
|
||||
defined(GCC_HAS_AVX2))
|
||||
#define HAS_ARGBMIRRORROW_AVX2
|
||||
#define HAS_RGB24MIRRORROW_AVX2
|
||||
#define HAS_ARGBTOUVMATRIXROW_AVX2
|
||||
@ -181,6 +181,7 @@ extern "C" {
|
||||
#define HAS_I422TOAR30ROW_SSSE3
|
||||
#define HAS_MERGEARGBROW_SSE2
|
||||
#define HAS_MERGEXRGBROW_SSE2
|
||||
#define HAS_MIRRORSPLITUVROW_SSSE3
|
||||
#define HAS_MIRRORUVROW_SSSE3
|
||||
#define HAS_NV21TOYUV24ROW_SSSE3
|
||||
#define HAS_P210TOAR30ROW_SSSE3
|
||||
@ -188,11 +189,6 @@ extern "C" {
|
||||
#define HAS_P210TOARGBROW_SSSE3
|
||||
#define HAS_P410TOAR30ROW_SSSE3
|
||||
#define HAS_P410TOARGBROW_SSSE3
|
||||
#define HAS_RAWTOARGBROW_AVX2
|
||||
#define HAS_RGB24TOARGBROW_AVX2
|
||||
#define HAS_RGB565TOARGBROW_AVX2
|
||||
#define HAS_ARGB1555TOARGBROW_AVX2
|
||||
#define HAS_ARGB4444TOARGBROW_AVX2
|
||||
#define HAS_SPLITARGBROW_SSE2
|
||||
#define HAS_SPLITARGBROW_SSSE3
|
||||
#define HAS_SPLITRGBROW_SSE41
|
||||
@ -233,11 +229,11 @@ extern "C" {
|
||||
#define HAS_ARGBTORGB565ROW_AVX2
|
||||
#define HAS_ARGBTOARGB1555ROW_AVX2
|
||||
#define HAS_ARGBTOARGB4444ROW_AVX2
|
||||
#define HAS_ARGBTOUVMATRIXROW_AVX2
|
||||
#define HAS_RGBTOUVMATRIXROW_AVX2
|
||||
#define HAS_RGB565TOUVMATRIXROW_AVX2
|
||||
#define HAS_ARGB1555TOUVMATRIXROW_AVX2
|
||||
#define HAS_ARGB4444TOUVMATRIXROW_AVX2
|
||||
#define HAS_RAWTOARGBROW_AVX2
|
||||
#define HAS_RGB24TOARGBROW_AVX2
|
||||
#define HAS_RGB565TOARGBROW_AVX2
|
||||
#define HAS_ARGB1555TOARGBROW_AVX2
|
||||
#define HAS_ARGB4444TOARGBROW_AVX2
|
||||
#define HAS_ARGBTOUV444MATRIXROW_AVX2
|
||||
#define HAS_RGBTOUV444MATRIXROW_AVX2
|
||||
#define HAS_ARGBTOYMATRIXROW_AVX2
|
||||
@ -318,7 +314,6 @@ extern "C" {
|
||||
#define HAS_MERGEXR64ROW_AVX2
|
||||
#define HAS_MERGEXRGB16TO8ROW_AVX2
|
||||
#define HAS_MERGEXRGBROW_AVX2
|
||||
#define HAS_MIRRORUVROW_AVX2
|
||||
#define HAS_MULTIPLYROW_16_AVX2
|
||||
#define HAS_NV21TOYUV24ROW_AVX2
|
||||
#define HAS_P210TOAR30ROW_AVX2
|
||||
@ -343,6 +338,10 @@ extern "C" {
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
#define HAS_RAWTORGB24ROW_SSSE3
|
||||
#define HAS_ARGBALPHABLENDROW_SSSE3
|
||||
#define HAS_ARGBORROW_SSE2
|
||||
#define HAS_MIRRORSPLITUVROW_SSSE3
|
||||
#if defined(VISUALC_HAS_AVX2)
|
||||
#define HAS_RAWTORGB24ROW_AVX2
|
||||
#define HAS_RAWTOARGBROW_AVX2
|
||||
#define HAS_I422TORGB24ROW_AVX2
|
||||
@ -360,6 +359,7 @@ extern "C" {
|
||||
#define HAS_I422TOARGB4444ROW_AVX2
|
||||
#define HAS_NV12TORGB565ROW_AVX2
|
||||
#define HAS_CONVERT8TO16ROW_AVX2
|
||||
#define HAS_CONVERT8TO8ROW_AVX2
|
||||
#define HAS_I422TOAR30ROW_AVX2
|
||||
#define HAS_AR30TOAB30ROW_AVX2
|
||||
#define HAS_AR30TOABGRROW_AVX2
|
||||
@ -367,7 +367,108 @@ extern "C" {
|
||||
#define HAS_AYUVTOUVROW_AVX2
|
||||
#define HAS_AYUVTOVUROW_AVX2
|
||||
#define HAS_AYUVTOYROW_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define HAS_ARGB1555TOUVROW_AVX2
|
||||
#define HAS_ARGB1555TOYROW_AVX2
|
||||
#define HAS_ARGB4444TOUVROW_AVX2
|
||||
#define HAS_ARGB4444TOYROW_AVX2
|
||||
#define HAS_ARGBAFFINEROW_AVX2
|
||||
#define HAS_ARGBCOLORMATRIXROW_AVX2
|
||||
#define HAS_ARGBCOLORTABLEROW_AVX2
|
||||
#define HAS_ARGBLUMACOLORTABLEROW_AVX2
|
||||
#define HAS_ARGBQUANTIZEROW_AVX2
|
||||
#define HAS_ARGBSEPIAROW_AVX2
|
||||
#define HAS_ARGBSETROW_AVX2
|
||||
#define HAS_ARGBSHADEROW_AVX2
|
||||
#define HAS_BYTETOFLOATROW_AVX2
|
||||
#define HAS_COPYROW_16_AVX2
|
||||
#define HAS_DETILEROW_16_AVX2
|
||||
#define HAS_DETILEROW_AVX2
|
||||
#define HAS_DETILESPLITUVROW_AVX2
|
||||
#define HAS_GAUSSROW_AVX2
|
||||
#define HAS_GAUSSROW_F32_AVX2
|
||||
#define HAS_RGB565TOUVROW_AVX2
|
||||
#define HAS_RGB565TOYROW_AVX2
|
||||
#define HAS_RGBCOLORTABLEROW_AVX2
|
||||
#define HAS_SETROW_AVX2
|
||||
#define HAS_SOBELROW_AVX2
|
||||
#define HAS_SOBELTOPLANEROW_AVX2
|
||||
#define HAS_SOBELXROW_AVX2
|
||||
#define HAS_SOBELXYROW_AVX2
|
||||
#define HAS_SOBELYROW_AVX2
|
||||
#define HAS_ARGBALPHABLENDROW_AVX2
|
||||
#define HAS_ARGBORROW_AVX2
|
||||
#define HAS_ARGBTOYMATRIXROW_AVX2
|
||||
#define HAS_RGBTOYMATRIXROW_AVX2
|
||||
#define HAS_RGB565TOYMATRIXROW_AVX2
|
||||
#define HAS_ARGB1555TOYMATRIXROW_AVX2
|
||||
#define HAS_ARGB4444TOYMATRIXROW_AVX2
|
||||
#define HAS_ARGBTOUV444MATRIXROW_AVX2
|
||||
#define HAS_RGBTOUV444MATRIXROW_AVX2
|
||||
#define HAS_J400TOARGBROW_AVX2
|
||||
#define HAS_MULTIPLYROW_16_AVX2
|
||||
#define HAS_SWAPUVROW_AVX2
|
||||
#define HAS_AB64TOARGBROW_AVX2
|
||||
#define HAS_AR64TOARGBROW_AVX2
|
||||
#define HAS_ARGBADDROW_AVX2
|
||||
#define HAS_ARGBBLENDROW_AVX2
|
||||
#define HAS_ARGBCOPYALPHAROW_AVX2
|
||||
#define HAS_ARGBGRAYROW_AVX2
|
||||
#define HAS_ARGBPOLYNOMIALROW_AVX2
|
||||
#define HAS_ARGBSUBTRACTROW_AVX2
|
||||
#define HAS_ARGBTOAB64ROW_AVX2
|
||||
#define HAS_ARGBTOAR64ROW_AVX2
|
||||
#define HAS_ARGBTORAWROW_AVX2
|
||||
#define HAS_ARGBTORGB24ROW_AVX2
|
||||
#define HAS_ARGBUNATTENUATEROW_AVX2
|
||||
#define HAS_COMPUTECUMULATIVESUMROW_AVX2
|
||||
#define HAS_CONVERT16TO8ROW_AVX2
|
||||
#define HAS_COPYROW_AVX
|
||||
#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
|
||||
#define HAS_DETILETOYUY2_AVX2
|
||||
#define HAS_DIVIDEROW_16_AVX2
|
||||
#define HAS_GAUSSCOL_AVX2
|
||||
#define HAS_GAUSSCOL_F32_AVX2
|
||||
#define HAS_HALFMERGEUVROW_AVX2
|
||||
#define HAS_HALFROW_16_AVX2
|
||||
#define HAS_HALFROW_16TO8_AVX2
|
||||
#define HAS_HALFROW_AVX2
|
||||
#define HAS_HALFWIDTHROW_16TO8_AVX2
|
||||
#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2
|
||||
#define HAS_I422TOUYVYROW_AVX2
|
||||
#define HAS_I422TOYUY2ROW_AVX2
|
||||
#define HAS_I444TOARGBROW_AVX2
|
||||
#define HAS_I444TORGB24ROW_AVX2
|
||||
#define HAS_MERGEAR64ROW_AVX2
|
||||
#define HAS_MERGEARGB16TO8ROW_AVX2
|
||||
#define HAS_MERGEARGBROW_AVX2
|
||||
#define HAS_MERGERGBROW_AVX2
|
||||
#define HAS_MERGEUVROW_16_AVX2
|
||||
#define HAS_MERGEXR30ROW_AVX2
|
||||
#define HAS_MERGEXR64ROW_AVX2
|
||||
#define HAS_MERGEXRGB16TO8ROW_AVX2
|
||||
#define HAS_MERGEXRGBROW_AVX2
|
||||
#define HAS_NV12TORGB24ROW_AVX2
|
||||
#define HAS_NV21TORGB24ROW_AVX2
|
||||
#define HAS_NV21TOYUV24ROW_AVX2
|
||||
#define HAS_RAWTORGBAROW_AVX2
|
||||
#define HAS_SCALEMAXSAMPLES_AVX2
|
||||
#define HAS_SCALESAMPLES_AVX2
|
||||
#define HAS_SCALESUMSAMPLES_AVX2
|
||||
#define HAS_SPLITARGBROW_AVX2
|
||||
#define HAS_SPLITRGBROW_AVX2
|
||||
#define HAS_SPLITUVROW_16_AVX2
|
||||
#define HAS_SPLITUVROW_AVX2
|
||||
#define HAS_SPLITXRGBROW_AVX2
|
||||
#define HAS_UNPACKMT2T_AVX2
|
||||
#define HAS_UYVYTOUV422ROW_AVX2
|
||||
#define HAS_UYVYTOUVROW_AVX2
|
||||
#define HAS_UYVYTOYROW_AVX2
|
||||
#define HAS_YUY2TONVUVROW_AVX2
|
||||
#define HAS_YUY2TOUV422ROW_AVX2
|
||||
#define HAS_YUY2TOUVROW_AVX2
|
||||
#define HAS_YUY2TOYROW_AVX2
|
||||
#endif
|
||||
#if defined(VISUALC_HAS_AVX512)
|
||||
#define HAS_RAWTOARGBROW_AVX512BW
|
||||
#define HAS_RGB24TOARGBROW_AVX512BW
|
||||
#define HAS_RAWTORGB24ROW_AVX512BW
|
||||
@ -397,6 +498,7 @@ extern "C" {
|
||||
#define HAS_NV21TOARGBROW_AVX512BW
|
||||
#define HAS_NV12TORGB24ROW_AVX512VBMI
|
||||
#define HAS_NV21TORGB24ROW_AVX512VBMI
|
||||
#define HAS_NV21TOYUV24ROW_AVX512VBMI
|
||||
#define HAS_I422ALPHATOARGBROW_AVX512BW
|
||||
#define HAS_SWAPUVROW_AVX512BW
|
||||
#define HAS_HALFMERGEUVROW_AVX512BW
|
||||
@ -453,120 +555,6 @@ extern "C" {
|
||||
#define HAS_YUY2TOARGBROW_AVX512BW
|
||||
#define HAS_YUY2TOUVROW_AVX512BW
|
||||
#define HAS_YUY2TOYROW_AVX512BW
|
||||
#endif
|
||||
#define HAS_ARGB1555TOUVROW_AVX2
|
||||
#define HAS_ARGB1555TOYROW_AVX2
|
||||
#define HAS_ARGB4444TOUVROW_AVX2
|
||||
#define HAS_ARGB4444TOYROW_AVX2
|
||||
#define HAS_ARGBAFFINEROW_AVX2
|
||||
#define HAS_ARGBCOLORMATRIXROW_AVX2
|
||||
#define HAS_ARGBCOLORTABLEROW_AVX2
|
||||
#define HAS_ARGBLUMACOLORTABLEROW_AVX2
|
||||
#define HAS_ARGBQUANTIZEROW_AVX2
|
||||
#define HAS_ARGBSEPIAROW_AVX2
|
||||
#define HAS_ARGBSETROW_AVX2
|
||||
#define HAS_ARGBSHADEROW_AVX2
|
||||
#define HAS_BYTETOFLOATROW_AVX2
|
||||
#define HAS_COPYROW_16_AVX2
|
||||
#define HAS_DETILEROW_16_AVX2
|
||||
#define HAS_DETILEROW_AVX2
|
||||
#define HAS_DETILESPLITUVROW_AVX2
|
||||
#define HAS_GAUSSROW_AVX2
|
||||
#define HAS_GAUSSROW_F32_AVX2
|
||||
#define HAS_RGB565TOUVROW_AVX2
|
||||
#define HAS_RGB565TOYROW_AVX2
|
||||
#define HAS_RGBCOLORTABLEROW_AVX2
|
||||
#define HAS_SETROW_AVX2
|
||||
#define HAS_SOBELROW_AVX2
|
||||
#define HAS_SOBELTOPLANEROW_AVX2
|
||||
#define HAS_SOBELXROW_AVX2
|
||||
#define HAS_SOBELXYROW_AVX2
|
||||
#define HAS_SOBELYROW_AVX2
|
||||
#define HAS_ARGBALPHABLENDROW_SSSE3
|
||||
#define HAS_ARGBORROW_SSE2
|
||||
#define HAS_ARGBALPHABLENDROW_AVX2
|
||||
#define HAS_ARGBORROW_AVX2
|
||||
#define HAS_ARGBTOYMATRIXROW_AVX2
|
||||
#define HAS_RGBTOYMATRIXROW_AVX2
|
||||
#define HAS_RGB565TOYMATRIXROW_AVX2
|
||||
#define HAS_ARGB1555TOYMATRIXROW_AVX2
|
||||
#define HAS_ARGB4444TOYMATRIXROW_AVX2
|
||||
#define HAS_ARGBTOUVMATRIXROW_AVX2
|
||||
#define HAS_ARGBTOUV444MATRIXROW_AVX2
|
||||
#define HAS_RGBTOUV444MATRIXROW_AVX2
|
||||
#define HAS_J400TOARGBROW_AVX2
|
||||
#define HAS_I422TOARGBROW_AVX2
|
||||
#define HAS_NV12TOARGBROW_AVX2
|
||||
#define HAS_NV21TOARGBROW_AVX2
|
||||
#define HAS_MERGEUVROW_AVX2
|
||||
#define HAS_MIRRORROW_AVX2
|
||||
#define HAS_MIRRORUVROW_AVX2
|
||||
#define HAS_MIRRORSPLITUVROW_AVX2
|
||||
#define HAS_RGB24MIRRORROW_AVX2
|
||||
#define HAS_ARGBMIRRORROW_AVX2
|
||||
#define HAS_INTERPOLATEROW_AVX2
|
||||
#define HAS_INTERPOLATEROW_16_AVX2
|
||||
#define HAS_MULTIPLYROW_16_AVX2
|
||||
#define HAS_SWAPUVROW_AVX2
|
||||
#define HAS_AB64TOARGBROW_AVX2
|
||||
#define HAS_AR64TOARGBROW_AVX2
|
||||
#define HAS_ARGBADDROW_AVX2
|
||||
#define HAS_ARGBBLENDROW_AVX2
|
||||
#define HAS_ARGBCOPYALPHAROW_AVX2
|
||||
#define HAS_ARGBGRAYROW_AVX2
|
||||
#define HAS_ARGBPOLYNOMIALROW_AVX2
|
||||
#define HAS_ARGBSUBTRACTROW_AVX2
|
||||
#define HAS_ARGBTOAB64ROW_AVX2
|
||||
#define HAS_ARGBTOAR64ROW_AVX2
|
||||
#define HAS_ARGBTORAWROW_AVX2
|
||||
#define HAS_ARGBTORGB24ROW_AVX2
|
||||
#define HAS_ARGBUNATTENUATEROW_AVX2
|
||||
#define HAS_COMPUTECUMULATIVESUMROW_AVX2
|
||||
#define HAS_CONVERT16TO8ROW_AVX2
|
||||
#define HAS_COPYROW_AVX
|
||||
#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2
|
||||
#define HAS_DETILETOYUY2_AVX2
|
||||
#define HAS_GAUSSCOL_AVX2
|
||||
#define HAS_GAUSSCOL_F32_AVX2
|
||||
#define HAS_HALFROW_16_AVX2
|
||||
#define HAS_HALFROW_16TO8_AVX2
|
||||
#define HAS_HALFROW_AVX2
|
||||
#define HAS_HALFWIDTHROW_16TO8_AVX2
|
||||
#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2
|
||||
#define HAS_I422TOUYVYROW_AVX2
|
||||
#define HAS_I422TOYUY2ROW_AVX2
|
||||
#define HAS_I444TOARGBROW_AVX2
|
||||
#define HAS_I444TORGB24ROW_AVX2
|
||||
#define HAS_MERGEAR64ROW_AVX2
|
||||
#define HAS_MERGEARGB16TO8ROW_AVX2
|
||||
#define HAS_MERGEARGBROW_AVX2
|
||||
#define HAS_MERGERGBROW_AVX2
|
||||
#define HAS_MERGEUVROW_16_AVX2
|
||||
#define HAS_MERGEXR30ROW_AVX2
|
||||
#define HAS_MERGEXR64ROW_AVX2
|
||||
#define HAS_MERGEXRGB16TO8ROW_AVX2
|
||||
#define HAS_MERGEXRGBROW_AVX2
|
||||
#define HAS_NV12TORGB24ROW_AVX2
|
||||
#define HAS_NV21TORGB24ROW_AVX2
|
||||
#define HAS_NV21TOYUV24ROW_AVX2
|
||||
#define HAS_RAWTORGBAROW_AVX2
|
||||
#define HAS_SCALEMAXSAMPLES_AVX2
|
||||
#define HAS_SCALESAMPLES_AVX2
|
||||
#define HAS_SCALESUMSAMPLES_AVX2
|
||||
#define HAS_SPLITARGBROW_AVX2
|
||||
#define HAS_SPLITRGBROW_AVX2
|
||||
#define HAS_SPLITUVROW_16_AVX2
|
||||
#define HAS_SPLITUVROW_AVX2
|
||||
#define HAS_SPLITXRGBROW_AVX2
|
||||
#define HAS_UNPACKMT2T_AVX2
|
||||
#define HAS_UYVYTOUV422ROW_AVX2
|
||||
#define HAS_UYVYTOUVROW_AVX2
|
||||
#define HAS_UYVYTOYROW_AVX2
|
||||
#define HAS_YUY2TONVUVROW_AVX2
|
||||
#define HAS_YUY2TOUV422ROW_AVX2
|
||||
#define HAS_YUY2TOUVROW_AVX2
|
||||
#define HAS_YUY2TOYROW_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define HAS_ARGBTORAWROW_AVX512BW
|
||||
#define HAS_ARGBTORGB24ROW_AVX512BW
|
||||
#define HAS_ARGBTOUV444MATRIXROW_AVX512BW
|
||||
@ -589,10 +577,10 @@ extern "C" {
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// The following are available for AVX512 clang x64 platforms:
|
||||
// TODO(fbarchard): Port to GCC and Visual C
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512)
|
||||
// The following are available for AVX512 GCC/clang x64 platforms:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_GCC) && \
|
||||
defined(__x86_64__) && \
|
||||
(defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512))
|
||||
#define HAS_COPYROW_AVX512BW
|
||||
#define HAS_RAWTOARGBROW_AVX512BW
|
||||
#define HAS_RGB24TOARGBROW_AVX512BW
|
||||
@ -605,12 +593,6 @@ extern "C" {
|
||||
#define HAS_HALFROW_16TO8_AVX512BW
|
||||
#define HAS_HALFWIDTHROW_16TO8_AVX512BW
|
||||
#define HAS_MULTIPLYROW_16_AVX512BW
|
||||
#endif
|
||||
|
||||
// The following are available for AVX512 clang x64 platforms:
|
||||
// TODO(fbarchard): Port to x86
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512)
|
||||
#define HAS_I422TOARGBROW_AVX512BW
|
||||
#define HAS_I422TOAR30ROW_AVX512BW
|
||||
#define HAS_ARGBSHUFFLEROW_AVX512BW
|
||||
@ -648,6 +630,7 @@ extern "C" {
|
||||
#define HAS_NV21TOARGBROW_AVX512BW
|
||||
#define HAS_NV12TORGB24ROW_AVX512VBMI
|
||||
#define HAS_NV21TORGB24ROW_AVX512VBMI
|
||||
#define HAS_NV21TOYUV24ROW_AVX512VBMI
|
||||
#define HAS_I422ALPHATOARGBROW_AVX512BW
|
||||
#define HAS_SWAPUVROW_AVX512BW
|
||||
#define HAS_HALFMERGEUVROW_AVX512BW
|
||||
@ -2611,18 +2594,6 @@ void ARGBToYMatrixRow_Any_AVX2(const uint8_t* src_argb,
|
||||
uint8_t* dst_y,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
void ARGBToYMatrixRow_Any_AVX512BW(const uint8_t* src_argb,
|
||||
uint8_t* dst_y,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
void ARGBToYMatrixRow_Any_AVX512VNNI(const uint8_t* src_argb,
|
||||
uint8_t* dst_y,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
void ARGBToYMatrixRow_Any_AVX512VNNIVBMI(const uint8_t* src_argb,
|
||||
uint8_t* dst_y,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
|
||||
void RGBToYMatrixRow_NEON(const uint8_t* src_rgb,
|
||||
uint8_t* dst_y,
|
||||
@ -2726,10 +2697,6 @@ void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
|
||||
uint8_t* dst_y,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
void RGBToYMatrixRow_Any_AVX512BW(const uint8_t* src_rgb,
|
||||
uint8_t* dst_y,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
|
||||
void ARGBToYMatrixRow_Any_NEON_DotProd(const uint8_t* src_argb,
|
||||
uint8_t* dst_y,
|
||||
@ -2826,21 +2793,6 @@ void ARGBToUV444MatrixRow_Any_AVX2(const uint8_t* src_argb,
|
||||
uint8_t* dst_v,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
void ARGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_argb,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
void ARGBToUV444MatrixRow_Any_AVX512VNNI(const uint8_t* src_argb,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
void ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI(const uint8_t* src_argb,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
void RGBToUV444MatrixRow_Any_SSSE3(const uint8_t* src_rgb,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
@ -2851,11 +2803,6 @@ void RGBToUV444MatrixRow_Any_AVX2(const uint8_t* src_rgb,
|
||||
uint8_t* dst_v,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
void RGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_rgb,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width,
|
||||
const struct ArgbConstants* c);
|
||||
|
||||
// Deprecated.
|
||||
void RGB565ToYRow_C(const uint8_t* src_rgb565, uint8_t* dst_y, int width);
|
||||
@ -3041,6 +2988,18 @@ void MirrorSplitUVRow_AVX2(const uint8_t* src,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width);
|
||||
void MirrorSplitUVRow_SSSE3(const uint8_t* src,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width);
|
||||
void MirrorSplitUVRow_Any_AVX2(const uint8_t* src_uv,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width);
|
||||
void MirrorSplitUVRow_Any_SSSE3(const uint8_t* src_uv,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width);
|
||||
void MirrorSplitUVRow_NEON(const uint8_t* src_uv,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
@ -3147,10 +3106,6 @@ void SplitUVRow_RVV(const uint8_t* src_uv,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width);
|
||||
void SplitUVRow_Any_AVX512BW(const uint8_t* src_ptr,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width);
|
||||
void SplitUVRow_Any_SSE2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
@ -3317,10 +3272,6 @@ void MergeUVRow_Any_AVX2(const uint8_t* y_buf,
|
||||
const uint8_t* uv_buf,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void MergeUVRow_Any_AVX512BW(const uint8_t* y_buf,
|
||||
const uint8_t* uv_buf,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void MergeUVRow_Any_NEON(const uint8_t* y_buf,
|
||||
const uint8_t* uv_buf,
|
||||
uint8_t* dst_ptr,
|
||||
@ -3374,6 +3325,18 @@ void HalfMergeUVRow_AVX512BW(const uint8_t* src_u,
|
||||
int src_stride_v,
|
||||
uint8_t* dst_uv,
|
||||
int width);
|
||||
void HalfMergeUVRow_Any_SSSE3(const uint8_t* src_u,
|
||||
int src_stride_u,
|
||||
const uint8_t* src_v,
|
||||
int src_stride_v,
|
||||
uint8_t* dst_uv,
|
||||
int width);
|
||||
void HalfMergeUVRow_Any_AVX2(const uint8_t* src_u,
|
||||
int src_stride_u,
|
||||
const uint8_t* src_v,
|
||||
int src_stride_v,
|
||||
uint8_t* dst_uv,
|
||||
int width);
|
||||
|
||||
void SplitRGBRow_C(const uint8_t* src_rgb,
|
||||
uint8_t* dst_r,
|
||||
@ -3917,10 +3880,6 @@ void MultiplyRow_16_AVX512BW(const uint16_t* src_y,
|
||||
uint16_t* dst_y,
|
||||
int scale,
|
||||
int width);
|
||||
void MultiplyRow_16_Any_AVX512BW(const uint16_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
int scale,
|
||||
int width);
|
||||
void MultiplyRow_16_NEON(const uint16_t* src_y,
|
||||
uint16_t* dst_y,
|
||||
int scale,
|
||||
@ -3991,10 +3950,6 @@ void Convert8To16Row_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
int bits,
|
||||
int width);
|
||||
void Convert8To16Row_Any_AVX512BW(const uint8_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
int bits,
|
||||
int width);
|
||||
void Convert8To16Row_NEON(const uint8_t* src_y,
|
||||
uint16_t* dst_y,
|
||||
int bits,
|
||||
@ -4036,10 +3991,6 @@ void Convert16To8Row_Any_AVX2(const uint16_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int scale,
|
||||
int width);
|
||||
void Convert16To8Row_Any_AVX512BW(const uint16_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int scale,
|
||||
int width);
|
||||
void Convert16To8Row_NEON(const uint16_t* src_y,
|
||||
uint8_t* dst_y,
|
||||
int scale,
|
||||
@ -4117,11 +4068,6 @@ void HalfRow_16To8_Any_AVX2(const uint16_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int scale,
|
||||
int width);
|
||||
void HalfRow_16To8_Any_AVX512BW(const uint16_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int scale,
|
||||
int width);
|
||||
void HalfRow_16To8_NEON(const uint16_t* src_uv,
|
||||
ptrdiff_t src_uv_stride,
|
||||
uint8_t* dst_uv,
|
||||
@ -4188,11 +4134,6 @@ void HalfWidthRow_16To8_Any_AVX2(const uint16_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int scale,
|
||||
int width);
|
||||
void HalfWidthRow_16To8_Any_AVX512BW(const uint16_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int scale,
|
||||
int width);
|
||||
void HalfWidthRow_16To8_NEON(const uint16_t* src_uv,
|
||||
ptrdiff_t src_uv_stride,
|
||||
uint8_t* dst_uv,
|
||||
@ -4603,12 +4544,6 @@ void RAWToRGB24Row_Any_SSSE3(const uint8_t* src_ptr,
|
||||
void RAWToRGB24Row_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void RAWToRGB24Row_Any_AVX512BW(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void RAWToRGB24Row_Any_AVX512VBMI(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void RGB565ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
@ -4867,9 +4802,6 @@ void J400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, int width);
|
||||
void J400ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, int width);
|
||||
void J400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, int width);
|
||||
void J400ToARGBRow_C(const uint8_t* src_y, uint8_t* dst_argb, int width);
|
||||
void J400ToARGBRow_Any_AVX512BW(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void J400ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
@ -5453,6 +5385,10 @@ void NV21ToYUV24Row_AVX2(const uint8_t* src_y,
|
||||
const uint8_t* src_vu,
|
||||
uint8_t* dst_yuv24,
|
||||
int width);
|
||||
void NV21ToYUV24Row_AVX512VBMI(const uint8_t* src_y,
|
||||
const uint8_t* src_vu,
|
||||
uint8_t* dst_yuv24,
|
||||
int width);
|
||||
|
||||
void NV21ToARGBRow_SSSE3(const uint8_t* y_buf,
|
||||
const uint8_t* vu_buf,
|
||||
@ -7825,6 +7761,14 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
int width);
|
||||
void ARGBPolynomialRow_Any_SSE2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
const float* param,
|
||||
int width);
|
||||
void ARGBPolynomialRow_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
const float* param,
|
||||
int width);
|
||||
void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
|
||||
@ -59,12 +59,9 @@ extern "C" {
|
||||
|
||||
// The following are available on all x86 platforms, but
|
||||
// require VS2012, clang 3.4 or gcc 4.7.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
|
||||
defined(GCC_HAS_AVX2) || \
|
||||
(defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || \
|
||||
defined(_M_X64) || defined(_M_IX86))))
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
|
||||
defined(GCC_HAS_AVX2))
|
||||
#define HAS_SCALEADDROW_AVX2
|
||||
#define HAS_SCALEARGBCOLS_AVX2
|
||||
#define HAS_SCALEARGBCOLSUP2_AVX2
|
||||
@ -96,11 +93,10 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
// The following are available for AVX512 x64 platforms:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(_M_X64)) && \
|
||||
(defined(CLANG_HAS_AVX512) || \
|
||||
(defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(_M_X64)) && \
|
||||
(defined(CLANG_HAS_AVX512) || defined(GCC_HAS_AVX512) || \
|
||||
defined(VISUALC_HAS_AVX512))
|
||||
#define HAS_SCALEADDROW_AVX512BW
|
||||
#define HAS_SCALEADDCOLS1_AVX512BW
|
||||
#define HAS_SCALEADDCOLS2_AVX512BW
|
||||
|
||||
@ -389,6 +389,7 @@ static uint64_t LowBytesMask64(int n) {
|
||||
#ifdef HAS_HAMMINGDISTANCE_AVX512BW
|
||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||
// loads that read zeros, so any count is supported.
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
uint32_t HammingDistance_AVX512BW(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
@ -527,6 +528,7 @@ uint32_t HammingDistance_AVX512VNNI(const uint8_t* src_a,
|
||||
#ifdef HAS_SUMSQUAREERROR_AVX512BW
|
||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||
// loads that read zeros, so any count is supported.
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
@ -596,6 +598,7 @@ uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
|
||||
#ifdef HAS_SUMSQUAREERROR_AVX512VNNI
|
||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||
// loads that read zeros, so any count is supported.
|
||||
LIBYUV_TARGET_AVX512VNNI
|
||||
uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
@ -811,6 +814,7 @@ static uint32_t Pow33(int n) {
|
||||
// Process 64 bytes per loop. The remainder of n = 1 to 63 bytes is loaded
|
||||
// into the high n bytes with a masked load ending at src + count, so the
|
||||
// same multipliers apply, and hash is multiplied by 33 ^ n.
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed) {
|
||||
const int n = count & 63;
|
||||
const uint64_t mask = n ? ~0ULL << (64 - n) : 0ULL;
|
||||
@ -1150,6 +1154,7 @@ void Ssim4x4Row_AVX2(const uint8_t* src_a,
|
||||
|
||||
// Process 64 pixels (16 4x4 tiles) per loop and the remainder of 4 to 60
|
||||
// pixels (1 to 15 tiles) with masked loads and stores.
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void Ssim4x4Row_AVX512BW(const uint8_t* src_a,
|
||||
int stride_a,
|
||||
const uint8_t* src_b,
|
||||
@ -1433,6 +1438,7 @@ float SsimWindowRow_AVX2(const uint32_t* row0,
|
||||
|
||||
// Process 16 8x8 windows per loop and the remainder of 1 to 15 windows with
|
||||
// masked loads and masked accumulation.
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows) {
|
||||
|
||||
@ -291,10 +291,7 @@ static int I41xToI420(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert16To8Row = Convert16To8Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_RVV)
|
||||
@ -345,10 +342,7 @@ static int I41xToI420(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_AVX512BW;
|
||||
if (IS_ALIGNED(uv_width, 32)) {
|
||||
HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX512BW;
|
||||
}
|
||||
HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_RVV)
|
||||
@ -466,10 +460,7 @@ static int I21xToI420(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert16To8Row = Convert16To8Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_RVV)
|
||||
@ -519,10 +510,7 @@ static int I21xToI420(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_HALFROW_16TO8_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
HalfRow_16To8 = HalfRow_16To8_Any_AVX512BW;
|
||||
if (IS_ALIGNED(uv_width, 64)) {
|
||||
HalfRow_16To8 = HalfRow_16To8_AVX512BW;
|
||||
}
|
||||
HalfRow_16To8 = HalfRow_16To8_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFROW_16TO8_RVV)
|
||||
@ -962,10 +950,7 @@ int I010ToNV12(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert16To8Row = Convert16To8Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 64)) {
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_HVX)
|
||||
@ -985,17 +970,14 @@ int I010ToNV12(const uint16_t* src_y,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
if (IS_ALIGNED(halfwidth, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -1503,17 +1485,14 @@ int I422ToNV21(const uint8_t* src_y,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
if (IS_ALIGNED(halfwidth, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -2096,10 +2075,7 @@ static int Px10ToI420(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert16To8Row = Convert16To8Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(uv_packed, 64)) {
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_RVV)
|
||||
@ -2130,10 +2106,7 @@ static int Px10ToI420(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
SplitUVRow = SplitUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(src_uv_width, 64)) {
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_NEON)
|
||||
@ -2262,10 +2235,7 @@ static int Px10ToI420(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_HALFROW_16TO8_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Half16To8 = HalfRow_16To8_Any_AVX512BW;
|
||||
if (IS_ALIGNED(dst_uv_width, 64)) {
|
||||
Half16To8 = HalfRow_16To8_AVX512BW;
|
||||
}
|
||||
Half16To8 = HalfRow_16To8_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFROW_16TO8_RVV)
|
||||
@ -2315,10 +2285,7 @@ static int Px10ToI420(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Half16To8 = HalfWidthRow_16To8_Any_AVX512BW;
|
||||
if (IS_ALIGNED(dst_uv_width, 32)) {
|
||||
Half16To8 = HalfWidthRow_16To8_AVX512BW;
|
||||
}
|
||||
Half16To8 = HalfWidthRow_16To8_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_RVV)
|
||||
@ -2919,26 +2886,17 @@ int ARGBToI420Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -3043,7 +3001,7 @@ int ARGBToI420Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -3187,26 +3145,17 @@ int ARGBToI420Alpha(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -3311,7 +3260,7 @@ int ARGBToI420Alpha(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -3598,26 +3547,17 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -3683,7 +3623,7 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
|
||||
}
|
||||
}
|
||||
@ -3691,7 +3631,7 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -3857,10 +3797,7 @@ int RGB24ToI420(const uint8_t* src_rgb24,
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
|
||||
@ -4018,10 +3955,7 @@ int RGB24ToJ420(const uint8_t* src_rgb24,
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
|
||||
@ -4178,10 +4112,7 @@ int RAWToI420(const uint8_t* src_raw,
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
|
||||
@ -4329,10 +4260,7 @@ int RAWToJ420(const uint8_t* src_raw,
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
|
||||
@ -4479,10 +4407,7 @@ static int RAWToI444Matrix(const uint8_t* src_raw,
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_NEON)
|
||||
@ -4549,10 +4474,7 @@ static int RAWToI444Matrix(const uint8_t* src_raw,
|
||||
#endif
|
||||
#if defined(HAS_RGBTOUV444MATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RGBToUV444MatrixRow = RGBToUV444MatrixRow_AVX512BW;
|
||||
}
|
||||
RGBToUV444MatrixRow = RGBToUV444MatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBTOUV444MATRIXROW_NEON)
|
||||
@ -4974,10 +4896,7 @@ int RGB24ToJ400(const uint8_t* src_rgb24,
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_NEON)
|
||||
@ -5079,10 +4998,7 @@ int RAWToJ400(const uint8_t* src_raw,
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_NEON)
|
||||
|
||||
@ -872,7 +872,7 @@ int I444ToRGB24Matrix(const uint8_t* src_y,
|
||||
#if defined(HAS_I444TORGB24ROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_SSSE3;
|
||||
}
|
||||
}
|
||||
@ -3710,10 +3710,7 @@ int J400ToARGB(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_J400TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
J400ToARGBRow = J400ToARGBRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
J400ToARGBRow = J400ToARGBRow_AVX512BW;
|
||||
}
|
||||
J400ToARGBRow = J400ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_J400TOARGBROW_NEON)
|
||||
@ -5446,6 +5443,11 @@ int NV21ToYUV24(const uint8_t* src_y,
|
||||
NV21ToYUV24Row = NV21ToYUV24Row_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_NV21TOYUV24ROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
NV21ToYUV24Row = NV21ToYUV24Row_AVX512VBMI;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
NV21ToYUV24Row(src_y, src_vu, dst_yuv24, width);
|
||||
@ -5491,7 +5493,7 @@ int YUY2ToARGBMatrix(const uint8_t* src_yuy2,
|
||||
#if defined(HAS_YUY2TOARGBROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
YUY2ToARGBRow = YUY2ToARGBRow_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
YUY2ToARGBRow = YUY2ToARGBRow_SSSE3;
|
||||
}
|
||||
}
|
||||
@ -5596,7 +5598,7 @@ int UYVYToARGBMatrix(const uint8_t* src_uyvy,
|
||||
#if defined(HAS_UYVYTOARGBROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
UYVYToARGBRow = UYVYToARGBRow_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
UYVYToARGBRow = UYVYToARGBRow_SSSE3;
|
||||
}
|
||||
}
|
||||
@ -6216,7 +6218,7 @@ int I420ToRGB24Matrix(const uint8_t* src_y,
|
||||
#if defined(HAS_I422TORGB24ROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_SSSE3;
|
||||
}
|
||||
}
|
||||
@ -6447,7 +6449,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y,
|
||||
#if defined(HAS_I422TORGB24ROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_SSSE3;
|
||||
}
|
||||
}
|
||||
@ -7758,7 +7760,7 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y,
|
||||
#if defined(HAS_I444TORGB24ROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_SSSE3;
|
||||
}
|
||||
}
|
||||
@ -9637,7 +9639,7 @@ static int I422ToRGB24MatrixLinear(const uint8_t* src_y,
|
||||
#if defined(HAS_I444TORGB24ROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_SSSE3;
|
||||
}
|
||||
}
|
||||
|
||||
@ -775,10 +775,7 @@ static int I4xxToPxxx(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert8To16Row = Convert8To16Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(uv_width, 64)) {
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_NEON)
|
||||
|
||||
@ -78,26 +78,17 @@ int ARGBToI444Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -179,26 +170,17 @@ int ARGBToI444Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_NEON)
|
||||
@ -312,26 +294,17 @@ int ARGBToI422Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -436,7 +409,7 @@ int ARGBToI422Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -547,26 +520,17 @@ int ARGBToNV12Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -671,7 +635,7 @@ int ARGBToNV12Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -732,17 +696,14 @@ int ARGBToNV12Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
if (IS_ALIGNED(halfwidth, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -875,26 +836,17 @@ int ARGBToNV16Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -999,7 +951,7 @@ int ARGBToNV16Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -1060,17 +1012,14 @@ int ARGBToNV16Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
if (IS_ALIGNED(halfwidth, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -1197,26 +1146,17 @@ int ARGBToNV24Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -1298,26 +1238,17 @@ int ARGBToNV24Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_NEON)
|
||||
@ -1377,17 +1308,14 @@ int ARGBToNV24Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -1502,26 +1430,17 @@ int ARGBToNV21Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -1626,7 +1545,7 @@ int ARGBToNV21Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -1687,17 +1606,14 @@ int ARGBToNV21Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
if (IS_ALIGNED(halfwidth, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -1816,26 +1732,17 @@ int ARGBToI400Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -1931,26 +1838,17 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -2003,7 +1901,7 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -2157,26 +2055,17 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -2229,7 +2118,7 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -2412,26 +2301,17 @@ int ARGBToP010Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -2536,7 +2416,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -2598,10 +2478,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert8To16Row = Convert8To16Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_NEON)
|
||||
@ -2645,10 +2522,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
MultiplyRow_16 = MultiplyRow_16_AVX512BW;
|
||||
}
|
||||
MultiplyRow_16 = MultiplyRow_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MULTIPLYROW_16_NEON)
|
||||
@ -2693,17 +2567,14 @@ int ARGBToP010Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
if (IS_ALIGNED(halfwidth, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -2869,26 +2740,17 @@ int ARGBToP210Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -2993,7 +2855,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
|
||||
}
|
||||
}
|
||||
@ -3055,10 +2917,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert8To16Row = Convert8To16Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_NEON)
|
||||
@ -3102,10 +2961,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
MultiplyRow_16 = MultiplyRow_16_AVX512BW;
|
||||
}
|
||||
MultiplyRow_16 = MultiplyRow_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MULTIPLYROW_16_NEON)
|
||||
@ -3150,17 +3006,14 @@ int ARGBToP210Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
if (IS_ALIGNED(halfwidth, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -3311,26 +3164,17 @@ int ARGBToP410Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -3412,26 +3256,17 @@ int ARGBToP410Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI;
|
||||
}
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOUV444MATRIXROW_NEON)
|
||||
@ -3492,10 +3327,7 @@ int ARGBToP410Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert8To16Row = Convert8To16Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_NEON)
|
||||
@ -3539,10 +3371,7 @@ int ARGBToP410Matrix(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
MultiplyRow_16 = MultiplyRow_16_AVX512BW;
|
||||
}
|
||||
MultiplyRow_16 = MultiplyRow_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MULTIPLYROW_16_NEON)
|
||||
@ -3587,17 +3416,14 @@ int ARGBToP410Matrix(const uint8_t* src_argb,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -4774,10 +4600,7 @@ static int RAWToNVMatrix(const uint8_t* src_raw,
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBTOYMATRIXROW_NEON)
|
||||
@ -4875,17 +4698,14 @@ static int RAWToNVMatrix(const uint8_t* src_raw,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(halfwidth, 32)) {
|
||||
if (IS_ALIGNED(halfwidth, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 64)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
|
||||
@ -240,10 +240,7 @@ static int NV24ToARGB(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
SplitUVRow = SplitUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_NEON)
|
||||
|
||||
@ -55,9 +55,11 @@ static int NV16ToI420(const uint8_t* src_y,
|
||||
if (height < 0) {
|
||||
height = -height;
|
||||
halfheight = (height + 1) >> 1;
|
||||
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
|
||||
if (src_y) {
|
||||
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
|
||||
src_stride_y = -src_stride_y;
|
||||
}
|
||||
src_uv = src_uv + (ptrdiff_t)(height - 1) * src_stride_uv;
|
||||
src_stride_y = -src_stride_y;
|
||||
src_stride_uv = -src_stride_uv;
|
||||
}
|
||||
if (dst_y) {
|
||||
@ -81,10 +83,7 @@ static int NV16ToI420(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
SplitUVRow = SplitUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(halfwidth, 64)) {
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_NEON)
|
||||
@ -226,9 +225,11 @@ static int NV24ToI420(const uint8_t* src_y,
|
||||
if (height < 0) {
|
||||
height = -height;
|
||||
halfheight = (height + 1) >> 1;
|
||||
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
|
||||
if (src_y) {
|
||||
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
|
||||
src_stride_y = -src_stride_y;
|
||||
}
|
||||
src_uv = src_uv + (ptrdiff_t)(height - 1) * src_stride_uv;
|
||||
src_stride_y = -src_stride_y;
|
||||
src_stride_uv = -src_stride_uv;
|
||||
}
|
||||
if (dst_y) {
|
||||
|
||||
@ -232,10 +232,7 @@ void Convert16To8Plane(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert16To8Row = Convert16To8Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
Convert16To8Row = Convert16To8Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_HVX)
|
||||
@ -309,10 +306,7 @@ void Convert8To16Plane(
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert8To16Row = Convert8To16Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
Convert8To16Row = Convert8To16Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO16ROW_NEON)
|
||||
@ -745,10 +739,7 @@ void SplitUVPlane(const uint8_t* src_uv,
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
SplitUVRow = SplitUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_NEON)
|
||||
@ -839,17 +830,14 @@ void MergeUVPlane(const uint8_t* src_u,
|
||||
#if defined(HAS_MERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
MergeUVRow = MergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow = MergeUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
MergeUVRow = MergeUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_NEON)
|
||||
@ -1092,10 +1080,7 @@ void ConvertToMSBPlane_16(const uint16_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
MultiplyRow_16 = MultiplyRow_16_AVX512BW;
|
||||
}
|
||||
MultiplyRow_16 = MultiplyRow_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MULTIPLYROW_16_NEON)
|
||||
@ -4353,18 +4338,12 @@ int RAWToRGB24(const uint8_t* src_raw,
|
||||
#endif
|
||||
#if defined(HAS_RAWTORGB24ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
RAWToRGB24Row = RAWToRGB24Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RAWToRGB24Row = RAWToRGB24Row_AVX512BW;
|
||||
}
|
||||
RAWToRGB24Row = RAWToRGB24Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
RAWToRGB24Row = RAWToRGB24Row_Any_AVX512VBMI;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
RAWToRGB24Row = RAWToRGB24Row_AVX512VBMI;
|
||||
}
|
||||
RAWToRGB24Row = RAWToRGB24Row_AVX512VBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RAWTORGB24ROW_NEON)
|
||||
@ -5565,7 +5544,7 @@ int InterpolatePlane_16(const uint16_t* src0,
|
||||
#if defined(HAS_INTERPOLATEROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
InterpolateRow_16 = InterpolateRow_16_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
InterpolateRow_16 = InterpolateRow_16_AVX2;
|
||||
}
|
||||
}
|
||||
@ -5981,10 +5960,17 @@ static int ARGBSobelize(const uint8_t* src_argb,
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI) && TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512VNNIVBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
|
||||
@ -6282,14 +6268,19 @@ int ARGBPolynomial(const uint8_t* src_argb,
|
||||
src_stride_argb = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBPOLYNOMIALROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 2)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_SSE2;
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_Any_SSE2;
|
||||
if (IS_ALIGNED(width, 2)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_SSE2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBPOLYNOMIALROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && TestCpuFlag(kCpuHasFMA3) &&
|
||||
IS_ALIGNED(width, 2)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_AVX2;
|
||||
if (TestCpuFlag(kCpuHasAVX2) && TestCpuFlag(kCpuHasFMA3)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 2)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBPOLYNOMIALROW_NEON)
|
||||
@ -6889,10 +6880,7 @@ int UYVYToNV12(const uint8_t* src_uyvy,
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
SplitUVRow = SplitUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(awidth, 64)) {
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
SplitUVRow = SplitUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SPLITUVROW_NEON)
|
||||
@ -7068,13 +7056,19 @@ void HalfMergeUVPlane(const uint8_t* src_u,
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFMERGEUVROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 16)) {
|
||||
HalfMergeUVRow = HalfMergeUVRow_SSSE3;
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
HalfMergeUVRow = HalfMergeUVRow_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
HalfMergeUVRow = HalfMergeUVRow_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFMERGEUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(width, 32)) {
|
||||
HalfMergeUVRow = HalfMergeUVRow_AVX2;
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
HalfMergeUVRow = HalfMergeUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
HalfMergeUVRow = HalfMergeUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFMERGEUVROW_AVX512BW)
|
||||
|
||||
@ -559,9 +559,20 @@ void SplitRotateUV180(const uint8_t* src,
|
||||
MirrorSplitUVRow = MirrorSplitUVRow_NEON;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORSPLITUVROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
MirrorSplitUVRow = MirrorSplitUVRow_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
MirrorSplitUVRow = MirrorSplitUVRow_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORSPLITUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(width, 32)) {
|
||||
MirrorSplitUVRow = MirrorSplitUVRow_AVX2;
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MirrorSplitUVRow = MirrorSplitUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
MirrorSplitUVRow = MirrorSplitUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORSPLITUVROW_AVX512BW)
|
||||
|
||||
@ -189,6 +189,7 @@ static const uint32_t kPermdTranspose_AVX512BW[32] = {
|
||||
// TODO(fbarchard): Use for ARGB (32 bit) and 16 bit channel transposes.
|
||||
// TODO(fbarchard): Consider removing in favor of TransposeWx16_Byte_AVX2,
|
||||
// which is within 10% of AVX512BW on most CPUs.
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
static void TransposeWx16_Byte_AVX512BW(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst_a,
|
||||
@ -748,6 +749,7 @@ void TransposeWx16_16_AVX2(const uint16_t* src,
|
||||
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
||||
// Transpose 32 rows of 16 bit plane (arbitrary width) using 8x32 tiles in
|
||||
// zmm0..zmm15 with k1-masked right tail.
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void TransposeWx32_16_AVX512BW(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
@ -1553,6 +1555,7 @@ void TransposeWxH_32_AVX2(const uint8_t* src,
|
||||
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
||||
// Transpose 16 rows of 32 bit plane (arbitrary width) using 4x16 ZMM vertical
|
||||
// tiles in zmm0..zmm7 with k1-masked right tail.
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void TransposeWx16_32_AVX512BW(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
|
||||
@ -398,7 +398,7 @@ ANY31C(I422ToARGB1555Row_Any_SSSE3, I422ToARGB1555Row_SSSE3, 1, 0, 2, 7)
|
||||
ANY31C(I422ToRGB565Row_Any_SSSE3, I422ToRGB565Row_SSSE3, 1, 0, 2, 7)
|
||||
#endif
|
||||
#ifdef HAS_I422TORGB24ROW_SSSE3
|
||||
ANY31C(I422ToRGB24Row_Any_SSSE3, I422ToRGB24Row_SSSE3, 1, 0, 3, 15)
|
||||
ANY31C(I422ToRGB24Row_Any_SSSE3, I422ToRGB24Row_SSSE3, 1, 0, 3, 7)
|
||||
#endif
|
||||
#ifdef HAS_I422TOAR30ROW_SSSE3
|
||||
ANY31C(I422ToAR30Row_Any_SSSE3, I422ToAR30Row_SSSE3, 1, 0, 4, 7)
|
||||
@ -410,7 +410,7 @@ ANY31C(I422ToAR30Row_Any_AVX2, I422ToAR30Row_AVX2, 1, 0, 4, 15)
|
||||
ANY31C(I444ToARGBRow_Any_SSSE3, I444ToARGBRow_SSSE3, 0, 0, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_I444TORGB24ROW_SSSE3
|
||||
ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 15)
|
||||
ANY31C(I444ToRGB24Row_Any_SSSE3, I444ToRGB24Row_SSSE3, 0, 0, 3, 7)
|
||||
#endif
|
||||
#ifdef HAS_I422TORGB24ROW_AVX2
|
||||
ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15)
|
||||
@ -676,10 +676,7 @@ ANY31PT(MergeXRGB16To8Row_Any_NEON,
|
||||
ANY21(MergeUVRow_Any_SSE2, MergeUVRow_SSE2, 0, 1, 1, 2, 15)
|
||||
#endif
|
||||
#ifdef HAS_MERGEUVROW_AVX2
|
||||
ANY21(MergeUVRow_Any_AVX2, MergeUVRow_AVX2, 0, 1, 1, 2, 31)
|
||||
#endif
|
||||
#ifdef HAS_MERGEUVROW_AVX512BW
|
||||
ANY21(MergeUVRow_Any_AVX512BW, MergeUVRow_AVX512BW, 0, 1, 1, 2, 31)
|
||||
ANY21(MergeUVRow_Any_AVX2, MergeUVRow_AVX2, 0, 1, 1, 2, 15)
|
||||
#endif
|
||||
#ifdef HAS_MERGEUVROW_NEON
|
||||
ANY21(MergeUVRow_Any_NEON, MergeUVRow_NEON, 0, 1, 1, 2, 15)
|
||||
@ -829,7 +826,7 @@ ANY21(MergeUVRow_Any_WASMSIMD, MergeUVRow_WASMSIMD, 0, 1, 1, 2, 15)
|
||||
memcpy(vin, src_yuy2 + np * SBPP, r * SBPP); \
|
||||
memcpy(vin + (MASK + 1) * SBPP, src_yuy2 + stride_yuy2 + np * SBPP, \
|
||||
r * SBPP); \
|
||||
ANY_SIMD(vin, (MASK + 1) * SBPP, vout, MASK + 1); \
|
||||
ANY_SIMD(vin, (MASK + 1) * SBPP, vout, (MASK + 1) * 2); \
|
||||
memcpy(dst_uv + np * BPP, vout, r * BPP); \
|
||||
}
|
||||
|
||||
@ -842,6 +839,46 @@ ANY21S(YUY2ToNVUVRow_Any_SSE2, YUY2ToNVUVRow_SSE2, 4, 2, 7)
|
||||
#ifdef HAS_YUY2TONVUVROW_AVX2
|
||||
ANY21S(YUY2ToNVUVRow_Any_AVX2, YUY2ToNVUVRow_AVX2, 4, 2, 15)
|
||||
#endif
|
||||
#undef ANY21S
|
||||
|
||||
// Any 2 planes to 1 half-size merged UV row with stride (2 rows of U and V).
|
||||
#define ANY21HMS(NAMEANY, ANY_SIMD, MASK) \
|
||||
void NAMEANY(const uint8_t* src_u, int src_stride_u, const uint8_t* src_v, \
|
||||
int src_stride_v, uint8_t* dst_uv, int width) { \
|
||||
SIMD_ALIGNED(uint8_t vin[64 * 4]); \
|
||||
static_assert(MASK + 1 <= sizeof(vin) / 4, "vin buffer too small"); \
|
||||
SIMD_ALIGNED(uint8_t vout[64]); \
|
||||
static_assert(SS(MASK + 1, 1) * 2 <= sizeof(vout), \
|
||||
"vout buffer too small"); \
|
||||
MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
|
||||
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
|
||||
int r = width & MASK; \
|
||||
int n = width & ~MASK; \
|
||||
if (n > 0) { \
|
||||
ANY_SIMD(src_u, src_stride_u, src_v, src_stride_v, dst_uv, n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
memcpy(vin, src_u + np, r); \
|
||||
memcpy(vin + 64, src_u + src_stride_u + np, r); \
|
||||
memcpy(vin + 128, src_v + np, r); \
|
||||
memcpy(vin + 192, src_v + src_stride_v + np, r); \
|
||||
if (r & 1) { \
|
||||
vin[r] = vin[r - 1]; \
|
||||
vin[64 + r] = vin[64 + r - 1]; \
|
||||
vin[128 + r] = vin[128 + r - 1]; \
|
||||
vin[192 + r] = vin[192 + r - 1]; \
|
||||
} \
|
||||
ANY_SIMD(vin, 64, vin + 128, 64, vout, MASK + 1); \
|
||||
memcpy(dst_uv + (np >> 1) * 2, vout, SS(r, 1) * 2); \
|
||||
}
|
||||
|
||||
#ifdef HAS_HALFMERGEUVROW_SSSE3
|
||||
ANY21HMS(HalfMergeUVRow_Any_SSSE3, HalfMergeUVRow_SSSE3, 15)
|
||||
#endif
|
||||
#ifdef HAS_HALFMERGEUVROW_AVX2
|
||||
ANY21HMS(HalfMergeUVRow_Any_AVX2, HalfMergeUVRow_AVX2, 31)
|
||||
#endif
|
||||
#undef ANY21HMS
|
||||
|
||||
// Any 2 planes to 1 with yuvconstants
|
||||
#define ANY21C(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, SBPP2, BPP, MASK) \
|
||||
@ -1133,9 +1170,6 @@ ANY11(ABGRToAR30Row_Any_AVX2, ABGRToAR30Row_AVX2, 0, 4, 4, 7)
|
||||
ANY11(ARGBToAR30Row_Any_AVX2, ARGBToAR30Row_AVX2, 0, 4, 4, 7)
|
||||
#endif
|
||||
|
||||
#if defined(HAS_J400TOARGBROW_AVX512BW)
|
||||
ANY11(J400ToARGBRow_Any_AVX512BW, J400ToARGBRow_AVX512BW, 0, 1, 4, 31)
|
||||
#endif
|
||||
#if defined(HAS_J400TOARGBROW_AVX2)
|
||||
ANY11(J400ToARGBRow_Any_AVX2, J400ToARGBRow_AVX2, 0, 1, 4, 15)
|
||||
#endif
|
||||
@ -1159,12 +1193,6 @@ ANY11(RAWToRGB24Row_Any_SSSE3, RAWToRGB24Row_SSSE3, 0, 3, 3, 7)
|
||||
#if defined(HAS_RAWTORGB24ROW_AVX2)
|
||||
ANY11(RAWToRGB24Row_Any_AVX2, RAWToRGB24Row_AVX2, 0, 3, 3, 31)
|
||||
#endif
|
||||
#if defined(HAS_RAWTORGB24ROW_AVX512BW)
|
||||
ANY11(RAWToRGB24Row_Any_AVX512BW, RAWToRGB24Row_AVX512BW, 0, 3, 3, 63)
|
||||
#endif
|
||||
#if defined(HAS_RAWTORGB24ROW_AVX512VBMI)
|
||||
ANY11(RAWToRGB24Row_Any_AVX512VBMI, RAWToRGB24Row_AVX512VBMI, 0, 3, 3, 63)
|
||||
#endif
|
||||
#if defined(HAS_RGB565TOARGBROW_AVX2)
|
||||
ANY11(RGB565ToARGBRow_Any_AVX2, RGB565ToARGBRow_AVX2, 0, 2, 4, 15)
|
||||
#endif
|
||||
@ -1614,6 +1642,22 @@ ANY11P(ARGBOrRow_Any_LASX, ARGBOrRow_LASX, const uint32_t, 4, 4, 7)
|
||||
#ifdef HAS_ARGBORROW_WASMSIMD
|
||||
ANY11P(ARGBOrRow_Any_WASMSIMD, ARGBOrRow_WASMSIMD, const uint32_t, 4, 4, 3)
|
||||
#endif
|
||||
#ifdef HAS_ARGBPOLYNOMIALROW_SSE2
|
||||
ANY11P(ARGBPolynomialRow_Any_SSE2,
|
||||
ARGBPolynomialRow_SSE2,
|
||||
const float*,
|
||||
4,
|
||||
4,
|
||||
1)
|
||||
#endif
|
||||
#ifdef HAS_ARGBPOLYNOMIALROW_AVX2
|
||||
ANY11P(ARGBPolynomialRow_Any_AVX2,
|
||||
ARGBPolynomialRow_AVX2,
|
||||
const float*,
|
||||
4,
|
||||
4,
|
||||
1)
|
||||
#endif
|
||||
#ifdef HAS_ARGBPOLYNOMIALROW_NEON
|
||||
ANY11P(ARGBPolynomialRow_Any_NEON,
|
||||
ARGBPolynomialRow_NEON,
|
||||
@ -1623,7 +1667,6 @@ ANY11P(ARGBPolynomialRow_Any_NEON,
|
||||
1)
|
||||
#endif
|
||||
#undef ANY11P
|
||||
#undef ANY11P
|
||||
|
||||
// Any 1 to 1 with type
|
||||
#define ANY11T(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \
|
||||
@ -1738,15 +1781,6 @@ ANY11C(Convert16To8Row_Any_AVX2,
|
||||
uint8_t,
|
||||
31)
|
||||
#endif
|
||||
#ifdef HAS_CONVERT16TO8ROW_AVX512BW
|
||||
ANY11C(Convert16To8Row_Any_AVX512BW,
|
||||
Convert16To8Row_AVX512BW,
|
||||
2,
|
||||
1,
|
||||
uint16_t,
|
||||
uint8_t,
|
||||
63)
|
||||
#endif
|
||||
#ifdef HAS_CONVERT16TO8ROW_NEON
|
||||
ANY11C(Convert16To8Row_Any_NEON,
|
||||
Convert16To8Row_NEON,
|
||||
@ -1774,15 +1808,6 @@ ANY11C(Convert8To16Row_Any_AVX2,
|
||||
uint16_t,
|
||||
31)
|
||||
#endif
|
||||
#ifdef HAS_CONVERT8TO16ROW_AVX512BW
|
||||
ANY11C(Convert8To16Row_Any_AVX512BW,
|
||||
Convert8To16Row_AVX512BW,
|
||||
1,
|
||||
2,
|
||||
uint8_t,
|
||||
uint16_t,
|
||||
63)
|
||||
#endif
|
||||
#ifdef HAS_CONVERT8TO16ROW_NEON
|
||||
ANY11C(Convert8To16Row_Any_NEON,
|
||||
Convert8To16Row_NEON,
|
||||
@ -1792,15 +1817,6 @@ ANY11C(Convert8To16Row_Any_NEON,
|
||||
uint16_t,
|
||||
15)
|
||||
#endif
|
||||
#ifdef HAS_MULTIPLYROW_16_AVX512BW
|
||||
ANY11C(MultiplyRow_16_Any_AVX512BW,
|
||||
MultiplyRow_16_AVX512BW,
|
||||
2,
|
||||
2,
|
||||
uint16_t,
|
||||
uint16_t,
|
||||
63)
|
||||
#endif
|
||||
#ifdef HAS_MULTIPLYROW_16_AVX2
|
||||
ANY11C(MultiplyRow_16_Any_AVX2,
|
||||
MultiplyRow_16_AVX2,
|
||||
@ -1959,8 +1975,8 @@ ANY11P16(HalfFloatRow_Any_LASX, HalfFloatRow_LASX, uint16_t, uint16_t, 2, 2, 15)
|
||||
}
|
||||
|
||||
#if defined(HAS_YUY2TOARGBROW_SSSE3)
|
||||
ANY11C(YUY2ToARGBRow_Any_SSSE3, YUY2ToARGBRow_SSSE3, 1, 4, 4, 15)
|
||||
ANY11C(UYVYToARGBRow_Any_SSSE3, UYVYToARGBRow_SSSE3, 1, 4, 4, 15)
|
||||
ANY11C(YUY2ToARGBRow_Any_SSSE3, YUY2ToARGBRow_SSSE3, 1, 4, 4, 7)
|
||||
ANY11C(UYVYToARGBRow_Any_SSSE3, UYVYToARGBRow_SSSE3, 1, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOARGBROW_AVX2)
|
||||
ANY11C(YUY2ToARGBRow_Any_AVX2, YUY2ToARGBRow_AVX2, 1, 4, 4, 31)
|
||||
@ -2121,13 +2137,6 @@ ANY11HS(HalfRow_16To8_Any_SSSE3, HalfRow_16To8_SSSE3, uint16_t, uint8_t, 15)
|
||||
#ifdef HAS_HALFROW_16TO8_AVX2
|
||||
ANY11HS(HalfRow_16To8_Any_AVX2, HalfRow_16To8_AVX2, uint16_t, uint8_t, 31)
|
||||
#endif
|
||||
#ifdef HAS_HALFROW_16TO8_AVX512BW
|
||||
ANY11HS(HalfRow_16To8_Any_AVX512BW,
|
||||
HalfRow_16To8_AVX512BW,
|
||||
uint16_t,
|
||||
uint8_t,
|
||||
63)
|
||||
#endif
|
||||
#ifdef HAS_HALFROW_16TO8_NEON
|
||||
ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15)
|
||||
#endif
|
||||
@ -2172,13 +2181,6 @@ ANY11HWS(HalfWidthRow_16To8_Any_AVX2,
|
||||
uint8_t,
|
||||
15)
|
||||
#endif
|
||||
#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW
|
||||
ANY11HWS(HalfWidthRow_16To8_Any_AVX512BW,
|
||||
HalfWidthRow_16To8_AVX512BW,
|
||||
uint16_t,
|
||||
uint8_t,
|
||||
31)
|
||||
#endif
|
||||
#ifdef HAS_HALFWIDTHROW_16TO8_NEON
|
||||
ANY11HWS(HalfWidthRow_16To8_Any_NEON,
|
||||
HalfWidthRow_16To8_NEON,
|
||||
@ -2302,6 +2304,36 @@ ANY11MT(MirrorRow_16_Any_NEON, MirrorRow_16_NEON, uint16_t, 2, 15)
|
||||
#endif
|
||||
#undef ANY11MT
|
||||
|
||||
// Any 1 to 2 mirror split UV.
|
||||
#define ANY12MR(NAMEANY, ANY_SIMD, MASK) \
|
||||
void NAMEANY(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, \
|
||||
int width) { \
|
||||
SIMD_ALIGNED(uint8_t vin[128]); \
|
||||
static_assert((MASK + 1) * 2 <= sizeof(vin), "vin buffer too small"); \
|
||||
SIMD_ALIGNED(uint8_t vout[64 * 2]); \
|
||||
static_assert(MASK + 1 <= sizeof(vout) / 2, "vout buffer too small"); \
|
||||
MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
|
||||
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
|
||||
int r = width & MASK; \
|
||||
int n = width & ~MASK; \
|
||||
if (n > 0) { \
|
||||
ANY_SIMD(src_uv + r * 2, dst_u, dst_v, n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
memcpy(vin, src_uv, r * 2); \
|
||||
ANY_SIMD(vin, vout, vout + 64, MASK + 1); \
|
||||
memcpy(dst_u + np, vout + (MASK + 1 - r), r); \
|
||||
memcpy(dst_v + np, vout + 64 + (MASK + 1 - r), r); \
|
||||
}
|
||||
|
||||
#ifdef HAS_MIRRORSPLITUVROW_SSSE3
|
||||
ANY12MR(MirrorSplitUVRow_Any_SSSE3, MirrorSplitUVRow_SSSE3, 7)
|
||||
#endif
|
||||
#ifdef HAS_MIRRORSPLITUVROW_AVX2
|
||||
ANY12MR(MirrorSplitUVRow_Any_AVX2, MirrorSplitUVRow_AVX2, 15)
|
||||
#endif
|
||||
#undef ANY12MR
|
||||
|
||||
// Any 1 plane. (memset)
|
||||
#define ANY1(NAMEANY, ANY_SIMD, T, BPP, MASK) \
|
||||
void NAMEANY(uint8_t* dst_ptr, T v32, int width) { \
|
||||
@ -2371,9 +2403,6 @@ ANY1(ARGBSetRow_Any_LASX, ARGBSetRow_LASX, uint32_t, 4, 7)
|
||||
#ifdef HAS_SPLITUVROW_SSE2
|
||||
ANY12(SplitUVRow_Any_SSE2, SplitUVRow_SSE2, 0, 2, 0, 15)
|
||||
#endif
|
||||
#ifdef HAS_SPLITUVROW_AVX512BW
|
||||
ANY12(SplitUVRow_Any_AVX512BW, SplitUVRow_AVX512BW, 0, 2, 0, 63)
|
||||
#endif
|
||||
#ifdef HAS_SPLITUVROW_AVX2
|
||||
ANY12(SplitUVRow_Any_AVX2, SplitUVRow_AVX2, 0, 2, 0, 31)
|
||||
#endif
|
||||
@ -2628,21 +2657,21 @@ ANY12MS(ARGB1555ToUVMatrixRow_Any_NEON, ARGB1555ToUVMatrixRow_NEON, 0, 2, 15)
|
||||
ANY12MS(ARGB4444ToUVMatrixRow_Any_NEON, ARGB4444ToUVMatrixRow_NEON, 0, 2, 15)
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOUVMATRIXROW_AVX2
|
||||
ANY12MS(ARGBToUVMatrixRow_Any_AVX2, ARGBToUVMatrixRow_AVX2, 0, 4, 31)
|
||||
ANY12MS(ARGBToUVMatrixRow_Any_AVX2, ARGBToUVMatrixRow_AVX2, 0, 4, 15)
|
||||
ANY12MS(RGBToUVMatrixRow_Any_AVX2, RGBToUVMatrixRow_AVX2, 0, 3, 31)
|
||||
ANY12MS(RGB565ToUVMatrixRow_Any_AVX2, RGB565ToUVMatrixRow_AVX2, 0, 2, 31)
|
||||
ANY12MS(RGB565ToUVMatrixRow_Any_AVX2, RGB565ToUVMatrixRow_AVX2, 0, 2, 15)
|
||||
#ifdef HAS_ARGB1555TOARGBROW_AVX2
|
||||
ANY12MS(ARGB1555ToUVMatrixRow_Any_AVX2, ARGB1555ToUVMatrixRow_AVX2, 0, 2, 31)
|
||||
ANY12MS(ARGB1555ToUVMatrixRow_Any_AVX2, ARGB1555ToUVMatrixRow_AVX2, 0, 2, 15)
|
||||
#endif
|
||||
#ifdef HAS_ARGB4444TOARGBROW_AVX2
|
||||
ANY12MS(ARGB4444ToUVMatrixRow_Any_AVX2, ARGB4444ToUVMatrixRow_AVX2, 0, 2, 31)
|
||||
ANY12MS(ARGB4444ToUVMatrixRow_Any_AVX2, ARGB4444ToUVMatrixRow_AVX2, 0, 2, 15)
|
||||
#endif
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOUVMATRIXROW_SSSE3
|
||||
ANY12MS(ARGBToUVMatrixRow_Any_SSSE3, ARGBToUVMatrixRow_SSSE3, 0, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_RGBTOUVMATRIXROW_SSSE3
|
||||
ANY12MS(RGBToUVMatrixRow_Any_SSSE3, RGBToUVMatrixRow_SSSE3, 0, 3, 7)
|
||||
ANY12MS(RGBToUVMatrixRow_Any_SSSE3, RGBToUVMatrixRow_SSSE3, 0, 3, 15)
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOUV444MATRIXROW_AVX2
|
||||
ANY12M(ARGBToUV444MatrixRow_Any_AVX2, ARGBToUV444MatrixRow_AVX2, 4, 31)
|
||||
@ -2650,24 +2679,6 @@ ANY12M(ARGBToUV444MatrixRow_Any_AVX2, ARGBToUV444MatrixRow_AVX2, 4, 31)
|
||||
#ifdef HAS_RGBTOUV444MATRIXROW_AVX2
|
||||
ANY12M(RGBToUV444MatrixRow_Any_AVX2, RGBToUV444MatrixRow_AVX2, 3, 31)
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW
|
||||
ANY12M(ARGBToUV444MatrixRow_Any_AVX512BW, ARGBToUV444MatrixRow_AVX512BW, 4, 63)
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNI
|
||||
ANY12M(ARGBToUV444MatrixRow_Any_AVX512VNNI,
|
||||
ARGBToUV444MatrixRow_AVX512VNNI,
|
||||
4,
|
||||
63)
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI
|
||||
ANY12M(ARGBToUV444MatrixRow_Any_AVX512VNNIVBMI,
|
||||
ARGBToUV444MatrixRow_AVX512VNNIVBMI,
|
||||
4,
|
||||
63)
|
||||
#endif
|
||||
#ifdef HAS_RGBTOUV444MATRIXROW_AVX512BW
|
||||
ANY12M(RGBToUV444MatrixRow_Any_AVX512BW, RGBToUV444MatrixRow_AVX512BW, 3, 63)
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOUV444MATRIXROW_SSSE3
|
||||
ANY12M(ARGBToUV444MatrixRow_Any_SSSE3, ARGBToUV444MatrixRow_SSSE3, 4, 15)
|
||||
#endif
|
||||
@ -2752,21 +2763,6 @@ ANY11MC(ARGB1555ToYMatrixRow_Any_AVX2, ARGB1555ToYMatrixRow_AVX2, 2, 31)
|
||||
ANY11MC(ARGB4444ToYMatrixRow_Any_AVX2, ARGB4444ToYMatrixRow_AVX2, 2, 31)
|
||||
#endif
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOYMATRIXROW_AVX512BW
|
||||
ANY11MC(ARGBToYMatrixRow_Any_AVX512BW, ARGBToYMatrixRow_AVX512BW, 4, 63)
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNI
|
||||
ANY11MC(ARGBToYMatrixRow_Any_AVX512VNNI, ARGBToYMatrixRow_AVX512VNNI, 4, 63)
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI
|
||||
ANY11MC(ARGBToYMatrixRow_Any_AVX512VNNIVBMI,
|
||||
ARGBToYMatrixRow_AVX512VNNIVBMI,
|
||||
4,
|
||||
63)
|
||||
#endif
|
||||
#ifdef HAS_RGBTOYMATRIXROW_AVX512BW
|
||||
ANY11MC(RGBToYMatrixRow_Any_AVX512BW, RGBToYMatrixRow_AVX512BW, 3, 63)
|
||||
#endif
|
||||
#ifdef HAS_ARGBTOYMATRIXROW_NEON
|
||||
ANY11MC(ARGBToYMatrixRow_Any_NEON, ARGBToYMatrixRow_NEON, 4, 15)
|
||||
#endif
|
||||
|
||||
@ -2878,7 +2878,7 @@ void DetileToYUY2_C(const uint8_t* src_y,
|
||||
// packed into 1x4 blocks, whereas the upper bits are packed in normal raster
|
||||
// order.
|
||||
void UnpackMT2T_C(const uint8_t* src, uint16_t* dst, size_t size) {
|
||||
for (size_t i = 0; i < size; i += 80) {
|
||||
while (size >= 80) {
|
||||
const uint8_t* src_lower_bits = src;
|
||||
const uint8_t* src_upper_bits = src + 16;
|
||||
|
||||
@ -2892,6 +2892,7 @@ void UnpackMT2T_C(const uint8_t* src, uint16_t* dst, size_t size) {
|
||||
}
|
||||
|
||||
src += 80;
|
||||
size -= 80;
|
||||
}
|
||||
}
|
||||
|
||||
@ -3942,7 +3943,7 @@ void ByteToFloatRow_C(const uint8_t* src, float* dst, float scale, int width) {
|
||||
int i;
|
||||
for (i = 0; i < width; ++i) {
|
||||
float value = src[i] * scale;
|
||||
dst[i] = value;
|
||||
memcpy(&dst[i], &value, sizeof(value));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
3012
source/row_gcc.cc
3012
source/row_gcc.cc
File diff suppressed because it is too large
Load Diff
@ -988,6 +988,12 @@ void DivideRow_16_HVX(const uint16_t* src_y,
|
||||
uint16_t* dst_y,
|
||||
int scale,
|
||||
int width) {
|
||||
if (scale == 65536) { // Divide by 1, so do memcpy.
|
||||
if (src_y != dst_y) {
|
||||
memcpy(dst_y, src_y, (size_t)width * 2);
|
||||
}
|
||||
return;
|
||||
}
|
||||
uint32_t scale32 = ((uint32_t)(uint16_t)scale << 16) | (uint16_t)scale;
|
||||
while (width > 0) {
|
||||
int w = width >= 64 ? 64 : width;
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "libyuv/convert_from_argb.h" // For ArgbConstants
|
||||
#include "libyuv/row.h"
|
||||
|
||||
@ -3819,6 +3821,12 @@ void DivideRow_16_NEON(const uint16_t* src_y,
|
||||
uint16_t* dst_y,
|
||||
int scale,
|
||||
int width) {
|
||||
if (scale == 65536) { // Divide by 1, so do memcpy.
|
||||
if (src_y != dst_y) {
|
||||
memcpy(dst_y, src_y, (size_t)width * 2);
|
||||
}
|
||||
return;
|
||||
}
|
||||
asm volatile(
|
||||
"vdup.16 d8, %3 \n"
|
||||
"1: \n"
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "libyuv/convert_from_argb.h"
|
||||
#include "libyuv/row.h"
|
||||
|
||||
@ -5464,6 +5466,12 @@ void DivideRow_16_NEON(const uint16_t* src_y,
|
||||
uint16_t* dst_y,
|
||||
int scale,
|
||||
int width) {
|
||||
if (scale == 65536) { // Divide by 1, so do memcpy.
|
||||
if (src_y != dst_y) {
|
||||
memcpy(dst_y, src_y, (size_t)width * 2);
|
||||
}
|
||||
return;
|
||||
}
|
||||
asm volatile(
|
||||
"dup v4.8h, %w3 \n"
|
||||
"1: \n"
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "libyuv/row_sve.h"
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/row.h"
|
||||
@ -772,6 +774,12 @@ void DivideRow_16_SVE2(const uint16_t* src_y,
|
||||
uint16_t* dst_y,
|
||||
int scale,
|
||||
int width) {
|
||||
if (scale == 65536) { // Divide by 1, so do memcpy.
|
||||
if (src_y != dst_y) {
|
||||
memcpy(dst_y, src_y, (size_t)width * 2);
|
||||
}
|
||||
return;
|
||||
}
|
||||
uint64_t vl;
|
||||
asm volatile(
|
||||
"cnth %x[vl] \n"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -415,6 +415,7 @@ void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr,
|
||||
#endif // HAS_SCALEROWDOWN2_16_AVX2
|
||||
|
||||
#ifdef HAS_SCALEROWDOWN2_AVX512BW
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
@ -455,6 +456,7 @@ void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr,
|
||||
: "memory", "cc", "xmm0", "k1");
|
||||
}
|
||||
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
@ -500,6 +502,7 @@ void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr,
|
||||
: "memory", "cc", "xmm0", "xmm4", "xmm5", "k1");
|
||||
}
|
||||
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void ScaleRowDown2Box_AVX512BW(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
@ -2200,6 +2203,7 @@ void ScaleAddRow_AVX2(const uint8_t* src_ptr,
|
||||
|
||||
#ifdef HAS_SCALEADDROW_AVX512BW
|
||||
// Reads 32xN bytes and accumulates to 32xN shorts at a time, with masked tail.
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
int src_width) {
|
||||
@ -2259,6 +2263,7 @@ static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7,
|
||||
#endif
|
||||
|
||||
#ifdef HAS_SCALEADDCOLS1_AVX512BW
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void ScaleAddCols1_AVX512BW(int dst_width,
|
||||
int boxheight,
|
||||
int x,
|
||||
@ -2511,6 +2516,7 @@ void ScaleAddCols1_AVX512BW(int dst_width,
|
||||
#endif // HAS_SCALEADDCOLS1_AVX512BW
|
||||
|
||||
#ifdef HAS_SCALEADDCOLS2_AVX512BW
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void ScaleAddCols2_AVX512BW(int dst_width,
|
||||
int boxheight,
|
||||
int x,
|
||||
@ -4246,6 +4252,7 @@ static const uint8_t kPackEvenBytes[64] = {
|
||||
// Scale UV columns by 3 box filter using AVX512VBMI.
|
||||
// Processes 32 UV pixels (384 input bytes -> 64 output bytes) per main loop
|
||||
// iteration, followed by a 16-pixel block and a k-masked tail (1..15 pixels).
|
||||
LIBYUV_TARGET_AVX512VBMI
|
||||
void ScaleUVAddCols3_AVX512VBMI(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
|
||||
@ -2668,8 +2668,11 @@ static int TestHalfFloatPlane(int benchmark_width,
|
||||
int mask,
|
||||
int invert,
|
||||
int off) {
|
||||
int i, j;
|
||||
const int y_plane_size = benchmark_width * benchmark_height * 2;
|
||||
if (benchmark_width <= 0 || benchmark_height <= 0) {
|
||||
return 0;
|
||||
}
|
||||
const size_t y_plane_size = static_cast<size_t>(benchmark_width) *
|
||||
static_cast<size_t>(benchmark_height) * 2;
|
||||
align_buffer_page_end(orig_y, y_plane_size + off);
|
||||
align_buffer_page_end(dst_c, y_plane_size);
|
||||
align_buffer_page_end(dst_opt, y_plane_size);
|
||||
@ -2678,13 +2681,13 @@ static int TestHalfFloatPlane(int benchmark_width,
|
||||
memset(dst_c, 1, y_plane_size);
|
||||
memset(dst_opt, 2, y_plane_size);
|
||||
|
||||
for (i = 0; i < y_plane_size / 2; ++i) {
|
||||
for (size_t i = 0; i < y_plane_size / 2; ++i) {
|
||||
reinterpret_cast<uint16_t*>(orig_y + off)[i] &= mask;
|
||||
}
|
||||
|
||||
// Disable all optimizations.
|
||||
MaskCpuFlags(disable_cpu_flags);
|
||||
for (j = 0; j < benchmark_iterations; j++) {
|
||||
for (int j = 0; j < benchmark_iterations; j++) {
|
||||
HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y + off),
|
||||
benchmark_width * 2, reinterpret_cast<uint16_t*>(dst_c),
|
||||
benchmark_width * 2, scale, benchmark_width,
|
||||
@ -2693,7 +2696,7 @@ static int TestHalfFloatPlane(int benchmark_width,
|
||||
|
||||
// Enable optimizations.
|
||||
MaskCpuFlags(benchmark_cpu_info);
|
||||
for (j = 0; j < benchmark_iterations; j++) {
|
||||
for (int j = 0; j < benchmark_iterations; j++) {
|
||||
HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y + off),
|
||||
benchmark_width * 2, reinterpret_cast<uint16_t*>(dst_opt),
|
||||
benchmark_width * 2, scale, benchmark_width,
|
||||
@ -2701,7 +2704,7 @@ static int TestHalfFloatPlane(int benchmark_width,
|
||||
}
|
||||
|
||||
int max_diff = 0;
|
||||
for (i = 0; i < y_plane_size / 2; ++i) {
|
||||
for (size_t i = 0; i < y_plane_size / 2; ++i) {
|
||||
int abs_diff =
|
||||
abs(static_cast<int>(reinterpret_cast<uint16_t*>(dst_c)[i]) -
|
||||
static_cast<int>(reinterpret_cast<uint16_t*>(dst_opt)[i]));
|
||||
@ -2838,9 +2841,11 @@ static float TestByteToFloat(int benchmark_width,
|
||||
int i, j;
|
||||
const int y_plane_size = benchmark_width * benchmark_height;
|
||||
|
||||
align_buffer_page_end(orig_y, y_plane_size * (1 + 4 + 4));
|
||||
float* dst_opt = reinterpret_cast<float*>(orig_y + y_plane_size);
|
||||
float* dst_c = reinterpret_cast<float*>(orig_y + y_plane_size * 5);
|
||||
align_buffer_page_end(orig_y, y_plane_size);
|
||||
align_buffer_page_end(dst_opt_mem, y_plane_size * 4);
|
||||
align_buffer_page_end(dst_c_mem, y_plane_size * 4);
|
||||
float* dst_opt = reinterpret_cast<float*>(dst_opt_mem);
|
||||
float* dst_c = reinterpret_cast<float*>(dst_c_mem);
|
||||
|
||||
MemRandomize(orig_y, y_plane_size);
|
||||
memset(dst_c, 0, y_plane_size * 4);
|
||||
@ -2865,6 +2870,8 @@ static float TestByteToFloat(int benchmark_width,
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end(orig_y);
|
||||
free_aligned_buffer_page_end(dst_opt_mem);
|
||||
free_aligned_buffer_page_end(dst_c_mem);
|
||||
return max_diff;
|
||||
}
|
||||
|
||||
@ -4228,9 +4235,9 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Any) {
|
||||
|
||||
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
|
||||
if (has_avx512) {
|
||||
MultiplyRow_16_Any_AVX512BW(
|
||||
reinterpret_cast<const uint16_t*>(src_pixels_y),
|
||||
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64, width);
|
||||
MultiplyRow_16_AVX512BW(reinterpret_cast<const uint16_t*>(src_pixels_y),
|
||||
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
|
||||
width);
|
||||
} else
|
||||
#endif
|
||||
#if defined(HAS_MULTIPLYROW_16_AVX2)
|
||||
@ -6184,4 +6191,442 @@ TEST_F(LibYUVPlanarTest, MSanUninitializedInputsAndTails) {
|
||||
free_aligned_buffer_page_end(uv_src);
|
||||
}
|
||||
|
||||
// Regression test for RGBToUVMatrixRow_Any_SSSE3 MASK=15 (previously MASK=7
|
||||
// overread 24 bytes past src_rgb24 at width=24).
|
||||
TEST_F(LibYUVPlanarTest, RGB24ToI420_Width24_OOB) {
|
||||
const int kWidth = 24;
|
||||
const int kHeight = 2;
|
||||
align_buffer_page_end(src_rgb24, kWidth * 3 * kHeight);
|
||||
align_buffer_page_end(dst_y_c, kWidth * kHeight);
|
||||
align_buffer_page_end(dst_u_c, (kWidth / 2) * (kHeight / 2));
|
||||
align_buffer_page_end(dst_v_c, (kWidth / 2) * (kHeight / 2));
|
||||
align_buffer_page_end(dst_y_opt, kWidth * kHeight);
|
||||
align_buffer_page_end(dst_u_opt, (kWidth / 2) * (kHeight / 2));
|
||||
align_buffer_page_end(dst_v_opt, (kWidth / 2) * (kHeight / 2));
|
||||
MemRandomize(src_rgb24, kWidth * 3 * kHeight);
|
||||
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
RGB24ToI420(src_rgb24, kWidth * 3, dst_y_c, kWidth, dst_u_c, kWidth / 2,
|
||||
dst_v_c, kWidth / 2, kWidth, kHeight);
|
||||
|
||||
const int kNoAvx512 =
|
||||
benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VL |
|
||||
kCpuHasAVX512VBMI | kCpuHasAVX512VNNI);
|
||||
const int kSsse3Only = kNoAvx512 & ~kCpuHasAVX2;
|
||||
const int kModes[3] = {benchmark_cpu_info_, kNoAvx512, kSsse3Only};
|
||||
for (int m = 0; m < 3; ++m) {
|
||||
MaskCpuFlags(kModes[m]);
|
||||
memset(dst_y_opt, 0, kWidth * kHeight);
|
||||
memset(dst_u_opt, 0, (kWidth / 2) * (kHeight / 2));
|
||||
memset(dst_v_opt, 0, (kWidth / 2) * (kHeight / 2));
|
||||
RGB24ToI420(src_rgb24, kWidth * 3, dst_y_opt, kWidth, dst_u_opt, kWidth / 2,
|
||||
dst_v_opt, kWidth / 2, kWidth, kHeight);
|
||||
for (int i = 0; i < kWidth * kHeight; ++i) {
|
||||
ASSERT_EQ(dst_y_c[i], dst_y_opt[i]);
|
||||
}
|
||||
for (int i = 0; i < (kWidth / 2) * (kHeight / 2); ++i) {
|
||||
ASSERT_EQ(dst_u_c[i], dst_u_opt[i]);
|
||||
ASSERT_EQ(dst_v_c[i], dst_v_opt[i]);
|
||||
}
|
||||
}
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
|
||||
free_aligned_buffer_page_end(src_rgb24);
|
||||
free_aligned_buffer_page_end(dst_y_c);
|
||||
free_aligned_buffer_page_end(dst_u_c);
|
||||
free_aligned_buffer_page_end(dst_v_c);
|
||||
free_aligned_buffer_page_end(dst_y_opt);
|
||||
free_aligned_buffer_page_end(dst_u_opt);
|
||||
free_aligned_buffer_page_end(dst_v_opt);
|
||||
}
|
||||
|
||||
#ifdef ENABLE_ROW_TESTS
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \
|
||||
defined(HAS_HALFWIDTHROW_16TO8_AVX2) || \
|
||||
defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) || \
|
||||
defined(HAS_HALFROW_16TO8_AVX512BW)
|
||||
// Regression test for HalfWidthRow_16To8 (>= 0x8000 unsigned MSB samples and
|
||||
// exact page-end tail bounds) and HalfRow_16To8 (non-power-of-two scale).
|
||||
TEST_F(LibYUVPlanarTest, Half16To8_HighBitAndNonPow2Scale) {
|
||||
const int kWidths[] = {1, 7, 15, 24, 31, 33, 63, 65};
|
||||
for (size_t w_idx = 0; w_idx < sizeof(kWidths) / sizeof(kWidths[0]);
|
||||
++w_idx) {
|
||||
const int w = kWidths[w_idx];
|
||||
align_buffer_page_end_16(src_uv_hw, w * 4);
|
||||
align_buffer_page_end(dst_c, w);
|
||||
align_buffer_page_end(dst_opt, w);
|
||||
for (int i = 0; i < w * 4; ++i) {
|
||||
src_uv_hw[i] = static_cast<uint16_t>(0x8000u + ((i * 257u) & 0x7fffu));
|
||||
}
|
||||
HalfWidthRow_16To8_C(src_uv_hw, w * 2, dst_c, 256, w);
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
memset(dst_opt, 0x5a, w);
|
||||
HalfWidthRow_16To8_AVX512BW(src_uv_hw, w * 2, dst_opt, 256, w);
|
||||
for (int i = 0; i < w; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]) << "AVX512BW w=" << w << " i=" << i;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
memset(dst_opt, 0x5a, w);
|
||||
HalfWidthRow_16To8_Any_AVX2(src_uv_hw, w * 2, dst_opt, 256, w);
|
||||
for (int i = 0; i < w; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]) << "AVX2 w=" << w << " i=" << i;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
memset(dst_opt, 0x5a, w);
|
||||
HalfWidthRow_16To8_Any_SSSE3(src_uv_hw, w * 2, dst_opt, 256, w);
|
||||
for (int i = 0; i < w; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]) << "SSSE3 w=" << w << " i=" << i;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// HalfRow_16To8 with non-power-of-two scale (e.g. 16320).
|
||||
align_buffer_page_end_16(src_uv_hr, w * 2);
|
||||
for (int i = 0; i < w * 2; ++i) {
|
||||
src_uv_hr[i] = static_cast<uint16_t>((i * 73u) & 1023u);
|
||||
}
|
||||
HalfRow_16To8_C(src_uv_hr, w, dst_c, 16320, w);
|
||||
#if defined(HAS_HALFROW_16TO8_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
memset(dst_opt, 0x5a, w);
|
||||
HalfRow_16To8_AVX512BW(src_uv_hr, w, dst_opt, 16320, w);
|
||||
for (int i = 0; i < w; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i])
|
||||
<< "HalfRow_16To8_AVX512BW w=" << w << " i=" << i;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
free_aligned_buffer_page_end_16(src_uv_hr);
|
||||
free_aligned_buffer_page_end_16(src_uv_hw);
|
||||
free_aligned_buffer_page_end(dst_c);
|
||||
free_aligned_buffer_page_end(dst_opt);
|
||||
}
|
||||
}
|
||||
#endif // HAS_HALFWIDTHROW_16TO8_* || HAS_HALFROW_16TO8_AVX512BW
|
||||
|
||||
#if defined(HAS_UNPACKMT2T_AVX2)
|
||||
// Regression test for UnpackMT2T_C and UnpackMT2T_AVX2 when size is not a
|
||||
// multiple of the 80-byte (64-pixel) MT2T block size.
|
||||
TEST_F(LibYUVPlanarTest, UnpackMT2T_PartialTail) {
|
||||
if (!TestCpuFlag(kCpuHasAVX2)) {
|
||||
return;
|
||||
}
|
||||
const size_t kSrcSize = 120; // 1 full 80-byte block + 40 trailing bytes
|
||||
const size_t kDstCount = (kSrcSize / 80) * 64;
|
||||
align_buffer_page_end(src, kSrcSize);
|
||||
align_buffer_page_end_16(dst_c, kDstCount);
|
||||
align_buffer_page_end_16(dst_opt, kDstCount);
|
||||
MemRandomize(src, kSrcSize);
|
||||
UnpackMT2T_C(src, dst_c, kSrcSize);
|
||||
UnpackMT2T_AVX2(src, dst_opt, kSrcSize);
|
||||
for (size_t i = 0; i < kDstCount; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]) << "i=" << i;
|
||||
}
|
||||
// Verify size < 80 performs zero out-of-bounds reads or writes.
|
||||
UnpackMT2T_C(src + 80, dst_c, 40);
|
||||
UnpackMT2T_AVX2(src + 80, dst_opt, 40);
|
||||
for (size_t i = 0; i < kDstCount; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]) << "i=" << i;
|
||||
}
|
||||
free_aligned_buffer_page_end(src);
|
||||
free_aligned_buffer_page_end_16(dst_c);
|
||||
free_aligned_buffer_page_end_16(dst_opt);
|
||||
}
|
||||
#endif // HAS_UNPACKMT2T_AVX2
|
||||
#endif // ENABLE_ROW_TESTS
|
||||
|
||||
// Regression test for ConvertToLSBPlane_16 with depth=16 (scale=65536).
|
||||
TEST_F(LibYUVPlanarTest, ConvertToLSBPlane_16_Depth16) {
|
||||
const int kWidth = 37;
|
||||
const int kHeight = 4;
|
||||
align_buffer_page_end_16(src_y, kWidth * kHeight);
|
||||
align_buffer_page_end_16(dst_c, kWidth * kHeight);
|
||||
align_buffer_page_end_16(dst_opt, kWidth * kHeight);
|
||||
for (int i = 0; i < kWidth * kHeight; ++i) {
|
||||
src_y[i] = static_cast<uint16_t>(0x1234u + i * 251u);
|
||||
}
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
ConvertToLSBPlane_16(src_y, kWidth, dst_c, kWidth, kWidth, kHeight, 16);
|
||||
|
||||
const int kNoAvx512 =
|
||||
benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VL);
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
memset(dst_opt, 0, kWidth * kHeight * 2);
|
||||
ConvertToLSBPlane_16(src_y, kWidth, dst_opt, kWidth, kWidth, kHeight, 16);
|
||||
for (int i = 0; i < kWidth * kHeight; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]);
|
||||
}
|
||||
|
||||
MaskCpuFlags(kNoAvx512);
|
||||
memset(dst_opt, 0, kWidth * kHeight * 2);
|
||||
ConvertToLSBPlane_16(src_y, kWidth, dst_opt, kWidth, kWidth, kHeight, 16);
|
||||
for (int i = 0; i < kWidth * kHeight; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]);
|
||||
}
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
|
||||
free_aligned_buffer_page_end_16(src_y);
|
||||
free_aligned_buffer_page_end_16(dst_c);
|
||||
free_aligned_buffer_page_end_16(dst_opt);
|
||||
}
|
||||
|
||||
// Regression test for ARGBPolynomial saturation on outputs >= 32768 / 65536,
|
||||
// odd widths, and 4-byte-aligned (non-16-byte-aligned) polynomial arrays.
|
||||
TEST_F(LibYUVPlanarTest, ARGBPolynomial_LargeOutputAndUnalignedPoly) {
|
||||
const int kWidth = 37;
|
||||
const int kHeight = 3;
|
||||
align_buffer_page_end(src_argb, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(dst_c, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(dst_opt, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(poly_buf, 17 * sizeof(float));
|
||||
// Offset by +1 float so poly is 4-byte aligned, not 16-byte aligned.
|
||||
float* poly = reinterpret_cast<float*>(poly_buf) + 1;
|
||||
for (int c = 0; c < 4; ++c) {
|
||||
poly[0 + c] = 0.0f;
|
||||
poly[4 + c] = 300.0f; // 255 * 300 = 76500 >= 65536 -> must clamp to 255
|
||||
poly[8 + c] = 1.0f; // 200^2 = 40000 in [32768, 65535] -> clamp to 255
|
||||
poly[12 + c] = 0.0f;
|
||||
}
|
||||
for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
|
||||
src_argb[i] = static_cast<uint8_t>(i & 255);
|
||||
}
|
||||
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
ARGBPolynomial(src_argb, kWidth * 4, dst_c, kWidth * 4, poly, kWidth,
|
||||
kHeight);
|
||||
|
||||
const int kSse2Only =
|
||||
benchmark_cpu_info_ & ~(kCpuHasAVX2 | kCpuHasFMA3 | kCpuHasAVX512BW);
|
||||
const int kModes[2] = {benchmark_cpu_info_, kSse2Only};
|
||||
for (int m = 0; m < 2; ++m) {
|
||||
MaskCpuFlags(kModes[m]);
|
||||
memset(dst_opt, 0, kWidth * 4 * kHeight);
|
||||
ARGBPolynomial(src_argb, kWidth * 4, dst_opt, kWidth * 4, poly, kWidth,
|
||||
kHeight);
|
||||
for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]) << "mode=" << m << " i=" << i;
|
||||
}
|
||||
}
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
|
||||
free_aligned_buffer_page_end(src_argb);
|
||||
free_aligned_buffer_page_end(dst_c);
|
||||
free_aligned_buffer_page_end(dst_opt);
|
||||
free_aligned_buffer_page_end(poly_buf);
|
||||
}
|
||||
|
||||
// Regression test for ARGBBlur / CumulativeSumToAverageRow_AVX2 when radius=0
|
||||
// or radius=1 produces small window areas (area == 1 or area == 2) on 0xff.
|
||||
TEST_F(LibYUVPlanarTest, ARGBBlur_SmallRadiusWhite) {
|
||||
const int kWidth = 37;
|
||||
const int kHeight = 4;
|
||||
const int kCumSumStride32 = (kWidth + 16) * 4;
|
||||
align_buffer_page_end(src_argb, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(dst_c, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(dst_opt, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(cumsum, kCumSumStride32 * (kHeight + 2) * 4);
|
||||
memset(src_argb, 0xff, kWidth * 4 * kHeight);
|
||||
|
||||
for (int radius = 1; radius <= 2; ++radius) {
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
ARGBBlur(src_argb, kWidth * 4, dst_c, kWidth * 4,
|
||||
reinterpret_cast<int32_t*>(cumsum), kCumSumStride32, kWidth,
|
||||
kHeight, radius);
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
memset(dst_opt, 0, kWidth * 4 * kHeight);
|
||||
ARGBBlur(src_argb, kWidth * 4, dst_opt, kWidth * 4,
|
||||
reinterpret_cast<int32_t*>(cumsum), kCumSumStride32, kWidth,
|
||||
kHeight, radius);
|
||||
for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]) << "radius=" << radius << " i=" << i;
|
||||
}
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end(src_argb);
|
||||
free_aligned_buffer_page_end(dst_c);
|
||||
free_aligned_buffer_page_end(dst_opt);
|
||||
free_aligned_buffer_page_end(cumsum);
|
||||
}
|
||||
|
||||
// Regression test for ARGBLumaColorTable with standard BT.601 green weight
|
||||
// 150 (> 127) and bright pixels where R*77 + G*150 + B*29 > 32767.
|
||||
TEST_F(LibYUVPlanarTest, ARGBLumaColorTable_BrightPixels) {
|
||||
const int kWidth = 37;
|
||||
const int kHeight = 3;
|
||||
align_buffer_page_end(src_argb, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(dst_c, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(dst_opt, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(luma, 32768);
|
||||
for (int i = 0; i < 32768; ++i) {
|
||||
luma[i] = static_cast<uint8_t>(((i >> 8) * 3 + (i & 255)) & 255);
|
||||
}
|
||||
for (int i = 0; i < kWidth * kHeight; ++i) {
|
||||
src_argb[i * 4 + 0] = static_cast<uint8_t>(200 + (i % 56));
|
||||
src_argb[i * 4 + 1] = static_cast<uint8_t>(200 + ((i * 3) % 56));
|
||||
src_argb[i * 4 + 2] = static_cast<uint8_t>(200 + ((i * 7) % 56));
|
||||
src_argb[i * 4 + 3] = static_cast<uint8_t>(i & 255);
|
||||
}
|
||||
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
ARGBLumaColorTable(src_argb, kWidth * 4, dst_c, kWidth * 4, luma, kWidth,
|
||||
kHeight);
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
ARGBLumaColorTable(src_argb, kWidth * 4, dst_opt, kWidth * 4, luma, kWidth,
|
||||
kHeight);
|
||||
for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
|
||||
ASSERT_EQ(dst_c[i], dst_opt[i]) << "i=" << i;
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end(src_argb);
|
||||
free_aligned_buffer_page_end(dst_c);
|
||||
free_aligned_buffer_page_end(dst_opt);
|
||||
free_aligned_buffer_page_end(luma);
|
||||
}
|
||||
|
||||
#if defined(ENABLE_ROW_TESTS) && !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) || \
|
||||
defined(__i386__))
|
||||
// Regression test for 2-byte-aligned (non-16/32-byte-aligned) Intel
|
||||
// YuvConstants and ArgbConstants structs and custom non-0x8000 kAddUV offsets.
|
||||
TEST_F(LibYUVPlanarTest, UnalignedMatrixConstantsAndCustomKAddUV) {
|
||||
const int kWidth = 38;
|
||||
const int kHeight = 4;
|
||||
align_buffer_page_end(yuvc_buf, sizeof(YuvConstants) + 16);
|
||||
align_buffer_page_end(argbc_buf, sizeof(ArgbConstants) + 16);
|
||||
// Offset by +2 bytes (matches alignof == 2, misaligned for movdqa/vmovdqa).
|
||||
YuvConstants* yuvc = reinterpret_cast<YuvConstants*>(yuvc_buf + 2);
|
||||
ArgbConstants* argbc = reinterpret_cast<ArgbConstants*>(argbc_buf + 2);
|
||||
memcpy(yuvc, &kYuvI601Constants, sizeof(YuvConstants));
|
||||
memcpy(argbc, &kArgbI601Constants, sizeof(ArgbConstants));
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
argbc->kAddUV[i] = 0x7200u; // Non-default UV bias (!= 0x8000)
|
||||
}
|
||||
|
||||
align_buffer_page_end(src_argb, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(y_c, kWidth * kHeight);
|
||||
align_buffer_page_end(u_c, kWidth * kHeight);
|
||||
align_buffer_page_end(v_c, kWidth * kHeight);
|
||||
align_buffer_page_end(y_opt, kWidth * kHeight);
|
||||
align_buffer_page_end(u_opt, kWidth * kHeight);
|
||||
align_buffer_page_end(v_opt, kWidth * kHeight);
|
||||
align_buffer_page_end(rgb_c, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(rgb_opt, kWidth * 4 * kHeight);
|
||||
MemRandomize(src_argb, kWidth * 4 * kHeight);
|
||||
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
ARGBToI444Matrix(src_argb, kWidth * 4, y_c, kWidth, u_c, kWidth, v_c, kWidth,
|
||||
argbc, kWidth, kHeight);
|
||||
I420ToARGBMatrix(y_c, kWidth, u_c, kWidth / 2, v_c, kWidth / 2, rgb_c,
|
||||
kWidth * 4, yuvc, kWidth, kHeight);
|
||||
|
||||
const int kNoAvx512 =
|
||||
benchmark_cpu_info_ & ~(kCpuHasAVX512BW | kCpuHasAVX512VL |
|
||||
kCpuHasAVX512VBMI | kCpuHasAVX512VNNI);
|
||||
const int kSsse3Only = kNoAvx512 & ~kCpuHasAVX2;
|
||||
const int kModes[3] = {benchmark_cpu_info_, kNoAvx512, kSsse3Only};
|
||||
for (int m = 0; m < 3; ++m) {
|
||||
MaskCpuFlags(kModes[m]);
|
||||
ARGBToI444Matrix(src_argb, kWidth * 4, y_opt, kWidth, u_opt, kWidth, v_opt,
|
||||
kWidth, argbc, kWidth, kHeight);
|
||||
I420ToARGBMatrix(y_c, kWidth, u_c, kWidth / 2, v_c, kWidth / 2, rgb_opt,
|
||||
kWidth * 4, yuvc, kWidth, kHeight);
|
||||
for (int i = 0; i < kWidth * kHeight; ++i) {
|
||||
ASSERT_EQ(y_c[i], y_opt[i]);
|
||||
ASSERT_EQ(u_c[i], u_opt[i]) << "mode=" << m << " i=" << i;
|
||||
ASSERT_EQ(v_c[i], v_opt[i]) << "mode=" << m << " i=" << i;
|
||||
}
|
||||
for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
|
||||
ASSERT_EQ(rgb_c[i], rgb_opt[i]) << "mode=" << m << " i=" << i;
|
||||
}
|
||||
}
|
||||
|
||||
// Also test ARGBToI420Matrix with custom kAddUV and I400ToARGBMatrix with
|
||||
// unaligned YuvConstants.
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
ARGBToI420Matrix(src_argb, kWidth * 4, y_c, kWidth, u_c, kWidth / 2, v_c,
|
||||
kWidth / 2, argbc, kWidth, kHeight);
|
||||
I400ToARGBMatrix(y_c, kWidth, rgb_c, kWidth * 4, yuvc, kWidth, kHeight);
|
||||
for (int m = 0; m < 3; ++m) {
|
||||
MaskCpuFlags(kModes[m]);
|
||||
ARGBToI420Matrix(src_argb, kWidth * 4, y_opt, kWidth, u_opt, kWidth / 2,
|
||||
v_opt, kWidth / 2, argbc, kWidth, kHeight);
|
||||
I400ToARGBMatrix(y_c, kWidth, rgb_opt, kWidth * 4, yuvc, kWidth, kHeight);
|
||||
for (int i = 0; i < (kWidth / 2) * (kHeight / 2); ++i) {
|
||||
ASSERT_NEAR(u_c[i], u_opt[i], 1) << "mode=" << m << " i=" << i;
|
||||
ASSERT_NEAR(v_c[i], v_opt[i], 1) << "mode=" << m << " i=" << i;
|
||||
}
|
||||
for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
|
||||
ASSERT_EQ(rgb_c[i], rgb_opt[i]) << "mode=" << m << " i=" << i;
|
||||
}
|
||||
}
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
|
||||
free_aligned_buffer_page_end(yuvc_buf);
|
||||
free_aligned_buffer_page_end(argbc_buf);
|
||||
free_aligned_buffer_page_end(src_argb);
|
||||
free_aligned_buffer_page_end(y_c);
|
||||
free_aligned_buffer_page_end(u_c);
|
||||
free_aligned_buffer_page_end(v_c);
|
||||
free_aligned_buffer_page_end(y_opt);
|
||||
free_aligned_buffer_page_end(u_opt);
|
||||
free_aligned_buffer_page_end(v_opt);
|
||||
free_aligned_buffer_page_end(rgb_c);
|
||||
free_aligned_buffer_page_end(rgb_opt);
|
||||
}
|
||||
#endif // ENABLE_ROW_TESTS
|
||||
|
||||
// Regression test for MergeAR64Plane depth=16 (UBSan shift) and
|
||||
// MergeARGB16To8Plane depth=8 saturation on out-of-range >255 samples.
|
||||
TEST_F(LibYUVPlanarTest, MergeAR64Depth16AndMergeARGB16To8Depth8Clamp) {
|
||||
const int kWidth = 37;
|
||||
const int kHeight = 2;
|
||||
align_buffer_page_end_16(src_r, kWidth * kHeight);
|
||||
align_buffer_page_end_16(src_g, kWidth * kHeight);
|
||||
align_buffer_page_end_16(src_b, kWidth * kHeight);
|
||||
align_buffer_page_end_16(src_a, kWidth * kHeight);
|
||||
align_buffer_page_end_16(ar64_c, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end_16(ar64_opt, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(argb_c, kWidth * 4 * kHeight);
|
||||
align_buffer_page_end(argb_opt, kWidth * 4 * kHeight);
|
||||
for (int i = 0; i < kWidth * kHeight; ++i) {
|
||||
src_r[i] = static_cast<uint16_t>(200u + (i * 19u));
|
||||
src_g[i] = static_cast<uint16_t>(100u + (i * 37u));
|
||||
src_b[i] = static_cast<uint16_t>(50u + (i * 53u));
|
||||
src_a[i] = static_cast<uint16_t>(0x8000u + (i * 101u));
|
||||
}
|
||||
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
MergeAR64Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a, kWidth,
|
||||
ar64_c, kWidth * 4, kWidth, kHeight, 16);
|
||||
MergeARGB16To8Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a,
|
||||
kWidth, argb_c, kWidth * 4, kWidth, kHeight, 8);
|
||||
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
MergeAR64Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a, kWidth,
|
||||
ar64_opt, kWidth * 4, kWidth, kHeight, 16);
|
||||
MergeARGB16To8Plane(src_r, kWidth, src_g, kWidth, src_b, kWidth, src_a,
|
||||
kWidth, argb_opt, kWidth * 4, kWidth, kHeight, 8);
|
||||
|
||||
for (int i = 0; i < kWidth * 4 * kHeight; ++i) {
|
||||
ASSERT_EQ(ar64_c[i], ar64_opt[i]);
|
||||
ASSERT_EQ(argb_c[i], argb_opt[i]);
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end_16(src_r);
|
||||
free_aligned_buffer_page_end_16(src_g);
|
||||
free_aligned_buffer_page_end_16(src_b);
|
||||
free_aligned_buffer_page_end_16(src_a);
|
||||
free_aligned_buffer_page_end_16(ar64_c);
|
||||
free_aligned_buffer_page_end_16(ar64_opt);
|
||||
free_aligned_buffer_page_end(argb_c);
|
||||
free_aligned_buffer_page_end(argb_opt);
|
||||
}
|
||||
|
||||
} // namespace libyuv
|
||||
|
||||
@ -329,9 +329,12 @@ int main(int argc, const char* argv[]) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (num_rec <= 0) {
|
||||
fclose(file_org);
|
||||
exit(1);
|
||||
}
|
||||
// Open all files to compare to
|
||||
FILE** file_rec = new FILE*[num_rec];
|
||||
memset(file_rec, 0, num_rec * sizeof(FILE*)); // NOLINT
|
||||
FILE** file_rec = new FILE*[num_rec]();
|
||||
for (int cur_rec = 0; cur_rec < num_rec; ++cur_rec) {
|
||||
file_rec[cur_rec] = fopen(argv[fileindex_rec + cur_rec], "rb");
|
||||
if (file_rec[cur_rec] == NULL) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user