mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-11-09 14:49:57 +08:00
[libyuv] Upgrade x86 SIMD to AVX2/AVX-512 with 32-bit & row_win parity
Upgrade all remaining SSE2 and SSSE3 max-ISA kernels across row, scale,
rotate, and compare to AVX2 while removing superseded pre-AVX2 versions.
Implement AVX2 for ARGBDetectRow, HalfRow, HalfRow_16,
HalfWidthRow_16To8_Odd, AYUVTo{Y,UV,VU}Row, AR30To{ARGB,ABGR,AB30}Row,
and TransposeWx8_16, and port all remaining Arm NEON (or higher) kernels
to AVX2 (TransposeWxH_32, ScaleRowDown2*_16, ScaleUVRowDown2*,
ScaleUVRowDownEven, ScaleColsUp2, GaussCol*, Scale*Samples, and
UnpackMT2T).
Port 32-bit x86 (-m32 -fPIC) inline assembly across row_gcc.cc,
rotate_gcc.cc, and compare_gcc.cc (including 5-GPR PIC support for
I422ToAR30Row and I444/I422/I210/I410AlphaToARGBRow) so x86_32 matches
x64 SIMD coverage across all 187 supported row/rotate/compare kernels.
Handle 1..7 remainder elements directly in inline assembly without
calling *_C in ARGBDetectRow_AVX2, ARGBSepiaRow_AVX2,
ARGBColorMatrixRow_AVX2, ARGBQuantizeRow_AVX2, ARGBShadeRow_AVX2,
ARGBAffineRow_AVX2, GaussRow*_AVX2, GaussCol*_AVX2, and
*ColorTableRow_AVX2. Fix unsigned 16-bit packing in
ScaleRowUp2_{Linear,Bilinear}_16_SSE2.
Add 64-bit AVX-512 (AVX512BW, AVX512VBMI, and AVX512VNNI, including
SumSquareError_AVX512VNNI) implementations in *_gcc.cc and *_win.cc
wherever Arm64 exceeds NEON, and handle arbitrary tail widths via
opmasks in Mirror*_AVX512BW without _Any_ wrappers. Bring row_win.cc,
scale_win.cc, rotate_win.cc, and compare_win.cc C++ intrinsics (and
64-bit VISUALC_HAS_AVX2) to 100% parity with *_gcc.cc for MSVC and
MemorySanitizer.
Wire YUY2ToUVRow_HVX and UYVYToUVRow_HVX in YUY2ToI420 and UYVYToI420,
implement all 13 missing ScaleUV*_16_C kernels so UVScale_16 supports
downscaling and general upscaling, make Ssim8x8, TransposeWxH,
TransposeUVWxH, and TransposeWxH_16 static internal helpers, clean up
HAS_* guards, enable direct row unittests so all 279 *_C functions have
active test coverage, and add Intel AMX-TF32 (kCpuHasAMXTF32, CPUID
leaf 7 subleaf 1 EAX bit 4, LIBYUV_DISABLE_AMXTF32) detection.
Mark 16-bit RGB formats (RGB565, ARGB1555, and ARGB4444) and legacy
fixed-matrix 16-bit RGB-to-YUV row functions as deprecated in headers
and GEMINI.md, using two-step ARGB wrapper row functions in
row_common.cc.
AMD Zen 6 Benchmark (1280 width, 10000 iterations):
----------------------------------------------------------------
Function Before (ms) After (ms) Improvement
----------------------------------------------------------------
ARGB1555ToARGBRow 0.83 0.49 1.69x
ARGB1555ToUVRow 16.00 3.21 4.99x
ARGB1555ToYRow 7.82 1.38 5.66x
ARGB4444ToUVRow 15.04 2.58 5.83x
ARGB4444ToYRow 7.42 1.07 6.94x
ARGBAffineRow 3.51 2.75 1.28x
ARGBColorMatrixRow 4.09 1.13 3.63x
ARGBColorTableRow 11.21 8.93 1.26x
ARGBLumaColorTableRow 10.01 10.21 0.98x
ARGBQuantizeRow 2.23 1.21 1.84x
ARGBSepiaRow 3.29 0.81 4.04x
ARGBSetRow 0.21 0.21 1.02x
ARGBShadeRow 1.66 0.87 1.91x
ARGBToRGB565DitherRow 1.22 0.50 2.43x
ARGBToRGB565Row 1.09 0.44 2.47x
ByteToFloatRow 0.88 0.41 2.14x
Convert8To8Row 0.21 0.11 1.88x
CopyRow_16 0.11 0.11 0.97x
DetileRow 0.21 0.12 1.75x
DetileRow_16 0.23 0.14 1.67x
DetileSplitUVRow 0.29 0.21 1.38x
DivideRow_16 0.14 0.11 1.30x
GaussRow 3.29 1.27 2.59x
GaussRow_F32 1.93 0.89 2.17x
HalfFloatRow 1.16 0.74 1.57x
I210AlphaToARGBRow 1.83 0.92 2.00x
I210ToAR30Row 2.21 0.96 2.30x
I210ToARGBRow 1.75 0.87 2.00x
I212ToAR30Row 2.21 0.96 2.31x
I212ToARGBRow 1.75 0.87 2.01x
I410AlphaToARGBRow 1.73 0.86 2.00x
I410ToAR30Row 2.04 0.89 2.28x
I410ToARGBRow 1.64 0.82 2.00x
I422ToARGB1555Row 2.94 2.27 1.30x
I422ToARGB4444Row 2.94 2.27 1.29x
I422ToRGB565Row 2.63 1.27 2.07x
I422ToRGBARow 1.51 0.82 1.83x
I444AlphaToARGBRow 1.52 0.87 1.73x
I444ToARGBRow 1.48 0.81 1.82x
I444ToRGB24Row 2.13 0.72 2.98x
MergeUVRow_16 0.87 0.32 2.74x
P210ToAR30Row 1.77 0.70 2.54x
P210ToARGBRow 1.38 0.62 2.24x
P410ToAR30Row 1.94 0.85 2.28x
P410ToARGBRow 1.45 0.77 1.90x
RGB565ToUVRow 15.84 2.88 5.49x
RGB565ToYRow 7.82 1.22 6.42x
RGBColorTableRow 8.42 6.95 1.21x
SetRow 0.11 0.11 1.04x
SobelRow 0.80 0.43 1.87x
SobelToPlaneRow 0.27 0.12 2.25x
SobelXRow 1.58 0.85 1.85x
SobelXYRow 0.80 0.55 1.44x
SobelYRow 1.58 0.85 1.85x
UYVYToARGBRow 1.28 0.65 1.96x
UYVYToUVRow 0.38 0.21 1.78x
UYVYToYRow 0.21 0.11 1.87x
YUY2ToARGBRow 1.28 0.65 1.96x
YUY2ToUVRow 0.40 0.21 1.87x
YUY2ToYRow 0.21 0.11 1.88x
----------------------------------------------------------------
Row/Rotate/Compare SIMD ISA per *_C (* = uncalled in library):
------------------------------------------------------------------------
C Function arm64 x86_32 x64 row_win
------------------------------------------------------------------------
AB64ToARGBRow_C NEON AVX2 AVX2 AVX2
ABGRToAR30Row_C NEON AVX2 AVX512BW AVX512BW
AR30ToAB30Row_C NEON AVX2 AVX512BW AVX512BW
AR30ToABGRRow_C NEON AVX2 AVX512BW AVX512BW
AR30ToARGBRow_C NEON AVX2 AVX512BW AVX512BW
AR64ShuffleRow_C NEON AVX2 AVX512BW AVX512BW
AR64ToAB64Row_C NEON AVX2 AVX512BW AVX512BW
AR64ToARGBRow_C NEON AVX2 AVX2 AVX2
ARGB1555ToARGBRow_C SVE2 AVX2 AVX512BW AVX512BW
ARGB1555ToUVMatrixRow_C NEON AVX2 AVX2 AVX2
ARGB1555ToUVRow_C * NEON AVX2 AVX2 AVX2
ARGB1555ToYMatrixRow_C NEON AVX2 AVX2 AVX2
ARGB1555ToYRow_C * NEON AVX2 AVX2 AVX2
ARGB4444ToARGBRow_C NEON AVX2 AVX2 AVX2
ARGB4444ToUVMatrixRow_C NEON AVX2 AVX2 AVX2
ARGB4444ToUVRow_C * NEON AVX2 AVX2 AVX2
ARGB4444ToYMatrixRow_C NEON AVX2 AVX2 AVX2
ARGB4444ToYRow_C * NEON AVX2 AVX2 AVX2
ARGBAddRow_C NEON AVX2 AVX2 AVX2
ARGBAffineRow_C * NEON AVX2 AVX2 AVX2
ARGBAlphaBlendRow_C SME AVX2 AVX512BW AVX512BW
ARGBAttenuateRow_C NEON AVX2 AVX512BW AVX512BW
ARGBBlendRow_C NEON AVX2 AVX512BW AVX512BW
ARGBColorMatrixRow_C I8MM AVX2 AVX512VNNI AVX512VNNI
ARGBColorTableRow_C NEON AVX2 AVX2 AVX2
ARGBCopyAlphaRow_C NEON AVX2 AVX2 AVX2
ARGBCopyYToAlphaRow_C NEON AVX2 AVX512BW AVX512BW
ARGBDetectRow_C - AVX2 AVX2 AVX2
ARGBExtractAlphaRow_C NEON AVX2 AVX512BW AVX512BW
ARGBGrayRow_C DOTPROD AVX2 AVX512VNNI AVX512VNNI
ARGBLumaColorTableRow_C NEON AVX2 AVX2 AVX2
ARGBMirrorRow_C NEON AVX2 AVX512BW AVX512BW
ARGBMultiplyRow_C SME AVX2 AVX512BW AVX512BW
ARGBOrRow_C SME AVX2 AVX512BW AVX512BW
ARGBPolynomialRow_C NEON AVX2 AVX2 AVX2
ARGBQuantizeRow_C NEON AVX2 AVX2 AVX2
ARGBSepiaRow_C DOTPROD AVX2 AVX512VNNI AVX512VNNI
ARGBSetRow_C NEON AVX2 AVX2 AVX2
ARGBShadeRow_C NEON AVX2 AVX2 AVX2
ARGBShuffleRow_C NEON AVX2 AVX512BW AVX512BW
ARGBSubtractRow_C NEON AVX2 AVX2 AVX2
ARGBToAB64Row_C NEON AVX2 AVX2 AVX2
ARGBToABGRRow_C NEON AVX2 AVX512BW AVX512BW
ARGBToAR30Row_C NEON AVX2 AVX512BW AVX512BW
ARGBToAR64Row_C NEON AVX2 AVX2 AVX2
ARGBToARGB1555Row_C NEON AVX2 AVX2 AVX2
ARGBToARGB4444Row_C NEON AVX2 AVX2 AVX2
ARGBToBGRARow_C NEON AVX2 AVX512BW AVX512BW
ARGBToRAWRow_C SVE2 AVX2 AVX512BW AVX512BW
ARGBToRGB24Row_C SVE2 AVX2 AVX512BW AVX512BW
ARGBToRGB565DitherRow_C SVE2 AVX2 AVX512BW AVX512BW
ARGBToRGB565Row_C SVE2 AVX2 AVX512BW AVX512BW
ARGBToRGBARow_C NEON AVX2 AVX512BW AVX512BW
ARGBToUV444MatrixRow_C I8MM AVX2 AVX512VNNI AVX512VNNI
ARGBToUVMatrixRow_C SME AVX2 AVX512VNNI AVX512VNNI
ARGBToYMatrixRow_C SME AVX2 AVX512VNNI AVX512VNNI
ARGBUnattenuateRow_C NEON AVX2 AVX2 AVX2
AYUVToUVRow_C SVE2 AVX2 AVX512VBMI AVX512VBMI
AYUVToVURow_C SVE2 AVX2 AVX512VBMI AVX512VBMI
AYUVToYRow_C NEON AVX2 AVX512VBMI AVX512VBMI
BlendPlaneRow_C NEON AVX2 AVX512BW AVX512BW
ByteToFloatRow_C NEON AVX2 AVX2 AVX2
ComputeCumulativeSumRow_C NEON AVX2 AVX2 AVX2
Convert16To8Row_C SME AVX2 AVX512BW AVX512BW
Convert8To16Row_C SME AVX2 AVX512BW AVX512BW
Convert8To8Row_C SME AVX2 AVX512BW AVX512BW
CopyRow_16_C NEON AVX2 AVX2 AVX2
CopyRow_C SME AVX AVX512BW AVX512BW
CumulativeSumToAverageRow_C NEON AVX2 AVX2 AVX2
DetileRow_16_C NEON AVX2 AVX2 AVX2
DetileRow_C NEON AVX2 AVX2 AVX2
DetileSplitUVRow_C NEON AVX2 AVX2 AVX2
DetileToYUY2_C NEON AVX2 AVX2 AVX2
DivideRow_16_C SVE2 AVX2 AVX512BW AVX512BW
GaussCol_C * NEON AVX2 AVX2 AVX2
GaussCol_F32_C NEON AVX2 AVX2 AVX2
GaussRow_C * NEON AVX2 AVX2 AVX2
GaussRow_F32_C NEON AVX2 AVX2 AVX2
HalfFloatRow_C SVE2 AVX2 AVX512BW AVX512BW
HalfMergeUVRow_C SVE2 AVX2 AVX512BW AVX512BW
HalfRow_16To8_C SME AVX2 AVX512BW AVX512BW
HalfRow_16_C - AVX2 AVX2 AVX2
HalfRow_C - AVX2 AVX2 AVX2
HalfWidthRow_16To8_C SME AVX2 AVX512BW AVX512BW
HalfWidthRow_16To8_Odd_C - AVX2 AVX2 AVX2
HammingDistance_C DOTPROD AVX2 AVX512BW AVX512BW
HashDjb2_C NEON AVX2 AVX512BW AVX512BW
I210AlphaToARGBRow_C SME AVX2 AVX512BW AVX512BW
I210ToAR30Row_C SME AVX2 AVX512BW AVX512BW
I210ToARGBRow_C SME AVX2 AVX512BW AVX512BW
I212ToAR30Row_C SME AVX2 AVX512BW AVX512BW
I212ToARGBRow_C SME AVX2 AVX512BW AVX512BW
I400ToARGBRow_C SME AVX2 AVX512BW AVX512BW
I410AlphaToARGBRow_C SME AVX2 AVX512BW AVX512BW
I410ToAR30Row_C SME AVX2 AVX512BW AVX512BW
I410ToARGBRow_C SME AVX2 AVX512BW AVX512BW
I422AlphaToARGBRow_C SME AVX2 AVX512BW AVX512BW
I422ToAR30Row_C SME AVX2 AVX512BW AVX512BW
I422ToARGB1555Row_C SME AVX2 AVX512BW AVX512BW
I422ToARGB4444Row_C SME AVX2 AVX512BW AVX512BW
I422ToARGBRow_C SME AVX2 AVX512BW AVX512BW
I422ToRGB24Row_C SME AVX2 AVX512VBMI AVX512VBMI
I422ToRGB565Row_C SME AVX2 AVX512BW AVX512BW
I422ToRGBARow_C SME AVX2 AVX512BW AVX512BW
I422ToUYVYRow_C NEON AVX2 AVX2 AVX2
I422ToYUY2Row_C NEON AVX2 AVX2 AVX2
I444AlphaToARGBRow_C SME AVX2 AVX512BW AVX512BW
I444ToARGBRow_C SME AVX2 AVX512BW AVX512BW
I444ToRGB24Row_C SME AVX2 AVX512VBMI AVX512VBMI
InterpolateRow_16_C SME AVX2 AVX512BW AVX512BW
InterpolateRow_C SME AVX2 AVX512BW AVX512BW
J400ToARGBRow_C NEON AVX2 AVX512BW AVX512BW
MergeAR64Row_C NEON AVX2 AVX2 AVX2
MergeARGB16To8Row_C NEON AVX2 AVX2 AVX2
MergeARGBRow_C NEON AVX2 AVX2 AVX2
MergeRGBRow_C NEON AVX2 AVX512VBMI AVX512VBMI
MergeUVRow_16_C SME AVX2 AVX512BW AVX512BW
MergeUVRow_C SME AVX2 AVX512BW AVX512BW
MergeXR30Row_C NEON AVX2 AVX512BW AVX512BW
MergeXR64Row_C NEON AVX2 AVX2 AVX2
MergeXRGB16To8Row_C NEON AVX2 AVX2 AVX2
MergeXRGBRow_C NEON AVX2 AVX2 AVX2
MirrorRow_16_C NEON AVX2 AVX512BW AVX512BW
MirrorRow_C NEON AVX2 AVX512BW AVX512BW
MirrorSplitUVRow_C NEON AVX2 AVX512BW AVX512BW
MirrorUVRow_C NEON AVX2 AVX512BW AVX512BW
MultiplyRow_16_C SME AVX2 AVX512BW AVX512BW
NV12ToARGBRow_C SME AVX2 AVX512BW AVX512BW
NV12ToRGB24Row_C SME AVX2 AVX512VBMI AVX512VBMI
NV12ToRGB565Row_C NEON AVX2 AVX2 AVX2
NV21ToARGBRow_C SME AVX2 AVX512BW AVX512BW
NV21ToRGB24Row_C SME AVX2 AVX512VBMI AVX512VBMI
NV21ToYUV24Row_C NEON AVX2 AVX2 AVX2
P210ToAR30Row_C SME AVX2 AVX512BW AVX512BW
P210ToARGBRow_C SME AVX2 AVX512BW AVX512BW
P410ToAR30Row_C SME AVX2 AVX512BW AVX512BW
P410ToARGBRow_C SME AVX2 AVX512BW AVX512BW
RAWToARGBRow_C SVE2 AVX2 AVX512BW AVX512BW
RAWToRGB24Row_C SVE2 AVX2 AVX512VBMI AVX512VBMI
RAWToRGBARow_C SVE2 AVX2 AVX512VBMI AVX512VBMI
RGB24MirrorRow_C NEON AVX2 AVX512VBMI AVX512VBMI
RGB24ToARGBRow_C SVE2 AVX2 AVX512BW AVX512BW
RGB565ToARGBRow_C NEON AVX2 AVX2 AVX2
RGB565ToUVMatrixRow_C NEON AVX2 AVX2 AVX2
RGB565ToUVRow_C * NEON AVX2 AVX2 AVX2
RGB565ToYMatrixRow_C NEON AVX2 AVX2 AVX2
RGB565ToYRow_C * NEON AVX2 AVX2 AVX2
RGBAToARGBRow_C NEON AVX2 AVX512BW AVX512BW
RGBColorTableRow_C NEON AVX2 AVX2 AVX2
RGBToUV444MatrixRow_C NEON AVX2 AVX512BW AVX512BW
RGBToUVMatrixRow_C NEON AVX2 AVX512BW AVX512BW
RGBToYMatrixRow_C DOTPROD AVX2 AVX512BW AVX512BW
ScaleMaxSamples_C * NEON AVX2 AVX2 AVX2
ScaleSamples_C * NEON AVX2 AVX2 AVX2
ScaleSumSamples_C * NEON AVX2 AVX2 AVX2
SetRow_C NEON AVX2 AVX2 AVX2
SobelRow_C NEON AVX2 AVX2 AVX2
SobelToPlaneRow_C NEON AVX2 AVX2 AVX2
SobelXRow_C NEON AVX2 AVX2 AVX2
SobelXYRow_C NEON AVX2 AVX2 AVX2
SobelYRow_C NEON AVX2 AVX2 AVX2
SplitARGBRow_C NEON AVX2 AVX2 AVX2
SplitRGBRow_C NEON AVX2 AVX2 AVX2
SplitUVRow_16_C NEON AVX2 AVX2 AVX2
SplitUVRow_C SVE2 AVX2 AVX512BW AVX512BW
SplitXRGBRow_C NEON AVX2 AVX2 AVX2
Ssim4x4Row_C SME AVX2 AVX512BW AVX512BW
SsimWindowRow_C SME AVX2 AVX512BW AVX512BW
SumSquareError_C DOTPROD AVX2 AVX512VNNI AVX512VNNI
SwapUVRow_C NEON AVX2 AVX512BW AVX512BW
TransposeUVWx16_C - AVX2 AVX512BW AVX512BW
TransposeUVWx8_C NEON AVX2 AVX2 AVX2
TransposeWx16_C NEON AVX2 AVX512BW AVX512BW
TransposeWx4_32_C * SVE2 AVX2 AVX512BW AVX512BW
TransposeWx8_16_C SVE2 AVX2 AVX512BW AVX512BW
TransposeWx8_C - AVX2 AVX2 AVX2
TransposeWxH_32_C NEON AVX2 AVX2 AVX2
UYVYToARGBRow_C SME AVX2 AVX512BW AVX512BW
UYVYToUV422Row_C NEON AVX2 AVX2 AVX2
UYVYToUVRow_C SVE2 AVX2 AVX512BW AVX512BW
UYVYToYRow_C SVE2 AVX2 AVX512BW AVX512BW
UnpackMT2T_C NEON AVX2 AVX2 AVX2
YUY2ToARGBRow_C SME AVX2 AVX512BW AVX512BW
YUY2ToNVUVRow_C NEON AVX2 AVX2 AVX2
YUY2ToUV422Row_C NEON AVX2 AVX2 AVX2
YUY2ToUVRow_C SVE2 AVX2 AVX512BW AVX512BW
YUY2ToYRow_C SVE2 AVX2 AVX512BW AVX512BW
------------------------------------------------------------------------
SIMD (187 row/rot/cmp) 181 97% 187 100% 187 100% 187 100%
------------------------------------------------------------------------
Scale SIMD ISA supported per *_C function:
------------------------------------------------------------------------
C Function arm64 x86_32 x64 row_win
------------------------------------------------------------------------
FixedDiv1_C - X86 X86 X86
FixedDiv_C - X86 X86 X86
ScaleARGBCols64_C - - - -
ScaleARGBColsUp2_C - AVX2 AVX2 AVX2
ScaleARGBCols_C NEON AVX2 AVX2 AVX2
ScaleARGBFilterCols64_C - - - -
ScaleARGBFilterCols_C NEON AVX2 AVX2 AVX2
ScaleARGBRowDown2Box_C SME AVX2 AVX2 AVX2
ScaleARGBRowDown2Linear_C SME AVX2 AVX2 AVX2
ScaleARGBRowDown2_C SME AVX2 AVX2 AVX2
ScaleARGBRowDownEvenBox_C NEON AVX2 AVX2 AVX2
ScaleARGBRowDownEven_C NEON AVX2 AVX2 AVX2
ScaleAddCols0_C - - - -
ScaleAddCols1_16_C - - - -
ScaleAddCols1_C - - AVX512BW AVX512BW
ScaleAddCols2_16_C - - - -
ScaleAddCols2_C - - AVX512BW AVX512BW
ScaleAddRow_16_C - - - -
ScaleAddRow_C NEON AVX2 AVX512BW AVX512BW
ScaleColsUp2_16_C - - - -
ScaleColsUp2_C - AVX2 AVX2 AVX2
ScaleCols_16_C - - - -
ScaleCols_C - - - -
ScaleFilterCols64_16_C - - - -
ScaleFilterCols64_C - - - -
ScaleFilterCols_16_C - - - -
ScaleFilterCols_C NEON AVX2 AVX2 AVX2
ScaleRowDown2Box_16_C SME AVX2 AVX2 AVX2
ScaleRowDown2Box_C SME AVX2 AVX512BW AVX512BW
ScaleRowDown2Box_Odd_C - - AVX512BW AVX512BW
ScaleRowDown2Linear_16_C SME AVX2 AVX2 AVX2
ScaleRowDown2Linear_C SME AVX2 AVX512BW AVX512BW
ScaleRowDown2_16_C SME AVX2 AVX2 AVX2
ScaleRowDown2_C SME AVX2 AVX512BW AVX512BW
ScaleRowDown34_0_Box_16_C - - - -
ScaleRowDown34_0_Box_C NEON AVX2 AVX2 AVX2
ScaleRowDown34_16_C - - - -
ScaleRowDown34_1_Box_16_C - - - -
ScaleRowDown34_1_Box_C NEON AVX2 AVX2 AVX2
ScaleRowDown34_C NEON AVX2 AVX2 AVX2
ScaleRowDown38_16_C - - - -
ScaleRowDown38_2_Box_16_C - - - -
ScaleRowDown38_2_Box_C NEON AVX2 AVX2 AVX2
ScaleRowDown38_3_Box_16_C - - - -
ScaleRowDown38_3_Box_C NEON AVX2 AVX2 AVX2
ScaleRowDown38_C NEON AVX2 AVX2 AVX2
ScaleRowDown4Box_16_C - - - -
ScaleRowDown4Box_C NEON AVX2 AVX2 AVX2
ScaleRowDown4_16_C - - - -
ScaleRowDown4_C NEON AVX2 AVX2 AVX2
ScaleRowUp2_Bilinear_16_Any_C NEON AVX2 AVX2 AVX2
ScaleRowUp2_Bilinear_16_C NEON AVX2 AVX2 AVX2
ScaleRowUp2_Bilinear_Any_C NEON AVX2 AVX2 AVX2
ScaleRowUp2_Bilinear_C NEON AVX2 AVX2 AVX2
ScaleRowUp2_Linear_16_Any_C NEON AVX2 AVX2 AVX2
ScaleRowUp2_Linear_16_C NEON AVX2 AVX2 AVX2
ScaleRowUp2_Linear_Any_C NEON AVX2 AVX2 AVX2
ScaleRowUp2_Linear_C NEON AVX2 AVX2 AVX2
ScaleUVAddCols1_16_C - - - -
ScaleUVAddCols1_C - - - -
ScaleUVAddCols2_16_C - - - -
ScaleUVAddCols2_C - - - -
ScaleUVAddCols3_16_C - - - -
ScaleUVAddCols3_C SME AVX2 AVX512VBMI AVX512VBMI
ScaleUVCols64_16_C - - - -
ScaleUVCols64_C - - - -
ScaleUVColsUp2_16_C - - - -
ScaleUVColsUp2_C - - - -
ScaleUVCols_16_C - - - -
ScaleUVCols_C - - - -
ScaleUVFilterCols64_16_C - - - -
ScaleUVFilterCols64_C - - - -
ScaleUVFilterCols_16_C - - - -
ScaleUVFilterCols_C - - - -
ScaleUVRowDown2Box_16_C - - - -
ScaleUVRowDown2Box_C SME AVX2 AVX2 AVX2
ScaleUVRowDown2Linear_16_C - - - -
ScaleUVRowDown2Linear_C SME AVX2 AVX2 AVX2
ScaleUVRowDown2_16_C - - - -
ScaleUVRowDown2_C SME AVX2 AVX2 AVX2
ScaleUVRowDownEvenBox_16_C - - - -
ScaleUVRowDownEvenBox_C - - - -
ScaleUVRowDownEven_16_C - - - -
ScaleUVRowDownEven_C NEON AVX2 AVX2 AVX2
ScaleUVRowUp2_Bilinear_16_Any_C NEON AVX2 AVX2 AVX2
ScaleUVRowUp2_Bilinear_16_C NEON AVX2 AVX2 AVX2
ScaleUVRowUp2_Bilinear_Any_C NEON AVX2 AVX2 AVX2
ScaleUVRowUp2_Bilinear_C NEON AVX2 AVX2 AVX2
ScaleUVRowUp2_Linear_16_Any_C NEON AVX2 AVX2 AVX2
ScaleUVRowUp2_Linear_16_C NEON AVX2 AVX2 AVX2
ScaleUVRowUp2_Linear_Any_C NEON AVX2 AVX2 AVX2
ScaleUVRowUp2_Linear_C NEON AVX2 AVX2 AVX2
------------------------------------------------------------------------
SIMD (92 scale *_C) 44 48% 48 52% 51 55% 51 55%
------------------------------------------------------------------------
Test: libyuv_unittest
Bug: none
Change-Id: I2b4a832f2a7fe1fe93487b453240cd15c1f5f940
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8676400
Commit-Queue: Frank Barchard <fbarchard@google.com>
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard
libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent
63e54f7374
commit
8d440c181c
31
GEMINI.md
31
GEMINI.md
@ -47,11 +47,30 @@ and compiler compatibility.
|
||||
|
||||
### Utility and Fallbacks
|
||||
|
||||
* **row_common.cc**: Portable C/C++ versions. This is the reference
|
||||
implementation.
|
||||
* **row_common.cc**: Portable C/C++ versions and multi-step `ARGB` wrapper
|
||||
row functions. This is the reference implementation.
|
||||
* **row_any.cc**: Handles "remainder" pixels for widths not multiples of SIMD
|
||||
register size. Used for x86, Neon, and MIPS. Not required for SVE2, SME,
|
||||
RVV, or HVX due to hardware-level masking.
|
||||
register size. Used for SSE/AVX2, Neon, and LoongArch (LSX/LASX). Not
|
||||
required for AVX-512, SVE2, SME, RVV, or HVX, which handle tail masking
|
||||
directly inside the Row function.
|
||||
|
||||
## Deprecated and Lower-Priority Formats
|
||||
|
||||
The following datatypes and functions are deprecated or lower priority and are
|
||||
not always fully hand-optimized across every SIMD extension:
|
||||
|
||||
* **16-bit RGB formats** (deprecated): `RGB565` (`FOURCC_RGBP`,
|
||||
`FOURCC_L565`), `ARGB1555` (`FOURCC_RGBO`, `FOURCC_L555`, `FOURCC_5551`),
|
||||
and `ARGB4444` (`FOURCC_R444`).
|
||||
* **Legacy fixed-matrix RGB-to-YUV row functions** (deprecated in favor of
|
||||
`*MatrixRow` variants): `RGB565ToYRow`, `RGB565ToUVRow`, `ARGB1555ToYRow`,
|
||||
`ARGB1555ToUVRow`, `ARGB4444ToYRow`, `ARGB4444ToUVRow`.
|
||||
* **Row-biplanar YUV** (deprecated): `M420` (`FOURCC_M420`).
|
||||
|
||||
When adding SIMD coverage for deprecated or less relevant formats, prefer
|
||||
two-step wrapper implementations in `source/row_common.cc` that convert to or
|
||||
from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing
|
||||
`*MatrixRow` functions) rather than writing large bespoke assembly kernels.
|
||||
|
||||
## Coding Guidelines
|
||||
|
||||
@ -72,7 +91,9 @@ and compiler compatibility.
|
||||
### x86 Architectures (32-bit and 64-bit)
|
||||
|
||||
1. **AVX512 Logic**: AVX512 row functions are strictly enabled for **64-bit x86
|
||||
only**.
|
||||
only**. Do not create `_Any_` wrappers for AVX512 row functions; handle
|
||||
arbitrary tail widths directly inside the Row function using `k1`-`k7`
|
||||
opmask registers.
|
||||
2. **MSVC Intrinsics (`row_win.cc`)**: When converting `row_gcc.cc` to
|
||||
`row_win.cc`, use `_mm512_permutex2var_epi8`, not `_mm512_permi2var_epi8`
|
||||
intrinsic.
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Name: libyuv
|
||||
URL: https://chromium.googlesource.com/libyuv/libyuv/
|
||||
Version: 1976
|
||||
Version: 1977
|
||||
Revision: DEPS
|
||||
License: BSD-3-Clause
|
||||
License File: LICENSE
|
||||
|
||||
@ -33,6 +33,8 @@ By default the cpu is detected and the most advanced form of SIMD is used. But
|
||||
LIBYUV_DISABLE_AVXVNNI
|
||||
LIBYUV_DISABLE_AVXVNNIINT8
|
||||
LIBYUV_DISABLE_AMXINT8
|
||||
LIBYUV_DISABLE_AVX512BMM
|
||||
LIBYUV_DISABLE_AMXTF32
|
||||
|
||||
## Arm CPUs
|
||||
|
||||
|
||||
@ -21,14 +21,14 @@ namespace libyuv {
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// The following are available for 32 bit x86, Visual C 64 bit, and for clang
|
||||
// with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These
|
||||
// are intrinsics in compare_win.cc.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__i386__) || defined(_M_IX86) || \
|
||||
((defined(__x86_64__) || defined(_M_X64)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))))
|
||||
// The following are available for Visual C 32/64 bit, and for clang with
|
||||
// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These are
|
||||
// intrinsics in compare_win.cc.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
||||
defined(_M_IX86)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
#define HAS_HAMMINGDISTANCE_SSE42
|
||||
#define HAS_SUMSQUAREERROR_SSE2
|
||||
#define HAS_HASHDJB2_SSE41
|
||||
@ -39,19 +39,26 @@ extern "C" {
|
||||
#define HAS_HASHDJB2_AVX2
|
||||
#define HAS_SSIM4X4ROW_AVX2
|
||||
#define HAS_SSIMWINDOWROW_AVX2
|
||||
#define HAS_ARGBDETECTROW_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define HAS_HAMMINGDISTANCE_AVX512BW
|
||||
#define HAS_SUMSQUAREERROR_AVX512BW
|
||||
#define HAS_SUMSQUAREERROR_AVX512VNNI
|
||||
#define HAS_HASHDJB2_AVX512BW
|
||||
#define HAS_SSIM4X4ROW_AVX512BW
|
||||
#define HAS_SSIMWINDOWROW_AVX512BW
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// The following are available for 64 bit GCC and clang, as inline assembly in
|
||||
// compare_gcc.cc:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
// The following are available for 32/64 bit GCC and clang, as inline assembly
|
||||
// in compare_gcc.cc:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(!defined(_MSC_VER) || defined(__clang__))
|
||||
#if defined(__x86_64__)
|
||||
#define HAS_HAMMINGDISTANCE_SSE42
|
||||
#endif
|
||||
#define HAS_HAMMINGDISTANCE_SSSE3
|
||||
#define HAS_SUMSQUAREERROR_SSE2
|
||||
#define HAS_HASHDJB2_SSE41
|
||||
@ -63,10 +70,12 @@ extern "C" {
|
||||
#define HAS_HASHDJB2_AVX2
|
||||
#define HAS_SSIM4X4ROW_AVX2
|
||||
#define HAS_SSIMWINDOWROW_AVX2
|
||||
#define HAS_ARGBDETECTROW_AVX2
|
||||
#endif
|
||||
#if defined(CLANG_HAS_AVX512)
|
||||
#if defined(__x86_64__) && defined(CLANG_HAS_AVX512)
|
||||
#define HAS_HAMMINGDISTANCE_AVX512BW
|
||||
#define HAS_SUMSQUAREERROR_AVX512BW
|
||||
#define HAS_SUMSQUAREERROR_AVX512VNNI
|
||||
#define HAS_HASHDJB2_AVX512BW
|
||||
#define HAS_SSIM4X4ROW_AVX512BW
|
||||
#define HAS_SSIMWINDOWROW_AVX512BW
|
||||
@ -177,6 +186,9 @@ uint32_t SumSquareError_AVX2(const uint8_t* src_a,
|
||||
uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
uint32_t SumSquareError_NEON(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count);
|
||||
@ -298,6 +310,9 @@ float SsimWindowRow_RVV(const uint32_t* row0,
|
||||
const uint32_t* row1,
|
||||
int num_windows);
|
||||
|
||||
uint32_t ARGBDetectRow_C(const uint8_t* argb, int width);
|
||||
uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -1152,7 +1152,7 @@ int RAWToJ444(const uint8_t* src_raw,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// RGB16 (RGBP fourcc) little endian to I420.
|
||||
// RGB16 (RGBP fourcc) little endian to I420. Deprecated.
|
||||
LIBYUV_API
|
||||
int RGB565ToI420(const uint8_t* src_rgb565,
|
||||
int src_stride_rgb565,
|
||||
@ -1165,7 +1165,7 @@ int RGB565ToI420(const uint8_t* src_rgb565,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// RGB15 (RGBO fourcc) little endian to I420.
|
||||
// RGB15 (RGBO fourcc) little endian to I420. Deprecated.
|
||||
LIBYUV_API
|
||||
int ARGB1555ToI420(const uint8_t* src_argb1555,
|
||||
int src_stride_argb1555,
|
||||
@ -1178,7 +1178,7 @@ int ARGB1555ToI420(const uint8_t* src_argb1555,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// RGB12 (R444 fourcc) little endian to I420.
|
||||
// RGB12 (R444 fourcc) little endian to I420. Deprecated.
|
||||
LIBYUV_API
|
||||
int ARGB4444ToI420(const uint8_t* src_argb4444,
|
||||
int src_stride_argb4444,
|
||||
|
||||
@ -1038,7 +1038,7 @@ int RAWToRGBA(const uint8_t* src_raw,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// RGB16 (RGBP fourcc) little endian to ARGB.
|
||||
// RGB16 (RGBP fourcc) little endian to ARGB. Deprecated.
|
||||
LIBYUV_API
|
||||
int RGB565ToARGB(const uint8_t* src_rgb565,
|
||||
int src_stride_rgb565,
|
||||
@ -1047,7 +1047,7 @@ int RGB565ToARGB(const uint8_t* src_rgb565,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// RGB15 (RGBO fourcc) little endian to ARGB.
|
||||
// RGB15 (RGBO fourcc) little endian to ARGB. Deprecated.
|
||||
LIBYUV_API
|
||||
int ARGB1555ToARGB(const uint8_t* src_argb1555,
|
||||
int src_stride_argb1555,
|
||||
@ -1056,7 +1056,7 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// RGB12 (R444 fourcc) little endian to ARGB.
|
||||
// RGB12 (R444 fourcc) little endian to ARGB. Deprecated.
|
||||
LIBYUV_API
|
||||
int ARGB4444ToARGB(const uint8_t* src_argb4444,
|
||||
int src_stride_argb4444,
|
||||
@ -1173,7 +1173,7 @@ int Android420ToABGR(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert NV12 to RGB565.
|
||||
// Convert NV12 to RGB565. Deprecated.
|
||||
LIBYUV_API
|
||||
int NV12ToRGB565(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
@ -1369,6 +1369,7 @@ int I422ToRAW(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert I420 to RGB565. Deprecated.
|
||||
LIBYUV_API
|
||||
int I420ToRGB565(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
@ -1381,6 +1382,7 @@ int I420ToRGB565(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert J420 to RGB565. Deprecated.
|
||||
LIBYUV_API
|
||||
int J420ToRGB565(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
@ -1393,6 +1395,7 @@ int J420ToRGB565(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert H420 to RGB565. Deprecated.
|
||||
LIBYUV_API
|
||||
int H420ToRGB565(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
@ -1405,6 +1408,7 @@ int H420ToRGB565(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert I422 to RGB565. Deprecated.
|
||||
LIBYUV_API
|
||||
int I422ToRGB565(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
@ -1417,7 +1421,7 @@ int I422ToRGB565(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert I420 To RGB565 with 4x4 dither matrix (16 bytes).
|
||||
// Convert I420 To RGB565 with 4x4 dither matrix (16 bytes). Deprecated.
|
||||
// Values in dither matrix from 0 to 7 recommended.
|
||||
// The order of the dither matrix is first byte is upper left.
|
||||
|
||||
@ -1434,6 +1438,7 @@ int I420ToRGB565Dither(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert I420 to ARGB1555. Deprecated.
|
||||
LIBYUV_API
|
||||
int I420ToARGB1555(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
@ -1446,6 +1451,7 @@ int I420ToARGB1555(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert I420 to ARGB4444. Deprecated.
|
||||
LIBYUV_API
|
||||
int I420ToARGB4444(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
@ -1903,7 +1909,7 @@ int NV21ToARGBMatrix(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert NV12 to RGB565 with matrix.
|
||||
// Convert NV12 to RGB565 with matrix. Deprecated.
|
||||
LIBYUV_API
|
||||
int NV12ToRGB565Matrix(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
@ -2030,7 +2036,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert I420 to RGB565 with specified color matrix.
|
||||
// Convert I420 to RGB565 with specified color matrix. Deprecated.
|
||||
LIBYUV_API
|
||||
int I420ToRGB565Matrix(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
@ -2044,7 +2050,7 @@ int I420ToRGB565Matrix(const uint8_t* src_y,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert I422 to RGB565 with specified color matrix.
|
||||
// Convert I422 to RGB565 with specified color matrix. Deprecated.
|
||||
LIBYUV_API
|
||||
int I422ToRGB565Matrix(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
|
||||
@ -142,7 +142,7 @@ int ARGBToRAW(const uint8_t* src_argb,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert ARGB To RGB565.
|
||||
// Convert ARGB To RGB565. Deprecated.
|
||||
LIBYUV_API
|
||||
int ARGBToRGB565(const uint8_t* src_argb,
|
||||
int src_stride_argb,
|
||||
@ -151,7 +151,7 @@ int ARGBToRGB565(const uint8_t* src_argb,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes).
|
||||
// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes). Deprecated.
|
||||
// Values in dither matrix from 0 to 7 recommended.
|
||||
// The order of the dither matrix is first byte is upper left.
|
||||
// TODO(fbarchard): Consider pointer to 2d array for dither4x4.
|
||||
@ -165,7 +165,7 @@ int ARGBToRGB565Dither(const uint8_t* src_argb,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert ARGB To ARGB1555.
|
||||
// Convert ARGB To ARGB1555. Deprecated.
|
||||
LIBYUV_API
|
||||
int ARGBToARGB1555(const uint8_t* src_argb,
|
||||
int src_stride_argb,
|
||||
@ -174,7 +174,7 @@ int ARGBToARGB1555(const uint8_t* src_argb,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
// Convert ARGB To ARGB4444.
|
||||
// Convert ARGB To ARGB4444. Deprecated.
|
||||
LIBYUV_API
|
||||
int ARGBToARGB4444(const uint8_t* src_argb,
|
||||
int src_stride_argb,
|
||||
|
||||
@ -63,6 +63,7 @@ static const int kCpuHasAVXVNNI = 0x4000000;
|
||||
static const int kCpuHasAVXVNNIINT8 = 0x8000000;
|
||||
static const int kCpuHasAMXINT8 = 0x10000000;
|
||||
static const int kCpuHasAVX512BMM = 0x20000000;
|
||||
static const int kCpuHasAMXTF32 = 0x40000000;
|
||||
|
||||
// These flags are only valid on LOONGARCH processors.
|
||||
static const int kCpuHasLOONGARCH = 0x20;
|
||||
|
||||
@ -81,7 +81,7 @@ extern "C" {
|
||||
#endif // __clang__
|
||||
|
||||
// Visual C 2012 required for AVX2.
|
||||
#if defined(_M_IX86) && \
|
||||
#if (defined(_M_IX86) || defined(_M_X64)) && \
|
||||
(!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN)) && \
|
||||
defined(_MSC_VER) && _MSC_VER >= 1700
|
||||
#define VISUALC_HAS_AVX2 1
|
||||
|
||||
@ -21,19 +21,21 @@ namespace libyuv {
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// The following are available for 32 bit x86, Visual C 64 bit, and for clang
|
||||
// with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These
|
||||
// are intrinsics in rotate_win.cc.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__i386__) || defined(_M_IX86) || \
|
||||
((defined(__x86_64__) || defined(_M_X64)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))))
|
||||
#define HAS_TRANSPOSEWX8_SSSE3
|
||||
#define HAS_TRANSPOSEUVWX8_SSE2
|
||||
// The following are available for Visual C 32/64 bit, and for clang with
|
||||
// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These are
|
||||
// intrinsics in rotate_win.cc.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
||||
defined(_M_IX86)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
#define HAS_TRANSPOSEWX8_AVX2
|
||||
#define HAS_TRANSPOSEUVWX8_AVX2
|
||||
#define HAS_TRANSPOSEWX8_16_AVX2
|
||||
#define HAS_TRANSPOSEWX16_16_AVX2
|
||||
#define HAS_TRANSPOSEWX4_32_SSE2
|
||||
#define HAS_TRANSPOSEWX4_32_AVX2
|
||||
#define HAS_TRANSPOSEWX8_32_AVX2
|
||||
#define HAS_TRANSPOSEWXH_32_AVX2
|
||||
#define HAS_TRANSPOSEWX16_AVX2
|
||||
#define HAS_TRANSPOSEUVWX16_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
@ -44,22 +46,27 @@ extern "C" {
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// The following are available for 64 bit GCC and clang, as inline assembly in
|
||||
// rotate_gcc.cc:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||
#define HAS_TRANSPOSEWX8_SSSE3
|
||||
#define HAS_TRANSPOSEUVWX8_SSE2
|
||||
// The following are available for 32/64 bit GCC and clang, as inline assembly
|
||||
// in rotate_gcc.cc:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(!defined(_MSC_VER) || defined(__clang__))
|
||||
#define HAS_TRANSPOSEWX8_AVX2
|
||||
#define HAS_TRANSPOSEUVWX8_AVX2
|
||||
#define HAS_TRANSPOSEWX8_16_AVX2
|
||||
#define HAS_TRANSPOSEWX16_16_AVX2
|
||||
#define HAS_TRANSPOSEWX4_32_SSE2
|
||||
#define HAS_TRANSPOSEWX4_32_AVX2
|
||||
#define HAS_TRANSPOSEWX8_32_AVX2
|
||||
#define HAS_TRANSPOSEWXH_32_AVX2
|
||||
#define HAS_TRANSPOSEWX16_AVX2
|
||||
#define HAS_TRANSPOSEUVWX16_AVX2
|
||||
#endif
|
||||
|
||||
// The following are available for AVX512 clang x64 platforms:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
||||
defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN)
|
||||
defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(!defined(_MSC_VER) || defined(__clang__))
|
||||
#define HAS_TRANSPOSEWX16_AVX512BW
|
||||
#define HAS_TRANSPOSEUVWX16_AVX512BW
|
||||
#define HAS_TRANSPOSEWX32_16_AVX512BW
|
||||
@ -128,13 +135,6 @@ extern "C" {
|
||||
#define HAS_TRANSPOSEWXH_32_HVX
|
||||
#endif
|
||||
|
||||
void TransposeWxH_C(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
void TransposeWx8_C(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
@ -167,11 +167,11 @@ void TransposeWxH_HVX(const uint8_t* src,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height);
|
||||
void TransposeWx8_SSSE3(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx16_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
@ -213,11 +213,11 @@ void TransposeWx16_Any_NEON(const uint8_t* src,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_Any_SSSE3(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_Any_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx16_Any_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
@ -244,15 +244,6 @@ void TransposeWx16_Any_WASMSIMD(const uint8_t* src,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeUVWxH_C(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst_a,
|
||||
int dst_stride_a,
|
||||
uint8_t* dst_b,
|
||||
int dst_stride_b,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
void TransposeUVWx8_C(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst_a,
|
||||
@ -267,7 +258,7 @@ void TransposeUVWx16_C(const uint8_t* src,
|
||||
uint8_t* dst_b,
|
||||
int dst_stride_b,
|
||||
int width);
|
||||
void TransposeUVWx8_SSE2(const uint8_t* src,
|
||||
void TransposeUVWx8_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst_a,
|
||||
int dst_stride_a,
|
||||
@ -340,7 +331,7 @@ void TransposeUVWx16_WASMSIMD(const uint8_t* src,
|
||||
int dst_stride_b,
|
||||
int width);
|
||||
|
||||
void TransposeUVWx8_Any_SSE2(const uint8_t* src,
|
||||
void TransposeUVWx8_Any_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst_a,
|
||||
int dst_stride_a,
|
||||
@ -389,18 +380,17 @@ void TransposeUVWx16_Any_WASMSIMD(const uint8_t* src,
|
||||
uint8_t* dst_b,
|
||||
int dst_stride_b,
|
||||
int width);
|
||||
void TransposeWxH_16_C(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
void TransposeWx8_16_C(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_16_AVX2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx16_16_AVX2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
@ -454,6 +444,11 @@ void TransposeWx8_16_WASMSIMD(const uint16_t* src,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx8_16_Any_AVX2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width);
|
||||
void TransposeWx16_16_Any_AVX2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
@ -513,7 +508,7 @@ void TransposeWx4_32_SVE2(const uint8_t* src,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeWx4_32_SSE2(const uint8_t* src,
|
||||
void TransposeWx4_32_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
@ -525,6 +520,13 @@ void TransposeWx8_32_AVX2(const uint8_t* src,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeWxH_32_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height);
|
||||
|
||||
void TransposeWx16_32_AVX512BW(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
@ -574,7 +576,7 @@ void TransposeWx4_32_C(const uint8_t* src,
|
||||
int dst_stride,
|
||||
int width);
|
||||
|
||||
void TransposeWx4_32_Any_SSE2(const uint8_t* src,
|
||||
void TransposeWx4_32_Any_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -23,31 +23,26 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
// The following are available on all x86 platforms:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(_M_IX86) || ((defined(__x86_64__) || defined(__i386__)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)))
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(_M_IX86) || defined(_M_X64) || defined(__x86_64__) || \
|
||||
defined(__i386__))
|
||||
#define HAS_FIXEDDIV1_X86
|
||||
#define HAS_FIXEDDIV_X86
|
||||
#define HAS_SCALEARGBCOLS_SSE2
|
||||
#define HAS_SCALEARGBCOLSUP2_SSE2
|
||||
#define HAS_SCALEARGBFILTERCOLS_SSSE3
|
||||
#define HAS_SCALEARGBROWDOWN2_SSE2
|
||||
#define HAS_SCALEARGBROWDOWNEVEN_SSE2
|
||||
#define HAS_SCALECOLSUP2_SSE2
|
||||
#define HAS_SCALEFILTERCOLS_SSSE3
|
||||
#define HAS_SCALEADDROW_SSE2
|
||||
#endif
|
||||
|
||||
// The following are available on 32-bit Visual C and gcc/clang x86 platforms:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && !defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
((defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__)) || \
|
||||
defined(__x86_64__) || defined(__i386__))
|
||||
#define HAS_SCALEROWDOWN2_SSSE3
|
||||
#define HAS_SCALEROWDOWN34_SSSE3
|
||||
#define HAS_SCALEROWDOWN4_SSSE3
|
||||
#endif
|
||||
|
||||
// The following are available for gcc/clang x86 platforms:
|
||||
// TODO(fbarchard): Port to Visual C
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||
#define HAS_SCALEADDROW_SSE2
|
||||
#define HAS_SCALEROWDOWN38_SSSE3
|
||||
#define HAS_SCALEUVROWDOWN2BOX_SSSE3
|
||||
#define HAS_SCALEROWUP2_LINEAR_SSE2
|
||||
#define HAS_SCALEROWUP2_LINEAR_SSSE3
|
||||
#define HAS_SCALEROWUP2_BILINEAR_SSE2
|
||||
@ -62,66 +57,57 @@ extern "C" {
|
||||
#define HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
|
||||
#endif
|
||||
|
||||
// The following are available for gcc/clang x86 platforms, but
|
||||
// require clang 3.4 or gcc 4.7.
|
||||
// TODO(fbarchard): Port to Visual C
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
(defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||
#define HAS_SCALEADDROW_AVX2
|
||||
#define HAS_SCALEUVADDCOLS3_AVX2
|
||||
#define HAS_SCALEUVROWDOWN2BOX_AVX2
|
||||
#define HAS_SCALEROWUP2_LINEAR_AVX2
|
||||
#define HAS_SCALEROWUP2_BILINEAR_AVX2
|
||||
#define HAS_SCALEROWUP2_LINEAR_12_AVX2
|
||||
#define HAS_SCALEROWUP2_BILINEAR_12_AVX2
|
||||
#define HAS_SCALEROWUP2_LINEAR_16_AVX2
|
||||
#define HAS_SCALEROWUP2_BILINEAR_16_AVX2
|
||||
#define HAS_SCALEUVROWUP2_LINEAR_AVX2
|
||||
#define HAS_SCALEUVROWUP2_BILINEAR_AVX2
|
||||
#define HAS_SCALEUVROWUP2_LINEAR_16_AVX2
|
||||
#define HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
|
||||
#endif
|
||||
|
||||
// The following are available on all x86 platforms, but
|
||||
// require VS2012, clang 3.4 or gcc 4.7.
|
||||
// The code supports NaCL but requires a new compiler and validator.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
|
||||
defined(GCC_HAS_AVX2))
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \
|
||||
defined(GCC_HAS_AVX2) || \
|
||||
(defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || \
|
||||
defined(_M_X64) || defined(_M_IX86))))
|
||||
#define HAS_SCALEADDROW_AVX2
|
||||
#define HAS_SCALEARGBCOLS_AVX2
|
||||
#define HAS_SCALEARGBCOLSUP2_AVX2
|
||||
#define HAS_SCALEARGBFILTERCOLS_AVX2
|
||||
#define HAS_SCALEARGBROWDOWN2_AVX2
|
||||
#define HAS_SCALEARGBROWDOWNEVEN_AVX2
|
||||
#define HAS_SCALECOLSUP2_AVX2
|
||||
#define HAS_SCALEFILTERCOLS_AVX2
|
||||
#define HAS_SCALEROWDOWN2_16_AVX2
|
||||
#define HAS_SCALEROWDOWN2_AVX2
|
||||
#define HAS_SCALEROWDOWN34_AVX2
|
||||
#define HAS_SCALEROWDOWN38_AVX2
|
||||
#define HAS_SCALEROWDOWN4_AVX2
|
||||
#define HAS_SCALEROWUP2_BILINEAR_12_AVX2
|
||||
#define HAS_SCALEROWUP2_BILINEAR_16_AVX2
|
||||
#define HAS_SCALEROWUP2_BILINEAR_AVX2
|
||||
#define HAS_SCALEROWUP2_LINEAR_12_AVX2
|
||||
#define HAS_SCALEROWUP2_LINEAR_16_AVX2
|
||||
#define HAS_SCALEROWUP2_LINEAR_AVX2
|
||||
#define HAS_SCALEUVADDCOLS3_AVX2
|
||||
#define HAS_SCALEUVROWDOWN2_AVX2
|
||||
#define HAS_SCALEUVROWDOWN2BOX_AVX2
|
||||
#define HAS_SCALEUVROWDOWN2LINEAR_AVX2
|
||||
#define HAS_SCALEUVROWDOWNEVEN_AVX2
|
||||
#define HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
|
||||
#define HAS_SCALEUVROWUP2_BILINEAR_AVX2
|
||||
#define HAS_SCALEUVROWUP2_LINEAR_16_AVX2
|
||||
#define HAS_SCALEUVROWUP2_LINEAR_AVX2
|
||||
#endif
|
||||
|
||||
// The following are available for AVX512 clang x64 platforms:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN)
|
||||
// The following are available for AVX512 x64 platforms:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(_M_X64)) && \
|
||||
(defined(CLANG_HAS_AVX512) || \
|
||||
(defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
#define HAS_SCALEADDROW_AVX512BW
|
||||
#define HAS_SCALEADDCOLS_AVX512BW
|
||||
#define HAS_SCALEADDCOLS1_AVX512BW
|
||||
#define HAS_SCALEADDCOLS2_AVX512BW
|
||||
#define HAS_SCALEUVADDCOLS3_AVX512VBMI
|
||||
#define HAS_SCALEROWDOWN2_AVX512BW
|
||||
#endif
|
||||
|
||||
// The following are available for Visual C 32/64 bit, and for clang with
|
||||
// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer:
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
||||
defined(_M_IX86)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
#define HAS_SCALEADDROW_SSE2
|
||||
#define HAS_SCALEROWDOWN38_SSSE3
|
||||
#define HAS_SCALEADDROW_AVX2
|
||||
#define HAS_SCALEUVADDCOLS3_AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define HAS_SCALEADDROW_AVX512BW
|
||||
#define HAS_SCALEADDCOLS_AVX512BW
|
||||
#define HAS_SCALEUVADDCOLS3_AVX512VBMI
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// The following are available on Neon platforms:
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && \
|
||||
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
|
||||
@ -532,24 +518,46 @@ void ScaleUVRowDown2_C(const uint8_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2Linear_C(const uint8_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2Linear_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2Box_C(const uint8_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2Box_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDownEven_C(const uint8_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDownEven_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDownEvenBox_C(const uint8_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDownEvenBox_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width);
|
||||
|
||||
void ScaleUVRowUp2_Linear_C(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
@ -589,30 +597,59 @@ void ScaleUVCols_C(uint8_t* dst_uv,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
void ScaleUVCols_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
void ScaleUVCols64_C(uint8_t* dst_uv,
|
||||
const uint8_t* src_uv,
|
||||
int dst_width,
|
||||
int x32,
|
||||
int dx);
|
||||
void ScaleUVCols64_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int x32,
|
||||
int dx);
|
||||
void ScaleUVColsUp2_C(uint8_t* dst_uv,
|
||||
const uint8_t* src_uv,
|
||||
int dst_width,
|
||||
int,
|
||||
int);
|
||||
void ScaleUVColsUp2_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int,
|
||||
int);
|
||||
void ScaleUVFilterCols_C(uint8_t* dst_uv,
|
||||
const uint8_t* src_uv,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
void ScaleUVFilterCols_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
void ScaleUVFilterCols64_C(uint8_t* dst_uv,
|
||||
const uint8_t* src_uv,
|
||||
int dst_width,
|
||||
int x32,
|
||||
int dx);
|
||||
void ScaleUVFilterCols64_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int x32,
|
||||
int dx);
|
||||
void ScaleUVAddCols3_C(int dst_width,
|
||||
int scaleval,
|
||||
const uint16_t* src_uv,
|
||||
uint8_t* dst_uv);
|
||||
void ScaleUVAddCols3_16_C(int dst_width,
|
||||
int scaleval,
|
||||
const uint32_t* src_uv,
|
||||
uint16_t* dst_uv);
|
||||
|
||||
// Specialized scalers for x86.
|
||||
void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,
|
||||
@ -639,6 +676,18 @@ void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
@ -672,30 +721,30 @@ void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
|
||||
void ScaleRowDown34_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
|
||||
void ScaleRowUp2_Linear_SSE2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
@ -842,6 +891,18 @@ void ScaleRowDown2Box_Odd_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown2_16_Any_AVX2(const uint16_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown2Linear_16_Any_AVX2(const uint16_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown2Box_16_Any_AVX2(const uint16_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown4_Any_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
@ -859,30 +920,30 @@ void ScaleRowDown4Box_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
|
||||
void ScaleRowDown34_Any_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_1_Box_Any_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_0_Box_Any_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_Any_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_3_Box_Any_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_2_Box_Any_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_1_Box_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown34_0_Box_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_3_Box_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleRowDown38_2_Box_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
|
||||
void ScaleAddRow_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
|
||||
void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
|
||||
@ -908,29 +969,29 @@ void ScaleAddCols2_AVX512BW(int dst_width,
|
||||
const uint16_t* src_ptr,
|
||||
uint8_t* dst_ptr);
|
||||
|
||||
void ScaleFilterCols_SSSE3(uint8_t* dst_ptr,
|
||||
const uint8_t* src_ptr,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
void ScaleColsUp2_SSE2(uint8_t* dst_ptr,
|
||||
void ScaleFilterCols_AVX2(uint8_t* dst_ptr,
|
||||
const uint8_t* src_ptr,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
void ScaleColsUp2_AVX2(uint8_t* dst_ptr,
|
||||
const uint8_t* src_ptr,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
|
||||
// ARGB Column functions
|
||||
void ScaleARGBCols_SSE2(uint8_t* dst_argb,
|
||||
void ScaleARGBCols_AVX2(uint8_t* dst_argb,
|
||||
const uint8_t* src_argb,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb,
|
||||
const uint8_t* src_argb,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb,
|
||||
void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb,
|
||||
const uint8_t* src_argb,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx);
|
||||
void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb,
|
||||
const uint8_t* src_argb,
|
||||
int dst_width,
|
||||
int x,
|
||||
@ -962,15 +1023,15 @@ void ScaleARGBFilterCols_RVV(uint8_t* dst_argb,
|
||||
int dx);
|
||||
|
||||
// ARGB Row functions
|
||||
void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb,
|
||||
void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_argb,
|
||||
int dst_width);
|
||||
void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb,
|
||||
void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_argb,
|
||||
int dst_width);
|
||||
void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb,
|
||||
void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_argb,
|
||||
int dst_width);
|
||||
@ -1022,15 +1083,15 @@ void ScaleARGBRowDown2Box_LSX(const uint8_t* src_argb,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_argb,
|
||||
int dst_width);
|
||||
void ScaleARGBRowDown2_Any_SSE2(const uint8_t* src_ptr,
|
||||
void ScaleARGBRowDown2_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleARGBRowDown2Linear_Any_SSE2(const uint8_t* src_ptr,
|
||||
void ScaleARGBRowDown2Linear_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleARGBRowDown2Box_Any_SSE2(const uint8_t* src_ptr,
|
||||
void ScaleARGBRowDown2Box_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
@ -1058,12 +1119,12 @@ void ScaleARGBRowDown2Box_Any_LSX(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb,
|
||||
void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint8_t* dst_argb,
|
||||
int dst_width);
|
||||
void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb,
|
||||
void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint8_t* dst_argb,
|
||||
@ -1098,12 +1159,12 @@ void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb,
|
||||
int src_stepx,
|
||||
uint8_t* dst_argb,
|
||||
int dst_width);
|
||||
void ScaleARGBRowDownEven_Any_SSE2(const uint8_t* src_ptr,
|
||||
void ScaleARGBRowDownEven_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleARGBRowDownEvenBox_Any_SSE2(const uint8_t* src_ptr,
|
||||
void ScaleARGBRowDownEvenBox_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint8_t* dst_ptr,
|
||||
@ -1130,10 +1191,14 @@ void ScaleARGBRowDownEvenBox_Any_LSX(const uint8_t* src_ptr,
|
||||
int dst_width);
|
||||
|
||||
// UV Row functions
|
||||
void ScaleUVRowDown2Box_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_uv,
|
||||
@ -1174,10 +1239,14 @@ void ScaleUVRowDown2Box_RVV(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2Box_Any_SSSE3(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2Linear_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleUVRowDown2Box_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
@ -1194,6 +1263,11 @@ void ScaleUVRowDown2Box_Any_NEON(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleUVRowDownEven_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
@ -1209,6 +1283,11 @@ void ScaleUVRowDownEven_RVV(const uint8_t* src_ptr,
|
||||
int32_t src_stepx,
|
||||
uint8_t* dst_uv,
|
||||
int dst_width);
|
||||
void ScaleUVRowDownEven_Any_AVX2(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width);
|
||||
void ScaleUVRowDownEven_Any_NEON(const uint8_t* src_ptr,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
|
||||
@ -33,7 +33,6 @@ int UVScale(const uint8_t* src_uv,
|
||||
enum FilterMode filtering);
|
||||
|
||||
// Scale a 16 bit UV image.
|
||||
// This function is currently incomplete, it can't handle all cases.
|
||||
LIBYUV_API
|
||||
int UVScale_16(const uint16_t* src_uv,
|
||||
int src_stride_uv,
|
||||
|
||||
@ -11,6 +11,6 @@
|
||||
#ifndef INCLUDE_LIBYUV_VERSION_H_
|
||||
#define INCLUDE_LIBYUV_VERSION_H_
|
||||
|
||||
#define LIBYUV_VERSION 1976
|
||||
#define LIBYUV_VERSION 1977
|
||||
|
||||
#endif // INCLUDE_LIBYUV_VERSION_H_
|
||||
|
||||
@ -84,9 +84,9 @@ enum FourCC {
|
||||
FOURCC_24BG = FOURCC('2', '4', 'B', 'G'),
|
||||
FOURCC_RAW = FOURCC('r', 'a', 'w', ' '),
|
||||
FOURCC_RGBA = FOURCC('R', 'G', 'B', 'A'),
|
||||
FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE.
|
||||
FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), // argb1555 LE.
|
||||
FOURCC_R444 = FOURCC('R', '4', '4', '4'), // argb4444 LE.
|
||||
FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE. deprecated.
|
||||
FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), // argb1555 LE. deprecated.
|
||||
FOURCC_R444 = FOURCC('R', '4', '4', '4'), // argb4444 LE. deprecated.
|
||||
|
||||
// 1 Primary Compressed YUV format.
|
||||
FOURCC_MJPG = FOURCC('M', 'J', 'P', 'G'),
|
||||
@ -135,9 +135,9 @@ enum FourCC {
|
||||
FOURCC_BGR3 = FOURCC('B', 'G', 'R', '3'), // Alias for 24BG.
|
||||
FOURCC_CM32 = FOURCC(0, 0, 0, 32), // Alias for BGRA kCMPixelFormat_32ARGB
|
||||
FOURCC_CM24 = FOURCC(0, 0, 0, 24), // Alias for RAW kCMPixelFormat_24RGB
|
||||
FOURCC_L555 = FOURCC('L', '5', '5', '5'), // Alias for RGBO.
|
||||
FOURCC_L565 = FOURCC('L', '5', '6', '5'), // Alias for RGBP.
|
||||
FOURCC_5551 = FOURCC('5', '5', '5', '1'), // Alias for RGBO.
|
||||
FOURCC_L555 = FOURCC('L', '5', '5', '5'), // Alias for RGBO. deprecated.
|
||||
FOURCC_L565 = FOURCC('L', '5', '6', '5'), // Alias for RGBP. deprecated.
|
||||
FOURCC_5551 = FOURCC('5', '5', '5', '1'), // Alias for RGBO. deprecated.
|
||||
|
||||
// deprecated formats. Not supported, but defined for backward compatibility.
|
||||
FOURCC_I411 = FOURCC('I', '4', '1', '1'),
|
||||
@ -177,9 +177,9 @@ enum FourCCBpp {
|
||||
FOURCC_BPP_AB64 = 64,
|
||||
FOURCC_BPP_24BG = 24,
|
||||
FOURCC_BPP_RAW = 24,
|
||||
FOURCC_BPP_RGBP = 16,
|
||||
FOURCC_BPP_RGBO = 16,
|
||||
FOURCC_BPP_R444 = 16,
|
||||
FOURCC_BPP_RGBP = 16, // deprecated
|
||||
FOURCC_BPP_RGBO = 16, // deprecated
|
||||
FOURCC_BPP_R444 = 16, // deprecated
|
||||
FOURCC_BPP_RGGB = 8,
|
||||
FOURCC_BPP_BGGR = 8,
|
||||
FOURCC_BPP_GRBG = 8,
|
||||
|
||||
@ -87,34 +87,6 @@ uint32_t HashDjb2(const uint8_t* src, uint64_t count, uint32_t seed) {
|
||||
return seed;
|
||||
}
|
||||
|
||||
static uint32_t ARGBDetectRow_C(const uint8_t* argb, int width) {
|
||||
int x;
|
||||
for (x = 0; x < width - 1; x += 2) {
|
||||
if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB.
|
||||
return FOURCC_BGRA;
|
||||
}
|
||||
if (argb[3] != 255) { // Fourth byte is not Alpha of 255, so not BGRA.
|
||||
return FOURCC_ARGB;
|
||||
}
|
||||
if (argb[4] != 255) { // Second pixel first byte is not Alpha of 255.
|
||||
return FOURCC_BGRA;
|
||||
}
|
||||
if (argb[7] != 255) { // Second pixel fourth byte is not Alpha of 255.
|
||||
return FOURCC_ARGB;
|
||||
}
|
||||
argb += 8;
|
||||
}
|
||||
if (width & 1) {
|
||||
if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB.
|
||||
return FOURCC_BGRA;
|
||||
}
|
||||
if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA.
|
||||
return FOURCC_ARGB;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Scan an opaque argb image and return fourcc based on alpha offset.
|
||||
// Returns FOURCC_ARGB, FOURCC_BGRA, or 0 if unknown.
|
||||
LIBYUV_API
|
||||
@ -122,6 +94,7 @@ uint32_t ARGBDetect(const uint8_t* argb,
|
||||
int stride_argb,
|
||||
int width,
|
||||
int height) {
|
||||
uint32_t (*ARGBDetectRow)(const uint8_t* argb, int width) = ARGBDetectRow_C;
|
||||
uint32_t fourcc = 0;
|
||||
int h;
|
||||
|
||||
@ -134,8 +107,13 @@ uint32_t ARGBDetect(const uint8_t* argb,
|
||||
height = 1;
|
||||
stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBDETECTROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBDetectRow = ARGBDetectRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
for (h = 0; h < height && fourcc == 0; ++h) {
|
||||
fourcc = ARGBDetectRow_C(argb, width);
|
||||
fourcc = ARGBDetectRow(argb, width);
|
||||
argb += stride_argb;
|
||||
}
|
||||
return fourcc;
|
||||
@ -262,6 +240,12 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a,
|
||||
simd_size = 1; // Any count.
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SUMSQUAREERROR_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
SumSquareError = SumSquareError_AVX512VNNI;
|
||||
simd_size = 1; // Any count.
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SUMSQUAREERROR_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
SumSquareError = SumSquareError_WASMSIMD;
|
||||
@ -380,10 +364,10 @@ double I420Psnr(const uint8_t* src_y_a,
|
||||
return SumSquareErrorToPsnr(sse, samples);
|
||||
}
|
||||
|
||||
static float Ssim8x8_C(const uint8_t* src_a,
|
||||
int stride_a,
|
||||
const uint8_t* src_b,
|
||||
int stride_b) {
|
||||
static float Ssim8x8(const uint8_t* src_a,
|
||||
int stride_a,
|
||||
const uint8_t* src_b,
|
||||
int stride_b) {
|
||||
int32_t sum_a = 0;
|
||||
int32_t sum_b = 0;
|
||||
int32_t sum_sq_ab = 0;
|
||||
@ -615,7 +599,7 @@ double CalcFrameSsim(const uint8_t* src_a,
|
||||
const uint8_t* b = src_b + (ptrdiff_t)i * stride_b;
|
||||
int j;
|
||||
for (j = 0; j < width - 8; j += 4) {
|
||||
ssim_total += Ssim8x8_C(a + j, stride_a, b + j, stride_b);
|
||||
ssim_total += Ssim8x8(a + j, stride_a, b + j, stride_b);
|
||||
samples++;
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,6 +12,8 @@
|
||||
|
||||
#include <float.h>
|
||||
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
namespace libyuv {
|
||||
extern "C" {
|
||||
@ -169,6 +171,34 @@ float SsimWindowRow_C(const uint32_t* row0,
|
||||
return ssim_total;
|
||||
}
|
||||
|
||||
uint32_t ARGBDetectRow_C(const uint8_t* argb, int width) {
|
||||
int x;
|
||||
for (x = 0; x < width - 1; x += 2) {
|
||||
if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB.
|
||||
return FOURCC_BGRA;
|
||||
}
|
||||
if (argb[3] != 255) { // Fourth byte is not Alpha of 255, so not BGRA.
|
||||
return FOURCC_ARGB;
|
||||
}
|
||||
if (argb[4] != 255) { // Second pixel first byte is not Alpha of 255.
|
||||
return FOURCC_BGRA;
|
||||
}
|
||||
if (argb[7] != 255) { // Second pixel fourth byte is not Alpha of 255.
|
||||
return FOURCC_ARGB;
|
||||
}
|
||||
argb += 8;
|
||||
}
|
||||
if (width & 1) {
|
||||
if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB.
|
||||
return FOURCC_BGRA;
|
||||
}
|
||||
if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA.
|
||||
return FOURCC_ARGB;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/row.h"
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
namespace libyuv {
|
||||
@ -23,13 +24,15 @@ extern "C" {
|
||||
// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
|
||||
// are callee-saved. List "memory" first, then "cc", then registers.
|
||||
|
||||
// This module is for GCC and clang x64. 32 bit x86 uses compare_win.cc.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
// This module is for GCC and clang x86 and x64.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__)) && \
|
||||
!defined(LIBYUV_ENABLE_ROWWIN) && \
|
||||
(!defined(_MSC_VER) || defined(__clang__))
|
||||
|
||||
// "memory" clobber prevents the reads from being removed
|
||||
|
||||
#ifdef HAS_HAMMINGDISTANCE_SSE42
|
||||
uint32_t HammingDistance_SSE42(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
@ -77,11 +80,16 @@ uint32_t HammingDistance_SSE42(const uint8_t* src_a,
|
||||
|
||||
return (uint32_t)(diff);
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_SSE42
|
||||
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSSE3) || defined(HAS_HAMMINGDISTANCE_AVX2) || \
|
||||
defined(HAS_HAMMINGDISTANCE_AVX512BW)
|
||||
static const vec8 kNibbleMask = {15, 15, 15, 15, 15, 15, 15, 15,
|
||||
15, 15, 15, 15, 15, 15, 15, 15};
|
||||
static const vec8 kBitCount = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4};
|
||||
#endif
|
||||
|
||||
#ifdef HAS_HAMMINGDISTANCE_SSSE3
|
||||
uint32_t HammingDistance_SSSE3(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
@ -141,6 +149,7 @@ uint32_t HammingDistance_SSSE3(const uint8_t* src_a,
|
||||
|
||||
return diff;
|
||||
}
|
||||
#endif // HAS_HAMMINGDISTANCE_SSSE3
|
||||
|
||||
#ifdef HAS_HAMMINGDISTANCE_AVX2
|
||||
uint32_t HammingDistance_AVX2(const uint8_t* src_a,
|
||||
@ -507,11 +516,78 @@ uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
|
||||
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
|
||||
[sse] "=r"(sse)
|
||||
: [mask] "m"(mask)
|
||||
: "cc", "memory", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
|
||||
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
|
||||
return sse;
|
||||
}
|
||||
#endif // HAS_SUMSQUAREERROR_AVX512BW
|
||||
|
||||
#ifdef HAS_SUMSQUAREERROR_AVX512VNNI
|
||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||
// loads that read zeros, so any count is supported.
|
||||
uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
uint32_t sse;
|
||||
const uint64_t mask = LowBytesMask64(count & 63);
|
||||
asm volatile(
|
||||
"vpxord %%zmm0,%%zmm0,%%zmm0 \n"
|
||||
"vpxord %%zmm4,%%zmm4,%%zmm4 \n"
|
||||
"vpxord %%zmm5,%%zmm5,%%zmm5 \n"
|
||||
"kmovq %[mask],%%k1 \n"
|
||||
"sub %[src_a],%[src_b] \n"
|
||||
"sub $0x40,%[count] \n"
|
||||
"jl 2f \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vmovdqu8 (%[src_a]),%%zmm1 \n"
|
||||
"vmovdqu8 (%[src_a],%[src_b]),%%zmm2 \n"
|
||||
"add $0x40,%[src_a] \n"
|
||||
"vpsubusb %%zmm2,%%zmm1,%%zmm3 \n" // abs difference trick
|
||||
"vpsubusb %%zmm1,%%zmm2,%%zmm2 \n"
|
||||
"vporq %%zmm3,%%zmm2,%%zmm1 \n"
|
||||
"vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n" // u16
|
||||
"vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n"
|
||||
"vpdpwssd %%zmm2,%%zmm2,%%zmm0 \n" // square + hadd + acc
|
||||
"vpdpwssd %%zmm1,%%zmm1,%%zmm4 \n"
|
||||
"sub $0x40,%[count] \n"
|
||||
"jge 1b \n"
|
||||
|
||||
// Remainder: 1 to 63 bytes, masked loads.
|
||||
"2: \n"
|
||||
"add $0x40,%[count] \n"
|
||||
"je 3f \n"
|
||||
"vmovdqu8 (%[src_a]),%%zmm1%{%%k1%}%{z%} \n"
|
||||
"vmovdqu8 (%[src_a],%[src_b]),%%zmm2%{%%k1%}%{z%} \n"
|
||||
"vpsubusb %%zmm2,%%zmm1,%%zmm3 \n"
|
||||
"vpsubusb %%zmm1,%%zmm2,%%zmm2 \n"
|
||||
"vporq %%zmm3,%%zmm2,%%zmm1 \n"
|
||||
"vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n"
|
||||
"vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n"
|
||||
"vpdpwssd %%zmm2,%%zmm2,%%zmm0 \n"
|
||||
"vpdpwssd %%zmm1,%%zmm1,%%zmm4 \n"
|
||||
|
||||
// Add 16 dwords.
|
||||
"3: \n"
|
||||
"vpaddd %%zmm4,%%zmm0,%%zmm0 \n"
|
||||
"vextracti64x4 $1,%%zmm0,%%ymm1 \n"
|
||||
"vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vextracti128 $1,%%ymm0,%%xmm1 \n"
|
||||
"vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vpshufd $0xee,%%xmm0,%%xmm1 \n"
|
||||
"vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vpshufd $0x1,%%xmm0,%%xmm1 \n"
|
||||
"vpaddd %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vmovd %%xmm0,%[sse] \n"
|
||||
"vzeroupper \n"
|
||||
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count),
|
||||
[sse] "=r"(sse)
|
||||
: [mask] "m"(mask)
|
||||
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||
return sse;
|
||||
}
|
||||
#endif // HAS_SUMSQUAREERROR_AVX512VNNI
|
||||
|
||||
#if defined(HAS_HASHDJB2_AVX2) || defined(HAS_HASHDJB2_AVX512BW)
|
||||
// HashDjb2 is hash = hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)).
|
||||
// The 32 bit multipliers 33 ^ (63 - i) are split into 16 bit halves so that
|
||||
@ -812,10 +888,10 @@ void Ssim4x4Row_SSE2(const uint8_t* src_a,
|
||||
"movdqu %%xmm3,0x20(%[dst]) \n"
|
||||
"movdqu %%xmm4,0x30(%[dst]) \n"
|
||||
"add $0x40,%[dst] \n"
|
||||
"sub $0x10,%[width] \n"
|
||||
"subl $0x10,%[width] \n"
|
||||
"jg 1b \n"
|
||||
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
|
||||
[width] "+r"(width)
|
||||
[width] "+rm"(width)
|
||||
: [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b)
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||
"xmm7");
|
||||
@ -916,11 +992,11 @@ void Ssim4x4Row_AVX2(const uint8_t* src_a,
|
||||
"vmovdqu %%ymm2,0x40(%[dst]) \n"
|
||||
"vmovdqu %%ymm3,0x60(%[dst]) \n"
|
||||
"add $0x80,%[dst] \n"
|
||||
"sub $0x20,%[width] \n"
|
||||
"subl $0x20,%[width] \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
: [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst),
|
||||
[width] "+r"(width)
|
||||
[width] "+rm"(width)
|
||||
: [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b)
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||
"xmm7");
|
||||
@ -1094,7 +1170,7 @@ float SsimWindowRow_SSE2(const uint32_t* row0,
|
||||
float total;
|
||||
asm volatile(
|
||||
"sub %[row0],%[row1] \n"
|
||||
"xorps %[total],%[total] \n"
|
||||
"xorps %%xmm0,%%xmm0 \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
@ -1151,19 +1227,20 @@ float SsimWindowRow_SSE2(const uint32_t* row0,
|
||||
"mulps %%xmm5,%%xmm2 \n"
|
||||
"mulps %%xmm3,%%xmm1 \n"
|
||||
"divps %%xmm1,%%xmm2 \n"
|
||||
"addps %%xmm2,%[total] \n"
|
||||
"addps %%xmm2,%%xmm0 \n"
|
||||
"sub $0x4,%[num_windows] \n"
|
||||
"jg 1b \n"
|
||||
|
||||
"movhlps %[total],%%xmm1 \n"
|
||||
"addps %%xmm1,%[total] \n"
|
||||
"movaps %[total],%%xmm1 \n"
|
||||
"shufps $0x1,%[total],%%xmm1 \n"
|
||||
"addss %%xmm1,%[total] \n"
|
||||
"movhlps %%xmm0,%%xmm1 \n"
|
||||
"addps %%xmm1,%%xmm0 \n"
|
||||
"movaps %%xmm0,%%xmm1 \n"
|
||||
"shufps $0x1,%%xmm0,%%xmm1 \n"
|
||||
"addss %%xmm1,%%xmm0 \n"
|
||||
"movss %%xmm0,%[total] \n"
|
||||
: [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows),
|
||||
[total] "=&x"(total)
|
||||
[total] "=m"(total)
|
||||
: [c1] "m"(kSsimC1), [c2] "m"(kSsimC2)
|
||||
: "memory", "cc", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||
return total;
|
||||
}
|
||||
#endif // HAS_SSIMWINDOWROW_SSE2
|
||||
@ -1176,7 +1253,7 @@ float SsimWindowRow_AVX2(const uint32_t* row0,
|
||||
float total;
|
||||
asm volatile(
|
||||
"sub %[row0],%[row1] \n"
|
||||
"vxorps %t[total],%t[total],%t[total] \n"
|
||||
"vxorps %%ymm0,%%ymm0,%%ymm0 \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
@ -1231,21 +1308,22 @@ float SsimWindowRow_AVX2(const uint32_t* row0,
|
||||
"vmulps %%ymm3,%%ymm1,%%ymm1 \n"
|
||||
"vmulps %%ymm5,%%ymm4,%%ymm4 \n"
|
||||
"vdivps %%ymm4,%%ymm1,%%ymm1 \n"
|
||||
"vaddps %%ymm1,%t[total],%t[total] \n"
|
||||
"vaddps %%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"sub $0x8,%[num_windows] \n"
|
||||
"jg 1b \n"
|
||||
|
||||
"vextractf128 $0x1,%t[total],%%xmm1 \n"
|
||||
"vaddps %%xmm1,%x[total],%x[total] \n"
|
||||
"vmovhlps %x[total],%%xmm1,%%xmm1 \n"
|
||||
"vaddps %%xmm1,%x[total],%x[total] \n"
|
||||
"vshufps $0x1,%x[total],%x[total],%%xmm1 \n"
|
||||
"vaddss %%xmm1,%x[total],%x[total] \n"
|
||||
"vextractf128 $0x1,%%ymm0,%%xmm1 \n"
|
||||
"vaddps %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vmovhlps %%xmm0,%%xmm1,%%xmm1 \n"
|
||||
"vaddps %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vshufps $0x1,%%xmm0,%%xmm0,%%xmm1 \n"
|
||||
"vaddss %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vmovss %%xmm0,%[total] \n"
|
||||
"vzeroupper \n"
|
||||
: [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows),
|
||||
[total] "=&x"(total)
|
||||
[total] "=m"(total)
|
||||
: [c1] "m"(kSsimC1), [c2] "m"(kSsimC2)
|
||||
: "memory", "cc", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
|
||||
return total;
|
||||
}
|
||||
#endif // HAS_SSIMWINDOWROW_AVX2
|
||||
@ -1370,8 +1448,51 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
||||
#undef SSIM_WINDOW_AVX512BW
|
||||
#endif // HAS_SSIMWINDOWROW_AVX512BW
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) &&
|
||||
// !defined(LIBYUV_ENABLE_ROWWIN)
|
||||
#ifdef HAS_ARGBDETECTROW_AVX2
|
||||
uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) {
|
||||
uint32_t diff = 0;
|
||||
asm volatile(
|
||||
"vpcmpeqd %%ymm1,%%ymm1,%%ymm1 \n"
|
||||
"sub $0x8,%[width] \n"
|
||||
"jl 2f \n"
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
"vpcmpeqb (%[argb]),%%ymm1,%%ymm0 \n"
|
||||
"vpmovmskb %%ymm0,%[diff] \n"
|
||||
"not %[diff] \n"
|
||||
"and $0x99999999,%[diff] \n"
|
||||
"jnz 99f \n"
|
||||
"add $0x20,%[argb] \n"
|
||||
"sub $0x8,%[width] \n"
|
||||
"jge 1b \n"
|
||||
"2: \n"
|
||||
"add $0x8,%[width] \n"
|
||||
"jle 99f \n"
|
||||
LABELALIGN
|
||||
"3: \n"
|
||||
"vmovd (%[argb]),%%xmm0 \n"
|
||||
"add $0x4,%[argb] \n"
|
||||
"vpcmpeqb %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vpmovmskb %%xmm0,%[diff] \n"
|
||||
"not %[diff] \n"
|
||||
"and $0x9,%[diff] \n"
|
||||
"jnz 99f \n"
|
||||
"sub $0x1,%[width] \n"
|
||||
"jg 3b \n"
|
||||
"99: \n"
|
||||
"vzeroupper \n"
|
||||
: [argb] "+r"(argb), [width] "+r"(width), [diff] "+r"(diff)
|
||||
:
|
||||
: "memory", "cc", "xmm0", "xmm1");
|
||||
if (diff) {
|
||||
return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // HAS_ARGBDETECTROW_AVX2
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) ||
|
||||
// defined(__i386__)) && !defined(LIBYUV_ENABLE_ROWWIN)
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
|
||||
@ -10,14 +10,15 @@
|
||||
|
||||
#include "libyuv/compare_row.h"
|
||||
#include "libyuv/row.h"
|
||||
#include "libyuv/video_common.h"
|
||||
|
||||
// This module is for 32 bit x86 with any compiler, Visual C 64 bit, and for
|
||||
// clang with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__i386__) || defined(_M_IX86) || \
|
||||
((defined(__x86_64__) || defined(_M_X64)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))))
|
||||
// This module is for Visual C 32/64 bit, and for clang with
|
||||
// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
||||
defined(_M_IX86)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
|
||||
#include <immintrin.h> // For SSE2 to AVX512 intrinsics
|
||||
#include <string.h> // For memcpy
|
||||
@ -39,12 +40,15 @@ extern "C" {
|
||||
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
|
||||
#define LIBYUV_TARGET_AVX512BW \
|
||||
__attribute__((target("avx512bw,avx512vl,avx512f")))
|
||||
#define LIBYUV_TARGET_AVX512VNNI \
|
||||
__attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f")))
|
||||
#else
|
||||
#define LIBYUV_TARGET_SSE2
|
||||
#define LIBYUV_TARGET_SSE41
|
||||
#define LIBYUV_TARGET_SSE42
|
||||
#define LIBYUV_TARGET_AVX2
|
||||
#define LIBYUV_TARGET_AVX512BW
|
||||
#define LIBYUV_TARGET_AVX512VNNI
|
||||
#endif // defined(__clang__) || defined(__GNUC__)
|
||||
|
||||
#if defined(HAS_HAMMINGDISTANCE_SSE42)
|
||||
@ -245,6 +249,44 @@ uint32_t SumSquareError_AVX512BW(const uint8_t* src_a,
|
||||
}
|
||||
#endif // HAS_SUMSQUAREERROR_AVX512BW
|
||||
|
||||
#if defined(HAS_SUMSQUAREERROR_AVX512VNNI)
|
||||
// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked
|
||||
// loads that read zeros, so any count is supported.
|
||||
LIBYUV_TARGET_AVX512VNNI
|
||||
uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a,
|
||||
const uint8_t* src_b,
|
||||
int count) {
|
||||
const __m512i zero = _mm512_setzero_si512();
|
||||
__m512i sum0 = _mm512_setzero_si512();
|
||||
__m512i sum1 = _mm512_setzero_si512();
|
||||
while (count >= 64) {
|
||||
__m512i a = _mm512_loadu_si512(src_a);
|
||||
__m512i b = _mm512_loadu_si512(src_b);
|
||||
__m512i d =
|
||||
_mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
|
||||
__m512i lo = _mm512_unpacklo_epi8(d, zero);
|
||||
__m512i hi = _mm512_unpackhi_epi8(d, zero);
|
||||
sum0 = _mm512_dpwssd_epi32(sum0, lo, lo);
|
||||
sum1 = _mm512_dpwssd_epi32(sum1, hi, hi);
|
||||
src_a += 64;
|
||||
src_b += 64;
|
||||
count -= 64;
|
||||
}
|
||||
if (count > 0) {
|
||||
const __mmask64 k = LowBytesMask64(count);
|
||||
__m512i a = _mm512_maskz_loadu_epi8(k, src_a);
|
||||
__m512i b = _mm512_maskz_loadu_epi8(k, src_b);
|
||||
__m512i d =
|
||||
_mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a));
|
||||
__m512i lo = _mm512_unpacklo_epi8(d, zero);
|
||||
__m512i hi = _mm512_unpackhi_epi8(d, zero);
|
||||
sum0 = _mm512_dpwssd_epi32(sum0, lo, lo);
|
||||
sum1 = _mm512_dpwssd_epi32(sum1, hi, hi);
|
||||
}
|
||||
return (uint32_t)_mm512_reduce_add_epi32(_mm512_add_epi32(sum0, sum1));
|
||||
}
|
||||
#endif // HAS_SUMSQUAREERROR_AVX512VNNI
|
||||
|
||||
#if defined(HAS_HASHDJB2_SSE41)
|
||||
// 33 ^ (15 - i) for byte i of 16.
|
||||
static const uint32_t kHashMul16[16] = {
|
||||
@ -892,6 +934,39 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0,
|
||||
}
|
||||
#endif // HAS_SSIMWINDOWROW_AVX512BW
|
||||
|
||||
#if defined(HAS_ARGBDETECTROW_AVX2)
|
||||
LIBYUV_TARGET_AVX2
|
||||
uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) {
|
||||
if (width >= 8) {
|
||||
const __m256i mask_rgb = _mm256_set1_epi32(0x00ffff00);
|
||||
const __m256i ones = _mm256_set1_epi8(-1);
|
||||
do {
|
||||
__m256i v =
|
||||
_mm256_or_si256(_mm256_loadu_si256((const __m256i*)argb), mask_rgb);
|
||||
uint32_t diff =
|
||||
~(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, ones));
|
||||
if (diff) {
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
unsigned long bit; // NOLINT(runtime/int)
|
||||
_BitScanForward(&bit, diff);
|
||||
#else
|
||||
int bit = __builtin_ctz(diff);
|
||||
#endif
|
||||
_mm256_zeroupper();
|
||||
return (bit & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB;
|
||||
}
|
||||
argb += 32;
|
||||
width -= 8;
|
||||
} while (width >= 8);
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
if (width > 0) {
|
||||
return ARGBDetectRow_C(argb, width);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // HAS_ARGBDETECTROW_AVX2
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
@ -355,6 +355,13 @@ static int I41xToI420(const uint16_t* src_y,
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
HalfWidthRow_16To8 = HalfWidthRow_16To8_RVV;
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \
|
||||
defined(HAS_HALFWIDTHROW_16TO8_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
HalfWidthRow_16To8 = HalfWidthRow_16To8_Odd_AVX2;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -1387,6 +1394,11 @@ int MT2TToP010(const uint8_t* src_y,
|
||||
if (!row_buf)
|
||||
return 1;
|
||||
|
||||
#if defined(HAS_UNPACKMT2T_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
UnpackMT2T = UnpackMT2T_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UNPACKMT2T_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
UnpackMT2T = UnpackMT2T_NEON;
|
||||
@ -2313,6 +2325,13 @@ static int Px10ToI420(const uint16_t* src_y,
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
Half16To8 = HalfWidthRow_16To8_RVV;
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \
|
||||
defined(HAS_HALFWIDTHROW_16TO8_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
Half16To8 = HalfWidthRow_16To8_Odd_AVX2;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -2516,6 +2535,12 @@ int YUY2ToI420(const uint8_t* src_yuy2,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOYROW_AVX512BW) && defined(HAS_YUY2TOUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
YUY2ToUVRow = YUY2ToUVRow_AVX512BW;
|
||||
YUY2ToYRow = YUY2ToYRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOYROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
YUY2ToYRow = YUY2ToYRow_Any_NEON;
|
||||
@ -2552,6 +2577,12 @@ int YUY2ToI420(const uint8_t* src_yuy2,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOYROW_HVX) && defined(HAS_YUY2TOUVROW_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
YUY2ToYRow = YUY2ToYRow_HVX;
|
||||
YUY2ToUVRow = YUY2ToUVRow_HVX;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < height - 1; y += 2) {
|
||||
YUY2ToUVRow(src_yuy2, src_stride_yuy2, dst_u, dst_v, width);
|
||||
@ -2616,6 +2647,12 @@ int UYVYToI420(const uint8_t* src_uyvy,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_AVX512BW) && defined(HAS_UYVYTOUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
UYVYToUVRow = UYVYToUVRow_AVX512BW;
|
||||
UYVYToYRow = UYVYToYRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
UYVYToYRow = UYVYToYRow_Any_NEON;
|
||||
@ -2652,6 +2689,12 @@ int UYVYToI420(const uint8_t* src_uyvy,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_HVX) && defined(HAS_UYVYTOUVROW_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
UYVYToYRow = UYVYToYRow_HVX;
|
||||
UYVYToUVRow = UYVYToUVRow_HVX;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < height - 1; y += 2) {
|
||||
UYVYToUVRow(src_uyvy, src_stride_uyvy, dst_u, dst_v, width);
|
||||
@ -2693,24 +2736,13 @@ int AYUVToNV12(const uint8_t* src_ayuv,
|
||||
src_ayuv = src_ayuv + (ptrdiff_t)(height - 1) * src_stride_ayuv;
|
||||
src_stride_ayuv = -src_stride_ayuv;
|
||||
}
|
||||
// place holders for future intel code
|
||||
#if defined(HAS_AYUVTOYROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
AYUVToUVRow = AYUVToUVRow_Any_SSE2;
|
||||
AYUVToYRow = AYUVToYRow_Any_SSE2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
AYUVToUVRow = AYUVToUVRow_SSE2;
|
||||
AYUVToYRow = AYUVToYRow_SSE2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AYUVTOYROW_AVX2)
|
||||
#if defined(HAS_AYUVTOYROW_AVX2) && defined(HAS_AYUVTOUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
AYUVToUVRow = AYUVToUVRow_Any_AVX2;
|
||||
AYUVToYRow = AYUVToYRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
AYUVToUVRow = AYUVToUVRow_AVX2;
|
||||
AYUVToUVRow = AYUVToUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
AYUVToYRow = AYUVToYRow_AVX2;
|
||||
AYUVToUVRow = AYUVToUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -2780,24 +2812,13 @@ int AYUVToNV21(const uint8_t* src_ayuv,
|
||||
src_ayuv = src_ayuv + (ptrdiff_t)(height - 1) * src_stride_ayuv;
|
||||
src_stride_ayuv = -src_stride_ayuv;
|
||||
}
|
||||
// place holders for future intel code
|
||||
#if defined(HAS_AYUVTOYROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
AYUVToVURow = AYUVToVURow_Any_SSE2;
|
||||
AYUVToYRow = AYUVToYRow_Any_SSE2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
AYUVToVURow = AYUVToVURow_SSE2;
|
||||
AYUVToYRow = AYUVToYRow_SSE2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AYUVTOYROW_AVX2)
|
||||
#if defined(HAS_AYUVTOYROW_AVX2) && defined(HAS_AYUVTOVUROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
AYUVToVURow = AYUVToVURow_Any_AVX2;
|
||||
AYUVToYRow = AYUVToYRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
AYUVToVURow = AYUVToVURow_AVX2;
|
||||
AYUVToVURow = AYUVToVURow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
AYUVToYRow = AYUVToYRow_AVX2;
|
||||
AYUVToVURow = AYUVToVURow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -3546,6 +3567,14 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_AVX512BW;
|
||||
@ -3791,12 +3820,6 @@ int AB30ToI420(const uint8_t* src_ab30,
|
||||
&kAbgrI601Constants, width, height);
|
||||
}
|
||||
|
||||
// Enabled if 1 pass is available
|
||||
#if (defined(HAS_RGB24TOYROW_NEON) || defined(HAS_RGB24TOYROW_LSX) || \
|
||||
defined(HAS_RGB24TOYROW_RVV))
|
||||
#define HAS_RGB24TOYROW
|
||||
#endif
|
||||
|
||||
// Convert RGB24 to I420.
|
||||
LIBYUV_API
|
||||
int RGB24ToI420(const uint8_t* src_rgb24,
|
||||
@ -4118,12 +4141,6 @@ int RGB24ToJ420(const uint8_t* src_rgb24,
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Enabled if 1 pass is available
|
||||
#if (defined(HAS_RAWTOYROW_NEON) || defined(HAS_RAWTOYROW_LSX) || \
|
||||
defined(HAS_RAWTOYROW_RVV))
|
||||
#define HAS_RAWTOYROW
|
||||
#endif
|
||||
|
||||
// Convert RAW to I420.
|
||||
LIBYUV_API
|
||||
int RAWToI420(const uint8_t* src_raw,
|
||||
|
||||
@ -631,10 +631,7 @@ int I444ToARGBMatrix(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_I444TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I444ToARGBRow = I444ToARGBRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I444ToARGBRow = I444ToARGBRow_AVX512BW;
|
||||
}
|
||||
I444ToARGBRow = I444ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TOARGBROW_NEON)
|
||||
@ -888,6 +885,11 @@ int I444ToRGB24Matrix(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_Any_NEON;
|
||||
@ -906,14 +908,6 @@ int I444ToRGB24Matrix(const uint8_t* src_y,
|
||||
I444ToRGB24Row = I444ToRGB24Row_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_Any_LSX;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_LSX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_RVV;
|
||||
@ -1030,6 +1024,11 @@ int I010ToAR30Matrix(const uint16_t* src_y,
|
||||
I210ToAR30Row = I210ToAR30Row_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I210TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I210ToAR30Row = I210ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
|
||||
@ -1192,6 +1191,11 @@ int I012ToAR30Matrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I212TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I212ToAR30Row = I212ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I212TOAR30ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I212ToAR30Row = I212ToAR30Row_Any_NEON;
|
||||
@ -1286,6 +1290,11 @@ int I210ToAR30Matrix(const uint16_t* src_y,
|
||||
I210ToAR30Row = I210ToAR30Row_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I210TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I210ToAR30Row = I210ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
|
||||
@ -1460,6 +1469,11 @@ int I410ToAR30Matrix(const uint16_t* src_y,
|
||||
I410ToAR30Row = I410ToAR30Row_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I410TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I410ToAR30Row = I410ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
I410ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
|
||||
@ -1533,6 +1547,11 @@ int I010ToARGBMatrix(const uint16_t* src_y,
|
||||
I210ToARGBRow = I210ToARGBRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I210TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I210ToARGBRow = I210ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
||||
@ -1699,6 +1718,11 @@ int I012ToARGBMatrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I212TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I212ToARGBRow = I212ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I212TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I212ToARGBRow = I212ToARGBRow_Any_NEON;
|
||||
@ -1799,6 +1823,11 @@ int I210ToARGBMatrix(const uint16_t* src_y,
|
||||
I210ToARGBRow = I210ToARGBRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I210TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I210ToARGBRow = I210ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
||||
@ -1986,6 +2015,11 @@ int I410ToARGBMatrix(const uint16_t* src_y,
|
||||
I410ToARGBRow = I410ToARGBRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I410TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I410ToARGBRow = I410ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
I410ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
|
||||
@ -2038,6 +2072,11 @@ int P010ToARGBMatrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P210TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P210ToARGBRow = P210ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P210TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P210ToARGBRow = P210ToARGBRow_Any_NEON;
|
||||
@ -2108,6 +2147,11 @@ int P210ToARGBMatrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P210TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P210ToARGBRow = P210ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P210TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P210ToARGBRow = P210ToARGBRow_Any_NEON;
|
||||
@ -2176,6 +2220,11 @@ int P410ToARGBMatrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P410ToARGBRow = P410ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P410ToARGBRow = P410ToARGBRow_Any_NEON;
|
||||
@ -2244,6 +2293,11 @@ int P010ToAR30Matrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P210TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P210ToAR30Row = P210ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P210TOAR30ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P210ToAR30Row = P210ToAR30Row_Any_NEON;
|
||||
@ -2314,6 +2368,11 @@ int P210ToAR30Matrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P210TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P210ToAR30Row = P210ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P210TOAR30ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P210ToAR30Row = P210ToAR30Row_Any_NEON;
|
||||
@ -2382,6 +2441,11 @@ int P410ToAR30Matrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P410ToAR30Row = P410ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOAR30ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P410ToAR30Row = P410ToAR30Row_Any_NEON;
|
||||
@ -2822,6 +2886,11 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON;
|
||||
@ -3119,6 +3188,11 @@ int I010AlphaToARGBMatrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I210ALPHATOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBATTENUATEROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
|
||||
@ -3261,6 +3335,11 @@ int I210AlphaToARGBMatrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I210ALPHATOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBATTENUATEROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
|
||||
@ -3401,6 +3480,11 @@ int I410AlphaToARGBMatrix(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I410ALPHATOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBATTENUATEROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
|
||||
@ -4214,11 +4298,11 @@ int RAWToRGBA(const uint8_t* src_raw,
|
||||
height = 1;
|
||||
src_stride_raw = dst_stride_rgba = 0;
|
||||
}
|
||||
#if defined(HAS_RAWTORGBAROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
RAWToRGBARow = RAWToRGBARow_Any_SSSE3;
|
||||
#if defined(HAS_RAWTORGBAROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
RAWToRGBARow = RAWToRGBARow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
RAWToRGBARow = RAWToRGBARow_SSSE3;
|
||||
RAWToRGBARow = RAWToRGBARow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -4370,6 +4454,11 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGB1555TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGB1555ToARGBRow = ARGB1555ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGB1555TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_NEON;
|
||||
@ -4530,6 +4619,14 @@ int AR30ToARGB(const uint8_t* src_ar30,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_AVX512BW;
|
||||
@ -4579,6 +4676,14 @@ int AR30ToABGR(const uint8_t* src_ar30,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOABGRROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
AR30ToABGRRow = AR30ToABGRRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToABGRRow = AR30ToABGRRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOABGRROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToABGRRow = AR30ToABGRRow_AVX512BW;
|
||||
@ -4628,6 +4733,14 @@ int AR30ToAB30(const uint8_t* src_ar30,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOAB30ROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
AR30ToAB30Row = AR30ToAB30Row_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToAB30Row = AR30ToAB30Row_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOAB30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToAB30Row = AR30ToAB30Row_AVX512BW;
|
||||
@ -5391,6 +5504,11 @@ int YUY2ToARGBMatrix(const uint8_t* src_yuy2,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
YUY2ToARGBRow = YUY2ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
YUY2ToARGBRow = YUY2ToARGBRow_Any_NEON;
|
||||
@ -5491,6 +5609,11 @@ int UYVYToARGBMatrix(const uint8_t* src_uyvy,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
UYVYToARGBRow = UYVYToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
UYVYToARGBRow = UYVYToARGBRow_Any_NEON;
|
||||
@ -5721,6 +5844,11 @@ int I422ToRGBAMatrix(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGBAROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I422ToRGBARow = I422ToRGBARow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGBAROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I422ToRGBARow = I422ToRGBARow_Any_NEON;
|
||||
@ -5958,6 +6086,11 @@ int I420ToRGBAMatrix(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGBAROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I422ToRGBARow = I422ToRGBARow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGBAROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I422ToRGBARow = I422ToRGBARow_Any_NEON;
|
||||
@ -6474,6 +6607,12 @@ int I420ToARGB1555(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TOARGB1555ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
|
||||
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
|
||||
I422ToARGB1555Row = I422ToARGB1555Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TOARGB1555ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I422ToARGB1555Row = I422ToARGB1555Row_Any_NEON;
|
||||
@ -6573,6 +6712,12 @@ int I420ToARGB4444(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TOARGB4444ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
|
||||
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
|
||||
I422ToARGB4444Row = I422ToARGB4444Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TOARGB4444ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I422ToARGB4444Row = I422ToARGB4444Row_Any_NEON;
|
||||
@ -6674,6 +6819,12 @@ int I420ToRGB565Matrix(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB565ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
|
||||
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
|
||||
I422ToRGB565Row = I422ToRGB565Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB565ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I422ToRGB565Row = I422ToRGB565Row_Any_NEON;
|
||||
@ -6833,6 +6984,12 @@ int I422ToRGB565Matrix(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB565ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
|
||||
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
|
||||
I422ToRGB565Row = I422ToRGB565Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB565ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I422ToRGB565Row = I422ToRGB565Row_Any_NEON;
|
||||
@ -7018,14 +7175,6 @@ int I420ToRGB565Dither(const uint8_t* src_y,
|
||||
I422ToARGBRow = I422ToARGBRow_HVX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTORGB565DITHERROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_SSE2;
|
||||
if (IS_ALIGNED(width, 4)) {
|
||||
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SSE2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTORGB565DITHERROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2;
|
||||
@ -7034,6 +7183,11 @@ int I420ToRGB565Dither(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTORGB565DITHERROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTORGB565DITHERROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON;
|
||||
@ -7306,10 +7460,7 @@ static int I420ToARGBMatrixBilinear(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_I444TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I444ToARGBRow = I444ToARGBRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I444ToARGBRow = I444ToARGBRow_AVX512BW;
|
||||
}
|
||||
I444ToARGBRow = I444ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TOARGBROW_NEON)
|
||||
@ -7477,10 +7628,7 @@ static int I422ToARGBMatrixLinear(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_I444TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I444ToARGBRow = I444ToARGBRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I444ToARGBRow = I444ToARGBRow_AVX512BW;
|
||||
}
|
||||
I444ToARGBRow = I444ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TOARGBROW_NEON)
|
||||
@ -7623,6 +7771,11 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_Any_NEON;
|
||||
@ -7641,14 +7794,6 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y,
|
||||
I444ToRGB24Row = I444ToRGB24Row_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_Any_LASX;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_LASX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_RVV;
|
||||
@ -7794,6 +7939,11 @@ static int I010ToAR30MatrixBilinear(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I410TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I410ToAR30Row = I410ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
@ -7919,6 +8069,11 @@ static int I210ToAR30MatrixLinear(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I410TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I410ToAR30Row = I410ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
@ -8023,6 +8178,11 @@ static int I010ToARGBMatrixBilinear(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I410TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I410ToARGBRow = I410ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
@ -8147,6 +8307,11 @@ static int I210ToARGBMatrixLinear(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I410TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I410ToARGBRow = I410ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
@ -8241,6 +8406,11 @@ static int I420AlphaToARGBMatrixBilinear(
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON;
|
||||
@ -8259,14 +8429,6 @@ static int I420AlphaToARGBMatrixBilinear(
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_LASX;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_LASX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_RVV;
|
||||
@ -8476,6 +8638,11 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON;
|
||||
@ -8494,14 +8661,6 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y,
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_LASX)
|
||||
if (TestCpuFlag(kCpuHasLASX)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_LASX;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_LASX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444ALPHATOARGBROW_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
I444AlphaToARGBRow = I444AlphaToARGBRow_RVV;
|
||||
@ -8694,6 +8853,11 @@ static int I010AlphaToARGBMatrixBilinear(
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I410ALPHATOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBATTENUATEROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
|
||||
@ -8898,6 +9062,11 @@ static int I210AlphaToARGBMatrixLinear(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I410ALPHATOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBATTENUATEROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3;
|
||||
@ -9039,6 +9208,11 @@ static int P010ToARGBMatrixBilinear(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P410ToARGBRow = P410ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P410ToARGBRow = P410ToARGBRow_Any_NEON;
|
||||
@ -9151,6 +9325,11 @@ static int P210ToARGBMatrixLinear(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P410ToARGBRow = P410ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P410ToARGBRow = P410ToARGBRow_Any_NEON;
|
||||
@ -9249,6 +9428,11 @@ static int P010ToAR30MatrixBilinear(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P410ToAR30Row = P410ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOAR30ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P410ToAR30Row = P410ToAR30Row_Any_NEON;
|
||||
@ -9361,6 +9545,11 @@ static int P210ToAR30MatrixLinear(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOAR30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
P410ToAR30Row = P410ToAR30Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_P410TOAR30ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
P410ToAR30Row = P410ToAR30Row_Any_NEON;
|
||||
@ -9461,6 +9650,11 @@ static int I422ToRGB24MatrixLinear(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TORGB24ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
I444ToRGB24Row = I444ToRGB24Row_Any_NEON;
|
||||
|
||||
@ -820,6 +820,11 @@ static int I4xxToPxxx(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow_16 = MergeUVRow_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
MergeUVRow_16 = MergeUVRow_16_Any_NEON;
|
||||
|
||||
@ -4040,6 +4040,11 @@ int ARGBToRGB565Dither(const uint8_t* src_argb,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTORGB565DITHERROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTORGB565DITHERROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON;
|
||||
@ -4123,6 +4128,11 @@ int ARGBToRGB565(const uint8_t* src_argb,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTORGB565ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
ARGBToRGB565Row = ARGBToRGB565Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBTORGB565ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBToRGB565Row = ARGBToRGB565Row_Any_NEON;
|
||||
|
||||
@ -174,10 +174,7 @@ static int NV24ToARGB(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_I444TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I444ToARGBRow = I444ToARGBRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I444ToARGBRow = I444ToARGBRow_AVX512BW;
|
||||
}
|
||||
I444ToARGBRow = I444ToARGBRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I444TOARGBROW_NEON)
|
||||
|
||||
@ -26,16 +26,6 @@ namespace libyuv {
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static void HalfRow_C(const uint8_t* src_uv,
|
||||
ptrdiff_t src_uv_stride,
|
||||
uint8_t* dst_uv,
|
||||
int width) {
|
||||
int x;
|
||||
for (x = 0; x < width; ++x) {
|
||||
dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1;
|
||||
}
|
||||
}
|
||||
|
||||
static int NV16ToI420(const uint8_t* src_y,
|
||||
int src_stride_y,
|
||||
const uint8_t* src_uv,
|
||||
@ -227,6 +217,8 @@ static int NV24ToI420(const uint8_t* src_y,
|
||||
int x, y;
|
||||
int halfwidth = (width + 1) >> 1;
|
||||
int halfheight = (height + 1) >> 1;
|
||||
void (*HalfRow)(const uint8_t* src_uv, ptrdiff_t src_uv_stride,
|
||||
uint8_t* dst_uv, int width) = HalfRow_C;
|
||||
if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 ||
|
||||
height == 0 || height == INT_MIN) {
|
||||
return -1;
|
||||
@ -242,6 +234,14 @@ static int NV24ToI420(const uint8_t* src_y,
|
||||
if (dst_y) {
|
||||
CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
|
||||
}
|
||||
#if defined(HAS_HALFROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
HalfRow = HalfRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width * 2, 32)) {
|
||||
HalfRow = HalfRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
align_buffer_64(row_uv, width * 2);
|
||||
if (!row_uv) {
|
||||
@ -249,7 +249,7 @@ static int NV24ToI420(const uint8_t* src_y,
|
||||
}
|
||||
for (y = 0; y < halfheight; ++y) {
|
||||
if (y * 2 + 1 < height) {
|
||||
HalfRow_C(src_uv, src_stride_uv, row_uv, width * 2);
|
||||
HalfRow(src_uv, src_stride_uv, row_uv, width * 2);
|
||||
} else {
|
||||
memcpy(row_uv, src_uv, (size_t)width * 2);
|
||||
}
|
||||
|
||||
@ -405,6 +405,7 @@ static SAFEBUFFERS int GetCpuFlags(void) {
|
||||
int cpu_info1[4] = {0, 0, 0, 0};
|
||||
int cpu_info7[4] = {0, 0, 0, 0};
|
||||
int cpu_einfo7[4] = {0, 0, 0, 0};
|
||||
int cpu_einfo1e[4] = {0, 0, 0, 0};
|
||||
int cpu_info24[4] = {0, 0, 0, 0};
|
||||
int cpu_info21[4] = {0, 0, 0, 0};
|
||||
int cpu_amdinfo21[4] = {0, 0, 0, 0};
|
||||
@ -425,6 +426,9 @@ static SAFEBUFFERS int GetCpuFlags(void) {
|
||||
if ((unsigned)cpu_extinfo0[0] >= 0x80000021u) {
|
||||
CpuId(0x80000021, 0, cpu_amdinfo21);
|
||||
}
|
||||
if (cpu_info0[0] >= 0x1e) {
|
||||
CpuId(0x1e, 1, cpu_einfo1e);
|
||||
}
|
||||
if (cpu_info0[0] >= 0x21) {
|
||||
CpuId(0x21, 0, cpu_info21);
|
||||
}
|
||||
@ -459,7 +463,8 @@ static SAFEBUFFERS int GetCpuFlags(void) {
|
||||
((cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0) |
|
||||
((cpu_einfo7[3] & 0x00080000) ? kCpuHasAVX10 : 0) |
|
||||
((cpu_info7[3] & 0x02000000) ? kCpuHasAMXINT8 : 0) |
|
||||
((cpu_info21[0] & 0x00800000) ? kCpuHasAVX512BMM : 0);
|
||||
((cpu_info21[0] & 0x00800000) ? kCpuHasAVX512BMM : 0) |
|
||||
((cpu_einfo1e[0] & 0x00000040) ? kCpuHasAMXTF32 : 0);
|
||||
if (cpu_info0[0] >= 0x24 && (cpu_einfo7[3] & 0x00080000)) {
|
||||
cpu_info |= ((cpu_info24[1] & 0xFF) >= 2) ? kCpuHasAVX10_2 : 0;
|
||||
}
|
||||
|
||||
@ -148,6 +148,11 @@ void CopyPlane_16(const uint16_t* src_y,
|
||||
if (src_y == dst_y && src_stride_y == dst_stride_y) {
|
||||
return;
|
||||
}
|
||||
#if defined(HAS_COPYROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
CopyRow = IS_ALIGNED(width, 32) ? CopyRow_16_AVX2 : CopyRow_16_Any_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_COPYROW_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
CopyRow = IS_ALIGNED(width, 16) ? CopyRow_16_NEON : CopyRow_16_Any_NEON;
|
||||
@ -406,6 +411,11 @@ void Convert8To8Plane(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO8ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
Convert8To8Row = Convert8To8Row_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CONVERT8TO8ROW_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
Convert8To8Row = Convert8To8Row_HVX;
|
||||
@ -1009,6 +1019,11 @@ void MergeUVPlane_16(const uint16_t* src_u,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MergeUVRow_16 = MergeUVRow_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
MergeUVRow_16 = MergeUVRow_16_Any_NEON;
|
||||
@ -1160,6 +1175,11 @@ void ConvertToLSBPlane_16(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_DIVIDEROW_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
DivideRow = DivideRow_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_DIVIDEROW_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
DivideRow = DivideRow_16_Any_NEON;
|
||||
@ -1337,11 +1357,11 @@ int DetilePlane(const uint8_t* src_y,
|
||||
dst_stride_y = -dst_stride_y;
|
||||
}
|
||||
|
||||
#if defined(HAS_DETILEROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
DetileRow = DetileRow_Any_SSE2;
|
||||
#if defined(HAS_DETILEROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
DetileRow = DetileRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
DetileRow = DetileRow_SSE2;
|
||||
DetileRow = DetileRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -1394,19 +1414,11 @@ int DetilePlane_16(const uint16_t* src_y,
|
||||
dst_stride_y = -dst_stride_y;
|
||||
}
|
||||
|
||||
#if defined(HAS_DETILEROW_16_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
DetileRow_16 = DetileRow_16_Any_SSE2;
|
||||
#if defined(HAS_DETILEROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
DetileRow_16 = DetileRow_16_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
DetileRow_16 = DetileRow_16_SSE2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_DETILEROW_16_AVX)
|
||||
if (TestCpuFlag(kCpuHasAVX)) {
|
||||
DetileRow_16 = DetileRow_16_Any_AVX;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
DetileRow_16 = DetileRow_16_AVX;
|
||||
DetileRow_16 = DetileRow_16_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -1466,11 +1478,11 @@ void DetileSplitUVPlane(const uint8_t* src_uv,
|
||||
dst_stride_v = -dst_stride_v;
|
||||
}
|
||||
|
||||
#if defined(HAS_DETILESPLITUVROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
DetileSplitUVRow = DetileSplitUVRow_Any_SSSE3;
|
||||
#if defined(HAS_DETILESPLITUVROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
DetileSplitUVRow = DetileSplitUVRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
DetileSplitUVRow = DetileSplitUVRow_SSSE3;
|
||||
DetileSplitUVRow = DetileSplitUVRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -1540,11 +1552,11 @@ void DetileToYUY2(const uint8_t* src_y,
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_DETILETOYUY2_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
DetileToYUY2 = DetileToYUY2_Any_SSE2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
DetileToYUY2 = DetileToYUY2_SSE2;
|
||||
#if defined(HAS_DETILETOYUY2_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
DetileToYUY2 = DetileToYUY2_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
DetileToYUY2 = DetileToYUY2_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -1684,11 +1696,11 @@ void MergeRGBPlane(const uint8_t* src_r,
|
||||
height = 1;
|
||||
src_stride_r = src_stride_g = src_stride_b = dst_stride_rgb = 0;
|
||||
}
|
||||
#if defined(HAS_MERGERGBROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
MergeRGBRow = MergeRGBRow_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
MergeRGBRow = MergeRGBRow_SSSE3;
|
||||
#if defined(HAS_MERGERGBROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
MergeRGBRow = MergeRGBRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
MergeRGBRow = MergeRGBRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -2536,6 +2548,11 @@ int YUY2ToI422(const uint8_t* src_yuy2,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOYROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
YUY2ToYRow = YUY2ToYRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOYROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
YUY2ToYRow = YUY2ToYRow_Any_NEON;
|
||||
@ -2644,6 +2661,11 @@ int UYVYToI422(const uint8_t* src_uyvy,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
UYVYToYRow = UYVYToYRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
UYVYToYRow = UYVYToYRow_Any_NEON;
|
||||
@ -2740,6 +2762,11 @@ int YUY2ToY(const uint8_t* src_yuy2,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOYROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
YUY2ToYRow = YUY2ToYRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOYROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
YUY2ToYRow = YUY2ToYRow_Any_NEON;
|
||||
@ -2810,6 +2837,11 @@ int UYVYToY(const uint8_t* src_uyvy,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
UYVYToYRow = UYVYToYRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
UYVYToYRow = UYVYToYRow_Any_NEON;
|
||||
@ -2899,10 +2931,7 @@ void MirrorPlane(const uint8_t* src_y,
|
||||
#endif
|
||||
#if defined(HAS_MIRRORROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MirrorRow = MirrorRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
MirrorRow = MirrorRow_AVX512BW;
|
||||
}
|
||||
MirrorRow = MirrorRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORROW_LSX)
|
||||
@ -3095,17 +3124,9 @@ void MirrorUVPlane(const uint8_t* src_uv,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORROW_16_AVX512BW)
|
||||
#if defined(HAS_MIRRORUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MirrorUVRow = reinterpret_cast<void (*)(const uint8_t*, uint8_t*, int)>(
|
||||
MirrorRow_16_AVX512BW);
|
||||
}
|
||||
#elif defined(HAS_MIRRORUVROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MirrorUVRow = MirrorUVRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
MirrorUVRow = MirrorUVRow_AVX512BW;
|
||||
}
|
||||
MirrorUVRow = MirrorUVRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORUVROW_LSX)
|
||||
@ -3447,9 +3468,9 @@ int ARGBBlend(const uint8_t* src_argb0,
|
||||
height = 1;
|
||||
src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBBLENDROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ARGBBlendRow = ARGBBlendRow_SSSE3;
|
||||
#if defined(HAS_ARGBBLENDROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBBlendRow = ARGBBlendRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBBLENDROW_AVX512BW)
|
||||
@ -4429,11 +4450,11 @@ void SetPlane(uint8_t* dst_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SETROW_X86)
|
||||
if (TestCpuFlag(kCpuHasX86)) {
|
||||
SetRow = SetRow_Any_X86;
|
||||
if (IS_ALIGNED(width, 4)) {
|
||||
SetRow = SetRow_X86;
|
||||
#if defined(HAS_SETROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
SetRow = SetRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
SetRow = SetRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -4541,9 +4562,12 @@ int ARGBRect(uint8_t* dst_argb,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBSETROW_X86)
|
||||
if (TestCpuFlag(kCpuHasX86)) {
|
||||
ARGBSetRow = ARGBSetRow_X86;
|
||||
#if defined(HAS_ARGBSETROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBSetRow = ARGBSetRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
ARGBSetRow = ARGBSetRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBSETROW_LSX)
|
||||
@ -4771,9 +4795,9 @@ int ARGBGrayTo(const uint8_t* src_argb,
|
||||
height = 1;
|
||||
src_stride_argb = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBGRAYROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) {
|
||||
ARGBGrayRow = ARGBGrayRow_SSSE3;
|
||||
#if defined(HAS_ARGBGRAYROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBGrayRow = ARGBGrayRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBGRAYROW_AVX512BW)
|
||||
@ -4836,9 +4860,9 @@ int ARGBGray(uint8_t* dst_argb,
|
||||
height = 1;
|
||||
dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBGRAYROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) {
|
||||
ARGBGrayRow = ARGBGrayRow_SSSE3;
|
||||
#if defined(HAS_ARGBGRAYROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBGrayRow = ARGBGrayRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBGRAYROW_AVX512BW)
|
||||
@ -4899,9 +4923,14 @@ int ARGBSepia(uint8_t* dst_argb,
|
||||
height = 1;
|
||||
dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBSEPIAROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) {
|
||||
ARGBSepiaRow = ARGBSepiaRow_SSSE3;
|
||||
#if defined(HAS_ARGBSEPIAROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBSepiaRow = ARGBSepiaRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBSEPIAROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBSepiaRow = ARGBSepiaRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBSEPIAROW_NEON)
|
||||
@ -4962,9 +4991,14 @@ int ARGBColorMatrix(const uint8_t* src_argb,
|
||||
height = 1;
|
||||
src_stride_argb = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBCOLORMATRIXROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) {
|
||||
ARGBColorMatrixRow = ARGBColorMatrixRow_SSSE3;
|
||||
#if defined(HAS_ARGBCOLORMATRIXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBColorMatrixRow = ARGBColorMatrixRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOLORMATRIXROW_AVX512VNNI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VNNI)) {
|
||||
ARGBColorMatrixRow = ARGBColorMatrixRow_AVX512VNNI;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOLORMATRIXROW_NEON)
|
||||
@ -5056,9 +5090,9 @@ int ARGBColorTable(uint8_t* dst_argb,
|
||||
height = 1;
|
||||
dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBCOLORTABLEROW_X86)
|
||||
if (TestCpuFlag(kCpuHasX86)) {
|
||||
ARGBColorTableRow = ARGBColorTableRow_X86;
|
||||
#if defined(HAS_ARGBCOLORTABLEROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBColorTableRow = ARGBColorTableRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOLORTABLEROW_NEON)
|
||||
@ -5097,9 +5131,9 @@ int RGBColorTable(uint8_t* dst_argb,
|
||||
height = 1;
|
||||
dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_RGBCOLORTABLEROW_X86)
|
||||
if (TestCpuFlag(kCpuHasX86)) {
|
||||
RGBColorTableRow = RGBColorTableRow_X86;
|
||||
#if defined(HAS_RGBCOLORTABLEROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
RGBColorTableRow = RGBColorTableRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBCOLORTABLEROW_NEON)
|
||||
@ -5147,9 +5181,9 @@ int ARGBQuantize(uint8_t* dst_argb,
|
||||
height = 1;
|
||||
dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBQUANTIZEROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) {
|
||||
ARGBQuantizeRow = ARGBQuantizeRow_SSE2;
|
||||
#if defined(HAS_ARGBQUANTIZEROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBQuantizeRow = ARGBQuantizeRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBQUANTIZEROW_NEON)
|
||||
@ -5191,11 +5225,6 @@ int ARGBComputeCumulativeSum(const uint8_t* src_argb,
|
||||
if (!dst_cumsum || !src_argb || width <= 0 || height <= 0) {
|
||||
return -1;
|
||||
}
|
||||
#if defined(HAS_COMPUTECUMULATIVESUMROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_COMPUTECUMULATIVESUMROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2;
|
||||
@ -5260,12 +5289,6 @@ int ARGBBlur(const uint8_t* src_argb,
|
||||
if (radius <= 0 || height <= 1) {
|
||||
return -1;
|
||||
}
|
||||
#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2;
|
||||
CumulativeSumToAverageRow = CumulativeSumToAverageRow_SSE2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2;
|
||||
@ -5371,9 +5394,9 @@ int ARGBShade(const uint8_t* src_argb,
|
||||
height = 1;
|
||||
src_stride_argb = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBSHADEROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) {
|
||||
ARGBShadeRow = ARGBShadeRow_SSE2;
|
||||
#if defined(HAS_ARGBSHADEROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBShadeRow = ARGBShadeRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBSHADEROW_NEON)
|
||||
@ -5539,14 +5562,6 @@ int InterpolatePlane_16(const uint16_t* src0,
|
||||
height = 1;
|
||||
src_stride0 = src_stride1 = dst_stride = 0;
|
||||
}
|
||||
#if defined(HAS_INTERPOLATEROW_16_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
InterpolateRow_16 = InterpolateRow_16_Any_SSSE3;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
InterpolateRow_16 = InterpolateRow_16_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
InterpolateRow_16 = InterpolateRow_16_Any_AVX2;
|
||||
@ -5573,14 +5588,6 @@ int InterpolatePlane_16(const uint16_t* src0,
|
||||
InterpolateRow_16 = InterpolateRow_16_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_LSX)
|
||||
if (TestCpuFlag(kCpuHasLSX)) {
|
||||
InterpolateRow_16 = InterpolateRow_16_Any_LSX;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
InterpolateRow_16 = InterpolateRow_16_LSX;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
InterpolateRow_16 = InterpolateRow_16_Any_WASMSIMD;
|
||||
@ -5868,11 +5875,21 @@ int GaussPlane_F32(const float* src,
|
||||
src_stride = -src_stride;
|
||||
}
|
||||
|
||||
#if defined(HAS_GAUSSCOL_F32_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
GaussCol_F32 = GaussCol_F32_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_GAUSSCOL_F32_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
|
||||
GaussCol_F32 = GaussCol_F32_NEON;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_GAUSSROW_F32_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
GaussRow_F32 = GaussRow_F32_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_GAUSSROW_F32_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
|
||||
GaussRow_F32 = GaussRow_F32_NEON;
|
||||
@ -6026,9 +6043,9 @@ static int ARGBSobelize(const uint8_t* src_argb,
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_SOBELYROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
SobelYRow = SobelYRow_SSE2;
|
||||
#if defined(HAS_SOBELYROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
SobelYRow = SobelYRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SOBELYROW_NEON)
|
||||
@ -6036,9 +6053,9 @@ static int ARGBSobelize(const uint8_t* src_argb,
|
||||
SobelYRow = SobelYRow_NEON;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SOBELXROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
SobelXRow = SobelXRow_SSE2;
|
||||
#if defined(HAS_SOBELXROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
SobelXRow = SobelXRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SOBELXROW_NEON)
|
||||
@ -6106,11 +6123,11 @@ int ARGBSobel(const uint8_t* src_argb,
|
||||
int height) {
|
||||
void (*SobelRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely,
|
||||
uint8_t* dst_argb, int width) = SobelRow_C;
|
||||
#if defined(HAS_SOBELROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
SobelRow = SobelRow_Any_SSE2;
|
||||
#if defined(HAS_SOBELROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
SobelRow = SobelRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
SobelRow = SobelRow_SSE2;
|
||||
SobelRow = SobelRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -6152,11 +6169,11 @@ int ARGBSobelToPlane(const uint8_t* src_argb,
|
||||
int height) {
|
||||
void (*SobelToPlaneRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely,
|
||||
uint8_t* dst_, int width) = SobelToPlaneRow_C;
|
||||
#if defined(HAS_SOBELTOPLANEROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
SobelToPlaneRow = SobelToPlaneRow_Any_SSE2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
SobelToPlaneRow = SobelToPlaneRow_SSE2;
|
||||
#if defined(HAS_SOBELTOPLANEROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
SobelToPlaneRow = SobelToPlaneRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
SobelToPlaneRow = SobelToPlaneRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -6199,11 +6216,11 @@ int ARGBSobelXY(const uint8_t* src_argb,
|
||||
int height) {
|
||||
void (*SobelXYRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely,
|
||||
uint8_t* dst_argb, int width) = SobelXYRow_C;
|
||||
#if defined(HAS_SOBELXYROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
SobelXYRow = SobelXYRow_Any_SSE2;
|
||||
#if defined(HAS_SOBELXYROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
SobelXYRow = SobelXYRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
SobelXYRow = SobelXYRow_SSE2;
|
||||
SobelXYRow = SobelXYRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -6340,6 +6357,11 @@ int HalfFloatPlane(const uint16_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFFLOATROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
HalfFloatRow = HalfFloatRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_HALFFLOATROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)
|
||||
#if defined(__arm__)
|
||||
@ -6394,6 +6416,14 @@ int ByteToFloat(const uint8_t* src_y, float* dst_y, float scale, int width) {
|
||||
if (!src_y || !dst_y || width <= 0) {
|
||||
return -1;
|
||||
}
|
||||
#if defined(HAS_BYTETOFLOATROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ByteToFloatRow = ByteToFloatRow_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
ByteToFloatRow = ByteToFloatRow_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_BYTETOFLOATROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ByteToFloatRow = ByteToFloatRow_Any_NEON;
|
||||
@ -6437,9 +6467,9 @@ int ARGBLumaColorTable(const uint8_t* src_argb,
|
||||
height = 1;
|
||||
src_stride_argb = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBLUMACOLORTABLEROW_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 4)) {
|
||||
ARGBLumaColorTableRow = ARGBLumaColorTableRow_SSSE3;
|
||||
#if defined(HAS_ARGBLUMACOLORTABLEROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ARGBLumaColorTableRow = ARGBLumaColorTableRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBLUMACOLORTABLEROW_NEON)
|
||||
@ -6730,6 +6760,11 @@ int YUY2ToNV12(const uint8_t* src_yuy2,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOYROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
YUY2ToYRow = YUY2ToYRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_YUY2TOYROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
YUY2ToYRow = YUY2ToYRow_Any_NEON;
|
||||
|
||||
127
source/rotate.cc
127
source/rotate.cc
@ -24,6 +24,21 @@ namespace libyuv {
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static void TransposeWxH(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
int i;
|
||||
for (i = 0; i < width; ++i) {
|
||||
int j;
|
||||
for (j = 0; j < height; ++j) {
|
||||
dst[i * dst_stride + j] = src[j * src_stride + i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LIBYUV_API
|
||||
void TransposePlane(const uint8_t* src,
|
||||
int src_stride,
|
||||
@ -33,8 +48,8 @@ void TransposePlane(const uint8_t* src,
|
||||
int height) {
|
||||
int i = height;
|
||||
#if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX)
|
||||
void (*TransposeWxH)(const uint8_t* src, int src_stride, uint8_t* dst,
|
||||
int dst_stride, int width, int height) = NULL;
|
||||
void (*TransposeWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst,
|
||||
int dst_stride, int width, int height) = NULL;
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \
|
||||
defined(HAS_TRANSPOSEWX16_LASX) || defined(HAS_TRANSPOSEWX16_WASMSIMD)
|
||||
@ -67,19 +82,19 @@ void TransposePlane(const uint8_t* src,
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_SME)
|
||||
if (TestCpuFlag(kCpuHasSME)) {
|
||||
TransposeWxH = TransposeWxH_SME;
|
||||
TransposeWxH_fn = TransposeWxH_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
TransposeWxH = TransposeWxH_HVX;
|
||||
TransposeWxH_fn = TransposeWxH_HVX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX8_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
TransposeWx8 = TransposeWx8_Any_SSSE3;
|
||||
#if defined(HAS_TRANSPOSEWX8_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
TransposeWx8 = TransposeWx8_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
TransposeWx8 = TransposeWx8_SSSE3;
|
||||
TransposeWx8 = TransposeWx8_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -130,8 +145,8 @@ void TransposePlane(const uint8_t* src,
|
||||
#endif
|
||||
|
||||
#if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX)
|
||||
if (TransposeWxH) {
|
||||
TransposeWxH(src, src_stride, dst, dst_stride, width, height);
|
||||
if (TransposeWxH_fn) {
|
||||
TransposeWxH_fn(src, src_stride, dst, dst_stride, width, height);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
@ -159,7 +174,7 @@ void TransposePlane(const uint8_t* src,
|
||||
#endif
|
||||
|
||||
if (i > 0) {
|
||||
TransposeWxH_C(src, src_stride, dst, dst_stride, width, i);
|
||||
TransposeWxH(src, src_stride, dst, dst_stride, width, i);
|
||||
}
|
||||
}
|
||||
|
||||
@ -237,10 +252,7 @@ void RotatePlane180(const uint8_t* src,
|
||||
#endif
|
||||
#if defined(HAS_MIRRORROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
MirrorRow = MirrorRow_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 64)) {
|
||||
MirrorRow = MirrorRow_AVX512BW;
|
||||
}
|
||||
MirrorRow = MirrorRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_MIRRORROW_LSX)
|
||||
@ -341,6 +353,24 @@ void RotatePlane180(const uint8_t* src,
|
||||
free_aligned_buffer_64(row);
|
||||
}
|
||||
|
||||
static void TransposeUVWxH(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst_a,
|
||||
int dst_stride_a,
|
||||
uint8_t* dst_b,
|
||||
int dst_stride_b,
|
||||
int width,
|
||||
int height) {
|
||||
int i;
|
||||
for (i = 0; i < width * 2; i += 2) {
|
||||
int j;
|
||||
for (j = 0; j < height; ++j) {
|
||||
dst_a[((i >> 1) * dst_stride_a) + j] = src[i + (j * src_stride)];
|
||||
dst_b[((i >> 1) * dst_stride_b) + j] = src[i + (j * src_stride) + 1];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LIBYUV_API
|
||||
void SplitTransposeUV(const uint8_t* src,
|
||||
int src_stride,
|
||||
@ -352,9 +382,9 @@ void SplitTransposeUV(const uint8_t* src,
|
||||
int height) {
|
||||
int i = height;
|
||||
#if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX)
|
||||
void (*TransposeUVWxH)(const uint8_t* src, int src_stride, uint8_t* dst_a,
|
||||
int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
|
||||
int width, int height) = NULL;
|
||||
void (*TransposeUVWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst_a,
|
||||
int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
|
||||
int width, int height) = NULL;
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEUVWX16_AVX2) || \
|
||||
defined(HAS_TRANSPOSEUVWX16_AVX512BW) || \
|
||||
@ -393,19 +423,19 @@ void SplitTransposeUV(const uint8_t* src,
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEUVWXH_SME)
|
||||
if (TestCpuFlag(kCpuHasSME)) {
|
||||
TransposeUVWxH = TransposeUVWxH_SME;
|
||||
TransposeUVWxH_fn = TransposeUVWxH_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEUVWXH_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
TransposeUVWxH = TransposeUVWxH_HVX;
|
||||
TransposeUVWxH_fn = TransposeUVWxH_HVX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEUVWX8_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
TransposeUVWx8 = TransposeUVWx8_Any_SSE2;
|
||||
#if defined(HAS_TRANSPOSEUVWX8_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
TransposeUVWx8 = TransposeUVWx8_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
TransposeUVWx8 = TransposeUVWx8_SSE2;
|
||||
TransposeUVWx8 = TransposeUVWx8_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -440,9 +470,9 @@ void SplitTransposeUV(const uint8_t* src,
|
||||
#endif
|
||||
|
||||
#if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX)
|
||||
if (TransposeUVWxH) {
|
||||
TransposeUVWxH(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
|
||||
width, i);
|
||||
if (TransposeUVWxH_fn) {
|
||||
TransposeUVWxH_fn(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
|
||||
width, i);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
@ -472,8 +502,8 @@ void SplitTransposeUV(const uint8_t* src,
|
||||
}
|
||||
|
||||
if (i > 0) {
|
||||
TransposeUVWxH_C(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
|
||||
width, i);
|
||||
TransposeUVWxH(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
|
||||
width, i);
|
||||
}
|
||||
}
|
||||
|
||||
@ -657,6 +687,21 @@ int RotatePlane(const uint8_t* src,
|
||||
return -1;
|
||||
}
|
||||
|
||||
static void TransposeWxH_16(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
int i;
|
||||
for (i = 0; i < width; ++i) {
|
||||
int j;
|
||||
for (j = 0; j < height; ++j) {
|
||||
dst[i * dst_stride + j] = src[j * src_stride + i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void TransposePlane_16(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
@ -666,8 +711,8 @@ static void TransposePlane_16(const uint16_t* src,
|
||||
int i = height;
|
||||
#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \
|
||||
defined(HAS_TRANSPOSEWXH_16_HVX)
|
||||
void (*TransposeWxH_16)(const uint16_t* src, int src_stride, uint16_t* dst,
|
||||
int dst_stride, int width, int height) = NULL;
|
||||
void (*TransposeWxH_16_fn)(const uint16_t* src, int src_stride, uint16_t* dst,
|
||||
int dst_stride, int width, int height) = NULL;
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
||||
void (*TransposeWx32_16)(const uint16_t* src, int src_stride, uint16_t* dst,
|
||||
@ -695,17 +740,25 @@ static void TransposePlane_16(const uint16_t* src,
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_16_SME)
|
||||
if (TestCpuFlag(kCpuHasSME)) {
|
||||
TransposeWxH_16 = TransposeWxH_16_SME;
|
||||
TransposeWxH_16_fn = TransposeWxH_16_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_16_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV)) {
|
||||
TransposeWxH_16 = TransposeWxH_16_RVV;
|
||||
TransposeWxH_16_fn = TransposeWxH_16_RVV;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_16_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
TransposeWxH_16 = TransposeWxH_16_HVX;
|
||||
TransposeWxH_16_fn = TransposeWxH_16_HVX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX8_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
TransposeWx8_16 = TransposeWx8_16_Any_AVX2;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
TransposeWx8_16 = TransposeWx8_16_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX16_16_AVX2)
|
||||
@ -748,8 +801,8 @@ static void TransposePlane_16(const uint16_t* src,
|
||||
|
||||
#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \
|
||||
defined(HAS_TRANSPOSEWXH_16_HVX)
|
||||
if (TransposeWxH_16) {
|
||||
TransposeWxH_16(src, src_stride, dst, dst_stride, width, height);
|
||||
if (TransposeWxH_16_fn) {
|
||||
TransposeWxH_16_fn(src, src_stride, dst, dst_stride, width, height);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
@ -782,7 +835,7 @@ static void TransposePlane_16(const uint16_t* src,
|
||||
}
|
||||
|
||||
if (i > 0) {
|
||||
TransposeWxH_16_C(src, src_stride, dst, dst_stride, width, i);
|
||||
TransposeWxH_16(src, src_stride, dst, dst_stride, width, i);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -34,8 +34,8 @@ TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7)
|
||||
#ifdef HAS_TRANSPOSEWX16_NEON
|
||||
TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEWX8_SSSE3
|
||||
TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15)
|
||||
#ifdef HAS_TRANSPOSEWX8_AVX2
|
||||
TANY(TransposeWx8_Any_AVX2, TransposeWx8_AVX2, TransposeWx8_C, 15)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEWX16_AVX2
|
||||
TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15)
|
||||
@ -71,8 +71,8 @@ TANY(TransposeWx16_Any_WASMSIMD, TransposeWx16_WASMSIMD, TransposeWx16_C, 15)
|
||||
#ifdef HAS_TRANSPOSEUVWX8_NEON
|
||||
TUVANY(TransposeUVWx8_Any_NEON, TransposeUVWx8_NEON, TransposeUVWx8_C, 7)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEUVWX8_SSE2
|
||||
TUVANY(TransposeUVWx8_Any_SSE2, TransposeUVWx8_SSE2, TransposeUVWx8_C, 7)
|
||||
#ifdef HAS_TRANSPOSEUVWX8_AVX2
|
||||
TUVANY(TransposeUVWx8_Any_AVX2, TransposeUVWx8_AVX2, TransposeUVWx8_C, 7)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEUVWX16_AVX2
|
||||
TUVANY(TransposeUVWx16_Any_AVX2, TransposeUVWx16_AVX2, TransposeUVWx16_C, 7)
|
||||
@ -111,8 +111,8 @@ TUVANY(TransposeUVWx16_Any_WASMSIMD,
|
||||
} \
|
||||
}
|
||||
|
||||
#ifdef HAS_TRANSPOSEWX4_32_SSE2
|
||||
TANY32(TransposeWx4_32_Any_SSE2, TransposeWx4_32_SSE2, TransposeWxH_32_C, 3)
|
||||
#ifdef HAS_TRANSPOSEWX4_32_AVX2
|
||||
TANY32(TransposeWx4_32_Any_AVX2, TransposeWx4_32_AVX2, TransposeWxH_32_C, 3)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEWX4_32_LSX
|
||||
TANY32(TransposeWx4_32_Any_LSX, TransposeWx4_32_LSX, TransposeWxH_32_C, 3)
|
||||
@ -155,13 +155,20 @@ TANY32_SRC(TransposeWx8_32_Any_AVX2, TransposeWx8_32_AVX2, 3, 8)
|
||||
TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
TransposeWxH_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \
|
||||
r, HEIGHT); \
|
||||
TransposeWx8_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \
|
||||
r); \
|
||||
if ((HEIGHT) == 16) { \
|
||||
TransposeWx8_16_C(src + np + 8 * src_stride, src_stride, \
|
||||
dst + np * dst_stride + 8, dst_stride, r); \
|
||||
} \
|
||||
}
|
||||
|
||||
#ifdef HAS_TRANSPOSEWX8_16_NEON
|
||||
TANY16(TransposeWx8_16_Any_NEON, TransposeWx8_16_NEON, 7, 8)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEWX8_16_AVX2
|
||||
TANY16(TransposeWx8_16_Any_AVX2, TransposeWx8_16_AVX2, 7, 8)
|
||||
#endif
|
||||
#ifdef HAS_TRANSPOSEWX16_16_AVX2
|
||||
TANY16(TransposeWx16_16_Any_AVX2, TransposeWx16_16_AVX2, 7, 16)
|
||||
#endif
|
||||
|
||||
@ -31,7 +31,8 @@ static int ARGBTranspose(const uint8_t* src_argb,
|
||||
int width,
|
||||
int height) {
|
||||
#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX)
|
||||
defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst,
|
||||
int dst_stride, int width, int height) = NULL;
|
||||
#endif
|
||||
@ -49,11 +50,11 @@ static int ARGBTranspose(const uint8_t* src_argb,
|
||||
if (src_stride_argb & 3) {
|
||||
return -1;
|
||||
}
|
||||
#if defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
TransposeWx4_32 = TransposeWx4_32_Any_SSE2;
|
||||
#if defined(HAS_TRANSPOSEWX4_32_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
TransposeWx4_32 = TransposeWx4_32_Any_AVX2;
|
||||
if (IS_ALIGNED(height, 4)) { // Width of dest.
|
||||
TransposeWx4_32 = TransposeWx4_32_SSE2;
|
||||
TransposeWx4_32 = TransposeWx4_32_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -65,6 +66,11 @@ static int ARGBTranspose(const uint8_t* src_argb,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
TransposeWxH_32 = TransposeWxH_32_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
TransposeWx16_32 = TransposeWx16_32_AVX512BW;
|
||||
@ -123,14 +129,6 @@ static int ARGBTranspose(const uint8_t* src_argb,
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX)
|
||||
if (TransposeWxH_32) {
|
||||
TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
|
||||
height);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
||||
if (TransposeWx16_32) {
|
||||
while (height >= 16) {
|
||||
@ -142,6 +140,17 @@ static int ARGBTranspose(const uint8_t* src_argb,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \
|
||||
defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
if (TransposeWxH_32) {
|
||||
if (height > 0) {
|
||||
TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
|
||||
width, height);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
|
||||
if (TransposeWx8_32) {
|
||||
while (height >= 8) {
|
||||
|
||||
@ -90,39 +90,6 @@ void TransposeUVWx16_C(const uint8_t* src,
|
||||
dst_stride_a, (dst_b + 8), dst_stride_b, width);
|
||||
}
|
||||
|
||||
void TransposeWxH_C(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
int i;
|
||||
for (i = 0; i < width; ++i) {
|
||||
int j;
|
||||
for (j = 0; j < height; ++j) {
|
||||
dst[i * dst_stride + j] = src[j * src_stride + i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TransposeUVWxH_C(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst_a,
|
||||
int dst_stride_a,
|
||||
uint8_t* dst_b,
|
||||
int dst_stride_b,
|
||||
int width,
|
||||
int height) {
|
||||
int i;
|
||||
for (i = 0; i < width * 2; i += 2) {
|
||||
int j;
|
||||
for (j = 0; j < height; ++j) {
|
||||
dst_a[((i >> 1) * dst_stride_a) + j] = src[i + (j * src_stride)];
|
||||
dst_b[((i >> 1) * dst_stride_b) + j] = src[i + (j * src_stride) + 1];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TransposeWx8_16_C(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
@ -143,21 +110,6 @@ void TransposeWx8_16_C(const uint16_t* src,
|
||||
}
|
||||
}
|
||||
|
||||
void TransposeWxH_16_C(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
int i;
|
||||
for (i = 0; i < width; ++i) {
|
||||
int j;
|
||||
for (j = 0; j < height; ++j) {
|
||||
dst[i * dst_stride + j] = src[j * src_stride + i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void Copy4(uint8_t* dst, const uint8_t* src) {
|
||||
dst[0] = src[0];
|
||||
dst[1] = src[1];
|
||||
|
||||
1030
source/rotate_gcc.cc
1030
source/rotate_gcc.cc
File diff suppressed because it is too large
Load Diff
@ -11,16 +11,17 @@
|
||||
#include "libyuv/rotate_row.h"
|
||||
#include "libyuv/row.h"
|
||||
|
||||
// This module is for 32 bit x86 with any compiler, Visual C 64 bit, and for
|
||||
// clang with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__i386__) || defined(_M_IX86) || \
|
||||
((defined(__x86_64__) || defined(_M_X64)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))))
|
||||
// This module is for Visual C 32/64 bit, and for clang with
|
||||
// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer.
|
||||
#if !defined(LIBYUV_DISABLE_X86) && \
|
||||
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \
|
||||
defined(_M_IX86)) && \
|
||||
((defined(_MSC_VER) && !defined(__clang__)) || \
|
||||
defined(LIBYUV_ENABLE_ROWWIN))
|
||||
|
||||
#include <emmintrin.h>
|
||||
#include <immintrin.h> // For AVX2 and AVX512 intrinsics
|
||||
#include <string.h>
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
#include <intrin.h>
|
||||
#endif
|
||||
@ -33,22 +34,21 @@ extern "C" {
|
||||
// Target attributes allow gcc and clang to build each function for its
|
||||
// instruction set regardless of the baseline, which may lack SSE2 for 32 bit.
|
||||
#if defined(__clang__) || defined(__GNUC__)
|
||||
#define LIBYUV_TARGET_SSE2 __attribute__((target("sse2")))
|
||||
#define LIBYUV_TARGET_AVX2 __attribute__((target("avx2")))
|
||||
#define LIBYUV_TARGET_AVX512BW \
|
||||
__attribute__((target("avx512bw,avx512vl,avx512f")))
|
||||
#else
|
||||
#define LIBYUV_TARGET_SSE2
|
||||
#define LIBYUV_TARGET_AVX2
|
||||
#define LIBYUV_TARGET_AVX512BW
|
||||
#endif // defined(__clang__) || defined(__GNUC__)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2)
|
||||
// Transpose 8 rows x 16 bytes with SSE2. Each output holds 2 columns x 8 rows.
|
||||
// The low 8 bytes (even column) go to dst_a and the high 8 bytes (odd column)
|
||||
// go to dst_b. Width is a multiple of 16 bytes.
|
||||
LIBYUV_TARGET_SSE2
|
||||
static void TransposeWx8_Byte_SSE2(const uint8_t* src,
|
||||
#if defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2)
|
||||
// Transpose 8 rows x 16 bytes using 2 lanes of AVX2. Source row r is in
|
||||
// lane r / 4 of register r % 4. Each 128-bit output lane holds 2 columns x 8
|
||||
// rows: low 8 bytes (even column) for dst_a and high 8 bytes (odd column) for
|
||||
// dst_b. Width is a multiple of 16 bytes.
|
||||
LIBYUV_TARGET_AVX2
|
||||
static void TransposeWx8_Byte_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst_a,
|
||||
int dst_stride_a,
|
||||
@ -59,97 +59,101 @@ static void TransposeWx8_Byte_SSE2(const uint8_t* src,
|
||||
const ptrdiff_t sa = dst_stride_a;
|
||||
const ptrdiff_t sb = dst_stride_b;
|
||||
while (byte_width > 0) {
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
|
||||
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
|
||||
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
|
||||
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
|
||||
__m128i r4 = _mm_loadu_si128((const __m128i*)(src + s * 4));
|
||||
__m128i r5 = _mm_loadu_si128((const __m128i*)(src + s * 5));
|
||||
__m128i r6 = _mm_loadu_si128((const __m128i*)(src + s * 6));
|
||||
__m128i r7 = _mm_loadu_si128((const __m128i*)(src + s * 7));
|
||||
__m256i r0 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
|
||||
__m256i r1 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 5)), 1);
|
||||
__m256i r2 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 6)), 1);
|
||||
__m256i r3 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
|
||||
|
||||
// Bytes of row pairs. a0 is columns 0..7 of rows 0, 1.
|
||||
__m128i a0 = _mm_unpacklo_epi8(r0, r1);
|
||||
__m128i a1 = _mm_unpackhi_epi8(r0, r1);
|
||||
__m128i a2 = _mm_unpacklo_epi8(r2, r3);
|
||||
__m128i a3 = _mm_unpackhi_epi8(r2, r3);
|
||||
__m128i a4 = _mm_unpacklo_epi8(r4, r5);
|
||||
__m128i a5 = _mm_unpackhi_epi8(r4, r5);
|
||||
__m128i a6 = _mm_unpacklo_epi8(r6, r7);
|
||||
__m128i a7 = _mm_unpackhi_epi8(r6, r7);
|
||||
__m256i a0 = _mm256_unpacklo_epi8(r0, r1);
|
||||
__m256i a1 = _mm256_unpackhi_epi8(r0, r1);
|
||||
__m256i a2 = _mm256_unpacklo_epi8(r2, r3);
|
||||
__m256i a3 = _mm256_unpackhi_epi8(r2, r3);
|
||||
|
||||
// Words. b0 is columns 0..3 of rows 0..3.
|
||||
__m128i b0 = _mm_unpacklo_epi16(a0, a2);
|
||||
__m128i b1 = _mm_unpackhi_epi16(a0, a2);
|
||||
__m128i b2 = _mm_unpacklo_epi16(a1, a3);
|
||||
__m128i b3 = _mm_unpackhi_epi16(a1, a3);
|
||||
__m128i b4 = _mm_unpacklo_epi16(a4, a6);
|
||||
__m128i b5 = _mm_unpackhi_epi16(a4, a6);
|
||||
__m128i b6 = _mm_unpacklo_epi16(a5, a7);
|
||||
__m128i b7 = _mm_unpackhi_epi16(a5, a7);
|
||||
__m256i b0 = _mm256_unpacklo_epi16(a0, a2);
|
||||
__m256i b1 = _mm256_unpackhi_epi16(a0, a2);
|
||||
__m256i b2 = _mm256_unpacklo_epi16(a1, a3);
|
||||
__m256i b3 = _mm256_unpackhi_epi16(a1, a3);
|
||||
|
||||
// Dwords. c0 is columns 0, 1 of rows 0..7.
|
||||
__m128i c0 = _mm_unpacklo_epi32(b0, b4);
|
||||
__m128i c1 = _mm_unpackhi_epi32(b0, b4);
|
||||
__m128i c2 = _mm_unpacklo_epi32(b1, b5);
|
||||
__m128i c3 = _mm_unpackhi_epi32(b1, b5);
|
||||
__m128i c4 = _mm_unpacklo_epi32(b2, b6);
|
||||
__m128i c5 = _mm_unpackhi_epi32(b2, b6);
|
||||
__m128i c6 = _mm_unpacklo_epi32(b3, b7);
|
||||
__m128i c7 = _mm_unpackhi_epi32(b3, b7);
|
||||
__m256i t0 = _mm256_permute2x128_si256(b0, b1, 0x20);
|
||||
__m256i t1 = _mm256_permute2x128_si256(b0, b1, 0x31);
|
||||
__m256i t2 = _mm256_permute2x128_si256(b2, b3, 0x20);
|
||||
__m256i t3 = _mm256_permute2x128_si256(b2, b3, 0x31);
|
||||
|
||||
_mm_storel_epi64((__m128i*)(dst_a), c0);
|
||||
_mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(c0));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa), c1);
|
||||
_mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(c1));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 2), c2);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(c2));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 3), c3);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(c3));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 4), c4);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(c4));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 5), c5);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(c5));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 6), c6);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(c6));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 7), c7);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(c7));
|
||||
__m256i c0 = _mm256_unpacklo_epi32(t0, t1);
|
||||
__m256i c1 = _mm256_unpackhi_epi32(t0, t1);
|
||||
__m256i c2 = _mm256_unpacklo_epi32(t2, t3);
|
||||
__m256i c3 = _mm256_unpackhi_epi32(t2, t3);
|
||||
|
||||
__m128i lo0 = _mm256_castsi256_si128(c0);
|
||||
__m128i lo1 = _mm256_castsi256_si128(c1);
|
||||
__m128i hi0 = _mm256_extracti128_si256(c0, 1);
|
||||
__m128i hi1 = _mm256_extracti128_si256(c1, 1);
|
||||
__m128i lo2 = _mm256_castsi256_si128(c2);
|
||||
__m128i lo3 = _mm256_castsi256_si128(c3);
|
||||
__m128i hi2 = _mm256_extracti128_si256(c2, 1);
|
||||
__m128i hi3 = _mm256_extracti128_si256(c3, 1);
|
||||
|
||||
_mm_storel_epi64((__m128i*)(dst_a), lo0);
|
||||
_mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(lo0));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa), lo1);
|
||||
_mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(lo1));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 2), hi0);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(hi0));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 3), hi1);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(hi1));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 4), lo2);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(lo2));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 5), lo3);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(lo3));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 6), hi2);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(hi2));
|
||||
_mm_storel_epi64((__m128i*)(dst_a + sa * 7), hi3);
|
||||
_mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(hi3));
|
||||
|
||||
src += 16;
|
||||
dst_a += sa * 8;
|
||||
dst_b += sb * 8;
|
||||
byte_width -= 16;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2)
|
||||
#endif // defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX8_SSSE3)
|
||||
#if defined(HAS_TRANSPOSEWX8_AVX2)
|
||||
// Transpose 16x8. Width is a multiple of 16.
|
||||
LIBYUV_TARGET_SSE2
|
||||
void TransposeWx8_SSSE3(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
TransposeWx8_Byte_SSE2(src, src_stride, dst, dst_stride * 2, dst + dst_stride,
|
||||
LIBYUV_TARGET_AVX2
|
||||
void TransposeWx8_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
TransposeWx8_Byte_AVX2(src, src_stride, dst, dst_stride * 2, dst + dst_stride,
|
||||
dst_stride * 2, width);
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWX8_SSSE3)
|
||||
#endif // defined(HAS_TRANSPOSEWX8_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSEUVWX8_SSE2)
|
||||
#if defined(HAS_TRANSPOSEUVWX8_AVX2)
|
||||
// Transpose UV 8x8. Width is a multiple of 8.
|
||||
LIBYUV_TARGET_SSE2
|
||||
void TransposeUVWx8_SSE2(const uint8_t* src,
|
||||
LIBYUV_TARGET_AVX2
|
||||
void TransposeUVWx8_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst_a,
|
||||
int dst_stride_a,
|
||||
uint8_t* dst_b,
|
||||
int dst_stride_b,
|
||||
int width) {
|
||||
TransposeWx8_Byte_SSE2(src, src_stride, dst_a, dst_stride_a, dst_b,
|
||||
TransposeWx8_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b,
|
||||
dst_stride_b, width * 2);
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEUVWX8_SSE2)
|
||||
#endif // defined(HAS_TRANSPOSEUVWX8_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
|
||||
// Load row r into lane 0 and row r + 8 into lane 1.
|
||||
@ -417,6 +421,55 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src,
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX8_16_AVX2)
|
||||
LIBYUV_TARGET_AVX2
|
||||
void TransposeWx8_16_AVX2(const uint16_t* src,
|
||||
int src_stride,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (width > 0) {
|
||||
__m256i r0 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
|
||||
__m256i r1 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 5)), 1);
|
||||
__m256i r2 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 6)), 1);
|
||||
__m256i r3 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
|
||||
|
||||
__m256i t0 = _mm256_unpacklo_epi16(r0, r1);
|
||||
__m256i t1 = _mm256_unpackhi_epi16(r0, r1);
|
||||
__m256i t2 = _mm256_unpacklo_epi16(r2, r3);
|
||||
__m256i t3 = _mm256_unpackhi_epi16(r2, r3);
|
||||
|
||||
__m256i u0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(t0, t2), 0xd8);
|
||||
__m256i u1 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(t0, t2), 0xd8);
|
||||
__m256i u2 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(t1, t3), 0xd8);
|
||||
__m256i u3 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(t1, t3), 0xd8);
|
||||
|
||||
_mm_storeu_si128((__m128i*)(dst), _mm256_castsi256_si128(u0));
|
||||
_mm_storeu_si128((__m128i*)(dst + d), _mm256_extracti128_si256(u0, 1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 2), _mm256_castsi256_si128(u1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 3), _mm256_extracti128_si256(u1, 1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 4), _mm256_castsi256_si128(u2));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 5), _mm256_extracti128_si256(u2, 1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 6), _mm256_castsi256_si128(u3));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 7), _mm256_extracti128_si256(u3, 1));
|
||||
src += 8;
|
||||
dst += d * 8;
|
||||
width -= 8;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWX8_16_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX16_16_AVX2)
|
||||
LIBYUV_TARGET_AVX2
|
||||
void TransposeWx16_16_AVX2(const uint16_t* src,
|
||||
@ -709,35 +762,61 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src,
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
// Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row.
|
||||
LIBYUV_TARGET_SSE2
|
||||
void TransposeWx4_32_SSE2(const uint8_t* src,
|
||||
#if defined(HAS_TRANSPOSEWX4_32_AVX2)
|
||||
// Transpose 32 bit values (ARGB). Read a column of 4 pixels across 8 rows per
|
||||
// AVX2 iteration (with a 4-row 128-bit tail) and write 4 destination rows.
|
||||
LIBYUV_TARGET_AVX2
|
||||
void TransposeWx4_32_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
while (width > 0) {
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src)); // a b c d
|
||||
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); // e f g h
|
||||
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); // i j k l
|
||||
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); // m n o p
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1); // a e b f
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r3); // i m j n
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1); // c g d h
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r3); // k o l p
|
||||
_mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); // a e i m
|
||||
while (width >= 8) {
|
||||
__m256i r0 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
|
||||
__m256i r1 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 5)), 1);
|
||||
__m256i r2 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 6)), 1);
|
||||
__m256i r3 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))),
|
||||
_mm_loadu_si128((const __m128i*)(src + s * 7)), 1);
|
||||
|
||||
__m256i t0 = _mm256_unpacklo_epi32(r0, r1);
|
||||
__m256i t1 = _mm256_unpacklo_epi32(r2, r3);
|
||||
__m256i t2 = _mm256_unpackhi_epi32(r0, r1);
|
||||
__m256i t3 = _mm256_unpackhi_epi32(r2, r3);
|
||||
|
||||
_mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3));
|
||||
_mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3));
|
||||
src += s * 8;
|
||||
dst += 32;
|
||||
width -= 8;
|
||||
}
|
||||
if (width > 0) {
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
|
||||
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
|
||||
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
|
||||
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
|
||||
_mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3));
|
||||
_mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3));
|
||||
src += s * 4;
|
||||
dst += 16;
|
||||
width -= 4;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
#endif // defined(HAS_TRANSPOSEWX4_32_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX8_32_AVX2)
|
||||
LIBYUV_TARGET_AVX2
|
||||
@ -779,6 +858,241 @@ void TransposeWx8_32_AVX2(const uint8_t* src,
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWX8_32_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
// Load n (1..3) 32-bit pixels into low lanes of __m128i with zeroed upper
|
||||
// lanes (safe at buffer boundaries and MemorySanitizer-clean).
|
||||
LIBYUV_TARGET_AVX2
|
||||
static __m128i LoadPartialRow_32_AVX2(const uint8_t* p, int n) {
|
||||
if (n == 1) {
|
||||
int v;
|
||||
memcpy(&v, p, 4);
|
||||
return _mm_cvtsi32_si128(v);
|
||||
}
|
||||
if (n == 2) {
|
||||
return _mm_loadl_epi64((const __m128i*)p);
|
||||
}
|
||||
int v;
|
||||
memcpy(&v, p + 8, 4);
|
||||
return _mm_insert_epi32(_mm_loadl_epi64((const __m128i*)p), v, 2);
|
||||
}
|
||||
|
||||
// Transpose arbitrary WxH 32-bit plane using AVX2 (4x8, 8x4, 4x4, and 4x{1..3}
|
||||
// tiles plus exact remainder handling).
|
||||
LIBYUV_TARGET_AVX2
|
||||
void TransposeWxH_32_AVX2(const uint8_t* src,
|
||||
int src_stride,
|
||||
uint8_t* dst,
|
||||
int dst_stride,
|
||||
int width,
|
||||
int height) {
|
||||
const ptrdiff_t s = src_stride;
|
||||
const ptrdiff_t d = dst_stride;
|
||||
const int w4 = width & ~3;
|
||||
const int rem_w = width & 3;
|
||||
|
||||
// Tier 1: 8-row strips using 4x8 YMM tiles + rem_w x 8 YMM tail.
|
||||
while (height >= 8) {
|
||||
if (w4 > 0) {
|
||||
TransposeWx8_32_AVX2(src, src_stride, dst, dst_stride, w4);
|
||||
}
|
||||
if (rem_w > 0) {
|
||||
const uint8_t* src_rem = src + (ptrdiff_t)w4 * 4;
|
||||
uint8_t* dst_rem = dst + (ptrdiff_t)w4 * d;
|
||||
__m256i r0 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(LoadPartialRow_32_AVX2(src_rem, rem_w)),
|
||||
LoadPartialRow_32_AVX2(src_rem + s * 4, rem_w), 1);
|
||||
__m256i r1 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(LoadPartialRow_32_AVX2(src_rem + s, rem_w)),
|
||||
LoadPartialRow_32_AVX2(src_rem + s * 5, rem_w), 1);
|
||||
__m256i r2 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(
|
||||
LoadPartialRow_32_AVX2(src_rem + s * 2, rem_w)),
|
||||
LoadPartialRow_32_AVX2(src_rem + s * 6, rem_w), 1);
|
||||
__m256i r3 = _mm256_inserti128_si256(
|
||||
_mm256_castsi128_si256(
|
||||
LoadPartialRow_32_AVX2(src_rem + s * 3, rem_w)),
|
||||
LoadPartialRow_32_AVX2(src_rem + s * 7, rem_w), 1);
|
||||
|
||||
__m256i t0 = _mm256_unpacklo_epi32(r0, r1);
|
||||
__m256i t1 = _mm256_unpacklo_epi32(r2, r3);
|
||||
_mm256_storeu_si256((__m256i*)(dst_rem), _mm256_unpacklo_epi64(t0, t1));
|
||||
if (rem_w > 1) {
|
||||
_mm256_storeu_si256((__m256i*)(dst_rem + d),
|
||||
_mm256_unpackhi_epi64(t0, t1));
|
||||
}
|
||||
if (rem_w > 2) {
|
||||
__m256i t2 = _mm256_unpackhi_epi32(r0, r1);
|
||||
__m256i t3 = _mm256_unpackhi_epi32(r2, r3);
|
||||
_mm256_storeu_si256((__m256i*)(dst_rem + d * 2),
|
||||
_mm256_unpacklo_epi64(t2, t3));
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
src += 8 * s;
|
||||
dst += 32;
|
||||
height -= 8;
|
||||
}
|
||||
|
||||
// Tier 2: 4-row strip using 8x4 YMM tiles + 4x4 XMM block + rem_w x 4 tail.
|
||||
if (height >= 4) {
|
||||
const int w8 = width & ~7;
|
||||
if (w8 > 0) {
|
||||
const uint8_t* src_p = src;
|
||||
uint8_t* dst_p = dst;
|
||||
int cnt = w8;
|
||||
while (cnt > 0) {
|
||||
__m256i r0 = _mm256_loadu_si256((const __m256i*)(src_p));
|
||||
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_p + s));
|
||||
__m256i r2 = _mm256_loadu_si256((const __m256i*)(src_p + s * 2));
|
||||
__m256i r3 = _mm256_loadu_si256((const __m256i*)(src_p + s * 3));
|
||||
|
||||
__m256i t0 = _mm256_unpacklo_epi32(r0, r1);
|
||||
__m256i t1 = _mm256_unpacklo_epi32(r2, r3);
|
||||
__m256i t2 = _mm256_unpackhi_epi32(r0, r1);
|
||||
__m256i t3 = _mm256_unpackhi_epi32(r2, r3);
|
||||
|
||||
__m256i o0 = _mm256_unpacklo_epi64(t0, t1);
|
||||
__m256i o1 = _mm256_unpackhi_epi64(t0, t1);
|
||||
__m256i o2 = _mm256_unpacklo_epi64(t2, t3);
|
||||
__m256i o3 = _mm256_unpackhi_epi64(t2, t3);
|
||||
|
||||
_mm_storeu_si128((__m128i*)(dst_p), _mm256_castsi256_si128(o0));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d), _mm256_castsi256_si128(o1));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d * 2), _mm256_castsi256_si128(o2));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d * 3), _mm256_castsi256_si128(o3));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d * 4),
|
||||
_mm256_extracti128_si256(o0, 1));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d * 5),
|
||||
_mm256_extracti128_si256(o1, 1));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d * 6),
|
||||
_mm256_extracti128_si256(o2, 1));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d * 7),
|
||||
_mm256_extracti128_si256(o3, 1));
|
||||
|
||||
src_p += 32;
|
||||
dst_p += d * 8;
|
||||
cnt -= 8;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
if (width & 4) {
|
||||
const uint8_t* src_p = src + (ptrdiff_t)w8 * 4;
|
||||
uint8_t* dst_p = dst + (ptrdiff_t)w8 * d;
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src_p));
|
||||
__m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s));
|
||||
__m128i r2 = _mm_loadu_si128((const __m128i*)(src_p + s * 2));
|
||||
__m128i r3 = _mm_loadu_si128((const __m128i*)(src_p + s * 3));
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
|
||||
_mm_storeu_si128((__m128i*)(dst_p), _mm_unpacklo_epi64(t0, t1));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d), _mm_unpackhi_epi64(t0, t1));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d * 2), _mm_unpacklo_epi64(t2, t3));
|
||||
_mm_storeu_si128((__m128i*)(dst_p + d * 3), _mm_unpackhi_epi64(t2, t3));
|
||||
}
|
||||
if (rem_w > 0) {
|
||||
const uint8_t* src_rem = src + (ptrdiff_t)w4 * 4;
|
||||
uint8_t* dst_rem = dst + (ptrdiff_t)w4 * d;
|
||||
__m128i r0 = LoadPartialRow_32_AVX2(src_rem, rem_w);
|
||||
__m128i r1 = LoadPartialRow_32_AVX2(src_rem + s, rem_w);
|
||||
__m128i r2 = LoadPartialRow_32_AVX2(src_rem + s * 2, rem_w);
|
||||
__m128i r3 = LoadPartialRow_32_AVX2(src_rem + s * 3, rem_w);
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
|
||||
_mm_storeu_si128((__m128i*)(dst_rem), _mm_unpacklo_epi64(t0, t1));
|
||||
if (rem_w > 1) {
|
||||
_mm_storeu_si128((__m128i*)(dst_rem + d), _mm_unpackhi_epi64(t0, t1));
|
||||
}
|
||||
if (rem_w > 2) {
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
|
||||
_mm_storeu_si128((__m128i*)(dst_rem + d * 2),
|
||||
_mm_unpacklo_epi64(t2, t3));
|
||||
}
|
||||
}
|
||||
src += 4 * s;
|
||||
dst += 16;
|
||||
height -= 4;
|
||||
}
|
||||
|
||||
// Tier 3: 1..3 remaining source rows using 4 x height XMM tiles + corner.
|
||||
if (height > 0) {
|
||||
if (w4 > 0) {
|
||||
const uint8_t* src_p = src;
|
||||
uint8_t* dst_p = dst;
|
||||
int cnt = w4;
|
||||
if (height == 1) {
|
||||
while (cnt > 0) {
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src_p));
|
||||
int v0 = _mm_cvtsi128_si32(r0);
|
||||
int v1 = _mm_extract_epi32(r0, 1);
|
||||
int v2 = _mm_extract_epi32(r0, 2);
|
||||
int v3 = _mm_extract_epi32(r0, 3);
|
||||
memcpy(dst_p, &v0, 4);
|
||||
memcpy(dst_p + d, &v1, 4);
|
||||
memcpy(dst_p + d * 2, &v2, 4);
|
||||
memcpy(dst_p + d * 3, &v3, 4);
|
||||
src_p += 16;
|
||||
dst_p += d * 4;
|
||||
cnt -= 4;
|
||||
}
|
||||
} else if (height == 2) {
|
||||
while (cnt > 0) {
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src_p));
|
||||
__m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s));
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpackhi_epi32(r0, r1);
|
||||
_mm_storel_epi64((__m128i*)(dst_p), t0);
|
||||
_mm_storeh_pd((double*)(dst_p + d), _mm_castsi128_pd(t0));
|
||||
_mm_storel_epi64((__m128i*)(dst_p + d * 2), t1);
|
||||
_mm_storeh_pd((double*)(dst_p + d * 3), _mm_castsi128_pd(t1));
|
||||
src_p += 16;
|
||||
dst_p += d * 4;
|
||||
cnt -= 4;
|
||||
}
|
||||
} else {
|
||||
const __m128i zero = _mm_setzero_si128();
|
||||
while (cnt > 0) {
|
||||
__m128i r0 = _mm_loadu_si128((const __m128i*)(src_p));
|
||||
__m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s));
|
||||
__m128i r2 = _mm_loadu_si128((const __m128i*)(src_p + s * 2));
|
||||
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
|
||||
__m128i t1 = _mm_unpacklo_epi32(r2, zero);
|
||||
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
|
||||
__m128i t3 = _mm_unpackhi_epi32(r2, zero);
|
||||
__m128i o0 = _mm_unpacklo_epi64(t0, t1);
|
||||
__m128i o1 = _mm_unpackhi_epi64(t0, t1);
|
||||
__m128i o2 = _mm_unpacklo_epi64(t2, t3);
|
||||
__m128i o3 = _mm_unpackhi_epi64(t2, t3);
|
||||
int v0 = _mm_extract_epi32(o0, 2);
|
||||
int v1 = _mm_extract_epi32(o1, 2);
|
||||
int v2 = _mm_extract_epi32(o2, 2);
|
||||
int v3 = _mm_extract_epi32(o3, 2);
|
||||
_mm_storel_epi64((__m128i*)(dst_p), o0);
|
||||
memcpy(dst_p + 8, &v0, 4);
|
||||
_mm_storel_epi64((__m128i*)(dst_p + d), o1);
|
||||
memcpy(dst_p + d + 8, &v1, 4);
|
||||
_mm_storel_epi64((__m128i*)(dst_p + d * 2), o2);
|
||||
memcpy(dst_p + d * 2 + 8, &v2, 4);
|
||||
_mm_storel_epi64((__m128i*)(dst_p + d * 3), o3);
|
||||
memcpy(dst_p + d * 3 + 8, &v3, 4);
|
||||
src_p += 16;
|
||||
dst_p += d * 4;
|
||||
cnt -= 4;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int x = w4; x < width; ++x) {
|
||||
uint8_t* dst_row = dst + (ptrdiff_t)x * d;
|
||||
for (int y = 0; y < height; ++y) {
|
||||
memcpy(dst_row + y * 4, src + (ptrdiff_t)y * s + x * 4, 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
|
||||
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void TransposeWx16_32_AVX512BW(const uint8_t* src,
|
||||
|
||||
@ -277,7 +277,7 @@ ANY41PT(MergeARGB16To8Row_Any_NEON,
|
||||
const uint8_t* v_buf, uint8_t* dst_ptr, int width) { \
|
||||
SIMD_ALIGNED(uint8_t vin[64 * 3]); \
|
||||
static_assert(MASK + 1 <= sizeof(vin) / 3, "vin buffer too small"); \
|
||||
SIMD_ALIGNED(uint8_t vout[64]); \
|
||||
SIMD_ALIGNED(uint8_t vout[64 * 4]); \
|
||||
static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \
|
||||
"vout buffer too small"); \
|
||||
memset(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \
|
||||
@ -296,8 +296,8 @@ ANY41PT(MergeARGB16To8Row_Any_NEON,
|
||||
}
|
||||
|
||||
// Merge functions.
|
||||
#ifdef HAS_MERGERGBROW_SSSE3
|
||||
ANY31(MergeRGBRow_Any_SSSE3, MergeRGBRow_SSSE3, 0, 0, 3, 15)
|
||||
#ifdef HAS_MERGERGBROW_AVX2
|
||||
ANY31(MergeRGBRow_Any_AVX2, MergeRGBRow_AVX2, 0, 0, 3, 31)
|
||||
#endif
|
||||
#ifdef HAS_MERGERGBROW_NEON
|
||||
ANY31(MergeRGBRow_Any_NEON, MergeRGBRow_NEON, 0, 0, 3, 15)
|
||||
@ -769,8 +769,8 @@ ANY21(ARGBAlphaBlendRow_Any_LASX, ARGBAlphaBlendRow_LASX, 0, 4, 4, 4, 7)
|
||||
#ifdef HAS_ARGBALPHABLENDROW_WASMSIMD
|
||||
ANY21(ARGBAlphaBlendRow_Any_WASMSIMD, ARGBAlphaBlendRow_WASMSIMD, 0, 4, 4, 4, 3)
|
||||
#endif
|
||||
#ifdef HAS_SOBELROW_SSE2
|
||||
ANY21(SobelRow_Any_SSE2, SobelRow_SSE2, 0, 1, 1, 4, 15)
|
||||
#ifdef HAS_SOBELROW_AVX2
|
||||
ANY21(SobelRow_Any_AVX2, SobelRow_AVX2, 0, 1, 1, 4, 15)
|
||||
#endif
|
||||
#ifdef HAS_SOBELROW_NEON
|
||||
ANY21(SobelRow_Any_NEON, SobelRow_NEON, 0, 1, 1, 4, 7)
|
||||
@ -781,8 +781,8 @@ ANY21(SobelRow_Any_LSX, SobelRow_LSX, 0, 1, 1, 4, 7)
|
||||
#ifdef HAS_SOBELROW_LASX
|
||||
ANY21(SobelRow_Any_LASX, SobelRow_LASX, 0, 1, 1, 4, 15)
|
||||
#endif
|
||||
#ifdef HAS_SOBELTOPLANEROW_SSE2
|
||||
ANY21(SobelToPlaneRow_Any_SSE2, SobelToPlaneRow_SSE2, 0, 1, 1, 1, 15)
|
||||
#ifdef HAS_SOBELTOPLANEROW_AVX2
|
||||
ANY21(SobelToPlaneRow_Any_AVX2, SobelToPlaneRow_AVX2, 0, 1, 1, 1, 31)
|
||||
#endif
|
||||
#ifdef HAS_SOBELTOPLANEROW_NEON
|
||||
ANY21(SobelToPlaneRow_Any_NEON, SobelToPlaneRow_NEON, 0, 1, 1, 1, 15)
|
||||
@ -793,8 +793,8 @@ ANY21(SobelToPlaneRow_Any_LSX, SobelToPlaneRow_LSX, 0, 1, 1, 1, 15)
|
||||
#ifdef HAS_SOBELTOPLANEROW_LASX
|
||||
ANY21(SobelToPlaneRow_Any_LASX, SobelToPlaneRow_LASX, 0, 1, 1, 1, 31)
|
||||
#endif
|
||||
#ifdef HAS_SOBELXYROW_SSE2
|
||||
ANY21(SobelXYRow_Any_SSE2, SobelXYRow_SSE2, 0, 1, 1, 4, 15)
|
||||
#ifdef HAS_SOBELXYROW_AVX2
|
||||
ANY21(SobelXYRow_Any_AVX2, SobelXYRow_AVX2, 0, 1, 1, 4, 15)
|
||||
#endif
|
||||
#ifdef HAS_SOBELXYROW_NEON
|
||||
ANY21(SobelXYRow_Any_NEON, SobelXYRow_NEON, 0, 1, 1, 4, 7)
|
||||
@ -1114,6 +1114,15 @@ ANY11(AR30ToABGRRow_Any_NEON, AR30ToABGRRow_NEON, 0, 4, 4, 7)
|
||||
#if defined(HAS_AR30TOAB30ROW_NEON)
|
||||
ANY11(AR30ToAB30Row_Any_NEON, AR30ToAB30Row_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_AVX2)
|
||||
ANY11(AR30ToARGBRow_Any_AVX2, AR30ToARGBRow_AVX2, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_AR30TOABGRROW_AVX2)
|
||||
ANY11(AR30ToABGRRow_Any_AVX2, AR30ToABGRRow_AVX2, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_AR30TOAB30ROW_AVX2)
|
||||
ANY11(AR30ToAB30Row_Any_AVX2, AR30ToAB30Row_AVX2, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOAR30ROW_SSSE3)
|
||||
ANY11(ARGBToAR30Row_Any_SSSE3, ARGBToAR30Row_SSSE3, 0, 4, 4, 3)
|
||||
#endif
|
||||
@ -1141,8 +1150,8 @@ ANY11(RAWToARGBRow_Any_AVX2, RAWToARGBRow_AVX2, 0, 3, 4, 31)
|
||||
#if defined(HAS_RGB24TOARGBROW_AVX2)
|
||||
ANY11(RGB24ToARGBRow_Any_AVX2, RGB24ToARGBRow_AVX2, 0, 3, 4, 31)
|
||||
#endif
|
||||
#if defined(HAS_RAWTORGBAROW_SSSE3)
|
||||
ANY11(RAWToRGBARow_Any_SSSE3, RAWToRGBARow_SSSE3, 0, 3, 4, 15)
|
||||
#if defined(HAS_RAWTORGBAROW_AVX2)
|
||||
ANY11(RAWToRGBARow_Any_AVX2, RAWToRGBARow_AVX2, 0, 3, 4, 15)
|
||||
#endif
|
||||
#if defined(HAS_RAWTORGB24ROW_SSSE3)
|
||||
ANY11(RAWToRGB24Row_Any_SSSE3, RAWToRGB24Row_SSSE3, 0, 3, 3, 7)
|
||||
@ -1217,6 +1226,15 @@ ANY11(YUY2ToYRow_Any_SSE2, YUY2ToYRow_SSE2, 1, 4, 1, 15)
|
||||
ANY11(UYVYToYRow_Any_SSE2, UYVYToYRow_SSE2, 1, 4, 1, 15)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_RGB565TOYROW_AVX2
|
||||
ANY11(RGB565ToYRow_Any_AVX2, RGB565ToYRow_AVX2, 0, 2, 1, 31)
|
||||
#endif
|
||||
#ifdef HAS_ARGB1555TOYROW_AVX2
|
||||
ANY11(ARGB1555ToYRow_Any_AVX2, ARGB1555ToYRow_AVX2, 0, 2, 1, 31)
|
||||
#endif
|
||||
#ifdef HAS_ARGB4444TOYROW_AVX2
|
||||
ANY11(ARGB4444ToYRow_Any_AVX2, ARGB4444ToYRow_AVX2, 0, 2, 1, 31)
|
||||
#endif
|
||||
#ifdef HAS_RGB565TOYROW_NEON
|
||||
ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 15)
|
||||
#endif
|
||||
@ -1264,6 +1282,9 @@ ANY11(UYVYToYRow_Any_LSX, UYVYToYRow_LSX, 1, 4, 1, 15)
|
||||
#ifdef HAS_UYVYTOYROW_LASX
|
||||
ANY11(UYVYToYRow_Any_LASX, UYVYToYRow_LASX, 1, 4, 1, 31)
|
||||
#endif
|
||||
#ifdef HAS_AYUVTOYROW_AVX2
|
||||
ANY11(AYUVToYRow_Any_AVX2, AYUVToYRow_AVX2, 0, 4, 1, 15)
|
||||
#endif
|
||||
#ifdef HAS_AYUVTOYROW_NEON
|
||||
ANY11(AYUVToYRow_Any_NEON, AYUVToYRow_NEON, 0, 4, 1, 15)
|
||||
#endif
|
||||
@ -1670,6 +1691,9 @@ ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
|
||||
ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_COPYROW_16_AVX2
|
||||
ANY11T(CopyRow_16_Any_AVX2, CopyRow_16_AVX2, 2, 2, uint16_t, uint16_t, 31)
|
||||
#endif
|
||||
#ifdef HAS_COPYROW_16_NEON
|
||||
ANY11T(CopyRow_16_Any_NEON, CopyRow_16_NEON, 2, 2, uint16_t, uint16_t, 15)
|
||||
#endif
|
||||
@ -1898,6 +1922,9 @@ ANY11P16(HalfFloat1Row_Any_F16C,
|
||||
#ifdef HAS_HALFFLOATROW_NEON
|
||||
ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 15)
|
||||
#endif
|
||||
#ifdef HAS_BYTETOFLOATROW_AVX2
|
||||
ANY11P16(ByteToFloatRow_Any_AVX2, ByteToFloatRow_AVX2, uint8_t, float, 1, 4, 15)
|
||||
#endif
|
||||
#ifdef HAS_BYTETOFLOATROW_NEON
|
||||
ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 4, 7)
|
||||
#endif
|
||||
@ -2031,6 +2058,39 @@ ANY11I(InterpolateRow_16_Any_WASMSIMD,
|
||||
#endif
|
||||
#undef ANY11I
|
||||
|
||||
// Any 1 to 1 half row
|
||||
#define ANY11H(NAMEANY, ANY_SIMD, T, MASK) \
|
||||
void NAMEANY(const T* src_ptr, ptrdiff_t src_stride, T* dst_ptr, \
|
||||
int width) { \
|
||||
SIMD_ALIGNED(T vin[64 * 2]); \
|
||||
static_assert((MASK + 1) * sizeof(T) <= sizeof(vin) / 2, \
|
||||
"vin buffer too small"); \
|
||||
SIMD_ALIGNED(T vout[64]); \
|
||||
static_assert((MASK + 1) * sizeof(T) <= sizeof(vout), \
|
||||
"vout buffer too small"); \
|
||||
MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
|
||||
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
|
||||
int r = width & MASK; \
|
||||
int n = width & ~MASK; \
|
||||
if (n > 0) { \
|
||||
ANY_SIMD(src_ptr, src_stride, dst_ptr, n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
memcpy(vin, src_ptr + np, r * sizeof(T)); \
|
||||
memcpy(vin + 64, src_ptr + src_stride + np, r * sizeof(T)); \
|
||||
ANY_SIMD(vin, 64, vout, MASK + 1); \
|
||||
memcpy(dst_ptr + np, vout, r * sizeof(T)); \
|
||||
}
|
||||
|
||||
#ifdef HAS_HALFROW_AVX2
|
||||
ANY11H(HalfRow_Any_AVX2, HalfRow_AVX2, uint8_t, 31)
|
||||
#endif
|
||||
#ifdef HAS_HALFROW_16_AVX2
|
||||
ANY11H(HalfRow_16_Any_AVX2, HalfRow_16_AVX2, uint16_t, 15)
|
||||
#endif
|
||||
|
||||
#undef ANY11H
|
||||
|
||||
// Any 1 to 1 half row with scale param
|
||||
#define ANY11HS(NAMEANY, ANY_SIMD, TS, TD, MASK) \
|
||||
void NAMEANY(const TS* src_ptr, ptrdiff_t src_stride, TD* dst_ptr, \
|
||||
@ -2149,9 +2209,6 @@ ANY11HWS(HalfWidthRow_16To8_Any_NEON,
|
||||
memcpy(dst_ptr + np * BPP, vout + (MASK + 1 - r) * BPP, r * BPP); \
|
||||
}
|
||||
|
||||
#ifdef HAS_MIRRORROW_AVX512BW
|
||||
ANY11M(MirrorRow_Any_AVX512BW, MirrorRow_AVX512BW, 1, 63)
|
||||
#endif
|
||||
#ifdef HAS_MIRRORROW_AVX2
|
||||
ANY11M(MirrorRow_Any_AVX2, MirrorRow_AVX2, 1, 31)
|
||||
#endif
|
||||
@ -2167,9 +2224,6 @@ ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 15)
|
||||
#ifdef HAS_MIRRORROW_LASX
|
||||
ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 31)
|
||||
#endif
|
||||
#ifdef HAS_MIRRORUVROW_AVX512BW
|
||||
ANY11M(MirrorUVRow_Any_AVX512BW, MirrorUVRow_AVX512BW, 2, 31)
|
||||
#endif
|
||||
#ifdef HAS_MIRRORUVROW_AVX2
|
||||
ANY11M(MirrorUVRow_Any_AVX2, MirrorUVRow_AVX2, 2, 15)
|
||||
#endif
|
||||
@ -2264,8 +2318,8 @@ ANY11MT(MirrorRow_16_Any_NEON, MirrorRow_16_NEON, uint16_t, 2, 15)
|
||||
memcpy(dst_ptr + np * BPP, vout, r * BPP); \
|
||||
}
|
||||
|
||||
#ifdef HAS_SETROW_X86
|
||||
ANY1(SetRow_Any_X86, SetRow_X86, uint8_t, 1, 3)
|
||||
#ifdef HAS_SETROW_AVX2
|
||||
ANY1(SetRow_Any_AVX2, SetRow_AVX2, uint8_t, 1, 31)
|
||||
#endif
|
||||
#ifdef HAS_SETROW_NEON
|
||||
ANY1(SetRow_Any_NEON, SetRow_NEON, uint8_t, 1, 15)
|
||||
@ -2276,6 +2330,9 @@ ANY1(SetRow_Any_LSX, SetRow_LSX, uint8_t, 1, 15)
|
||||
#ifdef HAS_SETROW_LASX
|
||||
ANY1(SetRow_Any_LASX, SetRow_LASX, uint8_t, 1, 31)
|
||||
#endif
|
||||
#ifdef HAS_ARGBSETROW_AVX2
|
||||
ANY1(ARGBSetRow_Any_AVX2, ARGBSetRow_AVX2, uint32_t, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_ARGBSETROW_NEON
|
||||
ANY1(ARGBSetRow_Any_NEON, ARGBSetRow_NEON, uint32_t, 4, 3)
|
||||
#endif
|
||||
@ -2788,6 +2845,15 @@ ANY12S(RAWToUVRow_Any_LSX, RAWToUVRow_LSX, 0, 3, 15)
|
||||
#ifdef HAS_RAWTOUVROW_LASX
|
||||
ANY12S(RAWToUVRow_Any_LASX, RAWToUVRow_LASX, 0, 3, 31)
|
||||
#endif
|
||||
#ifdef HAS_RGB565TOUVROW_AVX2
|
||||
ANY12S(RGB565ToUVRow_Any_AVX2, RGB565ToUVRow_AVX2, 0, 2, 31)
|
||||
#endif
|
||||
#ifdef HAS_ARGB1555TOUVROW_AVX2
|
||||
ANY12S(ARGB1555ToUVRow_Any_AVX2, ARGB1555ToUVRow_AVX2, 0, 2, 31)
|
||||
#endif
|
||||
#ifdef HAS_ARGB4444TOUVROW_AVX2
|
||||
ANY12S(ARGB4444ToUVRow_Any_AVX2, ARGB4444ToUVRow_AVX2, 0, 2, 31)
|
||||
#endif
|
||||
#ifdef HAS_RGB565TOUVROW_NEON
|
||||
ANY12S(RGB565ToUVRow_Any_NEON, RGB565ToUVRow_NEON, 0, 2, 15)
|
||||
#endif
|
||||
@ -2861,6 +2927,12 @@ ANY12S(UYVYToUVRow_Any_LASX, UYVYToUVRow_LASX, 1, 4, 31)
|
||||
memcpy(dst_vu + (np >> 1) * 2, vout, SS(r, 1) * 2); \
|
||||
}
|
||||
|
||||
#ifdef HAS_AYUVTOUVROW_AVX2
|
||||
ANY11S(AYUVToUVRow_Any_AVX2, AYUVToUVRow_AVX2, 0, 4, 15)
|
||||
#endif
|
||||
#ifdef HAS_AYUVTOVUROW_AVX2
|
||||
ANY11S(AYUVToVURow_Any_AVX2, AYUVToVURow_AVX2, 0, 4, 15)
|
||||
#endif
|
||||
#ifdef HAS_AYUVTOVUROW_NEON
|
||||
ANY11S(AYUVToUVRow_Any_NEON, AYUVToUVRow_NEON, 0, 4, 15)
|
||||
ANY11S(AYUVToVURow_Any_NEON, AYUVToVURow_NEON, 0, 4, 15)
|
||||
@ -2895,17 +2967,14 @@ ANY11S(AYUVToVURow_Any_SVE2, AYUVToVURow_SVE2, 0, 4, 1)
|
||||
#ifdef HAS_DETILEROW_NEON
|
||||
ANYDETILE(DetileRow_Any_NEON, DetileRow_NEON, uint8_t, 1, 15)
|
||||
#endif
|
||||
#ifdef HAS_DETILEROW_SSE2
|
||||
ANYDETILE(DetileRow_Any_SSE2, DetileRow_SSE2, uint8_t, 1, 15)
|
||||
#ifdef HAS_DETILEROW_AVX2
|
||||
ANYDETILE(DetileRow_Any_AVX2, DetileRow_AVX2, uint8_t, 1, 15)
|
||||
#endif
|
||||
#ifdef HAS_DETILEROW_16_NEON
|
||||
ANYDETILE(DetileRow_16_Any_NEON, DetileRow_16_NEON, uint16_t, 2, 15)
|
||||
#endif
|
||||
#ifdef HAS_DETILEROW_16_SSE2
|
||||
ANYDETILE(DetileRow_16_Any_SSE2, DetileRow_16_SSE2, uint16_t, 2, 15)
|
||||
#endif
|
||||
#ifdef HAS_DETILEROW_16_AVX
|
||||
ANYDETILE(DetileRow_16_Any_AVX, DetileRow_16_AVX, uint16_t, 2, 15)
|
||||
#ifdef HAS_DETILEROW_16_AVX2
|
||||
ANYDETILE(DetileRow_16_Any_AVX2, DetileRow_16_AVX2, uint16_t, 2, 15)
|
||||
#endif
|
||||
|
||||
// DetileSplitUVRow width is in bytes
|
||||
@ -2934,17 +3003,17 @@ ANYDETILE(DetileRow_16_Any_AVX, DetileRow_16_AVX, uint16_t, 2, 15)
|
||||
#ifdef HAS_DETILESPLITUVROW_NEON
|
||||
ANYDETILESPLITUV(DetileSplitUVRow_Any_NEON, DetileSplitUVRow_NEON, 15)
|
||||
#endif
|
||||
#ifdef HAS_DETILESPLITUVROW_SSSE3
|
||||
ANYDETILESPLITUV(DetileSplitUVRow_Any_SSSE3, DetileSplitUVRow_SSSE3, 15)
|
||||
#ifdef HAS_DETILESPLITUVROW_AVX2
|
||||
ANYDETILESPLITUV(DetileSplitUVRow_Any_AVX2, DetileSplitUVRow_AVX2, 15)
|
||||
#endif
|
||||
|
||||
#define ANYDETILEMERGE(NAMEANY, ANY_SIMD, MASK) \
|
||||
void NAMEANY(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, \
|
||||
const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, \
|
||||
uint8_t* dst_yuy2, int width) { \
|
||||
SIMD_ALIGNED(uint8_t vin[16 * 2]); \
|
||||
SIMD_ALIGNED(uint8_t vin[32 * 2]); \
|
||||
static_assert(MASK + 1 <= sizeof(vin) / 2, "vin buffer too small"); \
|
||||
SIMD_ALIGNED(uint8_t vout[16 * 2]); \
|
||||
SIMD_ALIGNED(uint8_t vout[32 * 2]); \
|
||||
static_assert(2 * (MASK + 1) <= sizeof(vout), "vout buffer too small"); \
|
||||
MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
|
||||
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
|
||||
@ -2955,9 +3024,14 @@ ANYDETILESPLITUV(DetileSplitUVRow_Any_SSSE3, DetileSplitUVRow_SSSE3, 15)
|
||||
n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
memcpy(vin, src_y + (np / 16) * src_y_tile_stride, r); \
|
||||
memcpy(vin + 16, src_uv + (np / 16) * src_uv_tile_stride, r); \
|
||||
ANY_SIMD(vin, src_y_tile_stride, vin + 16, src_uv_tile_stride, vout, r); \
|
||||
int r0 = (r < 16) ? r : 16; \
|
||||
memcpy(vin, src_y + (np / 16) * src_y_tile_stride, r0); \
|
||||
memcpy(vin + 32, src_uv + (np / 16) * src_uv_tile_stride, r0); \
|
||||
if (r > 16) { \
|
||||
memcpy(vin + 16, src_y + (np / 16 + 1) * src_y_tile_stride, r - 16); \
|
||||
memcpy(vin + 48, src_uv + (np / 16 + 1) * src_uv_tile_stride, r - 16); \
|
||||
} \
|
||||
ANY_SIMD(vin, 16, vin + 32, 16, vout, MASK + 1); \
|
||||
memcpy(dst_yuy2 + 2 * np, vout, 2 * r); \
|
||||
}
|
||||
|
||||
@ -2965,10 +3039,12 @@ ANYDETILESPLITUV(DetileSplitUVRow_Any_SSSE3, DetileSplitUVRow_SSSE3, 15)
|
||||
ANYDETILEMERGE(DetileToYUY2_Any_NEON, DetileToYUY2_NEON, 15)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_DETILETOYUY2_SSE2
|
||||
ANYDETILEMERGE(DetileToYUY2_Any_SSE2, DetileToYUY2_SSE2, 15)
|
||||
#ifdef HAS_DETILETOYUY2_AVX2
|
||||
ANYDETILEMERGE(DetileToYUY2_Any_AVX2, DetileToYUY2_AVX2, 31)
|
||||
#endif
|
||||
|
||||
#undef ANYDETILEMERGE
|
||||
|
||||
#undef MSAN_MEMSET
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -3643,20 +3643,20 @@ void ARGBAffineRow_C(const uint8_t* src_argb,
|
||||
}
|
||||
|
||||
// Blend 2 rows into 1.
|
||||
static void HalfRow_C(const uint8_t* src_uv,
|
||||
ptrdiff_t src_uv_stride,
|
||||
uint8_t* dst_uv,
|
||||
int width) {
|
||||
void HalfRow_C(const uint8_t* src_uv,
|
||||
ptrdiff_t src_uv_stride,
|
||||
uint8_t* dst_uv,
|
||||
int width) {
|
||||
int x;
|
||||
for (x = 0; x < width; ++x) {
|
||||
dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1;
|
||||
}
|
||||
}
|
||||
|
||||
static void HalfRow_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_uv_stride,
|
||||
uint16_t* dst_uv,
|
||||
int width) {
|
||||
void HalfRow_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_uv_stride,
|
||||
uint16_t* dst_uv,
|
||||
int width) {
|
||||
int x;
|
||||
for (x = 0; x < width; ++x) {
|
||||
dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1;
|
||||
@ -4197,6 +4197,77 @@ void NV21ToRGB24Row_AVX2(const uint8_t* src_y,
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTORGB565ROW_AVX512BW)
|
||||
void I422ToRGB565Row_AVX512BW(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_rgb565,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
|
||||
while (width > 0) {
|
||||
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
|
||||
I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
|
||||
ARGBToRGB565Row_AVX512BW(row, dst_rgb565, twidth);
|
||||
src_y += twidth;
|
||||
src_u += twidth / 2;
|
||||
src_v += twidth / 2;
|
||||
dst_rgb565 += twidth * 2;
|
||||
width -= twidth;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTOARGB1555ROW_AVX2)
|
||||
void I422ToARGB1555Row_AVX512BW(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_argb1555,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
|
||||
while (width > 0) {
|
||||
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
|
||||
I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
|
||||
if (IS_ALIGNED(twidth, 8)) {
|
||||
ARGBToARGB1555Row_AVX2(row, dst_argb1555, twidth);
|
||||
} else {
|
||||
ARGBToARGB1555Row_Any_AVX2(row, dst_argb1555, twidth);
|
||||
}
|
||||
src_y += twidth;
|
||||
src_u += twidth / 2;
|
||||
src_v += twidth / 2;
|
||||
dst_argb1555 += twidth * 2;
|
||||
width -= twidth;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTOARGB4444ROW_AVX2)
|
||||
void I422ToARGB4444Row_AVX512BW(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_argb4444,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
|
||||
while (width > 0) {
|
||||
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
|
||||
I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth);
|
||||
if (IS_ALIGNED(twidth, 8)) {
|
||||
ARGBToARGB4444Row_AVX2(row, dst_argb4444, twidth);
|
||||
} else {
|
||||
ARGBToARGB4444Row_Any_AVX2(row, dst_argb4444, twidth);
|
||||
}
|
||||
src_y += twidth;
|
||||
src_u += twidth / 2;
|
||||
src_v += twidth / 2;
|
||||
dst_argb4444 += twidth * 2;
|
||||
width -= twidth;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_I422TOARGBROW_AVX2) && defined(HAS_ARGBTORGB565ROW_AVX2)
|
||||
void I422ToRGB565Row_AVX2(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
@ -5056,6 +5127,61 @@ void ARGB4444ToUVMatrixRow_NEON(const uint8_t* src_argb4444,
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_RGB565TOYROW_AVX2) && defined(HAS_RGB565TOYMATRIXROW_AVX2)
|
||||
void RGB565ToYRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
|
||||
RGB565ToYMatrixRow_AVX2(src_rgb565, dst_y, width, &kArgbI601Constants);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_ARGB1555TOYROW_AVX2) && defined(HAS_ARGB1555TOYMATRIXROW_AVX2)
|
||||
void ARGB1555ToYRow_AVX2(const uint8_t* src_argb1555,
|
||||
uint8_t* dst_y,
|
||||
int width) {
|
||||
ARGB1555ToYMatrixRow_AVX2(src_argb1555, dst_y, width, &kArgbI601Constants);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_ARGB4444TOYROW_AVX2) && defined(HAS_ARGB4444TOYMATRIXROW_AVX2)
|
||||
void ARGB4444ToYRow_AVX2(const uint8_t* src_argb4444,
|
||||
uint8_t* dst_y,
|
||||
int width) {
|
||||
ARGB4444ToYMatrixRow_AVX2(src_argb4444, dst_y, width, &kArgbI601Constants);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_RGB565TOUVROW_AVX2) && defined(HAS_RGB565TOUVMATRIXROW_AVX2)
|
||||
void RGB565ToUVRow_AVX2(const uint8_t* src_rgb565,
|
||||
int src_stride_rgb565,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width) {
|
||||
RGB565ToUVMatrixRow_AVX2(src_rgb565, src_stride_rgb565, dst_u, dst_v, width,
|
||||
&kArgbI601Constants);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_ARGB1555TOUVROW_AVX2) && defined(HAS_ARGB1555TOUVMATRIXROW_AVX2)
|
||||
void ARGB1555ToUVRow_AVX2(const uint8_t* src_argb1555,
|
||||
int src_stride_argb1555,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width) {
|
||||
ARGB1555ToUVMatrixRow_AVX2(src_argb1555, src_stride_argb1555, dst_u, dst_v,
|
||||
width, &kArgbI601Constants);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_ARGB4444TOUVROW_AVX2) && defined(HAS_ARGB4444TOUVMATRIXROW_AVX2)
|
||||
void ARGB4444ToUVRow_AVX2(const uint8_t* src_argb4444,
|
||||
int src_stride_argb4444,
|
||||
uint8_t* dst_u,
|
||||
uint8_t* dst_v,
|
||||
int width) {
|
||||
ARGB4444ToUVMatrixRow_AVX2(src_argb4444, src_stride_argb4444, dst_u, dst_v,
|
||||
width, &kArgbI601Constants);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
5902
source/row_gcc.cc
5902
source/row_gcc.cc
File diff suppressed because it is too large
Load Diff
4997
source/row_win.cc
4997
source/row_win.cc
File diff suppressed because it is too large
Load Diff
207
source/scale.cc
207
source/scale.cc
@ -206,6 +206,20 @@ static void ScalePlaneDown2_16(int src_width,
|
||||
src_stride = 0;
|
||||
}
|
||||
|
||||
#if defined(HAS_SCALEROWDOWN2_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ScaleRowDown2 =
|
||||
filtering == kFilterNone
|
||||
? ScaleRowDown2_16_Any_AVX2
|
||||
: (filtering == kFilterLinear ? ScaleRowDown2Linear_16_Any_AVX2
|
||||
: ScaleRowDown2Box_16_Any_AVX2);
|
||||
if (IS_ALIGNED(dst_width, 16)) {
|
||||
ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_AVX2
|
||||
: filtering == kFilterLinear ? ScaleRowDown2Linear_16_AVX2
|
||||
: ScaleRowDown2Box_16_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEROWDOWN2_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 16)) {
|
||||
ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_NEON
|
||||
@ -220,15 +234,6 @@ static void ScalePlaneDown2_16(int src_width,
|
||||
: ScaleRowDown2Box_16_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEROWDOWN2_16_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 16)) {
|
||||
ScaleRowDown2 =
|
||||
filtering == kFilterNone
|
||||
? ScaleRowDown2_16_SSE2
|
||||
: (filtering == kFilterLinear ? ScaleRowDown2Linear_16_SSE2
|
||||
: ScaleRowDown2Box_16_SSE2);
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEROWDOWN2_16_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_HVX
|
||||
@ -360,18 +365,6 @@ static void ScalePlaneDown4_16(int src_width,
|
||||
src_ptr += src_stride * 2; // Point to row 2.
|
||||
src_stride = 0;
|
||||
}
|
||||
#if defined(HAS_SCALEROWDOWN4_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleRowDown4 =
|
||||
filtering ? ScaleRowDown4Box_16_NEON : ScaleRowDown4_16_NEON;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEROWDOWN4_16_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleRowDown4 =
|
||||
filtering ? ScaleRowDown4Box_16_SSE2 : ScaleRowDown4_16_SSE2;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (filtering == kFilterLinear) {
|
||||
src_stride = 0;
|
||||
@ -455,23 +448,23 @@ static void ScalePlaneDown34(int src_width,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEROWDOWN34_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
#if defined(HAS_SCALEROWDOWN34_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
if (dst_width % 24 == 0) {
|
||||
if (!filtering) {
|
||||
ScaleRowDown34_0 = ScaleRowDown34_SSSE3;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_SSSE3;
|
||||
ScaleRowDown34_0 = ScaleRowDown34_AVX2;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_AVX2;
|
||||
} else {
|
||||
ScaleRowDown34_0 = ScaleRowDown34_0_Box_SSSE3;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_1_Box_SSSE3;
|
||||
ScaleRowDown34_0 = ScaleRowDown34_0_Box_AVX2;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_1_Box_AVX2;
|
||||
}
|
||||
} else {
|
||||
if (!filtering) {
|
||||
ScaleRowDown34_0 = ScaleRowDown34_Any_SSSE3;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_Any_SSSE3;
|
||||
ScaleRowDown34_0 = ScaleRowDown34_Any_AVX2;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_Any_AVX2;
|
||||
} else {
|
||||
ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_SSSE3;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_SSSE3;
|
||||
ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_AVX2;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_AVX2;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -557,28 +550,6 @@ static void ScalePlaneDown34_16(int src_width,
|
||||
ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_C;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_C;
|
||||
}
|
||||
#if defined(HAS_SCALEROWDOWN34_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON) && (dst_width % 24 == 0)) {
|
||||
if (!filtering) {
|
||||
ScaleRowDown34_0 = ScaleRowDown34_16_NEON;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_16_NEON;
|
||||
} else {
|
||||
ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_NEON;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEROWDOWN34_16_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) {
|
||||
if (!filtering) {
|
||||
ScaleRowDown34_0 = ScaleRowDown34_16_SSSE3;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_16_SSSE3;
|
||||
} else {
|
||||
ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_SSSE3;
|
||||
ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < dst_height - 2; y += 3) {
|
||||
ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width);
|
||||
@ -664,22 +635,23 @@ static void ScalePlaneDown38(int src_width,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEROWDOWN38_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
#if defined(HAS_SCALEROWDOWN38_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
if (!filtering) {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_Any_SSSE3;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_Any_SSSE3;
|
||||
ScaleRowDown38_3 = ScaleRowDown38_Any_AVX2;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_Any_AVX2;
|
||||
} else {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_SSSE3;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_SSSE3;
|
||||
ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_AVX2;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_AVX2;
|
||||
}
|
||||
if (dst_width % 12 == 0 && !filtering) {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_SSSE3;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_SSSE3;
|
||||
}
|
||||
if (dst_width % 6 == 0 && filtering) {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_3_Box_SSSE3;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_2_Box_SSSE3;
|
||||
if (dst_width % 12 == 0) {
|
||||
if (!filtering) {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_AVX2;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_AVX2;
|
||||
} else {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_3_Box_AVX2;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_2_Box_AVX2;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -783,28 +755,6 @@ static void ScalePlaneDown38_16(int src_width,
|
||||
ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_C;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_C;
|
||||
}
|
||||
#if defined(HAS_SCALEROWDOWN38_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON) && (dst_width % 12 == 0)) {
|
||||
if (!filtering) {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_16_NEON;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_16_NEON;
|
||||
} else {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_NEON;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEROWDOWN38_16_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) {
|
||||
if (!filtering) {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_16_SSSE3;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_16_SSSE3;
|
||||
} else {
|
||||
ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_SSSE3;
|
||||
ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < dst_height - 2; y += 3) {
|
||||
ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width);
|
||||
@ -1002,11 +952,15 @@ static int ScalePlaneBox(int src_width,
|
||||
ScaleAddRow = ScaleAddRow_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEADDCOLS_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0) {
|
||||
ScaleAddCols = (dx & 0xffff) ? ScaleAddCols2_AVX512BW
|
||||
: ((dx != 0x10000) ? ScaleAddCols1_AVX512BW
|
||||
: ScaleAddCols0_C);
|
||||
#if defined(HAS_SCALEADDCOLS1_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0 && !(dx & 0xffff) &&
|
||||
dx != 0x10000) {
|
||||
ScaleAddCols = ScaleAddCols1_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEADDCOLS2_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0 && (dx & 0xffff)) {
|
||||
ScaleAddCols = ScaleAddCols2_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEADDROW_NEON)
|
||||
@ -1096,12 +1050,6 @@ static int ScalePlaneBox_16(int src_width,
|
||||
void (*ScaleAddRow)(const uint16_t* src_ptr, uint32_t* dst_ptr,
|
||||
int src_width) = ScaleAddRow_16_C;
|
||||
|
||||
#if defined(HAS_SCALEADDROW_16_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(src_width, 16)) {
|
||||
ScaleAddRow = ScaleAddRow_16_SSE2;
|
||||
}
|
||||
#endif
|
||||
|
||||
uint32_t uy = y;
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
int boxheight;
|
||||
@ -1224,9 +1172,9 @@ static int ScalePlaneBilinearDown(int src_width,
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_SCALEFILTERCOLS_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleFilterCols = ScaleFilterCols_SSSE3;
|
||||
#if defined(HAS_SCALEFILTERCOLS_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
|
||||
ScaleFilterCols = ScaleFilterCols_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEFILTERCOLS_NEON)
|
||||
@ -1307,14 +1255,6 @@ static int ScalePlaneBilinearDown_16(int src_width,
|
||||
&dx, &dy);
|
||||
src_width = Abs(src_width);
|
||||
|
||||
#if defined(HAS_INTERPOLATEROW_16_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_SSSE3;
|
||||
if (IS_ALIGNED(src_width, 16)) {
|
||||
InterpolateRow = InterpolateRow_16_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_AVX2;
|
||||
@ -1355,12 +1295,6 @@ static int ScalePlaneBilinearDown_16(int src_width,
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(HAS_SCALEFILTERCOLS_16_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleFilterCols = ScaleFilterCols_16_SSSE3;
|
||||
}
|
||||
#endif
|
||||
|
||||
uint32_t uy = y;
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
if (uy > max_y) {
|
||||
@ -1468,9 +1402,9 @@ static int ScalePlaneBilinearUp(int src_width,
|
||||
if (filtering && src_width >= 32768) {
|
||||
ScaleFilterCols = ScaleFilterCols64_C;
|
||||
}
|
||||
#if defined(HAS_SCALEFILTERCOLS_SSSE3)
|
||||
if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleFilterCols = ScaleFilterCols_SSSE3;
|
||||
#if defined(HAS_SCALEFILTERCOLS_AVX2)
|
||||
if (filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
|
||||
ScaleFilterCols = ScaleFilterCols_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEFILTERCOLS_NEON)
|
||||
@ -1499,9 +1433,9 @@ static int ScalePlaneBilinearUp(int src_width,
|
||||
#endif
|
||||
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleFilterCols = ScaleColsUp2_C;
|
||||
#if defined(HAS_SCALECOLSUP2_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleFilterCols = ScaleColsUp2_SSE2;
|
||||
#if defined(HAS_SCALECOLSUP2_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleFilterCols = ScaleColsUp2_AVX2;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@ -1937,14 +1871,6 @@ static int ScalePlaneBilinearUp_16(int src_width,
|
||||
assert(dy <= 65536);
|
||||
src_width = Abs(src_width);
|
||||
|
||||
#if defined(HAS_INTERPOLATEROW_16_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_SSSE3;
|
||||
if (IS_ALIGNED(dst_width, 16)) {
|
||||
InterpolateRow = InterpolateRow_16_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_AVX2;
|
||||
@ -1988,18 +1914,8 @@ static int ScalePlaneBilinearUp_16(int src_width,
|
||||
if (filtering && src_width >= 32768) {
|
||||
ScaleFilterCols = ScaleFilterCols64_16_C;
|
||||
}
|
||||
#if defined(HAS_SCALEFILTERCOLS_16_SSSE3)
|
||||
if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleFilterCols = ScaleFilterCols_16_SSSE3;
|
||||
}
|
||||
#endif
|
||||
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleFilterCols = ScaleColsUp2_16_C;
|
||||
#if defined(HAS_SCALECOLS_16_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleFilterCols = ScaleColsUp2_16_SSE2;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
uint32_t uy = y;
|
||||
@ -2093,9 +2009,9 @@ static void ScalePlaneSimple(int src_width,
|
||||
|
||||
if (src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleCols = ScaleColsUp2_C;
|
||||
#if defined(HAS_SCALECOLSUP2_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleCols = ScaleColsUp2_SSE2;
|
||||
#if defined(HAS_SCALECOLSUP2_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleCols = ScaleColsUp2_AVX2;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@ -2131,11 +2047,6 @@ static void ScalePlaneSimple_16(int src_width,
|
||||
|
||||
if (src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleCols = ScaleColsUp2_16_C;
|
||||
#if defined(HAS_SCALECOLS_16_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleCols = ScaleColsUp2_16_SSE2;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
uint32_t uy = y;
|
||||
|
||||
@ -50,6 +50,19 @@ extern "C" {
|
||||
dst_ptr + np * BPP, r + 1); \
|
||||
}
|
||||
|
||||
// Fixed scale down for 16-bit planar.
|
||||
#define SDANY16(NAMEANY, SCALEROWDOWN_SIMD, SCALEROWDOWN_C, FACTOR, MASK) \
|
||||
void NAMEANY(const uint16_t* src_ptr, ptrdiff_t src_stride, \
|
||||
uint16_t* dst_ptr, int dst_width) { \
|
||||
int r = (int)((unsigned int)dst_width % (MASK + 1)); /* NOLINT */ \
|
||||
int n = dst_width - r; \
|
||||
if (n > 0) { \
|
||||
SCALEROWDOWN_SIMD(src_ptr, src_stride, dst_ptr, n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
SCALEROWDOWN_C(src_ptr + (np * FACTOR), src_stride, dst_ptr + np, r); \
|
||||
}
|
||||
|
||||
#ifdef HAS_SCALEROWDOWN2_SSSE3
|
||||
SDANY(ScaleRowDown2_Any_SSSE3, ScaleRowDown2_SSSE3, ScaleRowDown2_C, 2, 1, 15)
|
||||
SDANY(ScaleRowDown2Linear_Any_SSSE3,
|
||||
@ -71,13 +84,21 @@ SDODD(ScaleRowDown2Box_Odd_SSSE3,
|
||||
1,
|
||||
15)
|
||||
#endif
|
||||
#ifdef HAS_SCALEUVROWDOWN2BOX_SSSE3
|
||||
SDANY(ScaleUVRowDown2Box_Any_SSSE3,
|
||||
ScaleUVRowDown2Box_SSSE3,
|
||||
ScaleUVRowDown2Box_C,
|
||||
#ifdef HAS_SCALEUVROWDOWN2_AVX2
|
||||
SDANY(ScaleUVRowDown2_Any_AVX2,
|
||||
ScaleUVRowDown2_AVX2,
|
||||
ScaleUVRowDown2_C,
|
||||
2,
|
||||
2,
|
||||
3)
|
||||
7)
|
||||
#endif
|
||||
#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2
|
||||
SDANY(ScaleUVRowDown2Linear_Any_AVX2,
|
||||
ScaleUVRowDown2Linear_AVX2,
|
||||
ScaleUVRowDown2Linear_C,
|
||||
2,
|
||||
2,
|
||||
7)
|
||||
#endif
|
||||
#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2
|
||||
SDANY(ScaleUVRowDown2Box_Any_AVX2,
|
||||
@ -108,6 +129,23 @@ SDODD(ScaleRowDown2Box_Odd_AVX2,
|
||||
1,
|
||||
31)
|
||||
#endif
|
||||
#ifdef HAS_SCALEROWDOWN2_16_AVX2
|
||||
SDANY16(ScaleRowDown2_16_Any_AVX2,
|
||||
ScaleRowDown2_16_AVX2,
|
||||
ScaleRowDown2_16_C,
|
||||
2,
|
||||
15)
|
||||
SDANY16(ScaleRowDown2Linear_16_Any_AVX2,
|
||||
ScaleRowDown2Linear_16_AVX2,
|
||||
ScaleRowDown2Linear_16_C,
|
||||
2,
|
||||
15)
|
||||
SDANY16(ScaleRowDown2Box_16_Any_AVX2,
|
||||
ScaleRowDown2Box_16_AVX2,
|
||||
ScaleRowDown2Box_16_C,
|
||||
2,
|
||||
15)
|
||||
#endif
|
||||
#ifdef HAS_SCALEROWDOWN2_NEON
|
||||
SDANY(ScaleRowDown2_Any_NEON, ScaleRowDown2_NEON, ScaleRowDown2_C, 2, 1, 15)
|
||||
SDANY(ScaleRowDown2Linear_Any_NEON,
|
||||
@ -204,21 +242,21 @@ SDANY(ScaleRowDown4Box_Any_LSX,
|
||||
1,
|
||||
7)
|
||||
#endif
|
||||
#ifdef HAS_SCALEROWDOWN34_SSSE3
|
||||
SDANY(ScaleRowDown34_Any_SSSE3,
|
||||
ScaleRowDown34_SSSE3,
|
||||
#ifdef HAS_SCALEROWDOWN34_AVX2
|
||||
SDANY(ScaleRowDown34_Any_AVX2,
|
||||
ScaleRowDown34_AVX2,
|
||||
ScaleRowDown34_C,
|
||||
4 / 3,
|
||||
1,
|
||||
23)
|
||||
SDANY(ScaleRowDown34_0_Box_Any_SSSE3,
|
||||
ScaleRowDown34_0_Box_SSSE3,
|
||||
SDANY(ScaleRowDown34_0_Box_Any_AVX2,
|
||||
ScaleRowDown34_0_Box_AVX2,
|
||||
ScaleRowDown34_0_Box_C,
|
||||
4 / 3,
|
||||
1,
|
||||
23)
|
||||
SDANY(ScaleRowDown34_1_Box_Any_SSSE3,
|
||||
ScaleRowDown34_1_Box_SSSE3,
|
||||
SDANY(ScaleRowDown34_1_Box_Any_AVX2,
|
||||
ScaleRowDown34_1_Box_AVX2,
|
||||
ScaleRowDown34_1_Box_C,
|
||||
4 / 3,
|
||||
1,
|
||||
@ -285,25 +323,25 @@ SDANY(ScaleRowDown34_1_Box_Any_LSX,
|
||||
1,
|
||||
11)
|
||||
#endif
|
||||
#ifdef HAS_SCALEROWDOWN38_SSSE3
|
||||
SDANY(ScaleRowDown38_Any_SSSE3,
|
||||
ScaleRowDown38_SSSE3,
|
||||
#ifdef HAS_SCALEROWDOWN38_AVX2
|
||||
SDANY(ScaleRowDown38_Any_AVX2,
|
||||
ScaleRowDown38_AVX2,
|
||||
ScaleRowDown38_C,
|
||||
8 / 3,
|
||||
1,
|
||||
11)
|
||||
SDANY(ScaleRowDown38_3_Box_Any_SSSE3,
|
||||
ScaleRowDown38_3_Box_SSSE3,
|
||||
SDANY(ScaleRowDown38_3_Box_Any_AVX2,
|
||||
ScaleRowDown38_3_Box_AVX2,
|
||||
ScaleRowDown38_3_Box_C,
|
||||
8 / 3,
|
||||
1,
|
||||
5)
|
||||
SDANY(ScaleRowDown38_2_Box_Any_SSSE3,
|
||||
ScaleRowDown38_2_Box_SSSE3,
|
||||
11)
|
||||
SDANY(ScaleRowDown38_2_Box_Any_AVX2,
|
||||
ScaleRowDown38_2_Box_AVX2,
|
||||
ScaleRowDown38_2_Box_C,
|
||||
8 / 3,
|
||||
1,
|
||||
5)
|
||||
11)
|
||||
#endif
|
||||
#ifdef HAS_SCALEROWDOWN38_NEON
|
||||
SDANY(ScaleRowDown38_Any_NEON,
|
||||
@ -346,25 +384,25 @@ SDANY(ScaleRowDown38_2_Box_Any_LSX,
|
||||
11)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_SCALEARGBROWDOWN2_SSE2
|
||||
SDANY(ScaleARGBRowDown2_Any_SSE2,
|
||||
ScaleARGBRowDown2_SSE2,
|
||||
#ifdef HAS_SCALEARGBROWDOWN2_AVX2
|
||||
SDANY(ScaleARGBRowDown2_Any_AVX2,
|
||||
ScaleARGBRowDown2_AVX2,
|
||||
ScaleARGBRowDown2_C,
|
||||
2,
|
||||
4,
|
||||
3)
|
||||
SDANY(ScaleARGBRowDown2Linear_Any_SSE2,
|
||||
ScaleARGBRowDown2Linear_SSE2,
|
||||
7)
|
||||
SDANY(ScaleARGBRowDown2Linear_Any_AVX2,
|
||||
ScaleARGBRowDown2Linear_AVX2,
|
||||
ScaleARGBRowDown2Linear_C,
|
||||
2,
|
||||
4,
|
||||
3)
|
||||
SDANY(ScaleARGBRowDown2Box_Any_SSE2,
|
||||
ScaleARGBRowDown2Box_SSE2,
|
||||
7)
|
||||
SDANY(ScaleARGBRowDown2Box_Any_AVX2,
|
||||
ScaleARGBRowDown2Box_AVX2,
|
||||
ScaleARGBRowDown2Box_C,
|
||||
2,
|
||||
4,
|
||||
3)
|
||||
7)
|
||||
#endif
|
||||
#ifdef HAS_SCALEARGBROWDOWN2_NEON
|
||||
SDANY(ScaleARGBRowDown2_Any_NEON,
|
||||
@ -506,6 +544,8 @@ SDANY(ScaleARGBRowDown2Box_Any_WASMSIMD,
|
||||
4,
|
||||
3)
|
||||
#endif
|
||||
#undef SDANY16
|
||||
#undef SDODD
|
||||
#undef SDANY
|
||||
|
||||
// Scale down by even scale factor.
|
||||
@ -522,17 +562,17 @@ SDANY(ScaleARGBRowDown2Box_Any_WASMSIMD,
|
||||
dst_ptr + np * BPP, r); \
|
||||
}
|
||||
|
||||
#ifdef HAS_SCALEARGBROWDOWNEVEN_SSE2
|
||||
SDAANY(ScaleARGBRowDownEven_Any_SSE2,
|
||||
ScaleARGBRowDownEven_SSE2,
|
||||
#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2
|
||||
SDAANY(ScaleARGBRowDownEven_Any_AVX2,
|
||||
ScaleARGBRowDownEven_AVX2,
|
||||
ScaleARGBRowDownEven_C,
|
||||
4,
|
||||
3)
|
||||
SDAANY(ScaleARGBRowDownEvenBox_Any_SSE2,
|
||||
ScaleARGBRowDownEvenBox_SSE2,
|
||||
7)
|
||||
SDAANY(ScaleARGBRowDownEvenBox_Any_AVX2,
|
||||
ScaleARGBRowDownEvenBox_AVX2,
|
||||
ScaleARGBRowDownEvenBox_C,
|
||||
4,
|
||||
3)
|
||||
7)
|
||||
#endif
|
||||
#ifdef HAS_SCALEARGBROWDOWNEVEN_NEON
|
||||
SDAANY(ScaleARGBRowDownEven_Any_NEON,
|
||||
@ -558,6 +598,13 @@ SDAANY(ScaleARGBRowDownEvenBox_Any_LSX,
|
||||
4,
|
||||
3)
|
||||
#endif
|
||||
#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2
|
||||
SDAANY(ScaleUVRowDownEven_Any_AVX2,
|
||||
ScaleUVRowDownEven_AVX2,
|
||||
ScaleUVRowDownEven_C,
|
||||
2,
|
||||
7)
|
||||
#endif
|
||||
#ifdef HAS_SCALEUVROWDOWNEVEN_NEON
|
||||
SDAANY(ScaleUVRowDownEven_Any_NEON,
|
||||
ScaleUVRowDownEven_NEON,
|
||||
@ -1107,7 +1154,7 @@ SBU2BLANY(ScaleUVRowUp2_Bilinear_Any_AVX2,
|
||||
SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_SSE41,
|
||||
ScaleUVRowUp2_Bilinear_16_SSE41,
|
||||
ScaleUVRowUp2_Bilinear_16_C,
|
||||
7,
|
||||
3,
|
||||
uint16_t)
|
||||
#endif
|
||||
|
||||
|
||||
@ -67,19 +67,19 @@ static void ScaleARGBDown2(int src_width,
|
||||
src_argb += (y >> 16) * src_stride + ((x >> 16) - 1) * 4;
|
||||
}
|
||||
|
||||
#if defined(HAS_SCALEARGBROWDOWN2_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
#if defined(HAS_SCALEARGBROWDOWN2_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ScaleARGBRowDown2 =
|
||||
filtering == kFilterNone
|
||||
? ScaleARGBRowDown2_Any_SSE2
|
||||
: (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_SSE2
|
||||
: ScaleARGBRowDown2Box_Any_SSE2);
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
? ScaleARGBRowDown2_Any_AVX2
|
||||
: (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_AVX2
|
||||
: ScaleARGBRowDown2Box_Any_AVX2);
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleARGBRowDown2 =
|
||||
filtering == kFilterNone
|
||||
? ScaleARGBRowDown2_SSE2
|
||||
: (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_SSE2
|
||||
: ScaleARGBRowDown2Box_SSE2);
|
||||
? ScaleARGBRowDown2_AVX2
|
||||
: (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_AVX2
|
||||
: ScaleARGBRowDown2Box_AVX2);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -205,11 +205,11 @@ static int ScaleARGBDown4Box(int src_width,
|
||||
(void)dx;
|
||||
assert(dx == 65536 * 4); // Test scale factor of 4.
|
||||
assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4.
|
||||
#if defined(HAS_SCALEARGBROWDOWN2_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_SSE2;
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
ScaleARGBRowDown2 = ScaleARGBRowDown2Box_SSE2;
|
||||
#if defined(HAS_SCALEARGBROWDOWN2_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_AVX2;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleARGBRowDown2 = ScaleARGBRowDown2Box_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -284,13 +284,13 @@ static void ScaleARGBDownEven(int src_width,
|
||||
assert(IS_ALIGNED(src_width, 2));
|
||||
assert(IS_ALIGNED(src_height, 2));
|
||||
src_argb += (y >> 16) * src_stride + (x >> 16) * 4;
|
||||
#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_SSE2
|
||||
: ScaleARGBRowDownEven_Any_SSE2;
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
#if defined(HAS_SCALEARGBROWDOWNEVEN_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_AVX2
|
||||
: ScaleARGBRowDownEven_Any_AVX2;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleARGBRowDownEven =
|
||||
filtering ? ScaleARGBRowDownEvenBox_SSE2 : ScaleARGBRowDownEven_SSE2;
|
||||
filtering ? ScaleARGBRowDownEvenBox_AVX2 : ScaleARGBRowDownEven_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -444,9 +444,9 @@ static int ScaleARGBBilinearDown(int src_width,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3;
|
||||
#if defined(HAS_SCALEARGBFILTERCOLS_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
|
||||
ScaleARGBFilterCols = ScaleARGBFilterCols_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEARGBFILTERCOLS_NEON)
|
||||
@ -607,9 +607,9 @@ static int ScaleARGBBilinearUp(int src_width,
|
||||
ScaleARGBFilterCols =
|
||||
filtering ? ScaleARGBFilterCols64_C : ScaleARGBCols64_C;
|
||||
}
|
||||
#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3)
|
||||
if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3;
|
||||
#if defined(HAS_SCALEARGBFILTERCOLS_AVX2)
|
||||
if (filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
|
||||
ScaleARGBFilterCols = ScaleARGBFilterCols_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEARGBFILTERCOLS_NEON)
|
||||
@ -633,9 +633,9 @@ static int ScaleARGBBilinearUp(int src_width,
|
||||
ScaleARGBFilterCols = ScaleARGBFilterCols_RVV;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEARGBCOLS_SSE2)
|
||||
if (!filtering && TestCpuFlag(kCpuHasSSE2) && src_width < 32768) {
|
||||
ScaleARGBFilterCols = ScaleARGBCols_SSE2;
|
||||
#if defined(HAS_SCALEARGBCOLS_AVX2)
|
||||
if (!filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
|
||||
ScaleARGBFilterCols = ScaleARGBCols_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEARGBCOLS_NEON)
|
||||
@ -672,9 +672,9 @@ static int ScaleARGBBilinearUp(int src_width,
|
||||
#endif
|
||||
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleARGBFilterCols = ScaleARGBColsUp2_C;
|
||||
#if defined(HAS_SCALEARGBCOLSUP2_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleARGBFilterCols = ScaleARGBColsUp2_SSE2;
|
||||
#if defined(HAS_SCALEARGBCOLSUP2_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleARGBFilterCols = ScaleARGBColsUp2_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD)
|
||||
@ -772,9 +772,9 @@ static void ScaleARGBSimple(int src_width,
|
||||
int dst_width, int x, int dx) =
|
||||
(src_width >= 32768) ? ScaleARGBCols64_C : ScaleARGBCols_C;
|
||||
const int64_t max_y = (int64_t)(src_height - 1) << 16;
|
||||
#if defined(HAS_SCALEARGBCOLS_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && src_width < 32768) {
|
||||
ScaleARGBCols = ScaleARGBCols_SSE2;
|
||||
#if defined(HAS_SCALEARGBCOLS_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) {
|
||||
ScaleARGBCols = ScaleARGBCols_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEARGBCOLS_NEON)
|
||||
@ -803,9 +803,9 @@ static void ScaleARGBSimple(int src_width,
|
||||
#endif
|
||||
if (src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleARGBCols = ScaleARGBColsUp2_C;
|
||||
#if defined(HAS_SCALEARGBCOLSUP2_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleARGBCols = ScaleARGBColsUp2_SSE2;
|
||||
#if defined(HAS_SCALEARGBCOLSUP2_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleARGBCols = ScaleARGBColsUp2_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD)
|
||||
|
||||
@ -1155,6 +1155,86 @@ void ScaleUVRowDownEvenBox_C(const uint8_t* src_uv,
|
||||
}
|
||||
}
|
||||
|
||||
void ScaleUVRowDown2_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width) {
|
||||
int x;
|
||||
(void)src_stride;
|
||||
for (x = 0; x < dst_width; ++x) {
|
||||
dst_uv[0] = src_uv[2]; // Store the 2nd UV
|
||||
dst_uv[1] = src_uv[3];
|
||||
src_uv += 4;
|
||||
dst_uv += 2;
|
||||
}
|
||||
}
|
||||
|
||||
void ScaleUVRowDown2Linear_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width) {
|
||||
int x;
|
||||
(void)src_stride;
|
||||
for (x = 0; x < dst_width; ++x) {
|
||||
dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] + 1) >> 1);
|
||||
dst_uv[1] = (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + 1) >> 1);
|
||||
src_uv += 4;
|
||||
dst_uv += 2;
|
||||
}
|
||||
}
|
||||
|
||||
void ScaleUVRowDown2Box_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width) {
|
||||
int x;
|
||||
for (x = 0; x < dst_width; ++x) {
|
||||
dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] +
|
||||
src_uv[src_stride] + src_uv[src_stride + 2] + 2) >>
|
||||
2);
|
||||
dst_uv[1] =
|
||||
(uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + src_uv[src_stride + 1] +
|
||||
src_uv[src_stride + 3] + 2) >>
|
||||
2);
|
||||
src_uv += 4;
|
||||
dst_uv += 2;
|
||||
}
|
||||
}
|
||||
|
||||
void ScaleUVRowDownEven_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width) {
|
||||
int x;
|
||||
(void)src_stride;
|
||||
for (x = 0; x < dst_width; ++x) {
|
||||
dst_uv[0] = src_uv[0];
|
||||
dst_uv[1] = src_uv[1];
|
||||
src_uv += src_stepx * 2;
|
||||
dst_uv += 2;
|
||||
}
|
||||
}
|
||||
|
||||
void ScaleUVRowDownEvenBox_16_C(const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride,
|
||||
int src_stepx,
|
||||
uint16_t* dst_uv,
|
||||
int dst_width) {
|
||||
int x;
|
||||
for (x = 0; x < dst_width; ++x) {
|
||||
dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] +
|
||||
src_uv[src_stride] + src_uv[src_stride + 2] + 2) >>
|
||||
2);
|
||||
dst_uv[1] =
|
||||
(uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + src_uv[src_stride + 1] +
|
||||
src_uv[src_stride + 3] + 2) >>
|
||||
2);
|
||||
src_uv += src_stepx * 2;
|
||||
dst_uv += 2;
|
||||
}
|
||||
}
|
||||
|
||||
void ScaleUVRowUp2_Linear_C(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int dst_width) {
|
||||
@ -1382,6 +1462,118 @@ void ScaleUVAddCols3_C(int dst_width,
|
||||
}
|
||||
}
|
||||
|
||||
// Scales a single row of 16-bit UV pixels using point sampling.
|
||||
void ScaleUVCols_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx) {
|
||||
int j;
|
||||
for (j = 0; j < dst_width; ++j) {
|
||||
const uint16_t* src = src_uv + (x >> 16) * 2;
|
||||
dst_uv[0] = src[0];
|
||||
dst_uv[1] = src[1];
|
||||
dst_uv += 2;
|
||||
x += dx;
|
||||
}
|
||||
}
|
||||
|
||||
void ScaleUVCols64_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int x32,
|
||||
int dx) {
|
||||
int64_t x = (int64_t)(x32);
|
||||
int j;
|
||||
for (j = 0; j < dst_width; ++j) {
|
||||
const uint16_t* src = src_uv + (x >> 16) * 2;
|
||||
dst_uv[0] = src[0];
|
||||
dst_uv[1] = src[1];
|
||||
dst_uv += 2;
|
||||
x += dx;
|
||||
}
|
||||
}
|
||||
|
||||
// Scales a single row of 16-bit UV pixels up by 2x using point sampling.
|
||||
void ScaleUVColsUp2_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx) {
|
||||
int j;
|
||||
(void)x;
|
||||
(void)dx;
|
||||
for (j = 0; j < dst_width - 1; j += 2) {
|
||||
dst_uv[0] = dst_uv[2] = src_uv[0];
|
||||
dst_uv[1] = dst_uv[3] = src_uv[1];
|
||||
src_uv += 2;
|
||||
dst_uv += 4;
|
||||
}
|
||||
if (dst_width & 1) {
|
||||
dst_uv[0] = src_uv[0];
|
||||
dst_uv[1] = src_uv[1];
|
||||
}
|
||||
}
|
||||
|
||||
#define BLENDER_16(a, b, f) \
|
||||
(uint16_t)((int)(a) + \
|
||||
(int)((((int64_t)((f)) * ((int64_t)(b) - (int)(a))) + 0x8000) >> \
|
||||
16))
|
||||
|
||||
void ScaleUVFilterCols_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int x,
|
||||
int dx) {
|
||||
int j;
|
||||
for (j = 0; j < dst_width; ++j) {
|
||||
int xi = (x >> 16) * 2;
|
||||
int xf = x & 0xffff;
|
||||
const uint16_t* a = src_uv + xi;
|
||||
const uint16_t* b = a + 2;
|
||||
dst_uv[0] = BLENDER_16(a[0], b[0], xf);
|
||||
dst_uv[1] = BLENDER_16(a[1], b[1], xf);
|
||||
dst_uv += 2;
|
||||
x += dx;
|
||||
}
|
||||
}
|
||||
|
||||
void ScaleUVFilterCols64_16_C(uint16_t* dst_uv,
|
||||
const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int x32,
|
||||
int dx) {
|
||||
int64_t x = (int64_t)(x32);
|
||||
int j;
|
||||
for (j = 0; j < dst_width; ++j) {
|
||||
int64_t xi = (x >> 16) * 2;
|
||||
int xf = x & 0xffff;
|
||||
const uint16_t* a = src_uv + xi;
|
||||
const uint16_t* b = a + 2;
|
||||
dst_uv[0] = BLENDER_16(a[0], b[0], xf);
|
||||
dst_uv[1] = BLENDER_16(a[1], b[1], xf);
|
||||
dst_uv += 2;
|
||||
x += dx;
|
||||
}
|
||||
}
|
||||
#undef BLENDER_16
|
||||
|
||||
// Scale 16-bit UV columns by 3 box filter.
|
||||
void ScaleUVAddCols3_16_C(int dst_width,
|
||||
int scaleval,
|
||||
const uint32_t* src_uv,
|
||||
uint16_t* dst_uv) {
|
||||
int i;
|
||||
for (i = 0; i < dst_width; ++i) {
|
||||
uint32_t u = src_uv[0] + src_uv[2] + src_uv[4];
|
||||
uint32_t v = src_uv[1] + src_uv[3] + src_uv[5];
|
||||
src_uv += 6;
|
||||
dst_uv[0] = (uint16_t)((u * (uint32_t)scaleval) >> 16);
|
||||
dst_uv[1] = (uint16_t)((v * (uint32_t)scaleval) >> 16);
|
||||
dst_uv += 2;
|
||||
}
|
||||
}
|
||||
|
||||
// Scale plane vertically with bilinear interpolation.
|
||||
void ScalePlaneVertical(int src_height,
|
||||
int dst_width,
|
||||
@ -1536,14 +1728,6 @@ void ScalePlaneVertical_16(int src_height,
|
||||
assert(dst_width > 0);
|
||||
assert(dst_height > 0);
|
||||
src_argb += (x >> 16) * wpp;
|
||||
#if defined(HAS_INTERPOLATEROW_16_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_SSSE3;
|
||||
if (IS_ALIGNED(dst_width_words, 16)) {
|
||||
InterpolateRow = InterpolateRow_16_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_AVX2;
|
||||
|
||||
1619
source/scale_gcc.cc
1619
source/scale_gcc.cc
File diff suppressed because it is too large
Load Diff
@ -84,25 +84,25 @@ static void ScaleUVDown2(int src_width,
|
||||
assert(dx == 65536 * 2); // Test scale factor of 2.
|
||||
assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2.
|
||||
// Advance to odd row, even column.
|
||||
if (filtering == kFilterBilinear) {
|
||||
if (filtering) {
|
||||
src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
|
||||
} else {
|
||||
src_uv += (y >> 16) * src_stride + ((x >> 16) - 1) * 2;
|
||||
}
|
||||
|
||||
#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && filtering) {
|
||||
ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_SSSE3;
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
ScaleUVRowDown2 = ScaleUVRowDown2Box_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVROWDOWN2BOX_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && filtering) {
|
||||
ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_AVX2;
|
||||
#if defined(HAS_SCALEUVROWDOWN2_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ScaleUVRowDown2 =
|
||||
filtering == kFilterNone
|
||||
? ScaleUVRowDown2_Any_AVX2
|
||||
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_AVX2
|
||||
: ScaleUVRowDown2Box_Any_AVX2);
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVRowDown2 = ScaleUVRowDown2Box_AVX2;
|
||||
ScaleUVRowDown2 =
|
||||
filtering == kFilterNone
|
||||
? ScaleUVRowDown2_AVX2
|
||||
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_AVX2
|
||||
: ScaleUVRowDown2Box_AVX2);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -148,23 +148,48 @@ static void ScaleUVDown2(int src_width,
|
||||
}
|
||||
#endif
|
||||
|
||||
// This code is not enabled. Only box filter is available at this time.
|
||||
#if defined(HAS_SCALEUVROWDOWN2_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ScaleUVRowDown2 =
|
||||
filtering == kFilterNone
|
||||
? ScaleUVRowDown2_Any_SSSE3
|
||||
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_SSSE3
|
||||
: ScaleUVRowDown2Box_Any_SSSE3);
|
||||
if (IS_ALIGNED(dst_width, 2)) {
|
||||
ScaleUVRowDown2 =
|
||||
filtering == kFilterNone
|
||||
? ScaleUVRowDown2_SSSE3
|
||||
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_SSSE3
|
||||
: ScaleUVRowDown2Box_SSSE3);
|
||||
}
|
||||
if (filtering == kFilterLinear) {
|
||||
src_stride = 0;
|
||||
}
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
ScaleUVRowDown2(src_uv, src_stride, dst_uv, dst_width);
|
||||
src_uv += row_stride;
|
||||
dst_uv += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static void ScaleUVDown2_16(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
ptrdiff_t src_stride,
|
||||
ptrdiff_t dst_stride,
|
||||
const uint16_t* src_uv,
|
||||
uint16_t* dst_uv,
|
||||
int x,
|
||||
int dx,
|
||||
int y,
|
||||
int dy,
|
||||
enum FilterMode filtering) {
|
||||
int j;
|
||||
ptrdiff_t row_stride = src_stride * (dy >> 16);
|
||||
void (*ScaleUVRowDown2)(const uint16_t* src_uv, ptrdiff_t src_stride,
|
||||
uint16_t* dst_uv, int dst_width) =
|
||||
filtering == kFilterNone
|
||||
? ScaleUVRowDown2_16_C
|
||||
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_16_C
|
||||
: ScaleUVRowDown2Box_16_C);
|
||||
(void)src_width;
|
||||
(void)src_height;
|
||||
(void)dx;
|
||||
assert(dx == 65536 * 2); // Test scale factor of 2.
|
||||
assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2.
|
||||
// Advance to odd row, even column.
|
||||
if (filtering) {
|
||||
src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
|
||||
} else {
|
||||
src_uv += (y >> 16) * src_stride + ((x >> 16) - 1) * 2;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (filtering == kFilterLinear) {
|
||||
src_stride = 0;
|
||||
@ -211,14 +236,6 @@ static int ScaleUVDown4Box(int src_width,
|
||||
assert(dx == 65536 * 4); // Test scale factor of 4.
|
||||
assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4.
|
||||
|
||||
#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_SSSE3;
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
ScaleUVRowDown2 = ScaleUVRowDown2Box_SSSE3;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVROWDOWN2BOX_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_AVX2;
|
||||
@ -262,6 +279,49 @@ static int ScaleUVDown4Box(int src_width,
|
||||
free_aligned_buffer_64(row);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ScaleUVDown4Box_16(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
ptrdiff_t src_stride,
|
||||
ptrdiff_t dst_stride,
|
||||
const uint16_t* src_uv,
|
||||
uint16_t* dst_uv,
|
||||
int x,
|
||||
int dx,
|
||||
int y,
|
||||
int dy) {
|
||||
int j;
|
||||
// Allocate 2 rows of 16-bit UV.
|
||||
const int row_stride_words = (dst_width * 2 * 2 + 15) & ~15;
|
||||
align_buffer_64(row, row_stride_words * 2 * (int)sizeof(uint16_t));
|
||||
if (!row)
|
||||
return 1;
|
||||
uint16_t* row16 = (uint16_t*)row;
|
||||
ptrdiff_t row_stride = src_stride * (dy >> 16);
|
||||
void (*ScaleUVRowDown2)(const uint16_t* src_uv, ptrdiff_t src_stride,
|
||||
uint16_t* dst_uv, int dst_width) =
|
||||
ScaleUVRowDown2Box_16_C;
|
||||
// Advance to odd row, even column.
|
||||
src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
|
||||
(void)src_width;
|
||||
(void)src_height;
|
||||
(void)dx;
|
||||
assert(dx == 65536 * 4); // Test scale factor of 4.
|
||||
assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4.
|
||||
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
ScaleUVRowDown2(src_uv, src_stride, row16, dst_width * 2);
|
||||
ScaleUVRowDown2(src_uv + src_stride * 2, src_stride,
|
||||
row16 + row_stride_words, dst_width * 2);
|
||||
ScaleUVRowDown2(row16, row_stride_words, dst_uv, dst_width);
|
||||
src_uv += row_stride;
|
||||
dst_uv += dst_stride;
|
||||
}
|
||||
free_aligned_buffer_64(row);
|
||||
return 0;
|
||||
}
|
||||
#endif // HAS_SCALEUVDOWN4BOX
|
||||
|
||||
// ScaleUV, any dimensions, box filter.
|
||||
@ -493,6 +553,119 @@ static int ScaleUVBox(int src_width,
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void ScaleUVAddCols1_16_C(int dst_width,
|
||||
int boxheight,
|
||||
int x,
|
||||
int dx,
|
||||
const uint32_t* src_uv,
|
||||
uint16_t* dst_uv) {
|
||||
int i, j;
|
||||
int boxwidth = MIN1(dx >> 16);
|
||||
uint32_t scaleval = 65536 / (boxwidth * boxheight);
|
||||
x >>= 16;
|
||||
for (i = 0; i < dst_width; ++i) {
|
||||
uint32_t u = 0;
|
||||
uint32_t v = 0;
|
||||
for (j = 0; j < boxwidth; ++j) {
|
||||
u += src_uv[(x + j) * 2];
|
||||
v += src_uv[(x + j) * 2 + 1];
|
||||
}
|
||||
x += boxwidth;
|
||||
*dst_uv++ = (uint16_t)((u * scaleval) >> 16);
|
||||
*dst_uv++ = (uint16_t)((v * scaleval) >> 16);
|
||||
}
|
||||
}
|
||||
|
||||
static void ScaleUVAddCols2_16_C(int dst_width,
|
||||
int boxheight,
|
||||
int x,
|
||||
int dx,
|
||||
const uint32_t* src_uv,
|
||||
uint16_t* dst_uv) {
|
||||
int i, j;
|
||||
int minboxwidth = dx >> 16;
|
||||
uint32_t scaletbl[2];
|
||||
scaletbl[0] = 65536 / (MIN1(minboxwidth) * boxheight);
|
||||
scaletbl[1] = 65536 / (MIN1(minboxwidth + 1) * boxheight);
|
||||
for (i = 0; i < dst_width; ++i) {
|
||||
int ix = x >> 16;
|
||||
uint32_t u = 0;
|
||||
uint32_t v = 0;
|
||||
int boxwidth;
|
||||
uint32_t scaleval;
|
||||
x += dx;
|
||||
boxwidth = MIN1((x >> 16) - ix);
|
||||
assert((boxwidth - minboxwidth == 0) || (boxwidth - minboxwidth == 1));
|
||||
scaleval = scaletbl[boxwidth - minboxwidth];
|
||||
for (j = 0; j < boxwidth; ++j) {
|
||||
u += src_uv[(ix + j) * 2];
|
||||
v += src_uv[(ix + j) * 2 + 1];
|
||||
}
|
||||
*dst_uv++ = (uint16_t)((u * scaleval) >> 16);
|
||||
*dst_uv++ = (uint16_t)((v * scaleval) >> 16);
|
||||
}
|
||||
}
|
||||
|
||||
static int ScaleUVBox_16(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
ptrdiff_t src_stride,
|
||||
ptrdiff_t dst_stride,
|
||||
const uint16_t* src_uv,
|
||||
uint16_t* dst_uv,
|
||||
int x,
|
||||
int dx,
|
||||
int y,
|
||||
int dy) {
|
||||
int j, k;
|
||||
const uint32_t max_y = (uint32_t)src_height << 16;
|
||||
const int row_samples = src_width * 2;
|
||||
{
|
||||
// Row accumulator, two uint32_t per source UV pixel.
|
||||
align_buffer_64(row32_buf, row_samples * (int)sizeof(uint32_t));
|
||||
if (!row32_buf)
|
||||
return 1;
|
||||
{
|
||||
uint32_t* row32 = (uint32_t*)row32_buf;
|
||||
void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx,
|
||||
const uint32_t* src_uv, uint16_t* dst_uv) =
|
||||
(dx & 0xffff) ? ScaleUVAddCols2_16_C : ScaleUVAddCols1_16_C;
|
||||
void (*ScaleUVAddCols3)(int dst_width, int scaleval,
|
||||
const uint32_t* src_uv, uint16_t* dst_uv) =
|
||||
(dx == 0x30000) ? ScaleUVAddCols3_16_C : NULL;
|
||||
void (*ScaleAddRow)(const uint16_t* src_ptr, uint32_t* dst_ptr,
|
||||
int src_width) = ScaleAddRow_16_C;
|
||||
|
||||
uint32_t uy = y;
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
int boxheight;
|
||||
int iy = uy >> 16;
|
||||
const uint16_t* src = src_uv + iy * src_stride;
|
||||
uy += dy;
|
||||
if (uy > max_y) {
|
||||
uy = max_y;
|
||||
}
|
||||
boxheight = MIN1((int)(uy >> 16) - iy);
|
||||
memset(row32, 0, row_samples * sizeof(uint32_t));
|
||||
for (k = 0; k < boxheight; ++k) {
|
||||
ScaleAddRow(src, row32, row_samples);
|
||||
src += src_stride;
|
||||
}
|
||||
if (ScaleUVAddCols3 && boxheight <= 21845) {
|
||||
ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight),
|
||||
row32 + (ptrdiff_t)(x >> 16) * 2, dst_uv);
|
||||
} else {
|
||||
ScaleUVAddCols(dst_width, boxheight, x, dx, row32, dst_uv);
|
||||
}
|
||||
dst_uv += dst_stride;
|
||||
}
|
||||
}
|
||||
free_aligned_buffer_64(row32_buf);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // HAS_SCALEUVBOX
|
||||
|
||||
// ScaleUV Even
|
||||
@ -523,13 +696,11 @@ static void ScaleUVDownEven(int src_width,
|
||||
assert(IS_ALIGNED(src_width, 2));
|
||||
assert(IS_ALIGNED(src_height, 2));
|
||||
src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
|
||||
#if defined(HAS_SCALEUVROWDOWNEVEN_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3)) {
|
||||
ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_SSSE3
|
||||
: ScaleUVRowDownEven_Any_SSSE3;
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
ScaleUVRowDownEven =
|
||||
filtering ? ScaleUVRowDownEvenBox_SSE2 : ScaleUVRowDownEven_SSSE3;
|
||||
#if defined(HAS_SCALEUVROWDOWNEVEN_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2) && !filtering) {
|
||||
ScaleUVRowDownEven = ScaleUVRowDownEven_Any_AVX2;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVRowDownEven = ScaleUVRowDownEven_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -541,16 +712,6 @@ static void ScaleUVDownEven(int src_width,
|
||||
}
|
||||
}
|
||||
#endif // TODO(fbarchard): Enable Box filter
|
||||
#if defined(HAS_SCALEUVROWDOWNEVENBOX_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_NEON
|
||||
: ScaleUVRowDownEven_Any_NEON;
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
ScaleUVRowDownEven =
|
||||
filtering ? ScaleUVRowDownEvenBox_NEON : ScaleUVRowDownEven_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVROWDOWNEVEN_RVV) || defined(HAS_SCALEUVROWDOWN4_RVV)
|
||||
if (TestCpuFlag(kCpuHasRVV) && !filtering) {
|
||||
#if defined(HAS_SCALEUVROWDOWNEVEN_RVV)
|
||||
@ -573,6 +734,41 @@ static void ScaleUVDownEven(int src_width,
|
||||
dst_uv += dst_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static void ScaleUVDownEven_16(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
ptrdiff_t src_stride,
|
||||
ptrdiff_t dst_stride,
|
||||
const uint16_t* src_uv,
|
||||
uint16_t* dst_uv,
|
||||
int x,
|
||||
int dx,
|
||||
int y,
|
||||
int dy,
|
||||
enum FilterMode filtering) {
|
||||
int j;
|
||||
int col_step = dx >> 16;
|
||||
ptrdiff_t row_stride = (dy >> 16) * src_stride;
|
||||
void (*ScaleUVRowDownEven)(const uint16_t* src_uv, ptrdiff_t src_stride,
|
||||
int src_step, uint16_t* dst_uv, int dst_width) =
|
||||
filtering ? ScaleUVRowDownEvenBox_16_C : ScaleUVRowDownEven_16_C;
|
||||
(void)src_width;
|
||||
(void)src_height;
|
||||
assert(IS_ALIGNED(src_width, 2));
|
||||
assert(IS_ALIGNED(src_height, 2));
|
||||
src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
|
||||
|
||||
if (filtering == kFilterLinear) {
|
||||
src_stride = 0;
|
||||
}
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
ScaleUVRowDownEven(src_uv, src_stride, col_step, dst_uv, dst_width);
|
||||
src_uv += row_stride;
|
||||
dst_uv += dst_stride;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// Scale UV down with bilinear interpolation.
|
||||
@ -674,19 +870,6 @@ static int ScaleUVBilinearDown(int src_width,
|
||||
InterpolateRow = InterpolateRow_WASMSIMD;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVFILTERCOLS_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleUVFilterCols = ScaleUVFilterCols_SSSE3;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVFILTERCOLS_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON;
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
ScaleUVFilterCols = ScaleUVFilterCols_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
|
||||
// Allocate a row of UV.
|
||||
@ -720,6 +903,110 @@ static int ScaleUVBilinearDown(int src_width,
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ScaleUVBilinearDown_16(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
ptrdiff_t src_stride,
|
||||
ptrdiff_t dst_stride,
|
||||
const uint16_t* src_uv,
|
||||
uint16_t* dst_uv,
|
||||
int x,
|
||||
int dx,
|
||||
int y,
|
||||
int dy,
|
||||
enum FilterMode filtering) {
|
||||
int j;
|
||||
void (*InterpolateRow)(uint16_t* dst_uv, const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride, int dst_width,
|
||||
int source_y_fraction) = InterpolateRow_16_C;
|
||||
void (*ScaleUVFilterCols)(uint16_t* dst_uv, const uint16_t* src_uv,
|
||||
int dst_width, int x, int dx) =
|
||||
(src_width >= 32768) ? ScaleUVFilterCols64_16_C : ScaleUVFilterCols_16_C;
|
||||
int64_t xlast = x + (int64_t)(dst_width - 1) * dx;
|
||||
int64_t xl = (dx >= 0) ? x : xlast;
|
||||
int64_t xr = (dx >= 0) ? xlast : x;
|
||||
int clip_src_width;
|
||||
xl = (xl >> 16) & ~3; // Left edge aligned.
|
||||
xr = (xr >> 16) + 1; // Right most pixel used. Bilinear uses 2 pixels.
|
||||
xr = (xr + 1 + 3) & ~3; // 1 beyond 4 pixel aligned right most pixel.
|
||||
if (xr > src_width) {
|
||||
xr = src_width;
|
||||
}
|
||||
clip_src_width = (int)(xr - xl) * 2; // Width in uint16_t words.
|
||||
src_uv += xl * 2;
|
||||
x -= (int)(xl << 16);
|
||||
#if defined(HAS_INTERPOLATEROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_AVX2;
|
||||
if (IS_ALIGNED(clip_src_width, 16)) {
|
||||
InterpolateRow = InterpolateRow_16_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
InterpolateRow = InterpolateRow_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_NEON;
|
||||
if (IS_ALIGNED(clip_src_width, 8)) {
|
||||
InterpolateRow = InterpolateRow_16_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_SME)
|
||||
if (TestCpuFlag(kCpuHasSME)) {
|
||||
InterpolateRow = InterpolateRow_16_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
InterpolateRow = InterpolateRow_16_HVX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_WASMSIMD;
|
||||
if (IS_ALIGNED(clip_src_width, 8)) {
|
||||
InterpolateRow = InterpolateRow_16_WASMSIMD;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// Allocate a row of 16-bit UV.
|
||||
{
|
||||
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
|
||||
uint32_t uy = y;
|
||||
align_buffer_64(row, clip_src_width * (int)sizeof(uint16_t));
|
||||
if (!row)
|
||||
return 1;
|
||||
uint16_t* row16 = (uint16_t*)row;
|
||||
if (uy > max_y) {
|
||||
uy = max_y;
|
||||
}
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
int yi = uy >> 16;
|
||||
const uint16_t* src = src_uv + yi * src_stride;
|
||||
if (filtering == kFilterLinear) {
|
||||
ScaleUVFilterCols(dst_uv, src, dst_width, x, dx);
|
||||
} else {
|
||||
int yf = (uy >> 8) & 255;
|
||||
InterpolateRow(row16, src, src_stride, clip_src_width, yf);
|
||||
ScaleUVFilterCols(dst_uv, row16, dst_width, x, dx);
|
||||
}
|
||||
dst_uv += dst_stride;
|
||||
uy += dy;
|
||||
if (uy > max_y) {
|
||||
uy = max_y;
|
||||
}
|
||||
}
|
||||
free_aligned_buffer_64(row);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Scale UV up with bilinear interpolation.
|
||||
@ -820,39 +1107,8 @@ static int ScaleUVBilinearUp(int src_width,
|
||||
if (src_width >= 32768) {
|
||||
ScaleUVFilterCols = filtering ? ScaleUVFilterCols64_C : ScaleUVCols64_C;
|
||||
}
|
||||
#if defined(HAS_SCALEUVFILTERCOLS_SSSE3)
|
||||
if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleUVFilterCols = ScaleUVFilterCols_SSSE3;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVFILTERCOLS_NEON)
|
||||
if (filtering && TestCpuFlag(kCpuHasNEON)) {
|
||||
ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVFilterCols = ScaleUVFilterCols_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVCOLS_SSSE3)
|
||||
if (!filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleUVFilterCols = ScaleUVCols_SSSE3;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVCOLS_NEON)
|
||||
if (!filtering && TestCpuFlag(kCpuHasNEON)) {
|
||||
ScaleUVFilterCols = ScaleUVCols_Any_NEON;
|
||||
if (IS_ALIGNED(dst_width, 16)) {
|
||||
ScaleUVFilterCols = ScaleUVCols_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleUVFilterCols = ScaleUVColsUp2_C;
|
||||
#if defined(HAS_SCALEUVCOLSUP2_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVFilterCols = ScaleUVColsUp2_SSSE3;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
if (uy > max_y) {
|
||||
@ -919,6 +1175,136 @@ static int ScaleUVBilinearUp(int src_width,
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ScaleUVBilinearUp_16(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
ptrdiff_t src_stride,
|
||||
ptrdiff_t dst_stride,
|
||||
const uint16_t* src_uv,
|
||||
uint16_t* dst_uv,
|
||||
int x,
|
||||
int dx,
|
||||
int y,
|
||||
int dy,
|
||||
enum FilterMode filtering) {
|
||||
assert(src_width > 0);
|
||||
assert(src_height > 0);
|
||||
assert(dst_width > 0);
|
||||
assert(dst_height > 0);
|
||||
assert(dy <= 65536);
|
||||
|
||||
int j;
|
||||
void (*InterpolateRow)(uint16_t* dst_uv, const uint16_t* src_uv,
|
||||
ptrdiff_t src_stride, int dst_width,
|
||||
int source_y_fraction) = InterpolateRow_16_C;
|
||||
void (*ScaleUVFilterCols)(uint16_t* dst_uv, const uint16_t* src_uv,
|
||||
int dst_width, int x, int dx) =
|
||||
filtering ? ScaleUVFilterCols_16_C : ScaleUVCols_16_C;
|
||||
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
|
||||
uint32_t uy = y;
|
||||
#if defined(HAS_INTERPOLATEROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_AVX2;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
InterpolateRow = InterpolateRow_16_AVX2;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
InterpolateRow = InterpolateRow_16_AVX512BW;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_NEON;
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
InterpolateRow = InterpolateRow_16_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_SME)
|
||||
if (TestCpuFlag(kCpuHasSME)) {
|
||||
InterpolateRow = InterpolateRow_16_SME;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_HVX)
|
||||
if (TestCpuFlag(kCpuHasHVX)) {
|
||||
InterpolateRow = InterpolateRow_16_HVX;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_INTERPOLATEROW_16_WASMSIMD)
|
||||
if (TestCpuFlag(kCpuHasWASMSIMD)) {
|
||||
InterpolateRow = InterpolateRow_16_Any_WASMSIMD;
|
||||
if (IS_ALIGNED(dst_width, 4)) {
|
||||
InterpolateRow = InterpolateRow_16_WASMSIMD;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (src_width >= 32768) {
|
||||
ScaleUVFilterCols =
|
||||
filtering ? ScaleUVFilterCols64_16_C : ScaleUVCols64_16_C;
|
||||
}
|
||||
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleUVFilterCols = ScaleUVColsUp2_16_C;
|
||||
}
|
||||
|
||||
if (uy > max_y) {
|
||||
uy = max_y;
|
||||
}
|
||||
|
||||
{
|
||||
int yi = uy >> 16;
|
||||
const uint16_t* src = src_uv + yi * src_stride;
|
||||
|
||||
// Allocate 2 rows of 16-bit UV.
|
||||
const int row_stride_words = (dst_width * 2 + 15) & ~15;
|
||||
align_buffer_64(row, row_stride_words * 2 * (int)sizeof(uint16_t));
|
||||
if (!row)
|
||||
return 1;
|
||||
|
||||
uint16_t* rowptr = (uint16_t*)row;
|
||||
ptrdiff_t rowstride = row_stride_words;
|
||||
int lasty = yi;
|
||||
|
||||
ScaleUVFilterCols(rowptr, src, dst_width, x, dx);
|
||||
if (src_height > 1) {
|
||||
src += src_stride;
|
||||
}
|
||||
ScaleUVFilterCols(rowptr + rowstride, src, dst_width, x, dx);
|
||||
if (src_height > 2) {
|
||||
src += src_stride;
|
||||
}
|
||||
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
if (uy > max_y) {
|
||||
uy = max_y;
|
||||
}
|
||||
yi = uy >> 16;
|
||||
if (yi != lasty) {
|
||||
ScaleUVFilterCols(rowptr, src, dst_width, x, dx);
|
||||
rowptr += rowstride;
|
||||
rowstride = -rowstride;
|
||||
lasty = yi;
|
||||
if ((uy + 65536) < max_y) {
|
||||
src += src_stride;
|
||||
}
|
||||
}
|
||||
if (filtering == kFilterLinear) {
|
||||
InterpolateRow(dst_uv, rowptr, 0, dst_width * 2, 0);
|
||||
} else {
|
||||
int yf = (uy >> 8) & 255;
|
||||
InterpolateRow(dst_uv, rowptr, rowstride, dst_width * 2, yf);
|
||||
}
|
||||
dst_uv += dst_stride;
|
||||
uy += dy;
|
||||
}
|
||||
free_aligned_buffer_64(row);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // HAS_SCALEUVBILINEARUP
|
||||
|
||||
// Scale UV, horizontally up by 2 times.
|
||||
@ -1169,26 +1555,38 @@ static void ScaleUVSimple(int src_width,
|
||||
int x, int dx) =
|
||||
(src_width >= 32768) ? ScaleUVCols64_C : ScaleUVCols_C;
|
||||
(void)src_height;
|
||||
#if defined(HAS_SCALEUVCOLS_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
|
||||
ScaleUVCols = ScaleUVCols_SSSE3;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_SCALEUVCOLS_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ScaleUVCols = ScaleUVCols_Any_NEON;
|
||||
if (IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVCols = ScaleUVCols_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleUVCols = ScaleUVColsUp2_C;
|
||||
#if defined(HAS_SCALEUVCOLSUP2_SSSE3)
|
||||
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) {
|
||||
ScaleUVCols = ScaleUVColsUp2_SSSE3;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
uint32_t uy = y;
|
||||
for (j = 0; j < dst_height; ++j) {
|
||||
int yi = uy >> 16;
|
||||
ScaleUVCols(dst_uv, src_uv + yi * src_stride, dst_width, x, dx);
|
||||
dst_uv += dst_stride;
|
||||
uy += dy;
|
||||
}
|
||||
}
|
||||
|
||||
static void ScaleUVSimple_16(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
ptrdiff_t src_stride,
|
||||
ptrdiff_t dst_stride,
|
||||
const uint16_t* src_uv,
|
||||
uint16_t* dst_uv,
|
||||
int x,
|
||||
int dx,
|
||||
int y,
|
||||
int dy) {
|
||||
int j;
|
||||
void (*ScaleUVCols)(uint16_t* dst_uv, const uint16_t* src_uv, int dst_width,
|
||||
int x, int dx) =
|
||||
(src_width >= 32768) ? ScaleUVCols64_16_C : ScaleUVCols_16_C;
|
||||
(void)src_height;
|
||||
if (src_width * 2 == dst_width && x < 0x8000) {
|
||||
ScaleUVCols = ScaleUVColsUp2_16_C;
|
||||
}
|
||||
|
||||
uint32_t uy = y;
|
||||
@ -1402,8 +1800,166 @@ int UVScale(const uint8_t* src_uv,
|
||||
dst_height, filtering);
|
||||
}
|
||||
|
||||
// Scale a 16-bit UV plane (from P010/P210)
|
||||
static int ScaleUV_16(const uint16_t* src,
|
||||
int src_stride,
|
||||
int src_width,
|
||||
int src_height,
|
||||
uint16_t* dst,
|
||||
int dst_stride,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
int clip_x,
|
||||
int clip_y,
|
||||
int clip_width,
|
||||
int clip_height,
|
||||
enum FilterMode filtering) {
|
||||
// Initial source x/y coordinate and step values as 16.16 fixed point.
|
||||
int x = 0;
|
||||
int y = 0;
|
||||
int dx = 0;
|
||||
int dy = 0;
|
||||
const enum FilterMode orig_filtering = filtering;
|
||||
// Simplify filtering when possible.
|
||||
filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height,
|
||||
filtering);
|
||||
|
||||
// Negative src_height means invert the image.
|
||||
if (src_height < 0) {
|
||||
src_height = -src_height;
|
||||
src = src + (src_height - 1) * (ptrdiff_t)src_stride;
|
||||
src_stride = -src_stride;
|
||||
}
|
||||
ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
|
||||
&dx, &dy);
|
||||
if (clip_x) {
|
||||
int64_t clipf = (int64_t)(clip_x)*dx;
|
||||
x += (clipf & 0xffff);
|
||||
src += (clipf >> 16) * 2;
|
||||
dst += clip_x * 2;
|
||||
}
|
||||
if (clip_y) {
|
||||
int64_t clipf = (int64_t)(clip_y)*dy;
|
||||
y += (clipf & 0xffff);
|
||||
src += (clipf >> 16) * (ptrdiff_t)src_stride;
|
||||
dst += clip_y * (ptrdiff_t)dst_stride;
|
||||
}
|
||||
|
||||
#if HAS_SCALEUVBOX
|
||||
// Exactly 4 goes to the faster ScaleUVDown4Box_16 below. Clipping is excluded
|
||||
// because whole source rows are accumulated.
|
||||
if (filtering == kFilterBox && !(dx == 0x40000 && dy == 0x40000) &&
|
||||
clip_x == 0 && clip_y == 0) {
|
||||
return ScaleUVBox_16(src_width, src_height, clip_width, clip_height,
|
||||
src_stride, dst_stride, src, dst, x, dx, y, dy);
|
||||
}
|
||||
#endif
|
||||
|
||||
// Special case for integer step values.
|
||||
if (((dx | dy) & 0xffff) == 0) {
|
||||
if (!dx || !dy) { // 1 pixel wide and/or tall.
|
||||
filtering = kFilterNone;
|
||||
} else {
|
||||
// Optimized even scale down. ie 2, 4, 6, 8, 10x.
|
||||
if (!(dx & 0x10000) && !(dy & 0x10000)) {
|
||||
#if HAS_SCALEUVDOWN2
|
||||
if (dx == 0x20000 && dy == 0x20000) {
|
||||
// Optimized 1/2 downsample.
|
||||
ScaleUVDown2_16(src_width, src_height, clip_width, clip_height,
|
||||
src_stride, dst_stride, src, dst, x, dx, y, dy,
|
||||
filtering);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#if HAS_SCALEUVDOWN4BOX
|
||||
if (dx == 0x40000 && dy == 0x40000 && filtering == kFilterBox) {
|
||||
// Optimized 1/4 box downsample.
|
||||
return ScaleUVDown4Box_16(src_width, src_height, clip_width,
|
||||
clip_height, src_stride, dst_stride, src,
|
||||
dst, x, dx, y, dy);
|
||||
}
|
||||
#endif
|
||||
#if HAS_SCALEUVDOWNEVEN
|
||||
ScaleUVDownEven_16(src_width, src_height, clip_width, clip_height,
|
||||
src_stride, dst_stride, src, dst, x, dx, y, dy,
|
||||
filtering);
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
// Optimized odd scale down. ie 3, 5, 7, 9x.
|
||||
if ((dx & 0x10000) && (dy & 0x10000)) {
|
||||
filtering = kFilterNone;
|
||||
#ifdef HAS_UVCOPY
|
||||
if (dx == 0x10000 && dy == 0x10000) {
|
||||
// Straight copy.
|
||||
return UVCopy_16(
|
||||
src + (y >> 16) * (ptrdiff_t)src_stride + (x >> 16) * 2,
|
||||
src_stride, dst, dst_stride, clip_width, clip_height);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone to avoid
|
||||
// reading a second source column. The Up2 kernels clamp their last pixel and
|
||||
// only read the 1 source column when dst_width <= 2, so restore Linear or
|
||||
// Bilinear for 2x upscales (e.g., P010ToP410 and P210ToP410 on 1-2 pixel wide
|
||||
// images) when the caller requested filtering.
|
||||
if (orig_filtering != kFilterNone && filtering == kFilterNone &&
|
||||
src_width == 1 && dst_width <= 2 && clip_x == 0 && clip_y == 0) {
|
||||
if (clip_height == src_height && (dst_width + 1) / 2 == src_width) {
|
||||
ScaleUVLinearUp2_16(src_width, src_height, clip_width, clip_height,
|
||||
src_stride, dst_stride, src, dst);
|
||||
return 0;
|
||||
}
|
||||
if ((clip_height + 1) / 2 == src_height &&
|
||||
(clip_width + 1) / 2 == src_width && orig_filtering != kFilterLinear) {
|
||||
ScaleUVBilinearUp2_16(src_width, src_height, clip_width, clip_height,
|
||||
src_stride, dst_stride, src, dst);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// HAS_SCALEPLANEVERTICAL
|
||||
if (dx == 0x10000 && (x & 0xffff) == 0) {
|
||||
// Arbitrary scale vertically, but unscaled horizontally.
|
||||
ScalePlaneVertical_16(src_height, clip_width, clip_height, src_stride,
|
||||
dst_stride, src, dst, x, y, dy, /*wpp=*/2, filtering);
|
||||
return 0;
|
||||
}
|
||||
if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) {
|
||||
ScaleUVLinearUp2_16(src_width, src_height, clip_width, clip_height,
|
||||
src_stride, dst_stride, src, dst);
|
||||
return 0;
|
||||
}
|
||||
if ((clip_height + 1) / 2 == src_height &&
|
||||
(clip_width + 1) / 2 == src_width &&
|
||||
(filtering == kFilterBilinear || filtering == kFilterBox)) {
|
||||
ScaleUVBilinearUp2_16(src_width, src_height, clip_width, clip_height,
|
||||
src_stride, dst_stride, src, dst);
|
||||
return 0;
|
||||
}
|
||||
#if HAS_SCALEUVBILINEARUP
|
||||
if (filtering && dy < 65536) {
|
||||
return ScaleUVBilinearUp_16(src_width, src_height, clip_width, clip_height,
|
||||
src_stride, dst_stride, src, dst, x, dx, y, dy,
|
||||
filtering);
|
||||
}
|
||||
#endif
|
||||
#if HAS_SCALEUVBILINEARDOWN
|
||||
if (filtering) {
|
||||
return ScaleUVBilinearDown_16(src_width, src_height, clip_width,
|
||||
clip_height, src_stride, dst_stride, src, dst,
|
||||
x, dx, y, dy, filtering);
|
||||
}
|
||||
#endif
|
||||
ScaleUVSimple_16(src_width, src_height, clip_width, clip_height, src_stride,
|
||||
dst_stride, src, dst, x, dx, y, dy);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Scale a 16 bit UV image.
|
||||
// This function is currently incomplete, it can't handle all cases.
|
||||
LIBYUV_API
|
||||
int UVScale_16(const uint16_t* src_uv,
|
||||
int src_stride_uv,
|
||||
@ -1414,65 +1970,14 @@ int UVScale_16(const uint16_t* src_uv,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
enum FilterMode filtering) {
|
||||
int dy = 0;
|
||||
|
||||
if (!src_uv || src_width <= 0 || src_height == 0 || src_width > 32768 ||
|
||||
src_height < -32768 || src_height > 32768 || !dst_uv || dst_width <= 0 ||
|
||||
dst_height <= 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
// UV does not support box filter yet, but allow the user to pass it.
|
||||
// Simplify filtering when possible.
|
||||
filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height,
|
||||
filtering);
|
||||
|
||||
// Negative src_height means invert the image.
|
||||
if (src_height < 0) {
|
||||
src_height = -src_height;
|
||||
src_uv = src_uv + (src_height - 1) * (ptrdiff_t)src_stride_uv;
|
||||
src_stride_uv = -src_stride_uv;
|
||||
}
|
||||
|
||||
#ifdef HAS_UVCOPY
|
||||
if (!filtering && src_width == dst_width && (src_height % dst_height == 0)) {
|
||||
if (dst_height == 1) {
|
||||
return UVCopy_16(
|
||||
src_uv + ((src_height - 1) / 2) * (ptrdiff_t)src_stride_uv,
|
||||
src_stride_uv, dst_uv, dst_stride_uv, dst_width, dst_height);
|
||||
}
|
||||
dy = src_height / dst_height;
|
||||
if (src_stride_uv > INT_MAX / dy) {
|
||||
return -1;
|
||||
}
|
||||
return UVCopy_16(src_uv + ((dy - 1) / 2) * (ptrdiff_t)src_stride_uv,
|
||||
dy * src_stride_uv, dst_uv, dst_stride_uv, dst_width,
|
||||
dst_height);
|
||||
}
|
||||
#endif
|
||||
|
||||
// ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone, which has
|
||||
// no 16 bit implementation other than the copy above. The Up2 kernels only
|
||||
// read the pixels they need, so restore Linear or Bilinear for a 2x upscale
|
||||
// so P010ToP410 and P210ToP410 work on 1 pixel wide images.
|
||||
if (filtering == kFilterNone && src_width == 1 && dst_width <= 2) {
|
||||
filtering = (src_height == dst_height) ? kFilterLinear : kFilterBilinear;
|
||||
}
|
||||
|
||||
if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) {
|
||||
ScaleUVLinearUp2_16(src_width, src_height, dst_width, dst_height,
|
||||
src_stride_uv, dst_stride_uv, src_uv, dst_uv);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ((dst_height + 1) / 2 == src_height && (dst_width + 1) / 2 == src_width &&
|
||||
(filtering == kFilterBilinear || filtering == kFilterBox)) {
|
||||
ScaleUVBilinearUp2_16(src_width, src_height, dst_width, dst_height,
|
||||
src_stride_uv, dst_stride_uv, src_uv, dst_uv);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return -1;
|
||||
return ScaleUV_16(src_uv, src_stride_uv, src_width, src_height, dst_uv,
|
||||
dst_stride_uv, dst_width, dst_height, 0, 0, dst_width,
|
||||
dst_height, filtering);
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
2658
source/scale_win.cc
2658
source/scale_win.cc
File diff suppressed because it is too large
Load Diff
@ -208,6 +208,61 @@ TEST_F(LibYUVCompareTest, BenchmarkARGBDetect_Unaligned) {
|
||||
free_aligned_buffer_page_end(src_a);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVCompareTest, TestARGBDetectAnyWidth) {
|
||||
const int kMaxWidth = 65;
|
||||
const int kBufSize = kMaxWidth * 4 + 4;
|
||||
align_buffer_page_end(src_a, kBufSize);
|
||||
memset(src_a, 255, kBufSize);
|
||||
|
||||
for (int width = 1; width <= kMaxWidth; ++width) {
|
||||
for (int offset = 0; offset < 4; ++offset) {
|
||||
uint8_t* row = src_a + kBufSize - width * 4 - offset;
|
||||
// All 255 -> 0 (unknown).
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
uint32_t c_fourcc = ARGBDetect(row, width * 4, width, 1);
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
uint32_t opt_fourcc = ARGBDetect(row, width * 4, width, 1);
|
||||
EXPECT_EQ(0u, c_fourcc);
|
||||
EXPECT_EQ(c_fourcc, opt_fourcc) << "width " << width << " off " << offset;
|
||||
|
||||
for (int x = 0; x < width; ++x) {
|
||||
// Byte 0 != 255 -> FOURCC_BGRA.
|
||||
row[x * 4 + 0] = 128;
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
c_fourcc = ARGBDetect(row, width * 4, width, 1);
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
opt_fourcc = ARGBDetect(row, width * 4, width, 1);
|
||||
EXPECT_EQ(static_cast<uint32_t>(FOURCC_BGRA), c_fourcc);
|
||||
EXPECT_EQ(c_fourcc, opt_fourcc)
|
||||
<< "width " << width << " x " << x << " byte 0";
|
||||
|
||||
// Both byte 0 and byte 3 != 255 -> FOURCC_BGRA (byte 0 checked first).
|
||||
row[x * 4 + 3] = 64;
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
c_fourcc = ARGBDetect(row, width * 4, width, 1);
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
opt_fourcc = ARGBDetect(row, width * 4, width, 1);
|
||||
EXPECT_EQ(static_cast<uint32_t>(FOURCC_BGRA), c_fourcc);
|
||||
EXPECT_EQ(c_fourcc, opt_fourcc)
|
||||
<< "width " << width << " x " << x << " byte 0+3";
|
||||
|
||||
// Byte 3 != 255 only -> FOURCC_ARGB.
|
||||
row[x * 4 + 0] = 255;
|
||||
MaskCpuFlags(disable_cpu_flags_);
|
||||
c_fourcc = ARGBDetect(row, width * 4, width, 1);
|
||||
MaskCpuFlags(benchmark_cpu_info_);
|
||||
opt_fourcc = ARGBDetect(row, width * 4, width, 1);
|
||||
EXPECT_EQ(static_cast<uint32_t>(FOURCC_ARGB), c_fourcc);
|
||||
EXPECT_EQ(c_fourcc, opt_fourcc)
|
||||
<< "width " << width << " x " << x << " byte 3";
|
||||
row[x * 4 + 3] = 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end(src_a);
|
||||
}
|
||||
|
||||
#ifdef ENABLE_ROW_TESTS
|
||||
TEST_F(LibYUVCompareTest, BenchmarkHammingDistance_Opt) {
|
||||
const int kMaxWidth = 4096 * 3;
|
||||
|
||||
@ -207,6 +207,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) {
|
||||
int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8);
|
||||
int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8);
|
||||
int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM);
|
||||
int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32);
|
||||
printf("Has X86 0x%x\n", has_x86);
|
||||
printf("Has SSE2 0x%x\n", has_sse2);
|
||||
printf("Has SSSE3 0x%x\n", has_ssse3);
|
||||
@ -230,6 +231,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) {
|
||||
printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8);
|
||||
printf("Has AMXINT8 0x%x\n", has_amxint8);
|
||||
printf("Has AVX512BMM 0x%x\n", has_avx512bmm);
|
||||
printf("Has AMXTF32 0x%x\n", has_amxtf32);
|
||||
}
|
||||
#endif // defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) ||
|
||||
// defined(_M_X64)
|
||||
|
||||
@ -23,6 +23,11 @@
|
||||
#include "libyuv/rotate.h"
|
||||
#include "libyuv/scale.h"
|
||||
|
||||
#if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \
|
||||
!defined(ENABLE_ROW_TESTS)
|
||||
#define ENABLE_ROW_TESTS
|
||||
#endif
|
||||
|
||||
#ifdef ENABLE_ROW_TESTS
|
||||
// row.h defines SIMD_ALIGNED, overriding unit_test.h
|
||||
// TODO(fbarchard): Remove row.h from unittests. Test public functions.
|
||||
@ -1573,21 +1578,21 @@ TEST_F(LibYUVPlanarTest, TestAffine) {
|
||||
ASSERT_EQ(96u, interpolate_pixels_C[128][0]);
|
||||
ASSERT_EQ(191u, interpolate_pixels_C[255][3]);
|
||||
|
||||
#if defined(HAS_ARGBAFFINEROW_SSE2)
|
||||
int has_sse2 = TestCpuFlag(kCpuHasSSE2);
|
||||
if (has_sse2) {
|
||||
#if defined(HAS_ARGBAFFINEROW_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
if (has_avx2) {
|
||||
SIMD_ALIGNED(uint8_t interpolate_pixels_Opt[1280][4]);
|
||||
ARGBAffineRow_SSE2(&orig_pixels_0[0][0][0], stride,
|
||||
ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride,
|
||||
&interpolate_pixels_Opt[0][0], uv_step, 1280);
|
||||
ASSERT_EQ(0,
|
||||
memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1280 * 4));
|
||||
ARGBAffineRow_SSE2(&orig_pixels_0[0][0][0], stride,
|
||||
ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride,
|
||||
&interpolate_pixels_Opt[0][0], uv_step, 1279);
|
||||
ASSERT_EQ(0,
|
||||
memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1279 * 4));
|
||||
|
||||
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
|
||||
ARGBAffineRow_SSE2(&orig_pixels_0[0][0][0], stride,
|
||||
ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride,
|
||||
&interpolate_pixels_Opt[0][0], uv_step, 1280);
|
||||
}
|
||||
}
|
||||
@ -3998,10 +4003,10 @@ TESTTPLANARTOP(MergeXR30, uint16_t, uint8_t, 12)
|
||||
TESTTPLANARTOP(MergeXR30, uint16_t, uint8_t, 16)
|
||||
|
||||
// TODO(fbarchard): improve test for platforms and cpu detect
|
||||
#ifdef HAS_MERGEUVROW_16_AVX2
|
||||
#if defined(HAS_MERGEUVROW_16_AVX512BW) || defined(HAS_MERGEUVROW_16_AVX2)
|
||||
TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) {
|
||||
// Round count up to multiple of 8
|
||||
const int kPixels = (benchmark_width_ * benchmark_height_ + 7) & ~7;
|
||||
// Round count up to multiple of 16
|
||||
const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
|
||||
|
||||
align_buffer_page_end(src_pixels_u, kPixels * 2);
|
||||
align_buffer_page_end(src_pixels_v, kPixels * 2);
|
||||
@ -4017,14 +4022,30 @@ TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) {
|
||||
reinterpret_cast<const uint16_t*>(src_pixels_v),
|
||||
reinterpret_cast<uint16_t*>(dst_pixels_uv_c), 16, kPixels);
|
||||
|
||||
#if defined(HAS_MERGEUVROW_16_AVX512BW)
|
||||
int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_16_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
#endif
|
||||
for (int i = 0; i < benchmark_iterations_; ++i) {
|
||||
if (has_avx2) {
|
||||
#if defined(HAS_MERGEUVROW_16_AVX512BW)
|
||||
if (has_avx512) {
|
||||
MergeUVRow_16_AVX512BW(reinterpret_cast<const uint16_t*>(src_pixels_u),
|
||||
reinterpret_cast<const uint16_t*>(src_pixels_v),
|
||||
reinterpret_cast<uint16_t*>(dst_pixels_uv_opt), 16,
|
||||
kPixels);
|
||||
} else
|
||||
#endif
|
||||
#if defined(HAS_MERGEUVROW_16_AVX2)
|
||||
if (has_avx2) {
|
||||
MergeUVRow_16_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_u),
|
||||
reinterpret_cast<const uint16_t*>(src_pixels_v),
|
||||
reinterpret_cast<uint16_t*>(dst_pixels_uv_opt), 16,
|
||||
kPixels);
|
||||
} else {
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
MergeUVRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_u),
|
||||
reinterpret_cast<const uint16_t*>(src_pixels_v),
|
||||
reinterpret_cast<uint16_t*>(dst_pixels_uv_opt), 16,
|
||||
@ -4331,10 +4352,10 @@ TEST_F(LibYUVPlanarTest, UYVYToY) {
|
||||
|
||||
#ifdef ENABLE_ROW_TESTS
|
||||
// TODO(fbarchard): Improve test for more platforms.
|
||||
#ifdef HAS_CONVERT16TO8ROW_AVX2
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX512BW) || defined(HAS_CONVERT16TO8ROW_AVX2)
|
||||
TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) {
|
||||
// AVX2 does multiple of 32, so round count up
|
||||
const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31;
|
||||
// AVX512 does multiple of 64, so round count up
|
||||
const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63;
|
||||
align_buffer_page_end(src_pixels_y, kPixels * 2);
|
||||
align_buffer_page_end(dst_pixels_y_opt, kPixels);
|
||||
align_buffer_page_end(dst_pixels_y_c, kPixels);
|
||||
@ -4351,16 +4372,26 @@ TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) {
|
||||
Convert16To8Row_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
|
||||
dst_pixels_y_c, 16384, kPixels);
|
||||
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
|
||||
int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
|
||||
#endif
|
||||
for (int i = 0; i < benchmark_iterations_; ++i) {
|
||||
if (has_avx2) {
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
|
||||
if (has_avx512) {
|
||||
Convert16To8Row_AVX512BW(reinterpret_cast<const uint16_t*>(src_pixels_y),
|
||||
dst_pixels_y_opt, 16384, kPixels);
|
||||
} else
|
||||
#endif
|
||||
#if defined(HAS_CONVERT16TO8ROW_AVX2)
|
||||
if (has_avx2) {
|
||||
Convert16To8Row_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_y),
|
||||
dst_pixels_y_opt, 16384, kPixels);
|
||||
} else if (has_ssse3) {
|
||||
Convert16To8Row_SSSE3(reinterpret_cast<const uint16_t*>(src_pixels_y),
|
||||
dst_pixels_y_opt, 16384, kPixels);
|
||||
} else {
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
Convert16To8Row_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
|
||||
dst_pixels_y_opt, 16384, kPixels);
|
||||
}
|
||||
@ -4374,7 +4405,7 @@ TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) {
|
||||
free_aligned_buffer_page_end(dst_pixels_y_opt);
|
||||
free_aligned_buffer_page_end(dst_pixels_y_c);
|
||||
}
|
||||
#endif // HAS_CONVERT16TO8ROW_AVX2
|
||||
#endif // HAS_CONVERT16TO8ROW_AVX512BW || HAS_CONVERT16TO8ROW_AVX2
|
||||
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) || \
|
||||
defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \
|
||||
@ -4481,8 +4512,8 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) {
|
||||
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
|
||||
if (has_avx512) {
|
||||
HalfWidthRow_16To8_Any_AVX512BW(src_pixels_uv, kMaxPixels * 2,
|
||||
dst_pixels_uv_opt, 16384, width);
|
||||
HalfWidthRow_16To8_AVX512BW(src_pixels_uv, kMaxPixels * 2,
|
||||
dst_pixels_uv_opt, 16384, width);
|
||||
} else
|
||||
#endif
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
|
||||
@ -4519,10 +4550,11 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) {
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAS_UYVYTOYROW_NEON
|
||||
#if defined(HAS_UYVYTOYROW_AVX512BW) || defined(HAS_UYVYTOYROW_AVX2) || \
|
||||
defined(HAS_UYVYTOYROW_NEON)
|
||||
TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) {
|
||||
// NEON does multiple of 16, so round count up
|
||||
const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15;
|
||||
// AVX2/AVX512 does multiple of 32, so round count up
|
||||
const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31;
|
||||
align_buffer_page_end(src_pixels_y, kPixels * 2);
|
||||
align_buffer_page_end(dst_pixels_y_opt, kPixels);
|
||||
align_buffer_page_end(dst_pixels_y_c, kPixels);
|
||||
@ -4533,8 +4565,34 @@ TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) {
|
||||
|
||||
UYVYToYRow_C(src_pixels_y, dst_pixels_y_c, kPixels);
|
||||
|
||||
#if defined(HAS_UYVYTOYROW_AVX512BW)
|
||||
int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_NEON)
|
||||
int has_neon = TestCpuFlag(kCpuHasNEON);
|
||||
#endif
|
||||
for (int i = 0; i < benchmark_iterations_; ++i) {
|
||||
UYVYToYRow_NEON(src_pixels_y, dst_pixels_y_opt, kPixels);
|
||||
#if defined(HAS_UYVYTOYROW_AVX512BW)
|
||||
if (has_avx512) {
|
||||
UYVYToYRow_AVX512BW(src_pixels_y, dst_pixels_y_opt, kPixels);
|
||||
} else
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_AVX2)
|
||||
if (has_avx2) {
|
||||
UYVYToYRow_AVX2(src_pixels_y, dst_pixels_y_opt, kPixels);
|
||||
} else
|
||||
#endif
|
||||
#if defined(HAS_UYVYTOYROW_NEON)
|
||||
if (has_neon) {
|
||||
UYVYToYRow_NEON(src_pixels_y, dst_pixels_y_opt, kPixels);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
UYVYToYRow_C(src_pixels_y, dst_pixels_y_opt, kPixels);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < kPixels; ++i) {
|
||||
@ -4545,7 +4603,99 @@ TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) {
|
||||
free_aligned_buffer_page_end(dst_pixels_y_opt);
|
||||
free_aligned_buffer_page_end(dst_pixels_y_c);
|
||||
}
|
||||
#endif // HAS_UYVYTOYROW_NEON
|
||||
#endif // HAS_UYVYTOYROW_...
|
||||
|
||||
#if defined(HAS_RGB565TOYROW_AVX2) && defined(HAS_ARGB1555TOYROW_AVX2) && \
|
||||
defined(HAS_ARGB4444TOYROW_AVX2)
|
||||
TEST_F(LibYUVPlanarTest, RGB16ToYUVRow_Opt) {
|
||||
if (!TestCpuFlag(kCpuHasAVX2)) {
|
||||
return;
|
||||
}
|
||||
const int kMaxWidth = 128;
|
||||
SIMD_ALIGNED(uint8_t src0[kMaxWidth * 2]);
|
||||
SIMD_ALIGNED(uint8_t src1[kMaxWidth * 2]);
|
||||
SIMD_ALIGNED(uint8_t dst_y_c[kMaxWidth]);
|
||||
SIMD_ALIGNED(uint8_t dst_y_opt[kMaxWidth]);
|
||||
SIMD_ALIGNED(uint8_t dst_u_c[kMaxWidth]);
|
||||
SIMD_ALIGNED(uint8_t dst_v_c[kMaxWidth]);
|
||||
SIMD_ALIGNED(uint8_t dst_u_opt[kMaxWidth]);
|
||||
SIMD_ALIGNED(uint8_t dst_v_opt[kMaxWidth]);
|
||||
MemRandomize(src0, sizeof(src0));
|
||||
MemRandomize(src1, sizeof(src1));
|
||||
|
||||
for (int width : {1, 2, 7, 15, 16, 17, 31, 32, 33, 64, 127, 128}) {
|
||||
const int halfwidth = (width + 1) / 2;
|
||||
const int stride = kMaxWidth * 2;
|
||||
|
||||
RGB565ToYRow_C(src0, dst_y_c, width);
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
RGB565ToYRow_AVX2(src0, dst_y_opt, width);
|
||||
} else {
|
||||
RGB565ToYRow_Any_AVX2(src0, dst_y_opt, width);
|
||||
}
|
||||
for (int i = 0; i < width; ++i) {
|
||||
ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1);
|
||||
}
|
||||
|
||||
ARGB1555ToYRow_C(src0, dst_y_c, width);
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
ARGB1555ToYRow_AVX2(src0, dst_y_opt, width);
|
||||
} else {
|
||||
ARGB1555ToYRow_Any_AVX2(src0, dst_y_opt, width);
|
||||
}
|
||||
for (int i = 0; i < width; ++i) {
|
||||
ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1);
|
||||
}
|
||||
|
||||
ARGB4444ToYRow_C(src0, dst_y_c, width);
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
ARGB4444ToYRow_AVX2(src0, dst_y_opt, width);
|
||||
} else {
|
||||
ARGB4444ToYRow_Any_AVX2(src0, dst_y_opt, width);
|
||||
}
|
||||
for (int i = 0; i < width; ++i) {
|
||||
ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1);
|
||||
}
|
||||
|
||||
SIMD_ALIGNED(uint8_t src2rows[kMaxWidth * 2 * 2]);
|
||||
memcpy(src2rows, src0, stride);
|
||||
memcpy(src2rows + stride, src1, stride);
|
||||
|
||||
RGB565ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width);
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
RGB565ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
|
||||
} else {
|
||||
RGB565ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
|
||||
}
|
||||
for (int i = 0; i < halfwidth; ++i) {
|
||||
ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2);
|
||||
ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2);
|
||||
}
|
||||
|
||||
ARGB1555ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width);
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
ARGB1555ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
|
||||
} else {
|
||||
ARGB1555ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
|
||||
}
|
||||
for (int i = 0; i < halfwidth; ++i) {
|
||||
ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2);
|
||||
ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2);
|
||||
}
|
||||
|
||||
ARGB4444ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width);
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
ARGB4444ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
|
||||
} else {
|
||||
ARGB4444ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width);
|
||||
}
|
||||
for (int i = 0; i < halfwidth; ++i) {
|
||||
ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2);
|
||||
ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // ENABLE_ROW_TESTS
|
||||
|
||||
@ -4601,7 +4751,6 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) {
|
||||
int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
|
||||
#endif
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
int has_sse2 = TestCpuFlag(kCpuHasSSE2);
|
||||
for (int i = 0; i < benchmark_iterations_; ++i) {
|
||||
#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
|
||||
if (has_avx512) {
|
||||
@ -4614,10 +4763,6 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) {
|
||||
Convert8To16Row_AVX2(src_pixels_y,
|
||||
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 10,
|
||||
kPixels);
|
||||
} else if (has_sse2) {
|
||||
Convert8To16Row_SSE2(src_pixels_y,
|
||||
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 10,
|
||||
kPixels);
|
||||
} else {
|
||||
Convert8To16Row_C(src_pixels_y,
|
||||
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 10,
|
||||
@ -4666,6 +4811,16 @@ float TestScaleMaxSamples(int benchmark_width,
|
||||
max_opt = ScaleMaxSamples_NEON(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale,
|
||||
kPixels);
|
||||
#elif defined(HAS_SCALEMAXSAMPLES_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
max_opt = ScaleMaxSamples_AVX2(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale,
|
||||
kPixels);
|
||||
} else {
|
||||
max_opt = ScaleMaxSamples_C(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale,
|
||||
kPixels);
|
||||
}
|
||||
#else
|
||||
max_opt =
|
||||
ScaleMaxSamples_C(reinterpret_cast<float*>(orig_y),
|
||||
@ -4734,6 +4889,16 @@ float TestScaleSumSamples(int benchmark_width,
|
||||
sum_opt = ScaleSumSamples_NEON(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale,
|
||||
kPixels);
|
||||
#elif defined(HAS_SCALESUMSAMPLES_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
sum_opt = ScaleSumSamples_AVX2(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale,
|
||||
kPixels);
|
||||
} else {
|
||||
sum_opt = ScaleSumSamples_C(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale,
|
||||
kPixels);
|
||||
}
|
||||
#else
|
||||
sum_opt =
|
||||
ScaleSumSamples_C(reinterpret_cast<float*>(orig_y),
|
||||
@ -4811,6 +4976,14 @@ float TestScaleSamples(int benchmark_width,
|
||||
#ifdef HAS_SCALESUMSAMPLES_NEON
|
||||
ScaleSamples_NEON(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale, kPixels);
|
||||
#elif defined(HAS_SCALESAMPLES_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
ScaleSamples_AVX2(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale, kPixels);
|
||||
} else {
|
||||
ScaleSamples_C(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale, kPixels);
|
||||
}
|
||||
#else
|
||||
ScaleSamples_C(reinterpret_cast<float*>(orig_y),
|
||||
reinterpret_cast<float*>(dst_opt), scale, kPixels);
|
||||
@ -4923,7 +5096,14 @@ TEST_F(LibYUVPlanarTest, TestGaussRow_Opt) {
|
||||
}
|
||||
GaussRow_C(&orig_pixels[0], &dst_pixels_c[0], 1280);
|
||||
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && \
|
||||
#if defined(HAS_GAUSSROW_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
if (has_avx2) {
|
||||
GaussRow_AVX2(&orig_pixels[0], &dst_pixels_opt[0], 1280);
|
||||
} else {
|
||||
GaussRow_C(&orig_pixels[0], &dst_pixels_opt[0], 1280);
|
||||
}
|
||||
#elif !defined(LIBYUV_DISABLE_NEON) && \
|
||||
(defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON))
|
||||
int has_neon = TestCpuFlag(kCpuHasNEON);
|
||||
if (has_neon) {
|
||||
@ -4977,7 +5157,18 @@ TEST_F(LibYUVPlanarTest, TestGaussCol_Opt) {
|
||||
&orig_pixels[1280 * 3], &orig_pixels[1280 * 4], &dst_pixels_c[0],
|
||||
1280);
|
||||
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && \
|
||||
#if defined(HAS_GAUSSCOL_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
if (has_avx2) {
|
||||
GaussCol_AVX2(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
|
||||
&orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
|
||||
&dst_pixels_opt[0], 1280);
|
||||
} else {
|
||||
GaussCol_C(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2],
|
||||
&orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
|
||||
&dst_pixels_opt[0], 1280);
|
||||
}
|
||||
#elif !defined(LIBYUV_DISABLE_NEON) && \
|
||||
(defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON))
|
||||
int has_neon = TestCpuFlag(kCpuHasNEON);
|
||||
if (has_neon) {
|
||||
@ -5015,7 +5206,14 @@ TEST_F(LibYUVPlanarTest, TestGaussRow_F32_Opt) {
|
||||
}
|
||||
GaussRow_F32_C(&orig_pixels[0], &dst_pixels_c[0], 1280);
|
||||
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
#if defined(HAS_GAUSSROW_F32_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
if (has_avx2) {
|
||||
GaussRow_F32_AVX2(&orig_pixels[0], &dst_pixels_opt[0], 1280);
|
||||
} else {
|
||||
GaussRow_F32_C(&orig_pixels[0], &dst_pixels_opt[0], 1280);
|
||||
}
|
||||
#elif !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
int has_neon = TestCpuFlag(kCpuHasNEON);
|
||||
if (has_neon) {
|
||||
GaussRow_F32_NEON(&orig_pixels[0], &dst_pixels_opt[0], 1280);
|
||||
@ -5049,7 +5247,18 @@ TEST_F(LibYUVPlanarTest, TestGaussCol_F32_Opt) {
|
||||
&orig_pixels[1280 * 3], &orig_pixels[1280 * 4],
|
||||
&dst_pixels_c[0], 1280);
|
||||
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
#if defined(HAS_GAUSSCOL_F32_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
if (has_avx2) {
|
||||
GaussCol_F32_AVX2(&orig_pixels[0], &orig_pixels[1280],
|
||||
&orig_pixels[1280 * 2], &orig_pixels[1280 * 3],
|
||||
&orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280);
|
||||
} else {
|
||||
GaussCol_F32_C(&orig_pixels[0], &orig_pixels[1280],
|
||||
&orig_pixels[1280 * 2], &orig_pixels[1280 * 3],
|
||||
&orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280);
|
||||
}
|
||||
#elif !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
int has_neon = TestCpuFlag(kCpuHasNEON);
|
||||
if (has_neon) {
|
||||
GaussCol_F32_NEON(&orig_pixels[0], &orig_pixels[1280],
|
||||
@ -5129,6 +5338,151 @@ TEST_F(LibYUVPlanarTest, SwapUVRow) {
|
||||
free_aligned_buffer_page_end(src_pixels_vu);
|
||||
free_aligned_buffer_page_end(dst_pixels_uv);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVPlanarTest, HalfRow_Opt) {
|
||||
const int kMaxPixels = 160;
|
||||
align_buffer_page_end(src_pixels, kMaxPixels * 2);
|
||||
align_buffer_page_end(dst_pixels_opt, kMaxPixels);
|
||||
align_buffer_page_end(dst_pixels_c, kMaxPixels);
|
||||
MemRandomize(src_pixels, kMaxPixels * 2);
|
||||
|
||||
for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels}}) {
|
||||
for (int width = 1; width <= 129; ++width) {
|
||||
memset(dst_pixels_opt, 0, kMaxPixels);
|
||||
memset(dst_pixels_c, 1, kMaxPixels);
|
||||
HalfRow_C(src_pixels, stride, dst_pixels_c, width);
|
||||
#if defined(HAS_HALFROW_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
HalfRow_AVX2(src_pixels, stride, dst_pixels_opt, width);
|
||||
} else {
|
||||
HalfRow_Any_AVX2(src_pixels, stride, dst_pixels_opt, width);
|
||||
}
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
HalfRow_C(src_pixels, stride, dst_pixels_opt, width);
|
||||
}
|
||||
ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width))
|
||||
<< "stride=" << stride << " width=" << width;
|
||||
}
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end(src_pixels);
|
||||
free_aligned_buffer_page_end(dst_pixels_opt);
|
||||
free_aligned_buffer_page_end(dst_pixels_c);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVPlanarTest, HalfRow_16_Opt) {
|
||||
const int kMaxPixels = 160;
|
||||
align_buffer_page_end_16(src_pixels, kMaxPixels * 2);
|
||||
align_buffer_page_end_16(dst_pixels_opt, kMaxPixels);
|
||||
align_buffer_page_end_16(dst_pixels_c, kMaxPixels);
|
||||
for (int i = 0; i < kMaxPixels * 2; ++i) {
|
||||
src_pixels[i] = static_cast<uint16_t>(fastrand());
|
||||
}
|
||||
|
||||
for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels}}) {
|
||||
for (int width = 1; width <= 129; ++width) {
|
||||
memset(dst_pixels_opt, 0, kMaxPixels * 2);
|
||||
memset(dst_pixels_c, 1, kMaxPixels * 2);
|
||||
HalfRow_16_C(src_pixels, stride, dst_pixels_c, width);
|
||||
#if defined(HAS_HALFROW_16_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
HalfRow_16_AVX2(src_pixels, stride, dst_pixels_opt, width);
|
||||
} else {
|
||||
HalfRow_16_Any_AVX2(src_pixels, stride, dst_pixels_opt, width);
|
||||
}
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
HalfRow_16_C(src_pixels, stride, dst_pixels_opt, width);
|
||||
}
|
||||
ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width * 2))
|
||||
<< "stride=" << stride << " width=" << width;
|
||||
}
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end_16(src_pixels);
|
||||
free_aligned_buffer_page_end_16(dst_pixels_opt);
|
||||
free_aligned_buffer_page_end_16(dst_pixels_c);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Odd_Opt) {
|
||||
const int kMaxPixels = 160;
|
||||
align_buffer_page_end_16(src_pixels, kMaxPixels * 4);
|
||||
align_buffer_page_end(dst_pixels_opt, kMaxPixels);
|
||||
align_buffer_page_end(dst_pixels_c, kMaxPixels);
|
||||
for (int i = 0; i < kMaxPixels * 4; ++i) {
|
||||
src_pixels[i] = fastrand() & 1023;
|
||||
}
|
||||
|
||||
for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels * 2}}) {
|
||||
for (int width = 1; width <= 129; ++width) {
|
||||
memset(dst_pixels_opt, 0, kMaxPixels);
|
||||
memset(dst_pixels_c, 1, kMaxPixels);
|
||||
HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_c, 16384, width);
|
||||
#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
HalfWidthRow_16To8_Odd_AVX2(src_pixels, stride, dst_pixels_opt, 16384,
|
||||
width);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_opt, 16384,
|
||||
width);
|
||||
}
|
||||
ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width))
|
||||
<< "stride=" << stride << " width=" << width;
|
||||
}
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end_16(src_pixels);
|
||||
free_aligned_buffer_page_end(dst_pixels_opt);
|
||||
free_aligned_buffer_page_end(dst_pixels_c);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVPlanarTest, ARGBChannelReorderRow_C_Test) {
|
||||
const int kPixels = 65;
|
||||
align_buffer_page_end(src_argb, kPixels * 4);
|
||||
align_buffer_page_end(dst_row, kPixels * 4);
|
||||
align_buffer_page_end(dst_api, kPixels * 4);
|
||||
MemRandomize(src_argb, kPixels * 4);
|
||||
|
||||
ARGBToABGRRow_C(src_argb, dst_row, kPixels);
|
||||
ARGBToABGR(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1);
|
||||
ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4));
|
||||
|
||||
ARGBToBGRARow_C(src_argb, dst_row, kPixels);
|
||||
ARGBToBGRA(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1);
|
||||
ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4));
|
||||
|
||||
ARGBToRGBARow_C(src_argb, dst_row, kPixels);
|
||||
ARGBToRGBA(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1);
|
||||
ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4));
|
||||
|
||||
RGBAToARGBRow_C(src_argb, dst_row, kPixels);
|
||||
RGBAToARGB(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1);
|
||||
ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4));
|
||||
|
||||
align_buffer_page_end_16(src_ar64, kPixels * 4);
|
||||
align_buffer_page_end_16(dst_ab64_row, kPixels * 4);
|
||||
align_buffer_page_end_16(dst_ab64_api, kPixels * 4);
|
||||
for (int i = 0; i < kPixels * 4; ++i) {
|
||||
src_ar64[i] = static_cast<uint16_t>(fastrand());
|
||||
}
|
||||
AR64ToAB64Row_C(src_ar64, dst_ab64_row, kPixels);
|
||||
AR64ToAB64(src_ar64, kPixels * 4, dst_ab64_api, kPixels * 4, kPixels, 1);
|
||||
ASSERT_EQ(0, memcmp(dst_ab64_row, dst_ab64_api, kPixels * 8));
|
||||
|
||||
free_aligned_buffer_page_end_16(src_ar64);
|
||||
free_aligned_buffer_page_end_16(dst_ab64_row);
|
||||
free_aligned_buffer_page_end_16(dst_ab64_api);
|
||||
free_aligned_buffer_page_end(src_argb);
|
||||
free_aligned_buffer_page_end(dst_row);
|
||||
free_aligned_buffer_page_end(dst_api);
|
||||
}
|
||||
#endif // ENABLE_ROW_TESTS
|
||||
|
||||
TEST_F(LibYUVPlanarTest, TestGaussPlane_F32) {
|
||||
|
||||
@ -14,6 +14,11 @@
|
||||
#include "libyuv/cpu_id.h"
|
||||
#include "libyuv/rotate.h"
|
||||
|
||||
#if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \
|
||||
!defined(ENABLE_ROW_TESTS)
|
||||
#define ENABLE_ROW_TESTS
|
||||
#endif
|
||||
|
||||
#ifdef ENABLE_ROW_TESTS
|
||||
#include "libyuv/rotate_row.h"
|
||||
#endif
|
||||
@ -892,9 +897,9 @@ TEST_F(LibYUVRotateTest, TransposeWx4_32_Test) {
|
||||
TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
} else
|
||||
#elif defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
TransposeWx4_32_SSE2((const uint8_t*)src_pixels, height * 4,
|
||||
#elif defined(HAS_TRANSPOSEWX4_32_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
TransposeWx4_32_AVX2((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
} else
|
||||
#endif
|
||||
@ -933,9 +938,9 @@ TEST_F(LibYUVRotateTest, TransposeWx4_32_Opt) {
|
||||
TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
} else
|
||||
#elif defined(HAS_TRANSPOSEWX4_32_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
TransposeWx4_32_SSE2((const uint8_t*)src_pixels, height * 4,
|
||||
#elif defined(HAS_TRANSPOSEWX4_32_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
TransposeWx4_32_AVX2((const uint8_t*)src_pixels, height * 4,
|
||||
(uint8_t*)dst_pixels_opt, width * 4, width);
|
||||
} else
|
||||
#endif
|
||||
@ -954,6 +959,97 @@ TEST_F(LibYUVRotateTest, TransposeWx4_32_Opt) {
|
||||
free_aligned_buffer_page_end(dst_pixels_opt);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVRotateTest, TransposeWx4_32_Any) {
|
||||
for (int width : {4, 8, 12, 16, 20, 28, 36}) {
|
||||
const int height = 4;
|
||||
align_buffer_page_end(src_pixels, height * width * 4);
|
||||
align_buffer_page_end(dst_pixels_c, width * height * 4);
|
||||
align_buffer_page_end(dst_pixels_opt, width * height * 4);
|
||||
|
||||
MemRandomize(src_pixels, height * width * 4);
|
||||
memset(dst_pixels_c, 1, width * height * 4);
|
||||
memset(dst_pixels_opt, 2, width * height * 4);
|
||||
|
||||
TransposeWx4_32_C(src_pixels, height * 4, dst_pixels_c, width * 4, width);
|
||||
#if defined(HAS_TRANSPOSEWX4_32_AVX2)
|
||||
if (TestCpuFlag(kCpuHasAVX2)) {
|
||||
if ((width & 7) == 0) {
|
||||
TransposeWx4_32_AVX2(src_pixels, height * 4, dst_pixels_opt, width * 4,
|
||||
width);
|
||||
} else {
|
||||
TransposeWx4_32_Any_AVX2(src_pixels, height * 4, dst_pixels_opt,
|
||||
width * 4, width);
|
||||
}
|
||||
} else
|
||||
#endif
|
||||
#if defined(HAS_TRANSPOSEWX4_32_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
TransposeWx4_32_NEON(src_pixels, height * 4, dst_pixels_opt, width * 4,
|
||||
width);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
TransposeWx4_32_C(src_pixels, height * 4, dst_pixels_opt, width * 4,
|
||||
width);
|
||||
}
|
||||
|
||||
ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width * height * 4))
|
||||
<< "width=" << width;
|
||||
|
||||
free_aligned_buffer_page_end(src_pixels);
|
||||
free_aligned_buffer_page_end(dst_pixels_c);
|
||||
free_aligned_buffer_page_end(dst_pixels_opt);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
TEST_F(LibYUVRotateTest, TransposeWxH_32_Test) {
|
||||
if (!TestCpuFlag(kCpuHasAVX2)) {
|
||||
return;
|
||||
}
|
||||
const int kMaxW = 35;
|
||||
const int kMaxH = 25;
|
||||
align_buffer_page_end(src_pixels, kMaxW * kMaxH * 4);
|
||||
align_buffer_page_end(dst_pixels_c, kMaxW * kMaxH * 4);
|
||||
align_buffer_page_end(dst_pixels_opt, kMaxW * kMaxH * 4);
|
||||
MemRandomize(src_pixels, kMaxW * kMaxH * 4);
|
||||
|
||||
for (int h = 1; h <= kMaxH; ++h) {
|
||||
for (int w = 1; w <= kMaxW; ++w) {
|
||||
const int src_stride = w * 4;
|
||||
const int dst_stride = h * 4;
|
||||
const int src_bytes = src_stride * h;
|
||||
const int dst_bytes = dst_stride * w;
|
||||
const uint8_t* src = src_pixels + kMaxW * kMaxH * 4 - src_bytes;
|
||||
uint8_t* dst_c = dst_pixels_c + kMaxW * kMaxH * 4 - dst_bytes;
|
||||
uint8_t* dst_opt = dst_pixels_opt + kMaxW * kMaxH * 4 - dst_bytes;
|
||||
|
||||
memset(dst_c, 1, dst_bytes);
|
||||
memset(dst_opt, 2, dst_bytes);
|
||||
TransposeWxH_32_C(src, src_stride, dst_c, dst_stride, w, h);
|
||||
TransposeWxH_32_AVX2(src, src_stride, dst_opt, dst_stride, w, h);
|
||||
ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << w << "x" << h;
|
||||
|
||||
// Also test negative src_stride (90-degree) and negative dst_stride
|
||||
// (270-degree).
|
||||
memset(dst_c, 3, dst_bytes);
|
||||
memset(dst_opt, 4, dst_bytes);
|
||||
TransposeWxH_32_C(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
|
||||
dst_c + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride, w,
|
||||
h);
|
||||
TransposeWxH_32_AVX2(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
|
||||
dst_opt + (ptrdiff_t)(w - 1) * dst_stride,
|
||||
-dst_stride, w, h);
|
||||
ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << "neg " << w << "x" << h;
|
||||
}
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end(src_pixels);
|
||||
free_aligned_buffer_page_end(dst_pixels_c);
|
||||
free_aligned_buffer_page_end(dst_pixels_opt);
|
||||
}
|
||||
#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
|
||||
|
||||
#endif // ENABLE_ROW_TESTS
|
||||
|
||||
} // namespace libyuv
|
||||
|
||||
@ -267,6 +267,13 @@ TEST_F(LibYUVScaleTest, TestScaleRowDown2Box_16) {
|
||||
} else {
|
||||
ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
|
||||
}
|
||||
#elif defined(HAS_SCALEROWDOWN2_16_AVX2)
|
||||
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
|
||||
if (has_avx2) {
|
||||
ScaleRowDown2Box_16_AVX2(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
|
||||
} else {
|
||||
ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
|
||||
}
|
||||
#else
|
||||
ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280);
|
||||
#endif
|
||||
|
||||
@ -39,6 +39,9 @@ static int UVTestFilter(int src_width,
|
||||
if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
|
||||
return 0;
|
||||
}
|
||||
if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int i;
|
||||
int64_t src_uv_plane_size = Abs(src_width) * Abs(src_height) * 2LL;
|
||||
@ -259,6 +262,9 @@ static int UVTestBoxAgainstPlanes(int src_width,
|
||||
if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
|
||||
return 0;
|
||||
}
|
||||
if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) {
|
||||
return 0;
|
||||
}
|
||||
int64_t src_uv_plane_size = src_width * src_height * 2LL;
|
||||
int64_t src_plane_size = src_width * src_height * 1LL;
|
||||
int64_t dst_uv_plane_size = dst_width * dst_height * 2LL;
|
||||
@ -369,4 +375,283 @@ TEST_F(LibYUVScaleTest, UVScaleBoxAverages) {
|
||||
free_aligned_buffer_page_end(src_uv);
|
||||
}
|
||||
|
||||
// Test 16-bit UVScale_16 with C vs Opt and return maximum pixel difference.
|
||||
static int UVTestFilter_16(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
FilterMode f,
|
||||
int benchmark_iterations,
|
||||
int disable_cpu_flags,
|
||||
int benchmark_cpu_info) {
|
||||
if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
|
||||
return 0;
|
||||
}
|
||||
if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int i;
|
||||
int64_t src_uv_samples = Abs(src_width) * Abs(src_height) * 2LL;
|
||||
int src_stride_uv = Abs(src_width) * 2;
|
||||
int64_t dst_uv_samples = dst_width * dst_height * 2LL;
|
||||
int dst_stride_uv = dst_width * 2;
|
||||
|
||||
align_buffer_page_end(src_uv_buf, src_uv_samples * 2);
|
||||
align_buffer_page_end(dst_uv_c_buf, dst_uv_samples * 2);
|
||||
align_buffer_page_end(dst_uv_opt_buf, dst_uv_samples * 2);
|
||||
|
||||
if (!src_uv_buf || !dst_uv_c_buf || !dst_uv_opt_buf) {
|
||||
printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
|
||||
return 0;
|
||||
}
|
||||
uint16_t* src_uv = reinterpret_cast<uint16_t*>(src_uv_buf);
|
||||
uint16_t* dst_uv_c = reinterpret_cast<uint16_t*>(dst_uv_c_buf);
|
||||
uint16_t* dst_uv_opt = reinterpret_cast<uint16_t*>(dst_uv_opt_buf);
|
||||
|
||||
MemRandomize(src_uv_buf, src_uv_samples * 2);
|
||||
memset(dst_uv_c_buf, 2, dst_uv_samples * 2);
|
||||
memset(dst_uv_opt_buf, 123, dst_uv_samples * 2);
|
||||
|
||||
MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
|
||||
int rc_c = UVScale_16(src_uv, src_stride_uv, src_width, src_height, dst_uv_c,
|
||||
dst_stride_uv, dst_width, dst_height, f);
|
||||
if (rc_c != 0) {
|
||||
free_aligned_buffer_page_end(dst_uv_c_buf);
|
||||
free_aligned_buffer_page_end(dst_uv_opt_buf);
|
||||
free_aligned_buffer_page_end(src_uv_buf);
|
||||
return 65536;
|
||||
}
|
||||
|
||||
MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
|
||||
int rc_opt = 0;
|
||||
for (i = 0; i < benchmark_iterations; ++i) {
|
||||
rc_opt |= UVScale_16(src_uv, src_stride_uv, src_width, src_height,
|
||||
dst_uv_opt, dst_stride_uv, dst_width, dst_height, f);
|
||||
}
|
||||
if (rc_opt != 0) {
|
||||
free_aligned_buffer_page_end(dst_uv_c_buf);
|
||||
free_aligned_buffer_page_end(dst_uv_opt_buf);
|
||||
free_aligned_buffer_page_end(src_uv_buf);
|
||||
return 65536;
|
||||
}
|
||||
|
||||
int max_diff = 0;
|
||||
for (i = 0; i < dst_uv_samples; ++i) {
|
||||
int abs_diff = Abs(static_cast<int>(dst_uv_c[i]) -
|
||||
static_cast<int>(dst_uv_opt[i]));
|
||||
if (abs_diff > max_diff) {
|
||||
max_diff = abs_diff;
|
||||
}
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end(dst_uv_c_buf);
|
||||
free_aligned_buffer_page_end(dst_uv_opt_buf);
|
||||
free_aligned_buffer_page_end(src_uv_buf);
|
||||
return max_diff;
|
||||
}
|
||||
|
||||
#define DX(x, nom, denom) static_cast<int>((Abs(x) / nom) * nom)
|
||||
#define SX(x, nom, denom) static_cast<int>((x / nom) * denom)
|
||||
|
||||
#define TEST_FACTOR1_16(name, filter, nom, denom) \
|
||||
TEST_F(LibYUVScaleTest, UVScale_16_DownBy##name##_##filter) { \
|
||||
int diff = UVTestFilter_16( \
|
||||
SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
|
||||
DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
|
||||
kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
|
||||
benchmark_cpu_info_); \
|
||||
ASSERT_EQ(0, diff); \
|
||||
}
|
||||
|
||||
#if defined(ENABLE_FULL_TESTS)
|
||||
#define TEST_FACTOR_16(name, nom, denom) \
|
||||
TEST_FACTOR1_16(name, None, nom, denom) \
|
||||
TEST_FACTOR1_16(name, Linear, nom, denom) \
|
||||
TEST_FACTOR1_16(name, Bilinear, nom, denom) \
|
||||
TEST_FACTOR1_16(name, Box, nom, denom)
|
||||
#else
|
||||
#define TEST_FACTOR_16(name, nom, denom) \
|
||||
TEST_FACTOR1_16(name, Bilinear, nom, denom)
|
||||
#endif
|
||||
|
||||
TEST_FACTOR_16(2, 1, 2)
|
||||
TEST_FACTOR_16(4, 1, 4)
|
||||
TEST_FACTOR_16(3by4, 3, 4)
|
||||
TEST_FACTOR_16(3by8, 3, 8)
|
||||
TEST_FACTOR_16(3, 1, 3)
|
||||
#undef TEST_FACTOR1_16
|
||||
#undef TEST_FACTOR_16
|
||||
#undef SX
|
||||
#undef DX
|
||||
|
||||
#define TEST_SCALETO1_16(name, width, height, filter, max_diff) \
|
||||
TEST_F(LibYUVScaleTest, name##To##width##x##height##_##filter) { \
|
||||
int diff = UVTestFilter_16(benchmark_width_, benchmark_height_, width, \
|
||||
height, kFilter##filter, benchmark_iterations_, \
|
||||
disable_cpu_flags_, benchmark_cpu_info_); \
|
||||
ASSERT_LE(diff, max_diff); \
|
||||
} \
|
||||
TEST_F(LibYUVScaleTest, name##From##width##x##height##_##filter) { \
|
||||
int diff = UVTestFilter_16(width, height, Abs(benchmark_width_), \
|
||||
Abs(benchmark_height_), kFilter##filter, \
|
||||
benchmark_iterations_, disable_cpu_flags_, \
|
||||
benchmark_cpu_info_); \
|
||||
ASSERT_LE(diff, max_diff); \
|
||||
}
|
||||
|
||||
#if defined(ENABLE_FULL_TESTS)
|
||||
#define TEST_SCALETO_16(name, width, height) \
|
||||
TEST_SCALETO1_16(name, width, height, None, 0) \
|
||||
TEST_SCALETO1_16(name, width, height, Linear, 0) \
|
||||
TEST_SCALETO1_16(name, width, height, Bilinear, 0)
|
||||
#else
|
||||
#define TEST_SCALETO_16(name, width, height) \
|
||||
TEST_SCALETO1_16(name, width, height, Bilinear, 0)
|
||||
#endif
|
||||
|
||||
TEST_SCALETO_16(UVScale_16, 1, 1)
|
||||
TEST_SCALETO_16(UVScale_16, 569, 480)
|
||||
TEST_SCALETO_16(UVScale_16, 640, 360)
|
||||
#undef TEST_SCALETO1_16
|
||||
#undef TEST_SCALETO_16
|
||||
|
||||
// Compare UVScale_16 against scaling U and V planes separately with
|
||||
// ScalePlane_16 across Box, Bilinear, Linear, and None filters.
|
||||
static int UVTestAgainstPlanes_16(int src_width,
|
||||
int src_height,
|
||||
int dst_width,
|
||||
int dst_height,
|
||||
FilterMode filtering,
|
||||
int cpu_info) {
|
||||
if (!SizeValid(src_width, src_height, dst_width, dst_height)) {
|
||||
return 0;
|
||||
}
|
||||
if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) {
|
||||
return 0;
|
||||
}
|
||||
const int abs_src_width = Abs(src_width);
|
||||
const int abs_src_height = Abs(src_height);
|
||||
int64_t src_uv_samples = abs_src_width * abs_src_height * 2LL;
|
||||
int64_t src_plane_samples = abs_src_width * abs_src_height * 1LL;
|
||||
int64_t dst_uv_samples = dst_width * dst_height * 2LL;
|
||||
int64_t dst_plane_samples = dst_width * dst_height * 1LL;
|
||||
|
||||
align_buffer_page_end(src_uv_buf, src_uv_samples * 2);
|
||||
align_buffer_page_end(src_u_buf, src_plane_samples * 2);
|
||||
align_buffer_page_end(src_v_buf, src_plane_samples * 2);
|
||||
align_buffer_page_end(dst_uv_buf, dst_uv_samples * 2);
|
||||
align_buffer_page_end(dst_u_buf, dst_plane_samples * 2);
|
||||
align_buffer_page_end(dst_v_buf, dst_plane_samples * 2);
|
||||
if (!src_uv_buf || !src_u_buf || !src_v_buf || !dst_uv_buf || !dst_u_buf ||
|
||||
!dst_v_buf) {
|
||||
printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n");
|
||||
return 0;
|
||||
}
|
||||
uint16_t* src_uv = reinterpret_cast<uint16_t*>(src_uv_buf);
|
||||
uint16_t* src_u = reinterpret_cast<uint16_t*>(src_u_buf);
|
||||
uint16_t* src_v = reinterpret_cast<uint16_t*>(src_v_buf);
|
||||
uint16_t* dst_uv = reinterpret_cast<uint16_t*>(dst_uv_buf);
|
||||
uint16_t* dst_u = reinterpret_cast<uint16_t*>(dst_u_buf);
|
||||
uint16_t* dst_v = reinterpret_cast<uint16_t*>(dst_v_buf);
|
||||
|
||||
MemRandomize(src_uv_buf, src_uv_samples * 2);
|
||||
memset(dst_uv_buf, 123, dst_uv_samples * 2);
|
||||
for (int i = 0; i < abs_src_width * abs_src_height; ++i) {
|
||||
src_u[i] = src_uv[i * 2 + 0];
|
||||
src_v[i] = src_uv[i * 2 + 1];
|
||||
}
|
||||
|
||||
MaskCpuFlags(cpu_info);
|
||||
int rc_uv = UVScale_16(src_uv, abs_src_width * 2, src_width, src_height,
|
||||
dst_uv, dst_width * 2, dst_width, dst_height,
|
||||
filtering);
|
||||
int rc_u = ScalePlane_16(src_u, abs_src_width, src_width, src_height, dst_u,
|
||||
dst_width, dst_width, dst_height, filtering);
|
||||
int rc_v = ScalePlane_16(src_v, abs_src_width, src_width, src_height, dst_v,
|
||||
dst_width, dst_width, dst_height, filtering);
|
||||
int max_diff = (rc_uv != 0 || rc_u != 0 || rc_v != 0) ? 65536 : 0;
|
||||
for (int i = 0; i < dst_width * dst_height; ++i) {
|
||||
int du = Abs(static_cast<int>(dst_uv[i * 2 + 0]) -
|
||||
static_cast<int>(dst_u[i]));
|
||||
int dv = Abs(static_cast<int>(dst_uv[i * 2 + 1]) -
|
||||
static_cast<int>(dst_v[i]));
|
||||
if (du > max_diff) {
|
||||
max_diff = du;
|
||||
}
|
||||
if (dv > max_diff) {
|
||||
max_diff = dv;
|
||||
}
|
||||
}
|
||||
|
||||
free_aligned_buffer_page_end(dst_v_buf);
|
||||
free_aligned_buffer_page_end(dst_u_buf);
|
||||
free_aligned_buffer_page_end(dst_uv_buf);
|
||||
free_aligned_buffer_page_end(src_v_buf);
|
||||
free_aligned_buffer_page_end(src_u_buf);
|
||||
free_aligned_buffer_page_end(src_uv_buf);
|
||||
return max_diff;
|
||||
}
|
||||
|
||||
#define TEST_UVPLANE16(name, swidth, sheight, dwidth, dheight, filter, \
|
||||
max_diff) \
|
||||
TEST_F(LibYUVScaleTest, UVScale_16_MatchesPlane_##name##_##filter) { \
|
||||
int diff_c = UVTestAgainstPlanes_16(swidth, sheight, dwidth, dheight, \
|
||||
kFilter##filter, disable_cpu_flags_); \
|
||||
ASSERT_LE(diff_c, max_diff); \
|
||||
int diff_opt = UVTestAgainstPlanes_16(swidth, sheight, dwidth, dheight, \
|
||||
kFilter##filter, \
|
||||
benchmark_cpu_info_); \
|
||||
ASSERT_LE(diff_opt, max_diff); \
|
||||
}
|
||||
|
||||
TEST_UVPLANE16(By2, 480, 270, 240, 135, Box, 0)
|
||||
TEST_UVPLANE16(By2, 480, 270, 240, 135, Bilinear, 0)
|
||||
TEST_UVPLANE16(By2, 480, 270, 240, 135, None, 0)
|
||||
TEST_UVPLANE16(By3, 480, 270, 160, 90, Box, 0)
|
||||
TEST_UVPLANE16(By3Rounded, 320, 180, 107, 60, Box, 0)
|
||||
TEST_UVPLANE16(By4, 480, 270, 120, 68, Box, 0)
|
||||
TEST_UVPLANE16(By4Exact, 480, 272, 120, 68, Box, 1)
|
||||
TEST_UVPLANE16(By6, 360, 180, 60, 30, Box, 0)
|
||||
TEST_UVPLANE16(By6, 360, 180, 60, 30, None, 0)
|
||||
TEST_UVPLANE16(OddTail, 33, 17, 7, 5, Box, 0)
|
||||
TEST_UVPLANE16(By3OddTail, 69, 15, 23, 5, Box, 0)
|
||||
TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, Bilinear, 0)
|
||||
TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, Linear, 0)
|
||||
TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, None, 0)
|
||||
TEST_UVPLANE16(Up2x, 37, 23, 74, 46, Bilinear, 0)
|
||||
TEST_UVPLANE16(Up2x, 37, 23, 74, 46, Linear, 0)
|
||||
TEST_UVPLANE16(Up2x, 37, 23, 74, 46, None, 0)
|
||||
TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, Bilinear, 0)
|
||||
TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, Linear, 0)
|
||||
TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, None, 0)
|
||||
TEST_UVPLANE16(VerticalOnlyUp, 41, 19, 41, 53, Bilinear, 0)
|
||||
TEST_UVPLANE16(VerticalOnlyDown, 41, 53, 41, 19, Bilinear, 0)
|
||||
TEST_UVPLANE16(InvertCopy, 31, -17, 31, 17, None, 0)
|
||||
TEST_UVPLANE16(InvertUp, 31, -17, 65, 39, Bilinear, 0)
|
||||
#undef TEST_UVPLANE16
|
||||
|
||||
// Verify 2x kFilterLinear downscale averages pairs horizontally on odd rows
|
||||
// without underflowing into the previous row.
|
||||
TEST_F(LibYUVScaleTest, UVScale_16_DownBy2_LinearAveragesOddRows) {
|
||||
const int kSrcWidth = 4;
|
||||
const int kSrcHeight = 2;
|
||||
const int kDstWidth = 2;
|
||||
const int kDstHeight = 1;
|
||||
uint16_t src_uv[kSrcWidth * kSrcHeight * 2] = {
|
||||
// Row 0 (even row - must not be read):
|
||||
60000, 60000, 60000, 60000, 60000, 60000, 60000, 60000,
|
||||
// Row 1 (odd row - horizontally averaged in pairs):
|
||||
100, 1000, 300, 3000, 500, 5000, 900, 7000,
|
||||
};
|
||||
uint16_t dst_uv[kDstWidth * kDstHeight * 2] = {0};
|
||||
|
||||
ASSERT_EQ(0, UVScale_16(src_uv, kSrcWidth * 2, kSrcWidth, kSrcHeight, dst_uv,
|
||||
kDstWidth * 2, kDstWidth, kDstHeight, kFilterLinear));
|
||||
EXPECT_EQ(200, dst_uv[0]);
|
||||
EXPECT_EQ(2000, dst_uv[1]);
|
||||
EXPECT_EQ(700, dst_uv[2]);
|
||||
EXPECT_EQ(6000, dst_uv[3]);
|
||||
}
|
||||
|
||||
} // namespace libyuv
|
||||
|
||||
@ -209,6 +209,12 @@ static int TestCpuEnv(int cpu_info) {
|
||||
if (TestEnv("LIBYUV_DISABLE_AMXINT8")) {
|
||||
cpu_info &= ~libyuv::kCpuHasAMXINT8;
|
||||
}
|
||||
if (TestEnv("LIBYUV_DISABLE_AVX512BMM")) {
|
||||
cpu_info &= ~libyuv::kCpuHasAVX512BMM;
|
||||
}
|
||||
if (TestEnv("LIBYUV_DISABLE_AMXTF32")) {
|
||||
cpu_info &= ~libyuv::kCpuHasAMXTF32;
|
||||
}
|
||||
#endif
|
||||
if (TestEnv("LIBYUV_DISABLE_ASM")) {
|
||||
cpu_info = libyuv::kCpuInitialized;
|
||||
|
||||
@ -217,6 +217,7 @@ int main(int argc, const char* argv[]) {
|
||||
int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8);
|
||||
int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8);
|
||||
int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM);
|
||||
int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32);
|
||||
printf("Has X86 0x%x\n", has_x86);
|
||||
printf("Has SSE2 0x%x\n", has_sse2);
|
||||
printf("Has SSSE3 0x%x\n", has_ssse3);
|
||||
@ -240,6 +241,7 @@ int main(int argc, const char* argv[]) {
|
||||
printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8);
|
||||
printf("Has AMXINT8 0x%x\n", has_amxint8);
|
||||
printf("Has AVX512BMM 0x%x\n", has_avx512bmm);
|
||||
printf("Has AMXTF32 0x%x\n", has_amxtf32);
|
||||
|
||||
#ifdef __linux__
|
||||
// Test VDPPHPS instruction
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user