mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
[libyuv] Add NEON implementations for all remaining C row functions
Implement 32-bit and 64-bit Arm NEON versions (in row_neon.cc and row_neon64.cc) for the 12 remaining row functions that previously fell back to C on Arm: - AR30ToAB30Row_NEON (and AR30ToAB30Row_Any_NEON) - AR30ToABGRRow_NEON (and AR30ToABGRRow_Any_NEON) - AR30ToARGBRow_NEON (and AR30ToARGBRow_Any_NEON) - ARGBAffineRow_NEON - ARGBColorTableRow_NEON - ARGBCopyAlphaRow_NEON (and ARGBCopyAlphaRow_Any_NEON) - ARGBLumaColorTableRow_NEON - ARGBPolynomialRow_NEON (and ARGBPolynomialRow_Any_NEON) - ARGBUnattenuateRow_NEON (and ARGBUnattenuateRow_Any_NEON) - CopyRow_16_NEON (and CopyRow_16_Any_NEON) - MirrorRow_16_NEON (and MirrorRow_16_Any_NEON) - RGBColorTableRow_NEON Add HAS_*_NEON macros, prototypes, _Any wrappers, NEON dispatches in convert.cc, convert_argb.cc, and planar_functions.cc, and enable unit tests in planar_test.cc. Arm row function SIMD coverage is now 164/164 (100%). Maximum SIMD ISA supported per C row function: ------------------------------------------------------------------------ C Row Function arm x86 riscv wasm loongson hexagon ------------------------------------------------------------------------ AB64ToARGBRow_C NEON AVX2 RVV - - HVX ABGRToAR30Row_C NEON AVX512BW - - - - AR30ToAB30Row_C NEON AVX512BW - - - - AR30ToABGRRow_C NEON AVX512BW - - - - AR30ToARGBRow_C NEON AVX512BW - - - - AR64ShuffleRow_C NEON AVX512BW RVV WASMSIMD - - AR64ToAB64Row_C NEON AVX512BW RVV WASMSIMD - HVX AR64ToARGBRow_C NEON AVX2 RVV - - HVX ARGB1555ToARGBRow_C SVE2 AVX2 - WASMSIMD LASX - ARGB1555ToUVMatrixRow_C NEON AVX2 - WASMSIMD - - ARGB1555ToUVRow_C NEON - - - LASX - ARGB1555ToYMatrixRow_C NEON AVX2 - WASMSIMD - - ARGB1555ToYRow_C NEON - - - LASX - ARGB4444ToARGBRow_C NEON AVX2 - WASMSIMD LASX - ARGB4444ToUVMatrixRow_C NEON AVX2 - WASMSIMD - - ARGB4444ToUVRow_C NEON - - - - - ARGB4444ToYMatrixRow_C NEON AVX2 - WASMSIMD - - ARGB4444ToYRow_C NEON - - - - - ARGBAddRow_C NEON AVX2 - - LASX HVX ARGBAffineRow_C NEON SSE2 - - - - ARGBAlphaBlendRow_C SME AVX512BW RVV WASMSIMD LASX HVX ARGBAttenuateRow_C NEON AVX512BW RVV - LASX HVX ARGBBlendRow_C NEON AVX512BW RVV WASMSIMD LASX HVX ARGBColorMatrixRow_C I8MM SSSE3 - - LASX - ARGBColorTableRow_C NEON X86 - - - - ARGBCopyAlphaRow_C NEON AVX2 - - - HVX ARGBCopyYToAlphaRow_C NEON AVX512BW RVV WASMSIMD - HVX ARGBExtractAlphaRow_C NEON AVX512BW RVV WASMSIMD LASX HVX ARGBGrayRow_C DOTPROD AVX512VNNI - - LASX - ARGBLumaColorTableRow_C NEON SSSE3 - - - - ARGBMirrorRow_C NEON AVX512BW RVV WASMSIMD LASX HVX ARGBMultiplyRow_C SME AVX512BW RVV - LASX HVX ARGBOrRow_C SME AVX512BW RVV WASMSIMD LASX HVX ARGBPolynomialRow_C NEON AVX2 - - - - ARGBQuantizeRow_C NEON SSE2 - - LASX - ARGBSepiaRow_C DOTPROD SSSE3 - - LASX - ARGBSetRow_C NEON X86 RVV - LASX HVX ARGBShadeRow_C NEON SSE2 - - LASX - ARGBShuffleRow_C NEON AVX512BW RVV WASMSIMD LASX - ARGBSubtractRow_C NEON AVX2 - - LASX HVX ARGBToAB64Row_C NEON AVX2 RVV - - HVX ARGBToABGRRow_C NEON AVX512BW RVV WASMSIMD LASX HVX ARGBToAR30Row_C NEON AVX512BW - - - - ARGBToAR64Row_C NEON AVX2 RVV - - HVX ARGBToARGB1555Row_C NEON AVX2 - WASMSIMD LASX - ARGBToARGB4444Row_C NEON AVX2 - WASMSIMD LASX - ARGBToBGRARow_C NEON AVX512BW RVV WASMSIMD LASX HVX ARGBToRAWRow_C SVE2 AVX512BW RVV - LASX - ARGBToRGB24Row_C SVE2 AVX512BW RVV - LASX - ARGBToRGB565DitherRow_C SVE2 AVX2 - WASMSIMD LASX - ARGBToRGB565Row_C SVE2 AVX2 - WASMSIMD LASX - ARGBToRGBARow_C NEON AVX512BW RVV WASMSIMD LASX HVX ARGBToUV444MatrixRow_C I8MM AVX512VNNI RVV WASMSIMD - HVX ARGBToUVMatrixRow_C SME AVX512VNNI RVV WASMSIMD - HVX ARGBToYMatrixRow_C SME AVX512VNNI RVV WASMSIMD LASX HVX ARGBUnattenuateRow_C NEON AVX2 - - - - AYUVToUVRow_C SVE2 AVX512VBMI - - - - AYUVToVURow_C SVE2 AVX512VBMI - - - - AYUVToYRow_C NEON AVX512VBMI - - - - BlendPlaneRow_C NEON AVX512BW RVV WASMSIMD - HVX ByteToFloatRow_C NEON - - - - - ComputeCumulativeSumRow_C NEON AVX2 - - - - Convert16To8Row_C SME AVX512BW RVV - - HVX Convert8To16Row_C SME AVX512BW RVV WASMSIMD - HVX Convert8To8Row_C SME AVX2 - - - HVX CopyRow_C SME AVX512BW RVV - LASX HVX CopyRow_16_C NEON - - - - HVX CumulativeSumToAverageRow_C NEON AVX2 - - - - DetileRow_C NEON SSE2 - - - - DetileRow_16_C NEON AVX - - - - DetileSplitUVRow_C NEON SSSE3 - - - - DivideRow_16_C SVE2 AVX2 - - - HVX GaussRow_C NEON - - - - - GaussRow_F32_C NEON - - - - - HalfFloatRow_C SVE2 AVX2 - - LASX - HalfMergeUVRow_C SVE2 AVX512BW - - - - HalfRow_16To8_C SME AVX512BW RVV - - HVX HalfWidthRow_16To8_C SME AVX512BW RVV - - - I210AlphaToARGBRow_C SME AVX2 - - - - I210ToAR30Row_C SME AVX2 - - - - I210ToARGBRow_C SME AVX2 - - - - I212ToAR30Row_C SME AVX2 - - - - I212ToARGBRow_C SME AVX2 - - - - I400ToARGBRow_C SME AVX512BW RVV - LASX HVX I410AlphaToARGBRow_C SME AVX2 - - - - I410ToAR30Row_C SME AVX2 - - - - I410ToARGBRow_C SME AVX2 - - - - I422AlphaToARGBRow_C SME AVX512BW RVV - LASX HVX I422ToAR30Row_C SME AVX512BW RVV WASMSIMD - - I422ToARGB1555Row_C SME AVX2 - WASMSIMD LASX - I422ToARGB4444Row_C SME AVX2 - WASMSIMD LASX - I422ToARGBRow_C SME AVX512BW RVV WASMSIMD LASX HVX I422ToRGB24Row_C SME AVX512VBMI RVV WASMSIMD LASX - I422ToRGB565Row_C SME AVX2 - WASMSIMD LASX - I422ToRGBARow_C SME AVX2 RVV - LASX HVX I422ToUYVYRow_C NEON AVX2 - - LASX - I422ToYUY2Row_C NEON AVX2 - - LASX - I444AlphaToARGBRow_C SME AVX2 RVV - - HVX I444ToARGBRow_C SME AVX2 RVV - LASX HVX I444ToRGB24Row_C SME AVX2 RVV - - - InterpolateRow_C SME AVX512BW RVV WASMSIMD LASX HVX InterpolateRow_16_C SME AVX512BW - WASMSIMD - HVX J400ToARGBRow_C NEON AVX512BW RVV WASMSIMD LASX HVX MergeAR64Row_C NEON AVX2 - - - - MergeARGB16To8Row_C NEON AVX2 - - - - MergeARGBRow_C NEON AVX2 RVV - - HVX MergeRGBRow_C NEON AVX512VBMI RVV - - - MergeUVRow_C SME AVX512BW RVV WASMSIMD LASX HVX MergeUVRow_16_C SME AVX2 - - - HVX MergeXR30Row_C NEON AVX512BW - - - - MergeXR64Row_C NEON AVX2 - - - - MergeXRGB16To8Row_C NEON AVX2 - - - - MergeXRGBRow_C NEON AVX2 RVV - - HVX MirrorRow_C NEON AVX512BW - WASMSIMD LASX HVX MirrorRow_16_C NEON AVX512BW - - - HVX MirrorSplitUVRow_C NEON AVX512BW - WASMSIMD LASX HVX MirrorUVRow_C NEON AVX512BW RVV WASMSIMD LASX HVX MultiplyRow_16_C SME AVX512BW RVV WASMSIMD - HVX NV12ToARGBRow_C SME AVX512BW RVV WASMSIMD LASX HVX NV12ToRGB24Row_C SME AVX512VBMI RVV - - - NV12ToRGB565Row_C NEON AVX2 - WASMSIMD LASX - NV21ToARGBRow_C SME AVX512BW RVV WASMSIMD LASX HVX NV21ToRGB24Row_C SME AVX512VBMI RVV - - - NV21ToYUV24Row_C NEON AVX2 - - - - P210ToAR30Row_C SME AVX2 - - - - P210ToARGBRow_C SME AVX2 - - - - P410ToAR30Row_C SME AVX2 - - - - P410ToARGBRow_C SME AVX2 - - - - RAWToARGBRow_C SVE2 AVX512BW RVV WASMSIMD LASX - RAWToRGB24Row_C SVE2 AVX512VBMI RVV WASMSIMD LASX - RAWToRGBARow_C SVE2 AVX512VBMI RVV - - - RGB24MirrorRow_C NEON AVX512VBMI - WASMSIMD - - RGB24ToARGBRow_C SVE2 AVX512BW RVV WASMSIMD LASX - RGB565ToARGBRow_C NEON AVX2 - WASMSIMD LASX - RGB565ToUVMatrixRow_C NEON AVX2 - WASMSIMD - - RGB565ToUVRow_C NEON - - - LASX - RGB565ToYMatrixRow_C NEON AVX2 - WASMSIMD - - RGB565ToYRow_C NEON - - - LASX - RGBAToARGBRow_C NEON AVX512BW RVV WASMSIMD LASX HVX RGBColorTableRow_C NEON X86 - - - - RGBToUV444MatrixRow_C NEON AVX512BW RVV WASMSIMD - - RGBToUVMatrixRow_C NEON AVX512BW RVV WASMSIMD - - RGBToYMatrixRow_C DOTPROD AVX512BW RVV WASMSIMD LASX - SetRow_C NEON ERMS - - LASX HVX SobelRow_C NEON SSE2 - - LASX - SobelToPlaneRow_C NEON SSE2 - - LASX - SobelXRow_C NEON SSE2 - - - - SobelXYRow_C NEON SSE2 - - LASX - SobelYRow_C NEON SSE2 - - - - SplitARGBRow_C NEON AVX2 RVV - - HVX SplitRGBRow_C NEON AVX2 RVV - - - SplitUVRow_C SVE2 AVX512BW RVV - LASX HVX SplitUVRow_16_C NEON AVX2 - - - HVX SplitXRGBRow_C NEON AVX2 RVV - - HVX SwapUVRow_C NEON AVX512BW RVV WASMSIMD - HVX UYVYToARGBRow_C SME AVX2 - - LASX - UYVYToUV422Row_C NEON AVX2 - - LASX HVX UYVYToUVRow_C SVE2 AVX2 - - LASX HVX UYVYToYRow_C SVE2 AVX2 - - LASX HVX YUY2ToARGBRow_C SME AVX2 - - LASX - YUY2ToNVUVRow_C NEON AVX2 - - - - YUY2ToUV422Row_C NEON AVX2 - - LASX HVX YUY2ToUVRow_C SVE2 AVX2 - - LASX HVX YUY2ToYRow_C SVE2 AVX2 - - LASX HVX ------------------------------------------------------------------------ SIMD (164/164 100%) 164 100% 154 94% 64 39% 56 34% 73 45% 64 39% ------------------------------------------------------------------------ Test: libyuv_unittest --gtest_filter=*AR30*:*Affine*:*ColorTable*:*CopyAlpha*:*Polynomial*:*Unattenuate*:*MirrorPlane_16* Bug: None Change-Id: Ib8b68beb77b47de6b996d41fd080397c61539ffb
This commit is contained in:
Frank Barchard
parent
4bc0419865
commit
10a3335a74
@ -1,6 +1,6 @@
|
||||
Name: libyuv
|
||||
URL: https://chromium.googlesource.com/libyuv/libyuv/
|
||||
Version: 1975
|
||||
Version: 1976
|
||||
Revision: DEPS
|
||||
License: BSD-3-Clause
|
||||
License File: LICENSE
|
||||
|
||||
@ -540,6 +540,9 @@ extern "C" {
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && \
|
||||
(defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON))
|
||||
#define HAS_AB64TOARGBROW_NEON
|
||||
#define HAS_AR30TOAB30ROW_NEON
|
||||
#define HAS_AR30TOABGRROW_NEON
|
||||
#define HAS_AR30TOARGBROW_NEON
|
||||
#define HAS_AR64TOARGBROW_NEON
|
||||
#define HAS_ARGB1555TOARGBROW_NEON
|
||||
#define HAS_ARGB1555TOUVROW_NEON
|
||||
@ -547,6 +550,7 @@ extern "C" {
|
||||
#define HAS_ARGB4444TOARGBROW_NEON
|
||||
#define HAS_ARGB4444TOUVROW_NEON
|
||||
#define HAS_ARGB4444TOYROW_NEON
|
||||
#define HAS_ARGBCOPYALPHAROW_NEON
|
||||
#define HAS_ARGBCOPYYTOALPHAROW_NEON
|
||||
#define HAS_ARGBEXTRACTALPHAROW_NEON
|
||||
#define HAS_ARGBSETROW_NEON
|
||||
@ -578,6 +582,7 @@ extern "C" {
|
||||
#define HAS_CONVERT16TO8ROW_NEON
|
||||
#define HAS_CONVERT8TO16ROW_NEON
|
||||
#define HAS_CONVERT8TO8ROW_NEON
|
||||
#define HAS_COPYROW_16_NEON
|
||||
#define HAS_COPYROW_NEON
|
||||
#define HAS_DETILEROW_16_NEON
|
||||
#define HAS_DETILEROW_NEON
|
||||
@ -613,6 +618,7 @@ extern "C" {
|
||||
#define HAS_MERGEXR64ROW_NEON
|
||||
#define HAS_MERGEXRGB16TO8ROW_NEON
|
||||
#define HAS_MERGEXRGBROW_NEON
|
||||
#define HAS_MIRRORROW_16_NEON
|
||||
#define HAS_MIRRORROW_NEON
|
||||
#define HAS_MIRRORSPLITUVROW_NEON
|
||||
#define HAS_MIRRORUVROW_NEON
|
||||
@ -651,21 +657,27 @@ extern "C" {
|
||||
|
||||
// Effects:
|
||||
#define HAS_ARGBADDROW_NEON
|
||||
#define HAS_ARGBAFFINEROW_NEON
|
||||
#define HAS_ARGBALPHABLENDROW_NEON
|
||||
#define HAS_ARGBATTENUATEROW_NEON
|
||||
#define HAS_ARGBBLENDROW_NEON
|
||||
#define HAS_BLENDPLANEROW_NEON
|
||||
#define HAS_ARGBCOLORMATRIXROW_NEON
|
||||
#define HAS_ARGBCOLORTABLEROW_NEON
|
||||
#define HAS_ARGBGRAYROW_NEON
|
||||
#define HAS_ARGBLUMACOLORTABLEROW_NEON
|
||||
#define HAS_ARGBMIRRORROW_NEON
|
||||
#define HAS_ARGBMULTIPLYROW_NEON
|
||||
#define HAS_ARGBORROW_NEON
|
||||
#define HAS_ARGBPOLYNOMIALROW_NEON
|
||||
#define HAS_ARGBQUANTIZEROW_NEON
|
||||
#define HAS_ARGBSEPIAROW_NEON
|
||||
#define HAS_ARGBSHADEROW_NEON
|
||||
#define HAS_ARGBSHUFFLEROW_NEON
|
||||
#define HAS_ARGBSUBTRACTROW_NEON
|
||||
#define HAS_ARGBUNATTENUATEROW_NEON
|
||||
#define HAS_RGB24MIRRORROW_NEON
|
||||
#define HAS_RGBCOLORTABLEROW_NEON
|
||||
#define HAS_SOBELROW_NEON
|
||||
#define HAS_SOBELTOPLANEROW_NEON
|
||||
#define HAS_SOBELXROW_NEON
|
||||
@ -2835,6 +2847,10 @@ void MirrorSplitUVRow_C(const uint8_t* src_uv,
|
||||
|
||||
void MirrorRow_16_C(const uint16_t* src, uint16_t* dst, int width);
|
||||
void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width);
|
||||
void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width);
|
||||
void MirrorRow_16_Any_NEON(const uint16_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
|
||||
@ -4011,16 +4027,22 @@ void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
|
||||
void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count);
|
||||
void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count);
|
||||
void CopyRow_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int width);
|
||||
|
||||
void ARGBCopyAlphaRow_C(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyAlphaRow_Any_SSE2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBCopyAlphaRow_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBCopyAlphaRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void ARGBExtractAlphaRow_C(const uint8_t* src_argb, uint8_t* dst_a, int width);
|
||||
void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb,
|
||||
@ -4276,6 +4298,18 @@ void AR30ToAB30Row_C(const uint8_t* src_ar30, uint8_t* dst_ab30, int width);
|
||||
void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30,
|
||||
uint8_t* dst_ab30,
|
||||
int width);
|
||||
void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width);
|
||||
void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width);
|
||||
void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width);
|
||||
void AR30ToARGBRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void AR30ToABGRRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void AR30ToAB30Row_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void RGB24ToARGBRow_Any_SSSE3(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
@ -6925,12 +6959,18 @@ void ARGBUnattenuateRow_SSE2(const uint8_t* src_argb,
|
||||
void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width);
|
||||
void ARGBUnattenuateRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width);
|
||||
void ARGBUnattenuateRow_Any_SSE2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBUnattenuateRow_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBUnattenuateRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void ARGBGrayRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width);
|
||||
void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width);
|
||||
@ -6985,6 +7025,9 @@ void ARGBColorTableRow_C(uint8_t* dst_argb,
|
||||
void ARGBColorTableRow_X86(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width);
|
||||
void ARGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width);
|
||||
|
||||
void RGBColorTableRow_C(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
@ -6992,6 +7035,9 @@ void RGBColorTableRow_C(uint8_t* dst_argb,
|
||||
void RGBColorTableRow_X86(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width);
|
||||
void RGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width);
|
||||
|
||||
void ARGBQuantizeRow_C(uint8_t* dst_argb,
|
||||
int scale,
|
||||
@ -7095,6 +7141,12 @@ void ARGBAffineRow_SSE2(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* src_dudv,
|
||||
int width);
|
||||
LIBYUV_API
|
||||
void ARGBAffineRow_NEON(const uint8_t* src_argb,
|
||||
int src_argb_stride,
|
||||
uint8_t* dst_argb,
|
||||
const float* src_dudv,
|
||||
int width);
|
||||
|
||||
// Used for I420Scale, ARGBScale, and ARGBInterpolate.
|
||||
void InterpolateRow_C(uint8_t* dst_ptr,
|
||||
@ -7348,6 +7400,14 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
int width);
|
||||
void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
int width);
|
||||
void ARGBPolynomialRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
const float* param,
|
||||
int width);
|
||||
|
||||
// Scale and convert to half float.
|
||||
void HalfFloatRow_C(const uint16_t* src, uint16_t* dst, float scale, int width);
|
||||
@ -7439,6 +7499,11 @@ void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb,
|
||||
int width,
|
||||
const uint8_t* luma,
|
||||
uint32_t lumacoeff);
|
||||
void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width,
|
||||
const uint8_t* luma,
|
||||
uint32_t lumacoeff);
|
||||
|
||||
float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width);
|
||||
float ScaleMaxSamples_NEON(const float* src,
|
||||
|
||||
@ -11,6 +11,6 @@
|
||||
#ifndef INCLUDE_LIBYUV_VERSION_H_
|
||||
#define INCLUDE_LIBYUV_VERSION_H_
|
||||
|
||||
#define LIBYUV_VERSION 1975
|
||||
#define LIBYUV_VERSION 1976
|
||||
|
||||
#endif // INCLUDE_LIBYUV_VERSION_H_
|
||||
|
||||
@ -3538,6 +3538,14 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30,
|
||||
src_ar30 = src_ar30 + (ptrdiff_t)(height - 1) * src_stride_ar30;
|
||||
src_stride_ar30 = -src_stride_ar30;
|
||||
}
|
||||
#if defined(HAS_AR30TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_AVX512BW;
|
||||
|
||||
@ -4522,6 +4522,14 @@ int AR30ToARGB(const uint8_t* src_ar30,
|
||||
}
|
||||
void (*AR30ToARGBRow)(const uint8_t* src_ar30, uint8_t* dst_argb, int width) =
|
||||
AR30ToARGBRow_C;
|
||||
#if defined(HAS_AR30TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_AVX512BW;
|
||||
@ -4563,6 +4571,14 @@ int AR30ToABGR(const uint8_t* src_ar30,
|
||||
height = 1;
|
||||
src_stride_ar30 = dst_stride_abgr = 0;
|
||||
}
|
||||
#if defined(HAS_AR30TOABGRROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
AR30ToABGRRow = AR30ToABGRRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToABGRRow = AR30ToABGRRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOABGRROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToABGRRow = AR30ToABGRRow_AVX512BW;
|
||||
@ -4604,6 +4620,14 @@ int AR30ToAB30(const uint8_t* src_ar30,
|
||||
height = 1;
|
||||
src_stride_ar30 = dst_stride_ab30 = 0;
|
||||
}
|
||||
#if defined(HAS_AR30TOAB30ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
AR30ToAB30Row = AR30ToAB30Row_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToAB30Row = AR30ToAB30Row_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOAB30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToAB30Row = AR30ToAB30Row_AVX512BW;
|
||||
|
||||
@ -2926,7 +2926,14 @@ void MirrorPlane_16(const uint16_t* src_y,
|
||||
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
|
||||
src_stride_y = -src_stride_y;
|
||||
}
|
||||
#if defined(HAS_MIRRORUVROW_NEON)
|
||||
#if defined(HAS_MIRRORROW_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
MirrorRow_16 = MirrorRow_16_Any_NEON;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
MirrorRow_16 = MirrorRow_16_NEON;
|
||||
}
|
||||
}
|
||||
#elif defined(HAS_MIRRORUVROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
MirrorRow_16 = reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
|
||||
MirrorUVRow_Any_NEON);
|
||||
@ -4693,7 +4700,14 @@ int ARGBUnattenuate(const uint8_t* src_argb,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// TODO(fbarchard): Neon version.
|
||||
#if defined(HAS_ARGBUNATTENUATEROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBUnattenuateRow = ARGBUnattenuateRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 4)) {
|
||||
ARGBUnattenuateRow = ARGBUnattenuateRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < height; ++y) {
|
||||
ARGBUnattenuateRow(src_argb, dst_argb, width);
|
||||
@ -5019,6 +5033,11 @@ int ARGBColorTable(uint8_t* dst_argb,
|
||||
if (TestCpuFlag(kCpuHasX86)) {
|
||||
ARGBColorTableRow = ARGBColorTableRow_X86;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOLORTABLEROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBColorTableRow = ARGBColorTableRow_NEON;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
ARGBColorTableRow(dst, table_argb, width);
|
||||
@ -5055,6 +5074,11 @@ int RGBColorTable(uint8_t* dst_argb,
|
||||
if (TestCpuFlag(kCpuHasX86)) {
|
||||
RGBColorTableRow = RGBColorTableRow_X86;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBCOLORTABLEROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
RGBColorTableRow = RGBColorTableRow_NEON;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
RGBColorTableRow(dst, table_argb, width);
|
||||
@ -6224,6 +6248,14 @@ int ARGBPolynomial(const uint8_t* src_argb,
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBPOLYNOMIALROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 2)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < height; ++y) {
|
||||
ARGBPolynomialRow(src_argb, dst_argb, poly, width);
|
||||
@ -6383,6 +6415,11 @@ int ARGBLumaColorTable(const uint8_t* src_argb,
|
||||
ARGBLumaColorTableRow = ARGBLumaColorTableRow_SSSE3;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBLUMACOLORTABLEROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 4)) {
|
||||
ARGBLumaColorTableRow = ARGBLumaColorTableRow_NEON;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < height; ++y) {
|
||||
ARGBLumaColorTableRow(src_argb, dst_argb, width, luma, 0x00264b0f);
|
||||
@ -6420,6 +6457,14 @@ int ARGBCopyAlpha(const uint8_t* src_argb,
|
||||
height = 1;
|
||||
src_stride_argb = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBCOPYALPHAROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
ARGBCopyAlphaRow = ARGBCopyAlphaRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOPYALPHAROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_SSE2;
|
||||
|
||||
@ -1105,6 +1105,15 @@ ANY11(ABGRToAR30Row_Any_NEON, ABGRToAR30Row_NEON, 0, 4, 4, 7)
|
||||
#if defined(HAS_ARGBTOAR30ROW_NEON)
|
||||
ANY11(ARGBToAR30Row_Any_NEON, ARGBToAR30Row_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_NEON)
|
||||
ANY11(AR30ToARGBRow_Any_NEON, AR30ToARGBRow_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_AR30TOABGRROW_NEON)
|
||||
ANY11(AR30ToABGRRow_Any_NEON, AR30ToABGRRow_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_AR30TOAB30ROW_NEON)
|
||||
ANY11(AR30ToAB30Row_Any_NEON, AR30ToAB30Row_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOAR30ROW_SSSE3)
|
||||
ANY11(ARGBToAR30Row_Any_SSSE3, ARGBToAR30Row_SSSE3, 0, 4, 4, 3)
|
||||
#endif
|
||||
@ -1330,6 +1339,9 @@ ANY11(ARGBUnattenuateRow_Any_AVX2, ARGBUnattenuateRow_AVX2, 0, 4, 4, 7)
|
||||
#ifdef HAS_ARGBATTENUATEROW_NEON
|
||||
ANY11(ARGBAttenuateRow_Any_NEON, ARGBAttenuateRow_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_ARGBUNATTENUATEROW_NEON
|
||||
ANY11(ARGBUnattenuateRow_Any_NEON, ARGBUnattenuateRow_NEON, 0, 4, 4, 3)
|
||||
#endif
|
||||
#ifdef HAS_ARGBATTENUATEROW_LSX
|
||||
ANY11(ARGBAttenuateRow_Any_LSX, ARGBAttenuateRow_LSX, 0, 4, 4, 3)
|
||||
#endif
|
||||
@ -1419,6 +1431,9 @@ ANY11B(ARGBCopyAlphaRow_Any_AVX2, ARGBCopyAlphaRow_AVX2, 0, 4, 4, 15)
|
||||
#ifdef HAS_ARGBCOPYALPHAROW_SSE2
|
||||
ANY11B(ARGBCopyAlphaRow_Any_SSE2, ARGBCopyAlphaRow_SSE2, 0, 4, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_ARGBCOPYALPHAROW_NEON
|
||||
ANY11B(ARGBCopyAlphaRow_Any_NEON, ARGBCopyAlphaRow_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2
|
||||
ANY11B(ARGBCopyYToAlphaRow_Any_AVX2, ARGBCopyYToAlphaRow_AVX2, 0, 1, 4, 15)
|
||||
#endif
|
||||
@ -1578,6 +1593,14 @@ ANY11P(ARGBOrRow_Any_LASX, ARGBOrRow_LASX, const uint32_t, 4, 4, 7)
|
||||
#ifdef HAS_ARGBORROW_WASMSIMD
|
||||
ANY11P(ARGBOrRow_Any_WASMSIMD, ARGBOrRow_WASMSIMD, const uint32_t, 4, 4, 3)
|
||||
#endif
|
||||
#ifdef HAS_ARGBPOLYNOMIALROW_NEON
|
||||
ANY11P(ARGBPolynomialRow_Any_NEON,
|
||||
ARGBPolynomialRow_NEON,
|
||||
const float*,
|
||||
4,
|
||||
4,
|
||||
1)
|
||||
#endif
|
||||
#undef ANY11P
|
||||
#undef ANY11P
|
||||
|
||||
@ -1647,6 +1670,10 @@ ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
|
||||
ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_COPYROW_16_NEON
|
||||
ANY11T(CopyRow_16_Any_NEON, CopyRow_16_NEON, 2, 2, uint16_t, uint16_t, 15)
|
||||
#endif
|
||||
|
||||
#undef ANY11T
|
||||
|
||||
// Any 1 to 1 with parameter and shorts. BPP measures in shorts.
|
||||
@ -2196,6 +2223,31 @@ ANY11M(RGB24MirrorRow_Any_WASMSIMD, RGB24MirrorRow_WASMSIMD, 3, 15)
|
||||
#endif
|
||||
#undef ANY11M
|
||||
|
||||
// Any 1 to 1 mirror with type.
|
||||
#define ANY11MT(NAMEANY, ANY_SIMD, T, BPP, MASK) \
|
||||
void NAMEANY(const T* src_ptr, T* dst_ptr, int width) { \
|
||||
SIMD_ALIGNED(T vin[64]); \
|
||||
static_assert((MASK + 1) * BPP <= sizeof(vin), "vin buffer too small"); \
|
||||
SIMD_ALIGNED(T vout[64]); \
|
||||
static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
|
||||
MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
|
||||
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
|
||||
int r = width & MASK; \
|
||||
int n = width & ~MASK; \
|
||||
if (n > 0) { \
|
||||
ANY_SIMD(src_ptr + r, dst_ptr, n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
memcpy(vin, src_ptr, r * BPP); \
|
||||
ANY_SIMD(vin, vout, MASK + 1); \
|
||||
memcpy(dst_ptr + np, vout + (MASK + 1 - r), r * BPP); \
|
||||
}
|
||||
|
||||
#ifdef HAS_MIRRORROW_16_NEON
|
||||
ANY11MT(MirrorRow_16_Any_NEON, MirrorRow_16_NEON, uint16_t, 2, 15)
|
||||
#endif
|
||||
#undef ANY11MT
|
||||
|
||||
// Any 1 plane. (memset)
|
||||
#define ANY1(NAMEANY, ANY_SIMD, T, BPP, MASK) \
|
||||
void NAMEANY(uint8_t* dst_ptr, T v32, int width) { \
|
||||
|
||||
@ -3992,6 +3992,433 @@ void Convert8To16Row_NEON(const uint8_t* src_y,
|
||||
: "cc", "memory", "q0", "q1", "q2");
|
||||
}
|
||||
|
||||
void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count) {
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"vld1.16 {d0, d1, d2, d3}, [%0]! \n" // load 16
|
||||
"subs %2, %2, #16 \n" // 16 processed per loop
|
||||
"vst1.16 {d0, d1, d2, d3}, [%1]! \n" // store 16
|
||||
"bgt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(count) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1");
|
||||
}
|
||||
|
||||
void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width) {
|
||||
asm volatile(
|
||||
// Start at end of source row.
|
||||
"add %0, %0, %2, lsl #1 \n"
|
||||
"sub %0, %0, #32 \n" // 32 bytes per loop
|
||||
"1: \n"
|
||||
"vld1.16 {q1, q2}, [%0], %3 \n" // src -= 32
|
||||
"subs %2, %2, #16 \n" // 16 pixels per loop.
|
||||
"vrev64.16 q0, q2 \n"
|
||||
"vrev64.16 q1, q1 \n"
|
||||
"vswp d0, d1 \n"
|
||||
"vswp d2, d3 \n"
|
||||
"vst1.16 {q0, q1}, [%1]! \n" // dst += 32
|
||||
"bgt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
: "r"(-32) // %3
|
||||
: "memory", "cc", "q0", "q1", "q2");
|
||||
}
|
||||
|
||||
void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width) {
|
||||
asm volatile(
|
||||
"vmov.i32 q8, #0x0000ff00 \n"
|
||||
"vmov.i32 q9, #0x000fffff \n"
|
||||
"vshl.i32 q9, q9, #8 \n" // 0x0fffff00
|
||||
"1: \n"
|
||||
"vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels
|
||||
"subs %2, %2, #8 \n" // 8 pixels per loop
|
||||
"vsri.32 q0, q0, #2 \n"
|
||||
"vsri.32 q1, q1, #2 \n"
|
||||
"vshr.u32 q2, q0, #2 \n"
|
||||
"vshr.u32 q3, q1, #2 \n"
|
||||
"vshr.u32 q4, q0, #4 \n"
|
||||
"vshr.u32 q5, q1, #4 \n"
|
||||
"vbit q0, q4, q9 \n"
|
||||
"vbit q1, q5, q9 \n"
|
||||
"vbit q0, q2, q8 \n"
|
||||
"vbit q1, q3, q8 \n"
|
||||
"vst1.8 {q0, q1}, [%1]! \n" // store 8 ARGB pixels
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9");
|
||||
}
|
||||
|
||||
void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) {
|
||||
asm volatile(
|
||||
"vmov.i64 q8, #0x00ffff0000ffff00 \n"
|
||||
"1: \n"
|
||||
"vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels
|
||||
"subs %2, %2, #8 \n" // 8 pixels per loop
|
||||
"vsri.32 q0, q0, #2 \n"
|
||||
"vsri.32 q1, q1, #2 \n"
|
||||
"vshr.u32 q2, q0, #2 \n"
|
||||
"vshr.u32 q3, q1, #2 \n"
|
||||
"vsli.32 q2, q0, #16 \n"
|
||||
"vsli.32 q3, q1, #16 \n"
|
||||
"vsri.32 q0, q0, #4 \n"
|
||||
"vsri.32 q1, q1, #4 \n"
|
||||
"vsri.32 q0, q0, #16 \n"
|
||||
"vsri.32 q1, q1, #16 \n"
|
||||
"vbit q0, q2, q8 \n"
|
||||
"vbit q1, q3, q8 \n"
|
||||
"vst1.8 {q0, q1}, [%1]! \n" // store 8 ABGR pixels
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_abgr), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q8");
|
||||
}
|
||||
|
||||
void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) {
|
||||
asm volatile(
|
||||
"vmov.i8 q9, #0xff \n"
|
||||
"vshr.u32 q9, q9, #22 \n" // 0x000003ff
|
||||
"vshl.i32 q8, q9, #20 \n" // 0x3ff00000
|
||||
"1: \n"
|
||||
"vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels
|
||||
"subs %2, %2, #8 \n" // 8 pixels per loop
|
||||
"vshl.i32 q2, q0, #20 \n"
|
||||
"vshl.i32 q3, q1, #20 \n"
|
||||
"vshr.u32 q4, q0, #20 \n"
|
||||
"vshr.u32 q5, q1, #20 \n"
|
||||
"vbit q0, q2, q8 \n"
|
||||
"vbit q1, q3, q8 \n"
|
||||
"vbit q0, q4, q9 \n"
|
||||
"vbit q1, q5, q9 \n"
|
||||
"vst1.8 {q0, q1}, [%1]! \n" // store 8 AB30 pixels
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_ab30), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9");
|
||||
}
|
||||
|
||||
void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
|
||||
asm volatile(
|
||||
"vmov.i32 q4, #0xff000000 \n"
|
||||
"1: \n"
|
||||
"vld1.8 {q0, q1}, [%0]! \n" // load 8 src pixels
|
||||
"vld1.8 {q2, q3}, [%1] \n" // load 8 dst pixels
|
||||
"subs %2, %2, #8 \n" // 8 pixels per loop
|
||||
"vbit q2, q0, q4 \n"
|
||||
"vbit q3, q1, q4 \n"
|
||||
"vst1.8 {q2, q3}, [%1]! \n" // store 8 dst pixels
|
||||
"bgt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4");
|
||||
}
|
||||
|
||||
void ARGBUnattenuateRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width) {
|
||||
uint32_t tmp, addr;
|
||||
asm volatile(
|
||||
"vmov.i32 q8, #0xff000000 \n"
|
||||
"1: \n"
|
||||
"ldrb %3, [%0, #3] \n"
|
||||
"add %4, %5, %3, lsl #2 \n"
|
||||
"vld1.16 {d6[0]}, [%4] \n"
|
||||
"ldrb %3, [%0, #7] \n"
|
||||
"add %4, %5, %3, lsl #2 \n"
|
||||
"vld1.16 {d6[1]}, [%4] \n"
|
||||
"ldrb %3, [%0, #11] \n"
|
||||
"add %4, %5, %3, lsl #2 \n"
|
||||
"vld1.16 {d6[2]}, [%4] \n"
|
||||
"ldrb %3, [%0, #15] \n"
|
||||
"add %4, %5, %3, lsl #2 \n"
|
||||
"vld1.16 {d6[3]}, [%4] \n"
|
||||
"vld1.8 {q0}, [%0]! \n"
|
||||
"subs %2, %2, #4 \n"
|
||||
"vmov q1, q0 \n"
|
||||
"vmov q2, q0 \n"
|
||||
"vzip.8 q1, q2 \n"
|
||||
"vmull.u16 q4, d2, d6[0] \n"
|
||||
"vmull.u16 q5, d3, d6[1] \n"
|
||||
"vmull.u16 q6, d4, d6[2] \n"
|
||||
"vmull.u16 q7, d5, d6[3] \n"
|
||||
"vshrn.i32 d8, q4, #16 \n"
|
||||
"vshrn.i32 d9, q5, #16 \n"
|
||||
"vshrn.i32 d12, q6, #16 \n"
|
||||
"vshrn.i32 d13, q7, #16 \n"
|
||||
"vqmovn.u16 d2, q4 \n"
|
||||
"vqmovn.u16 d3, q6 \n"
|
||||
"vbit q1, q0, q8 \n"
|
||||
"vst1.8 {q1}, [%1]! \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(tmp), // %3
|
||||
"=&r"(addr) // %4
|
||||
: "r"(fixed_invtbl8) // %5
|
||||
: "memory", "cc", "q0", "q1", "q2", "d6", "q4", "q5", "q6", "q7", "q8");
|
||||
}
|
||||
|
||||
void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
int width) {
|
||||
asm volatile(
|
||||
"vld1.32 {q8, q9}, [%3]! \n" // c0, c1
|
||||
"vld1.32 {q10, q11}, [%3] \n" // c2, c3
|
||||
"1: \n"
|
||||
"vld1.8 {d0}, [%0]! \n" // load 2 pixels
|
||||
"subs %2, %2, #2 \n" // 2 pixels per loop
|
||||
"vmovl.u8 q0, d0 \n"
|
||||
"vmovl.u16 q1, d0 \n"
|
||||
"vmovl.u16 q2, d1 \n"
|
||||
"vcvt.f32.u32 q1, q1 \n" // x0
|
||||
"vcvt.f32.u32 q2, q2 \n" // x1
|
||||
"vmov q3, q8 \n" // c0
|
||||
"vmov q4, q8 \n" // c0
|
||||
"vmla.f32 q3, q9, q1 \n" // c0 + c1 * x0
|
||||
"vmla.f32 q4, q9, q2 \n" // c0 + c1 * x1
|
||||
"vmul.f32 q5, q1, q1 \n" // x0 * x0
|
||||
"vmul.f32 q6, q2, q2 \n" // x1 * x1
|
||||
"vmla.f32 q3, q10, q5 \n" // + c2 * x0^2
|
||||
"vmla.f32 q4, q10, q6 \n" // + c2 * x1^2
|
||||
"vmul.f32 q5, q5, q1 \n" // x0^3
|
||||
"vmul.f32 q6, q6, q2 \n" // x1^3
|
||||
"vmla.f32 q3, q11, q5 \n" // + c3 * x0^3
|
||||
"vmla.f32 q4, q11, q6 \n" // + c3 * x1^3
|
||||
"vcvt.s32.f32 q3, q3 \n"
|
||||
"vcvt.s32.f32 q4, q4 \n"
|
||||
"vqmovn.s32 d6, q3 \n"
|
||||
"vqmovn.s32 d7, q4 \n"
|
||||
"vqmovun.s16 d6, q3 \n"
|
||||
"vst1.8 {d6}, [%1]! \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"+r"(poly) // %3
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q8", "q9",
|
||||
"q10", "q11");
|
||||
}
|
||||
|
||||
void ARGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width) {
|
||||
const uint8_t* table_g = table_argb + 1;
|
||||
const uint8_t* table_r = table_argb + 2;
|
||||
const uint8_t* table_a = table_argb + 3;
|
||||
uint32_t b, g, r, a;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldrb %2, [%0] \n"
|
||||
"ldrb %3, [%0, #1] \n"
|
||||
"ldrb %4, [%0, #2] \n"
|
||||
"ldrb %5, [%0, #3] \n"
|
||||
"ldrb %2, [%6, %2, lsl #2] \n"
|
||||
"ldrb %3, [%7, %3, lsl #2] \n"
|
||||
"ldrb %4, [%8, %4, lsl #2] \n"
|
||||
"ldrb %5, [%9, %5, lsl #2] \n"
|
||||
"subs %1, %1, #1 \n"
|
||||
"strb %2, [%0] \n"
|
||||
"strb %3, [%0, #1] \n"
|
||||
"strb %4, [%0, #2] \n"
|
||||
"strb %5, [%0, #3] \n"
|
||||
"add %0, %0, #4 \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(b), // %2
|
||||
"=&r"(g), // %3
|
||||
"=&r"(r), // %4
|
||||
"=&r"(a) // %5
|
||||
: "r"(table_argb), // %6
|
||||
"r"(table_g), // %7
|
||||
"r"(table_r), // %8
|
||||
"r"(table_a) // %9
|
||||
: "memory", "cc");
|
||||
}
|
||||
|
||||
void RGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width) {
|
||||
const uint8_t* table_g = table_argb + 1;
|
||||
const uint8_t* table_r = table_argb + 2;
|
||||
uint32_t b, g, r;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldrb %2, [%0] \n"
|
||||
"ldrb %3, [%0, #1] \n"
|
||||
"ldrb %4, [%0, #2] \n"
|
||||
"ldrb %2, [%5, %2, lsl #2] \n"
|
||||
"ldrb %3, [%6, %3, lsl #2] \n"
|
||||
"ldrb %4, [%7, %4, lsl #2] \n"
|
||||
"subs %1, %1, #1 \n"
|
||||
"strb %2, [%0] \n"
|
||||
"strb %3, [%0, #1] \n"
|
||||
"strb %4, [%0, #2] \n"
|
||||
"add %0, %0, #4 \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(b), // %2
|
||||
"=&r"(g), // %3
|
||||
"=&r"(r) // %4
|
||||
: "r"(table_argb), // %5
|
||||
"r"(table_g), // %6
|
||||
"r"(table_r) // %7
|
||||
: "memory", "cc");
|
||||
}
|
||||
|
||||
void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width,
|
||||
const uint8_t* luma,
|
||||
uint32_t lumacoeff) {
|
||||
uint32_t idx, addr;
|
||||
asm volatile(
|
||||
"vdup.32 q4, %6 \n" // lumacoeff
|
||||
"1: \n"
|
||||
"vld1.8 {q0}, [%0]! \n" // load 4 ARGB pixels
|
||||
"subs %2, %2, #4 \n" // 4 pixels per loop
|
||||
"vmull.u8 q1, d0, d8 \n"
|
||||
"vmull.u8 q2, d1, d9 \n"
|
||||
"vpadd.i16 d2, d2, d3 \n"
|
||||
"vpadd.i16 d3, d4, d5 \n"
|
||||
"vpadd.i16 d2, d2, d3 \n"
|
||||
"vbic.i16 d2, #0x8000 \n" // clear bit 15
|
||||
"vshrn.i16 d2, q1, #8 \n" // L_k & 0x7f
|
||||
"vmov d3, d2 \n"
|
||||
"vzip.8 d2, d3 \n"
|
||||
"vmov d3, d2 \n"
|
||||
"vzip.8 d2, d3 \n"
|
||||
"vmov q2, q0 \n"
|
||||
"vzip.8 q2, q1 \n" // q2: p0,1; q1: p2,3
|
||||
"vmov.u16 %3, d4[0] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[0]}, [%4] \n"
|
||||
"vmov.u16 %3, d4[1] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[1]}, [%4] \n"
|
||||
"vmov.u16 %3, d4[2] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[2]}, [%4] \n"
|
||||
"vmov.u16 %3, d5[0] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[4]}, [%4] \n"
|
||||
"vmov.u16 %3, d5[1] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[5]}, [%4] \n"
|
||||
"vmov.u16 %3, d5[2] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[6]}, [%4] \n"
|
||||
"vmov.u16 %3, d2[0] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[0]}, [%4] \n"
|
||||
"vmov.u16 %3, d2[1] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[1]}, [%4] \n"
|
||||
"vmov.u16 %3, d2[2] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[2]}, [%4] \n"
|
||||
"vmov.u16 %3, d3[0] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[4]}, [%4] \n"
|
||||
"vmov.u16 %3, d3[1] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[5]}, [%4] \n"
|
||||
"vmov.u16 %3, d3[2] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[6]}, [%4] \n"
|
||||
"vst1.8 {q0}, [%1]! \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(idx), // %3
|
||||
"=&r"(addr) // %4
|
||||
: "r"(luma), // %5
|
||||
"r"(lumacoeff) // %6
|
||||
: "memory", "cc", "q0", "q1", "q2", "q4");
|
||||
}
|
||||
|
||||
void ARGBAffineRow_NEON(const uint8_t* src_argb,
|
||||
int src_argb_stride,
|
||||
uint8_t* dst_argb,
|
||||
const float* uv_dudv,
|
||||
int width) {
|
||||
uint32_t off0, off1;
|
||||
asm volatile(
|
||||
"vld1.32 {d4, d5}, [%5] \n" // d4=[u,v], d5=[du,dv]
|
||||
"vmov d12, d5 \n" // d12=[du,dv]
|
||||
"vdup.32 d10, %4 \n" // [stride, stride]
|
||||
"vmov.i32 d11, #4 \n" // [4, 4]
|
||||
"vtrn.32 d11, d10 \n" // q5=[4, stride, 4,
|
||||
// stride]
|
||||
"subs %1, %1, #4 \n"
|
||||
"blt 2f \n"
|
||||
"vmov d13, d12 \n" // q6=[du, dv, du, dv]
|
||||
"vadd.f32 q4, q6, q6 \n" // 2 * [du, dv, du, dv]
|
||||
"vadd.f32 d5, d4, d12 \n" // [u+du, v+dv]
|
||||
"vadd.f32 q3, q2, q4 \n" // [u+2du, v+2dv, u+3du,
|
||||
// v+3dv]
|
||||
"vadd.f32 q4, q4, q4 \n" // 4 * [du, dv, du, dv]
|
||||
"1: \n"
|
||||
"vcvt.s32.f32 q0, q2 \n"
|
||||
"vcvt.s32.f32 q1, q3 \n"
|
||||
"vadd.f32 q2, q2, q4 \n"
|
||||
"vadd.f32 q3, q3, q4 \n"
|
||||
"vmul.i32 q0, q0, q5 \n"
|
||||
"vmul.i32 q1, q1, q5 \n"
|
||||
"vpadd.i32 d14, d0, d1 \n"
|
||||
"vpadd.i32 d15, d2, d3 \n"
|
||||
"vmov %2, %3, d14 \n"
|
||||
"ldr %2, [%6, %2] \n"
|
||||
"ldr %3, [%6, %3] \n"
|
||||
"str %2, [%0], #4 \n"
|
||||
"str %3, [%0], #4 \n"
|
||||
"vmov %2, %3, d15 \n"
|
||||
"ldr %2, [%6, %2] \n"
|
||||
"ldr %3, [%6, %3] \n"
|
||||
"str %2, [%0], #4 \n"
|
||||
"str %3, [%0], #4 \n"
|
||||
"subs %1, %1, #4 \n"
|
||||
"bge 1b \n"
|
||||
"2: \n"
|
||||
"adds %1, %1, #4 \n"
|
||||
"beq 4f \n"
|
||||
"3: \n"
|
||||
"vcvt.s32.f32 d0, d4 \n"
|
||||
"vadd.f32 d4, d4, d12 \n"
|
||||
"vmul.i32 d0, d0, d10 \n"
|
||||
"vpadd.i32 d0, d0, d0 \n"
|
||||
"vmov.32 %2, d0[0] \n"
|
||||
"ldr %2, [%6, %2] \n"
|
||||
"str %2, [%0], #4 \n"
|
||||
"subs %1, %1, #1 \n"
|
||||
"bgt 3b \n"
|
||||
"4: \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(off0), // %2
|
||||
"=&r"(off1) // %3
|
||||
: "r"(src_argb_stride), // %4
|
||||
"r"(uv_dudv), // %5
|
||||
"r"(src_argb) // %6
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7");
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__)..
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -5642,6 +5642,471 @@ void Convert8To16Row_NEON(const uint8_t* src_y,
|
||||
: "cc", "memory", "v0", "v1", "v2");
|
||||
}
|
||||
|
||||
void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count) {
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n"
|
||||
"subs %w2, %w2, #16 \n" // 16 processed per loop
|
||||
"stp q0, q1, [%1], #32 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(count) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1");
|
||||
}
|
||||
|
||||
void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width) {
|
||||
asm volatile(
|
||||
// Start at end of source row.
|
||||
"ld1 {v4.16b}, [%3] \n" // shuffler
|
||||
"add %0, %0, %w2, sxtw #1 \n"
|
||||
"sub %0, %0, #32 \n"
|
||||
"1: \n"
|
||||
"ldr q1, [%0, 16] \n"
|
||||
"ldr q0, [%0], -32 \n" // src -= 32
|
||||
"subs %w2, %w2, #16 \n" // 16 pixels per loop.
|
||||
"tbl v2.16b, {v1.16b}, v4.16b \n"
|
||||
"tbl v3.16b, {v0.16b}, v4.16b \n"
|
||||
"stp q2, q3, [%1], #32 \n" // dst += 32
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
: "r"(&kShuffleMirrorUV) // %3
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4");
|
||||
}
|
||||
|
||||
void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width) {
|
||||
asm volatile(
|
||||
"movi v16.4s, #0xff, lsl #8 \n" // 0x0000ff00
|
||||
"movi v17.4s, #0x0f, msl #16 \n" // 0x000fffff
|
||||
"shl v17.4s, v17.4s, #8 \n" // 0x0fffff00
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels
|
||||
"subs %w2, %w2, #8 \n" // 8 pixels per loop
|
||||
"sri v0.4s, v0.4s, #2 \n"
|
||||
"sri v1.4s, v1.4s, #2 \n"
|
||||
"ushr v2.4s, v0.4s, #2 \n"
|
||||
"ushr v3.4s, v1.4s, #2 \n"
|
||||
"ushr v4.4s, v0.4s, #4 \n"
|
||||
"ushr v5.4s, v1.4s, #4 \n"
|
||||
"bit v0.16b, v4.16b, v17.16b \n"
|
||||
"bit v1.16b, v5.16b, v17.16b \n"
|
||||
"bit v0.16b, v2.16b, v16.16b \n"
|
||||
"bit v1.16b, v3.16b, v16.16b \n"
|
||||
"stp q0, q1, [%1], #32 \n" // store 8 ARGB pixels
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17");
|
||||
}
|
||||
|
||||
void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) {
|
||||
asm volatile(
|
||||
"movi v16.2d, #0x00ffff0000ffff00 \n"
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels
|
||||
"subs %w2, %w2, #8 \n" // 8 pixels per loop
|
||||
"sri v0.4s, v0.4s, #2 \n"
|
||||
"sri v1.4s, v1.4s, #2 \n"
|
||||
"ushr v2.4s, v0.4s, #2 \n"
|
||||
"ushr v3.4s, v1.4s, #2 \n"
|
||||
"sli v2.4s, v0.4s, #16 \n"
|
||||
"sli v3.4s, v1.4s, #16 \n"
|
||||
"sri v0.4s, v0.4s, #4 \n"
|
||||
"sri v1.4s, v1.4s, #4 \n"
|
||||
"sri v0.4s, v0.4s, #16 \n"
|
||||
"sri v1.4s, v1.4s, #16 \n"
|
||||
"bit v0.16b, v2.16b, v16.16b \n"
|
||||
"bit v1.16b, v3.16b, v16.16b \n"
|
||||
"stp q0, q1, [%1], #32 \n" // store 8 ABGR pixels
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_abgr), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v16");
|
||||
}
|
||||
|
||||
void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) {
|
||||
asm volatile(
|
||||
"movi v17.16b, #0xff \n"
|
||||
"ushr v17.4s, v17.4s, #22 \n" // 0x000003ff
|
||||
"shl v16.4s, v17.4s, #20 \n" // 0x3ff00000
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels
|
||||
"subs %w2, %w2, #8 \n" // 8 pixels per loop
|
||||
"shl v2.4s, v0.4s, #20 \n"
|
||||
"shl v3.4s, v1.4s, #20 \n"
|
||||
"ushr v4.4s, v0.4s, #20 \n"
|
||||
"ushr v5.4s, v1.4s, #20 \n"
|
||||
"bit v0.16b, v2.16b, v16.16b \n"
|
||||
"bit v1.16b, v3.16b, v16.16b \n"
|
||||
"bit v0.16b, v4.16b, v17.16b \n"
|
||||
"bit v1.16b, v5.16b, v17.16b \n"
|
||||
"stp q0, q1, [%1], #32 \n" // store 8 AB30 pixels
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_ab30), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17");
|
||||
}
|
||||
|
||||
void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
|
||||
asm volatile(
|
||||
"movi v4.4s, #0xff, lsl #24 \n" // 0xff000000
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n" // load 8 src pixels
|
||||
"ldp q2, q3, [%1] \n" // load 8 dst pixels
|
||||
"subs %w2, %w2, #8 \n" // 8 pixels per loop
|
||||
"bit v2.16b, v0.16b, v4.16b \n"
|
||||
"bit v3.16b, v1.16b, v4.16b \n"
|
||||
"stp q2, q3, [%1], #32 \n" // store 8 dst pixels
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4");
|
||||
}
|
||||
|
||||
void ARGBUnattenuateRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width) {
|
||||
uint64_t tmp;
|
||||
asm volatile(
|
||||
"movi v16.4s, #0xff, lsl #24 \n" // 0xff000000
|
||||
"1: \n"
|
||||
"ldrb %w3, [%0, #3] \n"
|
||||
"ldr s4, [%4, %w3, uxtw #2] \n"
|
||||
"ldrb %w3, [%0, #7] \n"
|
||||
"ldr s5, [%4, %w3, uxtw #2] \n"
|
||||
"ldrb %w3, [%0, #11] \n"
|
||||
"ldr s6, [%4, %w3, uxtw #2] \n"
|
||||
"ldrb %w3, [%0, #15] \n"
|
||||
"ldr s7, [%4, %w3, uxtw #2] \n"
|
||||
"ldr q0, [%0], #16 \n"
|
||||
"subs %w2, %w2, #4 \n"
|
||||
"zip1 v1.16b, v0.16b, v0.16b \n"
|
||||
"zip2 v2.16b, v0.16b, v0.16b \n"
|
||||
"umull v3.4s, v1.4h, v4.h[0] \n"
|
||||
"umull2 v5.4s, v1.8h, v5.h[0] \n"
|
||||
"umull v6.4s, v2.4h, v6.h[0] \n"
|
||||
"umull2 v7.4s, v2.8h, v7.h[0] \n"
|
||||
"uzp2 v3.8h, v3.8h, v5.8h \n"
|
||||
"uzp2 v6.8h, v6.8h, v7.8h \n"
|
||||
"uqxtn v1.8b, v3.8h \n"
|
||||
"uqxtn2 v1.16b, v6.8h \n"
|
||||
"bit v1.16b, v0.16b, v16.16b \n"
|
||||
"str q1, [%1], #16 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(tmp) // %3
|
||||
: "r"(fixed_invtbl8) // %4
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
|
||||
}
|
||||
|
||||
void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
int width) {
|
||||
asm volatile(
|
||||
"ldp q16, q17, [%3] \n" // c0, c1
|
||||
"ldp q18, q19, [%3, #32] \n" // c2, c3
|
||||
"subs %w2, %w2, #4 \n"
|
||||
"b.lt 2f \n"
|
||||
"1: \n"
|
||||
"ldr q0, [%0], #16 \n" // load 4 pixels
|
||||
"subs %w2, %w2, #4 \n"
|
||||
"uxtl v1.8h, v0.8b \n"
|
||||
"uxtl2 v3.8h, v0.16b \n"
|
||||
"uxtl v0.4s, v1.4h \n"
|
||||
"uxtl2 v1.4s, v1.8h \n"
|
||||
"uxtl v2.4s, v3.4h \n"
|
||||
"uxtl2 v3.4s, v3.8h \n"
|
||||
"ucvtf v0.4s, v0.4s \n" // x0
|
||||
"ucvtf v1.4s, v1.4s \n" // x1
|
||||
"ucvtf v2.4s, v2.4s \n" // x2
|
||||
"ucvtf v3.4s, v3.4s \n" // x3
|
||||
"fmul v20.4s, v0.4s, v0.4s \n" // x0^2
|
||||
"fmul v21.4s, v1.4s, v1.4s \n" // x1^2
|
||||
"fmul v22.4s, v2.4s, v2.4s \n" // x2^2
|
||||
"fmul v23.4s, v3.4s, v3.4s \n" // x3^2
|
||||
"fmul v4.4s, v17.4s, v0.4s \n" // c1 * x0
|
||||
"fmul v5.4s, v17.4s, v1.4s \n" // c1 * x1
|
||||
"fmul v6.4s, v17.4s, v2.4s \n" // c1 * x2
|
||||
"fmul v7.4s, v17.4s, v3.4s \n" // c1 * x3
|
||||
"fmul v0.4s, v20.4s, v0.4s \n" // x0^3
|
||||
"fmul v1.4s, v21.4s, v1.4s \n" // x1^3
|
||||
"fmul v2.4s, v22.4s, v2.4s \n" // x2^3
|
||||
"fmul v3.4s, v23.4s, v3.4s \n" // x3^3
|
||||
"fadd v4.4s, v16.4s, v4.4s \n" // c0 + c1 * x0
|
||||
"fadd v5.4s, v16.4s, v5.4s \n" // c0 + c1 * x1
|
||||
"fadd v6.4s, v16.4s, v6.4s \n" // c0 + c1 * x2
|
||||
"fadd v7.4s, v16.4s, v7.4s \n" // c0 + c1 * x3
|
||||
"fmla v4.4s, v18.4s, v20.4s \n" // + c2 * x0^2
|
||||
"fmla v5.4s, v18.4s, v21.4s \n" // + c2 * x1^2
|
||||
"fmla v6.4s, v18.4s, v22.4s \n" // + c2 * x2^2
|
||||
"fmla v7.4s, v18.4s, v23.4s \n" // + c2 * x3^2
|
||||
"fmla v4.4s, v19.4s, v0.4s \n" // + c3 * x0^3
|
||||
"fmla v5.4s, v19.4s, v1.4s \n" // + c3 * x1^3
|
||||
"fmla v6.4s, v19.4s, v2.4s \n" // + c3 * x2^3
|
||||
"fmla v7.4s, v19.4s, v3.4s \n" // + c3 * x3^3
|
||||
"fcvtzs v4.4s, v4.4s \n"
|
||||
"fcvtzs v5.4s, v5.4s \n"
|
||||
"fcvtzs v6.4s, v6.4s \n"
|
||||
"fcvtzs v7.4s, v7.4s \n"
|
||||
"sqxtn v4.4h, v4.4s \n"
|
||||
"sqxtn2 v4.8h, v5.4s \n"
|
||||
"sqxtn v6.4h, v6.4s \n"
|
||||
"sqxtn2 v6.8h, v7.4s \n"
|
||||
"sqxtun v4.8b, v4.8h \n"
|
||||
"sqxtun2 v4.16b, v6.8h \n"
|
||||
"str q4, [%1], #16 \n"
|
||||
"b.ge 1b \n"
|
||||
"2: \n"
|
||||
"adds %w2, %w2, #4 \n"
|
||||
"b.eq 3f \n"
|
||||
"ldr d0, [%0], #8 \n" // load 2 pixels
|
||||
"uxtl v0.8h, v0.8b \n"
|
||||
"uxtl v1.4s, v0.4h \n"
|
||||
"uxtl2 v2.4s, v0.8h \n"
|
||||
"ucvtf v1.4s, v1.4s \n" // x0
|
||||
"ucvtf v2.4s, v2.4s \n" // x1
|
||||
"fmul v5.4s, v1.4s, v1.4s \n" // x0^2
|
||||
"fmul v6.4s, v2.4s, v2.4s \n" // x1^2
|
||||
"fmul v3.4s, v17.4s, v1.4s \n" // c1 * x0
|
||||
"fmul v4.4s, v17.4s, v2.4s \n" // c1 * x1
|
||||
"fmul v1.4s, v5.4s, v1.4s \n" // x0^3
|
||||
"fmul v2.4s, v6.4s, v2.4s \n" // x1^3
|
||||
"fadd v3.4s, v16.4s, v3.4s \n" // c0 + c1 * x0
|
||||
"fadd v4.4s, v16.4s, v4.4s \n" // c0 + c1 * x1
|
||||
"fmla v3.4s, v18.4s, v5.4s \n" // + c2 * x0^2
|
||||
"fmla v4.4s, v18.4s, v6.4s \n" // + c2 * x1^2
|
||||
"fmla v3.4s, v19.4s, v1.4s \n" // + c3 * x0^3
|
||||
"fmla v4.4s, v19.4s, v2.4s \n" // + c3 * x1^3
|
||||
"fcvtzs v3.4s, v3.4s \n"
|
||||
"fcvtzs v4.4s, v4.4s \n"
|
||||
"sqxtn v3.4h, v3.4s \n"
|
||||
"sqxtn2 v3.8h, v4.4s \n"
|
||||
"sqxtun v3.8b, v3.8h \n"
|
||||
"str d3, [%1], #8 \n"
|
||||
"3: \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width) // %2
|
||||
: "r"(poly) // %3
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
|
||||
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
|
||||
}
|
||||
|
||||
void ARGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width) {
|
||||
uint64_t b, g, r, a;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldrb %w2, [%0] \n"
|
||||
"ldrb %w3, [%0, #1] \n"
|
||||
"ldrb %w4, [%0, #2] \n"
|
||||
"ldrb %w5, [%0, #3] \n"
|
||||
"subs %w1, %w1, #1 \n"
|
||||
"ldr %w2, [%6, %w2, uxtw #2] \n"
|
||||
"ldr %w3, [%6, %w3, uxtw #2] \n"
|
||||
"ldr %w4, [%6, %w4, uxtw #2] \n"
|
||||
"ldr %w5, [%6, %w5, uxtw #2] \n"
|
||||
"bfi %w3, %w2, #0, #8 \n"
|
||||
"lsr %w4, %w4, #16 \n"
|
||||
"lsr %w5, %w5, #24 \n"
|
||||
"strh %w3, [%0] \n"
|
||||
"strb %w4, [%0, #2] \n"
|
||||
"strb %w5, [%0, #3] \n"
|
||||
"add %0, %0, #4 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(b), // %2
|
||||
"=&r"(g), // %3
|
||||
"=&r"(r), // %4
|
||||
"=&r"(a) // %5
|
||||
: "r"(table_argb) // %6
|
||||
: "memory", "cc");
|
||||
}
|
||||
|
||||
void RGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width) {
|
||||
uint64_t b, g, r;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldrb %w2, [%0] \n"
|
||||
"ldrb %w3, [%0, #1] \n"
|
||||
"ldrb %w4, [%0, #2] \n"
|
||||
"subs %w1, %w1, #1 \n"
|
||||
"ldr %w2, [%5, %w2, uxtw #2] \n"
|
||||
"ldr %w3, [%5, %w3, uxtw #2] \n"
|
||||
"ldr %w4, [%5, %w4, uxtw #2] \n"
|
||||
"bfi %w3, %w2, #0, #8 \n"
|
||||
"lsr %w4, %w4, #16 \n"
|
||||
"strh %w3, [%0] \n"
|
||||
"strb %w4, [%0, #2] \n"
|
||||
"add %0, %0, #4 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(b), // %2
|
||||
"=&r"(g), // %3
|
||||
"=&r"(r) // %4
|
||||
: "r"(table_argb) // %5
|
||||
: "memory", "cc");
|
||||
}
|
||||
|
||||
static const uvec8 kLumaTblIdx = {1, 1, 1, 1, 3, 3, 3, 3,
|
||||
5, 5, 5, 5, 7, 7, 7, 7};
|
||||
|
||||
void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width,
|
||||
const uint8_t* luma,
|
||||
uint32_t lumacoeff) {
|
||||
uint64_t p0, p1, t0, t1;
|
||||
asm volatile(
|
||||
"dup v4.4s, %w8 \n" // broadcast 4 byte coeffs
|
||||
"ldr q5, [%9] \n"
|
||||
"movi v6.4s, #0xff, lsl #24 \n"
|
||||
"1: \n"
|
||||
"ldr q0, [%0], #16 \n" // load 4 ARGB pixels
|
||||
"subs %w2, %w2, #4 \n" // 4 pixels per loop
|
||||
"umull v1.8h, v0.8b, v4.8b \n"
|
||||
"umull2 v2.8h, v0.16b, v4.16b \n"
|
||||
"addp v1.8h, v1.8h, v2.8h \n"
|
||||
"addp v1.8h, v1.8h, v1.8h \n"
|
||||
"bic v1.8h, #0x80, lsl #8 \n" // clear bit 15
|
||||
"tbl v1.16b, {v1.16b}, v5.16b \n"
|
||||
"zip2 v2.16b, v0.16b, v1.16b \n" // offsets pixels 2, 3
|
||||
"zip1 v1.16b, v0.16b, v1.16b \n" // offsets pixels 0, 1
|
||||
"umov %3, v1.d[0] \n"
|
||||
"umov %4, v1.d[1] \n"
|
||||
"add %5, %7, %w3, uxth \n"
|
||||
"ubfx %6, %3, #16, #16 \n"
|
||||
"ldr b3, [%5] \n"
|
||||
"add %6, %7, %6 \n"
|
||||
"ubfx %5, %3, #32, #16 \n"
|
||||
"ld1 {v3.b}[1], [%6] \n"
|
||||
"add %5, %7, %5 \n"
|
||||
"add %6, %7, %w4, uxth \n"
|
||||
"ld1 {v3.b}[2], [%5] \n"
|
||||
"ld1 {v3.b}[4], [%6] \n"
|
||||
"ubfx %5, %4, #16, #16 \n"
|
||||
"ubfx %6, %4, #32, #16 \n"
|
||||
"add %5, %7, %5 \n"
|
||||
"add %6, %7, %6 \n"
|
||||
"ld1 {v3.b}[5], [%5] \n"
|
||||
"ld1 {v3.b}[6], [%6] \n"
|
||||
"umov %3, v2.d[0] \n"
|
||||
"umov %4, v2.d[1] \n"
|
||||
"add %5, %7, %w3, uxth \n"
|
||||
"ubfx %6, %3, #16, #16 \n"
|
||||
"ld1 {v3.b}[8], [%5] \n"
|
||||
"add %6, %7, %6 \n"
|
||||
"ubfx %5, %3, #32, #16 \n"
|
||||
"ld1 {v3.b}[9], [%6] \n"
|
||||
"add %5, %7, %5 \n"
|
||||
"add %6, %7, %w4, uxth \n"
|
||||
"ld1 {v3.b}[10], [%5] \n"
|
||||
"ld1 {v3.b}[12], [%6] \n"
|
||||
"ubfx %5, %4, #16, #16 \n"
|
||||
"ubfx %6, %4, #32, #16 \n"
|
||||
"add %5, %7, %5 \n"
|
||||
"add %6, %7, %6 \n"
|
||||
"ld1 {v3.b}[13], [%5] \n"
|
||||
"ld1 {v3.b}[14], [%6] \n"
|
||||
"bit v3.16b, v0.16b, v6.16b \n"
|
||||
"str q3, [%1], #16 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(p0), // %3
|
||||
"=&r"(p1), // %4
|
||||
"=&r"(t0), // %5
|
||||
"=&r"(t1) // %6
|
||||
: "r"(luma), // %7
|
||||
"r"(lumacoeff), // %8
|
||||
"r"(&kLumaTblIdx) // %9
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
|
||||
}
|
||||
|
||||
void ARGBAffineRow_NEON(const uint8_t* src_argb,
|
||||
int src_argb_stride,
|
||||
uint8_t* dst_argb,
|
||||
const float* uv_dudv,
|
||||
int width) {
|
||||
int64_t off0, off1;
|
||||
asm volatile(
|
||||
"ld1r {v2.2d}, [%2], #8 \n" // [u, v, u, v]
|
||||
"ld1r {v6.2d}, [%2] \n" // [du, dv, du, dv]
|
||||
"dup v5.2s, %w5 \n" // [stride, stride]
|
||||
"movi v4.2s, #4 \n" // [4, 4]
|
||||
"zip1 v5.4s, v4.4s, v5.4s \n" // [4, stride, 4, stride]
|
||||
"subs %w1, %w1, #4 \n"
|
||||
"b.lt 2f \n"
|
||||
"fadd v4.4s, v6.4s, v6.4s \n" // 2 * [du, dv, du, dv]
|
||||
"fadd v3.2s, v2.2s, v6.2s \n" // [u+du, v+dv]
|
||||
"ins v2.d[1], v3.d[0] \n" // [u, v, u+du, v+dv]
|
||||
"fadd v3.4s, v2.4s, v4.4s \n" // [u+2du, v+2dv, u+3du,
|
||||
// v+3dv]
|
||||
"fadd v4.4s, v4.4s, v4.4s \n" // 4 * [du, dv, du, dv]
|
||||
"1: \n"
|
||||
"fcvtzs v0.4s, v2.4s \n"
|
||||
"fcvtzs v1.4s, v3.4s \n"
|
||||
"fadd v2.4s, v2.4s, v4.4s \n"
|
||||
"fadd v3.4s, v3.4s, v4.4s \n"
|
||||
"mul v0.4s, v0.4s, v5.4s \n"
|
||||
"mul v1.4s, v1.4s, v5.4s \n"
|
||||
"addp v7.4s, v0.4s, v1.4s \n"
|
||||
"smov %3, v7.s[0] \n"
|
||||
"smov %4, v7.s[1] \n"
|
||||
"ldr %w3, [%6, %3] \n"
|
||||
"ldr %w4, [%6, %4] \n"
|
||||
"stp %w3, %w4, [%0], #8 \n"
|
||||
"smov %3, v7.s[2] \n"
|
||||
"smov %4, v7.s[3] \n"
|
||||
"ldr %w3, [%6, %3] \n"
|
||||
"ldr %w4, [%6, %4] \n"
|
||||
"stp %w3, %w4, [%0], #8 \n"
|
||||
"subs %w1, %w1, #4 \n"
|
||||
"b.ge 1b \n"
|
||||
"2: \n"
|
||||
"adds %w1, %w1, #4 \n"
|
||||
"b.eq 4f \n"
|
||||
"3: \n"
|
||||
"fcvtzs v0.2s, v2.2s \n"
|
||||
"fadd v2.2s, v2.2s, v6.2s \n"
|
||||
"mul v0.2s, v0.2s, v5.2s \n"
|
||||
"addp v0.2s, v0.2s, v0.2s \n"
|
||||
"smov %3, v0.s[0] \n"
|
||||
"ldr %w3, [%6, %3] \n"
|
||||
"str %w3, [%0], #4 \n"
|
||||
"subs %w1, %w1, #1 \n"
|
||||
"b.gt 3b \n"
|
||||
"4: \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"+r"(uv_dudv), // %2
|
||||
"=&r"(off0), // %3
|
||||
"=&r"(off1) // %4
|
||||
: "r"(src_argb_stride), // %5
|
||||
"r"(src_argb) // %6
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -1574,6 +1574,20 @@ TEST_F(LibYUVPlanarTest, TestAffine) {
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBAFFINEROW_NEON)
|
||||
SIMD_ALIGNED(uint8_t interpolate_pixels_Opt[1280][4]);
|
||||
ARGBAffineRow_NEON(&orig_pixels_0[0][0], 0, &interpolate_pixels_Opt[0][0],
|
||||
uv_step, 1280);
|
||||
ASSERT_EQ(0, memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1280 * 4));
|
||||
|
||||
int has_neon = TestCpuFlag(kCpuHasNEON);
|
||||
if (has_neon) {
|
||||
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
|
||||
ARGBAffineRow_NEON(&orig_pixels_0[0][0], 0, &interpolate_pixels_Opt[0][0],
|
||||
uv_step, 1280);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif // ENABLE_ROW_TESTS
|
||||
|
||||
@ -2417,7 +2431,7 @@ TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Opt)) {
|
||||
ASSERT_LE(max_diff, 1);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVPlanarTest, DISABLED_ARM(TestARGBPolynomial)) {
|
||||
TEST_F(LibYUVPlanarTest, TestARGBPolynomial) {
|
||||
SIMD_ALIGNED(uint8_t orig_pixels[1280][4]);
|
||||
SIMD_ALIGNED(uint8_t dst_pixels_opt[1280][4]);
|
||||
SIMD_ALIGNED(uint8_t dst_pixels_c[1280][4]);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user