mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
[libyuv] Add NEON implementations for all remaining C row functions
Implement 32-bit and 64-bit Arm NEON versions (in row_neon.cc and row_neon64.cc) for the 12 remaining row functions that previously fell back to C on Arm: - AR30ToAB30Row_NEON (and AR30ToAB30Row_Any_NEON) - AR30ToABGRRow_NEON (and AR30ToABGRRow_Any_NEON) - AR30ToARGBRow_NEON (and AR30ToARGBRow_Any_NEON) - ARGBAffineRow_NEON - ARGBColorTableRow_NEON - ARGBCopyAlphaRow_NEON (and ARGBCopyAlphaRow_Any_NEON) - ARGBLumaColorTableRow_NEON - ARGBPolynomialRow_NEON (and ARGBPolynomialRow_Any_NEON) - ARGBUnattenuateRow_NEON (and ARGBUnattenuateRow_Any_NEON) - CopyRow_16_NEON (and CopyRow_16_Any_NEON) - MirrorRow_16_NEON (and MirrorRow_16_Any_NEON) - RGBColorTableRow_NEON Add HAS_*_NEON macros, prototypes, _Any wrappers, NEON dispatches in convert.cc, convert_argb.cc, and planar_functions.cc, and enable unit tests in planar_test.cc. Arm row function SIMD coverage is now 164/164 (100%). ARM64 (Apple M6) Benchmark (1280 width, 10000 iterations): ---------------------------------------------------------------- Function Before C (ms) After NEON (ms) Improvement ---------------------------------------------------------------- AR30ToAB30Row 1.20 0.70 1.72x AR30ToABGRRow 1.88 1.03 1.83x AR30ToARGBRow 1.88 0.87 2.17x ARGBAffineRow 7.96 2.31 3.44x ARGBColorTableRow 8.17 7.43 1.10x ARGBCopyAlphaRow 1.37 0.53 2.58x ARGBLumaColorTableRow 8.18 5.24 1.56x ARGBPolynomialRow 8.73 7.55 1.16x ARGBUnattenuateRow 6.00 2.05 2.93x CopyRow_16 0.18 0.18 1.00x MirrorRow_16 0.34 0.20 1.73x RGBColorTableRow 6.15 5.58 1.10x ---------------------------------------------------------------- Maximum SIMD ISA supported per C row function: ---------------------------------------------------------------- C Row Function arm32 arm64 x86 riscv ---------------------------------------------------------------- AB64ToARGBRow_C NEON NEON AVX2 RVV ABGRToAR30Row_C - NEON AVX512BW - AR30ToAB30Row_C NEON NEON AVX512BW - AR30ToABGRRow_C NEON NEON AVX512BW - AR30ToARGBRow_C NEON NEON AVX512BW - AR64ShuffleRow_C NEON NEON AVX512BW RVV AR64ToAB64Row_C NEON NEON AVX512BW RVV AR64ToARGBRow_C NEON NEON AVX2 RVV ARGB1555ToARGBRow_C NEON SVE2 AVX2 - ARGB1555ToUVMatrixRow_C NEON NEON AVX2 - ARGB1555ToUVRow_C NEON NEON - - ARGB1555ToYMatrixRow_C NEON NEON AVX2 - ARGB1555ToYRow_C NEON NEON - - ARGB4444ToARGBRow_C NEON NEON AVX2 - ARGB4444ToUVMatrixRow_C NEON NEON AVX2 - ARGB4444ToUVRow_C NEON NEON - - ARGB4444ToYMatrixRow_C NEON NEON AVX2 - ARGB4444ToYRow_C NEON NEON - - ARGBAddRow_C NEON NEON AVX2 - ARGBAffineRow_C NEON NEON SSE2 - ARGBAlphaBlendRow_C NEON SME AVX512BW RVV ARGBAttenuateRow_C NEON NEON AVX512BW RVV ARGBBlendRow_C NEON NEON AVX512BW RVV ARGBColorMatrixRow_C NEON I8MM SSSE3 - ARGBColorTableRow_C NEON NEON X86 - ARGBCopyAlphaRow_C NEON NEON AVX2 - ARGBCopyYToAlphaRow_C NEON NEON AVX512BW RVV ARGBExtractAlphaRow_C NEON NEON AVX512BW RVV ARGBGrayRow_C NEON DOTPROD AVX512VNNI - ARGBLumaColorTableRow_C NEON NEON SSSE3 - ARGBMirrorRow_C NEON NEON AVX512BW RVV ARGBMultiplyRow_C NEON SME AVX512BW RVV ARGBOrRow_C NEON SME AVX512BW RVV ARGBPolynomialRow_C NEON NEON AVX2 - ARGBQuantizeRow_C NEON NEON SSE2 - ARGBSepiaRow_C NEON DOTPROD SSSE3 - ARGBSetRow_C NEON NEON X86 RVV ARGBShadeRow_C NEON NEON SSE2 - ARGBShuffleRow_C NEON NEON AVX512BW RVV ARGBSubtractRow_C NEON NEON AVX2 - ARGBToAB64Row_C NEON NEON AVX2 RVV ARGBToABGRRow_C NEON NEON AVX512BW RVV ARGBToAR30Row_C - NEON AVX512BW - ARGBToAR64Row_C NEON NEON AVX2 RVV ARGBToARGB1555Row_C NEON NEON AVX2 - ARGBToARGB4444Row_C NEON NEON AVX2 - ARGBToBGRARow_C NEON NEON AVX512BW RVV ARGBToRAWRow_C NEON SVE2 AVX512BW RVV ARGBToRGB24Row_C NEON SVE2 AVX512BW RVV ARGBToRGB565DitherRow_C NEON SVE2 AVX2 - ARGBToRGB565Row_C NEON SVE2 AVX2 - ARGBToRGBARow_C NEON NEON AVX512BW RVV ARGBToUV444MatrixRow_C NEON I8MM AVX512VNNI RVV ARGBToUVMatrixRow_C NEON SME AVX512VNNI RVV ARGBToYMatrixRow_C NEON SME AVX512VNNI RVV ARGBUnattenuateRow_C NEON NEON AVX2 - AYUVToUVRow_C NEON SVE2 AVX512VBMI - AYUVToVURow_C NEON SVE2 AVX512VBMI - AYUVToYRow_C NEON NEON AVX512VBMI - BlendPlaneRow_C NEON NEON AVX512BW RVV ByteToFloatRow_C NEON NEON - - ComputeCumulativeSumRow_C - NEON AVX2 - Convert16To8Row_C NEON SME AVX512BW RVV Convert8To16Row_C NEON SME AVX512BW RVV Convert8To8Row_C NEON SME AVX2 - CopyRow_C NEON SME AVX512BW RVV CopyRow_16_C NEON NEON - - CumulativeSumToAverageRow_C - NEON AVX2 - DetileRow_C NEON NEON SSE2 - DetileRow_16_C NEON NEON AVX - DetileSplitUVRow_C NEON NEON SSSE3 - DivideRow_16_C NEON SVE2 AVX2 - GaussRow_C NEON NEON - - GaussRow_F32_C - NEON - - HalfFloatRow_C NEON SVE2 AVX2 - HalfMergeUVRow_C NEON SVE2 AVX512BW - HalfRow_16To8_C NEON SME AVX512BW RVV HalfWidthRow_16To8_C NEON SME AVX512BW RVV I210AlphaToARGBRow_C - SME AVX2 - I210ToAR30Row_C - SME AVX2 - I210ToARGBRow_C - SME AVX2 - I212ToAR30Row_C - SME AVX2 - I212ToARGBRow_C - SME AVX2 - I400ToARGBRow_C NEON SME AVX512BW RVV I410AlphaToARGBRow_C - SME AVX2 - I410ToAR30Row_C - SME AVX2 - I410ToARGBRow_C - SME AVX2 - I422AlphaToARGBRow_C NEON SME AVX512BW RVV I422ToAR30Row_C - SME AVX512BW RVV I422ToARGB1555Row_C NEON SME AVX2 - I422ToARGB4444Row_C NEON SME AVX2 - I422ToARGBRow_C NEON SME AVX512BW RVV I422ToRGB24Row_C NEON SME AVX512VBMI RVV I422ToRGB565Row_C NEON SME AVX2 - I422ToRGBARow_C NEON SME AVX2 RVV I422ToUYVYRow_C NEON NEON AVX2 - I422ToYUY2Row_C NEON NEON AVX2 - I444AlphaToARGBRow_C NEON SME AVX2 RVV I444ToARGBRow_C NEON SME AVX2 RVV I444ToRGB24Row_C NEON SME AVX2 RVV InterpolateRow_C NEON SME AVX512BW RVV InterpolateRow_16_C NEON SME AVX512BW - J400ToARGBRow_C NEON NEON AVX512BW RVV MergeAR64Row_C NEON NEON AVX2 - MergeARGB16To8Row_C NEON NEON AVX2 - MergeARGBRow_C NEON NEON AVX2 RVV MergeRGBRow_C NEON NEON AVX512VBMI RVV MergeUVRow_C NEON SME AVX512BW RVV MergeUVRow_16_C NEON SME AVX2 - MergeXR30Row_C NEON NEON AVX512BW - MergeXR64Row_C NEON NEON AVX2 - MergeXRGB16To8Row_C NEON NEON AVX2 - MergeXRGBRow_C NEON NEON AVX2 RVV MirrorRow_C NEON NEON AVX512BW - MirrorRow_16_C NEON NEON AVX512BW - MirrorSplitUVRow_C NEON NEON AVX512BW - MirrorUVRow_C NEON NEON AVX512BW RVV MultiplyRow_16_C NEON SME AVX512BW RVV NV12ToARGBRow_C NEON SME AVX512BW RVV NV12ToRGB24Row_C NEON SME AVX512VBMI RVV NV12ToRGB565Row_C NEON NEON AVX2 - NV21ToARGBRow_C NEON SME AVX512BW RVV NV21ToRGB24Row_C NEON SME AVX512VBMI RVV NV21ToYUV24Row_C NEON NEON AVX2 - P210ToAR30Row_C - SME AVX2 - P210ToARGBRow_C - SME AVX2 - P410ToAR30Row_C - SME AVX2 - P410ToARGBRow_C - SME AVX2 - RAWToARGBRow_C NEON SVE2 AVX512BW RVV RAWToRGB24Row_C NEON SVE2 AVX512VBMI RVV RAWToRGBARow_C NEON SVE2 AVX512VBMI RVV RGB24MirrorRow_C NEON NEON AVX512VBMI - RGB24ToARGBRow_C NEON SVE2 AVX512BW RVV RGB565ToARGBRow_C NEON NEON AVX2 - RGB565ToUVMatrixRow_C NEON NEON AVX2 - RGB565ToUVRow_C NEON NEON - - RGB565ToYMatrixRow_C NEON NEON AVX2 - RGB565ToYRow_C NEON NEON - - RGBAToARGBRow_C NEON NEON AVX512BW RVV RGBColorTableRow_C NEON NEON X86 - RGBToUV444MatrixRow_C NEON NEON AVX512BW RVV RGBToUVMatrixRow_C NEON NEON AVX512BW RVV RGBToYMatrixRow_C NEON DOTPROD AVX512BW RVV SetRow_C NEON NEON ERMS - SobelRow_C NEON NEON SSE2 - SobelToPlaneRow_C NEON NEON SSE2 - SobelXRow_C NEON NEON SSE2 - SobelXYRow_C NEON NEON SSE2 - SobelYRow_C NEON NEON SSE2 - SplitARGBRow_C NEON NEON AVX2 RVV SplitRGBRow_C NEON NEON AVX2 RVV SplitUVRow_C NEON SVE2 AVX512BW RVV SplitUVRow_16_C NEON NEON AVX2 - SplitXRGBRow_C NEON NEON AVX2 RVV SwapUVRow_C NEON NEON AVX512BW RVV UYVYToARGBRow_C NEON SME AVX2 - UYVYToUV422Row_C NEON NEON AVX2 - UYVYToUVRow_C NEON SVE2 AVX2 - UYVYToYRow_C NEON SVE2 AVX2 - YUY2ToARGBRow_C NEON SME AVX2 - YUY2ToNVUVRow_C NEON NEON AVX2 - YUY2ToUV422Row_C NEON NEON AVX2 - YUY2ToUVRow_C NEON SVE2 AVX2 - YUY2ToYRow_C NEON SVE2 AVX2 - ---------------------------------------------------------------- SIMD (164/164 100%) 146 89% 164 100% 154 94% 64 39% ---------------------------------------------------------------- Test: libyuv_unittest --gtest_filter=*AR30*:*Affine*:*ColorTable*:*CopyAlpha*:*Polynomial*:*Unattenuate*:*MirrorPlane_16* Bug: None Change-Id: Ib8b68beb77b47de6b996d41fd080397c61539ffb
This commit is contained in:
Frank Barchard
parent
f506767c5f
commit
1b1216abb6
@ -1,6 +1,6 @@
|
||||
Name: libyuv
|
||||
URL: https://chromium.googlesource.com/libyuv/libyuv/
|
||||
Version: 1975
|
||||
Version: 1976
|
||||
Revision: DEPS
|
||||
License: BSD-3-Clause
|
||||
License File: LICENSE
|
||||
|
||||
@ -540,6 +540,9 @@ extern "C" {
|
||||
#if !defined(LIBYUV_DISABLE_NEON) && \
|
||||
(defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON))
|
||||
#define HAS_AB64TOARGBROW_NEON
|
||||
#define HAS_AR30TOAB30ROW_NEON
|
||||
#define HAS_AR30TOABGRROW_NEON
|
||||
#define HAS_AR30TOARGBROW_NEON
|
||||
#define HAS_AR64TOARGBROW_NEON
|
||||
#define HAS_ARGB1555TOARGBROW_NEON
|
||||
#define HAS_ARGB1555TOUVROW_NEON
|
||||
@ -547,6 +550,7 @@ extern "C" {
|
||||
#define HAS_ARGB4444TOARGBROW_NEON
|
||||
#define HAS_ARGB4444TOUVROW_NEON
|
||||
#define HAS_ARGB4444TOYROW_NEON
|
||||
#define HAS_ARGBCOPYALPHAROW_NEON
|
||||
#define HAS_ARGBCOPYYTOALPHAROW_NEON
|
||||
#define HAS_ARGBEXTRACTALPHAROW_NEON
|
||||
#define HAS_ARGBSETROW_NEON
|
||||
@ -578,6 +582,7 @@ extern "C" {
|
||||
#define HAS_CONVERT16TO8ROW_NEON
|
||||
#define HAS_CONVERT8TO16ROW_NEON
|
||||
#define HAS_CONVERT8TO8ROW_NEON
|
||||
#define HAS_COPYROW_16_NEON
|
||||
#define HAS_COPYROW_NEON
|
||||
#define HAS_DETILEROW_16_NEON
|
||||
#define HAS_DETILEROW_NEON
|
||||
@ -614,6 +619,7 @@ extern "C" {
|
||||
#define HAS_MERGEXR64ROW_NEON
|
||||
#define HAS_MERGEXRGB16TO8ROW_NEON
|
||||
#define HAS_MERGEXRGBROW_NEON
|
||||
#define HAS_MIRRORROW_16_NEON
|
||||
#define HAS_MIRRORROW_NEON
|
||||
#define HAS_MIRRORSPLITUVROW_NEON
|
||||
#define HAS_MIRRORUVROW_NEON
|
||||
@ -652,21 +658,27 @@ extern "C" {
|
||||
|
||||
// Effects:
|
||||
#define HAS_ARGBADDROW_NEON
|
||||
#define HAS_ARGBAFFINEROW_NEON
|
||||
#define HAS_ARGBALPHABLENDROW_NEON
|
||||
#define HAS_ARGBATTENUATEROW_NEON
|
||||
#define HAS_ARGBBLENDROW_NEON
|
||||
#define HAS_BLENDPLANEROW_NEON
|
||||
#define HAS_ARGBCOLORMATRIXROW_NEON
|
||||
#define HAS_ARGBCOLORTABLEROW_NEON
|
||||
#define HAS_ARGBGRAYROW_NEON
|
||||
#define HAS_ARGBLUMACOLORTABLEROW_NEON
|
||||
#define HAS_ARGBMIRRORROW_NEON
|
||||
#define HAS_ARGBMULTIPLYROW_NEON
|
||||
#define HAS_ARGBORROW_NEON
|
||||
#define HAS_ARGBPOLYNOMIALROW_NEON
|
||||
#define HAS_ARGBQUANTIZEROW_NEON
|
||||
#define HAS_ARGBSEPIAROW_NEON
|
||||
#define HAS_ARGBSHADEROW_NEON
|
||||
#define HAS_ARGBSHUFFLEROW_NEON
|
||||
#define HAS_ARGBSUBTRACTROW_NEON
|
||||
#define HAS_ARGBUNATTENUATEROW_NEON
|
||||
#define HAS_RGB24MIRRORROW_NEON
|
||||
#define HAS_RGBCOLORTABLEROW_NEON
|
||||
#define HAS_SOBELROW_NEON
|
||||
#define HAS_SOBELTOPLANEROW_NEON
|
||||
#define HAS_SOBELXROW_NEON
|
||||
@ -2838,6 +2850,10 @@ void MirrorSplitUVRow_C(const uint8_t* src_uv,
|
||||
|
||||
void MirrorRow_16_C(const uint16_t* src, uint16_t* dst, int width);
|
||||
void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width);
|
||||
void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width);
|
||||
void MirrorRow_16_Any_NEON(const uint16_t* src_ptr,
|
||||
uint16_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
|
||||
@ -4014,16 +4030,22 @@ void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
|
||||
|
||||
void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count);
|
||||
void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count);
|
||||
void CopyRow_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int width);
|
||||
|
||||
void ARGBCopyAlphaRow_C(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width);
|
||||
void ARGBCopyAlphaRow_Any_SSE2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBCopyAlphaRow_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBCopyAlphaRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void ARGBExtractAlphaRow_C(const uint8_t* src_argb, uint8_t* dst_a, int width);
|
||||
void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb,
|
||||
@ -4279,6 +4301,18 @@ void AR30ToAB30Row_C(const uint8_t* src_ar30, uint8_t* dst_ab30, int width);
|
||||
void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30,
|
||||
uint8_t* dst_ab30,
|
||||
int width);
|
||||
void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width);
|
||||
void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width);
|
||||
void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width);
|
||||
void AR30ToARGBRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void AR30ToABGRRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void AR30ToAB30Row_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void RGB24ToARGBRow_Any_SSSE3(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
@ -6928,12 +6962,18 @@ void ARGBUnattenuateRow_SSE2(const uint8_t* src_argb,
|
||||
void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width);
|
||||
void ARGBUnattenuateRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width);
|
||||
void ARGBUnattenuateRow_Any_SSE2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBUnattenuateRow_Any_AVX2(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
void ARGBUnattenuateRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
int width);
|
||||
|
||||
void ARGBGrayRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width);
|
||||
void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width);
|
||||
@ -6988,6 +7028,9 @@ void ARGBColorTableRow_C(uint8_t* dst_argb,
|
||||
void ARGBColorTableRow_X86(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width);
|
||||
void ARGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width);
|
||||
|
||||
void RGBColorTableRow_C(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
@ -6995,6 +7038,9 @@ void RGBColorTableRow_C(uint8_t* dst_argb,
|
||||
void RGBColorTableRow_X86(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width);
|
||||
void RGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width);
|
||||
|
||||
void ARGBQuantizeRow_C(uint8_t* dst_argb,
|
||||
int scale,
|
||||
@ -7098,6 +7144,12 @@ void ARGBAffineRow_SSE2(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* src_dudv,
|
||||
int width);
|
||||
LIBYUV_API
|
||||
void ARGBAffineRow_NEON(const uint8_t* src_argb,
|
||||
int src_argb_stride,
|
||||
uint8_t* dst_argb,
|
||||
const float* src_dudv,
|
||||
int width);
|
||||
|
||||
// Used for I420Scale, ARGBScale, and ARGBInterpolate.
|
||||
void InterpolateRow_C(uint8_t* dst_ptr,
|
||||
@ -7351,6 +7403,14 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
int width);
|
||||
void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
int width);
|
||||
void ARGBPolynomialRow_Any_NEON(const uint8_t* src_ptr,
|
||||
uint8_t* dst_ptr,
|
||||
const float* param,
|
||||
int width);
|
||||
|
||||
// Scale and convert to half float.
|
||||
void HalfFloatRow_C(const uint16_t* src, uint16_t* dst, float scale, int width);
|
||||
@ -7442,6 +7502,11 @@ void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb,
|
||||
int width,
|
||||
const uint8_t* luma,
|
||||
uint32_t lumacoeff);
|
||||
void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width,
|
||||
const uint8_t* luma,
|
||||
uint32_t lumacoeff);
|
||||
|
||||
float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width);
|
||||
float ScaleMaxSamples_NEON(const float* src,
|
||||
|
||||
@ -11,6 +11,6 @@
|
||||
#ifndef INCLUDE_LIBYUV_VERSION_H_
|
||||
#define INCLUDE_LIBYUV_VERSION_H_
|
||||
|
||||
#define LIBYUV_VERSION 1975
|
||||
#define LIBYUV_VERSION 1976
|
||||
|
||||
#endif // INCLUDE_LIBYUV_VERSION_H_
|
||||
|
||||
@ -3538,6 +3538,14 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30,
|
||||
src_ar30 = src_ar30 + (ptrdiff_t)(height - 1) * src_stride_ar30;
|
||||
src_stride_ar30 = -src_stride_ar30;
|
||||
}
|
||||
#if defined(HAS_AR30TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_AVX512BW;
|
||||
|
||||
@ -4522,6 +4522,14 @@ int AR30ToARGB(const uint8_t* src_ar30,
|
||||
}
|
||||
void (*AR30ToARGBRow)(const uint8_t* src_ar30, uint8_t* dst_argb, int width) =
|
||||
AR30ToARGBRow_C;
|
||||
#if defined(HAS_AR30TOARGBROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToARGBRow = AR30ToARGBRow_AVX512BW;
|
||||
@ -4563,6 +4571,14 @@ int AR30ToABGR(const uint8_t* src_ar30,
|
||||
height = 1;
|
||||
src_stride_ar30 = dst_stride_abgr = 0;
|
||||
}
|
||||
#if defined(HAS_AR30TOABGRROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
AR30ToABGRRow = AR30ToABGRRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToABGRRow = AR30ToABGRRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOABGRROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToABGRRow = AR30ToABGRRow_AVX512BW;
|
||||
@ -4604,6 +4620,14 @@ int AR30ToAB30(const uint8_t* src_ar30,
|
||||
height = 1;
|
||||
src_stride_ar30 = dst_stride_ab30 = 0;
|
||||
}
|
||||
#if defined(HAS_AR30TOAB30ROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
AR30ToAB30Row = AR30ToAB30Row_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
AR30ToAB30Row = AR30ToAB30Row_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_AR30TOAB30ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
AR30ToAB30Row = AR30ToAB30Row_AVX512BW;
|
||||
|
||||
@ -2926,7 +2926,14 @@ void MirrorPlane_16(const uint16_t* src_y,
|
||||
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
|
||||
src_stride_y = -src_stride_y;
|
||||
}
|
||||
#if defined(HAS_MIRRORUVROW_NEON)
|
||||
#if defined(HAS_MIRRORROW_16_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
MirrorRow_16 = MirrorRow_16_Any_NEON;
|
||||
if (IS_ALIGNED(width, 16)) {
|
||||
MirrorRow_16 = MirrorRow_16_NEON;
|
||||
}
|
||||
}
|
||||
#elif defined(HAS_MIRRORUVROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
MirrorRow_16 = reinterpret_cast<void (*)(const uint16_t*, uint16_t*, int)>(
|
||||
MirrorUVRow_Any_NEON);
|
||||
@ -4693,7 +4700,14 @@ int ARGBUnattenuate(const uint8_t* src_argb,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// TODO(fbarchard): Neon version.
|
||||
#if defined(HAS_ARGBUNATTENUATEROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBUnattenuateRow = ARGBUnattenuateRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 4)) {
|
||||
ARGBUnattenuateRow = ARGBUnattenuateRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < height; ++y) {
|
||||
ARGBUnattenuateRow(src_argb, dst_argb, width);
|
||||
@ -5019,6 +5033,11 @@ int ARGBColorTable(uint8_t* dst_argb,
|
||||
if (TestCpuFlag(kCpuHasX86)) {
|
||||
ARGBColorTableRow = ARGBColorTableRow_X86;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOLORTABLEROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBColorTableRow = ARGBColorTableRow_NEON;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
ARGBColorTableRow(dst, table_argb, width);
|
||||
@ -5055,6 +5074,11 @@ int RGBColorTable(uint8_t* dst_argb,
|
||||
if (TestCpuFlag(kCpuHasX86)) {
|
||||
RGBColorTableRow = RGBColorTableRow_X86;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_RGBCOLORTABLEROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
RGBColorTableRow = RGBColorTableRow_NEON;
|
||||
}
|
||||
#endif
|
||||
for (y = 0; y < height; ++y) {
|
||||
RGBColorTableRow(dst, table_argb, width);
|
||||
@ -6224,6 +6248,14 @@ int ARGBPolynomial(const uint8_t* src_argb,
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_AVX2;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBPOLYNOMIALROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 2)) {
|
||||
ARGBPolynomialRow = ARGBPolynomialRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < height; ++y) {
|
||||
ARGBPolynomialRow(src_argb, dst_argb, poly, width);
|
||||
@ -6383,6 +6415,11 @@ int ARGBLumaColorTable(const uint8_t* src_argb,
|
||||
ARGBLumaColorTableRow = ARGBLumaColorTableRow_SSSE3;
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBLUMACOLORTABLEROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 4)) {
|
||||
ARGBLumaColorTableRow = ARGBLumaColorTableRow_NEON;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (y = 0; y < height; ++y) {
|
||||
ARGBLumaColorTableRow(src_argb, dst_argb, width, luma, 0x00264b0f);
|
||||
@ -6420,6 +6457,14 @@ int ARGBCopyAlpha(const uint8_t* src_argb,
|
||||
height = 1;
|
||||
src_stride_argb = dst_stride_argb = 0;
|
||||
}
|
||||
#if defined(HAS_ARGBCOPYALPHAROW_NEON)
|
||||
if (TestCpuFlag(kCpuHasNEON)) {
|
||||
ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_NEON;
|
||||
if (IS_ALIGNED(width, 8)) {
|
||||
ARGBCopyAlphaRow = ARGBCopyAlphaRow_NEON;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBCOPYALPHAROW_SSE2)
|
||||
if (TestCpuFlag(kCpuHasSSE2)) {
|
||||
ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_SSE2;
|
||||
|
||||
@ -1105,6 +1105,15 @@ ANY11(ABGRToAR30Row_Any_NEON, ABGRToAR30Row_NEON, 0, 4, 4, 7)
|
||||
#if defined(HAS_ARGBTOAR30ROW_NEON)
|
||||
ANY11(ARGBToAR30Row_Any_NEON, ARGBToAR30Row_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_AR30TOARGBROW_NEON)
|
||||
ANY11(AR30ToARGBRow_Any_NEON, AR30ToARGBRow_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_AR30TOABGRROW_NEON)
|
||||
ANY11(AR30ToABGRRow_Any_NEON, AR30ToABGRRow_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_AR30TOAB30ROW_NEON)
|
||||
ANY11(AR30ToAB30Row_Any_NEON, AR30ToAB30Row_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#if defined(HAS_ARGBTOAR30ROW_SSSE3)
|
||||
ANY11(ARGBToAR30Row_Any_SSSE3, ARGBToAR30Row_SSSE3, 0, 4, 4, 3)
|
||||
#endif
|
||||
@ -1330,6 +1339,9 @@ ANY11(ARGBUnattenuateRow_Any_AVX2, ARGBUnattenuateRow_AVX2, 0, 4, 4, 7)
|
||||
#ifdef HAS_ARGBATTENUATEROW_NEON
|
||||
ANY11(ARGBAttenuateRow_Any_NEON, ARGBAttenuateRow_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_ARGBUNATTENUATEROW_NEON
|
||||
ANY11(ARGBUnattenuateRow_Any_NEON, ARGBUnattenuateRow_NEON, 0, 4, 4, 3)
|
||||
#endif
|
||||
#ifdef HAS_ARGBATTENUATEROW_LSX
|
||||
ANY11(ARGBAttenuateRow_Any_LSX, ARGBAttenuateRow_LSX, 0, 4, 4, 3)
|
||||
#endif
|
||||
@ -1419,6 +1431,9 @@ ANY11B(ARGBCopyAlphaRow_Any_AVX2, ARGBCopyAlphaRow_AVX2, 0, 4, 4, 15)
|
||||
#ifdef HAS_ARGBCOPYALPHAROW_SSE2
|
||||
ANY11B(ARGBCopyAlphaRow_Any_SSE2, ARGBCopyAlphaRow_SSE2, 0, 4, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_ARGBCOPYALPHAROW_NEON
|
||||
ANY11B(ARGBCopyAlphaRow_Any_NEON, ARGBCopyAlphaRow_NEON, 0, 4, 4, 7)
|
||||
#endif
|
||||
#ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2
|
||||
ANY11B(ARGBCopyYToAlphaRow_Any_AVX2, ARGBCopyYToAlphaRow_AVX2, 0, 1, 4, 15)
|
||||
#endif
|
||||
@ -1578,6 +1593,14 @@ ANY11P(ARGBOrRow_Any_LASX, ARGBOrRow_LASX, const uint32_t, 4, 4, 7)
|
||||
#ifdef HAS_ARGBORROW_WASMSIMD
|
||||
ANY11P(ARGBOrRow_Any_WASMSIMD, ARGBOrRow_WASMSIMD, const uint32_t, 4, 4, 3)
|
||||
#endif
|
||||
#ifdef HAS_ARGBPOLYNOMIALROW_NEON
|
||||
ANY11P(ARGBPolynomialRow_Any_NEON,
|
||||
ARGBPolynomialRow_NEON,
|
||||
const float*,
|
||||
4,
|
||||
4,
|
||||
1)
|
||||
#endif
|
||||
#undef ANY11P
|
||||
#undef ANY11P
|
||||
|
||||
@ -1647,6 +1670,10 @@ ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
|
||||
ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_COPYROW_16_NEON
|
||||
ANY11T(CopyRow_16_Any_NEON, CopyRow_16_NEON, 2, 2, uint16_t, uint16_t, 15)
|
||||
#endif
|
||||
|
||||
#undef ANY11T
|
||||
|
||||
// Any 1 to 1 with parameter and shorts. BPP measures in shorts.
|
||||
@ -2196,6 +2223,31 @@ ANY11M(RGB24MirrorRow_Any_WASMSIMD, RGB24MirrorRow_WASMSIMD, 3, 15)
|
||||
#endif
|
||||
#undef ANY11M
|
||||
|
||||
// Any 1 to 1 mirror with type.
|
||||
#define ANY11MT(NAMEANY, ANY_SIMD, T, BPP, MASK) \
|
||||
void NAMEANY(const T* src_ptr, T* dst_ptr, int width) { \
|
||||
SIMD_ALIGNED(T vin[64]); \
|
||||
static_assert((MASK + 1) * BPP <= sizeof(vin), "vin buffer too small"); \
|
||||
SIMD_ALIGNED(T vout[64]); \
|
||||
static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \
|
||||
MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
|
||||
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
|
||||
int r = width & MASK; \
|
||||
int n = width & ~MASK; \
|
||||
if (n > 0) { \
|
||||
ANY_SIMD(src_ptr + r, dst_ptr, n); \
|
||||
} \
|
||||
ptrdiff_t np = n; \
|
||||
memcpy(vin, src_ptr, r * BPP); \
|
||||
ANY_SIMD(vin, vout, MASK + 1); \
|
||||
memcpy(dst_ptr + np, vout + (MASK + 1 - r), r * BPP); \
|
||||
}
|
||||
|
||||
#ifdef HAS_MIRRORROW_16_NEON
|
||||
ANY11MT(MirrorRow_16_Any_NEON, MirrorRow_16_NEON, uint16_t, 2, 15)
|
||||
#endif
|
||||
#undef ANY11MT
|
||||
|
||||
// Any 1 plane. (memset)
|
||||
#define ANY1(NAMEANY, ANY_SIMD, T, BPP, MASK) \
|
||||
void NAMEANY(uint8_t* dst_ptr, T v32, int width) { \
|
||||
|
||||
@ -3992,6 +3992,433 @@ void Convert8To16Row_NEON(const uint8_t* src_y,
|
||||
: "cc", "memory", "q0", "q1", "q2");
|
||||
}
|
||||
|
||||
void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count) {
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"vld1.16 {d0, d1, d2, d3}, [%0]! \n" // load 16
|
||||
"subs %2, %2, #16 \n" // 16 processed per loop
|
||||
"vst1.16 {d0, d1, d2, d3}, [%1]! \n" // store 16
|
||||
"bgt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(count) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1");
|
||||
}
|
||||
|
||||
void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width) {
|
||||
asm volatile(
|
||||
// Start at end of source row.
|
||||
"add %0, %0, %2, lsl #1 \n"
|
||||
"sub %0, %0, #32 \n" // 32 bytes per loop
|
||||
"1: \n"
|
||||
"vld1.16 {q1, q2}, [%0], %3 \n" // src -= 32
|
||||
"subs %2, %2, #16 \n" // 16 pixels per loop.
|
||||
"vrev64.16 q0, q2 \n"
|
||||
"vrev64.16 q1, q1 \n"
|
||||
"vswp d0, d1 \n"
|
||||
"vswp d2, d3 \n"
|
||||
"vst1.16 {q0, q1}, [%1]! \n" // dst += 32
|
||||
"bgt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
: "r"(-32) // %3
|
||||
: "memory", "cc", "q0", "q1", "q2");
|
||||
}
|
||||
|
||||
void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width) {
|
||||
asm volatile(
|
||||
"vmov.i32 q8, #0x0000ff00 \n"
|
||||
"vmov.i32 q9, #0x000fffff \n"
|
||||
"vshl.i32 q9, q9, #8 \n" // 0x0fffff00
|
||||
"1: \n"
|
||||
"vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels
|
||||
"subs %2, %2, #8 \n" // 8 pixels per loop
|
||||
"vsri.32 q0, q0, #2 \n"
|
||||
"vsri.32 q1, q1, #2 \n"
|
||||
"vshr.u32 q2, q0, #2 \n"
|
||||
"vshr.u32 q3, q1, #2 \n"
|
||||
"vshr.u32 q4, q0, #4 \n"
|
||||
"vshr.u32 q5, q1, #4 \n"
|
||||
"vbit q0, q4, q9 \n"
|
||||
"vbit q1, q5, q9 \n"
|
||||
"vbit q0, q2, q8 \n"
|
||||
"vbit q1, q3, q8 \n"
|
||||
"vst1.8 {q0, q1}, [%1]! \n" // store 8 ARGB pixels
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9");
|
||||
}
|
||||
|
||||
void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) {
|
||||
asm volatile(
|
||||
"vmov.i64 q8, #0x00ffff0000ffff00 \n"
|
||||
"1: \n"
|
||||
"vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels
|
||||
"subs %2, %2, #8 \n" // 8 pixels per loop
|
||||
"vsri.32 q0, q0, #2 \n"
|
||||
"vsri.32 q1, q1, #2 \n"
|
||||
"vshr.u32 q2, q0, #2 \n"
|
||||
"vshr.u32 q3, q1, #2 \n"
|
||||
"vsli.32 q2, q0, #16 \n"
|
||||
"vsli.32 q3, q1, #16 \n"
|
||||
"vsri.32 q0, q0, #4 \n"
|
||||
"vsri.32 q1, q1, #4 \n"
|
||||
"vsri.32 q0, q0, #16 \n"
|
||||
"vsri.32 q1, q1, #16 \n"
|
||||
"vbit q0, q2, q8 \n"
|
||||
"vbit q1, q3, q8 \n"
|
||||
"vst1.8 {q0, q1}, [%1]! \n" // store 8 ABGR pixels
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_abgr), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q8");
|
||||
}
|
||||
|
||||
void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) {
|
||||
asm volatile(
|
||||
"vmov.i8 q9, #0xff \n"
|
||||
"vshr.u32 q9, q9, #22 \n" // 0x000003ff
|
||||
"vshl.i32 q8, q9, #20 \n" // 0x3ff00000
|
||||
"1: \n"
|
||||
"vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels
|
||||
"subs %2, %2, #8 \n" // 8 pixels per loop
|
||||
"vshl.i32 q2, q0, #20 \n"
|
||||
"vshl.i32 q3, q1, #20 \n"
|
||||
"vshr.u32 q4, q0, #20 \n"
|
||||
"vshr.u32 q5, q1, #20 \n"
|
||||
"vbit q0, q2, q8 \n"
|
||||
"vbit q1, q3, q8 \n"
|
||||
"vbit q0, q4, q9 \n"
|
||||
"vbit q1, q5, q9 \n"
|
||||
"vst1.8 {q0, q1}, [%1]! \n" // store 8 AB30 pixels
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_ab30), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9");
|
||||
}
|
||||
|
||||
void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
|
||||
asm volatile(
|
||||
"vmov.i32 q4, #0xff000000 \n"
|
||||
"1: \n"
|
||||
"vld1.8 {q0, q1}, [%0]! \n" // load 8 src pixels
|
||||
"vld1.8 {q2, q3}, [%1] \n" // load 8 dst pixels
|
||||
"subs %2, %2, #8 \n" // 8 pixels per loop
|
||||
"vbit q2, q0, q4 \n"
|
||||
"vbit q3, q1, q4 \n"
|
||||
"vst1.8 {q2, q3}, [%1]! \n" // store 8 dst pixels
|
||||
"bgt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4");
|
||||
}
|
||||
|
||||
void ARGBUnattenuateRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width) {
|
||||
uint32_t tmp, addr;
|
||||
asm volatile(
|
||||
"vmov.i32 q8, #0xff000000 \n"
|
||||
"1: \n"
|
||||
"ldrb %3, [%0, #3] \n"
|
||||
"add %4, %5, %3, lsl #2 \n"
|
||||
"vld1.16 {d6[0]}, [%4] \n"
|
||||
"ldrb %3, [%0, #7] \n"
|
||||
"add %4, %5, %3, lsl #2 \n"
|
||||
"vld1.16 {d6[1]}, [%4] \n"
|
||||
"ldrb %3, [%0, #11] \n"
|
||||
"add %4, %5, %3, lsl #2 \n"
|
||||
"vld1.16 {d6[2]}, [%4] \n"
|
||||
"ldrb %3, [%0, #15] \n"
|
||||
"add %4, %5, %3, lsl #2 \n"
|
||||
"vld1.16 {d6[3]}, [%4] \n"
|
||||
"vld1.8 {q0}, [%0]! \n"
|
||||
"subs %2, %2, #4 \n"
|
||||
"vmov q1, q0 \n"
|
||||
"vmov q2, q0 \n"
|
||||
"vzip.8 q1, q2 \n"
|
||||
"vmull.u16 q4, d2, d6[0] \n"
|
||||
"vmull.u16 q5, d3, d6[1] \n"
|
||||
"vmull.u16 q6, d4, d6[2] \n"
|
||||
"vmull.u16 q7, d5, d6[3] \n"
|
||||
"vshrn.i32 d8, q4, #16 \n"
|
||||
"vshrn.i32 d9, q5, #16 \n"
|
||||
"vshrn.i32 d12, q6, #16 \n"
|
||||
"vshrn.i32 d13, q7, #16 \n"
|
||||
"vqmovn.u16 d2, q4 \n"
|
||||
"vqmovn.u16 d3, q6 \n"
|
||||
"vbit q1, q0, q8 \n"
|
||||
"vst1.8 {q1}, [%1]! \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(tmp), // %3
|
||||
"=&r"(addr) // %4
|
||||
: "r"(fixed_invtbl8) // %5
|
||||
: "memory", "cc", "q0", "q1", "q2", "d6", "q4", "q5", "q6", "q7", "q8");
|
||||
}
|
||||
|
||||
void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
int width) {
|
||||
asm volatile(
|
||||
"vld1.32 {q8, q9}, [%3]! \n" // c0, c1
|
||||
"vld1.32 {q10, q11}, [%3] \n" // c2, c3
|
||||
"1: \n"
|
||||
"vld1.8 {d0}, [%0]! \n" // load 2 pixels
|
||||
"subs %2, %2, #2 \n" // 2 pixels per loop
|
||||
"vmovl.u8 q0, d0 \n"
|
||||
"vmovl.u16 q1, d0 \n"
|
||||
"vmovl.u16 q2, d1 \n"
|
||||
"vcvt.f32.u32 q1, q1 \n" // x0
|
||||
"vcvt.f32.u32 q2, q2 \n" // x1
|
||||
"vmov q3, q8 \n" // c0
|
||||
"vmov q4, q8 \n" // c0
|
||||
"vmla.f32 q3, q9, q1 \n" // c0 + c1 * x0
|
||||
"vmla.f32 q4, q9, q2 \n" // c0 + c1 * x1
|
||||
"vmul.f32 q5, q1, q1 \n" // x0 * x0
|
||||
"vmul.f32 q6, q2, q2 \n" // x1 * x1
|
||||
"vmla.f32 q3, q10, q5 \n" // + c2 * x0^2
|
||||
"vmla.f32 q4, q10, q6 \n" // + c2 * x1^2
|
||||
"vmul.f32 q5, q5, q1 \n" // x0^3
|
||||
"vmul.f32 q6, q6, q2 \n" // x1^3
|
||||
"vmla.f32 q3, q11, q5 \n" // + c3 * x0^3
|
||||
"vmla.f32 q4, q11, q6 \n" // + c3 * x1^3
|
||||
"vcvt.s32.f32 q3, q3 \n"
|
||||
"vcvt.s32.f32 q4, q4 \n"
|
||||
"vqmovn.s32 d6, q3 \n"
|
||||
"vqmovn.s32 d7, q4 \n"
|
||||
"vqmovun.s16 d6, q3 \n"
|
||||
"vst1.8 {d6}, [%1]! \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"+r"(poly) // %3
|
||||
:
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q8", "q9",
|
||||
"q10", "q11");
|
||||
}
|
||||
|
||||
void ARGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width) {
|
||||
const uint8_t* table_g = table_argb + 1;
|
||||
const uint8_t* table_r = table_argb + 2;
|
||||
const uint8_t* table_a = table_argb + 3;
|
||||
uint32_t b, g, r, a;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldrb %2, [%0] \n"
|
||||
"ldrb %3, [%0, #1] \n"
|
||||
"ldrb %4, [%0, #2] \n"
|
||||
"ldrb %5, [%0, #3] \n"
|
||||
"ldrb %2, [%6, %2, lsl #2] \n"
|
||||
"ldrb %3, [%7, %3, lsl #2] \n"
|
||||
"ldrb %4, [%8, %4, lsl #2] \n"
|
||||
"ldrb %5, [%9, %5, lsl #2] \n"
|
||||
"subs %1, %1, #1 \n"
|
||||
"strb %2, [%0] \n"
|
||||
"strb %3, [%0, #1] \n"
|
||||
"strb %4, [%0, #2] \n"
|
||||
"strb %5, [%0, #3] \n"
|
||||
"add %0, %0, #4 \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(b), // %2
|
||||
"=&r"(g), // %3
|
||||
"=&r"(r), // %4
|
||||
"=&r"(a) // %5
|
||||
: "r"(table_argb), // %6
|
||||
"r"(table_g), // %7
|
||||
"r"(table_r), // %8
|
||||
"r"(table_a) // %9
|
||||
: "memory", "cc");
|
||||
}
|
||||
|
||||
void RGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width) {
|
||||
const uint8_t* table_g = table_argb + 1;
|
||||
const uint8_t* table_r = table_argb + 2;
|
||||
uint32_t b, g, r;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldrb %2, [%0] \n"
|
||||
"ldrb %3, [%0, #1] \n"
|
||||
"ldrb %4, [%0, #2] \n"
|
||||
"ldrb %2, [%5, %2, lsl #2] \n"
|
||||
"ldrb %3, [%6, %3, lsl #2] \n"
|
||||
"ldrb %4, [%7, %4, lsl #2] \n"
|
||||
"subs %1, %1, #1 \n"
|
||||
"strb %2, [%0] \n"
|
||||
"strb %3, [%0, #1] \n"
|
||||
"strb %4, [%0, #2] \n"
|
||||
"add %0, %0, #4 \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(b), // %2
|
||||
"=&r"(g), // %3
|
||||
"=&r"(r) // %4
|
||||
: "r"(table_argb), // %5
|
||||
"r"(table_g), // %6
|
||||
"r"(table_r) // %7
|
||||
: "memory", "cc");
|
||||
}
|
||||
|
||||
void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width,
|
||||
const uint8_t* luma,
|
||||
uint32_t lumacoeff) {
|
||||
uint32_t idx, addr;
|
||||
asm volatile(
|
||||
"vdup.32 q4, %6 \n" // lumacoeff
|
||||
"1: \n"
|
||||
"vld1.8 {q0}, [%0]! \n" // load 4 ARGB pixels
|
||||
"subs %2, %2, #4 \n" // 4 pixels per loop
|
||||
"vmull.u8 q1, d0, d8 \n"
|
||||
"vmull.u8 q2, d1, d9 \n"
|
||||
"vpadd.i16 d2, d2, d3 \n"
|
||||
"vpadd.i16 d3, d4, d5 \n"
|
||||
"vpadd.i16 d2, d2, d3 \n"
|
||||
"vbic.i16 d2, #0x8000 \n" // clear bit 15
|
||||
"vshrn.i16 d2, q1, #8 \n" // L_k & 0x7f
|
||||
"vmov d3, d2 \n"
|
||||
"vzip.8 d2, d3 \n"
|
||||
"vmov d3, d2 \n"
|
||||
"vzip.8 d2, d3 \n"
|
||||
"vmov q2, q0 \n"
|
||||
"vzip.8 q2, q1 \n" // q2: p0,1; q1: p2,3
|
||||
"vmov.u16 %3, d4[0] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[0]}, [%4] \n"
|
||||
"vmov.u16 %3, d4[1] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[1]}, [%4] \n"
|
||||
"vmov.u16 %3, d4[2] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[2]}, [%4] \n"
|
||||
"vmov.u16 %3, d5[0] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[4]}, [%4] \n"
|
||||
"vmov.u16 %3, d5[1] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[5]}, [%4] \n"
|
||||
"vmov.u16 %3, d5[2] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d0[6]}, [%4] \n"
|
||||
"vmov.u16 %3, d2[0] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[0]}, [%4] \n"
|
||||
"vmov.u16 %3, d2[1] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[1]}, [%4] \n"
|
||||
"vmov.u16 %3, d2[2] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[2]}, [%4] \n"
|
||||
"vmov.u16 %3, d3[0] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[4]}, [%4] \n"
|
||||
"vmov.u16 %3, d3[1] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[5]}, [%4] \n"
|
||||
"vmov.u16 %3, d3[2] \n"
|
||||
"add %4, %5, %3 \n"
|
||||
"vld1.8 {d1[6]}, [%4] \n"
|
||||
"vst1.8 {q0}, [%1]! \n"
|
||||
"bgt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(idx), // %3
|
||||
"=&r"(addr) // %4
|
||||
: "r"(luma), // %5
|
||||
"r"(lumacoeff) // %6
|
||||
: "memory", "cc", "q0", "q1", "q2", "q4");
|
||||
}
|
||||
|
||||
void ARGBAffineRow_NEON(const uint8_t* src_argb,
|
||||
int src_argb_stride,
|
||||
uint8_t* dst_argb,
|
||||
const float* uv_dudv,
|
||||
int width) {
|
||||
uint32_t off0, off1;
|
||||
asm volatile(
|
||||
"vld1.32 {d4, d5}, [%5] \n" // d4=[u,v], d5=[du,dv]
|
||||
"vmov d12, d5 \n" // d12=[du,dv]
|
||||
"vdup.32 d10, %4 \n" // [stride, stride]
|
||||
"vmov.i32 d11, #4 \n" // [4, 4]
|
||||
"vtrn.32 d11, d10 \n" // q5=[4, stride, 4,
|
||||
// stride]
|
||||
"subs %1, %1, #4 \n"
|
||||
"blt 2f \n"
|
||||
"vmov d13, d12 \n" // q6=[du, dv, du, dv]
|
||||
"vadd.f32 q4, q6, q6 \n" // 2 * [du, dv, du, dv]
|
||||
"vadd.f32 d5, d4, d12 \n" // [u+du, v+dv]
|
||||
"vadd.f32 q3, q2, q4 \n" // [u+2du, v+2dv, u+3du,
|
||||
// v+3dv]
|
||||
"vadd.f32 q4, q4, q4 \n" // 4 * [du, dv, du, dv]
|
||||
"1: \n"
|
||||
"vcvt.s32.f32 q0, q2 \n"
|
||||
"vcvt.s32.f32 q1, q3 \n"
|
||||
"vadd.f32 q2, q2, q4 \n"
|
||||
"vadd.f32 q3, q3, q4 \n"
|
||||
"vmul.i32 q0, q0, q5 \n"
|
||||
"vmul.i32 q1, q1, q5 \n"
|
||||
"vpadd.i32 d14, d0, d1 \n"
|
||||
"vpadd.i32 d15, d2, d3 \n"
|
||||
"vmov %2, %3, d14 \n"
|
||||
"ldr %2, [%6, %2] \n"
|
||||
"ldr %3, [%6, %3] \n"
|
||||
"str %2, [%0], #4 \n"
|
||||
"str %3, [%0], #4 \n"
|
||||
"vmov %2, %3, d15 \n"
|
||||
"ldr %2, [%6, %2] \n"
|
||||
"ldr %3, [%6, %3] \n"
|
||||
"str %2, [%0], #4 \n"
|
||||
"str %3, [%0], #4 \n"
|
||||
"subs %1, %1, #4 \n"
|
||||
"bge 1b \n"
|
||||
"2: \n"
|
||||
"adds %1, %1, #4 \n"
|
||||
"beq 4f \n"
|
||||
"3: \n"
|
||||
"vcvt.s32.f32 d0, d4 \n"
|
||||
"vadd.f32 d4, d4, d12 \n"
|
||||
"vmul.i32 d0, d0, d10 \n"
|
||||
"vpadd.i32 d0, d0, d0 \n"
|
||||
"vmov.32 %2, d0[0] \n"
|
||||
"ldr %2, [%6, %2] \n"
|
||||
"str %2, [%0], #4 \n"
|
||||
"subs %1, %1, #1 \n"
|
||||
"bgt 3b \n"
|
||||
"4: \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(off0), // %2
|
||||
"=&r"(off1) // %3
|
||||
: "r"(src_argb_stride), // %4
|
||||
"r"(uv_dudv), // %5
|
||||
"r"(src_argb) // %6
|
||||
: "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7");
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__)..
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -5642,6 +5642,471 @@ void Convert8To16Row_NEON(const uint8_t* src_y,
|
||||
: "cc", "memory", "v0", "v1", "v2");
|
||||
}
|
||||
|
||||
void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count) {
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n"
|
||||
"subs %w2, %w2, #16 \n" // 16 processed per loop
|
||||
"stp q0, q1, [%1], #32 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(count) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1");
|
||||
}
|
||||
|
||||
void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width) {
|
||||
asm volatile(
|
||||
// Start at end of source row.
|
||||
"ld1 {v4.16b}, [%3] \n" // shuffler
|
||||
"add %0, %0, %w2, sxtw #1 \n"
|
||||
"sub %0, %0, #32 \n"
|
||||
"1: \n"
|
||||
"ldr q1, [%0, 16] \n"
|
||||
"ldr q0, [%0], -32 \n" // src -= 32
|
||||
"subs %w2, %w2, #16 \n" // 16 pixels per loop.
|
||||
"tbl v2.16b, {v1.16b}, v4.16b \n"
|
||||
"tbl v3.16b, {v0.16b}, v4.16b \n"
|
||||
"stp q2, q3, [%1], #32 \n" // dst += 32
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
: "r"(&kShuffleMirrorUV) // %3
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4");
|
||||
}
|
||||
|
||||
void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width) {
|
||||
asm volatile(
|
||||
"movi v16.4s, #0xff, lsl #8 \n" // 0x0000ff00
|
||||
"movi v17.4s, #0x0f, msl #16 \n" // 0x000fffff
|
||||
"shl v17.4s, v17.4s, #8 \n" // 0x0fffff00
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels
|
||||
"subs %w2, %w2, #8 \n" // 8 pixels per loop
|
||||
"sri v0.4s, v0.4s, #2 \n"
|
||||
"sri v1.4s, v1.4s, #2 \n"
|
||||
"ushr v2.4s, v0.4s, #2 \n"
|
||||
"ushr v3.4s, v1.4s, #2 \n"
|
||||
"ushr v4.4s, v0.4s, #4 \n"
|
||||
"ushr v5.4s, v1.4s, #4 \n"
|
||||
"bit v0.16b, v4.16b, v17.16b \n"
|
||||
"bit v1.16b, v5.16b, v17.16b \n"
|
||||
"bit v0.16b, v2.16b, v16.16b \n"
|
||||
"bit v1.16b, v3.16b, v16.16b \n"
|
||||
"stp q0, q1, [%1], #32 \n" // store 8 ARGB pixels
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17");
|
||||
}
|
||||
|
||||
void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) {
|
||||
asm volatile(
|
||||
"movi v16.2d, #0x00ffff0000ffff00 \n"
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels
|
||||
"subs %w2, %w2, #8 \n" // 8 pixels per loop
|
||||
"sri v0.4s, v0.4s, #2 \n"
|
||||
"sri v1.4s, v1.4s, #2 \n"
|
||||
"ushr v2.4s, v0.4s, #2 \n"
|
||||
"ushr v3.4s, v1.4s, #2 \n"
|
||||
"sli v2.4s, v0.4s, #16 \n"
|
||||
"sli v3.4s, v1.4s, #16 \n"
|
||||
"sri v0.4s, v0.4s, #4 \n"
|
||||
"sri v1.4s, v1.4s, #4 \n"
|
||||
"sri v0.4s, v0.4s, #16 \n"
|
||||
"sri v1.4s, v1.4s, #16 \n"
|
||||
"bit v0.16b, v2.16b, v16.16b \n"
|
||||
"bit v1.16b, v3.16b, v16.16b \n"
|
||||
"stp q0, q1, [%1], #32 \n" // store 8 ABGR pixels
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_abgr), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v16");
|
||||
}
|
||||
|
||||
void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) {
|
||||
asm volatile(
|
||||
"movi v17.16b, #0xff \n"
|
||||
"ushr v17.4s, v17.4s, #22 \n" // 0x000003ff
|
||||
"shl v16.4s, v17.4s, #20 \n" // 0x3ff00000
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels
|
||||
"subs %w2, %w2, #8 \n" // 8 pixels per loop
|
||||
"shl v2.4s, v0.4s, #20 \n"
|
||||
"shl v3.4s, v1.4s, #20 \n"
|
||||
"ushr v4.4s, v0.4s, #20 \n"
|
||||
"ushr v5.4s, v1.4s, #20 \n"
|
||||
"bit v0.16b, v2.16b, v16.16b \n"
|
||||
"bit v1.16b, v3.16b, v16.16b \n"
|
||||
"bit v0.16b, v4.16b, v17.16b \n"
|
||||
"bit v1.16b, v5.16b, v17.16b \n"
|
||||
"stp q0, q1, [%1], #32 \n" // store 8 AB30 pixels
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_ar30), // %0
|
||||
"+r"(dst_ab30), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17");
|
||||
}
|
||||
|
||||
void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
|
||||
asm volatile(
|
||||
"movi v4.4s, #0xff, lsl #24 \n" // 0xff000000
|
||||
"1: \n"
|
||||
"ldp q0, q1, [%0], #32 \n" // load 8 src pixels
|
||||
"ldp q2, q3, [%1] \n" // load 8 dst pixels
|
||||
"subs %w2, %w2, #8 \n" // 8 pixels per loop
|
||||
"bit v2.16b, v0.16b, v4.16b \n"
|
||||
"bit v3.16b, v1.16b, v4.16b \n"
|
||||
"stp q2, q3, [%1], #32 \n" // store 8 dst pixels
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src), // %0
|
||||
"+r"(dst), // %1
|
||||
"+r"(width) // %2
|
||||
:
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4");
|
||||
}
|
||||
|
||||
void ARGBUnattenuateRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width) {
|
||||
uint64_t tmp;
|
||||
asm volatile(
|
||||
"movi v16.4s, #0xff, lsl #24 \n" // 0xff000000
|
||||
"1: \n"
|
||||
"ldrb %w3, [%0, #3] \n"
|
||||
"ldr s4, [%4, %w3, uxtw #2] \n"
|
||||
"ldrb %w3, [%0, #7] \n"
|
||||
"ldr s5, [%4, %w3, uxtw #2] \n"
|
||||
"ldrb %w3, [%0, #11] \n"
|
||||
"ldr s6, [%4, %w3, uxtw #2] \n"
|
||||
"ldrb %w3, [%0, #15] \n"
|
||||
"ldr s7, [%4, %w3, uxtw #2] \n"
|
||||
"ldr q0, [%0], #16 \n"
|
||||
"subs %w2, %w2, #4 \n"
|
||||
"zip1 v1.16b, v0.16b, v0.16b \n"
|
||||
"zip2 v2.16b, v0.16b, v0.16b \n"
|
||||
"umull v3.4s, v1.4h, v4.h[0] \n"
|
||||
"umull2 v5.4s, v1.8h, v5.h[0] \n"
|
||||
"umull v6.4s, v2.4h, v6.h[0] \n"
|
||||
"umull2 v7.4s, v2.8h, v7.h[0] \n"
|
||||
"uzp2 v3.8h, v3.8h, v5.8h \n"
|
||||
"uzp2 v6.8h, v6.8h, v7.8h \n"
|
||||
"uqxtn v1.8b, v3.8h \n"
|
||||
"uqxtn2 v1.16b, v6.8h \n"
|
||||
"bit v1.16b, v0.16b, v16.16b \n"
|
||||
"str q1, [%1], #16 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(tmp) // %3
|
||||
: "r"(fixed_invtbl8) // %4
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
|
||||
}
|
||||
|
||||
void ARGBPolynomialRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
const float* poly,
|
||||
int width) {
|
||||
asm volatile(
|
||||
"ldp q16, q17, [%3] \n" // c0, c1
|
||||
"ldp q18, q19, [%3, #32] \n" // c2, c3
|
||||
"subs %w2, %w2, #4 \n"
|
||||
"b.lt 2f \n"
|
||||
"1: \n"
|
||||
"ldr q0, [%0], #16 \n" // load 4 pixels
|
||||
"subs %w2, %w2, #4 \n"
|
||||
"uxtl v1.8h, v0.8b \n"
|
||||
"uxtl2 v3.8h, v0.16b \n"
|
||||
"uxtl v0.4s, v1.4h \n"
|
||||
"uxtl2 v1.4s, v1.8h \n"
|
||||
"uxtl v2.4s, v3.4h \n"
|
||||
"uxtl2 v3.4s, v3.8h \n"
|
||||
"ucvtf v0.4s, v0.4s \n" // x0
|
||||
"ucvtf v1.4s, v1.4s \n" // x1
|
||||
"ucvtf v2.4s, v2.4s \n" // x2
|
||||
"ucvtf v3.4s, v3.4s \n" // x3
|
||||
"fmul v20.4s, v0.4s, v0.4s \n" // x0^2
|
||||
"fmul v21.4s, v1.4s, v1.4s \n" // x1^2
|
||||
"fmul v22.4s, v2.4s, v2.4s \n" // x2^2
|
||||
"fmul v23.4s, v3.4s, v3.4s \n" // x3^2
|
||||
"fmul v4.4s, v17.4s, v0.4s \n" // c1 * x0
|
||||
"fmul v5.4s, v17.4s, v1.4s \n" // c1 * x1
|
||||
"fmul v6.4s, v17.4s, v2.4s \n" // c1 * x2
|
||||
"fmul v7.4s, v17.4s, v3.4s \n" // c1 * x3
|
||||
"fmul v0.4s, v20.4s, v0.4s \n" // x0^3
|
||||
"fmul v1.4s, v21.4s, v1.4s \n" // x1^3
|
||||
"fmul v2.4s, v22.4s, v2.4s \n" // x2^3
|
||||
"fmul v3.4s, v23.4s, v3.4s \n" // x3^3
|
||||
"fadd v4.4s, v16.4s, v4.4s \n" // c0 + c1 * x0
|
||||
"fadd v5.4s, v16.4s, v5.4s \n" // c0 + c1 * x1
|
||||
"fadd v6.4s, v16.4s, v6.4s \n" // c0 + c1 * x2
|
||||
"fadd v7.4s, v16.4s, v7.4s \n" // c0 + c1 * x3
|
||||
"fmla v4.4s, v18.4s, v20.4s \n" // + c2 * x0^2
|
||||
"fmla v5.4s, v18.4s, v21.4s \n" // + c2 * x1^2
|
||||
"fmla v6.4s, v18.4s, v22.4s \n" // + c2 * x2^2
|
||||
"fmla v7.4s, v18.4s, v23.4s \n" // + c2 * x3^2
|
||||
"fmla v4.4s, v19.4s, v0.4s \n" // + c3 * x0^3
|
||||
"fmla v5.4s, v19.4s, v1.4s \n" // + c3 * x1^3
|
||||
"fmla v6.4s, v19.4s, v2.4s \n" // + c3 * x2^3
|
||||
"fmla v7.4s, v19.4s, v3.4s \n" // + c3 * x3^3
|
||||
"fcvtzs v4.4s, v4.4s \n"
|
||||
"fcvtzs v5.4s, v5.4s \n"
|
||||
"fcvtzs v6.4s, v6.4s \n"
|
||||
"fcvtzs v7.4s, v7.4s \n"
|
||||
"sqxtn v4.4h, v4.4s \n"
|
||||
"sqxtn2 v4.8h, v5.4s \n"
|
||||
"sqxtn v6.4h, v6.4s \n"
|
||||
"sqxtn2 v6.8h, v7.4s \n"
|
||||
"sqxtun v4.8b, v4.8h \n"
|
||||
"sqxtun2 v4.16b, v6.8h \n"
|
||||
"str q4, [%1], #16 \n"
|
||||
"b.ge 1b \n"
|
||||
"2: \n"
|
||||
"adds %w2, %w2, #4 \n"
|
||||
"b.eq 3f \n"
|
||||
"ldr d0, [%0], #8 \n" // load 2 pixels
|
||||
"uxtl v0.8h, v0.8b \n"
|
||||
"uxtl v1.4s, v0.4h \n"
|
||||
"uxtl2 v2.4s, v0.8h \n"
|
||||
"ucvtf v1.4s, v1.4s \n" // x0
|
||||
"ucvtf v2.4s, v2.4s \n" // x1
|
||||
"fmul v5.4s, v1.4s, v1.4s \n" // x0^2
|
||||
"fmul v6.4s, v2.4s, v2.4s \n" // x1^2
|
||||
"fmul v3.4s, v17.4s, v1.4s \n" // c1 * x0
|
||||
"fmul v4.4s, v17.4s, v2.4s \n" // c1 * x1
|
||||
"fmul v1.4s, v5.4s, v1.4s \n" // x0^3
|
||||
"fmul v2.4s, v6.4s, v2.4s \n" // x1^3
|
||||
"fadd v3.4s, v16.4s, v3.4s \n" // c0 + c1 * x0
|
||||
"fadd v4.4s, v16.4s, v4.4s \n" // c0 + c1 * x1
|
||||
"fmla v3.4s, v18.4s, v5.4s \n" // + c2 * x0^2
|
||||
"fmla v4.4s, v18.4s, v6.4s \n" // + c2 * x1^2
|
||||
"fmla v3.4s, v19.4s, v1.4s \n" // + c3 * x0^3
|
||||
"fmla v4.4s, v19.4s, v2.4s \n" // + c3 * x1^3
|
||||
"fcvtzs v3.4s, v3.4s \n"
|
||||
"fcvtzs v4.4s, v4.4s \n"
|
||||
"sqxtn v3.4h, v3.4s \n"
|
||||
"sqxtn2 v3.8h, v4.4s \n"
|
||||
"sqxtun v3.8b, v3.8h \n"
|
||||
"str d3, [%1], #8 \n"
|
||||
"3: \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width) // %2
|
||||
: "r"(poly) // %3
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
|
||||
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
|
||||
}
|
||||
|
||||
void ARGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width) {
|
||||
uint64_t b, g, r, a;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldrb %w2, [%0] \n"
|
||||
"ldrb %w3, [%0, #1] \n"
|
||||
"ldrb %w4, [%0, #2] \n"
|
||||
"ldrb %w5, [%0, #3] \n"
|
||||
"subs %w1, %w1, #1 \n"
|
||||
"ldr %w2, [%6, %w2, uxtw #2] \n"
|
||||
"ldr %w3, [%6, %w3, uxtw #2] \n"
|
||||
"ldr %w4, [%6, %w4, uxtw #2] \n"
|
||||
"ldr %w5, [%6, %w5, uxtw #2] \n"
|
||||
"bfi %w3, %w2, #0, #8 \n"
|
||||
"lsr %w4, %w4, #16 \n"
|
||||
"lsr %w5, %w5, #24 \n"
|
||||
"strh %w3, [%0] \n"
|
||||
"strb %w4, [%0, #2] \n"
|
||||
"strb %w5, [%0, #3] \n"
|
||||
"add %0, %0, #4 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(b), // %2
|
||||
"=&r"(g), // %3
|
||||
"=&r"(r), // %4
|
||||
"=&r"(a) // %5
|
||||
: "r"(table_argb) // %6
|
||||
: "memory", "cc");
|
||||
}
|
||||
|
||||
void RGBColorTableRow_NEON(uint8_t* dst_argb,
|
||||
const uint8_t* table_argb,
|
||||
int width) {
|
||||
uint64_t b, g, r;
|
||||
asm volatile(
|
||||
"1: \n"
|
||||
"ldrb %w2, [%0] \n"
|
||||
"ldrb %w3, [%0, #1] \n"
|
||||
"ldrb %w4, [%0, #2] \n"
|
||||
"subs %w1, %w1, #1 \n"
|
||||
"ldr %w2, [%5, %w2, uxtw #2] \n"
|
||||
"ldr %w3, [%5, %w3, uxtw #2] \n"
|
||||
"ldr %w4, [%5, %w4, uxtw #2] \n"
|
||||
"bfi %w3, %w2, #0, #8 \n"
|
||||
"lsr %w4, %w4, #16 \n"
|
||||
"strh %w3, [%0] \n"
|
||||
"strb %w4, [%0, #2] \n"
|
||||
"add %0, %0, #4 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"=&r"(b), // %2
|
||||
"=&r"(g), // %3
|
||||
"=&r"(r) // %4
|
||||
: "r"(table_argb) // %5
|
||||
: "memory", "cc");
|
||||
}
|
||||
|
||||
static const uvec8 kLumaTblIdx = {1, 1, 1, 1, 3, 3, 3, 3,
|
||||
5, 5, 5, 5, 7, 7, 7, 7};
|
||||
|
||||
void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
|
||||
uint8_t* dst_argb,
|
||||
int width,
|
||||
const uint8_t* luma,
|
||||
uint32_t lumacoeff) {
|
||||
uint64_t p0, p1, t0, t1;
|
||||
asm volatile(
|
||||
"dup v4.4s, %w8 \n" // broadcast 4 byte coeffs
|
||||
"ldr q5, [%9] \n"
|
||||
"movi v6.4s, #0xff, lsl #24 \n"
|
||||
"1: \n"
|
||||
"ldr q0, [%0], #16 \n" // load 4 ARGB pixels
|
||||
"subs %w2, %w2, #4 \n" // 4 pixels per loop
|
||||
"umull v1.8h, v0.8b, v4.8b \n"
|
||||
"umull2 v2.8h, v0.16b, v4.16b \n"
|
||||
"addp v1.8h, v1.8h, v2.8h \n"
|
||||
"addp v1.8h, v1.8h, v1.8h \n"
|
||||
"bic v1.8h, #0x80, lsl #8 \n" // clear bit 15
|
||||
"tbl v1.16b, {v1.16b}, v5.16b \n"
|
||||
"zip2 v2.16b, v0.16b, v1.16b \n" // offsets pixels 2, 3
|
||||
"zip1 v1.16b, v0.16b, v1.16b \n" // offsets pixels 0, 1
|
||||
"umov %3, v1.d[0] \n"
|
||||
"umov %4, v1.d[1] \n"
|
||||
"add %5, %7, %w3, uxth \n"
|
||||
"ubfx %6, %3, #16, #16 \n"
|
||||
"ldr b3, [%5] \n"
|
||||
"add %6, %7, %6 \n"
|
||||
"ubfx %5, %3, #32, #16 \n"
|
||||
"ld1 {v3.b}[1], [%6] \n"
|
||||
"add %5, %7, %5 \n"
|
||||
"add %6, %7, %w4, uxth \n"
|
||||
"ld1 {v3.b}[2], [%5] \n"
|
||||
"ld1 {v3.b}[4], [%6] \n"
|
||||
"ubfx %5, %4, #16, #16 \n"
|
||||
"ubfx %6, %4, #32, #16 \n"
|
||||
"add %5, %7, %5 \n"
|
||||
"add %6, %7, %6 \n"
|
||||
"ld1 {v3.b}[5], [%5] \n"
|
||||
"ld1 {v3.b}[6], [%6] \n"
|
||||
"umov %3, v2.d[0] \n"
|
||||
"umov %4, v2.d[1] \n"
|
||||
"add %5, %7, %w3, uxth \n"
|
||||
"ubfx %6, %3, #16, #16 \n"
|
||||
"ld1 {v3.b}[8], [%5] \n"
|
||||
"add %6, %7, %6 \n"
|
||||
"ubfx %5, %3, #32, #16 \n"
|
||||
"ld1 {v3.b}[9], [%6] \n"
|
||||
"add %5, %7, %5 \n"
|
||||
"add %6, %7, %w4, uxth \n"
|
||||
"ld1 {v3.b}[10], [%5] \n"
|
||||
"ld1 {v3.b}[12], [%6] \n"
|
||||
"ubfx %5, %4, #16, #16 \n"
|
||||
"ubfx %6, %4, #32, #16 \n"
|
||||
"add %5, %7, %5 \n"
|
||||
"add %6, %7, %6 \n"
|
||||
"ld1 {v3.b}[13], [%5] \n"
|
||||
"ld1 {v3.b}[14], [%6] \n"
|
||||
"bit v3.16b, v0.16b, v6.16b \n"
|
||||
"str q3, [%1], #16 \n"
|
||||
"b.gt 1b \n"
|
||||
: "+r"(src_argb), // %0
|
||||
"+r"(dst_argb), // %1
|
||||
"+r"(width), // %2
|
||||
"=&r"(p0), // %3
|
||||
"=&r"(p1), // %4
|
||||
"=&r"(t0), // %5
|
||||
"=&r"(t1) // %6
|
||||
: "r"(luma), // %7
|
||||
"r"(lumacoeff), // %8
|
||||
"r"(&kLumaTblIdx) // %9
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
|
||||
}
|
||||
|
||||
void ARGBAffineRow_NEON(const uint8_t* src_argb,
|
||||
int src_argb_stride,
|
||||
uint8_t* dst_argb,
|
||||
const float* uv_dudv,
|
||||
int width) {
|
||||
int64_t off0, off1;
|
||||
asm volatile(
|
||||
"ld1r {v2.2d}, [%2], #8 \n" // [u, v, u, v]
|
||||
"ld1r {v6.2d}, [%2] \n" // [du, dv, du, dv]
|
||||
"dup v5.2s, %w5 \n" // [stride, stride]
|
||||
"movi v4.2s, #4 \n" // [4, 4]
|
||||
"zip1 v5.4s, v4.4s, v5.4s \n" // [4, stride, 4, stride]
|
||||
"subs %w1, %w1, #4 \n"
|
||||
"b.lt 2f \n"
|
||||
"fadd v4.4s, v6.4s, v6.4s \n" // 2 * [du, dv, du, dv]
|
||||
"fadd v3.2s, v2.2s, v6.2s \n" // [u+du, v+dv]
|
||||
"ins v2.d[1], v3.d[0] \n" // [u, v, u+du, v+dv]
|
||||
"fadd v3.4s, v2.4s, v4.4s \n" // [u+2du, v+2dv, u+3du,
|
||||
// v+3dv]
|
||||
"fadd v4.4s, v4.4s, v4.4s \n" // 4 * [du, dv, du, dv]
|
||||
"1: \n"
|
||||
"fcvtzs v0.4s, v2.4s \n"
|
||||
"fcvtzs v1.4s, v3.4s \n"
|
||||
"fadd v2.4s, v2.4s, v4.4s \n"
|
||||
"fadd v3.4s, v3.4s, v4.4s \n"
|
||||
"mul v0.4s, v0.4s, v5.4s \n"
|
||||
"mul v1.4s, v1.4s, v5.4s \n"
|
||||
"addp v7.4s, v0.4s, v1.4s \n"
|
||||
"smov %3, v7.s[0] \n"
|
||||
"smov %4, v7.s[1] \n"
|
||||
"ldr %w3, [%6, %3] \n"
|
||||
"ldr %w4, [%6, %4] \n"
|
||||
"stp %w3, %w4, [%0], #8 \n"
|
||||
"smov %3, v7.s[2] \n"
|
||||
"smov %4, v7.s[3] \n"
|
||||
"ldr %w3, [%6, %3] \n"
|
||||
"ldr %w4, [%6, %4] \n"
|
||||
"stp %w3, %w4, [%0], #8 \n"
|
||||
"subs %w1, %w1, #4 \n"
|
||||
"b.ge 1b \n"
|
||||
"2: \n"
|
||||
"adds %w1, %w1, #4 \n"
|
||||
"b.eq 4f \n"
|
||||
"3: \n"
|
||||
"fcvtzs v0.2s, v2.2s \n"
|
||||
"fadd v2.2s, v2.2s, v6.2s \n"
|
||||
"mul v0.2s, v0.2s, v5.2s \n"
|
||||
"addp v0.2s, v0.2s, v0.2s \n"
|
||||
"smov %3, v0.s[0] \n"
|
||||
"ldr %w3, [%6, %3] \n"
|
||||
"str %w3, [%0], #4 \n"
|
||||
"subs %w1, %w1, #1 \n"
|
||||
"b.gt 3b \n"
|
||||
"4: \n"
|
||||
: "+r"(dst_argb), // %0
|
||||
"+r"(width), // %1
|
||||
"+r"(uv_dudv), // %2
|
||||
"=&r"(off0), // %3
|
||||
"=&r"(off1) // %4
|
||||
: "r"(src_argb_stride), // %5
|
||||
"r"(src_argb) // %6
|
||||
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
|
||||
}
|
||||
|
||||
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -1574,6 +1574,20 @@ TEST_F(LibYUVPlanarTest, TestAffine) {
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_ARGBAFFINEROW_NEON)
|
||||
SIMD_ALIGNED(uint8_t interpolate_pixels_Opt[1280][4]);
|
||||
ARGBAffineRow_NEON(&orig_pixels_0[0][0], 0, &interpolate_pixels_Opt[0][0],
|
||||
uv_step, 1280);
|
||||
ASSERT_EQ(0, memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1280 * 4));
|
||||
|
||||
int has_neon = TestCpuFlag(kCpuHasNEON);
|
||||
if (has_neon) {
|
||||
for (int i = 0; i < benchmark_pixels_div1280_; ++i) {
|
||||
ARGBAffineRow_NEON(&orig_pixels_0[0][0], 0, &interpolate_pixels_Opt[0][0],
|
||||
uv_step, 1280);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif // ENABLE_ROW_TESTS
|
||||
|
||||
@ -2417,7 +2431,7 @@ TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Opt)) {
|
||||
ASSERT_LE(max_diff, 1);
|
||||
}
|
||||
|
||||
TEST_F(LibYUVPlanarTest, DISABLED_ARM(TestARGBPolynomial)) {
|
||||
TEST_F(LibYUVPlanarTest, TestARGBPolynomial) {
|
||||
SIMD_ALIGNED(uint8_t orig_pixels[1280][4]);
|
||||
SIMD_ALIGNED(uint8_t dst_pixels_opt[1280][4]);
|
||||
SIMD_ALIGNED(uint8_t dst_pixels_c[1280][4]);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user