diff --git a/README.chromium b/README.chromium index f1f089ed9..6dfc0d43d 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1975 +Version: 1976 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/row.h b/include/libyuv/row.h index b13512aac..74ccfc1ca 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -540,6 +540,9 @@ extern "C" { #if !defined(LIBYUV_DISABLE_NEON) && \ (defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON)) #define HAS_AB64TOARGBROW_NEON +#define HAS_AR30TOAB30ROW_NEON +#define HAS_AR30TOABGRROW_NEON +#define HAS_AR30TOARGBROW_NEON #define HAS_AR64TOARGBROW_NEON #define HAS_ARGB1555TOARGBROW_NEON #define HAS_ARGB1555TOUVROW_NEON @@ -547,6 +550,7 @@ extern "C" { #define HAS_ARGB4444TOARGBROW_NEON #define HAS_ARGB4444TOUVROW_NEON #define HAS_ARGB4444TOYROW_NEON +#define HAS_ARGBCOPYALPHAROW_NEON #define HAS_ARGBCOPYYTOALPHAROW_NEON #define HAS_ARGBEXTRACTALPHAROW_NEON #define HAS_ARGBSETROW_NEON @@ -578,6 +582,7 @@ extern "C" { #define HAS_CONVERT16TO8ROW_NEON #define HAS_CONVERT8TO16ROW_NEON #define HAS_CONVERT8TO8ROW_NEON +#define HAS_COPYROW_16_NEON #define HAS_COPYROW_NEON #define HAS_DETILEROW_16_NEON #define HAS_DETILEROW_NEON @@ -613,6 +618,7 @@ extern "C" { #define HAS_MERGEXR64ROW_NEON #define HAS_MERGEXRGB16TO8ROW_NEON #define HAS_MERGEXRGBROW_NEON +#define HAS_MIRRORROW_16_NEON #define HAS_MIRRORROW_NEON #define HAS_MIRRORSPLITUVROW_NEON #define HAS_MIRRORUVROW_NEON @@ -651,21 +657,27 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_NEON +#define HAS_ARGBAFFINEROW_NEON #define HAS_ARGBALPHABLENDROW_NEON #define HAS_ARGBATTENUATEROW_NEON #define HAS_ARGBBLENDROW_NEON #define HAS_BLENDPLANEROW_NEON #define HAS_ARGBCOLORMATRIXROW_NEON +#define HAS_ARGBCOLORTABLEROW_NEON #define HAS_ARGBGRAYROW_NEON +#define HAS_ARGBLUMACOLORTABLEROW_NEON #define HAS_ARGBMIRRORROW_NEON #define HAS_ARGBMULTIPLYROW_NEON #define HAS_ARGBORROW_NEON +#define HAS_ARGBPOLYNOMIALROW_NEON #define HAS_ARGBQUANTIZEROW_NEON #define HAS_ARGBSEPIAROW_NEON #define HAS_ARGBSHADEROW_NEON #define HAS_ARGBSHUFFLEROW_NEON #define HAS_ARGBSUBTRACTROW_NEON +#define HAS_ARGBUNATTENUATEROW_NEON #define HAS_RGB24MIRRORROW_NEON +#define HAS_RGBCOLORTABLEROW_NEON #define HAS_SOBELROW_NEON #define HAS_SOBELTOPLANEROW_NEON #define HAS_SOBELXROW_NEON @@ -2835,6 +2847,10 @@ void MirrorSplitUVRow_C(const uint8_t* src_uv, void MirrorRow_16_C(const uint16_t* src, uint16_t* dst, int width); void MirrorRow_16_AVX512BW(const uint16_t* src, uint16_t* dst, int width); +void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width); +void MirrorRow_16_Any_NEON(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int width); void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width); void ARGBMirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width); @@ -4011,16 +4027,22 @@ void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count); +void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count); +void CopyRow_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int width); void ARGBCopyAlphaRow_C(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width); +void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyAlphaRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void ARGBCopyAlphaRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBCopyAlphaRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void ARGBExtractAlphaRow_C(const uint8_t* src_argb, uint8_t* dst_a, int width); void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb, @@ -4276,6 +4298,18 @@ void AR30ToAB30Row_C(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); +void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width); +void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); +void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); +void AR30ToARGBRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToABGRRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToAB30Row_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void RGB24ToARGBRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -6925,12 +6959,18 @@ void ARGBUnattenuateRow_SSE2(const uint8_t* src_argb, void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width); +void ARGBUnattenuateRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); void ARGBUnattenuateRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void ARGBUnattenuateRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBUnattenuateRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void ARGBGrayRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width); @@ -6985,6 +7025,9 @@ void ARGBColorTableRow_C(uint8_t* dst_argb, void ARGBColorTableRow_X86(uint8_t* dst_argb, const uint8_t* table_argb, int width); +void ARGBColorTableRow_NEON(uint8_t* dst_argb, + const uint8_t* table_argb, + int width); void RGBColorTableRow_C(uint8_t* dst_argb, const uint8_t* table_argb, @@ -6992,6 +7035,9 @@ void RGBColorTableRow_C(uint8_t* dst_argb, void RGBColorTableRow_X86(uint8_t* dst_argb, const uint8_t* table_argb, int width); +void RGBColorTableRow_NEON(uint8_t* dst_argb, + const uint8_t* table_argb, + int width); void ARGBQuantizeRow_C(uint8_t* dst_argb, int scale, @@ -7095,6 +7141,12 @@ void ARGBAffineRow_SSE2(const uint8_t* src_argb, uint8_t* dst_argb, const float* src_dudv, int width); +LIBYUV_API +void ARGBAffineRow_NEON(const uint8_t* src_argb, + int src_argb_stride, + uint8_t* dst_argb, + const float* src_dudv, + int width); // Used for I420Scale, ARGBScale, and ARGBInterpolate. void InterpolateRow_C(uint8_t* dst_ptr, @@ -7348,6 +7400,14 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, const float* poly, int width); +void ARGBPolynomialRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + const float* poly, + int width); +void ARGBPolynomialRow_Any_NEON(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const float* param, + int width); // Scale and convert to half float. void HalfFloatRow_C(const uint16_t* src, uint16_t* dst, float scale, int width); @@ -7439,6 +7499,11 @@ void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb, int width, const uint8_t* luma, uint32_t lumacoeff); +void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff); float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width); float ScaleMaxSamples_NEON(const float* src, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index dc85ebbe1..caf8590aa 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1975 +#define LIBYUV_VERSION 1976 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/convert.cc b/source/convert.cc index 783071ce1..67385d9ff 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -3538,6 +3538,14 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, src_ar30 = src_ar30 + (ptrdiff_t)(height - 1) * src_stride_ar30; src_stride_ar30 = -src_stride_ar30; } +#if defined(HAS_AR30TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + AR30ToARGBRow = AR30ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + AR30ToARGBRow = AR30ToARGBRow_NEON; + } + } +#endif #if defined(HAS_AR30TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToARGBRow = AR30ToARGBRow_AVX512BW; diff --git a/source/convert_argb.cc b/source/convert_argb.cc index 9b7dea6bf..e9d8b9c1c 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -4522,6 +4522,14 @@ int AR30ToARGB(const uint8_t* src_ar30, } void (*AR30ToARGBRow)(const uint8_t* src_ar30, uint8_t* dst_argb, int width) = AR30ToARGBRow_C; +#if defined(HAS_AR30TOARGBROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + AR30ToARGBRow = AR30ToARGBRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + AR30ToARGBRow = AR30ToARGBRow_NEON; + } + } +#endif #if defined(HAS_AR30TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToARGBRow = AR30ToARGBRow_AVX512BW; @@ -4563,6 +4571,14 @@ int AR30ToABGR(const uint8_t* src_ar30, height = 1; src_stride_ar30 = dst_stride_abgr = 0; } +#if defined(HAS_AR30TOABGRROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + AR30ToABGRRow = AR30ToABGRRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + AR30ToABGRRow = AR30ToABGRRow_NEON; + } + } +#endif #if defined(HAS_AR30TOABGRROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToABGRRow = AR30ToABGRRow_AVX512BW; @@ -4604,6 +4620,14 @@ int AR30ToAB30(const uint8_t* src_ar30, height = 1; src_stride_ar30 = dst_stride_ab30 = 0; } +#if defined(HAS_AR30TOAB30ROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + AR30ToAB30Row = AR30ToAB30Row_Any_NEON; + if (IS_ALIGNED(width, 8)) { + AR30ToAB30Row = AR30ToAB30Row_NEON; + } + } +#endif #if defined(HAS_AR30TOAB30ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToAB30Row = AR30ToAB30Row_AVX512BW; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index dcf6b9227..500ecc8ce 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -2926,7 +2926,14 @@ void MirrorPlane_16(const uint16_t* src_y, src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y; src_stride_y = -src_stride_y; } -#if defined(HAS_MIRRORUVROW_NEON) +#if defined(HAS_MIRRORROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + MirrorRow_16 = MirrorRow_16_Any_NEON; + if (IS_ALIGNED(width, 16)) { + MirrorRow_16 = MirrorRow_16_NEON; + } + } +#elif defined(HAS_MIRRORUVROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { MirrorRow_16 = reinterpret_cast( MirrorUVRow_Any_NEON); @@ -4693,7 +4700,14 @@ int ARGBUnattenuate(const uint8_t* src_argb, } } #endif - // TODO(fbarchard): Neon version. +#if defined(HAS_ARGBUNATTENUATEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBUnattenuateRow = ARGBUnattenuateRow_Any_NEON; + if (IS_ALIGNED(width, 4)) { + ARGBUnattenuateRow = ARGBUnattenuateRow_NEON; + } + } +#endif for (y = 0; y < height; ++y) { ARGBUnattenuateRow(src_argb, dst_argb, width); @@ -5019,6 +5033,11 @@ int ARGBColorTable(uint8_t* dst_argb, if (TestCpuFlag(kCpuHasX86)) { ARGBColorTableRow = ARGBColorTableRow_X86; } +#endif +#if defined(HAS_ARGBCOLORTABLEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBColorTableRow = ARGBColorTableRow_NEON; + } #endif for (y = 0; y < height; ++y) { ARGBColorTableRow(dst, table_argb, width); @@ -5055,6 +5074,11 @@ int RGBColorTable(uint8_t* dst_argb, if (TestCpuFlag(kCpuHasX86)) { RGBColorTableRow = RGBColorTableRow_X86; } +#endif +#if defined(HAS_RGBCOLORTABLEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGBColorTableRow = RGBColorTableRow_NEON; + } #endif for (y = 0; y < height; ++y) { RGBColorTableRow(dst, table_argb, width); @@ -6224,6 +6248,14 @@ int ARGBPolynomial(const uint8_t* src_argb, ARGBPolynomialRow = ARGBPolynomialRow_AVX2; } #endif +#if defined(HAS_ARGBPOLYNOMIALROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBPolynomialRow = ARGBPolynomialRow_Any_NEON; + if (IS_ALIGNED(width, 2)) { + ARGBPolynomialRow = ARGBPolynomialRow_NEON; + } + } +#endif for (y = 0; y < height; ++y) { ARGBPolynomialRow(src_argb, dst_argb, poly, width); @@ -6383,6 +6415,11 @@ int ARGBLumaColorTable(const uint8_t* src_argb, ARGBLumaColorTableRow = ARGBLumaColorTableRow_SSSE3; } #endif +#if defined(HAS_ARGBLUMACOLORTABLEROW_NEON) + if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 4)) { + ARGBLumaColorTableRow = ARGBLumaColorTableRow_NEON; + } +#endif for (y = 0; y < height; ++y) { ARGBLumaColorTableRow(src_argb, dst_argb, width, luma, 0x00264b0f); @@ -6420,6 +6457,14 @@ int ARGBCopyAlpha(const uint8_t* src_argb, height = 1; src_stride_argb = dst_stride_argb = 0; } +#if defined(HAS_ARGBCOPYALPHAROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_NEON; + } + } +#endif #if defined(HAS_ARGBCOPYALPHAROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_SSE2; diff --git a/source/row_any.cc b/source/row_any.cc index 5dcf5325b..bb59eaf3e 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1105,6 +1105,15 @@ ANY11(ABGRToAR30Row_Any_NEON, ABGRToAR30Row_NEON, 0, 4, 4, 7) #if defined(HAS_ARGBTOAR30ROW_NEON) ANY11(ARGBToAR30Row_Any_NEON, ARGBToAR30Row_NEON, 0, 4, 4, 7) #endif +#if defined(HAS_AR30TOARGBROW_NEON) +ANY11(AR30ToARGBRow_Any_NEON, AR30ToARGBRow_NEON, 0, 4, 4, 7) +#endif +#if defined(HAS_AR30TOABGRROW_NEON) +ANY11(AR30ToABGRRow_Any_NEON, AR30ToABGRRow_NEON, 0, 4, 4, 7) +#endif +#if defined(HAS_AR30TOAB30ROW_NEON) +ANY11(AR30ToAB30Row_Any_NEON, AR30ToAB30Row_NEON, 0, 4, 4, 7) +#endif #if defined(HAS_ARGBTOAR30ROW_SSSE3) ANY11(ARGBToAR30Row_Any_SSSE3, ARGBToAR30Row_SSSE3, 0, 4, 4, 3) #endif @@ -1330,6 +1339,9 @@ ANY11(ARGBUnattenuateRow_Any_AVX2, ARGBUnattenuateRow_AVX2, 0, 4, 4, 7) #ifdef HAS_ARGBATTENUATEROW_NEON ANY11(ARGBAttenuateRow_Any_NEON, ARGBAttenuateRow_NEON, 0, 4, 4, 7) #endif +#ifdef HAS_ARGBUNATTENUATEROW_NEON +ANY11(ARGBUnattenuateRow_Any_NEON, ARGBUnattenuateRow_NEON, 0, 4, 4, 3) +#endif #ifdef HAS_ARGBATTENUATEROW_LSX ANY11(ARGBAttenuateRow_Any_LSX, ARGBAttenuateRow_LSX, 0, 4, 4, 3) #endif @@ -1419,6 +1431,9 @@ ANY11B(ARGBCopyAlphaRow_Any_AVX2, ARGBCopyAlphaRow_AVX2, 0, 4, 4, 15) #ifdef HAS_ARGBCOPYALPHAROW_SSE2 ANY11B(ARGBCopyAlphaRow_Any_SSE2, ARGBCopyAlphaRow_SSE2, 0, 4, 4, 7) #endif +#ifdef HAS_ARGBCOPYALPHAROW_NEON +ANY11B(ARGBCopyAlphaRow_Any_NEON, ARGBCopyAlphaRow_NEON, 0, 4, 4, 7) +#endif #ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2 ANY11B(ARGBCopyYToAlphaRow_Any_AVX2, ARGBCopyYToAlphaRow_AVX2, 0, 1, 4, 15) #endif @@ -1578,6 +1593,14 @@ ANY11P(ARGBOrRow_Any_LASX, ARGBOrRow_LASX, const uint32_t, 4, 4, 7) #ifdef HAS_ARGBORROW_WASMSIMD ANY11P(ARGBOrRow_Any_WASMSIMD, ARGBOrRow_WASMSIMD, const uint32_t, 4, 4, 3) #endif +#ifdef HAS_ARGBPOLYNOMIALROW_NEON +ANY11P(ARGBPolynomialRow_Any_NEON, + ARGBPolynomialRow_NEON, + const float*, + 4, + 4, + 1) +#endif #undef ANY11P #undef ANY11P @@ -1647,6 +1670,10 @@ ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) #endif +#ifdef HAS_COPYROW_16_NEON +ANY11T(CopyRow_16_Any_NEON, CopyRow_16_NEON, 2, 2, uint16_t, uint16_t, 15) +#endif + #undef ANY11T // Any 1 to 1 with parameter and shorts. BPP measures in shorts. @@ -2196,6 +2223,31 @@ ANY11M(RGB24MirrorRow_Any_WASMSIMD, RGB24MirrorRow_WASMSIMD, 3, 15) #endif #undef ANY11M +// Any 1 to 1 mirror with type. +#define ANY11MT(NAMEANY, ANY_SIMD, T, BPP, MASK) \ + void NAMEANY(const T* src_ptr, T* dst_ptr, int width) { \ + SIMD_ALIGNED(T vin[64]); \ + static_assert((MASK + 1) * BPP <= sizeof(vin), "vin buffer too small"); \ + SIMD_ALIGNED(T vout[64]); \ + static_assert((MASK + 1) * BPP <= sizeof(vout), "vout buffer too small"); \ + MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr + r, dst_ptr, n); \ + } \ + ptrdiff_t np = n; \ + memcpy(vin, src_ptr, r * BPP); \ + ANY_SIMD(vin, vout, MASK + 1); \ + memcpy(dst_ptr + np, vout + (MASK + 1 - r), r * BPP); \ + } + +#ifdef HAS_MIRRORROW_16_NEON +ANY11MT(MirrorRow_16_Any_NEON, MirrorRow_16_NEON, uint16_t, 2, 15) +#endif +#undef ANY11MT + // Any 1 plane. (memset) #define ANY1(NAMEANY, ANY_SIMD, T, BPP, MASK) \ void NAMEANY(uint8_t* dst_ptr, T v32, int width) { \ diff --git a/source/row_neon.cc b/source/row_neon.cc index ea072ee9b..fe6b747a9 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -3992,6 +3992,433 @@ void Convert8To16Row_NEON(const uint8_t* src_y, : "cc", "memory", "q0", "q1", "q2"); } +void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count) { + asm volatile( + "1: \n" + "vld1.16 {d0, d1, d2, d3}, [%0]! \n" // load 16 + "subs %2, %2, #16 \n" // 16 processed per loop + "vst1.16 {d0, d1, d2, d3}, [%1]! \n" // store 16 + "bgt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(count) // %2 + : + : "memory", "cc", "q0", "q1"); +} + +void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width) { + asm volatile( + // Start at end of source row. + "add %0, %0, %2, lsl #1 \n" + "sub %0, %0, #32 \n" // 32 bytes per loop + "1: \n" + "vld1.16 {q1, q2}, [%0], %3 \n" // src -= 32 + "subs %2, %2, #16 \n" // 16 pixels per loop. + "vrev64.16 q0, q2 \n" + "vrev64.16 q1, q1 \n" + "vswp d0, d1 \n" + "vswp d2, d3 \n" + "vst1.16 {q0, q1}, [%1]! \n" // dst += 32 + "bgt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "r"(-32) // %3 + : "memory", "cc", "q0", "q1", "q2"); +} + +void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width) { + asm volatile( + "vmov.i32 q8, #0x0000ff00 \n" + "vmov.i32 q9, #0x000fffff \n" + "vshl.i32 q9, q9, #8 \n" // 0x0fffff00 + "1: \n" + "vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels + "subs %2, %2, #8 \n" // 8 pixels per loop + "vsri.32 q0, q0, #2 \n" + "vsri.32 q1, q1, #2 \n" + "vshr.u32 q2, q0, #2 \n" + "vshr.u32 q3, q1, #2 \n" + "vshr.u32 q4, q0, #4 \n" + "vshr.u32 q5, q1, #4 \n" + "vbit q0, q4, q9 \n" + "vbit q1, q5, q9 \n" + "vbit q0, q2, q8 \n" + "vbit q1, q3, q8 \n" + "vst1.8 {q0, q1}, [%1]! \n" // store 8 ARGB pixels + "bgt 1b \n" + : "+r"(src_ar30), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9"); +} + +void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) { + asm volatile( + "vmov.i64 q8, #0x00ffff0000ffff00 \n" + "1: \n" + "vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels + "subs %2, %2, #8 \n" // 8 pixels per loop + "vsri.32 q0, q0, #2 \n" + "vsri.32 q1, q1, #2 \n" + "vshr.u32 q2, q0, #2 \n" + "vshr.u32 q3, q1, #2 \n" + "vsli.32 q2, q0, #16 \n" + "vsli.32 q3, q1, #16 \n" + "vsri.32 q0, q0, #4 \n" + "vsri.32 q1, q1, #4 \n" + "vsri.32 q0, q0, #16 \n" + "vsri.32 q1, q1, #16 \n" + "vbit q0, q2, q8 \n" + "vbit q1, q3, q8 \n" + "vst1.8 {q0, q1}, [%1]! \n" // store 8 ABGR pixels + "bgt 1b \n" + : "+r"(src_ar30), // %0 + "+r"(dst_abgr), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q8"); +} + +void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) { + asm volatile( + "vmov.i8 q9, #0xff \n" + "vshr.u32 q9, q9, #22 \n" // 0x000003ff + "vshl.i32 q8, q9, #20 \n" // 0x3ff00000 + "1: \n" + "vld1.8 {q0, q1}, [%0]! \n" // load 8 AR30 pixels + "subs %2, %2, #8 \n" // 8 pixels per loop + "vshl.i32 q2, q0, #20 \n" + "vshl.i32 q3, q1, #20 \n" + "vshr.u32 q4, q0, #20 \n" + "vshr.u32 q5, q1, #20 \n" + "vbit q0, q2, q8 \n" + "vbit q1, q3, q8 \n" + "vbit q0, q4, q9 \n" + "vbit q1, q5, q9 \n" + "vst1.8 {q0, q1}, [%1]! \n" // store 8 AB30 pixels + "bgt 1b \n" + : "+r"(src_ar30), // %0 + "+r"(dst_ab30), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9"); +} + +void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) { + asm volatile( + "vmov.i32 q4, #0xff000000 \n" + "1: \n" + "vld1.8 {q0, q1}, [%0]! \n" // load 8 src pixels + "vld1.8 {q2, q3}, [%1] \n" // load 8 dst pixels + "subs %2, %2, #8 \n" // 8 pixels per loop + "vbit q2, q0, q4 \n" + "vbit q3, q1, q4 \n" + "vst1.8 {q2, q3}, [%1]! \n" // store 8 dst pixels + "bgt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q4"); +} + +void ARGBUnattenuateRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + uint32_t tmp, addr; + asm volatile( + "vmov.i32 q8, #0xff000000 \n" + "1: \n" + "ldrb %3, [%0, #3] \n" + "add %4, %5, %3, lsl #2 \n" + "vld1.16 {d6[0]}, [%4] \n" + "ldrb %3, [%0, #7] \n" + "add %4, %5, %3, lsl #2 \n" + "vld1.16 {d6[1]}, [%4] \n" + "ldrb %3, [%0, #11] \n" + "add %4, %5, %3, lsl #2 \n" + "vld1.16 {d6[2]}, [%4] \n" + "ldrb %3, [%0, #15] \n" + "add %4, %5, %3, lsl #2 \n" + "vld1.16 {d6[3]}, [%4] \n" + "vld1.8 {q0}, [%0]! \n" + "subs %2, %2, #4 \n" + "vmov q1, q0 \n" + "vmov q2, q0 \n" + "vzip.8 q1, q2 \n" + "vmull.u16 q4, d2, d6[0] \n" + "vmull.u16 q5, d3, d6[1] \n" + "vmull.u16 q6, d4, d6[2] \n" + "vmull.u16 q7, d5, d6[3] \n" + "vshrn.i32 d8, q4, #16 \n" + "vshrn.i32 d9, q5, #16 \n" + "vshrn.i32 d12, q6, #16 \n" + "vshrn.i32 d13, q7, #16 \n" + "vqmovn.u16 d2, q4 \n" + "vqmovn.u16 d3, q6 \n" + "vbit q1, q0, q8 \n" + "vst1.8 {q1}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width), // %2 + "=&r"(tmp), // %3 + "=&r"(addr) // %4 + : "r"(fixed_invtbl8) // %5 + : "memory", "cc", "q0", "q1", "q2", "d6", "q4", "q5", "q6", "q7", "q8"); +} + +void ARGBPolynomialRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + const float* poly, + int width) { + asm volatile( + "vld1.32 {q8, q9}, [%3]! \n" // c0, c1 + "vld1.32 {q10, q11}, [%3] \n" // c2, c3 + "1: \n" + "vld1.8 {d0}, [%0]! \n" // load 2 pixels + "subs %2, %2, #2 \n" // 2 pixels per loop + "vmovl.u8 q0, d0 \n" + "vmovl.u16 q1, d0 \n" + "vmovl.u16 q2, d1 \n" + "vcvt.f32.u32 q1, q1 \n" // x0 + "vcvt.f32.u32 q2, q2 \n" // x1 + "vmov q3, q8 \n" // c0 + "vmov q4, q8 \n" // c0 + "vmla.f32 q3, q9, q1 \n" // c0 + c1 * x0 + "vmla.f32 q4, q9, q2 \n" // c0 + c1 * x1 + "vmul.f32 q5, q1, q1 \n" // x0 * x0 + "vmul.f32 q6, q2, q2 \n" // x1 * x1 + "vmla.f32 q3, q10, q5 \n" // + c2 * x0^2 + "vmla.f32 q4, q10, q6 \n" // + c2 * x1^2 + "vmul.f32 q5, q5, q1 \n" // x0^3 + "vmul.f32 q6, q6, q2 \n" // x1^3 + "vmla.f32 q3, q11, q5 \n" // + c3 * x0^3 + "vmla.f32 q4, q11, q6 \n" // + c3 * x1^3 + "vcvt.s32.f32 q3, q3 \n" + "vcvt.s32.f32 q4, q4 \n" + "vqmovn.s32 d6, q3 \n" + "vqmovn.s32 d7, q4 \n" + "vqmovun.s16 d6, q3 \n" + "vst1.8 {d6}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width), // %2 + "+r"(poly) // %3 + : + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q8", "q9", + "q10", "q11"); +} + +void ARGBColorTableRow_NEON(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + const uint8_t* table_g = table_argb + 1; + const uint8_t* table_r = table_argb + 2; + const uint8_t* table_a = table_argb + 3; + uint32_t b, g, r, a; + asm volatile( + "1: \n" + "ldrb %2, [%0] \n" + "ldrb %3, [%0, #1] \n" + "ldrb %4, [%0, #2] \n" + "ldrb %5, [%0, #3] \n" + "ldrb %2, [%6, %2, lsl #2] \n" + "ldrb %3, [%7, %3, lsl #2] \n" + "ldrb %4, [%8, %4, lsl #2] \n" + "ldrb %5, [%9, %5, lsl #2] \n" + "subs %1, %1, #1 \n" + "strb %2, [%0] \n" + "strb %3, [%0, #1] \n" + "strb %4, [%0, #2] \n" + "strb %5, [%0, #3] \n" + "add %0, %0, #4 \n" + "bgt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width), // %1 + "=&r"(b), // %2 + "=&r"(g), // %3 + "=&r"(r), // %4 + "=&r"(a) // %5 + : "r"(table_argb), // %6 + "r"(table_g), // %7 + "r"(table_r), // %8 + "r"(table_a) // %9 + : "memory", "cc"); +} + +void RGBColorTableRow_NEON(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + const uint8_t* table_g = table_argb + 1; + const uint8_t* table_r = table_argb + 2; + uint32_t b, g, r; + asm volatile( + "1: \n" + "ldrb %2, [%0] \n" + "ldrb %3, [%0, #1] \n" + "ldrb %4, [%0, #2] \n" + "ldrb %2, [%5, %2, lsl #2] \n" + "ldrb %3, [%6, %3, lsl #2] \n" + "ldrb %4, [%7, %4, lsl #2] \n" + "subs %1, %1, #1 \n" + "strb %2, [%0] \n" + "strb %3, [%0, #1] \n" + "strb %4, [%0, #2] \n" + "add %0, %0, #4 \n" + "bgt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width), // %1 + "=&r"(b), // %2 + "=&r"(g), // %3 + "=&r"(r) // %4 + : "r"(table_argb), // %5 + "r"(table_g), // %6 + "r"(table_r) // %7 + : "memory", "cc"); +} + +void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff) { + uint32_t idx, addr; + asm volatile( + "vdup.32 q4, %6 \n" // lumacoeff + "1: \n" + "vld1.8 {q0}, [%0]! \n" // load 4 ARGB pixels + "subs %2, %2, #4 \n" // 4 pixels per loop + "vmull.u8 q1, d0, d8 \n" + "vmull.u8 q2, d1, d9 \n" + "vpadd.i16 d2, d2, d3 \n" + "vpadd.i16 d3, d4, d5 \n" + "vpadd.i16 d2, d2, d3 \n" + "vbic.i16 d2, #0x8000 \n" // clear bit 15 + "vshrn.i16 d2, q1, #8 \n" // L_k & 0x7f + "vmov d3, d2 \n" + "vzip.8 d2, d3 \n" + "vmov d3, d2 \n" + "vzip.8 d2, d3 \n" + "vmov q2, q0 \n" + "vzip.8 q2, q1 \n" // q2: p0,1; q1: p2,3 + "vmov.u16 %3, d4[0] \n" + "add %4, %5, %3 \n" + "vld1.8 {d0[0]}, [%4] \n" + "vmov.u16 %3, d4[1] \n" + "add %4, %5, %3 \n" + "vld1.8 {d0[1]}, [%4] \n" + "vmov.u16 %3, d4[2] \n" + "add %4, %5, %3 \n" + "vld1.8 {d0[2]}, [%4] \n" + "vmov.u16 %3, d5[0] \n" + "add %4, %5, %3 \n" + "vld1.8 {d0[4]}, [%4] \n" + "vmov.u16 %3, d5[1] \n" + "add %4, %5, %3 \n" + "vld1.8 {d0[5]}, [%4] \n" + "vmov.u16 %3, d5[2] \n" + "add %4, %5, %3 \n" + "vld1.8 {d0[6]}, [%4] \n" + "vmov.u16 %3, d2[0] \n" + "add %4, %5, %3 \n" + "vld1.8 {d1[0]}, [%4] \n" + "vmov.u16 %3, d2[1] \n" + "add %4, %5, %3 \n" + "vld1.8 {d1[1]}, [%4] \n" + "vmov.u16 %3, d2[2] \n" + "add %4, %5, %3 \n" + "vld1.8 {d1[2]}, [%4] \n" + "vmov.u16 %3, d3[0] \n" + "add %4, %5, %3 \n" + "vld1.8 {d1[4]}, [%4] \n" + "vmov.u16 %3, d3[1] \n" + "add %4, %5, %3 \n" + "vld1.8 {d1[5]}, [%4] \n" + "vmov.u16 %3, d3[2] \n" + "add %4, %5, %3 \n" + "vld1.8 {d1[6]}, [%4] \n" + "vst1.8 {q0}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width), // %2 + "=&r"(idx), // %3 + "=&r"(addr) // %4 + : "r"(luma), // %5 + "r"(lumacoeff) // %6 + : "memory", "cc", "q0", "q1", "q2", "q4"); +} + +void ARGBAffineRow_NEON(const uint8_t* src_argb, + int src_argb_stride, + uint8_t* dst_argb, + const float* uv_dudv, + int width) { + uint32_t off0, off1; + asm volatile( + "vld1.32 {d4, d5}, [%5] \n" // d4=[u,v], d5=[du,dv] + "vmov d12, d5 \n" // d12=[du,dv] + "vdup.32 d10, %4 \n" // [stride, stride] + "vmov.i32 d11, #4 \n" // [4, 4] + "vtrn.32 d11, d10 \n" // q5=[4, stride, 4, + // stride] + "subs %1, %1, #4 \n" + "blt 2f \n" + "vmov d13, d12 \n" // q6=[du, dv, du, dv] + "vadd.f32 q4, q6, q6 \n" // 2 * [du, dv, du, dv] + "vadd.f32 d5, d4, d12 \n" // [u+du, v+dv] + "vadd.f32 q3, q2, q4 \n" // [u+2du, v+2dv, u+3du, + // v+3dv] + "vadd.f32 q4, q4, q4 \n" // 4 * [du, dv, du, dv] + "1: \n" + "vcvt.s32.f32 q0, q2 \n" + "vcvt.s32.f32 q1, q3 \n" + "vadd.f32 q2, q2, q4 \n" + "vadd.f32 q3, q3, q4 \n" + "vmul.i32 q0, q0, q5 \n" + "vmul.i32 q1, q1, q5 \n" + "vpadd.i32 d14, d0, d1 \n" + "vpadd.i32 d15, d2, d3 \n" + "vmov %2, %3, d14 \n" + "ldr %2, [%6, %2] \n" + "ldr %3, [%6, %3] \n" + "str %2, [%0], #4 \n" + "str %3, [%0], #4 \n" + "vmov %2, %3, d15 \n" + "ldr %2, [%6, %2] \n" + "ldr %3, [%6, %3] \n" + "str %2, [%0], #4 \n" + "str %3, [%0], #4 \n" + "subs %1, %1, #4 \n" + "bge 1b \n" + "2: \n" + "adds %1, %1, #4 \n" + "beq 4f \n" + "3: \n" + "vcvt.s32.f32 d0, d4 \n" + "vadd.f32 d4, d4, d12 \n" + "vmul.i32 d0, d0, d10 \n" + "vpadd.i32 d0, d0, d0 \n" + "vmov.32 %2, d0[0] \n" + "ldr %2, [%6, %2] \n" + "str %2, [%0], #4 \n" + "subs %1, %1, #1 \n" + "bgt 3b \n" + "4: \n" + : "+r"(dst_argb), // %0 + "+r"(width), // %1 + "=&r"(off0), // %2 + "=&r"(off1) // %3 + : "r"(src_argb_stride), // %4 + "r"(uv_dudv), // %5 + "r"(src_argb) // %6 + : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__).. #ifdef __cplusplus diff --git a/source/row_neon64.cc b/source/row_neon64.cc index f23ddcc8f..be554ae01 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -5642,6 +5642,471 @@ void Convert8To16Row_NEON(const uint8_t* src_y, : "cc", "memory", "v0", "v1", "v2"); } +void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count) { + asm volatile( + "1: \n" + "ldp q0, q1, [%0], #32 \n" + "subs %w2, %w2, #16 \n" // 16 processed per loop + "stp q0, q1, [%1], #32 \n" + "b.gt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(count) // %2 + : + : "memory", "cc", "v0", "v1"); +} + +void MirrorRow_16_NEON(const uint16_t* src, uint16_t* dst, int width) { + asm volatile( + // Start at end of source row. + "ld1 {v4.16b}, [%3] \n" // shuffler + "add %0, %0, %w2, sxtw #1 \n" + "sub %0, %0, #32 \n" + "1: \n" + "ldr q1, [%0, 16] \n" + "ldr q0, [%0], -32 \n" // src -= 32 + "subs %w2, %w2, #16 \n" // 16 pixels per loop. + "tbl v2.16b, {v1.16b}, v4.16b \n" + "tbl v3.16b, {v0.16b}, v4.16b \n" + "stp q2, q3, [%1], #32 \n" // dst += 32 + "b.gt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "r"(&kShuffleMirrorUV) // %3 + : "memory", "cc", "v0", "v1", "v2", "v3", "v4"); +} + +void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width) { + asm volatile( + "movi v16.4s, #0xff, lsl #8 \n" // 0x0000ff00 + "movi v17.4s, #0x0f, msl #16 \n" // 0x000fffff + "shl v17.4s, v17.4s, #8 \n" // 0x0fffff00 + "1: \n" + "ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels + "subs %w2, %w2, #8 \n" // 8 pixels per loop + "sri v0.4s, v0.4s, #2 \n" + "sri v1.4s, v1.4s, #2 \n" + "ushr v2.4s, v0.4s, #2 \n" + "ushr v3.4s, v1.4s, #2 \n" + "ushr v4.4s, v0.4s, #4 \n" + "ushr v5.4s, v1.4s, #4 \n" + "bit v0.16b, v4.16b, v17.16b \n" + "bit v1.16b, v5.16b, v17.16b \n" + "bit v0.16b, v2.16b, v16.16b \n" + "bit v1.16b, v3.16b, v16.16b \n" + "stp q0, q1, [%1], #32 \n" // store 8 ARGB pixels + "b.gt 1b \n" + : "+r"(src_ar30), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17"); +} + +void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) { + asm volatile( + "movi v16.2d, #0x00ffff0000ffff00 \n" + "1: \n" + "ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels + "subs %w2, %w2, #8 \n" // 8 pixels per loop + "sri v0.4s, v0.4s, #2 \n" + "sri v1.4s, v1.4s, #2 \n" + "ushr v2.4s, v0.4s, #2 \n" + "ushr v3.4s, v1.4s, #2 \n" + "sli v2.4s, v0.4s, #16 \n" + "sli v3.4s, v1.4s, #16 \n" + "sri v0.4s, v0.4s, #4 \n" + "sri v1.4s, v1.4s, #4 \n" + "sri v0.4s, v0.4s, #16 \n" + "sri v1.4s, v1.4s, #16 \n" + "bit v0.16b, v2.16b, v16.16b \n" + "bit v1.16b, v3.16b, v16.16b \n" + "stp q0, q1, [%1], #32 \n" // store 8 ABGR pixels + "b.gt 1b \n" + : "+r"(src_ar30), // %0 + "+r"(dst_abgr), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v16"); +} + +void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width) { + asm volatile( + "movi v17.16b, #0xff \n" + "ushr v17.4s, v17.4s, #22 \n" // 0x000003ff + "shl v16.4s, v17.4s, #20 \n" // 0x3ff00000 + "1: \n" + "ldp q0, q1, [%0], #32 \n" // load 8 AR30 pixels + "subs %w2, %w2, #8 \n" // 8 pixels per loop + "shl v2.4s, v0.4s, #20 \n" + "shl v3.4s, v1.4s, #20 \n" + "ushr v4.4s, v0.4s, #20 \n" + "ushr v5.4s, v1.4s, #20 \n" + "bit v0.16b, v2.16b, v16.16b \n" + "bit v1.16b, v3.16b, v16.16b \n" + "bit v0.16b, v4.16b, v17.16b \n" + "bit v1.16b, v5.16b, v17.16b \n" + "stp q0, q1, [%1], #32 \n" // store 8 AB30 pixels + "b.gt 1b \n" + : "+r"(src_ar30), // %0 + "+r"(dst_ab30), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17"); +} + +void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width) { + asm volatile( + "movi v4.4s, #0xff, lsl #24 \n" // 0xff000000 + "1: \n" + "ldp q0, q1, [%0], #32 \n" // load 8 src pixels + "ldp q2, q3, [%1] \n" // load 8 dst pixels + "subs %w2, %w2, #8 \n" // 8 pixels per loop + "bit v2.16b, v0.16b, v4.16b \n" + "bit v3.16b, v1.16b, v4.16b \n" + "stp q2, q3, [%1], #32 \n" // store 8 dst pixels + "b.gt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "v0", "v1", "v2", "v3", "v4"); +} + +void ARGBUnattenuateRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + uint64_t tmp; + asm volatile( + "movi v16.4s, #0xff, lsl #24 \n" // 0xff000000 + "1: \n" + "ldrb %w3, [%0, #3] \n" + "ldr s4, [%4, %w3, uxtw #2] \n" + "ldrb %w3, [%0, #7] \n" + "ldr s5, [%4, %w3, uxtw #2] \n" + "ldrb %w3, [%0, #11] \n" + "ldr s6, [%4, %w3, uxtw #2] \n" + "ldrb %w3, [%0, #15] \n" + "ldr s7, [%4, %w3, uxtw #2] \n" + "ldr q0, [%0], #16 \n" + "subs %w2, %w2, #4 \n" + "zip1 v1.16b, v0.16b, v0.16b \n" + "zip2 v2.16b, v0.16b, v0.16b \n" + "umull v3.4s, v1.4h, v4.h[0] \n" + "umull2 v5.4s, v1.8h, v5.h[0] \n" + "umull v6.4s, v2.4h, v6.h[0] \n" + "umull2 v7.4s, v2.8h, v7.h[0] \n" + "uzp2 v3.8h, v3.8h, v5.8h \n" + "uzp2 v6.8h, v6.8h, v7.8h \n" + "uqxtn v1.8b, v3.8h \n" + "uqxtn2 v1.16b, v6.8h \n" + "bit v1.16b, v0.16b, v16.16b \n" + "str q1, [%1], #16 \n" + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width), // %2 + "=&r"(tmp) // %3 + : "r"(fixed_invtbl8) // %4 + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16"); +} + +void ARGBPolynomialRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + const float* poly, + int width) { + asm volatile( + "ldp q16, q17, [%3] \n" // c0, c1 + "ldp q18, q19, [%3, #32] \n" // c2, c3 + "subs %w2, %w2, #4 \n" + "b.lt 2f \n" + "1: \n" + "ldr q0, [%0], #16 \n" // load 4 pixels + "subs %w2, %w2, #4 \n" + "uxtl v1.8h, v0.8b \n" + "uxtl2 v3.8h, v0.16b \n" + "uxtl v0.4s, v1.4h \n" + "uxtl2 v1.4s, v1.8h \n" + "uxtl v2.4s, v3.4h \n" + "uxtl2 v3.4s, v3.8h \n" + "ucvtf v0.4s, v0.4s \n" // x0 + "ucvtf v1.4s, v1.4s \n" // x1 + "ucvtf v2.4s, v2.4s \n" // x2 + "ucvtf v3.4s, v3.4s \n" // x3 + "fmul v20.4s, v0.4s, v0.4s \n" // x0^2 + "fmul v21.4s, v1.4s, v1.4s \n" // x1^2 + "fmul v22.4s, v2.4s, v2.4s \n" // x2^2 + "fmul v23.4s, v3.4s, v3.4s \n" // x3^2 + "fmul v4.4s, v17.4s, v0.4s \n" // c1 * x0 + "fmul v5.4s, v17.4s, v1.4s \n" // c1 * x1 + "fmul v6.4s, v17.4s, v2.4s \n" // c1 * x2 + "fmul v7.4s, v17.4s, v3.4s \n" // c1 * x3 + "fmul v0.4s, v20.4s, v0.4s \n" // x0^3 + "fmul v1.4s, v21.4s, v1.4s \n" // x1^3 + "fmul v2.4s, v22.4s, v2.4s \n" // x2^3 + "fmul v3.4s, v23.4s, v3.4s \n" // x3^3 + "fadd v4.4s, v16.4s, v4.4s \n" // c0 + c1 * x0 + "fadd v5.4s, v16.4s, v5.4s \n" // c0 + c1 * x1 + "fadd v6.4s, v16.4s, v6.4s \n" // c0 + c1 * x2 + "fadd v7.4s, v16.4s, v7.4s \n" // c0 + c1 * x3 + "fmla v4.4s, v18.4s, v20.4s \n" // + c2 * x0^2 + "fmla v5.4s, v18.4s, v21.4s \n" // + c2 * x1^2 + "fmla v6.4s, v18.4s, v22.4s \n" // + c2 * x2^2 + "fmla v7.4s, v18.4s, v23.4s \n" // + c2 * x3^2 + "fmla v4.4s, v19.4s, v0.4s \n" // + c3 * x0^3 + "fmla v5.4s, v19.4s, v1.4s \n" // + c3 * x1^3 + "fmla v6.4s, v19.4s, v2.4s \n" // + c3 * x2^3 + "fmla v7.4s, v19.4s, v3.4s \n" // + c3 * x3^3 + "fcvtzs v4.4s, v4.4s \n" + "fcvtzs v5.4s, v5.4s \n" + "fcvtzs v6.4s, v6.4s \n" + "fcvtzs v7.4s, v7.4s \n" + "sqxtn v4.4h, v4.4s \n" + "sqxtn2 v4.8h, v5.4s \n" + "sqxtn v6.4h, v6.4s \n" + "sqxtn2 v6.8h, v7.4s \n" + "sqxtun v4.8b, v4.8h \n" + "sqxtun2 v4.16b, v6.8h \n" + "str q4, [%1], #16 \n" + "b.ge 1b \n" + "2: \n" + "adds %w2, %w2, #4 \n" + "b.eq 3f \n" + "ldr d0, [%0], #8 \n" // load 2 pixels + "uxtl v0.8h, v0.8b \n" + "uxtl v1.4s, v0.4h \n" + "uxtl2 v2.4s, v0.8h \n" + "ucvtf v1.4s, v1.4s \n" // x0 + "ucvtf v2.4s, v2.4s \n" // x1 + "fmul v5.4s, v1.4s, v1.4s \n" // x0^2 + "fmul v6.4s, v2.4s, v2.4s \n" // x1^2 + "fmul v3.4s, v17.4s, v1.4s \n" // c1 * x0 + "fmul v4.4s, v17.4s, v2.4s \n" // c1 * x1 + "fmul v1.4s, v5.4s, v1.4s \n" // x0^3 + "fmul v2.4s, v6.4s, v2.4s \n" // x1^3 + "fadd v3.4s, v16.4s, v3.4s \n" // c0 + c1 * x0 + "fadd v4.4s, v16.4s, v4.4s \n" // c0 + c1 * x1 + "fmla v3.4s, v18.4s, v5.4s \n" // + c2 * x0^2 + "fmla v4.4s, v18.4s, v6.4s \n" // + c2 * x1^2 + "fmla v3.4s, v19.4s, v1.4s \n" // + c3 * x0^3 + "fmla v4.4s, v19.4s, v2.4s \n" // + c3 * x1^3 + "fcvtzs v3.4s, v3.4s \n" + "fcvtzs v4.4s, v4.4s \n" + "sqxtn v3.4h, v3.4s \n" + "sqxtn2 v3.8h, v4.4s \n" + "sqxtun v3.8b, v3.8h \n" + "str d3, [%1], #8 \n" + "3: \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(poly) // %3 + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} + +void ARGBColorTableRow_NEON(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + uint64_t b, g, r, a; + asm volatile( + "1: \n" + "ldrb %w2, [%0] \n" + "ldrb %w3, [%0, #1] \n" + "ldrb %w4, [%0, #2] \n" + "ldrb %w5, [%0, #3] \n" + "subs %w1, %w1, #1 \n" + "ldr %w2, [%6, %w2, uxtw #2] \n" + "ldr %w3, [%6, %w3, uxtw #2] \n" + "ldr %w4, [%6, %w4, uxtw #2] \n" + "ldr %w5, [%6, %w5, uxtw #2] \n" + "bfi %w3, %w2, #0, #8 \n" + "lsr %w4, %w4, #16 \n" + "lsr %w5, %w5, #24 \n" + "strh %w3, [%0] \n" + "strb %w4, [%0, #2] \n" + "strb %w5, [%0, #3] \n" + "add %0, %0, #4 \n" + "b.gt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width), // %1 + "=&r"(b), // %2 + "=&r"(g), // %3 + "=&r"(r), // %4 + "=&r"(a) // %5 + : "r"(table_argb) // %6 + : "memory", "cc"); +} + +void RGBColorTableRow_NEON(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + uint64_t b, g, r; + asm volatile( + "1: \n" + "ldrb %w2, [%0] \n" + "ldrb %w3, [%0, #1] \n" + "ldrb %w4, [%0, #2] \n" + "subs %w1, %w1, #1 \n" + "ldr %w2, [%5, %w2, uxtw #2] \n" + "ldr %w3, [%5, %w3, uxtw #2] \n" + "ldr %w4, [%5, %w4, uxtw #2] \n" + "bfi %w3, %w2, #0, #8 \n" + "lsr %w4, %w4, #16 \n" + "strh %w3, [%0] \n" + "strb %w4, [%0, #2] \n" + "add %0, %0, #4 \n" + "b.gt 1b \n" + : "+r"(dst_argb), // %0 + "+r"(width), // %1 + "=&r"(b), // %2 + "=&r"(g), // %3 + "=&r"(r) // %4 + : "r"(table_argb) // %5 + : "memory", "cc"); +} + +static const uvec8 kLumaTblIdx = {1, 1, 1, 1, 3, 3, 3, 3, + 5, 5, 5, 5, 7, 7, 7, 7}; + +void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff) { + uint64_t p0, p1, t0, t1; + asm volatile( + "dup v4.4s, %w8 \n" // broadcast 4 byte coeffs + "ldr q5, [%9] \n" + "movi v6.4s, #0xff, lsl #24 \n" + "1: \n" + "ldr q0, [%0], #16 \n" // load 4 ARGB pixels + "subs %w2, %w2, #4 \n" // 4 pixels per loop + "umull v1.8h, v0.8b, v4.8b \n" + "umull2 v2.8h, v0.16b, v4.16b \n" + "addp v1.8h, v1.8h, v2.8h \n" + "addp v1.8h, v1.8h, v1.8h \n" + "bic v1.8h, #0x80, lsl #8 \n" // clear bit 15 + "tbl v1.16b, {v1.16b}, v5.16b \n" + "zip2 v2.16b, v0.16b, v1.16b \n" // offsets pixels 2, 3 + "zip1 v1.16b, v0.16b, v1.16b \n" // offsets pixels 0, 1 + "umov %3, v1.d[0] \n" + "umov %4, v1.d[1] \n" + "add %5, %7, %w3, uxth \n" + "ubfx %6, %3, #16, #16 \n" + "ldr b3, [%5] \n" + "add %6, %7, %6 \n" + "ubfx %5, %3, #32, #16 \n" + "ld1 {v3.b}[1], [%6] \n" + "add %5, %7, %5 \n" + "add %6, %7, %w4, uxth \n" + "ld1 {v3.b}[2], [%5] \n" + "ld1 {v3.b}[4], [%6] \n" + "ubfx %5, %4, #16, #16 \n" + "ubfx %6, %4, #32, #16 \n" + "add %5, %7, %5 \n" + "add %6, %7, %6 \n" + "ld1 {v3.b}[5], [%5] \n" + "ld1 {v3.b}[6], [%6] \n" + "umov %3, v2.d[0] \n" + "umov %4, v2.d[1] \n" + "add %5, %7, %w3, uxth \n" + "ubfx %6, %3, #16, #16 \n" + "ld1 {v3.b}[8], [%5] \n" + "add %6, %7, %6 \n" + "ubfx %5, %3, #32, #16 \n" + "ld1 {v3.b}[9], [%6] \n" + "add %5, %7, %5 \n" + "add %6, %7, %w4, uxth \n" + "ld1 {v3.b}[10], [%5] \n" + "ld1 {v3.b}[12], [%6] \n" + "ubfx %5, %4, #16, #16 \n" + "ubfx %6, %4, #32, #16 \n" + "add %5, %7, %5 \n" + "add %6, %7, %6 \n" + "ld1 {v3.b}[13], [%5] \n" + "ld1 {v3.b}[14], [%6] \n" + "bit v3.16b, v0.16b, v6.16b \n" + "str q3, [%1], #16 \n" + "b.gt 1b \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width), // %2 + "=&r"(p0), // %3 + "=&r"(p1), // %4 + "=&r"(t0), // %5 + "=&r"(t1) // %6 + : "r"(luma), // %7 + "r"(lumacoeff), // %8 + "r"(&kLumaTblIdx) // %9 + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6"); +} + +void ARGBAffineRow_NEON(const uint8_t* src_argb, + int src_argb_stride, + uint8_t* dst_argb, + const float* uv_dudv, + int width) { + int64_t off0, off1; + asm volatile( + "ld1r {v2.2d}, [%2], #8 \n" // [u, v, u, v] + "ld1r {v6.2d}, [%2] \n" // [du, dv, du, dv] + "dup v5.2s, %w5 \n" // [stride, stride] + "movi v4.2s, #4 \n" // [4, 4] + "zip1 v5.4s, v4.4s, v5.4s \n" // [4, stride, 4, stride] + "subs %w1, %w1, #4 \n" + "b.lt 2f \n" + "fadd v4.4s, v6.4s, v6.4s \n" // 2 * [du, dv, du, dv] + "fadd v3.2s, v2.2s, v6.2s \n" // [u+du, v+dv] + "ins v2.d[1], v3.d[0] \n" // [u, v, u+du, v+dv] + "fadd v3.4s, v2.4s, v4.4s \n" // [u+2du, v+2dv, u+3du, + // v+3dv] + "fadd v4.4s, v4.4s, v4.4s \n" // 4 * [du, dv, du, dv] + "1: \n" + "fcvtzs v0.4s, v2.4s \n" + "fcvtzs v1.4s, v3.4s \n" + "fadd v2.4s, v2.4s, v4.4s \n" + "fadd v3.4s, v3.4s, v4.4s \n" + "mul v0.4s, v0.4s, v5.4s \n" + "mul v1.4s, v1.4s, v5.4s \n" + "addp v7.4s, v0.4s, v1.4s \n" + "smov %3, v7.s[0] \n" + "smov %4, v7.s[1] \n" + "ldr %w3, [%6, %3] \n" + "ldr %w4, [%6, %4] \n" + "stp %w3, %w4, [%0], #8 \n" + "smov %3, v7.s[2] \n" + "smov %4, v7.s[3] \n" + "ldr %w3, [%6, %3] \n" + "ldr %w4, [%6, %4] \n" + "stp %w3, %w4, [%0], #8 \n" + "subs %w1, %w1, #4 \n" + "b.ge 1b \n" + "2: \n" + "adds %w1, %w1, #4 \n" + "b.eq 4f \n" + "3: \n" + "fcvtzs v0.2s, v2.2s \n" + "fadd v2.2s, v2.2s, v6.2s \n" + "mul v0.2s, v0.2s, v5.2s \n" + "addp v0.2s, v0.2s, v0.2s \n" + "smov %3, v0.s[0] \n" + "ldr %w3, [%6, %3] \n" + "str %w3, [%0], #4 \n" + "subs %w1, %w1, #1 \n" + "b.gt 3b \n" + "4: \n" + : "+r"(dst_argb), // %0 + "+r"(width), // %1 + "+r"(uv_dudv), // %2 + "=&r"(off0), // %3 + "=&r"(off1) // %4 + : "r"(src_argb_stride), // %5 + "r"(src_argb) // %6 + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index d12be099b..1f4920fd5 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -1574,6 +1574,20 @@ TEST_F(LibYUVPlanarTest, TestAffine) { } } #endif +#if defined(HAS_ARGBAFFINEROW_NEON) + SIMD_ALIGNED(uint8_t interpolate_pixels_Opt[1280][4]); + ARGBAffineRow_NEON(&orig_pixels_0[0][0], 0, &interpolate_pixels_Opt[0][0], + uv_step, 1280); + ASSERT_EQ(0, memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1280 * 4)); + + int has_neon = TestCpuFlag(kCpuHasNEON); + if (has_neon) { + for (int i = 0; i < benchmark_pixels_div1280_; ++i) { + ARGBAffineRow_NEON(&orig_pixels_0[0][0], 0, &interpolate_pixels_Opt[0][0], + uv_step, 1280); + } + } +#endif } #endif // ENABLE_ROW_TESTS @@ -2417,7 +2431,7 @@ TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Opt)) { ASSERT_LE(max_diff, 1); } -TEST_F(LibYUVPlanarTest, DISABLED_ARM(TestARGBPolynomial)) { +TEST_F(LibYUVPlanarTest, TestARGBPolynomial) { SIMD_ALIGNED(uint8_t orig_pixels[1280][4]); SIMD_ALIGNED(uint8_t dst_pixels_opt[1280][4]); SIMD_ALIGNED(uint8_t dst_pixels_c[1280][4]);