From d2a64963531e80a329109705dc0ed2b575eba909 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Wed, 4 Nov 2026 03:09:41 -0800 Subject: [PATCH] [libyuv] Add LoongArch LASX parity, cpucfg CPUID, and 38 convert rows Follow up on the LoongArch *_la.cc consolidation with runtime CPU detection, build/test disable flags, full LASX parity for all LSX functions, and 38 new LSX/LASX convert and planar row functions: - Update LoongArchCpuCaps() in cpu_id.cc to detect LSX and LASX via the unprivileged cpucfg instruction combined with getauxval(AT_HWCAP) OS support, and wire GetCpuFlags() for __loongarch__. - Support LIBYUV_DISABLE_LSX, LIBYUV_DISABLE_LASX, and LIBYUV_DISABLE_ASM at build time in cpu_support.h and CMakeLists.txt, and fix the __loongarch__ environment variable check in unit_test.cc. - Port all 24 LSX-only Rotate and Scale kernels to 256-bit LASX so every LSX function has a matching LASX implementation. - Move ScaleMaxSamples, ScaleSumSamples, and ScaleSamples from row_* to scale_* across all platforms and add LSX and LASX implementations in scale_la.cc. - Add 38 LSX and LASX row implementations (and _Any wrappers): SwapUVRow, ARGBCopyYToAlphaRow, HalfMergeUVRow, Convert16To8Row, Convert8To16Row, MultiplyRow_16, DivideRow_16, MergeUVRow_16, SplitUVRow_16, CopyRow_16, HalfRow, HalfRow_16, ARGBCopyAlphaRow, YUY2ToNVUVRow, AR64ToARGBRow, AB64ToARGBRow, ARGBToAR64Row, ARGBToAB64Row, Convert8To8Row, HalfRow_16To8, AR30ToAB30Row, ARGBToAR30Row, ABGRToAR30Row, AR30ToARGBRow, AR30ToABGRRow, AYUVToYRow, AYUVToUVRow, AYUVToVURow, DetileRow, DetileRow_16, MergeARGBRow, MergeXRGBRow, SplitARGBRow, SplitXRGBRow, RAWToRGBARow, RGB24MirrorRow, BlendPlaneRow, and InterpolateRow_16. Compare SIMD ISA per *_C: ------------------------------------------------------------------------ C Function x86 LoongArch ------------------------------------------------------------------------ ARGBDetectRow_C AVX2 LSX+LASX HammingDistance_C SSSE3+AVX2+AVX512BW+AVX512VNNI LSX+LASX HashDjb2_C SSE41+AVX2+AVX512BW LSX+LASX Ssim4x4Row_C SSE2+AVX2+AVX512BW LSX+LASX SsimWindowRow_C SSE2+AVX2+AVX512BW LSX+LASX SumSquareError_C SSE2+AVX2+AVX512BW+AVX512VNNI LSX+LASX ------------------------------------------------------------------------ Compare: 6 (100%) 6 (100%) 6 (100%) ------------------------------------------------------------------------ Convert SIMD ISA per *_C: ------------------------------------------------------------------------ C Function x86 LoongArch ------------------------------------------------------------------------ AB64ToARGBRow_C SSSE3+AVX2 LSX+LASX ABGRToAR30Row_C SSSE3+AVX2+AVX512BW LSX+LASX AR30ToAB30Row_C AVX2+AVX512BW LSX+LASX AR30ToABGRRow_C AVX2+AVX512BW LSX+LASX AR30ToARGBRow_C AVX2+AVX512BW LSX+LASX AR64ShuffleRow_C SSSE3+AVX2+AVX512BW LSX+LASX AR64ToAB64Row_C SSSE3+AVX2+AVX512BW LSX+LASX AR64ToARGBRow_C SSSE3+AVX2 LSX+LASX ARGB1555ToARGBRow_C AVX2+AVX512BW LSX+LASX ARGB1555ToUVMatrixRow_C AVX2 ARGB1555ToUVRow_C AVX2 LSX+LASX ARGB1555ToYMatrixRow_C AVX2 ARGB1555ToYRow_C AVX2 LSX+LASX ARGB4444ToARGBRow_C AVX2 LSX+LASX ARGB4444ToUVMatrixRow_C AVX2 ARGB4444ToUVRow_C AVX2 ARGB4444ToYMatrixRow_C AVX2 ARGB4444ToYRow_C AVX2 ARGBAddRow_C SSE2+AVX2 LSX+LASX ARGBAffineRow_C AVX2 ARGBAlphaBlendRow_C SSSE3+AVX2+AVX512BW LSX+LASX ARGBAttenuateRow_C SSSE3+AVX2+AVX512BW LSX+LASX ARGBBlendRow_C AVX2+AVX512BW LSX+LASX ARGBColorMatrixRow_C AVX2+AVX512VNNI LSX+LASX ARGBColorTableRow_C AVX2 ARGBCopyAlphaRow_C SSE2+AVX2 LSX+LASX ARGBCopyYToAlphaRow_C SSE2+AVX2+AVX512BW LSX+LASX ARGBExtractAlphaRow_C SSE2+AVX2+AVX512BW LSX+LASX ARGBGrayRow_C AVX2+AVX512BW+AVX512VNNI LSX+LASX ARGBLumaColorTableRow_C AVX2 ARGBMirrorRow_C SSE2+AVX2+AVX512BW LSX+LASX ARGBMultiplyRow_C SSE2+AVX2+AVX512BW LSX+LASX ARGBOrRow_C SSE2+AVX2+AVX512BW LSX+LASX ARGBPolynomialRow_C SSE2+AVX2 ARGBQuantizeRow_C AVX2 LSX+LASX ARGBSepiaRow_C AVX2+AVX512VNNI LSX+LASX ARGBSetRow_C AVX2 LSX+LASX ARGBShadeRow_C AVX2 LSX+LASX ARGBShuffleRow_C SSSE3+AVX2+AVX512BW LSX+LASX ARGBSubtractRow_C SSE2+AVX2 LSX+LASX ARGBToAB64Row_C SSSE3+AVX2 LSX+LASX ARGBToABGRRow_C SSSE3+AVX2+AVX512BW LSX+LASX ARGBToAR30Row_C SSSE3+AVX2+AVX512BW LSX+LASX ARGBToAR64Row_C SSSE3+AVX2 LSX+LASX ARGBToARGB1555Row_C AVX2 LSX+LASX ARGBToARGB4444Row_C AVX2 LSX+LASX ARGBToBGRARow_C SSSE3+AVX2+AVX512BW LSX+LASX ARGBToRAWRow_C SSSE3+AVX2+AVX512BW LSX+LASX ARGBToRGB24Row_C SSSE3+AVX2+AVX512BW LSX+LASX ARGBToRGB565DitherRow_C AVX2+AVX512BW LSX+LASX ARGBToRGB565Row_C AVX2+AVX512BW LSX+LASX ARGBToRGBARow_C SSSE3+AVX2+AVX512BW LSX+LASX ARGBToUV444MatrixRow_C SSSE3+AVX2+AVX512BW+AVX512VNNI+AVX512VNNIVBMI ARGBToUVMatrixRow_C SSSE3+AVX2+AVX512BW+AVX512VNNI+AVX512VNNIVBMI ARGBToYMatrixRow_C SSSE3+AVX2+AVX512BW+AVX512VNNI+AVX512VNNIVBMI LSX+LASX ARGBUnattenuateRow_C SSE2+AVX2 AYUVToUVRow_C AVX2+AVX512VBMI LSX+LASX AYUVToVURow_C AVX2+AVX512VBMI LSX+LASX AYUVToYRow_C AVX2+AVX512VBMI LSX+LASX BlendPlaneRow_C SSSE3+AVX2+AVX512BW LSX+LASX ByteToFloatRow_C AVX2 ComputeCumulativeSumRow_C AVX2 Convert16To8Row_C SSSE3+AVX2+AVX512BW LSX+LASX Convert8To16Row_C SSE2+AVX2+AVX512BW LSX+LASX Convert8To8Row_C AVX2+AVX512BW LSX+LASX CopyRow_16_C AVX2 LSX+LASX CopyRow_C ERMS+SSE2+AVX+AVX512BW LSX+LASX CumulativeSumToAverageRow_C AVX2 DetileRow_16_C AVX2 LSX+LASX DetileRow_C AVX2 LSX+LASX DetileSplitUVRow_C AVX2 DetileToYUY2_C AVX2 DivideRow_16_C AVX2+AVX512BW LSX+LASX GaussCol_C AVX2 GaussCol_F32_C AVX2 GaussRow_C AVX2 GaussRow_F32_C AVX2 HalfFloatRow_C F16C+AVX2+AVX512BW LSX+LASX HalfMergeUVRow_C SSSE3+AVX2+AVX512BW LSX+LASX HalfRow_16To8_C SSSE3+AVX2+AVX512BW LSX+LASX HalfRow_16_C AVX2 LSX+LASX HalfRow_C AVX2 LSX+LASX HalfWidthRow_16To8_C SSSE3+AVX2+AVX512BW HalfWidthRow_16To8_Odd_C AVX2 I210AlphaToARGBRow_C SSSE3+AVX2+AVX512BW I210ToAR30Row_C SSSE3+AVX2+AVX512BW I210ToARGBRow_C SSSE3+AVX2+AVX512BW I212ToAR30Row_C SSSE3+AVX2+AVX512BW I212ToARGBRow_C SSSE3+AVX2+AVX512BW I400ToARGBRow_C SSE2+AVX2+AVX512BW LSX+LASX I410AlphaToARGBRow_C SSSE3+AVX2+AVX512BW I410ToAR30Row_C SSSE3+AVX2+AVX512BW I410ToARGBRow_C SSSE3+AVX2+AVX512BW I422AlphaToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX I422ToAR30Row_C SSSE3+AVX2+AVX512BW I422ToARGB1555Row_C SSSE3+AVX2+AVX512BW LSX+LASX I422ToARGB4444Row_C SSSE3+AVX2+AVX512BW LSX+LASX I422ToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX I422ToRGB24Row_C SSSE3+AVX2+AVX512BW+AVX512VBMI LSX+LASX I422ToRGB565Row_C SSSE3+AVX2+AVX512BW LSX+LASX I422ToRGBARow_C SSSE3+AVX2+AVX512BW LSX+LASX I422ToUYVYRow_C SSE2+AVX2 LSX+LASX I422ToYUY2Row_C SSE2+AVX2 LSX+LASX I444AlphaToARGBRow_C SSSE3+AVX2+AVX512BW I444ToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX I444ToRGB24Row_C SSSE3+AVX2+AVX512VBMI InterpolateRow_16_C AVX2+AVX512BW LSX+LASX InterpolateRow_C AVX2+AVX512BW LSX+LASX J400ToARGBRow_C AVX2+AVX512BW LSX+LASX MergeAR64Row_C AVX2 MergeARGB16To8Row_C AVX2 MergeARGBRow_C SSE2+AVX2 LSX+LASX MergeRGBRow_C AVX2+AVX512VBMI MergeUVRow_16_C AVX2+AVX512BW LSX+LASX MergeUVRow_C SSE2+AVX2+AVX512BW LSX+LASX MergeXR30Row_C AVX2 MergeXR64Row_C AVX2 MergeXRGB16To8Row_C AVX2 MergeXRGBRow_C SSE2+AVX2 LSX+LASX MirrorRow_16_C SSSE3+AVX2+AVX512BW LSX+LASX MirrorRow_C SSSE3+AVX2+AVX512BW LSX+LASX MirrorSplitUVRow_C SSSE3+AVX2+AVX512BW LSX+LASX MirrorUVRow_C SSSE3+AVX2+AVX512BW LSX+LASX MultiplyRow_16_C AVX2+AVX512BW LSX+LASX NV12ToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX NV12ToRGB24Row_C SSSE3+AVX2+AVX512VBMI NV12ToRGB565Row_C SSSE3+AVX2 LSX+LASX NV21ToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX NV21ToRGB24Row_C SSSE3+AVX2+AVX512VBMI NV21ToYUV24Row_C SSSE3+AVX2+AVX512VBMI P210ToAR30Row_C SSSE3+AVX2+AVX512BW P210ToARGBRow_C SSSE3+AVX2+AVX512BW P410ToAR30Row_C SSSE3+AVX2+AVX512BW P410ToARGBRow_C SSSE3+AVX2+AVX512BW RAWToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX RAWToRGB24Row_C SSSE3+AVX2+AVX512BW+AVX512VBMI LSX+LASX RAWToRGBARow_C AVX2+AVX512VBMI LSX+LASX RAWToUVRow LSX+LASX RGB24MirrorRow_C SSSE3+AVX2+AVX512VBMI LSX+LASX RGB24ToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX RGB24ToUVRow LSX+LASX RGB565ToARGBRow_C AVX2 LSX+LASX RGB565ToUVMatrixRow_C AVX2 RGB565ToUVRow_C AVX2 LSX+LASX RGB565ToYMatrixRow_C AVX2 RGB565ToYRow_C AVX2 LSX+LASX RGBAToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX RGBColorTableRow_C AVX2 RGBToUV444MatrixRow_C SSSE3+AVX2+AVX512BW RGBToUVMatrixRow_C SSSE3+AVX2+AVX512BW RGBToYMatrixRow_C SSSE3+AVX2+AVX512BW LSX+LASX SetRow_C ERMS+AVX2 LSX+LASX SobelRow_C AVX2 LSX+LASX SobelToPlaneRow_C AVX2 LSX+LASX SobelXRow_C AVX2 SobelXYRow_C AVX2 LSX+LASX SobelYRow_C AVX2 SplitARGBRow_C SSE2+SSSE3+AVX2 LSX+LASX SplitRGBRow_C SSSE3+SSE41+AVX2 SplitUVRow_16_C AVX2 LSX+LASX SplitUVRow_C SSE2+AVX2+AVX512BW LSX+LASX SplitXRGBRow_C SSE2+SSSE3+AVX2 LSX+LASX SwapUVRow_C SSSE3+AVX2+AVX512BW LSX+LASX UYVYToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX UYVYToUV422Row_C SSE2+AVX2+AVX512BW LSX+LASX UYVYToUVRow_C SSE2+AVX2+AVX512BW LSX+LASX UYVYToYRow_C SSE2+AVX2+AVX512BW LSX+LASX UnpackMT2T_C AVX2 YUY2ToARGBRow_C SSSE3+AVX2+AVX512BW LSX+LASX YUY2ToNVUVRow_C SSE2+AVX2 LSX+LASX YUY2ToUV422Row_C SSE2+AVX2+AVX512BW LSX+LASX YUY2ToUVRow_C SSE2+AVX2+AVX512BW LSX+LASX YUY2ToYRow_C SSE2+AVX2+AVX512BW LSX+LASX ------------------------------------------------------------------------ Convert: 173 (100%) 171 (99%) 116 (67%) ------------------------------------------------------------------------ Rotate SIMD ISA per *_C: ------------------------------------------------------------------------ C Function x86 LoongArch ------------------------------------------------------------------------ TransposeUVWx16_C AVX2+AVX512BW LSX+LASX TransposeUVWx8_C AVX2 LSX+LASX TransposeWx16_C AVX2+AVX512BW LSX+LASX TransposeWx4_32_C AVX2 LSX+LASX TransposeWx8_16_C AVX2 LSX+LASX TransposeWx8_C AVX2 LSX+LASX TransposeWxH_32_C AVX2 ------------------------------------------------------------------------ Rotate: 7 (100%) 7 (100%) 6 (86%) ------------------------------------------------------------------------ Scale SIMD ISA per *_C: ------------------------------------------------------------------------ C Function x86 LoongArch ------------------------------------------------------------------------ FixedDiv1_C X86 FixedDiv_C X86 ScaleARGBCols64_C ScaleARGBColsUp2_C AVX2 ScaleARGBCols_C AVX2 LSX+LASX ScaleARGBFilterCols64_C ScaleARGBFilterCols_C AVX2 LSX+LASX ScaleARGBRowDown2Box_C AVX2 LSX+LASX ScaleARGBRowDown2Linear_C AVX2 LSX+LASX ScaleARGBRowDown2_C AVX2 LSX+LASX ScaleARGBRowDownEvenBox_C AVX2 LSX+LASX ScaleARGBRowDownEven_C AVX2 LSX+LASX ScaleAddCols0_C ScaleAddCols1_16_C ScaleAddCols1_C AVX512BW ScaleAddCols2_16_C ScaleAddCols2_C AVX512BW ScaleAddRow_16_C ScaleAddRow_C SSE2+AVX2+AVX512BW LSX+LASX ScaleColsUp2_16_C ScaleColsUp2_C AVX2 ScaleCols_16_C ScaleCols_C ScaleFilterCols64_16_C ScaleFilterCols64_C ScaleFilterCols_16_C ScaleFilterCols_C AVX2 LSX+LASX ScaleMaxSamples_C AVX2 LSX+LASX ScaleRowDown2Box_16_C AVX2 ScaleRowDown2Box_C SSSE3+AVX2+AVX512BW LSX+LASX ScaleRowDown2Box_Odd_C SSSE3+AVX2+AVX512BW ScaleRowDown2Linear_16_C AVX2 ScaleRowDown2Linear_C SSSE3+AVX2+AVX512BW LSX+LASX ScaleRowDown2_16_C AVX2 ScaleRowDown2_C SSSE3+AVX2+AVX512BW LSX+LASX ScaleRowDown34_0_Box_16_C ScaleRowDown34_0_Box_C AVX2 LSX+LASX ScaleRowDown34_16_C ScaleRowDown34_1_Box_16_C ScaleRowDown34_1_Box_C AVX2 LSX+LASX ScaleRowDown34_C AVX2 LSX+LASX ScaleRowDown38_16_C ScaleRowDown38_2_Box_16_C ScaleRowDown38_2_Box_C AVX2 LSX+LASX ScaleRowDown38_3_Box_16_C ScaleRowDown38_3_Box_C AVX2 LSX+LASX ScaleRowDown38_C AVX2 LSX+LASX ScaleRowDown4Box_16_C ScaleRowDown4Box_C SSSE3+AVX2 LSX+LASX ScaleRowDown4_16_C ScaleRowDown4_C SSSE3+AVX2 LSX+LASX ScaleRowUp2_Bilinear_16_Any_C SSE2+AVX2 ScaleRowUp2_Bilinear_16_C SSE2+AVX2 ScaleRowUp2_Bilinear_Any_C SSE2+SSSE3+AVX2 ScaleRowUp2_Bilinear_C SSE2+SSSE3+AVX2 ScaleRowUp2_Linear_16_Any_C SSE2+AVX2 ScaleRowUp2_Linear_16_C SSE2+AVX2 ScaleRowUp2_Linear_Any_C SSE2+SSSE3+AVX2 ScaleRowUp2_Linear_C SSE2+SSSE3+AVX2 ScaleSamples_C AVX2 LSX+LASX ScaleSumSamples_C AVX2 LSX+LASX ScaleUVAddCols1_16_C ScaleUVAddCols1_C ScaleUVAddCols2_16_C ScaleUVAddCols2_C ScaleUVAddCols3_16_C ScaleUVAddCols3_C AVX2+AVX512VBMI LSX+LASX ScaleUVCols64_16_C ScaleUVCols64_C ScaleUVColsUp2_16_C ScaleUVColsUp2_C ScaleUVCols_16_C ScaleUVCols_C ScaleUVFilterCols64_16_C ScaleUVFilterCols64_C ScaleUVFilterCols_16_C ScaleUVFilterCols_C ScaleUVRowDown2Box_16_C ScaleUVRowDown2Box_C AVX2 ScaleUVRowDown2Linear_16_C ScaleUVRowDown2Linear_C AVX2 ScaleUVRowDown2_16_C ScaleUVRowDown2_C AVX2 ScaleUVRowDownEvenBox_16_C ScaleUVRowDownEvenBox_C ScaleUVRowDownEven_16_C ScaleUVRowDownEven_C AVX2 ScaleUVRowUp2_Bilinear_16_Any_C SSE41+AVX2 ScaleUVRowUp2_Bilinear_16_C SSE41+AVX2 ScaleUVRowUp2_Bilinear_Any_C SSSE3+AVX2 ScaleUVRowUp2_Bilinear_C SSSE3+AVX2 ScaleUVRowUp2_Linear_16_Any_C SSE41+AVX2 ScaleUVRowUp2_Linear_16_C SSE41+AVX2 ScaleUVRowUp2_Linear_Any_C SSSE3+AVX2 ScaleUVRowUp2_Linear_C SSSE3+AVX2 ------------------------------------------------------------------------ Scale: 95 (100%) 54 (57%) 24 (25%) ------------------------------------------------------------------------ Bug: 42280942 Change-Id: I663f277db5e8a6a73319df7e3153f9f78a147a7a Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8718861 Reviewed-by: richard winterton Commit-Queue: Frank Barchard --- CMakeLists.txt | 15 +- include/libyuv/cpu_support.h | 37 +- include/libyuv/rotate_row.h | 37 + include/libyuv/row.h | 631 ++++++++- include/libyuv/scale_row.h | 234 ++++ source/convert.cc | 168 +++ source/convert_argb.cc | 96 ++ source/convert_from.cc | 32 + source/convert_from_argb.cc | 160 +++ source/cpu_id.cc | 46 +- source/planar_functions.cc | 383 ++++++ source/rotate.cc | 24 + source/rotate_any.cc | 9 + source/rotate_la.cc | 398 ++++++ source/row_any.cc | 290 ++++- source/row_common.cc | 30 - source/row_gcc.cc | 155 --- source/row_la.cc | 2324 +++++++++++++++++++++++++++++++++- source/row_neon64.cc | 78 -- source/row_win.cc | 84 -- source/scale.cc | 137 ++ source/scale_any.cc | 118 ++ source/scale_argb.cc | 74 ++ source/scale_common.cc | 46 + source/scale_gcc.cc | 155 +++ source/scale_la.cc | 1292 ++++++++++++++++++- source/scale_neon64.cc | 78 ++ source/scale_uv.cc | 48 + source/scale_win.cc | 84 ++ unit_test/cpu_test.cc | 13 + unit_test/planar_test.cc | 96 +- unit_test/unit_test.cc | 2 +- 32 files changed, 6973 insertions(+), 401 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc9c162a9..d608e9e44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,19 @@ cmake_minimum_required( VERSION 3.16 ) project ( YUV C CXX ) # "C" is required even for C++ projects option( UNIT_TEST "Built unit tests" OFF ) option( LIBYUV_DISABLE_JPEG "Disable JPEG support" OFF ) +option( LIBYUV_DISABLE_ASM "Disable inline assembly" OFF ) +option( LIBYUV_DISABLE_LSX "Disable LoongArch LSX" OFF ) +option( LIBYUV_DISABLE_LASX "Disable LoongArch LASX" OFF ) + +if(LIBYUV_DISABLE_ASM) + add_definitions(-DLIBYUV_DISABLE_ASM) +endif() +if(LIBYUV_DISABLE_LSX) + add_definitions(-DLIBYUV_DISABLE_LSX) +endif() +if(LIBYUV_DISABLE_LASX) + add_definitions(-DLIBYUV_DISABLE_LASX) +endif() include(CheckCSourceCompiles) @@ -89,7 +102,7 @@ add_library( ${ly_lib_name}_common_objects OBJECT ${ly_common_source_files} ) set(ly_lib_parts $) string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" SYSPROC) -set(LOONGARCH64_ALIASES loongarch64) +set(LOONGARCH64_ALIASES loongarch64 loong64) list(FIND LOONGARCH64_ALIASES "${SYSPROC}" LOONGARCH64MATCH) if(LOONGARCH64MATCH GREATER "-1") diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index d35276f9d..c0e345f3e 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -41,12 +41,47 @@ extern "C" { #if !defined(LIBYUV_DISABLE_SVE) #define LIBYUV_DISABLE_SVE #endif +#if !defined(LIBYUV_DISABLE_RVV) +#define LIBYUV_DISABLE_RVV +#endif +#if !defined(LIBYUV_DISABLE_LSX) +#define LIBYUV_DISABLE_LSX +#endif +#if !defined(LIBYUV_DISABLE_LASX) +#define LIBYUV_DISABLE_LASX +#endif #if !defined(LIBYUV_ENABLE_ROWWIN) #define LIBYUV_ENABLE_ROWWIN #endif #endif // __has_feature(memory_sanitizer) #endif // defined(__has_feature) +// Disable all platforms that use inline assembly when LIBYUV_DISABLE_ASM is +// defined at build time. +#if defined(LIBYUV_DISABLE_ASM) +#if !defined(LIBYUV_DISABLE_X86) +#define LIBYUV_DISABLE_X86 +#endif +#if !defined(LIBYUV_DISABLE_NEON) +#define LIBYUV_DISABLE_NEON +#endif +#if !defined(LIBYUV_DISABLE_SVE) +#define LIBYUV_DISABLE_SVE +#endif +#if !defined(LIBYUV_DISABLE_SME) +#define LIBYUV_DISABLE_SME +#endif +#if !defined(LIBYUV_DISABLE_RVV) +#define LIBYUV_DISABLE_RVV +#endif +#if !defined(LIBYUV_DISABLE_LSX) +#define LIBYUV_DISABLE_LSX +#endif +#if !defined(LIBYUV_DISABLE_LASX) +#define LIBYUV_DISABLE_LASX +#endif +#endif // defined(LIBYUV_DISABLE_ASM) + // clang >= 3.5.0 required for Arm64. #if defined(__clang__) && defined(__aarch64__) && !defined(LIBYUV_DISABLE_NEON) #if (__clang_major__ < 3) || (__clang_major__ == 3 && (__clang_minor__ < 5)) @@ -146,7 +181,7 @@ extern "C" { #define VISUALC_HAS_AVX512 1 #endif // VisualStudio >= 2019 -#if defined(LIBYUV_ENABLE_SME) +#if defined(LIBYUV_ENABLE_SME) && !defined(LIBYUV_DISABLE_ASM) #undef LIBYUV_DISABLE_SME #endif diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index b6bd2ff29..06544a5a1 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -121,8 +121,11 @@ extern "C" { #endif #if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_lp64) +#define HAS_TRANSPOSEWX8_LASX #define HAS_TRANSPOSEWX16_LASX +#define HAS_TRANSPOSEUVWX8_LASX #define HAS_TRANSPOSEUVWX16_LASX +#define HAS_TRANSPOSEWX8_16_LASX #define HAS_TRANSPOSEWX16_16_LASX #define HAS_TRANSPOSEWX4_32_LASX #endif @@ -207,6 +210,11 @@ void TransposeWx8_LSX(const uint8_t* src, uint8_t* dst, int dst_stride, int width); +void TransposeWx8_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx16_LASX(const uint8_t* src, int src_stride, uint8_t* dst, @@ -248,6 +256,11 @@ void TransposeWx8_Any_LSX(const uint8_t* src, uint8_t* dst, int dst_stride, int width); +void TransposeWx8_Any_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx16_Any_LASX(const uint8_t* src, int src_stride, uint8_t* dst, @@ -337,6 +350,13 @@ void TransposeUVWx8_LSX(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); +void TransposeUVWx8_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); void TransposeUVWx16_LASX(const uint8_t* src, int src_stride, uint8_t* dst_a, @@ -387,6 +407,13 @@ void TransposeUVWx8_Any_LSX(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); +void TransposeUVWx8_Any_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); void TransposeUVWx16_Any_LASX(const uint8_t* src, int src_stride, uint8_t* dst_a, @@ -468,6 +495,11 @@ void TransposeWx8_16_LSX(const uint16_t* src, uint16_t* dst, int dst_stride, int width); +void TransposeWx8_16_LASX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); void TransposeWx16_16_LASX(const uint16_t* src, int src_stride, uint16_t* dst, @@ -498,6 +530,11 @@ void TransposeWx8_16_Any_LSX(const uint16_t* src, uint16_t* dst, int dst_stride, int width); +void TransposeWx8_16_Any_LASX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); void TransposeWx16_16_Any_LASX(const uint16_t* src, int src_stride, uint16_t* dst, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 2c98e3c3d..6c652fbb6 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -280,9 +280,6 @@ extern "C" { #define HAS_RGB565TOUVROW_AVX2 #define HAS_RGB565TOYROW_AVX2 #define HAS_RGBCOLORTABLEROW_AVX2 -#define HAS_SCALEMAXSAMPLES_AVX2 -#define HAS_SCALESAMPLES_AVX2 -#define HAS_SCALESUMSAMPLES_AVX2 #define HAS_SETROW_AVX2 #define HAS_SOBELROW_AVX2 #define HAS_SOBELTOPLANEROW_AVX2 @@ -455,9 +452,6 @@ extern "C" { #define HAS_NV21TORGB24ROW_AVX2 #define HAS_NV21TOYUV24ROW_AVX2 #define HAS_RAWTORGBAROW_AVX2 -#define HAS_SCALEMAXSAMPLES_AVX2 -#define HAS_SCALESAMPLES_AVX2 -#define HAS_SCALESUMSAMPLES_AVX2 #define HAS_SPLITARGBROW_AVX2 #define HAS_SPLITRGBROW_AVX2 #define HAS_SPLITUVROW_16_AVX2 @@ -1022,12 +1016,15 @@ extern "C" { #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #define HAS_GAUSSCOL_F32_NEON #define HAS_GAUSSROW_F32_NEON -#define HAS_SCALEMAXSAMPLES_NEON -#define HAS_SCALESAMPLES_NEON -#define HAS_SCALESUMSAMPLES_NEON #endif #if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_lp64) +#define HAS_AB64TOARGBROW_LSX +#define HAS_ABGRTOAR30ROW_LSX +#define HAS_AR30TOAB30ROW_LSX +#define HAS_AR30TOABGRROW_LSX +#define HAS_AR30TOARGBROW_LSX +#define HAS_AR64TOARGBROW_LSX #define HAS_ARGB1555TOARGBROW_LSX #define HAS_ARGB1555TOUVROW_LSX #define HAS_ARGB1555TOYROW_LSX @@ -1037,6 +1034,8 @@ extern "C" { #define HAS_ARGBATTENUATEROW_LSX #define HAS_ARGBBLENDROW_LSX #define HAS_ARGBCOLORMATRIXROW_LSX +#define HAS_ARGBCOPYALPHAROW_LSX +#define HAS_ARGBCOPYYTOALPHAROW_LSX #define HAS_ARGBEXTRACTALPHAROW_LSX #define HAS_ARGBGRAYROW_LSX #define HAS_ARGBMIRRORROW_LSX @@ -1048,6 +1047,9 @@ extern "C" { #define HAS_ARGBSHADEROW_LSX #define HAS_ARGBSHUFFLEROW_LSX #define HAS_ARGBSUBTRACTROW_LSX +#define HAS_ARGBTOAB64ROW_LSX +#define HAS_ARGBTOAR30ROW_LSX +#define HAS_ARGBTOAR64ROW_LSX #define HAS_ARGBTOARGB1555ROW_LSX #define HAS_ARGBTOARGB4444ROW_LSX #define HAS_ARGBTORAWROW_LSX @@ -1055,8 +1057,23 @@ extern "C" { #define HAS_ARGBTORGB565DITHERROW_LSX #define HAS_ARGBTORGB565ROW_LSX #define HAS_ARGBTOYMATRIXROW_LSX +#define HAS_AYUVTOUVROW_LSX +#define HAS_AYUVTOVUROW_LSX +#define HAS_AYUVTOYROW_LSX +#define HAS_BLENDPLANEROW_LSX +#define HAS_CONVERT16TO8ROW_LSX +#define HAS_CONVERT8TO16ROW_LSX +#define HAS_CONVERT8TO8ROW_LSX +#define HAS_COPYROW_16_LSX #define HAS_COPYROW_LSX +#define HAS_DETILEROW_16_LSX +#define HAS_DETILEROW_LSX +#define HAS_DIVIDEROW_16_LSX #define HAS_HALFFLOATROW_LSX +#define HAS_HALFMERGEUVROW_LSX +#define HAS_HALFROW_16_LSX +#define HAS_HALFROW_16TO8_LSX +#define HAS_HALFROW_LSX #define HAS_I400TOARGBROW_LSX #define HAS_I422ALPHATOARGBROW_LSX #define HAS_I422TOARGB1555ROW_LSX @@ -1068,18 +1085,25 @@ extern "C" { #define HAS_I422TOUYVYROW_LSX #define HAS_I422TOYUY2ROW_LSX #define HAS_I444TOARGBROW_LSX +#define HAS_INTERPOLATEROW_16_LSX #define HAS_INTERPOLATEROW_LSX #define HAS_J400TOARGBROW_LSX +#define HAS_MERGEARGBROW_LSX +#define HAS_MERGEUVROW_16_LSX #define HAS_MERGEUVROW_LSX +#define HAS_MERGEXRGBROW_LSX #define HAS_MIRRORROW_LSX #define HAS_MIRRORSPLITUVROW_LSX #define HAS_MIRRORUVROW_LSX +#define HAS_MULTIPLYROW_16_LSX #define HAS_NV12TOARGBROW_LSX #define HAS_NV12TORGB565ROW_LSX #define HAS_NV21TOARGBROW_LSX #define HAS_RAWTOARGBROW_LSX #define HAS_RAWTORGB24ROW_LSX +#define HAS_RAWTORGBAROW_LSX #define HAS_RAWTOUVROW_LSX +#define HAS_RGB24MIRRORROW_LSX #define HAS_RGB24TOARGBROW_LSX #define HAS_RGB24TOUVROW_LSX #define HAS_RGB565TOARGBROW_LSX @@ -1090,18 +1114,29 @@ extern "C" { #define HAS_SOBELROW_LSX #define HAS_SOBELTOPLANEROW_LSX #define HAS_SOBELXYROW_LSX +#define HAS_SPLITARGBROW_LSX +#define HAS_SPLITUVROW_16_LSX #define HAS_SPLITUVROW_LSX +#define HAS_SPLITXRGBROW_LSX +#define HAS_SWAPUVROW_LSX #define HAS_UYVYTOARGBROW_LSX #define HAS_UYVYTOUV422ROW_LSX #define HAS_UYVYTOUVROW_LSX #define HAS_UYVYTOYROW_LSX #define HAS_YUY2TOARGBROW_LSX +#define HAS_YUY2TONVUVROW_LSX #define HAS_YUY2TOUV422ROW_LSX #define HAS_YUY2TOUVROW_LSX #define HAS_YUY2TOYROW_LSX #endif #if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_lp64) +#define HAS_AB64TOARGBROW_LASX +#define HAS_ABGRTOAR30ROW_LASX +#define HAS_AR30TOAB30ROW_LASX +#define HAS_AR30TOABGRROW_LASX +#define HAS_AR30TOARGBROW_LASX +#define HAS_AR64TOARGBROW_LASX #define HAS_ARGB1555TOARGBROW_LASX #define HAS_ARGB1555TOUVROW_LASX #define HAS_ARGB1555TOYROW_LASX @@ -1111,6 +1146,8 @@ extern "C" { #define HAS_ARGBATTENUATEROW_LASX #define HAS_ARGBBLENDROW_LASX #define HAS_ARGBCOLORMATRIXROW_LASX +#define HAS_ARGBCOPYALPHAROW_LASX +#define HAS_ARGBCOPYYTOALPHAROW_LASX #define HAS_ARGBEXTRACTALPHAROW_LASX #define HAS_ARGBGRAYROW_LASX #define HAS_ARGBMIRRORROW_LASX @@ -1122,6 +1159,9 @@ extern "C" { #define HAS_ARGBSHADEROW_LASX #define HAS_ARGBSHUFFLEROW_LASX #define HAS_ARGBSUBTRACTROW_LASX +#define HAS_ARGBTOAB64ROW_LASX +#define HAS_ARGBTOAR30ROW_LASX +#define HAS_ARGBTOAR64ROW_LASX #define HAS_ARGBTOARGB1555ROW_LASX #define HAS_ARGBTOARGB4444ROW_LASX #define HAS_ARGBTORAWROW_LASX @@ -1129,8 +1169,23 @@ extern "C" { #define HAS_ARGBTORGB565DITHERROW_LASX #define HAS_ARGBTORGB565ROW_LASX #define HAS_ARGBTOYMATRIXROW_LASX +#define HAS_AYUVTOUVROW_LASX +#define HAS_AYUVTOVUROW_LASX +#define HAS_AYUVTOYROW_LASX +#define HAS_BLENDPLANEROW_LASX +#define HAS_CONVERT16TO8ROW_LASX +#define HAS_CONVERT8TO16ROW_LASX +#define HAS_CONVERT8TO8ROW_LASX +#define HAS_COPYROW_16_LASX #define HAS_COPYROW_LASX +#define HAS_DETILEROW_16_LASX +#define HAS_DETILEROW_LASX +#define HAS_DIVIDEROW_16_LASX #define HAS_HALFFLOATROW_LASX +#define HAS_HALFMERGEUVROW_LASX +#define HAS_HALFROW_16_LASX +#define HAS_HALFROW_16TO8_LASX +#define HAS_HALFROW_LASX #define HAS_I400TOARGBROW_LASX #define HAS_I422ALPHATOARGBROW_LASX #define HAS_I422TOARGB1555ROW_LASX @@ -1142,18 +1197,25 @@ extern "C" { #define HAS_I422TOUYVYROW_LASX #define HAS_I422TOYUY2ROW_LASX #define HAS_I444TOARGBROW_LASX +#define HAS_INTERPOLATEROW_16_LASX #define HAS_INTERPOLATEROW_LASX #define HAS_J400TOARGBROW_LASX +#define HAS_MERGEARGBROW_LASX +#define HAS_MERGEUVROW_16_LASX #define HAS_MERGEUVROW_LASX +#define HAS_MERGEXRGBROW_LASX #define HAS_MIRRORROW_LASX #define HAS_MIRRORSPLITUVROW_LASX #define HAS_MIRRORUVROW_LASX +#define HAS_MULTIPLYROW_16_LASX #define HAS_NV12TOARGBROW_LASX #define HAS_NV12TORGB565ROW_LASX #define HAS_NV21TOARGBROW_LASX #define HAS_RAWTOARGBROW_LASX #define HAS_RAWTORGB24ROW_LASX +#define HAS_RAWTORGBAROW_LASX #define HAS_RAWTOUVROW_LASX +#define HAS_RGB24MIRRORROW_LASX #define HAS_RGB24TOARGBROW_LASX #define HAS_RGB24TOUVROW_LASX #define HAS_RGB565TOARGBROW_LASX @@ -1164,12 +1226,17 @@ extern "C" { #define HAS_SOBELROW_LASX #define HAS_SOBELTOPLANEROW_LASX #define HAS_SOBELXYROW_LASX +#define HAS_SPLITARGBROW_LASX +#define HAS_SPLITUVROW_16_LASX #define HAS_SPLITUVROW_LASX +#define HAS_SPLITXRGBROW_LASX +#define HAS_SWAPUVROW_LASX #define HAS_UYVYTOARGBROW_LASX #define HAS_UYVYTOUV422ROW_LASX #define HAS_UYVYTOUVROW_LASX #define HAS_UYVYTOYROW_LASX #define HAS_YUY2TOARGBROW_LASX +#define HAS_YUY2TONVUVROW_LASX #define HAS_YUY2TOUV422ROW_LASX #define HAS_YUY2TOUVROW_LASX #define HAS_YUY2TOYROW_LASX @@ -3102,6 +3169,12 @@ void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, void RGB24MirrorRow_NEON(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width); +void RGB24MirrorRow_LSX(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width); +void RGB24MirrorRow_LASX(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width); void RGB24MirrorRow_C(const uint8_t* src_rgb24, uint8_t* dst_rgb24, int width); void RGB24MirrorRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -3112,6 +3185,12 @@ void RGB24MirrorRow_Any_AVX2(const uint8_t* src_ptr, void RGB24MirrorRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void RGB24MirrorRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void RGB24MirrorRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void SplitUVRow_C(const uint8_t* src_uv, uint8_t* dst_u, @@ -3193,6 +3272,22 @@ void DetileRow_Any_AVX2(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, int width); +void DetileRow_LSX(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width); +void DetileRow_Any_LSX(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width); +void DetileRow_LASX(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width); +void DetileRow_Any_LASX(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width); void DetileRow_16_C(const uint16_t* src, ptrdiff_t src_tile_stride, uint16_t* dst, @@ -3213,6 +3308,22 @@ void DetileRow_16_Any_AVX2(const uint16_t* src, ptrdiff_t src_tile_stride, uint16_t* dst, int width); +void DetileRow_16_LSX(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width); +void DetileRow_16_Any_LSX(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width); +void DetileRow_16_LASX(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width); +void DetileRow_16_Any_LASX(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width); void DetileSplitUVRow_C(const uint8_t* src_uv, ptrdiff_t src_tile_stride, uint8_t* dst_u, @@ -3396,6 +3507,30 @@ void HalfMergeUVRow_Any_NEON(const uint8_t* src_u, int src_stride_v, uint8_t* dst_uv, int width); +void HalfMergeUVRow_LSX(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); +void HalfMergeUVRow_Any_LSX(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); +void HalfMergeUVRow_LASX(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); +void HalfMergeUVRow_Any_LASX(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); void SplitRGBRow_C(const uint8_t* src_rgb, uint8_t* dst_r, @@ -3513,6 +3648,18 @@ void MergeARGBRow_RVV(const uint8_t* src_r, const uint8_t* src_a, uint8_t* dst_argb, int width); +void MergeARGBRow_LSX(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + const uint8_t* src_a, + uint8_t* dst_argb, + int width); +void MergeARGBRow_LASX(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + const uint8_t* src_a, + uint8_t* dst_argb, + int width); void MergeARGBRow_Any_SSE2(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -3531,6 +3678,18 @@ void MergeARGBRow_Any_NEON(const uint8_t* y_buf, const uint8_t* a_buf, uint8_t* dst_ptr, int width); +void MergeARGBRow_Any_LSX(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_ptr, + int width); +void MergeARGBRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_ptr, + int width); void SplitARGBRow_C(const uint8_t* src_argb, uint8_t* dst_r, uint8_t* dst_g, @@ -3567,6 +3726,18 @@ void SplitARGBRow_RVV(const uint8_t* src_rgba, uint8_t* dst_b, uint8_t* dst_a, int width); +void SplitARGBRow_LSX(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + uint8_t* dst_a, + int width); +void SplitARGBRow_LASX(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + uint8_t* dst_a, + int width); void SplitARGBRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_r, uint8_t* dst_g, @@ -3591,6 +3762,18 @@ void SplitARGBRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_b, uint8_t* dst_a, int width); +void SplitARGBRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + uint8_t* dst_a, + int width); +void SplitARGBRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + uint8_t* dst_a, + int width); void MergeXRGBRow_C(const uint8_t* src_r, const uint8_t* src_g, const uint8_t* src_b, @@ -3616,6 +3799,16 @@ void MergeXRGBRow_RVV(const uint8_t* src_r, const uint8_t* src_b, uint8_t* dst_argb, int width); +void MergeXRGBRow_LSX(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_argb, + int width); +void MergeXRGBRow_LASX(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_argb, + int width); void MergeXRGBRow_Any_SSE2(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -3631,6 +3824,16 @@ void MergeXRGBRow_Any_NEON(const uint8_t* y_buf, const uint8_t* v_buf, uint8_t* dst_ptr, int width); +void MergeXRGBRow_Any_LSX(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + int width); +void MergeXRGBRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + int width); void SplitXRGBRow_C(const uint8_t* src_argb, uint8_t* dst_r, uint8_t* dst_g, @@ -3661,6 +3864,16 @@ void SplitXRGBRow_RVV(const uint8_t* src_rgba, uint8_t* dst_g, uint8_t* dst_b, int width); +void SplitXRGBRow_LSX(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); +void SplitXRGBRow_LASX(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); void SplitXRGBRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_r, uint8_t* dst_g, @@ -3681,6 +3894,16 @@ void SplitXRGBRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_g, uint8_t* dst_b, int width); +void SplitXRGBRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); +void SplitXRGBRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); void MergeXR30Row_C(const uint16_t* src_r, const uint16_t* src_g, @@ -3896,6 +4119,26 @@ void MergeUVRow_16_SME(const uint16_t* src_u, uint16_t* dst_uv, int depth, int width); +void MergeUVRow_16_LSX(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width); +void MergeUVRow_16_Any_LSX(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width); +void MergeUVRow_16_LASX(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width); +void MergeUVRow_16_Any_LASX(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width); void SplitUVRow_16_C(const uint16_t* src_uv, uint16_t* dst_u, @@ -3922,6 +4165,26 @@ void SplitUVRow_16_Any_NEON(const uint16_t* src_uv, uint16_t* dst_v, int depth, int width); +void SplitUVRow_16_LSX(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width); +void SplitUVRow_16_Any_LSX(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width); +void SplitUVRow_16_LASX(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width); +void SplitUVRow_16_Any_LASX(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width); void MultiplyRow_16_C(const uint16_t* src_y, uint16_t* dst_y, @@ -3955,6 +4218,22 @@ void MultiplyRow_16_RVV(const uint16_t* src_y, uint16_t* dst_y, int scale, int width); +void MultiplyRow_16_LSX(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); +void MultiplyRow_16_Any_LSX(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int scale, + int width); +void MultiplyRow_16_LASX(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); +void MultiplyRow_16_Any_LASX(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int scale, + int width); void DivideRow_16_C(const uint16_t* src_y, uint16_t* dst_y, @@ -3988,6 +4267,22 @@ void DivideRow_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int scale, int width); +void DivideRow_16_LSX(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); +void DivideRow_16_Any_LSX(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int scale, + int width); +void DivideRow_16_LASX(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); +void DivideRow_16_Any_LASX(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int scale, + int width); void Convert8To16Row_C(const uint8_t* src_y, uint16_t* dst_y, @@ -4029,6 +4324,22 @@ void Convert8To16Row_RVV(const uint8_t* src_y, uint16_t* dst_y, int bits, int width); +void Convert8To16Row_LSX(const uint8_t* src_y, + uint16_t* dst_y, + int bits, + int width); +void Convert8To16Row_Any_LSX(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int bits, + int width); +void Convert8To16Row_LASX(const uint8_t* src_y, + uint16_t* dst_y, + int bits, + int width); +void Convert8To16Row_Any_LASX(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int bits, + int width); void Convert16To8Row_C(const uint16_t* src_y, uint8_t* dst_y, @@ -4074,6 +4385,22 @@ void Convert16To8Row_RVV(const uint16_t* src_y, uint8_t* dst_y, int scale, int width); +void Convert16To8Row_LSX(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width); +void Convert16To8Row_Any_LSX(const uint16_t* src_ptr, + uint8_t* dst_ptr, + int scale, + int width); +void Convert16To8Row_LASX(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width); +void Convert16To8Row_Any_LASX(const uint16_t* src_ptr, + uint8_t* dst_ptr, + int scale, + int width); void HalfRow_C(const uint8_t* src_uv, ptrdiff_t src_uv_stride, @@ -4095,6 +4422,22 @@ void HalfRow_SVE2(const uint8_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, int width); +void HalfRow_LSX(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); +void HalfRow_Any_LSX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int width); +void HalfRow_LASX(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); +void HalfRow_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int width); void HalfRow_16_C(const uint16_t* src_uv, ptrdiff_t src_uv_stride, @@ -4116,6 +4459,22 @@ void HalfRow_16_SVE2(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint16_t* dst_uv, int width); +void HalfRow_16_LSX(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); +void HalfRow_16_Any_LSX(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int width); +void HalfRow_16_LASX(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); +void HalfRow_16_Any_LASX(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int width); void HalfRow_16To8_C(const uint16_t* src_uv, ptrdiff_t src_uv_stride, @@ -4172,6 +4531,26 @@ void HalfRow_16To8_RVV(const uint16_t* src_uv, uint8_t* dst_uv, int scale, int width); +void HalfRow_16To8_LSX(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_Any_LSX(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); +void HalfRow_16To8_LASX(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_Any_LASX(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int scale, + int width); void HalfWidthRow_16To8_C(const uint16_t* src_uv, ptrdiff_t src_uv_stride, @@ -4284,6 +4663,26 @@ void Convert8To8Row_Any_AVX2(const uint8_t* src_ptr, int scale, int bias, int width); +void Convert8To8Row_LSX(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width); +void Convert8To8Row_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int scale, + int bias, + int width); +void Convert8To8Row_LASX(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width); +void Convert8To8Row_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int scale, + int bias, + int width); void CopyRow_SSE2(const uint8_t* src, uint8_t* dst, int width); void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width); @@ -4307,11 +4706,17 @@ void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int count); void CopyRow_16_Any_AVX2(const uint16_t* src_ptr, uint16_t* dst_ptr, int width); void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count); void CopyRow_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int width); +void CopyRow_16_LSX(const uint16_t* src, uint16_t* dst, int count); +void CopyRow_16_Any_LSX(const uint16_t* src_ptr, uint16_t* dst_ptr, int width); +void CopyRow_16_LASX(const uint16_t* src, uint16_t* dst, int count); +void CopyRow_16_Any_LASX(const uint16_t* src_ptr, uint16_t* dst_ptr, int width); void ARGBCopyAlphaRow_C(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyAlphaRow_SSE2(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyAlphaRow_NEON(const uint8_t* src, uint8_t* dst, int width); +void ARGBCopyAlphaRow_LSX(const uint8_t* src, uint8_t* dst, int width); +void ARGBCopyAlphaRow_LASX(const uint8_t* src, uint8_t* dst, int width); void ARGBCopyAlphaRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4321,6 +4726,12 @@ void ARGBCopyAlphaRow_Any_AVX2(const uint8_t* src_ptr, void ARGBCopyAlphaRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBCopyAlphaRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBCopyAlphaRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void ARGBExtractAlphaRow_C(const uint8_t* src_argb, uint8_t* dst_a, int width); void ARGBExtractAlphaRow_SSE2(const uint8_t* src_argb, @@ -4385,6 +4796,14 @@ void ARGBCopyYToAlphaRow_Any_NEON(const uint8_t* src_ptr, void ARGBCopyYToAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBCopyYToAlphaRow_LSX(const uint8_t* src, uint8_t* dst, int width); +void ARGBCopyYToAlphaRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBCopyYToAlphaRow_LASX(const uint8_t* src, uint8_t* dst, int width); +void ARGBCopyYToAlphaRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void SetRow_C(uint8_t* dst, uint8_t v8, int width); void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width); @@ -4517,6 +4936,8 @@ void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToRGBARow_SME(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width); void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width); +void RAWToRGBARow_LSX(const uint8_t* src_raw, uint8_t* dst_rgba, int width); +void RAWToRGBARow_LASX(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width); void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); @@ -4594,6 +5015,12 @@ void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width); void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); +void AR30ToARGBRow_LSX(const uint8_t* src_ar30, uint8_t* dst_argb, int width); +void AR30ToABGRRow_LSX(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); +void AR30ToAB30Row_LSX(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); +void AR30ToARGBRow_LASX(const uint8_t* src_ar30, uint8_t* dst_argb, int width); +void AR30ToABGRRow_LASX(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); +void AR30ToAB30Row_LASX(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); void AR30ToARGBRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4612,6 +5039,24 @@ void AR30ToABGRRow_Any_NEON(const uint8_t* src_ptr, void AR30ToAB30Row_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void AR30ToARGBRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToABGRRow_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToAB30Row_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToARGBRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToABGRRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToAB30Row_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void RGB24ToARGBRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -4659,6 +5104,8 @@ void RAWToARGBRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void RAWToRGBARow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void RAWToARGBRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void RAWToARGBRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void RAWToRGBARow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void RAWToRGBARow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void RAWToRGB24Row_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4804,6 +5251,10 @@ void ARGBToARGB4444Row_LSX(const uint8_t* src_argb, void ARGBToARGB4444Row_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, int width); +void ABGRToAR30Row_LSX(const uint8_t* src, uint8_t* dst, int width); +void ABGRToAR30Row_LASX(const uint8_t* src, uint8_t* dst, int width); +void ARGBToAR30Row_LSX(const uint8_t* src, uint8_t* dst, int width); +void ARGBToAR30Row_LASX(const uint8_t* src, uint8_t* dst, int width); void ARGBToRAWRow_RVV(const uint8_t* src_argb, uint8_t* dst_raw, int width); void ARGBToABGRRow_RVV(const uint8_t* src_argb, uint8_t* dst_abgr, int width); @@ -4852,6 +5303,14 @@ void ARGBToAR64Row_NEON(const uint8_t* src_argb, uint16_t* dst_ar64, int width); void ARGBToAB64Row_NEON(const uint8_t* src_argb, uint16_t* dst_ab64, int width); void AR64ToARGBRow_NEON(const uint16_t* src_ar64, uint8_t* dst_argb, int width); void AB64ToARGBRow_NEON(const uint16_t* src_ab64, uint8_t* dst_argb, int width); +void ARGBToAR64Row_LSX(const uint8_t* src_argb, uint16_t* dst_ar64, int width); +void ARGBToAB64Row_LSX(const uint8_t* src_argb, uint16_t* dst_ab64, int width); +void AR64ToARGBRow_LSX(const uint16_t* src_ar64, uint8_t* dst_argb, int width); +void AB64ToARGBRow_LSX(const uint16_t* src_ab64, uint8_t* dst_argb, int width); +void ARGBToAR64Row_LASX(const uint8_t* src_argb, uint16_t* dst_ar64, int width); +void ARGBToAB64Row_LASX(const uint8_t* src_argb, uint16_t* dst_ab64, int width); +void AR64ToARGBRow_LASX(const uint16_t* src_ar64, uint8_t* dst_argb, int width); +void AB64ToARGBRow_LASX(const uint16_t* src_ab64, uint8_t* dst_argb, int width); void ARGBToAR64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ar64, int width); void ARGBToAB64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ab64, int width); void AR64ToARGBRow_RVV(const uint16_t* src_ar64, uint8_t* dst_argb, int width); @@ -4894,6 +5353,30 @@ void AR64ToARGBRow_Any_NEON(const uint16_t* src_ptr, void AB64ToARGBRow_Any_NEON(const uint16_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBToAR64Row_Any_LSX(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int width); +void ARGBToAB64Row_Any_LSX(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int width); +void AR64ToARGBRow_Any_LSX(const uint16_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AB64ToARGBRow_Any_LSX(const uint16_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBToAR64Row_Any_LASX(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int width); +void ARGBToAB64Row_Any_LASX(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int width); +void AR64ToARGBRow_Any_LASX(const uint16_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AB64ToARGBRow_Any_LASX(const uint16_t* src_ptr, + uint8_t* dst_ptr, + int width); void J400ToARGBRow_AVX512BW(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width); @@ -6115,6 +6598,26 @@ void BlendPlaneRow_RVV(const uint8_t* src0, const uint8_t* alpha, uint8_t* dst, int width); +void BlendPlaneRow_LSX(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width); +void BlendPlaneRow_Any_LSX(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + int width); +void BlendPlaneRow_LASX(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width); +void BlendPlaneRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + int width); void BlendPlaneRow_C(const uint8_t* src0, const uint8_t* src1, const uint8_t* alpha, @@ -6511,6 +7014,18 @@ void ARGBToARGB4444Row_Any_LSX(const uint8_t* src_ptr, void ARGBToARGB4444Row_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ABGRToAR30Row_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ABGRToAR30Row_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBToAR30Row_Any_LSX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBToAR30Row_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void I444ToARGBRow_Any_NEON(const uint8_t* y_buf, const uint8_t* u_buf, @@ -6999,6 +7514,14 @@ void YUY2ToUVRow_LASX(const uint8_t* src_yuy2, uint8_t* dst_u, uint8_t* dst_v, int width); +void YUY2ToNVUVRow_LSX(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_uv, + int width); +void YUY2ToNVUVRow_LASX(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_uv, + int width); void YUY2ToUV422Row_LSX(const uint8_t* src_yuy2, uint8_t* dst_u, uint8_t* dst_v, @@ -7075,6 +7598,14 @@ void YUY2ToUVRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, int width); +void YUY2ToNVUVRow_Any_LSX(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_uv, + int width); +void YUY2ToNVUVRow_Any_LASX(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_uv, + int width); void YUY2ToUV422Row_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, @@ -7213,6 +7744,10 @@ void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width); void SwapUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void SwapUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_vu, int width); void SwapUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_vu, int width); +void SwapUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_vu, int width); +void SwapUVRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void SwapUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_vu, int width); +void SwapUVRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void AYUVToYRow_C(const uint8_t* src_ayuv, uint8_t* dst_y, int width); void AYUVToYRow_AVX2(const uint8_t* src_ayuv, uint8_t* dst_y, int width); void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width); @@ -7265,6 +7800,24 @@ void AYUVToVURow_SME(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, int width); +void AYUVToYRow_LSX(const uint8_t* src_ayuv, uint8_t* dst_y, int width); +void AYUVToUVRow_LSX(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); +void AYUVToVURow_LSX(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); +void AYUVToYRow_LASX(const uint8_t* src_ayuv, uint8_t* dst_y, int width); +void AYUVToUVRow_LASX(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); +void AYUVToVURow_LASX(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); void AYUVToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void AYUVToUVRow_Any_AVX2(const uint8_t* src_ptr, int src_stride, @@ -7283,6 +7836,24 @@ void AYUVToVURow_Any_NEON(const uint8_t* src_ptr, int src_stride, uint8_t* dst_vu, int width); +void AYUVToYRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void AYUVToUVRow_Any_LSX(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_vu, + int width); +void AYUVToVURow_Any_LSX(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_vu, + int width); +void AYUVToYRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void AYUVToUVRow_Any_LASX(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_vu, + int width); +void AYUVToVURow_Any_LASX(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_vu, + int width); void I422ToYUY2Row_C(const uint8_t* src_y, const uint8_t* src_u, @@ -7789,6 +8360,26 @@ void InterpolateRow_16_SVE2(uint16_t* dst_ptr, ptrdiff_t src_stride, int width, int source_y_fraction); +void InterpolateRow_16_LSX(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); +void InterpolateRow_16_Any_LSX(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); +void InterpolateRow_16_LASX(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); +void InterpolateRow_16_Any_LASX(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); // Sobel images. void SobelXRow_C(const uint8_t* src_y0, @@ -8077,28 +8668,6 @@ void ARGBLumaColorTableRow_Any_NEON(const uint8_t* src_ptr, const uint8_t* luma, uint32_t lumacoeff); -float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width); -float ScaleMaxSamples_AVX2(const float* src, - float* dst, - float scale, - int width); -float ScaleMaxSamples_NEON(const float* src, - float* dst, - float scale, - int width); -float ScaleSumSamples_C(const float* src, float* dst, float scale, int width); -float ScaleSumSamples_AVX2(const float* src, - float* dst, - float scale, - int width); -float ScaleSumSamples_NEON(const float* src, - float* dst, - float scale, - int width); -void ScaleSamples_C(const float* src, float* dst, float scale, int width); -void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width); -void ScaleSamples_NEON(const float* src, float* dst, float scale, int width); - void GaussRow_F32_AVX2(const float* src, float* dst, int width); void GaussRow_F32_NEON(const float* src, float* dst, int width); void GaussRow_F32_C(const float* src, float* dst, int width); diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 2f1f1365a..59d31e887 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -90,6 +90,9 @@ extern "C" { #define HAS_SCALEUVROWUP2_BILINEAR_AVX2 #define HAS_SCALEUVROWUP2_LINEAR_16_AVX2 #define HAS_SCALEUVROWUP2_LINEAR_AVX2 +#define HAS_SCALEMAXSAMPLES_AVX2 +#define HAS_SCALESAMPLES_AVX2 +#define HAS_SCALESUMSAMPLES_AVX2 #endif // The following are available for AVX512 x64 platforms: @@ -137,6 +140,9 @@ extern "C" { // The following are available on AArch64 Neon platforms: #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #define HAS_SCALEROWDOWN2_16_NEON +#define HAS_SCALEMAXSAMPLES_NEON +#define HAS_SCALESAMPLES_NEON +#define HAS_SCALESUMSAMPLES_NEON #endif // The following are available on AArch64 SME platforms: @@ -163,6 +169,26 @@ extern "C" { #define HAS_SCALEARGBFILTERCOLS_LSX #define HAS_SCALEROWDOWN34_LSX #define HAS_SCALEUVADDCOLS3_LSX +#define HAS_SCALEMAXSAMPLES_LSX +#define HAS_SCALESAMPLES_LSX +#define HAS_SCALESUMSAMPLES_LSX +#endif + +#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_lp64) +#define HAS_SCALEARGBROWDOWN2_LASX +#define HAS_SCALEARGBROWDOWNEVEN_LASX +#define HAS_SCALEROWDOWN2_LASX +#define HAS_SCALEROWDOWN4_LASX +#define HAS_SCALEROWDOWN38_LASX +#define HAS_SCALEFILTERCOLS_LASX +#define HAS_SCALEADDROW_LASX +#define HAS_SCALEARGBCOLS_LASX +#define HAS_SCALEARGBFILTERCOLS_LASX +#define HAS_SCALEROWDOWN34_LASX +#define HAS_SCALEUVADDCOLS3_LASX +#define HAS_SCALEMAXSAMPLES_LASX +#define HAS_SCALESAMPLES_LASX +#define HAS_SCALESUMSAMPLES_LASX #endif #if !defined(LIBYUV_DISABLE_RVV) && \ @@ -646,6 +672,9 @@ void ScaleUVAddCols3_16_C(int dst_width, int scaleval, const uint32_t* src_uv, uint16_t* dst_uv); +float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width); +float ScaleSumSamples_C(const float* src, float* dst, float scale, int width); +void ScaleSamples_C(const float* src, float* dst, float scale, int width); // Specialized scalers for x86. void ScaleRowDown2_SSSE3(const uint8_t* src_ptr, @@ -975,6 +1004,15 @@ void ScaleColsUp2_AVX2(uint8_t* dst_ptr, int dst_width, int x, int dx); +float ScaleMaxSamples_AVX2(const float* src, + float* dst, + float scale, + int width); +float ScaleSumSamples_AVX2(const float* src, + float* dst, + float scale, + int width); +void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width); // ARGB Column functions void ScaleARGBCols_AVX2(uint8_t* dst_argb, @@ -1079,6 +1117,18 @@ void ScaleARGBRowDown2Box_LSX(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); +void ScaleARGBRowDown2_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width); +void ScaleARGBRowDown2Linear_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width); +void ScaleARGBRowDown2Box_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width); void ScaleARGBRowDown2_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -1115,6 +1165,18 @@ void ScaleARGBRowDown2Box_Any_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); +void ScaleARGBRowDown2_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleARGBRowDown2Linear_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleARGBRowDown2Box_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, @@ -1145,6 +1207,16 @@ void ScaleARGBRowDownEvenBox_LSX(const uint8_t* src_argb, int src_stepx, uint8_t* dst_argb, int dst_width); +void ScaleARGBRowDownEven_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + int32_t src_stepx, + uint8_t* dst_argb, + int dst_width); +void ScaleARGBRowDownEvenBox_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width); void ScaleARGBRowDownEven_RVV(const uint8_t* src_argb, ptrdiff_t src_stride, int32_t src_stepx, @@ -1185,6 +1257,16 @@ void ScaleARGBRowDownEvenBox_Any_LSX(const uint8_t* src_ptr, int src_stepx, uint8_t* dst_ptr, int dst_width); +void ScaleARGBRowDownEven_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int32_t src_stepx, + uint8_t* dst_ptr, + int dst_width); +void ScaleARGBRowDownEvenBox_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_ptr, + int dst_width); // UV Row functions void ScaleUVRowDown2_AVX2(const uint8_t* src_ptr, @@ -1430,6 +1512,14 @@ void ScaleUVAddCols3_Any_LSX(int dst_width, int scaleval, const uint16_t* src_uv, uint8_t* dst_uv); +void ScaleUVAddCols3_LASX(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); +void ScaleUVAddCols3_Any_LASX(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv); void ScaleUVAddCols3_WASMSIMD(int dst_width, int scaleval, const uint16_t* src_uv, @@ -1652,6 +1742,15 @@ void ScaleFilterCols_Any_NEON(uint8_t* dst_ptr, int dst_width, int x, int dx); +float ScaleMaxSamples_NEON(const float* src, + float* dst, + float scale, + int width); +float ScaleSumSamples_NEON(const float* src, + float* dst, + float scale, + int width); +void ScaleSamples_NEON(const float* src, float* dst, float scale, int width); void ScaleRowDown2_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -1775,6 +1874,141 @@ void ScaleRowDown34_1_Box_Any_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); +float ScaleMaxSamples_LSX(const float* src, float* dst, float scale, int width); +float ScaleSumSamples_LSX(const float* src, float* dst, float scale, int width); +void ScaleSamples_LSX(const float* src, float* dst, float scale, int width); + +void ScaleRowDown2_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width); +void ScaleRowDown2Linear_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width); +void ScaleRowDown2Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width); +void ScaleRowDown4_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width); +void ScaleRowDown4Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width); +void ScaleRowDown38_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width); +void ScaleRowDown38_2_Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleAddRow_LASX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); +void ScaleFilterCols_LASX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleARGBFilterCols_LASX(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx); +void ScaleARGBCols_LASX(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx); +void ScaleRowDown34_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width); +void ScaleRowDown34_0_Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* d, + int dst_width); +void ScaleRowDown34_1_Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* d, + int dst_width); +void ScaleRowDown2_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown2Linear_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown2Box_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown4_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown4Box_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_2_Box_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleAddRow_Any_LASX(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width); +void ScaleFilterCols_Any_LASX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleARGBCols_Any_LASX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleARGBFilterCols_Any_LASX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleRowDown34_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_0_Box_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_1_Box_Any_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +float ScaleMaxSamples_LASX(const float* src, + float* dst, + float scale, + int width); +float ScaleSumSamples_LASX(const float* src, + float* dst, + float scale, + int width); +void ScaleSamples_LASX(const float* src, float* dst, float scale, int width); void ScaleAddRow_RVV(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); void ScaleRowDown2_RVV(const uint8_t* src_ptr, diff --git a/source/convert.cc b/source/convert.cc index d2f7e78e3..8437648d0 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -310,6 +310,22 @@ static int I41xToI420(const uint16_t* src_y, Convert16To8Row = Convert16To8Row_HVX; } #endif +#if defined(HAS_CONVERT16TO8ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert16To8Row = Convert16To8Row_Any_LSX; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert16To8Row = Convert16To8Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + Convert16To8Row = Convert16To8Row_LASX; + } + } +#endif if (IS_ALIGNED(width, 2)) { #if defined(HAS_HALFWIDTHROW_16TO8_NEON) @@ -484,6 +500,22 @@ static int I21xToI420(const uint16_t* src_y, Convert16To8Row = Convert16To8Row_HVX; } #endif +#if defined(HAS_CONVERT16TO8ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert16To8Row = Convert16To8Row_Any_LSX; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert16To8Row = Convert16To8Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + Convert16To8Row = Convert16To8Row_LASX; + } + } +#endif #if defined(HAS_HALFROW_16TO8_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -534,6 +566,22 @@ static int I21xToI420(const uint16_t* src_y, HalfRow_16To8 = HalfRow_16To8_HVX; } #endif +#if defined(HAS_HALFROW_16TO8_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + HalfRow_16To8 = HalfRow_16To8_Any_LSX; + if (IS_ALIGNED(uv_width, 16)) { + HalfRow_16To8 = HalfRow_16To8_LSX; + } + } +#endif +#if defined(HAS_HALFROW_16TO8_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + HalfRow_16To8 = HalfRow_16To8_Any_LASX; + if (IS_ALIGNED(uv_width, 32)) { + HalfRow_16To8 = HalfRow_16To8_LASX; + } + } +#endif for (y = 0; y < height - 1; y += 2) { HalfRow_16To8(src_u, src_stride_u, dst_u, scale, uv_width); @@ -984,6 +1032,22 @@ int I010ToNV12(const uint16_t* src_y, Convert16To8Row = Convert16To8Row_HVX; } #endif +#if defined(HAS_CONVERT16TO8ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert16To8Row = Convert16To8Row_Any_LSX; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert16To8Row = Convert16To8Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + Convert16To8Row = Convert16To8Row_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -2130,6 +2194,22 @@ static int Px10ToI420(const uint16_t* src_y, Convert16To8Row = Convert16To8Row_HVX; } #endif +#if defined(HAS_CONVERT16TO8ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert16To8Row = Convert16To8Row_Any_LSX; + if (IS_ALIGNED(uv_packed, 16)) { + Convert16To8Row = Convert16To8Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert16To8Row = Convert16To8Row_Any_LASX; + if (IS_ALIGNED(uv_packed, 32)) { + Convert16To8Row = Convert16To8Row_LASX; + } + } +#endif #if defined(HAS_SPLITUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { SplitUVRow = SplitUVRow_Any_SSE2; @@ -2244,6 +2324,22 @@ static int Px10ToI420(const uint16_t* src_y, if (TestCpuFlag(kCpuHasHVX)) { SplitUVRow_16 = SplitUVRow_16_HVX; } +#endif +#if defined(HAS_SPLITUVROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + SplitUVRow_16 = SplitUVRow_16_Any_LSX; + if (IS_ALIGNED(src_uv_width, 8)) { + SplitUVRow_16 = SplitUVRow_16_LSX; + } + } +#endif +#if defined(HAS_SPLITUVROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SplitUVRow_16 = SplitUVRow_16_Any_LASX; + if (IS_ALIGNED(src_uv_width, 16)) { + SplitUVRow_16 = SplitUVRow_16_LASX; + } + } #endif if (subsample_x) { #if defined(HAS_HALFROW_16TO8_NEON) @@ -2294,6 +2390,22 @@ static int Px10ToI420(const uint16_t* src_y, if (TestCpuFlag(kCpuHasHVX)) { Half16To8 = HalfRow_16To8_HVX; } +#endif +#if defined(HAS_HALFROW_16TO8_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Half16To8 = HalfRow_16To8_Any_LSX; + if (IS_ALIGNED(dst_uv_width, 16)) { + Half16To8 = HalfRow_16To8_LSX; + } + } +#endif +#if defined(HAS_HALFROW_16TO8_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Half16To8 = HalfRow_16To8_Any_LASX; + if (IS_ALIGNED(dst_uv_width, 32)) { + Half16To8 = HalfRow_16To8_LASX; + } + } #endif } else if (IS_ALIGNED(width, 2)) { #if defined(HAS_HALFWIDTHROW_16TO8_NEON) @@ -2808,6 +2920,26 @@ int AYUVToNV12(const uint8_t* src_ayuv, AYUVToUVRow = AYUVToUVRow_SME; } #endif +#if defined(HAS_AYUVTOYROW_LSX) && defined(HAS_AYUVTOUVROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + AYUVToYRow = AYUVToYRow_Any_LSX; + AYUVToUVRow = AYUVToUVRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + AYUVToYRow = AYUVToYRow_LSX; + AYUVToUVRow = AYUVToUVRow_LSX; + } + } +#endif +#if defined(HAS_AYUVTOYROW_LASX) && defined(HAS_AYUVTOUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + AYUVToYRow = AYUVToYRow_Any_LASX; + AYUVToUVRow = AYUVToUVRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + AYUVToYRow = AYUVToYRow_LASX; + AYUVToUVRow = AYUVToUVRow_LASX; + } + } +#endif for (y = 0; y < height - 1; y += 2) { AYUVToUVRow(src_ayuv, src_stride_ayuv, dst_uv, width); @@ -2887,6 +3019,26 @@ int AYUVToNV21(const uint8_t* src_ayuv, AYUVToVURow = AYUVToVURow_SME; } #endif +#if defined(HAS_AYUVTOYROW_LSX) && defined(HAS_AYUVTOVUROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + AYUVToYRow = AYUVToYRow_Any_LSX; + AYUVToVURow = AYUVToVURow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + AYUVToYRow = AYUVToYRow_LSX; + AYUVToVURow = AYUVToVURow_LSX; + } + } +#endif +#if defined(HAS_AYUVTOYROW_LASX) && defined(HAS_AYUVTOVUROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + AYUVToYRow = AYUVToYRow_Any_LASX; + AYUVToVURow = AYUVToVURow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + AYUVToYRow = AYUVToYRow_LASX; + AYUVToVURow = AYUVToVURow_LASX; + } + } +#endif for (y = 0; y < height - 1; y += 2) { AYUVToVURow(src_ayuv, src_stride_ayuv, dst_vu, width); @@ -3601,6 +3753,22 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, AR30ToARGBRow = AR30ToARGBRow_AVX512BW; } #endif +#if defined(HAS_AR30TOARGBROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + AR30ToARGBRow = AR30ToARGBRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + AR30ToARGBRow = AR30ToARGBRow_LSX; + } + } +#endif +#if defined(HAS_AR30TOARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + AR30ToARGBRow = AR30ToARGBRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + AR30ToARGBRow = AR30ToARGBRow_LASX; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; diff --git a/source/convert_argb.cc b/source/convert_argb.cc index be3d0a632..b10c18f5f 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -4385,6 +4385,22 @@ int RAWToRGBA(const uint8_t* src_raw, RAWToRGBARow = RAWToRGBARow_RVV; } #endif +#if defined(HAS_RAWTORGBAROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + RAWToRGBARow = RAWToRGBARow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + RAWToRGBARow = RAWToRGBARow_LSX; + } + } +#endif +#if defined(HAS_RAWTORGBAROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + RAWToRGBARow = RAWToRGBARow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + RAWToRGBARow = RAWToRGBARow_LASX; + } + } +#endif for (y = 0; y < height; ++y) { RAWToRGBARow(src_raw, dst_rgba, width); @@ -4691,6 +4707,22 @@ int AR30ToARGB(const uint8_t* src_ar30, if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToARGBRow = AR30ToARGBRow_AVX512BW; } +#endif +#if defined(HAS_AR30TOARGBROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + AR30ToARGBRow = AR30ToARGBRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + AR30ToARGBRow = AR30ToARGBRow_LSX; + } + } +#endif +#if defined(HAS_AR30TOARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + AR30ToARGBRow = AR30ToARGBRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + AR30ToARGBRow = AR30ToARGBRow_LASX; + } + } #endif for (y = 0; y < height; ++y) { AR30ToARGBRow(src_ar30, dst_argb, width); @@ -4748,6 +4780,22 @@ int AR30ToABGR(const uint8_t* src_ar30, if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToABGRRow = AR30ToABGRRow_AVX512BW; } +#endif +#if defined(HAS_AR30TOABGRROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + AR30ToABGRRow = AR30ToABGRRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + AR30ToABGRRow = AR30ToABGRRow_LSX; + } + } +#endif +#if defined(HAS_AR30TOABGRROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + AR30ToABGRRow = AR30ToABGRRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + AR30ToABGRRow = AR30ToABGRRow_LASX; + } + } #endif for (y = 0; y < height; ++y) { AR30ToABGRRow(src_ar30, dst_abgr, width); @@ -4805,6 +4853,22 @@ int AR30ToAB30(const uint8_t* src_ar30, if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToAB30Row = AR30ToAB30Row_AVX512BW; } +#endif +#if defined(HAS_AR30TOAB30ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + AR30ToAB30Row = AR30ToAB30Row_Any_LSX; + if (IS_ALIGNED(width, 4)) { + AR30ToAB30Row = AR30ToAB30Row_LSX; + } + } +#endif +#if defined(HAS_AR30TOAB30ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + AR30ToAB30Row = AR30ToAB30Row_Any_LASX; + if (IS_ALIGNED(width, 8)) { + AR30ToAB30Row = AR30ToAB30Row_LASX; + } + } #endif for (y = 0; y < height; ++y) { AR30ToAB30Row(src_ar30, dst_ab30, width); @@ -4876,6 +4940,22 @@ int AR64ToARGB(const uint16_t* src_ar64, AR64ToARGBRow = AR64ToARGBRow_HVX; } #endif +#if defined(HAS_AR64TOARGBROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + AR64ToARGBRow = AR64ToARGBRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + AR64ToARGBRow = AR64ToARGBRow_LSX; + } + } +#endif +#if defined(HAS_AR64TOARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + AR64ToARGBRow = AR64ToARGBRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + AR64ToARGBRow = AR64ToARGBRow_LASX; + } + } +#endif for (y = 0; y < height; ++y) { AR64ToARGBRow(src_ar64, dst_argb, width); @@ -4947,6 +5027,22 @@ int AB64ToARGB(const uint16_t* src_ab64, AB64ToARGBRow = AB64ToARGBRow_HVX; } #endif +#if defined(HAS_AB64TOARGBROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + AB64ToARGBRow = AB64ToARGBRow_Any_LSX; + if (IS_ALIGNED(width, 4)) { + AB64ToARGBRow = AB64ToARGBRow_LSX; + } + } +#endif +#if defined(HAS_AB64TOARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + AB64ToARGBRow = AB64ToARGBRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + AB64ToARGBRow = AB64ToARGBRow_LASX; + } + } +#endif for (y = 0; y < height; ++y) { AB64ToARGBRow(src_ab64, dst_argb, width); diff --git a/source/convert_from.cc b/source/convert_from.cc index 49402b09d..037dcec39 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -827,6 +827,22 @@ static int I4xxToPxxx(const uint8_t* src_y, } } #endif +#if defined(HAS_CONVERT8TO16ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert8To16Row = Convert8To16Row_Any_LSX; + if (IS_ALIGNED(uv_width, 16)) { + Convert8To16Row = Convert8To16Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT8TO16ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert8To16Row = Convert8To16Row_Any_LASX; + if (IS_ALIGNED(uv_width, 32)) { + Convert8To16Row = Convert8To16Row_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow_16 = MergeUVRow_16_Any_AVX2; @@ -858,6 +874,22 @@ static int I4xxToPxxx(const uint8_t* src_y, MergeUVRow_16 = MergeUVRow_16_HVX; } #endif +#if defined(HAS_MERGEUVROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + MergeUVRow_16 = MergeUVRow_16_Any_LSX; + if (IS_ALIGNED(uv_width, 8)) { + MergeUVRow_16 = MergeUVRow_16_LSX; + } + } +#endif +#if defined(HAS_MERGEUVROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow_16 = MergeUVRow_16_Any_LASX; + if (IS_ALIGNED(uv_width, 16)) { + MergeUVRow_16 = MergeUVRow_16_LASX; + } + } +#endif row_stride = (uv_width + 31) & ~31; align_buffer_64(row_u, (size_t)row_stride * sizeof(uint16_t) * 2u); diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 5d42dc2c9..623384718 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -2624,6 +2624,22 @@ int ARGBToP010Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_CONVERT8TO16ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert8To16Row = Convert8To16Row_Any_LSX; + if (IS_ALIGNED(width, 16)) { + Convert8To16Row = Convert8To16Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT8TO16ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert8To16Row = Convert8To16Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + Convert8To16Row = Convert8To16Row_LASX; + } + } +#endif #if defined(HAS_MULTIPLYROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MultiplyRow_16 = MultiplyRow_16_Any_AVX2; @@ -2668,6 +2684,22 @@ int ARGBToP010Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MULTIPLYROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + MultiplyRow_16 = MultiplyRow_16_Any_LSX; + if (IS_ALIGNED(width, 16)) { + MultiplyRow_16 = MultiplyRow_16_LSX; + } + } +#endif +#if defined(HAS_MULTIPLYROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MultiplyRow_16 = MultiplyRow_16_Any_LASX; + if (IS_ALIGNED(width, 32)) { + MultiplyRow_16 = MultiplyRow_16_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { MergeUVRow = MergeUVRow_Any_SSE2; @@ -3063,6 +3095,22 @@ int ARGBToP210Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_CONVERT8TO16ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert8To16Row = Convert8To16Row_Any_LSX; + if (IS_ALIGNED(width, 16)) { + Convert8To16Row = Convert8To16Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT8TO16ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert8To16Row = Convert8To16Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + Convert8To16Row = Convert8To16Row_LASX; + } + } +#endif #if defined(HAS_MULTIPLYROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MultiplyRow_16 = MultiplyRow_16_Any_AVX2; @@ -3107,6 +3155,22 @@ int ARGBToP210Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MULTIPLYROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + MultiplyRow_16 = MultiplyRow_16_Any_LSX; + if (IS_ALIGNED(width, 16)) { + MultiplyRow_16 = MultiplyRow_16_LSX; + } + } +#endif +#if defined(HAS_MULTIPLYROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MultiplyRow_16 = MultiplyRow_16_Any_LASX; + if (IS_ALIGNED(width, 32)) { + MultiplyRow_16 = MultiplyRow_16_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { MergeUVRow = MergeUVRow_Any_SSE2; @@ -3473,6 +3537,22 @@ int ARGBToP410Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_CONVERT8TO16ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert8To16Row = Convert8To16Row_Any_LSX; + if (IS_ALIGNED(width, 16)) { + Convert8To16Row = Convert8To16Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT8TO16ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert8To16Row = Convert8To16Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + Convert8To16Row = Convert8To16Row_LASX; + } + } +#endif #if defined(HAS_MULTIPLYROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MultiplyRow_16 = MultiplyRow_16_Any_AVX2; @@ -3517,6 +3597,22 @@ int ARGBToP410Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MULTIPLYROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + MultiplyRow_16 = MultiplyRow_16_Any_LSX; + if (IS_ALIGNED(width, 16)) { + MultiplyRow_16 = MultiplyRow_16_LSX; + } + } +#endif +#if defined(HAS_MULTIPLYROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MultiplyRow_16 = MultiplyRow_16_Any_LASX; + if (IS_ALIGNED(width, 32)) { + MultiplyRow_16 = MultiplyRow_16_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { MergeUVRow = MergeUVRow_Any_SSE2; @@ -4346,6 +4442,22 @@ int ABGRToAR30(const uint8_t* src_abgr, if (TestCpuFlag(kCpuHasAVX512BW)) { ABGRToAR30Row = ABGRToAR30Row_AVX512BW; } +#endif +#if defined(HAS_ABGRTOAR30ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ABGRToAR30Row = ABGRToAR30Row_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ABGRToAR30Row = ABGRToAR30Row_LSX; + } + } +#endif +#if defined(HAS_ABGRTOAR30ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ABGRToAR30Row = ABGRToAR30Row_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ABGRToAR30Row = ABGRToAR30Row_LASX; + } + } #endif for (y = 0; y < height; ++y) { ABGRToAR30Row(src_abgr, dst_ar30, width); @@ -4410,6 +4522,22 @@ int ARGBToAR30(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasAVX512BW)) { ARGBToAR30Row = ARGBToAR30Row_AVX512BW; } +#endif +#if defined(HAS_ARGBTOAR30ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBToAR30Row = ARGBToAR30Row_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBToAR30Row = ARGBToAR30Row_LSX; + } + } +#endif +#if defined(HAS_ARGBTOAR30ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBToAR30Row = ARGBToAR30Row_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBToAR30Row = ARGBToAR30Row_LASX; + } + } #endif for (y = 0; y < height; ++y) { ARGBToAR30Row(src_argb, dst_ar30, width); @@ -4602,6 +4730,22 @@ int ARGBToAR64(const uint8_t* src_argb, ARGBToAR64Row = ARGBToAR64Row_HVX; } #endif +#if defined(HAS_ARGBTOAR64ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBToAR64Row = ARGBToAR64Row_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBToAR64Row = ARGBToAR64Row_LSX; + } + } +#endif +#if defined(HAS_ARGBTOAR64ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBToAR64Row = ARGBToAR64Row_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBToAR64Row = ARGBToAR64Row_LASX; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToAR64Row(src_argb, dst_ar64, width); @@ -4673,6 +4817,22 @@ int ARGBToAB64(const uint8_t* src_argb, ARGBToAB64Row = ARGBToAB64Row_HVX; } #endif +#if defined(HAS_ARGBTOAB64ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBToAB64Row = ARGBToAB64Row_Any_LSX; + if (IS_ALIGNED(width, 4)) { + ARGBToAB64Row = ARGBToAB64Row_LSX; + } + } +#endif +#if defined(HAS_ARGBTOAB64ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBToAB64Row = ARGBToAB64Row_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBToAB64Row = ARGBToAB64Row_LASX; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToAB64Row(src_argb, dst_ab64, width); diff --git a/source/cpu_id.cc b/source/cpu_id.cc index ac987312b..cbc2b0af0 100644 --- a/source/cpu_id.cc +++ b/source/cpu_id.cc @@ -377,24 +377,48 @@ LIBYUV_API SAFEBUFFERS int RiscvCpuCaps(const char* cpuinfo_name) { return flag; } -#if defined(__loongarch__) && defined(__linux__) -// Define hwcap values ourselves: building with an old auxv header where these -// hwcap values are not defined should not prevent features from being enabled. +// Define cpucfg and hwcap values ourselves: building with an old auxv header +// where these hwcap values are not defined should not prevent features from +// being enabled. +#define YUV_LOONGARCH_CFG2_LSX (1 << 6) +#define YUV_LOONGARCH_CFG2_LASX (1 << 7) #define YUV_LOONGARCH_HWCAP_LSX (1 << 4) #define YUV_LOONGARCH_HWCAP_LASX (1 << 5) +// For LoongArch, but public to allow testing on any CPU. LIBYUV_API SAFEBUFFERS int LoongArchCpuCaps(void) { int flag = 0; - unsigned long hwcap = getauxval(AT_HWCAP); - - if (hwcap & YUV_LOONGARCH_HWCAP_LSX) +#if defined(__loongarch__) +#if !defined(LIBYUV_DISABLE_ASM) + uint32_t cfg2 = 0; + __asm__ volatile("cpucfg %0, %1\n" : "=r"(cfg2) : "r"(2)); + if (cfg2 & YUV_LOONGARCH_CFG2_LSX) { flag |= kCpuHasLSX; - - if (hwcap & YUV_LOONGARCH_HWCAP_LASX) - flag |= kCpuHasLASX; + if (cfg2 & YUV_LOONGARCH_CFG2_LASX) { + flag |= kCpuHasLASX; + } + } +#if defined(__linux__) + // Confirm OS kernel has enabled LSX/LASX context switching via AT_HWCAP. + unsigned long hwcap = getauxval(AT_HWCAP); + if (!(hwcap & YUV_LOONGARCH_HWCAP_LSX)) { + flag &= ~(kCpuHasLSX | kCpuHasLASX); + } else if (!(hwcap & YUV_LOONGARCH_HWCAP_LASX)) { + flag &= ~kCpuHasLASX; + } +#endif +#elif defined(__linux__) + unsigned long hwcap = getauxval(AT_HWCAP); + if (hwcap & YUV_LOONGARCH_HWCAP_LSX) { + flag |= kCpuHasLSX; + if (hwcap & YUV_LOONGARCH_HWCAP_LASX) { + flag |= kCpuHasLASX; + } + } +#endif +#endif // defined(__loongarch__) return flag; } -#endif static SAFEBUFFERS int GetCpuFlags(void) { int cpu_info = 0; @@ -491,7 +515,7 @@ static SAFEBUFFERS int GetCpuFlags(void) { } } #endif -#if defined(__loongarch__) && defined(__linux__) +#if defined(__loongarch__) cpu_info = LoongArchCpuCaps(); cpu_info |= kCpuHasLOONGARCH; #endif diff --git a/source/planar_functions.cc b/source/planar_functions.cc index de5bfbff8..b4f248f8b 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -158,6 +158,16 @@ void CopyPlane_16(const uint16_t* src_y, CopyRow = IS_ALIGNED(width, 16) ? CopyRow_16_NEON : CopyRow_16_Any_NEON; } #endif +#if defined(HAS_COPYROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + CopyRow = IS_ALIGNED(width, 16) ? CopyRow_16_LSX : CopyRow_16_Any_LSX; + } +#endif +#if defined(HAS_COPYROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_16_LASX : CopyRow_16_Any_LASX; + } +#endif #if defined(HAS_COPYROW_16_HVX) if (TestCpuFlag(kCpuHasHVX)) { CopyRow = CopyRow_16_HVX; @@ -240,6 +250,22 @@ void Convert16To8Plane(const uint16_t* src_y, Convert16To8Row = Convert16To8Row_HVX; } #endif +#if defined(HAS_CONVERT16TO8ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert16To8Row = Convert16To8Row_Any_LSX; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT16TO8ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert16To8Row = Convert16To8Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + Convert16To8Row = Convert16To8Row_LASX; + } + } +#endif // Convert plane for (y = 0; y < height; ++y) { @@ -340,6 +366,22 @@ void Convert8To16Plane( Convert8To16Row = Convert8To16Row_HVX; } #endif +#if defined(HAS_CONVERT8TO16ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert8To16Row = Convert8To16Row_Any_LSX; + if (IS_ALIGNED(width, 16)) { + Convert8To16Row = Convert8To16Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT8TO16ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert8To16Row = Convert8To16Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + Convert8To16Row = Convert8To16Row_LASX; + } + } +#endif // Convert plane for (y = 0; y < height; ++y) { @@ -415,6 +457,22 @@ void Convert8To8Plane(const uint8_t* src_y, Convert8To8Row = Convert8To8Row_HVX; } #endif +#if defined(HAS_CONVERT8TO8ROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Convert8To8Row = Convert8To8Row_Any_LSX; + if (IS_ALIGNED(width, 16)) { + Convert8To8Row = Convert8To8Row_LSX; + } + } +#endif +#if defined(HAS_CONVERT8TO8ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Convert8To8Row = Convert8To8Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + Convert8To8Row = Convert8To8Row_LASX; + } + } +#endif // Convert plane for (y = 0; y < height; ++y) { @@ -962,6 +1020,22 @@ void SplitUVPlane_16(const uint16_t* src_uv, SplitUVRow_16 = SplitUVRow_16_HVX; } #endif +#if defined(HAS_SPLITUVROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + SplitUVRow_16 = SplitUVRow_16_Any_LSX; + if (IS_ALIGNED(width, 8)) { + SplitUVRow_16 = SplitUVRow_16_LSX; + } + } +#endif +#if defined(HAS_SPLITUVROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SplitUVRow_16 = SplitUVRow_16_Any_LASX; + if (IS_ALIGNED(width, 16)) { + SplitUVRow_16 = SplitUVRow_16_LASX; + } + } +#endif for (y = 0; y < height; ++y) { // Copy a row of UV. @@ -1035,6 +1109,22 @@ void MergeUVPlane_16(const uint16_t* src_u, MergeUVRow_16 = MergeUVRow_16_HVX; } #endif +#if defined(HAS_MERGEUVROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + MergeUVRow_16 = MergeUVRow_16_Any_LSX; + if (IS_ALIGNED(width, 8)) { + MergeUVRow_16 = MergeUVRow_16_LSX; + } + } +#endif +#if defined(HAS_MERGEUVROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow_16 = MergeUVRow_16_Any_LASX; + if (IS_ALIGNED(width, 16)) { + MergeUVRow_16 = MergeUVRow_16_LASX; + } + } +#endif for (y = 0; y < height; ++y) { // Merge a row of U and V into a row of UV. @@ -1119,6 +1209,22 @@ void ConvertToMSBPlane_16(const uint16_t* src_y, MultiplyRow_16 = MultiplyRow_16_HVX; } #endif +#if defined(HAS_MULTIPLYROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + MultiplyRow_16 = MultiplyRow_16_Any_LSX; + if (IS_ALIGNED(width, 16)) { + MultiplyRow_16 = MultiplyRow_16_LSX; + } + } +#endif +#if defined(HAS_MULTIPLYROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MultiplyRow_16 = MultiplyRow_16_Any_LASX; + if (IS_ALIGNED(width, 32)) { + MultiplyRow_16 = MultiplyRow_16_LASX; + } + } +#endif for (y = 0; y < height; ++y) { MultiplyRow_16(src_y, dst_y, scale, width); @@ -1193,6 +1299,22 @@ void ConvertToLSBPlane_16(const uint16_t* src_y, DivideRow = DivideRow_16_HVX; } #endif +#if defined(HAS_DIVIDEROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + DivideRow = DivideRow_16_Any_LSX; + if (IS_ALIGNED(width, 16)) { + DivideRow = DivideRow_16_LSX; + } + } +#endif +#if defined(HAS_DIVIDEROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + DivideRow = DivideRow_16_Any_LASX; + if (IS_ALIGNED(width, 32)) { + DivideRow = DivideRow_16_LASX; + } + } +#endif for (y = 0; y < height; ++y) { DivideRow(src_y, dst_y, scale, width); @@ -1276,6 +1398,22 @@ void SwapUVPlane(const uint8_t* src_uv, SwapUVRow = SwapUVRow_HVX; } #endif +#if defined(HAS_SWAPUVROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + SwapUVRow = SwapUVRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + SwapUVRow = SwapUVRow_LSX; + } + } +#endif +#if defined(HAS_SWAPUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SwapUVRow = SwapUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + SwapUVRow = SwapUVRow_LASX; + } + } +#endif for (y = 0; y < height; ++y) { SwapUVRow(src_uv, dst_vu, width); @@ -1369,6 +1507,22 @@ int DetilePlane(const uint8_t* src_y, } } #endif +#if defined(HAS_DETILEROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + DetileRow = DetileRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + DetileRow = DetileRow_LSX; + } + } +#endif +#if defined(HAS_DETILEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + DetileRow = DetileRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + DetileRow = DetileRow_LASX; + } + } +#endif // Detile plane for (y = 0; y < height; ++y) { @@ -1426,6 +1580,22 @@ int DetilePlane_16(const uint16_t* src_y, } } #endif +#if defined(HAS_DETILEROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + DetileRow_16 = DetileRow_16_Any_LSX; + if (IS_ALIGNED(width, 16)) { + DetileRow_16 = DetileRow_16_LSX; + } + } +#endif +#if defined(HAS_DETILEROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + DetileRow_16 = DetileRow_16_Any_LASX; + if (IS_ALIGNED(width, 16)) { + DetileRow_16 = DetileRow_16_LASX; + } + } +#endif // Detile plane for (y = 0; y < height; ++y) { @@ -1799,6 +1969,22 @@ static void SplitARGBPlaneAlpha(const uint8_t* src_argb, SplitARGBRow = SplitARGBRow_RVV; } #endif +#if defined(HAS_SPLITARGBROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + SplitARGBRow = SplitARGBRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + SplitARGBRow = SplitARGBRow_LSX; + } + } +#endif +#if defined(HAS_SPLITARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SplitARGBRow = SplitARGBRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + SplitARGBRow = SplitARGBRow_LASX; + } + } +#endif #if defined(HAS_SPLITARGBROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { SplitARGBRow = SplitARGBRow_HVX; @@ -1879,6 +2065,22 @@ static void SplitARGBPlaneOpaque(const uint8_t* src_argb, SplitXRGBRow = SplitXRGBRow_RVV; } #endif +#if defined(HAS_SPLITXRGBROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + SplitXRGBRow = SplitXRGBRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + SplitXRGBRow = SplitXRGBRow_LSX; + } + } +#endif +#if defined(HAS_SPLITXRGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SplitXRGBRow = SplitXRGBRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + SplitXRGBRow = SplitXRGBRow_LASX; + } + } +#endif #if defined(HAS_SPLITXRGBROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { SplitXRGBRow = SplitXRGBRow_HVX; @@ -1995,6 +2197,22 @@ static void MergeARGBPlaneAlpha(const uint8_t* src_r, MergeARGBRow = MergeARGBRow_RVV; } #endif +#if defined(HAS_MERGEARGBROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + MergeARGBRow = MergeARGBRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + MergeARGBRow = MergeARGBRow_LSX; + } + } +#endif +#if defined(HAS_MERGEARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeARGBRow = MergeARGBRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + MergeARGBRow = MergeARGBRow_LASX; + } + } +#endif #if defined(HAS_MERGEARGBROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { MergeARGBRow = MergeARGBRow_HVX; @@ -2067,6 +2285,22 @@ static void MergeARGBPlaneOpaque(const uint8_t* src_r, MergeXRGBRow = MergeXRGBRow_RVV; } #endif +#if defined(HAS_MERGEXRGBROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + MergeXRGBRow = MergeXRGBRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + MergeXRGBRow = MergeXRGBRow_LSX; + } + } +#endif +#if defined(HAS_MERGEXRGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeXRGBRow = MergeXRGBRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + MergeXRGBRow = MergeXRGBRow_LASX; + } + } +#endif #if defined(HAS_MERGEXRGBROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { MergeXRGBRow = MergeXRGBRow_HVX; @@ -3462,6 +3696,22 @@ int RGB24Mirror(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGB24MIRRORROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + RGB24MirrorRow = RGB24MirrorRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + RGB24MirrorRow = RGB24MirrorRow_LSX; + } + } +#endif +#if defined(HAS_RGB24MIRRORROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + RGB24MirrorRow = RGB24MirrorRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + RGB24MirrorRow = RGB24MirrorRow_LASX; + } + } +#endif // Mirror plane for (y = 0; y < height; ++y) { @@ -3752,6 +4002,22 @@ int BlendPlane(const uint8_t* src_y0, BlendPlaneRow = BlendPlaneRow_HVX; } #endif +#if defined(HAS_BLENDPLANEROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + BlendPlaneRow = BlendPlaneRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + BlendPlaneRow = BlendPlaneRow_LSX; + } + } +#endif +#if defined(HAS_BLENDPLANEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + BlendPlaneRow = BlendPlaneRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + BlendPlaneRow = BlendPlaneRow_LASX; + } + } +#endif for (y = 0; y < height; ++y) { BlendPlaneRow(src_y0, src_y1, alpha, dst_y, width); @@ -3865,6 +4131,22 @@ int I420Blend(const uint8_t* src_y0, if (TestCpuFlag(kCpuHasHVX)) { BlendPlaneRow = BlendPlaneRow_HVX; } +#endif +#if defined(HAS_BLENDPLANEROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + BlendPlaneRow = BlendPlaneRow_Any_LSX; + if (IS_ALIGNED(halfwidth, 16)) { + BlendPlaneRow = BlendPlaneRow_LSX; + } + } +#endif +#if defined(HAS_BLENDPLANEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + BlendPlaneRow = BlendPlaneRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + BlendPlaneRow = BlendPlaneRow_LASX; + } + } #endif if (!IS_ALIGNED(width, 2)) { ScaleRowDown2 = ScaleRowDown2Box_Odd_C; @@ -3933,6 +4215,26 @@ int I420Blend(const uint8_t* src_y0, IS_ALIGNED(width, 2) ? ScaleRowDown2Box_HVX : ScaleRowDown2Box_Odd_HVX; } #endif +#if defined(HAS_SCALEROWDOWN2_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + if (IS_ALIGNED(width, 2)) { + ScaleRowDown2 = ScaleRowDown2Box_Any_LSX; + if (IS_ALIGNED(halfwidth, 16)) { + ScaleRowDown2 = ScaleRowDown2Box_LSX; + } + } + } +#endif +#if defined(HAS_SCALEROWDOWN2_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + if (IS_ALIGNED(width, 2)) { + ScaleRowDown2 = ScaleRowDown2Box_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + ScaleRowDown2 = ScaleRowDown2Box_LASX; + } + } + } +#endif // Row buffer for intermediate alpha pixels. align_buffer_64(halfalpha, halfwidth); @@ -5711,6 +6013,22 @@ int InterpolatePlane_16(const uint16_t* src0, InterpolateRow_16 = InterpolateRow_16_HVX; } #endif +#if defined(HAS_INTERPOLATEROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + InterpolateRow_16 = InterpolateRow_16_Any_LSX; + if (IS_ALIGNED(width, 8)) { + InterpolateRow_16 = InterpolateRow_16_LSX; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow_16 = InterpolateRow_16_Any_LASX; + if (IS_ALIGNED(width, 16)) { + InterpolateRow_16 = InterpolateRow_16_LASX; + } + } +#endif for (y = 0; y < height; ++y) { InterpolateRow_16(dst, src0, src1 - src0, width, interpolation); @@ -6694,6 +7012,22 @@ int ARGBCopyAlpha(const uint8_t* src_argb, ARGBCopyAlphaRow = ARGBCopyAlphaRow_HVX; } #endif +#if defined(HAS_ARGBCOPYALPHAROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_LSX; + if (IS_ALIGNED(width, 8)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_LSX; + } + } +#endif +#if defined(HAS_ARGBCOPYALPHAROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_LASX; + } + } +#endif for (y = 0; y < height; ++y) { ARGBCopyAlphaRow(src_argb, dst_argb, width); @@ -6861,6 +7195,22 @@ int ARGBCopyYToAlpha(const uint8_t* src_y, ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_HVX; } #endif +#if defined(HAS_ARGBCOPYYTOALPHAROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_Any_LSX; + if (IS_ALIGNED(width, 8)) { + ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_LSX; + } + } +#endif +#if defined(HAS_ARGBCOPYYTOALPHAROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_LASX; + } + } +#endif for (y = 0; y < height; ++y) { ARGBCopyYToAlphaRow(src_y, dst_argb, width); @@ -6981,6 +7331,23 @@ int YUY2ToNV12(const uint8_t* src_yuy2, } #endif +#if defined(HAS_YUY2TONVUVROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + YUY2ToNVUVRow = YUY2ToNVUVRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + YUY2ToNVUVRow = YUY2ToNVUVRow_LSX; + } + } +#endif +#if defined(HAS_YUY2TONVUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + YUY2ToNVUVRow = YUY2ToNVUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + YUY2ToNVUVRow = YUY2ToNVUVRow_LASX; + } + } +#endif + for (y = 0; y < height - 1; y += 2) { YUY2ToYRow(src_yuy2, dst_y, width); YUY2ToYRow(src_yuy2 + src_stride_yuy2, dst_y + dst_stride_y, width); @@ -7254,6 +7621,22 @@ void HalfMergeUVPlane(const uint8_t* src_u, HalfMergeUVRow = HalfMergeUVRow_AVX512BW; } #endif +#if defined(HAS_HALFMERGEUVROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + HalfMergeUVRow = HalfMergeUVRow_Any_LSX; + if (IS_ALIGNED(width, 16)) { + HalfMergeUVRow = HalfMergeUVRow_LSX; + } + } +#endif +#if defined(HAS_HALFMERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + HalfMergeUVRow = HalfMergeUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + HalfMergeUVRow = HalfMergeUVRow_LASX; + } + } +#endif for (y = 0; y < height - 1; y += 2) { // Merge a row of U and V into a row of UV. diff --git a/source/rotate.cc b/source/rotate.cc index 6574ac054..5ec9b9302 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -123,6 +123,14 @@ void TransposePlane(const uint8_t* src, } } #endif +#if defined(HAS_TRANSPOSEWX8_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + TransposeWx8 = TransposeWx8_Any_LASX; + if (IS_ALIGNED(width, 32)) { + TransposeWx8 = TransposeWx8_LASX; + } + } +#endif #if defined(HAS_TRANSPOSEWX16_LASX) if (TestCpuFlag(kCpuHasLASX)) { TransposeWx16 = TransposeWx16_Any_LASX; @@ -409,6 +417,14 @@ void SplitTransposeUV(const uint8_t* src, } } #endif +#if defined(HAS_TRANSPOSEUVWX8_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + TransposeUVWx8 = TransposeUVWx8_Any_LASX; + if (IS_ALIGNED(width, 16)) { + TransposeUVWx8 = TransposeUVWx8_LASX; + } + } +#endif #if defined(HAS_TRANSPOSEUVWX16_LASX) if (TestCpuFlag(kCpuHasLASX)) { TransposeUVWx16 = TransposeUVWx16_Any_LASX; @@ -820,6 +836,14 @@ static void TransposePlane_16(const uint16_t* src, } } #endif +#if defined(HAS_TRANSPOSEWX8_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + TransposeWx8_16 = TransposeWx8_16_Any_LASX; + if (IS_ALIGNED(width, 16)) { + TransposeWx8_16 = TransposeWx8_16_LASX; + } + } +#endif #if defined(HAS_TRANSPOSEWX16_16_LASX) if (TestCpuFlag(kCpuHasLASX)) { TransposeWx16_16 = TransposeWx16_16_Any_LASX; diff --git a/source/rotate_any.cc b/source/rotate_any.cc index bc203655a..1f75a2ed7 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -43,6 +43,9 @@ TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15) #ifdef HAS_TRANSPOSEWX8_LSX TANY(TransposeWx8_Any_LSX, TransposeWx8_LSX, TransposeWx8_C, 15) #endif +#ifdef HAS_TRANSPOSEWX8_LASX +TANY(TransposeWx8_Any_LASX, TransposeWx8_LASX, TransposeWx8_C, 31) +#endif #ifdef HAS_TRANSPOSEWX16_LASX TANY(TransposeWx16_Any_LASX, TransposeWx16_LASX, TransposeWx16_C, 15) #endif @@ -80,6 +83,9 @@ TUVANY(TransposeUVWx16_Any_AVX2, TransposeUVWx16_AVX2, TransposeUVWx16_C, 7) #ifdef HAS_TRANSPOSEUVWX8_LSX TUVANY(TransposeUVWx8_Any_LSX, TransposeUVWx8_LSX, TransposeUVWx8_C, 7) #endif +#ifdef HAS_TRANSPOSEUVWX8_LASX +TUVANY(TransposeUVWx8_Any_LASX, TransposeUVWx8_LASX, TransposeUVWx8_C, 15) +#endif #ifdef HAS_TRANSPOSEUVWX16_LASX TUVANY(TransposeUVWx16_Any_LASX, TransposeUVWx16_LASX, TransposeUVWx16_C, 7) #endif @@ -175,6 +181,9 @@ TANY16(TransposeWx16_16_Any_AVX2, TransposeWx16_16_AVX2, 7, 16) #ifdef HAS_TRANSPOSEWX8_16_LSX TANY16(TransposeWx8_16_Any_LSX, TransposeWx8_16_LSX, 7, 8) #endif +#ifdef HAS_TRANSPOSEWX8_16_LASX +TANY16(TransposeWx8_16_Any_LASX, TransposeWx8_16_LASX, 15, 8) +#endif #ifdef HAS_TRANSPOSEWX16_16_LASX TANY16(TransposeWx16_16_Any_LASX, TransposeWx16_16_LASX, 7, 16) #endif diff --git a/source/rotate_la.cc b/source/rotate_la.cc index 39c0a56bf..f25b6a1ca 100644 --- a/source/rotate_la.cc +++ b/source/rotate_la.cc @@ -385,6 +385,404 @@ void TransposeWx8_16_LSX(const uint16_t* src, } #endif // HAS_TRANSPOSEWX8_16_LSX +#ifdef HAS_TRANSPOSEWX8_LASX +void TransposeWx8_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 5 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "xvld $xr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr7, %[t], 0 \n" + "addi.d %[src], %[src], 32 \n" + + "xvilvl.b $xr8, $xr1, $xr0 \n" + "xvilvh.b $xr9, $xr1, $xr0 \n" + "xvilvl.b $xr10, $xr3, $xr2 \n" + "xvilvh.b $xr11, $xr3, $xr2 \n" + "xvilvl.b $xr12, $xr5, $xr4 \n" + "xvilvh.b $xr13, $xr5, $xr4 \n" + "xvilvl.b $xr14, $xr7, $xr6 \n" + "xvilvh.b $xr15, $xr7, $xr6 \n" + + "xvilvl.h $xr0, $xr10, $xr8 \n" + "xvilvh.h $xr1, $xr10, $xr8 \n" + "xvilvl.h $xr2, $xr11, $xr9 \n" + "xvilvh.h $xr3, $xr11, $xr9 \n" + "xvilvl.h $xr4, $xr14, $xr12 \n" + "xvilvh.h $xr5, $xr14, $xr12 \n" + "xvilvl.h $xr6, $xr15, $xr13 \n" + "xvilvh.h $xr7, $xr15, $xr13 \n" + + "xvilvl.w $xr8, $xr4, $xr0 \n" + "xvilvh.w $xr9, $xr4, $xr0 \n" + "xvilvl.w $xr10, $xr5, $xr1 \n" + "xvilvh.w $xr11, $xr5, $xr1 \n" + "xvilvl.w $xr12, $xr6, $xr2 \n" + "xvilvh.w $xr13, $xr6, $xr2 \n" + "xvilvl.w $xr14, $xr7, $xr3 \n" + "xvilvh.w $xr15, $xr7, $xr3 \n" + + "xvstelm.d $xr8, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr8, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr9, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr9, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr10, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr10, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr11, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr11, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr12, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr12, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr13, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr13, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr14, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr14, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr15, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr15, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + + "xvstelm.d $xr8, %[dst], 0, 2 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr8, %[dst], 0, 3 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr9, %[dst], 0, 2 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr9, %[dst], 0, 3 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr10, %[dst], 0, 2 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr10, %[dst], 0, 3 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr11, %[dst], 0, 2 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr11, %[dst], 0, 3 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr12, %[dst], 0, 2 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr12, %[dst], 0, 3 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr13, %[dst], 0, 2 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr13, %[dst], 0, 3 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr14, %[dst], 0, 2 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr14, %[dst], 0, 3 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr15, %[dst], 0, 2 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvstelm.d $xr15, %[dst], 0, 3 \n" + "add.d %[dst], %[dst], %[d] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15"); +} +#endif // HAS_TRANSPOSEWX8_LASX + +#ifdef HAS_TRANSPOSEUVWX8_LASX +void TransposeUVWx8_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t da = dst_stride_a; + const ptrdiff_t db = dst_stride_b; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "xvld $xr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr7, %[t], 0 \n" + "addi.d %[src], %[src], 32 \n" + + "xvilvl.b $xr8, $xr1, $xr0 \n" + "xvilvh.b $xr9, $xr1, $xr0 \n" + "xvilvl.b $xr10, $xr3, $xr2 \n" + "xvilvh.b $xr11, $xr3, $xr2 \n" + "xvilvl.b $xr12, $xr5, $xr4 \n" + "xvilvh.b $xr13, $xr5, $xr4 \n" + "xvilvl.b $xr14, $xr7, $xr6 \n" + "xvilvh.b $xr15, $xr7, $xr6 \n" + + "xvilvl.h $xr0, $xr10, $xr8 \n" + "xvilvh.h $xr1, $xr10, $xr8 \n" + "xvilvl.h $xr2, $xr11, $xr9 \n" + "xvilvh.h $xr3, $xr11, $xr9 \n" + "xvilvl.h $xr4, $xr14, $xr12 \n" + "xvilvh.h $xr5, $xr14, $xr12 \n" + "xvilvl.h $xr6, $xr15, $xr13 \n" + "xvilvh.h $xr7, $xr15, $xr13 \n" + + "xvilvl.w $xr8, $xr4, $xr0 \n" + "xvilvh.w $xr9, $xr4, $xr0 \n" + "xvilvl.w $xr10, $xr5, $xr1 \n" + "xvilvh.w $xr11, $xr5, $xr1 \n" + "xvilvl.w $xr12, $xr6, $xr2 \n" + "xvilvh.w $xr13, $xr6, $xr2 \n" + "xvilvl.w $xr14, $xr7, $xr3 \n" + "xvilvh.w $xr15, $xr7, $xr3 \n" + + "xvstelm.d $xr8, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr8, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr9, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr9, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr10, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr10, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr11, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr11, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr12, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr12, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr13, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr13, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr14, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr14, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr15, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr15, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + + "xvstelm.d $xr8, %[dst_a], 0, 2 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr8, %[dst_b], 0, 3 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr9, %[dst_a], 0, 2 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr9, %[dst_b], 0, 3 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr10, %[dst_a], 0, 2 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr10, %[dst_b], 0, 3 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr11, %[dst_a], 0, 2 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr11, %[dst_b], 0, 3 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr12, %[dst_a], 0, 2 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr12, %[dst_b], 0, 3 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr13, %[dst_a], 0, 2 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr13, %[dst_b], 0, 3 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr14, %[dst_a], 0, 2 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr14, %[dst_b], 0, 3 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "xvstelm.d $xr15, %[dst_a], 0, 2 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "xvstelm.d $xr15, %[dst_b], 0, 3 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst_a] "+r"(dst_a), // %[dst_a] + [dst_b] "+r"(dst_b), // %[dst_b] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [da] "r"(da), // %[da] + [db] "r"(db) // %[db] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15"); +} +#endif // HAS_TRANSPOSEUVWX8_LASX + +#ifdef HAS_TRANSPOSEWX8_16_LASX +void TransposeWx8_16_LASX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "xvld $xr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "xvld $xr7, %[t], 0 \n" + "addi.d %[src], %[src], 32 \n" + + "xvilvl.h $xr8, $xr1, $xr0 \n" + "xvilvh.h $xr9, $xr1, $xr0 \n" + "xvilvl.h $xr10, $xr3, $xr2 \n" + "xvilvh.h $xr11, $xr3, $xr2 \n" + "xvilvl.h $xr12, $xr5, $xr4 \n" + "xvilvh.h $xr13, $xr5, $xr4 \n" + "xvilvl.h $xr14, $xr7, $xr6 \n" + "xvilvh.h $xr15, $xr7, $xr6 \n" + + "xvilvl.w $xr0, $xr10, $xr8 \n" + "xvilvh.w $xr1, $xr10, $xr8 \n" + "xvilvl.w $xr2, $xr11, $xr9 \n" + "xvilvh.w $xr3, $xr11, $xr9 \n" + "xvilvl.w $xr4, $xr14, $xr12 \n" + "xvilvh.w $xr5, $xr14, $xr12 \n" + "xvilvl.w $xr6, $xr15, $xr13 \n" + "xvilvh.w $xr7, $xr15, $xr13 \n" + + "xvilvl.d $xr8, $xr4, $xr0 \n" + "xvilvh.d $xr9, $xr4, $xr0 \n" + "xvilvl.d $xr10, $xr5, $xr1 \n" + "xvilvh.d $xr11, $xr5, $xr1 \n" + "xvilvl.d $xr12, $xr6, $xr2 \n" + "xvilvh.d $xr13, $xr6, $xr2 \n" + "xvilvl.d $xr14, $xr7, $xr3 \n" + "xvilvh.d $xr15, $xr7, $xr3 \n" + + "vst $vr8, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr9, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr10, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr11, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr12, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr13, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr14, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr15, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + + "xvpermi.d $xr0, $xr8, 0x4e \n" + "xvpermi.d $xr1, $xr9, 0x4e \n" + "xvpermi.d $xr2, $xr10, 0x4e \n" + "xvpermi.d $xr3, $xr11, 0x4e \n" + "xvpermi.d $xr4, $xr12, 0x4e \n" + "xvpermi.d $xr5, $xr13, 0x4e \n" + "xvpermi.d $xr6, $xr14, 0x4e \n" + "xvpermi.d $xr7, $xr15, 0x4e \n" + + "vst $vr0, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr1, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr2, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr3, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr4, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr5, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr6, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr7, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_b), // %[src] + [dst] "+r"(dst_b), // %[dst] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15"); +} +#endif // HAS_TRANSPOSEWX8_16_LASX + #ifdef HAS_TRANSPOSEWX16_LASX void TransposeWx16_LASX(const uint8_t* src, int src_stride, diff --git a/source/row_any.cc b/source/row_any.cc index 7957c898e..480dda871 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -62,6 +62,12 @@ ANY41(MergeARGBRow_Any_AVX2, MergeARGBRow_AVX2, 0, 0, 4, 15) #ifdef HAS_MERGEARGBROW_NEON ANY41(MergeARGBRow_Any_NEON, MergeARGBRow_NEON, 0, 0, 4, 15) #endif +#ifdef HAS_MERGEARGBROW_LSX +ANY41(MergeARGBRow_Any_LSX, MergeARGBRow_LSX, 0, 0, 4, 15) +#endif +#ifdef HAS_MERGEARGBROW_LASX +ANY41(MergeARGBRow_Any_LASX, MergeARGBRow_LASX, 0, 0, 4, 15) +#endif // Note that odd width replication includes 444 due to implementation // on arm that subsamples 444 to 422 internally. @@ -311,6 +317,12 @@ ANY31(MergeXRGBRow_Any_AVX2, MergeXRGBRow_AVX2, 0, 0, 4, 15) #ifdef HAS_MERGEXRGBROW_NEON ANY31(MergeXRGBRow_Any_NEON, MergeXRGBRow_NEON, 0, 0, 4, 15) #endif +#ifdef HAS_MERGEXRGBROW_LSX +ANY31(MergeXRGBRow_Any_LSX, MergeXRGBRow_LSX, 0, 0, 4, 15) +#endif +#ifdef HAS_MERGEXRGBROW_LASX +ANY31(MergeXRGBRow_Any_LASX, MergeXRGBRow_LASX, 0, 0, 4, 15) +#endif #ifdef HAS_I422TOYUY2ROW_SSE2 ANY31(I422ToYUY2Row_Any_SSE2, I422ToYUY2Row_SSE2, 1, 1, 4, 15) ANY31(I422ToUYVYRow_Any_SSE2, I422ToUYVYRow_SSE2, 1, 1, 4, 15) @@ -346,6 +358,12 @@ ANY31(BlendPlaneRow_Any_SSSE3, BlendPlaneRow_SSSE3, 0, 0, 1, 7) #ifdef HAS_BLENDPLANEROW_NEON ANY31(BlendPlaneRow_Any_NEON, BlendPlaneRow_NEON, 0, 0, 1, 15) #endif +#ifdef HAS_BLENDPLANEROW_LSX +ANY31(BlendPlaneRow_Any_LSX, BlendPlaneRow_LSX, 0, 0, 1, 15) +#endif +#ifdef HAS_BLENDPLANEROW_LASX +ANY31(BlendPlaneRow_Any_LASX, BlendPlaneRow_LASX, 0, 0, 1, 31) +#endif #ifdef HAS_BLENDPLANEROW_WASMSIMD ANY31(BlendPlaneRow_Any_WASMSIMD, BlendPlaneRow_WASMSIMD, 0, 0, 1, 15) #endif @@ -839,6 +857,12 @@ ANY21S(YUY2ToNVUVRow_Any_SSE2, YUY2ToNVUVRow_SSE2, 4, 2, 7) #ifdef HAS_YUY2TONVUVROW_AVX2 ANY21S(YUY2ToNVUVRow_Any_AVX2, YUY2ToNVUVRow_AVX2, 4, 2, 15) #endif +#ifdef HAS_YUY2TONVUVROW_LSX +ANY21S(YUY2ToNVUVRow_Any_LSX, YUY2ToNVUVRow_LSX, 4, 2, 7) +#endif +#ifdef HAS_YUY2TONVUVROW_LASX +ANY21S(YUY2ToNVUVRow_Any_LASX, YUY2ToNVUVRow_LASX, 4, 2, 15) +#endif #undef ANY21S // Any 2 planes to 1 half-size merged UV row with stride (2 rows of U and V). @@ -881,6 +905,12 @@ ANY21HMS(HalfMergeUVRow_Any_AVX2, HalfMergeUVRow_AVX2, 31) #ifdef HAS_HALFMERGEUVROW_NEON ANY21HMS(HalfMergeUVRow_Any_NEON, HalfMergeUVRow_NEON, 15) #endif +#ifdef HAS_HALFMERGEUVROW_LSX +ANY21HMS(HalfMergeUVRow_Any_LSX, HalfMergeUVRow_LSX, 15) +#endif +#ifdef HAS_HALFMERGEUVROW_LASX +ANY21HMS(HalfMergeUVRow_Any_LASX, HalfMergeUVRow_LASX, 31) +#endif #undef ANY21HMS // Any 2 planes to 1 with yuvconstants @@ -1054,7 +1084,7 @@ ANY21CT(P410ToARGBRow_Any_NEON, P410ToARGBRow_NEON, 0, 0, uint16_t, 2, 4, 7) SIMD_ALIGNED(T vin[16 * 2]); \ static_assert((MASK + 1) * BPP <= sizeof(vin) / 2, \ "vin buffer too small"); \ - SIMD_ALIGNED(T vout[16]); \ + SIMD_ALIGNED(T vout[16 * 2]); \ static_assert((MASK + 1) * BPP * 2 <= sizeof(vout), \ "vout buffer too small"); \ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \ @@ -1077,6 +1107,12 @@ ANY21PT(MergeUVRow_16_Any_AVX2, MergeUVRow_16_AVX2, uint16_t, 2, 7) #ifdef HAS_MERGEUVROW_16_NEON ANY21PT(MergeUVRow_16_Any_NEON, MergeUVRow_16_NEON, uint16_t, 2, 7) #endif +#ifdef HAS_MERGEUVROW_16_LSX +ANY21PT(MergeUVRow_16_Any_LSX, MergeUVRow_16_LSX, uint16_t, 2, 7) +#endif +#ifdef HAS_MERGEUVROW_16_LASX +ANY21PT(MergeUVRow_16_Any_LASX, MergeUVRow_16_LASX, uint16_t, 2, 15) +#endif #undef ANY21CT @@ -1172,6 +1208,36 @@ ANY11(ABGRToAR30Row_Any_AVX2, ABGRToAR30Row_AVX2, 0, 4, 4, 7) #if defined(HAS_ARGBTOAR30ROW_AVX2) ANY11(ARGBToAR30Row_Any_AVX2, ARGBToAR30Row_AVX2, 0, 4, 4, 7) #endif +#if defined(HAS_ABGRTOAR30ROW_LSX) +ANY11(ABGRToAR30Row_Any_LSX, ABGRToAR30Row_LSX, 0, 4, 4, 3) +#endif +#if defined(HAS_ABGRTOAR30ROW_LASX) +ANY11(ABGRToAR30Row_Any_LASX, ABGRToAR30Row_LASX, 0, 4, 4, 7) +#endif +#if defined(HAS_ARGBTOAR30ROW_LSX) +ANY11(ARGBToAR30Row_Any_LSX, ARGBToAR30Row_LSX, 0, 4, 4, 3) +#endif +#if defined(HAS_ARGBTOAR30ROW_LASX) +ANY11(ARGBToAR30Row_Any_LASX, ARGBToAR30Row_LASX, 0, 4, 4, 7) +#endif +#if defined(HAS_AR30TOARGBROW_LSX) +ANY11(AR30ToARGBRow_Any_LSX, AR30ToARGBRow_LSX, 0, 4, 4, 3) +#endif +#if defined(HAS_AR30TOARGBROW_LASX) +ANY11(AR30ToARGBRow_Any_LASX, AR30ToARGBRow_LASX, 0, 4, 4, 7) +#endif +#if defined(HAS_AR30TOABGRROW_LSX) +ANY11(AR30ToABGRRow_Any_LSX, AR30ToABGRRow_LSX, 0, 4, 4, 3) +#endif +#if defined(HAS_AR30TOABGRROW_LASX) +ANY11(AR30ToABGRRow_Any_LASX, AR30ToABGRRow_LASX, 0, 4, 4, 7) +#endif +#if defined(HAS_AR30TOAB30ROW_LSX) +ANY11(AR30ToAB30Row_Any_LSX, AR30ToAB30Row_LSX, 0, 4, 4, 3) +#endif +#if defined(HAS_AR30TOAB30ROW_LASX) +ANY11(AR30ToAB30Row_Any_LASX, AR30ToAB30Row_LASX, 0, 4, 4, 7) +#endif #if defined(HAS_J400TOARGBROW_AVX2) ANY11(J400ToARGBRow_Any_AVX2, J400ToARGBRow_AVX2, 0, 1, 4, 15) @@ -1319,6 +1385,12 @@ ANY11(AYUVToYRow_Any_AVX2, AYUVToYRow_AVX2, 0, 4, 1, 15) #ifdef HAS_AYUVTOYROW_NEON ANY11(AYUVToYRow_Any_NEON, AYUVToYRow_NEON, 0, 4, 1, 15) #endif +#ifdef HAS_AYUVTOYROW_LSX +ANY11(AYUVToYRow_Any_LSX, AYUVToYRow_LSX, 0, 4, 1, 15) +#endif +#ifdef HAS_AYUVTOYROW_LASX +ANY11(AYUVToYRow_Any_LASX, AYUVToYRow_LASX, 0, 4, 1, 15) +#endif #ifdef HAS_SWAPUVROW_SSSE3 ANY11(SwapUVRow_Any_SSSE3, SwapUVRow_SSSE3, 0, 2, 2, 15) #endif @@ -1328,6 +1400,12 @@ ANY11(SwapUVRow_Any_AVX2, SwapUVRow_AVX2, 0, 2, 2, 31) #ifdef HAS_SWAPUVROW_NEON ANY11(SwapUVRow_Any_NEON, SwapUVRow_NEON, 0, 2, 2, 15) #endif +#ifdef HAS_SWAPUVROW_LSX +ANY11(SwapUVRow_Any_LSX, SwapUVRow_LSX, 0, 2, 2, 15) +#endif +#ifdef HAS_SWAPUVROW_LASX +ANY11(SwapUVRow_Any_LASX, SwapUVRow_LASX, 0, 2, 2, 31) +#endif #ifdef HAS_RGB24TOARGBROW_NEON ANY11(RGB24ToARGBRow_Any_NEON, RGB24ToARGBRow_NEON, 0, 3, 4, 7) #endif @@ -1349,6 +1427,12 @@ ANY11(RAWToARGBRow_Any_LSX, RAWToARGBRow_LSX, 0, 3, 4, 15) #ifdef HAS_RAWTOARGBROW_LASX ANY11(RAWToARGBRow_Any_LASX, RAWToARGBRow_LASX, 0, 3, 4, 31) #endif +#ifdef HAS_RAWTORGBAROW_LSX +ANY11(RAWToRGBARow_Any_LSX, RAWToRGBARow_LSX, 0, 3, 4, 15) +#endif +#ifdef HAS_RAWTORGBAROW_LASX +ANY11(RAWToRGBARow_Any_LASX, RAWToRGBARow_LASX, 0, 3, 4, 31) +#endif #ifdef HAS_RGB565TOARGBROW_NEON ANY11(RGB565ToARGBRow_Any_NEON, RGB565ToARGBRow_NEON, 0, 2, 4, 15) #endif @@ -1498,6 +1582,12 @@ ANY11B(ARGBCopyAlphaRow_Any_SSE2, ARGBCopyAlphaRow_SSE2, 0, 4, 4, 7) #ifdef HAS_ARGBCOPYALPHAROW_NEON ANY11B(ARGBCopyAlphaRow_Any_NEON, ARGBCopyAlphaRow_NEON, 0, 4, 4, 7) #endif +#ifdef HAS_ARGBCOPYALPHAROW_LSX +ANY11B(ARGBCopyAlphaRow_Any_LSX, ARGBCopyAlphaRow_LSX, 0, 4, 4, 7) +#endif +#ifdef HAS_ARGBCOPYALPHAROW_LASX +ANY11B(ARGBCopyAlphaRow_Any_LASX, ARGBCopyAlphaRow_LASX, 0, 4, 4, 15) +#endif #ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2 ANY11B(ARGBCopyYToAlphaRow_Any_AVX2, ARGBCopyYToAlphaRow_AVX2, 0, 1, 4, 15) #endif @@ -1507,6 +1597,12 @@ ANY11B(ARGBCopyYToAlphaRow_Any_SSE2, ARGBCopyYToAlphaRow_SSE2, 0, 1, 4, 7) #ifdef HAS_ARGBCOPYYTOALPHAROW_NEON ANY11B(ARGBCopyYToAlphaRow_Any_NEON, ARGBCopyYToAlphaRow_NEON, 0, 1, 4, 15) #endif +#ifdef HAS_ARGBCOPYYTOALPHAROW_LSX +ANY11B(ARGBCopyYToAlphaRow_Any_LSX, ARGBCopyYToAlphaRow_LSX, 0, 1, 4, 7) +#endif +#ifdef HAS_ARGBCOPYYTOALPHAROW_LASX +ANY11B(ARGBCopyYToAlphaRow_Any_LASX, ARGBCopyYToAlphaRow_LASX, 0, 1, 4, 15) +#endif #ifdef HAS_ARGBCOPYYTOALPHAROW_WASMSIMD ANY11B(ARGBCopyYToAlphaRow_Any_WASMSIMD, ARGBCopyYToAlphaRow_WASMSIMD, @@ -1888,6 +1984,30 @@ ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) #ifdef HAS_AB64TOARGBROW_NEON ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) #endif +#ifdef HAS_ARGBTOAR64ROW_LSX +ANY11T(ARGBToAR64Row_Any_LSX, ARGBToAR64Row_LSX, 4, 8, uint8_t, uint16_t, 3) +#endif +#ifdef HAS_ARGBTOAR64ROW_LASX +ANY11T(ARGBToAR64Row_Any_LASX, ARGBToAR64Row_LASX, 4, 8, uint8_t, uint16_t, 7) +#endif +#ifdef HAS_ARGBTOAB64ROW_LSX +ANY11T(ARGBToAB64Row_Any_LSX, ARGBToAB64Row_LSX, 4, 8, uint8_t, uint16_t, 3) +#endif +#ifdef HAS_ARGBTOAB64ROW_LASX +ANY11T(ARGBToAB64Row_Any_LASX, ARGBToAB64Row_LASX, 4, 8, uint8_t, uint16_t, 7) +#endif +#ifdef HAS_AR64TOARGBROW_LSX +ANY11T(AR64ToARGBRow_Any_LSX, AR64ToARGBRow_LSX, 8, 4, uint16_t, uint8_t, 3) +#endif +#ifdef HAS_AR64TOARGBROW_LASX +ANY11T(AR64ToARGBRow_Any_LASX, AR64ToARGBRow_LASX, 8, 4, uint16_t, uint8_t, 7) +#endif +#ifdef HAS_AB64TOARGBROW_LSX +ANY11T(AB64ToARGBRow_Any_LSX, AB64ToARGBRow_LSX, 8, 4, uint16_t, uint8_t, 3) +#endif +#ifdef HAS_AB64TOARGBROW_LASX +ANY11T(AB64ToARGBRow_Any_LASX, AB64ToARGBRow_LASX, 8, 4, uint16_t, uint8_t, 7) +#endif #ifdef HAS_COPYROW_16_AVX2 ANY11T(CopyRow_16_Any_AVX2, CopyRow_16_AVX2, 2, 2, uint16_t, uint16_t, 31) @@ -1895,6 +2015,12 @@ ANY11T(CopyRow_16_Any_AVX2, CopyRow_16_AVX2, 2, 2, uint16_t, uint16_t, 31) #ifdef HAS_COPYROW_16_NEON ANY11T(CopyRow_16_Any_NEON, CopyRow_16_NEON, 2, 2, uint16_t, uint16_t, 15) #endif +#ifdef HAS_COPYROW_16_LSX +ANY11T(CopyRow_16_Any_LSX, CopyRow_16_LSX, 2, 2, uint16_t, uint16_t, 15) +#endif +#ifdef HAS_COPYROW_16_LASX +ANY11T(CopyRow_16_Any_LASX, CopyRow_16_LASX, 2, 2, uint16_t, uint16_t, 31) +#endif #undef ANY11T @@ -1945,6 +2071,24 @@ ANY11C(Convert16To8Row_Any_NEON, uint8_t, 15) #endif +#ifdef HAS_CONVERT16TO8ROW_LSX +ANY11C(Convert16To8Row_Any_LSX, + Convert16To8Row_LSX, + 2, + 1, + uint16_t, + uint8_t, + 15) +#endif +#ifdef HAS_CONVERT16TO8ROW_LASX +ANY11C(Convert16To8Row_Any_LASX, + Convert16To8Row_LASX, + 2, + 1, + uint16_t, + uint8_t, + 31) +#endif #ifdef HAS_CONVERT8TO16ROW_SSE2 ANY11C(Convert8To16Row_Any_SSE2, Convert8To16Row_SSE2, @@ -1972,6 +2116,24 @@ ANY11C(Convert8To16Row_Any_NEON, uint16_t, 15) #endif +#ifdef HAS_CONVERT8TO16ROW_LSX +ANY11C(Convert8To16Row_Any_LSX, + Convert8To16Row_LSX, + 1, + 2, + uint8_t, + uint16_t, + 15) +#endif +#ifdef HAS_CONVERT8TO16ROW_LASX +ANY11C(Convert8To16Row_Any_LASX, + Convert8To16Row_LASX, + 1, + 2, + uint8_t, + uint16_t, + 31) +#endif #ifdef HAS_MULTIPLYROW_16_AVX2 ANY11C(MultiplyRow_16_Any_AVX2, MultiplyRow_16_AVX2, @@ -1990,12 +2152,36 @@ ANY11C(MultiplyRow_16_Any_NEON, uint16_t, 15) #endif +#ifdef HAS_MULTIPLYROW_16_LSX +ANY11C(MultiplyRow_16_Any_LSX, + MultiplyRow_16_LSX, + 2, + 2, + uint16_t, + uint16_t, + 15) +#endif +#ifdef HAS_MULTIPLYROW_16_LASX +ANY11C(MultiplyRow_16_Any_LASX, + MultiplyRow_16_LASX, + 2, + 2, + uint16_t, + uint16_t, + 31) +#endif #ifdef HAS_DIVIDEROW_16_AVX2 ANY11C(DivideRow_16_Any_AVX2, DivideRow_16_AVX2, 2, 2, uint16_t, uint16_t, 31) #endif #ifdef HAS_DIVIDEROW_16_NEON ANY11C(DivideRow_16_Any_NEON, DivideRow_16_NEON, 2, 2, uint16_t, uint16_t, 15) #endif +#ifdef HAS_DIVIDEROW_16_LSX +ANY11C(DivideRow_16_Any_LSX, DivideRow_16_LSX, 2, 2, uint16_t, uint16_t, 15) +#endif +#ifdef HAS_DIVIDEROW_16_LASX +ANY11C(DivideRow_16_Any_LASX, DivideRow_16_LASX, 2, 2, uint16_t, uint16_t, 31) +#endif #ifdef HAS_CONVERT8TO16ROW_WASMSIMD ANY11C(Convert8To16Row_Any_WASMSIMD, Convert8To16Row_WASMSIMD, @@ -2055,6 +2241,24 @@ ANY11SB(Convert8To8Row_Any_AVX2, uint8_t, 31) #endif +#ifdef HAS_CONVERT8TO8ROW_LSX +ANY11SB(Convert8To8Row_Any_LSX, + Convert8To8Row_LSX, + 1, + 1, + uint8_t, + uint8_t, + 15) +#endif +#ifdef HAS_CONVERT8TO8ROW_LASX +ANY11SB(Convert8To8Row_Any_LASX, + Convert8To8Row_LASX, + 1, + 1, + uint8_t, + uint8_t, + 31) +#endif #undef ANY11B // Any 1 to 1 with parameter and shorts to byte. BPP measures in shorts. @@ -2209,6 +2413,24 @@ ANY11I(InterpolateRow_16_Any_AVX2, 1, 15) #endif +#ifdef HAS_INTERPOLATEROW_16_LSX +ANY11I(InterpolateRow_16_Any_LSX, + InterpolateRow_16_LSX, + uint16_t, + uint16_t, + 1, + 1, + 7) +#endif +#ifdef HAS_INTERPOLATEROW_16_LASX +ANY11I(InterpolateRow_16_Any_LASX, + InterpolateRow_16_LASX, + uint16_t, + uint16_t, + 1, + 1, + 15) +#endif #ifdef HAS_INTERPOLATEROW_WASMSIMD ANY11I(InterpolateRow_Any_WASMSIMD, InterpolateRow_WASMSIMD, @@ -2259,6 +2481,18 @@ ANY11H(HalfRow_Any_AVX2, HalfRow_AVX2, uint8_t, 31) #ifdef HAS_HALFROW_16_AVX2 ANY11H(HalfRow_16_Any_AVX2, HalfRow_16_AVX2, uint16_t, 15) #endif +#ifdef HAS_HALFROW_LSX +ANY11H(HalfRow_Any_LSX, HalfRow_LSX, uint8_t, 15) +#endif +#ifdef HAS_HALFROW_LASX +ANY11H(HalfRow_Any_LASX, HalfRow_LASX, uint8_t, 31) +#endif +#ifdef HAS_HALFROW_16_LSX +ANY11H(HalfRow_16_Any_LSX, HalfRow_16_LSX, uint16_t, 7) +#endif +#ifdef HAS_HALFROW_16_LASX +ANY11H(HalfRow_16_Any_LASX, HalfRow_16_LASX, uint16_t, 15) +#endif #undef ANY11H @@ -2295,6 +2529,12 @@ ANY11HS(HalfRow_16To8_Any_AVX2, HalfRow_16To8_AVX2, uint16_t, uint8_t, 31) #ifdef HAS_HALFROW_16TO8_NEON ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15) #endif +#ifdef HAS_HALFROW_16TO8_LSX +ANY11HS(HalfRow_16To8_Any_LSX, HalfRow_16To8_LSX, uint16_t, uint8_t, 15) +#endif +#ifdef HAS_HALFROW_16TO8_LASX +ANY11HS(HalfRow_16To8_Any_LASX, HalfRow_16To8_LASX, uint16_t, uint8_t, 31) +#endif #undef ANY11HS @@ -2439,6 +2679,12 @@ ANY11M(RGB24MirrorRow_Any_SSSE3, RGB24MirrorRow_SSSE3, 3, 15) #ifdef HAS_RGB24MIRRORROW_NEON ANY11M(RGB24MirrorRow_Any_NEON, RGB24MirrorRow_NEON, 3, 15) #endif +#ifdef HAS_RGB24MIRRORROW_LSX +ANY11M(RGB24MirrorRow_Any_LSX, RGB24MirrorRow_LSX, 3, 15) +#endif +#ifdef HAS_RGB24MIRRORROW_LASX +ANY11M(RGB24MirrorRow_Any_LASX, RGB24MirrorRow_LASX, 3, 31) +#endif #ifdef HAS_MIRRORROW_WASMSIMD ANY11M(MirrorRow_Any_WASMSIMD, MirrorRow_WASMSIMD, 1, 15) #endif @@ -2655,6 +2901,12 @@ ANY12PT(SplitUVRow_16_Any_AVX2, SplitUVRow_16_AVX2, uint16_t, 2, 15) #ifdef HAS_SPLITUVROW_16_NEON ANY12PT(SplitUVRow_16_Any_NEON, SplitUVRow_16_NEON, uint16_t, 2, 7) #endif +#ifdef HAS_SPLITUVROW_16_LSX +ANY12PT(SplitUVRow_16_Any_LSX, SplitUVRow_16_LSX, uint16_t, 2, 7) +#endif +#ifdef HAS_SPLITUVROW_16_LASX +ANY12PT(SplitUVRow_16_Any_LASX, SplitUVRow_16_LASX, uint16_t, 2, 15) +#endif #undef ANY21CT @@ -2703,6 +2955,12 @@ ANY13(SplitXRGBRow_Any_AVX2, SplitXRGBRow_AVX2, 4, 15) #ifdef HAS_SPLITXRGBROW_NEON ANY13(SplitXRGBRow_Any_NEON, SplitXRGBRow_NEON, 4, 15) #endif +#ifdef HAS_SPLITXRGBROW_LSX +ANY13(SplitXRGBRow_Any_LSX, SplitXRGBRow_LSX, 4, 15) +#endif +#ifdef HAS_SPLITXRGBROW_LASX +ANY13(SplitXRGBRow_Any_LASX, SplitXRGBRow_LASX, 4, 15) +#endif // Any 1 to 4. Outputs ARGB planes. #define ANY14(NAMEANY, ANY_SIMD, BPP, MASK) \ @@ -2740,6 +2998,12 @@ ANY14(SplitARGBRow_Any_AVX2, SplitARGBRow_AVX2, 4, 15) #ifdef HAS_SPLITARGBROW_NEON ANY14(SplitARGBRow_Any_NEON, SplitARGBRow_NEON, 4, 15) #endif +#ifdef HAS_SPLITARGBROW_LSX +ANY14(SplitARGBRow_Any_LSX, SplitARGBRow_LSX, 4, 15) +#endif +#ifdef HAS_SPLITARGBROW_LASX +ANY14(SplitARGBRow_Any_LASX, SplitARGBRow_LASX, 4, 15) +#endif // Any 1 to 2 with source stride (2 rows of source). Outputs UV planes. // 128 byte row allows for 32 avx ARGB pixels. @@ -3120,6 +3384,18 @@ ANY11S(AYUVToVURow_Any_AVX2, AYUVToVURow_AVX2, 0, 4, 15) ANY11S(AYUVToUVRow_Any_NEON, AYUVToUVRow_NEON, 0, 4, 15) ANY11S(AYUVToVURow_Any_NEON, AYUVToVURow_NEON, 0, 4, 15) #endif +#ifdef HAS_AYUVTOUVROW_LSX +ANY11S(AYUVToUVRow_Any_LSX, AYUVToUVRow_LSX, 0, 4, 15) +#endif +#ifdef HAS_AYUVTOUVROW_LASX +ANY11S(AYUVToUVRow_Any_LASX, AYUVToUVRow_LASX, 0, 4, 15) +#endif +#ifdef HAS_AYUVTOVUROW_LSX +ANY11S(AYUVToVURow_Any_LSX, AYUVToVURow_LSX, 0, 4, 15) +#endif +#ifdef HAS_AYUVTOVUROW_LASX +ANY11S(AYUVToVURow_Any_LASX, AYUVToVURow_LASX, 0, 4, 15) +#endif #undef ANY11S #define ANYDETILE(NAMEANY, ANY_SIMD, T, BPP, MASK) \ @@ -3153,6 +3429,18 @@ ANYDETILE(DetileRow_16_Any_NEON, DetileRow_16_NEON, uint16_t, 2, 15) #ifdef HAS_DETILEROW_16_AVX2 ANYDETILE(DetileRow_16_Any_AVX2, DetileRow_16_AVX2, uint16_t, 2, 15) #endif +#ifdef HAS_DETILEROW_LSX +ANYDETILE(DetileRow_Any_LSX, DetileRow_LSX, uint8_t, 1, 15) +#endif +#ifdef HAS_DETILEROW_LASX +ANYDETILE(DetileRow_Any_LASX, DetileRow_LASX, uint8_t, 1, 15) +#endif +#ifdef HAS_DETILEROW_16_LSX +ANYDETILE(DetileRow_16_Any_LSX, DetileRow_16_LSX, uint16_t, 2, 15) +#endif +#ifdef HAS_DETILEROW_16_LASX +ANYDETILE(DetileRow_16_Any_LASX, DetileRow_16_LASX, uint16_t, 2, 15) +#endif // DetileSplitUVRow width is in bytes #define ANYDETILESPLITUV(NAMEANY, ANY_SIMD, MASK) \ diff --git a/source/row_common.cc b/source/row_common.cc index 19ebaa8db..3bd708c5c 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -4381,36 +4381,6 @@ void NV12ToRGB565Row_AVX2(const uint8_t* src_y, } #endif -float ScaleSumSamples_C(const float* src, float* dst, float scale, int width) { - float fsum = 0.f; - int i; - for (i = 0; i < width; ++i) { - float v = *src++; - fsum += v * v; - *dst++ = v * scale; - } - return fsum; -} - -float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width) { - float fmax = 0.f; - int i; - for (i = 0; i < width; ++i) { - float v = *src++; - float vs = v * scale; - fmax = (v > fmax) ? v : fmax; - *dst++ = vs; - } - return fmax; -} - -void ScaleSamples_C(const float* src, float* dst, float scale, int width) { - int i; - for (i = 0; i < width; ++i) { - *dst++ = *src++ * scale; - } -} - void GaussRow_C(const uint32_t* src, uint16_t* dst, int width) { int i; for (i = 0; i < width; ++i) { diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 0ed9f7939..9a22e4949 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -14646,161 +14646,6 @@ void GaussCol_F32_AVX2(const float* src0, } #endif // HAS_GAUSSCOL_F32_AVX2 -#ifdef HAS_SCALEMAXSAMPLES_AVX2 -float ScaleMaxSamples_AVX2(const float* src, - float* dst, - float scale, - int width) { - float fmax; - asm volatile( - "vpxor %%xmm3,%%xmm3,%%xmm3 \n" - "vbroadcastss %[scale],%%ymm2 \n" - "sub $0x8,%[width] \n" - "jl 2f \n" - - LABELALIGN - "1: \n" - "vmovups (%[src]),%%ymm0 \n" - "lea 0x20(%[src]),%[src] \n" - "vmaxps %%ymm3,%%ymm0,%%ymm3 \n" - "vmulps %%ymm2,%%ymm0,%%ymm0 \n" - "vmovups %%ymm0,(%[dst]) \n" - "lea 0x20(%[dst]),%[dst] \n" - "sub $0x8,%[width] \n" - "jge 1b \n" - - "vextractf128 $0x1,%%ymm3,%%xmm1 \n" - "vmaxps %%xmm3,%%xmm1,%%xmm3 \n" - "vpshufd $0x4e,%%xmm3,%%xmm1 \n" - "vmaxps %%xmm3,%%xmm1,%%xmm3 \n" - "vpshufd $0x55,%%xmm3,%%xmm1 \n" - "vmaxss %%xmm3,%%xmm1,%%xmm3 \n" - - "2: \n" - "add $0x8,%[width] \n" - "jle 99f \n" - - LABELALIGN - "3: \n" - "vmovss (%[src]),%%xmm0 \n" - "lea 0x4(%[src]),%[src] \n" - "vmaxss %%xmm3,%%xmm0,%%xmm3 \n" - "vmulss %%xmm2,%%xmm0,%%xmm0 \n" - "vmovss %%xmm0,(%[dst]) \n" - "lea 0x4(%[dst]),%[dst] \n" - "sub $0x1,%[width] \n" - "jg 3b \n" - - "99: \n" - "vmovss %%xmm3,%[fmax] \n" - "vzeroupper \n" - : [src] "+r"(src), - [dst] "+r"(dst), - [width] "+r"(width), - [fmax] "=m"(fmax) - : [scale] "m"(scale) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); - return fmax; -} -#endif // HAS_SCALEMAXSAMPLES_AVX2 - -#ifdef HAS_SCALESUMSAMPLES_AVX2 -float ScaleSumSamples_AVX2(const float* src, - float* dst, - float scale, - int width) { - float fsum; - asm volatile( - "vpxor %%xmm3,%%xmm3,%%xmm3 \n" - "vbroadcastss %[scale],%%ymm2 \n" - "sub $0x8,%[width] \n" - "jl 2f \n" - - LABELALIGN - "1: \n" - "vmovups (%[src]),%%ymm0 \n" - "lea 0x20(%[src]),%[src] \n" - "vmulps %%ymm0,%%ymm0,%%ymm1 \n" - "vaddps %%ymm1,%%ymm3,%%ymm3 \n" - "vmulps %%ymm2,%%ymm0,%%ymm0 \n" - "vmovups %%ymm0,(%[dst]) \n" - "lea 0x20(%[dst]),%[dst] \n" - "sub $0x8,%[width] \n" - "jge 1b \n" - - "vextractf128 $0x1,%%ymm3,%%xmm1 \n" - "vaddps %%xmm1,%%xmm3,%%xmm3 \n" - "vhaddps %%xmm3,%%xmm3,%%xmm3 \n" - "vhaddps %%xmm3,%%xmm3,%%xmm3 \n" - - "2: \n" - "add $0x8,%[width] \n" - "jle 99f \n" - - LABELALIGN - "3: \n" - "vmovss (%[src]),%%xmm0 \n" - "lea 0x4(%[src]),%[src] \n" - "vmulss %%xmm0,%%xmm0,%%xmm1 \n" - "vaddss %%xmm1,%%xmm3,%%xmm3 \n" - "vmulss %%xmm2,%%xmm0,%%xmm0 \n" - "vmovss %%xmm0,(%[dst]) \n" - "lea 0x4(%[dst]),%[dst] \n" - "sub $0x1,%[width] \n" - "jg 3b \n" - - "99: \n" - "vmovss %%xmm3,%[fsum] \n" - "vzeroupper \n" - : [src] "+r"(src), - [dst] "+r"(dst), - [width] "+r"(width), - [fsum] "=m"(fsum) - : [scale] "m"(scale) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); - return fsum; -} -#endif // HAS_SCALESUMSAMPLES_AVX2 - -#ifdef HAS_SCALESAMPLES_AVX2 -void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) { - asm volatile( - "vbroadcastss %[scale],%%ymm2 \n" - "sub $0x8,%[width] \n" - "jl 2f \n" - - LABELALIGN - "1: \n" - "vmovups (%[src]),%%ymm0 \n" - "lea 0x20(%[src]),%[src] \n" - "vmulps %%ymm2,%%ymm0,%%ymm0 \n" - "vmovups %%ymm0,(%[dst]) \n" - "lea 0x20(%[dst]),%[dst] \n" - "sub $0x8,%[width] \n" - "jge 1b \n" - - "2: \n" - "add $0x8,%[width] \n" - "jle 99f \n" - - LABELALIGN - "3: \n" - "vmovss (%[src]),%%xmm0 \n" - "lea 0x4(%[src]),%[src] \n" - "vmulss %%xmm2,%%xmm0,%%xmm0 \n" - "vmovss %%xmm0,(%[dst]) \n" - "lea 0x4(%[dst]),%[dst] \n" - "sub $0x1,%[width] \n" - "jg 3b \n" - - "99: \n" - "vzeroupper \n" - : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) - : [scale] "m"(scale) - : "memory", "cc", "xmm0", "xmm2"); -} -#endif // HAS_SCALESAMPLES_AVX2 - #ifdef HAS_ARGBCOLORTABLEROW_AVX2 // Transform ARGB pixels with color table. void ARGBColorTableRow_AVX2(uint8_t* dst_argb, diff --git a/source/row_la.cc b/source/row_la.cc index 88c5b4905..131781985 100644 --- a/source/row_la.cc +++ b/source/row_la.cc @@ -8,6 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include + #include "libyuv/basic_types.h" #include "libyuv/row.h" @@ -1303,7 +1305,8 @@ void ARGBToRAWRow_LSX(const uint8_t* src_argb, uint8_t* dst_raw, int width) { } #endif // HAS_ARGBTORAWROW_LSX -#if defined(HAS_RGB24TOARGBROW_LSX) || defined(HAS_RAWTOARGBROW_LSX) +#if defined(HAS_RGB24TOARGBROW_LSX) || defined(HAS_RAWTOARGBROW_LSX) || \ + defined(HAS_RAWTORGBAROW_LSX) static LIBYUV_NOINLINE void RGB24ToARGBHelper_LSX(const uint8_t* src_24, uint8_t* dst_argb, const uint8_t* shuf, @@ -1333,7 +1336,8 @@ static LIBYUV_NOINLINE void RGB24ToARGBHelper_LSX(const uint8_t* src_24, : [shuf] "r"(shuf) : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); } -#endif // defined(HAS_RGB24TOARGBROW_LSX) || defined(HAS_RAWTOARGBROW_LSX) +#endif // defined(HAS_RGB24TOARGBROW_LSX) || defined(HAS_RAWTOARGBROW_LSX) || + // defined(HAS_RAWTORGBAROW_LSX) #ifdef HAS_RGB24TOARGBROW_LSX void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24, @@ -1355,6 +1359,15 @@ void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { } #endif // HAS_RAWTOARGBROW_LSX +#ifdef HAS_RAWTORGBAROW_LSX +void RAWToRGBARow_LSX(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { + static const uint8_t kShuf[32] = {16, 2, 1, 0, 16, 5, 4, 3, 16, 8, 7, + 6, 16, 11, 10, 9, 16, 6, 5, 4, 16, 9, + 8, 7, 16, 12, 11, 10, 16, 15, 14, 13}; + RGB24ToARGBHelper_LSX(src_raw, dst_rgba, kShuf, width); +} +#endif // HAS_RAWTORGBAROW_LSX + #ifdef HAS_RAWTORGB24ROW_LSX void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { static const uint8_t kShuf[32] = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, @@ -2831,6 +2844,1132 @@ void RAWToUVRow_LSX(const uint8_t* src_raw, } #endif // HAS_RAWTOUVROW_LSX +#ifdef HAS_SWAPUVROW_LSX +void SwapUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_vu, int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vshuf4i.b $vr0, $vr0, 0xb1\n\t" + "vshuf4i.b $vr1, $vr1, 0xb1\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uv), [dst] "+r"(dst_vu), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_SWAPUVROW_LSX + +#ifdef HAS_ARGBCOPYYTOALPHAROW_LSX +void ARGBCopyYToAlphaRow_LSX(const uint8_t* src, uint8_t* dst, int width) { + asm volatile( + "vldi $vr4, 0xff\n\t" + "vslli.w $vr4, $vr4, 24\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[src], 0\n\t" + "vld $vr1, %[dst], 0\n\t" + "vld $vr2, %[dst], 16\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vilvh.h $vr3, $vr0, $vr0\n\t" + "vilvl.h $vr0, $vr0, $vr0\n\t" + "vslli.w $vr0, $vr0, 24\n\t" + "vslli.w $vr3, $vr3, 24\n\t" + "vandn.v $vr1, $vr4, $vr1\n\t" + "vandn.v $vr2, $vr4, $vr2\n\t" + "vor.v $vr1, $vr1, $vr0\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vst $vr1, %[dst], 0\n\t" + "vst $vr2, %[dst], 16\n\t" + "addi.d %[src], %[src], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_ARGBCOPYYTOALPHAROW_LSX + +#ifdef HAS_HALFMERGEUVROW_LSX +void HalfMergeUVRow_LSX(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[su], 0\n\t" + "vld $vr1, %[sv], 0\n\t" + "vldx $vr2, %[su], %[stu]\n\t" + "vldx $vr3, %[sv], %[stv]\n\t" + "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" + "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" + "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr2\n\t" + "vadd.h $vr1, $vr1, $vr3\n\t" + "vsrari.h $vr0, $vr0, 2\n\t" + "vsrari.h $vr1, $vr1, 2\n\t" + "vpackev.b $vr0, $vr1, $vr0\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[su], %[su], 16\n\t" + "addi.d %[sv], %[sv], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : [stu] "r"((ptrdiff_t)src_stride_u), [stv] "r"((ptrdiff_t)src_stride_v) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_HALFMERGEUVROW_LSX + +#ifdef HAS_CONVERT16TO8ROW_LSX +void Convert16To8Row_LSX(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width) { + asm volatile( + "vreplgr2vr.h $vr2, %[scl]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vmuh.hu $vr0, $vr0, $vr2\n\t" + "vmuh.hu $vr1, $vr1, $vr2\n\t" + "vssrani.bu.h $vr1, $vr0, 0\n\t" + "vst $vr1, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [scl] "r"(scale) + : "memory", "$vr0", "$vr1", "$vr2"); +} +#endif // HAS_CONVERT16TO8ROW_LSX + +#ifdef HAS_CONVERT8TO16ROW_LSX +void Convert8To16Row_LSX(const uint8_t* src_y, + uint16_t* dst_y, + int bits, + int width) { + const int shift = 16 - bits; + asm volatile( + "vreplgr2vr.h $vr3, %[shf]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vilvl.b $vr1, $vr0, $vr0\n\t" + "vilvh.b $vr2, $vr0, $vr0\n\t" + "vsrl.h $vr1, $vr1, $vr3\n\t" + "vsrl.h $vr2, $vr2, $vr3\n\t" + "vst $vr1, %[dst], 0\n\t" + "vst $vr2, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [shf] "r"(shift) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_CONVERT8TO16ROW_LSX + +#ifdef HAS_MULTIPLYROW_16_LSX +void MultiplyRow_16_LSX(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + asm volatile( + "vreplgr2vr.h $vr2, %[scl]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vmul.h $vr0, $vr0, $vr2\n\t" + "vmul.h $vr1, $vr1, $vr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [scl] "r"(scale) + : "memory", "$vr0", "$vr1", "$vr2"); +} +#endif // HAS_MULTIPLYROW_16_LSX + +#ifdef HAS_DIVIDEROW_16_LSX +void DivideRow_16_LSX(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } + asm volatile( + "vreplgr2vr.h $vr2, %[scl]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vmuh.hu $vr0, $vr0, $vr2\n\t" + "vmuh.hu $vr1, $vr1, $vr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [scl] "r"(scale) + : "memory", "$vr0", "$vr1", "$vr2"); +} +#endif // HAS_DIVIDEROW_16_LSX + +#ifdef HAS_MERGEUVROW_16_LSX +void MergeUVRow_16_LSX(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width) { + int shift = 16 - depth; + asm volatile( + "vreplgr2vr.h $vr4, %[shf]\n\t" + "1:\n\t" + "vld $vr0, %[su], 0\n\t" + "vld $vr1, %[sv], 0\n\t" + "vsll.h $vr0, $vr0, $vr4\n\t" + "vsll.h $vr1, $vr1, $vr4\n\t" + "vilvl.h $vr2, $vr1, $vr0\n\t" + "vilvh.h $vr3, $vr1, $vr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "vst $vr3, %[dst], 16\n\t" + "addi.d %[su], %[su], 16\n\t" + "addi.d %[sv], %[sv], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : [shf] "r"(shift) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_MERGEUVROW_16_LSX + +#ifdef HAS_SPLITUVROW_16_LSX +void SplitUVRow_16_LSX(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width) { + int shift = 16 - depth; + asm volatile( + "vreplgr2vr.h $vr4, %[shf]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.h $vr2, $vr1, $vr0\n\t" + "vpickod.h $vr3, $vr1, $vr0\n\t" + "vsrl.h $vr2, $vr2, $vr4\n\t" + "vsrl.h $vr3, $vr3, $vr4\n\t" + "vst $vr2, %[du], 0\n\t" + "vst $vr3, %[dv], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : [shf] "r"(shift) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_SPLITUVROW_16_LSX + +#ifdef HAS_COPYROW_16_LSX +void CopyRow_16_LSX(const uint16_t* src, uint16_t* dst, int count) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(count) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_COPYROW_16_LSX + +#ifdef HAS_HALFROW_LSX +void HalfRow_LSX(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width) { + const uint8_t* src_uv1 = src_uv + src_uv_stride; + asm volatile( + "1:\n\t" + "vld $vr0, %[src0], 0\n\t" + "vld $vr1, %[src1], 0\n\t" + "vavgr.bu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 16\n\t" + "addi.d %[src1], %[src1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_HALFROW_LSX + +#ifdef HAS_HALFROW_16_LSX +void HalfRow_16_LSX(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + asm volatile( + "1:\n\t" + "vld $vr0, %[src0], 0\n\t" + "vld $vr1, %[src1], 0\n\t" + "vavgr.hu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 16\n\t" + "addi.d %[src1], %[src1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_HALFROW_16_LSX + +#ifdef HAS_ARGBCOPYALPHAROW_LSX +void ARGBCopyAlphaRow_LSX(const uint8_t* src, uint8_t* dst, int width) { + asm volatile( + "vldi $vr4, 0xff\n\t" + "vslli.w $vr4, $vr4, 24\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr2, %[dst], 0\n\t" + "vld $vr3, %[dst], 16\n\t" + "vbitsel.v $vr2, $vr2, $vr0, $vr4\n\t" + "vbitsel.v $vr3, $vr3, $vr1, $vr4\n\t" + "vst $vr2, %[dst], 0\n\t" + "vst $vr3, %[dst], 16\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_ARGBCOPYALPHAROW_LSX + +#ifdef HAS_YUY2TONVUVROW_LSX +void YUY2ToNVUVRow_LSX(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_uv, + int width) { + const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2; + asm volatile( + "1:\n\t" + "vld $vr0, %[src0], 0\n\t" + "vld $vr1, %[src0], 16\n\t" + "vld $vr2, %[src1], 0\n\t" + "vld $vr3, %[src1], 16\n\t" + "vpickod.b $vr0, $vr1, $vr0\n\t" + "vpickod.b $vr2, $vr3, $vr2\n\t" + "vavgr.bu $vr0, $vr0, $vr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 32\n\t" + "addi.d %[src1], %[src1], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_yuy2), [src1] "+r"(src_yuy2_1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_YUY2TONVUVROW_LSX + +#ifdef HAS_AR64TOARGBROW_LSX +void AR64ToARGBRow_LSX(const uint16_t* src_ar64, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickod.b $vr0, $vr1, $vr0\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ar64), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_AR64TOARGBROW_LSX + +#ifdef HAS_AB64TOARGBROW_LSX +void AB64ToARGBRow_LSX(const uint16_t* src_ab64, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickod.b $vr0, $vr1, $vr0\n\t" + "vshuf4i.b $vr0, $vr0, 0xc6\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ab64), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_AB64TOARGBROW_LSX + +#ifdef HAS_ARGBTOAR64ROW_LSX +void ARGBToAR64Row_LSX(const uint8_t* src_argb, + uint16_t* dst_ar64, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vilvl.b $vr1, $vr0, $vr0\n\t" + "vilvh.b $vr2, $vr0, $vr0\n\t" + "vst $vr1, %[dst], 0\n\t" + "vst $vr2, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_ar64), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2"); +} +#endif // HAS_ARGBTOAR64ROW_LSX + +#ifdef HAS_ARGBTOAB64ROW_LSX +void ARGBToAB64Row_LSX(const uint8_t* src_argb, + uint16_t* dst_ab64, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf4i.b $vr0, $vr0, 0xc6\n\t" + "vilvl.b $vr1, $vr0, $vr0\n\t" + "vilvh.b $vr2, $vr0, $vr0\n\t" + "vst $vr1, %[dst], 0\n\t" + "vst $vr2, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_ab64), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2"); +} +#endif // HAS_ARGBTOAB64ROW_LSX + +#ifdef HAS_CONVERT8TO8ROW_LSX +void Convert8To8Row_LSX(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width) { + asm volatile( + "vreplgr2vr.b $vr2, %[scl]\n\t" + "vreplgr2vr.b $vr3, %[bia]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vmuh.bu $vr0, $vr0, $vr2\n\t" + "vadd.b $vr0, $vr0, $vr3\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [scl] "r"(scale), [bia] "r"(bias) + : "memory", "$vr0", "$vr2", "$vr3"); +} +#endif // HAS_CONVERT8TO8ROW_LSX + +#ifdef HAS_HALFROW_16TO8_LSX +void HalfRow_16To8_LSX(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + asm volatile( + "vreplgr2vr.h $vr4, %[scl]\n\t" + "1:\n\t" + "vld $vr0, %[src0], 0\n\t" + "vld $vr1, %[src0], 16\n\t" + "vld $vr2, %[src1], 0\n\t" + "vld $vr3, %[src1], 16\n\t" + "vavgr.hu $vr0, $vr0, $vr2\n\t" + "vavgr.hu $vr1, $vr1, $vr3\n\t" + "vmuh.hu $vr0, $vr0, $vr4\n\t" + "vmuh.hu $vr1, $vr1, $vr4\n\t" + "vssrani.bu.h $vr1, $vr0, 0\n\t" + "vst $vr1, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 32\n\t" + "addi.d %[src1], %[src1], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : [scl] "r"(scale) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_HALFROW_16TO8_LSX + +#ifdef HAS_AR30TOAB30ROW_LSX +void AR30ToAB30Row_LSX(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width) { + uint32_t m_ag = 0xc00ffc00u; + uint32_t m_r = 0x3ff00000u; + uint32_t m_b = 0x000003ffu; + asm volatile( + "vreplgr2vr.w $vr3, %[m_ag]\n\t" + "vreplgr2vr.w $vr4, %[m_r]\n\t" + "vreplgr2vr.w $vr5, %[m_b]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vsrli.w $vr1, $vr0, 20\n\t" + "vslli.w $vr2, $vr0, 20\n\t" + "vand.v $vr0, $vr0, $vr3\n\t" + "vand.v $vr1, $vr1, $vr5\n\t" + "vand.v $vr2, $vr2, $vr4\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vor.v $vr0, $vr0, $vr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ar30), [dst] "+r"(dst_ab30), [cnt] "+r"(width) + : [m_ag] "r"(m_ag), [m_r] "r"(m_r), [m_b] "r"(m_b) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); +} +#endif // HAS_AR30TOAB30ROW_LSX + +#ifdef HAS_ARGBTOAR30ROW_LSX +void ARGBToAR30Row_LSX(const uint8_t* src_argb, + uint8_t* dst_ar30, + int width) { + uint32_t m_a = 0xc0000000u; + uint32_t m_r = 0x3fc00000u; + uint32_t m_g = 0x000ff000u; + uint32_t m_b = 0x000003fcu; + uint32_t m_l = 0x00300c03u; + asm volatile( + "vreplgr2vr.w $vr4, %[m_a]\n\t" + "vreplgr2vr.w $vr5, %[m_r]\n\t" + "vreplgr2vr.w $vr6, %[m_g]\n\t" + "vreplgr2vr.w $vr7, %[m_b]\n\t" + "vreplgr2vr.w $vr8, %[m_l]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vslli.w $vr1, $vr0, 6\n\t" + "vslli.w $vr2, $vr0, 4\n\t" + "vslli.w $vr3, $vr0, 2\n\t" + "vand.v $vr0, $vr0, $vr4\n\t" + "vand.v $vr1, $vr1, $vr5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vand.v $vr3, $vr3, $vr7\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr0, $vr0, $vr2\n\t" + "vsrli.w $vr1, $vr0, 8\n\t" + "vand.v $vr1, $vr1, $vr8\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_ar30), [cnt] "+r"(width) + : [m_a] "r"(m_a), [m_r] "r"(m_r), [m_g] "r"(m_g), [m_b] "r"(m_b), + [m_l] "r"(m_l) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8"); +} +#endif // HAS_ARGBTOAR30ROW_LSX + +#ifdef HAS_ABGRTOAR30ROW_LSX +void ABGRToAR30Row_LSX(const uint8_t* src_abgr, + uint8_t* dst_ar30, + int width) { + uint32_t m_a = 0xc0000000u; + uint32_t m_r = 0x3fc00000u; + uint32_t m_g = 0x000ff000u; + uint32_t m_b = 0x000003fcu; + uint32_t m_l = 0x00300c03u; + asm volatile( + "vreplgr2vr.w $vr4, %[m_a]\n\t" + "vreplgr2vr.w $vr5, %[m_r]\n\t" + "vreplgr2vr.w $vr6, %[m_g]\n\t" + "vreplgr2vr.w $vr7, %[m_b]\n\t" + "vreplgr2vr.w $vr8, %[m_l]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf4i.b $vr0, $vr0, 0xc6\n\t" + "vslli.w $vr1, $vr0, 6\n\t" + "vslli.w $vr2, $vr0, 4\n\t" + "vslli.w $vr3, $vr0, 2\n\t" + "vand.v $vr0, $vr0, $vr4\n\t" + "vand.v $vr1, $vr1, $vr5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vand.v $vr3, $vr3, $vr7\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr0, $vr0, $vr2\n\t" + "vsrli.w $vr1, $vr0, 8\n\t" + "vand.v $vr1, $vr1, $vr8\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_abgr), [dst] "+r"(dst_ar30), [cnt] "+r"(width) + : [m_a] "r"(m_a), [m_r] "r"(m_r), [m_g] "r"(m_g), [m_b] "r"(m_b), + [m_l] "r"(m_l) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8"); +} +#endif // HAS_ABGRTOAR30ROW_LSX + +#ifdef HAS_AR30TOARGBROW_LSX +void AR30ToARGBRow_LSX(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width) { + uint32_t m_a = 0xc0000000u; + uint32_t m_2 = 0x300000ffu; + uint32_t m_4 = 0x0c00ff00u; + uint32_t m_6 = 0x03ff0000u; + asm volatile( + "vreplgr2vr.w $vr4, %[m_a]\n\t" + "vreplgr2vr.w $vr5, %[m_2]\n\t" + "vreplgr2vr.w $vr6, %[m_4]\n\t" + "vreplgr2vr.w $vr7, %[m_6]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vsrli.w $vr1, $vr0, 2\n\t" + "vsrli.w $vr2, $vr0, 4\n\t" + "vsrli.w $vr3, $vr0, 6\n\t" + "vand.v $vr0, $vr0, $vr4\n\t" + "vand.v $vr1, $vr1, $vr5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vand.v $vr3, $vr3, $vr7\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr0, $vr0, $vr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ar30), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [m_a] "r"(m_a), [m_2] "r"(m_2), [m_4] "r"(m_4), [m_6] "r"(m_6) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); +} +#endif // HAS_AR30TOARGBROW_LSX + +#ifdef HAS_AR30TOABGRROW_LSX +void AR30ToABGRRow_LSX(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width) { + uint32_t m_a = 0xc0000000u; + uint32_t m_2 = 0x300000ffu; + uint32_t m_4 = 0x0c00ff00u; + uint32_t m_6 = 0x03ff0000u; + asm volatile( + "vreplgr2vr.w $vr4, %[m_a]\n\t" + "vreplgr2vr.w $vr5, %[m_2]\n\t" + "vreplgr2vr.w $vr6, %[m_4]\n\t" + "vreplgr2vr.w $vr7, %[m_6]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vsrli.w $vr1, $vr0, 2\n\t" + "vsrli.w $vr2, $vr0, 4\n\t" + "vsrli.w $vr3, $vr0, 6\n\t" + "vand.v $vr0, $vr0, $vr4\n\t" + "vand.v $vr1, $vr1, $vr5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vand.v $vr3, $vr3, $vr7\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr0, $vr0, $vr2\n\t" + "vshuf4i.b $vr0, $vr0, 0xc6\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ar30), [dst] "+r"(dst_abgr), [cnt] "+r"(width) + : [m_a] "r"(m_a), [m_2] "r"(m_2), [m_4] "r"(m_4), [m_6] "r"(m_6) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); +} +#endif // HAS_AR30TOABGRROW_LSX + +#ifdef HAS_AYUVTOYROW_LSX +void AYUVToYRow_LSX(const uint8_t* src_ayuv, uint8_t* dst_y, int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr2, %[src], 32\n\t" + "vld $vr3, %[src], 48\n\t" + "vpickev.h $vr0, $vr1, $vr0\n\t" + "vpickev.h $vr2, $vr3, $vr2\n\t" + "vpickev.b $vr0, $vr2, $vr0\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ayuv), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_AYUVTOYROW_LSX + +#ifdef HAS_AYUVTOUVROW_LSX +void AYUVToUVRow_LSX(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv; + asm volatile( + "vldi $vr6, 0x02\n\t" + "1:\n\t" + "vld $vr0, %[src0], 0\n\t" + "vld $vr1, %[src0], 16\n\t" + "vld $vr2, %[src0], 32\n\t" + "vld $vr3, %[src0], 48\n\t" + "vld $vr4, %[src1], 0\n\t" + "vld $vr5, %[src1], 16\n\t" + "vaddwev.h.bu $vr0, $vr0, $vr4\n\t" + "vaddwod.h.bu $vr4, $vr1, $vr5\n\t" + "vld $vr1, %[src1], 32\n\t" + "vld $vr5, %[src1], 48\n\t" + "vaddwev.h.bu $vr2, $vr2, $vr1\n\t" + "vaddwod.h.bu $vr3, $vr3, $vr5\n\t" + "vadd.h $vr0, $vr0, $vr4\n\t" + "vadd.h $vr2, $vr2, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr6\n\t" + "vadd.h $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr0, $vr0, 2\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vpickev.b $vr0, $vr2, $vr0\n\t" + "vshuf4i.b $vr0, $vr0, 0xb1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 64\n\t" + "addi.d %[src1], %[src1], 64\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_ayuv), [src1] "+r"(src_ayuv1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} +#endif // HAS_AYUVTOUVROW_LSX + +#ifdef HAS_AYUVTOVUROW_LSX +void AYUVToVURow_LSX(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv; + asm volatile( + "vldi $vr6, 0x02\n\t" + "1:\n\t" + "vld $vr0, %[src0], 0\n\t" + "vld $vr1, %[src0], 16\n\t" + "vld $vr2, %[src0], 32\n\t" + "vld $vr3, %[src0], 48\n\t" + "vld $vr4, %[src1], 0\n\t" + "vld $vr5, %[src1], 16\n\t" + "vaddwev.h.bu $vr0, $vr0, $vr4\n\t" + "vaddwod.h.bu $vr4, $vr1, $vr5\n\t" + "vld $vr1, %[src1], 32\n\t" + "vld $vr5, %[src1], 48\n\t" + "vaddwev.h.bu $vr2, $vr2, $vr1\n\t" + "vaddwod.h.bu $vr3, $vr3, $vr5\n\t" + "vadd.h $vr0, $vr0, $vr4\n\t" + "vadd.h $vr2, $vr2, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr6\n\t" + "vadd.h $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr0, $vr0, 2\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vpickev.b $vr0, $vr2, $vr0\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 64\n\t" + "addi.d %[src1], %[src1], 64\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_ayuv), [src1] "+r"(src_ayuv1), [dst] "+r"(dst_vu), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} +#endif // HAS_AYUVTOVUROW_LSX + +#ifdef HAS_DETILEROW_LSX +void DetileRow_LSX(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vst $vr0, %[dst], 0\n\t" + "add.d %[src], %[src], %[str]\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : [str] "r"(src_tile_stride) + : "memory", "$vr0"); +} +#endif // HAS_DETILEROW_LSX + +#ifdef HAS_DETILEROW_16_LSX +void DetileRow_16_LSX(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width) { + ptrdiff_t byte_stride = src_tile_stride * 2; + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "add.d %[src], %[src], %[str]\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : [str] "r"(byte_stride) + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_DETILEROW_16_LSX + +#ifdef HAS_MERGEARGBROW_LSX +void MergeARGBRow_LSX(const uint8_t* src_b, + const uint8_t* src_g, + const uint8_t* src_r, + const uint8_t* src_a, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[sb], 0\n\t" + "vld $vr1, %[sg], 0\n\t" + "vld $vr2, %[sr], 0\n\t" + "vld $vr3, %[sa], 0\n\t" + "vilvl.b $vr4, $vr1, $vr0\n\t" + "vilvh.b $vr5, $vr1, $vr0\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvh.b $vr7, $vr3, $vr2\n\t" + "vilvl.h $vr0, $vr6, $vr4\n\t" + "vilvh.h $vr1, $vr6, $vr4\n\t" + "vilvl.h $vr2, $vr7, $vr5\n\t" + "vilvh.h $vr3, $vr7, $vr5\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "vst $vr2, %[dst], 32\n\t" + "vst $vr3, %[dst], 48\n\t" + "addi.d %[sb], %[sb], 16\n\t" + "addi.d %[sg], %[sg], 16\n\t" + "addi.d %[sr], %[sr], 16\n\t" + "addi.d %[sa], %[sa], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sb] "+r"(src_b), [sg] "+r"(src_g), [sr] "+r"(src_r), + [sa] "+r"(src_a), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); +} +#endif // HAS_MERGEARGBROW_LSX + +#ifdef HAS_MERGEXRGBROW_LSX +void MergeXRGBRow_LSX(const uint8_t* src_b, + const uint8_t* src_g, + const uint8_t* src_r, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr3, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sb], 0\n\t" + "vld $vr1, %[sg], 0\n\t" + "vld $vr2, %[sr], 0\n\t" + "vilvl.b $vr4, $vr1, $vr0\n\t" + "vilvh.b $vr5, $vr1, $vr0\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvh.b $vr7, $vr3, $vr2\n\t" + "vilvl.h $vr0, $vr6, $vr4\n\t" + "vilvh.h $vr1, $vr6, $vr4\n\t" + "vilvl.h $vr2, $vr7, $vr5\n\t" + "vilvh.h $vr4, $vr7, $vr5\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "vst $vr2, %[dst], 32\n\t" + "vst $vr4, %[dst], 48\n\t" + "addi.d %[sb], %[sb], 16\n\t" + "addi.d %[sg], %[sg], 16\n\t" + "addi.d %[sr], %[sr], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sb] "+r"(src_b), [sg] "+r"(src_g), [sr] "+r"(src_r), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); +} +#endif // HAS_MERGEXRGBROW_LSX + +#ifdef HAS_SPLITARGBROW_LSX +void SplitARGBRow_LSX(const uint8_t* src_argb, + uint8_t* dst_b, + uint8_t* dst_g, + uint8_t* dst_r, + uint8_t* dst_a, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr2, %[src], 32\n\t" + "vld $vr3, %[src], 48\n\t" + "vpickev.b $vr4, $vr1, $vr0\n\t" + "vpickod.b $vr5, $vr1, $vr0\n\t" + "vpickev.b $vr6, $vr3, $vr2\n\t" + "vpickod.b $vr7, $vr3, $vr2\n\t" + "vpickev.b $vr0, $vr6, $vr4\n\t" + "vpickod.b $vr1, $vr6, $vr4\n\t" + "vpickev.b $vr2, $vr7, $vr5\n\t" + "vpickod.b $vr3, $vr7, $vr5\n\t" + "vst $vr0, %[db], 0\n\t" + "vst $vr2, %[dg], 0\n\t" + "vst $vr1, %[dr], 0\n\t" + "vst $vr3, %[da], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[db], %[db], 16\n\t" + "addi.d %[dg], %[dg], 16\n\t" + "addi.d %[dr], %[dr], 16\n\t" + "addi.d %[da], %[da], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [db] "+r"(dst_b), [dg] "+r"(dst_g), + [dr] "+r"(dst_r), [da] "+r"(dst_a), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); +} +#endif // HAS_SPLITARGBROW_LSX + +#ifdef HAS_SPLITXRGBROW_LSX +void SplitXRGBRow_LSX(const uint8_t* src_argb, + uint8_t* dst_b, + uint8_t* dst_g, + uint8_t* dst_r, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr2, %[src], 32\n\t" + "vld $vr3, %[src], 48\n\t" + "vpickev.b $vr4, $vr1, $vr0\n\t" + "vpickod.b $vr5, $vr1, $vr0\n\t" + "vpickev.b $vr6, $vr3, $vr2\n\t" + "vpickod.b $vr7, $vr3, $vr2\n\t" + "vpickev.b $vr0, $vr6, $vr4\n\t" + "vpickod.b $vr1, $vr6, $vr4\n\t" + "vpickev.b $vr2, $vr7, $vr5\n\t" + "vst $vr0, %[db], 0\n\t" + "vst $vr2, %[dg], 0\n\t" + "vst $vr1, %[dr], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[db], %[db], 16\n\t" + "addi.d %[dg], %[dg], 16\n\t" + "addi.d %[dr], %[dr], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [db] "+r"(dst_b), [dg] "+r"(dst_g), + [dr] "+r"(dst_r), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); +} +#endif // HAS_SPLITXRGBROW_LSX + +#ifdef HAS_RGB24MIRRORROW_LSX +static const uvec8 kRGB24MirrorShuf0 = {13, 14, 15, 10, 11, 12, 7, 8, + 9, 4, 5, 6, 1, 2, 3, 0}; +static const uvec8 kRGB24MirrorShuf1 = {14, 15, 16, 11, 12, 13, 8, 9, + 10, 5, 6, 7, 2, 3, 4, 1}; +static const uvec8 kRGB24MirrorShuf2 = {15, 0, 1, 12, 13, 14, 9, 10, + 11, 6, 7, 8, 3, 4, 5, 2}; +void RGB24MirrorRow_LSX(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width) { + src_rgb24 += width * 3 - 48; + asm volatile( + "vld $vr4, %[sh0], 0\n\t" + "vld $vr5, %[sh1], 0\n\t" + "vld $vr6, %[sh2], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr2, %[src], 32\n\t" + "vshuf.b $vr3, $vr2, $vr1, $vr4\n\t" + "vshuf.b $vr1, $vr1, $vr0, $vr5\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr6\n\t" + "vst $vr3, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "vst $vr0, %[dst], 32\n\t" + "addi.d %[src], %[src], -48\n\t" + "addi.d %[dst], %[dst], 48\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgb24), [dst] "+r"(dst_rgb24), [cnt] "+r"(width) + : [sh0] "r"(&kRGB24MirrorShuf0), [sh1] "r"(&kRGB24MirrorShuf1), + [sh2] "r"(&kRGB24MirrorShuf2) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} +#endif // HAS_RGB24MIRRORROW_LSX + +#ifdef HAS_BLENDPLANEROW_LSX +void BlendPlaneRow_LSX(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width) { + asm volatile( + "vldi $vr6, 0x7f\n\t" + "vslli.h $vr6, $vr6, 1\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vld $vr2, %[alp], 0\n\t" + "vxori.b $vr3, $vr2, 0xff\n\t" + "vmulwev.h.bu $vr4, $vr0, $vr2\n\t" + "vmaddwev.h.bu $vr4, $vr1, $vr3\n\t" + "vmulwod.h.bu $vr5, $vr0, $vr2\n\t" + "vmaddwod.h.bu $vr5, $vr1, $vr3\n\t" + "vadd.h $vr4, $vr4, $vr6\n\t" + "vadd.h $vr5, $vr5, $vr6\n\t" + "vsrli.h $vr4, $vr4, 8\n\t" + "vsrli.h $vr5, $vr5, 8\n\t" + "vpackev.b $vr0, $vr5, $vr4\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[alp], %[alp], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src0), [s1] "+r"(src1), [alp] "+r"(alpha), [dst] "+r"(dst), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} +#endif // HAS_BLENDPLANEROW_LSX + +#ifdef HAS_INTERPOLATEROW_16_LSX +void InterpolateRow_16_LSX(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + int y1_fraction = source_y_fraction; + int y0_fraction = 256 - y1_fraction; + const uint16_t* src_ptr1 = src_ptr + src_stride; + if (y1_fraction == 0) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src0], 0\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_ptr), [dst] "+r"(dst_ptr), [cnt] "+r"(width) + : + : "memory", "$vr0"); + return; + } + if (y1_fraction == 128) { + HalfRow_16_LSX(src_ptr, src_stride, dst_ptr, width); + return; + } + asm volatile( + "vreplgr2vr.h $vr4, %[f0]\n\t" + "vreplgr2vr.h $vr5, %[f1]\n\t" + "1:\n\t" + "vld $vr0, %[src0], 0\n\t" + "vld $vr1, %[src1], 0\n\t" + "vmulwev.w.hu $vr2, $vr0, $vr4\n\t" + "vmaddwev.w.hu $vr2, $vr1, $vr5\n\t" + "vmulwod.w.hu $vr3, $vr0, $vr4\n\t" + "vmaddwod.w.hu $vr3, $vr1, $vr5\n\t" + "vsrari.w $vr2, $vr2, 8\n\t" + "vsrari.w $vr3, $vr3, 8\n\t" + "vpackev.h $vr3, $vr3, $vr2\n\t" + "vst $vr3, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 16\n\t" + "addi.d %[src1], %[src1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_ptr), [src1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), + [cnt] "+r"(width) + : [f0] "r"(y0_fraction), [f1] "r"(y1_fraction) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); +} +#endif // HAS_INTERPOLATEROW_16_LSX + #ifdef HAS_COPYROW_LASX void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int count) { asm volatile( @@ -4310,7 +5449,8 @@ void ARGBToRAWRow_LASX(const uint8_t* src_argb, uint8_t* dst_raw, int width) { } #endif // HAS_ARGBTORAWROW_LASX -#if defined(HAS_RGB24TOARGBROW_LASX) || defined(HAS_RAWTOARGBROW_LASX) +#if defined(HAS_RGB24TOARGBROW_LASX) || defined(HAS_RAWTOARGBROW_LASX) || \ + defined(HAS_RAWTORGBAROW_LASX) static LIBYUV_NOINLINE void RGB24ToARGBHelper_LASX(const uint8_t* src_24, uint8_t* dst_argb, const uint8_t* shuf, @@ -4359,7 +5499,8 @@ static LIBYUV_NOINLINE void RGB24ToARGBHelper_LASX(const uint8_t* src_24, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr31"); } -#endif // defined(HAS_RGB24TOARGBROW_LASX) || defined(HAS_RAWTOARGBROW_LASX) +#endif // defined(HAS_RGB24TOARGBROW_LASX) || defined(HAS_RAWTOARGBROW_LASX) || + // defined(HAS_RAWTORGBAROW_LASX) #ifdef HAS_RGB24TOARGBROW_LASX void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24, @@ -4381,6 +5522,15 @@ void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { } #endif // HAS_RAWTOARGBROW_LASX +#ifdef HAS_RAWTORGBAROW_LASX +void RAWToRGBARow_LASX(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { + static const uint8_t kShuf[32] = {16, 2, 1, 0, 16, 5, 4, 3, 16, 8, 7, + 6, 16, 11, 10, 9, 16, 6, 5, 4, 16, 9, + 8, 7, 16, 12, 11, 10, 16, 15, 14, 13}; + RGB24ToARGBHelper_LASX(src_raw, dst_rgba, kShuf, width); +} +#endif // HAS_RAWTORGBAROW_LASX + #ifdef HAS_RAWTORGB24ROW_LASX void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { static const uint8_t kShuf[32] = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, @@ -6016,6 +7166,1172 @@ void RAWToUVRow_LASX(const uint8_t* src_raw, } #endif // HAS_RAWTOUVROW_LASX +#ifdef HAS_SWAPUVROW_LASX +void SwapUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_vu, int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvshuf4i.b $xr0, $xr0, 0xb1\n\t" + "xvshuf4i.b $xr1, $xr1, 0xb1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "xvst $xr1, %[dst], 32\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uv), [dst] "+r"(dst_vu), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); +} +#endif // HAS_SWAPUVROW_LASX + +#ifdef HAS_ARGBCOPYYTOALPHAROW_LASX +void ARGBCopyYToAlphaRow_LASX(const uint8_t* src, uint8_t* dst, int width) { + asm volatile( + "xvldi $xr4, 0xff\n\t" + "xvslli.w $xr4, $xr4, 24\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "xvld $xr1, %[dst], 0\n\t" + "xvld $xr2, %[dst], 32\n\t" + "vilvl.b $vr3, $vr0, $vr0\n\t" + "vilvh.b $vr0, $vr0, $vr0\n\t" + "xvpermi.d $xr3, $xr3, 0xd8\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvilvl.h $xr3, $xr3, $xr3\n\t" + "xvilvl.h $xr0, $xr0, $xr0\n\t" + "xvslli.w $xr3, $xr3, 24\n\t" + "xvslli.w $xr0, $xr0, 24\n\t" + "xvandn.v $xr1, $xr4, $xr1\n\t" + "xvandn.v $xr2, $xr4, $xr2\n\t" + "xvor.v $xr1, $xr1, $xr3\n\t" + "xvor.v $xr2, $xr2, $xr0\n\t" + "xvst $xr1, %[dst], 0\n\t" + "xvst $xr2, %[dst], 32\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); +} +#endif // HAS_ARGBCOPYYTOALPHAROW_LASX + +#ifdef HAS_HALFMERGEUVROW_LASX +void HalfMergeUVRow_LASX(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[su], 0\n\t" + "xvld $xr1, %[sv], 0\n\t" + "xvldx $xr2, %[su], %[stu]\n\t" + "xvldx $xr3, %[sv], %[stv]\n\t" + "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t" + "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t" + "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t" + "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t" + "xvadd.h $xr0, $xr0, $xr2\n\t" + "xvadd.h $xr1, $xr1, $xr3\n\t" + "xvsrari.h $xr0, $xr0, 2\n\t" + "xvsrari.h $xr1, $xr1, 2\n\t" + "xvpackev.b $xr0, $xr1, $xr0\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[su], %[su], 32\n\t" + "addi.d %[sv], %[sv], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : [stu] "r"((ptrdiff_t)src_stride_u), [stv] "r"((ptrdiff_t)src_stride_v) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); +} +#endif // HAS_HALFMERGEUVROW_LASX + +#ifdef HAS_CONVERT16TO8ROW_LASX +void Convert16To8Row_LASX(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width) { + asm volatile( + "xvreplgr2vr.h $xr2, %[scl]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvmuh.hu $xr0, $xr0, $xr2\n\t" + "xvmuh.hu $xr1, $xr1, $xr2\n\t" + "xvssrani.bu.h $xr1, $xr0, 0\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvst $xr1, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [scl] "r"(scale) + : "memory", "$xr0", "$xr1", "$xr2"); +} +#endif // HAS_CONVERT16TO8ROW_LASX + +#ifdef HAS_CONVERT8TO16ROW_LASX +void Convert8To16Row_LASX(const uint8_t* src_y, + uint16_t* dst_y, + int bits, + int width) { + const int shift = 16 - bits; + asm volatile( + "xvreplgr2vr.h $xr3, %[shf]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvilvl.b $xr1, $xr0, $xr0\n\t" + "xvilvh.b $xr2, $xr0, $xr0\n\t" + "xvsrl.h $xr1, $xr1, $xr3\n\t" + "xvsrl.h $xr2, $xr2, $xr3\n\t" + "xvst $xr1, %[dst], 0\n\t" + "xvst $xr2, %[dst], 32\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [shf] "r"(shift) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); +} +#endif // HAS_CONVERT8TO16ROW_LASX + +#ifdef HAS_MULTIPLYROW_16_LASX +void MultiplyRow_16_LASX(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + asm volatile( + "xvreplgr2vr.h $xr2, %[scl]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvmul.h $xr0, $xr0, $xr2\n\t" + "xvmul.h $xr1, $xr1, $xr2\n\t" + "xvst $xr0, %[dst], 0\n\t" + "xvst $xr1, %[dst], 32\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [scl] "r"(scale) + : "memory", "$xr0", "$xr1", "$xr2"); +} +#endif // HAS_MULTIPLYROW_16_LASX + +#ifdef HAS_DIVIDEROW_16_LASX +void DivideRow_16_LASX(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + if (scale == 65536) { // Divide by 1, so do memcpy. + if (src_y != dst_y) { + memcpy(dst_y, src_y, (size_t)width * 2); + } + return; + } + asm volatile( + "xvreplgr2vr.h $xr2, %[scl]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvmuh.hu $xr0, $xr0, $xr2\n\t" + "xvmuh.hu $xr1, $xr1, $xr2\n\t" + "xvst $xr0, %[dst], 0\n\t" + "xvst $xr1, %[dst], 32\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [scl] "r"(scale) + : "memory", "$xr0", "$xr1", "$xr2"); +} +#endif // HAS_DIVIDEROW_16_LASX + +#ifdef HAS_MERGEUVROW_16_LASX +void MergeUVRow_16_LASX(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width) { + int shift = 16 - depth; + asm volatile( + "xvreplgr2vr.h $xr4, %[shf]\n\t" + "1:\n\t" + "xvld $xr0, %[su], 0\n\t" + "xvld $xr1, %[sv], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvsll.h $xr0, $xr0, $xr4\n\t" + "xvsll.h $xr1, $xr1, $xr4\n\t" + "xvilvl.h $xr2, $xr1, $xr0\n\t" + "xvilvh.h $xr3, $xr1, $xr0\n\t" + "xvst $xr2, %[dst], 0\n\t" + "xvst $xr3, %[dst], 32\n\t" + "addi.d %[su], %[su], 32\n\t" + "addi.d %[sv], %[sv], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : [shf] "r"(shift) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); +} +#endif // HAS_MERGEUVROW_16_LASX + +#ifdef HAS_SPLITUVROW_16_LASX +void SplitUVRow_16_LASX(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width) { + int shift = 16 - depth; + asm volatile( + "xvreplgr2vr.h $xr4, %[shf]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvpickev.h $xr2, $xr1, $xr0\n\t" + "xvpickod.h $xr3, $xr1, $xr0\n\t" + "xvpermi.d $xr2, $xr2, 0xd8\n\t" + "xvpermi.d $xr3, $xr3, 0xd8\n\t" + "xvsrl.h $xr2, $xr2, $xr4\n\t" + "xvsrl.h $xr3, $xr3, $xr4\n\t" + "xvst $xr2, %[du], 0\n\t" + "xvst $xr3, %[dv], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[du], %[du], 32\n\t" + "addi.d %[dv], %[dv], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : [shf] "r"(shift) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); +} +#endif // HAS_SPLITUVROW_16_LASX + +#ifdef HAS_COPYROW_16_LASX +void CopyRow_16_LASX(const uint16_t* src, uint16_t* dst, int count) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvst $xr0, %[dst], 0\n\t" + "xvst $xr1, %[dst], 32\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(count) + : + : "memory", "$xr0", "$xr1"); +} +#endif // HAS_COPYROW_16_LASX + +#ifdef HAS_HALFROW_LASX +void HalfRow_LASX(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width) { + const uint8_t* src_uv1 = src_uv + src_uv_stride; + asm volatile( + "1:\n\t" + "xvld $xr0, %[src0], 0\n\t" + "xvld $xr1, %[src1], 0\n\t" + "xvavgr.bu $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 32\n\t" + "addi.d %[src1], %[src1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); +} +#endif // HAS_HALFROW_LASX + +#ifdef HAS_HALFROW_16_LASX +void HalfRow_16_LASX(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + asm volatile( + "1:\n\t" + "xvld $xr0, %[src0], 0\n\t" + "xvld $xr1, %[src1], 0\n\t" + "xvavgr.hu $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 32\n\t" + "addi.d %[src1], %[src1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); +} +#endif // HAS_HALFROW_16_LASX + +#ifdef HAS_ARGBCOPYALPHAROW_LASX +void ARGBCopyAlphaRow_LASX(const uint8_t* src, uint8_t* dst, int width) { + asm volatile( + "xvldi $xr4, 0xff\n\t" + "xvslli.w $xr4, $xr4, 24\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvld $xr2, %[dst], 0\n\t" + "xvld $xr3, %[dst], 32\n\t" + "xvbitsel.v $xr2, $xr2, $xr0, $xr4\n\t" + "xvbitsel.v $xr3, $xr3, $xr1, $xr4\n\t" + "xvst $xr2, %[dst], 0\n\t" + "xvst $xr3, %[dst], 32\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); +} +#endif // HAS_ARGBCOPYALPHAROW_LASX + +#ifdef HAS_YUY2TONVUVROW_LASX +void YUY2ToNVUVRow_LASX(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_uv, + int width) { + const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2; + asm volatile( + "1:\n\t" + "xvld $xr0, %[src0], 0\n\t" + "xvld $xr1, %[src0], 32\n\t" + "xvld $xr2, %[src1], 0\n\t" + "xvld $xr3, %[src1], 32\n\t" + "xvpickod.b $xr0, $xr1, $xr0\n\t" + "xvpickod.b $xr2, $xr3, $xr2\n\t" + "xvavgr.bu $xr0, $xr0, $xr2\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 64\n\t" + "addi.d %[src1], %[src1], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_yuy2), [src1] "+r"(src_yuy2_1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); +} +#endif // HAS_YUY2TONVUVROW_LASX + +#ifdef HAS_AR64TOARGBROW_LASX +void AR64ToARGBRow_LASX(const uint16_t* src_ar64, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvpickod.b $xr0, $xr1, $xr0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ar64), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); +} +#endif // HAS_AR64TOARGBROW_LASX + +#ifdef HAS_AB64TOARGBROW_LASX +void AB64ToARGBRow_LASX(const uint16_t* src_ab64, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvpickod.b $xr0, $xr1, $xr0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvshuf4i.b $xr0, $xr0, 0xc6\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ab64), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); +} +#endif // HAS_AB64TOARGBROW_LASX + +#ifdef HAS_ARGBTOAR64ROW_LASX +void ARGBToAR64Row_LASX(const uint8_t* src_argb, + uint16_t* dst_ar64, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvilvl.b $xr1, $xr0, $xr0\n\t" + "xvilvh.b $xr2, $xr0, $xr0\n\t" + "xvst $xr1, %[dst], 0\n\t" + "xvst $xr2, %[dst], 32\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_ar64), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2"); +} +#endif // HAS_ARGBTOAR64ROW_LASX + +#ifdef HAS_ARGBTOAB64ROW_LASX +void ARGBToAB64Row_LASX(const uint8_t* src_argb, + uint16_t* dst_ab64, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvshuf4i.b $xr0, $xr0, 0xc6\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvilvl.b $xr1, $xr0, $xr0\n\t" + "xvilvh.b $xr2, $xr0, $xr0\n\t" + "xvst $xr1, %[dst], 0\n\t" + "xvst $xr2, %[dst], 32\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_ab64), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2"); +} +#endif // HAS_ARGBTOAB64ROW_LASX + +#ifdef HAS_CONVERT8TO8ROW_LASX +void Convert8To8Row_LASX(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width) { + asm volatile( + "xvreplgr2vr.b $xr2, %[scl]\n\t" + "xvreplgr2vr.b $xr3, %[bia]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvmuh.bu $xr0, $xr0, $xr2\n\t" + "xvadd.b $xr0, $xr0, $xr3\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [scl] "r"(scale), [bia] "r"(bias) + : "memory", "$xr0", "$xr2", "$xr3"); +} +#endif // HAS_CONVERT8TO8ROW_LASX + +#ifdef HAS_HALFROW_16TO8_LASX +void HalfRow_16To8_LASX(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + asm volatile( + "xvreplgr2vr.h $xr4, %[scl]\n\t" + "1:\n\t" + "xvld $xr0, %[src0], 0\n\t" + "xvld $xr1, %[src0], 32\n\t" + "xvld $xr2, %[src1], 0\n\t" + "xvld $xr3, %[src1], 32\n\t" + "xvavgr.hu $xr0, $xr0, $xr2\n\t" + "xvavgr.hu $xr1, $xr1, $xr3\n\t" + "xvmuh.hu $xr0, $xr0, $xr4\n\t" + "xvmuh.hu $xr1, $xr1, $xr4\n\t" + "xvssrani.bu.h $xr1, $xr0, 0\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvst $xr1, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 64\n\t" + "addi.d %[src1], %[src1], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_uv), [src1] "+r"(src_uv1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : [scl] "r"(scale) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); +} +#endif // HAS_HALFROW_16TO8_LASX + +#ifdef HAS_AR30TOAB30ROW_LASX +void AR30ToAB30Row_LASX(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width) { + uint32_t m_ag = 0xc00ffc00u; + uint32_t m_r = 0x3ff00000u; + uint32_t m_b = 0x000003ffu; + asm volatile( + "xvreplgr2vr.w $xr3, %[m_ag]\n\t" + "xvreplgr2vr.w $xr4, %[m_r]\n\t" + "xvreplgr2vr.w $xr5, %[m_b]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvsrli.w $xr1, $xr0, 20\n\t" + "xvslli.w $xr2, $xr0, 20\n\t" + "xvand.v $xr0, $xr0, $xr3\n\t" + "xvand.v $xr1, $xr1, $xr5\n\t" + "xvand.v $xr2, $xr2, $xr4\n\t" + "xvor.v $xr0, $xr0, $xr1\n\t" + "xvor.v $xr0, $xr0, $xr2\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ar30), [dst] "+r"(dst_ab30), [cnt] "+r"(width) + : [m_ag] "r"(m_ag), [m_r] "r"(m_r), [m_b] "r"(m_b) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5"); +} +#endif // HAS_AR30TOAB30ROW_LASX + +#ifdef HAS_ARGBTOAR30ROW_LASX +void ARGBToAR30Row_LASX(const uint8_t* src_argb, + uint8_t* dst_ar30, + int width) { + uint32_t m_a = 0xc0000000u; + uint32_t m_r = 0x3fc00000u; + uint32_t m_g = 0x000ff000u; + uint32_t m_b = 0x000003fcu; + uint32_t m_l = 0x00300c03u; + asm volatile( + "xvreplgr2vr.w $xr4, %[m_a]\n\t" + "xvreplgr2vr.w $xr5, %[m_r]\n\t" + "xvreplgr2vr.w $xr6, %[m_g]\n\t" + "xvreplgr2vr.w $xr7, %[m_b]\n\t" + "xvreplgr2vr.w $xr8, %[m_l]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvslli.w $xr1, $xr0, 6\n\t" + "xvslli.w $xr2, $xr0, 4\n\t" + "xvslli.w $xr3, $xr0, 2\n\t" + "xvand.v $xr0, $xr0, $xr4\n\t" + "xvand.v $xr1, $xr1, $xr5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvand.v $xr3, $xr3, $xr7\n\t" + "xvor.v $xr0, $xr0, $xr1\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvor.v $xr0, $xr0, $xr2\n\t" + "xvsrli.w $xr1, $xr0, 8\n\t" + "xvand.v $xr1, $xr1, $xr8\n\t" + "xvor.v $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_ar30), [cnt] "+r"(width) + : [m_a] "r"(m_a), [m_r] "r"(m_r), [m_g] "r"(m_g), [m_b] "r"(m_b), + [m_l] "r"(m_l) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8"); +} +#endif // HAS_ARGBTOAR30ROW_LASX + +#ifdef HAS_ABGRTOAR30ROW_LASX +void ABGRToAR30Row_LASX(const uint8_t* src_abgr, + uint8_t* dst_ar30, + int width) { + uint32_t m_a = 0xc0000000u; + uint32_t m_r = 0x3fc00000u; + uint32_t m_g = 0x000ff000u; + uint32_t m_b = 0x000003fcu; + uint32_t m_l = 0x00300c03u; + asm volatile( + "xvreplgr2vr.w $xr4, %[m_a]\n\t" + "xvreplgr2vr.w $xr5, %[m_r]\n\t" + "xvreplgr2vr.w $xr6, %[m_g]\n\t" + "xvreplgr2vr.w $xr7, %[m_b]\n\t" + "xvreplgr2vr.w $xr8, %[m_l]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvshuf4i.b $xr0, $xr0, 0xc6\n\t" + "xvslli.w $xr1, $xr0, 6\n\t" + "xvslli.w $xr2, $xr0, 4\n\t" + "xvslli.w $xr3, $xr0, 2\n\t" + "xvand.v $xr0, $xr0, $xr4\n\t" + "xvand.v $xr1, $xr1, $xr5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvand.v $xr3, $xr3, $xr7\n\t" + "xvor.v $xr0, $xr0, $xr1\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvor.v $xr0, $xr0, $xr2\n\t" + "xvsrli.w $xr1, $xr0, 8\n\t" + "xvand.v $xr1, $xr1, $xr8\n\t" + "xvor.v $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_abgr), [dst] "+r"(dst_ar30), [cnt] "+r"(width) + : [m_a] "r"(m_a), [m_r] "r"(m_r), [m_g] "r"(m_g), [m_b] "r"(m_b), + [m_l] "r"(m_l) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8"); +} +#endif // HAS_ABGRTOAR30ROW_LASX + +#ifdef HAS_AR30TOARGBROW_LASX +void AR30ToARGBRow_LASX(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width) { + uint32_t m_a = 0xc0000000u; + uint32_t m_2 = 0x300000ffu; + uint32_t m_4 = 0x0c00ff00u; + uint32_t m_6 = 0x03ff0000u; + asm volatile( + "xvreplgr2vr.w $xr4, %[m_a]\n\t" + "xvreplgr2vr.w $xr5, %[m_2]\n\t" + "xvreplgr2vr.w $xr6, %[m_4]\n\t" + "xvreplgr2vr.w $xr7, %[m_6]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvsrli.w $xr1, $xr0, 2\n\t" + "xvsrli.w $xr2, $xr0, 4\n\t" + "xvsrli.w $xr3, $xr0, 6\n\t" + "xvand.v $xr0, $xr0, $xr4\n\t" + "xvand.v $xr1, $xr1, $xr5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvand.v $xr3, $xr3, $xr7\n\t" + "xvor.v $xr0, $xr0, $xr1\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvor.v $xr0, $xr0, $xr2\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ar30), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [m_a] "r"(m_a), [m_2] "r"(m_2), [m_4] "r"(m_4), [m_6] "r"(m_6) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7"); +} +#endif // HAS_AR30TOARGBROW_LASX + +#ifdef HAS_AR30TOABGRROW_LASX +void AR30ToABGRRow_LASX(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width) { + uint32_t m_a = 0xc0000000u; + uint32_t m_2 = 0x300000ffu; + uint32_t m_4 = 0x0c00ff00u; + uint32_t m_6 = 0x03ff0000u; + asm volatile( + "xvreplgr2vr.w $xr4, %[m_a]\n\t" + "xvreplgr2vr.w $xr5, %[m_2]\n\t" + "xvreplgr2vr.w $xr6, %[m_4]\n\t" + "xvreplgr2vr.w $xr7, %[m_6]\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvsrli.w $xr1, $xr0, 2\n\t" + "xvsrli.w $xr2, $xr0, 4\n\t" + "xvsrli.w $xr3, $xr0, 6\n\t" + "xvand.v $xr0, $xr0, $xr4\n\t" + "xvand.v $xr1, $xr1, $xr5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvand.v $xr3, $xr3, $xr7\n\t" + "xvor.v $xr0, $xr0, $xr1\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvor.v $xr0, $xr0, $xr2\n\t" + "xvshuf4i.b $xr0, $xr0, 0xc6\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ar30), [dst] "+r"(dst_abgr), [cnt] "+r"(width) + : [m_a] "r"(m_a), [m_2] "r"(m_2), [m_4] "r"(m_4), [m_6] "r"(m_6) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7"); +} +#endif // HAS_AR30TOABGRROW_LASX + +#ifdef HAS_AYUVTOYROW_LASX +void AYUVToYRow_LASX(const uint8_t* src_ayuv, uint8_t* dst_y, int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvpickev.h $xr0, $xr1, $xr0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvpickev.b $xr0, $xr0, $xr0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_ayuv), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); +} +#endif // HAS_AYUVTOYROW_LASX + +#ifdef HAS_AYUVTOUVROW_LASX +void AYUVToUVRow_LASX(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv; + asm volatile( + "xvldi $xr6, 0x02\n\t" + "1:\n\t" + "xvld $xr0, %[src0], 0\n\t" + "xvld $xr1, %[src0], 32\n\t" + "xvld $xr2, %[src1], 0\n\t" + "xvld $xr3, %[src1], 32\n\t" + "xvaddwev.h.bu $xr4, $xr0, $xr2\n\t" + "xvaddwod.h.bu $xr0, $xr0, $xr2\n\t" + "xvaddwev.h.bu $xr5, $xr1, $xr3\n\t" + "xvaddwod.h.bu $xr1, $xr1, $xr3\n\t" + "xvpickev.d $xr2, $xr5, $xr4\n\t" + "xvpickod.d $xr3, $xr5, $xr4\n\t" + "xvpickev.d $xr4, $xr1, $xr0\n\t" + "xvpickod.d $xr5, $xr1, $xr0\n\t" + "xvadd.h $xr2, $xr2, $xr3\n\t" + "xvadd.h $xr4, $xr4, $xr5\n\t" + "xvadd.h $xr2, $xr2, $xr6\n\t" + "xvadd.h $xr4, $xr4, $xr6\n\t" + "xvsrli.h $xr2, $xr2, 2\n\t" + "xvsrli.h $xr4, $xr4, 2\n\t" + "xvpickev.b $xr0, $xr4, $xr2\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvshuf4i.b $xr0, $xr0, 0xb1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 64\n\t" + "addi.d %[src1], %[src1], 64\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_ayuv), [src1] "+r"(src_ayuv1), [dst] "+r"(dst_uv), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6"); +} +#endif // HAS_AYUVTOUVROW_LASX + +#ifdef HAS_AYUVTOVUROW_LASX +void AYUVToVURow_LASX(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv; + asm volatile( + "xvldi $xr6, 0x02\n\t" + "1:\n\t" + "xvld $xr0, %[src0], 0\n\t" + "xvld $xr1, %[src0], 32\n\t" + "xvld $xr2, %[src1], 0\n\t" + "xvld $xr3, %[src1], 32\n\t" + "xvaddwev.h.bu $xr4, $xr0, $xr2\n\t" + "xvaddwod.h.bu $xr0, $xr0, $xr2\n\t" + "xvaddwev.h.bu $xr5, $xr1, $xr3\n\t" + "xvaddwod.h.bu $xr1, $xr1, $xr3\n\t" + "xvpickev.d $xr2, $xr5, $xr4\n\t" + "xvpickod.d $xr3, $xr5, $xr4\n\t" + "xvpickev.d $xr4, $xr1, $xr0\n\t" + "xvpickod.d $xr5, $xr1, $xr0\n\t" + "xvadd.h $xr2, $xr2, $xr3\n\t" + "xvadd.h $xr4, $xr4, $xr5\n\t" + "xvadd.h $xr2, $xr2, $xr6\n\t" + "xvadd.h $xr4, $xr4, $xr6\n\t" + "xvsrli.h $xr2, $xr2, 2\n\t" + "xvsrli.h $xr4, $xr4, 2\n\t" + "xvpickev.b $xr0, $xr4, $xr2\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 64\n\t" + "addi.d %[src1], %[src1], 64\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_ayuv), [src1] "+r"(src_ayuv1), [dst] "+r"(dst_vu), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6"); +} +#endif // HAS_AYUVTOVUROW_LASX + +#ifdef HAS_DETILEROW_LASX +void DetileRow_LASX(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width) { + const uint8_t* src1 = src + src_tile_stride; + ptrdiff_t src_tile_stride2 = src_tile_stride * 2; + asm volatile( + "li.d $t0, 32\n\t" + "blt %[cnt], $t0, 2f\n\t" + "1:\n\t" + "vld $vr0, %[src0], 0\n\t" + "vld $vr1, %[src1], 0\n\t" + "xvpermi.q $xr0, $xr1, 0x20\n\t" + "xvst $xr0, %[dst], 0\n\t" + "add.d %[src0], %[src0], %[str2]\n\t" + "add.d %[src1], %[src1], %[str2]\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bge %[cnt], $t0, 1b\n\t" + "2:\n\t" + "beqz %[cnt], 3f\n\t" + "vld $vr0, %[src0], 0\n\t" + "vst $vr0, %[dst], 0\n\t" + "3:\n\t" + : [src0] "+r"(src), [src1] "+r"(src1), [dst] "+r"(dst), [cnt] "+r"(width) + : [str2] "r"(src_tile_stride2) + : "memory", "$t0", "$xr0", "$xr1"); +} +#endif // HAS_DETILEROW_LASX + +#ifdef HAS_DETILEROW_16_LASX +void DetileRow_16_LASX(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width) { + ptrdiff_t byte_stride = src_tile_stride * 2; + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvst $xr0, %[dst], 0\n\t" + "add.d %[src], %[src], %[str]\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : [str] "r"(byte_stride) + : "memory", "$xr0"); +} +#endif // HAS_DETILEROW_16_LASX + +#ifdef HAS_MERGEARGBROW_LASX +void MergeARGBRow_LASX(const uint8_t* src_b, + const uint8_t* src_g, + const uint8_t* src_r, + const uint8_t* src_a, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[sb], 0\n\t" + "vld $vr1, %[sg], 0\n\t" + "vld $vr2, %[sr], 0\n\t" + "vld $vr3, %[sa], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvpermi.d $xr2, $xr2, 0xd8\n\t" + "xvpermi.d $xr3, $xr3, 0xd8\n\t" + "xvilvl.b $xr4, $xr1, $xr0\n\t" + "xvilvl.b $xr5, $xr3, $xr2\n\t" + "xvpermi.d $xr4, $xr4, 0xd8\n\t" + "xvpermi.d $xr5, $xr5, 0xd8\n\t" + "xvilvl.h $xr0, $xr5, $xr4\n\t" + "xvilvh.h $xr1, $xr5, $xr4\n\t" + "xvst $xr0, %[dst], 0\n\t" + "xvst $xr1, %[dst], 32\n\t" + "addi.d %[sb], %[sb], 16\n\t" + "addi.d %[sg], %[sg], 16\n\t" + "addi.d %[sr], %[sr], 16\n\t" + "addi.d %[sa], %[sa], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sb] "+r"(src_b), [sg] "+r"(src_g), [sr] "+r"(src_r), + [sa] "+r"(src_a), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5"); +} +#endif // HAS_MERGEARGBROW_LASX + +#ifdef HAS_MERGEXRGBROW_LASX +void MergeXRGBRow_LASX(const uint8_t* src_b, + const uint8_t* src_g, + const uint8_t* src_r, + uint8_t* dst_argb, + int width) { + asm volatile( + "xvldi $xr3, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sb], 0\n\t" + "vld $vr1, %[sg], 0\n\t" + "vld $vr2, %[sr], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvpermi.d $xr2, $xr2, 0xd8\n\t" + "xvilvl.b $xr4, $xr1, $xr0\n\t" + "xvilvl.b $xr5, $xr3, $xr2\n\t" + "xvpermi.d $xr4, $xr4, 0xd8\n\t" + "xvpermi.d $xr5, $xr5, 0xd8\n\t" + "xvilvl.h $xr0, $xr5, $xr4\n\t" + "xvilvh.h $xr1, $xr5, $xr4\n\t" + "xvst $xr0, %[dst], 0\n\t" + "xvst $xr1, %[dst], 32\n\t" + "addi.d %[sb], %[sb], 16\n\t" + "addi.d %[sg], %[sg], 16\n\t" + "addi.d %[sr], %[sr], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sb] "+r"(src_b), [sg] "+r"(src_g), [sr] "+r"(src_r), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5"); +} +#endif // HAS_MERGEXRGBROW_LASX + +#ifdef HAS_SPLITARGBROW_LASX +void SplitARGBRow_LASX(const uint8_t* src_argb, + uint8_t* dst_b, + uint8_t* dst_g, + uint8_t* dst_r, + uint8_t* dst_a, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickod.b $xr3, $xr1, $xr0\n\t" + "xvpermi.d $xr2, $xr2, 0xd8\n\t" + "xvpermi.d $xr3, $xr3, 0xd8\n\t" + "xvpickev.b $xr0, $xr2, $xr2\n\t" + "xvpickod.b $xr1, $xr2, $xr2\n\t" + "xvpickev.b $xr4, $xr3, $xr3\n\t" + "xvpickod.b $xr5, $xr3, $xr3\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvpermi.d $xr4, $xr4, 0xd8\n\t" + "xvpermi.d $xr5, $xr5, 0xd8\n\t" + "vst $vr0, %[db], 0\n\t" + "vst $vr4, %[dg], 0\n\t" + "vst $vr1, %[dr], 0\n\t" + "vst $vr5, %[da], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[db], %[db], 16\n\t" + "addi.d %[dg], %[dg], 16\n\t" + "addi.d %[dr], %[dr], 16\n\t" + "addi.d %[da], %[da], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [db] "+r"(dst_b), [dg] "+r"(dst_g), + [dr] "+r"(dst_r), [da] "+r"(dst_a), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5"); +} +#endif // HAS_SPLITARGBROW_LASX + +#ifdef HAS_SPLITXRGBROW_LASX +void SplitXRGBRow_LASX(const uint8_t* src_argb, + uint8_t* dst_b, + uint8_t* dst_g, + uint8_t* dst_r, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvld $xr1, %[src], 32\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickod.b $xr3, $xr1, $xr0\n\t" + "xvpermi.d $xr2, $xr2, 0xd8\n\t" + "xvpermi.d $xr3, $xr3, 0xd8\n\t" + "xvpickev.b $xr0, $xr2, $xr2\n\t" + "xvpickod.b $xr1, $xr2, $xr2\n\t" + "xvpickev.b $xr4, $xr3, $xr3\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvpermi.d $xr4, $xr4, 0xd8\n\t" + "vst $vr0, %[db], 0\n\t" + "vst $vr4, %[dg], 0\n\t" + "vst $vr1, %[dr], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[db], %[db], 16\n\t" + "addi.d %[dg], %[dg], 16\n\t" + "addi.d %[dr], %[dr], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [db] "+r"(dst_b), [dg] "+r"(dst_g), + [dr] "+r"(dst_r), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); +} +#endif // HAS_SPLITXRGBROW_LASX + +#ifdef HAS_RGB24MIRRORROW_LASX +static const uvec8 kRGB24MirrorShuf0_LASX = {13, 14, 15, 10, 11, 12, 7, 8, + 9, 4, 5, 6, 1, 2, 3, 0}; +static const uvec8 kRGB24MirrorShuf1_LASX = {14, 15, 16, 11, 12, 13, 8, 9, + 10, 5, 6, 7, 2, 3, 4, 1}; +static const uvec8 kRGB24MirrorShuf2_LASX = {15, 0, 1, 12, 13, 14, 9, 10, + 11, 6, 7, 8, 3, 4, 5, 2}; +void RGB24MirrorRow_LASX(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width) { + src_rgb24 += width * 3 - 96; + asm volatile( + "vld $vr4, %[sh0], 0\n\t" + "vld $vr5, %[sh1], 0\n\t" + "vld $vr6, %[sh2], 0\n\t" + "xvpermi.q $xr4, $xr4, 0x00\n\t" + "xvpermi.q $xr5, $xr5, 0x00\n\t" + "xvpermi.q $xr6, $xr6, 0x00\n\t" + "1:\n\t" + "vld $vr2, %[src], 80\n\t" + "vld $vr1, %[src], 64\n\t" + "vld $vr0, %[src], 48\n\t" + "vld $vr7, %[src], 32\n\t" + "vld $vr8, %[src], 16\n\t" + "vld $vr9, %[src], 0\n\t" + "xvpermi.q $xr2, $xr7, 0x20\n\t" + "xvpermi.q $xr1, $xr8, 0x20\n\t" + "xvpermi.q $xr0, $xr9, 0x20\n\t" + "xvshuf.b $xr3, $xr2, $xr1, $xr4\n\t" + "xvshuf.b $xr1, $xr1, $xr0, $xr5\n\t" + "xvshuf.b $xr0, $xr0, $xr0, $xr6\n\t" + "vst $vr3, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "vst $vr0, %[dst], 32\n\t" + "xvpermi.q $xr3, $xr3, 0x01\n\t" + "xvpermi.q $xr1, $xr1, 0x01\n\t" + "xvpermi.q $xr0, $xr0, 0x01\n\t" + "vst $vr3, %[dst], 48\n\t" + "vst $vr1, %[dst], 64\n\t" + "vst $vr0, %[dst], 80\n\t" + "addi.d %[src], %[src], -96\n\t" + "addi.d %[dst], %[dst], 96\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgb24), [dst] "+r"(dst_rgb24), [cnt] "+r"(width) + : [sh0] "r"(&kRGB24MirrorShuf0_LASX), [sh1] "r"(&kRGB24MirrorShuf1_LASX), + [sh2] "r"(&kRGB24MirrorShuf2_LASX) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9"); +} +#endif // HAS_RGB24MIRRORROW_LASX + +#ifdef HAS_BLENDPLANEROW_LASX +void BlendPlaneRow_LASX(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width) { + asm volatile( + "xvldi $xr6, 0x7f\n\t" + "xvslli.h $xr6, $xr6, 1\n\t" + "1:\n\t" + "xvld $xr0, %[s0], 0\n\t" + "xvld $xr1, %[s1], 0\n\t" + "xvld $xr2, %[alp], 0\n\t" + "xvxori.b $xr3, $xr2, 0xff\n\t" + "xvmulwev.h.bu $xr4, $xr0, $xr2\n\t" + "xvmaddwev.h.bu $xr4, $xr1, $xr3\n\t" + "xvmulwod.h.bu $xr5, $xr0, $xr2\n\t" + "xvmaddwod.h.bu $xr5, $xr1, $xr3\n\t" + "xvadd.h $xr4, $xr4, $xr6\n\t" + "xvadd.h $xr5, $xr5, $xr6\n\t" + "xvsrli.h $xr4, $xr4, 8\n\t" + "xvsrli.h $xr5, $xr5, 8\n\t" + "xvpackev.b $xr0, $xr5, $xr4\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[alp], %[alp], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src0), [s1] "+r"(src1), [alp] "+r"(alpha), [dst] "+r"(dst), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6"); +} +#endif // HAS_BLENDPLANEROW_LASX + +#ifdef HAS_INTERPOLATEROW_16_LASX +void InterpolateRow_16_LASX(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + int y1_fraction = source_y_fraction; + int y0_fraction = 256 - y1_fraction; + const uint16_t* src_ptr1 = src_ptr + src_stride; + if (y1_fraction == 0) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src0], 0\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_ptr), [dst] "+r"(dst_ptr), [cnt] "+r"(width) + : + : "memory", "$xr0"); + return; + } + if (y1_fraction == 128) { + HalfRow_16_LASX(src_ptr, src_stride, dst_ptr, width); + return; + } + asm volatile( + "xvreplgr2vr.h $xr4, %[f0]\n\t" + "xvreplgr2vr.h $xr5, %[f1]\n\t" + "1:\n\t" + "xvld $xr0, %[src0], 0\n\t" + "xvld $xr1, %[src1], 0\n\t" + "xvmulwev.w.hu $xr2, $xr0, $xr4\n\t" + "xvmaddwev.w.hu $xr2, $xr1, $xr5\n\t" + "xvmulwod.w.hu $xr3, $xr0, $xr4\n\t" + "xvmaddwod.w.hu $xr3, $xr1, $xr5\n\t" + "xvsrari.w $xr2, $xr2, 8\n\t" + "xvsrari.w $xr3, $xr3, 8\n\t" + "xvpackev.h $xr3, $xr3, $xr2\n\t" + "xvst $xr3, %[dst], 0\n\t" + "addi.d %[src0], %[src0], 32\n\t" + "addi.d %[src1], %[src1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src0] "+r"(src_ptr), [src1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), + [cnt] "+r"(width) + : [f0] "r"(y0_fraction), [f1] "r"(y1_fraction) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5"); +} +#endif // HAS_INTERPOLATEROW_16_LASX + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_neon64.cc b/source/row_neon64.cc index c783ad5cd..c2513e1ff 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -4990,84 +4990,6 @@ void ConvertFP32ToFP16Row_NEON(const float* src, : "cc", "memory", "v1", "v2", "v3"); } -float ScaleMaxSamples_NEON(const float* src, - float* dst, - float scale, - int width) { - float fmax; - asm volatile( - "movi v5.4s, #0 \n" // max - "movi v6.4s, #0 \n" - - "1: \n" - "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples - "subs %w2, %w2, #8 \n" // 8 processed per loop - "fmul v3.4s, v1.4s, %4.s[0] \n" // scale - "prfm pldl1keep, [%0, 448] \n" - "fmul v4.4s, v2.4s, %4.s[0] \n" // scale - "fmax v5.4s, v5.4s, v1.4s \n" // max - "fmax v6.4s, v6.4s, v2.4s \n" - "st1 {v3.4s, v4.4s}, [%1], #32 \n" // store 8 samples - "b.gt 1b \n" - "fmax v5.4s, v5.4s, v6.4s \n" // max - "fmaxv %s3, v5.4s \n" // signed max acculator - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width), // %2 - "=w"(fmax) // %3 - : "w"(scale) // %4 - : "cc", "memory", "v1", "v2", "v3", "v4", "v5", "v6"); - return fmax; -} - -float ScaleSumSamples_NEON(const float* src, - float* dst, - float scale, - int width) { - float fsum; - asm volatile( - "movi v5.4s, #0 \n" // max - "movi v6.4s, #0 \n" // max - - "1: \n" - "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples - "subs %w2, %w2, #8 \n" // 8 processed per loop - "fmul v3.4s, v1.4s, %4.s[0] \n" // scale - "prfm pldl1keep, [%0, 448] \n" - "fmul v4.4s, v2.4s, %4.s[0] \n" - "fmla v5.4s, v1.4s, v1.4s \n" // sum of squares - "fmla v6.4s, v2.4s, v2.4s \n" - "st1 {v3.4s, v4.4s}, [%1], #32 \n" // store 8 samples - "b.gt 1b \n" - "faddp v5.4s, v5.4s, v6.4s \n" - "faddp v5.4s, v5.4s, v5.4s \n" - "faddp %3.4s, v5.4s, v5.4s \n" // sum - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width), // %2 - "=w"(fsum) // %3 - : "w"(scale) // %4 - : "cc", "memory", "v1", "v2", "v3", "v4", "v5", "v6"); - return fsum; -} - -void ScaleSamples_NEON(const float* src, float* dst, float scale, int width) { - asm volatile( - "1: \n" - "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples - "subs %w2, %w2, #8 \n" // 8 processed per loop - "prfm pldl1keep, [%0, 448] \n" - "fmul v1.4s, v1.4s, %3.s[0] \n" // scale - "fmul v2.4s, v2.4s, %3.s[0] \n" // scale - "st1 {v1.4s, v2.4s}, [%1], #32 \n" // store 8 samples - "b.gt 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "w"(scale) // %3 - : "cc", "memory", "v1", "v2"); -} - // filter 5 rows with 1, 4, 6, 4, 1 coefficients to produce 1 row. void GaussCol_NEON(const uint16_t* src0, const uint16_t* src1, diff --git a/source/row_win.cc b/source/row_win.cc index 18d5007fd..ef245ca27 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -9429,90 +9429,6 @@ void GaussCol_F32_AVX2(const float* src0, } #endif // HAS_GAUSSCOL_F32_AVX2 -#ifdef HAS_SCALEMAXSAMPLES_AVX2 -LIBYUV_TARGET_AVX2 -float ScaleMaxSamples_AVX2(const float* src, - float* dst, - float scale, - int width) { - __m256 vmax = _mm256_setzero_ps(); - const __m256 vscale = _mm256_set1_ps(scale); - while (width >= 8) { - __m256 v = _mm256_loadu_ps(src); - src += 8; - vmax = _mm256_max_ps(v, vmax); - _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); - dst += 8; - width -= 8; - } - __m128 m128 = - _mm_max_ps(_mm256_extractf128_ps(vmax, 1), _mm256_castps256_ps128(vmax)); - m128 = _mm_max_ps(_mm_movehl_ps(m128, m128), m128); - m128 = _mm_max_ss(_mm_shuffle_ps(m128, m128, 1), m128); - float fmax = _mm_cvtss_f32(m128); - _mm256_zeroupper(); - // TODO(fbarchard): Handle remainder with masking or SIMD Any function. - for (; width > 0; --width) { - float v = *src++; - float vs = v * scale; - fmax = (v > fmax) ? v : fmax; - *dst++ = vs; - } - return fmax; -} -#endif // HAS_SCALEMAXSAMPLES_AVX2 - -#ifdef HAS_SCALESUMSAMPLES_AVX2 -LIBYUV_TARGET_AVX2 -float ScaleSumSamples_AVX2(const float* src, - float* dst, - float scale, - int width) { - __m256 vsum = _mm256_setzero_ps(); - const __m256 vscale = _mm256_set1_ps(scale); - while (width >= 8) { - __m256 v = _mm256_loadu_ps(src); - src += 8; - vsum = _mm256_add_ps(vsum, _mm256_mul_ps(v, v)); - _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); - dst += 8; - width -= 8; - } - __m128 s128 = - _mm_add_ps(_mm256_castps256_ps128(vsum), _mm256_extractf128_ps(vsum, 1)); - s128 = _mm_add_ps(s128, _mm_movehl_ps(s128, s128)); - s128 = _mm_add_ss(s128, _mm_shuffle_ps(s128, s128, 1)); - float fsum = _mm_cvtss_f32(s128); - _mm256_zeroupper(); - // TODO(fbarchard): Handle remainder with masking or SIMD Any function. - for (; width > 0; --width) { - float v = *src++; - fsum += v * v; - *dst++ = v * scale; - } - return fsum; -} -#endif // HAS_SCALESUMSAMPLES_AVX2 - -#ifdef HAS_SCALESAMPLES_AVX2 -LIBYUV_TARGET_AVX2 -void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) { - const __m256 vscale = _mm256_set1_ps(scale); - while (width >= 8) { - __m256 v = _mm256_loadu_ps(src); - src += 8; - _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); - dst += 8; - width -= 8; - } - _mm256_zeroupper(); - if (width > 0) { - // TODO(fbarchard): Handle remainder with masking or SIMD Any function. - ScaleSamples_C(src, dst, scale, width); - } -} -#endif // HAS_SCALESAMPLES_AVX2 - #if defined(HAS_ARGBTORGB24ROW_AVX2) || defined(HAS_ARGBTORAWROW_AVX2) LIBYUV_TARGET_AVX2 static inline void ARGBToRGBRow_AVX2(const uint8_t* src, diff --git a/source/scale.cc b/source/scale.cc index 29745956a..f3cb00aa6 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -138,6 +138,21 @@ static void ScalePlaneDown2(int src_width, } } #endif +#if defined(HAS_SCALEROWDOWN2_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ScaleRowDown2 = + filtering == kFilterNone + ? ScaleRowDown2_Any_LASX + : (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_LASX + : ScaleRowDown2Box_Any_LASX); + if (IS_ALIGNED(dst_width, 32)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_LASX + : (filtering == kFilterLinear + ? ScaleRowDown2Linear_LASX + : ScaleRowDown2Box_LASX); + } + } +#endif #if defined(HAS_SCALEROWDOWN2_RVV) if (TestCpuFlag(kCpuHasRVV)) { ScaleRowDown2 = filtering == kFilterNone @@ -314,6 +329,15 @@ static void ScalePlaneDown4(int src_width, } } #endif +#if defined(HAS_SCALEROWDOWN4_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ScaleRowDown4 = + filtering ? ScaleRowDown4Box_Any_LASX : ScaleRowDown4_Any_LASX; + if (IS_ALIGNED(dst_width, 16)) { + ScaleRowDown4 = filtering ? ScaleRowDown4Box_LASX : ScaleRowDown4_LASX; + } + } +#endif #if defined(HAS_SCALEROWDOWN4_RVV) if (TestCpuFlag(kCpuHasRVV)) { ScaleRowDown4 = filtering ? ScaleRowDown4Box_RVV : ScaleRowDown4_RVV; @@ -448,6 +472,27 @@ static void ScalePlaneDown34(int src_width, } } #endif +#if defined(HAS_SCALEROWDOWN34_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + if (dst_width % 24 == 0) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_LASX; + ScaleRowDown34_1 = ScaleRowDown34_LASX; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_LASX; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_LASX; + } + } else { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_Any_LASX; + ScaleRowDown34_1 = ScaleRowDown34_Any_LASX; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_LASX; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_LASX; + } + } + } +#endif #if defined(HAS_SCALEROWDOWN34_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { if (dst_width % 24 == 0) { @@ -675,6 +720,26 @@ static void ScalePlaneDown38(int src_width, } } #endif +#if defined(HAS_SCALEROWDOWN38_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_Any_LASX; + ScaleRowDown38_2 = ScaleRowDown38_Any_LASX; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_LASX; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_LASX; + } + if (dst_width % 12 == 0) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_LASX; + ScaleRowDown38_2 = ScaleRowDown38_LASX; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_LASX; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_LASX; + } + } + } +#endif #if defined(HAS_SCALEROWDOWN38_RVV) if (TestCpuFlag(kCpuHasRVV)) { if (!filtering) { @@ -979,6 +1044,14 @@ static int ScalePlaneBox(int src_width, } } #endif +#if defined(HAS_SCALEADDROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ScaleAddRow = ScaleAddRow_Any_LASX; + if (IS_ALIGNED(src_width, 32)) { + ScaleAddRow = ScaleAddRow_LASX; + } + } +#endif #if defined(HAS_SCALEADDROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ScaleAddRow = ScaleAddRow_RVV; @@ -1193,6 +1266,14 @@ static int ScalePlaneBilinearDown(int src_width, } } #endif +#if defined(HAS_SCALEFILTERCOLS_LASX) + if (TestCpuFlag(kCpuHasLASX) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleFilterCols_LASX; + } + } +#endif #if defined(HAS_SCALEFILTERCOLS_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { ScaleFilterCols = ScaleFilterCols_Any_WASMSIMD; @@ -1294,6 +1375,22 @@ static int ScalePlaneBilinearDown_16(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + InterpolateRow = InterpolateRow_16_Any_LSX; + if (IS_ALIGNED(src_width, 8)) { + InterpolateRow = InterpolateRow_16_LSX; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_16_Any_LASX; + if (IS_ALIGNED(src_width, 16)) { + InterpolateRow = InterpolateRow_16_LASX; + } + } +#endif uint32_t uy = y; for (j = 0; j < dst_height; ++j) { @@ -1380,6 +1477,22 @@ static int ScalePlaneBilinearUp(int src_width, InterpolateRow = InterpolateRow_SME; } #endif +#if defined(HAS_INTERPOLATEROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + InterpolateRow = InterpolateRow_Any_LSX; + if (IS_ALIGNED(dst_width, 16)) { + InterpolateRow = InterpolateRow_LSX; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(dst_width, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; @@ -1423,6 +1536,14 @@ static int ScalePlaneBilinearUp(int src_width, } } #endif +#if defined(HAS_SCALEFILTERCOLS_LASX) + if (filtering && TestCpuFlag(kCpuHasLASX) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleFilterCols_LASX; + } + } +#endif #if defined(HAS_SCALEFILTERCOLS_WASMSIMD) if (filtering && TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { ScaleFilterCols = ScaleFilterCols_Any_WASMSIMD; @@ -1910,6 +2031,22 @@ static int ScalePlaneBilinearUp_16(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + InterpolateRow = InterpolateRow_16_Any_LSX; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_16_LSX; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_16_Any_LASX; + if (IS_ALIGNED(dst_width, 16)) { + InterpolateRow = InterpolateRow_16_LASX; + } + } +#endif if (filtering && src_width >= 32768) { ScaleFilterCols = ScaleFilterCols64_16_C; diff --git a/source/scale_any.cc b/source/scale_any.cc index ec903768e..bac11e3fe 100644 --- a/source/scale_any.cc +++ b/source/scale_any.cc @@ -206,6 +206,21 @@ SDANY(ScaleRowDown2Box_Any_LSX, 1, 15) #endif +#ifdef HAS_SCALEROWDOWN2_LASX +SDANY(ScaleRowDown2_Any_LASX, ScaleRowDown2_LASX, ScaleRowDown2_C, 2, 1, 31) +SDANY(ScaleRowDown2Linear_Any_LASX, + ScaleRowDown2Linear_LASX, + ScaleRowDown2Linear_C, + 2, + 1, + 31) +SDANY(ScaleRowDown2Box_Any_LASX, + ScaleRowDown2Box_LASX, + ScaleRowDown2Box_C, + 2, + 1, + 31) +#endif #ifdef HAS_SCALEROWDOWN4_SSSE3 SDANY(ScaleRowDown4_Any_SSSE3, ScaleRowDown4_SSSE3, ScaleRowDown4_C, 4, 1, 7) SDANY(ScaleRowDown4Box_Any_SSSE3, @@ -242,6 +257,15 @@ SDANY(ScaleRowDown4Box_Any_LSX, 1, 7) #endif +#ifdef HAS_SCALEROWDOWN4_LASX +SDANY(ScaleRowDown4_Any_LASX, ScaleRowDown4_LASX, ScaleRowDown4_C, 4, 1, 15) +SDANY(ScaleRowDown4Box_Any_LASX, + ScaleRowDown4Box_LASX, + ScaleRowDown4Box_C, + 4, + 1, + 15) +#endif #ifdef HAS_SCALEROWDOWN34_AVX2 SDANY(ScaleRowDown34_Any_AVX2, ScaleRowDown34_AVX2, @@ -323,6 +347,26 @@ SDANY(ScaleRowDown34_1_Box_Any_LSX, 1, 11) #endif +#ifdef HAS_SCALEROWDOWN34_LASX +SDANY(ScaleRowDown34_Any_LASX, + ScaleRowDown34_LASX, + ScaleRowDown34_C, + 4 / 3, + 1, + 23) +SDANY(ScaleRowDown34_0_Box_Any_LASX, + ScaleRowDown34_0_Box_LASX, + ScaleRowDown34_0_Box_C, + 4 / 3, + 1, + 23) +SDANY(ScaleRowDown34_1_Box_Any_LASX, + ScaleRowDown34_1_Box_LASX, + ScaleRowDown34_1_Box_C, + 4 / 3, + 1, + 23) +#endif #ifdef HAS_SCALEROWDOWN38_AVX2 SDANY(ScaleRowDown38_Any_AVX2, ScaleRowDown38_AVX2, @@ -383,6 +427,26 @@ SDANY(ScaleRowDown38_2_Box_Any_LSX, 1, 11) #endif +#ifdef HAS_SCALEROWDOWN38_LASX +SDANY(ScaleRowDown38_Any_LASX, + ScaleRowDown38_LASX, + ScaleRowDown38_C, + 8 / 3, + 1, + 11) +SDANY(ScaleRowDown38_3_Box_Any_LASX, + ScaleRowDown38_3_Box_LASX, + ScaleRowDown38_3_Box_C, + 8 / 3, + 1, + 11) +SDANY(ScaleRowDown38_2_Box_Any_LASX, + ScaleRowDown38_2_Box_LASX, + ScaleRowDown38_2_Box_C, + 8 / 3, + 1, + 11) +#endif #ifdef HAS_SCALEARGBROWDOWN2_AVX2 SDANY(ScaleARGBRowDown2_Any_AVX2, @@ -444,6 +508,26 @@ SDANY(ScaleARGBRowDown2Box_Any_LSX, 4, 3) #endif +#ifdef HAS_SCALEARGBROWDOWN2_LASX +SDANY(ScaleARGBRowDown2_Any_LASX, + ScaleARGBRowDown2_LASX, + ScaleARGBRowDown2_C, + 2, + 4, + 7) +SDANY(ScaleARGBRowDown2Linear_Any_LASX, + ScaleARGBRowDown2Linear_LASX, + ScaleARGBRowDown2Linear_C, + 2, + 4, + 7) +SDANY(ScaleARGBRowDown2Box_Any_LASX, + ScaleARGBRowDown2Box_LASX, + ScaleARGBRowDown2Box_C, + 2, + 4, + 7) +#endif #ifdef HAS_SCALEROWDOWN2_WASMSIMD SDANY(ScaleRowDown2_Any_WASMSIMD, ScaleRowDown2_WASMSIMD, @@ -598,6 +682,18 @@ SDAANY(ScaleARGBRowDownEvenBox_Any_LSX, 4, 3) #endif +#ifdef HAS_SCALEARGBROWDOWNEVEN_LASX +SDAANY(ScaleARGBRowDownEven_Any_LASX, + ScaleARGBRowDownEven_LASX, + ScaleARGBRowDownEven_C, + 4, + 7) +SDAANY(ScaleARGBRowDownEvenBox_Any_LASX, + ScaleARGBRowDownEvenBox_LASX, + ScaleARGBRowDownEvenBox_C, + 4, + 7) +#endif #ifdef HAS_SCALEUVROWDOWNEVEN_AVX2 SDAANY(ScaleUVRowDownEven_Any_AVX2, ScaleUVRowDownEven_AVX2, @@ -669,6 +765,9 @@ SAROW(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, 1, 2, 15) #ifdef HAS_SCALEADDROW_LSX SAROW(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, 1, 2, 15) #endif +#ifdef HAS_SCALEADDROW_LASX +SAROW(ScaleAddRow_Any_LASX, ScaleAddRow_LASX, 1, 2, 31) +#endif #ifdef HAS_SCALEADDROW_WASMSIMD SAROW(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, 1, 2, 15) #endif @@ -698,6 +797,9 @@ SAANY(ScaleAddRow_Any_NEON, ScaleAddRow_NEON, ScaleAddRow_C, 15) #ifdef HAS_SCALEADDROW_LSX SAANY(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, ScaleAddRow_C, 15) #endif +#ifdef HAS_SCALEADDROW_LASX +SAANY(ScaleAddRow_Any_LASX, ScaleAddRow_LASX, ScaleAddRow_C, 31) +#endif #ifdef HAS_SCALEADDROW_WASMSIMD SAANY(ScaleAddRow_Any_WASMSIMD, ScaleAddRow_WASMSIMD, ScaleAddRow_C, 15) #endif @@ -724,12 +826,18 @@ CANY(ScaleFilterCols_Any_NEON, ScaleFilterCols_NEON, ScaleFilterCols_C, 1, 7) #ifdef HAS_SCALEFILTERCOLS_LSX CANY(ScaleFilterCols_Any_LSX, ScaleFilterCols_LSX, ScaleFilterCols_C, 1, 3) #endif +#ifdef HAS_SCALEFILTERCOLS_LASX +CANY(ScaleFilterCols_Any_LASX, ScaleFilterCols_LASX, ScaleFilterCols_C, 1, 7) +#endif #ifdef HAS_SCALEARGBCOLS_NEON CANY(ScaleARGBCols_Any_NEON, ScaleARGBCols_NEON, ScaleARGBCols_C, 4, 7) #endif #ifdef HAS_SCALEARGBCOLS_LSX CANY(ScaleARGBCols_Any_LSX, ScaleARGBCols_LSX, ScaleARGBCols_C, 4, 3) #endif +#ifdef HAS_SCALEARGBCOLS_LASX +CANY(ScaleARGBCols_Any_LASX, ScaleARGBCols_LASX, ScaleARGBCols_C, 4, 7) +#endif #ifdef HAS_SCALEARGBFILTERCOLS_NEON CANY(ScaleARGBFilterCols_Any_NEON, ScaleARGBFilterCols_NEON, @@ -744,6 +852,13 @@ CANY(ScaleARGBFilterCols_Any_LSX, 4, 3) #endif +#ifdef HAS_SCALEARGBFILTERCOLS_LASX +CANY(ScaleARGBFilterCols_Any_LASX, + ScaleARGBFilterCols_LASX, + ScaleARGBFilterCols_C, + 4, + 7) +#endif #ifdef HAS_SCALEFILTERCOLS_WASMSIMD CANY(ScaleFilterCols_Any_WASMSIMD, ScaleFilterCols_WASMSIMD, @@ -1204,6 +1319,9 @@ SUVAC3ANY(ScaleUVAddCols3_Any_NEON, ScaleUVAddCols3_NEON, ScaleUVAddCols3_C, 7) #ifdef HAS_SCALEUVADDCOLS3_LSX SUVAC3ANY(ScaleUVAddCols3_Any_LSX, ScaleUVAddCols3_LSX, ScaleUVAddCols3_C, 7) #endif +#ifdef HAS_SCALEUVADDCOLS3_LASX +SUVAC3ANY(ScaleUVAddCols3_Any_LASX, ScaleUVAddCols3_LASX, ScaleUVAddCols3_C, 7) +#endif #ifdef HAS_SCALEUVADDCOLS3_WASMSIMD SUVAC3ANY(ScaleUVAddCols3_Any_WASMSIMD, ScaleUVAddCols3_WASMSIMD, diff --git a/source/scale_argb.cc b/source/scale_argb.cc index 70e2e69bf..3ae90b61a 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -123,6 +123,22 @@ static void ScaleARGBDown2(int src_width, } } #endif +#if defined(HAS_SCALEARGBROWDOWN2_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ScaleARGBRowDown2 = + filtering == kFilterNone + ? ScaleARGBRowDown2_Any_LASX + : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_LASX + : ScaleARGBRowDown2Box_Any_LASX); + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBRowDown2 = + filtering == kFilterNone + ? ScaleARGBRowDown2_LASX + : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_LASX + : ScaleARGBRowDown2Box_LASX); + } + } +#endif #if defined(HAS_SCALEARGBROWDOWN2_RVV) && \ defined(HAS_SCALEARGBROWDOWN2LINEAR_RVV) && \ defined(HAS_SCALEARGBROWDOWN2BOX_RVV) @@ -226,6 +242,22 @@ static int ScaleARGBDown4Box(int src_width, ScaleARGBRowDown2 = ScaleARGBRowDown2Box_SME; } #endif +#if defined(HAS_SCALEARGBROWDOWN2_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_LSX; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_LSX; + } + } +#endif +#if defined(HAS_SCALEARGBROWDOWN2_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_LASX; + } + } +#endif #if defined(HAS_SCALEARGBROWDOWN2BOX_RVV) if (TestCpuFlag(kCpuHasRVV)) { ScaleARGBRowDown2 = ScaleARGBRowDown2Box_RVV; @@ -314,6 +346,16 @@ static void ScaleARGBDownEven(int src_width, } } #endif +#if defined(HAS_SCALEARGBROWDOWNEVEN_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_LASX + : ScaleARGBRowDownEven_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBRowDownEven = + filtering ? ScaleARGBRowDownEvenBox_LASX : ScaleARGBRowDownEven_LASX; + } + } +#endif #if defined(HAS_SCALEARGBROWDOWNEVENBOX_RVV) if (filtering && TestCpuFlag(kCpuHasRVV)) { ScaleARGBRowDownEven = ScaleARGBRowDownEvenBox_RVV; @@ -465,6 +507,14 @@ static int ScaleARGBBilinearDown(int src_width, } } #endif +#if defined(HAS_SCALEARGBFILTERCOLS_LASX) + if (TestCpuFlag(kCpuHasLASX) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_LASX; + } + } +#endif #if defined(HAS_SCALEARGBFILTERCOLS_RVV) if (TestCpuFlag(kCpuHasRVV) && src_width < 32768) { ScaleARGBFilterCols = ScaleARGBFilterCols_RVV; @@ -628,6 +678,14 @@ static int ScaleARGBBilinearUp(int src_width, } } #endif +#if defined(HAS_SCALEARGBFILTERCOLS_LASX) + if (filtering && TestCpuFlag(kCpuHasLASX) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_LASX; + } + } +#endif #if defined(HAS_SCALEARGBFILTERCOLS_RVV) if (filtering && TestCpuFlag(kCpuHasRVV) && src_width < 32768) { ScaleARGBFilterCols = ScaleARGBFilterCols_RVV; @@ -654,6 +712,14 @@ static int ScaleARGBBilinearUp(int src_width, } } #endif +#if defined(HAS_SCALEARGBCOLS_LASX) + if (!filtering && TestCpuFlag(kCpuHasLASX) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBCols_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBCols_LASX; + } + } +#endif #if defined(HAS_SCALEARGBFILTERCOLS_WASMSIMD) if (filtering && TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { ScaleARGBFilterCols = ScaleARGBFilterCols_Any_WASMSIMD; @@ -793,6 +859,14 @@ static void ScaleARGBSimple(int src_width, } } #endif +#if defined(HAS_SCALEARGBCOLS_LASX) + if (TestCpuFlag(kCpuHasLASX) && src_width < 32768) { + ScaleARGBCols = ScaleARGBCols_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBCols = ScaleARGBCols_LASX; + } + } +#endif #if defined(HAS_SCALEARGBCOLS_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { ScaleARGBCols = ScaleARGBCols_Any_WASMSIMD; diff --git a/source/scale_common.cc b/source/scale_common.cc index a07a2183e..cdda834e8 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1766,6 +1766,22 @@ void ScalePlaneVertical_16(int src_height, InterpolateRow = InterpolateRow_16_WASMSIMD; } } +#endif +#if defined(HAS_INTERPOLATEROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + InterpolateRow = InterpolateRow_16_Any_LSX; + if (IS_ALIGNED(dst_width_words, 8)) { + InterpolateRow = InterpolateRow_16_LSX; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_16_Any_LASX; + if (IS_ALIGNED(dst_width_words, 16)) { + InterpolateRow = InterpolateRow_16_LASX; + } + } #endif for (j = 0; j < dst_height; ++j) { int yi; @@ -1915,6 +1931,36 @@ void ScaleSlope(int src_width, } #undef CENTERSTART +float ScaleSumSamples_C(const float* src, float* dst, float scale, int width) { + float fsum = 0.f; + int i; + for (i = 0; i < width; ++i) { + float v = *src++; + fsum += v * v; + *dst++ = v * scale; + } + return fsum; +} + +float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width) { + float fmax = 0.f; + int i; + for (i = 0; i < width; ++i) { + float v = *src++; + float vs = v * scale; + fmax = (v > fmax) ? v : fmax; + *dst++ = vs; + } + return fmax; +} + +void ScaleSamples_C(const float* src, float* dst, float scale, int width) { + int i; + for (i = 0; i < width; ++i) { + *dst++ = *src++ * scale; + } +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index 4c239b7d9..cde09b91b 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -4393,6 +4393,161 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, } #endif // HAS_SCALEUVADDCOLS3_AVX512VBMI +#ifdef HAS_SCALEMAXSAMPLES_AVX2 +float ScaleMaxSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + float fmax; + asm volatile( + "vpxor %%xmm3,%%xmm3,%%xmm3 \n" + "vbroadcastss %[scale],%%ymm2 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmaxps %%ymm3,%%ymm0,%%ymm3 \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "vextractf128 $0x1,%%ymm3,%%xmm1 \n" + "vmaxps %%xmm3,%%xmm1,%%xmm3 \n" + "vpshufd $0x4e,%%xmm3,%%xmm1 \n" + "vmaxps %%xmm3,%%xmm1,%%xmm3 \n" + "vpshufd $0x55,%%xmm3,%%xmm1 \n" + "vmaxss %%xmm3,%%xmm1,%%xmm3 \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vmaxss %%xmm3,%%xmm0,%%xmm3 \n" + "vmulss %%xmm2,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vmovss %%xmm3,%[fmax] \n" + "vzeroupper \n" + : [src] "+r"(src), + [dst] "+r"(dst), + [width] "+r"(width), + [fmax] "=m"(fmax) + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + return fmax; +} +#endif // HAS_SCALEMAXSAMPLES_AVX2 + +#ifdef HAS_SCALESUMSAMPLES_AVX2 +float ScaleSumSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + float fsum; + asm volatile( + "vpxor %%xmm3,%%xmm3,%%xmm3 \n" + "vbroadcastss %[scale],%%ymm2 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmulps %%ymm0,%%ymm0,%%ymm1 \n" + "vaddps %%ymm1,%%ymm3,%%ymm3 \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "vextractf128 $0x1,%%ymm3,%%xmm1 \n" + "vaddps %%xmm1,%%xmm3,%%xmm3 \n" + "vhaddps %%xmm3,%%xmm3,%%xmm3 \n" + "vhaddps %%xmm3,%%xmm3,%%xmm3 \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vmulss %%xmm0,%%xmm0,%%xmm1 \n" + "vaddss %%xmm1,%%xmm3,%%xmm3 \n" + "vmulss %%xmm2,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vmovss %%xmm3,%[fsum] \n" + "vzeroupper \n" + : [src] "+r"(src), + [dst] "+r"(dst), + [width] "+r"(width), + [fsum] "=m"(fsum) + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + return fsum; +} +#endif // HAS_SCALESUMSAMPLES_AVX2 + +#ifdef HAS_SCALESAMPLES_AVX2 +void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) { + asm volatile( + "vbroadcastss %[scale],%%ymm2 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vmulss %%xmm2,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm2"); +} +#endif // HAS_SCALESAMPLES_AVX2 + #endif // defined(__x86_64__) || defined(__i386__) #ifdef __cplusplus diff --git a/source/scale_la.cc b/source/scale_la.cc index ddda812eb..c7a4ddd0e 100644 --- a/source/scale_la.cc +++ b/source/scale_la.cc @@ -398,11 +398,14 @@ void ScaleRowDown4Box_LSX(const uint8_t* src_ptr, } #endif // HAS_SCALEROWDOWN4_LSX -#ifdef HAS_SCALEROWDOWN38_LSX +#if defined(HAS_SCALEROWDOWN38_LSX) || defined(HAS_SCALEROWDOWN38_LASX) static const uint64_t kScale38Shuf[2] = {0x13100e0b08060300ULL, 0x000000001e1b1816ULL}; static const uint64_t kScale38BoxShuf[2] = {0x0a08160604120200ULL, 0x000000001e0e0c1aULL}; +#endif + +#ifdef HAS_SCALEROWDOWN38_LSX void ScaleRowDown38_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -581,7 +584,7 @@ void ScaleAddRow_LSX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) { } #endif // HAS_SCALEADDROW_LSX -#ifdef HAS_SCALEUVADDCOLS3_LSX +#if defined(HAS_SCALEUVADDCOLS3_LSX) || defined(HAS_SCALEUVADDCOLS3_LASX) static const uint8_t kScaleUVAddCols3Shuf[] = { 0, 1, 2, 3, 12, 13, 14, 15, 24, 25, 26, 27, 4, 5, 6, 7, 4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31, 8, 9, 10, 11, @@ -590,6 +593,9 @@ static const uint8_t kScaleUVAddCols3Shuf[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 24, 25, 26, 27, 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31, }; +#endif + +#ifdef HAS_SCALEUVADDCOLS3_LSX void ScaleUVAddCols3_LSX(int dst_width, int scaleval, @@ -841,7 +847,7 @@ void ScaleARGBFilterCols_LSX(uint8_t* dst_argb, } #endif // HAS_SCALEARGBFILTERCOLS_LSX -#ifdef HAS_SCALEROWDOWN34_LSX +#if defined(HAS_SCALEROWDOWN34_LSX) || defined(HAS_SCALEROWDOWN34_LASX) static const uint64_t kScale34Shuf[2] = {0x0908070504030100ULL, 0x000000000f0d0c0bULL}; static const uint64_t kScale34BoxLo[2] = {0x0504030202010100ULL, @@ -852,6 +858,9 @@ static const uint64_t kScale34BoxWt[2] = {0x0103030102020103ULL, 0x0000000003010202ULL}; static const uint64_t kScale34Pack12[2] = {0x0908050403020100ULL, 0x000000000d0c0b0aULL}; +#endif + +#ifdef HAS_SCALEROWDOWN34_LSX void ScaleRowDown34_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -994,6 +1003,1283 @@ void ScaleRowDown34_1_Box_LSX(const uint8_t* src_ptr, } #endif // HAS_SCALEROWDOWN34_LSX +#ifdef HAS_SCALEMAXSAMPLES_LSX +float ScaleMaxSamples_LSX(const float* src, + float* dst, + float scale, + int width) { + float fmax = 0.f; + asm volatile( + "vldi $vr3, 0 \n" + "vldrepl.w $vr2, %[scale_ptr], 0 \n" + "addi.w %[w], %[w], -4 \n" + "bltz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "addi.d %[src], %[src], 16 \n" + "vfmax.s $vr3, $vr3, $vr0 \n" + "vfmul.s $vr0, $vr0, $vr2 \n" + "vst $vr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -4 \n" + "bgez %[w], 1b \n" + "vshuf4i.w $vr1, $vr3, 0x4e \n" + "vfmax.s $vr3, $vr3, $vr1 \n" + "vshuf4i.w $vr1, $vr3, 0xb1 \n" + "vfmax.s $vr3, $vr3, $vr1 \n" + "vstelm.w $vr3, %[fmax_ptr], 0, 0 \n" + "2: \n" + "addi.w %[w], %[w], 4 \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width) // %[w] + : [scale_ptr] "r"(&scale), + [fmax_ptr] "r"(&fmax) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); + for (; width > 0; --width) { + float v = *src++; + float vs = v * scale; + fmax = (v > fmax) ? v : fmax; + *dst++ = vs; + } + return fmax; +} +#endif // HAS_SCALEMAXSAMPLES_LSX + +#ifdef HAS_SCALESUMSAMPLES_LSX +float ScaleSumSamples_LSX(const float* src, + float* dst, + float scale, + int width) { + float fsum = 0.f; + asm volatile( + "vldi $vr3, 0 \n" + "vldrepl.w $vr2, %[scale_ptr], 0 \n" + "addi.w %[w], %[w], -4 \n" + "bltz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "addi.d %[src], %[src], 16 \n" + "vfmadd.s $vr3, $vr0, $vr0, $vr3 \n" + "vfmul.s $vr0, $vr0, $vr2 \n" + "vst $vr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -4 \n" + "bgez %[w], 1b \n" + "vshuf4i.w $vr1, $vr3, 0x4e \n" + "vfadd.s $vr3, $vr3, $vr1 \n" + "vshuf4i.w $vr1, $vr3, 0xb1 \n" + "vfadd.s $vr3, $vr3, $vr1 \n" + "vstelm.w $vr3, %[fsum_ptr], 0, 0 \n" + "2: \n" + "addi.w %[w], %[w], 4 \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width) // %[w] + : [scale_ptr] "r"(&scale), + [fsum_ptr] "r"(&fsum) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); + for (; width > 0; --width) { + float v = *src++; + fsum += v * v; + *dst++ = v * scale; + } + return fsum; +} +#endif // HAS_SCALESUMSAMPLES_LSX + +#ifdef HAS_SCALESAMPLES_LSX +void ScaleSamples_LSX(const float* src, float* dst, float scale, int width) { + asm volatile( + "vldrepl.w $vr2, %[scale_ptr], 0 \n" + "addi.w %[w], %[w], -4 \n" + "bltz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "addi.d %[src], %[src], 16 \n" + "vfmul.s $vr0, $vr0, $vr2 \n" + "vst $vr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -4 \n" + "bgez %[w], 1b \n" + "2: \n" + "addi.w %[w], %[w], 4 \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width) // %[w] + : [scale_ptr] "r"(&scale) + : "memory", "$vr0", "$vr2"); + if (width > 0) { + ScaleSamples_C(src, dst, scale, width); + } +} +#endif // HAS_SCALESAMPLES_LSX + +#ifdef HAS_SCALEARGBROWDOWN2_LASX +void ScaleARGBRowDown2_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "xvld $xr1, %[src], 32 \n" + "xvpickod.w $xr0, $xr1, $xr0 \n" + "xvpermi.d $xr0, $xr0, 0xd8 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[src], %[src], 64 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$xr0", "$xr1"); +} + +void ScaleARGBRowDown2Linear_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "xvld $xr1, %[src], 32 \n" + "xvpickev.w $xr2, $xr1, $xr0 \n" + "xvpickod.w $xr3, $xr1, $xr0 \n" + "xvavgr.bu $xr0, $xr3, $xr2 \n" + "xvpermi.d $xr0, $xr0, 0xd8 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[src], %[src], 64 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); +} + +void ScaleARGBRowDown2Box_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + const uint8_t* t = src_argb + src_stride; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[s], 0 \n" + "xvld $xr1, %[s], 32 \n" + "xvld $xr2, %[t], 0 \n" + "xvld $xr3, %[t], 32 \n" + "xvpickev.w $xr4, $xr1, $xr0 \n" + "xvpickod.w $xr5, $xr1, $xr0 \n" + "xvpickev.w $xr6, $xr3, $xr2 \n" + "xvpickod.w $xr7, $xr3, $xr2 \n" + "xvaddwev.h.bu $xr0, $xr4, $xr5 \n" + "xvaddwod.h.bu $xr1, $xr4, $xr5 \n" + "xvaddwev.h.bu $xr2, $xr6, $xr7 \n" + "xvaddwod.h.bu $xr3, $xr6, $xr7 \n" + "xvadd.h $xr0, $xr0, $xr2 \n" + "xvadd.h $xr1, $xr1, $xr3 \n" + "xvsrari.h $xr0, $xr0, 2 \n" + "xvsrari.h $xr1, $xr1, 2 \n" + "xvpackev.b $xr0, $xr1, $xr0 \n" + "xvpermi.d $xr0, $xr0, 0xd8 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[s], %[s], 64 \n" + "addi.d %[t], %[t], 64 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [s] "+r"(src_argb), // %[s] + [t] "+r"(t), // %[t] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7"); +} +#endif // HAS_SCALEARGBROWDOWN2_LASX + +#ifdef HAS_SCALEARGBROWDOWNEVEN_LASX +void ScaleARGBRowDownEven_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + int32_t src_stepx, + uint8_t* dst_argb, + int dst_width) { + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; + uint32_t r0, r1, r2, r3; + (void)src_stride; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "ld.w %[r0], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "ld.w %[r1], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "ld.w %[r2], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "ld.w %[r3], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "xvinsgr2vr.w $xr0, %[r0], 0 \n" + "xvinsgr2vr.w $xr0, %[r1], 1 \n" + "xvinsgr2vr.w $xr0, %[r2], 2 \n" + "xvinsgr2vr.w $xr0, %[r3], 3 \n" + "ld.w %[r0], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "ld.w %[r1], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "ld.w %[r2], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "ld.w %[r3], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "xvinsgr2vr.w $xr0, %[r0], 4 \n" + "xvinsgr2vr.w $xr0, %[r1], 5 \n" + "xvinsgr2vr.w $xr0, %[r2], 6 \n" + "xvinsgr2vr.w $xr0, %[r3], 7 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width), // %[w] + [r0] "=&r"(r0), // %[r0] + [r1] "=&r"(r1), // %[r1] + [r2] "=&r"(r2), // %[r2] + [r3] "=&r"(r3) // %[r3] + : [step] "r"(step) // %[step] + : "memory", "$xr0"); +} + +void ScaleARGBRowDownEvenBox_LASX(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width) { + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; + const uint8_t* next_argb = src_argb + src_stride; + uint64_t r0, r1; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "ld.d %[r0], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "ld.d %[r1], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "xvinsgr2vr.d $xr0, %[r0], 0 \n" + "xvinsgr2vr.d $xr0, %[r1], 1 \n" + "ld.d %[r0], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "ld.d %[r1], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "xvinsgr2vr.d $xr1, %[r0], 0 \n" + "xvinsgr2vr.d $xr1, %[r1], 1 \n" + "ld.d %[r0], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "ld.d %[r1], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "xvinsgr2vr.d $xr0, %[r0], 2 \n" + "xvinsgr2vr.d $xr0, %[r1], 3 \n" + "ld.d %[r0], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "ld.d %[r1], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "xvinsgr2vr.d $xr1, %[r0], 2 \n" + "xvinsgr2vr.d $xr1, %[r1], 3 \n" + + "ld.d %[r0], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "ld.d %[r1], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "xvinsgr2vr.d $xr2, %[r0], 0 \n" + "xvinsgr2vr.d $xr2, %[r1], 1 \n" + "ld.d %[r0], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "ld.d %[r1], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "xvinsgr2vr.d $xr3, %[r0], 0 \n" + "xvinsgr2vr.d $xr3, %[r1], 1 \n" + "ld.d %[r0], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "ld.d %[r1], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "xvinsgr2vr.d $xr2, %[r0], 2 \n" + "xvinsgr2vr.d $xr2, %[r1], 3 \n" + "ld.d %[r0], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "ld.d %[r1], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "xvinsgr2vr.d $xr3, %[r0], 2 \n" + "xvinsgr2vr.d $xr3, %[r1], 3 \n" + + "xvpickev.w $xr4, $xr1, $xr0 \n" + "xvpickod.w $xr5, $xr1, $xr0 \n" + "xvpickev.w $xr6, $xr3, $xr2 \n" + "xvpickod.w $xr7, $xr3, $xr2 \n" + "xvaddwev.h.bu $xr0, $xr4, $xr5 \n" + "xvaddwod.h.bu $xr1, $xr4, $xr5 \n" + "xvaddwev.h.bu $xr2, $xr6, $xr7 \n" + "xvaddwod.h.bu $xr3, $xr6, $xr7 \n" + "xvadd.h $xr0, $xr0, $xr2 \n" + "xvadd.h $xr1, $xr1, $xr3 \n" + "xvsrari.h $xr0, $xr0, 2 \n" + "xvsrari.h $xr1, $xr1, 2 \n" + "xvpackev.b $xr0, $xr1, $xr0 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [s] "+r"(src_argb), // %[s] + [t] "+r"(next_argb), // %[t] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width), // %[w] + [r0] "=&r"(r0), // %[r0] + [r1] "=&r"(r1) // %[r1] + : [step] "r"(step) // %[step] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7"); +} +#endif // HAS_SCALEARGBROWDOWNEVEN_LASX + +#ifdef HAS_SCALEROWDOWN2_LASX +void ScaleRowDown2_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width) { + (void)src_stride; + asm volatile( + "srli.w %[w], %[w], 5 \n" + "beqz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "xvld $xr1, %[src], 32 \n" + "xvpickod.b $xr0, $xr1, $xr0 \n" + "xvpermi.d $xr0, $xr0, 0xd8 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[src], %[src], 64 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$xr0", "$xr1"); +} + +void ScaleRowDown2Linear_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width) { + (void)src_stride; + asm volatile( + "srli.w %[w], %[w], 5 \n" + "beqz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "xvld $xr1, %[src], 32 \n" + "xvpickev.b $xr2, $xr1, $xr0 \n" + "xvpickod.b $xr3, $xr1, $xr0 \n" + "xvavgr.bu $xr0, $xr3, $xr2 \n" + "xvpermi.d $xr0, $xr0, 0xd8 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[src], %[src], 64 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); +} + +void ScaleRowDown2Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width) { + const uint8_t* src_nex = src_ptr + src_stride; + asm volatile( + "srli.w %[w], %[w], 5 \n" + "beqz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "xvld $xr1, %[src], 32 \n" + "xvld $xr2, %[nex], 0 \n" + "xvld $xr3, %[nex], 32 \n" + "xvaddwev.h.bu $xr4, $xr0, $xr2 \n" + "xvaddwod.h.bu $xr5, $xr0, $xr2 \n" + "xvaddwev.h.bu $xr6, $xr1, $xr3 \n" + "xvaddwod.h.bu $xr7, $xr1, $xr3 \n" + "xvadd.h $xr4, $xr4, $xr5 \n" + "xvadd.h $xr6, $xr6, $xr7 \n" + "xvsrarni.b.h $xr6, $xr4, 2 \n" + "xvpermi.d $xr6, $xr6, 0xd8 \n" + "xvst $xr6, %[dst], 0 \n" + "addi.d %[src], %[src], 64 \n" + "addi.d %[nex], %[nex], 64 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [nex] "+r"(src_nex), // %[nex] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7"); +} +#endif // HAS_SCALEROWDOWN2_LASX + +#ifdef HAS_SCALEROWDOWN4_LASX +void ScaleRowDown4_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width) { + (void)src_stride; + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "xvld $xr1, %[src], 32 \n" + "xvpickev.b $xr0, $xr1, $xr0 \n" + "xvpermi.d $xr0, $xr0, 0xd8 \n" + "xvpickod.b $xr0, $xr0, $xr0 \n" + "xvstelm.d $xr0, %[dst], 0, 0 \n" + "xvstelm.d $xr0, %[dst], 8, 2 \n" + "addi.d %[src], %[src], 64 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$xr0", "$xr1"); +} + +void ScaleRowDown4Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width) { + const uint8_t* ptr1 = src_ptr + src_stride; + const uint8_t* ptr2 = ptr1 + src_stride; + const uint8_t* ptr3 = ptr2 + src_stride; + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[p0], 0 \n" + "xvld $xr1, %[p0], 32 \n" + "xvld $xr2, %[p1], 0 \n" + "xvld $xr3, %[p1], 32 \n" + "xvld $xr4, %[p2], 0 \n" + "xvld $xr5, %[p2], 32 \n" + "xvld $xr6, %[p3], 0 \n" + "xvld $xr7, %[p3], 32 \n" + "xvhaddw.hu.bu $xr0, $xr0, $xr0 \n" + "xvhaddw.hu.bu $xr1, $xr1, $xr1 \n" + "xvhaddw.hu.bu $xr2, $xr2, $xr2 \n" + "xvhaddw.hu.bu $xr3, $xr3, $xr3 \n" + "xvhaddw.hu.bu $xr4, $xr4, $xr4 \n" + "xvhaddw.hu.bu $xr5, $xr5, $xr5 \n" + "xvhaddw.hu.bu $xr6, $xr6, $xr6 \n" + "xvhaddw.hu.bu $xr7, $xr7, $xr7 \n" + "xvadd.h $xr0, $xr0, $xr2 \n" + "xvadd.h $xr1, $xr1, $xr3 \n" + "xvadd.h $xr4, $xr4, $xr6 \n" + "xvadd.h $xr5, $xr5, $xr7 \n" + "xvadd.h $xr0, $xr0, $xr4 \n" + "xvadd.h $xr1, $xr1, $xr5 \n" + "xvhaddw.wu.hu $xr0, $xr0, $xr0 \n" + "xvhaddw.wu.hu $xr1, $xr1, $xr1 \n" + "xvsrarni.h.w $xr1, $xr0, 4 \n" + "xvpermi.d $xr1, $xr1, 0xd8 \n" + "xvsrlni.b.h $xr1, $xr1, 0 \n" + "xvstelm.d $xr1, %[dst], 0, 0 \n" + "xvstelm.d $xr1, %[dst], 8, 2 \n" + "addi.d %[p0], %[p0], 64 \n" + "addi.d %[p1], %[p1], 64 \n" + "addi.d %[p2], %[p2], 64 \n" + "addi.d %[p3], %[p3], 64 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [p0] "+r"(src_ptr), // %[p0] + [p1] "+r"(ptr1), // %[p1] + [p2] "+r"(ptr2), // %[p2] + [p3] "+r"(ptr3), // %[p3] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7"); +} +#endif // HAS_SCALEROWDOWN4_LASX + +#ifdef HAS_SCALEROWDOWN38_LASX +void ScaleRowDown38_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width) { + int count = dst_width / 12; + (void)src_stride; + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr2, %[shuf], 0 \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "xvpermi.d $xr1, $xr0, 0x4e \n" + "vshuf.b $vr0, $vr1, $vr0, $vr2 \n" + "vstelm.d $vr0, %[dst], 0, 0 \n" + "vstelm.w $vr0, %[dst], 8, 2 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [shuf] "r"(kScale38Shuf) // %[shuf] + : "memory", "$xr0", "$xr1", "$xr2"); +} + +void ScaleRowDown38_2_Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const uint8_t* src_nex = src_ptr + src_stride; + int count = dst_width / 12; + int c2aaa = 0x2aaa; + int c4000 = 0x4000; + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr8, %[shuf], 0 \n" + "vreplgr2vr.h $vr9, %[c2aaa] \n" + "vreplgr2vr.w $vr10, %[c4000] \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "xvld $xr2, %[nex], 0 \n" + "xvaddwev.h.bu $xr4, $xr0, $xr2 \n" + "xvaddwod.h.bu $xr5, $xr0, $xr2 \n" + "xvpermi.d $xr6, $xr4, 0x4e \n" + "xvpermi.d $xr7, $xr5, 0x4e \n" + "vpickev.h $vr0, $vr6, $vr4 \n" + "vpickev.h $vr1, $vr7, $vr5 \n" + "vpackod.h $vr2, $vr5, $vr4 \n" + "vpackod.h $vr3, $vr7, $vr6 \n" + "vpickev.w $vr4, $vr3, $vr2 \n" + "vadd.h $vr0, $vr0, $vr1 \n" + "vadd.h $vr0, $vr0, $vr4 \n" + "vmuh.h $vr0, $vr0, $vr9 \n" + "vpickod.w $vr2, $vr3, $vr2 \n" + "vhaddw.wu.hu $vr2, $vr2, $vr2 \n" + "vmul.w $vr2, $vr2, $vr10 \n" + "vshuf.b $vr0, $vr2, $vr0, $vr8 \n" + "vstelm.d $vr0, %[dst], 0, 0 \n" + "vstelm.w $vr0, %[dst], 8, 2 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[nex], %[nex], 32 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [nex] "+r"(src_nex), // %[nex] + [dst] "+r"(dst_ptr), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [shuf] "r"(kScale38BoxShuf), // %[shuf] + [c2aaa] "r"(c2aaa), // %[c2aaa] + [c4000] "r"(c4000) // %[c4000] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10"); +} + +void ScaleRowDown38_3_Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const uint8_t* ptr1 = src_ptr + src_stride; + const uint8_t* ptr2 = ptr1 + src_stride; + int count = dst_width / 12; + int c1c71 = 0x1c71; + int c2aaa = 0x2aaa; + asm volatile( + "beqz %[cnt], 2f \n" + "xvldi $xr11, 0 \n" + "vld $vr8, %[shuf], 0 \n" + "vreplgr2vr.h $vr9, %[c1c71] \n" + "vreplgr2vr.w $vr10, %[c2aaa] \n" + "1: \n" + "xvld $xr0, %[p0], 0 \n" + "xvld $xr2, %[p1], 0 \n" + "xvaddwev.h.bu $xr4, $xr0, $xr2 \n" + "xvaddwod.h.bu $xr5, $xr0, $xr2 \n" + "xvld $xr0, %[p2], 0 \n" + "xvpackev.b $xr2, $xr11, $xr0 \n" + "xvpackod.b $xr3, $xr11, $xr0 \n" + "xvadd.h $xr4, $xr4, $xr2 \n" + "xvadd.h $xr5, $xr5, $xr3 \n" + "xvpermi.d $xr6, $xr4, 0x4e \n" + "xvpermi.d $xr7, $xr5, 0x4e \n" + "vpickev.h $vr0, $vr6, $vr4 \n" + "vpickev.h $vr1, $vr7, $vr5 \n" + "vpackod.h $vr2, $vr5, $vr4 \n" + "vpackod.h $vr3, $vr7, $vr6 \n" + "vpickev.w $vr4, $vr3, $vr2 \n" + "vadd.h $vr0, $vr0, $vr1 \n" + "vadd.h $vr0, $vr0, $vr4 \n" + "vmuh.h $vr0, $vr0, $vr9 \n" + "vpickod.w $vr2, $vr3, $vr2 \n" + "vhaddw.wu.hu $vr2, $vr2, $vr2 \n" + "vmul.w $vr2, $vr2, $vr10 \n" + "vshuf.b $vr0, $vr2, $vr0, $vr8 \n" + "vstelm.d $vr0, %[dst], 0, 0 \n" + "vstelm.w $vr0, %[dst], 8, 2 \n" + "addi.d %[p0], %[p0], 32 \n" + "addi.d %[p1], %[p1], 32 \n" + "addi.d %[p2], %[p2], 32 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [p0] "+r"(src_ptr), // %[p0] + [p1] "+r"(ptr1), // %[p1] + [p2] "+r"(ptr2), // %[p2] + [dst] "+r"(dst_ptr), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [shuf] "r"(kScale38BoxShuf), // %[shuf] + [c1c71] "r"(c1c71), // %[c1c71] + [c2aaa] "r"(c2aaa) // %[c2aaa] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11"); +} +#endif // HAS_SCALEROWDOWN38_LASX + +#ifdef HAS_SCALEADDROW_LASX +void ScaleAddRow_LASX(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int src_width) { + asm volatile( + "srli.w %[w], %[w], 5 \n" + "beqz %[w], 2f \n" + "xvldi $xr0, 0 \n" + "1: \n" + "xvld $xr1, %[src], 0 \n" + "xvld $xr2, %[dst], 0 \n" + "xvld $xr3, %[dst], 32 \n" + "xvpermi.d $xr1, $xr1, 0xd8 \n" + "xvilvl.b $xr4, $xr0, $xr1 \n" + "xvilvh.b $xr5, $xr0, $xr1 \n" + "xvadd.h $xr2, $xr2, $xr4 \n" + "xvadd.h $xr3, $xr3, $xr5 \n" + "xvst $xr2, %[dst], 0 \n" + "xvst $xr3, %[dst], 32 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[dst], %[dst], 64 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst_ptr), // %[dst] + [w] "+r"(src_width) // %[w] + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5"); +} +#endif // HAS_SCALEADDROW_LASX + +#ifdef HAS_SCALEUVADDCOLS3_LASX +void ScaleUVAddCols3_LASX(int dst_width, + int scaleval, + const uint16_t* src_uv, + uint8_t* dst_uv) { + asm volatile( + "vld $vr6, %[shuf], 0 \n" + "vld $vr7, %[shuf], 16 \n" + "vld $vr8, %[shuf], 32 \n" + "vld $vr9, %[shuf], 48 \n" + "vld $vr10, %[shuf], 64 \n" + "vld $vr11, %[shuf], 80 \n" + "xvpermi.d $xr6, $xr6, 0x44 \n" + "xvpermi.d $xr7, $xr7, 0x44 \n" + "xvpermi.d $xr8, $xr8, 0x44 \n" + "xvpermi.d $xr9, $xr9, 0x44 \n" + "xvpermi.d $xr10, $xr10, 0x44 \n" + "xvpermi.d $xr11, $xr11, 0x44 \n" + "xvreplgr2vr.h $xr12, %[scaleval] \n" + "1: \n" + "vld $vr0, %[src_uv], 0 \n" + "vld $vr1, %[src_uv], 16 \n" + "vld $vr2, %[src_uv], 32 \n" + "vld $vr3, %[src_uv], 48 \n" + "vld $vr4, %[src_uv], 64 \n" + "vld $vr5, %[src_uv], 80 \n" + "addi.d %[src_uv], %[src_uv], 96 \n" + "xvpermi.q $xr0, $xr3, 0x02 \n" + "xvpermi.q $xr1, $xr4, 0x02 \n" + "xvpermi.q $xr2, $xr5, 0x02 \n" + "xvshuf.b $xr3, $xr1, $xr0, $xr6 \n" + "xvshuf.b $xr4, $xr1, $xr0, $xr7 \n" + "xvshuf.b $xr5, $xr1, $xr0, $xr8 \n" + "xvshuf.b $xr3, $xr2, $xr3, $xr9 \n" + "xvshuf.b $xr4, $xr2, $xr4, $xr10 \n" + "xvshuf.b $xr5, $xr2, $xr5, $xr11 \n" + "xvadd.h $xr3, $xr3, $xr4 \n" + "xvadd.h $xr3, $xr3, $xr5 \n" + "xvmuh.hu $xr3, $xr3, $xr12 \n" + "xvpermi.d $xr4, $xr3, 0x4e \n" + "vpickev.b $vr0, $vr4, $vr3 \n" + "vst $vr0, %[dst_uv], 0 \n" + "addi.d %[dst_uv], %[dst_uv], 16 \n" + "addi.w %[dst_width], %[dst_width], -8 \n" + "bgtz %[dst_width], 1b \n" + : [src_uv] "+r"(src_uv), // %[src_uv] + [dst_uv] "+r"(dst_uv), // %[dst_uv] + [dst_width] "+r"(dst_width) // %[dst_width] + : [scaleval] "r"(scaleval), // %[scaleval] + [shuf] "r"(kScaleUVAddCols3Shuf) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12"); +} +#endif // HAS_SCALEUVADDCOLS3_LASX + +#if defined(HAS_SCALEFILTERCOLS_LASX) || defined(HAS_SCALEARGBCOLS_LASX) || \ + defined(HAS_SCALEARGBFILTERCOLS_LASX) +static const uint64_t kRamp01234567[4] = { + 0x0000000100000000ULL, 0x0000000300000002ULL, 0x0000000500000004ULL, + 0x0000000700000006ULL}; +#endif + +#ifdef HAS_SCALEFILTERCOLS_LASX +void ScaleFilterCols_LASX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx) { + ptrdiff_t i0, i1, i2, i3; + uint32_t p0, p1; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "xvld $xr0, %[ramp], 0 \n" + "xvreplgr2vr.w $xr1, %[dx] \n" + "xvreplgr2vr.w $xr2, %[x] \n" + "xvmul.w $xr0, $xr1, $xr0 \n" + "xvslli.w $xr1, $xr1, 3 \n" + "xvadd.w $xr2, $xr2, $xr0 \n" + "xvldi $xr3, 0x7f \n" + "xvsrli.w $xr3, $xr3, 24 \n" + "xvaddi.wu $xr4, $xr3, 1 \n" + "1: \n" + "xvsrai.w $xr5, $xr2, 16 \n" + "xvsrai.w $xr6, $xr2, 9 \n" + "xvand.v $xr6, $xr6, $xr3 \n" + "xvsub.w $xr7, $xr4, $xr6 \n" + "xvslli.w $xr6, $xr6, 8 \n" + "xvor.v $xr6, $xr6, $xr7 \n" + "xvadd.w $xr2, $xr2, $xr1 \n" + "xvpickve2gr.w %[i0], $xr5, 0 \n" + "xvpickve2gr.w %[i1], $xr5, 1 \n" + "xvpickve2gr.w %[i2], $xr5, 2 \n" + "xvpickve2gr.w %[i3], $xr5, 3 \n" + "ldx.h %[p0], %[src], %[i0] \n" + "ldx.h %[p1], %[src], %[i1] \n" + "xvinsgr2vr.w $xr0, %[p0], 0 \n" + "xvinsgr2vr.w $xr0, %[p1], 1 \n" + "ldx.h %[p0], %[src], %[i2] \n" + "ldx.h %[p1], %[src], %[i3] \n" + "xvinsgr2vr.w $xr0, %[p0], 2 \n" + "xvinsgr2vr.w $xr0, %[p1], 3 \n" + "xvpickve2gr.w %[i0], $xr5, 4 \n" + "xvpickve2gr.w %[i1], $xr5, 5 \n" + "xvpickve2gr.w %[i2], $xr5, 6 \n" + "xvpickve2gr.w %[i3], $xr5, 7 \n" + "ldx.h %[p0], %[src], %[i0] \n" + "ldx.h %[p1], %[src], %[i1] \n" + "xvinsgr2vr.w $xr0, %[p0], 4 \n" + "xvinsgr2vr.w $xr0, %[p1], 5 \n" + "ldx.h %[p0], %[src], %[i2] \n" + "ldx.h %[p1], %[src], %[i3] \n" + "xvinsgr2vr.w $xr0, %[p0], 6 \n" + "xvinsgr2vr.w $xr0, %[p1], 7 \n" + "xvmulwev.h.bu $xr7, $xr0, $xr6 \n" + "xvmaddwod.h.bu $xr7, $xr0, $xr6 \n" + "xvsrari.h $xr5, $xr7, 7 \n" + "xvpickev.b $xr5, $xr5, $xr5 \n" + "xvpickev.b $xr5, $xr5, $xr5 \n" + "xvstelm.w $xr5, %[dst], 0, 0 \n" + "xvstelm.w $xr5, %[dst], 4, 4 \n" + "addi.d %[dst], %[dst], 8 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [dst] "+r"(dst_ptr), // %[dst] + [w] "+r"(dst_width), // %[w] + [i0] "=&r"(i0), // %[i0] + [i1] "=&r"(i1), // %[i1] + [i2] "=&r"(i2), // %[i2] + [i3] "=&r"(i3), // %[i3] + [p0] "=&r"(p0), // %[p0] + [p1] "=&r"(p1) // %[p1] + : [src] "r"(src_ptr), // %[src] + [x] "r"(x), // %[x] + [dx] "r"(dx), // %[dx] + [ramp] "r"(kRamp01234567) // %[ramp] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7"); +} +#endif // HAS_SCALEFILTERCOLS_LASX + +#ifdef HAS_SCALEARGBCOLS_LASX +void ScaleARGBCols_LASX(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + ptrdiff_t i0, i1, i2, i3; + uint32_t p0, p1; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "xvld $xr0, %[ramp], 0 \n" + "xvreplgr2vr.w $xr1, %[dx] \n" + "xvreplgr2vr.w $xr2, %[x] \n" + "xvmul.w $xr0, $xr1, $xr0 \n" + "xvslli.w $xr1, $xr1, 3 \n" + "xvadd.w $xr2, $xr2, $xr0 \n" + "1: \n" + "xvsrai.w $xr3, $xr2, 16 \n" + "xvslli.w $xr3, $xr3, 2 \n" + "xvadd.w $xr2, $xr2, $xr1 \n" + "xvpickve2gr.w %[i0], $xr3, 0 \n" + "xvpickve2gr.w %[i1], $xr3, 1 \n" + "xvpickve2gr.w %[i2], $xr3, 2 \n" + "xvpickve2gr.w %[i3], $xr3, 3 \n" + "ldx.w %[p0], %[src], %[i0] \n" + "ldx.w %[p1], %[src], %[i1] \n" + "xvinsgr2vr.w $xr0, %[p0], 0 \n" + "xvinsgr2vr.w $xr0, %[p1], 1 \n" + "ldx.w %[p0], %[src], %[i2] \n" + "ldx.w %[p1], %[src], %[i3] \n" + "xvinsgr2vr.w $xr0, %[p0], 2 \n" + "xvinsgr2vr.w $xr0, %[p1], 3 \n" + "xvpickve2gr.w %[i0], $xr3, 4 \n" + "xvpickve2gr.w %[i1], $xr3, 5 \n" + "xvpickve2gr.w %[i2], $xr3, 6 \n" + "xvpickve2gr.w %[i3], $xr3, 7 \n" + "ldx.w %[p0], %[src], %[i0] \n" + "ldx.w %[p1], %[src], %[i1] \n" + "xvinsgr2vr.w $xr0, %[p0], 4 \n" + "xvinsgr2vr.w $xr0, %[p1], 5 \n" + "ldx.w %[p0], %[src], %[i2] \n" + "ldx.w %[p1], %[src], %[i3] \n" + "xvinsgr2vr.w $xr0, %[p0], 6 \n" + "xvinsgr2vr.w $xr0, %[p1], 7 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width), // %[w] + [i0] "=&r"(i0), // %[i0] + [i1] "=&r"(i1), // %[i1] + [i2] "=&r"(i2), // %[i2] + [i3] "=&r"(i3), // %[i3] + [p0] "=&r"(p0), // %[p0] + [p1] "=&r"(p1) // %[p1] + : [src] "r"(src_argb), // %[src] + [x] "r"(x), // %[x] + [dx] "r"(dx), // %[dx] + [ramp] "r"(kRamp01234567) // %[ramp] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); +} +#endif // HAS_SCALEARGBCOLS_LASX + +#ifdef HAS_SCALEARGBFILTERCOLS_LASX +void ScaleARGBFilterCols_LASX(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + ptrdiff_t i0, i1, i2, i3; + uint64_t p0, p1; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "xvld $xr0, %[ramp], 0 \n" + "xvreplgr2vr.w $xr1, %[dx] \n" + "xvreplgr2vr.w $xr2, %[x] \n" + "xvmul.w $xr0, $xr1, $xr0 \n" + "xvslli.w $xr1, $xr1, 3 \n" + "xvadd.w $xr2, $xr2, $xr0 \n" + "xvldi $xr3, 0x7f \n" + "1: \n" + "xvsrai.w $xr0, $xr2, 16 \n" + "xvslli.w $xr0, $xr0, 2 \n" + "xvsrai.w $xr5, $xr2, 9 \n" + "xvadd.w $xr2, $xr2, $xr1 \n" + "xvand.v $xr5, $xr5, $xr3 \n" + "xvshuf4i.b $xr5, $xr5, 0x00 \n" + "xvxor.v $xr6, $xr5, $xr3 \n" + "xvilvl.b $xr7, $xr5, $xr6 \n" + "xvilvh.b $xr8, $xr5, $xr6 \n" + "xvpickve2gr.w %[i0], $xr0, 0 \n" + "xvpickve2gr.w %[i1], $xr0, 1 \n" + "xvpickve2gr.w %[i2], $xr0, 2 \n" + "xvpickve2gr.w %[i3], $xr0, 3 \n" + "ldx.d %[p0], %[src], %[i0] \n" + "ldx.d %[p1], %[src], %[i1] \n" + "xvinsgr2vr.d $xr4, %[p0], 0 \n" + "xvinsgr2vr.d $xr4, %[p1], 1 \n" + "ldx.d %[p0], %[src], %[i2] \n" + "ldx.d %[p1], %[src], %[i3] \n" + "xvinsgr2vr.d $xr5, %[p0], 0 \n" + "xvinsgr2vr.d $xr5, %[p1], 1 \n" + "xvpickve2gr.w %[i0], $xr0, 4 \n" + "xvpickve2gr.w %[i1], $xr0, 5 \n" + "xvpickve2gr.w %[i2], $xr0, 6 \n" + "xvpickve2gr.w %[i3], $xr0, 7 \n" + "ldx.d %[p0], %[src], %[i0] \n" + "ldx.d %[p1], %[src], %[i1] \n" + "xvinsgr2vr.d $xr4, %[p0], 2 \n" + "xvinsgr2vr.d $xr4, %[p1], 3 \n" + "ldx.d %[p0], %[src], %[i2] \n" + "ldx.d %[p1], %[src], %[i3] \n" + "xvinsgr2vr.d $xr5, %[p0], 2 \n" + "xvinsgr2vr.d $xr5, %[p1], 3 \n" + "xvpickev.w $xr6, $xr5, $xr4 \n" + "xvpickod.w $xr4, $xr5, $xr4 \n" + "xvilvl.b $xr5, $xr4, $xr6 \n" + "xvilvh.b $xr4, $xr4, $xr6 \n" + "xvmulwev.h.bu $xr6, $xr5, $xr7 \n" + "xvmaddwod.h.bu $xr6, $xr5, $xr7 \n" + "xvmulwev.h.bu $xr7, $xr4, $xr8 \n" + "xvmaddwod.h.bu $xr7, $xr4, $xr8 \n" + "xvsrani.b.h $xr7, $xr6, 7 \n" + "xvst $xr7, %[dst], 0 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width), // %[w] + [i0] "=&r"(i0), // %[i0] + [i1] "=&r"(i1), // %[i1] + [i2] "=&r"(i2), // %[i2] + [i3] "=&r"(i3), // %[i3] + [p0] "=&r"(p0), // %[p0] + [p1] "=&r"(p1) // %[p1] + : [src] "r"(src_argb), // %[src] + [x] "r"(x), // %[x] + [dx] "r"(dx), // %[dx] + [ramp] "r"(kRamp01234567) // %[ramp] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8"); +} +#endif // HAS_SCALEARGBFILTERCOLS_LASX + +#ifdef HAS_SCALEROWDOWN34_LASX +void ScaleRowDown34_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst, + int dst_width) { + int count = dst_width / 24; + (void)src_stride; + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr1, %[shuf], 0 \n" + "xvpermi.d $xr1, $xr1, 0x44 \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "xvshuf.b $xr0, $xr0, $xr0, $xr1 \n" + "xvstelm.d $xr0, %[dst], 0, 0 \n" + "xvstelm.w $xr0, %[dst], 8, 2 \n" + "addi.d %[dst], %[dst], 12 \n" + "xvstelm.d $xr0, %[dst], 0, 2 \n" + "xvstelm.w $xr0, %[dst], 8, 6 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [shuf] "r"(kScale34Shuf) // %[shuf] + : "memory", "$xr0", "$xr1"); +} + +void ScaleRowDown34_0_Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* d, + int dst_width) { + const uint8_t* src_nex = src_ptr + src_stride; + int count = dst_width / 24; + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr0, %[lo], 0 \n" + "vld $vr1, %[hi], 0 \n" + "vld $vr2, %[wt], 0 \n" + "vld $vr3, %[pack], 0 \n" + "xvpermi.d $xr0, $xr0, 0x44 \n" + "xvpermi.d $xr1, $xr1, 0x44 \n" + "xvpermi.d $xr2, $xr2, 0x44 \n" + "xvpermi.d $xr3, $xr3, 0x44 \n" + "1: \n" + "xvld $xr4, %[src], 0 \n" + "xvld $xr5, %[nex], 0 \n" + "xvshuf.b $xr6, $xr4, $xr4, $xr0 \n" + "xvshuf.b $xr7, $xr4, $xr4, $xr1 \n" + "xvshuf.b $xr8, $xr5, $xr5, $xr0 \n" + "xvshuf.b $xr9, $xr5, $xr5, $xr1 \n" + "xvmulwev.h.bu $xr4, $xr6, $xr2 \n" + "xvmaddwod.h.bu $xr4, $xr6, $xr2 \n" + "xvmulwev.h.bu $xr5, $xr7, $xr2 \n" + "xvmaddwod.h.bu $xr5, $xr7, $xr2 \n" + "xvmulwev.h.bu $xr6, $xr8, $xr2 \n" + "xvmaddwod.h.bu $xr6, $xr8, $xr2 \n" + "xvmulwev.h.bu $xr7, $xr9, $xr2 \n" + "xvmaddwod.h.bu $xr7, $xr9, $xr2 \n" + "xvsrari.h $xr4, $xr4, 2 \n" + "xvsrari.h $xr5, $xr5, 2 \n" + "xvsrari.h $xr6, $xr6, 2 \n" + "xvsrari.h $xr7, $xr7, 2 \n" + "xvslli.h $xr8, $xr4, 1 \n" + "xvslli.h $xr9, $xr5, 1 \n" + "xvadd.h $xr4, $xr4, $xr8 \n" + "xvadd.h $xr5, $xr5, $xr9 \n" + "xvadd.h $xr4, $xr4, $xr6 \n" + "xvadd.h $xr5, $xr5, $xr7 \n" + "xvsrarni.b.h $xr5, $xr4, 2 \n" + "xvshuf.b $xr5, $xr5, $xr5, $xr3 \n" + "xvstelm.d $xr5, %[dst], 0, 0 \n" + "xvstelm.w $xr5, %[dst], 8, 2 \n" + "addi.d %[dst], %[dst], 12 \n" + "xvstelm.d $xr5, %[dst], 0, 2 \n" + "xvstelm.w $xr5, %[dst], 8, 6 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[nex], %[nex], 32 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [nex] "+r"(src_nex), // %[nex] + [dst] "+r"(d), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [lo] "r"(kScale34BoxLo), // %[lo] + [hi] "r"(kScale34BoxHi), // %[hi] + [wt] "r"(kScale34BoxWt), // %[wt] + [pack] "r"(kScale34Pack12) // %[pack] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9"); +} + +void ScaleRowDown34_1_Box_LASX(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* d, + int dst_width) { + const uint8_t* src_nex = src_ptr + src_stride; + int count = dst_width / 24; + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr0, %[lo], 0 \n" + "vld $vr1, %[hi], 0 \n" + "vld $vr2, %[wt], 0 \n" + "vld $vr3, %[pack], 0 \n" + "xvpermi.d $xr0, $xr0, 0x44 \n" + "xvpermi.d $xr1, $xr1, 0x44 \n" + "xvpermi.d $xr2, $xr2, 0x44 \n" + "xvpermi.d $xr3, $xr3, 0x44 \n" + "1: \n" + "xvld $xr4, %[src], 0 \n" + "xvld $xr5, %[nex], 0 \n" + "xvshuf.b $xr6, $xr4, $xr4, $xr0 \n" + "xvshuf.b $xr7, $xr4, $xr4, $xr1 \n" + "xvshuf.b $xr8, $xr5, $xr5, $xr0 \n" + "xvshuf.b $xr9, $xr5, $xr5, $xr1 \n" + "xvmulwev.h.bu $xr4, $xr6, $xr2 \n" + "xvmaddwod.h.bu $xr4, $xr6, $xr2 \n" + "xvmulwev.h.bu $xr5, $xr7, $xr2 \n" + "xvmaddwod.h.bu $xr5, $xr7, $xr2 \n" + "xvmulwev.h.bu $xr6, $xr8, $xr2 \n" + "xvmaddwod.h.bu $xr6, $xr8, $xr2 \n" + "xvmulwev.h.bu $xr7, $xr9, $xr2 \n" + "xvmaddwod.h.bu $xr7, $xr9, $xr2 \n" + "xvsrari.h $xr4, $xr4, 2 \n" + "xvsrari.h $xr5, $xr5, 2 \n" + "xvsrari.h $xr6, $xr6, 2 \n" + "xvsrari.h $xr7, $xr7, 2 \n" + "xvadd.h $xr4, $xr4, $xr6 \n" + "xvadd.h $xr5, $xr5, $xr7 \n" + "xvsrarni.b.h $xr5, $xr4, 1 \n" + "xvshuf.b $xr5, $xr5, $xr5, $xr3 \n" + "xvstelm.d $xr5, %[dst], 0, 0 \n" + "xvstelm.w $xr5, %[dst], 8, 2 \n" + "addi.d %[dst], %[dst], 12 \n" + "xvstelm.d $xr5, %[dst], 0, 2 \n" + "xvstelm.w $xr5, %[dst], 8, 6 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[nex], %[nex], 32 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [nex] "+r"(src_nex), // %[nex] + [dst] "+r"(d), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [lo] "r"(kScale34BoxLo), // %[lo] + [hi] "r"(kScale34BoxHi), // %[hi] + [wt] "r"(kScale34BoxWt), // %[wt] + [pack] "r"(kScale34Pack12) // %[pack] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9"); +} +#endif // HAS_SCALEROWDOWN34_LASX + +#ifdef HAS_SCALEMAXSAMPLES_LASX +float ScaleMaxSamples_LASX(const float* src, + float* dst, + float scale, + int width) { + float fmax = 0.f; + asm volatile( + "xvldi $xr3, 0 \n" + "xvldrepl.w $xr2, %[scale_ptr], 0 \n" + "addi.w %[w], %[w], -8 \n" + "bltz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "addi.d %[src], %[src], 32 \n" + "xvfmax.s $xr3, $xr3, $xr0 \n" + "xvfmul.s $xr0, $xr0, $xr2 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -8 \n" + "bgez %[w], 1b \n" + "xvpermi.d $xr1, $xr3, 0x4e \n" + "vfmax.s $vr3, $vr3, $vr1 \n" + "vshuf4i.w $vr1, $vr3, 0x4e \n" + "vfmax.s $vr3, $vr3, $vr1 \n" + "vshuf4i.w $vr1, $vr3, 0xb1 \n" + "vfmax.s $vr3, $vr3, $vr1 \n" + "vstelm.w $vr3, %[fmax_ptr], 0, 0 \n" + "2: \n" + "addi.w %[w], %[w], 8 \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width) // %[w] + : [scale_ptr] "r"(&scale), + [fmax_ptr] "r"(&fmax) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); + for (; width > 0; --width) { + float v = *src++; + float vs = v * scale; + fmax = (v > fmax) ? v : fmax; + *dst++ = vs; + } + return fmax; +} +#endif // HAS_SCALEMAXSAMPLES_LASX + +#ifdef HAS_SCALESUMSAMPLES_LASX +float ScaleSumSamples_LASX(const float* src, + float* dst, + float scale, + int width) { + float fsum = 0.f; + asm volatile( + "xvldi $xr3, 0 \n" + "xvldrepl.w $xr2, %[scale_ptr], 0 \n" + "addi.w %[w], %[w], -8 \n" + "bltz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "addi.d %[src], %[src], 32 \n" + "xvfmadd.s $xr3, $xr0, $xr0, $xr3 \n" + "xvfmul.s $xr0, $xr0, $xr2 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -8 \n" + "bgez %[w], 1b \n" + "xvpermi.d $xr1, $xr3, 0x4e \n" + "vfadd.s $vr3, $vr3, $vr1 \n" + "vshuf4i.w $vr1, $vr3, 0x4e \n" + "vfadd.s $vr3, $vr3, $vr1 \n" + "vshuf4i.w $vr1, $vr3, 0xb1 \n" + "vfadd.s $vr3, $vr3, $vr1 \n" + "vstelm.w $vr3, %[fsum_ptr], 0, 0 \n" + "2: \n" + "addi.w %[w], %[w], 8 \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width) // %[w] + : [scale_ptr] "r"(&scale), + [fsum_ptr] "r"(&fsum) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); + for (; width > 0; --width) { + float v = *src++; + fsum += v * v; + *dst++ = v * scale; + } + return fsum; +} +#endif // HAS_SCALESUMSAMPLES_LASX + +#ifdef HAS_SCALESAMPLES_LASX +void ScaleSamples_LASX(const float* src, float* dst, float scale, int width) { + asm volatile( + "xvldrepl.w $xr2, %[scale_ptr], 0 \n" + "addi.w %[w], %[w], -8 \n" + "bltz %[w], 2f \n" + "1: \n" + "xvld $xr0, %[src], 0 \n" + "addi.d %[src], %[src], 32 \n" + "xvfmul.s $xr0, $xr0, $xr2 \n" + "xvst $xr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -8 \n" + "bgez %[w], 1b \n" + "2: \n" + "addi.w %[w], %[w], 8 \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width) // %[w] + : [scale_ptr] "r"(&scale) + : "memory", "$xr0", "$xr2"); + if (width > 0) { + ScaleSamples_C(src, dst, scale, width); + } +} +#endif // HAS_SCALESAMPLES_LASX + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale_neon64.cc b/source/scale_neon64.cc index 2bb04b6e9..0f2a2013a 100644 --- a/source/scale_neon64.cc +++ b/source/scale_neon64.cc @@ -1583,6 +1583,84 @@ void ScaleUVAddCols3_NEON(int dst_width, : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6"); } +float ScaleMaxSamples_NEON(const float* src, + float* dst, + float scale, + int width) { + float fmax; + asm volatile( + "movi v5.4s, #0 \n" // max + "movi v6.4s, #0 \n" + + "1: \n" + "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples + "subs %w2, %w2, #8 \n" // 8 processed per loop + "fmul v3.4s, v1.4s, %4.s[0] \n" // scale + "prfm pldl1keep, [%0, 448] \n" + "fmul v4.4s, v2.4s, %4.s[0] \n" // scale + "fmax v5.4s, v5.4s, v1.4s \n" // max + "fmax v6.4s, v6.4s, v2.4s \n" + "st1 {v3.4s, v4.4s}, [%1], #32 \n" // store 8 samples + "b.gt 1b \n" + "fmax v5.4s, v5.4s, v6.4s \n" // max + "fmaxv %s3, v5.4s \n" // signed max acculator + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + "=w"(fmax) // %3 + : "w"(scale) // %4 + : "memory", "cc", "v1", "v2", "v3", "v4", "v5", "v6"); + return fmax; +} + +float ScaleSumSamples_NEON(const float* src, + float* dst, + float scale, + int width) { + float fsum; + asm volatile( + "movi v5.4s, #0 \n" // max + "movi v6.4s, #0 \n" // max + + "1: \n" + "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples + "subs %w2, %w2, #8 \n" // 8 processed per loop + "fmul v3.4s, v1.4s, %4.s[0] \n" // scale + "prfm pldl1keep, [%0, 448] \n" + "fmul v4.4s, v2.4s, %4.s[0] \n" + "fmla v5.4s, v1.4s, v1.4s \n" // sum of squares + "fmla v6.4s, v2.4s, v2.4s \n" + "st1 {v3.4s, v4.4s}, [%1], #32 \n" // store 8 samples + "b.gt 1b \n" + "faddp v5.4s, v5.4s, v6.4s \n" + "faddp v5.4s, v5.4s, v5.4s \n" + "faddp %3.4s, v5.4s, v5.4s \n" // sum + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + "=w"(fsum) // %3 + : "w"(scale) // %4 + : "memory", "cc", "v1", "v2", "v3", "v4", "v5", "v6"); + return fsum; +} + +void ScaleSamples_NEON(const float* src, float* dst, float scale, int width) { + asm volatile( + "1: \n" + "ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples + "subs %w2, %w2, #8 \n" // 8 processed per loop + "prfm pldl1keep, [%0, 448] \n" + "fmul v1.4s, v1.4s, %3.s[0] \n" // scale + "fmul v2.4s, v2.4s, %3.s[0] \n" // scale + "st1 {v1.4s, v2.4s}, [%1], #32 \n" // store 8 samples + "b.gt 1b \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "w"(scale) // %3 + : "memory", "cc", "v1", "v2"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 6a3f2d181..81e64c042 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -453,6 +453,14 @@ static int ScaleUVBox(int src_width, } } #endif +#if defined(HAS_SCALEUVADDCOLS3_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ScaleUVAddCols3 = ScaleUVAddCols3_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVAddCols3 = ScaleUVAddCols3_LASX; + } + } +#endif #if defined(HAS_SCALEUVADDCOLS3_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { ScaleUVAddCols3 = ScaleUVAddCols3_Any_WASMSIMD; @@ -504,6 +512,14 @@ static int ScaleUVBox(int src_width, } } #endif +#if defined(HAS_SCALEADDROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ScaleAddRow = ScaleAddRow_Any_LASX; + if (IS_ALIGNED(row_samples, 32)) { + ScaleAddRow = ScaleAddRow_LASX; + } + } +#endif #if defined(HAS_SCALEADDROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ScaleAddRow = ScaleAddRow_RVV; @@ -975,6 +991,22 @@ static int ScaleUVBilinearDown_16(int src_width, InterpolateRow = InterpolateRow_16_WASMSIMD; } } +#endif +#if defined(HAS_INTERPOLATEROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + InterpolateRow = InterpolateRow_16_Any_LSX; + if (IS_ALIGNED(clip_src_width, 8)) { + InterpolateRow = InterpolateRow_16_LSX; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_16_Any_LASX; + if (IS_ALIGNED(clip_src_width, 16)) { + InterpolateRow = InterpolateRow_16_LASX; + } + } #endif // Allocate a row of 16-bit UV. { @@ -1242,6 +1274,22 @@ static int ScaleUVBilinearUp_16(int src_width, InterpolateRow = InterpolateRow_16_WASMSIMD; } } +#endif +#if defined(HAS_INTERPOLATEROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + InterpolateRow = InterpolateRow_16_Any_LSX; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_16_LSX; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_16_Any_LASX; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_16_LASX; + } + } #endif if (src_width >= 32768) { ScaleUVFilterCols = diff --git a/source/scale_win.cc b/source/scale_win.cc index 93921c794..4130df9b6 100644 --- a/source/scale_win.cc +++ b/source/scale_win.cc @@ -2609,6 +2609,90 @@ void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr, } #endif // HAS_SCALEUVROWUP2_BILINEAR_16_AVX2 +#ifdef HAS_SCALEMAXSAMPLES_AVX2 +LIBYUV_TARGET_AVX2 +float ScaleMaxSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + __m256 vmax = _mm256_setzero_ps(); + const __m256 vscale = _mm256_set1_ps(scale); + while (width >= 8) { + __m256 v = _mm256_loadu_ps(src); + src += 8; + vmax = _mm256_max_ps(v, vmax); + _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); + dst += 8; + width -= 8; + } + __m128 m128 = + _mm_max_ps(_mm256_extractf128_ps(vmax, 1), _mm256_castps256_ps128(vmax)); + m128 = _mm_max_ps(_mm_movehl_ps(m128, m128), m128); + m128 = _mm_max_ss(_mm_shuffle_ps(m128, m128, 1), m128); + float fmax = _mm_cvtss_f32(m128); + _mm256_zeroupper(); + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. + for (; width > 0; --width) { + float v = *src++; + float vs = v * scale; + fmax = (v > fmax) ? v : fmax; + *dst++ = vs; + } + return fmax; +} +#endif // HAS_SCALEMAXSAMPLES_AVX2 + +#ifdef HAS_SCALESUMSAMPLES_AVX2 +LIBYUV_TARGET_AVX2 +float ScaleSumSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + __m256 vsum = _mm256_setzero_ps(); + const __m256 vscale = _mm256_set1_ps(scale); + while (width >= 8) { + __m256 v = _mm256_loadu_ps(src); + src += 8; + vsum = _mm256_add_ps(vsum, _mm256_mul_ps(v, v)); + _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); + dst += 8; + width -= 8; + } + __m128 s128 = + _mm_add_ps(_mm256_castps256_ps128(vsum), _mm256_extractf128_ps(vsum, 1)); + s128 = _mm_add_ps(s128, _mm_movehl_ps(s128, s128)); + s128 = _mm_add_ss(s128, _mm_shuffle_ps(s128, s128, 1)); + float fsum = _mm_cvtss_f32(s128); + _mm256_zeroupper(); + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. + for (; width > 0; --width) { + float v = *src++; + fsum += v * v; + *dst++ = v * scale; + } + return fsum; +} +#endif // HAS_SCALESUMSAMPLES_AVX2 + +#ifdef HAS_SCALESAMPLES_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) { + const __m256 vscale = _mm256_set1_ps(scale); + while (width >= 8) { + __m256 v = _mm256_loadu_ps(src); + src += 8; + _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + if (width > 0) { + // TODO(fbarchard): Handle remainder with masking or SIMD Any function. + ScaleSamples_C(src, dst, scale, width); + } +} +#endif // HAS_SCALESAMPLES_AVX2 + #undef LIBYUV_TARGET_AVX2 #undef LIBYUV_TARGET_AVX512BW #undef LIBYUV_TARGET_AVX512VBMI diff --git a/unit_test/cpu_test.cc b/unit_test/cpu_test.cc index e74af5d79..ed05301db 100644 --- a/unit_test/cpu_test.cc +++ b/unit_test/cpu_test.cc @@ -310,6 +310,15 @@ TEST_F(LibYUVBaseTest, TestCompilerMacros) { #ifdef __loongarch__ printf("__loongarch__ %d\n", __loongarch__); #endif +#ifdef __loongarch_lp64 + printf("__loongarch_lp64 %d\n", __loongarch_lp64); +#endif +#ifdef __loongarch_sx + printf("__loongarch_sx %d\n", __loongarch_sx); +#endif +#ifdef __loongarch_asx + printf("__loongarch_asx %d\n", __loongarch_asx); +#endif #ifdef __wasm__ printf("__wasm__ %d\n", __wasm__); #endif @@ -459,6 +468,10 @@ TEST_F(LibYUVBaseTest, MAYBE_TestSetCpuFlags) { SetCpuFlags(cpu_flags); ASSERT_EQ(cpu_flags, TestCpuFlag(-1)); + cpu_flags = kCpuHasLOONGARCH | kCpuHasLSX | kCpuHasLASX | kCpuInitialized; + SetCpuFlags(cpu_flags); + ASSERT_EQ(cpu_flags, TestCpuFlag(-1)); + // Test that setting 0 turns auto-init back on. SetCpuFlags(0); ASSERT_EQ(original_cpu_flags, TestCpuFlag(-1)); diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 6439a9fce..68b265a74 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -32,7 +32,8 @@ #ifdef ENABLE_ROW_TESTS // row.h defines SIMD_ALIGNED, overriding unit_test.h // TODO(fbarchard): Remove row.h from unittests. Test public functions. -#include "libyuv/row.h" /* For ScaleSumSamples_Neon */ +#include "libyuv/row.h" +#include "libyuv/scale_row.h" /* For ScaleSumSamples_Neon */ #endif #if defined(__has_feature) @@ -4869,6 +4870,26 @@ float TestScaleMaxSamples(int benchmark_width, reinterpret_cast(dst_opt), scale, kPixels); } +#elif defined(HAS_SCALEMAXSAMPLES_LASX) || defined(HAS_SCALEMAXSAMPLES_LSX) +#ifdef HAS_SCALEMAXSAMPLES_LASX + if (TestCpuFlag(kCpuHasLASX)) { + max_opt = ScaleMaxSamples_LASX(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else +#endif +#ifdef HAS_SCALEMAXSAMPLES_LSX + if (TestCpuFlag(kCpuHasLSX)) { + max_opt = ScaleMaxSamples_LSX(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else +#endif + { + max_opt = ScaleMaxSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #else max_opt = ScaleMaxSamples_C(reinterpret_cast(orig_y), @@ -4956,6 +4977,26 @@ float TestScaleSumSamples(int benchmark_width, reinterpret_cast(dst_opt), scale, kPixels); } +#elif defined(HAS_SCALESUMSAMPLES_LASX) || defined(HAS_SCALESUMSAMPLES_LSX) +#ifdef HAS_SCALESUMSAMPLES_LASX + if (TestCpuFlag(kCpuHasLASX)) { + sum_opt = ScaleSumSamples_LASX(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else +#endif +#ifdef HAS_SCALESUMSAMPLES_LSX + if (TestCpuFlag(kCpuHasLSX)) { + sum_opt = ScaleSumSamples_LSX(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else +#endif + { + sum_opt = ScaleSumSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #else sum_opt = ScaleSumSamples_C(reinterpret_cast(orig_y), @@ -5049,6 +5090,23 @@ float TestScaleSamples(int benchmark_width, ScaleSamples_C(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); } +#elif defined(HAS_SCALESAMPLES_LASX) || defined(HAS_SCALESAMPLES_LSX) +#ifdef HAS_SCALESAMPLES_LASX + if (TestCpuFlag(kCpuHasLASX)) { + ScaleSamples_LASX(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } else +#endif +#ifdef HAS_SCALESAMPLES_LSX + if (TestCpuFlag(kCpuHasLSX)) { + ScaleSamples_LSX(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } else +#endif + { + ScaleSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } #else ScaleSamples_C(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); @@ -5450,6 +5508,24 @@ TEST_F(LibYUVPlanarTest, HalfRow_Opt) { HalfRow_Any_NEON(src_pixels, stride, dst_pixels_opt, width); } } else +#endif +#if defined(HAS_HALFROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + if (IS_ALIGNED(width, 32)) { + HalfRow_LASX(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_Any_LASX(src_pixels, stride, dst_pixels_opt, width); + } + } else +#endif +#if defined(HAS_HALFROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + if (IS_ALIGNED(width, 16)) { + HalfRow_LSX(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_Any_LSX(src_pixels, stride, dst_pixels_opt, width); + } + } else #endif { HalfRow_C(src_pixels, stride, dst_pixels_opt, width); @@ -5505,6 +5581,24 @@ TEST_F(LibYUVPlanarTest, HalfRow_16_Opt) { HalfRow_16_Any_NEON(src_pixels, stride, dst_pixels_opt, width); } } else +#endif +#if defined(HAS_HALFROW_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + if (IS_ALIGNED(width, 16)) { + HalfRow_16_LASX(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_16_Any_LASX(src_pixels, stride, dst_pixels_opt, width); + } + } else +#endif +#if defined(HAS_HALFROW_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + if (IS_ALIGNED(width, 8)) { + HalfRow_16_LSX(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_16_Any_LSX(src_pixels, stride, dst_pixels_opt, width); + } + } else #endif { HalfRow_16_C(src_pixels, stride, dst_pixels_opt, width); diff --git a/unit_test/unit_test.cc b/unit_test/unit_test.cc index 00b928d16..f3f3a79b1 100644 --- a/unit_test/unit_test.cc +++ b/unit_test/unit_test.cc @@ -120,7 +120,7 @@ static int TestCpuEnv(int cpu_info) { cpu_info &= ~libyuv::kCpuHasSME2; } #endif -#if defined(__longarch__) && defined(__linux__) +#if defined(__loongarch__) if (TestEnv("LIBYUV_DISABLE_LSX")) { cpu_info &= ~libyuv::kCpuHasLSX; }