From c6700f48ddf3fdda4afb3cc7163f36427733aba5 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Tue, 3 Nov 2026 10:44:29 -0800 Subject: [PATCH] [libyuv] Consolidate LoongArch LSX and LASX inline asm into *_la.cc Merge 64-bit LoongArch LSX and LASX inline assembly implementations from rotate_lsx.cc, rotate_lasx.cc, row_lsx.cc, row_lasx.cc, and scale_lsx.cc into compare_la.cc, rotate_la.cc, row_la.cc, and scale_la.cc, and remove the obsolete loongson_intrinsics.h header. Because all LSX and LASX kernels use inline assembly rather than compiler intrinsics, separate libyuv_lsx/libyuv_lasx build targets and -mlsx/-mlasx compiler flags are no longer required. Guard HAS_*_LSX and HAS_*_LASX macros with defined(__loongarch_lp64) so runtime CPU dispatch enables LASX and LSX without requiring global -mlasx/-mlsx flags, and wrap every function and helper in *_la.cc with its corresponding #ifdef HAS_* guard. Also fix bugs found during LoongArch verification: - Replace illegal vldi/xvldi -1 immediate encodings (si13 = 0x1fff, which raises SIGILL at runtime) in ARGBDetectRow_LSX and ARGBDetectRow_LASX with vseqi.b/xvseqi.b -1. - Set simd_size to 16 for LSX and 32 for LASX in ComputeHammingDistance and ComputeSumSquareError. - Remove orphaned dead BGRAToUVRow, ABGRToUVRow, RGBAToUVRow, and XRGBToUVHelper functions in row_la.cc. - Remove dead alpha shifts in ARGB1555ToYRow/UVRow_LSX/LASX and hoist channel masks out of the loop in RGB565ToUVRow/ARGB1555ToUVRow. Compare SIMD ISA per *_C: ------------------------------------------------------------------------ C Function LSX LASX ------------------------------------------------------------------------ ARGBDetectRow_C LSX LASX HammingDistance_C LSX LASX HashDjb2_C LSX LASX Ssim4x4Row_C LSX LASX SsimWindowRow_C LSX LASX SumSquareError_C LSX LASX ------------------------------------------------------------------------ Compare: 6 (100%) 6 (100%) 6 (100%) ------------------------------------------------------------------------ Convert SIMD ISA per *_C: ------------------------------------------------------------------------ C Function LSX LASX ------------------------------------------------------------------------ AB64ToARGBRow_C - - ABGRToAR30Row_C - - AR30ToAB30Row_C - - AR30ToABGRRow_C - - AR30ToARGBRow_C - - AR64ShuffleRow_C LSX LASX AR64ToAB64Row_C LSX LASX AR64ToARGBRow_C - - ARGB1555ToARGBRow_C LSX LASX ARGB1555ToUVMatrixRow_C - - ARGB1555ToUVRow_C LSX LASX ARGB1555ToYMatrixRow_C - - ARGB1555ToYRow_C LSX LASX ARGB4444ToARGBRow_C LSX LASX ARGB4444ToUVMatrixRow_C - - ARGB4444ToUVRow_C - - ARGB4444ToYMatrixRow_C - - ARGB4444ToYRow_C - - ARGBAddRow_C LSX LASX ARGBAffineRow_C - - ARGBAlphaBlendRow_C LSX LASX ARGBAttenuateRow_C LSX LASX ARGBBlendRow_C LSX LASX ARGBColorMatrixRow_C LSX LASX ARGBColorTableRow_C - - ARGBCopyAlphaRow_C - - ARGBCopyYToAlphaRow_C - - ARGBExtractAlphaRow_C LSX LASX ARGBGrayRow_C LSX LASX ARGBLumaColorTableRow_C - - ARGBMirrorRow_C LSX LASX ARGBMultiplyRow_C LSX LASX ARGBOrRow_C LSX LASX ARGBPolynomialRow_C - - ARGBQuantizeRow_C LSX LASX ARGBSepiaRow_C LSX LASX ARGBSetRow_C LSX LASX ARGBShadeRow_C LSX LASX ARGBShuffleRow_C LSX LASX ARGBSubtractRow_C LSX LASX ARGBToAB64Row_C - - ARGBToABGRRow_C LSX LASX ARGBToAR30Row_C - - ARGBToAR64Row_C - - ARGBToARGB1555Row_C LSX LASX ARGBToARGB4444Row_C LSX LASX ARGBToBGRARow_C LSX LASX ARGBToRAWRow_C LSX LASX ARGBToRGB24Row_C LSX LASX ARGBToRGB565DitherRow_C LSX LASX ARGBToRGB565Row_C LSX LASX ARGBToRGBARow_C LSX LASX ARGBToUV444MatrixRow_C - - ARGBToUVMatrixRow_C - - ARGBToYMatrixRow_C LSX LASX ARGBUnattenuateRow_C - - AYUVToUVRow_C - - AYUVToVURow_C - - AYUVToYRow_C - - BlendPlaneRow_C - - ByteToFloatRow_C - - ComputeCumulativeSumRow_C - - Convert16To8Row_C - - Convert8To16Row_C - - Convert8To8Row_C - - CopyRow_16_C - - CopyRow_C LSX LASX CumulativeSumToAverageRow_C - - DetileRow_16_C - - DetileRow_C - - DetileSplitUVRow_C - - DetileToYUY2_C - - DivideRow_16_C - - GaussCol_C - - GaussCol_F32_C - - GaussRow_C - - GaussRow_F32_C - - HalfFloatRow_C LSX LASX HalfMergeUVRow_C - - HalfRow_16To8_C - - HalfRow_16_C - - HalfRow_C - - HalfWidthRow_16To8_C - - HalfWidthRow_16To8_Odd_C - - I210AlphaToARGBRow_C - - I210ToAR30Row_C - - I210ToARGBRow_C - - I212ToAR30Row_C - - I212ToARGBRow_C - - I400ToARGBRow_C LSX LASX I410AlphaToARGBRow_C - - I410ToAR30Row_C - - I410ToARGBRow_C - - I422AlphaToARGBRow_C LSX LASX I422ToAR30Row_C - - I422ToARGB1555Row_C LSX LASX I422ToARGB4444Row_C LSX LASX I422ToARGBRow_C LSX LASX I422ToRGB24Row_C LSX LASX I422ToRGB565Row_C LSX LASX I422ToRGBARow_C LSX LASX I422ToUYVYRow_C LSX LASX I422ToYUY2Row_C LSX LASX I444AlphaToARGBRow_C - - I444ToARGBRow_C LSX LASX I444ToRGB24Row_C - - InterpolateRow_16_C - - InterpolateRow_C LSX LASX J400ToARGBRow_C LSX LASX MergeAR64Row_C - - MergeARGB16To8Row_C - - MergeARGBRow_C - - MergeRGBRow_C - - MergeUVRow_16_C - - MergeUVRow_C LSX LASX MergeXR30Row_C - - MergeXR64Row_C - - MergeXRGB16To8Row_C - - MergeXRGBRow_C - - MirrorRow_16_C LSX LASX MirrorRow_C LSX LASX MirrorSplitUVRow_C LSX LASX MirrorUVRow_C LSX LASX MultiplyRow_16_C - - NV12ToARGBRow_C LSX LASX NV12ToRGB24Row_C - - NV12ToRGB565Row_C LSX LASX NV21ToARGBRow_C LSX LASX NV21ToRGB24Row_C - - NV21ToYUV24Row_C - - P210ToAR30Row_C - - P210ToARGBRow_C - - P410ToAR30Row_C - - P410ToARGBRow_C - - RAWToARGBRow_C LSX LASX RAWToRGB24Row_C LSX LASX RAWToRGBARow_C - - RGB24MirrorRow_C - - RGB24ToARGBRow_C LSX LASX RGB565ToARGBRow_C LSX LASX RGB565ToUVMatrixRow_C - - RGB565ToUVRow_C LSX LASX RGB565ToYMatrixRow_C - - RGB565ToYRow_C LSX LASX RGBAToARGBRow_C LSX LASX RGBColorTableRow_C - - RGBToUV444MatrixRow_C - - RGBToUVMatrixRow_C - - RGBToYMatrixRow_C LSX LASX ScaleMaxSamples_C - - ScaleSamples_C - - ScaleSumSamples_C - - SetRow_C LSX LASX SobelRow_C LSX LASX SobelToPlaneRow_C LSX LASX SobelXRow_C - - SobelXYRow_C LSX LASX SobelYRow_C - - SplitARGBRow_C - - SplitRGBRow_C - - SplitUVRow_16_C - - SplitUVRow_C LSX LASX SplitXRGBRow_C - - SwapUVRow_C - - UYVYToARGBRow_C LSX LASX UYVYToUV422Row_C LSX LASX UYVYToUVRow_C LSX LASX UYVYToYRow_C LSX LASX UnpackMT2T_C - - YUY2ToARGBRow_C LSX LASX YUY2ToNVUVRow_C - - YUY2ToUV422Row_C LSX LASX YUY2ToUVRow_C LSX LASX YUY2ToYRow_C LSX LASX ------------------------------------------------------------------------ Convert: 174 (100%) 76 (44%) 76 (44%) ------------------------------------------------------------------------ Rotate SIMD ISA per *_C: ------------------------------------------------------------------------ C Function LSX LASX ------------------------------------------------------------------------ TransposeUVWx16_C - LASX TransposeUVWx8_C LSX - TransposeWx16_C - LASX TransposeWx4_32_C LSX LASX TransposeWx8_16_C LSX LASX TransposeWx8_C LSX - TransposeWxH_32_C - - ------------------------------------------------------------------------ Rotate: 7 (100%) 4 (57%) 4 (57%) ------------------------------------------------------------------------ Scale SIMD ISA per *_C: ------------------------------------------------------------------------ C Function LSX LASX ------------------------------------------------------------------------ FixedDiv1_C - - FixedDiv_C - - ScaleARGBCols64_C - - ScaleARGBColsUp2_C - - ScaleARGBCols_C LSX - ScaleARGBFilterCols64_C - - ScaleARGBFilterCols_C LSX - ScaleARGBRowDown2Box_C LSX - ScaleARGBRowDown2Linear_C LSX - ScaleARGBRowDown2_C LSX - ScaleARGBRowDownEvenBox_C LSX - ScaleARGBRowDownEven_C LSX - ScaleAddCols0_C - - ScaleAddCols1_16_C - - ScaleAddCols1_C - - ScaleAddCols2_16_C - - ScaleAddCols2_C - - ScaleAddRow_16_C - - ScaleAddRow_C LSX - ScaleColsUp2_16_C - - ScaleColsUp2_C - - ScaleCols_16_C - - ScaleCols_C - - ScaleFilterCols64_16_C - - ScaleFilterCols64_C - - ScaleFilterCols_16_C - - ScaleFilterCols_C LSX - ScaleRowDown2Box_16_C - - ScaleRowDown2Box_C LSX - ScaleRowDown2Box_Odd_C - - ScaleRowDown2Linear_16_C - - ScaleRowDown2Linear_C LSX - ScaleRowDown2_16_C - - ScaleRowDown2_C LSX - ScaleRowDown34_0_Box_16_C - - ScaleRowDown34_0_Box_C LSX - ScaleRowDown34_16_C - - ScaleRowDown34_1_Box_16_C - - ScaleRowDown34_1_Box_C LSX - ScaleRowDown34_C LSX - ScaleRowDown38_16_C - - ScaleRowDown38_2_Box_16_C - - ScaleRowDown38_2_Box_C LSX - ScaleRowDown38_3_Box_16_C - - ScaleRowDown38_3_Box_C LSX - ScaleRowDown38_C LSX - ScaleRowDown4Box_16_C - - ScaleRowDown4Box_C LSX - ScaleRowDown4_16_C - - ScaleRowDown4_C LSX - ScaleRowUp2_Bilinear_16_Any_C - - ScaleRowUp2_Bilinear_16_C - - ScaleRowUp2_Bilinear_Any_C - - ScaleRowUp2_Bilinear_C - - ScaleRowUp2_Linear_16_Any_C - - ScaleRowUp2_Linear_16_C - - ScaleRowUp2_Linear_Any_C - - ScaleRowUp2_Linear_C - - ScaleUVAddCols1_16_C - - ScaleUVAddCols1_C - - ScaleUVAddCols2_16_C - - ScaleUVAddCols2_C - - ScaleUVAddCols3_16_C - - ScaleUVAddCols3_C LSX - ScaleUVCols64_16_C - - ScaleUVCols64_C - - ScaleUVColsUp2_16_C - - ScaleUVColsUp2_C - - ScaleUVCols_16_C - - ScaleUVCols_C - - ScaleUVFilterCols64_16_C - - ScaleUVFilterCols64_C - - ScaleUVFilterCols_16_C - - ScaleUVFilterCols_C - - ScaleUVRowDown2Box_16_C - - ScaleUVRowDown2Box_C - - ScaleUVRowDown2Linear_16_C - - ScaleUVRowDown2Linear_C - - ScaleUVRowDown2_16_C - - ScaleUVRowDown2_C - - ScaleUVRowDownEvenBox_16_C - - ScaleUVRowDownEvenBox_C - - ScaleUVRowDownEven_16_C - - ScaleUVRowDownEven_C - - ScaleUVRowUp2_Bilinear_16_Any_C - - ScaleUVRowUp2_Bilinear_16_C - - ScaleUVRowUp2_Bilinear_Any_C - - ScaleUVRowUp2_Bilinear_C - - ScaleUVRowUp2_Linear_16_Any_C - - ScaleUVRowUp2_Linear_16_C - - ScaleUVRowUp2_Linear_Any_C - - ScaleUVRowUp2_Linear_C - - ------------------------------------------------------------------------ Scale: 92 (100%) 21 (23%) 0 (0%) ------------------------------------------------------------------------ Test: qemu-loongarch64 -cpu la464 ./libyuv_unittest Bug: none Change-Id: I7990bc8c2524dc9434441b5379cf839012e73d5f Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8701144 Reviewed-by: richard winterton Commit-Queue: Frank Barchard --- Android.bp | 4 + Android.mk | 4 + BUILD.gn | 52 +- CMakeLists.txt | 25 +- GEMINI.md | 3 +- README.chromium | 2 +- include/libyuv/compare_row.h | 4 +- include/libyuv/loongson_intrinsics.h | 14 - include/libyuv/rotate_row.h | 4 +- include/libyuv/row.h | 33 +- include/libyuv/scale_row.h | 2 +- include/libyuv/version.h | 2 +- libyuv.gni | 5 +- libyuv.gypi | 10 +- linux.mk | 9 +- source/compare.cc | 4 + source/compare_la.cc | 652 +++++ source/{rotate_lasx.cc => rotate_la.cc} | 383 ++- source/rotate_lsx.cc | 383 --- source/{row_lasx.cc => row_la.cc} | 3509 +++++++++++++++++++---- source/row_lsx.cc | 3112 -------------------- source/{scale_lsx.cc => scale_la.cc} | 70 +- unit_test/compare_test.cc | 24 +- 23 files changed, 4147 insertions(+), 4163 deletions(-) delete mode 100644 include/libyuv/loongson_intrinsics.h create mode 100644 source/compare_la.cc rename source/{rotate_lasx.cc => rotate_la.cc} (57%) delete mode 100644 source/rotate_lsx.cc rename source/{row_lasx.cc => row_la.cc} (51%) delete mode 100644 source/row_lsx.cc rename source/{scale_lsx.cc => scale_la.cc} (97%) diff --git a/Android.bp b/Android.bp index 186a1cbab..cf590f6ad 100644 --- a/Android.bp +++ b/Android.bp @@ -35,6 +35,7 @@ cc_library { "source/compare_common.cc", "source/compare_gcc.cc", "source/compare_hvx.cc", + "source/compare_la.cc", "source/compare_neon.cc", "source/compare_neon64.cc", "source/convert.cc", @@ -54,6 +55,7 @@ cc_library { "source/rotate_common.cc", "source/rotate_gcc.cc", "source/rotate_hvx.cc", + "source/rotate_la.cc", "source/rotate_neon.cc", "source/rotate_neon64.cc", "source/rotate_rvv.cc", @@ -63,6 +65,7 @@ cc_library { "source/row_common.cc", "source/row_gcc.cc", "source/row_hvx.cc", + "source/row_la.cc", "source/row_neon.cc", "source/row_neon64.cc", "source/row_rvv.cc", @@ -74,6 +77,7 @@ cc_library { "source/scale_common.cc", "source/scale_gcc.cc", "source/scale_hvx.cc", + "source/scale_la.cc", "source/scale_neon.cc", "source/scale_neon64.cc", "source/scale_rgb.cc", diff --git a/Android.mk b/Android.mk index 8f8d1e442..5b13a3222 100644 --- a/Android.mk +++ b/Android.mk @@ -13,6 +13,7 @@ LOCAL_SRC_FILES := \ source/compare_common.cc \ source/compare_gcc.cc \ source/compare_hvx.cc \ + source/compare_la.cc \ source/compare_neon.cc \ source/compare_neon64.cc \ source/compare_win.cc \ @@ -30,6 +31,7 @@ LOCAL_SRC_FILES := \ source/rotate_common.cc \ source/rotate_gcc.cc \ source/rotate_hvx.cc \ + source/rotate_la.cc \ source/rotate_neon.cc \ source/rotate_neon64.cc \ source/rotate_rvv.cc \ @@ -38,6 +40,7 @@ LOCAL_SRC_FILES := \ source/row_common.cc \ source/row_gcc.cc \ source/row_hvx.cc \ + source/row_la.cc \ source/row_neon.cc \ source/row_neon64.cc \ source/row_rvv.cc \ @@ -48,6 +51,7 @@ LOCAL_SRC_FILES := \ source/scale_common.cc \ source/scale_gcc.cc \ source/scale_hvx.cc \ + source/scale_la.cc \ source/scale_neon.cc \ source/scale_neon64.cc \ source/scale_rgb.cc \ diff --git a/BUILD.gn b/BUILD.gn index 3735aa338..3b905a032 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -98,14 +98,6 @@ group("libyuv") { deps += [ ":libyuv_rvv" ] } - if (libyuv_use_lsx) { - deps += [ ":libyuv_lsx" ] - } - - if (libyuv_use_lasx) { - deps += [ ":libyuv_lasx" ] - } - if (!is_ios && !libyuv_disable_jpeg) { # Make sure that clients of libyuv link with libjpeg. This can't go in # libyuv_internal because in Windows x64 builds that will generate a clang @@ -129,7 +121,6 @@ static_library("libyuv_internal") { "include/libyuv/convert_from_argb.h", "include/libyuv/cpu_id.h", "include/libyuv/cpu_support.h", - "include/libyuv/loongson_intrinsics.h", "include/libyuv/mjpeg_decoder.h", "include/libyuv/planar_functions.h", "include/libyuv/rotate.h", @@ -150,6 +141,7 @@ static_library("libyuv_internal") { "source/compare_common.cc", "source/compare_gcc.cc", "source/compare_hvx.cc", + "source/compare_la.cc", "source/compare_wasm.cc", "source/compare_win.cc", "source/convert.cc", @@ -169,12 +161,14 @@ static_library("libyuv_internal") { "source/rotate_common.cc", "source/rotate_gcc.cc", "source/rotate_hvx.cc", + "source/rotate_la.cc", "source/rotate_wasm.cc", "source/rotate_win.cc", "source/row_any.cc", "source/row_common.cc", "source/row_gcc.cc", "source/row_hvx.cc", + "source/row_la.cc", "source/row_wasm.cc", "source/row_win.cc", "source/scale.cc", @@ -183,6 +177,7 @@ static_library("libyuv_internal") { "source/scale_common.cc", "source/scale_gcc.cc", "source/scale_hvx.cc", + "source/scale_la.cc", "source/scale_rgb.cc", "source/scale_uv.cc", "source/scale_wasm.cc", @@ -328,45 +323,6 @@ if (libyuv_use_rvv && !libyuv_disable_rvv) { } } -if (libyuv_use_lsx) { - static_library("libyuv_lsx") { - sources = [ - # LSX Source Files - "source/rotate_lsx.cc", - "source/row_lsx.cc", - "source/scale_lsx.cc", - ] - - cflags_cc = [ - "-mlsx", - "-Wno-c++11-narrowing", - ] - - deps = [ ":libyuv_internal" ] - - public_configs = [ ":libyuv_config" ] - } -} - -if (libyuv_use_lasx) { - static_library("libyuv_lasx") { - sources = [ - # LASX Source Files - "source/rotate_lasx.cc", - "source/row_lasx.cc", - ] - - cflags_cc = [ - "-mlasx", - "-Wno-c++11-narrowing", - ] - - deps = [ ":libyuv_internal" ] - - public_configs = [ ":libyuv_config" ] - } -} - if (libyuv_include_tests) { config("libyuv_unittest_warnings_config") { if (!is_win) { diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c84dc1b4..bc9c162a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,7 @@ set ( ly_common_source_files ${ly_src_dir}/compare_common.cc ${ly_src_dir}/compare_gcc.cc ${ly_src_dir}/compare_hvx.cc + ${ly_src_dir}/compare_la.cc ${ly_src_dir}/compare_wasm.cc ${ly_src_dir}/compare_win.cc ${ly_src_dir}/convert.cc @@ -44,6 +45,7 @@ set ( ly_common_source_files ${ly_src_dir}/rotate_common.cc ${ly_src_dir}/rotate_gcc.cc ${ly_src_dir}/rotate_hvx.cc + ${ly_src_dir}/rotate_la.cc ${ly_src_dir}/rotate_rvv.cc ${ly_src_dir}/rotate_wasm.cc ${ly_src_dir}/rotate_win.cc @@ -51,6 +53,7 @@ set ( ly_common_source_files ${ly_src_dir}/row_common.cc ${ly_src_dir}/row_gcc.cc ${ly_src_dir}/row_hvx.cc + ${ly_src_dir}/row_la.cc ${ly_src_dir}/row_rvv.cc ${ly_src_dir}/row_wasm.cc ${ly_src_dir}/row_win.cc @@ -60,6 +63,7 @@ set ( ly_common_source_files ${ly_src_dir}/scale_common.cc ${ly_src_dir}/scale_gcc.cc ${ly_src_dir}/scale_hvx.cc + ${ly_src_dir}/scale_la.cc ${ly_src_dir}/scale_rgb.cc ${ly_src_dir}/scale_rvv.cc ${ly_src_dir}/scale_uv.cc @@ -198,7 +202,6 @@ endif() if(LOONGARCH64) include(CheckCXXSourceCompiles) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-narrowing") check_cxx_source_compiles(" int main(int argc, char **argv) { __asm__ volatile ( @@ -213,23 +216,11 @@ if(LOONGARCH64) ); return 0; }" SUPPORTS_LASX) - if(SUPPORTS_LSX) - # These kernels use only LSX (128-bit) instructions and must not be - # compiled with -mlasx, which would let the compiler emit 256-bit LASX - # code (e.g. for stack-array copies) and crash on LSX-only CPUs. - add_library(${ly_lib_name}_lsx OBJECT - ${ly_src_dir}/rotate_lsx.cc - ${ly_src_dir}/row_lsx.cc - ${ly_src_dir}/scale_lsx.cc) - target_compile_options(${ly_lib_name}_lsx PRIVATE -mlsx) - list(APPEND ly_lib_parts $) + if(NOT SUPPORTS_LSX) + add_definitions(-DLIBYUV_DISABLE_LSX) endif() - if(SUPPORTS_LASX) - add_library(${ly_lib_name}_lasx OBJECT - ${ly_src_dir}/rotate_lasx.cc - ${ly_src_dir}/row_lasx.cc) - target_compile_options(${ly_lib_name}_lasx PRIVATE -mlasx) - list(APPEND ly_lib_parts $) + if(NOT SUPPORTS_LASX) + add_definitions(-DLIBYUV_DISABLE_LASX) endif() endif() diff --git a/GEMINI.md b/GEMINI.md index d9cfd977a..30c4866aa 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -43,7 +43,8 @@ and compiler compatibility. for SiFive X280. * **row_hvx.cc**: Qualcomm Hexagon Vector Extensions (HVX). Implemented using intrinsics. -* **row_lsx.cc / row_lasx.cc**: Loongarch MIPS-like extensions. +* **row_la.cc**: 64-bit LoongArch LSX and LASX extensions (written in inline + assembly). ### Utility and Fallbacks diff --git a/README.chromium b/README.chromium index 0bac66a43..019c74818 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1977 +Version: 1978 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index a3c529128..540d388cc 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -132,7 +132,7 @@ extern "C" { #endif // The following are available for LoongArch LSX: -#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) +#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_lp64) #define HAS_HAMMINGDISTANCE_LSX #define HAS_SUMSQUAREERROR_LSX #define HAS_HASHDJB2_LSX @@ -142,7 +142,7 @@ extern "C" { #endif // The following are available for LoongArch LASX: -#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_lp64) #define HAS_HAMMINGDISTANCE_LASX #define HAS_SUMSQUAREERROR_LASX #define HAS_HASHDJB2_LASX diff --git a/include/libyuv/loongson_intrinsics.h b/include/libyuv/loongson_intrinsics.h deleted file mode 100644 index 7dd44c575..000000000 --- a/include/libyuv/loongson_intrinsics.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright 2022 The LibYuv Project Authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H_ -#define INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H_ - -#endif // INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H_ diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index fa75aa314..b6bd2ff29 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -113,14 +113,14 @@ extern "C" { #define HAS_TRANSPOSEWXH_32_SME #endif -#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) +#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_lp64) #define HAS_TRANSPOSEWX8_LSX #define HAS_TRANSPOSEUVWX8_LSX #define HAS_TRANSPOSEWX8_16_LSX #define HAS_TRANSPOSEWX4_32_LSX #endif -#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_lp64) #define HAS_TRANSPOSEWX16_LASX #define HAS_TRANSPOSEUVWX16_LASX #define HAS_TRANSPOSEWX16_16_LASX diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 9f73b9be2..2c98e3c3d 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -1027,7 +1027,7 @@ extern "C" { #define HAS_SCALESUMSAMPLES_NEON #endif -#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) +#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_lp64) #define HAS_ARGB1555TOARGBROW_LSX #define HAS_ARGB1555TOUVROW_LSX #define HAS_ARGB1555TOYROW_LSX @@ -1039,38 +1039,41 @@ extern "C" { #define HAS_ARGBCOLORMATRIXROW_LSX #define HAS_ARGBEXTRACTALPHAROW_LSX #define HAS_ARGBGRAYROW_LSX +#define HAS_ARGBMIRRORROW_LSX +#define HAS_ARGBMULTIPLYROW_LSX #define HAS_ARGBORROW_LSX +#define HAS_ARGBQUANTIZEROW_LSX #define HAS_ARGBSEPIAROW_LSX +#define HAS_ARGBSETROW_LSX #define HAS_ARGBSHADEROW_LSX #define HAS_ARGBSHUFFLEROW_LSX #define HAS_ARGBSUBTRACTROW_LSX -#define HAS_ARGBQUANTIZEROW_LSX -#define HAS_ARGBSETROW_LSX #define HAS_ARGBTOARGB1555ROW_LSX #define HAS_ARGBTOARGB4444ROW_LSX #define HAS_ARGBTORAWROW_LSX #define HAS_ARGBTORGB24ROW_LSX -#define HAS_ARGBTORGB565ROW_LSX #define HAS_ARGBTORGB565DITHERROW_LSX -#define HAS_ARGBMIRRORROW_LSX -#define HAS_ARGBMULTIPLYROW_LSX +#define HAS_ARGBTORGB565ROW_LSX +#define HAS_ARGBTOYMATRIXROW_LSX +#define HAS_COPYROW_LSX #define HAS_HALFFLOATROW_LSX #define HAS_I400TOARGBROW_LSX -#define HAS_I444TOARGBROW_LSX -#define HAS_INTERPOLATEROW_LSX #define HAS_I422ALPHATOARGBROW_LSX #define HAS_I422TOARGB1555ROW_LSX #define HAS_I422TOARGB4444ROW_LSX +#define HAS_I422TOARGBROW_LSX #define HAS_I422TORGB24ROW_LSX #define HAS_I422TORGB565ROW_LSX #define HAS_I422TORGBAROW_LSX #define HAS_I422TOUYVYROW_LSX #define HAS_I422TOYUY2ROW_LSX +#define HAS_I444TOARGBROW_LSX +#define HAS_INTERPOLATEROW_LSX #define HAS_J400TOARGBROW_LSX #define HAS_MERGEUVROW_LSX #define HAS_MIRRORROW_LSX -#define HAS_MIRRORUVROW_LSX #define HAS_MIRRORSPLITUVROW_LSX +#define HAS_MIRRORUVROW_LSX #define HAS_NV12TOARGBROW_LSX #define HAS_NV12TORGB565ROW_LSX #define HAS_NV21TOARGBROW_LSX @@ -1082,6 +1085,7 @@ extern "C" { #define HAS_RGB565TOARGBROW_LSX #define HAS_RGB565TOUVROW_LSX #define HAS_RGB565TOYROW_LSX +#define HAS_RGBTOYMATRIXROW_LSX #define HAS_SETROW_LSX #define HAS_SOBELROW_LSX #define HAS_SOBELTOPLANEROW_LSX @@ -1092,19 +1096,12 @@ extern "C" { #define HAS_UYVYTOUVROW_LSX #define HAS_UYVYTOYROW_LSX #define HAS_YUY2TOARGBROW_LSX -#define HAS_YUY2TOUVROW_LSX #define HAS_YUY2TOUV422ROW_LSX +#define HAS_YUY2TOUVROW_LSX #define HAS_YUY2TOYROW_LSX -#define HAS_ARGBTOYMATRIXROW_LSX -#define HAS_COPYROW_LSX -#define HAS_RGBTOYMATRIXROW_LSX #endif -#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) -#define HAS_I422TOARGBROW_LSX -#endif - -#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_lp64) #define HAS_ARGB1555TOARGBROW_LASX #define HAS_ARGB1555TOUVROW_LASX #define HAS_ARGB1555TOYROW_LASX diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 94ef003b8..2f1f1365a 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -151,7 +151,7 @@ extern "C" { #define HAS_SCALEUVROWDOWN2_SME #endif -#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) +#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_lp64) #define HAS_SCALEARGBROWDOWN2_LSX #define HAS_SCALEARGBROWDOWNEVEN_LSX #define HAS_SCALEROWDOWN2_LSX diff --git a/include/libyuv/version.h b/include/libyuv/version.h index 5ba507c6d..5c69922cb 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1977 +#define LIBYUV_VERSION 1978 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/libyuv.gni b/libyuv.gni index 23e38c1b4..bd57fad6f 100644 --- a/libyuv.gni +++ b/libyuv.gni @@ -7,7 +7,6 @@ # be found in the AUTHORS file in the root of the source tree. import("//build/config/arm.gni") -import("//build/config/loongarch64.gni") import("//build/config/mips.gni") import("//build_overrides/build.gni") @@ -28,7 +27,7 @@ declare_args() { (current_cpu == "mips64el" || current_cpu == "mipsel") && mips_use_msa libyuv_use_mmi = (current_cpu == "mips64el" || current_cpu == "mipsel") && mips_use_mmi - libyuv_use_lsx = current_cpu == "loong64" && loongarch64_use_lsx - libyuv_use_lasx = current_cpu == "loong64" && loongarch64_use_lasx + libyuv_use_lsx = current_cpu == "loong64" + libyuv_use_lasx = current_cpu == "loong64" libyuv_use_rvv = current_cpu == "riscv64" || current_cpu == "riscv32" } diff --git a/libyuv.gypi b/libyuv.gypi index 91d9ff67f..3c2268b4c 100644 --- a/libyuv.gypi +++ b/libyuv.gypi @@ -20,7 +20,6 @@ 'include/libyuv/convert_from_argb.h', 'include/libyuv/cpu_id.h', 'include/libyuv/cpu_support.h', - 'include/libyuv/loongson_intrinsics.h', 'include/libyuv/mjpeg_decoder.h', 'include/libyuv/planar_functions.h', 'include/libyuv/rotate.h', @@ -41,6 +40,7 @@ 'source/compare_common.cc', 'source/compare_gcc.cc', 'source/compare_hvx.cc', + 'source/compare_la.cc', 'source/compare_neon.cc', 'source/compare_neon64.cc', 'source/compare_win.cc', @@ -61,8 +61,7 @@ 'source/rotate_common.cc', 'source/rotate_gcc.cc', 'source/rotate_hvx.cc', - 'source/rotate_lasx.cc', - 'source/rotate_lsx.cc', + 'source/rotate_la.cc', 'source/rotate_neon.cc', 'source/rotate_neon64.cc', 'source/rotate_rvv.cc', @@ -71,8 +70,7 @@ 'source/row_common.cc', 'source/row_gcc.cc', 'source/row_hvx.cc', - 'source/row_lasx.cc', - 'source/row_lsx.cc', + 'source/row_la.cc', 'source/row_neon.cc', 'source/row_neon64.cc', 'source/row_rvv.cc', @@ -83,7 +81,7 @@ 'source/scale_common.cc', 'source/scale_gcc.cc', 'source/scale_hvx.cc', - 'source/scale_lsx.cc', + 'source/scale_la.cc', 'source/scale_neon.cc', 'source/scale_neon64.cc', 'source/scale_rgb.cc', diff --git a/linux.mk b/linux.mk index b458e84b7..0bf693c2f 100644 --- a/linux.mk +++ b/linux.mk @@ -14,6 +14,7 @@ LOCAL_OBJ_FILES := \ source/compare_common.o \ source/compare_gcc.o \ source/compare_hvx.o \ + source/compare_la.o \ source/compare_neon.o \ source/compare_neon64.o \ source/compare_win.o \ @@ -34,8 +35,7 @@ LOCAL_OBJ_FILES := \ source/rotate_common.o \ source/rotate_gcc.o \ source/rotate_hvx.o \ - source/rotate_lasx.o \ - source/rotate_lsx.o \ + source/rotate_la.o \ source/rotate_neon.o \ source/rotate_neon64.o \ source/rotate_rvv.o \ @@ -46,8 +46,7 @@ LOCAL_OBJ_FILES := \ source/row_common.o \ source/row_gcc.o \ source/row_hvx.o \ - source/row_lasx.o \ - source/row_lsx.o \ + source/row_la.o \ source/row_neon.o \ source/row_neon64.o \ source/row_rvv.o \ @@ -60,7 +59,7 @@ LOCAL_OBJ_FILES := \ source/scale_common.o \ source/scale_gcc.o \ source/scale_hvx.o \ - source/scale_lsx.o \ + source/scale_la.o \ source/scale_neon.o \ source/scale_neon64.o \ source/scale_rgb.o \ diff --git a/source/compare.cc b/source/compare.cc index 12c573896..4ad159ad2 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -238,11 +238,13 @@ uint64_t ComputeHammingDistance(const uint8_t* src_a, #if defined(HAS_HAMMINGDISTANCE_LSX) if (TestCpuFlag(kCpuHasLSX)) { HammingDistance = HammingDistance_LSX; + simd_size = 16; } #endif #if defined(HAS_HAMMINGDISTANCE_LASX) if (TestCpuFlag(kCpuHasLASX)) { HammingDistance = HammingDistance_LASX; + simd_size = 32; } #endif #if defined(HAS_HAMMINGDISTANCE_WASMSIMD) @@ -346,11 +348,13 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a, #if defined(HAS_SUMSQUAREERROR_LSX) if (TestCpuFlag(kCpuHasLSX)) { SumSquareError = SumSquareError_LSX; + simd_size = 16; } #endif #if defined(HAS_SUMSQUAREERROR_LASX) if (TestCpuFlag(kCpuHasLASX)) { SumSquareError = SumSquareError_LASX; + simd_size = 32; } #endif #if defined(HAS_SUMSQUAREERROR_WASMSIMD) diff --git a/source/compare_la.cc b/source/compare_la.cc new file mode 100644 index 000000000..1ee23b3bd --- /dev/null +++ b/source/compare_la.cc @@ -0,0 +1,652 @@ +/* + * Copyright 2022 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/compare_row.h" +#include "libyuv/row.h" +#include "libyuv/video_common.h" + +// Note: every register written by an inline asm block (vector, general +// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be +// listed in its clobber list, even when the ABI treats it as caller-saved. +// With LTO/LTCG the compiler may keep values live in any register across the +// asm statement, and other OS ABIs (e.g. Windows) differ in which registers +// are callee-saved. List "memory" first, then "cc", then registers. + +#if (!defined(LIBYUV_DISABLE_LSX) || !defined(LIBYUV_DISABLE_LASX)) && \ + defined(__loongarch_lp64) + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#if defined(HAS_HASHDJB2_LSX) || defined(HAS_HASHDJB2_LASX) +static const ulvec32 kDjb2Multiplicands[] = { + {0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, 0x30f35d61, 0x855cb541, + 0x040a9121, 0x747c7101}, + {0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881, 0x00008c61, 0x00000441, + 0x00000021, 0x00000001}, +}; +#endif // defined(HAS_HASHDJB2_LSX) || defined(HAS_HASHDJB2_LASX) + +#ifdef HAS_HAMMINGDISTANCE_LSX +uint32_t HammingDistance_LSX(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + uint32_t diff; + asm volatile( + "vxor.v $vr4, $vr4, $vr4\n\t" + "1:\n\t" + "vld $vr0, %[sa], 0\n\t" + "vld $vr1, %[sb], 0\n\t" + "vxor.v $vr0, $vr0, $vr1\n\t" + "vpcnt.b $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" + "vhaddw.wu.hu $vr0, $vr0, $vr0\n\t" + "vadd.w $vr4, $vr4, $vr0\n\t" + "addi.d %[sa], %[sa], 16\n\t" + "addi.d %[sb], %[sb], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + "vhaddw.du.wu $vr4, $vr4, $vr4\n\t" + "vhaddw.qu.du $vr4, $vr4, $vr4\n\t" + "vpickve2gr.wu %[diff], $vr4, 0\n\t" + : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [diff] "=r"(diff) + : + : "memory", "$vr0", "$vr1", "$vr4"); + return diff; +} +#endif // HAS_HAMMINGDISTANCE_LSX + +#ifdef HAS_SUMSQUAREERROR_LSX +uint32_t SumSquareError_LSX(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + uint32_t sse; + asm volatile( + "vxor.v $vr4, $vr4, $vr4\n\t" + "1:\n\t" + "vld $vr0, %[sa], 0\n\t" + "vld $vr1, %[sb], 0\n\t" + "vabsd.bu $vr0, $vr0, $vr1\n\t" + "vmulwev.h.bu $vr2, $vr0, $vr0\n\t" + "vmulwod.h.bu $vr3, $vr0, $vr0\n\t" + "vhaddw.wu.hu $vr2, $vr2, $vr2\n\t" + "vhaddw.wu.hu $vr3, $vr3, $vr3\n\t" + "vadd.w $vr4, $vr4, $vr2\n\t" + "vadd.w $vr4, $vr4, $vr3\n\t" + "addi.d %[sa], %[sa], 16\n\t" + "addi.d %[sb], %[sb], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + "vhaddw.du.wu $vr4, $vr4, $vr4\n\t" + "vhaddw.qu.du $vr4, $vr4, $vr4\n\t" + "vpickve2gr.wu %[sse], $vr4, 0\n\t" + : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [sse] "=r"(sse) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); + return sse; +} +#endif // HAS_SUMSQUAREERROR_LSX + +#ifdef HAS_HASHDJB2_LSX +uint32_t HashDjb2_LSX(const uint8_t* src, int count, uint32_t seed) { + uint32_t hash = seed; + uint32_t tmp; + uint32_t c16 = 0x92d9e201u; + asm volatile( + "vld $vr4, %[muls], 0\n\t" + "vld $vr5, %[muls], 16\n\t" + "vld $vr6, %[muls], 32\n\t" + "vld $vr7, %[muls], 48\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vsllwil.hu.bu $vr1, $vr0, 0\n\t" + "vexth.hu.bu $vr2, $vr0\n\t" + "vsllwil.wu.hu $vr0, $vr1, 0\n\t" + "vexth.wu.hu $vr1, $vr1\n\t" + "vsllwil.wu.hu $vr3, $vr2, 0\n\t" + "vexth.wu.hu $vr2, $vr2\n\t" + "vmul.w $vr0, $vr0, $vr4\n\t" + "vmadd.w $vr0, $vr1, $vr5\n\t" + "vmadd.w $vr0, $vr3, $vr6\n\t" + "vmadd.w $vr0, $vr2, $vr7\n\t" + "vhaddw.du.wu $vr0, $vr0, $vr0\n\t" + "vhaddw.qu.du $vr0, $vr0, $vr0\n\t" + "vpickve2gr.wu %[tmp], $vr0, 0\n\t" + "mul.w %[hash], %[hash], %[c16]\n\t" + "add.w %[hash], %[hash], %[tmp]\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [cnt] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp) + : [muls] "r"(kDjb2Multiplicands), [c16] "r"(c16) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); + return hash; +} +#endif // HAS_HASHDJB2_LSX + +#ifdef HAS_SSIM4X4ROW_LSX +void Ssim4x4Row_LSX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const uint8_t* pa; + const uint8_t* pb; + int r; + asm volatile( + "1:\n\t" + "move %[pa], %[sa]\n\t" + "move %[pb], %[sb]\n\t" + "li.d %[r], 4\n\t" + "vxor.v $vr8, $vr8, $vr8\n\t" + "vxor.v $vr9, $vr9, $vr9\n\t" + "vxor.v $vr10, $vr10, $vr10\n\t" + "vxor.v $vr11, $vr11, $vr11\n\t" + "2:\n\t" + "vld $vr0, %[pa], 0\n\t" + "vld $vr1, %[pb], 0\n\t" + "add.d %[pa], %[pa], %[sta]\n\t" + "add.d %[pb], %[pb], %[stb]\n\t" + "vhaddw.hu.bu $vr2, $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr3, $vr1, $vr1\n\t" + "vadd.h $vr8, $vr8, $vr2\n\t" + "vadd.h $vr9, $vr9, $vr3\n\t" + "vmulwev.h.bu $vr2, $vr0, $vr1\n\t" + "vmulwod.h.bu $vr3, $vr0, $vr1\n\t" + "vhaddw.wu.hu $vr2, $vr2, $vr2\n\t" + "vhaddw.wu.hu $vr3, $vr3, $vr3\n\t" + "vadd.w $vr11, $vr11, $vr2\n\t" + "vadd.w $vr11, $vr11, $vr3\n\t" + "vmulwev.h.bu $vr2, $vr0, $vr0\n\t" + "vmulwod.h.bu $vr3, $vr0, $vr0\n\t" + "vmulwev.h.bu $vr4, $vr1, $vr1\n\t" + "vmulwod.h.bu $vr5, $vr1, $vr1\n\t" + "vhaddw.wu.hu $vr2, $vr2, $vr2\n\t" + "vhaddw.wu.hu $vr3, $vr3, $vr3\n\t" + "vhaddw.wu.hu $vr4, $vr4, $vr4\n\t" + "vhaddw.wu.hu $vr5, $vr5, $vr5\n\t" + "vadd.w $vr2, $vr2, $vr3\n\t" + "vadd.w $vr4, $vr4, $vr5\n\t" + "vadd.w $vr10, $vr10, $vr2\n\t" + "vadd.w $vr10, $vr10, $vr4\n\t" + "addi.d %[r], %[r], -1\n\t" + "bnez %[r], 2b\n\t" + "vhaddw.wu.hu $vr8, $vr8, $vr8\n\t" + "vhaddw.wu.hu $vr9, $vr9, $vr9\n\t" + "vilvl.w $vr0, $vr9, $vr8\n\t" + "vilvh.w $vr1, $vr9, $vr8\n\t" + "vilvl.w $vr2, $vr11, $vr10\n\t" + "vilvh.w $vr3, $vr11, $vr10\n\t" + "vilvl.d $vr4, $vr2, $vr0\n\t" + "vilvh.d $vr5, $vr2, $vr0\n\t" + "vilvl.d $vr6, $vr3, $vr1\n\t" + "vilvh.d $vr7, $vr3, $vr1\n\t" + "vst $vr4, %[dst], 0\n\t" + "vst $vr5, %[dst], 16\n\t" + "vst $vr6, %[dst], 32\n\t" + "vst $vr7, %[dst], 48\n\t" + "addi.d %[sa], %[sa], 16\n\t" + "addi.d %[sb], %[sb], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst), [cnt] "+r"(width), + [pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r) + : [sta] "r"((ptrdiff_t)stride_a), [stb] "r"((ptrdiff_t)stride_b) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11"); +} +#endif // HAS_SSIM4X4ROW_LSX + +#ifdef HAS_SSIMWINDOWROW_LSX +float SsimWindowRow_LSX(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + const uint32_t* row0_next = row0 + 4; + const uint32_t* row1_next = row1 + 4; + float total; + uint32_t c1 = 26634; + uint32_t c2 = 239708; + uint32_t c0 = 0; + asm volatile( + "vreplgr2vr.w $vr6, %[c0]\n\t" + "vreplgr2vr.w $vr4, %[c1]\n\t" + "vreplgr2vr.w $vr5, %[c2]\n\t" + "1:\n\t" + "vld $vr0, %[r0], 0\n\t" + "vld $vr1, %[r0n], 0\n\t" + "vld $vr2, %[r1], 0\n\t" + "vld $vr3, %[r1n], 0\n\t" + "vadd.w $vr0, $vr0, $vr1\n\t" + "vadd.w $vr2, $vr2, $vr3\n\t" + "vadd.w $vr0, $vr0, $vr2\n\t" + "vshuf4i.w $vr1, $vr0, 0x00\n\t" + "vshuf4i.w $vr2, $vr0, 0x55\n\t" + "vshuf4i.w $vr3, $vr0, 0xaa\n\t" + "vshuf4i.w $vr0, $vr0, 0xff\n\t" + "vmul.w $vr7, $vr1, $vr2\n\t" + "vslli.w $vr7, $vr7, 1\n\t" + "vmul.w $vr1, $vr1, $vr1\n\t" + "vmadd.w $vr1, $vr2, $vr2\n\t" + "vslli.w $vr0, $vr0, 7\n\t" + "vsub.w $vr0, $vr0, $vr7\n\t" + "vadd.w $vr7, $vr7, $vr4\n\t" + "vadd.w $vr0, $vr0, $vr5\n\t" + "vslli.w $vr3, $vr3, 6\n\t" + "vsub.w $vr3, $vr3, $vr1\n\t" + "vadd.w $vr1, $vr1, $vr4\n\t" + "vadd.w $vr3, $vr3, $vr5\n\t" + "vffint.s.w $vr7, $vr7\n\t" + "vffint.s.w $vr0, $vr0\n\t" + "vffint.s.w $vr1, $vr1\n\t" + "vffint.s.w $vr3, $vr3\n\t" + "vfmul.s $vr7, $vr7, $vr0\n\t" + "vfmul.s $vr1, $vr1, $vr3\n\t" + "vfdiv.s $vr7, $vr7, $vr1\n\t" + "vfadd.s $vr6, $vr6, $vr7\n\t" + "addi.d %[r0], %[r0], 16\n\t" + "addi.d %[r0n], %[r0n], 16\n\t" + "addi.d %[r1], %[r1], 16\n\t" + "addi.d %[r1n], %[r1n], 16\n\t" + "addi.d %[cnt], %[cnt], -1\n\t" + "bnez %[cnt], 1b\n\t" + "vstelm.w $vr6, %[tot], 0, 0\n\t" + : [r0] "+r"(row0), [r0n] "+r"(row0_next), [r1] "+r"(row1), + [r1n] "+r"(row1_next), [cnt] "+r"(num_windows) + : [c0] "r"(c0), [c1] "r"(c1), [c2] "r"(c2), [tot] "r"(&total) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); + return total; +} +#endif // HAS_SSIMWINDOWROW_LSX + +#ifdef HAS_ARGBDETECTROW_LSX +uint32_t ARGBDetectRow_LSX(const uint8_t* argb, int width) { + uint32_t diff = 0; + uint32_t rgb_mask = 0x00ffff00u; + asm volatile( + "vreplgr2vr.w $vr2, %[rgb_mask]\n\t" + "addi.d %[w], %[w], -4\n\t" + "bltz %[w], 2f\n\t" + "1:\n\t" + "vld $vr0, %[argb], 0\n\t" + "addi.d %[argb], %[argb], 16\n\t" + "vor.v $vr0, $vr0, $vr2\n\t" + "vseqi.b $vr0, $vr0, -1\n\t" + "vmskltz.b $vr0, $vr0\n\t" + "vpickve2gr.hu %[diff], $vr0, 0\n\t" + "nor %[diff], %[diff], $zero\n\t" + "bstrpick.w %[diff], %[diff], 15, 0\n\t" + "bnez %[diff], 4f\n\t" + "addi.d %[w], %[w], -4\n\t" + "bgez %[w], 1b\n\t" + "2:\n\t" + "addi.d %[w], %[w], 4\n\t" + "beqz %[w], 4f\n\t" + "3:\n\t" + "vldrepl.w $vr0, %[argb], 0\n\t" + "addi.d %[argb], %[argb], 4\n\t" + "vor.v $vr0, $vr0, $vr2\n\t" + "vseqi.b $vr0, $vr0, -1\n\t" + "vmskltz.b $vr0, $vr0\n\t" + "vpickve2gr.hu %[diff], $vr0, 0\n\t" + "nor %[diff], %[diff], $zero\n\t" + "bstrpick.w %[diff], %[diff], 15, 0\n\t" + "bnez %[diff], 4f\n\t" + "addi.d %[w], %[w], -1\n\t" + "bnez %[w], 3b\n\t" + "4:\n\t" + : [argb] "+r"(argb), [w] "+r"(width), [diff] "+r"(diff) + : [rgb_mask] "r"(rgb_mask) + : "memory", "$vr0", "$vr2"); + if (diff) { + return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB; + } + return 0; +} +#endif // HAS_ARGBDETECTROW_LSX + +#ifdef HAS_HAMMINGDISTANCE_LASX +uint32_t HammingDistance_LASX(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + uint32_t diff; + asm volatile( + "xvxor.v $xr4, $xr4, $xr4\n\t" + "1:\n\t" + "xvld $xr0, %[sa], 0\n\t" + "xvld $xr1, %[sb], 0\n\t" + "xvxor.v $xr0, $xr0, $xr1\n\t" + "xvpcnt.b $xr0, $xr0\n\t" + "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t" + "xvhaddw.wu.hu $xr0, $xr0, $xr0\n\t" + "xvadd.w $xr4, $xr4, $xr0\n\t" + "addi.d %[sa], %[sa], 32\n\t" + "addi.d %[sb], %[sb], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + "xvhaddw.du.wu $xr4, $xr4, $xr4\n\t" + "xvhaddw.qu.du $xr4, $xr4, $xr4\n\t" + "xvpermi.q $xr0, $xr4, 0x01\n\t" + "vadd.w $vr4, $vr4, $vr0\n\t" + "vpickve2gr.wu %[diff], $vr4, 0\n\t" + : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [diff] "=r"(diff) + : + : "memory", "$xr0", "$xr1", "$xr4"); + return diff; +} +#endif // HAS_HAMMINGDISTANCE_LASX + +#ifdef HAS_SUMSQUAREERROR_LASX +uint32_t SumSquareError_LASX(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + uint32_t sse; + asm volatile( + "xvxor.v $xr4, $xr4, $xr4\n\t" + "1:\n\t" + "xvld $xr0, %[sa], 0\n\t" + "xvld $xr1, %[sb], 0\n\t" + "xvabsd.bu $xr0, $xr0, $xr1\n\t" + "xvmulwev.h.bu $xr2, $xr0, $xr0\n\t" + "xvmulwod.h.bu $xr3, $xr0, $xr0\n\t" + "xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t" + "xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t" + "xvadd.w $xr4, $xr4, $xr2\n\t" + "xvadd.w $xr4, $xr4, $xr3\n\t" + "addi.d %[sa], %[sa], 32\n\t" + "addi.d %[sb], %[sb], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + "xvhaddw.du.wu $xr4, $xr4, $xr4\n\t" + "xvhaddw.qu.du $xr4, $xr4, $xr4\n\t" + "xvpermi.q $xr0, $xr4, 0x01\n\t" + "vadd.w $vr4, $vr4, $vr0\n\t" + "vpickve2gr.wu %[sse], $vr4, 0\n\t" + : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [sse] "=r"(sse) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); + return sse; +} +#endif // HAS_SUMSQUAREERROR_LASX + +#ifdef HAS_HASHDJB2_LASX +uint32_t HashDjb2_LASX(const uint8_t* src, int count, uint32_t seed) { + uint32_t hash = seed; + uint32_t tmp; + uint32_t c16 = 0x92d9e201u; + asm volatile( + "xvld $xr4, %[muls], 0\n\t" + "xvld $xr5, %[muls], 32\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vbsrl.v $vr1, $vr0, 8\n\t" + "vext2xv.wu.bu $xr0, $xr0\n\t" + "vext2xv.wu.bu $xr1, $xr1\n\t" + "xvmul.w $xr0, $xr0, $xr4\n\t" + "xvmadd.w $xr0, $xr1, $xr5\n\t" + "xvhaddw.du.wu $xr0, $xr0, $xr0\n\t" + "xvhaddw.qu.du $xr0, $xr0, $xr0\n\t" + "xvpermi.q $xr1, $xr0, 0x01\n\t" + "vadd.w $vr0, $vr0, $vr1\n\t" + "vpickve2gr.wu %[tmp], $vr0, 0\n\t" + "mul.w %[hash], %[hash], %[c16]\n\t" + "add.w %[hash], %[hash], %[tmp]\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [cnt] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp) + : [muls] "r"(kDjb2Multiplicands), [c16] "r"(c16) + : "memory", "$xr0", "$xr1", "$xr4", "$xr5"); + return hash; +} +#endif // HAS_HASHDJB2_LASX + +#ifdef HAS_SSIM4X4ROW_LASX +void Ssim4x4Row_LASX(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b, + uint32_t* dst, + int width) { + const uint8_t* pa; + const uint8_t* pb; + int r; + asm volatile( + "1:\n\t" + "move %[pa], %[sa]\n\t" + "move %[pb], %[sb]\n\t" + "li.d %[r], 4\n\t" + "xvxor.v $xr8, $xr8, $xr8\n\t" + "xvxor.v $xr9, $xr9, $xr9\n\t" + "xvxor.v $xr10, $xr10, $xr10\n\t" + "xvxor.v $xr11, $xr11, $xr11\n\t" + "2:\n\t" + "xvld $xr0, %[pa], 0\n\t" + "xvld $xr1, %[pb], 0\n\t" + "add.d %[pa], %[pa], %[sta]\n\t" + "add.d %[pb], %[pb], %[stb]\n\t" + "xvhaddw.hu.bu $xr2, $xr0, $xr0\n\t" + "xvhaddw.hu.bu $xr3, $xr1, $xr1\n\t" + "xvadd.h $xr8, $xr8, $xr2\n\t" + "xvadd.h $xr9, $xr9, $xr3\n\t" + "xvmulwev.h.bu $xr2, $xr0, $xr1\n\t" + "xvmulwod.h.bu $xr3, $xr0, $xr1\n\t" + "xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t" + "xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t" + "xvadd.w $xr11, $xr11, $xr2\n\t" + "xvadd.w $xr11, $xr11, $xr3\n\t" + "xvmulwev.h.bu $xr2, $xr0, $xr0\n\t" + "xvmulwod.h.bu $xr3, $xr0, $xr0\n\t" + "xvmulwev.h.bu $xr4, $xr1, $xr1\n\t" + "xvmulwod.h.bu $xr5, $xr1, $xr1\n\t" + "xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t" + "xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t" + "xvhaddw.wu.hu $xr4, $xr4, $xr4\n\t" + "xvhaddw.wu.hu $xr5, $xr5, $xr5\n\t" + "xvadd.w $xr2, $xr2, $xr3\n\t" + "xvadd.w $xr4, $xr4, $xr5\n\t" + "xvadd.w $xr10, $xr10, $xr2\n\t" + "xvadd.w $xr10, $xr10, $xr4\n\t" + "addi.d %[r], %[r], -1\n\t" + "bnez %[r], 2b\n\t" + "xvhaddw.wu.hu $xr8, $xr8, $xr8\n\t" + "xvhaddw.wu.hu $xr9, $xr9, $xr9\n\t" + "xvilvl.w $xr0, $xr9, $xr8\n\t" + "xvilvh.w $xr1, $xr9, $xr8\n\t" + "xvilvl.w $xr2, $xr11, $xr10\n\t" + "xvilvh.w $xr3, $xr11, $xr10\n\t" + "xvilvl.d $xr4, $xr2, $xr0\n\t" + "xvilvh.d $xr5, $xr2, $xr0\n\t" + "xvilvl.d $xr6, $xr3, $xr1\n\t" + "xvilvh.d $xr7, $xr3, $xr1\n\t" + "xvor.v $xr0, $xr5, $xr5\n\t" + "xvpermi.q $xr0, $xr4, 0x20\n\t" + "xvpermi.q $xr5, $xr4, 0x31\n\t" + "xvor.v $xr1, $xr7, $xr7\n\t" + "xvpermi.q $xr1, $xr6, 0x20\n\t" + "xvpermi.q $xr7, $xr6, 0x31\n\t" + "xvst $xr0, %[dst], 0\n\t" + "xvst $xr1, %[dst], 32\n\t" + "xvst $xr5, %[dst], 64\n\t" + "xvst $xr7, %[dst], 96\n\t" + "addi.d %[sa], %[sa], 32\n\t" + "addi.d %[sb], %[sb], 32\n\t" + "addi.d %[dst], %[dst], 128\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst), [cnt] "+r"(width), + [pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r) + : [sta] "r"((ptrdiff_t)stride_a), [stb] "r"((ptrdiff_t)stride_b) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11"); +} +#endif // HAS_SSIM4X4ROW_LASX + +#ifdef HAS_SSIMWINDOWROW_LASX +float SsimWindowRow_LASX(const uint32_t* row0, + const uint32_t* row1, + int num_windows) { + const uint32_t* row0_next = row0 + 4; + const uint32_t* row1_next = row1 + 4; + float total; + uint32_t c1 = 26634; + uint32_t c2 = 239708; + uint32_t c0 = 0; + asm volatile( + "xvreplgr2vr.w $xr6, %[c0]\n\t" + "xvreplgr2vr.w $xr4, %[c1]\n\t" + "xvreplgr2vr.w $xr5, %[c2]\n\t" + "addi.d %[cnt], %[cnt], -2\n\t" + "bltz %[cnt], 2f\n\t" + "1:\n\t" + "xvld $xr0, %[r0], 0\n\t" + "xvld $xr1, %[r0n], 0\n\t" + "xvld $xr2, %[r1], 0\n\t" + "xvld $xr3, %[r1n], 0\n\t" + "xvadd.w $xr0, $xr0, $xr1\n\t" + "xvadd.w $xr2, $xr2, $xr3\n\t" + "xvadd.w $xr0, $xr0, $xr2\n\t" + "xvshuf4i.w $xr1, $xr0, 0x00\n\t" + "xvshuf4i.w $xr2, $xr0, 0x55\n\t" + "xvshuf4i.w $xr3, $xr0, 0xaa\n\t" + "xvshuf4i.w $xr0, $xr0, 0xff\n\t" + "xvmul.w $xr7, $xr1, $xr2\n\t" + "xvslli.w $xr7, $xr7, 1\n\t" + "xvmul.w $xr1, $xr1, $xr1\n\t" + "xvmadd.w $xr1, $xr2, $xr2\n\t" + "xvslli.w $xr0, $xr0, 7\n\t" + "xvsub.w $xr0, $xr0, $xr7\n\t" + "xvadd.w $xr7, $xr7, $xr4\n\t" + "xvadd.w $xr0, $xr0, $xr5\n\t" + "xvslli.w $xr3, $xr3, 6\n\t" + "xvsub.w $xr3, $xr3, $xr1\n\t" + "xvadd.w $xr1, $xr1, $xr4\n\t" + "xvadd.w $xr3, $xr3, $xr5\n\t" + "xvffint.s.w $xr7, $xr7\n\t" + "xvffint.s.w $xr0, $xr0\n\t" + "xvffint.s.w $xr1, $xr1\n\t" + "xvffint.s.w $xr3, $xr3\n\t" + "xvfmul.s $xr7, $xr7, $xr0\n\t" + "xvfmul.s $xr1, $xr1, $xr3\n\t" + "xvfdiv.s $xr7, $xr7, $xr1\n\t" + "xvfadd.s $xr6, $xr6, $xr7\n\t" + "addi.d %[r0], %[r0], 32\n\t" + "addi.d %[r0n], %[r0n], 32\n\t" + "addi.d %[r1], %[r1], 32\n\t" + "addi.d %[r1n], %[r1n], 32\n\t" + "addi.d %[cnt], %[cnt], -2\n\t" + "bgez %[cnt], 1b\n\t" + "xvpermi.q $xr0, $xr6, 0x01\n\t" + "vfadd.s $vr6, $vr6, $vr0\n\t" + "2:\n\t" + "addi.d %[cnt], %[cnt], 2\n\t" + "beqz %[cnt], 3f\n\t" + "vld $vr0, %[r0], 0\n\t" + "vld $vr1, %[r0n], 0\n\t" + "vld $vr2, %[r1], 0\n\t" + "vld $vr3, %[r1n], 0\n\t" + "vadd.w $vr0, $vr0, $vr1\n\t" + "vadd.w $vr2, $vr2, $vr3\n\t" + "vadd.w $vr0, $vr0, $vr2\n\t" + "vshuf4i.w $vr1, $vr0, 0x00\n\t" + "vshuf4i.w $vr2, $vr0, 0x55\n\t" + "vshuf4i.w $vr3, $vr0, 0xaa\n\t" + "vshuf4i.w $vr0, $vr0, 0xff\n\t" + "vmul.w $vr7, $vr1, $vr2\n\t" + "vslli.w $vr7, $vr7, 1\n\t" + "vmul.w $vr1, $vr1, $vr1\n\t" + "vmadd.w $vr1, $vr2, $vr2\n\t" + "vslli.w $vr0, $vr0, 7\n\t" + "vsub.w $vr0, $vr0, $vr7\n\t" + "vadd.w $vr7, $vr7, $vr4\n\t" + "vadd.w $vr0, $vr0, $vr5\n\t" + "vslli.w $vr3, $vr3, 6\n\t" + "vsub.w $vr3, $vr3, $vr1\n\t" + "vadd.w $vr1, $vr1, $vr4\n\t" + "vadd.w $vr3, $vr3, $vr5\n\t" + "vffint.s.w $vr7, $vr7\n\t" + "vffint.s.w $vr0, $vr0\n\t" + "vffint.s.w $vr1, $vr1\n\t" + "vffint.s.w $vr3, $vr3\n\t" + "vfmul.s $vr7, $vr7, $vr0\n\t" + "vfmul.s $vr1, $vr1, $vr3\n\t" + "vfdiv.s $vr7, $vr7, $vr1\n\t" + "vfadd.s $vr6, $vr6, $vr7\n\t" + "3:\n\t" + "vstelm.w $vr6, %[tot], 0, 0\n\t" + : [r0] "+r"(row0), [r0n] "+r"(row0_next), [r1] "+r"(row1), + [r1n] "+r"(row1_next), [cnt] "+r"(num_windows) + : [c0] "r"(c0), [c1] "r"(c1), [c2] "r"(c2), [tot] "r"(&total) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7"); + return total; +} +#endif // HAS_SSIMWINDOWROW_LASX + +#ifdef HAS_ARGBDETECTROW_LASX +uint32_t ARGBDetectRow_LASX(const uint8_t* argb, int width) { + uint32_t diff = 0; + uint32_t tmp; + uint32_t rgb_mask = 0x00ffff00u; + asm volatile( + "xvreplgr2vr.w $xr2, %[rgb_mask]\n\t" + "addi.d %[w], %[w], -8\n\t" + "bltz %[w], 2f\n\t" + "1:\n\t" + "xvld $xr0, %[argb], 0\n\t" + "addi.d %[argb], %[argb], 32\n\t" + "xvor.v $xr0, $xr0, $xr2\n\t" + "xvseqi.b $xr0, $xr0, -1\n\t" + "xvmskltz.b $xr0, $xr0\n\t" + "xvpickve2gr.wu %[diff], $xr0, 0\n\t" + "xvpickve2gr.wu %[tmp], $xr0, 4\n\t" + "bstrins.w %[diff], %[tmp], 31, 16\n\t" + "nor %[diff], %[diff], $zero\n\t" + "bnez %[diff], 4f\n\t" + "addi.d %[w], %[w], -8\n\t" + "bgez %[w], 1b\n\t" + "2:\n\t" + "addi.d %[w], %[w], 8\n\t" + "beqz %[w], 4f\n\t" + "3:\n\t" + "vldrepl.w $vr0, %[argb], 0\n\t" + "addi.d %[argb], %[argb], 4\n\t" + "vor.v $vr0, $vr0, $vr2\n\t" + "vseqi.b $vr0, $vr0, -1\n\t" + "vmskltz.b $vr0, $vr0\n\t" + "vpickve2gr.hu %[diff], $vr0, 0\n\t" + "nor %[diff], %[diff], $zero\n\t" + "bstrpick.w %[diff], %[diff], 15, 0\n\t" + "bnez %[diff], 4f\n\t" + "addi.d %[w], %[w], -1\n\t" + "bnez %[w], 3b\n\t" + "4:\n\t" + : [argb] "+r"(argb), [w] "+r"(width), [diff] "+r"(diff), [tmp] "=&r"(tmp) + : [rgb_mask] "r"(rgb_mask) + : "memory", "$xr0", "$xr2"); + if (diff) { + return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB; + } + return 0; +} +#endif // HAS_ARGBDETECTROW_LASX + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // (!defined(LIBYUV_DISABLE_LSX) || !defined(LIBYUV_DISABLE_LASX)) && + // defined(__loongarch_lp64) diff --git a/source/rotate_lasx.cc b/source/rotate_la.cc similarity index 57% rename from source/rotate_lasx.cc rename to source/rotate_la.cc index 8a50cbe2a..39c0a56bf 100644 --- a/source/rotate_lasx.cc +++ b/source/rotate_la.cc @@ -1,5 +1,5 @@ /* - * Copyright 2026 The LibYuv Project Authors. All rights reserved. + * Copyright 2022 The LibYuv Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -9,14 +9,383 @@ */ #include "libyuv/rotate_row.h" +#include "libyuv/row.h" -#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#if (!defined(LIBYUV_DISABLE_LSX) || !defined(LIBYUV_DISABLE_LASX)) && \ + defined(__loongarch_lp64) #ifdef __cplusplus namespace libyuv { extern "C" { #endif +#ifdef HAS_TRANSPOSEWX8_LSX +void TransposeWx8_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" + + "vilvl.b $vr8, $vr1, $vr0 \n" + "vilvh.b $vr9, $vr1, $vr0 \n" + "vilvl.b $vr10, $vr3, $vr2 \n" + "vilvh.b $vr11, $vr3, $vr2 \n" + "vilvl.b $vr12, $vr5, $vr4 \n" + "vilvh.b $vr13, $vr5, $vr4 \n" + "vilvl.b $vr14, $vr7, $vr6 \n" + "vilvh.b $vr15, $vr7, $vr6 \n" + + "vilvl.h $vr0, $vr10, $vr8 \n" + "vilvh.h $vr1, $vr10, $vr8 \n" + "vilvl.h $vr2, $vr11, $vr9 \n" + "vilvh.h $vr3, $vr11, $vr9 \n" + "vilvl.h $vr4, $vr14, $vr12 \n" + "vilvh.h $vr5, $vr14, $vr12 \n" + "vilvl.h $vr6, $vr15, $vr13 \n" + "vilvh.h $vr7, $vr15, $vr13 \n" + + "vilvl.w $vr8, $vr4, $vr0 \n" + "vilvh.w $vr9, $vr4, $vr0 \n" + "vilvl.w $vr10, $vr5, $vr1 \n" + "vilvh.w $vr11, $vr5, $vr1 \n" + "vilvl.w $vr12, $vr6, $vr2 \n" + "vilvh.w $vr13, $vr6, $vr2 \n" + "vilvl.w $vr14, $vr7, $vr3 \n" + "vilvh.w $vr15, $vr7, $vr3 \n" + + "vstelm.d $vr8, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr8, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr9, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr9, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr10, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr10, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr11, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr11, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr12, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr12, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr13, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr13, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr14, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr14, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr15, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr15, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15"); +} +#endif // HAS_TRANSPOSEWX8_LSX + +#ifdef HAS_TRANSPOSEUVWX8_LSX +void TransposeUVWx8_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t da = dst_stride_a; + const ptrdiff_t db = dst_stride_b; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" + + "vilvl.b $vr8, $vr1, $vr0 \n" + "vilvh.b $vr9, $vr1, $vr0 \n" + "vilvl.b $vr10, $vr3, $vr2 \n" + "vilvh.b $vr11, $vr3, $vr2 \n" + "vilvl.b $vr12, $vr5, $vr4 \n" + "vilvh.b $vr13, $vr5, $vr4 \n" + "vilvl.b $vr14, $vr7, $vr6 \n" + "vilvh.b $vr15, $vr7, $vr6 \n" + + "vilvl.h $vr0, $vr10, $vr8 \n" + "vilvh.h $vr1, $vr10, $vr8 \n" + "vilvl.h $vr2, $vr11, $vr9 \n" + "vilvh.h $vr3, $vr11, $vr9 \n" + "vilvl.h $vr4, $vr14, $vr12 \n" + "vilvh.h $vr5, $vr14, $vr12 \n" + "vilvl.h $vr6, $vr15, $vr13 \n" + "vilvh.h $vr7, $vr15, $vr13 \n" + + "vilvl.w $vr8, $vr4, $vr0 \n" + "vilvh.w $vr9, $vr4, $vr0 \n" + "vilvl.w $vr10, $vr5, $vr1 \n" + "vilvh.w $vr11, $vr5, $vr1 \n" + "vilvl.w $vr12, $vr6, $vr2 \n" + "vilvh.w $vr13, $vr6, $vr2 \n" + "vilvl.w $vr14, $vr7, $vr3 \n" + "vilvh.w $vr15, $vr7, $vr3 \n" + + "vstelm.d $vr8, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr8, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr9, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr9, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr10, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr10, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr11, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr11, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr12, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr12, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr13, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr13, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr14, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr14, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr15, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr15, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst_a] "+r"(dst_a), // %[dst_a] + [dst_b] "+r"(dst_b), // %[dst_b] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [da] "r"(da), // %[da] + [db] "r"(db) // %[db] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15"); +} +#endif // HAS_TRANSPOSEUVWX8_LSX + +#ifdef HAS_TRANSPOSEWX4_32_LSX +void TransposeWx4_32_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t s2 = s * 2; + const ptrdiff_t s3 = s + s2; + const ptrdiff_t s4 = s * 4; + uint8_t* dst0 = dst; + uint8_t* dst1 = dst0 + dst_stride; + uint8_t* dst2 = dst1 + dst_stride; + uint8_t* dst3 = dst2 + dst_stride; + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vldx $vr1, %[src], %[s] \n" + "vldx $vr2, %[src], %[s2] \n" + "vldx $vr3, %[src], %[s3] \n" + "add.d %[src], %[src], %[s4] \n" + + "vilvl.w $vr4, $vr1, $vr0 \n" + "vilvh.w $vr5, $vr1, $vr0 \n" + "vilvl.w $vr6, $vr3, $vr2 \n" + "vilvh.w $vr7, $vr3, $vr2 \n" + + "vilvl.d $vr0, $vr6, $vr4 \n" + "vilvh.d $vr1, $vr6, $vr4 \n" + "vilvl.d $vr2, $vr7, $vr5 \n" + "vilvh.d $vr3, $vr7, $vr5 \n" + + "vst $vr0, %[dst0], 0 \n" + "vst $vr1, %[dst1], 0 \n" + "vst $vr2, %[dst2], 0 \n" + "vst $vr3, %[dst3], 0 \n" + "addi.d %[dst0], %[dst0], 16 \n" + "addi.d %[dst1], %[dst1], 16 \n" + "addi.d %[dst2], %[dst2], 16 \n" + "addi.d %[dst3], %[dst3], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst0] "+r"(dst0), // %[dst0] + [dst1] "+r"(dst1), // %[dst1] + [dst2] "+r"(dst2), // %[dst2] + [dst3] "+r"(dst3), // %[dst3] + [w] "+r"(width) // %[w] + : [s] "r"(s), // %[s] + [s2] "r"(s2), // %[s2] + [s3] "r"(s3), // %[s3] + [s4] "r"(s4) // %[s4] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); +} +#endif // HAS_TRANSPOSEWX4_32_LSX + +#ifdef HAS_TRANSPOSEWX8_16_LSX +void TransposeWx8_16_LSX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" + + "vilvl.h $vr8, $vr1, $vr0 \n" + "vilvh.h $vr9, $vr1, $vr0 \n" + "vilvl.h $vr10, $vr3, $vr2 \n" + "vilvh.h $vr11, $vr3, $vr2 \n" + "vilvl.h $vr12, $vr5, $vr4 \n" + "vilvh.h $vr13, $vr5, $vr4 \n" + "vilvl.h $vr14, $vr7, $vr6 \n" + "vilvh.h $vr15, $vr7, $vr6 \n" + + "vilvl.w $vr0, $vr10, $vr8 \n" + "vilvh.w $vr1, $vr10, $vr8 \n" + "vilvl.w $vr2, $vr11, $vr9 \n" + "vilvh.w $vr3, $vr11, $vr9 \n" + "vilvl.w $vr4, $vr14, $vr12 \n" + "vilvh.w $vr5, $vr14, $vr12 \n" + "vilvl.w $vr6, $vr15, $vr13 \n" + "vilvh.w $vr7, $vr15, $vr13 \n" + + "vilvl.d $vr8, $vr4, $vr0 \n" + "vilvh.d $vr9, $vr4, $vr0 \n" + "vilvl.d $vr10, $vr5, $vr1 \n" + "vilvh.d $vr11, $vr5, $vr1 \n" + "vilvl.d $vr12, $vr6, $vr2 \n" + "vilvh.d $vr13, $vr6, $vr2 \n" + "vilvl.d $vr14, $vr7, $vr3 \n" + "vilvh.d $vr15, $vr7, $vr3 \n" + + "vst $vr8, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr9, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr10, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr11, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr12, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr13, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr14, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr15, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_b), // %[src] + [dst] "+r"(dst_b), // %[dst] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15"); +} +#endif // HAS_TRANSPOSEWX8_16_LSX + +#ifdef HAS_TRANSPOSEWX16_LASX void TransposeWx16_LASX(const uint8_t* src, int src_stride, uint8_t* dst, @@ -161,7 +530,9 @@ void TransposeWx16_LASX(const uint8_t* src, "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15"); } +#endif // HAS_TRANSPOSEWX16_LASX +#ifdef HAS_TRANSPOSEUVWX16_LASX void TransposeUVWx16_LASX(const uint8_t* src, int src_stride, uint8_t* dst_a, @@ -311,7 +682,9 @@ void TransposeUVWx16_LASX(const uint8_t* src, "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15"); } +#endif // HAS_TRANSPOSEUVWX16_LASX +#ifdef HAS_TRANSPOSEWX4_32_LASX void TransposeWx4_32_LASX(const uint8_t* src, int src_stride, uint8_t* dst, @@ -382,7 +755,9 @@ void TransposeWx4_32_LASX(const uint8_t* src, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7"); } +#endif // HAS_TRANSPOSEWX4_32_LASX +#ifdef HAS_TRANSPOSEWX16_16_LASX void TransposeWx16_16_LASX(const uint16_t* src, int src_stride, uint16_t* dst, @@ -497,10 +872,12 @@ void TransposeWx16_16_LASX(const uint16_t* src, "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15"); } +#endif // HAS_TRANSPOSEWX16_16_LASX #ifdef __cplusplus } // extern "C" } // namespace libyuv #endif -#endif // !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#endif // (!defined(LIBYUV_DISABLE_LSX) || !defined(LIBYUV_DISABLE_LASX)) && + // defined(__loongarch_lp64) diff --git a/source/rotate_lsx.cc b/source/rotate_lsx.cc deleted file mode 100644 index 531145bf1..000000000 --- a/source/rotate_lsx.cc +++ /dev/null @@ -1,383 +0,0 @@ -/* - * Copyright 2022 The LibYuv Project Authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "libyuv/rotate_row.h" - -#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) - -#ifdef __cplusplus -namespace libyuv { -extern "C" { -#endif - -void TransposeWx8_LSX(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - const ptrdiff_t s = src_stride; - const ptrdiff_t d = dst_stride; - const uint8_t* t; - asm volatile( - "srli.w %[w], %[w], 4 \n" - "beqz %[w], 2f \n" - "1: \n" - "move %[t], %[src] \n" - "vld $vr0, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr1, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr2, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr3, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr4, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr5, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr6, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr7, %[t], 0 \n" - "addi.d %[src], %[src], 16 \n" - - "vilvl.b $vr8, $vr1, $vr0 \n" - "vilvh.b $vr9, $vr1, $vr0 \n" - "vilvl.b $vr10, $vr3, $vr2 \n" - "vilvh.b $vr11, $vr3, $vr2 \n" - "vilvl.b $vr12, $vr5, $vr4 \n" - "vilvh.b $vr13, $vr5, $vr4 \n" - "vilvl.b $vr14, $vr7, $vr6 \n" - "vilvh.b $vr15, $vr7, $vr6 \n" - - "vilvl.h $vr0, $vr10, $vr8 \n" - "vilvh.h $vr1, $vr10, $vr8 \n" - "vilvl.h $vr2, $vr11, $vr9 \n" - "vilvh.h $vr3, $vr11, $vr9 \n" - "vilvl.h $vr4, $vr14, $vr12 \n" - "vilvh.h $vr5, $vr14, $vr12 \n" - "vilvl.h $vr6, $vr15, $vr13 \n" - "vilvh.h $vr7, $vr15, $vr13 \n" - - "vilvl.w $vr8, $vr4, $vr0 \n" - "vilvh.w $vr9, $vr4, $vr0 \n" - "vilvl.w $vr10, $vr5, $vr1 \n" - "vilvh.w $vr11, $vr5, $vr1 \n" - "vilvl.w $vr12, $vr6, $vr2 \n" - "vilvh.w $vr13, $vr6, $vr2 \n" - "vilvl.w $vr14, $vr7, $vr3 \n" - "vilvh.w $vr15, $vr7, $vr3 \n" - - "vstelm.d $vr8, %[dst], 0, 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr8, %[dst], 0, 1 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr9, %[dst], 0, 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr9, %[dst], 0, 1 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr10, %[dst], 0, 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr10, %[dst], 0, 1 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr11, %[dst], 0, 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr11, %[dst], 0, 1 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr12, %[dst], 0, 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr12, %[dst], 0, 1 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr13, %[dst], 0, 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr13, %[dst], 0, 1 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr14, %[dst], 0, 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr14, %[dst], 0, 1 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr15, %[dst], 0, 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vstelm.d $vr15, %[dst], 0, 1 \n" - "add.d %[dst], %[dst], %[d] \n" - "addi.w %[w], %[w], -1 \n" - "bnez %[w], 1b \n" - "2: \n" - : [src] "+r"(src), // %[src] - [dst] "+r"(dst), // %[dst] - [w] "+r"(width), // %[w] - [t] "=&r"(t) // %[t] - : [s] "r"(s), // %[s] - [d] "r"(d) // %[d] - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", - "$vr15"); -} - -void TransposeUVWx8_LSX(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width) { - const ptrdiff_t s = src_stride; - const ptrdiff_t da = dst_stride_a; - const ptrdiff_t db = dst_stride_b; - const uint8_t* t; - asm volatile( - "srli.w %[w], %[w], 3 \n" - "beqz %[w], 2f \n" - "1: \n" - "move %[t], %[src] \n" - "vld $vr0, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr1, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr2, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr3, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr4, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr5, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr6, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr7, %[t], 0 \n" - "addi.d %[src], %[src], 16 \n" - - "vilvl.b $vr8, $vr1, $vr0 \n" - "vilvh.b $vr9, $vr1, $vr0 \n" - "vilvl.b $vr10, $vr3, $vr2 \n" - "vilvh.b $vr11, $vr3, $vr2 \n" - "vilvl.b $vr12, $vr5, $vr4 \n" - "vilvh.b $vr13, $vr5, $vr4 \n" - "vilvl.b $vr14, $vr7, $vr6 \n" - "vilvh.b $vr15, $vr7, $vr6 \n" - - "vilvl.h $vr0, $vr10, $vr8 \n" - "vilvh.h $vr1, $vr10, $vr8 \n" - "vilvl.h $vr2, $vr11, $vr9 \n" - "vilvh.h $vr3, $vr11, $vr9 \n" - "vilvl.h $vr4, $vr14, $vr12 \n" - "vilvh.h $vr5, $vr14, $vr12 \n" - "vilvl.h $vr6, $vr15, $vr13 \n" - "vilvh.h $vr7, $vr15, $vr13 \n" - - "vilvl.w $vr8, $vr4, $vr0 \n" - "vilvh.w $vr9, $vr4, $vr0 \n" - "vilvl.w $vr10, $vr5, $vr1 \n" - "vilvh.w $vr11, $vr5, $vr1 \n" - "vilvl.w $vr12, $vr6, $vr2 \n" - "vilvh.w $vr13, $vr6, $vr2 \n" - "vilvl.w $vr14, $vr7, $vr3 \n" - "vilvh.w $vr15, $vr7, $vr3 \n" - - "vstelm.d $vr8, %[dst_a], 0, 0 \n" - "add.d %[dst_a], %[dst_a], %[da] \n" - "vstelm.d $vr8, %[dst_b], 0, 1 \n" - "add.d %[dst_b], %[dst_b], %[db] \n" - "vstelm.d $vr9, %[dst_a], 0, 0 \n" - "add.d %[dst_a], %[dst_a], %[da] \n" - "vstelm.d $vr9, %[dst_b], 0, 1 \n" - "add.d %[dst_b], %[dst_b], %[db] \n" - "vstelm.d $vr10, %[dst_a], 0, 0 \n" - "add.d %[dst_a], %[dst_a], %[da] \n" - "vstelm.d $vr10, %[dst_b], 0, 1 \n" - "add.d %[dst_b], %[dst_b], %[db] \n" - "vstelm.d $vr11, %[dst_a], 0, 0 \n" - "add.d %[dst_a], %[dst_a], %[da] \n" - "vstelm.d $vr11, %[dst_b], 0, 1 \n" - "add.d %[dst_b], %[dst_b], %[db] \n" - "vstelm.d $vr12, %[dst_a], 0, 0 \n" - "add.d %[dst_a], %[dst_a], %[da] \n" - "vstelm.d $vr12, %[dst_b], 0, 1 \n" - "add.d %[dst_b], %[dst_b], %[db] \n" - "vstelm.d $vr13, %[dst_a], 0, 0 \n" - "add.d %[dst_a], %[dst_a], %[da] \n" - "vstelm.d $vr13, %[dst_b], 0, 1 \n" - "add.d %[dst_b], %[dst_b], %[db] \n" - "vstelm.d $vr14, %[dst_a], 0, 0 \n" - "add.d %[dst_a], %[dst_a], %[da] \n" - "vstelm.d $vr14, %[dst_b], 0, 1 \n" - "add.d %[dst_b], %[dst_b], %[db] \n" - "vstelm.d $vr15, %[dst_a], 0, 0 \n" - "add.d %[dst_a], %[dst_a], %[da] \n" - "vstelm.d $vr15, %[dst_b], 0, 1 \n" - "add.d %[dst_b], %[dst_b], %[db] \n" - "addi.w %[w], %[w], -1 \n" - "bnez %[w], 1b \n" - "2: \n" - : [src] "+r"(src), // %[src] - [dst_a] "+r"(dst_a), // %[dst_a] - [dst_b] "+r"(dst_b), // %[dst_b] - [w] "+r"(width), // %[w] - [t] "=&r"(t) // %[t] - : [s] "r"(s), // %[s] - [da] "r"(da), // %[da] - [db] "r"(db) // %[db] - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", - "$vr15"); -} - -void TransposeWx4_32_LSX(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - const ptrdiff_t s = src_stride; - const ptrdiff_t s2 = s * 2; - const ptrdiff_t s3 = s + s2; - const ptrdiff_t s4 = s * 4; - uint8_t* dst0 = dst; - uint8_t* dst1 = dst0 + dst_stride; - uint8_t* dst2 = dst1 + dst_stride; - uint8_t* dst3 = dst2 + dst_stride; - asm volatile( - "srli.w %[w], %[w], 2 \n" - "beqz %[w], 2f \n" - "1: \n" - "vld $vr0, %[src], 0 \n" - "vldx $vr1, %[src], %[s] \n" - "vldx $vr2, %[src], %[s2] \n" - "vldx $vr3, %[src], %[s3] \n" - "add.d %[src], %[src], %[s4] \n" - - "vilvl.w $vr4, $vr1, $vr0 \n" - "vilvh.w $vr5, $vr1, $vr0 \n" - "vilvl.w $vr6, $vr3, $vr2 \n" - "vilvh.w $vr7, $vr3, $vr2 \n" - - "vilvl.d $vr0, $vr6, $vr4 \n" - "vilvh.d $vr1, $vr6, $vr4 \n" - "vilvl.d $vr2, $vr7, $vr5 \n" - "vilvh.d $vr3, $vr7, $vr5 \n" - - "vst $vr0, %[dst0], 0 \n" - "vst $vr1, %[dst1], 0 \n" - "vst $vr2, %[dst2], 0 \n" - "vst $vr3, %[dst3], 0 \n" - "addi.d %[dst0], %[dst0], 16 \n" - "addi.d %[dst1], %[dst1], 16 \n" - "addi.d %[dst2], %[dst2], 16 \n" - "addi.d %[dst3], %[dst3], 16 \n" - "addi.w %[w], %[w], -1 \n" - "bnez %[w], 1b \n" - "2: \n" - : [src] "+r"(src), // %[src] - [dst0] "+r"(dst0), // %[dst0] - [dst1] "+r"(dst1), // %[dst1] - [dst2] "+r"(dst2), // %[dst2] - [dst3] "+r"(dst3), // %[dst3] - [w] "+r"(width) // %[w] - : [s] "r"(s), // %[s] - [s2] "r"(s2), // %[s2] - [s3] "r"(s3), // %[s3] - [s4] "r"(s4) // %[s4] - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7"); -} - -void TransposeWx8_16_LSX(const uint16_t* src, - int src_stride, - uint16_t* dst, - int dst_stride, - int width) { - const ptrdiff_t s = (ptrdiff_t)src_stride * 2; - const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; - const uint8_t* src_b = (const uint8_t*)src; - uint8_t* dst_b = (uint8_t*)dst; - const uint8_t* t; - asm volatile( - "srli.w %[w], %[w], 3 \n" - "beqz %[w], 2f \n" - "1: \n" - "move %[t], %[src] \n" - "vld $vr0, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr1, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr2, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr3, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr4, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr5, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr6, %[t], 0 \n" - "add.d %[t], %[t], %[s] \n" - "vld $vr7, %[t], 0 \n" - "addi.d %[src], %[src], 16 \n" - - "vilvl.h $vr8, $vr1, $vr0 \n" - "vilvh.h $vr9, $vr1, $vr0 \n" - "vilvl.h $vr10, $vr3, $vr2 \n" - "vilvh.h $vr11, $vr3, $vr2 \n" - "vilvl.h $vr12, $vr5, $vr4 \n" - "vilvh.h $vr13, $vr5, $vr4 \n" - "vilvl.h $vr14, $vr7, $vr6 \n" - "vilvh.h $vr15, $vr7, $vr6 \n" - - "vilvl.w $vr0, $vr10, $vr8 \n" - "vilvh.w $vr1, $vr10, $vr8 \n" - "vilvl.w $vr2, $vr11, $vr9 \n" - "vilvh.w $vr3, $vr11, $vr9 \n" - "vilvl.w $vr4, $vr14, $vr12 \n" - "vilvh.w $vr5, $vr14, $vr12 \n" - "vilvl.w $vr6, $vr15, $vr13 \n" - "vilvh.w $vr7, $vr15, $vr13 \n" - - "vilvl.d $vr8, $vr4, $vr0 \n" - "vilvh.d $vr9, $vr4, $vr0 \n" - "vilvl.d $vr10, $vr5, $vr1 \n" - "vilvh.d $vr11, $vr5, $vr1 \n" - "vilvl.d $vr12, $vr6, $vr2 \n" - "vilvh.d $vr13, $vr6, $vr2 \n" - "vilvl.d $vr14, $vr7, $vr3 \n" - "vilvh.d $vr15, $vr7, $vr3 \n" - - "vst $vr8, %[dst], 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vst $vr9, %[dst], 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vst $vr10, %[dst], 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vst $vr11, %[dst], 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vst $vr12, %[dst], 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vst $vr13, %[dst], 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vst $vr14, %[dst], 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "vst $vr15, %[dst], 0 \n" - "add.d %[dst], %[dst], %[d] \n" - "addi.w %[w], %[w], -1 \n" - "bnez %[w], 1b \n" - "2: \n" - : [src] "+r"(src_b), // %[src] - [dst] "+r"(dst_b), // %[dst] - [w] "+r"(width), // %[w] - [t] "=&r"(t) // %[t] - : [s] "r"(s), // %[s] - [d] "r"(d) // %[d] - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", - "$vr15"); -} - -#ifdef __cplusplus -} // extern "C" -} // namespace libyuv -#endif - -#endif // !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) diff --git a/source/row_lasx.cc b/source/row_la.cc similarity index 51% rename from source/row_lasx.cc rename to source/row_la.cc index 12d267ad2..88c5b4905 100644 --- a/source/row_lasx.cc +++ b/source/row_la.cc @@ -8,9 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "libyuv/compare_row.h" +#include "libyuv/basic_types.h" #include "libyuv/row.h" -#include "libyuv/video_common.h" // Note: every register written by an inline asm block (vector, general // purpose, mask/predicate, "cc" for flags and "memory" for stores) must be @@ -19,15 +18,2820 @@ // asm statement, and other OS ABIs (e.g. Windows) differ in which registers // are callee-saved. List "memory" first, then "cc", then registers. -#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) - -#include "libyuv/basic_types.h" +#if (!defined(LIBYUV_DISABLE_LSX) || !defined(LIBYUV_DISABLE_LASX)) && \ + defined(__loongarch_lp64) #ifdef __cplusplus namespace libyuv { extern "C" { #endif +#ifdef HAS_COPYROW_LSX +void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int count) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(count) + : + : "memory", "$vr0"); +} +#endif // HAS_COPYROW_LSX + +#ifdef HAS_SETROW_LSX +void SetRow_LSX(uint8_t* dst, uint8_t v8, int count) { + asm volatile( + "vreplgr2vr.b $vr0, %[val]\n\t" + "1:\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst), [cnt] "+r"(count) + : [val] "r"(v8) + : "memory", "$vr0"); +} +#endif // HAS_SETROW_LSX + +#ifdef HAS_ARGBSETROW_LSX +void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width) { + asm volatile( + "vreplgr2vr.w $vr0, %[val]\n\t" + "1:\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(v32) + : "memory", "$vr0"); +} +#endif // HAS_ARGBSETROW_LSX + +#ifdef HAS_MIRRORROW_LSX +void MirrorRow_LSX(const uint8_t* src_y, uint8_t* dst_y, int width) { + static const uint8_t kRev[16] = {15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0}; + src_y += width - 16; + asm volatile( + "vld $vr1, %[rev], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], -16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [rev] "r"(kRev) + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_MIRRORROW_LSX + +#ifdef HAS_MIRRORUVROW_LSX +void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1}; + src_uv += width * 2 - 16; + asm volatile( + "vld $vr1, %[rev], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], -16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uv), [dst] "+r"(dst_uv), [cnt] "+r"(width) + : [rev] "r"(kRevUV) + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_MIRRORUVROW_LSX + +#ifdef HAS_ARGBMIRRORROW_LSX +void ARGBMirrorRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + src_argb += width * 4 - 16; + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf4i.w $vr0, $vr0, 0x1b\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], -16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0"); +} +#endif // HAS_ARGBMIRRORROW_LSX + +#ifdef HAS_MIRRORSPLITUVROW_LSX +void MirrorSplitUVRow_LSX(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1}; + src_uv += width * 2 - 32; + asm volatile( + "vld $vr4, %[rev], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 16\n\t" + "vld $vr1, %[src], 0\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" + "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vst $vr2, %[du], 0\n\t" + "vst $vr3, %[dv], 0\n\t" + "addi.d %[src], %[src], -32\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : + [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width) + : [rev] "r"(kRevUV) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_MIRRORSPLITUVROW_LSX + +#ifdef HAS_SPLITUVROW_LSX +void SplitUVRow_LSX(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vst $vr2, %[du], 0\n\t" + "vst $vr3, %[dv], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : + [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_SPLITUVROW_LSX + +#ifdef HAS_MERGEUVROW_LSX +void MergeUVRow_LSX(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[su], 0\n\t" + "vld $vr1, %[sv], 0\n\t" + "vilvl.b $vr2, $vr1, $vr0\n\t" + "vilvh.b $vr3, $vr1, $vr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "vst $vr3, %[dst], 16\n\t" + "addi.d %[su], %[su], 16\n\t" + "addi.d %[sv], %[sv], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : + [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_MERGEUVROW_LSX + +#ifdef HAS_ARGBEXTRACTALPHAROW_LSX +void ARGBExtractAlphaRow_LSX(const uint8_t* src_argb, + uint8_t* dst_a, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickod.b $vr0, $vr1, $vr0\n\t" + "vpickod.b $vr0, $vr0, $vr0\n\t" + "vstelm.d $vr0, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_a), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_ARGBEXTRACTALPHAROW_LSX + +#ifdef HAS_I422TOYUY2ROW_LSX +void I422ToYUY2Row_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_yuy2, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[su], 0\n\t" + "vldrepl.d $vr2, %[sv], 0\n\t" + "vilvl.b $vr3, $vr2, $vr1\n\t" + "vilvl.b $vr4, $vr3, $vr0\n\t" + "vilvh.b $vr5, $vr3, $vr0\n\t" + "vst $vr4, %[dst], 0\n\t" + "vst $vr5, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_yuy2), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); +} +#endif // HAS_I422TOYUY2ROW_LSX + +#ifdef HAS_I422TOUYVYROW_LSX +void I422ToUYVYRow_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uyvy, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[su], 0\n\t" + "vldrepl.d $vr2, %[sv], 0\n\t" + "vilvl.b $vr3, $vr2, $vr1\n\t" + "vilvl.b $vr4, $vr0, $vr3\n\t" + "vilvh.b $vr5, $vr0, $vr3\n\t" + "vst $vr4, %[dst], 0\n\t" + "vst $vr5, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_uyvy), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); +} +#endif // HAS_I422TOUYVYROW_LSX + +#ifdef HAS_YUY2TOYROW_LSX +void YUY2ToYRow_LSX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_yuy2), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2"); +} +#endif // HAS_YUY2TOYROW_LSX + +#ifdef HAS_UYVYTOYROW_LSX +void UYVYToYRow_LSX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickod.b $vr2, $vr1, $vr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uyvy), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2"); +} +#endif // HAS_UYVYTOYROW_LSX + +#ifdef HAS_YUY2TOUV422ROW_LSX +void YUY2ToUV422Row_LSX(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickod.b $vr2, $vr1, $vr0\n\t" + "vpickev.b $vr3, $vr2, $vr2\n\t" + "vpickod.b $vr4, $vr2, $vr2\n\t" + "vstelm.d $vr3, %[du], 0, 0\n\t" + "vstelm.d $vr4, %[dv], 0, 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_yuy2), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_YUY2TOUV422ROW_LSX + +#ifdef HAS_UYVYTOUV422ROW_LSX +void UYVYToUV422Row_LSX(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickev.b $vr3, $vr2, $vr2\n\t" + "vpickod.b $vr4, $vr2, $vr2\n\t" + "vstelm.d $vr3, %[du], 0, 0\n\t" + "vstelm.d $vr4, %[dv], 0, 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uyvy), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_UYVYTOUV422ROW_LSX + +#ifdef HAS_YUY2TOUVROW_LSX +void YUY2ToUVRow_LSX(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_yuy2 = src_yuy2 + src_stride_yuy2; + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s0], 16\n\t" + "vld $vr2, %[s1], 0\n\t" + "vld $vr3, %[s1], 16\n\t" + "vpickod.b $vr0, $vr1, $vr0\n\t" + "vpickod.b $vr2, $vr3, $vr2\n\t" + "vavgr.bu $vr0, $vr0, $vr2\n\t" + "vpickev.b $vr1, $vr0, $vr0\n\t" + "vpickod.b $vr2, $vr0, $vr0\n\t" + "vstelm.d $vr1, %[du], 0, 0\n\t" + "vstelm.d $vr2, %[dv], 0, 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_yuy2), [s1] "+r"(next_yuy2), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_YUY2TOUVROW_LSX + +#ifdef HAS_UYVYTOUVROW_LSX +void UYVYToUVRow_LSX(const uint8_t* src_uyvy, + int src_stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_uyvy = src_uyvy + src_stride_uyvy; + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s0], 16\n\t" + "vld $vr2, %[s1], 0\n\t" + "vld $vr3, %[s1], 16\n\t" + "vpickev.b $vr0, $vr1, $vr0\n\t" + "vpickev.b $vr2, $vr3, $vr2\n\t" + "vavgr.bu $vr0, $vr0, $vr2\n\t" + "vpickev.b $vr1, $vr0, $vr0\n\t" + "vpickod.b $vr2, $vr0, $vr0\n\t" + "vstelm.d $vr1, %[du], 0, 0\n\t" + "vstelm.d $vr2, %[dv], 0, 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_uyvy), [s1] "+r"(next_uyvy), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_UYVYTOUVROW_LSX + +#ifdef HAS_I422TOARGBROW_LSX +void I422ToARGBRow_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", + "$vr23", "$vr24"); +} +#endif // HAS_I422TOARGBROW_LSX + +#ifdef HAS_I422TORGBAROW_LSX +void I422ToRGBARow_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr2, $vr5\n\t" + "vilvl.b $vr7, $vr4, $vr3\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", + "$vr23", "$vr24"); +} +#endif // HAS_I422TORGBAROW_LSX + +#ifdef HAS_I422ALPHATOARGBROW_LSX +void I422AlphaToARGBRow_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + const uint8_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vldrepl.d $vr5, %[sa], 0\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[sa], %[sa], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width), [sa] "+r"(src_a) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", + "$vr23", "$vr24"); +} +#endif // HAS_I422ALPHATOARGBROW_LSX + +#ifdef HAS_I422TORGB24ROW_LSX +void I422ToRGB24Row_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + static const uint8_t kShuf0[16] = {0, 1, 2, 4, 5, 6, 8, 9, + 10, 12, 13, 14, 16, 17, 18, 20}; + static const uint8_t kShuf1[16] = {21, 22, 24, 25, 26, 28, 29, 30, + 0, 0, 0, 0, 0, 0, 0, 0}; + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vld $vr12, %[s0], 0\n\t" + "vld $vr13, %[s1], 0\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr4, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vshuf.b $vr10, $vr9, $vr8, $vr12\n\t" + "vshuf.b $vr11, $vr9, $vr8, $vr13\n\t" + "vst $vr10, %[dst], 0\n\t" + "vstelm.d $vr11, %[dst], 16, 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 24\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants), [s0] "r"(kShuf0), [s1] "r"(kShuf1) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7", + "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr20", "$vr21", + "$vr22", "$vr23", "$vr24"); +} +#endif // HAS_I422TORGB24ROW_LSX + +#ifdef HAS_I422TORGB565ROW_LSX +void I422ToRGB565Row_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr14, 31\n\t" + "vsrli.h $vr14, $vr14, 8\n\t" + "vldi $vr15, 63\n\t" + "vsrli.h $vr15, $vr15, 8\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 2\n\t" + "vsrli.b $vr4, $vr4, 3\n\t" + "vilvl.b $vr2, $vr2, $vr2\n\t" + "vilvl.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr4, $vr4, $vr4\n\t" + "vand.v $vr2, $vr2, $vr14\n\t" + "vand.v $vr3, $vr3, $vr15\n\t" + "vand.v $vr4, $vr4, $vr14\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr4, $vr4, 11\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11", + "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); +} +#endif // HAS_I422TORGB565ROW_LSX + +#ifdef HAS_I422TOARGB4444ROW_LSX +void I422ToARGB4444Row_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr14, 15\n\t" + "vsrli.h $vr14, $vr14, 8\n\t" + "vldi $vr15, 240\n\t" + "vsrli.h $vr15, $vr15, 8\n\t" + "vslli.h $vr15, $vr15, 8\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vsrli.b $vr2, $vr2, 4\n\t" + "vsrli.b $vr3, $vr3, 4\n\t" + "vsrli.b $vr4, $vr4, 4\n\t" + "vilvl.b $vr2, $vr2, $vr2\n\t" + "vilvl.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr4, $vr4, $vr4\n\t" + "vand.v $vr2, $vr2, $vr14\n\t" + "vand.v $vr3, $vr3, $vr14\n\t" + "vand.v $vr4, $vr4, $vr14\n\t" + "vslli.h $vr3, $vr3, 4\n\t" + "vslli.h $vr4, $vr4, 8\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vor.v $vr2, $vr2, $vr15\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11", + "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); +} +#endif // HAS_I422TOARGB4444ROW_LSX + +#ifdef HAS_I422TOARGB1555ROW_LSX +void I422ToARGB1555Row_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr14, 31\n\t" + "vsrli.h $vr14, $vr14, 8\n\t" + "vldi $vr15, 128\n\t" + "vsrli.h $vr15, $vr15, 8\n\t" + "vslli.h $vr15, $vr15, 8\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 3\n\t" + "vsrli.b $vr4, $vr4, 3\n\t" + "vilvl.b $vr2, $vr2, $vr2\n\t" + "vilvl.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr4, $vr4, $vr4\n\t" + "vand.v $vr2, $vr2, $vr14\n\t" + "vand.v $vr3, $vr3, $vr14\n\t" + "vand.v $vr4, $vr4, $vr14\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr4, $vr4, 10\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vor.v $vr2, $vr2, $vr15\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11", + "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); +} +#endif // HAS_I422TOARGB1555ROW_LSX + +#ifdef HAS_NV12TOARGBROW_LSX +void NV12ToARGBRow_LSX(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[suv], 0\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[suv], %[suv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); +} +#endif // HAS_NV12TOARGBROW_LSX + +#ifdef HAS_NV21TOARGBROW_LSX +void NV21ToARGBRow_LSX(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[suv], 0\n\t" + "vshuf4i.b $vr1, $vr1, 0xb1\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[suv], %[suv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [suv] "+r"(src_vu), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); +} +#endif // HAS_NV21TOARGBROW_LSX + +#ifdef HAS_NV12TORGB565ROW_LSX +void NV12ToRGB565Row_LSX(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr14, 31\n\t" + "vsrli.h $vr14, $vr14, 8\n\t" + "vldi $vr15, 63\n\t" + "vsrli.h $vr15, $vr15, 8\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[suv], 0\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 2\n\t" + "vsrli.b $vr4, $vr4, 3\n\t" + "vilvl.b $vr2, $vr2, $vr2\n\t" + "vilvl.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr4, $vr4, $vr4\n\t" + "vand.v $vr2, $vr2, $vr14\n\t" + "vand.v $vr3, $vr3, $vr15\n\t" + "vand.v $vr4, $vr4, $vr14\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr4, $vr4, 11\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[suv], %[suv], 8\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr14", "$vr15", + "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); +} +#endif // HAS_NV12TORGB565ROW_LSX + +#ifdef HAS_I444TOARGBROW_LSX +void I444ToARGBRow_LSX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.d $vr10, %[su], 0\n\t" + "vldrepl.d $vr11, %[sv], 0\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", + "$vr23", "$vr24"); +} +#endif // HAS_I444TOARGBROW_LSX + +#ifdef HAS_I400TOARGBROW_LSX +void I400ToARGBRow_LSX(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vld $vr23, %[c], 96\n\t" + "vld $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vssrani.bu.h $vr2, $vr0, 6\n\t" + "vilvl.b $vr6, $vr2, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr2\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr2", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9", + "$vr23", "$vr24"); +} +#endif // HAS_I400TOARGBROW_LSX + +#ifdef HAS_J400TOARGBROW_LSX +void J400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, int width) { + asm volatile( + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr2, %[sy], 0\n\t" + "vilvl.b $vr6, $vr2, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr2\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr2", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9"); +} +#endif // HAS_J400TOARGBROW_LSX + +#ifdef HAS_YUY2TOARGBROW_LSX +void YUY2ToARGBRow_LSX(const uint8_t* src_yuy2, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vld $vr10, %[src], 0\n\t" + "vpickev.b $vr0, $vr10, $vr10\n\t" + "vpickod.b $vr1, $vr10, $vr10\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_yuy2), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr20", "$vr21", "$vr22", "$vr23", + "$vr24"); +} +#endif // HAS_YUY2TOARGBROW_LSX + +#ifdef HAS_UYVYTOARGBROW_LSX +void UYVYToARGBRow_LSX(const uint8_t* src_uyvy, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vld $vr10, %[src], 0\n\t" + "vpickod.b $vr0, $vr10, $vr10\n\t" + "vpickev.b $vr1, $vr10, $vr10\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uyvy), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr20", "$vr21", "$vr22", "$vr23", + "$vr24"); +} +#endif // HAS_UYVYTOARGBROW_LSX + +#if defined(HAS_ARGBTORGB24ROW_LSX) || defined(HAS_ARGBTORAWROW_LSX) +static LIBYUV_NOINLINE void ARGBTo24Helper_LSX(const uint8_t* src_argb, + uint8_t* dst_24, + const uint8_t* shuf, + int width) { + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr2, %[src], 32\n\t" + "vld $vr3, %[src], 48\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" + "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t" + "vshuf.b $vr2, $vr2, $vr2, $vr4\n\t" + "vshuf.b $vr3, $vr3, $vr3, $vr4\n\t" + "vstelm.d $vr0, %[dst], 0, 0\n\t" + "vstelm.w $vr0, %[dst], 8, 2\n\t" + "vstelm.w $vr1, %[dst], 12, 0\n\t" + "vstelm.w $vr1, %[dst], 16, 1\n\t" + "vstelm.w $vr1, %[dst], 20, 2\n\t" + "vstelm.d $vr2, %[dst], 24, 0\n\t" + "vstelm.w $vr2, %[dst], 32, 2\n\t" + "vstelm.w $vr3, %[dst], 36, 0\n\t" + "vstelm.w $vr3, %[dst], 40, 1\n\t" + "vstelm.w $vr3, %[dst], 44, 2\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 48\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_24), [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // defined(HAS_ARGBTORGB24ROW_LSX) || defined(HAS_ARGBTORAWROW_LSX) + +#ifdef HAS_ARGBTORGB24ROW_LSX +void ARGBToRGB24Row_LSX(const uint8_t* src_argb, + uint8_t* dst_rgb24, + int width) { + static const uint8_t kShuf[16] = {0, 1, 2, 4, 5, 6, 8, 9, + 10, 12, 13, 14, 0, 0, 0, 0}; + ARGBTo24Helper_LSX(src_argb, dst_rgb24, kShuf, width); +} +#endif // HAS_ARGBTORGB24ROW_LSX + +#ifdef HAS_ARGBTORAWROW_LSX +void ARGBToRAWRow_LSX(const uint8_t* src_argb, uint8_t* dst_raw, int width) { + static const uint8_t kShuf[16] = {2, 1, 0, 6, 5, 4, 10, 9, + 8, 14, 13, 12, 0, 0, 0, 0}; + ARGBTo24Helper_LSX(src_argb, dst_raw, kShuf, width); +} +#endif // HAS_ARGBTORAWROW_LSX + +#if defined(HAS_RGB24TOARGBROW_LSX) || defined(HAS_RAWTOARGBROW_LSX) +static LIBYUV_NOINLINE void RGB24ToARGBHelper_LSX(const uint8_t* src_24, + uint8_t* dst_argb, + const uint8_t* shuf, + int width) { + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "vld $vr5, %[shuf], 16\n\t" + "vldi $vr6, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 12\n\t" + "vld $vr2, %[src], 24\n\t" + "vld $vr3, %[src], 32\n\t" + "vshuf.b $vr0, $vr6, $vr0, $vr4\n\t" + "vshuf.b $vr1, $vr6, $vr1, $vr4\n\t" + "vshuf.b $vr2, $vr6, $vr2, $vr4\n\t" + "vshuf.b $vr3, $vr6, $vr3, $vr5\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "vst $vr2, %[dst], 32\n\t" + "vst $vr3, %[dst], 48\n\t" + "addi.d %[src], %[src], 48\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_24), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} +#endif // defined(HAS_RGB24TOARGBROW_LSX) || defined(HAS_RAWTOARGBROW_LSX) + +#ifdef HAS_RGB24TOARGBROW_LSX +void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24, + uint8_t* dst_argb, + int width) { + static const uint8_t kShuf[32] = {0, 1, 2, 16, 3, 4, 5, 16, 6, 7, 8, + 16, 9, 10, 11, 16, 4, 5, 6, 16, 7, 8, + 9, 16, 10, 11, 12, 16, 13, 14, 15, 16}; + RGB24ToARGBHelper_LSX(src_rgb24, dst_argb, kShuf, width); +} +#endif // HAS_RGB24TOARGBROW_LSX + +#ifdef HAS_RAWTOARGBROW_LSX +void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { + static const uint8_t kShuf[32] = {2, 1, 0, 16, 5, 4, 3, 16, 8, 7, 6, + 16, 11, 10, 9, 16, 6, 5, 4, 16, 9, 8, + 7, 16, 12, 11, 10, 16, 15, 14, 13, 16}; + RGB24ToARGBHelper_LSX(src_raw, dst_argb, kShuf, width); +} +#endif // HAS_RAWTOARGBROW_LSX + +#ifdef HAS_RAWTORGB24ROW_LSX +void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { + static const uint8_t kShuf[32] = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, + 9, 0, 0, 0, 0, 6, 5, 4, 9, 8, 7, + 12, 11, 10, 15, 14, 13, 0, 0, 0, 0}; + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "vld $vr5, %[shuf], 16\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 12\n\t" + "vld $vr2, %[src], 24\n\t" + "vld $vr3, %[src], 32\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" + "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t" + "vshuf.b $vr2, $vr2, $vr2, $vr4\n\t" + "vshuf.b $vr3, $vr3, $vr3, $vr5\n\t" + "vstelm.d $vr0, %[dst], 0, 0\n\t" + "vstelm.w $vr0, %[dst], 8, 2\n\t" + "vstelm.w $vr1, %[dst], 12, 0\n\t" + "vstelm.w $vr1, %[dst], 16, 1\n\t" + "vstelm.w $vr1, %[dst], 20, 2\n\t" + "vstelm.d $vr2, %[dst], 24, 0\n\t" + "vstelm.w $vr2, %[dst], 32, 2\n\t" + "vstelm.w $vr3, %[dst], 36, 0\n\t" + "vstelm.w $vr3, %[dst], 40, 1\n\t" + "vstelm.w $vr3, %[dst], 44, 2\n\t" + "addi.d %[src], %[src], 48\n\t" + "addi.d %[dst], %[dst], 48\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_raw), [dst] "+r"(dst_rgb24), [cnt] "+r"(width) + : [shuf] "r"(kShuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); +} +#endif // HAS_RAWTORGB24ROW_LSX + +#ifdef HAS_ARGBTORGB565ROW_LSX +void ARGBToRGB565Row_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 2\n\t" + "vand.v $vr4, $vr2, $vr6\n\t" + "vsrli.h $vr2, $vr2, 8\n\t" + "vand.v $vr3, $vr3, $vr7\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr2, $vr2, 11\n\t" + "vor.v $vr4, $vr4, $vr3\n\t" + "vor.v $vr4, $vr4, $vr2\n\t" + "vst $vr4, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7"); +} +#endif // HAS_ARGBTORGB565ROW_LSX + +#ifdef HAS_ARGBTORGB565DITHERROW_LSX +void ARGBToRGB565DitherRow_LSX(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width) { + asm volatile( + "vreplgr2vr.w $vr5, %[dith]\n\t" + "vilvl.b $vr5, $vr5, $vr5\n\t" + "vilvl.h $vr5, $vr5, $vr5\n\t" + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vsadd.bu $vr0, $vr0, $vr5\n\t" + "vpickev.b $vr2, $vr0, $vr0\n\t" + "vpickod.b $vr3, $vr0, $vr0\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 2\n\t" + "vand.v $vr4, $vr2, $vr6\n\t" + "vsrli.h $vr2, $vr2, 8\n\t" + "vand.v $vr3, $vr3, $vr7\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr2, $vr2, 11\n\t" + "vor.v $vr4, $vr4, $vr3\n\t" + "vor.v $vr4, $vr4, $vr2\n\t" + "vstelm.d $vr4, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : [dith] "r"(dither4) + : "memory", "$vr0", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7"); +} +#endif // HAS_ARGBTORGB565DITHERROW_LSX + +#ifdef HAS_ARGBTOARGB1555ROW_LSX +void ARGBToARGB1555Row_LSX(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vand.v $vr4, $vr2, $vr6\n\t" + "vsrli.h $vr2, $vr2, 8\n\t" + "vsrli.h $vr5, $vr3, 15\n\t" + "vsrli.b $vr3, $vr3, 3\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr2, $vr2, 10\n\t" + "vslli.h $vr5, $vr5, 15\n\t" + "vor.v $vr4, $vr4, $vr3\n\t" + "vor.v $vr4, $vr4, $vr2\n\t" + "vor.v $vr4, $vr4, $vr5\n\t" + "vst $vr4, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} +#endif // HAS_ARGBTOARGB1555ROW_LSX + +#ifdef HAS_ARGBTOARGB4444ROW_LSX +void ARGBToARGB4444Row_LSX(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vsrli.b $vr2, $vr2, 4\n\t" + "vsrli.b $vr3, $vr3, 4\n\t" + "vslli.b $vr3, $vr3, 4\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_ARGBTOARGB4444ROW_LSX + +#ifdef HAS_RGB565TOARGBROW_LSX +void RGB565ToARGBRow_LSX(const uint8_t* src_rgb565, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr1, $vr1, $vr1\n\t" + "vpickev.b $vr2, $vr2, $vr2\n\t" + "vpickev.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr8, $vr2, $vr1\n\t" + "vilvl.b $vr9, $vr5, $vr3\n\t" + "vilvl.h $vr10, $vr9, $vr8\n\t" + "vilvh.h $vr11, $vr9, $vr8\n\t" + "vst $vr10, %[dst], 0\n\t" + "vst $vr11, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgb565), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11"); +} +#endif // HAS_RGB565TOARGBROW_LSX + +#ifdef HAS_ARGB1555TOARGBROW_LSX +void ARGB1555ToARGBRow_LSX(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vsrai.h $vr5, $vr0, 15\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr1, $vr1, $vr1\n\t" + "vpickev.b $vr2, $vr2, $vr2\n\t" + "vpickev.b $vr3, $vr3, $vr3\n\t" + "vpickev.b $vr5, $vr5, $vr5\n\t" + "vilvl.b $vr8, $vr2, $vr1\n\t" + "vilvl.b $vr9, $vr5, $vr3\n\t" + "vilvl.h $vr10, $vr9, $vr8\n\t" + "vilvh.h $vr11, $vr9, $vr8\n\t" + "vst $vr10, %[dst], 0\n\t" + "vst $vr11, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb1555), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr8", "$vr9", "$vr10", "$vr11"); +} +#endif // HAS_ARGB1555TOARGBROW_LSX + +#ifdef HAS_ARGB4444TOARGBROW_LSX +void ARGB4444ToARGBRow_LSX(const uint8_t* src_argb4444, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr6, 0x0f\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.b $vr2, $vr0, 4\n\t" + "vslli.b $vr3, $vr1, 4\n\t" + "vor.v $vr1, $vr1, $vr3\n\t" + "vslli.b $vr3, $vr2, 4\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vilvl.b $vr4, $vr2, $vr1\n\t" + "vilvh.b $vr5, $vr2, $vr1\n\t" + "vst $vr4, %[dst], 0\n\t" + "vst $vr5, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb4444), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} +#endif // HAS_ARGB4444TOARGBROW_LSX + +#ifdef HAS_ARGBMULTIPLYROW_LSX +void ARGBMultiplyRow_LSX(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vmulwev.h.bu $vr2, $vr0, $vr1\n\t" + "vmulwod.h.bu $vr3, $vr0, $vr1\n\t" + "vsrari.h $vr2, $vr2, 8\n\t" + "vsrari.h $vr3, $vr3, 8\n\t" + "vpackev.b $vr2, $vr3, $vr2\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} +#endif // HAS_ARGBMULTIPLYROW_LSX + +#ifdef HAS_ARGBADDROW_LSX +void ARGBAddRow_LSX(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vsadd.bu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_ARGBADDROW_LSX + +#ifdef HAS_ARGBSUBTRACTROW_LSX +void ARGBSubtractRow_LSX(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vssub.bu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_ARGBSUBTRACTROW_LSX + +#ifdef HAS_ARGBATTENUATEROW_LSX +void ARGBAttenuateRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const uint8_t kAlpha[16] = {3, 3, 3, 16, 7, 7, 7, 16, + 11, 11, 11, 16, 15, 15, 15, 16}; + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "vldi $vr5, 0xff\n\t" + "vldi $vr6, 255\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf.b $vr1, $vr5, $vr0, $vr4\n\t" + "vmulwev.h.bu $vr2, $vr0, $vr1\n\t" + "vmulwod.h.bu $vr3, $vr0, $vr1\n\t" + "vadd.h $vr2, $vr2, $vr6\n\t" + "vadd.h $vr3, $vr3, $vr6\n\t" + "vpackod.b $vr2, $vr3, $vr2\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [shuf] "r"(kAlpha) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} +#endif // HAS_ARGBATTENUATEROW_LSX + +#ifdef HAS_ARGBBLENDROW_LSX +void ARGBBlendRow_LSX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + static const uint8_t kAlpha[16] = {3, 3, 3, 3, 7, 7, 7, 7, + 11, 11, 11, 11, 15, 15, 15, 15}; + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "vldi $vr5, 0xff\n\t" + "vslli.w $vr5, $vr5, 24\n\t" + "vldi $vr6, 1\n\t" + "vslli.h $vr6, $vr6, 8\n\t" + "vldi $vr8, 0\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vshuf.b $vr2, $vr0, $vr0, $vr4\n\t" + "vilvl.b $vr3, $vr8, $vr2\n\t" + "vilvh.b $vr2, $vr8, $vr2\n\t" + "vsub.h $vr3, $vr6, $vr3\n\t" + "vsub.h $vr2, $vr6, $vr2\n\t" + "vilvl.b $vr7, $vr8, $vr1\n\t" + "vilvh.b $vr1, $vr8, $vr1\n\t" + "vmul.h $vr7, $vr7, $vr3\n\t" + "vmul.h $vr1, $vr1, $vr2\n\t" + "vsrlni.b.h $vr1, $vr7, 8\n\t" + "vsadd.bu $vr0, $vr0, $vr1\n\t" + "vor.v $vr0, $vr0, $vr5\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [shuf] "r"(kAlpha) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8"); +} +#endif // HAS_ARGBBLENDROW_LSX + +#ifdef HAS_ARGBORROW_LSX +void ARGBOrRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + uint32_t value, + int width) { + asm volatile( + "vreplgr2vr.w $vr1, %[val]\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vor.v $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(value) + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_ARGBORROW_LSX + +#ifdef HAS_ARGBALPHABLENDROW_LSX +void ARGBAlphaBlendRow_LSX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + static const uint8_t kShufRatio[16] = {0, 0, 0, 0, 4, 4, 4, 4, + 8, 8, 8, 8, 12, 12, 12, 12}; + asm volatile( + "vld $vr8, %[shuf], 0\n\t" + "vldi $vr9, 1\n\t" + "vsrli.h $vr10, $vr9, 8\n\t" + "vldi $vr11, 0xff\n\t" + "vsrli.w $vr11, $vr11, 8\n\t" + "vldi $vr12, 0\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vsrli.w $vr2, $vr0, 24\n\t" + "vsrli.w $vr3, $vr1, 24\n\t" + "vmul.h $vr4, $vr3, $vr2\n\t" + "vadd.h $vr4, $vr4, $vr10\n\t" + "vmuh.hu $vr4, $vr4, $vr9\n\t" + "vsub.w $vr4, $vr2, $vr4\n\t" + "vadd.w $vr4, $vr3, $vr4\n\t" + "vseq.w $vr5, $vr3, $vr12\n\t" + "vsrli.w $vr5, $vr5, 24\n\t" + "vor.v $vr6, $vr2, $vr5\n\t" + "vor.v $vr7, $vr4, $vr5\n\t" + "vslli.w $vr5, $vr6, 8\n\t" + "vsub.w $vr5, $vr5, $vr6\n\t" + "vffint.s.wu $vr5, $vr5\n\t" + "vffint.s.wu $vr7, $vr7\n\t" + "vfdiv.s $vr5, $vr5, $vr7\n\t" + "vftintrz.wu.s $vr5, $vr5\n\t" + "vshuf.b $vr5, $vr5, $vr5, $vr8\n\t" + "vnor.v $vr6, $vr5, $vr5\n\t" + "vmulwev.h.bu $vr2, $vr1, $vr6\n\t" + "vmulwod.h.bu $vr3, $vr1, $vr6\n\t" + "vmulwev.h.bu $vr7, $vr0, $vr5\n\t" + "vmulwod.h.bu $vr0, $vr0, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr7\n\t" + "vadd.h $vr3, $vr3, $vr0\n\t" + "vadd.h $vr2, $vr2, $vr10\n\t" + "vadd.h $vr3, $vr3, $vr10\n\t" + "vmuh.hu $vr2, $vr2, $vr9\n\t" + "vmuh.hu $vr3, $vr3, $vr9\n\t" + "vslli.h $vr3, $vr3, 8\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vand.v $vr2, $vr2, $vr11\n\t" + "vslli.w $vr4, $vr4, 24\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [shuf] "r"(kShufRatio) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12"); +} +#endif // HAS_ARGBALPHABLENDROW_LSX + +#ifdef HAS_ARGBSHADEROW_LSX +void ARGBShadeRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + uint32_t value) { + asm volatile( + "vreplgr2vr.w $vr4, %[val]\n\t" + "vilvl.b $vr5, $vr4, $vr4\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vilvl.b $vr1, $vr0, $vr0\n\t" + "vilvh.b $vr2, $vr0, $vr0\n\t" + "vmuh.hu $vr1, $vr1, $vr5\n\t" + "vmuh.hu $vr2, $vr2, $vr5\n\t" + "vsrlni.b.h $vr2, $vr1, 8\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(value) + : "memory", "$vr0", "$vr1", "$vr2", "$vr4", "$vr5"); +} +#endif // HAS_ARGBSHADEROW_LSX + +#ifdef HAS_ARGBSHUFFLEROW_LSX +void ARGBShuffleRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width) { + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [shuf] "r"(shuffler) + : "memory", "$vr0", "$vr4"); +} +#endif // HAS_ARGBSHUFFLEROW_LSX + +#ifdef HAS_ARGBGRAYROW_LSX +void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + asm volatile( + "vldi $vr4, 29\n\t" + "vldi $vr5, 150\n\t" + "vldi $vr6, 77\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vmulwev.h.bu $vr7, $vr2, $vr4\n\t" + "vmaddwod.h.bu $vr7, $vr2, $vr6\n\t" + "vmaddwev.h.bu $vr7, $vr3, $vr5\n\t" + "vsrlri.h $vr7, $vr7, 8\n\t" + "vsrli.h $vr3, $vr3, 8\n\t" + "vslli.h $vr3, $vr3, 8\n\t" + "vor.v $vr3, $vr7, $vr3\n\t" + "vslli.h $vr8, $vr7, 8\n\t" + "vor.v $vr7, $vr7, $vr8\n\t" + "vilvl.h $vr0, $vr3, $vr7\n\t" + "vilvh.h $vr1, $vr3, $vr7\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8"); +} +#endif // HAS_ARGBGRAYROW_LSX + +#ifdef HAS_ARGBSEPIAROW_LSX +void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width) { + asm volatile( + "vldi $vr4, 17\n\t" + "vldi $vr5, 68\n\t" + "vldi $vr6, 35\n\t" + "vldi $vr7, 22\n\t" + "vldi $vr8, 88\n\t" + "vldi $vr9, 45\n\t" + "vldi $vr10, 24\n\t" + "vldi $vr11, 98\n\t" + "vldi $vr12, 50\n\t" + "1:\n\t" + "vld $vr0, %[dst], 0\n\t" + "vld $vr1, %[dst], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vmulwev.h.bu $vr13, $vr2, $vr4\n\t" + "vmaddwod.h.bu $vr13, $vr2, $vr6\n\t" + "vmaddwev.h.bu $vr13, $vr3, $vr5\n\t" + "vmulwev.h.bu $vr14, $vr2, $vr7\n\t" + "vmaddwod.h.bu $vr14, $vr2, $vr9\n\t" + "vmaddwev.h.bu $vr14, $vr3, $vr8\n\t" + "vmulwev.h.bu $vr15, $vr2, $vr10\n\t" + "vmaddwod.h.bu $vr15, $vr2, $vr12\n\t" + "vmaddwev.h.bu $vr15, $vr3, $vr11\n\t" + "vsrli.h $vr13, $vr13, 7\n\t" + "vsrli.h $vr14, $vr14, 7\n\t" + "vsrli.h $vr15, $vr15, 7\n\t" + "vsat.hu $vr13, $vr13, 7\n\t" + "vsat.hu $vr14, $vr14, 7\n\t" + "vsat.hu $vr15, $vr15, 7\n\t" + "vpickev.b $vr13, $vr13, $vr13\n\t" + "vpickev.b $vr14, $vr14, $vr14\n\t" + "vpickev.b $vr15, $vr15, $vr15\n\t" + "vpickod.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr2, $vr14, $vr13\n\t" + "vilvl.b $vr3, $vr3, $vr15\n\t" + "vilvl.h $vr0, $vr3, $vr2\n\t" + "vilvh.h $vr1, $vr3, $vr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15"); +} +#endif // HAS_ARGBSEPIAROW_LSX + +#ifdef HAS_ARGBCOLORMATRIXROW_LSX +void ARGBColorMatrixRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + asm volatile( + "vldrepl.w $vr4, %[mat], 0\n\t" + "vldrepl.w $vr5, %[mat], 4\n\t" + "vldrepl.w $vr6, %[mat], 8\n\t" + "vldrepl.w $vr7, %[mat], 12\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vmulwev.h.bu.b $vr8, $vr0, $vr4\n\t" + "vmaddwod.h.bu.b $vr8, $vr0, $vr4\n\t" + "vmulwev.h.bu.b $vr9, $vr0, $vr5\n\t" + "vmaddwod.h.bu.b $vr9, $vr0, $vr5\n\t" + "vmulwev.h.bu.b $vr10, $vr0, $vr6\n\t" + "vmaddwod.h.bu.b $vr10, $vr0, $vr6\n\t" + "vmulwev.h.bu.b $vr11, $vr0, $vr7\n\t" + "vmaddwod.h.bu.b $vr11, $vr0, $vr7\n\t" + "vhaddw.w.h $vr8, $vr8, $vr8\n\t" + "vhaddw.w.h $vr9, $vr9, $vr9\n\t" + "vhaddw.w.h $vr10, $vr10, $vr10\n\t" + "vhaddw.w.h $vr11, $vr11, $vr11\n\t" + "vilvl.w $vr12, $vr9, $vr8\n\t" + "vilvh.w $vr13, $vr9, $vr8\n\t" + "vilvl.w $vr14, $vr11, $vr10\n\t" + "vilvh.w $vr15, $vr11, $vr10\n\t" + "vilvl.d $vr8, $vr14, $vr12\n\t" + "vilvh.d $vr9, $vr14, $vr12\n\t" + "vilvl.d $vr10, $vr15, $vr13\n\t" + "vilvh.d $vr11, $vr15, $vr13\n\t" + "vssrani.h.w $vr9, $vr8, 6\n\t" + "vssrani.h.w $vr11, $vr10, 6\n\t" + "vssrani.bu.h $vr11, $vr9, 0\n\t" + "vst $vr11, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [mat] "r"(matrix_argb) + : "memory", "$vr0", "$vr4", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9", + "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15"); +} +#endif // HAS_ARGBCOLORMATRIXROW_LSX + +#ifdef HAS_ARGBQUANTIZEROW_LSX +void ARGBQuantizeRow_LSX(uint8_t* dst_argb, + int scale, + int interval_size, + int interval_offset, + int width) { + asm volatile( + "vreplgr2vr.h $vr4, %[scl]\n\t" + "vreplgr2vr.b $vr5, %[isz]\n\t" + "vreplgr2vr.b $vr6, %[ioff]\n\t" + "vldi $vr7, 0xff\n\t" + "vslli.w $vr7, $vr7, 24\n\t" + "1:\n\t" + "vld $vr0, %[dst], 0\n\t" + "vilvl.b $vr1, $vr0, $vr0\n\t" + "vilvh.b $vr2, $vr0, $vr0\n\t" + "vsrli.h $vr1, $vr1, 8\n\t" + "vsrli.h $vr2, $vr2, 8\n\t" + "vmuh.hu $vr1, $vr1, $vr4\n\t" + "vmuh.hu $vr2, $vr2, $vr4\n\t" + "vsrlni.b.h $vr2, $vr1, 0\n\t" + "vmul.b $vr2, $vr2, $vr5\n\t" + "vadd.b $vr2, $vr2, $vr6\n\t" + "vand.v $vr0, $vr0, $vr7\n\t" + "vandn.v $vr2, $vr7, $vr2\n\t" + "vor.v $vr2, $vr2, $vr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [scl] "r"(scale), [isz] "r"(interval_size), [ioff] "r"(interval_offset) + : "memory", "$vr0", "$vr1", "$vr2", "$vr4", "$vr5", "$vr6", "$vr7"); +} +#endif // HAS_ARGBQUANTIZEROW_LSX + +#ifdef HAS_SOBELROW_LSX +void SobelRow_LSX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr4, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sx], 0\n\t" + "vldrepl.d $vr1, %[sy], 0\n\t" + "vsadd.bu $vr0, $vr0, $vr1\n\t" + "vilvl.b $vr2, $vr0, $vr0\n\t" + "vilvl.b $vr3, $vr4, $vr0\n\t" + "vilvl.h $vr0, $vr3, $vr2\n\t" + "vilvh.h $vr1, $vr3, $vr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[sx], %[sx], 8\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_SOBELROW_LSX + +#ifdef HAS_SOBELTOPLANEROW_LSX +void SobelToPlaneRow_LSX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_y, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[sx], 0\n\t" + "vld $vr1, %[sy], 0\n\t" + "vsadd.bu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[sx], %[sx], 16\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_y), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} +#endif // HAS_SOBELTOPLANEROW_LSX + +#ifdef HAS_SOBELXYROW_LSX +void SobelXYRow_LSX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr4, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sx], 0\n\t" + "vldrepl.d $vr1, %[sy], 0\n\t" + "vsadd.bu $vr2, $vr0, $vr1\n\t" + "vilvl.b $vr1, $vr2, $vr1\n\t" + "vilvl.b $vr0, $vr4, $vr0\n\t" + "vilvl.h $vr2, $vr0, $vr1\n\t" + "vilvh.h $vr3, $vr0, $vr1\n\t" + "vst $vr2, %[dst], 0\n\t" + "vst $vr3, %[dst], 16\n\t" + "addi.d %[sx], %[sx], 8\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_SOBELXYROW_LSX + +#ifdef HAS_INTERPOLATEROW_LSX +void InterpolateRow_LSX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int32_t source_y_fraction) { + int y1_fraction = source_y_fraction; + const uint8_t* src_ptr1 = src_ptr + src_stride; + if (y1_fraction == 0) { + CopyRow_LSX(src_ptr, dst_ptr, width); + return; + } + if (y1_fraction == 128) { + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vavgr.bu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); + return; + } + uint16_t y_frac = (uint16_t)((256 - y1_fraction) + (y1_fraction << 8)); + asm volatile( + "vreplgr2vr.h $vr3, %[yf]\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vilvl.b $vr2, $vr1, $vr0\n\t" + "vilvh.b $vr4, $vr1, $vr0\n\t" + "vmulwev.h.bu $vr0, $vr2, $vr3\n\t" + "vmaddwod.h.bu $vr0, $vr2, $vr3\n\t" + "vmulwev.h.bu $vr1, $vr4, $vr3\n\t" + "vmaddwod.h.bu $vr1, $vr4, $vr3\n\t" + "vsrarni.b.h $vr1, $vr0, 8\n\t" + "vst $vr1, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), + [cnt] "+r"(width) + : [yf] "r"(y_frac) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} +#endif // HAS_INTERPOLATEROW_LSX + +#ifdef HAS_HALFFLOATROW_LSX +void HalfFloatRow_LSX(const uint16_t* src, + uint16_t* dst, + float scale, + int width) { + const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C + asm volatile( + "vldrepl.w $vr3, %[scale], 0\n\t" + "vldrepl.w $vr5, %[bias], 0\n\t" + "vldi $vr4, 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vilvl.h $vr1, $vr4, $vr0\n\t" + "vilvh.h $vr2, $vr4, $vr0\n\t" + "vffint.s.wu $vr1, $vr1\n\t" + "vffint.s.wu $vr2, $vr2\n\t" + "vfmul.s $vr1, $vr1, $vr3\n\t" + "vfmul.s $vr2, $vr2, $vr3\n\t" + "vfmul.s $vr1, $vr1, $vr5\n\t" + "vfmul.s $vr2, $vr2, $vr5\n\t" + "vsrlni.h.w $vr2, $vr1, 13\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : [scale] "r"(&scale), [bias] "r"(&bias) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); +} +#endif // HAS_HALFFLOATROW_LSX + +#ifdef HAS_ARGBTOYMATRIXROW_LSX +void ARGBToYMatrixRow_LSX(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + asm volatile( + "vldrepl.b $vr0, %[c], 0\n\t" + "vldrepl.b $vr1, %[c], 1\n\t" + "vldrepl.b $vr2, %[c], 2\n\t" + "vldrepl.b $vr3, %[c], 3\n\t" + "vldrepl.h $vr4, %[c], 96\n\t" + "1:\n\t" + "vld $vr5, %[src], 0\n\t" + "vld $vr6, %[src], 16\n\t" + "vor.v $vr9, $vr4, $vr4\n\t" + "vpickev.b $vr7, $vr6, $vr5\n\t" + "vpickod.b $vr8, $vr6, $vr5\n\t" + "vmaddwev.h.bu $vr9, $vr7, $vr0\n\t" + "vmaddwev.h.bu $vr9, $vr8, $vr1\n\t" + "vmaddwod.h.bu $vr9, $vr7, $vr2\n\t" + "vmaddwod.h.bu $vr9, $vr8, $vr3\n\t" + "vpickod.b $vr9, $vr9, $vr9\n\t" + "vstelm.d $vr9, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [c] "r"(c) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9"); +} +#endif // HAS_ARGBTOYMATRIXROW_LSX + +#ifdef HAS_RGBTOYMATRIXROW_LSX +void RGBToYMatrixRow_LSX(const uint8_t* src_rgba, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8, + 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8, + 9, 0, 10, 11, 12, 0, 13, 14, 15, 0}; + asm volatile( + "vldrepl.b $vr0, %[c], 0\n\t" + "vldrepl.b $vr1, %[c], 1\n\t" + "vldrepl.b $vr2, %[c], 2\n\t" + "vldrepl.h $vr3, %[c], 96\n\t" + "vld $vr4, %[shuf], 0\n\t" + "vld $vr5, %[shuf], 16\n\t" + "1:\n\t" + "vld $vr6, %[src], 0\n\t" + "vld $vr7, %[src], 12\n\t" + "vld $vr8, %[src], 24\n\t" + "vld $vr9, %[src], 32\n\t" + "vshuf.b $vr6, $vr6, $vr6, $vr4\n\t" + "vshuf.b $vr7, $vr7, $vr7, $vr4\n\t" + "vshuf.b $vr8, $vr8, $vr8, $vr4\n\t" + "vshuf.b $vr9, $vr9, $vr9, $vr5\n\t" + "vor.v $vr12, $vr3, $vr3\n\t" + "vor.v $vr13, $vr3, $vr3\n\t" + "vpickev.b $vr10, $vr7, $vr6\n\t" + "vpickod.b $vr11, $vr7, $vr6\n\t" + "vmaddwev.h.bu $vr12, $vr10, $vr0\n\t" + "vmaddwev.h.bu $vr12, $vr11, $vr1\n\t" + "vmaddwod.h.bu $vr12, $vr10, $vr2\n\t" + "vpickev.b $vr10, $vr9, $vr8\n\t" + "vpickod.b $vr11, $vr9, $vr8\n\t" + "vmaddwev.h.bu $vr13, $vr10, $vr0\n\t" + "vmaddwev.h.bu $vr13, $vr11, $vr1\n\t" + "vmaddwod.h.bu $vr13, $vr10, $vr2\n\t" + "vpickod.b $vr12, $vr13, $vr12\n\t" + "vst $vr12, %[dst], 0\n\t" + "addi.d %[src], %[src], 48\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgba), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [c] "r"(c), [shuf] "r"(kShuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13"); +} +#endif // HAS_RGBTOYMATRIXROW_LSX + +#ifdef HAS_RGB565TOYROW_LSX +void RGB565ToYRow_LSX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "vldi $vr8, 25\n\t" + "vsrli.h $vr8, $vr8, 8\n\t" + "vldi $vr9, 129\n\t" + "vsrli.h $vr9, $vr9, 8\n\t" + "vldi $vr10, 66\n\t" + "vsrli.h $vr10, $vr10, 8\n\t" + "vldi $vr11, 33\n\t" + "vsrli.h $vr11, $vr11, 8\n\t" + "vslli.h $vr11, $vr11, 7\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vmul.h $vr1, $vr1, $vr8\n\t" + "vmadd.h $vr1, $vr2, $vr9\n\t" + "vmadd.h $vr1, $vr3, $vr10\n\t" + "vadd.h $vr1, $vr1, $vr11\n\t" + "vpickod.b $vr1, $vr1, $vr1\n\t" + "vstelm.d $vr1, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgb565), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7", + "$vr8", "$vr9", "$vr10", "$vr11"); +} +#endif // HAS_RGB565TOYROW_LSX + +#ifdef HAS_ARGB1555TOYROW_LSX +void ARGB1555ToYRow_LSX(const uint8_t* src_argb1555, + uint8_t* dst_y, + int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr8, 25\n\t" + "vsrli.h $vr8, $vr8, 8\n\t" + "vldi $vr9, 129\n\t" + "vsrli.h $vr9, $vr9, 8\n\t" + "vldi $vr10, 66\n\t" + "vsrli.h $vr10, $vr10, 8\n\t" + "vldi $vr11, 33\n\t" + "vsrli.h $vr11, $vr11, 8\n\t" + "vslli.h $vr11, $vr11, 7\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vmul.h $vr1, $vr1, $vr8\n\t" + "vmadd.h $vr1, $vr2, $vr9\n\t" + "vmadd.h $vr1, $vr3, $vr10\n\t" + "vadd.h $vr1, $vr1, $vr11\n\t" + "vpickod.b $vr1, $vr1, $vr1\n\t" + "vstelm.d $vr1, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb1555), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr8", + "$vr9", "$vr10", "$vr11"); +} +#endif // HAS_ARGB1555TOYROW_LSX + +#ifdef HAS_RGB565TOUVROW_LSX +void RGB565ToUVRow_LSX(const uint8_t* src_rgb565, + int src_stride_rgb565, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_rgb565 = src_rgb565 + src_stride_rgb565; + asm volatile( + "vldi $vr20, 112\n\t" + "vsrli.h $vr20, $vr20, 8\n\t" + "vldi $vr21, 74\n\t" + "vsrli.h $vr21, $vr21, 8\n\t" + "vldi $vr22, 38\n\t" + "vsrli.h $vr22, $vr22, 8\n\t" + "vldi $vr23, 94\n\t" + "vsrli.h $vr23, $vr23, 8\n\t" + "vldi $vr24, 18\n\t" + "vsrli.h $vr24, $vr24, 8\n\t" + "vldi $vr25, 128\n\t" + "vsrli.h $vr25, $vr25, 8\n\t" + "vslli.h $vr25, $vr25, 8\n\t" + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vor.v $vr10, $vr1, $vr1\n\t" + "vor.v $vr11, $vr2, $vr2\n\t" + "vor.v $vr12, $vr3, $vr3\n\t" + "vld $vr0, %[s0], 16\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr10, $vr1, $vr10\n\t" + "vpickev.b $vr11, $vr2, $vr11\n\t" + "vpickev.b $vr12, $vr3, $vr12\n\t" + "vld $vr0, %[s1], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vor.v $vr13, $vr1, $vr1\n\t" + "vor.v $vr14, $vr2, $vr2\n\t" + "vor.v $vr15, $vr3, $vr3\n\t" + "vld $vr0, %[s1], 16\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr5, $vr3, $vr15\n\t" + "vpickev.b $vr4, $vr2, $vr14\n\t" + "vpickev.b $vr3, $vr1, $vr13\n\t" + "vor.v $vr0, $vr10, $vr10\n\t" + "vor.v $vr1, $vr11, $vr11\n\t" + "vor.v $vr2, $vr12, $vr12\n\t" + "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr3\n\t" + "vsrari.h $vr0, $vr0, 2\n\t" + "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" + "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" + "vadd.h $vr1, $vr1, $vr4\n\t" + "vsrari.h $vr1, $vr1, 2\n\t" + "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" + "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr5\n\t" + "vsrari.h $vr2, $vr2, 2\n\t" + "vor.v $vr8, $vr25, $vr25\n\t" + "vor.v $vr9, $vr25, $vr25\n\t" + "vmadd.h $vr8, $vr0, $vr20\n\t" + "vmsub.h $vr8, $vr1, $vr21\n\t" + "vmsub.h $vr8, $vr2, $vr22\n\t" + "vmadd.h $vr9, $vr2, $vr20\n\t" + "vmsub.h $vr9, $vr1, $vr23\n\t" + "vmsub.h $vr9, $vr0, $vr24\n\t" + "vpickod.b $vr8, $vr9, $vr8\n\t" + "vstelm.d $vr8, %[du], 0, 0\n\t" + "vstelm.d $vr8, %[dv], 0, 1\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_rgb565), [s1] "+r"(next_rgb565), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24", "$vr25"); +} +#endif // HAS_RGB565TOUVROW_LSX + +#ifdef HAS_ARGB1555TOUVROW_LSX +void ARGB1555ToUVRow_LSX(const uint8_t* src_argb1555, + int src_stride_argb1555, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_argb1555 = src_argb1555 + src_stride_argb1555; + asm volatile( + "vldi $vr20, 112\n\t" + "vsrli.h $vr20, $vr20, 8\n\t" + "vldi $vr21, 74\n\t" + "vsrli.h $vr21, $vr21, 8\n\t" + "vldi $vr22, 38\n\t" + "vsrli.h $vr22, $vr22, 8\n\t" + "vldi $vr23, 94\n\t" + "vsrli.h $vr23, $vr23, 8\n\t" + "vldi $vr24, 18\n\t" + "vsrli.h $vr24, $vr24, 8\n\t" + "vldi $vr25, 128\n\t" + "vsrli.h $vr25, $vr25, 8\n\t" + "vslli.h $vr25, $vr25, 8\n\t" + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vor.v $vr10, $vr1, $vr1\n\t" + "vor.v $vr11, $vr2, $vr2\n\t" + "vor.v $vr12, $vr3, $vr3\n\t" + "vld $vr0, %[s0], 16\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr10, $vr1, $vr10\n\t" + "vpickev.b $vr11, $vr2, $vr11\n\t" + "vpickev.b $vr12, $vr3, $vr12\n\t" + "vld $vr0, %[s1], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vor.v $vr13, $vr1, $vr1\n\t" + "vor.v $vr14, $vr2, $vr2\n\t" + "vor.v $vr15, $vr3, $vr3\n\t" + "vld $vr0, %[s1], 16\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr5, $vr3, $vr15\n\t" + "vpickev.b $vr4, $vr2, $vr14\n\t" + "vpickev.b $vr3, $vr1, $vr13\n\t" + "vor.v $vr0, $vr10, $vr10\n\t" + "vor.v $vr1, $vr11, $vr11\n\t" + "vor.v $vr2, $vr12, $vr12\n\t" + "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr3\n\t" + "vsrari.h $vr0, $vr0, 2\n\t" + "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" + "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" + "vadd.h $vr1, $vr1, $vr4\n\t" + "vsrari.h $vr1, $vr1, 2\n\t" + "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" + "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr5\n\t" + "vsrari.h $vr2, $vr2, 2\n\t" + "vor.v $vr8, $vr25, $vr25\n\t" + "vor.v $vr9, $vr25, $vr25\n\t" + "vmadd.h $vr8, $vr0, $vr20\n\t" + "vmsub.h $vr8, $vr1, $vr21\n\t" + "vmsub.h $vr8, $vr2, $vr22\n\t" + "vmadd.h $vr9, $vr2, $vr20\n\t" + "vmsub.h $vr9, $vr1, $vr23\n\t" + "vmsub.h $vr9, $vr0, $vr24\n\t" + "vpickod.b $vr8, $vr9, $vr8\n\t" + "vstelm.d $vr8, %[du], 0, 0\n\t" + "vstelm.d $vr8, %[dv], 0, 1\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb1555), [s1] "+r"(next_argb1555), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15", + "$vr20", "$vr21", "$vr22", "$vr23", "$vr24", "$vr25"); +} +#endif // HAS_ARGB1555TOUVROW_LSX + +#if defined(HAS_RGB24TOUVROW_LSX) || defined(HAS_RAWTOUVROW_LSX) +static LIBYUV_NOINLINE void RGB24ToUVHelper_LSX(const uint8_t* src, + int stride, + uint8_t* dst_u, + uint8_t* dst_v, + const uint8_t* shuf, + int width) { + const uint8_t* next = src + stride; + asm volatile( + "vldi $vr20, 112\n\t" + "vsrli.h $vr20, $vr20, 8\n\t" + "vldi $vr21, 74\n\t" + "vsrli.h $vr21, $vr21, 8\n\t" + "vldi $vr22, 38\n\t" + "vsrli.h $vr22, $vr22, 8\n\t" + "vldi $vr23, 94\n\t" + "vsrli.h $vr23, $vr23, 8\n\t" + "vldi $vr24, 18\n\t" + "vsrli.h $vr24, $vr24, 8\n\t" + "vldi $vr25, 128\n\t" + "vsrli.h $vr25, $vr25, 8\n\t" + "vslli.h $vr25, $vr25, 8\n\t" + "vld $vr16, %[shuf], 0\n\t" + "vld $vr17, %[shuf], 16\n\t" + "1:\n\t" + "vld $vr8, %[s0], 0\n\t" + "vld $vr9, %[s0], 12\n\t" + "vld $vr10, %[s0], 24\n\t" + "vld $vr11, %[s0], 32\n\t" + "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" + "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" + "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" + "vshuf.b $vr11, $vr11, $vr11, $vr17\n\t" + "vpickev.b $vr12, $vr9, $vr8\n\t" + "vpickod.b $vr13, $vr9, $vr8\n\t" + "vpickev.b $vr14, $vr11, $vr10\n\t" + "vpickod.b $vr15, $vr11, $vr10\n\t" + "vpickev.b $vr0, $vr14, $vr12\n\t" + "vpickod.b $vr2, $vr14, $vr12\n\t" + "vpickev.b $vr1, $vr15, $vr13\n\t" + "vld $vr8, %[s1], 0\n\t" + "vld $vr9, %[s1], 12\n\t" + "vld $vr10, %[s1], 24\n\t" + "vld $vr11, %[s1], 32\n\t" + "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" + "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" + "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" + "vshuf.b $vr11, $vr11, $vr11, $vr17\n\t" + "vpickev.b $vr12, $vr9, $vr8\n\t" + "vpickod.b $vr13, $vr9, $vr8\n\t" + "vpickev.b $vr14, $vr11, $vr10\n\t" + "vpickod.b $vr15, $vr11, $vr10\n\t" + "vpickev.b $vr3, $vr14, $vr12\n\t" + "vpickod.b $vr5, $vr14, $vr12\n\t" + "vpickev.b $vr4, $vr15, $vr13\n\t" + "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr3\n\t" + "vsrari.h $vr0, $vr0, 2\n\t" + "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" + "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" + "vadd.h $vr1, $vr1, $vr4\n\t" + "vsrari.h $vr1, $vr1, 2\n\t" + "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" + "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr5\n\t" + "vsrari.h $vr2, $vr2, 2\n\t" + "vor.v $vr6, $vr25, $vr25\n\t" + "vor.v $vr7, $vr25, $vr25\n\t" + "vmadd.h $vr6, $vr0, $vr20\n\t" + "vmsub.h $vr6, $vr1, $vr21\n\t" + "vmsub.h $vr6, $vr2, $vr22\n\t" + "vmadd.h $vr7, $vr2, $vr20\n\t" + "vmsub.h $vr7, $vr1, $vr23\n\t" + "vmsub.h $vr7, $vr0, $vr24\n\t" + "vpickod.b $vr6, $vr7, $vr6\n\t" + "vstelm.d $vr6, %[du], 0, 0\n\t" + "vstelm.d $vr6, %[dv], 0, 1\n\t" + "addi.d %[s0], %[s0], 48\n\t" + "addi.d %[s1], %[s1], 48\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15", "$vr16", "$vr17", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24", + "$vr25"); +} +#endif // defined(HAS_RGB24TOUVROW_LSX) || defined(HAS_RAWTOUVROW_LSX) + +#ifdef HAS_RGB24TOUVROW_LSX +void RGB24ToUVRow_LSX(const uint8_t* src_rgb24, + int src_stride_rgb24, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8, + 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8, + 9, 0, 10, 11, 12, 0, 13, 14, 15, 0}; + RGB24ToUVHelper_LSX(src_rgb24, src_stride_rgb24, dst_u, dst_v, kShuf, width); +} +#endif // HAS_RGB24TOUVROW_LSX + +#ifdef HAS_RAWTOUVROW_LSX +void RAWToUVRow_LSX(const uint8_t* src_raw, + int src_stride_raw, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[32] = {2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6, + 0, 11, 10, 9, 0, 6, 5, 4, 0, 9, 8, + 7, 0, 12, 11, 10, 0, 15, 14, 13, 0}; + RGB24ToUVHelper_LSX(src_raw, src_stride_raw, dst_u, dst_v, kShuf, width); +} +#endif // HAS_RAWTOUVROW_LSX + +#ifdef HAS_COPYROW_LASX void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int count) { asm volatile( "1:\n\t" @@ -41,7 +2845,9 @@ void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int count) { : : "memory", "$xr0"); } +#endif // HAS_COPYROW_LASX +#ifdef HAS_SETROW_LASX void SetRow_LASX(uint8_t* dst, uint8_t v8, int count) { asm volatile( "xvreplgr2vr.b $xr0, %[val]\n\t" @@ -54,7 +2860,9 @@ void SetRow_LASX(uint8_t* dst, uint8_t v8, int count) { : [val] "r"(v8) : "memory", "$xr0"); } +#endif // HAS_SETROW_LASX +#ifdef HAS_ARGBSETROW_LASX void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width) { asm volatile( "xvreplgr2vr.w $xr0, %[val]\n\t" @@ -67,7 +2875,9 @@ void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width) { : [val] "r"(v32) : "memory", "$xr0"); } +#endif // HAS_ARGBSETROW_LASX +#ifdef HAS_MIRRORROW_LASX void MirrorRow_LASX(const uint8_t* src_y, uint8_t* dst_y, int width) { static const uint8_t kRev[16] = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; @@ -89,7 +2899,9 @@ void MirrorRow_LASX(const uint8_t* src_y, uint8_t* dst_y, int width) { : [rev] "r"(kRev) : "memory", "$xr0", "$xr1", "$xr31"); } +#endif // HAS_MIRRORROW_LASX +#ifdef HAS_MIRRORUVROW_LASX void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width) { static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1}; @@ -111,7 +2923,9 @@ void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width) { : [rev] "r"(kRevUV) : "memory", "$xr0", "$xr1", "$xr31"); } +#endif // HAS_MIRRORUVROW_LASX +#ifdef HAS_ARGBMIRRORROW_LASX void ARGBMirrorRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { src_argb += width * 4 - 32; asm volatile( @@ -129,7 +2943,9 @@ void ARGBMirrorRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { : : "memory", "$xr0", "$xr31"); } +#endif // HAS_ARGBMIRRORROW_LASX +#ifdef HAS_MIRRORSPLITUVROW_LASX void MirrorSplitUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, @@ -163,7 +2979,9 @@ void MirrorSplitUVRow_LASX(const uint8_t* src_uv, : [rev] "r"(kRevUV) : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); } +#endif // HAS_MIRRORSPLITUVROW_LASX +#ifdef HAS_SPLITUVROW_LASX void SplitUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, @@ -190,7 +3008,9 @@ void SplitUVRow_LASX(const uint8_t* src_uv, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); } +#endif // HAS_SPLITUVROW_LASX +#ifdef HAS_MERGEUVROW_LASX void MergeUVRow_LASX(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, @@ -217,7 +3037,9 @@ void MergeUVRow_LASX(const uint8_t* src_u, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); } +#endif // HAS_MERGEUVROW_LASX +#ifdef HAS_ARGBEXTRACTALPHAROW_LASX void ARGBExtractAlphaRow_LASX(const uint8_t* src_argb, uint8_t* dst_a, int width) { @@ -241,7 +3063,9 @@ void ARGBExtractAlphaRow_LASX(const uint8_t* src_argb, : : "memory", "$xr0", "$xr1", "$xr31"); } +#endif // HAS_ARGBEXTRACTALPHAROW_LASX +#ifdef HAS_I422TOYUY2ROW_LASX void I422ToYUY2Row_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -274,7 +3098,9 @@ void I422ToYUY2Row_LASX(const uint8_t* src_y, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr31"); } +#endif // HAS_I422TOYUY2ROW_LASX +#ifdef HAS_I422TOUYVYROW_LASX void I422ToUYVYRow_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -307,7 +3133,9 @@ void I422ToUYVYRow_LASX(const uint8_t* src_y, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr31"); } +#endif // HAS_I422TOUYVYROW_LASX +#ifdef HAS_YUY2TOYROW_LASX void YUY2ToYRow_LASX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { asm volatile( "1:\n\t" @@ -327,7 +3155,9 @@ void YUY2ToYRow_LASX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { : : "memory", "$xr0", "$xr1", "$xr2", "$xr31"); } +#endif // HAS_YUY2TOYROW_LASX +#ifdef HAS_UYVYTOYROW_LASX void UYVYToYRow_LASX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { asm volatile( "1:\n\t" @@ -347,7 +3177,9 @@ void UYVYToYRow_LASX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { : : "memory", "$xr0", "$xr1", "$xr2", "$xr31"); } +#endif // HAS_UYVYTOYROW_LASX +#ifdef HAS_YUY2TOUV422ROW_LASX void YUY2ToUV422Row_LASX(const uint8_t* src_yuy2, uint8_t* dst_u, uint8_t* dst_v, @@ -377,7 +3209,9 @@ void YUY2ToUV422Row_LASX(const uint8_t* src_yuy2, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); } +#endif // HAS_YUY2TOUV422ROW_LASX +#ifdef HAS_UYVYTOUV422ROW_LASX void UYVYToUV422Row_LASX(const uint8_t* src_uyvy, uint8_t* dst_u, uint8_t* dst_v, @@ -407,7 +3241,9 @@ void UYVYToUV422Row_LASX(const uint8_t* src_uyvy, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); } +#endif // HAS_UYVYTOUV422ROW_LASX +#ifdef HAS_YUY2TOUVROW_LASX void YUY2ToUVRow_LASX(const uint8_t* src_yuy2, int src_stride_yuy2, uint8_t* dst_u, @@ -448,7 +3284,9 @@ void YUY2ToUVRow_LASX(const uint8_t* src_yuy2, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); } +#endif // HAS_YUY2TOUVROW_LASX +#ifdef HAS_UYVYTOUVROW_LASX void UYVYToUVRow_LASX(const uint8_t* src_uyvy, int src_stride_uyvy, uint8_t* dst_u, @@ -489,7 +3327,9 @@ void UYVYToUVRow_LASX(const uint8_t* src_uyvy, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); } +#endif // HAS_UYVYTOUVROW_LASX +#ifdef HAS_I422TOARGBROW_LASX void I422ToARGBRow_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -552,7 +3392,9 @@ void I422ToARGBRow_LASX(const uint8_t* src_y, "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_I422TOARGBROW_LASX +#ifdef HAS_I422TORGBAROW_LASX void I422ToRGBARow_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -615,7 +3457,9 @@ void I422ToRGBARow_LASX(const uint8_t* src_y, "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_I422TORGBAROW_LASX +#ifdef HAS_I422ALPHATOARGBROW_LASX void I422AlphaToARGBRow_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -681,7 +3525,9 @@ void I422AlphaToARGBRow_LASX(const uint8_t* src_y, "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_I422ALPHATOARGBROW_LASX +#ifdef HAS_I422TORGB24ROW_LASX void I422ToRGB24Row_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -753,7 +3599,9 @@ void I422ToRGB24Row_LASX(const uint8_t* src_y, "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_I422TORGB24ROW_LASX +#ifdef HAS_I422TORGB565ROW_LASX void I422ToRGB565Row_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -822,7 +3670,9 @@ void I422ToRGB565Row_LASX(const uint8_t* src_y, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr10", "$xr11", "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_I422TORGB565ROW_LASX +#ifdef HAS_I422TOARGB4444ROW_LASX void I422ToARGB4444Row_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -893,7 +3743,9 @@ void I422ToARGB4444Row_LASX(const uint8_t* src_y, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr10", "$xr11", "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_I422TOARGB4444ROW_LASX +#ifdef HAS_I422TOARGB1555ROW_LASX void I422ToARGB1555Row_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -964,7 +3816,9 @@ void I422ToARGB1555Row_LASX(const uint8_t* src_y, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr10", "$xr11", "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_I422TOARGB1555ROW_LASX +#ifdef HAS_NV12TOARGBROW_LASX void NV12ToARGBRow_LASX(const uint8_t* src_y, const uint8_t* src_uv, uint8_t* dst_argb, @@ -1019,7 +3873,9 @@ void NV12ToARGBRow_LASX(const uint8_t* src_y, "$xr7", "$xr8", "$xr9", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_NV12TOARGBROW_LASX +#ifdef HAS_NV21TOARGBROW_LASX void NV21ToARGBRow_LASX(const uint8_t* src_y, const uint8_t* src_vu, uint8_t* dst_argb, @@ -1075,7 +3931,9 @@ void NV21ToARGBRow_LASX(const uint8_t* src_y, "$xr7", "$xr8", "$xr9", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_NV21TOARGBROW_LASX +#ifdef HAS_NV12TORGB565ROW_LASX void NV12ToRGB565Row_LASX(const uint8_t* src_y, const uint8_t* src_uv, uint8_t* dst_argb, @@ -1136,7 +3994,9 @@ void NV12ToRGB565Row_LASX(const uint8_t* src_y, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24"); } +#endif // HAS_NV12TORGB565ROW_LASX +#ifdef HAS_I444TOARGBROW_LASX void I444ToARGBRow_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -1195,7 +4055,9 @@ void I444ToARGBRow_LASX(const uint8_t* src_y, "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_I444TOARGBROW_LASX +#ifdef HAS_I400TOARGBROW_LASX void I400ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, @@ -1232,7 +4094,9 @@ void I400ToARGBRow_LASX(const uint8_t* src_y, : "memory", "$xr0", "$xr2", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_I400TOARGBROW_LASX +#ifdef HAS_J400TOARGBROW_LASX void J400ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, int width) { asm volatile( "xvldi $xr5, 0xff\n\t" @@ -1257,7 +4121,9 @@ void J400ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, int width) { : : "memory", "$xr2", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr31"); } +#endif // HAS_J400TOARGBROW_LASX +#ifdef HAS_YUY2TOARGBROW_LASX void YUY2ToARGBRow_LASX(const uint8_t* src_yuy2, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, @@ -1308,7 +4174,9 @@ void YUY2ToARGBRow_LASX(const uint8_t* src_yuy2, "$xr7", "$xr8", "$xr9", "$xr10", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_YUY2TOARGBROW_LASX +#ifdef HAS_UYVYTOARGBROW_LASX void UYVYToARGBRow_LASX(const uint8_t* src_uyvy, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, @@ -1359,7 +4227,9 @@ void UYVYToARGBRow_LASX(const uint8_t* src_uyvy, "$xr7", "$xr8", "$xr9", "$xr10", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } +#endif // HAS_UYVYTOARGBROW_LASX +#if defined(HAS_ARGBTORGB24ROW_LASX) || defined(HAS_ARGBTORAWROW_LASX) static LIBYUV_NOINLINE void ARGBTo24Helper_LASX(const uint8_t* src_argb, uint8_t* dst_24, const uint8_t* shuf, @@ -1420,7 +4290,9 @@ static LIBYUV_NOINLINE void ARGBTo24Helper_LASX(const uint8_t* src_argb, : [shuf] "r"(shuf) : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); } +#endif // defined(HAS_ARGBTORGB24ROW_LASX) || defined(HAS_ARGBTORAWROW_LASX) +#ifdef HAS_ARGBTORGB24ROW_LASX void ARGBToRGB24Row_LASX(const uint8_t* src_argb, uint8_t* dst_rgb24, int width) { @@ -1428,13 +4300,17 @@ void ARGBToRGB24Row_LASX(const uint8_t* src_argb, 10, 12, 13, 14, 0, 0, 0, 0}; ARGBTo24Helper_LASX(src_argb, dst_rgb24, kShuf, width); } +#endif // HAS_ARGBTORGB24ROW_LASX +#ifdef HAS_ARGBTORAWROW_LASX void ARGBToRAWRow_LASX(const uint8_t* src_argb, uint8_t* dst_raw, int width) { static const uint8_t kShuf[16] = {2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 0, 0, 0, 0}; ARGBTo24Helper_LASX(src_argb, dst_raw, kShuf, width); } +#endif // HAS_ARGBTORAWROW_LASX +#if defined(HAS_RGB24TOARGBROW_LASX) || defined(HAS_RAWTOARGBROW_LASX) static LIBYUV_NOINLINE void RGB24ToARGBHelper_LASX(const uint8_t* src_24, uint8_t* dst_argb, const uint8_t* shuf, @@ -1483,7 +4359,9 @@ static LIBYUV_NOINLINE void RGB24ToARGBHelper_LASX(const uint8_t* src_24, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr31"); } +#endif // defined(HAS_RGB24TOARGBROW_LASX) || defined(HAS_RAWTOARGBROW_LASX) +#ifdef HAS_RGB24TOARGBROW_LASX void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24, uint8_t* dst_argb, int width) { @@ -1492,14 +4370,18 @@ void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24, 9, 16, 10, 11, 12, 16, 13, 14, 15, 16}; RGB24ToARGBHelper_LASX(src_rgb24, dst_argb, kShuf, width); } +#endif // HAS_RGB24TOARGBROW_LASX +#ifdef HAS_RAWTOARGBROW_LASX void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { static const uint8_t kShuf[32] = {2, 1, 0, 16, 5, 4, 3, 16, 8, 7, 6, 16, 11, 10, 9, 16, 6, 5, 4, 16, 9, 8, 7, 16, 12, 11, 10, 16, 15, 14, 13, 16}; RGB24ToARGBHelper_LASX(src_raw, dst_argb, kShuf, width); } +#endif // HAS_RAWTOARGBROW_LASX +#ifdef HAS_RAWTORGB24ROW_LASX void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { static const uint8_t kShuf[32] = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 0, 0, 0, 0, 6, 5, 4, 9, 8, 7, @@ -1562,7 +4444,9 @@ void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { : [shuf] "r"(kShuf) : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr31"); } +#endif // HAS_RAWTORGB24ROW_LASX +#ifdef HAS_ARGBTORGB565ROW_LASX void ARGBToRGB565Row_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { @@ -1599,7 +4483,9 @@ void ARGBToRGB565Row_LASX(const uint8_t* src_argb, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr6", "$xr7", "$xr31"); } +#endif // HAS_ARGBTORGB565ROW_LASX +#ifdef HAS_ARGBTORGB565DITHERROW_LASX void ARGBToRGB565DitherRow_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, uint32_t dither4, @@ -1637,7 +4523,9 @@ void ARGBToRGB565DitherRow_LASX(const uint8_t* src_argb, : "memory", "$xr0", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7", "$xr31"); } +#endif // HAS_ARGBTORGB565DITHERROW_LASX +#ifdef HAS_ARGBTOARGB1555ROW_LASX void ARGBToARGB1555Row_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { @@ -1675,7 +4563,9 @@ void ARGBToARGB1555Row_LASX(const uint8_t* src_argb, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr31"); } +#endif // HAS_ARGBTOARGB1555ROW_LASX +#ifdef HAS_ARGBTOARGB4444ROW_LASX void ARGBToARGB4444Row_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { @@ -1702,7 +4592,9 @@ void ARGBToARGB4444Row_LASX(const uint8_t* src_argb, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); } +#endif // HAS_ARGBTOARGB4444ROW_LASX +#ifdef HAS_RGB565TOARGBROW_LASX void RGB565ToARGBRow_LASX(const uint8_t* src_rgb565, uint8_t* dst_argb, int width) { @@ -1749,7 +4641,9 @@ void RGB565ToARGBRow_LASX(const uint8_t* src_rgb565, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr31"); } +#endif // HAS_RGB565TOARGBROW_LASX +#ifdef HAS_ARGB1555TOARGBROW_LASX void ARGB1555ToARGBRow_LASX(const uint8_t* src_argb1555, uint8_t* dst_argb, int width) { @@ -1796,7 +4690,9 @@ void ARGB1555ToARGBRow_LASX(const uint8_t* src_argb1555, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr8", "$xr9", "$xr10", "$xr11", "$xr31"); } +#endif // HAS_ARGB1555TOARGBROW_LASX +#ifdef HAS_ARGB4444TOARGBROW_LASX void ARGB4444ToARGBRow_LASX(const uint8_t* src_argb4444, uint8_t* dst_argb, int width) { @@ -1827,7 +4723,9 @@ void ARGB4444ToARGBRow_LASX(const uint8_t* src_argb4444, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr31"); } +#endif // HAS_ARGB4444TOARGBROW_LASX +#ifdef HAS_ARGBMULTIPLYROW_LASX void ARGBMultiplyRow_LASX(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -1852,7 +4750,9 @@ void ARGBMultiplyRow_LASX(const uint8_t* src_argb, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); } +#endif // HAS_ARGBMULTIPLYROW_LASX +#ifdef HAS_ARGBADDROW_LASX void ARGBAddRow_LASX(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -1873,7 +4773,9 @@ void ARGBAddRow_LASX(const uint8_t* src_argb, : : "memory", "$xr0", "$xr1"); } +#endif // HAS_ARGBADDROW_LASX +#ifdef HAS_ARGBSUBTRACTROW_LASX void ARGBSubtractRow_LASX(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -1894,7 +4796,9 @@ void ARGBSubtractRow_LASX(const uint8_t* src_argb, : : "memory", "$xr0", "$xr1"); } +#endif // HAS_ARGBSUBTRACTROW_LASX +#ifdef HAS_ARGBATTENUATEROW_LASX void ARGBAttenuateRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { @@ -1923,7 +4827,9 @@ void ARGBAttenuateRow_LASX(const uint8_t* src_argb, : [shuf] "r"(kAlpha) : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6"); } +#endif // HAS_ARGBATTENUATEROW_LASX +#ifdef HAS_ARGBBLENDROW_LASX void ARGBBlendRow_LASX(const uint8_t* src_argb0, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -1965,7 +4871,9 @@ void ARGBBlendRow_LASX(const uint8_t* src_argb0, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8"); } +#endif // HAS_ARGBBLENDROW_LASX +#ifdef HAS_ARGBORROW_LASX void ARGBOrRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, uint32_t value, @@ -1984,7 +4892,9 @@ void ARGBOrRow_LASX(const uint8_t* src_argb, : [val] "r"(value) : "memory", "$xr0", "$xr1"); } +#endif // HAS_ARGBORROW_LASX +#ifdef HAS_ARGBALPHABLENDROW_LASX void ARGBAlphaBlendRow_LASX(const uint8_t* src_argb0, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -2048,7 +4958,9 @@ void ARGBAlphaBlendRow_LASX(const uint8_t* src_argb0, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12"); } +#endif // HAS_ARGBALPHABLENDROW_LASX +#ifdef HAS_ARGBSHADEROW_LASX void ARGBShadeRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width, @@ -2072,7 +4984,9 @@ void ARGBShadeRow_LASX(const uint8_t* src_argb, : [val] "r"(value) : "memory", "$xr0", "$xr1", "$xr2", "$xr4", "$xr5"); } +#endif // HAS_ARGBSHADEROW_LASX +#ifdef HAS_ARGBSHUFFLEROW_LASX void ARGBShuffleRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, const uint8_t* shuffler, @@ -2092,7 +5006,9 @@ void ARGBShuffleRow_LASX(const uint8_t* src_argb, : [shuf] "r"(shuffler) : "memory", "$xr0", "$xr4"); } +#endif // HAS_ARGBSHUFFLEROW_LASX +#ifdef HAS_ARGBGRAYROW_LASX void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { asm volatile( "xvldi $xr4, 29\n\t" @@ -2133,7 +5049,9 @@ void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8", "$xr31"); } +#endif // HAS_ARGBGRAYROW_LASX +#ifdef HAS_ARGBSEPIAROW_LASX void ARGBSepiaRow_LASX(uint8_t* dst_argb, int width) { asm volatile( "xvldi $xr4, 17\n\t" @@ -2192,7 +5110,9 @@ void ARGBSepiaRow_LASX(uint8_t* dst_argb, int width) { "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr31"); } +#endif // HAS_ARGBSEPIAROW_LASX +#ifdef HAS_ARGBCOLORMATRIXROW_LASX void ARGBColorMatrixRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, const int8_t* matrix_argb, @@ -2245,7 +5165,9 @@ void ARGBColorMatrixRow_LASX(const uint8_t* src_argb, : "memory", "$xr0", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr31"); } +#endif // HAS_ARGBCOLORMATRIXROW_LASX +#ifdef HAS_ARGBQUANTIZEROW_LASX void ARGBQuantizeRow_LASX(uint8_t* dst_argb, int scale, int interval_size, @@ -2279,7 +5201,9 @@ void ARGBQuantizeRow_LASX(uint8_t* dst_argb, : [scl] "r"(scale), [isz] "r"(interval_size), [ioff] "r"(interval_offset) : "memory", "$xr0", "$xr1", "$xr2", "$xr4", "$xr5", "$xr6", "$xr7"); } +#endif // HAS_ARGBQUANTIZEROW_LASX +#ifdef HAS_SOBELROW_LASX void SobelRow_LASX(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, @@ -2312,7 +5236,9 @@ void SobelRow_LASX(const uint8_t* src_sobelx, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); } +#endif // HAS_SOBELROW_LASX +#ifdef HAS_SOBELTOPLANEROW_LASX void SobelToPlaneRow_LASX(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, @@ -2333,7 +5259,9 @@ void SobelToPlaneRow_LASX(const uint8_t* src_sobelx, : : "memory", "$xr0", "$xr1"); } +#endif // HAS_SOBELTOPLANEROW_LASX +#ifdef HAS_SOBELXYROW_LASX void SobelXYRow_LASX(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, @@ -2366,7 +5294,9 @@ void SobelXYRow_LASX(const uint8_t* src_sobelx, : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); } +#endif // HAS_SOBELXYROW_LASX +#ifdef HAS_INTERPOLATEROW_LASX void InterpolateRow_LASX(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -2420,7 +5350,9 @@ void InterpolateRow_LASX(uint8_t* dst_ptr, : [yf] "r"(y_frac) : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); } +#endif // HAS_INTERPOLATEROW_LASX +#ifdef HAS_HALFFLOATROW_LASX void HalfFloatRow_LASX(const uint16_t* src, uint16_t* dst, float scale, @@ -2450,7 +5382,9 @@ void HalfFloatRow_LASX(const uint16_t* src, : [scale] "r"(&scale), [bias] "r"(&bias) : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5"); } +#endif // HAS_HALFFLOATROW_LASX +#ifdef HAS_ARGBTOYMATRIXROW_LASX void ARGBToYMatrixRow_LASX(const uint8_t* src_argb, uint8_t* dst_y, int width, @@ -2487,7 +5421,9 @@ void ARGBToYMatrixRow_LASX(const uint8_t* src_argb, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr31"); } +#endif // HAS_ARGBTOYMATRIXROW_LASX +#ifdef HAS_RGBTOYMATRIXROW_LASX void RGBToYMatrixRow_LASX(const uint8_t* src_rgba, uint8_t* dst_y, int width, @@ -2544,7 +5480,9 @@ void RGBToYMatrixRow_LASX(const uint8_t* src_rgba, : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr31"); } +#endif // HAS_RGBTOYMATRIXROW_LASX +#ifdef HAS_RGB565TOYROW_LASX void RGB565ToYRow_LASX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { asm volatile( "xvldi $xr6, 31\n\t" @@ -2591,7 +5529,9 @@ void RGB565ToYRow_LASX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr6", "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr31"); } +#endif // HAS_RGB565TOYROW_LASX +#ifdef HAS_ARGB1555TOYROW_LASX void ARGB1555ToYRow_LASX(const uint8_t* src_argb1555, uint8_t* dst_y, int width) { @@ -2614,7 +5554,6 @@ void ARGB1555ToYRow_LASX(const uint8_t* src_argb1555, "xvand.v $xr2, $xr2, $xr6\n\t" "xvsrli.h $xr3, $xr0, 10\n\t" "xvand.v $xr3, $xr3, $xr6\n\t" - "xvsrai.h $xr5, $xr0, 15\n\t" "xvslli.h $xr4, $xr1, 3\n\t" "xvsrli.h $xr1, $xr1, 2\n\t" "xvor.v $xr1, $xr4, $xr1\n\t" @@ -2637,10 +5576,12 @@ void ARGB1555ToYRow_LASX(const uint8_t* src_argb1555, "bnez %[cnt], 1b\n\t" : [src] "+r"(src_argb1555), [dst] "+r"(dst_y), [cnt] "+r"(width) : - : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", - "$xr8", "$xr9", "$xr10", "$xr11", "$xr31"); + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr6", "$xr8", + "$xr9", "$xr10", "$xr11", "$xr31"); } +#endif // HAS_ARGB1555TOYROW_LASX +#ifdef HAS_RGB565TOUVROW_LASX void RGB565ToUVRow_LASX(const uint8_t* src_rgb565, int src_stride_rgb565, uint8_t* dst_u, @@ -2757,24 +5698,20 @@ void RGB565ToUVRow_LASX(const uint8_t* src_rgb565, "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t" "xvadd.h $xr2, $xr2, $xr5\n\t" "xvsrari.h $xr2, $xr2, 2\n\t" - "xvor.v $xr6, $xr25, $xr25\n\t" - "xvor.v $xr7, $xr25, $xr25\n\t" - "xvmadd.h $xr6, $xr0, $xr20\n\t" - "xvmsub.h $xr6, $xr1, $xr21\n\t" - "xvmsub.h $xr6, $xr2, $xr22\n\t" - "xvmadd.h $xr7, $xr2, $xr20\n\t" - "xvmsub.h $xr7, $xr1, $xr23\n\t" - "xvmsub.h $xr7, $xr0, $xr24\n\t" - "xvpickod.b $xr6, $xr7, $xr6\n\t" - "xvpermi.d $xr31, $xr6, 0xd8\n\t" + "xvor.v $xr8, $xr25, $xr25\n\t" + "xvor.v $xr9, $xr25, $xr25\n\t" + "xvmadd.h $xr8, $xr0, $xr20\n\t" + "xvmsub.h $xr8, $xr1, $xr21\n\t" + "xvmsub.h $xr8, $xr2, $xr22\n\t" + "xvmadd.h $xr9, $xr2, $xr20\n\t" + "xvmsub.h $xr9, $xr1, $xr23\n\t" + "xvmsub.h $xr9, $xr0, $xr24\n\t" + "xvpickod.b $xr8, $xr9, $xr8\n\t" + "xvpermi.d $xr31, $xr8, 0xd8\n\t" "vst $vr31, %[du], 0\n\t" - "vstelm.d $vr6, %[dv], 0, 1\n\t" - "xvpermi.q $xr31, $xr6, 0x01\n\t" + "vstelm.d $vr8, %[dv], 0, 1\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" "vstelm.d $vr31, %[dv], 8, 1\n\t" - "xvldi $xr6, 31\n\t" - "xvsrli.h $xr6, $xr6, 8\n\t" - "xvldi $xr7, 63\n\t" - "xvsrli.h $xr7, $xr7, 8\n\t" "addi.d %[s0], %[s0], 64\n\t" "addi.d %[s1], %[s1], 64\n\t" "addi.d %[du], %[du], 16\n\t" @@ -2785,10 +5722,12 @@ void RGB565ToUVRow_LASX(const uint8_t* src_rgb565, [dv] "+r"(dst_v), [cnt] "+r"(width) : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", - "$xr7", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr20", - "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", "$xr31"); + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", "$xr31"); } +#endif // HAS_RGB565TOUVROW_LASX +#ifdef HAS_ARGB1555TOUVROW_LASX void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555, int src_stride_argb1555, uint8_t* dst_u, @@ -2820,7 +5759,6 @@ void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555, "xvand.v $xr2, $xr2, $xr6\n\t" "xvsrli.h $xr3, $xr0, 10\n\t" "xvand.v $xr3, $xr3, $xr6\n\t" - "xvsrai.h $xr5, $xr0, 15\n\t" "xvslli.h $xr4, $xr1, 3\n\t" "xvsrli.h $xr1, $xr1, 2\n\t" "xvor.v $xr1, $xr4, $xr1\n\t" @@ -2841,7 +5779,6 @@ void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555, "xvand.v $xr2, $xr2, $xr6\n\t" "xvsrli.h $xr3, $xr0, 10\n\t" "xvand.v $xr3, $xr3, $xr6\n\t" - "xvsrai.h $xr5, $xr0, 15\n\t" "xvslli.h $xr4, $xr1, 3\n\t" "xvsrli.h $xr1, $xr1, 2\n\t" "xvor.v $xr1, $xr4, $xr1\n\t" @@ -2862,7 +5799,6 @@ void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555, "xvand.v $xr2, $xr2, $xr6\n\t" "xvsrli.h $xr3, $xr0, 10\n\t" "xvand.v $xr3, $xr3, $xr6\n\t" - "xvsrai.h $xr5, $xr0, 15\n\t" "xvslli.h $xr4, $xr1, 3\n\t" "xvsrli.h $xr1, $xr1, 2\n\t" "xvor.v $xr1, $xr4, $xr1\n\t" @@ -2883,7 +5819,6 @@ void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555, "xvand.v $xr2, $xr2, $xr6\n\t" "xvsrli.h $xr3, $xr0, 10\n\t" "xvand.v $xr3, $xr3, $xr6\n\t" - "xvsrai.h $xr5, $xr0, 15\n\t" "xvslli.h $xr4, $xr1, 3\n\t" "xvsrli.h $xr1, $xr1, 2\n\t" "xvor.v $xr1, $xr4, $xr1\n\t" @@ -2911,22 +5846,20 @@ void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555, "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t" "xvadd.h $xr2, $xr2, $xr5\n\t" "xvsrari.h $xr2, $xr2, 2\n\t" - "xvor.v $xr6, $xr25, $xr25\n\t" - "xvor.v $xr7, $xr25, $xr25\n\t" - "xvmadd.h $xr6, $xr0, $xr20\n\t" - "xvmsub.h $xr6, $xr1, $xr21\n\t" - "xvmsub.h $xr6, $xr2, $xr22\n\t" - "xvmadd.h $xr7, $xr2, $xr20\n\t" - "xvmsub.h $xr7, $xr1, $xr23\n\t" - "xvmsub.h $xr7, $xr0, $xr24\n\t" - "xvpickod.b $xr6, $xr7, $xr6\n\t" - "xvpermi.d $xr31, $xr6, 0xd8\n\t" + "xvor.v $xr8, $xr25, $xr25\n\t" + "xvor.v $xr9, $xr25, $xr25\n\t" + "xvmadd.h $xr8, $xr0, $xr20\n\t" + "xvmsub.h $xr8, $xr1, $xr21\n\t" + "xvmsub.h $xr8, $xr2, $xr22\n\t" + "xvmadd.h $xr9, $xr2, $xr20\n\t" + "xvmsub.h $xr9, $xr1, $xr23\n\t" + "xvmsub.h $xr9, $xr0, $xr24\n\t" + "xvpickod.b $xr8, $xr9, $xr8\n\t" + "xvpermi.d $xr31, $xr8, 0xd8\n\t" "vst $vr31, %[du], 0\n\t" - "vstelm.d $vr6, %[dv], 0, 1\n\t" - "xvpermi.q $xr31, $xr6, 0x01\n\t" + "vstelm.d $vr8, %[dv], 0, 1\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" "vstelm.d $vr31, %[dv], 8, 1\n\t" - "xvldi $xr6, 31\n\t" - "xvsrli.h $xr6, $xr6, 8\n\t" "addi.d %[s0], %[s0], 64\n\t" "addi.d %[s1], %[s1], 64\n\t" "addi.d %[du], %[du], 16\n\t" @@ -2937,151 +5870,12 @@ void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555, [dv] "+r"(dst_v), [cnt] "+r"(width) : : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", - "$xr7", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr20", - "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", "$xr31"); -} - -static LIBYUV_NOINLINE void XRGBToUVHelper_LASX(const uint8_t* src, - int stride, - uint8_t* dst_u, - uint8_t* dst_v, - const uint8_t* shuf, - int width) { - const uint8_t* next = src + stride; - asm volatile( - "xvldi $xr20, 112\n\t" - "xvsrli.h $xr20, $xr20, 8\n\t" - "xvldi $xr21, 74\n\t" - "xvsrli.h $xr21, $xr21, 8\n\t" - "xvldi $xr22, 38\n\t" - "xvsrli.h $xr22, $xr22, 8\n\t" - "xvldi $xr23, 94\n\t" - "xvsrli.h $xr23, $xr23, 8\n\t" - "xvldi $xr24, 18\n\t" - "xvsrli.h $xr24, $xr24, 8\n\t" - "xvldi $xr25, 128\n\t" - "xvsrli.h $xr25, $xr25, 8\n\t" - "xvslli.h $xr25, $xr25, 8\n\t" - "vld $vr16, %[shuf], 0\n\t" - "xvpermi.q $xr16, $xr16, 0x00\n\t" - "1:\n\t" - "vld $vr8, %[s0], 0\n\t" - "vld $vr31, %[s0], 64\n\t" - "xvpermi.q $xr8, $xr31, 0x02\n\t" - "vld $vr9, %[s0], 16\n\t" - "vld $vr31, %[s0], 80\n\t" - "xvpermi.q $xr9, $xr31, 0x02\n\t" - "vld $vr10, %[s0], 32\n\t" - "vld $vr31, %[s0], 96\n\t" - "xvpermi.q $xr10, $xr31, 0x02\n\t" - "vld $vr11, %[s0], 48\n\t" - "vld $vr31, %[s0], 112\n\t" - "xvpermi.q $xr11, $xr31, 0x02\n\t" - "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t" - "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t" - "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t" - "xvshuf.b $xr11, $xr11, $xr11, $xr16\n\t" - "xvpickev.b $xr12, $xr9, $xr8\n\t" - "xvpickod.b $xr13, $xr9, $xr8\n\t" - "xvpickev.b $xr14, $xr11, $xr10\n\t" - "xvpickod.b $xr15, $xr11, $xr10\n\t" - "xvpickev.b $xr0, $xr14, $xr12\n\t" - "xvpickod.b $xr2, $xr14, $xr12\n\t" - "xvpickev.b $xr1, $xr15, $xr13\n\t" - "vld $vr8, %[s1], 0\n\t" - "vld $vr31, %[s1], 64\n\t" - "xvpermi.q $xr8, $xr31, 0x02\n\t" - "vld $vr9, %[s1], 16\n\t" - "vld $vr31, %[s1], 80\n\t" - "xvpermi.q $xr9, $xr31, 0x02\n\t" - "vld $vr10, %[s1], 32\n\t" - "vld $vr31, %[s1], 96\n\t" - "xvpermi.q $xr10, $xr31, 0x02\n\t" - "vld $vr11, %[s1], 48\n\t" - "vld $vr31, %[s1], 112\n\t" - "xvpermi.q $xr11, $xr31, 0x02\n\t" - "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t" - "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t" - "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t" - "xvshuf.b $xr11, $xr11, $xr11, $xr16\n\t" - "xvpickev.b $xr12, $xr9, $xr8\n\t" - "xvpickod.b $xr13, $xr9, $xr8\n\t" - "xvpickev.b $xr14, $xr11, $xr10\n\t" - "xvpickod.b $xr15, $xr11, $xr10\n\t" - "xvpickev.b $xr3, $xr14, $xr12\n\t" - "xvpickod.b $xr5, $xr14, $xr12\n\t" - "xvpickev.b $xr4, $xr15, $xr13\n\t" - "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t" - "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t" - "xvadd.h $xr0, $xr0, $xr3\n\t" - "xvsrari.h $xr0, $xr0, 2\n\t" - "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t" - "xvhaddw.hu.bu $xr4, $xr4, $xr4\n\t" - "xvadd.h $xr1, $xr1, $xr4\n\t" - "xvsrari.h $xr1, $xr1, 2\n\t" - "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t" - "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t" - "xvadd.h $xr2, $xr2, $xr5\n\t" - "xvsrari.h $xr2, $xr2, 2\n\t" - "xvor.v $xr6, $xr25, $xr25\n\t" - "xvor.v $xr7, $xr25, $xr25\n\t" - "xvmadd.h $xr6, $xr0, $xr20\n\t" - "xvmsub.h $xr6, $xr1, $xr21\n\t" - "xvmsub.h $xr6, $xr2, $xr22\n\t" - "xvmadd.h $xr7, $xr2, $xr20\n\t" - "xvmsub.h $xr7, $xr1, $xr23\n\t" - "xvmsub.h $xr7, $xr0, $xr24\n\t" - "xvpickod.b $xr6, $xr7, $xr6\n\t" - "xvpermi.d $xr31, $xr6, 0xd8\n\t" - "vst $vr31, %[du], 0\n\t" - "vstelm.d $vr6, %[dv], 0, 1\n\t" - "xvpermi.q $xr31, $xr6, 0x01\n\t" - "vstelm.d $vr31, %[dv], 8, 1\n\t" - "addi.d %[s0], %[s0], 128\n\t" - "addi.d %[s1], %[s1], 128\n\t" - "addi.d %[du], %[du], 16\n\t" - "addi.d %[dv], %[dv], 16\n\t" - "addi.d %[cnt], %[cnt], -32\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v), - [cnt] "+r"(width) - : [shuf] "r"(shuf) - : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", - "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", - "$xr15", "$xr16", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", - "$xr31"); -} - -void BGRAToUVRow_LASX(const uint8_t* src_bgra, - int src_stride_bgra, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - static const uint8_t kShuf[16] = {3, 2, 1, 0, 7, 6, 5, 4, - 11, 10, 9, 8, 15, 14, 13, 12}; - XRGBToUVHelper_LASX(src_bgra, src_stride_bgra, dst_u, dst_v, kShuf, width); -} - -void ABGRToUVRow_LASX(const uint8_t* src_abgr, - int src_stride_abgr, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - static const uint8_t kShuf[16] = {2, 1, 0, 3, 6, 5, 4, 7, - 10, 9, 8, 11, 14, 13, 12, 15}; - XRGBToUVHelper_LASX(src_abgr, src_stride_abgr, dst_u, dst_v, kShuf, width); -} - -void RGBAToUVRow_LASX(const uint8_t* src_rgba, - int src_stride_rgba, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - static const uint8_t kShuf[16] = {1, 2, 3, 0, 5, 6, 7, 4, - 9, 10, 11, 8, 13, 14, 15, 12}; - XRGBToUVHelper_LASX(src_rgba, src_stride_rgba, dst_u, dst_v, kShuf, width); + "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", + "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", "$xr31"); } +#endif // HAS_ARGB1555TOUVROW_LASX +#if defined(HAS_RGB24TOUVROW_LASX) || defined(HAS_RAWTOUVROW_LASX) static LIBYUV_NOINLINE void RGB24ToUVHelper_LASX(const uint8_t* src, int stride, uint8_t* dst_u, @@ -3194,7 +5988,9 @@ static LIBYUV_NOINLINE void RGB24ToUVHelper_LASX(const uint8_t* src, "$xr15", "$xr16", "$xr17", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", "$xr31"); } +#endif // defined(HAS_RGB24TOUVROW_LASX) || defined(HAS_RAWTOUVROW_LASX) +#ifdef HAS_RGB24TOUVROW_LASX void RGB24ToUVRow_LASX(const uint8_t* src_rgb24, int src_stride_rgb24, uint8_t* dst_u, @@ -3205,7 +6001,9 @@ void RGB24ToUVRow_LASX(const uint8_t* src_rgb24, 9, 0, 10, 11, 12, 0, 13, 14, 15, 0}; RGB24ToUVHelper_LASX(src_rgb24, src_stride_rgb24, dst_u, dst_v, kShuf, width); } +#endif // HAS_RGB24TOUVROW_LASX +#ifdef HAS_RAWTOUVROW_LASX void RAWToUVRow_LASX(const uint8_t* src_raw, int src_stride_raw, uint8_t* dst_u, @@ -3216,343 +6014,12 @@ void RAWToUVRow_LASX(const uint8_t* src_raw, 7, 0, 12, 11, 10, 0, 15, 14, 13, 0}; RGB24ToUVHelper_LASX(src_raw, src_stride_raw, dst_u, dst_v, kShuf, width); } - -#ifdef HAS_SSIM4X4ROW_LASX -void Ssim4x4Row_LASX(const uint8_t* src_a, - int stride_a, - const uint8_t* src_b, - int stride_b, - uint32_t* dst, - int width) { - const uint8_t* pa; - const uint8_t* pb; - int r; - asm volatile( - "1:\n\t" - "move %[pa], %[sa]\n\t" - "move %[pb], %[sb]\n\t" - "li.d %[r], 4\n\t" - "xvxor.v $xr8, $xr8, $xr8\n\t" - "xvxor.v $xr9, $xr9, $xr9\n\t" - "xvxor.v $xr10, $xr10, $xr10\n\t" - "xvxor.v $xr11, $xr11, $xr11\n\t" - "2:\n\t" - "xvld $xr0, %[pa], 0\n\t" - "xvld $xr1, %[pb], 0\n\t" - "add.d %[pa], %[pa], %[sta]\n\t" - "add.d %[pb], %[pb], %[stb]\n\t" - "xvhaddw.hu.bu $xr2, $xr0, $xr0\n\t" - "xvhaddw.hu.bu $xr3, $xr1, $xr1\n\t" - "xvadd.h $xr8, $xr8, $xr2\n\t" - "xvadd.h $xr9, $xr9, $xr3\n\t" - "xvmulwev.h.bu $xr2, $xr0, $xr1\n\t" - "xvmulwod.h.bu $xr3, $xr0, $xr1\n\t" - "xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t" - "xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t" - "xvadd.w $xr11, $xr11, $xr2\n\t" - "xvadd.w $xr11, $xr11, $xr3\n\t" - "xvmulwev.h.bu $xr2, $xr0, $xr0\n\t" - "xvmulwod.h.bu $xr3, $xr0, $xr0\n\t" - "xvmulwev.h.bu $xr4, $xr1, $xr1\n\t" - "xvmulwod.h.bu $xr5, $xr1, $xr1\n\t" - "xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t" - "xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t" - "xvhaddw.wu.hu $xr4, $xr4, $xr4\n\t" - "xvhaddw.wu.hu $xr5, $xr5, $xr5\n\t" - "xvadd.w $xr2, $xr2, $xr3\n\t" - "xvadd.w $xr4, $xr4, $xr5\n\t" - "xvadd.w $xr10, $xr10, $xr2\n\t" - "xvadd.w $xr10, $xr10, $xr4\n\t" - "addi.d %[r], %[r], -1\n\t" - "bnez %[r], 2b\n\t" - "xvhaddw.wu.hu $xr8, $xr8, $xr8\n\t" - "xvhaddw.wu.hu $xr9, $xr9, $xr9\n\t" - "xvilvl.w $xr0, $xr9, $xr8\n\t" - "xvilvh.w $xr1, $xr9, $xr8\n\t" - "xvilvl.w $xr2, $xr11, $xr10\n\t" - "xvilvh.w $xr3, $xr11, $xr10\n\t" - "xvilvl.d $xr4, $xr2, $xr0\n\t" - "xvilvh.d $xr5, $xr2, $xr0\n\t" - "xvilvl.d $xr6, $xr3, $xr1\n\t" - "xvilvh.d $xr7, $xr3, $xr1\n\t" - "xvor.v $xr0, $xr5, $xr5\n\t" - "xvpermi.q $xr0, $xr4, 0x20\n\t" - "xvpermi.q $xr5, $xr4, 0x31\n\t" - "xvor.v $xr1, $xr7, $xr7\n\t" - "xvpermi.q $xr1, $xr6, 0x20\n\t" - "xvpermi.q $xr7, $xr6, 0x31\n\t" - "xvst $xr0, %[dst], 0\n\t" - "xvst $xr1, %[dst], 32\n\t" - "xvst $xr5, %[dst], 64\n\t" - "xvst $xr7, %[dst], 96\n\t" - "addi.d %[sa], %[sa], 32\n\t" - "addi.d %[sb], %[sb], 32\n\t" - "addi.d %[dst], %[dst], 128\n\t" - "addi.d %[cnt], %[cnt], -32\n\t" - "bnez %[cnt], 1b\n\t" - : [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst), [cnt] "+r"(width), - [pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r) - : [sta] "r"((ptrdiff_t)stride_a), [stb] "r"((ptrdiff_t)stride_b) - : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", - "$xr7", "$xr8", "$xr9", "$xr10", "$xr11"); -} -#endif // HAS_SSIM4X4ROW_LASX - -#ifdef HAS_SSIMWINDOWROW_LASX -float SsimWindowRow_LASX(const uint32_t* row0, - const uint32_t* row1, - int num_windows) { - const uint32_t* row0_next = row0 + 4; - const uint32_t* row1_next = row1 + 4; - float total; - uint32_t c1 = 26634; - uint32_t c2 = 239708; - uint32_t c0 = 0; - asm volatile( - "xvreplgr2vr.w $xr6, %[c0]\n\t" - "xvreplgr2vr.w $xr4, %[c1]\n\t" - "xvreplgr2vr.w $xr5, %[c2]\n\t" - "addi.d %[cnt], %[cnt], -2\n\t" - "bltz %[cnt], 2f\n\t" - "1:\n\t" - "xvld $xr0, %[r0], 0\n\t" - "xvld $xr1, %[r0n], 0\n\t" - "xvld $xr2, %[r1], 0\n\t" - "xvld $xr3, %[r1n], 0\n\t" - "xvadd.w $xr0, $xr0, $xr1\n\t" - "xvadd.w $xr2, $xr2, $xr3\n\t" - "xvadd.w $xr0, $xr0, $xr2\n\t" - "xvshuf4i.w $xr1, $xr0, 0x00\n\t" - "xvshuf4i.w $xr2, $xr0, 0x55\n\t" - "xvshuf4i.w $xr3, $xr0, 0xaa\n\t" - "xvshuf4i.w $xr0, $xr0, 0xff\n\t" - "xvmul.w $xr7, $xr1, $xr2\n\t" - "xvslli.w $xr7, $xr7, 1\n\t" - "xvmul.w $xr1, $xr1, $xr1\n\t" - "xvmadd.w $xr1, $xr2, $xr2\n\t" - "xvslli.w $xr0, $xr0, 7\n\t" - "xvsub.w $xr0, $xr0, $xr7\n\t" - "xvadd.w $xr7, $xr7, $xr4\n\t" - "xvadd.w $xr0, $xr0, $xr5\n\t" - "xvslli.w $xr3, $xr3, 6\n\t" - "xvsub.w $xr3, $xr3, $xr1\n\t" - "xvadd.w $xr1, $xr1, $xr4\n\t" - "xvadd.w $xr3, $xr3, $xr5\n\t" - "xvffint.s.w $xr7, $xr7\n\t" - "xvffint.s.w $xr0, $xr0\n\t" - "xvffint.s.w $xr1, $xr1\n\t" - "xvffint.s.w $xr3, $xr3\n\t" - "xvfmul.s $xr7, $xr7, $xr0\n\t" - "xvfmul.s $xr1, $xr1, $xr3\n\t" - "xvfdiv.s $xr7, $xr7, $xr1\n\t" - "xvfadd.s $xr6, $xr6, $xr7\n\t" - "addi.d %[r0], %[r0], 32\n\t" - "addi.d %[r0n], %[r0n], 32\n\t" - "addi.d %[r1], %[r1], 32\n\t" - "addi.d %[r1n], %[r1n], 32\n\t" - "addi.d %[cnt], %[cnt], -2\n\t" - "bgez %[cnt], 1b\n\t" - "xvpermi.q $xr0, $xr6, 0x01\n\t" - "vfadd.s $vr6, $vr6, $vr0\n\t" - "2:\n\t" - "addi.d %[cnt], %[cnt], 2\n\t" - "beqz %[cnt], 3f\n\t" - "vld $vr0, %[r0], 0\n\t" - "vld $vr1, %[r0n], 0\n\t" - "vld $vr2, %[r1], 0\n\t" - "vld $vr3, %[r1n], 0\n\t" - "vadd.w $vr0, $vr0, $vr1\n\t" - "vadd.w $vr2, $vr2, $vr3\n\t" - "vadd.w $vr0, $vr0, $vr2\n\t" - "vshuf4i.w $vr1, $vr0, 0x00\n\t" - "vshuf4i.w $vr2, $vr0, 0x55\n\t" - "vshuf4i.w $vr3, $vr0, 0xaa\n\t" - "vshuf4i.w $vr0, $vr0, 0xff\n\t" - "vmul.w $vr7, $vr1, $vr2\n\t" - "vslli.w $vr7, $vr7, 1\n\t" - "vmul.w $vr1, $vr1, $vr1\n\t" - "vmadd.w $vr1, $vr2, $vr2\n\t" - "vslli.w $vr0, $vr0, 7\n\t" - "vsub.w $vr0, $vr0, $vr7\n\t" - "vadd.w $vr7, $vr7, $vr4\n\t" - "vadd.w $vr0, $vr0, $vr5\n\t" - "vslli.w $vr3, $vr3, 6\n\t" - "vsub.w $vr3, $vr3, $vr1\n\t" - "vadd.w $vr1, $vr1, $vr4\n\t" - "vadd.w $vr3, $vr3, $vr5\n\t" - "vffint.s.w $vr7, $vr7\n\t" - "vffint.s.w $vr0, $vr0\n\t" - "vffint.s.w $vr1, $vr1\n\t" - "vffint.s.w $vr3, $vr3\n\t" - "vfmul.s $vr7, $vr7, $vr0\n\t" - "vfmul.s $vr1, $vr1, $vr3\n\t" - "vfdiv.s $vr7, $vr7, $vr1\n\t" - "vfadd.s $vr6, $vr6, $vr7\n\t" - "3:\n\t" - "vstelm.w $vr6, %[tot], 0, 0\n\t" - : [r0] "+r"(row0), [r0n] "+r"(row0_next), [r1] "+r"(row1), - [r1n] "+r"(row1_next), [cnt] "+r"(num_windows) - : [c0] "r"(c0), [c1] "r"(c1), [c2] "r"(c2), [tot] "r"(&total) - : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", - "$xr7"); - return total; -} -#endif // HAS_SSIMWINDOWROW_LASX - -#ifdef HAS_ARGBDETECTROW_LASX -uint32_t ARGBDetectRow_LASX(const uint8_t* argb, int width) { - uint32_t diff = 0; - uint32_t tmp; - uint32_t rgb_mask = 0x00ffff00u; - asm volatile( - "xvreplgr2vr.w $xr2, %[rgb_mask]\n\t" - "xvldi $xr3, -1\n\t" - "addi.d %[w], %[w], -8\n\t" - "bltz %[w], 2f\n\t" - "1:\n\t" - "xvld $xr0, %[argb], 0\n\t" - "addi.d %[argb], %[argb], 32\n\t" - "xvor.v $xr0, $xr0, $xr2\n\t" - "xvseq.b $xr0, $xr0, $xr3\n\t" - "xvmskltz.b $xr0, $xr0\n\t" - "xvpickve2gr.wu %[diff], $xr0, 0\n\t" - "xvpickve2gr.wu %[tmp], $xr0, 4\n\t" - "bstrins.w %[diff], %[tmp], 31, 16\n\t" - "nor %[diff], %[diff], $zero\n\t" - "bnez %[diff], 4f\n\t" - "addi.d %[w], %[w], -8\n\t" - "bgez %[w], 1b\n\t" - "2:\n\t" - "addi.d %[w], %[w], 8\n\t" - "beqz %[w], 4f\n\t" - "3:\n\t" - "vldrepl.w $vr0, %[argb], 0\n\t" - "addi.d %[argb], %[argb], 4\n\t" - "vor.v $vr0, $vr0, $vr2\n\t" - "vseq.b $vr0, $vr0, $vr3\n\t" - "vmskltz.b $vr0, $vr0\n\t" - "vpickve2gr.hu %[diff], $vr0, 0\n\t" - "nor %[diff], %[diff], $zero\n\t" - "bstrpick.w %[diff], %[diff], 15, 0\n\t" - "bnez %[diff], 4f\n\t" - "addi.d %[w], %[w], -1\n\t" - "bnez %[w], 3b\n\t" - "4:\n\t" - : [argb] "+r"(argb), [w] "+r"(width), [diff] "+r"(diff), [tmp] "=&r"(tmp) - : [rgb_mask] "r"(rgb_mask) - : "memory", "$xr0", "$xr2", "$xr3"); - if (diff) { - return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB; - } - return 0; -} -#endif // HAS_ARGBDETECTROW_LASX - -#ifdef HAS_HAMMINGDISTANCE_LASX -uint32_t HammingDistance_LASX(const uint8_t* src_a, - const uint8_t* src_b, - int count) { - uint32_t diff; - asm volatile( - "xvxor.v $xr4, $xr4, $xr4\n\t" - "1:\n\t" - "xvld $xr0, %[sa], 0\n\t" - "xvld $xr1, %[sb], 0\n\t" - "xvxor.v $xr0, $xr0, $xr1\n\t" - "xvpcnt.b $xr0, $xr0\n\t" - "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t" - "xvhaddw.wu.hu $xr0, $xr0, $xr0\n\t" - "xvadd.w $xr4, $xr4, $xr0\n\t" - "addi.d %[sa], %[sa], 32\n\t" - "addi.d %[sb], %[sb], 32\n\t" - "addi.d %[cnt], %[cnt], -32\n\t" - "bnez %[cnt], 1b\n\t" - "xvhaddw.du.wu $xr4, $xr4, $xr4\n\t" - "xvhaddw.qu.du $xr4, $xr4, $xr4\n\t" - "xvpermi.q $xr0, $xr4, 0x01\n\t" - "vadd.w $vr4, $vr4, $vr0\n\t" - "vpickve2gr.wu %[diff], $vr4, 0\n\t" - : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [diff] "=r"(diff) - : - : "memory", "$xr0", "$xr1", "$xr4"); - return diff; -} -#endif // HAS_HAMMINGDISTANCE_LASX - -#ifdef HAS_SUMSQUAREERROR_LASX -uint32_t SumSquareError_LASX(const uint8_t* src_a, - const uint8_t* src_b, - int count) { - uint32_t sse; - asm volatile( - "xvxor.v $xr4, $xr4, $xr4\n\t" - "1:\n\t" - "xvld $xr0, %[sa], 0\n\t" - "xvld $xr1, %[sb], 0\n\t" - "xvabsd.bu $xr0, $xr0, $xr1\n\t" - "xvmulwev.h.bu $xr2, $xr0, $xr0\n\t" - "xvmulwod.h.bu $xr3, $xr0, $xr0\n\t" - "xvhaddw.wu.hu $xr2, $xr2, $xr2\n\t" - "xvhaddw.wu.hu $xr3, $xr3, $xr3\n\t" - "xvadd.w $xr4, $xr4, $xr2\n\t" - "xvadd.w $xr4, $xr4, $xr3\n\t" - "addi.d %[sa], %[sa], 32\n\t" - "addi.d %[sb], %[sb], 32\n\t" - "addi.d %[cnt], %[cnt], -32\n\t" - "bnez %[cnt], 1b\n\t" - "xvhaddw.du.wu $xr4, $xr4, $xr4\n\t" - "xvhaddw.qu.du $xr4, $xr4, $xr4\n\t" - "xvpermi.q $xr0, $xr4, 0x01\n\t" - "vadd.w $vr4, $vr4, $vr0\n\t" - "vpickve2gr.wu %[sse], $vr4, 0\n\t" - : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [sse] "=r"(sse) - : - : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); - return sse; -} -#endif // HAS_SUMSQUAREERROR_LASX - -#ifdef HAS_HASHDJB2_LASX -static const uint32_t kDjb2Muls_LASX[16] = { - 0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, 0x30f35d61, 0x855cb541, - 0x040a9121, 0x747c7101, 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881, - 0x00008c61, 0x00000441, 0x00000021, 0x00000001, -}; - -uint32_t HashDjb2_LASX(const uint8_t* src, int count, uint32_t seed) { - uint32_t hash = seed; - uint32_t tmp; - uint32_t c16 = 0x92d9e201u; - asm volatile( - "xvld $xr4, %[muls], 0\n\t" - "xvld $xr5, %[muls], 32\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vbsrl.v $vr1, $vr0, 8\n\t" - "vext2xv.wu.bu $xr0, $xr0\n\t" - "vext2xv.wu.bu $xr1, $xr1\n\t" - "xvmul.w $xr0, $xr0, $xr4\n\t" - "xvmadd.w $xr0, $xr1, $xr5\n\t" - "xvhaddw.du.wu $xr0, $xr0, $xr0\n\t" - "xvhaddw.qu.du $xr0, $xr0, $xr0\n\t" - "xvpermi.q $xr1, $xr0, 0x01\n\t" - "vadd.w $vr0, $vr0, $vr1\n\t" - "vpickve2gr.wu %[tmp], $vr0, 0\n\t" - "mul.w %[hash], %[hash], %[c16]\n\t" - "add.w %[hash], %[hash], %[tmp]\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src), [cnt] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp) - : [muls] "r"(kDjb2Muls_LASX), [c16] "r"(c16) - : "memory", "$xr0", "$xr1", "$xr4", "$xr5"); - return hash; -} -#endif // HAS_HASHDJB2_LASX +#endif // HAS_RAWTOUVROW_LASX #ifdef __cplusplus } // extern "C" } // namespace libyuv #endif -#endif // !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#endif // (!defined(LIBYUV_DISABLE_LSX) || !defined(LIBYUV_DISABLE_LASX)) && + // defined(__loongarch_lp64) diff --git a/source/row_lsx.cc b/source/row_lsx.cc deleted file mode 100644 index cb0ad4137..000000000 --- a/source/row_lsx.cc +++ /dev/null @@ -1,3112 +0,0 @@ -/* - * Copyright 2022 The LibYuv Project Authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "libyuv/compare_row.h" -#include "libyuv/row.h" -#include "libyuv/video_common.h" - -// Note: every register written by an inline asm block (vector, general -// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be -// listed in its clobber list, even when the ABI treats it as caller-saved. -// With LTO/LTCG the compiler may keep values live in any register across the -// asm statement, and other OS ABIs (e.g. Windows) differ in which registers -// are callee-saved. List "memory" first, then "cc", then registers. - -#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) - -#include "libyuv/basic_types.h" - -#ifdef __cplusplus -namespace libyuv { -extern "C" { -#endif - -void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int count) { - asm volatile( - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(count) - : - : "memory", "$vr0"); -} - -void SetRow_LSX(uint8_t* dst, uint8_t v8, int count) { - asm volatile( - "vreplgr2vr.b $vr0, %[val]\n\t" - "1:\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [dst] "+r"(dst), [cnt] "+r"(count) - : [val] "r"(v8) - : "memory", "$vr0"); -} - -void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width) { - asm volatile( - "vreplgr2vr.w $vr0, %[val]\n\t" - "1:\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [val] "r"(v32) - : "memory", "$vr0"); -} - -void MirrorRow_LSX(const uint8_t* src_y, uint8_t* dst_y, int width) { - static const uint8_t kRev[16] = {15, 14, 13, 12, 11, 10, 9, 8, - 7, 6, 5, 4, 3, 2, 1, 0}; - src_y += width - 16; - asm volatile( - "vld $vr1, %[rev], 0\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vshuf.b $vr0, $vr0, $vr0, $vr1\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[src], %[src], -16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) - : [rev] "r"(kRev) - : "memory", "$vr0", "$vr1"); -} - -void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width) { - static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, - 6, 7, 4, 5, 2, 3, 0, 1}; - src_uv += width * 2 - 16; - asm volatile( - "vld $vr1, %[rev], 0\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vshuf.b $vr0, $vr0, $vr0, $vr1\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[src], %[src], -16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_uv), [dst] "+r"(dst_uv), [cnt] "+r"(width) - : [rev] "r"(kRevUV) - : "memory", "$vr0", "$vr1"); -} - -void ARGBMirrorRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { - src_argb += width * 4 - 16; - asm volatile( - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vshuf4i.w $vr0, $vr0, 0x1b\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[src], %[src], -16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : - : "memory", "$vr0"); -} - -void MirrorSplitUVRow_LSX(const uint8_t* src_uv, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, - 6, 7, 4, 5, 2, 3, 0, 1}; - src_uv += width * 2 - 32; - asm volatile( - "vld $vr4, %[rev], 0\n\t" - "1:\n\t" - "vld $vr0, %[src], 16\n\t" - "vld $vr1, %[src], 0\n\t" - "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" - "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t" - "vpickev.b $vr2, $vr1, $vr0\n\t" - "vpickod.b $vr3, $vr1, $vr0\n\t" - "vst $vr2, %[du], 0\n\t" - "vst $vr3, %[dv], 0\n\t" - "addi.d %[src], %[src], -32\n\t" - "addi.d %[du], %[du], 16\n\t" - "addi.d %[dv], %[dv], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : - [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width) - : [rev] "r"(kRevUV) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); -} - -void SplitUVRow_LSX(const uint8_t* src_uv, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickev.b $vr2, $vr1, $vr0\n\t" - "vpickod.b $vr3, $vr1, $vr0\n\t" - "vst $vr2, %[du], 0\n\t" - "vst $vr3, %[dv], 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[du], %[du], 16\n\t" - "addi.d %[dv], %[dv], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : - [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); -} - -void MergeUVRow_LSX(const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_uv, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[su], 0\n\t" - "vld $vr1, %[sv], 0\n\t" - "vilvl.b $vr2, $vr1, $vr0\n\t" - "vilvh.b $vr3, $vr1, $vr0\n\t" - "vst $vr2, %[dst], 0\n\t" - "vst $vr3, %[dst], 16\n\t" - "addi.d %[su], %[su], 16\n\t" - "addi.d %[sv], %[sv], 16\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : - [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); -} - -void ARGBExtractAlphaRow_LSX(const uint8_t* src_argb, - uint8_t* dst_a, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickod.b $vr0, $vr1, $vr0\n\t" - "vpickod.b $vr0, $vr0, $vr0\n\t" - "vstelm.d $vr0, %[dst], 0, 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[dst], %[dst], 8\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_a), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1"); -} - -void I422ToYUY2Row_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_yuy2, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[sy], 0\n\t" - "vldrepl.d $vr1, %[su], 0\n\t" - "vldrepl.d $vr2, %[sv], 0\n\t" - "vilvl.b $vr3, $vr2, $vr1\n\t" - "vilvl.b $vr4, $vr3, $vr0\n\t" - "vilvh.b $vr5, $vr3, $vr0\n\t" - "vst $vr4, %[dst], 0\n\t" - "vst $vr5, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 16\n\t" - "addi.d %[su], %[su], 8\n\t" - "addi.d %[sv], %[sv], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_yuy2), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); -} - -void I422ToUYVYRow_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_uyvy, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[sy], 0\n\t" - "vldrepl.d $vr1, %[su], 0\n\t" - "vldrepl.d $vr2, %[sv], 0\n\t" - "vilvl.b $vr3, $vr2, $vr1\n\t" - "vilvl.b $vr4, $vr0, $vr3\n\t" - "vilvh.b $vr5, $vr0, $vr3\n\t" - "vst $vr4, %[dst], 0\n\t" - "vst $vr5, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 16\n\t" - "addi.d %[su], %[su], 8\n\t" - "addi.d %[sv], %[sv], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_uyvy), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); -} - -void YUY2ToYRow_LSX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickev.b $vr2, $vr1, $vr0\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_yuy2), [dst] "+r"(dst_y), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2"); -} - -void UYVYToYRow_LSX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickod.b $vr2, $vr1, $vr0\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_uyvy), [dst] "+r"(dst_y), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2"); -} - -void YUY2ToUV422Row_LSX(const uint8_t* src_yuy2, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickod.b $vr2, $vr1, $vr0\n\t" - "vpickev.b $vr3, $vr2, $vr2\n\t" - "vpickod.b $vr4, $vr2, $vr2\n\t" - "vstelm.d $vr3, %[du], 0, 0\n\t" - "vstelm.d $vr4, %[dv], 0, 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[du], %[du], 8\n\t" - "addi.d %[dv], %[dv], 8\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_yuy2), [du] "+r"(dst_u), [dv] "+r"(dst_v), - [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); -} - -void UYVYToUV422Row_LSX(const uint8_t* src_uyvy, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickev.b $vr2, $vr1, $vr0\n\t" - "vpickev.b $vr3, $vr2, $vr2\n\t" - "vpickod.b $vr4, $vr2, $vr2\n\t" - "vstelm.d $vr3, %[du], 0, 0\n\t" - "vstelm.d $vr4, %[dv], 0, 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[du], %[du], 8\n\t" - "addi.d %[dv], %[dv], 8\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_uyvy), [du] "+r"(dst_u), [dv] "+r"(dst_v), - [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); -} - -void YUY2ToUVRow_LSX(const uint8_t* src_yuy2, - int src_stride_yuy2, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - const uint8_t* next_yuy2 = src_yuy2 + src_stride_yuy2; - asm volatile( - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vld $vr1, %[s0], 16\n\t" - "vld $vr2, %[s1], 0\n\t" - "vld $vr3, %[s1], 16\n\t" - "vpickod.b $vr0, $vr1, $vr0\n\t" - "vpickod.b $vr2, $vr3, $vr2\n\t" - "vavgr.bu $vr0, $vr0, $vr2\n\t" - "vpickev.b $vr1, $vr0, $vr0\n\t" - "vpickod.b $vr2, $vr0, $vr0\n\t" - "vstelm.d $vr1, %[du], 0, 0\n\t" - "vstelm.d $vr2, %[dv], 0, 0\n\t" - "addi.d %[s0], %[s0], 32\n\t" - "addi.d %[s1], %[s1], 32\n\t" - "addi.d %[du], %[du], 8\n\t" - "addi.d %[dv], %[dv], 8\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_yuy2), [s1] "+r"(next_yuy2), [du] "+r"(dst_u), - [dv] "+r"(dst_v), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); -} - -void UYVYToUVRow_LSX(const uint8_t* src_uyvy, - int src_stride_uyvy, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - const uint8_t* next_uyvy = src_uyvy + src_stride_uyvy; - asm volatile( - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vld $vr1, %[s0], 16\n\t" - "vld $vr2, %[s1], 0\n\t" - "vld $vr3, %[s1], 16\n\t" - "vpickev.b $vr0, $vr1, $vr0\n\t" - "vpickev.b $vr2, $vr3, $vr2\n\t" - "vavgr.bu $vr0, $vr0, $vr2\n\t" - "vpickev.b $vr1, $vr0, $vr0\n\t" - "vpickod.b $vr2, $vr0, $vr0\n\t" - "vstelm.d $vr1, %[du], 0, 0\n\t" - "vstelm.d $vr2, %[dv], 0, 0\n\t" - "addi.d %[s0], %[s0], 32\n\t" - "addi.d %[s1], %[s1], 32\n\t" - "addi.d %[du], %[du], 8\n\t" - "addi.d %[dv], %[dv], 8\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_uyvy), [s1] "+r"(next_uyvy), [du] "+r"(dst_u), - [dv] "+r"(dst_v), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); -} - -void I422ToARGBRow_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.w $vr10, %[su], 0\n\t" - "vldrepl.w $vr11, %[sv], 0\n\t" - "vilvl.b $vr10, $vr10, $vr10\n\t" - "vilvl.b $vr11, $vr11, $vr11\n\t" - "vilvl.b $vr1, $vr11, $vr10\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vilvl.b $vr6, $vr3, $vr2\n\t" - "vilvl.b $vr7, $vr5, $vr4\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[su], %[su], 4\n\t" - "addi.d %[sv], %[sv], 4\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", - "$vr23", "$vr24"); -} - -void I422ToRGBARow_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.w $vr10, %[su], 0\n\t" - "vldrepl.w $vr11, %[sv], 0\n\t" - "vilvl.b $vr10, $vr10, $vr10\n\t" - "vilvl.b $vr11, $vr11, $vr11\n\t" - "vilvl.b $vr1, $vr11, $vr10\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vilvl.b $vr6, $vr2, $vr5\n\t" - "vilvl.b $vr7, $vr4, $vr3\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[su], %[su], 4\n\t" - "addi.d %[sv], %[sv], 4\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", - "$vr23", "$vr24"); -} - -void I422AlphaToARGBRow_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - const uint8_t* src_a, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.w $vr10, %[su], 0\n\t" - "vldrepl.w $vr11, %[sv], 0\n\t" - "vilvl.b $vr10, $vr10, $vr10\n\t" - "vilvl.b $vr11, $vr11, $vr11\n\t" - "vilvl.b $vr1, $vr11, $vr10\n\t" - "vldrepl.d $vr5, %[sa], 0\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vilvl.b $vr6, $vr3, $vr2\n\t" - "vilvl.b $vr7, $vr5, $vr4\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[su], %[su], 4\n\t" - "addi.d %[sv], %[sv], 4\n\t" - "addi.d %[sa], %[sa], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_argb), [cnt] "+r"(width), [sa] "+r"(src_a) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", - "$vr23", "$vr24"); -} - -void I422ToRGB24Row_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - static const uint8_t kShuf0[16] = {0, 1, 2, 4, 5, 6, 8, 9, - 10, 12, 13, 14, 16, 17, 18, 20}; - static const uint8_t kShuf1[16] = {21, 22, 24, 25, 26, 28, 29, 30, - 0, 0, 0, 0, 0, 0, 0, 0}; - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vld $vr12, %[s0], 0\n\t" - "vld $vr13, %[s1], 0\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.w $vr10, %[su], 0\n\t" - "vldrepl.w $vr11, %[sv], 0\n\t" - "vilvl.b $vr10, $vr10, $vr10\n\t" - "vilvl.b $vr11, $vr11, $vr11\n\t" - "vilvl.b $vr1, $vr11, $vr10\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vilvl.b $vr6, $vr3, $vr2\n\t" - "vilvl.b $vr7, $vr4, $vr4\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vshuf.b $vr10, $vr9, $vr8, $vr12\n\t" - "vshuf.b $vr11, $vr9, $vr8, $vr13\n\t" - "vst $vr10, %[dst], 0\n\t" - "vstelm.d $vr11, %[dst], 16, 0\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[su], %[su], 4\n\t" - "addi.d %[sv], %[sv], 4\n\t" - "addi.d %[dst], %[dst], 24\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants), [s0] "r"(kShuf0), [s1] "r"(kShuf1) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7", - "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr20", "$vr21", - "$vr22", "$vr23", "$vr24"); -} - -void I422ToRGB565Row_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr14, 31\n\t" - "vsrli.h $vr14, $vr14, 8\n\t" - "vldi $vr15, 63\n\t" - "vsrli.h $vr15, $vr15, 8\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.w $vr10, %[su], 0\n\t" - "vldrepl.w $vr11, %[sv], 0\n\t" - "vilvl.b $vr10, $vr10, $vr10\n\t" - "vilvl.b $vr11, $vr11, $vr11\n\t" - "vilvl.b $vr1, $vr11, $vr10\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vsrli.b $vr2, $vr2, 3\n\t" - "vsrli.b $vr3, $vr3, 2\n\t" - "vsrli.b $vr4, $vr4, 3\n\t" - "vilvl.b $vr2, $vr2, $vr2\n\t" - "vilvl.b $vr3, $vr3, $vr3\n\t" - "vilvl.b $vr4, $vr4, $vr4\n\t" - "vand.v $vr2, $vr2, $vr14\n\t" - "vand.v $vr3, $vr3, $vr15\n\t" - "vand.v $vr4, $vr4, $vr14\n\t" - "vslli.h $vr3, $vr3, 5\n\t" - "vslli.h $vr4, $vr4, 11\n\t" - "vor.v $vr2, $vr2, $vr3\n\t" - "vor.v $vr2, $vr2, $vr4\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[su], %[su], 4\n\t" - "addi.d %[sv], %[sv], 4\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11", - "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); -} - -void I422ToARGB4444Row_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr14, 15\n\t" - "vsrli.h $vr14, $vr14, 8\n\t" - "vldi $vr15, 240\n\t" - "vsrli.h $vr15, $vr15, 8\n\t" - "vslli.h $vr15, $vr15, 8\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.w $vr10, %[su], 0\n\t" - "vldrepl.w $vr11, %[sv], 0\n\t" - "vilvl.b $vr10, $vr10, $vr10\n\t" - "vilvl.b $vr11, $vr11, $vr11\n\t" - "vilvl.b $vr1, $vr11, $vr10\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vsrli.b $vr2, $vr2, 4\n\t" - "vsrli.b $vr3, $vr3, 4\n\t" - "vsrli.b $vr4, $vr4, 4\n\t" - "vilvl.b $vr2, $vr2, $vr2\n\t" - "vilvl.b $vr3, $vr3, $vr3\n\t" - "vilvl.b $vr4, $vr4, $vr4\n\t" - "vand.v $vr2, $vr2, $vr14\n\t" - "vand.v $vr3, $vr3, $vr14\n\t" - "vand.v $vr4, $vr4, $vr14\n\t" - "vslli.h $vr3, $vr3, 4\n\t" - "vslli.h $vr4, $vr4, 8\n\t" - "vor.v $vr2, $vr2, $vr3\n\t" - "vor.v $vr2, $vr2, $vr4\n\t" - "vor.v $vr2, $vr2, $vr15\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[su], %[su], 4\n\t" - "addi.d %[sv], %[sv], 4\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11", - "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); -} - -void I422ToARGB1555Row_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr14, 31\n\t" - "vsrli.h $vr14, $vr14, 8\n\t" - "vldi $vr15, 128\n\t" - "vsrli.h $vr15, $vr15, 8\n\t" - "vslli.h $vr15, $vr15, 8\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.w $vr10, %[su], 0\n\t" - "vldrepl.w $vr11, %[sv], 0\n\t" - "vilvl.b $vr10, $vr10, $vr10\n\t" - "vilvl.b $vr11, $vr11, $vr11\n\t" - "vilvl.b $vr1, $vr11, $vr10\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vsrli.b $vr2, $vr2, 3\n\t" - "vsrli.b $vr3, $vr3, 3\n\t" - "vsrli.b $vr4, $vr4, 3\n\t" - "vilvl.b $vr2, $vr2, $vr2\n\t" - "vilvl.b $vr3, $vr3, $vr3\n\t" - "vilvl.b $vr4, $vr4, $vr4\n\t" - "vand.v $vr2, $vr2, $vr14\n\t" - "vand.v $vr3, $vr3, $vr14\n\t" - "vand.v $vr4, $vr4, $vr14\n\t" - "vslli.h $vr3, $vr3, 5\n\t" - "vslli.h $vr4, $vr4, 10\n\t" - "vor.v $vr2, $vr2, $vr3\n\t" - "vor.v $vr2, $vr2, $vr4\n\t" - "vor.v $vr2, $vr2, $vr15\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[su], %[su], 4\n\t" - "addi.d %[sv], %[sv], 4\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11", - "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); -} - -void NV12ToARGBRow_LSX(const uint8_t* src_y, - const uint8_t* src_uv, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.d $vr1, %[suv], 0\n\t" - "vilvl.h $vr1, $vr1, $vr1\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vilvl.b $vr6, $vr3, $vr2\n\t" - "vilvl.b $vr7, $vr5, $vr4\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[suv], %[suv], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); -} - -void NV21ToARGBRow_LSX(const uint8_t* src_y, - const uint8_t* src_vu, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.d $vr1, %[suv], 0\n\t" - "vshuf4i.b $vr1, $vr1, 0xb1\n\t" - "vilvl.h $vr1, $vr1, $vr1\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vilvl.b $vr6, $vr3, $vr2\n\t" - "vilvl.b $vr7, $vr5, $vr4\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[suv], %[suv], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [suv] "+r"(src_vu), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); -} - -void NV12ToRGB565Row_LSX(const uint8_t* src_y, - const uint8_t* src_uv, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr14, 31\n\t" - "vsrli.h $vr14, $vr14, 8\n\t" - "vldi $vr15, 63\n\t" - "vsrli.h $vr15, $vr15, 8\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.d $vr1, %[suv], 0\n\t" - "vilvl.h $vr1, $vr1, $vr1\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vsrli.b $vr2, $vr2, 3\n\t" - "vsrli.b $vr3, $vr3, 2\n\t" - "vsrli.b $vr4, $vr4, 3\n\t" - "vilvl.b $vr2, $vr2, $vr2\n\t" - "vilvl.b $vr3, $vr3, $vr3\n\t" - "vilvl.b $vr4, $vr4, $vr4\n\t" - "vand.v $vr2, $vr2, $vr14\n\t" - "vand.v $vr3, $vr3, $vr15\n\t" - "vand.v $vr4, $vr4, $vr14\n\t" - "vslli.h $vr3, $vr3, 5\n\t" - "vslli.h $vr4, $vr4, 11\n\t" - "vor.v $vr2, $vr2, $vr3\n\t" - "vor.v $vr2, $vr2, $vr4\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[suv], %[suv], 8\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr14", "$vr15", - "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); -} - -void I444ToARGBRow_LSX(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vldrepl.d $vr10, %[su], 0\n\t" - "vldrepl.d $vr11, %[sv], 0\n\t" - "vilvl.b $vr1, $vr11, $vr10\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vilvl.b $vr6, $vr3, $vr2\n\t" - "vilvl.b $vr7, $vr5, $vr4\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[su], %[su], 8\n\t" - "addi.d %[sv], %[sv], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), - [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", - "$vr23", "$vr24"); -} - -void I400ToARGBRow_LSX(const uint8_t* src_y, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vld $vr23, %[c], 96\n\t" - "vld $vr24, %[c], 128\n\t" - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sy], 0\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vssrani.bu.h $vr2, $vr0, 6\n\t" - "vilvl.b $vr6, $vr2, $vr2\n\t" - "vilvl.b $vr7, $vr5, $vr2\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr2", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9", - "$vr23", "$vr24"); -} - -void J400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, int width) { - asm volatile( - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vldrepl.d $vr2, %[sy], 0\n\t" - "vilvl.b $vr6, $vr2, $vr2\n\t" - "vilvl.b $vr7, $vr5, $vr2\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : - : "memory", "$vr2", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9"); -} - -void YUY2ToARGBRow_LSX(const uint8_t* src_yuy2, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vld $vr10, %[src], 0\n\t" - "vpickev.b $vr0, $vr10, $vr10\n\t" - "vpickod.b $vr1, $vr10, $vr10\n\t" - "vilvl.h $vr1, $vr1, $vr1\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vilvl.b $vr6, $vr3, $vr2\n\t" - "vilvl.b $vr7, $vr5, $vr4\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_yuy2), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr20", "$vr21", "$vr22", "$vr23", - "$vr24"); -} - -void UYVYToARGBRow_LSX(const uint8_t* src_uyvy, - uint8_t* dst_argb, - const struct YuvConstants* yuvconstants, - int width) { - asm volatile( - "vldrepl.w $vr20, %[c], 0\n\t" - "vldrepl.w $vr21, %[c], 32\n\t" - "vldrepl.w $vr22, %[c], 64\n\t" - "vldrepl.h $vr23, %[c], 96\n\t" - "vldrepl.h $vr24, %[c], 128\n\t" - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vld $vr10, %[src], 0\n\t" - "vpickod.b $vr0, $vr10, $vr10\n\t" - "vpickev.b $vr1, $vr10, $vr10\n\t" - "vilvl.h $vr1, $vr1, $vr1\n\t" - "vilvl.b $vr0, $vr0, $vr0\n\t" - "vxori.b $vr1, $vr1, 0x80\n\t" - "vmuh.hu $vr0, $vr0, $vr23\n\t" - "vadd.h $vr0, $vr0, $vr24\n\t" - "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" - "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" - "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" - "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" - "vsadd.h $vr2, $vr0, $vr2\n\t" - "vsadd.h $vr4, $vr0, $vr4\n\t" - "vssub.h $vr3, $vr0, $vr3\n\t" - "vssrani.bu.h $vr2, $vr2, 6\n\t" - "vssrani.bu.h $vr3, $vr3, 6\n\t" - "vssrani.bu.h $vr4, $vr4, 6\n\t" - "vilvl.b $vr6, $vr3, $vr2\n\t" - "vilvl.b $vr7, $vr5, $vr4\n\t" - "vilvl.h $vr8, $vr7, $vr6\n\t" - "vilvh.h $vr9, $vr7, $vr6\n\t" - "vst $vr8, %[dst], 0\n\t" - "vst $vr9, %[dst], 16\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_uyvy), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [c] "r"(yuvconstants) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr20", "$vr21", "$vr22", "$vr23", - "$vr24"); -} - -static LIBYUV_NOINLINE void ARGBTo24Helper_LSX(const uint8_t* src_argb, - uint8_t* dst_24, - const uint8_t* shuf, - int width) { - asm volatile( - "vld $vr4, %[shuf], 0\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vld $vr2, %[src], 32\n\t" - "vld $vr3, %[src], 48\n\t" - "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" - "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t" - "vshuf.b $vr2, $vr2, $vr2, $vr4\n\t" - "vshuf.b $vr3, $vr3, $vr3, $vr4\n\t" - "vstelm.d $vr0, %[dst], 0, 0\n\t" - "vstelm.w $vr0, %[dst], 8, 2\n\t" - "vstelm.w $vr1, %[dst], 12, 0\n\t" - "vstelm.w $vr1, %[dst], 16, 1\n\t" - "vstelm.w $vr1, %[dst], 20, 2\n\t" - "vstelm.d $vr2, %[dst], 24, 0\n\t" - "vstelm.w $vr2, %[dst], 32, 2\n\t" - "vstelm.w $vr3, %[dst], 36, 0\n\t" - "vstelm.w $vr3, %[dst], 40, 1\n\t" - "vstelm.w $vr3, %[dst], 44, 2\n\t" - "addi.d %[src], %[src], 64\n\t" - "addi.d %[dst], %[dst], 48\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_24), [cnt] "+r"(width) - : [shuf] "r"(shuf) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); -} - -void ARGBToRGB24Row_LSX(const uint8_t* src_argb, - uint8_t* dst_rgb24, - int width) { - static const uint8_t kShuf[16] = {0, 1, 2, 4, 5, 6, 8, 9, - 10, 12, 13, 14, 0, 0, 0, 0}; - ARGBTo24Helper_LSX(src_argb, dst_rgb24, kShuf, width); -} - -void ARGBToRAWRow_LSX(const uint8_t* src_argb, uint8_t* dst_raw, int width) { - static const uint8_t kShuf[16] = {2, 1, 0, 6, 5, 4, 10, 9, - 8, 14, 13, 12, 0, 0, 0, 0}; - ARGBTo24Helper_LSX(src_argb, dst_raw, kShuf, width); -} - -static LIBYUV_NOINLINE void RGB24ToARGBHelper_LSX(const uint8_t* src_24, - uint8_t* dst_argb, - const uint8_t* shuf, - int width) { - asm volatile( - "vld $vr4, %[shuf], 0\n\t" - "vld $vr5, %[shuf], 16\n\t" - "vldi $vr6, 0xff\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 12\n\t" - "vld $vr2, %[src], 24\n\t" - "vld $vr3, %[src], 32\n\t" - "vshuf.b $vr0, $vr6, $vr0, $vr4\n\t" - "vshuf.b $vr1, $vr6, $vr1, $vr4\n\t" - "vshuf.b $vr2, $vr6, $vr2, $vr4\n\t" - "vshuf.b $vr3, $vr6, $vr3, $vr5\n\t" - "vst $vr0, %[dst], 0\n\t" - "vst $vr1, %[dst], 16\n\t" - "vst $vr2, %[dst], 32\n\t" - "vst $vr3, %[dst], 48\n\t" - "addi.d %[src], %[src], 48\n\t" - "addi.d %[dst], %[dst], 64\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_24), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [shuf] "r"(shuf) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); -} - -void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24, - uint8_t* dst_argb, - int width) { - static const uint8_t kShuf[32] = {0, 1, 2, 16, 3, 4, 5, 16, 6, 7, 8, - 16, 9, 10, 11, 16, 4, 5, 6, 16, 7, 8, - 9, 16, 10, 11, 12, 16, 13, 14, 15, 16}; - RGB24ToARGBHelper_LSX(src_rgb24, dst_argb, kShuf, width); -} - -void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { - static const uint8_t kShuf[32] = {2, 1, 0, 16, 5, 4, 3, 16, 8, 7, 6, - 16, 11, 10, 9, 16, 6, 5, 4, 16, 9, 8, - 7, 16, 12, 11, 10, 16, 15, 14, 13, 16}; - RGB24ToARGBHelper_LSX(src_raw, dst_argb, kShuf, width); -} - -void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { - static const uint8_t kShuf[32] = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, - 9, 0, 0, 0, 0, 6, 5, 4, 9, 8, 7, - 12, 11, 10, 15, 14, 13, 0, 0, 0, 0}; - asm volatile( - "vld $vr4, %[shuf], 0\n\t" - "vld $vr5, %[shuf], 16\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 12\n\t" - "vld $vr2, %[src], 24\n\t" - "vld $vr3, %[src], 32\n\t" - "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" - "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t" - "vshuf.b $vr2, $vr2, $vr2, $vr4\n\t" - "vshuf.b $vr3, $vr3, $vr3, $vr5\n\t" - "vstelm.d $vr0, %[dst], 0, 0\n\t" - "vstelm.w $vr0, %[dst], 8, 2\n\t" - "vstelm.w $vr1, %[dst], 12, 0\n\t" - "vstelm.w $vr1, %[dst], 16, 1\n\t" - "vstelm.w $vr1, %[dst], 20, 2\n\t" - "vstelm.d $vr2, %[dst], 24, 0\n\t" - "vstelm.w $vr2, %[dst], 32, 2\n\t" - "vstelm.w $vr3, %[dst], 36, 0\n\t" - "vstelm.w $vr3, %[dst], 40, 1\n\t" - "vstelm.w $vr3, %[dst], 44, 2\n\t" - "addi.d %[src], %[src], 48\n\t" - "addi.d %[dst], %[dst], 48\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_raw), [dst] "+r"(dst_rgb24), [cnt] "+r"(width) - : [shuf] "r"(kShuf) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); -} - -void ARGBToRGB565Row_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { - asm volatile( - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "vldi $vr7, 63\n\t" - "vsrli.h $vr7, $vr7, 8\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickev.b $vr2, $vr1, $vr0\n\t" - "vpickod.b $vr3, $vr1, $vr0\n\t" - "vsrli.b $vr2, $vr2, 3\n\t" - "vsrli.b $vr3, $vr3, 2\n\t" - "vand.v $vr4, $vr2, $vr6\n\t" - "vsrli.h $vr2, $vr2, 8\n\t" - "vand.v $vr3, $vr3, $vr7\n\t" - "vslli.h $vr3, $vr3, 5\n\t" - "vslli.h $vr2, $vr2, 11\n\t" - "vor.v $vr4, $vr4, $vr3\n\t" - "vor.v $vr4, $vr4, $vr2\n\t" - "vst $vr4, %[dst], 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7"); -} - -void ARGBToRGB565DitherRow_LSX(const uint8_t* src_argb, - uint8_t* dst_rgb, - uint32_t dither4, - int width) { - asm volatile( - "vreplgr2vr.w $vr5, %[dith]\n\t" - "vilvl.b $vr5, $vr5, $vr5\n\t" - "vilvl.h $vr5, $vr5, $vr5\n\t" - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "vldi $vr7, 63\n\t" - "vsrli.h $vr7, $vr7, 8\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vsadd.bu $vr0, $vr0, $vr5\n\t" - "vpickev.b $vr2, $vr0, $vr0\n\t" - "vpickod.b $vr3, $vr0, $vr0\n\t" - "vsrli.b $vr2, $vr2, 3\n\t" - "vsrli.b $vr3, $vr3, 2\n\t" - "vand.v $vr4, $vr2, $vr6\n\t" - "vsrli.h $vr2, $vr2, 8\n\t" - "vand.v $vr3, $vr3, $vr7\n\t" - "vslli.h $vr3, $vr3, 5\n\t" - "vslli.h $vr2, $vr2, 11\n\t" - "vor.v $vr4, $vr4, $vr3\n\t" - "vor.v $vr4, $vr4, $vr2\n\t" - "vstelm.d $vr4, %[dst], 0, 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 8\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) - : [dith] "r"(dither4) - : "memory", "$vr0", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7"); -} - -void ARGBToARGB1555Row_LSX(const uint8_t* src_argb, - uint8_t* dst_rgb, - int width) { - asm volatile( - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickev.b $vr2, $vr1, $vr0\n\t" - "vpickod.b $vr3, $vr1, $vr0\n\t" - "vsrli.b $vr2, $vr2, 3\n\t" - "vand.v $vr4, $vr2, $vr6\n\t" - "vsrli.h $vr2, $vr2, 8\n\t" - "vsrli.h $vr5, $vr3, 15\n\t" - "vsrli.b $vr3, $vr3, 3\n\t" - "vand.v $vr3, $vr3, $vr6\n\t" - "vslli.h $vr3, $vr3, 5\n\t" - "vslli.h $vr2, $vr2, 10\n\t" - "vslli.h $vr5, $vr5, 15\n\t" - "vor.v $vr4, $vr4, $vr3\n\t" - "vor.v $vr4, $vr4, $vr2\n\t" - "vor.v $vr4, $vr4, $vr5\n\t" - "vst $vr4, %[dst], 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); -} - -void ARGBToARGB4444Row_LSX(const uint8_t* src_argb, - uint8_t* dst_rgb, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickev.b $vr2, $vr1, $vr0\n\t" - "vpickod.b $vr3, $vr1, $vr0\n\t" - "vsrli.b $vr2, $vr2, 4\n\t" - "vsrli.b $vr3, $vr3, 4\n\t" - "vslli.b $vr3, $vr3, 4\n\t" - "vor.v $vr2, $vr2, $vr3\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); -} - -void RGB565ToARGBRow_LSX(const uint8_t* src_rgb565, - uint8_t* dst_argb, - int width) { - asm volatile( - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "vldi $vr7, 63\n\t" - "vsrli.h $vr7, $vr7, 8\n\t" - "vldi $vr5, 0xff\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr7\n\t" - "vsrli.h $vr3, $vr0, 11\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 2\n\t" - "vsrli.h $vr2, $vr2, 4\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vpickev.b $vr1, $vr1, $vr1\n\t" - "vpickev.b $vr2, $vr2, $vr2\n\t" - "vpickev.b $vr3, $vr3, $vr3\n\t" - "vilvl.b $vr8, $vr2, $vr1\n\t" - "vilvl.b $vr9, $vr5, $vr3\n\t" - "vilvl.h $vr10, $vr9, $vr8\n\t" - "vilvh.h $vr11, $vr9, $vr8\n\t" - "vst $vr10, %[dst], 0\n\t" - "vst $vr11, %[dst], 16\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_rgb565), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11"); -} - -void ARGB1555ToARGBRow_LSX(const uint8_t* src_argb1555, - uint8_t* dst_argb, - int width) { - asm volatile( - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr6\n\t" - "vsrli.h $vr3, $vr0, 10\n\t" - "vand.v $vr3, $vr3, $vr6\n\t" - "vsrai.h $vr5, $vr0, 15\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 3\n\t" - "vsrli.h $vr2, $vr2, 2\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vpickev.b $vr1, $vr1, $vr1\n\t" - "vpickev.b $vr2, $vr2, $vr2\n\t" - "vpickev.b $vr3, $vr3, $vr3\n\t" - "vpickev.b $vr5, $vr5, $vr5\n\t" - "vilvl.b $vr8, $vr2, $vr1\n\t" - "vilvl.b $vr9, $vr5, $vr3\n\t" - "vilvl.h $vr10, $vr9, $vr8\n\t" - "vilvh.h $vr11, $vr9, $vr8\n\t" - "vst $vr10, %[dst], 0\n\t" - "vst $vr11, %[dst], 16\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb1555), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr8", "$vr9", "$vr10", "$vr11"); -} - -void ARGB4444ToARGBRow_LSX(const uint8_t* src_argb4444, - uint8_t* dst_argb, - int width) { - asm volatile( - "vldi $vr6, 0x0f\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.b $vr2, $vr0, 4\n\t" - "vslli.b $vr3, $vr1, 4\n\t" - "vor.v $vr1, $vr1, $vr3\n\t" - "vslli.b $vr3, $vr2, 4\n\t" - "vor.v $vr2, $vr2, $vr3\n\t" - "vilvl.b $vr4, $vr2, $vr1\n\t" - "vilvh.b $vr5, $vr2, $vr1\n\t" - "vst $vr4, %[dst], 0\n\t" - "vst $vr5, %[dst], 16\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb4444), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); -} - -void ARGBMultiplyRow_LSX(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vld $vr1, %[s1], 0\n\t" - "vmulwev.h.bu $vr2, $vr0, $vr1\n\t" - "vmulwod.h.bu $vr3, $vr0, $vr1\n\t" - "vsrari.h $vr2, $vr2, 8\n\t" - "vsrari.h $vr3, $vr3, 8\n\t" - "vpackev.b $vr2, $vr3, $vr2\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[s0], %[s0], 16\n\t" - "addi.d %[s1], %[s1], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); -} - -void ARGBAddRow_LSX(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vld $vr1, %[s1], 0\n\t" - "vsadd.bu $vr0, $vr0, $vr1\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[s0], %[s0], 16\n\t" - "addi.d %[s1], %[s1], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1"); -} - -void ARGBSubtractRow_LSX(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vld $vr1, %[s1], 0\n\t" - "vssub.bu $vr0, $vr0, $vr1\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[s0], %[s0], 16\n\t" - "addi.d %[s1], %[s1], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1"); -} - -void ARGBAttenuateRow_LSX(const uint8_t* src_argb, - uint8_t* dst_argb, - int width) { - static const uint8_t kAlpha[16] = {3, 3, 3, 16, 7, 7, 7, 16, - 11, 11, 11, 16, 15, 15, 15, 16}; - asm volatile( - "vld $vr4, %[shuf], 0\n\t" - "vldi $vr5, 0xff\n\t" - "vldi $vr6, 255\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vshuf.b $vr1, $vr5, $vr0, $vr4\n\t" - "vmulwev.h.bu $vr2, $vr0, $vr1\n\t" - "vmulwod.h.bu $vr3, $vr0, $vr1\n\t" - "vadd.h $vr2, $vr2, $vr6\n\t" - "vadd.h $vr3, $vr3, $vr6\n\t" - "vpackod.b $vr2, $vr3, $vr2\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [shuf] "r"(kAlpha) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); -} - -void ARGBBlendRow_LSX(const uint8_t* src_argb0, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - static const uint8_t kAlpha[16] = {3, 3, 3, 3, 7, 7, 7, 7, - 11, 11, 11, 11, 15, 15, 15, 15}; - asm volatile( - "vld $vr4, %[shuf], 0\n\t" - "vldi $vr5, 0xff\n\t" - "vslli.w $vr5, $vr5, 24\n\t" - "vldi $vr6, 1\n\t" - "vslli.h $vr6, $vr6, 8\n\t" - "vldi $vr8, 0\n\t" - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vld $vr1, %[s1], 0\n\t" - "vshuf.b $vr2, $vr0, $vr0, $vr4\n\t" - "vilvl.b $vr3, $vr8, $vr2\n\t" - "vilvh.b $vr2, $vr8, $vr2\n\t" - "vsub.h $vr3, $vr6, $vr3\n\t" - "vsub.h $vr2, $vr6, $vr2\n\t" - "vilvl.b $vr7, $vr8, $vr1\n\t" - "vilvh.b $vr1, $vr8, $vr1\n\t" - "vmul.h $vr7, $vr7, $vr3\n\t" - "vmul.h $vr1, $vr1, $vr2\n\t" - "vsrlni.b.h $vr1, $vr7, 8\n\t" - "vsadd.bu $vr0, $vr0, $vr1\n\t" - "vor.v $vr0, $vr0, $vr5\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[s0], %[s0], 16\n\t" - "addi.d %[s1], %[s1], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : [shuf] "r"(kAlpha) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8"); -} - -void ARGBOrRow_LSX(const uint8_t* src_argb, - uint8_t* dst_argb, - uint32_t value, - int width) { - asm volatile( - "vreplgr2vr.w $vr1, %[val]\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vor.v $vr0, $vr0, $vr1\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [val] "r"(value) - : "memory", "$vr0", "$vr1"); -} - -void ARGBAlphaBlendRow_LSX(const uint8_t* src_argb0, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - static const uint8_t kShufRatio[16] = {0, 0, 0, 0, 4, 4, 4, 4, - 8, 8, 8, 8, 12, 12, 12, 12}; - asm volatile( - "vld $vr8, %[shuf], 0\n\t" - "vldi $vr9, 1\n\t" - "vsrli.h $vr10, $vr9, 8\n\t" - "vldi $vr11, 0xff\n\t" - "vsrli.w $vr11, $vr11, 8\n\t" - "vldi $vr12, 0\n\t" - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vld $vr1, %[s1], 0\n\t" - "vsrli.w $vr2, $vr0, 24\n\t" - "vsrli.w $vr3, $vr1, 24\n\t" - "vmul.h $vr4, $vr3, $vr2\n\t" - "vadd.h $vr4, $vr4, $vr10\n\t" - "vmuh.hu $vr4, $vr4, $vr9\n\t" - "vsub.w $vr4, $vr2, $vr4\n\t" - "vadd.w $vr4, $vr3, $vr4\n\t" - "vseq.w $vr5, $vr3, $vr12\n\t" - "vsrli.w $vr5, $vr5, 24\n\t" - "vor.v $vr6, $vr2, $vr5\n\t" - "vor.v $vr7, $vr4, $vr5\n\t" - "vslli.w $vr5, $vr6, 8\n\t" - "vsub.w $vr5, $vr5, $vr6\n\t" - "vffint.s.wu $vr5, $vr5\n\t" - "vffint.s.wu $vr7, $vr7\n\t" - "vfdiv.s $vr5, $vr5, $vr7\n\t" - "vftintrz.wu.s $vr5, $vr5\n\t" - "vshuf.b $vr5, $vr5, $vr5, $vr8\n\t" - "vnor.v $vr6, $vr5, $vr5\n\t" - "vmulwev.h.bu $vr2, $vr1, $vr6\n\t" - "vmulwod.h.bu $vr3, $vr1, $vr6\n\t" - "vmulwev.h.bu $vr7, $vr0, $vr5\n\t" - "vmulwod.h.bu $vr0, $vr0, $vr5\n\t" - "vadd.h $vr2, $vr2, $vr7\n\t" - "vadd.h $vr3, $vr3, $vr0\n\t" - "vadd.h $vr2, $vr2, $vr10\n\t" - "vadd.h $vr3, $vr3, $vr10\n\t" - "vmuh.hu $vr2, $vr2, $vr9\n\t" - "vmuh.hu $vr3, $vr3, $vr9\n\t" - "vslli.h $vr3, $vr3, 8\n\t" - "vor.v $vr2, $vr2, $vr3\n\t" - "vand.v $vr2, $vr2, $vr11\n\t" - "vslli.w $vr4, $vr4, 24\n\t" - "vor.v $vr2, $vr2, $vr4\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[s0], %[s0], 16\n\t" - "addi.d %[s1], %[s1], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : [shuf] "r"(kShufRatio) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12"); -} - -void ARGBShadeRow_LSX(const uint8_t* src_argb, - uint8_t* dst_argb, - int width, - uint32_t value) { - asm volatile( - "vreplgr2vr.w $vr4, %[val]\n\t" - "vilvl.b $vr5, $vr4, $vr4\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vilvl.b $vr1, $vr0, $vr0\n\t" - "vilvh.b $vr2, $vr0, $vr0\n\t" - "vmuh.hu $vr1, $vr1, $vr5\n\t" - "vmuh.hu $vr2, $vr2, $vr5\n\t" - "vsrlni.b.h $vr2, $vr1, 8\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [val] "r"(value) - : "memory", "$vr0", "$vr1", "$vr2", "$vr4", "$vr5"); -} - -void ARGBShuffleRow_LSX(const uint8_t* src_argb, - uint8_t* dst_argb, - const uint8_t* shuffler, - int width) { - asm volatile( - "vld $vr4, %[shuf], 0\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [shuf] "r"(shuffler) - : "memory", "$vr0", "$vr4"); -} - -void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { - asm volatile( - "vldi $vr4, 29\n\t" - "vldi $vr5, 150\n\t" - "vldi $vr6, 77\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vld $vr1, %[src], 16\n\t" - "vpickev.b $vr2, $vr1, $vr0\n\t" - "vpickod.b $vr3, $vr1, $vr0\n\t" - "vmulwev.h.bu $vr7, $vr2, $vr4\n\t" - "vmaddwod.h.bu $vr7, $vr2, $vr6\n\t" - "vmaddwev.h.bu $vr7, $vr3, $vr5\n\t" - "vsrlri.h $vr7, $vr7, 8\n\t" - "vsrli.h $vr3, $vr3, 8\n\t" - "vslli.h $vr3, $vr3, 8\n\t" - "vor.v $vr3, $vr7, $vr3\n\t" - "vslli.h $vr8, $vr7, 8\n\t" - "vor.v $vr7, $vr7, $vr8\n\t" - "vilvl.h $vr0, $vr3, $vr7\n\t" - "vilvh.h $vr1, $vr3, $vr7\n\t" - "vst $vr0, %[dst], 0\n\t" - "vst $vr1, %[dst], 16\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8"); -} - -void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width) { - asm volatile( - "vldi $vr4, 17\n\t" - "vldi $vr5, 68\n\t" - "vldi $vr6, 35\n\t" - "vldi $vr7, 22\n\t" - "vldi $vr8, 88\n\t" - "vldi $vr9, 45\n\t" - "vldi $vr10, 24\n\t" - "vldi $vr11, 98\n\t" - "vldi $vr12, 50\n\t" - "1:\n\t" - "vld $vr0, %[dst], 0\n\t" - "vld $vr1, %[dst], 16\n\t" - "vpickev.b $vr2, $vr1, $vr0\n\t" - "vpickod.b $vr3, $vr1, $vr0\n\t" - "vmulwev.h.bu $vr13, $vr2, $vr4\n\t" - "vmaddwod.h.bu $vr13, $vr2, $vr6\n\t" - "vmaddwev.h.bu $vr13, $vr3, $vr5\n\t" - "vmulwev.h.bu $vr14, $vr2, $vr7\n\t" - "vmaddwod.h.bu $vr14, $vr2, $vr9\n\t" - "vmaddwev.h.bu $vr14, $vr3, $vr8\n\t" - "vmulwev.h.bu $vr15, $vr2, $vr10\n\t" - "vmaddwod.h.bu $vr15, $vr2, $vr12\n\t" - "vmaddwev.h.bu $vr15, $vr3, $vr11\n\t" - "vsrli.h $vr13, $vr13, 7\n\t" - "vsrli.h $vr14, $vr14, 7\n\t" - "vsrli.h $vr15, $vr15, 7\n\t" - "vsat.hu $vr13, $vr13, 7\n\t" - "vsat.hu $vr14, $vr14, 7\n\t" - "vsat.hu $vr15, $vr15, 7\n\t" - "vpickev.b $vr13, $vr13, $vr13\n\t" - "vpickev.b $vr14, $vr14, $vr14\n\t" - "vpickev.b $vr15, $vr15, $vr15\n\t" - "vpickod.b $vr3, $vr3, $vr3\n\t" - "vilvl.b $vr2, $vr14, $vr13\n\t" - "vilvl.b $vr3, $vr3, $vr15\n\t" - "vilvl.h $vr0, $vr3, $vr2\n\t" - "vilvh.h $vr1, $vr3, $vr2\n\t" - "vst $vr0, %[dst], 0\n\t" - "vst $vr1, %[dst], 16\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [dst] "+r"(dst_argb), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", - "$vr15"); -} - -void ARGBColorMatrixRow_LSX(const uint8_t* src_argb, - uint8_t* dst_argb, - const int8_t* matrix_argb, - int width) { - asm volatile( - "vldrepl.w $vr4, %[mat], 0\n\t" - "vldrepl.w $vr5, %[mat], 4\n\t" - "vldrepl.w $vr6, %[mat], 8\n\t" - "vldrepl.w $vr7, %[mat], 12\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vmulwev.h.bu.b $vr8, $vr0, $vr4\n\t" - "vmaddwod.h.bu.b $vr8, $vr0, $vr4\n\t" - "vmulwev.h.bu.b $vr9, $vr0, $vr5\n\t" - "vmaddwod.h.bu.b $vr9, $vr0, $vr5\n\t" - "vmulwev.h.bu.b $vr10, $vr0, $vr6\n\t" - "vmaddwod.h.bu.b $vr10, $vr0, $vr6\n\t" - "vmulwev.h.bu.b $vr11, $vr0, $vr7\n\t" - "vmaddwod.h.bu.b $vr11, $vr0, $vr7\n\t" - "vhaddw.w.h $vr8, $vr8, $vr8\n\t" - "vhaddw.w.h $vr9, $vr9, $vr9\n\t" - "vhaddw.w.h $vr10, $vr10, $vr10\n\t" - "vhaddw.w.h $vr11, $vr11, $vr11\n\t" - "vilvl.w $vr12, $vr9, $vr8\n\t" - "vilvh.w $vr13, $vr9, $vr8\n\t" - "vilvl.w $vr14, $vr11, $vr10\n\t" - "vilvh.w $vr15, $vr11, $vr10\n\t" - "vilvl.d $vr8, $vr14, $vr12\n\t" - "vilvh.d $vr9, $vr14, $vr12\n\t" - "vilvl.d $vr10, $vr15, $vr13\n\t" - "vilvh.d $vr11, $vr15, $vr13\n\t" - "vssrani.h.w $vr9, $vr8, 6\n\t" - "vssrani.h.w $vr11, $vr10, 6\n\t" - "vssrani.bu.h $vr11, $vr9, 0\n\t" - "vst $vr11, %[dst], 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [mat] "r"(matrix_argb) - : "memory", "$vr0", "$vr4", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9", - "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15"); -} - -void ARGBQuantizeRow_LSX(uint8_t* dst_argb, - int scale, - int interval_size, - int interval_offset, - int width) { - asm volatile( - "vreplgr2vr.h $vr4, %[scl]\n\t" - "vreplgr2vr.b $vr5, %[isz]\n\t" - "vreplgr2vr.b $vr6, %[ioff]\n\t" - "vldi $vr7, 0xff\n\t" - "vslli.w $vr7, $vr7, 24\n\t" - "1:\n\t" - "vld $vr0, %[dst], 0\n\t" - "vilvl.b $vr1, $vr0, $vr0\n\t" - "vilvh.b $vr2, $vr0, $vr0\n\t" - "vsrli.h $vr1, $vr1, 8\n\t" - "vsrli.h $vr2, $vr2, 8\n\t" - "vmuh.hu $vr1, $vr1, $vr4\n\t" - "vmuh.hu $vr2, $vr2, $vr4\n\t" - "vsrlni.b.h $vr2, $vr1, 0\n\t" - "vmul.b $vr2, $vr2, $vr5\n\t" - "vadd.b $vr2, $vr2, $vr6\n\t" - "vand.v $vr0, $vr0, $vr7\n\t" - "vandn.v $vr2, $vr7, $vr2\n\t" - "vor.v $vr2, $vr2, $vr0\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -4\n\t" - "bnez %[cnt], 1b\n\t" - : [dst] "+r"(dst_argb), [cnt] "+r"(width) - : [scl] "r"(scale), [isz] "r"(interval_size), [ioff] "r"(interval_offset) - : "memory", "$vr0", "$vr1", "$vr2", "$vr4", "$vr5", "$vr6", "$vr7"); -} - -void SobelRow_LSX(const uint8_t* src_sobelx, - const uint8_t* src_sobely, - uint8_t* dst_argb, - int width) { - asm volatile( - "vldi $vr4, 0xff\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sx], 0\n\t" - "vldrepl.d $vr1, %[sy], 0\n\t" - "vsadd.bu $vr0, $vr0, $vr1\n\t" - "vilvl.b $vr2, $vr0, $vr0\n\t" - "vilvl.b $vr3, $vr4, $vr0\n\t" - "vilvl.h $vr0, $vr3, $vr2\n\t" - "vilvh.h $vr1, $vr3, $vr2\n\t" - "vst $vr0, %[dst], 0\n\t" - "vst $vr1, %[dst], 16\n\t" - "addi.d %[sx], %[sx], 8\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); -} - -void SobelToPlaneRow_LSX(const uint8_t* src_sobelx, - const uint8_t* src_sobely, - uint8_t* dst_y, - int width) { - asm volatile( - "1:\n\t" - "vld $vr0, %[sx], 0\n\t" - "vld $vr1, %[sy], 0\n\t" - "vsadd.bu $vr0, $vr0, $vr1\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[sx], %[sx], 16\n\t" - "addi.d %[sy], %[sy], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_y), - [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1"); -} - -void SobelXYRow_LSX(const uint8_t* src_sobelx, - const uint8_t* src_sobely, - uint8_t* dst_argb, - int width) { - asm volatile( - "vldi $vr4, 0xff\n\t" - "1:\n\t" - "vldrepl.d $vr0, %[sx], 0\n\t" - "vldrepl.d $vr1, %[sy], 0\n\t" - "vsadd.bu $vr2, $vr0, $vr1\n\t" - "vilvl.b $vr1, $vr2, $vr1\n\t" - "vilvl.b $vr0, $vr4, $vr0\n\t" - "vilvl.h $vr2, $vr0, $vr1\n\t" - "vilvh.h $vr3, $vr0, $vr1\n\t" - "vst $vr2, %[dst], 0\n\t" - "vst $vr3, %[dst], 16\n\t" - "addi.d %[sx], %[sx], 8\n\t" - "addi.d %[sy], %[sy], 8\n\t" - "addi.d %[dst], %[dst], 32\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb), - [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); -} - -void InterpolateRow_LSX(uint8_t* dst_ptr, - const uint8_t* src_ptr, - ptrdiff_t src_stride, - int width, - int32_t source_y_fraction) { - int y1_fraction = source_y_fraction; - const uint8_t* src_ptr1 = src_ptr + src_stride; - if (y1_fraction == 0) { - CopyRow_LSX(src_ptr, dst_ptr, width); - return; - } - if (y1_fraction == 128) { - asm volatile( - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vld $vr1, %[s1], 0\n\t" - "vavgr.bu $vr0, $vr0, $vr1\n\t" - "vst $vr0, %[dst], 0\n\t" - "addi.d %[s0], %[s0], 16\n\t" - "addi.d %[s1], %[s1], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), - [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1"); - return; - } - uint16_t y_frac = (uint16_t)((256 - y1_fraction) + (y1_fraction << 8)); - asm volatile( - "vreplgr2vr.h $vr3, %[yf]\n\t" - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vld $vr1, %[s1], 0\n\t" - "vilvl.b $vr2, $vr1, $vr0\n\t" - "vilvh.b $vr4, $vr1, $vr0\n\t" - "vmulwev.h.bu $vr0, $vr2, $vr3\n\t" - "vmaddwod.h.bu $vr0, $vr2, $vr3\n\t" - "vmulwev.h.bu $vr1, $vr4, $vr3\n\t" - "vmaddwod.h.bu $vr1, $vr4, $vr3\n\t" - "vsrarni.b.h $vr1, $vr0, 8\n\t" - "vst $vr1, %[dst], 0\n\t" - "addi.d %[s0], %[s0], 16\n\t" - "addi.d %[s1], %[s1], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), - [cnt] "+r"(width) - : [yf] "r"(y_frac) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); -} - -void HalfFloatRow_LSX(const uint16_t* src, - uint16_t* dst, - float scale, - int width) { - const float bias = 1.9259299444e-34f; // 2^-112, see HalfFloatRow_C - asm volatile( - "vldrepl.w $vr3, %[scale], 0\n\t" - "vldrepl.w $vr5, %[bias], 0\n\t" - "vldi $vr4, 0\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vilvl.h $vr1, $vr4, $vr0\n\t" - "vilvh.h $vr2, $vr4, $vr0\n\t" - "vffint.s.wu $vr1, $vr1\n\t" - "vffint.s.wu $vr2, $vr2\n\t" - "vfmul.s $vr1, $vr1, $vr3\n\t" - "vfmul.s $vr2, $vr2, $vr3\n\t" - "vfmul.s $vr1, $vr1, $vr5\n\t" - "vfmul.s $vr2, $vr2, $vr5\n\t" - "vsrlni.h.w $vr2, $vr1, 13\n\t" - "vst $vr2, %[dst], 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) - : [scale] "r"(&scale), [bias] "r"(&bias) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); -} - -void ARGBToYMatrixRow_LSX(const uint8_t* src_argb, - uint8_t* dst_y, - int width, - const struct ArgbConstants* c) { - asm volatile( - "vldrepl.b $vr0, %[c], 0\n\t" - "vldrepl.b $vr1, %[c], 1\n\t" - "vldrepl.b $vr2, %[c], 2\n\t" - "vldrepl.b $vr3, %[c], 3\n\t" - "vldrepl.h $vr4, %[c], 96\n\t" - "1:\n\t" - "vld $vr5, %[src], 0\n\t" - "vld $vr6, %[src], 16\n\t" - "vor.v $vr9, $vr4, $vr4\n\t" - "vpickev.b $vr7, $vr6, $vr5\n\t" - "vpickod.b $vr8, $vr6, $vr5\n\t" - "vmaddwev.h.bu $vr9, $vr7, $vr0\n\t" - "vmaddwev.h.bu $vr9, $vr8, $vr1\n\t" - "vmaddwod.h.bu $vr9, $vr7, $vr2\n\t" - "vmaddwod.h.bu $vr9, $vr8, $vr3\n\t" - "vpickod.b $vr9, $vr9, $vr9\n\t" - "vstelm.d $vr9, %[dst], 0, 0\n\t" - "addi.d %[src], %[src], 32\n\t" - "addi.d %[dst], %[dst], 8\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb), [dst] "+r"(dst_y), [cnt] "+r"(width) - : [c] "r"(c) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9"); -} - -void RGBToYMatrixRow_LSX(const uint8_t* src_rgba, - uint8_t* dst_y, - int width, - const struct ArgbConstants* c) { - static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8, - 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8, - 9, 0, 10, 11, 12, 0, 13, 14, 15, 0}; - asm volatile( - "vldrepl.b $vr0, %[c], 0\n\t" - "vldrepl.b $vr1, %[c], 1\n\t" - "vldrepl.b $vr2, %[c], 2\n\t" - "vldrepl.h $vr3, %[c], 96\n\t" - "vld $vr4, %[shuf], 0\n\t" - "vld $vr5, %[shuf], 16\n\t" - "1:\n\t" - "vld $vr6, %[src], 0\n\t" - "vld $vr7, %[src], 12\n\t" - "vld $vr8, %[src], 24\n\t" - "vld $vr9, %[src], 32\n\t" - "vshuf.b $vr6, $vr6, $vr6, $vr4\n\t" - "vshuf.b $vr7, $vr7, $vr7, $vr4\n\t" - "vshuf.b $vr8, $vr8, $vr8, $vr4\n\t" - "vshuf.b $vr9, $vr9, $vr9, $vr5\n\t" - "vor.v $vr12, $vr3, $vr3\n\t" - "vor.v $vr13, $vr3, $vr3\n\t" - "vpickev.b $vr10, $vr7, $vr6\n\t" - "vpickod.b $vr11, $vr7, $vr6\n\t" - "vmaddwev.h.bu $vr12, $vr10, $vr0\n\t" - "vmaddwev.h.bu $vr12, $vr11, $vr1\n\t" - "vmaddwod.h.bu $vr12, $vr10, $vr2\n\t" - "vpickev.b $vr10, $vr9, $vr8\n\t" - "vpickod.b $vr11, $vr9, $vr8\n\t" - "vmaddwev.h.bu $vr13, $vr10, $vr0\n\t" - "vmaddwev.h.bu $vr13, $vr11, $vr1\n\t" - "vmaddwod.h.bu $vr13, $vr10, $vr2\n\t" - "vpickod.b $vr12, $vr13, $vr12\n\t" - "vst $vr12, %[dst], 0\n\t" - "addi.d %[src], %[src], 48\n\t" - "addi.d %[dst], %[dst], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_rgba), [dst] "+r"(dst_y), [cnt] "+r"(width) - : [c] "r"(c), [shuf] "r"(kShuf) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13"); -} - -void RGB565ToYRow_LSX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { - asm volatile( - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "vldi $vr7, 63\n\t" - "vsrli.h $vr7, $vr7, 8\n\t" - "vldi $vr8, 25\n\t" - "vsrli.h $vr8, $vr8, 8\n\t" - "vldi $vr9, 129\n\t" - "vsrli.h $vr9, $vr9, 8\n\t" - "vldi $vr10, 66\n\t" - "vsrli.h $vr10, $vr10, 8\n\t" - "vldi $vr11, 33\n\t" - "vsrli.h $vr11, $vr11, 8\n\t" - "vslli.h $vr11, $vr11, 7\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr7\n\t" - "vsrli.h $vr3, $vr0, 11\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 2\n\t" - "vsrli.h $vr2, $vr2, 4\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vmul.h $vr1, $vr1, $vr8\n\t" - "vmadd.h $vr1, $vr2, $vr9\n\t" - "vmadd.h $vr1, $vr3, $vr10\n\t" - "vadd.h $vr1, $vr1, $vr11\n\t" - "vpickod.b $vr1, $vr1, $vr1\n\t" - "vstelm.d $vr1, %[dst], 0, 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 8\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_rgb565), [dst] "+r"(dst_y), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7", - "$vr8", "$vr9", "$vr10", "$vr11"); -} - -void ARGB1555ToYRow_LSX(const uint8_t* src_argb1555, - uint8_t* dst_y, - int width) { - asm volatile( - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "vldi $vr8, 25\n\t" - "vsrli.h $vr8, $vr8, 8\n\t" - "vldi $vr9, 129\n\t" - "vsrli.h $vr9, $vr9, 8\n\t" - "vldi $vr10, 66\n\t" - "vsrli.h $vr10, $vr10, 8\n\t" - "vldi $vr11, 33\n\t" - "vsrli.h $vr11, $vr11, 8\n\t" - "vslli.h $vr11, $vr11, 7\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr6\n\t" - "vsrli.h $vr3, $vr0, 10\n\t" - "vand.v $vr3, $vr3, $vr6\n\t" - "vsrai.h $vr5, $vr0, 15\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 3\n\t" - "vsrli.h $vr2, $vr2, 2\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vmul.h $vr1, $vr1, $vr8\n\t" - "vmadd.h $vr1, $vr2, $vr9\n\t" - "vmadd.h $vr1, $vr3, $vr10\n\t" - "vadd.h $vr1, $vr1, $vr11\n\t" - "vpickod.b $vr1, $vr1, $vr1\n\t" - "vstelm.d $vr1, %[dst], 0, 0\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[dst], %[dst], 8\n\t" - "addi.d %[cnt], %[cnt], -8\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src_argb1555), [dst] "+r"(dst_y), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr8", "$vr9", "$vr10", "$vr11"); -} - -void RGB565ToUVRow_LSX(const uint8_t* src_rgb565, - int src_stride_rgb565, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - const uint8_t* next_rgb565 = src_rgb565 + src_stride_rgb565; - asm volatile( - "vldi $vr20, 112\n\t" - "vsrli.h $vr20, $vr20, 8\n\t" - "vldi $vr21, 74\n\t" - "vsrli.h $vr21, $vr21, 8\n\t" - "vldi $vr22, 38\n\t" - "vsrli.h $vr22, $vr22, 8\n\t" - "vldi $vr23, 94\n\t" - "vsrli.h $vr23, $vr23, 8\n\t" - "vldi $vr24, 18\n\t" - "vsrli.h $vr24, $vr24, 8\n\t" - "vldi $vr25, 128\n\t" - "vsrli.h $vr25, $vr25, 8\n\t" - "vslli.h $vr25, $vr25, 8\n\t" - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "vldi $vr7, 63\n\t" - "vsrli.h $vr7, $vr7, 8\n\t" - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr7\n\t" - "vsrli.h $vr3, $vr0, 11\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 2\n\t" - "vsrli.h $vr2, $vr2, 4\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vor.v $vr10, $vr1, $vr1\n\t" - "vor.v $vr11, $vr2, $vr2\n\t" - "vor.v $vr12, $vr3, $vr3\n\t" - "vld $vr0, %[s0], 16\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr7\n\t" - "vsrli.h $vr3, $vr0, 11\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 2\n\t" - "vsrli.h $vr2, $vr2, 4\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vpickev.b $vr10, $vr1, $vr10\n\t" - "vpickev.b $vr11, $vr2, $vr11\n\t" - "vpickev.b $vr12, $vr3, $vr12\n\t" - "vld $vr0, %[s1], 0\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr7\n\t" - "vsrli.h $vr3, $vr0, 11\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 2\n\t" - "vsrli.h $vr2, $vr2, 4\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vor.v $vr13, $vr1, $vr1\n\t" - "vor.v $vr14, $vr2, $vr2\n\t" - "vor.v $vr15, $vr3, $vr3\n\t" - "vld $vr0, %[s1], 16\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr7\n\t" - "vsrli.h $vr3, $vr0, 11\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 2\n\t" - "vsrli.h $vr2, $vr2, 4\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vpickev.b $vr5, $vr3, $vr15\n\t" - "vpickev.b $vr4, $vr2, $vr14\n\t" - "vpickev.b $vr3, $vr1, $vr13\n\t" - "vor.v $vr0, $vr10, $vr10\n\t" - "vor.v $vr1, $vr11, $vr11\n\t" - "vor.v $vr2, $vr12, $vr12\n\t" - "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" - "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" - "vadd.h $vr0, $vr0, $vr3\n\t" - "vsrari.h $vr0, $vr0, 2\n\t" - "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" - "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" - "vadd.h $vr1, $vr1, $vr4\n\t" - "vsrari.h $vr1, $vr1, 2\n\t" - "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" - "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" - "vadd.h $vr2, $vr2, $vr5\n\t" - "vsrari.h $vr2, $vr2, 2\n\t" - "vor.v $vr6, $vr25, $vr25\n\t" - "vor.v $vr7, $vr25, $vr25\n\t" - "vmadd.h $vr6, $vr0, $vr20\n\t" - "vmsub.h $vr6, $vr1, $vr21\n\t" - "vmsub.h $vr6, $vr2, $vr22\n\t" - "vmadd.h $vr7, $vr2, $vr20\n\t" - "vmsub.h $vr7, $vr1, $vr23\n\t" - "vmsub.h $vr7, $vr0, $vr24\n\t" - "vpickod.b $vr6, $vr7, $vr6\n\t" - "vstelm.d $vr6, %[du], 0, 0\n\t" - "vstelm.d $vr6, %[dv], 0, 1\n\t" - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "vldi $vr7, 63\n\t" - "vsrli.h $vr7, $vr7, 8\n\t" - "addi.d %[s0], %[s0], 32\n\t" - "addi.d %[s1], %[s1], 32\n\t" - "addi.d %[du], %[du], 8\n\t" - "addi.d %[dv], %[dv], 8\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_rgb565), [s1] "+r"(next_rgb565), [du] "+r"(dst_u), - [dv] "+r"(dst_v), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15", "$vr20", - "$vr21", "$vr22", "$vr23", "$vr24", "$vr25"); -} - -void ARGB1555ToUVRow_LSX(const uint8_t* src_argb1555, - int src_stride_argb1555, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - const uint8_t* next_argb1555 = src_argb1555 + src_stride_argb1555; - asm volatile( - "vldi $vr20, 112\n\t" - "vsrli.h $vr20, $vr20, 8\n\t" - "vldi $vr21, 74\n\t" - "vsrli.h $vr21, $vr21, 8\n\t" - "vldi $vr22, 38\n\t" - "vsrli.h $vr22, $vr22, 8\n\t" - "vldi $vr23, 94\n\t" - "vsrli.h $vr23, $vr23, 8\n\t" - "vldi $vr24, 18\n\t" - "vsrli.h $vr24, $vr24, 8\n\t" - "vldi $vr25, 128\n\t" - "vsrli.h $vr25, $vr25, 8\n\t" - "vslli.h $vr25, $vr25, 8\n\t" - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "1:\n\t" - "vld $vr0, %[s0], 0\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr6\n\t" - "vsrli.h $vr3, $vr0, 10\n\t" - "vand.v $vr3, $vr3, $vr6\n\t" - "vsrai.h $vr5, $vr0, 15\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 3\n\t" - "vsrli.h $vr2, $vr2, 2\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vor.v $vr10, $vr1, $vr1\n\t" - "vor.v $vr11, $vr2, $vr2\n\t" - "vor.v $vr12, $vr3, $vr3\n\t" - "vld $vr0, %[s0], 16\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr6\n\t" - "vsrli.h $vr3, $vr0, 10\n\t" - "vand.v $vr3, $vr3, $vr6\n\t" - "vsrai.h $vr5, $vr0, 15\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 3\n\t" - "vsrli.h $vr2, $vr2, 2\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vpickev.b $vr10, $vr1, $vr10\n\t" - "vpickev.b $vr11, $vr2, $vr11\n\t" - "vpickev.b $vr12, $vr3, $vr12\n\t" - "vld $vr0, %[s1], 0\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr6\n\t" - "vsrli.h $vr3, $vr0, 10\n\t" - "vand.v $vr3, $vr3, $vr6\n\t" - "vsrai.h $vr5, $vr0, 15\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 3\n\t" - "vsrli.h $vr2, $vr2, 2\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vor.v $vr13, $vr1, $vr1\n\t" - "vor.v $vr14, $vr2, $vr2\n\t" - "vor.v $vr15, $vr3, $vr3\n\t" - "vld $vr0, %[s1], 16\n\t" - "vand.v $vr1, $vr0, $vr6\n\t" - "vsrli.h $vr2, $vr0, 5\n\t" - "vand.v $vr2, $vr2, $vr6\n\t" - "vsrli.h $vr3, $vr0, 10\n\t" - "vand.v $vr3, $vr3, $vr6\n\t" - "vsrai.h $vr5, $vr0, 15\n\t" - "vslli.h $vr4, $vr1, 3\n\t" - "vsrli.h $vr1, $vr1, 2\n\t" - "vor.v $vr1, $vr4, $vr1\n\t" - "vslli.h $vr4, $vr2, 3\n\t" - "vsrli.h $vr2, $vr2, 2\n\t" - "vor.v $vr2, $vr4, $vr2\n\t" - "vslli.h $vr4, $vr3, 3\n\t" - "vsrli.h $vr3, $vr3, 2\n\t" - "vor.v $vr3, $vr4, $vr3\n\t" - "vpickev.b $vr5, $vr3, $vr15\n\t" - "vpickev.b $vr4, $vr2, $vr14\n\t" - "vpickev.b $vr3, $vr1, $vr13\n\t" - "vor.v $vr0, $vr10, $vr10\n\t" - "vor.v $vr1, $vr11, $vr11\n\t" - "vor.v $vr2, $vr12, $vr12\n\t" - "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" - "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" - "vadd.h $vr0, $vr0, $vr3\n\t" - "vsrari.h $vr0, $vr0, 2\n\t" - "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" - "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" - "vadd.h $vr1, $vr1, $vr4\n\t" - "vsrari.h $vr1, $vr1, 2\n\t" - "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" - "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" - "vadd.h $vr2, $vr2, $vr5\n\t" - "vsrari.h $vr2, $vr2, 2\n\t" - "vor.v $vr6, $vr25, $vr25\n\t" - "vor.v $vr7, $vr25, $vr25\n\t" - "vmadd.h $vr6, $vr0, $vr20\n\t" - "vmsub.h $vr6, $vr1, $vr21\n\t" - "vmsub.h $vr6, $vr2, $vr22\n\t" - "vmadd.h $vr7, $vr2, $vr20\n\t" - "vmsub.h $vr7, $vr1, $vr23\n\t" - "vmsub.h $vr7, $vr0, $vr24\n\t" - "vpickod.b $vr6, $vr7, $vr6\n\t" - "vstelm.d $vr6, %[du], 0, 0\n\t" - "vstelm.d $vr6, %[dv], 0, 1\n\t" - "vldi $vr6, 31\n\t" - "vsrli.h $vr6, $vr6, 8\n\t" - "addi.d %[s0], %[s0], 32\n\t" - "addi.d %[s1], %[s1], 32\n\t" - "addi.d %[du], %[du], 8\n\t" - "addi.d %[dv], %[dv], 8\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src_argb1555), [s1] "+r"(next_argb1555), [du] "+r"(dst_u), - [dv] "+r"(dst_v), [cnt] "+r"(width) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15", "$vr20", - "$vr21", "$vr22", "$vr23", "$vr24", "$vr25"); -} - -static LIBYUV_NOINLINE void XRGBToUVHelper_LSX(const uint8_t* src, - int stride, - uint8_t* dst_u, - uint8_t* dst_v, - const uint8_t* shuf, - int width) { - const uint8_t* next = src + stride; - asm volatile( - "vldi $vr20, 112\n\t" - "vsrli.h $vr20, $vr20, 8\n\t" - "vldi $vr21, 74\n\t" - "vsrli.h $vr21, $vr21, 8\n\t" - "vldi $vr22, 38\n\t" - "vsrli.h $vr22, $vr22, 8\n\t" - "vldi $vr23, 94\n\t" - "vsrli.h $vr23, $vr23, 8\n\t" - "vldi $vr24, 18\n\t" - "vsrli.h $vr24, $vr24, 8\n\t" - "vldi $vr25, 128\n\t" - "vsrli.h $vr25, $vr25, 8\n\t" - "vslli.h $vr25, $vr25, 8\n\t" - "vld $vr16, %[shuf], 0\n\t" - "1:\n\t" - "vld $vr8, %[s0], 0\n\t" - "vld $vr9, %[s0], 16\n\t" - "vld $vr10, %[s0], 32\n\t" - "vld $vr11, %[s0], 48\n\t" - "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" - "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" - "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" - "vshuf.b $vr11, $vr11, $vr11, $vr16\n\t" - "vpickev.b $vr12, $vr9, $vr8\n\t" - "vpickod.b $vr13, $vr9, $vr8\n\t" - "vpickev.b $vr14, $vr11, $vr10\n\t" - "vpickod.b $vr15, $vr11, $vr10\n\t" - "vpickev.b $vr0, $vr14, $vr12\n\t" - "vpickod.b $vr2, $vr14, $vr12\n\t" - "vpickev.b $vr1, $vr15, $vr13\n\t" - "vld $vr8, %[s1], 0\n\t" - "vld $vr9, %[s1], 16\n\t" - "vld $vr10, %[s1], 32\n\t" - "vld $vr11, %[s1], 48\n\t" - "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" - "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" - "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" - "vshuf.b $vr11, $vr11, $vr11, $vr16\n\t" - "vpickev.b $vr12, $vr9, $vr8\n\t" - "vpickod.b $vr13, $vr9, $vr8\n\t" - "vpickev.b $vr14, $vr11, $vr10\n\t" - "vpickod.b $vr15, $vr11, $vr10\n\t" - "vpickev.b $vr3, $vr14, $vr12\n\t" - "vpickod.b $vr5, $vr14, $vr12\n\t" - "vpickev.b $vr4, $vr15, $vr13\n\t" - "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" - "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" - "vadd.h $vr0, $vr0, $vr3\n\t" - "vsrari.h $vr0, $vr0, 2\n\t" - "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" - "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" - "vadd.h $vr1, $vr1, $vr4\n\t" - "vsrari.h $vr1, $vr1, 2\n\t" - "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" - "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" - "vadd.h $vr2, $vr2, $vr5\n\t" - "vsrari.h $vr2, $vr2, 2\n\t" - "vor.v $vr6, $vr25, $vr25\n\t" - "vor.v $vr7, $vr25, $vr25\n\t" - "vmadd.h $vr6, $vr0, $vr20\n\t" - "vmsub.h $vr6, $vr1, $vr21\n\t" - "vmsub.h $vr6, $vr2, $vr22\n\t" - "vmadd.h $vr7, $vr2, $vr20\n\t" - "vmsub.h $vr7, $vr1, $vr23\n\t" - "vmsub.h $vr7, $vr0, $vr24\n\t" - "vpickod.b $vr6, $vr7, $vr6\n\t" - "vstelm.d $vr6, %[du], 0, 0\n\t" - "vstelm.d $vr6, %[dv], 0, 1\n\t" - "addi.d %[s0], %[s0], 64\n\t" - "addi.d %[s1], %[s1], 64\n\t" - "addi.d %[du], %[du], 8\n\t" - "addi.d %[dv], %[dv], 8\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v), - [cnt] "+r"(width) - : [shuf] "r"(shuf) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", - "$vr15", "$vr16", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24", "$vr25"); -} - -void BGRAToUVRow_LSX(const uint8_t* src_bgra, - int src_stride_bgra, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - static const uint8_t kShuf[16] = {3, 2, 1, 0, 7, 6, 5, 4, - 11, 10, 9, 8, 15, 14, 13, 12}; - XRGBToUVHelper_LSX(src_bgra, src_stride_bgra, dst_u, dst_v, kShuf, width); -} - -void ABGRToUVRow_LSX(const uint8_t* src_abgr, - int src_stride_abgr, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - static const uint8_t kShuf[16] = {2, 1, 0, 3, 6, 5, 4, 7, - 10, 9, 8, 11, 14, 13, 12, 15}; - XRGBToUVHelper_LSX(src_abgr, src_stride_abgr, dst_u, dst_v, kShuf, width); -} - -void RGBAToUVRow_LSX(const uint8_t* src_rgba, - int src_stride_rgba, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - static const uint8_t kShuf[16] = {1, 2, 3, 0, 5, 6, 7, 4, - 9, 10, 11, 8, 13, 14, 15, 12}; - XRGBToUVHelper_LSX(src_rgba, src_stride_rgba, dst_u, dst_v, kShuf, width); -} - -static LIBYUV_NOINLINE void RGB24ToUVHelper_LSX(const uint8_t* src, - int stride, - uint8_t* dst_u, - uint8_t* dst_v, - const uint8_t* shuf, - int width) { - const uint8_t* next = src + stride; - asm volatile( - "vldi $vr20, 112\n\t" - "vsrli.h $vr20, $vr20, 8\n\t" - "vldi $vr21, 74\n\t" - "vsrli.h $vr21, $vr21, 8\n\t" - "vldi $vr22, 38\n\t" - "vsrli.h $vr22, $vr22, 8\n\t" - "vldi $vr23, 94\n\t" - "vsrli.h $vr23, $vr23, 8\n\t" - "vldi $vr24, 18\n\t" - "vsrli.h $vr24, $vr24, 8\n\t" - "vldi $vr25, 128\n\t" - "vsrli.h $vr25, $vr25, 8\n\t" - "vslli.h $vr25, $vr25, 8\n\t" - "vld $vr16, %[shuf], 0\n\t" - "vld $vr17, %[shuf], 16\n\t" - "1:\n\t" - "vld $vr8, %[s0], 0\n\t" - "vld $vr9, %[s0], 12\n\t" - "vld $vr10, %[s0], 24\n\t" - "vld $vr11, %[s0], 32\n\t" - "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" - "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" - "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" - "vshuf.b $vr11, $vr11, $vr11, $vr17\n\t" - "vpickev.b $vr12, $vr9, $vr8\n\t" - "vpickod.b $vr13, $vr9, $vr8\n\t" - "vpickev.b $vr14, $vr11, $vr10\n\t" - "vpickod.b $vr15, $vr11, $vr10\n\t" - "vpickev.b $vr0, $vr14, $vr12\n\t" - "vpickod.b $vr2, $vr14, $vr12\n\t" - "vpickev.b $vr1, $vr15, $vr13\n\t" - "vld $vr8, %[s1], 0\n\t" - "vld $vr9, %[s1], 12\n\t" - "vld $vr10, %[s1], 24\n\t" - "vld $vr11, %[s1], 32\n\t" - "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" - "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" - "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" - "vshuf.b $vr11, $vr11, $vr11, $vr17\n\t" - "vpickev.b $vr12, $vr9, $vr8\n\t" - "vpickod.b $vr13, $vr9, $vr8\n\t" - "vpickev.b $vr14, $vr11, $vr10\n\t" - "vpickod.b $vr15, $vr11, $vr10\n\t" - "vpickev.b $vr3, $vr14, $vr12\n\t" - "vpickod.b $vr5, $vr14, $vr12\n\t" - "vpickev.b $vr4, $vr15, $vr13\n\t" - "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" - "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" - "vadd.h $vr0, $vr0, $vr3\n\t" - "vsrari.h $vr0, $vr0, 2\n\t" - "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" - "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" - "vadd.h $vr1, $vr1, $vr4\n\t" - "vsrari.h $vr1, $vr1, 2\n\t" - "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" - "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" - "vadd.h $vr2, $vr2, $vr5\n\t" - "vsrari.h $vr2, $vr2, 2\n\t" - "vor.v $vr6, $vr25, $vr25\n\t" - "vor.v $vr7, $vr25, $vr25\n\t" - "vmadd.h $vr6, $vr0, $vr20\n\t" - "vmsub.h $vr6, $vr1, $vr21\n\t" - "vmsub.h $vr6, $vr2, $vr22\n\t" - "vmadd.h $vr7, $vr2, $vr20\n\t" - "vmsub.h $vr7, $vr1, $vr23\n\t" - "vmsub.h $vr7, $vr0, $vr24\n\t" - "vpickod.b $vr6, $vr7, $vr6\n\t" - "vstelm.d $vr6, %[du], 0, 0\n\t" - "vstelm.d $vr6, %[dv], 0, 1\n\t" - "addi.d %[s0], %[s0], 48\n\t" - "addi.d %[s1], %[s1], 48\n\t" - "addi.d %[du], %[du], 8\n\t" - "addi.d %[dv], %[dv], 8\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v), - [cnt] "+r"(width) - : [shuf] "r"(shuf) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", - "$vr15", "$vr16", "$vr17", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24", - "$vr25"); -} - -void RGB24ToUVRow_LSX(const uint8_t* src_rgb24, - int src_stride_rgb24, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8, - 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8, - 9, 0, 10, 11, 12, 0, 13, 14, 15, 0}; - RGB24ToUVHelper_LSX(src_rgb24, src_stride_rgb24, dst_u, dst_v, kShuf, width); -} - -void RAWToUVRow_LSX(const uint8_t* src_raw, - int src_stride_raw, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - static const uint8_t kShuf[32] = {2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6, - 0, 11, 10, 9, 0, 6, 5, 4, 0, 9, 8, - 7, 0, 12, 11, 10, 0, 15, 14, 13, 0}; - RGB24ToUVHelper_LSX(src_raw, src_stride_raw, dst_u, dst_v, kShuf, width); -} - -#ifdef HAS_SSIM4X4ROW_LSX -void Ssim4x4Row_LSX(const uint8_t* src_a, - int stride_a, - const uint8_t* src_b, - int stride_b, - uint32_t* dst, - int width) { - const uint8_t* pa; - const uint8_t* pb; - int r; - asm volatile( - "1:\n\t" - "move %[pa], %[sa]\n\t" - "move %[pb], %[sb]\n\t" - "li.d %[r], 4\n\t" - "vxor.v $vr8, $vr8, $vr8\n\t" - "vxor.v $vr9, $vr9, $vr9\n\t" - "vxor.v $vr10, $vr10, $vr10\n\t" - "vxor.v $vr11, $vr11, $vr11\n\t" - "2:\n\t" - "vld $vr0, %[pa], 0\n\t" - "vld $vr1, %[pb], 0\n\t" - "add.d %[pa], %[pa], %[sta]\n\t" - "add.d %[pb], %[pb], %[stb]\n\t" - "vhaddw.hu.bu $vr2, $vr0, $vr0\n\t" - "vhaddw.hu.bu $vr3, $vr1, $vr1\n\t" - "vadd.h $vr8, $vr8, $vr2\n\t" - "vadd.h $vr9, $vr9, $vr3\n\t" - "vmulwev.h.bu $vr2, $vr0, $vr1\n\t" - "vmulwod.h.bu $vr3, $vr0, $vr1\n\t" - "vhaddw.wu.hu $vr2, $vr2, $vr2\n\t" - "vhaddw.wu.hu $vr3, $vr3, $vr3\n\t" - "vadd.w $vr11, $vr11, $vr2\n\t" - "vadd.w $vr11, $vr11, $vr3\n\t" - "vmulwev.h.bu $vr2, $vr0, $vr0\n\t" - "vmulwod.h.bu $vr3, $vr0, $vr0\n\t" - "vmulwev.h.bu $vr4, $vr1, $vr1\n\t" - "vmulwod.h.bu $vr5, $vr1, $vr1\n\t" - "vhaddw.wu.hu $vr2, $vr2, $vr2\n\t" - "vhaddw.wu.hu $vr3, $vr3, $vr3\n\t" - "vhaddw.wu.hu $vr4, $vr4, $vr4\n\t" - "vhaddw.wu.hu $vr5, $vr5, $vr5\n\t" - "vadd.w $vr2, $vr2, $vr3\n\t" - "vadd.w $vr4, $vr4, $vr5\n\t" - "vadd.w $vr10, $vr10, $vr2\n\t" - "vadd.w $vr10, $vr10, $vr4\n\t" - "addi.d %[r], %[r], -1\n\t" - "bnez %[r], 2b\n\t" - "vhaddw.wu.hu $vr8, $vr8, $vr8\n\t" - "vhaddw.wu.hu $vr9, $vr9, $vr9\n\t" - "vilvl.w $vr0, $vr9, $vr8\n\t" - "vilvh.w $vr1, $vr9, $vr8\n\t" - "vilvl.w $vr2, $vr11, $vr10\n\t" - "vilvh.w $vr3, $vr11, $vr10\n\t" - "vilvl.d $vr4, $vr2, $vr0\n\t" - "vilvh.d $vr5, $vr2, $vr0\n\t" - "vilvl.d $vr6, $vr3, $vr1\n\t" - "vilvh.d $vr7, $vr3, $vr1\n\t" - "vst $vr4, %[dst], 0\n\t" - "vst $vr5, %[dst], 16\n\t" - "vst $vr6, %[dst], 32\n\t" - "vst $vr7, %[dst], 48\n\t" - "addi.d %[sa], %[sa], 16\n\t" - "addi.d %[sb], %[sb], 16\n\t" - "addi.d %[dst], %[dst], 64\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [sa] "+r"(src_a), [sb] "+r"(src_b), [dst] "+r"(dst), [cnt] "+r"(width), - [pa] "=&r"(pa), [pb] "=&r"(pb), [r] "=&r"(r) - : [sta] "r"((ptrdiff_t)stride_a), [stb] "r"((ptrdiff_t)stride_b) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11"); -} -#endif // HAS_SSIM4X4ROW_LSX - -#ifdef HAS_SSIMWINDOWROW_LSX -float SsimWindowRow_LSX(const uint32_t* row0, - const uint32_t* row1, - int num_windows) { - const uint32_t* row0_next = row0 + 4; - const uint32_t* row1_next = row1 + 4; - float total; - uint32_t c1 = 26634; - uint32_t c2 = 239708; - uint32_t c0 = 0; - asm volatile( - "vreplgr2vr.w $vr6, %[c0]\n\t" - "vreplgr2vr.w $vr4, %[c1]\n\t" - "vreplgr2vr.w $vr5, %[c2]\n\t" - "1:\n\t" - "vld $vr0, %[r0], 0\n\t" - "vld $vr1, %[r0n], 0\n\t" - "vld $vr2, %[r1], 0\n\t" - "vld $vr3, %[r1n], 0\n\t" - "vadd.w $vr0, $vr0, $vr1\n\t" - "vadd.w $vr2, $vr2, $vr3\n\t" - "vadd.w $vr0, $vr0, $vr2\n\t" - "vshuf4i.w $vr1, $vr0, 0x00\n\t" - "vshuf4i.w $vr2, $vr0, 0x55\n\t" - "vshuf4i.w $vr3, $vr0, 0xaa\n\t" - "vshuf4i.w $vr0, $vr0, 0xff\n\t" - "vmul.w $vr7, $vr1, $vr2\n\t" - "vslli.w $vr7, $vr7, 1\n\t" - "vmul.w $vr1, $vr1, $vr1\n\t" - "vmadd.w $vr1, $vr2, $vr2\n\t" - "vslli.w $vr0, $vr0, 7\n\t" - "vsub.w $vr0, $vr0, $vr7\n\t" - "vadd.w $vr7, $vr7, $vr4\n\t" - "vadd.w $vr0, $vr0, $vr5\n\t" - "vslli.w $vr3, $vr3, 6\n\t" - "vsub.w $vr3, $vr3, $vr1\n\t" - "vadd.w $vr1, $vr1, $vr4\n\t" - "vadd.w $vr3, $vr3, $vr5\n\t" - "vffint.s.w $vr7, $vr7\n\t" - "vffint.s.w $vr0, $vr0\n\t" - "vffint.s.w $vr1, $vr1\n\t" - "vffint.s.w $vr3, $vr3\n\t" - "vfmul.s $vr7, $vr7, $vr0\n\t" - "vfmul.s $vr1, $vr1, $vr3\n\t" - "vfdiv.s $vr7, $vr7, $vr1\n\t" - "vfadd.s $vr6, $vr6, $vr7\n\t" - "addi.d %[r0], %[r0], 16\n\t" - "addi.d %[r0n], %[r0n], 16\n\t" - "addi.d %[r1], %[r1], 16\n\t" - "addi.d %[r1n], %[r1n], 16\n\t" - "addi.d %[cnt], %[cnt], -1\n\t" - "bnez %[cnt], 1b\n\t" - "vstelm.w $vr6, %[tot], 0, 0\n\t" - : [r0] "+r"(row0), [r0n] "+r"(row0_next), [r1] "+r"(row1), - [r1n] "+r"(row1_next), [cnt] "+r"(num_windows) - : [c0] "r"(c0), [c1] "r"(c1), [c2] "r"(c2), [tot] "r"(&total) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7"); - return total; -} -#endif // HAS_SSIMWINDOWROW_LSX - -#ifdef HAS_ARGBDETECTROW_LSX -uint32_t ARGBDetectRow_LSX(const uint8_t* argb, int width) { - uint32_t diff = 0; - uint32_t rgb_mask = 0x00ffff00u; - asm volatile( - "vreplgr2vr.w $vr2, %[rgb_mask]\n\t" - "vldi $vr3, -1\n\t" - "addi.d %[w], %[w], -4\n\t" - "bltz %[w], 2f\n\t" - "1:\n\t" - "vld $vr0, %[argb], 0\n\t" - "addi.d %[argb], %[argb], 16\n\t" - "vor.v $vr0, $vr0, $vr2\n\t" - "vseq.b $vr0, $vr0, $vr3\n\t" - "vmskltz.b $vr0, $vr0\n\t" - "vpickve2gr.hu %[diff], $vr0, 0\n\t" - "nor %[diff], %[diff], $zero\n\t" - "bstrpick.w %[diff], %[diff], 15, 0\n\t" - "bnez %[diff], 4f\n\t" - "addi.d %[w], %[w], -4\n\t" - "bgez %[w], 1b\n\t" - "2:\n\t" - "addi.d %[w], %[w], 4\n\t" - "beqz %[w], 4f\n\t" - "3:\n\t" - "vldrepl.w $vr0, %[argb], 0\n\t" - "addi.d %[argb], %[argb], 4\n\t" - "vor.v $vr0, $vr0, $vr2\n\t" - "vseq.b $vr0, $vr0, $vr3\n\t" - "vmskltz.b $vr0, $vr0\n\t" - "vpickve2gr.hu %[diff], $vr0, 0\n\t" - "nor %[diff], %[diff], $zero\n\t" - "bstrpick.w %[diff], %[diff], 15, 0\n\t" - "bnez %[diff], 4f\n\t" - "addi.d %[w], %[w], -1\n\t" - "bnez %[w], 3b\n\t" - "4:\n\t" - : [argb] "+r"(argb), [w] "+r"(width), [diff] "+r"(diff) - : [rgb_mask] "r"(rgb_mask) - : "memory", "$vr0", "$vr2", "$vr3"); - if (diff) { - return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB; - } - return 0; -} -#endif // HAS_ARGBDETECTROW_LSX - -#ifdef HAS_HAMMINGDISTANCE_LSX -uint32_t HammingDistance_LSX(const uint8_t* src_a, - const uint8_t* src_b, - int count) { - uint32_t diff; - asm volatile( - "vxor.v $vr4, $vr4, $vr4\n\t" - "1:\n\t" - "vld $vr0, %[sa], 0\n\t" - "vld $vr1, %[sb], 0\n\t" - "vxor.v $vr0, $vr0, $vr1\n\t" - "vpcnt.b $vr0, $vr0\n\t" - "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" - "vhaddw.wu.hu $vr0, $vr0, $vr0\n\t" - "vadd.w $vr4, $vr4, $vr0\n\t" - "addi.d %[sa], %[sa], 16\n\t" - "addi.d %[sb], %[sb], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - "vhaddw.du.wu $vr4, $vr4, $vr4\n\t" - "vhaddw.qu.du $vr4, $vr4, $vr4\n\t" - "vpickve2gr.wu %[diff], $vr4, 0\n\t" - : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [diff] "=r"(diff) - : - : "memory", "$vr0", "$vr1", "$vr4"); - return diff; -} -#endif // HAS_HAMMINGDISTANCE_LSX - -#ifdef HAS_SUMSQUAREERROR_LSX -uint32_t SumSquareError_LSX(const uint8_t* src_a, - const uint8_t* src_b, - int count) { - uint32_t sse; - asm volatile( - "vxor.v $vr4, $vr4, $vr4\n\t" - "1:\n\t" - "vld $vr0, %[sa], 0\n\t" - "vld $vr1, %[sb], 0\n\t" - "vabsd.bu $vr0, $vr0, $vr1\n\t" - "vmulwev.h.bu $vr2, $vr0, $vr0\n\t" - "vmulwod.h.bu $vr3, $vr0, $vr0\n\t" - "vhaddw.wu.hu $vr2, $vr2, $vr2\n\t" - "vhaddw.wu.hu $vr3, $vr3, $vr3\n\t" - "vadd.w $vr4, $vr4, $vr2\n\t" - "vadd.w $vr4, $vr4, $vr3\n\t" - "addi.d %[sa], %[sa], 16\n\t" - "addi.d %[sb], %[sb], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - "vhaddw.du.wu $vr4, $vr4, $vr4\n\t" - "vhaddw.qu.du $vr4, $vr4, $vr4\n\t" - "vpickve2gr.wu %[sse], $vr4, 0\n\t" - : [sa] "+r"(src_a), [sb] "+r"(src_b), [cnt] "+r"(count), [sse] "=r"(sse) - : - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); - return sse; -} -#endif // HAS_SUMSQUAREERROR_LSX - -#ifdef HAS_HASHDJB2_LSX -static const uint32_t kDjb2Muls_LSX[16] = { - 0x0c3525e1, 0xa3476dc1, 0x3b4039a1, 0x4f5f0981, 0x30f35d61, 0x855cb541, - 0x040a9121, 0x747c7101, 0xec41d4e1, 0x4cfa3cc1, 0x025528a1, 0x00121881, - 0x00008c61, 0x00000441, 0x00000021, 0x00000001, -}; - -uint32_t HashDjb2_LSX(const uint8_t* src, int count, uint32_t seed) { - uint32_t hash = seed; - uint32_t tmp; - uint32_t c16 = 0x92d9e201u; - asm volatile( - "vld $vr4, %[muls], 0\n\t" - "vld $vr5, %[muls], 16\n\t" - "vld $vr6, %[muls], 32\n\t" - "vld $vr7, %[muls], 48\n\t" - "1:\n\t" - "vld $vr0, %[src], 0\n\t" - "vsllwil.hu.bu $vr1, $vr0, 0\n\t" - "vexth.hu.bu $vr2, $vr0\n\t" - "vsllwil.wu.hu $vr0, $vr1, 0\n\t" - "vexth.wu.hu $vr1, $vr1\n\t" - "vsllwil.wu.hu $vr3, $vr2, 0\n\t" - "vexth.wu.hu $vr2, $vr2\n\t" - "vmul.w $vr0, $vr0, $vr4\n\t" - "vmadd.w $vr0, $vr1, $vr5\n\t" - "vmadd.w $vr0, $vr3, $vr6\n\t" - "vmadd.w $vr0, $vr2, $vr7\n\t" - "vhaddw.du.wu $vr0, $vr0, $vr0\n\t" - "vhaddw.qu.du $vr0, $vr0, $vr0\n\t" - "vpickve2gr.wu %[tmp], $vr0, 0\n\t" - "mul.w %[hash], %[hash], %[c16]\n\t" - "add.w %[hash], %[hash], %[tmp]\n\t" - "addi.d %[src], %[src], 16\n\t" - "addi.d %[cnt], %[cnt], -16\n\t" - "bnez %[cnt], 1b\n\t" - : [src] "+r"(src), [cnt] "+r"(count), [hash] "+r"(hash), [tmp] "=&r"(tmp) - : [muls] "r"(kDjb2Muls_LSX), [c16] "r"(c16) - : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7"); - return hash; -} -#endif // HAS_HASHDJB2_LSX - -#ifdef __cplusplus -} // extern "C" -} // namespace libyuv -#endif - -#endif // !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) diff --git a/source/scale_lsx.cc b/source/scale_la.cc similarity index 97% rename from source/scale_lsx.cc rename to source/scale_la.cc index cdb333ee4..ddda812eb 100644 --- a/source/scale_lsx.cc +++ b/source/scale_la.cc @@ -8,34 +8,18 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include - +#include "libyuv/row.h" #include "libyuv/scale_row.h" -#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) +#if (!defined(LIBYUV_DISABLE_LSX) || !defined(LIBYUV_DISABLE_LASX)) && \ + defined(__loongarch_lp64) #ifdef __cplusplus namespace libyuv { extern "C" { #endif -static const uint64_t kScale38Shuf[2] = {0x13100e0b08060300ULL, - 0x000000001e1b1816ULL}; -static const uint64_t kScale38BoxShuf[2] = {0x0a08160604120200ULL, - 0x000000001e0e0c1aULL}; -static const uint64_t kScale34Shuf[2] = {0x0908070504030100ULL, - 0x000000000f0d0c0bULL}; -static const uint64_t kScale34BoxLo[2] = {0x0504030202010100ULL, - 0x0000000007060605ULL}; -static const uint64_t kScale34BoxHi[2] = {0x0d0c0b0a0a090908ULL, - 0x000000000f0e0e0dULL}; -static const uint64_t kScale34BoxWt[2] = {0x0103030102020103ULL, - 0x0000000003010202ULL}; -static const uint64_t kScale34Pack12[2] = {0x0908050403020100ULL, - 0x000000000d0c0b0aULL}; -static const uint64_t kRamp0123[2] = {0x0000000100000000ULL, - 0x0000000300000002ULL}; - +#ifdef HAS_SCALEARGBROWDOWN2_LSX void ScaleARGBRowDown2_LSX(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, @@ -129,7 +113,9 @@ void ScaleARGBRowDown2Box_LSX(const uint8_t* src_argb, : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7"); } +#endif // HAS_SCALEARGBROWDOWN2_LSX +#ifdef HAS_SCALEARGBROWDOWNEVEN_LSX void ScaleARGBRowDownEven_LSX(const uint8_t* src_argb, ptrdiff_t src_stride, int32_t src_stepx, @@ -235,7 +221,9 @@ void ScaleARGBRowDownEvenBox_LSX(const uint8_t* src_argb, : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7"); } +#endif // HAS_SCALEARGBROWDOWNEVEN_LSX +#ifdef HAS_SCALEROWDOWN2_LSX void ScaleRowDown2_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, @@ -323,7 +311,9 @@ void ScaleRowDown2Box_LSX(const uint8_t* src_ptr, : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7"); } +#endif // HAS_SCALEROWDOWN2_LSX +#ifdef HAS_SCALEROWDOWN4_LSX void ScaleRowDown4_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, @@ -406,6 +396,13 @@ void ScaleRowDown4Box_LSX(const uint8_t* src_ptr, : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7"); } +#endif // HAS_SCALEROWDOWN4_LSX + +#ifdef HAS_SCALEROWDOWN38_LSX +static const uint64_t kScale38Shuf[2] = {0x13100e0b08060300ULL, + 0x000000001e1b1816ULL}; +static const uint64_t kScale38BoxShuf[2] = {0x0a08160604120200ULL, + 0x000000001e0e0c1aULL}; void ScaleRowDown38_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -553,7 +550,9 @@ void ScaleRowDown38_3_Box_LSX(const uint8_t* src_ptr, : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9", "$vr10", "$vr11"); } +#endif // HAS_SCALEROWDOWN38_LSX +#ifdef HAS_SCALEADDROW_LSX void ScaleAddRow_LSX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) { asm volatile( "srli.w %[w], %[w], 4 \n" @@ -580,7 +579,9 @@ void ScaleAddRow_LSX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) { : : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); } +#endif // HAS_SCALEADDROW_LSX +#ifdef HAS_SCALEUVADDCOLS3_LSX static const uint8_t kScaleUVAddCols3Shuf[] = { 0, 1, 2, 3, 12, 13, 14, 15, 24, 25, 26, 27, 4, 5, 6, 7, 4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31, 8, 9, 10, 11, @@ -641,7 +642,15 @@ void ScaleUVAddCols3_LSX(int dst_width, : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13"); } +#endif // HAS_SCALEUVADDCOLS3_LSX +#if defined(HAS_SCALEFILTERCOLS_LSX) || defined(HAS_SCALEARGBCOLS_LSX) || \ + defined(HAS_SCALEARGBFILTERCOLS_LSX) +static const uint64_t kRamp0123[2] = {0x0000000100000000ULL, + 0x0000000300000002ULL}; +#endif + +#ifdef HAS_SCALEFILTERCOLS_LSX void ScaleFilterCols_LSX(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, @@ -706,7 +715,9 @@ void ScaleFilterCols_LSX(uint8_t* dst_ptr, : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7"); } +#endif // HAS_SCALEFILTERCOLS_LSX +#ifdef HAS_SCALEARGBCOLS_LSX void ScaleARGBCols_LSX(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, @@ -757,7 +768,9 @@ void ScaleARGBCols_LSX(uint8_t* dst_argb, [ramp] "r"(kRamp0123) // %[ramp] : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); } +#endif // HAS_SCALEARGBCOLS_LSX +#ifdef HAS_SCALEARGBFILTERCOLS_LSX void ScaleARGBFilterCols_LSX(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, @@ -826,6 +839,19 @@ void ScaleARGBFilterCols_LSX(uint8_t* dst_argb, : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7", "$vr8"); } +#endif // HAS_SCALEARGBFILTERCOLS_LSX + +#ifdef HAS_SCALEROWDOWN34_LSX +static const uint64_t kScale34Shuf[2] = {0x0908070504030100ULL, + 0x000000000f0d0c0bULL}; +static const uint64_t kScale34BoxLo[2] = {0x0504030202010100ULL, + 0x0000000007060605ULL}; +static const uint64_t kScale34BoxHi[2] = {0x0d0c0b0a0a090908ULL, + 0x000000000f0e0e0dULL}; +static const uint64_t kScale34BoxWt[2] = {0x0103030102020103ULL, + 0x0000000003010202ULL}; +static const uint64_t kScale34Pack12[2] = {0x0908050403020100ULL, + 0x000000000d0c0b0aULL}; void ScaleRowDown34_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -966,10 +992,12 @@ void ScaleRowDown34_1_Box_LSX(const uint8_t* src_ptr, : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9"); } +#endif // HAS_SCALEROWDOWN34_LSX #ifdef __cplusplus } // extern "C" } // namespace libyuv #endif -#endif // !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) +#endif // (!defined(LIBYUV_DISABLE_LSX) || !defined(LIBYUV_DISABLE_LASX)) && + // defined(__loongarch_lp64) diff --git a/unit_test/compare_test.cc b/unit_test/compare_test.cc index a3a402aa1..976a39a67 100644 --- a/unit_test/compare_test.cc +++ b/unit_test/compare_test.cc @@ -338,11 +338,19 @@ TEST_F(LibYUVCompareTest, BenchmarkHammingDistance_Opt) { #elif defined(HAS_HAMMINGDISTANCE_LASX) if (TestCpuFlag(kCpuHasLASX)) { h1 = HammingDistance_LASX(src_a, src_b, kMaxWidth); - } else { +#if defined(HAS_HAMMINGDISTANCE_LSX) + } else if (TestCpuFlag(kCpuHasLSX)) { h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth); +#endif + } else { + h1 = HammingDistance_C(src_a, src_b, kMaxWidth); } #elif defined(HAS_HAMMINGDISTANCE_LSX) - h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth); + if (TestCpuFlag(kCpuHasLSX)) { + h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth); + } else { + h1 = HammingDistance_C(src_a, src_b, kMaxWidth); + } #elif defined(HAS_HAMMINGDISTANCE_WASMSIMD) h1 = HammingDistance_WASMSIMD(src_a, src_b, kMaxWidth); #elif defined(HAS_HAMMINGDISTANCE_HVX) @@ -493,11 +501,19 @@ TEST_F(LibYUVCompareTest, TestHammingDistance_Opt) { #elif defined(HAS_HAMMINGDISTANCE_LASX) if (TestCpuFlag(kCpuHasLASX)) { h1 = HammingDistance_LASX(src_a, src_b, kMaxWidth); - } else { +#if defined(HAS_HAMMINGDISTANCE_LSX) + } else if (TestCpuFlag(kCpuHasLSX)) { h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth); +#endif + } else { + h1 = HammingDistance_C(src_a, src_b, kMaxWidth); } #elif defined(HAS_HAMMINGDISTANCE_LSX) - h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth); + if (TestCpuFlag(kCpuHasLSX)) { + h1 = HammingDistance_LSX(src_a, src_b, kMaxWidth); + } else { + h1 = HammingDistance_C(src_a, src_b, kMaxWidth); + } #elif defined(HAS_HAMMINGDISTANCE_WASMSIMD) h1 = HammingDistance_WASMSIMD(src_a, src_b, kMaxWidth); #elif defined(HAS_HAMMINGDISTANCE_HVX)