From d4074c4e187af2455253b164b368e2f28c2e398c Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Wed, 4 Nov 2026 03:09:42 -0800 Subject: [PATCH] [libyuv] Expand WebAssembly SIMD128 and Relaxed SIMD support Expand WebAssembly SIMD128 (_WASMSIMD) and WebAssembly Relaxed SIMD (_RELAXEDSIMD) optimizations across compare, rotate, convert, and scale row functions. Add a distinct kCpuHasRELAXEDSIMD (0x200) capability flag and runtime WasmCpuCaps() detection modeled after XNNPACK, probing whether wasm_i8x16_relaxed_swizzle or wasm_f32x4_relaxed_madd exhibit hardware-specific Relaxed SIMD behavior at runtime. Support disabling Relaxed SIMD alone (LIBYUV_DISABLE_RELAXEDSIMD), Wasm SIMD128 alone (LIBYUV_DISABLE_WASMSIMD), or all Wasm SIMD (LIBYUV_DISABLE_WASM) via compile-time macros or runtime environment variables. Compare (6 functions): C Function x86 Arm Wasm ----------------- ---------- ------------ ----------- HammingDistance_C AVX512VNNI NEON_DOTPROD RELAXEDSIMD SumSquareError_C AVX512VNNI NEON_DOTPROD WASMSIMD HashDjb2_C AVX512BW NEON WASMSIMD Ssim4x4Row_C AVX512BW SME WASMSIMD SsimWindowRow_C AVX512BW SME WASMSIMD ARGBDetectRow_C AVX2 NEON WASMSIMD ----------------- ---------- ------------ ----------- Count (6) 6/6 6/6 6/6 Percentage 100.0% 100.0% 100.0% Rotate (7 functions): C Function x86 Arm Wasm ----------------- -------- ------ -------- TransposeWx8_C AVX2 NEON WASMSIMD TransposeWx16_C AVX512BW NEON WASMSIMD TransposeUVWx8_C AVX2 NEON WASMSIMD TransposeUVWx16_C AVX512BW - WASMSIMD TransposeWx8_16_C AVX2 SVE2 WASMSIMD TransposeWxH_32_C AVX2 SME - TransposeWx4_32_C AVX2 SVE2 WASMSIMD ----------------- -------- ------ -------- Count (7) 7/7 6/7 6/7 Percentage 100.0% 85.7% 85.7% Convert (171 functions): C Function x86 Arm Wasm --------------------------- ---------- ------------ ----------- ARGBToYMatrixRow_C AVX512VNNI SME WASMSIMD ARGBToUVMatrixRow_C AVX512VNNI SME RELAXEDSIMD RGBToYMatrixRow_C AVX512BW NEON_DOTPROD RELAXEDSIMD RGBToUVMatrixRow_C AVX512BW NEON RELAXEDSIMD RGBToUV444MatrixRow_C AVX512BW NEON RELAXEDSIMD RGB565ToYMatrixRow_C AVX2 NEON WASMSIMD ARGB1555ToYMatrixRow_C AVX2 NEON WASMSIMD ARGB1555ToUVMatrixRow_C AVX2 NEON WASMSIMD ARGB4444ToYMatrixRow_C AVX2 NEON WASMSIMD ARGB4444ToUVMatrixRow_C AVX2 NEON WASMSIMD RGB565ToUVMatrixRow_C AVX2 NEON WASMSIMD ARGBToUV444MatrixRow_C AVX512VNNI NEON_I8MM RELAXEDSIMD RGB565ToYRow_C AVX2 NEON - ARGB1555ToYRow_C AVX2 NEON - ARGB4444ToYRow_C AVX2 NEON - RGB565ToUVRow_C AVX2 NEON - ARGB1555ToUVRow_C AVX2 NEON - ARGB4444ToUVRow_C AVX2 NEON - MirrorRow_C AVX512BW NEON RELAXEDSIMD MirrorUVRow_C AVX512BW NEON RELAXEDSIMD MirrorSplitUVRow_C AVX512BW NEON RELAXEDSIMD MirrorRow_16_C AVX512BW NEON - ARGBMirrorRow_C AVX512BW NEON WASMSIMD RGB24MirrorRow_C AVX512VBMI NEON RELAXEDSIMD SplitUVRow_C AVX512BW SME WASMSIMD DetileRow_C AVX2 NEON WASMSIMD DetileRow_16_C AVX2 NEON WASMSIMD DetileSplitUVRow_C AVX2 NEON WASMSIMD DetileToYUY2_C AVX2 NEON WASMSIMD UnpackMT2T_C AVX2 NEON - MergeUVRow_C AVX512BW SME WASMSIMD HalfMergeUVRow_C AVX512BW SME WASMSIMD SplitRGBRow_C AVX2 NEON RELAXEDSIMD MergeRGBRow_C AVX512VBMI NEON RELAXEDSIMD MergeARGBRow_C AVX2 NEON WASMSIMD SplitARGBRow_C AVX2 NEON WASMSIMD MergeXRGBRow_C AVX2 NEON WASMSIMD SplitXRGBRow_C AVX2 NEON WASMSIMD MergeXR30Row_C AVX2 NEON - MergeAR64Row_C AVX2 NEON - MergeARGB16To8Row_C AVX2 NEON - MergeXR64Row_C AVX2 NEON - MergeXRGB16To8Row_C AVX2 NEON - MergeUVRow_16_C AVX512BW SME WASMSIMD SplitUVRow_16_C AVX2 NEON WASMSIMD MultiplyRow_16_C AVX512BW SME WASMSIMD DivideRow_16_C AVX512BW SME WASMSIMD Convert8To16Row_C AVX512BW SME WASMSIMD Convert16To8Row_C AVX512BW SME WASMSIMD HalfRow_C AVX2 SME WASMSIMD HalfRow_16_C AVX2 SME WASMSIMD HalfRow_16To8_C AVX512BW SME WASMSIMD HalfWidthRow_16To8_C AVX512BW SME WASMSIMD HalfWidthRow_16To8_Odd_C AVX2 NEON - Convert8To8Row_C AVX512BW SME WASMSIMD CopyRow_C AVX512BW SME WASMSIMD CopyRow_16_C AVX2 NEON WASMSIMD ARGBCopyAlphaRow_C AVX2 NEON WASMSIMD ARGBExtractAlphaRow_C AVX512BW NEON WASMSIMD ARGBCopyYToAlphaRow_C AVX512BW NEON RELAXEDSIMD SetRow_C AVX2 NEON WASMSIMD ARGBSetRow_C AVX2 NEON WASMSIMD ARGBShuffleRow_C AVX512BW NEON RELAXEDSIMD RGB24ToARGBRow_C AVX512BW SME RELAXEDSIMD RAWToARGBRow_C AVX512BW SME RELAXEDSIMD RAWToRGBARow_C AVX512VBMI SME RELAXEDSIMD RAWToRGB24Row_C AVX512VBMI SME RELAXEDSIMD RGB565ToARGBRow_C AVX2 NEON WASMSIMD ARGB1555ToARGBRow_C AVX512BW SME WASMSIMD ARGB4444ToARGBRow_C AVX2 NEON WASMSIMD AR30ToARGBRow_C AVX512BW NEON WASMSIMD AR30ToABGRRow_C AVX512BW NEON WASMSIMD ARGBToAR30Row_C AVX512BW NEON WASMSIMD AR30ToAB30Row_C AVX512BW NEON WASMSIMD ARGBToRGB565DitherRow_C AVX512BW SME WASMSIMD ARGBToABGRRow_C - - - ARGBToBGRARow_C - - - ARGBToRGBARow_C - - - ARGBToRGB24Row_C AVX512BW SME RELAXEDSIMD ARGBToRAWRow_C AVX512BW SME RELAXEDSIMD ARGBToRGB565Row_C AVX512BW SME WASMSIMD ARGBToARGB1555Row_C AVX2 NEON WASMSIMD ARGBToARGB4444Row_C AVX2 NEON WASMSIMD ABGRToAR30Row_C AVX512BW NEON WASMSIMD ARGBToAR64Row_C AVX2 NEON WASMSIMD ARGBToAB64Row_C AVX2 NEON WASMSIMD AR64ToARGBRow_C AVX2 NEON WASMSIMD AB64ToARGBRow_C AVX2 NEON WASMSIMD AR64ToAB64Row_C - - - RGBAToARGBRow_C - - - AR64ShuffleRow_C - - - J400ToARGBRow_C AVX512BW NEON RELAXEDSIMD I444ToARGBRow_C AVX512BW SME WASMSIMD I444ToRGB24Row_C AVX512VBMI SME RELAXEDSIMD I422ToARGBRow_C AVX512BW SME WASMSIMD I422ToAR30Row_C AVX512BW SME WASMSIMD I210ToAR30Row_C AVX512BW SME WASMSIMD I210ToARGBRow_C AVX512BW SME WASMSIMD I212ToAR30Row_C AVX512BW SME - I212ToARGBRow_C AVX512BW SME - I410ToAR30Row_C AVX512BW SME WASMSIMD I410ToARGBRow_C AVX512BW SME WASMSIMD I210AlphaToARGBRow_C AVX512BW SME WASMSIMD I410AlphaToARGBRow_C AVX512BW SME WASMSIMD I444AlphaToARGBRow_C AVX512BW SME WASMSIMD I422AlphaToARGBRow_C AVX512BW SME WASMSIMD NV12ToARGBRow_C AVX512BW SME WASMSIMD NV12ToRGB565Row_C AVX2 NEON WASMSIMD NV21ToARGBRow_C AVX512BW SME WASMSIMD NV12ToRGB24Row_C AVX512VBMI SME RELAXEDSIMD NV21ToRGB24Row_C AVX512VBMI SME RELAXEDSIMD NV21ToYUV24Row_C AVX512VBMI NEON - YUY2ToARGBRow_C AVX512BW SME WASMSIMD UYVYToARGBRow_C AVX512BW SME WASMSIMD P210ToARGBRow_C AVX512BW SME - P410ToARGBRow_C AVX512BW SME - P210ToAR30Row_C AVX512BW SME - P410ToAR30Row_C AVX512BW SME - I422ToRGBARow_C AVX512BW SME WASMSIMD I422ToRGB24Row_C AVX512VBMI SME RELAXEDSIMD I422ToARGB4444Row_C AVX512BW SME WASMSIMD I422ToARGB1555Row_C AVX512BW SME WASMSIMD I422ToRGB565Row_C AVX512BW SME WASMSIMD I400ToARGBRow_C AVX512BW SME RELAXEDSIMD ARGBBlendRow_C AVX512BW NEON RELAXEDSIMD BlendPlaneRow_C AVX512BW NEON WASMSIMD ARGBMultiplyRow_C AVX512BW SME WASMSIMD ARGBAddRow_C AVX2 NEON WASMSIMD ARGBSubtractRow_C AVX2 NEON WASMSIMD ARGBOrRow_C AVX512BW SME WASMSIMD ARGBAlphaBlendRow_C AVX512BW SME WASMSIMD YUY2ToYRow_C AVX512BW SME WASMSIMD YUY2ToUVRow_C AVX512BW SME WASMSIMD YUY2ToNVUVRow_C AVX2 NEON WASMSIMD YUY2ToUV422Row_C AVX512BW NEON WASMSIMD UYVYToYRow_C AVX512BW SME WASMSIMD UYVYToUVRow_C AVX512BW SME WASMSIMD UYVYToUV422Row_C AVX512BW NEON WASMSIMD SwapUVRow_C AVX512BW NEON WASMSIMD AYUVToYRow_C AVX512VBMI NEON WASMSIMD AYUVToUVRow_C AVX512VBMI SME RELAXEDSIMD AYUVToVURow_C AVX512VBMI SME RELAXEDSIMD I422ToYUY2Row_C AVX2 NEON WASMSIMD I422ToUYVYRow_C AVX2 NEON WASMSIMD ARGBAttenuateRow_C AVX512BW NEON RELAXEDSIMD ARGBUnattenuateRow_C AVX2 NEON - ARGBGrayRow_C AVX512VNNI NEON_DOTPROD RELAXEDSIMD ARGBSepiaRow_C AVX512VNNI NEON_DOTPROD RELAXEDSIMD ARGBColorMatrixRow_C AVX512VNNI NEON_I8MM WASMSIMD ARGBColorTableRow_C AVX2 NEON - RGBColorTableRow_C AVX2 NEON - ARGBQuantizeRow_C AVX2 NEON WASMSIMD ARGBShadeRow_C AVX2 NEON WASMSIMD CumulativeSumToAverageRow_C AVX2 NEON - ComputeCumulativeSumRow_C AVX2 NEON - ARGBAffineRow_C AVX2 NEON - InterpolateRow_C AVX512BW SME WASMSIMD InterpolateRow_16_C AVX512BW SME WASMSIMD SobelXRow_C AVX2 NEON WASMSIMD SobelYRow_C AVX2 NEON WASMSIMD SobelRow_C AVX2 NEON RELAXEDSIMD SobelToPlaneRow_C AVX2 NEON WASMSIMD SobelXYRow_C AVX2 NEON WASMSIMD ARGBPolynomialRow_C AVX2 NEON WASMSIMD HalfFloatRow_C AVX512BW SME WASMSIMD ByteToFloatRow_C AVX2 NEON WASMSIMD ARGBLumaColorTableRow_C AVX2 NEON WASMSIMD GaussRow_F32_C AVX2 NEON - GaussCol_F32_C AVX2 NEON - GaussRow_C AVX2 NEON - GaussCol_C AVX2 NEON - --------------------------- ---------- ------------ ----------- Count (171) 165/171 165/171 134/171 Percentage 96.5% 96.5% 78.4% Scale (86 functions): C Function x86 Arm Wasm ------------------------------- ---------- ------ ----------- FixedDiv_C - - - FixedDiv1_C - - - ScaleRowDown2_C AVX512BW SME WASMSIMD ScaleRowDown2_16_C AVX2 SME - ScaleRowDown2Linear_C AVX512BW SME WASMSIMD ScaleRowDown2Linear_16_C AVX2 SME - ScaleRowDown2Box_C AVX512BW SME WASMSIMD ScaleRowDown2Box_Odd_C AVX512BW NEON WASMSIMD ScaleRowDown2Box_16_C AVX2 SME - ScaleRowDown4_C AVX2 NEON WASMSIMD ScaleRowDown4_16_C - - - ScaleRowDown4Box_C AVX2 NEON WASMSIMD ScaleRowDown4Box_16_C - - - ScaleRowDown34_C AVX2 NEON RELAXEDSIMD ScaleRowDown34_16_C - - - ScaleRowDown34_0_Box_C AVX2 NEON RELAXEDSIMD ScaleRowDown34_0_Box_16_C - - - ScaleRowDown34_1_Box_C AVX2 NEON RELAXEDSIMD ScaleRowDown34_1_Box_16_C - - - ScaleRowUp2_Linear_C AVX2 NEON RELAXEDSIMD ScaleRowUp2_Bilinear_C AVX2 NEON RELAXEDSIMD ScaleRowUp2_Linear_16_C AVX2 NEON - ScaleRowUp2_Bilinear_16_C AVX2 NEON - ScaleRowUp2_Linear_Any_C AVX2 NEON RELAXEDSIMD ScaleRowUp2_Bilinear_Any_C AVX2 NEON RELAXEDSIMD ScaleRowUp2_Linear_16_Any_C AVX2 NEON - ScaleRowUp2_Bilinear_16_Any_C AVX2 NEON - ScaleCols_C - - - ScaleCols_16_C - - - ScaleColsUp2_C AVX2 - - ScaleColsUp2_16_C - - - ScaleFilterCols_C AVX2 NEON WASMSIMD ScaleFilterCols_16_C - - - ScaleFilterCols64_C - - - ScaleFilterCols64_16_C - - - ScaleRowDown38_C AVX2 NEON RELAXEDSIMD ScaleRowDown38_16_C - - - ScaleRowDown38_3_Box_C AVX2 NEON RELAXEDSIMD ScaleRowDown38_3_Box_16_C - - - ScaleRowDown38_2_Box_C AVX2 NEON RELAXEDSIMD ScaleRowDown38_2_Box_16_C - - - ScaleAddRow_C AVX512BW NEON WASMSIMD ScaleAddRow_16_C - - - ScaleARGBRowDown2_C AVX2 SME WASMSIMD ScaleARGBRowDown2Linear_C AVX2 SME WASMSIMD ScaleARGBRowDown2Box_C AVX2 SME WASMSIMD ScaleARGBRowDownEven_C AVX2 NEON WASMSIMD ScaleARGBRowDownEvenBox_C AVX2 NEON WASMSIMD ScaleARGBCols_C AVX2 NEON WASMSIMD ScaleARGBCols64_C - - - ScaleARGBColsUp2_C AVX2 - WASMSIMD ScaleARGBFilterCols_C AVX2 NEON WASMSIMD ScaleARGBFilterCols64_C - - - ScaleUVRowDown2_C AVX2 SME WASMSIMD ScaleUVRowDown2_16_C - - - ScaleUVRowDown2Linear_C AVX2 SME WASMSIMD ScaleUVRowDown2Linear_16_C - - - ScaleUVRowDown2Box_C AVX2 SME WASMSIMD ScaleUVRowDown2Box_16_C - - - ScaleUVRowDownEven_C AVX2 NEON WASMSIMD ScaleUVRowDownEven_16_C - - - ScaleUVRowDownEvenBox_C - - - ScaleUVRowDownEvenBox_16_C - - - ScaleUVRowUp2_Linear_C AVX2 NEON RELAXEDSIMD ScaleUVRowUp2_Bilinear_C AVX2 NEON RELAXEDSIMD ScaleUVRowUp2_Linear_Any_C AVX2 NEON RELAXEDSIMD ScaleUVRowUp2_Bilinear_Any_C AVX2 NEON RELAXEDSIMD ScaleUVRowUp2_Linear_16_C AVX2 NEON - ScaleUVRowUp2_Bilinear_16_C AVX2 NEON - ScaleUVRowUp2_Linear_16_Any_C AVX2 NEON - ScaleUVRowUp2_Bilinear_16_Any_C AVX2 NEON - ScaleUVCols_C - - - ScaleUVCols_16_C - - - ScaleUVCols64_C - - - ScaleUVCols64_16_C - - - ScaleUVColsUp2_C - - - ScaleUVColsUp2_16_C - - - ScaleUVFilterCols_C - - - ScaleUVFilterCols_16_C - - - ScaleUVFilterCols64_C - - - ScaleUVFilterCols64_16_C - - - ScaleUVAddCols3_C AVX512VBMI SME WASMSIMD ScaleUVAddCols3_16_C - - - ScaleMaxSamples_C AVX2 NEON WASMSIMD ScaleSumSamples_C AVX2 NEON WASMSIMD ScaleSamples_C AVX2 NEON WASMSIMD ------------------------------- ---------- ------ ----------- Count (86) 50/86 48/86 38/86 Percentage 58.1% 55.8% 44.2% Bug: 42280942 Change-Id: I4420a570327a95cc998c6aa023286a0b3552f742 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8724066 Reviewed-by: richard winterton Commit-Queue: Frank Barchard --- BUILD.gn | 5 +- CMakeLists.txt | 12 +- README.chromium | 2 +- include/libyuv/compare_row.h | 10 +- include/libyuv/cpu_id.h | 3 + include/libyuv/cpu_support.h | 9 + include/libyuv/rotate_row.h | 2 +- include/libyuv/row.h | 1068 +++++++++- include/libyuv/scale_row.h | 181 +- include/libyuv/version.h | 2 +- source/compare.cc | 5 + source/compare_wasm.cc | 40 +- source/convert.cc | 264 +++ source/convert_argb.cc | 654 +++++++ source/convert_from.cc | 40 + source/convert_from_argb.cc | 184 ++ source/convert_to_argb.cc | 16 + source/convert_to_i420.cc | 24 + source/cpu_id.cc | 46 +- source/planar_functions.cc | 504 +++++ source/rotate.cc | 21 + source/rotate_wasm.cc | 2 +- source/row_any.cc | 534 +++++ source/row_wasm.cc | 3568 ++++++++++++++++++++++++++++++++-- source/scale.cc | 61 + source/scale_any.cc | 135 ++ source/scale_uv.cc | 56 + source/scale_wasm.cc | 703 ++++++- unit_test/cpu_test.cc | 10 + unit_test/planar_test.cc | 35 + unit_test/unit_test.cc | 6 + util/cpuid.c | 2 + 32 files changed, 7910 insertions(+), 294 deletions(-) diff --git a/BUILD.gn b/BUILD.gn index 3b905a032..c39a3b6bc 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -50,7 +50,10 @@ config("libyuv_config") { defines += [ "LIBYUV_DISABLE_LASX" ] } if (current_cpu == "wasm") { - cflags_cc = [ "-msimd128" ] + cflags_cc = [ + "-msimd128", + "-mrelaxed-simd", + ] } } diff --git a/CMakeLists.txt b/CMakeLists.txt index d608e9e44..6bb297a22 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,8 @@ option( LIBYUV_DISABLE_JPEG "Disable JPEG support" OFF ) option( LIBYUV_DISABLE_ASM "Disable inline assembly" OFF ) option( LIBYUV_DISABLE_LSX "Disable LoongArch LSX" OFF ) option( LIBYUV_DISABLE_LASX "Disable LoongArch LASX" OFF ) +option( LIBYUV_DISABLE_WASMSIMD "Disable WebAssembly SIMD128" OFF ) +option( LIBYUV_DISABLE_RELAXEDSIMD "Disable WebAssembly Relaxed SIMD" OFF ) if(LIBYUV_DISABLE_ASM) add_definitions(-DLIBYUV_DISABLE_ASM) @@ -19,6 +21,12 @@ endif() if(LIBYUV_DISABLE_LASX) add_definitions(-DLIBYUV_DISABLE_LASX) endif() +if(LIBYUV_DISABLE_WASMSIMD) + add_definitions(-DLIBYUV_DISABLE_WASMSIMD) +endif() +if(LIBYUV_DISABLE_RELAXEDSIMD) + add_definitions(-DLIBYUV_DISABLE_RELAXEDSIMD) +endif() include(CheckCSourceCompiles) @@ -208,8 +216,8 @@ int main(void) { return 0; } endif() if(arch_lowercase MATCHES "^wasm" OR EMSCRIPTEN) - set(CMAKE_C_FLAGS "-msimd128 ${CMAKE_C_FLAGS}") - set(CMAKE_CXX_FLAGS "-msimd128 ${CMAKE_CXX_FLAGS}") + set(CMAKE_C_FLAGS "-msimd128 -mrelaxed-simd ${CMAKE_C_FLAGS}") + set(CMAKE_CXX_FLAGS "-msimd128 -mrelaxed-simd ${CMAKE_CXX_FLAGS}") endif() endif() diff --git a/README.chromium b/README.chromium index 019c74818..6192e3ffa 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1978 +Version: 1979 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index 540d388cc..8abcc7a09 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -152,7 +152,7 @@ extern "C" { #endif // The following are available for WebAssembly SIMD128: -#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#if !defined(LIBYUV_DISABLE_WASMSIMD) && defined(__wasm_simd128__) #define HAS_HAMMINGDISTANCE_WASMSIMD #define HAS_SUMSQUAREERROR_WASMSIMD #define HAS_HASHDJB2_WASMSIMD @@ -161,6 +161,11 @@ extern "C" { #define HAS_ARGBDETECTROW_WASMSIMD #endif +// The following are available for WebAssembly Relaxed SIMD: +#if !defined(LIBYUV_DISABLE_RELAXEDSIMD) && defined(__wasm_relaxed_simd__) +#define HAS_HAMMINGDISTANCE_RELAXEDSIMD +#endif + // The following are available for Hexagon HVX: #if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__) #define HAS_HAMMINGDISTANCE_HVX @@ -207,6 +212,9 @@ uint32_t HammingDistance_LASX(const uint8_t* src_a, uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a, const uint8_t* src_b, int count); +uint32_t HammingDistance_RELAXEDSIMD(const uint8_t* src_a, + const uint8_t* src_b, + int count); uint32_t HammingDistance_HVX(const uint8_t* src_a, const uint8_t* src_b, int count); diff --git a/include/libyuv/cpu_id.h b/include/libyuv/cpu_id.h index 4ff61c87d..ed814fc7b 100644 --- a/include/libyuv/cpu_id.h +++ b/include/libyuv/cpu_id.h @@ -74,6 +74,7 @@ static const int kCpuHasLASX = 0x200; // These flags are only valid on WASM processors. static const int kCpuHasWASM = 0x40; static const int kCpuHasWASMSIMD = 0x100; +static const int kCpuHasRELAXEDSIMD = 0x200; // These flags are only valid on Hexagon processors. static const int kCpuHasHexagon = 0x80; @@ -102,6 +103,8 @@ LIBYUV_API int ArmCpuCaps(const char* cpuinfo_name); LIBYUV_API int RiscvCpuCaps(const char* cpuinfo_name); +LIBYUV_API +int WasmCpuCaps(void); #ifdef __linux__ // On Linux, parse AArch64 features from getauxval(AT_HWCAP{,2}). diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index c0e345f3e..63c6d665f 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -26,6 +26,15 @@ extern "C" { #define LIBYUV_DISABLE_NEON #endif +#if defined(LIBYUV_DISABLE_WASM) +#if !defined(LIBYUV_DISABLE_WASMSIMD) +#define LIBYUV_DISABLE_WASMSIMD +#endif +#if !defined(LIBYUV_DISABLE_RELAXEDSIMD) +#define LIBYUV_DISABLE_RELAXEDSIMD +#endif +#endif + // MemorySanitizer does not support assembly code yet. http://crbug.com/344505 #if defined(__has_feature) #if __has_feature(memory_sanitizer) diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index 06544a5a1..e4702f221 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -136,7 +136,7 @@ extern "C" { #define HAS_TRANSPOSEWXH_32_RVV #endif -#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#if !defined(LIBYUV_DISABLE_WASMSIMD) && defined(__wasm_simd128__) #define HAS_TRANSPOSEWX8_WASMSIMD #define HAS_TRANSPOSEUVWX8_WASMSIMD #define HAS_TRANSPOSEWX16_WASMSIMD diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 6c652fbb6..3302f7b6c 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -1309,48 +1309,112 @@ extern "C" { #define HAS_RGBATOARGBROW_RVV #endif -#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#if !defined(LIBYUV_DISABLE_WASMSIMD) && defined(__wasm_simd128__) +#define HAS_AB64TOARGBROW_WASMSIMD +#define HAS_ABGRTOAR30ROW_WASMSIMD +#define HAS_AR30TOAB30ROW_WASMSIMD +#define HAS_AR30TOABGRROW_WASMSIMD +#define HAS_AR30TOARGBROW_WASMSIMD +#define HAS_AR64TOARGBROW_WASMSIMD #define HAS_ARGB1555TOARGBROW_WASMSIMD #define HAS_ARGB1555TOUVMATRIXROW_WASMSIMD #define HAS_ARGB1555TOYMATRIXROW_WASMSIMD #define HAS_ARGB4444TOARGBROW_WASMSIMD #define HAS_ARGB4444TOUVMATRIXROW_WASMSIMD #define HAS_ARGB4444TOYMATRIXROW_WASMSIMD +#define HAS_ARGBADDROW_WASMSIMD #define HAS_ARGBALPHABLENDROW_WASMSIMD +#define HAS_ARGBATTENUATEROW_WASMSIMD #define HAS_ARGBBLENDROW_WASMSIMD +#define HAS_ARGBCOLORMATRIXROW_WASMSIMD +#define HAS_ARGBCOPYALPHAROW_WASMSIMD #define HAS_ARGBCOPYYTOALPHAROW_WASMSIMD #define HAS_ARGBEXTRACTALPHAROW_WASMSIMD +#define HAS_ARGBGRAYROW_WASMSIMD +#define HAS_ARGBLUMACOLORTABLEROW_WASMSIMD #define HAS_ARGBMIRRORROW_WASMSIMD +#define HAS_ARGBMULTIPLYROW_WASMSIMD #define HAS_ARGBORROW_WASMSIMD +#define HAS_ARGBPOLYNOMIALROW_WASMSIMD +#define HAS_ARGBQUANTIZEROW_WASMSIMD +#define HAS_ARGBSEPIAROW_WASMSIMD +#define HAS_ARGBSETROW_WASMSIMD +#define HAS_ARGBSHADEROW_WASMSIMD #define HAS_ARGBSHUFFLEROW_WASMSIMD +#define HAS_ARGBSUBTRACTROW_WASMSIMD +#define HAS_ARGBTOAB64ROW_WASMSIMD +#define HAS_ARGBTOAR30ROW_WASMSIMD +#define HAS_ARGBTOAR64ROW_WASMSIMD #define HAS_ARGBTOARGB1555ROW_WASMSIMD #define HAS_ARGBTOARGB4444ROW_WASMSIMD +#define HAS_ARGBTORAWROW_WASMSIMD +#define HAS_ARGBTORGB24ROW_WASMSIMD #define HAS_ARGBTORGB565DITHERROW_WASMSIMD #define HAS_ARGBTORGB565ROW_WASMSIMD #define HAS_ARGBTOUV444MATRIXROW_WASMSIMD #define HAS_ARGBTOUVMATRIXROW_WASMSIMD #define HAS_ARGBTOYMATRIXROW_WASMSIMD +#define HAS_AYUVTOUVROW_WASMSIMD +#define HAS_AYUVTOVUROW_WASMSIMD +#define HAS_AYUVTOYROW_WASMSIMD #define HAS_BLENDPLANEROW_WASMSIMD +#define HAS_BYTETOFLOATROW_WASMSIMD +#define HAS_CONVERT16TO8ROW_WASMSIMD #define HAS_CONVERT8TO16ROW_WASMSIMD +#define HAS_CONVERT8TO8ROW_WASMSIMD +#define HAS_COPYROW_16_WASMSIMD +#define HAS_COPYROW_WASMSIMD +#define HAS_DETILEROW_16_WASMSIMD +#define HAS_DETILEROW_WASMSIMD +#define HAS_DETILESPLITUVROW_WASMSIMD +#define HAS_DETILETOYUY2_WASMSIMD +#define HAS_DIVIDEROW_16_WASMSIMD +#define HAS_HALFFLOATROW_WASMSIMD +#define HAS_HALFMERGEUVROW_WASMSIMD +#define HAS_HALFROW_16_WASMSIMD +#define HAS_HALFROW_16TO8_WASMSIMD +#define HAS_HALFROW_WASMSIMD +#define HAS_HALFWIDTHROW_16TO8_WASMSIMD +#define HAS_I210ALPHATOARGBROW_WASMSIMD +#define HAS_I210TOAR30ROW_WASMSIMD +#define HAS_I210TOARGBROW_WASMSIMD +#define HAS_I400TOARGBROW_WASMSIMD +#define HAS_I410ALPHATOARGBROW_WASMSIMD +#define HAS_I410TOAR30ROW_WASMSIMD +#define HAS_I410TOARGBROW_WASMSIMD +#define HAS_I422ALPHATOARGBROW_WASMSIMD #define HAS_I422TOAR30ROW_WASMSIMD #define HAS_I422TOARGB1555ROW_WASMSIMD #define HAS_I422TOARGB4444ROW_WASMSIMD #define HAS_I422TOARGBROW_WASMSIMD #define HAS_I422TORGB24ROW_WASMSIMD #define HAS_I422TORGB565ROW_WASMSIMD +#define HAS_I422TORGBAROW_WASMSIMD +#define HAS_I422TOUYVYROW_WASMSIMD +#define HAS_I422TOYUY2ROW_WASMSIMD +#define HAS_I444ALPHATOARGBROW_WASMSIMD +#define HAS_I444TOARGBROW_WASMSIMD +#define HAS_I444TORGB24ROW_WASMSIMD #define HAS_INTERPOLATEROW_16_WASMSIMD #define HAS_INTERPOLATEROW_WASMSIMD #define HAS_J400TOARGBROW_WASMSIMD +#define HAS_MERGEARGBROW_WASMSIMD +#define HAS_MERGERGBROW_WASMSIMD +#define HAS_MERGEUVROW_16_WASMSIMD #define HAS_MERGEUVROW_WASMSIMD +#define HAS_MERGEXRGBROW_WASMSIMD #define HAS_MIRRORROW_WASMSIMD #define HAS_MIRRORSPLITUVROW_WASMSIMD #define HAS_MIRRORUVROW_WASMSIMD #define HAS_MULTIPLYROW_16_WASMSIMD #define HAS_NV12TOARGBROW_WASMSIMD +#define HAS_NV12TORGB24ROW_WASMSIMD #define HAS_NV12TORGB565ROW_WASMSIMD #define HAS_NV21TOARGBROW_WASMSIMD +#define HAS_NV21TORGB24ROW_WASMSIMD #define HAS_RAWTOARGBROW_WASMSIMD #define HAS_RAWTORGB24ROW_WASMSIMD +#define HAS_RAWTORGBAROW_WASMSIMD #define HAS_RGB24MIRRORROW_WASMSIMD #define HAS_RGB24TOARGBROW_WASMSIMD #define HAS_RGB565TOARGBROW_WASMSIMD @@ -1359,7 +1423,62 @@ extern "C" { #define HAS_RGBTOUV444MATRIXROW_WASMSIMD #define HAS_RGBTOUVMATRIXROW_WASMSIMD #define HAS_RGBTOYMATRIXROW_WASMSIMD +#define HAS_SETROW_WASMSIMD +#define HAS_SOBELROW_WASMSIMD +#define HAS_SOBELTOPLANEROW_WASMSIMD +#define HAS_SOBELXROW_WASMSIMD +#define HAS_SOBELXYROW_WASMSIMD +#define HAS_SOBELYROW_WASMSIMD +#define HAS_SPLITARGBROW_WASMSIMD +#define HAS_SPLITRGBROW_WASMSIMD +#define HAS_SPLITUVROW_16_WASMSIMD +#define HAS_SPLITUVROW_WASMSIMD +#define HAS_SPLITXRGBROW_WASMSIMD #define HAS_SWAPUVROW_WASMSIMD +#define HAS_UYVYTOARGBROW_WASMSIMD +#define HAS_UYVYTOUV422ROW_WASMSIMD +#define HAS_UYVYTOUVROW_WASMSIMD +#define HAS_UYVYTOYROW_WASMSIMD +#define HAS_YUY2TOARGBROW_WASMSIMD +#define HAS_YUY2TONVUVROW_WASMSIMD +#define HAS_YUY2TOUV422ROW_WASMSIMD +#define HAS_YUY2TOUVROW_WASMSIMD +#define HAS_YUY2TOYROW_WASMSIMD +#endif + +#if !defined(LIBYUV_DISABLE_RELAXEDSIMD) && defined(__wasm_relaxed_simd__) +#define HAS_ARGBATTENUATEROW_RELAXEDSIMD +#define HAS_ARGBBLENDROW_RELAXEDSIMD +#define HAS_ARGBCOPYYTOALPHAROW_RELAXEDSIMD +#define HAS_ARGBGRAYROW_RELAXEDSIMD +#define HAS_ARGBSEPIAROW_RELAXEDSIMD +#define HAS_ARGBSHUFFLEROW_RELAXEDSIMD +#define HAS_ARGBTORAWROW_RELAXEDSIMD +#define HAS_ARGBTORGB24ROW_RELAXEDSIMD +#define HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD +#define HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD +#define HAS_AYUVTOUVROW_RELAXEDSIMD +#define HAS_AYUVTOVUROW_RELAXEDSIMD +#define HAS_I400TOARGBROW_RELAXEDSIMD +#define HAS_I422TORGB24ROW_RELAXEDSIMD +#define HAS_I444TORGB24ROW_RELAXEDSIMD +#define HAS_J400TOARGBROW_RELAXEDSIMD +#define HAS_MERGERGBROW_RELAXEDSIMD +#define HAS_MIRRORROW_RELAXEDSIMD +#define HAS_MIRRORSPLITUVROW_RELAXEDSIMD +#define HAS_MIRRORUVROW_RELAXEDSIMD +#define HAS_NV12TORGB24ROW_RELAXEDSIMD +#define HAS_NV21TORGB24ROW_RELAXEDSIMD +#define HAS_RAWTOARGBROW_RELAXEDSIMD +#define HAS_RAWTORGB24ROW_RELAXEDSIMD +#define HAS_RAWTORGBAROW_RELAXEDSIMD +#define HAS_RGB24MIRRORROW_RELAXEDSIMD +#define HAS_RGB24TOARGBROW_RELAXEDSIMD +#define HAS_RGBTOUV444MATRIXROW_RELAXEDSIMD +#define HAS_RGBTOUVMATRIXROW_RELAXEDSIMD +#define HAS_RGBTOYMATRIXROW_RELAXEDSIMD +#define HAS_SOBELROW_RELAXEDSIMD +#define HAS_SPLITRGBROW_RELAXEDSIMD #endif #if !defined(LIBYUV_DISABLE_HVX) && defined(__HVX__) @@ -9102,6 +9221,953 @@ void BlendPlaneRow_Any_WASMSIMD(const uint8_t* y_buf, const uint8_t* v_buf, uint8_t* dst_ptr, int width); +void CopyRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int count); +void CopyRow_16_WASMSIMD(const uint16_t* src, uint16_t* dst, int count); +void CopyRow_Any_WASMSIMD(const uint8_t* src, uint8_t* dst, int count); +void CopyRow_16_Any_WASMSIMD(const uint16_t* src, uint16_t* dst, int count); +void SetRow_WASMSIMD(uint8_t* dst, uint8_t v8, int width); +void SetRow_Any_WASMSIMD(uint8_t* dst, uint8_t v8, int width); +void ARGBSetRow_WASMSIMD(uint8_t* dst_argb, uint32_t v32, int width); +void ARGBSetRow_Any_WASMSIMD(uint8_t* dst_argb, uint32_t v32, int width); +void SplitUVRow_WASMSIMD(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void SplitUVRow_Any_WASMSIMD(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void SplitUVRow_16_WASMSIMD(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width); +void SplitUVRow_16_Any_WASMSIMD(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width); +void MergeUVRow_16_WASMSIMD(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width); +void MergeUVRow_16_Any_WASMSIMD(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width); +void HalfMergeUVRow_WASMSIMD(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); +void HalfMergeUVRow_Any_WASMSIMD(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width); +void SplitRGBRow_WASMSIMD(const uint8_t* src_rgb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); +void SplitRGBRow_Any_WASMSIMD(const uint8_t* src_rgb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); +void MergeRGBRow_WASMSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width); +void MergeRGBRow_Any_WASMSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width); +void SplitARGBRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + uint8_t* dst_a, + int width); +void SplitARGBRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + uint8_t* dst_a, + int width); +void MergeARGBRow_WASMSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + const uint8_t* src_a, + uint8_t* dst_argb, + int width); +void MergeARGBRow_Any_WASMSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + const uint8_t* src_a, + uint8_t* dst_argb, + int width); +void SplitXRGBRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); +void SplitXRGBRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); +void MergeXRGBRow_WASMSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_argb, + int width); +void MergeXRGBRow_Any_WASMSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_argb, + int width); +void HalfRow_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); +void HalfRow_Any_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); +void HalfRow_16_WASMSIMD(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); +void HalfRow_16_Any_WASMSIMD(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); +void HalfRow_16To8_WASMSIMD(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfRow_16To8_Any_WASMSIMD(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_WASMSIMD(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void HalfWidthRow_16To8_Any_WASMSIMD(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); +void DivideRow_16_WASMSIMD(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); +void DivideRow_16_Any_WASMSIMD(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); +void Convert16To8Row_WASMSIMD(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width); +void Convert16To8Row_Any_WASMSIMD(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width); +void Convert8To8Row_WASMSIMD(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width); +void Convert8To8Row_Any_WASMSIMD(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width); +void ARGBToRGB24Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb24, + int width); +void ARGBToRGB24Row_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb24, + int width); +void ARGBToRAWRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_raw, + int width); +void ARGBToRAWRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_raw, + int width); +void RAWToRGBARow_WASMSIMD(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width); +void RAWToRGBARow_Any_WASMSIMD(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width); +void ARGBToAR64Row_WASMSIMD(const uint8_t* src_argb, + uint16_t* dst_ar64, + int width); +void ARGBToAR64Row_Any_WASMSIMD(const uint8_t* src_argb, + uint16_t* dst_ar64, + int width); +void ARGBToAB64Row_WASMSIMD(const uint8_t* src_argb, + uint16_t* dst_ab64, + int width); +void ARGBToAB64Row_Any_WASMSIMD(const uint8_t* src_argb, + uint16_t* dst_ab64, + int width); +void AR64ToARGBRow_WASMSIMD(const uint16_t* src_ar64, + uint8_t* dst_argb, + int width); +void AR64ToARGBRow_Any_WASMSIMD(const uint16_t* src_ar64, + uint8_t* dst_argb, + int width); +void AB64ToARGBRow_WASMSIMD(const uint16_t* src_ab64, + uint8_t* dst_argb, + int width); +void AB64ToARGBRow_Any_WASMSIMD(const uint16_t* src_ab64, + uint8_t* dst_argb, + int width); +void AR30ToARGBRow_WASMSIMD(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width); +void AR30ToARGBRow_Any_WASMSIMD(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width); +void AR30ToABGRRow_WASMSIMD(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width); +void AR30ToABGRRow_Any_WASMSIMD(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width); +void AR30ToAB30Row_WASMSIMD(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width); +void AR30ToAB30Row_Any_WASMSIMD(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width); +void ARGBToAR30Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_ar30, + int width); +void ARGBToAR30Row_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_ar30, + int width); +void ABGRToAR30Row_WASMSIMD(const uint8_t* src_abgr, + uint8_t* dst_ar30, + int width); +void ABGRToAR30Row_Any_WASMSIMD(const uint8_t* src_abgr, + uint8_t* dst_ar30, + int width); +void I444ToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I444ToARGBRow_Any_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I444AlphaToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + const uint8_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I444AlphaToARGBRow_Any_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + const uint8_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I422AlphaToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + const uint8_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I422AlphaToARGBRow_Any_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + const uint8_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I422ToRGBARow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgba, + const struct YuvConstants* yuvconstants, + int width); +void I422ToRGBARow_Any_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgba, + const struct YuvConstants* yuvconstants, + int width); +void I400ToARGBRow_WASMSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I400ToARGBRow_Any_WASMSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void YUY2ToARGBRow_WASMSIMD(const uint8_t* src_yuy2, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void YUY2ToARGBRow_Any_WASMSIMD(const uint8_t* src_yuy2, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void UYVYToARGBRow_WASMSIMD(const uint8_t* src_uyvy, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void UYVYToARGBRow_Any_WASMSIMD(const uint8_t* src_uyvy, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I444ToRGB24Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void I444ToRGB24Row_Any_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void NV12ToRGB24Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void NV12ToRGB24Row_Any_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void NV21ToRGB24Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void NV21ToRGB24Row_Any_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void YUY2ToYRow_WASMSIMD(const uint8_t* src_yuy2, uint8_t* dst_y, int width); +void YUY2ToYRow_Any_WASMSIMD(const uint8_t* src_yuy2, + uint8_t* dst_y, + int width); +void YUY2ToUVRow_WASMSIMD(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void YUY2ToUVRow_Any_WASMSIMD(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void YUY2ToUV422Row_WASMSIMD(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void YUY2ToUV422Row_Any_WASMSIMD(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void YUY2ToNVUVRow_WASMSIMD(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_uv, + int width); +void YUY2ToNVUVRow_Any_WASMSIMD(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_uv, + int width); +void UYVYToYRow_WASMSIMD(const uint8_t* src_uyvy, uint8_t* dst_y, int width); +void UYVYToYRow_Any_WASMSIMD(const uint8_t* src_uyvy, + uint8_t* dst_y, + int width); +void UYVYToUVRow_WASMSIMD(const uint8_t* src_uyvy, + int src_stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void UYVYToUVRow_Any_WASMSIMD(const uint8_t* src_uyvy, + int src_stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void UYVYToUV422Row_WASMSIMD(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void UYVYToUV422Row_Any_WASMSIMD(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void I422ToYUY2Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_yuy2, + int width); +void I422ToYUY2Row_Any_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_yuy2, + int width); +void I422ToUYVYRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uyvy, + int width); +void I422ToUYVYRow_Any_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uyvy, + int width); +void AYUVToYRow_WASMSIMD(const uint8_t* src_ayuv, uint8_t* dst_y, int width); +void AYUVToYRow_Any_WASMSIMD(const uint8_t* src_ayuv, + uint8_t* dst_y, + int width); +void AYUVToUVRow_WASMSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); +void AYUVToUVRow_Any_WASMSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); +void AYUVToVURow_WASMSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); +void AYUVToVURow_Any_WASMSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); +void ARGBCopyAlphaRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void ARGBCopyAlphaRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void ARGBAttenuateRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void ARGBAttenuateRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void ARGBMultiplyRow_WASMSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBMultiplyRow_Any_WASMSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAddRow_WASMSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAddRow_Any_WASMSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBSubtractRow_WASMSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBSubtractRow_Any_WASMSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBGrayRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void ARGBGrayRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void ARGBSepiaRow_WASMSIMD(uint8_t* dst_argb, int width); +void ARGBSepiaRow_Any_WASMSIMD(uint8_t* dst_argb, int width); +void ARGBColorMatrixRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width); +void ARGBColorMatrixRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width); +void ARGBQuantizeRow_WASMSIMD(uint8_t* dst_argb, + int scale, + int interval_size, + int interval_offset, + int width); +void ARGBQuantizeRow_Any_WASMSIMD(uint8_t* dst_argb, + int scale, + int interval_size, + int interval_offset, + int width); +void ARGBShadeRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + uint32_t value); +void ARGBShadeRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + uint32_t value); +void I210ToARGBRow_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I210ToARGBRow_Any_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I210AlphaToARGBRow_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + const uint16_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I210AlphaToARGBRow_Any_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + const uint16_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I410ToARGBRow_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I410ToARGBRow_Any_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I410AlphaToARGBRow_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + const uint16_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I410AlphaToARGBRow_Any_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + const uint16_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I210ToAR30Row_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); +void I210ToAR30Row_Any_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); +void I410ToAR30Row_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); +void I410ToAR30Row_Any_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); +void SobelXRow_WASMSIMD(const uint8_t* src_y0, + const uint8_t* src_y1, + const uint8_t* src_y2, + uint8_t* dst_sobelx, + int width); +void SobelYRow_WASMSIMD(const uint8_t* src_y0, + const uint8_t* src_y1, + uint8_t* dst_sobely, + int width); +void SobelRow_WASMSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width); +void SobelRow_Any_WASMSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width); +void SobelToPlaneRow_WASMSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_y, + int width); +void SobelToPlaneRow_Any_WASMSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_y, + int width); +void SobelXYRow_WASMSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width); +void SobelXYRow_Any_WASMSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width); +void ARGBPolynomialRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const float* poly, + int width); +void ARGBPolynomialRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const float* poly, + int width); +void ARGBLumaColorTableRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff); +void ARGBLumaColorTableRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff); +void HalfFloatRow_WASMSIMD(const uint16_t* src, + uint16_t* dst, + float scale, + int width); +void HalfFloatRow_Any_WASMSIMD(const uint16_t* src, + uint16_t* dst, + float scale, + int width); +void ByteToFloatRow_WASMSIMD(const uint8_t* src, + float* dst, + float scale, + int width); +void ByteToFloatRow_Any_WASMSIMD(const uint8_t* src, + float* dst, + float scale, + int width); +void DetileRow_WASMSIMD(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width); +void DetileRow_Any_WASMSIMD(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width); +void DetileRow_16_WASMSIMD(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width); +void DetileRow_16_Any_WASMSIMD(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width); +void DetileSplitUVRow_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void DetileSplitUVRow_Any_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void DetileToYUY2_WASMSIMD(const uint8_t* src_y, + ptrdiff_t src_y_tile_stride, + const uint8_t* src_uv, + ptrdiff_t src_uv_tile_stride, + uint8_t* dst_yuy2, + int width); +void DetileToYUY2_Any_WASMSIMD(const uint8_t* src_y, + ptrdiff_t src_y_tile_stride, + const uint8_t* src_uv, + ptrdiff_t src_uv_tile_stride, + uint8_t* dst_yuy2, + int width); + +void ARGBToUVMatrixRow_RELAXEDSIMD(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGBToUV444MatrixRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_RELAXEDSIMD(const uint8_t* src_rgb24, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void RGBToUVMatrixRow_RELAXEDSIMD(const uint8_t* src_rgb24, + int src_stride_rgb24, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_RELAXEDSIMD(const uint8_t* src_rgb24, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGB24ToARGBRow_RELAXEDSIMD(const uint8_t* src_rgb24, + uint8_t* dst_argb, + int width); +void RAWToARGBRow_RELAXEDSIMD(const uint8_t* src_raw, + uint8_t* dst_argb, + int width); +void RAWToRGBARow_RELAXEDSIMD(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width); +void RAWToRGB24Row_RELAXEDSIMD(const uint8_t* src_raw, + uint8_t* dst_rgb24, + int width); +void ARGBToRGB24Row_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb24, + int width); +void ARGBToRAWRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_raw, + int width); +void ARGBCopyYToAlphaRow_RELAXEDSIMD(const uint8_t* src, + uint8_t* dst, + int width); +void ARGBBlendRow_RELAXEDSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAttenuateRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void MirrorRow_RELAXEDSIMD(const uint8_t* src, uint8_t* dst, int width); +void MirrorUVRow_RELAXEDSIMD(const uint8_t* src_uv, uint8_t* dst_uv, int width); +void MirrorSplitUVRow_RELAXEDSIMD(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void RGB24MirrorRow_RELAXEDSIMD(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width); +void J400ToARGBRow_RELAXEDSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + int width); +void I400ToARGBRow_RELAXEDSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void ARGBShuffleRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width); +void I422ToRGB24Row_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void I444ToRGB24Row_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void NV12ToRGB24Row_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void NV21ToRGB24Row_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void SplitRGBRow_RELAXEDSIMD(const uint8_t* src_rgb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); +void MergeRGBRow_RELAXEDSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width); +void AYUVToUVRow_RELAXEDSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); +void AYUVToVURow_RELAXEDSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); +void ARGBGrayRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void ARGBSepiaRow_RELAXEDSIMD(uint8_t* dst_argb, int width); +void SobelRow_RELAXEDSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width); + +void ARGBToUVMatrixRow_Any_RELAXEDSIMD(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGBToUV444MatrixRow_Any_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_Any_RELAXEDSIMD(const uint8_t* src_rgb24, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void RGBToUVMatrixRow_Any_RELAXEDSIMD(const uint8_t* src_rgb24, + int src_stride_rgb24, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_Any_RELAXEDSIMD(const uint8_t* src_rgb24, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGB24ToARGBRow_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void RAWToARGBRow_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void RAWToRGBARow_Any_RELAXEDSIMD(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width); +void RAWToRGB24Row_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBToRGB24Row_Any_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb24, + int width); +void ARGBToRAWRow_Any_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_raw, + int width); +void ARGBCopyYToAlphaRow_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBBlendRow_Any_RELAXEDSIMD(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void ARGBAttenuateRow_Any_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void MirrorRow_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void MirrorUVRow_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void MirrorSplitUVRow_Any_RELAXEDSIMD(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void RGB24MirrorRow_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void J400ToARGBRow_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void I400ToARGBRow_Any_RELAXEDSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void ARGBShuffleRow_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const uint8_t* shuffler, + int width); +void I422ToRGB24Row_Any_RELAXEDSIMD(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void I444ToRGB24Row_Any_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void NV12ToRGB24Row_Any_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void NV21ToRGB24Row_Any_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void SplitRGBRow_Any_RELAXEDSIMD(const uint8_t* src_rgb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width); +void MergeRGBRow_Any_RELAXEDSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width); +void AYUVToUVRow_Any_RELAXEDSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); +void AYUVToVURow_Any_RELAXEDSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); +void ARGBGrayRow_Any_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width); +void ARGBSepiaRow_Any_RELAXEDSIMD(uint8_t* dst_argb, int width); +void SobelRow_Any_RELAXEDSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width); void CopyRow_HVX(const uint8_t* src, uint8_t* dst, int count); void CopyRow_16_HVX(const uint16_t* src, uint16_t* dst, int count); diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 59d31e887..9ef48770e 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -232,7 +232,7 @@ extern "C" { #endif // The following are available for WebAssembly SIMD128: -#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#if !defined(LIBYUV_DISABLE_WASMSIMD) && defined(__wasm_simd128__) #define HAS_SCALEADDROW_WASMSIMD #define HAS_SCALEARGBCOLS_WASMSIMD #define HAS_SCALEARGBCOLSUP2_WASMSIMD @@ -240,11 +240,32 @@ extern "C" { #define HAS_SCALEARGBROWDOWN2_WASMSIMD #define HAS_SCALEARGBROWDOWNEVEN_WASMSIMD #define HAS_SCALEFILTERCOLS_WASMSIMD +#define HAS_SCALEMAXSAMPLES_WASMSIMD #define HAS_SCALEROWDOWN2_WASMSIMD #define HAS_SCALEROWDOWN34_WASMSIMD #define HAS_SCALEROWDOWN38_WASMSIMD #define HAS_SCALEROWDOWN4_WASMSIMD +#define HAS_SCALEROWUP2_BILINEAR_WASMSIMD +#define HAS_SCALEROWUP2_LINEAR_WASMSIMD +#define HAS_SCALESAMPLES_WASMSIMD +#define HAS_SCALESUMSAMPLES_WASMSIMD #define HAS_SCALEUVADDCOLS3_WASMSIMD +#define HAS_SCALEUVROWDOWN2_WASMSIMD +#define HAS_SCALEUVROWDOWN2BOX_WASMSIMD +#define HAS_SCALEUVROWDOWN2LINEAR_WASMSIMD +#define HAS_SCALEUVROWDOWNEVEN_WASMSIMD +#define HAS_SCALEUVROWUP2_BILINEAR_WASMSIMD +#define HAS_SCALEUVROWUP2_LINEAR_WASMSIMD +#endif + +// The following are available for WebAssembly Relaxed SIMD: +#if !defined(LIBYUV_DISABLE_RELAXEDSIMD) && defined(__wasm_relaxed_simd__) +#define HAS_SCALEROWDOWN34_RELAXEDSIMD +#define HAS_SCALEROWDOWN38_RELAXEDSIMD +#define HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD +#define HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD +#define HAS_SCALEUVROWUP2_BILINEAR_RELAXEDSIMD +#define HAS_SCALEUVROWUP2_LINEAR_RELAXEDSIMD #endif // The following are available for Hexagon HVX: @@ -2247,6 +2268,164 @@ void ScaleARGBColsUp2_WASMSIMD(uint8_t* dst_argb, int dst_width, int x, int dx); +void ScaleRowDown34_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_0_Box_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_1_Box_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_2_Box_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_Any_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_0_Box_Any_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_1_Box_Any_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_Any_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_Any_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_2_Box_Any_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowDown2_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width); +void ScaleUVRowDown2Linear_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width); +void ScaleUVRowDown2Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width); +void ScaleUVRowDown2_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowDown2Linear_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowDown2Box_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowDownEven_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowDownEven_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowUp2_Linear_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowUp2_Linear_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowUp2_Linear_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowUp2_Linear_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowUp2_Bilinear_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width); +void ScaleRowUp2_Bilinear_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width); +void ScaleRowUp2_Bilinear_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width); +void ScaleRowUp2_Bilinear_Any_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width); +void ScaleUVRowUp2_Linear_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowUp2_Linear_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowUp2_Linear_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowUp2_Linear_Any_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowUp2_Bilinear_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width); +void ScaleUVRowUp2_Bilinear_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width); +void ScaleUVRowUp2_Bilinear_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width); +void ScaleUVRowUp2_Bilinear_Any_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width); +void ScaleSamples_WASMSIMD(const float* src, + float* dst, + float scale, + int width); +float ScaleSumSamples_WASMSIMD(const float* src, + float* dst, + float scale, + int width); +float ScaleMaxSamples_WASMSIMD(const float* src, + float* dst, + float scale, + int width); void ScaleAddRow_HVX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); void ScaleRowDown2_HVX(const uint8_t* src_ptr, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index 5c69922cb..e3aa6940a 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1978 +#define LIBYUV_VERSION 1979 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/compare.cc b/source/compare.cc index 4ad159ad2..10459bfbe 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -252,6 +252,11 @@ uint64_t ComputeHammingDistance(const uint8_t* src_a, HammingDistance = HammingDistance_WASMSIMD; } #endif +#if defined(HAS_HAMMINGDISTANCE_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + HammingDistance = HammingDistance_RELAXEDSIMD; + } +#endif #if defined(HAS_HAMMINGDISTANCE_HVX) if (TestCpuFlag(kCpuHasHVX)) { HammingDistance = HammingDistance_HVX; diff --git a/source/compare_wasm.cc b/source/compare_wasm.cc index 5447cf84b..0c952195e 100644 --- a/source/compare_wasm.cc +++ b/source/compare_wasm.cc @@ -12,7 +12,8 @@ #include "libyuv/row.h" #include "libyuv/video_common.h" -#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#if (!defined(LIBYUV_DISABLE_WASMSIMD) && defined(__wasm_simd128__)) || \ + (!defined(LIBYUV_DISABLE_RELAXEDSIMD) && defined(__wasm_relaxed_simd__)) #include #ifdef __cplusplus @@ -29,15 +30,11 @@ static inline uint32_t SumLanes_WASMSIMD(v128_t sum) { #if defined(HAS_HAMMINGDISTANCE_WASMSIMD) // Bit counts of 64 bytes are added as bytes, then accumulated into 32 bit -// lanes using relaxed i8x16 dot product when available or pairwise widening -// adds. Count is a multiple of 64. +// lanes using pairwise widening adds. Count is a multiple of 64. uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a, const uint8_t* src_b, int count) { v128_t sum = wasm_i32x4_splat(0); -#ifdef __wasm_relaxed_simd__ - const v128_t ones = wasm_i8x16_splat(1); -#endif while (count > 0) { v128_t c0 = wasm_i8x16_popcnt( wasm_v128_xor(wasm_v128_load(src_a), wasm_v128_load(src_b))); @@ -48,12 +45,8 @@ uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a, v128_t c3 = wasm_i8x16_popcnt( wasm_v128_xor(wasm_v128_load(src_a + 48), wasm_v128_load(src_b + 48))); v128_t c = wasm_i8x16_add(wasm_i8x16_add(c0, c1), wasm_i8x16_add(c2, c3)); -#ifdef __wasm_relaxed_simd__ - sum = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(c, ones, sum); -#else sum = wasm_i32x4_add(sum, wasm_u32x4_extadd_pairwise_u16x8( wasm_u16x8_extadd_pairwise_u8x16(c))); -#endif src_a += 64; src_b += 64; count -= 64; @@ -62,6 +55,33 @@ uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a, } #endif // HAS_HAMMINGDISTANCE_WASMSIMD +#if defined(HAS_HAMMINGDISTANCE_RELAXEDSIMD) +// Bit counts of 64 bytes are added as bytes, then accumulated into 32 bit +// lanes using relaxed i8x16 dot product. Count is a multiple of 64. +uint32_t HammingDistance_RELAXEDSIMD(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + v128_t sum = wasm_i32x4_splat(0); + const v128_t ones = wasm_i8x16_splat(1); + while (count > 0) { + v128_t c0 = wasm_i8x16_popcnt( + wasm_v128_xor(wasm_v128_load(src_a), wasm_v128_load(src_b))); + v128_t c1 = wasm_i8x16_popcnt( + wasm_v128_xor(wasm_v128_load(src_a + 16), wasm_v128_load(src_b + 16))); + v128_t c2 = wasm_i8x16_popcnt( + wasm_v128_xor(wasm_v128_load(src_a + 32), wasm_v128_load(src_b + 32))); + v128_t c3 = wasm_i8x16_popcnt( + wasm_v128_xor(wasm_v128_load(src_a + 48), wasm_v128_load(src_b + 48))); + v128_t c = wasm_i8x16_add(wasm_i8x16_add(c0, c1), wasm_i8x16_add(c2, c3)); + sum = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(c, ones, sum); + src_a += 64; + src_b += 64; + count -= 64; + } + return SumLanes_WASMSIMD(sum); +} +#endif // HAS_HAMMINGDISTANCE_RELAXEDSIMD + #if defined(HAS_SUMSQUAREERROR_WASMSIMD) // Differences are widened to signed 16 bits and squared with i32x4.dot. // Count is a multiple of 16. diff --git a/source/convert.cc b/source/convert.cc index 8437648d0..308e77835 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -326,6 +326,14 @@ static int I41xToI420(const uint16_t* src_y, } } #endif +#if defined(HAS_CONVERT16TO8ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert16To8Row = Convert16To8Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_WASMSIMD; + } + } +#endif if (IS_ALIGNED(width, 2)) { #if defined(HAS_HALFWIDTHROW_16TO8_NEON) @@ -371,6 +379,14 @@ static int I41xToI420(const uint16_t* src_y, if (TestCpuFlag(kCpuHasRVV)) { HalfWidthRow_16To8 = HalfWidthRow_16To8_RVV; } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_WASMSIMD; + if (IS_ALIGNED(uv_width, 8)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_WASMSIMD; + } + } #endif } else { #if defined(HAS_HALFWIDTHROW_16TO8_NEON) @@ -516,6 +532,14 @@ static int I21xToI420(const uint16_t* src_y, } } #endif +#if defined(HAS_CONVERT16TO8ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert16To8Row = Convert16To8Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_WASMSIMD; + } + } +#endif #if defined(HAS_HALFROW_16TO8_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -582,6 +606,14 @@ static int I21xToI420(const uint16_t* src_y, } } #endif +#if defined(HAS_HALFROW_16TO8_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + HalfRow_16To8 = HalfRow_16To8_Any_WASMSIMD; + if (IS_ALIGNED(uv_width, 16)) { + HalfRow_16To8 = HalfRow_16To8_WASMSIMD; + } + } +#endif for (y = 0; y < height - 1; y += 2) { HalfRow_16To8(src_u, src_stride_u, dst_u, scale, uv_width); @@ -1048,6 +1080,14 @@ int I010ToNV12(const uint16_t* src_y, } } #endif +#if defined(HAS_CONVERT16TO8ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert16To8Row = Convert16To8Row_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + Convert16To8Row = Convert16To8Row_WASMSIMD; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -2210,6 +2250,14 @@ static int Px10ToI420(const uint16_t* src_y, } } #endif +#if defined(HAS_CONVERT16TO8ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert16To8Row = Convert16To8Row_Any_WASMSIMD; + if (IS_ALIGNED(uv_packed, 16)) { + Convert16To8Row = Convert16To8Row_WASMSIMD; + } + } +#endif #if defined(HAS_SPLITUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { SplitUVRow = SplitUVRow_Any_SSE2; @@ -2275,6 +2323,14 @@ static int Px10ToI420(const uint16_t* src_y, SplitUVRow = SplitUVRow_HVX; } #endif +#if defined(HAS_SPLITUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitUVRow = SplitUVRow_Any_WASMSIMD; + if (IS_ALIGNED(src_uv_width, 16)) { + SplitUVRow = SplitUVRow_WASMSIMD; + } + } +#endif align_buffer_64(row_uv, ((uv_packed + 63) & ~63)); if (!row_uv) { @@ -2340,6 +2396,14 @@ static int Px10ToI420(const uint16_t* src_y, SplitUVRow_16 = SplitUVRow_16_LASX; } } +#endif +#if defined(HAS_SPLITUVROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitUVRow_16 = SplitUVRow_16_Any_WASMSIMD; + if (IS_ALIGNED(src_uv_width, 8)) { + SplitUVRow_16 = SplitUVRow_16_WASMSIMD; + } + } #endif if (subsample_x) { #if defined(HAS_HALFROW_16TO8_NEON) @@ -2406,6 +2470,14 @@ static int Px10ToI420(const uint16_t* src_y, Half16To8 = HalfRow_16To8_LASX; } } +#endif +#if defined(HAS_HALFROW_16TO8_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Half16To8 = HalfRow_16To8_Any_WASMSIMD; + if (IS_ALIGNED(dst_uv_width, 16)) { + Half16To8 = HalfRow_16To8_WASMSIMD; + } + } #endif } else if (IS_ALIGNED(width, 2)) { #if defined(HAS_HALFWIDTHROW_16TO8_NEON) @@ -2451,6 +2523,14 @@ static int Px10ToI420(const uint16_t* src_y, if (TestCpuFlag(kCpuHasRVV)) { Half16To8 = HalfWidthRow_16To8_RVV; } +#endif +#if defined(HAS_HALFWIDTHROW_16TO8_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Half16To8 = HalfWidthRow_16To8_Any_WASMSIMD; + if (IS_ALIGNED(dst_uv_width, 8)) { + Half16To8 = HalfWidthRow_16To8_WASMSIMD; + } + } #endif } else { #if defined(HAS_HALFWIDTHROW_16TO8_NEON) @@ -2721,6 +2801,16 @@ int YUY2ToI420(const uint8_t* src_yuy2, YUY2ToUVRow = YUY2ToUVRow_HVX; } #endif +#if defined(HAS_YUY2TOYROW_WASMSIMD) && defined(HAS_YUY2TOUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + YUY2ToYRow = YUY2ToYRow_Any_WASMSIMD; + YUY2ToUVRow = YUY2ToUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + YUY2ToYRow = YUY2ToYRow_WASMSIMD; + YUY2ToUVRow = YUY2ToUVRow_WASMSIMD; + } + } +#endif for (y = 0; y < height - 1; y += 2) { YUY2ToUVRow(src_yuy2, src_stride_yuy2, dst_u, dst_v, width); @@ -2840,6 +2930,16 @@ int UYVYToI420(const uint8_t* src_uyvy, UYVYToUVRow = UYVYToUVRow_HVX; } #endif +#if defined(HAS_UYVYTOYROW_WASMSIMD) && defined(HAS_UYVYTOUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + UYVYToYRow = UYVYToYRow_Any_WASMSIMD; + UYVYToUVRow = UYVYToUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + UYVYToYRow = UYVYToYRow_WASMSIMD; + UYVYToUVRow = UYVYToUVRow_WASMSIMD; + } + } +#endif for (y = 0; y < height - 1; y += 2) { UYVYToUVRow(src_uyvy, src_stride_uyvy, dst_u, dst_v, width); @@ -2940,6 +3040,24 @@ int AYUVToNV12(const uint8_t* src_ayuv, } } #endif +#if defined(HAS_AYUVTOYROW_WASMSIMD) && defined(HAS_AYUVTOUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + AYUVToYRow = AYUVToYRow_Any_WASMSIMD; + AYUVToUVRow = AYUVToUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + AYUVToYRow = AYUVToYRow_WASMSIMD; + AYUVToUVRow = AYUVToUVRow_WASMSIMD; + } + } +#endif +#if defined(HAS_AYUVTOUVROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + AYUVToUVRow = AYUVToUVRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + AYUVToUVRow = AYUVToUVRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height - 1; y += 2) { AYUVToUVRow(src_ayuv, src_stride_ayuv, dst_uv, width); @@ -3039,6 +3157,24 @@ int AYUVToNV21(const uint8_t* src_ayuv, } } #endif +#if defined(HAS_AYUVTOYROW_WASMSIMD) && defined(HAS_AYUVTOVUROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + AYUVToYRow = AYUVToYRow_Any_WASMSIMD; + AYUVToVURow = AYUVToVURow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + AYUVToYRow = AYUVToYRow_WASMSIMD; + AYUVToVURow = AYUVToVURow_WASMSIMD; + } + } +#endif +#if defined(HAS_AYUVTOVUROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + AYUVToVURow = AYUVToVURow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + AYUVToVURow = AYUVToVURow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height - 1; y += 2) { AYUVToVURow(src_ayuv, src_stride_ayuv, dst_vu, width); @@ -3258,6 +3394,14 @@ int ARGBToI420Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX; @@ -3517,6 +3661,14 @@ int ARGBToI420Alpha(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX; @@ -3769,6 +3921,14 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, } } #endif +#if defined(HAS_AR30TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + AR30ToARGBRow = AR30ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + AR30ToARGBRow = AR30ToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; @@ -3934,6 +4094,14 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX; @@ -4119,6 +4287,14 @@ int RGB24ToI420(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; @@ -4127,6 +4303,14 @@ int RGB24ToI420(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_RELAXEDSIMD; + } + } +#endif if (!src_rgb24 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -4277,6 +4461,14 @@ int RGB24ToJ420(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; @@ -4285,6 +4477,14 @@ int RGB24ToJ420(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_RELAXEDSIMD; + } + } +#endif if (!src_rgb24 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -4424,6 +4624,14 @@ int RAWToI420(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; @@ -4432,6 +4640,14 @@ int RAWToI420(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_RGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { RGBToUVMatrixRow = RGBToUVMatrixRow_RVV; @@ -4582,6 +4798,14 @@ int RAWToJ420(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; @@ -4590,6 +4814,14 @@ int RAWToJ420(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_RGBTOUVMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { RGBToUVMatrixRow = RGBToUVMatrixRow_RVV; @@ -4715,6 +4947,14 @@ static int RAWToI444Matrix(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_RGBTOUV444MATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -4758,6 +4998,14 @@ static int RAWToI444Matrix(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOUV444MATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_RELAXEDSIMD; + } + } +#endif if (!src_raw || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -5204,6 +5452,14 @@ int RGB24ToJ400(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { RGBToYMatrixRow(src_rgb24, dst_yj, width, &kArgbJPEGConstants); @@ -5306,6 +5562,14 @@ int RAWToJ400(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { RGBToYMatrixRow(src_raw, dst_yj, width, &kAbgrJPEGConstants); diff --git a/source/convert_argb.cc b/source/convert_argb.cc index b10c18f5f..4258754f5 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -678,6 +678,14 @@ int I444ToARGBMatrix(const uint8_t* src_y, I444ToARGBRow = I444ToARGBRow_HVX; } #endif +#if defined(HAS_I444TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444ToARGBRow = I444ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I444ToARGBRow = I444ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I444ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -913,6 +921,22 @@ int I444ToRGB24Matrix(const uint8_t* src_y, I444ToRGB24Row = I444ToRGB24Row_RVV; } #endif +#if defined(HAS_I444TORGB24ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444ToRGB24Row = I444ToRGB24Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I444ToRGB24Row = I444ToRGB24Row_WASMSIMD; + } + } +#endif +#if defined(HAS_I444TORGB24ROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + I444ToRGB24Row = I444ToRGB24Row_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + I444ToRGB24Row = I444ToRGB24Row_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I444ToRGB24Row(src_y, src_u, src_v, dst_rgb24, yuvconstants, width); @@ -1029,6 +1053,14 @@ int I010ToAR30Matrix(const uint16_t* src_y, if (TestCpuFlag(kCpuHasAVX512BW)) { I210ToAR30Row = I210ToAR30Row_AVX512BW; } +#endif +#if defined(HAS_I210TOAR30ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I210ToAR30Row = I210ToAR30Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I210ToAR30Row = I210ToAR30Row_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); @@ -1303,6 +1335,14 @@ int I210ToAR30Matrix(const uint16_t* src_y, if (TestCpuFlag(kCpuHasAVX512BW)) { I210ToAR30Row = I210ToAR30Row_AVX512BW; } +#endif +#if defined(HAS_I210TOAR30ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I210ToAR30Row = I210ToAR30Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I210ToAR30Row = I210ToAR30Row_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); @@ -1489,6 +1529,14 @@ int I410ToAR30Matrix(const uint16_t* src_y, if (TestCpuFlag(kCpuHasAVX512BW)) { I410ToAR30Row = I410ToAR30Row_AVX512BW; } +#endif +#if defined(HAS_I410TOAR30ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I410ToAR30Row = I410ToAR30Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I410ToAR30Row = I410ToAR30Row_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { I410ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); @@ -1567,6 +1615,14 @@ int I010ToARGBMatrix(const uint16_t* src_y, if (TestCpuFlag(kCpuHasAVX512BW)) { I210ToARGBRow = I210ToARGBRow_AVX512BW; } +#endif +#if defined(HAS_I210TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I210ToARGBRow = I210ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I210ToARGBRow = I210ToARGBRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -1843,6 +1899,14 @@ int I210ToARGBMatrix(const uint16_t* src_y, if (TestCpuFlag(kCpuHasAVX512BW)) { I210ToARGBRow = I210ToARGBRow_AVX512BW; } +#endif +#if defined(HAS_I210TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I210ToARGBRow = I210ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I210ToARGBRow = I210ToARGBRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -2035,6 +2099,14 @@ int I410ToARGBMatrix(const uint16_t* src_y, if (TestCpuFlag(kCpuHasAVX512BW)) { I410ToARGBRow = I410ToARGBRow_AVX512BW; } +#endif +#if defined(HAS_I410TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I410ToARGBRow = I410ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I410ToARGBRow = I410ToARGBRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { I410ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -2616,6 +2688,14 @@ int I420AlphaToARGBMatrix(const uint8_t* src_y, I422AlphaToARGBRow = I422AlphaToARGBRow_HVX; } #endif +#if defined(HAS_I422ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422AlphaToARGBRow = I422AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422AlphaToARGBRow = I422AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -2671,6 +2751,22 @@ int I420AlphaToARGBMatrix(const uint8_t* src_y, ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, @@ -2798,6 +2894,14 @@ int I422AlphaToARGBMatrix(const uint8_t* src_y, I422AlphaToARGBRow = I422AlphaToARGBRow_HVX; } #endif +#if defined(HAS_I422ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422AlphaToARGBRow = I422AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422AlphaToARGBRow = I422AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -2853,6 +2957,22 @@ int I422AlphaToARGBMatrix(const uint8_t* src_y, ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, @@ -2962,6 +3082,14 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y, I444AlphaToARGBRow = I444AlphaToARGBRow_HVX; } #endif +#if defined(HAS_I444ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -3017,6 +3145,22 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y, ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I444AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, @@ -3236,6 +3380,14 @@ int I010AlphaToARGBMatrix(const uint16_t* src_y, I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW; } #endif +#if defined(HAS_I210ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I210AlphaToARGBRow = I210AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I210AlphaToARGBRow = I210AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -3291,6 +3443,22 @@ int I010AlphaToARGBMatrix(const uint16_t* src_y, ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I210AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, @@ -3383,6 +3551,14 @@ int I210AlphaToARGBMatrix(const uint16_t* src_y, I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW; } #endif +#if defined(HAS_I210ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I210AlphaToARGBRow = I210AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I210AlphaToARGBRow = I210AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -3438,6 +3614,22 @@ int I210AlphaToARGBMatrix(const uint16_t* src_y, ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I210AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, @@ -3528,6 +3720,14 @@ int I410AlphaToARGBMatrix(const uint16_t* src_y, I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW; } #endif +#if defined(HAS_I410ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -3583,6 +3783,22 @@ int I410AlphaToARGBMatrix(const uint16_t* src_y, ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I410AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, @@ -3694,6 +3910,22 @@ int I400ToARGBMatrix(const uint8_t* src_y, I400ToARGBRow = I400ToARGBRow_HVX; } #endif +#if defined(HAS_I400TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I400ToARGBRow = I400ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I400ToARGBRow = I400ToARGBRow_WASMSIMD; + } + } +#endif +#if defined(HAS_I400TOARGBROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + I400ToARGBRow = I400ToARGBRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + I400ToARGBRow = I400ToARGBRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I400ToARGBRow(src_y, dst_argb, yuvconstants, width); @@ -3793,6 +4025,14 @@ int J400ToARGB(const uint8_t* src_y, } } #endif +#if defined(HAS_J400TOARGBROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + J400ToARGBRow = J400ToARGBRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + J400ToARGBRow = J400ToARGBRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_J400TOARGBROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { J400ToARGBRow = J400ToARGBRow_HVX; @@ -4208,6 +4448,14 @@ int RGB24ToARGB(const uint8_t* src_rgb24, RGB24ToARGBRow = RGB24ToARGBRow_WASMSIMD; } } +#endif +#if defined(HAS_RGB24TOARGBROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGB24ToARGBRow = RGB24ToARGBRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGB24ToARGBRow = RGB24ToARGBRow_RELAXEDSIMD; + } + } #endif for (y = 0; y < height; ++y) { RGB24ToARGBRow(src_rgb24, dst_argb, width); @@ -4312,6 +4560,14 @@ int RAWToARGB(const uint8_t* src_raw, } } #endif +#if defined(HAS_RAWTOARGBROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RAWToARGBRow = RAWToARGBRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RAWToARGBRow = RAWToARGBRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { RAWToARGBRow(src_raw, dst_argb, width); @@ -4401,6 +4657,22 @@ int RAWToRGBA(const uint8_t* src_raw, } } #endif +#if defined(HAS_RAWTORGBAROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RAWToRGBARow = RAWToRGBARow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RAWToRGBARow = RAWToRGBARow_WASMSIMD; + } + } +#endif +#if defined(HAS_RAWTORGBAROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RAWToRGBARow = RAWToRGBARow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RAWToRGBARow = RAWToRGBARow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { RAWToRGBARow(src_raw, dst_rgba, width); @@ -4723,6 +4995,14 @@ int AR30ToARGB(const uint8_t* src_ar30, AR30ToARGBRow = AR30ToARGBRow_LASX; } } +#endif +#if defined(HAS_AR30TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + AR30ToARGBRow = AR30ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + AR30ToARGBRow = AR30ToARGBRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { AR30ToARGBRow(src_ar30, dst_argb, width); @@ -4796,6 +5076,14 @@ int AR30ToABGR(const uint8_t* src_ar30, AR30ToABGRRow = AR30ToABGRRow_LASX; } } +#endif +#if defined(HAS_AR30TOABGRROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + AR30ToABGRRow = AR30ToABGRRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + AR30ToABGRRow = AR30ToABGRRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { AR30ToABGRRow(src_ar30, dst_abgr, width); @@ -4869,6 +5157,14 @@ int AR30ToAB30(const uint8_t* src_ar30, AR30ToAB30Row = AR30ToAB30Row_LASX; } } +#endif +#if defined(HAS_AR30TOAB30ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + AR30ToAB30Row = AR30ToAB30Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + AR30ToAB30Row = AR30ToAB30Row_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { AR30ToAB30Row(src_ar30, dst_ab30, width); @@ -4956,6 +5252,14 @@ int AR64ToARGB(const uint16_t* src_ar64, } } #endif +#if defined(HAS_AR64TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + AR64ToARGBRow = AR64ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + AR64ToARGBRow = AR64ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { AR64ToARGBRow(src_ar64, dst_argb, width); @@ -5043,6 +5347,14 @@ int AB64ToARGB(const uint16_t* src_ab64, } } #endif +#if defined(HAS_AB64TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + AB64ToARGBRow = AB64ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + AB64ToARGBRow = AB64ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { AB64ToARGBRow(src_ab64, dst_argb, width); @@ -5404,6 +5716,22 @@ int NV12ToRGB24Matrix(const uint8_t* src_y, NV12ToRGB24Row = NV12ToRGB24Row_RVV; } #endif +#if defined(HAS_NV12TORGB24ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + NV12ToRGB24Row = NV12ToRGB24Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + NV12ToRGB24Row = NV12ToRGB24Row_WASMSIMD; + } + } +#endif +#if defined(HAS_NV12TORGB24ROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + NV12ToRGB24Row = NV12ToRGB24Row_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + NV12ToRGB24Row = NV12ToRGB24Row_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { NV12ToRGB24Row(src_y, src_uv, dst_rgb24, yuvconstants, width); @@ -5487,6 +5815,22 @@ int NV21ToRGB24Matrix(const uint8_t* src_y, NV21ToRGB24Row = NV21ToRGB24Row_RVV; } #endif +#if defined(HAS_NV21TORGB24ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + NV21ToRGB24Row = NV21ToRGB24Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + NV21ToRGB24Row = NV21ToRGB24Row_WASMSIMD; + } + } +#endif +#if defined(HAS_NV21TORGB24ROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + NV21ToRGB24Row = NV21ToRGB24Row_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + NV21ToRGB24Row = NV21ToRGB24Row_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { NV21ToRGB24Row(src_y, src_vu, dst_rgb24, yuvconstants, width); @@ -5703,6 +6047,14 @@ int YUY2ToARGBMatrix(const uint8_t* src_yuy2, YUY2ToARGBRow = YUY2ToARGBRow_LASX; } } +#endif +#if defined(HAS_YUY2TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + YUY2ToARGBRow = YUY2ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + YUY2ToARGBRow = YUY2ToARGBRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { YUY2ToARGBRow(src_yuy2, dst_argb, yuvconstants, width); @@ -5808,6 +6160,14 @@ int UYVYToARGBMatrix(const uint8_t* src_uyvy, UYVYToARGBRow = UYVYToARGBRow_LASX; } } +#endif +#if defined(HAS_UYVYTOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + UYVYToARGBRow = UYVYToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + UYVYToARGBRow = UYVYToARGBRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { UYVYToARGBRow(src_uyvy, dst_argb, yuvconstants, width); @@ -6054,6 +6414,14 @@ int I422ToRGBAMatrix(const uint8_t* src_y, I422ToRGBARow = I422ToRGBARow_HVX; } #endif +#if defined(HAS_I422TORGBAROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToRGBARow = I422ToRGBARow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToRGBARow = I422ToRGBARow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToRGBARow(src_y, src_u, src_v, dst_rgba, yuvconstants, width); @@ -6296,6 +6664,14 @@ int I420ToRGBAMatrix(const uint8_t* src_y, I422ToRGBARow = I422ToRGBARow_HVX; } #endif +#if defined(HAS_I422TORGBAROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToRGBARow = I422ToRGBARow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToRGBARow = I422ToRGBARow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToRGBARow(src_y, src_u, src_v, dst_rgba, yuvconstants, width); @@ -6447,6 +6823,14 @@ int I420ToRGB24Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB24ROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + I422ToRGB24Row = I422ToRGB24Row_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + I422ToRGB24Row = I422ToRGB24Row_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToRGB24Row(src_y, src_u, src_v, dst_rgb24, yuvconstants, width); @@ -6678,6 +7062,14 @@ int I422ToRGB24Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB24ROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + I422ToRGB24Row = I422ToRGB24Row_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + I422ToRGB24Row = I422ToRGB24Row_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToRGB24Row(src_y, src_u, src_v, dst_rgb24, yuvconstants, width); @@ -7673,6 +8065,14 @@ static int I420ToARGBMatrixBilinear(const uint8_t* src_y, I444ToARGBRow = I444ToARGBRow_HVX; } #endif +#if defined(HAS_I444TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444ToARGBRow = I444ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I444ToARGBRow = I444ToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -7707,6 +8107,18 @@ static int I420ToARGBMatrixBilinear(const uint8_t* src_y, ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV; } #endif +#if defined(HAS_SCALEROWUP2_BILINEAR_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_WASMSIMD; + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_WASMSIMD; + } +#endif +#if defined(HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_RELAXEDSIMD; + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_RELAXEDSIMD; + } +#endif // alloc 4 lines temp const int row_size = (width + 31) & ~31; @@ -7841,6 +8253,14 @@ static int I422ToARGBMatrixLinear(const uint8_t* src_y, I444ToARGBRow = I444ToARGBRow_HVX; } #endif +#if defined(HAS_I444TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444ToARGBRow = I444ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I444ToARGBRow = I444ToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSE2; @@ -7866,6 +8286,16 @@ static int I422ToARGBMatrixLinear(const uint8_t* src_y, ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV; } #endif +#if defined(HAS_SCALEROWUP2_LINEAR_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_WASMSIMD; + } +#endif +#if defined(HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_RELAXEDSIMD; + } +#endif // alloc 2 lines temp const int row_size = (width + 31) & ~31; @@ -7965,6 +8395,22 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y, I444ToRGB24Row = I444ToRGB24Row_RVV; } #endif +#if defined(HAS_I444TORGB24ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444ToRGB24Row = I444ToRGB24Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I444ToRGB24Row = I444ToRGB24Row_WASMSIMD; + } + } +#endif +#if defined(HAS_I444TORGB24ROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + I444ToRGB24Row = I444ToRGB24Row_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + I444ToRGB24Row = I444ToRGB24Row_RELAXEDSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -7999,6 +8445,18 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y, ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV; } #endif +#if defined(HAS_SCALEROWUP2_BILINEAR_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_WASMSIMD; + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_WASMSIMD; + } +#endif +#if defined(HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_RELAXEDSIMD; + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_RELAXEDSIMD; + } +#endif // alloc 4 lines temp const int row_size = (width + 31) & ~31; @@ -8110,6 +8568,14 @@ static int I010ToAR30MatrixBilinear(const uint16_t* src_y, I410ToAR30Row = I410ToAR30Row_AVX512BW; } #endif +#if defined(HAS_I410TOAR30ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I410ToAR30Row = I410ToAR30Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I410ToAR30Row = I410ToAR30Row_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8240,6 +8706,14 @@ static int I210ToAR30MatrixLinear(const uint16_t* src_y, I410ToAR30Row = I410ToAR30Row_AVX512BW; } #endif +#if defined(HAS_I410TOAR30ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I410ToAR30Row = I410ToAR30Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I410ToAR30Row = I410ToAR30Row_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8349,6 +8823,14 @@ static int I010ToARGBMatrixBilinear(const uint16_t* src_y, I410ToARGBRow = I410ToARGBRow_AVX512BW; } #endif +#if defined(HAS_I410TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I410ToARGBRow = I410ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I410ToARGBRow = I410ToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8478,6 +8960,14 @@ static int I210ToARGBMatrixLinear(const uint16_t* src_y, I410ToARGBRow = I410ToARGBRow_AVX512BW; } #endif +#if defined(HAS_I410TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I410ToARGBRow = I410ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I410ToARGBRow = I410ToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8605,6 +9095,14 @@ static int I420AlphaToARGBMatrixBilinear( I444AlphaToARGBRow = I444AlphaToARGBRow_HVX; } #endif +#if defined(HAS_I444ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -8660,6 +9158,22 @@ static int I420AlphaToARGBMatrixBilinear( ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -8694,6 +9208,18 @@ static int I420AlphaToARGBMatrixBilinear( ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV; } #endif +#if defined(HAS_SCALEROWUP2_BILINEAR_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_WASMSIMD; + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_WASMSIMD; + } +#endif +#if defined(HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + Scale2RowUp_Bilinear = ScaleRowUp2_Bilinear_Any_RELAXEDSIMD; + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_RELAXEDSIMD; + } +#endif // alloc 4 lines temp const int row_size = (width + 31) & ~31; @@ -8837,6 +9363,14 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, I444AlphaToARGBRow = I444AlphaToARGBRow_HVX; } #endif +#if defined(HAS_I444ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -8892,6 +9426,22 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -8918,6 +9468,16 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV; } #endif +#if defined(HAS_SCALEROWUP2_LINEAR_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_WASMSIMD; + } +#endif +#if defined(HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_RELAXEDSIMD; + } +#endif // alloc 2 lines temp const int row_size = (width + 31) & ~31; @@ -9024,6 +9584,14 @@ static int I010AlphaToARGBMatrixBilinear( I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW; } #endif +#if defined(HAS_I410ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -9079,6 +9647,22 @@ static int I010AlphaToARGBMatrixBilinear( ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -9233,6 +9817,14 @@ static int I210AlphaToARGBMatrixLinear(const uint16_t* src_y, I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW; } #endif +#if defined(HAS_I410ALPHATOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -9288,6 +9880,22 @@ static int I210AlphaToARGBMatrixLinear(const uint16_t* src_y, ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -9415,6 +10023,11 @@ static int P010ToARGBMatrixBilinear(const uint16_t* src_y, Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_NEON; } #endif +#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_WASMSIMD + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_WASMSIMD; + } +#endif // alloc 2 lines temp const int row_size = (2 * width + 31) & ~31; @@ -9532,6 +10145,11 @@ static int P210ToARGBMatrixLinear(const uint16_t* src_y, ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_NEON; } #endif +#ifdef HAS_SCALEUVROWUP2_LINEAR_16_WASMSIMD + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_WASMSIMD; + } +#endif const int row_size = (2 * width + 31) & ~31; align_buffer_64(row, row_size * sizeof(uint16_t)); @@ -9635,6 +10253,11 @@ static int P010ToAR30MatrixBilinear(const uint16_t* src_y, Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_NEON; } #endif +#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_WASMSIMD + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Scale2RowUp_Bilinear_16 = ScaleUVRowUp2_Bilinear_16_Any_WASMSIMD; + } +#endif // alloc 2 lines temp const int row_size = (2 * width + 31) & ~31; @@ -9752,6 +10375,11 @@ static int P210ToAR30MatrixLinear(const uint16_t* src_y, ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_NEON; } #endif +#ifdef HAS_SCALEUVROWUP2_LINEAR_16_WASMSIMD + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowUp2_Linear = ScaleUVRowUp2_Linear_16_Any_WASMSIMD; + } +#endif const int row_size = (2 * width + 31) & ~31; align_buffer_64(row, row_size * sizeof(uint16_t)); @@ -9844,6 +10472,22 @@ static int I422ToRGB24MatrixLinear(const uint8_t* src_y, I444ToRGB24Row = I444ToRGB24Row_RVV; } #endif +#if defined(HAS_I444TORGB24ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444ToRGB24Row = I444ToRGB24Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I444ToRGB24Row = I444ToRGB24Row_WASMSIMD; + } + } +#endif +#if defined(HAS_I444TORGB24ROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + I444ToRGB24Row = I444ToRGB24Row_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + I444ToRGB24Row = I444ToRGB24Row_RELAXEDSIMD; + } + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_SSE2; @@ -9869,6 +10513,16 @@ static int I422ToRGB24MatrixLinear(const uint8_t* src_y, ScaleRowUp2_Linear = ScaleRowUp2_Linear_RVV; } #endif +#if defined(HAS_SCALEROWUP2_LINEAR_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_WASMSIMD; + } +#endif +#if defined(HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ScaleRowUp2_Linear = ScaleRowUp2_Linear_Any_RELAXEDSIMD; + } +#endif // alloc 2 lines temp const int row_size = (width + 31) & ~31; diff --git a/source/convert_from.cc b/source/convert_from.cc index 037dcec39..d501ea3dc 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -414,6 +414,14 @@ int I422ToYUY2(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOYUY2ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToYUY2Row = I422ToYUY2Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToYUY2Row(src_y, src_u, src_v, dst_yuy2, width); @@ -490,6 +498,14 @@ int I420ToYUY2(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOYUY2ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToYUY2Row = I422ToYUY2Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_WASMSIMD; + } + } +#endif for (y = 0; y < height - 1; y += 2) { I422ToYUY2Row(src_y, src_u, src_v, dst_yuy2, width); @@ -579,6 +595,14 @@ int I422ToUYVY(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOUYVYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToUYVYRow = I422ToUYVYRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToUYVYRow(src_y, src_u, src_v, dst_uyvy, width); @@ -655,6 +679,14 @@ int I420ToUYVY(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOUYVYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToUYVYRow = I422ToUYVYRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_WASMSIMD; + } + } +#endif for (y = 0; y < height - 1; y += 2) { I422ToUYVYRow(src_y, src_u, src_v, dst_uyvy, width); @@ -890,6 +922,14 @@ static int I4xxToPxxx(const uint8_t* src_y, } } #endif +#if defined(HAS_MERGEUVROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow_16 = MergeUVRow_16_Any_WASMSIMD; + if (IS_ALIGNED(uv_width, 8)) { + MergeUVRow_16 = MergeUVRow_16_WASMSIMD; + } + } +#endif row_stride = (uv_width + 31) & ~31; align_buffer_64(row_u, (size_t)row_stride * sizeof(uint16_t) * 2u); diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 623384718..b4fa6b881 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -216,6 +216,14 @@ int ARGBToI444Matrix(const uint8_t* src_argb, ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_WASMSIMD; } } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RELAXEDSIMD; + } + } #endif if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -446,6 +454,14 @@ int ARGBToI422Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } #endif if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -672,6 +688,14 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) = MergeUVRow_C; @@ -988,6 +1012,14 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) = MergeUVRow_C; @@ -1284,6 +1316,14 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_WASMSIMD; } } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RELAXEDSIMD; + } + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) = MergeUVRow_C; @@ -1582,6 +1622,14 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_vu, int width) = MergeUVRow_C; @@ -2013,6 +2061,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX; @@ -2058,6 +2114,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_I422TOYUY2ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToYUY2Row = I422ToYUY2Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I422ToYUY2Row = I422ToYUY2Row_WASMSIMD; + } + } +#endif { align_buffer_64(row_y, ((width + 63) & ~63) * 2); @@ -2272,6 +2336,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_HVX; @@ -2317,6 +2389,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_I422TOUYVYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToUYVYRow = I422ToUYVYRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + I422ToUYVYRow = I422ToUYVYRow_WASMSIMD; + } + } +#endif { align_buffer_64(row_y, ((width + 63) & ~63) * 2); @@ -2565,6 +2645,14 @@ int ARGBToP010Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } #endif // Negative height means invert the image. if (height < 0) { @@ -3036,6 +3124,14 @@ int ARGBToP210Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RELAXEDSIMD; + } + } #endif // Negative height means invert the image. if (height < 0) { @@ -3478,6 +3574,14 @@ int ARGBToP410Matrix(const uint8_t* src_argb, ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_WASMSIMD; } } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RELAXEDSIMD; + } + } #endif // Negative height means invert the image. if (height < 0) { @@ -3939,6 +4043,22 @@ int ARGBToRGB24(const uint8_t* src_argb, ARGBToRGB24Row = ARGBToRGB24Row_RVV; } #endif +#if defined(HAS_ARGBTORGB24ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToRGB24Row = ARGBToRGB24Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToRGB24Row = ARGBToRGB24Row_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBTORGB24ROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToRGB24Row = ARGBToRGB24Row_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToRGB24Row = ARGBToRGB24Row_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToRGB24Row(src_argb, dst_rgb24, width); @@ -4034,6 +4154,22 @@ int ARGBToRAW(const uint8_t* src_argb, ARGBToRAWRow = ARGBToRAWRow_RVV; } #endif +#if defined(HAS_ARGBTORAWROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToRAWRow = ARGBToRAWRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToRAWRow = ARGBToRAWRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBTORAWROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBToRAWRow = ARGBToRAWRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToRAWRow = ARGBToRAWRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToRAWRow(src_argb, dst_raw, width); @@ -4458,6 +4594,14 @@ int ABGRToAR30(const uint8_t* src_abgr, ABGRToAR30Row = ABGRToAR30Row_LASX; } } +#endif +#if defined(HAS_ABGRTOAR30ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ABGRToAR30Row = ABGRToAR30Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ABGRToAR30Row = ABGRToAR30Row_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { ABGRToAR30Row(src_abgr, dst_ar30, width); @@ -4538,6 +4682,14 @@ int ARGBToAR30(const uint8_t* src_argb, ARGBToAR30Row = ARGBToAR30Row_LASX; } } +#endif +#if defined(HAS_ARGBTOAR30ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToAR30Row = ARGBToAR30Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBToAR30Row = ARGBToAR30Row_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { ARGBToAR30Row(src_argb, dst_ar30, width); @@ -4746,6 +4898,14 @@ int ARGBToAR64(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOAR64ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToAR64Row = ARGBToAR64Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGBToAR64Row = ARGBToAR64Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToAR64Row(src_argb, dst_ar64, width); @@ -4833,6 +4993,14 @@ int ARGBToAB64(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOAB64ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToAB64Row = ARGBToAB64Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGBToAB64Row = ARGBToAB64Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToAB64Row(src_argb, dst_ab64, width); @@ -4940,6 +5108,14 @@ static int RAWToNVMatrix(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_RGBTOUVMATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -4983,6 +5159,14 @@ static int RAWToNVMatrix(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOUVMATRIXROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index 7d29fd392..567f2a80a 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -221,6 +221,14 @@ static int NV24ToARGB(const uint8_t* src_y, I444ToARGBRow = I444ToARGBRow_HVX; } #endif +#if defined(HAS_I444TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I444ToARGBRow = I444ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I444ToARGBRow = I444ToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_SPLITUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -287,6 +295,14 @@ static int NV24ToARGB(const uint8_t* src_y, SplitUVRow = SplitUVRow_HVX; } #endif +#if defined(HAS_SPLITUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitUVRow = SplitUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SplitUVRow = SplitUVRow_WASMSIMD; + } + } +#endif align_buffer_64(row_u, width); align_buffer_64(row_v, width); diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc index e1adc5fb5..17b98f4c4 100644 --- a/source/convert_to_i420.cc +++ b/source/convert_to_i420.cc @@ -126,6 +126,14 @@ static int NV16ToI420(const uint8_t* src_y, SplitUVRow = SplitUVRow_RVV; } #endif +#if defined(HAS_SPLITUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitUVRow = SplitUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + SplitUVRow = SplitUVRow_WASMSIMD; + } + } +#endif #if defined(HAS_INTERPOLATEROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_Any_AVX2; @@ -343,6 +351,14 @@ static int NV24ToI420(const uint8_t* src_y, ScaleUVRowDown2Linear = ScaleUVRowDown2Linear_HVX; } #endif +#if defined(HAS_SCALEUVROWDOWN2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleUVRowDown2Linear = ScaleUVRowDown2Linear_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 8)) { + ScaleUVRowDown2Linear = ScaleUVRowDown2Linear_WASMSIMD; + } + } +#endif #if defined(HAS_SPLITUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { SplitUVRow = SplitUVRow_Any_SSE2; @@ -408,6 +424,14 @@ static int NV24ToI420(const uint8_t* src_y, SplitUVRow = SplitUVRow_HVX; } #endif +#if defined(HAS_SPLITUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitUVRow = SplitUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + SplitUVRow = SplitUVRow_WASMSIMD; + } + } +#endif align_buffer_64(row_uv, halfwidth * 6); if (!row_uv) { diff --git a/source/cpu_id.cc b/source/cpu_id.cc index cbc2b0af0..f39ecbab4 100644 --- a/source/cpu_id.cc +++ b/source/cpu_id.cc @@ -9,6 +9,11 @@ */ #include "libyuv/cpu_id.h" +#include "libyuv/cpu_support.h" + +#if defined(__wasm__) && defined(__wasm_simd128__) +#include +#endif #if defined(_MSC_VER) #include // For __cpuidex() @@ -420,6 +425,41 @@ LIBYUV_API SAFEBUFFERS int LoongArchCpuCaps(void) { return flag; } +// For WebAssembly, but public to allow testing on any CPU. +LIBYUV_API SAFEBUFFERS int WasmCpuCaps(void) { + int flag = 0; +#if defined(__wasm__) +#if !defined(LIBYUV_DISABLE_WASMSIMD) && defined(__wasm_simd128__) + flag |= kCpuHasWASMSIMD; +#endif +#if !defined(LIBYUV_DISABLE_RELAXEDSIMD) && defined(__wasm_relaxed_simd__) + // Probe whether the runtime executes Relaxed SIMD instructions natively on + // the host CPU: + // - On x86 (SSSE3/AVX2/AVX-512), i8x16.relaxed_swizzle lowers to vpshufb, + // which masks out-of-range index 16 by 0x0f and selects lane 0 (non-zero), + // whereas deterministic i8x16.swizzle zeroes out-of-range indices. + // - On AArch64 (Neon fmla) and x86 (FMA3/AVX2 vfmadd213ps), + // f32x4.relaxed_madd(16777218.f, 3.f, 3.f) computes a single-rounded + // fused multiply-add (50331656.f), whereas unfused mul+add rounds twice + // (50331660.f). + const v128_t kNonZero = wasm_i8x16_splat(1); + const volatile v128_t kIndex16 = wasm_i8x16_splat(16); + const v128_t swizzle_diff = + wasm_v128_xor(wasm_i8x16_relaxed_swizzle(kNonZero, kIndex16), + wasm_i8x16_swizzle(kNonZero, kIndex16)); + const volatile v128_t kFmaA = wasm_f32x4_splat(16777218.0f); + const v128_t kFmaB = wasm_f32x4_splat(3.0f); + const v128_t fma_diff = + wasm_v128_xor(wasm_f32x4_relaxed_madd(kFmaA, kFmaB, kFmaB), + wasm_f32x4_add(kFmaB, wasm_f32x4_mul(kFmaA, kFmaB))); + if (wasm_v128_any_true(wasm_v128_or(swizzle_diff, fma_diff))) { + flag |= kCpuHasRELAXEDSIMD; + } +#endif +#endif // defined(__wasm__) + return flag; +} + static SAFEBUFFERS int GetCpuFlags(void) { int cpu_info = 0; #if !defined(__pnacl__) && !defined(__CLR_VER) && \ @@ -552,10 +592,8 @@ static SAFEBUFFERS int GetCpuFlags(void) { cpu_info |= kCpuHasRISCV; #endif // __riscv #if defined(__wasm__) - cpu_info = kCpuHasWASM; -#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) - cpu_info |= kCpuHasWASMSIMD; -#endif + cpu_info = WasmCpuCaps(); + cpu_info |= kCpuHasWASM; #endif // __wasm__ #if defined(__hexagon__) cpu_info = kCpuHasHexagon; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index b4f248f8b..ce4619b85 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -110,6 +110,11 @@ void CopyPlane(const uint8_t* src_y, CopyRow = CopyRow_HVX; } #endif +#if defined(HAS_COPYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + CopyRow = IS_ALIGNED(width, 64) ? CopyRow_WASMSIMD : CopyRow_Any_WASMSIMD; + } +#endif // Copy plane for (y = 0; y < height; ++y) { @@ -173,6 +178,12 @@ void CopyPlane_16(const uint16_t* src_y, CopyRow = CopyRow_16_HVX; } #endif +#if defined(HAS_COPYROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + CopyRow = + IS_ALIGNED(width, 32) ? CopyRow_16_WASMSIMD : CopyRow_16_Any_WASMSIMD; + } +#endif // Copy plane for (y = 0; y < height; ++y) { @@ -266,6 +277,14 @@ void Convert16To8Plane(const uint16_t* src_y, } } #endif +#if defined(HAS_CONVERT16TO8ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert16To8Row = Convert16To8Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + Convert16To8Row = Convert16To8Row_WASMSIMD; + } + } +#endif // Convert plane for (y = 0; y < height; ++y) { @@ -473,6 +492,14 @@ void Convert8To8Plane(const uint8_t* src_y, } } #endif +#if defined(HAS_CONVERT8TO8ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert8To8Row = Convert8To8Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + Convert8To8Row = Convert8To8Row_WASMSIMD; + } + } +#endif // Convert plane for (y = 0; y < height; ++y) { @@ -844,6 +871,14 @@ void SplitUVPlane(const uint8_t* src_uv, SplitUVRow = SplitUVRow_HVX; } #endif +#if defined(HAS_SPLITUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitUVRow = SplitUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SplitUVRow = SplitUVRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { // Copy a row of UV. @@ -1036,6 +1071,14 @@ void SplitUVPlane_16(const uint16_t* src_uv, } } #endif +#if defined(HAS_SPLITUVROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitUVRow_16 = SplitUVRow_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + SplitUVRow_16 = SplitUVRow_16_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { // Copy a row of UV. @@ -1125,6 +1168,14 @@ void MergeUVPlane_16(const uint16_t* src_u, } } #endif +#if defined(HAS_MERGEUVROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow_16 = MergeUVRow_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + MergeUVRow_16 = MergeUVRow_16_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { // Merge a row of U and V into a row of UV. @@ -1315,6 +1366,14 @@ void ConvertToLSBPlane_16(const uint16_t* src_y, } } #endif +#if defined(HAS_DIVIDEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + DivideRow = DivideRow_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + DivideRow = DivideRow_16_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { DivideRow(src_y, dst_y, scale, width); @@ -1523,6 +1582,14 @@ int DetilePlane(const uint8_t* src_y, } } #endif +#if defined(HAS_DETILEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + DetileRow = DetileRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + DetileRow = DetileRow_WASMSIMD; + } + } +#endif // Detile plane for (y = 0; y < height; ++y) { @@ -1596,6 +1663,14 @@ int DetilePlane_16(const uint16_t* src_y, } } #endif +#if defined(HAS_DETILEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + DetileRow_16 = DetileRow_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + DetileRow_16 = DetileRow_16_WASMSIMD; + } + } +#endif // Detile plane for (y = 0; y < height; ++y) { @@ -1660,6 +1735,14 @@ void DetileSplitUVPlane(const uint8_t* src_uv, } } #endif +#if defined(HAS_DETILESPLITUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + DetileSplitUVRow = DetileSplitUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + DetileSplitUVRow = DetileSplitUVRow_WASMSIMD; + } + } +#endif // Detile plane for (y = 0; y < height; ++y) { @@ -1726,6 +1809,14 @@ void DetileToYUY2(const uint8_t* src_y, } } #endif +#if defined(HAS_DETILETOYUY2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + DetileToYUY2 = DetileToYUY2_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + DetileToYUY2 = DetileToYUY2_WASMSIMD; + } + } +#endif // Detile plane for (y = 0; y < height; ++y) { @@ -1819,6 +1910,22 @@ void SplitRGBPlane(const uint8_t* src_rgb, SplitRGBRow = SplitRGBRow_RVV; } #endif +#if defined(HAS_SPLITRGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitRGBRow = SplitRGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SplitRGBRow = SplitRGBRow_WASMSIMD; + } + } +#endif +#if defined(HAS_SPLITRGBROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + SplitRGBRow = SplitRGBRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + SplitRGBRow = SplitRGBRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { // Copy a row of RGB. @@ -1889,6 +1996,22 @@ void MergeRGBPlane(const uint8_t* src_r, MergeRGBRow = MergeRGBRow_RVV; } #endif +#if defined(HAS_MERGERGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeRGBRow = MergeRGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MergeRGBRow = MergeRGBRow_WASMSIMD; + } + } +#endif +#if defined(HAS_MERGERGBROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + MergeRGBRow = MergeRGBRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + MergeRGBRow = MergeRGBRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { // Merge a row of U and V into a row of RGB. @@ -1990,6 +2113,14 @@ static void SplitARGBPlaneAlpha(const uint8_t* src_argb, SplitARGBRow = SplitARGBRow_HVX; } #endif +#if defined(HAS_SPLITARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitARGBRow = SplitARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SplitARGBRow = SplitARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { SplitARGBRow(src_argb, dst_r, dst_g, dst_b, dst_a, width); @@ -2086,6 +2217,14 @@ static void SplitARGBPlaneOpaque(const uint8_t* src_argb, SplitXRGBRow = SplitXRGBRow_HVX; } #endif +#if defined(HAS_SPLITXRGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitXRGBRow = SplitXRGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SplitXRGBRow = SplitXRGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { SplitXRGBRow(src_argb, dst_r, dst_g, dst_b, width); @@ -2218,6 +2357,14 @@ static void MergeARGBPlaneAlpha(const uint8_t* src_r, MergeARGBRow = MergeARGBRow_HVX; } #endif +#if defined(HAS_MERGEARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeARGBRow = MergeARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MergeARGBRow = MergeARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { MergeARGBRow(src_r, src_g, src_b, src_a, dst_argb, width); @@ -2306,6 +2453,14 @@ static void MergeARGBPlaneOpaque(const uint8_t* src_r, MergeXRGBRow = MergeXRGBRow_HVX; } #endif +#if defined(HAS_MERGEXRGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeXRGBRow = MergeXRGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MergeXRGBRow = MergeXRGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { MergeXRGBRow(src_r, src_g, src_b, dst_argb, width); @@ -2830,6 +2985,16 @@ int YUY2ToI422(const uint8_t* src_yuy2, YUY2ToUV422Row = YUY2ToUV422Row_HVX; } #endif +#if defined(HAS_YUY2TOYROW_WASMSIMD) && defined(HAS_YUY2TOUV422ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + YUY2ToYRow = YUY2ToYRow_Any_WASMSIMD; + YUY2ToUV422Row = YUY2ToUV422Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + YUY2ToYRow = YUY2ToYRow_WASMSIMD; + YUY2ToUV422Row = YUY2ToUV422Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { YUY2ToUV422Row(src_yuy2, dst_u, dst_v, width); @@ -2949,6 +3114,16 @@ int UYVYToI422(const uint8_t* src_uyvy, UYVYToUV422Row = UYVYToUV422Row_HVX; } #endif +#if defined(HAS_UYVYTOYROW_WASMSIMD) && defined(HAS_UYVYTOUV422ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + UYVYToYRow = UYVYToYRow_Any_WASMSIMD; + UYVYToUV422Row = UYVYToUV422Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + UYVYToYRow = UYVYToYRow_WASMSIMD; + UYVYToUV422Row = UYVYToUV422Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { UYVYToUV422Row(src_uyvy, dst_u, dst_v, width); @@ -3048,6 +3223,14 @@ int YUY2ToY(const uint8_t* src_yuy2, YUY2ToYRow = YUY2ToYRow_HVX; } #endif +#if defined(HAS_YUY2TOYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + YUY2ToYRow = YUY2ToYRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + YUY2ToYRow = YUY2ToYRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { YUY2ToYRow(src_yuy2, dst_y, width); @@ -3144,6 +3327,14 @@ int UYVYToY(const uint8_t* src_uyvy, UYVYToYRow = UYVYToYRow_HVX; } #endif +#if defined(HAS_UYVYTOYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + UYVYToYRow = UYVYToYRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + UYVYToYRow = UYVYToYRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { UYVYToYRow(src_uyvy, dst_y, width); @@ -3226,6 +3417,14 @@ void MirrorPlane(const uint8_t* src_y, } } #endif +#if defined(HAS_MIRRORROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + MirrorRow = MirrorRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_MIRRORROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { MirrorRow = MirrorRow_HVX; @@ -3334,6 +3533,17 @@ void MirrorPlane_16(const uint16_t* src_y, } } #endif +#if defined(HAS_MIRRORUVROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + MirrorRow_16 = reinterpret_cast( + MirrorUVRow_Any_RELAXEDSIMD); + if (IS_ALIGNED(width, 8)) { + MirrorRow_16 = + reinterpret_cast( + MirrorUVRow_RELAXEDSIMD); + } + } +#endif #if defined(HAS_MIRRORROW_16_HVX) if (TestCpuFlag(kCpuHasHVX)) { MirrorRow_16 = MirrorRow_16_HVX; @@ -3426,6 +3636,14 @@ void MirrorUVPlane(const uint8_t* src_uv, } } #endif +#if defined(HAS_MIRRORUVROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + MirrorUVRow = MirrorUVRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 8)) { + MirrorUVRow = MirrorUVRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_MIRRORUVROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { MirrorUVRow = MirrorUVRow_HVX; @@ -3696,6 +3914,14 @@ int RGB24Mirror(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGB24MIRRORROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RGB24MirrorRow = RGB24MirrorRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RGB24MirrorRow = RGB24MirrorRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_RGB24MIRRORROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { RGB24MirrorRow = RGB24MirrorRow_Any_LSX; @@ -3793,6 +4019,11 @@ int ARGBBlend(const uint8_t* src_argb0, ARGBBlendRow = ARGBBlendRow_WASMSIMD; } #endif +#if defined(HAS_ARGBBLENDROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBBlendRow = ARGBBlendRow_RELAXEDSIMD; + } +#endif #if defined(HAS_ARGBBLENDROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { ARGBBlendRow = ARGBBlendRow_HVX; @@ -4356,6 +4587,14 @@ int ARGBMultiply(const uint8_t* src_argb0, ARGBMultiplyRow = ARGBMultiplyRow_HVX; } #endif +#if defined(HAS_ARGBMULTIPLYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBMultiplyRow = ARGBMultiplyRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBMultiplyRow = ARGBMultiplyRow_WASMSIMD; + } + } +#endif // Multiply plane for (y = 0; y < height; ++y) { @@ -4442,6 +4681,14 @@ int ARGBAdd(const uint8_t* src_argb0, ARGBAddRow = ARGBAddRow_HVX; } #endif +#if defined(HAS_ARGBADDROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAddRow = ARGBAddRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAddRow = ARGBAddRow_WASMSIMD; + } + } +#endif // Add plane for (y = 0; y < height; ++y) { @@ -4528,6 +4775,14 @@ int ARGBSubtract(const uint8_t* src_argb0, ARGBSubtractRow = ARGBSubtractRow_HVX; } #endif +#if defined(HAS_ARGBSUBTRACTROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBSubtractRow = ARGBSubtractRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBSubtractRow = ARGBSubtractRow_WASMSIMD; + } + } +#endif // Subtract plane for (y = 0; y < height; ++y) { @@ -4750,6 +5005,14 @@ int RAWToRGB24(const uint8_t* src_raw, } } #endif +#if defined(HAS_RAWTORGB24ROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + RAWToRGB24Row = RAWToRGB24Row_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + RAWToRGB24Row = RAWToRGB24Row_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { RAWToRGB24Row(src_raw, dst_rgb24, width); @@ -4825,6 +5088,14 @@ void SetPlane(uint8_t* dst_y, SetRow = SetRow_HVX; } #endif +#if defined(HAS_SETROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SetRow = SetRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SetRow = SetRow_WASMSIMD; + } + } +#endif // Set plane for (y = 0; y < height; ++y) { @@ -4937,6 +5208,14 @@ int ARGBRect(uint8_t* dst_argb, ARGBSetRow = ARGBSetRow_RVV; } #endif +#if defined(HAS_ARGBSETROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBSetRow = ARGBSetRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBSetRow = ARGBSetRow_WASMSIMD; + } + } +#endif // Set plane for (y = 0; y < height; ++y) { @@ -5040,6 +5319,22 @@ int ARGBAttenuate(const uint8_t* src_argb, ARGBAttenuateRow = ARGBAttenuateRow_HVX; } #endif +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBAttenuateRow = ARGBAttenuateRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBAttenuateRow = ARGBAttenuateRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBAttenuateRow(src_argb, dst_argb, width); @@ -5183,6 +5478,22 @@ int ARGBGrayTo(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBGRAYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBGrayRow = ARGBGrayRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBGrayRow = ARGBGrayRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBGRAYROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBGrayRow = ARGBGrayRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBGrayRow = ARGBGrayRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBGrayRow(src_argb, dst_argb, width); @@ -5260,6 +5571,22 @@ int ARGBGray(uint8_t* dst_argb, } } #endif +#if defined(HAS_ARGBGRAYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBGrayRow = ARGBGrayRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBGrayRow = ARGBGrayRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBGRAYROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBGrayRow = ARGBGrayRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBGrayRow = ARGBGrayRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBGrayRow(dst, dst, width); @@ -5330,6 +5657,22 @@ int ARGBSepia(uint8_t* dst_argb, } } #endif +#if defined(HAS_ARGBSEPIAROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBSepiaRow = ARGBSepiaRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBSepiaRow = ARGBSepiaRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGBSEPIAROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBSepiaRow = ARGBSepiaRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBSepiaRow = ARGBSepiaRow_RELAXEDSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBSepiaRow(dst, width); @@ -5409,6 +5752,14 @@ int ARGBColorMatrix(const uint8_t* src_argb, ARGBColorMatrixRow = ARGBColorMatrixRow_LASX; } } +#endif +#if defined(HAS_ARGBCOLORMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { ARGBColorMatrixRow(src_argb, dst_argb, matrix_argb, width); @@ -5598,6 +5949,14 @@ int ARGBQuantize(uint8_t* dst_argb, ARGBQuantizeRow = ARGBQuantizeRow_LASX; } } +#endif +#if defined(HAS_ARGBQUANTIZEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBQuantizeRow = ARGBQuantizeRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBQuantizeRow = ARGBQuantizeRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { ARGBQuantizeRow(dst, scale, interval_size, interval_offset, width); @@ -5821,6 +6180,14 @@ int ARGBShade(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBSHADEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBShadeRow = ARGBShadeRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBShadeRow = ARGBShadeRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBShadeRow(src_argb, dst_argb, width, value); @@ -6178,6 +6545,14 @@ int ARGBShuffle(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBSHUFFLEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBShuffleRow = ARGBShuffleRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBShuffleRow = ARGBShuffleRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_ARGBSHUFFLEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBShuffleRow = ARGBShuffleRow_RVV; @@ -6275,6 +6650,14 @@ int AR64Shuffle(const uint16_t* src_ar64, } } #endif +#if defined(HAS_ARGBSHUFFLEROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + AR64ShuffleRow = ARGBShuffleRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 4)) { + AR64ShuffleRow = ARGBShuffleRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_ARGBSHUFFLEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { AR64ShuffleRow = ARGBShuffleRow_RVV; @@ -6509,6 +6892,11 @@ static int ARGBSobelize(const uint8_t* src_argb, SobelYRow = SobelYRow_NEON; } #endif +#if defined(HAS_SOBELYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SobelYRow = SobelYRow_WASMSIMD; + } +#endif #if defined(HAS_SOBELXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { SobelXRow = SobelXRow_AVX2; @@ -6518,6 +6906,11 @@ static int ARGBSobelize(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasNEON)) { SobelXRow = SobelXRow_NEON; } +#endif +#if defined(HAS_SOBELXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SobelXRow = SobelXRow_WASMSIMD; + } #endif { // 3 rows with edges before/after. @@ -6610,6 +7003,22 @@ int ARGBSobel(const uint8_t* src_argb, SobelRow = SobelRow_LASX; } } +#endif +#if defined(HAS_SOBELROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SobelRow = SobelRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SobelRow = SobelRow_WASMSIMD; + } + } +#endif +#if defined(HAS_SOBELROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + SobelRow = SobelRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + SobelRow = SobelRow_RELAXEDSIMD; + } + } #endif return ARGBSobelize(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width, height, SobelRow); @@ -6656,6 +7065,14 @@ int ARGBSobelToPlane(const uint8_t* src_argb, SobelToPlaneRow = SobelToPlaneRow_LASX; } } +#endif +#if defined(HAS_SOBELTOPLANEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SobelToPlaneRow = SobelToPlaneRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SobelToPlaneRow = SobelToPlaneRow_WASMSIMD; + } + } #endif return ARGBSobelize(src_argb, src_stride_argb, dst_y, dst_stride_y, width, height, SobelToPlaneRow); @@ -6703,6 +7120,14 @@ int ARGBSobelXY(const uint8_t* src_argb, SobelXYRow = SobelXYRow_LASX; } } +#endif +#if defined(HAS_SOBELXYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SobelXYRow = SobelXYRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SobelXYRow = SobelXYRow_WASMSIMD; + } + } #endif return ARGBSobelize(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width, height, SobelXYRow); @@ -6761,6 +7186,14 @@ int ARGBPolynomial(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBPOLYNOMIALROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBPolynomialRow = ARGBPolynomialRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBPolynomialRow = ARGBPolynomialRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBPolynomialRow(src_argb, dst_argb, poly, width); @@ -6865,6 +7298,14 @@ int HalfFloatPlane(const uint16_t* src_y, } } #endif +#if defined(HAS_HALFFLOATROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + HalfFloatRow = HalfFloatRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + HalfFloatRow = HalfFloatRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { HalfFloatRow(src_y, dst_y, scale, width); @@ -6898,6 +7339,14 @@ int ByteToFloat(const uint8_t* src_y, float* dst_y, float scale, int width) { } } #endif +#if defined(HAS_BYTETOFLOATROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ByteToFloatRow = ByteToFloatRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ByteToFloatRow = ByteToFloatRow_WASMSIMD; + } + } +#endif ByteToFloatRow(src_y, dst_y, scale, width); return 0; @@ -6946,6 +7395,14 @@ int ARGBLumaColorTable(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBLUMACOLORTABLEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBLumaColorTableRow = ARGBLumaColorTableRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBLumaColorTableRow = ARGBLumaColorTableRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBLumaColorTableRow(src_argb, dst_argb, width, luma, 0x00264b0f); @@ -7028,6 +7485,14 @@ int ARGBCopyAlpha(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBCOPYALPHAROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGBCopyAlphaRow = ARGBCopyAlphaRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBCopyAlphaRow(src_argb, dst_argb, width); @@ -7185,6 +7650,13 @@ int ARGBCopyYToAlpha(const uint8_t* src_y, : ARGBCopyYToAlphaRow_Any_WASMSIMD; } #endif +#if defined(HAS_ARGBCOPYYTOALPHAROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ARGBCopyYToAlphaRow = IS_ALIGNED(width, 16) + ? ARGBCopyYToAlphaRow_RELAXEDSIMD + : ARGBCopyYToAlphaRow_Any_RELAXEDSIMD; + } +#endif #if defined(HAS_ARGBCOPYYTOALPHAROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBCopyYToAlphaRow = ARGBCopyYToAlphaRow_RVV; @@ -7305,6 +7777,14 @@ int YUY2ToNV12(const uint8_t* src_yuy2, YUY2ToYRow = YUY2ToYRow_HVX; } #endif +#if defined(HAS_YUY2TOYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + YUY2ToYRow = YUY2ToYRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + YUY2ToYRow = YUY2ToYRow_WASMSIMD; + } + } +#endif #if defined(HAS_YUY2TONVUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -7347,6 +7827,14 @@ int YUY2ToNV12(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TONVUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + YUY2ToNVUVRow = YUY2ToNVUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + YUY2ToNVUVRow = YUY2ToNVUVRow_WASMSIMD; + } + } +#endif for (y = 0; y < height - 1; y += 2) { YUY2ToYRow(src_yuy2, dst_y, width); @@ -7459,6 +7947,14 @@ int UYVYToNV12(const uint8_t* src_uyvy, SplitUVRow = SplitUVRow_HVX; } #endif +#if defined(HAS_SPLITUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SplitUVRow = SplitUVRow_Any_WASMSIMD; + if (IS_ALIGNED(awidth, 16)) { + SplitUVRow = SplitUVRow_WASMSIMD; + } + } +#endif #if defined(HAS_INTERPOLATEROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { @@ -7637,6 +8133,14 @@ void HalfMergeUVPlane(const uint8_t* src_u, } } #endif +#if defined(HAS_HALFMERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + HalfMergeUVRow = HalfMergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + HalfMergeUVRow = HalfMergeUVRow_WASMSIMD; + } + } +#endif for (y = 0; y < height - 1; y += 2) { // Merge a row of U and V into a row of UV. diff --git a/source/rotate.cc b/source/rotate.cc index 5ec9b9302..a9f48d74c 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -290,6 +290,14 @@ void RotatePlane180(const uint8_t* src, } } #endif +#if defined(HAS_MIRRORROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + MirrorRow = MirrorRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_MIRRORROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { MirrorRow = MirrorRow_HVX; @@ -350,6 +358,11 @@ void RotatePlane180(const uint8_t* src, CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LASX : CopyRow_Any_LASX; } #endif +#if defined(HAS_COPYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_WASMSIMD : CopyRow_Any_WASMSIMD; + } +#endif // Odd height will harmlessly mirror the middle row twice. for (y = 0; y < half_height; ++y) { @@ -633,6 +646,14 @@ void SplitRotateUV180(const uint8_t* src, } } #endif +#if defined(HAS_MIRRORSPLITUVROW_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + MirrorSplitUVRow = MirrorSplitUVRow_Any_RELAXEDSIMD; + if (IS_ALIGNED(width, 16)) { + MirrorSplitUVRow = MirrorSplitUVRow_RELAXEDSIMD; + } + } +#endif #if defined(HAS_MIRRORSPLITUVROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { MirrorSplitUVRow = MirrorSplitUVRow_HVX; diff --git a/source/rotate_wasm.cc b/source/rotate_wasm.cc index 5a8adc95d..3827494bd 100644 --- a/source/rotate_wasm.cc +++ b/source/rotate_wasm.cc @@ -11,7 +11,7 @@ #include "libyuv/rotate_row.h" #include "libyuv/row.h" -#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#if !defined(LIBYUV_DISABLE_WASMSIMD) && defined(__wasm_simd128__) #include #ifdef __cplusplus diff --git a/source/row_any.cc b/source/row_any.cc index 480dda871..8b74ec050 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -68,6 +68,9 @@ ANY41(MergeARGBRow_Any_LSX, MergeARGBRow_LSX, 0, 0, 4, 15) #ifdef HAS_MERGEARGBROW_LASX ANY41(MergeARGBRow_Any_LASX, MergeARGBRow_LASX, 0, 0, 4, 15) #endif +#ifdef HAS_MERGEARGBROW_WASMSIMD +ANY41(MergeARGBRow_Any_WASMSIMD, MergeARGBRow_WASMSIMD, 0, 0, 4, 15) +#endif // Note that odd width replication includes 444 due to implementation // on arm that subsamples 444 to 422 internally. @@ -126,6 +129,12 @@ ANY41C(I422AlphaToARGBRow_Any_LSX, I422AlphaToARGBRow_LSX, 1, 0, 4, 7) #ifdef HAS_I422ALPHATOARGBROW_LASX ANY41C(I422AlphaToARGBRow_Any_LASX, I422AlphaToARGBRow_LASX, 1, 0, 4, 15) #endif +#ifdef HAS_I444ALPHATOARGBROW_WASMSIMD +ANY41C(I444AlphaToARGBRow_Any_WASMSIMD, I444AlphaToARGBRow_WASMSIMD, 0, 0, 4, 7) +#endif +#ifdef HAS_I422ALPHATOARGBROW_WASMSIMD +ANY41C(I422AlphaToARGBRow_Any_WASMSIMD, I422AlphaToARGBRow_WASMSIMD, 1, 0, 4, 7) +#endif #undef ANY41C // Any 4 planes to 1 plane of 8 bit with yuvconstants @@ -220,6 +229,27 @@ ANY41CT(I410AlphaToARGBRow_Any_AVX2, 15) #endif +#ifdef HAS_I210ALPHATOARGBROW_WASMSIMD +ANY41CT(I210AlphaToARGBRow_Any_WASMSIMD, + I210AlphaToARGBRow_WASMSIMD, + 1, + 0, + uint16_t, + 2, + 4, + 7) +#endif +#ifdef HAS_I410ALPHATOARGBROW_WASMSIMD +ANY41CT(I410AlphaToARGBRow_Any_WASMSIMD, + I410AlphaToARGBRow_WASMSIMD, + 0, + 0, + uint16_t, + 2, + 4, + 7) +#endif + #undef ANY41CT // Any 4 planes to 1 plane with parameter @@ -367,6 +397,21 @@ ANY31(BlendPlaneRow_Any_LASX, BlendPlaneRow_LASX, 0, 0, 1, 31) #ifdef HAS_BLENDPLANEROW_WASMSIMD ANY31(BlendPlaneRow_Any_WASMSIMD, BlendPlaneRow_WASMSIMD, 0, 0, 1, 15) #endif +#ifdef HAS_MERGERGBROW_WASMSIMD +ANY31(MergeRGBRow_Any_WASMSIMD, MergeRGBRow_WASMSIMD, 0, 0, 3, 15) +#endif +#ifdef HAS_MERGERGBROW_RELAXEDSIMD +ANY31(MergeRGBRow_Any_RELAXEDSIMD, MergeRGBRow_RELAXEDSIMD, 0, 0, 3, 15) +#endif +#ifdef HAS_MERGEXRGBROW_WASMSIMD +ANY31(MergeXRGBRow_Any_WASMSIMD, MergeXRGBRow_WASMSIMD, 0, 0, 4, 15) +#endif +#ifdef HAS_I422TOYUY2ROW_WASMSIMD +ANY31(I422ToYUY2Row_Any_WASMSIMD, I422ToYUY2Row_WASMSIMD, 1, 1, 4, 15) +#endif +#ifdef HAS_I422TOUYVYROW_WASMSIMD +ANY31(I422ToUYVYRow_Any_WASMSIMD, I422ToUYVYRow_WASMSIMD, 1, 1, 4, 15) +#endif #undef ANY31 // Note that odd width replication includes 444 due to implementation @@ -509,6 +554,21 @@ ANY31C(I422ToARGB1555Row_Any_WASMSIMD, I422ToARGB1555Row_WASMSIMD, 1, 0, 2, 7) #ifdef HAS_I422TOARGB4444ROW_WASMSIMD ANY31C(I422ToARGB4444Row_Any_WASMSIMD, I422ToARGB4444Row_WASMSIMD, 1, 0, 2, 7) #endif +#ifdef HAS_I422TORGBAROW_WASMSIMD +ANY31C(I422ToRGBARow_Any_WASMSIMD, I422ToRGBARow_WASMSIMD, 1, 0, 4, 7) +#endif +#ifdef HAS_I444TOARGBROW_WASMSIMD +ANY31C(I444ToARGBRow_Any_WASMSIMD, I444ToARGBRow_WASMSIMD, 0, 0, 4, 7) +#endif +#ifdef HAS_I444TORGB24ROW_WASMSIMD +ANY31C(I444ToRGB24Row_Any_WASMSIMD, I444ToRGB24Row_WASMSIMD, 0, 0, 3, 15) +#endif +#ifdef HAS_I444TORGB24ROW_RELAXEDSIMD +ANY31C(I444ToRGB24Row_Any_RELAXEDSIMD, I444ToRGB24Row_RELAXEDSIMD, 0, 0, 3, 15) +#endif +#ifdef HAS_I422TORGB24ROW_RELAXEDSIMD +ANY31C(I422ToRGB24Row_Any_RELAXEDSIMD, I422ToRGB24Row_RELAXEDSIMD, 1, 0, 3, 15) +#endif #undef ANY31C // Any 3 planes of 16 bit to 1 with yuvconstants @@ -592,6 +652,46 @@ ANY31CT(I212ToARGBRow_Any_NEON, I212ToARGBRow_NEON, 1, 0, uint16_t, 2, 4, 7) #ifdef HAS_I212TOAR30ROW_NEON ANY31CT(I212ToAR30Row_Any_NEON, I212ToAR30Row_NEON, 1, 0, uint16_t, 2, 4, 7) #endif +#ifdef HAS_I210TOARGBROW_WASMSIMD +ANY31CT(I210ToARGBRow_Any_WASMSIMD, + I210ToARGBRow_WASMSIMD, + 1, + 0, + uint16_t, + 2, + 4, + 7) +#endif +#ifdef HAS_I410TOARGBROW_WASMSIMD +ANY31CT(I410ToARGBRow_Any_WASMSIMD, + I410ToARGBRow_WASMSIMD, + 0, + 0, + uint16_t, + 2, + 4, + 7) +#endif +#ifdef HAS_I210TOAR30ROW_WASMSIMD +ANY31CT(I210ToAR30Row_Any_WASMSIMD, + I210ToAR30Row_WASMSIMD, + 1, + 0, + uint16_t, + 2, + 4, + 7) +#endif +#ifdef HAS_I410TOAR30ROW_WASMSIMD +ANY31CT(I410ToAR30Row_Any_WASMSIMD, + I410ToAR30Row_WASMSIMD, + 0, + 0, + uint16_t, + 2, + 4, + 7) +#endif #undef ANY31CT // Any 3 planes to 1 plane with parameter @@ -823,6 +923,33 @@ ANY21(SobelXYRow_Any_LASX, SobelXYRow_LASX, 0, 1, 1, 4, 15) #ifdef HAS_MERGEUVROW_WASMSIMD ANY21(MergeUVRow_Any_WASMSIMD, MergeUVRow_WASMSIMD, 0, 1, 1, 2, 15) #endif +#ifdef HAS_ARGBBLENDROW_WASMSIMD +ANY21(ARGBBlendRow_Any_WASMSIMD, ARGBBlendRow_WASMSIMD, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBBLENDROW_RELAXEDSIMD +ANY21(ARGBBlendRow_Any_RELAXEDSIMD, ARGBBlendRow_RELAXEDSIMD, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBMULTIPLYROW_WASMSIMD +ANY21(ARGBMultiplyRow_Any_WASMSIMD, ARGBMultiplyRow_WASMSIMD, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBADDROW_WASMSIMD +ANY21(ARGBAddRow_Any_WASMSIMD, ARGBAddRow_WASMSIMD, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBSUBTRACTROW_WASMSIMD +ANY21(ARGBSubtractRow_Any_WASMSIMD, ARGBSubtractRow_WASMSIMD, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_SOBELROW_WASMSIMD +ANY21(SobelRow_Any_WASMSIMD, SobelRow_WASMSIMD, 0, 1, 1, 4, 15) +#endif +#ifdef HAS_SOBELROW_RELAXEDSIMD +ANY21(SobelRow_Any_RELAXEDSIMD, SobelRow_RELAXEDSIMD, 0, 1, 1, 4, 15) +#endif +#ifdef HAS_SOBELTOPLANEROW_WASMSIMD +ANY21(SobelToPlaneRow_Any_WASMSIMD, SobelToPlaneRow_WASMSIMD, 0, 1, 1, 1, 15) +#endif +#ifdef HAS_SOBELXYROW_WASMSIMD +ANY21(SobelXYRow_Any_WASMSIMD, SobelXYRow_WASMSIMD, 0, 1, 1, 4, 15) +#endif #undef ANY21 // Any 2 planes to 1 with stride @@ -863,6 +990,9 @@ ANY21S(YUY2ToNVUVRow_Any_LSX, YUY2ToNVUVRow_LSX, 4, 2, 7) #ifdef HAS_YUY2TONVUVROW_LASX ANY21S(YUY2ToNVUVRow_Any_LASX, YUY2ToNVUVRow_LASX, 4, 2, 15) #endif +#ifdef HAS_YUY2TONVUVROW_WASMSIMD +ANY21S(YUY2ToNVUVRow_Any_WASMSIMD, YUY2ToNVUVRow_WASMSIMD, 4, 2, 7) +#endif #undef ANY21S // Any 2 planes to 1 half-size merged UV row with stride (2 rows of U and V). @@ -911,6 +1041,9 @@ ANY21HMS(HalfMergeUVRow_Any_LSX, HalfMergeUVRow_LSX, 15) #ifdef HAS_HALFMERGEUVROW_LASX ANY21HMS(HalfMergeUVRow_Any_LASX, HalfMergeUVRow_LASX, 31) #endif +#ifdef HAS_HALFMERGEUVROW_WASMSIMD +ANY21HMS(HalfMergeUVRow_Any_WASMSIMD, HalfMergeUVRow_WASMSIMD, 15) +#endif #undef ANY21HMS // Any 2 planes to 1 with yuvconstants @@ -1012,6 +1145,30 @@ ANY21C(NV21ToARGBRow_Any_WASMSIMD, NV21ToARGBRow_WASMSIMD, 1, 1, 2, 4, 7) #ifdef HAS_NV12TORGB565ROW_WASMSIMD ANY21C(NV12ToRGB565Row_Any_WASMSIMD, NV12ToRGB565Row_WASMSIMD, 1, 1, 2, 2, 7) #endif +#ifdef HAS_NV12TORGB24ROW_WASMSIMD +ANY21C(NV12ToRGB24Row_Any_WASMSIMD, NV12ToRGB24Row_WASMSIMD, 1, 1, 2, 3, 15) +#endif +#ifdef HAS_NV12TORGB24ROW_RELAXEDSIMD +ANY21C(NV12ToRGB24Row_Any_RELAXEDSIMD, + NV12ToRGB24Row_RELAXEDSIMD, + 1, + 1, + 2, + 3, + 15) +#endif +#ifdef HAS_NV21TORGB24ROW_WASMSIMD +ANY21C(NV21ToRGB24Row_Any_WASMSIMD, NV21ToRGB24Row_WASMSIMD, 1, 1, 2, 3, 15) +#endif +#ifdef HAS_NV21TORGB24ROW_RELAXEDSIMD +ANY21C(NV21ToRGB24Row_Any_RELAXEDSIMD, + NV21ToRGB24Row_RELAXEDSIMD, + 1, + 1, + 2, + 3, + 15) +#endif #undef ANY21C // Any 2 planes of 16 bit to 1 with yuvconstants @@ -1113,6 +1270,9 @@ ANY21PT(MergeUVRow_16_Any_LSX, MergeUVRow_16_LSX, uint16_t, 2, 7) #ifdef HAS_MERGEUVROW_16_LASX ANY21PT(MergeUVRow_16_Any_LASX, MergeUVRow_16_LASX, uint16_t, 2, 15) #endif +#ifdef HAS_MERGEUVROW_16_WASMSIMD +ANY21PT(MergeUVRow_16_Any_WASMSIMD, MergeUVRow_16_WASMSIMD, uint16_t, 2, 7) +#endif #undef ANY21CT @@ -1549,6 +1709,80 @@ ANY11(ARGBGrayRow_Any_LSX, ARGBGrayRow_LSX, 0, 4, 4, 7) #ifdef HAS_ARGBGRAYROW_LASX ANY11(ARGBGrayRow_Any_LASX, ARGBGrayRow_LASX, 0, 4, 4, 15) #endif +#ifdef HAS_COPYROW_WASMSIMD +ANY11(CopyRow_Any_WASMSIMD, CopyRow_WASMSIMD, 0, 1, 1, 63) +#endif +#ifdef HAS_ARGBTORGB24ROW_WASMSIMD +ANY11(ARGBToRGB24Row_Any_WASMSIMD, ARGBToRGB24Row_WASMSIMD, 0, 4, 3, 15) +#endif +#ifdef HAS_ARGBTORGB24ROW_RELAXEDSIMD +ANY11(ARGBToRGB24Row_Any_RELAXEDSIMD, ARGBToRGB24Row_RELAXEDSIMD, 0, 4, 3, 15) +#endif +#ifdef HAS_ARGBTORAWROW_WASMSIMD +ANY11(ARGBToRAWRow_Any_WASMSIMD, ARGBToRAWRow_WASMSIMD, 0, 4, 3, 15) +#endif +#ifdef HAS_ARGBTORAWROW_RELAXEDSIMD +ANY11(ARGBToRAWRow_Any_RELAXEDSIMD, ARGBToRAWRow_RELAXEDSIMD, 0, 4, 3, 15) +#endif +#ifdef HAS_RAWTORGBAROW_WASMSIMD +ANY11(RAWToRGBARow_Any_WASMSIMD, RAWToRGBARow_WASMSIMD, 0, 3, 4, 15) +#endif +#ifdef HAS_RAWTORGBAROW_RELAXEDSIMD +ANY11(RAWToRGBARow_Any_RELAXEDSIMD, RAWToRGBARow_RELAXEDSIMD, 0, 3, 4, 15) +#endif +#ifdef HAS_RAWTOARGBROW_RELAXEDSIMD +ANY11(RAWToARGBRow_Any_RELAXEDSIMD, RAWToARGBRow_RELAXEDSIMD, 0, 3, 4, 15) +#endif +#ifdef HAS_RGB24TOARGBROW_RELAXEDSIMD +ANY11(RGB24ToARGBRow_Any_RELAXEDSIMD, RGB24ToARGBRow_RELAXEDSIMD, 0, 3, 4, 15) +#endif +#ifdef HAS_RAWTORGB24ROW_RELAXEDSIMD +ANY11(RAWToRGB24Row_Any_RELAXEDSIMD, RAWToRGB24Row_RELAXEDSIMD, 0, 3, 3, 15) +#endif +#ifdef HAS_J400TOARGBROW_RELAXEDSIMD +ANY11(J400ToARGBRow_Any_RELAXEDSIMD, J400ToARGBRow_RELAXEDSIMD, 0, 1, 4, 15) +#endif +#ifdef HAS_AR30TOARGBROW_WASMSIMD +ANY11(AR30ToARGBRow_Any_WASMSIMD, AR30ToARGBRow_WASMSIMD, 0, 4, 4, 3) +#endif +#ifdef HAS_AR30TOABGRROW_WASMSIMD +ANY11(AR30ToABGRRow_Any_WASMSIMD, AR30ToABGRRow_WASMSIMD, 0, 4, 4, 3) +#endif +#ifdef HAS_AR30TOAB30ROW_WASMSIMD +ANY11(AR30ToAB30Row_Any_WASMSIMD, AR30ToAB30Row_WASMSIMD, 0, 4, 4, 3) +#endif +#ifdef HAS_ARGBTOAR30ROW_WASMSIMD +ANY11(ARGBToAR30Row_Any_WASMSIMD, ARGBToAR30Row_WASMSIMD, 0, 4, 4, 3) +#endif +#ifdef HAS_ABGRTOAR30ROW_WASMSIMD +ANY11(ABGRToAR30Row_Any_WASMSIMD, ABGRToAR30Row_WASMSIMD, 0, 4, 4, 3) +#endif +#ifdef HAS_YUY2TOYROW_WASMSIMD +ANY11(YUY2ToYRow_Any_WASMSIMD, YUY2ToYRow_WASMSIMD, 1, 4, 1, 15) +#endif +#ifdef HAS_UYVYTOYROW_WASMSIMD +ANY11(UYVYToYRow_Any_WASMSIMD, UYVYToYRow_WASMSIMD, 1, 4, 1, 15) +#endif +#ifdef HAS_AYUVTOYROW_WASMSIMD +ANY11(AYUVToYRow_Any_WASMSIMD, AYUVToYRow_WASMSIMD, 0, 4, 1, 15) +#endif +#ifdef HAS_ARGBATTENUATEROW_WASMSIMD +ANY11(ARGBAttenuateRow_Any_WASMSIMD, ARGBAttenuateRow_WASMSIMD, 0, 4, 4, 3) +#endif +#ifdef HAS_ARGBATTENUATEROW_RELAXEDSIMD +ANY11(ARGBAttenuateRow_Any_RELAXEDSIMD, + ARGBAttenuateRow_RELAXEDSIMD, + 0, + 4, + 4, + 3) +#endif +#ifdef HAS_ARGBGRAYROW_WASMSIMD +ANY11(ARGBGrayRow_Any_WASMSIMD, ARGBGrayRow_WASMSIMD, 0, 4, 4, 3) +#endif +#ifdef HAS_ARGBGRAYROW_RELAXEDSIMD +ANY11(ARGBGrayRow_Any_RELAXEDSIMD, ARGBGrayRow_RELAXEDSIMD, 0, 4, 4, 3) +#endif #undef ANY11 // Any 1 to 1 blended. Destination is read, modify, write. @@ -1611,6 +1845,17 @@ ANY11B(ARGBCopyYToAlphaRow_Any_WASMSIMD, 4, 15) #endif +#ifdef HAS_ARGBCOPYYTOALPHAROW_RELAXEDSIMD +ANY11B(ARGBCopyYToAlphaRow_Any_RELAXEDSIMD, + ARGBCopyYToAlphaRow_RELAXEDSIMD, + 0, + 1, + 4, + 15) +#endif +#ifdef HAS_ARGBCOPYALPHAROW_WASMSIMD +ANY11B(ARGBCopyAlphaRow_Any_WASMSIMD, ARGBCopyAlphaRow_WASMSIMD, 0, 4, 4, 7) +#endif #undef ANY11B // Any 1 to 1 with parameter. @@ -1809,6 +2054,46 @@ ANY11P(ARGBColorMatrixRow_Any_LASX, 4, 7) #endif +#ifdef HAS_I400TOARGBROW_WASMSIMD +ANY11P(I400ToARGBRow_Any_WASMSIMD, + I400ToARGBRow_WASMSIMD, + const struct YuvConstants*, + 1, + 4, + 15) +#endif +#ifdef HAS_I400TOARGBROW_RELAXEDSIMD +ANY11P(I400ToARGBRow_Any_RELAXEDSIMD, + I400ToARGBRow_RELAXEDSIMD, + const struct YuvConstants*, + 1, + 4, + 15) +#endif +#ifdef HAS_ARGBSHUFFLEROW_RELAXEDSIMD +ANY11P(ARGBShuffleRow_Any_RELAXEDSIMD, + ARGBShuffleRow_RELAXEDSIMD, + const uint8_t*, + 4, + 4, + 3) +#endif +#ifdef HAS_ARGBPOLYNOMIALROW_WASMSIMD +ANY11P(ARGBPolynomialRow_Any_WASMSIMD, + ARGBPolynomialRow_WASMSIMD, + const float*, + 4, + 4, + 3) +#endif +#ifdef HAS_ARGBCOLORMATRIXROW_WASMSIMD +ANY11P(ARGBColorMatrixRow_Any_WASMSIMD, + ARGBColorMatrixRow_WASMSIMD, + const int8_t*, + 4, + 4, + 3) +#endif #undef ANY11P // Any 1 in-place. @@ -1839,6 +2124,12 @@ ANY1I(ARGBSepiaRow_Any_LSX, ARGBSepiaRow_LSX, 4, 7) #ifdef HAS_ARGBSEPIAROW_LASX ANY1I(ARGBSepiaRow_Any_LASX, ARGBSepiaRow_LASX, 4, 15) #endif +#ifdef HAS_ARGBSEPIAROW_WASMSIMD +ANY1I(ARGBSepiaRow_Any_WASMSIMD, ARGBSepiaRow_WASMSIMD, 4, 3) +#endif +#ifdef HAS_ARGBSEPIAROW_RELAXEDSIMD +ANY1I(ARGBSepiaRow_Any_RELAXEDSIMD, ARGBSepiaRow_RELAXEDSIMD, 4, 3) +#endif #undef ANY1I // Any 1 in-place with 3 int parameters. @@ -1866,6 +2157,9 @@ ANY1I3(ARGBQuantizeRow_Any_LSX, ARGBQuantizeRow_LSX, 4, 3) #ifdef HAS_ARGBQUANTIZEROW_LASX ANY1I3(ARGBQuantizeRow_Any_LASX, ARGBQuantizeRow_LASX, 4, 7) #endif +#ifdef HAS_ARGBQUANTIZEROW_WASMSIMD +ANY1I3(ARGBQuantizeRow_Any_WASMSIMD, ARGBQuantizeRow_WASMSIMD, 4, 3) +#endif #undef ANY1I3 // Any 1 to 1 with trailing parameter. @@ -1895,6 +2189,9 @@ ANY11PT(ARGBShadeRow_Any_LSX, ARGBShadeRow_LSX, uint32_t, 4, 4, 3) #ifdef HAS_ARGBSHADEROW_LASX ANY11PT(ARGBShadeRow_Any_LASX, ARGBShadeRow_LASX, uint32_t, 4, 4, 7) #endif +#ifdef HAS_ARGBSHADEROW_WASMSIMD +ANY11PT(ARGBShadeRow_Any_WASMSIMD, ARGBShadeRow_WASMSIMD, uint32_t, 4, 4, 3) +#endif #undef ANY11PT #ifdef HAS_ARGBLUMACOLORTABLEROW_NEON @@ -1918,6 +2215,27 @@ void ARGBLumaColorTableRow_Any_NEON(const uint8_t* src_argb, memcpy(dst_argb + np * 4, vout, r * 4); } #endif +#ifdef HAS_ARGBLUMACOLORTABLEROW_WASMSIMD +void ARGBLumaColorTableRow_Any_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + const uint32_t lumacoeff) { + SIMD_ALIGNED(uint8_t vin[4 * 4]); + SIMD_ALIGNED(uint8_t vout[4 * 4]); + MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ + int r = width & 3; + int n = width & ~3; + if (n > 0) { + ARGBLumaColorTableRow_WASMSIMD(src_argb, dst_argb, n, luma, lumacoeff); + } + ptrdiff_t np = n; + memcpy(vin, src_argb + np * 4, r * 4); + ARGBLumaColorTableRow_WASMSIMD(vin, vout, 4, luma, lumacoeff); + memcpy(dst_argb + np * 4, vout, r * 4); +} +#endif // Any 1 to 1 with type #define ANY11T(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \ @@ -2021,6 +2339,51 @@ ANY11T(CopyRow_16_Any_LSX, CopyRow_16_LSX, 2, 2, uint16_t, uint16_t, 15) #ifdef HAS_COPYROW_16_LASX ANY11T(CopyRow_16_Any_LASX, CopyRow_16_LASX, 2, 2, uint16_t, uint16_t, 31) #endif +#ifdef HAS_ARGBTOAR64ROW_WASMSIMD +ANY11T(ARGBToAR64Row_Any_WASMSIMD, + ARGBToAR64Row_WASMSIMD, + 4, + 8, + uint8_t, + uint16_t, + 7) +#endif +#ifdef HAS_ARGBTOAB64ROW_WASMSIMD +ANY11T(ARGBToAB64Row_Any_WASMSIMD, + ARGBToAB64Row_WASMSIMD, + 4, + 8, + uint8_t, + uint16_t, + 7) +#endif +#ifdef HAS_AR64TOARGBROW_WASMSIMD +ANY11T(AR64ToARGBRow_Any_WASMSIMD, + AR64ToARGBRow_WASMSIMD, + 8, + 4, + uint16_t, + uint8_t, + 7) +#endif +#ifdef HAS_AB64TOARGBROW_WASMSIMD +ANY11T(AB64ToARGBRow_Any_WASMSIMD, + AB64ToARGBRow_WASMSIMD, + 8, + 4, + uint16_t, + uint8_t, + 7) +#endif +#ifdef HAS_COPYROW_16_WASMSIMD +ANY11T(CopyRow_16_Any_WASMSIMD, + CopyRow_16_WASMSIMD, + 2, + 2, + uint16_t, + uint16_t, + 31) +#endif #undef ANY11T @@ -2200,6 +2563,24 @@ ANY11C(MultiplyRow_16_Any_WASMSIMD, uint16_t, 15) #endif +#ifdef HAS_CONVERT16TO8ROW_WASMSIMD +ANY11C(Convert16To8Row_Any_WASMSIMD, + Convert16To8Row_WASMSIMD, + 2, + 1, + uint16_t, + uint8_t, + 15) +#endif +#ifdef HAS_DIVIDEROW_16_WASMSIMD +ANY11C(DivideRow_16_Any_WASMSIMD, + DivideRow_16_WASMSIMD, + 2, + 2, + uint16_t, + uint16_t, + 15) +#endif #undef ANY11C // Any 1 to 1 with parameter and shorts. BPP measures in shorts. @@ -2259,6 +2640,15 @@ ANY11SB(Convert8To8Row_Any_LASX, uint8_t, 31) #endif +#ifdef HAS_CONVERT8TO8ROW_WASMSIMD +ANY11SB(Convert8To8Row_Any_WASMSIMD, + Convert8To8Row_WASMSIMD, + 1, + 1, + uint8_t, + uint8_t, + 15) +#endif #undef ANY11B // Any 1 to 1 with parameter and shorts to byte. BPP measures in shorts. @@ -2309,6 +2699,24 @@ ANY11P16(HalfFloatRow_Any_LSX, HalfFloatRow_LSX, uint16_t, uint16_t, 2, 2, 7) #ifdef HAS_HALFFLOATROW_LASX ANY11P16(HalfFloatRow_Any_LASX, HalfFloatRow_LASX, uint16_t, uint16_t, 2, 2, 15) #endif +#ifdef HAS_HALFFLOATROW_WASMSIMD +ANY11P16(HalfFloatRow_Any_WASMSIMD, + HalfFloatRow_WASMSIMD, + uint16_t, + uint16_t, + 2, + 2, + 15) +#endif +#ifdef HAS_BYTETOFLOATROW_WASMSIMD +ANY11P16(ByteToFloatRow_Any_WASMSIMD, + ByteToFloatRow_WASMSIMD, + uint8_t, + float, + 1, + 4, + 15) +#endif #undef ANY11P16 // Any 1 to 1 with yuvconstants @@ -2353,6 +2761,10 @@ ANY11C(UYVYToARGBRow_Any_LSX, UYVYToARGBRow_LSX, 1, 4, 4, 7) ANY11C(YUY2ToARGBRow_Any_LASX, YUY2ToARGBRow_LASX, 1, 4, 4, 15) ANY11C(UYVYToARGBRow_Any_LASX, UYVYToARGBRow_LASX, 1, 4, 4, 15) #endif +#if defined(HAS_YUY2TOARGBROW_WASMSIMD) +ANY11C(YUY2ToARGBRow_Any_WASMSIMD, YUY2ToARGBRow_WASMSIMD, 1, 4, 4, 7) +ANY11C(UYVYToARGBRow_Any_WASMSIMD, UYVYToARGBRow_WASMSIMD, 1, 4, 4, 7) +#endif #undef ANY11C // Any 1 to 1 interpolate. Takes 2 rows of source via stride. @@ -2493,6 +2905,12 @@ ANY11H(HalfRow_16_Any_LSX, HalfRow_16_LSX, uint16_t, 7) #ifdef HAS_HALFROW_16_LASX ANY11H(HalfRow_16_Any_LASX, HalfRow_16_LASX, uint16_t, 15) #endif +#ifdef HAS_HALFROW_WASMSIMD +ANY11H(HalfRow_Any_WASMSIMD, HalfRow_WASMSIMD, uint8_t, 15) +#endif +#ifdef HAS_HALFROW_16_WASMSIMD +ANY11H(HalfRow_16_Any_WASMSIMD, HalfRow_16_WASMSIMD, uint16_t, 7) +#endif #undef ANY11H @@ -2535,6 +2953,13 @@ ANY11HS(HalfRow_16To8_Any_LSX, HalfRow_16To8_LSX, uint16_t, uint8_t, 15) #ifdef HAS_HALFROW_16TO8_LASX ANY11HS(HalfRow_16To8_Any_LASX, HalfRow_16To8_LASX, uint16_t, uint8_t, 31) #endif +#ifdef HAS_HALFROW_16TO8_WASMSIMD +ANY11HS(HalfRow_16To8_Any_WASMSIMD, + HalfRow_16To8_WASMSIMD, + uint16_t, + uint8_t, + 15) +#endif #undef ANY11HS @@ -2602,6 +3027,13 @@ void HalfWidthRow_16To8_Odd_NEON(const uint16_t* src_uv, dst_uv[0] = (uint8_t)((avg * (uint32_t)scale) >> 17); } #endif +#ifdef HAS_HALFWIDTHROW_16TO8_WASMSIMD +ANY11HWS(HalfWidthRow_16To8_Any_WASMSIMD, + HalfWidthRow_16To8_WASMSIMD, + uint16_t, + uint8_t, + 7) +#endif #undef ANY11HWS @@ -2697,6 +3129,15 @@ ANY11M(ARGBMirrorRow_Any_WASMSIMD, ARGBMirrorRow_WASMSIMD, 4, 3) #ifdef HAS_RGB24MIRRORROW_WASMSIMD ANY11M(RGB24MirrorRow_Any_WASMSIMD, RGB24MirrorRow_WASMSIMD, 3, 15) #endif +#ifdef HAS_MIRRORROW_RELAXEDSIMD +ANY11M(MirrorRow_Any_RELAXEDSIMD, MirrorRow_RELAXEDSIMD, 1, 15) +#endif +#ifdef HAS_MIRRORUVROW_RELAXEDSIMD +ANY11M(MirrorUVRow_Any_RELAXEDSIMD, MirrorUVRow_RELAXEDSIMD, 2, 7) +#endif +#ifdef HAS_RGB24MIRRORROW_RELAXEDSIMD +ANY11M(RGB24MirrorRow_Any_RELAXEDSIMD, RGB24MirrorRow_RELAXEDSIMD, 3, 15) +#endif #undef ANY11M // Any 1 to 1 mirror with type. @@ -2764,6 +3205,9 @@ ANY12MR(MirrorSplitUVRow_Any_LASX, MirrorSplitUVRow_LASX, 31) #ifdef HAS_MIRRORSPLITUVROW_WASMSIMD ANY12MR(MirrorSplitUVRow_Any_WASMSIMD, MirrorSplitUVRow_WASMSIMD, 15) #endif +#ifdef HAS_MIRRORSPLITUVROW_RELAXEDSIMD +ANY12MR(MirrorSplitUVRow_Any_RELAXEDSIMD, MirrorSplitUVRow_RELAXEDSIMD, 15) +#endif #undef ANY12MR // Any 1 plane. (memset) @@ -2794,6 +3238,9 @@ ANY1(SetRow_Any_LSX, SetRow_LSX, uint8_t, 1, 15) #ifdef HAS_SETROW_LASX ANY1(SetRow_Any_LASX, SetRow_LASX, uint8_t, 1, 31) #endif +#ifdef HAS_SETROW_WASMSIMD +ANY1(SetRow_Any_WASMSIMD, SetRow_WASMSIMD, uint8_t, 1, 15) +#endif #ifdef HAS_ARGBSETROW_AVX2 ANY1(ARGBSetRow_Any_AVX2, ARGBSetRow_AVX2, uint32_t, 4, 7) #endif @@ -2806,6 +3253,9 @@ ANY1(ARGBSetRow_Any_LSX, ARGBSetRow_LSX, uint32_t, 4, 3) #ifdef HAS_ARGBSETROW_LASX ANY1(ARGBSetRow_Any_LASX, ARGBSetRow_LASX, uint32_t, 4, 7) #endif +#ifdef HAS_ARGBSETROW_WASMSIMD +ANY1(ARGBSetRow_Any_WASMSIMD, ARGBSetRow_WASMSIMD, uint32_t, 4, 3) +#endif #undef ANY1 // Any 1 to 2. Outputs UV planes. @@ -2869,6 +3319,15 @@ ANY12(UYVYToUV422Row_Any_LSX, UYVYToUV422Row_LSX, 1, 4, 1, 15) ANY12(YUY2ToUV422Row_Any_LASX, YUY2ToUV422Row_LASX, 1, 4, 1, 31) ANY12(UYVYToUV422Row_Any_LASX, UYVYToUV422Row_LASX, 1, 4, 1, 31) #endif +#ifdef HAS_SPLITUVROW_WASMSIMD +ANY12(SplitUVRow_Any_WASMSIMD, SplitUVRow_WASMSIMD, 0, 2, 0, 15) +#endif +#ifdef HAS_YUY2TOUV422ROW_WASMSIMD +ANY12(YUY2ToUV422Row_Any_WASMSIMD, YUY2ToUV422Row_WASMSIMD, 1, 4, 1, 15) +#endif +#ifdef HAS_UYVYTOUV422ROW_WASMSIMD +ANY12(UYVYToUV422Row_Any_WASMSIMD, UYVYToUV422Row_WASMSIMD, 1, 4, 1, 15) +#endif #undef ANY12 // Any 2 16 bit planes with parameter to 1 @@ -2907,6 +3366,9 @@ ANY12PT(SplitUVRow_16_Any_LSX, SplitUVRow_16_LSX, uint16_t, 2, 7) #ifdef HAS_SPLITUVROW_16_LASX ANY12PT(SplitUVRow_16_Any_LASX, SplitUVRow_16_LASX, uint16_t, 2, 15) #endif +#ifdef HAS_SPLITUVROW_16_WASMSIMD +ANY12PT(SplitUVRow_16_Any_WASMSIMD, SplitUVRow_16_WASMSIMD, uint16_t, 2, 7) +#endif #undef ANY21CT @@ -2961,6 +3423,15 @@ ANY13(SplitXRGBRow_Any_LSX, SplitXRGBRow_LSX, 4, 15) #ifdef HAS_SPLITXRGBROW_LASX ANY13(SplitXRGBRow_Any_LASX, SplitXRGBRow_LASX, 4, 15) #endif +#ifdef HAS_SPLITRGBROW_WASMSIMD +ANY13(SplitRGBRow_Any_WASMSIMD, SplitRGBRow_WASMSIMD, 3, 15) +#endif +#ifdef HAS_SPLITRGBROW_RELAXEDSIMD +ANY13(SplitRGBRow_Any_RELAXEDSIMD, SplitRGBRow_RELAXEDSIMD, 3, 15) +#endif +#ifdef HAS_SPLITXRGBROW_WASMSIMD +ANY13(SplitXRGBRow_Any_WASMSIMD, SplitXRGBRow_WASMSIMD, 4, 15) +#endif // Any 1 to 4. Outputs ARGB planes. #define ANY14(NAMEANY, ANY_SIMD, BPP, MASK) \ @@ -3004,6 +3475,9 @@ ANY14(SplitARGBRow_Any_LSX, SplitARGBRow_LSX, 4, 15) #ifdef HAS_SPLITARGBROW_LASX ANY14(SplitARGBRow_Any_LASX, SplitARGBRow_LASX, 4, 15) #endif +#ifdef HAS_SPLITARGBROW_WASMSIMD +ANY14(SplitARGBRow_Any_WASMSIMD, SplitARGBRow_WASMSIMD, 4, 15) +#endif // Any 1 to 2 with source stride (2 rows of source). Outputs UV planes. // 128 byte row allows for 32 avx ARGB pixels. @@ -3179,6 +3653,32 @@ ANY12M(ARGBToUV444MatrixRow_Any_WASMSIMD, ARGBToUV444MatrixRow_WASMSIMD, 4, 15) #ifdef HAS_RGBTOUV444MATRIXROW_WASMSIMD ANY12M(RGBToUV444MatrixRow_Any_WASMSIMD, RGBToUV444MatrixRow_WASMSIMD, 3, 15) #endif +#ifdef HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD +ANY12MS(ARGBToUVMatrixRow_Any_RELAXEDSIMD, + ARGBToUVMatrixRow_RELAXEDSIMD, + 0, + 4, + 15) +#endif +#ifdef HAS_RGBTOUVMATRIXROW_RELAXEDSIMD +ANY12MS(RGBToUVMatrixRow_Any_RELAXEDSIMD, + RGBToUVMatrixRow_RELAXEDSIMD, + 0, + 3, + 15) +#endif +#ifdef HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD +ANY12M(ARGBToUV444MatrixRow_Any_RELAXEDSIMD, + ARGBToUV444MatrixRow_RELAXEDSIMD, + 4, + 15) +#endif +#ifdef HAS_RGBTOUV444MATRIXROW_RELAXEDSIMD +ANY12M(RGBToUV444MatrixRow_Any_RELAXEDSIMD, + RGBToUV444MatrixRow_RELAXEDSIMD, + 3, + 15) +#endif #define ANY11MC(NAMEANY, ANY_SIMD, BPP, MASK) \ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width, \ @@ -3261,6 +3761,9 @@ ANY11MC(ARGB1555ToYMatrixRow_Any_WASMSIMD, ARGB1555ToYMatrixRow_WASMSIMD, 2, 15) #ifdef HAS_ARGB4444TOYMATRIXROW_WASMSIMD ANY11MC(ARGB4444ToYMatrixRow_Any_WASMSIMD, ARGB4444ToYMatrixRow_WASMSIMD, 2, 15) #endif +#ifdef HAS_RGBTOYMATRIXROW_RELAXEDSIMD +ANY11MC(RGBToYMatrixRow_Any_RELAXEDSIMD, RGBToYMatrixRow_RELAXEDSIMD, 3, 15) +#endif #undef ANY11MC #ifdef HAS_YUY2TOUVROW_AVX2 @@ -3340,6 +3843,12 @@ ANY12S(UYVYToUVRow_Any_LSX, UYVYToUVRow_LSX, 1, 4, 15) #ifdef HAS_UYVYTOUVROW_LASX ANY12S(UYVYToUVRow_Any_LASX, UYVYToUVRow_LASX, 1, 4, 31) #endif +#ifdef HAS_YUY2TOUVROW_WASMSIMD +ANY12S(YUY2ToUVRow_Any_WASMSIMD, YUY2ToUVRow_WASMSIMD, 1, 4, 15) +#endif +#ifdef HAS_UYVYTOUVROW_WASMSIMD +ANY12S(UYVYToUVRow_Any_WASMSIMD, UYVYToUVRow_WASMSIMD, 1, 4, 15) +#endif #undef ANY12S // Any 1 to 1 with source stride (2 rows of source). Outputs UV plane. @@ -3396,6 +3905,18 @@ ANY11S(AYUVToVURow_Any_LSX, AYUVToVURow_LSX, 0, 4, 15) #ifdef HAS_AYUVTOVUROW_LASX ANY11S(AYUVToVURow_Any_LASX, AYUVToVURow_LASX, 0, 4, 15) #endif +#ifdef HAS_AYUVTOUVROW_WASMSIMD +ANY11S(AYUVToUVRow_Any_WASMSIMD, AYUVToUVRow_WASMSIMD, 0, 4, 15) +#endif +#ifdef HAS_AYUVTOUVROW_RELAXEDSIMD +ANY11S(AYUVToUVRow_Any_RELAXEDSIMD, AYUVToUVRow_RELAXEDSIMD, 0, 4, 15) +#endif +#ifdef HAS_AYUVTOVUROW_WASMSIMD +ANY11S(AYUVToVURow_Any_WASMSIMD, AYUVToVURow_WASMSIMD, 0, 4, 15) +#endif +#ifdef HAS_AYUVTOVUROW_RELAXEDSIMD +ANY11S(AYUVToVURow_Any_RELAXEDSIMD, AYUVToVURow_RELAXEDSIMD, 0, 4, 15) +#endif #undef ANY11S #define ANYDETILE(NAMEANY, ANY_SIMD, T, BPP, MASK) \ @@ -3441,6 +3962,12 @@ ANYDETILE(DetileRow_16_Any_LSX, DetileRow_16_LSX, uint16_t, 2, 15) #ifdef HAS_DETILEROW_16_LASX ANYDETILE(DetileRow_16_Any_LASX, DetileRow_16_LASX, uint16_t, 2, 15) #endif +#ifdef HAS_DETILEROW_WASMSIMD +ANYDETILE(DetileRow_Any_WASMSIMD, DetileRow_WASMSIMD, uint8_t, 1, 15) +#endif +#ifdef HAS_DETILEROW_16_WASMSIMD +ANYDETILE(DetileRow_16_Any_WASMSIMD, DetileRow_16_WASMSIMD, uint16_t, 2, 15) +#endif // DetileSplitUVRow width is in bytes #define ANYDETILESPLITUV(NAMEANY, ANY_SIMD, MASK) \ @@ -3471,6 +3998,9 @@ ANYDETILESPLITUV(DetileSplitUVRow_Any_NEON, DetileSplitUVRow_NEON, 15) #ifdef HAS_DETILESPLITUVROW_AVX2 ANYDETILESPLITUV(DetileSplitUVRow_Any_AVX2, DetileSplitUVRow_AVX2, 15) #endif +#ifdef HAS_DETILESPLITUVROW_WASMSIMD +ANYDETILESPLITUV(DetileSplitUVRow_Any_WASMSIMD, DetileSplitUVRow_WASMSIMD, 15) +#endif #define ANYDETILEMERGE(NAMEANY, ANY_SIMD, MASK) \ void NAMEANY(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, \ @@ -3508,6 +4038,10 @@ ANYDETILEMERGE(DetileToYUY2_Any_NEON, DetileToYUY2_NEON, 15) ANYDETILEMERGE(DetileToYUY2_Any_AVX2, DetileToYUY2_AVX2, 31) #endif +#ifdef HAS_DETILETOYUY2_WASMSIMD +ANYDETILEMERGE(DetileToYUY2_Any_WASMSIMD, DetileToYUY2_WASMSIMD, 15) +#endif + #undef ANYDETILEMERGE #undef MSAN_MEMSET diff --git a/source/row_wasm.cc b/source/row_wasm.cc index df9e3917a..0ebc2326a 100644 --- a/source/row_wasm.cc +++ b/source/row_wasm.cc @@ -10,19 +10,24 @@ #include "libyuv/row.h" -#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#if (!defined(LIBYUV_DISABLE_WASMSIMD) || \ + !defined(LIBYUV_DISABLE_RELAXEDSIMD)) && \ + defined(__wasm_simd128__) #include #ifdef __cplusplus namespace libyuv { -extern "C" { #endif -#ifdef __wasm_relaxed_simd__ -#define LIBYUV_WASM_SWIZZLE wasm_i8x16_relaxed_swizzle -#else -#define LIBYUV_WASM_SWIZZLE wasm_i8x16_swizzle +template +static inline v128_t WasmSwizzle(v128_t v, v128_t m) { +#if !defined(LIBYUV_DISABLE_RELAXEDSIMD) && defined(__wasm_relaxed_simd__) + if (kRelaxed) { + return wasm_i8x16_relaxed_swizzle(v, m); + } #endif + return wasm_i8x16_swizzle(v, m); +} // V8 materializes v128.const values and arbitrary i8x16.shuffle masks with // several instructions at every use, including inside loops. These helpers @@ -50,7 +55,8 @@ static inline v128_t SplatConst32(int v) { return wasm_i32x4_splat(v); } -#ifdef HAS_RAWTORGB24ROW_WASMSIMD +#if defined(HAS_RAWTORGB24ROW_WASMSIMD) || \ + defined(HAS_RAWTORGB24ROW_RELAXEDSIMD) static const uint8_t kRawToRGB24SwizzleTable[96] = { 2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, @@ -60,9 +66,10 @@ static const uint8_t kRawToRGB24SwizzleTable[96] = { 11, 10, 15, 14, 13, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}; -void RAWToRGB24Row_WASMSIMD(const uint8_t* src_raw, - uint8_t* dst_rgb24, - int width) { +template +static inline void RAWToRGB24Row_Wasm(const uint8_t* src_raw, + uint8_t* dst_rgb24, + int width) { const uint8_t* shuf = kRawToRGB24SwizzleTable; // Load shuffle masks into Wasm locals outside the loop so Binaryen's // precompute-propagate pass does not sink v128.const into the loop. @@ -90,18 +97,18 @@ void RAWToRGB24Row_WASMSIMD(const uint8_t* src_raw, v128_t s35 = wasm_i8x16_shuffle(s3, s5, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29); - v128_t d0 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s0, kShuf0_0), - LIBYUV_WASM_SWIZZLE(s1, kShuf0_1)); - v128_t d1 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s1, kShuf1_1), - LIBYUV_WASM_SWIZZLE(s02, kShuf1_02)); - v128_t d2 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s2, kShuf2_2), - LIBYUV_WASM_SWIZZLE(s1, kShuf2_1)); - v128_t d3 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s3, kShuf0_0), - LIBYUV_WASM_SWIZZLE(s4, kShuf0_1)); - v128_t d4 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s4, kShuf1_1), - LIBYUV_WASM_SWIZZLE(s35, kShuf1_02)); - v128_t d5 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s5, kShuf2_2), - LIBYUV_WASM_SWIZZLE(s4, kShuf2_1)); + v128_t d0 = wasm_v128_or(WasmSwizzle(s0, kShuf0_0), + WasmSwizzle(s1, kShuf0_1)); + v128_t d1 = wasm_v128_or(WasmSwizzle(s1, kShuf1_1), + WasmSwizzle(s02, kShuf1_02)); + v128_t d2 = wasm_v128_or(WasmSwizzle(s2, kShuf2_2), + WasmSwizzle(s1, kShuf2_1)); + v128_t d3 = wasm_v128_or(WasmSwizzle(s3, kShuf0_0), + WasmSwizzle(s4, kShuf0_1)); + v128_t d4 = wasm_v128_or(WasmSwizzle(s4, kShuf1_1), + WasmSwizzle(s35, kShuf1_02)); + v128_t d5 = wasm_v128_or(WasmSwizzle(s5, kShuf2_2), + WasmSwizzle(s4, kShuf2_1)); wasm_v128_store(dst_rgb24, d0); wasm_v128_store(dst_rgb24 + 16, d1); @@ -123,36 +130,58 @@ void RAWToRGB24Row_WASMSIMD(const uint8_t* src_raw, v128_t s02 = wasm_i8x16_shuffle(s0, s2, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29); - v128_t d0 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s0, kShuf0_0), - LIBYUV_WASM_SWIZZLE(s1, kShuf0_1)); - v128_t d1 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s1, kShuf1_1), - LIBYUV_WASM_SWIZZLE(s02, kShuf1_02)); - v128_t d2 = wasm_v128_or(LIBYUV_WASM_SWIZZLE(s2, kShuf2_2), - LIBYUV_WASM_SWIZZLE(s1, kShuf2_1)); + v128_t d0 = wasm_v128_or(WasmSwizzle(s0, kShuf0_0), + WasmSwizzle(s1, kShuf0_1)); + v128_t d1 = wasm_v128_or(WasmSwizzle(s1, kShuf1_1), + WasmSwizzle(s02, kShuf1_02)); + v128_t d2 = wasm_v128_or(WasmSwizzle(s2, kShuf2_2), + WasmSwizzle(s1, kShuf2_1)); wasm_v128_store(dst_rgb24, d0); wasm_v128_store(dst_rgb24 + 16, d1); wasm_v128_store(dst_rgb24 + 32, d2); } } -#endif // HAS_RAWTORGB24ROW_WASMSIMD -#if defined(HAS_RAWTOARGBROW_WASMSIMD) || defined(HAS_RGB24TOARGBROW_WASMSIMD) +#ifdef HAS_RAWTORGB24ROW_WASMSIMD +void RAWToRGB24Row_WASMSIMD(const uint8_t* src_raw, + uint8_t* dst_rgb24, + int width) { + RAWToRGB24Row_Wasm(src_raw, dst_rgb24, width); +} +#endif // HAS_RAWTORGB24ROW_WASMSIMD +#ifdef HAS_RAWTORGB24ROW_RELAXEDSIMD +void RAWToRGB24Row_RELAXEDSIMD(const uint8_t* src_raw, + uint8_t* dst_rgb24, + int width) { + RAWToRGB24Row_Wasm(src_raw, dst_rgb24, width); +} +#endif // HAS_RAWTORGB24ROW_RELAXEDSIMD +#endif // HAS_RAWTORGB24ROW_WASMSIMD || HAS_RAWTORGB24ROW_RELAXEDSIMD + +#if defined(HAS_RAWTOARGBROW_WASMSIMD) || \ + defined(HAS_RAWTOARGBROW_RELAXEDSIMD) || \ + defined(HAS_RAWTORGBAROW_WASMSIMD) || \ + defined(HAS_RAWTORGBAROW_RELAXEDSIMD) || \ + defined(HAS_RGB24TOARGBROW_WASMSIMD) || \ + defined(HAS_RGB24TOARGBROW_RELAXEDSIMD) // 16 pixels of 3 bytes to 4 bytes. shuf holds 2 masks: one for loads at // offsets 0, 12, 24 and one for the load at offset 32 (avoids over-read). -static void RGB3ToARGBRow_WASMSIMD(const uint8_t* src, - uint8_t* dst_argb, - int width, - const uint8_t* shuf) { +template +static void RGB3ToARGBRow_Wasm(const uint8_t* src, + uint8_t* dst_argb, + int width, + const uint8_t* shuf, + uint32_t alpha_mask) { v128_t m0 = LoadConst(shuf); v128_t m1 = LoadConst(shuf + 16); - v128_t alpha = SplatConst32((int)0xff000000); + v128_t alpha = SplatConst32((int)alpha_mask); do { asm("" : "+r"(src), "+r"(dst_argb)); - v128_t d0 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src), m0); - v128_t d1 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src + 12), m0); - v128_t d2 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src + 24), m0); - v128_t d3 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src + 32), m1); + v128_t d0 = WasmSwizzle(wasm_v128_load(src), m0); + v128_t d1 = WasmSwizzle(wasm_v128_load(src + 12), m0); + v128_t d2 = WasmSwizzle(wasm_v128_load(src + 24), m0); + v128_t d3 = WasmSwizzle(wasm_v128_load(src + 32), m1); wasm_v128_store(dst_argb, wasm_v128_or(d0, alpha)); wasm_v128_store(dst_argb + 16, wasm_v128_or(d1, alpha)); wasm_v128_store(dst_argb + 32, wasm_v128_or(d2, alpha)); @@ -164,39 +193,178 @@ static void RGB3ToARGBRow_WASMSIMD(const uint8_t* src, } #endif -#ifdef HAS_RAWTOARGBROW_WASMSIMD +#if defined(HAS_RAWTOARGBROW_WASMSIMD) || defined(HAS_RAWTOARGBROW_RELAXEDSIMD) static const uint8_t kShuffleRAWToARGB[32] = { 2, 1, 0, 128, 5, 4, 3, 128, 8, 7, 6, 128, 11, 10, 9, 128, 6, 5, 4, 128, 9, 8, 7, 128, 12, 11, 10, 128, 15, 14, 13, 128}; +#ifdef HAS_RAWTOARGBROW_WASMSIMD void RAWToARGBRow_WASMSIMD(const uint8_t* src_raw, uint8_t* dst_argb, int width) { - RGB3ToARGBRow_WASMSIMD(src_raw, dst_argb, width, kShuffleRAWToARGB); + RGB3ToARGBRow_Wasm(src_raw, dst_argb, width, kShuffleRAWToARGB, + 0xff000000u); } #endif // HAS_RAWTOARGBROW_WASMSIMD +#ifdef HAS_RAWTOARGBROW_RELAXEDSIMD +void RAWToARGBRow_RELAXEDSIMD(const uint8_t* src_raw, + uint8_t* dst_argb, + int width) { + RGB3ToARGBRow_Wasm(src_raw, dst_argb, width, kShuffleRAWToARGB, + 0xff000000u); +} +#endif // HAS_RAWTOARGBROW_RELAXEDSIMD +#endif // HAS_RAWTOARGBROW_WASMSIMD || HAS_RAWTOARGBROW_RELAXEDSIMD -#ifdef HAS_RGB24TOARGBROW_WASMSIMD +#if defined(HAS_RAWTORGBAROW_WASMSIMD) || defined(HAS_RAWTORGBAROW_RELAXEDSIMD) +static const uint8_t kShuffleRAWToRGBA[32] = { + 128, 2, 1, 0, 128, 5, 4, 3, 128, 8, 7, 6, 128, 11, 10, 9, + 128, 6, 5, 4, 128, 9, 8, 7, 128, 12, 11, 10, 128, 15, 14, 13}; +#ifdef HAS_RAWTORGBAROW_WASMSIMD +void RAWToRGBARow_WASMSIMD(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width) { + RGB3ToARGBRow_Wasm(src_raw, dst_rgba, width, kShuffleRAWToRGBA, + 0x000000ffu); +} +#endif // HAS_RAWTORGBAROW_WASMSIMD +#ifdef HAS_RAWTORGBAROW_RELAXEDSIMD +void RAWToRGBARow_RELAXEDSIMD(const uint8_t* src_raw, + uint8_t* dst_rgba, + int width) { + RGB3ToARGBRow_Wasm(src_raw, dst_rgba, width, kShuffleRAWToRGBA, + 0x000000ffu); +} +#endif // HAS_RAWTORGBAROW_RELAXEDSIMD +#endif // HAS_RAWTORGBAROW_WASMSIMD || HAS_RAWTORGBAROW_RELAXEDSIMD + +#if defined(HAS_RGB24TOARGBROW_WASMSIMD) || \ + defined(HAS_RGB24TOARGBROW_RELAXEDSIMD) static const uint8_t kShuffleRGB24ToARGB[32] = { 0, 1, 2, 128, 3, 4, 5, 128, 6, 7, 8, 128, 9, 10, 11, 128, 4, 5, 6, 128, 7, 8, 9, 128, 10, 11, 12, 128, 13, 14, 15, 128}; +#ifdef HAS_RGB24TOARGBROW_WASMSIMD void RGB24ToARGBRow_WASMSIMD(const uint8_t* src_rgb24, uint8_t* dst_argb, int width) { - RGB3ToARGBRow_WASMSIMD(src_rgb24, dst_argb, width, kShuffleRGB24ToARGB); + RGB3ToARGBRow_Wasm(src_rgb24, dst_argb, width, kShuffleRGB24ToARGB, + 0xff000000u); } #endif // HAS_RGB24TOARGBROW_WASMSIMD +#ifdef HAS_RGB24TOARGBROW_RELAXEDSIMD +void RGB24ToARGBRow_RELAXEDSIMD(const uint8_t* src_rgb24, + uint8_t* dst_argb, + int width) { + RGB3ToARGBRow_Wasm(src_rgb24, dst_argb, width, kShuffleRGB24ToARGB, + 0xff000000u); +} +#endif // HAS_RGB24TOARGBROW_RELAXEDSIMD +#endif // HAS_RGB24TOARGBROW_WASMSIMD || HAS_RGB24TOARGBROW_RELAXEDSIMD -#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) || \ - defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) || \ - defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) || \ - defined(HAS_RGBTOYMATRIXROW_WASMSIMD) || \ - defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD) || \ - defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) || \ - defined(HAS_RGB565TOYMATRIXROW_WASMSIMD) || \ - defined(HAS_ARGB1555TOYMATRIXROW_WASMSIMD) || \ - defined(HAS_ARGB4444TOYMATRIXROW_WASMSIMD) || \ - defined(HAS_RGB565TOUVMATRIXROW_WASMSIMD) || \ - defined(HAS_ARGB1555TOUVMATRIXROW_WASMSIMD) || \ +#if defined(HAS_ARGBTORGB24ROW_WASMSIMD) || \ + defined(HAS_ARGBTORGB24ROW_RELAXEDSIMD) || \ + defined(HAS_ARGBTORAWROW_WASMSIMD) || \ + defined(HAS_ARGBTORAWROW_RELAXEDSIMD) +// 16 pixels of 4 bytes to 3 bytes (48 bytes output). +template +static void ARGBToRGB3Row_Wasm(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width, + const uint8_t* shuf) { + v128_t m0a = LoadConst(shuf); + v128_t m0b = LoadConst(shuf + 16); + v128_t m1a = LoadConst(shuf + 32); + v128_t m1b = LoadConst(shuf + 48); + v128_t m2a = LoadConst(shuf + 64); + v128_t m2b = LoadConst(shuf + 80); + do { + asm("" : "+r"(src_argb), "+r"(dst_rgb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + v128_t s2 = wasm_v128_load(src_argb + 32); + v128_t s3 = wasm_v128_load(src_argb + 48); + v128_t d0 = wasm_v128_or(WasmSwizzle(s0, m0a), + WasmSwizzle(s1, m0b)); + v128_t d1 = wasm_v128_or(WasmSwizzle(s1, m1a), + WasmSwizzle(s2, m1b)); + v128_t d2 = wasm_v128_or(WasmSwizzle(s2, m2a), + WasmSwizzle(s3, m2b)); + wasm_v128_store(dst_rgb, d0); + wasm_v128_store(dst_rgb + 16, d1); + wasm_v128_store(dst_rgb + 32, d2); + src_argb += 64; + dst_rgb += 48; + width -= 16; + } while (width > 0); +} +#endif + +#if defined(HAS_ARGBTORGB24ROW_WASMSIMD) || \ + defined(HAS_ARGBTORGB24ROW_RELAXEDSIMD) +static const uint8_t kShuffleARGBToRGB24[96] = { + 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14}; +#ifdef HAS_ARGBTORGB24ROW_WASMSIMD +void ARGBToRGB24Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb24, + int width) { + ARGBToRGB3Row_Wasm(src_argb, dst_rgb24, width, kShuffleARGBToRGB24); +} +#endif // HAS_ARGBTORGB24ROW_WASMSIMD +#ifdef HAS_ARGBTORGB24ROW_RELAXEDSIMD +void ARGBToRGB24Row_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb24, + int width) { + ARGBToRGB3Row_Wasm(src_argb, dst_rgb24, width, kShuffleARGBToRGB24); +} +#endif // HAS_ARGBTORGB24ROW_RELAXEDSIMD +#endif // HAS_ARGBTORGB24ROW_WASMSIMD || HAS_ARGBTORGB24ROW_RELAXEDSIMD + +#if defined(HAS_ARGBTORAWROW_WASMSIMD) || defined(HAS_ARGBTORAWROW_RELAXEDSIMD) +static const uint8_t kShuffleARGBToRAW[96] = { + 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12}; +#ifdef HAS_ARGBTORAWROW_WASMSIMD +void ARGBToRAWRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_raw, + int width) { + ARGBToRGB3Row_Wasm(src_argb, dst_raw, width, kShuffleARGBToRAW); +} +#endif // HAS_ARGBTORAWROW_WASMSIMD +#ifdef HAS_ARGBTORAWROW_RELAXEDSIMD +void ARGBToRAWRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_raw, + int width) { + ARGBToRGB3Row_Wasm(src_argb, dst_raw, width, kShuffleARGBToRAW); +} +#endif // HAS_ARGBTORAWROW_RELAXEDSIMD +#endif // HAS_ARGBTORAWROW_WASMSIMD || HAS_ARGBTORAWROW_RELAXEDSIMD + +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) || \ + defined(HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD) || \ + defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) || \ + defined(HAS_RGBTOYMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) || \ + defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUV444MATRIXROW_RELAXEDSIMD) || \ + defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUVMATRIXROW_RELAXEDSIMD) || \ + defined(HAS_RGB565TOYMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGB1555TOYMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGB4444TOYMATRIXROW_WASMSIMD) || \ + defined(HAS_RGB565TOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGB1555TOUVMATRIXROW_WASMSIMD) || \ defined(HAS_ARGB4444TOUVMATRIXROW_WASMSIMD) // RGB to YUV. Math is in 16 bit lanes, which wrap, but sum + bias fits in // 16 bits so the results match the C code. i32x4.dot_i16x8_s would be best @@ -225,9 +393,12 @@ static inline v128_t SubShr8(v128_t bias, v128_t sum) { } #endif -#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) || \ - defined(HAS_RGBTOYMATRIXROW_WASMSIMD) || \ - defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD) +#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) || \ + defined(HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD) || \ + defined(HAS_RGBTOYMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) || \ + defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUV444MATRIXROW_RELAXEDSIMD) // Swizzles 4 pixels to 16 bit {c0 x4, c2 x4} and {c1 x4, c3 x4}, so the 2 // halves of the products are summed with a 64 bit interleave. static const uint8_t kShuffleARGBToPlanar[32] = { @@ -255,6 +426,7 @@ static inline v128_t HAddHalves64(v128_t t0, v128_t t1) { // 16 U and V from 16 pixels swizzled by m (m[0], m[1] for the first 3 // vectors, m[2], m[3] for the last). +template static inline void PlanarToUV16(const v128_t* s, const v128_t* m, v128_t ku02, @@ -267,8 +439,8 @@ static inline void PlanarToUV16(const v128_t* s, uint8_t* dst_v) { v128_t u[4], v[4]; for (int i = 0; i < 4; ++i) { - v128_t lo = LIBYUV_WASM_SWIZZLE(s[i], m[i == 3 ? 2 : 0]); - v128_t hi = LIBYUV_WASM_SWIZZLE(s[i], m[i == 3 ? 3 : 1]); + v128_t lo = WasmSwizzle(s[i], m[i == 3 ? 2 : 0]); + v128_t hi = WasmSwizzle(s[i], m[i == 3 ? 3 : 1]); u[i] = MulAdd16(lo, hi, ku02, ku13); v[i] = MulAdd16(lo, hi, kv02, kv13); } @@ -281,8 +453,10 @@ static inline void PlanarToUV16(const v128_t* s, } #endif -#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) || \ - defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD) || \ + defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUVMATRIXROW_RELAXEDSIMD) // Rounded 2x2 average of 8 pixels held as 16 bit {c0, c2} or {c1, c3} // pairs. a = row 0, b = row 1, 4 pixels per vector. Returns 4 pixels. static inline v128_t Avg2x2Pairs(v128_t a0, @@ -344,12 +518,14 @@ void ARGBToYMatrixRow_WASMSIMD(const uint8_t* src_argb, } #endif // HAS_ARGBTOYMATRIXROW_WASMSIMD -#ifdef HAS_RGBTOYMATRIXROW_WASMSIMD +#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOYMATRIXROW_RELAXEDSIMD) // RGB24 is alpha 255 in the C version, so 255 * kRGBToY[3] is in the bias. -void RGBToYMatrixRow_WASMSIMD(const uint8_t* src_rgb, - uint8_t* dst_y, - int width, - const struct ArgbConstants* c) { +template +static inline void RGBToYMatrixRow_Wasm(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { v128_t m[4]; for (int i = 0; i < 4; ++i) { m[i] = LoadConst(kShuffleRGBToPlanar + i * 16); @@ -361,8 +537,8 @@ void RGBToYMatrixRow_WASMSIMD(const uint8_t* src_rgb, v128_t t[4]; for (int i = 0; i < 4; ++i) { v128_t s = wasm_v128_load(src_rgb + (i == 3 ? 32 : i * 12)); - t[i] = MulAdd16(LIBYUV_WASM_SWIZZLE(s, m[i == 3 ? 2 : 0]), - LIBYUV_WASM_SWIZZLE(s, m[i == 3 ? 3 : 1]), k02, k13); + t[i] = MulAdd16(WasmSwizzle(s, m[i == 3 ? 2 : 0]), + WasmSwizzle(s, m[i == 3 ? 3 : 1]), k02, k13); } v128_t y0 = wasm_i16x8_add(HAddHalves64(t[0], t[1]), bias); v128_t y1 = wasm_i16x8_add(HAddHalves64(t[2], t[3]), bias); @@ -373,14 +549,32 @@ void RGBToYMatrixRow_WASMSIMD(const uint8_t* src_rgb, width -= 16; } while (width > 0); } +#ifdef HAS_RGBTOYMATRIXROW_WASMSIMD +void RGBToYMatrixRow_WASMSIMD(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + RGBToYMatrixRow_Wasm(src_rgb, dst_y, width, c); +} #endif // HAS_RGBTOYMATRIXROW_WASMSIMD +#ifdef HAS_RGBTOYMATRIXROW_RELAXEDSIMD +void RGBToYMatrixRow_RELAXEDSIMD(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + RGBToYMatrixRow_Wasm(src_rgb, dst_y, width, c); +} +#endif // HAS_RGBTOYMATRIXROW_RELAXEDSIMD +#endif // HAS_RGBTOYMATRIXROW_WASMSIMD || HAS_RGBTOYMATRIXROW_RELAXEDSIMD -#ifdef HAS_ARGBTOUV444MATRIXROW_WASMSIMD -void ARGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_argb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c) { +#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) || \ + defined(HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD) +template +static inline void ARGBToUV444MatrixRow_Wasm(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { v128_t m[4]; m[0] = m[2] = LoadConst(kShuffleARGBToPlanar); m[1] = m[3] = LoadConst(kShuffleARGBToPlanar + 16); @@ -394,21 +588,43 @@ void ARGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_argb, for (int i = 0; i < 4; ++i) { s[i] = wasm_v128_load(src_argb + i * 16); } - PlanarToUV16(s, m, ku02, ku13, kv02, kv13, bias, bias, dst_u, dst_v); + PlanarToUV16(s, m, ku02, ku13, kv02, kv13, bias, bias, dst_u, + dst_v); src_argb += 64; dst_u += 16; dst_v += 16; width -= 16; } while (width > 0); } +#ifdef HAS_ARGBTOUV444MATRIXROW_WASMSIMD +void ARGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + ARGBToUV444MatrixRow_Wasm(src_argb, dst_u, dst_v, width, c); +} #endif // HAS_ARGBTOUV444MATRIXROW_WASMSIMD +#ifdef HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD +void ARGBToUV444MatrixRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + ARGBToUV444MatrixRow_Wasm(src_argb, dst_u, dst_v, width, c); +} +#endif // HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD +#endif // HAS_ARGBTOUV444MATRIXROW_WASMSIMD || + // HAS_ARGBTOUV444MATRIXROW_RELAXEDSIMD -#ifdef HAS_RGBTOUV444MATRIXROW_WASMSIMD -void RGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_rgb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c) { +#if defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUV444MATRIXROW_RELAXEDSIMD) +template +static inline void RGBToUV444MatrixRow_Wasm(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { v128_t m[4]; for (int i = 0; i < 4; ++i) { m[i] = LoadConst(kShuffleRGBToPlanar + i * 16); @@ -424,14 +640,34 @@ void RGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_rgb, for (int i = 0; i < 4; ++i) { s[i] = wasm_v128_load(src_rgb + (i == 3 ? 32 : i * 12)); } - PlanarToUV16(s, m, ku02, ku13, kv02, kv13, bias_u, bias_v, dst_u, dst_v); + PlanarToUV16(s, m, ku02, ku13, kv02, kv13, bias_u, bias_v, dst_u, + dst_v); src_rgb += 48; dst_u += 16; dst_v += 16; width -= 16; } while (width > 0); } +#ifdef HAS_RGBTOUV444MATRIXROW_WASMSIMD +void RGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + RGBToUV444MatrixRow_Wasm(src_rgb, dst_u, dst_v, width, c); +} #endif // HAS_RGBTOUV444MATRIXROW_WASMSIMD +#ifdef HAS_RGBTOUV444MATRIXROW_RELAXEDSIMD +void RGBToUV444MatrixRow_RELAXEDSIMD(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + RGBToUV444MatrixRow_Wasm(src_rgb, dst_u, dst_v, width, c); +} +#endif // HAS_RGBTOUV444MATRIXROW_RELAXEDSIMD +#endif // HAS_RGBTOUV444MATRIXROW_WASMSIMD || + // HAS_RGBTOUV444MATRIXROW_RELAXEDSIMD #ifdef HAS_ARGBTOUVMATRIXROW_WASMSIMD void ARGBToUVMatrixRow_WASMSIMD(const uint8_t* src_argb, @@ -471,7 +707,59 @@ void ARGBToUVMatrixRow_WASMSIMD(const uint8_t* src_argb, } #endif // HAS_ARGBTOUVMATRIXROW_WASMSIMD -#ifdef HAS_RGBTOUVMATRIXROW_WASMSIMD +#ifdef HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD +static const uint8_t kShuffleARGBPairs[32] = {0, 4, 2, 6, 8, 12, 10, 14, + 1, 5, 3, 7, 9, 13, 11, 15}; +void ARGBToUVMatrixRow_RELAXEDSIMD(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const uint8_t* src_argb1 = src_argb + src_stride_argb; + v128_t shuf = LoadConst(kShuffleARGBPairs); + v128_t x80 = SplatConst8(-128); + v128_t one = SplatConst8(1); + v128_t bias514 = SplatConst16(512 + 2); + v128_t ku02 = SplatPair16(c->kRGBToU[0], c->kRGBToU[2]); + v128_t ku13 = SplatPair16(c->kRGBToU[1], c->kRGBToU[3]); + v128_t kv02 = SplatPair16(c->kRGBToV[0], c->kRGBToV[2]); + v128_t kv13 = SplatPair16(c->kRGBToV[1], c->kRGBToV[3]); + v128_t bias = SplatConst16(c->kAddUV[0]); + do { + v128_t lo[2], hi[2]; + for (int i = 0; i < 2; ++i) { + v128_t a0 = + wasm_i8x16_relaxed_swizzle(wasm_v128_load(src_argb + i * 32), shuf); + v128_t a1 = wasm_i8x16_relaxed_swizzle( + wasm_v128_load(src_argb + i * 32 + 16), shuf); + v128_t b0 = + wasm_i8x16_relaxed_swizzle(wasm_v128_load(src_argb1 + i * 32), shuf); + v128_t b1 = wasm_i8x16_relaxed_swizzle( + wasm_v128_load(src_argb1 + i * 32 + 16), shuf); + v128_t s0 = wasm_i16x8_add( + wasm_i16x8_relaxed_dot_i8x16_i7x16(wasm_v128_xor(a0, x80), one), + wasm_i16x8_relaxed_dot_i8x16_i7x16(wasm_v128_xor(b0, x80), one)); + v128_t s1 = wasm_i16x8_add( + wasm_i16x8_relaxed_dot_i8x16_i7x16(wasm_v128_xor(a1, x80), one), + wasm_i16x8_relaxed_dot_i8x16_i7x16(wasm_v128_xor(b1, x80), one)); + s0 = wasm_u16x8_shr(wasm_i16x8_add(s0, bias514), 2); + s1 = wasm_u16x8_shr(wasm_i16x8_add(s1, bias514), 2); + lo[i] = wasm_i64x2_shuffle(s0, s1, 0, 2); + hi[i] = wasm_i64x2_shuffle(s0, s1, 1, 3); + } + PairsToUV8(lo, hi, ku02, ku13, kv02, kv13, bias, bias, dst_u, dst_v); + src_argb += 64; + src_argb1 += 64; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGBTOUVMATRIXROW_RELAXEDSIMD + +#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUVMATRIXROW_RELAXEDSIMD) // Swizzles 4 RGB24 pixels to 16 bit {b, r} and {g, 0} pairs. The second // pair of masks is for the load at offset 32 (avoids over-read). static const uint8_t kShuffleRGBToPairs[64] = { @@ -480,12 +768,13 @@ static const uint8_t kShuffleRGBToPairs[64] = { 4, 128, 6, 128, 7, 128, 9, 128, 10, 128, 12, 128, 13, 128, 15, 128, 5, 128, 128, 128, 8, 128, 128, 128, 11, 128, 128, 128, 14, 128, 128, 128}; -void RGBToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb, - int src_stride_rgb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c) { +template +static inline void RGBToUVMatrixRow_Wasm(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { const uint8_t* src_rgb1 = src_rgb + src_stride_rgb; v128_t m[4]; for (int i = 0; i < 4; ++i) { @@ -510,11 +799,11 @@ void RGBToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb, v128_t ml = m[i * 2]; v128_t mh = m[i * 2 + 1]; lo[i] = Avg2x2Pairs( - LIBYUV_WASM_SWIZZLE(a0, m[0]), LIBYUV_WASM_SWIZZLE(a1, ml), - LIBYUV_WASM_SWIZZLE(b0, m[0]), LIBYUV_WASM_SWIZZLE(b1, ml), two); + WasmSwizzle(a0, m[0]), WasmSwizzle(a1, ml), + WasmSwizzle(b0, m[0]), WasmSwizzle(b1, ml), two); hi[i] = Avg2x2Pairs( - LIBYUV_WASM_SWIZZLE(a0, m[1]), LIBYUV_WASM_SWIZZLE(a1, mh), - LIBYUV_WASM_SWIZZLE(b0, m[1]), LIBYUV_WASM_SWIZZLE(b1, mh), two); + WasmSwizzle(a0, m[1]), WasmSwizzle(a1, mh), + WasmSwizzle(b0, m[1]), WasmSwizzle(b1, mh), two); } PairsToUV8(lo, hi, ku02, ku13, kv02, kv13, bias_u, bias_v, dst_u, dst_v); src_rgb += 48; @@ -524,7 +813,27 @@ void RGBToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb, width -= 16; } while (width > 0); } +#ifdef HAS_RGBTOUVMATRIXROW_WASMSIMD +void RGBToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + RGBToUVMatrixRow_Wasm(src_rgb, src_stride_rgb, dst_u, dst_v, width, c); +} #endif // HAS_RGBTOUVMATRIXROW_WASMSIMD +#ifdef HAS_RGBTOUVMATRIXROW_RELAXEDSIMD +void RGBToUVMatrixRow_RELAXEDSIMD(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + RGBToUVMatrixRow_Wasm(src_rgb, src_stride_rgb, dst_u, dst_v, width, c); +} +#endif // HAS_RGBTOUVMATRIXROW_RELAXEDSIMD +#endif // HAS_RGBTOUVMATRIXROW_WASMSIMD || HAS_RGBTOUVMATRIXROW_RELAXEDSIMD // 16 bit pixel formats keep each channel in its own 16 bit lane. // 5 and 6 bit channels expand as (v * 33) >> 2 and (v * 65) >> 4, and 4 bit @@ -837,6 +1146,86 @@ void MultiplyRow_16_WASMSIMD(const uint16_t* src_y, } #endif // HAS_MULTIPLYROW_16_WASMSIMD +#ifdef HAS_DIVIDEROW_16_WASMSIMD +void DivideRow_16_WASMSIMD(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + v128_t vscale = wasm_i32x4_splat(scale); + do { + asm("" : "+r"(src_y), "+r"(dst_y)); + v128_t y0 = wasm_v128_load(src_y); + v128_t y1 = wasm_v128_load(src_y + 8); + v128_t lo0 = wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(y0), vscale); + v128_t hi0 = wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(y0), vscale); + v128_t lo1 = wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(y1), vscale); + v128_t hi1 = wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(y1), vscale); + wasm_v128_store(dst_y, + wasm_i16x8_shuffle(lo0, hi0, 1, 3, 5, 7, 9, 11, 13, 15)); + wasm_v128_store(dst_y + 8, + wasm_i16x8_shuffle(lo1, hi1, 1, 3, 5, 7, 9, 11, 13, 15)); + src_y += 16; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_DIVIDEROW_16_WASMSIMD + +#ifdef HAS_CONVERT16TO8ROW_WASMSIMD +void Convert16To8Row_WASMSIMD(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width) { + v128_t vscale = wasm_i16x8_splat(scale); + do { + asm("" : "+r"(src_y)); + v128_t y0 = wasm_v128_load(src_y); + v128_t y1 = wasm_v128_load(src_y + 8); + v128_t lo0 = wasm_u32x4_extmul_low_u16x8(y0, vscale); + v128_t hi0 = wasm_u32x4_extmul_high_u16x8(y0, vscale); + v128_t lo1 = wasm_u32x4_extmul_low_u16x8(y1, vscale); + v128_t hi1 = wasm_u32x4_extmul_high_u16x8(y1, vscale); + v128_t r0 = wasm_i16x8_shuffle(lo0, hi0, 1, 3, 5, 7, 9, 11, 13, 15); + v128_t r1 = wasm_i16x8_shuffle(lo1, hi1, 1, 3, 5, 7, 9, 11, 13, 15); + wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(r0, r1)); + src_y += 16; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_CONVERT16TO8ROW_WASMSIMD + +#ifdef HAS_HALFMERGEUVROW_WASMSIMD +void HalfMergeUVRow_WASMSIMD(const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_uv, + int width) { + const uint8_t* src_u1 = src_u + src_stride_u; + const uint8_t* src_v1 = src_v + src_stride_v; + v128_t c2 = SplatConst16(2); + do { + asm("" : "+r"(src_u), "+r"(src_u1), "+r"(src_v), "+r"(src_v1)); + v128_t u = wasm_i16x8_add( + wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_u)), + wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_u1))); + v128_t v = wasm_i16x8_add( + wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_v)), + wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_v1))); + u = wasm_u16x8_shr(wasm_i16x8_add(u, c2), 2); + v = wasm_u16x8_shr(wasm_i16x8_add(v, c2), 2); + wasm_v128_store(dst_uv, wasm_v128_or(u, wasm_i16x8_shl(v, 8))); + src_u += 16; + src_u1 += 16; + src_v += 16; + src_v1 += 16; + dst_uv += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_HALFMERGEUVROW_WASMSIMD + #ifdef HAS_MERGEUVROW_WASMSIMD void MergeUVRow_WASMSIMD(const uint8_t* src_u, const uint8_t* src_v, @@ -860,36 +1249,411 @@ void MergeUVRow_WASMSIMD(const uint8_t* src_u, } #endif // HAS_MERGEUVROW_WASMSIMD -#ifdef HAS_MIRRORROW_WASMSIMD +#ifdef HAS_MERGEUVROW_16_WASMSIMD +void MergeUVRow_16_WASMSIMD(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width) { + int shift = 16 - depth; + do { + asm("" : "+r"(dst_uv)); + v128_t u = wasm_i16x8_shl(wasm_v128_load(src_u), shift); + v128_t v = wasm_i16x8_shl(wasm_v128_load(src_v), shift); + v128_t uv0 = wasm_i16x8_shuffle(u, v, 0, 8, 1, 9, 2, 10, 3, 11); + v128_t uv1 = wasm_i16x8_shuffle(u, v, 4, 12, 5, 13, 6, 14, 7, 15); + wasm_v128_store(dst_uv, uv0); + wasm_v128_store(dst_uv + 8, uv1); + src_u += 8; + src_v += 8; + dst_uv += 16; + width -= 8; + } while (width > 0); +} +#endif // HAS_MERGEUVROW_16_WASMSIMD + +#ifdef HAS_SPLITUVROW_WASMSIMD +void SplitUVRow_WASMSIMD(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + do { + asm("" : "+r"(src_uv)); + v128_t uv0 = wasm_v128_load(src_uv); + v128_t uv1 = wasm_v128_load(src_uv + 16); + v128_t u = wasm_i8x16_shuffle(uv0, uv1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, + 20, 22, 24, 26, 28, 30); + v128_t v = wasm_i8x16_shuffle(uv0, uv1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, + 21, 23, 25, 27, 29, 31); + wasm_v128_store(dst_u, u); + wasm_v128_store(dst_v, v); + src_uv += 32; + dst_u += 16; + dst_v += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_SPLITUVROW_WASMSIMD + +#ifdef HAS_SPLITUVROW_16_WASMSIMD +void SplitUVRow_16_WASMSIMD(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width) { + int shift = 16 - depth; + do { + asm("" : "+r"(src_uv)); + v128_t uv0 = wasm_v128_load(src_uv); + v128_t uv1 = wasm_v128_load(src_uv + 8); + v128_t u = wasm_u16x8_shr( + wasm_i16x8_shuffle(uv0, uv1, 0, 2, 4, 6, 8, 10, 12, 14), shift); + v128_t v = wasm_u16x8_shr( + wasm_i16x8_shuffle(uv0, uv1, 1, 3, 5, 7, 9, 11, 13, 15), shift); + wasm_v128_store(dst_u, u); + wasm_v128_store(dst_v, v); + src_uv += 16; + dst_u += 8; + dst_v += 8; + width -= 8; + } while (width > 0); +} +#endif // HAS_SPLITUVROW_16_WASMSIMD + +#if defined(HAS_SPLITRGBROW_WASMSIMD) || defined(HAS_SPLITRGBROW_RELAXEDSIMD) +static const uint8_t kShuffleSplitRGB[16] = {1, 4, 7, 10, 13, 0, 3, 6, + 9, 12, 15, 2, 5, 8, 11, 14}; + +template +static inline void SplitRGBRow_Wasm(const uint8_t* src_rgb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width) { + v128_t shuf = LoadConst(kShuffleSplitRGB); + do { + asm("" : "+r"(src_rgb)); + v128_t s0 = WasmSwizzle(wasm_v128_load(src_rgb), shuf); + v128_t s1 = WasmSwizzle(wasm_v128_load(src_rgb + 16), shuf); + v128_t s2 = WasmSwizzle(wasm_v128_load(src_rgb + 32), shuf); + v128_t s12_a = wasm_i8x16_shuffle(s1, s2, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 0, 0, 0, 0, 0, 0); + v128_t r = wasm_i8x16_shuffle(s0, s12_a, 5, 6, 7, 8, 9, 10, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25); + v128_t s01_b = wasm_i8x16_shuffle(s0, s1, 0, 1, 2, 3, 4, 21, 22, 23, 24, 25, + 26, 0, 0, 0, 0, 0); + v128_t g = wasm_i8x16_shuffle(s01_b, s2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 27, 28, 29, 30, 31); + v128_t s01_c = wasm_i8x16_shuffle(s0, s1, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 0, 0, 0, 0, 0, 0); + v128_t b = wasm_i8x16_shuffle(s01_c, s2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 21, + 22, 23, 24, 25, 26); + wasm_v128_store(dst_r, r); + wasm_v128_store(dst_g, g); + wasm_v128_store(dst_b, b); + src_rgb += 48; + dst_r += 16; + dst_g += 16; + dst_b += 16; + width -= 16; + } while (width > 0); +} + +#ifdef HAS_SPLITRGBROW_WASMSIMD +void SplitRGBRow_WASMSIMD(const uint8_t* src_rgb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width) { + SplitRGBRow_Wasm(src_rgb, dst_r, dst_g, dst_b, width); +} +#endif // HAS_SPLITRGBROW_WASMSIMD + +#ifdef HAS_SPLITRGBROW_RELAXEDSIMD +void SplitRGBRow_RELAXEDSIMD(const uint8_t* src_rgb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width) { + SplitRGBRow_Wasm(src_rgb, dst_r, dst_g, dst_b, width); +} +#endif // HAS_SPLITRGBROW_RELAXEDSIMD +#endif // HAS_SPLITRGBROW_WASMSIMD || HAS_SPLITRGBROW_RELAXEDSIMD + +#if defined(HAS_MERGERGBROW_WASMSIMD) || defined(HAS_MERGERGBROW_RELAXEDSIMD) +static const uint8_t kMergeRGBSwizzle[96] = { + 0, 1, 128, 2, 3, 128, 4, 5, 128, 6, 7, 128, 8, 9, + 128, 10, 128, 128, 0, 128, 128, 1, 128, 128, 2, 128, 128, 3, + 128, 128, 4, 128, 3, 128, 4, 5, 128, 6, 7, 128, 8, 9, + 128, 10, 11, 128, 12, 13, 128, 5, 128, 128, 6, 128, 128, 7, + 128, 128, 8, 128, 128, 9, 128, 128, 128, 6, 7, 128, 8, 9, + 128, 10, 11, 128, 12, 13, 128, 14, 15, 128, 10, 128, 128, 11, + 128, 128, 12, 128, 128, 13, 128, 128, 14, 128, 128, 15}; + +template +static inline void MergeRGBRow_Wasm(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width) { + v128_t m0 = LoadConst(kMergeRGBSwizzle); + v128_t m1 = LoadConst(kMergeRGBSwizzle + 16); + v128_t m2 = LoadConst(kMergeRGBSwizzle + 32); + v128_t m3 = LoadConst(kMergeRGBSwizzle + 48); + v128_t m4 = LoadConst(kMergeRGBSwizzle + 64); + v128_t m5 = LoadConst(kMergeRGBSwizzle + 80); + do { + asm("" : "+r"(dst_rgb)); + v128_t r = wasm_v128_load(src_r); + v128_t g = wasm_v128_load(src_g); + v128_t b = wasm_v128_load(src_b); + v128_t rg0 = wasm_i8x16_shuffle(r, g, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + v128_t rg2 = wasm_i8x16_shuffle(r, g, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31); + v128_t rg1 = wasm_i8x16_shuffle(rg0, rg2, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23); + wasm_v128_store(dst_rgb, wasm_v128_or(WasmSwizzle(rg0, m0), + WasmSwizzle(b, m1))); + wasm_v128_store(dst_rgb + 16, wasm_v128_or(WasmSwizzle(rg1, m2), + WasmSwizzle(b, m3))); + wasm_v128_store(dst_rgb + 32, wasm_v128_or(WasmSwizzle(rg2, m4), + WasmSwizzle(b, m5))); + src_r += 16; + src_g += 16; + src_b += 16; + dst_rgb += 48; + width -= 16; + } while (width > 0); +} + +#ifdef HAS_MERGERGBROW_WASMSIMD +void MergeRGBRow_WASMSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width) { + MergeRGBRow_Wasm(src_r, src_g, src_b, dst_rgb, width); +} +#endif // HAS_MERGERGBROW_WASMSIMD + +#ifdef HAS_MERGERGBROW_RELAXEDSIMD +void MergeRGBRow_RELAXEDSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width) { + MergeRGBRow_Wasm(src_r, src_g, src_b, dst_rgb, width); +} +#endif // HAS_MERGERGBROW_RELAXEDSIMD +#endif // HAS_MERGERGBROW_WASMSIMD || HAS_MERGERGBROW_RELAXEDSIMD + +#ifdef HAS_SPLITARGBROW_WASMSIMD +void SplitARGBRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + uint8_t* dst_a, + int width) { + do { + asm("" : "+r"(src_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + v128_t s2 = wasm_v128_load(src_argb + 32); + v128_t s3 = wasm_v128_load(src_argb + 48); + v128_t bg01 = wasm_i8x16_shuffle(s0, s1, 0, 4, 8, 12, 16, 20, 24, 28, 1, 5, + 9, 13, 17, 21, 25, 29); + v128_t bg23 = wasm_i8x16_shuffle(s2, s3, 0, 4, 8, 12, 16, 20, 24, 28, 1, 5, + 9, 13, 17, 21, 25, 29); + v128_t ra01 = wasm_i8x16_shuffle(s0, s1, 2, 6, 10, 14, 18, 22, 26, 30, 3, 7, + 11, 15, 19, 23, 27, 31); + v128_t ra23 = wasm_i8x16_shuffle(s2, s3, 2, 6, 10, 14, 18, 22, 26, 30, 3, 7, + 11, 15, 19, 23, 27, 31); + wasm_v128_store(dst_r, wasm_i64x2_shuffle(bg01, bg23, 0, 2)); + wasm_v128_store(dst_g, wasm_i64x2_shuffle(bg01, bg23, 1, 3)); + wasm_v128_store(dst_b, wasm_i64x2_shuffle(ra01, ra23, 0, 2)); + wasm_v128_store(dst_a, wasm_i64x2_shuffle(ra01, ra23, 1, 3)); + src_argb += 64; + dst_r += 16; + dst_g += 16; + dst_b += 16; + dst_a += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_SPLITARGBROW_WASMSIMD + +#ifdef HAS_MERGEARGBROW_WASMSIMD +void MergeARGBRow_WASMSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + const uint8_t* src_a, + uint8_t* dst_argb, + int width) { + do { + asm("" : "+r"(dst_argb)); + v128_t r = wasm_v128_load(src_r); + v128_t g = wasm_v128_load(src_g); + v128_t b = wasm_v128_load(src_b); + v128_t a = wasm_v128_load(src_a); + v128_t rg0 = wasm_i8x16_shuffle(r, g, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + v128_t rg1 = wasm_i8x16_shuffle(r, g, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31); + v128_t ba0 = wasm_i8x16_shuffle(b, a, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + v128_t ba1 = wasm_i8x16_shuffle(b, a, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31); + wasm_v128_store(dst_argb, + wasm_i16x8_shuffle(rg0, ba0, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_v128_store(dst_argb + 16, + wasm_i16x8_shuffle(rg0, ba0, 4, 12, 5, 13, 6, 14, 7, 15)); + wasm_v128_store(dst_argb + 32, + wasm_i16x8_shuffle(rg1, ba1, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_v128_store(dst_argb + 48, + wasm_i16x8_shuffle(rg1, ba1, 4, 12, 5, 13, 6, 14, 7, 15)); + src_r += 16; + src_g += 16; + src_b += 16; + src_a += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_MERGEARGBROW_WASMSIMD + +#ifdef HAS_SPLITXRGBROW_WASMSIMD +void SplitXRGBRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width) { + do { + asm("" : "+r"(src_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + v128_t s2 = wasm_v128_load(src_argb + 32); + v128_t s3 = wasm_v128_load(src_argb + 48); + v128_t bg01 = wasm_i8x16_shuffle(s0, s1, 0, 4, 8, 12, 16, 20, 24, 28, 1, 5, + 9, 13, 17, 21, 25, 29); + v128_t bg23 = wasm_i8x16_shuffle(s2, s3, 0, 4, 8, 12, 16, 20, 24, 28, 1, 5, + 9, 13, 17, 21, 25, 29); + v128_t r01 = wasm_i8x16_shuffle(s0, s1, 2, 6, 10, 14, 18, 22, 26, 30, 2, 6, + 10, 14, 18, 22, 26, 30); + v128_t r23 = wasm_i8x16_shuffle(s2, s3, 2, 6, 10, 14, 18, 22, 26, 30, 2, 6, + 10, 14, 18, 22, 26, 30); + wasm_v128_store(dst_r, wasm_i64x2_shuffle(bg01, bg23, 0, 2)); + wasm_v128_store(dst_g, wasm_i64x2_shuffle(bg01, bg23, 1, 3)); + wasm_v128_store(dst_b, wasm_i64x2_shuffle(r01, r23, 0, 2)); + src_argb += 64; + dst_r += 16; + dst_g += 16; + dst_b += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_SPLITXRGBROW_WASMSIMD + +#ifdef HAS_MERGEXRGBROW_WASMSIMD +void MergeXRGBRow_WASMSIMD(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_argb, + int width) { + v128_t a = SplatConst8(0xff); + do { + asm("" : "+r"(dst_argb)); + v128_t r = wasm_v128_load(src_r); + v128_t g = wasm_v128_load(src_g); + v128_t b = wasm_v128_load(src_b); + v128_t rg0 = wasm_i8x16_shuffle(r, g, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + v128_t rg1 = wasm_i8x16_shuffle(r, g, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31); + v128_t ba0 = wasm_i8x16_shuffle(b, a, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + v128_t ba1 = wasm_i8x16_shuffle(b, a, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31); + wasm_v128_store(dst_argb, + wasm_i16x8_shuffle(rg0, ba0, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_v128_store(dst_argb + 16, + wasm_i16x8_shuffle(rg0, ba0, 4, 12, 5, 13, 6, 14, 7, 15)); + wasm_v128_store(dst_argb + 32, + wasm_i16x8_shuffle(rg1, ba1, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_v128_store(dst_argb + 48, + wasm_i16x8_shuffle(rg1, ba1, 4, 12, 5, 13, 6, 14, 7, 15)); + src_r += 16; + src_g += 16; + src_b += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_MERGEXRGBROW_WASMSIMD + +#if defined(HAS_MIRRORROW_WASMSIMD) || defined(HAS_MIRRORROW_RELAXEDSIMD) static const uint8_t kShuffleMirror[16] = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; -void MirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { + +template +static inline void MirrorRow_Wasm(const uint8_t* src, uint8_t* dst, int width) { v128_t shuf = LoadConst(kShuffleMirror); src += width - 16; do { - wasm_v128_store(dst, LIBYUV_WASM_SWIZZLE(wasm_v128_load(src), shuf)); + wasm_v128_store(dst, WasmSwizzle(wasm_v128_load(src), shuf)); src -= 16; dst += 16; width -= 16; } while (width > 0); } + +#ifdef HAS_MIRRORROW_WASMSIMD +void MirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { + MirrorRow_Wasm(src, dst, width); +} #endif // HAS_MIRRORROW_WASMSIMD -#ifdef HAS_MIRRORUVROW_WASMSIMD +#ifdef HAS_MIRRORROW_RELAXEDSIMD +void MirrorRow_RELAXEDSIMD(const uint8_t* src, uint8_t* dst, int width) { + MirrorRow_Wasm(src, dst, width); +} +#endif // HAS_MIRRORROW_RELAXEDSIMD +#endif // HAS_MIRRORROW_WASMSIMD || HAS_MIRRORROW_RELAXEDSIMD + +#if defined(HAS_MIRRORUVROW_WASMSIMD) || defined(HAS_MIRRORUVROW_RELAXEDSIMD) static const uint8_t kShuffleMirrorUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1}; -void MirrorUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + +template +static inline void MirrorUVRow_Wasm(const uint8_t* src_uv, + uint8_t* dst_uv, + int width) { v128_t shuf = LoadConst(kShuffleMirrorUV); src_uv += (width - 8) * 2; do { - wasm_v128_store(dst_uv, LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_uv), shuf)); + wasm_v128_store(dst_uv, + WasmSwizzle(wasm_v128_load(src_uv), shuf)); src_uv -= 16; dst_uv += 16; width -= 8; } while (width > 0); } + +#ifdef HAS_MIRRORUVROW_WASMSIMD +void MirrorUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + MirrorUVRow_Wasm(src_uv, dst_uv, width); +} #endif // HAS_MIRRORUVROW_WASMSIMD +#ifdef HAS_MIRRORUVROW_RELAXEDSIMD +void MirrorUVRow_RELAXEDSIMD(const uint8_t* src_uv, + uint8_t* dst_uv, + int width) { + MirrorUVRow_Wasm(src_uv, dst_uv, width); +} +#endif // HAS_MIRRORUVROW_RELAXEDSIMD +#endif // HAS_MIRRORUVROW_WASMSIMD || HAS_MIRRORUVROW_RELAXEDSIMD + #ifdef HAS_SWAPUVROW_WASMSIMD void SwapUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_vu, int width) { do { @@ -909,20 +1673,23 @@ void SwapUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_vu, int width) { } #endif // HAS_SWAPUVROW_WASMSIMD -#ifdef HAS_MIRRORSPLITUVROW_WASMSIMD +#if defined(HAS_MIRRORSPLITUVROW_WASMSIMD) || \ + defined(HAS_MIRRORSPLITUVROW_RELAXEDSIMD) // Reverses 8 UV pairs into 8 U then 8 V. static const uint8_t kShuffleMirrorSplitUV[16] = {14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1}; -void MirrorSplitUVRow_WASMSIMD(const uint8_t* src_uv, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { + +template +static inline void MirrorSplitUVRow_Wasm(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { v128_t shuf = LoadConst(kShuffleMirrorSplitUV); src_uv += (width - 16) * 2; do { asm("" : "+r"(src_uv)); - v128_t a = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_uv), shuf); - v128_t b = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_uv + 16), shuf); + v128_t a = WasmSwizzle(wasm_v128_load(src_uv), shuf); + v128_t b = WasmSwizzle(wasm_v128_load(src_uv + 16), shuf); wasm_v128_store(dst_u, wasm_i64x2_shuffle(b, a, 0, 2)); wasm_v128_store(dst_v, wasm_i64x2_shuffle(b, a, 1, 3)); src_uv -= 32; @@ -931,9 +1698,28 @@ void MirrorSplitUVRow_WASMSIMD(const uint8_t* src_uv, width -= 16; } while (width > 0); } + +#ifdef HAS_MIRRORSPLITUVROW_WASMSIMD +void MirrorSplitUVRow_WASMSIMD(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + MirrorSplitUVRow_Wasm(src_uv, dst_u, dst_v, width); +} #endif // HAS_MIRRORSPLITUVROW_WASMSIMD -#ifdef HAS_RGB24MIRRORROW_WASMSIMD +#ifdef HAS_MIRRORSPLITUVROW_RELAXEDSIMD +void MirrorSplitUVRow_RELAXEDSIMD(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + MirrorSplitUVRow_Wasm(src_uv, dst_u, dst_v, width); +} +#endif // HAS_MIRRORSPLITUVROW_RELAXEDSIMD +#endif // HAS_MIRRORSPLITUVROW_WASMSIMD || HAS_MIRRORSPLITUVROW_RELAXEDSIMD + +#if defined(HAS_RGB24MIRRORROW_WASMSIMD) || \ + defined(HAS_RGB24MIRRORROW_RELAXEDSIMD) // Each 16 byte output needs 18 source bytes, so it is the OR of 2 swizzles. static const uint8_t kShuffleRGB24Mirror[96] = { 13, 14, 15, 10, 11, 12, 7, 8, 9, 4, 5, 6, 1, 2, @@ -943,9 +1729,11 @@ static const uint8_t kShuffleRGB24Mirror[96] = { 128, 128, 128, 128, 128, 128, 0, 128, 128, 12, 13, 14, 9, 10, 11, 6, 7, 8, 3, 4, 5, 0, 1, 2, 1, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}; -void RGB24MirrorRow_WASMSIMD(const uint8_t* src_rgb24, - uint8_t* dst_rgb24, - int width) { + +template +static inline void RGB24MirrorRow_Wasm(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width) { const uint8_t* shuf = kShuffleRGB24Mirror; v128_t m0a = LoadConst(shuf); v128_t m0b = LoadConst(shuf + 16); @@ -962,20 +1750,37 @@ void RGB24MirrorRow_WASMSIMD(const uint8_t* src_rgb24, // Bytes 15 and 32 of the 48 source bytes. v128_t s02 = wasm_i8x16_shuffle(s0, s2, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30); - wasm_v128_store(dst_rgb24, wasm_v128_or(LIBYUV_WASM_SWIZZLE(s2, m0a), - LIBYUV_WASM_SWIZZLE(s1, m0b))); + wasm_v128_store(dst_rgb24, wasm_v128_or(WasmSwizzle(s2, m0a), + WasmSwizzle(s1, m0b))); wasm_v128_store(dst_rgb24 + 16, - wasm_v128_or(LIBYUV_WASM_SWIZZLE(s1, m1a), - LIBYUV_WASM_SWIZZLE(s02, m1b))); - wasm_v128_store(dst_rgb24 + 32, wasm_v128_or(LIBYUV_WASM_SWIZZLE(s0, m2a), - LIBYUV_WASM_SWIZZLE(s1, m2b))); + wasm_v128_or(WasmSwizzle(s1, m1a), + WasmSwizzle(s02, m1b))); + wasm_v128_store(dst_rgb24 + 32, + wasm_v128_or(WasmSwizzle(s0, m2a), + WasmSwizzle(s1, m2b))); src_rgb24 -= 48; dst_rgb24 += 48; width -= 16; } while (width > 0); } + +#ifdef HAS_RGB24MIRRORROW_WASMSIMD +void RGB24MirrorRow_WASMSIMD(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width) { + RGB24MirrorRow_Wasm(src_rgb24, dst_rgb24, width); +} #endif // HAS_RGB24MIRRORROW_WASMSIMD +#ifdef HAS_RGB24MIRRORROW_RELAXEDSIMD +void RGB24MirrorRow_RELAXEDSIMD(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width) { + RGB24MirrorRow_Wasm(src_rgb24, dst_rgb24, width); +} +#endif // HAS_RGB24MIRRORROW_RELAXEDSIMD +#endif // HAS_RGB24MIRRORROW_WASMSIMD || HAS_RGB24MIRRORROW_RELAXEDSIMD + #ifdef HAS_ARGBMIRRORROW_WASMSIMD void ARGBMirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { src += (width - 4) * 4; @@ -1095,15 +1900,24 @@ void InterpolateRow_16_WASMSIMD(uint16_t* dst_ptr, } #endif // HAS_INTERPOLATEROW_16_WASMSIMD -#ifdef HAS_J400TOARGBROW_WASMSIMD +#if defined(HAS_J400TOARGBROW_WASMSIMD) || \ + defined(HAS_J400TOARGBROW_RELAXEDSIMD) || \ + defined(HAS_I400TOARGBROW_WASMSIMD) || \ + defined(HAS_I400TOARGBROW_RELAXEDSIMD) || \ + defined(HAS_SOBELROW_WASMSIMD) || defined(HAS_SOBELROW_RELAXEDSIMD) static const uint8_t kShuffleJ400ToARGB[64] = { 0, 0, 0, 128, 1, 1, 1, 128, 2, 2, 2, 128, 3, 3, 3, 128, 4, 4, 4, 128, 5, 5, 5, 128, 6, 6, 6, 128, 7, 7, 7, 128, 8, 8, 8, 128, 9, 9, 9, 128, 10, 10, 10, 128, 11, 11, 11, 128, 12, 12, 12, 128, 13, 13, 13, 128, 14, 14, 14, 128, 15, 15, 15, 128}; -void J400ToARGBRow_WASMSIMD(const uint8_t* src_y, - uint8_t* dst_argb, - int width) { +#endif + +#if defined(HAS_J400TOARGBROW_WASMSIMD) || \ + defined(HAS_J400TOARGBROW_RELAXEDSIMD) +template +static inline void J400ToARGBRow_Wasm(const uint8_t* src_y, + uint8_t* dst_argb, + int width) { v128_t shuf0 = LoadConst(kShuffleJ400ToARGB); v128_t shuf1 = LoadConst(kShuffleJ400ToARGB + 16); v128_t shuf2 = LoadConst(kShuffleJ400ToARGB + 32); @@ -1113,20 +1927,106 @@ void J400ToARGBRow_WASMSIMD(const uint8_t* src_y, asm("" : "+r"(dst_argb)); v128_t y = wasm_v128_load(src_y); wasm_v128_store(dst_argb, - wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf0), alpha)); + wasm_v128_or(WasmSwizzle(y, shuf0), alpha)); wasm_v128_store(dst_argb + 16, - wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf1), alpha)); + wasm_v128_or(WasmSwizzle(y, shuf1), alpha)); wasm_v128_store(dst_argb + 32, - wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf2), alpha)); + wasm_v128_or(WasmSwizzle(y, shuf2), alpha)); wasm_v128_store(dst_argb + 48, - wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf3), alpha)); + wasm_v128_or(WasmSwizzle(y, shuf3), alpha)); src_y += 16; dst_argb += 64; width -= 16; } while (width > 0); } + +#ifdef HAS_J400TOARGBROW_WASMSIMD +void J400ToARGBRow_WASMSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + int width) { + J400ToARGBRow_Wasm(src_y, dst_argb, width); +} #endif // HAS_J400TOARGBROW_WASMSIMD +#ifdef HAS_J400TOARGBROW_RELAXEDSIMD +void J400ToARGBRow_RELAXEDSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + int width) { + J400ToARGBRow_Wasm(src_y, dst_argb, width); +} +#endif // HAS_J400TOARGBROW_RELAXEDSIMD +#endif // HAS_J400TOARGBROW_WASMSIMD || HAS_J400TOARGBROW_RELAXEDSIMD + +#if defined(HAS_I400TOARGBROW_WASMSIMD) || \ + defined(HAS_I400TOARGBROW_RELAXEDSIMD) +template +static inline void I400ToARGBRow_Wasm(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + v128_t shuf0 = LoadConst(kShuffleJ400ToARGB); + v128_t shuf1 = LoadConst(kShuffleJ400ToARGB + 16); + v128_t shuf2 = LoadConst(kShuffleJ400ToARGB + 32); + v128_t shuf3 = LoadConst(kShuffleJ400ToARGB + 48); + v128_t yg = wasm_i16x8_splat(yuvconstants->kYToRgb[0]); + v128_t yb = wasm_i16x8_splat(yuvconstants->kYBiasToRgb[0]); + v128_t k0101 = SplatConst16(0x0101); + v128_t alpha = SplatConst32((int)0xff000000); + do { + asm("" : "+r"(dst_argb)); + asm volatile("" : "+r"(yg)); + v128_t y = wasm_v128_load(src_y); + v128_t y0 = wasm_i16x8_mul(wasm_u16x8_extend_low_u8x16(y), k0101); + v128_t y1 = wasm_i16x8_mul(wasm_u16x8_extend_high_u8x16(y), k0101); + v128_t lo0 = wasm_u32x4_extmul_low_u16x8(y0, yg); + v128_t hi0 = wasm_u32x4_extmul_high_u16x8(y0, yg); + v128_t lo1 = wasm_u32x4_extmul_low_u16x8(y1, yg); + v128_t hi1 = wasm_u32x4_extmul_high_u16x8(y1, yg); + y0 = wasm_i16x8_shr( + wasm_i16x8_add(wasm_i16x8_shuffle(lo0, hi0, 1, 3, 5, 7, 9, 11, 13, 15), + yb), + 6); + y1 = wasm_i16x8_shr( + wasm_i16x8_add(wasm_i16x8_shuffle(lo1, hi1, 1, 3, 5, 7, 9, 11, 13, 15), + yb), + 6); + v128_t y_scaled = wasm_u8x16_narrow_i16x8(y0, y1); + wasm_v128_store( + dst_argb, wasm_v128_or(WasmSwizzle(y_scaled, shuf0), alpha)); + wasm_v128_store( + dst_argb + 16, + wasm_v128_or(WasmSwizzle(y_scaled, shuf1), alpha)); + wasm_v128_store( + dst_argb + 32, + wasm_v128_or(WasmSwizzle(y_scaled, shuf2), alpha)); + wasm_v128_store( + dst_argb + 48, + wasm_v128_or(WasmSwizzle(y_scaled, shuf3), alpha)); + src_y += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} + +#ifdef HAS_I400TOARGBROW_WASMSIMD +void I400ToARGBRow_WASMSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + I400ToARGBRow_Wasm(src_y, dst_argb, yuvconstants, width); +} +#endif // HAS_I400TOARGBROW_WASMSIMD + +#ifdef HAS_I400TOARGBROW_RELAXEDSIMD +void I400ToARGBRow_RELAXEDSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + I400ToARGBRow_Wasm(src_y, dst_argb, yuvconstants, width); +} +#endif // HAS_I400TOARGBROW_RELAXEDSIMD +#endif // HAS_I400TOARGBROW_WASMSIMD || HAS_I400TOARGBROW_RELAXEDSIMD + #ifdef HAS_RGB565TOARGBROW_WASMSIMD void RGB565ToARGBRow_WASMSIMD(const uint8_t* src_rgb565, uint8_t* dst_argb, @@ -1332,38 +2232,752 @@ void ARGBToARGB4444Row_WASMSIMD(const uint8_t* src_argb, } #endif // HAS_ARGBTOARGB4444ROW_WASMSIMD -#ifdef HAS_ARGBSHUFFLEROW_WASMSIMD -void ARGBShuffleRow_WASMSIMD(const uint8_t* src_argb, - uint8_t* dst_argb, - const uint8_t* shuffler, - int width) { +#if defined(HAS_ARGBSHUFFLEROW_WASMSIMD) || \ + defined(HAS_ARGBSHUFFLEROW_RELAXEDSIMD) +template +static inline void ARGBShuffleRow_Wasm(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width) { v128_t shuf = wasm_v128_load(shuffler); while (width >= 8) { asm("" : "+r"(src_argb), "+r"(dst_argb)); v128_t s0 = wasm_v128_load(src_argb); v128_t s1 = wasm_v128_load(src_argb + 16); - wasm_v128_store(dst_argb, LIBYUV_WASM_SWIZZLE(s0, shuf)); - wasm_v128_store(dst_argb + 16, LIBYUV_WASM_SWIZZLE(s1, shuf)); + wasm_v128_store(dst_argb, WasmSwizzle(s0, shuf)); + wasm_v128_store(dst_argb + 16, WasmSwizzle(s1, shuf)); src_argb += 32; dst_argb += 32; width -= 8; } if (width > 0) { v128_t s0 = wasm_v128_load(src_argb); - wasm_v128_store(dst_argb, LIBYUV_WASM_SWIZZLE(s0, shuf)); + wasm_v128_store(dst_argb, WasmSwizzle(s0, shuf)); } } + +#ifdef HAS_ARGBSHUFFLEROW_WASMSIMD +void ARGBShuffleRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width) { + ARGBShuffleRow_Wasm(src_argb, dst_argb, shuffler, width); +} #endif // HAS_ARGBSHUFFLEROW_WASMSIMD -#if defined(HAS_I422TOARGBROW_WASMSIMD) || \ - defined(HAS_I422TOAR30ROW_WASMSIMD) || \ - defined(HAS_NV12TOARGBROW_WASMSIMD) || \ - defined(HAS_NV21TOARGBROW_WASMSIMD) || \ - defined(HAS_I422TORGB24ROW_WASMSIMD) || \ - defined(HAS_I422TORGB565ROW_WASMSIMD) || \ - defined(HAS_I422TOARGB1555ROW_WASMSIMD) || \ - defined(HAS_I422TOARGB4444ROW_WASMSIMD) || \ - defined(HAS_NV12TORGB565ROW_WASMSIMD) +#ifdef HAS_ARGBSHUFFLEROW_RELAXEDSIMD +void ARGBShuffleRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width) { + ARGBShuffleRow_Wasm(src_argb, dst_argb, shuffler, width); +} +#endif // HAS_ARGBSHUFFLEROW_RELAXEDSIMD +#endif // HAS_ARGBSHUFFLEROW_WASMSIMD || HAS_ARGBSHUFFLEROW_RELAXEDSIMD + +#ifdef HAS_COPYROW_WASMSIMD +void CopyRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int count) { + do { + asm("" : "+r"(src), "+r"(dst)); + v128_t v0 = wasm_v128_load(src); + v128_t v1 = wasm_v128_load(src + 16); + v128_t v2 = wasm_v128_load(src + 32); + v128_t v3 = wasm_v128_load(src + 48); + wasm_v128_store(dst, v0); + wasm_v128_store(dst + 16, v1); + wasm_v128_store(dst + 32, v2); + wasm_v128_store(dst + 48, v3); + src += 64; + dst += 64; + count -= 64; + } while (count > 0); +} +#endif // HAS_COPYROW_WASMSIMD + +#ifdef HAS_COPYROW_16_WASMSIMD +void CopyRow_16_WASMSIMD(const uint16_t* src, uint16_t* dst, int count) { + do { + asm("" : "+r"(src), "+r"(dst)); + v128_t v0 = wasm_v128_load(src); + v128_t v1 = wasm_v128_load(src + 8); + v128_t v2 = wasm_v128_load(src + 16); + v128_t v3 = wasm_v128_load(src + 24); + wasm_v128_store(dst, v0); + wasm_v128_store(dst + 8, v1); + wasm_v128_store(dst + 16, v2); + wasm_v128_store(dst + 24, v3); + src += 32; + dst += 32; + count -= 32; + } while (count > 0); +} +#endif // HAS_COPYROW_16_WASMSIMD + +#ifdef HAS_SETROW_WASMSIMD +void SetRow_WASMSIMD(uint8_t* dst, uint8_t v8, int width) { + v128_t v = wasm_i8x16_splat(v8); + do { + wasm_v128_store(dst, v); + dst += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_SETROW_WASMSIMD + +#ifdef HAS_ARGBSETROW_WASMSIMD +void ARGBSetRow_WASMSIMD(uint8_t* dst_argb, uint32_t v32, int width) { + v128_t v = wasm_i32x4_splat((int32_t)v32); + do { + wasm_v128_store(dst_argb, v); + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBSETROW_WASMSIMD + +#ifdef HAS_HALFROW_WASMSIMD +void HalfRow_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width) { + const uint8_t* src_uv1 = src_uv + src_uv_stride; + do { + wasm_v128_store(dst_uv, wasm_u8x16_avgr(wasm_v128_load(src_uv), + wasm_v128_load(src_uv1))); + src_uv += 16; + src_uv1 += 16; + dst_uv += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_HALFROW_WASMSIMD + +#ifdef HAS_HALFROW_16_WASMSIMD +void HalfRow_16_WASMSIMD(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + do { + wasm_v128_store(dst_uv, wasm_u16x8_avgr(wasm_v128_load(src_uv), + wasm_v128_load(src_uv1))); + src_uv += 8; + src_uv1 += 8; + dst_uv += 8; + width -= 8; + } while (width > 0); +} +#endif // HAS_HALFROW_16_WASMSIMD + +#ifdef HAS_HALFROW_16TO8_WASMSIMD +void HalfRow_16To8_WASMSIMD(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + v128_t vscale = wasm_i16x8_splat(scale); + do { + asm("" : "+r"(src_uv), "+r"(src_uv1)); + v128_t a0 = + wasm_u16x8_avgr(wasm_v128_load(src_uv), wasm_v128_load(src_uv1)); + v128_t a1 = wasm_u16x8_avgr(wasm_v128_load(src_uv + 8), + wasm_v128_load(src_uv1 + 8)); + v128_t s0 = wasm_i16x8_shuffle(wasm_u32x4_extmul_low_u16x8(a0, vscale), + wasm_u32x4_extmul_high_u16x8(a0, vscale), 1, + 3, 5, 7, 9, 11, 13, 15); + v128_t s1 = wasm_i16x8_shuffle(wasm_u32x4_extmul_low_u16x8(a1, vscale), + wasm_u32x4_extmul_high_u16x8(a1, vscale), 1, + 3, 5, 7, 9, 11, 13, 15); + wasm_v128_store(dst_uv, wasm_u8x16_narrow_i16x8(s0, s1)); + src_uv += 16; + src_uv1 += 16; + dst_uv += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_HALFROW_16TO8_WASMSIMD + +#ifdef HAS_HALFWIDTHROW_16TO8_WASMSIMD +void HalfWidthRow_16To8_WASMSIMD(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const uint16_t* src_uv1 = src_uv + src_uv_stride; + v128_t vscale = wasm_i16x8_splat(scale); + do { + asm("" : "+r"(src_uv), "+r"(src_uv1)); + v128_t r0a = wasm_v128_load(src_uv); + v128_t r0b = wasm_v128_load(src_uv + 8); + v128_t r1a = wasm_v128_load(src_uv1); + v128_t r1b = wasm_v128_load(src_uv1 + 8); + v128_t sum0 = wasm_i32x4_add(wasm_u32x4_extadd_pairwise_u16x8(r0a), + wasm_u32x4_extadd_pairwise_u16x8(r1a)); + v128_t sum1 = wasm_i32x4_add(wasm_u32x4_extadd_pairwise_u16x8(r0b), + wasm_u32x4_extadd_pairwise_u16x8(r1b)); + v128_t avg0 = wasm_u32x4_shr(wasm_i32x4_add(sum0, SplatConst32(2)), 2); + v128_t avg1 = wasm_u32x4_shr(wasm_i32x4_add(sum1, SplatConst32(2)), 2); + v128_t avg = wasm_i16x8_shuffle(avg0, avg1, 0, 2, 4, 6, 8, 10, 12, 14); + v128_t scaled = wasm_i16x8_shuffle( + wasm_u32x4_extmul_low_u16x8(avg, vscale), + wasm_u32x4_extmul_high_u16x8(avg, vscale), 1, 3, 5, 7, 9, 11, 13, 15); + wasm_v128_store64_lane(dst_uv, wasm_u8x16_narrow_i16x8(scaled, scaled), 0); + src_uv += 16; + src_uv1 += 16; + dst_uv += 8; + width -= 8; + } while (width > 0); +} +#endif // HAS_HALFWIDTHROW_16TO8_WASMSIMD + +#ifdef HAS_CONVERT8TO8ROW_WASMSIMD +void Convert8To8Row_WASMSIMD(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width) { + v128_t vscale = wasm_i8x16_splat(scale); + v128_t vbias = wasm_i16x8_splat(bias); + do { + v128_t y = wasm_v128_load(src_y); + v128_t s0 = wasm_i16x8_add( + wasm_u16x8_shr(wasm_u16x8_extmul_low_u8x16(y, vscale), 8), vbias); + v128_t s1 = wasm_i16x8_add( + wasm_u16x8_shr(wasm_u16x8_extmul_high_u8x16(y, vscale), 8), vbias); + wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(s0, s1)); + src_y += 16; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_CONVERT8TO8ROW_WASMSIMD + +#ifdef HAS_I422TOYUY2ROW_WASMSIMD +void I422ToYUY2Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_yuy2, + int width) { + do { + asm("" : "+r"(dst_yuy2)); + v128_t y = wasm_v128_load(src_y); + v128_t u = wasm_v128_load64_zero(src_u); + v128_t v = wasm_v128_load64_zero(src_v); + v128_t uv = wasm_i8x16_shuffle(u, v, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + wasm_v128_store(dst_yuy2, + wasm_i8x16_shuffle(y, uv, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, + 5, 21, 6, 22, 7, 23)); + wasm_v128_store(dst_yuy2 + 16, + wasm_i8x16_shuffle(y, uv, 8, 24, 9, 25, 10, 26, 11, 27, 12, + 28, 13, 29, 14, 30, 15, 31)); + src_y += 16; + src_u += 8; + src_v += 8; + dst_yuy2 += 32; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422TOYUY2ROW_WASMSIMD + +#ifdef HAS_I422TOUYVYROW_WASMSIMD +void I422ToUYVYRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uyvy, + int width) { + do { + asm("" : "+r"(dst_uyvy)); + v128_t y = wasm_v128_load(src_y); + v128_t u = wasm_v128_load64_zero(src_u); + v128_t v = wasm_v128_load64_zero(src_v); + v128_t uv = wasm_i8x16_shuffle(u, v, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + wasm_v128_store(dst_uyvy, + wasm_i8x16_shuffle(uv, y, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, + 5, 21, 6, 22, 7, 23)); + wasm_v128_store(dst_uyvy + 16, + wasm_i8x16_shuffle(uv, y, 8, 24, 9, 25, 10, 26, 11, 27, 12, + 28, 13, 29, 14, 30, 15, 31)); + src_y += 16; + src_u += 8; + src_v += 8; + dst_uyvy += 32; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422TOUYVYROW_WASMSIMD + +#ifdef HAS_YUY2TONVUVROW_WASMSIMD +void YUY2ToNVUVRow_WASMSIMD(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_uv, + int width) { + const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2; + do { + asm("" : "+r"(src_yuy2), "+r"(src_yuy2_1)); + v128_t r0 = + wasm_u8x16_avgr(wasm_v128_load(src_yuy2), wasm_v128_load(src_yuy2_1)); + v128_t r1 = wasm_u8x16_avgr(wasm_v128_load(src_yuy2 + 16), + wasm_v128_load(src_yuy2_1 + 16)); + wasm_v128_store( + dst_uv, wasm_i8x16_shuffle(r0, r1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, + 21, 23, 25, 27, 29, 31)); + src_yuy2 += 32; + src_yuy2_1 += 32; + dst_uv += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_YUY2TONVUVROW_WASMSIMD + +#ifdef HAS_AYUVTOYROW_WASMSIMD +void AYUVToYRow_WASMSIMD(const uint8_t* src_ayuv, uint8_t* dst_y, int width) { + do { + asm("" : "+r"(src_ayuv)); + v128_t s0 = wasm_v128_load(src_ayuv); + v128_t s1 = wasm_v128_load(src_ayuv + 16); + v128_t s2 = wasm_v128_load(src_ayuv + 32); + v128_t s3 = wasm_v128_load(src_ayuv + 48); + v128_t y01 = wasm_i8x16_shuffle(s0, s1, 2, 6, 10, 14, 18, 22, 26, 30, 2, 6, + 10, 14, 18, 22, 26, 30); + v128_t y23 = wasm_i8x16_shuffle(s2, s3, 2, 6, 10, 14, 18, 22, 26, 30, 2, 6, + 10, 14, 18, 22, 26, 30); + wasm_v128_store(dst_y, wasm_i64x2_shuffle(y01, y23, 0, 2)); + src_ayuv += 64; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_AYUVTOYROW_WASMSIMD + +#if defined(HAS_AYUVTOUVROW_WASMSIMD) || \ + defined(HAS_AYUVTOUVROW_RELAXEDSIMD) || \ + defined(HAS_AYUVTOVUROW_WASMSIMD) || defined(HAS_AYUVTOVUROW_RELAXEDSIMD) +static const uint8_t kShuffleAYUVToUV[16] = {1, 0, 5, 4, 9, 8, 13, 12, + 1, 0, 5, 4, 9, 8, 13, 12}; +static const uint8_t kShuffleAYUVToVU[16] = {0, 1, 4, 5, 8, 9, 12, 13, + 0, 1, 4, 5, 8, 9, 12, 13}; + +template +static inline void AYUVToUVRowImpl_Wasm(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width, + const uint8_t* shuf_table) { + const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv; + v128_t shuf = LoadConst(shuf_table); + do { + asm("" : "+r"(src_ayuv), "+r"(src_ayuv1)); + v128_t r00 = wasm_v128_load(src_ayuv); + v128_t r01 = wasm_v128_load(src_ayuv + 16); + v128_t r02 = wasm_v128_load(src_ayuv + 32); + v128_t r03 = wasm_v128_load(src_ayuv + 48); + v128_t r10 = wasm_v128_load(src_ayuv1); + v128_t r11 = wasm_v128_load(src_ayuv1 + 16); + v128_t r12 = wasm_v128_load(src_ayuv1 + 32); + v128_t r13 = wasm_v128_load(src_ayuv1 + 48); + v128_t uv00 = WasmSwizzle(r00, shuf); + v128_t uv01 = WasmSwizzle(r01, shuf); + v128_t uv02 = WasmSwizzle(r02, shuf); + v128_t uv03 = WasmSwizzle(r03, shuf); + v128_t uv10 = WasmSwizzle(r10, shuf); + v128_t uv11 = WasmSwizzle(r11, shuf); + v128_t uv12 = WasmSwizzle(r12, shuf); + v128_t uv13 = WasmSwizzle(r13, shuf); + v128_t row0_a = wasm_i64x2_shuffle(uv00, uv01, 0, 2); + v128_t row0_b = wasm_i64x2_shuffle(uv02, uv03, 0, 2); + v128_t row1_a = wasm_i64x2_shuffle(uv10, uv11, 0, 2); + v128_t row1_b = wasm_i64x2_shuffle(uv12, uv13, 0, 2); + v128_t even_0 = + wasm_i16x8_shuffle(row0_a, row0_b, 0, 2, 4, 6, 8, 10, 12, 14); + v128_t odd_0 = + wasm_i16x8_shuffle(row0_a, row0_b, 1, 3, 5, 7, 9, 11, 13, 15); + v128_t even_1 = + wasm_i16x8_shuffle(row1_a, row1_b, 0, 2, 4, 6, 8, 10, 12, 14); + v128_t odd_1 = + wasm_i16x8_shuffle(row1_a, row1_b, 1, 3, 5, 7, 9, 11, 13, 15); + v128_t s_lo = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_low_u8x16(even_0), + wasm_u16x8_extend_low_u8x16(odd_0)), + wasm_i16x8_add(wasm_u16x8_extend_low_u8x16(even_1), + wasm_u16x8_extend_low_u8x16(odd_1))); + v128_t s_hi = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_high_u8x16(even_0), + wasm_u16x8_extend_high_u8x16(odd_0)), + wasm_i16x8_add(wasm_u16x8_extend_high_u8x16(even_1), + wasm_u16x8_extend_high_u8x16(odd_1))); + s_lo = wasm_u16x8_shr(wasm_i16x8_add(s_lo, SplatConst16(2)), 2); + s_hi = wasm_u16x8_shr(wasm_i16x8_add(s_hi, SplatConst16(2)), 2); + wasm_v128_store(dst_uv, wasm_u8x16_narrow_i16x8(s_lo, s_hi)); + src_ayuv += 64; + src_ayuv1 += 64; + dst_uv += 16; + width -= 16; + } while (width > 0); +} + +#ifdef HAS_AYUVTOUVROW_WASMSIMD +void AYUVToUVRow_WASMSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + AYUVToUVRowImpl_Wasm(src_ayuv, src_stride_ayuv, dst_uv, width, + kShuffleAYUVToUV); +} +#endif // HAS_AYUVTOUVROW_WASMSIMD + +#ifdef HAS_AYUVTOUVROW_RELAXEDSIMD +void AYUVToUVRow_RELAXEDSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + AYUVToUVRowImpl_Wasm(src_ayuv, src_stride_ayuv, dst_uv, width, + kShuffleAYUVToUV); +} +#endif // HAS_AYUVTOUVROW_RELAXEDSIMD + +#ifdef HAS_AYUVTOVUROW_WASMSIMD +void AYUVToVURow_WASMSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + AYUVToUVRowImpl_Wasm(src_ayuv, src_stride_ayuv, dst_vu, width, + kShuffleAYUVToVU); +} +#endif // HAS_AYUVTOVUROW_WASMSIMD + +#ifdef HAS_AYUVTOVUROW_RELAXEDSIMD +void AYUVToVURow_RELAXEDSIMD(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + AYUVToUVRowImpl_Wasm(src_ayuv, src_stride_ayuv, dst_vu, width, + kShuffleAYUVToVU); +} +#endif // HAS_AYUVTOVUROW_RELAXEDSIMD +#endif // HAS_AYUVTOUVROW_* || HAS_AYUVTOVUROW_* + +#ifdef HAS_DETILEROW_WASMSIMD +void DetileRow_WASMSIMD(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width) { + do { + wasm_v128_store(dst, wasm_v128_load(src)); + src += src_tile_stride; + dst += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_DETILEROW_WASMSIMD + +#ifdef HAS_DETILEROW_16_WASMSIMD +void DetileRow_16_WASMSIMD(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width) { + do { + asm("" : "+r"(dst)); + v128_t v0 = wasm_v128_load(src); + v128_t v1 = wasm_v128_load(src + 8); + wasm_v128_store(dst, v0); + wasm_v128_store(dst + 8, v1); + src += src_tile_stride; + dst += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_DETILEROW_16_WASMSIMD + +#ifdef HAS_DETILESPLITUVROW_WASMSIMD +void DetileSplitUVRow_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + do { + v128_t uv = wasm_v128_load(src_uv); + v128_t u_v = wasm_i8x16_shuffle(uv, uv, 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, + 7, 9, 11, 13, 15); + wasm_v128_store64_lane(dst_u, u_v, 0); + wasm_v128_store64_lane(dst_v, u_v, 1); + src_uv += src_tile_stride; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} +#endif // HAS_DETILESPLITUVROW_WASMSIMD + +#ifdef HAS_DETILETOYUY2_WASMSIMD +void DetileToYUY2_WASMSIMD(const uint8_t* src_y, + ptrdiff_t src_y_tile_stride, + const uint8_t* src_uv, + ptrdiff_t src_uv_tile_stride, + uint8_t* dst_yuy2, + int width) { + do { + asm("" : "+r"(dst_yuy2)); + v128_t y = wasm_v128_load(src_y); + v128_t uv = wasm_v128_load(src_uv); + wasm_v128_store(dst_yuy2, + wasm_i8x16_shuffle(y, uv, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, + 5, 21, 6, 22, 7, 23)); + wasm_v128_store(dst_yuy2 + 16, + wasm_i8x16_shuffle(y, uv, 8, 24, 9, 25, 10, 26, 11, 27, 12, + 28, 13, 29, 14, 30, 15, 31)); + src_y += src_y_tile_stride; + src_uv += src_uv_tile_stride; + dst_yuy2 += 32; + width -= 16; + } while (width > 0); +} +#endif // HAS_DETILETOYUY2_WASMSIMD + +#if defined(HAS_AB64TOARGBROW_WASMSIMD) || defined(HAS_AR64TOARGBROW_WASMSIMD) +void AR64ToARGBRow_WASMSIMD(const uint16_t* src_ar64, + uint8_t* dst_argb, + int width) { + do { + asm("" : "+r"(src_ar64), "+r"(dst_argb)); + v128_t s0 = wasm_v128_load(src_ar64); + v128_t s1 = wasm_v128_load(src_ar64 + 8); + v128_t s2 = wasm_v128_load(src_ar64 + 16); + v128_t s3 = wasm_v128_load(src_ar64 + 24); + wasm_v128_store( + dst_argb, wasm_i8x16_shuffle(s0, s1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, + 21, 23, 25, 27, 29, 31)); + wasm_v128_store(dst_argb + 16, + wasm_i8x16_shuffle(s2, s3, 1, 3, 5, 7, 9, 11, 13, 15, 17, + 19, 21, 23, 25, 27, 29, 31)); + src_ar64 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); +} + +void AB64ToARGBRow_WASMSIMD(const uint16_t* src_ab64, + uint8_t* dst_argb, + int width) { + do { + asm("" : "+r"(src_ab64), "+r"(dst_argb)); + v128_t s0 = wasm_v128_load(src_ab64); + v128_t s1 = wasm_v128_load(src_ab64 + 8); + v128_t s2 = wasm_v128_load(src_ab64 + 16); + v128_t s3 = wasm_v128_load(src_ab64 + 24); + wasm_v128_store( + dst_argb, wasm_i8x16_shuffle(s0, s1, 5, 3, 1, 7, 13, 11, 9, 15, 21, 19, + 17, 23, 29, 27, 25, 31)); + wasm_v128_store(dst_argb + 16, + wasm_i8x16_shuffle(s2, s3, 5, 3, 1, 7, 13, 11, 9, 15, 21, + 19, 17, 23, 29, 27, 25, 31)); + src_ab64 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); +} +#endif // HAS_AB64TOARGBROW_WASMSIMD || HAS_AR64TOARGBROW_WASMSIMD + +#if defined(HAS_ARGBTOAR64ROW_WASMSIMD) || defined(HAS_ARGBTOAB64ROW_WASMSIMD) +void ARGBToAR64Row_WASMSIMD(const uint8_t* src_argb, + uint16_t* dst_ar64, + int width) { + do { + asm("" : "+r"(src_argb), "+r"(dst_ar64)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + wasm_v128_store(dst_ar64, wasm_i8x16_shuffle(s0, s0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7)); + wasm_v128_store(dst_ar64 + 8, + wasm_i8x16_shuffle(s0, s0, 8, 8, 9, 9, 10, 10, 11, 11, 12, + 12, 13, 13, 14, 14, 15, 15)); + wasm_v128_store(dst_ar64 + 16, + wasm_i8x16_shuffle(s1, s1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, + 5, 6, 6, 7, 7)); + wasm_v128_store(dst_ar64 + 24, + wasm_i8x16_shuffle(s1, s1, 8, 8, 9, 9, 10, 10, 11, 11, 12, + 12, 13, 13, 14, 14, 15, 15)); + src_argb += 32; + dst_ar64 += 32; + width -= 8; + } while (width > 0); +} + +void ARGBToAB64Row_WASMSIMD(const uint8_t* src_argb, + uint16_t* dst_ab64, + int width) { + do { + asm("" : "+r"(src_argb), "+r"(dst_ab64)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + wasm_v128_store(dst_ab64, wasm_i8x16_shuffle(s0, s0, 2, 2, 1, 1, 0, 0, 3, 3, + 6, 6, 5, 5, 4, 4, 7, 7)); + wasm_v128_store(dst_ab64 + 8, + wasm_i8x16_shuffle(s0, s0, 10, 10, 9, 9, 8, 8, 11, 11, 14, + 14, 13, 13, 12, 12, 15, 15)); + wasm_v128_store(dst_ab64 + 16, + wasm_i8x16_shuffle(s1, s1, 2, 2, 1, 1, 0, 0, 3, 3, 6, 6, 5, + 5, 4, 4, 7, 7)); + wasm_v128_store(dst_ab64 + 24, + wasm_i8x16_shuffle(s1, s1, 10, 10, 9, 9, 8, 8, 11, 11, 14, + 14, 13, 13, 12, 12, 15, 15)); + src_argb += 32; + dst_ab64 += 32; + width -= 8; + } while (width > 0); +} +#endif // HAS_ARGBTOAR64ROW_WASMSIMD || HAS_ARGBTOAB64ROW_WASMSIMD + +#if defined(HAS_AR30TOARGBROW_WASMSIMD) || \ + defined(HAS_AR30TOABGRROW_WASMSIMD) || defined(HAS_AR30TOAB30ROW_WASMSIMD) +void AR30ToARGBRow_WASMSIMD(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width) { + v128_t mask_b = SplatConst32(0x000000ff); + v128_t mask_g = SplatConst32(0x0000ff00); + v128_t mask_r = SplatConst32(0x00ff0000); + v128_t mask_a3 = SplatConst32((int)0xc0000000); + v128_t mul_a = SplatConst32(0x01550000); + do { + v128_t v = wasm_v128_load(src_ar30); + v128_t b = wasm_v128_and(wasm_u32x4_shr(v, 2), mask_b); + v128_t g = wasm_v128_and(wasm_u32x4_shr(v, 4), mask_g); + v128_t r = wasm_v128_and(wasm_u32x4_shr(v, 6), mask_r); + v128_t a = + wasm_i32x4_mul(wasm_u32x4_shr(wasm_v128_and(v, mask_a3), 22), mul_a); + wasm_v128_store(dst_argb, + wasm_v128_or(wasm_v128_or(b, g), wasm_v128_or(r, a))); + src_ar30 += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} + +void AR30ToABGRRow_WASMSIMD(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width) { + v128_t mask_r = SplatConst32(0x000000ff); + v128_t mask_g = SplatConst32(0x0000ff00); + v128_t mask_b = SplatConst32(0x00ff0000); + v128_t mask_a3 = SplatConst32((int)0xc0000000); + v128_t mul_a = SplatConst32(0x01550000); + do { + v128_t v = wasm_v128_load(src_ar30); + v128_t b = wasm_v128_and(wasm_i32x4_shl(v, 14), mask_b); + v128_t g = wasm_v128_and(wasm_u32x4_shr(v, 4), mask_g); + v128_t r = wasm_v128_and(wasm_u32x4_shr(v, 22), mask_r); + v128_t a = + wasm_i32x4_mul(wasm_u32x4_shr(wasm_v128_and(v, mask_a3), 22), mul_a); + wasm_v128_store(dst_abgr, + wasm_v128_or(wasm_v128_or(b, g), wasm_v128_or(r, a))); + src_ar30 += 16; + dst_abgr += 16; + width -= 4; + } while (width > 0); +} + +void AR30ToAB30Row_WASMSIMD(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width) { + v128_t mask_b = SplatConst32(0x000003ff); + v128_t mask_ga = SplatConst32((int)0xc00ffc00); + v128_t mask_r = SplatConst32(0x3ff00000); + do { + v128_t v = wasm_v128_load(src_ar30); + v128_t b_to_r = wasm_i32x4_shl(wasm_v128_and(v, mask_b), 20); + v128_t r_to_b = wasm_u32x4_shr(wasm_v128_and(v, mask_r), 20); + v128_t ga = wasm_v128_and(v, mask_ga); + wasm_v128_store(dst_ab30, wasm_v128_or(ga, wasm_v128_or(b_to_r, r_to_b))); + src_ar30 += 16; + dst_ab30 += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_AR30TOARGBROW_WASMSIMD + +#if defined(HAS_ARGBTOAR30ROW_WASMSIMD) || defined(HAS_ABGRTOAR30ROW_WASMSIMD) +void ARGBToAR30Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_ar30, + int width) { + v128_t mask_b = SplatConst32(0x000000ff); + v128_t mask_g = SplatConst32(0x0000ff00); + v128_t mask_r = SplatConst32(0x00ff0000); + v128_t mask_a = SplatConst32((int)0xc0000000); + do { + v128_t v = wasm_v128_load(src_argb); + v128_t b = wasm_v128_and(v, mask_b); + b = wasm_v128_or(wasm_i32x4_shl(b, 2), wasm_u32x4_shr(b, 6)); + v128_t g = wasm_v128_and(v, mask_g); + g = wasm_v128_or(wasm_i32x4_shl(g, 4), + wasm_i32x4_shl(wasm_u32x4_shr(g, 14), 10)); + v128_t r = wasm_v128_and(v, mask_r); + r = wasm_v128_or(wasm_i32x4_shl(r, 6), + wasm_i32x4_shl(wasm_u32x4_shr(r, 22), 20)); + v128_t a = wasm_v128_and(v, mask_a); + wasm_v128_store(dst_ar30, + wasm_v128_or(wasm_v128_or(b, g), wasm_v128_or(r, a))); + src_argb += 16; + dst_ar30 += 16; + width -= 4; + } while (width > 0); +} + +void ABGRToAR30Row_WASMSIMD(const uint8_t* src_abgr, + uint8_t* dst_ar30, + int width) { + v128_t mask_r = SplatConst32(0x000000ff); + v128_t mask_g = SplatConst32(0x0000ff00); + v128_t mask_b = SplatConst32(0x00ff0000); + v128_t mask_a = SplatConst32((int)0xc0000000); + do { + v128_t v = wasm_v128_load(src_abgr); + v128_t r = wasm_v128_and(v, mask_r); + r = wasm_v128_or(wasm_i32x4_shl(r, 22), + wasm_i32x4_shl(wasm_u32x4_shr(r, 6), 20)); + v128_t g = wasm_v128_and(v, mask_g); + g = wasm_v128_or(wasm_i32x4_shl(g, 4), + wasm_i32x4_shl(wasm_u32x4_shr(g, 14), 10)); + v128_t b = wasm_v128_and(v, mask_b); + b = wasm_v128_or(wasm_u32x4_shr(b, 14), wasm_u32x4_shr(b, 22)); + v128_t a = wasm_v128_and(v, mask_a); + wasm_v128_store(dst_ar30, + wasm_v128_or(wasm_v128_or(b, g), wasm_v128_or(r, a))); + src_abgr += 16; + dst_ar30 += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBTOAR30ROW_WASMSIMD || HAS_ABGRTOAR30ROW_WASMSIMD + +#if defined(HAS_I422TOARGBROW_WASMSIMD) || \ + defined(HAS_I422ALPHATOARGBROW_WASMSIMD) || \ + defined(HAS_I422TORGBAROW_WASMSIMD) || \ + defined(HAS_I422TOAR30ROW_WASMSIMD) || \ + defined(HAS_NV12TOARGBROW_WASMSIMD) || \ + defined(HAS_NV21TOARGBROW_WASMSIMD) || \ + defined(HAS_I444TOARGBROW_WASMSIMD) || \ + defined(HAS_I444ALPHATOARGBROW_WASMSIMD) || \ + defined(HAS_I422TORGB24ROW_WASMSIMD) || \ + defined(HAS_I422TORGB24ROW_RELAXEDSIMD) || \ + defined(HAS_NV12TORGB24ROW_WASMSIMD) || \ + defined(HAS_NV12TORGB24ROW_RELAXEDSIMD) || \ + defined(HAS_NV21TORGB24ROW_WASMSIMD) || \ + defined(HAS_NV21TORGB24ROW_RELAXEDSIMD) || \ + defined(HAS_I444TORGB24ROW_WASMSIMD) || \ + defined(HAS_I444TORGB24ROW_RELAXEDSIMD) || \ + defined(HAS_YUY2TOARGBROW_WASMSIMD) || \ + defined(HAS_UYVYTOARGBROW_WASMSIMD) || \ + defined(HAS_I422TORGB565ROW_WASMSIMD) || \ + defined(HAS_I422TOARGB1555ROW_WASMSIMD) || \ + defined(HAS_I422TOARGB4444ROW_WASMSIMD) || \ + defined(HAS_NV12TORGB565ROW_WASMSIMD) || \ + defined(HAS_I210TOARGBROW_WASMSIMD) || \ + defined(HAS_I210ALPHATOARGBROW_WASMSIMD) || \ + defined(HAS_I410TOARGBROW_WASMSIMD) || \ + defined(HAS_I410ALPHATOARGBROW_WASMSIMD) || \ + defined(HAS_I210TOAR30ROW_WASMSIMD) || defined(HAS_I410TOAR30ROW_WASMSIMD) // Interleaves map to punpck on x64 and zip on arm64 when the two inputs // differ. V8 arm64 lowers shuffle(x, x) to tbl with a rebuilt mask. static __inline v128_t ZipLo8_WASMSIMD(v128_t a, v128_t b) { @@ -1394,20 +3008,25 @@ static __inline void LoadYuvCoefs_WASMSIMD(const struct YuvConstants* yc, c->k0101 = SplatConst16(0x0101); } -// Returns ((y * 0x0101 * yg) >> 16) + yb for 8 Y in 16-bit lanes. -static __inline v128_t YToRgb_WASMSIMD(v128_t y, - const struct YuvCoefs_WASMSIMD* c) { +// Returns ((y16 * yg) >> 16) + yb for 8 16-bit Y values. +static __inline v128_t Y16ToRgb_WASMSIMD(v128_t y16, + const struct YuvCoefs_WASMSIMD* c) { // The barrier keeps clang from widening yg outside the loop, which would // turn extmul (umull on arm64) into extend + i32x4.mul. v128_t yg = c->yg; asm volatile("" : "+r"(yg)); - v128_t y16 = wasm_i16x8_mul(y, c->k0101); v128_t lo = wasm_u32x4_extmul_low_u16x8(y16, yg); v128_t hi = wasm_u32x4_extmul_high_u16x8(y16, yg); return wasm_i16x8_add(wasm_i16x8_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15), c->yb); } +// Returns ((y * 0x0101 * yg) >> 16) + yb for 8 Y in 16-bit lanes. +static __inline v128_t YToRgb_WASMSIMD(v128_t y, + const struct YuvCoefs_WASMSIMD* c) { + return Y16ToRgb_WASMSIMD(wasm_i16x8_mul(y, c->k0101), c); +} + // Converts 16 pixels to B, G, R with 6 fractional bits (unclamped int16). // y = 16 Y. uv = 8 U/V pairs, U in the low byte of each 16-bit lane (V if vu). // Even and odd pixels are kept apart so each pair shares its chroma terms: @@ -1437,6 +3056,30 @@ static __inline void YuvToRgb16_WASMSIMD(v128_t y, rgb[5] = wasm_i16x8_add_sat(yo, r); } +// Converts 16 4:4:4 pixels to even/odd B, G, R with 6 fractional bits. +static __inline void Yuv444ToRgb16_WASMSIMD(v128_t y, + v128_t u, + v128_t v, + const struct YuvCoefs_WASMSIMD* c, + v128_t* rgb) { + v128_t ye = YToRgb_WASMSIMD(wasm_v128_and(y, c->k00ff), c); + v128_t yo = YToRgb_WASMSIMD(wasm_u16x8_shr(y, 8), c); + v128_t us = wasm_v128_xor(u, c->x80); + v128_t vs = wasm_v128_xor(v, c->x80); + v128_t ue = wasm_i16x8_shr(wasm_i16x8_shl(us, 8), 8); + v128_t uo = wasm_i16x8_shr(us, 8); + v128_t ve = wasm_i16x8_shr(wasm_i16x8_shl(vs, 8), 8); + v128_t vo = wasm_i16x8_shr(vs, 8); + rgb[0] = wasm_i16x8_add_sat(ye, wasm_i16x8_mul(ue, c->ub)); + rgb[1] = wasm_i16x8_add_sat(yo, wasm_i16x8_mul(uo, c->ub)); + rgb[2] = wasm_i16x8_sub_sat( + ye, wasm_i16x8_add(wasm_i16x8_mul(ue, c->ug), wasm_i16x8_mul(ve, c->vg))); + rgb[3] = wasm_i16x8_sub_sat( + yo, wasm_i16x8_add(wasm_i16x8_mul(uo, c->ug), wasm_i16x8_mul(vo, c->vg))); + rgb[4] = wasm_i16x8_add_sat(ye, wasm_i16x8_mul(ve, c->vr)); + rgb[5] = wasm_i16x8_add_sat(yo, wasm_i16x8_mul(vo, c->vr)); +} + // Loads 16 Y and 8 U/V, or 8 Y and 4 U/V when width < 16. static __inline void ReadI422_WASMSIMD(const uint8_t* src_y, const uint8_t* src_u, @@ -1469,12 +3112,41 @@ static __inline void ReadNV12_WASMSIMD(const uint8_t* src_y, *uv = wasm_v128_load64_zero(src_uv); } } + +// Loads 16 Y, 16 U, and 16 V, or 8 of each when width < 16. +static __inline void ReadI444_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + int width, + v128_t* y, + v128_t* u, + v128_t* v) { + if (width >= 16) { + *y = wasm_v128_load(src_y); + *u = wasm_v128_load(src_u); + *v = wasm_v128_load(src_v); + } else { + *y = wasm_v128_load64_zero(src_y); + *u = wasm_v128_load64_zero(src_u); + *v = wasm_v128_load64_zero(src_v); + } +} #endif -#if defined(HAS_I422TOARGBROW_WASMSIMD) || \ - defined(HAS_NV12TOARGBROW_WASMSIMD) || defined(HAS_NV21TOARGBROW_WASMSIMD) +#if defined(HAS_I422TOARGBROW_WASMSIMD) || \ + defined(HAS_I422ALPHATOARGBROW_WASMSIMD) || \ + defined(HAS_NV12TOARGBROW_WASMSIMD) || \ + defined(HAS_NV21TOARGBROW_WASMSIMD) || \ + defined(HAS_I444TOARGBROW_WASMSIMD) || \ + defined(HAS_I444ALPHATOARGBROW_WASMSIMD) || \ + defined(HAS_YUY2TOARGBROW_WASMSIMD) || \ + defined(HAS_UYVYTOARGBROW_WASMSIMD) || \ + defined(HAS_I210TOARGBROW_WASMSIMD) || \ + defined(HAS_I210ALPHATOARGBROW_WASMSIMD) || \ + defined(HAS_I410TOARGBROW_WASMSIMD) || \ + defined(HAS_I410ALPHATOARGBROW_WASMSIMD) // Clamps even/odd B, G, R to 8 bits and stores 16 ARGB pixels, or 8 when -// width < 16. +// width < 16. ff holds 8 odd alphas in bytes 0..7 and 8 even alphas in 8..15. static __inline void StoreARGB_WASMSIMD(const v128_t* rgb, v128_t ff, uint8_t* dst_argb, @@ -1500,6 +3172,41 @@ static __inline void StoreARGB_WASMSIMD(const v128_t* rgb, wasm_v128_store(dst_argb + 48, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); } } + +static __inline v128_t PackAlphaOddEven_WASMSIMD(v128_t a) { + return wasm_i8x16_shuffle(a, a, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14); +} +#endif + +#ifdef HAS_I422TORGBAROW_WASMSIMD +// Clamps even/odd B, G, R to 8 bits and stores 16 RGBA pixels, or 8 when +// width < 16. +static __inline void StoreRGBA_WASMSIMD(const v128_t* rgb, + v128_t ff, + uint8_t* dst_rgba, + int width) { + v128_t bg_e = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[0], 6), + wasm_i16x8_shr(rgb[2], 6)); + v128_t gb_o = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[3], 6), + wasm_i16x8_shr(rgb[1], 6)); + v128_t rr = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[4], 6), + wasm_i16x8_shr(rgb[5], 6)); + v128_t br_e = ZipLo8_WASMSIMD(bg_e, rr); + v128_t ag_e = ZipHi8_WASMSIMD(ff, bg_e); + v128_t br_o = ZipHi8_WASMSIMD(gb_o, rr); + v128_t ag_o = ZipLo8_WASMSIMD(ff, gb_o); + v128_t e = ZipLo8_WASMSIMD(ag_e, br_e); // Pixels 0, 2, 4, 6. + v128_t o = ZipLo8_WASMSIMD(ag_o, br_o); // Pixels 1, 3, 5, 7. + wasm_v128_store(dst_rgba, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); + wasm_v128_store(dst_rgba + 16, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); + if (width >= 16) { + e = ZipHi8_WASMSIMD(ag_e, br_e); + o = ZipHi8_WASMSIMD(ag_o, br_o); + wasm_v128_store(dst_rgba + 32, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); + wasm_v128_store(dst_rgba + 48, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); + } +} #endif #ifdef HAS_I422TOARGBROW_WASMSIMD @@ -1526,7 +3233,59 @@ void I422ToARGBRow_WASMSIMD(const uint8_t* src_y, } #endif // HAS_I422TOARGBROW_WASMSIMD -#ifdef HAS_I422TOAR30ROW_WASMSIMD +#ifdef HAS_I422ALPHATOARGBROW_WASMSIMD +void I422AlphaToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + const uint8_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv); + v128_t a = + (width >= 16) ? wasm_v128_load(src_a) : wasm_v128_load64_zero(src_a); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + StoreARGB_WASMSIMD(rgb, PackAlphaOddEven_WASMSIMD(a), dst_argb, width); + src_y += 16; + src_u += 8; + src_v += 8; + src_a += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422ALPHATOARGBROW_WASMSIMD + +#ifdef HAS_I422TORGBAROW_WASMSIMD +void I422ToRGBARow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgba, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t ff = SplatConst8(0xff); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + StoreRGBA_WASMSIMD(rgb, ff, dst_rgba, width); + src_y += 16; + src_u += 8; + src_v += 8; + dst_rgba += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422TORGBAROW_WASMSIMD + +#if defined(HAS_I422TOAR30ROW_WASMSIMD) || \ + defined(HAS_I210TOAR30ROW_WASMSIMD) || defined(HAS_I410TOAR30ROW_WASMSIMD) // Clamps B, G, R with 6 fractional bits to 10 bits and packs them into the // low and high 16 bits of AR30, without the alpha bits. static __inline void RgbToAR30_WASMSIMD(v128_t b, @@ -1544,6 +3303,32 @@ static __inline void RgbToAR30_WASMSIMD(v128_t b, *hi = wasm_v128_or(wasm_v128_and(r, k3ff0), wasm_u16x8_shr(g, 10)); } +static __inline void StoreAR30_WASMSIMD(const v128_t* rgb, + v128_t zero, + v128_t k3ff0, + v128_t kc000, + uint8_t* dst_ar30, + int width) { + v128_t lo_e, hi_e, lo_o, hi_o; + RgbToAR30_WASMSIMD(rgb[0], rgb[2], rgb[4], zero, k3ff0, &lo_e, &hi_e); + RgbToAR30_WASMSIMD(rgb[1], rgb[3], rgb[5], zero, k3ff0, &lo_o, &hi_o); + hi_e = wasm_v128_or(hi_e, kc000); + hi_o = wasm_v128_or(hi_o, kc000); + // Even pixels 0, 2, 4, 6 and odd pixels 1, 3, 5, 7 as 32 bits. + v128_t e = wasm_i16x8_shuffle(lo_e, hi_e, 0, 8, 1, 9, 2, 10, 3, 11); + v128_t o = wasm_i16x8_shuffle(lo_o, hi_o, 0, 8, 1, 9, 2, 10, 3, 11); + wasm_v128_store(dst_ar30, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); + wasm_v128_store(dst_ar30 + 16, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); + if (width >= 16) { + e = wasm_i16x8_shuffle(lo_e, hi_e, 4, 12, 5, 13, 6, 14, 7, 15); + o = wasm_i16x8_shuffle(lo_o, hi_o, 4, 12, 5, 13, 6, 14, 7, 15); + wasm_v128_store(dst_ar30 + 32, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); + wasm_v128_store(dst_ar30 + 48, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); + } +} +#endif + +#ifdef HAS_I422TOAR30ROW_WASMSIMD void I422ToAR30Row_WASMSIMD(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -1556,24 +3341,10 @@ void I422ToAR30Row_WASMSIMD(const uint8_t* src_y, v128_t kc000 = SplatConst16(0xc000); LoadYuvCoefs_WASMSIMD(yuvconstants, 24, &c); // AR30 bias. do { - v128_t y, uv, rgb[6], lo_e, hi_e, lo_o, hi_o; + v128_t y, uv, rgb[6]; ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv); YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); - RgbToAR30_WASMSIMD(rgb[0], rgb[2], rgb[4], zero, k3ff0, &lo_e, &hi_e); - RgbToAR30_WASMSIMD(rgb[1], rgb[3], rgb[5], zero, k3ff0, &lo_o, &hi_o); - hi_e = wasm_v128_or(hi_e, kc000); - hi_o = wasm_v128_or(hi_o, kc000); - // Even pixels 0, 2, 4, 6 and odd pixels 1, 3, 5, 7 as 32 bits. - v128_t e = wasm_i16x8_shuffle(lo_e, hi_e, 0, 8, 1, 9, 2, 10, 3, 11); - v128_t o = wasm_i16x8_shuffle(lo_o, hi_o, 0, 8, 1, 9, 2, 10, 3, 11); - wasm_v128_store(dst_ar30, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); - wasm_v128_store(dst_ar30 + 16, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); - if (width >= 16) { - e = wasm_i16x8_shuffle(lo_e, hi_e, 4, 12, 5, 13, 6, 14, 7, 15); - o = wasm_i16x8_shuffle(lo_o, hi_o, 4, 12, 5, 13, 6, 14, 7, 15); - wasm_v128_store(dst_ar30 + 32, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); - wasm_v128_store(dst_ar30 + 48, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); - } + StoreAR30_WASMSIMD(rgb, zero, k3ff0, kc000, dst_ar30, width); src_y += 16; src_u += 8; src_v += 8; @@ -1627,7 +3398,65 @@ void NV21ToARGBRow_WASMSIMD(const uint8_t* src_y, } #endif // HAS_NV21TOARGBROW_WASMSIMD -#ifdef HAS_I422TORGB24ROW_WASMSIMD +#ifdef HAS_I444TOARGBROW_WASMSIMD +void I444ToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t ff = SplatConst8(0xff); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, u, v, rgb[6]; + ReadI444_WASMSIMD(src_y, src_u, src_v, width, &y, &u, &v); + Yuv444ToRgb16_WASMSIMD(y, u, v, &c, rgb); + StoreARGB_WASMSIMD(rgb, ff, dst_argb, width); + src_y += 16; + src_u += 16; + src_v += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I444TOARGBROW_WASMSIMD + +#ifdef HAS_I444ALPHATOARGBROW_WASMSIMD +void I444AlphaToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + const uint8_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, u, v, rgb[6]; + ReadI444_WASMSIMD(src_y, src_u, src_v, width, &y, &u, &v); + v128_t a = + (width >= 16) ? wasm_v128_load(src_a) : wasm_v128_load64_zero(src_a); + Yuv444ToRgb16_WASMSIMD(y, u, v, &c, rgb); + StoreARGB_WASMSIMD(rgb, PackAlphaOddEven_WASMSIMD(a), dst_argb, width); + src_y += 16; + src_u += 16; + src_v += 16; + src_a += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I444ALPHATOARGBROW_WASMSIMD + +#if defined(HAS_I422TORGB24ROW_WASMSIMD) || \ + defined(HAS_I422TORGB24ROW_RELAXEDSIMD) || \ + defined(HAS_NV12TORGB24ROW_WASMSIMD) || \ + defined(HAS_NV12TORGB24ROW_RELAXEDSIMD) || \ + defined(HAS_NV21TORGB24ROW_WASMSIMD) || \ + defined(HAS_NV21TORGB24ROW_RELAXEDSIMD) || \ + defined(HAS_I444TORGB24ROW_WASMSIMD) || \ + defined(HAS_I444TORGB24ROW_RELAXEDSIMD) // Swizzles that interleave B/G pairs and R into 3 vectors of RGB24. R is // 8 even pixels followed by 8 odd pixels. static const uint8_t kI422ToRGB24Swizzle[96] = { @@ -1639,42 +3468,55 @@ static const uint8_t kI422ToRGB24Swizzle[96] = { 128, 10, 11, 128, 12, 13, 128, 14, 15, 128, 5, 128, 128, 13, 128, 128, 6, 128, 128, 14, 128, 128, 7, 128, 128, 15}; -void I422ToRGB24Row_WASMSIMD(const uint8_t* src_y, - const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_rgb24, - const struct YuvConstants* yuvconstants, - int width) { +template +static __inline void StoreRGB24_Wasm(const v128_t* rgb, + const v128_t* m, + uint8_t* dst_rgb24) { + v128_t bg_e = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[0], 6), + wasm_i16x8_shr(rgb[2], 6)); + v128_t bg_o = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[1], 6), + wasm_i16x8_shr(rgb[3], 6)); + v128_t r = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[4], 6), + wasm_i16x8_shr(rgb[5], 6)); + v128_t b = ZipLo8_WASMSIMD(bg_e, bg_o); + v128_t g = ZipHi8_WASMSIMD(bg_e, bg_o); + v128_t bg0 = ZipLo8_WASMSIMD(b, g); + v128_t bg2 = ZipHi8_WASMSIMD(b, g); + v128_t bg1 = wasm_i8x16_shuffle(bg0, bg2, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23); + wasm_v128_store(dst_rgb24, wasm_v128_or(WasmSwizzle(bg0, m[0]), + WasmSwizzle(r, m[1]))); + wasm_v128_store(dst_rgb24 + 16, wasm_v128_or(WasmSwizzle(bg1, m[2]), + WasmSwizzle(r, m[3]))); + wasm_v128_store(dst_rgb24 + 32, wasm_v128_or(WasmSwizzle(bg2, m[4]), + WasmSwizzle(r, m[5]))); +} + +static __inline void LoadRGB24Swizzle_WASMSIMD(v128_t* m) { + for (int i = 0; i < 6; ++i) { + m[i] = LoadConst(kI422ToRGB24Swizzle + i * 16); + } +} +#endif + +#if defined(HAS_I422TORGB24ROW_WASMSIMD) || \ + defined(HAS_I422TORGB24ROW_RELAXEDSIMD) +template +static inline void I422ToRGB24Row_Wasm(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { struct YuvCoefs_WASMSIMD c; - v128_t m0 = LoadConst(kI422ToRGB24Swizzle); - v128_t m1 = LoadConst(kI422ToRGB24Swizzle + 16); - v128_t m2 = LoadConst(kI422ToRGB24Swizzle + 32); - v128_t m3 = LoadConst(kI422ToRGB24Swizzle + 48); - v128_t m4 = LoadConst(kI422ToRGB24Swizzle + 64); - v128_t m5 = LoadConst(kI422ToRGB24Swizzle + 80); + v128_t m[6]; + LoadRGB24Swizzle_WASMSIMD(m); LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); do { v128_t y, uv, rgb[6]; ReadI422_WASMSIMD(src_y, src_u, src_v, 16, &y, &uv); YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); - v128_t bg_e = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[0], 6), - wasm_i16x8_shr(rgb[2], 6)); - v128_t bg_o = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[1], 6), - wasm_i16x8_shr(rgb[3], 6)); - v128_t r = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[4], 6), - wasm_i16x8_shr(rgb[5], 6)); - v128_t b = ZipLo8_WASMSIMD(bg_e, bg_o); - v128_t g = ZipHi8_WASMSIMD(bg_e, bg_o); - v128_t bg0 = ZipLo8_WASMSIMD(b, g); - v128_t bg2 = ZipHi8_WASMSIMD(b, g); - v128_t bg1 = wasm_i8x16_shuffle(bg0, bg2, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20, 21, 22, 23); - wasm_v128_store(dst_rgb24, wasm_v128_or(LIBYUV_WASM_SWIZZLE(bg0, m0), - LIBYUV_WASM_SWIZZLE(r, m1))); - wasm_v128_store(dst_rgb24 + 16, wasm_v128_or(LIBYUV_WASM_SWIZZLE(bg1, m2), - LIBYUV_WASM_SWIZZLE(r, m3))); - wasm_v128_store(dst_rgb24 + 32, wasm_v128_or(LIBYUV_WASM_SWIZZLE(bg2, m4), - LIBYUV_WASM_SWIZZLE(r, m5))); + StoreRGB24_Wasm(rgb, m, dst_rgb24); src_y += 16; src_u += 8; src_v += 8; @@ -1682,8 +3524,556 @@ void I422ToRGB24Row_WASMSIMD(const uint8_t* src_y, width -= 16; } while (width > 0); } + +#ifdef HAS_I422TORGB24ROW_WASMSIMD +void I422ToRGB24Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + I422ToRGB24Row_Wasm(src_y, src_u, src_v, dst_rgb24, yuvconstants, + width); +} #endif // HAS_I422TORGB24ROW_WASMSIMD +#ifdef HAS_I422TORGB24ROW_RELAXEDSIMD +void I422ToRGB24Row_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + I422ToRGB24Row_Wasm(src_y, src_u, src_v, dst_rgb24, yuvconstants, + width); +} +#endif // HAS_I422TORGB24ROW_RELAXEDSIMD +#endif // HAS_I422TORGB24ROW_WASMSIMD || HAS_I422TORGB24ROW_RELAXEDSIMD + +#if defined(HAS_NV12TORGB24ROW_WASMSIMD) || \ + defined(HAS_NV12TORGB24ROW_RELAXEDSIMD) || \ + defined(HAS_NV21TORGB24ROW_WASMSIMD) || \ + defined(HAS_NV21TORGB24ROW_RELAXEDSIMD) +template +static inline void NVToRGB24Row_Wasm(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t m[6]; + LoadRGB24Swizzle_WASMSIMD(m); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadNV12_WASMSIMD(src_y, src_uv, 16, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, kVu, &c, rgb); + StoreRGB24_Wasm(rgb, m, dst_rgb24); + src_y += 16; + src_uv += 16; + dst_rgb24 += 48; + width -= 16; + } while (width > 0); +} + +#ifdef HAS_NV12TORGB24ROW_WASMSIMD +void NV12ToRGB24Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + NVToRGB24Row_Wasm(src_y, src_uv, dst_rgb24, yuvconstants, width); +} +#endif // HAS_NV12TORGB24ROW_WASMSIMD + +#ifdef HAS_NV12TORGB24ROW_RELAXEDSIMD +void NV12ToRGB24Row_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + NVToRGB24Row_Wasm(src_y, src_uv, dst_rgb24, yuvconstants, width); +} +#endif // HAS_NV12TORGB24ROW_RELAXEDSIMD + +#ifdef HAS_NV21TORGB24ROW_WASMSIMD +void NV21ToRGB24Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + NVToRGB24Row_Wasm(src_y, src_vu, dst_rgb24, yuvconstants, width); +} +#endif // HAS_NV21TORGB24ROW_WASMSIMD + +#ifdef HAS_NV21TORGB24ROW_RELAXEDSIMD +void NV21ToRGB24Row_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + NVToRGB24Row_Wasm(src_y, src_vu, dst_rgb24, yuvconstants, width); +} +#endif // HAS_NV21TORGB24ROW_RELAXEDSIMD +#endif + +#if defined(HAS_I444TORGB24ROW_WASMSIMD) || \ + defined(HAS_I444TORGB24ROW_RELAXEDSIMD) +template +static inline void I444ToRGB24Row_Wasm(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t m[6]; + LoadRGB24Swizzle_WASMSIMD(m); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, u, v, rgb[6]; + ReadI444_WASMSIMD(src_y, src_u, src_v, 16, &y, &u, &v); + Yuv444ToRgb16_WASMSIMD(y, u, v, &c, rgb); + StoreRGB24_Wasm(rgb, m, dst_rgb24); + src_y += 16; + src_u += 16; + src_v += 16; + dst_rgb24 += 48; + width -= 16; + } while (width > 0); +} + +#ifdef HAS_I444TORGB24ROW_WASMSIMD +void I444ToRGB24Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + I444ToRGB24Row_Wasm(src_y, src_u, src_v, dst_rgb24, yuvconstants, + width); +} +#endif // HAS_I444TORGB24ROW_WASMSIMD + +#ifdef HAS_I444TORGB24ROW_RELAXEDSIMD +void I444ToRGB24Row_RELAXEDSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + I444ToRGB24Row_Wasm(src_y, src_u, src_v, dst_rgb24, yuvconstants, + width); +} +#endif // HAS_I444TORGB24ROW_RELAXEDSIMD +#endif // HAS_I444TORGB24ROW_WASMSIMD || HAS_I444TORGB24ROW_RELAXEDSIMD + +#ifdef HAS_YUY2TOARGBROW_WASMSIMD +void YUY2ToARGBRow_WASMSIMD(const uint8_t* src_yuy2, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t ff = SplatConst8(0xff); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t s0 = wasm_v128_load(src_yuy2); + v128_t s1 = (width >= 16) ? wasm_v128_load(src_yuy2 + 16) : s0; + v128_t y = wasm_i8x16_shuffle(s0, s1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, + 22, 24, 26, 28, 30); + v128_t uv = wasm_i8x16_shuffle(s0, s1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, + 21, 23, 25, 27, 29, 31); + v128_t rgb[6]; + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + StoreARGB_WASMSIMD(rgb, ff, dst_argb, width); + src_yuy2 += 32; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_YUY2TOARGBROW_WASMSIMD + +#ifdef HAS_UYVYTOARGBROW_WASMSIMD +void UYVYToARGBRow_WASMSIMD(const uint8_t* src_uyvy, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t ff = SplatConst8(0xff); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t s0 = wasm_v128_load(src_uyvy); + v128_t s1 = (width >= 16) ? wasm_v128_load(src_uyvy + 16) : s0; + v128_t uv = wasm_i8x16_shuffle(s0, s1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, + 20, 22, 24, 26, 28, 30); + v128_t y = wasm_i8x16_shuffle(s0, s1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, + 23, 25, 27, 29, 31); + v128_t rgb[6]; + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + StoreARGB_WASMSIMD(rgb, ff, dst_argb, width); + src_uyvy += 32; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_UYVYTOARGBROW_WASMSIMD + +#if defined(HAS_I210TOARGBROW_WASMSIMD) || \ + defined(HAS_I210ALPHATOARGBROW_WASMSIMD) || \ + defined(HAS_I410TOARGBROW_WASMSIMD) || \ + defined(HAS_I410ALPHATOARGBROW_WASMSIMD) || \ + defined(HAS_I210TOAR30ROW_WASMSIMD) || defined(HAS_I410TOAR30ROW_WASMSIMD) +// Loads 16 10-bit Y (or 8 when width < 16) and separates even and odd pixels, +// expanding 10-bit Y to 16 bits via (y << 6) | (y >> 4). +static __inline void ReadY10_WASMSIMD(const uint16_t* src_y, + int width, + const struct YuvCoefs_WASMSIMD* c, + v128_t* ye, + v128_t* yo) { + v128_t y0 = wasm_v128_load(src_y); + v128_t y1 = (width >= 16) ? wasm_v128_load(src_y + 8) : y0; + y0 = wasm_v128_or(wasm_i16x8_shl(y0, 6), wasm_u16x8_shr(y0, 4)); + y1 = wasm_v128_or(wasm_i16x8_shl(y1, 6), wasm_u16x8_shr(y1, 4)); + *ye = Y16ToRgb_WASMSIMD(wasm_i16x8_shuffle(y0, y1, 0, 2, 4, 6, 8, 10, 12, 14), + c); + *yo = Y16ToRgb_WASMSIMD(wasm_i16x8_shuffle(y0, y1, 1, 3, 5, 7, 9, 11, 13, 15), + c); +} + +// Loads 8 10-bit U and V (or 4 when width < 16), shifts right by 2, clamps to +// [0, 255], and interleaves into 8 U/V pairs. +static __inline v128_t ReadUV210_WASMSIMD(const uint16_t* src_u, + const uint16_t* src_v, + int width) { + v128_t u16 = wasm_u16x8_shr( + (width >= 16) ? wasm_v128_load(src_u) : wasm_v128_load64_zero(src_u), 2); + v128_t v16 = wasm_u16x8_shr( + (width >= 16) ? wasm_v128_load(src_v) : wasm_v128_load64_zero(src_v), 2); + v128_t uv8 = wasm_u8x16_narrow_i16x8(u16, v16); + return wasm_i8x16_shuffle(uv8, uv8, 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, + 14, 7, 15); +} + +// Loads 16 10-bit samples (or 8 when width < 16), shifts right by 2, and +// clamps to [0, 255]. +static __inline v128_t Read16_10To8_WASMSIMD(const uint16_t* src, int width) { + v128_t s0 = wasm_u16x8_shr(wasm_v128_load(src), 2); + v128_t s1 = (width >= 16) ? wasm_u16x8_shr(wasm_v128_load(src + 8), 2) : s0; + return wasm_u8x16_narrow_i16x8(s0, s1); +} + +static __inline void Yuv210ToRgb16_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + int width, + const struct YuvCoefs_WASMSIMD* c, + v128_t* rgb) { + v128_t ye, yo; + ReadY10_WASMSIMD(src_y, width, c, &ye, &yo); + v128_t uvs = wasm_v128_xor(ReadUV210_WASMSIMD(src_u, src_v, width), c->x80); + v128_t ui = wasm_i16x8_shr(wasm_i16x8_shl(uvs, 8), 8); + v128_t vi = wasm_i16x8_shr(uvs, 8); + v128_t b = wasm_i16x8_mul(ui, c->ub); + v128_t g = + wasm_i16x8_add(wasm_i16x8_mul(ui, c->ug), wasm_i16x8_mul(vi, c->vg)); + v128_t r = wasm_i16x8_mul(vi, c->vr); + rgb[0] = wasm_i16x8_add_sat(ye, b); + rgb[1] = wasm_i16x8_add_sat(yo, b); + rgb[2] = wasm_i16x8_sub_sat(ye, g); + rgb[3] = wasm_i16x8_sub_sat(yo, g); + rgb[4] = wasm_i16x8_add_sat(ye, r); + rgb[5] = wasm_i16x8_add_sat(yo, r); +} + +static __inline void Yuv410ToRgb16_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + int width, + const struct YuvCoefs_WASMSIMD* c, + v128_t* rgb) { + v128_t ye, yo; + ReadY10_WASMSIMD(src_y, width, c, &ye, &yo); + v128_t us = wasm_v128_xor(Read16_10To8_WASMSIMD(src_u, width), c->x80); + v128_t vs = wasm_v128_xor(Read16_10To8_WASMSIMD(src_v, width), c->x80); + v128_t ue = wasm_i16x8_shr(wasm_i16x8_shl(us, 8), 8); + v128_t uo = wasm_i16x8_shr(us, 8); + v128_t ve = wasm_i16x8_shr(wasm_i16x8_shl(vs, 8), 8); + v128_t vo = wasm_i16x8_shr(vs, 8); + rgb[0] = wasm_i16x8_add_sat(ye, wasm_i16x8_mul(ue, c->ub)); + rgb[1] = wasm_i16x8_add_sat(yo, wasm_i16x8_mul(uo, c->ub)); + rgb[2] = wasm_i16x8_sub_sat( + ye, wasm_i16x8_add(wasm_i16x8_mul(ue, c->ug), wasm_i16x8_mul(ve, c->vg))); + rgb[3] = wasm_i16x8_sub_sat( + yo, wasm_i16x8_add(wasm_i16x8_mul(uo, c->ug), wasm_i16x8_mul(vo, c->vg))); + rgb[4] = wasm_i16x8_add_sat(ye, wasm_i16x8_mul(ve, c->vr)); + rgb[5] = wasm_i16x8_add_sat(yo, wasm_i16x8_mul(vo, c->vr)); +} +#endif + +#ifdef HAS_I210TOARGBROW_WASMSIMD +void I210ToARGBRow_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t ff = SplatConst8(0xff); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t rgb[6]; + Yuv210ToRgb16_WASMSIMD(src_y, src_u, src_v, width, &c, rgb); + StoreARGB_WASMSIMD(rgb, ff, dst_argb, width); + src_y += 16; + src_u += 8; + src_v += 8; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I210TOARGBROW_WASMSIMD + +#ifdef HAS_I210ALPHATOARGBROW_WASMSIMD +void I210AlphaToARGBRow_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + const uint16_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t rgb[6]; + Yuv210ToRgb16_WASMSIMD(src_y, src_u, src_v, width, &c, rgb); + v128_t a = Read16_10To8_WASMSIMD(src_a, width); + StoreARGB_WASMSIMD(rgb, PackAlphaOddEven_WASMSIMD(a), dst_argb, width); + src_y += 16; + src_u += 8; + src_v += 8; + src_a += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I210ALPHATOARGBROW_WASMSIMD + +#ifdef HAS_I410TOARGBROW_WASMSIMD +void I410ToARGBRow_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t ff = SplatConst8(0xff); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t rgb[6]; + Yuv410ToRgb16_WASMSIMD(src_y, src_u, src_v, width, &c, rgb); + StoreARGB_WASMSIMD(rgb, ff, dst_argb, width); + src_y += 16; + src_u += 16; + src_v += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I410TOARGBROW_WASMSIMD + +#ifdef HAS_I410ALPHATOARGBROW_WASMSIMD +void I410AlphaToARGBRow_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + const uint16_t* src_a, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t rgb[6]; + Yuv410ToRgb16_WASMSIMD(src_y, src_u, src_v, width, &c, rgb); + v128_t a = Read16_10To8_WASMSIMD(src_a, width); + StoreARGB_WASMSIMD(rgb, PackAlphaOddEven_WASMSIMD(a), dst_argb, width); + src_y += 16; + src_u += 16; + src_v += 16; + src_a += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I410ALPHATOARGBROW_WASMSIMD + +#ifdef HAS_I210TOAR30ROW_WASMSIMD +void I210ToAR30Row_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t zero = SplatConst16(0); + v128_t k3ff0 = SplatConst16(0x3ff0); + v128_t kc000 = SplatConst16(0xc000); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t rgb[6]; + Yuv210ToRgb16_WASMSIMD(src_y, src_u, src_v, width, &c, rgb); + StoreAR30_WASMSIMD(rgb, zero, k3ff0, kc000, dst_ar30, width); + src_y += 16; + src_u += 8; + src_v += 8; + dst_ar30 += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I210TOAR30ROW_WASMSIMD + +#ifdef HAS_I410TOAR30ROW_WASMSIMD +void I410ToAR30Row_WASMSIMD(const uint16_t* src_y, + const uint16_t* src_u, + const uint16_t* src_v, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t zero = SplatConst16(0); + v128_t k3ff0 = SplatConst16(0x3ff0); + v128_t kc000 = SplatConst16(0xc000); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t rgb[6]; + Yuv410ToRgb16_WASMSIMD(src_y, src_u, src_v, width, &c, rgb); + StoreAR30_WASMSIMD(rgb, zero, k3ff0, kc000, dst_ar30, width); + src_y += 16; + src_u += 16; + src_v += 16; + dst_ar30 += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I410TOAR30ROW_WASMSIMD + +#if defined(HAS_YUY2TOYROW_WASMSIMD) || defined(HAS_UYVYTOYROW_WASMSIMD) +void YUY2ToYRow_WASMSIMD(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + do { + asm("" : "+r"(src_yuy2)); + v128_t s0 = wasm_v128_load(src_yuy2); + v128_t s1 = wasm_v128_load(src_yuy2 + 16); + wasm_v128_store(dst_y, wasm_i8x16_shuffle(s0, s1, 0, 2, 4, 6, 8, 10, 12, 14, + 16, 18, 20, 22, 24, 26, 28, 30)); + src_yuy2 += 32; + dst_y += 16; + width -= 16; + } while (width > 0); +} + +void UYVYToYRow_WASMSIMD(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + do { + asm("" : "+r"(src_uyvy)); + v128_t s0 = wasm_v128_load(src_uyvy); + v128_t s1 = wasm_v128_load(src_uyvy + 16); + wasm_v128_store(dst_y, wasm_i8x16_shuffle(s0, s1, 1, 3, 5, 7, 9, 11, 13, 15, + 17, 19, 21, 23, 25, 27, 29, 31)); + src_uyvy += 32; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_YUY2TOYROW_WASMSIMD || HAS_UYVYTOYROW_WASMSIMD + +#if defined(HAS_YUY2TOUVROW_WASMSIMD) || \ + defined(HAS_YUY2TOUV422ROW_WASMSIMD) || \ + defined(HAS_UYVYTOUVROW_WASMSIMD) || defined(HAS_UYVYTOUV422ROW_WASMSIMD) +void YUY2ToUV422Row_WASMSIMD(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + do { + asm("" : "+r"(src_yuy2)); + v128_t s0 = wasm_v128_load(src_yuy2); + v128_t s1 = wasm_v128_load(src_yuy2 + 16); + v128_t uv = wasm_i8x16_shuffle(s0, s1, 1, 5, 9, 13, 17, 21, 25, 29, 3, 7, + 11, 15, 19, 23, 27, 31); + wasm_v128_store64_lane(dst_u, uv, 0); + wasm_v128_store64_lane(dst_v, uv, 1); + src_yuy2 += 32; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} + +void YUY2ToUVRow_WASMSIMD(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* src_yuy2_1 = src_yuy2 + src_stride_yuy2; + do { + asm("" : "+r"(src_yuy2), "+r"(src_yuy2_1)); + v128_t s0 = + wasm_u8x16_avgr(wasm_v128_load(src_yuy2), wasm_v128_load(src_yuy2_1)); + v128_t s1 = wasm_u8x16_avgr(wasm_v128_load(src_yuy2 + 16), + wasm_v128_load(src_yuy2_1 + 16)); + v128_t uv = wasm_i8x16_shuffle(s0, s1, 1, 5, 9, 13, 17, 21, 25, 29, 3, 7, + 11, 15, 19, 23, 27, 31); + wasm_v128_store64_lane(dst_u, uv, 0); + wasm_v128_store64_lane(dst_v, uv, 1); + src_yuy2 += 32; + src_yuy2_1 += 32; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} + +void UYVYToUV422Row_WASMSIMD(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + do { + asm("" : "+r"(src_uyvy)); + v128_t s0 = wasm_v128_load(src_uyvy); + v128_t s1 = wasm_v128_load(src_uyvy + 16); + v128_t uv = wasm_i8x16_shuffle(s0, s1, 0, 4, 8, 12, 16, 20, 24, 28, 2, 6, + 10, 14, 18, 22, 26, 30); + wasm_v128_store64_lane(dst_u, uv, 0); + wasm_v128_store64_lane(dst_v, uv, 1); + src_uyvy += 32; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} + +void UYVYToUVRow_WASMSIMD(const uint8_t* src_uyvy, + int src_stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* src_uyvy_1 = src_uyvy + src_stride_uyvy; + do { + asm("" : "+r"(src_uyvy), "+r"(src_uyvy_1)); + v128_t s0 = + wasm_u8x16_avgr(wasm_v128_load(src_uyvy), wasm_v128_load(src_uyvy_1)); + v128_t s1 = wasm_u8x16_avgr(wasm_v128_load(src_uyvy + 16), + wasm_v128_load(src_uyvy_1 + 16)); + v128_t uv = wasm_i8x16_shuffle(s0, s1, 0, 4, 8, 12, 16, 20, 24, 28, 2, 6, + 10, 14, 18, 22, 26, 30); + wasm_v128_store64_lane(dst_u, uv, 0); + wasm_v128_store64_lane(dst_v, uv, 1); + src_uyvy += 32; + src_uyvy_1 += 32; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} +#endif + #if defined(HAS_I422TORGB565ROW_WASMSIMD) || \ defined(HAS_I422TOARGB1555ROW_WASMSIMD) || \ defined(HAS_I422TOARGB4444ROW_WASMSIMD) || \ @@ -1824,14 +4214,16 @@ void NV12ToRGB565Row_WASMSIMD(const uint8_t* src_y, } #endif // HAS_NV12TORGB565ROW_WASMSIMD -#ifdef HAS_ARGBBLENDROW_WASMSIMD +#if defined(HAS_ARGBBLENDROW_WASMSIMD) || defined(HAS_ARGBBLENDROW_RELAXEDSIMD) // Broadcast 255 - alpha (from byte 3 of each pixel) to 16 bit lanes. static const uint8_t kShuffleAlpha16[16] = {3, 128, 3, 128, 7, 128, 7, 128, 11, 128, 11, 128, 15, 128, 15, 128}; -void ARGBBlendRow_WASMSIMD(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { + +template +static inline void ARGBBlendRow_Wasm(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { v128_t shuf_a = LoadConst(kShuffleAlpha16); v128_t one = SplatConst16(1); v128_t mask_lo = SplatConst16(0x00ff); @@ -1845,7 +4237,7 @@ void ARGBBlendRow_WASMSIMD(const uint8_t* src_argb, // a = 256 - alpha. v128_t a = wasm_i16x8_add( - LIBYUV_WASM_SWIZZLE(wasm_v128_xor(s0, mask_alpha), shuf_a), one); + WasmSwizzle(wasm_v128_xor(s0, mask_alpha), shuf_a), one); v128_t s1_lo = wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(s1, mask_lo), a), 8); v128_t s1_hi = @@ -1880,8 +4272,26 @@ void ARGBBlendRow_WASMSIMD(const uint8_t* src_argb, --width; } } + +#ifdef HAS_ARGBBLENDROW_WASMSIMD +void ARGBBlendRow_WASMSIMD(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + ARGBBlendRow_Wasm(src_argb, src_argb1, dst_argb, width); +} #endif // HAS_ARGBBLENDROW_WASMSIMD +#ifdef HAS_ARGBBLENDROW_RELAXEDSIMD +void ARGBBlendRow_RELAXEDSIMD(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + ARGBBlendRow_Wasm(src_argb, src_argb1, dst_argb, width); +} +#endif // HAS_ARGBBLENDROW_RELAXEDSIMD +#endif // HAS_ARGBBLENDROW_WASMSIMD || HAS_ARGBBLENDROW_RELAXEDSIMD + #ifdef HAS_BLENDPLANEROW_WASMSIMD void BlendPlaneRow_WASMSIMD(const uint8_t* src0, const uint8_t* src1, @@ -1914,6 +4324,52 @@ void BlendPlaneRow_WASMSIMD(const uint8_t* src0, } #endif // HAS_BLENDPLANEROW_WASMSIMD +#if defined(HAS_ARGBATTENUATEROW_WASMSIMD) || \ + defined(HAS_ARGBATTENUATEROW_RELAXEDSIMD) +static const uint8_t kShuffleAttenuateAlpha[16] = { + 3, 3, 3, 128, 7, 7, 7, 128, 11, 11, 11, 128, 15, 15, 15, 128}; + +template +static inline void ARGBAttenuateRow_Wasm(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + v128_t shuf_a = LoadConst(kShuffleAttenuateAlpha); + v128_t mask_a = SplatConst32((int)0xff000000); + v128_t c255 = SplatConst16(255); + do { + asm("" : "+r"(src_argb), "+r"(dst_argb)); + v128_t s = wasm_v128_load(src_argb); + v128_t a = WasmSwizzle(s, shuf_a); + v128_t p_lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(s, a), c255), 8); + v128_t p_hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(s, a), c255), 8); + v128_t res = wasm_v128_or(wasm_u8x16_narrow_i16x8(p_lo, p_hi), + wasm_v128_and(s, mask_a)); + wasm_v128_store(dst_argb, res); + src_argb += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} + +#ifdef HAS_ARGBATTENUATEROW_WASMSIMD +void ARGBAttenuateRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + ARGBAttenuateRow_Wasm(src_argb, dst_argb, width); +} +#endif // HAS_ARGBATTENUATEROW_WASMSIMD + +#ifdef HAS_ARGBATTENUATEROW_RELAXEDSIMD +void ARGBAttenuateRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + ARGBAttenuateRow_Wasm(src_argb, dst_argb, width); +} +#endif // HAS_ARGBATTENUATEROW_RELAXEDSIMD +#endif // HAS_ARGBATTENUATEROW_WASMSIMD || HAS_ARGBATTENUATEROW_RELAXEDSIMD + #ifdef HAS_ARGBEXTRACTALPHAROW_WASMSIMD void ARGBExtractAlphaRow_WASMSIMD(const uint8_t* src_argb, uint8_t* dst_a, @@ -1938,7 +4394,8 @@ void ARGBExtractAlphaRow_WASMSIMD(const uint8_t* src_argb, } #endif // HAS_ARGBEXTRACTALPHAROW_WASMSIMD -#ifdef HAS_ARGBCOPYYTOALPHAROW_WASMSIMD +#if defined(HAS_ARGBCOPYYTOALPHAROW_WASMSIMD) || \ + defined(HAS_ARGBCOPYYTOALPHAROW_RELAXEDSIMD) // Swizzle Y[i] to the alpha byte of 32 bit lane i; 128 indices give 0. static const uint8_t kYToAlphaSwizzleTable[64] = { 128, 128, 128, 0, 128, 128, 128, 1, 128, 128, 128, 2, 128, 128, 128, 3, @@ -1946,7 +4403,10 @@ static const uint8_t kYToAlphaSwizzleTable[64] = { 128, 128, 128, 8, 128, 128, 128, 9, 128, 128, 128, 10, 128, 128, 128, 11, 128, 128, 128, 12, 128, 128, 128, 13, 128, 128, 128, 14, 128, 128, 128, 15}; -void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { +template +static inline void ARGBCopyYToAlphaRow_Wasm(const uint8_t* src, + uint8_t* dst, + int width) { v128_t rgb_mask = SplatConst32(0x00ffffff); v128_t idx0 = LoadConst(kYToAlphaSwizzleTable); v128_t idx1 = LoadConst(kYToAlphaSwizzleTable + 16); @@ -1959,17 +4419,52 @@ void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { v128_t d1 = wasm_v128_and(wasm_v128_load(dst + 16), rgb_mask); v128_t d2 = wasm_v128_and(wasm_v128_load(dst + 32), rgb_mask); v128_t d3 = wasm_v128_and(wasm_v128_load(dst + 48), rgb_mask); - wasm_v128_store(dst, wasm_v128_or(d0, LIBYUV_WASM_SWIZZLE(y, idx0))); - wasm_v128_store(dst + 16, wasm_v128_or(d1, LIBYUV_WASM_SWIZZLE(y, idx1))); - wasm_v128_store(dst + 32, wasm_v128_or(d2, LIBYUV_WASM_SWIZZLE(y, idx2))); - wasm_v128_store(dst + 48, wasm_v128_or(d3, LIBYUV_WASM_SWIZZLE(y, idx3))); + wasm_v128_store(dst, wasm_v128_or(d0, WasmSwizzle(y, idx0))); + wasm_v128_store(dst + 16, wasm_v128_or(d1, WasmSwizzle(y, idx1))); + wasm_v128_store(dst + 32, wasm_v128_or(d2, WasmSwizzle(y, idx2))); + wasm_v128_store(dst + 48, wasm_v128_or(d3, WasmSwizzle(y, idx3))); src += 16; dst += 64; width -= 16; } while (width > 0); } + +#ifdef HAS_ARGBCOPYYTOALPHAROW_WASMSIMD +void ARGBCopyYToAlphaRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { + ARGBCopyYToAlphaRow_Wasm(src, dst, width); +} #endif // HAS_ARGBCOPYYTOALPHAROW_WASMSIMD +#ifdef HAS_ARGBCOPYYTOALPHAROW_RELAXEDSIMD +void ARGBCopyYToAlphaRow_RELAXEDSIMD(const uint8_t* src, + uint8_t* dst, + int width) { + ARGBCopyYToAlphaRow_Wasm(src, dst, width); +} +#endif // HAS_ARGBCOPYYTOALPHAROW_RELAXEDSIMD +#endif // HAS_ARGBCOPYYTOALPHAROW_WASMSIMD || + // HAS_ARGBCOPYYTOALPHAROW_RELAXEDSIMD + +#ifdef HAS_ARGBCOPYALPHAROW_WASMSIMD +void ARGBCopyAlphaRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + v128_t mask_alpha = SplatConst32((int)0xff000000); + do { + asm("" : "+r"(src_argb), "+r"(dst_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + v128_t d0 = wasm_v128_load(dst_argb); + v128_t d1 = wasm_v128_load(dst_argb + 16); + wasm_v128_store(dst_argb, wasm_v128_bitselect(s0, d0, mask_alpha)); + wasm_v128_store(dst_argb + 16, wasm_v128_bitselect(s1, d1, mask_alpha)); + src_argb += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); +} +#endif // HAS_ARGBCOPYALPHAROW_WASMSIMD + #ifdef HAS_ARGBORROW_WASMSIMD void ARGBOrRow_WASMSIMD(const uint8_t* src_argb, uint8_t* dst_argb, @@ -2037,11 +4532,662 @@ void ARGBAlphaBlendRow_WASMSIMD(const uint8_t* src_argb0, } #endif // HAS_ARGBALPHABLENDROW_WASMSIMD -#undef LIBYUV_WASM_SWIZZLE +#ifdef HAS_ARGBMULTIPLYROW_WASMSIMD +void ARGBMultiplyRow_WASMSIMD(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + v128_t c128 = SplatConst16(128); + do { + asm("" : "+r"(src_argb), "+r"(src_argb1), "+r"(dst_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb1); + v128_t lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(s0, s1), c128), 8); + v128_t hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(s0, s1), c128), 8); + wasm_v128_store(dst_argb, wasm_u8x16_narrow_i16x8(lo, hi)); + src_argb += 16; + src_argb1 += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBMULTIPLYROW_WASMSIMD + +#ifdef HAS_ARGBADDROW_WASMSIMD +void ARGBAddRow_WASMSIMD(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + do { + asm("" : "+r"(src_argb), "+r"(src_argb1), "+r"(dst_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb1); + wasm_v128_store(dst_argb, wasm_u8x16_add_sat(s0, s1)); + src_argb += 16; + src_argb1 += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBADDROW_WASMSIMD + +#ifdef HAS_ARGBSUBTRACTROW_WASMSIMD +void ARGBSubtractRow_WASMSIMD(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + do { + asm("" : "+r"(src_argb), "+r"(src_argb1), "+r"(dst_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb1); + wasm_v128_store(dst_argb, wasm_u8x16_sub_sat(s0, s1)); + src_argb += 16; + src_argb1 += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBSUBTRACTROW_WASMSIMD + +#if defined(HAS_ARGBGRAYROW_WASMSIMD) || defined(HAS_ARGBGRAYROW_RELAXEDSIMD) +static const uint16_t kGrayCoeffs[8] = {29, 150, 77, 0, 29, 150, 77, 0}; +static const uint8_t kShuffleGray[16] = {0, 0, 0, 128, 1, 1, 1, 128, + 2, 2, 2, 128, 3, 3, 3, 128}; + +template +static inline void ARGBGrayRow_Wasm(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + v128_t coeffs = LoadConst(kGrayCoeffs); + v128_t shuf = LoadConst(kShuffleGray); + v128_t mask_a = SplatConst32((int)0xff000000); + v128_t c128 = SplatConst32(128); + do { + asm("" : "+r"(src_argb), "+r"(dst_argb)); + v128_t s = wasm_v128_load(src_argb); + v128_t lo = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_low_u8x16(s), coeffs); + v128_t hi = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_high_u8x16(s), coeffs); + v128_t y32 = wasm_u32x4_shr( + wasm_i32x4_add(wasm_i32x4_add(wasm_i32x4_shuffle(lo, hi, 0, 2, 4, 6), + wasm_i32x4_shuffle(lo, hi, 1, 3, 5, 7)), + c128), + 8); + v128_t y16 = wasm_i16x8_narrow_i32x4(y32, y32); + v128_t y = wasm_u8x16_narrow_i16x8(y16, y16); + wasm_v128_store(dst_argb, wasm_v128_or(WasmSwizzle(y, shuf), + wasm_v128_and(s, mask_a))); + src_argb += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} + +#ifdef HAS_ARGBGRAYROW_WASMSIMD +void ARGBGrayRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + ARGBGrayRow_Wasm(src_argb, dst_argb, width); +} +#endif // HAS_ARGBGRAYROW_WASMSIMD + +#ifdef HAS_ARGBGRAYROW_RELAXEDSIMD +void ARGBGrayRow_RELAXEDSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + ARGBGrayRow_Wasm(src_argb, dst_argb, width); +} +#endif // HAS_ARGBGRAYROW_RELAXEDSIMD +#endif // HAS_ARGBGRAYROW_WASMSIMD || HAS_ARGBGRAYROW_RELAXEDSIMD + +#if defined(HAS_ARGBSEPIAROW_WASMSIMD) || defined(HAS_ARGBSEPIAROW_RELAXEDSIMD) +static const uint16_t kSepiaCoeffsB16[8] = {17, 68, 35, 0, 17, 68, 35, 0}; +static const uint16_t kSepiaCoeffsG16[8] = {22, 88, 45, 0, 22, 88, 45, 0}; +static const uint16_t kSepiaCoeffsR16[8] = {24, 98, 50, 0, 24, 98, 50, 0}; + +#ifdef HAS_ARGBSEPIAROW_WASMSIMD +void ARGBSepiaRow_WASMSIMD(uint8_t* dst_argb, int width) { + v128_t cb = LoadConst(kSepiaCoeffsB16); + v128_t cg = LoadConst(kSepiaCoeffsG16); + v128_t cr = LoadConst(kSepiaCoeffsR16); + v128_t mask_a = SplatConst32((int)0xff000000); + do { + v128_t s = wasm_v128_load(dst_argb); + v128_t s_lo = wasm_u16x8_extend_low_u8x16(s); + v128_t s_hi = wasm_u16x8_extend_high_u8x16(s); + v128_t b0 = wasm_i32x4_dot_i16x8(s_lo, cb); + v128_t b1 = wasm_i32x4_dot_i16x8(s_hi, cb); + v128_t g0 = wasm_i32x4_dot_i16x8(s_lo, cg); + v128_t g1 = wasm_i32x4_dot_i16x8(s_hi, cg); + v128_t r0 = wasm_i32x4_dot_i16x8(s_lo, cr); + v128_t r1 = wasm_i32x4_dot_i16x8(s_hi, cr); + v128_t b32 = + wasm_u32x4_shr(wasm_i32x4_add(wasm_i32x4_shuffle(b0, b1, 0, 2, 4, 6), + wasm_i32x4_shuffle(b0, b1, 1, 3, 5, 7)), + 7); + v128_t g32 = + wasm_u32x4_shr(wasm_i32x4_add(wasm_i32x4_shuffle(g0, g1, 0, 2, 4, 6), + wasm_i32x4_shuffle(g0, g1, 1, 3, 5, 7)), + 7); + v128_t r32 = + wasm_u32x4_shr(wasm_i32x4_add(wasm_i32x4_shuffle(r0, r1, 0, 2, 4, 6), + wasm_i32x4_shuffle(r0, r1, 1, 3, 5, 7)), + 7); + v128_t bg = wasm_u8x16_narrow_i16x8(wasm_i16x8_narrow_i32x4(b32, b32), + wasm_i16x8_narrow_i32x4(g32, g32)); + v128_t r = wasm_u8x16_narrow_i16x8(wasm_i16x8_narrow_i32x4(r32, r32), + wasm_i16x8_const_splat(0)); + v128_t bgr = wasm_i8x16_shuffle(bg, r, 0, 8, 16, 31, 1, 9, 17, 31, 2, 10, + 18, 31, 3, 11, 19, 31); + wasm_v128_store(dst_argb, wasm_v128_or(bgr, wasm_v128_and(s, mask_a))); + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBSEPIAROW_WASMSIMD + +#ifdef HAS_ARGBSEPIAROW_RELAXEDSIMD +static const uint8_t kSepiaCoeffsB8[16] = {17, 68, 35, 0, 17, 68, 35, 0, + 17, 68, 35, 0, 17, 68, 35, 0}; +static const uint8_t kSepiaCoeffsG8[16] = {22, 88, 45, 0, 22, 88, 45, 0, + 22, 88, 45, 0, 22, 88, 45, 0}; +static const uint8_t kSepiaCoeffsR8[16] = {24, 98, 50, 0, 24, 98, 50, 0, + 24, 98, 50, 0, 24, 98, 50, 0}; + +void ARGBSepiaRow_RELAXEDSIMD(uint8_t* dst_argb, int width) { + v128_t cb = LoadConst(kSepiaCoeffsB8); + v128_t cg = LoadConst(kSepiaCoeffsG8); + v128_t cr = LoadConst(kSepiaCoeffsR8); + // All weights are in [0, 98] <= 127. Biasing pixels by -128 makes the + // first operand signed and the second i7x16, which is deterministic on + // both x86 and Arm. + v128_t x80 = SplatConst8(-128); + v128_t bias_b = SplatConst32(128 * (17 + 68 + 35)); + v128_t bias_g = SplatConst32(128 * (22 + 88 + 45)); + v128_t bias_r = SplatConst32(128 * (24 + 98 + 50)); + v128_t mask_a = SplatConst32((int)0xff000000); + do { + v128_t s = wasm_v128_load(dst_argb); + v128_t ss = wasm_v128_xor(s, x80); + v128_t b = wasm_u32x4_shr( + wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ss, cb, bias_b), 7); + v128_t g = wasm_u32x4_shr( + wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ss, cg, bias_g), 7); + v128_t r = wasm_u32x4_shr( + wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ss, cr, bias_r), 7); + v128_t bg = wasm_u8x16_narrow_i16x8(wasm_i16x8_narrow_i32x4(b, b), + wasm_i16x8_narrow_i32x4(g, g)); + v128_t r8 = wasm_u8x16_narrow_i16x8(wasm_i16x8_narrow_i32x4(r, r), + wasm_i16x8_const_splat(0)); + v128_t bgr = wasm_i8x16_shuffle(bg, r8, 0, 8, 16, 31, 1, 9, 17, 31, 2, 10, + 18, 31, 3, 11, 19, 31); + wasm_v128_store(dst_argb, wasm_v128_or(bgr, wasm_v128_and(s, mask_a))); + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBSEPIAROW_RELAXEDSIMD +#endif // HAS_ARGBSEPIAROW_WASMSIMD || HAS_ARGBSEPIAROW_RELAXEDSIMD + +#ifdef HAS_ARGBCOLORMATRIXROW_WASMSIMD +void ARGBColorMatrixRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + v128_t m = wasm_v128_load(matrix_argb); + v128_t m_lo = wasm_i16x8_extend_low_i8x16(m); + v128_t m_hi = wasm_i16x8_extend_high_i8x16(m); + v128_t mb = wasm_i64x2_shuffle(m_lo, m_lo, 0, 0); + v128_t mg = wasm_i64x2_shuffle(m_lo, m_lo, 1, 1); + v128_t mr = wasm_i64x2_shuffle(m_hi, m_hi, 0, 0); + v128_t ma = wasm_i64x2_shuffle(m_hi, m_hi, 1, 1); + do { + asm("" : "+r"(src_argb), "+r"(dst_argb)); + v128_t s = wasm_v128_load(src_argb); + v128_t s_lo = wasm_u16x8_extend_low_u8x16(s); + v128_t s_hi = wasm_u16x8_extend_high_u8x16(s); + v128_t b0 = wasm_i32x4_dot_i16x8(s_lo, mb); + v128_t b1 = wasm_i32x4_dot_i16x8(s_hi, mb); + v128_t g0 = wasm_i32x4_dot_i16x8(s_lo, mg); + v128_t g1 = wasm_i32x4_dot_i16x8(s_hi, mg); + v128_t r0 = wasm_i32x4_dot_i16x8(s_lo, mr); + v128_t r1 = wasm_i32x4_dot_i16x8(s_hi, mr); + v128_t a0 = wasm_i32x4_dot_i16x8(s_lo, ma); + v128_t a1 = wasm_i32x4_dot_i16x8(s_hi, ma); + v128_t b32 = + wasm_i32x4_shr(wasm_i32x4_add(wasm_i32x4_shuffle(b0, b1, 0, 2, 4, 6), + wasm_i32x4_shuffle(b0, b1, 1, 3, 5, 7)), + 6); + v128_t g32 = + wasm_i32x4_shr(wasm_i32x4_add(wasm_i32x4_shuffle(g0, g1, 0, 2, 4, 6), + wasm_i32x4_shuffle(g0, g1, 1, 3, 5, 7)), + 6); + v128_t r32 = + wasm_i32x4_shr(wasm_i32x4_add(wasm_i32x4_shuffle(r0, r1, 0, 2, 4, 6), + wasm_i32x4_shuffle(r0, r1, 1, 3, 5, 7)), + 6); + v128_t a32 = + wasm_i32x4_shr(wasm_i32x4_add(wasm_i32x4_shuffle(a0, a1, 0, 2, 4, 6), + wasm_i32x4_shuffle(a0, a1, 1, 3, 5, 7)), + 6); + v128_t b = wasm_i16x8_narrow_i32x4(b32, b32); + v128_t g = wasm_i16x8_narrow_i32x4(g32, g32); + v128_t r = wasm_i16x8_narrow_i32x4(r32, r32); + v128_t a = wasm_i16x8_narrow_i32x4(a32, a32); + v128_t bg = wasm_u8x16_narrow_i16x8(b, g); + v128_t ra = wasm_u8x16_narrow_i16x8(r, a); + wasm_v128_store( + dst_argb, wasm_i8x16_shuffle(bg, ra, 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, + 18, 26, 3, 11, 19, 27)); + src_argb += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBCOLORMATRIXROW_WASMSIMD + +#ifdef HAS_ARGBQUANTIZEROW_WASMSIMD +void ARGBQuantizeRow_WASMSIMD(uint8_t* dst_argb, + int scale, + int interval_size, + int interval_offset, + int width) { + v128_t vscale = wasm_i16x8_splat(scale); + v128_t vsize = wasm_i16x8_splat(interval_size); + v128_t voffset = wasm_i8x16_splat(interval_offset); + v128_t mask_a = SplatConst32((int)0xff000000); + do { + v128_t s = wasm_v128_load(dst_argb); + v128_t s_lo = wasm_u16x8_extend_low_u8x16(s); + v128_t s_hi = wasm_u16x8_extend_high_u8x16(s); + v128_t q_lo = wasm_i16x8_mul( + wasm_i16x8_shuffle(wasm_u32x4_extmul_low_u16x8(s_lo, vscale), + wasm_u32x4_extmul_high_u16x8(s_lo, vscale), 1, 3, 5, + 7, 9, 11, 13, 15), + vsize); + v128_t q_hi = wasm_i16x8_mul( + wasm_i16x8_shuffle(wasm_u32x4_extmul_low_u16x8(s_hi, vscale), + wasm_u32x4_extmul_high_u16x8(s_hi, vscale), 1, 3, 5, + 7, 9, 11, 13, 15), + vsize); + v128_t q = wasm_i8x16_add(wasm_u8x16_narrow_i16x8(q_lo, q_hi), voffset); + wasm_v128_store(dst_argb, wasm_v128_bitselect(s, q, mask_a)); + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBQUANTIZEROW_WASMSIMD + +#ifdef HAS_ARGBSHADEROW_WASMSIMD +void ARGBShadeRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + uint32_t value) { + v128_t v = wasm_i32x4_splat((int32_t)value); + v128_t c128 = SplatConst16(128); + do { + asm("" : "+r"(src_argb), "+r"(dst_argb)); + v128_t s = wasm_v128_load(src_argb); + v128_t lo = wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(s, v), c128); + v128_t hi = wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(s, v), c128); + lo = wasm_u16x8_shr(wasm_i16x8_add(lo, wasm_u16x8_shr(lo, 8)), 8); + hi = wasm_u16x8_shr(wasm_i16x8_add(hi, wasm_u16x8_shr(hi, 8)), 8); + wasm_v128_store(dst_argb, wasm_u8x16_narrow_i16x8(lo, hi)); + src_argb += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBSHADEROW_WASMSIMD + +#if defined(HAS_SOBELROW_WASMSIMD) || defined(HAS_SOBELROW_RELAXEDSIMD) || \ + defined(HAS_SOBELTOPLANEROW_WASMSIMD) || \ + defined(HAS_SOBELXROW_WASMSIMD) || defined(HAS_SOBELYROW_WASMSIMD) || \ + defined(HAS_SOBELXYROW_WASMSIMD) +void SobelXRow_WASMSIMD(const uint8_t* src_y0, + const uint8_t* src_y1, + const uint8_t* src_y2, + uint8_t* dst_sobelx, + int width) { + do { + v128_t a0 = wasm_v128_load(src_y0); + v128_t a1 = wasm_v128_load(src_y1); + v128_t a2 = wasm_v128_load(src_y2); + v128_t b0 = wasm_v128_load(src_y0 + 2); + v128_t b1 = wasm_v128_load(src_y1 + 2); + v128_t b2 = wasm_v128_load(src_y2 + 2); + v128_t a_lo = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_low_u8x16(a0), + wasm_u16x8_extend_low_u8x16(a2)), + wasm_i16x8_shl(wasm_u16x8_extend_low_u8x16(a1), 1)); + v128_t a_hi = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_high_u8x16(a0), + wasm_u16x8_extend_high_u8x16(a2)), + wasm_i16x8_shl(wasm_u16x8_extend_high_u8x16(a1), 1)); + v128_t b_lo = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_low_u8x16(b0), + wasm_u16x8_extend_low_u8x16(b2)), + wasm_i16x8_shl(wasm_u16x8_extend_low_u8x16(b1), 1)); + v128_t b_hi = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_high_u8x16(b0), + wasm_u16x8_extend_high_u8x16(b2)), + wasm_i16x8_shl(wasm_u16x8_extend_high_u8x16(b1), 1)); + v128_t diff_lo = wasm_i16x8_abs(wasm_i16x8_sub(a_lo, b_lo)); + v128_t diff_hi = wasm_i16x8_abs(wasm_i16x8_sub(a_hi, b_hi)); + wasm_v128_store(dst_sobelx, wasm_u8x16_narrow_i16x8(diff_lo, diff_hi)); + src_y0 += 16; + src_y1 += 16; + src_y2 += 16; + dst_sobelx += 16; + width -= 16; + } while (width > 0); +} + +void SobelYRow_WASMSIMD(const uint8_t* src_y0, + const uint8_t* src_y1, + uint8_t* dst_sobely, + int width) { + do { + v128_t a0 = wasm_v128_load(src_y0); + v128_t a1 = wasm_v128_load(src_y0 + 1); + v128_t a2 = wasm_v128_load(src_y0 + 2); + v128_t b0 = wasm_v128_load(src_y1); + v128_t b1 = wasm_v128_load(src_y1 + 1); + v128_t b2 = wasm_v128_load(src_y1 + 2); + v128_t a_lo = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_low_u8x16(a0), + wasm_u16x8_extend_low_u8x16(a2)), + wasm_i16x8_shl(wasm_u16x8_extend_low_u8x16(a1), 1)); + v128_t a_hi = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_high_u8x16(a0), + wasm_u16x8_extend_high_u8x16(a2)), + wasm_i16x8_shl(wasm_u16x8_extend_high_u8x16(a1), 1)); + v128_t b_lo = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_low_u8x16(b0), + wasm_u16x8_extend_low_u8x16(b2)), + wasm_i16x8_shl(wasm_u16x8_extend_low_u8x16(b1), 1)); + v128_t b_hi = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_high_u8x16(b0), + wasm_u16x8_extend_high_u8x16(b2)), + wasm_i16x8_shl(wasm_u16x8_extend_high_u8x16(b1), 1)); + v128_t diff_lo = wasm_i16x8_abs(wasm_i16x8_sub(a_lo, b_lo)); + v128_t diff_hi = wasm_i16x8_abs(wasm_i16x8_sub(a_hi, b_hi)); + wasm_v128_store(dst_sobely, wasm_u8x16_narrow_i16x8(diff_lo, diff_hi)); + src_y0 += 16; + src_y1 += 16; + dst_sobely += 16; + width -= 16; + } while (width > 0); +} + +template +static inline void SobelRow_Wasm(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + v128_t shuf0 = LoadConst(kShuffleJ400ToARGB); + v128_t shuf1 = LoadConst(kShuffleJ400ToARGB + 16); + v128_t shuf2 = LoadConst(kShuffleJ400ToARGB + 32); + v128_t shuf3 = LoadConst(kShuffleJ400ToARGB + 48); + v128_t alpha = SplatConst32((int)0xff000000); + do { + asm("" : "+r"(dst_argb)); + v128_t v = wasm_u8x16_add_sat(wasm_v128_load(src_sobelx), + wasm_v128_load(src_sobely)); + wasm_v128_store(dst_argb, + wasm_v128_or(WasmSwizzle(v, shuf0), alpha)); + wasm_v128_store(dst_argb + 16, + wasm_v128_or(WasmSwizzle(v, shuf1), alpha)); + wasm_v128_store(dst_argb + 32, + wasm_v128_or(WasmSwizzle(v, shuf2), alpha)); + wasm_v128_store(dst_argb + 48, + wasm_v128_or(WasmSwizzle(v, shuf3), alpha)); + src_sobelx += 16; + src_sobely += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} + +#ifdef HAS_SOBELROW_WASMSIMD +void SobelRow_WASMSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + SobelRow_Wasm(src_sobelx, src_sobely, dst_argb, width); +} +#endif // HAS_SOBELROW_WASMSIMD + +#ifdef HAS_SOBELROW_RELAXEDSIMD +void SobelRow_RELAXEDSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + SobelRow_Wasm(src_sobelx, src_sobely, dst_argb, width); +} +#endif // HAS_SOBELROW_RELAXEDSIMD + +void SobelToPlaneRow_WASMSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_y, + int width) { + do { + wasm_v128_store(dst_y, wasm_u8x16_add_sat(wasm_v128_load(src_sobelx), + wasm_v128_load(src_sobely))); + src_sobelx += 16; + src_sobely += 16; + dst_y += 16; + width -= 16; + } while (width > 0); +} + +void SobelXYRow_WASMSIMD(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + v128_t a = SplatConst8(0xff); + do { + asm("" : "+r"(dst_argb)); + v128_t r = wasm_v128_load(src_sobelx); + v128_t b = wasm_v128_load(src_sobely); + v128_t g = wasm_u8x16_add_sat(r, b); + v128_t bg0 = wasm_i8x16_shuffle(b, g, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + v128_t bg1 = wasm_i8x16_shuffle(b, g, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31); + v128_t ra0 = wasm_i8x16_shuffle(r, a, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + v128_t ra1 = wasm_i8x16_shuffle(r, a, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31); + wasm_v128_store(dst_argb, + wasm_i16x8_shuffle(bg0, ra0, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_v128_store(dst_argb + 16, + wasm_i16x8_shuffle(bg0, ra0, 4, 12, 5, 13, 6, 14, 7, 15)); + wasm_v128_store(dst_argb + 32, + wasm_i16x8_shuffle(bg1, ra1, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_v128_store(dst_argb + 48, + wasm_i16x8_shuffle(bg1, ra1, 4, 12, 5, 13, 6, 14, 7, 15)); + src_sobelx += 16; + src_sobely += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif + +#ifdef HAS_ARGBPOLYNOMIALROW_WASMSIMD +void ARGBPolynomialRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const float* poly, + int width) { + v128_t c0 = wasm_v128_load(poly); + v128_t c1 = wasm_v128_load(poly + 4); + v128_t c2 = wasm_v128_load(poly + 8); + v128_t c3 = wasm_v128_load(poly + 12); + do { + asm("" : "+r"(src_argb), "+r"(dst_argb)); + v128_t s = wasm_v128_load(src_argb); + v128_t s16_0 = wasm_u16x8_extend_low_u8x16(s); + v128_t s16_1 = wasm_u16x8_extend_high_u8x16(s); + v128_t f0 = wasm_f32x4_convert_u32x4(wasm_u32x4_extend_low_u16x8(s16_0)); + v128_t f1 = wasm_f32x4_convert_u32x4(wasm_u32x4_extend_high_u16x8(s16_0)); + v128_t f2 = wasm_f32x4_convert_u32x4(wasm_u32x4_extend_low_u16x8(s16_1)); + v128_t f3 = wasm_f32x4_convert_u32x4(wasm_u32x4_extend_high_u16x8(s16_1)); + v128_t r0 = wasm_f32x4_add( + c0, + wasm_f32x4_mul( + f0, wasm_f32x4_add( + c1, wasm_f32x4_mul( + f0, wasm_f32x4_add(c2, wasm_f32x4_mul(f0, c3)))))); + v128_t r1 = wasm_f32x4_add( + c0, + wasm_f32x4_mul( + f1, wasm_f32x4_add( + c1, wasm_f32x4_mul( + f1, wasm_f32x4_add(c2, wasm_f32x4_mul(f1, c3)))))); + v128_t r2 = wasm_f32x4_add( + c0, + wasm_f32x4_mul( + f2, wasm_f32x4_add( + c1, wasm_f32x4_mul( + f2, wasm_f32x4_add(c2, wasm_f32x4_mul(f2, c3)))))); + v128_t r3 = wasm_f32x4_add( + c0, + wasm_f32x4_mul( + f3, wasm_f32x4_add( + c1, wasm_f32x4_mul( + f3, wasm_f32x4_add(c2, wasm_f32x4_mul(f3, c3)))))); + v128_t i0 = wasm_i16x8_narrow_i32x4(wasm_i32x4_trunc_sat_f32x4(r0), + wasm_i32x4_trunc_sat_f32x4(r1)); + v128_t i1 = wasm_i16x8_narrow_i32x4(wasm_i32x4_trunc_sat_f32x4(r2), + wasm_i32x4_trunc_sat_f32x4(r3)); + wasm_v128_store(dst_argb, wasm_u8x16_narrow_i16x8(i0, i1)); + src_argb += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBPOLYNOMIALROW_WASMSIMD + +#ifdef HAS_ARGBLUMACOLORTABLEROW_WASMSIMD +void ARGBLumaColorTableRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff) { + v128_t coeffs = wasm_u16x8_extend_low_u8x16(wasm_i32x4_splat(lumacoeff)); + do { + v128_t s = wasm_v128_load(src_argb); + v128_t y01 = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_low_u8x16(s), coeffs); + v128_t y23 = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_high_u8x16(s), coeffs); + v128_t y = + wasm_v128_and(wasm_i32x4_add(wasm_i32x4_shuffle(y01, y23, 0, 2, 4, 6), + wasm_i32x4_shuffle(y01, y23, 1, 3, 5, 7)), + SplatConst32(0x7f00)); + uint32_t y_lane0 = (uint32_t)wasm_i32x4_extract_lane(y, 0); + uint32_t y_lane1 = (uint32_t)wasm_i32x4_extract_lane(y, 1); + uint32_t y_lane2 = (uint32_t)wasm_i32x4_extract_lane(y, 2); + uint32_t y_lane3 = (uint32_t)wasm_i32x4_extract_lane(y, 3); + const uint8_t* l0 = luma + y_lane0; + const uint8_t* l1 = luma + y_lane1; + const uint8_t* l2 = luma + y_lane2; + const uint8_t* l3 = luma + y_lane3; + dst_argb[0] = l0[src_argb[0]]; + dst_argb[1] = l0[src_argb[1]]; + dst_argb[2] = l0[src_argb[2]]; + dst_argb[3] = src_argb[3]; + dst_argb[4] = l1[src_argb[4]]; + dst_argb[5] = l1[src_argb[5]]; + dst_argb[6] = l1[src_argb[6]]; + dst_argb[7] = src_argb[7]; + dst_argb[8] = l2[src_argb[8]]; + dst_argb[9] = l2[src_argb[9]]; + dst_argb[10] = l2[src_argb[10]]; + dst_argb[11] = src_argb[11]; + dst_argb[12] = l3[src_argb[12]]; + dst_argb[13] = l3[src_argb[13]]; + dst_argb[14] = l3[src_argb[14]]; + dst_argb[15] = src_argb[15]; + src_argb += 16; + dst_argb += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBLUMACOLORTABLEROW_WASMSIMD + +#ifdef HAS_HALFFLOATROW_WASMSIMD +void HalfFloatRow_WASMSIMD(const uint16_t* src, + uint16_t* dst, + float scale, + int width) { + v128_t vmult = wasm_f32x4_splat(1.9259299444e-34f * scale); + do { + asm("" : "+r"(src), "+r"(dst)); + v128_t s0 = wasm_v128_load(src); + v128_t s1 = wasm_v128_load(src + 8); + v128_t f0 = wasm_u32x4_shr( + wasm_f32x4_mul( + wasm_f32x4_convert_u32x4(wasm_u32x4_extend_low_u16x8(s0)), vmult), + 13); + v128_t f1 = wasm_u32x4_shr( + wasm_f32x4_mul( + wasm_f32x4_convert_u32x4(wasm_u32x4_extend_high_u16x8(s0)), vmult), + 13); + v128_t f2 = wasm_u32x4_shr( + wasm_f32x4_mul( + wasm_f32x4_convert_u32x4(wasm_u32x4_extend_low_u16x8(s1)), vmult), + 13); + v128_t f3 = wasm_u32x4_shr( + wasm_f32x4_mul( + wasm_f32x4_convert_u32x4(wasm_u32x4_extend_high_u16x8(s1)), vmult), + 13); + wasm_v128_store(dst, wasm_i16x8_shuffle(f0, f1, 0, 2, 4, 6, 8, 10, 12, 14)); + wasm_v128_store(dst + 8, + wasm_i16x8_shuffle(f2, f3, 0, 2, 4, 6, 8, 10, 12, 14)); + src += 16; + dst += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_HALFFLOATROW_WASMSIMD + +#ifdef HAS_BYTETOFLOATROW_WASMSIMD +void ByteToFloatRow_WASMSIMD(const uint8_t* src, + float* dst, + float scale, + int width) { + v128_t vscale = wasm_f32x4_splat(scale); + do { + asm("" : "+r"(src), "+r"(dst)); + v128_t s = wasm_v128_load(src); + v128_t s0 = wasm_u16x8_extend_low_u8x16(s); + v128_t s1 = wasm_u16x8_extend_high_u8x16(s); + wasm_v128_store( + dst, + wasm_f32x4_mul( + wasm_f32x4_convert_u32x4(wasm_u32x4_extend_low_u16x8(s0)), vscale)); + wasm_v128_store( + dst + 4, wasm_f32x4_mul( + wasm_f32x4_convert_u32x4(wasm_u32x4_extend_high_u16x8(s0)), + vscale)); + wasm_v128_store( + dst + 8, + wasm_f32x4_mul( + wasm_f32x4_convert_u32x4(wasm_u32x4_extend_low_u16x8(s1)), vscale)); + wasm_v128_store(dst + 12, + wasm_f32x4_mul(wasm_f32x4_convert_u32x4( + wasm_u32x4_extend_high_u16x8(s1)), + vscale)); + src += 16; + dst += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_BYTETOFLOATROW_WASMSIMD #ifdef __cplusplus -} // extern "C" } // namespace libyuv #endif -#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#endif // !defined(LIBYUV_DISABLE_WASMSIMD) && defined(__wasm_simd128__) diff --git a/source/scale.cc b/source/scale.cc index f3cb00aa6..0a583e436 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -546,6 +546,27 @@ static void ScalePlaneDown34(int src_width, } } #endif +#if defined(HAS_SCALEROWDOWN34_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + if (dst_width % 24 == 0) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_RELAXEDSIMD; + ScaleRowDown34_1 = ScaleRowDown34_RELAXEDSIMD; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_RELAXEDSIMD; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_RELAXEDSIMD; + } + } else { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_Any_RELAXEDSIMD; + ScaleRowDown34_1 = ScaleRowDown34_Any_RELAXEDSIMD; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_RELAXEDSIMD; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_RELAXEDSIMD; + } + } + } +#endif for (y = 0; y < dst_height - 2; y += 3) { ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width); @@ -771,6 +792,26 @@ static void ScalePlaneDown38(int src_width, } } #endif +#if defined(HAS_SCALEROWDOWN38_RELAXEDSIMD) + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_Any_RELAXEDSIMD; + ScaleRowDown38_2 = ScaleRowDown38_Any_RELAXEDSIMD; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_RELAXEDSIMD; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_RELAXEDSIMD; + } + if (dst_width % 12 == 0) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_RELAXEDSIMD; + ScaleRowDown38_2 = ScaleRowDown38_RELAXEDSIMD; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_RELAXEDSIMD; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_RELAXEDSIMD; + } + } + } +#endif for (y = 0; y < dst_height - 2; y += 3) { ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); @@ -1675,6 +1716,16 @@ static void ScalePlaneUp2_Linear(int src_width, ScaleRowUp = ScaleRowUp2_Linear_RVV; } #endif +#ifdef HAS_SCALEROWUP2_LINEAR_WASMSIMD + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowUp = ScaleRowUp2_Linear_Any_WASMSIMD; + } +#endif +#ifdef HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ScaleRowUp = ScaleRowUp2_Linear_Any_RELAXEDSIMD; + } +#endif if (dst_height == 1) { ScaleRowUp(src_ptr + ((src_height - 1) / 2) * src_stride, dst_ptr, @@ -1741,6 +1792,16 @@ static void ScalePlaneUp2_Bilinear(int src_width, Scale2RowUp = ScaleRowUp2_Bilinear_RVV; } #endif +#ifdef HAS_SCALEROWUP2_BILINEAR_WASMSIMD + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Scale2RowUp = ScaleRowUp2_Bilinear_Any_WASMSIMD; + } +#endif +#ifdef HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + Scale2RowUp = ScaleRowUp2_Bilinear_Any_RELAXEDSIMD; + } +#endif Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width); dst_ptr += dst_stride; diff --git a/source/scale_any.cc b/source/scale_any.cc index bac11e3fe..1e2705a47 100644 --- a/source/scale_any.cc +++ b/source/scale_any.cc @@ -588,6 +588,26 @@ SDANY(ScaleRowDown34_1_Box_Any_WASMSIMD, 1, 23) #endif +#ifdef HAS_SCALEROWDOWN34_RELAXEDSIMD +SDANY(ScaleRowDown34_Any_RELAXEDSIMD, + ScaleRowDown34_RELAXEDSIMD, + ScaleRowDown34_C, + 4 / 3, + 1, + 23) +SDANY(ScaleRowDown34_0_Box_Any_RELAXEDSIMD, + ScaleRowDown34_0_Box_RELAXEDSIMD, + ScaleRowDown34_0_Box_C, + 4 / 3, + 1, + 23) +SDANY(ScaleRowDown34_1_Box_Any_RELAXEDSIMD, + ScaleRowDown34_1_Box_RELAXEDSIMD, + ScaleRowDown34_1_Box_C, + 4 / 3, + 1, + 23) +#endif #ifdef HAS_SCALEROWDOWN38_WASMSIMD SDANY(ScaleRowDown38_Any_WASMSIMD, ScaleRowDown38_WASMSIMD, @@ -608,6 +628,50 @@ SDANY(ScaleRowDown38_2_Box_Any_WASMSIMD, 1, 11) #endif +#ifdef HAS_SCALEROWDOWN38_RELAXEDSIMD +SDANY(ScaleRowDown38_Any_RELAXEDSIMD, + ScaleRowDown38_RELAXEDSIMD, + ScaleRowDown38_C, + 8 / 3, + 1, + 11) +SDANY(ScaleRowDown38_3_Box_Any_RELAXEDSIMD, + ScaleRowDown38_3_Box_RELAXEDSIMD, + ScaleRowDown38_3_Box_C, + 8 / 3, + 1, + 11) +SDANY(ScaleRowDown38_2_Box_Any_RELAXEDSIMD, + ScaleRowDown38_2_Box_RELAXEDSIMD, + ScaleRowDown38_2_Box_C, + 8 / 3, + 1, + 11) +#endif +#ifdef HAS_SCALEUVROWDOWN2_WASMSIMD +SDANY(ScaleUVRowDown2_Any_WASMSIMD, + ScaleUVRowDown2_WASMSIMD, + ScaleUVRowDown2_C, + 2, + 2, + 7) +#endif +#ifdef HAS_SCALEUVROWDOWN2LINEAR_WASMSIMD +SDANY(ScaleUVRowDown2Linear_Any_WASMSIMD, + ScaleUVRowDown2Linear_WASMSIMD, + ScaleUVRowDown2Linear_C, + 2, + 2, + 7) +#endif +#ifdef HAS_SCALEUVROWDOWN2BOX_WASMSIMD +SDANY(ScaleUVRowDown2Box_Any_WASMSIMD, + ScaleUVRowDown2Box_WASMSIMD, + ScaleUVRowDown2Box_C, + 2, + 2, + 7) +#endif #ifdef HAS_SCALEARGBROWDOWN2_WASMSIMD SDANY(ScaleARGBRowDown2_Any_WASMSIMD, ScaleARGBRowDown2_WASMSIMD, @@ -708,6 +772,13 @@ SDAANY(ScaleUVRowDownEven_Any_NEON, 2, 3) #endif +#ifdef HAS_SCALEUVROWDOWNEVEN_WASMSIMD +SDAANY(ScaleUVRowDownEven_Any_WASMSIMD, + ScaleUVRowDownEven_WASMSIMD, + ScaleUVRowDownEven_C, + 2, + 3) +#endif #ifdef HAS_SCALEARGBROWDOWNEVEN_WASMSIMD SDAANY(ScaleARGBRowDownEven_Any_WASMSIMD, ScaleARGBRowDownEven_WASMSIMD, @@ -997,6 +1068,22 @@ SUH2LANY(ScaleRowUp2_Linear_16_Any_NEON, uint16_t) #endif +#ifdef HAS_SCALEROWUP2_LINEAR_WASMSIMD +SUH2LANY(ScaleRowUp2_Linear_Any_WASMSIMD, + ScaleRowUp2_Linear_WASMSIMD, + ScaleRowUp2_Linear_C, + 15, + uint8_t) +#endif + +#ifdef HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD +SUH2LANY(ScaleRowUp2_Linear_Any_RELAXEDSIMD, + ScaleRowUp2_Linear_RELAXEDSIMD, + ScaleRowUp2_Linear_C, + 15, + uint8_t) +#endif + #undef SUH2LANY // Scale up 2 times using bilinear filter. @@ -1117,6 +1204,22 @@ SU2BLANY(ScaleRowUp2_Bilinear_16_Any_NEON, uint16_t) #endif +#ifdef HAS_SCALEROWUP2_BILINEAR_WASMSIMD +SU2BLANY(ScaleRowUp2_Bilinear_Any_WASMSIMD, + ScaleRowUp2_Bilinear_WASMSIMD, + ScaleRowUp2_Bilinear_C, + 15, + uint8_t) +#endif + +#ifdef HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD +SU2BLANY(ScaleRowUp2_Bilinear_Any_RELAXEDSIMD, + ScaleRowUp2_Bilinear_RELAXEDSIMD, + ScaleRowUp2_Bilinear_C, + 15, + uint8_t) +#endif + #undef SU2BLANY // Scale bi-planar plane up horizontally 2 times using linear filter. @@ -1198,6 +1301,22 @@ SBUH2LANY(ScaleUVRowUp2_Linear_16_Any_NEON, uint16_t) #endif +#ifdef HAS_SCALEUVROWUP2_LINEAR_WASMSIMD +SBUH2LANY(ScaleUVRowUp2_Linear_Any_WASMSIMD, + ScaleUVRowUp2_Linear_WASMSIMD, + ScaleUVRowUp2_Linear_C, + 7, + uint8_t) +#endif + +#ifdef HAS_SCALEUVROWUP2_LINEAR_RELAXEDSIMD +SBUH2LANY(ScaleUVRowUp2_Linear_Any_RELAXEDSIMD, + ScaleUVRowUp2_Linear_RELAXEDSIMD, + ScaleUVRowUp2_Linear_C, + 7, + uint8_t) +#endif + #undef SBUH2LANY // Scale bi-planar plane up 2 times using bilinear filter. @@ -1297,6 +1416,22 @@ SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_NEON, uint16_t) #endif +#ifdef HAS_SCALEUVROWUP2_BILINEAR_WASMSIMD +SBU2BLANY(ScaleUVRowUp2_Bilinear_Any_WASMSIMD, + ScaleUVRowUp2_Bilinear_WASMSIMD, + ScaleUVRowUp2_Bilinear_C, + 7, + uint8_t) +#endif + +#ifdef HAS_SCALEUVROWUP2_BILINEAR_RELAXEDSIMD +SBU2BLANY(ScaleUVRowUp2_Bilinear_Any_RELAXEDSIMD, + ScaleUVRowUp2_Bilinear_RELAXEDSIMD, + ScaleUVRowUp2_Bilinear_C, + 7, + uint8_t) +#endif + #undef SBU2BLANY // Scale UV columns by 3 box filter. diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 81e64c042..9f4d2fa83 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -147,6 +147,22 @@ static void ScaleUVDown2(int src_width, : ScaleUVRowDown2Box_HVX); } #endif +#if defined(HAS_SCALEUVROWDOWN2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleUVRowDown2 = + filtering == kFilterNone + ? ScaleUVRowDown2_Any_WASMSIMD + : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_WASMSIMD + : ScaleUVRowDown2Box_Any_WASMSIMD); + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVRowDown2 = + filtering == kFilterNone + ? ScaleUVRowDown2_WASMSIMD + : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_WASMSIMD + : ScaleUVRowDown2Box_WASMSIMD); + } + } +#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -267,6 +283,14 @@ static int ScaleUVDown4Box(int src_width, ScaleUVRowDown2 = ScaleUVRowDown2Box_HVX; } #endif +#if defined(HAS_SCALEUVROWDOWN2BOX_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVRowDown2 = ScaleUVRowDown2Box_WASMSIMD; + } + } +#endif for (j = 0; j < dst_height; ++j) { ScaleUVRowDown2(src_uv, src_stride, row, dst_width * 2); @@ -740,6 +764,14 @@ static void ScaleUVDownEven(int src_width, #endif } #endif +#if defined(HAS_SCALEUVROWDOWNEVEN_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD) && !filtering) { + ScaleUVRowDownEven = ScaleUVRowDownEven_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + ScaleUVRowDownEven = ScaleUVRowDownEven_WASMSIMD; + } + } +#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -1400,6 +1432,18 @@ static void ScaleUVLinearUp2(int src_width, } #endif +#ifdef HAS_SCALEUVROWUP2_LINEAR_WASMSIMD + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowUp = ScaleUVRowUp2_Linear_Any_WASMSIMD; + } +#endif + +#ifdef HAS_SCALEUVROWUP2_LINEAR_RELAXEDSIMD + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + ScaleRowUp = ScaleUVRowUp2_Linear_Any_RELAXEDSIMD; + } +#endif + if (dst_height == 1) { ScaleRowUp(src_uv + ((src_height - 1) / 2) * src_stride, dst_uv, dst_width); } else { @@ -1460,6 +1504,18 @@ static void ScaleUVBilinearUp2(int src_width, } #endif +#ifdef HAS_SCALEUVROWUP2_BILINEAR_WASMSIMD + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Scale2RowUp = ScaleUVRowUp2_Bilinear_Any_WASMSIMD; + } +#endif + +#ifdef HAS_SCALEUVROWUP2_BILINEAR_RELAXEDSIMD + if (TestCpuFlag(kCpuHasRELAXEDSIMD)) { + Scale2RowUp = ScaleUVRowUp2_Bilinear_Any_RELAXEDSIMD; + } +#endif + Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width); dst_ptr += dst_stride; for (x = 0; x < src_height - 1; ++x) { diff --git a/source/scale_wasm.cc b/source/scale_wasm.cc index 62e613ea3..804deb445 100644 --- a/source/scale_wasm.cc +++ b/source/scale_wasm.cc @@ -11,22 +11,27 @@ #include "libyuv/row.h" #include "libyuv/scale_row.h" -#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#if (!defined(LIBYUV_DISABLE_WASMSIMD) || \ + !defined(LIBYUV_DISABLE_RELAXEDSIMD)) && \ + defined(__wasm_simd128__) #include #ifdef __cplusplus namespace libyuv { -extern "C" { #endif // Arbitrary i8x16.shuffle masks are rematerialized on every use by V8, so // shuffles that do not map to a single instruction use swizzle with masks // loaded before the loop. Mask indices of 128 produce 0. +template +static inline v128_t WasmSwizzle(v128_t a, v128_t b) { #ifdef __wasm_relaxed_simd__ -#define LIBYUV_WASM_SWIZZLE wasm_i8x16_relaxed_swizzle -#else -#define LIBYUV_WASM_SWIZZLE wasm_i8x16_swizzle + if (kRelaxed) { + return wasm_i8x16_relaxed_swizzle(a, b); + } #endif + return wasm_i8x16_swizzle(a, b); +} // Load a constant vector. The empty asm hides the address so the compiler // keeps the vector in a local instead of sinking a v128.const into the loop. @@ -148,7 +153,8 @@ void ScaleRowDown4Box_WASMSIMD(const uint8_t* src_ptr, } #endif // HAS_SCALEROWDOWN4_WASMSIMD -#ifdef HAS_SCALEROWDOWN34_WASMSIMD +#if defined(HAS_SCALEROWDOWN34_WASMSIMD) || \ + defined(HAS_SCALEROWDOWN34_RELAXEDSIMD) static const uint8_t kShuf34[8][16] = { // ScaleRowDown34: 16 bytes from a, then b, then 8 bytes from b. {0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128}, @@ -164,10 +170,11 @@ static const uint8_t kShuf34[8][16] = { }; // Point sample 32 pixels to 24, keeping pixels 0, 1 and 3 of every 4. -void ScaleRowDown34_WASMSIMD(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +template +static inline void ScaleRowDown34_Wasm(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { const v128_t m0 = LoadConst(kShuf34[0]); const v128_t m1 = LoadConst(kShuf34[1]); const v128_t m2 = LoadConst(kShuf34[2]); @@ -175,9 +182,9 @@ void ScaleRowDown34_WASMSIMD(const uint8_t* src_ptr, while (dst_width > 0) { v128_t a = wasm_v128_load(src_ptr); v128_t b = wasm_v128_load(src_ptr + 16); - wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0), - LIBYUV_WASM_SWIZZLE(b, m1))); - wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0); + wasm_v128_store(dst_ptr, wasm_v128_or(WasmSwizzle(a, m0), + WasmSwizzle(b, m1))); + wasm_v128_store64_lane(dst_ptr + 16, WasmSwizzle(b, m2), 0); src_ptr += 32; dst_ptr += 24; dst_width -= 24; @@ -194,18 +201,20 @@ static inline v128_t AvgFloor(v128_t a, v128_t b) { // Horizontal 4 to 3 filter of 16 pixels. Result bytes 4n+0..2 are // (3 * s0 + s1 + 2) >> 2, (s1 + s2 + 1) >> 1, (s2 + 3 * s3 + 2) >> 2 // using (3 * a + b + 2) >> 2 == avgr(a, (a + b) >> 1). Byte 4n+3 is unused. +template static inline v128_t Filter34(v128_t s, v128_t mp, v128_t ma) { - v128_t p = LIBYUV_WASM_SWIZZLE(s, mp); // s0 s1 s3 - v128_t a = LIBYUV_WASM_SWIZZLE(s, ma); // s0 s2 s2 - v128_t b = wasm_u32x4_shr(s, 8); // s1 s2 s3 + v128_t p = WasmSwizzle(s, mp); // s0 s1 s3 + v128_t a = WasmSwizzle(s, ma); // s0 s2 s2 + v128_t b = wasm_u32x4_shr(s, 8); // s1 s2 s3 return wasm_u8x16_avgr(p, AvgFloor(a, b)); } // Filter rows 1 and 2 together, 1 : 1 -void ScaleRowDown34_1_Box_WASMSIMD(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +template +static inline void ScaleRowDown34_1_Box_Wasm(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { const uint8_t* t = src_ptr + src_stride; const v128_t mp = LoadConst(kShuf34[3]); const v128_t ma = LoadConst(kShuf34[4]); @@ -213,13 +222,15 @@ void ScaleRowDown34_1_Box_WASMSIMD(const uint8_t* src_ptr, const v128_t m1 = LoadConst(kShuf34[6]); const v128_t m2 = LoadConst(kShuf34[7]); while (dst_width > 0) { - v128_t a = wasm_u8x16_avgr(Filter34(wasm_v128_load(src_ptr), mp, ma), - Filter34(wasm_v128_load(t), mp, ma)); - v128_t b = wasm_u8x16_avgr(Filter34(wasm_v128_load(src_ptr + 16), mp, ma), - Filter34(wasm_v128_load(t + 16), mp, ma)); - wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0), - LIBYUV_WASM_SWIZZLE(b, m1))); - wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0); + v128_t a = + wasm_u8x16_avgr(Filter34(wasm_v128_load(src_ptr), mp, ma), + Filter34(wasm_v128_load(t), mp, ma)); + v128_t b = wasm_u8x16_avgr( + Filter34(wasm_v128_load(src_ptr + 16), mp, ma), + Filter34(wasm_v128_load(t + 16), mp, ma)); + wasm_v128_store(dst_ptr, wasm_v128_or(WasmSwizzle(a, m0), + WasmSwizzle(b, m1))); + wasm_v128_store64_lane(dst_ptr + 16, WasmSwizzle(b, m2), 0); src_ptr += 32; t += 32; dst_ptr += 24; @@ -228,10 +239,11 @@ void ScaleRowDown34_1_Box_WASMSIMD(const uint8_t* src_ptr, } // Filter rows 0 and 1 together, 3 : 1 -void ScaleRowDown34_0_Box_WASMSIMD(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +template +static inline void ScaleRowDown34_0_Box_Wasm(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { const uint8_t* t = src_ptr + src_stride; const v128_t mp = LoadConst(kShuf34[3]); const v128_t ma = LoadConst(kShuf34[4]); @@ -239,24 +251,67 @@ void ScaleRowDown34_0_Box_WASMSIMD(const uint8_t* src_ptr, const v128_t m1 = LoadConst(kShuf34[6]); const v128_t m2 = LoadConst(kShuf34[7]); while (dst_width > 0) { - v128_t a0 = Filter34(wasm_v128_load(src_ptr), mp, ma); - v128_t b0 = Filter34(wasm_v128_load(src_ptr + 16), mp, ma); - v128_t a1 = Filter34(wasm_v128_load(t), mp, ma); - v128_t b1 = Filter34(wasm_v128_load(t + 16), mp, ma); + v128_t a0 = Filter34(wasm_v128_load(src_ptr), mp, ma); + v128_t b0 = Filter34(wasm_v128_load(src_ptr + 16), mp, ma); + v128_t a1 = Filter34(wasm_v128_load(t), mp, ma); + v128_t b1 = Filter34(wasm_v128_load(t + 16), mp, ma); v128_t a = wasm_u8x16_avgr(a0, AvgFloor(a0, a1)); v128_t b = wasm_u8x16_avgr(b0, AvgFloor(b0, b1)); - wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0), - LIBYUV_WASM_SWIZZLE(b, m1))); - wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0); + wasm_v128_store(dst_ptr, wasm_v128_or(WasmSwizzle(a, m0), + WasmSwizzle(b, m1))); + wasm_v128_store64_lane(dst_ptr + 16, WasmSwizzle(b, m2), 0); src_ptr += 32; t += 32; dst_ptr += 24; dst_width -= 24; } } + +#ifdef HAS_SCALEROWDOWN34_WASMSIMD +void ScaleRowDown34_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown34_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +void ScaleRowDown34_1_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown34_1_Box_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +void ScaleRowDown34_0_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown34_0_Box_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} #endif // HAS_SCALEROWDOWN34_WASMSIMD -#ifdef HAS_SCALEROWDOWN38_WASMSIMD +#ifdef HAS_SCALEROWDOWN34_RELAXEDSIMD +void ScaleRowDown34_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown34_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +void ScaleRowDown34_1_Box_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown34_1_Box_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +void ScaleRowDown34_0_Box_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown34_0_Box_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +#endif // HAS_SCALEROWDOWN34_RELAXEDSIMD +#endif // HAS_SCALEROWDOWN34_WASMSIMD || HAS_SCALEROWDOWN34_RELAXEDSIMD + +#if defined(HAS_SCALEROWDOWN38_WASMSIMD) || \ + defined(HAS_SCALEROWDOWN38_RELAXEDSIMD) static const uint8_t kShuf38[3][16] = { // ScaleRowDown38: 6 bytes from a, then 6 bytes from b. {0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, @@ -266,17 +321,18 @@ static const uint8_t kShuf38[3][16] = { }; // Point sample 32 pixels to 12, keeping pixels 0, 3 and 6 of every 8. -void ScaleRowDown38_WASMSIMD(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +template +static inline void ScaleRowDown38_Wasm(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { const v128_t m0 = LoadConst(kShuf38[0]); const v128_t m1 = LoadConst(kShuf38[1]); (void)src_stride; while (dst_width > 0) { v128_t d = - wasm_v128_or(LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_ptr), m0), - LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_ptr + 16), m1)); + wasm_v128_or(WasmSwizzle(wasm_v128_load(src_ptr), m0), + WasmSwizzle(wasm_v128_load(src_ptr + 16), m1)); wasm_v128_store64_lane(dst_ptr, d, 0); wasm_v128_store32_lane(dst_ptr + 8, d, 2); src_ptr += 32; @@ -299,6 +355,7 @@ static inline v128_t Sum38(v128_t s, v128_t k) { } // Pack 12 results from Sum38 and store them. +template static inline void Store38(uint8_t* dst_ptr, v128_t m, v128_t a, @@ -307,16 +364,17 @@ static inline void Store38(uint8_t* dst_ptr, v128_t d) { v128_t r = wasm_u8x16_narrow_i16x8(wasm_u16x8_narrow_i32x4(a, b), wasm_u16x8_narrow_i32x4(c, d)); - r = LIBYUV_WASM_SWIZZLE(r, m); + r = WasmSwizzle(r, m); wasm_v128_store64_lane(dst_ptr, r, 0); wasm_v128_store32_lane(dst_ptr + 8, r, 2); } // 8x3 -> 3x1 -void ScaleRowDown38_3_Box_WASMSIMD(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +template +static inline void ScaleRowDown38_3_Box_Wasm(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { const v128_t k = wasm_i16x8_const(65536 / 9, 0, 0, 65536 / 9, 0, 0, 65536 / 6, 0); const v128_t m = LoadConst(kShuf38[2]); @@ -328,8 +386,8 @@ void ScaleRowDown38_3_Box_WASMSIMD(const uint8_t* src_ptr, wasm_u16x8_load8x8(p + src_stride)), wasm_u16x8_load8x8(p + src_stride * 2)); } - Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), Sum38(s[2], k), - Sum38(s[3], k)); + Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), + Sum38(s[2], k), Sum38(s[3], k)); src_ptr += 32; dst_ptr += 12; dst_width -= 12; @@ -337,10 +395,11 @@ void ScaleRowDown38_3_Box_WASMSIMD(const uint8_t* src_ptr, } // 8x2 -> 3x1 -void ScaleRowDown38_2_Box_WASMSIMD(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +template +static inline void ScaleRowDown38_2_Box_Wasm(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { const v128_t k = wasm_i16x8_const(65536 / 6, 0, 0, 65536 / 6, 0, 0, 65536 / 4, 0); const v128_t m = LoadConst(kShuf38[2]); @@ -351,15 +410,57 @@ void ScaleRowDown38_2_Box_WASMSIMD(const uint8_t* src_ptr, s[i] = wasm_i16x8_add(wasm_u16x8_load8x8(p), wasm_u16x8_load8x8(p + src_stride)); } - Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), Sum38(s[2], k), - Sum38(s[3], k)); + Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), + Sum38(s[2], k), Sum38(s[3], k)); src_ptr += 32; dst_ptr += 12; dst_width -= 12; } } + +#ifdef HAS_SCALEROWDOWN38_WASMSIMD +void ScaleRowDown38_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown38_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +void ScaleRowDown38_3_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown38_3_Box_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +void ScaleRowDown38_2_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown38_2_Box_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} #endif // HAS_SCALEROWDOWN38_WASMSIMD +#ifdef HAS_SCALEROWDOWN38_RELAXEDSIMD +void ScaleRowDown38_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown38_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +void ScaleRowDown38_3_Box_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown38_3_Box_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +void ScaleRowDown38_2_Box_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + ScaleRowDown38_2_Box_Wasm(src_ptr, src_stride, dst_ptr, dst_width); +} +#endif // HAS_SCALEROWDOWN38_RELAXEDSIMD +#endif // HAS_SCALEROWDOWN38_WASMSIMD || HAS_SCALEROWDOWN38_RELAXEDSIMD + #ifdef HAS_SCALEFILTERCOLS_WASMSIMD // Bilinear column filtering with 7 bit fractions, matching ScaleFilterCols_C: // a + ((f * (b - a) + 64) >> 7). 8 pixels per loop. The fraction @@ -705,8 +806,492 @@ void ScaleUVAddCols3_WASMSIMD(int dst_width, } #endif // HAS_SCALEUVADDCOLS3_WASMSIMD +#ifdef HAS_SCALEUVROWDOWN2_WASMSIMD +void ScaleUVRowDown2_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + int x; + (void)src_stride; + for (x = 0; x < dst_width - 7; x += 8) { + const v128_t s0 = wasm_v128_load(src_uv); + const v128_t s1 = wasm_v128_load(src_uv + 16); + src_uv += 32; + const v128_t d = wasm_i16x8_shuffle(s0, s1, 1, 3, 5, 7, 9, 11, 13, 15); + wasm_v128_store(dst_uv, d); + dst_uv += 16; + } +} +#endif // HAS_SCALEUVROWDOWN2_WASMSIMD + +#ifdef HAS_SCALEUVROWDOWN2LINEAR_WASMSIMD +void ScaleUVRowDown2Linear_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + int x; + (void)src_stride; + for (x = 0; x < dst_width - 7; x += 8) { + const v128_t s0 = wasm_v128_load(src_uv); + const v128_t s1 = wasm_v128_load(src_uv + 16); + src_uv += 32; + const v128_t even = wasm_i16x8_shuffle(s0, s1, 0, 2, 4, 6, 8, 10, 12, 14); + const v128_t odd = wasm_i16x8_shuffle(s0, s1, 1, 3, 5, 7, 9, 11, 13, 15); + wasm_v128_store(dst_uv, wasm_u8x16_avgr(even, odd)); + dst_uv += 16; + } +} +#endif // HAS_SCALEUVROWDOWN2LINEAR_WASMSIMD + +#ifdef HAS_SCALEUVROWDOWN2BOX_WASMSIMD +void ScaleUVRowDown2Box_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + int x; + const uint8_t* t_uv = src_uv + src_stride; + const v128_t kTwo = wasm_i16x8_splat(2); + for (x = 0; x < dst_width - 7; x += 8) { + const v128_t s0 = wasm_v128_load(src_uv); + const v128_t s1 = wasm_v128_load(src_uv + 16); + const v128_t t0 = wasm_v128_load(t_uv); + const v128_t t1 = wasm_v128_load(t_uv + 16); + src_uv += 32; + t_uv += 32; + const v128_t s_even = wasm_i16x8_shuffle(s0, s1, 0, 2, 4, 6, 8, 10, 12, 14); + const v128_t s_odd = wasm_i16x8_shuffle(s0, s1, 1, 3, 5, 7, 9, 11, 13, 15); + const v128_t t_even = wasm_i16x8_shuffle(t0, t1, 0, 2, 4, 6, 8, 10, 12, 14); + const v128_t t_odd = wasm_i16x8_shuffle(t0, t1, 1, 3, 5, 7, 9, 11, 13, 15); + const v128_t sum_lo = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_low_u8x16(s_even), + wasm_u16x8_extend_low_u8x16(s_odd)), + wasm_i16x8_add(wasm_u16x8_extend_low_u8x16(t_even), + wasm_u16x8_extend_low_u8x16(t_odd))); + const v128_t sum_hi = + wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_extend_high_u8x16(s_even), + wasm_u16x8_extend_high_u8x16(s_odd)), + wasm_i16x8_add(wasm_u16x8_extend_high_u8x16(t_even), + wasm_u16x8_extend_high_u8x16(t_odd))); + const v128_t avg_lo = wasm_u16x8_shr(wasm_i16x8_add(sum_lo, kTwo), 2); + const v128_t avg_hi = wasm_u16x8_shr(wasm_i16x8_add(sum_hi, kTwo), 2); + wasm_v128_store(dst_uv, wasm_u8x16_narrow_i16x8(avg_lo, avg_hi)); + dst_uv += 16; + } +} +#endif // HAS_SCALEUVROWDOWN2BOX_WASMSIMD + +#ifdef HAS_SCALEUVROWDOWNEVEN_WASMSIMD +void ScaleUVRowDownEven_WASMSIMD(const uint8_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_uv, + int dst_width) { + int x; + const ptrdiff_t step = (ptrdiff_t)src_stepx * 2; + (void)src_stride; + v128_t v = wasm_i32x4_splat(0); + for (x = 0; x < dst_width - 3; x += 4) { + v = wasm_v128_load16_lane(src_uv, v, 0); + v = wasm_v128_load16_lane(src_uv + step, v, 1); + v = wasm_v128_load16_lane(src_uv + step * 2, v, 2); + v = wasm_v128_load16_lane(src_uv + step * 3, v, 3); + wasm_v128_store64_lane(dst_uv, v, 0); + src_uv += step * 4; + dst_uv += 8; + } +} +#endif // HAS_SCALEUVROWDOWNEVEN_WASMSIMD + +#if defined(HAS_SCALEROWUP2_LINEAR_WASMSIMD) || \ + defined(HAS_SCALEROWUP2_BILINEAR_WASMSIMD) +static inline void ScaleRowUp2Madd31_Wasm(const uint8_t* src, + v128_t* out_lo, + v128_t* out_hi) { + const v128_t s0 = wasm_u16x8_load8x8(src); + const v128_t s1 = wasm_u16x8_load8x8(src + 1); + const v128_t near_lo = wasm_i16x8_shuffle(s0, s1, 0, 8, 1, 9, 2, 10, 3, 11); + const v128_t far_lo = wasm_i16x8_shuffle(s0, s1, 8, 0, 9, 1, 10, 2, 11, 3); + const v128_t near_hi = wasm_i16x8_shuffle(s0, s1, 4, 12, 5, 13, 6, 14, 7, 15); + const v128_t far_hi = wasm_i16x8_shuffle(s0, s1, 12, 4, 13, 5, 14, 6, 15, 7); + *out_lo = wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(near_lo, 1), near_lo), + far_lo); + *out_hi = wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(near_hi, 1), near_hi), + far_hi); +} +#endif // HAS_SCALEROWUP2_LINEAR_WASMSIMD || HAS_SCALEROWUP2_BILINEAR_WASMSIMD + +#if defined(HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD) || \ + defined(HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD) +static inline void ScaleRowUp2Madd31_Relaxed(const uint8_t* src, + v128_t* out_lo, + v128_t* out_hi) { + const v128_t kBias80 = wasm_i8x16_splat(-128); + const v128_t kMadd31 = + wasm_i8x16_const(3, 1, 1, 3, 3, 1, 1, 3, 3, 1, 1, 3, 3, 1, 1, 3); + const v128_t s0 = wasm_v128_xor(wasm_v128_load64_zero(src), kBias80); + const v128_t s1 = wasm_v128_xor(wasm_v128_load64_zero(src + 1), kBias80); + const v128_t lo = wasm_i8x16_shuffle(s0, s1, 0, 16, 0, 16, 1, 17, 1, 17, 2, + 18, 2, 18, 3, 19, 3, 19); + const v128_t hi = wasm_i8x16_shuffle(s0, s1, 4, 20, 4, 20, 5, 21, 5, 21, 6, + 22, 6, 22, 7, 23, 7, 23); + *out_lo = wasm_i16x8_relaxed_dot_i8x16_i7x16(lo, kMadd31); + *out_hi = wasm_i16x8_relaxed_dot_i8x16_i7x16(hi, kMadd31); +} +#endif // HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD || + // HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD + +#ifdef HAS_SCALEROWUP2_LINEAR_WASMSIMD +void ScaleRowUp2_Linear_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width) { + int x; + const v128_t kTwo = wasm_i16x8_splat(2); + for (x = 0; x < dst_width - 15; x += 16) { + v128_t lo, hi; + ScaleRowUp2Madd31_Wasm(src_ptr, &lo, &hi); + src_ptr += 8; + lo = wasm_u16x8_shr(wasm_i16x8_add(lo, kTwo), 2); + hi = wasm_u16x8_shr(wasm_i16x8_add(hi, kTwo), 2); + wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(lo, hi)); + dst_ptr += 16; + } +} +#endif // HAS_SCALEROWUP2_LINEAR_WASMSIMD + +#ifdef HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD +void ScaleRowUp2_Linear_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width) { + int x; + const v128_t kBias514 = wasm_i16x8_splat(512 + 2); + for (x = 0; x < dst_width - 15; x += 16) { + v128_t lo, hi; + ScaleRowUp2Madd31_Relaxed(src_ptr, &lo, &hi); + src_ptr += 8; + lo = wasm_u16x8_shr(wasm_i16x8_add(lo, kBias514), 2); + hi = wasm_u16x8_shr(wasm_i16x8_add(hi, kBias514), 2); + wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(lo, hi)); + dst_ptr += 16; + } +} +#endif // HAS_SCALEROWUP2_LINEAR_RELAXEDSIMD + +#ifdef HAS_SCALEROWUP2_BILINEAR_WASMSIMD +void ScaleRowUp2_Bilinear_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + int x; + const uint8_t* s = src_ptr; + const uint8_t* t = src_ptr + src_stride; + uint8_t* d = dst_ptr; + uint8_t* e = dst_ptr + dst_stride; + const v128_t kEight = wasm_i16x8_splat(8); + for (x = 0; x < dst_width - 15; x += 16) { + v128_t s_lo, s_hi, t_lo, t_hi; + ScaleRowUp2Madd31_Wasm(s, &s_lo, &s_hi); + ScaleRowUp2Madd31_Wasm(t, &t_lo, &t_hi); + s += 8; + t += 8; + const v128_t d_lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(s_lo, 1), s_lo), + wasm_i16x8_add(t_lo, kEight)), + 4); + const v128_t d_hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(s_hi, 1), s_hi), + wasm_i16x8_add(t_hi, kEight)), + 4); + const v128_t e_lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(t_lo, 1), t_lo), + wasm_i16x8_add(s_lo, kEight)), + 4); + const v128_t e_hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(t_hi, 1), t_hi), + wasm_i16x8_add(s_hi, kEight)), + 4); + wasm_v128_store(d, wasm_u8x16_narrow_i16x8(d_lo, d_hi)); + wasm_v128_store(e, wasm_u8x16_narrow_i16x8(e_lo, e_hi)); + d += 16; + e += 16; + } +} +#endif // HAS_SCALEROWUP2_BILINEAR_WASMSIMD + +#ifdef HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD +void ScaleRowUp2_Bilinear_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + int x; + const uint8_t* s = src_ptr; + const uint8_t* t = src_ptr + src_stride; + uint8_t* d = dst_ptr; + uint8_t* e = dst_ptr + dst_stride; + const v128_t kBias2056 = wasm_i16x8_splat(2048 + 8); + for (x = 0; x < dst_width - 15; x += 16) { + v128_t s_lo, s_hi, t_lo, t_hi; + ScaleRowUp2Madd31_Relaxed(s, &s_lo, &s_hi); + ScaleRowUp2Madd31_Relaxed(t, &t_lo, &t_hi); + s += 8; + t += 8; + const v128_t d_lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(s_lo, 1), s_lo), + wasm_i16x8_add(t_lo, kBias2056)), + 4); + const v128_t d_hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(s_hi, 1), s_hi), + wasm_i16x8_add(t_hi, kBias2056)), + 4); + const v128_t e_lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(t_lo, 1), t_lo), + wasm_i16x8_add(s_lo, kBias2056)), + 4); + const v128_t e_hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(t_hi, 1), t_hi), + wasm_i16x8_add(s_hi, kBias2056)), + 4); + wasm_v128_store(d, wasm_u8x16_narrow_i16x8(d_lo, d_hi)); + wasm_v128_store(e, wasm_u8x16_narrow_i16x8(e_lo, e_hi)); + d += 16; + e += 16; + } +} +#endif // HAS_SCALEROWUP2_BILINEAR_RELAXEDSIMD + +#if defined(HAS_SCALEUVROWUP2_LINEAR_WASMSIMD) || \ + defined(HAS_SCALEUVROWUP2_BILINEAR_WASMSIMD) +static inline void ScaleUVRowUp2Madd31_Wasm(const uint8_t* src, + v128_t* out_lo, + v128_t* out_hi) { + const v128_t s0 = wasm_u16x8_load8x8(src); + const v128_t s1 = wasm_u16x8_load8x8(src + 2); + const v128_t near_lo = wasm_i16x8_shuffle(s0, s1, 0, 1, 8, 9, 2, 3, 10, 11); + const v128_t far_lo = wasm_i16x8_shuffle(s0, s1, 8, 9, 0, 1, 10, 11, 2, 3); + const v128_t near_hi = wasm_i16x8_shuffle(s0, s1, 4, 5, 12, 13, 6, 7, 14, 15); + const v128_t far_hi = wasm_i16x8_shuffle(s0, s1, 12, 13, 4, 5, 14, 15, 6, 7); + *out_lo = wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(near_lo, 1), near_lo), + far_lo); + *out_hi = wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(near_hi, 1), near_hi), + far_hi); +} +#endif // HAS_SCALEUVROWUP2_LINEAR_WASMSIMD || + // HAS_SCALEUVROWUP2_BILINEAR_WASMSIMD + +#if defined(HAS_SCALEUVROWUP2_LINEAR_RELAXEDSIMD) || \ + defined(HAS_SCALEUVROWUP2_BILINEAR_RELAXEDSIMD) +static inline void ScaleUVRowUp2Madd31_Relaxed(const uint8_t* src, + v128_t* out_lo, + v128_t* out_hi) { + const v128_t kBias80 = wasm_i8x16_splat(-128); + const v128_t kUVMadd31 = + wasm_i8x16_const(3, 1, 3, 1, 1, 3, 1, 3, 3, 1, 3, 1, 1, 3, 1, 3); + const v128_t s0 = wasm_v128_xor(wasm_v128_load64_zero(src), kBias80); + const v128_t s1 = wasm_v128_xor(wasm_v128_load64_zero(src + 2), kBias80); + const v128_t lo = wasm_i8x16_shuffle(s0, s1, 0, 16, 1, 17, 0, 16, 1, 17, 2, + 18, 3, 19, 2, 18, 3, 19); + const v128_t hi = wasm_i8x16_shuffle(s0, s1, 4, 20, 5, 21, 4, 20, 5, 21, 6, + 22, 7, 23, 6, 22, 7, 23); + *out_lo = wasm_i16x8_relaxed_dot_i8x16_i7x16(lo, kUVMadd31); + *out_hi = wasm_i16x8_relaxed_dot_i8x16_i7x16(hi, kUVMadd31); +} +#endif // HAS_SCALEUVROWUP2_LINEAR_RELAXEDSIMD || + // HAS_SCALEUVROWUP2_BILINEAR_RELAXEDSIMD + +#ifdef HAS_SCALEUVROWUP2_LINEAR_WASMSIMD +void ScaleUVRowUp2_Linear_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width) { + int x; + const v128_t kTwo = wasm_i16x8_splat(2); + for (x = 0; x < dst_width - 7; x += 8) { + v128_t lo, hi; + ScaleUVRowUp2Madd31_Wasm(src_ptr, &lo, &hi); + src_ptr += 8; + lo = wasm_u16x8_shr(wasm_i16x8_add(lo, kTwo), 2); + hi = wasm_u16x8_shr(wasm_i16x8_add(hi, kTwo), 2); + wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(lo, hi)); + dst_ptr += 16; + } +} +#endif // HAS_SCALEUVROWUP2_LINEAR_WASMSIMD + +#ifdef HAS_SCALEUVROWUP2_LINEAR_RELAXEDSIMD +void ScaleUVRowUp2_Linear_RELAXEDSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width) { + int x; + const v128_t kBias514 = wasm_i16x8_splat(512 + 2); + for (x = 0; x < dst_width - 7; x += 8) { + v128_t lo, hi; + ScaleUVRowUp2Madd31_Relaxed(src_ptr, &lo, &hi); + src_ptr += 8; + lo = wasm_u16x8_shr(wasm_i16x8_add(lo, kBias514), 2); + hi = wasm_u16x8_shr(wasm_i16x8_add(hi, kBias514), 2); + wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(lo, hi)); + dst_ptr += 16; + } +} +#endif // HAS_SCALEUVROWUP2_LINEAR_RELAXEDSIMD + +#ifdef HAS_SCALEUVROWUP2_BILINEAR_WASMSIMD +void ScaleUVRowUp2_Bilinear_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + int x; + const uint8_t* s = src_ptr; + const uint8_t* t = src_ptr + src_stride; + uint8_t* d = dst_ptr; + uint8_t* e = dst_ptr + dst_stride; + const v128_t kEight = wasm_i16x8_splat(8); + for (x = 0; x < dst_width - 7; x += 8) { + v128_t s_lo, s_hi, t_lo, t_hi; + ScaleUVRowUp2Madd31_Wasm(s, &s_lo, &s_hi); + ScaleUVRowUp2Madd31_Wasm(t, &t_lo, &t_hi); + s += 8; + t += 8; + const v128_t d_lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(s_lo, 1), s_lo), + wasm_i16x8_add(t_lo, kEight)), + 4); + const v128_t d_hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(s_hi, 1), s_hi), + wasm_i16x8_add(t_hi, kEight)), + 4); + const v128_t e_lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(t_lo, 1), t_lo), + wasm_i16x8_add(s_lo, kEight)), + 4); + const v128_t e_hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(t_hi, 1), t_hi), + wasm_i16x8_add(s_hi, kEight)), + 4); + wasm_v128_store(d, wasm_u8x16_narrow_i16x8(d_lo, d_hi)); + wasm_v128_store(e, wasm_u8x16_narrow_i16x8(e_lo, e_hi)); + d += 16; + e += 16; + } +} +#endif // HAS_SCALEUVROWUP2_BILINEAR_WASMSIMD + +#ifdef HAS_SCALEUVROWUP2_BILINEAR_RELAXEDSIMD +void ScaleUVRowUp2_Bilinear_RELAXEDSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + int x; + const uint8_t* s = src_ptr; + const uint8_t* t = src_ptr + src_stride; + uint8_t* d = dst_ptr; + uint8_t* e = dst_ptr + dst_stride; + const v128_t kBias2056 = wasm_i16x8_splat(2048 + 8); + for (x = 0; x < dst_width - 7; x += 8) { + v128_t s_lo, s_hi, t_lo, t_hi; + ScaleUVRowUp2Madd31_Relaxed(s, &s_lo, &s_hi); + ScaleUVRowUp2Madd31_Relaxed(t, &t_lo, &t_hi); + s += 8; + t += 8; + const v128_t d_lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(s_lo, 1), s_lo), + wasm_i16x8_add(t_lo, kBias2056)), + 4); + const v128_t d_hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(s_hi, 1), s_hi), + wasm_i16x8_add(t_hi, kBias2056)), + 4); + const v128_t e_lo = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(t_lo, 1), t_lo), + wasm_i16x8_add(s_lo, kBias2056)), + 4); + const v128_t e_hi = wasm_u16x8_shr( + wasm_i16x8_add(wasm_i16x8_add(wasm_i16x8_shl(t_hi, 1), t_hi), + wasm_i16x8_add(s_hi, kBias2056)), + 4); + wasm_v128_store(d, wasm_u8x16_narrow_i16x8(d_lo, d_hi)); + wasm_v128_store(e, wasm_u8x16_narrow_i16x8(e_lo, e_hi)); + d += 16; + e += 16; + } +} +#endif // HAS_SCALEUVROWUP2_BILINEAR_RELAXEDSIMD + +#ifdef HAS_SCALESAMPLES_WASMSIMD +void ScaleSamples_WASMSIMD(const float* src, + float* dst, + float scale, + int width) { + const v128_t vscale = wasm_f32x4_splat(scale); + for (; width >= 4; width -= 4) { + const v128_t v = wasm_v128_load(src); + src += 4; + wasm_v128_store(dst, wasm_f32x4_mul(v, vscale)); + dst += 4; + } + for (; width > 0; --width) { + *dst++ = *src++ * scale; + } +} +#endif // HAS_SCALESAMPLES_WASMSIMD + +#ifdef HAS_SCALESUMSAMPLES_WASMSIMD +float ScaleSumSamples_WASMSIMD(const float* src, + float* dst, + float scale, + int width) { + v128_t vsum = wasm_f32x4_splat(0.f); + const v128_t vscale = wasm_f32x4_splat(scale); + for (; width >= 4; width -= 4) { + const v128_t v = wasm_v128_load(src); + src += 4; + vsum = wasm_f32x4_add(vsum, wasm_f32x4_mul(v, v)); + wasm_v128_store(dst, wasm_f32x4_mul(v, vscale)); + dst += 4; + } + vsum = wasm_f32x4_add(vsum, wasm_i32x4_shuffle(vsum, vsum, 2, 3, 0, 1)); + vsum = wasm_f32x4_add(vsum, wasm_i32x4_shuffle(vsum, vsum, 1, 0, 3, 2)); + float fsum = wasm_f32x4_extract_lane(vsum, 0); + for (; width > 0; --width) { + const float v = *src++; + fsum += v * v; + *dst++ = v * scale; + } + return fsum; +} +#endif // HAS_SCALESUMSAMPLES_WASMSIMD + +#ifdef HAS_SCALEMAXSAMPLES_WASMSIMD +float ScaleMaxSamples_WASMSIMD(const float* src, + float* dst, + float scale, + int width) { + v128_t vmax = wasm_f32x4_splat(0.f); + const v128_t vscale = wasm_f32x4_splat(scale); + for (; width >= 4; width -= 4) { + const v128_t v = wasm_v128_load(src); + src += 4; + vmax = wasm_f32x4_pmax(vmax, v); + wasm_v128_store(dst, wasm_f32x4_mul(v, vscale)); + dst += 4; + } + vmax = wasm_f32x4_pmax(vmax, wasm_i32x4_shuffle(vmax, vmax, 2, 3, 0, 1)); + vmax = wasm_f32x4_pmax(vmax, wasm_i32x4_shuffle(vmax, vmax, 1, 0, 3, 2)); + float fmax = wasm_f32x4_extract_lane(vmax, 0); + for (; width > 0; --width) { + const float v = *src++; + if (v > fmax) { + fmax = v; + } + *dst++ = v * scale; + } + return fmax; +} +#endif // HAS_SCALEMAXSAMPLES_WASMSIMD + #ifdef __cplusplus -} // extern "C" } // namespace libyuv #endif diff --git a/unit_test/cpu_test.cc b/unit_test/cpu_test.cc index ed05301db..c6cd0ce74 100644 --- a/unit_test/cpu_test.cc +++ b/unit_test/cpu_test.cc @@ -167,8 +167,10 @@ TEST_F(LibYUVBaseTest, TestCpuHas) { int has_wasm = TestCpuFlag(kCpuHasWASM); if (has_wasm) { int has_wasm_simd = TestCpuFlag(kCpuHasWASMSIMD); + int has_relaxed_simd = TestCpuFlag(kCpuHasRELAXEDSIMD); printf("Has WASM 0x%x\n", has_wasm); printf("Has WASM SIMD 0x%x\n", has_wasm_simd); + printf("Has RELAXED SIMD 0x%x\n", has_relaxed_simd); } #endif // defined(__wasm__) @@ -325,6 +327,9 @@ TEST_F(LibYUVBaseTest, TestCompilerMacros) { #ifdef __wasm_simd128__ printf("__wasm_simd128__ %d\n", __wasm_simd128__); #endif +#ifdef __wasm_relaxed_simd__ + printf("__wasm_relaxed_simd__ %d\n", __wasm_relaxed_simd__); +#endif #ifdef __hexagon__ printf("__hexagon__ %d\n", __hexagon__); #endif @@ -472,6 +477,11 @@ TEST_F(LibYUVBaseTest, MAYBE_TestSetCpuFlags) { SetCpuFlags(cpu_flags); ASSERT_EQ(cpu_flags, TestCpuFlag(-1)); + cpu_flags = + kCpuHasWASM | kCpuHasWASMSIMD | kCpuHasRELAXEDSIMD | kCpuInitialized; + SetCpuFlags(cpu_flags); + ASSERT_EQ(cpu_flags, TestCpuFlag(-1)); + // Test that setting 0 turns auto-init back on. SetCpuFlags(0); ASSERT_EQ(original_cpu_flags, TestCpuFlag(-1)); diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 68b265a74..1d21e8c2d 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -4890,6 +4890,16 @@ float TestScaleMaxSamples(int benchmark_width, reinterpret_cast(dst_opt), scale, kPixels); } +#elif defined(HAS_SCALEMAXSAMPLES_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + max_opt = ScaleMaxSamples_WASMSIMD(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), + scale, kPixels); + } else { + max_opt = ScaleMaxSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #else max_opt = ScaleMaxSamples_C(reinterpret_cast(orig_y), @@ -4997,6 +5007,16 @@ float TestScaleSumSamples(int benchmark_width, reinterpret_cast(dst_opt), scale, kPixels); } +#elif defined(HAS_SCALESUMSAMPLES_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + sum_opt = ScaleSumSamples_WASMSIMD(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), + scale, kPixels); + } else { + sum_opt = ScaleSumSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #else sum_opt = ScaleSumSamples_C(reinterpret_cast(orig_y), @@ -5107,6 +5127,15 @@ float TestScaleSamples(int benchmark_width, ScaleSamples_C(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); } +#elif defined(HAS_SCALESAMPLES_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleSamples_WASMSIMD(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else { + ScaleSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } #else ScaleSamples_C(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); @@ -5176,6 +5205,12 @@ float TestCopySamples(int benchmark_width, } else { CopyRow_C(orig_y, dst_opt, kPixels * 4); } +#elif defined(HAS_COPYROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + CopyRow_WASMSIMD(orig_y, dst_opt, kPixels * 4); + } else { + CopyRow_C(orig_y, dst_opt, kPixels * 4); + } #else CopyRow_C(orig_y, dst_opt, kPixels * 4); #endif diff --git a/unit_test/unit_test.cc b/unit_test/unit_test.cc index f3f3a79b1..d51eeec60 100644 --- a/unit_test/unit_test.cc +++ b/unit_test/unit_test.cc @@ -135,8 +135,14 @@ static int TestCpuEnv(int cpu_info) { #endif #if defined(__wasm__) if (TestEnv("LIBYUV_DISABLE_WASM")) { + cpu_info &= ~(libyuv::kCpuHasWASMSIMD | libyuv::kCpuHasRELAXEDSIMD); + } + if (TestEnv("LIBYUV_DISABLE_WASMSIMD")) { cpu_info &= ~libyuv::kCpuHasWASMSIMD; } + if (TestEnv("LIBYUV_DISABLE_RELAXEDSIMD")) { + cpu_info &= ~libyuv::kCpuHasRELAXEDSIMD; + } #endif #if defined(__hexagon__) if (TestEnv("LIBYUV_DISABLE_HVX")) { diff --git a/util/cpuid.c b/util/cpuid.c index e12c05729..2ae73bb45 100644 --- a/util/cpuid.c +++ b/util/cpuid.c @@ -140,8 +140,10 @@ int main(int argc, const char* argv[]) { int has_wasm = TestCpuFlag(kCpuHasWASM); if (has_wasm) { int has_wasm_simd = TestCpuFlag(kCpuHasWASMSIMD); + int has_relaxed_simd = TestCpuFlag(kCpuHasRELAXEDSIMD); printf("Has WASM 0x%x\n", has_wasm); printf("Has WASM SIMD 0x%x\n", has_wasm_simd); + printf("Has RELAXED SIMD 0x%x\n", has_relaxed_simd); } #endif // defined(__wasm__)