From 98697ab033ff9fb56dbe42c6bb214e1d285640c4 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Mon, 5 Oct 2026 23:37:52 -0700 Subject: [PATCH] [libyuv] Add WASM SIMD128 compare, rotate, scale and row functions d8 geomean speedup vs C compiled with -O2 -msimd128: File AMD Zen 3 ARM Neoverse-V2 row_wasm.cc 5.67x 3.24x scale_wasm.cc 5.47x 3.17x compare_wasm.cc 4.58x 5.77x rotate_wasm.cc 6.83x 6.03x Bug: None Change-Id: Ia06c805dc4029f94842c40e7301b2933beb8c1e5 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8524106 Reviewed-by: richard winterton --- BUILD.gn | 4 + CMakeLists.txt | 3 + include/libyuv/compare_row.h | 14 + include/libyuv/rotate_row.h | 62 ++ include/libyuv/row.h | 421 ++++++++ include/libyuv/scale_row.h | 187 ++++ source/compare.cc | 15 + source/compare_wasm.cc | 137 +++ source/convert.cc | 200 ++++ source/convert_argb.cc | 160 +++ source/convert_from.cc | 8 + source/convert_from_argb.cc | 344 +++++++ source/convert_to_argb.cc | 8 + source/planar_functions.cc | 144 +++ source/rotate.cc | 60 +- source/rotate_any.cc | 18 + source/rotate_argb.cc | 16 + source/rotate_wasm.cc | 238 +++++ source/row_any.cc | 175 ++++ source/row_wasm.cc | 1808 +++++++++++++++++++++++++++++++++- source/scale.cc | 115 +++ source/scale_any.cc | 129 +++ source/scale_argb.cc | 92 ++ source/scale_common.cc | 16 + source/scale_uv.cc | 16 + source/scale_wasm.cc | 645 ++++++++++++ 26 files changed, 5028 insertions(+), 7 deletions(-) create mode 100644 source/compare_wasm.cc create mode 100644 source/rotate_wasm.cc create mode 100644 source/scale_wasm.cc diff --git a/BUILD.gn b/BUILD.gn index 389a08940..059bf2269 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -148,6 +148,7 @@ static_library("libyuv_internal") { "source/compare.cc", "source/compare_common.cc", "source/compare_gcc.cc", + "source/compare_wasm.cc", "source/compare_win.cc", "source/convert.cc", "source/convert_argb.cc", @@ -165,6 +166,7 @@ static_library("libyuv_internal") { "source/rotate_argb.cc", "source/rotate_common.cc", "source/rotate_gcc.cc", + "source/rotate_wasm.cc", "source/rotate_win.cc", "source/row_any.cc", "source/row_common.cc", @@ -178,6 +180,7 @@ static_library("libyuv_internal") { "source/scale_gcc.cc", "source/scale_rgb.cc", "source/scale_uv.cc", + "source/scale_wasm.cc", "source/scale_win.cc", "source/video_common.cc", ] @@ -366,6 +369,7 @@ if (libyuv_include_tests) { ] } } + config("libyuv_unittest_config") { defines = [ "GTEST_RELATIVE_PATH" ] } diff --git a/CMakeLists.txt b/CMakeLists.txt index 29357a89f..1bf9b00d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ set ( ly_common_source_files ${ly_src_dir}/compare.cc ${ly_src_dir}/compare_common.cc ${ly_src_dir}/compare_gcc.cc + ${ly_src_dir}/compare_wasm.cc ${ly_src_dir}/compare_win.cc ${ly_src_dir}/convert.cc ${ly_src_dir}/convert_argb.cc @@ -41,6 +42,7 @@ set ( ly_common_source_files ${ly_src_dir}/rotate_argb.cc ${ly_src_dir}/rotate_common.cc ${ly_src_dir}/rotate_gcc.cc + ${ly_src_dir}/rotate_wasm.cc ${ly_src_dir}/rotate_win.cc ${ly_src_dir}/row_any.cc ${ly_src_dir}/row_common.cc @@ -56,6 +58,7 @@ set ( ly_common_source_files ${ly_src_dir}/scale_rgb.cc ${ly_src_dir}/scale_rvv.cc ${ly_src_dir}/scale_uv.cc + ${ly_src_dir}/scale_wasm.cc ${ly_src_dir}/scale_win.cc ${ly_src_dir}/video_common.cc) diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index fe2abdf45..3c7722baf 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -76,6 +76,13 @@ extern "C" { #define HAS_SUMSQUAREERROR_NEON_DOTPROD #endif +// The following are available for WebAssembly SIMD128: +#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#define HAS_HAMMINGDISTANCE_WASMSIMD +#define HAS_SUMSQUAREERROR_WASMSIMD +#define HAS_HASHDJB2_WASMSIMD +#endif + uint32_t HammingDistance_C(const uint8_t* src_a, const uint8_t* src_b, int count); @@ -97,6 +104,9 @@ uint32_t HammingDistance_NEON(const uint8_t* src_a, uint32_t HammingDistance_NEON_DotProd(const uint8_t* src_a, const uint8_t* src_b, int count); +uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a, + const uint8_t* src_b, + int count); uint32_t SumSquareError_C(const uint8_t* src_a, const uint8_t* src_b, int count); @@ -115,12 +125,16 @@ uint32_t SumSquareError_NEON(const uint8_t* src_a, uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a, const uint8_t* src_b, int count); +uint32_t SumSquareError_WASMSIMD(const uint8_t* src_a, + const uint8_t* src_b, + int count); uint32_t HashDjb2_C(const uint8_t* src, int count, uint32_t seed); uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed); uint32_t HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed); uint32_t HashDjb2_AVX512BW(const uint8_t* src, int count, uint32_t seed); uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed); +uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed); #ifdef __cplusplus } // extern "C" diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index aa376579b..46de57e41 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -82,6 +82,14 @@ extern "C" { #define HAS_TRANSPOSEUVWX16_LSX #endif +#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#define HAS_TRANSPOSEWX8_WASMSIMD +#define HAS_TRANSPOSEUVWX8_WASMSIMD +#define HAS_TRANSPOSEWX16_WASMSIMD +#define HAS_TRANSPOSEUVWX16_WASMSIMD +#define HAS_TRANSPOSE4X4_32_WASMSIMD +#endif + void TransposeWxH_C(const uint8_t* src, int src_stride, uint8_t* dst, @@ -135,6 +143,16 @@ void TransposeWx16_LSX(const uint8_t* src, uint8_t* dst, int dst_stride, int width); +void TransposeWx8_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); +void TransposeWx16_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx8_Any_NEON(const uint8_t* src, int src_stride, @@ -161,6 +179,16 @@ void TransposeWx16_Any_LSX(const uint8_t* src, uint8_t* dst, int dst_stride, int width); +void TransposeWx8_Any_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); +void TransposeWx16_Any_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeUVWxH_C(const uint8_t* src, int src_stride, @@ -228,6 +256,20 @@ void TransposeUVWx16_LSX(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); +void TransposeUVWx8_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); +void TransposeUVWx16_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); void TransposeUVWx8_Any_SSE2(const uint8_t* src, int src_stride, @@ -257,6 +299,20 @@ void TransposeUVWx16_Any_LSX(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); +void TransposeUVWx8_Any_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); +void TransposeUVWx16_Any_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); void TransposeWxH_16_C(const uint16_t* src, int src_stride, uint16_t* dst, @@ -294,6 +350,12 @@ void Transpose4x4_32_AVX2(const uint8_t* src, int dst_stride, int width); +void Transpose4x4_32_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + void Transpose4x4_32_C(const uint8_t* src, int src_stride, uint8_t* dst, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index fd2e80430..b10d18218 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -930,7 +930,52 @@ extern "C" { #endif #if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#define HAS_ARGB1555TOARGBROW_WASMSIMD +#define HAS_ARGB1555TOUVMATRIXROW_WASMSIMD +#define HAS_ARGB1555TOYMATRIXROW_WASMSIMD +#define HAS_ARGB4444TOARGBROW_WASMSIMD +#define HAS_ARGB4444TOUVMATRIXROW_WASMSIMD +#define HAS_ARGB4444TOYMATRIXROW_WASMSIMD +#define HAS_ARGBBLENDROW_WASMSIMD +#define HAS_ARGBMIRRORROW_WASMSIMD +#define HAS_ARGBSHUFFLEROW_WASMSIMD +#define HAS_ARGBTOARGB1555ROW_WASMSIMD +#define HAS_ARGBTOARGB4444ROW_WASMSIMD +#define HAS_ARGBTORGB565DITHERROW_WASMSIMD +#define HAS_ARGBTORGB565ROW_WASMSIMD +#define HAS_ARGBTOUV444MATRIXROW_WASMSIMD +#define HAS_ARGBTOUVMATRIXROW_WASMSIMD +#define HAS_ARGBTOYMATRIXROW_WASMSIMD +#define HAS_BLENDPLANEROW_WASMSIMD +#define HAS_CONVERT8TO16ROW_WASMSIMD +#define HAS_I422TOAR30ROW_WASMSIMD +#define HAS_I422TOARGB1555ROW_WASMSIMD +#define HAS_I422TOARGB4444ROW_WASMSIMD +#define HAS_I422TOARGBROW_WASMSIMD +#define HAS_I422TORGB24ROW_WASMSIMD +#define HAS_I422TORGB565ROW_WASMSIMD +#define HAS_INTERPOLATEROW_16_WASMSIMD +#define HAS_INTERPOLATEROW_WASMSIMD +#define HAS_J400TOARGBROW_WASMSIMD +#define HAS_MERGEUVROW_WASMSIMD +#define HAS_MIRRORROW_WASMSIMD +#define HAS_MIRRORSPLITUVROW_WASMSIMD +#define HAS_MIRRORUVROW_WASMSIMD +#define HAS_MULTIPLYROW_16_WASMSIMD +#define HAS_NV12TOARGBROW_WASMSIMD +#define HAS_NV12TORGB565ROW_WASMSIMD +#define HAS_NV21TOARGBROW_WASMSIMD +#define HAS_RAWTOARGBROW_WASMSIMD #define HAS_RAWTORGB24ROW_WASMSIMD +#define HAS_RGB24MIRRORROW_WASMSIMD +#define HAS_RGB24TOARGBROW_WASMSIMD +#define HAS_RGB565TOARGBROW_WASMSIMD +#define HAS_RGB565TOUVMATRIXROW_WASMSIMD +#define HAS_RGB565TOYMATRIXROW_WASMSIMD +#define HAS_RGBTOUV444MATRIXROW_WASMSIMD +#define HAS_RGBTOUVMATRIXROW_WASMSIMD +#define HAS_RGBTOYMATRIXROW_WASMSIMD +#define HAS_SWAPUVROW_WASMSIMD #endif #if defined(_MSC_VER) && !defined(__CLR_VER) && !defined(__clang__) @@ -7035,6 +7080,382 @@ void GaussCol_NEON(const uint16_t* src0, void ClampFloatToZero_SSE2(const float* src_x, float* dst_y, int width); +void RAWToARGBRow_WASMSIMD(const uint8_t* src_raw, + uint8_t* dst_argb, + int width); +void RGB24ToARGBRow_WASMSIMD(const uint8_t* src_rgb24, + uint8_t* dst_argb, + int width); +void ARGBToYMatrixRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_WASMSIMD(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void RGB565ToYMatrixRow_WASMSIMD(const uint8_t* src_rgb565, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void ARGB1555ToYMatrixRow_WASMSIMD(const uint8_t* src_argb1555, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void ARGB4444ToYMatrixRow_WASMSIMD(const uint8_t* src_argb4444, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void ARGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGBToUVMatrixRow_WASMSIMD(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGB565ToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb565, + int src_stride_rgb565, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGB1555ToUVMatrixRow_WASMSIMD(const uint8_t* src_argb1555, + int src_stride_argb1555, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGB4444ToUVMatrixRow_WASMSIMD(const uint8_t* src_argb4444, + int src_stride_argb4444, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void Convert8To16Row_WASMSIMD(const uint8_t* src_y, + uint16_t* dst_y, + int bits, + int width); +void MultiplyRow_16_WASMSIMD(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); +void MergeUVRow_WASMSIMD(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width); +void MirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width); +void MirrorUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_uv, int width); +void SwapUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_vu, int width); +void MirrorSplitUVRow_WASMSIMD(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void RGB24MirrorRow_WASMSIMD(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width); +void ARGBMirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width); +void InterpolateRow_WASMSIMD(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); +void InterpolateRow_16_WASMSIMD(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); +void J400ToARGBRow_WASMSIMD(const uint8_t* src_y, uint8_t* dst_argb, int width); +void RGB565ToARGBRow_WASMSIMD(const uint8_t* src_rgb565, + uint8_t* dst_argb, + int width); +void ARGB1555ToARGBRow_WASMSIMD(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width); +void ARGB4444ToARGBRow_WASMSIMD(const uint8_t* src_argb4444, + uint8_t* dst_argb, + int width); +void ARGBToRGB565DitherRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width); +void ARGBToRGB565Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width); +void ARGBToARGB1555Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb1555, + int width); +void ARGBToARGB4444Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb4444, + int width); +void ARGBShuffleRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width); +void I422ToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I422ToAR30Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); +void NV12ToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void NV21ToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); +void I422ToRGB24Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); +void I422ToRGB565Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb565, + const struct YuvConstants* yuvconstants, + int width); +void I422ToARGB1555Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb1555, + const struct YuvConstants* yuvconstants, + int width); +void I422ToARGB4444Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb4444, + const struct YuvConstants* yuvconstants, + int width); +void NV12ToRGB565Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb565, + const struct YuvConstants* yuvconstants, + int width); +void ARGBBlendRow_WASMSIMD(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); +void BlendPlaneRow_WASMSIMD(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width); + +void RAWToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void RGB24ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width, + const struct ArgbConstants* c); +void RGB565ToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width, + const struct ArgbConstants* c); +void ARGB1555ToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width, + const struct ArgbConstants* c); +void ARGB4444ToYMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width, + const struct ArgbConstants* c); +void ARGBToUV444MatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGBToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGB565ToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGB1555ToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGB4444ToUVMatrixRow_Any_WASMSIMD(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void Convert8To16Row_Any_WASMSIMD(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int scale, + int width); +void MultiplyRow_16_Any_WASMSIMD(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int scale, + int width); +void MergeUVRow_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); +void MirrorRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void MirrorUVRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void SwapUVRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void RGB24MirrorRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBMirrorRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void InterpolateRow_Any_WASMSIMD(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); +void InterpolateRow_16_Any_WASMSIMD(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); +void J400ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void RGB565ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGB1555ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGB4444ToARGBRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBToRGB565DitherRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const uint32_t dither4, + int width); +void ARGBToRGB565Row_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBToARGB1555Row_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBToARGB4444Row_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGBShuffleRow_Any_WASMSIMD(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const uint8_t* shuffler, + int width); +void I422ToARGBRow_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void I422ToAR30Row_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void NV12ToARGBRow_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void NV21ToARGBRow_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void I422ToRGB24Row_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void I422ToRGB565Row_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void I422ToARGB1555Row_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void I422ToARGB4444Row_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void NV12ToRGB565Row_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); +void BlendPlaneRow_Any_WASMSIMD(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + int width); + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index 5dd4a36c8..5a4681163 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -198,6 +198,20 @@ extern "C" { #define HAS_SCALEARGBFILTERCOLS_RVV #endif +// The following are available for WebAssembly SIMD128: +#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#define HAS_SCALEARGBCOLS_WASMSIMD +#define HAS_SCALEARGBCOLSUP2_WASMSIMD +#define HAS_SCALEARGBFILTERCOLS_WASMSIMD +#define HAS_SCALEARGBROWDOWN2_WASMSIMD +#define HAS_SCALEARGBROWDOWNEVEN_WASMSIMD +#define HAS_SCALEFILTERCOLS_WASMSIMD +#define HAS_SCALEROWDOWN2_WASMSIMD +#define HAS_SCALEROWDOWN34_WASMSIMD +#define HAS_SCALEROWDOWN38_WASMSIMD +#define HAS_SCALEROWDOWN4_WASMSIMD +#endif + // Scale ARGB vertically with bilinear interpolation. void ScalePlaneVertical(int src_height, int dst_width, @@ -1696,6 +1710,179 @@ void ScaleRowUp2_Bilinear_RVV(const uint8_t* src_ptr, uint8_t* dst_ptr, ptrdiff_t dst_stride, int dst_width); + +void ScaleRowDown2_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown2Linear_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown2Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown4_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown4Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_0_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_1_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_2_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown2_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown2Linear_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown2Box_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown2Box_Odd_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown4_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown4Box_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_0_Box_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_1_Box_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_2_Box_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleFilterCols_WASMSIMD(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleFilterCols_Any_WASMSIMD(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleARGBRowDown2_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width); +void ScaleARGBRowDown2Linear_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width); +void ScaleARGBRowDown2Box_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width); +void ScaleARGBRowDown2_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleARGBRowDown2Linear_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleARGBRowDown2Box_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleARGBRowDownEven_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width); +void ScaleARGBRowDownEvenBox_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width); +void ScaleARGBRowDownEven_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_ptr, + int dst_width); +void ScaleARGBRowDownEvenBox_Any_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_ptr, + int dst_width); +void ScaleARGBCols_WASMSIMD(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx); +void ScaleARGBCols_Any_WASMSIMD(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleARGBFilterCols_WASMSIMD(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx); +void ScaleARGBFilterCols_Any_WASMSIMD(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleARGBColsUp2_WASMSIMD(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx); + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/compare.cc b/source/compare.cc index a6729dc19..9c3a5ed16 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -57,6 +57,11 @@ uint32_t HashDjb2(const uint8_t* src, uint64_t count, uint32_t seed) { simd_size = 1; // Any count. } #endif +#if defined(HAS_HASHDJB2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + HashDjb2_SSE = HashDjb2_WASMSIMD; + } +#endif while (count >= (uint64_t)kBlockSize) { seed = HashDjb2_SSE(src, kBlockSize, seed); @@ -174,6 +179,11 @@ uint64_t ComputeHammingDistance(const uint8_t* src_a, HammingDistance = HammingDistance_AVX512BW; simd_size = 1; // Any count. } +#endif +#if defined(HAS_HAMMINGDISTANCE_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + HammingDistance = HammingDistance_WASMSIMD; + } #endif remainder = count & (kBlockSize - 1) & ~(simd_size - 1); @@ -239,6 +249,11 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a, SumSquareError = SumSquareError_AVX512BW; simd_size = 1; // Any count. } +#endif +#if defined(HAS_SUMSQUAREERROR_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SumSquareError = SumSquareError_WASMSIMD; + } #endif remainder = count & (kBlockSize - 1) & ~(simd_size - 1); #ifdef _OPENMP diff --git a/source/compare_wasm.cc b/source/compare_wasm.cc new file mode 100644 index 000000000..6400e1ae9 --- /dev/null +++ b/source/compare_wasm.cc @@ -0,0 +1,137 @@ +/* + * Copyright 2026 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/compare_row.h" +#include "libyuv/row.h" + +#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#include + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Add the 4 32 bit lanes. +static inline uint32_t SumLanes_WASMSIMD(v128_t sum) { + sum = wasm_i32x4_add(sum, wasm_i64x2_shuffle(sum, sum, 1, 0)); + sum = wasm_i32x4_add(sum, wasm_i32x4_shuffle(sum, sum, 1, 0, 3, 2)); + return wasm_u32x4_extract_lane(sum, 0); +} + +#if defined(HAS_HAMMINGDISTANCE_WASMSIMD) +// Bit counts of 64 bytes are added as bytes, then pairs of bytes are added +// to 16 bit sums, which hold up to 64 per 64 bytes, so count can be up to +// 65535. Count is a multiple of 64. +uint32_t HammingDistance_WASMSIMD(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + v128_t sum = wasm_i16x8_splat(0); + while (count > 0) { + v128_t c = wasm_i8x16_splat(0); + int i; + for (i = 0; i < 64; i += 16) { + c = wasm_i8x16_add( + c, wasm_i8x16_popcnt(wasm_v128_xor(wasm_v128_load(src_a + i), + wasm_v128_load(src_b + i)))); + } + sum = wasm_i16x8_add(sum, wasm_u16x8_extadd_pairwise_u8x16(c)); + src_a += 64; + src_b += 64; + count -= 64; + } + return SumLanes_WASMSIMD(wasm_u32x4_extadd_pairwise_u16x8(sum)); +} +#endif // HAS_HAMMINGDISTANCE_WASMSIMD + +#if defined(HAS_SUMSQUAREERROR_WASMSIMD) +// Differences are widened to signed 16 bits and squared with i32x4.dot. +// Count is a multiple of 16. +uint32_t SumSquareError_WASMSIMD(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + v128_t sum = wasm_i32x4_splat(0); + while (count > 0) { + v128_t a = wasm_v128_load(src_a); + v128_t b = wasm_v128_load(src_b); + v128_t lo = wasm_i16x8_sub(wasm_u16x8_extend_low_u8x16(a), + wasm_u16x8_extend_low_u8x16(b)); + v128_t hi = wasm_i16x8_sub(wasm_u16x8_extend_high_u8x16(a), + wasm_u16x8_extend_high_u8x16(b)); + sum = wasm_i32x4_add(sum, wasm_i32x4_add(wasm_i32x4_dot_i16x8(lo, lo), + wasm_i32x4_dot_i16x8(hi, hi))); + src_a += 16; + src_b += 16; + count -= 16; + } + return SumLanes_WASMSIMD(sum); +} +#endif // HAS_SUMSQUAREERROR_WASMSIMD + +#if defined(HAS_HASHDJB2_WASMSIMD) +static const uint32_t kHashMul_WASMSIMD[24] = { + // 33 ^ (30 - 2 * i) for byte pair i of 32 bytes. + 0xccc4cfc1, 0xd61beb81, 0x13791741, 0xac185301, // 33 ^ 30 .. 24 + 0xc8359ec1, 0x510cfa81, 0xb0da6641, 0x92d9e201, // 33 ^ 22 .. 16 + 0xa3476dc1, 0x4f5f0981, 0x855cb541, 0x747c7101, // 33 ^ 14 .. 8 + 0x4cfa3cc1, 0x00121881, 0x00000441, 0x00000001, // 33 ^ 6 .. 0 + 0x1137c401, 0x1137c401, 0x1137c401, 0x1137c401, // 33 ^ 32 + 0x00010021, 0x00010021, 0x00010021, 0x00010021, // 16 bit 33, 1 +}; + +// Sum of 16 bytes times 33 ^ n, as 4 lanes. Byte pairs are combined as +// src[2 * i] * 33 + src[2 * i + 1] with i32x4.dot, then multiplied by +// 33 ^ n for the pair. +static inline v128_t HashMul16_WASMSIMD(const uint8_t* src, + const v128_t* m, + v128_t k33) { + v128_t s = wasm_v128_load(src); + v128_t lo = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_low_u8x16(s), k33); + v128_t hi = wasm_i32x4_dot_i16x8(wasm_u16x8_extend_high_u8x16(s), k33); + return wasm_i32x4_add(wasm_i32x4_mul(lo, m[0]), wasm_i32x4_mul(hi, m[1])); +} + +// HashDjb2 of n bytes is hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)). +// Products are summed per lane, multiplying the sums by 33 ^ 32 for each 32 +// bytes, so one horizontal add is done at the end. +// Count is a multiple of 16. +uint32_t HashDjb2_WASMSIMD(const uint8_t* src, int count, uint32_t seed) { + const v128_t* m = (const v128_t*)kHashMul_WASMSIMD; + v128_t sum = wasm_i32x4_make(0, 0, 0, (int32_t)seed); + // Load constants into Wasm locals outside the loop, instead of v128.const + // in the loop. + asm("" : "+r"(m)); + const v128_t k33_32 = m[4]; + const v128_t k33 = m[5]; + if (count & 16) { + // Lane 3 of m[1] is 33 ^ 16, and lanes 0..2 of sum are 0. + sum = wasm_i32x4_add(wasm_i32x4_mul(sum, m[1]), + HashMul16_WASMSIMD(src, m + 2, k33)); + src += 16; + count -= 16; + } + while (count > 0) { + sum = wasm_i32x4_add( + wasm_i32x4_mul(sum, k33_32), + wasm_i32x4_add(HashMul16_WASMSIMD(src, m, k33), + HashMul16_WASMSIMD(src + 16, m + 2, k33))); + src += 32; + count -= 32; + } + return SumLanes_WASMSIMD(sum); +} +#endif // HAS_HASHDJB2_WASMSIMD + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) diff --git a/source/convert.cc b/source/convert.cc index b55f2f821..39b7374f7 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -1002,6 +1002,14 @@ int I010ToNV12(const uint16_t* src_y, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif // Convert Y plane. if (dst_y) { @@ -1493,6 +1501,14 @@ int I422ToNV21(const uint8_t* src_y, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif #if defined(HAS_INTERPOLATEROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_Any_AVX2; @@ -1537,6 +1553,14 @@ int I422ToNV21(const uint8_t* src_y, InterpolateRow = InterpolateRow_RVV; } #endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } +#endif if (dst_y) { CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, halfwidth, height); @@ -2845,6 +2869,14 @@ int ARGBToI420Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -2904,6 +2936,14 @@ int ARGBToI420Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } #endif if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -3064,6 +3104,14 @@ int ARGBToI420Alpha(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -3124,6 +3172,14 @@ int ARGBToI420Alpha(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBEXTRACTALPHAROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -3388,6 +3444,22 @@ int RGB24ToI420(const uint8_t* src_rgb24, RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif +#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD; + } + } +#endif if (!src_rgb24 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -3520,6 +3592,22 @@ int RGB24ToJ420(const uint8_t* src_rgb24, RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif +#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD; + } + } +#endif if (!src_rgb24 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -3647,6 +3735,22 @@ int RAWToI420(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD; + } + } +#endif if (!src_raw || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -3769,6 +3873,22 @@ int RAWToJ420(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD; + } + } +#endif if (!src_raw || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -3879,6 +3999,14 @@ static int RAWToI444Matrix(const uint8_t* src_raw, RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif +#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_RGBTOUV444MATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -3917,6 +4045,14 @@ static int RAWToI444Matrix(const uint8_t* src_raw, RGBToUV444MatrixRow = RGBToUV444MatrixRow_RVV; } #endif +#if defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_WASMSIMD; + } + } +#endif if (!src_raw || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -4026,6 +4162,22 @@ int RGB565ToI420(const uint8_t* src_rgb565, } } #endif +#if defined(HAS_RGB565TOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGB565ToUVMatrixRow = RGB565ToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGB565ToUVMatrixRow = RGB565ToUVMatrixRow_WASMSIMD; + } + } +#endif +#if defined(HAS_RGB565TOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGB565ToYMatrixRow = RGB565ToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGB565ToYMatrixRow = RGB565ToYMatrixRow_WASMSIMD; + } + } +#endif if (!src_rgb565 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -4109,6 +4261,22 @@ int ARGB1555ToI420(const uint8_t* src_argb1555, } } #endif +#if defined(HAS_ARGB1555TOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGB1555ToUVMatrixRow = ARGB1555ToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGB1555ToUVMatrixRow = ARGB1555ToUVMatrixRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGB1555TOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGB1555ToYMatrixRow = ARGB1555ToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGB1555ToYMatrixRow = ARGB1555ToYMatrixRow_WASMSIMD; + } + } +#endif if (!src_argb1555 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -4192,6 +4360,22 @@ int ARGB4444ToI420(const uint8_t* src_argb4444, } } #endif +#if defined(HAS_ARGB4444TOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGB4444ToUVMatrixRow = ARGB4444ToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGB4444ToUVMatrixRow = ARGB4444ToUVMatrixRow_WASMSIMD; + } + } +#endif +#if defined(HAS_ARGB4444TOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGB4444ToYMatrixRow = ARGB4444ToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGB4444ToYMatrixRow = ARGB4444ToYMatrixRow_WASMSIMD; + } + } +#endif if (!src_argb4444 || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { @@ -4310,6 +4494,14 @@ int RGB24ToJ400(const uint8_t* src_rgb24, RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif +#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { RGBToYMatrixRow(src_rgb24, dst_yj, width, &kArgbJPEGConstants); @@ -4407,6 +4599,14 @@ int RAWToJ400(const uint8_t* src_raw, RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif +#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { RGBToYMatrixRow(src_raw, dst_yj, width, &kAbgrJPEGConstants); diff --git a/source/convert_argb.cc b/source/convert_argb.cc index 172b1f6c2..a47659625 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -144,6 +144,14 @@ int I420ToARGBMatrix(const uint8_t* src_y, I422ToARGBRow = I422ToARGBRow_RVV; } #endif +#if defined(HAS_I422TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToARGBRow = I422ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToARGBRow = I422ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -402,6 +410,14 @@ int I422ToARGBMatrix(const uint8_t* src_y, I422ToARGBRow = I422ToARGBRow_RVV; } #endif +#if defined(HAS_I422TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToARGBRow = I422ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToARGBRow = I422ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -3509,6 +3525,14 @@ int J400ToARGB(const uint8_t* src_y, J400ToARGBRow = J400ToARGBRow_RVV; } #endif +#if defined(HAS_J400TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + J400ToARGBRow = J400ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + J400ToARGBRow = J400ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { J400ToARGBRow(src_y, dst_argb, width); @@ -3889,6 +3913,14 @@ int RGB24ToARGB(const uint8_t* src_rgb24, if (TestCpuFlag(kCpuHasRVV)) { RGB24ToARGBRow = RGB24ToARGBRow_RVV; } +#endif +#if defined(HAS_RGB24TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGB24ToARGBRow = RGB24ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGB24ToARGBRow = RGB24ToARGBRow_WASMSIMD; + } + } #endif for (y = 0; y < height; ++y) { RGB24ToARGBRow(src_rgb24, dst_argb, width); @@ -3983,6 +4015,14 @@ int RAWToARGB(const uint8_t* src_raw, RAWToARGBRow = RAWToARGBRow_RVV; } #endif +#if defined(HAS_RAWTOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RAWToARGBRow = RAWToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RAWToARGBRow = RAWToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { RAWToARGBRow(src_raw, dst_argb, width); @@ -4115,6 +4155,14 @@ int RGB565ToARGB(const uint8_t* src_rgb565, } } #endif +#if defined(HAS_RGB565TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGB565ToARGBRow = RGB565ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + RGB565ToARGBRow = RGB565ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { RGB565ToARGBRow(src_rgb565, dst_argb, width); @@ -4190,6 +4238,14 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555, } } #endif +#if defined(HAS_ARGB1555TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGB1555ToARGBRow(src_argb1555, dst_argb, width); @@ -4260,6 +4316,14 @@ int ARGB4444ToARGB(const uint8_t* src_argb4444, } } #endif +#if defined(HAS_ARGB4444TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGB4444ToARGBRow = ARGB4444ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGB4444ToARGBRow(src_argb4444, dst_argb, width); @@ -4584,6 +4648,14 @@ int NV12ToARGBMatrix(const uint8_t* src_y, NV12ToARGBRow = NV12ToARGBRow_RVV; } #endif +#if defined(HAS_NV12TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + NV12ToARGBRow = NV12ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + NV12ToARGBRow = NV12ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { NV12ToARGBRow(src_y, src_uv, dst_argb, yuvconstants, width); @@ -4677,6 +4749,14 @@ int NV21ToARGBMatrix(const uint8_t* src_y, NV21ToARGBRow = NV21ToARGBRow_RVV; } #endif +#if defined(HAS_NV21TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + NV21ToARGBRow = NV21ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + NV21ToARGBRow = NV21ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { NV21ToARGBRow(src_y, src_vu, dst_argb, yuvconstants, width); @@ -5529,6 +5609,14 @@ int NV12ToRGB565Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_NV12TORGB565ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + NV12ToRGB565Row = NV12ToRGB565Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + NV12ToRGB565Row = NV12ToRGB565Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { NV12ToRGB565Row(src_y, src_uv, dst_rgb565, yuvconstants, width); @@ -5783,6 +5871,14 @@ int I420ToRGB24Matrix(const uint8_t* src_y, I422ToRGB24Row = I422ToRGB24Row_RVV; } #endif +#if defined(HAS_I422TORGB24ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToRGB24Row = I422ToRGB24Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToRGB24Row = I422ToRGB24Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToRGB24Row(src_y, src_u, src_v, dst_rgb24, yuvconstants, width); @@ -6006,6 +6102,14 @@ int I422ToRGB24Matrix(const uint8_t* src_y, I422ToRGB24Row = I422ToRGB24Row_RVV; } #endif +#if defined(HAS_I422TORGB24ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToRGB24Row = I422ToRGB24Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToRGB24Row = I422ToRGB24Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToRGB24Row(src_y, src_u, src_v, dst_rgb24, yuvconstants, width); @@ -6130,6 +6234,14 @@ int I420ToARGB1555(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOARGB1555ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToARGB1555Row = I422ToARGB1555Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToARGB1555Row = I422ToARGB1555Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToARGB1555Row(src_y, src_u, src_v, dst_argb1555, &kYuvI601Constants, @@ -6221,6 +6333,14 @@ int I420ToARGB4444(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOARGB4444ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToARGB4444Row = I422ToARGB4444Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToARGB4444Row = I422ToARGB4444Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToARGB4444Row(src_y, src_u, src_v, dst_argb4444, &kYuvI601Constants, @@ -6314,6 +6434,14 @@ int I420ToRGB565Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB565ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToRGB565Row = I422ToRGB565Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToRGB565Row = I422ToRGB565Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToRGB565Row(src_y, src_u, src_v, dst_rgb565, yuvconstants, width); @@ -6465,6 +6593,14 @@ int I422ToRGB565Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB565ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToRGB565Row = I422ToRGB565Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToRGB565Row = I422ToRGB565Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToRGB565Row(src_y, src_u, src_v, dst_rgb565, yuvconstants, width); @@ -6598,6 +6734,14 @@ int I420ToRGB565Dither(const uint8_t* src_y, I422ToARGBRow = I422ToARGBRow_RVV; } #endif +#if defined(HAS_I422TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToARGBRow = I422ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToARGBRow = I422ToARGBRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTORGB565DITHERROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_SSE2; @@ -6642,6 +6786,14 @@ int I420ToRGB565Dither(const uint8_t* src_y, ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_LASX; } } +#endif +#if defined(HAS_ARGBTORGB565DITHERROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_WASMSIMD; + } + } #endif { // Allocate a row of argb. @@ -6742,6 +6894,14 @@ int I420ToAR30Matrix(const uint8_t* src_y, I422ToAR30Row = I422ToAR30Row_RVV; } #endif +#if defined(HAS_I422TOAR30ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + I422ToAR30Row = I422ToAR30Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + I422ToAR30Row = I422ToAR30Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { I422ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); diff --git a/source/convert_from.cc b/source/convert_from.cc index 6c0132b9f..f454c1704 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -799,6 +799,14 @@ static int I4xxToPxxx(const uint8_t* src_y, Convert8To16Row = Convert8To16Row_RVV; } #endif +#if defined(HAS_CONVERT8TO16ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert8To16Row = Convert8To16Row_Any_WASMSIMD; + if (IS_ALIGNED(uv_width, 16)) { + Convert8To16Row = Convert8To16Row_WASMSIMD; + } + } +#endif #if defined(HAS_MERGEUVROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MergeUVRow_16 = MergeUVRow_16_Any_AVX2; diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index e9912ddb5..ae25009ae 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -131,6 +131,14 @@ int ARGBToI444Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -176,6 +184,14 @@ int ARGBToI444Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_WASMSIMD; + } + } #endif if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -307,6 +323,14 @@ int ARGBToI422Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -366,6 +390,14 @@ int ARGBToI422Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } #endif if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -493,6 +525,14 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -544,6 +584,14 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) = MergeUVRow_C; @@ -612,6 +660,14 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif // Allocate a rows of uv. align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2); @@ -743,6 +799,14 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -802,6 +866,14 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) = MergeUVRow_C; @@ -870,6 +942,14 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif size_t row_size = (static_cast(halfwidth) + 31) & ~static_cast(31); @@ -995,6 +1075,14 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -1040,6 +1128,14 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_WASMSIMD; + } + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) = MergeUVRow_C; @@ -1108,6 +1204,14 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif size_t row_size = (static_cast(width) + 31) & ~static_cast(31); @@ -1221,6 +1325,14 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -1272,6 +1384,14 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_vu, int width) = MergeUVRow_C; @@ -1340,6 +1460,14 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif // Allocate a rows of uv. align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2); @@ -1428,6 +1556,14 @@ int ARGBToI400Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_SME; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToYMatrixRow(src_argb, dst_y, width, constants); @@ -1506,6 +1642,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_SME; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; @@ -1552,6 +1696,14 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_I422TOYUY2ROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { I422ToYUY2Row = I422ToYUY2Row_Any_AVX2; @@ -1659,6 +1811,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_SME; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; @@ -1705,6 +1865,14 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_I422TOUYVYROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { I422ToUYVYRow = I422ToUYVYRow_Any_AVX2; @@ -1870,6 +2038,14 @@ int ARGBToP010Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -1929,6 +2105,14 @@ int ARGBToP010Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } #endif // Negative height means invert the image. if (height < 0) { @@ -1978,6 +2162,14 @@ int ARGBToP010Matrix(const uint8_t* src_argb, Convert8To16Row = Convert8To16Row_RVV; } #endif +#if defined(HAS_CONVERT8TO16ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert8To16Row = Convert8To16Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + Convert8To16Row = Convert8To16Row_WASMSIMD; + } + } +#endif #if defined(HAS_MULTIPLYROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MultiplyRow_16 = MultiplyRow_16_Any_AVX2; @@ -2012,6 +2204,14 @@ int ARGBToP010Matrix(const uint8_t* src_argb, MultiplyRow_16 = MultiplyRow_16_RVV; } #endif +#if defined(HAS_MULTIPLYROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MultiplyRow_16 = MultiplyRow_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MultiplyRow_16 = MultiplyRow_16_WASMSIMD; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { MergeUVRow = MergeUVRow_Any_SSE2; @@ -2067,6 +2267,14 @@ int ARGBToP010Matrix(const uint8_t* src_argb, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif size_t row_y_size = (static_cast(width) + 31) & ~static_cast(31); @@ -2231,6 +2439,14 @@ int ARGBToP210Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { @@ -2290,6 +2506,14 @@ int ARGBToP210Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_WASMSIMD; + } + } #endif // Negative height means invert the image. if (height < 0) { @@ -2339,6 +2563,14 @@ int ARGBToP210Matrix(const uint8_t* src_argb, Convert8To16Row = Convert8To16Row_RVV; } #endif +#if defined(HAS_CONVERT8TO16ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert8To16Row = Convert8To16Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + Convert8To16Row = Convert8To16Row_WASMSIMD; + } + } +#endif #if defined(HAS_MULTIPLYROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MultiplyRow_16 = MultiplyRow_16_Any_AVX2; @@ -2373,6 +2605,14 @@ int ARGBToP210Matrix(const uint8_t* src_argb, MultiplyRow_16 = MultiplyRow_16_RVV; } #endif +#if defined(HAS_MULTIPLYROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MultiplyRow_16 = MultiplyRow_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MultiplyRow_16 = MultiplyRow_16_WASMSIMD; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { MergeUVRow = MergeUVRow_Any_SSE2; @@ -2428,6 +2668,14 @@ int ARGBToP210Matrix(const uint8_t* src_argb, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif size_t row_y_size = (static_cast(width) + 31) & ~static_cast(31); @@ -2577,6 +2825,14 @@ int ARGBToP410Matrix(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -2622,6 +2878,14 @@ int ARGBToP410Matrix(const uint8_t* src_argb, if (TestCpuFlag(kCpuHasRVV)) { ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RVV; } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_WASMSIMD; + } + } #endif // Negative height means invert the image. if (height < 0) { @@ -2671,6 +2935,14 @@ int ARGBToP410Matrix(const uint8_t* src_argb, Convert8To16Row = Convert8To16Row_RVV; } #endif +#if defined(HAS_CONVERT8TO16ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert8To16Row = Convert8To16Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + Convert8To16Row = Convert8To16Row_WASMSIMD; + } + } +#endif #if defined(HAS_MULTIPLYROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { MultiplyRow_16 = MultiplyRow_16_Any_AVX2; @@ -2705,6 +2977,14 @@ int ARGBToP410Matrix(const uint8_t* src_argb, MultiplyRow_16 = MultiplyRow_16_RVV; } #endif +#if defined(HAS_MULTIPLYROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MultiplyRow_16 = MultiplyRow_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MultiplyRow_16 = MultiplyRow_16_WASMSIMD; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { MergeUVRow = MergeUVRow_Any_SSE2; @@ -2760,6 +3040,14 @@ int ARGBToP410Matrix(const uint8_t* src_argb, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif size_t row_size = (static_cast(width) + 31) & ~static_cast(31); @@ -3172,6 +3460,14 @@ int ARGBToRGB565Dither(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTORGB565DITHERROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToRGB565DitherRow(src_argb, dst_rgb565, dither4[y & 3], width); @@ -3248,6 +3544,14 @@ int ARGBToRGB565(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTORGB565ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToRGB565Row = ARGBToRGB565Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGBToRGB565Row = ARGBToRGB565Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToRGB565Row(src_argb, dst_rgb565, width); @@ -3317,6 +3621,14 @@ int ARGBToARGB1555(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOARGB1555ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToARGB1555Row = ARGBToARGB1555Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGBToARGB1555Row = ARGBToARGB1555Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToARGB1555Row(src_argb, dst_argb1555, width); @@ -3386,6 +3698,14 @@ int ARGBToARGB4444(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOARGB4444ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToARGB4444Row = ARGBToARGB4444Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + ARGBToARGB4444Row = ARGBToARGB4444Row_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBToARGB4444Row(src_argb, dst_argb4444, width); @@ -3859,6 +4179,14 @@ static int RAWToNVMatrix(const uint8_t* src_raw, RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif +#if defined(HAS_RGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_RGBTOUVMATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -3892,6 +4220,14 @@ static int RAWToNVMatrix(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_MERGEUVROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -3948,6 +4284,14 @@ static int RAWToNVMatrix(const uint8_t* src_raw, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif { // Allocate 1 row of U and 1 row of V. diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index 669154e79..cea1e4499 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -104,6 +104,14 @@ static int NV16ToARGB(const uint8_t* src_y, NV12ToARGBRow = NV12ToARGBRow_RVV; } #endif +#if defined(HAS_NV12TOARGBROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + NV12ToARGBRow = NV12ToARGBRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + NV12ToARGBRow = NV12ToARGBRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { NV12ToARGBRow(src_y, src_uv, dst_argb, &kYuvI601Constants, width); diff --git a/source/planar_functions.cc b/source/planar_functions.cc index cfc65ecfe..d0c09de37 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -271,6 +271,14 @@ void Convert8To16Plane( Convert8To16Row = Convert8To16Row_RVV; } #endif +#if defined(HAS_CONVERT8TO16ROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Convert8To16Row = Convert8To16Row_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + Convert8To16Row = Convert8To16Row_WASMSIMD; + } + } +#endif // Convert plane for (y = 0; y < height; ++y) { @@ -785,6 +793,14 @@ void MergeUVPlane(const uint8_t* src_u, MergeUVRow = MergeUVRow_RVV; } #endif +#if defined(HAS_MERGEUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MergeUVRow = MergeUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MergeUVRow = MergeUVRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { // Merge a row of U and V into a row of UV. @@ -982,6 +998,14 @@ void ConvertToMSBPlane_16(const uint16_t* src_y, MultiplyRow_16 = MultiplyRow_16_RVV; } #endif +#if defined(HAS_MULTIPLYROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MultiplyRow_16 = MultiplyRow_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MultiplyRow_16 = MultiplyRow_16_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { MultiplyRow_16(src_y, dst_y, scale, width); @@ -1106,6 +1130,14 @@ void SwapUVPlane(const uint8_t* src_uv, SwapUVRow = SwapUVRow_RVV; } #endif +#if defined(HAS_SWAPUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + SwapUVRow = SwapUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + SwapUVRow = SwapUVRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { SwapUVRow(src_uv, dst_vu, width); @@ -2696,6 +2728,14 @@ void MirrorPlane(const uint8_t* src_y, } } #endif +#if defined(HAS_MIRRORROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MirrorRow = MirrorRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_WASMSIMD; + } + } +#endif // Mirror plane for (y = 0; y < height; ++y) { @@ -2764,6 +2804,14 @@ void MirrorUVPlane(const uint8_t* src_uv, } } #endif +#if defined(HAS_MIRRORUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MirrorUVRow = MirrorUVRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + MirrorUVRow = MirrorUVRow_WASMSIMD; + } + } +#endif // MirrorUV plane for (y = 0; y < height; ++y) { @@ -2938,6 +2986,14 @@ int ARGBMirror(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBMIRRORROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBMirrorRow = ARGBMirrorRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBMirrorRow = ARGBMirrorRow_WASMSIMD; + } + } +#endif // Mirror plane for (y = 0; y < height; ++y) { @@ -2985,6 +3041,14 @@ int RGB24Mirror(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGB24MIRRORROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + RGB24MirrorRow = RGB24MirrorRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + RGB24MirrorRow = RGB24MirrorRow_WASMSIMD; + } + } +#endif // Mirror plane for (y = 0; y < height; ++y) { @@ -3047,6 +3111,11 @@ int ARGBBlend(const uint8_t* src_argb0, if (TestCpuFlag(kCpuHasRVV)) { ARGBBlendRow = ARGBBlendRow_RVV; } +#endif +#if defined(HAS_ARGBBLENDROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBBlendRow = ARGBBlendRow_WASMSIMD; + } #endif for (y = 0; y < height; ++y) { ARGBBlendRow(src_argb0, src_argb1, dst_argb, width); @@ -3130,6 +3199,14 @@ int BlendPlane(const uint8_t* src_y0, BlendPlaneRow = BlendPlaneRow_RVV; } #endif +#if defined(HAS_BLENDPLANEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + BlendPlaneRow = BlendPlaneRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + BlendPlaneRow = BlendPlaneRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { BlendPlaneRow(src_y0, src_y1, alpha, dst_y, width); @@ -3228,6 +3305,14 @@ int I420Blend(const uint8_t* src_y0, if (TestCpuFlag(kCpuHasRVV)) { BlendPlaneRow = BlendPlaneRow_RVV; } +#endif +#if defined(HAS_BLENDPLANEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + BlendPlaneRow = BlendPlaneRow_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 8)) { + BlendPlaneRow = BlendPlaneRow_WASMSIMD; + } + } #endif if (!IS_ALIGNED(width, 2)) { ScaleRowDown2 = ScaleRowDown2Box_Odd_C; @@ -3278,6 +3363,17 @@ int I420Blend(const uint8_t* src_y0, ScaleRowDown2 = ScaleRowDown2Box_RVV; } #endif +#if defined(HAS_SCALEROWDOWN2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowDown2 = ScaleRowDown2Box_Odd_WASMSIMD; + if (IS_ALIGNED(width, 2)) { + ScaleRowDown2 = ScaleRowDown2Box_Any_WASMSIMD; + if (IS_ALIGNED(halfwidth, 16)) { + ScaleRowDown2 = ScaleRowDown2Box_WASMSIMD; + } + } + } +#endif // Row buffer for intermediate alpha pixels. align_buffer_64(halfalpha, halfwidth); @@ -4664,6 +4760,14 @@ int InterpolatePlane(const uint8_t* src0, InterpolateRow = InterpolateRow_RVV; } #endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { InterpolateRow(dst, src0, src1 - src0, width, interpolation); @@ -4743,6 +4847,14 @@ int InterpolatePlane_16(const uint16_t* src0, } } #endif +#if defined(HAS_INTERPOLATEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow_16 = InterpolateRow_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + InterpolateRow_16 = InterpolateRow_16_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { InterpolateRow_16(dst, src0, src1 - src0, width, interpolation); @@ -4887,6 +4999,14 @@ int ARGBShuffle(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBSHUFFLEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBShuffleRow = ARGBShuffleRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBShuffleRow = ARGBShuffleRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { ARGBShuffleRow(src_argb, dst_argb, shuffler, width); @@ -4958,6 +5078,14 @@ int AR64Shuffle(const uint16_t* src_ar64, } } #endif +#if defined(HAS_ARGBSHUFFLEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + AR64ShuffleRow = ARGBShuffleRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + AR64ShuffleRow = ARGBShuffleRow_WASMSIMD; + } + } +#endif for (y = 0; y < height; ++y) { AR64ShuffleRow((uint8_t*)(src_ar64), (uint8_t*)(dst_ar64), shuffler, @@ -5141,6 +5269,14 @@ static int ARGBSobelize(const uint8_t* src_argb, ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; } #endif +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_WASMSIMD; + } + } +#endif #if defined(HAS_SOBELYROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -5927,6 +6063,14 @@ int UYVYToNV12(const uint8_t* src_uyvy, InterpolateRow = InterpolateRow_RVV; } #endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } +#endif { int awidth = halfwidth * 2; diff --git a/source/rotate.cc b/source/rotate.cc index 95767b4ce..c44ca1917 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -36,7 +36,8 @@ void TransposePlane(const uint8_t* src, void (*TransposeWxH)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width, int height) = NULL; #endif -#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) +#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \ + defined(HAS_TRANSPOSEWX16_WASMSIMD) void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) = NULL; #elif defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) @@ -98,6 +99,22 @@ void TransposePlane(const uint8_t* src, } } #endif +#if defined(HAS_TRANSPOSEWX8_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + TransposeWx8 = TransposeWx8_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + TransposeWx8 = TransposeWx8_WASMSIMD; + } + } +#endif +#if defined(HAS_TRANSPOSEWX16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + TransposeWx16 = TransposeWx16_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + TransposeWx16 = TransposeWx16_WASMSIMD; + } + } +#endif #if defined(HAS_TRANSPOSEWXH_SME) if (TransposeWxH) { @@ -106,7 +123,8 @@ void TransposePlane(const uint8_t* src, } #endif #if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \ - defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) + defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) || \ + defined(HAS_TRANSPOSEWX16_WASMSIMD) // Work across the source in 16x16 tiles if (TransposeWx16) { while (i >= 16) { @@ -220,6 +238,14 @@ void RotatePlane180(const uint8_t* src, } } #endif +#if defined(HAS_MIRRORROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + MirrorRow = MirrorRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 16)) { + MirrorRow = MirrorRow_WASMSIMD; + } + } +#endif #if defined(HAS_COPYROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { CopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2; @@ -289,7 +315,9 @@ void SplitTransposeUV(const uint8_t* src, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width, int height) = TransposeUVWxH_C; #endif -#if defined(HAS_TRANSPOSEUVWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX512BW) +#if defined(HAS_TRANSPOSEUVWX16_AVX2) || \ + defined(HAS_TRANSPOSEUVWX16_AVX512BW) || \ + defined(HAS_TRANSPOSEUVWX16_WASMSIMD) void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width) = NULL; @@ -346,6 +374,22 @@ void SplitTransposeUV(const uint8_t* src, TransposeUVWx16 = TransposeUVWx16_AVX512BW; } #endif +#if defined(HAS_TRANSPOSEUVWX8_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + TransposeUVWx8 = TransposeUVWx8_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + TransposeUVWx8 = TransposeUVWx8_WASMSIMD; + } + } +#endif +#if defined(HAS_TRANSPOSEUVWX16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + TransposeUVWx16 = TransposeUVWx16_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + TransposeUVWx16 = TransposeUVWx16_WASMSIMD; + } + } +#endif #if defined(HAS_TRANSPOSEUVWXH_SME) if (TestCpuFlag(kCpuHasSME)) { @@ -354,8 +398,9 @@ void SplitTransposeUV(const uint8_t* src, return; } #endif -#if defined(HAS_TRANSPOSEUVWX16_AVX2) || \ - defined(HAS_TRANSPOSEUVWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_LSX) +#if defined(HAS_TRANSPOSEUVWX16_AVX2) || \ + defined(HAS_TRANSPOSEUVWX16_AVX512BW) || \ + defined(HAS_TRANSPOSEUVWX16_LSX) || defined(HAS_TRANSPOSEUVWX16_WASMSIMD) // Work through the source in 16x16 tiles. if (TransposeUVWx16) { while (i >= 16) { @@ -453,6 +498,11 @@ void SplitRotateUV180(const uint8_t* src, MirrorSplitUVRow = MirrorSplitUVRow_LSX; } #endif +#if defined(HAS_MIRRORSPLITUVROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD) && IS_ALIGNED(width, 16)) { + MirrorSplitUVRow = MirrorSplitUVRow_WASMSIMD; + } +#endif dst_a += dst_stride_a * (height - 1); dst_b += dst_stride_b * (height - 1); diff --git a/source/rotate_any.cc b/source/rotate_any.cc index 52dd8ac2b..986b8c9ed 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -43,6 +43,12 @@ TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15) #ifdef HAS_TRANSPOSEWX16_LSX TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, TransposeWx16_C, 15) #endif +#ifdef HAS_TRANSPOSEWX8_WASMSIMD +TANY(TransposeWx8_Any_WASMSIMD, TransposeWx8_WASMSIMD, TransposeWx8_C, 15) +#endif +#ifdef HAS_TRANSPOSEWX16_WASMSIMD +TANY(TransposeWx16_Any_WASMSIMD, TransposeWx16_WASMSIMD, TransposeWx16_C, 15) +#endif #undef TANY #define TUVANY(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \ @@ -71,6 +77,18 @@ TUVANY(TransposeUVWx16_Any_AVX2, TransposeUVWx16_AVX2, TransposeUVWx16_C, 7) #ifdef HAS_TRANSPOSEUVWX16_LSX TUVANY(TransposeUVWx16_Any_LSX, TransposeUVWx16_LSX, TransposeUVWx16_C, 7) #endif +#ifdef HAS_TRANSPOSEUVWX8_WASMSIMD +TUVANY(TransposeUVWx8_Any_WASMSIMD, + TransposeUVWx8_WASMSIMD, + TransposeUVWx8_C, + 7) +#endif +#ifdef HAS_TRANSPOSEUVWX16_WASMSIMD +TUVANY(TransposeUVWx16_Any_WASMSIMD, + TransposeUVWx16_WASMSIMD, + TransposeUVWx16_C, + 7) +#endif #undef TUVANY #ifdef __cplusplus diff --git a/source/rotate_argb.cc b/source/rotate_argb.cc index f6603cfd0..211119ae3 100644 --- a/source/rotate_argb.cc +++ b/source/rotate_argb.cc @@ -68,6 +68,14 @@ static int ARGBTranspose(const uint8_t* src_argb, ScaleARGBRowDownEven = ScaleARGBRowDownEven_RVV; } #endif +#if defined(HAS_SCALEARGBROWDOWNEVEN_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_WASMSIMD; + if (IS_ALIGNED(height, 4)) { // Width of dest. + ScaleARGBRowDownEven = ScaleARGBRowDownEven_WASMSIMD; + } + } +#endif for (i = 0; i < width; ++i) { // column of source to row of dest. ScaleARGBRowDownEven(src_argb, 0, src_pixel_step, dst_argb, height); @@ -168,6 +176,14 @@ static int ARGBRotate180(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBMIRRORROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ARGBMirrorRow = ARGBMirrorRow_Any_WASMSIMD; + if (IS_ALIGNED(width, 4)) { + ARGBMirrorRow = ARGBMirrorRow_WASMSIMD; + } + } +#endif #if defined(HAS_COPYROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2; diff --git a/source/rotate_wasm.cc b/source/rotate_wasm.cc new file mode 100644 index 000000000..f478a8bff --- /dev/null +++ b/source/rotate_wasm.cc @@ -0,0 +1,238 @@ +/* + * Copyright 2026 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/rotate_row.h" +#include "libyuv/row.h" + +#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#include + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Interleave the low or high halves of 2 vectors as 8, 16, 32 or 64 bit +// elements. These map to punpck on x86 and zip on Arm. +#define ZIPLO8(a, b) \ + wasm_i8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, \ + 23) +#define ZIPHI8(a, b) \ + wasm_i8x16_shuffle(a, b, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, \ + 30, 15, 31) +#define ZIPLO16(a, b) wasm_i16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11) +#define ZIPHI16(a, b) wasm_i16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15) +#define ZIPLO32(a, b) wasm_i32x4_shuffle(a, b, 0, 4, 1, 5) +#define ZIPHI32(a, b) wasm_i32x4_shuffle(a, b, 2, 6, 3, 7) +#define ZIPLO64(a, b) wasm_i64x2_shuffle(a, b, 0, 2) +#define ZIPHI64(a, b) wasm_i64x2_shuffle(a, b, 1, 3) + +// Transpose 8 rows x 16 bytes. c[m] holds columns 2m and 2m + 1 of the 8 rows +// in its low and high 8 bytes. +static inline void Transpose8x16_WASMSIMD(const uint8_t* src, + ptrdiff_t s, + v128_t* c) { + v128_t r0 = wasm_v128_load(src); + v128_t r1 = wasm_v128_load(src + s); + v128_t r2 = wasm_v128_load(src + s * 2); + v128_t r3 = wasm_v128_load(src + s * 3); + v128_t r4 = wasm_v128_load(src + s * 4); + v128_t r5 = wasm_v128_load(src + s * 5); + v128_t r6 = wasm_v128_load(src + s * 6); + v128_t r7 = wasm_v128_load(src + s * 7); + + // Bytes of row pairs. a0 is columns 0..7 of rows 0, 1. + v128_t a0 = ZIPLO8(r0, r1); + v128_t a1 = ZIPHI8(r0, r1); + v128_t a2 = ZIPLO8(r2, r3); + v128_t a3 = ZIPHI8(r2, r3); + v128_t a4 = ZIPLO8(r4, r5); + v128_t a5 = ZIPHI8(r4, r5); + v128_t a6 = ZIPLO8(r6, r7); + v128_t a7 = ZIPHI8(r6, r7); + + // Words. b0 is columns 0..3 of rows 0..3. + v128_t b0 = ZIPLO16(a0, a2); + v128_t b1 = ZIPHI16(a0, a2); + v128_t b2 = ZIPLO16(a1, a3); + v128_t b3 = ZIPHI16(a1, a3); + v128_t b4 = ZIPLO16(a4, a6); + v128_t b5 = ZIPHI16(a4, a6); + v128_t b6 = ZIPLO16(a5, a7); + v128_t b7 = ZIPHI16(a5, a7); + + // Dwords. c0 is columns 0, 1 of rows 0..7. + c[0] = ZIPLO32(b0, b4); + c[1] = ZIPHI32(b0, b4); + c[2] = ZIPLO32(b1, b5); + c[3] = ZIPHI32(b1, b5); + c[4] = ZIPLO32(b2, b6); + c[5] = ZIPHI32(b2, b6); + c[6] = ZIPLO32(b3, b7); + c[7] = ZIPHI32(b3, b7); +} + +#if defined(HAS_TRANSPOSEWX8_WASMSIMD) || defined(HAS_TRANSPOSEUVWX8_WASMSIMD) +// Transpose 8 rows x 16 bytes. Even columns go to dst_a and odd columns go to +// dst_b. Width is a multiple of 16 bytes. +static void TransposeWx8_Byte_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int byte_width) { + const ptrdiff_t sa = dst_stride_a; + const ptrdiff_t sb = dst_stride_b; + while (byte_width > 0) { + v128_t c[8]; + int m; + Transpose8x16_WASMSIMD(src, src_stride, c); + for (m = 0; m < 8; ++m) { + wasm_v128_store64_lane(dst_a + sa * m, c[m], 0); + wasm_v128_store64_lane(dst_b + sb * m, c[m], 1); + } + src += 16; + dst_a += sa * 8; + dst_b += sb * 8; + byte_width -= 16; + } +} +#endif // HAS_TRANSPOSEWX8_WASMSIMD || HAS_TRANSPOSEUVWX8_WASMSIMD + +#if defined(HAS_TRANSPOSEWX8_WASMSIMD) +// Transpose 16x8. Width is a multiple of 16. +void TransposeWx8_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx8_Byte_WASMSIMD(src, src_stride, dst, dst_stride * 2, + dst + dst_stride, dst_stride * 2, width); +} +#endif // HAS_TRANSPOSEWX8_WASMSIMD + +#if defined(HAS_TRANSPOSEUVWX8_WASMSIMD) +// Transpose UV 8x8. Width is a multiple of 8. +void TransposeUVWx8_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeWx8_Byte_WASMSIMD(src, src_stride, dst_a, dst_stride_a, dst_b, + dst_stride_b, width * 2); +} +#endif // HAS_TRANSPOSEUVWX8_WASMSIMD + +#if defined(HAS_TRANSPOSEWX16_WASMSIMD) || defined(HAS_TRANSPOSEUVWX16_WASMSIMD) +// Transpose 16 rows x 16 bytes as 2 halves of 8 rows, which are joined with +// 64 bit interleaves to store full 16 byte rows. Even columns go to dst_a and +// odd columns go to dst_b. Width is a multiple of 16 bytes. +static void TransposeWx16_Byte_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int byte_width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t sa = dst_stride_a; + const ptrdiff_t sb = dst_stride_b; + while (byte_width > 0) { + v128_t lo[8]; + v128_t hi[8]; + int m; + Transpose8x16_WASMSIMD(src, s, lo); + Transpose8x16_WASMSIMD(src + s * 8, s, hi); + for (m = 0; m < 8; ++m) { + wasm_v128_store(dst_a + sa * m, ZIPLO64(lo[m], hi[m])); + wasm_v128_store(dst_b + sb * m, ZIPHI64(lo[m], hi[m])); + } + src += 16; + dst_a += sa * 8; + dst_b += sb * 8; + byte_width -= 16; + } +} +#endif // HAS_TRANSPOSEWX16_WASMSIMD || HAS_TRANSPOSEUVWX16_WASMSIMD + +#if defined(HAS_TRANSPOSEWX16_WASMSIMD) +// Transpose 16x16. Width is a multiple of 16. +void TransposeWx16_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx16_Byte_WASMSIMD(src, src_stride, dst, dst_stride * 2, + dst + dst_stride, dst_stride * 2, width); +} +#endif // HAS_TRANSPOSEWX16_WASMSIMD + +#if defined(HAS_TRANSPOSEUVWX16_WASMSIMD) +// Transpose UV 8x16. Width is a multiple of 8. +void TransposeUVWx16_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeWx16_Byte_WASMSIMD(src, src_stride, dst_a, dst_stride_a, dst_b, + dst_stride_b, width * 2); +} +#endif // HAS_TRANSPOSEUVWX16_WASMSIMD + +#if defined(HAS_TRANSPOSE4X4_32_WASMSIMD) +// Transpose 32 bit values (ARGB) in 4x4 tiles, moving down 4 rows of source +// and across 4 columns of destination. Width is a multiple of 4. +void Transpose4x4_32_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + v128_t r0 = wasm_v128_load(src); + v128_t r1 = wasm_v128_load(src + s); + v128_t r2 = wasm_v128_load(src + s * 2); + v128_t r3 = wasm_v128_load(src + s * 3); + v128_t a0 = ZIPLO32(r0, r1); + v128_t a1 = ZIPHI32(r0, r1); + v128_t a2 = ZIPLO32(r2, r3); + v128_t a3 = ZIPHI32(r2, r3); + wasm_v128_store(dst, ZIPLO64(a0, a2)); + wasm_v128_store(dst + d, ZIPHI64(a0, a2)); + wasm_v128_store(dst + d * 2, ZIPLO64(a1, a3)); + wasm_v128_store(dst + d * 3, ZIPHI64(a1, a3)); + src += s * 4; + dst += 16; + width -= 4; + } +} +#endif // HAS_TRANSPOSE4X4_32_WASMSIMD + +#undef ZIPLO8 +#undef ZIPHI8 +#undef ZIPLO16 +#undef ZIPHI16 +#undef ZIPLO32 +#undef ZIPHI32 +#undef ZIPLO64 +#undef ZIPHI64 + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) diff --git a/source/row_any.cc b/source/row_any.cc index 74c9d7134..ad16d9670 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -349,6 +349,9 @@ ANY31(BlendPlaneRow_Any_SSSE3, BlendPlaneRow_SSSE3, 0, 0, 1, 7) #ifdef HAS_BLENDPLANEROW_NEON ANY31(BlendPlaneRow_Any_NEON, BlendPlaneRow_NEON, 0, 0, 1, 15) #endif +#ifdef HAS_BLENDPLANEROW_WASMSIMD +ANY31(BlendPlaneRow_Any_WASMSIMD, BlendPlaneRow_WASMSIMD, 0, 0, 1, 15) +#endif #undef ANY31 // Note that odd width replication includes 444 due to implementation @@ -476,6 +479,24 @@ ANY31C(I422ToARGB1555Row_Any_LASX, I422ToARGB1555Row_LASX, 1, 0, 2, 31) #ifdef HAS_I444TOARGBROW_LSX ANY31C(I444ToARGBRow_Any_LSX, I444ToARGBRow_LSX, 0, 0, 4, 15) #endif +#ifdef HAS_I422TOARGBROW_WASMSIMD +ANY31C(I422ToARGBRow_Any_WASMSIMD, I422ToARGBRow_WASMSIMD, 1, 0, 4, 7) +#endif +#ifdef HAS_I422TOAR30ROW_WASMSIMD +ANY31C(I422ToAR30Row_Any_WASMSIMD, I422ToAR30Row_WASMSIMD, 1, 0, 4, 7) +#endif +#ifdef HAS_I422TORGB24ROW_WASMSIMD +ANY31C(I422ToRGB24Row_Any_WASMSIMD, I422ToRGB24Row_WASMSIMD, 1, 0, 3, 15) +#endif +#ifdef HAS_I422TORGB565ROW_WASMSIMD +ANY31C(I422ToRGB565Row_Any_WASMSIMD, I422ToRGB565Row_WASMSIMD, 1, 0, 2, 7) +#endif +#ifdef HAS_I422TOARGB1555ROW_WASMSIMD +ANY31C(I422ToARGB1555Row_Any_WASMSIMD, I422ToARGB1555Row_WASMSIMD, 1, 0, 2, 7) +#endif +#ifdef HAS_I422TOARGB4444ROW_WASMSIMD +ANY31C(I422ToARGB4444Row_Any_WASMSIMD, I422ToARGB4444Row_WASMSIMD, 1, 0, 2, 7) +#endif #undef ANY31C // Any 3 planes of 16 bit to 1 with yuvconstants @@ -757,6 +778,9 @@ ANY21(SobelXYRow_Any_NEON, SobelXYRow_NEON, 0, 1, 1, 4, 7) #ifdef HAS_SOBELXYROW_LSX ANY21(SobelXYRow_Any_LSX, SobelXYRow_LSX, 0, 1, 1, 4, 15) #endif +#ifdef HAS_MERGEUVROW_WASMSIMD +ANY21(MergeUVRow_Any_WASMSIMD, MergeUVRow_WASMSIMD, 0, 1, 1, 2, 15) +#endif #undef ANY21 // Any 2 planes to 1 with stride @@ -882,6 +906,15 @@ ANY21C(NV12ToRGB565Row_Any_LSX, NV12ToRGB565Row_LSX, 1, 1, 2, 2, 7) #ifdef HAS_NV12TORGB565ROW_LASX ANY21C(NV12ToRGB565Row_Any_LASX, NV12ToRGB565Row_LASX, 1, 1, 2, 2, 15) #endif +#ifdef HAS_NV12TOARGBROW_WASMSIMD +ANY21C(NV12ToARGBRow_Any_WASMSIMD, NV12ToARGBRow_WASMSIMD, 1, 1, 2, 4, 7) +#endif +#ifdef HAS_NV21TOARGBROW_WASMSIMD +ANY21C(NV21ToARGBRow_Any_WASMSIMD, NV21ToARGBRow_WASMSIMD, 1, 1, 2, 4, 7) +#endif +#ifdef HAS_NV12TORGB565ROW_WASMSIMD +ANY21C(NV12ToRGB565Row_Any_WASMSIMD, NV12ToRGB565Row_WASMSIMD, 1, 1, 2, 2, 7) +#endif #undef ANY21C // Any 2 planes of 16 bit to 1 with yuvconstants @@ -1288,6 +1321,36 @@ ANY11(ARGBExtractAlphaRow_Any_NEON, ARGBExtractAlphaRow_NEON, 0, 4, 1, 15) #ifdef HAS_ARGBEXTRACTALPHAROW_LSX ANY11(ARGBExtractAlphaRow_Any_LSX, ARGBExtractAlphaRow_LSX, 0, 4, 1, 15) #endif +#ifdef HAS_RAWTOARGBROW_WASMSIMD +ANY11(RAWToARGBRow_Any_WASMSIMD, RAWToARGBRow_WASMSIMD, 0, 3, 4, 15) +#endif +#ifdef HAS_RGB24TOARGBROW_WASMSIMD +ANY11(RGB24ToARGBRow_Any_WASMSIMD, RGB24ToARGBRow_WASMSIMD, 0, 3, 4, 15) +#endif +#ifdef HAS_SWAPUVROW_WASMSIMD +ANY11(SwapUVRow_Any_WASMSIMD, SwapUVRow_WASMSIMD, 0, 2, 2, 15) +#endif +#ifdef HAS_J400TOARGBROW_WASMSIMD +ANY11(J400ToARGBRow_Any_WASMSIMD, J400ToARGBRow_WASMSIMD, 0, 1, 4, 15) +#endif +#ifdef HAS_RGB565TOARGBROW_WASMSIMD +ANY11(RGB565ToARGBRow_Any_WASMSIMD, RGB565ToARGBRow_WASMSIMD, 0, 2, 4, 7) +#endif +#ifdef HAS_ARGB1555TOARGBROW_WASMSIMD +ANY11(ARGB1555ToARGBRow_Any_WASMSIMD, ARGB1555ToARGBRow_WASMSIMD, 0, 2, 4, 7) +#endif +#ifdef HAS_ARGB4444TOARGBROW_WASMSIMD +ANY11(ARGB4444ToARGBRow_Any_WASMSIMD, ARGB4444ToARGBRow_WASMSIMD, 0, 2, 4, 7) +#endif +#ifdef HAS_ARGBTORGB565ROW_WASMSIMD +ANY11(ARGBToRGB565Row_Any_WASMSIMD, ARGBToRGB565Row_WASMSIMD, 0, 4, 2, 7) +#endif +#ifdef HAS_ARGBTOARGB1555ROW_WASMSIMD +ANY11(ARGBToARGB1555Row_Any_WASMSIMD, ARGBToARGB1555Row_WASMSIMD, 0, 4, 2, 7) +#endif +#ifdef HAS_ARGBTOARGB4444ROW_WASMSIMD +ANY11(ARGBToARGB4444Row_Any_WASMSIMD, ARGBToARGB4444Row_WASMSIMD, 0, 4, 2, 7) +#endif #undef ANY11 // Any 1 to 1 blended. Destination is read, modify, write. @@ -1432,6 +1495,22 @@ ANY11P(ARGBShuffleRow_Any_LSX, ARGBShuffleRow_LSX, const uint8_t*, 4, 4, 7) #ifdef HAS_ARGBSHUFFLEROW_LASX ANY11P(ARGBShuffleRow_Any_LASX, ARGBShuffleRow_LASX, const uint8_t*, 4, 4, 15) #endif +#if defined(HAS_ARGBTORGB565DITHERROW_WASMSIMD) +ANY11P(ARGBToRGB565DitherRow_Any_WASMSIMD, + ARGBToRGB565DitherRow_WASMSIMD, + const uint32_t, + 4, + 2, + 7) +#endif +#ifdef HAS_ARGBSHUFFLEROW_WASMSIMD +ANY11P(ARGBShuffleRow_Any_WASMSIMD, + ARGBShuffleRow_WASMSIMD, + const uint8_t*, + 4, + 4, + 3) +#endif #undef ANY11P #undef ANY11P @@ -1628,6 +1707,24 @@ ANY11C(DivideRow_16_Any_AVX2, DivideRow_16_AVX2, 2, 2, uint16_t, uint16_t, 31) #ifdef HAS_DIVIDEROW_16_NEON ANY11C(DivideRow_16_Any_NEON, DivideRow_16_NEON, 2, 2, uint16_t, uint16_t, 15) #endif +#ifdef HAS_CONVERT8TO16ROW_WASMSIMD +ANY11C(Convert8To16Row_Any_WASMSIMD, + Convert8To16Row_WASMSIMD, + 1, + 2, + uint8_t, + uint16_t, + 15) +#endif +#ifdef HAS_MULTIPLYROW_16_WASMSIMD +ANY11C(MultiplyRow_16_Any_WASMSIMD, + MultiplyRow_16_WASMSIMD, + 2, + 2, + uint16_t, + uint16_t, + 15) +#endif #undef ANY11C // Any 1 to 1 with parameter and shorts. BPP measures in shorts. @@ -1810,6 +1907,24 @@ ANY11I(InterpolateRow_16_Any_AVX2, 1, 15) #endif +#ifdef HAS_INTERPOLATEROW_WASMSIMD +ANY11I(InterpolateRow_Any_WASMSIMD, + InterpolateRow_WASMSIMD, + uint8_t, + uint8_t, + 1, + 1, + 15) +#endif +#ifdef HAS_INTERPOLATEROW_16_WASMSIMD +ANY11I(InterpolateRow_16_Any_WASMSIMD, + InterpolateRow_16_WASMSIMD, + uint16_t, + uint16_t, + 1, + 1, + 7) +#endif #undef ANY11I // Any 1 to 1 half row with scale param @@ -1981,6 +2096,18 @@ ANY11M(RGB24MirrorRow_Any_AVX2, RGB24MirrorRow_AVX2, 3, 31) #ifdef HAS_RGB24MIRRORROW_NEON ANY11M(RGB24MirrorRow_Any_NEON, RGB24MirrorRow_NEON, 3, 15) #endif +#ifdef HAS_MIRRORROW_WASMSIMD +ANY11M(MirrorRow_Any_WASMSIMD, MirrorRow_WASMSIMD, 1, 15) +#endif +#ifdef HAS_MIRRORUVROW_WASMSIMD +ANY11M(MirrorUVRow_Any_WASMSIMD, MirrorUVRow_WASMSIMD, 2, 7) +#endif +#ifdef HAS_ARGBMIRRORROW_WASMSIMD +ANY11M(ARGBMirrorRow_Any_WASMSIMD, ARGBMirrorRow_WASMSIMD, 4, 3) +#endif +#ifdef HAS_RGB24MIRRORROW_WASMSIMD +ANY11M(RGB24MirrorRow_Any_WASMSIMD, RGB24MirrorRow_WASMSIMD, 3, 15) +#endif #undef ANY11M // Any 1 plane. (memset) @@ -2346,6 +2473,39 @@ ANY12M(RGBToUV444MatrixRow_Any_NEON, RGBToUV444MatrixRow_NEON, 3, 7) #ifdef HAS_ARGBTOUV444MATRIXROW_NEON_I8MM ANY12M(ARGBToUV444MatrixRow_Any_NEON_I8MM, ARGBToUV444MatrixRow_NEON_I8MM, 4, 7) #endif +#ifdef HAS_ARGBTOUVMATRIXROW_WASMSIMD +ANY12MS(ARGBToUVMatrixRow_Any_WASMSIMD, ARGBToUVMatrixRow_WASMSIMD, 0, 4, 15) +#endif +#ifdef HAS_RGBTOUVMATRIXROW_WASMSIMD +ANY12MS(RGBToUVMatrixRow_Any_WASMSIMD, RGBToUVMatrixRow_WASMSIMD, 0, 3, 15) +#endif +#ifdef HAS_RGB565TOUVMATRIXROW_WASMSIMD +ANY12MS(RGB565ToUVMatrixRow_Any_WASMSIMD, + RGB565ToUVMatrixRow_WASMSIMD, + 0, + 2, + 15) +#endif +#ifdef HAS_ARGB1555TOUVMATRIXROW_WASMSIMD +ANY12MS(ARGB1555ToUVMatrixRow_Any_WASMSIMD, + ARGB1555ToUVMatrixRow_WASMSIMD, + 0, + 2, + 15) +#endif +#ifdef HAS_ARGB4444TOUVMATRIXROW_WASMSIMD +ANY12MS(ARGB4444ToUVMatrixRow_Any_WASMSIMD, + ARGB4444ToUVMatrixRow_WASMSIMD, + 0, + 2, + 15) +#endif +#ifdef HAS_ARGBTOUV444MATRIXROW_WASMSIMD +ANY12M(ARGBToUV444MatrixRow_Any_WASMSIMD, ARGBToUV444MatrixRow_WASMSIMD, 4, 15) +#endif +#ifdef HAS_RGBTOUV444MATRIXROW_WASMSIMD +ANY12M(RGBToUV444MatrixRow_Any_WASMSIMD, RGBToUV444MatrixRow_WASMSIMD, 3, 15) +#endif #define ANY11MC(NAMEANY, ANY_SIMD, BPP, MASK) \ void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width, \ @@ -2420,6 +2580,21 @@ ANY11MC(RGBToYMatrixRow_Any_LASX, RGBToYMatrixRow_LASX, 3, 31) #ifdef HAS_ARGBTOYMATRIXROW_RVV ANY11MC(ARGBToYMatrixRow_Any_RVV, ARGBToYMatrixRow_RVV, 4, 15) #endif +#ifdef HAS_ARGBTOYMATRIXROW_WASMSIMD +ANY11MC(ARGBToYMatrixRow_Any_WASMSIMD, ARGBToYMatrixRow_WASMSIMD, 4, 15) +#endif +#ifdef HAS_RGBTOYMATRIXROW_WASMSIMD +ANY11MC(RGBToYMatrixRow_Any_WASMSIMD, RGBToYMatrixRow_WASMSIMD, 3, 15) +#endif +#ifdef HAS_RGB565TOYMATRIXROW_WASMSIMD +ANY11MC(RGB565ToYMatrixRow_Any_WASMSIMD, RGB565ToYMatrixRow_WASMSIMD, 2, 15) +#endif +#ifdef HAS_ARGB1555TOYMATRIXROW_WASMSIMD +ANY11MC(ARGB1555ToYMatrixRow_Any_WASMSIMD, ARGB1555ToYMatrixRow_WASMSIMD, 2, 15) +#endif +#ifdef HAS_ARGB4444TOYMATRIXROW_WASMSIMD +ANY11MC(ARGB4444ToYMatrixRow_Any_WASMSIMD, ARGB4444ToYMatrixRow_WASMSIMD, 2, 15) +#endif #undef ANY11MC #ifdef HAS_YUY2TOUVROW_AVX2 diff --git a/source/row_wasm.cc b/source/row_wasm.cc index 5814eb1cb..4376d912c 100644 --- a/source/row_wasm.cc +++ b/source/row_wasm.cc @@ -18,13 +18,39 @@ namespace libyuv { extern "C" { #endif -#ifdef HAS_RAWTORGB24ROW_WASMSIMD #ifdef __wasm_relaxed_simd__ #define LIBYUV_WASM_SWIZZLE wasm_i8x16_relaxed_swizzle #else #define LIBYUV_WASM_SWIZZLE wasm_i8x16_swizzle #endif +// V8 materializes v128.const values and arbitrary i8x16.shuffle masks with +// several instructions at every use, including inside loops. These helpers +// make constants opaque to clang so they are built once, before the loop, +// and kept in registers. Use swizzle with a LoadConst() table instead of +// i8x16.shuffle unless the pattern maps to a native instruction (unpack, +// 32-bit lane shuffle, concat/palignr, even/odd byte unzip). +// Loops that access ptr + 16 etc. use asm("" : "+r"(ptr)) so LLVM keeps the +// constant offsets in the load/store instructions instead of creating extra +// pointer induction variables. +static inline v128_t LoadConst(const void* p) { + asm("" : "+r"(p)); + return wasm_v128_load(p); +} +static inline v128_t SplatConst8(int v) { + asm("" : "+r"(v)); + return wasm_i8x16_splat(v); +} +static inline v128_t SplatConst16(int v) { + asm("" : "+r"(v)); + return wasm_i16x8_splat(v); +} +static inline v128_t SplatConst32(int v) { + asm("" : "+r"(v)); + return wasm_i32x4_splat(v); +} + +#ifdef HAS_RAWTORGB24ROW_WASMSIMD static const uint8_t kRawToRGB24SwizzleTable[96] = { 2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, @@ -109,9 +135,1787 @@ void RAWToRGB24Row_WASMSIMD(const uint8_t* src_raw, wasm_v128_store(dst_rgb24 + 32, d2); } } -#undef LIBYUV_WASM_SWIZZLE #endif // HAS_RAWTORGB24ROW_WASMSIMD +#if defined(HAS_RAWTOARGBROW_WASMSIMD) || defined(HAS_RGB24TOARGBROW_WASMSIMD) +// 16 pixels of 3 bytes to 4 bytes. shuf holds 2 masks: one for loads at +// offsets 0, 12, 24 and one for the load at offset 32 (avoids over-read). +static void RGB3ToARGBRow_WASMSIMD(const uint8_t* src, + uint8_t* dst_argb, + int width, + const uint8_t* shuf) { + v128_t m0 = LoadConst(shuf); + v128_t m1 = LoadConst(shuf + 16); + v128_t alpha = SplatConst32((int)0xff000000); + do { + asm("" : "+r"(src), "+r"(dst_argb)); + v128_t d0 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src), m0); + v128_t d1 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src + 12), m0); + v128_t d2 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src + 24), m0); + v128_t d3 = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src + 32), m1); + wasm_v128_store(dst_argb, wasm_v128_or(d0, alpha)); + wasm_v128_store(dst_argb + 16, wasm_v128_or(d1, alpha)); + wasm_v128_store(dst_argb + 32, wasm_v128_or(d2, alpha)); + wasm_v128_store(dst_argb + 48, wasm_v128_or(d3, alpha)); + src += 48; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif + +#ifdef HAS_RAWTOARGBROW_WASMSIMD +static const uint8_t kShuffleRAWToARGB[32] = { + 2, 1, 0, 128, 5, 4, 3, 128, 8, 7, 6, 128, 11, 10, 9, 128, + 6, 5, 4, 128, 9, 8, 7, 128, 12, 11, 10, 128, 15, 14, 13, 128}; +void RAWToARGBRow_WASMSIMD(const uint8_t* src_raw, + uint8_t* dst_argb, + int width) { + RGB3ToARGBRow_WASMSIMD(src_raw, dst_argb, width, kShuffleRAWToARGB); +} +#endif // HAS_RAWTOARGBROW_WASMSIMD + +#ifdef HAS_RGB24TOARGBROW_WASMSIMD +static const uint8_t kShuffleRGB24ToARGB[32] = { + 0, 1, 2, 128, 3, 4, 5, 128, 6, 7, 8, 128, 9, 10, 11, 128, + 4, 5, 6, 128, 7, 8, 9, 128, 10, 11, 12, 128, 13, 14, 15, 128}; +void RGB24ToARGBRow_WASMSIMD(const uint8_t* src_rgb24, + uint8_t* dst_argb, + int width) { + RGB3ToARGBRow_WASMSIMD(src_rgb24, dst_argb, width, kShuffleRGB24ToARGB); +} +#endif // HAS_RGB24TOARGBROW_WASMSIMD + +#if defined(HAS_ARGBTOYMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) || \ + defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOYMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_RGB565TOYMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGB1555TOYMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGB4444TOYMATRIXROW_WASMSIMD) || \ + defined(HAS_RGB565TOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGB1555TOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGB4444TOUVMATRIXROW_WASMSIMD) +// RGB to YUV. Math is in 16 bit lanes, which wrap, but sum + bias fits in +// 16 bits so the results match the C code. i32x4.dot_i16x8_s would be best +// on x86 but is 3 instructions on Arm, so i16x8.mul (mul + mla on Arm) and +// horizontal adds that are cheap on both are used instead. + +// lo * k0 + hi * k1. +static inline v128_t MulAdd16(v128_t lo, v128_t hi, v128_t k0, v128_t k1) { + return wasm_i16x8_add(wasm_i16x8_mul(lo, k0), wasm_i16x8_mul(hi, k1)); +} + +// Sums adjacent 16 bit lanes of t0 and t1. +static inline v128_t HAddPairs16(v128_t t0, v128_t t1) { + return wasm_i16x8_add(wasm_i16x8_shuffle(t0, t1, 0, 2, 4, 6, 8, 10, 12, 14), + wasm_i16x8_shuffle(t0, t1, 1, 3, 5, 7, 9, 11, 13, 15)); +} + +// Splats 16 bit {lo, hi} into each 32 bit lane. +static inline v128_t SplatPair16(int lo, int hi) { + return SplatConst32((lo & 0xffff) | (hi << 16)); +} + +// (bias - sum) >> 8 for U and V. +static inline v128_t SubShr8(v128_t bias, v128_t sum) { + return wasm_u16x8_shr(wasm_i16x8_sub(bias, sum), 8); +} +#endif + +#if defined(HAS_ARGBTOUV444MATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOYMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUV444MATRIXROW_WASMSIMD) +// Swizzles 4 pixels to 16 bit {c0 x4, c2 x4} and {c1 x4, c3 x4}, so the 2 +// halves of the products are summed with a 64 bit interleave. +static const uint8_t kShuffleARGBToPlanar[32] = { + 0, 128, 4, 128, 8, 128, 12, 128, 2, 128, 6, 128, 10, 128, 14, 128, + 1, 128, 5, 128, 9, 128, 13, 128, 3, 128, 7, 128, 11, 128, 15, 128}; +// RGB24 to {b x4, r x4} and {g x4, 0 x4}. The second pair of masks is for +// the load at offset 32 (avoids over-read). +static const uint8_t kShuffleRGBToPlanar[64] = { + 0, 128, 3, 128, 6, 128, 9, 128, 2, 128, 5, 128, 8, 128, 11, 128, + 1, 128, 4, 128, 7, 128, 10, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 4, 128, 7, 128, 10, 128, 13, 128, 6, 128, 9, 128, 12, 128, 15, 128, + 5, 128, 8, 128, 11, 128, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128}; + +// {a x4, b x4} as 16 bit lanes. +static inline v128_t Splat4x2(int a, int b) { + asm("" : "+r"(a), "+r"(b)); + return wasm_i16x8_make(a, a, a, a, b, b, b, b); +} + +// Sums the 64 bit halves of t0 and t1. +static inline v128_t HAddHalves64(v128_t t0, v128_t t1) { + return wasm_i16x8_add(wasm_i64x2_shuffle(t0, t1, 0, 2), + wasm_i64x2_shuffle(t0, t1, 1, 3)); +} + +// 16 U and V from 16 pixels swizzled by m (m[0], m[1] for the first 3 +// vectors, m[2], m[3] for the last). +static inline void PlanarToUV16(const v128_t* s, + const v128_t* m, + v128_t ku02, + v128_t ku13, + v128_t kv02, + v128_t kv13, + v128_t bias_u, + v128_t bias_v, + uint8_t* dst_u, + uint8_t* dst_v) { + v128_t u[4], v[4]; + for (int i = 0; i < 4; ++i) { + v128_t lo = LIBYUV_WASM_SWIZZLE(s[i], m[i == 3 ? 2 : 0]); + v128_t hi = LIBYUV_WASM_SWIZZLE(s[i], m[i == 3 ? 3 : 1]); + u[i] = MulAdd16(lo, hi, ku02, ku13); + v[i] = MulAdd16(lo, hi, kv02, kv13); + } + wasm_v128_store(dst_u, wasm_u8x16_narrow_i16x8( + SubShr8(bias_u, HAddHalves64(u[0], u[1])), + SubShr8(bias_u, HAddHalves64(u[2], u[3])))); + wasm_v128_store(dst_v, wasm_u8x16_narrow_i16x8( + SubShr8(bias_v, HAddHalves64(v[0], v[1])), + SubShr8(bias_v, HAddHalves64(v[2], v[3])))); +} +#endif + +#if defined(HAS_ARGBTOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_RGBTOUVMATRIXROW_WASMSIMD) +// Rounded 2x2 average of 8 pixels held as 16 bit {c0, c2} or {c1, c3} +// pairs. a = row 0, b = row 1, 4 pixels per vector. Returns 4 pixels. +static inline v128_t Avg2x2Pairs(v128_t a0, + v128_t a1, + v128_t b0, + v128_t b1, + v128_t two) { + v128_t s0 = wasm_i16x8_add(a0, b0); + v128_t s1 = wasm_i16x8_add(a1, b1); + v128_t s = wasm_i16x8_add(wasm_i32x4_shuffle(s0, s1, 0, 2, 4, 6), + wasm_i32x4_shuffle(s0, s1, 1, 3, 5, 7)); + return wasm_u16x8_shr(wasm_i16x8_add(s, two), 2); +} + +// Stores 8 U and 8 V from 2 vectors each of {c0, c2} and {c1, c3} pairs. +static inline void PairsToUV8(const v128_t* lo, + const v128_t* hi, + v128_t ku02, + v128_t ku13, + v128_t kv02, + v128_t kv13, + v128_t bias_u, + v128_t bias_v, + uint8_t* dst_u, + uint8_t* dst_v) { + v128_t u = HAddPairs16(MulAdd16(lo[0], hi[0], ku02, ku13), + MulAdd16(lo[1], hi[1], ku02, ku13)); + v128_t v = HAddPairs16(MulAdd16(lo[0], hi[0], kv02, kv13), + MulAdd16(lo[1], hi[1], kv02, kv13)); + v128_t uv = wasm_u8x16_narrow_i16x8(SubShr8(bias_u, u), SubShr8(bias_v, v)); + wasm_v128_store64_lane(dst_u, uv, 0); + wasm_v128_store64_lane(dst_v, uv, 1); +} +#endif + +#ifdef HAS_ARGBTOYMATRIXROW_WASMSIMD +void ARGBToYMatrixRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + v128_t mask = SplatConst16(0xff); + v128_t k02 = SplatPair16(c->kRGBToY[0], c->kRGBToY[2]); + v128_t k13 = SplatPair16(c->kRGBToY[1], c->kRGBToY[3]); + v128_t bias = SplatConst16(c->kAddY[0]); + do { + v128_t t[4]; + for (int i = 0; i < 4; ++i) { + v128_t s = wasm_v128_load(src_argb + i * 16); + t[i] = MulAdd16(wasm_v128_and(s, mask), wasm_u16x8_shr(s, 8), k02, k13); + } + v128_t y0 = wasm_i16x8_add(HAddPairs16(t[0], t[1]), bias); + v128_t y1 = wasm_i16x8_add(HAddPairs16(t[2], t[3]), bias); + wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(wasm_u16x8_shr(y0, 8), + wasm_u16x8_shr(y1, 8))); + src_argb += 64; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGBTOYMATRIXROW_WASMSIMD + +#ifdef HAS_RGBTOYMATRIXROW_WASMSIMD +// RGB24 is alpha 255 in the C version, so 255 * kRGBToY[3] is in the bias. +void RGBToYMatrixRow_WASMSIMD(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + v128_t m[4]; + for (int i = 0; i < 4; ++i) { + m[i] = LoadConst(kShuffleRGBToPlanar + i * 16); + } + v128_t k02 = Splat4x2(c->kRGBToY[0], c->kRGBToY[2]); + v128_t k13 = Splat4x2(c->kRGBToY[1], 0); + v128_t bias = SplatConst16(c->kAddY[0] + 255 * c->kRGBToY[3]); + do { + v128_t t[4]; + for (int i = 0; i < 4; ++i) { + v128_t s = wasm_v128_load(src_rgb + (i == 3 ? 32 : i * 12)); + t[i] = MulAdd16(LIBYUV_WASM_SWIZZLE(s, m[i == 3 ? 2 : 0]), + LIBYUV_WASM_SWIZZLE(s, m[i == 3 ? 3 : 1]), k02, k13); + } + v128_t y0 = wasm_i16x8_add(HAddHalves64(t[0], t[1]), bias); + v128_t y1 = wasm_i16x8_add(HAddHalves64(t[2], t[3]), bias); + wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(wasm_u16x8_shr(y0, 8), + wasm_u16x8_shr(y1, 8))); + src_rgb += 48; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_RGBTOYMATRIXROW_WASMSIMD + +#ifdef HAS_ARGBTOUV444MATRIXROW_WASMSIMD +void ARGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + v128_t m[4]; + m[0] = m[2] = LoadConst(kShuffleARGBToPlanar); + m[1] = m[3] = LoadConst(kShuffleARGBToPlanar + 16); + v128_t ku02 = Splat4x2(c->kRGBToU[0], c->kRGBToU[2]); + v128_t ku13 = Splat4x2(c->kRGBToU[1], c->kRGBToU[3]); + v128_t kv02 = Splat4x2(c->kRGBToV[0], c->kRGBToV[2]); + v128_t kv13 = Splat4x2(c->kRGBToV[1], c->kRGBToV[3]); + v128_t bias = SplatConst16(c->kAddUV[0]); + do { + v128_t s[4]; + for (int i = 0; i < 4; ++i) { + s[i] = wasm_v128_load(src_argb + i * 16); + } + PlanarToUV16(s, m, ku02, ku13, kv02, kv13, bias, bias, dst_u, dst_v); + src_argb += 64; + dst_u += 16; + dst_v += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGBTOUV444MATRIXROW_WASMSIMD + +#ifdef HAS_RGBTOUV444MATRIXROW_WASMSIMD +void RGBToUV444MatrixRow_WASMSIMD(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + v128_t m[4]; + for (int i = 0; i < 4; ++i) { + m[i] = LoadConst(kShuffleRGBToPlanar + i * 16); + } + v128_t ku02 = Splat4x2(c->kRGBToU[0], c->kRGBToU[2]); + v128_t ku13 = Splat4x2(c->kRGBToU[1], 0); + v128_t kv02 = Splat4x2(c->kRGBToV[0], c->kRGBToV[2]); + v128_t kv13 = Splat4x2(c->kRGBToV[1], 0); + v128_t bias_u = SplatConst16(c->kAddUV[0] - 255 * c->kRGBToU[3]); + v128_t bias_v = SplatConst16(c->kAddUV[0] - 255 * c->kRGBToV[3]); + do { + v128_t s[4]; + for (int i = 0; i < 4; ++i) { + s[i] = wasm_v128_load(src_rgb + (i == 3 ? 32 : i * 12)); + } + PlanarToUV16(s, m, ku02, ku13, kv02, kv13, bias_u, bias_v, dst_u, dst_v); + src_rgb += 48; + dst_u += 16; + dst_v += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_RGBTOUV444MATRIXROW_WASMSIMD + +#ifdef HAS_ARGBTOUVMATRIXROW_WASMSIMD +void ARGBToUVMatrixRow_WASMSIMD(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const uint8_t* src_argb1 = src_argb + src_stride_argb; + v128_t mask = SplatConst16(0xff); + v128_t two = SplatConst16(2); + v128_t ku02 = SplatPair16(c->kRGBToU[0], c->kRGBToU[2]); + v128_t ku13 = SplatPair16(c->kRGBToU[1], c->kRGBToU[3]); + v128_t kv02 = SplatPair16(c->kRGBToV[0], c->kRGBToV[2]); + v128_t kv13 = SplatPair16(c->kRGBToV[1], c->kRGBToV[3]); + v128_t bias = SplatConst16(c->kAddUV[0]); + do { + v128_t lo[2], hi[2]; + for (int i = 0; i < 2; ++i) { + v128_t a0 = wasm_v128_load(src_argb + i * 32); + v128_t a1 = wasm_v128_load(src_argb + i * 32 + 16); + v128_t b0 = wasm_v128_load(src_argb1 + i * 32); + v128_t b1 = wasm_v128_load(src_argb1 + i * 32 + 16); + lo[i] = + Avg2x2Pairs(wasm_v128_and(a0, mask), wasm_v128_and(a1, mask), + wasm_v128_and(b0, mask), wasm_v128_and(b1, mask), two); + hi[i] = Avg2x2Pairs(wasm_u16x8_shr(a0, 8), wasm_u16x8_shr(a1, 8), + wasm_u16x8_shr(b0, 8), wasm_u16x8_shr(b1, 8), two); + } + PairsToUV8(lo, hi, ku02, ku13, kv02, kv13, bias, bias, dst_u, dst_v); + src_argb += 64; + src_argb1 += 64; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGBTOUVMATRIXROW_WASMSIMD + +#ifdef HAS_RGBTOUVMATRIXROW_WASMSIMD +// Swizzles 4 RGB24 pixels to 16 bit {b, r} and {g, 0} pairs. The second +// pair of masks is for the load at offset 32 (avoids over-read). +static const uint8_t kShuffleRGBToPairs[64] = { + 0, 128, 2, 128, 3, 128, 5, 128, 6, 128, 8, 128, 9, 128, 11, 128, + 1, 128, 128, 128, 4, 128, 128, 128, 7, 128, 128, 128, 10, 128, 128, 128, + 4, 128, 6, 128, 7, 128, 9, 128, 10, 128, 12, 128, 13, 128, 15, 128, + 5, 128, 128, 128, 8, 128, 128, 128, 11, 128, 128, 128, 14, 128, 128, 128}; + +void RGBToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const uint8_t* src_rgb1 = src_rgb + src_stride_rgb; + v128_t m[4]; + for (int i = 0; i < 4; ++i) { + m[i] = LoadConst(kShuffleRGBToPairs + i * 16); + } + v128_t two = SplatConst16(2); + v128_t ku02 = SplatPair16(c->kRGBToU[0], c->kRGBToU[2]); + v128_t ku13 = SplatPair16(c->kRGBToU[1], 0); + v128_t kv02 = SplatPair16(c->kRGBToV[0], c->kRGBToV[2]); + v128_t kv13 = SplatPair16(c->kRGBToV[1], 0); + v128_t bias_u = SplatConst16(c->kAddUV[0] - 255 * c->kRGBToU[3]); + v128_t bias_v = SplatConst16(c->kAddUV[0] - 255 * c->kRGBToV[3]); + do { + v128_t lo[2], hi[2]; + for (int i = 0; i < 2; ++i) { + // Pixels 8 * i to 8 * i + 3 at offset 24 * i, the rest at 12 or 32. + int off = i ? 32 : 12; + v128_t a0 = wasm_v128_load(src_rgb + i * 24); + v128_t a1 = wasm_v128_load(src_rgb + off); + v128_t b0 = wasm_v128_load(src_rgb1 + i * 24); + v128_t b1 = wasm_v128_load(src_rgb1 + off); + v128_t ml = m[i * 2]; + v128_t mh = m[i * 2 + 1]; + lo[i] = Avg2x2Pairs( + LIBYUV_WASM_SWIZZLE(a0, m[0]), LIBYUV_WASM_SWIZZLE(a1, ml), + LIBYUV_WASM_SWIZZLE(b0, m[0]), LIBYUV_WASM_SWIZZLE(b1, ml), two); + hi[i] = Avg2x2Pairs( + LIBYUV_WASM_SWIZZLE(a0, m[1]), LIBYUV_WASM_SWIZZLE(a1, mh), + LIBYUV_WASM_SWIZZLE(b0, m[1]), LIBYUV_WASM_SWIZZLE(b1, mh), two); + } + PairsToUV8(lo, hi, ku02, ku13, kv02, kv13, bias_u, bias_v, dst_u, dst_v); + src_rgb += 48; + src_rgb1 += 48; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} +#endif // HAS_RGBTOUVMATRIXROW_WASMSIMD + +// 16 bit pixel formats keep each channel in its own 16 bit lane. +// 5 and 6 bit channels expand as (v * 33) >> 2 and (v * 65) >> 4, and 4 bit +// channels as v * 17, which match the C bit replication. + +#if defined(HAS_RGB565TOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGB1555TOUVMATRIXROW_WASMSIMD) || \ + defined(HAS_ARGB4444TOUVMATRIXROW_WASMSIMD) +// Rounded 2x2 average of 16 pixels of 1 channel. a = row 0, b = row 1. +static inline v128_t Avg2x2Lanes(v128_t a0, + v128_t a1, + v128_t b0, + v128_t b1, + v128_t two) { + v128_t s = HAddPairs16(wasm_i16x8_add(a0, b0), wasm_i16x8_add(a1, b1)); + return wasm_u16x8_shr(wasm_i16x8_add(s, two), 2); +} + +// Stores 8 U and 8 V from 8 pixels of 16 bit channels. k holds U +// coefficients for b, g, r, a, then V coefficients, then bias U and V. +static inline void ChannelsToUV8(v128_t b, + v128_t g, + v128_t r, + v128_t a, + const v128_t* k, + uint8_t* dst_u, + uint8_t* dst_v) { + v128_t u = + wasm_i16x8_add(MulAdd16(b, g, k[0], k[1]), MulAdd16(r, a, k[2], k[3])); + v128_t v = + wasm_i16x8_add(MulAdd16(b, g, k[4], k[5]), MulAdd16(r, a, k[6], k[7])); + v128_t uv = wasm_u8x16_narrow_i16x8(SubShr8(k[8], u), SubShr8(k[9], v)); + wasm_v128_store64_lane(dst_u, uv, 0); + wasm_v128_store64_lane(dst_v, uv, 1); +} + +// U and V coefficients for ChannelsToUV8. The alpha term is folded into the +// bias if alpha is always 255 (opaque). +static inline void LoadUVChannelConstants(const struct ArgbConstants* c, + int opaque, + v128_t* k) { + for (int i = 0; i < 4; ++i) { + k[i] = SplatConst16(c->kRGBToU[i]); + k[i + 4] = SplatConst16(c->kRGBToV[i]); + } + k[8] = SplatConst16(c->kAddUV[0] - opaque * 255 * c->kRGBToU[3]); + k[9] = SplatConst16(c->kAddUV[0] - opaque * 255 * c->kRGBToV[3]); +} +#endif +#ifdef HAS_RGB565TOYMATRIXROW_WASMSIMD +void RGB565ToYMatrixRow_WASMSIMD(const uint8_t* src_rgb565, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + v128_t m5 = SplatConst16(0x1f); + v128_t m6 = SplatConst16(0x3f); + v128_t k33 = SplatConst16(33); + v128_t k65 = SplatConst16(65); + v128_t kb = SplatConst16(c->kRGBToY[0]); + v128_t kg = SplatConst16(c->kRGBToY[1]); + v128_t kr = SplatConst16(c->kRGBToY[2]); + v128_t bias = SplatConst16(c->kAddY[0] + 255 * c->kRGBToY[3]); + do { + v128_t y[2]; + for (int i = 0; i < 2; ++i) { + v128_t x = wasm_v128_load(src_rgb565 + i * 16); + v128_t b = wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(x, m5), k33), 2); + v128_t g = wasm_u16x8_shr( + wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 5), m6), k65), 4); + v128_t r = wasm_u16x8_shr(wasm_i16x8_mul(wasm_u16x8_shr(x, 11), k33), 2); + v128_t s = wasm_i16x8_add(wasm_i16x8_mul(b, kb), bias); + y[i] = wasm_u16x8_shr(wasm_i16x8_add(s, MulAdd16(g, r, kg, kr)), 8); + } + wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(y[0], y[1])); + src_rgb565 += 32; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_RGB565TOYMATRIXROW_WASMSIMD + +#ifdef HAS_ARGB1555TOYMATRIXROW_WASMSIMD +void ARGB1555ToYMatrixRow_WASMSIMD(const uint8_t* src_argb1555, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + v128_t m5 = SplatConst16(0x1f); + v128_t k33 = SplatConst16(33); + v128_t kb = SplatConst16(c->kRGBToY[0]); + v128_t kg = SplatConst16(c->kRGBToY[1]); + v128_t kr = SplatConst16(c->kRGBToY[2]); + v128_t ka = SplatConst16(255 * c->kRGBToY[3]); + v128_t bias = SplatConst16(c->kAddY[0]); + do { + v128_t y[2]; + for (int i = 0; i < 2; ++i) { + v128_t x = wasm_v128_load(src_argb1555 + i * 16); + v128_t b = wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(x, m5), k33), 2); + v128_t g = wasm_u16x8_shr( + wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 5), m5), k33), 2); + v128_t r = wasm_u16x8_shr( + wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 10), m5), k33), 2); + // Alpha is 0 or 255. + v128_t s = wasm_i16x8_add(wasm_v128_and(wasm_i16x8_shr(x, 15), ka), bias); + s = wasm_i16x8_add(s, wasm_i16x8_mul(b, kb)); + y[i] = wasm_u16x8_shr(wasm_i16x8_add(s, MulAdd16(g, r, kg, kr)), 8); + } + wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(y[0], y[1])); + src_argb1555 += 32; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGB1555TOYMATRIXROW_WASMSIMD + +#ifdef HAS_ARGB4444TOYMATRIXROW_WASMSIMD +// 4 bit channels expand as v * 17, so the coefficients are scaled by 17. +void ARGB4444ToYMatrixRow_WASMSIMD(const uint8_t* src_argb4444, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + v128_t m4 = SplatConst16(0x0f); + v128_t kb = SplatConst16(c->kRGBToY[0] * 17); + v128_t kg = SplatConst16(c->kRGBToY[1] * 17); + v128_t kr = SplatConst16(c->kRGBToY[2] * 17); + v128_t ka = SplatConst16(c->kRGBToY[3] * 17); + v128_t bias = SplatConst16(c->kAddY[0]); + do { + v128_t y[2]; + for (int i = 0; i < 2; ++i) { + v128_t x = wasm_v128_load(src_argb4444 + i * 16); + v128_t s = wasm_i16x8_add( + MulAdd16(wasm_v128_and(x, m4), wasm_u16x8_shr(x, 12), kb, ka), bias); + s = wasm_i16x8_add( + s, MulAdd16(wasm_v128_and(wasm_u16x8_shr(x, 4), m4), + wasm_v128_and(wasm_u16x8_shr(x, 8), m4), kg, kr)); + y[i] = wasm_u16x8_shr(s, 8); + } + wasm_v128_store(dst_y, wasm_u8x16_narrow_i16x8(y[0], y[1])); + src_argb4444 += 32; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGB4444TOYMATRIXROW_WASMSIMD + +#ifdef HAS_RGB565TOUVMATRIXROW_WASMSIMD +void RGB565ToUVMatrixRow_WASMSIMD(const uint8_t* src_rgb565, + int src_stride_rgb565, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const uint8_t* src_rgb565_1 = src_rgb565 + src_stride_rgb565; + v128_t m5 = SplatConst16(0x1f); + v128_t m6 = SplatConst16(0x3f); + v128_t k33 = SplatConst16(33); + v128_t k65 = SplatConst16(65); + v128_t two = SplatConst16(2); + v128_t k[10]; + LoadUVChannelConstants(c, 1, k); + do { + v128_t b[4], g[4], r[4]; + for (int i = 0; i < 4; ++i) { + v128_t x = + wasm_v128_load((i & 2 ? src_rgb565_1 : src_rgb565) + (i & 1) * 16); + b[i] = wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(x, m5), k33), 2); + g[i] = wasm_u16x8_shr( + wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 5), m6), k65), 4); + r[i] = wasm_u16x8_shr(wasm_i16x8_mul(wasm_u16x8_shr(x, 11), k33), 2); + } + ChannelsToUV8(Avg2x2Lanes(b[0], b[1], b[2], b[3], two), + Avg2x2Lanes(g[0], g[1], g[2], g[3], two), + Avg2x2Lanes(r[0], r[1], r[2], r[3], two), + wasm_i16x8_const_splat(0), k, dst_u, dst_v); + src_rgb565 += 32; + src_rgb565_1 += 32; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} +#endif // HAS_RGB565TOUVMATRIXROW_WASMSIMD + +#ifdef HAS_ARGB1555TOUVMATRIXROW_WASMSIMD +void ARGB1555ToUVMatrixRow_WASMSIMD(const uint8_t* src_argb1555, + int src_stride_argb1555, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const uint8_t* src_argb1555_1 = src_argb1555 + src_stride_argb1555; + v128_t m5 = SplatConst16(0x1f); + v128_t k33 = SplatConst16(33); + v128_t two = SplatConst16(2); + v128_t k[10]; + LoadUVChannelConstants(c, 0, k); + do { + v128_t b[4], g[4], r[4], a[4]; + for (int i = 0; i < 4; ++i) { + v128_t x = wasm_v128_load((i & 2 ? src_argb1555_1 : src_argb1555) + + (i & 1) * 16); + b[i] = wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(x, m5), k33), 2); + g[i] = wasm_u16x8_shr( + wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 5), m5), k33), 2); + r[i] = wasm_u16x8_shr( + wasm_i16x8_mul(wasm_v128_and(wasm_u16x8_shr(x, 10), m5), k33), 2); + a[i] = wasm_u16x8_shr(wasm_i16x8_shr(x, 15), 8); + } + ChannelsToUV8(Avg2x2Lanes(b[0], b[1], b[2], b[3], two), + Avg2x2Lanes(g[0], g[1], g[2], g[3], two), + Avg2x2Lanes(r[0], r[1], r[2], r[3], two), + Avg2x2Lanes(a[0], a[1], a[2], a[3], two), k, dst_u, dst_v); + src_argb1555 += 32; + src_argb1555_1 += 32; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGB1555TOUVMATRIXROW_WASMSIMD + +#ifdef HAS_ARGB4444TOUVMATRIXROW_WASMSIMD +// Sums the nibble pairs {b, r} and {g, a} of 2x2 pixels in bytes, then +// expands the sums by 17 before rounding. +void ARGB4444ToUVMatrixRow_WASMSIMD(const uint8_t* src_argb4444, + int src_stride_argb4444, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const uint8_t* src_argb4444_1 = src_argb4444 + src_stride_argb4444; + v128_t m4 = SplatConst16(0x0f0f); + v128_t m8 = SplatConst16(0xff); + v128_t k17 = SplatConst16(17); + v128_t two = SplatConst16(2); + v128_t k[10]; + LoadUVChannelConstants(c, 0, k); + do { + v128_t s[2], t[2]; + for (int i = 0; i < 2; ++i) { + v128_t a = wasm_v128_load(src_argb4444 + i * 16); + v128_t b = wasm_v128_load(src_argb4444_1 + i * 16); + s[i] = wasm_i16x8_add(wasm_v128_and(a, m4), wasm_v128_and(b, m4)); + t[i] = wasm_i16x8_add(wasm_v128_and(wasm_u16x8_shr(a, 4), m4), + wasm_v128_and(wasm_u16x8_shr(b, 4), m4)); + } + v128_t br = HAddPairs16(s[0], s[1]); + v128_t ga = HAddPairs16(t[0], t[1]); + v128_t ch[4] = {wasm_v128_and(br, m8), wasm_v128_and(ga, m8), + wasm_u16x8_shr(br, 8), wasm_u16x8_shr(ga, 8)}; + for (int i = 0; i < 4; ++i) { + ch[i] = + wasm_u16x8_shr(wasm_i16x8_add(wasm_i16x8_mul(ch[i], k17), two), 2); + } + ChannelsToUV8(ch[0], ch[1], ch[2], ch[3], k, dst_u, dst_v); + src_argb4444 += 32; + src_argb4444_1 += 32; + dst_u += 8; + dst_v += 8; + width -= 16; + } while (width > 0); +} +#endif // HAS_ARGB4444TOUVMATRIXROW_WASMSIMD + +#ifdef HAS_CONVERT8TO16ROW_WASMSIMD +// For bits 8 to 16, (y * 0x0101) >> (16 - bits) == p + (p >> 8) where +// p = y << (bits - 8). This avoids a variable shift count. +void Convert8To16Row_WASMSIMD(const uint8_t* src_y, + uint16_t* dst_y, + int bits, + int width) { + if (bits < 8) { + Convert8To16Row_C(src_y, dst_y, bits, width); + return; + } + v128_t scale = SplatConst16(1 << (bits - 8)); + do { + asm("" : "+r"(dst_y)); + v128_t y = wasm_v128_load(src_y); + v128_t y0 = wasm_i16x8_mul(wasm_u16x8_extend_low_u8x16(y), scale); + v128_t y1 = wasm_i16x8_mul(wasm_u16x8_extend_high_u8x16(y), scale); + wasm_v128_store(dst_y, wasm_i16x8_add(y0, wasm_u16x8_shr(y0, 8))); + wasm_v128_store(dst_y + 8, wasm_i16x8_add(y1, wasm_u16x8_shr(y1, 8))); + src_y += 16; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_CONVERT8TO16ROW_WASMSIMD + +#ifdef HAS_MULTIPLYROW_16_WASMSIMD +void MultiplyRow_16_WASMSIMD(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + v128_t vscale = wasm_i16x8_splat(scale); + do { + asm("" : "+r"(src_y), "+r"(dst_y)); + v128_t y0 = wasm_v128_load(src_y); + v128_t y1 = wasm_v128_load(src_y + 8); + y0 = wasm_i16x8_mul(y0, vscale); + y1 = wasm_i16x8_mul(y1, vscale); + wasm_v128_store(dst_y, y0); + wasm_v128_store(dst_y + 8, y1); + src_y += 16; + dst_y += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_MULTIPLYROW_16_WASMSIMD + +#ifdef HAS_MERGEUVROW_WASMSIMD +void MergeUVRow_WASMSIMD(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width) { + do { + asm("" : "+r"(dst_uv)); + v128_t u = wasm_v128_load(src_u); + v128_t v = wasm_v128_load(src_v); + v128_t uv0 = wasm_i8x16_shuffle(u, v, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + 21, 6, 22, 7, 23); + v128_t uv1 = wasm_i8x16_shuffle(u, v, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31); + wasm_v128_store(dst_uv, uv0); + wasm_v128_store(dst_uv + 16, uv1); + src_u += 16; + src_v += 16; + dst_uv += 32; + width -= 16; + } while (width > 0); +} +#endif // HAS_MERGEUVROW_WASMSIMD + +#ifdef HAS_MIRRORROW_WASMSIMD +static const uint8_t kShuffleMirror[16] = {15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0}; +void MirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { + v128_t shuf = LoadConst(kShuffleMirror); + src += width - 16; + do { + wasm_v128_store(dst, LIBYUV_WASM_SWIZZLE(wasm_v128_load(src), shuf)); + src -= 16; + dst += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_MIRRORROW_WASMSIMD + +#ifdef HAS_MIRRORUVROW_WASMSIMD +static const uint8_t kShuffleMirrorUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1}; +void MirrorUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + v128_t shuf = LoadConst(kShuffleMirrorUV); + src_uv += (width - 8) * 2; + do { + wasm_v128_store(dst_uv, LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_uv), shuf)); + src_uv -= 16; + dst_uv += 16; + width -= 8; + } while (width > 0); +} +#endif // HAS_MIRRORUVROW_WASMSIMD + +#ifdef HAS_SWAPUVROW_WASMSIMD +void SwapUVRow_WASMSIMD(const uint8_t* src_uv, uint8_t* dst_vu, int width) { + do { + // Keep constant offsets in the load/store instructions. + asm("" : "+r"(src_uv), "+r"(dst_vu)); + v128_t uv0 = wasm_v128_load(src_uv); + v128_t uv1 = wasm_v128_load(src_uv + 16); + // 16 bit byte swap: rev16 on Arm, shifts on x86. + wasm_v128_store( + dst_vu, wasm_v128_or(wasm_i16x8_shl(uv0, 8), wasm_u16x8_shr(uv0, 8))); + wasm_v128_store(dst_vu + 16, wasm_v128_or(wasm_i16x8_shl(uv1, 8), + wasm_u16x8_shr(uv1, 8))); + src_uv += 32; + dst_vu += 32; + width -= 16; + } while (width > 0); +} +#endif // HAS_SWAPUVROW_WASMSIMD + +#ifdef HAS_MIRRORSPLITUVROW_WASMSIMD +// Reverses 8 UV pairs into 8 U then 8 V. +static const uint8_t kShuffleMirrorSplitUV[16] = {14, 12, 10, 8, 6, 4, 2, 0, + 15, 13, 11, 9, 7, 5, 3, 1}; +void MirrorSplitUVRow_WASMSIMD(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + v128_t shuf = LoadConst(kShuffleMirrorSplitUV); + src_uv += (width - 16) * 2; + do { + asm("" : "+r"(src_uv)); + v128_t a = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_uv), shuf); + v128_t b = LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_uv + 16), shuf); + wasm_v128_store(dst_u, wasm_i64x2_shuffle(b, a, 0, 2)); + wasm_v128_store(dst_v, wasm_i64x2_shuffle(b, a, 1, 3)); + src_uv -= 32; + dst_u += 16; + dst_v += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_MIRRORSPLITUVROW_WASMSIMD + +#ifdef HAS_RGB24MIRRORROW_WASMSIMD +// Each 16 byte output needs 18 source bytes, so it is the OR of 2 swizzles. +static const uint8_t kShuffleRGB24Mirror[96] = { + 13, 14, 15, 10, 11, 12, 7, 8, 9, 4, 5, 6, 1, 2, + 3, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 14, 15, 128, 11, 12, 13, 8, 9, 10, 5, 6, + 7, 2, 3, 4, 128, 0, 128, 1, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 0, 128, 128, 12, 13, 14, 9, 10, + 11, 6, 7, 8, 3, 4, 5, 0, 1, 2, 1, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}; +void RGB24MirrorRow_WASMSIMD(const uint8_t* src_rgb24, + uint8_t* dst_rgb24, + int width) { + const uint8_t* shuf = kShuffleRGB24Mirror; + v128_t m0a = LoadConst(shuf); + v128_t m0b = LoadConst(shuf + 16); + v128_t m1a = LoadConst(shuf + 32); + v128_t m1b = LoadConst(shuf + 48); + v128_t m2a = LoadConst(shuf + 64); + v128_t m2b = LoadConst(shuf + 80); + src_rgb24 += (width - 16) * 3; + do { + asm("" : "+r"(src_rgb24), "+r"(dst_rgb24)); + v128_t s0 = wasm_v128_load(src_rgb24); + v128_t s1 = wasm_v128_load(src_rgb24 + 16); + v128_t s2 = wasm_v128_load(src_rgb24 + 32); + // Bytes 15 and 32 of the 48 source bytes. + v128_t s02 = wasm_i8x16_shuffle(s0, s2, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30); + wasm_v128_store(dst_rgb24, wasm_v128_or(LIBYUV_WASM_SWIZZLE(s2, m0a), + LIBYUV_WASM_SWIZZLE(s1, m0b))); + wasm_v128_store(dst_rgb24 + 16, + wasm_v128_or(LIBYUV_WASM_SWIZZLE(s1, m1a), + LIBYUV_WASM_SWIZZLE(s02, m1b))); + wasm_v128_store(dst_rgb24 + 32, wasm_v128_or(LIBYUV_WASM_SWIZZLE(s0, m2a), + LIBYUV_WASM_SWIZZLE(s1, m2b))); + src_rgb24 -= 48; + dst_rgb24 += 48; + width -= 16; + } while (width > 0); +} +#endif // HAS_RGB24MIRRORROW_WASMSIMD + +#ifdef HAS_ARGBMIRRORROW_WASMSIMD +void ARGBMirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width) { + src += (width - 4) * 4; + do { + v128_t v = wasm_v128_load(src); + v = wasm_i32x4_shuffle(v, v, 3, 2, 1, 0); + wasm_v128_store(dst, v); + src -= 16; + dst += 16; + width -= 4; + } while (width > 0); +} +#endif // HAS_ARGBMIRRORROW_WASMSIMD + +#ifdef HAS_INTERPOLATEROW_WASMSIMD +void InterpolateRow_WASMSIMD(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + const uint8_t* src_ptr1 = src_ptr + src_stride; + if (source_y_fraction == 0) { + do { + wasm_v128_store(dst_ptr, wasm_v128_load(src_ptr)); + src_ptr += 16; + dst_ptr += 16; + width -= 16; + } while (width > 0); + return; + } + if (source_y_fraction == 128) { + do { + v128_t s0 = wasm_v128_load(src_ptr); + v128_t s1 = wasm_v128_load(src_ptr1); + wasm_v128_store(dst_ptr, wasm_u8x16_avgr(s0, s1)); + src_ptr += 16; + src_ptr1 += 16; + dst_ptr += 16; + width -= 16; + } while (width > 0); + return; + } + // s0 * (256 - f) + s1 * f + 128 fits in 16 bits. Both weights fit in 8 + // bits since f is 1 to 255. The high bytes are the result (uzp2 on Arm). + v128_t w0 = SplatConst8(256 - source_y_fraction); + v128_t w1 = SplatConst8(source_y_fraction); + v128_t round = SplatConst16(128); + do { + v128_t s0 = wasm_v128_load(src_ptr); + v128_t s1 = wasm_v128_load(src_ptr1); + v128_t lo = wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(s0, w0), + wasm_u16x8_extmul_low_u8x16(s1, w1)); + v128_t hi = wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(s0, w0), + wasm_u16x8_extmul_high_u8x16(s1, w1)); + lo = wasm_i16x8_add(lo, round); + hi = wasm_i16x8_add(hi, round); + wasm_v128_store( + dst_ptr, wasm_i8x16_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, + 21, 23, 25, 27, 29, 31)); + src_ptr += 16; + src_ptr1 += 16; + dst_ptr += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_INTERPOLATEROW_WASMSIMD + +#ifdef HAS_INTERPOLATEROW_16_WASMSIMD +void InterpolateRow_16_WASMSIMD(uint16_t* dst_ptr, + const uint16_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + const uint16_t* src_ptr1 = src_ptr + src_stride; + if (source_y_fraction == 0) { + do { + wasm_v128_store(dst_ptr, wasm_v128_load(src_ptr)); + src_ptr += 8; + dst_ptr += 8; + width -= 8; + } while (width > 0); + return; + } + if (source_y_fraction == 128) { + do { + v128_t s0 = wasm_v128_load(src_ptr); + v128_t s1 = wasm_v128_load(src_ptr1); + wasm_v128_store(dst_ptr, wasm_u16x8_avgr(s0, s1)); + src_ptr += 8; + src_ptr1 += 8; + dst_ptr += 8; + width -= 8; + } while (width > 0); + return; + } + // s0 * (256 - f) + s1 * f + 128 fits in 24 bits. Shift the result into + // the high 16 bits of each lane, then take the odd 16 bit lanes. + v128_t w0 = SplatConst16(256 - source_y_fraction); + v128_t w1 = SplatConst16(source_y_fraction); + v128_t round = SplatConst32(128); + do { + v128_t s0 = wasm_v128_load(src_ptr); + v128_t s1 = wasm_v128_load(src_ptr1); + v128_t lo = wasm_i32x4_add(wasm_u32x4_extmul_low_u16x8(s0, w0), + wasm_u32x4_extmul_low_u16x8(s1, w1)); + v128_t hi = wasm_i32x4_add(wasm_u32x4_extmul_high_u16x8(s0, w0), + wasm_u32x4_extmul_high_u16x8(s1, w1)); + lo = wasm_i32x4_shl(wasm_i32x4_add(lo, round), 8); + hi = wasm_i32x4_shl(wasm_i32x4_add(hi, round), 8); + wasm_v128_store(dst_ptr, + wasm_i16x8_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15)); + src_ptr += 8; + src_ptr1 += 8; + dst_ptr += 8; + width -= 8; + } while (width > 0); +} +#endif // HAS_INTERPOLATEROW_16_WASMSIMD + +#ifdef HAS_J400TOARGBROW_WASMSIMD +static const uint8_t kShuffleJ400ToARGB[64] = { + 0, 0, 0, 128, 1, 1, 1, 128, 2, 2, 2, 128, 3, 3, 3, 128, + 4, 4, 4, 128, 5, 5, 5, 128, 6, 6, 6, 128, 7, 7, 7, 128, + 8, 8, 8, 128, 9, 9, 9, 128, 10, 10, 10, 128, 11, 11, 11, 128, + 12, 12, 12, 128, 13, 13, 13, 128, 14, 14, 14, 128, 15, 15, 15, 128}; +void J400ToARGBRow_WASMSIMD(const uint8_t* src_y, + uint8_t* dst_argb, + int width) { + v128_t shuf0 = LoadConst(kShuffleJ400ToARGB); + v128_t shuf1 = LoadConst(kShuffleJ400ToARGB + 16); + v128_t shuf2 = LoadConst(kShuffleJ400ToARGB + 32); + v128_t shuf3 = LoadConst(kShuffleJ400ToARGB + 48); + v128_t alpha = SplatConst32((int)0xff000000); + do { + asm("" : "+r"(dst_argb)); + v128_t y = wasm_v128_load(src_y); + wasm_v128_store(dst_argb, + wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf0), alpha)); + wasm_v128_store(dst_argb + 16, + wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf1), alpha)); + wasm_v128_store(dst_argb + 32, + wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf2), alpha)); + wasm_v128_store(dst_argb + 48, + wasm_v128_or(LIBYUV_WASM_SWIZZLE(y, shuf3), alpha)); + src_y += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_J400TOARGBROW_WASMSIMD + +#ifdef HAS_RGB565TOARGBROW_WASMSIMD +void RGB565ToARGBRow_WASMSIMD(const uint8_t* src_rgb565, + uint8_t* dst_argb, + int width) { + v128_t mask_b5 = SplatConst16(0x001f); + v128_t mask_g6 = SplatConst16(0x07e0); + v128_t mask_r5 = SplatConst16(0xf800); + v128_t mask_hi = SplatConst16(0xff00); + do { + asm("" : "+r"(dst_argb)); + v128_t p = wasm_v128_load(src_rgb565); + v128_t b5 = wasm_v128_and(p, mask_b5); + v128_t b8 = wasm_v128_or(wasm_i16x8_shl(b5, 3), wasm_u16x8_shr(b5, 2)); + v128_t g6 = wasm_v128_and(p, mask_g6); + v128_t g8_hi = wasm_v128_and( + wasm_v128_or(wasm_i16x8_shl(g6, 5), wasm_u16x8_shr(g6, 1)), mask_hi); + v128_t bg = wasm_v128_or(b8, g8_hi); + + v128_t r8 = wasm_v128_or(wasm_u16x8_shr(wasm_v128_and(p, mask_r5), 8), + wasm_u16x8_shr(p, 13)); + v128_t ra = wasm_v128_or(r8, mask_hi); + + v128_t d0 = wasm_i16x8_shuffle(bg, ra, 0, 8, 1, 9, 2, 10, 3, 11); + v128_t d1 = wasm_i16x8_shuffle(bg, ra, 4, 12, 5, 13, 6, 14, 7, 15); + wasm_v128_store(dst_argb, d0); + wasm_v128_store(dst_argb + 16, d1); + src_rgb565 += 16; + dst_argb += 32; + width -= 8; + } while (width > 0); +} +#endif // HAS_RGB565TOARGBROW_WASMSIMD + +#ifdef HAS_ARGB1555TOARGBROW_WASMSIMD +// 5 to 8 bit replication with multiplies: (v * 0x108) >> 5 == v << 3 | v >> 2. +void ARGB1555ToARGBRow_WASMSIMD(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width) { + v128_t mask_b5 = SplatConst16(0x001f); + v128_t mask_g5 = SplatConst16(0x03e0); + v128_t mask_hi = SplatConst16(0xff00); + v128_t mul_br = SplatConst16(0x0108); + v128_t mul_g = SplatConst16(0x0042); + do { + asm("" : "+r"(dst_argb)); + v128_t p = wasm_v128_load(src_argb1555); + v128_t b8 = + wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(p, mask_b5), mul_br), 5); + // g5 << 5 times 0x42 is g5 << 11 | g5 << 6; the high byte is g8. + v128_t g8_hi = wasm_v128_and( + wasm_i16x8_mul(wasm_v128_and(p, mask_g5), mul_g), mask_hi); + v128_t bg = wasm_v128_or(b8, g8_hi); + v128_t r5 = wasm_u16x8_shr(wasm_i16x8_shl(p, 1), 11); + v128_t r8 = wasm_u16x8_shr(wasm_i16x8_mul(r5, mul_br), 5); + v128_t a8_hi = wasm_v128_and(wasm_i16x8_shr(p, 15), mask_hi); + v128_t ra = wasm_v128_or(r8, a8_hi); + + v128_t d0 = wasm_i16x8_shuffle(bg, ra, 0, 8, 1, 9, 2, 10, 3, 11); + v128_t d1 = wasm_i16x8_shuffle(bg, ra, 4, 12, 5, 13, 6, 14, 7, 15); + wasm_v128_store(dst_argb, d0); + wasm_v128_store(dst_argb + 16, d1); + src_argb1555 += 16; + dst_argb += 32; + width -= 8; + } while (width > 0); +} +#endif // HAS_ARGB1555TOARGBROW_WASMSIMD + +#ifdef HAS_ARGB4444TOARGBROW_WASMSIMD +void ARGB4444ToARGBRow_WASMSIMD(const uint8_t* src_argb4444, + uint8_t* dst_argb, + int width) { + v128_t mask_lo = SplatConst8(0x0f); + v128_t mask_hi = SplatConst8(0xf0); + do { + asm("" : "+r"(dst_argb)); + v128_t v = wasm_v128_load(src_argb4444); + v128_t v_lo = wasm_v128_and(v, mask_lo); + v_lo = wasm_v128_or(v_lo, wasm_i16x8_shl(v_lo, 4)); + v128_t v_hi = wasm_v128_and(v, mask_hi); + v_hi = wasm_v128_or(v_hi, wasm_u16x8_shr(v_hi, 4)); + + v128_t d0 = wasm_i8x16_shuffle(v_lo, v_hi, 0, 16, 1, 17, 2, 18, 3, 19, 4, + 20, 5, 21, 6, 22, 7, 23); + v128_t d1 = wasm_i8x16_shuffle(v_lo, v_hi, 8, 24, 9, 25, 10, 26, 11, 27, 12, + 28, 13, 29, 14, 30, 15, 31); + wasm_v128_store(dst_argb, d0); + wasm_v128_store(dst_argb + 16, d1); + src_argb4444 += 16; + dst_argb += 32; + width -= 8; + } while (width > 0); +} +#endif // HAS_ARGB4444TOARGBROW_WASMSIMD + +#ifdef HAS_ARGBTORGB565DITHERROW_WASMSIMD +void ARGBToRGB565DitherRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width) { + v128_t dither = wasm_i32x4_splat((int32_t)dither4); + dither = wasm_i8x16_shuffle(dither, dither, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, + 2, 3, 3, 3, 3); + v128_t mask_b = SplatConst32(0x0000001f); + v128_t mask_g = SplatConst32(0x000007e0); + v128_t mask_r = SplatConst32(0x0000f800); + do { + asm("" : "+r"(src_argb)); + v128_t s0 = wasm_u8x16_add_sat(wasm_v128_load(src_argb), dither); + v128_t s1 = wasm_u8x16_add_sat(wasm_v128_load(src_argb + 16), dither); + v128_t b0 = wasm_v128_and(wasm_u32x4_shr(s0, 3), mask_b); + v128_t g0 = wasm_v128_and(wasm_u32x4_shr(s0, 5), mask_g); + v128_t r0 = wasm_v128_and(wasm_u32x4_shr(s0, 8), mask_r); + v128_t b1 = wasm_v128_and(wasm_u32x4_shr(s1, 3), mask_b); + v128_t g1 = wasm_v128_and(wasm_u32x4_shr(s1, 5), mask_g); + v128_t r1 = wasm_v128_and(wasm_u32x4_shr(s1, 8), mask_r); + v128_t p0 = wasm_v128_or(wasm_v128_or(b0, g0), r0); + v128_t p1 = wasm_v128_or(wasm_v128_or(b1, g1), r1); + wasm_v128_store(dst_rgb, wasm_u16x8_narrow_i32x4(p0, p1)); + src_argb += 32; + dst_rgb += 16; + width -= 8; + } while (width > 0); +} +#endif // HAS_ARGBTORGB565DITHERROW_WASMSIMD + +#ifdef HAS_ARGBTORGB565ROW_WASMSIMD +void ARGBToRGB565Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + v128_t mask_b = SplatConst32(0x0000001f); + v128_t mask_g = SplatConst32(0x000007e0); + v128_t mask_r = SplatConst32(0x0000f800); + do { + asm("" : "+r"(src_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + v128_t b0 = wasm_v128_and(wasm_u32x4_shr(s0, 3), mask_b); + v128_t g0 = wasm_v128_and(wasm_u32x4_shr(s0, 5), mask_g); + v128_t r0 = wasm_v128_and(wasm_u32x4_shr(s0, 8), mask_r); + v128_t b1 = wasm_v128_and(wasm_u32x4_shr(s1, 3), mask_b); + v128_t g1 = wasm_v128_and(wasm_u32x4_shr(s1, 5), mask_g); + v128_t r1 = wasm_v128_and(wasm_u32x4_shr(s1, 8), mask_r); + v128_t p0 = wasm_v128_or(wasm_v128_or(b0, g0), r0); + v128_t p1 = wasm_v128_or(wasm_v128_or(b1, g1), r1); + wasm_v128_store(dst_rgb, wasm_u16x8_narrow_i32x4(p0, p1)); + src_argb += 32; + dst_rgb += 16; + width -= 8; + } while (width > 0); +} +#endif // HAS_ARGBTORGB565ROW_WASMSIMD + +#ifdef HAS_ARGBTOARGB1555ROW_WASMSIMD +void ARGBToARGB1555Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb1555, + int width) { + v128_t mask_b = SplatConst32(0x0000001f); + v128_t mask_g = SplatConst32(0x000003e0); + v128_t mask_r = SplatConst32(0x00007c00); + v128_t mask_a = SplatConst32(0x00008000); + do { + asm("" : "+r"(src_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + v128_t b0 = wasm_v128_and(wasm_u32x4_shr(s0, 3), mask_b); + v128_t g0 = wasm_v128_and(wasm_u32x4_shr(s0, 6), mask_g); + v128_t r0 = wasm_v128_and(wasm_u32x4_shr(s0, 9), mask_r); + v128_t a0 = wasm_v128_and(wasm_u32x4_shr(s0, 16), mask_a); + v128_t b1 = wasm_v128_and(wasm_u32x4_shr(s1, 3), mask_b); + v128_t g1 = wasm_v128_and(wasm_u32x4_shr(s1, 6), mask_g); + v128_t r1 = wasm_v128_and(wasm_u32x4_shr(s1, 9), mask_r); + v128_t a1 = wasm_v128_and(wasm_u32x4_shr(s1, 16), mask_a); + v128_t p0 = wasm_v128_or(wasm_v128_or(b0, g0), wasm_v128_or(r0, a0)); + v128_t p1 = wasm_v128_or(wasm_v128_or(b1, g1), wasm_v128_or(r1, a1)); + wasm_v128_store(dst_argb1555, wasm_u16x8_narrow_i32x4(p0, p1)); + src_argb += 32; + dst_argb1555 += 16; + width -= 8; + } while (width > 0); +} +#endif // HAS_ARGBTOARGB1555ROW_WASMSIMD + +#ifdef HAS_ARGBTOARGB4444ROW_WASMSIMD +void ARGBToARGB4444Row_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb4444, + int width) { + v128_t mask_lo = SplatConst16(0x000f); + v128_t mask_hi = SplatConst16(0x00f0); + do { + asm("" : "+r"(src_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + v128_t n0 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(s0, 4), mask_lo), + wasm_v128_and(wasm_u16x8_shr(s0, 8), mask_hi)); + v128_t n1 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(s1, 4), mask_lo), + wasm_v128_and(wasm_u16x8_shr(s1, 8), mask_hi)); + wasm_v128_store(dst_argb4444, wasm_u8x16_narrow_i16x8(n0, n1)); + src_argb += 32; + dst_argb4444 += 16; + width -= 8; + } while (width > 0); +} +#endif // HAS_ARGBTOARGB4444ROW_WASMSIMD + +#ifdef HAS_ARGBSHUFFLEROW_WASMSIMD +void ARGBShuffleRow_WASMSIMD(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width) { + v128_t shuf = wasm_v128_load(shuffler); + while (width >= 8) { + asm("" : "+r"(src_argb), "+r"(dst_argb)); + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb + 16); + wasm_v128_store(dst_argb, LIBYUV_WASM_SWIZZLE(s0, shuf)); + wasm_v128_store(dst_argb + 16, LIBYUV_WASM_SWIZZLE(s1, shuf)); + src_argb += 32; + dst_argb += 32; + width -= 8; + } + if (width > 0) { + v128_t s0 = wasm_v128_load(src_argb); + wasm_v128_store(dst_argb, LIBYUV_WASM_SWIZZLE(s0, shuf)); + } +} +#endif // HAS_ARGBSHUFFLEROW_WASMSIMD + +#if defined(HAS_I422TOARGBROW_WASMSIMD) || \ + defined(HAS_I422TOAR30ROW_WASMSIMD) || \ + defined(HAS_NV12TOARGBROW_WASMSIMD) || \ + defined(HAS_NV21TOARGBROW_WASMSIMD) || \ + defined(HAS_I422TORGB24ROW_WASMSIMD) || \ + defined(HAS_I422TORGB565ROW_WASMSIMD) || \ + defined(HAS_I422TOARGB1555ROW_WASMSIMD) || \ + defined(HAS_I422TOARGB4444ROW_WASMSIMD) || \ + defined(HAS_NV12TORGB565ROW_WASMSIMD) +// Interleaves map to punpck on x64 and zip on arm64 when the two inputs +// differ. V8 arm64 lowers shuffle(x, x) to tbl with a rebuilt mask. +static __inline v128_t ZipLo8_WASMSIMD(v128_t a, v128_t b) { + return wasm_i8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, + 22, 7, 23); +} +static __inline v128_t ZipHi8_WASMSIMD(v128_t a, v128_t b) { + return wasm_i8x16_shuffle(a, b, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, + 14, 30, 15, 31); +} + +// YuvConstants splatted to 16-bit lanes, built once per row. +struct YuvCoefs_WASMSIMD { + v128_t ub, ug, vg, vr, yg, yb, x80, k00ff, k0101; +}; + +static __inline void LoadYuvCoefs_WASMSIMD(const struct YuvConstants* yc, + int yb_adjust, + struct YuvCoefs_WASMSIMD* c) { + c->ub = wasm_i16x8_splat(yc->kUVToB[0]); + c->ug = wasm_i16x8_splat(yc->kUVToG[0]); + c->vg = wasm_i16x8_splat(yc->kUVToG[1]); + c->vr = wasm_i16x8_splat(yc->kUVToR[1]); + c->yg = wasm_i16x8_splat(yc->kYToRgb[0]); + c->yb = wasm_i16x8_splat(yc->kYBiasToRgb[0] - yb_adjust); + c->x80 = SplatConst8(0x80); + c->k00ff = SplatConst16(0x00ff); + c->k0101 = SplatConst16(0x0101); +} + +// Returns ((y * 0x0101 * yg) >> 16) + yb for 8 Y in 16-bit lanes. +static __inline v128_t YToRgb_WASMSIMD(v128_t y, + const struct YuvCoefs_WASMSIMD* c) { + // The barrier keeps clang from widening yg outside the loop, which would + // turn extmul (umull on arm64) into extend + i32x4.mul. + v128_t yg = c->yg; + asm volatile("" : "+r"(yg)); + v128_t y16 = wasm_i16x8_mul(y, c->k0101); + v128_t lo = wasm_u32x4_extmul_low_u16x8(y16, yg); + v128_t hi = wasm_u32x4_extmul_high_u16x8(y16, yg); + return wasm_i16x8_add(wasm_i16x8_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15), + c->yb); +} + +// Converts 16 pixels to B, G, R with 6 fractional bits (unclamped int16). +// y = 16 Y. uv = 8 U/V pairs, U in the low byte of each 16-bit lane (V if vu). +// Even and odd pixels are kept apart so each pair shares its chroma terms: +// rgb[0], rgb[2], rgb[4] = even B, G, R and rgb[1], rgb[3], rgb[5] = odd. +static __inline void YuvToRgb16_WASMSIMD(v128_t y, + v128_t uv, + int vu, + const struct YuvCoefs_WASMSIMD* c, + v128_t* rgb) { + v128_t ye = YToRgb_WASMSIMD(wasm_v128_and(y, c->k00ff), c); + v128_t yo = YToRgb_WASMSIMD(wasm_u16x8_shr(y, 8), c); + // u - 128 and v - 128 as int8, sign extended to int16. + v128_t uvs = wasm_v128_xor(uv, c->x80); + v128_t lo = wasm_i16x8_shr(wasm_i16x8_shl(uvs, 8), 8); + v128_t hi = wasm_i16x8_shr(uvs, 8); + v128_t ui = vu ? hi : lo; + v128_t vi = vu ? lo : hi; + v128_t b = wasm_i16x8_mul(ui, c->ub); + v128_t g = + wasm_i16x8_add(wasm_i16x8_mul(ui, c->ug), wasm_i16x8_mul(vi, c->vg)); + v128_t r = wasm_i16x8_mul(vi, c->vr); + rgb[0] = wasm_i16x8_add_sat(ye, b); + rgb[1] = wasm_i16x8_add_sat(yo, b); + rgb[2] = wasm_i16x8_sub_sat(ye, g); + rgb[3] = wasm_i16x8_sub_sat(yo, g); + rgb[4] = wasm_i16x8_add_sat(ye, r); + rgb[5] = wasm_i16x8_add_sat(yo, r); +} + +// Loads 16 Y and 8 U/V, or 8 Y and 4 U/V when width < 16. +static __inline void ReadI422_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + int width, + v128_t* y, + v128_t* uv) { + if (width >= 16) { + *y = wasm_v128_load(src_y); + *uv = ZipLo8_WASMSIMD(wasm_v128_load64_zero(src_u), + wasm_v128_load64_zero(src_v)); + } else { + *y = wasm_v128_load64_zero(src_y); + *uv = ZipLo8_WASMSIMD(wasm_v128_load32_zero(src_u), + wasm_v128_load32_zero(src_v)); + } +} + +// Loads 16 Y and 8 UV pairs, or 8 Y and 4 UV pairs when width < 16. +static __inline void ReadNV12_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + int width, + v128_t* y, + v128_t* uv) { + if (width >= 16) { + *y = wasm_v128_load(src_y); + *uv = wasm_v128_load(src_uv); + } else { + *y = wasm_v128_load64_zero(src_y); + *uv = wasm_v128_load64_zero(src_uv); + } +} +#endif + +#if defined(HAS_I422TOARGBROW_WASMSIMD) || \ + defined(HAS_NV12TOARGBROW_WASMSIMD) || defined(HAS_NV21TOARGBROW_WASMSIMD) +// Clamps even/odd B, G, R to 8 bits and stores 16 ARGB pixels, or 8 when +// width < 16. +static __inline void StoreARGB_WASMSIMD(const v128_t* rgb, + v128_t ff, + uint8_t* dst_argb, + int width) { + v128_t bg_e = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[0], 6), + wasm_i16x8_shr(rgb[2], 6)); + v128_t gb_o = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[3], 6), + wasm_i16x8_shr(rgb[1], 6)); + v128_t rr = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[4], 6), + wasm_i16x8_shr(rgb[5], 6)); + v128_t br_e = ZipLo8_WASMSIMD(bg_e, rr); + v128_t ga_e = ZipHi8_WASMSIMD(bg_e, ff); + v128_t br_o = ZipHi8_WASMSIMD(gb_o, rr); + v128_t ga_o = ZipLo8_WASMSIMD(gb_o, ff); + v128_t e = ZipLo8_WASMSIMD(br_e, ga_e); // Pixels 0, 2, 4, 6. + v128_t o = ZipLo8_WASMSIMD(br_o, ga_o); // Pixels 1, 3, 5, 7. + wasm_v128_store(dst_argb, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); + wasm_v128_store(dst_argb + 16, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); + if (width >= 16) { + e = ZipHi8_WASMSIMD(br_e, ga_e); + o = ZipHi8_WASMSIMD(br_o, ga_o); + wasm_v128_store(dst_argb + 32, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); + wasm_v128_store(dst_argb + 48, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); + } +} +#endif + +#ifdef HAS_I422TOARGBROW_WASMSIMD +void I422ToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t ff = SplatConst8(0xff); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + StoreARGB_WASMSIMD(rgb, ff, dst_argb, width); + src_y += 16; + src_u += 8; + src_v += 8; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422TOARGBROW_WASMSIMD + +#ifdef HAS_I422TOAR30ROW_WASMSIMD +// Clamps B, G, R with 6 fractional bits to 10 bits and packs them into the +// low and high 16 bits of AR30, without the alpha bits. +static __inline void RgbToAR30_WASMSIMD(v128_t b, + v128_t g, + v128_t r, + v128_t zero, + v128_t k3ff0, + v128_t* lo, + v128_t* hi) { + b = wasm_i16x8_max(wasm_i16x8_min(b, k3ff0), zero); + g = wasm_i16x8_max(wasm_i16x8_min(g, k3ff0), zero); + r = wasm_i16x8_max(wasm_i16x8_min(r, k3ff0), zero); + *lo = wasm_v128_or(wasm_u16x8_shr(b, 4), + wasm_i16x8_shl(wasm_v128_and(g, k3ff0), 6)); + *hi = wasm_v128_or(wasm_v128_and(r, k3ff0), wasm_u16x8_shr(g, 10)); +} + +void I422ToAR30Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t zero = SplatConst16(0); + v128_t k3ff0 = SplatConst16(0x3ff0); + v128_t kc000 = SplatConst16(0xc000); + LoadYuvCoefs_WASMSIMD(yuvconstants, 24, &c); // AR30 bias. + do { + v128_t y, uv, rgb[6], lo_e, hi_e, lo_o, hi_o; + ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + RgbToAR30_WASMSIMD(rgb[0], rgb[2], rgb[4], zero, k3ff0, &lo_e, &hi_e); + RgbToAR30_WASMSIMD(rgb[1], rgb[3], rgb[5], zero, k3ff0, &lo_o, &hi_o); + hi_e = wasm_v128_or(hi_e, kc000); + hi_o = wasm_v128_or(hi_o, kc000); + // Even pixels 0, 2, 4, 6 and odd pixels 1, 3, 5, 7 as 32 bits. + v128_t e = wasm_i16x8_shuffle(lo_e, hi_e, 0, 8, 1, 9, 2, 10, 3, 11); + v128_t o = wasm_i16x8_shuffle(lo_o, hi_o, 0, 8, 1, 9, 2, 10, 3, 11); + wasm_v128_store(dst_ar30, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); + wasm_v128_store(dst_ar30 + 16, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); + if (width >= 16) { + e = wasm_i16x8_shuffle(lo_e, hi_e, 4, 12, 5, 13, 6, 14, 7, 15); + o = wasm_i16x8_shuffle(lo_o, hi_o, 4, 12, 5, 13, 6, 14, 7, 15); + wasm_v128_store(dst_ar30 + 32, wasm_i32x4_shuffle(e, o, 0, 4, 1, 5)); + wasm_v128_store(dst_ar30 + 48, wasm_i32x4_shuffle(e, o, 2, 6, 3, 7)); + } + src_y += 16; + src_u += 8; + src_v += 8; + dst_ar30 += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422TOAR30ROW_WASMSIMD + +#ifdef HAS_NV12TOARGBROW_WASMSIMD +void NV12ToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t ff = SplatConst8(0xff); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadNV12_WASMSIMD(src_y, src_uv, width, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + StoreARGB_WASMSIMD(rgb, ff, dst_argb, width); + src_y += 16; + src_uv += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_NV12TOARGBROW_WASMSIMD + +#ifdef HAS_NV21TOARGBROW_WASMSIMD +void NV21ToARGBRow_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t ff = SplatConst8(0xff); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, vu, rgb[6]; + ReadNV12_WASMSIMD(src_y, src_vu, width, &y, &vu); + YuvToRgb16_WASMSIMD(y, vu, 1, &c, rgb); + StoreARGB_WASMSIMD(rgb, ff, dst_argb, width); + src_y += 16; + src_vu += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); +} +#endif // HAS_NV21TOARGBROW_WASMSIMD + +#ifdef HAS_I422TORGB24ROW_WASMSIMD +// Swizzles that interleave B/G pairs and R into 3 vectors of RGB24. R is +// 8 even pixels followed by 8 odd pixels. +static const uint8_t kI422ToRGB24Swizzle[96] = { + 0, 1, 128, 2, 3, 128, 4, 5, 128, 6, 7, 128, 8, 9, + 128, 10, 128, 128, 0, 128, 128, 8, 128, 128, 1, 128, 128, 9, + 128, 128, 2, 128, 3, 128, 4, 5, 128, 6, 7, 128, 8, 9, + 128, 10, 11, 128, 12, 13, 128, 10, 128, 128, 3, 128, 128, 11, + 128, 128, 4, 128, 128, 12, 128, 128, 128, 6, 7, 128, 8, 9, + 128, 10, 11, 128, 12, 13, 128, 14, 15, 128, 5, 128, 128, 13, + 128, 128, 6, 128, 128, 14, 128, 128, 7, 128, 128, 15}; + +void I422ToRGB24Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t m0 = LoadConst(kI422ToRGB24Swizzle); + v128_t m1 = LoadConst(kI422ToRGB24Swizzle + 16); + v128_t m2 = LoadConst(kI422ToRGB24Swizzle + 32); + v128_t m3 = LoadConst(kI422ToRGB24Swizzle + 48); + v128_t m4 = LoadConst(kI422ToRGB24Swizzle + 64); + v128_t m5 = LoadConst(kI422ToRGB24Swizzle + 80); + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadI422_WASMSIMD(src_y, src_u, src_v, 16, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + v128_t bg_e = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[0], 6), + wasm_i16x8_shr(rgb[2], 6)); + v128_t bg_o = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[1], 6), + wasm_i16x8_shr(rgb[3], 6)); + v128_t r = wasm_u8x16_narrow_i16x8(wasm_i16x8_shr(rgb[4], 6), + wasm_i16x8_shr(rgb[5], 6)); + v128_t b = ZipLo8_WASMSIMD(bg_e, bg_o); + v128_t g = ZipHi8_WASMSIMD(bg_e, bg_o); + v128_t bg0 = ZipLo8_WASMSIMD(b, g); + v128_t bg2 = ZipHi8_WASMSIMD(b, g); + v128_t bg1 = wasm_i8x16_shuffle(bg0, bg2, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23); + wasm_v128_store(dst_rgb24, wasm_v128_or(LIBYUV_WASM_SWIZZLE(bg0, m0), + LIBYUV_WASM_SWIZZLE(r, m1))); + wasm_v128_store(dst_rgb24 + 16, wasm_v128_or(LIBYUV_WASM_SWIZZLE(bg1, m2), + LIBYUV_WASM_SWIZZLE(r, m3))); + wasm_v128_store(dst_rgb24 + 32, wasm_v128_or(LIBYUV_WASM_SWIZZLE(bg2, m4), + LIBYUV_WASM_SWIZZLE(r, m5))); + src_y += 16; + src_u += 8; + src_v += 8; + dst_rgb24 += 48; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422TORGB24ROW_WASMSIMD + +#if defined(HAS_I422TORGB565ROW_WASMSIMD) || \ + defined(HAS_I422TOARGB1555ROW_WASMSIMD) || \ + defined(HAS_I422TOARGB4444ROW_WASMSIMD) || \ + defined(HAS_NV12TORGB565ROW_WASMSIMD) +// Clamps x with 6 fractional bits to 8 bits, keeps the top 'bits' and moves +// them to bit 'pos'. +static __inline v128_t PackChannel_WASMSIMD(v128_t x, + v128_t zero, + v128_t k3fff, + int bits, + int pos) { + x = wasm_i16x8_max(wasm_i16x8_min(x, k3fff), zero); + return wasm_i16x8_shl(wasm_u16x8_shr(x, 14 - bits), pos); +} + +// Packs even/odd B, G, R into 16 bit pixels with the given channel sizes and +// stores 16 pixels, or 8 when width < 16. k = {0, 0x3fff, alpha} in 16-bit +// lanes. Alpha is set if the channels use fewer than 16 bits. +static __inline void Store16Bit_WASMSIMD(const v128_t* rgb, + int bbits, + int gbits, + int rbits, + const v128_t* k, + uint8_t* dst, + int width) { + v128_t p[2]; + int i; + for (i = 0; i < 2; ++i) { + p[i] = wasm_v128_or( + PackChannel_WASMSIMD(rgb[i], k[0], k[1], bbits, 0), + wasm_v128_or(PackChannel_WASMSIMD(rgb[2 + i], k[0], k[1], gbits, bbits), + PackChannel_WASMSIMD(rgb[4 + i], k[0], k[1], rbits, + bbits + gbits))); + if (bbits + gbits + rbits < 16) { + p[i] = wasm_v128_or(p[i], k[2]); + } + } + wasm_v128_store(dst, + wasm_i16x8_shuffle(p[0], p[1], 0, 8, 1, 9, 2, 10, 3, 11)); + if (width >= 16) { + wasm_v128_store(dst + 16, + wasm_i16x8_shuffle(p[0], p[1], 4, 12, 5, 13, 6, 14, 7, 15)); + } +} +#endif + +#ifdef HAS_I422TORGB565ROW_WASMSIMD +void I422ToRGB565Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb565, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t k[2] = {SplatConst16(0), SplatConst16(0x3fff)}; + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + Store16Bit_WASMSIMD(rgb, 5, 6, 5, k, dst_rgb565, width); + src_y += 16; + src_u += 8; + src_v += 8; + dst_rgb565 += 32; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422TORGB565ROW_WASMSIMD + +#ifdef HAS_I422TOARGB1555ROW_WASMSIMD +void I422ToARGB1555Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb1555, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t k[3] = {SplatConst16(0), SplatConst16(0x3fff), SplatConst16(0x8000)}; + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + Store16Bit_WASMSIMD(rgb, 5, 5, 5, k, dst_argb1555, width); + src_y += 16; + src_u += 8; + src_v += 8; + dst_argb1555 += 32; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422TOARGB1555ROW_WASMSIMD + +#ifdef HAS_I422TOARGB4444ROW_WASMSIMD +void I422ToARGB4444Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb4444, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t k[3] = {SplatConst16(0), SplatConst16(0x3fff), SplatConst16(0xf000)}; + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadI422_WASMSIMD(src_y, src_u, src_v, width, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + Store16Bit_WASMSIMD(rgb, 4, 4, 4, k, dst_argb4444, width); + src_y += 16; + src_u += 8; + src_v += 8; + dst_argb4444 += 32; + width -= 16; + } while (width > 0); +} +#endif // HAS_I422TOARGB4444ROW_WASMSIMD + +#ifdef HAS_NV12TORGB565ROW_WASMSIMD +void NV12ToRGB565Row_WASMSIMD(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_rgb565, + const struct YuvConstants* yuvconstants, + int width) { + struct YuvCoefs_WASMSIMD c; + v128_t k[2] = {SplatConst16(0), SplatConst16(0x3fff)}; + LoadYuvCoefs_WASMSIMD(yuvconstants, 0, &c); + do { + v128_t y, uv, rgb[6]; + ReadNV12_WASMSIMD(src_y, src_uv, width, &y, &uv); + YuvToRgb16_WASMSIMD(y, uv, 0, &c, rgb); + Store16Bit_WASMSIMD(rgb, 5, 6, 5, k, dst_rgb565, width); + src_y += 16; + src_uv += 16; + dst_rgb565 += 32; + width -= 16; + } while (width > 0); +} +#endif // HAS_NV12TORGB565ROW_WASMSIMD + +#ifdef HAS_ARGBBLENDROW_WASMSIMD +// Broadcast 255 - alpha (from byte 3 of each pixel) to 16 bit lanes. +static const uint8_t kShuffleAlpha16[16] = {3, 128, 3, 128, 7, 128, 7, 128, + 11, 128, 11, 128, 15, 128, 15, 128}; +void ARGBBlendRow_WASMSIMD(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + v128_t shuf_a = LoadConst(kShuffleAlpha16); + v128_t one = SplatConst16(1); + v128_t mask_lo = SplatConst16(0x00ff); + v128_t mask_alpha = SplatConst32((int)0xff000000); + + while (width >= 4) { + v128_t s0 = wasm_v128_load(src_argb); + v128_t s1 = wasm_v128_load(src_argb1); + src_argb += 16; + src_argb1 += 16; + + // a = 256 - alpha. + v128_t a = wasm_i16x8_add( + LIBYUV_WASM_SWIZZLE(wasm_v128_xor(s0, mask_alpha), shuf_a), one); + v128_t s1_lo = + wasm_u16x8_shr(wasm_i16x8_mul(wasm_v128_and(s1, mask_lo), a), 8); + v128_t s1_hi = + wasm_v128_andnot(wasm_i16x8_mul(wasm_u16x8_shr(s1, 8), a), mask_lo); + + v128_t res = wasm_v128_or(s0, mask_alpha); + res = wasm_u8x16_add_sat(res, s1_lo); + res = wasm_u8x16_add_sat(res, s1_hi); + + wasm_v128_store(dst_argb, res); + dst_argb += 16; + width -= 4; + } + + while (width > 0) { + uint32_t s0 = *(const uint32_t*)src_argb; + uint32_t s1 = *(const uint32_t*)src_argb1; + src_argb += 4; + src_argb1 += 4; + + uint32_t a = 256 - (s0 >> 24); + uint32_t b = (((s1 & 0x000000ff) * a) >> 8) + (s0 & 0x000000ff); + uint32_t g = (((s1 & 0x0000ff00) * a) >> 8) + (s0 & 0x0000ff00); + uint32_t r = (((s1 & 0x00ff0000) >> 8) * a) + (s0 & 0x00ff0000); + + b = (b > 255) ? 255 : b; + g = (g > 0xff00) ? 0xff00 : (g & 0xff00); + r = (r > 0x00ff0000) ? 0x00ff0000 : (r & 0x00ff0000); + + *(uint32_t*)dst_argb = 0xff000000u | r | g | b; + dst_argb += 4; + --width; + } +} +#endif // HAS_ARGBBLENDROW_WASMSIMD + +#ifdef HAS_BLENDPLANEROW_WASMSIMD +void BlendPlaneRow_WASMSIMD(const uint8_t* src0, + const uint8_t* src1, + const uint8_t* alpha, + uint8_t* dst, + int width) { + // src0 * a + src1 * (255 - a) + 255 fits in 16 bits. 255 - a is a ^ 255. + // The high bytes are the result (uzp2 on Arm). + v128_t c255 = SplatConst16(255); + v128_t ff = SplatConst8(255); + do { + v128_t s0 = wasm_v128_load(src0); + v128_t s1 = wasm_v128_load(src1); + v128_t a = wasm_v128_load(alpha); + v128_t ia = wasm_v128_xor(a, ff); + src0 += 16; + src1 += 16; + alpha += 16; + v128_t lo = wasm_i16x8_add(wasm_u16x8_extmul_low_u8x16(s0, a), + wasm_u16x8_extmul_low_u8x16(s1, ia)); + v128_t hi = wasm_i16x8_add(wasm_u16x8_extmul_high_u8x16(s0, a), + wasm_u16x8_extmul_high_u8x16(s1, ia)); + lo = wasm_i16x8_add(lo, c255); + hi = wasm_i16x8_add(hi, c255); + wasm_v128_store(dst, wasm_i8x16_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15, + 17, 19, 21, 23, 25, 27, 29, 31)); + dst += 16; + width -= 16; + } while (width > 0); +} +#endif // HAS_BLENDPLANEROW_WASMSIMD + +#undef LIBYUV_WASM_SWIZZLE + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale.cc b/source/scale.cc index 7e77aca89..de815ab3d 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -146,6 +146,22 @@ static void ScalePlaneDown2(int src_width, : ScaleRowDown2Box_RVV); } #endif +#if defined(HAS_SCALEROWDOWN2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowDown2 = + filtering == kFilterNone + ? ScaleRowDown2_Any_WASMSIMD + : (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_WASMSIMD + : ScaleRowDown2Box_Any_WASMSIMD); + if (IS_ALIGNED(dst_width, 16)) { + ScaleRowDown2 = + filtering == kFilterNone + ? ScaleRowDown2_WASMSIMD + : (filtering == kFilterLinear ? ScaleRowDown2Linear_WASMSIMD + : ScaleRowDown2Box_WASMSIMD); + } + } +#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -282,6 +298,16 @@ static void ScalePlaneDown4(int src_width, ScaleRowDown4 = filtering ? ScaleRowDown4Box_RVV : ScaleRowDown4_RVV; } #endif +#if defined(HAS_SCALEROWDOWN4_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleRowDown4 = + filtering ? ScaleRowDown4Box_Any_WASMSIMD : ScaleRowDown4_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 16)) { + ScaleRowDown4 = + filtering ? ScaleRowDown4Box_WASMSIMD : ScaleRowDown4_WASMSIMD; + } + } +#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -440,6 +466,27 @@ static void ScalePlaneDown34(int src_width, } } #endif +#if defined(HAS_SCALEROWDOWN34_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + if (dst_width % 24 == 0) { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_WASMSIMD; + ScaleRowDown34_1 = ScaleRowDown34_WASMSIMD; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_WASMSIMD; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_WASMSIMD; + } + } else { + if (!filtering) { + ScaleRowDown34_0 = ScaleRowDown34_Any_WASMSIMD; + ScaleRowDown34_1 = ScaleRowDown34_Any_WASMSIMD; + } else { + ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_WASMSIMD; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_WASMSIMD; + } + } + } +#endif for (y = 0; y < dst_height - 2; y += 3) { ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width); @@ -646,6 +693,26 @@ static void ScalePlaneDown38(int src_width, } } #endif +#if defined(HAS_SCALEROWDOWN38_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_Any_WASMSIMD; + ScaleRowDown38_2 = ScaleRowDown38_Any_WASMSIMD; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_WASMSIMD; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_WASMSIMD; + } + if (dst_width % 12 == 0) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_WASMSIMD; + ScaleRowDown38_2 = ScaleRowDown38_WASMSIMD; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_WASMSIMD; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_WASMSIMD; + } + } + } +#endif for (y = 0; y < dst_height - 2; y += 3) { ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); @@ -1098,6 +1165,14 @@ static int ScalePlaneBilinearDown(int src_width, InterpolateRow = InterpolateRow_RVV; } #endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(src_width, 16)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEFILTERCOLS_SSSE3) if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { @@ -1119,6 +1194,14 @@ static int ScalePlaneBilinearDown(int src_width, ScaleFilterCols = ScaleFilterCols_LSX; } } +#endif +#if defined(HAS_SCALEFILTERCOLS_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleFilterCols_WASMSIMD; + } + } #endif if (y > max_y) { y = max_y; @@ -1205,6 +1288,14 @@ static int ScalePlaneBilinearDown_16(int src_width, InterpolateRow = InterpolateRow_16_SME; } #endif +#if defined(HAS_INTERPOLATEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_16_Any_WASMSIMD; + if (IS_ALIGNED(src_width, 8)) { + InterpolateRow = InterpolateRow_16_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEFILTERCOLS_16_SSSE3) if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { @@ -1304,6 +1395,14 @@ static int ScalePlaneBilinearUp(int src_width, InterpolateRow = InterpolateRow_RVV; } #endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 16)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } +#endif if (filtering && src_width >= 32768) { ScaleFilterCols = ScaleFilterCols64_C; @@ -1328,6 +1427,14 @@ static int ScalePlaneBilinearUp(int src_width, ScaleFilterCols = ScaleFilterCols_LSX; } } +#endif +#if defined(HAS_SCALEFILTERCOLS_WASMSIMD) + if (filtering && TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleFilterCols_WASMSIMD; + } + } #endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleFilterCols = ScaleColsUp2_C; @@ -1800,6 +1907,14 @@ static int ScalePlaneBilinearUp_16(int src_width, InterpolateRow = InterpolateRow_16_SME; } #endif +#if defined(HAS_INTERPOLATEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_16_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_16_WASMSIMD; + } + } +#endif if (filtering && src_width >= 32768) { ScaleFilterCols = ScaleFilterCols64_16_C; diff --git a/source/scale_any.cc b/source/scale_any.cc index 9ea05d036..8d17e62c7 100644 --- a/source/scale_any.cc +++ b/source/scale_any.cc @@ -406,6 +406,106 @@ SDANY(ScaleARGBRowDown2Box_Any_LSX, 4, 3) #endif +#ifdef HAS_SCALEROWDOWN2_WASMSIMD +SDANY(ScaleRowDown2_Any_WASMSIMD, + ScaleRowDown2_WASMSIMD, + ScaleRowDown2_C, + 2, + 1, + 15) +SDANY(ScaleRowDown2Linear_Any_WASMSIMD, + ScaleRowDown2Linear_WASMSIMD, + ScaleRowDown2Linear_C, + 2, + 1, + 15) +SDANY(ScaleRowDown2Box_Any_WASMSIMD, + ScaleRowDown2Box_WASMSIMD, + ScaleRowDown2Box_C, + 2, + 1, + 15) +SDODD(ScaleRowDown2Box_Odd_WASMSIMD, + ScaleRowDown2Box_WASMSIMD, + ScaleRowDown2Box_Odd_C, + 2, + 1, + 15) +#endif +#ifdef HAS_SCALEROWDOWN4_WASMSIMD +SDANY(ScaleRowDown4_Any_WASMSIMD, + ScaleRowDown4_WASMSIMD, + ScaleRowDown4_C, + 4, + 1, + 15) +SDANY(ScaleRowDown4Box_Any_WASMSIMD, + ScaleRowDown4Box_WASMSIMD, + ScaleRowDown4Box_C, + 4, + 1, + 7) +#endif +#ifdef HAS_SCALEROWDOWN34_WASMSIMD +SDANY(ScaleRowDown34_Any_WASMSIMD, + ScaleRowDown34_WASMSIMD, + ScaleRowDown34_C, + 4 / 3, + 1, + 23) +SDANY(ScaleRowDown34_0_Box_Any_WASMSIMD, + ScaleRowDown34_0_Box_WASMSIMD, + ScaleRowDown34_0_Box_C, + 4 / 3, + 1, + 23) +SDANY(ScaleRowDown34_1_Box_Any_WASMSIMD, + ScaleRowDown34_1_Box_WASMSIMD, + ScaleRowDown34_1_Box_C, + 4 / 3, + 1, + 23) +#endif +#ifdef HAS_SCALEROWDOWN38_WASMSIMD +SDANY(ScaleRowDown38_Any_WASMSIMD, + ScaleRowDown38_WASMSIMD, + ScaleRowDown38_C, + 8 / 3, + 1, + 11) +SDANY(ScaleRowDown38_3_Box_Any_WASMSIMD, + ScaleRowDown38_3_Box_WASMSIMD, + ScaleRowDown38_3_Box_C, + 8 / 3, + 1, + 11) +SDANY(ScaleRowDown38_2_Box_Any_WASMSIMD, + ScaleRowDown38_2_Box_WASMSIMD, + ScaleRowDown38_2_Box_C, + 8 / 3, + 1, + 11) +#endif +#ifdef HAS_SCALEARGBROWDOWN2_WASMSIMD +SDANY(ScaleARGBRowDown2_Any_WASMSIMD, + ScaleARGBRowDown2_WASMSIMD, + ScaleARGBRowDown2_C, + 2, + 4, + 3) +SDANY(ScaleARGBRowDown2Linear_Any_WASMSIMD, + ScaleARGBRowDown2Linear_WASMSIMD, + ScaleARGBRowDown2Linear_C, + 2, + 4, + 3) +SDANY(ScaleARGBRowDown2Box_Any_WASMSIMD, + ScaleARGBRowDown2Box_WASMSIMD, + ScaleARGBRowDown2Box_C, + 2, + 4, + 3) +#endif #undef SDANY // Scale down by even scale factor. @@ -465,6 +565,18 @@ SDAANY(ScaleUVRowDownEven_Any_NEON, 2, 3) #endif +#ifdef HAS_SCALEARGBROWDOWNEVEN_WASMSIMD +SDAANY(ScaleARGBRowDownEven_Any_WASMSIMD, + ScaleARGBRowDownEven_WASMSIMD, + ScaleARGBRowDownEven_C, + 4, + 3) +SDAANY(ScaleARGBRowDownEvenBox_Any_WASMSIMD, + ScaleARGBRowDownEvenBox_WASMSIMD, + ScaleARGBRowDownEvenBox_C, + 4, + 3) +#endif #ifdef SASIMDONLY // This also works and uses memcpy and SIMD instead of C, but is slower on ARM @@ -579,6 +691,23 @@ CANY(ScaleARGBFilterCols_Any_LSX, 4, 7) #endif +#ifdef HAS_SCALEFILTERCOLS_WASMSIMD +CANY(ScaleFilterCols_Any_WASMSIMD, + ScaleFilterCols_WASMSIMD, + ScaleFilterCols_C, + 1, + 7) +#endif +#ifdef HAS_SCALEARGBCOLS_WASMSIMD +CANY(ScaleARGBCols_Any_WASMSIMD, ScaleARGBCols_WASMSIMD, ScaleARGBCols_C, 4, 3) +#endif +#ifdef HAS_SCALEARGBFILTERCOLS_WASMSIMD +CANY(ScaleARGBFilterCols_Any_WASMSIMD, + ScaleARGBFilterCols_WASMSIMD, + ScaleARGBFilterCols_C, + 4, + 3) +#endif #undef CANY // Scale up horizontally 2 times using linear filter. diff --git a/source/scale_argb.cc b/source/scale_argb.cc index 70b2c3de4..a1650dc72 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -134,6 +134,22 @@ static void ScaleARGBDown2(int src_width, : ScaleARGBRowDown2Box_RVV); } #endif +#if defined(HAS_SCALEARGBROWDOWN2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleARGBRowDown2 = + filtering == kFilterNone + ? ScaleARGBRowDown2_Any_WASMSIMD + : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_WASMSIMD + : ScaleARGBRowDown2Box_Any_WASMSIMD); + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBRowDown2 = + filtering == kFilterNone + ? ScaleARGBRowDown2_WASMSIMD + : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_WASMSIMD + : ScaleARGBRowDown2Box_WASMSIMD); + } + } +#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -206,6 +222,14 @@ static int ScaleARGBDown4Box(int src_width, ScaleARGBRowDown2 = ScaleARGBRowDown2Box_RVV; } #endif +#if defined(HAS_SCALEARGBROWDOWN2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_WASMSIMD; + } + } +#endif for (j = 0; j < dst_height; ++j) { ScaleARGBRowDown2(src_argb, src_stride, row, dst_width * 2); @@ -286,6 +310,16 @@ static void ScaleARGBDownEven(int src_width, ScaleARGBRowDownEven = ScaleARGBRowDownEven_RVV; } #endif +#if defined(HAS_SCALEARGBROWDOWNEVEN_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_WASMSIMD + : ScaleARGBRowDownEven_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_WASMSIMD + : ScaleARGBRowDownEven_WASMSIMD; + } + } +#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -375,6 +409,14 @@ static int ScaleARGBBilinearDown(int src_width, InterpolateRow = InterpolateRow_RVV; } #endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(clip_src_width, 16)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEARGBFILTERCOLS_SSSE3) if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3; @@ -401,6 +443,14 @@ static int ScaleARGBBilinearDown(int src_width, ScaleARGBFilterCols = ScaleARGBFilterCols_RVV; } #endif +#if defined(HAS_SCALEARGBFILTERCOLS_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_WASMSIMD; + } + } +#endif // TODO(fbarchard): Consider not allocating row buffer for kFilterLinear. // Allocate a row of ARGB. @@ -503,6 +553,14 @@ static int ScaleARGBBilinearUp(int src_width, if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; } +#endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } #endif if (src_width >= 32768) { ScaleARGBFilterCols = @@ -554,6 +612,22 @@ static int ScaleARGBBilinearUp(int src_width, ScaleARGBFilterCols = ScaleARGBCols_LSX; } } +#endif +#if defined(HAS_SCALEARGBFILTERCOLS_WASMSIMD) + if (filtering && TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBFilterCols = ScaleARGBFilterCols_WASMSIMD; + } + } +#endif +#if defined(HAS_SCALEARGBCOLS_WASMSIMD) + if (!filtering && TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBCols_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBFilterCols = ScaleARGBCols_WASMSIMD; + } + } #endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleARGBFilterCols = ScaleARGBColsUp2_C; @@ -561,6 +635,11 @@ static int ScaleARGBBilinearUp(int src_width, if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { ScaleARGBFilterCols = ScaleARGBColsUp2_SSE2; } +#endif +#if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD) && IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBColsUp2_WASMSIMD; + } #endif } @@ -672,6 +751,14 @@ static void ScaleARGBSimple(int src_width, ScaleARGBCols = ScaleARGBCols_LSX; } } +#endif +#if defined(HAS_SCALEARGBCOLS_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD) && src_width < 32768) { + ScaleARGBCols = ScaleARGBCols_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + ScaleARGBCols = ScaleARGBCols_WASMSIMD; + } + } #endif if (src_width * 2 == dst_width && x < 0x8000) { ScaleARGBCols = ScaleARGBColsUp2_C; @@ -679,6 +766,11 @@ static void ScaleARGBSimple(int src_width, if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { ScaleARGBCols = ScaleARGBColsUp2_SSE2; } +#endif +#if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD) && IS_ALIGNED(dst_width, 8)) { + ScaleARGBCols = ScaleARGBColsUp2_WASMSIMD; + } #endif } diff --git a/source/scale_common.cc b/source/scale_common.cc index 19dd1d797..6c4c6dac1 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1447,6 +1447,14 @@ void ScalePlaneVertical(int src_height, InterpolateRow = InterpolateRow_RVV; } #endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(dst_width_bytes, 16)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } +#endif for (j = 0; j < dst_height; ++j) { int yi; @@ -1527,6 +1535,14 @@ void ScalePlaneVertical_16(int src_height, if (TestCpuFlag(kCpuHasSME)) { InterpolateRow = InterpolateRow_16_SME; } +#endif +#if defined(HAS_INTERPOLATEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_16_Any_WASMSIMD; + if (IS_ALIGNED(dst_width_words, 8)) { + InterpolateRow = InterpolateRow_16_WASMSIMD; + } + } #endif for (j = 0; j < dst_height; ++j) { int yi; diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 88a975dd4..7094cd0e6 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -561,6 +561,14 @@ static int ScaleUVBilinearDown(int src_width, InterpolateRow = InterpolateRow_RVV; } #endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(clip_src_width, 16)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } +#endif #if defined(HAS_SCALEUVFILTERCOLS_SSSE3) if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { ScaleUVFilterCols = ScaleUVFilterCols_SSSE3; @@ -678,6 +686,14 @@ static int ScaleUVBilinearUp(int src_width, if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; } +#endif +#if defined(HAS_INTERPOLATEROW_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_WASMSIMD; + } + } #endif if (src_width >= 32768) { ScaleUVFilterCols = filtering ? ScaleUVFilterCols64_C : ScaleUVCols64_C; diff --git a/source/scale_wasm.cc b/source/scale_wasm.cc new file mode 100644 index 000000000..661f923eb --- /dev/null +++ b/source/scale_wasm.cc @@ -0,0 +1,645 @@ +/* + * Copyright 2026 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/row.h" +#include "libyuv/scale_row.h" + +#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) +#include + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Arbitrary i8x16.shuffle masks are rematerialized on every use by V8, so +// shuffles that do not map to a single instruction use swizzle with masks +// loaded before the loop. Mask indices of 128 produce 0. +#ifdef __wasm_relaxed_simd__ +#define LIBYUV_WASM_SWIZZLE wasm_i8x16_relaxed_swizzle +#else +#define LIBYUV_WASM_SWIZZLE wasm_i8x16_swizzle +#endif + +// Load a constant vector. The empty asm hides the address so the compiler +// keeps the vector in a local instead of sinking a v128.const into the loop. +static inline v128_t LoadConst(const uint8_t* p) { + asm("" : "+r"(p)); + return wasm_v128_load(p); +} + +#ifdef HAS_SCALEROWDOWN2_WASMSIMD +// Point sample 32 pixels to 16, keeping the odd pixels. +void ScaleRowDown2_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + v128_t a = wasm_u16x8_shr(wasm_v128_load(src_ptr), 8); + v128_t b = wasm_u16x8_shr(wasm_v128_load(src_ptr + 16), 8); + wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(a, b)); + src_ptr += 32; + dst_ptr += 16; + dst_width -= 16; + } +} + +// Average pairs of pixels: (s0 + s1 + 1) >> 1. +void ScaleRowDown2Linear_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const v128_t zero = wasm_i16x8_const_splat(0); + (void)src_stride; + while (dst_width > 0) { + v128_t a = wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr)); + v128_t b = wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr + 16)); + a = wasm_u16x8_avgr(a, zero); + b = wasm_u16x8_avgr(b, zero); + wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(a, b)); + src_ptr += 32; + dst_ptr += 16; + dst_width -= 16; + } +} + +// Average 2x2 boxes: (s0 + s1 + t0 + t1 + 2) >> 2. +void ScaleRowDown2Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const uint8_t* t = src_ptr + src_stride; + const v128_t two = wasm_i16x8_const_splat(2); + while (dst_width > 0) { + v128_t a = wasm_i16x8_add( + wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr)), + wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(t))); + v128_t b = wasm_i16x8_add( + wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr + 16)), + wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(t + 16))); + a = wasm_u16x8_shr(wasm_i16x8_add(a, two), 2); + b = wasm_u16x8_shr(wasm_i16x8_add(b, two), 2); + wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(a, b)); + src_ptr += 32; + t += 32; + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWDOWN2_WASMSIMD + +#ifdef HAS_SCALEROWDOWN4_WASMSIMD +// Point sample 64 pixels to 16, keeping pixel 2 of every 4. Odd 16 bit +// lanes then even bytes, which are unzips (uzp on Arm, pack on x86). +void ScaleRowDown4_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const v128_t mask = wasm_i16x8_const_splat(0xff); + (void)src_stride; + while (dst_width > 0) { + v128_t a = wasm_i16x8_shuffle(wasm_v128_load(src_ptr), + wasm_v128_load(src_ptr + 16), 1, 3, 5, 7, 9, + 11, 13, 15); + v128_t b = wasm_i16x8_shuffle(wasm_v128_load(src_ptr + 32), + wasm_v128_load(src_ptr + 48), 1, 3, 5, 7, 9, + 11, 13, 15); + wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(wasm_v128_and(a, mask), + wasm_v128_and(b, mask))); + src_ptr += 64; + dst_ptr += 16; + dst_width -= 16; + } +} + +// Average 4x4 boxes: (sum + 8) >> 4. 8 pixels per loop. +void ScaleRowDown4Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const v128_t eight = wasm_i16x8_const_splat(8); + while (dst_width > 0) { + v128_t a = wasm_i16x8_const_splat(0); + v128_t b = wasm_i16x8_const_splat(0); + const uint8_t* s = src_ptr; + for (int i = 0; i < 4; ++i) { + a = wasm_i16x8_add(a, + wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(s))); + b = wasm_i16x8_add( + b, wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(s + 16))); + s += src_stride; + } + a = wasm_u16x8_narrow_i32x4(wasm_u32x4_extadd_pairwise_u16x8(a), + wasm_u32x4_extadd_pairwise_u16x8(b)); + a = wasm_u16x8_shr(wasm_i16x8_add(a, eight), 4); + wasm_v128_store64_lane(dst_ptr, wasm_u8x16_narrow_i16x8(a, a), 0); + src_ptr += 32; + dst_ptr += 8; + dst_width -= 8; + } +} +#endif // HAS_SCALEROWDOWN4_WASMSIMD + +#ifdef HAS_SCALEROWDOWN34_WASMSIMD +static const uint8_t kShuf34[8][16] = { + // ScaleRowDown34: 16 bytes from a, then b, then 8 bytes from b. + {0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128}, + {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 1, 3, 4}, + {5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128, 128, 128, 128, 128}, + // Filter34: s0 s1 s3 and s0 s2 s2 of every 4. + {0, 1, 3, 3, 4, 5, 7, 7, 8, 9, 11, 11, 12, 13, 15, 15}, + {0, 2, 2, 3, 4, 6, 6, 7, 8, 10, 10, 11, 12, 14, 14, 15}, + // Store34: bytes 0, 1, 2 of every 4 from a, then b, then 8 from b. + {0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 128, 128, 128, 128}, + {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 1, 2, 4}, + {5, 6, 8, 9, 10, 12, 13, 14, 128, 128, 128, 128, 128, 128, 128, 128}, +}; + +// Point sample 32 pixels to 24, keeping pixels 0, 1 and 3 of every 4. +void ScaleRowDown34_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const v128_t m0 = LoadConst(kShuf34[0]); + const v128_t m1 = LoadConst(kShuf34[1]); + const v128_t m2 = LoadConst(kShuf34[2]); + (void)src_stride; + while (dst_width > 0) { + v128_t a = wasm_v128_load(src_ptr); + v128_t b = wasm_v128_load(src_ptr + 16); + wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0), + LIBYUV_WASM_SWIZZLE(b, m1))); + wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0); + src_ptr += 32; + dst_ptr += 24; + dst_width -= 24; + } +} + +// Rounding-down average: (a + b) >> 1. +static inline v128_t AvgFloor(v128_t a, v128_t b) { + return wasm_i8x16_sub( + wasm_u8x16_avgr(a, b), + wasm_v128_and(wasm_v128_xor(a, b), wasm_i8x16_const_splat(1))); +} + +// Horizontal 4 to 3 filter of 16 pixels. Result bytes 4n+0..2 are +// (3 * s0 + s1 + 2) >> 2, (s1 + s2 + 1) >> 1, (s2 + 3 * s3 + 2) >> 2 +// using (3 * a + b + 2) >> 2 == avgr(a, (a + b) >> 1). Byte 4n+3 is unused. +static inline v128_t Filter34(v128_t s, v128_t mp, v128_t ma) { + v128_t p = LIBYUV_WASM_SWIZZLE(s, mp); // s0 s1 s3 + v128_t a = LIBYUV_WASM_SWIZZLE(s, ma); // s0 s2 s2 + v128_t b = wasm_u32x4_shr(s, 8); // s1 s2 s3 + return wasm_u8x16_avgr(p, AvgFloor(a, b)); +} + +// Filter rows 1 and 2 together, 1 : 1 +void ScaleRowDown34_1_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const uint8_t* t = src_ptr + src_stride; + const v128_t mp = LoadConst(kShuf34[3]); + const v128_t ma = LoadConst(kShuf34[4]); + const v128_t m0 = LoadConst(kShuf34[5]); + const v128_t m1 = LoadConst(kShuf34[6]); + const v128_t m2 = LoadConst(kShuf34[7]); + while (dst_width > 0) { + v128_t a = wasm_u8x16_avgr(Filter34(wasm_v128_load(src_ptr), mp, ma), + Filter34(wasm_v128_load(t), mp, ma)); + v128_t b = wasm_u8x16_avgr(Filter34(wasm_v128_load(src_ptr + 16), mp, ma), + Filter34(wasm_v128_load(t + 16), mp, ma)); + wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0), + LIBYUV_WASM_SWIZZLE(b, m1))); + wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0); + src_ptr += 32; + t += 32; + dst_ptr += 24; + dst_width -= 24; + } +} + +// Filter rows 0 and 1 together, 3 : 1 +void ScaleRowDown34_0_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const uint8_t* t = src_ptr + src_stride; + const v128_t mp = LoadConst(kShuf34[3]); + const v128_t ma = LoadConst(kShuf34[4]); + const v128_t m0 = LoadConst(kShuf34[5]); + const v128_t m1 = LoadConst(kShuf34[6]); + const v128_t m2 = LoadConst(kShuf34[7]); + while (dst_width > 0) { + v128_t a0 = Filter34(wasm_v128_load(src_ptr), mp, ma); + v128_t b0 = Filter34(wasm_v128_load(src_ptr + 16), mp, ma); + v128_t a1 = Filter34(wasm_v128_load(t), mp, ma); + v128_t b1 = Filter34(wasm_v128_load(t + 16), mp, ma); + v128_t a = wasm_u8x16_avgr(a0, AvgFloor(a0, a1)); + v128_t b = wasm_u8x16_avgr(b0, AvgFloor(b0, b1)); + wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0), + LIBYUV_WASM_SWIZZLE(b, m1))); + wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0); + src_ptr += 32; + t += 32; + dst_ptr += 24; + dst_width -= 24; + } +} +#endif // HAS_SCALEROWDOWN34_WASMSIMD + +#ifdef HAS_SCALEROWDOWN38_WASMSIMD +static const uint8_t kShuf38[3][16] = { + // ScaleRowDown38: 6 bytes from a, then 6 bytes from b. + {0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, + {128, 128, 128, 128, 128, 128, 0, 3, 6, 8, 11, 14, 128, 128, 128, 128}, + // Store38: bytes 0, 1, 3 of every 4. + {0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128}, +}; + +// Point sample 32 pixels to 12, keeping pixels 0, 3 and 6 of every 8. +void ScaleRowDown38_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const v128_t m0 = LoadConst(kShuf38[0]); + const v128_t m1 = LoadConst(kShuf38[1]); + (void)src_stride; + while (dst_width > 0) { + v128_t d = + wasm_v128_or(LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_ptr), m0), + LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_ptr + 16), m1)); + wasm_v128_store64_lane(dst_ptr, d, 0); + wasm_v128_store32_lane(dst_ptr + 8, d, 2); + src_ptr += 32; + dst_ptr += 12; + dst_width -= 12; + } +} + +// Sums of columns 0-2, 3-5 and 6-7 of 8 columns of u16 are scaled by the +// i16 factors in lanes 0, 3 and 6 of k. Returns the high 16 bits of the +// products in 32 bit lanes 0, 1 and 3. +static inline v128_t Sum38(v128_t s, v128_t k) { + const v128_t zero = wasm_i16x8_const_splat(0); + v128_t s1 = wasm_i8x16_shuffle(s, zero, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17); + v128_t s2 = wasm_i8x16_shuffle(s, zero, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19); + s = wasm_i16x8_add(wasm_i16x8_add(s, s1), s2); + return wasm_u32x4_shr(wasm_i32x4_dot_i16x8(s, k), 16); +} + +// Pack 12 results from Sum38 and store them. +static inline void Store38(uint8_t* dst_ptr, + v128_t m, + v128_t a, + v128_t b, + v128_t c, + v128_t d) { + v128_t r = wasm_u8x16_narrow_i16x8(wasm_u16x8_narrow_i32x4(a, b), + wasm_u16x8_narrow_i32x4(c, d)); + r = LIBYUV_WASM_SWIZZLE(r, m); + wasm_v128_store64_lane(dst_ptr, r, 0); + wasm_v128_store32_lane(dst_ptr + 8, r, 2); +} + +// 8x3 -> 3x1 +void ScaleRowDown38_3_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const v128_t k = + wasm_i16x8_const(65536 / 9, 0, 0, 65536 / 9, 0, 0, 65536 / 6, 0); + const v128_t m = LoadConst(kShuf38[2]); + while (dst_width > 0) { + v128_t s[4]; + for (int i = 0; i < 4; ++i) { + const uint8_t* p = src_ptr + i * 8; + s[i] = wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_load8x8(p), + wasm_u16x8_load8x8(p + src_stride)), + wasm_u16x8_load8x8(p + src_stride * 2)); + } + Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), Sum38(s[2], k), + Sum38(s[3], k)); + src_ptr += 32; + dst_ptr += 12; + dst_width -= 12; + } +} + +// 8x2 -> 3x1 +void ScaleRowDown38_2_Box_WASMSIMD(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const v128_t k = + wasm_i16x8_const(65536 / 6, 0, 0, 65536 / 6, 0, 0, 65536 / 4, 0); + const v128_t m = LoadConst(kShuf38[2]); + while (dst_width > 0) { + v128_t s[4]; + for (int i = 0; i < 4; ++i) { + const uint8_t* p = src_ptr + i * 8; + s[i] = wasm_i16x8_add(wasm_u16x8_load8x8(p), + wasm_u16x8_load8x8(p + src_stride)); + } + Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), Sum38(s[2], k), + Sum38(s[3], k)); + src_ptr += 32; + dst_ptr += 12; + dst_width -= 12; + } +} +#endif // HAS_SCALEROWDOWN38_WASMSIMD + +#ifdef HAS_SCALEFILTERCOLS_WASMSIMD +// Bilinear column filtering with 7 bit fractions, matching ScaleFilterCols_C: +// a + ((f * (b - a) + 64) >> 7). 8 pixels per loop. The fraction +// (x >> 9) & 0x7f only uses the low 16 bits of x, so x is tracked in 16 bit +// lanes. +void ScaleFilterCols_WASMSIMD(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx) { + uint32_t ux = (uint32_t)x; + const uint32_t udx = (uint32_t)dx; + v128_t xv = wasm_i16x8_make((int16_t)ux, (int16_t)(ux + udx), + (int16_t)(ux + udx * 2), (int16_t)(ux + udx * 3), + (int16_t)(ux + udx * 4), (int16_t)(ux + udx * 5), + (int16_t)(ux + udx * 6), (int16_t)(ux + udx * 7)); + const v128_t dx8 = wasm_i16x8_splat((int16_t)(udx * 8)); + const v128_t maskff = wasm_i16x8_const_splat(0xff); + const v128_t round = wasm_i16x8_const_splat(64); + while (dst_width > 0) { + v128_t p = wasm_i16x8_const_splat(0); + p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 0); + ux += udx; + p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 1); + ux += udx; + p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 2); + ux += udx; + p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 3); + ux += udx; + p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 4); + ux += udx; + p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 5); + ux += udx; + p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 6); + ux += udx; + p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 7); + ux += udx; + v128_t f = wasm_u16x8_shr(xv, 9); + xv = wasm_i16x8_add(xv, dx8); + v128_t a = wasm_v128_and(p, maskff); + v128_t d = wasm_i16x8_sub(wasm_u16x8_shr(p, 8), a); + d = wasm_i16x8_shr(wasm_i16x8_add(wasm_i16x8_mul(d, f), round), 7); + d = wasm_i16x8_add(d, a); + wasm_v128_store64_lane(dst_ptr, wasm_u8x16_narrow_i16x8(d, d), 0); + dst_ptr += 8; + dst_width -= 8; + } +} +#endif // HAS_SCALEFILTERCOLS_WASMSIMD + +#if defined(HAS_SCALEARGBROWDOWN2_WASMSIMD) || \ + defined(HAS_SCALEARGBROWDOWNEVEN_WASMSIMD) +// Average 2x2 boxes of ARGB pixels. s0 s1 hold 4 pairs of pixels from row +// 0 and t0 t1 hold the same pairs from row 1. Returns 4 pixels. +static inline v128_t BoxARGB(v128_t s0, v128_t s1, v128_t t0, v128_t t1) { + v128_t se = wasm_i32x4_shuffle(s0, s1, 0, 2, 4, 6); + v128_t so = wasm_i32x4_shuffle(s0, s1, 1, 3, 5, 7); + v128_t te = wasm_i32x4_shuffle(t0, t1, 0, 2, 4, 6); + v128_t to = wasm_i32x4_shuffle(t0, t1, 1, 3, 5, 7); + // Interleave even and odd pixels so pairwise adds sum the same channel. + v128_t a = wasm_i16x8_add( + wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle( + se, so, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23)), + wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle( + te, to, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23))); + v128_t b = wasm_i16x8_add(wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle( + se, so, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31)), + wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle( + te, to, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + 13, 29, 14, 30, 15, 31))); + a = wasm_u16x8_shr(wasm_i16x8_add(a, wasm_i16x8_const_splat(2)), 2); + b = wasm_u16x8_shr(wasm_i16x8_add(b, wasm_i16x8_const_splat(2)), 2); + return wasm_u8x16_narrow_i16x8(a, b); +} +#endif + +#ifdef HAS_SCALEARGBROWDOWN2_WASMSIMD +// Point sample 8 ARGB pixels to 4, keeping the odd pixels. +void ScaleARGBRowDown2_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + v128_t a = wasm_v128_load(src_argb); + v128_t b = wasm_v128_load(src_argb + 16); + wasm_v128_store(dst_argb, wasm_i32x4_shuffle(a, b, 1, 3, 5, 7)); + src_argb += 32; + dst_argb += 16; + dst_width -= 4; + } +} + +// Average pairs of ARGB pixels. +void ScaleARGBRowDown2Linear_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + v128_t a = wasm_v128_load(src_argb); + v128_t b = wasm_v128_load(src_argb + 16); + wasm_v128_store(dst_argb, + wasm_u8x16_avgr(wasm_i32x4_shuffle(a, b, 0, 2, 4, 6), + wasm_i32x4_shuffle(a, b, 1, 3, 5, 7))); + src_argb += 32; + dst_argb += 16; + dst_width -= 4; + } +} + +// Average 2x2 boxes of ARGB pixels. +void ScaleARGBRowDown2Box_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + const uint8_t* t = src_argb + src_stride; + while (dst_width > 0) { + wasm_v128_store( + dst_argb, + BoxARGB(wasm_v128_load(src_argb), wasm_v128_load(src_argb + 16), + wasm_v128_load(t), wasm_v128_load(t + 16))); + src_argb += 32; + t += 32; + dst_argb += 16; + dst_width -= 4; + } +} +#endif // HAS_SCALEARGBROWDOWN2_WASMSIMD + +#ifdef HAS_SCALEARGBROWDOWNEVEN_WASMSIMD +// Point sample ARGB pixels with an even step. 4 pixels per loop. +void ScaleARGBRowDownEven_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width) { + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; + (void)src_stride; + while (dst_width > 0) { + v128_t v = wasm_v128_load32_zero(src_argb); + v = wasm_v128_load32_lane(src_argb + step, v, 1); + v = wasm_v128_load32_lane(src_argb + step * 2, v, 2); + v = wasm_v128_load32_lane(src_argb + step * 3, v, 3); + wasm_v128_store(dst_argb, v); + src_argb += step * 4; + dst_argb += 16; + dst_width -= 4; + } +} + +// Average 2x2 boxes of ARGB pixels with an even step. 4 pixels per loop. +void ScaleARGBRowDownEvenBox_WASMSIMD(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width) { + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; + while (dst_width > 0) { + const uint8_t* t = src_argb + src_stride; + v128_t s0 = wasm_v128_load64_zero(src_argb); + v128_t t0 = wasm_v128_load64_zero(t); + s0 = wasm_v128_load64_lane(src_argb + step, s0, 1); + t0 = wasm_v128_load64_lane(t + step, t0, 1); + v128_t s1 = wasm_v128_load64_zero(src_argb + step * 2); + v128_t t1 = wasm_v128_load64_zero(t + step * 2); + s1 = wasm_v128_load64_lane(src_argb + step * 3, s1, 1); + t1 = wasm_v128_load64_lane(t + step * 3, t1, 1); + wasm_v128_store(dst_argb, BoxARGB(s0, s1, t0, t1)); + src_argb += step * 4; + dst_argb += 16; + dst_width -= 4; + } +} +#endif // HAS_SCALEARGBROWDOWNEVEN_WASMSIMD + +#ifdef HAS_SCALEARGBCOLS_WASMSIMD +// Point sample ARGB columns. 4 pixels per loop. +void ScaleARGBCols_WASMSIMD(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + uint32_t ux = (uint32_t)x; + const uint32_t udx = (uint32_t)dx; + while (dst_width > 0) { + v128_t v = wasm_v128_load32_zero(src_argb + ((int)ux >> 16) * 4); + ux += udx; + v = wasm_v128_load32_lane(src_argb + ((int)ux >> 16) * 4, v, 1); + ux += udx; + v = wasm_v128_load32_lane(src_argb + ((int)ux >> 16) * 4, v, 2); + ux += udx; + v = wasm_v128_load32_lane(src_argb + ((int)ux >> 16) * 4, v, 3); + ux += udx; + wasm_v128_store(dst_argb, v); + dst_argb += 16; + dst_width -= 4; + } +} +#endif // HAS_SCALEARGBCOLS_WASMSIMD + +#ifdef HAS_SCALEARGBFILTERCOLS_WASMSIMD +// Bilinear ARGB column filtering with 7 bit fractions, matching +// ScaleARGBFilterCols_C: (a * (127 - f) + b * f) >> 7. 4 pixels per loop. +// The fraction (x >> 9) & 0x7f only uses the low 16 bits of x, so x is +// tracked in 16 bit lanes, 4 per pixel to match the channels. +void ScaleARGBFilterCols_WASMSIMD(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + uint32_t ux = (uint32_t)x; + const uint32_t udx = (uint32_t)dx; + int16_t x0 = (int16_t)ux; + int16_t x1 = (int16_t)(ux + udx); + int16_t x2 = (int16_t)(ux + udx * 2); + int16_t x3 = (int16_t)(ux + udx * 3); + v128_t x01 = wasm_i16x8_make(x0, x0, x0, x0, x1, x1, x1, x1); + v128_t x23 = wasm_i16x8_make(x2, x2, x2, x2, x3, x3, x3, x3); + const v128_t dx4 = wasm_i16x8_splat((int16_t)(udx * 4)); + const v128_t k7f = wasm_i16x8_const_splat(0x7f); + while (dst_width > 0) { + // Load pixel pairs (a, b) for 4 pixels. + v128_t p0 = wasm_v128_load64_zero(src_argb + ((int)ux >> 16) * 4); + ux += udx; + p0 = wasm_v128_load64_lane(src_argb + ((int)ux >> 16) * 4, p0, 1); + ux += udx; + v128_t p1 = wasm_v128_load64_zero(src_argb + ((int)ux >> 16) * 4); + ux += udx; + p1 = wasm_v128_load64_lane(src_argb + ((int)ux >> 16) * 4, p1, 1); + ux += udx; + v128_t a = wasm_i32x4_shuffle(p0, p1, 0, 2, 4, 6); // a0 a1 a2 a3 + v128_t b = wasm_i32x4_shuffle(p0, p1, 1, 3, 5, 7); // b0 b1 b2 b3 + v128_t f01 = wasm_u16x8_shr(x01, 9); + v128_t f23 = wasm_u16x8_shr(x23, 9); + x01 = wasm_i16x8_add(x01, dx4); + x23 = wasm_i16x8_add(x23, dx4); + v128_t r0 = wasm_i16x8_add( + wasm_i16x8_mul(wasm_u16x8_extend_low_u8x16(a), wasm_v128_xor(f01, k7f)), + wasm_i16x8_mul(wasm_u16x8_extend_low_u8x16(b), f01)); + v128_t r1 = + wasm_i16x8_add(wasm_i16x8_mul(wasm_u16x8_extend_high_u8x16(a), + wasm_v128_xor(f23, k7f)), + wasm_i16x8_mul(wasm_u16x8_extend_high_u8x16(b), f23)); + wasm_v128_store(dst_argb, wasm_u8x16_narrow_i16x8(wasm_u16x8_shr(r0, 7), + wasm_u16x8_shr(r1, 7))); + dst_argb += 16; + dst_width -= 4; + } +} +#endif // HAS_SCALEARGBFILTERCOLS_WASMSIMD + +#ifdef HAS_SCALEARGBCOLSUP2_WASMSIMD +// Point sample ARGB up by 2x. 8 pixels per loop. +void ScaleARGBColsUp2_WASMSIMD(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + (void)x; + (void)dx; + while (dst_width > 0) { + // V8 lowers an Arm zip of a vector with itself to several instructions, + // so zip with copies of the pixels loaded by load64_splat instead. + v128_t s = wasm_v128_load(src_argb); + v128_t t = wasm_v128_load64_splat(src_argb); + v128_t u = wasm_v128_load64_splat(src_argb + 8); + wasm_v128_store(dst_argb, wasm_i32x4_shuffle(s, t, 0, 4, 1, 5)); + wasm_v128_store(dst_argb + 16, wasm_i32x4_shuffle(s, u, 2, 6, 3, 7)); + src_argb += 16; + dst_argb += 32; + dst_width -= 8; + } +} +#endif // HAS_SCALEARGBCOLSUP2_WASMSIMD + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)