coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-11-09 14:49:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

[libyuv] Add GCC 15 SME support, SVE2/SME ports, and next 10 SIMD fixes

Browse Source 
1. Enable ARMv9 SVE2 and SME compilation with GCC 15 and Clang across
   Android.bp, BUILD.gn, CMakeLists.txt, and linux.mk using portable
   LIBYUV_SVE_ATTR, LIBYUV_SME_ATTR, and STREAMING_COMPATIBLE macros.
2. Port 46 SVE2 kernels to streaming-compatible SVE_SC functions in
   row_sve.h and expose _SME entry points for processors with SME but
   no non-streaming SVE2. Upgrade 15 1x SVE2 kernels to 2x unrolled,
   use full-vector st1b in ARGBToRGB24Row/ARGBToRAWRow_SVE2, contiguous
   ld1h/st1h in HalfFloatRow_SVE_SC, and frecpe+frecps reciprocal in
   ARGBAlphaBlendRow_SVE_SC.
3. Add missing kCpuHasNEON runtime checks in planar_test.cc and define
   missing HAS_*_NEON macros in row.h for existing NEON kernels.
4. Handle odd width directly in AYUVToUVRow_SVE_SC and
   AYUVToVURow_SVE_SC and remove redundant width alignment guards in
   AYUVToNV12 and AYUVToNV21.
5. Implement TransposeWx8_NEON for AArch64 in rotate_neon64.cc so
   TransposePlane and RotatePlane90/270 cascade 16x -> 8x -> C tail.
6. Add _Any_ wrappers for 9 ARGB/float filter functions (ARGBGrayTo,
   ARGBGray, ARGBSepia, ARGBColorMatrix, ARGBQuantize, ARGBShade,
   GaussRow_F32, GaussCol_F32, and ARGBLumaColorTable).
7. Add HalfRow and HalfRow_16 wrappers for AVX2, NEON, SVE2, and SME
   delegating to InterpolateRow{,_16} with fraction 0x80.
8. Add YUY2ToUV422Row_AVX512BW and UYVYToUV422Row_AVX512BW in
   row_gcc.cc and row_win.cc.
9. Add HalfWidthRow_16To8_Odd_NEON in row_any.cc and wire into
   I41xToI420 and Px10ToI420 for odd chroma widths.
10. Fix inline asm clobbers and vzeroupper placement in
    TransposeWx32_16_AVX512BW and TransposeWx16_32_AVX512BW, and
    re-enable 36 passing unit tests.

Test: libyuv_unittest
Bug: 579659306
Change-Id: I6b808f8b03ed424d3af4bb7941c34af05c1cb774
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8701301
Commit-Queue: Frank Barchard <fbarchard@google.com>
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard 2026-11-02 10:21:09 -08:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent c15967905d
commit d088b8f2bd
34 changed files with 3667 additions and 2220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Android.bp
View File

@ -57,6 +57,8 @@ cc_library {
"source/rotate_neon.cc",
"source/rotate_neon64.cc",
"source/rotate_rvv.cc",
"source/rotate_sme.cc",
"source/rotate_sve.cc",
"source/row_any.cc",
"source/row_common.cc",
"source/row_gcc.cc",
@ -64,6 +66,8 @@ cc_library {
"source/row_neon.cc",
"source/row_neon64.cc",
"source/row_rvv.cc",
"source/row_sme.cc",
"source/row_sve.cc",
"source/scale.cc",
"source/scale_any.cc",
"source/scale_argb.cc",
@ -74,6 +78,7 @@ cc_library {
"source/scale_neon64.cc",
"source/scale_rgb.cc",
"source/scale_rvv.cc",
"source/scale_sme.cc",
"source/scale_uv.cc",
"source/video_common.cc",
],

8
BUILD.gn
View File

@ -288,8 +288,12 @@ if (libyuv_use_sme) {
use_libcxx_modules = false
# SME is an Armv9-A feature, but use Armv8.5-A for compatibility with
# processors like Apple M4/M5 that support SME without SVE.
cflags = [ "-march=armv8.5-a+i8mm+sme" ]
# processors like Apple M4/M5/M6 that support SME without SVE.
if (is_clang) {
cflags = [ "-march=armv8.5-a+i8mm+sme" ]
} else {
cflags = [ "-march=armv8.5-a+i8mm+sve2+sme" ]
}
}
}

9
CMakeLists.txt
View File

@ -128,7 +128,12 @@ if(NOT MSVC)
# with Clang for Windows.
set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
set(OLD_CMAKE_TRY_COMPILE_TARGET_TYPE ${CMAKE_TRY_COMPILE_TARGET_TYPE})
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -march=armv8.5-a+i8mm+sme")
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(SME_COMPILER_FLAGS "-march=armv8.5-a+i8mm+sve2+sme")
else()
set(SME_COMPILER_FLAGS "-march=armv8.5-a+i8mm+sme")
endif()
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${SME_COMPILER_FLAGS}")
set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
check_c_source_compiles("
__arm_locally_streaming void func(void) { }
@ -143,7 +148,7 @@ int main(void) { return 0; }
${ly_src_dir}/rotate_sme.cc
${ly_src_dir}/row_sme.cc
${ly_src_dir}/scale_sme.cc)
target_compile_options(${ly_lib_name}_sme PRIVATE -march=armv8.5-a+i8mm+sme)
target_compile_options(${ly_lib_name}_sme PRIVATE ${SME_COMPILER_FLAGS})
list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_sme>)
else()
add_definitions(-DLIBYUV_DISABLE_SME)

4
include/libyuv/compare_row.h
View File

@ -112,8 +112,8 @@ extern "C" {
#endif
// The following are available for AArch64 SME:
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
defined(__aarch64__)
#if !defined(LIBYUV_DISABLE_SME) && \
(defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__)
#define HAS_SSIM4X4ROW_SME
#define HAS_SSIMWINDOWROW_SME
#endif

21
include/libyuv/cpu_support.h
View File

@ -150,11 +150,24 @@ extern "C" {
#undef LIBYUV_DISABLE_SME
#endif
// Clang 18+ required for SME due to needing __arm_tpidr2_save from compiler-rt,
// only enabled on Linux and Android (both define __linux__) for now.
#if !defined(LIBYUV_DISABLE_SME) && defined(__aarch64__) && \
defined(__linux__) && defined(__clang__) && (__clang_major__ >= 18)
// clang >= 11.0.0 or gcc >= 10.0.0 required for SVE2.
#if defined(__aarch64__) && !defined(LIBYUV_DISABLE_SVE)
#if (defined(__clang__) && (__clang_major__ < 11)) || \
(!defined(__clang__) && defined(__GNUC__) && (__GNUC__ < 10))
#define LIBYUV_DISABLE_SVE
#endif
#endif
// Clang 18+ on Linux/Android, Apple Clang 17+ on macOS, or GCC 15+ on Linux
// required for SME (__arm_tpidr2_save from compiler-rt or libgcc).
#if !defined(LIBYUV_DISABLE_SME) && defined(__aarch64__)
#if defined(__clang__) && ((defined(__linux__) && (__clang_major__ >= 18)) || \
(defined(__APPLE__) && (__clang_major__ >= 17)))
#define CLANG_HAS_SME 1
#elif defined(__linux__) && !defined(__clang__) && defined(__GNUC__) && \
(__GNUC__ >= 15)
#define GCC_HAS_SME 1
#endif
#endif
#ifdef __cplusplus

7
include/libyuv/rotate_row.h
View File

@ -92,9 +92,8 @@ extern "C" {
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
#if defined(__aarch64__)
#define HAS_TRANSPOSEWX16_NEON
#else
#define HAS_TRANSPOSEWX8_NEON
#endif
#define HAS_TRANSPOSEWX8_NEON
#define HAS_TRANSPOSEUVWX8_NEON
#define HAS_TRANSPOSEWX8_16_NEON
#define HAS_TRANSPOSEWX4_32_NEON
@ -106,8 +105,8 @@ extern "C" {
#define HAS_TRANSPOSEWX4_32_SVE2
#endif
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
defined(__aarch64__)
#if !defined(LIBYUV_DISABLE_SME) && \
(defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__)
#define HAS_TRANSPOSEWXH_SME
#define HAS_TRANSPOSEUVWXH_SME
#define HAS_TRANSPOSEWXH_16_SME

192
include/libyuv/row.h
View File

@ -554,9 +554,11 @@ extern "C" {
#define HAS_P410TOAR30ROW_AVX512BW
#define HAS_P410TOARGBROW_AVX512BW
#define HAS_UYVYTOARGBROW_AVX512BW
#define HAS_UYVYTOUV422ROW_AVX512BW
#define HAS_UYVYTOUVROW_AVX512BW
#define HAS_UYVYTOYROW_AVX512BW
#define HAS_YUY2TOARGBROW_AVX512BW
#define HAS_YUY2TOUV422ROW_AVX512BW
#define HAS_YUY2TOUVROW_AVX512BW
#define HAS_YUY2TOYROW_AVX512BW
#define HAS_ARGBTORAWROW_AVX512BW
@ -685,9 +687,11 @@ extern "C" {
#define HAS_P410TOAR30ROW_AVX512BW
#define HAS_P410TOARGBROW_AVX512BW
#define HAS_UYVYTOARGBROW_AVX512BW
#define HAS_UYVYTOUV422ROW_AVX512BW
#define HAS_UYVYTOUVROW_AVX512BW
#define HAS_UYVYTOYROW_AVX512BW
#define HAS_YUY2TOARGBROW_AVX512BW
#define HAS_YUY2TOUV422ROW_AVX512BW
#define HAS_YUY2TOUVROW_AVX512BW
#define HAS_YUY2TOYROW_AVX512BW
#endif
@ -892,6 +896,8 @@ extern "C" {
#define HAS_DIVIDEROW_16_SVE2
#define HAS_HALFFLOATROW_SVE2
#define HAS_HALFMERGEUVROW_SVE2
#define HAS_HALFROW_16_SVE2
#define HAS_HALFROW_SVE2
// Disabled: NEON version is faster for 128 bit vectors.
// #define HAS_HALFROW_16TO8_SVE2
// Disabled: NEON version is faster for 128 bit vectors.
@ -916,6 +922,7 @@ extern "C" {
#define HAS_I444ALPHATOARGBROW_SVE2
#define HAS_I444TOARGBROW_SVE2
#define HAS_I444TORGB24ROW_SVE2
#define HAS_INTERPOLATEROW_16_SVE2
#define HAS_INTERPOLATEROW_SVE2
#define HAS_MERGEUVROW_SVE2
#define HAS_NV12TOARGBROW_SVE2
@ -940,18 +947,30 @@ extern "C" {
#endif
// The following are available on AArch64 SME platforms:
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
defined(__aarch64__)
#if !defined(LIBYUV_DISABLE_SME) && \
(defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__)
#define HAS_ARGB1555TOARGBROW_SME
#define HAS_ARGBALPHABLENDROW_SME
#define HAS_ARGBMULTIPLYROW_SME
#define HAS_ARGBORROW_SME
#define HAS_ARGBTORAWROW_SME
#define HAS_ARGBTORGB24ROW_SME
#define HAS_ARGBTORGB565DITHERROW_SME
#define HAS_ARGBTORGB565ROW_SME
#define HAS_ARGBTOUVMATRIXROW_SME
#define HAS_ARGBTOYMATRIXROW_SME
#define HAS_AYUVTOUVROW_SME
#define HAS_AYUVTOVUROW_SME
#define HAS_CONVERT16TO8ROW_SME
#define HAS_CONVERT8TO16ROW_SME
#define HAS_CONVERT8TO8ROW_SME
#define HAS_COPYROW_SME
#define HAS_DIVIDEROW_16_SME
#define HAS_HALFFLOATROW_SME
#define HAS_HALFMERGEUVROW_SME
#define HAS_HALFROW_16_SME
#define HAS_HALFROW_16TO8_SME
#define HAS_HALFROW_SME
#define HAS_HALFWIDTHROW_16TO8_SME
#define HAS_I210ALPHATOARGBROW_SME
#define HAS_I210TOAR30ROW_SME
@ -986,14 +1005,25 @@ extern "C" {
#define HAS_P210TOARGBROW_SME
#define HAS_P410TOAR30ROW_SME
#define HAS_P410TOARGBROW_SME
#define HAS_RAWTOARGBROW_SME
#define HAS_RAWTORGB24ROW_SME
#define HAS_RAWTORGBAROW_SME
#define HAS_RGB24TOARGBROW_SME
#define HAS_SPLITUVROW_SME
#define HAS_UYVYTOARGBROW_SME
#define HAS_UYVYTOUVROW_SME
#define HAS_UYVYTOYROW_SME
#define HAS_YUY2TOARGBROW_SME
#define HAS_YUY2TOUVROW_SME
#define HAS_YUY2TOYROW_SME
#endif
// The following are available on AArch64 platforms:
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#define HAS_GAUSSCOL_F32_NEON
#define HAS_GAUSSROW_F32_NEON
#define HAS_SCALEMAXSAMPLES_NEON
#define HAS_SCALESAMPLES_NEON
#define HAS_SCALESUMSAMPLES_NEON
#endif
@ -3110,6 +3140,10 @@ void SplitUVRow_SVE2(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void SplitUVRow_SME(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void SplitUVRow_LSX(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
@ -3321,6 +3355,12 @@ void HalfMergeUVRow_SVE2(const uint8_t* src_u,
int src_stride_v,
uint8_t* dst_uv,
int width);
void HalfMergeUVRow_SME(const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_uv,
int width);
void HalfMergeUVRow_SSSE3(const uint8_t* src_u,
int src_stride_u,
@ -3943,6 +3983,10 @@ void DivideRow_16_SVE2(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width);
void DivideRow_16_SME(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width);
void DivideRow_16_Any_NEON(const uint16_t* src_ptr,
uint16_t* dst_ptr,
int scale,
@ -4046,6 +4090,14 @@ void HalfRow_Any_AVX2(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int width);
void HalfRow_SME(const uint8_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int width);
void HalfRow_SVE2(const uint8_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int width);
void HalfRow_16_C(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
@ -4059,6 +4111,14 @@ void HalfRow_16_Any_AVX2(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
int width);
void HalfRow_16_SME(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint16_t* dst_uv,
int width);
void HalfRow_16_SVE2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint16_t* dst_uv,
int width);
void HalfRow_16To8_C(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
@ -4131,6 +4191,11 @@ void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_Odd_NEON(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
@ -4441,6 +4506,7 @@ void RGB24ToARGBRow_NEON(const uint8_t* src_rgb24,
void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width);
void RGB24ToARGBRow_SME(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24,
uint8_t* dst_argb,
@ -4448,14 +4514,17 @@ void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24,
void RGB24ToARGBRow_RVV(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToARGBRow_SME(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToRGBARow_SME(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_SME(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
@ -4477,6 +4546,9 @@ void ARGB1555ToARGBRow_NEON(const uint8_t* src_argb1555,
void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width);
void ARGB1555ToARGBRow_SME(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width);
void ARGB1555ToARGBRow_LSX(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width);
@ -4677,14 +4749,19 @@ void ARGBToRGB24Row_NEON(const uint8_t* src_argb,
void ARGBToRGB24Row_SVE2(const uint8_t* src_argb,
uint8_t* dst_rgb24,
int width);
void ARGBToRGB24Row_SME(const uint8_t* src_argb, uint8_t* dst_rgb24, int width);
void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width);
void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_raw, int width);
void ARGBToRAWRow_SME(const uint8_t* src_argb, uint8_t* dst_raw, int width);
void ARGBToRGB565Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb565,
int width);
void ARGBToRGB565Row_SVE2(const uint8_t* src_argb,
uint8_t* dst_rgb565,
int width);
void ARGBToRGB565Row_SME(const uint8_t* src_argb,
uint8_t* dst_rgb565,
int width);
void ARGBToARGB1555Row_NEON(const uint8_t* src_argb,
uint8_t* dst_argb1555,
int width);
@ -4699,6 +4776,10 @@ void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb,
uint8_t* dst_rgb,
uint32_t dither4,
int width);
void ARGBToRGB565DitherRow_SME(const uint8_t* src_argb,
uint8_t* dst_rgb,
uint32_t dither4,
int width);
void ARGBToRGB565DitherRow_LSX(const uint8_t* src_argb,
uint8_t* dst_rgb,
uint32_t dither4,
@ -6853,6 +6934,7 @@ void YUY2ToUV422Row_SSE2(const uint8_t* src_yuy2,
int width);
void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
void YUY2ToYRow_SME(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width);
void YUY2ToUVRow_NEON(const uint8_t* src_yuy2,
int stride_yuy2,
@ -6864,23 +6946,42 @@ void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void YUY2ToUVRow_SME(const uint8_t* src_yuy2,
int stride_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2,
int stride_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void YUY2ToUV422Row_AVX512BW(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
void UYVYToYRow_SME(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width);
void UYVYToUVRow_SVE2(const uint8_t* src_uyvy,
int stride_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void UYVYToUVRow_SME(const uint8_t* src_uyvy,
int stride_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy,
int stride_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void UYVYToUV422Row_AVX512BW(const uint8_t* src_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2,
int stride_yuy2,
uint8_t* dst_uv,
@ -7151,6 +7252,10 @@ void AYUVToUVRow_SVE2(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_uv,
int width);
void AYUVToUVRow_SME(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_uv,
int width);
void AYUVToVURow_NEON(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_vu,
@ -7159,6 +7264,10 @@ void AYUVToVURow_SVE2(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_vu,
int width);
void AYUVToVURow_SME(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_vu,
int width);
void AYUVToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void AYUVToUVRow_Any_AVX2(const uint8_t* src_ptr,
int src_stride,
@ -7173,18 +7282,10 @@ void AYUVToUVRow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_vu,
int width);
void AYUVToUVRow_Any_SVE2(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_vu,
int width);
void AYUVToVURow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_vu,
int width);
void AYUVToVURow_Any_SVE2(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_vu,
int width);
void I422ToYUY2Row_C(const uint8_t* src_y,
const uint8_t* src_u,
@ -7374,6 +7475,12 @@ void ARGBGrayRow_NEON_DotProd(const uint8_t* src_argb,
int width);
void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width);
void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width);
void ARGBGrayRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void ARGBGrayRow_Any_NEON_DotProd(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void ARGBGrayRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void ARGBGrayRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void ARGBSepiaRow_C(uint8_t* dst_argb, int width);
void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width);
@ -7382,6 +7489,10 @@ void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width);
void ARGBSepiaRow_NEON_DotProd(uint8_t* dst_argb, int width);
void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width);
void ARGBSepiaRow_LASX(uint8_t* dst_argb, int width);
void ARGBSepiaRow_Any_NEON(uint8_t* dst_ptr, int width);
void ARGBSepiaRow_Any_NEON_DotProd(uint8_t* dst_ptr, int width);
void ARGBSepiaRow_Any_LSX(uint8_t* dst_ptr, int width);
void ARGBSepiaRow_Any_LASX(uint8_t* dst_ptr, int width);
void ARGBColorMatrixRow_C(const uint8_t* src_argb,
uint8_t* dst_argb,
@ -7411,6 +7522,22 @@ void ARGBColorMatrixRow_LASX(const uint8_t* src_argb,
uint8_t* dst_argb,
const int8_t* matrix_argb,
int width);
void ARGBColorMatrixRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const int8_t* matrix_argb,
int width);
void ARGBColorMatrixRow_Any_NEON_I8MM(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const int8_t* matrix_argb,
int width);
void ARGBColorMatrixRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const int8_t* matrix_argb,
int width);
void ARGBColorMatrixRow_Any_LASX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const int8_t* matrix_argb,
int width);
void ARGBColorTableRow_C(uint8_t* dst_argb,
const uint8_t* table_argb,
@ -7457,6 +7584,21 @@ void ARGBQuantizeRow_LASX(uint8_t* dst_argb,
int interval_size,
int interval_offset,
int width);
void ARGBQuantizeRow_Any_NEON(uint8_t* dst_ptr,
int scale,
int interval_size,
int interval_offset,
int width);
void ARGBQuantizeRow_Any_LSX(uint8_t* dst_ptr,
int scale,
int interval_size,
int interval_offset,
int width);
void ARGBQuantizeRow_Any_LASX(uint8_t* dst_ptr,
int scale,
int interval_size,
int interval_offset,
int width);
void ARGBShadeRow_C(const uint8_t* src_argb,
uint8_t* dst_argb,
@ -7478,6 +7620,18 @@ void ARGBShadeRow_LASX(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
uint32_t value);
void ARGBShadeRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width,
uint32_t value);
void ARGBShadeRow_Any_LSX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width,
uint32_t value);
void ARGBShadeRow_Any_LASX(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width,
uint32_t value);
// Used for blur.
void CumulativeSumToAverageRow_AVX2(const int32_t* topleft,
@ -7633,6 +7787,11 @@ void InterpolateRow_16_SME(uint16_t* dst_ptr,
ptrdiff_t src_stride,
int width,
int source_y_fraction);
void InterpolateRow_16_SVE2(uint16_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int width,
int source_y_fraction);
// Sobel images.
void SobelXRow_C(const uint8_t* src_y0,
@ -7846,6 +8005,14 @@ void HalfFloat1Row_SVE2(const uint16_t* src,
uint16_t* dst,
float scale,
int width);
void HalfFloatRow_SME(const uint16_t* src,
uint16_t* dst,
float scale,
int width);
void HalfFloat1Row_SME(const uint16_t* src,
uint16_t* dst,
float scale,
int width);
void HalfFloatRow_LSX(const uint16_t* src,
uint16_t* dst,
float scale,
@ -7907,6 +8074,11 @@ void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb,
int width,
const uint8_t* luma,
uint32_t lumacoeff);
void ARGBLumaColorTableRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width,
const uint8_t* luma,
uint32_t lumacoeff);
float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width);
float ScaleMaxSamples_AVX2(const float* src,

2751
include/libyuv/row_sve.h
View File

File diff suppressed because it is too large Load Diff

4
include/libyuv/scale_row.h
View File

@ -140,8 +140,8 @@ extern "C" {
#endif
// The following are available on AArch64 SME platforms:
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
defined(__aarch64__)
#if !defined(LIBYUV_DISABLE_SME) && \
(defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__)
#define HAS_SCALEARGBROWDOWN2_SME
#define HAS_SCALEROWDOWN2_16_SME
#define HAS_SCALEROWDOWN2_SME

5
linux.mk
View File

@ -39,6 +39,8 @@ LOCAL_OBJ_FILES := \
source/rotate_neon.o \
source/rotate_neon64.o \
source/rotate_rvv.o \
source/rotate_sme.o \
source/rotate_sve.o \
source/rotate_win.o \
source/row_any.o \
source/row_common.o \
@ -49,6 +51,8 @@ LOCAL_OBJ_FILES := \
source/row_neon.o \
source/row_neon64.o \
source/row_rvv.o \
source/row_sme.o \
source/row_sve.o \
source/row_win.o \
source/scale.o \
source/scale_any.o \
@ -61,6 +65,7 @@ LOCAL_OBJ_FILES := \
source/scale_neon64.o \
source/scale_rgb.o \
source/scale_rvv.o \
source/scale_sme.o \
source/scale_uv.o \
source/scale_win.o \
source/video_common.o

14
source/compare_neon64.cc
View File

@ -26,6 +26,14 @@ extern "C" {
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#if defined(__clang__)
#pragma clang attribute push(__attribute__((target("dotprod,i8mm"))), \
apply_to = function)
#elif defined(__GNUC__)
#pragma GCC push_options
#pragma GCC target("+dotprod+i8mm")
#endif
// 256 bits at a time
// uses short accumulator which restricts count to 131 KB
uint32_t HammingDistance_NEON(const uint8_t* src_a,
@ -451,6 +459,12 @@ uint32_t ARGBDetectRow_NEON(const uint8_t* argb, int width) {
return 0;
}
#if defined(__clang__)
#pragma clang attribute pop
#elif defined(__GNUC__)
#pragma GCC pop_options
#endif
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus

47
source/convert.cc
View File

@ -357,6 +357,11 @@ static int I41xToI420(const uint16_t* src_y,
}
#endif
} else {
#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_Odd_NEON;
}
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \
defined(HAS_HALFWIDTHROW_16TO8_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
@ -2159,6 +2164,11 @@ static int Px10ToI420(const uint16_t* src_y,
SplitUVRow = SplitUVRow_SVE2;
}
#endif
#if defined(HAS_SPLITUVROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
SplitUVRow = SplitUVRow_SME;
}
#endif
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;
@ -2331,6 +2341,11 @@ static int Px10ToI420(const uint16_t* src_y,
}
#endif
} else {
#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
Half16To8 = HalfWidthRow_16To8_Odd_NEON;
}
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \
defined(HAS_HALFWIDTHROW_16TO8_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
@ -2562,6 +2577,12 @@ int YUY2ToI420(const uint8_t* src_yuy2,
YUY2ToUVRow = YUY2ToUVRow_SVE2;
}
#endif
#if defined(HAS_YUY2TOYROW_SME) && defined(HAS_YUY2TOUVROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
YUY2ToYRow = YUY2ToYRow_SME;
YUY2ToUVRow = YUY2ToUVRow_SME;
}
#endif
#if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
YUY2ToYRow = YUY2ToYRow_Any_LSX;
@ -2675,6 +2696,12 @@ int UYVYToI420(const uint8_t* src_uyvy,
UYVYToUVRow = UYVYToUVRow_SVE2;
}
#endif
#if defined(HAS_UYVYTOYROW_SME) && defined(HAS_UYVYTOUVROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
UYVYToYRow = UYVYToYRow_SME;
UYVYToUVRow = UYVYToUVRow_SME;
}
#endif
#if defined(HAS_UYVYTOYROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
UYVYToYRow = UYVYToYRow_Any_LSX;
@ -2773,10 +2800,12 @@ int AYUVToNV12(const uint8_t* src_ayuv,
#endif
#if defined(HAS_AYUVTOUVROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
AYUVToUVRow = AYUVToUVRow_Any_SVE2;
if (IS_ALIGNED(width, 2)) {
AYUVToUVRow = AYUVToUVRow_SVE2;
}
AYUVToUVRow = AYUVToUVRow_SVE2;
}
#endif
#if defined(HAS_AYUVTOUVROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
AYUVToUVRow = AYUVToUVRow_SME;
}
#endif
@ -2850,10 +2879,12 @@ int AYUVToNV21(const uint8_t* src_ayuv,
#endif
#if defined(HAS_AYUVTOVUROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
AYUVToVURow = AYUVToVURow_Any_SVE2;
if (IS_ALIGNED(width, 2)) {
AYUVToVURow = AYUVToVURow_SVE2;
}
AYUVToVURow = AYUVToVURow_SVE2;
}
#endif
#if defined(HAS_AYUVTOVUROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
AYUVToVURow = AYUVToVURow_SME;
}
#endif

68
source/convert_argb.cc
View File

@ -1257,6 +1257,14 @@ int I210ToAR30Matrix(const uint16_t* src_y,
dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
dst_stride_ar30 = -dst_stride_ar30;
}
// Coalesce rows.
if (src_stride_y == width && src_stride_u * 2 == width &&
src_stride_v * 2 == width && dst_stride_ar30 == width * 4 &&
(ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = src_stride_u = src_stride_v = dst_stride_ar30 = 0;
}
#if defined(HAS_I210TOAR30ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
I210ToAR30Row = I210ToAR30Row_Any_NEON;
@ -1436,6 +1444,13 @@ int I410ToAR30Matrix(const uint16_t* src_y,
dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
dst_stride_ar30 = -dst_stride_ar30;
}
// Coalesce rows.
if (src_stride_y == width && src_stride_u == width && src_stride_v == width &&
dst_stride_ar30 == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = src_stride_u = src_stride_v = dst_stride_ar30 = 0;
}
#if defined(HAS_I410TOAR30ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
I410ToAR30Row = I410ToAR30Row_Any_NEON;
@ -2131,6 +2146,13 @@ int P210ToARGBMatrix(const uint16_t* src_y,
dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
if (IS_ALIGNED(width, 2) && src_stride_y == width && src_stride_uv == width &&
dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = src_stride_uv = dst_stride_argb = 0;
}
#if defined(HAS_P210TOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
P210ToARGBRow = P210ToARGBRow_Any_SSSE3;
@ -2204,6 +2226,13 @@ int P410ToARGBMatrix(const uint16_t* src_y,
dst_argb = dst_argb + (ptrdiff_t)(height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
// Coalesce rows.
if (src_stride_y == width && src_stride_uv == width * 2 &&
dst_stride_argb == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = src_stride_uv = dst_stride_argb = 0;
}
#if defined(HAS_P410TOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
P410ToARGBRow = P410ToARGBRow_Any_SSSE3;
@ -2352,6 +2381,13 @@ int P210ToAR30Matrix(const uint16_t* src_y,
dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
dst_stride_ar30 = -dst_stride_ar30;
}
// Coalesce rows.
if (IS_ALIGNED(width, 2) && src_stride_y == width && src_stride_uv == width &&
dst_stride_ar30 == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = src_stride_uv = dst_stride_ar30 = 0;
}
#if defined(HAS_P210TOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
P210ToAR30Row = P210ToAR30Row_Any_SSSE3;
@ -2425,6 +2461,13 @@ int P410ToAR30Matrix(const uint16_t* src_y,
dst_ar30 = dst_ar30 + (ptrdiff_t)(height - 1) * dst_stride_ar30;
dst_stride_ar30 = -dst_stride_ar30;
}
// Coalesce rows.
if (src_stride_y == width && src_stride_uv == width * 2 &&
dst_stride_ar30 == width * 4 && (ptrdiff_t)width * height <= INT_MAX) {
width *= height;
height = 1;
src_stride_y = src_stride_uv = dst_stride_ar30 = 0;
}
#if defined(HAS_P410TOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
P410ToAR30Row = P410ToAR30Row_Any_SSSE3;
@ -4132,6 +4175,11 @@ int RGB24ToARGB(const uint8_t* src_rgb24,
RGB24ToARGBRow = RGB24ToARGBRow_SVE2;
}
#endif
#if defined(HAS_RGB24TOARGBROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
RGB24ToARGBRow = RGB24ToARGBRow_SME;
}
#endif
#if defined(HAS_RGB24TOARGBROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
RGB24ToARGBRow = RGB24ToARGBRow_Any_LSX;
@ -4230,6 +4278,11 @@ int RAWToARGB(const uint8_t* src_raw,
RAWToARGBRow = RAWToARGBRow_SVE2;
}
#endif
#if defined(HAS_RAWTOARGBROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
RAWToARGBRow = RAWToARGBRow_SME;
}
#endif
#if defined(HAS_RAWTOARGBROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
RAWToARGBRow = RAWToARGBRow_Any_LSX;
@ -4322,6 +4375,11 @@ int RAWToRGBA(const uint8_t* src_raw,
RAWToRGBARow = RAWToRGBARow_SVE2;
}
#endif
#if defined(HAS_RAWTORGBAROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
RAWToRGBARow = RAWToRGBARow_SME;
}
#endif
#if defined(HAS_RAWTORGBAROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
RAWToRGBARow = RAWToRGBARow_RVV;
@ -4469,6 +4527,11 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555,
ARGB1555ToARGBRow = ARGB1555ToARGBRow_SVE2;
}
#endif
#if defined(HAS_ARGB1555TOARGBROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGB1555ToARGBRow = ARGB1555ToARGBRow_SME;
}
#endif
#if defined(HAS_ARGB1555TOARGBROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_LSX;
@ -7203,6 +7266,11 @@ int I420ToRGB565Dither(const uint8_t* src_y,
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SVE2;
}
#endif
#if defined(HAS_ARGBTORGB565DITHERROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SME;
}
#endif
#if defined(HAS_ARGBTORGB565DITHERROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LSX;

20
source/convert_from_argb.cc
View File

@ -3817,6 +3817,11 @@ int ARGBToRGB24(const uint8_t* src_argb,
ARGBToRGB24Row = ARGBToRGB24Row_SVE2;
}
#endif
#if defined(HAS_ARGBTORGB24ROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToRGB24Row = ARGBToRGB24Row_SME;
}
#endif
#if defined(HAS_ARGBTORGB24ROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToRGB24Row = ARGBToRGB24Row_Any_LSX;
@ -3907,6 +3912,11 @@ int ARGBToRAW(const uint8_t* src_argb,
ARGBToRAWRow = ARGBToRAWRow_SVE2;
}
#endif
#if defined(HAS_ARGBTORAWROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToRAWRow = ARGBToRAWRow_SME;
}
#endif
#if defined(HAS_ARGBTORAWROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToRAWRow = ARGBToRAWRow_Any_LSX;
@ -3996,6 +4006,11 @@ int ARGBToRGB565Dither(const uint8_t* src_argb,
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SVE2;
}
#endif
#if defined(HAS_ARGBTORGB565DITHERROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SME;
}
#endif
#if defined(HAS_ARGBTORGB565DITHERROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LSX;
@ -4084,6 +4099,11 @@ int ARGBToRGB565(const uint8_t* src_argb,
ARGBToRGB565Row = ARGBToRGB565Row_SVE2;
}
#endif
#if defined(HAS_ARGBTORGB565ROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToRGB565Row = ARGBToRGB565Row_SME;
}
#endif
#if defined(HAS_ARGBTORGB565ROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToRGB565Row = ARGBToRGB565Row_Any_LSX;

5
source/convert_to_argb.cc
View File

@ -256,6 +256,11 @@ static int NV24ToARGB(const uint8_t* src_y,
SplitUVRow = SplitUVRow_SVE2;
}
#endif
#if defined(HAS_SPLITUVROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
SplitUVRow = SplitUVRow_SME;
}
#endif
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;

10
source/convert_to_i420.cc
View File

@ -100,6 +100,11 @@ static int NV16ToI420(const uint8_t* src_y,
SplitUVRow = SplitUVRow_SVE2;
}
#endif
#if defined(HAS_SPLITUVROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
SplitUVRow = SplitUVRow_SME;
}
#endif
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;
@ -372,6 +377,11 @@ static int NV24ToI420(const uint8_t* src_y,
SplitUVRow = SplitUVRow_SVE2;
}
#endif
#if defined(HAS_SPLITUVROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
SplitUVRow = SplitUVRow_SME;
}
#endif
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;

237
source/planar_functions.cc
View File

@ -755,6 +755,11 @@ void SplitUVPlane(const uint8_t* src_uv,
SplitUVRow = SplitUVRow_SVE2;
}
#endif
#if defined(HAS_SPLITUVROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
SplitUVRow = SplitUVRow_SME;
}
#endif
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;
@ -1178,6 +1183,11 @@ void ConvertToLSBPlane_16(const uint16_t* src_y,
DivideRow = DivideRow_16_SVE2;
}
#endif
#if defined(HAS_DIVIDEROW_16_SME)
if (TestCpuFlag(kCpuHasSME)) {
DivideRow = DivideRow_16_SME;
}
#endif
#if defined(HAS_DIVIDEROW_16_HVX)
if (TestCpuFlag(kCpuHasHVX)) {
DivideRow = DivideRow_16_HVX;
@ -2534,8 +2544,9 @@ int YUY2ToI422(const uint8_t* src_yuy2,
}
}
#endif
#if defined(HAS_YUY2TOYROW_AVX512BW)
#if defined(HAS_YUY2TOYROW_AVX512BW) && defined(HAS_YUY2TOUV422ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
YUY2ToUV422Row = YUY2ToUV422Row_AVX512BW;
YUY2ToYRow = YUY2ToYRow_AVX512BW;
}
#endif
@ -2554,6 +2565,11 @@ int YUY2ToI422(const uint8_t* src_yuy2,
YUY2ToYRow = YUY2ToYRow_SVE2;
}
#endif
#if defined(HAS_YUY2TOYROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
YUY2ToYRow = YUY2ToYRow_SME;
}
#endif
#if defined(HAS_YUY2TOYROW_LSX) && defined(HAS_YUY2TOUV422ROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
YUY2ToYRow = YUY2ToYRow_Any_LSX;
@ -2647,8 +2663,9 @@ int UYVYToI422(const uint8_t* src_uyvy,
}
}
#endif
#if defined(HAS_UYVYTOYROW_AVX512BW)
#if defined(HAS_UYVYTOYROW_AVX512BW) && defined(HAS_UYVYTOUV422ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
UYVYToUV422Row = UYVYToUV422Row_AVX512BW;
UYVYToYRow = UYVYToYRow_AVX512BW;
}
#endif
@ -2667,6 +2684,11 @@ int UYVYToI422(const uint8_t* src_uyvy,
UYVYToYRow = UYVYToYRow_SVE2;
}
#endif
#if defined(HAS_UYVYTOYROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
UYVYToYRow = UYVYToYRow_SME;
}
#endif
#if defined(HAS_UYVYTOYROW_LSX) && defined(HAS_UYVYTOUV422ROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
UYVYToYRow = UYVYToYRow_Any_LSX;
@ -2766,6 +2788,11 @@ int YUY2ToY(const uint8_t* src_yuy2,
YUY2ToYRow = YUY2ToYRow_SVE2;
}
#endif
#if defined(HAS_YUY2TOYROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
YUY2ToYRow = YUY2ToYRow_SME;
}
#endif
#if defined(HAS_YUY2TOYROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
YUY2ToYRow = YUY2ToYRow_Any_LSX;
@ -2857,6 +2884,11 @@ int UYVYToY(const uint8_t* src_uyvy,
UYVYToYRow = UYVYToYRow_SVE2;
}
#endif
#if defined(HAS_UYVYTOYROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
UYVYToYRow = UYVYToYRow_SME;
}
#endif
#if defined(HAS_UYVYTOYROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
UYVYToYRow = UYVYToYRow_Any_LSX;
@ -3986,6 +4018,11 @@ int ARGBMultiply(const uint8_t* src_argb0,
}
}
#endif
#if defined(HAS_ARGBMULTIPLYROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
ARGBMultiplyRow = ARGBMultiplyRow_SVE2;
}
#endif
#if defined(HAS_ARGBMULTIPLYROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBMultiplyRow = ARGBMultiplyRow_SME;
@ -4058,11 +4095,6 @@ int ARGBAdd(const uint8_t* src_argb0,
height = 1;
src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0;
}
#if defined(HAS_ARGBADDROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGBAddRow = ARGBAddRow_SSE2;
}
#endif
#if defined(HAS_ARGBADDROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGBAddRow = ARGBAddRow_Any_SSE2;
@ -4382,6 +4414,11 @@ int RAWToRGB24(const uint8_t* src_raw,
RAWToRGB24Row = RAWToRGB24Row_SVE2;
}
#endif
#if defined(HAS_RAWTORGB24ROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
RAWToRGB24Row = RAWToRGB24Row_SME;
}
#endif
#if defined(HAS_RAWTORGB24ROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
RAWToRGB24Row = RAWToRGB24Row_Any_LSX;
@ -4813,23 +4850,35 @@ int ARGBGrayTo(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBGRAYROW_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_NEON;
if (TestCpuFlag(kCpuHasNEON)) {
ARGBGrayRow = ARGBGrayRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_NEON;
}
}
#endif
#if defined(HAS_ARGBGRAYROW_NEON_DOTPROD)
if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_NEON_DotProd;
if (TestCpuFlag(kCpuHasNeonDotProd)) {
ARGBGrayRow = ARGBGrayRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_NEON_DotProd;
}
}
#endif
#if defined(HAS_ARGBGRAYROW_LSX)
if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_LSX;
if (TestCpuFlag(kCpuHasLSX)) {
ARGBGrayRow = ARGBGrayRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_LSX;
}
}
#endif
#if defined(HAS_ARGBGRAYROW_LASX)
if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 16)) {
ARGBGrayRow = ARGBGrayRow_LASX;
if (TestCpuFlag(kCpuHasLASX)) {
ARGBGrayRow = ARGBGrayRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
ARGBGrayRow = ARGBGrayRow_LASX;
}
}
#endif
@ -4878,23 +4927,35 @@ int ARGBGray(uint8_t* dst_argb,
}
#endif
#if defined(HAS_ARGBGRAYROW_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_NEON;
if (TestCpuFlag(kCpuHasNEON)) {
ARGBGrayRow = ARGBGrayRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_NEON;
}
}
#endif
#if defined(HAS_ARGBGRAYROW_NEON_DOTPROD)
if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_NEON_DotProd;
if (TestCpuFlag(kCpuHasNeonDotProd)) {
ARGBGrayRow = ARGBGrayRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_NEON_DotProd;
}
}
#endif
#if defined(HAS_ARGBGRAYROW_LSX)
if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_LSX;
if (TestCpuFlag(kCpuHasLSX)) {
ARGBGrayRow = ARGBGrayRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_LSX;
}
}
#endif
#if defined(HAS_ARGBGRAYROW_LASX)
if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 16)) {
ARGBGrayRow = ARGBGrayRow_LASX;
if (TestCpuFlag(kCpuHasLASX)) {
ARGBGrayRow = ARGBGrayRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
ARGBGrayRow = ARGBGrayRow_LASX;
}
}
#endif
@ -4936,23 +4997,35 @@ int ARGBSepia(uint8_t* dst_argb,
}
#endif
#if defined(HAS_ARGBSEPIAROW_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
ARGBSepiaRow = ARGBSepiaRow_NEON;
if (TestCpuFlag(kCpuHasNEON)) {
ARGBSepiaRow = ARGBSepiaRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGBSepiaRow = ARGBSepiaRow_NEON;
}
}
#endif
#if defined(HAS_ARGBSEPIAROW_NEON_DOTPROD)
if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) {
ARGBSepiaRow = ARGBSepiaRow_NEON_DotProd;
if (TestCpuFlag(kCpuHasNeonDotProd)) {
ARGBSepiaRow = ARGBSepiaRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 8)) {
ARGBSepiaRow = ARGBSepiaRow_NEON_DotProd;
}
}
#endif
#if defined(HAS_ARGBSEPIAROW_LSX)
if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) {
ARGBSepiaRow = ARGBSepiaRow_LSX;
if (TestCpuFlag(kCpuHasLSX)) {
ARGBSepiaRow = ARGBSepiaRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
ARGBSepiaRow = ARGBSepiaRow_LSX;
}
}
#endif
#if defined(HAS_ARGBSEPIAROW_LASX)
if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 16)) {
ARGBSepiaRow = ARGBSepiaRow_LASX;
if (TestCpuFlag(kCpuHasLASX)) {
ARGBSepiaRow = ARGBSepiaRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
ARGBSepiaRow = ARGBSepiaRow_LASX;
}
}
#endif
@ -5004,23 +5077,35 @@ int ARGBColorMatrix(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBCOLORMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_NEON;
if (TestCpuFlag(kCpuHasNEON)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_NEON;
}
}
#endif
#if defined(HAS_ARGBCOLORMATRIXROW_NEON_I8MM)
if (TestCpuFlag(kCpuHasNeonI8MM) && IS_ALIGNED(width, 8)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_NEON_I8MM;
if (TestCpuFlag(kCpuHasNeonI8MM)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 8)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_NEON_I8MM;
}
}
#endif
#if defined(HAS_ARGBCOLORMATRIXROW_LSX)
if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_LSX;
if (TestCpuFlag(kCpuHasLSX)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 4)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_LSX;
}
}
#endif
#if defined(HAS_ARGBCOLORMATRIXROW_LASX)
if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_LASX;
if (TestCpuFlag(kCpuHasLASX)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_Any_LASX;
if (IS_ALIGNED(width, 8)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_LASX;
}
}
#endif
for (y = 0; y < height; ++y) {
@ -5189,18 +5274,27 @@ int ARGBQuantize(uint8_t* dst_argb,
}
#endif
#if defined(HAS_ARGBQUANTIZEROW_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
ARGBQuantizeRow = ARGBQuantizeRow_NEON;
if (TestCpuFlag(kCpuHasNEON)) {
ARGBQuantizeRow = ARGBQuantizeRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGBQuantizeRow = ARGBQuantizeRow_NEON;
}
}
#endif
#if defined(HAS_ARGBQUANTIZEROW_LSX)
if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) {
ARGBQuantizeRow = ARGBQuantizeRow_LSX;
if (TestCpuFlag(kCpuHasLSX)) {
ARGBQuantizeRow = ARGBQuantizeRow_Any_LSX;
if (IS_ALIGNED(width, 4)) {
ARGBQuantizeRow = ARGBQuantizeRow_LSX;
}
}
#endif
#if defined(HAS_ARGBQUANTIZEROW_LASX)
if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) {
ARGBQuantizeRow = ARGBQuantizeRow_LASX;
if (TestCpuFlag(kCpuHasLASX)) {
ARGBQuantizeRow = ARGBQuantizeRow_Any_LASX;
if (IS_ALIGNED(width, 8)) {
ARGBQuantizeRow = ARGBQuantizeRow_LASX;
}
}
#endif
for (y = 0; y < height; ++y) {
@ -5402,18 +5496,27 @@ int ARGBShade(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBSHADEROW_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) {
ARGBShadeRow = ARGBShadeRow_NEON;
if (TestCpuFlag(kCpuHasNEON)) {
ARGBShadeRow = ARGBShadeRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGBShadeRow = ARGBShadeRow_NEON;
}
}
#endif
#if defined(HAS_ARGBSHADEROW_LSX)
if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) {
ARGBShadeRow = ARGBShadeRow_LSX;
if (TestCpuFlag(kCpuHasLSX)) {
ARGBShadeRow = ARGBShadeRow_Any_LSX;
if (IS_ALIGNED(width, 4)) {
ARGBShadeRow = ARGBShadeRow_LSX;
}
}
#endif
#if defined(HAS_ARGBSHADEROW_LASX)
if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) {
ARGBShadeRow = ARGBShadeRow_LASX;
if (TestCpuFlag(kCpuHasLASX)) {
ARGBShadeRow = ARGBShadeRow_Any_LASX;
if (IS_ALIGNED(width, 8)) {
ARGBShadeRow = ARGBShadeRow_LASX;
}
}
#endif
@ -5585,6 +5688,11 @@ int InterpolatePlane_16(const uint16_t* src0,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_16_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
InterpolateRow_16 = InterpolateRow_16_SVE2;
}
#endif
#if defined(HAS_INTERPOLATEROW_16_SME)
if (TestCpuFlag(kCpuHasSME)) {
InterpolateRow_16 = InterpolateRow_16_SME;
@ -6418,6 +6526,11 @@ int HalfFloatPlane(const uint16_t* src_y,
HalfFloatRow = scale == 1.0f ? HalfFloat1Row_SVE2 : HalfFloatRow_SVE2;
}
#endif
#if defined(HAS_HALFFLOATROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
HalfFloatRow = scale == 1.0f ? HalfFloat1Row_SME : HalfFloatRow_SME;
}
#endif
#if defined(HAS_HALFFLOATROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
HalfFloatRow = HalfFloatRow_Any_LSX;
@ -6508,8 +6621,11 @@ int ARGBLumaColorTable(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBLUMACOLORTABLEROW_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 4)) {
ARGBLumaColorTableRow = ARGBLumaColorTableRow_NEON;
if (TestCpuFlag(kCpuHasNEON)) {
ARGBLumaColorTableRow = ARGBLumaColorTableRow_Any_NEON;
if (IS_ALIGNED(width, 4)) {
ARGBLumaColorTableRow = ARGBLumaColorTableRow_NEON;
}
}
#endif
@ -6813,6 +6929,11 @@ int YUY2ToNV12(const uint8_t* src_yuy2,
YUY2ToYRow = YUY2ToYRow_SVE2;
}
#endif
#if defined(HAS_YUY2TOYROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
YUY2ToYRow = YUY2ToYRow_SME;
}
#endif
#if defined(HAS_YUY2TOYROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
YUY2ToYRow = YUY2ToYRow_Any_LSX;
@ -6940,6 +7061,11 @@ int UYVYToNV12(const uint8_t* src_uyvy,
SplitUVRow = SplitUVRow_SVE2;
}
#endif
#if defined(HAS_SPLITUVROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
SplitUVRow = SplitUVRow_SME;
}
#endif
#if defined(HAS_SPLITUVROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
SplitUVRow = SplitUVRow_Any_LSX;
@ -7102,6 +7228,11 @@ void HalfMergeUVPlane(const uint8_t* src_u,
HalfMergeUVRow = HalfMergeUVRow_SVE2;
}
#endif
#if defined(HAS_HALFMERGEUVROW_SME)
if (TestCpuFlag(kCpuHasSME) && IS_ALIGNED(width, 2)) {
HalfMergeUVRow = HalfMergeUVRow_SME;
}
#endif
#if defined(HAS_HALFMERGEUVROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
HalfMergeUVRow = HalfMergeUVRow_Any_SSSE3;

4
source/rotate.cc
View File

@ -60,10 +60,8 @@ void TransposePlane(const uint8_t* src,
void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = TransposeWx16_C;
#endif
#if !defined(HAS_TRANSPOSEWX16_NEON)
void (*TransposeWx8)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = TransposeWx8_C;
#endif
#if defined(HAS_TRANSPOSEWX8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
@ -170,7 +168,6 @@ void TransposePlane(const uint8_t* src,
}
}
#endif
#if !defined(HAS_TRANSPOSEWX16_NEON)
// Work across the source in 8x8 tiles
while (i >= 8) {
TransposeWx8(src, src_stride, dst, dst_stride, width);
@ -178,7 +175,6 @@ void TransposePlane(const uint8_t* src,
dst += 8; // Move over 8 columns.
i -= 8;
}
#endif
if (i > 0) {
TransposeWxH(src, src_stride, dst, dst_stride, width, i);

94
source/rotate_gcc.cc
View File

@ -1078,6 +1078,7 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src,
"lea (%[dst],%[d],2),%[dst] \n"
"sub $0x8,%[cnt] \n"
"jg 1b \n"
"vzeroupper \n"
: [src] "+r"(src_b), // %[src]
[dst] "+r"(dst_b), // %[dst]
[cnt] "+r"(cnt), // %[cnt]
@ -1222,6 +1223,7 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src,
"lea (%[dst],%[d],2),%[dst] \n"
"vmovdqu16 %%zmm14,(%[dst]) \n"
"2: \n"
"vzeroupper \n"
: [dst] "+r"(dst_b), // %[dst]
[temp] "=&r"(temp) // %[temp]
: [src] "r"(src_b), // %[src]
@ -1233,7 +1235,6 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src,
"xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13",
"xmm14", "xmm15", "xmm16");
}
asm volatile("vzeroupper \n");
}
#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW)
@ -1915,16 +1916,16 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src,
"add $0x10,%[src] \n"
// Stage 1: 32-bit unpack across all 4 128-bit lanes
"vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n"
"vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n"
"vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n"
"vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n"
"vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n"
"vpunpckldq %%zmm3,%%zmm2,%%zmm17 \n"
"vpunpckhdq %%zmm1,%%zmm0,%%zmm18 \n"
"vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n"
// Stage 2: 64-bit unpack across all 4 128-bit lanes
"vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n"
"vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n"
"vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n"
"vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n"
"vpunpcklqdq %%zmm17,%%zmm16,%%zmm0 \n"
"vpunpckhqdq %%zmm17,%%zmm16,%%zmm1 \n"
"vpunpcklqdq %%zmm19,%%zmm18,%%zmm2 \n"
"vpunpckhqdq %%zmm19,%%zmm18,%%zmm3 \n"
// Store 4 full 512-bit (64 B) rows = 16 32-bit pixels each
"vmovdqu32 %%zmm0,(%[dst]) \n"
@ -1935,14 +1936,15 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src,
"lea (%[dst],%[d],2),%[dst] \n"
"sub $0x4,%[cnt] \n"
"jg 1b \n"
"vzeroupper \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[cnt] "+r"(cnt), // %[cnt]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7");
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16", "xmm17",
"xmm18", "xmm19");
width -= n;
}
if (width > 0) {
@ -1957,47 +1959,47 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src,
"vmovdqu32 (%[temp],%[s]),%%xmm3%{%%k1%}%{z%} \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm8,%%zmm0,%%zmm0 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm8,%%zmm1,%%zmm1 \n"
"vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm20,%%zmm0,%%zmm0 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm20,%%zmm1,%%zmm1 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm8,%%zmm2,%%zmm2 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm8,%%zmm3,%%zmm3 \n"
"vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm20,%%zmm2,%%zmm2 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $1,%%xmm20,%%zmm3,%%zmm3 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm8,%%zmm0,%%zmm0 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm8,%%zmm1,%%zmm1 \n"
"vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm20,%%zmm0,%%zmm0 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm20,%%zmm1,%%zmm1 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm8,%%zmm2,%%zmm2 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm8,%%zmm3,%%zmm3 \n"
"vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm20,%%zmm2,%%zmm2 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $2,%%xmm20,%%zmm3,%%zmm3 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm8,%%zmm0,%%zmm0 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm8,%%zmm1,%%zmm1 \n"
"vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm20,%%zmm0,%%zmm0 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm20,%%zmm1,%%zmm1 \n"
"lea (%[temp],%[s],2),%[temp] \n"
"vmovdqu32 (%[temp]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm8,%%zmm2,%%zmm2 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm8%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm8,%%zmm3,%%zmm3 \n"
"vmovdqu32 (%[temp]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm20,%%zmm2,%%zmm2 \n"
"vmovdqu32 (%[temp],%[s]),%%xmm20%{%%k1%}%{z%} \n"
"vinserti32x4 $3,%%xmm20,%%zmm3,%%zmm3 \n"
"vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n"
"vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n"
"vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n"
"vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n"
"vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n"
"vpunpckldq %%zmm3,%%zmm2,%%zmm17 \n"
"vpunpckhdq %%zmm1,%%zmm0,%%zmm18 \n"
"vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n"
"vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n"
"vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n"
"vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n"
"vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n"
"vpunpcklqdq %%zmm17,%%zmm16,%%zmm0 \n"
"vpunpckhqdq %%zmm17,%%zmm16,%%zmm1 \n"
"vpunpcklqdq %%zmm19,%%zmm18,%%zmm2 \n"
"vpunpckhqdq %%zmm19,%%zmm18,%%zmm3 \n"
"vmovdqu32 %%zmm0,(%[dst]) \n"
"cmp $1,%[width] \n"
@ -2008,6 +2010,7 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src,
"lea (%[dst],%[d],2),%[dst] \n"
"vmovdqu32 %%zmm2,(%[dst]) \n"
"2: \n"
"vzeroupper \n"
: [dst] "+r"(dst), // %[dst]
[temp] "=&r"(temp) // %[temp]
: [src] "r"(src), // %[src]
@ -2015,10 +2018,9 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src,
[d] "r"(d), // %[d]
[col_mask] "r"(col_mask), // %[col_mask]
[width] "r"(width) // %[width]
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7", "xmm8");
: "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm16",
"xmm17", "xmm18", "xmm19", "xmm20");
}
asm volatile("vzeroupper \n");
}
#endif // defined(HAS_TRANSPOSEWX16_32_AVX512BW)

71
source/rotate_neon64.cc
View File

@ -26,6 +26,77 @@ extern "C" {
// This module is for GCC Neon armv8 64 bit.
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
void TransposeWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
const uint8_t* temp;
asm volatile(
"1: \n"
"mov %[temp], %[src] \n"
"ld1 {v0.8b}, [%[temp]], %[s] \n"
"ld1 {v1.8b}, [%[temp]], %[s] \n"
"ld1 {v2.8b}, [%[temp]], %[s] \n"
"ld1 {v3.8b}, [%[temp]], %[s] \n"
"ld1 {v4.8b}, [%[temp]], %[s] \n"
"ld1 {v5.8b}, [%[temp]], %[s] \n"
"ld1 {v6.8b}, [%[temp]], %[s] \n"
"ld1 {v7.8b}, [%[temp]] \n"
"add %[src], %[src], #8 \n"
// Stage 1: 8-bit transpose
"trn1 v16.8b, v0.8b, v1.8b \n"
"trn2 v17.8b, v0.8b, v1.8b \n"
"trn1 v18.8b, v2.8b, v3.8b \n"
"trn2 v19.8b, v2.8b, v3.8b \n"
"trn1 v20.8b, v4.8b, v5.8b \n"
"trn2 v21.8b, v4.8b, v5.8b \n"
"trn1 v22.8b, v6.8b, v7.8b \n"
"trn2 v23.8b, v6.8b, v7.8b \n"
// Stage 2: 16-bit transpose
"trn1 v0.4h, v16.4h, v18.4h \n"
"trn1 v1.4h, v17.4h, v19.4h \n"
"trn2 v2.4h, v16.4h, v18.4h \n"
"trn2 v3.4h, v17.4h, v19.4h \n"
"trn1 v4.4h, v20.4h, v22.4h \n"
"trn1 v5.4h, v21.4h, v23.4h \n"
"trn2 v6.4h, v20.4h, v22.4h \n"
"trn2 v7.4h, v21.4h, v23.4h \n"
// Stage 3: 32-bit transpose
"trn1 v16.2s, v0.2s, v4.2s \n"
"trn1 v17.2s, v1.2s, v5.2s \n"
"trn1 v18.2s, v2.2s, v6.2s \n"
"trn1 v19.2s, v3.2s, v7.2s \n"
"trn2 v20.2s, v0.2s, v4.2s \n"
"trn2 v21.2s, v1.2s, v5.2s \n"
"trn2 v22.2s, v2.2s, v6.2s \n"
"trn2 v23.2s, v3.2s, v7.2s \n"
"subs %w[w], %w[w], #8 \n"
"st1 {v16.8b}, [%[dst]], %[d] \n"
"st1 {v17.8b}, [%[dst]], %[d] \n"
"st1 {v18.8b}, [%[dst]], %[d] \n"
"st1 {v19.8b}, [%[dst]], %[d] \n"
"st1 {v20.8b}, [%[dst]], %[d] \n"
"st1 {v21.8b}, [%[dst]], %[d] \n"
"st1 {v22.8b}, [%[dst]], %[d] \n"
"st1 {v23.8b}, [%[dst]], %[d] \n"
"b.gt 1b \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[w] "+r"(width), // %[w]
[temp] "=&r"(temp) // %[temp]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
void TransposeWx16_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,

28
source/rotate_sme.cc
View File

@ -23,8 +23,16 @@ extern "C" {
// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
// are callee-saved. List "memory" first, then "cc", then registers.
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
defined(__aarch64__)
#if !defined(LIBYUV_DISABLE_SME) && \
(defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__)
#if defined(__clang__)
#pragma clang attribute push( \
__attribute__((target("arch=armv8.5-a+i8mm+sme"))), apply_to = function)
#elif defined(__GNUC__)
#pragma GCC push_options
#pragma GCC target("arch=armv8.5-a+i8mm+sve2+sme")
#endif
__arm_locally_streaming __arm_new("za") void TransposeWxH_SME(
const uint8_t* src,
@ -81,7 +89,7 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_SME(
[dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride]
[block_width] "r"(block_width), // %[block_width]
[block_height] "r"(block_height) // %[block_height]
: "cc", "memory", "p0", "w12", "za");
: "memory", "cc", "p0", "w12", "za");
src2 += vl;
width2 -= vl;
@ -157,7 +165,7 @@ __arm_locally_streaming __arm_new("za") void TransposeUVWxH_SME(
[dst_stride_b] "r"((ptrdiff_t)dst_stride_b), // %[dst_stride_b]
[block_width] "r"(block_width * 2), // %[block_width]
[block_height] "r"(block_height) // %[block_height]
: "cc", "memory", "p0", "w12", "za");
: "memory", "cc", "p0", "w12", "za");
src2 += 2 * vl;
width2 -= vl;
@ -225,7 +233,7 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_32_SME(
[dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride]
[block_width] "r"(block_width), // %[block_width]
[block_height] "r"(block_height) // %[block_height]
: "cc", "memory", "p0", "w12", "za");
: "memory", "cc", "p0", "w12", "za");
src2 += vl * 4;
width2 -= vl;
@ -304,8 +312,14 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_16_SME(
} while (height > 0);
}
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
// defined(__aarch64__)
#if defined(__clang__)
#pragma clang attribute pop
#elif defined(__GNUC__)
#pragma GCC pop_options
#endif
#endif // !defined(LIBYUV_DISABLE_SME) && (defined(CLANG_HAS_SME) ||
// defined(GCC_HAS_SME)) && defined(__aarch64__)
#ifdef __cplusplus
} // extern "C"

16
source/rotate_sve.cc
View File

@ -18,6 +18,14 @@ extern "C" {
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#if defined(__clang__)
#pragma clang attribute push( \
__attribute__((target("arch=armv8.5-a+i8mm+sve2"))), apply_to = function)
#elif defined(__GNUC__)
#pragma GCC push_options
#pragma GCC target("arch=armv8.5-a+i8mm+sve2")
#endif
// Transpose 32 bit values (ARGB) in Wx4 tiles using SVE2 with predication
// for tail widths.
void TransposeWx4_32_SVE2(const uint8_t* src,
@ -106,7 +114,7 @@ void TransposeWx4_32_SVE2(const uint8_t* src,
[dst3] "+r"(dst3), // %[dst3]
[width] "+r"(width) // %[width]
: [s4] "r"(s4) // %[s4]
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
}
void TransposeWx8_16_SVE2(const uint16_t* src,
@ -271,6 +279,12 @@ void TransposeWx8_16_SVE2(const uint16_t* src,
"z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23");
}
#if defined(__clang__)
#pragma clang attribute pop
#elif defined(__GNUC__)
#pragma GCC pop_options
#endif
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#ifdef __cplusplus

180
source/row_any.cc
View File

@ -1453,6 +1453,18 @@ ANY11(ARGBToARGB1555Row_Any_WASMSIMD, ARGBToARGB1555Row_WASMSIMD, 0, 4, 2, 7)
#ifdef HAS_ARGBTOARGB4444ROW_WASMSIMD
ANY11(ARGBToARGB4444Row_Any_WASMSIMD, ARGBToARGB4444Row_WASMSIMD, 0, 4, 2, 7)
#endif
#ifdef HAS_ARGBGRAYROW_NEON
ANY11(ARGBGrayRow_Any_NEON, ARGBGrayRow_NEON, 0, 4, 4, 7)
#endif
#ifdef HAS_ARGBGRAYROW_NEON_DOTPROD
ANY11(ARGBGrayRow_Any_NEON_DotProd, ARGBGrayRow_NEON_DotProd, 0, 4, 4, 7)
#endif
#ifdef HAS_ARGBGRAYROW_LSX
ANY11(ARGBGrayRow_Any_LSX, ARGBGrayRow_LSX, 0, 4, 4, 7)
#endif
#ifdef HAS_ARGBGRAYROW_LASX
ANY11(ARGBGrayRow_Any_LASX, ARGBGrayRow_LASX, 0, 4, 4, 15)
#endif
#undef ANY11
// Any 1 to 1 blended. Destination is read, modify, write.
@ -1669,8 +1681,148 @@ ANY11P(ARGBPolynomialRow_Any_NEON,
4,
1)
#endif
#ifdef HAS_ARGBCOLORMATRIXROW_NEON
ANY11P(ARGBColorMatrixRow_Any_NEON,
ARGBColorMatrixRow_NEON,
const int8_t*,
4,
4,
7)
#endif
#ifdef HAS_ARGBCOLORMATRIXROW_NEON_I8MM
ANY11P(ARGBColorMatrixRow_Any_NEON_I8MM,
ARGBColorMatrixRow_NEON_I8MM,
const int8_t*,
4,
4,
7)
#endif
#ifdef HAS_ARGBCOLORMATRIXROW_LSX
ANY11P(ARGBColorMatrixRow_Any_LSX,
ARGBColorMatrixRow_LSX,
const int8_t*,
4,
4,
3)
#endif
#ifdef HAS_ARGBCOLORMATRIXROW_LASX
ANY11P(ARGBColorMatrixRow_Any_LASX,
ARGBColorMatrixRow_LASX,
const int8_t*,
4,
4,
7)
#endif
#undef ANY11P
// Any 1 in-place.
#define ANY1I(NAMEANY, ANY_SIMD, BPP, MASK) \
void NAMEANY(uint8_t* dst_ptr, int width) { \
SIMD_ALIGNED(uint8_t vout[(MASK + 1) * BPP]); \
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(dst_ptr, n); \
} \
ptrdiff_t np = n; \
memcpy(vout, dst_ptr + np * BPP, r * BPP); \
ANY_SIMD(vout, MASK + 1); \
memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
#ifdef HAS_ARGBSEPIAROW_NEON
ANY1I(ARGBSepiaRow_Any_NEON, ARGBSepiaRow_NEON, 4, 7)
#endif
#ifdef HAS_ARGBSEPIAROW_NEON_DOTPROD
ANY1I(ARGBSepiaRow_Any_NEON_DotProd, ARGBSepiaRow_NEON_DotProd, 4, 7)
#endif
#ifdef HAS_ARGBSEPIAROW_LSX
ANY1I(ARGBSepiaRow_Any_LSX, ARGBSepiaRow_LSX, 4, 7)
#endif
#ifdef HAS_ARGBSEPIAROW_LASX
ANY1I(ARGBSepiaRow_Any_LASX, ARGBSepiaRow_LASX, 4, 15)
#endif
#undef ANY1I
// Any 1 in-place with 3 int parameters.
#define ANY1I3(NAMEANY, ANY_SIMD, BPP, MASK) \
void NAMEANY(uint8_t* dst_ptr, int p0, int p1, int p2, int width) { \
SIMD_ALIGNED(uint8_t vout[(MASK + 1) * BPP]); \
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(dst_ptr, p0, p1, p2, n); \
} \
ptrdiff_t np = n; \
memcpy(vout, dst_ptr + np * BPP, r * BPP); \
ANY_SIMD(vout, p0, p1, p2, MASK + 1); \
memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
#ifdef HAS_ARGBQUANTIZEROW_NEON
ANY1I3(ARGBQuantizeRow_Any_NEON, ARGBQuantizeRow_NEON, 4, 7)
#endif
#ifdef HAS_ARGBQUANTIZEROW_LSX
ANY1I3(ARGBQuantizeRow_Any_LSX, ARGBQuantizeRow_LSX, 4, 3)
#endif
#ifdef HAS_ARGBQUANTIZEROW_LASX
ANY1I3(ARGBQuantizeRow_Any_LASX, ARGBQuantizeRow_LASX, 4, 7)
#endif
#undef ANY1I3
// Any 1 to 1 with trailing parameter.
#define ANY11PT(NAMEANY, ANY_SIMD, T, SBPP, BPP, MASK) \
void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width, T param) { \
SIMD_ALIGNED(uint8_t vin[(MASK + 1) * SBPP]); \
SIMD_ALIGNED(uint8_t vout[(MASK + 1) * BPP]); \
MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(src_ptr, dst_ptr, n, param); \
} \
ptrdiff_t np = n; \
memcpy(vin, src_ptr + np * SBPP, r * SBPP); \
ANY_SIMD(vin, vout, MASK + 1, param); \
memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
#ifdef HAS_ARGBSHADEROW_NEON
ANY11PT(ARGBShadeRow_Any_NEON, ARGBShadeRow_NEON, uint32_t, 4, 4, 7)
#endif
#ifdef HAS_ARGBSHADEROW_LSX
ANY11PT(ARGBShadeRow_Any_LSX, ARGBShadeRow_LSX, uint32_t, 4, 4, 3)
#endif
#ifdef HAS_ARGBSHADEROW_LASX
ANY11PT(ARGBShadeRow_Any_LASX, ARGBShadeRow_LASX, uint32_t, 4, 4, 7)
#endif
#undef ANY11PT
#ifdef HAS_ARGBLUMACOLORTABLEROW_NEON
void ARGBLumaColorTableRow_Any_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
const uint8_t* luma,
const uint32_t lumacoeff) {
SIMD_ALIGNED(uint8_t vin[4 * 4]);
SIMD_ALIGNED(uint8_t vout[4 * 4]);
MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */
MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */
int r = width & 3;
int n = width & ~3;
if (n > 0) {
ARGBLumaColorTableRow_NEON(src_argb, dst_argb, n, luma, lumacoeff);
}
ptrdiff_t np = n;
memcpy(vin, src_argb + np * 4, r * 4);
ARGBLumaColorTableRow_NEON(vin, vout, 4, luma, lumacoeff);
memcpy(dst_argb + np * 4, vout, r * 4);
}
#endif
// Any 1 to 1 with type
#define ANY11T(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \
void NAMEANY(const STYPE* src_ptr, DTYPE* dst_ptr, int width) { \
@ -2190,6 +2342,25 @@ ANY11HWS(HalfWidthRow_16To8_Any_NEON,
uint16_t,
uint8_t,
7)
void HalfWidthRow_16To8_Odd_NEON(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
int n = width - 1;
if (n > 0) {
if (IS_ALIGNED(n, 8)) {
HalfWidthRow_16To8_NEON(src_uv, src_uv_stride, dst_uv, scale, n);
} else {
HalfWidthRow_16To8_Any_NEON(src_uv, src_uv_stride, dst_uv, scale, n);
}
src_uv += (ptrdiff_t)n * 2;
dst_uv += n;
}
const uint16_t* src_uv1 = src_uv + src_uv_stride;
uint32_t avg = (uint32_t)src_uv[0] + (uint32_t)src_uv1[0] + 1u;
dst_uv[0] = (uint8_t)((avg * (uint32_t)scale) >> 17);
}
#endif
#undef ANY11HWS
@ -2811,9 +2982,6 @@ ANY11MC(RGBToYMatrixRow_Any_LSX, RGBToYMatrixRow_LSX, 3, 15)
#ifdef HAS_RGBTOYMATRIXROW_LASX
ANY11MC(RGBToYMatrixRow_Any_LASX, RGBToYMatrixRow_LASX, 3, 31)
#endif
#ifdef HAS_ARGBTOYMATRIXROW_RVV
ANY11MC(ARGBToYMatrixRow_Any_RVV, ARGBToYMatrixRow_RVV, 4, 15)
#endif
#ifdef HAS_ARGBTOYMATRIXROW_WASMSIMD
ANY11MC(ARGBToYMatrixRow_Any_WASMSIMD, ARGBToYMatrixRow_WASMSIMD, 4, 15)
#endif
@ -2952,12 +3120,6 @@ ANY11S(AYUVToVURow_Any_AVX2, AYUVToVURow_AVX2, 0, 4, 15)
ANY11S(AYUVToUVRow_Any_NEON, AYUVToUVRow_NEON, 0, 4, 15)
ANY11S(AYUVToVURow_Any_NEON, AYUVToVURow_NEON, 0, 4, 15)
#endif
#ifdef HAS_AYUVTOUVROW_SVE2
ANY11S(AYUVToUVRow_Any_SVE2, AYUVToUVRow_SVE2, 0, 4, 1)
#endif
#ifdef HAS_AYUVTOVUROW_SVE2
ANY11S(AYUVToVURow_Any_SVE2, AYUVToVURow_SVE2, 0, 4, 1)
#endif
#undef ANY11S
#define ANYDETILE(NAMEANY, ANY_SIMD, T, BPP, MASK) \

98
source/row_gcc.cc
View File

@ -11431,6 +11431,56 @@ void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2,
}
#endif // HAS_YUY2TOUVROW_AVX512BW
#ifdef HAS_YUY2TOUV422ROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void YUY2ToUV422Row_AVX512BW(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
uintptr_t temp;
asm volatile(
"sub %1,%2 \n"
"sub $0x20,%3 \n"
"jl 2f \n"
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0 \n"
"lea 0x40(%0),%0 \n"
"vpsrld $0x8,%%zmm0,%%zmm1 \n"
"vpsrld $0x18,%%zmm0,%%zmm0 \n"
"vpmovdb %%zmm1,(%1) \n"
"vpmovdb %%zmm0,0x00(%1,%2,1) \n"
"lea 0x10(%1),%1 \n"
"sub $0x20,%3 \n"
"jge 1b \n"
"2: \n"
"add $0x20,%3 \n"
"jle 99f \n"
"add $0x1,%k3 \n"
"shr $0x1,%k3 \n"
"mov $-1,%k[temp] \n"
"bzhi %k3,%k[temp],%k[temp] \n"
"kmovw %k[temp],%%k1 \n"
"vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
"vpsrld $0x8,%%zmm0,%%zmm1 \n"
"vpsrld $0x18,%%zmm0,%%zmm0 \n"
"vpmovdb %%zmm1,(%1)%{%%k1%} \n"
"vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(src_yuy2), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
"+r"(width), // %3
[temp] "=&r"(temp) // %[temp]
:
: "memory", "cc", "xmm0", "xmm1", "k1");
}
#endif // HAS_YUY2TOUV422ROW_AVX512BW
#ifdef HAS_UYVYTOYROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
@ -11536,6 +11586,54 @@ void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy,
}
#endif // HAS_UYVYTOUVROW_AVX512BW
#ifdef HAS_UYVYTOUV422ROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void UYVYToUV422Row_AVX512BW(const uint8_t* src_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
uintptr_t temp;
asm volatile(
"sub %1,%2 \n"
"sub $0x20,%3 \n"
"jl 2f \n"
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0 \n"
"lea 0x40(%0),%0 \n"
"vpmovdb %%zmm0,(%1) \n"
"vpsrld $0x10,%%zmm0,%%zmm0 \n"
"vpmovdb %%zmm0,0x00(%1,%2,1) \n"
"lea 0x10(%1),%1 \n"
"sub $0x20,%3 \n"
"jge 1b \n"
"2: \n"
"add $0x20,%3 \n"
"jle 99f \n"
"add $0x1,%k3 \n"
"shr $0x1,%k3 \n"
"mov $-1,%k[temp] \n"
"bzhi %k3,%k[temp],%k[temp] \n"
"kmovw %k[temp],%%k1 \n"
"vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
"vpmovdb %%zmm0,(%1)%{%%k1%} \n"
"vpsrld $0x10,%%zmm0,%%zmm0 \n"
"vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(src_uyvy), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
"+r"(width), // %3
[temp] "=&r"(temp) // %[temp]
:
: "memory", "cc", "xmm0", "k1");
}
#endif // HAS_UYVYTOUV422ROW_AVX512BW
#if defined(HAS_ARGBBLENDROW_AVX2) || defined(HAS_ARGBBLENDROW_AVX512BW)
// Shuffle table for isolating alpha.
SIMD_ALIGNED(static const uvec8 kShuffleAlpha[2]) = {

14
source/row_neon64.cc
View File

@ -32,6 +32,14 @@ extern "C" {
// This module is for GCC Neon armv8 64 bit.
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#if defined(__clang__)
#pragma clang attribute push(__attribute__((target("dotprod,i8mm"))), \
apply_to = function)
#elif defined(__GNUC__)
#pragma GCC push_options
#pragma GCC target("+dotprod+i8mm")
#endif
// v0.8h: Y
// v1.16b: 8U, 8V
@ -6115,6 +6123,12 @@ void ARGBAffineRow_NEON(const uint8_t* src_argb,
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
#if defined(__clang__)
#pragma clang attribute pop
#elif defined(__GNUC__)
#pragma GCC pop_options
#endif
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus

514
source/row_sme.cc
View File

@ -10,6 +10,18 @@
#include "libyuv/compare_row.h"
#include "libyuv/row.h"
#if !defined(LIBYUV_DISABLE_SME) && \
(defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__)
#if defined(__clang__)
#pragma clang attribute push( \
__attribute__((target("arch=armv8.5-a+i8mm+sme"))), apply_to = function)
#elif defined(__GNUC__)
#pragma GCC push_options
#pragma GCC target("arch=armv8.5-a+i8mm+sve2+sme")
#endif
#endif
#include "libyuv/row_sve.h"
#ifdef __cplusplus
@ -24,17 +36,8 @@ extern "C" {
// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
// are callee-saved. List "memory" first, then "cc", then registers.
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
defined(__aarch64__)
#define RGBTOARGB8_SVE_2X \
/* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
"uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
"uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \
"uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
"uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \
"uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \
"uqshrnt z18.b, z22.h, #6 \n" /* R1 */
#if !defined(LIBYUV_DISABLE_SME) && \
(defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__)
__arm_locally_streaming void I444ToARGBRow_SME(
const uint8_t* src_y,
@ -44,44 +47,7 @@ __arm_locally_streaming void I444ToARGBRow_SME(
const struct YuvConstants* yuvconstants,
int width) {
// Streaming-SVE only, no use of ZA tile.
uint64_t vl;
asm volatile(
"cntb %[vl] \n"
"ptrue p0.b \n" //
YUVTORGB_SVE_SETUP
"dup z19.b, #255 \n" // A
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p1.b \n"
"1: \n" //
READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X
"subs %w[width], %w[width], %w[vl] \n"
"st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
"incb %[dst_argb], all, mul #4 \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p1.b, wzr, %w[width] \n" //
READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X
"st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
"99: \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
: "cc", "memory", YUVTORGB_SVE_REGS);
I444ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
}
__arm_locally_streaming void I444ToRGB24Row_SME(
@ -465,51 +431,31 @@ __arm_locally_streaming void ARGBMultiplyRow_SME(const uint8_t* src_argb,
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "z2", "p0", "p1");
: "memory", "cc", "z0", "z1", "z2", "p0");
}
__arm_locally_streaming void MergeUVRow_SME(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
int width) {
// Streaming-SVE only, no use of ZA tile.
int vl;
asm volatile(
"cntb %x[vl] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
MergeUVRow_SVE_SC(src_u, src_v, dst_uv, width);
}
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p0.b \n"
"1: \n"
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
"ld1b {z2.b}, p0/z, [%[src_v]] \n"
"incb %[src_u] \n"
"incb %[src_v] \n"
"subs %w[width], %w[width], %w[vl] \n"
"st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
"incb %[dst_uv], all, mul #2 \n"
"b.ge 1b \n"
__arm_locally_streaming void SplitUVRow_SME(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
SplitUVRow_SVE_SC(src_uv, dst_u, dst_v, width);
}
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p0.b, wzr, %w[width] \n"
"ld1b {z1.b}, p0/z, [%[src_u]] \n"
"ld1b {z2.b}, p0/z, [%[src_v]] \n"
"st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
"99: \n"
: [src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z1", "z2", "p0");
__arm_locally_streaming void HalfMergeUVRow_SME(const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_uv,
int width) {
HalfMergeUVRow_SVE_SC(src_u, src_stride_u, src_v, src_stride_v, dst_uv,
width);
}
__arm_locally_streaming void MergeUVRow_16_SME(const uint16_t* src_u,
@ -628,87 +574,14 @@ __arm_locally_streaming void Convert16To8Row_SME(const uint16_t* src_y,
__arm_locally_streaming void CopyRow_SME(const uint8_t* src,
uint8_t* dst,
int width) {
// Streaming-SVE only, no use of ZA tile.
int vl;
asm volatile(
"cntb %x[vl] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p0.b \n"
"1: \n"
"ld1b {z0.b}, p0/z, [%[src]] \n"
"incb %[src] \n"
"subs %w[width], %w[width], %w[vl] \n"
"st1b {z0.b}, p0, [%[dst]] \n"
"incb %[dst] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p0.b, wzr, %w[width] \n"
"ld1b {z0.b}, p0/z, [%[src]] \n"
"st1b {z0.b}, p0, [%[dst]] \n"
"99: \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "p0");
CopyRow_SVE_SC(src, dst, width);
}
__arm_locally_streaming static void HalfRow_SME(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
int width) {
const uint8_t* src_ptr1 = src_ptr + src_stride;
int vl;
asm volatile(
"cntb %x[vl] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p0.b \n"
"1: \n"
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
"incb %[src_ptr] \n"
"incb %[src_ptr1] \n"
"urhadd z2.b, p0/m, z2.b, z3.b \n"
"subs %w[width], %w[width], %w[vl] \n"
"st1b {z2.b}, p0, [%[dst_ptr]] \n"
"incb %[dst_ptr] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p0.b, wzr, %w[width] \n"
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
"urhadd z2.b, p0/m, z2.b, z3.b \n"
"st1b {z2.b}, p0, [%[dst_ptr]] \n"
"99: \n"
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z2", "z3", "p0");
__arm_locally_streaming void HalfRow_SME(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int width) {
HalfRow_SVE_SC(src_ptr, src_stride, dst_ptr, width);
}
__arm_locally_streaming void InterpolateRow_SME(uint8_t* dst_ptr,
@ -716,122 +589,14 @@ __arm_locally_streaming void InterpolateRow_SME(uint8_t* dst_ptr,
ptrdiff_t src_stride,
int width,
int source_y_fraction) {
int y1_fraction = source_y_fraction;
int y0_fraction = 256 - y1_fraction;
const uint8_t* src_ptr1 = src_ptr + src_stride;
if (y0_fraction == 0) {
CopyRow_SME(src_ptr1, dst_ptr, width);
return;
}
if (y0_fraction == 128) {
HalfRow_SME(dst_ptr, src_ptr, src_stride, width);
return;
}
if (y0_fraction == 256) {
CopyRow_SME(src_ptr, dst_ptr, width);
return;
}
int vl;
asm volatile(
"cntb %x[vl] \n"
"dup z0.b, %w[y0_fraction] \n"
"dup z1.b, %w[y1_fraction] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p0.b \n"
"1: \n"
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
"incb %[src_ptr] \n"
"incb %[src_ptr1] \n"
"umullb z4.h, z2.b, z0.b \n"
"umullt z2.h, z2.b, z0.b \n"
"subs %w[width], %w[width], %w[vl] \n"
"umlalb z4.h, z3.b, z1.b \n"
"umlalt z2.h, z3.b, z1.b \n"
"rshrnb z3.b, z4.h, #8 \n"
"rshrnt z3.b, z2.h, #8 \n"
"st1b {z3.b}, p0, [%[dst_ptr]] \n"
"incb %[dst_ptr] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p0.b, wzr, %w[width] \n"
"ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
"ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
"umullb z4.h, z2.b, z0.b \n"
"umullt z2.h, z2.b, z0.b \n"
"umlalb z4.h, z3.b, z1.b \n"
"umlalt z2.h, z3.b, z1.b \n"
"rshrnb z3.b, z4.h, #8 \n"
"rshrnt z3.b, z2.h, #8 \n"
"st1b {z3.b}, p0, [%[dst_ptr]] \n"
"99: \n"
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
: [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
[y1_fraction] "r"(y1_fraction) // %[y1_fraction]
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "p0");
InterpolateRow_SVE_SC(dst_ptr, src_ptr, src_stride, width, source_y_fraction);
}
__arm_locally_streaming static void HalfRow_16_SME(uint16_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int width) {
const uint16_t* src_ptr1 = src_ptr + src_stride;
int vl;
asm volatile(
"cnth %x[vl] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p0.h \n"
"1: \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
"incb %[src_ptr] \n"
"incb %[src_ptr1] \n"
"urhadd z2.h, p0/m, z2.h, z3.h \n"
"subs %w[width], %w[width], %w[vl] \n"
"st1h {z2.h}, p0, [%[dst_ptr]] \n"
"incb %[dst_ptr] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p0.h, wzr, %w[width] \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
"urhadd z2.h, p0/m, z2.h, z3.h \n"
"st1h {z2.h}, p0, [%[dst_ptr]] \n"
"99: \n"
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z2", "z3", "p0");
__arm_locally_streaming void HalfRow_16_SME(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst_ptr,
int width) {
HalfRow_16_SVE_SC(src_ptr, src_stride, dst_ptr, width);
}
__arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr,
@ -839,77 +604,126 @@ __arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr,
ptrdiff_t src_stride,
int width,
int source_y_fraction) {
int y1_fraction = source_y_fraction;
int y0_fraction = 256 - y1_fraction;
const uint16_t* src_ptr1 = src_ptr + src_stride;
InterpolateRow_16_SVE_SC(dst_ptr, src_ptr, src_stride, width,
source_y_fraction);
}
if (y0_fraction == 0) {
CopyRow_SME((const uint8_t*)src_ptr1, (uint8_t*)dst_ptr,
width * sizeof(uint16_t));
return;
}
if (y0_fraction == 128) {
HalfRow_16_SME(dst_ptr, src_ptr, src_stride, width);
return;
}
if (y0_fraction == 256) {
CopyRow_SME((const uint8_t*)src_ptr, (uint8_t*)dst_ptr,
width * sizeof(uint16_t));
return;
}
__arm_locally_streaming void DivideRow_16_SME(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
DivideRow_16_SVE_SC(src_y, dst_y, scale, width);
}
int vl;
asm volatile(
"cnth %x[vl] \n"
"subs %w[width], %w[width], %w[vl] \n"
"dup z0.h, %w[y0_fraction] \n"
"dup z1.h, %w[y1_fraction] \n"
"b.lt 2f \n"
__arm_locally_streaming void HalfFloatRow_SME(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
HalfFloatRow_SVE_SC(src, dst, scale, width);
}
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p0.h \n"
"1: \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
"incb %[src_ptr] \n"
"incb %[src_ptr1] \n"
"umullb z4.s, z2.h, z0.h \n"
"umullt z2.s, z2.h, z0.h \n"
"subs %w[width], %w[width], %w[vl] \n"
"umlalb z4.s, z3.h, z1.h \n"
"umlalt z2.s, z3.h, z1.h \n"
"rshrnb z3.h, z4.s, #8 \n"
"rshrnt z3.h, z2.s, #8 \n"
"st1h {z3.h}, p0, [%[dst_ptr]] \n"
"incb %[dst_ptr] \n"
"b.ge 1b \n"
__arm_locally_streaming void HalfFloat1Row_SME(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
HalfFloat1Row_SVE_SC(src, dst, scale, width);
}
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
__arm_locally_streaming void ARGBToRGB565Row_SME(const uint8_t* src_argb,
uint8_t* dst_rgb,
int width) {
ARGBToRGB565Row_SVE_SC(src_argb, dst_rgb, width);
}
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p0.h, wzr, %w[width] \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
"umullb z4.s, z2.h, z0.h \n"
"umullt z2.s, z2.h, z0.h \n"
"umlalb z4.s, z3.h, z1.h \n"
"umlalt z2.s, z3.h, z1.h \n"
"rshrnb z3.h, z4.s, #8 \n"
"rshrnt z3.h, z2.s, #8 \n"
"st1h {z3.h}, p0, [%[dst_ptr]] \n"
__arm_locally_streaming void ARGBToRGB565DitherRow_SME(const uint8_t* src_argb,
uint8_t* dst_rgb,
uint32_t dither4,
int width) {
ARGBToRGB565DitherRow_SVE_SC(src_argb, dst_rgb, dither4, width);
}
"99: \n"
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
: [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
[y1_fraction] "r"(y1_fraction) // %[y1_fraction]
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
__arm_locally_streaming void ARGB1555ToARGBRow_SME(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width) {
ARGB1555ToARGBRow_SVE_SC(src_argb1555, dst_argb, width);
}
__arm_locally_streaming void AYUVToUVRow_SME(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_uv,
int width) {
AYUVToUVRow_SVE_SC(src_ayuv, src_stride_ayuv, dst_uv, width);
}
__arm_locally_streaming void AYUVToVURow_SME(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_vu,
int width) {
AYUVToVURow_SVE_SC(src_ayuv, src_stride_ayuv, dst_vu, width);
}
__arm_locally_streaming void RAWToARGBRow_SME(const uint8_t* src_raw,
uint8_t* dst_argb,
int width) {
RAWToARGBRow_SVE_SC(src_raw, dst_argb, width);
}
__arm_locally_streaming void RAWToRGBARow_SME(const uint8_t* src_raw,
uint8_t* dst_rgba,
int width) {
RAWToRGBARow_SVE_SC(src_raw, dst_rgba, width);
}
__arm_locally_streaming void RGB24ToARGBRow_SME(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width) {
RGB24ToARGBRow_SVE_SC(src_rgb24, dst_argb, width);
}
__arm_locally_streaming void RAWToRGB24Row_SME(const uint8_t* src_raw,
uint8_t* dst_rgb24,
int width) {
RAWToRGB24Row_SVE_SC(src_raw, dst_rgb24, width);
}
__arm_locally_streaming void ARGBToRGB24Row_SME(const uint8_t* src_argb,
uint8_t* dst_rgb,
int width) {
ARGBToRGB24Row_SVE_SC(src_argb, dst_rgb, width);
}
__arm_locally_streaming void ARGBToRAWRow_SME(const uint8_t* src_argb,
uint8_t* dst_rgb,
int width) {
ARGBToRAWRow_SVE_SC(src_argb, dst_rgb, width);
}
__arm_locally_streaming void YUY2ToYRow_SME(const uint8_t* src_yuy2,
uint8_t* dst_y,
int width) {
YUY2ToYRow_SVE_SC(src_yuy2, dst_y, width);
}
__arm_locally_streaming void UYVYToYRow_SME(const uint8_t* src_uyvy,
uint8_t* dst_y,
int width) {
UYVYToYRow_SVE_SC(src_uyvy, dst_y, width);
}
__arm_locally_streaming void YUY2ToUVRow_SME(const uint8_t* src_yuy2,
int src_stride_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
YUY2ToUVRow_SVE_SC(src_yuy2, src_stride_yuy2, dst_u, dst_v, width);
}
__arm_locally_streaming void UYVYToUVRow_SME(const uint8_t* src_uyvy,
int src_stride_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
UYVYToUVRow_SVE_SC(src_uyvy, src_stride_uyvy, dst_u, dst_v, width);
}
__arm_locally_streaming void HalfRow_16To8_SME(const uint16_t* src_ptr,
@ -967,7 +781,7 @@ __arm_locally_streaming void HalfRow_16To8_SME(const uint16_t* src_ptr,
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
: [shift] "r"(shift) // %[shift]
: "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0");
: "memory", "cc", "z2", "z3", "z31", "p0");
}
__arm_locally_streaming void HalfWidthRow_16To8_SME(const uint16_t* src_ptr,
@ -1157,8 +971,14 @@ __arm_locally_streaming void ARGBAlphaBlendRow_SME(const uint8_t* src_argb0,
}
#endif // HAS_ARGBALPHABLENDROW_SME
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
// defined(__aarch64__)
#if defined(__clang__)
#pragma clang attribute pop
#elif defined(__GNUC__)
#pragma GCC pop_options
#endif
#endif // !defined(LIBYUV_DISABLE_SME) && (defined(CLANG_HAS_SME) ||
// defined(GCC_HAS_SME)) && defined(__aarch64__)
#ifdef __cplusplus
} // extern "C"

1181
source/row_sve.cc
View File

File diff suppressed because it is too large Load Diff

56
source/row_win.cc
View File

@ -6778,6 +6778,35 @@ void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2,
}
#endif // HAS_YUY2TOUVROW_AVX512BW
#ifdef HAS_YUY2TOUV422ROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void YUY2ToUV422Row_AVX512BW(const uint8_t* src_yuy2,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
while (width >= 32) {
__m512i r0 = _mm512_loadu_si512((const __m512i*)src_yuy2);
src_yuy2 += 64;
_mm_storeu_si128((__m128i*)dst_u,
_mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 8)));
_mm_storeu_si128((__m128i*)dst_v,
_mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 24)));
dst_u += 16;
dst_v += 16;
width -= 32;
}
if (width > 0) {
__mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
__m512i r0 = _mm512_maskz_loadu_epi32(k, src_yuy2);
_mm_mask_storeu_epi8(dst_u, k,
_mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 8)));
_mm_mask_storeu_epi8(dst_v, k,
_mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 24)));
}
_mm256_zeroupper();
}
#endif // HAS_YUY2TOUV422ROW_AVX512BW
#ifdef HAS_UYVYTOYROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
@ -6832,6 +6861,33 @@ void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy,
}
#endif // HAS_UYVYTOUVROW_AVX512BW
#ifdef HAS_UYVYTOUV422ROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void UYVYToUV422Row_AVX512BW(const uint8_t* src_uyvy,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
while (width >= 32) {
__m512i r0 = _mm512_loadu_si512((const __m512i*)src_uyvy);
src_uyvy += 64;
_mm_storeu_si128((__m128i*)dst_u, _mm512_cvtepi32_epi8(r0));
_mm_storeu_si128((__m128i*)dst_v,
_mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 16)));
dst_u += 16;
dst_v += 16;
width -= 32;
}
if (width > 0) {
__mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
__m512i r0 = _mm512_maskz_loadu_epi32(k, src_uyvy);
_mm_mask_storeu_epi8(dst_u, k, _mm512_cvtepi32_epi8(r0));
_mm_mask_storeu_epi8(dst_v, k,
_mm512_cvtepi32_epi8(_mm512_srli_epi32(r0, 16)));
}
_mm256_zeroupper();
}
#endif // HAS_UYVYTOUV422ROW_AVX512BW
#ifdef HAS_ARGBSEPIAROW_AVX512VNNI
LIBYUV_TARGET_AVX512VNNI
void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) {

46
source/scale_sme.cc
View File

@ -22,8 +22,16 @@ extern "C" {
// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
// are callee-saved. List "memory" first, then "cc", then registers.
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
defined(__aarch64__)
#if !defined(LIBYUV_DISABLE_SME) && \
(defined(CLANG_HAS_SME) || defined(GCC_HAS_SME)) && defined(__aarch64__)
#if defined(__clang__)
#pragma clang attribute push( \
__attribute__((target("arch=armv8.5-a+i8mm+sme"))), apply_to = function)
#elif defined(__GNUC__)
#pragma GCC push_options
#pragma GCC target("arch=armv8.5-a+i8mm+sve2+sme")
#endif
__arm_locally_streaming void ScaleRowDown2_SME(const uint8_t* src_ptr,
ptrdiff_t src_stride,
@ -58,7 +66,7 @@ __arm_locally_streaming void ScaleRowDown2_SME(const uint8_t* src_ptr,
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[dst_ptr] "+r"(dst), // %[dst_ptr]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "p0");
}
@ -96,7 +104,7 @@ __arm_locally_streaming void ScaleRowDown2_16_SME(const uint16_t* src_ptr,
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[dst_ptr] "+r"(dst), // %[dst_ptr]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "p0");
}
@ -136,7 +144,7 @@ __arm_locally_streaming void ScaleRowDown2Linear_SME(const uint8_t* src_ptr,
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[dst_ptr] "+r"(dst), // %[dst_ptr]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "p0");
}
@ -176,7 +184,7 @@ __arm_locally_streaming void ScaleRowDown2Linear_16_SME(const uint16_t* src_ptr,
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[dst_ptr] "+r"(dst), // %[dst_ptr]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "p0");
}
@ -227,7 +235,7 @@ __arm_locally_streaming void ScaleRowDown2Box_SME(const uint8_t* src_ptr,
[src2_ptr] "+r"(src2_ptr), // %[src2_ptr]
[dst_ptr] "+r"(dst), // %[dst_ptr]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
}
@ -280,7 +288,7 @@ __arm_locally_streaming void ScaleRowDown2Box_16_SME(const uint16_t* src_ptr,
[src2_ptr] "+r"(src2_ptr), // %[src2_ptr]
[dst_ptr] "+r"(dst), // %[dst_ptr]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
}
@ -320,7 +328,7 @@ __arm_locally_streaming void ScaleUVRowDown2_SME(const uint8_t* src_uv,
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "p0");
}
@ -361,7 +369,7 @@ __arm_locally_streaming void ScaleUVRowDown2Linear_SME(const uint8_t* src_uv,
: [src_uv] "+r"(src_uv), // %[src_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "p0", "p1");
}
@ -413,7 +421,7 @@ __arm_locally_streaming void ScaleUVRowDown2Box_SME(const uint8_t* src_uv,
[src2_uv] "+r"(src2_uv), // %[src2_uv]
[dst_uv] "+r"(dst_uv), // %[dst_uv]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
"p1");
@ -454,7 +462,7 @@ __arm_locally_streaming void ScaleARGBRowDown2_SME(const uint8_t* src_argb,
: [src_argb] "+r"(src_argb), // %[src_argb]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "p0");
}
@ -496,7 +504,7 @@ __arm_locally_streaming void ScaleARGBRowDown2Linear_SME(
: [src_argb] "+r"(src_argb), // %[src_argb]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "p0", "p1");
}
@ -548,7 +556,7 @@ __arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb,
[src2_argb] "+r"(src2_argb), // %[src2_argb]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[dst_width] "+r"(dst_width), // %[dst_width]
[vl] "=r"(vl) // %[vl]
[vl] "=&r"(vl) // %[vl]
:
: "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
"p1");
@ -619,8 +627,14 @@ __arm_locally_streaming void ScaleUVAddCols3_SME(int dst_width,
"p2", "p3");
}
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
// defined(__aarch64__)
#if defined(__clang__)
#pragma clang attribute pop
#elif defined(__GNUC__)
#pragma GCC pop_options
#endif
#endif // !defined(LIBYUV_DISABLE_SME) && (defined(CLANG_HAS_SME) ||
// defined(GCC_HAS_SME)) && defined(__aarch64__)
#ifdef __cplusplus
} // extern "C"

10
unit_test/convert_argb_test.cc
View File

@ -247,21 +247,15 @@ TESTBPTOP(P410, uint16_t, 2, 1, 1, I420, uint8_t, 1, 2, 2, 10, 1, 1)
memset(dst_argb_c + OFF, 1, kStrideB * kHeight); \
memset(dst_argb_opt + OFF, 101, kStrideB * kHeight); \
MaskCpuFlags(disable_cpu_flags_); \
double time0 = get_time(); \
FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
src_v + OFF, kStrideUV, dst_argb_c + OFF, kStrideB, \
kWidth, NEG kHeight); \
double time1 = get_time(); \
MaskCpuFlags(benchmark_cpu_info_); \
for (int i = 0; i < benchmark_iterations_; ++i) { \
FMT_PLANAR##To##FMT_B(src_y + OFF, kWidth, src_u + OFF, kStrideUV, \
src_v + OFF, kStrideUV, dst_argb_opt + OFF, \
kStrideB, kWidth, NEG kHeight); \
} \
double time2 = get_time(); \
printf(" %8d us C - %8d us OPT\n", \
static_cast<int>((time1 - time0) * 1e6), \
static_cast<int>((time2 - time1) * 1e6 / benchmark_iterations_)); \
for (int i = 0; i < kWidth * BPP_B * kHeight; ++i) { \
ASSERT_EQ(dst_argb_c[i + OFF], dst_argb_opt[i + OFF]); \
} \
@ -2982,6 +2976,7 @@ TEST_F(LibYUVConvertTest, P210ToAR30MatrixFilterOverflow) {
#ifdef ENABLE_ROW_TESTS
TEST_F(LibYUVConvertTest, TestARGBToUVMatrixRow_Opt) {
#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
const int kMaxWidth = 128;
SIMD_ALIGNED(uint8_t orig_argb_pixels[kMaxWidth * 4 * 2]);
SIMD_ALIGNED(uint8_t dest_u_c[kMaxWidth]);
@ -2993,7 +2988,6 @@ TEST_F(LibYUVConvertTest, TestARGBToUVMatrixRow_Opt) {
orig_argb_pixels[i] = i * 43;
}
#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
int has_neon = TestCpuFlag(kCpuHasNEON);
if (has_neon) {
for (int width = 1; width <= kMaxWidth; ++width) {
@ -3090,7 +3084,7 @@ TEST_F(LibYUVConvertTest, TestI400LargeSize) {
free_aligned_buffer_page_end(orig_i400);
}
#endif // DISABLE_SLOW_TESTS
#endif // !defined(DISABLE_SLOW_TESTS) && \
#endif // !defined(DISABLE_SLOW_TESTS) &&
// (defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__))
#endif // !defined(LEAN_TESTS)

120
unit_test/planar_test.cc
View File

@ -4849,10 +4849,16 @@ float TestScaleMaxSamples(int benchmark_width,
for (j = 0; j < benchmark_iterations; j++) {
if (opt) {
#ifdef HAS_SCALESUMSAMPLES_NEON
max_opt = ScaleMaxSamples_NEON(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale,
kPixels);
#ifdef HAS_SCALEMAXSAMPLES_NEON
if (TestCpuFlag(kCpuHasNEON)) {
max_opt = ScaleMaxSamples_NEON(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale,
kPixels);
} else {
max_opt = ScaleMaxSamples_C(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale,
kPixels);
}
#elif defined(HAS_SCALEMAXSAMPLES_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
max_opt = ScaleMaxSamples_AVX2(reinterpret_cast<float*>(orig_y),
@ -4909,6 +4915,9 @@ float TestScaleSumSamples(int benchmark_width,
float sum_c, sum_opt = 0.f;
// NEON does multiple of 8, so round count up
const int kPixels = (benchmark_width * benchmark_height + 7) & ~7;
if (kPixels <= 0) {
return 0.f;
}
align_buffer_page_end(orig_y, kPixels * 4 * 3);
uint8_t* dst_c = orig_y + kPixels * 4;
uint8_t* dst_opt = orig_y + kPixels * 4 * 2;
@ -4928,9 +4937,15 @@ float TestScaleSumSamples(int benchmark_width,
for (j = 0; j < benchmark_iterations; j++) {
if (opt) {
#ifdef HAS_SCALESUMSAMPLES_NEON
sum_opt = ScaleSumSamples_NEON(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale,
kPixels);
if (TestCpuFlag(kCpuHasNEON)) {
sum_opt = ScaleSumSamples_NEON(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale,
kPixels);
} else {
sum_opt = ScaleSumSamples_C(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale,
kPixels);
}
#elif defined(HAS_SCALESUMSAMPLES_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
sum_opt = ScaleSumSamples_AVX2(reinterpret_cast<float*>(orig_y),
@ -4997,6 +5012,9 @@ float TestScaleSamples(int benchmark_width,
int i, j;
// NEON does multiple of 8, so round count up
const int kPixels = (benchmark_width * benchmark_height + 7) & ~7;
if (kPixels <= 0) {
return 0.f;
}
align_buffer_page_end(orig_y, kPixels * 4 * 3);
uint8_t* dst_c = orig_y + kPixels * 4;
uint8_t* dst_opt = orig_y + kPixels * 4 * 2;
@ -5007,17 +5025,22 @@ float TestScaleSamples(int benchmark_width,
for (i = 0; i < kPixels; ++i) {
(reinterpret_cast<float*>(orig_y))[i] = sinf(static_cast<float>(i) * 0.1f);
}
memset(dst_c, 0, kPixels * 4);
memset(dst_opt, 1, kPixels * 4);
memset(dst_c, 0, static_cast<size_t>(kPixels) * 4);
memset(dst_opt, 1, static_cast<size_t>(kPixels) * 4);
ScaleSamples_C(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_c), scale, kPixels);
for (j = 0; j < benchmark_iterations; j++) {
if (opt) {
#ifdef HAS_SCALESUMSAMPLES_NEON
ScaleSamples_NEON(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale, kPixels);
#ifdef HAS_SCALESAMPLES_NEON
if (TestCpuFlag(kCpuHasNEON)) {
ScaleSamples_NEON(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale, kPixels);
} else {
ScaleSamples_C(reinterpret_cast<float*>(orig_y),
reinterpret_cast<float*>(dst_opt), scale, kPixels);
}
#elif defined(HAS_SCALESAMPLES_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ScaleSamples_AVX2(reinterpret_cast<float*>(orig_y),
@ -5068,6 +5091,9 @@ float TestCopySamples(int benchmark_width,
int i, j;
// NEON does multiple of 16 floats, so round count up
const int kPixels = (benchmark_width * benchmark_height + 15) & ~15;
if (kPixels <= 0) {
return 0.f;
}
align_buffer_page_end(orig_y, kPixels * 4 * 3);
uint8_t* dst_c = orig_y + kPixels * 4;
uint8_t* dst_opt = orig_y + kPixels * 4 * 2;
@ -5078,16 +5104,20 @@ float TestCopySamples(int benchmark_width,
for (i = 0; i < kPixels; ++i) {
(reinterpret_cast<float*>(orig_y))[i] = sinf(static_cast<float>(i) * 0.1f);
}
memset(dst_c, 0, kPixels * 4);
memset(dst_opt, 1, kPixels * 4);
memset(dst_c, 0, static_cast<size_t>(kPixels) * 4);
memset(dst_opt, 1, static_cast<size_t>(kPixels) * 4);
memcpy(reinterpret_cast<void*>(dst_c), reinterpret_cast<void*>(orig_y),
kPixels * 4);
static_cast<size_t>(kPixels) * 4);
for (j = 0; j < benchmark_iterations; j++) {
if (opt) {
#ifdef HAS_COPYROW_NEON
CopyRow_NEON(orig_y, dst_opt, kPixels * 4);
if (TestCpuFlag(kCpuHasNEON)) {
CopyRow_NEON(orig_y, dst_opt, kPixels * 4);
} else {
CopyRow_C(orig_y, dst_opt, kPixels * 4);
}
#else
CopyRow_C(orig_y, dst_opt, kPixels * 4);
#endif
@ -5401,6 +5431,25 @@ TEST_F(LibYUVPlanarTest, HalfRow_Opt) {
HalfRow_Any_AVX2(src_pixels, stride, dst_pixels_opt, width);
}
} else
#endif
#if defined(HAS_HALFROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
HalfRow_SME(src_pixels, stride, dst_pixels_opt, width);
} else
#endif
#if defined(HAS_HALFROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
HalfRow_SVE2(src_pixels, stride, dst_pixels_opt, width);
} else
#endif
#if defined(HAS_HALFROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
if (IS_ALIGNED(width, 16)) {
HalfRow_NEON(src_pixels, stride, dst_pixels_opt, width);
} else {
HalfRow_Any_NEON(src_pixels, stride, dst_pixels_opt, width);
}
} else
#endif
{
HalfRow_C(src_pixels, stride, dst_pixels_opt, width);
@ -5437,6 +5486,25 @@ TEST_F(LibYUVPlanarTest, HalfRow_16_Opt) {
HalfRow_16_Any_AVX2(src_pixels, stride, dst_pixels_opt, width);
}
} else
#endif
#if defined(HAS_HALFROW_16_SME)
if (TestCpuFlag(kCpuHasSME)) {
HalfRow_16_SME(src_pixels, stride, dst_pixels_opt, width);
} else
#endif
#if defined(HAS_HALFROW_16_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
HalfRow_16_SVE2(src_pixels, stride, dst_pixels_opt, width);
} else
#endif
#if defined(HAS_HALFROW_16_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
if (IS_ALIGNED(width, 8)) {
HalfRow_16_NEON(src_pixels, stride, dst_pixels_opt, width);
} else {
HalfRow_16_Any_NEON(src_pixels, stride, dst_pixels_opt, width);
}
} else
#endif
{
HalfRow_16_C(src_pixels, stride, dst_pixels_opt, width);
@ -5470,6 +5538,12 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Odd_Opt) {
HalfWidthRow_16To8_Odd_AVX2(src_pixels, stride, dst_pixels_opt, 16384,
width);
} else
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
HalfWidthRow_16To8_Odd_NEON(src_pixels, stride, dst_pixels_opt, 16384,
width);
} else
#endif
{
HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_opt, 16384,
@ -5665,6 +5739,7 @@ TEST_F(LibYUVPlanarTest, NV21Copy) {
TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32) {
int i, j;
const int y_plane_size = benchmark_width_ * benchmark_height_;
ASSERT_GT(y_plane_size, 0);
align_buffer_page_end(orig_f, y_plane_size * 4);
align_buffer_page_end(orig_y, y_plane_size * 2);
@ -5674,9 +5749,9 @@ TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32) {
for (i = 0; i < y_plane_size; ++i) {
((float*)orig_f)[i] = (float)(i % 10000) * 3.14f;
}
memset(orig_y, 1, y_plane_size * 2);
memset(dst_opt, 2, y_plane_size * 4);
memset(rec_opt, 3, y_plane_size * 2);
memset(orig_y, 1, static_cast<size_t>(y_plane_size) * 2);
memset(dst_opt, 2, static_cast<size_t>(y_plane_size) * 4);
memset(rec_opt, 3, static_cast<size_t>(y_plane_size) * 2);
ConvertFP32ToFP16Row_NEON((const float*)orig_f, (uint16_t*)orig_y,
y_plane_size);
@ -5702,6 +5777,7 @@ TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32) {
TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32Column) {
int i, j;
const int y_plane_size = benchmark_width_ * benchmark_height_;
ASSERT_GT(y_plane_size, 0);
align_buffer_page_end(orig_f, y_plane_size * 4);
align_buffer_page_end(orig_y, y_plane_size * 2);
@ -5711,9 +5787,9 @@ TEST_F(LibYUVPlanarTest, TestConvertFP16ToFP32Column) {
for (i = 0; i < y_plane_size; ++i) {
((float*)orig_f)[i] = (float)(i % 10000) * 3.14f;
}
memset(orig_y, 1, y_plane_size * 2);
memset(dst_opt, 2, y_plane_size * 4);
memset(rec_opt, 3, y_plane_size * 2);
memset(orig_y, 1, static_cast<size_t>(y_plane_size) * 2);
memset(dst_opt, 2, static_cast<size_t>(y_plane_size) * 4);
memset(rec_opt, 3, static_cast<size_t>(y_plane_size) * 2);
ConvertFP32ToFP16Row_NEON((const float*)orig_f, (uint16_t*)orig_y,
y_plane_size);

8
unit_test/rotate_argb_test.cc
View File

@ -170,28 +170,28 @@ TEST_F(LibYUVRotateTest, RotatePlane270_Opt) {
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane0_Odd) {
TEST_F(LibYUVRotateTest, RotatePlane0_Odd) {
TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane90_Odd) {
TEST_F(LibYUVRotateTest, RotatePlane90_Odd) {
TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane180_Odd) {
TEST_F(LibYUVRotateTest, RotatePlane180_Odd) {
TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane270_Odd) {
TEST_F(LibYUVRotateTest, RotatePlane270_Odd) {
TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,

24
unit_test/rotate_test.cc
View File

@ -118,28 +118,28 @@ TEST_F(LibYUVRotateTest, I420Rotate270_Opt) {
// TODO(fbarchard): Remove odd width tests.
// Odd width tests work but disabled because they use C code and can be
// tested by passing an odd width command line or environment variable.
TEST_F(LibYUVRotateTest, DISABLED_I420Rotate0_Odd) {
TEST_F(LibYUVRotateTest, I420Rotate0_Odd) {
I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I420Rotate90_Odd) {
TEST_F(LibYUVRotateTest, I420Rotate90_Odd) {
I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I420Rotate180_Odd) {
TEST_F(LibYUVRotateTest, I420Rotate180_Odd) {
I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I420Rotate270_Odd) {
TEST_F(LibYUVRotateTest, I420Rotate270_Odd) {
I420TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
@ -323,28 +323,28 @@ TEST_F(LibYUVRotateTest, I444Rotate270_Opt) {
// TODO(fbarchard): Remove odd width tests.
// Odd width tests work but disabled because they use C code and can be
// tested by passing an odd width command line or environment variable.
TEST_F(LibYUVRotateTest, DISABLED_I444Rotate0_Odd) {
TEST_F(LibYUVRotateTest, I444Rotate0_Odd) {
I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I444Rotate90_Odd) {
TEST_F(LibYUVRotateTest, I444Rotate90_Odd) {
I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I444Rotate180_Odd) {
TEST_F(LibYUVRotateTest, I444Rotate180_Odd) {
I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_I444Rotate270_Odd) {
TEST_F(LibYUVRotateTest, I444Rotate270_Odd) {
I444TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
@ -438,28 +438,28 @@ TEST_F(LibYUVRotateTest, NV12Rotate270_Opt) {
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate0_Odd) {
TEST_F(LibYUVRotateTest, NV12Rotate0_Odd) {
NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate90_Odd) {
TEST_F(LibYUVRotateTest, NV12Rotate90_Odd) {
NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate180_Odd) {
TEST_F(LibYUVRotateTest, NV12Rotate180_Odd) {
NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_NV12Rotate270_Odd) {
TEST_F(LibYUVRotateTest, NV12Rotate270_Odd) {
NV12TestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,

16
unit_test/scale_plane_test.cc
View File

@ -357,14 +357,14 @@ static int TestPlaneFilter_16(int src_width,
#define DX(x, nom, denom) static_cast<int>(((Abs(x) / nom + 1) / 2) * nom * 2)
#define SX(x, nom, denom) static_cast<int>(((x / nom + 1) / 2) * denom * 2)
#define TEST_FACTOR1(name, filter, nom, denom, max_diff) \
TEST_F(LibYUVScaleTest, DISABLED_##ScalePlaneDownBy##name##_##filter##_16) { \
int diff = TestPlaneFilter_16( \
SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
benchmark_cpu_info_); \
ASSERT_LE(diff, max_diff); \
#define TEST_FACTOR1(name, filter, nom, denom, max_diff) \
TEST_F(LibYUVScaleTest, ScalePlaneDownBy##name##_##filter##_16) { \
int diff = TestPlaneFilter_16( \
SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \
DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \
kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \
benchmark_cpu_info_); \
ASSERT_LE(diff, max_diff); \
}
// Test a scale factor with all 4 filters. Expect unfiltered to be exact, but