diff --git a/Android.bp b/Android.bp index f0914dae6..59d57b03a 100644 --- a/Android.bp +++ b/Android.bp @@ -54,6 +54,7 @@ cc_library { "source/rotate_gcc.cc", "source/rotate_neon.cc", "source/rotate_neon64.cc", + "source/rotate_rvv.cc", "source/row_any.cc", "source/row_common.cc", "source/row_gcc.cc", diff --git a/Android.mk b/Android.mk index 601f27601..d370360ec 100644 --- a/Android.mk +++ b/Android.mk @@ -30,6 +30,7 @@ LOCAL_SRC_FILES := \ source/rotate_gcc.cc \ source/rotate_neon.cc \ source/rotate_neon64.cc \ + source/rotate_rvv.cc \ source/rotate_win.cc \ source/row_any.cc \ source/row_common.cc \ diff --git a/BUILD.gn b/BUILD.gn index 059bf2269..97503d052 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -250,7 +250,10 @@ if (libyuv_use_neon) { if (libyuv_use_sve) { static_library("libyuv_sve") { - sources = [ "source/row_sve.cc" ] + sources = [ + "source/rotate_sve.cc", + "source/row_sve.cc", + ] deps = [ ":libyuv_internal" ] @@ -288,6 +291,7 @@ if (libyuv_use_sme) { if (libyuv_use_rvv && !libyuv_disable_rvv) { static_library("libyuv_rvv") { sources = [ + "source/rotate_rvv.cc", "source/row_rvv.cc", "source/scale_rvv.cc", ] @@ -339,6 +343,7 @@ if (libyuv_use_lasx) { static_library("libyuv_lasx") { sources = [ # LASX Source Files + "source/rotate_lasx.cc", "source/row_lasx.cc", ] diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bf9b00d0..5b3842178 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ set ( ly_common_source_files ${ly_src_dir}/rotate_argb.cc ${ly_src_dir}/rotate_common.cc ${ly_src_dir}/rotate_gcc.cc + ${ly_src_dir}/rotate_rvv.cc ${ly_src_dir}/rotate_wasm.cc ${ly_src_dir}/rotate_win.cc ${ly_src_dir}/row_any.cc @@ -114,6 +115,7 @@ if(NOT MSVC) # Enable AArch64 SVE kernels. add_library(${ly_lib_name}_sve OBJECT + ${ly_src_dir}/rotate_sve.cc ${ly_src_dir}/row_sve.cc) target_compile_options(${ly_lib_name}_sve PRIVATE -march=armv8.5-a+i8mm+sve2) list(APPEND ly_lib_parts $) @@ -215,6 +217,7 @@ if(LOONGARCH64) endif() if(SUPPORTS_LASX) add_library(${ly_lib_name}_lasx OBJECT + ${ly_src_dir}/rotate_lasx.cc ${ly_src_dir}/row_lasx.cc) target_compile_options(${ly_lib_name}_lasx PRIVATE -mlasx) list(APPEND ly_lib_parts $) diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index 46de57e41..b50e5fd3c 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -32,12 +32,15 @@ extern "C" { #define HAS_TRANSPOSEWX8_SSSE3 #define HAS_TRANSPOSEUVWX8_SSE2 #define HAS_TRANSPOSE4X4_32_SSE2 -#define HAS_TRANSPOSE4X4_32_AVX2 +#define HAS_TRANSPOSENX4_32_AVX2 +#define HAS_TRANSPOSEWXH_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 #if defined(__x86_64__) || defined(_M_X64) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW +#define HAS_TRANSPOSENX4_32_AVX512BW +#define HAS_TRANSPOSEWXH_32_AVX512BW #endif #endif @@ -48,7 +51,8 @@ extern "C" { #define HAS_TRANSPOSEWX8_SSSE3 #define HAS_TRANSPOSEUVWX8_SSE2 #define HAS_TRANSPOSE4X4_32_SSE2 -#define HAS_TRANSPOSE4X4_32_AVX2 +#define HAS_TRANSPOSENX4_32_AVX2 +#define HAS_TRANSPOSEWXH_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 #endif @@ -58,6 +62,8 @@ extern "C" { defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW +#define HAS_TRANSPOSENX4_32_AVX512BW +#define HAS_TRANSPOSEWXH_32_AVX512BW #endif #if !defined(LIBYUV_DISABLE_NEON) && \ @@ -68,18 +74,34 @@ extern "C" { #define HAS_TRANSPOSEWX8_NEON #endif #define HAS_TRANSPOSEUVWX8_NEON -#define HAS_TRANSPOSE4X4_32_NEON +#define HAS_TRANSPOSENX4_32_NEON +#define HAS_TRANSPOSEWXH_32_NEON +#endif + +#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) +#define HAS_TRANSPOSENX4_32_SVE2 #endif #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ defined(__aarch64__) #define HAS_TRANSPOSEWXH_SME #define HAS_TRANSPOSEUVWXH_SME +#define HAS_TRANSPOSEWXH_32_SME #endif #if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) #define HAS_TRANSPOSEWX16_LSX #define HAS_TRANSPOSEUVWX16_LSX +#define HAS_TRANSPOSE4X4_32_LSX +#endif + +#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#define HAS_TRANSPOSE4X4_32_LASX +#endif + +#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) +#define HAS_TRANSPOSENX4_32_RVV +#define HAS_TRANSPOSEWXH_32_RVV #endif #if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) @@ -331,8 +353,34 @@ void TransposeWx1_16_C(const uint16_t* src, int dst_stride, int width); -// Transpose 32 bit values (ARGB) -void Transpose4x4_32_NEON(const uint8_t* src, +void TransposeWxH_32_C(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height); + +void TransposeWxH_32_SME(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height); + +void TransposeNx4_32_NEON(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + +void TransposeWxH_32_NEON(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height); + +void TransposeNx4_32_SVE2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, @@ -344,7 +392,52 @@ void Transpose4x4_32_SSE2(const uint8_t* src, int dst_stride, int width); -void Transpose4x4_32_AVX2(const uint8_t* src, +void TransposeNx4_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + +void TransposeWxH_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height); + +void TransposeNx4_32_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + +void TransposeWxH_32_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height); + +void TransposeNx4_32_RVV(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + +void TransposeWxH_32_RVV(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height); + +void Transpose4x4_32_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + +void Transpose4x4_32_LASX(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, @@ -362,6 +455,30 @@ void Transpose4x4_32_C(const uint8_t* src, int dst_stride, int width); +void Transpose4x4_32_Any_SSE2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + +void Transpose4x4_32_Any_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + +void Transpose4x4_32_Any_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + +void Transpose4x4_32_Any_WASMSIMD(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/include/libyuv/row.h b/include/libyuv/row.h index a430db8e5..e70570758 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -818,6 +818,7 @@ extern "C" { #define HAS_YUY2TOUV422ROW_LSX #define HAS_YUY2TOYROW_LSX #define HAS_ARGBTOYMATRIXROW_LSX +#define HAS_COPYROW_LSX #define HAS_RGBTOYMATRIXROW_LSX #endif @@ -846,6 +847,7 @@ extern "C" { #define HAS_ARGBTORGB565DITHERROW_LASX #define HAS_ARGBTORGB565ROW_LASX #define HAS_ARGBTOYMATRIXROW_LASX +#define HAS_COPYROW_LASX #define HAS_I422ALPHATOARGBROW_LASX #define HAS_I422TOARGB1555ROW_LASX #define HAS_I422TOARGB4444ROW_LASX @@ -884,6 +886,7 @@ extern "C" { #define HAS_ARGBEXTRACTALPHAROW_RVV #define HAS_ARGBMULTIPLYROW_RVV #define HAS_ARGBTOAR64ROW_RVV +#define HAS_ARGBMIRRORROW_RVV #define HAS_ARGBTOUV444MATRIXROW_RVV #define HAS_ARGBTOUVMATRIXROW_RVV #define HAS_ARGBTOYMATRIXROW_RVV @@ -2786,6 +2789,8 @@ void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width); void ARGBMirrorRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBMirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width); void ARGBMirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width); +void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width); +void ARGBMirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width); void ARGBMirrorRow_C(const uint8_t* src, uint8_t* dst, int width); void ARGBMirrorRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -3912,11 +3917,15 @@ void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width); void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width); void CopyRow_SME(const uint8_t* src, uint8_t* dst, int width); void CopyRow_RVV(const uint8_t* src, uint8_t* dst, int count); +void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int width); +void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int width); void CopyRow_C(const uint8_t* src, uint8_t* dst, int count); void CopyRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_AVX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_AVX512BW(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count); diff --git a/libyuv.gypi b/libyuv.gypi index a73ba4b71..7de8fb786 100644 --- a/libyuv.gypi +++ b/libyuv.gypi @@ -59,9 +59,11 @@ 'source/rotate_argb.cc', 'source/rotate_common.cc', 'source/rotate_gcc.cc', + 'source/rotate_lasx.cc', 'source/rotate_lsx.cc', 'source/rotate_neon.cc', 'source/rotate_neon64.cc', + 'source/rotate_rvv.cc', 'source/rotate_win.cc', 'source/row_any.cc', 'source/row_common.cc', diff --git a/linux.mk b/linux.mk index 6999810da..2dd55b24c 100644 --- a/linux.mk +++ b/linux.mk @@ -32,9 +32,11 @@ LOCAL_OBJ_FILES := \ source/rotate_argb.o \ source/rotate_common.o \ source/rotate_gcc.o \ + source/rotate_lasx.o \ source/rotate_lsx.o \ source/rotate_neon.o \ source/rotate_neon64.o \ + source/rotate_rvv.o \ source/rotate_win.o \ source/row_any.o \ source/row_common.o \ diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 6cde151f6..f3ee9e3f5 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -95,6 +95,16 @@ void CopyPlane(const uint8_t* src_y, CopyRow = CopyRow_RVV; } #endif +#if defined(HAS_COPYROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LSX : CopyRow_Any_LSX; + } +#endif +#if defined(HAS_COPYROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + CopyRow = IS_ALIGNED(width, 64) ? CopyRow_LASX : CopyRow_Any_LASX; + } +#endif // Copy plane for (y = 0; y < height; ++y) { @@ -2999,6 +3009,11 @@ int ARGBMirror(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBMIRRORROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBMirrorRow = ARGBMirrorRow_RVV; + } +#endif // Mirror plane for (y = 0; y < height; ++y) { diff --git a/source/rotate.cc b/source/rotate.cc index c44ca1917..e8844f1b7 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -286,6 +286,16 @@ void RotatePlane180(const uint8_t* src, CopyRow = CopyRow_RVV; } #endif +#if defined(HAS_COPYROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LSX : CopyRow_Any_LSX; + } +#endif +#if defined(HAS_COPYROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + CopyRow = IS_ALIGNED(width, 64) ? CopyRow_LASX : CopyRow_Any_LASX; + } +#endif // Odd height will harmlessly mirror the middle row twice. for (y = 0; y < half_height; ++y) { diff --git a/source/rotate_any.cc b/source/rotate_any.cc index 986b8c9ed..c474bc262 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -91,6 +91,37 @@ TUVANY(TransposeUVWx16_Any_WASMSIMD, #endif #undef TUVANY +#define TANY32(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \ + void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \ + int dst_stride, int width) { \ + int r = width & MASK; \ + int n = width - r; \ + if (n > 0) { \ + TPOS_SIMD(src, src_stride, dst, dst_stride, n); \ + } \ + if (r > 0) { \ + TPOS_C(src + (ptrdiff_t)n * src_stride, src_stride, \ + dst + (ptrdiff_t)n * 4, dst_stride, 4, r); \ + } \ + } + +#ifdef HAS_TRANSPOSE4X4_32_SSE2 +TANY32(Transpose4x4_32_Any_SSE2, Transpose4x4_32_SSE2, TransposeWxH_32_C, 3) +#endif +#ifdef HAS_TRANSPOSE4X4_32_LSX +TANY32(Transpose4x4_32_Any_LSX, Transpose4x4_32_LSX, TransposeWxH_32_C, 3) +#endif +#ifdef HAS_TRANSPOSE4X4_32_LASX +TANY32(Transpose4x4_32_Any_LASX, Transpose4x4_32_LASX, TransposeWxH_32_C, 7) +#endif +#ifdef HAS_TRANSPOSE4X4_32_WASMSIMD +TANY32(Transpose4x4_32_Any_WASMSIMD, + Transpose4x4_32_WASMSIMD, + TransposeWxH_32_C, + 3) +#endif +#undef TANY32 + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_argb.cc b/source/rotate_argb.cc index 3220ab99e..9a58cb33c 100644 --- a/source/rotate_argb.cc +++ b/source/rotate_argb.cc @@ -16,8 +16,8 @@ #include "libyuv/cpu_id.h" #include "libyuv/planar_functions.h" #include "libyuv/rotate.h" +#include "libyuv/rotate_row.h" #include "libyuv/row.h" -#include "libyuv/scale_row.h" /* for ScaleARGBRowDownEven_ */ #ifdef __cplusplus namespace libyuv { @@ -30,57 +30,104 @@ static int ARGBTranspose(const uint8_t* src_argb, int dst_stride_argb, int width, int height) { - int i; - int src_pixel_step = src_stride_argb >> 2; - void (*ScaleARGBRowDownEven)( - const uint8_t* src_argb, ptrdiff_t src_stride_argb, int src_step, - uint8_t* dst_argb, int dst_width) = ScaleARGBRowDownEven_C; +#if defined(HAS_TRANSPOSEWXH_32_AVX2) || \ + defined(HAS_TRANSPOSEWXH_32_AVX512BW) || \ + defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \ + defined(HAS_TRANSPOSEWXH_32_RVV) + void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst, + int dst_stride, int width, int height) = NULL; +#endif + void (*Transpose4x4_32)(const uint8_t* src, int src_stride, uint8_t* dst, + int dst_stride, int width) = NULL; // Check stride is a multiple of 4. if (src_stride_argb & 3) { return -1; } -#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2) +#if defined(HAS_TRANSPOSE4X4_32_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { - ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_SSE2; + Transpose4x4_32 = Transpose4x4_32_Any_SSE2; if (IS_ALIGNED(height, 4)) { // Width of dest. - ScaleARGBRowDownEven = ScaleARGBRowDownEven_SSE2; + Transpose4x4_32 = Transpose4x4_32_SSE2; } } #endif -#if defined(HAS_SCALEARGBROWDOWNEVEN_NEON) +#if defined(HAS_TRANSPOSEWXH_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWxH_32 = TransposeWxH_32_AVX2; + } +#endif +#if defined(HAS_TRANSPOSEWXH_32_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + TransposeWxH_32 = TransposeWxH_32_AVX512BW; + } +#endif +#if defined(HAS_TRANSPOSEWXH_32_NEON) if (TestCpuFlag(kCpuHasNEON)) { - ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_NEON; - if (IS_ALIGNED(height, 4)) { // Width of dest. - ScaleARGBRowDownEven = ScaleARGBRowDownEven_NEON; - } + TransposeWxH_32 = TransposeWxH_32_NEON; } #endif -#if defined(HAS_SCALEARGBROWDOWNEVEN_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_LSX; - if (IS_ALIGNED(height, 4)) { // Width of dest. - ScaleARGBRowDownEven = ScaleARGBRowDownEven_LSX; - } +#if defined(HAS_TRANSPOSENX4_32_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + Transpose4x4_32 = TransposeNx4_32_SVE2; } #endif -#if defined(HAS_SCALEARGBROWDOWNEVEN_RVV) +#if defined(HAS_TRANSPOSEWXH_32_SME) + if (TestCpuFlag(kCpuHasSME)) { + TransposeWxH_32 = TransposeWxH_32_SME; + } +#endif +#if defined(HAS_TRANSPOSEWXH_32_RVV) if (TestCpuFlag(kCpuHasRVV)) { - ScaleARGBRowDownEven = ScaleARGBRowDownEven_RVV; + TransposeWxH_32 = TransposeWxH_32_RVV; } #endif -#if defined(HAS_SCALEARGBROWDOWNEVEN_WASMSIMD) - if (TestCpuFlag(kCpuHasWASMSIMD)) { - ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_WASMSIMD; +#if defined(HAS_TRANSPOSE4X4_32_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + Transpose4x4_32 = Transpose4x4_32_Any_LSX; if (IS_ALIGNED(height, 4)) { // Width of dest. - ScaleARGBRowDownEven = ScaleARGBRowDownEven_WASMSIMD; + Transpose4x4_32 = Transpose4x4_32_LSX; + } + } +#endif +#if defined(HAS_TRANSPOSE4X4_32_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + Transpose4x4_32 = Transpose4x4_32_Any_LASX; + if (IS_ALIGNED(height, 8)) { // Width of dest. + Transpose4x4_32 = Transpose4x4_32_LASX; + } + } +#endif +#if defined(HAS_TRANSPOSE4X4_32_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + Transpose4x4_32 = Transpose4x4_32_Any_WASMSIMD; + if (IS_ALIGNED(height, 4)) { // Width of dest. + Transpose4x4_32 = Transpose4x4_32_WASMSIMD; } } #endif - for (i = 0; i < width; ++i) { // column of source to row of dest. - ScaleARGBRowDownEven(src_argb, 0, src_pixel_step, dst_argb, height); - dst_argb += dst_stride_argb; - src_argb += 4; +#if defined(HAS_TRANSPOSEWXH_32_AVX2) || \ + defined(HAS_TRANSPOSEWXH_32_AVX512BW) || \ + defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \ + defined(HAS_TRANSPOSEWXH_32_RVV) + if (TransposeWxH_32) { + TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width, + height); + return 0; + } +#endif + if (Transpose4x4_32) { + while (width >= 4) { + Transpose4x4_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, + height); + src_argb += 4 * 4; + dst_argb += 4 * (ptrdiff_t)dst_stride_argb; + width -= 4; + } + } + if (width > 0) { + TransposeWxH_32_C(src_argb, src_stride_argb, dst_argb, dst_stride_argb, + width, height); } return 0; } @@ -121,128 +168,9 @@ static int ARGBRotate180(const uint8_t* src_argb, int dst_stride_argb, int width, int height) { - // Swap first and last row and mirror the content. Uses a temporary row. - const uint8_t* src_bot = src_argb + (ptrdiff_t)src_stride_argb * (height - 1); - uint8_t* dst_bot = dst_argb + (ptrdiff_t)dst_stride_argb * (height - 1); - int half_height = (height + 1) >> 1; - int y; - void (*ARGBMirrorRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width) = - ARGBMirrorRow_C; - void (*CopyRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width) = - CopyRow_C; - if (width > INT_MAX / 4) { - return -1; - } - align_buffer_64(row, width * 4); - if (!row) - return 1; -#if defined(HAS_ARGBMIRRORROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ARGBMirrorRow = ARGBMirrorRow_Any_NEON; - if (IS_ALIGNED(width, 8)) { - ARGBMirrorRow = ARGBMirrorRow_NEON; - } - } -#endif -#if defined(HAS_ARGBMIRRORROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ARGBMirrorRow = ARGBMirrorRow_Any_SSE2; - if (IS_ALIGNED(width, 4)) { - ARGBMirrorRow = ARGBMirrorRow_SSE2; - } - } -#endif -#if defined(HAS_ARGBMIRRORROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - ARGBMirrorRow = ARGBMirrorRow_Any_AVX2; - if (IS_ALIGNED(width, 8)) { - ARGBMirrorRow = ARGBMirrorRow_AVX2; - } - } -#endif -#if defined(HAS_ARGBMIRRORROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBMirrorRow = ARGBMirrorRow_AVX512BW; - } -#endif -#if defined(HAS_ARGBMIRRORROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - ARGBMirrorRow = ARGBMirrorRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { - ARGBMirrorRow = ARGBMirrorRow_LSX; - } - } -#endif -#if defined(HAS_ARGBMIRRORROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - ARGBMirrorRow = ARGBMirrorRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { - ARGBMirrorRow = ARGBMirrorRow_LASX; - } - } -#endif -#if defined(HAS_ARGBMIRRORROW_WASMSIMD) - if (TestCpuFlag(kCpuHasWASMSIMD)) { - ARGBMirrorRow = ARGBMirrorRow_Any_WASMSIMD; - if (IS_ALIGNED(width, 4)) { - ARGBMirrorRow = ARGBMirrorRow_WASMSIMD; - } - } -#endif -#if defined(HAS_COPYROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2; - } -#endif -#if defined(HAS_COPYROW_AVX) - if (TestCpuFlag(kCpuHasAVX)) { - CopyRow = IS_ALIGNED(width * 4, 64) ? CopyRow_AVX : CopyRow_Any_AVX; - } -#endif -#if defined(HAS_COPYROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - CopyRow = - IS_ALIGNED(width * 4, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW; - } -#endif -#if defined(HAS_COPYROW_ERMS) - if (TestCpuFlag(kCpuHasERMS)) { - CopyRow = CopyRow_ERMS; - } -#endif -#if defined(HAS_COPYROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_NEON : CopyRow_Any_NEON; - } -#endif -#if defined(HAS_COPYROW_SVE2) - if (TestCpuFlag(kCpuHasSVE2)) { - CopyRow = CopyRow_SVE2; - } -#endif -#if defined(HAS_COPYROW_SME) - if (TestCpuFlag(kCpuHasSME)) { - CopyRow = CopyRow_SME; - } -#endif -#if defined(HAS_COPYROW_RVV) - if (TestCpuFlag(kCpuHasRVV)) { - CopyRow = CopyRow_RVV; - } -#endif - - // Odd height will harmlessly mirror the middle row twice. - for (y = 0; y < half_height; ++y) { - ARGBMirrorRow(src_argb, row, width); // Mirror first row into a buffer - ARGBMirrorRow(src_bot, dst_argb, width); // Mirror last row into first row - CopyRow(row, dst_bot, width * 4); // Copy first mirrored row into last - src_argb += src_stride_argb; - dst_argb += dst_stride_argb; - src_bot -= src_stride_argb; - dst_bot -= dst_stride_argb; - } - free_aligned_buffer_64(row); - return 0; + // Rotate by 180 is a vertical flip (negative height) and horizontal mirror. + return ARGBMirror(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width, + -height); } LIBYUV_API diff --git a/source/rotate_common.cc b/source/rotate_common.cc index 20e5faf2b..636161a9a 100644 --- a/source/rotate_common.cc +++ b/source/rotate_common.cc @@ -206,6 +206,22 @@ void Transpose4x4_32_C(const uint8_t* src, } } +void TransposeWxH_32_C(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + int i; + for (i = 0; i < width; ++i) { + int j; + for (j = 0; j < height; ++j) { + Copy4(dst + (ptrdiff_t)i * dst_stride + j * 4, + src + (ptrdiff_t)j * src_stride + i * 4); + } + } +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index 64181160a..3b9d0bcf6 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -730,61 +730,845 @@ void Transpose4x4_32_SSE2(const uint8_t* src, } #endif // defined(HAS_TRANSPOSE4X4_32_SSE2) -#if defined(HAS_TRANSPOSE4X4_32_AVX2) - -// Transpose 32 bit values (ARGB) -void Transpose4x4_32_AVX2(const uint8_t* src, +#if defined(HAS_TRANSPOSENX4_32_AVX2) +// Transpose 32 bit values (ARGB) in Nx4 tiles with 2 blocks of 4x4 at a time +// in ymm0..ymm3 and vpmaskmovd tail for arbitrary N. +void TransposeNx4_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + uint8_t* dst1 = dst + d; + uint8_t* dst2 = dst1 + d; + uint8_t* dst3 = dst2 + d; asm volatile( - // Main loop transpose 2 blocks of 4x4. Read a column, write a row. - "1: \n" - "vmovdqu (%0),%%xmm0 \n" // a b c d - "vmovdqu (%0,%3),%%xmm1 \n" // e f g h - "lea (%0,%3,2),%0 \n" // src += stride * 2 - "vmovdqu (%0),%%xmm2 \n" // i j k l - "vmovdqu (%0,%3),%%xmm3 \n" // m n o p - "lea (%0,%3,2),%0 \n" // src += stride * 2 + "sub $0x8,%[w] \n" + "jl 2f \n" - "vinserti128 $1,(%0),%%ymm0,%%ymm0 \n" // a b c d - "vinserti128 $1,(%0,%3),%%ymm1,%%ymm1 \n" // e f g h - "lea (%0,%3,2),%0 \n" // src += stride * 2 - "vinserti128 $1,(%0),%%ymm2,%%ymm2 \n" // i j k l - "vinserti128 $1,(%0,%3),%%ymm3,%%ymm3 \n" // m n o p - "lea (%0,%3,2),%0 \n" // src += stride * 2 + // Main loop: 2 blocks of 4x4 (8 source rows -> 8 dest columns / 32 B). + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" // a b c d + "vmovdqu (%[src],%[s]),%%xmm1 \n" // e f g h + "lea (%[src],%[s],2),%[src] \n" // src += stride * 2 + "vmovdqu (%[src]),%%xmm2 \n" // i j k l + "vmovdqu (%[src],%[s]),%%xmm3 \n" // m n o p + "lea (%[src],%[s],2),%[src] \n" // src += stride * 2 - // Transpose 2x2 - "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" // a e b f from row 0, 1 - "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" // i m j n from row 2, 3 - "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" // c g d h from row 0, 1 - "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" // k o l p from row 2, 3 + "vinserti128 $1,(%[src]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[src],%[s]),%%ymm1,%%ymm1 \n" + "lea (%[src],%[s],2),%[src] \n" + "vinserti128 $1,(%[src]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[src],%[s]),%%ymm3,%%ymm3 \n" + "lea (%[src],%[s],2),%[src] \n" - // Transpose 4x4 - "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" // a e i m from row 0, 1 - "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" // b f j n from row 0, 1 - "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" // c g k o from row 2, 3 - "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" // d h l p from row 2, 3 + // Transpose 2x2 across both 128-bit lanes + "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" - "vmovdqu %%ymm0,(%1) \n" - "lea 32(%1,%4),%1 \n" // dst += stride + 32 - "vmovdqu %%ymm1,-32(%1) \n" - "vmovdqu %%ymm2,-32(%1,%4) \n" - "vmovdqu %%ymm3,-32(%1,%4,2) \n" - "sub %4,%1 \n" - "sub $0x8,%2 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+rm"(width) // %2 - : "r"((ptrdiff_t)(src_stride)), // %3 - "r"((ptrdiff_t)(dst_stride)) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + // Transpose 4x4 across both 128-bit lanes + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" + + "vmovdqu %%ymm0,(%[dst0]) \n" + "vmovdqu %%ymm1,(%[dst1]) \n" + "vmovdqu %%ymm2,(%[dst2]) \n" + "vmovdqu %%ymm3,(%[dst3]) \n" + "add $0x20,%[dst0] \n" + "add $0x20,%[dst1] \n" + "add $0x20,%[dst2] \n" + "add $0x20,%[dst3] \n" + "sub $0x8,%[w] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[w] \n" + "jz 5f \n" + + // 4x4 block for 4..7 remainder elements + "cmp $0x4,%[w] \n" + "jl 3f \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[src] \n" + "vmovdqu (%[src]),%%xmm2 \n" + "vmovdqu (%[src],%[s]),%%xmm3 \n" + "lea (%[src],%[s],2),%[src] \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" + "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" + "vmovdqu %%xmm0,(%[dst0]) \n" + "vmovdqu %%xmm1,(%[dst1]) \n" + "vmovdqu %%xmm2,(%[dst2]) \n" + "vmovdqu %%xmm3,(%[dst3]) \n" + "add $0x10,%[dst0] \n" + "add $0x10,%[dst1] \n" + "add $0x10,%[dst2] \n" + "add $0x10,%[dst3] \n" + "sub $0x4,%[w] \n" + "jz 5f \n" + + // Masked tail for 1..3 remainder elements using vpmaskmovd + "3: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqa %%xmm0,%%xmm1 \n" + "vmovdqa %%xmm0,%%xmm2 \n" + "mov $0x000000ff,%%eax \n" + "cmp $2,%[w] \n" + "jl 4f \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "mov $0x0000ffff,%%eax \n" + "je 4f \n" + "vmovdqu (%[src],%[s],2),%%xmm2 \n" + "mov $0x00ffffff,%%eax \n" + "4: \n" + "vmovd %%eax,%%xmm7 \n" + "vpmovsxbd %%xmm7,%%xmm7 \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm2,%%xmm2,%%xmm3 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n" + "vpunpckhqdq %%xmm3,%%xmm6,%%xmm3 \n" + "vpmaskmovd %%xmm0,%%xmm7,(%[dst0]) \n" + "vpmaskmovd %%xmm1,%%xmm7,(%[dst1]) \n" + "vpmaskmovd %%xmm2,%%xmm7,(%[dst2]) \n" + "vpmaskmovd %%xmm3,%%xmm7,(%[dst3]) \n" + "5: \n" + "vzeroupper \n" + : [src] "+r"(src), // %[src] + [dst0] "+r"(dst), // %[dst0] + [dst1] "+r"(dst1), // %[dst1] + [dst2] "+r"(dst2), // %[dst2] + [dst3] "+r"(dst3), // %[dst3] + [w] "+r"(width) // %[w] + : [s] "r"(s) // %[s] + : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); } -#endif // defined(HAS_TRANSPOSE4X4_32_AVX2) +#endif // defined(HAS_TRANSPOSENX4_32_AVX2) + +#if defined(HAS_TRANSPOSEWXH_32_AVX2) +// Transpose 32 bit plane of any width x height using AVX2 vpmaskmovd. +void TransposeWxH_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (height >= 8) { + const uint8_t* src_row = src; + uint8_t* dst_col = dst; + int w = width; + while (w >= 8) { + asm volatile( + "vmovdqu (%[s0]),%%ymm0 \n" + "vmovdqu (%[s0],%[s]),%%ymm1 \n" + "lea (%[s0],%[s],2),%%rax \n" + "vmovdqu (%%rax),%%ymm2 \n" + "vmovdqu (%%rax,%[s]),%%ymm3 \n" + "lea (%%rax,%[s],2),%%rax \n" + "vmovdqu (%%rax),%%ymm4 \n" + "vmovdqu (%%rax,%[s]),%%ymm5 \n" + "lea (%%rax,%[s],2),%%rax \n" + "vmovdqu (%%rax),%%ymm6 \n" + "vmovdqu (%%rax,%[s]),%%ymm7 \n" + + "vpunpckldq %%ymm1,%%ymm0,%%ymm8 \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm9 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm10 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm11 \n" + "vpunpckldq %%ymm5,%%ymm4,%%ymm12 \n" + "vpunpckhdq %%ymm5,%%ymm4,%%ymm13 \n" + "vpunpckldq %%ymm7,%%ymm6,%%ymm14 \n" + "vpunpckhdq %%ymm7,%%ymm6,%%ymm15 \n" + + "vpunpcklqdq %%ymm10,%%ymm8,%%ymm0 \n" + "vpunpckhqdq %%ymm10,%%ymm8,%%ymm1 \n" + "vpunpcklqdq %%ymm11,%%ymm9,%%ymm2 \n" + "vpunpckhqdq %%ymm11,%%ymm9,%%ymm3 \n" + "vpunpcklqdq %%ymm14,%%ymm12,%%ymm4 \n" + "vpunpckhqdq %%ymm14,%%ymm12,%%ymm5 \n" + "vpunpcklqdq %%ymm15,%%ymm13,%%ymm6 \n" + "vpunpckhqdq %%ymm15,%%ymm13,%%ymm7 \n" + + "vperm2i128 $0x20,%%ymm4,%%ymm0,%%ymm8 \n" + "vperm2i128 $0x20,%%ymm5,%%ymm1,%%ymm9 \n" + "vperm2i128 $0x20,%%ymm6,%%ymm2,%%ymm10 \n" + "vperm2i128 $0x20,%%ymm7,%%ymm3,%%ymm11 \n" + "vperm2i128 $0x31,%%ymm4,%%ymm0,%%ymm12 \n" + "vperm2i128 $0x31,%%ymm5,%%ymm1,%%ymm13 \n" + "vperm2i128 $0x31,%%ymm6,%%ymm2,%%ymm14 \n" + "vperm2i128 $0x31,%%ymm7,%%ymm3,%%ymm15 \n" + + "vmovdqu %%ymm8,(%[d0]) \n" + "vmovdqu %%ymm9,(%[d0],%[d]) \n" + "lea (%[d0],%[d],2),%%rax \n" + "vmovdqu %%ymm10,(%%rax) \n" + "vmovdqu %%ymm11,(%%rax,%[d]) \n" + "lea (%%rax,%[d],2),%%rax \n" + "vmovdqu %%ymm12,(%%rax) \n" + "vmovdqu %%ymm13,(%%rax,%[d]) \n" + "lea (%%rax,%[d],2),%%rax \n" + "vmovdqu %%ymm14,(%%rax) \n" + "vmovdqu %%ymm15,(%%rax,%[d]) \n" + : + : [s0] "r"(src_row), // %[s0] + [d0] "r"(dst_col), // %[d0] + [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "rax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", + "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", + "xmm13", "xmm14", "xmm15"); + src_row += 32; + dst_col += 8 * d; + w -= 8; + } + if (w >= 4) { + TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, 8); + src_row += 16; + dst_col += 4 * d; + w -= 4; + } + if (w > 0) { + TransposeWxH_32_C(src_row, src_stride, dst_col, dst_stride, w, 8); + } + src += 8 * s; + dst += 32; + height -= 8; + } + asm volatile("vzeroupper \n"); + if (height > 0) { + while (width >= 4) { + TransposeNx4_32_AVX2(src, src_stride, dst, dst_stride, height); + src += 16; + dst += 4 * d; + width -= 4; + } + if (width > 0) { + const uint8_t* src_rem = src; + uint8_t* dst0 = dst; + uint8_t* dst1 = (width > 1) ? dst0 + d : dst0; + uint8_t* dst2 = (width > 2) ? dst1 + d : dst1; + int col_mask_bits = (1 << (width * 8)) - 1; + int h_rem = height; + asm volatile( + "vmovd %[col_mask],%%xmm7 \n" + "vpmovsxbd %%xmm7,%%xmm7 \n" + "sub $0x4,%[h] \n" + "jl 2f \n" + "1: \n" + "vpmaskmovd (%[src]),%%xmm7,%%xmm0 \n" + "vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm1 \n" + "lea (%[src],%[s],2),%[src] \n" + "vpmaskmovd (%[src]),%%xmm7,%%xmm2 \n" + "vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm3 \n" + "lea (%[src],%[s],2),%[src] \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm3 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n" + "vmovdqu %%xmm2,(%[dst2]) \n" + "vmovdqu %%xmm1,(%[dst1]) \n" + "vmovdqu %%xmm0,(%[dst0]) \n" + "add $0x10,%[dst2] \n" + "add $0x10,%[dst1] \n" + "add $0x10,%[dst0] \n" + "sub $0x4,%[h] \n" + "jge 1b \n" + "2: \n" + "add $0x4,%[h] \n" + "jz 4f \n" + "vpmaskmovd (%[src]),%%xmm7,%%xmm0 \n" + "vmovdqa %%xmm0,%%xmm1 \n" + "vmovdqa %%xmm0,%%xmm2 \n" + "mov $0x000000ff,%%eax \n" + "cmp $2,%[h] \n" + "jl 3f \n" + "vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm1 \n" + "mov $0x0000ffff,%%eax \n" + "je 3f \n" + "vpmaskmovd (%[src],%[s],2),%%xmm7,%%xmm2 \n" + "mov $0x00ffffff,%%eax \n" + "3: \n" + "vmovd %%eax,%%xmm7 \n" + "vpmovsxbd %%xmm7,%%xmm7 \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm2,%%xmm2,%%xmm3 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n" + "vpmaskmovd %%xmm2,%%xmm7,(%[dst2]) \n" + "vpmaskmovd %%xmm1,%%xmm7,(%[dst1]) \n" + "vpmaskmovd %%xmm0,%%xmm7,(%[dst0]) \n" + "4: \n" + "vzeroupper \n" + : [src] "+r"(src_rem), // %[src] + [dst0] "+r"(dst0), // %[dst0] + [dst1] "+r"(dst1), // %[dst1] + [dst2] "+r"(dst2), // %[dst2] + [h] "+r"(h_rem) // %[h] + : [s] "r"(s), // %[s] + [col_mask] "r"(col_mask_bits) // %[col_mask] + : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", + "xmm5", "xmm6", "xmm7"); + } + } +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX2) + +#if defined(HAS_TRANSPOSENX4_32_AVX512BW) +// Transpose 32 bit values (ARGB) in Nx4 tiles with 4 blocks of 4x4 at a time +// in zmm0..zmm3 and masked tail for arbitrary N. +void TransposeNx4_32_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + uint8_t* dst1 = dst + d; + uint8_t* dst2 = dst1 + d; + uint8_t* dst3 = dst2 + d; + asm volatile( + "sub $0x10,%[w] \n" + "jl 2f \n" + + // Main loop: 4 blocks of 4x4 (16 source rows -> 16 dest columns / 64 B). + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[src] \n" + "vmovdqu (%[src]),%%xmm2 \n" + "vmovdqu (%[src],%[s]),%%xmm3 \n" + "lea (%[src],%[s],2),%[src] \n" + + "vinserti32x4 $1,(%[src]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $1,(%[src],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[src],%[s],2),%[src] \n" + "vinserti32x4 $1,(%[src]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $1,(%[src],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[src],%[s],2),%[src] \n" + + "vinserti32x4 $2,(%[src]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $2,(%[src],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[src],%[s],2),%[src] \n" + "vinserti32x4 $2,(%[src]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $2,(%[src],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[src],%[s],2),%[src] \n" + + "vinserti32x4 $3,(%[src]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $3,(%[src],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[src],%[s],2),%[src] \n" + "vinserti32x4 $3,(%[src]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $3,(%[src],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[src],%[s],2),%[src] \n" + + // Transpose 2x2 across all 4 128-bit lanes + "vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n" + "vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n" + "vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n" + "vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n" + + // Transpose 4x4 across all 4 128-bit lanes + "vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n" + "vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n" + "vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n" + "vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n" + + "vmovdqu32 %%zmm0,(%[dst0]) \n" + "vmovdqu32 %%zmm1,(%[dst1]) \n" + "vmovdqu32 %%zmm2,(%[dst2]) \n" + "vmovdqu32 %%zmm3,(%[dst3]) \n" + "add $0x40,%[dst0] \n" + "add $0x40,%[dst1] \n" + "add $0x40,%[dst2] \n" + "add $0x40,%[dst3] \n" + "sub $0x10,%[w] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[w] \n" + "jz 5f \n" + + // 4x4 loop for 4..15 remainder elements + "sub $0x4,%[w] \n" + "jl 4f \n" + "3: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[src] \n" + "vmovdqu (%[src]),%%xmm2 \n" + "vmovdqu (%[src],%[s]),%%xmm3 \n" + "lea (%[src],%[s],2),%[src] \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" + "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" + "vmovdqu %%xmm0,(%[dst0]) \n" + "vmovdqu %%xmm1,(%[dst1]) \n" + "vmovdqu %%xmm2,(%[dst2]) \n" + "vmovdqu %%xmm3,(%[dst3]) \n" + "add $0x10,%[dst0] \n" + "add $0x10,%[dst1] \n" + "add $0x10,%[dst2] \n" + "add $0x10,%[dst3] \n" + "sub $0x4,%[w] \n" + "jge 3b \n" + + "4: \n" + "add $0x4,%[w] \n" + "jz 5f \n" + + // Masked tail for 1..3 remainder elements (in-bounds row loads + k1 mask) + "mov $1,%%eax \n" + "shlx %[w],%%eax,%%eax \n" + "dec %%eax \n" + "kmovw %%eax,%%k1 \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqa %%xmm0,%%xmm1 \n" + "vmovdqa %%xmm0,%%xmm2 \n" + "cmp $1,%[w] \n" + "jle 6f \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "cmp $2,%[w] \n" + "jle 6f \n" + "vmovdqu (%[src],%[s],2),%%xmm2 \n" + "6: \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm2,%%xmm2,%%xmm7 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" + "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" + "vmovdqu32 %%xmm0,(%[dst0])%{%%k1%} \n" + "vmovdqu32 %%xmm1,(%[dst1])%{%%k1%} \n" + "vmovdqu32 %%xmm2,(%[dst2])%{%%k1%} \n" + "vmovdqu32 %%xmm3,(%[dst3])%{%%k1%} \n" + "5: \n" + "vzeroupper \n" + : [src] "+r"(src), // %[src] + [dst0] "+r"(dst), // %[dst0] + [dst1] "+r"(dst1), // %[dst1] + [dst2] "+r"(dst2), // %[dst2] + [dst3] "+r"(dst3), // %[dst3] + [w] "+r"(width) // %[w] + : [s] "r"(s) // %[s] + : "memory", "cc", "eax", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", + "xmm5", "xmm6", "xmm7"); +} +#endif // defined(HAS_TRANSPOSENX4_32_AVX512BW) + +#if defined(HAS_TRANSPOSEWXH_32_AVX512BW) +// Transpose 32 bit plane of any width x height using 16x16 ZMM horizontal tiles +// in zmm0..zmm31 with k-masked right and bottom tails. +void TransposeWxH_32_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (height > 0) { + const int bh = (height >= 16) ? 16 : height; + const uint8_t* src_row = src; + uint8_t* dst_col = dst; + int w = width; + if (bh == 16) { + while (w >= 16) { + const uint8_t* s_ptr = src_row; + uint8_t* d_ptr = dst_col; + asm volatile( + "vmovdqu32 (%[s_ptr]),%%zmm0 \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm1 \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm2 \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm3 \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm4 \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm5 \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm6 \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm7 \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm8 \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm9 \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm10 \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm11 \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm12 \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm13 \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm14 \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm15 \n" + + // Stage 1: 32-bit unpack + "vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n" + "vpunpckhdq %%zmm1,%%zmm0,%%zmm17 \n" + "vpunpckldq %%zmm3,%%zmm2,%%zmm18 \n" + "vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n" + "vpunpckldq %%zmm5,%%zmm4,%%zmm20 \n" + "vpunpckhdq %%zmm5,%%zmm4,%%zmm21 \n" + "vpunpckldq %%zmm7,%%zmm6,%%zmm22 \n" + "vpunpckhdq %%zmm7,%%zmm6,%%zmm23 \n" + "vpunpckldq %%zmm9,%%zmm8,%%zmm24 \n" + "vpunpckhdq %%zmm9,%%zmm8,%%zmm25 \n" + "vpunpckldq %%zmm11,%%zmm10,%%zmm26 \n" + "vpunpckhdq %%zmm11,%%zmm10,%%zmm27 \n" + "vpunpckldq %%zmm13,%%zmm12,%%zmm28 \n" + "vpunpckhdq %%zmm13,%%zmm12,%%zmm29 \n" + "vpunpckldq %%zmm15,%%zmm14,%%zmm30 \n" + "vpunpckhdq %%zmm15,%%zmm14,%%zmm31 \n" + + // Stage 2: 64-bit unpack + "vpunpcklqdq %%zmm18,%%zmm16,%%zmm0 \n" + "vpunpckhqdq %%zmm18,%%zmm16,%%zmm1 \n" + "vpunpcklqdq %%zmm19,%%zmm17,%%zmm2 \n" + "vpunpckhqdq %%zmm19,%%zmm17,%%zmm3 \n" + "vpunpcklqdq %%zmm22,%%zmm20,%%zmm4 \n" + "vpunpckhqdq %%zmm22,%%zmm20,%%zmm5 \n" + "vpunpcklqdq %%zmm23,%%zmm21,%%zmm6 \n" + "vpunpckhqdq %%zmm23,%%zmm21,%%zmm7 \n" + "vpunpcklqdq %%zmm26,%%zmm24,%%zmm8 \n" + "vpunpckhqdq %%zmm26,%%zmm24,%%zmm9 \n" + "vpunpcklqdq %%zmm27,%%zmm25,%%zmm10 \n" + "vpunpckhqdq %%zmm27,%%zmm25,%%zmm11 \n" + "vpunpcklqdq %%zmm30,%%zmm28,%%zmm12 \n" + "vpunpckhqdq %%zmm30,%%zmm28,%%zmm13 \n" + "vpunpcklqdq %%zmm31,%%zmm29,%%zmm14 \n" + "vpunpckhqdq %%zmm31,%%zmm29,%%zmm15 \n" + + // Stage 3: 128-bit lane shuffle + "vshufi32x4 $0x88,%%zmm4,%%zmm0,%%zmm16 \n" + "vshufi32x4 $0x88,%%zmm5,%%zmm1,%%zmm17 \n" + "vshufi32x4 $0x88,%%zmm6,%%zmm2,%%zmm18 \n" + "vshufi32x4 $0x88,%%zmm7,%%zmm3,%%zmm19 \n" + "vshufi32x4 $0xdd,%%zmm4,%%zmm0,%%zmm20 \n" + "vshufi32x4 $0xdd,%%zmm5,%%zmm1,%%zmm21 \n" + "vshufi32x4 $0xdd,%%zmm6,%%zmm2,%%zmm22 \n" + "vshufi32x4 $0xdd,%%zmm7,%%zmm3,%%zmm23 \n" + "vshufi32x4 $0x88,%%zmm12,%%zmm8,%%zmm24 \n" + "vshufi32x4 $0x88,%%zmm13,%%zmm9,%%zmm25 \n" + "vshufi32x4 $0x88,%%zmm14,%%zmm10,%%zmm26 \n" + "vshufi32x4 $0x88,%%zmm15,%%zmm11,%%zmm27 \n" + "vshufi32x4 $0xdd,%%zmm12,%%zmm8,%%zmm28 \n" + "vshufi32x4 $0xdd,%%zmm13,%%zmm9,%%zmm29 \n" + "vshufi32x4 $0xdd,%%zmm14,%%zmm10,%%zmm30 \n" + "vshufi32x4 $0xdd,%%zmm15,%%zmm11,%%zmm31 \n" + + // Stage 4: 256-bit half shuffle -> columns 0..15 in zmm0..zmm15 + "vshufi32x4 $0x88,%%zmm24,%%zmm16,%%zmm0 \n" + "vshufi32x4 $0x88,%%zmm25,%%zmm17,%%zmm1 \n" + "vshufi32x4 $0x88,%%zmm26,%%zmm18,%%zmm2 \n" + "vshufi32x4 $0x88,%%zmm27,%%zmm19,%%zmm3 \n" + "vshufi32x4 $0x88,%%zmm28,%%zmm20,%%zmm4 \n" + "vshufi32x4 $0x88,%%zmm29,%%zmm21,%%zmm5 \n" + "vshufi32x4 $0x88,%%zmm30,%%zmm22,%%zmm6 \n" + "vshufi32x4 $0x88,%%zmm31,%%zmm23,%%zmm7 \n" + "vshufi32x4 $0xdd,%%zmm24,%%zmm16,%%zmm8 \n" + "vshufi32x4 $0xdd,%%zmm25,%%zmm17,%%zmm9 \n" + "vshufi32x4 $0xdd,%%zmm26,%%zmm18,%%zmm10 \n" + "vshufi32x4 $0xdd,%%zmm27,%%zmm19,%%zmm11 \n" + "vshufi32x4 $0xdd,%%zmm28,%%zmm20,%%zmm12 \n" + "vshufi32x4 $0xdd,%%zmm29,%%zmm21,%%zmm13 \n" + "vshufi32x4 $0xdd,%%zmm30,%%zmm22,%%zmm14 \n" + "vshufi32x4 $0xdd,%%zmm31,%%zmm23,%%zmm15 \n" + + "vmovdqu32 %%zmm0,(%[d_ptr]) \n" + "vmovdqu32 %%zmm1,(%[d_ptr],%[d]) \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm2,(%[d_ptr]) \n" + "vmovdqu32 %%zmm3,(%[d_ptr],%[d]) \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm4,(%[d_ptr]) \n" + "vmovdqu32 %%zmm5,(%[d_ptr],%[d]) \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm6,(%[d_ptr]) \n" + "vmovdqu32 %%zmm7,(%[d_ptr],%[d]) \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm8,(%[d_ptr]) \n" + "vmovdqu32 %%zmm9,(%[d_ptr],%[d]) \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm10,(%[d_ptr]) \n" + "vmovdqu32 %%zmm11,(%[d_ptr],%[d]) \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm12,(%[d_ptr]) \n" + "vmovdqu32 %%zmm13,(%[d_ptr],%[d]) \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm14,(%[d_ptr]) \n" + "vmovdqu32 %%zmm15,(%[d_ptr],%[d]) \n" + : [s_ptr] "+r"(s_ptr), // %[s_ptr] + [d_ptr] "+r"(d_ptr) // %[d_ptr] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", + "xmm13", "xmm14", "xmm15", "xmm16", "xmm17", "xmm18", "xmm19", + "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26", + "xmm27", "xmm28", "xmm29", "xmm30", "xmm31"); + src_row += 64; + dst_col += 16 * d; + w -= 16; + } + } + const uint32_t row_mask = (1u << bh) - 1; + while (w > 0) { + const int bw = (w >= 16) ? 16 : w; + const uint32_t col_mask = (1u << bw) - 1; + const uint8_t* s_ptr = src_row; + uint8_t* d_ptr = dst_col; + asm volatile( + "kmovw %[col_mask],%%k1 \n" + "kmovw %[row_mask],%%k2 \n" + "vpxord %%zmm0,%%zmm0,%%zmm0 \n" + "vmovdqa64 %%zmm0,%%zmm1 \n" + "vmovdqa64 %%zmm0,%%zmm2 \n" + "vmovdqa64 %%zmm0,%%zmm3 \n" + "vmovdqa64 %%zmm0,%%zmm4 \n" + "vmovdqa64 %%zmm0,%%zmm5 \n" + "vmovdqa64 %%zmm0,%%zmm6 \n" + "vmovdqa64 %%zmm0,%%zmm7 \n" + "vmovdqa64 %%zmm0,%%zmm8 \n" + "vmovdqa64 %%zmm0,%%zmm9 \n" + "vmovdqa64 %%zmm0,%%zmm10 \n" + "vmovdqa64 %%zmm0,%%zmm11 \n" + "vmovdqa64 %%zmm0,%%zmm12 \n" + "vmovdqa64 %%zmm0,%%zmm13 \n" + "vmovdqa64 %%zmm0,%%zmm14 \n" + "vmovdqa64 %%zmm0,%%zmm15 \n" + + "vmovdqu32 (%[s_ptr]),%%zmm0%{%%k1%}%{z%} \n" + "cmp $1,%[bh] \n" + "jle 1f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm1%{%%k1%}%{z%} \n" + "cmp $2,%[bh] \n" + "jle 1f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm2%{%%k1%}%{z%} \n" + "cmp $3,%[bh] \n" + "jle 1f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm3%{%%k1%}%{z%} \n" + "cmp $4,%[bh] \n" + "jle 1f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm4%{%%k1%}%{z%} \n" + "cmp $5,%[bh] \n" + "jle 1f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm5%{%%k1%}%{z%} \n" + "cmp $6,%[bh] \n" + "jle 1f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm6%{%%k1%}%{z%} \n" + "cmp $7,%[bh] \n" + "jle 1f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm7%{%%k1%}%{z%} \n" + "cmp $8,%[bh] \n" + "jle 1f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm8%{%%k1%}%{z%} \n" + "cmp $9,%[bh] \n" + "jle 1f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm9%{%%k1%}%{z%} \n" + "cmp $10,%[bh] \n" + "jle 1f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm10%{%%k1%}%{z%} \n" + "cmp $11,%[bh] \n" + "jle 1f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm11%{%%k1%}%{z%} \n" + "cmp $12,%[bh] \n" + "jle 1f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm12%{%%k1%}%{z%} \n" + "cmp $13,%[bh] \n" + "jle 1f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm13%{%%k1%}%{z%} \n" + "cmp $14,%[bh] \n" + "jle 1f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%zmm14%{%%k1%}%{z%} \n" + "cmp $15,%[bh] \n" + "jle 1f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%zmm15%{%%k1%}%{z%} \n" + "1: \n" + + // Stage 1: 32-bit unpack + "vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n" + "vpunpckhdq %%zmm1,%%zmm0,%%zmm17 \n" + "vpunpckldq %%zmm3,%%zmm2,%%zmm18 \n" + "vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n" + "vpunpckldq %%zmm5,%%zmm4,%%zmm20 \n" + "vpunpckhdq %%zmm5,%%zmm4,%%zmm21 \n" + "vpunpckldq %%zmm7,%%zmm6,%%zmm22 \n" + "vpunpckhdq %%zmm7,%%zmm6,%%zmm23 \n" + "vpunpckldq %%zmm9,%%zmm8,%%zmm24 \n" + "vpunpckhdq %%zmm9,%%zmm8,%%zmm25 \n" + "vpunpckldq %%zmm11,%%zmm10,%%zmm26 \n" + "vpunpckhdq %%zmm11,%%zmm10,%%zmm27 \n" + "vpunpckldq %%zmm13,%%zmm12,%%zmm28 \n" + "vpunpckhdq %%zmm13,%%zmm12,%%zmm29 \n" + "vpunpckldq %%zmm15,%%zmm14,%%zmm30 \n" + "vpunpckhdq %%zmm15,%%zmm14,%%zmm31 \n" + + // Stage 2: 64-bit unpack + "vpunpcklqdq %%zmm18,%%zmm16,%%zmm0 \n" + "vpunpckhqdq %%zmm18,%%zmm16,%%zmm1 \n" + "vpunpcklqdq %%zmm19,%%zmm17,%%zmm2 \n" + "vpunpckhqdq %%zmm19,%%zmm17,%%zmm3 \n" + "vpunpcklqdq %%zmm22,%%zmm20,%%zmm4 \n" + "vpunpckhqdq %%zmm22,%%zmm20,%%zmm5 \n" + "vpunpcklqdq %%zmm23,%%zmm21,%%zmm6 \n" + "vpunpckhqdq %%zmm23,%%zmm21,%%zmm7 \n" + "vpunpcklqdq %%zmm26,%%zmm24,%%zmm8 \n" + "vpunpckhqdq %%zmm26,%%zmm24,%%zmm9 \n" + "vpunpcklqdq %%zmm27,%%zmm25,%%zmm10 \n" + "vpunpckhqdq %%zmm27,%%zmm25,%%zmm11 \n" + "vpunpcklqdq %%zmm30,%%zmm28,%%zmm12 \n" + "vpunpckhqdq %%zmm30,%%zmm28,%%zmm13 \n" + "vpunpcklqdq %%zmm31,%%zmm29,%%zmm14 \n" + "vpunpckhqdq %%zmm31,%%zmm29,%%zmm15 \n" + + // Stage 3: 128-bit lane shuffle + "vshufi32x4 $0x88,%%zmm4,%%zmm0,%%zmm16 \n" + "vshufi32x4 $0x88,%%zmm5,%%zmm1,%%zmm17 \n" + "vshufi32x4 $0x88,%%zmm6,%%zmm2,%%zmm18 \n" + "vshufi32x4 $0x88,%%zmm7,%%zmm3,%%zmm19 \n" + "vshufi32x4 $0xdd,%%zmm4,%%zmm0,%%zmm20 \n" + "vshufi32x4 $0xdd,%%zmm5,%%zmm1,%%zmm21 \n" + "vshufi32x4 $0xdd,%%zmm6,%%zmm2,%%zmm22 \n" + "vshufi32x4 $0xdd,%%zmm7,%%zmm3,%%zmm23 \n" + "vshufi32x4 $0x88,%%zmm12,%%zmm8,%%zmm24 \n" + "vshufi32x4 $0x88,%%zmm13,%%zmm9,%%zmm25 \n" + "vshufi32x4 $0x88,%%zmm14,%%zmm10,%%zmm26 \n" + "vshufi32x4 $0x88,%%zmm15,%%zmm11,%%zmm27 \n" + "vshufi32x4 $0xdd,%%zmm12,%%zmm8,%%zmm28 \n" + "vshufi32x4 $0xdd,%%zmm13,%%zmm9,%%zmm29 \n" + "vshufi32x4 $0xdd,%%zmm14,%%zmm10,%%zmm30 \n" + "vshufi32x4 $0xdd,%%zmm15,%%zmm11,%%zmm31 \n" + + // Stage 4: 256-bit half shuffle -> columns 0..15 in zmm0..zmm15 + "vshufi32x4 $0x88,%%zmm24,%%zmm16,%%zmm0 \n" + "vshufi32x4 $0x88,%%zmm25,%%zmm17,%%zmm1 \n" + "vshufi32x4 $0x88,%%zmm26,%%zmm18,%%zmm2 \n" + "vshufi32x4 $0x88,%%zmm27,%%zmm19,%%zmm3 \n" + "vshufi32x4 $0x88,%%zmm28,%%zmm20,%%zmm4 \n" + "vshufi32x4 $0x88,%%zmm29,%%zmm21,%%zmm5 \n" + "vshufi32x4 $0x88,%%zmm30,%%zmm22,%%zmm6 \n" + "vshufi32x4 $0x88,%%zmm31,%%zmm23,%%zmm7 \n" + "vshufi32x4 $0xdd,%%zmm24,%%zmm16,%%zmm8 \n" + "vshufi32x4 $0xdd,%%zmm25,%%zmm17,%%zmm9 \n" + "vshufi32x4 $0xdd,%%zmm26,%%zmm18,%%zmm10 \n" + "vshufi32x4 $0xdd,%%zmm27,%%zmm19,%%zmm11 \n" + "vshufi32x4 $0xdd,%%zmm28,%%zmm20,%%zmm12 \n" + "vshufi32x4 $0xdd,%%zmm29,%%zmm21,%%zmm13 \n" + "vshufi32x4 $0xdd,%%zmm30,%%zmm22,%%zmm14 \n" + "vshufi32x4 $0xdd,%%zmm31,%%zmm23,%%zmm15 \n" + + "vmovdqu32 %%zmm0,(%[d_ptr])%{%%k2%} \n" + "cmp $1,%[bw] \n" + "jle 2f \n" + "vmovdqu32 %%zmm1,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $2,%[bw] \n" + "jle 2f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm2,(%[d_ptr])%{%%k2%} \n" + "cmp $3,%[bw] \n" + "jle 2f \n" + "vmovdqu32 %%zmm3,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $4,%[bw] \n" + "jle 2f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm4,(%[d_ptr])%{%%k2%} \n" + "cmp $5,%[bw] \n" + "jle 2f \n" + "vmovdqu32 %%zmm5,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $6,%[bw] \n" + "jle 2f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm6,(%[d_ptr])%{%%k2%} \n" + "cmp $7,%[bw] \n" + "jle 2f \n" + "vmovdqu32 %%zmm7,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $8,%[bw] \n" + "jle 2f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm8,(%[d_ptr])%{%%k2%} \n" + "cmp $9,%[bw] \n" + "jle 2f \n" + "vmovdqu32 %%zmm9,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $10,%[bw] \n" + "jle 2f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm10,(%[d_ptr])%{%%k2%} \n" + "cmp $11,%[bw] \n" + "jle 2f \n" + "vmovdqu32 %%zmm11,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $12,%[bw] \n" + "jle 2f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm12,(%[d_ptr])%{%%k2%} \n" + "cmp $13,%[bw] \n" + "jle 2f \n" + "vmovdqu32 %%zmm13,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $14,%[bw] \n" + "jle 2f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm14,(%[d_ptr])%{%%k2%} \n" + "cmp $15,%[bw] \n" + "jle 2f \n" + "vmovdqu32 %%zmm15,(%[d_ptr],%[d])%{%%k2%} \n" + "2: \n" + : [s_ptr] "+r"(s_ptr), // %[s_ptr] + [d_ptr] "+r"(d_ptr) // %[d_ptr] + : [s] "r"(s), // %[s] + [d] "r"(d), // %[d] + [col_mask] "r"(col_mask), // %[col_mask] + [row_mask] "r"(row_mask), // %[row_mask] + [bh] "r"(bh), // %[bh] + [bw] "r"(bw) // %[bw] + : "memory", "cc", "k1", "k2", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", + "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", + "xmm13", "xmm14", "xmm15", "xmm16", "xmm17", "xmm18", "xmm19", + "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26", + "xmm27", "xmm28", "xmm29", "xmm30", "xmm31"); + src_row += 64; + dst_col += 16 * d; + w -= 16; + } + src += 16 * s; + dst += 64; + height -= 16; + } + asm volatile("vzeroupper \n"); +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX512BW) #endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && // !defined(LIBYUV_ENABLE_ROWWIN) diff --git a/source/rotate_lasx.cc b/source/rotate_lasx.cc new file mode 100644 index 000000000..b4361809e --- /dev/null +++ b/source/rotate_lasx.cc @@ -0,0 +1,103 @@ +/* + * Copyright 2026 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/rotate_row.h" + +#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#include "libyuv/loongson_intrinsics.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +// Transpose 32 bit values (ARGB) in 2x 4x4 tiles per step (4 cols x 8 rows -> +// 4 rows x 8 cols), matching Transpose4x4_32_AVX2. Width is a multiple of 8. +void Transpose4x4_32_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + ptrdiff_t s = src_stride; + ptrdiff_t s2 = s * 2; + ptrdiff_t s3 = s + s2; + ptrdiff_t s4 = s * 4; + uint8_t* dst0 = dst; + uint8_t* dst1 = dst0 + dst_stride; + uint8_t* dst2 = dst1 + dst_stride; + uint8_t* dst3 = dst2 + dst_stride; + const uint8_t* src4; + + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vldx $vr1, %[src], %[s] \n" + "vldx $vr2, %[src], %[s2] \n" + "vldx $vr3, %[src], %[s3] \n" + "add.d %[src4], %[src], %[s4] \n" + "vld $vr4, %[src4], 0 \n" + "vldx $vr5, %[src4], %[s] \n" + "vldx $vr6, %[src4], %[s2] \n" + "vldx $vr7, %[src4], %[s3] \n" + "add.d %[src], %[src4], %[s4] \n" + + // Combine rows (0,4), (1,5), (2,6), (3,7) into 256-bit xr0..xr3 + "xvpermi.q $xr0, $xr4, 0x02 \n" + "xvpermi.q $xr1, $xr5, 0x02 \n" + "xvpermi.q $xr2, $xr6, 0x02 \n" + "xvpermi.q $xr3, $xr7, 0x02 \n" + + // Transpose 2x2 across both 128-bit lanes + "xvilvl.w $xr4, $xr1, $xr0 \n" + "xvilvh.w $xr5, $xr1, $xr0 \n" + "xvilvl.w $xr6, $xr3, $xr2 \n" + "xvilvh.w $xr7, $xr3, $xr2 \n" + + // Transpose 4x4 across both 128-bit lanes + "xvilvl.d $xr0, $xr6, $xr4 \n" + "xvilvh.d $xr1, $xr6, $xr4 \n" + "xvilvl.d $xr2, $xr7, $xr5 \n" + "xvilvh.d $xr3, $xr7, $xr5 \n" + + // Store 4 full 256-bit (32-byte) destination rows + "xvst $xr0, %[dst0], 0 \n" + "xvst $xr1, %[dst1], 0 \n" + "xvst $xr2, %[dst2], 0 \n" + "xvst $xr3, %[dst3], 0 \n" + "addi.d %[dst0], %[dst0], 32 \n" + "addi.d %[dst1], %[dst1], 32 \n" + "addi.d %[dst2], %[dst2], 32 \n" + "addi.d %[dst3], %[dst3], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [src4] "=&r"(src4), // %[src4] + [dst0] "+r"(dst0), // %[dst0] + [dst1] "+r"(dst1), // %[dst1] + [dst2] "+r"(dst2), // %[dst2] + [dst3] "+r"(dst3), // %[dst3] + [w] "+r"(width) // %[w] + : [s] "r"(s), // %[s] + [s2] "r"(s2), // %[s2] + [s3] "r"(s3), // %[s3] + [s4] "r"(s4) // %[s4] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7"); +} + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) diff --git a/source/rotate_lsx.cc b/source/rotate_lsx.cc index 1e8b6f962..9f330e32e 100644 --- a/source/rotate_lsx.cc +++ b/source/rotate_lsx.cc @@ -212,6 +212,36 @@ void TransposeUVWx16_LSX(const uint8_t* src, } } +void Transpose4x4_32_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + int x; + int len = width / 4; + ptrdiff_t src_stride2 = src_stride * 2; + ptrdiff_t src_stride3 = src_stride + src_stride2; + ptrdiff_t src_stride4 = src_stride * 4; + ptrdiff_t dst_stride2 = dst_stride * 2; + ptrdiff_t dst_stride3 = dst_stride + dst_stride2; + __m128i src0, src1, src2, src3; + __m128i dst0, dst1, dst2, dst3; + + for (x = 0; x < len; x++) { + src0 = __lsx_vld(src, 0); + src1 = __lsx_vldx(src, src_stride); + src2 = __lsx_vldx(src, src_stride2); + src3 = __lsx_vldx(src, src_stride3); + LSX_TRANSPOSE4x4_W(src0, src1, src2, src3, dst0, dst1, dst2, dst3); + __lsx_vst(dst0, dst, 0); + __lsx_vstx(dst1, dst, dst_stride); + __lsx_vstx(dst2, dst, dst_stride2); + __lsx_vstx(dst3, dst, dst_stride3); + dst += 16; + src += src_stride4; + } +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_neon.cc b/source/rotate_neon.cc index 3394a10f7..54a993144 100644 --- a/source/rotate_neon.cc +++ b/source/rotate_neon.cc @@ -170,45 +170,248 @@ void TransposeUVWx8_NEON(const uint8_t* src, : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11"); } -// Transpose 32 bit values (ARGB) -void Transpose4x4_32_NEON(const uint8_t* src, +// Transpose 32 bit values (ARGB) in Nx4 tiles using vld4.32 with lane insertion +// and lane stores for 1..3 remainder rows. +void TransposeNx4_32_NEON(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) { - const uint8_t* src1 = src + src_stride; - const uint8_t* src2 = src1 + src_stride; - const uint8_t* src3 = src2 + src_stride; - uint8_t* dst1 = dst + dst_stride; - uint8_t* dst2 = dst1 + dst_stride; - uint8_t* dst3 = dst2 + dst_stride; + const ptrdiff_t s = src_stride; + const uint8_t* src1 = src + s; + const uint8_t* src2 = src1 + s; + const uint8_t* src3 = src2 + s; + const ptrdiff_t d = dst_stride; + uint8_t* dst1 = dst + d; + uint8_t* dst2 = dst1 + d; + uint8_t* dst3 = dst2 + d; asm volatile( - // Main loop transpose 4x4. Read a column, write a row. - "1: \n" + "subs %8, %8, #4 \n" + "blt 2f \n" + + // Main loop transpose 4x4. Read a column, write a row. + "1: \n" "vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0], %9 \n" "vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1], %9 \n" "vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2], %9 \n" "vld4.32 {d1[1], d3[1], d5[1], d7[1]}, [%3], %9 \n" - "subs %8, %8, #4 \n" // w -= 4 + "subs %8, %8, #4 \n" "vst1.8 {q0}, [%4]! \n" "vst1.8 {q1}, [%5]! \n" "vst1.8 {q2}, [%6]! \n" "vst1.8 {q3}, [%7]! \n" - "bgt 1b \n" + "bge 1b \n" - : "+r"(src), // %0 - "+r"(src1), // %1 - "+r"(src2), // %2 - "+r"(src3), // %3 - "+r"(dst), // %4 - "+r"(dst1), // %5 - "+r"(dst2), // %6 - "+r"(dst3), // %7 - "+r"(width) // %8 - : "r"((ptrdiff_t)src_stride * 4) // %9 + "2: \n" + "adds %8, %8, #4 \n" + "beq 5f \n" + + // Remainder of 1, 2, or 3 source rows. + "vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0] \n" + "cmp %8, #2 \n" + "blt 4f \n" + "vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1] \n" + "bgt 3f \n" + "vst1.32 {d0}, [%4] \n" + "vst1.32 {d2}, [%5] \n" + "vst1.32 {d4}, [%6] \n" + "vst1.32 {d6}, [%7] \n" + "b 5f \n" + + "3: \n" + "vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2] \n" + "vst1.32 {d0}, [%4]! \n" + "vst1.32 {d2}, [%5]! \n" + "vst1.32 {d4}, [%6]! \n" + "vst1.32 {d6}, [%7]! \n" + "vst1.32 {d1[0]}, [%4] \n" + "vst1.32 {d3[0]}, [%5] \n" + "vst1.32 {d5[0]}, [%6] \n" + "vst1.32 {d7[0]}, [%7] \n" + "b 5f \n" + + "4: \n" + "vst1.32 {d0[0]}, [%4] \n" + "vst1.32 {d2[0]}, [%5] \n" + "vst1.32 {d4[0]}, [%6] \n" + "vst1.32 {d6[0]}, [%7] \n" + "5: \n" + : "+r"(src), // %0 + "+r"(src1), // %1 + "+r"(src2), // %2 + "+r"(src3), // %3 + "+r"(dst), // %4 + "+r"(dst1), // %5 + "+r"(dst2), // %6 + "+r"(dst3), // %7 + "+r"(width) // %8 + : "r"(s * 4) // %9 : "memory", "cc", "q0", "q1", "q2", "q3"); } +void TransposeWxH_32_NEON(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t d = dst_stride; + while (width >= 4) { + TransposeNx4_32_NEON(src, src_stride, dst, dst_stride, height); + src += 16; + dst += 4 * d; + width -= 4; + } + if (width == 3) { + const ptrdiff_t s = src_stride; + const uint8_t* src1 = src + s; + const uint8_t* src2 = src1 + s; + const uint8_t* src3 = src2 + s; + uint8_t* dst1 = dst + d; + uint8_t* dst2 = dst1 + d; + asm volatile( + "subs %7, %7, #4 \n" + "blt 2f \n" + "1: \n" + "vld3.32 {d0[0], d2[0], d4[0]}, [%0], %8 \n" + "vld3.32 {d0[1], d2[1], d4[1]}, [%1], %8 \n" + "vld3.32 {d1[0], d3[0], d5[0]}, [%2], %8 \n" + "vld3.32 {d1[1], d3[1], d5[1]}, [%3], %8 \n" + "subs %7, %7, #4 \n" + "vst1.8 {q0}, [%4]! \n" + "vst1.8 {q1}, [%5]! \n" + "vst1.8 {q2}, [%6]! \n" + "bge 1b \n" + "2: \n" + "adds %7, %7, #4 \n" + "beq 5f \n" + "vld3.32 {d0[0], d2[0], d4[0]}, [%0] \n" + "cmp %7, #2 \n" + "blt 4f \n" + "vld3.32 {d0[1], d2[1], d4[1]}, [%1] \n" + "bgt 3f \n" + "vst1.32 {d0}, [%4] \n" + "vst1.32 {d2}, [%5] \n" + "vst1.32 {d4}, [%6] \n" + "b 5f \n" + "3: \n" + "vld3.32 {d1[0], d3[0], d5[0]}, [%2] \n" + "vst1.32 {d0}, [%4]! \n" + "vst1.32 {d2}, [%5]! \n" + "vst1.32 {d4}, [%6]! \n" + "vst1.32 {d1[0]}, [%4] \n" + "vst1.32 {d3[0]}, [%5] \n" + "vst1.32 {d5[0]}, [%6] \n" + "b 5f \n" + "4: \n" + "vst1.32 {d0[0]}, [%4] \n" + "vst1.32 {d2[0]}, [%5] \n" + "vst1.32 {d4[0]}, [%6] \n" + "5: \n" + : "+r"(src), // %0 + "+r"(src1), // %1 + "+r"(src2), // %2 + "+r"(src3), // %3 + "+r"(dst), // %4 + "+r"(dst1), // %5 + "+r"(dst2), // %6 + "+r"(height) // %7 + : "r"(s * 4) // %8 + : "memory", "cc", "q0", "q1", "q2"); + } else if (width == 2) { + const ptrdiff_t s = src_stride; + const uint8_t* src1 = src + s; + const uint8_t* src2 = src1 + s; + const uint8_t* src3 = src2 + s; + uint8_t* dst1 = dst + d; + asm volatile( + "subs %6, %6, #4 \n" + "blt 2f \n" + "1: \n" + "vld2.32 {d0[0], d2[0]}, [%0], %7 \n" + "vld2.32 {d0[1], d2[1]}, [%1], %7 \n" + "vld2.32 {d1[0], d3[0]}, [%2], %7 \n" + "vld2.32 {d1[1], d3[1]}, [%3], %7 \n" + "subs %6, %6, #4 \n" + "vst1.8 {q0}, [%4]! \n" + "vst1.8 {q1}, [%5]! \n" + "bge 1b \n" + "2: \n" + "adds %6, %6, #4 \n" + "beq 5f \n" + "vld2.32 {d0[0], d2[0]}, [%0] \n" + "cmp %6, #2 \n" + "blt 4f \n" + "vld2.32 {d0[1], d2[1]}, [%1] \n" + "bgt 3f \n" + "vst1.32 {d0}, [%4] \n" + "vst1.32 {d2}, [%5] \n" + "b 5f \n" + "3: \n" + "vld2.32 {d1[0], d3[0]}, [%2] \n" + "vst1.32 {d0}, [%4]! \n" + "vst1.32 {d2}, [%5]! \n" + "vst1.32 {d1[0]}, [%4] \n" + "vst1.32 {d3[0]}, [%5] \n" + "b 5f \n" + "4: \n" + "vst1.32 {d0[0]}, [%4] \n" + "vst1.32 {d2[0]}, [%5] \n" + "5: \n" + : "+r"(src), // %0 + "+r"(src1), // %1 + "+r"(src2), // %2 + "+r"(src3), // %3 + "+r"(dst), // %4 + "+r"(dst1), // %5 + "+r"(height) // %6 + : "r"(s * 4) // %7 + : "memory", "cc", "q0", "q1"); + } else if (width == 1) { + const ptrdiff_t s = src_stride; + const uint8_t* src1 = src + s; + const uint8_t* src2 = src1 + s; + const uint8_t* src3 = src2 + s; + asm volatile( + "subs %5, %5, #4 \n" + "blt 2f \n" + "1: \n" + "vld1.32 {d0[0]}, [%0], %6 \n" + "vld1.32 {d0[1]}, [%1], %6 \n" + "vld1.32 {d1[0]}, [%2], %6 \n" + "vld1.32 {d1[1]}, [%3], %6 \n" + "subs %5, %5, #4 \n" + "vst1.8 {q0}, [%4]! \n" + "bge 1b \n" + "2: \n" + "adds %5, %5, #4 \n" + "beq 5f \n" + "vld1.32 {d0[0]}, [%0] \n" + "cmp %5, #2 \n" + "blt 4f \n" + "vld1.32 {d0[1]}, [%1] \n" + "bgt 3f \n" + "vst1.32 {d0}, [%4] \n" + "b 5f \n" + "3: \n" + "vld1.32 {d1[0]}, [%2] \n" + "vst1.32 {d0}, [%4]! \n" + "vst1.32 {d1[0]}, [%4] \n" + "b 5f \n" + "4: \n" + "vst1.32 {d0[0]}, [%4] \n" + "5: \n" + : "+r"(src), // %0 + "+r"(src1), // %1 + "+r"(src2), // %2 + "+r"(src3), // %3 + "+r"(dst), // %4 + "+r"(height) // %5 + : "r"(s * 4) // %6 + : "memory", "cc", "q0"); + } +} + #endif // defined(__ARM_NEON__) && !defined(__aarch64__) #ifdef __cplusplus diff --git a/source/rotate_neon64.cc b/source/rotate_neon64.cc index f81a419c5..33e283a6e 100644 --- a/source/rotate_neon64.cc +++ b/source/rotate_neon64.cc @@ -225,44 +225,248 @@ void TransposeUVWx8_NEON(const uint8_t* src, "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v30", "v31"); } -// Transpose 32 bit values (ARGB) -void Transpose4x4_32_NEON(const uint8_t* src, +// Transpose 32 bit values (ARGB) in Nx4 tiles using ld4 with lane insertion +// and lane stores for 1..3 remainder rows. +void TransposeNx4_32_NEON(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) { - const uint8_t* src1 = src + src_stride; - const uint8_t* src2 = src1 + src_stride; - const uint8_t* src3 = src2 + src_stride; - uint8_t* dst1 = dst + dst_stride; - uint8_t* dst2 = dst1 + dst_stride; - uint8_t* dst3 = dst2 + dst_stride; + const ptrdiff_t s = src_stride; + const uint8_t* src1 = src + s; + const uint8_t* src2 = src1 + s; + const uint8_t* src3 = src2 + s; + const ptrdiff_t d = dst_stride; + uint8_t* dst1 = dst + d; + uint8_t* dst2 = dst1 + d; + uint8_t* dst3 = dst2 + d; asm volatile( - // Main loop transpose 4x4. Read a column, write a row. - "1: \n" + "subs %w8, %w8, #4 \n" + "b.lt 2f \n" + + // Main loop transpose 4x4. Read a column, write a row. + "1: \n" "ld4 {v0.s, v1.s, v2.s, v3.s}[0], [%0], %9 \n" "ld4 {v0.s, v1.s, v2.s, v3.s}[1], [%1], %9 \n" "ld4 {v0.s, v1.s, v2.s, v3.s}[2], [%2], %9 \n" "ld4 {v0.s, v1.s, v2.s, v3.s}[3], [%3], %9 \n" - "subs %w8, %w8, #4 \n" // w -= 4 - "st1 {v0.4s}, [%4], 16 \n" - "st1 {v1.4s}, [%5], 16 \n" - "st1 {v2.4s}, [%6], 16 \n" - "st1 {v3.4s}, [%7], 16 \n" - "b.gt 1b \n" - : "+r"(src), // %0 - "+r"(src1), // %1 - "+r"(src2), // %2 - "+r"(src3), // %3 - "+r"(dst), // %4 - "+r"(dst1), // %5 - "+r"(dst2), // %6 - "+r"(dst3), // %7 - "+r"(width) // %8 - : "r"((ptrdiff_t)src_stride * 4) // %9 + "subs %w8, %w8, #4 \n" + "st1 {v0.4s}, [%4], 16 \n" + "st1 {v1.4s}, [%5], 16 \n" + "st1 {v2.4s}, [%6], 16 \n" + "st1 {v3.4s}, [%7], 16 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w8, %w8, #4 \n" + "b.eq 5f \n" + + // Remainder of 1, 2, or 3 source rows. + "ld4 {v0.s, v1.s, v2.s, v3.s}[0], [%0] \n" + "cmp %w8, #2 \n" + "b.lt 4f \n" + "ld4 {v0.s, v1.s, v2.s, v3.s}[1], [%1] \n" + "b.gt 3f \n" + "st1 {v0.d}[0], [%4] \n" + "st1 {v1.d}[0], [%5] \n" + "st1 {v2.d}[0], [%6] \n" + "st1 {v3.d}[0], [%7] \n" + "b 5f \n" + + "3: \n" + "ld4 {v0.s, v1.s, v2.s, v3.s}[2], [%2] \n" + "st1 {v0.d}[0], [%4], 8 \n" + "st1 {v1.d}[0], [%5], 8 \n" + "st1 {v2.d}[0], [%6], 8 \n" + "st1 {v3.d}[0], [%7], 8 \n" + "st1 {v0.s}[2], [%4] \n" + "st1 {v1.s}[2], [%5] \n" + "st1 {v2.s}[2], [%6] \n" + "st1 {v3.s}[2], [%7] \n" + "b 5f \n" + + "4: \n" + "st1 {v0.s}[0], [%4] \n" + "st1 {v1.s}[0], [%5] \n" + "st1 {v2.s}[0], [%6] \n" + "st1 {v3.s}[0], [%7] \n" + "5: \n" + : "+r"(src), // %0 + "+r"(src1), // %1 + "+r"(src2), // %2 + "+r"(src3), // %3 + "+r"(dst), // %4 + "+r"(dst1), // %5 + "+r"(dst2), // %6 + "+r"(dst3), // %7 + "+r"(width) // %8 + : "r"(s * 4) // %9 : "memory", "cc", "v0", "v1", "v2", "v3"); } +void TransposeWxH_32_NEON(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t d = dst_stride; + while (width >= 4) { + TransposeNx4_32_NEON(src, src_stride, dst, dst_stride, height); + src += 16; + dst += 4 * d; + width -= 4; + } + if (width == 3) { + const ptrdiff_t s = src_stride; + const uint8_t* src1 = src + s; + const uint8_t* src2 = src1 + s; + const uint8_t* src3 = src2 + s; + uint8_t* dst1 = dst + d; + uint8_t* dst2 = dst1 + d; + asm volatile( + "subs %w7, %w7, #4 \n" + "b.lt 2f \n" + "1: \n" + "ld3 {v0.s, v1.s, v2.s}[0], [%0], %8 \n" + "ld3 {v0.s, v1.s, v2.s}[1], [%1], %8 \n" + "ld3 {v0.s, v1.s, v2.s}[2], [%2], %8 \n" + "ld3 {v0.s, v1.s, v2.s}[3], [%3], %8 \n" + "subs %w7, %w7, #4 \n" + "st1 {v0.4s}, [%4], 16 \n" + "st1 {v1.4s}, [%5], 16 \n" + "st1 {v2.4s}, [%6], 16 \n" + "b.ge 1b \n" + "2: \n" + "adds %w7, %w7, #4 \n" + "b.eq 5f \n" + "ld3 {v0.s, v1.s, v2.s}[0], [%0] \n" + "cmp %w7, #2 \n" + "b.lt 4f \n" + "ld3 {v0.s, v1.s, v2.s}[1], [%1] \n" + "b.gt 3f \n" + "st1 {v0.d}[0], [%4] \n" + "st1 {v1.d}[0], [%5] \n" + "st1 {v2.d}[0], [%6] \n" + "b 5f \n" + "3: \n" + "ld3 {v0.s, v1.s, v2.s}[2], [%2] \n" + "st1 {v0.d}[0], [%4], 8 \n" + "st1 {v1.d}[0], [%5], 8 \n" + "st1 {v2.d}[0], [%6], 8 \n" + "st1 {v0.s}[2], [%4] \n" + "st1 {v1.s}[2], [%5] \n" + "st1 {v2.s}[2], [%6] \n" + "b 5f \n" + "4: \n" + "st1 {v0.s}[0], [%4] \n" + "st1 {v1.s}[0], [%5] \n" + "st1 {v2.s}[0], [%6] \n" + "5: \n" + : "+r"(src), // %0 + "+r"(src1), // %1 + "+r"(src2), // %2 + "+r"(src3), // %3 + "+r"(dst), // %4 + "+r"(dst1), // %5 + "+r"(dst2), // %6 + "+r"(height) // %7 + : "r"(s * 4) // %8 + : "memory", "cc", "v0", "v1", "v2"); + } else if (width == 2) { + const ptrdiff_t s = src_stride; + const uint8_t* src1 = src + s; + const uint8_t* src2 = src1 + s; + const uint8_t* src3 = src2 + s; + uint8_t* dst1 = dst + d; + asm volatile( + "subs %w6, %w6, #4 \n" + "b.lt 2f \n" + "1: \n" + "ld2 {v0.s, v1.s}[0], [%0], %7 \n" + "ld2 {v0.s, v1.s}[1], [%1], %7 \n" + "ld2 {v0.s, v1.s}[2], [%2], %7 \n" + "ld2 {v0.s, v1.s}[3], [%3], %7 \n" + "subs %w6, %w6, #4 \n" + "st1 {v0.4s}, [%4], 16 \n" + "st1 {v1.4s}, [%5], 16 \n" + "b.ge 1b \n" + "2: \n" + "adds %w6, %w6, #4 \n" + "b.eq 5f \n" + "ld2 {v0.s, v1.s}[0], [%0] \n" + "cmp %w6, #2 \n" + "b.lt 4f \n" + "ld2 {v0.s, v1.s}[1], [%1] \n" + "b.gt 3f \n" + "st1 {v0.d}[0], [%4] \n" + "st1 {v1.d}[0], [%5] \n" + "b 5f \n" + "3: \n" + "ld2 {v0.s, v1.s}[2], [%2] \n" + "st1 {v0.d}[0], [%4], 8 \n" + "st1 {v1.d}[0], [%5], 8 \n" + "st1 {v0.s}[2], [%4] \n" + "st1 {v1.s}[2], [%5] \n" + "b 5f \n" + "4: \n" + "st1 {v0.s}[0], [%4] \n" + "st1 {v1.s}[0], [%5] \n" + "5: \n" + : "+r"(src), // %0 + "+r"(src1), // %1 + "+r"(src2), // %2 + "+r"(src3), // %3 + "+r"(dst), // %4 + "+r"(dst1), // %5 + "+r"(height) // %6 + : "r"(s * 4) // %7 + : "memory", "cc", "v0", "v1"); + } else if (width == 1) { + const ptrdiff_t s = src_stride; + const uint8_t* src1 = src + s; + const uint8_t* src2 = src1 + s; + const uint8_t* src3 = src2 + s; + asm volatile( + "subs %w5, %w5, #4 \n" + "b.lt 2f \n" + "1: \n" + "ld1 {v0.s}[0], [%0], %6 \n" + "ld1 {v0.s}[1], [%1], %6 \n" + "ld1 {v0.s}[2], [%2], %6 \n" + "ld1 {v0.s}[3], [%3], %6 \n" + "subs %w5, %w5, #4 \n" + "st1 {v0.4s}, [%4], 16 \n" + "b.ge 1b \n" + "2: \n" + "adds %w5, %w5, #4 \n" + "b.eq 5f \n" + "ld1 {v0.s}[0], [%0] \n" + "cmp %w5, #2 \n" + "b.lt 4f \n" + "ld1 {v0.s}[1], [%1] \n" + "b.gt 3f \n" + "st1 {v0.d}[0], [%4] \n" + "b 5f \n" + "3: \n" + "ld1 {v0.s}[2], [%2] \n" + "st1 {v0.d}[0], [%4], 8 \n" + "st1 {v0.s}[2], [%4] \n" + "b 5f \n" + "4: \n" + "st1 {v0.s}[0], [%4] \n" + "5: \n" + : "+r"(src), // %0 + "+r"(src1), // %1 + "+r"(src2), // %2 + "+r"(src3), // %3 + "+r"(dst), // %4 + "+r"(height) // %5 + : "r"(s * 4) // %6 + : "memory", "cc", "v0"); + } +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/rotate_rvv.cc b/source/rotate_rvv.cc new file mode 100644 index 000000000..3b9e2b635 --- /dev/null +++ b/source/rotate_rvv.cc @@ -0,0 +1,151 @@ +/* + * Copyright 2026 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/rotate_row.h" +#include "libyuv/row.h" + +// This module is for RVV (RISC-V Vector extension) +#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#ifndef RVV_VL_CLOBBER +#ifdef __clang__ +#define RVV_VL_CLOBBER +#else +#define RVV_VL_CLOBBER "vl", +#endif +#endif + +#ifdef HAS_TRANSPOSENX4_32_RVV +// Transpose 32 bit values (ARGB) in Nx4 tiles, moving down N rows of source +// (VL at a time) and across N columns of destination (4 rows). +// Tuned for 512-bit VL (SiFive X280). +void TransposeNx4_32_RVV(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + int vl; + ptrdiff_t src_step; + ptrdiff_t dst_step; + uint8_t* dst1 = dst + (ptrdiff_t)dst_stride; + uint8_t* dst2 = dst1 + (ptrdiff_t)dst_stride; + uint8_t* dst3 = dst2 + (ptrdiff_t)dst_stride; + asm volatile( + "vsetvli %[vl], %[w], e32, m2, ta, ma \n" + "slli %[dst_step], %[vl], 2 \n" + "mul %[src_step], %[vl], %[src_stride] \n" + "sub %[w], %[w], %[vl] \n" + "bltz %[w], 2f \n" + "1: \n" + "vlsseg4e32.v v8, (%[src]), %[src_stride] \n" + "add %[src], %[src], %[src_step] \n" + "vse32.v v8, (%[dst0]) \n" + "vse32.v v10, (%[dst1]) \n" + "vse32.v v12, (%[dst2]) \n" + "vse32.v v14, (%[dst3]) \n" + "add %[dst0], %[dst0], %[dst_step] \n" + "add %[dst1], %[dst1], %[dst_step] \n" + "add %[dst2], %[dst2], %[dst_step] \n" + "add %[dst3], %[dst3], %[dst_step] \n" + "sub %[w], %[w], %[vl] \n" + "bgez %[w], 1b \n" + "2: \n" + "add %[w], %[w], %[vl] \n" + "beqz %[w], 3f \n" + "vsetvli %[vl], %[w], e32, m2, ta, ma \n" + "vlsseg4e32.v v8, (%[src]), %[src_stride] \n" + "vse32.v v8, (%[dst0]) \n" + "vse32.v v10, (%[dst1]) \n" + "vse32.v v12, (%[dst2]) \n" + "vse32.v v14, (%[dst3]) \n" + "3: \n" + : [w] "+r"(width), // %[w] + [src] "+r"(src), // %[src] + [dst0] "+r"(dst), // %[dst0] + [dst1] "+r"(dst1), // %[dst1] + [dst2] "+r"(dst2), // %[dst2] + [dst3] "+r"(dst3), // %[dst3] + [vl] "=&r"(vl), // %[vl] + [src_step] "=&r"(src_step), // %[src_step] + [dst_step] "=&r"(dst_step) // %[dst_step] + : [src_stride] "r"((ptrdiff_t)src_stride) // %[src_stride] + : RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15"); +} +#endif // HAS_TRANSPOSE4X4_32_RVV + +#ifdef HAS_TRANSPOSEWXH_32_RVV +// Transpose 32 bit plane of any width x height using RVV without scalar +// fallback. +void TransposeWxH_32_RVV(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + while (width >= 4) { + TransposeNx4_32_RVV(src, src_stride, dst, dst_stride, height); + src += 16; + dst += 4 * (ptrdiff_t)dst_stride; + width -= 4; + } + while (width > 0) { + const uint8_t* s_col = src; + uint8_t* d_row = dst; + int h = height; + int vl; + ptrdiff_t src_step; + ptrdiff_t dst_step; + asm volatile( + "vsetvli %[vl], %[h], e32, m8, ta, ma \n" + "slli %[d_step], %[vl], 2 \n" + "mul %[s_step], %[vl], %[src_stride] \n" + "sub %[h], %[h], %[vl] \n" + "bltz %[h], 2f \n" + "1: \n" + "vlse32.v v8, (%[s]), %[src_stride] \n" + "add %[s], %[s], %[s_step] \n" + "vse32.v v8, (%[d]) \n" + "add %[d], %[d], %[d_step] \n" + "sub %[h], %[h], %[vl] \n" + "bgez %[h], 1b \n" + "2: \n" + "add %[h], %[h], %[vl] \n" + "beqz %[h], 3f \n" + "vsetvli %[vl], %[h], e32, m8, ta, ma \n" + "vlse32.v v8, (%[s]), %[src_stride] \n" + "vse32.v v8, (%[d]) \n" + "3: \n" + : [h] "+r"(h), // %[h] + [s] "+r"(s_col), // %[s] + [d] "+r"(d_row), // %[d] + [vl] "=&r"(vl), // %[vl] + [s_step] "=&r"(src_step), // %[s_step] + [d_step] "=&r"(dst_step) // %[d_step] + : [src_stride] "r"((ptrdiff_t)src_stride) // %[src_stride] + : RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15"); + src += 4; + dst += (ptrdiff_t)dst_stride; + width -= 1; + } +} +#endif // HAS_TRANSPOSEWXH_32_RVV + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif + +#endif // !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) diff --git a/source/rotate_sme.cc b/source/rotate_sme.cc index 6cf0a1528..22b6f7d7f 100644 --- a/source/rotate_sme.cc +++ b/source/rotate_sme.cc @@ -163,6 +163,73 @@ __arm_locally_streaming __arm_new("za") void TransposeUVWxH_SME( } while (height > 0); } +__arm_locally_streaming __arm_new("za") void TransposeWxH_32_SME( + const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + int vl; + asm("cntw %x0" : "=r"(vl)); + + do { + const uint8_t* src2 = src; + uint8_t* dst2 = dst; + + // Process up to VL 32-bit rows per iteration of the outer loop. + int block_height = height > vl ? vl : height; + + int width2 = width; + do { + const uint8_t* src3 = src2; + + // Process up to VL 32-bit columns per iteration of the inner loop. + int block_width = width2 > vl ? vl : width2; + + asm volatile( + "mov w12, #0 \n" + + // Create a predicate to handle loading partial rows. + "whilelt p0.s, wzr, %w[block_width] \n" + + // Load H <= VL rows into ZA0.S. + "1: \n" + "ld1w {za0h.s[w12, 0]}, p0/z, [%[src3]] \n" + "add %[src3], %[src3], %[src_stride] \n" + "add w12, w12, #1 \n" + "cmp w12, %w[block_height] \n" + "b.ne 1b \n" + + // Create a predicate to handle storing partial columns. + "whilelt p0.s, wzr, %w[block_height] \n" + "mov w12, #0 \n" + + // Store W <= VL columns from ZA0.S. + "2: \n" + "st1w {za0v.s[w12, 0]}, p0, [%[dst2]] \n" + "add %[dst2], %[dst2], %[dst_stride] \n" + "add w12, w12, #1 \n" + "cmp w12, %w[block_width] \n" + "b.ne 2b \n" + : [src3] "+r"(src3), // %[src3] + [dst2] "+r"(dst2) // %[dst2] + : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride] + [dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride] + [block_width] "r"(block_width), // %[block_width] + [block_height] "r"(block_height) // %[block_height] + : "cc", "memory", "p0", "w12", "za"); + + src2 += vl * 4; + width2 -= vl; + } while (width2 > 0); + + src += (ptrdiff_t)vl * src_stride; + dst += vl * 4; + height -= vl; + } while (height > 0); +} + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/rotate_sve.cc b/source/rotate_sve.cc new file mode 100644 index 000000000..84d63bc7e --- /dev/null +++ b/source/rotate_sve.cc @@ -0,0 +1,117 @@ +/* + * Copyright 2026 The LibYuv Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "libyuv/rotate_row.h" +#include "libyuv/row.h" + +#ifdef __cplusplus +namespace libyuv { +extern "C" { +#endif + +#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) + +// Transpose 32 bit values (ARGB) in Nx4 tiles using SVE2 with predication +// for tail widths. +void TransposeNx4_32_SVE2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t s4 = s * 4; + const ptrdiff_t d = dst_stride; + const uint8_t* src1 = src + s; + const uint8_t* src2 = src1 + s; + const uint8_t* src3 = src2 + s; + uint8_t* dst1 = dst + d; + uint8_t* dst2 = dst1 + d; + uint8_t* dst3 = dst2 + d; + asm volatile( + "ptrue p0.s, vl4 \n" + "subs %w[width], %w[width], #4 \n" + "b.lt 2f \n" + + // Main loop: full 4x4 blocks. + "1: \n" + "ld1w {z0.s}, p0/z, [%[src0]] \n" + "ld1w {z1.s}, p0/z, [%[src1]] \n" + "ld1w {z2.s}, p0/z, [%[src2]] \n" + "ld1w {z3.s}, p0/z, [%[src3]] \n" + "add %[src0], %[src0], %[s4] \n" + "add %[src1], %[src1], %[s4] \n" + "add %[src2], %[src2], %[s4] \n" + "add %[src3], %[src3], %[s4] \n" + "trn1 z4.s, z0.s, z1.s \n" + "trn2 z5.s, z0.s, z1.s \n" + "trn1 z6.s, z2.s, z3.s \n" + "trn2 z7.s, z2.s, z3.s \n" + "trn1 z0.d, z4.d, z6.d \n" + "trn2 z1.d, z4.d, z6.d \n" + "trn1 z2.d, z5.d, z7.d \n" + "trn2 z3.d, z5.d, z7.d \n" + "subs %w[width], %w[width], #4 \n" + "st1w {z0.s}, p0, [%[dst0]] \n" + "st1w {z1.s}, p0, [%[dst1]] \n" + "st1w {z2.s}, p0, [%[dst2]] \n" + "st1w {z3.s}, p0, [%[dst3]] \n" + "add %[dst0], %[dst0], #16 \n" + "add %[dst1], %[dst1], #16 \n" + "add %[dst2], %[dst2], #16 \n" + "add %[dst3], %[dst3], #16 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[width], %w[width], #4 \n" + "b.eq 3f \n" + + // Tail loop: 1..3 remaining source rows / destination columns. + "cmp %w[width], #1 \n" + "csel %[src1], %[src0], %[src1], eq \n" + "cmp %w[width], #2 \n" + "csel %[src2], %[src1], %[src2], le \n" + "mov %[src3], %[src2] \n" + "ld1w {z0.s}, p0/z, [%[src0]] \n" + "ld1w {z1.s}, p0/z, [%[src1]] \n" + "ld1w {z2.s}, p0/z, [%[src2]] \n" + "ld1w {z3.s}, p0/z, [%[src3]] \n" + "trn1 z4.s, z0.s, z1.s \n" + "trn2 z5.s, z0.s, z1.s \n" + "trn1 z6.s, z2.s, z3.s \n" + "trn2 z7.s, z2.s, z3.s \n" + "trn1 z0.d, z4.d, z6.d \n" + "trn2 z1.d, z4.d, z6.d \n" + "trn1 z2.d, z5.d, z7.d \n" + "trn2 z3.d, z5.d, z7.d \n" + "whilelt p0.s, wzr, %w[width] \n" + "st1w {z0.s}, p0, [%[dst0]] \n" + "st1w {z1.s}, p0, [%[dst1]] \n" + "st1w {z2.s}, p0, [%[dst2]] \n" + "st1w {z3.s}, p0, [%[dst3]] \n" + "3: \n" + : [src0] "+r"(src), // %[src0] + [src1] "+r"(src1), // %[src1] + [src2] "+r"(src2), // %[src2] + [src3] "+r"(src3), // %[src3] + [dst0] "+r"(dst), // %[dst0] + [dst1] "+r"(dst1), // %[dst1] + [dst2] "+r"(dst2), // %[dst2] + [dst3] "+r"(dst3), // %[dst3] + [width] "+r"(width) // %[width] + : [s4] "r"(s4) // %[s4] + : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0"); +} + +#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) + +#ifdef __cplusplus +} // extern "C" +} // namespace libyuv +#endif diff --git a/source/rotate_wasm.cc b/source/rotate_wasm.cc index f478a8bff..093a37e39 100644 --- a/source/rotate_wasm.cc +++ b/source/rotate_wasm.cc @@ -201,6 +201,42 @@ void Transpose4x4_32_WASMSIMD(const uint8_t* src, int width) { const ptrdiff_t s = src_stride; const ptrdiff_t d = dst_stride; + uint8_t* dst1 = dst + d; + uint8_t* dst2 = dst1 + d; + uint8_t* dst3 = dst2 + d; + while (width >= 8) { + v128_t r0 = wasm_v128_load(src); + v128_t r1 = wasm_v128_load(src + s); + v128_t r2 = wasm_v128_load(src + s * 2); + v128_t r3 = wasm_v128_load(src + s * 3); + src += s * 4; + v128_t r4 = wasm_v128_load(src); + v128_t r5 = wasm_v128_load(src + s); + v128_t r6 = wasm_v128_load(src + s * 2); + v128_t r7 = wasm_v128_load(src + s * 3); + src += s * 4; + v128_t a0 = ZIPLO32(r0, r1); + v128_t a1 = ZIPHI32(r0, r1); + v128_t a2 = ZIPLO32(r2, r3); + v128_t a3 = ZIPHI32(r2, r3); + v128_t b0 = ZIPLO32(r4, r5); + v128_t b1 = ZIPHI32(r4, r5); + v128_t b2 = ZIPLO32(r6, r7); + v128_t b3 = ZIPHI32(r6, r7); + wasm_v128_store(dst, ZIPLO64(a0, a2)); + wasm_v128_store(dst1, ZIPHI64(a0, a2)); + wasm_v128_store(dst2, ZIPLO64(a1, a3)); + wasm_v128_store(dst3, ZIPHI64(a1, a3)); + wasm_v128_store(dst + 16, ZIPLO64(b0, b2)); + wasm_v128_store(dst1 + 16, ZIPHI64(b0, b2)); + wasm_v128_store(dst2 + 16, ZIPLO64(b1, b3)); + wasm_v128_store(dst3 + 16, ZIPHI64(b1, b3)); + dst += 32; + dst1 += 32; + dst2 += 32; + dst3 += 32; + width -= 8; + } while (width > 0) { v128_t r0 = wasm_v128_load(src); v128_t r1 = wasm_v128_load(src + s); @@ -211,11 +247,14 @@ void Transpose4x4_32_WASMSIMD(const uint8_t* src, v128_t a2 = ZIPLO32(r2, r3); v128_t a3 = ZIPHI32(r2, r3); wasm_v128_store(dst, ZIPLO64(a0, a2)); - wasm_v128_store(dst + d, ZIPHI64(a0, a2)); - wasm_v128_store(dst + d * 2, ZIPLO64(a1, a3)); - wasm_v128_store(dst + d * 3, ZIPHI64(a1, a3)); + wasm_v128_store(dst1, ZIPHI64(a0, a2)); + wasm_v128_store(dst2, ZIPLO64(a1, a3)); + wasm_v128_store(dst3, ZIPHI64(a1, a3)); src += s * 4; dst += 16; + dst1 += 16; + dst2 += 16; + dst3 += 16; width -= 4; } } diff --git a/source/rotate_win.cc b/source/rotate_win.cc index 12eb63862..f5eb65dad 100644 --- a/source/rotate_win.cc +++ b/source/rotate_win.cc @@ -447,17 +447,18 @@ void Transpose4x4_32_SSE2(const uint8_t* src, } #endif // defined(HAS_TRANSPOSE4X4_32_SSE2) -#if defined(HAS_TRANSPOSE4X4_32_AVX2) -// Transpose 32 bit values (ARGB). Read a column of 8 pixels, write a row. +#if defined(HAS_TRANSPOSENX4_32_AVX2) +// Transpose 32 bit values (ARGB) in Nx4 tiles with 2 blocks of 4x4 at a time +// in ymm0..ymm3 and _mm_maskstore_epi32 tail for arbitrary N. LIBYUV_TARGET_AVX2 -void Transpose4x4_32_AVX2(const uint8_t* src, +void TransposeNx4_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) { const ptrdiff_t s = src_stride; const ptrdiff_t d = dst_stride; - while (width > 0) { + while (width >= 8) { __m256i r0 = _mm256_inserti128_si256( _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), _mm_loadu_si128((const __m128i*)(src + s * 4)), 1); @@ -482,9 +483,436 @@ void Transpose4x4_32_AVX2(const uint8_t* src, dst += 32; width -= 8; } + if (width >= 4) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); + __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3)); + _mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3)); + src += s * 4; + dst += 16; + width -= 4; + } + if (width > 0) { + const __m128i mask = + _mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (width * 8)) - 1)); + __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); + __m128i r1 = (width > 1) ? _mm_loadu_si128((const __m128i*)(src + s)) : r0; + __m128i r2 = + (width > 2) ? _mm_loadu_si128((const __m128i*)(src + s * 2)) : r1; + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r2); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r2); + _mm_maskstore_epi32((int*)(dst), mask, _mm_unpacklo_epi64(t0, t1)); + _mm_maskstore_epi32((int*)(dst + d), mask, _mm_unpackhi_epi64(t0, t1)); + _mm_maskstore_epi32((int*)(dst + d * 2), mask, _mm_unpacklo_epi64(t2, t3)); + _mm_maskstore_epi32((int*)(dst + d * 3), mask, _mm_unpackhi_epi64(t2, t3)); + } _mm256_zeroupper(); } -#endif // defined(HAS_TRANSPOSE4X4_32_AVX2) +#endif // defined(HAS_TRANSPOSENX4_32_AVX2) + +#if defined(HAS_TRANSPOSEWXH_32_AVX2) +LIBYUV_TARGET_AVX2 +void TransposeWxH_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (height >= 8) { + const uint8_t* src_row = src; + uint8_t* dst_col = dst; + int w = width; + while (w >= 8) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)(src_row)); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_row + s)); + __m256i r2 = _mm256_loadu_si256((const __m256i*)(src_row + s * 2)); + __m256i r3 = _mm256_loadu_si256((const __m256i*)(src_row + s * 3)); + __m256i r4 = _mm256_loadu_si256((const __m256i*)(src_row + s * 4)); + __m256i r5 = _mm256_loadu_si256((const __m256i*)(src_row + s * 5)); + __m256i r6 = _mm256_loadu_si256((const __m256i*)(src_row + s * 6)); + __m256i r7 = _mm256_loadu_si256((const __m256i*)(src_row + s * 7)); + + __m256i t0 = _mm256_unpacklo_epi32(r0, r1); + __m256i t1 = _mm256_unpackhi_epi32(r0, r1); + __m256i t2 = _mm256_unpacklo_epi32(r2, r3); + __m256i t3 = _mm256_unpackhi_epi32(r2, r3); + __m256i t4 = _mm256_unpacklo_epi32(r4, r5); + __m256i t5 = _mm256_unpackhi_epi32(r4, r5); + __m256i t6 = _mm256_unpacklo_epi32(r6, r7); + __m256i t7 = _mm256_unpackhi_epi32(r6, r7); + + __m256i u0 = _mm256_unpacklo_epi64(t0, t2); + __m256i u1 = _mm256_unpackhi_epi64(t0, t2); + __m256i u2 = _mm256_unpacklo_epi64(t1, t3); + __m256i u3 = _mm256_unpackhi_epi64(t1, t3); + __m256i u4 = _mm256_unpacklo_epi64(t4, t6); + __m256i u5 = _mm256_unpackhi_epi64(t4, t6); + __m256i u6 = _mm256_unpacklo_epi64(t5, t7); + __m256i u7 = _mm256_unpackhi_epi64(t5, t7); + + _mm256_storeu_si256((__m256i*)(dst_col), + _mm256_permute2x128_si256(u0, u4, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_col + d), + _mm256_permute2x128_si256(u1, u5, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_col + d * 2), + _mm256_permute2x128_si256(u2, u6, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_col + d * 3), + _mm256_permute2x128_si256(u3, u7, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_col + d * 4), + _mm256_permute2x128_si256(u0, u4, 0x31)); + _mm256_storeu_si256((__m256i*)(dst_col + d * 5), + _mm256_permute2x128_si256(u1, u5, 0x31)); + _mm256_storeu_si256((__m256i*)(dst_col + d * 6), + _mm256_permute2x128_si256(u2, u6, 0x31)); + _mm256_storeu_si256((__m256i*)(dst_col + d * 7), + _mm256_permute2x128_si256(u3, u7, 0x31)); + src_row += 32; + dst_col += 8 * d; + w -= 8; + } + if (w >= 4) { + TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, 8); + src_row += 16; + dst_col += 4 * d; + w -= 4; + } + if (w > 0) { + const __m128i col_mask = + _mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (w * 8)) - 1)); + uint8_t* dst0 = dst_col; + uint8_t* dst1 = (w > 1) ? dst0 + d : dst0; + uint8_t* dst2 = (w > 2) ? dst1 + d : dst1; + for (int y = 0; y < 8; y += 4) { + __m128i r0 = + _mm_maskload_epi32((const int*)(src_row + y * s), col_mask); + __m128i r1 = + _mm_maskload_epi32((const int*)(src_row + (y + 1) * s), col_mask); + __m128i r2 = + _mm_maskload_epi32((const int*)(src_row + (y + 2) * s), col_mask); + __m128i r3 = + _mm_maskload_epi32((const int*)(src_row + (y + 3) * s), col_mask); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst2 + y * 4), _mm_unpacklo_epi64(t2, t3)); + _mm_storeu_si128((__m128i*)(dst1 + y * 4), _mm_unpackhi_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst0 + y * 4), _mm_unpacklo_epi64(t0, t1)); + } + } + src += 8 * s; + dst += 32; + height -= 8; + } + while (height > 0) { + const int bh = (height >= 4) ? 4 : height; + const uint8_t* src_row = src; + uint8_t* dst_col = dst; + int w = width; + while (w >= 4) { + TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, bh); + src_row += 16; + dst_col += 4 * d; + w -= 4; + } + if (w > 0) { + const __m128i col_mask = + _mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (w * 8)) - 1)); + const __m128i row_mask = + _mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (bh * 8)) - 1)); + uint8_t* dst0 = dst_col; + uint8_t* dst1 = (w > 1) ? dst0 + d : dst0; + uint8_t* dst2 = (w > 2) ? dst1 + d : dst1; + __m128i r0 = _mm_maskload_epi32((const int*)(src_row), col_mask); + __m128i r1 = (bh > 1) + ? _mm_maskload_epi32((const int*)(src_row + s), col_mask) + : r0; + __m128i r2 = + (bh > 2) ? _mm_maskload_epi32((const int*)(src_row + s * 2), col_mask) + : r1; + __m128i r3 = + (bh > 3) ? _mm_maskload_epi32((const int*)(src_row + s * 3), col_mask) + : r2; + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_maskstore_epi32((int*)dst2, row_mask, _mm_unpacklo_epi64(t2, t3)); + _mm_maskstore_epi32((int*)dst1, row_mask, _mm_unpackhi_epi64(t0, t1)); + _mm_maskstore_epi32((int*)dst0, row_mask, _mm_unpacklo_epi64(t0, t1)); + } + src += bh * s; + dst += bh * 4; + height -= bh; + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX2) + +#if defined(HAS_TRANSPOSENX4_32_AVX512BW) +// Transpose 32 bit values (ARGB) in Nx4 tiles with 4 blocks of 4x4 at a time +// in zmm0..zmm3 and masked tail for arbitrary N. +LIBYUV_TARGET_AVX512BW +void TransposeNx4_32_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width >= 16) { + __m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src))); + __m512i r1 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s))); + __m512i r2 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2))); + __m512i r3 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3))); + r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 4)), + 1); + r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 5)), + 1); + r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 6)), + 1); + r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 7)), + 1); + r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)), + 2); + r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)), + 2); + r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)), + 2); + r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)), + 2); + r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 12)), + 3); + r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 13)), + 3); + r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 14)), + 3); + r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 15)), + 3); + __m512i t0 = _mm512_unpacklo_epi32(r0, r1); + __m512i t1 = _mm512_unpacklo_epi32(r2, r3); + __m512i t2 = _mm512_unpackhi_epi32(r0, r1); + __m512i t3 = _mm512_unpackhi_epi32(r2, r3); + _mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(t0, t1)); + _mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(t0, t1)); + _mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(t2, t3)); + _mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(t2, t3)); + src += s * 16; + dst += 64; + width -= 16; + } + while (width >= 4) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); + __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3)); + _mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3)); + src += s * 4; + dst += 16; + width -= 4; + } + if (width > 0) { + const __mmask8 mask = (1u << width) - 1; + __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); + __m128i r1 = (width > 1) ? _mm_loadu_si128((const __m128i*)(src + s)) : r0; + __m128i r2 = + (width > 2) ? _mm_loadu_si128((const __m128i*)(src + s * 2)) : r1; + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r2); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r2); + _mm_mask_storeu_epi32(dst, mask, _mm_unpacklo_epi64(t0, t1)); + _mm_mask_storeu_epi32(dst + d, mask, _mm_unpackhi_epi64(t0, t1)); + _mm_mask_storeu_epi32(dst + d * 2, mask, _mm_unpacklo_epi64(t2, t3)); + _mm_mask_storeu_epi32(dst + d * 3, mask, _mm_unpackhi_epi64(t2, t3)); + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSENX4_32_AVX512BW) + +#if defined(HAS_TRANSPOSEWXH_32_AVX512BW) +LIBYUV_TARGET_AVX512BW +void TransposeWxH_32_AVX512BW(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (height > 0) { + const int bh = (height >= 16) ? 16 : height; + const __mmask16 row_mask = (1u << bh) - 1; + const uint8_t* src_row = src; + uint8_t* dst_col = dst; + int w = width; + while (w > 0) { + const int bw = (w >= 16) ? 16 : w; + const __mmask16 col_mask = (1u << bw) - 1; + __m512i z = _mm512_setzero_si512(); + __m512i r0 = _mm512_maskz_loadu_epi32(col_mask, src_row); + __m512i r1 = + (bh > 1) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s) : z; + __m512i r2 = + (bh > 2) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 2) : z; + __m512i r3 = + (bh > 3) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 3) : z; + __m512i r4 = + (bh > 4) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 4) : z; + __m512i r5 = + (bh > 5) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 5) : z; + __m512i r6 = + (bh > 6) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 6) : z; + __m512i r7 = + (bh > 7) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 7) : z; + __m512i r8 = + (bh > 8) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 8) : z; + __m512i r9 = + (bh > 9) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 9) : z; + __m512i r10 = + (bh > 10) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 10) : z; + __m512i r11 = + (bh > 11) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 11) : z; + __m512i r12 = + (bh > 12) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 12) : z; + __m512i r13 = + (bh > 13) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 13) : z; + __m512i r14 = + (bh > 14) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 14) : z; + __m512i r15 = + (bh > 15) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 15) : z; + + __m512i t0 = _mm512_unpacklo_epi32(r0, r1); + __m512i t1 = _mm512_unpackhi_epi32(r0, r1); + __m512i t2 = _mm512_unpacklo_epi32(r2, r3); + __m512i t3 = _mm512_unpackhi_epi32(r2, r3); + __m512i t4 = _mm512_unpacklo_epi32(r4, r5); + __m512i t5 = _mm512_unpackhi_epi32(r4, r5); + __m512i t6 = _mm512_unpacklo_epi32(r6, r7); + __m512i t7 = _mm512_unpackhi_epi32(r6, r7); + __m512i t8 = _mm512_unpacklo_epi32(r8, r9); + __m512i t9 = _mm512_unpackhi_epi32(r8, r9); + __m512i t10 = _mm512_unpacklo_epi32(r10, r11); + __m512i t11 = _mm512_unpackhi_epi32(r10, r11); + __m512i t12 = _mm512_unpacklo_epi32(r12, r13); + __m512i t13 = _mm512_unpackhi_epi32(r12, r13); + __m512i t14 = _mm512_unpacklo_epi32(r14, r15); + __m512i t15 = _mm512_unpackhi_epi32(r14, r15); + + r0 = _mm512_unpacklo_epi64(t0, t2); + r1 = _mm512_unpackhi_epi64(t0, t2); + r2 = _mm512_unpacklo_epi64(t1, t3); + r3 = _mm512_unpackhi_epi64(t1, t3); + r4 = _mm512_unpacklo_epi64(t4, t6); + r5 = _mm512_unpackhi_epi64(t4, t6); + r6 = _mm512_unpacklo_epi64(t5, t7); + r7 = _mm512_unpackhi_epi64(t5, t7); + r8 = _mm512_unpacklo_epi64(t8, t10); + r9 = _mm512_unpackhi_epi64(t8, t10); + r10 = _mm512_unpacklo_epi64(t9, t11); + r11 = _mm512_unpackhi_epi64(t9, t11); + r12 = _mm512_unpacklo_epi64(t12, t14); + r13 = _mm512_unpackhi_epi64(t12, t14); + r14 = _mm512_unpacklo_epi64(t13, t15); + r15 = _mm512_unpackhi_epi64(t13, t15); + + t0 = _mm512_shuffle_i32x4(r0, r4, 0x88); + t1 = _mm512_shuffle_i32x4(r1, r5, 0x88); + t2 = _mm512_shuffle_i32x4(r2, r6, 0x88); + t3 = _mm512_shuffle_i32x4(r3, r7, 0x88); + t4 = _mm512_shuffle_i32x4(r0, r4, 0xdd); + t5 = _mm512_shuffle_i32x4(r1, r5, 0xdd); + t6 = _mm512_shuffle_i32x4(r2, r6, 0xdd); + t7 = _mm512_shuffle_i32x4(r3, r7, 0xdd); + t8 = _mm512_shuffle_i32x4(r8, r12, 0x88); + t9 = _mm512_shuffle_i32x4(r9, r13, 0x88); + t10 = _mm512_shuffle_i32x4(r10, r14, 0x88); + t11 = _mm512_shuffle_i32x4(r11, r15, 0x88); + t12 = _mm512_shuffle_i32x4(r8, r12, 0xdd); + t13 = _mm512_shuffle_i32x4(r9, r13, 0xdd); + t14 = _mm512_shuffle_i32x4(r10, r14, 0xdd); + t15 = _mm512_shuffle_i32x4(r11, r15, 0xdd); + + _mm512_mask_storeu_epi32(dst_col, row_mask, + _mm512_shuffle_i32x4(t0, t8, 0x88)); + if (bw > 1) + _mm512_mask_storeu_epi32(dst_col + d, row_mask, + _mm512_shuffle_i32x4(t1, t9, 0x88)); + if (bw > 2) + _mm512_mask_storeu_epi32(dst_col + d * 2, row_mask, + _mm512_shuffle_i32x4(t2, t10, 0x88)); + if (bw > 3) + _mm512_mask_storeu_epi32(dst_col + d * 3, row_mask, + _mm512_shuffle_i32x4(t3, t11, 0x88)); + if (bw > 4) + _mm512_mask_storeu_epi32(dst_col + d * 4, row_mask, + _mm512_shuffle_i32x4(t4, t12, 0x88)); + if (bw > 5) + _mm512_mask_storeu_epi32(dst_col + d * 5, row_mask, + _mm512_shuffle_i32x4(t5, t13, 0x88)); + if (bw > 6) + _mm512_mask_storeu_epi32(dst_col + d * 6, row_mask, + _mm512_shuffle_i32x4(t6, t14, 0x88)); + if (bw > 7) + _mm512_mask_storeu_epi32(dst_col + d * 7, row_mask, + _mm512_shuffle_i32x4(t7, t15, 0x88)); + if (bw > 8) + _mm512_mask_storeu_epi32(dst_col + d * 8, row_mask, + _mm512_shuffle_i32x4(t0, t8, 0xdd)); + if (bw > 9) + _mm512_mask_storeu_epi32(dst_col + d * 9, row_mask, + _mm512_shuffle_i32x4(t1, t9, 0xdd)); + if (bw > 10) + _mm512_mask_storeu_epi32(dst_col + d * 10, row_mask, + _mm512_shuffle_i32x4(t2, t10, 0xdd)); + if (bw > 11) + _mm512_mask_storeu_epi32(dst_col + d * 11, row_mask, + _mm512_shuffle_i32x4(t3, t11, 0xdd)); + if (bw > 12) + _mm512_mask_storeu_epi32(dst_col + d * 12, row_mask, + _mm512_shuffle_i32x4(t4, t12, 0xdd)); + if (bw > 13) + _mm512_mask_storeu_epi32(dst_col + d * 13, row_mask, + _mm512_shuffle_i32x4(t5, t13, 0xdd)); + if (bw > 14) + _mm512_mask_storeu_epi32(dst_col + d * 14, row_mask, + _mm512_shuffle_i32x4(t6, t14, 0xdd)); + if (bw > 15) + _mm512_mask_storeu_epi32(dst_col + d * 15, row_mask, + _mm512_shuffle_i32x4(t7, t15, 0xdd)); + src_row += 64; + dst_col += 16 * d; + w -= 16; + } + src += 16 * s; + dst += 64; + height -= 16; + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX512BW) #ifdef __cplusplus } // extern "C" diff --git a/source/row_any.cc b/source/row_any.cc index 43adb4758..2bd2957c5 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1046,6 +1046,12 @@ ANY11(CopyRow_Any_SSE2, CopyRow_SSE2, 0, 1, 1, 31) #ifdef HAS_COPYROW_NEON ANY11(CopyRow_Any_NEON, CopyRow_NEON, 0, 1, 1, 31) #endif +#ifdef HAS_COPYROW_LSX +ANY11(CopyRow_Any_LSX, CopyRow_LSX, 0, 1, 1, 31) +#endif +#ifdef HAS_COPYROW_LASX +ANY11(CopyRow_Any_LASX, CopyRow_LASX, 0, 1, 1, 63) +#endif #if defined(HAS_ARGBTORGB24ROW_SSSE3) ANY11(ARGBToRGB24Row_Any_SSSE3, ARGBToRGB24Row_SSSE3, 0, 4, 3, 15) ANY11(ARGBToRAWRow_Any_SSSE3, ARGBToRAWRow_SSSE3, 0, 4, 3, 15) diff --git a/source/row_lasx.cc b/source/row_lasx.cc index 0ae67b12a..d445ae09d 100644 --- a/source/row_lasx.cc +++ b/source/row_lasx.cc @@ -282,6 +282,20 @@ void ARGBMirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width) { } } +void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int width) { + int x; + int len = width / 64; + __m256i src0, src1; + + for (x = 0; x < len; x++) { + DUP2_ARG2(__lasx_xvld, src, 0, src, 32, src0, src1); + __lasx_xvst(src0, dst, 0); + __lasx_xvst(src1, dst, 32); + src += 64; + dst += 64; + } +} + void I422ToYUY2Row_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, diff --git a/source/row_lsx.cc b/source/row_lsx.cc index c54196100..a4c1b994c 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -322,6 +322,20 @@ void ARGBMirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width) { } } +void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int width) { + int x; + int len = width / 32; + __m128i src0, src1; + + for (x = 0; x < len; x++) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + __lsx_vst(src0, dst, 0); + __lsx_vst(src1, dst, 16); + src += 32; + dst += 32; + } +} + void I422ToYUY2Row_LSX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 2206c53e9..875bb12f0 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -2569,6 +2569,53 @@ void HalfWidthRow_16To8_RVV(const uint16_t* src_uv, } #endif +#ifdef HAS_ARGBMIRRORROW_RVV +void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) { + int vl; + ptrdiff_t byte_vl; + src += (ptrdiff_t)width * 4; + asm volatile( + "vsetvli %[vl], %[w], e32, m8, ta, ma \n" + "slli %[byte_vl], %[vl], 2 \n" + "vid.v v16 \n" + "addi %[vl], %[vl], -1 \n" + "vrsub.vx v16, v16, %[vl] \n" + "addi %[vl], %[vl], 1 \n" + "sub %[w], %[w], %[vl] \n" + "bltz %[w], 2f \n" + "1: \n" + "sub %[src], %[src], %[byte_vl] \n" + "vle32.v v8, (%[src]) \n" + "vrgather.vv v0, v8, v16 \n" + "vse32.v v0, (%[dst]) \n" + "add %[dst], %[dst], %[byte_vl] \n" + "sub %[w], %[w], %[vl] \n" + "bgez %[w], 1b \n" + "2: \n" + "add %[w], %[w], %[vl] \n" + "beqz %[w], 3f \n" + "vsetvli %[vl], %[w], e32, m8, ta, ma \n" + "slli %[byte_vl], %[vl], 2 \n" + "vid.v v16 \n" + "addi %[vl], %[vl], -1 \n" + "vrsub.vx v16, v16, %[vl] \n" + "sub %[src], %[src], %[byte_vl] \n" + "vle32.v v8, (%[src]) \n" + "vrgather.vv v0, v8, v16 \n" + "vse32.v v0, (%[dst]) \n" + "3: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [byte_vl] "=&r"(byte_vl) // %[byte_vl] + : + : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", + "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} +#endif // HAS_ARGBMIRRORROW_RVV + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/unit_test/rotate_argb_test.cc b/unit_test/rotate_argb_test.cc index 701e57a01..af7333512 100644 --- a/unit_test/rotate_argb_test.cc +++ b/unit_test/rotate_argb_test.cc @@ -119,6 +119,20 @@ TEST_F(LibYUVRotateTest, ARGBRotate270_Opt) { disable_cpu_flags_, benchmark_cpu_info_); } +TEST_F(LibYUVRotateTest, ARGBRotate90_Odd) { + ARGBTestRotate(benchmark_width_ + 1, benchmark_height_ + 1, + benchmark_height_ + 1, benchmark_width_ + 1, kRotate90, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_); +} + +TEST_F(LibYUVRotateTest, ARGBRotate270_Odd) { + ARGBTestRotate(benchmark_width_ + 1, benchmark_height_ + 1, + benchmark_height_ + 1, benchmark_width_ + 1, kRotate270, + benchmark_iterations_, disable_cpu_flags_, + benchmark_cpu_info_); +} + static void TestRotatePlane(int src_width, int src_height, int dst_width,