diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index b50e5fd3c..34ebe20f3 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -31,15 +31,15 @@ extern "C" { defined(LIBYUV_ENABLE_ROWWIN)))) #define HAS_TRANSPOSEWX8_SSSE3 #define HAS_TRANSPOSEUVWX8_SSE2 +#define HAS_TRANSPOSEWX16_16_AVX2 #define HAS_TRANSPOSE4X4_32_SSE2 -#define HAS_TRANSPOSENX4_32_AVX2 #define HAS_TRANSPOSEWXH_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 #if defined(__x86_64__) || defined(_M_X64) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW -#define HAS_TRANSPOSENX4_32_AVX512BW +#define HAS_TRANSPOSEWXH_16_AVX512BW #define HAS_TRANSPOSEWXH_32_AVX512BW #endif #endif @@ -50,8 +50,8 @@ extern "C" { !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_TRANSPOSEWX8_SSSE3 #define HAS_TRANSPOSEUVWX8_SSE2 +#define HAS_TRANSPOSEWX16_16_AVX2 #define HAS_TRANSPOSE4X4_32_SSE2 -#define HAS_TRANSPOSENX4_32_AVX2 #define HAS_TRANSPOSEWXH_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 @@ -62,7 +62,7 @@ extern "C" { defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW -#define HAS_TRANSPOSENX4_32_AVX512BW +#define HAS_TRANSPOSEWXH_16_AVX512BW #define HAS_TRANSPOSEWXH_32_AVX512BW #endif @@ -74,11 +74,13 @@ extern "C" { #define HAS_TRANSPOSEWX8_NEON #endif #define HAS_TRANSPOSEUVWX8_NEON +#define HAS_TRANSPOSEWX8_16_NEON #define HAS_TRANSPOSENX4_32_NEON #define HAS_TRANSPOSEWXH_32_NEON #endif #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) +#define HAS_TRANSPOSEWX8_16_SVE2 #define HAS_TRANSPOSENX4_32_SVE2 #endif @@ -86,20 +88,24 @@ extern "C" { defined(__aarch64__) #define HAS_TRANSPOSEWXH_SME #define HAS_TRANSPOSEUVWXH_SME +#define HAS_TRANSPOSEWXH_16_SME #define HAS_TRANSPOSEWXH_32_SME #endif #if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) #define HAS_TRANSPOSEWX16_LSX #define HAS_TRANSPOSEUVWX16_LSX +#define HAS_TRANSPOSEWX8_16_LSX #define HAS_TRANSPOSE4X4_32_LSX #endif #if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#define HAS_TRANSPOSEWX16_16_LASX #define HAS_TRANSPOSE4X4_32_LASX #endif #if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) +#define HAS_TRANSPOSEWXH_16_RVV #define HAS_TRANSPOSENX4_32_RVV #define HAS_TRANSPOSEWXH_32_RVV #endif @@ -109,6 +115,7 @@ extern "C" { #define HAS_TRANSPOSEUVWX8_WASMSIMD #define HAS_TRANSPOSEWX16_WASMSIMD #define HAS_TRANSPOSEUVWX16_WASMSIMD +#define HAS_TRANSPOSEWX8_16_WASMSIMD #define HAS_TRANSPOSE4X4_32_WASMSIMD #endif @@ -352,6 +359,79 @@ void TransposeWx1_16_C(const uint16_t* src, uint16_t* dst, int dst_stride, int width); +void TransposeWx16_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWxH_16_AVX512BW(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height); +void TransposeWx8_16_NEON(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_SVE2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWxH_16_SME(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height); +void TransposeWxH_16_RVV(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height); +void TransposeWx8_16_LSX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx16_16_LASX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_WASMSIMD(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx16_16_Any_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_Any_NEON(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_Any_LSX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx16_16_Any_LASX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_Any_WASMSIMD(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); void TransposeWxH_32_C(const uint8_t* src, int src_stride, @@ -392,12 +472,6 @@ void Transpose4x4_32_SSE2(const uint8_t* src, int dst_stride, int width); -void TransposeNx4_32_AVX2(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); - void TransposeWxH_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, @@ -405,12 +479,6 @@ void TransposeWxH_32_AVX2(const uint8_t* src, int width, int height); -void TransposeNx4_32_AVX512BW(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); - void TransposeWxH_32_AVX512BW(const uint8_t* src, int src_stride, uint8_t* dst, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index e70570758..dd9a6e108 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -412,6 +412,9 @@ extern "C" { #define HAS_INTERPOLATEROW_16_AVX2 #define HAS_MULTIPLYROW_16_AVX2 #define HAS_SWAPUVROW_AVX2 +#if defined(__x86_64__) || defined(_M_X64) +#define HAS_MIRRORUVROW_AVX512BW +#endif #endif // The following are available for AVX512 clang x64 platforms: @@ -452,6 +455,7 @@ extern "C" { #define HAS_J400TOARGBROW_AVX512BW #define HAS_MERGEUVROW_AVX512BW #define HAS_MIRRORROW_AVX512BW +#define HAS_MIRRORUVROW_AVX512BW #define HAS_MIRRORSPLITUVROW_AVX512BW #define HAS_SPLITUVROW_AVX512BW #define HAS_RGBTOUVMATRIXROW_AVX512BW @@ -930,6 +934,7 @@ extern "C" { #define HAS_MERGERGBROW_RVV #define HAS_MERGEUVROW_RVV #define HAS_MERGEXRGBROW_RVV +#define HAS_MIRRORUVROW_RVV #define HAS_NV12TOARGBROW_RVV #define HAS_NV12TORGB24ROW_RVV #define HAS_NV21TOARGBROW_RVV @@ -2749,13 +2754,19 @@ void MirrorRow_Any_SSE2(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width); +void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_C(const uint8_t* src_uv, uint8_t* dst_uv, int width); +void MirrorUVRow_Any_AVX512BW(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void MirrorUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void MirrorUVRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); diff --git a/source/planar_functions.cc b/source/planar_functions.cc index f3ee9e3f5..9f1a10062 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -2785,6 +2785,7 @@ void MirrorUVPlane(const uint8_t* src_uv, #endif #if defined(HAS_MIRRORUVROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { + MirrorUVRow = MirrorUVRow_Any_SSSE3; if (IS_ALIGNED(width, 8)) { MirrorUVRow = MirrorUVRow_SSSE3; } @@ -2798,6 +2799,14 @@ void MirrorUVPlane(const uint8_t* src_uv, } } #endif +#if defined(HAS_MIRRORUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MirrorUVRow = MirrorUVRow_Any_AVX512BW; + if (IS_ALIGNED(width, 32)) { + MirrorUVRow = MirrorUVRow_AVX512BW; + } + } +#endif #if defined(HAS_MIRRORUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { MirrorUVRow = MirrorUVRow_Any_LSX; @@ -2814,6 +2823,11 @@ void MirrorUVPlane(const uint8_t* src_uv, } } #endif +#if defined(HAS_MIRRORUVROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + MirrorUVRow = MirrorUVRow_RVV; + } +#endif #if defined(HAS_MIRRORUVROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { MirrorUVRow = MirrorUVRow_Any_WASMSIMD; diff --git a/source/rotate.cc b/source/rotate.cc index e8844f1b7..fab166711 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -618,11 +618,101 @@ static void TransposePlane_16(const uint16_t* src, int width, int height) { int i = height; +#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \ + defined(HAS_TRANSPOSEWXH_16_AVX512BW) + void (*TransposeWxH_16)(const uint16_t* src, int src_stride, uint16_t* dst, + int dst_stride, int width, int height) = NULL; +#endif +#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX) + void (*TransposeWx16_16)(const uint16_t* src, int src_stride, uint16_t* dst, + int dst_stride, int width) = NULL; +#endif + void (*TransposeWx8_16)(const uint16_t* src, int src_stride, uint16_t* dst, + int dst_stride, int width) = TransposeWx8_16_C; + +#if defined(HAS_TRANSPOSEWX8_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + TransposeWx8_16 = TransposeWx8_16_Any_NEON; + if (IS_ALIGNED(width, 8)) { + TransposeWx8_16 = TransposeWx8_16_NEON; + } + } +#endif +#if defined(HAS_TRANSPOSEWX8_16_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + TransposeWx8_16 = TransposeWx8_16_SVE2; + } +#endif +#if defined(HAS_TRANSPOSEWXH_16_SME) + if (TestCpuFlag(kCpuHasSME)) { + TransposeWxH_16 = TransposeWxH_16_SME; + } +#endif +#if defined(HAS_TRANSPOSEWXH_16_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + TransposeWxH_16 = TransposeWxH_16_RVV; + } +#endif +#if defined(HAS_TRANSPOSEWX16_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx16_16 = TransposeWx16_16_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + TransposeWx16_16 = TransposeWx16_16_AVX2; + } + } +#endif +#if defined(HAS_TRANSPOSEWXH_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + TransposeWxH_16 = TransposeWxH_16_AVX512BW; + } +#endif +#if defined(HAS_TRANSPOSEWX8_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + TransposeWx8_16 = TransposeWx8_16_Any_LSX; + if (IS_ALIGNED(width, 8)) { + TransposeWx8_16 = TransposeWx8_16_LSX; + } + } +#endif +#if defined(HAS_TRANSPOSEWX16_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + TransposeWx16_16 = TransposeWx16_16_Any_LASX; + if (IS_ALIGNED(width, 8)) { + TransposeWx16_16 = TransposeWx16_16_LASX; + } + } +#endif +#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + TransposeWx8_16 = TransposeWx8_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + TransposeWx8_16 = TransposeWx8_16_WASMSIMD; + } + } +#endif + +#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \ + defined(HAS_TRANSPOSEWXH_16_AVX512BW) + if (TransposeWxH_16) { + TransposeWxH_16(src, src_stride, dst, dst_stride, width, height); + return; + } +#endif +#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX) + if (TransposeWx16_16) { + while (i >= 16) { + TransposeWx16_16(src, src_stride, dst, dst_stride, width); + src += 16 * (ptrdiff_t)src_stride; + dst += 16; + i -= 16; + } + } +#endif // Work across the source in 8x8 tiles while (i >= 8) { - TransposeWx8_16_C(src, src_stride, dst, dst_stride, width); - src += 8 * src_stride; // Go down 8 rows. - dst += 8; // Move over 8 columns. + TransposeWx8_16(src, src_stride, dst, dst_stride, width); + src += 8 * (ptrdiff_t)src_stride; // Go down 8 rows. + dst += 8; // Move over 8 columns. i -= 8; } @@ -665,29 +755,9 @@ static void RotatePlane180_16(const uint16_t* src, int dst_stride, int width, int height) { - const uint16_t* src_bot = src + (ptrdiff_t)src_stride * (height - 1); - uint16_t* dst_bot = dst + (ptrdiff_t)dst_stride * (height - 1); - int half_height = (height + 1) >> 1; - int y; - - // Swap top and bottom row and mirror the content. Uses a temporary row. - align_buffer_64(row, width * 2); - uint16_t* row_tmp = (uint16_t*)row; - assert(row); - if (!row) - return; - - // Odd height will harmlessly mirror the middle row twice. - for (y = 0; y < half_height; ++y) { - CopyRow_16_C(src, row_tmp, width); // Copy top row into buffer - MirrorRow_16_C(src_bot, dst, width); // Mirror bottom row into top row - MirrorRow_16_C(row_tmp, dst_bot, width); // Mirror buffer into bottom row - src += src_stride; - dst += dst_stride; - src_bot -= src_stride; - dst_bot -= dst_stride; - } - free_aligned_buffer_64(row); + // Rotate by 180 is a vertical flip (negative height) and horizontal mirror. + MirrorUVPlane((const uint8_t*)src, src_stride * 2, (uint8_t*)dst, + dst_stride * 2, width, -height); } LIBYUV_API diff --git a/source/rotate_any.cc b/source/rotate_any.cc index c474bc262..8a4288400 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -122,6 +122,36 @@ TANY32(Transpose4x4_32_Any_WASMSIMD, #endif #undef TANY32 +#define TANY16(NAMEANY, TPOS_SIMD, MASK, HEIGHT) \ + void NAMEANY(const uint16_t* src, int src_stride, uint16_t* dst, \ + int dst_stride, int width) { \ + int r = width & MASK; \ + int n = width - r; \ + if (n > 0) { \ + TPOS_SIMD(src, src_stride, dst, dst_stride, n); \ + } \ + ptrdiff_t np = n; \ + TransposeWxH_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \ + r, HEIGHT); \ + } + +#ifdef HAS_TRANSPOSEWX8_16_NEON +TANY16(TransposeWx8_16_Any_NEON, TransposeWx8_16_NEON, 7, 8) +#endif +#ifdef HAS_TRANSPOSEWX16_16_AVX2 +TANY16(TransposeWx16_16_Any_AVX2, TransposeWx16_16_AVX2, 7, 16) +#endif +#ifdef HAS_TRANSPOSEWX8_16_LSX +TANY16(TransposeWx8_16_Any_LSX, TransposeWx8_16_LSX, 7, 8) +#endif +#ifdef HAS_TRANSPOSEWX16_16_LASX +TANY16(TransposeWx16_16_Any_LASX, TransposeWx16_16_LASX, 7, 16) +#endif +#ifdef HAS_TRANSPOSEWX8_16_WASMSIMD +TANY16(TransposeWx8_16_Any_WASMSIMD, TransposeWx8_16_WASMSIMD, 7, 8) +#endif +#undef TANY16 + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index a66697403..471028f12 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -664,6 +664,474 @@ void TransposeUVWx8_SSE2(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEUVWX8_SSE2) +#if defined(HAS_TRANSPOSEWX16_16_AVX2) +void TransposeWx16_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm4 \n" + "vmovdqu (%[temp],%[s]),%%xmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm6 \n" + "vmovdqu (%[temp],%[s]),%%xmm7 \n" + "lea (%[temp],%[s],2),%[temp] \n" + + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm4,%%ymm4 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm5,%%ymm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm6,%%ymm6 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm7,%%ymm7 \n" + "add $0x10,%[src] \n" + + // Stage 1: 16-bit unpack -> ymm8..ymm15 + "vpunpcklwd %%ymm1,%%ymm0,%%ymm8 \n" + "vpunpckhwd %%ymm1,%%ymm0,%%ymm9 \n" + "vpunpcklwd %%ymm3,%%ymm2,%%ymm10 \n" + "vpunpckhwd %%ymm3,%%ymm2,%%ymm11 \n" + "vpunpcklwd %%ymm5,%%ymm4,%%ymm12 \n" + "vpunpckhwd %%ymm5,%%ymm4,%%ymm13 \n" + "vpunpcklwd %%ymm7,%%ymm6,%%ymm14 \n" + "vpunpckhwd %%ymm7,%%ymm6,%%ymm15 \n" + + // Stage 2: 32-bit unpack -> ymm0..ymm7 + "vpunpckldq %%ymm10,%%ymm8,%%ymm0 \n" + "vpunpckhdq %%ymm10,%%ymm8,%%ymm1 \n" + "vpunpckldq %%ymm11,%%ymm9,%%ymm2 \n" + "vpunpckhdq %%ymm11,%%ymm9,%%ymm3 \n" + "vpunpckldq %%ymm14,%%ymm12,%%ymm4 \n" + "vpunpckhdq %%ymm14,%%ymm12,%%ymm5 \n" + "vpunpckldq %%ymm15,%%ymm13,%%ymm6 \n" + "vpunpckhdq %%ymm15,%%ymm13,%%ymm7 \n" + + // Stage 3: 64-bit unpack -> ymm8..ymm15 + "vpunpcklqdq %%ymm4,%%ymm0,%%ymm8 \n" + "vpunpckhqdq %%ymm4,%%ymm0,%%ymm9 \n" + "vpunpcklqdq %%ymm5,%%ymm1,%%ymm10 \n" + "vpunpckhqdq %%ymm5,%%ymm1,%%ymm11 \n" + "vpunpcklqdq %%ymm6,%%ymm2,%%ymm12 \n" + "vpunpckhqdq %%ymm6,%%ymm2,%%ymm13 \n" + "vpunpcklqdq %%ymm7,%%ymm3,%%ymm14 \n" + "vpunpckhqdq %%ymm7,%%ymm3,%%ymm15 \n" + + "vmovdqu %%ymm8,(%[dst]) \n" + "vmovdqu %%ymm9,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%ymm10,(%[dst]) \n" + "vmovdqu %%ymm11,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%ymm12,(%[dst]) \n" + "vmovdqu %%ymm13,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%ymm14,(%[dst]) \n" + "vmovdqu %%ymm15,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x8,%[w] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_b), // %[src] + [dst] "+r"(dst_b), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", + "xmm15"); +} +#endif // defined(HAS_TRANSPOSEWX16_16_AVX2) + +#if defined(HAS_TRANSPOSEWXH_16_AVX512BW) +// Transpose 16 bit plane of any width x height using 8x32 tiles in zmm0..zmm15 +// with k1/k2-masked right and bottom tails. +void TransposeWxH_16_AVX512BW(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + while (height > 0) { + const int bh = (height >= 32) ? 32 : height; + const uint8_t* src_row = src_b; + uint8_t* dst_col = dst_b; + int w = width; + if (bh == 32) { + const int n = w & ~7; + if (n > 0) { + const uint8_t* s_ptr = src_row; + uint8_t* d_ptr = dst_col; + int cnt = n; + const uint8_t* temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm4 \n" + "vmovdqu (%[temp],%[s]),%%xmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm6 \n" + "vmovdqu (%[temp],%[s]),%%xmm7 \n" + "lea (%[temp],%[s],2),%[temp] \n" + + "vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $1,(%[temp]),%%zmm4,%%zmm4 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm5,%%zmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $1,(%[temp]),%%zmm6,%%zmm6 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm7,%%zmm7 \n" + "lea (%[temp],%[s],2),%[temp] \n" + + "vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $2,(%[temp]),%%zmm4,%%zmm4 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm5,%%zmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $2,(%[temp]),%%zmm6,%%zmm6 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm7,%%zmm7 \n" + "lea (%[temp],%[s],2),%[temp] \n" + + "vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $3,(%[temp]),%%zmm4,%%zmm4 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm5,%%zmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $3,(%[temp]),%%zmm6,%%zmm6 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm7,%%zmm7 \n" + "add $0x10,%[src] \n" + + // Stage 1: 16-bit unpack -> zmm8..zmm15 + "vpunpcklwd %%zmm1,%%zmm0,%%zmm8 \n" + "vpunpckhwd %%zmm1,%%zmm0,%%zmm9 \n" + "vpunpcklwd %%zmm3,%%zmm2,%%zmm10 \n" + "vpunpckhwd %%zmm3,%%zmm2,%%zmm11 \n" + "vpunpcklwd %%zmm5,%%zmm4,%%zmm12 \n" + "vpunpckhwd %%zmm5,%%zmm4,%%zmm13 \n" + "vpunpcklwd %%zmm7,%%zmm6,%%zmm14 \n" + "vpunpckhwd %%zmm7,%%zmm6,%%zmm15 \n" + + // Stage 2: 32-bit unpack -> zmm0..zmm7 + "vpunpckldq %%zmm10,%%zmm8,%%zmm0 \n" + "vpunpckhdq %%zmm10,%%zmm8,%%zmm1 \n" + "vpunpckldq %%zmm11,%%zmm9,%%zmm2 \n" + "vpunpckhdq %%zmm11,%%zmm9,%%zmm3 \n" + "vpunpckldq %%zmm14,%%zmm12,%%zmm4 \n" + "vpunpckhdq %%zmm14,%%zmm12,%%zmm5 \n" + "vpunpckldq %%zmm15,%%zmm13,%%zmm6 \n" + "vpunpckhdq %%zmm15,%%zmm13,%%zmm7 \n" + + // Stage 3: 64-bit unpack -> zmm8..zmm15 + "vpunpcklqdq %%zmm4,%%zmm0,%%zmm8 \n" + "vpunpckhqdq %%zmm4,%%zmm0,%%zmm9 \n" + "vpunpcklqdq %%zmm5,%%zmm1,%%zmm10 \n" + "vpunpckhqdq %%zmm5,%%zmm1,%%zmm11 \n" + "vpunpcklqdq %%zmm6,%%zmm2,%%zmm12 \n" + "vpunpckhqdq %%zmm6,%%zmm2,%%zmm13 \n" + "vpunpcklqdq %%zmm7,%%zmm3,%%zmm14 \n" + "vpunpckhqdq %%zmm7,%%zmm3,%%zmm15 \n" + + "vmovdqu16 %%zmm8,(%[dst]) \n" + "vmovdqu16 %%zmm9,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu16 %%zmm10,(%[dst]) \n" + "vmovdqu16 %%zmm11,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu16 %%zmm12,(%[dst]) \n" + "vmovdqu16 %%zmm13,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu16 %%zmm14,(%[dst]) \n" + "vmovdqu16 %%zmm15,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x8,%[cnt] \n" + "jg 1b \n" + : [src] "+r"(s_ptr), // %[src] + [dst] "+r"(d_ptr), // %[dst] + [cnt] "+r"(cnt), // %[cnt] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", + "xmm13", "xmm14", "xmm15"); + src_row += (ptrdiff_t)n * 2; + dst_col += (ptrdiff_t)n * d; + w -= n; + } + } + const uint32_t row_mask = (bh == 32) ? 0xFFFFFFFFu : ((1u << bh) - 1); + while (w > 0) { + const int bw = (w >= 8) ? 8 : w; + const uint32_t col_mask = (1u << bw) - 1; + const uint8_t* s_ptr = src_row; + uint8_t* d_ptr = dst_col; + asm volatile( + "kmovw %[col_mask],%%k1 \n" + "kmovd %[row_mask],%%k2 \n" + "vpxord %%zmm0,%%zmm0,%%zmm0 \n" + "vmovdqa64 %%zmm0,%%zmm1 \n" + "vmovdqa64 %%zmm0,%%zmm2 \n" + "vmovdqa64 %%zmm0,%%zmm3 \n" + "vmovdqa64 %%zmm0,%%zmm4 \n" + "vmovdqa64 %%zmm0,%%zmm5 \n" + "vmovdqa64 %%zmm0,%%zmm6 \n" + "vmovdqa64 %%zmm0,%%zmm7 \n" + + "vmovdqu16 (%[s_ptr]),%%xmm0%{%%k1%}%{z%} \n" + "cmp $1,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm1%{%%k1%}%{z%} \n" + "cmp $2,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm2%{%%k1%}%{z%} \n" + "cmp $3,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm3%{%%k1%}%{z%} \n" + "cmp $4,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm4%{%%k1%}%{z%} \n" + "cmp $5,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm5%{%%k1%}%{z%} \n" + "cmp $6,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm6%{%%k1%}%{z%} \n" + "cmp $7,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm7%{%%k1%}%{z%} \n" + "cmp $8,%[bh] \n" + "jle 2f \n" + + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm16,%%zmm0,%%zmm0 \n" + "cmp $9,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm16,%%zmm1,%%zmm1 \n" + "cmp $10,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm16,%%zmm2,%%zmm2 \n" + "cmp $11,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm16,%%zmm3,%%zmm3 \n" + "cmp $12,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm16,%%zmm4,%%zmm4 \n" + "cmp $13,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm16,%%zmm5,%%zmm5 \n" + "cmp $14,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm16,%%zmm6,%%zmm6 \n" + "cmp $15,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm16,%%zmm7,%%zmm7 \n" + "cmp $16,%[bh] \n" + "jle 2f \n" + + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm16,%%zmm0,%%zmm0 \n" + "cmp $17,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm16,%%zmm1,%%zmm1 \n" + "cmp $18,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm16,%%zmm2,%%zmm2 \n" + "cmp $19,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm16,%%zmm3,%%zmm3 \n" + "cmp $20,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm16,%%zmm4,%%zmm4 \n" + "cmp $21,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm16,%%zmm5,%%zmm5 \n" + "cmp $22,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm16,%%zmm6,%%zmm6 \n" + "cmp $23,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm16,%%zmm7,%%zmm7 \n" + "cmp $24,%[bh] \n" + "jle 2f \n" + + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm16,%%zmm0,%%zmm0 \n" + "cmp $25,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm16,%%zmm1,%%zmm1 \n" + "cmp $26,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm16,%%zmm2,%%zmm2 \n" + "cmp $27,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm16,%%zmm3,%%zmm3 \n" + "cmp $28,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm16,%%zmm4,%%zmm4 \n" + "cmp $29,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm16,%%zmm5,%%zmm5 \n" + "cmp $30,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu16 (%[s_ptr]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm16,%%zmm6,%%zmm6 \n" + "cmp $31,%[bh] \n" + "jle 2f \n" + "vmovdqu16 (%[s_ptr],%[s]),%%xmm16%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm16,%%zmm7,%%zmm7 \n" + "2: \n" + + // Stage 1: 16-bit unpack -> zmm8..zmm15 + "vpunpcklwd %%zmm1,%%zmm0,%%zmm8 \n" + "vpunpckhwd %%zmm1,%%zmm0,%%zmm9 \n" + "vpunpcklwd %%zmm3,%%zmm2,%%zmm10 \n" + "vpunpckhwd %%zmm3,%%zmm2,%%zmm11 \n" + "vpunpcklwd %%zmm5,%%zmm4,%%zmm12 \n" + "vpunpckhwd %%zmm5,%%zmm4,%%zmm13 \n" + "vpunpcklwd %%zmm7,%%zmm6,%%zmm14 \n" + "vpunpckhwd %%zmm7,%%zmm6,%%zmm15 \n" + + // Stage 2: 32-bit unpack -> zmm0..zmm7 + "vpunpckldq %%zmm10,%%zmm8,%%zmm0 \n" + "vpunpckhdq %%zmm10,%%zmm8,%%zmm1 \n" + "vpunpckldq %%zmm11,%%zmm9,%%zmm2 \n" + "vpunpckhdq %%zmm11,%%zmm9,%%zmm3 \n" + "vpunpckldq %%zmm14,%%zmm12,%%zmm4 \n" + "vpunpckhdq %%zmm14,%%zmm12,%%zmm5 \n" + "vpunpckldq %%zmm15,%%zmm13,%%zmm6 \n" + "vpunpckhdq %%zmm15,%%zmm13,%%zmm7 \n" + + // Stage 3: 64-bit unpack -> zmm8..zmm15 + "vpunpcklqdq %%zmm4,%%zmm0,%%zmm8 \n" + "vpunpckhqdq %%zmm4,%%zmm0,%%zmm9 \n" + "vpunpcklqdq %%zmm5,%%zmm1,%%zmm10 \n" + "vpunpckhqdq %%zmm5,%%zmm1,%%zmm11 \n" + "vpunpcklqdq %%zmm6,%%zmm2,%%zmm12 \n" + "vpunpckhqdq %%zmm6,%%zmm2,%%zmm13 \n" + "vpunpcklqdq %%zmm7,%%zmm3,%%zmm14 \n" + "vpunpckhqdq %%zmm7,%%zmm3,%%zmm15 \n" + + "vmovdqu16 %%zmm8,(%[d_ptr])%{%%k2%} \n" + "cmp $1,%[bw] \n" + "jle 3f \n" + "vmovdqu16 %%zmm9,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $2,%[bw] \n" + "jle 3f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu16 %%zmm10,(%[d_ptr])%{%%k2%} \n" + "cmp $3,%[bw] \n" + "jle 3f \n" + "vmovdqu16 %%zmm11,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $4,%[bw] \n" + "jle 3f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu16 %%zmm12,(%[d_ptr])%{%%k2%} \n" + "cmp $5,%[bw] \n" + "jle 3f \n" + "vmovdqu16 %%zmm13,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $6,%[bw] \n" + "jle 3f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu16 %%zmm14,(%[d_ptr])%{%%k2%} \n" + "cmp $7,%[bw] \n" + "jle 3f \n" + "vmovdqu16 %%zmm15,(%[d_ptr],%[d])%{%%k2%} \n" + "3: \n" + : [s_ptr] "+r"(s_ptr), // %[s_ptr] + [d_ptr] "+r"(d_ptr) // %[d_ptr] + : [s] "r"(s), // %[s] + [d] "r"(d), // %[d] + [col_mask] "r"(col_mask), // %[col_mask] + [row_mask] "r"(row_mask), // %[row_mask] + [bh] "r"(bh), // %[bh] + [bw] "r"(bw) // %[bw] + : "memory", "cc", "k1", "k2", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", + "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", + "xmm13", "xmm14", "xmm15", "xmm16"); + src_row += 16; + dst_col += 8 * d; + w -= 8; + } + src_b += 32 * s; + dst_b += 64; + height -= 32; + } + asm volatile("vzeroupper \n"); +} +#endif // defined(HAS_TRANSPOSEWXH_16_AVX512BW) + #if defined(HAS_TRANSPOSE4X4_32_SSE2) // 4 values, little endian view // a b c d @@ -737,138 +1205,9 @@ void Transpose4x4_32_SSE2(const uint8_t* src, } #endif // defined(HAS_TRANSPOSE4X4_32_SSE2) -#if defined(HAS_TRANSPOSENX4_32_AVX2) -// Transpose 32 bit values (ARGB) in Nx4 tiles with 2 blocks of 4x4 at a time -// in ymm0..ymm3 and vpmaskmovd tail for arbitrary N. -void TransposeNx4_32_AVX2(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - const ptrdiff_t s = src_stride; - const ptrdiff_t d = dst_stride; - uint8_t* dst1 = dst + d; - uint8_t* dst2 = dst1 + d; - uint8_t* dst3 = dst2 + d; - asm volatile( - "sub $0x8,%[w] \n" - "jl 2f \n" - - // Main loop: 2 blocks of 4x4 (8 source rows -> 8 dest columns / 32 B). - "1: \n" - "vmovdqu (%[src]),%%xmm0 \n" // a b c d - "vmovdqu (%[src],%[s]),%%xmm1 \n" // e f g h - "lea (%[src],%[s],2),%[src] \n" // src += stride * 2 - "vmovdqu (%[src]),%%xmm2 \n" // i j k l - "vmovdqu (%[src],%[s]),%%xmm3 \n" // m n o p - "lea (%[src],%[s],2),%[src] \n" // src += stride * 2 - - "vinserti128 $1,(%[src]),%%ymm0,%%ymm0 \n" - "vinserti128 $1,(%[src],%[s]),%%ymm1,%%ymm1 \n" - "lea (%[src],%[s],2),%[src] \n" - "vinserti128 $1,(%[src]),%%ymm2,%%ymm2 \n" - "vinserti128 $1,(%[src],%[s]),%%ymm3,%%ymm3 \n" - "lea (%[src],%[s],2),%[src] \n" - - // Transpose 2x2 across both 128-bit lanes - "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" - "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" - "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" - "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" - - // Transpose 4x4 across both 128-bit lanes - "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" - "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" - "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" - "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" - - "vmovdqu %%ymm0,(%[dst0]) \n" - "vmovdqu %%ymm1,(%[dst1]) \n" - "vmovdqu %%ymm2,(%[dst2]) \n" - "vmovdqu %%ymm3,(%[dst3]) \n" - "add $0x20,%[dst0] \n" - "add $0x20,%[dst1] \n" - "add $0x20,%[dst2] \n" - "add $0x20,%[dst3] \n" - "sub $0x8,%[w] \n" - "jge 1b \n" - - "2: \n" - "add $0x8,%[w] \n" - "jz 5f \n" - - // 4x4 block for 4..7 remainder elements - "cmp $0x4,%[w] \n" - "jl 3f \n" - "vmovdqu (%[src]),%%xmm0 \n" - "vmovdqu (%[src],%[s]),%%xmm1 \n" - "lea (%[src],%[s],2),%[src] \n" - "vmovdqu (%[src]),%%xmm2 \n" - "vmovdqu (%[src],%[s]),%%xmm3 \n" - "lea (%[src],%[s],2),%[src] \n" - "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" - "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" - "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" - "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" - "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" - "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" - "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" - "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" - "vmovdqu %%xmm0,(%[dst0]) \n" - "vmovdqu %%xmm1,(%[dst1]) \n" - "vmovdqu %%xmm2,(%[dst2]) \n" - "vmovdqu %%xmm3,(%[dst3]) \n" - "add $0x10,%[dst0] \n" - "add $0x10,%[dst1] \n" - "add $0x10,%[dst2] \n" - "add $0x10,%[dst3] \n" - "sub $0x4,%[w] \n" - "jz 5f \n" - - // Masked tail for 1..3 remainder elements using vpmaskmovd - "3: \n" - "vmovdqu (%[src]),%%xmm0 \n" - "vmovdqa %%xmm0,%%xmm1 \n" - "vmovdqa %%xmm0,%%xmm2 \n" - "mov $0x000000ff,%%eax \n" - "cmp $2,%[w] \n" - "jl 4f \n" - "vmovdqu (%[src],%[s]),%%xmm1 \n" - "mov $0x0000ffff,%%eax \n" - "je 4f \n" - "vmovdqu (%[src],%[s],2),%%xmm2 \n" - "mov $0x00ffffff,%%eax \n" - "4: \n" - "vmovd %%eax,%%xmm7 \n" - "vpmovsxbd %%xmm7,%%xmm7 \n" - "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" - "vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n" - "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" - "vpunpckhdq %%xmm2,%%xmm2,%%xmm3 \n" - "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" - "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" - "vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n" - "vpunpckhqdq %%xmm3,%%xmm6,%%xmm3 \n" - "vpmaskmovd %%xmm0,%%xmm7,(%[dst0]) \n" - "vpmaskmovd %%xmm1,%%xmm7,(%[dst1]) \n" - "vpmaskmovd %%xmm2,%%xmm7,(%[dst2]) \n" - "vpmaskmovd %%xmm3,%%xmm7,(%[dst3]) \n" - "5: \n" - "vzeroupper \n" - : [src] "+r"(src), // %[src] - [dst0] "+r"(dst), // %[dst0] - [dst1] "+r"(dst1), // %[dst1] - [dst2] "+r"(dst2), // %[dst2] - [dst3] "+r"(dst3), // %[dst3] - [w] "+r"(width) // %[w] - : [s] "r"(s) // %[s] - : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6", "xmm7"); -} -#endif // defined(HAS_TRANSPOSENX4_32_AVX2) - #if defined(HAS_TRANSPOSEWXH_32_AVX2) -// Transpose 32 bit plane of any width x height using AVX2 vpmaskmovd. +// Transpose 32 bit plane of any width x height using 4x8 YMM vertical tiles +// in ymm0..ymm7 with vpmaskmovd right and bottom tail masking. void TransposeWxH_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, @@ -877,323 +1216,162 @@ void TransposeWxH_32_AVX2(const uint8_t* src, int height) { const ptrdiff_t s = src_stride; const ptrdiff_t d = dst_stride; - while (height >= 8) { + while (height > 0) { + const int bh = (height >= 8) ? 8 : height; const uint8_t* src_row = src; uint8_t* dst_col = dst; int w = width; - while (w >= 8) { - asm volatile( - "vmovdqu (%[s0]),%%ymm0 \n" - "vmovdqu (%[s0],%[s]),%%ymm1 \n" - "lea (%[s0],%[s],2),%%rax \n" - "vmovdqu (%%rax),%%ymm2 \n" - "vmovdqu (%%rax,%[s]),%%ymm3 \n" - "lea (%%rax,%[s],2),%%rax \n" - "vmovdqu (%%rax),%%ymm4 \n" - "vmovdqu (%%rax,%[s]),%%ymm5 \n" - "lea (%%rax,%[s],2),%%rax \n" - "vmovdqu (%%rax),%%ymm6 \n" - "vmovdqu (%%rax,%[s]),%%ymm7 \n" + if (bh == 8) { + const int n = w & ~3; + if (n > 0) { + const uint8_t* s_ptr = src_row; + uint8_t* d_ptr = dst_col; + int cnt = n; + const uint8_t* temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" - "vpunpckldq %%ymm1,%%ymm0,%%ymm8 \n" - "vpunpckhdq %%ymm1,%%ymm0,%%ymm9 \n" - "vpunpckldq %%ymm3,%%ymm2,%%ymm10 \n" - "vpunpckhdq %%ymm3,%%ymm2,%%ymm11 \n" - "vpunpckldq %%ymm5,%%ymm4,%%ymm12 \n" - "vpunpckhdq %%ymm5,%%ymm4,%%ymm13 \n" - "vpunpckldq %%ymm7,%%ymm6,%%ymm14 \n" - "vpunpckhdq %%ymm7,%%ymm6,%%ymm15 \n" + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n" + "add $0x10,%[src] \n" - "vpunpcklqdq %%ymm10,%%ymm8,%%ymm0 \n" - "vpunpckhqdq %%ymm10,%%ymm8,%%ymm1 \n" - "vpunpcklqdq %%ymm11,%%ymm9,%%ymm2 \n" - "vpunpckhqdq %%ymm11,%%ymm9,%%ymm3 \n" - "vpunpcklqdq %%ymm14,%%ymm12,%%ymm4 \n" - "vpunpckhqdq %%ymm14,%%ymm12,%%ymm5 \n" - "vpunpcklqdq %%ymm15,%%ymm13,%%ymm6 \n" - "vpunpckhqdq %%ymm15,%%ymm13,%%ymm7 \n" + // Stage 1: 32-bit unpack across both 128-bit lanes + "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" - "vperm2i128 $0x20,%%ymm4,%%ymm0,%%ymm8 \n" - "vperm2i128 $0x20,%%ymm5,%%ymm1,%%ymm9 \n" - "vperm2i128 $0x20,%%ymm6,%%ymm2,%%ymm10 \n" - "vperm2i128 $0x20,%%ymm7,%%ymm3,%%ymm11 \n" - "vperm2i128 $0x31,%%ymm4,%%ymm0,%%ymm12 \n" - "vperm2i128 $0x31,%%ymm5,%%ymm1,%%ymm13 \n" - "vperm2i128 $0x31,%%ymm6,%%ymm2,%%ymm14 \n" - "vperm2i128 $0x31,%%ymm7,%%ymm3,%%ymm15 \n" + // Stage 2: 64-bit unpack across both 128-bit lanes + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" - "vmovdqu %%ymm8,(%[d0]) \n" - "vmovdqu %%ymm9,(%[d0],%[d]) \n" - "lea (%[d0],%[d],2),%%rax \n" - "vmovdqu %%ymm10,(%%rax) \n" - "vmovdqu %%ymm11,(%%rax,%[d]) \n" - "lea (%%rax,%[d],2),%%rax \n" - "vmovdqu %%ymm12,(%%rax) \n" - "vmovdqu %%ymm13,(%%rax,%[d]) \n" - "lea (%%rax,%[d],2),%%rax \n" - "vmovdqu %%ymm14,(%%rax) \n" - "vmovdqu %%ymm15,(%%rax,%[d]) \n" - : - : [s0] "r"(src_row), // %[s0] - [d0] "r"(dst_col), // %[d0] - [s] "r"(s), // %[s] - [d] "r"(d) // %[d] - : "memory", "cc", "rax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", - "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", - "xmm13", "xmm14", "xmm15"); - src_row += 32; - dst_col += 8 * d; - w -= 8; + // Store 4 full 256-bit (32 B) rows = 8 32-bit pixels each + "vmovdqu %%ymm0,(%[dst]) \n" + "vmovdqu %%ymm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%ymm2,(%[dst]) \n" + "vmovdqu %%ymm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x4,%[cnt] \n" + "jg 1b \n" + : [src] "+r"(s_ptr), // %[src] + [dst] "+r"(d_ptr), // %[dst] + [cnt] "+r"(cnt), // %[cnt] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); + src_row += (ptrdiff_t)n * 4; + dst_col += (ptrdiff_t)n * d; + w -= n; + } } - if (w >= 4) { - TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, 8); + const uint64_t row_mask = (bh == 8) ? ~0ULL : ((1ULL << (bh * 8)) - 1); + while (w > 0) { + const int bw = (w >= 4) ? 4 : w; + const uint32_t col_mask = (bw == 4) ? ~0u : ((1u << (bw * 8)) - 1); + const uint8_t* s_ptr = src_row; + uint8_t* d_ptr = dst_col; + asm volatile( + "vmovd %[col_mask],%%xmm8 \n" + "vpmovsxbd %%xmm8,%%xmm8 \n" + "vmovq %[row_mask],%%xmm9 \n" + "vpmovsxbd %%xmm9,%%ymm9 \n" + "vpxor %%ymm0,%%ymm0,%%ymm0 \n" + "vmovdqa %%ymm0,%%ymm1 \n" + "vmovdqa %%ymm0,%%ymm2 \n" + "vmovdqa %%ymm0,%%ymm3 \n" + + "vpmaskmovd (%[s_ptr]),%%xmm8,%%xmm0 \n" + "cmp $1,%[bh] \n" + "jle 2f \n" + "vpmaskmovd (%[s_ptr],%[s]),%%xmm8,%%xmm1 \n" + "cmp $2,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vpmaskmovd (%[s_ptr]),%%xmm8,%%xmm2 \n" + "cmp $3,%[bh] \n" + "jle 2f \n" + "vpmaskmovd (%[s_ptr],%[s]),%%xmm8,%%xmm3 \n" + "cmp $4,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vpmaskmovd (%[s_ptr]),%%xmm8,%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n" + "cmp $5,%[bh] \n" + "jle 2f \n" + "vpmaskmovd (%[s_ptr],%[s]),%%xmm8,%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm1,%%ymm1 \n" + "cmp $6,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vpmaskmovd (%[s_ptr]),%%xmm8,%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n" + "cmp $7,%[bh] \n" + "jle 2f \n" + "vpmaskmovd (%[s_ptr],%[s]),%%xmm8,%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm3,%%ymm3 \n" + "2: \n" + + "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" + + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" + + "vpmaskmovd %%ymm0,%%ymm9,(%[d_ptr]) \n" + "cmp $1,%[bw] \n" + "jle 3f \n" + "vpmaskmovd %%ymm1,%%ymm9,(%[d_ptr],%[d]) \n" + "cmp $2,%[bw] \n" + "jle 3f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vpmaskmovd %%ymm2,%%ymm9,(%[d_ptr]) \n" + "cmp $3,%[bw] \n" + "jle 3f \n" + "vpmaskmovd %%ymm3,%%ymm9,(%[d_ptr],%[d]) \n" + "3: \n" + : [s_ptr] "+r"(s_ptr), // %[s_ptr] + [d_ptr] "+r"(d_ptr) // %[d_ptr] + : [s] "r"(s), // %[s] + [d] "r"(d), // %[d] + [col_mask] "r"(col_mask), // %[col_mask] + [row_mask] "r"(row_mask), // %[row_mask] + [bh] "r"(bh), // %[bh] + [bw] "r"(bw) // %[bw] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7", "xmm8", "xmm9"); src_row += 16; dst_col += 4 * d; w -= 4; } - if (w > 0) { - TransposeWxH_32_C(src_row, src_stride, dst_col, dst_stride, w, 8); - } src += 8 * s; dst += 32; height -= 8; } asm volatile("vzeroupper \n"); - if (height > 0) { - while (width >= 4) { - TransposeNx4_32_AVX2(src, src_stride, dst, dst_stride, height); - src += 16; - dst += 4 * d; - width -= 4; - } - if (width > 0) { - const uint8_t* src_rem = src; - uint8_t* dst0 = dst; - uint8_t* dst1 = (width > 1) ? dst0 + d : dst0; - uint8_t* dst2 = (width > 2) ? dst1 + d : dst1; - int col_mask_bits = (1 << (width * 8)) - 1; - int h_rem = height; - asm volatile( - "vmovd %[col_mask],%%xmm7 \n" - "vpmovsxbd %%xmm7,%%xmm7 \n" - "sub $0x4,%[h] \n" - "jl 2f \n" - "1: \n" - "vpmaskmovd (%[src]),%%xmm7,%%xmm0 \n" - "vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm1 \n" - "lea (%[src],%[s],2),%[src] \n" - "vpmaskmovd (%[src]),%%xmm7,%%xmm2 \n" - "vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm3 \n" - "lea (%[src],%[s],2),%[src] \n" - "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" - "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" - "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" - "vpunpckhdq %%xmm3,%%xmm2,%%xmm3 \n" - "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" - "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" - "vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n" - "vmovdqu %%xmm2,(%[dst2]) \n" - "vmovdqu %%xmm1,(%[dst1]) \n" - "vmovdqu %%xmm0,(%[dst0]) \n" - "add $0x10,%[dst2] \n" - "add $0x10,%[dst1] \n" - "add $0x10,%[dst0] \n" - "sub $0x4,%[h] \n" - "jge 1b \n" - "2: \n" - "add $0x4,%[h] \n" - "jz 4f \n" - "vpmaskmovd (%[src]),%%xmm7,%%xmm0 \n" - "vmovdqa %%xmm0,%%xmm1 \n" - "vmovdqa %%xmm0,%%xmm2 \n" - "mov $0x000000ff,%%eax \n" - "cmp $2,%[h] \n" - "jl 3f \n" - "vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm1 \n" - "mov $0x0000ffff,%%eax \n" - "je 3f \n" - "vpmaskmovd (%[src],%[s],2),%%xmm7,%%xmm2 \n" - "mov $0x00ffffff,%%eax \n" - "3: \n" - "vmovd %%eax,%%xmm7 \n" - "vpmovsxbd %%xmm7,%%xmm7 \n" - "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" - "vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n" - "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" - "vpunpckhdq %%xmm2,%%xmm2,%%xmm3 \n" - "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" - "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" - "vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n" - "vpmaskmovd %%xmm2,%%xmm7,(%[dst2]) \n" - "vpmaskmovd %%xmm1,%%xmm7,(%[dst1]) \n" - "vpmaskmovd %%xmm0,%%xmm7,(%[dst0]) \n" - "4: \n" - "vzeroupper \n" - : [src] "+r"(src_rem), // %[src] - [dst0] "+r"(dst0), // %[dst0] - [dst1] "+r"(dst1), // %[dst1] - [dst2] "+r"(dst2), // %[dst2] - [h] "+r"(h_rem) // %[h] - : [s] "r"(s), // %[s] - [col_mask] "r"(col_mask_bits) // %[col_mask] - : "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", - "xmm5", "xmm6", "xmm7"); - } - } } #endif // defined(HAS_TRANSPOSEWXH_32_AVX2) -#if defined(HAS_TRANSPOSENX4_32_AVX512BW) -// Transpose 32 bit values (ARGB) in Nx4 tiles with 4 blocks of 4x4 at a time -// in zmm0..zmm3 and masked tail for arbitrary N. -void TransposeNx4_32_AVX512BW(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - const ptrdiff_t s = src_stride; - const ptrdiff_t d = dst_stride; - uint8_t* dst1 = dst + d; - uint8_t* dst2 = dst1 + d; - uint8_t* dst3 = dst2 + d; - asm volatile( - "sub $0x10,%[w] \n" - "jl 2f \n" - - // Main loop: 4 blocks of 4x4 (16 source rows -> 16 dest columns / 64 B). - "1: \n" - "vmovdqu (%[src]),%%xmm0 \n" - "vmovdqu (%[src],%[s]),%%xmm1 \n" - "lea (%[src],%[s],2),%[src] \n" - "vmovdqu (%[src]),%%xmm2 \n" - "vmovdqu (%[src],%[s]),%%xmm3 \n" - "lea (%[src],%[s],2),%[src] \n" - - "vinserti32x4 $1,(%[src]),%%zmm0,%%zmm0 \n" - "vinserti32x4 $1,(%[src],%[s]),%%zmm1,%%zmm1 \n" - "lea (%[src],%[s],2),%[src] \n" - "vinserti32x4 $1,(%[src]),%%zmm2,%%zmm2 \n" - "vinserti32x4 $1,(%[src],%[s]),%%zmm3,%%zmm3 \n" - "lea (%[src],%[s],2),%[src] \n" - - "vinserti32x4 $2,(%[src]),%%zmm0,%%zmm0 \n" - "vinserti32x4 $2,(%[src],%[s]),%%zmm1,%%zmm1 \n" - "lea (%[src],%[s],2),%[src] \n" - "vinserti32x4 $2,(%[src]),%%zmm2,%%zmm2 \n" - "vinserti32x4 $2,(%[src],%[s]),%%zmm3,%%zmm3 \n" - "lea (%[src],%[s],2),%[src] \n" - - "vinserti32x4 $3,(%[src]),%%zmm0,%%zmm0 \n" - "vinserti32x4 $3,(%[src],%[s]),%%zmm1,%%zmm1 \n" - "lea (%[src],%[s],2),%[src] \n" - "vinserti32x4 $3,(%[src]),%%zmm2,%%zmm2 \n" - "vinserti32x4 $3,(%[src],%[s]),%%zmm3,%%zmm3 \n" - "lea (%[src],%[s],2),%[src] \n" - - // Transpose 2x2 across all 4 128-bit lanes - "vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n" - "vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n" - "vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n" - "vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n" - - // Transpose 4x4 across all 4 128-bit lanes - "vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n" - "vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n" - "vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n" - "vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n" - - "vmovdqu32 %%zmm0,(%[dst0]) \n" - "vmovdqu32 %%zmm1,(%[dst1]) \n" - "vmovdqu32 %%zmm2,(%[dst2]) \n" - "vmovdqu32 %%zmm3,(%[dst3]) \n" - "add $0x40,%[dst0] \n" - "add $0x40,%[dst1] \n" - "add $0x40,%[dst2] \n" - "add $0x40,%[dst3] \n" - "sub $0x10,%[w] \n" - "jge 1b \n" - - "2: \n" - "add $0x10,%[w] \n" - "jz 5f \n" - - // 4x4 loop for 4..15 remainder elements - "sub $0x4,%[w] \n" - "jl 4f \n" - "3: \n" - "vmovdqu (%[src]),%%xmm0 \n" - "vmovdqu (%[src],%[s]),%%xmm1 \n" - "lea (%[src],%[s],2),%[src] \n" - "vmovdqu (%[src]),%%xmm2 \n" - "vmovdqu (%[src],%[s]),%%xmm3 \n" - "lea (%[src],%[s],2),%[src] \n" - "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" - "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" - "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" - "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" - "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" - "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" - "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" - "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" - "vmovdqu %%xmm0,(%[dst0]) \n" - "vmovdqu %%xmm1,(%[dst1]) \n" - "vmovdqu %%xmm2,(%[dst2]) \n" - "vmovdqu %%xmm3,(%[dst3]) \n" - "add $0x10,%[dst0] \n" - "add $0x10,%[dst1] \n" - "add $0x10,%[dst2] \n" - "add $0x10,%[dst3] \n" - "sub $0x4,%[w] \n" - "jge 3b \n" - - "4: \n" - "add $0x4,%[w] \n" - "jz 5f \n" - - // Masked tail for 1..3 remainder elements (in-bounds row loads + k1 mask) - "mov $1,%%eax \n" - "shlx %[w],%%eax,%%eax \n" - "dec %%eax \n" - "kmovw %%eax,%%k1 \n" - "vmovdqu (%[src]),%%xmm0 \n" - "vmovdqa %%xmm0,%%xmm1 \n" - "vmovdqa %%xmm0,%%xmm2 \n" - "cmp $1,%[w] \n" - "jle 6f \n" - "vmovdqu (%[src],%[s]),%%xmm1 \n" - "cmp $2,%[w] \n" - "jle 6f \n" - "vmovdqu (%[src],%[s],2),%%xmm2 \n" - "6: \n" - "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" - "vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n" - "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" - "vpunpckhdq %%xmm2,%%xmm2,%%xmm7 \n" - "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" - "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" - "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" - "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" - "vmovdqu32 %%xmm0,(%[dst0])%{%%k1%} \n" - "vmovdqu32 %%xmm1,(%[dst1])%{%%k1%} \n" - "vmovdqu32 %%xmm2,(%[dst2])%{%%k1%} \n" - "vmovdqu32 %%xmm3,(%[dst3])%{%%k1%} \n" - "5: \n" - "vzeroupper \n" - : [src] "+r"(src), // %[src] - [dst0] "+r"(dst), // %[dst0] - [dst1] "+r"(dst1), // %[dst1] - [dst2] "+r"(dst2), // %[dst2] - [dst3] "+r"(dst3), // %[dst3] - [w] "+r"(width) // %[w] - : [s] "r"(s) // %[s] - : "memory", "cc", "eax", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", - "xmm5", "xmm6", "xmm7"); -} -#endif // defined(HAS_TRANSPOSENX4_32_AVX512BW) - #if defined(HAS_TRANSPOSEWXH_32_AVX512BW) -// Transpose 32 bit plane of any width x height using 16x16 ZMM horizontal tiles -// in zmm0..zmm31 with k-masked right and bottom tails. +// Transpose 32 bit plane of any width x height using 4x16 ZMM vertical tiles +// in zmm0..zmm7 with k-masked right and bottom tails. void TransposeWxH_32_AVX512BW(const uint8_t* src, int src_stride, uint8_t* dst, @@ -1208,350 +1386,179 @@ void TransposeWxH_32_AVX512BW(const uint8_t* src, uint8_t* dst_col = dst; int w = width; if (bh == 16) { - while (w >= 16) { + const int n = w & ~3; + if (n > 0) { const uint8_t* s_ptr = src_row; uint8_t* d_ptr = dst_col; + int cnt = n; + const uint8_t* temp; asm volatile( - "vmovdqu32 (%[s_ptr]),%%zmm0 \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm1 \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm2 \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm3 \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm4 \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm5 \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm6 \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm7 \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm8 \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm9 \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm10 \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm11 \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm12 \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm13 \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm14 \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm15 \n" + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" - // Stage 1: 32-bit unpack - "vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n" - "vpunpckhdq %%zmm1,%%zmm0,%%zmm17 \n" - "vpunpckldq %%zmm3,%%zmm2,%%zmm18 \n" - "vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n" - "vpunpckldq %%zmm5,%%zmm4,%%zmm20 \n" - "vpunpckhdq %%zmm5,%%zmm4,%%zmm21 \n" - "vpunpckldq %%zmm7,%%zmm6,%%zmm22 \n" - "vpunpckhdq %%zmm7,%%zmm6,%%zmm23 \n" - "vpunpckldq %%zmm9,%%zmm8,%%zmm24 \n" - "vpunpckhdq %%zmm9,%%zmm8,%%zmm25 \n" - "vpunpckldq %%zmm11,%%zmm10,%%zmm26 \n" - "vpunpckhdq %%zmm11,%%zmm10,%%zmm27 \n" - "vpunpckldq %%zmm13,%%zmm12,%%zmm28 \n" - "vpunpckhdq %%zmm13,%%zmm12,%%zmm29 \n" - "vpunpckldq %%zmm15,%%zmm14,%%zmm30 \n" - "vpunpckhdq %%zmm15,%%zmm14,%%zmm31 \n" + "vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" - // Stage 2: 64-bit unpack - "vpunpcklqdq %%zmm18,%%zmm16,%%zmm0 \n" - "vpunpckhqdq %%zmm18,%%zmm16,%%zmm1 \n" - "vpunpcklqdq %%zmm19,%%zmm17,%%zmm2 \n" - "vpunpckhqdq %%zmm19,%%zmm17,%%zmm3 \n" - "vpunpcklqdq %%zmm22,%%zmm20,%%zmm4 \n" - "vpunpckhqdq %%zmm22,%%zmm20,%%zmm5 \n" - "vpunpcklqdq %%zmm23,%%zmm21,%%zmm6 \n" - "vpunpckhqdq %%zmm23,%%zmm21,%%zmm7 \n" - "vpunpcklqdq %%zmm26,%%zmm24,%%zmm8 \n" - "vpunpckhqdq %%zmm26,%%zmm24,%%zmm9 \n" - "vpunpcklqdq %%zmm27,%%zmm25,%%zmm10 \n" - "vpunpckhqdq %%zmm27,%%zmm25,%%zmm11 \n" - "vpunpcklqdq %%zmm30,%%zmm28,%%zmm12 \n" - "vpunpckhqdq %%zmm30,%%zmm28,%%zmm13 \n" - "vpunpcklqdq %%zmm31,%%zmm29,%%zmm14 \n" - "vpunpckhqdq %%zmm31,%%zmm29,%%zmm15 \n" + "vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" - // Stage 3: 128-bit lane shuffle - "vshufi32x4 $0x88,%%zmm4,%%zmm0,%%zmm16 \n" - "vshufi32x4 $0x88,%%zmm5,%%zmm1,%%zmm17 \n" - "vshufi32x4 $0x88,%%zmm6,%%zmm2,%%zmm18 \n" - "vshufi32x4 $0x88,%%zmm7,%%zmm3,%%zmm19 \n" - "vshufi32x4 $0xdd,%%zmm4,%%zmm0,%%zmm20 \n" - "vshufi32x4 $0xdd,%%zmm5,%%zmm1,%%zmm21 \n" - "vshufi32x4 $0xdd,%%zmm6,%%zmm2,%%zmm22 \n" - "vshufi32x4 $0xdd,%%zmm7,%%zmm3,%%zmm23 \n" - "vshufi32x4 $0x88,%%zmm12,%%zmm8,%%zmm24 \n" - "vshufi32x4 $0x88,%%zmm13,%%zmm9,%%zmm25 \n" - "vshufi32x4 $0x88,%%zmm14,%%zmm10,%%zmm26 \n" - "vshufi32x4 $0x88,%%zmm15,%%zmm11,%%zmm27 \n" - "vshufi32x4 $0xdd,%%zmm12,%%zmm8,%%zmm28 \n" - "vshufi32x4 $0xdd,%%zmm13,%%zmm9,%%zmm29 \n" - "vshufi32x4 $0xdd,%%zmm14,%%zmm10,%%zmm30 \n" - "vshufi32x4 $0xdd,%%zmm15,%%zmm11,%%zmm31 \n" + "vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm3,%%zmm3 \n" + "add $0x10,%[src] \n" - // Stage 4: 256-bit half shuffle -> columns 0..15 in zmm0..zmm15 - "vshufi32x4 $0x88,%%zmm24,%%zmm16,%%zmm0 \n" - "vshufi32x4 $0x88,%%zmm25,%%zmm17,%%zmm1 \n" - "vshufi32x4 $0x88,%%zmm26,%%zmm18,%%zmm2 \n" - "vshufi32x4 $0x88,%%zmm27,%%zmm19,%%zmm3 \n" - "vshufi32x4 $0x88,%%zmm28,%%zmm20,%%zmm4 \n" - "vshufi32x4 $0x88,%%zmm29,%%zmm21,%%zmm5 \n" - "vshufi32x4 $0x88,%%zmm30,%%zmm22,%%zmm6 \n" - "vshufi32x4 $0x88,%%zmm31,%%zmm23,%%zmm7 \n" - "vshufi32x4 $0xdd,%%zmm24,%%zmm16,%%zmm8 \n" - "vshufi32x4 $0xdd,%%zmm25,%%zmm17,%%zmm9 \n" - "vshufi32x4 $0xdd,%%zmm26,%%zmm18,%%zmm10 \n" - "vshufi32x4 $0xdd,%%zmm27,%%zmm19,%%zmm11 \n" - "vshufi32x4 $0xdd,%%zmm28,%%zmm20,%%zmm12 \n" - "vshufi32x4 $0xdd,%%zmm29,%%zmm21,%%zmm13 \n" - "vshufi32x4 $0xdd,%%zmm30,%%zmm22,%%zmm14 \n" - "vshufi32x4 $0xdd,%%zmm31,%%zmm23,%%zmm15 \n" + // Stage 1: 32-bit unpack across all 4 128-bit lanes + "vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n" + "vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n" + "vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n" + "vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n" - "vmovdqu32 %%zmm0,(%[d_ptr]) \n" - "vmovdqu32 %%zmm1,(%[d_ptr],%[d]) \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm2,(%[d_ptr]) \n" - "vmovdqu32 %%zmm3,(%[d_ptr],%[d]) \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm4,(%[d_ptr]) \n" - "vmovdqu32 %%zmm5,(%[d_ptr],%[d]) \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm6,(%[d_ptr]) \n" - "vmovdqu32 %%zmm7,(%[d_ptr],%[d]) \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm8,(%[d_ptr]) \n" - "vmovdqu32 %%zmm9,(%[d_ptr],%[d]) \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm10,(%[d_ptr]) \n" - "vmovdqu32 %%zmm11,(%[d_ptr],%[d]) \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm12,(%[d_ptr]) \n" - "vmovdqu32 %%zmm13,(%[d_ptr],%[d]) \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm14,(%[d_ptr]) \n" - "vmovdqu32 %%zmm15,(%[d_ptr],%[d]) \n" - : [s_ptr] "+r"(s_ptr), // %[s_ptr] - [d_ptr] "+r"(d_ptr) // %[d_ptr] - : [s] "r"(s), // %[s] - [d] "r"(d) // %[d] + // Stage 2: 64-bit unpack across all 4 128-bit lanes + "vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n" + "vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n" + "vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n" + "vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n" + + // Store 4 full 512-bit (64 B) rows = 16 32-bit pixels each + "vmovdqu32 %%zmm0,(%[dst]) \n" + "vmovdqu32 %%zmm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu32 %%zmm2,(%[dst]) \n" + "vmovdqu32 %%zmm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x4,%[cnt] \n" + "jg 1b \n" + : [src] "+r"(s_ptr), // %[src] + [dst] "+r"(d_ptr), // %[dst] + [cnt] "+r"(cnt), // %[cnt] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", - "xmm13", "xmm14", "xmm15", "xmm16", "xmm17", "xmm18", "xmm19", - "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26", - "xmm27", "xmm28", "xmm29", "xmm30", "xmm31"); - src_row += 64; - dst_col += 16 * d; - w -= 16; + "xmm6", "xmm7"); + src_row += (ptrdiff_t)n * 4; + dst_col += (ptrdiff_t)n * d; + w -= n; } } const uint32_t row_mask = (1u << bh) - 1; while (w > 0) { - const int bw = (w >= 16) ? 16 : w; + const int bw = (w >= 4) ? 4 : w; const uint32_t col_mask = (1u << bw) - 1; const uint8_t* s_ptr = src_row; uint8_t* d_ptr = dst_col; asm volatile( - "kmovw %[col_mask],%%k1 \n" - "kmovw %[row_mask],%%k2 \n" - "vpxord %%zmm0,%%zmm0,%%zmm0 \n" - "vmovdqa64 %%zmm0,%%zmm1 \n" - "vmovdqa64 %%zmm0,%%zmm2 \n" - "vmovdqa64 %%zmm0,%%zmm3 \n" - "vmovdqa64 %%zmm0,%%zmm4 \n" - "vmovdqa64 %%zmm0,%%zmm5 \n" - "vmovdqa64 %%zmm0,%%zmm6 \n" - "vmovdqa64 %%zmm0,%%zmm7 \n" - "vmovdqa64 %%zmm0,%%zmm8 \n" - "vmovdqa64 %%zmm0,%%zmm9 \n" - "vmovdqa64 %%zmm0,%%zmm10 \n" - "vmovdqa64 %%zmm0,%%zmm11 \n" - "vmovdqa64 %%zmm0,%%zmm12 \n" - "vmovdqa64 %%zmm0,%%zmm13 \n" - "vmovdqa64 %%zmm0,%%zmm14 \n" - "vmovdqa64 %%zmm0,%%zmm15 \n" + "kmovw %[col_mask],%%k1 \n" + "kmovw %[row_mask],%%k2 \n" + "vpxord %%zmm0,%%zmm0,%%zmm0 \n" + "vmovdqa64 %%zmm0,%%zmm1 \n" + "vmovdqa64 %%zmm0,%%zmm2 \n" + "vmovdqa64 %%zmm0,%%zmm3 \n" - "vmovdqu32 (%[s_ptr]),%%zmm0%{%%k1%}%{z%} \n" - "cmp $1,%[bh] \n" - "jle 1f \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm1%{%%k1%}%{z%} \n" - "cmp $2,%[bh] \n" - "jle 1f \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm2%{%%k1%}%{z%} \n" - "cmp $3,%[bh] \n" - "jle 1f \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm3%{%%k1%}%{z%} \n" - "cmp $4,%[bh] \n" - "jle 1f \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm4%{%%k1%}%{z%} \n" - "cmp $5,%[bh] \n" - "jle 1f \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm5%{%%k1%}%{z%} \n" - "cmp $6,%[bh] \n" - "jle 1f \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm6%{%%k1%}%{z%} \n" - "cmp $7,%[bh] \n" - "jle 1f \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm7%{%%k1%}%{z%} \n" - "cmp $8,%[bh] \n" - "jle 1f \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm8%{%%k1%}%{z%} \n" - "cmp $9,%[bh] \n" - "jle 1f \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm9%{%%k1%}%{z%} \n" - "cmp $10,%[bh] \n" - "jle 1f \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm10%{%%k1%}%{z%} \n" - "cmp $11,%[bh] \n" - "jle 1f \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm11%{%%k1%}%{z%} \n" - "cmp $12,%[bh] \n" - "jle 1f \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm12%{%%k1%}%{z%} \n" - "cmp $13,%[bh] \n" - "jle 1f \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm13%{%%k1%}%{z%} \n" - "cmp $14,%[bh] \n" - "jle 1f \n" - "lea (%[s_ptr],%[s],2),%[s_ptr] \n" - "vmovdqu32 (%[s_ptr]),%%zmm14%{%%k1%}%{z%} \n" - "cmp $15,%[bh] \n" - "jle 1f \n" - "vmovdqu32 (%[s_ptr],%[s]),%%zmm15%{%%k1%}%{z%} \n" - "1: \n" + "vmovdqu32 (%[s_ptr]),%%xmm0%{%%k1%}%{z%} \n" + "cmp $1,%[bh] \n" + "jle 2f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%xmm1%{%%k1%}%{z%} \n" + "cmp $2,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%xmm2%{%%k1%}%{z%} \n" + "cmp $3,%[bh] \n" + "jle 2f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%xmm3%{%%k1%}%{z%} \n" + "cmp $4,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm8,%%zmm0,%%zmm0 \n" + "cmp $5,%[bh] \n" + "jle 2f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm8,%%zmm1,%%zmm1 \n" + "cmp $6,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm8,%%zmm2,%%zmm2 \n" + "cmp $7,%[bh] \n" + "jle 2f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $1,%%xmm8,%%zmm3,%%zmm3 \n" + "cmp $8,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm8,%%zmm0,%%zmm0 \n" + "cmp $9,%[bh] \n" + "jle 2f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm8,%%zmm1,%%zmm1 \n" + "cmp $10,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm8,%%zmm2,%%zmm2 \n" + "cmp $11,%[bh] \n" + "jle 2f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $2,%%xmm8,%%zmm3,%%zmm3 \n" + "cmp $12,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm8,%%zmm0,%%zmm0 \n" + "cmp $13,%[bh] \n" + "jle 2f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm8,%%zmm1,%%zmm1 \n" + "cmp $14,%[bh] \n" + "jle 2f \n" + "lea (%[s_ptr],%[s],2),%[s_ptr] \n" + "vmovdqu32 (%[s_ptr]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm8,%%zmm2,%%zmm2 \n" + "cmp $15,%[bh] \n" + "jle 2f \n" + "vmovdqu32 (%[s_ptr],%[s]),%%xmm8%{%%k1%}%{z%} \n" + "vinserti32x4 $3,%%xmm8,%%zmm3,%%zmm3 \n" + "2: \n" - // Stage 1: 32-bit unpack - "vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n" - "vpunpckhdq %%zmm1,%%zmm0,%%zmm17 \n" - "vpunpckldq %%zmm3,%%zmm2,%%zmm18 \n" - "vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n" - "vpunpckldq %%zmm5,%%zmm4,%%zmm20 \n" - "vpunpckhdq %%zmm5,%%zmm4,%%zmm21 \n" - "vpunpckldq %%zmm7,%%zmm6,%%zmm22 \n" - "vpunpckhdq %%zmm7,%%zmm6,%%zmm23 \n" - "vpunpckldq %%zmm9,%%zmm8,%%zmm24 \n" - "vpunpckhdq %%zmm9,%%zmm8,%%zmm25 \n" - "vpunpckldq %%zmm11,%%zmm10,%%zmm26 \n" - "vpunpckhdq %%zmm11,%%zmm10,%%zmm27 \n" - "vpunpckldq %%zmm13,%%zmm12,%%zmm28 \n" - "vpunpckhdq %%zmm13,%%zmm12,%%zmm29 \n" - "vpunpckldq %%zmm15,%%zmm14,%%zmm30 \n" - "vpunpckhdq %%zmm15,%%zmm14,%%zmm31 \n" + "vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n" + "vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n" + "vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n" + "vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n" - // Stage 2: 64-bit unpack - "vpunpcklqdq %%zmm18,%%zmm16,%%zmm0 \n" - "vpunpckhqdq %%zmm18,%%zmm16,%%zmm1 \n" - "vpunpcklqdq %%zmm19,%%zmm17,%%zmm2 \n" - "vpunpckhqdq %%zmm19,%%zmm17,%%zmm3 \n" - "vpunpcklqdq %%zmm22,%%zmm20,%%zmm4 \n" - "vpunpckhqdq %%zmm22,%%zmm20,%%zmm5 \n" - "vpunpcklqdq %%zmm23,%%zmm21,%%zmm6 \n" - "vpunpckhqdq %%zmm23,%%zmm21,%%zmm7 \n" - "vpunpcklqdq %%zmm26,%%zmm24,%%zmm8 \n" - "vpunpckhqdq %%zmm26,%%zmm24,%%zmm9 \n" - "vpunpcklqdq %%zmm27,%%zmm25,%%zmm10 \n" - "vpunpckhqdq %%zmm27,%%zmm25,%%zmm11 \n" - "vpunpcklqdq %%zmm30,%%zmm28,%%zmm12 \n" - "vpunpckhqdq %%zmm30,%%zmm28,%%zmm13 \n" - "vpunpcklqdq %%zmm31,%%zmm29,%%zmm14 \n" - "vpunpckhqdq %%zmm31,%%zmm29,%%zmm15 \n" + "vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n" + "vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n" + "vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n" + "vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n" - // Stage 3: 128-bit lane shuffle - "vshufi32x4 $0x88,%%zmm4,%%zmm0,%%zmm16 \n" - "vshufi32x4 $0x88,%%zmm5,%%zmm1,%%zmm17 \n" - "vshufi32x4 $0x88,%%zmm6,%%zmm2,%%zmm18 \n" - "vshufi32x4 $0x88,%%zmm7,%%zmm3,%%zmm19 \n" - "vshufi32x4 $0xdd,%%zmm4,%%zmm0,%%zmm20 \n" - "vshufi32x4 $0xdd,%%zmm5,%%zmm1,%%zmm21 \n" - "vshufi32x4 $0xdd,%%zmm6,%%zmm2,%%zmm22 \n" - "vshufi32x4 $0xdd,%%zmm7,%%zmm3,%%zmm23 \n" - "vshufi32x4 $0x88,%%zmm12,%%zmm8,%%zmm24 \n" - "vshufi32x4 $0x88,%%zmm13,%%zmm9,%%zmm25 \n" - "vshufi32x4 $0x88,%%zmm14,%%zmm10,%%zmm26 \n" - "vshufi32x4 $0x88,%%zmm15,%%zmm11,%%zmm27 \n" - "vshufi32x4 $0xdd,%%zmm12,%%zmm8,%%zmm28 \n" - "vshufi32x4 $0xdd,%%zmm13,%%zmm9,%%zmm29 \n" - "vshufi32x4 $0xdd,%%zmm14,%%zmm10,%%zmm30 \n" - "vshufi32x4 $0xdd,%%zmm15,%%zmm11,%%zmm31 \n" - - // Stage 4: 256-bit half shuffle -> columns 0..15 in zmm0..zmm15 - "vshufi32x4 $0x88,%%zmm24,%%zmm16,%%zmm0 \n" - "vshufi32x4 $0x88,%%zmm25,%%zmm17,%%zmm1 \n" - "vshufi32x4 $0x88,%%zmm26,%%zmm18,%%zmm2 \n" - "vshufi32x4 $0x88,%%zmm27,%%zmm19,%%zmm3 \n" - "vshufi32x4 $0x88,%%zmm28,%%zmm20,%%zmm4 \n" - "vshufi32x4 $0x88,%%zmm29,%%zmm21,%%zmm5 \n" - "vshufi32x4 $0x88,%%zmm30,%%zmm22,%%zmm6 \n" - "vshufi32x4 $0x88,%%zmm31,%%zmm23,%%zmm7 \n" - "vshufi32x4 $0xdd,%%zmm24,%%zmm16,%%zmm8 \n" - "vshufi32x4 $0xdd,%%zmm25,%%zmm17,%%zmm9 \n" - "vshufi32x4 $0xdd,%%zmm26,%%zmm18,%%zmm10 \n" - "vshufi32x4 $0xdd,%%zmm27,%%zmm19,%%zmm11 \n" - "vshufi32x4 $0xdd,%%zmm28,%%zmm20,%%zmm12 \n" - "vshufi32x4 $0xdd,%%zmm29,%%zmm21,%%zmm13 \n" - "vshufi32x4 $0xdd,%%zmm30,%%zmm22,%%zmm14 \n" - "vshufi32x4 $0xdd,%%zmm31,%%zmm23,%%zmm15 \n" - - "vmovdqu32 %%zmm0,(%[d_ptr])%{%%k2%} \n" - "cmp $1,%[bw] \n" - "jle 2f \n" - "vmovdqu32 %%zmm1,(%[d_ptr],%[d])%{%%k2%} \n" - "cmp $2,%[bw] \n" - "jle 2f \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm2,(%[d_ptr])%{%%k2%} \n" - "cmp $3,%[bw] \n" - "jle 2f \n" - "vmovdqu32 %%zmm3,(%[d_ptr],%[d])%{%%k2%} \n" - "cmp $4,%[bw] \n" - "jle 2f \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm4,(%[d_ptr])%{%%k2%} \n" - "cmp $5,%[bw] \n" - "jle 2f \n" - "vmovdqu32 %%zmm5,(%[d_ptr],%[d])%{%%k2%} \n" - "cmp $6,%[bw] \n" - "jle 2f \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm6,(%[d_ptr])%{%%k2%} \n" - "cmp $7,%[bw] \n" - "jle 2f \n" - "vmovdqu32 %%zmm7,(%[d_ptr],%[d])%{%%k2%} \n" - "cmp $8,%[bw] \n" - "jle 2f \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm8,(%[d_ptr])%{%%k2%} \n" - "cmp $9,%[bw] \n" - "jle 2f \n" - "vmovdqu32 %%zmm9,(%[d_ptr],%[d])%{%%k2%} \n" - "cmp $10,%[bw] \n" - "jle 2f \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm10,(%[d_ptr])%{%%k2%} \n" - "cmp $11,%[bw] \n" - "jle 2f \n" - "vmovdqu32 %%zmm11,(%[d_ptr],%[d])%{%%k2%} \n" - "cmp $12,%[bw] \n" - "jle 2f \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm12,(%[d_ptr])%{%%k2%} \n" - "cmp $13,%[bw] \n" - "jle 2f \n" - "vmovdqu32 %%zmm13,(%[d_ptr],%[d])%{%%k2%} \n" - "cmp $14,%[bw] \n" - "jle 2f \n" - "lea (%[d_ptr],%[d],2),%[d_ptr] \n" - "vmovdqu32 %%zmm14,(%[d_ptr])%{%%k2%} \n" - "cmp $15,%[bw] \n" - "jle 2f \n" - "vmovdqu32 %%zmm15,(%[d_ptr],%[d])%{%%k2%} \n" - "2: \n" + "vmovdqu32 %%zmm0,(%[d_ptr])%{%%k2%} \n" + "cmp $1,%[bw] \n" + "jle 3f \n" + "vmovdqu32 %%zmm1,(%[d_ptr],%[d])%{%%k2%} \n" + "cmp $2,%[bw] \n" + "jle 3f \n" + "lea (%[d_ptr],%[d],2),%[d_ptr] \n" + "vmovdqu32 %%zmm2,(%[d_ptr])%{%%k2%} \n" + "cmp $3,%[bw] \n" + "jle 3f \n" + "vmovdqu32 %%zmm3,(%[d_ptr],%[d])%{%%k2%} \n" + "3: \n" : [s_ptr] "+r"(s_ptr), // %[s_ptr] [d_ptr] "+r"(d_ptr) // %[d_ptr] : [s] "r"(s), // %[s] @@ -1561,13 +1568,10 @@ void TransposeWxH_32_AVX512BW(const uint8_t* src, [bh] "r"(bh), // %[bh] [bw] "r"(bw) // %[bw] : "memory", "cc", "k1", "k2", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", - "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", - "xmm13", "xmm14", "xmm15", "xmm16", "xmm17", "xmm18", "xmm19", - "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26", - "xmm27", "xmm28", "xmm29", "xmm30", "xmm31"); - src_row += 64; - dst_col += 16 * d; - w -= 16; + "xmm5", "xmm6", "xmm7", "xmm8"); + src_row += 16; + dst_col += 4 * d; + w -= 4; } src += 16 * s; dst += 64; diff --git a/source/rotate_lasx.cc b/source/rotate_lasx.cc index b4361809e..74e54c008 100644 --- a/source/rotate_lasx.cc +++ b/source/rotate_lasx.cc @@ -95,6 +95,121 @@ void Transpose4x4_32_LASX(const uint8_t* src, "$xr7"); } +void TransposeWx16_16_LASX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + + "vld $vr8, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr9, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr10, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr11, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr12, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr13, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr14, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr15, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" + + "xvpermi.q $xr0, $xr8, 0x02 \n" + "xvpermi.q $xr1, $xr9, 0x02 \n" + "xvpermi.q $xr2, $xr10, 0x02 \n" + "xvpermi.q $xr3, $xr11, 0x02 \n" + "xvpermi.q $xr4, $xr12, 0x02 \n" + "xvpermi.q $xr5, $xr13, 0x02 \n" + "xvpermi.q $xr6, $xr14, 0x02 \n" + "xvpermi.q $xr7, $xr15, 0x02 \n" + + "xvilvl.h $xr8, $xr1, $xr0 \n" + "xvilvh.h $xr9, $xr1, $xr0 \n" + "xvilvl.h $xr10, $xr3, $xr2 \n" + "xvilvh.h $xr11, $xr3, $xr2 \n" + "xvilvl.h $xr12, $xr5, $xr4 \n" + "xvilvh.h $xr13, $xr5, $xr4 \n" + "xvilvl.h $xr14, $xr7, $xr6 \n" + "xvilvh.h $xr15, $xr7, $xr6 \n" + + "xvilvl.w $xr0, $xr10, $xr8 \n" + "xvilvh.w $xr1, $xr10, $xr8 \n" + "xvilvl.w $xr2, $xr11, $xr9 \n" + "xvilvh.w $xr3, $xr11, $xr9 \n" + "xvilvl.w $xr4, $xr14, $xr12 \n" + "xvilvh.w $xr5, $xr14, $xr12 \n" + "xvilvl.w $xr6, $xr15, $xr13 \n" + "xvilvh.w $xr7, $xr15, $xr13 \n" + + "xvilvl.d $xr8, $xr4, $xr0 \n" + "xvilvh.d $xr9, $xr4, $xr0 \n" + "xvilvl.d $xr10, $xr5, $xr1 \n" + "xvilvh.d $xr11, $xr5, $xr1 \n" + "xvilvl.d $xr12, $xr6, $xr2 \n" + "xvilvh.d $xr13, $xr6, $xr2 \n" + "xvilvl.d $xr14, $xr7, $xr3 \n" + "xvilvh.d $xr15, $xr7, $xr3 \n" + + "xvst $xr8, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr9, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr10, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr11, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr12, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr13, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr14, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr15, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_b), // %[src] + [dst] "+r"(dst_b), // %[dst] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15"); +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_lsx.cc b/source/rotate_lsx.cc index 9f330e32e..931b108b2 100644 --- a/source/rotate_lsx.cc +++ b/source/rotate_lsx.cc @@ -242,6 +242,57 @@ void Transpose4x4_32_LSX(const uint8_t* src, } } +void TransposeWx8_16_LSX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + while (width > 0) { + __m128i r0 = __lsx_vld(src_b, 0); + __m128i r1 = __lsx_vldx(src_b, s); + __m128i r2 = __lsx_vldx(src_b, s * 2); + __m128i r3 = __lsx_vldx(src_b, s * 3); + __m128i r4 = __lsx_vldx(src_b, s * 4); + __m128i r5 = __lsx_vldx(src_b, s * 5); + __m128i r6 = __lsx_vldx(src_b, s * 6); + __m128i r7 = __lsx_vldx(src_b, s * 7); + + __m128i t0 = __lsx_vilvl_h(r1, r0); + __m128i t1 = __lsx_vilvh_h(r1, r0); + __m128i t2 = __lsx_vilvl_h(r3, r2); + __m128i t3 = __lsx_vilvh_h(r3, r2); + __m128i t4 = __lsx_vilvl_h(r5, r4); + __m128i t5 = __lsx_vilvh_h(r5, r4); + __m128i t6 = __lsx_vilvl_h(r7, r6); + __m128i t7 = __lsx_vilvh_h(r7, r6); + + __m128i u0 = __lsx_vilvl_w(t2, t0); + __m128i u1 = __lsx_vilvh_w(t2, t0); + __m128i u2 = __lsx_vilvl_w(t3, t1); + __m128i u3 = __lsx_vilvh_w(t3, t1); + __m128i u4 = __lsx_vilvl_w(t6, t4); + __m128i u5 = __lsx_vilvh_w(t6, t4); + __m128i u6 = __lsx_vilvl_w(t7, t5); + __m128i u7 = __lsx_vilvh_w(t7, t5); + + __lsx_vst(__lsx_vilvl_d(u4, u0), dst_b, 0); + __lsx_vstx(__lsx_vilvh_d(u4, u0), dst_b, d); + __lsx_vstx(__lsx_vilvl_d(u5, u1), dst_b, d * 2); + __lsx_vstx(__lsx_vilvh_d(u5, u1), dst_b, d * 3); + __lsx_vstx(__lsx_vilvl_d(u6, u2), dst_b, d * 4); + __lsx_vstx(__lsx_vilvh_d(u6, u2), dst_b, d * 5); + __lsx_vstx(__lsx_vilvl_d(u7, u3), dst_b, d * 6); + __lsx_vstx(__lsx_vilvh_d(u7, u3), dst_b, d * 7); + src_b += 16; + dst_b += d * 8; + width -= 8; + } +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_neon.cc b/source/rotate_neon.cc index 6106787f0..49faf1ff1 100644 --- a/source/rotate_neon.cc +++ b/source/rotate_neon.cc @@ -419,6 +419,64 @@ void TransposeWxH_32_NEON(const uint8_t* src, } } +void TransposeWx8_16_NEON(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* temp; + asm volatile( + "1: \n" + "mov %[temp], %[src] \n" + "vld1.8 {q0}, [%[temp]], %[s] \n" + "vld1.8 {q1}, [%[temp]], %[s] \n" + "vld1.8 {q2}, [%[temp]], %[s] \n" + "vld1.8 {q3}, [%[temp]], %[s] \n" + "vld1.8 {q8}, [%[temp]], %[s] \n" + "vld1.8 {q9}, [%[temp]], %[s] \n" + "vld1.8 {q10}, [%[temp]], %[s] \n" + "vld1.8 {q11}, [%[temp]] \n" + "add %[src], %[src], #16 \n" + + // Stage 1: 16-bit transpose + "vtrn.16 q0, q1 \n" + "vtrn.16 q2, q3 \n" + "vtrn.16 q8, q9 \n" + "vtrn.16 q10, q11 \n" + + // Stage 2: 32-bit transpose + "vtrn.32 q0, q2 \n" + "vtrn.32 q1, q3 \n" + "vtrn.32 q8, q10 \n" + "vtrn.32 q9, q11 \n" + + // Stage 3: 64-bit swap + "vswp d1, d16 \n" + "vswp d3, d18 \n" + "vswp d5, d20 \n" + "vswp d7, d22 \n" + + "subs %[w], %[w], #8 \n" + "vst1.8 {q0}, [%[dst]], %[d] \n" + "vst1.8 {q1}, [%[dst]], %[d] \n" + "vst1.8 {q2}, [%[dst]], %[d] \n" + "vst1.8 {q3}, [%[dst]], %[d] \n" + "vst1.8 {q8}, [%[dst]], %[d] \n" + "vst1.8 {q9}, [%[dst]], %[d] \n" + "vst1.8 {q10}, [%[dst]], %[d] \n" + "vst1.8 {q11}, [%[dst]], %[d] \n" + "bgt 1b \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11"); +} + #endif // defined(__ARM_NEON__) && !defined(__aarch64__) #ifdef __cplusplus diff --git a/source/rotate_neon64.cc b/source/rotate_neon64.cc index 3960e28d9..e751488b5 100644 --- a/source/rotate_neon64.cc +++ b/source/rotate_neon64.cc @@ -474,6 +474,77 @@ void TransposeWxH_32_NEON(const uint8_t* src, } } +void TransposeWx8_16_NEON(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint16_t* temp; + asm volatile( + "1: \n" + "mov %[temp], %[src] \n" + "ld1 {v0.8h}, [%[temp]], %[s] \n" + "ld1 {v1.8h}, [%[temp]], %[s] \n" + "ld1 {v2.8h}, [%[temp]], %[s] \n" + "ld1 {v3.8h}, [%[temp]], %[s] \n" + "ld1 {v4.8h}, [%[temp]], %[s] \n" + "ld1 {v5.8h}, [%[temp]], %[s] \n" + "ld1 {v6.8h}, [%[temp]], %[s] \n" + "ld1 {v7.8h}, [%[temp]] \n" + "add %[src], %[src], #16 \n" + + // Stage 1: 16-bit transpose + "trn1 v16.8h, v0.8h, v1.8h \n" + "trn2 v17.8h, v0.8h, v1.8h \n" + "trn1 v18.8h, v2.8h, v3.8h \n" + "trn2 v19.8h, v2.8h, v3.8h \n" + "trn1 v20.8h, v4.8h, v5.8h \n" + "trn2 v21.8h, v4.8h, v5.8h \n" + "trn1 v22.8h, v6.8h, v7.8h \n" + "trn2 v23.8h, v6.8h, v7.8h \n" + + // Stage 2: 32-bit transpose + "trn1 v0.4s, v16.4s, v18.4s \n" + "trn1 v1.4s, v17.4s, v19.4s \n" + "trn2 v2.4s, v16.4s, v18.4s \n" + "trn2 v3.4s, v17.4s, v19.4s \n" + "trn1 v4.4s, v20.4s, v22.4s \n" + "trn1 v5.4s, v21.4s, v23.4s \n" + "trn2 v6.4s, v20.4s, v22.4s \n" + "trn2 v7.4s, v21.4s, v23.4s \n" + + // Stage 3: 64-bit transpose + "trn1 v16.2d, v0.2d, v4.2d \n" + "trn1 v17.2d, v1.2d, v5.2d \n" + "trn1 v18.2d, v2.2d, v6.2d \n" + "trn1 v19.2d, v3.2d, v7.2d \n" + "trn2 v20.2d, v0.2d, v4.2d \n" + "trn2 v21.2d, v1.2d, v5.2d \n" + "trn2 v22.2d, v2.2d, v6.2d \n" + "trn2 v23.2d, v3.2d, v7.2d \n" + + "subs %w[w], %w[w], #8 \n" + "st1 {v16.8h}, [%[dst]], %[d] \n" + "st1 {v17.8h}, [%[dst]], %[d] \n" + "st1 {v18.8h}, [%[dst]], %[d] \n" + "st1 {v19.8h}, [%[dst]], %[d] \n" + "st1 {v20.8h}, [%[dst]], %[d] \n" + "st1 {v21.8h}, [%[dst]], %[d] \n" + "st1 {v22.8h}, [%[dst]], %[d] \n" + "st1 {v23.8h}, [%[dst]], %[d] \n" + "b.gt 1b \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/rotate_rvv.cc b/source/rotate_rvv.cc index 3b9e2b635..6a352e352 100644 --- a/source/rotate_rvv.cc +++ b/source/rotate_rvv.cc @@ -143,6 +143,130 @@ void TransposeWxH_32_RVV(const uint8_t* src, } #endif // HAS_TRANSPOSEWXH_32_RVV +#ifdef HAS_TRANSPOSEWXH_16_RVV +void TransposeWxH_16_RVV(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s_stride = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d_stride = (ptrdiff_t)dst_stride * 2; + while (width >= 4) { + const uint16_t* s0 = src; + const uint16_t* s1 = src + 1; + const uint16_t* s2 = src + 2; + const uint16_t* s3 = src + 3; + uint8_t* d0 = (uint8_t*)dst; + uint8_t* d1 = d0 + d_stride; + uint8_t* d2 = d1 + d_stride; + uint8_t* d3 = d2 + d_stride; + int h = height; + int vl; + ptrdiff_t src_step; + ptrdiff_t dst_step; + asm volatile( + "vsetvli %[vl], %[h], e16, m8, ta, ma \n" + "slli %[d_step], %[vl], 1 \n" + "mul %[s_step], %[vl], %[s_stride] \n" + "sub %[h], %[h], %[vl] \n" + "bltz %[h], 2f \n" + "1: \n" + "vlse16.v v0, (%[s0]), %[s_stride] \n" + "vlse16.v v8, (%[s1]), %[s_stride] \n" + "vlse16.v v16, (%[s2]), %[s_stride] \n" + "vlse16.v v24, (%[s3]), %[s_stride] \n" + "add %[s0], %[s0], %[s_step] \n" + "add %[s1], %[s1], %[s_step] \n" + "add %[s2], %[s2], %[s_step] \n" + "add %[s3], %[s3], %[s_step] \n" + "vse16.v v0, (%[d0]) \n" + "vse16.v v8, (%[d1]) \n" + "vse16.v v16, (%[d2]) \n" + "vse16.v v24, (%[d3]) \n" + "add %[d0], %[d0], %[d_step] \n" + "add %[d1], %[d1], %[d_step] \n" + "add %[d2], %[d2], %[d_step] \n" + "add %[d3], %[d3], %[d_step] \n" + "sub %[h], %[h], %[vl] \n" + "bgez %[h], 1b \n" + "2: \n" + "add %[h], %[h], %[vl] \n" + "beqz %[h], 3f \n" + "vsetvli %[vl], %[h], e16, m8, ta, ma \n" + "vlse16.v v0, (%[s0]), %[s_stride] \n" + "vlse16.v v8, (%[s1]), %[s_stride] \n" + "vlse16.v v16, (%[s2]), %[s_stride] \n" + "vlse16.v v24, (%[s3]), %[s_stride] \n" + "vse16.v v0, (%[d0]) \n" + "vse16.v v8, (%[d1]) \n" + "vse16.v v16, (%[d2]) \n" + "vse16.v v24, (%[d3]) \n" + "3: \n" + : [h] "+r"(h), // %[h] + [s0] "+r"(s0), // %[s0] + [s1] "+r"(s1), // %[s1] + [s2] "+r"(s2), // %[s2] + [s3] "+r"(s3), // %[s3] + [d0] "+r"(d0), // %[d0] + [d1] "+r"(d1), // %[d1] + [d2] "+r"(d2), // %[d2] + [d3] "+r"(d3), // %[d3] + [vl] "=&r"(vl), // %[vl] + [s_step] "=&r"(src_step), // %[s_step] + [d_step] "=&r"(dst_step) // %[d_step] + : [s_stride] "r"(s_stride) // %[s_stride] + : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", + "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", + "v26", "v27", "v28", "v29", "v30", "v31"); + src += 4; + dst += 4 * (ptrdiff_t)dst_stride; + width -= 4; + } + while (width > 0) { + const uint16_t* s_col = src; + uint16_t* d_row = dst; + int h = height; + int vl; + ptrdiff_t src_step; + ptrdiff_t dst_step; + asm volatile( + "vsetvli %[vl], %[h], e16, m8, ta, ma \n" + "slli %[d_step], %[vl], 1 \n" + "mul %[s_step], %[vl], %[s_stride] \n" + "sub %[h], %[h], %[vl] \n" + "bltz %[h], 2f \n" + "1: \n" + "vlse16.v v8, (%[s]), %[s_stride] \n" + "add %[s], %[s], %[s_step] \n" + "vse16.v v8, (%[d]) \n" + "add %[d], %[d], %[d_step] \n" + "sub %[h], %[h], %[vl] \n" + "bgez %[h], 1b \n" + "2: \n" + "add %[h], %[h], %[vl] \n" + "beqz %[h], 3f \n" + "vsetvli %[vl], %[h], e16, m8, ta, ma \n" + "vlse16.v v8, (%[s]), %[s_stride] \n" + "vse16.v v8, (%[d]) \n" + "3: \n" + : [h] "+r"(h), // %[h] + [s] "+r"(s_col), // %[s] + [d] "+r"(d_row), // %[d] + [vl] "=&r"(vl), // %[vl] + [s_step] "=&r"(src_step), // %[s_step] + [d_step] "=&r"(dst_step) // %[d_step] + : [s_stride] "r"(s_stride) // %[s_stride] + : RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15"); + src += 1; + dst += (ptrdiff_t)dst_stride; + width -= 1; + } +} +#endif // HAS_TRANSPOSEWXH_16_RVV + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_sme.cc b/source/rotate_sme.cc index 2182198c9..85d5bf604 100644 --- a/source/rotate_sme.cc +++ b/source/rotate_sme.cc @@ -237,6 +237,73 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_32_SME( } while (height > 0); } +__arm_locally_streaming __arm_new("za") void TransposeWxH_16_SME( + const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height) { + int vl; + asm("cnth %x0" : "=r"(vl)); + + do { + const uint16_t* src2 = src; + uint16_t* dst2 = dst; + + // Process up to VL 16-bit rows per iteration of the outer loop. + int block_height = height > vl ? vl : height; + + int width2 = width; + do { + const uint16_t* src3 = src2; + + // Process up to VL 16-bit columns per iteration of the inner loop. + int block_width = width2 > vl ? vl : width2; + + asm volatile( + "mov w12, #0 \n" + + // Create a predicate to handle loading partial rows. + "whilelt p0.h, wzr, %w[block_width] \n" + + // Load H <= VL rows into ZA0.H. + "1: \n" + "ld1h {za0h.h[w12, 0]}, p0/z, [%[src3]] \n" + "add %[src3], %[src3], %[src_stride] \n" + "add w12, w12, #1 \n" + "cmp w12, %w[block_height] \n" + "b.ne 1b \n" + + // Create a predicate to handle storing partial columns. + "whilelt p0.h, wzr, %w[block_height] \n" + "mov w12, #0 \n" + + // Store W <= VL columns from ZA0.H. + "2: \n" + "st1h {za0v.h[w12, 0]}, p0, [%[dst2]] \n" + "add %[dst2], %[dst2], %[dst_stride] \n" + "add w12, w12, #1 \n" + "cmp w12, %w[block_width] \n" + "b.ne 2b \n" + : [src3] "+r"(src3), // %[src3] + [dst2] "+r"(dst2) // %[dst2] + : [src_stride] "r"((ptrdiff_t)src_stride * 2), // %[src_stride] + [dst_stride] "r"((ptrdiff_t)dst_stride * 2), // %[dst_stride] + [block_width] "r"(block_width), // %[block_width] + [block_height] "r"(block_height) // %[block_height] + : "memory", "cc", "w12", "p0", "za"); + + src2 += vl; + width2 -= vl; + } while (width2 > 0); + + src += (ptrdiff_t)vl * src_stride; + dst += vl; + height -= vl; + } while (height > 0); +} + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/rotate_sve.cc b/source/rotate_sve.cc index 84d63bc7e..efe243689 100644 --- a/source/rotate_sve.cc +++ b/source/rotate_sve.cc @@ -109,6 +109,168 @@ void TransposeNx4_32_SVE2(const uint8_t* src, : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0"); } +void TransposeWx8_16_SVE2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint16_t* temp; + asm volatile( + "ptrue p0.h, vl8 \n" + "subs %w[w], %w[w], #8 \n" + "b.lt 2f \n" + + "1: \n" + "mov %[temp], %[src] \n" + "ld1h {z0.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z1.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z2.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z3.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z4.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z5.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z6.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z7.h}, p0/z, [%[temp]] \n" + "add %[src], %[src], #16 \n" + + "trn1 z16.h, z0.h, z1.h \n" + "trn2 z17.h, z0.h, z1.h \n" + "trn1 z18.h, z2.h, z3.h \n" + "trn2 z19.h, z2.h, z3.h \n" + "trn1 z20.h, z4.h, z5.h \n" + "trn2 z21.h, z4.h, z5.h \n" + "trn1 z22.h, z6.h, z7.h \n" + "trn2 z23.h, z6.h, z7.h \n" + + "trn1 z0.s, z16.s, z18.s \n" + "trn1 z1.s, z17.s, z19.s \n" + "trn2 z2.s, z16.s, z18.s \n" + "trn2 z3.s, z17.s, z19.s \n" + "trn1 z4.s, z20.s, z22.s \n" + "trn1 z5.s, z21.s, z23.s \n" + "trn2 z6.s, z20.s, z22.s \n" + "trn2 z7.s, z21.s, z23.s \n" + + "trn1 z16.d, z0.d, z4.d \n" + "trn1 z17.d, z1.d, z5.d \n" + "trn1 z18.d, z2.d, z6.d \n" + "trn1 z19.d, z3.d, z7.d \n" + "trn2 z20.d, z0.d, z4.d \n" + "trn2 z21.d, z1.d, z5.d \n" + "trn2 z22.d, z2.d, z6.d \n" + "trn2 z23.d, z3.d, z7.d \n" + + "st1h {z16.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z17.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z18.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z19.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z20.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z21.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z22.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z23.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "subs %w[w], %w[w], #8 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[w], %w[w], #8 \n" + "b.eq 3f \n" + "whilelt p1.h, wzr, %w[w] \n" + "mov %[temp], %[src] \n" + "ld1h {z0.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z1.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z2.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z3.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z4.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z5.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z6.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z7.h}, p1/z, [%[temp]] \n" + + "trn1 z16.h, z0.h, z1.h \n" + "trn2 z17.h, z0.h, z1.h \n" + "trn1 z18.h, z2.h, z3.h \n" + "trn2 z19.h, z2.h, z3.h \n" + "trn1 z20.h, z4.h, z5.h \n" + "trn2 z21.h, z4.h, z5.h \n" + "trn1 z22.h, z6.h, z7.h \n" + "trn2 z23.h, z6.h, z7.h \n" + + "trn1 z0.s, z16.s, z18.s \n" + "trn1 z1.s, z17.s, z19.s \n" + "trn2 z2.s, z16.s, z18.s \n" + "trn2 z3.s, z17.s, z19.s \n" + "trn1 z4.s, z20.s, z22.s \n" + "trn1 z5.s, z21.s, z23.s \n" + "trn2 z6.s, z20.s, z22.s \n" + "trn2 z7.s, z21.s, z23.s \n" + + "trn1 z16.d, z0.d, z4.d \n" + "trn1 z17.d, z1.d, z5.d \n" + "trn1 z18.d, z2.d, z6.d \n" + "trn1 z19.d, z3.d, z7.d \n" + "trn2 z20.d, z0.d, z4.d \n" + "trn2 z21.d, z1.d, z5.d \n" + "trn2 z22.d, z2.d, z6.d \n" + "trn2 z23.d, z3.d, z7.d \n" + + "st1h {z16.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z17.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z18.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z19.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z20.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z21.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z22.h}, p0, [%[dst]] \n" + "3: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "p0", "p1", "z0", "z1", "z2", "z3", "z4", "z5", "z6", + "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23"); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/rotate_wasm.cc b/source/rotate_wasm.cc index 093a37e39..9c3cbac64 100644 --- a/source/rotate_wasm.cc +++ b/source/rotate_wasm.cc @@ -260,6 +260,57 @@ void Transpose4x4_32_WASMSIMD(const uint8_t* src, } #endif // HAS_TRANSPOSE4X4_32_WASMSIMD +#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD) +void TransposeWx8_16_WASMSIMD(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + v128_t r0 = wasm_v128_load(src); + v128_t r1 = wasm_v128_load(src + s); + v128_t r2 = wasm_v128_load(src + s * 2); + v128_t r3 = wasm_v128_load(src + s * 3); + v128_t r4 = wasm_v128_load(src + s * 4); + v128_t r5 = wasm_v128_load(src + s * 5); + v128_t r6 = wasm_v128_load(src + s * 6); + v128_t r7 = wasm_v128_load(src + s * 7); + + v128_t t0 = ZIPLO16(r0, r1); + v128_t t1 = ZIPHI16(r0, r1); + v128_t t2 = ZIPLO16(r2, r3); + v128_t t3 = ZIPHI16(r2, r3); + v128_t t4 = ZIPLO16(r4, r5); + v128_t t5 = ZIPHI16(r4, r5); + v128_t t6 = ZIPLO16(r6, r7); + v128_t t7 = ZIPHI16(r6, r7); + + v128_t u0 = ZIPLO32(t0, t2); + v128_t u1 = ZIPHI32(t0, t2); + v128_t u2 = ZIPLO32(t1, t3); + v128_t u3 = ZIPHI32(t1, t3); + v128_t u4 = ZIPLO32(t4, t6); + v128_t u5 = ZIPHI32(t4, t6); + v128_t u6 = ZIPLO32(t5, t7); + v128_t u7 = ZIPHI32(t5, t7); + + wasm_v128_store(dst, ZIPLO64(u0, u4)); + wasm_v128_store(dst + d, ZIPHI64(u0, u4)); + wasm_v128_store(dst + d * 2, ZIPLO64(u1, u5)); + wasm_v128_store(dst + d * 3, ZIPHI64(u1, u5)); + wasm_v128_store(dst + d * 4, ZIPLO64(u2, u6)); + wasm_v128_store(dst + d * 5, ZIPHI64(u2, u6)); + wasm_v128_store(dst + d * 6, ZIPLO64(u3, u7)); + wasm_v128_store(dst + d * 7, ZIPHI64(u3, u7)); + src += 8; + dst += d * 8; + width -= 8; + } +} +#endif // HAS_TRANSPOSEWX8_16_WASMSIMD + #undef ZIPLO8 #undef ZIPHI8 #undef ZIPLO16 diff --git a/source/rotate_win.cc b/source/rotate_win.cc index f5eb65dad..ec3772084 100644 --- a/source/rotate_win.cc +++ b/source/rotate_win.cc @@ -417,6 +417,373 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW) +#if defined(HAS_TRANSPOSEWX16_16_AVX2) +LIBYUV_TARGET_AVX2 +void TransposeWx16_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), + _mm_loadu_si128((const __m128i*)(src + s * 8)), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), + _mm_loadu_si128((const __m128i*)(src + s * 9)), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), + _mm_loadu_si128((const __m128i*)(src + s * 10)), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), + _mm_loadu_si128((const __m128i*)(src + s * 11)), 1); + __m256i r4 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 4))), + _mm_loadu_si128((const __m128i*)(src + s * 12)), 1); + __m256i r5 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 5))), + _mm_loadu_si128((const __m128i*)(src + s * 13)), 1); + __m256i r6 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 6))), + _mm_loadu_si128((const __m128i*)(src + s * 14)), 1); + __m256i r7 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 7))), + _mm_loadu_si128((const __m128i*)(src + s * 15)), 1); + + __m256i t0 = _mm256_unpacklo_epi16(r0, r1); + __m256i t1 = _mm256_unpackhi_epi16(r0, r1); + __m256i t2 = _mm256_unpacklo_epi16(r2, r3); + __m256i t3 = _mm256_unpackhi_epi16(r2, r3); + __m256i t4 = _mm256_unpacklo_epi16(r4, r5); + __m256i t5 = _mm256_unpackhi_epi16(r4, r5); + __m256i t6 = _mm256_unpacklo_epi16(r6, r7); + __m256i t7 = _mm256_unpackhi_epi16(r6, r7); + + __m256i u0 = _mm256_unpacklo_epi32(t0, t2); + __m256i u1 = _mm256_unpackhi_epi32(t0, t2); + __m256i u2 = _mm256_unpacklo_epi32(t1, t3); + __m256i u3 = _mm256_unpackhi_epi32(t1, t3); + __m256i u4 = _mm256_unpacklo_epi32(t4, t6); + __m256i u5 = _mm256_unpackhi_epi32(t4, t6); + __m256i u6 = _mm256_unpacklo_epi32(t5, t7); + __m256i u7 = _mm256_unpackhi_epi32(t5, t7); + + _mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(u0, u4)); + _mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(u0, u4)); + _mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(u1, u5)); + _mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(u1, u5)); + _mm256_storeu_si256((__m256i*)(dst + d * 4), _mm256_unpacklo_epi64(u2, u6)); + _mm256_storeu_si256((__m256i*)(dst + d * 5), _mm256_unpackhi_epi64(u2, u6)); + _mm256_storeu_si256((__m256i*)(dst + d * 6), _mm256_unpacklo_epi64(u3, u7)); + _mm256_storeu_si256((__m256i*)(dst + d * 7), _mm256_unpackhi_epi64(u3, u7)); + src += 8; + dst += d * 8; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSEWX16_16_AVX2) + +#if defined(HAS_TRANSPOSEWXH_16_AVX512BW) +LIBYUV_TARGET_AVX512BW +void TransposeWxH_16_AVX512BW(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (height > 0) { + const int bh = (height >= 32) ? 32 : height; + const uint16_t* src_row = src; + uint16_t* dst_col = dst; + int w = width; + if (bh == 32) { + while (w >= 8) { + __m512i r0 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src_row))); + __m512i r1 = _mm512_castsi128_si512( + _mm_loadu_si128((const __m128i*)(src_row + s))); + __m512i r2 = _mm512_castsi128_si512( + _mm_loadu_si128((const __m128i*)(src_row + s * 2))); + __m512i r3 = _mm512_castsi128_si512( + _mm_loadu_si128((const __m128i*)(src_row + s * 3))); + __m512i r4 = _mm512_castsi128_si512( + _mm_loadu_si128((const __m128i*)(src_row + s * 4))); + __m512i r5 = _mm512_castsi128_si512( + _mm_loadu_si128((const __m128i*)(src_row + s * 5))); + __m512i r6 = _mm512_castsi128_si512( + _mm_loadu_si128((const __m128i*)(src_row + s * 6))); + __m512i r7 = _mm512_castsi128_si512( + _mm_loadu_si128((const __m128i*)(src_row + s * 7))); + + r0 = _mm512_inserti32x4( + r0, _mm_loadu_si128((const __m128i*)(src_row + s * 8)), 1); + r1 = _mm512_inserti32x4( + r1, _mm_loadu_si128((const __m128i*)(src_row + s * 9)), 1); + r2 = _mm512_inserti32x4( + r2, _mm_loadu_si128((const __m128i*)(src_row + s * 10)), 1); + r3 = _mm512_inserti32x4( + r3, _mm_loadu_si128((const __m128i*)(src_row + s * 11)), 1); + r4 = _mm512_inserti32x4( + r4, _mm_loadu_si128((const __m128i*)(src_row + s * 12)), 1); + r5 = _mm512_inserti32x4( + r5, _mm_loadu_si128((const __m128i*)(src_row + s * 13)), 1); + r6 = _mm512_inserti32x4( + r6, _mm_loadu_si128((const __m128i*)(src_row + s * 14)), 1); + r7 = _mm512_inserti32x4( + r7, _mm_loadu_si128((const __m128i*)(src_row + s * 15)), 1); + + r0 = _mm512_inserti32x4( + r0, _mm_loadu_si128((const __m128i*)(src_row + s * 16)), 2); + r1 = _mm512_inserti32x4( + r1, _mm_loadu_si128((const __m128i*)(src_row + s * 17)), 2); + r2 = _mm512_inserti32x4( + r2, _mm_loadu_si128((const __m128i*)(src_row + s * 18)), 2); + r3 = _mm512_inserti32x4( + r3, _mm_loadu_si128((const __m128i*)(src_row + s * 19)), 2); + r4 = _mm512_inserti32x4( + r4, _mm_loadu_si128((const __m128i*)(src_row + s * 20)), 2); + r5 = _mm512_inserti32x4( + r5, _mm_loadu_si128((const __m128i*)(src_row + s * 21)), 2); + r6 = _mm512_inserti32x4( + r6, _mm_loadu_si128((const __m128i*)(src_row + s * 22)), 2); + r7 = _mm512_inserti32x4( + r7, _mm_loadu_si128((const __m128i*)(src_row + s * 23)), 2); + + r0 = _mm512_inserti32x4( + r0, _mm_loadu_si128((const __m128i*)(src_row + s * 24)), 3); + r1 = _mm512_inserti32x4( + r1, _mm_loadu_si128((const __m128i*)(src_row + s * 25)), 3); + r2 = _mm512_inserti32x4( + r2, _mm_loadu_si128((const __m128i*)(src_row + s * 26)), 3); + r3 = _mm512_inserti32x4( + r3, _mm_loadu_si128((const __m128i*)(src_row + s * 27)), 3); + r4 = _mm512_inserti32x4( + r4, _mm_loadu_si128((const __m128i*)(src_row + s * 28)), 3); + r5 = _mm512_inserti32x4( + r5, _mm_loadu_si128((const __m128i*)(src_row + s * 29)), 3); + r6 = _mm512_inserti32x4( + r6, _mm_loadu_si128((const __m128i*)(src_row + s * 30)), 3); + r7 = _mm512_inserti32x4( + r7, _mm_loadu_si128((const __m128i*)(src_row + s * 31)), 3); + + __m512i t0 = _mm512_unpacklo_epi16(r0, r1); + __m512i t1 = _mm512_unpackhi_epi16(r0, r1); + __m512i t2 = _mm512_unpacklo_epi16(r2, r3); + __m512i t3 = _mm512_unpackhi_epi16(r2, r3); + __m512i t4 = _mm512_unpacklo_epi16(r4, r5); + __m512i t5 = _mm512_unpackhi_epi16(r4, r5); + __m512i t6 = _mm512_unpacklo_epi16(r6, r7); + __m512i t7 = _mm512_unpackhi_epi16(r6, r7); + + __m512i u0 = _mm512_unpacklo_epi32(t0, t2); + __m512i u1 = _mm512_unpackhi_epi32(t0, t2); + __m512i u2 = _mm512_unpacklo_epi32(t1, t3); + __m512i u3 = _mm512_unpackhi_epi32(t1, t3); + __m512i u4 = _mm512_unpacklo_epi32(t4, t6); + __m512i u5 = _mm512_unpackhi_epi32(t4, t6); + __m512i u6 = _mm512_unpacklo_epi32(t5, t7); + __m512i u7 = _mm512_unpackhi_epi32(t5, t7); + + _mm512_storeu_si512((__m512i*)(dst_col), _mm512_unpacklo_epi64(u0, u4)); + _mm512_storeu_si512((__m512i*)(dst_col + d), + _mm512_unpackhi_epi64(u0, u4)); + _mm512_storeu_si512((__m512i*)(dst_col + d * 2), + _mm512_unpacklo_epi64(u1, u5)); + _mm512_storeu_si512((__m512i*)(dst_col + d * 3), + _mm512_unpackhi_epi64(u1, u5)); + _mm512_storeu_si512((__m512i*)(dst_col + d * 4), + _mm512_unpacklo_epi64(u2, u6)); + _mm512_storeu_si512((__m512i*)(dst_col + d * 5), + _mm512_unpackhi_epi64(u2, u6)); + _mm512_storeu_si512((__m512i*)(dst_col + d * 6), + _mm512_unpacklo_epi64(u3, u7)); + _mm512_storeu_si512((__m512i*)(dst_col + d * 7), + _mm512_unpackhi_epi64(u3, u7)); + src_row += 8; + dst_col += d * 8; + w -= 8; + } + } + const __mmask32 row_mask = + (bh == 32) ? (__mmask32)0xFFFFFFFFu : ((1u << bh) - 1); + while (w > 0) { + const int bw = (w >= 8) ? 8 : w; + const __mmask8 col_mask = (1u << bw) - 1; + const __m128i z128 = _mm_setzero_si128(); + __m512i r0 = + _mm512_castsi128_si512(_mm_maskz_loadu_epi16(col_mask, src_row)); + __m512i r1 = _mm512_castsi128_si512( + (bh > 1) ? _mm_maskz_loadu_epi16(col_mask, src_row + s) : z128); + __m512i r2 = _mm512_castsi128_si512( + (bh > 2) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 2) : z128); + __m512i r3 = _mm512_castsi128_si512( + (bh > 3) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 3) : z128); + __m512i r4 = _mm512_castsi128_si512( + (bh > 4) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 4) : z128); + __m512i r5 = _mm512_castsi128_si512( + (bh > 5) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 5) : z128); + __m512i r6 = _mm512_castsi128_si512( + (bh > 6) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 6) : z128); + __m512i r7 = _mm512_castsi128_si512( + (bh > 7) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 7) : z128); + + r0 = _mm512_inserti32x4( + r0, + (bh > 8) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 8) : z128, + 1); + r1 = _mm512_inserti32x4( + r1, + (bh > 9) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 9) : z128, + 1); + r2 = _mm512_inserti32x4( + r2, + (bh > 10) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 10) : z128, + 1); + r3 = _mm512_inserti32x4( + r3, + (bh > 11) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 11) : z128, + 1); + r4 = _mm512_inserti32x4( + r4, + (bh > 12) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 12) : z128, + 1); + r5 = _mm512_inserti32x4( + r5, + (bh > 13) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 13) : z128, + 1); + r6 = _mm512_inserti32x4( + r6, + (bh > 14) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 14) : z128, + 1); + r7 = _mm512_inserti32x4( + r7, + (bh > 15) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 15) : z128, + 1); + + r0 = _mm512_inserti32x4( + r0, + (bh > 16) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 16) : z128, + 2); + r1 = _mm512_inserti32x4( + r1, + (bh > 17) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 17) : z128, + 2); + r2 = _mm512_inserti32x4( + r2, + (bh > 18) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 18) : z128, + 2); + r3 = _mm512_inserti32x4( + r3, + (bh > 19) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 19) : z128, + 2); + r4 = _mm512_inserti32x4( + r4, + (bh > 20) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 20) : z128, + 2); + r5 = _mm512_inserti32x4( + r5, + (bh > 21) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 21) : z128, + 2); + r6 = _mm512_inserti32x4( + r6, + (bh > 22) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 22) : z128, + 2); + r7 = _mm512_inserti32x4( + r7, + (bh > 23) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 23) : z128, + 2); + + r0 = _mm512_inserti32x4( + r0, + (bh > 24) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 24) : z128, + 3); + r1 = _mm512_inserti32x4( + r1, + (bh > 25) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 25) : z128, + 3); + r2 = _mm512_inserti32x4( + r2, + (bh > 26) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 26) : z128, + 3); + r3 = _mm512_inserti32x4( + r3, + (bh > 27) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 27) : z128, + 3); + r4 = _mm512_inserti32x4( + r4, + (bh > 28) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 28) : z128, + 3); + r5 = _mm512_inserti32x4( + r5, + (bh > 29) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 29) : z128, + 3); + r6 = _mm512_inserti32x4( + r6, + (bh > 30) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 30) : z128, + 3); + r7 = _mm512_inserti32x4( + r7, + (bh > 31) ? _mm_maskz_loadu_epi16(col_mask, src_row + s * 31) : z128, + 3); + + __m512i t0 = _mm512_unpacklo_epi16(r0, r1); + __m512i t1 = _mm512_unpackhi_epi16(r0, r1); + __m512i t2 = _mm512_unpacklo_epi16(r2, r3); + __m512i t3 = _mm512_unpackhi_epi16(r2, r3); + __m512i t4 = _mm512_unpacklo_epi16(r4, r5); + __m512i t5 = _mm512_unpackhi_epi16(r4, r5); + __m512i t6 = _mm512_unpacklo_epi16(r6, r7); + __m512i t7 = _mm512_unpackhi_epi16(r6, r7); + + __m512i u0 = _mm512_unpacklo_epi32(t0, t2); + __m512i u1 = _mm512_unpackhi_epi32(t0, t2); + __m512i u2 = _mm512_unpacklo_epi32(t1, t3); + __m512i u3 = _mm512_unpackhi_epi32(t1, t3); + __m512i u4 = _mm512_unpacklo_epi32(t4, t6); + __m512i u5 = _mm512_unpackhi_epi32(t4, t6); + __m512i u6 = _mm512_unpacklo_epi32(t5, t7); + __m512i u7 = _mm512_unpackhi_epi32(t5, t7); + + _mm512_mask_storeu_epi16(dst_col, row_mask, + _mm512_unpacklo_epi64(u0, u4)); + if (bw > 1) + _mm512_mask_storeu_epi16(dst_col + d, row_mask, + _mm512_unpackhi_epi64(u0, u4)); + if (bw > 2) + _mm512_mask_storeu_epi16(dst_col + d * 2, row_mask, + _mm512_unpacklo_epi64(u1, u5)); + if (bw > 3) + _mm512_mask_storeu_epi16(dst_col + d * 3, row_mask, + _mm512_unpackhi_epi64(u1, u5)); + if (bw > 4) + _mm512_mask_storeu_epi16(dst_col + d * 4, row_mask, + _mm512_unpacklo_epi64(u2, u6)); + if (bw > 5) + _mm512_mask_storeu_epi16(dst_col + d * 5, row_mask, + _mm512_unpackhi_epi64(u2, u6)); + if (bw > 6) + _mm512_mask_storeu_epi16(dst_col + d * 6, row_mask, + _mm512_unpacklo_epi64(u3, u7)); + if (bw > 7) + _mm512_mask_storeu_epi16(dst_col + d * 7, row_mask, + _mm512_unpackhi_epi64(u3, u7)); + src_row += 8; + dst_col += d * 8; + w -= 8; + } + src += 32 * s; + dst += 32; + height -= 32; + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSEWXH_16_AVX512BW) + #if defined(HAS_TRANSPOSE4X4_32_SSE2) // Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row. LIBYUV_TARGET_SSE2 @@ -447,79 +814,6 @@ void Transpose4x4_32_SSE2(const uint8_t* src, } #endif // defined(HAS_TRANSPOSE4X4_32_SSE2) -#if defined(HAS_TRANSPOSENX4_32_AVX2) -// Transpose 32 bit values (ARGB) in Nx4 tiles with 2 blocks of 4x4 at a time -// in ymm0..ymm3 and _mm_maskstore_epi32 tail for arbitrary N. -LIBYUV_TARGET_AVX2 -void TransposeNx4_32_AVX2(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - const ptrdiff_t s = src_stride; - const ptrdiff_t d = dst_stride; - while (width >= 8) { - __m256i r0 = _mm256_inserti128_si256( - _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), - _mm_loadu_si128((const __m128i*)(src + s * 4)), 1); - __m256i r1 = _mm256_inserti128_si256( - _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), - _mm_loadu_si128((const __m128i*)(src + s * 5)), 1); - __m256i r2 = _mm256_inserti128_si256( - _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), - _mm_loadu_si128((const __m128i*)(src + s * 6)), 1); - __m256i r3 = _mm256_inserti128_si256( - _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), - _mm_loadu_si128((const __m128i*)(src + s * 7)), 1); - __m256i t0 = _mm256_unpacklo_epi32(r0, r1); // a e b f - __m256i t1 = _mm256_unpacklo_epi32(r2, r3); // i m j n - __m256i t2 = _mm256_unpackhi_epi32(r0, r1); // c g d h - __m256i t3 = _mm256_unpackhi_epi32(r2, r3); // k o l p - _mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1)); - _mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1)); - _mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3)); - _mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3)); - src += s * 8; - dst += 32; - width -= 8; - } - if (width >= 4) { - __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); - __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); - __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); - __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); - __m128i t0 = _mm_unpacklo_epi32(r0, r1); - __m128i t1 = _mm_unpacklo_epi32(r2, r3); - __m128i t2 = _mm_unpackhi_epi32(r0, r1); - __m128i t3 = _mm_unpackhi_epi32(r2, r3); - _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); - _mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1)); - _mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3)); - _mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3)); - src += s * 4; - dst += 16; - width -= 4; - } - if (width > 0) { - const __m128i mask = - _mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (width * 8)) - 1)); - __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); - __m128i r1 = (width > 1) ? _mm_loadu_si128((const __m128i*)(src + s)) : r0; - __m128i r2 = - (width > 2) ? _mm_loadu_si128((const __m128i*)(src + s * 2)) : r1; - __m128i t0 = _mm_unpacklo_epi32(r0, r1); - __m128i t1 = _mm_unpacklo_epi32(r2, r2); - __m128i t2 = _mm_unpackhi_epi32(r0, r1); - __m128i t3 = _mm_unpackhi_epi32(r2, r2); - _mm_maskstore_epi32((int*)(dst), mask, _mm_unpacklo_epi64(t0, t1)); - _mm_maskstore_epi32((int*)(dst + d), mask, _mm_unpackhi_epi64(t0, t1)); - _mm_maskstore_epi32((int*)(dst + d * 2), mask, _mm_unpacklo_epi64(t2, t3)); - _mm_maskstore_epi32((int*)(dst + d * 3), mask, _mm_unpackhi_epi64(t2, t3)); - } - _mm256_zeroupper(); -} -#endif // defined(HAS_TRANSPOSENX4_32_AVX2) - #if defined(HAS_TRANSPOSEWXH_32_AVX2) LIBYUV_TARGET_AVX2 void TransposeWxH_32_AVX2(const uint8_t* src, @@ -530,228 +824,77 @@ void TransposeWxH_32_AVX2(const uint8_t* src, int height) { const ptrdiff_t s = src_stride; const ptrdiff_t d = dst_stride; - while (height >= 8) { + const __m256i idx8 = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + const __m128i idx4 = _mm_setr_epi32(0, 1, 2, 3); + while (height > 0) { + const int bh = (height >= 8) ? 8 : height; + const __m256i row_mask = _mm256_cmpgt_epi32(_mm256_set1_epi32(bh), idx8); const uint8_t* src_row = src; uint8_t* dst_col = dst; int w = width; - while (w >= 8) { - __m256i r0 = _mm256_loadu_si256((const __m256i*)(src_row)); - __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_row + s)); - __m256i r2 = _mm256_loadu_si256((const __m256i*)(src_row + s * 2)); - __m256i r3 = _mm256_loadu_si256((const __m256i*)(src_row + s * 3)); - __m256i r4 = _mm256_loadu_si256((const __m256i*)(src_row + s * 4)); - __m256i r5 = _mm256_loadu_si256((const __m256i*)(src_row + s * 5)); - __m256i r6 = _mm256_loadu_si256((const __m256i*)(src_row + s * 6)); - __m256i r7 = _mm256_loadu_si256((const __m256i*)(src_row + s * 7)); + while (w > 0) { + const int bw = (w >= 4) ? 4 : w; + const __m128i col_mask = _mm_cmpgt_epi32(_mm_set1_epi32(bw), idx4); + const __m128i z = _mm_setzero_si128(); + __m128i x0 = _mm_maskload_epi32((const int*)(src_row), col_mask); + __m128i x1 = (bh > 1) + ? _mm_maskload_epi32((const int*)(src_row + s), col_mask) + : z; + __m128i x2 = + (bh > 2) ? _mm_maskload_epi32((const int*)(src_row + s * 2), col_mask) + : z; + __m128i x3 = + (bh > 3) ? _mm_maskload_epi32((const int*)(src_row + s * 3), col_mask) + : z; + __m128i x4 = + (bh > 4) ? _mm_maskload_epi32((const int*)(src_row + s * 4), col_mask) + : z; + __m128i x5 = + (bh > 5) ? _mm_maskload_epi32((const int*)(src_row + s * 5), col_mask) + : z; + __m128i x6 = + (bh > 6) ? _mm_maskload_epi32((const int*)(src_row + s * 6), col_mask) + : z; + __m128i x7 = + (bh > 7) ? _mm_maskload_epi32((const int*)(src_row + s * 7), col_mask) + : z; + + __m256i r0 = _mm256_inserti128_si256(_mm256_castsi128_si256(x0), x4, 1); + __m256i r1 = _mm256_inserti128_si256(_mm256_castsi128_si256(x1), x5, 1); + __m256i r2 = _mm256_inserti128_si256(_mm256_castsi128_si256(x2), x6, 1); + __m256i r3 = _mm256_inserti128_si256(_mm256_castsi128_si256(x3), x7, 1); __m256i t0 = _mm256_unpacklo_epi32(r0, r1); - __m256i t1 = _mm256_unpackhi_epi32(r0, r1); - __m256i t2 = _mm256_unpacklo_epi32(r2, r3); + __m256i t1 = _mm256_unpacklo_epi32(r2, r3); + __m256i t2 = _mm256_unpackhi_epi32(r0, r1); __m256i t3 = _mm256_unpackhi_epi32(r2, r3); - __m256i t4 = _mm256_unpacklo_epi32(r4, r5); - __m256i t5 = _mm256_unpackhi_epi32(r4, r5); - __m256i t6 = _mm256_unpacklo_epi32(r6, r7); - __m256i t7 = _mm256_unpackhi_epi32(r6, r7); - __m256i u0 = _mm256_unpacklo_epi64(t0, t2); - __m256i u1 = _mm256_unpackhi_epi64(t0, t2); - __m256i u2 = _mm256_unpacklo_epi64(t1, t3); - __m256i u3 = _mm256_unpackhi_epi64(t1, t3); - __m256i u4 = _mm256_unpacklo_epi64(t4, t6); - __m256i u5 = _mm256_unpackhi_epi64(t4, t6); - __m256i u6 = _mm256_unpacklo_epi64(t5, t7); - __m256i u7 = _mm256_unpackhi_epi64(t5, t7); - - _mm256_storeu_si256((__m256i*)(dst_col), - _mm256_permute2x128_si256(u0, u4, 0x20)); - _mm256_storeu_si256((__m256i*)(dst_col + d), - _mm256_permute2x128_si256(u1, u5, 0x20)); - _mm256_storeu_si256((__m256i*)(dst_col + d * 2), - _mm256_permute2x128_si256(u2, u6, 0x20)); - _mm256_storeu_si256((__m256i*)(dst_col + d * 3), - _mm256_permute2x128_si256(u3, u7, 0x20)); - _mm256_storeu_si256((__m256i*)(dst_col + d * 4), - _mm256_permute2x128_si256(u0, u4, 0x31)); - _mm256_storeu_si256((__m256i*)(dst_col + d * 5), - _mm256_permute2x128_si256(u1, u5, 0x31)); - _mm256_storeu_si256((__m256i*)(dst_col + d * 6), - _mm256_permute2x128_si256(u2, u6, 0x31)); - _mm256_storeu_si256((__m256i*)(dst_col + d * 7), - _mm256_permute2x128_si256(u3, u7, 0x31)); - src_row += 32; - dst_col += 8 * d; - w -= 8; - } - if (w >= 4) { - TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, 8); + _mm256_maskstore_epi32((int*)(dst_col), row_mask, + _mm256_unpacklo_epi64(t0, t1)); + if (bw > 1) { + _mm256_maskstore_epi32((int*)(dst_col + d), row_mask, + _mm256_unpackhi_epi64(t0, t1)); + } + if (bw > 2) { + _mm256_maskstore_epi32((int*)(dst_col + d * 2), row_mask, + _mm256_unpacklo_epi64(t2, t3)); + } + if (bw > 3) { + _mm256_maskstore_epi32((int*)(dst_col + d * 3), row_mask, + _mm256_unpackhi_epi64(t2, t3)); + } src_row += 16; dst_col += 4 * d; w -= 4; } - if (w > 0) { - const __m128i col_mask = - _mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (w * 8)) - 1)); - uint8_t* dst0 = dst_col; - uint8_t* dst1 = (w > 1) ? dst0 + d : dst0; - uint8_t* dst2 = (w > 2) ? dst1 + d : dst1; - for (int y = 0; y < 8; y += 4) { - __m128i r0 = - _mm_maskload_epi32((const int*)(src_row + y * s), col_mask); - __m128i r1 = - _mm_maskload_epi32((const int*)(src_row + (y + 1) * s), col_mask); - __m128i r2 = - _mm_maskload_epi32((const int*)(src_row + (y + 2) * s), col_mask); - __m128i r3 = - _mm_maskload_epi32((const int*)(src_row + (y + 3) * s), col_mask); - __m128i t0 = _mm_unpacklo_epi32(r0, r1); - __m128i t1 = _mm_unpacklo_epi32(r2, r3); - __m128i t2 = _mm_unpackhi_epi32(r0, r1); - __m128i t3 = _mm_unpackhi_epi32(r2, r3); - _mm_storeu_si128((__m128i*)(dst2 + y * 4), _mm_unpacklo_epi64(t2, t3)); - _mm_storeu_si128((__m128i*)(dst1 + y * 4), _mm_unpackhi_epi64(t0, t1)); - _mm_storeu_si128((__m128i*)(dst0 + y * 4), _mm_unpacklo_epi64(t0, t1)); - } - } src += 8 * s; dst += 32; height -= 8; } - while (height > 0) { - const int bh = (height >= 4) ? 4 : height; - const uint8_t* src_row = src; - uint8_t* dst_col = dst; - int w = width; - while (w >= 4) { - TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, bh); - src_row += 16; - dst_col += 4 * d; - w -= 4; - } - if (w > 0) { - const __m128i col_mask = - _mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (w * 8)) - 1)); - const __m128i row_mask = - _mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (bh * 8)) - 1)); - uint8_t* dst0 = dst_col; - uint8_t* dst1 = (w > 1) ? dst0 + d : dst0; - uint8_t* dst2 = (w > 2) ? dst1 + d : dst1; - __m128i r0 = _mm_maskload_epi32((const int*)(src_row), col_mask); - __m128i r1 = (bh > 1) - ? _mm_maskload_epi32((const int*)(src_row + s), col_mask) - : r0; - __m128i r2 = - (bh > 2) ? _mm_maskload_epi32((const int*)(src_row + s * 2), col_mask) - : r1; - __m128i r3 = - (bh > 3) ? _mm_maskload_epi32((const int*)(src_row + s * 3), col_mask) - : r2; - __m128i t0 = _mm_unpacklo_epi32(r0, r1); - __m128i t1 = _mm_unpacklo_epi32(r2, r3); - __m128i t2 = _mm_unpackhi_epi32(r0, r1); - __m128i t3 = _mm_unpackhi_epi32(r2, r3); - _mm_maskstore_epi32((int*)dst2, row_mask, _mm_unpacklo_epi64(t2, t3)); - _mm_maskstore_epi32((int*)dst1, row_mask, _mm_unpackhi_epi64(t0, t1)); - _mm_maskstore_epi32((int*)dst0, row_mask, _mm_unpacklo_epi64(t0, t1)); - } - src += bh * s; - dst += bh * 4; - height -= bh; - } _mm256_zeroupper(); } #endif // defined(HAS_TRANSPOSEWXH_32_AVX2) -#if defined(HAS_TRANSPOSENX4_32_AVX512BW) -// Transpose 32 bit values (ARGB) in Nx4 tiles with 4 blocks of 4x4 at a time -// in zmm0..zmm3 and masked tail for arbitrary N. -LIBYUV_TARGET_AVX512BW -void TransposeNx4_32_AVX512BW(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - const ptrdiff_t s = src_stride; - const ptrdiff_t d = dst_stride; - while (width >= 16) { - __m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src))); - __m512i r1 = - _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s))); - __m512i r2 = - _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2))); - __m512i r3 = - _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3))); - r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 4)), - 1); - r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 5)), - 1); - r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 6)), - 1); - r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 7)), - 1); - r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)), - 2); - r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)), - 2); - r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)), - 2); - r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)), - 2); - r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 12)), - 3); - r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 13)), - 3); - r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 14)), - 3); - r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 15)), - 3); - __m512i t0 = _mm512_unpacklo_epi32(r0, r1); - __m512i t1 = _mm512_unpacklo_epi32(r2, r3); - __m512i t2 = _mm512_unpackhi_epi32(r0, r1); - __m512i t3 = _mm512_unpackhi_epi32(r2, r3); - _mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(t0, t1)); - _mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(t0, t1)); - _mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(t2, t3)); - _mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(t2, t3)); - src += s * 16; - dst += 64; - width -= 16; - } - while (width >= 4) { - __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); - __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); - __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); - __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); - __m128i t0 = _mm_unpacklo_epi32(r0, r1); - __m128i t1 = _mm_unpacklo_epi32(r2, r3); - __m128i t2 = _mm_unpackhi_epi32(r0, r1); - __m128i t3 = _mm_unpackhi_epi32(r2, r3); - _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); - _mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1)); - _mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3)); - _mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3)); - src += s * 4; - dst += 16; - width -= 4; - } - if (width > 0) { - const __mmask8 mask = (1u << width) - 1; - __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); - __m128i r1 = (width > 1) ? _mm_loadu_si128((const __m128i*)(src + s)) : r0; - __m128i r2 = - (width > 2) ? _mm_loadu_si128((const __m128i*)(src + s * 2)) : r1; - __m128i t0 = _mm_unpacklo_epi32(r0, r1); - __m128i t1 = _mm_unpacklo_epi32(r2, r2); - __m128i t2 = _mm_unpackhi_epi32(r0, r1); - __m128i t3 = _mm_unpackhi_epi32(r2, r2); - _mm_mask_storeu_epi32(dst, mask, _mm_unpacklo_epi64(t0, t1)); - _mm_mask_storeu_epi32(dst + d, mask, _mm_unpackhi_epi64(t0, t1)); - _mm_mask_storeu_epi32(dst + d * 2, mask, _mm_unpacklo_epi64(t2, t3)); - _mm_mask_storeu_epi32(dst + d * 3, mask, _mm_unpackhi_epi64(t2, t3)); - } - _mm256_zeroupper(); -} -#endif // defined(HAS_TRANSPOSENX4_32_AVX512BW) - #if defined(HAS_TRANSPOSEWXH_32_AVX512BW) LIBYUV_TARGET_AVX512BW void TransposeWxH_32_AVX512BW(const uint8_t* src, @@ -769,142 +912,79 @@ void TransposeWxH_32_AVX512BW(const uint8_t* src, uint8_t* dst_col = dst; int w = width; while (w > 0) { - const int bw = (w >= 16) ? 16 : w; - const __mmask16 col_mask = (1u << bw) - 1; - __m512i z = _mm512_setzero_si512(); - __m512i r0 = _mm512_maskz_loadu_epi32(col_mask, src_row); - __m512i r1 = - (bh > 1) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s) : z; - __m512i r2 = - (bh > 2) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 2) : z; - __m512i r3 = - (bh > 3) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 3) : z; - __m512i r4 = - (bh > 4) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 4) : z; - __m512i r5 = - (bh > 5) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 5) : z; - __m512i r6 = - (bh > 6) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 6) : z; - __m512i r7 = - (bh > 7) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 7) : z; - __m512i r8 = - (bh > 8) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 8) : z; - __m512i r9 = - (bh > 9) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 9) : z; - __m512i r10 = - (bh > 10) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 10) : z; - __m512i r11 = - (bh > 11) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 11) : z; - __m512i r12 = - (bh > 12) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 12) : z; - __m512i r13 = - (bh > 13) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 13) : z; - __m512i r14 = - (bh > 14) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 14) : z; - __m512i r15 = - (bh > 15) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 15) : z; + const int bw = (w >= 4) ? 4 : w; + const __mmask8 col_mask = (1u << bw) - 1; + const __m128i z = _mm_setzero_si128(); + __m512i r0 = + _mm512_castsi128_si512(_mm_maskz_loadu_epi32(col_mask, src_row)); + __m512i r1 = _mm512_castsi128_si512( + (bh > 1) ? _mm_maskz_loadu_epi32(col_mask, src_row + s) : z); + __m512i r2 = _mm512_castsi128_si512( + (bh > 2) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 2) : z); + __m512i r3 = _mm512_castsi128_si512( + (bh > 3) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 3) : z); + + r0 = _mm512_inserti32x4( + r0, (bh > 4) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 4) : z, + 1); + r1 = _mm512_inserti32x4( + r1, (bh > 5) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 5) : z, + 1); + r2 = _mm512_inserti32x4( + r2, (bh > 6) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 6) : z, + 1); + r3 = _mm512_inserti32x4( + r3, (bh > 7) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 7) : z, + 1); + + r0 = _mm512_inserti32x4( + r0, (bh > 8) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 8) : z, + 2); + r1 = _mm512_inserti32x4( + r1, (bh > 9) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 9) : z, + 2); + r2 = _mm512_inserti32x4( + r2, (bh > 10) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 10) : z, + 2); + r3 = _mm512_inserti32x4( + r3, (bh > 11) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 11) : z, + 2); + + r0 = _mm512_inserti32x4( + r0, (bh > 12) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 12) : z, + 3); + r1 = _mm512_inserti32x4( + r1, (bh > 13) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 13) : z, + 3); + r2 = _mm512_inserti32x4( + r2, (bh > 14) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 14) : z, + 3); + r3 = _mm512_inserti32x4( + r3, (bh > 15) ? _mm_maskz_loadu_epi32(col_mask, src_row + s * 15) : z, + 3); __m512i t0 = _mm512_unpacklo_epi32(r0, r1); - __m512i t1 = _mm512_unpackhi_epi32(r0, r1); - __m512i t2 = _mm512_unpacklo_epi32(r2, r3); + __m512i t1 = _mm512_unpacklo_epi32(r2, r3); + __m512i t2 = _mm512_unpackhi_epi32(r0, r1); __m512i t3 = _mm512_unpackhi_epi32(r2, r3); - __m512i t4 = _mm512_unpacklo_epi32(r4, r5); - __m512i t5 = _mm512_unpackhi_epi32(r4, r5); - __m512i t6 = _mm512_unpacklo_epi32(r6, r7); - __m512i t7 = _mm512_unpackhi_epi32(r6, r7); - __m512i t8 = _mm512_unpacklo_epi32(r8, r9); - __m512i t9 = _mm512_unpackhi_epi32(r8, r9); - __m512i t10 = _mm512_unpacklo_epi32(r10, r11); - __m512i t11 = _mm512_unpackhi_epi32(r10, r11); - __m512i t12 = _mm512_unpacklo_epi32(r12, r13); - __m512i t13 = _mm512_unpackhi_epi32(r12, r13); - __m512i t14 = _mm512_unpacklo_epi32(r14, r15); - __m512i t15 = _mm512_unpackhi_epi32(r14, r15); - - r0 = _mm512_unpacklo_epi64(t0, t2); - r1 = _mm512_unpackhi_epi64(t0, t2); - r2 = _mm512_unpacklo_epi64(t1, t3); - r3 = _mm512_unpackhi_epi64(t1, t3); - r4 = _mm512_unpacklo_epi64(t4, t6); - r5 = _mm512_unpackhi_epi64(t4, t6); - r6 = _mm512_unpacklo_epi64(t5, t7); - r7 = _mm512_unpackhi_epi64(t5, t7); - r8 = _mm512_unpacklo_epi64(t8, t10); - r9 = _mm512_unpackhi_epi64(t8, t10); - r10 = _mm512_unpacklo_epi64(t9, t11); - r11 = _mm512_unpackhi_epi64(t9, t11); - r12 = _mm512_unpacklo_epi64(t12, t14); - r13 = _mm512_unpackhi_epi64(t12, t14); - r14 = _mm512_unpacklo_epi64(t13, t15); - r15 = _mm512_unpackhi_epi64(t13, t15); - - t0 = _mm512_shuffle_i32x4(r0, r4, 0x88); - t1 = _mm512_shuffle_i32x4(r1, r5, 0x88); - t2 = _mm512_shuffle_i32x4(r2, r6, 0x88); - t3 = _mm512_shuffle_i32x4(r3, r7, 0x88); - t4 = _mm512_shuffle_i32x4(r0, r4, 0xdd); - t5 = _mm512_shuffle_i32x4(r1, r5, 0xdd); - t6 = _mm512_shuffle_i32x4(r2, r6, 0xdd); - t7 = _mm512_shuffle_i32x4(r3, r7, 0xdd); - t8 = _mm512_shuffle_i32x4(r8, r12, 0x88); - t9 = _mm512_shuffle_i32x4(r9, r13, 0x88); - t10 = _mm512_shuffle_i32x4(r10, r14, 0x88); - t11 = _mm512_shuffle_i32x4(r11, r15, 0x88); - t12 = _mm512_shuffle_i32x4(r8, r12, 0xdd); - t13 = _mm512_shuffle_i32x4(r9, r13, 0xdd); - t14 = _mm512_shuffle_i32x4(r10, r14, 0xdd); - t15 = _mm512_shuffle_i32x4(r11, r15, 0xdd); _mm512_mask_storeu_epi32(dst_col, row_mask, - _mm512_shuffle_i32x4(t0, t8, 0x88)); - if (bw > 1) + _mm512_unpacklo_epi64(t0, t1)); + if (bw > 1) { _mm512_mask_storeu_epi32(dst_col + d, row_mask, - _mm512_shuffle_i32x4(t1, t9, 0x88)); - if (bw > 2) + _mm512_unpackhi_epi64(t0, t1)); + } + if (bw > 2) { _mm512_mask_storeu_epi32(dst_col + d * 2, row_mask, - _mm512_shuffle_i32x4(t2, t10, 0x88)); - if (bw > 3) + _mm512_unpacklo_epi64(t2, t3)); + } + if (bw > 3) { _mm512_mask_storeu_epi32(dst_col + d * 3, row_mask, - _mm512_shuffle_i32x4(t3, t11, 0x88)); - if (bw > 4) - _mm512_mask_storeu_epi32(dst_col + d * 4, row_mask, - _mm512_shuffle_i32x4(t4, t12, 0x88)); - if (bw > 5) - _mm512_mask_storeu_epi32(dst_col + d * 5, row_mask, - _mm512_shuffle_i32x4(t5, t13, 0x88)); - if (bw > 6) - _mm512_mask_storeu_epi32(dst_col + d * 6, row_mask, - _mm512_shuffle_i32x4(t6, t14, 0x88)); - if (bw > 7) - _mm512_mask_storeu_epi32(dst_col + d * 7, row_mask, - _mm512_shuffle_i32x4(t7, t15, 0x88)); - if (bw > 8) - _mm512_mask_storeu_epi32(dst_col + d * 8, row_mask, - _mm512_shuffle_i32x4(t0, t8, 0xdd)); - if (bw > 9) - _mm512_mask_storeu_epi32(dst_col + d * 9, row_mask, - _mm512_shuffle_i32x4(t1, t9, 0xdd)); - if (bw > 10) - _mm512_mask_storeu_epi32(dst_col + d * 10, row_mask, - _mm512_shuffle_i32x4(t2, t10, 0xdd)); - if (bw > 11) - _mm512_mask_storeu_epi32(dst_col + d * 11, row_mask, - _mm512_shuffle_i32x4(t3, t11, 0xdd)); - if (bw > 12) - _mm512_mask_storeu_epi32(dst_col + d * 12, row_mask, - _mm512_shuffle_i32x4(t4, t12, 0xdd)); - if (bw > 13) - _mm512_mask_storeu_epi32(dst_col + d * 13, row_mask, - _mm512_shuffle_i32x4(t5, t13, 0xdd)); - if (bw > 14) - _mm512_mask_storeu_epi32(dst_col + d * 14, row_mask, - _mm512_shuffle_i32x4(t6, t14, 0xdd)); - if (bw > 15) - _mm512_mask_storeu_epi32(dst_col + d * 15, row_mask, - _mm512_shuffle_i32x4(t7, t15, 0xdd)); - src_row += 64; - dst_col += 16 * d; - w -= 16; + _mm512_unpackhi_epi64(t2, t3)); + } + src_row += 16; + dst_col += 4 * d; + w -= 4; } src += 16 * s; dst += 64; diff --git a/source/row_any.cc b/source/row_any.cc index 2bd2957c5..c12ad5e10 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -2088,9 +2088,15 @@ ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 31) #ifdef HAS_MIRRORROW_LASX ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 63) #endif +#ifdef HAS_MIRRORUVROW_AVX512BW +ANY11M(MirrorUVRow_Any_AVX512BW, MirrorUVRow_AVX512BW, 2, 31) +#endif #ifdef HAS_MIRRORUVROW_AVX2 ANY11M(MirrorUVRow_Any_AVX2, MirrorUVRow_AVX2, 2, 15) #endif +#ifdef HAS_MIRRORUVROW_SSSE3 +ANY11M(MirrorUVRow_Any_SSSE3, MirrorUVRow_SSSE3, 2, 7) +#endif #ifdef HAS_MIRRORUVROW_NEON ANY11M(MirrorUVRow_Any_NEON, MirrorUVRow_NEON, 2, 31) #endif diff --git a/source/row_gcc.cc b/source/row_gcc.cc index a4b959a50..52ec255e4 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -5357,6 +5357,31 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { } #endif // HAS_MIRRORUVROW_AVX2 +#ifdef HAS_MIRRORUVROW_AVX512BW +static const uint16_t kMirrorUVPermute_AVX512[32] = { + 31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u, + 20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u, + 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u}; +void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + ptrdiff_t temp_width = (ptrdiff_t)(width); + asm volatile("vmovdqu16 %3,%%zmm5 \n" + + LABELALIGN + "1: \n" + "vpermw -0x40(%0,%2,2),%%zmm5,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(temp_width) // %2 + : "m"(kMirrorUVPermute_AVX512) // %3 + : "memory", "cc", "xmm0", "xmm5"); +} +#endif // HAS_MIRRORUVROW_AVX512BW + #ifdef HAS_RGB24MIRRORROW_SSSE3 // Shuffle first 5 pixels to last 5 mirrored. first byte zero diff --git a/source/row_rvv.cc b/source/row_rvv.cc index a22f239bc..d7898ea18 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -2628,6 +2628,53 @@ void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBMIRRORROW_RVV +#ifdef HAS_MIRRORUVROW_RVV +void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + int vl; + ptrdiff_t byte_vl; + src_uv += (ptrdiff_t)width * 2; + asm volatile( + "vsetvli %[vl], %[w], e16, m8, ta, ma \n" + "slli %[byte_vl], %[vl], 1 \n" + "vid.v v16 \n" + "addi %[vl], %[vl], -1 \n" + "vrsub.vx v16, v16, %[vl] \n" + "addi %[vl], %[vl], 1 \n" + "sub %[w], %[w], %[vl] \n" + "bltz %[w], 2f \n" + "1: \n" + "sub %[src], %[src], %[byte_vl] \n" + "vle16.v v8, (%[src]) \n" + "vrgather.vv v0, v8, v16 \n" + "vse16.v v0, (%[dst]) \n" + "add %[dst], %[dst], %[byte_vl] \n" + "sub %[w], %[w], %[vl] \n" + "bgez %[w], 1b \n" + "2: \n" + "add %[w], %[w], %[vl] \n" + "beqz %[w], 3f \n" + "vsetvli %[vl], %[w], e16, m8, ta, ma \n" + "slli %[byte_vl], %[vl], 1 \n" + "vid.v v16 \n" + "addi %[vl], %[vl], -1 \n" + "vrsub.vx v16, v16, %[vl] \n" + "sub %[src], %[src], %[byte_vl] \n" + "vle16.v v8, (%[src]) \n" + "vrgather.vv v0, v8, v16 \n" + "vse16.v v0, (%[dst]) \n" + "3: \n" + : [src] "+r"(src_uv), // %[src] + [dst] "+r"(dst_uv), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [byte_vl] "=&r"(byte_vl) // %[byte_vl] + : + : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", + "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} +#endif // HAS_MIRRORUVROW_RVV + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_win.cc b/source/row_win.cc index 077b4e077..24d4d888f 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -915,6 +915,25 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { } #endif // HAS_MIRRORUVROW_AVX2 +#ifdef HAS_MIRRORUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + const __m512i zmm_idx = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + src_uv += width * 2; + while (width > 0) { + src_uv -= 64; + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_uv); + zmm0 = _mm512_permutexvar_epi16(zmm_idx, zmm0); + _mm512_storeu_si512((__m512i*)dst_uv, zmm0); + dst_uv += 64; + width -= 32; + } + _mm256_zeroupper(); +} +#endif // HAS_MIRRORUVROW_AVX512BW + #ifdef HAS_SWAPUVROW_AVX2 LIBYUV_TARGET_AVX2 void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) { diff --git a/unit_test/rotate_argb_test.cc b/unit_test/rotate_argb_test.cc index af7333512..902aa8129 100644 --- a/unit_test/rotate_argb_test.cc +++ b/unit_test/rotate_argb_test.cc @@ -12,6 +12,7 @@ #include "../unit_test/unit_test.h" #include "libyuv/cpu_id.h" +#include "libyuv/planar_functions.h" #include "libyuv/rotate_argb.h" namespace libyuv { @@ -263,15 +264,15 @@ static void TestRotatePlane_16(int src_width, int src_plane_size = src_stride * abs(src_height); align_buffer_page_end_16(src, src_plane_size); for (int i = 0; i < src_plane_size; ++i) { - src[i] = fastrand() & 0xff; + src[i] = fastrand() & 0xffff; } int dst_stride = dst_width; int dst_plane_size = dst_stride * dst_height; align_buffer_page_end_16(dst_c, dst_plane_size); align_buffer_page_end_16(dst_opt, dst_plane_size); - memset(dst_c, 2, dst_plane_size); - memset(dst_opt, 3, dst_plane_size); + memset(dst_c, 2, dst_plane_size * 2); + memset(dst_opt, 3, dst_plane_size * 2); MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization. RotatePlane_16(src, src_stride, dst_c, dst_stride, src_width, src_height, @@ -345,4 +346,96 @@ TEST_F(LibYUVRotateTest, RotatePlane270_16_Odd) { benchmark_cpu_info_); } +static void TestRotatePlane16_SplitMergeUV(int src_width, + int src_height, + int dst_width, + int dst_height, + libyuv::RotationMode mode, + int benchmark_cpu_info) { + if (src_width < 1) { + src_width = 1; + } + if (src_height < 1) { + src_height = 1; + } + if (dst_width < 1) { + dst_width = 1; + } + if (dst_height < 1) { + dst_height = 1; + } + const int src_pixels = src_width * src_height; + const int dst_pixels = dst_width * dst_height; + align_buffer_page_end(src_16, src_pixels * 2); + align_buffer_page_end(dst_16, dst_pixels * 2); + align_buffer_page_end(ref_16, dst_pixels * 2); + align_buffer_page_end(src_lo, src_pixels); + align_buffer_page_end(src_hi, src_pixels); + align_buffer_page_end(dst_lo, dst_pixels); + align_buffer_page_end(dst_hi, dst_pixels); + for (int i = 0; i < src_pixels * 2; ++i) { + src_16[i] = fastrand() & 0xff; + } + memset(dst_16, 2, dst_pixels * 2); + memset(ref_16, 3, dst_pixels * 2); + + MaskCpuFlags(benchmark_cpu_info); + SplitUVPlane(src_16, src_width * 2, src_lo, src_width, src_hi, src_width, + src_width, src_height); + RotatePlane(src_lo, src_width, dst_lo, dst_width, src_width, src_height, + mode); + RotatePlane(src_hi, src_width, dst_hi, dst_width, src_width, src_height, + mode); + MergeUVPlane(dst_lo, dst_width, dst_hi, dst_width, ref_16, dst_width * 2, + dst_width, dst_height); + + I410Rotate(reinterpret_cast(src_16), src_width, + reinterpret_cast(src_16), src_width, + reinterpret_cast(src_16), src_width, + reinterpret_cast(dst_16), dst_width, + reinterpret_cast(dst_16), dst_width, + reinterpret_cast(dst_16), dst_width, src_width, + src_height, mode); + + EXPECT_EQ(0, memcmp(ref_16, dst_16, dst_pixels * 2)); + + free_aligned_buffer_page_end(dst_hi); + free_aligned_buffer_page_end(dst_lo); + free_aligned_buffer_page_end(src_hi); + free_aligned_buffer_page_end(src_lo); + free_aligned_buffer_page_end(ref_16); + free_aligned_buffer_page_end(dst_16); + free_aligned_buffer_page_end(src_16); +} + +TEST_F(LibYUVRotateTest, RotatePlane16_SplitMergeUV_Opt) { + TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_, + benchmark_width_, benchmark_height_, kRotate0, + benchmark_cpu_info_); + TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_, + benchmark_height_, benchmark_width_, kRotate90, + benchmark_cpu_info_); + TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_, + benchmark_width_, benchmark_height_, + kRotate180, benchmark_cpu_info_); + TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_, + benchmark_height_, benchmark_width_, + kRotate270, benchmark_cpu_info_); +} + +TEST_F(LibYUVRotateTest, RotatePlane16_SplitMergeUV_Odd) { + TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5, + benchmark_width_ + 3, benchmark_height_ + 5, + kRotate0, benchmark_cpu_info_); + TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5, + benchmark_height_ + 5, benchmark_width_ + 3, + kRotate90, benchmark_cpu_info_); + TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5, + benchmark_width_ + 3, benchmark_height_ + 5, + kRotate180, benchmark_cpu_info_); + TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5, + benchmark_height_ + 5, benchmark_width_ + 3, + kRotate270, benchmark_cpu_info_); +} + } // namespace libyuv