diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index b50e5fd3c..132e95aa2 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -31,6 +31,8 @@ extern "C" { defined(LIBYUV_ENABLE_ROWWIN)))) #define HAS_TRANSPOSEWX8_SSSE3 #define HAS_TRANSPOSEUVWX8_SSE2 +#define HAS_TRANSPOSEWX8_16_SSE2 +#define HAS_TRANSPOSEWX16_16_AVX2 #define HAS_TRANSPOSE4X4_32_SSE2 #define HAS_TRANSPOSENX4_32_AVX2 #define HAS_TRANSPOSEWXH_32_AVX2 @@ -39,6 +41,7 @@ extern "C" { #if defined(__x86_64__) || defined(_M_X64) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW +#define HAS_TRANSPOSEWX32_16_AVX512BW #define HAS_TRANSPOSENX4_32_AVX512BW #define HAS_TRANSPOSEWXH_32_AVX512BW #endif @@ -50,6 +53,8 @@ extern "C" { !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_TRANSPOSEWX8_SSSE3 #define HAS_TRANSPOSEUVWX8_SSE2 +#define HAS_TRANSPOSEWX8_16_SSE2 +#define HAS_TRANSPOSEWX16_16_AVX2 #define HAS_TRANSPOSE4X4_32_SSE2 #define HAS_TRANSPOSENX4_32_AVX2 #define HAS_TRANSPOSEWXH_32_AVX2 @@ -62,6 +67,7 @@ extern "C" { defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW +#define HAS_TRANSPOSEWX32_16_AVX512BW #define HAS_TRANSPOSENX4_32_AVX512BW #define HAS_TRANSPOSEWXH_32_AVX512BW #endif @@ -74,11 +80,13 @@ extern "C" { #define HAS_TRANSPOSEWX8_NEON #endif #define HAS_TRANSPOSEUVWX8_NEON +#define HAS_TRANSPOSEWX8_16_NEON #define HAS_TRANSPOSENX4_32_NEON #define HAS_TRANSPOSEWXH_32_NEON #endif #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) +#define HAS_TRANSPOSEWX8_16_SVE2 #define HAS_TRANSPOSENX4_32_SVE2 #endif @@ -86,20 +94,24 @@ extern "C" { defined(__aarch64__) #define HAS_TRANSPOSEWXH_SME #define HAS_TRANSPOSEUVWXH_SME +#define HAS_TRANSPOSEWXH_16_SME #define HAS_TRANSPOSEWXH_32_SME #endif #if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) #define HAS_TRANSPOSEWX16_LSX #define HAS_TRANSPOSEUVWX16_LSX +#define HAS_TRANSPOSEWX8_16_LSX #define HAS_TRANSPOSE4X4_32_LSX #endif #if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#define HAS_TRANSPOSEWX16_16_LASX #define HAS_TRANSPOSE4X4_32_LASX #endif #if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) +#define HAS_TRANSPOSEWXH_16_RVV #define HAS_TRANSPOSENX4_32_RVV #define HAS_TRANSPOSEWXH_32_RVV #endif @@ -109,6 +121,7 @@ extern "C" { #define HAS_TRANSPOSEUVWX8_WASMSIMD #define HAS_TRANSPOSEWX16_WASMSIMD #define HAS_TRANSPOSEUVWX16_WASMSIMD +#define HAS_TRANSPOSEWX8_16_WASMSIMD #define HAS_TRANSPOSE4X4_32_WASMSIMD #endif @@ -352,6 +365,93 @@ void TransposeWx1_16_C(const uint16_t* src, uint16_t* dst, int dst_stride, int width); +void TransposeWx8_16_SSE2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx16_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx32_16_AVX512BW(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_NEON(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_SVE2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWxH_16_SME(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height); +void TransposeWxH_16_RVV(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height); +void TransposeWx8_16_LSX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx16_16_LASX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_WASMSIMD(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_Any_SSE2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx16_16_Any_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx32_16_Any_AVX512BW(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_Any_NEON(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_Any_LSX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx16_16_Any_LASX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); +void TransposeWx8_16_Any_WASMSIMD(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); void TransposeWxH_32_C(const uint8_t* src, int src_stride, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index e70570758..dd9a6e108 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -412,6 +412,9 @@ extern "C" { #define HAS_INTERPOLATEROW_16_AVX2 #define HAS_MULTIPLYROW_16_AVX2 #define HAS_SWAPUVROW_AVX2 +#if defined(__x86_64__) || defined(_M_X64) +#define HAS_MIRRORUVROW_AVX512BW +#endif #endif // The following are available for AVX512 clang x64 platforms: @@ -452,6 +455,7 @@ extern "C" { #define HAS_J400TOARGBROW_AVX512BW #define HAS_MERGEUVROW_AVX512BW #define HAS_MIRRORROW_AVX512BW +#define HAS_MIRRORUVROW_AVX512BW #define HAS_MIRRORSPLITUVROW_AVX512BW #define HAS_SPLITUVROW_AVX512BW #define HAS_RGBTOUVMATRIXROW_AVX512BW @@ -930,6 +934,7 @@ extern "C" { #define HAS_MERGERGBROW_RVV #define HAS_MERGEUVROW_RVV #define HAS_MERGEXRGBROW_RVV +#define HAS_MIRRORUVROW_RVV #define HAS_NV12TOARGBROW_RVV #define HAS_NV12TORGB24ROW_RVV #define HAS_NV21TOARGBROW_RVV @@ -2749,13 +2754,19 @@ void MirrorRow_Any_SSE2(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width); +void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_C(const uint8_t* src_uv, uint8_t* dst_uv, int width); +void MirrorUVRow_Any_AVX512BW(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void MirrorUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void MirrorUVRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); diff --git a/source/planar_functions.cc b/source/planar_functions.cc index f3ee9e3f5..9f1a10062 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -2785,6 +2785,7 @@ void MirrorUVPlane(const uint8_t* src_uv, #endif #if defined(HAS_MIRRORUVROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { + MirrorUVRow = MirrorUVRow_Any_SSSE3; if (IS_ALIGNED(width, 8)) { MirrorUVRow = MirrorUVRow_SSSE3; } @@ -2798,6 +2799,14 @@ void MirrorUVPlane(const uint8_t* src_uv, } } #endif +#if defined(HAS_MIRRORUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MirrorUVRow = MirrorUVRow_Any_AVX512BW; + if (IS_ALIGNED(width, 32)) { + MirrorUVRow = MirrorUVRow_AVX512BW; + } + } +#endif #if defined(HAS_MIRRORUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { MirrorUVRow = MirrorUVRow_Any_LSX; @@ -2814,6 +2823,11 @@ void MirrorUVPlane(const uint8_t* src_uv, } } #endif +#if defined(HAS_MIRRORUVROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + MirrorUVRow = MirrorUVRow_RVV; + } +#endif #if defined(HAS_MIRRORUVROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { MirrorUVRow = MirrorUVRow_Any_WASMSIMD; diff --git a/source/rotate.cc b/source/rotate.cc index e8844f1b7..eab8d26ee 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -618,11 +618,124 @@ static void TransposePlane_16(const uint16_t* src, int width, int height) { int i = height; +#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) + void (*TransposeWxH_16)(const uint16_t* src, int src_stride, uint16_t* dst, + int dst_stride, int width, int height) = NULL; +#endif +#if defined(HAS_TRANSPOSEWX32_16_AVX512BW) + void (*TransposeWx32_16)(const uint16_t* src, int src_stride, uint16_t* dst, + int dst_stride, int width) = NULL; +#endif +#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX) + void (*TransposeWx16_16)(const uint16_t* src, int src_stride, uint16_t* dst, + int dst_stride, int width) = NULL; +#endif + void (*TransposeWx8_16)(const uint16_t* src, int src_stride, uint16_t* dst, + int dst_stride, int width) = TransposeWx8_16_C; + +#if defined(HAS_TRANSPOSEWX8_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + TransposeWx8_16 = TransposeWx8_16_Any_NEON; + if (IS_ALIGNED(width, 8)) { + TransposeWx8_16 = TransposeWx8_16_NEON; + } + } +#endif +#if defined(HAS_TRANSPOSEWX8_16_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + TransposeWx8_16 = TransposeWx8_16_SVE2; + } +#endif +#if defined(HAS_TRANSPOSEWXH_16_SME) + if (TestCpuFlag(kCpuHasSME)) { + TransposeWxH_16 = TransposeWxH_16_SME; + } +#endif +#if defined(HAS_TRANSPOSEWXH_16_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + TransposeWxH_16 = TransposeWxH_16_RVV; + } +#endif +#if defined(HAS_TRANSPOSEWX8_16_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + TransposeWx8_16 = TransposeWx8_16_Any_SSE2; + if (IS_ALIGNED(width, 8)) { + TransposeWx8_16 = TransposeWx8_16_SSE2; + } + } +#endif +#if defined(HAS_TRANSPOSEWX16_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx16_16 = TransposeWx16_16_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + TransposeWx16_16 = TransposeWx16_16_AVX2; + } + } +#endif +#if defined(HAS_TRANSPOSEWX32_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + TransposeWx32_16 = TransposeWx32_16_Any_AVX512BW; + if (IS_ALIGNED(width, 8)) { + TransposeWx32_16 = TransposeWx32_16_AVX512BW; + } + } +#endif +#if defined(HAS_TRANSPOSEWX8_16_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + TransposeWx8_16 = TransposeWx8_16_Any_LSX; + if (IS_ALIGNED(width, 8)) { + TransposeWx8_16 = TransposeWx8_16_LSX; + } + } +#endif +#if defined(HAS_TRANSPOSEWX16_16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + TransposeWx16_16 = TransposeWx16_16_Any_LASX; + if (IS_ALIGNED(width, 8)) { + TransposeWx16_16 = TransposeWx16_16_LASX; + } + } +#endif +#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + TransposeWx8_16 = TransposeWx8_16_Any_WASMSIMD; + if (IS_ALIGNED(width, 8)) { + TransposeWx8_16 = TransposeWx8_16_WASMSIMD; + } + } +#endif + +#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) + if (TransposeWxH_16) { + TransposeWxH_16(src, src_stride, dst, dst_stride, width, height); + return; + } +#endif +#if defined(HAS_TRANSPOSEWX32_16_AVX512BW) + if (TransposeWx32_16) { + while (i >= 32) { + TransposeWx32_16(src, src_stride, dst, dst_stride, width); + src += 32 * (ptrdiff_t)src_stride; + dst += 32; + i -= 32; + } + } +#endif +#if defined(HAS_TRANSPOSEWX16_16_AVX2) || defined(HAS_TRANSPOSEWX16_16_LASX) + if (TransposeWx16_16) { + while (i >= 16) { + TransposeWx16_16(src, src_stride, dst, dst_stride, width); + src += 16 * (ptrdiff_t)src_stride; + dst += 16; + i -= 16; + } + } +#endif // Work across the source in 8x8 tiles while (i >= 8) { - TransposeWx8_16_C(src, src_stride, dst, dst_stride, width); - src += 8 * src_stride; // Go down 8 rows. - dst += 8; // Move over 8 columns. + TransposeWx8_16(src, src_stride, dst, dst_stride, width); + src += 8 * (ptrdiff_t)src_stride; // Go down 8 rows. + dst += 8; // Move over 8 columns. i -= 8; } @@ -665,29 +778,9 @@ static void RotatePlane180_16(const uint16_t* src, int dst_stride, int width, int height) { - const uint16_t* src_bot = src + (ptrdiff_t)src_stride * (height - 1); - uint16_t* dst_bot = dst + (ptrdiff_t)dst_stride * (height - 1); - int half_height = (height + 1) >> 1; - int y; - - // Swap top and bottom row and mirror the content. Uses a temporary row. - align_buffer_64(row, width * 2); - uint16_t* row_tmp = (uint16_t*)row; - assert(row); - if (!row) - return; - - // Odd height will harmlessly mirror the middle row twice. - for (y = 0; y < half_height; ++y) { - CopyRow_16_C(src, row_tmp, width); // Copy top row into buffer - MirrorRow_16_C(src_bot, dst, width); // Mirror bottom row into top row - MirrorRow_16_C(row_tmp, dst_bot, width); // Mirror buffer into bottom row - src += src_stride; - dst += dst_stride; - src_bot -= src_stride; - dst_bot -= dst_stride; - } - free_aligned_buffer_64(row); + // Rotate by 180 is a vertical flip (negative height) and horizontal mirror. + MirrorUVPlane((const uint8_t*)src, src_stride * 2, (uint8_t*)dst, + dst_stride * 2, width, -height); } LIBYUV_API diff --git a/source/rotate_any.cc b/source/rotate_any.cc index c474bc262..52d5a243f 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -122,6 +122,42 @@ TANY32(Transpose4x4_32_Any_WASMSIMD, #endif #undef TANY32 +#define TANY16(NAMEANY, TPOS_SIMD, MASK, HEIGHT) \ + void NAMEANY(const uint16_t* src, int src_stride, uint16_t* dst, \ + int dst_stride, int width) { \ + int r = width & MASK; \ + int n = width - r; \ + if (n > 0) { \ + TPOS_SIMD(src, src_stride, dst, dst_stride, n); \ + } \ + ptrdiff_t np = n; \ + TransposeWxH_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \ + r, HEIGHT); \ + } + +#ifdef HAS_TRANSPOSEWX8_16_NEON +TANY16(TransposeWx8_16_Any_NEON, TransposeWx8_16_NEON, 7, 8) +#endif +#ifdef HAS_TRANSPOSEWX8_16_SSE2 +TANY16(TransposeWx8_16_Any_SSE2, TransposeWx8_16_SSE2, 7, 8) +#endif +#ifdef HAS_TRANSPOSEWX16_16_AVX2 +TANY16(TransposeWx16_16_Any_AVX2, TransposeWx16_16_AVX2, 7, 16) +#endif +#ifdef HAS_TRANSPOSEWX32_16_AVX512BW +TANY16(TransposeWx32_16_Any_AVX512BW, TransposeWx32_16_AVX512BW, 7, 32) +#endif +#ifdef HAS_TRANSPOSEWX8_16_LSX +TANY16(TransposeWx8_16_Any_LSX, TransposeWx8_16_LSX, 7, 8) +#endif +#ifdef HAS_TRANSPOSEWX16_16_LASX +TANY16(TransposeWx16_16_Any_LASX, TransposeWx16_16_LASX, 7, 16) +#endif +#ifdef HAS_TRANSPOSEWX8_16_WASMSIMD +TANY16(TransposeWx8_16_Any_WASMSIMD, TransposeWx8_16_WASMSIMD, 7, 8) +#endif +#undef TANY16 + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index a66697403..116f05b6b 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -664,6 +664,330 @@ void TransposeUVWx8_SSE2(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEUVWX8_SSE2) +#if defined(HAS_TRANSPOSEWX8_16_SSE2) +void TransposeWx8_16_SSE2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* temp; + asm volatile( + "1: \n" + "movdqu (%[src]),%%xmm0 \n" + "movdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "movdqu (%[temp]),%%xmm2 \n" + "movdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "movdqu (%[temp]),%%xmm4 \n" + "movdqu (%[temp],%[s]),%%xmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "movdqu (%[temp]),%%xmm6 \n" + "movdqu (%[temp],%[s]),%%xmm7 \n" + "add $0x10,%[src] \n" + + // Stage 1: 16-bit unpack -> xmm8..xmm15 + "movdqa %%xmm0,%%xmm8 \n" + "punpcklwd %%xmm1,%%xmm8 \n" + "movdqa %%xmm0,%%xmm9 \n" + "punpckhwd %%xmm1,%%xmm9 \n" + "movdqa %%xmm2,%%xmm10 \n" + "punpcklwd %%xmm3,%%xmm10 \n" + "movdqa %%xmm2,%%xmm11 \n" + "punpckhwd %%xmm3,%%xmm11 \n" + "movdqa %%xmm4,%%xmm12 \n" + "punpcklwd %%xmm5,%%xmm12 \n" + "movdqa %%xmm4,%%xmm13 \n" + "punpckhwd %%xmm5,%%xmm13 \n" + "movdqa %%xmm6,%%xmm14 \n" + "punpcklwd %%xmm7,%%xmm14 \n" + "movdqa %%xmm6,%%xmm15 \n" + "punpckhwd %%xmm7,%%xmm15 \n" + + // Stage 2: 32-bit unpack -> xmm0..xmm7 + "movdqa %%xmm8,%%xmm0 \n" + "punpckldq %%xmm10,%%xmm0 \n" + "movdqa %%xmm8,%%xmm1 \n" + "punpckhdq %%xmm10,%%xmm1 \n" + "movdqa %%xmm9,%%xmm2 \n" + "punpckldq %%xmm11,%%xmm2 \n" + "movdqa %%xmm9,%%xmm3 \n" + "punpckhdq %%xmm11,%%xmm3 \n" + "movdqa %%xmm12,%%xmm4 \n" + "punpckldq %%xmm14,%%xmm4 \n" + "movdqa %%xmm12,%%xmm5 \n" + "punpckhdq %%xmm14,%%xmm5 \n" + "movdqa %%xmm13,%%xmm6 \n" + "punpckldq %%xmm15,%%xmm6 \n" + "movdqa %%xmm13,%%xmm7 \n" + "punpckhdq %%xmm15,%%xmm7 \n" + + // Stage 3: 64-bit unpack -> xmm8..xmm15 + "movdqa %%xmm0,%%xmm8 \n" + "punpcklqdq %%xmm4,%%xmm8 \n" + "movdqa %%xmm0,%%xmm9 \n" + "punpckhqdq %%xmm4,%%xmm9 \n" + "movdqa %%xmm1,%%xmm10 \n" + "punpcklqdq %%xmm5,%%xmm10 \n" + "movdqa %%xmm1,%%xmm11 \n" + "punpckhqdq %%xmm5,%%xmm11 \n" + "movdqa %%xmm2,%%xmm12 \n" + "punpcklqdq %%xmm6,%%xmm12 \n" + "movdqa %%xmm2,%%xmm13 \n" + "punpckhqdq %%xmm6,%%xmm13 \n" + "movdqa %%xmm3,%%xmm14 \n" + "punpcklqdq %%xmm7,%%xmm14 \n" + "movdqa %%xmm3,%%xmm15 \n" + "punpckhqdq %%xmm7,%%xmm15 \n" + + "movdqu %%xmm8,(%[dst]) \n" + "movdqu %%xmm9,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "movdqu %%xmm10,(%[dst]) \n" + "movdqu %%xmm11,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "movdqu %%xmm12,(%[dst]) \n" + "movdqu %%xmm13,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "movdqu %%xmm14,(%[dst]) \n" + "movdqu %%xmm15,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x8,%[w] \n" + "jg 1b \n" + : [src] "+r"(src_b), // %[src] + [dst] "+r"(dst_b), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", + "xmm15"); +} +#endif // defined(HAS_TRANSPOSEWX8_16_SSE2) + +#if defined(HAS_TRANSPOSEWX16_16_AVX2) +void TransposeWx16_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm4 \n" + "vmovdqu (%[temp],%[s]),%%xmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm6 \n" + "vmovdqu (%[temp],%[s]),%%xmm7 \n" + "lea (%[temp],%[s],2),%[temp] \n" + + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm4,%%ymm4 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm5,%%ymm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm6,%%ymm6 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm7,%%ymm7 \n" + "add $0x10,%[src] \n" + + // Stage 1: 16-bit unpack -> ymm8..ymm15 + "vpunpcklwd %%ymm1,%%ymm0,%%ymm8 \n" + "vpunpckhwd %%ymm1,%%ymm0,%%ymm9 \n" + "vpunpcklwd %%ymm3,%%ymm2,%%ymm10 \n" + "vpunpckhwd %%ymm3,%%ymm2,%%ymm11 \n" + "vpunpcklwd %%ymm5,%%ymm4,%%ymm12 \n" + "vpunpckhwd %%ymm5,%%ymm4,%%ymm13 \n" + "vpunpcklwd %%ymm7,%%ymm6,%%ymm14 \n" + "vpunpckhwd %%ymm7,%%ymm6,%%ymm15 \n" + + // Stage 2: 32-bit unpack -> ymm0..ymm7 + "vpunpckldq %%ymm10,%%ymm8,%%ymm0 \n" + "vpunpckhdq %%ymm10,%%ymm8,%%ymm1 \n" + "vpunpckldq %%ymm11,%%ymm9,%%ymm2 \n" + "vpunpckhdq %%ymm11,%%ymm9,%%ymm3 \n" + "vpunpckldq %%ymm14,%%ymm12,%%ymm4 \n" + "vpunpckhdq %%ymm14,%%ymm12,%%ymm5 \n" + "vpunpckldq %%ymm15,%%ymm13,%%ymm6 \n" + "vpunpckhdq %%ymm15,%%ymm13,%%ymm7 \n" + + // Stage 3: 64-bit unpack -> ymm8..ymm15 + "vpunpcklqdq %%ymm4,%%ymm0,%%ymm8 \n" + "vpunpckhqdq %%ymm4,%%ymm0,%%ymm9 \n" + "vpunpcklqdq %%ymm5,%%ymm1,%%ymm10 \n" + "vpunpckhqdq %%ymm5,%%ymm1,%%ymm11 \n" + "vpunpcklqdq %%ymm6,%%ymm2,%%ymm12 \n" + "vpunpckhqdq %%ymm6,%%ymm2,%%ymm13 \n" + "vpunpcklqdq %%ymm7,%%ymm3,%%ymm14 \n" + "vpunpckhqdq %%ymm7,%%ymm3,%%ymm15 \n" + + "vmovdqu %%ymm8,(%[dst]) \n" + "vmovdqu %%ymm9,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%ymm10,(%[dst]) \n" + "vmovdqu %%ymm11,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%ymm12,(%[dst]) \n" + "vmovdqu %%ymm13,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%ymm14,(%[dst]) \n" + "vmovdqu %%ymm15,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x8,%[w] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_b), // %[src] + [dst] "+r"(dst_b), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", + "xmm15"); +} +#endif // defined(HAS_TRANSPOSEWX16_16_AVX2) + +#if defined(HAS_TRANSPOSEWX32_16_AVX512BW) +void TransposeWx32_16_AVX512BW(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm4 \n" + "vmovdqu (%[temp],%[s]),%%xmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm6 \n" + "vmovdqu (%[temp],%[s]),%%xmm7 \n" + "lea (%[temp],%[s],2),%[temp] \n" + + "vinserti32x4 $1,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $1,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $1,(%[temp]),%%zmm4,%%zmm4 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm5,%%zmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $1,(%[temp]),%%zmm6,%%zmm6 \n" + "vinserti32x4 $1,(%[temp],%[s]),%%zmm7,%%zmm7 \n" + "lea (%[temp],%[s],2),%[temp] \n" + + "vinserti32x4 $2,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $2,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $2,(%[temp]),%%zmm4,%%zmm4 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm5,%%zmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $2,(%[temp]),%%zmm6,%%zmm6 \n" + "vinserti32x4 $2,(%[temp],%[s]),%%zmm7,%%zmm7 \n" + "lea (%[temp],%[s],2),%[temp] \n" + + "vinserti32x4 $3,(%[temp]),%%zmm0,%%zmm0 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm1,%%zmm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $3,(%[temp]),%%zmm2,%%zmm2 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm3,%%zmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $3,(%[temp]),%%zmm4,%%zmm4 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm5,%%zmm5 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti32x4 $3,(%[temp]),%%zmm6,%%zmm6 \n" + "vinserti32x4 $3,(%[temp],%[s]),%%zmm7,%%zmm7 \n" + "add $0x10,%[src] \n" + + // Stage 1: 16-bit unpack -> zmm8..zmm15 + "vpunpcklwd %%zmm1,%%zmm0,%%zmm8 \n" + "vpunpckhwd %%zmm1,%%zmm0,%%zmm9 \n" + "vpunpcklwd %%zmm3,%%zmm2,%%zmm10 \n" + "vpunpckhwd %%zmm3,%%zmm2,%%zmm11 \n" + "vpunpcklwd %%zmm5,%%zmm4,%%zmm12 \n" + "vpunpckhwd %%zmm5,%%zmm4,%%zmm13 \n" + "vpunpcklwd %%zmm7,%%zmm6,%%zmm14 \n" + "vpunpckhwd %%zmm7,%%zmm6,%%zmm15 \n" + + // Stage 2: 32-bit unpack -> zmm0..zmm7 + "vpunpckldq %%zmm10,%%zmm8,%%zmm0 \n" + "vpunpckhdq %%zmm10,%%zmm8,%%zmm1 \n" + "vpunpckldq %%zmm11,%%zmm9,%%zmm2 \n" + "vpunpckhdq %%zmm11,%%zmm9,%%zmm3 \n" + "vpunpckldq %%zmm14,%%zmm12,%%zmm4 \n" + "vpunpckhdq %%zmm14,%%zmm12,%%zmm5 \n" + "vpunpckldq %%zmm15,%%zmm13,%%zmm6 \n" + "vpunpckhdq %%zmm15,%%zmm13,%%zmm7 \n" + + // Stage 3: 64-bit unpack -> zmm8..zmm15 + "vpunpcklqdq %%zmm4,%%zmm0,%%zmm8 \n" + "vpunpckhqdq %%zmm4,%%zmm0,%%zmm9 \n" + "vpunpcklqdq %%zmm5,%%zmm1,%%zmm10 \n" + "vpunpckhqdq %%zmm5,%%zmm1,%%zmm11 \n" + "vpunpcklqdq %%zmm6,%%zmm2,%%zmm12 \n" + "vpunpckhqdq %%zmm6,%%zmm2,%%zmm13 \n" + "vpunpcklqdq %%zmm7,%%zmm3,%%zmm14 \n" + "vpunpckhqdq %%zmm7,%%zmm3,%%zmm15 \n" + + "vmovdqu16 %%zmm8,(%[dst]) \n" + "vmovdqu16 %%zmm9,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu16 %%zmm10,(%[dst]) \n" + "vmovdqu16 %%zmm11,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu16 %%zmm12,(%[dst]) \n" + "vmovdqu16 %%zmm13,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu16 %%zmm14,(%[dst]) \n" + "vmovdqu16 %%zmm15,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x8,%[w] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_b), // %[src] + [dst] "+r"(dst_b), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", + "xmm15"); +} +#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW) + #if defined(HAS_TRANSPOSE4X4_32_SSE2) // 4 values, little endian view // a b c d diff --git a/source/rotate_lasx.cc b/source/rotate_lasx.cc index b4361809e..74e54c008 100644 --- a/source/rotate_lasx.cc +++ b/source/rotate_lasx.cc @@ -95,6 +95,121 @@ void Transpose4x4_32_LASX(const uint8_t* src, "$xr7"); } +void TransposeWx16_16_LASX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + + "vld $vr8, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr9, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr10, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr11, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr12, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr13, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr14, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr15, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" + + "xvpermi.q $xr0, $xr8, 0x02 \n" + "xvpermi.q $xr1, $xr9, 0x02 \n" + "xvpermi.q $xr2, $xr10, 0x02 \n" + "xvpermi.q $xr3, $xr11, 0x02 \n" + "xvpermi.q $xr4, $xr12, 0x02 \n" + "xvpermi.q $xr5, $xr13, 0x02 \n" + "xvpermi.q $xr6, $xr14, 0x02 \n" + "xvpermi.q $xr7, $xr15, 0x02 \n" + + "xvilvl.h $xr8, $xr1, $xr0 \n" + "xvilvh.h $xr9, $xr1, $xr0 \n" + "xvilvl.h $xr10, $xr3, $xr2 \n" + "xvilvh.h $xr11, $xr3, $xr2 \n" + "xvilvl.h $xr12, $xr5, $xr4 \n" + "xvilvh.h $xr13, $xr5, $xr4 \n" + "xvilvl.h $xr14, $xr7, $xr6 \n" + "xvilvh.h $xr15, $xr7, $xr6 \n" + + "xvilvl.w $xr0, $xr10, $xr8 \n" + "xvilvh.w $xr1, $xr10, $xr8 \n" + "xvilvl.w $xr2, $xr11, $xr9 \n" + "xvilvh.w $xr3, $xr11, $xr9 \n" + "xvilvl.w $xr4, $xr14, $xr12 \n" + "xvilvh.w $xr5, $xr14, $xr12 \n" + "xvilvl.w $xr6, $xr15, $xr13 \n" + "xvilvh.w $xr7, $xr15, $xr13 \n" + + "xvilvl.d $xr8, $xr4, $xr0 \n" + "xvilvh.d $xr9, $xr4, $xr0 \n" + "xvilvl.d $xr10, $xr5, $xr1 \n" + "xvilvh.d $xr11, $xr5, $xr1 \n" + "xvilvl.d $xr12, $xr6, $xr2 \n" + "xvilvh.d $xr13, $xr6, $xr2 \n" + "xvilvl.d $xr14, $xr7, $xr3 \n" + "xvilvh.d $xr15, $xr7, $xr3 \n" + + "xvst $xr8, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr9, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr10, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr11, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr12, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr13, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr14, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "xvst $xr15, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_b), // %[src] + [dst] "+r"(dst_b), // %[dst] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15"); +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_lsx.cc b/source/rotate_lsx.cc index 9f330e32e..931b108b2 100644 --- a/source/rotate_lsx.cc +++ b/source/rotate_lsx.cc @@ -242,6 +242,57 @@ void Transpose4x4_32_LSX(const uint8_t* src, } } +void TransposeWx8_16_LSX(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + while (width > 0) { + __m128i r0 = __lsx_vld(src_b, 0); + __m128i r1 = __lsx_vldx(src_b, s); + __m128i r2 = __lsx_vldx(src_b, s * 2); + __m128i r3 = __lsx_vldx(src_b, s * 3); + __m128i r4 = __lsx_vldx(src_b, s * 4); + __m128i r5 = __lsx_vldx(src_b, s * 5); + __m128i r6 = __lsx_vldx(src_b, s * 6); + __m128i r7 = __lsx_vldx(src_b, s * 7); + + __m128i t0 = __lsx_vilvl_h(r1, r0); + __m128i t1 = __lsx_vilvh_h(r1, r0); + __m128i t2 = __lsx_vilvl_h(r3, r2); + __m128i t3 = __lsx_vilvh_h(r3, r2); + __m128i t4 = __lsx_vilvl_h(r5, r4); + __m128i t5 = __lsx_vilvh_h(r5, r4); + __m128i t6 = __lsx_vilvl_h(r7, r6); + __m128i t7 = __lsx_vilvh_h(r7, r6); + + __m128i u0 = __lsx_vilvl_w(t2, t0); + __m128i u1 = __lsx_vilvh_w(t2, t0); + __m128i u2 = __lsx_vilvl_w(t3, t1); + __m128i u3 = __lsx_vilvh_w(t3, t1); + __m128i u4 = __lsx_vilvl_w(t6, t4); + __m128i u5 = __lsx_vilvh_w(t6, t4); + __m128i u6 = __lsx_vilvl_w(t7, t5); + __m128i u7 = __lsx_vilvh_w(t7, t5); + + __lsx_vst(__lsx_vilvl_d(u4, u0), dst_b, 0); + __lsx_vstx(__lsx_vilvh_d(u4, u0), dst_b, d); + __lsx_vstx(__lsx_vilvl_d(u5, u1), dst_b, d * 2); + __lsx_vstx(__lsx_vilvh_d(u5, u1), dst_b, d * 3); + __lsx_vstx(__lsx_vilvl_d(u6, u2), dst_b, d * 4); + __lsx_vstx(__lsx_vilvh_d(u6, u2), dst_b, d * 5); + __lsx_vstx(__lsx_vilvl_d(u7, u3), dst_b, d * 6); + __lsx_vstx(__lsx_vilvh_d(u7, u3), dst_b, d * 7); + src_b += 16; + dst_b += d * 8; + width -= 8; + } +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_neon.cc b/source/rotate_neon.cc index 6106787f0..49faf1ff1 100644 --- a/source/rotate_neon.cc +++ b/source/rotate_neon.cc @@ -419,6 +419,64 @@ void TransposeWxH_32_NEON(const uint8_t* src, } } +void TransposeWx8_16_NEON(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* temp; + asm volatile( + "1: \n" + "mov %[temp], %[src] \n" + "vld1.8 {q0}, [%[temp]], %[s] \n" + "vld1.8 {q1}, [%[temp]], %[s] \n" + "vld1.8 {q2}, [%[temp]], %[s] \n" + "vld1.8 {q3}, [%[temp]], %[s] \n" + "vld1.8 {q8}, [%[temp]], %[s] \n" + "vld1.8 {q9}, [%[temp]], %[s] \n" + "vld1.8 {q10}, [%[temp]], %[s] \n" + "vld1.8 {q11}, [%[temp]] \n" + "add %[src], %[src], #16 \n" + + // Stage 1: 16-bit transpose + "vtrn.16 q0, q1 \n" + "vtrn.16 q2, q3 \n" + "vtrn.16 q8, q9 \n" + "vtrn.16 q10, q11 \n" + + // Stage 2: 32-bit transpose + "vtrn.32 q0, q2 \n" + "vtrn.32 q1, q3 \n" + "vtrn.32 q8, q10 \n" + "vtrn.32 q9, q11 \n" + + // Stage 3: 64-bit swap + "vswp d1, d16 \n" + "vswp d3, d18 \n" + "vswp d5, d20 \n" + "vswp d7, d22 \n" + + "subs %[w], %[w], #8 \n" + "vst1.8 {q0}, [%[dst]], %[d] \n" + "vst1.8 {q1}, [%[dst]], %[d] \n" + "vst1.8 {q2}, [%[dst]], %[d] \n" + "vst1.8 {q3}, [%[dst]], %[d] \n" + "vst1.8 {q8}, [%[dst]], %[d] \n" + "vst1.8 {q9}, [%[dst]], %[d] \n" + "vst1.8 {q10}, [%[dst]], %[d] \n" + "vst1.8 {q11}, [%[dst]], %[d] \n" + "bgt 1b \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11"); +} + #endif // defined(__ARM_NEON__) && !defined(__aarch64__) #ifdef __cplusplus diff --git a/source/rotate_neon64.cc b/source/rotate_neon64.cc index 3960e28d9..e751488b5 100644 --- a/source/rotate_neon64.cc +++ b/source/rotate_neon64.cc @@ -474,6 +474,77 @@ void TransposeWxH_32_NEON(const uint8_t* src, } } +void TransposeWx8_16_NEON(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint16_t* temp; + asm volatile( + "1: \n" + "mov %[temp], %[src] \n" + "ld1 {v0.8h}, [%[temp]], %[s] \n" + "ld1 {v1.8h}, [%[temp]], %[s] \n" + "ld1 {v2.8h}, [%[temp]], %[s] \n" + "ld1 {v3.8h}, [%[temp]], %[s] \n" + "ld1 {v4.8h}, [%[temp]], %[s] \n" + "ld1 {v5.8h}, [%[temp]], %[s] \n" + "ld1 {v6.8h}, [%[temp]], %[s] \n" + "ld1 {v7.8h}, [%[temp]] \n" + "add %[src], %[src], #16 \n" + + // Stage 1: 16-bit transpose + "trn1 v16.8h, v0.8h, v1.8h \n" + "trn2 v17.8h, v0.8h, v1.8h \n" + "trn1 v18.8h, v2.8h, v3.8h \n" + "trn2 v19.8h, v2.8h, v3.8h \n" + "trn1 v20.8h, v4.8h, v5.8h \n" + "trn2 v21.8h, v4.8h, v5.8h \n" + "trn1 v22.8h, v6.8h, v7.8h \n" + "trn2 v23.8h, v6.8h, v7.8h \n" + + // Stage 2: 32-bit transpose + "trn1 v0.4s, v16.4s, v18.4s \n" + "trn1 v1.4s, v17.4s, v19.4s \n" + "trn2 v2.4s, v16.4s, v18.4s \n" + "trn2 v3.4s, v17.4s, v19.4s \n" + "trn1 v4.4s, v20.4s, v22.4s \n" + "trn1 v5.4s, v21.4s, v23.4s \n" + "trn2 v6.4s, v20.4s, v22.4s \n" + "trn2 v7.4s, v21.4s, v23.4s \n" + + // Stage 3: 64-bit transpose + "trn1 v16.2d, v0.2d, v4.2d \n" + "trn1 v17.2d, v1.2d, v5.2d \n" + "trn1 v18.2d, v2.2d, v6.2d \n" + "trn1 v19.2d, v3.2d, v7.2d \n" + "trn2 v20.2d, v0.2d, v4.2d \n" + "trn2 v21.2d, v1.2d, v5.2d \n" + "trn2 v22.2d, v2.2d, v6.2d \n" + "trn2 v23.2d, v3.2d, v7.2d \n" + + "subs %w[w], %w[w], #8 \n" + "st1 {v16.8h}, [%[dst]], %[d] \n" + "st1 {v17.8h}, [%[dst]], %[d] \n" + "st1 {v18.8h}, [%[dst]], %[d] \n" + "st1 {v19.8h}, [%[dst]], %[d] \n" + "st1 {v20.8h}, [%[dst]], %[d] \n" + "st1 {v21.8h}, [%[dst]], %[d] \n" + "st1 {v22.8h}, [%[dst]], %[d] \n" + "st1 {v23.8h}, [%[dst]], %[d] \n" + "b.gt 1b \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/rotate_rvv.cc b/source/rotate_rvv.cc index 3b9e2b635..6a352e352 100644 --- a/source/rotate_rvv.cc +++ b/source/rotate_rvv.cc @@ -143,6 +143,130 @@ void TransposeWxH_32_RVV(const uint8_t* src, } #endif // HAS_TRANSPOSEWXH_32_RVV +#ifdef HAS_TRANSPOSEWXH_16_RVV +void TransposeWxH_16_RVV(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s_stride = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d_stride = (ptrdiff_t)dst_stride * 2; + while (width >= 4) { + const uint16_t* s0 = src; + const uint16_t* s1 = src + 1; + const uint16_t* s2 = src + 2; + const uint16_t* s3 = src + 3; + uint8_t* d0 = (uint8_t*)dst; + uint8_t* d1 = d0 + d_stride; + uint8_t* d2 = d1 + d_stride; + uint8_t* d3 = d2 + d_stride; + int h = height; + int vl; + ptrdiff_t src_step; + ptrdiff_t dst_step; + asm volatile( + "vsetvli %[vl], %[h], e16, m8, ta, ma \n" + "slli %[d_step], %[vl], 1 \n" + "mul %[s_step], %[vl], %[s_stride] \n" + "sub %[h], %[h], %[vl] \n" + "bltz %[h], 2f \n" + "1: \n" + "vlse16.v v0, (%[s0]), %[s_stride] \n" + "vlse16.v v8, (%[s1]), %[s_stride] \n" + "vlse16.v v16, (%[s2]), %[s_stride] \n" + "vlse16.v v24, (%[s3]), %[s_stride] \n" + "add %[s0], %[s0], %[s_step] \n" + "add %[s1], %[s1], %[s_step] \n" + "add %[s2], %[s2], %[s_step] \n" + "add %[s3], %[s3], %[s_step] \n" + "vse16.v v0, (%[d0]) \n" + "vse16.v v8, (%[d1]) \n" + "vse16.v v16, (%[d2]) \n" + "vse16.v v24, (%[d3]) \n" + "add %[d0], %[d0], %[d_step] \n" + "add %[d1], %[d1], %[d_step] \n" + "add %[d2], %[d2], %[d_step] \n" + "add %[d3], %[d3], %[d_step] \n" + "sub %[h], %[h], %[vl] \n" + "bgez %[h], 1b \n" + "2: \n" + "add %[h], %[h], %[vl] \n" + "beqz %[h], 3f \n" + "vsetvli %[vl], %[h], e16, m8, ta, ma \n" + "vlse16.v v0, (%[s0]), %[s_stride] \n" + "vlse16.v v8, (%[s1]), %[s_stride] \n" + "vlse16.v v16, (%[s2]), %[s_stride] \n" + "vlse16.v v24, (%[s3]), %[s_stride] \n" + "vse16.v v0, (%[d0]) \n" + "vse16.v v8, (%[d1]) \n" + "vse16.v v16, (%[d2]) \n" + "vse16.v v24, (%[d3]) \n" + "3: \n" + : [h] "+r"(h), // %[h] + [s0] "+r"(s0), // %[s0] + [s1] "+r"(s1), // %[s1] + [s2] "+r"(s2), // %[s2] + [s3] "+r"(s3), // %[s3] + [d0] "+r"(d0), // %[d0] + [d1] "+r"(d1), // %[d1] + [d2] "+r"(d2), // %[d2] + [d3] "+r"(d3), // %[d3] + [vl] "=&r"(vl), // %[vl] + [s_step] "=&r"(src_step), // %[s_step] + [d_step] "=&r"(dst_step) // %[d_step] + : [s_stride] "r"(s_stride) // %[s_stride] + : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", + "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", + "v26", "v27", "v28", "v29", "v30", "v31"); + src += 4; + dst += 4 * (ptrdiff_t)dst_stride; + width -= 4; + } + while (width > 0) { + const uint16_t* s_col = src; + uint16_t* d_row = dst; + int h = height; + int vl; + ptrdiff_t src_step; + ptrdiff_t dst_step; + asm volatile( + "vsetvli %[vl], %[h], e16, m8, ta, ma \n" + "slli %[d_step], %[vl], 1 \n" + "mul %[s_step], %[vl], %[s_stride] \n" + "sub %[h], %[h], %[vl] \n" + "bltz %[h], 2f \n" + "1: \n" + "vlse16.v v8, (%[s]), %[s_stride] \n" + "add %[s], %[s], %[s_step] \n" + "vse16.v v8, (%[d]) \n" + "add %[d], %[d], %[d_step] \n" + "sub %[h], %[h], %[vl] \n" + "bgez %[h], 1b \n" + "2: \n" + "add %[h], %[h], %[vl] \n" + "beqz %[h], 3f \n" + "vsetvli %[vl], %[h], e16, m8, ta, ma \n" + "vlse16.v v8, (%[s]), %[s_stride] \n" + "vse16.v v8, (%[d]) \n" + "3: \n" + : [h] "+r"(h), // %[h] + [s] "+r"(s_col), // %[s] + [d] "+r"(d_row), // %[d] + [vl] "=&r"(vl), // %[vl] + [s_step] "=&r"(src_step), // %[s_step] + [d_step] "=&r"(dst_step) // %[d_step] + : [s_stride] "r"(s_stride) // %[s_stride] + : RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15"); + src += 1; + dst += (ptrdiff_t)dst_stride; + width -= 1; + } +} +#endif // HAS_TRANSPOSEWXH_16_RVV + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/rotate_sme.cc b/source/rotate_sme.cc index 2182198c9..85d5bf604 100644 --- a/source/rotate_sme.cc +++ b/source/rotate_sme.cc @@ -237,6 +237,73 @@ __arm_locally_streaming __arm_new("za") void TransposeWxH_32_SME( } while (height > 0); } +__arm_locally_streaming __arm_new("za") void TransposeWxH_16_SME( + const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height) { + int vl; + asm("cnth %x0" : "=r"(vl)); + + do { + const uint16_t* src2 = src; + uint16_t* dst2 = dst; + + // Process up to VL 16-bit rows per iteration of the outer loop. + int block_height = height > vl ? vl : height; + + int width2 = width; + do { + const uint16_t* src3 = src2; + + // Process up to VL 16-bit columns per iteration of the inner loop. + int block_width = width2 > vl ? vl : width2; + + asm volatile( + "mov w12, #0 \n" + + // Create a predicate to handle loading partial rows. + "whilelt p0.h, wzr, %w[block_width] \n" + + // Load H <= VL rows into ZA0.H. + "1: \n" + "ld1h {za0h.h[w12, 0]}, p0/z, [%[src3]] \n" + "add %[src3], %[src3], %[src_stride] \n" + "add w12, w12, #1 \n" + "cmp w12, %w[block_height] \n" + "b.ne 1b \n" + + // Create a predicate to handle storing partial columns. + "whilelt p0.h, wzr, %w[block_height] \n" + "mov w12, #0 \n" + + // Store W <= VL columns from ZA0.H. + "2: \n" + "st1h {za0v.h[w12, 0]}, p0, [%[dst2]] \n" + "add %[dst2], %[dst2], %[dst_stride] \n" + "add w12, w12, #1 \n" + "cmp w12, %w[block_width] \n" + "b.ne 2b \n" + : [src3] "+r"(src3), // %[src3] + [dst2] "+r"(dst2) // %[dst2] + : [src_stride] "r"((ptrdiff_t)src_stride * 2), // %[src_stride] + [dst_stride] "r"((ptrdiff_t)dst_stride * 2), // %[dst_stride] + [block_width] "r"(block_width), // %[block_width] + [block_height] "r"(block_height) // %[block_height] + : "memory", "cc", "w12", "p0", "za"); + + src2 += vl; + width2 -= vl; + } while (width2 > 0); + + src += (ptrdiff_t)vl * src_stride; + dst += vl; + height -= vl; + } while (height > 0); +} + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/rotate_sve.cc b/source/rotate_sve.cc index 84d63bc7e..efe243689 100644 --- a/source/rotate_sve.cc +++ b/source/rotate_sve.cc @@ -109,6 +109,168 @@ void TransposeNx4_32_SVE2(const uint8_t* src, : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0"); } +void TransposeWx8_16_SVE2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint16_t* temp; + asm volatile( + "ptrue p0.h, vl8 \n" + "subs %w[w], %w[w], #8 \n" + "b.lt 2f \n" + + "1: \n" + "mov %[temp], %[src] \n" + "ld1h {z0.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z1.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z2.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z3.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z4.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z5.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z6.h}, p0/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z7.h}, p0/z, [%[temp]] \n" + "add %[src], %[src], #16 \n" + + "trn1 z16.h, z0.h, z1.h \n" + "trn2 z17.h, z0.h, z1.h \n" + "trn1 z18.h, z2.h, z3.h \n" + "trn2 z19.h, z2.h, z3.h \n" + "trn1 z20.h, z4.h, z5.h \n" + "trn2 z21.h, z4.h, z5.h \n" + "trn1 z22.h, z6.h, z7.h \n" + "trn2 z23.h, z6.h, z7.h \n" + + "trn1 z0.s, z16.s, z18.s \n" + "trn1 z1.s, z17.s, z19.s \n" + "trn2 z2.s, z16.s, z18.s \n" + "trn2 z3.s, z17.s, z19.s \n" + "trn1 z4.s, z20.s, z22.s \n" + "trn1 z5.s, z21.s, z23.s \n" + "trn2 z6.s, z20.s, z22.s \n" + "trn2 z7.s, z21.s, z23.s \n" + + "trn1 z16.d, z0.d, z4.d \n" + "trn1 z17.d, z1.d, z5.d \n" + "trn1 z18.d, z2.d, z6.d \n" + "trn1 z19.d, z3.d, z7.d \n" + "trn2 z20.d, z0.d, z4.d \n" + "trn2 z21.d, z1.d, z5.d \n" + "trn2 z22.d, z2.d, z6.d \n" + "trn2 z23.d, z3.d, z7.d \n" + + "st1h {z16.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z17.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z18.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z19.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z20.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z21.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z22.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z23.h}, p0, [%[dst]] \n" + "add %[dst], %[dst], %[d] \n" + "subs %w[w], %w[w], #8 \n" + "b.ge 1b \n" + + "2: \n" + "adds %w[w], %w[w], #8 \n" + "b.eq 3f \n" + "whilelt p1.h, wzr, %w[w] \n" + "mov %[temp], %[src] \n" + "ld1h {z0.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z1.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z2.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z3.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z4.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z5.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z6.h}, p1/z, [%[temp]] \n" + "add %[temp], %[temp], %[s] \n" + "ld1h {z7.h}, p1/z, [%[temp]] \n" + + "trn1 z16.h, z0.h, z1.h \n" + "trn2 z17.h, z0.h, z1.h \n" + "trn1 z18.h, z2.h, z3.h \n" + "trn2 z19.h, z2.h, z3.h \n" + "trn1 z20.h, z4.h, z5.h \n" + "trn2 z21.h, z4.h, z5.h \n" + "trn1 z22.h, z6.h, z7.h \n" + "trn2 z23.h, z6.h, z7.h \n" + + "trn1 z0.s, z16.s, z18.s \n" + "trn1 z1.s, z17.s, z19.s \n" + "trn2 z2.s, z16.s, z18.s \n" + "trn2 z3.s, z17.s, z19.s \n" + "trn1 z4.s, z20.s, z22.s \n" + "trn1 z5.s, z21.s, z23.s \n" + "trn2 z6.s, z20.s, z22.s \n" + "trn2 z7.s, z21.s, z23.s \n" + + "trn1 z16.d, z0.d, z4.d \n" + "trn1 z17.d, z1.d, z5.d \n" + "trn1 z18.d, z2.d, z6.d \n" + "trn1 z19.d, z3.d, z7.d \n" + "trn2 z20.d, z0.d, z4.d \n" + "trn2 z21.d, z1.d, z5.d \n" + "trn2 z22.d, z2.d, z6.d \n" + "trn2 z23.d, z3.d, z7.d \n" + + "st1h {z16.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z17.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z18.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z19.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z20.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z21.h}, p0, [%[dst]] \n" + "subs %w[w], %w[w], #1 \n" + "b.eq 3f \n" + "add %[dst], %[dst], %[d] \n" + "st1h {z22.h}, p0, [%[dst]] \n" + "3: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "cc", "p0", "p1", "z0", "z1", "z2", "z3", "z4", "z5", "z6", + "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23"); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/source/rotate_wasm.cc b/source/rotate_wasm.cc index 093a37e39..9c3cbac64 100644 --- a/source/rotate_wasm.cc +++ b/source/rotate_wasm.cc @@ -260,6 +260,57 @@ void Transpose4x4_32_WASMSIMD(const uint8_t* src, } #endif // HAS_TRANSPOSE4X4_32_WASMSIMD +#if defined(HAS_TRANSPOSEWX8_16_WASMSIMD) +void TransposeWx8_16_WASMSIMD(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + v128_t r0 = wasm_v128_load(src); + v128_t r1 = wasm_v128_load(src + s); + v128_t r2 = wasm_v128_load(src + s * 2); + v128_t r3 = wasm_v128_load(src + s * 3); + v128_t r4 = wasm_v128_load(src + s * 4); + v128_t r5 = wasm_v128_load(src + s * 5); + v128_t r6 = wasm_v128_load(src + s * 6); + v128_t r7 = wasm_v128_load(src + s * 7); + + v128_t t0 = ZIPLO16(r0, r1); + v128_t t1 = ZIPHI16(r0, r1); + v128_t t2 = ZIPLO16(r2, r3); + v128_t t3 = ZIPHI16(r2, r3); + v128_t t4 = ZIPLO16(r4, r5); + v128_t t5 = ZIPHI16(r4, r5); + v128_t t6 = ZIPLO16(r6, r7); + v128_t t7 = ZIPHI16(r6, r7); + + v128_t u0 = ZIPLO32(t0, t2); + v128_t u1 = ZIPHI32(t0, t2); + v128_t u2 = ZIPLO32(t1, t3); + v128_t u3 = ZIPHI32(t1, t3); + v128_t u4 = ZIPLO32(t4, t6); + v128_t u5 = ZIPHI32(t4, t6); + v128_t u6 = ZIPLO32(t5, t7); + v128_t u7 = ZIPHI32(t5, t7); + + wasm_v128_store(dst, ZIPLO64(u0, u4)); + wasm_v128_store(dst + d, ZIPHI64(u0, u4)); + wasm_v128_store(dst + d * 2, ZIPLO64(u1, u5)); + wasm_v128_store(dst + d * 3, ZIPHI64(u1, u5)); + wasm_v128_store(dst + d * 4, ZIPLO64(u2, u6)); + wasm_v128_store(dst + d * 5, ZIPHI64(u2, u6)); + wasm_v128_store(dst + d * 6, ZIPLO64(u3, u7)); + wasm_v128_store(dst + d * 7, ZIPHI64(u3, u7)); + src += 8; + dst += d * 8; + width -= 8; + } +} +#endif // HAS_TRANSPOSEWX8_16_WASMSIMD + #undef ZIPLO8 #undef ZIPHI8 #undef ZIPLO16 diff --git a/source/rotate_win.cc b/source/rotate_win.cc index f5eb65dad..08791bb9e 100644 --- a/source/rotate_win.cc +++ b/source/rotate_win.cc @@ -417,6 +417,238 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW) +#if defined(HAS_TRANSPOSEWX8_16_SSE2) +LIBYUV_TARGET_SSE2 +void TransposeWx8_16_SSE2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); + __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); + __m128i r4 = _mm_loadu_si128((const __m128i*)(src + s * 4)); + __m128i r5 = _mm_loadu_si128((const __m128i*)(src + s * 5)); + __m128i r6 = _mm_loadu_si128((const __m128i*)(src + s * 6)); + __m128i r7 = _mm_loadu_si128((const __m128i*)(src + s * 7)); + + __m128i t0 = _mm_unpacklo_epi16(r0, r1); + __m128i t1 = _mm_unpackhi_epi16(r0, r1); + __m128i t2 = _mm_unpacklo_epi16(r2, r3); + __m128i t3 = _mm_unpackhi_epi16(r2, r3); + __m128i t4 = _mm_unpacklo_epi16(r4, r5); + __m128i t5 = _mm_unpackhi_epi16(r4, r5); + __m128i t6 = _mm_unpacklo_epi16(r6, r7); + __m128i t7 = _mm_unpackhi_epi16(r6, r7); + + __m128i u0 = _mm_unpacklo_epi32(t0, t2); + __m128i u1 = _mm_unpackhi_epi32(t0, t2); + __m128i u2 = _mm_unpacklo_epi32(t1, t3); + __m128i u3 = _mm_unpackhi_epi32(t1, t3); + __m128i u4 = _mm_unpacklo_epi32(t4, t6); + __m128i u5 = _mm_unpackhi_epi32(t4, t6); + __m128i u6 = _mm_unpacklo_epi32(t5, t7); + __m128i u7 = _mm_unpackhi_epi32(t5, t7); + + _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(u0, u4)); + _mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(u0, u4)); + _mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(u1, u5)); + _mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(u1, u5)); + _mm_storeu_si128((__m128i*)(dst + d * 4), _mm_unpacklo_epi64(u2, u6)); + _mm_storeu_si128((__m128i*)(dst + d * 5), _mm_unpackhi_epi64(u2, u6)); + _mm_storeu_si128((__m128i*)(dst + d * 6), _mm_unpacklo_epi64(u3, u7)); + _mm_storeu_si128((__m128i*)(dst + d * 7), _mm_unpackhi_epi64(u3, u7)); + src += 8; + dst += d * 8; + width -= 8; + } +} +#endif // defined(HAS_TRANSPOSEWX8_16_SSE2) + +#if defined(HAS_TRANSPOSEWX16_16_AVX2) +LIBYUV_TARGET_AVX2 +void TransposeWx16_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), + _mm_loadu_si128((const __m128i*)(src + s * 8)), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), + _mm_loadu_si128((const __m128i*)(src + s * 9)), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), + _mm_loadu_si128((const __m128i*)(src + s * 10)), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), + _mm_loadu_si128((const __m128i*)(src + s * 11)), 1); + __m256i r4 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 4))), + _mm_loadu_si128((const __m128i*)(src + s * 12)), 1); + __m256i r5 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 5))), + _mm_loadu_si128((const __m128i*)(src + s * 13)), 1); + __m256i r6 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 6))), + _mm_loadu_si128((const __m128i*)(src + s * 14)), 1); + __m256i r7 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 7))), + _mm_loadu_si128((const __m128i*)(src + s * 15)), 1); + + __m256i t0 = _mm256_unpacklo_epi16(r0, r1); + __m256i t1 = _mm256_unpackhi_epi16(r0, r1); + __m256i t2 = _mm256_unpacklo_epi16(r2, r3); + __m256i t3 = _mm256_unpackhi_epi16(r2, r3); + __m256i t4 = _mm256_unpacklo_epi16(r4, r5); + __m256i t5 = _mm256_unpackhi_epi16(r4, r5); + __m256i t6 = _mm256_unpacklo_epi16(r6, r7); + __m256i t7 = _mm256_unpackhi_epi16(r6, r7); + + __m256i u0 = _mm256_unpacklo_epi32(t0, t2); + __m256i u1 = _mm256_unpackhi_epi32(t0, t2); + __m256i u2 = _mm256_unpacklo_epi32(t1, t3); + __m256i u3 = _mm256_unpackhi_epi32(t1, t3); + __m256i u4 = _mm256_unpacklo_epi32(t4, t6); + __m256i u5 = _mm256_unpackhi_epi32(t4, t6); + __m256i u6 = _mm256_unpacklo_epi32(t5, t7); + __m256i u7 = _mm256_unpackhi_epi32(t5, t7); + + _mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(u0, u4)); + _mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(u0, u4)); + _mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(u1, u5)); + _mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(u1, u5)); + _mm256_storeu_si256((__m256i*)(dst + d * 4), _mm256_unpacklo_epi64(u2, u6)); + _mm256_storeu_si256((__m256i*)(dst + d * 5), _mm256_unpackhi_epi64(u2, u6)); + _mm256_storeu_si256((__m256i*)(dst + d * 6), _mm256_unpacklo_epi64(u3, u7)); + _mm256_storeu_si256((__m256i*)(dst + d * 7), _mm256_unpackhi_epi64(u3, u7)); + src += 8; + dst += d * 8; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSEWX16_16_AVX2) + +#if defined(HAS_TRANSPOSEWX32_16_AVX512BW) +LIBYUV_TARGET_AVX512BW +void TransposeWx32_16_AVX512BW(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + __m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src))); + __m512i r1 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s))); + __m512i r2 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2))); + __m512i r3 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3))); + __m512i r4 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 4))); + __m512i r5 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 5))); + __m512i r6 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 6))); + __m512i r7 = + _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 7))); + + r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)), + 1); + r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)), + 1); + r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)), + 1); + r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)), + 1); + r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 12)), + 1); + r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 13)), + 1); + r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 14)), + 1); + r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 15)), + 1); + + r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 16)), + 2); + r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 17)), + 2); + r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 18)), + 2); + r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 19)), + 2); + r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 20)), + 2); + r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 21)), + 2); + r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 22)), + 2); + r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 23)), + 2); + + r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 24)), + 3); + r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 25)), + 3); + r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 26)), + 3); + r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 27)), + 3); + r4 = _mm512_inserti32x4(r4, _mm_loadu_si128((const __m128i*)(src + s * 28)), + 3); + r5 = _mm512_inserti32x4(r5, _mm_loadu_si128((const __m128i*)(src + s * 29)), + 3); + r6 = _mm512_inserti32x4(r6, _mm_loadu_si128((const __m128i*)(src + s * 30)), + 3); + r7 = _mm512_inserti32x4(r7, _mm_loadu_si128((const __m128i*)(src + s * 31)), + 3); + + __m512i t0 = _mm512_unpacklo_epi16(r0, r1); + __m512i t1 = _mm512_unpackhi_epi16(r0, r1); + __m512i t2 = _mm512_unpacklo_epi16(r2, r3); + __m512i t3 = _mm512_unpackhi_epi16(r2, r3); + __m512i t4 = _mm512_unpacklo_epi16(r4, r5); + __m512i t5 = _mm512_unpackhi_epi16(r4, r5); + __m512i t6 = _mm512_unpacklo_epi16(r6, r7); + __m512i t7 = _mm512_unpackhi_epi16(r6, r7); + + __m512i u0 = _mm512_unpacklo_epi32(t0, t2); + __m512i u1 = _mm512_unpackhi_epi32(t0, t2); + __m512i u2 = _mm512_unpacklo_epi32(t1, t3); + __m512i u3 = _mm512_unpackhi_epi32(t1, t3); + __m512i u4 = _mm512_unpacklo_epi32(t4, t6); + __m512i u5 = _mm512_unpackhi_epi32(t4, t6); + __m512i u6 = _mm512_unpacklo_epi32(t5, t7); + __m512i u7 = _mm512_unpackhi_epi32(t5, t7); + + _mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(u0, u4)); + _mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(u0, u4)); + _mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(u1, u5)); + _mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(u1, u5)); + _mm512_storeu_si512((__m512i*)(dst + d * 4), _mm512_unpacklo_epi64(u2, u6)); + _mm512_storeu_si512((__m512i*)(dst + d * 5), _mm512_unpackhi_epi64(u2, u6)); + _mm512_storeu_si512((__m512i*)(dst + d * 6), _mm512_unpacklo_epi64(u3, u7)); + _mm512_storeu_si512((__m512i*)(dst + d * 7), _mm512_unpackhi_epi64(u3, u7)); + src += 8; + dst += d * 8; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW) + #if defined(HAS_TRANSPOSE4X4_32_SSE2) // Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row. LIBYUV_TARGET_SSE2 diff --git a/source/row_any.cc b/source/row_any.cc index 2bd2957c5..c12ad5e10 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -2088,9 +2088,15 @@ ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 31) #ifdef HAS_MIRRORROW_LASX ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 63) #endif +#ifdef HAS_MIRRORUVROW_AVX512BW +ANY11M(MirrorUVRow_Any_AVX512BW, MirrorUVRow_AVX512BW, 2, 31) +#endif #ifdef HAS_MIRRORUVROW_AVX2 ANY11M(MirrorUVRow_Any_AVX2, MirrorUVRow_AVX2, 2, 15) #endif +#ifdef HAS_MIRRORUVROW_SSSE3 +ANY11M(MirrorUVRow_Any_SSSE3, MirrorUVRow_SSSE3, 2, 7) +#endif #ifdef HAS_MIRRORUVROW_NEON ANY11M(MirrorUVRow_Any_NEON, MirrorUVRow_NEON, 2, 31) #endif diff --git a/source/row_gcc.cc b/source/row_gcc.cc index a4b959a50..52ec255e4 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -5357,6 +5357,31 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { } #endif // HAS_MIRRORUVROW_AVX2 +#ifdef HAS_MIRRORUVROW_AVX512BW +static const uint16_t kMirrorUVPermute_AVX512[32] = { + 31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u, + 20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u, + 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u}; +void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + ptrdiff_t temp_width = (ptrdiff_t)(width); + asm volatile("vmovdqu16 %3,%%zmm5 \n" + + LABELALIGN + "1: \n" + "vpermw -0x40(%0,%2,2),%%zmm5,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(temp_width) // %2 + : "m"(kMirrorUVPermute_AVX512) // %3 + : "memory", "cc", "xmm0", "xmm5"); +} +#endif // HAS_MIRRORUVROW_AVX512BW + #ifdef HAS_RGB24MIRRORROW_SSSE3 // Shuffle first 5 pixels to last 5 mirrored. first byte zero diff --git a/source/row_rvv.cc b/source/row_rvv.cc index a22f239bc..d7898ea18 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -2628,6 +2628,53 @@ void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBMIRRORROW_RVV +#ifdef HAS_MIRRORUVROW_RVV +void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + int vl; + ptrdiff_t byte_vl; + src_uv += (ptrdiff_t)width * 2; + asm volatile( + "vsetvli %[vl], %[w], e16, m8, ta, ma \n" + "slli %[byte_vl], %[vl], 1 \n" + "vid.v v16 \n" + "addi %[vl], %[vl], -1 \n" + "vrsub.vx v16, v16, %[vl] \n" + "addi %[vl], %[vl], 1 \n" + "sub %[w], %[w], %[vl] \n" + "bltz %[w], 2f \n" + "1: \n" + "sub %[src], %[src], %[byte_vl] \n" + "vle16.v v8, (%[src]) \n" + "vrgather.vv v0, v8, v16 \n" + "vse16.v v0, (%[dst]) \n" + "add %[dst], %[dst], %[byte_vl] \n" + "sub %[w], %[w], %[vl] \n" + "bgez %[w], 1b \n" + "2: \n" + "add %[w], %[w], %[vl] \n" + "beqz %[w], 3f \n" + "vsetvli %[vl], %[w], e16, m8, ta, ma \n" + "slli %[byte_vl], %[vl], 1 \n" + "vid.v v16 \n" + "addi %[vl], %[vl], -1 \n" + "vrsub.vx v16, v16, %[vl] \n" + "sub %[src], %[src], %[byte_vl] \n" + "vle16.v v8, (%[src]) \n" + "vrgather.vv v0, v8, v16 \n" + "vse16.v v0, (%[dst]) \n" + "3: \n" + : [src] "+r"(src_uv), // %[src] + [dst] "+r"(dst_uv), // %[dst] + [w] "+r"(width), // %[w] + [vl] "=&r"(vl), // %[vl] + [byte_vl] "=&r"(byte_vl) // %[byte_vl] + : + : RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5", + "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23"); +} +#endif // HAS_MIRRORUVROW_RVV + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_win.cc b/source/row_win.cc index 077b4e077..24d4d888f 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -915,6 +915,25 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { } #endif // HAS_MIRRORUVROW_AVX2 +#ifdef HAS_MIRRORUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { + const __m512i zmm_idx = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + src_uv += width * 2; + while (width > 0) { + src_uv -= 64; + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_uv); + zmm0 = _mm512_permutexvar_epi16(zmm_idx, zmm0); + _mm512_storeu_si512((__m512i*)dst_uv, zmm0); + dst_uv += 64; + width -= 32; + } + _mm256_zeroupper(); +} +#endif // HAS_MIRRORUVROW_AVX512BW + #ifdef HAS_SWAPUVROW_AVX2 LIBYUV_TARGET_AVX2 void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) { diff --git a/unit_test/rotate_argb_test.cc b/unit_test/rotate_argb_test.cc index af7333512..3f6895b71 100644 --- a/unit_test/rotate_argb_test.cc +++ b/unit_test/rotate_argb_test.cc @@ -263,15 +263,15 @@ static void TestRotatePlane_16(int src_width, int src_plane_size = src_stride * abs(src_height); align_buffer_page_end_16(src, src_plane_size); for (int i = 0; i < src_plane_size; ++i) { - src[i] = fastrand() & 0xff; + src[i] = fastrand() & 0xffff; } int dst_stride = dst_width; int dst_plane_size = dst_stride * dst_height; align_buffer_page_end_16(dst_c, dst_plane_size); align_buffer_page_end_16(dst_opt, dst_plane_size); - memset(dst_c, 2, dst_plane_size); - memset(dst_opt, 3, dst_plane_size); + memset(dst_c, 2, dst_plane_size * 2); + memset(dst_opt, 3, dst_plane_size * 2); MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization. RotatePlane_16(src, src_stride, dst_c, dst_stride, src_width, src_height,