From 20b0a91bc1fc76370568ea53c2fb862b1bffb51a Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Wed, 30 Sep 2026 22:53:24 -0700 Subject: [PATCH] Transpose 16x16 for AVX2 - TransposeWx16_AVX2 and TransposeUVWx16_AVX2 using 2 lanes of 8 ymm registers, with Any for tail. - Move TransposeUVWx16_C to rotate_common.cc and pass the C function to TUVANY, so the LSX 16 row Any version transposes all 16 rows. I420Rotate90_Opt (LIBYUV_WIDTH=1279 LIBYUV_HEIGHT=719 LIBYUV_REPEAT=1000): CPU SSSE3 AVX2 AVX512 AVX2 x 512 x Intel Haswell 1812 916 945 1.98x 1.92x AMD Milan (Zen 3) 177 118 118 1.50x 1.50x Intel Granite Rapids 245 186 186 1.32x 1.32x Intel Emerald Rapids 246 193 176 1.27x 1.40x Intel Sapphire Rapids 278 233 225 1.19x 1.24x Intel Ice Lake 308 213 221 1.45x 1.39x Intel Skylake-X 365 267 246 1.37x 1.48x AMD Turin (Zen 5) 106 75 75 1.41x 1.41x AMD Genoa (Zen 4) 149 98 93 1.52x 1.60x AMD Venice (Zen 6) 106 74 73 1.43x 1.45x NV12Rotate90_Opt (LIBYUV_WIDTH=1279 LIBYUV_HEIGHT=719 LIBYUV_REPEAT=1000): CPU SSSE3 AVX2 AVX512 AVX2 x 512 x Intel Haswell 1771 963 935 1.84x 1.89x AMD Milan (Zen 3) 173 118 118 1.47x 1.47x Intel Granite Rapids 275 189 186 1.46x 1.48x Intel Emerald Rapids 256 188 179 1.36x 1.43x Intel Sapphire Rapids 286 232 218 1.23x 1.31x Intel Ice Lake 309 208 218 1.49x 1.42x Intel Skylake-X 380 269 251 1.41x 1.51x AMD Turin (Zen 5) 112 74 73 1.51x 1.53x AMD Genoa (Zen 4) 146 97 93 1.51x 1.57x AMD Venice (Zen 6) 114 73 72 1.56x 1.58x Bug: None Change-Id: Ia89219e3336f39a817aa662e1452824d781664a3 TAG=agy CONV=bea53b37-b2bf-46fa-9e44-a708b438fabc Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8489456 Reviewed-by: richard winterton --- include/libyuv/rotate_row.h | 26 +++++++ source/rotate.cc | 27 +++++-- source/rotate_any.cc | 19 +++-- source/rotate_common.cc | 13 ++++ source/rotate_gcc.cc | 141 ++++++++++++++++++++++++++++++++++++ source/rotate_lsx.cc | 13 ---- 6 files changed, 214 insertions(+), 25 deletions(-) diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index 2950ee3fb..b1b4f2a7b 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -42,6 +42,8 @@ extern "C" { #if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_TRANSPOSEUVWX8_SSE2 +#define HAS_TRANSPOSEWX16_AVX2 +#define HAS_TRANSPOSEUVWX16_AVX2 #endif // The following are available for AVX512 clang x64 platforms: @@ -111,6 +113,11 @@ void TransposeWx8_SSSE3(const uint8_t* src, uint8_t* dst, int dst_stride, int width); +void TransposeWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx16_AVX512BW(const uint8_t* src, int src_stride, uint8_t* dst, @@ -137,6 +144,11 @@ void TransposeWx8_Any_SSSE3(const uint8_t* src, uint8_t* dst, int dst_stride, int width); +void TransposeWx16_Any_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx16_Any_LSX(const uint8_t* src, int src_stride, uint8_t* dst, @@ -173,6 +185,13 @@ void TransposeUVWx8_SSE2(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); +void TransposeUVWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); void TransposeUVWx16_AVX512BW(const uint8_t* src, int src_stride, uint8_t* dst_a, @@ -217,6 +236,13 @@ void TransposeUVWx8_Any_NEON(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); +void TransposeUVWx16_Any_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); void TransposeUVWx16_Any_LSX(const uint8_t* src, int src_stride, uint8_t* dst_a, diff --git a/source/rotate.cc b/source/rotate.cc index 2cff13945..aa71b0b19 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -36,7 +36,7 @@ void TransposePlane(const uint8_t* src, void (*TransposeWxH)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width, int height) = NULL; #endif -#if defined(HAS_TRANSPOSEWX16_AVX512BW) +#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) = NULL; #elif defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) @@ -83,6 +83,14 @@ void TransposePlane(const uint8_t* src, #endif } #endif +#if defined(HAS_TRANSPOSEWX16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx16 = TransposeWx16_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + TransposeWx16 = TransposeWx16_AVX2; + } + } +#endif #if defined(HAS_TRANSPOSEWX16_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { TransposeWx16 = TransposeWx16_AVX512BW; @@ -103,8 +111,8 @@ void TransposePlane(const uint8_t* src, return; } #endif -#if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEWX16_LSX) || \ - defined(HAS_TRANSPOSEWX16_NEON) +#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \ + defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) // Work across the source in 16x16 tiles if (TransposeWx16) { while (i >= 16) { @@ -287,7 +295,7 @@ void SplitTransposeUV(const uint8_t* src, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width, int height) = TransposeUVWxH_C; #endif -#if defined(HAS_TRANSPOSEUVWX16_AVX512BW) +#if defined(HAS_TRANSPOSEUVWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX512BW) void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width) = NULL; @@ -331,6 +339,14 @@ void SplitTransposeUV(const uint8_t* src, } } #endif +#if defined(HAS_TRANSPOSEUVWX16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeUVWx16 = TransposeUVWx16_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + TransposeUVWx16 = TransposeUVWx16_AVX2; + } + } +#endif #if defined(HAS_TRANSPOSEUVWX16_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { TransposeUVWx16 = TransposeUVWx16_AVX512BW; @@ -344,7 +360,8 @@ void SplitTransposeUV(const uint8_t* src, return; } #endif -#if defined(HAS_TRANSPOSEUVWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_LSX) +#if defined(HAS_TRANSPOSEUVWX16_AVX2) || \ + defined(HAS_TRANSPOSEUVWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_LSX) // Work through the source in 16x16 tiles. if (TransposeUVWx16) { while (i >= 16) { diff --git a/source/rotate_any.cc b/source/rotate_any.cc index 07c2443c8..00af1dd0b 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -41,12 +41,15 @@ TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15) TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 7) #endif #endif +#ifdef HAS_TRANSPOSEWX16_AVX2 +TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15) +#endif #ifdef HAS_TRANSPOSEWX16_LSX TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, TransposeWx16_C, 15) #endif #undef TANY -#define TUVANY(NAMEANY, TPOS_SIMD, MASK) \ +#define TUVANY(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \ void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst_a, \ int dst_stride_a, uint8_t* dst_b, int dst_stride_b, \ int width) { \ @@ -56,19 +59,21 @@ TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, TransposeWx16_C, 15) TPOS_SIMD(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, n); \ } \ ptrdiff_t np = n; \ - TransposeUVWx8_C(src + np * 2, src_stride, dst_a + np * dst_stride_a, \ - dst_stride_a, dst_b + np * dst_stride_b, dst_stride_b, \ - r); \ + TPOS_C(src + np * 2, src_stride, dst_a + np * dst_stride_a, dst_stride_a, \ + dst_b + np * dst_stride_b, dst_stride_b, r); \ } #ifdef HAS_TRANSPOSEUVWX8_NEON -TUVANY(TransposeUVWx8_Any_NEON, TransposeUVWx8_NEON, 7) +TUVANY(TransposeUVWx8_Any_NEON, TransposeUVWx8_NEON, TransposeUVWx8_C, 7) #endif #ifdef HAS_TRANSPOSEUVWX8_SSE2 -TUVANY(TransposeUVWx8_Any_SSE2, TransposeUVWx8_SSE2, 7) +TUVANY(TransposeUVWx8_Any_SSE2, TransposeUVWx8_SSE2, TransposeUVWx8_C, 7) +#endif +#ifdef HAS_TRANSPOSEUVWX16_AVX2 +TUVANY(TransposeUVWx16_Any_AVX2, TransposeUVWx16_AVX2, TransposeUVWx16_C, 7) #endif #ifdef HAS_TRANSPOSEUVWX16_LSX -TUVANY(TransposeUVWx16_Any_LSX, TransposeUVWx16_LSX, 7) +TUVANY(TransposeUVWx16_Any_LSX, TransposeUVWx16_LSX, TransposeUVWx16_C, 7) #endif #undef TUVANY diff --git a/source/rotate_common.cc b/source/rotate_common.cc index af1bea016..20e5faf2b 100644 --- a/source/rotate_common.cc +++ b/source/rotate_common.cc @@ -77,6 +77,19 @@ void TransposeUVWx8_C(const uint8_t* src, } } +void TransposeUVWx16_C(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeUVWx8_C(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, + width); + TransposeUVWx8_C((src + 8 * src_stride), src_stride, (dst_a + 8), + dst_stride_a, (dst_b + 8), dst_stride_b, width); +} + void TransposeWxH_C(const uint8_t* src, int src_stride, uint8_t* dst, diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index e11eb2c47..7d0e21942 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -286,6 +286,8 @@ static const uint32_t kPermdTranspose_AVX512BW[32] = { // or 16-bit UV split transpose (dst_a = U plane, dst_b = V plane). // TODO(fbarchard): Port to rotate_win.cc using intrinsics. // TODO(fbarchard): Use for ARGB (32 bit) and 16 bit channel transposes. +// TODO(fbarchard): Consider removing in favor of TransposeWx16_Byte_AVX2, +// which is within 10% of AVX512BW on most CPUs. static void TransposeWx16_Byte_AVX512BW(const uint8_t* src, int src_stride, uint8_t* dst_a, @@ -493,6 +495,145 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW) +#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2) +// Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8 +// of ymm(r % 8). 3 unpack stages leave each lane with 2 columns x 8 rows, and +// vpermq joins the rows 0..7 and 8..15 halves. Lane 0 of each output is an +// even column for dst_a and lane 1 an odd column for dst_b. +// Width is a multiple of 16 bytes. +static void TransposeWx16_Byte_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int byte_width) { + uintptr_t temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[src_stride]),%%xmm1 \n" + "lea (%[src],%[src_stride],2),%[temp] \n" + "lea 0x10(%[src]),%[src] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[src_stride]),%%xmm3 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm4 \n" + "vmovdqu (%[temp],%[src_stride]),%%xmm5 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm6 \n" + "vmovdqu (%[temp],%[src_stride]),%%xmm7 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[src_stride]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[src_stride]),%%ymm3,%%ymm3 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm4,%%ymm4 \n" + "vinserti128 $1,(%[temp],%[src_stride]),%%ymm5,%%ymm5 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm6,%%ymm6 \n" + "vinserti128 $1,(%[temp],%[src_stride]),%%ymm7,%%ymm7 \n" + + "vpunpcklbw %%ymm1,%%ymm0,%%ymm8 \n" + "vpunpckhbw %%ymm1,%%ymm0,%%ymm9 \n" + "vpunpcklbw %%ymm3,%%ymm2,%%ymm10 \n" + "vpunpckhbw %%ymm3,%%ymm2,%%ymm11 \n" + "vpunpcklbw %%ymm5,%%ymm4,%%ymm12 \n" + "vpunpckhbw %%ymm5,%%ymm4,%%ymm13 \n" + "vpunpcklbw %%ymm7,%%ymm6,%%ymm14 \n" + "vpunpckhbw %%ymm7,%%ymm6,%%ymm15 \n" + + "vpunpcklwd %%ymm10,%%ymm8,%%ymm0 \n" + "vpunpckhwd %%ymm10,%%ymm8,%%ymm1 \n" + "vpunpcklwd %%ymm11,%%ymm9,%%ymm2 \n" + "vpunpckhwd %%ymm11,%%ymm9,%%ymm3 \n" + "vpunpcklwd %%ymm14,%%ymm12,%%ymm4 \n" + "vpunpckhwd %%ymm14,%%ymm12,%%ymm5 \n" + "vpunpcklwd %%ymm15,%%ymm13,%%ymm6 \n" + "vpunpckhwd %%ymm15,%%ymm13,%%ymm7 \n" + + "vpunpckldq %%ymm4,%%ymm0,%%ymm8 \n" + "vpunpckhdq %%ymm4,%%ymm0,%%ymm9 \n" + "vpunpckldq %%ymm5,%%ymm1,%%ymm10 \n" + "vpunpckhdq %%ymm5,%%ymm1,%%ymm11 \n" + "vpunpckldq %%ymm6,%%ymm2,%%ymm12 \n" + "vpunpckhdq %%ymm6,%%ymm2,%%ymm13 \n" + "vpunpckldq %%ymm7,%%ymm3,%%ymm14 \n" + "vpunpckhdq %%ymm7,%%ymm3,%%ymm15 \n" + + "vpermq $0xd8,%%ymm8,%%ymm8 \n" + "vpermq $0xd8,%%ymm9,%%ymm9 \n" + "vpermq $0xd8,%%ymm10,%%ymm10 \n" + "vpermq $0xd8,%%ymm11,%%ymm11 \n" + "vpermq $0xd8,%%ymm12,%%ymm12 \n" + "vpermq $0xd8,%%ymm13,%%ymm13 \n" + "vpermq $0xd8,%%ymm14,%%ymm14 \n" + "vpermq $0xd8,%%ymm15,%%ymm15 \n" + + "vmovdqu %%xmm8,(%[dst_a]) \n" + "vextracti128 $1,%%ymm8,(%[dst_b]) \n" + "vmovdqu %%xmm9,(%[dst_a],%[dst_stride_a]) \n" + "vextracti128 $1,%%ymm9,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "vmovdqu %%xmm10,(%[dst_a]) \n" + "vextracti128 $1,%%ymm10,(%[dst_b]) \n" + "vmovdqu %%xmm11,(%[dst_a],%[dst_stride_a]) \n" + "vextracti128 $1,%%ymm11,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "vmovdqu %%xmm12,(%[dst_a]) \n" + "vextracti128 $1,%%ymm12,(%[dst_b]) \n" + "vmovdqu %%xmm13,(%[dst_a],%[dst_stride_a]) \n" + "vextracti128 $1,%%ymm13,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "vmovdqu %%xmm14,(%[dst_a]) \n" + "vextracti128 $1,%%ymm14,(%[dst_b]) \n" + "vmovdqu %%xmm15,(%[dst_a],%[dst_stride_a]) \n" + "vextracti128 $1,%%ymm15,(%[dst_b],%[dst_stride_b]) \n" + "lea (%[dst_a],%[dst_stride_a],2),%[dst_a] \n" + "lea (%[dst_b],%[dst_stride_b],2),%[dst_b] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b), + [width] "+r"(byte_width), [temp] "=&r"(temp) + : [src_stride] "r"((ptrdiff_t)src_stride), + [dst_stride_a] "r"((ptrdiff_t)dst_stride_a), + [dst_stride_b] "r"((ptrdiff_t)dst_stride_b) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", + "xmm15"); +} +#endif // defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2) + +#if defined(HAS_TRANSPOSEWX16_AVX2) +void TransposeWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx16_Byte_AVX2(src, src_stride, dst, dst_stride * 2, + dst + dst_stride, dst_stride * 2, width); +} +#endif // defined(HAS_TRANSPOSEWX16_AVX2) + +#if defined(HAS_TRANSPOSEUVWX16_AVX2) +void TransposeUVWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeWx16_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b, + dst_stride_b, width * 2); +} +#endif // defined(HAS_TRANSPOSEUVWX16_AVX2) + // Transpose UV 8x8. 64 bit. #if defined(HAS_TRANSPOSEUVWX8_SSE2) void TransposeUVWx8_SSE2(const uint8_t* src, diff --git a/source/rotate_lsx.cc b/source/rotate_lsx.cc index b292803a0..1e8b6f962 100644 --- a/source/rotate_lsx.cc +++ b/source/rotate_lsx.cc @@ -61,19 +61,6 @@ extern "C" { _dst += _stride2; \ } -void TransposeUVWx16_C(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width) { - TransposeUVWx8_C(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, - width); - TransposeUVWx8_C((src + 8 * src_stride), src_stride, (dst_a + 8), - dst_stride_a, (dst_b + 8), dst_stride_b, width); -} - void TransposeWx16_LSX(const uint8_t* src, int src_stride, uint8_t* dst,