From b25fa8992056629c99d7815516e7be6b82509897 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Mon, 28 Sep 2026 19:28:10 -0700 Subject: [PATCH] InterpolateRow_AVX512BW with tail masking I420ScaleTo569x480_Bilinear (LIBYUV_WIDTH=569 LIBYUV_REPEAT=5000): CPU AVX2 AVX512 Speedup Intel Emerald Rapids 124 ms 63 ms 1.97x Intel Sapphire Rapids 134 ms 69 ms 1.94x Intel Skylake-X 219 ms 129 ms 1.70x Intel Granite Rapids 124 ms 76 ms 1.63x Intel Ice Lake 164 ms 101 ms 1.62x Bug: None Change-Id: I7a10446eb4dbcebae68380c2a186d4e3badec551 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8483371 Commit-Queue: Frank Barchard Reviewed-by: richard winterton --- include/libyuv/row.h | 7 +++ source/convert.cc | 5 ++ source/planar_functions.cc | 10 +++ source/row_gcc.cc | 126 +++++++++++++++++++++++++++++++++++++ source/row_win.cc | 75 ++++++++++++++++++++++ source/scale.cc | 10 +++ source/scale_argb.cc | 10 +++ source/scale_common.cc | 5 ++ source/scale_uv.cc | 10 +++ 9 files changed, 258 insertions(+) diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 37a4124d8..39171f2cb 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -383,6 +383,7 @@ extern "C" { #define HAS_I422TOAR30ROW_AVX512BW #define HAS_RGBTOYMATRIXROW_AVX512BW #define HAS_BLENDPLANEROW_AVX512BW +#define HAS_INTERPOLATEROW_AVX512BW #endif #define HAS_ARGBTOYMATRIXROW_AVX2 #define HAS_RGBTOYMATRIXROW_AVX2 @@ -436,6 +437,7 @@ extern "C" { #define HAS_ARGBSHUFFLEROW_AVX512BW #define HAS_ARGBTOUV444MATRIXROW_AVX512BW #define HAS_BLENDPLANEROW_AVX512BW +#define HAS_INTERPOLATEROW_AVX512BW #define HAS_RGBTOUV444MATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512VBMI @@ -6682,6 +6684,11 @@ void InterpolateRow_AVX2(uint8_t* dst_ptr, ptrdiff_t src_stride, int dst_width, int source_y_fraction); +void InterpolateRow_AVX512BW(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int dst_width, + int source_y_fraction); void InterpolateRow_NEON(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, diff --git a/source/convert.cc b/source/convert.cc index de0e62f3c..b55f2f821 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -1501,6 +1501,11 @@ int I422ToNV21(const uint8_t* src_y, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 09b9e9e8f..e8ad41dbc 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -4604,6 +4604,11 @@ int InterpolatePlane(const uint8_t* src0, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; @@ -5862,6 +5867,11 @@ int UYVYToNV12(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 695a5e215..71fe63b47 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -9978,6 +9978,132 @@ void InterpolateRow_AVX2(uint8_t* dst_ptr, } #endif // HAS_INTERPOLATEROW_AVX2 +#ifdef HAS_INTERPOLATEROW_AVX512BW +// Bilinear filter 64x2 -> 64x1 with masked tail for any width. +void InterpolateRow_AVX512BW(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + asm volatile( + "sub %1,%0 \n" + "cmp $0x0,%3 \n" + "je 100f \n" + "cmp $0x80,%3 \n" + "je 50f \n" + + "imul $0xff,%3,%3 \n" + "add $0x100,%3 \n" + "vpbroadcastw %3,%%zmm5 \n" + "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n" + "vpabsb %%zmm4,%%zmm4 \n" + "vpsllw $7,%%zmm4,%%zmm4 \n" + + "sub $0x40,%2 \n" + "jl 2f \n" + + // General purpose row blend. + LABELALIGN + "1: \n" + "vmovdqu64 (%1),%%zmm0 \n" + "vmovdqu64 0x00(%1,%4,1),%%zmm2 \n" + "vpunpckhbw %%zmm2,%%zmm0,%%zmm1 \n" + "vpunpcklbw %%zmm2,%%zmm0,%%zmm0 \n" + "vpsubb %%zmm4,%%zmm1,%%zmm1 \n" + "vpsubb %%zmm4,%%zmm0,%%zmm0 \n" + "vpmaddubsw %%zmm1,%%zmm5,%%zmm1 \n" + "vpmaddubsw %%zmm0,%%zmm5,%%zmm0 \n" + "vpaddw %%zmm4,%%zmm1,%%zmm1 \n" + "vpaddw %%zmm4,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm1,%%zmm1 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu64 %%zmm0,0x00(%1,%0,1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "jle 99f \n" + "mov $-1,%q3 \n" + "bzhi %q2,%q3,%q3 \n" + "kmovq %q3,%%k1 \n" + "vmovdqu8 (%1),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu8 0x00(%1,%4,1),%%zmm2%{%%k1%}%{z%}\n" + "vpunpckhbw %%zmm2,%%zmm0,%%zmm1 \n" + "vpunpcklbw %%zmm2,%%zmm0,%%zmm0 \n" + "vpsubb %%zmm4,%%zmm1,%%zmm1 \n" + "vpsubb %%zmm4,%%zmm0,%%zmm0 \n" + "vpmaddubsw %%zmm1,%%zmm5,%%zmm1 \n" + "vpmaddubsw %%zmm0,%%zmm5,%%zmm0 \n" + "vpaddw %%zmm4,%%zmm1,%%zmm1 \n" + "vpaddw %%zmm4,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm1,%%zmm1 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,0x00(%1,%0,1)%{%%k1%} \n" + "jmp 99f \n" + + // Blend 50 / 50. + LABELALIGN + "50: \n" + "sub $0x40,%2 \n" + "jl 52f \n" + LABELALIGN + "51: \n" + "vmovdqu64 (%1),%%zmm0 \n" + "vpavgb 0x00(%1,%4,1),%%zmm0,%%zmm0 \n" + "vmovdqu64 %%zmm0,0x00(%1,%0,1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 51b \n" + + "52: \n" + "add $0x40,%2 \n" + "jle 99f \n" + "mov $-1,%q3 \n" + "bzhi %q2,%q3,%q3 \n" + "kmovq %q3,%%k1 \n" + "vmovdqu8 (%1),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu8 0x00(%1,%4,1),%%zmm2%{%%k1%}%{z%}\n" + "vpavgb %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,0x00(%1,%0,1)%{%%k1%} \n" + "jmp 99f \n" + + // Blend 100 / 0 - Copy row unchanged. + LABELALIGN + "100: \n" + "sub $0x40,%2 \n" + "jl 102f \n" + LABELALIGN + "101: \n" + "vmovdqu64 (%1),%%zmm0 \n" + "vmovdqu64 %%zmm0,0x00(%1,%0,1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 101b \n" + + "102: \n" + "add $0x40,%2 \n" + "jle 99f \n" + "mov $-1,%q3 \n" + "bzhi %q2,%q3,%q3 \n" + "kmovq %q3,%%k1 \n" + "vmovdqu8 (%1),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu8 %%zmm0,0x00(%1,%0,1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(dst_ptr), // %0 + "+r"(src_ptr), // %1 + "+r"(width), // %2 + "+r"(source_y_fraction) // %3 + : "r"(src_stride) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm4", "xmm5", "k1"); +} +#endif // HAS_INTERPOLATEROW_AVX512BW + #ifdef HAS_INTERPOLATEROW_16_AVX2 // Bilinear filter 16x2 -> 16x1 void InterpolateRow_16_AVX2(uint16_t* dst_ptr, diff --git a/source/row_win.cc b/source/row_win.cc index 214767b0e..ad5db9206 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -1018,6 +1018,81 @@ void InterpolateRow_AVX2(uint8_t* dst_ptr, } #endif // HAS_INTERPOLATEROW_AVX2 +#ifdef HAS_INTERPOLATEROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void InterpolateRow_AVX512BW(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction) { + int y1 = source_y_fraction; + int y0 = 256 - y1; + const uint8_t* src_ptr1 = src_ptr + src_stride; + __m512i zmm_y = _mm512_set1_epi16((short)((y1 << 8) | y0)); + __m512i zmm_8080 = _mm512_set1_epi16((short)0x8080); + int i; + + if (y1 == 0) { + for (i = 0; i <= width - 64; i += 64) { + _mm512_storeu_si512((__m512i*)(dst_ptr + i), + _mm512_loadu_si512((const __m512i*)(src_ptr + i))); + } + if (i < width) { + __mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull); + _mm512_mask_storeu_epi8(dst_ptr + i, mask, + _mm512_maskz_loadu_epi8(mask, src_ptr + i)); + } + } else if (y1 == 128) { + for (i = 0; i <= width - 64; i += 64) { + __m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i)); + __m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i)); + _mm512_storeu_si512((__m512i*)(dst_ptr + i), _mm512_avg_epu8(row0, row1)); + } + if (i < width) { + __mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull); + __m512i row0 = _mm512_maskz_loadu_epi8(mask, src_ptr + i); + __m512i row1 = _mm512_maskz_loadu_epi8(mask, src_ptr1 + i); + _mm512_mask_storeu_epi8(dst_ptr + i, mask, _mm512_avg_epu8(row0, row1)); + } + } else { + for (i = 0; i <= width - 64; i += 64) { + __m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i)); + __m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i)); + __m512i low = _mm512_unpacklo_epi8(row0, row1); + __m512i high = _mm512_unpackhi_epi8(row0, row1); + low = _mm512_sub_epi8(low, zmm_8080); + high = _mm512_sub_epi8(high, zmm_8080); + low = _mm512_maddubs_epi16(zmm_y, low); + high = _mm512_maddubs_epi16(zmm_y, high); + low = _mm512_add_epi16(low, zmm_8080); + high = _mm512_add_epi16(high, zmm_8080); + low = _mm512_srli_epi16(low, 8); + high = _mm512_srli_epi16(high, 8); + _mm512_storeu_si512((__m512i*)(dst_ptr + i), + _mm512_packus_epi16(low, high)); + } + if (i < width) { + __mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull); + __m512i row0 = _mm512_maskz_loadu_epi8(mask, src_ptr + i); + __m512i row1 = _mm512_maskz_loadu_epi8(mask, src_ptr1 + i); + __m512i low = _mm512_unpacklo_epi8(row0, row1); + __m512i high = _mm512_unpackhi_epi8(row0, row1); + low = _mm512_sub_epi8(low, zmm_8080); + high = _mm512_sub_epi8(high, zmm_8080); + low = _mm512_maddubs_epi16(zmm_y, low); + high = _mm512_maddubs_epi16(zmm_y, high); + low = _mm512_add_epi16(low, zmm_8080); + high = _mm512_add_epi16(high, zmm_8080); + low = _mm512_srli_epi16(low, 8); + high = _mm512_srli_epi16(high, 8); + _mm512_mask_storeu_epi8(dst_ptr + i, mask, + _mm512_packus_epi16(low, high)); + } + } + _mm256_zeroupper(); +} +#endif // HAS_INTERPOLATEROW_AVX512BW + #ifdef HAS_INTERPOLATEROW_16_AVX2 LIBYUV_TARGET_AVX2 void InterpolateRow_16_AVX2(uint16_t* dst_ptr, diff --git a/source/scale.cc b/source/scale.cc index aaaa52949..404768b40 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -1053,6 +1053,11 @@ static int ScalePlaneBilinearDown(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; @@ -1262,6 +1267,11 @@ static int ScalePlaneBilinearUp(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; diff --git a/source/scale_argb.cc b/source/scale_argb.cc index b35581ca6..70b2c3de4 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -339,6 +339,11 @@ static int ScaleARGBBilinearDown(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; @@ -463,6 +468,11 @@ static int ScaleARGBBilinearUp(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; diff --git a/source/scale_common.cc b/source/scale_common.cc index 292a852c4..19dd1d797 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1411,6 +1411,11 @@ void ScalePlaneVertical(int src_height, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 095f96e12..8ea8e0100 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -372,6 +372,11 @@ static int ScaleUVBilinearDown(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON; @@ -485,6 +490,11 @@ static int ScaleUVBilinearUp(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_AVX512BW; + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { InterpolateRow = InterpolateRow_Any_NEON;