coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

InterpolateRow_AVX512BW with tail masking

Browse Source 
I420ScaleTo569x480_Bilinear (LIBYUV_WIDTH=569 LIBYUV_REPEAT=5000):
CPU                     AVX2      AVX512     Speedup
Intel Emerald Rapids   124 ms      63 ms      1.97x
Intel Sapphire Rapids  134 ms      69 ms      1.94x
Intel Skylake-X        219 ms     129 ms      1.70x
Intel Granite Rapids   124 ms      76 ms      1.63x
Intel Ice Lake         164 ms     101 ms      1.62x

Bug: None
Change-Id: I7a10446eb4dbcebae68380c2a186d4e3badec551
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8483371
Commit-Queue: Frank Barchard <fbarchard@google.com>
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard 2026-09-28 19:28:10 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent 39445fa162
commit b25fa89920
9 changed files with 258 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/libyuv/row.h
View File

@ -383,6 +383,7 @@ extern "C" {
#define HAS_I422TOAR30ROW_AVX512BW
#define HAS_RGBTOYMATRIXROW_AVX512BW
#define HAS_BLENDPLANEROW_AVX512BW
#define HAS_INTERPOLATEROW_AVX512BW
#endif
#define HAS_ARGBTOYMATRIXROW_AVX2
#define HAS_RGBTOYMATRIXROW_AVX2
@ -436,6 +437,7 @@ extern "C" {
#define HAS_ARGBSHUFFLEROW_AVX512BW
#define HAS_ARGBTOUV444MATRIXROW_AVX512BW
#define HAS_BLENDPLANEROW_AVX512BW
#define HAS_INTERPOLATEROW_AVX512BW
#define HAS_RGBTOUV444MATRIXROW_AVX512BW
#define HAS_ARGBTOYMATRIXROW_AVX512BW
#define HAS_I422TORGB24ROW_AVX512VBMI
@ -6682,6 +6684,11 @@ void InterpolateRow_AVX2(uint8_t* dst_ptr,
ptrdiff_t src_stride,
int dst_width,
int source_y_fraction);
void InterpolateRow_AVX512BW(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
int dst_width,
int source_y_fraction);
void InterpolateRow_NEON(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,

5
source/convert.cc
View File

@ -1501,6 +1501,11 @@ int I422ToNV21(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;

10
source/planar_functions.cc
View File

@ -4604,6 +4604,11 @@ int InterpolatePlane(const uint8_t* src0,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@ -5862,6 +5867,11 @@ int UYVYToNV12(const uint8_t* src_uyvy,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;

126
source/row_gcc.cc
View File

@ -9978,6 +9978,132 @@ void InterpolateRow_AVX2(uint8_t* dst_ptr,
}
#endif // HAS_INTERPOLATEROW_AVX2
#ifdef HAS_INTERPOLATEROW_AVX512BW
// Bilinear filter 64x2 -> 64x1 with masked tail for any width.
void InterpolateRow_AVX512BW(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
int width,
int source_y_fraction) {
asm volatile(
"sub %1,%0 \n"
"cmp $0x0,%3 \n"
"je 100f \n"
"cmp $0x80,%3 \n"
"je 50f \n"
"imul $0xff,%3,%3 \n"
"add $0x100,%3 \n"
"vpbroadcastw %3,%%zmm5 \n"
"vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n"
"vpabsb %%zmm4,%%zmm4 \n"
"vpsllw $7,%%zmm4,%%zmm4 \n"
"sub $0x40,%2 \n"
"jl 2f \n"
// General purpose row blend.
LABELALIGN
"1: \n"
"vmovdqu64 (%1),%%zmm0 \n"
"vmovdqu64 0x00(%1,%4,1),%%zmm2 \n"
"vpunpckhbw %%zmm2,%%zmm0,%%zmm1 \n"
"vpunpcklbw %%zmm2,%%zmm0,%%zmm0 \n"
"vpsubb %%zmm4,%%zmm1,%%zmm1 \n"
"vpsubb %%zmm4,%%zmm0,%%zmm0 \n"
"vpmaddubsw %%zmm1,%%zmm5,%%zmm1 \n"
"vpmaddubsw %%zmm0,%%zmm5,%%zmm0 \n"
"vpaddw %%zmm4,%%zmm1,%%zmm1 \n"
"vpaddw %%zmm4,%%zmm0,%%zmm0 \n"
"vpsrlw $0x8,%%zmm1,%%zmm1 \n"
"vpsrlw $0x8,%%zmm0,%%zmm0 \n"
"vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
"vmovdqu64 %%zmm0,0x00(%1,%0,1) \n"
"lea 0x40(%1),%1 \n"
"sub $0x40,%2 \n"
"jge 1b \n"
"2: \n"
"add $0x40,%2 \n"
"jle 99f \n"
"mov $-1,%q3 \n"
"bzhi %q2,%q3,%q3 \n"
"kmovq %q3,%%k1 \n"
"vmovdqu8 (%1),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu8 0x00(%1,%4,1),%%zmm2%{%%k1%}%{z%}\n"
"vpunpckhbw %%zmm2,%%zmm0,%%zmm1 \n"
"vpunpcklbw %%zmm2,%%zmm0,%%zmm0 \n"
"vpsubb %%zmm4,%%zmm1,%%zmm1 \n"
"vpsubb %%zmm4,%%zmm0,%%zmm0 \n"
"vpmaddubsw %%zmm1,%%zmm5,%%zmm1 \n"
"vpmaddubsw %%zmm0,%%zmm5,%%zmm0 \n"
"vpaddw %%zmm4,%%zmm1,%%zmm1 \n"
"vpaddw %%zmm4,%%zmm0,%%zmm0 \n"
"vpsrlw $0x8,%%zmm1,%%zmm1 \n"
"vpsrlw $0x8,%%zmm0,%%zmm0 \n"
"vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
"vmovdqu8 %%zmm0,0x00(%1,%0,1)%{%%k1%} \n"
"jmp 99f \n"
// Blend 50 / 50.
LABELALIGN
"50: \n"
"sub $0x40,%2 \n"
"jl 52f \n"
LABELALIGN
"51: \n"
"vmovdqu64 (%1),%%zmm0 \n"
"vpavgb 0x00(%1,%4,1),%%zmm0,%%zmm0 \n"
"vmovdqu64 %%zmm0,0x00(%1,%0,1) \n"
"lea 0x40(%1),%1 \n"
"sub $0x40,%2 \n"
"jge 51b \n"
"52: \n"
"add $0x40,%2 \n"
"jle 99f \n"
"mov $-1,%q3 \n"
"bzhi %q2,%q3,%q3 \n"
"kmovq %q3,%%k1 \n"
"vmovdqu8 (%1),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu8 0x00(%1,%4,1),%%zmm2%{%%k1%}%{z%}\n"
"vpavgb %%zmm2,%%zmm0,%%zmm0 \n"
"vmovdqu8 %%zmm0,0x00(%1,%0,1)%{%%k1%} \n"
"jmp 99f \n"
// Blend 100 / 0 - Copy row unchanged.
LABELALIGN
"100: \n"
"sub $0x40,%2 \n"
"jl 102f \n"
LABELALIGN
"101: \n"
"vmovdqu64 (%1),%%zmm0 \n"
"vmovdqu64 %%zmm0,0x00(%1,%0,1) \n"
"lea 0x40(%1),%1 \n"
"sub $0x40,%2 \n"
"jge 101b \n"
"102: \n"
"add $0x40,%2 \n"
"jle 99f \n"
"mov $-1,%q3 \n"
"bzhi %q2,%q3,%q3 \n"
"kmovq %q3,%%k1 \n"
"vmovdqu8 (%1),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu8 %%zmm0,0x00(%1,%0,1)%{%%k1%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(width), // %2
"+r"(source_y_fraction) // %3
: "r"(src_stride) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm4", "xmm5", "k1");
}
#endif // HAS_INTERPOLATEROW_AVX512BW
#ifdef HAS_INTERPOLATEROW_16_AVX2
// Bilinear filter 16x2 -> 16x1
void InterpolateRow_16_AVX2(uint16_t* dst_ptr,

75
source/row_win.cc
View File

@ -1018,6 +1018,81 @@ void InterpolateRow_AVX2(uint8_t* dst_ptr,
}
#endif // HAS_INTERPOLATEROW_AVX2
#ifdef HAS_INTERPOLATEROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void InterpolateRow_AVX512BW(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
int width,
int source_y_fraction) {
int y1 = source_y_fraction;
int y0 = 256 - y1;
const uint8_t* src_ptr1 = src_ptr + src_stride;
__m512i zmm_y = _mm512_set1_epi16((short)((y1 << 8) | y0));
__m512i zmm_8080 = _mm512_set1_epi16((short)0x8080);
int i;
if (y1 == 0) {
for (i = 0; i <= width - 64; i += 64) {
_mm512_storeu_si512((__m512i*)(dst_ptr + i),
_mm512_loadu_si512((const __m512i*)(src_ptr + i)));
}
if (i < width) {
__mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull);
_mm512_mask_storeu_epi8(dst_ptr + i, mask,
_mm512_maskz_loadu_epi8(mask, src_ptr + i));
}
} else if (y1 == 128) {
for (i = 0; i <= width - 64; i += 64) {
__m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
__m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
_mm512_storeu_si512((__m512i*)(dst_ptr + i), _mm512_avg_epu8(row0, row1));
}
if (i < width) {
__mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull);
__m512i row0 = _mm512_maskz_loadu_epi8(mask, src_ptr + i);
__m512i row1 = _mm512_maskz_loadu_epi8(mask, src_ptr1 + i);
_mm512_mask_storeu_epi8(dst_ptr + i, mask, _mm512_avg_epu8(row0, row1));
}
} else {
for (i = 0; i <= width - 64; i += 64) {
__m512i row0 = _mm512_loadu_si512((const __m512i*)(src_ptr + i));
__m512i row1 = _mm512_loadu_si512((const __m512i*)(src_ptr1 + i));
__m512i low = _mm512_unpacklo_epi8(row0, row1);
__m512i high = _mm512_unpackhi_epi8(row0, row1);
low = _mm512_sub_epi8(low, zmm_8080);
high = _mm512_sub_epi8(high, zmm_8080);
low = _mm512_maddubs_epi16(zmm_y, low);
high = _mm512_maddubs_epi16(zmm_y, high);
low = _mm512_add_epi16(low, zmm_8080);
high = _mm512_add_epi16(high, zmm_8080);
low = _mm512_srli_epi16(low, 8);
high = _mm512_srli_epi16(high, 8);
_mm512_storeu_si512((__m512i*)(dst_ptr + i),
_mm512_packus_epi16(low, high));
}
if (i < width) {
__mmask64 mask = (__mmask64)((1ull << (width - i)) - 1ull);
__m512i row0 = _mm512_maskz_loadu_epi8(mask, src_ptr + i);
__m512i row1 = _mm512_maskz_loadu_epi8(mask, src_ptr1 + i);
__m512i low = _mm512_unpacklo_epi8(row0, row1);
__m512i high = _mm512_unpackhi_epi8(row0, row1);
low = _mm512_sub_epi8(low, zmm_8080);
high = _mm512_sub_epi8(high, zmm_8080);
low = _mm512_maddubs_epi16(zmm_y, low);
high = _mm512_maddubs_epi16(zmm_y, high);
low = _mm512_add_epi16(low, zmm_8080);
high = _mm512_add_epi16(high, zmm_8080);
low = _mm512_srli_epi16(low, 8);
high = _mm512_srli_epi16(high, 8);
_mm512_mask_storeu_epi8(dst_ptr + i, mask,
_mm512_packus_epi16(low, high));
}
}
_mm256_zeroupper();
}
#endif // HAS_INTERPOLATEROW_AVX512BW
#ifdef HAS_INTERPOLATEROW_16_AVX2
LIBYUV_TARGET_AVX2
void InterpolateRow_16_AVX2(uint16_t* dst_ptr,

10
source/scale.cc
View File

@ -1053,6 +1053,11 @@ static int ScalePlaneBilinearDown(int src_width,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@ -1262,6 +1267,11 @@ static int ScalePlaneBilinearUp(int src_width,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;

10
source/scale_argb.cc
View File

@ -339,6 +339,11 @@ static int ScaleARGBBilinearDown(int src_width,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@ -463,6 +468,11 @@ static int ScaleARGBBilinearUp(int src_width,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;

5
source/scale_common.cc
View File

@ -1411,6 +1411,11 @@ void ScalePlaneVertical(int src_height,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;

10
source/scale_uv.cc
View File

@ -372,6 +372,11 @@ static int ScaleUVBilinearDown(int src_width,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
@ -485,6 +490,11 @@ static int ScaleUVBilinearUp(int src_width,
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;