coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

AVX2 column kernel for ScaleUVBox at 3x

Browse Source 
The box path costs +34% over point sampling, all of it the scalar column
pass. A UV pair is 32 bits, so vpaddw keeps U and V in separate
  halves and vpblendd + vpermd compacts every third pair. 96 bytes in, 16 out.

The +34% perf regression from the previous CL becomes +3%. Box width 3
only, AVX2 only; other widths need their own permute constants.

Bug: libyuv:564129034
Change-Id: Ic20c798645be94548f74cb450e04b2e28c6dc9b2
This commit is contained in:
Philipp Hancke 2026-10-01 12:36:56 +02:00
parent 2404f9730e
commit c1f2a326d8
3 changed files with 89 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
include/libyuv/scale_row.h
View File

@ -92,6 +92,13 @@ extern "C" {
#define HAS_SCALEROWDOWN4_AVX2
#endif
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__)) && \
(defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \
!defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_SCALEUVADDCOLS3_AVX2
#endif
// The following are available for AVX512 clang x64 platforms:
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \
@ -845,6 +852,10 @@ void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width);
void ScaleAddRow_AVX512BW(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);
void ScaleUVAddCols3_AVX2(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv);
void ScaleAddRow_Any_SSE2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width);

48
source/scale_gcc.cc
View File

@ -2598,6 +2598,54 @@ void ScaleAddCols2_AVX512BW(int dst_width,
}
#endif // HAS_SCALEADDCOLS_AVX512BW
#ifdef HAS_SCALEUVADDCOLS3_AVX2
static const ulvec32 kPermUVBox3 = {0, 3, 6, 2, 5, 1, 4, 7};
// A UV pair is 32 bits, so vpblendd + vpermd compacts every third pair and
// vpaddw keeps U and V in separate halves. dst_width must be a multiple of 8.
// Consumes exactly 24 accumulator entries (96 bytes) per iteration with no
// over-read.
void ScaleUVAddCols3_AVX2(int dst_width,
int scaleval,
const uint16_t* src_uv,
uint8_t* dst_uv) {
asm volatile(
"vmovd %3,%%xmm7 \n"
"vpbroadcastw %%xmm7,%%ymm7 \n"
"vmovdqu %4,%%ymm4 \n"
LABELALIGN
"1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vpaddw 0x4(%0),%%ymm0,%%ymm0 \n"
"vpaddw 0x8(%0),%%ymm0,%%ymm0 \n"
"vmovdqu 0x1c(%0),%%ymm1 \n"
"vpaddw 0x20(%0),%%ymm1,%%ymm1 \n"
"vpaddw 0x24(%0),%%ymm1,%%ymm1 \n"
"vmovdqu 0x38(%0),%%ymm2 \n"
"vpaddw 0x3c(%0),%%ymm2,%%ymm2 \n"
"vpaddw 0x40(%0),%%ymm2,%%ymm2 \n"
"vpblendd $0x24,%%ymm1,%%ymm0,%%ymm0 \n"
"vpblendd $0x92,%%ymm2,%%ymm0,%%ymm0 \n"
"vpermd %%ymm0,%%ymm4,%%ymm0 \n"
"vpmulhuw %%ymm7,%%ymm0,%%ymm0 \n"
"vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vmovdqu %%xmm0,(%1) \n"
"lea 0x60(%0),%0 \n"
"lea 0x10(%1),%1 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_uv), // %0
"+r"(dst_uv), // %1
"+r"(dst_width) // %2
: "r"(scaleval), // %3
"m"(kPermUVBox3) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm4", "xmm7");
}
#endif // HAS_SCALEUVADDCOLS3_AVX2
static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040,
0x0040, 0x0040, 0x0040, 0x0040};

30
source/scale_uv.cc
View File

@ -307,6 +307,31 @@ static void ScaleUVAddCols2_C(int dst_width,
}
}
#if defined(HAS_SCALEUVADDCOLS3_AVX2)
static void ScaleUVAddCols3_Any_AVX2(int dst_width,
int boxheight,
int x,
int dx,
const uint16_t* src_uv,
uint8_t* dst_uv) {
int n = dst_width & ~7;
// 255 * boxheight * 3 has to stay inside the 16 bit accumulator.
if (boxheight > 85) {
ScaleUVAddCols1_C(dst_width, boxheight, x, dx, src_uv, dst_uv);
return;
}
if (n) {
ScaleUVAddCols3_AVX2(n, 65536 / (3 * boxheight),
src_uv + (ptrdiff_t)(x >> 16) * 2, dst_uv);
}
if (n < dst_width) {
ScaleUVAddCols1_C(dst_width - n, boxheight, 0, dx,
src_uv + (ptrdiff_t)((x >> 16) + n * 3) * 2,
dst_uv + (ptrdiff_t)n * 2);
}
}
#endif
static int ScaleUVBox(int src_width,
int src_height,
int dst_width,
@ -332,6 +357,11 @@ static int ScaleUVBox(int src_width,
void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx,
const uint16_t* src_uv, uint8_t* dst_uv) =
(dx & 0xffff) ? ScaleUVAddCols2_C : ScaleUVAddCols1_C;
#if defined(HAS_SCALEUVADDCOLS3_AVX2)
if (!(dx & 0xffff) && (dx >> 16) == 3 && TestCpuFlag(kCpuHasAVX2)) {
ScaleUVAddCols = ScaleUVAddCols3_Any_AVX2;
}
#endif
// Summing a row is position wise, so an interleaved UV row can be fed to
// the plane accumulator as src_width * 2 samples.
void (*ScaleAddRow)(const uint8_t* src_ptr, uint16_t* dst_ptr,