coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Multiply_16 port to avx512, rvv, neon 32 bit, and windows

Browse Source 
- Also remove redundant neon check for I8MM kernels

======================================================================
CPU                                 Before      After          Speedup
----------------------------------------------------------------------
Intel Granite Rapids (AVX10.1)      251 ms     141 ms   1.78x (+43.8%)
Intel Emerald Rapids (AVX512)       199 ms     108 ms   1.84x (+45.7%)
Intel Sapphire Rapids (AVX512)      230 ms     112 ms   2.05x (+51.3%)
AMD Turin Zen 5 (AVX512)            156 ms      99 ms   1.58x (+36.5%)
AMD Venice Zen 6 (AVX512)           204 ms     106 ms   1.92x (+48.0%)
Intel Skylake-X (AVX512)            320 ms     156 ms   2.05x (+51.2%)
AMD Genoa Zen 4 (AVX512)            236 ms     346 ms   0.68x (-46.6%)
Intel Ice Lake (AVX512)             242 ms     316 ms   0.77x (-30.6%)
AMD Milan Zen 3 (AVX2)              252 ms     230 ms    1.10x (+8.7%)
Intel Haswell (AVX2)                369 ms     456 ms   0.81x (-23.6%)
NVIDIA Grace (SVE2)                 336 ms     337 ms    1.00x (-0.3%)
ARM Neoverse-N1 (NEON)             1081 ms    1105 ms    0.98x (-2.2%)
======================================================================


Bug: 537818862
Change-Id: I3ddeddb040fc52a30dabe7ae2c71d16dda021568
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8343467
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard 2026-09-01 18:53:39 -07:00
parent e111db837a
commit f51ad56a3e
9 changed files with 232 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
include/libyuv/row.h
View File

@ -404,6 +404,7 @@ extern "C" {
#define HAS_ARGBMIRRORROW_AVX2
#define HAS_INTERPOLATEROW_AVX2
#define HAS_INTERPOLATEROW_16_AVX2
#define HAS_MULTIPLYROW_16_AVX2
#endif
// The following are available for AVX512 clang x64 platforms:
@ -422,6 +423,7 @@ extern "C" {
#define HAS_CONVERT8TO16ROW_AVX512BW
#define HAS_HALFROW_16TO8_AVX512BW
#define HAS_HALFWIDTHROW_16TO8_AVX512BW
#define HAS_MULTIPLYROW_16_AVX512BW
#endif
// The following are available for AVX512 clang x64 platforms:
@ -887,6 +889,7 @@ extern "C" {
#define HAS_HALFWIDTHROW_16TO8_RVV
#define HAS_I422TOAR30ROW_RVV
#define HAS_INTERPOLATEROW_RVV
#define HAS_MULTIPLYROW_16_RVV
#define HAS_RGBTOUV444MATRIXROW_RVV
#define HAS_RGBTOUVMATRIXROW_RVV
#define HAS_RGBTOYMATRIXROW_RVV
@ -3539,6 +3542,14 @@ void MultiplyRow_16_Any_AVX2(const uint16_t* src_ptr,
uint16_t* dst_ptr,
int scale,
int width);
void MultiplyRow_16_AVX512BW(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width);
void MultiplyRow_16_Any_AVX512BW(const uint16_t* src_ptr,
uint16_t* dst_ptr,
int scale,
int width);
void MultiplyRow_16_NEON(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
@ -3551,6 +3562,10 @@ void MultiplyRow_16_SME(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width);
void MultiplyRow_16_RVV(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width);
void DivideRow_16_C(const uint16_t* src_y,
uint16_t* dst_y,

4
source/convert.cc
View File

@ -2395,7 +2395,7 @@ int ARGBToI420Matrix(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) {
if (TestCpuFlag(kCpuHasNeonI8MM)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 16)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
@ -2614,7 +2614,7 @@ int ARGBToI420Alpha(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) {
if (TestCpuFlag(kCpuHasNeonI8MM)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 16)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;

8
source/convert_from_argb.cc
View File

@ -316,7 +316,7 @@ int ARGBToI422Matrix(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) {
if (TestCpuFlag(kCpuHasNeonI8MM)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 16)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
@ -502,7 +502,7 @@ int ARGBToNV12Matrix(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) {
if (TestCpuFlag(kCpuHasNeonI8MM)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 16)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
@ -752,7 +752,7 @@ int ARGBToNV16Matrix(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) {
if (TestCpuFlag(kCpuHasNeonI8MM)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 16)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
@ -1230,7 +1230,7 @@ int ARGBToNV21Matrix(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) {
if (TestCpuFlag(kCpuHasNeonI8MM)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 16)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;

13
source/planar_functions.cc
View File

@ -955,6 +955,14 @@ void ConvertToMSBPlane_16(const uint16_t* src_y,
}
}
#endif
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
MultiplyRow_16 = MultiplyRow_16_AVX512BW;
}
}
#endif
#if defined(HAS_MULTIPLYROW_16_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
MultiplyRow_16 = MultiplyRow_16_Any_NEON;
@ -968,6 +976,11 @@ void ConvertToMSBPlane_16(const uint16_t* src_y,
MultiplyRow_16 = MultiplyRow_16_SME;
}
#endif
#if defined(HAS_MULTIPLYROW_16_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
MultiplyRow_16 = MultiplyRow_16_RVV;
}
#endif
for (y = 0; y < height; ++y) {
MultiplyRow_16(src_y, dst_y, scale, width);

9
source/row_any.cc
View File

@ -1583,6 +1583,15 @@ ANY11C(Convert8To16Row_Any_NEON,
uint16_t,
15)
#endif
#ifdef HAS_MULTIPLYROW_16_AVX512BW
ANY11C(MultiplyRow_16_Any_AVX512BW,
MultiplyRow_16_AVX512BW,
2,
2,
uint16_t,
uint16_t,
63)
#endif
#ifdef HAS_MULTIPLYROW_16_AVX2
ANY11C(MultiplyRow_16_Any_AVX2,
MultiplyRow_16_AVX2,

34
source/row_gcc.cc
View File

@ -5789,10 +5789,8 @@ void MultiplyRow_16_AVX2(const uint16_t* src_y,
// 32 pixels per loop.
LABELALIGN
"1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vpmullw %%ymm3,%%ymm0,%%ymm0 \n"
"vpmullw %%ymm3,%%ymm1,%%ymm1 \n"
"vpmullw (%0),%%ymm3,%%ymm0 \n"
"vpmullw 0x20(%0),%%ymm3,%%ymm1 \n"
"vmovdqu %%ymm0,(%0,%1) \n"
"vmovdqu %%ymm1,0x20(%0,%1) \n"
"add $0x40,%0 \n"
@ -5807,6 +5805,34 @@ void MultiplyRow_16_AVX2(const uint16_t* src_y,
}
#endif // HAS_MULTIPLYROW_16_AVX2
#ifdef HAS_MULTIPLYROW_16_AVX512BW
void MultiplyRow_16_AVX512BW(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
asm volatile(
"vpbroadcastw %3,%%zmm2 \n"
// 64 pixels per loop.
LABELALIGN
"1: \n"
"vpmullw (%0),%%zmm2,%%zmm0 \n"
"vpmullw 0x40(%0),%%zmm2,%%zmm1 \n"
"add $0x80,%0 \n"
"vmovdqu16 %%zmm0,(%1) \n"
"vmovdqu16 %%zmm1,0x40(%1) \n"
"add $0x80,%1 \n"
"sub $0x40,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_y), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "r"(scale) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_MULTIPLYROW_16_AVX512BW
// Use scale to convert msb formats to lsb, depending how many bits there are:
// 512 = 9 bits
// 1024 = 10 bits

18
source/row_rvv.cc
View File

@ -1906,6 +1906,24 @@ void Convert8To16Row_RVV(const uint8_t* src_y,
}
#endif
#ifdef HAS_MULTIPLYROW_16_RVV
void MultiplyRow_16_RVV(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
size_t w = (size_t)width;
do {
size_t vl = __riscv_vsetvl_e16m8(w);
vuint16m8_t v_src = __riscv_vle16_v_u16m8(src_y, vl);
vuint16m8_t v_dst = __riscv_vmul_vx_u16m8(v_src, (uint16_t)scale, vl);
__riscv_vse16_v_u16m8(dst_y, v_dst, vl);
w -= vl;
src_y += vl;
dst_y += vl;
} while (w > 0);
}
#endif
#ifdef HAS_HALFROW_16TO8_RVV
void HalfRow_16To8_RVV(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,

22
source/row_win.cc
View File

@ -790,6 +790,28 @@ void Convert8To16Row_AVX2(const uint8_t* src_y,
}
#endif // HAS_CONVERT8TO16ROW_AVX2
#ifdef HAS_MULTIPLYROW_16_AVX2
LIBYUV_TARGET_AVX2
void MultiplyRow_16_AVX2(const uint16_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
__m256i scale256 = _mm256_set1_epi16((short)scale);
do {
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y);
__m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_y + 16));
ymm0 = _mm256_mullo_epi16(ymm0, scale256);
ymm1 = _mm256_mullo_epi16(ymm1, scale256);
_mm256_storeu_si256((__m256i*)dst_y, ymm0);
_mm256_storeu_si256((__m256i*)(dst_y + 16), ymm1);
src_y += 32;
dst_y += 32;
width -= 32;
} while (width > 0);
_mm256_zeroupper();
}
#endif // HAS_MULTIPLYROW_16_AVX2
#ifdef HAS_MERGEUVROW_AVX2
LIBYUV_TARGET_AVX2
void MergeUVRow_AVX2(const uint8_t* src_u,

125
unit_test/planar_test.cc
View File

@ -3768,11 +3768,12 @@ TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) {
}
#endif
// TODO(fbarchard): Improve test for more platforms.
#ifdef HAS_MULTIPLYROW_16_AVX2
#if defined(HAS_MULTIPLYROW_16_AVX512BW) || defined(HAS_MULTIPLYROW_16_AVX2) || \
defined(HAS_MULTIPLYROW_16_NEON) || defined(HAS_MULTIPLYROW_16_SME) || \
defined(HAS_MULTIPLYROW_16_RVV)
TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) {
// Round count up to multiple of 32
const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31;
// Round count up to multiple of 64
const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63;
align_buffer_page_end(src_pixels_y, kPixels * 2);
align_buffer_page_end(dst_pixels_y_opt, kPixels * 2);
@ -3785,13 +3786,59 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) {
MultiplyRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_c), 64, kPixels);
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
#endif
#if defined(HAS_MULTIPLYROW_16_AVX2)
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
#endif
#if defined(HAS_MULTIPLYROW_16_NEON)
int has_neon = TestCpuFlag(kCpuHasNEON);
#endif
#if defined(HAS_MULTIPLYROW_16_SME)
int has_sme = TestCpuFlag(kCpuHasSME);
#endif
#if defined(HAS_MULTIPLYROW_16_RVV)
int has_rvv = TestCpuFlag(kCpuHasRVV);
#endif
for (int i = 0; i < benchmark_iterations_; ++i) {
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
if (has_avx512) {
MultiplyRow_16_AVX512BW(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
kPixels);
} else
#endif
#if defined(HAS_MULTIPLYROW_16_AVX2)
if (has_avx2) {
MultiplyRow_16_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
kPixels);
} else {
} else
#endif
#if defined(HAS_MULTIPLYROW_16_SME)
if (has_sme) {
MultiplyRow_16_SME(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
kPixels);
} else
#endif
#if defined(HAS_MULTIPLYROW_16_NEON)
if (has_neon) {
MultiplyRow_16_NEON(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
kPixels);
} else
#endif
#if defined(HAS_MULTIPLYROW_16_RVV)
if (has_rvv) {
MultiplyRow_16_RVV(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
kPixels);
} else
#endif
{
MultiplyRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64,
kPixels);
@ -3806,7 +3853,73 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) {
free_aligned_buffer_page_end(dst_pixels_y_opt);
free_aligned_buffer_page_end(dst_pixels_y_c);
}
#endif // HAS_MULTIPLYROW_16_AVX2
#if defined(HAS_MULTIPLYROW_16_AVX512BW) || defined(HAS_MULTIPLYROW_16_AVX2) || \
defined(HAS_MULTIPLYROW_16_NEON)
TEST_F(LibYUVPlanarTest, MultiplyRow_16_Any) {
const int kMaxPixels = 256;
align_buffer_page_end(src_pixels_y, kMaxPixels * 2);
align_buffer_page_end(dst_pixels_y_opt, kMaxPixels * 2);
align_buffer_page_end(dst_pixels_y_c, kMaxPixels * 2);
for (int i = 0; i < kMaxPixels; ++i) {
reinterpret_cast<uint16_t*>(src_pixels_y)[i] = fastrand() & 1023;
}
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
#endif
#if defined(HAS_MULTIPLYROW_16_AVX2)
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
#endif
#if defined(HAS_MULTIPLYROW_16_NEON)
int has_neon = TestCpuFlag(kCpuHasNEON);
#endif
for (int width = 1; width <= 129; ++width) {
memset(dst_pixels_y_opt, 0, kMaxPixels * 2);
memset(dst_pixels_y_c, 1, kMaxPixels * 2);
MultiplyRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_c), 64, width);
#if defined(HAS_MULTIPLYROW_16_AVX512BW)
if (has_avx512) {
MultiplyRow_16_Any_AVX512BW(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt),
64, width);
} else
#endif
#if defined(HAS_MULTIPLYROW_16_AVX2)
if (has_avx2) {
MultiplyRow_16_Any_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt),
64, width);
} else
#endif
#if defined(HAS_MULTIPLYROW_16_NEON)
if (has_neon) {
MultiplyRow_16_Any_NEON(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt),
64, width);
} else
#endif
{
MultiplyRow_16_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 64, width);
}
for (int i = 0; i < width * 2; ++i) {
ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
}
}
free_aligned_buffer_page_end(src_pixels_y);
free_aligned_buffer_page_end(dst_pixels_y_opt);
free_aligned_buffer_page_end(dst_pixels_y_c);
}
#endif // HAS_MULTIPLYROW_16_AVX512BW || HAS_MULTIPLYROW_16_AVX2 || HAS_MULTIPLYROW_16_NEON
#endif // HAS_MULTIPLYROW_16_...
TEST_F(LibYUVPlanarTest, Convert16To8Plane) {
const int kPixels = benchmark_width_ * benchmark_height_;