coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Convert8To16 use shift to allow scale=65536

Browse Source 
- Add avx512bw, neon 32 bit and RVV versions
- Adapt SSE2 to psrlw instead of pmulhuw

CPU                             Before   After    Improvement
---------------------------------------------------------------
Granite Rapids (AVX10)           77 ms    75 ms   +2.6% (1.03x)
Emerald Rapids (AVX512)          75 ms    67 ms  +10.7% (1.12x)
Sapphire Rapids (AVX512)         91 ms    85 ms   +6.6% (1.07x)
AMD Venice Zen 6 (AVX512)        30 ms    30 ms    0.0% (1.00x)
AMD Turin Zen 5 (AVX512)         32 ms    32 ms    0.0% (1.00x)
AMD Genoa Zen 4 (AVX512)         35 ms    36 ms   -2.9% (0.97x)
Skylake-X (AVX512)              119 ms   119 ms    0.0% (1.00x)
Ice Lake (AVX512)                93 ms   104 ms  -11.8% (0.89x)
AMD Milan Zen 3 (AVX2)           43 ms    44 ms   -2.3% (0.98x)
Haswell (AVX2)                  129 ms   138 ms   -7.0% (0.93x)
NVIDIA Grace (SVE2)              42 ms    43 ms   -2.4% (0.98x)
Neoverse-N1 (NEON)               87 ms    87 ms    0.0% (1.00x)
---------------------------------------------------------------


Bug: None
Change-Id: Ia8f310a70c8b1229927fcdf117fe22e131d70ea9
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8330912
Commit-Queue: Frank Barchard <fbarchard@google.com>
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard 2026-09-01 11:10:30 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent 2f74e7a784
commit af714f0810
12 changed files with 199 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.chromium
View File

@ -1,6 +1,6 @@
Name: libyuv
URL: https://chromium.googlesource.com/libyuv/libyuv/
Version: 1966
Version: 1967
Revision: DEPS
License: BSD-3-Clause
License File: LICENSE

17
include/libyuv/row.h
View File

@ -369,6 +369,7 @@ extern "C" {
#define HAS_I422TOARGB1555ROW_AVX2
#define HAS_I422TOARGB4444ROW_AVX2
#define HAS_NV12TORGB565ROW_AVX2
#define HAS_CONVERT8TO16ROW_AVX2
#if defined(__x86_64__) || defined(_M_X64)
#define HAS_I422TOAR30ROW_AVX2
#define HAS_RAWTORGB24ROW_AVX2
@ -418,6 +419,7 @@ extern "C" {
#define HAS_RAWTORGB24ROW_AVX512VBMI
#define HAS_ARGBTORGB24ROW_AVX512VBMI
#define HAS_CONVERT16TO8ROW_AVX512BW
#define HAS_CONVERT8TO16ROW_AVX512BW
#define HAS_HALFROW_16TO8_AVX512BW
#define HAS_HALFWIDTHROW_16TO8_AVX512BW
#endif
@ -485,6 +487,7 @@ extern "C" {
#define HAS_AYUVTOYROW_NEON
#define HAS_BYTETOFLOATROW_NEON
#define HAS_CONVERT16TO8ROW_NEON
#define HAS_CONVERT8TO16ROW_NEON
#define HAS_CONVERT8TO8ROW_NEON
#define HAS_COPYROW_NEON
#define HAS_DETILEROW_16_NEON
@ -583,7 +586,6 @@ extern "C" {
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#define HAS_ABGRTOAR30ROW_NEON
#define HAS_ARGBTOAR30ROW_NEON
#define HAS_CONVERT8TO16ROW_NEON
#define HAS_I210ALPHATOARGBROW_NEON
#define HAS_I210TOAR30ROW_NEON
#define HAS_I210TOARGBROW_NEON
@ -879,6 +881,7 @@ extern "C" {
#define HAS_ARGBTOUVMATRIXROW_RVV
#define HAS_ARGBTOYMATRIXROW_RVV
#define HAS_CONVERT16TO8ROW_RVV
#define HAS_CONVERT8TO16ROW_RVV
#define HAS_COPYROW_RVV
#define HAS_HALFROW_16TO8_RVV
#define HAS_HALFWIDTHROW_16TO8_RVV
@ -3579,6 +3582,10 @@ void Convert8To16Row_AVX2(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
int width);
void Convert8To16Row_AVX512BW(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
int width);
void Convert8To16Row_Any_SSE2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int scale,
@ -3587,6 +3594,10 @@ void Convert8To16Row_Any_AVX2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int scale,
int width);
void Convert8To16Row_Any_AVX512BW(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int scale,
int width);
void Convert8To16Row_NEON(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
@ -3599,6 +3610,10 @@ void Convert8To16Row_SME(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
int width);
void Convert8To16Row_RVV(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
int width);
void Convert16To8Row_C(const uint16_t* src_y,
uint8_t* dst_y,

2
include/libyuv/version.h
View File

@ -11,6 +11,6 @@
#ifndef INCLUDE_LIBYUV_VERSION_H_
#define INCLUDE_LIBYUV_VERSION_H_
#define LIBYUV_VERSION 1967
#define LIBYUV_VERSION 1968
#endif // INCLUDE_LIBYUV_VERSION_H_

13
source/planar_functions.cc
View File

@ -235,6 +235,14 @@ void Convert8To16Plane(const uint8_t* src_y,
}
}
#endif
#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
Convert8To16Row = Convert8To16Row_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
Convert8To16Row = Convert8To16Row_AVX512BW;
}
}
#endif
#if defined(HAS_CONVERT8TO16ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
Convert8To16Row = Convert8To16Row_Any_NEON;
@ -248,6 +256,11 @@ void Convert8To16Plane(const uint8_t* src_y,
Convert8To16Row = Convert8To16Row_SME;
}
#endif
#if defined(HAS_CONVERT8TO16ROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
Convert8To16Row = Convert8To16Row_RVV;
}
#endif
// Convert plane
for (y = 0; y < height; ++y) {

9
source/row_any.cc
View File

@ -1565,6 +1565,15 @@ ANY11C(Convert8To16Row_Any_AVX2,
uint16_t,
31)
#endif
#ifdef HAS_CONVERT8TO16ROW_AVX512BW
ANY11C(Convert8To16Row_Any_AVX512BW,
Convert8To16Row_AVX512BW,
1,
2,
uint8_t,
uint16_t,
63)
#endif
#ifdef HAS_CONVERT8TO16ROW_NEON
ANY11C(Convert8To16Row_Any_NEON,
Convert8To16Row_NEON,

2
source/row_common.cc
View File

@ -3243,6 +3243,8 @@ void Convert16To8Row_C(const uint16_t* src_y,
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 1024 = 10 bits
// 4096 = 12 bits
// 65536 = 16 bits
void Convert8To16Row_C(const uint8_t* src_y,
uint16_t* dst_y,
int scale,

52
source/row_gcc.cc
View File

@ -6215,16 +6215,16 @@ void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv,
// 512 = 9 bits
// 1024 = 10 bits
// 4096 = 12 bits
// 65536 = 16 bits
void Convert8To16Row_SSE2(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
const int shift = __builtin_clz(scale) - 15;
asm volatile(
"movd %3,%%xmm2 \n"
"punpcklwd %%xmm2,%%xmm2 \n"
"pshufd $0x0,%%xmm2,%%xmm2 \n"
// 32 pixels per loop.
// 16 pixels per loop.
LABELALIGN
"1: \n"
"movdqu (%0),%%xmm0 \n"
@ -6232,8 +6232,8 @@ void Convert8To16Row_SSE2(const uint8_t* src_y,
"punpcklbw %%xmm0,%%xmm0 \n"
"punpckhbw %%xmm1,%%xmm1 \n"
"add $0x10,%0 \n"
"pmulhuw %%xmm2,%%xmm0 \n"
"pmulhuw %%xmm2,%%xmm1 \n"
"psrlw %%xmm2,%%xmm0 \n"
"psrlw %%xmm2,%%xmm1 \n"
"movdqu %%xmm0,(%1) \n"
"movdqu %%xmm1,0x10(%1) \n"
"add $0x20,%1 \n"
@ -6242,7 +6242,7 @@ void Convert8To16Row_SSE2(const uint8_t* src_y,
: "+r"(src_y), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "r"(scale) // %3
: "r"(shift) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2");
}
@ -6278,6 +6278,46 @@ void Convert8To16Row_AVX2(const uint8_t* src_y,
}
#endif // HAS_CONVERT8TO16ROW_AVX2
#ifdef HAS_CONVERT8TO16ROW_AVX512BW
// Use scale to convert to lsb formats depending how many bits there are:
// 512 = 9 bits
// 1024 = 10 bits
// 4096 = 12 bits
// 65536 = 16 bits
void Convert8To16Row_AVX512BW(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
const int shift = __builtin_clz(scale) - 15;
asm volatile(
"vpbroadcastw %4,%%zmm2 \n"
"vmovd %3,%%xmm3 \n"
// 64 pixels per loop.
LABELALIGN
"1: \n"
"vpmovzxbw (%0),%%zmm0 \n"
"vpmovzxbw 0x20(%0),%%zmm1 \n"
"add $0x40,%0 \n"
"vpmullw %%zmm2,%%zmm0,%%zmm0 \n"
"vpmullw %%zmm2,%%zmm1,%%zmm1 \n"
"vpsrlw %%xmm3,%%zmm0,%%zmm0 \n"
"vpsrlw %%xmm3,%%zmm1,%%zmm1 \n"
"vmovdqu16 %%zmm0,(%1) \n"
"vmovdqu16 %%zmm1,0x40(%1) \n"
"add $0x80,%1 \n"
"sub $0x40,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_y), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "r"(shift), // %3
"r"(0x0101) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
}
#endif // HAS_CONVERT8TO16ROW_AVX512BW
#ifdef HAS_SPLITRGBROW_SSSE3
// Shuffle table for converting RGB to Planar.
static const uvec8 kSplitRGBShuffle[9] = {

31
source/row_neon.cc
View File

@ -3831,6 +3831,37 @@ void Convert8To8Row_NEON(const uint8_t* src_y,
: "cc", "memory", "q0", "q1", "q2", "q3", "d8", "q5");
}
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 512 = 9 bits
// 1024 = 10 bits
// 4096 = 12 bits
// 65536 = 16 bits
void Convert8To16Row_NEON(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
// (src * 0x0101 * scale) >> 16.
// Since scale is a power of two, compute the shift to use to avoid needing
// to widen to int32.
const int shift = 15 - __builtin_clz(scale);
asm volatile(
"vdup.16 q2, %3 \n"
"1: \n"
"vld1.8 {q0}, [%0]! \n"
"vmov q1, q0 \n"
"vzip.8 q0, q1 \n"
"subs %2, %2, #16 \n"
"vshl.u16 q0, q0, q2 \n"
"vshl.u16 q1, q1, q2 \n"
"vst1.16 {q0, q1}, [%1]! \n"
"bgt 1b \n"
: "+r"(src_y), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "r"(shift) // %3
: "cc", "memory", "q0", "q1", "q2");
}
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__)..
#ifdef __cplusplus

2
source/row_neon64.cc
View File

@ -5269,6 +5269,8 @@ void Convert8To8Row_NEON(const uint8_t* src_y,
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 1024 = 10 bits
// 4096 = 12 bits
// 65536 = 16 bits
void Convert8To16Row_NEON(const uint8_t* src_y,
uint16_t* dst_y,
int scale,

26
source/row_rvv.cc
View File

@ -1813,6 +1813,32 @@ void Convert16To8Row_RVV(const uint16_t* src_y,
}
#endif
#ifdef HAS_CONVERT8TO16ROW_RVV
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 512 = 9 bits
// 1024 = 10 bits
// 4096 = 12 bits
// 65536 = 16 bits
void Convert8To16Row_RVV(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
size_t w = (size_t)width;
const int shift = __builtin_clz((int32_t)scale) - 15;
do {
size_t vl = __riscv_vsetvl_e8m2(w);
vuint8m2_t v_src = __riscv_vle8_v_u8m2(src_y, vl);
vuint16m4_t v_dst = __riscv_vwaddu_vx_u16m4(v_src, 0, vl);
v_dst = __riscv_vmul_vx_u16m4(v_dst, 0x0101, vl);
v_dst = __riscv_vsrl_vx_u16m4(v_dst, shift, vl);
__riscv_vse16_v_u16m4(dst_y, v_dst, vl);
w -= vl;
src_y += vl;
dst_y += vl;
} while (w > 0);
}
#endif
#ifdef HAS_HALFROW_16TO8_RVV
void HalfRow_16To8_RVV(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,

39
source/row_win.cc
View File

@ -21,6 +21,9 @@
#include <emmintrin.h>
#include <immintrin.h> // For AVX2 intrinsics
#include <tmmintrin.h> // For _mm_maddubs_epi16
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
#endif
#ifdef __cplusplus
namespace libyuv {
@ -762,6 +765,42 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
}
#endif // HAS_ARGBTOUVMATRIXROW_AVX2
#ifdef HAS_CONVERT8TO16ROW_AVX2
// Use scale to convert to lsb formats depending how many bits there are:
// 512 = 9 bits
// 1024 = 10 bits
// 4096 = 12 bits
// 65536 = 16 bits
LIBYUV_TARGET_AVX2
void Convert8To16Row_AVX2(const uint8_t* src_y,
uint16_t* dst_y,
int scale,
int width) {
#if defined(_MSC_VER) && !defined(__clang__)
unsigned long index;
_BitScanReverse(&index, (unsigned long)scale);
const int shift = 16 - (int)index;
#else
const int shift = __builtin_clz(scale) - 15;
#endif
__m128i shift128 = _mm_cvtsi32_si128(shift);
do {
__m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y);
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
__m256i row0 = _mm256_unpacklo_epi8(ymm0, ymm0);
__m256i row1 = _mm256_unpackhi_epi8(ymm0, ymm0);
row0 = _mm256_srl_epi16(row0, shift128);
row1 = _mm256_srl_epi16(row1, shift128);
_mm256_storeu_si256((__m256i*)dst_y, row0);
_mm256_storeu_si256((__m256i*)(dst_y + 16), row1);
src_y += 32;
dst_y += 32;
width -= 32;
} while (width > 0);
_mm256_zeroupper();
}
#endif // HAS_CONVERT8TO16ROW_AVX2
#ifdef HAS_MERGEUVROW_AVX2
LIBYUV_TARGET_AVX2
void MergeUVRow_AVX2(const uint8_t* src_u,

16
unit_test/planar_test.cc
View File

@ -4181,9 +4181,9 @@ TEST_F(LibYUVPlanarTest, Convert8To16Plane) {
#ifdef ENABLE_ROW_TESTS
// TODO(fbarchard): Improve test for more platforms.
#ifdef HAS_CONVERT8TO16ROW_AVX2
#if defined(HAS_CONVERT8TO16ROW_AVX512BW) || defined(HAS_CONVERT8TO16ROW_AVX2)
TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) {
const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31;
const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63;
align_buffer_page_end(src_pixels_y, kPixels);
align_buffer_page_end(dst_pixels_y_opt, kPixels * 2);
align_buffer_page_end(dst_pixels_y_c, kPixels * 2);
@ -4195,9 +4195,19 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) {
Convert8To16Row_C(src_pixels_y, reinterpret_cast<uint16_t*>(dst_pixels_y_c),
1024, kPixels);
#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
#endif
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
int has_sse2 = TestCpuFlag(kCpuHasSSE2);
for (int i = 0; i < benchmark_iterations_; ++i) {
#if defined(HAS_CONVERT8TO16ROW_AVX512BW)
if (has_avx512) {
Convert8To16Row_AVX512BW(src_pixels_y,
reinterpret_cast<uint16_t*>(dst_pixels_y_opt),
1024, kPixels);
} else
#endif
if (has_avx2) {
Convert8To16Row_AVX2(src_pixels_y,
reinterpret_cast<uint16_t*>(dst_pixels_y_opt), 1024,
@ -4221,7 +4231,7 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) {
free_aligned_buffer_page_end(dst_pixels_y_opt);
free_aligned_buffer_page_end(dst_pixels_y_c);
}
#endif // HAS_CONVERT8TO16ROW_AVX2
#endif // HAS_CONVERT8TO16ROW_AVX512BW || HAS_CONVERT8TO16ROW_AVX2
float TestScaleMaxSamples(int benchmark_width,
int benchmark_height,