coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

I410ToI420 use 1 step HalfWidthRow function for UV

Browse Source 
On Icelake
Was C+AVX512
I410ToI420_Opt (593 ms)
I412ToI420_Opt (581 ms)
Now AVX512
I410ToI420_Opt (241 ms)
I412ToI420_Opt (227 ms)
 60.28% HalfWidthRow_16To8_AVX512BW
 32.34% Convert16To8Row_AVX512BW

Samsung S25 Oryon
Was C+NEON
I410ToI420_Opt (141 ms)
I412ToI420_Opt (127 ms)
Now NEON
I410ToI420_Opt (89 ms)
I412ToI420_Opt (88 ms)

Vivo X300 lunix c1
Was C+NEON
I410ToI420_Opt (682 ms)
I412ToI420_Opt (679 ms)
With SVE2 (slow)
I410ToI420_Opt (622 ms)
I412ToI420_Opt (584 ms)
 65.59% HalfWidthRow_16To8_SVE2
 26.74% Convert16To8Row_SVE2
Now NEON
I410ToI420_Opt (412 ms)
I412ToI420_Opt (406 ms)

Bug: chromium:531172025
Change-Id: I1732157f6be660d2a01086f21daf85fe7f7d166f
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8230200
Commit-Queue: Frank Barchard <fbarchard@google.com>
Reviewed-by: Wan-Teh Chang <wtc@google.com>
This commit is contained in:
Frank Barchard 2026-08-10 18:03:00 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent c409140ff6
commit eac1ea9744
16 changed files with 895 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.chromium
View File

@ -1,6 +1,6 @@
Name: libyuv
URL: https://chromium.googlesource.com/libyuv/libyuv/
Version: 1956
Version: 1957
Revision: DEPS
License: BSD-3-Clause
License File: LICENSE

77
include/libyuv/row.h
View File

@ -206,6 +206,7 @@ extern "C" {
#define HAS_DETILETOYUY2_SSE2
#define HAS_HALFMERGEUVROW_SSSE3
#define HAS_HALFROW_16TO8_SSSE3
#define HAS_HALFWIDTHROW_16TO8_SSSE3
#define HAS_I210TOAR30ROW_SSSE3
#define HAS_I210TOARGBROW_SSSE3
#define HAS_I212TOAR30ROW_SSSE3
@ -289,6 +290,7 @@ extern "C" {
#define HAS_DIVIDEROW_16_AVX2
#define HAS_HALFMERGEUVROW_AVX2
#define HAS_HALFROW_16TO8_AVX2
#define HAS_HALFWIDTHROW_16TO8_AVX2
#define HAS_I210TOAR30ROW_AVX2
#define HAS_I210TOARGBROW_AVX2
#define HAS_I212TOAR30ROW_AVX2
@ -383,6 +385,7 @@ extern "C" {
#define HAS_ARGBTORGB24ROW_AVX512VBMI
#define HAS_CONVERT16TO8ROW_AVX512BW
#define HAS_HALFROW_16TO8_AVX512BW
#define HAS_HALFWIDTHROW_16TO8_AVX512BW
#endif
// The following are available for AVX512 clang x64 platforms:
@ -454,6 +457,7 @@ extern "C" {
#define HAS_HALFFLOATROW_NEON
#define HAS_HALFMERGEUVROW_NEON
#define HAS_HALFROW_16TO8_NEON
#define HAS_HALFWIDTHROW_16TO8_NEON
#define HAS_I400TOARGBROW_NEON
#define HAS_I422ALPHATOARGBROW_NEON
#define HAS_I422TOARGB1555ROW_NEON
@ -581,13 +585,14 @@ extern "C" {
#define HAS_ARGBTOYMATRIXROW_SVE2
#define HAS_AYUVTOUVROW_SVE2
#define HAS_AYUVTOVUROW_SVE2
#define HAS_CONVERT16TO8ROW_SVE2
// #define HAS_CONVERT16TO8ROW_SVE2 // Disabled: NEON version is faster for 128 bit vectors.
#define HAS_CONVERT8TO8ROW_SVE2
#define HAS_COPYROW_SVE2
#define HAS_DIVIDEROW_16_SVE2
#define HAS_HALFFLOATROW_SVE2
#define HAS_HALFMERGEUVROW_SVE2
#define HAS_HALFROW_16TO8_SVE2
// #define HAS_HALFROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors.
// #define HAS_HALFWIDTHROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors.
#define HAS_I210ALPHATOARGBROW_SVE2
#define HAS_I210TOAR30ROW_SVE2
#define HAS_I210TOARGBROW_SVE2
@ -642,6 +647,7 @@ extern "C" {
#define HAS_CONVERT8TO8ROW_SME
#define HAS_COPYROW_SME
#define HAS_HALFROW_16TO8_SME
#define HAS_HALFWIDTHROW_16TO8_SME
#define HAS_I210ALPHATOARGBROW_SME
#define HAS_I210TOAR30ROW_SME
#define HAS_I210TOARGBROW_SME
@ -835,6 +841,7 @@ extern "C" {
#define HAS_CONVERT16TO8ROW_RVV
#define HAS_COPYROW_RVV
#define HAS_HALFROW_16TO8_RVV
#define HAS_HALFWIDTHROW_16TO8_RVV
#define HAS_INTERPOLATEROW_RVV
#define HAS_RGBTOUV444MATRIXROW_RVV
#define HAS_RGBTOUVMATRIXROW_RVV
@ -3663,6 +3670,72 @@ void HalfRow_16To8_RVV(const uint16_t* src_uv,
int scale,
int width);
void HalfWidthRow_16To8_C(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_Odd_C(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_Any_SSSE3(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width);
void HalfWidthRow_16To8_Any_AVX2(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width);
void HalfWidthRow_16To8_Any_AVX512BW(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width);
void HalfWidthRow_16To8_NEON(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_Any_NEON(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width);
void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_SME(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfWidthRow_16To8_RVV(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void Convert8To8Row_C(const uint8_t* src_y,
uint8_t* dst_y,
int scale,

40
include/libyuv/scale_row.h
View File

@ -223,16 +223,6 @@ void ScalePlaneVertical_16(int src_height,
int wpp,
enum FilterMode filtering);
void ScalePlaneDown2_16To8(int src_width,
int src_height,
int dst_width,
int dst_height,
int src_stride,
int dst_stride,
const uint16_t* src_ptr,
uint8_t* dst_ptr,
int scale,
enum FilterMode filtering);
// Simplify the filtering based on scale factors.
enum FilterMode ScaleFilterReduce(int src_width,
@ -274,16 +264,6 @@ void ScaleRowDown2_16_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst,
int dst_width);
void ScaleRowDown2_16To8_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale);
void ScaleRowDown2_16To8_Odd_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale);
void ScaleRowDown2Linear_C(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
@ -292,16 +272,6 @@ void ScaleRowDown2Linear_16_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst,
int dst_width);
void ScaleRowDown2Linear_16To8_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale);
void ScaleRowDown2Linear_16To8_Odd_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale);
void ScaleRowDown2Box_C(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
@ -314,16 +284,6 @@ void ScaleRowDown2Box_16_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint16_t* dst,
int dst_width);
void ScaleRowDown2Box_16To8_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale);
void ScaleRowDown2Box_16To8_Odd_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale);
void ScaleRowDown4_C(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,

2
include/libyuv/version.h
View File

@ -11,6 +11,6 @@
#ifndef INCLUDE_LIBYUV_VERSION_H_
#define INCLUDE_LIBYUV_VERSION_H_
#define LIBYUV_VERSION 1956
#define LIBYUV_VERSION 1957
#endif // INCLUDE_LIBYUV_VERSION_H_

135
source/convert.cc
View File

@ -230,7 +230,14 @@ static int I41xToI420(const uint16_t* src_y,
int width,
int height,
int depth) {
int y;
const int scale = 1 << (24 - depth);
const int uv_width = SUBSAMPLE(width, 1, 1);
void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale,
int width) = Convert16To8Row_C;
void (*HalfWidthRow_16To8)(const uint16_t* src_uv, ptrdiff_t src_uv_stride,
uint8_t* dst_uv, int scale, int width) =
(width & 1) ? HalfWidthRow_16To8_Odd_C : HalfWidthRow_16To8_C;
if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
height == 0 || height == INT_MIN) {
@ -249,18 +256,124 @@ static int I41xToI420(const uint16_t* src_y,
src_stride_v = -src_stride_v;
}
{
const int uv_width = SUBSAMPLE(width, 1, 1);
const int uv_height = SUBSAMPLE(height, 1, 1);
if (dst_y) {
Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
height);
#if defined(HAS_CONVERT16TO8ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
Convert16To8Row = Convert16To8Row_Any_NEON;
if (IS_ALIGNED(width, 16)) {
Convert16To8Row = Convert16To8Row_NEON;
}
}
#endif
#if defined(HAS_CONVERT16TO8ROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
Convert16To8Row = Convert16To8Row_SVE2;
}
#endif
#if defined(HAS_CONVERT16TO8ROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
Convert16To8Row = Convert16To8Row_SME;
}
#endif
#if defined(HAS_CONVERT16TO8ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
Convert16To8Row = Convert16To8Row_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
Convert16To8Row = Convert16To8Row_SSSE3;
}
}
#endif
#if defined(HAS_CONVERT16TO8ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
Convert16To8Row = Convert16To8Row_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
Convert16To8Row = Convert16To8Row_AVX2;
}
}
#endif
#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
Convert16To8Row = Convert16To8Row_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
Convert16To8Row = Convert16To8Row_AVX512BW;
}
}
#endif
#if defined(HAS_CONVERT16TO8ROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
Convert16To8Row = Convert16To8Row_RVV;
}
#endif
if (IS_ALIGNED(width, 2)) {
#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_NEON;
if (IS_ALIGNED(uv_width, 8)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_NEON;
}
}
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_SVE2;
}
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_SME)
if (TestCpuFlag(kCpuHasSME)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_SME;
}
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_SSSE3;
if (IS_ALIGNED(uv_width, 8)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_SSSE3;
}
}
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_AVX2;
if (IS_ALIGNED(uv_width, 16)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX2;
}
}
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_Any_AVX512BW;
if (IS_ALIGNED(uv_width, 32)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_AVX512BW;
}
}
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
HalfWidthRow_16To8 = HalfWidthRow_16To8_RVV;
}
#endif
}
for (y = 0; y < height - 1; y += 2) {
HalfWidthRow_16To8(src_u, src_stride_u, dst_u, scale, uv_width);
HalfWidthRow_16To8(src_v, src_stride_v, dst_v, scale, uv_width);
if (dst_y) {
Convert16To8Row(src_y, dst_y, scale, width);
Convert16To8Row(src_y + src_stride_y, dst_y + dst_stride_y, scale, width);
src_y += src_stride_y * 2;
dst_y += dst_stride_y * 2;
}
src_u += src_stride_u * 2;
src_v += src_stride_v * 2;
dst_u += dst_stride_u;
dst_v += dst_stride_v;
}
if (height & 1) {
HalfWidthRow_16To8(src_u, 0, dst_u, scale, uv_width);
HalfWidthRow_16To8(src_v, 0, dst_v, scale, uv_width);
if (dst_y) {
Convert16To8Row(src_y, dst_y, scale, width);
}
ScalePlaneDown2_16To8(width, height, uv_width, uv_height, src_stride_u,
dst_stride_u, src_u, dst_u, scale, kFilterBilinear);
ScalePlaneDown2_16To8(width, height, uv_width, uv_height, src_stride_v,
dst_stride_v, src_v, dst_v, scale, kFilterBilinear);
}
return 0;
}

38
source/row_any.cc
View File

@ -1732,6 +1732,44 @@ ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15)
#undef ANY11HS
// Any 2 to 1 half-width row with scale param
#define ANY11HWS(NAMEANY, ANY_SIMD, TS, TD, MASK) \
void NAMEANY(const TS* src_ptr, ptrdiff_t src_stride, TD* dst_ptr, \
int scale, int width) { \
SIMD_ALIGNED(TS vin[128 * 2]); \
static_assert((MASK + 1) * 2 * sizeof(TS) <= sizeof(vin), \
"vin buffer too small"); \
SIMD_ALIGNED(TD vout[64]); \
static_assert((MASK + 1) * sizeof(TD) <= sizeof(vout), \
"vout buffer too small"); \
memset(vin, 0, sizeof(vin)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(src_ptr, src_stride, dst_ptr, scale, n); \
} \
ptrdiff_t np = n; \
memcpy(vin, src_ptr + np * 2, r * 2 * sizeof(TS)); \
memcpy(vin + 128, src_ptr + src_stride + np * 2, r * 2 * sizeof(TS)); \
ANY_SIMD(vin, 128, vout, scale, MASK + 1); \
memcpy(dst_ptr + np, vout, r * sizeof(TD)); \
}
#ifdef HAS_HALFWIDTHROW_16TO8_SSSE3
ANY11HWS(HalfWidthRow_16To8_Any_SSSE3, HalfWidthRow_16To8_SSSE3, uint16_t, uint8_t, 7)
#endif
#ifdef HAS_HALFWIDTHROW_16TO8_AVX2
ANY11HWS(HalfWidthRow_16To8_Any_AVX2, HalfWidthRow_16To8_AVX2, uint16_t, uint8_t, 15)
#endif
#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW
ANY11HWS(HalfWidthRow_16To8_Any_AVX512BW, HalfWidthRow_16To8_AVX512BW, uint16_t, uint8_t, 31)
#endif
#ifdef HAS_HALFWIDTHROW_16TO8_NEON
ANY11HWS(HalfWidthRow_16To8_Any_NEON, HalfWidthRow_16To8_NEON, uint16_t, uint8_t, 7)
#endif
#undef ANY11HWS
// Any 1 to 1 mirror.
#define ANY11M(NAMEANY, ANY_SIMD, BPP, MASK) \
void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \

57
source/row_common.cc
View File

@ -3677,6 +3677,63 @@ void HalfRow_16To8_C(const uint16_t* src_uv,
}
}
void HalfWidthRow_16To8_C(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* s = src_uv;
const uint16_t* t = src_uv + src_uv_stride;
int x;
for (x = 0; x < width - 1; x += 2) {
dst_uv[0] = STATIC_CAST(
uint8_t,
C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
dst_uv[1] = STATIC_CAST(
uint8_t,
C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale));
dst_uv += 2;
s += 4;
t += 4;
}
if (width & 1) {
dst_uv[0] = STATIC_CAST(
uint8_t,
C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
}
}
void HalfWidthRow_16To8_Odd_C(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* s = src_uv;
const uint16_t* t = src_uv + src_uv_stride;
int x;
width -= 1;
for (x = 0; x < width - 1; x += 2) {
dst_uv[0] = STATIC_CAST(
uint8_t,
C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
dst_uv[1] = STATIC_CAST(
uint8_t,
C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale));
dst_uv += 2;
s += 4;
t += 4;
}
if (width & 1) {
dst_uv[0] = STATIC_CAST(
uint8_t,
C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
dst_uv += 1;
s += 2;
t += 2;
}
dst_uv[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + t[0] + 1) >> 1, scale));
}
// C version 2x2 -> 2x1.
void InterpolateRow_C(uint8_t* dst_ptr,
const uint8_t* src_ptr,

166
source/row_gcc.cc
View File

@ -5438,6 +5438,65 @@ void HalfRow_16To8_SSSE3(const uint16_t* src_uv,
}
#endif // HAS_HALFROW_16TO8_SSSE3
#ifdef HAS_HALFWIDTHROW_16TO8_SSSE3
static const uvec8 kShuffleMaskHalfWidth16To8 = {
0u, 1u, 4u, 5u, 8u, 9u, 12u, 13u,
128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
asm volatile(
"movd %4,%%xmm2 \n"
"punpcklwd %%xmm2,%%xmm2 \n"
"pshufd $0x0,%%xmm2,%%xmm2 \n"
"pcmpeqw %%xmm5,%%xmm5 \n"
"psrlw $15,%%xmm5 \n"
"mov $2,%%eax \n"
"movd %%eax,%%xmm6 \n"
"pshufd $0x0,%%xmm6,%%xmm6 \n"
"movdqa %5,%%xmm7 \n"
// 8 pixels per loop.
LABELALIGN
"1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu (%0,%3,2),%%xmm3 \n"
"movdqu 0x10(%0,%3,2),%%xmm4 \n"
"add $0x20,%0 \n"
"pmaddwd %%xmm5,%%xmm0 \n"
"pmaddwd %%xmm5,%%xmm1 \n"
"pmaddwd %%xmm5,%%xmm3 \n"
"pmaddwd %%xmm5,%%xmm4 \n"
"paddd %%xmm3,%%xmm0 \n"
"paddd %%xmm4,%%xmm1 \n"
"paddd %%xmm6,%%xmm0 \n"
"paddd %%xmm6,%%xmm1 \n"
"psrld $2,%%xmm0 \n"
"psrld $2,%%xmm1 \n"
"pshufb %%xmm7,%%xmm0 \n"
"pshufb %%xmm7,%%xmm1 \n"
"punpcklqdq %%xmm1,%%xmm0 \n"
"pmulhuw %%xmm2,%%xmm0 \n"
"packuswb %%xmm0,%%xmm0 \n"
"movq %%xmm0,(%1) \n"
"add $0x08,%1 \n"
"sub $0x08,%2 \n"
"jg 1b \n"
: "+r"(src_uv), // %0
"+r"(dst_uv), // %1
"+r"(width) // %2
: "r"(src_uv_stride), // %3
"r"(scale), // %4
"m"(kShuffleMaskHalfWidth16To8) // %5
: "eax", "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7");
}
#endif // HAS_HALFWIDTHROW_16TO8_SSSE3
#ifdef HAS_HALFROW_16TO8_AVX2
void HalfRow_16To8_AVX2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
@ -5474,6 +5533,59 @@ void HalfRow_16To8_AVX2(const uint16_t* src_uv,
}
#endif // HAS_HALFROW_16TO8_AVX2
#ifdef HAS_HALFWIDTHROW_16TO8_AVX2
void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
asm volatile(
"vmovd %4,%%xmm2 \n"
"vpbroadcastw %%xmm2,%%ymm2 \n"
"vpcmpeqw %%ymm5,%%ymm5,%%ymm5 \n"
"vpsrlw $15,%%ymm5,%%ymm5 \n"
"mov $2,%%eax \n"
"vmovd %%eax,%%xmm6 \n"
"vpbroadcastd %%xmm6,%%ymm6 \n"
// 16 pixels per loop.
LABELALIGN
"1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vmovdqu (%0,%3,2),%%ymm3 \n"
"vmovdqu 0x20(%0,%3,2),%%ymm4 \n"
"add $0x40,%0 \n"
"vpmaddwd %%ymm5,%%ymm0,%%ymm0 \n"
"vpmaddwd %%ymm5,%%ymm1,%%ymm1 \n"
"vpmaddwd %%ymm5,%%ymm3,%%ymm3 \n"
"vpmaddwd %%ymm5,%%ymm4,%%ymm4 \n"
"vpaddd %%ymm3,%%ymm0,%%ymm0 \n"
"vpaddd %%ymm4,%%ymm1,%%ymm1 \n"
"vpaddd %%ymm6,%%ymm0,%%ymm0 \n"
"vpaddd %%ymm6,%%ymm1,%%ymm1 \n"
"vpsrld $2,%%ymm0,%%ymm0 \n"
"vpsrld $2,%%ymm1,%%ymm1 \n"
"vpackusdw %%ymm1,%%ymm0,%%ymm0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
"vpackuswb %%ymm0,%%ymm0,%%ymm0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vmovdqu %%xmm0,(%1) \n"
"add $0x10,%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_uv), // %0
"+r"(dst_uv), // %1
"+r"(width) // %2
: "r"(src_uv_stride), // %3
"r"(scale) // %4
: "eax", "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6");
}
#endif // HAS_HALFWIDTHROW_16TO8_AVX2
#ifdef HAS_HALFROW_16TO8_AVX512BW
void HalfRow_16To8_AVX512BW(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
@ -5511,6 +5623,60 @@ void HalfRow_16To8_AVX512BW(const uint16_t* src_uv,
}
#endif // HAS_HALFROW_16TO8_AVX512BW
#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW
void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* src_uv1 = src_uv + src_uv_stride;
asm volatile(
"vpbroadcastw %4,%%zmm2 \n"
"mov $1,%%eax \n"
"vmovd %%eax,%%xmm5 \n"
"vpbroadcastw %%xmm5,%%zmm5 \n"
"mov $2,%%eax \n"
"vmovd %%eax,%%xmm6 \n"
"vpbroadcastd %%xmm6,%%zmm6 \n"
// 32 pixels per loop.
LABELALIGN
"1: \n"
"vmovdqu64 (%0),%%zmm0 \n"
"vmovdqu64 0x40(%0),%%zmm1 \n"
"vmovdqu64 (%1),%%zmm3 \n"
"vmovdqu64 0x40(%1),%%zmm4 \n"
"add $0x80,%0 \n"
"add $0x80,%1 \n"
"vpmaddwd %%zmm5,%%zmm0,%%zmm0 \n"
"vpmaddwd %%zmm5,%%zmm1,%%zmm1 \n"
"vpmaddwd %%zmm5,%%zmm3,%%zmm3 \n"
"vpmaddwd %%zmm5,%%zmm4,%%zmm4 \n"
"vpaddd %%zmm3,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm4,%%zmm1,%%zmm1 \n"
"vpaddd %%zmm6,%%zmm0,%%zmm0 \n"
"vpaddd %%zmm6,%%zmm1,%%zmm1 \n"
"vpsrld $2,%%zmm0,%%zmm0 \n"
"vpsrld $2,%%zmm1,%%zmm1 \n"
"vpmovusdw %%zmm0,%%ymm0 \n"
"vpmovusdw %%zmm1,%%ymm1 \n"
"vinserti64x4 $1,%%ymm1,%%zmm0,%%zmm0 \n"
"vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
"vpmovuswb %%zmm0,(%2) \n"
"add $0x20,%2 \n"
"sub $0x20,%3 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_uv), // %0
"+r"(src_uv1), // %1
"+r"(dst_uv), // %2
"+r"(width) // %3
: "r"(scale) // %4
: "eax", "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6");
}
#endif // HAS_HALFWIDTHROW_16TO8_AVX512BW
// Use scale to convert to lsb formats depending how many bits there are:
// 512 = 9 bits
// 1024 = 10 bits

31
source/row_neon.cc
View File

@ -3710,6 +3710,37 @@ void HalfRow_16To8_NEON(const uint16_t* src_uv,
: "cc", "memory", "q0", "q1", "q2", "q3", "q4");
}
void HalfWidthRow_16To8_NEON(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* src_uv1 = src_uv + src_uv_stride;
const int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr
asm volatile(
"vdup.16 q4, %4 \n"
"1: \n"
"vld1.16 {q0, q1}, [%0]! \n"
"vld1.16 {q2, q3}, [%1]! \n"
"subs %3, %3, #8 \n" // 8 dst pixels per loop
"vpaddl.u16 q0, q0 \n"
"vpaddl.u16 q1, q1 \n"
"vpadal.u16 q0, q2 \n"
"vpadal.u16 q1, q3 \n"
"vrshrn.u32 d0, q0, #2 \n"
"vrshrn.u32 d1, q1, #2 \n"
"vshl.u16 q0, q0, q4 \n" // shr = q4 is negative
"vqmovn.u16 d0, q0 \n"
"vst1.8 {d0}, [%2]! \n"
"bgt 1b \n"
: "+r"(src_uv), // %0
"+r"(src_uv1), // %1
"+r"(dst_uv), // %2
"+r"(width) // %3
: "r"(shift) // %4
: "cc", "memory", "q0", "q1", "q2", "q3", "q4");
}
// Use scale to convert J420 to I420
// scale parameter is 8.8 fixed point but limited to 0 to 255
// Function is based on DivideRow, but adds a bias

33
source/row_neon64.cc
View File

@ -5135,6 +5135,39 @@ void HalfRow_16To8_NEON(const uint16_t* src_uv,
: "cc", "memory", "v0", "v1", "v2", "v3", "v4");
}
void HalfWidthRow_16To8_NEON(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* src_uv1 = src_uv + src_uv_stride;
const int shift = 23 - __builtin_clz((int32_t)scale);
asm volatile(
"dup v4.8h, %w4 \n"
"1: \n"
"ldp q0, q1, [%0], #32 \n"
"ldp q2, q3, [%1], #32 \n"
"subs %w3, %w3, #8 \n"
"uaddlp v0.4s, v0.8h \n"
"uaddlp v1.4s, v1.8h \n"
"prfm pldl1keep, [%0, 448] \n"
"uadalp v0.4s, v2.8h \n"
"uadalp v1.4s, v3.8h \n"
"prfm pldl1keep, [%1, 448] \n"
"rshrn v0.4h, v0.4s, #2 \n"
"rshrn2 v0.8h, v1.4s, #2 \n"
"uqshl v0.8h, v0.8h, v4.8h \n"
"uzp2 v0.16b, v0.16b, v0.16b \n"
"str d0, [%2], #8 \n"
"b.gt 1b \n"
: "+r"(src_uv), // %0
"+r"(src_uv1), // %1
"+r"(dst_uv), // %2
"+r"(width) // %3
: "r"(shift) // %4
: "cc", "memory", "v0", "v1", "v2", "v3", "v4");
}
// Use scale to convert J420 to I420
// scale parameter is 8.8 fixed point but limited to 0 to 255
// Function is based on DivideRow, but adds a bias

66
source/row_rvv.cc
View File

@ -1830,7 +1830,8 @@ void HalfRow_16To8_RVV(const uint16_t* src_uv,
vuint16m4_t v_src0 = __riscv_vle16_v_u16m4(src_uv, vl);
vuint16m4_t v_src1 = __riscv_vle16_v_u16m4(src_uv1, vl);
#ifdef LIBYUV_RVV_HAS_VXRM_ARG
vuint16m4_t v_avg = __riscv_vaaddu_vv_u16m4_vxrm(v_src0, v_src1, 0, vl);
vuint16m4_t v_avg =
__riscv_vaaddu_vv_u16m4(v_src0, v_src1, __RISCV_VXRM_RNU, vl);
#else
vuint16m4_t v_avg = __riscv_vaaddu_vv_u16m4(v_src0, v_src1, vl);
#endif
@ -1844,6 +1845,69 @@ void HalfRow_16To8_RVV(const uint16_t* src_uv,
}
#endif
#ifdef HAS_HALFWIDTHROW_16TO8_RVV
#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
void HalfWidthRow_16To8_RVV(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* s = src_uv;
const uint16_t* t = src_uv + src_uv_stride;
size_t w = (size_t)width;
const int shift = __builtin_clz((int32_t)scale) - 15;
do {
size_t vl = __riscv_vsetvl_e16m4(w);
vuint16m4x2_t v_s = __riscv_vlseg2e16_v_u16m4x2(s, vl);
vuint16m4x2_t v_t = __riscv_vlseg2e16_v_u16m4x2(t, vl);
vuint16m4_t v_s0 = __riscv_vget_v_u16m4x2_u16m4(v_s, 0);
vuint16m4_t v_s1 = __riscv_vget_v_u16m4x2_u16m4(v_s, 1);
vuint16m4_t v_t0 = __riscv_vget_v_u16m4x2_u16m4(v_t, 0);
vuint16m4_t v_t1 = __riscv_vget_v_u16m4x2_u16m4(v_t, 1);
vuint32m8_t v_s01 = __riscv_vwaddu_vv_u32m8(v_s0, v_s1, vl);
vuint32m8_t v_t01 = __riscv_vwaddu_vv_u32m8(v_t0, v_t1, vl);
vuint32m8_t v_st01 = __riscv_vadd_vv_u32m8(v_s01, v_t01, vl);
vuint16m4_t v_avg =
__riscv_vnclipu_wx_u16m4(v_st01, 2, __RISCV_VXRM_RNU, vl);
vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_avg, shift, vl);
__riscv_vse8_v_u8m2(dst_uv, v_dst, vl);
w -= vl;
s += 2 * vl;
t += 2 * vl;
dst_uv += vl;
} while (w > 0);
}
#else
void HalfWidthRow_16To8_RVV(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* s = src_uv;
const uint16_t* t = src_uv + src_uv_stride;
size_t w = (size_t)width;
const int shift = __builtin_clz((int32_t)scale) - 15;
asm volatile("csrwi vxrm, 0");
do {
size_t vl = __riscv_vsetvl_e16m4(w);
vuint16m4_t v_s0, v_s1, v_t0, v_t1;
__riscv_vlseg2e16_v_u16m4(&v_s0, &v_s1, s, vl);
__riscv_vlseg2e16_v_u16m4(&v_t0, &v_t1, t, vl);
vuint32m8_t v_s01 = __riscv_vwaddu_vv_u32m8(v_s0, v_s1, vl);
vuint32m8_t v_t01 = __riscv_vwaddu_vv_u32m8(v_t0, v_t1, vl);
vuint32m8_t v_st01 = __riscv_vadd_vv_u32m8(v_s01, v_t01, vl);
vuint16m4_t v_avg = __riscv_vnclipu_wx_u16m4(v_st01, 2, vl);
vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_avg, shift, vl);
__riscv_vse8_v_u8m2(dst_uv, v_dst, vl);
w -= vl;
s += 2 * vl;
t += 2 * vl;
dst_uv += vl;
} while (w > 0);
}
#endif
#endif
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

65
source/row_sme.cc
View File

@ -965,6 +965,71 @@ __arm_locally_streaming void HalfRow_16To8_SME(const uint16_t* src_ptr,
: "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0");
}
__arm_locally_streaming void HalfWidthRow_16To8_SME(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width) {
const uint16_t* src_ptr1 = src_ptr + src_stride;
const int shift = 23 - __builtin_clz((int32_t)scale);
int vl;
asm volatile(
"cnth %x[vl] \n"
"dup z31.h, %w[shift] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
"ptrue p0.h \n"
"1: \n"
"ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
"ld2h {z2.h, z3.h}, p0/z, [%[src_ptr1]] \n"
"incb %[src_ptr], all, mul #2 \n"
"incb %[src_ptr1], all, mul #2 \n"
"uaddlb z4.s, z0.h, z1.h \n"
"uaddlt z5.s, z0.h, z1.h \n"
"uaddlb z6.s, z2.h, z3.h \n"
"uaddlt z7.s, z2.h, z3.h \n"
"add z4.s, z4.s, z6.s \n"
"add z5.s, z5.s, z7.s \n"
"rshrnb z0.h, z4.s, #2 \n"
"rshrnt z0.h, z5.s, #2 \n"
"subs %w[width], %w[width], %w[vl] \n"
"uqshl z0.h, p0/m, z0.h, z31.h \n"
"shrnb z0.b, z0.h, #8 \n"
"st1b {z0.h}, p0, [%[dst_ptr]] \n"
"inch %[dst_ptr] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
"whilelt p0.h, wzr, %w[width] \n"
"ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
"ld2h {z2.h, z3.h}, p0/z, [%[src_ptr1]] \n"
"uaddlb z4.s, z0.h, z1.h \n"
"uaddlt z5.s, z0.h, z1.h \n"
"uaddlb z6.s, z2.h, z3.h \n"
"uaddlt z7.s, z2.h, z3.h \n"
"add z4.s, z4.s, z6.s \n"
"add z5.s, z5.s, z7.s \n"
"rshrnb z0.h, z4.s, #2 \n"
"rshrnt z0.h, z5.s, #2 \n"
"uqshl z0.h, p0/m, z0.h, z31.h \n"
"shrnb z0.b, z0.h, #8 \n"
"st1b {z0.h}, p0, [%[dst_ptr]] \n"
"99: \n"
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
: [shift] "r"(shift) // %[shift]
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31",
"p0");
}
__arm_locally_streaming void Convert8To8Row_SME(const uint8_t* src_y,
uint8_t* dst_y,
int scale,

91
source/row_sve.cc
View File

@ -1530,6 +1530,8 @@ void UYVYToUVRow_SVE2(const uint8_t* src_uyvy,
: "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
}
#ifdef HAS_CONVERT16TO8ROW_SVE2
// Disabled: NEON version is faster for 128 bit vectors.
void Convert16To8Row_SVE2(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
@ -1571,7 +1573,10 @@ void Convert16To8Row_SVE2(const uint16_t* src_y,
: [shift] "r"(shift) // %[shift]
: "cc", "memory", "z2", "z31", "p0");
}
#endif // HAS_CONVERT16TO8ROW_SVE2
#ifdef HAS_HALFROW_16TO8_SVE2
// Disabled: NEON version is faster for 128 bit vectors.
void HalfRow_16To8_SVE2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
@ -1621,6 +1626,92 @@ void HalfRow_16To8_SVE2(const uint16_t* src_uv,
: [shift] "r"(shift) // %[shift]
: "cc", "memory", "z2", "z3", "z31", "p0");
}
#endif // HAS_HALFROW_16TO8_SVE2
#ifdef HAS_HALFWIDTHROW_16TO8_SVE2
// Disabled: NEON version is faster for 128 bit vectors.
void HalfWidthRow_16To8_SVE2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* src_uv1 = src_uv + src_uv_stride;
const int shift = 23 - __builtin_clz((int32_t)scale);
int vl;
int vl_input;
asm volatile(
"cnth %x[vl] \n"
"dup z31.h, %w[shift] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
"ptrue p0.h \n"
"1: \n"
"ld1h {z0.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z1.h}, p0/z, [%[src_ptr], #1, mul vl] \n"
"ld1h {z2.h}, p0/z, [%[src_ptr1]] \n"
"ld1h {z3.h}, p0/z, [%[src_ptr1], #1, mul vl] \n"
"incb %[src_ptr], all, mul #4 \n"
"incb %[src_ptr1], all, mul #4 \n"
"uzp1 z4.h, z0.h, z1.h \n"
"uzp2 z5.h, z0.h, z1.h \n"
"uzp1 z6.h, z2.h, z3.h \n"
"uzp2 z7.h, z2.h, z3.h \n"
"uaddlb z0.s, z4.h, z5.h \n"
"uaddlt z1.s, z4.h, z5.h \n"
"uaddlb z2.s, z6.h, z7.h \n"
"uaddlt z3.s, z6.h, z7.h \n"
"add z0.s, z0.s, z2.s \n"
"add z1.s, z1.s, z3.s \n"
"rshrnb z0.h, z0.s, #2 \n"
"rshrnt z0.h, z1.s, #2 \n"
"subs %w[width], %w[width], %w[vl] \n"
"uqshl z0.h, p0/m, z0.h, z31.h \n"
"shrnb z0.b, z0.h, #8 \n"
"st1b {z0.h}, p0, [%[dst_ptr]] \n"
"inch %[dst_ptr] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
"lsl %w[vl_input], %w[width], #1 \n"
"whilelt p0.h, wzr, %w[vl_input] \n"
"whilelt p1.h, %w[vl], %w[vl_input] \n"
"whilelt p2.h, wzr, %w[width] \n"
"ld1h {z0.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z1.h}, p1/z, [%[src_ptr], #1, mul vl] \n"
"ld1h {z2.h}, p0/z, [%[src_ptr1]] \n"
"ld1h {z3.h}, p1/z, [%[src_ptr1], #1, mul vl] \n"
"uzp1 z4.h, z0.h, z1.h \n"
"uzp2 z5.h, z0.h, z1.h \n"
"uzp1 z6.h, z2.h, z3.h \n"
"uzp2 z7.h, z2.h, z3.h \n"
"uaddlb z0.s, z4.h, z5.h \n"
"uaddlt z1.s, z4.h, z5.h \n"
"uaddlb z2.s, z6.h, z7.h \n"
"uaddlt z3.s, z6.h, z7.h \n"
"add z0.s, z0.s, z2.s \n"
"add z1.s, z1.s, z3.s \n"
"rshrnb z0.h, z0.s, #2 \n"
"rshrnt z0.h, z1.s, #2 \n"
"uqshl z0.h, p2/m, z0.h, z31.h \n"
"shrnb z0.b, z0.h, #8 \n"
"st1b {z0.h}, p2, [%[dst_ptr]] \n"
"99: \n"
: [src_ptr] "+r"(src_uv), // %[src_ptr]
[src_ptr1] "+r"(src_uv1), // %[src_ptr1]
[dst_ptr] "+r"(dst_uv), // %[dst_ptr]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl), // %[vl]
[vl_input] "=&r"(vl_input) // %[vl_input]
: [shift] "r"(shift) // %[shift]
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z31",
"p0", "p1", "p2");
}
#endif // HAS_HALFWIDTHROW_16TO8_SVE2
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)

44
source/scale.cc
View File

@ -207,50 +207,6 @@ static void ScalePlaneDown2_16(int src_width,
}
}
void ScalePlaneDown2_16To8(int src_width,
int src_height,
int dst_width,
int dst_height,
int src_stride,
int dst_stride,
const uint16_t* src_ptr,
uint8_t* dst_ptr,
int scale,
enum FilterMode filtering) {
int y;
void (*ScaleRowDown2)(const uint16_t* src_ptr, ptrdiff_t src_stride,
uint8_t* dst_ptr, int dst_width, int scale) =
(src_width & 1)
? (filtering == kFilterNone
? ScaleRowDown2_16To8_Odd_C
: (filtering == kFilterLinear ? ScaleRowDown2Linear_16To8_Odd_C
: ScaleRowDown2Box_16To8_Odd_C))
: (filtering == kFilterNone
? ScaleRowDown2_16To8_C
: (filtering == kFilterLinear ? ScaleRowDown2Linear_16To8_C
: ScaleRowDown2Box_16To8_C));
ptrdiff_t row_stride = (ptrdiff_t)src_stride * 2;
(void)dst_height;
if (!filtering) {
src_ptr += src_stride; // Point to odd rows.
src_stride = 0;
}
if (filtering == kFilterLinear) {
src_stride = 0;
}
for (y = 0; y < src_height / 2; ++y) {
ScaleRowDown2(src_ptr, src_stride, dst_ptr, dst_width, scale);
src_ptr += row_stride;
dst_ptr += dst_stride;
}
if (src_height & 1) {
if (!filtering) {
src_ptr -= src_stride; // Point to last row.
}
ScaleRowDown2(src_ptr, 0, dst_ptr, dst_width, scale);
}
}
// Scale plane, 1/4
// This is an optimized version for scaling down a plane to 1/4 of

155
source/scale_common.cc
View File

@ -29,19 +29,6 @@ extern "C" {
#define STATIC_CAST(type, expr) (type)(expr)
#endif
// TODO(fbarchard): make clamp255 preserve negative values.
static __inline int32_t clamp255(int32_t v) {
return (-(v >= 255) | v) & 255;
}
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 32768 = 9 bits
// 16384 = 10 bits
// 4096 = 12 bits
// 256 = 16 bits
// TODO(fbarchard): change scale to bits
#define C16TO8(v, scale) clamp255(((v) * (scale)) >> 16)
static __inline int Abs(int v) {
return v >= 0 ? v : -v;
}
@ -81,49 +68,6 @@ void ScaleRowDown2_16_C(const uint16_t* src_ptr,
}
}
void ScaleRowDown2_16To8_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale) {
int x;
(void)src_stride;
assert(scale >= 256);
assert(scale <= 32768);
for (x = 0; x < dst_width - 1; x += 2) {
dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[1], scale));
dst[1] = STATIC_CAST(uint8_t, C16TO8(src_ptr[3], scale));
dst += 2;
src_ptr += 4;
}
if (dst_width & 1) {
dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[1], scale));
}
}
void ScaleRowDown2_16To8_Odd_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale) {
int x;
(void)src_stride;
assert(scale >= 256);
assert(scale <= 32768);
dst_width -= 1;
for (x = 0; x < dst_width - 1; x += 2) {
dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[1], scale));
dst[1] = STATIC_CAST(uint8_t, C16TO8(src_ptr[3], scale));
dst += 2;
src_ptr += 4;
}
if (dst_width & 1) {
dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[1], scale));
dst += 1;
src_ptr += 2;
}
dst[0] = STATIC_CAST(uint8_t, C16TO8(src_ptr[0], scale));
}
void ScaleRowDown2Linear_C(const uint8_t* src_ptr,
ptrdiff_t src_stride,
@ -161,51 +105,6 @@ void ScaleRowDown2Linear_16_C(const uint16_t* src_ptr,
}
}
void ScaleRowDown2Linear_16To8_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale) {
const uint16_t* s = src_ptr;
int x;
(void)src_stride;
assert(scale >= 256);
assert(scale <= 32768);
for (x = 0; x < dst_width - 1; x += 2) {
dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + s[1] + 1) >> 1, scale));
dst[1] = STATIC_CAST(uint8_t, C16TO8((s[2] + s[3] + 1) >> 1, scale));
dst += 2;
s += 4;
}
if (dst_width & 1) {
dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + s[1] + 1) >> 1, scale));
}
}
void ScaleRowDown2Linear_16To8_Odd_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale) {
const uint16_t* s = src_ptr;
int x;
(void)src_stride;
assert(scale >= 256);
assert(scale <= 32768);
dst_width -= 1;
for (x = 0; x < dst_width - 1; x += 2) {
dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + s[1] + 1) >> 1, scale));
dst[1] = STATIC_CAST(uint8_t, C16TO8((s[2] + s[3] + 1) >> 1, scale));
dst += 2;
s += 4;
}
if (dst_width & 1) {
dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + s[1] + 1) >> 1, scale));
dst += 1;
s += 2;
}
dst[0] = STATIC_CAST(uint8_t, C16TO8(s[0], scale));
}
void ScaleRowDown2Box_C(const uint8_t* src_ptr,
ptrdiff_t src_stride,
@ -269,60 +168,6 @@ void ScaleRowDown2Box_16_C(const uint16_t* src_ptr,
}
}
void ScaleRowDown2Box_16To8_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale) {
const uint16_t* s = src_ptr;
const uint16_t* t = src_ptr + src_stride;
int x;
assert(scale >= 256);
assert(scale <= 32768);
for (x = 0; x < dst_width - 1; x += 2) {
dst[0] = STATIC_CAST(uint8_t,
C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
dst[1] = STATIC_CAST(uint8_t,
C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale));
dst += 2;
s += 4;
t += 4;
}
if (dst_width & 1) {
dst[0] = STATIC_CAST(uint8_t,
C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
}
}
void ScaleRowDown2Box_16To8_Odd_C(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width,
int scale) {
const uint16_t* s = src_ptr;
const uint16_t* t = src_ptr + src_stride;
int x;
assert(scale >= 256);
assert(scale <= 32768);
dst_width -= 1;
for (x = 0; x < dst_width - 1; x += 2) {
dst[0] = STATIC_CAST(uint8_t,
C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
dst[1] = STATIC_CAST(uint8_t,
C16TO8((s[2] + s[3] + t[2] + t[3] + 2) >> 2, scale));
dst += 2;
s += 4;
t += 4;
}
if (dst_width & 1) {
dst[0] = STATIC_CAST(uint8_t,
C16TO8((s[0] + s[1] + t[0] + t[1] + 2) >> 2, scale));
dst += 1;
s += 2;
t += 2;
}
dst[0] = STATIC_CAST(uint8_t, C16TO8((s[0] + t[0] + 1) >> 1, scale));
}
void ScaleRowDown4_C(const uint8_t* src_ptr,
ptrdiff_t src_stride,

148
unit_test/planar_test.cc
View File

@ -3973,6 +3973,154 @@ TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) {
}
#endif // HAS_CONVERT16TO8ROW_AVX2
#if defined(HAS_HALFWIDTHROW_16TO8_AVX2) || defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \
defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) || defined(HAS_HALFWIDTHROW_16TO8_NEON)
TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Opt) {
const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63;
align_buffer_page_end(src_pixels_y, kPixels * 4);
align_buffer_page_end(dst_pixels_y_opt, kPixels);
align_buffer_page_end(dst_pixels_y_c, kPixels);
MemRandomize(src_pixels_y, kPixels * 4);
for (int i = 0; i < kPixels * 2; ++i) {
reinterpret_cast<uint16_t*>(src_pixels_y)[i] &= 1023;
}
memset(dst_pixels_y_opt, 0, kPixels);
memset(dst_pixels_y_c, 1, kPixels);
HalfWidthRow_16To8_C(reinterpret_cast<const uint16_t*>(src_pixels_y), kPixels,
dst_pixels_y_c, 16384, kPixels);
#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
int has_neon = TestCpuFlag(kCpuHasNEON);
#endif
for (int i = 0; i < benchmark_iterations_; ++i) {
#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
if (has_avx512) {
HalfWidthRow_16To8_AVX512BW(
reinterpret_cast<const uint16_t*>(src_pixels_y), kPixels,
dst_pixels_y_opt, 16384, kPixels);
} else
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
if (has_avx2) {
HalfWidthRow_16To8_AVX2(reinterpret_cast<const uint16_t*>(src_pixels_y),
kPixels, dst_pixels_y_opt, 16384, kPixels);
} else
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
if (has_ssse3) {
HalfWidthRow_16To8_SSSE3(reinterpret_cast<const uint16_t*>(src_pixels_y),
kPixels, dst_pixels_y_opt, 16384, kPixels);
} else
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
if (has_neon) {
HalfWidthRow_16To8_NEON(reinterpret_cast<const uint16_t*>(src_pixels_y),
kPixels, dst_pixels_y_opt, 16384, kPixels);
} else
#endif
{
HalfWidthRow_16To8_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
kPixels, dst_pixels_y_opt, 16384, kPixels);
}
}
for (int i = 0; i < kPixels; ++i) {
ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
}
free_aligned_buffer_page_end(src_pixels_y);
free_aligned_buffer_page_end(dst_pixels_y_opt);
free_aligned_buffer_page_end(dst_pixels_y_c);
}
TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) {
const int kMaxPixels = 256;
align_buffer_page_end(src_pixels_y, kMaxPixels * 4);
align_buffer_page_end(dst_pixels_y_opt, kMaxPixels);
align_buffer_page_end(dst_pixels_y_c, kMaxPixels);
MemRandomize(src_pixels_y, kMaxPixels * 4);
for (int i = 0; i < kMaxPixels * 2; ++i) {
reinterpret_cast<uint16_t*>(src_pixels_y)[i] &= 1023;
}
#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
int has_avx512 = TestCpuFlag(kCpuHasAVX512BW);
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
int has_neon = TestCpuFlag(kCpuHasNEON);
#endif
for (int width = 1; width <= 129; ++width) {
memset(dst_pixels_y_opt, 0, kMaxPixels);
memset(dst_pixels_y_c, 1, kMaxPixels);
HalfWidthRow_16To8_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
kMaxPixels, dst_pixels_y_c, 16384, width);
#if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW)
if (has_avx512) {
HalfWidthRow_16To8_Any_AVX512BW(
reinterpret_cast<const uint16_t*>(src_pixels_y), kMaxPixels,
dst_pixels_y_opt, 16384, width);
} else
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_AVX2)
if (has_avx2) {
HalfWidthRow_16To8_Any_AVX2(
reinterpret_cast<const uint16_t*>(src_pixels_y), kMaxPixels,
dst_pixels_y_opt, 16384, width);
} else
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_SSSE3)
if (has_ssse3) {
HalfWidthRow_16To8_Any_SSSE3(
reinterpret_cast<const uint16_t*>(src_pixels_y), kMaxPixels,
dst_pixels_y_opt, 16384, width);
} else
#endif
#if defined(HAS_HALFWIDTHROW_16TO8_NEON)
if (has_neon) {
HalfWidthRow_16To8_Any_NEON(
reinterpret_cast<const uint16_t*>(src_pixels_y), kMaxPixels,
dst_pixels_y_opt, 16384, width);
} else
#endif
{
HalfWidthRow_16To8_C(reinterpret_cast<const uint16_t*>(src_pixels_y),
kMaxPixels, dst_pixels_y_opt, 16384, width);
}
for (int i = 0; i < width; ++i) {
ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
}
}
free_aligned_buffer_page_end(src_pixels_y);
free_aligned_buffer_page_end(dst_pixels_y_opt);
free_aligned_buffer_page_end(dst_pixels_y_c);
}
#endif
#ifdef HAS_UYVYTOYROW_NEON
TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) {
// NEON does multiple of 16, so round count up