coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Harden I21xToI420 against oversize heights

Browse Source 
I21xToI420 (reached via I210ToI420 / I212ToI420): fix integer overflows
in ScalePlaneVertical_16To8() by switching to HalfRow_16To8.

Add a unit test covering the I21x vertical-scale path. Co-authored-by:
Timothy Nikkel <tnikkel@mozilla.com> Test: libyuv_unittest
--gtest_filter=*.I21xToI420NoHeightOverflow

AMD Zen4
 Was LibYUVConvertTest.I210ToI420_Opt (81 ms)
  30.32% Convert16To8Row_AVX2
  29.81% Convert16To8Row_AVX512BW
  25.72% InterpolateRow_16To8_AVX2

 Now LibYUVConvertTest.I210ToI420_Opt (56 ms)
  41.32% Convert16To8Row_AVX512BW
  40.13% HalfRow_16To8_AVX512BW

Vivo X300 (ARM Lunix C1 Ultra)
 Was LibYUVConvertTest.I210ToI420_Opt (89 ms)
  38.95% InterpolateRow_16To8_NEON
  37.25% Convert16To8Row_NEON

 Now LibYUVConvertTest.I210ToI420_Opt (85 ms)
  47.26% Convert16To8Row_SVE2
  30.39% HalfRow_16To8_SVE2

Bug: chromium:531172025
Change-Id: I23e1215030c40b8577d58617098a8aeec27dd99a
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8221883
Reviewed-by: Wan-Teh Chang <wtc@google.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard 2026-08-08 11:52:15 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent 149568401f
commit 5dc477899b
16 changed files with 663 additions and 443 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.chromium
View File

@ -1,6 +1,6 @@
Name: libyuv
URL: https://chromium.googlesource.com/libyuv/libyuv/
Version: 1954
Version: 1955
Revision: DEPS
License: BSD-3-Clause
License File: LICENSE

118
include/libyuv/row.h
View File

@ -205,6 +205,7 @@ extern "C" {
#define HAS_DETILESPLITUVROW_SSSE3
#define HAS_DETILETOYUY2_SSE2
#define HAS_HALFMERGEUVROW_SSSE3
#define HAS_HALFROW_16TO8_SSSE3
#define HAS_I210TOAR30ROW_SSSE3
#define HAS_I210TOARGBROW_SSSE3
#define HAS_I212TOAR30ROW_SSSE3
@ -287,6 +288,7 @@ extern "C" {
#define HAS_DETILEROW_16_AVX
#define HAS_DIVIDEROW_16_AVX2
#define HAS_HALFMERGEUVROW_AVX2
#define HAS_HALFROW_16TO8_AVX2
#define HAS_I210TOAR30ROW_AVX2
#define HAS_I210TOARGBROW_AVX2
#define HAS_I212TOAR30ROW_AVX2
@ -297,7 +299,6 @@ extern "C" {
#define HAS_I422TOAR30ROW_AVX2
#define HAS_I422TOUYVYROW_AVX2
#define HAS_I422TOYUY2ROW_AVX2
#define HAS_INTERPOLATEROW_16TO8_AVX2
#define HAS_MERGEAR64ROW_AVX2
#define HAS_MERGEARGB16TO8ROW_AVX2
#define HAS_MERGEARGBROW_AVX2
@ -381,6 +382,7 @@ extern "C" {
#define HAS_RGB24TOARGBROW_AVX512BW
#define HAS_ARGBTORGB24ROW_AVX512VBMI
#define HAS_CONVERT16TO8ROW_AVX512BW
#define HAS_HALFROW_16TO8_AVX512BW
#endif
// The following are available for AVX512 clang x64 platforms:
@ -451,6 +453,7 @@ extern "C" {
#define HAS_DIVIDEROW_16_NEON
#define HAS_HALFFLOATROW_NEON
#define HAS_HALFMERGEUVROW_NEON
#define HAS_HALFROW_16TO8_NEON
#define HAS_I400TOARGBROW_NEON
#define HAS_I422ALPHATOARGBROW_NEON
#define HAS_I422TOARGB1555ROW_NEON
@ -578,11 +581,13 @@ extern "C" {
#define HAS_ARGBTOYMATRIXROW_SVE2
#define HAS_AYUVTOUVROW_SVE2
#define HAS_AYUVTOVUROW_SVE2
#define HAS_CONVERT16TO8ROW_SVE2
#define HAS_CONVERT8TO8ROW_SVE2
#define HAS_COPYROW_SVE2
#define HAS_DIVIDEROW_16_SVE2
#define HAS_HALFFLOATROW_SVE2
#define HAS_HALFMERGEUVROW_SVE2
#define HAS_HALFROW_16TO8_SVE2
#define HAS_I210ALPHATOARGBROW_SVE2
#define HAS_I210TOAR30ROW_SVE2
#define HAS_I210TOARGBROW_SVE2
@ -636,6 +641,7 @@ extern "C" {
#define HAS_CONVERT8TO16ROW_SME
#define HAS_CONVERT8TO8ROW_SME
#define HAS_COPYROW_SME
#define HAS_HALFROW_16TO8_SME
#define HAS_I210ALPHATOARGBROW_SME
#define HAS_I210TOAR30ROW_SME
#define HAS_I210TOARGBROW_SME
@ -657,7 +663,6 @@ extern "C" {
#define HAS_I444TOARGBROW_SME
#define HAS_I444TORGB24ROW_SME
#define HAS_INTERPOLATEROW_16_SME
#define HAS_INTERPOLATEROW_16TO8_SME
#define HAS_INTERPOLATEROW_SME
#define HAS_MERGEUVROW_16_SME
#define HAS_MERGEUVROW_SME
@ -677,7 +682,6 @@ extern "C" {
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#define HAS_GAUSSCOL_F32_NEON
#define HAS_GAUSSROW_F32_NEON
#define HAS_INTERPOLATEROW_16TO8_NEON
#define HAS_SCALESUMSAMPLES_NEON
#endif
@ -828,7 +832,9 @@ extern "C" {
#define HAS_ARGBTOUV444MATRIXROW_RVV
#define HAS_ARGBTOUVMATRIXROW_RVV
#define HAS_ARGBTOYMATRIXROW_RVV
#define HAS_CONVERT16TO8ROW_RVV
#define HAS_COPYROW_RVV
#define HAS_HALFROW_16TO8_RVV
#define HAS_INTERPOLATEROW_RVV
#define HAS_RGBTOUV444MATRIXROW_RVV
#define HAS_RGBTOUVMATRIXROW_RVV
@ -3587,6 +3593,75 @@ void Convert16To8Row_SME(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
int width);
void Convert16To8Row_SVE2(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
int width);
void Convert16To8Row_RVV(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
int width);
void HalfRow_16To8_C(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfRow_16To8_SSSE3(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfRow_16To8_AVX2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfRow_16To8_AVX512BW(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfRow_16To8_Any_SSSE3(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width);
void HalfRow_16To8_Any_AVX2(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width);
void HalfRow_16To8_Any_AVX512BW(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width);
void HalfRow_16To8_NEON(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfRow_16To8_Any_NEON(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width);
void HalfRow_16To8_SVE2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfRow_16To8_SME(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void HalfRow_16To8_RVV(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width);
void Convert8To8Row_C(const uint8_t* src_y,
uint8_t* dst_y,
@ -6379,43 +6454,6 @@ void InterpolateRow_16_SME(uint16_t* dst_ptr,
int width,
int source_y_fraction);
void InterpolateRow_16To8_C(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width,
int source_y_fraction);
void InterpolateRow_16To8_NEON(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width,
int source_y_fraction);
void InterpolateRow_16To8_Any_NEON(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width,
int source_y_fraction);
void InterpolateRow_16To8_SME(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width,
int source_y_fraction);
void InterpolateRow_16To8_AVX2(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width,
int source_y_fraction);
void InterpolateRow_16To8_Any_AVX2(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width,
int source_y_fraction);
// Sobel images.
void SobelXRow_C(const uint8_t* src_y0,
const uint8_t* src_y1,

14
include/libyuv/scale_row.h
View File

@ -223,20 +223,6 @@ void ScalePlaneVertical_16(int src_height,
int wpp,
enum FilterMode filtering);
void ScalePlaneVertical_16To8(int src_height,
int dst_width,
int dst_height,
int src_stride,
int dst_stride,
const uint16_t* src_argb,
uint8_t* dst_argb,
int x,
int y,
int dy,
int wpp,
int scale,
enum FilterMode filtering);
void ScalePlaneDown2_16To8(int src_width,
int src_height,
int dst_width,

2
include/libyuv/version.h
View File

@ -11,6 +11,6 @@
#ifndef INCLUDE_LIBYUV_VERSION_H_
#define INCLUDE_LIBYUV_VERSION_H_
#define LIBYUV_VERSION 1954
#define LIBYUV_VERSION 1955
#endif // INCLUDE_LIBYUV_VERSION_H_

158
source/convert.cc
View File

@ -232,16 +232,19 @@ static int I41xToI420(const uint16_t* src_y,
int depth) {
const int scale = 1 << (24 - depth);
if (width <= 0 || height == 0 || height == INT_MIN) {
if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
if (src_y) {
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_stride_y = -src_stride_y;
}
src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
}
@ -250,8 +253,10 @@ static int I41xToI420(const uint16_t* src_y,
const int uv_width = SUBSAMPLE(width, 1, 1);
const int uv_height = SUBSAMPLE(height, 1, 1);
Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
height);
if (dst_y) {
Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
height);
}
ScalePlaneDown2_16To8(width, height, uv_width, uv_height, src_stride_u,
dst_stride_u, src_u, dst_u, scale, kFilterBilinear);
ScalePlaneDown2_16To8(width, height, uv_width, uv_height, src_stride_v,
@ -275,35 +280,148 @@ static int I21xToI420(const uint16_t* src_y,
int width,
int height,
int depth) {
int y;
const int scale = 1 << (24 - depth);
const int uv_width = SUBSAMPLE(width, 1, 1);
void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale,
int width) = Convert16To8Row_C;
void (*HalfRow_16To8)(const uint16_t* src_uv, ptrdiff_t src_uv_stride,
uint8_t* dst_uv, int scale, int width) =
HalfRow_16To8_C;
if (width <= 0 || height == 0 || height == INT_MIN) {
if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
if (src_y) {
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_stride_y = -src_stride_y;
}
src_u = src_u + (ptrdiff_t)(height - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(height - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
}
{
const int uv_width = SUBSAMPLE(width, 1, 1);
const int uv_height = SUBSAMPLE(height, 1, 1);
const int dy = FixedDiv(height, uv_height);
#if defined(HAS_CONVERT16TO8ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
Convert16To8Row = Convert16To8Row_Any_NEON;
if (IS_ALIGNED(width, 16)) {
Convert16To8Row = Convert16To8Row_NEON;
}
}
#endif
#if defined(HAS_CONVERT16TO8ROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
Convert16To8Row = Convert16To8Row_SVE2;
}
#endif
#if defined(HAS_CONVERT16TO8ROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
Convert16To8Row = Convert16To8Row_SME;
}
#endif
#if defined(HAS_CONVERT16TO8ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
Convert16To8Row = Convert16To8Row_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
Convert16To8Row = Convert16To8Row_SSSE3;
}
}
#endif
#if defined(HAS_CONVERT16TO8ROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
Convert16To8Row = Convert16To8Row_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
Convert16To8Row = Convert16To8Row_AVX2;
}
}
#endif
#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
Convert16To8Row = Convert16To8Row_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
Convert16To8Row = Convert16To8Row_AVX512BW;
}
}
#endif
#if defined(HAS_CONVERT16TO8ROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
Convert16To8Row = Convert16To8Row_RVV;
}
#endif
Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
height);
ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_u,
dst_stride_u, src_u, dst_u, 0, 32768, dy,
/*bpp=*/1, scale, kFilterBilinear);
ScalePlaneVertical_16To8(height, uv_width, uv_height, src_stride_v,
dst_stride_v, src_v, dst_v, 0, 32768, dy,
/*bpp=*/1, scale, kFilterBilinear);
#if defined(HAS_HALFROW_16TO8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
HalfRow_16To8 = HalfRow_16To8_Any_NEON;
if (IS_ALIGNED(uv_width, 16)) {
HalfRow_16To8 = HalfRow_16To8_NEON;
}
}
#endif
#if defined(HAS_HALFROW_16TO8_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
HalfRow_16To8 = HalfRow_16To8_SVE2;
}
#endif
#if defined(HAS_HALFROW_16TO8_SME)
if (TestCpuFlag(kCpuHasSME)) {
HalfRow_16To8 = HalfRow_16To8_SME;
}
#endif
#if defined(HAS_HALFROW_16TO8_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
HalfRow_16To8 = HalfRow_16To8_Any_SSSE3;
if (IS_ALIGNED(uv_width, 16)) {
HalfRow_16To8 = HalfRow_16To8_SSSE3;
}
}
#endif
#if defined(HAS_HALFROW_16TO8_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
HalfRow_16To8 = HalfRow_16To8_Any_AVX2;
if (IS_ALIGNED(uv_width, 32)) {
HalfRow_16To8 = HalfRow_16To8_AVX2;
}
}
#endif
#if defined(HAS_HALFROW_16TO8_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
HalfRow_16To8 = HalfRow_16To8_Any_AVX512BW;
if (IS_ALIGNED(uv_width, 64)) {
HalfRow_16To8 = HalfRow_16To8_AVX512BW;
}
}
#endif
#if defined(HAS_HALFROW_16TO8_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
HalfRow_16To8 = HalfRow_16To8_RVV;
}
#endif
for (y = 0; y < height - 1; y += 2) {
HalfRow_16To8(src_u, src_stride_u, dst_u, scale, uv_width);
HalfRow_16To8(src_v, src_stride_v, dst_v, scale, uv_width);
if (dst_y) {
Convert16To8Row(src_y, dst_y, scale, width);
Convert16To8Row(src_y + src_stride_y, dst_y + dst_stride_y, scale, width);
src_y += src_stride_y * 2;
dst_y += dst_stride_y * 2;
}
src_u += src_stride_u * 2;
src_v += src_stride_v * 2;
dst_u += dst_stride_u;
dst_v += dst_stride_v;
}
if (height & 1) {
HalfRow_16To8(src_u, 0, dst_u, scale, uv_width);
HalfRow_16To8(src_v, 0, dst_v, scale, uv_width);
if (dst_y) {
Convert16To8Row(src_y, dst_y, scale, width);
}
}
return 0;
}

24
source/convert_from.cc
View File

@ -96,17 +96,21 @@ int I420ToI010(const uint8_t* src_y,
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
if (src_y) {
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_stride_y = -src_stride_y;
}
src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
}
// Convert Y plane.
Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 1024, width,
height);
if (dst_y) {
Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 1024, width,
height);
}
// Convert UV planes.
Convert8To16Plane(src_u, src_stride_u, dst_u, dst_stride_u, 1024, halfwidth,
halfheight);
@ -141,17 +145,21 @@ int I420ToI012(const uint8_t* src_y,
if (height < 0) {
height = -height;
halfheight = (height + 1) >> 1;
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
if (src_y) {
src_y = src_y + (ptrdiff_t)(height - 1) * src_stride_y;
src_stride_y = -src_stride_y;
}
src_u = src_u + (ptrdiff_t)(halfheight - 1) * src_stride_u;
src_v = src_v + (ptrdiff_t)(halfheight - 1) * src_stride_v;
src_stride_y = -src_stride_y;
src_stride_u = -src_stride_u;
src_stride_v = -src_stride_v;
}
// Convert Y plane.
Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 4096, width,
height);
if (dst_y) {
Convert8To16Plane(src_y, src_stride_y, dst_y, dst_stride_y, 4096, width,
height);
}
// Convert UV planes.
Convert8To16Plane(src_u, src_stride_u, dst_u, dst_stride_u, 4096, halfwidth,
halfheight);

69
source/row_any.cc
View File

@ -1689,48 +1689,43 @@ ANY11I(InterpolateRow_16_Any_AVX2,
#endif
#undef ANY11I
// Any 1 to 1 interpolate with scale param
#define ANY11IS(NAMEANY, ANY_SIMD, TD, TS, SBPP, BPP, MASK) \
void NAMEANY(TD* dst_ptr, const TS* src_ptr, ptrdiff_t src_stride, \
int scale, int width, int source_y_fraction) { \
SIMD_ALIGNED(TS vin[64 * 2]); \
SIMD_ALIGNED(TD vout[64]); \
memset(vin, 0, sizeof(vin)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(dst_ptr, src_ptr, src_stride, scale, n, source_y_fraction); \
} \
ptrdiff_t np = n; \
memcpy(vin, src_ptr + np * SBPP, r * SBPP * sizeof(TS)); \
if (source_y_fraction) { \
memcpy(vin + 64, src_ptr + src_stride + np * SBPP, \
r * SBPP * sizeof(TS)); \
} \
ANY_SIMD(vout, vin, 64, scale, MASK + 1, source_y_fraction); \
memcpy(dst_ptr + np * BPP, vout, r * BPP * sizeof(TD)); \
// Any 1 to 1 half row with scale param
#define ANY11HS(NAMEANY, ANY_SIMD, TS, TD, MASK) \
void NAMEANY(const TS* src_ptr, ptrdiff_t src_stride, TD* dst_ptr, \
int scale, int width) { \
SIMD_ALIGNED(TS vin[64 * 2]); \
static_assert((MASK + 1) * sizeof(TS) <= sizeof(vin) / 2, \
"vin buffer too small"); \
SIMD_ALIGNED(TD vout[64]); \
static_assert((MASK + 1) * sizeof(TD) <= sizeof(vout), \
"vout buffer too small"); \
memset(vin, 0, sizeof(vin)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
if (n > 0) { \
ANY_SIMD(src_ptr, src_stride, dst_ptr, scale, n); \
} \
ptrdiff_t np = n; \
memcpy(vin, src_ptr + np, r * sizeof(TS)); \
memcpy(vin + 64, src_ptr + src_stride + np, r * sizeof(TS)); \
ANY_SIMD(vin, 64, vout, scale, MASK + 1); \
memcpy(dst_ptr + np, vout, r * sizeof(TD)); \
}
#ifdef HAS_INTERPOLATEROW_16TO8_NEON
ANY11IS(InterpolateRow_16To8_Any_NEON,
InterpolateRow_16To8_NEON,
uint8_t,
uint16_t,
1,
1,
7)
#ifdef HAS_HALFROW_16TO8_SSSE3
ANY11HS(HalfRow_16To8_Any_SSSE3, HalfRow_16To8_SSSE3, uint16_t, uint8_t, 15)
#endif
#ifdef HAS_INTERPOLATEROW_16TO8_AVX2
ANY11IS(InterpolateRow_16To8_Any_AVX2,
InterpolateRow_16To8_AVX2,
uint8_t,
uint16_t,
1,
1,
31)
#ifdef HAS_HALFROW_16TO8_AVX2
ANY11HS(HalfRow_16To8_Any_AVX2, HalfRow_16To8_AVX2, uint16_t, uint8_t, 31)
#endif
#ifdef HAS_HALFROW_16TO8_AVX512BW
ANY11HS(HalfRow_16To8_Any_AVX512BW, HalfRow_16To8_AVX512BW, uint16_t, uint8_t, 63)
#endif
#ifdef HAS_HALFROW_16TO8_NEON
ANY11HS(HalfRow_16To8_Any_NEON, HalfRow_16To8_NEON, uint16_t, uint8_t, 15)
#endif
#undef ANY11IS
#undef ANY11HS
// Any 1 to 1 mirror.
#define ANY11M(NAMEANY, ANY_SIMD, BPP, MASK) \

71
source/row_common.cc
View File

@ -3664,11 +3664,11 @@ static void HalfRow_16_C(const uint16_t* src_uv,
}
}
static void HalfRow_16To8_C(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
void HalfRow_16To8_C(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
int x;
for (x = 0; x < width; ++x) {
dst_uv[x] = STATIC_CAST(
@ -3739,47 +3739,6 @@ void InterpolateRow_16_C(uint16_t* dst_ptr,
}
}
// C version 2x2 16 bit-> 2x1 8 bit.
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 32768 = 9 bits
// 16384 = 10 bits
// 4096 = 12 bits
// 256 = 16 bits
// TODO(fbarchard): change scale to bits
void InterpolateRow_16To8_C(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width,
int source_y_fraction) {
int y1_fraction = source_y_fraction;
int y0_fraction = 256 - y1_fraction;
const uint16_t* src_ptr1 = src_ptr + src_stride;
int x;
assert(source_y_fraction >= 0);
assert(source_y_fraction < 256);
if (source_y_fraction == 0) {
Convert16To8Row_C(src_ptr, dst_ptr, scale, width);
return;
}
if (source_y_fraction == 128) {
HalfRow_16To8_C(src_ptr, src_stride, dst_ptr, scale, width);
return;
}
for (x = 0; x < width; ++x) {
dst_ptr[0] = STATIC_CAST(
uint8_t,
C16TO8(
(src_ptr[0] * y0_fraction + src_ptr1[0] * y1_fraction + 128) >> 8,
scale));
src_ptr += 1;
src_ptr1 += 1;
dst_ptr += 1;
}
}
// Use first 4 shuffler values to reorder ARGB channels.
void ARGBShuffleRow_C(const uint8_t* src_argb,
uint8_t* dst_argb,
@ -4291,26 +4250,6 @@ void NV12ToRGB565Row_AVX2(const uint8_t* src_y,
}
#endif
#ifdef HAS_INTERPOLATEROW_16TO8_AVX2
void InterpolateRow_16To8_AVX2(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width,
int source_y_fraction) {
// Row buffer for intermediate 16 bit pixels.
SIMD_ALIGNED(uint16_t row[MAXTWIDTH]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
InterpolateRow_16_AVX2(row, src_ptr, src_stride, twidth, source_y_fraction);
Convert16To8Row_AVX2(row, dst_ptr, scale, twidth);
src_ptr += twidth;
dst_ptr += twidth;
width -= twidth;
}
}
#endif // HAS_INTERPOLATEROW_16TO8_AVX2
float ScaleSumSamples_C(const float* src, float* dst, float scale, int width) {
float fsum = 0.f;
int i;

116
source/row_gcc.cc
View File

@ -241,6 +241,7 @@ void RGB24ToARGBRow_SSSE3(const uint8_t* src_rgb24,
void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width) {
// TODO(fbarchard): Fix ASAN issue with kShuffleMaskRGB24ToARGB[1]
// Reference to prevent discarding of kShuffleMaskRGB24ToARGB[1] which is
// accessed via offset in assembly.
const uvec8* dummy = &kShuffleMaskRGB24ToARGB[1];
@ -2458,6 +2459,7 @@ void OMITFP I422ToRGB24Row_SSSE3(const uint8_t* y_buf,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
// TODO(fbarchard): Fix ASAN issue with kShuffleMaskARGBToRGB24[1]
// Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is
// accessed via offset in assembly.
const uvec8* dummy = &kShuffleMaskARGBToRGB24[1];
@ -2497,6 +2499,7 @@ void OMITFP I444ToRGB24Row_SSSE3(const uint8_t* y_buf,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
// TODO(fbarchard): Fix ASAN issue with kShuffleMaskARGBToRGB24[1]
// Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is
// accessed via offset in assembly.
const uvec8* dummy = &kShuffleMaskARGBToRGB24[1];
@ -3551,6 +3554,7 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width) {
// TODO(fbarchard): Fix ASAN issue with kShuffleMaskARGBToRGB24[1]
// Reference to prevent discarding of kShuffleMaskARGBToRGB24[1] which is
// accessed via offset in assembly.
const uvec8* dummy = &kShuffleMaskARGBToRGB24[1];
@ -5414,7 +5418,117 @@ void Convert16To8Row_AVX512BW(const uint16_t* src_y,
: "r"(scale) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_CONVERT16TO8ROW_AVX2
#endif // HAS_CONVERT16TO8ROW_AVX512BW
#ifdef HAS_HALFROW_16TO8_SSSE3
void HalfRow_16To8_SSSE3(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
asm volatile(
"movd %4,%%xmm2 \n"
"punpcklwd %%xmm2,%%xmm2 \n"
"pshufd $0x0,%%xmm2,%%xmm2 \n"
// 16 pixels per loop.
LABELALIGN
"1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu (%0,%3,2),%%xmm3 \n"
"movdqu 0x10(%0,%3,2),%%xmm4 \n"
"add $0x20,%0 \n"
"pavgw %%xmm3,%%xmm0 \n"
"pavgw %%xmm4,%%xmm1 \n"
"pmulhuw %%xmm2,%%xmm0 \n"
"pmulhuw %%xmm2,%%xmm1 \n"
"packuswb %%xmm1,%%xmm0 \n"
"movdqu %%xmm0,(%1) \n"
"add $0x10,%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
: "+r"(src_uv), // %0
"+r"(dst_uv), // %1
"+r"(width) // %2
: "r"(src_uv_stride), // %3
"r"(scale) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
}
#endif // HAS_HALFROW_16TO8_SSSE3
#ifdef HAS_HALFROW_16TO8_AVX2
void HalfRow_16To8_AVX2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
asm volatile(
"vmovd %4,%%xmm2 \n"
"vpbroadcastw %%xmm2,%%ymm2 \n"
// 32 pixels per loop.
LABELALIGN
"1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vpavgw (%0,%3,2),%%ymm0,%%ymm0 \n"
"vpavgw 0x20(%0,%3,2),%%ymm1,%%ymm1 \n"
"add $0x40,%0 \n"
"vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
"vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n"
"vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vmovdqu %%ymm0,(%1) \n"
"add $0x20,%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_uv), // %0
"+r"(dst_uv), // %1
"+r"(width) // %2
: "r"(src_uv_stride), // %3
"r"(scale) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_HALFROW_16TO8_AVX2
#ifdef HAS_HALFROW_16TO8_AVX512BW
void HalfRow_16To8_AVX512BW(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* src_uv1 = src_uv + src_uv_stride;
const int shift = 16 - __builtin_ctz((uint32_t)scale);
asm volatile(
"vpbroadcastw %4,%%zmm2 \n"
// 64 pixels per loop.
LABELALIGN
"1: \n"
"vmovdqu64 (%0),%%zmm0 \n"
"vmovdqu64 0x40(%0),%%zmm1 \n"
"vpavgw (%1),%%zmm0,%%zmm0 \n"
"vpavgw 0x40(%1),%%zmm1,%%zmm1 \n"
"add $0x80,%0 \n"
"add $0x80,%1 \n"
"vpsrlvw %%zmm2,%%zmm0,%%zmm0 \n"
"vpsrlvw %%zmm2,%%zmm1,%%zmm1 \n"
"vpmovuswb %%zmm0,(%2) \n"
"vpmovuswb %%zmm1,0x20(%2) \n"
"add $0x40,%2 \n"
"sub $0x40,%3 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_uv), // %0
"+r"(src_uv1), // %1
"+r"(dst_uv), // %2
"+r"(width) // %3
: "r"(shift) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_HALFROW_16TO8_AVX512BW
// Use scale to convert to lsb formats depending how many bits there are:
// 512 = 9 bits

29
source/row_neon.cc
View File

@ -3681,6 +3681,35 @@ void Convert16To8Row_NEON(const uint16_t* src_y,
: "cc", "memory", "q0", "q1", "q2");
}
void HalfRow_16To8_NEON(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* src_uv1 = src_uv + src_uv_stride;
const int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr
asm volatile(
"vdup.16 q4, %4 \n"
"1: \n"
"vld1.16 {q0, q1}, [%0]! \n"
"vld1.16 {q2, q3}, [%1]! \n"
"subs %3, %3, #16 \n" // 16 src pixels per loop
"vrhadd.u16 q0, q0, q2 \n"
"vrhadd.u16 q1, q1, q3 \n"
"vshl.u16 q0, q0, q4 \n" // shr = q4 is negative
"vshl.u16 q1, q1, q4 \n"
"vqmovn.u16 d0, q0 \n"
"vqmovn.u16 d1, q1 \n"
"vst1.8 {q0}, [%2]! \n"
"bgt 1b \n"
: "+r"(src_uv), // %0
"+r"(src_uv1), // %1
"+r"(dst_uv), // %2
"+r"(width) // %3
: "r"(shift) // %4
: "cc", "memory", "q0", "q1", "q2", "q3", "q4");
}
// Use scale to convert J420 to I420
// scale parameter is 8.8 fixed point but limited to 0 to 255
// Function is based on DivideRow, but adds a bias

110
source/row_neon64.cc
View File

@ -3728,86 +3728,6 @@ void InterpolateRow_16_NEON(uint16_t* dst_ptr,
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
}
// Bilinear filter 8x2 -> 8x1
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 32768 = 9 bits
// 16384 = 10 bits
// 4096 = 12 bits
// 256 = 16 bits
void InterpolateRow_16To8_NEON(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int dst_width,
int source_y_fraction) {
const int y1_fraction = source_y_fraction;
const int y0_fraction = 256 - y1_fraction;
const uint16_t* src_ptr1 = src_ptr + src_stride;
const int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr
asm volatile(
"dup v6.8h, %w6 \n"
"cmp %w4, #0 \n"
"b.eq 100f \n"
"cmp %w4, #128 \n"
"b.eq 50f \n"
"dup v5.8h, %w4 \n"
"dup v4.8h, %w5 \n"
// General purpose row blend.
"1: \n"
"ld1 {v0.8h}, [%1], #16 \n"
"ld1 {v1.8h}, [%2], #16 \n"
"subs %w3, %w3, #8 \n"
"umull v2.4s, v0.4h, v4.4h \n"
"prfm pldl1keep, [%1, 448] \n"
"umull2 v3.4s, v0.8h, v4.8h \n"
"prfm pldl1keep, [%2, 448] \n"
"umlal v2.4s, v1.4h, v5.4h \n"
"umlal2 v3.4s, v1.8h, v5.8h \n"
"rshrn v0.4h, v2.4s, #8 \n"
"rshrn2 v0.8h, v3.4s, #8 \n"
"ushl v0.8h, v0.8h, v6.8h \n"
"uqxtn v0.8b, v0.8h \n"
"st1 {v0.8b}, [%0], #8 \n"
"b.gt 1b \n"
"b 99f \n"
// Blend 50 / 50.
"50: \n"
"ld1 {v0.8h}, [%1], #16 \n"
"ld1 {v1.8h}, [%2], #16 \n"
"subs %w3, %w3, #8 \n"
"prfm pldl1keep, [%1, 448] \n"
"urhadd v0.8h, v0.8h, v1.8h \n"
"prfm pldl1keep, [%2, 448] \n"
"ushl v0.8h, v0.8h, v6.8h \n"
"uqxtn v0.8b, v0.8h \n"
"st1 {v0.8b}, [%0], #8 \n"
"b.gt 50b \n"
"b 99f \n"
// Blend 100 / 0 - Copy row unchanged.
"100: \n"
"ldr q0, [%1], #16 \n"
"ushl v0.8h, v0.8h, v2.8h \n" // shr = v2 is negative
"prfm pldl1keep, [%1, 448] \n"
"uqxtn v0.8b, v0.8h \n"
"subs %w3, %w3, #8 \n" // 8 src pixels per loop
"str d0, [%0], #8 \n" // store 8 pixels
"b.gt 100b \n"
"99: \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(src_ptr1), // %2
"+r"(dst_width) // %3
: "r"(y1_fraction), // %4
"r"(y0_fraction), // %5
"r"(shift) // %6
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
}
// dr * (256 - sa) / 256 + sr = dr - dr * sa / 256 + sr
void ARGBBlendRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
@ -5169,6 +5089,36 @@ void Convert16To8Row_NEON(const uint16_t* src_y,
: "cc", "memory", "v0", "v1", "v2");
}
void HalfRow_16To8_NEON(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* src_uv1 = src_uv + src_uv_stride;
const int shift = 23 - __builtin_clz((int32_t)scale);
asm volatile(
"dup v4.8h, %w4 \n"
"1: \n"
"ldp q0, q1, [%0], #32 \n"
"ldp q2, q3, [%1], #32 \n"
"subs %w3, %w3, #16 \n"
"urhadd v0.8h, v0.8h, v2.8h \n"
"urhadd v1.8h, v1.8h, v3.8h \n"
"prfm pldl1keep, [%0, 448] \n"
"prfm pldl1keep, [%1, 448] \n"
"uqshl v0.8h, v0.8h, v4.8h \n"
"uqshl v1.8h, v1.8h, v4.8h \n"
"uzp2 v0.16b, v0.16b, v1.16b \n"
"str q0, [%2], #16 \n"
"b.gt 1b \n"
: "+r"(src_uv), // %0
"+r"(src_uv1), // %1
"+r"(dst_uv), // %2
"+r"(width) // %3
: "r"(shift) // %4
: "cc", "memory", "v0", "v1", "v2", "v3", "v4");
}
// Use scale to convert J420 to I420
// scale parameter is 8.8 fixed point but limited to 0 to 255
// Function is based on DivideRow, but adds a bias

50
source/row_rvv.cc
View File

@ -1794,6 +1794,56 @@ void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
}
#endif
#ifdef HAS_CONVERT16TO8ROW_RVV
void Convert16To8Row_RVV(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
int width) {
size_t w = (size_t)width;
const int shift = __builtin_clz((int32_t)scale) - 15;
do {
size_t vl = __riscv_vsetvl_e16m4(w);
vuint16m4_t v_src = __riscv_vle16_v_u16m4(src_y, vl);
vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_src, shift, vl);
__riscv_vse8_v_u8m2(dst_y, v_dst, vl);
w -= vl;
src_y += vl;
dst_y += vl;
} while (w > 0);
}
#endif
#ifdef HAS_HALFROW_16TO8_RVV
void HalfRow_16To8_RVV(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
size_t w = (size_t)width;
const uint16_t* src_uv1 = src_uv + src_uv_stride;
const int shift = __builtin_clz((int32_t)scale) - 15;
#ifndef LIBYUV_RVV_HAS_VXRM_ARG
asm volatile("csrwi vxrm, 0");
#endif
do {
size_t vl = __riscv_vsetvl_e16m4(w);
vuint16m4_t v_src0 = __riscv_vle16_v_u16m4(src_uv, vl);
vuint16m4_t v_src1 = __riscv_vle16_v_u16m4(src_uv1, vl);
#ifdef LIBYUV_RVV_HAS_VXRM_ARG
vuint16m4_t v_avg = __riscv_vaaddu_vv_u16m4_vxrm(v_src0, v_src1, 0, vl);
#else
vuint16m4_t v_avg = __riscv_vaaddu_vv_u16m4(v_src0, v_src1, vl);
#endif
vuint8m2_t v_dst = __riscv_vnsrl_wx_u8m2(v_avg, shift, vl);
__riscv_vse8_v_u8m2(dst_uv, v_dst, vl);
w -= vl;
src_uv += vl;
src_uv1 += vl;
dst_uv += vl;
} while (w > 0);
}
#endif
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

101
source/row_sme.cc
View File

@ -907,11 +907,11 @@ __arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr,
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
}
__arm_locally_streaming static void HalfRow_16To8_SME(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width) {
__arm_locally_streaming void HalfRow_16To8_SME(const uint16_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int scale,
int width) {
const uint16_t* src_ptr1 = src_ptr + src_stride;
// 15 - clz(scale), + 8 to shift result into the high half of the lane to
@ -965,97 +965,6 @@ __arm_locally_streaming static void HalfRow_16To8_SME(uint8_t* dst_ptr,
: "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0");
}
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 32768 = 9 bits
// 16384 = 10 bits
// 4096 = 12 bits
// 256 = 16 bits
// TODO(fbarchard): change scale to bits
__arm_locally_streaming void InterpolateRow_16To8_SME(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
int scale,
int width,
int source_y_fraction) {
const int y1_fraction = source_y_fraction;
const int y0_fraction = 256 - y1_fraction;
const uint16_t* src_ptr1 = src_ptr + src_stride;
// y0_fraction == 0 is never called here.
if (y0_fraction == 128) {
HalfRow_16To8_SME(dst_ptr, src_ptr, src_stride, scale, width);
return;
}
if (y0_fraction == 256) {
Convert16To8Row_SME(src_ptr, dst_ptr, scale, width);
return;
}
// 15 - clz(scale), + 8 to shift result into the high half of the lane to
// saturate, then we can just use UZP2 to narrow rather than a pair of
// saturating narrow instructions.
const int shift = 23 - __builtin_clz((int32_t)scale);
int vl;
asm volatile(
"cnth %x[vl] \n"
"dup z31.h, %w[shift] \n"
"dup z0.h, %w[y0_fraction] \n"
"dup z1.h, %w[y1_fraction] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
// Run bulk of computation with an all-true predicate to avoid predicate
// generation overhead.
"ptrue p0.h \n"
"1: \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
"incb %[src_ptr] \n"
"incb %[src_ptr1] \n"
"umullb z4.s, z2.h, z0.h \n"
"umullt z2.s, z2.h, z0.h \n"
"subs %w[width], %w[width], %w[vl] \n"
"umlalb z4.s, z3.h, z1.h \n"
"umlalt z2.s, z3.h, z1.h \n"
"rshrnb z3.h, z4.s, #8 \n"
"rshrnt z3.h, z2.s, #8 \n"
"uqshl z3.h, p0/m, z3.h, z31.h \n"
"shrnb z3.b, z3.h, #8 \n"
"st1b {z3.h}, p0, [%[dst_ptr]] \n"
"inch %[dst_ptr] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
// Calculate a predicate for the final iteration to deal with the tail.
"whilelt p0.h, wzr, %w[width] \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
"umullb z4.s, z2.h, z0.h \n"
"umullt z2.s, z2.h, z0.h \n"
"umlalb z4.s, z3.h, z1.h \n"
"umlalt z2.s, z3.h, z1.h \n"
"rshrnb z3.h, z4.s, #8 \n"
"rshrnt z3.h, z2.s, #8 \n"
"uqshl z3.h, p0/m, z3.h, z31.h \n"
"shrnb z3.b, z3.h, #8 \n"
"st1b {z3.h}, p0, [%[dst_ptr]] \n"
"99: \n"
: [src_ptr] "+r"(src_ptr), // %[src_ptr]
[src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
[dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
: [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
[y1_fraction] "r"(y1_fraction), // %[y1_fraction]
[shift] "r"(shift) // %[shift]
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z31", "p0");
}
__arm_locally_streaming void Convert8To8Row_SME(const uint8_t* src_y,
uint8_t* dst_y,
int scale,

92
source/row_sve.cc
View File

@ -1474,6 +1474,98 @@ void UYVYToUVRow_SVE2(const uint8_t* src_uyvy,
: "memory", "cc", "z0", "z1", "z2", "z4", "z5", "p0", "p1");
}
void Convert16To8Row_SVE2(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
int width) {
const int shift = 23 - __builtin_clz((int32_t)scale);
int vl;
asm volatile(
"cnth %x[vl] \n"
"dup z31.h, %w[shift] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
"ptrue p0.h \n"
"1: \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"incb %[src_ptr] \n"
"subs %w[width], %w[width], %w[vl] \n"
"uqshl z2.h, p0/m, z2.h, z31.h \n"
"shrnb z2.b, z2.h, #8 \n"
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
"inch %[dst_ptr] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
"whilelt p0.h, wzr, %w[width] \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"uqshl z2.h, p0/m, z2.h, z31.h \n"
"shrnb z2.b, z2.h, #8 \n"
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
"99: \n"
: [src_ptr] "+r"(src_y), // %[src_ptr]
[dst_ptr] "+r"(dst_y), // %[dst_ptr]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
: [shift] "r"(shift) // %[shift]
: "cc", "memory", "z2", "z31", "p0");
}
void HalfRow_16To8_SVE2(const uint16_t* src_uv,
ptrdiff_t src_uv_stride,
uint8_t* dst_uv,
int scale,
int width) {
const uint16_t* src_uv1 = src_uv + src_uv_stride;
const int shift = 23 - __builtin_clz((int32_t)scale);
int vl;
asm volatile(
"cnth %x[vl] \n"
"dup z31.h, %w[shift] \n"
"subs %w[width], %w[width], %w[vl] \n"
"b.lt 2f \n"
"ptrue p0.h \n"
"1: \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
"incb %[src_ptr] \n"
"incb %[src_ptr1] \n"
"urhadd z2.h, p0/m, z2.h, z3.h \n"
"subs %w[width], %w[width], %w[vl] \n"
"uqshl z2.h, p0/m, z2.h, z31.h \n"
"shrnb z2.b, z2.h, #8 \n"
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
"inch %[dst_ptr] \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl] \n"
"b.eq 99f \n"
"whilelt p0.h, wzr, %w[width] \n"
"ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
"ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
"urhadd z2.h, p0/m, z2.h, z3.h \n"
"uqshl z2.h, p0/m, z2.h, z31.h \n"
"shrnb z2.b, z2.h, #8 \n"
"st1b {z2.h}, p0, [%[dst_ptr]] \n"
"99: \n"
: [src_ptr] "+r"(src_uv), // %[src_ptr]
[src_ptr1] "+r"(src_uv1), // %[src_ptr1]
[dst_ptr] "+r"(dst_uv), // %[dst_ptr]
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
: [shift] "r"(shift) // %[shift]
: "cc", "memory", "z2", "z3", "z31", "p0");
}
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#ifdef __cplusplus

69
source/scale_common.cc
View File

@ -1759,75 +1759,6 @@ void ScalePlaneVertical_16(int src_height,
}
}
// Use scale to convert lsb formats to msb, depending how many bits there are:
// 32768 = 9 bits
// 16384 = 10 bits
// 4096 = 12 bits
// 256 = 16 bits
// TODO(fbarchard): change scale to bits
void ScalePlaneVertical_16To8(int src_height,
int dst_width,
int dst_height,
int src_stride,
int dst_stride,
const uint16_t* src_argb,
uint8_t* dst_argb,
int x,
int y,
int dy,
int wpp, /* words per pixel. normally 1 */
int scale,
enum FilterMode filtering) {
// TODO(fbarchard): Allow higher wpp.
int dst_width_words = dst_width * wpp;
// TODO(https://crbug.com/libyuv/931): Add NEON 32 bit and AVX2 versions.
void (*InterpolateRow_16To8)(uint8_t* dst_argb, const uint16_t* src_argb,
ptrdiff_t src_stride, int scale, int dst_width,
int source_y_fraction) = InterpolateRow_16To8_C;
const int max_y = (src_height > 1) ? ((src_height - 1) << 16) - 1 : 0;
int j;
assert(wpp >= 1 && wpp <= 2);
assert(src_height != 0);
assert(dst_width > 0);
assert(dst_height > 0);
src_argb += (x >> 16) * wpp;
#if defined(HAS_INTERPOLATEROW_16TO8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow_16To8 = InterpolateRow_16To8_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
InterpolateRow_16To8 = InterpolateRow_16To8_NEON;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_16TO8_SME)
if (TestCpuFlag(kCpuHasSME)) {
InterpolateRow_16To8 = InterpolateRow_16To8_SME;
}
#endif
#if defined(HAS_INTERPOLATEROW_16TO8_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow_16To8 = InterpolateRow_16To8_Any_AVX2;
if (IS_ALIGNED(dst_width, 32)) {
InterpolateRow_16To8 = InterpolateRow_16To8_AVX2;
}
}
#endif
for (j = 0; j < dst_height; ++j) {
int yi;
int yf;
if (y > max_y) {
y = max_y;
}
yi = y >> 16;
yf = filtering ? ((y >> 8) & 255) : 0;
InterpolateRow_16To8(dst_argb, src_argb + yi * (ptrdiff_t)src_stride,
src_stride, scale, dst_width_words, yf);
dst_argb += dst_stride;
y += dy;
}
}
// Simplify the filtering based on scale factors.
enum FilterMode ScaleFilterReduce(int src_width,
int src_height,

81
unit_test/convert_test.cc
View File

@ -59,7 +59,7 @@ namespace libyuv {
#define TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \
DST_SUBSAMP_X, DST_SUBSAMP_Y, W1280, N, NEG, OFF, \
DST_SUBSAMP_X, DST_SUBSAMP_Y, W1280, N, NEG, OFF, DOY, \
SRC_DEPTH) \
TEST_F(LibYUVConvertTest, SRC_FMT_PLANAR##To##FMT_PLANAR##N) { \
static_assert(SRC_BPC == 1 || SRC_BPC == 2, "SRC BPC unsupported"); \
@ -113,7 +113,7 @@ namespace libyuv {
MaskCpuFlags(disable_cpu_flags_); \
SRC_FMT_PLANAR##To##FMT_PLANAR( \
src_y_p, kWidth, src_u_p, kSrcHalfWidth, src_v_p, kSrcHalfWidth, \
reinterpret_cast<DST_T*>(dst_y_c), kWidth, \
DOY ? reinterpret_cast<DST_T*>(dst_y_c) : NULL, kWidth, \
reinterpret_cast<DST_T*>(dst_u_c), kDstHalfWidth, \
reinterpret_cast<DST_T*>(dst_v_c), kDstHalfWidth, kWidth, \
NEG kHeight); \
@ -121,13 +121,15 @@ namespace libyuv {
for (int i = 0; i < benchmark_iterations_; ++i) { \
SRC_FMT_PLANAR##To##FMT_PLANAR( \
src_y_p, kWidth, src_u_p, kSrcHalfWidth, src_v_p, kSrcHalfWidth, \
reinterpret_cast<DST_T*>(dst_y_opt), kWidth, \
DOY ? reinterpret_cast<DST_T*>(dst_y_opt) : NULL, kWidth, \
reinterpret_cast<DST_T*>(dst_u_opt), kDstHalfWidth, \
reinterpret_cast<DST_T*>(dst_v_opt), kDstHalfWidth, kWidth, \
NEG kHeight); \
} \
for (int i = 0; i < kHeight * kWidth * DST_BPC; ++i) { \
ASSERT_EQ(dst_y_c[i], dst_y_opt[i]); \
if (DOY) { \
for (int i = 0; i < kHeight * kWidth * DST_BPC; ++i) { \
ASSERT_EQ(dst_y_c[i], dst_y_opt[i]); \
} \
} \
for (int i = 0; i < kDstHalfWidth * kDstHalfHeight * DST_BPC; ++i) { \
ASSERT_EQ(dst_u_c[i], dst_u_opt[i]); \
@ -150,23 +152,29 @@ namespace libyuv {
DST_SUBSAMP_X, DST_SUBSAMP_Y, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
benchmark_width_ + 1, _Any, +, 0, SRC_DEPTH) \
benchmark_width_ + 1, _Any, +, 0, 1, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
benchmark_width_, _Unaligned, +, 2, SRC_DEPTH) \
benchmark_width_, _Unaligned, +, 2, 1, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
benchmark_width_, _Invert, -, 0, SRC_DEPTH) \
benchmark_width_, _Invert, -, 0, 1, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
benchmark_width_, _Opt, +, 0, SRC_DEPTH)
benchmark_width_, _Opt, +, 0, 1, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
benchmark_width_, _NullY, +, 0, 0, SRC_DEPTH)
#else
#define TESTPLANARTOP(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, \
SRC_SUBSAMP_Y, FMT_PLANAR, DST_T, DST_BPC, \
DST_SUBSAMP_X, DST_SUBSAMP_Y, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
benchmark_width_, _Opt, +, 0, SRC_DEPTH)
benchmark_width_, _NullY, +, 0, 0, SRC_DEPTH) \
TESTPLANARTOPI(SRC_FMT_PLANAR, SRC_T, SRC_BPC, SRC_SUBSAMP_X, SRC_SUBSAMP_Y, \
FMT_PLANAR, DST_T, DST_BPC, DST_SUBSAMP_X, DST_SUBSAMP_Y, \
benchmark_width_, _Opt, +, 0, 1, SRC_DEPTH)
#endif
TESTPLANARTOP(I420, uint8_t, 1, 2, 2, I420, uint8_t, 1, 2, 2, 8)
@ -2789,6 +2797,59 @@ TEST_F(LibYUVConvertTest, ABGRToI420_Check) {
benchmark_cpu_info_);
}
// I210ToI420/I212ToI420 funnel through I21xToI420, which scales chroma
// vertically. Verify oversize heights do not overflow a signed int.
TEST_F(LibYUVConvertTest, I21xToI420NoHeightOverflow) {
int width = 4;
int height = 40000;
int half_width = (width + 1) / 2;
int half_height = (height + 1) / 2;
// I21x: 16-bit 4:2:2
align_buffer_page_end_16(src_y, width * height);
align_buffer_page_end_16(src_u, half_width * height);
align_buffer_page_end_16(src_v, half_width * height);
// I420: 8-bit 4:2:0
align_buffer_page_end(dst_y, width * height);
align_buffer_page_end(dst_u, half_width * half_height);
align_buffer_page_end(dst_v, half_width * half_height);
memset(src_y, 0, width * height * sizeof(uint16_t));
memset(src_u, 0, half_width * height * sizeof(uint16_t));
memset(src_v, 0, half_width * height * sizeof(uint16_t));
EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, 32767));
EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, 32768));
EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, 32769));
EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, -32767));
EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, -32768));
EXPECT_EQ(0, I210ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, -32769));
EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, 32767));
EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, 32768));
EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, 32769));
EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, -32767));
EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, -32768));
EXPECT_EQ(0, I212ToI420(src_y, 4, src_u, 2, src_v, 2, dst_y, 4, dst_u, 2,
dst_v, 2, 4, -32769));
free_aligned_buffer_page_end_16(src_y);
free_aligned_buffer_page_end_16(src_u);
free_aligned_buffer_page_end_16(src_v);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_u);
free_aligned_buffer_page_end(dst_v);
}
#endif // !defined(LEAN_TESTS)
} // namespace libyuv