coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Make RGBToY a 1 pass avx512bw

Browse Source 
- RGBToARGB use vpermi2q+vpshufb
- direct RGBToY for avx512 and rvv

+----------+----------+-------------+----------+---------+-----------+
| ARGBToY  | ARGBToUV | ARGBToUV444 | RGBToY   | RGBToUV | RGBToUV444|
+----------+----------+-------------+----------+---------+-----------+
| avx512bw | avx512bw | avx512bw    | avx512bw | 2 step  | 2 step    |
| avx2     | avx2     | avx2        | 2 step   | 2 step  | 2 step    |
| ssse3    | ssse3    | ssse3       | 2 step   | 2 step  | 2 step    |
| dot      | i8mm     | i8mm        | dot      | neon    | neon      |
| sve2     | sve2     | -           | -        | -       | -         |
| sme      | sme      | -           | -        | -       | -         |
| rvv      | rvv      | rvv         | rvv      | rvv     | rvv       |
+----------+----------+-------------+----------+---------+-----------+


Bug: 42280902
Change-Id: I9f83bf3039431f9fb5391bd7d9eb393e186d6bcf
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8175442
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
This commit is contained in:
Frank Barchard 2026-08-03 17:38:31 -07:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent 0148582bb8
commit 7138e0a2a7
17 changed files with 2312 additions and 977 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.chromium
View File

@ -1,6 +1,6 @@
Name: libyuv
URL: https://chromium.googlesource.com/libyuv/libyuv/
Version: 1952
Version: 1953
Revision: DEPS
License: BSD-3-Clause
License File: LICENSE

8
include/libyuv/cpu_support.h
View File

@ -84,10 +84,14 @@ extern "C" {
#define VISUALC_HAS_AVX2 1
#endif // VisualStudio >= 2012
// Clang 19 required for SME due to needing __arm_tpidr2_save from compiler-rt,
#if defined(LIBYUV_ENABLE_SME)
#undef LIBYUV_DISABLE_SME
#endif
// Clang 18+ required for SME due to needing __arm_tpidr2_save from compiler-rt,
// only enabled on Linux and Android (both define __linux__) for now.
#if !defined(LIBYUV_DISABLE_SME) && defined(__aarch64__) && \
defined(__linux__) && defined(__clang__) && (__clang_major__ >= 19)
defined(__linux__) && defined(__clang__) && (__clang_major__ >= 18)
#define CLANG_HAS_SME 1
#endif

153
include/libyuv/row.h
View File

@ -138,6 +138,9 @@ extern "C" {
#define HAS_MIRRORUVROW_AVX2
#define HAS_INTERPOLATEROW_16_AVX2
#define HAS_INTERPOLATEROW_AVX2
#define HAS_I422TOARGBROW_AVX2
#define HAS_NV12TOARGBROW_AVX2
#define HAS_NV21TOARGBROW_AVX2
#endif
#if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_GCC) && \
@ -236,8 +239,11 @@ extern "C" {
#define HAS_YUY2TONVUVROW_SSE2
// TODO: port row_win to use 8 bit coefficients.
#define HAS_ARGBTOYMATRIXROW_SSSE3
#define HAS_RGBTOYMATRIXROW_SSSE3
#define HAS_ARGBTOUVMATRIXROW_SSSE3
#define HAS_RGBTOUVMATRIXROW_SSSE3
#define HAS_ARGBTOUV444MATRIXROW_SSSE3
#define HAS_RGBTOUV444MATRIXROW_SSSE3
#if defined(__x86_64__) || !defined(__pic__)
// TODO(fbarchard): fix build error on android_full_debug=1
@ -268,6 +274,7 @@ extern "C" {
#define HAS_ARGB1555TOUVMATRIXROW_AVX2
#define HAS_ARGB4444TOUVMATRIXROW_AVX2
#define HAS_ARGBTOUV444MATRIXROW_AVX2
#define HAS_RGBTOUV444MATRIXROW_AVX2
#define HAS_ARGBTOYMATRIXROW_AVX2
#define HAS_RGBTOYMATRIXROW_AVX2
#define HAS_RGB565TOYMATRIXROW_AVX2
@ -339,38 +346,51 @@ extern "C" {
#define HAS_RGB24TOARGBROW_AVX512BW
#define HAS_ARGBSHUFFLEROW_AVX512BW
#define HAS_I422TORGB24ROW_AVX512VBMI
#define HAS_RGBTOYMATRIXROW_AVX512BW
#endif
#define HAS_ARGBTOYMATRIXROW_AVX2
#define HAS_RGBTOYMATRIXROW_AVX2
#define HAS_RGB565TOYMATRIXROW_AVX2
#define HAS_ARGB1555TOYMATRIXROW_AVX2
#define HAS_ARGB4444TOYMATRIXROW_AVX2
#define HAS_ARGBTOUVMATRIXROW_AVX2
#define HAS_ARGBTOUV444MATRIXROW_AVX2
#define HAS_RGBTOUV444MATRIXROW_AVX2
#define HAS_J400TOARGBROW_AVX2
#define HAS_I422TOARGBROW_AVX2
#define HAS_NV12TOARGBROW_AVX2
#define HAS_NV21TOARGBROW_AVX2
#define HAS_MERGEUVROW_AVX2
#define HAS_MIRRORROW_AVX2
#define HAS_MIRRORUVROW_AVX2
#define HAS_MIRRORSPLITUVROW_AVX2
#define HAS_RGB24MIRRORROW_AVX2
#define HAS_ARGBMIRRORROW_AVX2
#define HAS_INTERPOLATEROW_AVX2
#define HAS_INTERPOLATEROW_16_AVX2
#endif
// The following are available for AVX512 clang x86 platforms:
// The following are available for AVX512 clang x64 platforms:
// TODO(fbarchard): Port to GCC and Visual C
// TODO(b/42280744): re-enable HAS_ARGBTORGB24ROW_AVX512VBMI.
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(__i386__)) && defined(CLANG_HAS_AVX512) && \
(defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \
!defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_COPYROW_AVX512BW
#if defined(__x86_64__) || defined(_M_X64)
#define HAS_RAWTOARGBROW_AVX512BW
#define HAS_RGB24TOARGBROW_AVX512BW
#endif
#define HAS_ARGBTORGB24ROW_AVX512VBMI
#define HAS_CONVERT16TO8ROW_AVX512BW
#endif
// The following are available for AVX512 clang x64 platforms:
// TODO(fbarchard): Port to x86
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
(defined(CLANG_HAS_AVX512))
#if !defined(LIBYUV_DISABLE_X86) && \
(defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512)
#define HAS_I422TOARGBROW_AVX512BW
#define HAS_ARGBSHUFFLEROW_AVX512BW
#define HAS_ARGBTOUV444MATRIXROW_AVX512BW
#define HAS_RGBTOUV444MATRIXROW_AVX512BW
#define HAS_ARGBTOYMATRIXROW_AVX512BW
#define HAS_I422TORGB24ROW_AVX512VBMI
#define HAS_ARGBTOUVMATRIXROW_AVX512BW
@ -380,6 +400,7 @@ extern "C" {
#define HAS_MIRRORSPLITUVROW_AVX512BW
#define HAS_SPLITUVROW_AVX512BW
#define HAS_RGBTOUVMATRIXROW_AVX512BW
#define HAS_RGBTOYMATRIXROW_AVX512BW
#endif
// The following are available on Neon platforms:
@ -404,6 +425,7 @@ extern "C" {
#define HAS_ARGBTORGB565DITHERROW_NEON
#define HAS_ARGBTORGB565ROW_NEON
#define HAS_ARGBTOUV444MATRIXROW_NEON
#define HAS_RGBTOUV444MATRIXROW_NEON
#if !defined(__GNUC__) || defined(__clang__)
#define HAS_ARGBTOUVMATRIXROW_NEON
#define HAS_RGBTOUVMATRIXROW_NEON
@ -538,6 +560,7 @@ extern "C" {
#define HAS_BGRATOYROW_NEON_DOTPROD
#define HAS_RGBATOYJROW_NEON_DOTPROD
#define HAS_RGBATOYROW_NEON_DOTPROD
#define HAS_RGBTOYMATRIXROW_NEON_DOTPROD
#define HAS_ARGBCOLORMATRIXROW_NEON_I8MM
#define HAS_ARGBTOUVMATRIXROW_NEON_I8MM
@ -552,6 +575,7 @@ extern "C" {
#define HAS_ARGBTORGB565DITHERROW_SVE2
#define HAS_ARGBTORGB565ROW_SVE2
#define HAS_ARGBTOUVMATRIXROW_SVE2
#define HAS_ARGBTOYMATRIXROW_SVE2
#define HAS_AYUVTOUVROW_SVE2
#define HAS_AYUVTOVUROW_SVE2
#define HAS_CONVERT8TO8ROW_SVE2
@ -598,6 +622,7 @@ extern "C" {
defined(__aarch64__)
#define HAS_ARGBMULTIPLYROW_SME
#define HAS_ARGBTOUVMATRIXROW_SME
#define HAS_ARGBTOYMATRIXROW_SME
#define HAS_CONVERT16TO8ROW_SME
#define HAS_CONVERT8TO16ROW_SME
#define HAS_CONVERT8TO8ROW_SME
@ -760,6 +785,7 @@ extern "C" {
#define HAS_RAWTOUVROW_LASX
#define HAS_RGB24TOARGBROW_LASX
#define HAS_RGB24TOUVROW_LASX
#define HAS_RGBTOYMATRIXROW_LASX
#define HAS_RGB565TOARGBROW_LASX
#define HAS_RGB565TOUVROW_LASX
#define HAS_RGB565TOYROW_LASX
@ -790,9 +816,13 @@ extern "C" {
#define HAS_ARGBCOPYYTOALPHAROW_RVV
#define HAS_ARGBEXTRACTALPHAROW_RVV
#define HAS_ARGBTOAR64ROW_RVV
#define HAS_ARGBTOUV444MATRIXROW_RVV
#define HAS_ARGBTOUVMATRIXROW_RVV
#define HAS_ARGBTOYMATRIXROW_RVV
#define HAS_COPYROW_RVV
#define HAS_INTERPOLATEROW_RVV
#define HAS_RGBTOUV444MATRIXROW_RVV
#define HAS_RGBTOUVMATRIXROW_RVV
#define HAS_RGBTOYMATRIXROW_RVV
#define HAS_SPLITARGBROW_RVV
#define HAS_SPLITRGBROW_RVV
@ -1723,6 +1753,14 @@ void ARGBToUVMatrixRow_SME(const uint8_t* src_argb,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c);
void ARGBToYMatrixRow_SME(const uint8_t* src_argb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c);
void RGB24ToUVRow_NEON(const uint8_t* src_rgb24,
int src_stride_rgb24,
@ -1828,6 +1866,11 @@ void RGBToUVMatrixRow_C(const uint8_t* src_rgb,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_C(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGB565ToYMatrixRow_C(const uint8_t* src_rgb565,
uint8_t* dst_y,
int width,
@ -2080,6 +2123,24 @@ void RGBToUVMatrixRow_Any_NEON(const uint8_t* src_rgb,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_Any_NEON(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToYMatrixRow_NEON_DotProd(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c);
void RGBToYMatrixRow_Any_NEON_DotProd(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c);
void ARGBToYMatrixRow_NEON_DotProd(const uint8_t* src_argb,
uint8_t* dst_y,
@ -2109,6 +2170,44 @@ void RGBToYMatrixRow_RVV(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c);
void ARGBToUV444MatrixRow_RVV(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void ARGBToUVMatrixRow_RVV(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_RVV(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUVMatrixRow_RVV(const uint8_t* src_rgb,
int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToYMatrixRow_SSSE3(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c);
void RGBToYMatrixRow_Any_SSSE3(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c);
void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c);
void RGBToYMatrixRow_Any_AVX512BW(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c);
void ARGBToYMatrixRow_Any_NEON_DotProd(const uint8_t* src_argb,
uint8_t* dst_y,
@ -2146,6 +2245,33 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_SSSE3(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_AVX2(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_AVX512BW(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUVMatrixRow_SSSE3(const uint8_t* src_rgb,
int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUVMatrixRow_Any_SSSE3(const uint8_t* src_rgb,
int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void ARGBToUVMatrixRow_Any_SSSE3(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_u,
@ -2179,6 +2305,21 @@ void ARGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_argb,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_Any_SSSE3(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_Any_AVX2(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void ABGRToYJRow_C(const uint8_t* src_rgb, uint8_t* dst_y, int width);
void RGBAToYJRow_C(const uint8_t* src_rgb, uint8_t* dst_y, int width);

73
include/libyuv/row_sve.h
View File

@ -2144,6 +2144,79 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb,
"z27", "p0", "p1", "p2", "p3", "p4", "p5");
}
#define ARGBTOYMATRIX_SVE \
"ld1w {z0.s}, p1/z, [%[src]] \n" /* load 4*vl pixels */\
"ld1w {z1.s}, p2/z, [%[src], #1, mul vl] \n" \
"ld1w {z2.s}, p3/z, [%[src], #2, mul vl] \n" \
"ld1w {z3.s}, p4/z, [%[src], #3, mul vl] \n" \
"incb %[src], all, mul #4 \n" \
"fmov s16, wzr \n" \
"fmov s17, wzr \n" \
"fmov s18, wzr \n" \
"fmov s19, wzr \n" \
"udot z16.s, z0.b, z24.b \n" \
"udot z17.s, z1.b, z24.b \n" \
"udot z18.s, z2.b, z24.b \n" \
"udot z19.s, z3.b, z24.b \n" \
"uzp1 z16.h, z16.h, z17.h \n" \
"uzp1 z18.h, z18.h, z19.h \n" \
"add z16.h, z16.h, z25.h \n" \
"add z18.h, z18.h, z25.h \n" \
"uzp2 z16.b, z16.b, z18.b \n" \
"st1b {z16.b}, p5, [%[dst_y]] \n" \
"incb %[dst_y] \n" \
"subs %w[width], %w[width], %w[vl], lsl #2 \n"
static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c)
STREAMING_COMPATIBLE {
uint64_t vl;
asm("cntw %x0" : "=r"(vl));
asm volatile(
"ptrue p0.b \n"
"ld1rw {z24.s}, p0/z, [%[c]] \n"
"ld1rh {z25.h}, p0/z, [%[c], #48] \n"
"subs %w[width], %w[width], %w[vl], lsl #2 \n"
"b.lt 2f \n"
"ptrue p1.s \n"
"ptrue p2.s \n"
"ptrue p3.s \n"
"ptrue p4.s \n"
"ptrue p5.b \n"
"1: \n" //
ARGBTOYMATRIX_SVE
"b.gt 1b \n"
"2: \n"
"adds %w[width], %w[width], %w[vl], lsl #2 \n"
"b.eq 99f \n"
"3: \n"
"whilelt p1.s, wzr, %w[width] \n"
"whilelt p2.s, %w[vl], %w[width] \n"
"whilelt p3.s, %w[vl2], %w[width] \n"
"whilelt p4.s, %w[vl3], %w[width] \n"
"whilelt p5.b, wzr, %w[width] \n" //
ARGBTOYMATRIX_SVE
"b.gt 3b \n"
"99: \n"
: [src] "+r"(src_argb), // %[src]
[dst_y] "+r"(dst_y), // %[dst_y]
[width] "+r"(width) // %[width]
: [c] "r"(c), // %[c]
[vl] "r"(vl), // %[vl]
[vl2] "r"(vl * 2), // %[vl2]
[vl3] "r"(vl * 3) // %[vl3]
: "cc", "memory", "z0", "z1", "z2", "z3", "z16", "z17", "z18", "z19",
"z24", "z25", "p0", "p1", "p2", "p3", "p4", "p5");
}
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#ifdef __cplusplus

2
include/libyuv/version.h
View File

@ -11,6 +11,6 @@
#ifndef INCLUDE_LIBYUV_VERSION_H_
#define INCLUDE_LIBYUV_VERSION_H_
#define LIBYUV_VERSION 1952
#define LIBYUV_VERSION 1953
#endif // INCLUDE_LIBYUV_VERSION_H_

1055
source/convert.cc
View File

File diff suppressed because it is too large Load Diff

335
source/convert_from_argb.cc
View File

@ -98,6 +98,16 @@ int ARGBToI444Matrix(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
@ -159,6 +169,11 @@ int ARGBToI444Matrix(const uint8_t* src_argb,
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON_I8MM;
}
}
#endif
#if defined(HAS_ARGBTOUV444MATRIXROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RVV;
}
#endif
if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 ||
height == 0 || height == INT_MIN) {
@ -259,6 +274,16 @@ int ARGBToI422Matrix(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
@ -334,6 +359,11 @@ int ARGBToI422Matrix(const uint8_t* src_argb,
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
}
#endif
if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 ||
height == 0 || height == INT_MIN) {
@ -430,6 +460,16 @@ int ARGBToNV12Matrix(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
@ -497,6 +537,11 @@ int ARGBToNV12Matrix(const uint8_t* src_argb,
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
}
#endif
void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
uint8_t* dst_uv, int width) = MergeUVRow_C;
@ -645,6 +690,16 @@ int ARGBToNV21Matrix(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
@ -712,6 +767,11 @@ int ARGBToNV21Matrix(const uint8_t* src_argb,
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV;
}
#endif
void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
uint8_t* dst_vu, int width) = MergeUVRow_C;
@ -853,6 +913,16 @@ int ARGBToI400Matrix(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2;
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SME;
}
#endif
for (y = 0; y < height; ++y) {
ARGBToYMatrixRow(src_argb, dst_y, width, constants);
@ -2072,91 +2142,28 @@ int ARGBToAB64(const uint8_t* src_argb,
}
// Convert RAW to NV21 with Matrix.
LIBYUV_API
int RAWToNV21Matrix(const uint8_t* src_raw,
int src_stride_raw,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_vu,
int dst_stride_vu,
const struct ArgbConstants* argbconstants,
int width,
int height) {
static int RAWToNVMatrix(const uint8_t* src_raw,
int src_stride_raw,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_uv,
int dst_stride_uv,
const struct ArgbConstants* argbconstants,
int width,
int height,
int is_nv21) {
int y;
int halfwidth = (width + 1) >> 1;
void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
RAWToARGBRow_C;
void (*ARGBToUVMatrixRow)(const uint8_t* src_argb0, int src_stride_argb,
uint8_t* dst_u, uint8_t* dst_v, int width,
const struct ArgbConstants* c) =
ARGBToUVMatrixRow_C;
void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width,
const struct ArgbConstants* c) = ARGBToYMatrixRow_C;
void (*RGBToUVMatrixRow)(const uint8_t* src_rgb, int src_stride_rgb,
uint8_t* dst_u, uint8_t* dst_v, int width,
const struct ArgbConstants* c) =
RGBToUVMatrixRow_C;
void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width,
const struct ArgbConstants* c) = RGBToYMatrixRow_C;
void (*MergeUVRow)(const uint8_t* src_uj, const uint8_t* src_vj,
uint8_t* dst_vu, int width) = MergeUVRow_C;
#if defined(HAS_ARGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_NEON;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD)
if (TestCpuFlag(kCpuHasNeonDotProd)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_LSX;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_LASX;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBToYMatrixRow = ARGBToYMatrixRow_RVV;
}
#endif
if (!src_raw || !dst_y || !dst_vu || !argbconstants || width <= 0 ||
if (!src_raw || !dst_y || !dst_uv || !argbconstants || width <= 0 ||
height == 0 || height == INT_MIN) {
return -1;
}
@ -2167,115 +2174,89 @@ int RAWToNV21Matrix(const uint8_t* src_raw,
src_stride_raw = -src_stride_raw;
}
#if defined(HAS_RAWTOARGBROW_SSSE3)
#if defined(HAS_RGBTOYMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
RAWToARGBRow = RAWToARGBRow_Any_SSSE3;
RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
RAWToARGBRow = RAWToARGBRow_SSSE3;
RGBToYMatrixRow = RGBToYMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_RAWTOARGBROW_AVX2)
#if defined(HAS_RGBTOYMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
RAWToARGBRow = RAWToARGBRow_Any_AVX2;
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
RAWToARGBRow = RAWToARGBRow_AVX2;
RGBToYMatrixRow = RGBToYMatrixRow_AVX2;
}
}
#endif
#if defined(HAS_RAWTOARGBROW_AVX512BW)
#if defined(HAS_RGBTOYMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
RAWToARGBRow = RAWToARGBRow_Any_AVX512BW;
RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
RAWToARGBRow = RAWToARGBRow_AVX512BW;
RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW;
}
}
#endif
#if defined(HAS_RAWTOARGBROW_NEON)
#if defined(HAS_RGBTOYMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
RAWToARGBRow = RAWToARGBRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
RAWToARGBRow = RAWToARGBRow_NEON;
}
}
#endif
#if defined(HAS_RAWTOARGBROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
RAWToARGBRow = RAWToARGBRow_SVE2;
}
#endif
#if defined(HAS_RAWTOARGBROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
RAWToARGBRow = RAWToARGBRow_Any_LSX;
RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
RAWToARGBRow = RAWToARGBRow_LSX;
RGBToYMatrixRow = RGBToYMatrixRow_NEON;
}
}
#endif
#if defined(HAS_RAWTOARGBROW_LASX)
#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD)
if (TestCpuFlag(kCpuHasNeonDotProd)) {
RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd;
if (IS_ALIGNED(width, 16)) {
RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd;
}
}
#endif
#if defined(HAS_RGBTOYMATRIXROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
RAWToARGBRow = RAWToARGBRow_Any_LASX;
RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
RAWToARGBRow = RAWToARGBRow_LASX;
RGBToYMatrixRow = RGBToYMatrixRow_LASX;
}
}
#endif
#if defined(HAS_RAWTOARGBROW_RVV)
#if defined(HAS_RGBTOYMATRIXROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
RAWToARGBRow = RAWToARGBRow_RVV;
RGBToYMatrixRow = RGBToYMatrixRow_RVV;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM)
if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM;
if (IS_ALIGNED(width, 16)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
if (IS_ALIGNED(width, 2)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
if (IS_ALIGNED(width, 2)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3)
#if defined(HAS_RGBTOUVMATRIXROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 8)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3;
RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3;
if (IS_ALIGNED(width, 16)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2)
#if defined(HAS_RGBTOUVMATRIXROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2;
RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2;
if (IS_ALIGNED(width, 32)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2;
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX2;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW;
RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
}
#endif
#if defined(HAS_RGBTOUVMATRIXROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_Any_NEON;
if (IS_ALIGNED(width, 16)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_NEON;
}
}
#endif
@ -2332,45 +2313,55 @@ int RAWToNV21Matrix(const uint8_t* src_raw,
#endif
{
// Allocate 2 rows of ARGB.
const int row_size = (width * 4 + 31) & ~31;
align_buffer_64(row, row_size * 2);
// Allocate 1 row of U and 1 row of V.
align_buffer_64(row_u, halfwidth);
align_buffer_64(row_v, halfwidth);
if (!row || !row_u || !row_v) {
free_aligned_buffer_64(row);
if (!row_u || !row_v) {
free_aligned_buffer_64(row_u);
free_aligned_buffer_64(row_v);
return 1;
}
const uint8_t* first_chroma = is_nv21 ? row_v : row_u;
const uint8_t* second_chroma = is_nv21 ? row_u : row_v;
for (y = 0; y < height - 1; y += 2) {
RAWToARGBRow(src_raw, row, width);
RAWToARGBRow(src_raw + src_stride_raw, row + row_size, width);
ARGBToUVMatrixRow(row, row_size, row_u, row_v, width, argbconstants);
MergeUVRow(row_v, row_u, dst_vu, halfwidth);
ARGBToYMatrixRow(row, dst_y, width, argbconstants);
ARGBToYMatrixRow(row + row_size, dst_y + dst_stride_y, width,
RGBToUVMatrixRow(src_raw, src_stride_raw, row_u, row_v, width,
argbconstants);
MergeUVRow(first_chroma, second_chroma, dst_uv, halfwidth);
RGBToYMatrixRow(src_raw, dst_y, width, argbconstants);
RGBToYMatrixRow(src_raw + src_stride_raw, dst_y + dst_stride_y, width,
argbconstants);
src_raw += src_stride_raw * 2;
dst_y += dst_stride_y * 2;
dst_vu += dst_stride_vu;
dst_uv += dst_stride_uv;
}
if (height & 1) {
RAWToARGBRow(src_raw, row, width);
ARGBToUVMatrixRow(row, 0, row_u, row_v, width, argbconstants);
MergeUVRow(row_v, row_u, dst_vu, halfwidth);
ARGBToYMatrixRow(row, dst_y, width, argbconstants);
RGBToUVMatrixRow(src_raw, 0, row_u, row_v, width, argbconstants);
MergeUVRow(first_chroma, second_chroma, dst_uv, halfwidth);
RGBToYMatrixRow(src_raw, dst_y, width, argbconstants);
}
free_aligned_buffer_64(row_v);
free_aligned_buffer_64(row_u);
free_aligned_buffer_64(row);
}
return 0;
}
LIBYUV_API
int RAWToNV21Matrix(const uint8_t* src_raw,
int src_stride_raw,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_vu,
int dst_stride_vu,
const struct ArgbConstants* argbconstants,
int width,
int height) {
return RAWToNVMatrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_vu,
dst_stride_vu, argbconstants, width, height, 1);
}
LIBYUV_API
int RAWToJNV21(const uint8_t* src_raw,
int src_stride_raw,
@ -2381,7 +2372,7 @@ int RAWToJNV21(const uint8_t* src_raw,
int width,
int height) {
return RAWToNV21Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_vu,
dst_stride_vu, &kArgbJPEGConstants, width, height);
dst_stride_vu, &kAbgrJPEGConstants, width, height);
}
LIBYUV_API
@ -2394,7 +2385,7 @@ int RAWToNV21(const uint8_t* src_raw,
int width,
int height) {
return RAWToNV21Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_vu,
dst_stride_vu, &kArgbI601Constants, width, height);
dst_stride_vu, &kAbgrI601Constants, width, height);
}
LIBYUV_API
@ -2406,9 +2397,9 @@ int RGB24ToNV12(const uint8_t* src_rgb24,
int dst_stride_uv,
int width,
int height) {
return RAWToNV21Matrix(src_rgb24, src_stride_rgb24, dst_y, dst_stride_y,
dst_uv, dst_stride_uv, &kAbgrI601Constants, width,
height);
return RAWToNVMatrix(src_rgb24, src_stride_rgb24, dst_y, dst_stride_y,
dst_uv, dst_stride_uv, &kArgbI601Constants, width,
height, 0);
}
#ifdef __cplusplus

20
source/row_any.cc
View File

@ -2107,18 +2107,33 @@ ANY12MS(RGBToUVMatrixRow_Any_AVX512BW, RGBToUVMatrixRow_AVX512BW, 0, 3, 63)
#ifdef HAS_ARGBTOUVMATRIXROW_SSSE3
ANY12MS(ARGBToUVMatrixRow_Any_SSSE3, ARGBToUVMatrixRow_SSSE3, 0, 4, 7)
#endif
#ifdef HAS_RGBTOUVMATRIXROW_SSSE3
ANY12MS(RGBToUVMatrixRow_Any_SSSE3, RGBToUVMatrixRow_SSSE3, 0, 3, 7)
#endif
#ifdef HAS_ARGBTOUV444MATRIXROW_AVX2
ANY12M(ARGBToUV444MatrixRow_Any_AVX2, ARGBToUV444MatrixRow_AVX2, 4, 31)
#endif
#ifdef HAS_RGBTOUV444MATRIXROW_AVX2
ANY12M(RGBToUV444MatrixRow_Any_AVX2, RGBToUV444MatrixRow_AVX2, 3, 31)
#endif
#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW
ANY12M(ARGBToUV444MatrixRow_Any_AVX512BW, ARGBToUV444MatrixRow_AVX512BW, 4, 63)
#endif
#ifdef HAS_RGBTOUV444MATRIXROW_AVX512BW
ANY12M(RGBToUV444MatrixRow_Any_AVX512BW, RGBToUV444MatrixRow_AVX512BW, 3, 63)
#endif
#ifdef HAS_ARGBTOUV444MATRIXROW_SSSE3
ANY12M(ARGBToUV444MatrixRow_Any_SSSE3, ARGBToUV444MatrixRow_SSSE3, 4, 15)
#endif
#ifdef HAS_RGBTOUV444MATRIXROW_SSSE3
ANY12M(RGBToUV444MatrixRow_Any_SSSE3, RGBToUV444MatrixRow_SSSE3, 3, 15)
#endif
#ifdef HAS_ARGBTOUV444MATRIXROW_NEON
ANY12M(ARGBToUV444MatrixRow_Any_NEON, ARGBToUV444MatrixRow_NEON, 4, 7)
#endif
#ifdef HAS_RGBTOUV444MATRIXROW_NEON
ANY12M(RGBToUV444MatrixRow_Any_NEON, RGBToUV444MatrixRow_NEON, 3, 7)
#endif
#ifdef HAS_ARGBTOUV444MATRIXROW_NEON_I8MM
ANY12M(ARGBToUV444MatrixRow_Any_NEON_I8MM,
ARGBToUV444MatrixRow_NEON_I8MM,
@ -2144,6 +2159,7 @@ ANY12M(ARGBToUV444MatrixRow_Any_NEON_I8MM,
#ifdef HAS_ARGBTOYMATRIXROW_SSSE3
ANY11MC(ARGBToYMatrixRow_Any_SSSE3, ARGBToYMatrixRow_SSSE3, 4, 15)
ANY11MC(RGBToYMatrixRow_Any_SSSE3, RGBToYMatrixRow_SSSE3, 3, 15)
#endif
#ifdef HAS_ARGBTOYMATRIXROW_AVX2
ANY11MC(ARGBToYMatrixRow_Any_AVX2, ARGBToYMatrixRow_AVX2, 4, 31)
@ -2159,11 +2175,15 @@ ANY11MC(ARGB4444ToYMatrixRow_Any_AVX2, ARGB4444ToYMatrixRow_AVX2, 2, 31)
#ifdef HAS_ARGBTOYMATRIXROW_AVX512BW
ANY11MC(ARGBToYMatrixRow_Any_AVX512BW, ARGBToYMatrixRow_AVX512BW, 4, 63)
#endif
#ifdef HAS_RGBTOYMATRIXROW_AVX512BW
ANY11MC(RGBToYMatrixRow_Any_AVX512BW, RGBToYMatrixRow_AVX512BW, 3, 63)
#endif
#ifdef HAS_ARGBTOYMATRIXROW_NEON
ANY11MC(ARGBToYMatrixRow_Any_NEON, ARGBToYMatrixRow_NEON, 4, 15)
#endif
#ifdef HAS_ARGBTOYMATRIXROW_NEON_DOTPROD
ANY11MC(ARGBToYMatrixRow_Any_NEON_DotProd, ARGBToYMatrixRow_NEON_DotProd, 4, 15)
ANY11MC(RGBToYMatrixRow_Any_NEON_DotProd, RGBToYMatrixRow_NEON_DotProd, 3, 15)
#endif
#ifdef HAS_RGBTOYMATRIXROW_NEON
ANY11MC(RGBToYMatrixRow_Any_NEON, RGBToYMatrixRow_NEON, 3, 15)

120
source/row_common.cc
View File

@ -4546,6 +4546,40 @@ void RGBToUVMatrixRow_C(const uint8_t* src_rgb,
}
}
void RGBToUV444MatrixRow_C(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
RGB24ToARGBRow_C(src_rgb, row, twidth);
ARGBToUV444MatrixRow_C(row, dst_u, dst_v, twidth, c);
src_rgb += twidth * 3;
dst_u += twidth;
dst_v += twidth;
width -= twidth;
}
}
#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) && defined(HAS_RGB24TOARGBROW_SSSE3)
void RGBToYMatrixRow_SSSE3(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c) {
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
RGB24ToARGBRow_SSSE3(src_rgb, row, twidth);
ARGBToYMatrixRow_SSSE3(row, dst_y, twidth, c);
src_rgb += twidth * 3;
dst_y += twidth;
width -= twidth;
}
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX2) && defined(HAS_RGB24TOARGBROW_AVX2)
void RGBToYMatrixRow_AVX2(const uint8_t* src_rgb,
uint8_t* dst_y,
@ -4563,6 +4597,27 @@ void RGBToYMatrixRow_AVX2(const uint8_t* src_rgb,
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3) && defined(HAS_RGB24TOARGBROW_SSSE3)
void RGBToUVMatrixRow_SSSE3(const uint8_t* src_rgb,
int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
RGB24ToARGBRow_SSSE3(src_rgb, row, twidth);
RGB24ToARGBRow_SSSE3(src_rgb + src_stride_rgb, row + MAXTWIDTH * 4, twidth);
ARGBToUVMatrixRow_SSSE3(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
src_rgb += twidth * 3;
dst_u += twidth / 2;
dst_v += twidth / 2;
width -= twidth;
}
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX2) && defined(HAS_RGB24TOARGBROW_AVX2)
void RGBToUVMatrixRow_AVX2(const uint8_t* src_rgb,
int src_stride_rgb,
@ -4607,22 +4662,61 @@ void RGBToUVMatrixRow_AVX512BW(const uint8_t* src_rgb,
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON) && defined(HAS_RGB24TOARGBROW_NEON)
void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb,
int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]);
#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) && \
defined(HAS_RGB24TOARGBROW_SSSE3)
void RGBToUV444MatrixRow_SSSE3(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
RGB24ToARGBRow_NEON(src_rgb, row, twidth);
RGB24ToARGBRow_NEON(src_rgb + src_stride_rgb, row + MAXTWIDTH * 4, twidth);
ARGBToUVMatrixRow_NEON(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c);
RGB24ToARGBRow_SSSE3(src_rgb, row, twidth);
ARGBToUV444MatrixRow_SSSE3(row, dst_u, dst_v, twidth, c);
src_rgb += twidth * 3;
dst_u += twidth / 2;
dst_v += twidth / 2;
dst_u += twidth;
dst_v += twidth;
width -= twidth;
}
}
#endif
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2) && \
defined(HAS_RGB24TOARGBROW_AVX2)
void RGBToUV444MatrixRow_AVX2(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
RGB24ToARGBRow_AVX2(src_rgb, row, twidth);
ARGBToUV444MatrixRow_AVX2(row, dst_u, dst_v, twidth, c);
src_rgb += twidth * 3;
dst_u += twidth;
dst_v += twidth;
width -= twidth;
}
}
#endif
#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) && \
defined(HAS_RGB24TOARGBROW_AVX512BW)
void RGBToUV444MatrixRow_AVX512BW(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
RGB24ToARGBRow_AVX512BW(src_rgb, row, twidth);
ARGBToUV444MatrixRow_AVX512BW(row, dst_u, dst_v, twidth, c);
src_rgb += twidth * 3;
dst_u += twidth;
dst_v += twidth;
width -= twidth;
}
}

294
source/row_gcc.cc
View File

@ -63,6 +63,12 @@ static const uvec8 kShuffleMaskRAWToARGB = {
static const uvec8 kShuffleMaskRAWToARGB_0 = {6u, 5u, 4u, 128u, 9u, 8u,
7u, 128u, 12u, 11u, 10u, 128u,
15u, 14u, 13u, 128u};
#if defined(HAS_RAWTOARGBROW_AVX512BW)
static const uvec8 kShuffleMaskRAWToARGB_AVX512BW[2] = {
{2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u},
{6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u,
128u}};
#endif
// Shuffle table for converting RAW to RGBA.
static const uvec8 kShuffleMaskRAWToRGBA = {
@ -163,8 +169,8 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y,
uint8_t* dst_argb,
int width) {
asm volatile(
"vpternlogd $0xff,%%zmm7,%%zmm7,%%zmm7 \n" // 0xffffffff
"vpslld $0x18,%%zmm7,%%zmm7 \n" // 0xff000000
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // 0xffffffff
"vpslld $0x18,%%zmm16,%%zmm16 \n" // 0xff000000
"vmovdqu64 %3,%%zmm5 \n"
LABELALIGN
@ -173,8 +179,8 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y,
"vbroadcasti32x4 0x10(%0),%%zmm1 \n"
"vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
"vpshufb %%zmm5,%%zmm1,%%zmm1 \n"
"vpord %%zmm7,%%zmm0,%%zmm0 \n"
"vpord %%zmm7,%%zmm1,%%zmm1 \n"
"vpord %%zmm16,%%zmm0,%%zmm0 \n"
"vpord %%zmm16,%%zmm1,%%zmm1 \n"
"vmovdqu64 %%zmm0,(%1) \n"
"vmovdqu64 %%zmm1,0x40(%1) \n"
"lea 0x20(%0),%0 \n"
@ -186,7 +192,7 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y,
"+r"(dst_argb), // %1
"+r"(width) // %2
: "m"(kShuffleMaskJ400ToARGB) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm5", "xmm7");
: "memory", "cc", "xmm0", "xmm1", "xmm5", "xmm16");
}
#endif // HAS_J400TOARGBROW_AVX512BW
@ -359,41 +365,55 @@ void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
#ifdef HAS_RAWTOARGBROW_AVX512BW
static const uint32_t kPermdRAWToARGB_AVX512BW[16] = {
0, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9, 9, 10, 11, 12};
#if defined(HAS_RAWTOARGBROW_AVX512BW) || \
defined(HAS_RGB24TOARGBROW_AVX512BW) || \
defined(HAS_RGBTOYMATRIXROW_AVX512BW)
static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2,
3, 4, 4, 5};
static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8,
9, 10, 10, 11};
static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6,
7, 8, 8, 9};
static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4,
5, 6, 6, 7};
#endif
#ifdef HAS_RAWTOARGBROW_AVX512BW
void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_argb,
const uint32_t* shuffler,
int width) {
asm volatile(
"vpternlogd $0xff,%%zmm6,%%zmm6,%%zmm6 \n" // 0xffffffff
"vpslld $0x18,%%zmm6,%%zmm6 \n" // 0xff000000
"movabs $0xffffffffffff,%%rax \n" // 48 bytes mask
"kmovq %%rax,%%k1 \n"
"vmovdqu32 %3,%%zmm5 \n"
"vbroadcasti32x4 %4,%%zmm4 \n"
"vmovdqu64 %4,%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW
"vmovdqu64 %5,%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW
"vmovdqu64 %6,%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW
"vmovdqu64 %7,%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW
"vbroadcasti64x4 %3,%%zmm4 \n" // shuffler (32 bytes)
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // 0xffffffff
"vpslld $0x18,%%zmm16,%%zmm16 \n" // 0xff000000
LABELALIGN //
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu8 48(%0),%%zmm1%{%%k1%}%{z%} \n"
"vmovdqu8 96(%0),%%zmm2%{%%k1%}%{z%} \n"
"vmovdqu8 144(%0),%%zmm3%{%%k1%}%{z%} \n"
"vmovdqu8 (%0),%%zmm0 \n"
"vmovdqu8 64(%0),%%zmm1 \n"
"vmovdqu8 128(%0),%%zmm2 \n"
"lea 192(%0),%0 \n"
"vpermd %%zmm0,%%zmm5,%%zmm0 \n"
"vpermd %%zmm1,%%zmm5,%%zmm1 \n"
"vpermd %%zmm2,%%zmm5,%%zmm2 \n"
"vpermd %%zmm3,%%zmm5,%%zmm3 \n"
"vpermq %%zmm2,%%zmm23,%%zmm3 \n"
"vmovdqa64 %%zmm22,%%zmm18 \n"
"vpermi2q %%zmm2,%%zmm1,%%zmm18 \n"
"vmovdqa64 %%zmm21,%%zmm19 \n"
"vpermi2q %%zmm1,%%zmm0,%%zmm19 \n"
"vpermq %%zmm0,%%zmm20,%%zmm0 \n"
"vmovdqa64 %%zmm19,%%zmm1 \n"
"vmovdqa64 %%zmm18,%%zmm2 \n"
"vpshufb %%zmm4,%%zmm0,%%zmm0 \n"
"vpshufb %%zmm4,%%zmm1,%%zmm1 \n"
"vpshufb %%zmm4,%%zmm2,%%zmm2 \n"
"vpshufb %%zmm4,%%zmm3,%%zmm3 \n"
"vpord %%zmm6,%%zmm0,%%zmm0 \n"
"vpord %%zmm6,%%zmm1,%%zmm1 \n"
"vpord %%zmm6,%%zmm2,%%zmm2 \n"
"vpord %%zmm6,%%zmm3,%%zmm3 \n"
"vpord %%zmm16,%%zmm0,%%zmm0 \n"
"vpord %%zmm16,%%zmm1,%%zmm1 \n"
"vpord %%zmm16,%%zmm2,%%zmm2 \n"
"vpord %%zmm16,%%zmm3,%%zmm3 \n"
"vmovdqu32 %%zmm0,(%1) \n"
"vmovdqu32 %%zmm1,0x40(%1) \n"
"vmovdqu32 %%zmm2,0x80(%1) \n"
@ -401,21 +421,25 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
"lea 0x100(%1),%1 \n"
"sub $0x40,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_raw), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
: "m"(kPermdRAWToARGB_AVX512BW), // %3
"m"(*shuffler) // %4
: "memory", "cc", "rax", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
"xmm5", "xmm6");
"vzeroupper \n"
: "+r"(src_raw), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
: "m"(*shuffler), // %3
"m"(kPermqRGB24ToARGB0_AVX512BW), // %4
"m"(kPermqRGB24ToARGB1_AVX512BW), // %5
"m"(kPermqRGB24ToARGB2_AVX512BW), // %6
"m"(kPermqRGB24ToARGB3_AVX512BW) // %7
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16",
"xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23");
}
void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_argb,
int width) {
RGBToARGBRow_AVX512BW(src_raw, dst_argb,
(const uint32_t*)&kShuffleMaskRAWToARGB, width);
(const uint32_t*)&kShuffleMaskRAWToARGB_AVX512BW[0],
width);
}
void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
@ -777,9 +801,9 @@ static const ulvec8 kPermARGBToRGB24_2 = {
void ARGBToRGB24Row_AVX512VBMI(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
"vmovdqa %3,%%ymm5 \n"
"vmovdqa %4,%%ymm6 \n"
"vmovdqa %5,%%ymm7 \n"
"vmovdqa32 %3,%%ymm5 \n"
"vmovdqa32 %4,%%ymm16 \n"
"vmovdqa32 %5,%%ymm17 \n"
LABELALIGN
"1: \n"
@ -789,8 +813,8 @@ void ARGBToRGB24Row_AVX512VBMI(const uint8_t* src, uint8_t* dst, int width) {
"vmovdqu 0x60(%0),%%ymm3 \n"
"lea 0x80(%0),%0 \n"
"vpermt2b %%ymm1,%%ymm5,%%ymm0 \n"
"vpermt2b %%ymm2,%%ymm6,%%ymm1 \n"
"vpermt2b %%ymm3,%%ymm7,%%ymm2 \n"
"vpermt2b %%ymm2,%%ymm16,%%ymm1 \n"
"vpermt2b %%ymm3,%%ymm17,%%ymm2 \n"
"vmovdqu %%ymm0,(%1) \n"
"vmovdqu %%ymm1,0x20(%1) \n"
"vmovdqu %%ymm2,0x40(%1) \n"
@ -804,7 +828,8 @@ void ARGBToRGB24Row_AVX512VBMI(const uint8_t* src, uint8_t* dst, int width) {
: "m"(kPermARGBToRGB24_0), // %3
"m"(kPermARGBToRGB24_1), // %4
"m"(kPermARGBToRGB24_2) // %5
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7");
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm16",
"xmm17");
}
#endif
@ -1419,11 +1444,12 @@ void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
}
#endif
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \
#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \
defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \
defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \
defined(HAS_RGBTOYMATRIXROW_AVX512BW)
static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13,
2, 6, 10, 14, 3, 7, 11, 15};
2, 6, 10, 14, 3, 7, 11, 15};
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
@ -1439,15 +1465,16 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
asm volatile(
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
"vpsllw $15,%%zmm16,%%zmm5 \n"
"vpacksswb %%zmm5,%%zmm5,%%zmm5 \n"
"vpsrlw $15,%%zmm16,%%zmm16 \n" // zmm16 = 1
"vbroadcasti64x4 0(%3),%%zmm4 \n"
"vbroadcasti64x4 0x60(%3),%%zmm7 \n"
"vpmaddubsw %%zmm5,%%zmm4,%%zmm6 \n"
"vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n"
"vpackssdw %%zmm6,%%zmm6,%%zmm6 \n"
"vpsubw %%zmm6,%%zmm7,%%zmm7 \n"
"vmovups %4,%%zmm6 \n" LABELALIGN
"vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80
"vpabsw %%zmm16,%%zmm16 \n" // zmm16 = 1
"vbroadcasti32x4 0(%3),%%zmm4 \n"
"vbroadcasti32x4 0x60(%3),%%zmm17 \n"
"vpmaddubsw %%zmm5,%%zmm4,%%zmm18 \n"
"vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n"
"vpackssdw %%zmm18,%%zmm18,%%zmm18 \n"
"vpsubw %%zmm18,%%zmm17,%%zmm17 \n"
"vmovups %4,%%zmm18 \n"
LABELALIGN
"1: \n"
"vmovups (%0),%%zmm0 \n"
"vmovups 0x40(%0),%%zmm1 \n"
@ -1468,12 +1495,12 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
"vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
"vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n"
"vpackssdw %%zmm3,%%zmm2,%%zmm2 \n"
"vpaddw %%zmm7,%%zmm0,%%zmm0 \n"
"vpaddw %%zmm7,%%zmm2,%%zmm2 \n"
"vpaddw %%zmm17,%%zmm0,%%zmm0 \n"
"vpaddw %%zmm17,%%zmm2,%%zmm2 \n"
"vpsrlw $0x8,%%zmm0,%%zmm0 \n"
"vpsrlw $0x8,%%zmm2,%%zmm2 \n"
"vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
"vpermd %%zmm0,%%zmm6,%%zmm0 \n"
"vpermd %%zmm0,%%zmm18,%%zmm0 \n"
"vmovups %%zmm0,(%1) \n"
"lea 0x40(%1),%1 \n"
"sub $0x40,%2 \n"
@ -1484,8 +1511,89 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb,
"+r"(width) // %2
: "r"(c), // %3
"m"(kPermdARGBToY_AVX512BW) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "xmm16");
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm16", "xmm17", "xmm18");
}
#endif
#ifdef HAS_RGBTOYMATRIXROW_AVX512BW
void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c) {
asm volatile(
"vmovdqu64 %5,%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW
"vmovdqu64 %6,%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW
"vmovdqu64 %7,%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW
"vmovdqu64 %8,%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW
"vbroadcasti64x4 %9,%%zmm24 \n" // kShuffleMaskRGB24ToARGB
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
"vpsllw $15,%%zmm16,%%zmm5 \n"
"vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80
"vpabsw %%zmm16,%%zmm16 \n" // zmm16 = 1
"vbroadcasti32x4 0(%3),%%zmm4 \n" // zmm4 = kRGBToY
"vbroadcasti32x4 0x60(%3),%%zmm17 \n" // zmm17 = kAddY
"vpmaddubsw %%zmm5,%%zmm4,%%zmm18 \n"
"vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n"
"vpackssdw %%zmm18,%%zmm18,%%zmm18 \n"
"vpsubw %%zmm18,%%zmm17,%%zmm17 \n" // zmm17 = adjusted bias
"vmovups %4,%%zmm25 \n" // zmm25 = kPermdARGBToY_AVX512BW
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0 \n"
"vmovdqu8 64(%0),%%zmm1 \n"
"vmovdqu8 128(%0),%%zmm2 \n"
"lea 192(%0),%0 \n"
"vpermq %%zmm2,%%zmm23,%%zmm3 \n"
"vmovdqa64 %%zmm22,%%zmm18 \n"
"vpermi2q %%zmm2,%%zmm1,%%zmm18 \n"
"vmovdqa64 %%zmm21,%%zmm19 \n"
"vpermi2q %%zmm1,%%zmm0,%%zmm19 \n"
"vpermq %%zmm0,%%zmm20,%%zmm0 \n"
"vmovdqa64 %%zmm19,%%zmm1 \n"
"vmovdqa64 %%zmm18,%%zmm2 \n"
"vpshufb %%zmm24,%%zmm0,%%zmm0 \n"
"vpshufb %%zmm24,%%zmm1,%%zmm1 \n"
"vpshufb %%zmm24,%%zmm2,%%zmm2 \n"
"vpshufb %%zmm24,%%zmm3,%%zmm3 \n"
"vpsubb %%zmm5,%%zmm0,%%zmm0 \n"
"vpsubb %%zmm5,%%zmm1,%%zmm1 \n"
"vpsubb %%zmm5,%%zmm2,%%zmm2 \n"
"vpsubb %%zmm5,%%zmm3,%%zmm3 \n"
"vpmaddubsw %%zmm0,%%zmm4,%%zmm0 \n"
"vpmaddubsw %%zmm1,%%zmm4,%%zmm1 \n"
"vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n"
"vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n"
"vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
"vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
"vpackssdw %%zmm1,%%zmm0,%%zmm0 \n"
"vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
"vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n"
"vpackssdw %%zmm3,%%zmm2,%%zmm2 \n"
"vpaddw %%zmm17,%%zmm0,%%zmm0 \n"
"vpaddw %%zmm17,%%zmm2,%%zmm2 \n"
"vpsrlw $0x8,%%zmm0,%%zmm0 \n"
"vpsrlw $0x8,%%zmm2,%%zmm2 \n"
"vpackuswb %%zmm2,%%zmm0,%%zmm0 \n"
"vpermd %%zmm0,%%zmm25,%%zmm0 \n"
"vmovups %%zmm0,(%1) \n"
"lea 0x40(%1),%1 \n"
"sub $0x40,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_rgb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "r"(c), // %3
"m"(kPermdARGBToY_AVX512BW), // %4
"m"(kPermqRGB24ToARGB0_AVX512BW), // %5
"m"(kPermqRGB24ToARGB1_AVX512BW), // %6
"m"(kPermqRGB24ToARGB2_AVX512BW), // %7
"m"(kPermqRGB24ToARGB3_AVX512BW), // %8
"m"(kShuffleMaskRGB24ToARGB) // %9
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22",
"xmm23", "xmm24", "xmm25");
}
#endif
@ -1639,11 +1747,11 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
int width,
const struct ArgbConstants* c) {
asm volatile(
"vbroadcasti64x4 0x20(%4),%%zmm3 \n" // kRGBToU
"vbroadcasti64x4 0x40(%4),%%zmm4 \n" // kRGBToV
"vbroadcasti32x4 0x20(%4),%%zmm3 \n" // kRGBToU
"vbroadcasti32x4 0x40(%4),%%zmm4 \n" // kRGBToV
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // -1
"vpsllw $15,%%zmm16,%%zmm5 \n" // 0x8000
"vmovups %5,%%zmm7 \n"
"vmovups %5,%%zmm17 \n"
"sub %1,%2 \n"
LABELALIGN
@ -1651,45 +1759,45 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
"vmovups (%0),%%zmm0 \n"
"vmovups 0x40(%0),%%zmm1 \n"
"vmovups 0x80(%0),%%zmm2 \n"
"vmovups 0xc0(%0),%%zmm6 \n"
"vmovups 0xc0(%0),%%zmm18 \n"
"vpmaddubsw %%zmm3,%%zmm0,%%zmm0 \n"
"vpmaddubsw %%zmm3,%%zmm1,%%zmm1 \n"
"vpmaddubsw %%zmm3,%%zmm2,%%zmm2 \n"
"vpmaddubsw %%zmm3,%%zmm6,%%zmm6 \n"
"vpmaddubsw %%zmm3,%%zmm18,%%zmm18 \n"
"vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
"vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
"vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
"vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n"
"vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n"
"vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates
"vpackssdw %%zmm6,%%zmm2,%%zmm2 \n"
"vpackssdw %%zmm18,%%zmm2,%%zmm2 \n"
"vpsubw %%zmm5,%%zmm0,%%zmm0 \n"
"vpsubw %%zmm5,%%zmm2,%%zmm2 \n"
"vpsrlw $0x8,%%zmm0,%%zmm0 \n"
"vpsrlw $0x8,%%zmm2,%%zmm2 \n"
"vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates
"vpermd %%zmm0,%%zmm7,%%zmm0 \n" // unmutate.
"vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate.
"vmovups %%zmm0,(%1) \n"
"vmovups (%0),%%zmm0 \n"
"vmovups 0x40(%0),%%zmm1 \n"
"vmovups 0x80(%0),%%zmm2 \n"
"vmovups 0xc0(%0),%%zmm6 \n"
"vmovups 0xc0(%0),%%zmm18 \n"
"vpmaddubsw %%zmm4,%%zmm0,%%zmm0 \n"
"vpmaddubsw %%zmm4,%%zmm1,%%zmm1 \n"
"vpmaddubsw %%zmm4,%%zmm2,%%zmm2 \n"
"vpmaddubsw %%zmm4,%%zmm6,%%zmm6 \n"
"vpmaddubsw %%zmm4,%%zmm18,%%zmm18 \n"
"vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
"vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
"vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n"
"vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n"
"vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n"
"vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates
"vpackssdw %%zmm6,%%zmm2,%%zmm2 \n"
"vpackssdw %%zmm18,%%zmm2,%%zmm2 \n"
"vpsubw %%zmm5,%%zmm0,%%zmm0 \n"
"vpsubw %%zmm5,%%zmm2,%%zmm2 \n"
"vpsrlw $0x8,%%zmm0,%%zmm0 \n"
"vpsrlw $0x8,%%zmm2,%%zmm2 \n"
"vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates
"vpermd %%zmm0,%%zmm7,%%zmm0 \n" // unmutate.
"vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate.
"vmovups %%zmm0,(%1,%2,1) \n"
"lea 0x100(%0),%0 \n"
"lea 0x40(%1),%1 \n"
@ -1706,8 +1814,8 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
#endif
: "r"(c), // %4
"m"(kPermdARGBToY_AVX512BW) // %5
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "xmm16");
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm16", "xmm17", "xmm18");
}
#endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW
@ -1885,12 +1993,12 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
int width,
const struct ArgbConstants* c) {
asm volatile(
"vbroadcasti64x4 0x20(%5),%%zmm4 \n" // RGBToU
"vbroadcasti64x4 0x40(%5),%%zmm5 \n" // RGBToV
"vbroadcasti32x4 0x20(%5),%%zmm4 \n" // RGBToU
"vbroadcasti32x4 0x40(%5),%%zmm5 \n" // RGBToV
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
"vpabsb %%zmm16,%%zmm6 \n" // 0x0101
"vpabsb %%zmm16,%%zmm20 \n" // 0x0101
"vpsllw $15,%%zmm16,%%zmm17 \n" // 0x8000
"vbroadcasti64x4 %6,%%zmm7 \n" // kShuffleAARRGGBB
"vbroadcasti32x4 %6,%%zmm21 \n" // kShuffleAARRGGBB
"vmovups %7,%%zmm18 \n" // kPermdARGBToY_AVX512BW
"vmovups %8,%%zmm19 \n" // kPermdARGBToUV_AVX512BW
"sub %1,%2 \n"
@ -1901,14 +2009,14 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
"vmovups 0x40(%0),%%zmm1 \n"
"vmovups 0x00(%0,%4,1),%%zmm2 \n"
"vmovups 0x40(%0,%4,1),%%zmm3 \n"
"vpshufb %%zmm7,%%zmm0,%%zmm0 \n" // aarrggbb
"vpshufb %%zmm7,%%zmm1,%%zmm1 \n"
"vpshufb %%zmm7,%%zmm2,%%zmm2 \n"
"vpshufb %%zmm7,%%zmm3,%%zmm3 \n"
"vpmaddubsw %%zmm6,%%zmm0,%%zmm0 \n" // 32x2 -> 16x2
"vpmaddubsw %%zmm6,%%zmm1,%%zmm1 \n"
"vpmaddubsw %%zmm6,%%zmm2,%%zmm2 \n"
"vpmaddubsw %%zmm6,%%zmm3,%%zmm3 \n"
"vpshufb %%zmm21,%%zmm0,%%zmm0 \n" // aarrggbb
"vpshufb %%zmm21,%%zmm1,%%zmm1 \n"
"vpshufb %%zmm21,%%zmm2,%%zmm2 \n"
"vpshufb %%zmm21,%%zmm3,%%zmm3 \n"
"vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" // 32x2 -> 16x2
"vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n"
"vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n"
"vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm0,%%zmm2,%%zmm0 \n" // 16x2 -> 16x1
"vpaddw %%zmm1,%%zmm3,%%zmm1 \n"
"vpxorq %%zmm2,%%zmm2,%%zmm2 \n" // 0 for vpavgw
@ -1952,8 +2060,8 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
"m"(kShuffleAARRGGBB), // %6
"m"(kPermdARGBToY_AVX512BW), // %7
"m"(kPermdARGBToUV_AVX512BW) // %8
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7", "xmm16", "xmm17", "xmm18", "xmm19");
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21");
}
#endif // HAS_ARGBTOUVMATRIXROW_AVX512BW
@ -9446,9 +9554,9 @@ void NV21ToYUV24Row_AVX512(const uint8_t* src_y,
int width) {
asm volatile(
"sub %0,%1 \n"
"vmovdqa (%4),%%ymm4 \n" // 3 shuffler constants
"vmovdqa 32(%4),%%ymm5 \n"
"vmovdqa 64(%4),%%ymm6 \n" LABELALIGN
"vmovdqa32 (%4),%%ymm4 \n" // 3 shuffler constants
"vmovdqa32 32(%4),%%ymm5 \n"
"vmovdqa32 64(%4),%%ymm16 \n" LABELALIGN
"1: \n"
"vmovdqu (%0),%%ymm2 \n" // load 32 Y values
"vmovdqu (%0,%1),%%ymm3 \n" // load 16 VU values
@ -9457,7 +9565,7 @@ void NV21ToYUV24Row_AVX512(const uint8_t* src_y,
"vmovdqa %%ymm2, %%ymm1 \n"
"vpermt2b %%ymm3,%%ymm4,%%ymm0 \n"
"vpermt2b %%ymm3,%%ymm5,%%ymm1 \n"
"vpermt2b %%ymm3,%%ymm6,%%ymm2 \n"
"vpermt2b %%ymm3,%%ymm16,%%ymm2 \n"
"vmovdqu %%ymm0,(%2) \n"
"vmovdqu %%ymm1,32(%2) \n"
"vmovdqu %%ymm2,64(%2) \n"
@ -9470,7 +9578,7 @@ void NV21ToYUV24Row_AVX512(const uint8_t* src_y,
"+r"(dst_yuv24), // %2
"+r"(width) // %3
: "r"(&kYUV24Perm[0]) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16");
}
#endif // HAS_NV21ToYUV24ROW_AVX512

114
source/row_neon.cc
View File

@ -1897,6 +1897,58 @@ void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb,
"q10", "q11", "q12");
}
// 8x1 pixels.
void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
asm volatile(
"vld1.8 {d24}, [%4] \n" // load kRGBToU
"vld1.8 {d25}, [%5] \n" // load kRGBToV
"vld1.16 {d26[0]}, [%6] \n" // load kAddUV[0]
"vmovl.s8 q10, d24 \n" // U coeffs (8 shorts)
"vmovl.s8 q11, d25 \n" // V coeffs (8 shorts)
"vdup.16 q6, d26[0] \n" // bias
"1: \n"
"vld3.8 {d0, d1, d2}, [%0]! \n" // load 8 RGB pixels.
"subs %3, %3, #8 \n" // 8 processed per loop.
"vmovl.u8 q4, d0 \n" // B
"vmovl.u8 q5, d1 \n" // G
"vmovl.u8 q7, d2 \n" // R
"vdup.16 q12, d20[0] \n"
"vmul.s16 q2, q4, q12 \n" // U = B * U0
"vdup.16 q12, d20[1] \n"
"vmla.s16 q2, q5, q12 \n" // U += G * U1
"vdup.16 q12, d20[2] \n"
"vmla.s16 q2, q7, q12 \n" // U += R * U2
"vdup.16 q12, d22[0] \n"
"vmul.s16 q3, q4, q12 \n" // V = B * V0
"vdup.16 q12, d22[1] \n"
"vmla.s16 q3, q5, q12 \n" // V += G * V1
"vdup.16 q12, d22[2] \n"
"vmla.s16 q3, q7, q12 \n" // V += R * V2
"vsubhn.s16 d0, q6, q2 \n" // 128.0 - U
"vsubhn.s16 d1, q6, q3 \n" // 128.0 - V
"vst1.8 {d0}, [%1]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%2]! \n" // store 8 pixels V.
"bgt 1b \n"
: "+r"(src_rgb), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
"+r"(width) // %3
: "r"(&c->kRGBToU), // %4
"r"(&c->kRGBToV), // %5
"r"(&c->kAddUV) // %6
: "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7",
"q10", "q11", "q12");
}
// clang-format off
// 16x2 pixels -> 8x1. width is number of argb pixels. e.g. 16.
#define RGBTOUV(QB, QG, QR) \
@ -1982,6 +2034,68 @@ void ARGBToUVMatrixRow_NEON(const uint8_t* src_argb,
"q9", "q11", "q12", "q14", "q15");
}
void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb,
int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
const uint8_t* src_rgb_1 = src_rgb + src_stride_rgb;
asm volatile(
"vld1.8 {d24}, [%5] \n" // load kRGBToU
"vld1.8 {d25}, [%6] \n" // load kRGBToV
"vmovl.s8 q14, d24 \n" // U coeffs in d28
"vmovl.s8 q15, d25 \n" // V coeffs in d30
"vmov.u16 q11, #0x8000 \n" // 128.0 bias
"1: \n"
"vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB pixels.
"vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RGB pixels.
"subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
"vld3.8 {d6, d8, d10}, [%1]! \n" // load 8 more pixels.
"vld3.8 {d7, d9, d11}, [%1]! \n"
"vpadal.u8 q0, q3 \n" // B
"vpadal.u8 q1, q4 \n" // G
"vpadal.u8 q2, q5 \n" // R
"vrshr.u16 q0, q0, #2 \n" // average of 4
"vrshr.u16 q1, q1, #2 \n"
"vrshr.u16 q2, q2, #2 \n"
"vdup.16 q12, d28[0] \n"
"vmul.s16 q8, q0, q12 \n" // U = B * U0
"vdup.16 q12, d28[1] \n"
"vmla.s16 q8, q1, q12 \n" // U += G * U1
"vdup.16 q12, d28[2] \n"
"vmla.s16 q8, q2, q12 \n" // U += R * U2
"vdup.16 q12, d30[0] \n"
"vmul.s16 q9, q0, q12 \n" // V = B * V0
"vdup.16 q12, d30[1] \n"
"vmla.s16 q9, q1, q12 \n" // V += G * V1
"vdup.16 q12, d30[2] \n"
"vmla.s16 q9, q2, q12 \n" // V += R * V2
"vsubhn.s16 d0, q11, q8 \n" // 128.0 - U
"vsubhn.s16 d1, q11, q9 \n" // 128.0 - V
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
"bgt 1b \n"
: "+r"(src_rgb), // %0
"+r"(src_rgb_1), // %1
"+r"(dst_u), // %2
"+r"(dst_v), // %3
"+r"(width) // %4
: "r"(&c->kRGBToU), // %5
"r"(&c->kRGBToV) // %6
: "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q8",
"q9", "q11", "q12", "q14", "q15");
}
void RGB24ToUVRow_NEON(const uint8_t* src_rgb24,
int src_stride_rgb24,
uint8_t* dst_u,

164
source/row_neon64.cc
View File

@ -2793,6 +2793,61 @@ void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb,
"v27", "v28");
}
// 8x1 pixels.
void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
asm volatile(
"ldr q16, [%[c], #16] \n" // kRGBToU
"ldr q17, [%[c], #32] \n" // kRGBToV
"ldr s0, [%[c], #64] \n" // kAddUV
"sxtl v16.8h, v16.8b \n" // sign extend U coeffs
// to 16-bit
"sxtl v17.8h, v17.8b \n" // sign extend V coeffs
// to 16-bit
"dup v20.8h, v16.h[0] \n" // U0
"dup v21.8h, v16.h[1] \n" // U1
"dup v22.8h, v16.h[2] \n" // U2
"dup v24.8h, v17.h[0] \n" // V0
"dup v26.8h, v17.h[1] \n" // V1
"dup v27.8h, v17.h[2] \n" // V2
"dup v25.8h, v0.h[0] \n" // kAddUV
"1: \n"
"ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // load 8 RGB
"subs %w3, %w3, #8 \n" // 8 processed per loop.
"uxtl v4.8h, v0.8b \n" // B
"uxtl v5.8h, v1.8b \n" // G
"uxtl v6.8h, v2.8b \n" // R
// U = B*U0 + G*U1 + R*U2
"mul v18.8h, v4.8h, v20.8h \n"
"mla v18.8h, v5.8h, v21.8h \n"
"mla v18.8h, v6.8h, v22.8h \n"
// V = B*V0 + G*V1 + R*V2
"mul v19.8h, v4.8h, v24.8h \n"
"mla v19.8h, v5.8h, v26.8h \n"
"mla v19.8h, v6.8h, v27.8h \n"
"subhn v0.8b, v25.8h, v18.8h \n"
"subhn v1.8b, v25.8h, v19.8h \n"
"st1 {v0.8b}, [%1], #8 \n"
"st1 {v1.8b}, [%2], #8 \n"
"b.gt 1b \n"
: "+r"(src_rgb), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
"+r"(width) // %3
: [c] "r"(c) // %4
: "cc", "memory", "v0", "v1", "v2", "v4", "v5", "v6", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v24", "v25", "v26",
"v27");
}
void ARGBToUV444MatrixRow_NEON_I8MM(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
@ -2940,6 +2995,71 @@ void ARGBToUVMatrixRow_NEON(const uint8_t* src_argb,
"v28");
}
void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb,
int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
const uint8_t* src_rgb_1 = src_rgb + src_stride_rgb;
asm volatile(
"ldr q16, [%[c], #16] \n" // kRGBToU
"ldr q17, [%[c], #32] \n" // kRGBToV
"sxtl v16.8h, v16.8b \n" // sign extend U coeffs
"sxtl v17.8h, v17.8b \n" // sign extend V coeffs
"dup v20.8h, v16.h[0] \n" // U0
"dup v21.8h, v16.h[1] \n" // U1
"dup v22.8h, v16.h[2] \n" // U2
"dup v24.8h, v17.h[0] \n" // V0
"dup v26.8h, v17.h[1] \n" // V1
"dup v27.8h, v17.h[2] \n" // V2
"movi v25.8h, #0x80, lsl #8 \n" // 128.0 in 16-bit (0x8000)
"1: \n"
"ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels.
"subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
"uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
"ld3 {v4.16b,v5.16b,v6.16b}, [%1], #48 \n" // load 16 more.
"uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%1, 448] \n"
"uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
"urshr v0.8h, v0.8h, #2 \n" // average of 4
"urshr v1.8h, v1.8h, #2 \n"
"urshr v2.8h, v2.8h, #2 \n"
// U = B*U0 + G*U1 + R*U2
"mul v3.8h, v0.8h, v20.8h \n"
"mla v3.8h, v1.8h, v21.8h \n"
"mla v3.8h, v2.8h, v22.8h \n"
// V = B*V0 + G*V1 + R*V2
"mul v4.8h, v0.8h, v24.8h \n"
"mla v4.8h, v1.8h, v26.8h \n"
"mla v4.8h, v2.8h, v27.8h \n"
// U = (128.0 - U) >> 8, V = (128.0 - V) >> 8
"subhn v0.8b, v25.8h, v3.8h \n"
"subhn v1.8b, v25.8h, v4.8h \n"
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
"b.gt 1b \n"
: "+r"(src_rgb), // %0
"+r"(src_rgb_1), // %1
"+r"(dst_u), // %2
"+r"(dst_v), // %3
"+r"(width) // %4
: [c] "r"(c) // %5
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v20", "v21", "v22", "v24", "v25", "v26", "v27");
}
void RGB24ToUVRow_NEON(const uint8_t* src_rgb24,
int src_stride_rgb24,
uint8_t* dst_u,
@ -3401,6 +3521,50 @@ void ARGBToYMatrixRow_NEON_DotProd(const uint8_t* src_argb,
"v17", "v18", "v19");
}
void RGBToYMatrixRow_NEON_DotProd(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c) {
asm volatile(
"ldr s16, [%3] \n" // load 4 coeffs
"ldr s17, [%3, #48] \n" // load kAddY[0]
"dup v18.4s, v16.s[0] \n"
"dup v19.8h, v17.h[0] \n"
"movi v7.16b, #0 \n" // zero alpha
"1: \n"
"ld3 {v4.16b, v5.16b, v6.16b}, [%0], #48 \n" // load 16 pixels
"subs %w2, %w2, #16 \n" // 16 processed per loop.
"zip1 v24.16b, v4.16b, v5.16b \n" // B0 G0 B1 G1 ...
"zip1 v25.16b, v6.16b, v7.16b \n" // R0 0 R1 0 ...
"zip2 v26.16b, v4.16b, v5.16b \n" // B8 G8 B9 G9 ...
"zip2 v27.16b, v6.16b, v7.16b \n" // R8 0 R9 0 ...
"zip1 v20.8h, v24.8h, v25.8h \n" // pixels 0..3
"zip2 v21.8h, v24.8h, v25.8h \n" // pixels 4..7
"zip1 v22.8h, v26.8h, v27.8h \n" // pixels 8..11
"zip2 v23.8h, v26.8h, v27.8h \n" // pixels 12..15
"movi v0.16b, #0 \n"
"movi v1.16b, #0 \n"
"movi v2.16b, #0 \n"
"movi v3.16b, #0 \n"
"udot v0.4s, v20.16b, v18.16b \n"
"udot v1.4s, v21.16b, v18.16b \n"
"udot v2.4s, v22.16b, v18.16b \n"
"udot v3.4s, v23.16b, v18.16b \n"
"uzp1 v0.8h, v0.8h, v1.8h \n"
"uzp1 v1.8h, v2.8h, v3.8h \n"
"addhn v0.8b, v0.8h, v19.8h \n"
"addhn v1.8b, v1.8h, v19.8h \n"
"st1 {v0.8b, v1.8b}, [%1], #16 \n" // store 16 pixels Y.
"b.gt 1b \n"
: "+r"(src_rgb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "r"(c) // %3
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27");
}
// RGBA expects first value to be A and ignored, then 3 values to contain RGB.
void RGBToYMatrixRow_NEON(const uint8_t* src_rgb,
uint8_t* dst_y,

341
source/row_rvv.cc
View File

@ -1257,24 +1257,27 @@ void ARGBToYMatrixRow_RVV(const uint8_t* src_argb,
const struct ArgbConstants* c) {
assert(width != 0);
size_t w = (size_t)width;
vuint8m2_t v_by, v_gy, v_ry; // vectors are to store RGBToY constant
vuint16m4_t v_addy; // vector is to store kAddY
vuint8m2_t v_y0, v_y1, v_y2, v_y3;
vuint16m4_t v_addy;
size_t vl = __riscv_vsetvl_e8m2(w);
v_by = __riscv_vmv_v_x_u8m2(c->kRGBToY[0], vl);
v_gy = __riscv_vmv_v_x_u8m2(c->kRGBToY[1], vl);
v_ry = __riscv_vmv_v_x_u8m2(c->kRGBToY[2], vl);
v_y0 = __riscv_vmv_v_x_u8m2(c->kRGBToY[0], vl);
v_y1 = __riscv_vmv_v_x_u8m2(c->kRGBToY[1], vl);
v_y2 = __riscv_vmv_v_x_u8m2(c->kRGBToY[2], vl);
v_y3 = __riscv_vmv_v_x_u8m2(c->kRGBToY[3], vl);
v_addy = __riscv_vmv_v_x_u16m4(c->kAddY[0], vl);
do {
vuint8m2_t v_y;
vuint16m4_t v_y_u16;
size_t vl = __riscv_vsetvl_e8m2(w);
vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
v_y_u16 = __riscv_vwmulu_vv_u16m4(v_r, v_ry, vl);
v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_gy, v_g, vl);
v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_by, v_b, vl);
vuint8m2_t v_b0 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
vuint8m2_t v_b1 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
vuint8m2_t v_b2 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
vuint8m2_t v_b3 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3);
v_y_u16 = __riscv_vwmulu_vv_u16m4(v_b0, v_y0, vl);
v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_y1, v_b1, vl);
v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_y2, v_b2, vl);
v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_y3, v_b3, vl);
v_y_u16 = __riscv_vadd_vv_u16m4(v_y_u16, v_addy, vl);
v_y = __riscv_vnsrl_wx_u8m2(v_y_u16, 8, vl);
__riscv_vse8_v_u8m2(dst_y, v_y, vl);
@ -1320,6 +1323,322 @@ void RGBToYMatrixRow_RVV(const uint8_t* src_rgb,
}
#endif
#ifdef HAS_ARGBTOUV444MATRIXROW_RVV
void ARGBToUV444MatrixRow_RVV(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
assert(width != 0);
size_t w = (size_t)width;
vint8m2_t v_u0, v_u1, v_u2, v_u3;
vint8m2_t v_v0, v_v1, v_v2, v_v3;
vint16m4_t v_adduv;
size_t vl = __riscv_vsetvl_e8m2(w);
v_u0 = __riscv_vmv_v_x_i8m2(c->kRGBToU[0], vl);
v_u1 = __riscv_vmv_v_x_i8m2(c->kRGBToU[1], vl);
v_u2 = __riscv_vmv_v_x_i8m2(c->kRGBToU[2], vl);
v_u3 = __riscv_vmv_v_x_i8m2(c->kRGBToU[3], vl);
v_v0 = __riscv_vmv_v_x_i8m2(c->kRGBToV[0], vl);
v_v1 = __riscv_vmv_v_x_i8m2(c->kRGBToV[1], vl);
v_v2 = __riscv_vmv_v_x_i8m2(c->kRGBToV[2], vl);
v_v3 = __riscv_vmv_v_x_i8m2(c->kRGBToV[3], vl);
v_adduv = __riscv_vmv_v_x_i16m4((int16_t)c->kAddUV[0], vl);
do {
size_t vl = __riscv_vsetvl_e8m2(w);
vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
vuint8m2_t v_b0 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
vuint8m2_t v_b1 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
vuint8m2_t v_b2 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
vuint8m2_t v_b3 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3);
vint16m4_t v_u_i16 = __riscv_vwmulsu_vv_i16m4(v_u0, v_b0, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_u1, v_b1, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_u2, v_b2, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_u3, v_b3, vl);
v_u_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_u_i16, vl);
vuint8m2_t v_u = __riscv_vnsrl_wx_u8m2(
__riscv_vreinterpret_v_i16m4_u16m4(v_u_i16), 8, vl);
__riscv_vse8_v_u8m2(dst_u, v_u, vl);
vint16m4_t v_v_i16 = __riscv_vwmulsu_vv_i16m4(v_v0, v_b0, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_v1, v_b1, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_v2, v_b2, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_v3, v_b3, vl);
v_v_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_v_i16, vl);
vuint8m2_t v_v = __riscv_vnsrl_wx_u8m2(
__riscv_vreinterpret_v_i16m4_u16m4(v_v_i16), 8, vl);
__riscv_vse8_v_u8m2(dst_v, v_v, vl);
w -= vl;
src_argb += 4 * vl;
dst_u += vl;
dst_v += vl;
} while (w > 0);
}
#endif
#ifdef HAS_ARGBTOUVMATRIXROW_RVV
void ARGBToUVMatrixRow_RVV(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
assert(width != 0);
const uint8_t* src_argb_0 = src_argb;
const uint8_t* src_argb_1 = src_argb + src_stride_argb;
int w = width >> 1;
if (w > 0) {
size_t w_len = (size_t)w;
vint8m1_t v_u0, v_u1, v_u2, v_u3;
vint8m1_t v_v0, v_v1, v_v2, v_v3;
vint16m2_t v_adduv;
size_t vl = __riscv_vsetvl_e8m1(w_len);
v_u0 = __riscv_vmv_v_x_i8m1(c->kRGBToU[0], vl);
v_u1 = __riscv_vmv_v_x_i8m1(c->kRGBToU[1], vl);
v_u2 = __riscv_vmv_v_x_i8m1(c->kRGBToU[2], vl);
v_u3 = __riscv_vmv_v_x_i8m1(c->kRGBToU[3], vl);
v_v0 = __riscv_vmv_v_x_i8m1(c->kRGBToV[0], vl);
v_v1 = __riscv_vmv_v_x_i8m1(c->kRGBToV[1], vl);
v_v2 = __riscv_vmv_v_x_i8m1(c->kRGBToV[2], vl);
v_v3 = __riscv_vmv_v_x_i8m1(c->kRGBToV[3], vl);
v_adduv = __riscv_vmv_v_x_i16m2((int16_t)c->kAddUV[0], vl);
do {
size_t vl = __riscv_vsetvl_e8m1(w_len);
vuint8m1x8_t v_src0 = __riscv_vlseg8e8_v_u8m1x8(src_argb_0, vl);
vuint8m1x8_t v_src1 = __riscv_vlseg8e8_v_u8m1x8(src_argb_1, vl);
vuint8m1_t b0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 0);
vuint8m1_t g0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 1);
vuint8m1_t r0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 2);
vuint8m1_t a0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 3);
vuint8m1_t b1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 4);
vuint8m1_t g1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 5);
vuint8m1_t r1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 6);
vuint8m1_t a1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 7);
vuint8m1_t b0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 0);
vuint8m1_t g0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 1);
vuint8m1_t r0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 2);
vuint8m1_t a0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 3);
vuint8m1_t b1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 4);
vuint8m1_t g1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 5);
vuint8m1_t r1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 6);
vuint8m1_t a1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 7);
vuint16m2_t sum_b = __riscv_vwaddu_vv_u16m2(b0, b1, vl);
sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b0_1, vl);
sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b1_1, vl);
sum_b = __riscv_vadd_vx_u16m2(sum_b, 2, vl);
vuint8m1_t b = __riscv_vnsrl_wx_u8m1(sum_b, 2, vl);
vuint16m2_t sum_g = __riscv_vwaddu_vv_u16m2(g0, g1, vl);
sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g0_1, vl);
sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g1_1, vl);
sum_g = __riscv_vadd_vx_u16m2(sum_g, 2, vl);
vuint8m1_t g = __riscv_vnsrl_wx_u8m1(sum_g, 2, vl);
vuint16m2_t sum_r = __riscv_vwaddu_vv_u16m2(r0, r1, vl);
sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r0_1, vl);
sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r1_1, vl);
sum_r = __riscv_vadd_vx_u16m2(sum_r, 2, vl);
vuint8m1_t r = __riscv_vnsrl_wx_u8m1(sum_r, 2, vl);
vuint16m2_t sum_a = __riscv_vwaddu_vv_u16m2(a0, a1, vl);
sum_a = __riscv_vwaddu_wv_u16m2(sum_a, a0_1, vl);
sum_a = __riscv_vwaddu_wv_u16m2(sum_a, a1_1, vl);
sum_a = __riscv_vadd_vx_u16m2(sum_a, 2, vl);
vuint8m1_t a = __riscv_vnsrl_wx_u8m1(sum_a, 2, vl);
vint16m2_t v_u_i16 = __riscv_vwmulsu_vv_i16m2(v_u0, b, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_u1, g, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_u2, r, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_u3, a, vl);
v_u_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_u_i16, vl);
vuint8m1_t v_u = __riscv_vnsrl_wx_u8m1(
__riscv_vreinterpret_v_i16m2_u16m2(v_u_i16), 8, vl);
__riscv_vse8_v_u8m1(dst_u, v_u, vl);
vint16m2_t v_v_i16 = __riscv_vwmulsu_vv_i16m2(v_v0, b, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_v1, g, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_v2, r, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_v3, a, vl);
v_v_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_v_i16, vl);
vuint8m1_t v_v = __riscv_vnsrl_wx_u8m1(
__riscv_vreinterpret_v_i16m2_u16m2(v_v_i16), 8, vl);
__riscv_vse8_v_u8m1(dst_v, v_v, vl);
w_len -= vl;
src_argb_0 += 8 * vl;
src_argb_1 += 8 * vl;
dst_u += vl;
dst_v += vl;
} while (w_len > 0);
}
if (width & 1) {
uint8_t b = (src_argb_0[0] + src_argb_1[0] + 1) >> 1;
uint8_t g = (src_argb_0[1] + src_argb_1[1] + 1) >> 1;
uint8_t r = (src_argb_0[2] + src_argb_1[2] + 1) >> 1;
uint8_t a = (src_argb_0[3] + src_argb_1[3] + 1) >> 1;
dst_u[0] = (c->kAddUV[0] - (c->kRGBToU[0] * b + c->kRGBToU[1] * g +
c->kRGBToU[2] * r + c->kRGBToU[3] * a)) >>
8;
dst_v[0] = (c->kAddUV[0] - (c->kRGBToV[0] * b + c->kRGBToV[1] * g +
c->kRGBToV[2] * r + c->kRGBToV[3] * a)) >>
8;
}
}
#endif
#ifdef HAS_RGBTOUV444MATRIXROW_RVV
void RGBToUV444MatrixRow_RVV(const uint8_t* src_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
assert(width != 0);
size_t w = (size_t)width;
vint8m2_t v_bu, v_gu, v_ru;
vint8m2_t v_bv, v_gv, v_rv;
vint16m4_t v_adduv;
size_t vl = __riscv_vsetvl_e8m2(w);
v_bu = __riscv_vmv_v_x_i8m2(c->kRGBToU[0], vl);
v_gu = __riscv_vmv_v_x_i8m2(c->kRGBToU[1], vl);
v_ru = __riscv_vmv_v_x_i8m2(c->kRGBToU[2], vl);
v_bv = __riscv_vmv_v_x_i8m2(c->kRGBToV[0], vl);
v_gv = __riscv_vmv_v_x_i8m2(c->kRGBToV[1], vl);
v_rv = __riscv_vmv_v_x_i8m2(c->kRGBToV[2], vl);
v_adduv = __riscv_vmv_v_x_i16m4((int16_t)c->kAddUV[0], vl);
do {
size_t vl = __riscv_vsetvl_e8m2(w);
vuint8m2x3_t v_src_rgb = __riscv_vlseg3e8_v_u8m2x3(src_rgb, vl);
vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 0);
vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 1);
vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 2);
vint16m4_t v_u_i16 = __riscv_vwmulsu_vv_i16m4(v_ru, v_r, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_gu, v_g, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_bu, v_b, vl);
v_u_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_u_i16, vl);
vuint8m2_t v_u = __riscv_vnsrl_wx_u8m2(
__riscv_vreinterpret_v_i16m4_u16m4(v_u_i16), 8, vl);
__riscv_vse8_v_u8m2(dst_u, v_u, vl);
vint16m4_t v_v_i16 = __riscv_vwmulsu_vv_i16m4(v_rv, v_r, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_gv, v_g, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_bv, v_b, vl);
v_v_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_v_i16, vl);
vuint8m2_t v_v = __riscv_vnsrl_wx_u8m2(
__riscv_vreinterpret_v_i16m4_u16m4(v_v_i16), 8, vl);
__riscv_vse8_v_u8m2(dst_v, v_v, vl);
w -= vl;
src_rgb += 3 * vl;
dst_u += vl;
dst_v += vl;
} while (w > 0);
}
#endif
#ifdef HAS_RGBTOUVMATRIXROW_RVV
void RGBToUVMatrixRow_RVV(const uint8_t* src_rgb,
int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
assert(width != 0);
const uint8_t* src_rgb_0 = src_rgb;
const uint8_t* src_rgb_1 = src_rgb + src_stride_rgb;
int w = width >> 1;
if (w > 0) {
size_t w_len = (size_t)w;
vint8m1_t v_bu, v_gu, v_ru;
vint8m1_t v_bv, v_gv, v_rv;
vint16m2_t v_adduv;
size_t vl = __riscv_vsetvl_e8m1(w_len);
v_bu = __riscv_vmv_v_x_i8m1(c->kRGBToU[0], vl);
v_gu = __riscv_vmv_v_x_i8m1(c->kRGBToU[1], vl);
v_ru = __riscv_vmv_v_x_i8m1(c->kRGBToU[2], vl);
v_bv = __riscv_vmv_v_x_i8m1(c->kRGBToV[0], vl);
v_gv = __riscv_vmv_v_x_i8m1(c->kRGBToV[1], vl);
v_rv = __riscv_vmv_v_x_i8m1(c->kRGBToV[2], vl);
v_adduv = __riscv_vmv_v_x_i16m2((int16_t)c->kAddUV[0], vl);
do {
size_t vl = __riscv_vsetvl_e8m1(w_len);
vuint8m1x6_t v_src0 = __riscv_vlseg6e8_v_u8m1x6(src_rgb_0, vl);
vuint8m1x6_t v_src1 = __riscv_vlseg6e8_v_u8m1x6(src_rgb_1, vl);
vuint8m1_t b0 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 0);
vuint8m1_t g0 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 1);
vuint8m1_t r0 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 2);
vuint8m1_t b1 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 3);
vuint8m1_t g1 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 4);
vuint8m1_t r1 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 5);
vuint8m1_t b0_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 0);
vuint8m1_t g0_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 1);
vuint8m1_t r0_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 2);
vuint8m1_t b1_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 3);
vuint8m1_t g1_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 4);
vuint8m1_t r1_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 5);
vuint16m2_t sum_b = __riscv_vwaddu_vv_u16m2(b0, b1, vl);
sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b0_1, vl);
sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b1_1, vl);
sum_b = __riscv_vadd_vx_u16m2(sum_b, 2, vl);
vuint8m1_t b = __riscv_vnsrl_wx_u8m1(sum_b, 2, vl);
vuint16m2_t sum_g = __riscv_vwaddu_vv_u16m2(g0, g1, vl);
sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g0_1, vl);
sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g1_1, vl);
sum_g = __riscv_vadd_vx_u16m2(sum_g, 2, vl);
vuint8m1_t g = __riscv_vnsrl_wx_u8m1(sum_g, 2, vl);
vuint16m2_t sum_r = __riscv_vwaddu_vv_u16m2(r0, r1, vl);
sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r0_1, vl);
sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r1_1, vl);
sum_r = __riscv_vadd_vx_u16m2(sum_r, 2, vl);
vuint8m1_t r = __riscv_vnsrl_wx_u8m1(sum_r, 2, vl);
vint16m2_t v_u_i16 = __riscv_vwmulsu_vv_i16m2(v_ru, r, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_gu, g, vl);
v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_bu, b, vl);
v_u_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_u_i16, vl);
vuint8m1_t v_u = __riscv_vnsrl_wx_u8m1(
__riscv_vreinterpret_v_i16m2_u16m2(v_u_i16), 8, vl);
__riscv_vse8_v_u8m1(dst_u, v_u, vl);
vint16m2_t v_v_i16 = __riscv_vwmulsu_vv_i16m2(v_rv, r, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_gv, g, vl);
v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_bv, b, vl);
v_v_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_v_i16, vl);
vuint8m1_t v_v = __riscv_vnsrl_wx_u8m1(
__riscv_vreinterpret_v_i16m2_u16m2(v_v_i16), 8, vl);
__riscv_vse8_v_u8m1(dst_v, v_v, vl);
w_len -= vl;
src_rgb_0 += 6 * vl;
src_rgb_1 += 6 * vl;
dst_u += vl;
dst_v += vl;
} while (w_len > 0);
}
if (width & 1) {
uint8_t b = (src_rgb_0[0] + src_rgb_1[0] + 1) >> 1;
uint8_t g = (src_rgb_0[1] + src_rgb_1[1] + 1) >> 1;
uint8_t r = (src_rgb_0[2] + src_rgb_1[2] + 1) >> 1;
dst_u[0] = (c->kAddUV[0] - (c->kRGBToU[0] * b + c->kRGBToU[1] * g +
c->kRGBToU[2] * r)) >>
8;
dst_v[0] = (c->kAddUV[0] - (c->kRGBToV[0] * b + c->kRGBToV[1] * g +
c->kRGBToV[2] * r)) >>
8;
}
}
#endif
// Blend src_argb over src_argb1 and store to dst_argb.
// dst_argb may be src_argb or src_argb1.
// src_argb: RGB values have already been pre-multiplied by the a.

8
source/row_sme.cc
View File

@ -1135,6 +1135,14 @@ __arm_locally_streaming void ARGBToUVMatrixRow_SME(
uvconstants);
}
__arm_locally_streaming void ARGBToYMatrixRow_SME(
const uint8_t* src_argb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c) {
ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c);
}
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
// defined(__aarch64__)

7
source/row_sve.cc
View File

@ -231,6 +231,13 @@ void ARGBToUVMatrixRow_SVE2(const uint8_t* src_argb,
uvconstants);
}
void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c) {
ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c);
}
#define ARGBTORGB565_SVE \
/* Inputs: \
* z0: rrrrrxxxbbbbbxxx \

397
source/row_win.cc
View File

@ -178,6 +178,7 @@ void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb,
dst_v += 32;
width -= 32;
}
_mm256_zeroupper();
}
#endif
LIBYUV_TARGET_AVX2
@ -228,6 +229,7 @@ void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb,
dst_y += 32;
width -= 32;
}
_mm256_zeroupper();
}
#ifdef HAS_RAWTOARGBROW_AVX2
@ -281,30 +283,59 @@ void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
dst_argb += 128;
width -= 32;
}
_mm256_zeroupper();
}
#endif
#if defined(HAS_RAWTOARGBROW_AVX512BW) || \
defined(HAS_RGB24TOARGBROW_AVX512BW) || \
defined(HAS_RGBTOYMATRIXROW_AVX512BW)
static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2,
3, 4, 4, 5};
static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8,
9, 10, 10, 11};
static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6,
7, 8, 8, 9};
static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4,
5, 6, 6, 7};
static const uint8_t kShuffleMaskRGB24ToARGB_AVX512BW[32] = {
0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u,
4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, 128u};
#endif
#if defined(HAS_RAWTOARGBROW_AVX512BW)
static const uint8_t kShuffleMaskRAWToARGB_AVX512BW[32] = {
2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u,
6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u};
#endif
#ifdef HAS_RAWTOARGBROW_AVX512BW
LIBYUV_TARGET_AVX512BW
void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_argb,
const __m128i* shuffler,
int width) {
static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_argb,
const uint32_t* shuffler,
int width) {
__m512i zmm_alpha = _mm512_set1_epi32(0xff000000);
__m512i zmm_perm =
_mm512_set_epi32(12, 11, 10, 9, 9, 8, 7, 6, 6, 5, 4, 3, 3, 2, 1, 0);
__m512i zmm_shuf = _mm512_broadcast_i32x4(_mm_loadu_si128(shuffler));
__m512i zmm_q0 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
__m512i zmm_q1 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
__m512i zmm_q2 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
__m512i zmm_q3 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
__m512i zmm_shuf =
_mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)shuffler));
while (width > 0) {
__m512i zmm0 = _mm512_maskz_loadu_epi8(0xffffffffffffull, src_raw);
__m512i zmm1 = _mm512_maskz_loadu_epi8(0xffffffffffffull, src_raw + 48);
__m512i zmm2 = _mm512_maskz_loadu_epi8(0xffffffffffffull, src_raw + 96);
__m512i zmm3 = _mm512_maskz_loadu_epi8(0xffffffffffffull, src_raw + 144);
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
zmm0 = _mm512_permutexvar_epi32(zmm_perm, zmm0);
zmm1 = _mm512_permutexvar_epi32(zmm_perm, zmm1);
zmm2 = _mm512_permutexvar_epi32(zmm_perm, zmm2);
zmm3 = _mm512_permutexvar_epi32(zmm_perm, zmm3);
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
@ -325,25 +356,114 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
dst_argb += 256;
width -= 64;
}
_mm256_zeroupper();
}
LIBYUV_TARGET_AVX512BW
void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_argb,
int width) {
__m128i shuf =
_mm_set_epi8(-1, 9, 10, 11, -1, 6, 7, 8, -1, 3, 4, 5, -1, 0, 1, 2);
RGBToARGBRow_AVX512BW(src_raw, dst_argb, &shuf, width);
RGBToARGBRow_AVX512BW(
src_raw, dst_argb,
(const uint32_t*)kShuffleMaskRAWToARGB_AVX512BW, width);
}
LIBYUV_TARGET_AVX512BW
void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width) {
__m128i shuf =
_mm_set_epi8(-1, 11, 10, 9, -1, 8, 7, 6, -1, 5, 4, 3, -1, 2, 1, 0);
RGBToARGBRow_AVX512BW(src_rgb24, dst_argb, &shuf, width);
RGBToARGBRow_AVX512BW(
src_rgb24, dst_argb,
(const uint32_t*)kShuffleMaskRGB24ToARGB_AVX512BW, width);
}
#ifdef HAS_RGBTOYMATRIXROW_AVX512BW
static const uint32_t kPermdARGBToY_AVX512BW[16] = {
0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15};
LIBYUV_TARGET_AVX512BW
void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb,
uint8_t* dst_y,
int width,
const struct ArgbConstants* c) {
__m512i zmm_q0 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW);
__m512i zmm_q1 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW);
__m512i zmm_q2 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW);
__m512i zmm_q3 =
_mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW);
__m512i zmm_shuf = _mm512_broadcast_i64x4(
_mm256_loadu_si256((const __m256i*)kShuffleMaskRGB24ToARGB_AVX512BW));
__m512i zmm_perm_y =
_mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW);
__m512i zmm_80 = _mm512_set1_epi8((char)0x80);
__m512i zmm_one = _mm512_set1_epi16(1);
__m512i zmm_coeff = _mm512_broadcast_i32x4(
_mm_loadu_si128((const __m128i*)c->kRGBToY));
__m512i zmm_add = _mm512_broadcast_i32x4(
_mm_loadu_si128((const __m128i*)c->kAddY));
// Compute bias adjusted for 0x80 subtraction:
// bias = kAddY - pmaddwd(pmaddubsw(kRGBToY, 0x80), 1)
__m512i zmm_bias = _mm512_maddubs_epi16(zmm_coeff, zmm_80);
zmm_bias = _mm512_madd_epi16(zmm_bias, zmm_one);
zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias);
zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias);
while (width > 0) {
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_rgb);
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_rgb + 64));
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_rgb + 128));
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
zmm0 = _mm512_xor_si512(zmm0, zmm_80);
zmm1 = _mm512_xor_si512(zmm1, zmm_80);
zmm2 = _mm512_xor_si512(zmm2, zmm_80);
zmm3 = _mm512_xor_si512(zmm3, zmm_80);
zmm0 = _mm512_maddubs_epi16(zmm_coeff, zmm0);
zmm1 = _mm512_maddubs_epi16(zmm_coeff, zmm1);
zmm2 = _mm512_maddubs_epi16(zmm_coeff, zmm2);
zmm3 = _mm512_maddubs_epi16(zmm_coeff, zmm3);
zmm0 = _mm512_madd_epi16(zmm0, zmm_one);
zmm1 = _mm512_madd_epi16(zmm1, zmm_one);
zmm0 = _mm512_packs_epi32(zmm0, zmm1);
zmm2 = _mm512_madd_epi16(zmm2, zmm_one);
zmm3 = _mm512_madd_epi16(zmm3, zmm_one);
zmm2 = _mm512_packs_epi32(zmm2, zmm3);
zmm0 = _mm512_add_epi16(zmm0, zmm_add);
zmm2 = _mm512_add_epi16(zmm2, zmm_add);
zmm0 = _mm512_srli_epi16(zmm0, 8);
zmm2 = _mm512_srli_epi16(zmm2, 8);
zmm0 = _mm512_packus_epi16(zmm0, zmm2);
zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0);
_mm512_storeu_si512((__m512i*)dst_y, zmm0);
src_rgb += 192;
dst_y += 64;
width -= 64;
}
_mm256_zeroupper();
}
#endif
#endif
#ifdef HAS_ARGBTOUVMATRIXROW_AVX2
@ -412,6 +532,7 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
dst_v += 8;
width -= 16;
}
_mm256_zeroupper();
}
#endif
@ -435,6 +556,7 @@ void MergeUVRow_AVX2(const uint8_t* src_u,
dst_uv += 32;
width -= 16;
}
_mm256_zeroupper();
}
#endif
@ -453,6 +575,7 @@ void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
dst += 32;
width -= 32;
}
_mm256_zeroupper();
}
#endif
@ -471,6 +594,7 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
dst_uv += 32;
width -= 16;
}
_mm256_zeroupper();
}
#endif
@ -494,6 +618,7 @@ void MirrorSplitUVRow_AVX2(const uint8_t* src_uv,
dst_v += 16;
width -= 16;
}
_mm256_zeroupper();
}
#endif
@ -547,6 +672,7 @@ void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24,
dst_rgb24 += 96;
width -= 32;
}
_mm256_zeroupper();
}
#endif
@ -659,6 +785,7 @@ void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
dst += 32;
width -= 8;
}
_mm256_zeroupper();
}
#endif
@ -696,6 +823,7 @@ void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) {
dst_argb += 64;
width -= 16;
}
_mm256_zeroupper();
}
#endif // HAS_J400TOARGBROW_AVX2
@ -914,6 +1042,7 @@ void ARGBShuffleRow_AVX2(const uint8_t* src_argb,
dst_argb += 64;
width -= 16;
}
_mm256_zeroupper();
}
#endif
@ -941,6 +1070,226 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
#endif
#ifdef HAS_I422TOARGBROW_AVX2
LIBYUV_TARGET_AVX2
void I422ToARGBRow_AVX2(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
__m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
__m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
__m256i ymm_u_zero = _mm256_setzero_si256();
ptrdiff_t offset = src_v - src_u;
while (width >= 16) {
__m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u);
__m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset));
src_u += 8;
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0);
__m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0);
ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1);
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
src_y += 16;
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
ymm0 = _mm256_srai_epi16(ymm0, 6);
ymm1 = _mm256_srai_epi16(ymm1, 6);
ymm2 = _mm256_srai_epi16(ymm2, 6);
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
__m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
__m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
_mm256_storeu_si256((__m256i*)dst_argb, argb0);
_mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
dst_argb += 64;
width -= 16;
}
_mm256_zeroupper();
}
#endif
#ifdef HAS_NV12TOARGBROW_AVX2
LIBYUV_TARGET_AVX2
void NV12ToARGBRow_AVX2(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
__m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
__m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
__m256i ymm_u_zero = _mm256_setzero_si256();
while (width >= 16) {
__m128i xmm_uv = _mm_loadu_si128((const __m128i*)src_uv);
src_uv += 16;
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_uv, 0);
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
src_y += 16;
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
__m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
ymm0 = _mm256_srai_epi16(ymm0, 6);
ymm1 = _mm256_srai_epi16(ymm1, 6);
ymm2 = _mm256_srai_epi16(ymm2, 6);
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
__m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
__m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
_mm256_storeu_si256((__m256i*)dst_argb, argb0);
_mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
dst_argb += 64;
width -= 16;
}
_mm256_zeroupper();
}
#endif
#ifdef HAS_NV21TOARGBROW_AVX2
static const uint8_t kShuffleNV21_AVX2[32] = {
1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u,
1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u};
LIBYUV_TARGET_AVX2
void NV21ToARGBRow_AVX2(const uint8_t* src_y,
const uint8_t* src_vu,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
__m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB);
__m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG);
__m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR);
__m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb);
__m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb);
__m256i ymm_128 = _mm256_set1_epi8((char)0x80);
__m256i ymm_alpha = _mm256_set1_epi8((char)0xff);
__m256i ymm_shuf_nv21 = _mm256_loadu_si256((const __m256i*)kShuffleNV21_AVX2);
__m256i ymm_u_zero = _mm256_setzero_si256();
while (width >= 16) {
__m128i xmm_vu = _mm_loadu_si128((const __m128i*)src_vu);
src_vu += 16;
__m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_vu, 0);
ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8);
ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf_nv21);
ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3);
__m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y);
src_y += 16;
__m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0);
ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8);
ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4);
ymm3 = _mm256_sub_epi8(ymm3, ymm_128);
ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb);
__m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3);
__m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3);
__m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3);
ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb);
ymm0 = _mm256_adds_epi16(ymm0, ymm4);
ymm1 = _mm256_subs_epi16(ymm4, ymm1);
ymm2 = _mm256_adds_epi16(ymm2, ymm4);
ymm0 = _mm256_srai_epi16(ymm0, 6);
ymm1 = _mm256_srai_epi16(ymm1, 6);
ymm2 = _mm256_srai_epi16(ymm2, 6);
ymm0 = _mm256_packus_epi16(ymm0, ymm0);
ymm1 = _mm256_packus_epi16(ymm1, ymm1);
ymm2 = _mm256_packus_epi16(ymm2, ymm2);
ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1);
ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8);
ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha);
ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8);
__m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2);
__m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2);
_mm256_storeu_si256((__m256i*)dst_argb, argb0);
_mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1);
dst_argb += 64;
width -= 16;
}
_mm256_zeroupper();
}
#endif
#ifdef HAS_I422TORGB24ROW_AVX2
static const uint8_t kShuffleMaskARGBToRGB24[2][16] = {
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
@ -1138,9 +1487,9 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
// STORERGB24_AVX512VBMI
__m512i zmm_BG = _mm512_permi2var_epi8(zmm0, zmm_mask_BG, zmm1);
__m512i zmm_dst0 = _mm512_permi2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
__m512i zmm_dst1 = _mm512_permi2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
__m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1);
__m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2);
__m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2);
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1));

196
unit_test/convert_test.cc
View File

@ -2216,6 +2216,202 @@ TEST_F(LibYUVConvertTest, TestRAWToI420) {
free_aligned_buffer_page_end(ref_v);
}
TEST_F(LibYUVConvertTest, TestRAWToI444) {
const int kWidth = 16;
const int kHeight = 16;
align_buffer_page_end(src_raw, kWidth * kHeight * 3);
align_buffer_page_end(dst_y, kWidth * kHeight);
align_buffer_page_end(dst_u, kWidth * kHeight);
align_buffer_page_end(dst_v, kWidth * kHeight);
align_buffer_page_end(src_argb, kWidth * kHeight * 4);
align_buffer_page_end(ref_y, kWidth * kHeight);
align_buffer_page_end(ref_u, kWidth * kHeight);
align_buffer_page_end(ref_v, kWidth * kHeight);
MemRandomize(src_raw, kWidth * kHeight * 3);
RAWToI444(src_raw, kWidth * 3, dst_y, kWidth, dst_u, kWidth, dst_v, kWidth,
kWidth, kHeight);
// Verify against RAWToARGB + ARGBToI444
RAWToARGB(src_raw, kWidth * 3, src_argb, kWidth * 4, kWidth, kHeight);
ARGBToI444(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth, ref_v, kWidth,
kWidth, kHeight);
for (int i = 0; i < kWidth * kHeight; ++i) {
ASSERT_EQ(dst_y[i], ref_y[i]);
ASSERT_EQ(dst_u[i], ref_u[i]);
ASSERT_EQ(dst_v[i], ref_v[i]);
}
free_aligned_buffer_page_end(src_raw);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_u);
free_aligned_buffer_page_end(dst_v);
free_aligned_buffer_page_end(src_argb);
free_aligned_buffer_page_end(ref_y);
free_aligned_buffer_page_end(ref_u);
free_aligned_buffer_page_end(ref_v);
}
TEST_F(LibYUVConvertTest, TestRAWToJ444) {
const int kWidth = 16;
const int kHeight = 16;
align_buffer_page_end(src_raw, kWidth * kHeight * 3);
align_buffer_page_end(dst_y, kWidth * kHeight);
align_buffer_page_end(dst_u, kWidth * kHeight);
align_buffer_page_end(dst_v, kWidth * kHeight);
align_buffer_page_end(src_argb, kWidth * kHeight * 4);
align_buffer_page_end(ref_y, kWidth * kHeight);
align_buffer_page_end(ref_u, kWidth * kHeight);
align_buffer_page_end(ref_v, kWidth * kHeight);
MemRandomize(src_raw, kWidth * kHeight * 3);
RAWToJ444(src_raw, kWidth * 3, dst_y, kWidth, dst_u, kWidth, dst_v, kWidth,
kWidth, kHeight);
// Verify against RAWToARGB + ARGBToJ444
RAWToARGB(src_raw, kWidth * 3, src_argb, kWidth * 4, kWidth, kHeight);
ARGBToJ444(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth, ref_v, kWidth,
kWidth, kHeight);
for (int i = 0; i < kWidth * kHeight; ++i) {
ASSERT_EQ(dst_y[i], ref_y[i]);
ASSERT_EQ(dst_u[i], ref_u[i]);
ASSERT_EQ(dst_v[i], ref_v[i]);
}
free_aligned_buffer_page_end(src_raw);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_u);
free_aligned_buffer_page_end(dst_v);
free_aligned_buffer_page_end(src_argb);
free_aligned_buffer_page_end(ref_y);
free_aligned_buffer_page_end(ref_u);
free_aligned_buffer_page_end(ref_v);
}
TEST_F(LibYUVConvertTest, TestRAWToNV21) {
const int kWidth = 16;
const int kHeight = 16;
align_buffer_page_end(src_raw, kWidth * kHeight * 3);
align_buffer_page_end(dst_y, kWidth * kHeight);
align_buffer_page_end(dst_vu, kWidth * kHeight / 2);
align_buffer_page_end(ref_y, kWidth * kHeight);
align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2);
align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2);
align_buffer_page_end(ref_vu, kWidth * kHeight / 2);
MemRandomize(src_raw, kWidth * kHeight * 3);
RAWToNV21(src_raw, kWidth * 3, dst_y, kWidth, dst_vu, kWidth, kWidth,
kHeight);
// Verify against RAWToI420 + interleaving V and U
RAWToI420(src_raw, kWidth * 3, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
kWidth / 2, kWidth, kHeight);
for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
ref_vu[i * 2] = ref_v[i];
ref_vu[i * 2 + 1] = ref_u[i];
}
for (int i = 0; i < kWidth * kHeight; ++i) {
ASSERT_EQ(dst_y[i], ref_y[i]);
}
for (int i = 0; i < kWidth * kHeight / 2; ++i) {
ASSERT_EQ(dst_vu[i], ref_vu[i]);
}
free_aligned_buffer_page_end(src_raw);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_vu);
free_aligned_buffer_page_end(ref_y);
free_aligned_buffer_page_end(ref_u);
free_aligned_buffer_page_end(ref_v);
free_aligned_buffer_page_end(ref_vu);
}
TEST_F(LibYUVConvertTest, TestRAWToJNV21) {
const int kWidth = 16;
const int kHeight = 16;
align_buffer_page_end(src_raw, kWidth * kHeight * 3);
align_buffer_page_end(dst_y, kWidth * kHeight);
align_buffer_page_end(dst_vu, kWidth * kHeight / 2);
align_buffer_page_end(ref_y, kWidth * kHeight);
align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2);
align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2);
align_buffer_page_end(ref_vu, kWidth * kHeight / 2);
MemRandomize(src_raw, kWidth * kHeight * 3);
RAWToJNV21(src_raw, kWidth * 3, dst_y, kWidth, dst_vu, kWidth, kWidth,
kHeight);
// Verify against RAWToJ420 + interleaving V and U
RAWToJ420(src_raw, kWidth * 3, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
kWidth / 2, kWidth, kHeight);
for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
ref_vu[i * 2] = ref_v[i];
ref_vu[i * 2 + 1] = ref_u[i];
}
for (int i = 0; i < kWidth * kHeight; ++i) {
ASSERT_EQ(dst_y[i], ref_y[i]);
}
for (int i = 0; i < kWidth * kHeight / 2; ++i) {
ASSERT_EQ(dst_vu[i], ref_vu[i]);
}
free_aligned_buffer_page_end(src_raw);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_vu);
free_aligned_buffer_page_end(ref_y);
free_aligned_buffer_page_end(ref_u);
free_aligned_buffer_page_end(ref_v);
free_aligned_buffer_page_end(ref_vu);
}
TEST_F(LibYUVConvertTest, TestRGB24ToNV12) {
const int kWidth = 16;
const int kHeight = 16;
align_buffer_page_end(src_rgb24, kWidth * kHeight * 3);
align_buffer_page_end(dst_y, kWidth * kHeight);
align_buffer_page_end(dst_uv, kWidth * kHeight / 2);
align_buffer_page_end(ref_y, kWidth * kHeight);
align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2);
align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2);
align_buffer_page_end(ref_uv, kWidth * kHeight / 2);
MemRandomize(src_rgb24, kWidth * kHeight * 3);
RGB24ToNV12(src_rgb24, kWidth * 3, dst_y, kWidth, dst_uv, kWidth, kWidth,
kHeight);
// Verify against RGB24ToI420 + interleaving U and V
RGB24ToI420(src_rgb24, kWidth * 3, ref_y, kWidth, ref_u, kWidth / 2, ref_v,
kWidth / 2, kWidth, kHeight);
for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) {
ref_uv[i * 2] = ref_u[i];
ref_uv[i * 2 + 1] = ref_v[i];
}
for (int i = 0; i < kWidth * kHeight; ++i) {
ASSERT_EQ(dst_y[i], ref_y[i]);
}
for (int i = 0; i < kWidth * kHeight / 2; ++i) {
ASSERT_EQ(dst_uv[i], ref_uv[i]);
}
free_aligned_buffer_page_end(src_rgb24);
free_aligned_buffer_page_end(dst_y);
free_aligned_buffer_page_end(dst_uv);
free_aligned_buffer_page_end(ref_y);
free_aligned_buffer_page_end(ref_u);
free_aligned_buffer_page_end(ref_v);
free_aligned_buffer_page_end(ref_uv);
}
TEST_F(LibYUVConvertTest, TestARGBToNV12Matrix) {
const int kWidth = 16;
const int kHeight = 16;