From 7138e0a2a7eab0ea7e200e49f047b80e6d7c474e Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Mon, 3 Aug 2026 17:38:31 -0700 Subject: [PATCH] Make RGBToY a 1 pass avx512bw - RGBToARGB use vpermi2q+vpshufb - direct RGBToY for avx512 and rvv +----------+----------+-------------+----------+---------+-----------+ | ARGBToY | ARGBToUV | ARGBToUV444 | RGBToY | RGBToUV | RGBToUV444| +----------+----------+-------------+----------+---------+-----------+ | avx512bw | avx512bw | avx512bw | avx512bw | 2 step | 2 step | | avx2 | avx2 | avx2 | 2 step | 2 step | 2 step | | ssse3 | ssse3 | ssse3 | 2 step | 2 step | 2 step | | dot | i8mm | i8mm | dot | neon | neon | | sve2 | sve2 | - | - | - | - | | sme | sme | - | - | - | - | | rvv | rvv | rvv | rvv | rvv | rvv | +----------+----------+-------------+----------+---------+-----------+ Bug: 42280902 Change-Id: I9f83bf3039431f9fb5391bd7d9eb393e186d6bcf Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8175442 Reviewed-by: Dale Curtis Reviewed-by: richard winterton Commit-Queue: Frank Barchard --- README.chromium | 2 +- include/libyuv/cpu_support.h | 8 +- include/libyuv/row.h | 153 ++++- include/libyuv/row_sve.h | 73 +++ include/libyuv/version.h | 2 +- source/convert.cc | 1055 +++++++++++++--------------------- source/convert_from_argb.cc | 335 ++++++----- source/row_any.cc | 20 + source/row_common.cc | 120 +++- source/row_gcc.cc | 294 +++++++--- source/row_neon.cc | 114 ++++ source/row_neon64.cc | 164 ++++++ source/row_rvv.cc | 341 ++++++++++- source/row_sme.cc | 8 + source/row_sve.cc | 7 + source/row_win.cc | 397 ++++++++++++- unit_test/convert_test.cc | 196 +++++++ 17 files changed, 2312 insertions(+), 977 deletions(-) diff --git a/README.chromium b/README.chromium index 3d4e00c57..f46e4d0f5 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1952 +Version: 1953 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index 79c6fb943..c8378dcfb 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -84,10 +84,14 @@ extern "C" { #define VISUALC_HAS_AVX2 1 #endif // VisualStudio >= 2012 -// Clang 19 required for SME due to needing __arm_tpidr2_save from compiler-rt, +#if defined(LIBYUV_ENABLE_SME) +#undef LIBYUV_DISABLE_SME +#endif + +// Clang 18+ required for SME due to needing __arm_tpidr2_save from compiler-rt, // only enabled on Linux and Android (both define __linux__) for now. #if !defined(LIBYUV_DISABLE_SME) && defined(__aarch64__) && \ - defined(__linux__) && defined(__clang__) && (__clang_major__ >= 19) + defined(__linux__) && defined(__clang__) && (__clang_major__ >= 18) #define CLANG_HAS_SME 1 #endif diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 2cff19758..569bd6bad 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -138,6 +138,9 @@ extern "C" { #define HAS_MIRRORUVROW_AVX2 #define HAS_INTERPOLATEROW_16_AVX2 #define HAS_INTERPOLATEROW_AVX2 +#define HAS_I422TOARGBROW_AVX2 +#define HAS_NV12TOARGBROW_AVX2 +#define HAS_NV21TOARGBROW_AVX2 #endif #if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_GCC) && \ @@ -236,8 +239,11 @@ extern "C" { #define HAS_YUY2TONVUVROW_SSE2 // TODO: port row_win to use 8 bit coefficients. #define HAS_ARGBTOYMATRIXROW_SSSE3 +#define HAS_RGBTOYMATRIXROW_SSSE3 #define HAS_ARGBTOUVMATRIXROW_SSSE3 +#define HAS_RGBTOUVMATRIXROW_SSSE3 #define HAS_ARGBTOUV444MATRIXROW_SSSE3 +#define HAS_RGBTOUV444MATRIXROW_SSSE3 #if defined(__x86_64__) || !defined(__pic__) // TODO(fbarchard): fix build error on android_full_debug=1 @@ -268,6 +274,7 @@ extern "C" { #define HAS_ARGB1555TOUVMATRIXROW_AVX2 #define HAS_ARGB4444TOUVMATRIXROW_AVX2 #define HAS_ARGBTOUV444MATRIXROW_AVX2 +#define HAS_RGBTOUV444MATRIXROW_AVX2 #define HAS_ARGBTOYMATRIXROW_AVX2 #define HAS_RGBTOYMATRIXROW_AVX2 #define HAS_RGB565TOYMATRIXROW_AVX2 @@ -339,38 +346,51 @@ extern "C" { #define HAS_RGB24TOARGBROW_AVX512BW #define HAS_ARGBSHUFFLEROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512VBMI +#define HAS_RGBTOYMATRIXROW_AVX512BW #endif #define HAS_ARGBTOYMATRIXROW_AVX2 #define HAS_RGBTOYMATRIXROW_AVX2 #define HAS_RGB565TOYMATRIXROW_AVX2 #define HAS_ARGB1555TOYMATRIXROW_AVX2 #define HAS_ARGB4444TOYMATRIXROW_AVX2 +#define HAS_ARGBTOUVMATRIXROW_AVX2 #define HAS_ARGBTOUV444MATRIXROW_AVX2 +#define HAS_RGBTOUV444MATRIXROW_AVX2 #define HAS_J400TOARGBROW_AVX2 +#define HAS_I422TOARGBROW_AVX2 +#define HAS_NV12TOARGBROW_AVX2 +#define HAS_NV21TOARGBROW_AVX2 +#define HAS_MERGEUVROW_AVX2 +#define HAS_MIRRORROW_AVX2 +#define HAS_MIRRORUVROW_AVX2 +#define HAS_MIRRORSPLITUVROW_AVX2 +#define HAS_RGB24MIRRORROW_AVX2 +#define HAS_ARGBMIRRORROW_AVX2 +#define HAS_INTERPOLATEROW_AVX2 +#define HAS_INTERPOLATEROW_16_AVX2 #endif -// The following are available for AVX512 clang x86 platforms: +// The following are available for AVX512 clang x64 platforms: // TODO(fbarchard): Port to GCC and Visual C // TODO(b/42280744): re-enable HAS_ARGBTORGB24ROW_AVX512VBMI. #if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(__i386__)) && defined(CLANG_HAS_AVX512) && \ + (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \ !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_COPYROW_AVX512BW -#if defined(__x86_64__) || defined(_M_X64) #define HAS_RAWTOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW -#endif #define HAS_ARGBTORGB24ROW_AVX512VBMI #define HAS_CONVERT16TO8ROW_AVX512BW #endif // The following are available for AVX512 clang x64 platforms: // TODO(fbarchard): Port to x86 -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - (defined(CLANG_HAS_AVX512)) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) #define HAS_I422TOARGBROW_AVX512BW #define HAS_ARGBSHUFFLEROW_AVX512BW #define HAS_ARGBTOUV444MATRIXROW_AVX512BW +#define HAS_RGBTOUV444MATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512VBMI #define HAS_ARGBTOUVMATRIXROW_AVX512BW @@ -380,6 +400,7 @@ extern "C" { #define HAS_MIRRORSPLITUVROW_AVX512BW #define HAS_SPLITUVROW_AVX512BW #define HAS_RGBTOUVMATRIXROW_AVX512BW +#define HAS_RGBTOYMATRIXROW_AVX512BW #endif // The following are available on Neon platforms: @@ -404,6 +425,7 @@ extern "C" { #define HAS_ARGBTORGB565DITHERROW_NEON #define HAS_ARGBTORGB565ROW_NEON #define HAS_ARGBTOUV444MATRIXROW_NEON +#define HAS_RGBTOUV444MATRIXROW_NEON #if !defined(__GNUC__) || defined(__clang__) #define HAS_ARGBTOUVMATRIXROW_NEON #define HAS_RGBTOUVMATRIXROW_NEON @@ -538,6 +560,7 @@ extern "C" { #define HAS_BGRATOYROW_NEON_DOTPROD #define HAS_RGBATOYJROW_NEON_DOTPROD #define HAS_RGBATOYROW_NEON_DOTPROD +#define HAS_RGBTOYMATRIXROW_NEON_DOTPROD #define HAS_ARGBCOLORMATRIXROW_NEON_I8MM #define HAS_ARGBTOUVMATRIXROW_NEON_I8MM @@ -552,6 +575,7 @@ extern "C" { #define HAS_ARGBTORGB565DITHERROW_SVE2 #define HAS_ARGBTORGB565ROW_SVE2 #define HAS_ARGBTOUVMATRIXROW_SVE2 +#define HAS_ARGBTOYMATRIXROW_SVE2 #define HAS_AYUVTOUVROW_SVE2 #define HAS_AYUVTOVUROW_SVE2 #define HAS_CONVERT8TO8ROW_SVE2 @@ -598,6 +622,7 @@ extern "C" { defined(__aarch64__) #define HAS_ARGBMULTIPLYROW_SME #define HAS_ARGBTOUVMATRIXROW_SME +#define HAS_ARGBTOYMATRIXROW_SME #define HAS_CONVERT16TO8ROW_SME #define HAS_CONVERT8TO16ROW_SME #define HAS_CONVERT8TO8ROW_SME @@ -760,6 +785,7 @@ extern "C" { #define HAS_RAWTOUVROW_LASX #define HAS_RGB24TOARGBROW_LASX #define HAS_RGB24TOUVROW_LASX +#define HAS_RGBTOYMATRIXROW_LASX #define HAS_RGB565TOARGBROW_LASX #define HAS_RGB565TOUVROW_LASX #define HAS_RGB565TOYROW_LASX @@ -790,9 +816,13 @@ extern "C" { #define HAS_ARGBCOPYYTOALPHAROW_RVV #define HAS_ARGBEXTRACTALPHAROW_RVV #define HAS_ARGBTOAR64ROW_RVV +#define HAS_ARGBTOUV444MATRIXROW_RVV +#define HAS_ARGBTOUVMATRIXROW_RVV #define HAS_ARGBTOYMATRIXROW_RVV #define HAS_COPYROW_RVV #define HAS_INTERPOLATEROW_RVV +#define HAS_RGBTOUV444MATRIXROW_RVV +#define HAS_RGBTOUVMATRIXROW_RVV #define HAS_RGBTOYMATRIXROW_RVV #define HAS_SPLITARGBROW_RVV #define HAS_SPLITRGBROW_RVV @@ -1723,6 +1753,14 @@ void ARGBToUVMatrixRow_SME(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c); +void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void ARGBToYMatrixRow_SME(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); void RGB24ToUVRow_NEON(const uint8_t* src_rgb24, int src_stride_rgb24, @@ -1828,6 +1866,11 @@ void RGBToUVMatrixRow_C(const uint8_t* src_rgb, uint8_t* dst_v, int width, const struct ArgbConstants* c); +void RGBToUV444MatrixRow_C(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); void RGB565ToYMatrixRow_C(const uint8_t* src_rgb565, uint8_t* dst_y, int width, @@ -2080,6 +2123,24 @@ void RGBToUVMatrixRow_Any_NEON(const uint8_t* src_rgb, uint8_t* dst_v, int width, const struct ArgbConstants* c); +void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_Any_NEON(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_NEON_DotProd(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_Any_NEON_DotProd(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); void ARGBToYMatrixRow_NEON_DotProd(const uint8_t* src_argb, uint8_t* dst_y, @@ -2109,6 +2170,44 @@ void RGBToYMatrixRow_RVV(const uint8_t* src_rgb, uint8_t* dst_y, int width, const struct ArgbConstants* c); +void ARGBToUV444MatrixRow_RVV(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void ARGBToUVMatrixRow_RVV(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_RVV(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUVMatrixRow_RVV(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_SSSE3(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_Any_SSSE3(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); +void RGBToYMatrixRow_Any_AVX512BW(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c); void ARGBToYMatrixRow_Any_NEON_DotProd(const uint8_t* src_argb, uint8_t* dst_y, @@ -2146,6 +2245,33 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c); +void RGBToUV444MatrixRow_SSSE3(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_AVX2(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_AVX512BW(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUVMatrixRow_SSSE3(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUVMatrixRow_Any_SSSE3(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); void ARGBToUVMatrixRow_Any_SSSE3(const uint8_t* src_argb, int src_stride_argb, uint8_t* dst_u, @@ -2179,6 +2305,21 @@ void ARGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_argb, uint8_t* dst_v, int width, const struct ArgbConstants* c); +void RGBToUV444MatrixRow_Any_SSSE3(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_Any_AVX2(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); +void RGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c); void ABGRToYJRow_C(const uint8_t* src_rgb, uint8_t* dst_y, int width); void RGBAToYJRow_C(const uint8_t* src_rgb, uint8_t* dst_y, int width); diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h index 280d635b9..b39f59a79 100644 --- a/include/libyuv/row_sve.h +++ b/include/libyuv/row_sve.h @@ -2144,6 +2144,79 @@ static inline void ARGBToUVMatrixRow_SVE_SC(const uint8_t* src_argb, "z27", "p0", "p1", "p2", "p3", "p4", "p5"); } +#define ARGBTOYMATRIX_SVE \ + "ld1w {z0.s}, p1/z, [%[src]] \n" /* load 4*vl pixels */\ + "ld1w {z1.s}, p2/z, [%[src], #1, mul vl] \n" \ + "ld1w {z2.s}, p3/z, [%[src], #2, mul vl] \n" \ + "ld1w {z3.s}, p4/z, [%[src], #3, mul vl] \n" \ + "incb %[src], all, mul #4 \n" \ + "fmov s16, wzr \n" \ + "fmov s17, wzr \n" \ + "fmov s18, wzr \n" \ + "fmov s19, wzr \n" \ + "udot z16.s, z0.b, z24.b \n" \ + "udot z17.s, z1.b, z24.b \n" \ + "udot z18.s, z2.b, z24.b \n" \ + "udot z19.s, z3.b, z24.b \n" \ + "uzp1 z16.h, z16.h, z17.h \n" \ + "uzp1 z18.h, z18.h, z19.h \n" \ + "add z16.h, z16.h, z25.h \n" \ + "add z18.h, z18.h, z25.h \n" \ + "uzp2 z16.b, z16.b, z18.b \n" \ + "st1b {z16.b}, p5, [%[dst_y]] \n" \ + "incb %[dst_y] \n" \ + "subs %w[width], %w[width], %w[vl], lsl #2 \n" + +static inline void ARGBToYMatrixRow_SVE_SC(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) + STREAMING_COMPATIBLE { + uint64_t vl; + asm("cntw %x0" : "=r"(vl)); + + asm volatile( + "ptrue p0.b \n" + "ld1rw {z24.s}, p0/z, [%[c]] \n" + "ld1rh {z25.h}, p0/z, [%[c], #48] \n" + + "subs %w[width], %w[width], %w[vl], lsl #2 \n" + "b.lt 2f \n" + + "ptrue p1.s \n" + "ptrue p2.s \n" + "ptrue p3.s \n" + "ptrue p4.s \n" + "ptrue p5.b \n" + "1: \n" // + ARGBTOYMATRIX_SVE + "b.gt 1b \n" + + "2: \n" + "adds %w[width], %w[width], %w[vl], lsl #2 \n" + "b.eq 99f \n" + + "3: \n" + "whilelt p1.s, wzr, %w[width] \n" + "whilelt p2.s, %w[vl], %w[width] \n" + "whilelt p3.s, %w[vl2], %w[width] \n" + "whilelt p4.s, %w[vl3], %w[width] \n" + "whilelt p5.b, wzr, %w[width] \n" // + ARGBTOYMATRIX_SVE + "b.gt 3b \n" + + "99: \n" + : [src] "+r"(src_argb), // %[src] + [dst_y] "+r"(dst_y), // %[dst_y] + [width] "+r"(width) // %[width] + : [c] "r"(c), // %[c] + [vl] "r"(vl), // %[vl] + [vl2] "r"(vl * 2), // %[vl2] + [vl3] "r"(vl * 3) // %[vl3] + : "cc", "memory", "z0", "z1", "z2", "z3", "z16", "z17", "z18", "z19", + "z24", "z25", "p0", "p1", "p2", "p3", "p4", "p5"); +} + #endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__) #ifdef __cplusplus diff --git a/include/libyuv/version.h b/include/libyuv/version.h index 5e34c512c..7f9da35e7 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1952 +#define LIBYUV_VERSION 1953 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/convert.cc b/source/convert.cc index ad194d2ae..0e023fcf1 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -2097,6 +2097,16 @@ int ARGBToI420Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; @@ -2172,6 +2182,11 @@ int ARGBToI420Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; + } #endif if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -2301,6 +2316,16 @@ int ARGBToI420Alpha(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; @@ -2377,6 +2402,11 @@ int ARGBToI420Alpha(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; + } +#endif #if defined(HAS_ARGBEXTRACTALPHAROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { @@ -2559,6 +2589,14 @@ int RGB24ToI420(const uint8_t* src_rgb24, void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, const struct ArgbConstants* c) = RGBToYMatrixRow_C; +#if defined(HAS_RGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_RGBTOYMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2; @@ -2567,6 +2605,22 @@ int RGB24ToI420(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; + } + } +#endif +#if defined(HAS_RGBTOUVMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_RGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2; @@ -2599,9 +2653,22 @@ int RGB24ToI420(const uint8_t* src_rgb24, } } #endif -#if defined(HAS_RGBTOYMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - RGBToYMatrixRow = RGBToYMatrixRow_LSX; // This uses the NEON/LSX names +#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd; + } + } +#endif +#if defined(HAS_RGBTOUVMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_RVV; + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif @@ -2654,6 +2721,14 @@ int RGB24ToJ420(const uint8_t* src_rgb24, void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, const struct ArgbConstants* c) = RGBToYMatrixRow_C; +#if defined(HAS_RGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_RGBTOYMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2; @@ -2662,6 +2737,22 @@ int RGB24ToJ420(const uint8_t* src_rgb24, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; + } + } +#endif +#if defined(HAS_RGBTOUVMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_RGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2; @@ -2694,9 +2785,22 @@ int RGB24ToJ420(const uint8_t* src_rgb24, } } #endif -#if defined(HAS_RGBTOYMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - RGBToYMatrixRow = RGBToYMatrixRow_LSX; // This uses the NEON/LSX names +#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd; + } + } +#endif +#if defined(HAS_RGBTOUVMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_RVV; + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif @@ -2754,6 +2858,14 @@ int RAWToI420(const uint8_t* src_raw, void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, const struct ArgbConstants* c) = RGBToYMatrixRow_C; +#if defined(HAS_RGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_RGBTOYMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2; @@ -2762,6 +2874,22 @@ int RAWToI420(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; + } + } +#endif +#if defined(HAS_RGBTOUVMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_RGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2; @@ -2794,9 +2922,12 @@ int RAWToI420(const uint8_t* src_raw, } } #endif -#if defined(HAS_RGBTOYMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - RGBToYMatrixRow = RGBToYMatrixRow_LSX; // This uses the NEON/LSX names +#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd; + } } #endif @@ -2849,6 +2980,14 @@ int RAWToJ420(const uint8_t* src_raw, void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, const struct ArgbConstants* c) = RGBToYMatrixRow_C; +#if defined(HAS_RGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_RGBTOYMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2; @@ -2857,6 +2996,22 @@ int RAWToJ420(const uint8_t* src_raw, } } #endif +#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; + } + } +#endif +#if defined(HAS_RGBTOUVMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3; + } + } +#endif #if defined(HAS_RGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2; @@ -2889,9 +3044,12 @@ int RAWToJ420(const uint8_t* src_raw, } } #endif -#if defined(HAS_RGBTOYMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - RGBToYMatrixRow = RGBToYMatrixRow_LSX; // This uses the NEON/LSX names +#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd; + } } #endif @@ -2924,8 +3082,140 @@ int RAWToJ420(const uint8_t* src_raw, return 0; } -// RAW big endian (rgb in memory) to I444 -// 2 step conversion of RAWToARGB then ARGBToY and ARGBToUV444 +// Convert RAW to I444 with matrix. +static int RAWToI444Matrix(const uint8_t* src_raw, + int src_stride_raw, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + const struct ArgbConstants* argbconstants, + int width, + int height) { + int y; + void (*RGBToUV444MatrixRow)(const uint8_t* src_rgb, uint8_t* dst_u, + uint8_t* dst_v, int width, + const struct ArgbConstants* c) = + RGBToUV444MatrixRow_C; + void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, + const struct ArgbConstants* c) = RGBToYMatrixRow_C; + +#if defined(HAS_RGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_SSSE3; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + RGBToYMatrixRow = RGBToYMatrixRow_AVX2; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + RGBToYMatrixRow = RGBToYMatrixRow_LASX; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + RGBToYMatrixRow = RGBToYMatrixRow_RVV; + } +#endif + +#if defined(HAS_RGBTOUV444MATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_SSSE3; + } + } +#endif +#if defined(HAS_RGBTOUV444MATRIXROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_AVX2; + } + } +#endif +#if defined(HAS_RGBTOUV444MATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_AVX512BW; + } + } +#endif +#if defined(HAS_RGBTOUV444MATRIXROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_Any_NEON; + if (IS_ALIGNED(width, 8)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_NEON; + } + } +#endif +#if defined(HAS_RGBTOUV444MATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + RGBToUV444MatrixRow = RGBToUV444MatrixRow_RVV; + } +#endif + + if (!src_raw || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || + height == 0 || height == INT_MIN) { + return -1; + } + if (height < 0) { + height = -height; + src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw; + src_stride_raw = -src_stride_raw; + } + + for (y = 0; y < height; ++y) { + RGBToYMatrixRow(src_raw, dst_y, width, argbconstants); + RGBToUV444MatrixRow(src_raw, dst_u, dst_v, width, argbconstants); + src_raw += src_stride_raw; + dst_y += dst_stride_y; + dst_u += dst_stride_u; + dst_v += dst_stride_v; + } + return 0; +} + +// Convert RAW to I444. LIBYUV_API int RAWToI444(const uint8_t* src_raw, int src_stride_raw, @@ -2937,210 +3227,12 @@ int RAWToI444(const uint8_t* src_raw, int dst_stride_v, int width, int height) { - int y; - void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) = - RAWToARGBRow_C; - void (*ARGBToYMatrixRow)(const uint8_t* src_raw, uint8_t* dst_y, int width, - const struct ArgbConstants* c) = ARGBToYMatrixRow_C; - void (*ARGBToUV444MatrixRow)(const uint8_t* src_raw, uint8_t* dst_u, - uint8_t* dst_v, int width, - const struct ArgbConstants* c) = - ARGBToUV444MatrixRow_C; - if (!src_raw || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || - height == INT_MIN) { - return -1; - } - if (height < 0) { - height = -height; - src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw; - src_stride_raw = -src_stride_raw; - } - // TODO: add row coalesce when main loop handles large width in blocks -#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_SSSE3; - } - } -#endif -#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX2; - } - } -#endif -#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW; - } - } -#endif -#if defined(HAS_ARGBTOUV444MATRIXROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON; - if (IS_ALIGNED(width, 8)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON; - } - } -#endif -#if defined(HAS_ARGBTOUV444MATRIXROW_NEON_I8MM) - if (TestCpuFlag(kCpuHasNeonI8MM)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON_I8MM; - if (IS_ALIGNED(width, 8)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON_I8MM; - } - } -#endif - -#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD) - if (TestCpuFlag(kCpuHasNeonDotProd)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_RVV) - if (TestCpuFlag(kCpuHasRVV)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; - } -#endif - -#if defined(HAS_RAWTOARGBROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - RAWToARGBRow = RAWToARGBRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - RAWToARGBRow = RAWToARGBRow_SSSE3; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - RAWToARGBRow = RAWToARGBRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - RAWToARGBRow = RAWToARGBRow_AVX2; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - RAWToARGBRow = RAWToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RAWToARGBRow = RAWToARGBRow_AVX512BW; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - RAWToARGBRow = RAWToARGBRow_Any_NEON; - if (IS_ALIGNED(width, 8)) { - RAWToARGBRow = RAWToARGBRow_NEON; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_SVE2) - if (TestCpuFlag(kCpuHasSVE2)) { - RAWToARGBRow = RAWToARGBRow_SVE2; - } -#endif -#if defined(HAS_RAWTOARGBROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - RAWToARGBRow = RAWToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { - RAWToARGBRow = RAWToARGBRow_LSX; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - RAWToARGBRow = RAWToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { - RAWToARGBRow = RAWToARGBRow_LASX; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_RVV) - if (TestCpuFlag(kCpuHasRVV)) { - RAWToARGBRow = RAWToARGBRow_RVV; - } -#endif - - { - // Allocate a row of ARGB. - const int row_size = width * 4; - align_buffer_64(row, row_size); - if (!row) - return 1; - - for (y = 0; y < height; ++y) { - RAWToARGBRow(src_raw, row, width); - ARGBToUV444MatrixRow(row, dst_u, dst_v, width, &kArgbI601Constants); - ARGBToYMatrixRow(row, dst_y, width, &kArgbI601Constants); - src_raw += src_stride_raw; - dst_y += dst_stride_y; - dst_u += dst_stride_u; - dst_v += dst_stride_v; - } - free_aligned_buffer_64(row); - } - return 0; + return RAWToI444Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, + &kAbgrI601Constants, width, height); } -// RAW big endian (rgb in memory) to J444 -// 2 step conversion of RAWToARGB then ARGBToYJ and ARGBToUVJ444 +// Convert RAW to J444. LIBYUV_API int RAWToJ444(const uint8_t* src_raw, int src_stride_raw, @@ -3152,198 +3244,9 @@ int RAWToJ444(const uint8_t* src_raw, int dst_stride_v, int width, int height) { - int y; - void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) = - RAWToARGBRow_C; - void (*ARGBToYMatrixRow)(const uint8_t* src_raw, uint8_t* dst_y, int width, - const struct ArgbConstants* c) = ARGBToYMatrixRow_C; - void (*ARGBToUV444MatrixRow)(const uint8_t* src_raw, uint8_t* dst_u, - uint8_t* dst_v, int width, - const struct ArgbConstants* c) = - ARGBToUV444MatrixRow_C; - if (!src_raw || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0 || - height == INT_MIN) { - return -1; - } - if (height < 0) { - height = -height; - src_raw = src_raw + (ptrdiff_t)(height - 1) * src_stride_raw; - src_stride_raw = -src_stride_raw; - } - // TODO: add row coalesce when main loop handles large width in blocks -#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_SSSE3; - } - } -#endif -#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX2; - } - } -#endif -#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_AVX512BW; - } - } -#endif -#if defined(HAS_ARGBTOUV444MATRIXROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_Any_NEON; - if (IS_ALIGNED(width, 8)) { - ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON; - } - } -#endif - -#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD) - if (TestCpuFlag(kCpuHasNeonDotProd)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_RVV) - if (TestCpuFlag(kCpuHasRVV)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; - } -#endif - -#if defined(HAS_RAWTOARGBROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - RAWToARGBRow = RAWToARGBRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - RAWToARGBRow = RAWToARGBRow_SSSE3; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - RAWToARGBRow = RAWToARGBRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - RAWToARGBRow = RAWToARGBRow_AVX2; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - RAWToARGBRow = RAWToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RAWToARGBRow = RAWToARGBRow_AVX512BW; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - RAWToARGBRow = RAWToARGBRow_Any_NEON; - if (IS_ALIGNED(width, 8)) { - RAWToARGBRow = RAWToARGBRow_NEON; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_SVE2) - if (TestCpuFlag(kCpuHasSVE2)) { - RAWToARGBRow = RAWToARGBRow_SVE2; - } -#endif -#if defined(HAS_RAWTOARGBROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - RAWToARGBRow = RAWToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { - RAWToARGBRow = RAWToARGBRow_LSX; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - RAWToARGBRow = RAWToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { - RAWToARGBRow = RAWToARGBRow_LASX; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_RVV) - if (TestCpuFlag(kCpuHasRVV)) { - RAWToARGBRow = RAWToARGBRow_RVV; - } -#endif - - { - // Allocate a row of ARGB. - const int row_size = width * 4; - align_buffer_64(row, row_size); - if (!row) - return 1; - - for (y = 0; y < height; ++y) { - RAWToARGBRow(src_raw, row, width); - ARGBToUV444MatrixRow(row, dst_u, dst_v, width, &kArgbJPEGConstants); - ARGBToYMatrixRow(row, dst_y, width, &kArgbJPEGConstants); - src_raw += src_stride_raw; - dst_y += dst_stride_y; - dst_u += dst_stride_u; - dst_v += dst_stride_v; - } - free_aligned_buffer_64(row); - } - return 0; + return RAWToI444Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, + &kAbgrJPEGConstants, width, height); } // Convert RGB565 to I420. @@ -3604,72 +3507,8 @@ int RGB24ToJ400(const uint8_t* src_rgb24, int width, int height) { int y; - void (*RGB24ToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) = - RGB24ToARGBRow_C; - void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width, - const struct ArgbConstants* c) = ARGBToYMatrixRow_C; -#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD) - if (TestCpuFlag(kCpuHasNeonDotProd)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_RVV) - if (TestCpuFlag(kCpuHasRVV)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; - } -#endif - + void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, + const struct ArgbConstants* c) = RGBToYMatrixRow_C; if (!src_rgb24 || !dst_yj || width <= 0 || height == 0 || height == INT_MIN) { return -1; } @@ -3685,78 +3524,65 @@ int RGB24ToJ400(const uint8_t* src_rgb24, height = 1; src_stride_rgb24 = dst_stride_yj = 0; } -#if defined(HAS_RGB24TOARGBROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - RGB24ToARGBRow = RGB24ToARGBRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - RGB24ToARGBRow = RGB24ToARGBRow_SSSE3; - } - } -#endif -#if defined(HAS_RGB24TOARGBROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - RGB24ToARGBRow = RGB24ToARGBRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - RGB24ToARGBRow = RGB24ToARGBRow_AVX2; - } - } -#endif -#if defined(HAS_RGB24TOARGBROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - RGB24ToARGBRow = RGB24ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - RGB24ToARGBRow = RGB24ToARGBRow_AVX512BW; - } - } -#endif -#if defined(HAS_RGB24TOARGBROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - RGB24ToARGBRow = RGB24ToARGBRow_Any_NEON; - if (IS_ALIGNED(width, 8)) { - RGB24ToARGBRow = RGB24ToARGBRow_NEON; - } - } -#endif -#if defined(HAS_RGB24TOARGBROW_SVE2) - if (TestCpuFlag(kCpuHasSVE2)) { - RGB24ToARGBRow = RGB24ToARGBRow_SVE2; - } -#endif -#if defined(HAS_RGB24TOARGBROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - RGB24ToARGBRow = RGB24ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { - RGB24ToARGBRow = RGB24ToARGBRow_LSX; - } - } -#endif -#if defined(HAS_RGB24TOARGBROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - RGB24ToARGBRow = RGB24ToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { - RGB24ToARGBRow = RGB24ToARGBRow_LASX; - } - } -#endif -#if defined(HAS_RGB24TOARGBROW_RVV) - if (TestCpuFlag(kCpuHasRVV)) { - RGB24ToARGBRow = RGB24ToARGBRow_RVV; - } -#endif - { - // Allocate 1 row of ARGB. - const int row_size = (width * 4 + 31) & ~31; - align_buffer_64(row, row_size); - if (!row) - return 1; - for (y = 0; y < height; ++y) { - RGB24ToARGBRow(src_rgb24, row, width); - ARGBToYMatrixRow(row, dst_yj, width, &kArgbJPEGConstants); - src_rgb24 += src_stride_rgb24; - dst_yj += dst_stride_yj; +#if defined(HAS_RGBTOYMATRIXROW_SSSE3) + if (TestCpuFlag(kCpuHasSSSE3)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_SSSE3; } - free_aligned_buffer_64(row); + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + RGBToYMatrixRow = RGBToYMatrixRow_AVX2; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + RGBToYMatrixRow = RGBToYMatrixRow_LASX; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + RGBToYMatrixRow = RGBToYMatrixRow_RVV; + } +#endif + + for (y = 0; y < height; ++y) { + RGBToYMatrixRow(src_rgb24, dst_yj, width, &kArgbJPEGConstants); + src_rgb24 += src_stride_rgb24; + dst_yj += dst_stride_yj; } return 0; } @@ -3770,72 +3596,8 @@ int RAWToJ400(const uint8_t* src_raw, int width, int height) { int y; - void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) = - RAWToARGBRow_C; - void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width, - const struct ArgbConstants* c) = ARGBToYMatrixRow_C; -#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD) - if (TestCpuFlag(kCpuHasNeonDotProd)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_RVV) - if (TestCpuFlag(kCpuHasRVV)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; - } -#endif - + void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, + const struct ArgbConstants* c) = RGBToYMatrixRow_C; if (!src_raw || !dst_yj || width <= 0 || height == 0 || height == INT_MIN) { return -1; } @@ -3852,79 +3614,64 @@ int RAWToJ400(const uint8_t* src_raw, src_stride_raw = dst_stride_yj = 0; } -#if defined(HAS_RAWTOARGBROW_SSSE3) +#if defined(HAS_RGBTOYMATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { - RAWToARGBRow = RAWToARGBRow_Any_SSSE3; + RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3; if (IS_ALIGNED(width, 16)) { - RAWToARGBRow = RAWToARGBRow_SSSE3; + RGBToYMatrixRow = RGBToYMatrixRow_SSSE3; } } #endif -#if defined(HAS_RAWTOARGBROW_AVX2) +#if defined(HAS_RGBTOYMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { - RAWToARGBRow = RAWToARGBRow_Any_AVX2; + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2; if (IS_ALIGNED(width, 32)) { - RAWToARGBRow = RAWToARGBRow_AVX2; + RGBToYMatrixRow = RGBToYMatrixRow_AVX2; } } #endif -#if defined(HAS_RAWTOARGBROW_AVX512BW) +#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RAWToARGBRow = RAWToARGBRow_Any_AVX512BW; + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; if (IS_ALIGNED(width, 64)) { - RAWToARGBRow = RAWToARGBRow_AVX512BW; + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } } #endif -#if defined(HAS_RAWTOARGBROW_NEON) +#if defined(HAS_RGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { - RAWToARGBRow = RAWToARGBRow_Any_NEON; - if (IS_ALIGNED(width, 8)) { - RAWToARGBRow = RAWToARGBRow_NEON; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_SVE2) - if (TestCpuFlag(kCpuHasSVE2)) { - RAWToARGBRow = RAWToARGBRow_SVE2; - } -#endif -#if defined(HAS_RAWTOARGBROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - RAWToARGBRow = RAWToARGBRow_Any_LSX; + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON; if (IS_ALIGNED(width, 16)) { - RAWToARGBRow = RAWToARGBRow_LSX; + RGBToYMatrixRow = RGBToYMatrixRow_NEON; } } #endif -#if defined(HAS_RAWTOARGBROW_LASX) +#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { - RAWToARGBRow = RAWToARGBRow_Any_LASX; + RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX; if (IS_ALIGNED(width, 32)) { - RAWToARGBRow = RAWToARGBRow_LASX; + RGBToYMatrixRow = RGBToYMatrixRow_LASX; } } #endif -#if defined(HAS_RAWTOARGBROW_RVV) +#if defined(HAS_RGBTOYMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { - RAWToARGBRow = RAWToARGBRow_RVV; + RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif - { - // Allocate 1 row of ARGB. - const int row_size = (width * 4 + 31) & ~31; - align_buffer_64(row, row_size); - if (!row) - return 1; - - for (y = 0; y < height; ++y) { - RAWToARGBRow(src_raw, row, width); - ARGBToYMatrixRow(row, dst_yj, width, &kArgbJPEGConstants); - src_raw += src_stride_raw; - dst_yj += dst_stride_yj; - } - free_aligned_buffer_64(row); + for (y = 0; y < height; ++y) { + RGBToYMatrixRow(src_raw, dst_yj, width, &kAbgrJPEGConstants); + src_raw += src_stride_raw; + dst_yj += dst_stride_yj; } return 0; } diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 187d5a9c7..d03b0ab61 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -98,6 +98,16 @@ int ARGBToI444Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; @@ -159,6 +169,11 @@ int ARGBToI444Matrix(const uint8_t* src_argb, ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_NEON_I8MM; } } +#endif +#if defined(HAS_ARGBTOUV444MATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBToUV444MatrixRow = ARGBToUV444MatrixRow_RVV; + } #endif if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -259,6 +274,16 @@ int ARGBToI422Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; @@ -334,6 +359,11 @@ int ARGBToI422Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; + } #endif if (!src_argb || !dst_y || !dst_u || !dst_v || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { @@ -430,6 +460,16 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; @@ -497,6 +537,11 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) = MergeUVRow_C; @@ -645,6 +690,16 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; @@ -712,6 +767,11 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; } } +#endif +#if defined(HAS_ARGBTOUVMATRIXROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ARGBToUVMatrixRow = ARGBToUVMatrixRow_RVV; + } #endif void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_vu, int width) = MergeUVRow_C; @@ -853,6 +913,16 @@ int ARGBToI400Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTOYMATRIXROW_SVE2) + if (TestCpuFlag(kCpuHasSVE2)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SVE2; + } +#endif +#if defined(HAS_ARGBTOYMATRIXROW_SME) + if (TestCpuFlag(kCpuHasSME)) { + ARGBToYMatrixRow = ARGBToYMatrixRow_SME; + } +#endif for (y = 0; y < height; ++y) { ARGBToYMatrixRow(src_argb, dst_y, width, constants); @@ -2072,91 +2142,28 @@ int ARGBToAB64(const uint8_t* src_argb, } // Convert RAW to NV21 with Matrix. -LIBYUV_API -int RAWToNV21Matrix(const uint8_t* src_raw, - int src_stride_raw, - uint8_t* dst_y, - int dst_stride_y, - uint8_t* dst_vu, - int dst_stride_vu, - const struct ArgbConstants* argbconstants, - int width, - int height) { +static int RAWToNVMatrix(const uint8_t* src_raw, + int src_stride_raw, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + const struct ArgbConstants* argbconstants, + int width, + int height, + int is_nv21) { int y; int halfwidth = (width + 1) >> 1; - void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) = - RAWToARGBRow_C; - void (*ARGBToUVMatrixRow)(const uint8_t* src_argb0, int src_stride_argb, - uint8_t* dst_u, uint8_t* dst_v, int width, - const struct ArgbConstants* c) = - ARGBToUVMatrixRow_C; - void (*ARGBToYMatrixRow)(const uint8_t* src_argb, uint8_t* dst_y, int width, - const struct ArgbConstants* c) = ARGBToYMatrixRow_C; + void (*RGBToUVMatrixRow)(const uint8_t* src_rgb, int src_stride_rgb, + uint8_t* dst_u, uint8_t* dst_v, int width, + const struct ArgbConstants* c) = + RGBToUVMatrixRow_C; + void (*RGBToYMatrixRow)(const uint8_t* src_rgb, uint8_t* dst_y, int width, + const struct ArgbConstants* c) = RGBToYMatrixRow_C; void (*MergeUVRow)(const uint8_t* src_uj, const uint8_t* src_vj, uint8_t* dst_vu, int width) = MergeUVRow_C; -#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_SSSE3; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX2) - if (TestCpuFlag(kCpuHasAVX2)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX2; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_AVX512BW; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_NEON_DOTPROD) - if (TestCpuFlag(kCpuHasNeonDotProd)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_NEON_DotProd; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_NEON_DotProd; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; - } - } -#endif -#if defined(HAS_ARGBTOYMATRIXROW_RVV) - if (TestCpuFlag(kCpuHasRVV)) { - ARGBToYMatrixRow = ARGBToYMatrixRow_RVV; - } -#endif - if (!src_raw || !dst_y || !dst_vu || !argbconstants || width <= 0 || + if (!src_raw || !dst_y || !dst_uv || !argbconstants || width <= 0 || height == 0 || height == INT_MIN) { return -1; } @@ -2167,115 +2174,89 @@ int RAWToNV21Matrix(const uint8_t* src_raw, src_stride_raw = -src_stride_raw; } -#if defined(HAS_RAWTOARGBROW_SSSE3) +#if defined(HAS_RGBTOYMATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { - RAWToARGBRow = RAWToARGBRow_Any_SSSE3; + RGBToYMatrixRow = RGBToYMatrixRow_Any_SSSE3; if (IS_ALIGNED(width, 16)) { - RAWToARGBRow = RAWToARGBRow_SSSE3; + RGBToYMatrixRow = RGBToYMatrixRow_SSSE3; } } #endif -#if defined(HAS_RAWTOARGBROW_AVX2) +#if defined(HAS_RGBTOYMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { - RAWToARGBRow = RAWToARGBRow_Any_AVX2; + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX2; if (IS_ALIGNED(width, 32)) { - RAWToARGBRow = RAWToARGBRow_AVX2; + RGBToYMatrixRow = RGBToYMatrixRow_AVX2; } } #endif -#if defined(HAS_RAWTOARGBROW_AVX512BW) +#if defined(HAS_RGBTOYMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - RAWToARGBRow = RAWToARGBRow_Any_AVX512BW; + RGBToYMatrixRow = RGBToYMatrixRow_Any_AVX512BW; if (IS_ALIGNED(width, 64)) { - RAWToARGBRow = RAWToARGBRow_AVX512BW; + RGBToYMatrixRow = RGBToYMatrixRow_AVX512BW; } } #endif -#if defined(HAS_RAWTOARGBROW_NEON) +#if defined(HAS_RGBTOYMATRIXROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { - RAWToARGBRow = RAWToARGBRow_Any_NEON; - if (IS_ALIGNED(width, 8)) { - RAWToARGBRow = RAWToARGBRow_NEON; - } - } -#endif -#if defined(HAS_RAWTOARGBROW_SVE2) - if (TestCpuFlag(kCpuHasSVE2)) { - RAWToARGBRow = RAWToARGBRow_SVE2; - } -#endif -#if defined(HAS_RAWTOARGBROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - RAWToARGBRow = RAWToARGBRow_Any_LSX; + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON; if (IS_ALIGNED(width, 16)) { - RAWToARGBRow = RAWToARGBRow_LSX; + RGBToYMatrixRow = RGBToYMatrixRow_NEON; } } #endif -#if defined(HAS_RAWTOARGBROW_LASX) +#if defined(HAS_RGBTOYMATRIXROW_NEON_DOTPROD) + if (TestCpuFlag(kCpuHasNeonDotProd)) { + RGBToYMatrixRow = RGBToYMatrixRow_Any_NEON_DotProd; + if (IS_ALIGNED(width, 16)) { + RGBToYMatrixRow = RGBToYMatrixRow_NEON_DotProd; + } + } +#endif +#if defined(HAS_RGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { - RAWToARGBRow = RAWToARGBRow_Any_LASX; + RGBToYMatrixRow = RGBToYMatrixRow_Any_LASX; if (IS_ALIGNED(width, 32)) { - RAWToARGBRow = RAWToARGBRow_LASX; + RGBToYMatrixRow = RGBToYMatrixRow_LASX; } } #endif -#if defined(HAS_RAWTOARGBROW_RVV) +#if defined(HAS_RGBTOYMATRIXROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { - RAWToARGBRow = RAWToARGBRow_RVV; + RGBToYMatrixRow = RGBToYMatrixRow_RVV; } #endif -#if defined(HAS_ARGBTOUVMATRIXROW_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON; - if (IS_ALIGNED(width, 16)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON; - } - } -#endif -#if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) - if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; - if (IS_ALIGNED(width, 16)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; - } - } -#endif -#if defined(HAS_ARGBTOUVMATRIXROW_SVE2) - if (TestCpuFlag(kCpuHasSVE2)) { - if (IS_ALIGNED(width, 2)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_SVE2; - } - } -#endif -#if defined(HAS_ARGBTOUVMATRIXROW_SME) - if (TestCpuFlag(kCpuHasSME)) { - if (IS_ALIGNED(width, 2)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_SME; - } - } -#endif -#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3) + +#if defined(HAS_RGBTOUVMATRIXROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_SSSE3; - if (IS_ALIGNED(width, 8)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_SSSE3; + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_SSSE3; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_SSSE3; } } #endif -#if defined(HAS_ARGBTOUVMATRIXROW_AVX2) +#if defined(HAS_RGBTOUVMATRIXROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX2; + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX2; if (IS_ALIGNED(width, 32)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX2; + RGBToUVMatrixRow = RGBToUVMatrixRow_AVX2; } } #endif -#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) +#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW; + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW; if (IS_ALIGNED(width, 64)) { - ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW; + RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW; + } + } +#endif +#if defined(HAS_RGBTOUVMATRIXROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_Any_NEON; + if (IS_ALIGNED(width, 16)) { + RGBToUVMatrixRow = RGBToUVMatrixRow_NEON; } } #endif @@ -2332,45 +2313,55 @@ int RAWToNV21Matrix(const uint8_t* src_raw, #endif { - // Allocate 2 rows of ARGB. - const int row_size = (width * 4 + 31) & ~31; - align_buffer_64(row, row_size * 2); // Allocate 1 row of U and 1 row of V. align_buffer_64(row_u, halfwidth); align_buffer_64(row_v, halfwidth); - if (!row || !row_u || !row_v) { - free_aligned_buffer_64(row); + if (!row_u || !row_v) { free_aligned_buffer_64(row_u); free_aligned_buffer_64(row_v); return 1; } + const uint8_t* first_chroma = is_nv21 ? row_v : row_u; + const uint8_t* second_chroma = is_nv21 ? row_u : row_v; + for (y = 0; y < height - 1; y += 2) { - RAWToARGBRow(src_raw, row, width); - RAWToARGBRow(src_raw + src_stride_raw, row + row_size, width); - ARGBToUVMatrixRow(row, row_size, row_u, row_v, width, argbconstants); - MergeUVRow(row_v, row_u, dst_vu, halfwidth); - ARGBToYMatrixRow(row, dst_y, width, argbconstants); - ARGBToYMatrixRow(row + row_size, dst_y + dst_stride_y, width, + RGBToUVMatrixRow(src_raw, src_stride_raw, row_u, row_v, width, argbconstants); + MergeUVRow(first_chroma, second_chroma, dst_uv, halfwidth); + RGBToYMatrixRow(src_raw, dst_y, width, argbconstants); + RGBToYMatrixRow(src_raw + src_stride_raw, dst_y + dst_stride_y, width, + argbconstants); src_raw += src_stride_raw * 2; dst_y += dst_stride_y * 2; - dst_vu += dst_stride_vu; + dst_uv += dst_stride_uv; } if (height & 1) { - RAWToARGBRow(src_raw, row, width); - ARGBToUVMatrixRow(row, 0, row_u, row_v, width, argbconstants); - MergeUVRow(row_v, row_u, dst_vu, halfwidth); - ARGBToYMatrixRow(row, dst_y, width, argbconstants); + RGBToUVMatrixRow(src_raw, 0, row_u, row_v, width, argbconstants); + MergeUVRow(first_chroma, second_chroma, dst_uv, halfwidth); + RGBToYMatrixRow(src_raw, dst_y, width, argbconstants); } free_aligned_buffer_64(row_v); free_aligned_buffer_64(row_u); - free_aligned_buffer_64(row); } return 0; } +LIBYUV_API +int RAWToNV21Matrix(const uint8_t* src_raw, + int src_stride_raw, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_vu, + int dst_stride_vu, + const struct ArgbConstants* argbconstants, + int width, + int height) { + return RAWToNVMatrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_vu, + dst_stride_vu, argbconstants, width, height, 1); +} + LIBYUV_API int RAWToJNV21(const uint8_t* src_raw, int src_stride_raw, @@ -2381,7 +2372,7 @@ int RAWToJNV21(const uint8_t* src_raw, int width, int height) { return RAWToNV21Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_vu, - dst_stride_vu, &kArgbJPEGConstants, width, height); + dst_stride_vu, &kAbgrJPEGConstants, width, height); } LIBYUV_API @@ -2394,7 +2385,7 @@ int RAWToNV21(const uint8_t* src_raw, int width, int height) { return RAWToNV21Matrix(src_raw, src_stride_raw, dst_y, dst_stride_y, dst_vu, - dst_stride_vu, &kArgbI601Constants, width, height); + dst_stride_vu, &kAbgrI601Constants, width, height); } LIBYUV_API @@ -2406,9 +2397,9 @@ int RGB24ToNV12(const uint8_t* src_rgb24, int dst_stride_uv, int width, int height) { - return RAWToNV21Matrix(src_rgb24, src_stride_rgb24, dst_y, dst_stride_y, - dst_uv, dst_stride_uv, &kAbgrI601Constants, width, - height); + return RAWToNVMatrix(src_rgb24, src_stride_rgb24, dst_y, dst_stride_y, + dst_uv, dst_stride_uv, &kArgbI601Constants, width, + height, 0); } #ifdef __cplusplus diff --git a/source/row_any.cc b/source/row_any.cc index ad8485848..4c19d8e15 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -2107,18 +2107,33 @@ ANY12MS(RGBToUVMatrixRow_Any_AVX512BW, RGBToUVMatrixRow_AVX512BW, 0, 3, 63) #ifdef HAS_ARGBTOUVMATRIXROW_SSSE3 ANY12MS(ARGBToUVMatrixRow_Any_SSSE3, ARGBToUVMatrixRow_SSSE3, 0, 4, 7) #endif +#ifdef HAS_RGBTOUVMATRIXROW_SSSE3 +ANY12MS(RGBToUVMatrixRow_Any_SSSE3, RGBToUVMatrixRow_SSSE3, 0, 3, 7) +#endif #ifdef HAS_ARGBTOUV444MATRIXROW_AVX2 ANY12M(ARGBToUV444MatrixRow_Any_AVX2, ARGBToUV444MatrixRow_AVX2, 4, 31) #endif +#ifdef HAS_RGBTOUV444MATRIXROW_AVX2 +ANY12M(RGBToUV444MatrixRow_Any_AVX2, RGBToUV444MatrixRow_AVX2, 3, 31) +#endif #ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW ANY12M(ARGBToUV444MatrixRow_Any_AVX512BW, ARGBToUV444MatrixRow_AVX512BW, 4, 63) #endif +#ifdef HAS_RGBTOUV444MATRIXROW_AVX512BW +ANY12M(RGBToUV444MatrixRow_Any_AVX512BW, RGBToUV444MatrixRow_AVX512BW, 3, 63) +#endif #ifdef HAS_ARGBTOUV444MATRIXROW_SSSE3 ANY12M(ARGBToUV444MatrixRow_Any_SSSE3, ARGBToUV444MatrixRow_SSSE3, 4, 15) #endif +#ifdef HAS_RGBTOUV444MATRIXROW_SSSE3 +ANY12M(RGBToUV444MatrixRow_Any_SSSE3, RGBToUV444MatrixRow_SSSE3, 3, 15) +#endif #ifdef HAS_ARGBTOUV444MATRIXROW_NEON ANY12M(ARGBToUV444MatrixRow_Any_NEON, ARGBToUV444MatrixRow_NEON, 4, 7) #endif +#ifdef HAS_RGBTOUV444MATRIXROW_NEON +ANY12M(RGBToUV444MatrixRow_Any_NEON, RGBToUV444MatrixRow_NEON, 3, 7) +#endif #ifdef HAS_ARGBTOUV444MATRIXROW_NEON_I8MM ANY12M(ARGBToUV444MatrixRow_Any_NEON_I8MM, ARGBToUV444MatrixRow_NEON_I8MM, @@ -2144,6 +2159,7 @@ ANY12M(ARGBToUV444MatrixRow_Any_NEON_I8MM, #ifdef HAS_ARGBTOYMATRIXROW_SSSE3 ANY11MC(ARGBToYMatrixRow_Any_SSSE3, ARGBToYMatrixRow_SSSE3, 4, 15) +ANY11MC(RGBToYMatrixRow_Any_SSSE3, RGBToYMatrixRow_SSSE3, 3, 15) #endif #ifdef HAS_ARGBTOYMATRIXROW_AVX2 ANY11MC(ARGBToYMatrixRow_Any_AVX2, ARGBToYMatrixRow_AVX2, 4, 31) @@ -2159,11 +2175,15 @@ ANY11MC(ARGB4444ToYMatrixRow_Any_AVX2, ARGB4444ToYMatrixRow_AVX2, 2, 31) #ifdef HAS_ARGBTOYMATRIXROW_AVX512BW ANY11MC(ARGBToYMatrixRow_Any_AVX512BW, ARGBToYMatrixRow_AVX512BW, 4, 63) #endif +#ifdef HAS_RGBTOYMATRIXROW_AVX512BW +ANY11MC(RGBToYMatrixRow_Any_AVX512BW, RGBToYMatrixRow_AVX512BW, 3, 63) +#endif #ifdef HAS_ARGBTOYMATRIXROW_NEON ANY11MC(ARGBToYMatrixRow_Any_NEON, ARGBToYMatrixRow_NEON, 4, 15) #endif #ifdef HAS_ARGBTOYMATRIXROW_NEON_DOTPROD ANY11MC(ARGBToYMatrixRow_Any_NEON_DotProd, ARGBToYMatrixRow_NEON_DotProd, 4, 15) +ANY11MC(RGBToYMatrixRow_Any_NEON_DotProd, RGBToYMatrixRow_NEON_DotProd, 3, 15) #endif #ifdef HAS_RGBTOYMATRIXROW_NEON ANY11MC(RGBToYMatrixRow_Any_NEON, RGBToYMatrixRow_NEON, 3, 15) diff --git a/source/row_common.cc b/source/row_common.cc index 3f1ff4456..c547556a8 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -4546,6 +4546,40 @@ void RGBToUVMatrixRow_C(const uint8_t* src_rgb, } } +void RGBToUV444MatrixRow_C(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + RGB24ToARGBRow_C(src_rgb, row, twidth); + ARGBToUV444MatrixRow_C(row, dst_u, dst_v, twidth, c); + src_rgb += twidth * 3; + dst_u += twidth; + dst_v += twidth; + width -= twidth; + } +} + +#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) && defined(HAS_RGB24TOARGBROW_SSSE3) +void RGBToYMatrixRow_SSSE3(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + RGB24ToARGBRow_SSSE3(src_rgb, row, twidth); + ARGBToYMatrixRow_SSSE3(row, dst_y, twidth, c); + src_rgb += twidth * 3; + dst_y += twidth; + width -= twidth; + } +} +#endif + #if defined(HAS_ARGBTOYMATRIXROW_AVX2) && defined(HAS_RGB24TOARGBROW_AVX2) void RGBToYMatrixRow_AVX2(const uint8_t* src_rgb, uint8_t* dst_y, @@ -4563,6 +4597,27 @@ void RGBToYMatrixRow_AVX2(const uint8_t* src_rgb, } #endif +#if defined(HAS_ARGBTOUVMATRIXROW_SSSE3) && defined(HAS_RGB24TOARGBROW_SSSE3) +void RGBToUVMatrixRow_SSSE3(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + RGB24ToARGBRow_SSSE3(src_rgb, row, twidth); + RGB24ToARGBRow_SSSE3(src_rgb + src_stride_rgb, row + MAXTWIDTH * 4, twidth); + ARGBToUVMatrixRow_SSSE3(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c); + src_rgb += twidth * 3; + dst_u += twidth / 2; + dst_v += twidth / 2; + width -= twidth; + } +} +#endif + #if defined(HAS_ARGBTOUVMATRIXROW_AVX2) && defined(HAS_RGB24TOARGBROW_AVX2) void RGBToUVMatrixRow_AVX2(const uint8_t* src_rgb, int src_stride_rgb, @@ -4607,22 +4662,61 @@ void RGBToUVMatrixRow_AVX512BW(const uint8_t* src_rgb, } #endif -#if defined(HAS_ARGBTOUVMATRIXROW_NEON) && defined(HAS_RGB24TOARGBROW_NEON) -void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb, - int src_stride_rgb, - uint8_t* dst_u, - uint8_t* dst_v, - int width, - const struct ArgbConstants* c) { - SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4 * 2]); +#if defined(HAS_ARGBTOUV444MATRIXROW_SSSE3) && \ + defined(HAS_RGB24TOARGBROW_SSSE3) +void RGBToUV444MatrixRow_SSSE3(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); while (width > 0) { int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; - RGB24ToARGBRow_NEON(src_rgb, row, twidth); - RGB24ToARGBRow_NEON(src_rgb + src_stride_rgb, row + MAXTWIDTH * 4, twidth); - ARGBToUVMatrixRow_NEON(row, MAXTWIDTH * 4, dst_u, dst_v, twidth, c); + RGB24ToARGBRow_SSSE3(src_rgb, row, twidth); + ARGBToUV444MatrixRow_SSSE3(row, dst_u, dst_v, twidth, c); src_rgb += twidth * 3; - dst_u += twidth / 2; - dst_v += twidth / 2; + dst_u += twidth; + dst_v += twidth; + width -= twidth; + } +} +#endif + +#if defined(HAS_ARGBTOUV444MATRIXROW_AVX2) && \ + defined(HAS_RGB24TOARGBROW_AVX2) +void RGBToUV444MatrixRow_AVX2(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + RGB24ToARGBRow_AVX2(src_rgb, row, twidth); + ARGBToUV444MatrixRow_AVX2(row, dst_u, dst_v, twidth, c); + src_rgb += twidth * 3; + dst_u += twidth; + dst_v += twidth; + width -= twidth; + } +} +#endif + +#if defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) && \ + defined(HAS_RGB24TOARGBROW_AVX512BW) +void RGBToUV444MatrixRow_AVX512BW(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + RGB24ToARGBRow_AVX512BW(src_rgb, row, twidth); + ARGBToUV444MatrixRow_AVX512BW(row, dst_u, dst_v, twidth, c); + src_rgb += twidth * 3; + dst_u += twidth; + dst_v += twidth; width -= twidth; } } diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 87e1e18b8..4318cbf93 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -63,6 +63,12 @@ static const uvec8 kShuffleMaskRAWToARGB = { static const uvec8 kShuffleMaskRAWToARGB_0 = {6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u}; +#if defined(HAS_RAWTOARGBROW_AVX512BW) +static const uvec8 kShuffleMaskRAWToARGB_AVX512BW[2] = { + {2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u}, + {6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, + 128u}}; +#endif // Shuffle table for converting RAW to RGBA. static const uvec8 kShuffleMaskRAWToRGBA = { @@ -163,8 +169,8 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y, uint8_t* dst_argb, int width) { asm volatile( - "vpternlogd $0xff,%%zmm7,%%zmm7,%%zmm7 \n" // 0xffffffff - "vpslld $0x18,%%zmm7,%%zmm7 \n" // 0xff000000 + "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // 0xffffffff + "vpslld $0x18,%%zmm16,%%zmm16 \n" // 0xff000000 "vmovdqu64 %3,%%zmm5 \n" LABELALIGN @@ -173,8 +179,8 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y, "vbroadcasti32x4 0x10(%0),%%zmm1 \n" "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" "vpshufb %%zmm5,%%zmm1,%%zmm1 \n" - "vpord %%zmm7,%%zmm0,%%zmm0 \n" - "vpord %%zmm7,%%zmm1,%%zmm1 \n" + "vpord %%zmm16,%%zmm0,%%zmm0 \n" + "vpord %%zmm16,%%zmm1,%%zmm1 \n" "vmovdqu64 %%zmm0,(%1) \n" "vmovdqu64 %%zmm1,0x40(%1) \n" "lea 0x20(%0),%0 \n" @@ -186,7 +192,7 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y, "+r"(dst_argb), // %1 "+r"(width) // %2 : "m"(kShuffleMaskJ400ToARGB) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm5", "xmm7"); + : "memory", "cc", "xmm0", "xmm1", "xmm5", "xmm16"); } #endif // HAS_J400TOARGBROW_AVX512BW @@ -359,41 +365,55 @@ void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) { : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); } -#ifdef HAS_RAWTOARGBROW_AVX512BW -static const uint32_t kPermdRAWToARGB_AVX512BW[16] = { - 0, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9, 9, 10, 11, 12}; +#if defined(HAS_RAWTOARGBROW_AVX512BW) || \ + defined(HAS_RGB24TOARGBROW_AVX512BW) || \ + defined(HAS_RGBTOYMATRIXROW_AVX512BW) +static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, + 3, 4, 4, 5}; +static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8, + 9, 10, 10, 11}; +static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6, + 7, 8, 8, 9}; +static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, + 5, 6, 6, 7}; +#endif +#ifdef HAS_RAWTOARGBROW_AVX512BW void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, uint8_t* dst_argb, const uint32_t* shuffler, int width) { asm volatile( - "vpternlogd $0xff,%%zmm6,%%zmm6,%%zmm6 \n" // 0xffffffff - "vpslld $0x18,%%zmm6,%%zmm6 \n" // 0xff000000 - "movabs $0xffffffffffff,%%rax \n" // 48 bytes mask - "kmovq %%rax,%%k1 \n" - "vmovdqu32 %3,%%zmm5 \n" - "vbroadcasti32x4 %4,%%zmm4 \n" + "vmovdqu64 %4,%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW + "vmovdqu64 %5,%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW + "vmovdqu64 %6,%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW + "vmovdqu64 %7,%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW + "vbroadcasti64x4 %3,%%zmm4 \n" // shuffler (32 bytes) + "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // 0xffffffff + "vpslld $0x18,%%zmm16,%%zmm16 \n" // 0xff000000 - LABELALIGN // + LABELALIGN "1: \n" - "vmovdqu8 (%0),%%zmm0%{%%k1%}%{z%} \n" - "vmovdqu8 48(%0),%%zmm1%{%%k1%}%{z%} \n" - "vmovdqu8 96(%0),%%zmm2%{%%k1%}%{z%} \n" - "vmovdqu8 144(%0),%%zmm3%{%%k1%}%{z%} \n" + "vmovdqu8 (%0),%%zmm0 \n" + "vmovdqu8 64(%0),%%zmm1 \n" + "vmovdqu8 128(%0),%%zmm2 \n" "lea 192(%0),%0 \n" - "vpermd %%zmm0,%%zmm5,%%zmm0 \n" - "vpermd %%zmm1,%%zmm5,%%zmm1 \n" - "vpermd %%zmm2,%%zmm5,%%zmm2 \n" - "vpermd %%zmm3,%%zmm5,%%zmm3 \n" + "vpermq %%zmm2,%%zmm23,%%zmm3 \n" + "vmovdqa64 %%zmm22,%%zmm18 \n" + "vpermi2q %%zmm2,%%zmm1,%%zmm18 \n" + "vmovdqa64 %%zmm21,%%zmm19 \n" + "vpermi2q %%zmm1,%%zmm0,%%zmm19 \n" + "vpermq %%zmm0,%%zmm20,%%zmm0 \n" + "vmovdqa64 %%zmm19,%%zmm1 \n" + "vmovdqa64 %%zmm18,%%zmm2 \n" "vpshufb %%zmm4,%%zmm0,%%zmm0 \n" "vpshufb %%zmm4,%%zmm1,%%zmm1 \n" "vpshufb %%zmm4,%%zmm2,%%zmm2 \n" "vpshufb %%zmm4,%%zmm3,%%zmm3 \n" - "vpord %%zmm6,%%zmm0,%%zmm0 \n" - "vpord %%zmm6,%%zmm1,%%zmm1 \n" - "vpord %%zmm6,%%zmm2,%%zmm2 \n" - "vpord %%zmm6,%%zmm3,%%zmm3 \n" + "vpord %%zmm16,%%zmm0,%%zmm0 \n" + "vpord %%zmm16,%%zmm1,%%zmm1 \n" + "vpord %%zmm16,%%zmm2,%%zmm2 \n" + "vpord %%zmm16,%%zmm3,%%zmm3 \n" "vmovdqu32 %%zmm0,(%1) \n" "vmovdqu32 %%zmm1,0x40(%1) \n" "vmovdqu32 %%zmm2,0x80(%1) \n" @@ -401,21 +421,25 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, "lea 0x100(%1),%1 \n" "sub $0x40,%2 \n" "jg 1b \n" - "vzeroupper \n" - : "+r"(src_raw), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "m"(kPermdRAWToARGB_AVX512BW), // %3 - "m"(*shuffler) // %4 - : "memory", "cc", "rax", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", - "xmm5", "xmm6"); + "vzeroupper \n" + : "+r"(src_raw), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "m"(*shuffler), // %3 + "m"(kPermqRGB24ToARGB0_AVX512BW), // %4 + "m"(kPermqRGB24ToARGB1_AVX512BW), // %5 + "m"(kPermqRGB24ToARGB2_AVX512BW), // %6 + "m"(kPermqRGB24ToARGB3_AVX512BW) // %7 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", + "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23"); } void RAWToARGBRow_AVX512BW(const uint8_t* src_raw, uint8_t* dst_argb, int width) { RGBToARGBRow_AVX512BW(src_raw, dst_argb, - (const uint32_t*)&kShuffleMaskRAWToARGB, width); + (const uint32_t*)&kShuffleMaskRAWToARGB_AVX512BW[0], + width); } void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, @@ -777,9 +801,9 @@ static const ulvec8 kPermARGBToRGB24_2 = { void ARGBToRGB24Row_AVX512VBMI(const uint8_t* src, uint8_t* dst, int width) { asm volatile( - "vmovdqa %3,%%ymm5 \n" - "vmovdqa %4,%%ymm6 \n" - "vmovdqa %5,%%ymm7 \n" + "vmovdqa32 %3,%%ymm5 \n" + "vmovdqa32 %4,%%ymm16 \n" + "vmovdqa32 %5,%%ymm17 \n" LABELALIGN "1: \n" @@ -789,8 +813,8 @@ void ARGBToRGB24Row_AVX512VBMI(const uint8_t* src, uint8_t* dst, int width) { "vmovdqu 0x60(%0),%%ymm3 \n" "lea 0x80(%0),%0 \n" "vpermt2b %%ymm1,%%ymm5,%%ymm0 \n" - "vpermt2b %%ymm2,%%ymm6,%%ymm1 \n" - "vpermt2b %%ymm3,%%ymm7,%%ymm2 \n" + "vpermt2b %%ymm2,%%ymm16,%%ymm1 \n" + "vpermt2b %%ymm3,%%ymm17,%%ymm2 \n" "vmovdqu %%ymm0,(%1) \n" "vmovdqu %%ymm1,0x20(%1) \n" "vmovdqu %%ymm2,0x40(%1) \n" @@ -804,7 +828,8 @@ void ARGBToRGB24Row_AVX512VBMI(const uint8_t* src, uint8_t* dst, int width) { : "m"(kPermARGBToRGB24_0), // %3 "m"(kPermARGBToRGB24_1), // %4 "m"(kPermARGBToRGB24_2) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm16", + "xmm17"); } #endif @@ -1419,11 +1444,12 @@ void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb, } #endif -#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \ +#if defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \ defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \ - defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) + defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \ + defined(HAS_RGBTOYMATRIXROW_AVX512BW) static const uint32_t kPermdARGBToY_AVX512BW[16] = {0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15}; + 2, 6, 10, 14, 3, 7, 11, 15}; #endif #if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) @@ -1439,15 +1465,16 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, asm volatile( "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" "vpsllw $15,%%zmm16,%%zmm5 \n" - "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" - "vpsrlw $15,%%zmm16,%%zmm16 \n" // zmm16 = 1 - "vbroadcasti64x4 0(%3),%%zmm4 \n" - "vbroadcasti64x4 0x60(%3),%%zmm7 \n" - "vpmaddubsw %%zmm5,%%zmm4,%%zmm6 \n" - "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n" - "vpackssdw %%zmm6,%%zmm6,%%zmm6 \n" - "vpsubw %%zmm6,%%zmm7,%%zmm7 \n" - "vmovups %4,%%zmm6 \n" LABELALIGN + "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80 + "vpabsw %%zmm16,%%zmm16 \n" // zmm16 = 1 + "vbroadcasti32x4 0(%3),%%zmm4 \n" + "vbroadcasti32x4 0x60(%3),%%zmm17 \n" + "vpmaddubsw %%zmm5,%%zmm4,%%zmm18 \n" + "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" + "vpackssdw %%zmm18,%%zmm18,%%zmm18 \n" + "vpsubw %%zmm18,%%zmm17,%%zmm17 \n" + "vmovups %4,%%zmm18 \n" + LABELALIGN "1: \n" "vmovups (%0),%%zmm0 \n" "vmovups 0x40(%0),%%zmm1 \n" @@ -1468,12 +1495,12 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" "vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n" "vpackssdw %%zmm3,%%zmm2,%%zmm2 \n" - "vpaddw %%zmm7,%%zmm0,%%zmm0 \n" - "vpaddw %%zmm7,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm17,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm17,%%zmm2,%%zmm2 \n" "vpsrlw $0x8,%%zmm0,%%zmm0 \n" "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" - "vpermd %%zmm0,%%zmm6,%%zmm0 \n" + "vpermd %%zmm0,%%zmm18,%%zmm0 \n" "vmovups %%zmm0,(%1) \n" "lea 0x40(%1),%1 \n" "sub $0x40,%2 \n" @@ -1484,8 +1511,89 @@ void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, "+r"(width) // %2 : "r"(c), // %3 "m"(kPermdARGBToY_AVX512BW) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7", "xmm16"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm16", "xmm17", "xmm18"); +} +#endif + +#ifdef HAS_RGBTOYMATRIXROW_AVX512BW +void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + asm volatile( + "vmovdqu64 %5,%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW + "vmovdqu64 %6,%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW + "vmovdqu64 %7,%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW + "vmovdqu64 %8,%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW + "vbroadcasti64x4 %9,%%zmm24 \n" // kShuffleMaskRGB24ToARGB + "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" + "vpsllw $15,%%zmm16,%%zmm5 \n" + "vpacksswb %%zmm5,%%zmm5,%%zmm5 \n" // zmm5 = 0x80 + "vpabsw %%zmm16,%%zmm16 \n" // zmm16 = 1 + "vbroadcasti32x4 0(%3),%%zmm4 \n" // zmm4 = kRGBToY + "vbroadcasti32x4 0x60(%3),%%zmm17 \n" // zmm17 = kAddY + "vpmaddubsw %%zmm5,%%zmm4,%%zmm18 \n" + "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" + "vpackssdw %%zmm18,%%zmm18,%%zmm18 \n" + "vpsubw %%zmm18,%%zmm17,%%zmm17 \n" // zmm17 = adjusted bias + "vmovups %4,%%zmm25 \n" // zmm25 = kPermdARGBToY_AVX512BW + LABELALIGN + "1: \n" + "vmovdqu8 (%0),%%zmm0 \n" + "vmovdqu8 64(%0),%%zmm1 \n" + "vmovdqu8 128(%0),%%zmm2 \n" + "lea 192(%0),%0 \n" + "vpermq %%zmm2,%%zmm23,%%zmm3 \n" + "vmovdqa64 %%zmm22,%%zmm18 \n" + "vpermi2q %%zmm2,%%zmm1,%%zmm18 \n" + "vmovdqa64 %%zmm21,%%zmm19 \n" + "vpermi2q %%zmm1,%%zmm0,%%zmm19 \n" + "vpermq %%zmm0,%%zmm20,%%zmm0 \n" + "vmovdqa64 %%zmm19,%%zmm1 \n" + "vmovdqa64 %%zmm18,%%zmm2 \n" + "vpshufb %%zmm24,%%zmm0,%%zmm0 \n" + "vpshufb %%zmm24,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm24,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm24,%%zmm3,%%zmm3 \n" + "vpsubb %%zmm5,%%zmm0,%%zmm0 \n" + "vpsubb %%zmm5,%%zmm1,%%zmm1 \n" + "vpsubb %%zmm5,%%zmm2,%%zmm2 \n" + "vpsubb %%zmm5,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm0,%%zmm4,%%zmm0 \n" + "vpmaddubsw %%zmm1,%%zmm4,%%zmm1 \n" + "vpmaddubsw %%zmm2,%%zmm4,%%zmm2 \n" + "vpmaddubsw %%zmm3,%%zmm4,%%zmm3 \n" + "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" + "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" + "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" + "vpmaddwd %%zmm16,%%zmm3,%%zmm3 \n" + "vpackssdw %%zmm3,%%zmm2,%%zmm2 \n" + "vpaddw %%zmm17,%%zmm0,%%zmm0 \n" + "vpaddw %%zmm17,%%zmm2,%%zmm2 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm2,%%zmm2 \n" + "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" + "vpermd %%zmm0,%%zmm25,%%zmm0 \n" + "vmovups %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_rgb), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(c), // %3 + "m"(kPermdARGBToY_AVX512BW), // %4 + "m"(kPermqRGB24ToARGB0_AVX512BW), // %5 + "m"(kPermqRGB24ToARGB1_AVX512BW), // %6 + "m"(kPermqRGB24ToARGB2_AVX512BW), // %7 + "m"(kPermqRGB24ToARGB3_AVX512BW), // %8 + "m"(kShuffleMaskRGB24ToARGB) // %9 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", + "xmm23", "xmm24", "xmm25"); } #endif @@ -1639,11 +1747,11 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, int width, const struct ArgbConstants* c) { asm volatile( - "vbroadcasti64x4 0x20(%4),%%zmm3 \n" // kRGBToU - "vbroadcasti64x4 0x40(%4),%%zmm4 \n" // kRGBToV + "vbroadcasti32x4 0x20(%4),%%zmm3 \n" // kRGBToU + "vbroadcasti32x4 0x40(%4),%%zmm4 \n" // kRGBToV "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // -1 "vpsllw $15,%%zmm16,%%zmm5 \n" // 0x8000 - "vmovups %5,%%zmm7 \n" + "vmovups %5,%%zmm17 \n" "sub %1,%2 \n" LABELALIGN @@ -1651,45 +1759,45 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, "vmovups (%0),%%zmm0 \n" "vmovups 0x40(%0),%%zmm1 \n" "vmovups 0x80(%0),%%zmm2 \n" - "vmovups 0xc0(%0),%%zmm6 \n" + "vmovups 0xc0(%0),%%zmm18 \n" "vpmaddubsw %%zmm3,%%zmm0,%%zmm0 \n" "vpmaddubsw %%zmm3,%%zmm1,%%zmm1 \n" "vpmaddubsw %%zmm3,%%zmm2,%%zmm2 \n" - "vpmaddubsw %%zmm3,%%zmm6,%%zmm6 \n" + "vpmaddubsw %%zmm3,%%zmm18,%%zmm18 \n" "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" - "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n" + "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates - "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n" + "vpackssdw %%zmm18,%%zmm2,%%zmm2 \n" "vpsubw %%zmm5,%%zmm0,%%zmm0 \n" "vpsubw %%zmm5,%%zmm2,%%zmm2 \n" "vpsrlw $0x8,%%zmm0,%%zmm0 \n" "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates - "vpermd %%zmm0,%%zmm7,%%zmm0 \n" // unmutate. + "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate. "vmovups %%zmm0,(%1) \n" "vmovups (%0),%%zmm0 \n" "vmovups 0x40(%0),%%zmm1 \n" "vmovups 0x80(%0),%%zmm2 \n" - "vmovups 0xc0(%0),%%zmm6 \n" + "vmovups 0xc0(%0),%%zmm18 \n" "vpmaddubsw %%zmm4,%%zmm0,%%zmm0 \n" "vpmaddubsw %%zmm4,%%zmm1,%%zmm1 \n" "vpmaddubsw %%zmm4,%%zmm2,%%zmm2 \n" - "vpmaddubsw %%zmm4,%%zmm6,%%zmm6 \n" + "vpmaddubsw %%zmm4,%%zmm18,%%zmm18 \n" "vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n" "vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n" "vpmaddwd %%zmm16,%%zmm2,%%zmm2 \n" - "vpmaddwd %%zmm16,%%zmm6,%%zmm6 \n" + "vpmaddwd %%zmm16,%%zmm18,%%zmm18 \n" "vpackssdw %%zmm1,%%zmm0,%%zmm0 \n" // mutates - "vpackssdw %%zmm6,%%zmm2,%%zmm2 \n" + "vpackssdw %%zmm18,%%zmm2,%%zmm2 \n" "vpsubw %%zmm5,%%zmm0,%%zmm0 \n" "vpsubw %%zmm5,%%zmm2,%%zmm2 \n" "vpsrlw $0x8,%%zmm0,%%zmm0 \n" "vpsrlw $0x8,%%zmm2,%%zmm2 \n" "vpackuswb %%zmm2,%%zmm0,%%zmm0 \n" // mutates - "vpermd %%zmm0,%%zmm7,%%zmm0 \n" // unmutate. + "vpermd %%zmm0,%%zmm17,%%zmm0 \n" // unmutate. "vmovups %%zmm0,(%1,%2,1) \n" "lea 0x100(%0),%0 \n" "lea 0x40(%1),%1 \n" @@ -1706,8 +1814,8 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, #endif : "r"(c), // %4 "m"(kPermdARGBToY_AVX512BW) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7", "xmm16"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm16", "xmm17", "xmm18"); } #endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW @@ -1885,12 +1993,12 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, int width, const struct ArgbConstants* c) { asm volatile( - "vbroadcasti64x4 0x20(%5),%%zmm4 \n" // RGBToU - "vbroadcasti64x4 0x40(%5),%%zmm5 \n" // RGBToV + "vbroadcasti32x4 0x20(%5),%%zmm4 \n" // RGBToU + "vbroadcasti32x4 0x40(%5),%%zmm5 \n" // RGBToV "vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" - "vpabsb %%zmm16,%%zmm6 \n" // 0x0101 + "vpabsb %%zmm16,%%zmm20 \n" // 0x0101 "vpsllw $15,%%zmm16,%%zmm17 \n" // 0x8000 - "vbroadcasti64x4 %6,%%zmm7 \n" // kShuffleAARRGGBB + "vbroadcasti32x4 %6,%%zmm21 \n" // kShuffleAARRGGBB "vmovups %7,%%zmm18 \n" // kPermdARGBToY_AVX512BW "vmovups %8,%%zmm19 \n" // kPermdARGBToUV_AVX512BW "sub %1,%2 \n" @@ -1901,14 +2009,14 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, "vmovups 0x40(%0),%%zmm1 \n" "vmovups 0x00(%0,%4,1),%%zmm2 \n" "vmovups 0x40(%0,%4,1),%%zmm3 \n" - "vpshufb %%zmm7,%%zmm0,%%zmm0 \n" // aarrggbb - "vpshufb %%zmm7,%%zmm1,%%zmm1 \n" - "vpshufb %%zmm7,%%zmm2,%%zmm2 \n" - "vpshufb %%zmm7,%%zmm3,%%zmm3 \n" - "vpmaddubsw %%zmm6,%%zmm0,%%zmm0 \n" // 32x2 -> 16x2 - "vpmaddubsw %%zmm6,%%zmm1,%%zmm1 \n" - "vpmaddubsw %%zmm6,%%zmm2,%%zmm2 \n" - "vpmaddubsw %%zmm6,%%zmm3,%%zmm3 \n" + "vpshufb %%zmm21,%%zmm0,%%zmm0 \n" // aarrggbb + "vpshufb %%zmm21,%%zmm1,%%zmm1 \n" + "vpshufb %%zmm21,%%zmm2,%%zmm2 \n" + "vpshufb %%zmm21,%%zmm3,%%zmm3 \n" + "vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n" // 32x2 -> 16x2 + "vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n" + "vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n" + "vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n" "vpaddw %%zmm0,%%zmm2,%%zmm0 \n" // 16x2 -> 16x1 "vpaddw %%zmm1,%%zmm3,%%zmm1 \n" "vpxorq %%zmm2,%%zmm2,%%zmm2 \n" // 0 for vpavgw @@ -1952,8 +2060,8 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, "m"(kShuffleAARRGGBB), // %6 "m"(kPermdARGBToY_AVX512BW), // %7 "m"(kPermdARGBToUV_AVX512BW) // %8 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7", "xmm16", "xmm17", "xmm18", "xmm19"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21"); } #endif // HAS_ARGBTOUVMATRIXROW_AVX512BW @@ -9446,9 +9554,9 @@ void NV21ToYUV24Row_AVX512(const uint8_t* src_y, int width) { asm volatile( "sub %0,%1 \n" - "vmovdqa (%4),%%ymm4 \n" // 3 shuffler constants - "vmovdqa 32(%4),%%ymm5 \n" - "vmovdqa 64(%4),%%ymm6 \n" LABELALIGN + "vmovdqa32 (%4),%%ymm4 \n" // 3 shuffler constants + "vmovdqa32 32(%4),%%ymm5 \n" + "vmovdqa32 64(%4),%%ymm16 \n" LABELALIGN "1: \n" "vmovdqu (%0),%%ymm2 \n" // load 32 Y values "vmovdqu (%0,%1),%%ymm3 \n" // load 16 VU values @@ -9457,7 +9565,7 @@ void NV21ToYUV24Row_AVX512(const uint8_t* src_y, "vmovdqa %%ymm2, %%ymm1 \n" "vpermt2b %%ymm3,%%ymm4,%%ymm0 \n" "vpermt2b %%ymm3,%%ymm5,%%ymm1 \n" - "vpermt2b %%ymm3,%%ymm6,%%ymm2 \n" + "vpermt2b %%ymm3,%%ymm16,%%ymm2 \n" "vmovdqu %%ymm0,(%2) \n" "vmovdqu %%ymm1,32(%2) \n" "vmovdqu %%ymm2,64(%2) \n" @@ -9470,7 +9578,7 @@ void NV21ToYUV24Row_AVX512(const uint8_t* src_y, "+r"(dst_yuv24), // %2 "+r"(width) // %3 : "r"(&kYUV24Perm[0]) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16"); } #endif // HAS_NV21ToYUV24ROW_AVX512 diff --git a/source/row_neon.cc b/source/row_neon.cc index dcdf68fef..50ba7dbde 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -1897,6 +1897,58 @@ void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb, "q10", "q11", "q12"); } +// 8x1 pixels. +void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + asm volatile( + "vld1.8 {d24}, [%4] \n" // load kRGBToU + "vld1.8 {d25}, [%5] \n" // load kRGBToV + "vld1.16 {d26[0]}, [%6] \n" // load kAddUV[0] + "vmovl.s8 q10, d24 \n" // U coeffs (8 shorts) + "vmovl.s8 q11, d25 \n" // V coeffs (8 shorts) + "vdup.16 q6, d26[0] \n" // bias + "1: \n" + "vld3.8 {d0, d1, d2}, [%0]! \n" // load 8 RGB pixels. + "subs %3, %3, #8 \n" // 8 processed per loop. + + "vmovl.u8 q4, d0 \n" // B + "vmovl.u8 q5, d1 \n" // G + "vmovl.u8 q7, d2 \n" // R + + "vdup.16 q12, d20[0] \n" + "vmul.s16 q2, q4, q12 \n" // U = B * U0 + "vdup.16 q12, d20[1] \n" + "vmla.s16 q2, q5, q12 \n" // U += G * U1 + "vdup.16 q12, d20[2] \n" + "vmla.s16 q2, q7, q12 \n" // U += R * U2 + + "vdup.16 q12, d22[0] \n" + "vmul.s16 q3, q4, q12 \n" // V = B * V0 + "vdup.16 q12, d22[1] \n" + "vmla.s16 q3, q5, q12 \n" // V += G * V1 + "vdup.16 q12, d22[2] \n" + "vmla.s16 q3, q7, q12 \n" // V += R * V2 + + "vsubhn.s16 d0, q6, q2 \n" // 128.0 - U + "vsubhn.s16 d1, q6, q3 \n" // 128.0 - V + + "vst1.8 {d0}, [%1]! \n" // store 8 pixels U. + "vst1.8 {d1}, [%2]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_rgb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width) // %3 + : "r"(&c->kRGBToU), // %4 + "r"(&c->kRGBToV), // %5 + "r"(&c->kAddUV) // %6 + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q10", "q11", "q12"); +} + // clang-format off // 16x2 pixels -> 8x1. width is number of argb pixels. e.g. 16. #define RGBTOUV(QB, QG, QR) \ @@ -1982,6 +2034,68 @@ void ARGBToUVMatrixRow_NEON(const uint8_t* src_argb, "q9", "q11", "q12", "q14", "q15"); } +void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const uint8_t* src_rgb_1 = src_rgb + src_stride_rgb; + asm volatile( + "vld1.8 {d24}, [%5] \n" // load kRGBToU + "vld1.8 {d25}, [%6] \n" // load kRGBToV + "vmovl.s8 q14, d24 \n" // U coeffs in d28 + "vmovl.s8 q15, d25 \n" // V coeffs in d30 + "vmov.u16 q11, #0x8000 \n" // 128.0 bias + + "1: \n" + "vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB pixels. + "vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RGB pixels. + "subs %4, %4, #16 \n" // 16 processed per loop. + "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts. + "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts. + "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts. + "vld3.8 {d6, d8, d10}, [%1]! \n" // load 8 more pixels. + "vld3.8 {d7, d9, d11}, [%1]! \n" + "vpadal.u8 q0, q3 \n" // B + "vpadal.u8 q1, q4 \n" // G + "vpadal.u8 q2, q5 \n" // R + + "vrshr.u16 q0, q0, #2 \n" // average of 4 + "vrshr.u16 q1, q1, #2 \n" + "vrshr.u16 q2, q2, #2 \n" + + "vdup.16 q12, d28[0] \n" + "vmul.s16 q8, q0, q12 \n" // U = B * U0 + "vdup.16 q12, d28[1] \n" + "vmla.s16 q8, q1, q12 \n" // U += G * U1 + "vdup.16 q12, d28[2] \n" + "vmla.s16 q8, q2, q12 \n" // U += R * U2 + + "vdup.16 q12, d30[0] \n" + "vmul.s16 q9, q0, q12 \n" // V = B * V0 + "vdup.16 q12, d30[1] \n" + "vmla.s16 q9, q1, q12 \n" // V += G * V1 + "vdup.16 q12, d30[2] \n" + "vmla.s16 q9, q2, q12 \n" // V += R * V2 + + "vsubhn.s16 d0, q11, q8 \n" // 128.0 - U + "vsubhn.s16 d1, q11, q9 \n" // 128.0 - V + + "vst1.8 {d0}, [%2]! \n" // store 8 pixels U. + "vst1.8 {d1}, [%3]! \n" // store 8 pixels V. + "bgt 1b \n" + : "+r"(src_rgb), // %0 + "+r"(src_rgb_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(width) // %4 + : "r"(&c->kRGBToU), // %5 + "r"(&c->kRGBToV) // %6 + : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q8", + "q9", "q11", "q12", "q14", "q15"); +} + void RGB24ToUVRow_NEON(const uint8_t* src_rgb24, int src_stride_rgb24, uint8_t* dst_u, diff --git a/source/row_neon64.cc b/source/row_neon64.cc index ced4193ba..782a6653f 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -2793,6 +2793,61 @@ void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb, "v27", "v28"); } +// 8x1 pixels. +void RGBToUV444MatrixRow_NEON(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + asm volatile( + "ldr q16, [%[c], #16] \n" // kRGBToU + "ldr q17, [%[c], #32] \n" // kRGBToV + "ldr s0, [%[c], #64] \n" // kAddUV + "sxtl v16.8h, v16.8b \n" // sign extend U coeffs + // to 16-bit + "sxtl v17.8h, v17.8b \n" // sign extend V coeffs + // to 16-bit + "dup v20.8h, v16.h[0] \n" // U0 + "dup v21.8h, v16.h[1] \n" // U1 + "dup v22.8h, v16.h[2] \n" // U2 + "dup v24.8h, v17.h[0] \n" // V0 + "dup v26.8h, v17.h[1] \n" // V1 + "dup v27.8h, v17.h[2] \n" // V2 + "dup v25.8h, v0.h[0] \n" // kAddUV + "1: \n" + "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // load 8 RGB + "subs %w3, %w3, #8 \n" // 8 processed per loop. + + "uxtl v4.8h, v0.8b \n" // B + "uxtl v5.8h, v1.8b \n" // G + "uxtl v6.8h, v2.8b \n" // R + + // U = B*U0 + G*U1 + R*U2 + "mul v18.8h, v4.8h, v20.8h \n" + "mla v18.8h, v5.8h, v21.8h \n" + "mla v18.8h, v6.8h, v22.8h \n" + + // V = B*V0 + G*V1 + R*V2 + "mul v19.8h, v4.8h, v24.8h \n" + "mla v19.8h, v5.8h, v26.8h \n" + "mla v19.8h, v6.8h, v27.8h \n" + + "subhn v0.8b, v25.8h, v18.8h \n" + "subhn v1.8b, v25.8h, v19.8h \n" + + "st1 {v0.8b}, [%1], #8 \n" + "st1 {v1.8b}, [%2], #8 \n" + "b.gt 1b \n" + : "+r"(src_rgb), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width) // %3 + : [c] "r"(c) // %4 + : "cc", "memory", "v0", "v1", "v2", "v4", "v5", "v6", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v24", "v25", "v26", + "v27"); +} + void ARGBToUV444MatrixRow_NEON_I8MM(const uint8_t* src_argb, uint8_t* dst_u, uint8_t* dst_v, @@ -2940,6 +2995,71 @@ void ARGBToUVMatrixRow_NEON(const uint8_t* src_argb, "v28"); } +void RGBToUVMatrixRow_NEON(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const uint8_t* src_rgb_1 = src_rgb + src_stride_rgb; + asm volatile( + "ldr q16, [%[c], #16] \n" // kRGBToU + "ldr q17, [%[c], #32] \n" // kRGBToV + "sxtl v16.8h, v16.8b \n" // sign extend U coeffs + "sxtl v17.8h, v17.8b \n" // sign extend V coeffs + "dup v20.8h, v16.h[0] \n" // U0 + "dup v21.8h, v16.h[1] \n" // U1 + "dup v22.8h, v16.h[2] \n" // U2 + "dup v24.8h, v17.h[0] \n" // V0 + "dup v26.8h, v17.h[1] \n" // V1 + "dup v27.8h, v17.h[2] \n" // V2 + "movi v25.8h, #0x80, lsl #8 \n" // 128.0 in 16-bit (0x8000) + + "1: \n" + "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels. + "subs %w4, %w4, #16 \n" // 16 processed per loop. + "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts. + "prfm pldl1keep, [%0, 448] \n" + "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts. + "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts. + + "ld3 {v4.16b,v5.16b,v6.16b}, [%1], #48 \n" // load 16 more. + "uadalp v0.8h, v4.16b \n" // B 16 bytes -> 8 shorts. + "prfm pldl1keep, [%1, 448] \n" + "uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts. + "uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts. + + "urshr v0.8h, v0.8h, #2 \n" // average of 4 + "urshr v1.8h, v1.8h, #2 \n" + "urshr v2.8h, v2.8h, #2 \n" + + // U = B*U0 + G*U1 + R*U2 + "mul v3.8h, v0.8h, v20.8h \n" + "mla v3.8h, v1.8h, v21.8h \n" + "mla v3.8h, v2.8h, v22.8h \n" + + // V = B*V0 + G*V1 + R*V2 + "mul v4.8h, v0.8h, v24.8h \n" + "mla v4.8h, v1.8h, v26.8h \n" + "mla v4.8h, v2.8h, v27.8h \n" + + // U = (128.0 - U) >> 8, V = (128.0 - V) >> 8 + "subhn v0.8b, v25.8h, v3.8h \n" + "subhn v1.8b, v25.8h, v4.8h \n" + + "st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U. + "st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V. + "b.gt 1b \n" + : "+r"(src_rgb), // %0 + "+r"(src_rgb_1), // %1 + "+r"(dst_u), // %2 + "+r"(dst_v), // %3 + "+r"(width) // %4 + : [c] "r"(c) // %5 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v20", "v21", "v22", "v24", "v25", "v26", "v27"); +} + void RGB24ToUVRow_NEON(const uint8_t* src_rgb24, int src_stride_rgb24, uint8_t* dst_u, @@ -3401,6 +3521,50 @@ void ARGBToYMatrixRow_NEON_DotProd(const uint8_t* src_argb, "v17", "v18", "v19"); } +void RGBToYMatrixRow_NEON_DotProd(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + asm volatile( + "ldr s16, [%3] \n" // load 4 coeffs + "ldr s17, [%3, #48] \n" // load kAddY[0] + "dup v18.4s, v16.s[0] \n" + "dup v19.8h, v17.h[0] \n" + "movi v7.16b, #0 \n" // zero alpha + "1: \n" + "ld3 {v4.16b, v5.16b, v6.16b}, [%0], #48 \n" // load 16 pixels + "subs %w2, %w2, #16 \n" // 16 processed per loop. + "zip1 v24.16b, v4.16b, v5.16b \n" // B0 G0 B1 G1 ... + "zip1 v25.16b, v6.16b, v7.16b \n" // R0 0 R1 0 ... + "zip2 v26.16b, v4.16b, v5.16b \n" // B8 G8 B9 G9 ... + "zip2 v27.16b, v6.16b, v7.16b \n" // R8 0 R9 0 ... + "zip1 v20.8h, v24.8h, v25.8h \n" // pixels 0..3 + "zip2 v21.8h, v24.8h, v25.8h \n" // pixels 4..7 + "zip1 v22.8h, v26.8h, v27.8h \n" // pixels 8..11 + "zip2 v23.8h, v26.8h, v27.8h \n" // pixels 12..15 + "movi v0.16b, #0 \n" + "movi v1.16b, #0 \n" + "movi v2.16b, #0 \n" + "movi v3.16b, #0 \n" + "udot v0.4s, v20.16b, v18.16b \n" + "udot v1.4s, v21.16b, v18.16b \n" + "udot v2.4s, v22.16b, v18.16b \n" + "udot v3.4s, v23.16b, v18.16b \n" + "uzp1 v0.8h, v0.8h, v1.8h \n" + "uzp1 v1.8h, v2.8h, v3.8h \n" + "addhn v0.8b, v0.8h, v19.8h \n" + "addhn v1.8b, v1.8h, v19.8h \n" + "st1 {v0.8b, v1.8b}, [%1], #16 \n" // store 16 pixels Y. + "b.gt 1b \n" + : "+r"(src_rgb), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(c) // %3 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", + "v27"); +} + // RGBA expects first value to be A and ignored, then 3 values to contain RGB. void RGBToYMatrixRow_NEON(const uint8_t* src_rgb, uint8_t* dst_y, diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 9e1ed7687..7ef2c64ed 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -1257,24 +1257,27 @@ void ARGBToYMatrixRow_RVV(const uint8_t* src_argb, const struct ArgbConstants* c) { assert(width != 0); size_t w = (size_t)width; - vuint8m2_t v_by, v_gy, v_ry; // vectors are to store RGBToY constant - vuint16m4_t v_addy; // vector is to store kAddY + vuint8m2_t v_y0, v_y1, v_y2, v_y3; + vuint16m4_t v_addy; size_t vl = __riscv_vsetvl_e8m2(w); - v_by = __riscv_vmv_v_x_u8m2(c->kRGBToY[0], vl); - v_gy = __riscv_vmv_v_x_u8m2(c->kRGBToY[1], vl); - v_ry = __riscv_vmv_v_x_u8m2(c->kRGBToY[2], vl); + v_y0 = __riscv_vmv_v_x_u8m2(c->kRGBToY[0], vl); + v_y1 = __riscv_vmv_v_x_u8m2(c->kRGBToY[1], vl); + v_y2 = __riscv_vmv_v_x_u8m2(c->kRGBToY[2], vl); + v_y3 = __riscv_vmv_v_x_u8m2(c->kRGBToY[3], vl); v_addy = __riscv_vmv_v_x_u16m4(c->kAddY[0], vl); do { vuint8m2_t v_y; vuint16m4_t v_y_u16; size_t vl = __riscv_vsetvl_e8m2(w); vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); - vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); - vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); - vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); - v_y_u16 = __riscv_vwmulu_vv_u16m4(v_r, v_ry, vl); - v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_gy, v_g, vl); - v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_by, v_b, vl); + vuint8m2_t v_b0 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); + vuint8m2_t v_b1 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); + vuint8m2_t v_b2 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); + vuint8m2_t v_b3 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3); + v_y_u16 = __riscv_vwmulu_vv_u16m4(v_b0, v_y0, vl); + v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_y1, v_b1, vl); + v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_y2, v_b2, vl); + v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_y3, v_b3, vl); v_y_u16 = __riscv_vadd_vv_u16m4(v_y_u16, v_addy, vl); v_y = __riscv_vnsrl_wx_u8m2(v_y_u16, 8, vl); __riscv_vse8_v_u8m2(dst_y, v_y, vl); @@ -1320,6 +1323,322 @@ void RGBToYMatrixRow_RVV(const uint8_t* src_rgb, } #endif +#ifdef HAS_ARGBTOUV444MATRIXROW_RVV +void ARGBToUV444MatrixRow_RVV(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + assert(width != 0); + size_t w = (size_t)width; + vint8m2_t v_u0, v_u1, v_u2, v_u3; + vint8m2_t v_v0, v_v1, v_v2, v_v3; + vint16m4_t v_adduv; + size_t vl = __riscv_vsetvl_e8m2(w); + v_u0 = __riscv_vmv_v_x_i8m2(c->kRGBToU[0], vl); + v_u1 = __riscv_vmv_v_x_i8m2(c->kRGBToU[1], vl); + v_u2 = __riscv_vmv_v_x_i8m2(c->kRGBToU[2], vl); + v_u3 = __riscv_vmv_v_x_i8m2(c->kRGBToU[3], vl); + v_v0 = __riscv_vmv_v_x_i8m2(c->kRGBToV[0], vl); + v_v1 = __riscv_vmv_v_x_i8m2(c->kRGBToV[1], vl); + v_v2 = __riscv_vmv_v_x_i8m2(c->kRGBToV[2], vl); + v_v3 = __riscv_vmv_v_x_i8m2(c->kRGBToV[3], vl); + v_adduv = __riscv_vmv_v_x_i16m4((int16_t)c->kAddUV[0], vl); + do { + size_t vl = __riscv_vsetvl_e8m2(w); + vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl); + vuint8m2_t v_b0 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0); + vuint8m2_t v_b1 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1); + vuint8m2_t v_b2 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2); + vuint8m2_t v_b3 = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3); + + vint16m4_t v_u_i16 = __riscv_vwmulsu_vv_i16m4(v_u0, v_b0, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_u1, v_b1, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_u2, v_b2, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_u3, v_b3, vl); + v_u_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_u_i16, vl); + vuint8m2_t v_u = __riscv_vnsrl_wx_u8m2( + __riscv_vreinterpret_v_i16m4_u16m4(v_u_i16), 8, vl); + __riscv_vse8_v_u8m2(dst_u, v_u, vl); + + vint16m4_t v_v_i16 = __riscv_vwmulsu_vv_i16m4(v_v0, v_b0, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_v1, v_b1, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_v2, v_b2, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_v3, v_b3, vl); + v_v_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_v_i16, vl); + vuint8m2_t v_v = __riscv_vnsrl_wx_u8m2( + __riscv_vreinterpret_v_i16m4_u16m4(v_v_i16), 8, vl); + __riscv_vse8_v_u8m2(dst_v, v_v, vl); + + w -= vl; + src_argb += 4 * vl; + dst_u += vl; + dst_v += vl; + } while (w > 0); +} +#endif + +#ifdef HAS_ARGBTOUVMATRIXROW_RVV +void ARGBToUVMatrixRow_RVV(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + assert(width != 0); + const uint8_t* src_argb_0 = src_argb; + const uint8_t* src_argb_1 = src_argb + src_stride_argb; + int w = width >> 1; + if (w > 0) { + size_t w_len = (size_t)w; + vint8m1_t v_u0, v_u1, v_u2, v_u3; + vint8m1_t v_v0, v_v1, v_v2, v_v3; + vint16m2_t v_adduv; + size_t vl = __riscv_vsetvl_e8m1(w_len); + v_u0 = __riscv_vmv_v_x_i8m1(c->kRGBToU[0], vl); + v_u1 = __riscv_vmv_v_x_i8m1(c->kRGBToU[1], vl); + v_u2 = __riscv_vmv_v_x_i8m1(c->kRGBToU[2], vl); + v_u3 = __riscv_vmv_v_x_i8m1(c->kRGBToU[3], vl); + v_v0 = __riscv_vmv_v_x_i8m1(c->kRGBToV[0], vl); + v_v1 = __riscv_vmv_v_x_i8m1(c->kRGBToV[1], vl); + v_v2 = __riscv_vmv_v_x_i8m1(c->kRGBToV[2], vl); + v_v3 = __riscv_vmv_v_x_i8m1(c->kRGBToV[3], vl); + v_adduv = __riscv_vmv_v_x_i16m2((int16_t)c->kAddUV[0], vl); + do { + size_t vl = __riscv_vsetvl_e8m1(w_len); + vuint8m1x8_t v_src0 = __riscv_vlseg8e8_v_u8m1x8(src_argb_0, vl); + vuint8m1x8_t v_src1 = __riscv_vlseg8e8_v_u8m1x8(src_argb_1, vl); + + vuint8m1_t b0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 0); + vuint8m1_t g0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 1); + vuint8m1_t r0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 2); + vuint8m1_t a0 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 3); + vuint8m1_t b1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 4); + vuint8m1_t g1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 5); + vuint8m1_t r1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 6); + vuint8m1_t a1 = __riscv_vget_v_u8m1x8_u8m1(v_src0, 7); + + vuint8m1_t b0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 0); + vuint8m1_t g0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 1); + vuint8m1_t r0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 2); + vuint8m1_t a0_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 3); + vuint8m1_t b1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 4); + vuint8m1_t g1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 5); + vuint8m1_t r1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 6); + vuint8m1_t a1_1 = __riscv_vget_v_u8m1x8_u8m1(v_src1, 7); + + vuint16m2_t sum_b = __riscv_vwaddu_vv_u16m2(b0, b1, vl); + sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b0_1, vl); + sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b1_1, vl); + sum_b = __riscv_vadd_vx_u16m2(sum_b, 2, vl); + vuint8m1_t b = __riscv_vnsrl_wx_u8m1(sum_b, 2, vl); + + vuint16m2_t sum_g = __riscv_vwaddu_vv_u16m2(g0, g1, vl); + sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g0_1, vl); + sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g1_1, vl); + sum_g = __riscv_vadd_vx_u16m2(sum_g, 2, vl); + vuint8m1_t g = __riscv_vnsrl_wx_u8m1(sum_g, 2, vl); + + vuint16m2_t sum_r = __riscv_vwaddu_vv_u16m2(r0, r1, vl); + sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r0_1, vl); + sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r1_1, vl); + sum_r = __riscv_vadd_vx_u16m2(sum_r, 2, vl); + vuint8m1_t r = __riscv_vnsrl_wx_u8m1(sum_r, 2, vl); + + vuint16m2_t sum_a = __riscv_vwaddu_vv_u16m2(a0, a1, vl); + sum_a = __riscv_vwaddu_wv_u16m2(sum_a, a0_1, vl); + sum_a = __riscv_vwaddu_wv_u16m2(sum_a, a1_1, vl); + sum_a = __riscv_vadd_vx_u16m2(sum_a, 2, vl); + vuint8m1_t a = __riscv_vnsrl_wx_u8m1(sum_a, 2, vl); + + vint16m2_t v_u_i16 = __riscv_vwmulsu_vv_i16m2(v_u0, b, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_u1, g, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_u2, r, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_u3, a, vl); + v_u_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_u_i16, vl); + vuint8m1_t v_u = __riscv_vnsrl_wx_u8m1( + __riscv_vreinterpret_v_i16m2_u16m2(v_u_i16), 8, vl); + __riscv_vse8_v_u8m1(dst_u, v_u, vl); + + vint16m2_t v_v_i16 = __riscv_vwmulsu_vv_i16m2(v_v0, b, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_v1, g, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_v2, r, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_v3, a, vl); + v_v_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_v_i16, vl); + vuint8m1_t v_v = __riscv_vnsrl_wx_u8m1( + __riscv_vreinterpret_v_i16m2_u16m2(v_v_i16), 8, vl); + __riscv_vse8_v_u8m1(dst_v, v_v, vl); + + w_len -= vl; + src_argb_0 += 8 * vl; + src_argb_1 += 8 * vl; + dst_u += vl; + dst_v += vl; + } while (w_len > 0); + } + if (width & 1) { + uint8_t b = (src_argb_0[0] + src_argb_1[0] + 1) >> 1; + uint8_t g = (src_argb_0[1] + src_argb_1[1] + 1) >> 1; + uint8_t r = (src_argb_0[2] + src_argb_1[2] + 1) >> 1; + uint8_t a = (src_argb_0[3] + src_argb_1[3] + 1) >> 1; + dst_u[0] = (c->kAddUV[0] - (c->kRGBToU[0] * b + c->kRGBToU[1] * g + + c->kRGBToU[2] * r + c->kRGBToU[3] * a)) >> + 8; + dst_v[0] = (c->kAddUV[0] - (c->kRGBToV[0] * b + c->kRGBToV[1] * g + + c->kRGBToV[2] * r + c->kRGBToV[3] * a)) >> + 8; + } +} +#endif + +#ifdef HAS_RGBTOUV444MATRIXROW_RVV +void RGBToUV444MatrixRow_RVV(const uint8_t* src_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + assert(width != 0); + size_t w = (size_t)width; + vint8m2_t v_bu, v_gu, v_ru; + vint8m2_t v_bv, v_gv, v_rv; + vint16m4_t v_adduv; + size_t vl = __riscv_vsetvl_e8m2(w); + v_bu = __riscv_vmv_v_x_i8m2(c->kRGBToU[0], vl); + v_gu = __riscv_vmv_v_x_i8m2(c->kRGBToU[1], vl); + v_ru = __riscv_vmv_v_x_i8m2(c->kRGBToU[2], vl); + v_bv = __riscv_vmv_v_x_i8m2(c->kRGBToV[0], vl); + v_gv = __riscv_vmv_v_x_i8m2(c->kRGBToV[1], vl); + v_rv = __riscv_vmv_v_x_i8m2(c->kRGBToV[2], vl); + v_adduv = __riscv_vmv_v_x_i16m4((int16_t)c->kAddUV[0], vl); + do { + size_t vl = __riscv_vsetvl_e8m2(w); + vuint8m2x3_t v_src_rgb = __riscv_vlseg3e8_v_u8m2x3(src_rgb, vl); + vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 0); + vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 1); + vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 2); + + vint16m4_t v_u_i16 = __riscv_vwmulsu_vv_i16m4(v_ru, v_r, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_gu, v_g, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m4(v_u_i16, v_bu, v_b, vl); + v_u_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_u_i16, vl); + vuint8m2_t v_u = __riscv_vnsrl_wx_u8m2( + __riscv_vreinterpret_v_i16m4_u16m4(v_u_i16), 8, vl); + __riscv_vse8_v_u8m2(dst_u, v_u, vl); + + vint16m4_t v_v_i16 = __riscv_vwmulsu_vv_i16m4(v_rv, v_r, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_gv, v_g, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m4(v_v_i16, v_bv, v_b, vl); + v_v_i16 = __riscv_vsub_vv_i16m4(v_adduv, v_v_i16, vl); + vuint8m2_t v_v = __riscv_vnsrl_wx_u8m2( + __riscv_vreinterpret_v_i16m4_u16m4(v_v_i16), 8, vl); + __riscv_vse8_v_u8m2(dst_v, v_v, vl); + + w -= vl; + src_rgb += 3 * vl; + dst_u += vl; + dst_v += vl; + } while (w > 0); +} +#endif + +#ifdef HAS_RGBTOUVMATRIXROW_RVV +void RGBToUVMatrixRow_RVV(const uint8_t* src_rgb, + int src_stride_rgb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + assert(width != 0); + const uint8_t* src_rgb_0 = src_rgb; + const uint8_t* src_rgb_1 = src_rgb + src_stride_rgb; + int w = width >> 1; + if (w > 0) { + size_t w_len = (size_t)w; + vint8m1_t v_bu, v_gu, v_ru; + vint8m1_t v_bv, v_gv, v_rv; + vint16m2_t v_adduv; + size_t vl = __riscv_vsetvl_e8m1(w_len); + v_bu = __riscv_vmv_v_x_i8m1(c->kRGBToU[0], vl); + v_gu = __riscv_vmv_v_x_i8m1(c->kRGBToU[1], vl); + v_ru = __riscv_vmv_v_x_i8m1(c->kRGBToU[2], vl); + v_bv = __riscv_vmv_v_x_i8m1(c->kRGBToV[0], vl); + v_gv = __riscv_vmv_v_x_i8m1(c->kRGBToV[1], vl); + v_rv = __riscv_vmv_v_x_i8m1(c->kRGBToV[2], vl); + v_adduv = __riscv_vmv_v_x_i16m2((int16_t)c->kAddUV[0], vl); + do { + size_t vl = __riscv_vsetvl_e8m1(w_len); + vuint8m1x6_t v_src0 = __riscv_vlseg6e8_v_u8m1x6(src_rgb_0, vl); + vuint8m1x6_t v_src1 = __riscv_vlseg6e8_v_u8m1x6(src_rgb_1, vl); + + vuint8m1_t b0 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 0); + vuint8m1_t g0 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 1); + vuint8m1_t r0 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 2); + vuint8m1_t b1 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 3); + vuint8m1_t g1 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 4); + vuint8m1_t r1 = __riscv_vget_v_u8m1x6_u8m1(v_src0, 5); + + vuint8m1_t b0_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 0); + vuint8m1_t g0_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 1); + vuint8m1_t r0_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 2); + vuint8m1_t b1_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 3); + vuint8m1_t g1_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 4); + vuint8m1_t r1_1 = __riscv_vget_v_u8m1x6_u8m1(v_src1, 5); + + vuint16m2_t sum_b = __riscv_vwaddu_vv_u16m2(b0, b1, vl); + sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b0_1, vl); + sum_b = __riscv_vwaddu_wv_u16m2(sum_b, b1_1, vl); + sum_b = __riscv_vadd_vx_u16m2(sum_b, 2, vl); + vuint8m1_t b = __riscv_vnsrl_wx_u8m1(sum_b, 2, vl); + + vuint16m2_t sum_g = __riscv_vwaddu_vv_u16m2(g0, g1, vl); + sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g0_1, vl); + sum_g = __riscv_vwaddu_wv_u16m2(sum_g, g1_1, vl); + sum_g = __riscv_vadd_vx_u16m2(sum_g, 2, vl); + vuint8m1_t g = __riscv_vnsrl_wx_u8m1(sum_g, 2, vl); + + vuint16m2_t sum_r = __riscv_vwaddu_vv_u16m2(r0, r1, vl); + sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r0_1, vl); + sum_r = __riscv_vwaddu_wv_u16m2(sum_r, r1_1, vl); + sum_r = __riscv_vadd_vx_u16m2(sum_r, 2, vl); + vuint8m1_t r = __riscv_vnsrl_wx_u8m1(sum_r, 2, vl); + + vint16m2_t v_u_i16 = __riscv_vwmulsu_vv_i16m2(v_ru, r, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_gu, g, vl); + v_u_i16 = __riscv_vwmaccsu_vv_i16m2(v_u_i16, v_bu, b, vl); + v_u_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_u_i16, vl); + vuint8m1_t v_u = __riscv_vnsrl_wx_u8m1( + __riscv_vreinterpret_v_i16m2_u16m2(v_u_i16), 8, vl); + __riscv_vse8_v_u8m1(dst_u, v_u, vl); + + vint16m2_t v_v_i16 = __riscv_vwmulsu_vv_i16m2(v_rv, r, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_gv, g, vl); + v_v_i16 = __riscv_vwmaccsu_vv_i16m2(v_v_i16, v_bv, b, vl); + v_v_i16 = __riscv_vsub_vv_i16m2(v_adduv, v_v_i16, vl); + vuint8m1_t v_v = __riscv_vnsrl_wx_u8m1( + __riscv_vreinterpret_v_i16m2_u16m2(v_v_i16), 8, vl); + __riscv_vse8_v_u8m1(dst_v, v_v, vl); + + w_len -= vl; + src_rgb_0 += 6 * vl; + src_rgb_1 += 6 * vl; + dst_u += vl; + dst_v += vl; + } while (w_len > 0); + } + if (width & 1) { + uint8_t b = (src_rgb_0[0] + src_rgb_1[0] + 1) >> 1; + uint8_t g = (src_rgb_0[1] + src_rgb_1[1] + 1) >> 1; + uint8_t r = (src_rgb_0[2] + src_rgb_1[2] + 1) >> 1; + dst_u[0] = (c->kAddUV[0] - (c->kRGBToU[0] * b + c->kRGBToU[1] * g + + c->kRGBToU[2] * r)) >> + 8; + dst_v[0] = (c->kAddUV[0] - (c->kRGBToV[0] * b + c->kRGBToV[1] * g + + c->kRGBToV[2] * r)) >> + 8; + } +} +#endif + // Blend src_argb over src_argb1 and store to dst_argb. // dst_argb may be src_argb or src_argb1. // src_argb: RGB values have already been pre-multiplied by the a. diff --git a/source/row_sme.cc b/source/row_sme.cc index 553936cf8..9ac3fb7f4 100644 --- a/source/row_sme.cc +++ b/source/row_sme.cc @@ -1135,6 +1135,14 @@ __arm_locally_streaming void ARGBToUVMatrixRow_SME( uvconstants); } +__arm_locally_streaming void ARGBToYMatrixRow_SME( + const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c); +} + #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) diff --git a/source/row_sve.cc b/source/row_sve.cc index f5b5d32ba..5c7ddc425 100644 --- a/source/row_sve.cc +++ b/source/row_sve.cc @@ -231,6 +231,13 @@ void ARGBToUVMatrixRow_SVE2(const uint8_t* src_argb, uvconstants); } +void ARGBToYMatrixRow_SVE2(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + ARGBToYMatrixRow_SVE_SC(src_argb, dst_y, width, c); +} + #define ARGBTORGB565_SVE \ /* Inputs: \ * z0: rrrrrxxxbbbbbxxx \ diff --git a/source/row_win.cc b/source/row_win.cc index a71345107..1f2c7e672 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -178,6 +178,7 @@ void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb, dst_v += 32; width -= 32; } + _mm256_zeroupper(); } #endif LIBYUV_TARGET_AVX2 @@ -228,6 +229,7 @@ void ARGBToYMatrixRow_AVX2(const uint8_t* src_argb, dst_y += 32; width -= 32; } + _mm256_zeroupper(); } #ifdef HAS_RAWTOARGBROW_AVX2 @@ -281,30 +283,59 @@ void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) { dst_argb += 128; width -= 32; } + _mm256_zeroupper(); } #endif +#if defined(HAS_RAWTOARGBROW_AVX512BW) || \ + defined(HAS_RGB24TOARGBROW_AVX512BW) || \ + defined(HAS_RGBTOYMATRIXROW_AVX512BW) +static const uint64_t kPermqRGB24ToARGB0_AVX512BW[8] = {0, 1, 1, 2, + 3, 4, 4, 5}; +static const uint64_t kPermqRGB24ToARGB1_AVX512BW[8] = {6, 7, 7, 8, + 9, 10, 10, 11}; +static const uint64_t kPermqRGB24ToARGB2_AVX512BW[8] = {4, 5, 5, 6, + 7, 8, 8, 9}; +static const uint64_t kPermqRGB24ToARGB3_AVX512BW[8] = {2, 3, 3, 4, + 5, 6, 6, 7}; +static const uint8_t kShuffleMaskRGB24ToARGB_AVX512BW[32] = { + 0u, 1u, 2u, 128u, 3u, 4u, 5u, 128u, 6u, 7u, 8u, 128u, 9u, 10u, 11u, 128u, + 4u, 5u, 6u, 128u, 7u, 8u, 9u, 128u, 10u, 11u, 12u, 128u, 13u, 14u, 15u, 128u}; +#endif + +#if defined(HAS_RAWTOARGBROW_AVX512BW) +static const uint8_t kShuffleMaskRAWToARGB_AVX512BW[32] = { + 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u, + 6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u}; +#endif + #ifdef HAS_RAWTOARGBROW_AVX512BW LIBYUV_TARGET_AVX512BW -void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, - uint8_t* dst_argb, - const __m128i* shuffler, - int width) { +static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, + uint8_t* dst_argb, + const uint32_t* shuffler, + int width) { __m512i zmm_alpha = _mm512_set1_epi32(0xff000000); - __m512i zmm_perm = - _mm512_set_epi32(12, 11, 10, 9, 9, 8, 7, 6, 6, 5, 4, 3, 3, 2, 1, 0); - __m512i zmm_shuf = _mm512_broadcast_i32x4(_mm_loadu_si128(shuffler)); + __m512i zmm_q0 = + _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW); + __m512i zmm_q1 = + _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW); + __m512i zmm_q2 = + _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW); + __m512i zmm_q3 = + _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW); + __m512i zmm_shuf = + _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)shuffler)); while (width > 0) { - __m512i zmm0 = _mm512_maskz_loadu_epi8(0xffffffffffffull, src_raw); - __m512i zmm1 = _mm512_maskz_loadu_epi8(0xffffffffffffull, src_raw + 48); - __m512i zmm2 = _mm512_maskz_loadu_epi8(0xffffffffffffull, src_raw + 96); - __m512i zmm3 = _mm512_maskz_loadu_epi8(0xffffffffffffull, src_raw + 144); + __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw); + __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64)); + __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128)); - zmm0 = _mm512_permutexvar_epi32(zmm_perm, zmm0); - zmm1 = _mm512_permutexvar_epi32(zmm_perm, zmm1); - zmm2 = _mm512_permutexvar_epi32(zmm_perm, zmm2); - zmm3 = _mm512_permutexvar_epi32(zmm_perm, zmm3); + __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); + __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); + __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); @@ -325,25 +356,114 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw, dst_argb += 256; width -= 64; } + _mm256_zeroupper(); } LIBYUV_TARGET_AVX512BW void RAWToARGBRow_AVX512BW(const uint8_t* src_raw, uint8_t* dst_argb, int width) { - __m128i shuf = - _mm_set_epi8(-1, 9, 10, 11, -1, 6, 7, 8, -1, 3, 4, 5, -1, 0, 1, 2); - RGBToARGBRow_AVX512BW(src_raw, dst_argb, &shuf, width); + RGBToARGBRow_AVX512BW( + src_raw, dst_argb, + (const uint32_t*)kShuffleMaskRAWToARGB_AVX512BW, width); } LIBYUV_TARGET_AVX512BW void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, uint8_t* dst_argb, int width) { - __m128i shuf = - _mm_set_epi8(-1, 11, 10, 9, -1, 8, 7, 6, -1, 5, 4, 3, -1, 2, 1, 0); - RGBToARGBRow_AVX512BW(src_rgb24, dst_argb, &shuf, width); + RGBToARGBRow_AVX512BW( + src_rgb24, dst_argb, + (const uint32_t*)kShuffleMaskRGB24ToARGB_AVX512BW, width); } + +#ifdef HAS_RGBTOYMATRIXROW_AVX512BW +static const uint32_t kPermdARGBToY_AVX512BW[16] = { + 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15}; + +LIBYUV_TARGET_AVX512BW +void RGBToYMatrixRow_AVX512BW(const uint8_t* src_rgb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + __m512i zmm_q0 = + _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB0_AVX512BW); + __m512i zmm_q1 = + _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB1_AVX512BW); + __m512i zmm_q2 = + _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB2_AVX512BW); + __m512i zmm_q3 = + _mm512_loadu_si512((const __m512i*)kPermqRGB24ToARGB3_AVX512BW); + __m512i zmm_shuf = _mm512_broadcast_i64x4( + _mm256_loadu_si256((const __m256i*)kShuffleMaskRGB24ToARGB_AVX512BW)); + __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + + __m512i zmm_80 = _mm512_set1_epi8((char)0x80); + __m512i zmm_one = _mm512_set1_epi16(1); + __m512i zmm_coeff = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)c->kRGBToY)); + __m512i zmm_add = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)c->kAddY)); + + // Compute bias adjusted for 0x80 subtraction: + // bias = kAddY - pmaddwd(pmaddubsw(kRGBToY, 0x80), 1) + __m512i zmm_bias = _mm512_maddubs_epi16(zmm_coeff, zmm_80); + zmm_bias = _mm512_madd_epi16(zmm_bias, zmm_one); + zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias); + zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias); + + while (width > 0) { + __m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_rgb); + __m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_rgb + 64)); + __m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_rgb + 128)); + + __m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0); + __m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1); + __m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2); + __m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_xor_si512(zmm0, zmm_80); + zmm1 = _mm512_xor_si512(zmm1, zmm_80); + zmm2 = _mm512_xor_si512(zmm2, zmm_80); + zmm3 = _mm512_xor_si512(zmm3, zmm_80); + + zmm0 = _mm512_maddubs_epi16(zmm_coeff, zmm0); + zmm1 = _mm512_maddubs_epi16(zmm_coeff, zmm1); + zmm2 = _mm512_maddubs_epi16(zmm_coeff, zmm2); + zmm3 = _mm512_maddubs_epi16(zmm_coeff, zmm3); + + zmm0 = _mm512_madd_epi16(zmm0, zmm_one); + zmm1 = _mm512_madd_epi16(zmm1, zmm_one); + zmm0 = _mm512_packs_epi32(zmm0, zmm1); + + zmm2 = _mm512_madd_epi16(zmm2, zmm_one); + zmm3 = _mm512_madd_epi16(zmm3, zmm_one); + zmm2 = _mm512_packs_epi32(zmm2, zmm3); + + zmm0 = _mm512_add_epi16(zmm0, zmm_add); + zmm2 = _mm512_add_epi16(zmm2, zmm_add); + + zmm0 = _mm512_srli_epi16(zmm0, 8); + zmm2 = _mm512_srli_epi16(zmm2, 8); + + zmm0 = _mm512_packus_epi16(zmm0, zmm2); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, zmm0); + + _mm512_storeu_si512((__m512i*)dst_y, zmm0); + + src_rgb += 192; + dst_y += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif #endif #ifdef HAS_ARGBTOUVMATRIXROW_AVX2 @@ -412,6 +532,7 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, dst_v += 8; width -= 16; } + _mm256_zeroupper(); } #endif @@ -435,6 +556,7 @@ void MergeUVRow_AVX2(const uint8_t* src_u, dst_uv += 32; width -= 16; } + _mm256_zeroupper(); } #endif @@ -453,6 +575,7 @@ void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { dst += 32; width -= 32; } + _mm256_zeroupper(); } #endif @@ -471,6 +594,7 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { dst_uv += 32; width -= 16; } + _mm256_zeroupper(); } #endif @@ -494,6 +618,7 @@ void MirrorSplitUVRow_AVX2(const uint8_t* src_uv, dst_v += 16; width -= 16; } + _mm256_zeroupper(); } #endif @@ -547,6 +672,7 @@ void RGB24MirrorRow_AVX2(const uint8_t* src_rgb24, dst_rgb24 += 96; width -= 32; } + _mm256_zeroupper(); } #endif @@ -659,6 +785,7 @@ void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { dst += 32; width -= 8; } + _mm256_zeroupper(); } #endif @@ -696,6 +823,7 @@ void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width) { dst_argb += 64; width -= 16; } + _mm256_zeroupper(); } #endif // HAS_J400TOARGBROW_AVX2 @@ -914,6 +1042,7 @@ void ARGBShuffleRow_AVX2(const uint8_t* src_argb, dst_argb += 64; width -= 16; } + _mm256_zeroupper(); } #endif @@ -941,6 +1070,226 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb, #endif +#ifdef HAS_I422TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void I422ToARGBRow_AVX2(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); + __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); + __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); + __m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + __m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); + __m256i ymm_128 = _mm256_set1_epi8((char)0x80); + __m256i ymm_alpha = _mm256_set1_epi8((char)0xff); + __m256i ymm_u_zero = _mm256_setzero_si256(); + + ptrdiff_t offset = src_v - src_u; + + while (width >= 16) { + __m128i xmm_u = _mm_loadl_epi64((const __m128i*)src_u); + __m128i xmm_v = _mm_loadl_epi64((const __m128i*)(src_u + offset)); + src_u += 8; + + __m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_u, 0); + __m256i ymm1 = _mm256_insertf128_si256(ymm_u_zero, xmm_v, 0); + + ymm3 = _mm256_unpacklo_epi8(ymm3, ymm1); + ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8); + ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3); + + __m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y); + src_y += 16; + __m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0); + ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8); + ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4); + + ymm3 = _mm256_sub_epi8(ymm3, ymm_128); + ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb); + + __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3); + ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3); + __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3); + + ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb); + + ymm0 = _mm256_adds_epi16(ymm0, ymm4); + ymm1 = _mm256_subs_epi16(ymm4, ymm1); + ymm2 = _mm256_adds_epi16(ymm2, ymm4); + + ymm0 = _mm256_srai_epi16(ymm0, 6); + ymm1 = _mm256_srai_epi16(ymm1, 6); + ymm2 = _mm256_srai_epi16(ymm2, 6); + + ymm0 = _mm256_packus_epi16(ymm0, ymm0); + ymm1 = _mm256_packus_epi16(ymm1, ymm1); + ymm2 = _mm256_packus_epi16(ymm2, ymm2); + + ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1); + ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8); + ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha); + ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8); + + __m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2); + __m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2); + + _mm256_storeu_si256((__m256i*)dst_argb, argb0); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1); + + dst_argb += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif + +#ifdef HAS_NV12TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void NV12ToARGBRow_AVX2(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); + __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); + __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); + __m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + __m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); + __m256i ymm_128 = _mm256_set1_epi8((char)0x80); + __m256i ymm_alpha = _mm256_set1_epi8((char)0xff); + __m256i ymm_u_zero = _mm256_setzero_si256(); + + while (width >= 16) { + __m128i xmm_uv = _mm_loadu_si128((const __m128i*)src_uv); + src_uv += 16; + + __m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_uv, 0); + ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8); + ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3); + + __m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y); + src_y += 16; + __m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0); + ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8); + ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4); + + ymm3 = _mm256_sub_epi8(ymm3, ymm_128); + ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb); + + __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3); + __m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3); + __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3); + + ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb); + + ymm0 = _mm256_adds_epi16(ymm0, ymm4); + ymm1 = _mm256_subs_epi16(ymm4, ymm1); + ymm2 = _mm256_adds_epi16(ymm2, ymm4); + + ymm0 = _mm256_srai_epi16(ymm0, 6); + ymm1 = _mm256_srai_epi16(ymm1, 6); + ymm2 = _mm256_srai_epi16(ymm2, 6); + + ymm0 = _mm256_packus_epi16(ymm0, ymm0); + ymm1 = _mm256_packus_epi16(ymm1, ymm1); + ymm2 = _mm256_packus_epi16(ymm2, ymm2); + + ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1); + ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8); + ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha); + ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8); + + __m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2); + __m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2); + + _mm256_storeu_si256((__m256i*)dst_argb, argb0); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1); + + dst_argb += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif + +#ifdef HAS_NV21TOARGBROW_AVX2 +static const uint8_t kShuffleNV21_AVX2[32] = { + 1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u, + 1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u, 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u}; + +LIBYUV_TARGET_AVX2 +void NV21ToARGBRow_AVX2(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + __m256i ymm_kUVToB = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); + __m256i ymm_kUVToG = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); + __m256i ymm_kUVToR = _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); + __m256i ymm_kYToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + __m256i ymm_kYBiasToRgb = _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); + __m256i ymm_128 = _mm256_set1_epi8((char)0x80); + __m256i ymm_alpha = _mm256_set1_epi8((char)0xff); + __m256i ymm_shuf_nv21 = _mm256_loadu_si256((const __m256i*)kShuffleNV21_AVX2); + __m256i ymm_u_zero = _mm256_setzero_si256(); + + while (width >= 16) { + __m128i xmm_vu = _mm_loadu_si128((const __m128i*)src_vu); + src_vu += 16; + + __m256i ymm3 = _mm256_insertf128_si256(ymm_u_zero, xmm_vu, 0); + ymm3 = _mm256_permute4x64_epi64(ymm3, 0xd8); + ymm3 = _mm256_shuffle_epi8(ymm3, ymm_shuf_nv21); + ymm3 = _mm256_unpacklo_epi16(ymm3, ymm3); + + __m128i xmm_y = _mm_loadu_si128((const __m128i*)src_y); + src_y += 16; + __m256i ymm4 = _mm256_insertf128_si256(ymm_u_zero, xmm_y, 0); + ymm4 = _mm256_permute4x64_epi64(ymm4, 0xd8); + ymm4 = _mm256_unpacklo_epi8(ymm4, ymm4); + + ymm3 = _mm256_sub_epi8(ymm3, ymm_128); + ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb); + + __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3); + __m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3); + __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3); + + ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb); + + ymm0 = _mm256_adds_epi16(ymm0, ymm4); + ymm1 = _mm256_subs_epi16(ymm4, ymm1); + ymm2 = _mm256_adds_epi16(ymm2, ymm4); + + ymm0 = _mm256_srai_epi16(ymm0, 6); + ymm1 = _mm256_srai_epi16(ymm1, 6); + ymm2 = _mm256_srai_epi16(ymm2, 6); + + ymm0 = _mm256_packus_epi16(ymm0, ymm0); + ymm1 = _mm256_packus_epi16(ymm1, ymm1); + ymm2 = _mm256_packus_epi16(ymm2, ymm2); + + ymm0 = _mm256_unpacklo_epi8(ymm0, ymm1); + ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8); + ymm2 = _mm256_unpacklo_epi8(ymm2, ymm_alpha); + ymm2 = _mm256_permute4x64_epi64(ymm2, 0xd8); + + __m256i argb0 = _mm256_unpacklo_epi16(ymm0, ymm2); + __m256i argb1 = _mm256_unpackhi_epi16(ymm0, ymm2); + + _mm256_storeu_si256((__m256i*)dst_argb, argb0); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), argb1); + + dst_argb += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif + #ifdef HAS_I422TORGB24ROW_AVX2 static const uint8_t kShuffleMaskARGBToRGB24[2][16] = { {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, @@ -1138,9 +1487,9 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y, zmm2 = _mm512_packus_epi16(zmm2, zmm2); // STORERGB24_AVX512VBMI - __m512i zmm_BG = _mm512_permi2var_epi8(zmm0, zmm_mask_BG, zmm1); - __m512i zmm_dst0 = _mm512_permi2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); - __m512i zmm_dst1 = _mm512_permi2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1)); diff --git a/unit_test/convert_test.cc b/unit_test/convert_test.cc index 12398fc08..ba1102e50 100644 --- a/unit_test/convert_test.cc +++ b/unit_test/convert_test.cc @@ -2216,6 +2216,202 @@ TEST_F(LibYUVConvertTest, TestRAWToI420) { free_aligned_buffer_page_end(ref_v); } +TEST_F(LibYUVConvertTest, TestRAWToI444) { + const int kWidth = 16; + const int kHeight = 16; + align_buffer_page_end(src_raw, kWidth * kHeight * 3); + align_buffer_page_end(dst_y, kWidth * kHeight); + align_buffer_page_end(dst_u, kWidth * kHeight); + align_buffer_page_end(dst_v, kWidth * kHeight); + align_buffer_page_end(src_argb, kWidth * kHeight * 4); + align_buffer_page_end(ref_y, kWidth * kHeight); + align_buffer_page_end(ref_u, kWidth * kHeight); + align_buffer_page_end(ref_v, kWidth * kHeight); + + MemRandomize(src_raw, kWidth * kHeight * 3); + + RAWToI444(src_raw, kWidth * 3, dst_y, kWidth, dst_u, kWidth, dst_v, kWidth, + kWidth, kHeight); + + // Verify against RAWToARGB + ARGBToI444 + RAWToARGB(src_raw, kWidth * 3, src_argb, kWidth * 4, kWidth, kHeight); + ARGBToI444(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth, ref_v, kWidth, + kWidth, kHeight); + + for (int i = 0; i < kWidth * kHeight; ++i) { + ASSERT_EQ(dst_y[i], ref_y[i]); + ASSERT_EQ(dst_u[i], ref_u[i]); + ASSERT_EQ(dst_v[i], ref_v[i]); + } + + free_aligned_buffer_page_end(src_raw); + free_aligned_buffer_page_end(dst_y); + free_aligned_buffer_page_end(dst_u); + free_aligned_buffer_page_end(dst_v); + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(ref_y); + free_aligned_buffer_page_end(ref_u); + free_aligned_buffer_page_end(ref_v); +} + +TEST_F(LibYUVConvertTest, TestRAWToJ444) { + const int kWidth = 16; + const int kHeight = 16; + align_buffer_page_end(src_raw, kWidth * kHeight * 3); + align_buffer_page_end(dst_y, kWidth * kHeight); + align_buffer_page_end(dst_u, kWidth * kHeight); + align_buffer_page_end(dst_v, kWidth * kHeight); + align_buffer_page_end(src_argb, kWidth * kHeight * 4); + align_buffer_page_end(ref_y, kWidth * kHeight); + align_buffer_page_end(ref_u, kWidth * kHeight); + align_buffer_page_end(ref_v, kWidth * kHeight); + + MemRandomize(src_raw, kWidth * kHeight * 3); + + RAWToJ444(src_raw, kWidth * 3, dst_y, kWidth, dst_u, kWidth, dst_v, kWidth, + kWidth, kHeight); + + // Verify against RAWToARGB + ARGBToJ444 + RAWToARGB(src_raw, kWidth * 3, src_argb, kWidth * 4, kWidth, kHeight); + ARGBToJ444(src_argb, kWidth * 4, ref_y, kWidth, ref_u, kWidth, ref_v, kWidth, + kWidth, kHeight); + + for (int i = 0; i < kWidth * kHeight; ++i) { + ASSERT_EQ(dst_y[i], ref_y[i]); + ASSERT_EQ(dst_u[i], ref_u[i]); + ASSERT_EQ(dst_v[i], ref_v[i]); + } + + free_aligned_buffer_page_end(src_raw); + free_aligned_buffer_page_end(dst_y); + free_aligned_buffer_page_end(dst_u); + free_aligned_buffer_page_end(dst_v); + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(ref_y); + free_aligned_buffer_page_end(ref_u); + free_aligned_buffer_page_end(ref_v); +} + +TEST_F(LibYUVConvertTest, TestRAWToNV21) { + const int kWidth = 16; + const int kHeight = 16; + align_buffer_page_end(src_raw, kWidth * kHeight * 3); + align_buffer_page_end(dst_y, kWidth * kHeight); + align_buffer_page_end(dst_vu, kWidth * kHeight / 2); + align_buffer_page_end(ref_y, kWidth * kHeight); + align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2); + align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2); + align_buffer_page_end(ref_vu, kWidth * kHeight / 2); + + MemRandomize(src_raw, kWidth * kHeight * 3); + + RAWToNV21(src_raw, kWidth * 3, dst_y, kWidth, dst_vu, kWidth, kWidth, + kHeight); + + // Verify against RAWToI420 + interleaving V and U + RAWToI420(src_raw, kWidth * 3, ref_y, kWidth, ref_u, kWidth / 2, ref_v, + kWidth / 2, kWidth, kHeight); + for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) { + ref_vu[i * 2] = ref_v[i]; + ref_vu[i * 2 + 1] = ref_u[i]; + } + + for (int i = 0; i < kWidth * kHeight; ++i) { + ASSERT_EQ(dst_y[i], ref_y[i]); + } + for (int i = 0; i < kWidth * kHeight / 2; ++i) { + ASSERT_EQ(dst_vu[i], ref_vu[i]); + } + + free_aligned_buffer_page_end(src_raw); + free_aligned_buffer_page_end(dst_y); + free_aligned_buffer_page_end(dst_vu); + free_aligned_buffer_page_end(ref_y); + free_aligned_buffer_page_end(ref_u); + free_aligned_buffer_page_end(ref_v); + free_aligned_buffer_page_end(ref_vu); +} + +TEST_F(LibYUVConvertTest, TestRAWToJNV21) { + const int kWidth = 16; + const int kHeight = 16; + align_buffer_page_end(src_raw, kWidth * kHeight * 3); + align_buffer_page_end(dst_y, kWidth * kHeight); + align_buffer_page_end(dst_vu, kWidth * kHeight / 2); + align_buffer_page_end(ref_y, kWidth * kHeight); + align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2); + align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2); + align_buffer_page_end(ref_vu, kWidth * kHeight / 2); + + MemRandomize(src_raw, kWidth * kHeight * 3); + + RAWToJNV21(src_raw, kWidth * 3, dst_y, kWidth, dst_vu, kWidth, kWidth, + kHeight); + + // Verify against RAWToJ420 + interleaving V and U + RAWToJ420(src_raw, kWidth * 3, ref_y, kWidth, ref_u, kWidth / 2, ref_v, + kWidth / 2, kWidth, kHeight); + for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) { + ref_vu[i * 2] = ref_v[i]; + ref_vu[i * 2 + 1] = ref_u[i]; + } + + for (int i = 0; i < kWidth * kHeight; ++i) { + ASSERT_EQ(dst_y[i], ref_y[i]); + } + for (int i = 0; i < kWidth * kHeight / 2; ++i) { + ASSERT_EQ(dst_vu[i], ref_vu[i]); + } + + free_aligned_buffer_page_end(src_raw); + free_aligned_buffer_page_end(dst_y); + free_aligned_buffer_page_end(dst_vu); + free_aligned_buffer_page_end(ref_y); + free_aligned_buffer_page_end(ref_u); + free_aligned_buffer_page_end(ref_v); + free_aligned_buffer_page_end(ref_vu); +} + +TEST_F(LibYUVConvertTest, TestRGB24ToNV12) { + const int kWidth = 16; + const int kHeight = 16; + align_buffer_page_end(src_rgb24, kWidth * kHeight * 3); + align_buffer_page_end(dst_y, kWidth * kHeight); + align_buffer_page_end(dst_uv, kWidth * kHeight / 2); + align_buffer_page_end(ref_y, kWidth * kHeight); + align_buffer_page_end(ref_u, kWidth / 2 * kHeight / 2); + align_buffer_page_end(ref_v, kWidth / 2 * kHeight / 2); + align_buffer_page_end(ref_uv, kWidth * kHeight / 2); + + MemRandomize(src_rgb24, kWidth * kHeight * 3); + + RGB24ToNV12(src_rgb24, kWidth * 3, dst_y, kWidth, dst_uv, kWidth, kWidth, + kHeight); + + // Verify against RGB24ToI420 + interleaving U and V + RGB24ToI420(src_rgb24, kWidth * 3, ref_y, kWidth, ref_u, kWidth / 2, ref_v, + kWidth / 2, kWidth, kHeight); + for (int i = 0; i < kWidth / 2 * kHeight / 2; ++i) { + ref_uv[i * 2] = ref_u[i]; + ref_uv[i * 2 + 1] = ref_v[i]; + } + + for (int i = 0; i < kWidth * kHeight; ++i) { + ASSERT_EQ(dst_y[i], ref_y[i]); + } + for (int i = 0; i < kWidth * kHeight / 2; ++i) { + ASSERT_EQ(dst_uv[i], ref_uv[i]); + } + + free_aligned_buffer_page_end(src_rgb24); + free_aligned_buffer_page_end(dst_y); + free_aligned_buffer_page_end(dst_uv); + free_aligned_buffer_page_end(ref_y); + free_aligned_buffer_page_end(ref_u); + free_aligned_buffer_page_end(ref_v); + free_aligned_buffer_page_end(ref_uv); +} + TEST_F(LibYUVConvertTest, TestARGBToNV12Matrix) { const int kWidth = 16; const int kHeight = 16;