diff --git a/GEMINI.md b/GEMINI.md index a82327f5b..923077777 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -47,11 +47,30 @@ and compiler compatibility. ### Utility and Fallbacks -* **row_common.cc**: Portable C/C++ versions. This is the reference - implementation. +* **row_common.cc**: Portable C/C++ versions and multi-step `ARGB` wrapper + row functions. This is the reference implementation. * **row_any.cc**: Handles "remainder" pixels for widths not multiples of SIMD - register size. Used for x86, Neon, and MIPS. Not required for SVE2, SME, - RVV, or HVX due to hardware-level masking. + register size. Used for SSE/AVX2, Neon, and LoongArch (LSX/LASX). Not + required for AVX-512, SVE2, SME, RVV, or HVX, which handle tail masking + directly inside the Row function. + +## Deprecated and Lower-Priority Formats + +The following datatypes and functions are deprecated or lower priority and are +not always fully hand-optimized across every SIMD extension: + +* **16-bit RGB formats** (deprecated): `RGB565` (`FOURCC_RGBP`, + `FOURCC_L565`), `ARGB1555` (`FOURCC_RGBO`, `FOURCC_L555`, `FOURCC_5551`), + and `ARGB4444` (`FOURCC_R444`). +* **Legacy fixed-matrix RGB-to-YUV row functions** (deprecated in favor of + `*MatrixRow` variants): `RGB565ToYRow`, `RGB565ToUVRow`, `ARGB1555ToYRow`, + `ARGB1555ToUVRow`, `ARGB4444ToYRow`, `ARGB4444ToUVRow`. +* **Row-biplanar YUV** (deprecated): `M420` (`FOURCC_M420`). + +When adding SIMD coverage for deprecated or less relevant formats, prefer +two-step wrapper implementations in `source/row_common.cc` that convert to or +from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing +`*MatrixRow` functions) rather than writing large bespoke assembly kernels. ## Coding Guidelines @@ -72,7 +91,9 @@ and compiler compatibility. ### x86 Architectures (32-bit and 64-bit) 1. **AVX512 Logic**: AVX512 row functions are strictly enabled for **64-bit x86 - only**. + only**. Do not create `_Any_` wrappers for AVX512 row functions; handle + arbitrary tail widths directly inside the Row function using `k1`-`k7` + opmask registers. 2. **MSVC Intrinsics (`row_win.cc`)**: When converting `row_gcc.cc` to `row_win.cc`, use `_mm512_permutex2var_epi8`, not `_mm512_permi2var_epi8` intrinsic. diff --git a/README.chromium b/README.chromium index 6dfc0d43d..0bac66a43 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1976 +Version: 1977 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/docs/environment_variables.md b/docs/environment_variables.md index a8c7aef6b..a46744e45 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -33,6 +33,8 @@ By default the cpu is detected and the most advanced form of SIMD is used. But LIBYUV_DISABLE_AVXVNNI LIBYUV_DISABLE_AVXVNNIINT8 LIBYUV_DISABLE_AMXINT8 + LIBYUV_DISABLE_AVX512BMM + LIBYUV_DISABLE_AMXTF32 ## Arm CPUs diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index 301a103e9..03caa5c01 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -21,14 +21,14 @@ namespace libyuv { extern "C" { #endif -// The following are available for 32 bit x86, Visual C 64 bit, and for clang -// with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These -// are intrinsics in compare_win.cc. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__i386__) || defined(_M_IX86) || \ - ((defined(__x86_64__) || defined(_M_X64)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)))) +// The following are available for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These are +// intrinsics in compare_win.cc. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) #define HAS_HAMMINGDISTANCE_SSE42 #define HAS_SUMSQUAREERROR_SSE2 #define HAS_HASHDJB2_SSE41 @@ -39,19 +39,26 @@ extern "C" { #define HAS_HASHDJB2_AVX2 #define HAS_SSIM4X4ROW_AVX2 #define HAS_SSIMWINDOWROW_AVX2 +#define HAS_ARGBDETECTROW_AVX2 +#if defined(__x86_64__) || defined(_M_X64) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW +#define HAS_SUMSQUAREERROR_AVX512VNNI #define HAS_HASHDJB2_AVX512BW #define HAS_SSIM4X4ROW_AVX512BW #define HAS_SSIMWINDOWROW_AVX512BW #endif +#endif -// The following are available for 64 bit GCC and clang, as inline assembly in -// compare_gcc.cc: -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) && \ +// The following are available for 32/64 bit GCC and clang, as inline assembly +// in compare_gcc.cc: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ (!defined(_MSC_VER) || defined(__clang__)) +#if defined(__x86_64__) #define HAS_HAMMINGDISTANCE_SSE42 +#endif #define HAS_HAMMINGDISTANCE_SSSE3 #define HAS_SUMSQUAREERROR_SSE2 #define HAS_HASHDJB2_SSE41 @@ -63,10 +70,12 @@ extern "C" { #define HAS_HASHDJB2_AVX2 #define HAS_SSIM4X4ROW_AVX2 #define HAS_SSIMWINDOWROW_AVX2 +#define HAS_ARGBDETECTROW_AVX2 #endif -#if defined(CLANG_HAS_AVX512) +#if defined(__x86_64__) && defined(CLANG_HAS_AVX512) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW +#define HAS_SUMSQUAREERROR_AVX512VNNI #define HAS_HASHDJB2_AVX512BW #define HAS_SSIM4X4ROW_AVX512BW #define HAS_SSIMWINDOWROW_AVX512BW @@ -177,6 +186,9 @@ uint32_t SumSquareError_AVX2(const uint8_t* src_a, uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count); +uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a, + const uint8_t* src_b, + int count); uint32_t SumSquareError_NEON(const uint8_t* src_a, const uint8_t* src_b, int count); @@ -298,6 +310,9 @@ float SsimWindowRow_RVV(const uint32_t* row0, const uint32_t* row1, int num_windows); +uint32_t ARGBDetectRow_C(const uint8_t* argb, int width); +uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width); + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/include/libyuv/convert.h b/include/libyuv/convert.h index adf35df7f..fdaec5eaf 100644 --- a/include/libyuv/convert.h +++ b/include/libyuv/convert.h @@ -1152,7 +1152,7 @@ int RAWToJ444(const uint8_t* src_raw, int width, int height); -// RGB16 (RGBP fourcc) little endian to I420. +// RGB16 (RGBP fourcc) little endian to I420. Deprecated. LIBYUV_API int RGB565ToI420(const uint8_t* src_rgb565, int src_stride_rgb565, @@ -1165,7 +1165,7 @@ int RGB565ToI420(const uint8_t* src_rgb565, int width, int height); -// RGB15 (RGBO fourcc) little endian to I420. +// RGB15 (RGBO fourcc) little endian to I420. Deprecated. LIBYUV_API int ARGB1555ToI420(const uint8_t* src_argb1555, int src_stride_argb1555, @@ -1178,7 +1178,7 @@ int ARGB1555ToI420(const uint8_t* src_argb1555, int width, int height); -// RGB12 (R444 fourcc) little endian to I420. +// RGB12 (R444 fourcc) little endian to I420. Deprecated. LIBYUV_API int ARGB4444ToI420(const uint8_t* src_argb4444, int src_stride_argb4444, diff --git a/include/libyuv/convert_argb.h b/include/libyuv/convert_argb.h index bcb2099d6..59b690f3a 100644 --- a/include/libyuv/convert_argb.h +++ b/include/libyuv/convert_argb.h @@ -1038,7 +1038,7 @@ int RAWToRGBA(const uint8_t* src_raw, int width, int height); -// RGB16 (RGBP fourcc) little endian to ARGB. +// RGB16 (RGBP fourcc) little endian to ARGB. Deprecated. LIBYUV_API int RGB565ToARGB(const uint8_t* src_rgb565, int src_stride_rgb565, @@ -1047,7 +1047,7 @@ int RGB565ToARGB(const uint8_t* src_rgb565, int width, int height); -// RGB15 (RGBO fourcc) little endian to ARGB. +// RGB15 (RGBO fourcc) little endian to ARGB. Deprecated. LIBYUV_API int ARGB1555ToARGB(const uint8_t* src_argb1555, int src_stride_argb1555, @@ -1056,7 +1056,7 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555, int width, int height); -// RGB12 (R444 fourcc) little endian to ARGB. +// RGB12 (R444 fourcc) little endian to ARGB. Deprecated. LIBYUV_API int ARGB4444ToARGB(const uint8_t* src_argb4444, int src_stride_argb4444, @@ -1173,7 +1173,7 @@ int Android420ToABGR(const uint8_t* src_y, int width, int height); -// Convert NV12 to RGB565. +// Convert NV12 to RGB565. Deprecated. LIBYUV_API int NV12ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1369,6 +1369,7 @@ int I422ToRAW(const uint8_t* src_y, int width, int height); +// Convert I420 to RGB565. Deprecated. LIBYUV_API int I420ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1381,6 +1382,7 @@ int I420ToRGB565(const uint8_t* src_y, int width, int height); +// Convert J420 to RGB565. Deprecated. LIBYUV_API int J420ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1393,6 +1395,7 @@ int J420ToRGB565(const uint8_t* src_y, int width, int height); +// Convert H420 to RGB565. Deprecated. LIBYUV_API int H420ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1405,6 +1408,7 @@ int H420ToRGB565(const uint8_t* src_y, int width, int height); +// Convert I422 to RGB565. Deprecated. LIBYUV_API int I422ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1417,7 +1421,7 @@ int I422ToRGB565(const uint8_t* src_y, int width, int height); -// Convert I420 To RGB565 with 4x4 dither matrix (16 bytes). +// Convert I420 To RGB565 with 4x4 dither matrix (16 bytes). Deprecated. // Values in dither matrix from 0 to 7 recommended. // The order of the dither matrix is first byte is upper left. @@ -1434,6 +1438,7 @@ int I420ToRGB565Dither(const uint8_t* src_y, int width, int height); +// Convert I420 to ARGB1555. Deprecated. LIBYUV_API int I420ToARGB1555(const uint8_t* src_y, int src_stride_y, @@ -1446,6 +1451,7 @@ int I420ToARGB1555(const uint8_t* src_y, int width, int height); +// Convert I420 to ARGB4444. Deprecated. LIBYUV_API int I420ToARGB4444(const uint8_t* src_y, int src_stride_y, @@ -1903,7 +1909,7 @@ int NV21ToARGBMatrix(const uint8_t* src_y, int width, int height); -// Convert NV12 to RGB565 with matrix. +// Convert NV12 to RGB565 with matrix. Deprecated. LIBYUV_API int NV12ToRGB565Matrix(const uint8_t* src_y, int src_stride_y, @@ -2030,7 +2036,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y, int width, int height); -// Convert I420 to RGB565 with specified color matrix. +// Convert I420 to RGB565 with specified color matrix. Deprecated. LIBYUV_API int I420ToRGB565Matrix(const uint8_t* src_y, int src_stride_y, @@ -2044,7 +2050,7 @@ int I420ToRGB565Matrix(const uint8_t* src_y, int width, int height); -// Convert I422 to RGB565 with specified color matrix. +// Convert I422 to RGB565 with specified color matrix. Deprecated. LIBYUV_API int I422ToRGB565Matrix(const uint8_t* src_y, int src_stride_y, diff --git a/include/libyuv/convert_from_argb.h b/include/libyuv/convert_from_argb.h index 29d656d0a..817df6f9e 100644 --- a/include/libyuv/convert_from_argb.h +++ b/include/libyuv/convert_from_argb.h @@ -142,7 +142,7 @@ int ARGBToRAW(const uint8_t* src_argb, int width, int height); -// Convert ARGB To RGB565. +// Convert ARGB To RGB565. Deprecated. LIBYUV_API int ARGBToRGB565(const uint8_t* src_argb, int src_stride_argb, @@ -151,7 +151,7 @@ int ARGBToRGB565(const uint8_t* src_argb, int width, int height); -// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes). +// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes). Deprecated. // Values in dither matrix from 0 to 7 recommended. // The order of the dither matrix is first byte is upper left. // TODO(fbarchard): Consider pointer to 2d array for dither4x4. @@ -165,7 +165,7 @@ int ARGBToRGB565Dither(const uint8_t* src_argb, int width, int height); -// Convert ARGB To ARGB1555. +// Convert ARGB To ARGB1555. Deprecated. LIBYUV_API int ARGBToARGB1555(const uint8_t* src_argb, int src_stride_argb, @@ -174,7 +174,7 @@ int ARGBToARGB1555(const uint8_t* src_argb, int width, int height); -// Convert ARGB To ARGB4444. +// Convert ARGB To ARGB4444. Deprecated. LIBYUV_API int ARGBToARGB4444(const uint8_t* src_argb, int src_stride_argb, diff --git a/include/libyuv/cpu_id.h b/include/libyuv/cpu_id.h index 917df1cb4..54512b766 100644 --- a/include/libyuv/cpu_id.h +++ b/include/libyuv/cpu_id.h @@ -63,6 +63,7 @@ static const int kCpuHasAVXVNNI = 0x4000000; static const int kCpuHasAVXVNNIINT8 = 0x8000000; static const int kCpuHasAMXINT8 = 0x10000000; static const int kCpuHasAVX512BMM = 0x20000000; +static const int kCpuHasAMXTF32 = 0x40000000; // These flags are only valid on LOONGARCH processors. static const int kCpuHasLOONGARCH = 0x20; diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index adb50c42d..227bf3bd1 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -81,7 +81,7 @@ extern "C" { #endif // __clang__ // Visual C 2012 required for AVX2. -#if defined(_M_IX86) && \ +#if (defined(_M_IX86) || defined(_M_X64)) && \ (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN)) && \ defined(_MSC_VER) && _MSC_VER >= 1700 #define VISUALC_HAS_AVX2 1 diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index 4897256db..4ac43d388 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -21,19 +21,21 @@ namespace libyuv { extern "C" { #endif -// The following are available for 32 bit x86, Visual C 64 bit, and for clang -// with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These -// are intrinsics in rotate_win.cc. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__i386__) || defined(_M_IX86) || \ - ((defined(__x86_64__) || defined(_M_X64)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)))) -#define HAS_TRANSPOSEWX8_SSSE3 -#define HAS_TRANSPOSEUVWX8_SSE2 +// The following are available for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These are +// intrinsics in rotate_win.cc. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) +#define HAS_TRANSPOSEWX8_AVX2 +#define HAS_TRANSPOSEUVWX8_AVX2 +#define HAS_TRANSPOSEWX8_16_AVX2 #define HAS_TRANSPOSEWX16_16_AVX2 -#define HAS_TRANSPOSEWX4_32_SSE2 +#define HAS_TRANSPOSEWX4_32_AVX2 #define HAS_TRANSPOSEWX8_32_AVX2 +#define HAS_TRANSPOSEWXH_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 #if defined(__x86_64__) || defined(_M_X64) @@ -44,22 +46,27 @@ extern "C" { #endif #endif -// The following are available for 64 bit GCC and clang, as inline assembly in -// rotate_gcc.cc: -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) -#define HAS_TRANSPOSEWX8_SSSE3 -#define HAS_TRANSPOSEUVWX8_SSE2 +// The following are available for 32/64 bit GCC and clang, as inline assembly +// in rotate_gcc.cc: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ + (!defined(_MSC_VER) || defined(__clang__)) +#define HAS_TRANSPOSEWX8_AVX2 +#define HAS_TRANSPOSEUVWX8_AVX2 +#define HAS_TRANSPOSEWX8_16_AVX2 #define HAS_TRANSPOSEWX16_16_AVX2 -#define HAS_TRANSPOSEWX4_32_SSE2 +#define HAS_TRANSPOSEWX4_32_AVX2 #define HAS_TRANSPOSEWX8_32_AVX2 +#define HAS_TRANSPOSEWXH_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 #endif // The following are available for AVX512 clang x64 platforms: #if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) + defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) && \ + (!defined(_MSC_VER) || defined(__clang__)) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW #define HAS_TRANSPOSEWX32_16_AVX512BW @@ -128,13 +135,6 @@ extern "C" { #define HAS_TRANSPOSEWXH_32_HVX #endif -void TransposeWxH_C(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width, - int height); - void TransposeWx8_C(const uint8_t* src, int src_stride, uint8_t* dst, @@ -167,11 +167,11 @@ void TransposeWxH_HVX(const uint8_t* src, int dst_stride, int width, int height); -void TransposeWx8_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); +void TransposeWx8_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx16_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, @@ -213,11 +213,11 @@ void TransposeWx16_Any_NEON(const uint8_t* src, uint8_t* dst, int dst_stride, int width); -void TransposeWx8_Any_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); +void TransposeWx8_Any_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx16_Any_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, @@ -244,15 +244,6 @@ void TransposeWx16_Any_WASMSIMD(const uint8_t* src, int dst_stride, int width); -void TransposeUVWxH_C(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width, - int height); - void TransposeUVWx8_C(const uint8_t* src, int src_stride, uint8_t* dst_a, @@ -267,7 +258,7 @@ void TransposeUVWx16_C(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); -void TransposeUVWx8_SSE2(const uint8_t* src, +void TransposeUVWx8_AVX2(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, @@ -340,7 +331,7 @@ void TransposeUVWx16_WASMSIMD(const uint8_t* src, int dst_stride_b, int width); -void TransposeUVWx8_Any_SSE2(const uint8_t* src, +void TransposeUVWx8_Any_AVX2(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, @@ -389,18 +380,17 @@ void TransposeUVWx16_Any_WASMSIMD(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); -void TransposeWxH_16_C(const uint16_t* src, - int src_stride, - uint16_t* dst, - int dst_stride, - int width, - int height); void TransposeWx8_16_C(const uint16_t* src, int src_stride, uint16_t* dst, int dst_stride, int width); +void TransposeWx8_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); void TransposeWx16_16_AVX2(const uint16_t* src, int src_stride, uint16_t* dst, @@ -454,6 +444,11 @@ void TransposeWx8_16_WASMSIMD(const uint16_t* src, uint16_t* dst, int dst_stride, int width); +void TransposeWx8_16_Any_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); void TransposeWx16_16_Any_AVX2(const uint16_t* src, int src_stride, uint16_t* dst, @@ -513,7 +508,7 @@ void TransposeWx4_32_SVE2(const uint8_t* src, int dst_stride, int width); -void TransposeWx4_32_SSE2(const uint8_t* src, +void TransposeWx4_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, @@ -525,6 +520,13 @@ void TransposeWx8_32_AVX2(const uint8_t* src, int dst_stride, int width); +void TransposeWxH_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height); + void TransposeWx16_32_AVX512BW(const uint8_t* src, int src_stride, uint8_t* dst, @@ -574,7 +576,7 @@ void TransposeWx4_32_C(const uint8_t* src, int dst_stride, int width); -void TransposeWx4_32_Any_SSE2(const uint8_t* src, +void TransposeWx4_32_Any_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index f3c46bb2a..3fe8341b9 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -39,7 +39,6 @@ extern "C" { #if defined(USE_ROW_GCC) // Conversions: #define HAS_ARGBEXTRACTALPHAROW_SSE2 -#define HAS_ARGBSETROW_X86 #define HAS_ARGBSHUFFLEROW_SSSE3 #define HAS_ARGBTORAWROW_SSSE3 #define HAS_ARGBTORGB24ROW_SSSE3 @@ -66,7 +65,6 @@ extern "C" { #define HAS_RAWTORGB24ROW_SSSE3 #define HAS_RGB24TOARGBROW_SSSE3 #define HAS_SETROW_ERMS -#define HAS_SETROW_X86 #define HAS_SPLITUVROW_SSE2 #define HAS_UYVYTOARGBROW_SSSE3 #define HAS_UYVYTOYROW_SSE2 @@ -77,53 +75,17 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_SSE2 -#define HAS_ARGBAFFINEROW_SSE2 #define HAS_ARGBALPHABLENDROW_SSSE3 -#define HAS_ARGBBLENDROW_SSSE3 -#define HAS_ARGBCOLORMATRIXROW_SSSE3 -#define HAS_ARGBCOLORTABLEROW_X86 #define HAS_ARGBCOPYALPHAROW_SSE2 #define HAS_ARGBCOPYYTOALPHAROW_SSE2 -#define HAS_ARGBGRAYROW_SSSE3 -#define HAS_ARGBLUMACOLORTABLEROW_SSSE3 #define HAS_ARGBMIRRORROW_SSE2 #define HAS_ARGBMULTIPLYROW_SSE2 #define HAS_ARGBORROW_SSE2 #define HAS_ARGBPOLYNOMIALROW_SSE2 -#define HAS_ARGBQUANTIZEROW_SSE2 -#define HAS_ARGBSEPIAROW_SSSE3 -#define HAS_ARGBSHADEROW_SSE2 #define HAS_ARGBSUBTRACTROW_SSE2 #define HAS_BLENDPLANEROW_SSSE3 -#define HAS_COMPUTECUMULATIVESUMROW_SSE2 -#define HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 -#define HAS_RGBCOLORTABLEROW_X86 -#define HAS_SOBELROW_SSE2 -#define HAS_SOBELTOPLANEROW_SSE2 -#define HAS_SOBELXROW_SSE2 -#define HAS_SOBELXYROW_SSE2 -#define HAS_SOBELYROW_SSE2 - -// The following functions fail on gcc/clang 32 bit with fpic and framepointer. -// caveat: clangcl uses row_win.cc which works. -#if (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \ - defined(_MSC_VER)) && \ - !defined(LIBYUV_ENABLE_ROWWIN) -// TODO(fbarchard): fix build error on android_full_debug=1 -// https://code.google.com/p/libyuv/issues/detail?id=517 #define HAS_I422ALPHATOARGBROW_SSSE3 #define HAS_I444ALPHATOARGBROW_SSSE3 -#endif -#if (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \ - defined(_MSC_VER)) && \ - !defined(LIBYUV_ENABLE_ROWWIN) -// TODO(fbarchard): fix build error on android_full_debug=1 -// https://code.google.com/p/libyuv/issues/detail?id=517 -// TODO(fbarchard): fix LIBYUV_ENABLE_ROWWIN with clang -#define HAS_I422ALPHATOARGBROW_AVX2 -#define HAS_I444ALPHATOARGBROW_AVX2 -#endif - #endif // The following are available on all x86 platforms, but @@ -160,11 +122,13 @@ extern "C" { #define HAS_ARGBTORGB565DITHERROW_AVX2 #define HAS_COPYROW_AVX #define HAS_HALFFLOATROW_AVX2 +#define HAS_I422ALPHATOARGBROW_AVX2 #define HAS_I422TORGB24ROW_AVX2 #define HAS_I422TORGBAROW_AVX2 #define HAS_I422TORGB565ROW_AVX2 #define HAS_I422TOARGB1555ROW_AVX2 #define HAS_I422TOARGB4444ROW_AVX2 +#define HAS_I444ALPHATOARGBROW_AVX2 #define HAS_I444TOARGBROW_AVX2 #define HAS_I444TORGB24ROW_AVX2 #define HAS_J400TOARGBROW_AVX2 @@ -204,10 +168,6 @@ extern "C" { #define HAS_ARGBUNATTENUATEROW_SSE2 #define HAS_CONVERT16TO8ROW_SSSE3 #define HAS_CONVERT8TO16ROW_SSE2 -#define HAS_DETILEROW_16_SSE2 -#define HAS_DETILEROW_SSE2 -#define HAS_DETILESPLITUVROW_SSSE3 -#define HAS_DETILETOYUY2_SSE2 #define HAS_HALFMERGEUVROW_SSSE3 #define HAS_HALFROW_16TO8_SSSE3 #define HAS_HALFWIDTHROW_16TO8_SSSE3 @@ -218,11 +178,8 @@ extern "C" { #define HAS_I400TOARGBROW_SSE2 #define HAS_I410TOAR30ROW_SSSE3 #define HAS_I410TOARGBROW_SSSE3 -#if defined(__x86_64__) #define HAS_I422TOAR30ROW_SSSE3 -#endif #define HAS_MERGEARGBROW_SSE2 -#define HAS_MERGERGBROW_SSSE3 #define HAS_MERGEXRGBROW_SSE2 #define HAS_MIRRORUVROW_SSSE3 #define HAS_NV21TOYUV24ROW_SSSE3 @@ -236,7 +193,6 @@ extern "C" { #define HAS_RGB565TOARGBROW_AVX2 #define HAS_ARGB1555TOARGBROW_AVX2 #define HAS_ARGB4444TOARGBROW_AVX2 -#define HAS_RAWTORGBAROW_SSSE3 #define HAS_SPLITARGBROW_SSE2 #define HAS_SPLITARGBROW_SSSE3 #define HAS_SPLITRGBROW_SSE41 @@ -252,14 +208,9 @@ extern "C" { #define HAS_RGBTOUVMATRIXROW_SSSE3 #define HAS_ARGBTOUV444MATRIXROW_SSSE3 #define HAS_RGBTOUV444MATRIXROW_SSSE3 - -#if defined(__x86_64__) || !defined(__pic__) -// TODO(fbarchard): fix build error on android_full_debug=1 -// https://code.google.com/p/libyuv/issues/detail?id=517 #define HAS_I210ALPHATOARGBROW_SSSE3 #define HAS_I410ALPHATOARGBROW_SSSE3 #endif -#endif // The following are available for AVX2 gcc/clang x86 platforms: // TODO(fbarchard): Port to Visual C @@ -269,6 +220,9 @@ extern "C" { !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_AB64TOARGBROW_AVX2 #define HAS_ABGRTOAR30ROW_AVX2 +#define HAS_AR30TOAB30ROW_AVX2 +#define HAS_AR30TOABGRROW_AVX2 +#define HAS_AR30TOARGBROW_AVX2 #define HAS_AR64TOARGBROW_AVX2 #define HAS_ARGBATTENUATEROW_AVX2 #define HAS_ARGBTOAB64ROW_AVX2 @@ -291,15 +245,58 @@ extern "C" { #define HAS_RGB565TOYMATRIXROW_AVX2 #define HAS_ARGB1555TOYMATRIXROW_AVX2 #define HAS_ARGB4444TOYMATRIXROW_AVX2 +#define HAS_ARGB1555TOUVROW_AVX2 +#define HAS_ARGB1555TOYROW_AVX2 +#define HAS_ARGB4444TOUVROW_AVX2 +#define HAS_ARGB4444TOYROW_AVX2 +#define HAS_ARGBAFFINEROW_AVX2 +#define HAS_ARGBCOLORMATRIXROW_AVX2 +#define HAS_ARGBCOLORTABLEROW_AVX2 +#define HAS_ARGBLUMACOLORTABLEROW_AVX2 +#define HAS_ARGBQUANTIZEROW_AVX2 +#define HAS_ARGBSEPIAROW_AVX2 +#define HAS_ARGBSETROW_AVX2 +#define HAS_ARGBSHADEROW_AVX2 #define HAS_ARGBUNATTENUATEROW_AVX2 +#define HAS_ARGBBLENDROW_AVX2 +#define HAS_ARGBGRAYROW_AVX2 +#define HAS_AYUVTOUVROW_AVX2 +#define HAS_AYUVTOVUROW_AVX2 +#define HAS_AYUVTOYROW_AVX2 +#define HAS_BYTETOFLOATROW_AVX2 +#define HAS_COMPUTECUMULATIVESUMROW_AVX2 #define HAS_CONVERT16TO8ROW_AVX2 #define HAS_CONVERT8TO16ROW_AVX2 #define HAS_CONVERT8TO8ROW_AVX2 -#define HAS_DETILEROW_16_AVX +#define HAS_COPYROW_16_AVX2 +#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 +#define HAS_DETILEROW_16_AVX2 +#define HAS_DETILEROW_AVX2 +#define HAS_DETILESPLITUVROW_AVX2 +#define HAS_DETILETOYUY2_AVX2 #define HAS_DIVIDEROW_16_AVX2 +#define HAS_GAUSSCOL_AVX2 +#define HAS_GAUSSCOL_F32_AVX2 +#define HAS_GAUSSROW_AVX2 +#define HAS_GAUSSROW_F32_AVX2 +#define HAS_RGB565TOUVROW_AVX2 +#define HAS_RGB565TOYROW_AVX2 +#define HAS_RGBCOLORTABLEROW_AVX2 +#define HAS_SCALEMAXSAMPLES_AVX2 +#define HAS_SCALESAMPLES_AVX2 +#define HAS_SCALESUMSAMPLES_AVX2 +#define HAS_SETROW_AVX2 +#define HAS_SOBELROW_AVX2 +#define HAS_SOBELTOPLANEROW_AVX2 +#define HAS_SOBELXROW_AVX2 +#define HAS_SOBELXYROW_AVX2 +#define HAS_SOBELYROW_AVX2 #define HAS_HALFMERGEUVROW_AVX2 +#define HAS_HALFROW_16_AVX2 #define HAS_HALFROW_16TO8_AVX2 +#define HAS_HALFROW_AVX2 #define HAS_HALFWIDTHROW_16TO8_AVX2 +#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2 #define HAS_I210TOAR30ROW_AVX2 #define HAS_I210TOARGBROW_AVX2 #define HAS_I212TOAR30ROW_AVX2 @@ -307,17 +304,15 @@ extern "C" { #define HAS_I400TOARGBROW_AVX2 #define HAS_I410TOAR30ROW_AVX2 #define HAS_I410TOARGBROW_AVX2 -#if defined(__x86_64__) -#define HAS_COMPUTECUMULATIVESUMROW_AVX2 -#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 #define HAS_I422TOAR30ROW_AVX2 -#endif #define HAS_RAWTORGB24ROW_AVX2 +#define HAS_RAWTORGBAROW_AVX2 #define HAS_I422TOUYVYROW_AVX2 #define HAS_I422TOYUY2ROW_AVX2 #define HAS_MERGEAR64ROW_AVX2 #define HAS_MERGEARGB16TO8ROW_AVX2 #define HAS_MERGEARGBROW_AVX2 +#define HAS_MERGERGBROW_AVX2 #define HAS_MERGEUVROW_16_AVX2 #define HAS_MERGEXR30ROW_AVX2 #define HAS_MERGEXR64ROW_AVX2 @@ -335,15 +330,11 @@ extern "C" { #define HAS_SPLITUVROW_16_AVX2 #define HAS_SPLITXRGBROW_AVX2 #define HAS_SWAPUVROW_AVX2 +#define HAS_UNPACKMT2T_AVX2 #define HAS_YUY2TONVUVROW_AVX2 - -#if defined(__x86_64__) || !defined(__pic__) -// TODO(fbarchard): fix build error on android_full_debug=1 -// https://code.google.com/p/libyuv/issues/detail?id=517 #define HAS_I210ALPHATOARGBROW_AVX2 #define HAS_I410ALPHATOARGBROW_AVX2 #endif -#endif // This module is for Visual C 32/64 bit #if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_WIN) && \ @@ -369,8 +360,14 @@ extern "C" { #define HAS_I422TOARGB4444ROW_AVX2 #define HAS_NV12TORGB565ROW_AVX2 #define HAS_CONVERT8TO16ROW_AVX2 -#if defined(__x86_64__) || defined(_M_X64) #define HAS_I422TOAR30ROW_AVX2 +#define HAS_AR30TOAB30ROW_AVX2 +#define HAS_AR30TOABGRROW_AVX2 +#define HAS_AR30TOARGBROW_AVX2 +#define HAS_AYUVTOUVROW_AVX2 +#define HAS_AYUVTOVUROW_AVX2 +#define HAS_AYUVTOYROW_AVX2 +#if defined(__x86_64__) || defined(_M_X64) #define HAS_RAWTOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW #define HAS_RAWTORGB24ROW_AVX512BW @@ -421,7 +418,70 @@ extern "C" { #define HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI #define HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI #define HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI +#define HAS_I422TOARGBROW_AVX512BW +#define HAS_ARGB1555TOARGBROW_AVX512BW +#define HAS_ARGBCOLORMATRIXROW_AVX512VNNI +#define HAS_ARGBSEPIAROW_AVX512VNNI +#define HAS_ARGBTORGB565DITHERROW_AVX512BW +#define HAS_ARGBTORGB565ROW_AVX512BW +#define HAS_CONVERT8TO8ROW_AVX512BW +#define HAS_DIVIDEROW_16_AVX512BW +#define HAS_HALFFLOATROW_AVX512BW +#define HAS_I210ALPHATOARGBROW_AVX512BW +#define HAS_I210TOAR30ROW_AVX512BW +#define HAS_I210TOARGBROW_AVX512BW +#define HAS_I212TOAR30ROW_AVX512BW +#define HAS_I212TOARGBROW_AVX512BW +#define HAS_I410ALPHATOARGBROW_AVX512BW +#define HAS_I410TOAR30ROW_AVX512BW +#define HAS_I410TOARGBROW_AVX512BW +#define HAS_I422TOARGB1555ROW_AVX512BW +#define HAS_I422TOARGB4444ROW_AVX512BW +#define HAS_I422TORGB565ROW_AVX512BW +#define HAS_I422TORGBAROW_AVX512BW +#define HAS_I444ALPHATOARGBROW_AVX512BW +#define HAS_I444TOARGBROW_AVX512BW +#define HAS_I444TORGB24ROW_AVX512VBMI +#define HAS_MERGEUVROW_16_AVX512BW +#define HAS_P210TOAR30ROW_AVX512BW +#define HAS_P210TOARGBROW_AVX512BW +#define HAS_P410TOAR30ROW_AVX512BW +#define HAS_P410TOARGBROW_AVX512BW +#define HAS_UYVYTOARGBROW_AVX512BW +#define HAS_UYVYTOUVROW_AVX512BW +#define HAS_UYVYTOYROW_AVX512BW +#define HAS_YUY2TOARGBROW_AVX512BW +#define HAS_YUY2TOUVROW_AVX512BW +#define HAS_YUY2TOYROW_AVX512BW #endif +#define HAS_ARGB1555TOUVROW_AVX2 +#define HAS_ARGB1555TOYROW_AVX2 +#define HAS_ARGB4444TOUVROW_AVX2 +#define HAS_ARGB4444TOYROW_AVX2 +#define HAS_ARGBAFFINEROW_AVX2 +#define HAS_ARGBCOLORMATRIXROW_AVX2 +#define HAS_ARGBCOLORTABLEROW_AVX2 +#define HAS_ARGBLUMACOLORTABLEROW_AVX2 +#define HAS_ARGBQUANTIZEROW_AVX2 +#define HAS_ARGBSEPIAROW_AVX2 +#define HAS_ARGBSETROW_AVX2 +#define HAS_ARGBSHADEROW_AVX2 +#define HAS_BYTETOFLOATROW_AVX2 +#define HAS_COPYROW_16_AVX2 +#define HAS_DETILEROW_16_AVX2 +#define HAS_DETILEROW_AVX2 +#define HAS_DETILESPLITUVROW_AVX2 +#define HAS_GAUSSROW_AVX2 +#define HAS_GAUSSROW_F32_AVX2 +#define HAS_RGB565TOUVROW_AVX2 +#define HAS_RGB565TOYROW_AVX2 +#define HAS_RGBCOLORTABLEROW_AVX2 +#define HAS_SETROW_AVX2 +#define HAS_SOBELROW_AVX2 +#define HAS_SOBELTOPLANEROW_AVX2 +#define HAS_SOBELXROW_AVX2 +#define HAS_SOBELXYROW_AVX2 +#define HAS_SOBELYROW_AVX2 #define HAS_ARGBALPHABLENDROW_SSSE3 #define HAS_ARGBORROW_SSE2 #define HAS_ARGBALPHABLENDROW_AVX2 @@ -448,16 +508,91 @@ extern "C" { #define HAS_INTERPOLATEROW_16_AVX2 #define HAS_MULTIPLYROW_16_AVX2 #define HAS_SWAPUVROW_AVX2 +#define HAS_AB64TOARGBROW_AVX2 +#define HAS_AR64TOARGBROW_AVX2 +#define HAS_ARGBADDROW_AVX2 +#define HAS_ARGBBLENDROW_AVX2 +#define HAS_ARGBCOPYALPHAROW_AVX2 +#define HAS_ARGBGRAYROW_AVX2 +#define HAS_ARGBPOLYNOMIALROW_AVX2 +#define HAS_ARGBSUBTRACTROW_AVX2 +#define HAS_ARGBTOAB64ROW_AVX2 +#define HAS_ARGBTOAR64ROW_AVX2 +#define HAS_ARGBTORAWROW_AVX2 +#define HAS_ARGBTORGB24ROW_AVX2 +#define HAS_ARGBUNATTENUATEROW_AVX2 +#define HAS_COMPUTECUMULATIVESUMROW_AVX2 +#define HAS_CONVERT16TO8ROW_AVX2 +#define HAS_COPYROW_AVX +#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 +#define HAS_DETILETOYUY2_AVX2 +#define HAS_GAUSSCOL_AVX2 +#define HAS_GAUSSCOL_F32_AVX2 +#define HAS_HALFROW_16_AVX2 +#define HAS_HALFROW_16TO8_AVX2 +#define HAS_HALFROW_AVX2 +#define HAS_HALFWIDTHROW_16TO8_AVX2 +#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2 +#define HAS_I422TOUYVYROW_AVX2 +#define HAS_I422TOYUY2ROW_AVX2 +#define HAS_I444TOARGBROW_AVX2 +#define HAS_I444TORGB24ROW_AVX2 +#define HAS_MERGEAR64ROW_AVX2 +#define HAS_MERGEARGB16TO8ROW_AVX2 +#define HAS_MERGEARGBROW_AVX2 +#define HAS_MERGERGBROW_AVX2 +#define HAS_MERGEUVROW_16_AVX2 +#define HAS_MERGEXR30ROW_AVX2 +#define HAS_MERGEXR64ROW_AVX2 +#define HAS_MERGEXRGB16TO8ROW_AVX2 +#define HAS_MERGEXRGBROW_AVX2 +#define HAS_NV12TORGB24ROW_AVX2 +#define HAS_NV21TORGB24ROW_AVX2 +#define HAS_NV21TOYUV24ROW_AVX2 +#define HAS_RAWTORGBAROW_AVX2 +#define HAS_SCALEMAXSAMPLES_AVX2 +#define HAS_SCALESAMPLES_AVX2 +#define HAS_SCALESUMSAMPLES_AVX2 +#define HAS_SPLITARGBROW_AVX2 +#define HAS_SPLITRGBROW_AVX2 +#define HAS_SPLITUVROW_16_AVX2 +#define HAS_SPLITUVROW_AVX2 +#define HAS_SPLITXRGBROW_AVX2 +#define HAS_UNPACKMT2T_AVX2 +#define HAS_UYVYTOUV422ROW_AVX2 +#define HAS_UYVYTOUVROW_AVX2 +#define HAS_UYVYTOYROW_AVX2 +#define HAS_YUY2TONVUVROW_AVX2 +#define HAS_YUY2TOUV422ROW_AVX2 +#define HAS_YUY2TOUVROW_AVX2 +#define HAS_YUY2TOYROW_AVX2 #if defined(__x86_64__) || defined(_M_X64) +#define HAS_ARGBTORAWROW_AVX512BW +#define HAS_ARGBTORGB24ROW_AVX512BW +#define HAS_ARGBTOUV444MATRIXROW_AVX512BW +#define HAS_ARGBTOUVMATRIXROW_AVX512BW +#define HAS_ARGBTOYMATRIXROW_AVX512BW +#define HAS_CONVERT16TO8ROW_AVX512BW +#define HAS_CONVERT8TO16ROW_AVX512BW +#define HAS_COPYROW_AVX512BW +#define HAS_HALFROW_16TO8_AVX512BW +#define HAS_HALFWIDTHROW_16TO8_AVX512BW +#define HAS_J400TOARGBROW_AVX512BW +#define HAS_MERGEUVROW_AVX512BW +#define HAS_MIRRORROW_AVX512BW #define HAS_MIRRORUVROW_AVX512BW +#define HAS_MIRRORSPLITUVROW_AVX512BW +#define HAS_MULTIPLYROW_16_AVX512BW +#define HAS_RGBTOUV444MATRIXROW_AVX512BW +#define HAS_RGBTOUVMATRIXROW_AVX512BW +#define HAS_SPLITUVROW_AVX512BW #endif #endif // The following are available for AVX512 clang x64 platforms: // TODO(fbarchard): Port to GCC and Visual C -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \ - !defined(LIBYUV_ENABLE_ROWWIN) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) #define HAS_COPYROW_AVX512BW #define HAS_RAWTOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW @@ -534,6 +669,40 @@ extern "C" { #define HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI #define HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI #define HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI +#define HAS_ARGB1555TOARGBROW_AVX512BW +#define HAS_ARGBCOLORMATRIXROW_AVX512VNNI +#define HAS_ARGBSEPIAROW_AVX512VNNI +#define HAS_ARGBTORGB565DITHERROW_AVX512BW +#define HAS_ARGBTORGB565ROW_AVX512BW +#define HAS_CONVERT8TO8ROW_AVX512BW +#define HAS_DIVIDEROW_16_AVX512BW +#define HAS_HALFFLOATROW_AVX512BW +#define HAS_I210ALPHATOARGBROW_AVX512BW +#define HAS_I210TOAR30ROW_AVX512BW +#define HAS_I210TOARGBROW_AVX512BW +#define HAS_I212TOAR30ROW_AVX512BW +#define HAS_I212TOARGBROW_AVX512BW +#define HAS_I410ALPHATOARGBROW_AVX512BW +#define HAS_I410TOAR30ROW_AVX512BW +#define HAS_I410TOARGBROW_AVX512BW +#define HAS_I422TOARGB1555ROW_AVX512BW +#define HAS_I422TOARGB4444ROW_AVX512BW +#define HAS_I422TORGB565ROW_AVX512BW +#define HAS_I422TORGBAROW_AVX512BW +#define HAS_I444ALPHATOARGBROW_AVX512BW +#define HAS_I444TOARGBROW_AVX512BW +#define HAS_I444TORGB24ROW_AVX512VBMI +#define HAS_MERGEUVROW_16_AVX512BW +#define HAS_P210TOAR30ROW_AVX512BW +#define HAS_P210TOARGBROW_AVX512BW +#define HAS_P410TOAR30ROW_AVX512BW +#define HAS_P410TOARGBROW_AVX512BW +#define HAS_UYVYTOARGBROW_AVX512BW +#define HAS_UYVYTOUVROW_AVX512BW +#define HAS_UYVYTOYROW_AVX512BW +#define HAS_YUY2TOARGBROW_AVX512BW +#define HAS_YUY2TOUVROW_AVX512BW +#define HAS_YUY2TOYROW_AVX512BW #endif // The following are available on Neon platforms: @@ -1179,6 +1348,7 @@ extern "C" { #define HAS_YUY2TOYROW_HVX #endif +#undef SIMD_ALIGNED #if defined(_MSC_VER) && !defined(__CLR_VER) && !defined(__clang__) #if defined(VISUALC_HAS_AVX2) #define SIMD_ALIGNED(var) __declspec(align(32)) var @@ -2687,9 +2857,24 @@ void RGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_rgb, int width, const struct ArgbConstants* c); +// Deprecated. void RGB565ToYRow_C(const uint8_t* src_rgb565, uint8_t* dst_y, int width); void ARGB1555ToYRow_C(const uint8_t* src_argb1555, uint8_t* dst_y, int width); void ARGB4444ToYRow_C(const uint8_t* src_argb4444, uint8_t* dst_y, int width); +void RGB565ToYRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_y, int width); +void ARGB1555ToYRow_AVX2(const uint8_t* src_argb1555, + uint8_t* dst_y, + int width); +void ARGB4444ToYRow_AVX2(const uint8_t* src_argb4444, + uint8_t* dst_y, + int width); +void RGB565ToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGB1555ToYRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGB4444ToYRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void RGB565ToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void ARGB1555ToYRow_Any_NEON(const uint8_t* src_ptr, @@ -2775,6 +2960,7 @@ void RAWToUVRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, int width); +// Deprecated. void RGB565ToUVRow_C(const uint8_t* src_rgb565, int src_stride_rgb565, uint8_t* dst_u, @@ -2790,6 +2976,36 @@ void ARGB4444ToUVRow_C(const uint8_t* src_argb4444, uint8_t* dst_u, uint8_t* dst_v, int width); +void RGB565ToUVRow_AVX2(const uint8_t* src_rgb565, + int src_stride_rgb565, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void ARGB1555ToUVRow_AVX2(const uint8_t* src_argb1555, + int src_stride_argb1555, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void ARGB4444ToUVRow_AVX2(const uint8_t* src_argb4444, + int src_stride_argb4444, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void RGB565ToUVRow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void ARGB1555ToUVRow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void ARGB4444ToUVRow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width); @@ -2798,9 +3014,6 @@ void MirrorRow_NEON(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_C(const uint8_t* src, uint8_t* dst, int width); -void MirrorRow_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); void MirrorRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -2814,9 +3027,6 @@ void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_C(const uint8_t* src_uv, uint8_t* dst_uv, int width); -void MirrorUVRow_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); void MirrorUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -2973,11 +3183,11 @@ void DetileRow_Any_NEON(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, int width); -void DetileRow_SSE2(const uint8_t* src, +void DetileRow_AVX2(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, int width); -void DetileRow_Any_SSE2(const uint8_t* src, +void DetileRow_Any_AVX2(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, int width); @@ -2993,37 +3203,29 @@ void DetileRow_16_Any_NEON(const uint16_t* src, ptrdiff_t src_tile_stride, uint16_t* dst, int width); -void DetileRow_16_SSE2(const uint16_t* src, +void DetileRow_16_AVX2(const uint16_t* src, ptrdiff_t src_tile_stride, uint16_t* dst, int width); -void DetileRow_16_Any_SSE2(const uint16_t* src, +void DetileRow_16_Any_AVX2(const uint16_t* src, ptrdiff_t src_tile_stride, uint16_t* dst, int width); -void DetileRow_16_AVX(const uint16_t* src, - ptrdiff_t src_tile_stride, - uint16_t* dst, - int width); -void DetileRow_16_Any_AVX(const uint16_t* src, - ptrdiff_t src_tile_stride, - uint16_t* dst, - int width); void DetileSplitUVRow_C(const uint8_t* src_uv, ptrdiff_t src_tile_stride, uint8_t* dst_u, uint8_t* dst_v, int width); -void DetileSplitUVRow_SSSE3(const uint8_t* src_uv, - ptrdiff_t src_tile_stride, - uint8_t* dst_u, - uint8_t* dst_v, - int width); -void DetileSplitUVRow_Any_SSSE3(const uint8_t* src_uv, - ptrdiff_t src_tile_stride, - uint8_t* dst_u, - uint8_t* dst_v, - int width); +void DetileSplitUVRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void DetileSplitUVRow_Any_AVX2(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void DetileSplitUVRow_NEON(const uint8_t* src_uv, ptrdiff_t src_tile_stride, uint8_t* dst_u, @@ -3040,13 +3242,13 @@ void DetileToYUY2_C(const uint8_t* src_y, ptrdiff_t src_uv_tile_stride, uint8_t* dst_yuy2, int width); -void DetileToYUY2_SSE2(const uint8_t* src_y, +void DetileToYUY2_AVX2(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, uint8_t* dst_yuy2, int width); -void DetileToYUY2_Any_SSE2(const uint8_t* src_y, +void DetileToYUY2_Any_AVX2(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, @@ -3065,6 +3267,7 @@ void DetileToYUY2_Any_NEON(const uint8_t* src_y, uint8_t* dst_yuy2, int width); void UnpackMT2T_C(const uint8_t* src, uint16_t* dst, size_t size); +void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size); void UnpackMT2T_NEON(const uint8_t* src, uint16_t* dst, size_t size); void MergeUVRow_C(const uint8_t* src_u, const uint8_t* src_v, @@ -3228,11 +3431,11 @@ void MergeRGBRow_C(const uint8_t* src_r, const uint8_t* src_b, uint8_t* dst_rgb, int width); -void MergeRGBRow_SSSE3(const uint8_t* src_r, - const uint8_t* src_g, - const uint8_t* src_b, - uint8_t* dst_rgb, - int width); +void MergeRGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width); void MergeRGBRow_NEON(const uint8_t* src_r, const uint8_t* src_g, const uint8_t* src_b, @@ -3248,11 +3451,11 @@ void MergeRGBRow_AVX512VBMI(const uint8_t* src_r, const uint8_t* src_b, uint8_t* dst_rgb, int width); -void MergeRGBRow_Any_SSSE3(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - int width); +void MergeRGBRow_Any_AVX2(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + int width); void MergeRGBRow_Any_NEON(const uint8_t* src_r, const uint8_t* src_g, const uint8_t* src_b, @@ -3646,6 +3849,11 @@ void MergeUVRow_16_AVX2(const uint16_t* src_u, uint16_t* dst_uv, int depth, int width); +void MergeUVRow_16_AVX512BW(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width); void MergeUVRow_16_Any_AVX2(const uint16_t* src_u, const uint16_t* src_v, uint16_t* dst_uv, @@ -3738,6 +3946,10 @@ void DivideRow_16_AVX2(const uint16_t* src_y, uint16_t* dst_y, int scale, int width); +void DivideRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); void DivideRow_16_Any_AVX2(const uint16_t* src_ptr, uint16_t* dst_ptr, int scale, @@ -3849,6 +4061,32 @@ void Convert16To8Row_RVV(const uint16_t* src_y, int scale, int width); +void HalfRow_C(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); +void HalfRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); +void HalfRow_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int width); + +void HalfRow_16_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); +void HalfRow_16_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); +void HalfRow_16_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int width); + void HalfRow_16To8_C(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, @@ -3920,6 +4158,11 @@ void HalfWidthRow_16To8_Odd_C(const uint16_t* src_uv, uint8_t* dst_uv, int scale, int width); +void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, @@ -4006,6 +4249,11 @@ void Convert8To8Row_AVX2(const uint8_t* src_y, int scale, int bias, int width); +void Convert8To8Row_AVX512BW(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width); void Convert8To8Row_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int scale, @@ -4030,6 +4278,8 @@ void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count); +void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int count); +void CopyRow_16_Any_AVX2(const uint16_t* src_ptr, uint16_t* dst_ptr, int width); void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count); void CopyRow_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int width); @@ -4112,20 +4362,21 @@ void ARGBCopyYToAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr, int width); void SetRow_C(uint8_t* dst, uint8_t v8, int width); -void SetRow_X86(uint8_t* dst, uint8_t v8, int width); void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width); +void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width); void SetRow_NEON(uint8_t* dst, uint8_t v8, int width); void SetRow_LSX(uint8_t* dst, uint8_t v8, int width); void SetRow_LASX(uint8_t* dst, uint8_t v8, int width); -void SetRow_Any_X86(uint8_t* dst_ptr, uint8_t v32, int width); +void SetRow_Any_AVX2(uint8_t* dst_ptr, uint8_t v32, int width); void SetRow_Any_NEON(uint8_t* dst_ptr, uint8_t v32, int width); void SetRow_Any_LSX(uint8_t* dst_ptr, uint8_t v32, int width); void SetRow_Any_LASX(uint8_t* dst_ptr, uint8_t v32, int width); void ARGBSetRow_C(uint8_t* dst_argb, uint32_t v32, int width); -void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width); +void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width); void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width); +void ARGBSetRow_Any_AVX2(uint8_t* dst_ptr, uint32_t v32, int width); void ARGBSetRow_Any_NEON(uint8_t* dst_ptr, uint32_t v32, int width); void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width); @@ -4197,7 +4448,7 @@ void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24, void RAWToARGBRow_AVX512BW(const uint8_t* src_raw, uint8_t* dst_argb, int width); -void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width); +void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, @@ -4207,12 +4458,16 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); +// Deprecated. void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_argb, int width); void ARGB1555ToARGBRow_AVX2(const uint8_t* src_argb1555, uint8_t* dst_argb, int width); +void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width); void ARGB4444ToARGBRow_AVX2(const uint8_t* src_argb4444, uint8_t* dst_argb, int width); @@ -4289,21 +4544,33 @@ void ARGB4444ToARGBRow_C(const uint8_t* src_argb4444, uint8_t* dst_argb, int width); void AR30ToARGBRow_C(const uint8_t* src_ar30, uint8_t* dst_argb, int width); +void AR30ToARGBRow_AVX2(const uint8_t* src_ar30, uint8_t* dst_argb, int width); void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_argb, int width); void AR30ToABGRRow_C(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); +void AR30ToABGRRow_AVX2(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); void ARGBToAR30Row_C(const uint8_t* src_argb, uint8_t* dst_ar30, int width); void AR30ToAB30Row_C(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); +void AR30ToAB30Row_AVX2(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width); void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); +void AR30ToARGBRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToABGRRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToAB30Row_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void AR30ToARGBRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4327,9 +4594,9 @@ void RGB24ToARGBRow_Any_AVX2(const uint8_t* src_ptr, void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, uint8_t* dst_argb, int width); -void RAWToRGBARow_Any_SSSE3(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); +void RAWToRGBARow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void RAWToRGB24Row_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4417,6 +4684,7 @@ void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width); void ARGBToRAWRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width); void ARGBToRGB24Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width); +// Deprecated. void ARGBToRGB565DitherRow_C(const uint8_t* src_argb, uint8_t* dst_rgb, uint32_t dither4, @@ -4426,8 +4694,15 @@ void ARGBToRGB565DitherRow_AVX2(const uint8_t* src, uint8_t* dst, uint32_t dither4, int width); +void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width); void ARGBToRGB565Row_AVX2(const uint8_t* src_argb, uint8_t* dst_rgb, int width); +void ARGBToRGB565Row_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width); void ARGBToARGB1555Row_AVX2(const uint8_t* src_argb, uint8_t* dst_rgb, int width); @@ -4806,24 +5081,49 @@ void I422ToRGBARow_AVX2(const uint8_t* y_buf, uint8_t* dst_rgba, const struct YuvConstants* yuvconstants, int width); +void I422ToRGBARow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgba, + const struct YuvConstants* yuvconstants, + int width); +// Deprecated. void I422ToRGB565Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_rgb565, const struct YuvConstants* yuvconstants, int width); +void I422ToRGB565Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb565, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGB1555Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb1555, const struct YuvConstants* yuvconstants, int width); +void I422ToARGB1555Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb1555, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGB4444Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb4444, const struct YuvConstants* yuvconstants, int width); +void I422ToARGB4444Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb4444, + const struct YuvConstants* yuvconstants, + int width); void I422ToRGB565Row_SSSE3(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -4854,6 +5154,12 @@ void I444ToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I444ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I444ToRGB24Row_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -4866,6 +5172,12 @@ void I444ToRGB24Row_AVX2(const uint8_t* y_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width); +void I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGBRow_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -4941,36 +5253,72 @@ void I210ToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I210ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void I210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void I212ToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I212ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I212ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void I212ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void I410ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void I410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void I410ToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I210AlphaToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -4978,6 +5326,13 @@ void I210AlphaToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I410AlphaToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -4985,6 +5340,13 @@ void I410AlphaToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I444AlphaToARGBRow_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -4999,6 +5361,13 @@ void I444AlphaToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -5112,10 +5481,18 @@ void YUY2ToARGBRow_AVX2(const uint8_t* yuy2_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void YUY2ToARGBRow_AVX512BW(const uint8_t* yuy2_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void UYVYToARGBRow_AVX2(const uint8_t* uyvy_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void UYVYToARGBRow_AVX512BW(const uint8_t* uyvy_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void P210ToARGBRow_SSSE3(const uint16_t* y_buf, const uint16_t* uv_buf, @@ -5142,21 +5519,41 @@ void P210ToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void P210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void P410ToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void P410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void P210ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void P210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void P410ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void P410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void I422ToRGBARow_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, @@ -5604,10 +6001,10 @@ void I400ToARGBRow_Any_LASX(const uint8_t* src_ptr, int width); // ARGB preattenuated alpha blend. -void ARGBBlendRow_SSSE3(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width); +void ARGBBlendRow_AVX2(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); void ARGBBlendRow_AVX512BW(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -6498,6 +6895,7 @@ void YUY2ToUV422Row_SSE2(const uint8_t* src_yuy2, int width); void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width); void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width); +void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width); void YUY2ToUVRow_NEON(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_u, @@ -6508,12 +6906,23 @@ void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_u, uint8_t* dst_v, int width); +void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width); +void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width); void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, int stride_uyvy, uint8_t* dst_u, uint8_t* dst_v, int width); +void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_uv, @@ -6749,11 +7158,16 @@ void SwapUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void SwapUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_vu, int width); void SwapUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_vu, int width); void AYUVToYRow_C(const uint8_t* src_ayuv, uint8_t* dst_y, int width); +void AYUVToYRow_AVX2(const uint8_t* src_ayuv, uint8_t* dst_y, int width); void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width); void AYUVToUVRow_C(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_uv, int width); +void AYUVToUVRow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); void AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_uv, @@ -6762,6 +7176,10 @@ void AYUVToVURow_C(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, int width); +void AYUVToVURow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); void AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, @@ -6783,6 +7201,15 @@ void AYUVToVURow_SVE2(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, int width); +void AYUVToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void AYUVToUVRow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_vu, + int width); +void AYUVToVURow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_vu, + int width); void AYUVToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void AYUVToUVRow_Any_NEON(const uint8_t* src_ptr, int src_stride, @@ -6976,7 +7403,7 @@ void ARGBUnattenuateRow_Any_NEON(const uint8_t* src_ptr, int width); void ARGBGrayRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width); -void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width); +void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, int width); @@ -6991,7 +7418,8 @@ void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBSepiaRow_C(uint8_t* dst_argb, int width); -void ARGBSepiaRow_SSSE3(uint8_t* dst_argb, int width); +void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width); +void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width); void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width); void ARGBSepiaRow_NEON_DotProd(uint8_t* dst_argb, int width); void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width); @@ -7001,10 +7429,14 @@ void ARGBColorMatrixRow_C(const uint8_t* src_argb, uint8_t* dst_argb, const int8_t* matrix_argb, int width); -void ARGBColorMatrixRow_SSSE3(const uint8_t* src_argb, - uint8_t* dst_argb, - const int8_t* matrix_argb, - int width); +void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width); +void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width); void ARGBColorMatrixRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, const int8_t* matrix_argb, @@ -7025,9 +7457,9 @@ void ARGBColorMatrixRow_LASX(const uint8_t* src_argb, void ARGBColorTableRow_C(uint8_t* dst_argb, const uint8_t* table_argb, int width); -void ARGBColorTableRow_X86(uint8_t* dst_argb, - const uint8_t* table_argb, - int width); +void ARGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width); void ARGBColorTableRow_NEON(uint8_t* dst_argb, const uint8_t* table_argb, int width); @@ -7035,9 +7467,9 @@ void ARGBColorTableRow_NEON(uint8_t* dst_argb, void RGBColorTableRow_C(uint8_t* dst_argb, const uint8_t* table_argb, int width); -void RGBColorTableRow_X86(uint8_t* dst_argb, - const uint8_t* table_argb, - int width); +void RGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width); void RGBColorTableRow_NEON(uint8_t* dst_argb, const uint8_t* table_argb, int width); @@ -7047,7 +7479,7 @@ void ARGBQuantizeRow_C(uint8_t* dst_argb, int interval_size, int interval_offset, int width); -void ARGBQuantizeRow_SSE2(uint8_t* dst_argb, +void ARGBQuantizeRow_AVX2(uint8_t* dst_argb, int scale, int interval_size, int interval_offset, @@ -7072,7 +7504,7 @@ void ARGBShadeRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width, uint32_t value); -void ARGBShadeRow_SSE2(const uint8_t* src_argb, +void ARGBShadeRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width, uint32_t value); @@ -7090,16 +7522,6 @@ void ARGBShadeRow_LASX(const uint8_t* src_argb, uint32_t value); // Used for blur. -void CumulativeSumToAverageRow_SSE2(const int32_t* topleft, - const int32_t* botleft, - int width, - int area, - uint8_t* dst, - int count); -void ComputeCumulativeSumRow_SSE2(const uint8_t* row, - int32_t* cumsum, - const int32_t* previous_cumsum, - int width); void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, const int32_t* botleft, int width, @@ -7139,7 +7561,7 @@ void ARGBAffineRow_C(const uint8_t* src_argb, const float* uv_dudv, int width); LIBYUV_API -void ARGBAffineRow_SSE2(const uint8_t* src_argb, +void ARGBAffineRow_AVX2(const uint8_t* src_argb, int src_argb_stride, uint8_t* dst_argb, const float* src_dudv, @@ -7260,7 +7682,7 @@ void SobelXRow_C(const uint8_t* src_y0, const uint8_t* src_y2, uint8_t* dst_sobelx, int width); -void SobelXRow_SSE2(const uint8_t* src_y0, +void SobelXRow_AVX2(const uint8_t* src_y0, const uint8_t* src_y1, const uint8_t* src_y2, uint8_t* dst_sobelx, @@ -7274,7 +7696,7 @@ void SobelYRow_C(const uint8_t* src_y0, const uint8_t* src_y1, uint8_t* dst_sobely, int width); -void SobelYRow_SSE2(const uint8_t* src_y0, +void SobelYRow_AVX2(const uint8_t* src_y0, const uint8_t* src_y1, uint8_t* dst_sobely, int width); @@ -7286,7 +7708,7 @@ void SobelRow_C(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); -void SobelRow_SSE2(const uint8_t* src_sobelx, +void SobelRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); @@ -7306,7 +7728,7 @@ void SobelToPlaneRow_C(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, int width); -void SobelToPlaneRow_SSE2(const uint8_t* src_sobelx, +void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, int width); @@ -7326,7 +7748,7 @@ void SobelXYRow_C(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); -void SobelXYRow_SSE2(const uint8_t* src_sobelx, +void SobelXYRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); @@ -7342,7 +7764,7 @@ void SobelXYRow_LASX(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); -void SobelRow_Any_SSE2(const uint8_t* y_buf, +void SobelRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); @@ -7358,7 +7780,7 @@ void SobelRow_Any_LASX(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); -void SobelToPlaneRow_Any_SSE2(const uint8_t* y_buf, +void SobelToPlaneRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); @@ -7374,7 +7796,7 @@ void SobelToPlaneRow_Any_LASX(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); -void SobelXYRow_Any_SSE2(const uint8_t* y_buf, +void SobelXYRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); @@ -7418,6 +7840,10 @@ void HalfFloatRow_AVX2(const uint16_t* src, uint16_t* dst, float scale, int width); +void HalfFloatRow_AVX512BW(const uint16_t* src, + uint16_t* dst, + float scale, + int width); void HalfFloatRow_Any_AVX2(const uint16_t* src_ptr, uint16_t* dst_ptr, float param, @@ -7471,6 +7897,14 @@ void HalfFloatRow_Any_LASX(const uint16_t* src_ptr, float param, int width); void ByteToFloatRow_C(const uint8_t* src, float* dst, float scale, int width); +void ByteToFloatRow_AVX2(const uint8_t* src, + float* dst, + float scale, + int width); +void ByteToFloatRow_Any_AVX2(const uint8_t* src_ptr, + float* dst_ptr, + float param, + int width); void ByteToFloatRow_NEON(const uint8_t* src, float* dst, float scale, @@ -7497,11 +7931,11 @@ void ARGBLumaColorTableRow_C(const uint8_t* src_argb, int width, const uint8_t* luma, uint32_t lumacoeff); -void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb, - uint8_t* dst_argb, - int width, - const uint8_t* luma, - uint32_t lumacoeff); +void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff); void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width, @@ -7509,21 +7943,39 @@ void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb, uint32_t lumacoeff); float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width); +float ScaleMaxSamples_AVX2(const float* src, + float* dst, + float scale, + int width); float ScaleMaxSamples_NEON(const float* src, float* dst, float scale, int width); float ScaleSumSamples_C(const float* src, float* dst, float scale, int width); +float ScaleSumSamples_AVX2(const float* src, + float* dst, + float scale, + int width); float ScaleSumSamples_NEON(const float* src, float* dst, float scale, int width); void ScaleSamples_C(const float* src, float* dst, float scale, int width); +void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width); void ScaleSamples_NEON(const float* src, float* dst, float scale, int width); +void GaussRow_F32_AVX2(const float* src, float* dst, int width); void GaussRow_F32_NEON(const float* src, float* dst, int width); void GaussRow_F32_C(const float* src, float* dst, int width); +void GaussCol_F32_AVX2(const float* src0, + const float* src1, + const float* src2, + const float* src3, + const float* src4, + float* dst, + int width); + void GaussCol_F32_NEON(const float* src0, const float* src1, const float* src2, @@ -7541,6 +7993,7 @@ void GaussCol_F32_C(const float* src0, int width); void GaussRow_C(const uint32_t* src, uint16_t* dst, int width); +void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width); void GaussRow_NEON(const uint32_t* src, uint16_t* dst, int width); void GaussCol_C(const uint16_t* src0, const uint16_t* src1, @@ -7549,6 +8002,13 @@ void GaussCol_C(const uint16_t* src0, const uint16_t* src4, uint32_t* dst, int width); +void GaussCol_AVX2(const uint16_t* src0, + const uint16_t* src1, + const uint16_t* src2, + const uint16_t* src3, + const uint16_t* src4, + uint32_t* dst, + int width); void GaussCol_NEON(const uint16_t* src0, const uint16_t* src1, const uint16_t* src2, diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index e58962e7e..a4d34c515 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -23,31 +23,26 @@ extern "C" { #endif // The following are available on all x86 platforms: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(_M_IX86) || ((defined(__x86_64__) || defined(__i386__)) && \ - !defined(LIBYUV_ENABLE_ROWWIN))) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || defined(_M_X64) || defined(__x86_64__) || \ + defined(__i386__)) #define HAS_FIXEDDIV1_X86 #define HAS_FIXEDDIV_X86 -#define HAS_SCALEARGBCOLS_SSE2 -#define HAS_SCALEARGBCOLSUP2_SSE2 -#define HAS_SCALEARGBFILTERCOLS_SSSE3 -#define HAS_SCALEARGBROWDOWN2_SSE2 -#define HAS_SCALEARGBROWDOWNEVEN_SSE2 -#define HAS_SCALECOLSUP2_SSE2 -#define HAS_SCALEFILTERCOLS_SSSE3 +#define HAS_SCALEADDROW_SSE2 +#endif + +// The following are available on 32-bit Visual C and gcc/clang x86 platforms: +#if !defined(LIBYUV_DISABLE_X86) && !defined(LIBYUV_ENABLE_ROWWIN) && \ + ((defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__)) || \ + defined(__x86_64__) || defined(__i386__)) #define HAS_SCALEROWDOWN2_SSSE3 -#define HAS_SCALEROWDOWN34_SSSE3 #define HAS_SCALEROWDOWN4_SSSE3 #endif // The following are available for gcc/clang x86 platforms: -// TODO(fbarchard): Port to Visual C #if !defined(LIBYUV_DISABLE_X86) && \ (defined(__x86_64__) || defined(__i386__)) && \ !defined(LIBYUV_ENABLE_ROWWIN) -#define HAS_SCALEADDROW_SSE2 -#define HAS_SCALEROWDOWN38_SSSE3 -#define HAS_SCALEUVROWDOWN2BOX_SSSE3 #define HAS_SCALEROWUP2_LINEAR_SSE2 #define HAS_SCALEROWUP2_LINEAR_SSSE3 #define HAS_SCALEROWUP2_BILINEAR_SSE2 @@ -62,66 +57,57 @@ extern "C" { #define HAS_SCALEUVROWUP2_BILINEAR_16_SSE41 #endif -// The following are available for gcc/clang x86 platforms, but -// require clang 3.4 or gcc 4.7. -// TODO(fbarchard): Port to Visual C -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(__i386__)) && \ - (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \ - !defined(LIBYUV_ENABLE_ROWWIN) -#define HAS_SCALEADDROW_AVX2 -#define HAS_SCALEUVADDCOLS3_AVX2 -#define HAS_SCALEUVROWDOWN2BOX_AVX2 -#define HAS_SCALEROWUP2_LINEAR_AVX2 -#define HAS_SCALEROWUP2_BILINEAR_AVX2 -#define HAS_SCALEROWUP2_LINEAR_12_AVX2 -#define HAS_SCALEROWUP2_BILINEAR_12_AVX2 -#define HAS_SCALEROWUP2_LINEAR_16_AVX2 -#define HAS_SCALEROWUP2_BILINEAR_16_AVX2 -#define HAS_SCALEUVROWUP2_LINEAR_AVX2 -#define HAS_SCALEUVROWUP2_BILINEAR_AVX2 -#define HAS_SCALEUVROWUP2_LINEAR_16_AVX2 -#define HAS_SCALEUVROWUP2_BILINEAR_16_AVX2 -#endif - // The following are available on all x86 platforms, but // require VS2012, clang 3.4 or gcc 4.7. -// The code supports NaCL but requires a new compiler and validator. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \ - defined(GCC_HAS_AVX2)) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \ + defined(GCC_HAS_AVX2) || \ + (defined(LIBYUV_ENABLE_ROWWIN) && \ + (defined(__x86_64__) || defined(__i386__) || \ + defined(_M_X64) || defined(_M_IX86)))) +#define HAS_SCALEADDROW_AVX2 +#define HAS_SCALEARGBCOLS_AVX2 +#define HAS_SCALEARGBCOLSUP2_AVX2 +#define HAS_SCALEARGBFILTERCOLS_AVX2 +#define HAS_SCALEARGBROWDOWN2_AVX2 +#define HAS_SCALEARGBROWDOWNEVEN_AVX2 +#define HAS_SCALECOLSUP2_AVX2 +#define HAS_SCALEFILTERCOLS_AVX2 +#define HAS_SCALEROWDOWN2_16_AVX2 #define HAS_SCALEROWDOWN2_AVX2 +#define HAS_SCALEROWDOWN34_AVX2 +#define HAS_SCALEROWDOWN38_AVX2 #define HAS_SCALEROWDOWN4_AVX2 +#define HAS_SCALEROWUP2_BILINEAR_12_AVX2 +#define HAS_SCALEROWUP2_BILINEAR_16_AVX2 +#define HAS_SCALEROWUP2_BILINEAR_AVX2 +#define HAS_SCALEROWUP2_LINEAR_12_AVX2 +#define HAS_SCALEROWUP2_LINEAR_16_AVX2 +#define HAS_SCALEROWUP2_LINEAR_AVX2 +#define HAS_SCALEUVADDCOLS3_AVX2 +#define HAS_SCALEUVROWDOWN2_AVX2 +#define HAS_SCALEUVROWDOWN2BOX_AVX2 +#define HAS_SCALEUVROWDOWN2LINEAR_AVX2 +#define HAS_SCALEUVROWDOWNEVEN_AVX2 +#define HAS_SCALEUVROWUP2_BILINEAR_16_AVX2 +#define HAS_SCALEUVROWUP2_BILINEAR_AVX2 +#define HAS_SCALEUVROWUP2_LINEAR_16_AVX2 +#define HAS_SCALEUVROWUP2_LINEAR_AVX2 #endif -// The following are available for AVX512 clang x64 platforms: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \ - !defined(LIBYUV_ENABLE_ROWWIN) +// The following are available for AVX512 x64 platforms: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(_M_X64)) && \ + (defined(CLANG_HAS_AVX512) || \ + (defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) #define HAS_SCALEADDROW_AVX512BW -#define HAS_SCALEADDCOLS_AVX512BW +#define HAS_SCALEADDCOLS1_AVX512BW +#define HAS_SCALEADDCOLS2_AVX512BW #define HAS_SCALEUVADDCOLS3_AVX512VBMI #define HAS_SCALEROWDOWN2_AVX512BW #endif -// The following are available for Visual C 32/64 bit, and for clang with -// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ - defined(_M_IX86)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)) -#define HAS_SCALEADDROW_SSE2 -#define HAS_SCALEROWDOWN38_SSSE3 -#define HAS_SCALEADDROW_AVX2 -#define HAS_SCALEUVADDCOLS3_AVX2 -#if defined(__x86_64__) || defined(_M_X64) -#define HAS_SCALEADDROW_AVX512BW -#define HAS_SCALEADDCOLS_AVX512BW -#define HAS_SCALEUVADDCOLS3_AVX512VBMI -#endif -#endif - // The following are available on Neon platforms: #if !defined(LIBYUV_DISABLE_NEON) && \ (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) @@ -532,24 +518,46 @@ void ScaleUVRowDown2_C(const uint8_t* src_uv, ptrdiff_t src_stride, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDown2_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowDown2Linear_C(const uint8_t* src_uv, ptrdiff_t src_stride, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDown2Linear_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowDown2Box_C(const uint8_t* src_uv, ptrdiff_t src_stride, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDown2Box_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowDownEven_C(const uint8_t* src_uv, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDownEven_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowDownEvenBox_C(const uint8_t* src_uv, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDownEvenBox_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowUp2_Linear_C(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -589,30 +597,59 @@ void ScaleUVCols_C(uint8_t* dst_uv, int dst_width, int x, int dx); +void ScaleUVCols_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx); void ScaleUVCols64_C(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width, int x32, int dx); +void ScaleUVCols64_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x32, + int dx); void ScaleUVColsUp2_C(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width, int, int); +void ScaleUVColsUp2_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int, + int); void ScaleUVFilterCols_C(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width, int x, int dx); +void ScaleUVFilterCols_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx); void ScaleUVFilterCols64_C(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width, int x32, int dx); +void ScaleUVFilterCols64_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x32, + int dx); void ScaleUVAddCols3_C(int dst_width, int scaleval, const uint16_t* src_uv, uint8_t* dst_uv); +void ScaleUVAddCols3_16_C(int dst_width, + int scaleval, + const uint32_t* src_uv, + uint16_t* dst_uv); // Specialized scalers for x86. void ScaleRowDown2_SSSE3(const uint8_t* src_ptr, @@ -639,6 +676,18 @@ void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); +void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); +void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); +void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -672,30 +721,30 @@ void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width); -void ScaleRowDown34_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); +void ScaleRowDown34_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); void ScaleRowUp2_Linear_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -842,6 +891,18 @@ void ScaleRowDown2Box_Odd_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); +void ScaleRowDown2_16_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); +void ScaleRowDown2Linear_16_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); +void ScaleRowDown2Box_16_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); void ScaleRowDown4_Any_SSSE3(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -859,30 +920,30 @@ void ScaleRowDown4Box_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width); -void ScaleRowDown34_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown34_1_Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown34_0_Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_3_Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_2_Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); +void ScaleRowDown34_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_1_Box_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_0_Box_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_2_Box_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); void ScaleAddRow_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); @@ -908,29 +969,29 @@ void ScaleAddCols2_AVX512BW(int dst_width, const uint16_t* src_ptr, uint8_t* dst_ptr); -void ScaleFilterCols_SSSE3(uint8_t* dst_ptr, - const uint8_t* src_ptr, - int dst_width, - int x, - int dx); -void ScaleColsUp2_SSE2(uint8_t* dst_ptr, +void ScaleFilterCols_AVX2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleColsUp2_AVX2(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, int x, int dx); // ARGB Column functions -void ScaleARGBCols_SSE2(uint8_t* dst_argb, +void ScaleARGBCols_AVX2(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, int dx); -void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx); -void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb, +void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx); +void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, @@ -962,15 +1023,15 @@ void ScaleARGBFilterCols_RVV(uint8_t* dst_argb, int dx); // ARGB Row functions -void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); @@ -1022,15 +1083,15 @@ void ScaleARGBRowDown2Box_LSX(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDown2_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDown2_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); -void ScaleARGBRowDown2Linear_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDown2Linear_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); -void ScaleARGBRowDown2Box_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDown2Box_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); @@ -1058,12 +1119,12 @@ void ScaleARGBRowDown2Box_Any_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); -void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, @@ -1098,12 +1159,12 @@ void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb, int src_stepx, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDownEven_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDownEven_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_ptr, int dst_width); -void ScaleARGBRowDownEvenBox_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDownEvenBox_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_ptr, @@ -1130,10 +1191,14 @@ void ScaleARGBRowDownEvenBox_Any_LSX(const uint8_t* src_ptr, int dst_width); // UV Row functions -void ScaleUVRowDown2Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_uv, - int dst_width); +void ScaleUVRowDown2_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width); +void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width); void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_uv, @@ -1174,10 +1239,14 @@ void ScaleUVRowDown2Box_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width); -void ScaleUVRowDown2Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); +void ScaleUVRowDown2_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowDown2Linear_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); void ScaleUVRowDown2Box_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -1194,6 +1263,11 @@ void ScaleUVRowDown2Box_Any_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); +void ScaleUVRowDownEven_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_uv, + int dst_width); void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, int src_stepx, @@ -1209,6 +1283,11 @@ void ScaleUVRowDownEven_RVV(const uint8_t* src_ptr, int32_t src_stepx, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDownEven_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_ptr, + int dst_width); void ScaleUVRowDownEven_Any_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, int src_stepx, diff --git a/include/libyuv/scale_uv.h b/include/libyuv/scale_uv.h index f6cc8e40f..168c5d949 100644 --- a/include/libyuv/scale_uv.h +++ b/include/libyuv/scale_uv.h @@ -33,7 +33,6 @@ int UVScale(const uint8_t* src_uv, enum FilterMode filtering); // Scale a 16 bit UV image. -// This function is currently incomplete, it can't handle all cases. LIBYUV_API int UVScale_16(const uint16_t* src_uv, int src_stride_uv, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index caf8590aa..5ba507c6d 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1976 +#define LIBYUV_VERSION 1977 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/include/libyuv/video_common.h b/include/libyuv/video_common.h index e60707ae5..ee989f786 100644 --- a/include/libyuv/video_common.h +++ b/include/libyuv/video_common.h @@ -84,9 +84,9 @@ enum FourCC { FOURCC_24BG = FOURCC('2', '4', 'B', 'G'), FOURCC_RAW = FOURCC('r', 'a', 'w', ' '), FOURCC_RGBA = FOURCC('R', 'G', 'B', 'A'), - FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE. - FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), // argb1555 LE. - FOURCC_R444 = FOURCC('R', '4', '4', '4'), // argb4444 LE. + FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE. deprecated. + FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), // argb1555 LE. deprecated. + FOURCC_R444 = FOURCC('R', '4', '4', '4'), // argb4444 LE. deprecated. // 1 Primary Compressed YUV format. FOURCC_MJPG = FOURCC('M', 'J', 'P', 'G'), @@ -135,9 +135,9 @@ enum FourCC { FOURCC_BGR3 = FOURCC('B', 'G', 'R', '3'), // Alias for 24BG. FOURCC_CM32 = FOURCC(0, 0, 0, 32), // Alias for BGRA kCMPixelFormat_32ARGB FOURCC_CM24 = FOURCC(0, 0, 0, 24), // Alias for RAW kCMPixelFormat_24RGB - FOURCC_L555 = FOURCC('L', '5', '5', '5'), // Alias for RGBO. - FOURCC_L565 = FOURCC('L', '5', '6', '5'), // Alias for RGBP. - FOURCC_5551 = FOURCC('5', '5', '5', '1'), // Alias for RGBO. + FOURCC_L555 = FOURCC('L', '5', '5', '5'), // Alias for RGBO. deprecated. + FOURCC_L565 = FOURCC('L', '5', '6', '5'), // Alias for RGBP. deprecated. + FOURCC_5551 = FOURCC('5', '5', '5', '1'), // Alias for RGBO. deprecated. // deprecated formats. Not supported, but defined for backward compatibility. FOURCC_I411 = FOURCC('I', '4', '1', '1'), @@ -177,9 +177,9 @@ enum FourCCBpp { FOURCC_BPP_AB64 = 64, FOURCC_BPP_24BG = 24, FOURCC_BPP_RAW = 24, - FOURCC_BPP_RGBP = 16, - FOURCC_BPP_RGBO = 16, - FOURCC_BPP_R444 = 16, + FOURCC_BPP_RGBP = 16, // deprecated + FOURCC_BPP_RGBO = 16, // deprecated + FOURCC_BPP_R444 = 16, // deprecated FOURCC_BPP_RGGB = 8, FOURCC_BPP_BGGR = 8, FOURCC_BPP_GRBG = 8, diff --git a/source/compare.cc b/source/compare.cc index 352cedb1b..c5ef00699 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -87,34 +87,6 @@ uint32_t HashDjb2(const uint8_t* src, uint64_t count, uint32_t seed) { return seed; } -static uint32_t ARGBDetectRow_C(const uint8_t* argb, int width) { - int x; - for (x = 0; x < width - 1; x += 2) { - if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. - return FOURCC_BGRA; - } - if (argb[3] != 255) { // Fourth byte is not Alpha of 255, so not BGRA. - return FOURCC_ARGB; - } - if (argb[4] != 255) { // Second pixel first byte is not Alpha of 255. - return FOURCC_BGRA; - } - if (argb[7] != 255) { // Second pixel fourth byte is not Alpha of 255. - return FOURCC_ARGB; - } - argb += 8; - } - if (width & 1) { - if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. - return FOURCC_BGRA; - } - if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA. - return FOURCC_ARGB; - } - } - return 0; -} - // Scan an opaque argb image and return fourcc based on alpha offset. // Returns FOURCC_ARGB, FOURCC_BGRA, or 0 if unknown. LIBYUV_API @@ -122,6 +94,7 @@ uint32_t ARGBDetect(const uint8_t* argb, int stride_argb, int width, int height) { + uint32_t (*ARGBDetectRow)(const uint8_t* argb, int width) = ARGBDetectRow_C; uint32_t fourcc = 0; int h; @@ -134,8 +107,13 @@ uint32_t ARGBDetect(const uint8_t* argb, height = 1; stride_argb = 0; } +#if defined(HAS_ARGBDETECTROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBDetectRow = ARGBDetectRow_AVX2; + } +#endif for (h = 0; h < height && fourcc == 0; ++h) { - fourcc = ARGBDetectRow_C(argb, width); + fourcc = ARGBDetectRow(argb, width); argb += stride_argb; } return fourcc; @@ -262,6 +240,12 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a, simd_size = 1; // Any count. } #endif +#if defined(HAS_SUMSQUAREERROR_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + SumSquareError = SumSquareError_AVX512VNNI; + simd_size = 1; // Any count. + } +#endif #if defined(HAS_SUMSQUAREERROR_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { SumSquareError = SumSquareError_WASMSIMD; @@ -380,10 +364,10 @@ double I420Psnr(const uint8_t* src_y_a, return SumSquareErrorToPsnr(sse, samples); } -static float Ssim8x8_C(const uint8_t* src_a, - int stride_a, - const uint8_t* src_b, - int stride_b) { +static float Ssim8x8(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b) { int32_t sum_a = 0; int32_t sum_b = 0; int32_t sum_sq_ab = 0; @@ -615,7 +599,7 @@ double CalcFrameSsim(const uint8_t* src_a, const uint8_t* b = src_b + (ptrdiff_t)i * stride_b; int j; for (j = 0; j < width - 8; j += 4) { - ssim_total += Ssim8x8_C(a + j, stride_a, b + j, stride_b); + ssim_total += Ssim8x8(a + j, stride_a, b + j, stride_b); samples++; } } diff --git a/source/compare_common.cc b/source/compare_common.cc index cd667fdd2..f50bc4708 100644 --- a/source/compare_common.cc +++ b/source/compare_common.cc @@ -12,6 +12,8 @@ #include +#include "libyuv/video_common.h" + #ifdef __cplusplus namespace libyuv { extern "C" { @@ -169,6 +171,34 @@ float SsimWindowRow_C(const uint32_t* row0, return ssim_total; } +uint32_t ARGBDetectRow_C(const uint8_t* argb, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. + return FOURCC_BGRA; + } + if (argb[3] != 255) { // Fourth byte is not Alpha of 255, so not BGRA. + return FOURCC_ARGB; + } + if (argb[4] != 255) { // Second pixel first byte is not Alpha of 255. + return FOURCC_BGRA; + } + if (argb[7] != 255) { // Second pixel fourth byte is not Alpha of 255. + return FOURCC_ARGB; + } + argb += 8; + } + if (width & 1) { + if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. + return FOURCC_BGRA; + } + if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA. + return FOURCC_ARGB; + } + } + return 0; +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/compare_gcc.cc b/source/compare_gcc.cc index 123d92fea..4ddaa12f9 100644 --- a/source/compare_gcc.cc +++ b/source/compare_gcc.cc @@ -10,6 +10,7 @@ #include "libyuv/compare_row.h" #include "libyuv/row.h" +#include "libyuv/video_common.h" #ifdef __cplusplus namespace libyuv { @@ -23,13 +24,15 @@ extern "C" { // asm statement, and other OS ABIs (e.g. Windows) differ in which registers // are callee-saved. List "memory" first, then "cc", then registers. -// This module is for GCC and clang x64. 32 bit x86 uses compare_win.cc. -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) && \ +// This module is for GCC and clang x86 and x64. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ (!defined(_MSC_VER) || defined(__clang__)) // "memory" clobber prevents the reads from being removed +#ifdef HAS_HAMMINGDISTANCE_SSE42 uint32_t HammingDistance_SSE42(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -77,11 +80,16 @@ uint32_t HammingDistance_SSE42(const uint8_t* src_a, return (uint32_t)(diff); } +#endif // HAS_HAMMINGDISTANCE_SSE42 +#if defined(HAS_HAMMINGDISTANCE_SSSE3) || defined(HAS_HAMMINGDISTANCE_AVX2) || \ + defined(HAS_HAMMINGDISTANCE_AVX512BW) static const vec8 kNibbleMask = {15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; static const vec8 kBitCount = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4}; +#endif +#ifdef HAS_HAMMINGDISTANCE_SSSE3 uint32_t HammingDistance_SSSE3(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -141,6 +149,7 @@ uint32_t HammingDistance_SSSE3(const uint8_t* src_a, return diff; } +#endif // HAS_HAMMINGDISTANCE_SSSE3 #ifdef HAS_HAMMINGDISTANCE_AVX2 uint32_t HammingDistance_AVX2(const uint8_t* src_a, @@ -507,11 +516,78 @@ uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count), [sse] "=r"(sse) : [mask] "m"(mask) - : "cc", "memory", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); return sse; } #endif // HAS_SUMSQUAREERROR_AVX512BW +#ifdef HAS_SUMSQUAREERROR_AVX512VNNI +// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked +// loads that read zeros, so any count is supported. +uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + uint32_t sse; + const uint64_t mask = LowBytesMask64(count & 63); + asm volatile( + "vpxord %%zmm0,%%zmm0,%%zmm0 \n" + "vpxord %%zmm4,%%zmm4,%%zmm4 \n" + "vpxord %%zmm5,%%zmm5,%%zmm5 \n" + "kmovq %[mask],%%k1 \n" + "sub %[src_a],%[src_b] \n" + "sub $0x40,%[count] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%[src_a]),%%zmm1 \n" + "vmovdqu8 (%[src_a],%[src_b]),%%zmm2 \n" + "add $0x40,%[src_a] \n" + "vpsubusb %%zmm2,%%zmm1,%%zmm3 \n" // abs difference trick + "vpsubusb %%zmm1,%%zmm2,%%zmm2 \n" + "vporq %%zmm3,%%zmm2,%%zmm1 \n" + "vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n" // u16 + "vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n" + "vpdpwssd %%zmm2,%%zmm2,%%zmm0 \n" // square + hadd + acc + "vpdpwssd %%zmm1,%%zmm1,%%zmm4 \n" + "sub $0x40,%[count] \n" + "jge 1b \n" + + // Remainder: 1 to 63 bytes, masked loads. + "2: \n" + "add $0x40,%[count] \n" + "je 3f \n" + "vmovdqu8 (%[src_a]),%%zmm1%{%%k1%}%{z%} \n" + "vmovdqu8 (%[src_a],%[src_b]),%%zmm2%{%%k1%}%{z%} \n" + "vpsubusb %%zmm2,%%zmm1,%%zmm3 \n" + "vpsubusb %%zmm1,%%zmm2,%%zmm2 \n" + "vporq %%zmm3,%%zmm2,%%zmm1 \n" + "vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n" + "vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n" + "vpdpwssd %%zmm2,%%zmm2,%%zmm0 \n" + "vpdpwssd %%zmm1,%%zmm1,%%zmm4 \n" + + // Add 16 dwords. + "3: \n" + "vpaddd %%zmm4,%%zmm0,%%zmm0 \n" + "vextracti64x4 $1,%%zmm0,%%ymm1 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vextracti128 $1,%%ymm0,%%xmm1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpshufd $0xee,%%xmm0,%%xmm1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpshufd $0x1,%%xmm0,%%xmm1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,%[sse] \n" + "vzeroupper \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count), + [sse] "=r"(sse) + : [mask] "m"(mask) + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + return sse; +} +#endif // HAS_SUMSQUAREERROR_AVX512VNNI + #if defined(HAS_HASHDJB2_AVX2) || defined(HAS_HASHDJB2_AVX512BW) // HashDjb2 is hash = hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)). // The 32 bit multipliers 33 ^ (63 - i) are split into 16 bit halves so that @@ -812,10 +888,10 @@ void Ssim4x4Row_SSE2(const uint8_t* src_a, "movdqu %%xmm3,0x20(%[dst]) \n" "movdqu %%xmm4,0x30(%[dst]) \n" "add $0x40,%[dst] \n" - "sub $0x10,%[width] \n" + "subl $0x10,%[width] \n" "jg 1b \n" : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), - [width] "+r"(width) + [width] "+rm"(width) : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); @@ -916,11 +992,11 @@ void Ssim4x4Row_AVX2(const uint8_t* src_a, "vmovdqu %%ymm2,0x40(%[dst]) \n" "vmovdqu %%ymm3,0x60(%[dst]) \n" "add $0x80,%[dst] \n" - "sub $0x20,%[width] \n" + "subl $0x20,%[width] \n" "jg 1b \n" "vzeroupper \n" : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), - [width] "+r"(width) + [width] "+rm"(width) : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); @@ -1094,7 +1170,7 @@ float SsimWindowRow_SSE2(const uint32_t* row0, float total; asm volatile( "sub %[row0],%[row1] \n" - "xorps %[total],%[total] \n" + "xorps %%xmm0,%%xmm0 \n" LABELALIGN "1: \n" @@ -1151,19 +1227,20 @@ float SsimWindowRow_SSE2(const uint32_t* row0, "mulps %%xmm5,%%xmm2 \n" "mulps %%xmm3,%%xmm1 \n" "divps %%xmm1,%%xmm2 \n" - "addps %%xmm2,%[total] \n" + "addps %%xmm2,%%xmm0 \n" "sub $0x4,%[num_windows] \n" "jg 1b \n" - "movhlps %[total],%%xmm1 \n" - "addps %%xmm1,%[total] \n" - "movaps %[total],%%xmm1 \n" - "shufps $0x1,%[total],%%xmm1 \n" - "addss %%xmm1,%[total] \n" + "movhlps %%xmm0,%%xmm1 \n" + "addps %%xmm1,%%xmm0 \n" + "movaps %%xmm0,%%xmm1 \n" + "shufps $0x1,%%xmm0,%%xmm1 \n" + "addss %%xmm1,%%xmm0 \n" + "movss %%xmm0,%[total] \n" : [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows), - [total] "=&x"(total) + [total] "=m"(total) : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2) - : "memory", "cc", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); return total; } #endif // HAS_SSIMWINDOWROW_SSE2 @@ -1176,7 +1253,7 @@ float SsimWindowRow_AVX2(const uint32_t* row0, float total; asm volatile( "sub %[row0],%[row1] \n" - "vxorps %t[total],%t[total],%t[total] \n" + "vxorps %%ymm0,%%ymm0,%%ymm0 \n" LABELALIGN "1: \n" @@ -1231,21 +1308,22 @@ float SsimWindowRow_AVX2(const uint32_t* row0, "vmulps %%ymm3,%%ymm1,%%ymm1 \n" "vmulps %%ymm5,%%ymm4,%%ymm4 \n" "vdivps %%ymm4,%%ymm1,%%ymm1 \n" - "vaddps %%ymm1,%t[total],%t[total] \n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" "sub $0x8,%[num_windows] \n" "jg 1b \n" - "vextractf128 $0x1,%t[total],%%xmm1 \n" - "vaddps %%xmm1,%x[total],%x[total] \n" - "vmovhlps %x[total],%%xmm1,%%xmm1 \n" - "vaddps %%xmm1,%x[total],%x[total] \n" - "vshufps $0x1,%x[total],%x[total],%%xmm1 \n" - "vaddss %%xmm1,%x[total],%x[total] \n" + "vextractf128 $0x1,%%ymm0,%%xmm1 \n" + "vaddps %%xmm1,%%xmm0,%%xmm0 \n" + "vmovhlps %%xmm0,%%xmm1,%%xmm1 \n" + "vaddps %%xmm1,%%xmm0,%%xmm0 \n" + "vshufps $0x1,%%xmm0,%%xmm0,%%xmm1 \n" + "vaddss %%xmm1,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,%[total] \n" "vzeroupper \n" : [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows), - [total] "=&x"(total) + [total] "=m"(total) : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2) - : "memory", "cc", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); return total; } #endif // HAS_SSIMWINDOWROW_AVX2 @@ -1370,8 +1448,51 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, #undef SSIM_WINDOW_AVX512BW #endif // HAS_SSIMWINDOWROW_AVX512BW -#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && - // !defined(LIBYUV_ENABLE_ROWWIN) +#ifdef HAS_ARGBDETECTROW_AVX2 +uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) { + uint32_t diff = 0; + asm volatile( + "vpcmpeqd %%ymm1,%%ymm1,%%ymm1 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + LABELALIGN + "1: \n" + "vpcmpeqb (%[argb]),%%ymm1,%%ymm0 \n" + "vpmovmskb %%ymm0,%[diff] \n" + "not %[diff] \n" + "and $0x99999999,%[diff] \n" + "jnz 99f \n" + "add $0x20,%[argb] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + LABELALIGN + "3: \n" + "vmovd (%[argb]),%%xmm0 \n" + "add $0x4,%[argb] \n" + "vpcmpeqb %%xmm1,%%xmm0,%%xmm0 \n" + "vpmovmskb %%xmm0,%[diff] \n" + "not %[diff] \n" + "and $0x9,%[diff] \n" + "jnz 99f \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + "99: \n" + "vzeroupper \n" + : [argb] "+r"(argb), [width] "+r"(width), [diff] "+r"(diff) + : + : "memory", "cc", "xmm0", "xmm1"); + if (diff) { + return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB; + } + return 0; +} +#endif // HAS_ARGBDETECTROW_AVX2 + +#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || + // defined(__i386__)) && !defined(LIBYUV_ENABLE_ROWWIN) #ifdef __cplusplus } // extern "C" diff --git a/source/compare_win.cc b/source/compare_win.cc index d1166ad46..b969a3091 100644 --- a/source/compare_win.cc +++ b/source/compare_win.cc @@ -10,14 +10,15 @@ #include "libyuv/compare_row.h" #include "libyuv/row.h" +#include "libyuv/video_common.h" -// This module is for 32 bit x86 with any compiler, Visual C 64 bit, and for -// clang with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__i386__) || defined(_M_IX86) || \ - ((defined(__x86_64__) || defined(_M_X64)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)))) +// This module is for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) #include // For SSE2 to AVX512 intrinsics #include // For memcpy @@ -39,12 +40,15 @@ extern "C" { #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) +#define LIBYUV_TARGET_AVX512VNNI \ + __attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f"))) #else #define LIBYUV_TARGET_SSE2 #define LIBYUV_TARGET_SSE41 #define LIBYUV_TARGET_SSE42 #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW +#define LIBYUV_TARGET_AVX512VNNI #endif // defined(__clang__) || defined(__GNUC__) #if defined(HAS_HAMMINGDISTANCE_SSE42) @@ -245,6 +249,44 @@ uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, } #endif // HAS_SUMSQUAREERROR_AVX512BW +#if defined(HAS_SUMSQUAREERROR_AVX512VNNI) +// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked +// loads that read zeros, so any count is supported. +LIBYUV_TARGET_AVX512VNNI +uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + const __m512i zero = _mm512_setzero_si512(); + __m512i sum0 = _mm512_setzero_si512(); + __m512i sum1 = _mm512_setzero_si512(); + while (count >= 64) { + __m512i a = _mm512_loadu_si512(src_a); + __m512i b = _mm512_loadu_si512(src_b); + __m512i d = + _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a)); + __m512i lo = _mm512_unpacklo_epi8(d, zero); + __m512i hi = _mm512_unpackhi_epi8(d, zero); + sum0 = _mm512_dpwssd_epi32(sum0, lo, lo); + sum1 = _mm512_dpwssd_epi32(sum1, hi, hi); + src_a += 64; + src_b += 64; + count -= 64; + } + if (count > 0) { + const __mmask64 k = LowBytesMask64(count); + __m512i a = _mm512_maskz_loadu_epi8(k, src_a); + __m512i b = _mm512_maskz_loadu_epi8(k, src_b); + __m512i d = + _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a)); + __m512i lo = _mm512_unpacklo_epi8(d, zero); + __m512i hi = _mm512_unpackhi_epi8(d, zero); + sum0 = _mm512_dpwssd_epi32(sum0, lo, lo); + sum1 = _mm512_dpwssd_epi32(sum1, hi, hi); + } + return (uint32_t)_mm512_reduce_add_epi32(_mm512_add_epi32(sum0, sum1)); +} +#endif // HAS_SUMSQUAREERROR_AVX512VNNI + #if defined(HAS_HASHDJB2_SSE41) // 33 ^ (15 - i) for byte i of 16. static const uint32_t kHashMul16[16] = { @@ -892,6 +934,39 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, } #endif // HAS_SSIMWINDOWROW_AVX512BW +#if defined(HAS_ARGBDETECTROW_AVX2) +LIBYUV_TARGET_AVX2 +uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) { + if (width >= 8) { + const __m256i mask_rgb = _mm256_set1_epi32(0x00ffff00); + const __m256i ones = _mm256_set1_epi8(-1); + do { + __m256i v = + _mm256_or_si256(_mm256_loadu_si256((const __m256i*)argb), mask_rgb); + uint32_t diff = + ~(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, ones)); + if (diff) { +#if defined(_MSC_VER) && !defined(__clang__) + unsigned long bit; // NOLINT(runtime/int) + _BitScanForward(&bit, diff); +#else + int bit = __builtin_ctz(diff); +#endif + _mm256_zeroupper(); + return (bit & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB; + } + argb += 32; + width -= 8; + } while (width >= 8); + _mm256_zeroupper(); + } + if (width > 0) { + return ARGBDetectRow_C(argb, width); + } + return 0; +} +#endif // HAS_ARGBDETECTROW_AVX2 + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/convert.cc b/source/convert.cc index 67385d9ff..50ea99484 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -355,6 +355,13 @@ static int I41xToI420(const uint16_t* src_y, if (TestCpuFlag(kCpuHasRVV)) { HalfWidthRow_16To8 = HalfWidthRow_16To8_RVV; } +#endif + } else { +#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_Odd_AVX2; + } #endif } @@ -1387,6 +1394,11 @@ int MT2TToP010(const uint8_t* src_y, if (!row_buf) return 1; +#if defined(HAS_UNPACKMT2T_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + UnpackMT2T = UnpackMT2T_AVX2; + } +#endif #if defined(HAS_UNPACKMT2T_NEON) if (TestCpuFlag(kCpuHasNEON)) { UnpackMT2T = UnpackMT2T_NEON; @@ -2313,6 +2325,13 @@ static int Px10ToI420(const uint16_t* src_y, if (TestCpuFlag(kCpuHasRVV)) { Half16To8 = HalfWidthRow_16To8_RVV; } +#endif + } else { +#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + Half16To8 = HalfWidthRow_16To8_Odd_AVX2; + } #endif } @@ -2516,6 +2535,12 @@ int YUY2ToI420(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_AVX512BW) && defined(HAS_YUY2TOUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToUVRow = YUY2ToUVRow_AVX512BW; + YUY2ToYRow = YUY2ToYRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToYRow = YUY2ToYRow_Any_NEON; @@ -2552,6 +2577,12 @@ int YUY2ToI420(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_HVX) && defined(HAS_YUY2TOUVROW_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + YUY2ToYRow = YUY2ToYRow_HVX; + YUY2ToUVRow = YUY2ToUVRow_HVX; + } +#endif for (y = 0; y < height - 1; y += 2) { YUY2ToUVRow(src_yuy2, src_stride_yuy2, dst_u, dst_v, width); @@ -2616,6 +2647,12 @@ int UYVYToI420(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_AVX512BW) && defined(HAS_UYVYTOUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + UYVYToUVRow = UYVYToUVRow_AVX512BW; + UYVYToYRow = UYVYToYRow_AVX512BW; + } +#endif #if defined(HAS_UYVYTOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { UYVYToYRow = UYVYToYRow_Any_NEON; @@ -2652,6 +2689,12 @@ int UYVYToI420(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_HVX) && defined(HAS_UYVYTOUVROW_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + UYVYToYRow = UYVYToYRow_HVX; + UYVYToUVRow = UYVYToUVRow_HVX; + } +#endif for (y = 0; y < height - 1; y += 2) { UYVYToUVRow(src_uyvy, src_stride_uyvy, dst_u, dst_v, width); @@ -2693,24 +2736,13 @@ int AYUVToNV12(const uint8_t* src_ayuv, src_ayuv = src_ayuv + (ptrdiff_t)(height - 1) * src_stride_ayuv; src_stride_ayuv = -src_stride_ayuv; } -// place holders for future intel code -#if defined(HAS_AYUVTOYROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - AYUVToUVRow = AYUVToUVRow_Any_SSE2; - AYUVToYRow = AYUVToYRow_Any_SSE2; - if (IS_ALIGNED(width, 16)) { - AYUVToUVRow = AYUVToUVRow_SSE2; - AYUVToYRow = AYUVToYRow_SSE2; - } - } -#endif -#if defined(HAS_AYUVTOYROW_AVX2) +#if defined(HAS_AYUVTOYROW_AVX2) && defined(HAS_AYUVTOUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { - AYUVToUVRow = AYUVToUVRow_Any_AVX2; AYUVToYRow = AYUVToYRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - AYUVToUVRow = AYUVToUVRow_AVX2; + AYUVToUVRow = AYUVToUVRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { AYUVToYRow = AYUVToYRow_AVX2; + AYUVToUVRow = AYUVToUVRow_AVX2; } } #endif @@ -2780,24 +2812,13 @@ int AYUVToNV21(const uint8_t* src_ayuv, src_ayuv = src_ayuv + (ptrdiff_t)(height - 1) * src_stride_ayuv; src_stride_ayuv = -src_stride_ayuv; } -// place holders for future intel code -#if defined(HAS_AYUVTOYROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - AYUVToVURow = AYUVToVURow_Any_SSE2; - AYUVToYRow = AYUVToYRow_Any_SSE2; - if (IS_ALIGNED(width, 16)) { - AYUVToVURow = AYUVToVURow_SSE2; - AYUVToYRow = AYUVToYRow_SSE2; - } - } -#endif -#if defined(HAS_AYUVTOYROW_AVX2) +#if defined(HAS_AYUVTOYROW_AVX2) && defined(HAS_AYUVTOVUROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { - AYUVToVURow = AYUVToVURow_Any_AVX2; AYUVToYRow = AYUVToYRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - AYUVToVURow = AYUVToVURow_AVX2; + AYUVToVURow = AYUVToVURow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { AYUVToYRow = AYUVToYRow_AVX2; + AYUVToVURow = AYUVToVURow_AVX2; } } #endif @@ -3546,6 +3567,14 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, } } #endif +#if defined(HAS_AR30TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + AR30ToARGBRow = AR30ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + AR30ToARGBRow = AR30ToARGBRow_AVX2; + } + } +#endif #if defined(HAS_AR30TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToARGBRow = AR30ToARGBRow_AVX512BW; @@ -3791,12 +3820,6 @@ int AB30ToI420(const uint8_t* src_ab30, &kAbgrI601Constants, width, height); } -// Enabled if 1 pass is available -#if (defined(HAS_RGB24TOYROW_NEON) || defined(HAS_RGB24TOYROW_LSX) || \ - defined(HAS_RGB24TOYROW_RVV)) -#define HAS_RGB24TOYROW -#endif - // Convert RGB24 to I420. LIBYUV_API int RGB24ToI420(const uint8_t* src_rgb24, @@ -4118,12 +4141,6 @@ int RGB24ToJ420(const uint8_t* src_rgb24, return 0; } -// Enabled if 1 pass is available -#if (defined(HAS_RAWTOYROW_NEON) || defined(HAS_RAWTOYROW_LSX) || \ - defined(HAS_RAWTOYROW_RVV)) -#define HAS_RAWTOYROW -#endif - // Convert RAW to I420. LIBYUV_API int RAWToI420(const uint8_t* src_raw, diff --git a/source/convert_argb.cc b/source/convert_argb.cc index e9d8b9c1c..e79dcb8ad 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -631,10 +631,7 @@ int I444ToARGBMatrix(const uint8_t* src_y, #endif #if defined(HAS_I444TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I444ToARGBRow = I444ToARGBRow_AVX512BW; - } + I444ToARGBRow = I444ToARGBRow_AVX512BW; } #endif #if defined(HAS_I444TOARGBROW_NEON) @@ -888,6 +885,11 @@ int I444ToRGB24Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TORGB24ROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI; + } +#endif #if defined(HAS_I444TORGB24ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToRGB24Row = I444ToRGB24Row_Any_NEON; @@ -906,14 +908,6 @@ int I444ToRGB24Matrix(const uint8_t* src_y, I444ToRGB24Row = I444ToRGB24Row_SME; } #endif -#if defined(HAS_I444TORGB24ROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - I444ToRGB24Row = I444ToRGB24Row_Any_LSX; - if (IS_ALIGNED(width, 16)) { - I444ToRGB24Row = I444ToRGB24Row_LSX; - } - } -#endif #if defined(HAS_I444TORGB24ROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I444ToRGB24Row = I444ToRGB24Row_RVV; @@ -1030,6 +1024,11 @@ int I010ToAR30Matrix(const uint16_t* src_y, I210ToAR30Row = I210ToAR30Row_AVX2; } } +#endif +#if defined(HAS_I210TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210ToAR30Row = I210ToAR30Row_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); @@ -1192,6 +1191,11 @@ int I012ToAR30Matrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I212TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I212ToAR30Row = I212ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_I212TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I212ToAR30Row = I212ToAR30Row_Any_NEON; @@ -1286,6 +1290,11 @@ int I210ToAR30Matrix(const uint16_t* src_y, I210ToAR30Row = I210ToAR30Row_AVX2; } } +#endif +#if defined(HAS_I210TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210ToAR30Row = I210ToAR30Row_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); @@ -1460,6 +1469,11 @@ int I410ToAR30Matrix(const uint16_t* src_y, I410ToAR30Row = I410ToAR30Row_AVX2; } } +#endif +#if defined(HAS_I410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToAR30Row = I410ToAR30Row_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I410ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); @@ -1533,6 +1547,11 @@ int I010ToARGBMatrix(const uint16_t* src_y, I210ToARGBRow = I210ToARGBRow_AVX2; } } +#endif +#if defined(HAS_I210TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210ToARGBRow = I210ToARGBRow_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -1699,6 +1718,11 @@ int I012ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I212TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I212ToARGBRow = I212ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I212TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I212ToARGBRow = I212ToARGBRow_Any_NEON; @@ -1799,6 +1823,11 @@ int I210ToARGBMatrix(const uint16_t* src_y, I210ToARGBRow = I210ToARGBRow_AVX2; } } +#endif +#if defined(HAS_I210TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210ToARGBRow = I210ToARGBRow_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -1986,6 +2015,11 @@ int I410ToARGBMatrix(const uint16_t* src_y, I410ToARGBRow = I410ToARGBRow_AVX2; } } +#endif +#if defined(HAS_I410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToARGBRow = I410ToARGBRow_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I410ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -2038,6 +2072,11 @@ int P010ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P210TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P210ToARGBRow = P210ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P210TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P210ToARGBRow = P210ToARGBRow_Any_NEON; @@ -2108,6 +2147,11 @@ int P210ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P210TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P210ToARGBRow = P210ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P210TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P210ToARGBRow = P210ToARGBRow_Any_NEON; @@ -2176,6 +2220,11 @@ int P410ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToARGBRow = P410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P410TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToARGBRow = P410ToARGBRow_Any_NEON; @@ -2244,6 +2293,11 @@ int P010ToAR30Matrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P210TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P210ToAR30Row = P210ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P210TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P210ToAR30Row = P210ToAR30Row_Any_NEON; @@ -2314,6 +2368,11 @@ int P210ToAR30Matrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P210TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P210ToAR30Row = P210ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P210TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P210ToAR30Row = P210ToAR30Row_Any_NEON; @@ -2382,6 +2441,11 @@ int P410ToAR30Matrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToAR30Row = P410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P410TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToAR30Row = P410ToAR30Row_Any_NEON; @@ -2822,6 +2886,11 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I444ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I444ALPHATOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON; @@ -3119,6 +3188,11 @@ int I010AlphaToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I210ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -3261,6 +3335,11 @@ int I210AlphaToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I210ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -3401,6 +3480,11 @@ int I410AlphaToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I410ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -4214,11 +4298,11 @@ int RAWToRGBA(const uint8_t* src_raw, height = 1; src_stride_raw = dst_stride_rgba = 0; } -#if defined(HAS_RAWTORGBAROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - RAWToRGBARow = RAWToRGBARow_Any_SSSE3; +#if defined(HAS_RAWTORGBAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + RAWToRGBARow = RAWToRGBARow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - RAWToRGBARow = RAWToRGBARow_SSSE3; + RAWToRGBARow = RAWToRGBARow_AVX2; } } #endif @@ -4370,6 +4454,11 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555, } } #endif +#if defined(HAS_ARGB1555TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGB1555TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_NEON; @@ -4530,6 +4619,14 @@ int AR30ToARGB(const uint8_t* src_ar30, } } #endif +#if defined(HAS_AR30TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + AR30ToARGBRow = AR30ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + AR30ToARGBRow = AR30ToARGBRow_AVX2; + } + } +#endif #if defined(HAS_AR30TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToARGBRow = AR30ToARGBRow_AVX512BW; @@ -4579,6 +4676,14 @@ int AR30ToABGR(const uint8_t* src_ar30, } } #endif +#if defined(HAS_AR30TOABGRROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + AR30ToABGRRow = AR30ToABGRRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + AR30ToABGRRow = AR30ToABGRRow_AVX2; + } + } +#endif #if defined(HAS_AR30TOABGRROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToABGRRow = AR30ToABGRRow_AVX512BW; @@ -4628,6 +4733,14 @@ int AR30ToAB30(const uint8_t* src_ar30, } } #endif +#if defined(HAS_AR30TOAB30ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + AR30ToAB30Row = AR30ToAB30Row_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + AR30ToAB30Row = AR30ToAB30Row_AVX2; + } + } +#endif #if defined(HAS_AR30TOAB30ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToAB30Row = AR30ToAB30Row_AVX512BW; @@ -5391,6 +5504,11 @@ int YUY2ToARGBMatrix(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToARGBRow = YUY2ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToARGBRow = YUY2ToARGBRow_Any_NEON; @@ -5491,6 +5609,11 @@ int UYVYToARGBMatrix(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + UYVYToARGBRow = UYVYToARGBRow_AVX512BW; + } +#endif #if defined(HAS_UYVYTOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { UYVYToARGBRow = UYVYToARGBRow_Any_NEON; @@ -5721,6 +5844,11 @@ int I422ToRGBAMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGBAROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I422ToRGBARow = I422ToRGBARow_AVX512BW; + } +#endif #if defined(HAS_I422TORGBAROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToRGBARow = I422ToRGBARow_Any_NEON; @@ -5958,6 +6086,11 @@ int I420ToRGBAMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGBAROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I422ToRGBARow = I422ToRGBARow_AVX512BW; + } +#endif #if defined(HAS_I422TORGBAROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToRGBARow = I422ToRGBARow_Any_NEON; @@ -6474,6 +6607,12 @@ int I420ToARGB1555(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOARGB1555ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { + I422ToARGB1555Row = I422ToARGB1555Row_AVX512BW; + } +#endif #if defined(HAS_I422TOARGB1555ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToARGB1555Row = I422ToARGB1555Row_Any_NEON; @@ -6573,6 +6712,12 @@ int I420ToARGB4444(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOARGB4444ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { + I422ToARGB4444Row = I422ToARGB4444Row_AVX512BW; + } +#endif #if defined(HAS_I422TOARGB4444ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToARGB4444Row = I422ToARGB4444Row_Any_NEON; @@ -6674,6 +6819,12 @@ int I420ToRGB565Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB565ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { + I422ToRGB565Row = I422ToRGB565Row_AVX512BW; + } +#endif #if defined(HAS_I422TORGB565ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToRGB565Row = I422ToRGB565Row_Any_NEON; @@ -6833,6 +6984,12 @@ int I422ToRGB565Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB565ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { + I422ToRGB565Row = I422ToRGB565Row_AVX512BW; + } +#endif #if defined(HAS_I422TORGB565ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToRGB565Row = I422ToRGB565Row_Any_NEON; @@ -7018,14 +7175,6 @@ int I420ToRGB565Dither(const uint8_t* src_y, I422ToARGBRow = I422ToARGBRow_HVX; } #endif -#if defined(HAS_ARGBTORGB565DITHERROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_SSE2; - if (IS_ALIGNED(width, 4)) { - ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SSE2; - } - } -#endif #if defined(HAS_ARGBTORGB565DITHERROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2; @@ -7034,6 +7183,11 @@ int I420ToRGB565Dither(const uint8_t* src_y, } } #endif +#if defined(HAS_ARGBTORGB565DITHERROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX512BW; + } +#endif #if defined(HAS_ARGBTORGB565DITHERROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON; @@ -7306,10 +7460,7 @@ static int I420ToARGBMatrixBilinear(const uint8_t* src_y, #endif #if defined(HAS_I444TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I444ToARGBRow = I444ToARGBRow_AVX512BW; - } + I444ToARGBRow = I444ToARGBRow_AVX512BW; } #endif #if defined(HAS_I444TOARGBROW_NEON) @@ -7477,10 +7628,7 @@ static int I422ToARGBMatrixLinear(const uint8_t* src_y, #endif #if defined(HAS_I444TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I444ToARGBRow = I444ToARGBRow_AVX512BW; - } + I444ToARGBRow = I444ToARGBRow_AVX512BW; } #endif #if defined(HAS_I444TOARGBROW_NEON) @@ -7623,6 +7771,11 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TORGB24ROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI; + } +#endif #if defined(HAS_I444TORGB24ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToRGB24Row = I444ToRGB24Row_Any_NEON; @@ -7641,14 +7794,6 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y, I444ToRGB24Row = I444ToRGB24Row_SME; } #endif -#if defined(HAS_I444TORGB24ROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - I444ToRGB24Row = I444ToRGB24Row_Any_LASX; - if (IS_ALIGNED(width, 32)) { - I444ToRGB24Row = I444ToRGB24Row_LASX; - } - } -#endif #if defined(HAS_I444TORGB24ROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I444ToRGB24Row = I444ToRGB24Row_RVV; @@ -7794,6 +7939,11 @@ static int I010ToAR30MatrixBilinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToAR30Row = I410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -7919,6 +8069,11 @@ static int I210ToAR30MatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToAR30Row = I410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8023,6 +8178,11 @@ static int I010ToARGBMatrixBilinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToARGBRow = I410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8147,6 +8307,11 @@ static int I210ToARGBMatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToARGBRow = I410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8241,6 +8406,11 @@ static int I420AlphaToARGBMatrixBilinear( } } #endif +#if defined(HAS_I444ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I444ALPHATOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON; @@ -8259,14 +8429,6 @@ static int I420AlphaToARGBMatrixBilinear( I444AlphaToARGBRow = I444AlphaToARGBRow_SME; } #endif -#if defined(HAS_I444ALPHATOARGBROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - I444AlphaToARGBRow = I444AlphaToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { - I444AlphaToARGBRow = I444AlphaToARGBRow_LASX; - } - } -#endif #if defined(HAS_I444ALPHATOARGBROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I444AlphaToARGBRow = I444AlphaToARGBRow_RVV; @@ -8476,6 +8638,11 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, } } #endif +#if defined(HAS_I444ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I444ALPHATOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON; @@ -8494,14 +8661,6 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, I444AlphaToARGBRow = I444AlphaToARGBRow_SME; } #endif -#if defined(HAS_I444ALPHATOARGBROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - I444AlphaToARGBRow = I444AlphaToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { - I444AlphaToARGBRow = I444AlphaToARGBRow_LASX; - } - } -#endif #if defined(HAS_I444ALPHATOARGBROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I444AlphaToARGBRow = I444AlphaToARGBRow_RVV; @@ -8694,6 +8853,11 @@ static int I010AlphaToARGBMatrixBilinear( } } #endif +#if defined(HAS_I410ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -8898,6 +9062,11 @@ static int I210AlphaToARGBMatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -9039,6 +9208,11 @@ static int P010ToARGBMatrixBilinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToARGBRow = P410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P410TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToARGBRow = P410ToARGBRow_Any_NEON; @@ -9151,6 +9325,11 @@ static int P210ToARGBMatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToARGBRow = P410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P410TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToARGBRow = P410ToARGBRow_Any_NEON; @@ -9249,6 +9428,11 @@ static int P010ToAR30MatrixBilinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToAR30Row = P410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P410TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToAR30Row = P410ToAR30Row_Any_NEON; @@ -9361,6 +9545,11 @@ static int P210ToAR30MatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToAR30Row = P410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P410TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToAR30Row = P410ToAR30Row_Any_NEON; @@ -9461,6 +9650,11 @@ static int I422ToRGB24MatrixLinear(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TORGB24ROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI; + } +#endif #if defined(HAS_I444TORGB24ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToRGB24Row = I444ToRGB24Row_Any_NEON; diff --git a/source/convert_from.cc b/source/convert_from.cc index 4bca7602f..c4f35cfea 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -820,6 +820,11 @@ static int I4xxToPxxx(const uint8_t* src_y, } } #endif +#if defined(HAS_MERGEUVROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MergeUVRow_16 = MergeUVRow_16_AVX512BW; + } +#endif #if defined(HAS_MERGEUVROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { MergeUVRow_16 = MergeUVRow_16_Any_NEON; diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 2337d54f7..cc7691314 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -4040,6 +4040,11 @@ int ARGBToRGB565Dither(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTORGB565DITHERROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX512BW; + } +#endif #if defined(HAS_ARGBTORGB565DITHERROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON; @@ -4123,6 +4128,11 @@ int ARGBToRGB565(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTORGB565ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToRGB565Row = ARGBToRGB565Row_AVX512BW; + } +#endif #if defined(HAS_ARGBTORGB565ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToRGB565Row = ARGBToRGB565Row_Any_NEON; diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index bc699b647..d3b55687a 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -174,10 +174,7 @@ static int NV24ToARGB(const uint8_t* src_y, #endif #if defined(HAS_I444TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I444ToARGBRow = I444ToARGBRow_AVX512BW; - } + I444ToARGBRow = I444ToARGBRow_AVX512BW; } #endif #if defined(HAS_I444TOARGBROW_NEON) diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc index 67f78b170..0d80d0c0d 100644 --- a/source/convert_to_i420.cc +++ b/source/convert_to_i420.cc @@ -26,16 +26,6 @@ namespace libyuv { extern "C" { #endif -static void HalfRow_C(const uint8_t* src_uv, - ptrdiff_t src_uv_stride, - uint8_t* dst_uv, - int width) { - int x; - for (x = 0; x < width; ++x) { - dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1; - } -} - static int NV16ToI420(const uint8_t* src_y, int src_stride_y, const uint8_t* src_uv, @@ -227,6 +217,8 @@ static int NV24ToI420(const uint8_t* src_y, int x, y; int halfwidth = (width + 1) >> 1; int halfheight = (height + 1) >> 1; + void (*HalfRow)(const uint8_t* src_uv, ptrdiff_t src_uv_stride, + uint8_t* dst_uv, int width) = HalfRow_C; if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { return -1; @@ -242,6 +234,14 @@ static int NV24ToI420(const uint8_t* src_y, if (dst_y) { CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); } +#if defined(HAS_HALFROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + HalfRow = HalfRow_Any_AVX2; + if (IS_ALIGNED(width * 2, 32)) { + HalfRow = HalfRow_AVX2; + } + } +#endif align_buffer_64(row_uv, width * 2); if (!row_uv) { @@ -249,7 +249,7 @@ static int NV24ToI420(const uint8_t* src_y, } for (y = 0; y < halfheight; ++y) { if (y * 2 + 1 < height) { - HalfRow_C(src_uv, src_stride_uv, row_uv, width * 2); + HalfRow(src_uv, src_stride_uv, row_uv, width * 2); } else { memcpy(row_uv, src_uv, (size_t)width * 2); } diff --git a/source/cpu_id.cc b/source/cpu_id.cc index 7bfb66de2..f78283a97 100644 --- a/source/cpu_id.cc +++ b/source/cpu_id.cc @@ -405,6 +405,7 @@ static SAFEBUFFERS int GetCpuFlags(void) { int cpu_info1[4] = {0, 0, 0, 0}; int cpu_info7[4] = {0, 0, 0, 0}; int cpu_einfo7[4] = {0, 0, 0, 0}; + int cpu_einfo1e[4] = {0, 0, 0, 0}; int cpu_info24[4] = {0, 0, 0, 0}; int cpu_info21[4] = {0, 0, 0, 0}; int cpu_amdinfo21[4] = {0, 0, 0, 0}; @@ -425,6 +426,9 @@ static SAFEBUFFERS int GetCpuFlags(void) { if ((unsigned)cpu_extinfo0[0] >= 0x80000021u) { CpuId(0x80000021, 0, cpu_amdinfo21); } + if (cpu_info0[0] >= 0x1e) { + CpuId(0x1e, 1, cpu_einfo1e); + } if (cpu_info0[0] >= 0x21) { CpuId(0x21, 0, cpu_info21); } @@ -459,7 +463,8 @@ static SAFEBUFFERS int GetCpuFlags(void) { ((cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0) | ((cpu_einfo7[3] & 0x00080000) ? kCpuHasAVX10 : 0) | ((cpu_info7[3] & 0x02000000) ? kCpuHasAMXINT8 : 0) | - ((cpu_info21[0] & 0x00800000) ? kCpuHasAVX512BMM : 0); + ((cpu_info21[0] & 0x00800000) ? kCpuHasAVX512BMM : 0) | + ((cpu_einfo1e[0] & 0x00000040) ? kCpuHasAMXTF32 : 0); if (cpu_info0[0] >= 0x24 && (cpu_einfo7[3] & 0x00080000)) { cpu_info |= ((cpu_info24[1] & 0xFF) >= 2) ? kCpuHasAVX10_2 : 0; } diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 1521e3f05..0c8a42542 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -148,6 +148,11 @@ void CopyPlane_16(const uint16_t* src_y, if (src_y == dst_y && src_stride_y == dst_stride_y) { return; } +#if defined(HAS_COPYROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_16_AVX2 : CopyRow_16_Any_AVX2; + } +#endif #if defined(HAS_COPYROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { CopyRow = IS_ALIGNED(width, 16) ? CopyRow_16_NEON : CopyRow_16_Any_NEON; @@ -406,6 +411,11 @@ void Convert8To8Plane(const uint8_t* src_y, } } #endif +#if defined(HAS_CONVERT8TO8ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + Convert8To8Row = Convert8To8Row_AVX512BW; + } +#endif #if defined(HAS_CONVERT8TO8ROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { Convert8To8Row = Convert8To8Row_HVX; @@ -1009,6 +1019,11 @@ void MergeUVPlane_16(const uint16_t* src_u, } } #endif +#if defined(HAS_MERGEUVROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MergeUVRow_16 = MergeUVRow_16_AVX512BW; + } +#endif #if defined(HAS_MERGEUVROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { MergeUVRow_16 = MergeUVRow_16_Any_NEON; @@ -1160,6 +1175,11 @@ void ConvertToLSBPlane_16(const uint16_t* src_y, } } #endif +#if defined(HAS_DIVIDEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + DivideRow = DivideRow_16_AVX512BW; + } +#endif #if defined(HAS_DIVIDEROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { DivideRow = DivideRow_16_Any_NEON; @@ -1337,11 +1357,11 @@ int DetilePlane(const uint8_t* src_y, dst_stride_y = -dst_stride_y; } -#if defined(HAS_DETILEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - DetileRow = DetileRow_Any_SSE2; +#if defined(HAS_DETILEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + DetileRow = DetileRow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - DetileRow = DetileRow_SSE2; + DetileRow = DetileRow_AVX2; } } #endif @@ -1394,19 +1414,11 @@ int DetilePlane_16(const uint16_t* src_y, dst_stride_y = -dst_stride_y; } -#if defined(HAS_DETILEROW_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - DetileRow_16 = DetileRow_16_Any_SSE2; +#if defined(HAS_DETILEROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + DetileRow_16 = DetileRow_16_Any_AVX2; if (IS_ALIGNED(width, 16)) { - DetileRow_16 = DetileRow_16_SSE2; - } - } -#endif -#if defined(HAS_DETILEROW_16_AVX) - if (TestCpuFlag(kCpuHasAVX)) { - DetileRow_16 = DetileRow_16_Any_AVX; - if (IS_ALIGNED(width, 16)) { - DetileRow_16 = DetileRow_16_AVX; + DetileRow_16 = DetileRow_16_AVX2; } } #endif @@ -1466,11 +1478,11 @@ void DetileSplitUVPlane(const uint8_t* src_uv, dst_stride_v = -dst_stride_v; } -#if defined(HAS_DETILESPLITUVROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - DetileSplitUVRow = DetileSplitUVRow_Any_SSSE3; +#if defined(HAS_DETILESPLITUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + DetileSplitUVRow = DetileSplitUVRow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - DetileSplitUVRow = DetileSplitUVRow_SSSE3; + DetileSplitUVRow = DetileSplitUVRow_AVX2; } } #endif @@ -1540,11 +1552,11 @@ void DetileToYUY2(const uint8_t* src_y, } #endif -#if defined(HAS_DETILETOYUY2_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - DetileToYUY2 = DetileToYUY2_Any_SSE2; - if (IS_ALIGNED(width, 16)) { - DetileToYUY2 = DetileToYUY2_SSE2; +#if defined(HAS_DETILETOYUY2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + DetileToYUY2 = DetileToYUY2_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + DetileToYUY2 = DetileToYUY2_AVX2; } } #endif @@ -1684,11 +1696,11 @@ void MergeRGBPlane(const uint8_t* src_r, height = 1; src_stride_r = src_stride_g = src_stride_b = dst_stride_rgb = 0; } -#if defined(HAS_MERGERGBROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - MergeRGBRow = MergeRGBRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - MergeRGBRow = MergeRGBRow_SSSE3; +#if defined(HAS_MERGERGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + MergeRGBRow = MergeRGBRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + MergeRGBRow = MergeRGBRow_AVX2; } } #endif @@ -2536,6 +2548,11 @@ int YUY2ToI422(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToYRow = YUY2ToYRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToYRow = YUY2ToYRow_Any_NEON; @@ -2644,6 +2661,11 @@ int UYVYToI422(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + UYVYToYRow = UYVYToYRow_AVX512BW; + } +#endif #if defined(HAS_UYVYTOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { UYVYToYRow = UYVYToYRow_Any_NEON; @@ -2740,6 +2762,11 @@ int YUY2ToY(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToYRow = YUY2ToYRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToYRow = YUY2ToYRow_Any_NEON; @@ -2810,6 +2837,11 @@ int UYVYToY(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + UYVYToYRow = UYVYToYRow_AVX512BW; + } +#endif #if defined(HAS_UYVYTOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { UYVYToYRow = UYVYToYRow_Any_NEON; @@ -2899,10 +2931,7 @@ void MirrorPlane(const uint8_t* src_y, #endif #if defined(HAS_MIRRORROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MirrorRow = MirrorRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - MirrorRow = MirrorRow_AVX512BW; - } + MirrorRow = MirrorRow_AVX512BW; } #endif #if defined(HAS_MIRRORROW_LSX) @@ -3095,17 +3124,9 @@ void MirrorUVPlane(const uint8_t* src_uv, } } #endif -#if defined(HAS_MIRRORROW_16_AVX512BW) +#if defined(HAS_MIRRORUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MirrorUVRow = reinterpret_cast( - MirrorRow_16_AVX512BW); - } -#elif defined(HAS_MIRRORUVROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - MirrorUVRow = MirrorUVRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - MirrorUVRow = MirrorUVRow_AVX512BW; - } + MirrorUVRow = MirrorUVRow_AVX512BW; } #endif #if defined(HAS_MIRRORUVROW_LSX) @@ -3447,9 +3468,9 @@ int ARGBBlend(const uint8_t* src_argb0, height = 1; src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; } -#if defined(HAS_ARGBBLENDROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBBlendRow = ARGBBlendRow_SSSE3; +#if defined(HAS_ARGBBLENDROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBBlendRow = ARGBBlendRow_AVX2; } #endif #if defined(HAS_ARGBBLENDROW_AVX512BW) @@ -4429,11 +4450,11 @@ void SetPlane(uint8_t* dst_y, } } #endif -#if defined(HAS_SETROW_X86) - if (TestCpuFlag(kCpuHasX86)) { - SetRow = SetRow_Any_X86; - if (IS_ALIGNED(width, 4)) { - SetRow = SetRow_X86; +#if defined(HAS_SETROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SetRow = SetRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + SetRow = SetRow_AVX2; } } #endif @@ -4541,9 +4562,12 @@ int ARGBRect(uint8_t* dst_argb, } } #endif -#if defined(HAS_ARGBSETROW_X86) - if (TestCpuFlag(kCpuHasX86)) { - ARGBSetRow = ARGBSetRow_X86; +#if defined(HAS_ARGBSETROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBSetRow = ARGBSetRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBSetRow = ARGBSetRow_AVX2; + } } #endif #if defined(HAS_ARGBSETROW_LSX) @@ -4771,9 +4795,9 @@ int ARGBGrayTo(const uint8_t* src_argb, height = 1; src_stride_argb = dst_stride_argb = 0; } -#if defined(HAS_ARGBGRAYROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_SSSE3; +#if defined(HAS_ARGBGRAYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBGrayRow = ARGBGrayRow_AVX2; } #endif #if defined(HAS_ARGBGRAYROW_AVX512BW) @@ -4836,9 +4860,9 @@ int ARGBGray(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_ARGBGRAYROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_SSSE3; +#if defined(HAS_ARGBGRAYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBGrayRow = ARGBGrayRow_AVX2; } #endif #if defined(HAS_ARGBGRAYROW_AVX512BW) @@ -4899,9 +4923,14 @@ int ARGBSepia(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_ARGBSEPIAROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { - ARGBSepiaRow = ARGBSepiaRow_SSSE3; +#if defined(HAS_ARGBSEPIAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBSepiaRow = ARGBSepiaRow_AVX2; + } +#endif +#if defined(HAS_ARGBSEPIAROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBSepiaRow = ARGBSepiaRow_AVX512VNNI; } #endif #if defined(HAS_ARGBSEPIAROW_NEON) @@ -4962,9 +4991,14 @@ int ARGBColorMatrix(const uint8_t* src_argb, height = 1; src_stride_argb = dst_stride_argb = 0; } -#if defined(HAS_ARGBCOLORMATRIXROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { - ARGBColorMatrixRow = ARGBColorMatrixRow_SSSE3; +#if defined(HAS_ARGBCOLORMATRIXROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_AVX2; + } +#endif +#if defined(HAS_ARGBCOLORMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBCOLORMATRIXROW_NEON) @@ -5056,9 +5090,9 @@ int ARGBColorTable(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_ARGBCOLORTABLEROW_X86) - if (TestCpuFlag(kCpuHasX86)) { - ARGBColorTableRow = ARGBColorTableRow_X86; +#if defined(HAS_ARGBCOLORTABLEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBColorTableRow = ARGBColorTableRow_AVX2; } #endif #if defined(HAS_ARGBCOLORTABLEROW_NEON) @@ -5097,9 +5131,9 @@ int RGBColorTable(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_RGBCOLORTABLEROW_X86) - if (TestCpuFlag(kCpuHasX86)) { - RGBColorTableRow = RGBColorTableRow_X86; +#if defined(HAS_RGBCOLORTABLEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + RGBColorTableRow = RGBColorTableRow_AVX2; } #endif #if defined(HAS_RGBCOLORTABLEROW_NEON) @@ -5147,9 +5181,9 @@ int ARGBQuantize(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_ARGBQUANTIZEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) { - ARGBQuantizeRow = ARGBQuantizeRow_SSE2; +#if defined(HAS_ARGBQUANTIZEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBQuantizeRow = ARGBQuantizeRow_AVX2; } #endif #if defined(HAS_ARGBQUANTIZEROW_NEON) @@ -5191,11 +5225,6 @@ int ARGBComputeCumulativeSum(const uint8_t* src_argb, if (!dst_cumsum || !src_argb || width <= 0 || height <= 0) { return -1; } -#if defined(HAS_COMPUTECUMULATIVESUMROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2; - } -#endif #if defined(HAS_COMPUTECUMULATIVESUMROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2; @@ -5260,12 +5289,6 @@ int ARGBBlur(const uint8_t* src_argb, if (radius <= 0 || height <= 1) { return -1; } -#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2; - CumulativeSumToAverageRow = CumulativeSumToAverageRow_SSE2; - } -#endif #if defined(HAS_CUMULATIVESUMTOAVERAGEROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2; @@ -5371,9 +5394,9 @@ int ARGBShade(const uint8_t* src_argb, height = 1; src_stride_argb = dst_stride_argb = 0; } -#if defined(HAS_ARGBSHADEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) { - ARGBShadeRow = ARGBShadeRow_SSE2; +#if defined(HAS_ARGBSHADEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBShadeRow = ARGBShadeRow_AVX2; } #endif #if defined(HAS_ARGBSHADEROW_NEON) @@ -5539,14 +5562,6 @@ int InterpolatePlane_16(const uint16_t* src0, height = 1; src_stride0 = src_stride1 = dst_stride = 0; } -#if defined(HAS_INTERPOLATEROW_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - InterpolateRow_16 = InterpolateRow_16_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - InterpolateRow_16 = InterpolateRow_16_SSSE3; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow_16 = InterpolateRow_16_Any_AVX2; @@ -5573,14 +5588,6 @@ int InterpolatePlane_16(const uint16_t* src0, InterpolateRow_16 = InterpolateRow_16_SME; } #endif -#if defined(HAS_INTERPOLATEROW_16_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - InterpolateRow_16 = InterpolateRow_16_Any_LSX; - if (IS_ALIGNED(width, 32)) { - InterpolateRow_16 = InterpolateRow_16_LSX; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { InterpolateRow_16 = InterpolateRow_16_Any_WASMSIMD; @@ -5868,11 +5875,21 @@ int GaussPlane_F32(const float* src, src_stride = -src_stride; } +#if defined(HAS_GAUSSCOL_F32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + GaussCol_F32 = GaussCol_F32_AVX2; + } +#endif #if defined(HAS_GAUSSCOL_F32_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { GaussCol_F32 = GaussCol_F32_NEON; } #endif +#if defined(HAS_GAUSSROW_F32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + GaussRow_F32 = GaussRow_F32_AVX2; + } +#endif #if defined(HAS_GAUSSROW_F32_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { GaussRow_F32 = GaussRow_F32_NEON; @@ -6026,9 +6043,9 @@ static int ARGBSobelize(const uint8_t* src_argb, } #endif -#if defined(HAS_SOBELYROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelYRow = SobelYRow_SSE2; +#if defined(HAS_SOBELYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelYRow = SobelYRow_AVX2; } #endif #if defined(HAS_SOBELYROW_NEON) @@ -6036,9 +6053,9 @@ static int ARGBSobelize(const uint8_t* src_argb, SobelYRow = SobelYRow_NEON; } #endif -#if defined(HAS_SOBELXROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelXRow = SobelXRow_SSE2; +#if defined(HAS_SOBELXROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelXRow = SobelXRow_AVX2; } #endif #if defined(HAS_SOBELXROW_NEON) @@ -6106,11 +6123,11 @@ int ARGBSobel(const uint8_t* src_argb, int height) { void (*SobelRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width) = SobelRow_C; -#if defined(HAS_SOBELROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelRow = SobelRow_Any_SSE2; +#if defined(HAS_SOBELROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelRow = SobelRow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - SobelRow = SobelRow_SSE2; + SobelRow = SobelRow_AVX2; } } #endif @@ -6152,11 +6169,11 @@ int ARGBSobelToPlane(const uint8_t* src_argb, int height) { void (*SobelToPlaneRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_, int width) = SobelToPlaneRow_C; -#if defined(HAS_SOBELTOPLANEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelToPlaneRow = SobelToPlaneRow_Any_SSE2; - if (IS_ALIGNED(width, 16)) { - SobelToPlaneRow = SobelToPlaneRow_SSE2; +#if defined(HAS_SOBELTOPLANEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelToPlaneRow = SobelToPlaneRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + SobelToPlaneRow = SobelToPlaneRow_AVX2; } } #endif @@ -6199,11 +6216,11 @@ int ARGBSobelXY(const uint8_t* src_argb, int height) { void (*SobelXYRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width) = SobelXYRow_C; -#if defined(HAS_SOBELXYROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelXYRow = SobelXYRow_Any_SSE2; +#if defined(HAS_SOBELXYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelXYRow = SobelXYRow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - SobelXYRow = SobelXYRow_SSE2; + SobelXYRow = SobelXYRow_AVX2; } } #endif @@ -6340,6 +6357,11 @@ int HalfFloatPlane(const uint16_t* src_y, } } #endif +#if defined(HAS_HALFFLOATROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + HalfFloatRow = HalfFloatRow_AVX512BW; + } +#endif #if defined(HAS_HALFFLOATROW_NEON) if (TestCpuFlag(kCpuHasNEON) #if defined(__arm__) @@ -6394,6 +6416,14 @@ int ByteToFloat(const uint8_t* src_y, float* dst_y, float scale, int width) { if (!src_y || !dst_y || width <= 0) { return -1; } +#if defined(HAS_BYTETOFLOATROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ByteToFloatRow = ByteToFloatRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + ByteToFloatRow = ByteToFloatRow_AVX2; + } + } +#endif #if defined(HAS_BYTETOFLOATROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ByteToFloatRow = ByteToFloatRow_Any_NEON; @@ -6437,9 +6467,9 @@ int ARGBLumaColorTable(const uint8_t* src_argb, height = 1; src_stride_argb = dst_stride_argb = 0; } -#if defined(HAS_ARGBLUMACOLORTABLEROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 4)) { - ARGBLumaColorTableRow = ARGBLumaColorTableRow_SSSE3; +#if defined(HAS_ARGBLUMACOLORTABLEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBLumaColorTableRow = ARGBLumaColorTableRow_AVX2; } #endif #if defined(HAS_ARGBLUMACOLORTABLEROW_NEON) @@ -6730,6 +6760,11 @@ int YUY2ToNV12(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToYRow = YUY2ToYRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToYRow = YUY2ToYRow_Any_NEON; diff --git a/source/rotate.cc b/source/rotate.cc index 59bf00824..7b27333f1 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -24,6 +24,21 @@ namespace libyuv { extern "C" { #endif +static void TransposeWxH(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + int i; + for (i = 0; i < width; ++i) { + int j; + for (j = 0; j < height; ++j) { + dst[i * dst_stride + j] = src[j * src_stride + i]; + } + } +} + LIBYUV_API void TransposePlane(const uint8_t* src, int src_stride, @@ -33,8 +48,8 @@ void TransposePlane(const uint8_t* src, int height) { int i = height; #if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX) - void (*TransposeWxH)(const uint8_t* src, int src_stride, uint8_t* dst, - int dst_stride, int width, int height) = NULL; + void (*TransposeWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst, + int dst_stride, int width, int height) = NULL; #endif #if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \ defined(HAS_TRANSPOSEWX16_LASX) || defined(HAS_TRANSPOSEWX16_WASMSIMD) @@ -67,19 +82,19 @@ void TransposePlane(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEWXH_SME) if (TestCpuFlag(kCpuHasSME)) { - TransposeWxH = TransposeWxH_SME; + TransposeWxH_fn = TransposeWxH_SME; } #endif #if defined(HAS_TRANSPOSEWXH_HVX) if (TestCpuFlag(kCpuHasHVX)) { - TransposeWxH = TransposeWxH_HVX; + TransposeWxH_fn = TransposeWxH_HVX; } #endif -#if defined(HAS_TRANSPOSEWX8_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - TransposeWx8 = TransposeWx8_Any_SSSE3; +#if defined(HAS_TRANSPOSEWX8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx8 = TransposeWx8_Any_AVX2; if (IS_ALIGNED(width, 16)) { - TransposeWx8 = TransposeWx8_SSSE3; + TransposeWx8 = TransposeWx8_AVX2; } } #endif @@ -130,8 +145,8 @@ void TransposePlane(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX) - if (TransposeWxH) { - TransposeWxH(src, src_stride, dst, dst_stride, width, height); + if (TransposeWxH_fn) { + TransposeWxH_fn(src, src_stride, dst, dst_stride, width, height); return; } #endif @@ -159,7 +174,7 @@ void TransposePlane(const uint8_t* src, #endif if (i > 0) { - TransposeWxH_C(src, src_stride, dst, dst_stride, width, i); + TransposeWxH(src, src_stride, dst, dst_stride, width, i); } } @@ -237,10 +252,7 @@ void RotatePlane180(const uint8_t* src, #endif #if defined(HAS_MIRRORROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MirrorRow = MirrorRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - MirrorRow = MirrorRow_AVX512BW; - } + MirrorRow = MirrorRow_AVX512BW; } #endif #if defined(HAS_MIRRORROW_LSX) @@ -341,6 +353,24 @@ void RotatePlane180(const uint8_t* src, free_aligned_buffer_64(row); } +static void TransposeUVWxH(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width, + int height) { + int i; + for (i = 0; i < width * 2; i += 2) { + int j; + for (j = 0; j < height; ++j) { + dst_a[((i >> 1) * dst_stride_a) + j] = src[i + (j * src_stride)]; + dst_b[((i >> 1) * dst_stride_b) + j] = src[i + (j * src_stride) + 1]; + } + } +} + LIBYUV_API void SplitTransposeUV(const uint8_t* src, int src_stride, @@ -352,9 +382,9 @@ void SplitTransposeUV(const uint8_t* src, int height) { int i = height; #if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX) - void (*TransposeUVWxH)(const uint8_t* src, int src_stride, uint8_t* dst_a, - int dst_stride_a, uint8_t* dst_b, int dst_stride_b, - int width, int height) = NULL; + void (*TransposeUVWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst_a, + int dst_stride_a, uint8_t* dst_b, int dst_stride_b, + int width, int height) = NULL; #endif #if defined(HAS_TRANSPOSEUVWX16_AVX2) || \ defined(HAS_TRANSPOSEUVWX16_AVX512BW) || \ @@ -393,19 +423,19 @@ void SplitTransposeUV(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEUVWXH_SME) if (TestCpuFlag(kCpuHasSME)) { - TransposeUVWxH = TransposeUVWxH_SME; + TransposeUVWxH_fn = TransposeUVWxH_SME; } #endif #if defined(HAS_TRANSPOSEUVWXH_HVX) if (TestCpuFlag(kCpuHasHVX)) { - TransposeUVWxH = TransposeUVWxH_HVX; + TransposeUVWxH_fn = TransposeUVWxH_HVX; } #endif -#if defined(HAS_TRANSPOSEUVWX8_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - TransposeUVWx8 = TransposeUVWx8_Any_SSE2; +#if defined(HAS_TRANSPOSEUVWX8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeUVWx8 = TransposeUVWx8_Any_AVX2; if (IS_ALIGNED(width, 8)) { - TransposeUVWx8 = TransposeUVWx8_SSE2; + TransposeUVWx8 = TransposeUVWx8_AVX2; } } #endif @@ -440,9 +470,9 @@ void SplitTransposeUV(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX) - if (TransposeUVWxH) { - TransposeUVWxH(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, - width, i); + if (TransposeUVWxH_fn) { + TransposeUVWxH_fn(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, + width, i); return; } #endif @@ -472,8 +502,8 @@ void SplitTransposeUV(const uint8_t* src, } if (i > 0) { - TransposeUVWxH_C(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, - width, i); + TransposeUVWxH(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, + width, i); } } @@ -657,6 +687,21 @@ int RotatePlane(const uint8_t* src, return -1; } +static void TransposeWxH_16(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height) { + int i; + for (i = 0; i < width; ++i) { + int j; + for (j = 0; j < height; ++j) { + dst[i * dst_stride + j] = src[j * src_stride + i]; + } + } +} + static void TransposePlane_16(const uint16_t* src, int src_stride, uint16_t* dst, @@ -666,8 +711,8 @@ static void TransposePlane_16(const uint16_t* src, int i = height; #if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \ defined(HAS_TRANSPOSEWXH_16_HVX) - void (*TransposeWxH_16)(const uint16_t* src, int src_stride, uint16_t* dst, - int dst_stride, int width, int height) = NULL; + void (*TransposeWxH_16_fn)(const uint16_t* src, int src_stride, uint16_t* dst, + int dst_stride, int width, int height) = NULL; #endif #if defined(HAS_TRANSPOSEWX32_16_AVX512BW) void (*TransposeWx32_16)(const uint16_t* src, int src_stride, uint16_t* dst, @@ -695,17 +740,25 @@ static void TransposePlane_16(const uint16_t* src, #endif #if defined(HAS_TRANSPOSEWXH_16_SME) if (TestCpuFlag(kCpuHasSME)) { - TransposeWxH_16 = TransposeWxH_16_SME; + TransposeWxH_16_fn = TransposeWxH_16_SME; } #endif #if defined(HAS_TRANSPOSEWXH_16_RVV) if (TestCpuFlag(kCpuHasRVV)) { - TransposeWxH_16 = TransposeWxH_16_RVV; + TransposeWxH_16_fn = TransposeWxH_16_RVV; } #endif #if defined(HAS_TRANSPOSEWXH_16_HVX) if (TestCpuFlag(kCpuHasHVX)) { - TransposeWxH_16 = TransposeWxH_16_HVX; + TransposeWxH_16_fn = TransposeWxH_16_HVX; + } +#endif +#if defined(HAS_TRANSPOSEWX8_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx8_16 = TransposeWx8_16_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + TransposeWx8_16 = TransposeWx8_16_AVX2; + } } #endif #if defined(HAS_TRANSPOSEWX16_16_AVX2) @@ -748,8 +801,8 @@ static void TransposePlane_16(const uint16_t* src, #if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \ defined(HAS_TRANSPOSEWXH_16_HVX) - if (TransposeWxH_16) { - TransposeWxH_16(src, src_stride, dst, dst_stride, width, height); + if (TransposeWxH_16_fn) { + TransposeWxH_16_fn(src, src_stride, dst, dst_stride, width, height); return; } #endif @@ -782,7 +835,7 @@ static void TransposePlane_16(const uint16_t* src, } if (i > 0) { - TransposeWxH_16_C(src, src_stride, dst, dst_stride, width, i); + TransposeWxH_16(src, src_stride, dst, dst_stride, width, i); } } diff --git a/source/rotate_any.cc b/source/rotate_any.cc index 499047cb4..bc203655a 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -34,8 +34,8 @@ TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7) #ifdef HAS_TRANSPOSEWX16_NEON TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15) #endif -#ifdef HAS_TRANSPOSEWX8_SSSE3 -TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15) +#ifdef HAS_TRANSPOSEWX8_AVX2 +TANY(TransposeWx8_Any_AVX2, TransposeWx8_AVX2, TransposeWx8_C, 15) #endif #ifdef HAS_TRANSPOSEWX16_AVX2 TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15) @@ -71,8 +71,8 @@ TANY(TransposeWx16_Any_WASMSIMD, TransposeWx16_WASMSIMD, TransposeWx16_C, 15) #ifdef HAS_TRANSPOSEUVWX8_NEON TUVANY(TransposeUVWx8_Any_NEON, TransposeUVWx8_NEON, TransposeUVWx8_C, 7) #endif -#ifdef HAS_TRANSPOSEUVWX8_SSE2 -TUVANY(TransposeUVWx8_Any_SSE2, TransposeUVWx8_SSE2, TransposeUVWx8_C, 7) +#ifdef HAS_TRANSPOSEUVWX8_AVX2 +TUVANY(TransposeUVWx8_Any_AVX2, TransposeUVWx8_AVX2, TransposeUVWx8_C, 7) #endif #ifdef HAS_TRANSPOSEUVWX16_AVX2 TUVANY(TransposeUVWx16_Any_AVX2, TransposeUVWx16_AVX2, TransposeUVWx16_C, 7) @@ -111,8 +111,8 @@ TUVANY(TransposeUVWx16_Any_WASMSIMD, } \ } -#ifdef HAS_TRANSPOSEWX4_32_SSE2 -TANY32(TransposeWx4_32_Any_SSE2, TransposeWx4_32_SSE2, TransposeWxH_32_C, 3) +#ifdef HAS_TRANSPOSEWX4_32_AVX2 +TANY32(TransposeWx4_32_Any_AVX2, TransposeWx4_32_AVX2, TransposeWxH_32_C, 3) #endif #ifdef HAS_TRANSPOSEWX4_32_LSX TANY32(TransposeWx4_32_Any_LSX, TransposeWx4_32_LSX, TransposeWxH_32_C, 3) @@ -155,13 +155,20 @@ TANY32_SRC(TransposeWx8_32_Any_AVX2, TransposeWx8_32_AVX2, 3, 8) TPOS_SIMD(src, src_stride, dst, dst_stride, n); \ } \ ptrdiff_t np = n; \ - TransposeWxH_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \ - r, HEIGHT); \ + TransposeWx8_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \ + r); \ + if ((HEIGHT) == 16) { \ + TransposeWx8_16_C(src + np + 8 * src_stride, src_stride, \ + dst + np * dst_stride + 8, dst_stride, r); \ + } \ } #ifdef HAS_TRANSPOSEWX8_16_NEON TANY16(TransposeWx8_16_Any_NEON, TransposeWx8_16_NEON, 7, 8) #endif +#ifdef HAS_TRANSPOSEWX8_16_AVX2 +TANY16(TransposeWx8_16_Any_AVX2, TransposeWx8_16_AVX2, 7, 8) +#endif #ifdef HAS_TRANSPOSEWX16_16_AVX2 TANY16(TransposeWx16_16_Any_AVX2, TransposeWx16_16_AVX2, 7, 16) #endif diff --git a/source/rotate_argb.cc b/source/rotate_argb.cc index 00900d175..1b0bf7417 100644 --- a/source/rotate_argb.cc +++ b/source/rotate_argb.cc @@ -31,7 +31,8 @@ static int ARGBTranspose(const uint8_t* src_argb, int width, int height) { #if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \ - defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) + defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \ + defined(HAS_TRANSPOSEWXH_32_AVX2) void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width, int height) = NULL; #endif @@ -49,11 +50,11 @@ static int ARGBTranspose(const uint8_t* src_argb, if (src_stride_argb & 3) { return -1; } -#if defined(HAS_TRANSPOSEWX4_32_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - TransposeWx4_32 = TransposeWx4_32_Any_SSE2; +#if defined(HAS_TRANSPOSEWX4_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx4_32 = TransposeWx4_32_Any_AVX2; if (IS_ALIGNED(height, 4)) { // Width of dest. - TransposeWx4_32 = TransposeWx4_32_SSE2; + TransposeWx4_32 = TransposeWx4_32_AVX2; } } #endif @@ -65,6 +66,11 @@ static int ARGBTranspose(const uint8_t* src_argb, } } #endif +#if defined(HAS_TRANSPOSEWXH_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWxH_32 = TransposeWxH_32_AVX2; + } +#endif #if defined(HAS_TRANSPOSEWX16_32_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { TransposeWx16_32 = TransposeWx16_32_AVX512BW; @@ -123,14 +129,6 @@ static int ARGBTranspose(const uint8_t* src_argb, } #endif -#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \ - defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) - if (TransposeWxH_32) { - TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width, - height); - return 0; - } -#endif #if defined(HAS_TRANSPOSEWX16_32_AVX512BW) if (TransposeWx16_32) { while (height >= 16) { @@ -142,6 +140,17 @@ static int ARGBTranspose(const uint8_t* src_argb, } } #endif +#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \ + defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \ + defined(HAS_TRANSPOSEWXH_32_AVX2) + if (TransposeWxH_32) { + if (height > 0) { + TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, + width, height); + } + return 0; + } +#endif #if defined(HAS_TRANSPOSEWX8_32_AVX2) if (TransposeWx8_32) { while (height >= 8) { diff --git a/source/rotate_common.cc b/source/rotate_common.cc index 0abab4f7e..e4aab102f 100644 --- a/source/rotate_common.cc +++ b/source/rotate_common.cc @@ -90,39 +90,6 @@ void TransposeUVWx16_C(const uint8_t* src, dst_stride_a, (dst_b + 8), dst_stride_b, width); } -void TransposeWxH_C(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width, - int height) { - int i; - for (i = 0; i < width; ++i) { - int j; - for (j = 0; j < height; ++j) { - dst[i * dst_stride + j] = src[j * src_stride + i]; - } - } -} - -void TransposeUVWxH_C(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width, - int height) { - int i; - for (i = 0; i < width * 2; i += 2) { - int j; - for (j = 0; j < height; ++j) { - dst_a[((i >> 1) * dst_stride_a) + j] = src[i + (j * src_stride)]; - dst_b[((i >> 1) * dst_stride_b) + j] = src[i + (j * src_stride) + 1]; - } - } -} - void TransposeWx8_16_C(const uint16_t* src, int src_stride, uint16_t* dst, @@ -143,21 +110,6 @@ void TransposeWx8_16_C(const uint16_t* src, } } -void TransposeWxH_16_C(const uint16_t* src, - int src_stride, - uint16_t* dst, - int dst_stride, - int width, - int height) { - int i; - for (i = 0; i < width; ++i) { - int j; - for (j = 0; j < height; ++j) { - dst[i * dst_stride + j] = src[j * src_stride + i]; - } - } -} - static inline void Copy4(uint8_t* dst, const uint8_t* src) { dst[0] = src[0]; dst[1] = src[1]; diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index d9456a842..251054f2a 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -11,6 +11,8 @@ #include "libyuv/rotate_row.h" #include "libyuv/row.h" +#include + #ifdef __cplusplus namespace libyuv { extern "C" { @@ -23,152 +25,135 @@ extern "C" { // asm statement, and other OS ABIs (e.g. Windows) differ in which registers // are callee-saved. List "memory" first, then "cc", then registers. -// This module is for GCC and clang x64. 32 bit x86 uses rotate_win.cc. -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) +// This module is for GCC and clang x86 and x64. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ + (!defined(_MSC_VER) || defined(__clang__)) -#if defined(HAS_TRANSPOSEWX8_SSSE3) -// Transpose 16x8. 64 bit. -void TransposeWx8_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { +#if defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2) +// Transpose 16x8 bytes using 2 lanes of AVX2. Source row r is in lane r / 4 +// of ymm(r % 4). Byte and word unpacks produce 4-row column chunks in each +// lane; vperm2i128 + dword unpack join rows 0..3 and 4..7. Each 128-bit lane +// holds 8 bytes for dst_a (low 64 bits) and 8 bytes for dst_b (high 64 bits). +// Width is a multiple of 16 bytes. +static void TransposeWx8_Byte_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int byte_width) { + const ptrdiff_t d_a = dst_stride_a; + const ptrdiff_t d_b = dst_stride_b; + uintptr_t temp; asm volatile( - // Read in the data from the source pointer. - // First round of bit swap. - LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%0,%3),%%xmm1 \n" - "lea (%0,%3,2),%0 \n" - "movdqa %%xmm0,%%xmm8 \n" - "punpcklbw %%xmm1,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm8 \n" - "movdqu (%0),%%xmm2 \n" - "movdqa %%xmm0,%%xmm1 \n" - "movdqa %%xmm8,%%xmm9 \n" - "palignr $0x8,%%xmm1,%%xmm1 \n" - "palignr $0x8,%%xmm9,%%xmm9 \n" - "movdqu (%0,%3),%%xmm3 \n" - "lea (%0,%3,2),%0 \n" - "movdqa %%xmm2,%%xmm10 \n" - "punpcklbw %%xmm3,%%xmm2 \n" - "punpckhbw %%xmm3,%%xmm10 \n" - "movdqa %%xmm2,%%xmm3 \n" - "movdqa %%xmm10,%%xmm11 \n" - "movdqu (%0),%%xmm4 \n" - "palignr $0x8,%%xmm3,%%xmm3 \n" - "palignr $0x8,%%xmm11,%%xmm11 \n" - "movdqu (%0,%3),%%xmm5 \n" - "lea (%0,%3,2),%0 \n" - "movdqa %%xmm4,%%xmm12 \n" - "punpcklbw %%xmm5,%%xmm4 \n" - "punpckhbw %%xmm5,%%xmm12 \n" - "movdqa %%xmm4,%%xmm5 \n" - "movdqa %%xmm12,%%xmm13 \n" - "movdqu (%0),%%xmm6 \n" - "palignr $0x8,%%xmm5,%%xmm5 \n" - "palignr $0x8,%%xmm13,%%xmm13 \n" - "movdqu (%0,%3),%%xmm7 \n" - "lea (%0,%3,2),%0 \n" - "movdqa %%xmm6,%%xmm14 \n" - "punpcklbw %%xmm7,%%xmm6 \n" - "punpckhbw %%xmm7,%%xmm14 \n" - "neg %3 \n" - "movdqa %%xmm6,%%xmm7 \n" - "movdqa %%xmm14,%%xmm15 \n" - "lea 0x10(%0,%3,8),%0 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "palignr $0x8,%%xmm15,%%xmm15 \n" - "neg %3 \n" - // Second round of bit swap. - "punpcklwd %%xmm2,%%xmm0 \n" - "punpcklwd %%xmm3,%%xmm1 \n" - "movdqa %%xmm0,%%xmm2 \n" - "movdqa %%xmm1,%%xmm3 \n" - "palignr $0x8,%%xmm2,%%xmm2 \n" - "palignr $0x8,%%xmm3,%%xmm3 \n" - "punpcklwd %%xmm6,%%xmm4 \n" - "punpcklwd %%xmm7,%%xmm5 \n" - "movdqa %%xmm4,%%xmm6 \n" - "movdqa %%xmm5,%%xmm7 \n" - "palignr $0x8,%%xmm6,%%xmm6 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "punpcklwd %%xmm10,%%xmm8 \n" - "punpcklwd %%xmm11,%%xmm9 \n" - "movdqa %%xmm8,%%xmm10 \n" - "movdqa %%xmm9,%%xmm11 \n" - "palignr $0x8,%%xmm10,%%xmm10 \n" - "palignr $0x8,%%xmm11,%%xmm11 \n" - "punpcklwd %%xmm14,%%xmm12 \n" - "punpcklwd %%xmm15,%%xmm13 \n" - "movdqa %%xmm12,%%xmm14 \n" - "movdqa %%xmm13,%%xmm15 \n" - "palignr $0x8,%%xmm14,%%xmm14 \n" - "palignr $0x8,%%xmm15,%%xmm15 \n" - // Third round of bit swap. - // Write to the destination pointer. - "punpckldq %%xmm4,%%xmm0 \n" - "movq %%xmm0,(%1) \n" - "movdqa %%xmm0,%%xmm4 \n" - "palignr $0x8,%%xmm4,%%xmm4 \n" - "movq %%xmm4,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm6,%%xmm2 \n" - "movdqa %%xmm2,%%xmm6 \n" - "movq %%xmm2,(%1) \n" - "palignr $0x8,%%xmm6,%%xmm6 \n" - "punpckldq %%xmm5,%%xmm1 \n" - "movq %%xmm6,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "movdqa %%xmm1,%%xmm5 \n" - "movq %%xmm1,(%1) \n" - "palignr $0x8,%%xmm5,%%xmm5 \n" - "movq %%xmm5,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm7,%%xmm3 \n" - "movq %%xmm3,(%1) \n" - "movdqa %%xmm3,%%xmm7 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "movq %%xmm7,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm12,%%xmm8 \n" - "movq %%xmm8,(%1) \n" - "movdqa %%xmm8,%%xmm12 \n" - "palignr $0x8,%%xmm12,%%xmm12 \n" - "movq %%xmm12,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm14,%%xmm10 \n" - "movdqa %%xmm10,%%xmm14 \n" - "movq %%xmm10,(%1) \n" - "palignr $0x8,%%xmm14,%%xmm14 \n" - "punpckldq %%xmm13,%%xmm9 \n" - "movq %%xmm14,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "movdqa %%xmm9,%%xmm13 \n" - "movq %%xmm9,(%1) \n" - "palignr $0x8,%%xmm13,%%xmm13 \n" - "movq %%xmm13,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm15,%%xmm11 \n" - "movq %%xmm11,(%1) \n" - "movdqa %%xmm11,%%xmm15 \n" - "palignr $0x8,%%xmm15,%%xmm15 \n" - "sub $0x10,%2 \n" - "movq %%xmm15,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[src_stride]),%%xmm1 \n" + "lea (%[src],%[src_stride],2),%[temp] \n" + "lea 0x10(%[src]),%[src] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[src_stride]),%%xmm3 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[src_stride]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[src_stride]),%%ymm3,%%ymm3 \n" + + "vpunpcklbw %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckhbw %%ymm1,%%ymm0,%%ymm5 \n" + "vpunpcklbw %%ymm3,%%ymm2,%%ymm6 \n" + "vpunpckhbw %%ymm3,%%ymm2,%%ymm7 \n" + + "vpunpcklwd %%ymm6,%%ymm4,%%ymm0 \n" + "vpunpckhwd %%ymm6,%%ymm4,%%ymm1 \n" + "vpunpcklwd %%ymm7,%%ymm5,%%ymm2 \n" + "vpunpckhwd %%ymm7,%%ymm5,%%ymm3 \n" + + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm4 \n" + "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm5 \n" + "vperm2i128 $0x20,%%ymm3,%%ymm2,%%ymm6 \n" + "vperm2i128 $0x31,%%ymm3,%%ymm2,%%ymm7 \n" + + "vpunpckldq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpckldq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhdq %%ymm7,%%ymm6,%%ymm3 \n" + + "vmovq %%xmm0,(%[dst_a]) \n" + "vmovhpd %%xmm0,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm1,(%[dst_a]) \n" + "vmovhpd %%xmm1,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vextracti128 $1,%%ymm0,%%xmm4 \n" + "vextracti128 $1,%%ymm1,%%xmm5 \n" + "vmovq %%xmm4,(%[dst_a]) \n" + "vmovhpd %%xmm4,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm5,(%[dst_a]) \n" + "vmovhpd %%xmm5,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm2,(%[dst_a]) \n" + "vmovhpd %%xmm2,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm3,(%[dst_a]) \n" + "vmovhpd %%xmm3,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vextracti128 $1,%%ymm2,%%xmm6 \n" + "vextracti128 $1,%%ymm3,%%xmm7 \n" + "vmovq %%xmm6,(%[dst_a]) \n" + "vmovhpd %%xmm6,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm7,(%[dst_a]) \n" + "vmovhpd %%xmm7,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "subl $0x10,%[width] \n" "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "r"((ptrdiff_t)(src_stride)), // %3 - "r"((ptrdiff_t)(dst_stride)) // %4 + "vzeroupper \n" + : [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b), + [width] "+rm"(byte_width), [temp] "=&r"(temp) + : [src_stride] "r"((ptrdiff_t)src_stride), + [dst_stride_a] "rm"(d_a), + [dst_stride_b] "rm"(d_b) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", - "xmm15"); + "xmm7"); } -#endif // defined(HAS_TRANSPOSEWX8_SSSE3) +#endif // defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2) + +#if defined(HAS_TRANSPOSEWX8_AVX2) +void TransposeWx8_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx8_Byte_AVX2(src, src_stride, dst, dst_stride * 2, dst + dst_stride, + dst_stride * 2, width); +} +#endif // defined(HAS_TRANSPOSEWX8_AVX2) + +#if defined(HAS_TRANSPOSEUVWX8_AVX2) +void TransposeUVWx8_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeWx8_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b, + dst_stride_b, width * 2); +} +#endif // defined(HAS_TRANSPOSEUVWX8_AVX2) #if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW) // Dword permutes to gather 16 rows of a column from 4 lanes. @@ -411,6 +396,7 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW) +#if defined(__x86_64__) #if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2) // Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8 // of ymm(r % 8). 3 unpack stages leave each lane with 2 columns x 8 rows, and @@ -549,122 +535,37 @@ void TransposeUVWx16_AVX2(const uint8_t* src, dst_stride_b, width * 2); } #endif // defined(HAS_TRANSPOSEUVWX16_AVX2) - -// Transpose UV 8x8. 64 bit. -#if defined(HAS_TRANSPOSEUVWX8_SSE2) -void TransposeUVWx8_SSE2(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width) { - asm volatile( - // Read in the data from the source pointer. - // First round of bit swap. - LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%0,%4),%%xmm1 \n" - "lea (%0,%4,2),%0 \n" - "movdqa %%xmm0,%%xmm8 \n" - "punpcklbw %%xmm1,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm8 \n" - "movdqa %%xmm8,%%xmm1 \n" - "movdqu (%0),%%xmm2 \n" - "movdqu (%0,%4),%%xmm3 \n" - "lea (%0,%4,2),%0 \n" - "movdqa %%xmm2,%%xmm8 \n" - "punpcklbw %%xmm3,%%xmm2 \n" - "punpckhbw %%xmm3,%%xmm8 \n" - "movdqa %%xmm8,%%xmm3 \n" - "movdqu (%0),%%xmm4 \n" - "movdqu (%0,%4),%%xmm5 \n" - "lea (%0,%4,2),%0 \n" - "movdqa %%xmm4,%%xmm8 \n" - "punpcklbw %%xmm5,%%xmm4 \n" - "punpckhbw %%xmm5,%%xmm8 \n" - "movdqa %%xmm8,%%xmm5 \n" - "movdqu (%0),%%xmm6 \n" - "movdqu (%0,%4),%%xmm7 \n" - "lea (%0,%4,2),%0 \n" - "movdqa %%xmm6,%%xmm8 \n" - "punpcklbw %%xmm7,%%xmm6 \n" - "neg %4 \n" - "lea 0x10(%0,%4,8),%0 \n" - "punpckhbw %%xmm7,%%xmm8 \n" - "movdqa %%xmm8,%%xmm7 \n" - "neg %4 \n" - // Second round of bit swap. - "movdqa %%xmm0,%%xmm8 \n" - "movdqa %%xmm1,%%xmm9 \n" - "punpckhwd %%xmm2,%%xmm8 \n" - "punpckhwd %%xmm3,%%xmm9 \n" - "punpcklwd %%xmm2,%%xmm0 \n" - "punpcklwd %%xmm3,%%xmm1 \n" - "movdqa %%xmm8,%%xmm2 \n" - "movdqa %%xmm9,%%xmm3 \n" - "movdqa %%xmm4,%%xmm8 \n" - "movdqa %%xmm5,%%xmm9 \n" - "punpckhwd %%xmm6,%%xmm8 \n" - "punpckhwd %%xmm7,%%xmm9 \n" - "punpcklwd %%xmm6,%%xmm4 \n" - "punpcklwd %%xmm7,%%xmm5 \n" - "movdqa %%xmm8,%%xmm6 \n" - "movdqa %%xmm9,%%xmm7 \n" - // Third round of bit swap. - // Write to the destination pointer. - "movdqa %%xmm0,%%xmm8 \n" - "punpckldq %%xmm4,%%xmm0 \n" - "movlpd %%xmm0,(%1) \n" // Write back U channel - "movhpd %%xmm0,(%2) \n" // Write back V channel - "punpckhdq %%xmm4,%%xmm8 \n" - "movlpd %%xmm8,(%1,%5) \n" - "lea (%1,%5,2),%1 \n" - "movhpd %%xmm8,(%2,%6) \n" - "lea (%2,%6,2),%2 \n" - "movdqa %%xmm2,%%xmm8 \n" - "punpckldq %%xmm6,%%xmm2 \n" - "movlpd %%xmm2,(%1) \n" - "movhpd %%xmm2,(%2) \n" - "punpckhdq %%xmm6,%%xmm8 \n" - "movlpd %%xmm8,(%1,%5) \n" - "lea (%1,%5,2),%1 \n" - "movhpd %%xmm8,(%2,%6) \n" - "lea (%2,%6,2),%2 \n" - "movdqa %%xmm1,%%xmm8 \n" - "punpckldq %%xmm5,%%xmm1 \n" - "movlpd %%xmm1,(%1) \n" - "movhpd %%xmm1,(%2) \n" - "punpckhdq %%xmm5,%%xmm8 \n" - "movlpd %%xmm8,(%1,%5) \n" - "lea (%1,%5,2),%1 \n" - "movhpd %%xmm8,(%2,%6) \n" - "lea (%2,%6,2),%2 \n" - "movdqa %%xmm3,%%xmm8 \n" - "punpckldq %%xmm7,%%xmm3 \n" - "movlpd %%xmm3,(%1) \n" - "movhpd %%xmm3,(%2) \n" - "punpckhdq %%xmm7,%%xmm8 \n" - "sub $0x8,%3 \n" - "movlpd %%xmm8,(%1,%5) \n" - "lea (%1,%5,2),%1 \n" - "movhpd %%xmm8,(%2,%6) \n" - "lea (%2,%6,2),%2 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst_a), // %1 - "+r"(dst_b), // %2 - "+r"(width) // %3 - : "r"((ptrdiff_t)(src_stride)), // %4 - "r"((ptrdiff_t)(dst_stride_a)), // %5 - "r"((ptrdiff_t)(dst_stride_b)) // %6 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7", "xmm8", "xmm9"); +#else // !defined(__x86_64__) +#if defined(HAS_TRANSPOSEWX16_AVX2) +void TransposeWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx8_AVX2(src, src_stride, dst, dst_stride, width); + TransposeWx8_AVX2(src + 8 * (ptrdiff_t)src_stride, src_stride, dst + 8, + dst_stride, width); } -#endif // defined(HAS_TRANSPOSEUVWX8_SSE2) +#endif // defined(HAS_TRANSPOSEWX16_AVX2) + +#if defined(HAS_TRANSPOSEUVWX16_AVX2) +void TransposeUVWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeUVWx8_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, + width); + TransposeUVWx8_AVX2(src + 8 * (ptrdiff_t)src_stride, src_stride, dst_a + 8, + dst_stride_a, dst_b + 8, dst_stride_b, width); +} +#endif // defined(HAS_TRANSPOSEUVWX16_AVX2) +#endif // defined(__x86_64__) #if defined(HAS_TRANSPOSEWX16_16_AVX2) +#if defined(__x86_64__) void TransposeWx16_16_AVX2(const uint16_t* src, int src_stride, uint16_t* dst, @@ -758,8 +659,92 @@ void TransposeWx16_16_AVX2(const uint16_t* src, "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); } +#endif // defined(__x86_64__) #endif // defined(HAS_TRANSPOSEWX16_16_AVX2) +#if defined(HAS_TRANSPOSEWX8_16_AVX2) +void TransposeWx8_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n" + "add $0x10,%[src] \n" + + "vpunpcklwd %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckhwd %%ymm1,%%ymm0,%%ymm5 \n" + "vpunpcklwd %%ymm3,%%ymm2,%%ymm6 \n" + "vpunpckhwd %%ymm3,%%ymm2,%%ymm7 \n" + + "vpunpckldq %%ymm6,%%ymm4,%%ymm0 \n" + "vpunpckhdq %%ymm6,%%ymm4,%%ymm1 \n" + "vpunpckldq %%ymm7,%%ymm5,%%ymm2 \n" + "vpunpckhdq %%ymm7,%%ymm5,%%ymm3 \n" + + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vpermq $0xd8,%%ymm3,%%ymm3 \n" + + "vmovdqu %%xmm0,(%[dst]) \n" + "vextracti128 $1,%%ymm0,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%xmm1,(%[dst]) \n" + "vextracti128 $1,%%ymm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%xmm2,(%[dst]) \n" + "vextracti128 $1,%%ymm2,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%xmm3,(%[dst]) \n" + "vextracti128 $1,%%ymm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "subl $0x8,%[w] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_b), + [dst] "+r"(dst_b), +#if defined(__i386__) + [w] "+m"(width), +#else + [w] "+rm"(width), +#endif + [temp] "=&r"(temp) + : [s] "r"(s), + [d] "r"(d) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // defined(HAS_TRANSPOSEWX8_16_AVX2) + +#if defined(HAS_TRANSPOSEWX16_16_AVX2) && !defined(__x86_64__) +void TransposeWx16_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + TransposeWx8_16_AVX2(src, src_stride, dst, dst_stride, width); + TransposeWx8_16_AVX2(src + 8 * (ptrdiff_t)src_stride, src_stride, dst + 8, + dst_stride, width); +} +#endif // defined(HAS_TRANSPOSEWX16_16_AVX2) && !defined(__x86_64__) + #if defined(HAS_TRANSPOSEWX32_16_AVX512BW) // Transpose 32 rows of 16 bit plane (arbitrary width) using 8x32 tiles in // zmm0..zmm15 with k1-masked right tail. @@ -1033,78 +1018,93 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src, } #endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW) -#if defined(HAS_TRANSPOSEWX4_32_SSE2) -// 4 values, little endian view -// a b c d -// e f g h -// i j k l -// m n o p - -// transpose 2x2 -// a e b f from row 0, 1 -// i m j n from row 2, 3 -// c g d h from row 0, 1 -// k o l p from row 2, 3 - -// transpose 4x4 -// a e i m from row 0, 1 -// b f j n from row 0, 1 -// c g k o from row 2, 3 -// d h l p from row 2, 3 - -// Transpose 32 bit values (ARGB) -void TransposeWx4_32_SSE2(const uint8_t* src, +#if defined(HAS_TRANSPOSEWX4_32_AVX2) +// Transpose 32 bit values (ARGB). Read a column of 4 pixels across 8 rows per +// AVX2 iteration (with a 4-row VEX 128-bit tail) and write 4 destination rows. +void TransposeWx4_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + uintptr_t temp; asm volatile( - // Main loop transpose 4x4. Read a column, write a row. - "1: \n" - "movdqu (%0),%%xmm0 \n" // a b c d - "movdqu (%0,%3),%%xmm1 \n" // e f g h - "lea (%0,%3,2),%0 \n" // src += stride * 2 - "movdqu (%0),%%xmm2 \n" // i j k l - "movdqu (%0,%3),%%xmm3 \n" // m n o p - "lea (%0,%3,2),%0 \n" // src += stride * 2 + "subl $0x8,%[width] \n" + "jl 2f \n" - // Transpose 2x2 - "movdqa %%xmm0,%%xmm4 \n" - "movdqa %%xmm2,%%xmm5 \n" - "movdqa %%xmm0,%%xmm6 \n" - "movdqa %%xmm2,%%xmm7 \n" - "punpckldq %%xmm1,%%xmm4 \n" // a e b f from row 0, 1 - "punpckldq %%xmm3,%%xmm5 \n" // i m j n from row 2, 3 - "punpckhdq %%xmm1,%%xmm6 \n" // c g d h from row 0, 1 - "punpckhdq %%xmm3,%%xmm7 \n" // k o l p from row 2, 3 + // Main loop: 8 source rows of 4 pixels -> 4 dst rows of 8 pixels (32 B). + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n" + "lea (%[temp],%[s],2),%[src] \n" - // Transpose 4x4 - "movdqa %%xmm4,%%xmm0 \n" - "movdqa %%xmm4,%%xmm1 \n" - "movdqa %%xmm6,%%xmm2 \n" - "movdqa %%xmm6,%%xmm3 \n" - "punpcklqdq %%xmm5,%%xmm0 \n" // a e i m from row 0, 1 - "punpckhqdq %%xmm5,%%xmm1 \n" // b f j n from row 0, 1 - "punpcklqdq %%xmm7,%%xmm2 \n" // c g k o from row 2, 3 - "punpckhqdq %%xmm7,%%xmm3 \n" // d h l p from row 2, 3 + "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" - "movdqu %%xmm0,(%1) \n" - "lea 16(%1,%4),%1 \n" // dst += stride + 16 - "movdqu %%xmm1,-16(%1) \n" - "movdqu %%xmm2,-16(%1,%4) \n" - "movdqu %%xmm3,-16(%1,%4,2) \n" - "sub %4,%1 \n" - "sub $0x4,%2 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+rm"(width) // %2 - : "r"((ptrdiff_t)(src_stride)), // %3 - "r"((ptrdiff_t)(dst_stride)) // %4 + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" + + "vmovdqu %%ymm0,(%[dst]) \n" + "vmovdqu %%ymm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[temp] \n" + "vmovdqu %%ymm2,(%[temp]) \n" + "vmovdqu %%ymm3,(%[temp],%[d]) \n" + "add $0x20,%[dst] \n" + "subl $0x8,%[width] \n" + "jge 1b \n" + + // Tail: 4 source rows of 4 pixels -> 4 dst rows of 4 pixels (16 B). + "2: \n" + "addl $0x8,%[width] \n" + "je 99f \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" + + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" + "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" + + "vmovdqu %%xmm0,(%[dst]) \n" + "vmovdqu %%xmm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[temp] \n" + "vmovdqu %%xmm2,(%[temp]) \n" + "vmovdqu %%xmm3,(%[temp],%[d]) \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [width] "+rm"(width), // %[width] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } -#endif // defined(HAS_TRANSPOSEWX4_32_SSE2) +#endif // defined(HAS_TRANSPOSEWX4_32_AVX2) #if defined(HAS_TRANSPOSEWX8_32_AVX2) // Transpose 8 rows of 32 bit plane (width multiple of 4) using 4x8 YMM vertical @@ -1152,20 +1152,404 @@ void TransposeWx8_32_AVX2(const uint8_t* src, "vmovdqu %%ymm2,(%[dst]) \n" "vmovdqu %%ymm3,(%[dst],%[d]) \n" "lea (%[dst],%[d],2),%[dst] \n" - "sub $0x4,%[width] \n" + "subl $0x4,%[width] \n" "jg 1b \n" "vzeroupper \n" - : [src] "+r"(src), // %[src] - [dst] "+r"(dst), // %[dst] - [width] "+r"(width), // %[width] - [temp] "=&r"(temp) // %[temp] - : [s] "r"(s), // %[s] - [d] "r"(d) // %[d] + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [width] "+rm"(width), // %[width] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } #endif // defined(HAS_TRANSPOSEWX8_32_AVX2) +#if defined(HAS_TRANSPOSEWXH_32_AVX2) +// Transpose rem_w (1..3) columns x 8 rows of 32-bit pixels using AVX2. +static void TransposeRemWx8_32_AVX2(const uint8_t* src, + ptrdiff_t s, + uint8_t* dst, + ptrdiff_t d, + int rem_w) { + const uint8_t* temp; + asm volatile( + "cmpl $2,%[rem_w] \n" + "jl 1f \n" + "je 2f \n" + + // rem_w == 3: load 12 bytes per row from 8 rows. + "vmovq (%[src]),%%xmm0 \n" + "vpinsrd $2,8(%[src]),%%xmm0,%%xmm0 \n" + "vmovq (%[src],%[s]),%%xmm1 \n" + "vpinsrd $2,8(%[src],%[s]),%%xmm1,%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm2 \n" + "vpinsrd $2,8(%[temp]),%%xmm2,%%xmm2 \n" + "vmovq (%[temp],%[s]),%%xmm3 \n" + "vpinsrd $2,8(%[temp],%[s]),%%xmm3,%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm4 \n" + "vpinsrd $2,8(%[temp]),%%xmm4,%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n" + "vmovq (%[temp],%[s]),%%xmm5 \n" + "vpinsrd $2,8(%[temp],%[s]),%%xmm5,%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm4 \n" + "vpinsrd $2,8(%[temp]),%%xmm4,%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n" + "vmovq (%[temp],%[s]),%%xmm5 \n" + "vpinsrd $2,8(%[temp],%[s]),%%xmm5,%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm3,%%ymm3 \n" + "jmp 3f \n" + + // rem_w == 2: load 8 bytes per row from 8 rows. + "2: \n" + "vmovq (%[src]),%%xmm0 \n" + "vmovq (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm2 \n" + "vmovq (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n" + "vmovq (%[temp],%[s]),%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n" + "vmovq (%[temp],%[s]),%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm3,%%ymm3 \n" + "jmp 3f \n" + + // rem_w == 1: load 4 bytes per row from 8 rows. + "1: \n" + "vmovd (%[src]),%%xmm0 \n" + "vmovd (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovd (%[temp]),%%xmm2 \n" + "vmovd (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovd (%[temp]),%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n" + "vmovd (%[temp],%[s]),%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovd (%[temp]),%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n" + "vmovd (%[temp],%[s]),%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm3,%%ymm3 \n" + + "3: \n" + "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm6 \n" + "vmovdqu %%ymm6,(%[dst]) \n" + "cmpl $1,%[rem_w] \n" + "jle 4f \n" + "vpunpckhqdq %%ymm5,%%ymm4,%%ymm7 \n" + "vmovdqu %%ymm7,(%[dst],%[d]) \n" + "cmpl $2,%[rem_w] \n" + "jle 4f \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm6 \n" + "vmovdqu %%ymm6,(%[dst],%[d],2) \n" + "4: \n" + "vzeroupper \n" + : [temp] "=&r"(temp) + : [src] "r"(src), + [s] "r"(s), + [dst] "r"(dst), + [d] "r"(d), + [rem_w] "rm"(rem_w) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} + +// Transpose rem_w (1..3) columns x 4 rows of 32-bit pixels using VEX 128-bit. +static void TransposeRemWx4_32_AVX2(const uint8_t* src, + ptrdiff_t s, + uint8_t* dst, + ptrdiff_t d, + int rem_w) { + const uint8_t* temp; + asm volatile( + "cmpl $2,%[rem_w] \n" + "jl 1f \n" + "je 2f \n" + + // rem_w == 3: load 12 bytes per row from 4 rows. + "vmovq (%[src]),%%xmm0 \n" + "vpinsrd $2,8(%[src]),%%xmm0,%%xmm0 \n" + "vmovq (%[src],%[s]),%%xmm1 \n" + "vpinsrd $2,8(%[src],%[s]),%%xmm1,%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm2 \n" + "vpinsrd $2,8(%[temp]),%%xmm2,%%xmm2 \n" + "vmovq (%[temp],%[s]),%%xmm3 \n" + "vpinsrd $2,8(%[temp],%[s]),%%xmm3,%%xmm3 \n" + "jmp 3f \n" + + // rem_w == 2: load 8 bytes per row from 4 rows. + "2: \n" + "vmovq (%[src]),%%xmm0 \n" + "vmovq (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm2 \n" + "vmovq (%[temp],%[s]),%%xmm3 \n" + "jmp 3f \n" + + // rem_w == 1: load 4 bytes per row from 4 rows. + "1: \n" + "vmovd (%[src]),%%xmm0 \n" + "vmovd (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovd (%[temp]),%%xmm2 \n" + "vmovd (%[temp],%[s]),%%xmm3 \n" + + "3: \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm6 \n" + "vmovdqu %%xmm6,(%[dst]) \n" + "cmpl $1,%[rem_w] \n" + "jle 4f \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm7 \n" + "vmovdqu %%xmm7,(%[dst],%[d]) \n" + "cmpl $2,%[rem_w] \n" + "jle 4f \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm6 \n" + "vmovdqu %%xmm6,(%[dst],%[d],2) \n" + "4: \n" + : [temp] "=&r"(temp) + : [src] "r"(src), + [s] "r"(s), + [dst] "r"(dst), + [d] "r"(d), + [rem_w] "rm"(rem_w) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} + +// Transpose arbitrary WxH 32-bit plane using AVX2 (4x8, 8x4, 4x4, and 4x{1..3} +// tiles plus exact remainder handling). +void TransposeWxH_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + const int w4 = width & ~3; + const int rem_w = width & 3; + + // Tier 1: 8-row strips using 4x8 YMM tiles + rem_w x 8 YMM tail. + while (height >= 8) { + if (w4 > 0) { + TransposeWx8_32_AVX2(src, src_stride, dst, dst_stride, w4); + } + if (rem_w > 0) { + TransposeRemWx8_32_AVX2(src + (ptrdiff_t)w4 * 4, s, + dst + (ptrdiff_t)w4 * d, d, rem_w); + } + src += 8 * s; + dst += 32; + height -= 8; + } + + // Tier 2: 4-row strip using 8x4 YMM tiles + 4x4 XMM block + rem_w x 4 tail. + if (height >= 4) { + const int w8 = width & ~7; + if (w8 > 0) { + const uint8_t* src_p = src; + uint8_t* dst_p = dst; + int cnt = w8; + const uint8_t* temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vmovdqu (%[src],%[s]),%%ymm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%ymm2 \n" + "vmovdqu (%[temp],%[s]),%%ymm3 \n" + "add $0x20,%[src] \n" + + "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" + + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" + + "vmovdqu %%xmm0,(%[dst]) \n" + "vmovdqu %%xmm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%xmm2,(%[dst]) \n" + "vmovdqu %%xmm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vextracti128 $1,%%ymm0,(%[dst]) \n" + "vextracti128 $1,%%ymm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vextracti128 $1,%%ymm2,(%[dst]) \n" + "vextracti128 $1,%%ymm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "subl $0x8,%[cnt] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_p), + [dst] "+r"(dst_p), + [cnt] "+rm"(cnt), + [temp] "=&r"(temp) + : [s] "r"(s), + [d] "r"(d) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); + } + if (width & 4) { + const uint8_t* src_p = src + (ptrdiff_t)w8 * 4; + uint8_t* dst_p = dst + (ptrdiff_t)w8 * d; + uintptr_t temp; + asm volatile( + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" + + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" + "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" + + "vmovdqu %%xmm0,(%[dst]) \n" + "vmovdqu %%xmm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[temp] \n" + "vmovdqu %%xmm2,(%[temp]) \n" + "vmovdqu %%xmm3,(%[temp],%[d]) \n" + : [temp] "=&r"(temp) + : [src] "r"(src_p), + [dst] "r"(dst_p), + [s] "r"(s), + [d] "r"(d) + : "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); + } + if (rem_w > 0) { + TransposeRemWx4_32_AVX2(src + (ptrdiff_t)w4 * 4, s, + dst + (ptrdiff_t)w4 * d, d, rem_w); + } + src += 4 * s; + dst += 16; + height -= 4; + } + + // Tier 3: 1..3 remaining source rows using 4 x height XMM tiles + corner. + if (height > 0) { + if (w4 > 0) { + const uint8_t* src_p = src; + uint8_t* dst_p = dst; + int cnt = w4; + if (height == 1) { + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "add $0x10,%[src] \n" + "vmovd %%xmm0,(%[dst]) \n" + "vpextrd $1,%%xmm0,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vpextrd $2,%%xmm0,(%[dst]) \n" + "vpextrd $3,%%xmm0,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x4,%[cnt] \n" + "jg 1b \n" + : [src] "+r"(src_p), + [dst] "+r"(dst_p), + [cnt] "+r"(cnt) + : [d] "r"(d) + : "memory", "cc", "xmm0"); + } else if (height == 2) { + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "add $0x10,%[src] \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm2 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm3 \n" + "vmovq %%xmm2,(%[dst]) \n" + "vmovhpd %%xmm2,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovq %%xmm3,(%[dst]) \n" + "vmovhpd %%xmm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x4,%[cnt] \n" + "jg 1b \n" + : [src] "+r"(src_p), + [dst] "+r"(dst_p), + [cnt] "+r"(cnt) + : [s] "r"(s), + [d] "r"(d) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + } else { + asm volatile( + "vpxor %%xmm3,%%xmm3,%%xmm3 \n" + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "vmovdqu (%[src],%[s],2),%%xmm2 \n" + "add $0x10,%[src] \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" + "vmovq %%xmm0,(%[dst]) \n" + "vpextrd $2,%%xmm0,8(%[dst]) \n" + "vmovq %%xmm1,(%[dst],%[d]) \n" + "vpextrd $2,%%xmm1,8(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovq %%xmm2,(%[dst]) \n" + "vpextrd $2,%%xmm2,8(%[dst]) \n" + "vpunpckhqdq %%xmm7,%%xmm6,%%xmm4 \n" + "vmovq %%xmm4,(%[dst],%[d]) \n" + "vpextrd $2,%%xmm4,8(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x4,%[cnt] \n" + "jg 1b \n" + : [src] "+r"(src_p), + [dst] "+r"(dst_p), + [cnt] "+r"(cnt) + : [s] "r"(s), + [d] "r"(d) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); + } + } + for (int x = w4; x < width; ++x) { + uint8_t* dst_row = dst + (ptrdiff_t)x * d; + for (int y = 0; y < height; ++y) { + memcpy(dst_row + y * 4, src + (ptrdiff_t)y * s + x * 4, 4); + } + } + } +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX2) + #if defined(HAS_TRANSPOSEWX16_32_AVX512BW) // Transpose 16 rows of 32 bit plane (arbitrary width) using 4x16 ZMM vertical // tiles in zmm0..zmm7 with k1-masked right tail. @@ -1318,8 +1702,8 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEWX16_32_AVX512BW) -#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && - // !defined(LIBYUV_ENABLE_ROWWIN) +#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || + // defined(__i386__)) && !defined(LIBYUV_ENABLE_ROWWIN) #ifdef __cplusplus } // extern "C" diff --git a/source/rotate_win.cc b/source/rotate_win.cc index 707286a0f..d1d48dfea 100644 --- a/source/rotate_win.cc +++ b/source/rotate_win.cc @@ -11,16 +11,17 @@ #include "libyuv/rotate_row.h" #include "libyuv/row.h" -// This module is for 32 bit x86 with any compiler, Visual C 64 bit, and for -// clang with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__i386__) || defined(_M_IX86) || \ - ((defined(__x86_64__) || defined(_M_X64)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)))) +// This module is for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) #include #include // For AVX2 and AVX512 intrinsics +#include #if defined(_MSC_VER) && !defined(__clang__) #include #endif @@ -33,22 +34,21 @@ extern "C" { // Target attributes allow gcc and clang to build each function for its // instruction set regardless of the baseline, which may lack SSE2 for 32 bit. #if defined(__clang__) || defined(__GNUC__) -#define LIBYUV_TARGET_SSE2 __attribute__((target("sse2"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) #else -#define LIBYUV_TARGET_SSE2 #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW #endif // defined(__clang__) || defined(__GNUC__) -#if defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2) -// Transpose 8 rows x 16 bytes with SSE2. Each output holds 2 columns x 8 rows. -// The low 8 bytes (even column) go to dst_a and the high 8 bytes (odd column) -// go to dst_b. Width is a multiple of 16 bytes. -LIBYUV_TARGET_SSE2 -static void TransposeWx8_Byte_SSE2(const uint8_t* src, +#if defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2) +// Transpose 8 rows x 16 bytes using 2 lanes of AVX2. Source row r is in +// lane r / 4 of register r % 4. Each 128-bit output lane holds 2 columns x 8 +// rows: low 8 bytes (even column) for dst_a and high 8 bytes (odd column) for +// dst_b. Width is a multiple of 16 bytes. +LIBYUV_TARGET_AVX2 +static void TransposeWx8_Byte_AVX2(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, @@ -59,97 +59,101 @@ static void TransposeWx8_Byte_SSE2(const uint8_t* src, const ptrdiff_t sa = dst_stride_a; const ptrdiff_t sb = dst_stride_b; while (byte_width > 0) { - __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); - __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); - __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); - __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); - __m128i r4 = _mm_loadu_si128((const __m128i*)(src + s * 4)); - __m128i r5 = _mm_loadu_si128((const __m128i*)(src + s * 5)); - __m128i r6 = _mm_loadu_si128((const __m128i*)(src + s * 6)); - __m128i r7 = _mm_loadu_si128((const __m128i*)(src + s * 7)); + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), + _mm_loadu_si128((const __m128i*)(src + s * 4)), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), + _mm_loadu_si128((const __m128i*)(src + s * 5)), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), + _mm_loadu_si128((const __m128i*)(src + s * 6)), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), + _mm_loadu_si128((const __m128i*)(src + s * 7)), 1); - // Bytes of row pairs. a0 is columns 0..7 of rows 0, 1. - __m128i a0 = _mm_unpacklo_epi8(r0, r1); - __m128i a1 = _mm_unpackhi_epi8(r0, r1); - __m128i a2 = _mm_unpacklo_epi8(r2, r3); - __m128i a3 = _mm_unpackhi_epi8(r2, r3); - __m128i a4 = _mm_unpacklo_epi8(r4, r5); - __m128i a5 = _mm_unpackhi_epi8(r4, r5); - __m128i a6 = _mm_unpacklo_epi8(r6, r7); - __m128i a7 = _mm_unpackhi_epi8(r6, r7); + __m256i a0 = _mm256_unpacklo_epi8(r0, r1); + __m256i a1 = _mm256_unpackhi_epi8(r0, r1); + __m256i a2 = _mm256_unpacklo_epi8(r2, r3); + __m256i a3 = _mm256_unpackhi_epi8(r2, r3); - // Words. b0 is columns 0..3 of rows 0..3. - __m128i b0 = _mm_unpacklo_epi16(a0, a2); - __m128i b1 = _mm_unpackhi_epi16(a0, a2); - __m128i b2 = _mm_unpacklo_epi16(a1, a3); - __m128i b3 = _mm_unpackhi_epi16(a1, a3); - __m128i b4 = _mm_unpacklo_epi16(a4, a6); - __m128i b5 = _mm_unpackhi_epi16(a4, a6); - __m128i b6 = _mm_unpacklo_epi16(a5, a7); - __m128i b7 = _mm_unpackhi_epi16(a5, a7); + __m256i b0 = _mm256_unpacklo_epi16(a0, a2); + __m256i b1 = _mm256_unpackhi_epi16(a0, a2); + __m256i b2 = _mm256_unpacklo_epi16(a1, a3); + __m256i b3 = _mm256_unpackhi_epi16(a1, a3); - // Dwords. c0 is columns 0, 1 of rows 0..7. - __m128i c0 = _mm_unpacklo_epi32(b0, b4); - __m128i c1 = _mm_unpackhi_epi32(b0, b4); - __m128i c2 = _mm_unpacklo_epi32(b1, b5); - __m128i c3 = _mm_unpackhi_epi32(b1, b5); - __m128i c4 = _mm_unpacklo_epi32(b2, b6); - __m128i c5 = _mm_unpackhi_epi32(b2, b6); - __m128i c6 = _mm_unpacklo_epi32(b3, b7); - __m128i c7 = _mm_unpackhi_epi32(b3, b7); + __m256i t0 = _mm256_permute2x128_si256(b0, b1, 0x20); + __m256i t1 = _mm256_permute2x128_si256(b0, b1, 0x31); + __m256i t2 = _mm256_permute2x128_si256(b2, b3, 0x20); + __m256i t3 = _mm256_permute2x128_si256(b2, b3, 0x31); - _mm_storel_epi64((__m128i*)(dst_a), c0); - _mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(c0)); - _mm_storel_epi64((__m128i*)(dst_a + sa), c1); - _mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(c1)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 2), c2); - _mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(c2)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 3), c3); - _mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(c3)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 4), c4); - _mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(c4)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 5), c5); - _mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(c5)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 6), c6); - _mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(c6)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 7), c7); - _mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(c7)); + __m256i c0 = _mm256_unpacklo_epi32(t0, t1); + __m256i c1 = _mm256_unpackhi_epi32(t0, t1); + __m256i c2 = _mm256_unpacklo_epi32(t2, t3); + __m256i c3 = _mm256_unpackhi_epi32(t2, t3); + + __m128i lo0 = _mm256_castsi256_si128(c0); + __m128i lo1 = _mm256_castsi256_si128(c1); + __m128i hi0 = _mm256_extracti128_si256(c0, 1); + __m128i hi1 = _mm256_extracti128_si256(c1, 1); + __m128i lo2 = _mm256_castsi256_si128(c2); + __m128i lo3 = _mm256_castsi256_si128(c3); + __m128i hi2 = _mm256_extracti128_si256(c2, 1); + __m128i hi3 = _mm256_extracti128_si256(c3, 1); + + _mm_storel_epi64((__m128i*)(dst_a), lo0); + _mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(lo0)); + _mm_storel_epi64((__m128i*)(dst_a + sa), lo1); + _mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(lo1)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 2), hi0); + _mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(hi0)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 3), hi1); + _mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(hi1)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 4), lo2); + _mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(lo2)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 5), lo3); + _mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(lo3)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 6), hi2); + _mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(hi2)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 7), hi3); + _mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(hi3)); src += 16; dst_a += sa * 8; dst_b += sb * 8; byte_width -= 16; } + _mm256_zeroupper(); } -#endif // defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2) +#endif // defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2) -#if defined(HAS_TRANSPOSEWX8_SSSE3) +#if defined(HAS_TRANSPOSEWX8_AVX2) // Transpose 16x8. Width is a multiple of 16. -LIBYUV_TARGET_SSE2 -void TransposeWx8_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - TransposeWx8_Byte_SSE2(src, src_stride, dst, dst_stride * 2, dst + dst_stride, +LIBYUV_TARGET_AVX2 +void TransposeWx8_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx8_Byte_AVX2(src, src_stride, dst, dst_stride * 2, dst + dst_stride, dst_stride * 2, width); } -#endif // defined(HAS_TRANSPOSEWX8_SSSE3) +#endif // defined(HAS_TRANSPOSEWX8_AVX2) -#if defined(HAS_TRANSPOSEUVWX8_SSE2) +#if defined(HAS_TRANSPOSEUVWX8_AVX2) // Transpose UV 8x8. Width is a multiple of 8. -LIBYUV_TARGET_SSE2 -void TransposeUVWx8_SSE2(const uint8_t* src, +LIBYUV_TARGET_AVX2 +void TransposeUVWx8_AVX2(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width) { - TransposeWx8_Byte_SSE2(src, src_stride, dst_a, dst_stride_a, dst_b, + TransposeWx8_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, width * 2); } -#endif // defined(HAS_TRANSPOSEUVWX8_SSE2) +#endif // defined(HAS_TRANSPOSEUVWX8_AVX2) #if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2) // Load row r into lane 0 and row r + 8 into lane 1. @@ -417,6 +421,55 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW) +#if defined(HAS_TRANSPOSEWX8_16_AVX2) +LIBYUV_TARGET_AVX2 +void TransposeWx8_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), + _mm_loadu_si128((const __m128i*)(src + s * 4)), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), + _mm_loadu_si128((const __m128i*)(src + s * 5)), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), + _mm_loadu_si128((const __m128i*)(src + s * 6)), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), + _mm_loadu_si128((const __m128i*)(src + s * 7)), 1); + + __m256i t0 = _mm256_unpacklo_epi16(r0, r1); + __m256i t1 = _mm256_unpackhi_epi16(r0, r1); + __m256i t2 = _mm256_unpacklo_epi16(r2, r3); + __m256i t3 = _mm256_unpackhi_epi16(r2, r3); + + __m256i u0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(t0, t2), 0xd8); + __m256i u1 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(t0, t2), 0xd8); + __m256i u2 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(t1, t3), 0xd8); + __m256i u3 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(t1, t3), 0xd8); + + _mm_storeu_si128((__m128i*)(dst), _mm256_castsi256_si128(u0)); + _mm_storeu_si128((__m128i*)(dst + d), _mm256_extracti128_si256(u0, 1)); + _mm_storeu_si128((__m128i*)(dst + d * 2), _mm256_castsi256_si128(u1)); + _mm_storeu_si128((__m128i*)(dst + d * 3), _mm256_extracti128_si256(u1, 1)); + _mm_storeu_si128((__m128i*)(dst + d * 4), _mm256_castsi256_si128(u2)); + _mm_storeu_si128((__m128i*)(dst + d * 5), _mm256_extracti128_si256(u2, 1)); + _mm_storeu_si128((__m128i*)(dst + d * 6), _mm256_castsi256_si128(u3)); + _mm_storeu_si128((__m128i*)(dst + d * 7), _mm256_extracti128_si256(u3, 1)); + src += 8; + dst += d * 8; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSEWX8_16_AVX2) + #if defined(HAS_TRANSPOSEWX16_16_AVX2) LIBYUV_TARGET_AVX2 void TransposeWx16_16_AVX2(const uint16_t* src, @@ -709,35 +762,61 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src, } #endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW) -#if defined(HAS_TRANSPOSEWX4_32_SSE2) -// Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row. -LIBYUV_TARGET_SSE2 -void TransposeWx4_32_SSE2(const uint8_t* src, +#if defined(HAS_TRANSPOSEWX4_32_AVX2) +// Transpose 32 bit values (ARGB). Read a column of 4 pixels across 8 rows per +// AVX2 iteration (with a 4-row 128-bit tail) and write 4 destination rows. +LIBYUV_TARGET_AVX2 +void TransposeWx4_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) { const ptrdiff_t s = src_stride; const ptrdiff_t d = dst_stride; - while (width > 0) { - __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); // a b c d - __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); // e f g h - __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); // i j k l - __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); // m n o p - __m128i t0 = _mm_unpacklo_epi32(r0, r1); // a e b f - __m128i t1 = _mm_unpacklo_epi32(r2, r3); // i m j n - __m128i t2 = _mm_unpackhi_epi32(r0, r1); // c g d h - __m128i t3 = _mm_unpackhi_epi32(r2, r3); // k o l p - _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); // a e i m + while (width >= 8) { + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), + _mm_loadu_si128((const __m128i*)(src + s * 4)), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), + _mm_loadu_si128((const __m128i*)(src + s * 5)), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), + _mm_loadu_si128((const __m128i*)(src + s * 6)), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), + _mm_loadu_si128((const __m128i*)(src + s * 7)), 1); + + __m256i t0 = _mm256_unpacklo_epi32(r0, r1); + __m256i t1 = _mm256_unpacklo_epi32(r2, r3); + __m256i t2 = _mm256_unpackhi_epi32(r0, r1); + __m256i t3 = _mm256_unpackhi_epi32(r2, r3); + + _mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1)); + _mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1)); + _mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3)); + _mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3)); + src += s * 8; + dst += 32; + width -= 8; + } + if (width > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); + __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); _mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1)); _mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3)); _mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3)); - src += s * 4; - dst += 16; - width -= 4; } + _mm256_zeroupper(); } -#endif // defined(HAS_TRANSPOSEWX4_32_SSE2) +#endif // defined(HAS_TRANSPOSEWX4_32_AVX2) #if defined(HAS_TRANSPOSEWX8_32_AVX2) LIBYUV_TARGET_AVX2 @@ -779,6 +858,241 @@ void TransposeWx8_32_AVX2(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEWX8_32_AVX2) +#if defined(HAS_TRANSPOSEWXH_32_AVX2) +// Load n (1..3) 32-bit pixels into low lanes of __m128i with zeroed upper +// lanes (safe at buffer boundaries and MemorySanitizer-clean). +LIBYUV_TARGET_AVX2 +static __m128i LoadPartialRow_32_AVX2(const uint8_t* p, int n) { + if (n == 1) { + int v; + memcpy(&v, p, 4); + return _mm_cvtsi32_si128(v); + } + if (n == 2) { + return _mm_loadl_epi64((const __m128i*)p); + } + int v; + memcpy(&v, p + 8, 4); + return _mm_insert_epi32(_mm_loadl_epi64((const __m128i*)p), v, 2); +} + +// Transpose arbitrary WxH 32-bit plane using AVX2 (4x8, 8x4, 4x4, and 4x{1..3} +// tiles plus exact remainder handling). +LIBYUV_TARGET_AVX2 +void TransposeWxH_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + const int w4 = width & ~3; + const int rem_w = width & 3; + + // Tier 1: 8-row strips using 4x8 YMM tiles + rem_w x 8 YMM tail. + while (height >= 8) { + if (w4 > 0) { + TransposeWx8_32_AVX2(src, src_stride, dst, dst_stride, w4); + } + if (rem_w > 0) { + const uint8_t* src_rem = src + (ptrdiff_t)w4 * 4; + uint8_t* dst_rem = dst + (ptrdiff_t)w4 * d; + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(LoadPartialRow_32_AVX2(src_rem, rem_w)), + LoadPartialRow_32_AVX2(src_rem + s * 4, rem_w), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(LoadPartialRow_32_AVX2(src_rem + s, rem_w)), + LoadPartialRow_32_AVX2(src_rem + s * 5, rem_w), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256( + LoadPartialRow_32_AVX2(src_rem + s * 2, rem_w)), + LoadPartialRow_32_AVX2(src_rem + s * 6, rem_w), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256( + LoadPartialRow_32_AVX2(src_rem + s * 3, rem_w)), + LoadPartialRow_32_AVX2(src_rem + s * 7, rem_w), 1); + + __m256i t0 = _mm256_unpacklo_epi32(r0, r1); + __m256i t1 = _mm256_unpacklo_epi32(r2, r3); + _mm256_storeu_si256((__m256i*)(dst_rem), _mm256_unpacklo_epi64(t0, t1)); + if (rem_w > 1) { + _mm256_storeu_si256((__m256i*)(dst_rem + d), + _mm256_unpackhi_epi64(t0, t1)); + } + if (rem_w > 2) { + __m256i t2 = _mm256_unpackhi_epi32(r0, r1); + __m256i t3 = _mm256_unpackhi_epi32(r2, r3); + _mm256_storeu_si256((__m256i*)(dst_rem + d * 2), + _mm256_unpacklo_epi64(t2, t3)); + } + _mm256_zeroupper(); + } + src += 8 * s; + dst += 32; + height -= 8; + } + + // Tier 2: 4-row strip using 8x4 YMM tiles + 4x4 XMM block + rem_w x 4 tail. + if (height >= 4) { + const int w8 = width & ~7; + if (w8 > 0) { + const uint8_t* src_p = src; + uint8_t* dst_p = dst; + int cnt = w8; + while (cnt > 0) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)(src_p)); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_p + s)); + __m256i r2 = _mm256_loadu_si256((const __m256i*)(src_p + s * 2)); + __m256i r3 = _mm256_loadu_si256((const __m256i*)(src_p + s * 3)); + + __m256i t0 = _mm256_unpacklo_epi32(r0, r1); + __m256i t1 = _mm256_unpacklo_epi32(r2, r3); + __m256i t2 = _mm256_unpackhi_epi32(r0, r1); + __m256i t3 = _mm256_unpackhi_epi32(r2, r3); + + __m256i o0 = _mm256_unpacklo_epi64(t0, t1); + __m256i o1 = _mm256_unpackhi_epi64(t0, t1); + __m256i o2 = _mm256_unpacklo_epi64(t2, t3); + __m256i o3 = _mm256_unpackhi_epi64(t2, t3); + + _mm_storeu_si128((__m128i*)(dst_p), _mm256_castsi256_si128(o0)); + _mm_storeu_si128((__m128i*)(dst_p + d), _mm256_castsi256_si128(o1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 2), _mm256_castsi256_si128(o2)); + _mm_storeu_si128((__m128i*)(dst_p + d * 3), _mm256_castsi256_si128(o3)); + _mm_storeu_si128((__m128i*)(dst_p + d * 4), + _mm256_extracti128_si256(o0, 1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 5), + _mm256_extracti128_si256(o1, 1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 6), + _mm256_extracti128_si256(o2, 1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 7), + _mm256_extracti128_si256(o3, 1)); + + src_p += 32; + dst_p += d * 8; + cnt -= 8; + } + _mm256_zeroupper(); + } + if (width & 4) { + const uint8_t* src_p = src + (ptrdiff_t)w8 * 4; + uint8_t* dst_p = dst + (ptrdiff_t)w8 * d; + __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src_p + s * 2)); + __m128i r3 = _mm_loadu_si128((const __m128i*)(src_p + s * 3)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst_p), _mm_unpacklo_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst_p + d), _mm_unpackhi_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 2), _mm_unpacklo_epi64(t2, t3)); + _mm_storeu_si128((__m128i*)(dst_p + d * 3), _mm_unpackhi_epi64(t2, t3)); + } + if (rem_w > 0) { + const uint8_t* src_rem = src + (ptrdiff_t)w4 * 4; + uint8_t* dst_rem = dst + (ptrdiff_t)w4 * d; + __m128i r0 = LoadPartialRow_32_AVX2(src_rem, rem_w); + __m128i r1 = LoadPartialRow_32_AVX2(src_rem + s, rem_w); + __m128i r2 = LoadPartialRow_32_AVX2(src_rem + s * 2, rem_w); + __m128i r3 = LoadPartialRow_32_AVX2(src_rem + s * 3, rem_w); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst_rem), _mm_unpacklo_epi64(t0, t1)); + if (rem_w > 1) { + _mm_storeu_si128((__m128i*)(dst_rem + d), _mm_unpackhi_epi64(t0, t1)); + } + if (rem_w > 2) { + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst_rem + d * 2), + _mm_unpacklo_epi64(t2, t3)); + } + } + src += 4 * s; + dst += 16; + height -= 4; + } + + // Tier 3: 1..3 remaining source rows using 4 x height XMM tiles + corner. + if (height > 0) { + if (w4 > 0) { + const uint8_t* src_p = src; + uint8_t* dst_p = dst; + int cnt = w4; + if (height == 1) { + while (cnt > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p)); + int v0 = _mm_cvtsi128_si32(r0); + int v1 = _mm_extract_epi32(r0, 1); + int v2 = _mm_extract_epi32(r0, 2); + int v3 = _mm_extract_epi32(r0, 3); + memcpy(dst_p, &v0, 4); + memcpy(dst_p + d, &v1, 4); + memcpy(dst_p + d * 2, &v2, 4); + memcpy(dst_p + d * 3, &v3, 4); + src_p += 16; + dst_p += d * 4; + cnt -= 4; + } + } else if (height == 2) { + while (cnt > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpackhi_epi32(r0, r1); + _mm_storel_epi64((__m128i*)(dst_p), t0); + _mm_storeh_pd((double*)(dst_p + d), _mm_castsi128_pd(t0)); + _mm_storel_epi64((__m128i*)(dst_p + d * 2), t1); + _mm_storeh_pd((double*)(dst_p + d * 3), _mm_castsi128_pd(t1)); + src_p += 16; + dst_p += d * 4; + cnt -= 4; + } + } else { + const __m128i zero = _mm_setzero_si128(); + while (cnt > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src_p + s * 2)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, zero); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, zero); + __m128i o0 = _mm_unpacklo_epi64(t0, t1); + __m128i o1 = _mm_unpackhi_epi64(t0, t1); + __m128i o2 = _mm_unpacklo_epi64(t2, t3); + __m128i o3 = _mm_unpackhi_epi64(t2, t3); + int v0 = _mm_extract_epi32(o0, 2); + int v1 = _mm_extract_epi32(o1, 2); + int v2 = _mm_extract_epi32(o2, 2); + int v3 = _mm_extract_epi32(o3, 2); + _mm_storel_epi64((__m128i*)(dst_p), o0); + memcpy(dst_p + 8, &v0, 4); + _mm_storel_epi64((__m128i*)(dst_p + d), o1); + memcpy(dst_p + d + 8, &v1, 4); + _mm_storel_epi64((__m128i*)(dst_p + d * 2), o2); + memcpy(dst_p + d * 2 + 8, &v2, 4); + _mm_storel_epi64((__m128i*)(dst_p + d * 3), o3); + memcpy(dst_p + d * 3 + 8, &v3, 4); + src_p += 16; + dst_p += d * 4; + cnt -= 4; + } + } + } + for (int x = w4; x < width; ++x) { + uint8_t* dst_row = dst + (ptrdiff_t)x * d; + for (int y = 0; y < height; ++y) { + memcpy(dst_row + y * 4, src + (ptrdiff_t)y * s + x * 4, 4); + } + } + } +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX2) + #if defined(HAS_TRANSPOSEWX16_32_AVX512BW) LIBYUV_TARGET_AVX512BW void TransposeWx16_32_AVX512BW(const uint8_t* src, diff --git a/source/row_any.cc b/source/row_any.cc index bb59eaf3e..3ac640620 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -277,7 +277,7 @@ ANY41PT(MergeARGB16To8Row_Any_NEON, const uint8_t* v_buf, uint8_t* dst_ptr, int width) { \ SIMD_ALIGNED(uint8_t vin[64 * 3]); \ static_assert(MASK + 1 <= sizeof(vin) / 3, "vin buffer too small"); \ - SIMD_ALIGNED(uint8_t vout[64]); \ + SIMD_ALIGNED(uint8_t vout[64 * 4]); \ static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \ "vout buffer too small"); \ memset(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \ @@ -296,8 +296,8 @@ ANY41PT(MergeARGB16To8Row_Any_NEON, } // Merge functions. -#ifdef HAS_MERGERGBROW_SSSE3 -ANY31(MergeRGBRow_Any_SSSE3, MergeRGBRow_SSSE3, 0, 0, 3, 15) +#ifdef HAS_MERGERGBROW_AVX2 +ANY31(MergeRGBRow_Any_AVX2, MergeRGBRow_AVX2, 0, 0, 3, 31) #endif #ifdef HAS_MERGERGBROW_NEON ANY31(MergeRGBRow_Any_NEON, MergeRGBRow_NEON, 0, 0, 3, 15) @@ -769,8 +769,8 @@ ANY21(ARGBAlphaBlendRow_Any_LASX, ARGBAlphaBlendRow_LASX, 0, 4, 4, 4, 7) #ifdef HAS_ARGBALPHABLENDROW_WASMSIMD ANY21(ARGBAlphaBlendRow_Any_WASMSIMD, ARGBAlphaBlendRow_WASMSIMD, 0, 4, 4, 4, 3) #endif -#ifdef HAS_SOBELROW_SSE2 -ANY21(SobelRow_Any_SSE2, SobelRow_SSE2, 0, 1, 1, 4, 15) +#ifdef HAS_SOBELROW_AVX2 +ANY21(SobelRow_Any_AVX2, SobelRow_AVX2, 0, 1, 1, 4, 15) #endif #ifdef HAS_SOBELROW_NEON ANY21(SobelRow_Any_NEON, SobelRow_NEON, 0, 1, 1, 4, 7) @@ -781,8 +781,8 @@ ANY21(SobelRow_Any_LSX, SobelRow_LSX, 0, 1, 1, 4, 7) #ifdef HAS_SOBELROW_LASX ANY21(SobelRow_Any_LASX, SobelRow_LASX, 0, 1, 1, 4, 15) #endif -#ifdef HAS_SOBELTOPLANEROW_SSE2 -ANY21(SobelToPlaneRow_Any_SSE2, SobelToPlaneRow_SSE2, 0, 1, 1, 1, 15) +#ifdef HAS_SOBELTOPLANEROW_AVX2 +ANY21(SobelToPlaneRow_Any_AVX2, SobelToPlaneRow_AVX2, 0, 1, 1, 1, 31) #endif #ifdef HAS_SOBELTOPLANEROW_NEON ANY21(SobelToPlaneRow_Any_NEON, SobelToPlaneRow_NEON, 0, 1, 1, 1, 15) @@ -793,8 +793,8 @@ ANY21(SobelToPlaneRow_Any_LSX, SobelToPlaneRow_LSX, 0, 1, 1, 1, 15) #ifdef HAS_SOBELTOPLANEROW_LASX ANY21(SobelToPlaneRow_Any_LASX, SobelToPlaneRow_LASX, 0, 1, 1, 1, 31) #endif -#ifdef HAS_SOBELXYROW_SSE2 -ANY21(SobelXYRow_Any_SSE2, SobelXYRow_SSE2, 0, 1, 1, 4, 15) +#ifdef HAS_SOBELXYROW_AVX2 +ANY21(SobelXYRow_Any_AVX2, SobelXYRow_AVX2, 0, 1, 1, 4, 15) #endif #ifdef HAS_SOBELXYROW_NEON ANY21(SobelXYRow_Any_NEON, SobelXYRow_NEON, 0, 1, 1, 4, 7) @@ -1114,6 +1114,15 @@ ANY11(AR30ToABGRRow_Any_NEON, AR30ToABGRRow_NEON, 0, 4, 4, 7) #if defined(HAS_AR30TOAB30ROW_NEON) ANY11(AR30ToAB30Row_Any_NEON, AR30ToAB30Row_NEON, 0, 4, 4, 7) #endif +#if defined(HAS_AR30TOARGBROW_AVX2) +ANY11(AR30ToARGBRow_Any_AVX2, AR30ToARGBRow_AVX2, 0, 4, 4, 7) +#endif +#if defined(HAS_AR30TOABGRROW_AVX2) +ANY11(AR30ToABGRRow_Any_AVX2, AR30ToABGRRow_AVX2, 0, 4, 4, 7) +#endif +#if defined(HAS_AR30TOAB30ROW_AVX2) +ANY11(AR30ToAB30Row_Any_AVX2, AR30ToAB30Row_AVX2, 0, 4, 4, 7) +#endif #if defined(HAS_ARGBTOAR30ROW_SSSE3) ANY11(ARGBToAR30Row_Any_SSSE3, ARGBToAR30Row_SSSE3, 0, 4, 4, 3) #endif @@ -1141,8 +1150,8 @@ ANY11(RAWToARGBRow_Any_AVX2, RAWToARGBRow_AVX2, 0, 3, 4, 31) #if defined(HAS_RGB24TOARGBROW_AVX2) ANY11(RGB24ToARGBRow_Any_AVX2, RGB24ToARGBRow_AVX2, 0, 3, 4, 31) #endif -#if defined(HAS_RAWTORGBAROW_SSSE3) -ANY11(RAWToRGBARow_Any_SSSE3, RAWToRGBARow_SSSE3, 0, 3, 4, 15) +#if defined(HAS_RAWTORGBAROW_AVX2) +ANY11(RAWToRGBARow_Any_AVX2, RAWToRGBARow_AVX2, 0, 3, 4, 15) #endif #if defined(HAS_RAWTORGB24ROW_SSSE3) ANY11(RAWToRGB24Row_Any_SSSE3, RAWToRGB24Row_SSSE3, 0, 3, 3, 7) @@ -1217,6 +1226,15 @@ ANY11(YUY2ToYRow_Any_SSE2, YUY2ToYRow_SSE2, 1, 4, 1, 15) ANY11(UYVYToYRow_Any_SSE2, UYVYToYRow_SSE2, 1, 4, 1, 15) #endif +#ifdef HAS_RGB565TOYROW_AVX2 +ANY11(RGB565ToYRow_Any_AVX2, RGB565ToYRow_AVX2, 0, 2, 1, 31) +#endif +#ifdef HAS_ARGB1555TOYROW_AVX2 +ANY11(ARGB1555ToYRow_Any_AVX2, ARGB1555ToYRow_AVX2, 0, 2, 1, 31) +#endif +#ifdef HAS_ARGB4444TOYROW_AVX2 +ANY11(ARGB4444ToYRow_Any_AVX2, ARGB4444ToYRow_AVX2, 0, 2, 1, 31) +#endif #ifdef HAS_RGB565TOYROW_NEON ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 15) #endif @@ -1264,6 +1282,9 @@ ANY11(UYVYToYRow_Any_LSX, UYVYToYRow_LSX, 1, 4, 1, 15) #ifdef HAS_UYVYTOYROW_LASX ANY11(UYVYToYRow_Any_LASX, UYVYToYRow_LASX, 1, 4, 1, 31) #endif +#ifdef HAS_AYUVTOYROW_AVX2 +ANY11(AYUVToYRow_Any_AVX2, AYUVToYRow_AVX2, 0, 4, 1, 15) +#endif #ifdef HAS_AYUVTOYROW_NEON ANY11(AYUVToYRow_Any_NEON, AYUVToYRow_NEON, 0, 4, 1, 15) #endif @@ -1670,6 +1691,9 @@ ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) #endif +#ifdef HAS_COPYROW_16_AVX2 +ANY11T(CopyRow_16_Any_AVX2, CopyRow_16_AVX2, 2, 2, uint16_t, uint16_t, 31) +#endif #ifdef HAS_COPYROW_16_NEON ANY11T(CopyRow_16_Any_NEON, CopyRow_16_NEON, 2, 2, uint16_t, uint16_t, 15) #endif @@ -1898,6 +1922,9 @@ ANY11P16(HalfFloat1Row_Any_F16C, #ifdef HAS_HALFFLOATROW_NEON ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 15) #endif +#ifdef HAS_BYTETOFLOATROW_AVX2 +ANY11P16(ByteToFloatRow_Any_AVX2, ByteToFloatRow_AVX2, uint8_t, float, 1, 4, 15) +#endif #ifdef HAS_BYTETOFLOATROW_NEON ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 4, 7) #endif @@ -2031,6 +2058,39 @@ ANY11I(InterpolateRow_16_Any_WASMSIMD, #endif #undef ANY11I +// Any 1 to 1 half row +#define ANY11H(NAMEANY, ANY_SIMD, T, MASK) \ + void NAMEANY(const T* src_ptr, ptrdiff_t src_stride, T* dst_ptr, \ + int width) { \ + SIMD_ALIGNED(T vin[64 * 2]); \ + static_assert((MASK + 1) * sizeof(T) <= sizeof(vin) / 2, \ + "vin buffer too small"); \ + SIMD_ALIGNED(T vout[64]); \ + static_assert((MASK + 1) * sizeof(T) <= sizeof(vout), \ + "vout buffer too small"); \ + MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr, src_stride, dst_ptr, n); \ + } \ + ptrdiff_t np = n; \ + memcpy(vin, src_ptr + np, r * sizeof(T)); \ + memcpy(vin + 64, src_ptr + src_stride + np, r * sizeof(T)); \ + ANY_SIMD(vin, 64, vout, MASK + 1); \ + memcpy(dst_ptr + np, vout, r * sizeof(T)); \ + } + +#ifdef HAS_HALFROW_AVX2 +ANY11H(HalfRow_Any_AVX2, HalfRow_AVX2, uint8_t, 31) +#endif +#ifdef HAS_HALFROW_16_AVX2 +ANY11H(HalfRow_16_Any_AVX2, HalfRow_16_AVX2, uint16_t, 15) +#endif + +#undef ANY11H + // Any 1 to 1 half row with scale param #define ANY11HS(NAMEANY, ANY_SIMD, TS, TD, MASK) \ void NAMEANY(const TS* src_ptr, ptrdiff_t src_stride, TD* dst_ptr, \ @@ -2149,9 +2209,6 @@ ANY11HWS(HalfWidthRow_16To8_Any_NEON, memcpy(dst_ptr + np * BPP, vout + (MASK + 1 - r) * BPP, r * BPP); \ } -#ifdef HAS_MIRRORROW_AVX512BW -ANY11M(MirrorRow_Any_AVX512BW, MirrorRow_AVX512BW, 1, 63) -#endif #ifdef HAS_MIRRORROW_AVX2 ANY11M(MirrorRow_Any_AVX2, MirrorRow_AVX2, 1, 31) #endif @@ -2167,9 +2224,6 @@ ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 15) #ifdef HAS_MIRRORROW_LASX ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 31) #endif -#ifdef HAS_MIRRORUVROW_AVX512BW -ANY11M(MirrorUVRow_Any_AVX512BW, MirrorUVRow_AVX512BW, 2, 31) -#endif #ifdef HAS_MIRRORUVROW_AVX2 ANY11M(MirrorUVRow_Any_AVX2, MirrorUVRow_AVX2, 2, 15) #endif @@ -2264,8 +2318,8 @@ ANY11MT(MirrorRow_16_Any_NEON, MirrorRow_16_NEON, uint16_t, 2, 15) memcpy(dst_ptr + np * BPP, vout, r * BPP); \ } -#ifdef HAS_SETROW_X86 -ANY1(SetRow_Any_X86, SetRow_X86, uint8_t, 1, 3) +#ifdef HAS_SETROW_AVX2 +ANY1(SetRow_Any_AVX2, SetRow_AVX2, uint8_t, 1, 31) #endif #ifdef HAS_SETROW_NEON ANY1(SetRow_Any_NEON, SetRow_NEON, uint8_t, 1, 15) @@ -2276,6 +2330,9 @@ ANY1(SetRow_Any_LSX, SetRow_LSX, uint8_t, 1, 15) #ifdef HAS_SETROW_LASX ANY1(SetRow_Any_LASX, SetRow_LASX, uint8_t, 1, 31) #endif +#ifdef HAS_ARGBSETROW_AVX2 +ANY1(ARGBSetRow_Any_AVX2, ARGBSetRow_AVX2, uint32_t, 4, 7) +#endif #ifdef HAS_ARGBSETROW_NEON ANY1(ARGBSetRow_Any_NEON, ARGBSetRow_NEON, uint32_t, 4, 3) #endif @@ -2788,6 +2845,15 @@ ANY12S(RAWToUVRow_Any_LSX, RAWToUVRow_LSX, 0, 3, 15) #ifdef HAS_RAWTOUVROW_LASX ANY12S(RAWToUVRow_Any_LASX, RAWToUVRow_LASX, 0, 3, 31) #endif +#ifdef HAS_RGB565TOUVROW_AVX2 +ANY12S(RGB565ToUVRow_Any_AVX2, RGB565ToUVRow_AVX2, 0, 2, 31) +#endif +#ifdef HAS_ARGB1555TOUVROW_AVX2 +ANY12S(ARGB1555ToUVRow_Any_AVX2, ARGB1555ToUVRow_AVX2, 0, 2, 31) +#endif +#ifdef HAS_ARGB4444TOUVROW_AVX2 +ANY12S(ARGB4444ToUVRow_Any_AVX2, ARGB4444ToUVRow_AVX2, 0, 2, 31) +#endif #ifdef HAS_RGB565TOUVROW_NEON ANY12S(RGB565ToUVRow_Any_NEON, RGB565ToUVRow_NEON, 0, 2, 15) #endif @@ -2861,6 +2927,12 @@ ANY12S(UYVYToUVRow_Any_LASX, UYVYToUVRow_LASX, 1, 4, 31) memcpy(dst_vu + (np >> 1) * 2, vout, SS(r, 1) * 2); \ } +#ifdef HAS_AYUVTOUVROW_AVX2 +ANY11S(AYUVToUVRow_Any_AVX2, AYUVToUVRow_AVX2, 0, 4, 15) +#endif +#ifdef HAS_AYUVTOVUROW_AVX2 +ANY11S(AYUVToVURow_Any_AVX2, AYUVToVURow_AVX2, 0, 4, 15) +#endif #ifdef HAS_AYUVTOVUROW_NEON ANY11S(AYUVToUVRow_Any_NEON, AYUVToUVRow_NEON, 0, 4, 15) ANY11S(AYUVToVURow_Any_NEON, AYUVToVURow_NEON, 0, 4, 15) @@ -2895,17 +2967,14 @@ ANY11S(AYUVToVURow_Any_SVE2, AYUVToVURow_SVE2, 0, 4, 1) #ifdef HAS_DETILEROW_NEON ANYDETILE(DetileRow_Any_NEON, DetileRow_NEON, uint8_t, 1, 15) #endif -#ifdef HAS_DETILEROW_SSE2 -ANYDETILE(DetileRow_Any_SSE2, DetileRow_SSE2, uint8_t, 1, 15) +#ifdef HAS_DETILEROW_AVX2 +ANYDETILE(DetileRow_Any_AVX2, DetileRow_AVX2, uint8_t, 1, 15) #endif #ifdef HAS_DETILEROW_16_NEON ANYDETILE(DetileRow_16_Any_NEON, DetileRow_16_NEON, uint16_t, 2, 15) #endif -#ifdef HAS_DETILEROW_16_SSE2 -ANYDETILE(DetileRow_16_Any_SSE2, DetileRow_16_SSE2, uint16_t, 2, 15) -#endif -#ifdef HAS_DETILEROW_16_AVX -ANYDETILE(DetileRow_16_Any_AVX, DetileRow_16_AVX, uint16_t, 2, 15) +#ifdef HAS_DETILEROW_16_AVX2 +ANYDETILE(DetileRow_16_Any_AVX2, DetileRow_16_AVX2, uint16_t, 2, 15) #endif // DetileSplitUVRow width is in bytes @@ -2934,17 +3003,17 @@ ANYDETILE(DetileRow_16_Any_AVX, DetileRow_16_AVX, uint16_t, 2, 15) #ifdef HAS_DETILESPLITUVROW_NEON ANYDETILESPLITUV(DetileSplitUVRow_Any_NEON, DetileSplitUVRow_NEON, 15) #endif -#ifdef HAS_DETILESPLITUVROW_SSSE3 -ANYDETILESPLITUV(DetileSplitUVRow_Any_SSSE3, DetileSplitUVRow_SSSE3, 15) +#ifdef HAS_DETILESPLITUVROW_AVX2 +ANYDETILESPLITUV(DetileSplitUVRow_Any_AVX2, DetileSplitUVRow_AVX2, 15) #endif #define ANYDETILEMERGE(NAMEANY, ANY_SIMD, MASK) \ void NAMEANY(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, \ const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, \ uint8_t* dst_yuy2, int width) { \ - SIMD_ALIGNED(uint8_t vin[16 * 2]); \ + SIMD_ALIGNED(uint8_t vin[32 * 2]); \ static_assert(MASK + 1 <= sizeof(vin) / 2, "vin buffer too small"); \ - SIMD_ALIGNED(uint8_t vout[16 * 2]); \ + SIMD_ALIGNED(uint8_t vout[32 * 2]); \ static_assert(2 * (MASK + 1) <= sizeof(vout), "vout buffer too small"); \ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ @@ -2955,9 +3024,14 @@ ANYDETILESPLITUV(DetileSplitUVRow_Any_SSSE3, DetileSplitUVRow_SSSE3, 15) n); \ } \ ptrdiff_t np = n; \ - memcpy(vin, src_y + (np / 16) * src_y_tile_stride, r); \ - memcpy(vin + 16, src_uv + (np / 16) * src_uv_tile_stride, r); \ - ANY_SIMD(vin, src_y_tile_stride, vin + 16, src_uv_tile_stride, vout, r); \ + int r0 = (r < 16) ? r : 16; \ + memcpy(vin, src_y + (np / 16) * src_y_tile_stride, r0); \ + memcpy(vin + 32, src_uv + (np / 16) * src_uv_tile_stride, r0); \ + if (r > 16) { \ + memcpy(vin + 16, src_y + (np / 16 + 1) * src_y_tile_stride, r - 16); \ + memcpy(vin + 48, src_uv + (np / 16 + 1) * src_uv_tile_stride, r - 16); \ + } \ + ANY_SIMD(vin, 16, vin + 32, 16, vout, MASK + 1); \ memcpy(dst_yuy2 + 2 * np, vout, 2 * r); \ } @@ -2965,10 +3039,12 @@ ANYDETILESPLITUV(DetileSplitUVRow_Any_SSSE3, DetileSplitUVRow_SSSE3, 15) ANYDETILEMERGE(DetileToYUY2_Any_NEON, DetileToYUY2_NEON, 15) #endif -#ifdef HAS_DETILETOYUY2_SSE2 -ANYDETILEMERGE(DetileToYUY2_Any_SSE2, DetileToYUY2_SSE2, 15) +#ifdef HAS_DETILETOYUY2_AVX2 +ANYDETILEMERGE(DetileToYUY2_Any_AVX2, DetileToYUY2_AVX2, 31) #endif +#undef ANYDETILEMERGE + #undef MSAN_MEMSET #ifdef __cplusplus diff --git a/source/row_common.cc b/source/row_common.cc index 94de60ab3..951d3c927 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -3643,20 +3643,20 @@ void ARGBAffineRow_C(const uint8_t* src_argb, } // Blend 2 rows into 1. -static void HalfRow_C(const uint8_t* src_uv, - ptrdiff_t src_uv_stride, - uint8_t* dst_uv, - int width) { +void HalfRow_C(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width) { int x; for (x = 0; x < width; ++x) { dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1; } } -static void HalfRow_16_C(const uint16_t* src_uv, - ptrdiff_t src_uv_stride, - uint16_t* dst_uv, - int width) { +void HalfRow_16_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width) { int x; for (x = 0; x < width; ++x) { dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1; @@ -4197,6 +4197,77 @@ void NV21ToRGB24Row_AVX2(const uint8_t* src_y, } #endif +#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTORGB565ROW_AVX512BW) +void I422ToRGB565Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb565, + const struct YuvConstants* yuvconstants, + int width) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth); + ARGBToRGB565Row_AVX512BW(row, dst_rgb565, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_rgb565 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTOARGB1555ROW_AVX2) +void I422ToARGB1555Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb1555, + const struct YuvConstants* yuvconstants, + int width) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth); + if (IS_ALIGNED(twidth, 8)) { + ARGBToARGB1555Row_AVX2(row, dst_argb1555, twidth); + } else { + ARGBToARGB1555Row_Any_AVX2(row, dst_argb1555, twidth); + } + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_argb1555 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTOARGB4444ROW_AVX2) +void I422ToARGB4444Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb4444, + const struct YuvConstants* yuvconstants, + int width) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth); + if (IS_ALIGNED(twidth, 8)) { + ARGBToARGB4444Row_AVX2(row, dst_argb4444, twidth); + } else { + ARGBToARGB4444Row_Any_AVX2(row, dst_argb4444, twidth); + } + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_argb4444 += twidth * 2; + width -= twidth; + } +} +#endif + #if defined(HAS_I422TOARGBROW_AVX2) && defined(HAS_ARGBTORGB565ROW_AVX2) void I422ToRGB565Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, @@ -5056,6 +5127,61 @@ void ARGB4444ToUVMatrixRow_NEON(const uint8_t* src_argb4444, } #endif +#if defined(HAS_RGB565TOYROW_AVX2) && defined(HAS_RGB565TOYMATRIXROW_AVX2) +void RGB565ToYRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { + RGB565ToYMatrixRow_AVX2(src_rgb565, dst_y, width, &kArgbI601Constants); +} +#endif + +#if defined(HAS_ARGB1555TOYROW_AVX2) && defined(HAS_ARGB1555TOYMATRIXROW_AVX2) +void ARGB1555ToYRow_AVX2(const uint8_t* src_argb1555, + uint8_t* dst_y, + int width) { + ARGB1555ToYMatrixRow_AVX2(src_argb1555, dst_y, width, &kArgbI601Constants); +} +#endif + +#if defined(HAS_ARGB4444TOYROW_AVX2) && defined(HAS_ARGB4444TOYMATRIXROW_AVX2) +void ARGB4444ToYRow_AVX2(const uint8_t* src_argb4444, + uint8_t* dst_y, + int width) { + ARGB4444ToYMatrixRow_AVX2(src_argb4444, dst_y, width, &kArgbI601Constants); +} +#endif + +#if defined(HAS_RGB565TOUVROW_AVX2) && defined(HAS_RGB565TOUVMATRIXROW_AVX2) +void RGB565ToUVRow_AVX2(const uint8_t* src_rgb565, + int src_stride_rgb565, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + RGB565ToUVMatrixRow_AVX2(src_rgb565, src_stride_rgb565, dst_u, dst_v, width, + &kArgbI601Constants); +} +#endif + +#if defined(HAS_ARGB1555TOUVROW_AVX2) && defined(HAS_ARGB1555TOUVMATRIXROW_AVX2) +void ARGB1555ToUVRow_AVX2(const uint8_t* src_argb1555, + int src_stride_argb1555, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + ARGB1555ToUVMatrixRow_AVX2(src_argb1555, src_stride_argb1555, dst_u, dst_v, + width, &kArgbI601Constants); +} +#endif + +#if defined(HAS_ARGB4444TOUVROW_AVX2) && defined(HAS_ARGB4444TOUVMATRIXROW_AVX2) +void ARGB4444ToUVRow_AVX2(const uint8_t* src_argb4444, + int src_stride_argb4444, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + ARGB4444ToUVMatrixRow_AVX2(src_argb4444, src_stride_argb4444, dst_u, dst_v, + width, &kArgbI601Constants); +} +#endif + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 40f990dc0..876fdd274 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -32,30 +32,6 @@ extern "C" { // clang for windows needing to preserve xmm registers but not saving // them if declared as ymm or zmm. -#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3) - -// Constants for ARGB - -// JPeg full range. -static const uvec8 kARGBToYJ = {29u, 150u, 77u, 0u, 29u, 150u, 77u, 0u, - 29u, 150u, 77u, 0u, 29u, 150u, 77u, 0u}; - -#endif // defined(HAS_ARGBTOYMATRIXROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3) - -#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3) -// Constants for BGRA - -// Constants for ABGR - -// Constants for RGBA. -// 126 (7e) - (-109..110) = 16..235 - -static const uvec16 kSub128 = {0x8080u, 0x8080u, 0x8080u, 0x8080u, - 0x8080u, 0x8080u, 0x8080u, 0x8080u}; - -#endif // defined(HAS_ARGBTOYMATRIXROW_SSSE3) || - // defined(HAS_I422TOARGBROW_SSSE3) - #if defined(HAS_RGB24TOARGBROW_SSSE3) || defined(HAS_RGB24TOARGBROW_AVX2) || \ defined(HAS_RGB24TOARGBROW_AVX512BW) || \ defined(HAS_RGBTOYMATRIXROW_AVX512BW) @@ -139,11 +115,13 @@ static const uint8_t kPermRAWToRGB24_3[64] = { 0x79, 0x78, 0x77, 0x7c, 0x7b, 0x7a, 0x7f, 0x7e, 0x7d}; #endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI) -#if defined(HAS_RAWTORGBAROW_SSSE3) +#if defined(HAS_RAWTORGBAROW_AVX2) // Shuffle table for converting RAW to RGBA. -static const uvec8 kShuffleMaskRAWToRGBA = { - 128u, 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u}; -#endif // defined(HAS_RAWTORGBAROW_SSSE3) +static const uvec8 kShuffleMaskRAWToRGBA_AVX2[2] = { + {128u, 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u}, + {128u, 6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, + 13u}}; +#endif // defined(HAS_RAWTORGBAROW_AVX2) #if defined(HAS_RAWTORGB24ROW_SSSE3) // Shuffle table for converting RAW to RGB24. First 8. @@ -576,46 +554,38 @@ void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, } #endif // HAS_RAWTOARGBROW_AVX512BW -#ifdef HAS_RAWTORGBAROW_SSSE3 -// Same code as RAWToARGB with different shuffler and A in low bits -void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { +#ifdef HAS_RAWTORGBAROW_AVX2 +void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { asm volatile( - "pcmpeqb %%xmm5,%%xmm5 \n" // 0x000000ff - "psrld $24,%%xmm5 \n" - "movdqa %3,%%xmm4 \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsrld $24,%%ymm5,%%ymm5 \n" // 0x000000ff + "vbroadcasti128 %3,%%ymm3 \n" + "vmovdqu %3,%%ymm4 \n" LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "movdqu 0x20(%0),%%xmm3 \n" + "vmovdqu (%0),%%xmm0 \n" // first 12 bytes + "vinserti128 $1,12(%0),%%ymm0,%%ymm0 \n" // second 12 bytes + "vmovdqu 24(%0),%%xmm1 \n" // third 12 bytes + "vinserti128 $1,32(%0),%%ymm1,%%ymm1 \n" // fourth 12 bytes "lea 0x30(%0),%0 \n" - "movdqa %%xmm3,%%xmm2 \n" - "palignr $0x8,%%xmm1,%%xmm2 \n" - "pshufb %%xmm4,%%xmm2 \n" - "por %%xmm5,%%xmm2 \n" - "palignr $0xc,%%xmm0,%%xmm1 \n" - "pshufb %%xmm4,%%xmm0 \n" - "movdqu %%xmm2,0x20(%1) \n" - "por %%xmm5,%%xmm0 \n" - "pshufb %%xmm4,%%xmm1 \n" - "movdqu %%xmm0,(%1) \n" - "por %%xmm5,%%xmm1 \n" - "palignr $0x4,%%xmm3,%%xmm3 \n" - "pshufb %%xmm4,%%xmm3 \n" - "movdqu %%xmm1,0x10(%1) \n" - "por %%xmm5,%%xmm3 \n" - "movdqu %%xmm3,0x30(%1) \n" + "vpshufb %%ymm3,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm4,%%ymm1,%%ymm1 \n" + "vpor %%ymm5,%%ymm0,%%ymm0 \n" + "vpor %%ymm5,%%ymm1,%%ymm1 \n" + "vmovdqu %%ymm0,(%1) \n" + "vmovdqu %%ymm1,0x20(%1) \n" "lea 0x40(%1),%1 \n" "sub $0x10,%2 \n" "jg 1b \n" - : "+r"(src_raw), // %0 - "+r"(dst_rgba), // %1 - "+r"(width) // %2 - : "m"(kShuffleMaskRAWToRGBA) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + "vzeroupper \n" + : "+r"(src_raw), // %0 + "+r"(dst_rgba), // %1 + "+r"(width) // %2 + : "m"(kShuffleMaskRAWToRGBA_AVX2[0]) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm3", "xmm4", "xmm5"); } -#endif // HAS_RAWTORGBAROW_SSSE3 +#endif // HAS_RAWTORGBAROW_AVX2 #ifdef HAS_RAWTORGB24ROW_SSSE3 void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, @@ -908,6 +878,97 @@ void ARGB1555ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGB1555TOARGBROW_AVX2 +#ifdef HAS_ARGB1555TOARGBROW_AVX512BW +void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + uintptr_t temp; + asm volatile( + "mov $0x01080108,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm5 \n" + "mov $0x42004200,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm6 \n" + "vpternlogd $0xff,%%zmm3,%%zmm3,%%zmm3 \n" + "vpsllw $0xb,%%zmm3,%%zmm3 \n" + "vpsrlw $0x6,%%zmm3,%%zmm4 \n" + "vpternlogd $0xff,%%zmm7,%%zmm7,%%zmm7 \n" + "vpsllw $0x8,%%zmm7,%%zmm7 \n" + "sub $0x20,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu16 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsllw $0x1,%%zmm0,%%zmm1 \n" + "vpsllw $0xb,%%zmm0,%%zmm2 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpmulhuw %%zmm5,%%zmm2,%%zmm2 \n" + "vpmulhuw %%zmm5,%%zmm1,%%zmm1 \n" + "vpsllw $0x8,%%zmm1,%%zmm1 \n" + "vpord %%zmm2,%%zmm1,%%zmm1 \n" + "vpsraw $0x8,%%zmm0,%%zmm2 \n" + "vpandd %%zmm4,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n" + "vpandd %%zmm7,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vpunpcklbw %%zmm0,%%zmm1,%%zmm2 \n" + "vpunpckhbw %%zmm0,%%zmm1,%%zmm1 \n" + "vshufi64x2 $0x44,%%zmm1,%%zmm2,%%zmm0 \n" + "vshufi64x2 $0xee,%%zmm1,%%zmm2,%%zmm1 \n" + "vshufi64x2 $0xd8,%%zmm0,%%zmm0,%%zmm0 \n" + "vshufi64x2 $0xd8,%%zmm1,%%zmm1,%%zmm1 \n" + "vmovdqu32 %%zmm0,(%1) \n" + "vmovdqu32 %%zmm1,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x20,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "je 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpsllw $0x1,%%zmm0,%%zmm1 \n" + "vpsllw $0xb,%%zmm0,%%zmm2 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpmulhuw %%zmm5,%%zmm2,%%zmm2 \n" + "vpmulhuw %%zmm5,%%zmm1,%%zmm1 \n" + "vpsllw $0x8,%%zmm1,%%zmm1 \n" + "vpord %%zmm2,%%zmm1,%%zmm1 \n" + "vpsraw $0x8,%%zmm0,%%zmm2 \n" + "vpandd %%zmm4,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n" + "vpandd %%zmm7,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vpunpcklbw %%zmm0,%%zmm1,%%zmm2 \n" + "vpunpckhbw %%zmm0,%%zmm1,%%zmm1 \n" + "vshufi64x2 $0x44,%%zmm1,%%zmm2,%%zmm0 \n" + "vshufi64x2 $0xee,%%zmm1,%%zmm2,%%zmm1 \n" + "vshufi64x2 $0xd8,%%zmm0,%%zmm0,%%zmm0 \n" + "vshufi64x2 $0xd8,%%zmm1,%%zmm1,%%zmm1 \n" + "cmp $0x10,%2 \n" + "jle 3f \n" + "vmovdqu32 %%zmm0,(%1) \n" + "kshiftrd $0x10,%%k1,%%k1 \n" + "vmovdqu32 %%zmm1,0x40(%1)%{%%k1%} \n" + "jmp 99f \n" + + "3: \n" + "vmovdqu32 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "k1"); +} +#endif // HAS_ARGB1555TOARGBROW_AVX512BW + #ifdef HAS_ARGB4444TOARGBROW_AVX2 void ARGB4444ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { asm volatile( @@ -1273,6 +1334,72 @@ void ARGBToRGB565DitherRow_AVX2(const uint8_t* src, } #endif // HAS_ARGBTORGB565DITHERROW_AVX2 +#ifdef HAS_ARGBTORGB565DITHERROW_AVX512BW +void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src, + uint8_t* dst, + uint32_t dither4, + int width) { + uintptr_t temp; + asm volatile( + "vmovd %[dither4],%%xmm6 \n" + "vpunpcklbw %%xmm6,%%xmm6,%%xmm6 \n" + "vpunpcklwd %%xmm6,%%xmm6,%%xmm6 \n" + "vshufi32x4 $0x00,%%zmm6,%%zmm6,%%zmm6 \n" + "vpternlogd $0xff,%%zmm3,%%zmm3,%%zmm3 \n" + "vpsrld $0x1b,%%zmm3,%%zmm3 \n" + "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n" + "vpsrld $0x1a,%%zmm4,%%zmm4 \n" + "vpslld $0x5,%%zmm4,%%zmm4 \n" + "vpslld $0xb,%%zmm3,%%zmm5 \n" + "sub $0x10,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpaddusb %%zmm6,%%zmm0,%%zmm0 \n" + "vpsrld $0x3,%%zmm0,%%zmm1 \n" + "vpsrld $0x5,%%zmm0,%%zmm2 \n" + "vpsrld $0x8,%%zmm0,%%zmm0 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpandd %%zmm4,%%zmm2,%%zmm2 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n" + "vpmovdw %%zmm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%2 \n" + "je 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpaddusb %%zmm6,%%zmm0,%%zmm0 \n" + "vpsrld $0x3,%%zmm0,%%zmm1 \n" + "vpsrld $0x5,%%zmm0,%%zmm2 \n" + "vpsrld $0x8,%%zmm0,%%zmm0 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpandd %%zmm4,%%zmm2,%%zmm2 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n" + "vpmovdw %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : [dither4] "r"(dither4) // %[dither4] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "k1"); +} +#endif // HAS_ARGBTORGB565DITHERROW_AVX512BW + #ifdef HAS_ARGBTORGB565ROW_AVX2 void ARGBToRGB565Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { asm volatile( @@ -1310,6 +1437,62 @@ void ARGBToRGB565Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBTORGB565ROW_AVX2 +#ifdef HAS_ARGBTORGB565ROW_AVX512BW +void ARGBToRGB565Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + uintptr_t temp; + asm volatile( + "vpternlogd $0xff,%%zmm3,%%zmm3,%%zmm3 \n" + "vpsrld $0x1b,%%zmm3,%%zmm3 \n" + "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n" + "vpsrld $0x1a,%%zmm4,%%zmm4 \n" + "vpslld $0x5,%%zmm4,%%zmm4 \n" + "vpslld $0xb,%%zmm3,%%zmm5 \n" + "sub $0x10,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x3,%%zmm0,%%zmm1 \n" + "vpsrld $0x5,%%zmm0,%%zmm2 \n" + "vpsrld $0x8,%%zmm0,%%zmm0 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpandd %%zmm4,%%zmm2,%%zmm2 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n" + "vpmovdw %%zmm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%2 \n" + "je 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpsrld $0x3,%%zmm0,%%zmm1 \n" + "vpsrld $0x5,%%zmm0,%%zmm2 \n" + "vpsrld $0x8,%%zmm0,%%zmm0 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpandd %%zmm4,%%zmm2,%%zmm2 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n" + "vpmovdw %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1"); +} +#endif // HAS_ARGBTORGB565ROW_AVX512BW + #ifdef HAS_ARGBTOARGB1555ROW_AVX2 void ARGBToARGB1555Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { asm volatile( @@ -3365,6 +3548,8 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "psubsw %%xmm1,%%xmm4 \n" \ "movdqa %%xmm4,%%xmm1 \n" +#define YUVTORGB16_AR30(yuvconstants) YUVTORGB16(yuvconstants) + #define YUVTORGB_REGS "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", #else @@ -3391,6 +3576,29 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "psubsw %%xmm1,%%xmm4 \n" \ "movdqa %%xmm4,%%xmm1 \n" +#define YUVTORGB16_AR30(yuvconstants) \ + "pcmpeqb %%xmm0,%%xmm0 \n" \ + "pabsb %%xmm0,%%xmm0 \n" \ + "psllw $7,%%xmm0 \n" \ + "psubb %%xmm0,%%xmm3 \n" \ + "pmulhuw 96(%[yuvconstants]),%%xmm4 \n" \ + "movdqa (%[yuvconstants]),%%xmm0 \n" \ + "movdqa 32(%[yuvconstants]),%%xmm1 \n" \ + "movdqa 64(%[yuvconstants]),%%xmm2 \n" \ + "pmaddubsw %%xmm3,%%xmm0 \n" \ + "pmaddubsw %%xmm3,%%xmm1 \n" \ + "pmaddubsw %%xmm3,%%xmm2 \n" \ + "movdqa 128(%[yuvconstants]),%%xmm3 \n" \ + "paddw %%xmm3,%%xmm4 \n" \ + "pcmpeqb %%xmm3,%%xmm3 \n" \ + "psrlw $14,%%xmm3 \n" \ + "psllw $3,%%xmm3 \n" \ + "psubw %%xmm3,%%xmm4 \n" \ + "paddsw %%xmm4,%%xmm0 \n" \ + "paddsw %%xmm4,%%xmm2 \n" \ + "psubsw %%xmm1,%%xmm4 \n" \ + "movdqa %%xmm4,%%xmm1 \n" + #define YUVTORGB_REGS #endif // defined(__x86_64__) @@ -3501,6 +3709,43 @@ void OMITFP I444AlphaToARGBRow_SSSE3(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "movd %[temp],%%xmm7 \n" + "movd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "movd %%xmm7,%[temp] \n" + "movq (%[u_buf]),%%xmm3 \n" + "movq 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x8(%[u_buf]),%[u_buf] \n" + "punpcklbw %%xmm1,%%xmm3 \n" + "movq (%[y_buf]),%%xmm4 \n" + "punpcklbw %%xmm4,%%xmm4 \n" + "lea 0x8(%[y_buf]),%[y_buf] \n" + "movd %%xmm6,%[temp] \n" + "movq (%[temp]),%%xmm5 \n" + "lea 0x8(%[temp]),%[temp] \n" + "movd %[temp],%%xmm6 \n" + YUVTORGB(yuvconstants) + STOREARGB + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -3513,14 +3758,11 @@ void OMITFP I444AlphaToARGBRow_SSSE3(const uint8_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // HAS_I444ALPHATOARGBROW_SSSE3 @@ -3625,6 +3867,7 @@ void OMITFP I422ToARGBRow_SSSE3(const uint8_t* y_buf, ); } +#ifdef HAS_I422TOAR30ROW_SSSE3 void OMITFP I422ToAR30Row_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -3644,20 +3887,25 @@ void OMITFP I422ToAR30Row_SSSE3(const uint8_t* y_buf, LABELALIGN "1: \n" READYUV422 - YUVTORGB16(yuvconstants) + YUVTORGB16_AR30(yuvconstants) STOREAR30 - "sub $0x8,%[width] \n" + "subl $0x8,%[width] \n" "jg 1b \n" : [y_buf]"+r"(y_buf), // %[y_buf] [u_buf]"+r"(u_buf), // %[u_buf] [v_buf]"+r"(v_buf), // %[v_buf] [dst_ar30]"+r"(dst_ar30), // %[dst_ar30] +#if defined(__i386__) + [width]"+m"(width) // %[width] +#else [width]"+rm"(width) // %[width] +#endif : [yuvconstants]"r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" ); } +#endif // HAS_I422TOAR30ROW_SSSE3 // 10 bit YUV to ARGB void OMITFP I210ToARGBRow_SSSE3(const uint16_t* y_buf, @@ -3827,6 +4075,51 @@ void OMITFP I210AlphaToARGBRow_SSSE3(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "movd %[temp],%%xmm7 \n" + "movd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "movd %%xmm7,%[temp] \n" + "movq (%[u_buf]),%%xmm3 \n" + "movq 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x8(%[u_buf]),%[u_buf] \n" + "punpcklwd %%xmm1,%%xmm3 \n" + "psraw $2,%%xmm3 \n" + "packuswb %%xmm3,%%xmm3 \n" + "punpcklwd %%xmm3,%%xmm3 \n" + "movdqu (%[y_buf]),%%xmm4 \n" + "movdqa %%xmm4,%%xmm2 \n" + "psllw $6,%%xmm4 \n" + "psrlw $4,%%xmm2 \n" + "paddw %%xmm2,%%xmm4 \n" + "lea 0x10(%[y_buf]),%[y_buf] \n" + "movd %%xmm6,%[temp] \n" + "movdqu (%[temp]),%%xmm5 \n" + "psraw $2,%%xmm5 \n" + "packuswb %%xmm5,%%xmm5 \n" + "lea 0x10(%[temp]),%[temp] \n" + "movd %[temp],%%xmm6 \n" + YUVTORGB(yuvconstants) + STOREARGB + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -3839,14 +4132,11 @@ void OMITFP I210AlphaToARGBRow_SSSE3(const uint16_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // HAS_I210ALPHATOARGBROW_SSSE3 @@ -3859,6 +4149,53 @@ void OMITFP I410AlphaToARGBRow_SSSE3(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "movd %[temp],%%xmm7 \n" + "movd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "movd %%xmm7,%[temp] \n" + "movdqu (%[u_buf]),%%xmm3 \n" + "movdqu 0x00(%[u_buf],%[temp],1),%%xmm2 \n" + "lea 0x10(%[u_buf]),%[u_buf] \n" + "psraw $2,%%xmm3 \n" + "psraw $2,%%xmm2 \n" + "movdqa %%xmm3,%%xmm1 \n" + "punpcklwd %%xmm2,%%xmm3 \n" + "punpckhwd %%xmm2,%%xmm1 \n" + "packuswb %%xmm1,%%xmm3 \n" + "movdqu (%[y_buf]),%%xmm4 \n" + "movdqa %%xmm4,%%xmm2 \n" + "psllw $6,%%xmm4 \n" + "psrlw $4,%%xmm2 \n" + "paddw %%xmm2,%%xmm4 \n" + "lea 0x10(%[y_buf]),%[y_buf] \n" + "movd %%xmm6,%[temp] \n" + "movdqu (%[temp]),%%xmm5 \n" + "psraw $2,%%xmm5 \n" + "packuswb %%xmm5,%%xmm5 \n" + "lea 0x10(%[temp]),%[temp] \n" + "movd %[temp],%%xmm6 \n" + YUVTORGB(yuvconstants) + STOREARGB + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -3871,14 +4208,11 @@ void OMITFP I410AlphaToARGBRow_SSSE3(const uint16_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // HAS_I410ALPHATOARGBROW_SSSE3 @@ -3925,6 +4259,44 @@ void OMITFP I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "movd %[temp],%%xmm7 \n" + "movd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "movd %%xmm7,%[temp] \n" + "movd (%[u_buf]),%%xmm3 \n" + "movd 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x4(%[u_buf]),%[u_buf] \n" + "punpcklbw %%xmm1,%%xmm3 \n" + "punpcklwd %%xmm3,%%xmm3 \n" + "movq (%[y_buf]),%%xmm4 \n" + "punpcklbw %%xmm4,%%xmm4 \n" + "lea 0x8(%[y_buf]),%[y_buf] \n" + "movd %%xmm6,%[temp] \n" + "movq (%[temp]),%%xmm5 \n" + "lea 0x8(%[temp]),%[temp] \n" + "movd %[temp],%%xmm6 \n" + YUVTORGB(yuvconstants) + STOREARGB + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -3937,14 +4309,11 @@ void OMITFP I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // HAS_I422ALPHATOARGBROW_SSSE3 @@ -4466,6 +4835,8 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \ "vpaddsw %%ymm4,%%ymm2,%%ymm2 \n" +#define YUVTORGB16_AR30_AVX2(yuvconstants) YUVTORGB16_AVX2(yuvconstants) + #define YUVTORGB16_AVX512BW(yuvconstants) \ "vpsubb %%zmm13,%%zmm3,%%zmm3 \n" \ "vpmulhuw %%zmm11,%%zmm4,%%zmm4 \n" \ @@ -4504,6 +4875,28 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \ "vpaddsw %%ymm4,%%ymm2,%%ymm2 \n" +#define YUVTORGB16_AR30_AVX2(yuvconstants) \ + "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" \ + "vpabsb %%ymm0,%%ymm0 \n" \ + "vpsllw $7,%%ymm0,%%ymm0 \n" \ + "vpsubb %%ymm0,%%ymm3,%%ymm3 \n" \ + "vpmulhuw 96(%[yuvconstants]),%%ymm4,%%ymm4 \n" \ + "vmovdqa (%[yuvconstants]),%%ymm0 \n" \ + "vmovdqa 32(%[yuvconstants]),%%ymm1 \n" \ + "vmovdqa 64(%[yuvconstants]),%%ymm2 \n" \ + "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n" \ + "vpmaddubsw %%ymm3,%%ymm1,%%ymm1 \n" \ + "vpmaddubsw %%ymm3,%%ymm2,%%ymm2 \n" \ + "vmovdqa 128(%[yuvconstants]),%%ymm3 \n" \ + "vpaddw %%ymm4,%%ymm3,%%ymm4 \n" \ + "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpsrlw $14,%%ymm3,%%ymm3 \n" \ + "vpsllw $3,%%ymm3,%%ymm3 \n" \ + "vpsubw %%ymm3,%%ymm4,%%ymm4 \n" \ + "vpaddsw %%ymm4,%%ymm0,%%ymm0 \n" \ + "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \ + "vpaddsw %%ymm4,%%ymm2,%%ymm2 \n" + #define YUVTORGB_REGS_AVX2 #endif // defined(__x86_64__) @@ -4827,28 +5220,46 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf, } #endif // defined(HAS_I422TORGB24ROW_AVX2) -#if defined(HAS_I422TOARGBROW_AVX512BW) || \ - defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ - defined(HAS_I422TORGB24ROW_AVX512BW) || \ - defined(HAS_I422TOAR30ROW_AVX512BW) +#if defined(HAS_I422TOARGBROW_AVX512BW) || \ + defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I422TORGB24ROW_AVX512BW) || \ + defined(HAS_I422TOAR30ROW_AVX512BW) || \ + defined(HAS_I444TOARGBROW_AVX512BW) || \ + defined(HAS_I444ALPHATOARGBROW_AVX512BW) || \ + defined(HAS_I444TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I422TORGBAROW_AVX512BW) || \ + defined(HAS_YUY2TOARGBROW_AVX512BW) || \ + defined(HAS_UYVYTOARGBROW_AVX512BW) || \ + defined(HAS_I210TOARGBROW_AVX512BW) || \ + defined(HAS_I210TOAR30ROW_AVX512BW) || \ + defined(HAS_I210ALPHATOARGBROW_AVX512BW) || \ + defined(HAS_I212TOARGBROW_AVX512BW) || \ + defined(HAS_I212TOAR30ROW_AVX512BW) || \ + defined(HAS_I410TOARGBROW_AVX512BW) || \ + defined(HAS_I410TOAR30ROW_AVX512BW) || \ + defined(HAS_I410ALPHATOARGBROW_AVX512BW) || \ + defined(HAS_P210TOARGBROW_AVX512BW) || \ + defined(HAS_P210TOAR30ROW_AVX512BW) || \ + defined(HAS_P410TOARGBROW_AVX512BW) || defined(HAS_P410TOAR30ROW_AVX512BW) static const uint16_t kSplitQuadWords[32] = { 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15}; static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; static const uint64_t kUnpermuteAVX512[8] = {0, 4, 1, 5, 2, 6, 3, 7}; -#endif // defined(HAS_I422TOARGBROW_AVX512BW) || - // defined(HAS_I422TORGB24ROW_AVX512VBMI) || - // defined(HAS_I422TORGB24ROW_AVX512BW) || - // defined(HAS_I422TOAR30ROW_AVX512BW) +#endif -#if defined(HAS_I422TOAR30ROW_AVX512BW) +#if defined(HAS_I422TOAR30ROW_AVX512BW) || \ + defined(HAS_I210TOAR30ROW_AVX512BW) || \ + defined(HAS_I212TOAR30ROW_AVX512BW) || \ + defined(HAS_I410TOAR30ROW_AVX512BW) || \ + defined(HAS_P210TOAR30ROW_AVX512BW) || defined(HAS_P410TOAR30ROW_AVX512BW) static const uint16_t kPermAR30_0[32] = { 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47}; static const uint16_t kPermAR30_1[32] = { 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63}; -#endif // defined(HAS_I422TOAR30ROW_AVX512BW) +#endif #if defined(HAS_I422TOARGBROW_AVX512BW) // 32 pixels @@ -4903,7 +5314,8 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf, } #endif // defined(HAS_I422TOARGBROW_AVX512BW) -#if defined(HAS_I422TORGB24ROW_AVX512VBMI) +#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I444TORGB24ROW_AVX512VBMI) static const uint8_t kMaskBG[64] = { 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, @@ -4925,7 +5337,10 @@ static const uint8_t kMaskDST1[64] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +#endif // defined(HAS_I422TORGB24ROW_AVX512VBMI) || + // defined(HAS_I444TORGB24ROW_AVX512VBMI) +#if defined(HAS_I422TORGB24ROW_AVX512VBMI) // 32 pixels // 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96 // bytes). @@ -5131,9 +5546,9 @@ void OMITFP I422ToAR30Row_AVX2(const uint8_t* y_buf, LABELALIGN "1: \n" READYUV422_AVX2 - YUVTORGB16_AVX2(yuvconstants) + YUVTORGB16_AR30_AVX2(yuvconstants) STOREAR30_AVX2 - "sub $0x10,%[width] \n" + "subl $0x10,%[width] \n" "jg 1b \n" "vzeroupper \n" @@ -5141,7 +5556,11 @@ void OMITFP I422ToAR30Row_AVX2(const uint8_t* y_buf, [u_buf]"+r"(u_buf), // %[u_buf] [v_buf]"+r"(v_buf), // %[v_buf] [dst_ar30]"+r"(dst_ar30), // %[dst_ar30] +#if defined(__i386__) + [width]"+m"(width) // %[width] +#else [width]"+rm"(width) // %[width] +#endif : [yuvconstants]"r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" @@ -5342,6 +5761,53 @@ void OMITFP I210AlphaToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "vmovd %[temp],%%xmm7 \n" + "vmovd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "vmovd %%xmm7,%[temp] \n" + "vmovdqu (%[u_buf]),%%xmm3 \n" + "vmovdqu 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x10(%[u_buf]),%[u_buf] \n" + "vpermq $0xd8,%%ymm3,%%ymm3 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpunpcklwd %%ymm1,%%ymm3,%%ymm3 \n" + "vpsraw $2,%%ymm3,%%ymm3 \n" + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" + "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" + "vmovdqu (%[y_buf]),%%ymm4 \n" + "vpsllw $6,%%ymm4,%%ymm2 \n" + "vpsrlw $4,%%ymm4,%%ymm4 \n" + "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" + "lea 0x20(%[y_buf]),%[y_buf] \n" + "vmovd %%xmm6,%[temp] \n" + "vmovdqu (%[temp]),%%ymm5 \n" + "vpsraw $2,%%ymm5,%%ymm5 \n" + "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n" + "lea 0x20(%[temp]),%[temp] \n" + "vmovd %[temp],%%xmm6 \n" + YUVTORGB_AVX2(yuvconstants) + STOREARGB_AVX2 + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP_AVX2( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -5356,14 +5822,11 @@ void OMITFP I210AlphaToARGBRow_AVX2(const uint16_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // defined(HAS_I210ALPHATOARGBROW_AVX2) @@ -5377,6 +5840,52 @@ void OMITFP I410AlphaToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "vmovd %[temp],%%xmm7 \n" + "vmovd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "vmovd %%xmm7,%[temp] \n" + "vmovdqu (%[u_buf]),%%ymm3 \n" + "vmovdqu 0x00(%[u_buf],%[temp],1),%%ymm2 \n" + "lea 0x20(%[u_buf]),%[u_buf] \n" + "vpsraw $2,%%ymm3,%%ymm3 \n" + "vpsraw $2,%%ymm2,%%ymm2 \n" + "vpunpckhwd %%ymm2,%%ymm3,%%ymm1 \n" + "vpunpcklwd %%ymm2,%%ymm3,%%ymm3 \n" + "vpackuswb %%ymm1,%%ymm3,%%ymm3 \n" + "vmovdqu (%[y_buf]),%%ymm4 \n" + "vpsllw $6,%%ymm4,%%ymm2 \n" + "vpsrlw $4,%%ymm4,%%ymm4 \n" + "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" + "lea 0x20(%[y_buf]),%[y_buf] \n" + "vmovd %%xmm6,%[temp] \n" + "vmovdqu (%[temp]),%%ymm5 \n" + "vpsraw $2,%%ymm5,%%ymm5 \n" + "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n" + "lea 0x20(%[temp]),%[temp] \n" + "vmovd %[temp],%%xmm6 \n" + YUVTORGB_AVX2(yuvconstants) + STOREARGB_AVX2 + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP_AVX2( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -5391,14 +5900,11 @@ void OMITFP I410AlphaToARGBRow_AVX2(const uint16_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // defined(HAS_I410ALPHATOARGBROW_AVX2) @@ -5451,6 +5957,48 @@ void OMITFP I444AlphaToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "vmovd %[temp],%%xmm7 \n" + "vmovd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "vmovd %%xmm7,%[temp] \n" + "vmovdqu (%[u_buf]),%%xmm3 \n" + "vmovdqu 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x10(%[u_buf]),%[u_buf] \n" + "vpermq $0xd8,%%ymm3,%%ymm3 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" + "vmovdqu (%[y_buf]),%%xmm4 \n" + "vpermq $0xd8,%%ymm4,%%ymm4 \n" + "vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" + "lea 0x10(%[y_buf]),%[y_buf] \n" + "vmovd %%xmm6,%[temp] \n" + "vmovdqu (%[temp]),%%xmm5 \n" + "vpermq $0xd8,%%ymm5,%%ymm5 \n" + "lea 0x10(%[temp]),%[temp] \n" + "vmovd %[temp],%%xmm6 \n" + YUVTORGB_AVX2(yuvconstants) + STOREARGB_AVX2 + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP_AVX2( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -5464,14 +6012,11 @@ void OMITFP I444AlphaToARGBRow_AVX2(const uint8_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // defined(HAS_I444ALPHATOARGBROW_AVX2) @@ -5485,6 +6030,48 @@ void OMITFP I422AlphaToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "vmovd %[temp],%%xmm7 \n" + "vmovd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "vmovd %%xmm7,%[temp] \n" + "vmovq (%[u_buf]),%%xmm3 \n" + "vmovq 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x8(%[u_buf]),%[u_buf] \n" + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" + "vpermq $0xd8,%%ymm3,%%ymm3 \n" + "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" + "vmovdqu (%[y_buf]),%%xmm4 \n" + "vpermq $0xd8,%%ymm4,%%ymm4 \n" + "vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" + "lea 0x10(%[y_buf]),%[y_buf] \n" + "vmovd %%xmm6,%[temp] \n" + "vmovdqu (%[temp]),%%xmm5 \n" + "vpermq $0xd8,%%ymm5,%%ymm5 \n" + "lea 0x10(%[temp]),%[temp] \n" + "vmovd %[temp],%%xmm6 \n" + YUVTORGB_AVX2(yuvconstants) + STOREARGB_AVX2 + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP_AVX2( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -5498,14 +6085,11 @@ void OMITFP I422AlphaToARGBRow_AVX2(const uint8_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // defined(HAS_I422ALPHATOARGBROW_AVX2) @@ -5896,26 +6480,69 @@ void MirrorRow_SSSE3(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_MIRRORROW_SSSE3 +#if defined(HAS_MIRRORROW_AVX512BW) || \ + defined(HAS_MIRRORSPLITUVROW_AVX512BW) || \ + defined(HAS_MIRRORUVROW_AVX512BW) +static const uint16_t kMirrorUVPermute_AVX512[32] = { + 31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u, + 20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u, + 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u}; +#endif // defined(HAS_MIRRORROW_AVX512BW) || + // defined(HAS_MIRRORSPLITUVROW_AVX512BW) || + // defined(HAS_MIRRORUVROW_AVX512BW) + #ifdef HAS_MIRRORROW_AVX512BW void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); - asm volatile("vbroadcasti32x4 %3,%%zmm5 \n" + uintptr_t temp; + asm volatile( + "vbroadcasti32x4 %[shuf],%%zmm5 \n" + "sub $0x40,%[width] \n" + "jl 2f \n" - LABELALIGN - "1: \n" - "vmovdqu8 -0x40(%0,%2,1),%%zmm0 \n" - "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" - "vshufi64x2 $0x1b,%%zmm0,%%zmm0,%%zmm0 \n" - "vmovdqu8 %%zmm0,(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(temp_width) // %2 - : "m"(kShuffleMirror) // %3 - : "memory", "cc", "xmm0", "xmm5"); + LABELALIGN + "1: \n" + "vmovdqu8 (%[src],%[width],1),%%zmm0 \n" + "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" + "vshufi64x2 $0x1b,%%zmm0,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "vmovdqu8 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpsrlw $0x8,%%zmm0,%%zmm1 \n" + "vpsllw $0x8,%%zmm0,%%zmm0 \n" + "vporq %%zmm1,%%zmm0,%%zmm0 \n" + "mov $0x40,%k[temp] \n" + "sub %k[width],%k[temp] \n" + "shr $0x1,%k[temp] \n" + "vpbroadcastw %k[temp],%%zmm1 \n" + "vmovdqu16 %[rev],%%zmm2 \n" + "vpsubw %%zmm1,%%zmm2,%%zmm1 \n" + "vpermw %%zmm0,%%zmm1,%%zmm0 \n" + "test $0x1,%b[width] \n" + "je 3f \n" + "valignq $0x2,%%zmm0,%%zmm0,%%zmm1 \n" + "vpalignr $0x1,%%zmm0,%%zmm1,%%zmm0 \n" + "3: \n" + "vmovdqu8 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst), + [width] "+r"(temp_width), + [temp] "=&r"(temp) + : [src] "r"(src), + [shuf] "m"(kShuffleMirror), + [rev] "m"(kMirrorUVPermute_AVX512) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm5", "k1"); } #endif // HAS_MIRRORROW_AVX512BW @@ -5949,14 +6576,6 @@ static const uvec8 kShuffleMirrorSplitUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u, #endif // defined(HAS_MIRRORSPLITUVROW_AVX2) || // defined(HAS_MIRRORSPLITUVROW_AVX512BW) -#if defined(HAS_MIRRORSPLITUVROW_AVX512BW) || defined(HAS_MIRRORUVROW_AVX512BW) -static const uint16_t kMirrorUVPermute_AVX512[32] = { - 31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u, - 20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u, - 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u}; -#endif // defined(HAS_MIRRORSPLITUVROW_AVX512BW) || - // defined(HAS_MIRRORUVROW_AVX512BW) - #ifdef HAS_MIRRORSPLITUVROW_AVX512BW static const uint64_t kMirrorSplitUVPermute[8] = {6, 4, 2, 0, 7, 5, 3, 1}; @@ -6100,21 +6719,42 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { #ifdef HAS_MIRRORUVROW_AVX512BW void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); - asm volatile("vmovdqu16 %3,%%zmm5 \n" + uintptr_t temp; + asm volatile( + "vmovdqu16 %[rev],%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" - LABELALIGN - "1: \n" - "vpermw -0x40(%0,%2,2),%%zmm5,%%zmm0 \n" - "vmovdqu16 %%zmm0,(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x20,%2 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_uv), // %0 - "+r"(dst_uv), // %1 - "+r"(temp_width) // %2 - : "m"(kMirrorUVPermute_AVX512) // %3 - : "memory", "cc", "xmm0", "xmm5"); + LABELALIGN + "1: \n" + "vpermw (%[src_uv],%[width],2),%%zmm5,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst_uv]) \n" + "lea 0x40(%[dst_uv]),%[dst_uv] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k[width],%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%[src_uv]),%%zmm0%{%%k1%}%{z%}\n" + "mov $0x20,%k[temp] \n" + "sub %k[width],%k[temp] \n" + "vpbroadcastw %k[temp],%%zmm1 \n" + "vpsubw %%zmm1,%%zmm5,%%zmm1 \n" + "vpermw %%zmm0,%%zmm1,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst_uv])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [dst_uv] "+r"(dst_uv), + [width] "+r"(temp_width), + [temp] "=&r"(temp) + : [src_uv] "r"(src_uv), + [rev] "m"(kMirrorUVPermute_AVX512) + : "memory", "cc", "xmm0", "xmm1", "xmm5", "k1"); } #endif // HAS_MIRRORUVROW_AVX512BW @@ -6560,140 +7200,216 @@ void SplitUVRow_SSE2(const uint8_t* src_uv, } #endif // HAS_SPLITUVROW_SSE2 -#ifdef HAS_DETILEROW_SSE2 -void DetileRow_SSE2(const uint8_t* src, +#ifdef HAS_DETILEROW_AVX2 +void DetileRow_AVX2(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, int width) { asm volatile( - "1: \n" - "movdqu (%0),%%xmm0 \n" - "sub $0x10,%2 \n" - "lea (%0,%3),%0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "r"(src_tile_stride) // %3 - : "cc", "memory", "xmm0"); -} -#endif // HAS_DETILEROW_SSE2 + "sub $0x20,%2 \n" + "jl 2f \n" -#ifdef HAS_DETILEROW_16_SSE2 -void DetileRow_16_SSE2(const uint16_t* src, - ptrdiff_t src_tile_stride, - uint16_t* dst, - int width) { - asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "lea (%0,%3,2),%0 \n" - "movdqu %%xmm0,(%1) \n" - "movdqu %%xmm1,0x10(%1) \n" - "lea 0x20(%1),%1 \n" - "sub $0x10,%2 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "r"(src_tile_stride) // %3 - : "cc", "memory", "xmm0", "xmm1"); -} -#endif // HAS_DETILEROW_16_SSE2 - -#ifdef HAS_DETILEROW_16_AVX -void DetileRow_16_AVX(const uint16_t* src, - ptrdiff_t src_tile_stride, - uint16_t* dst, - int width) { - asm volatile( - "1: \n" - "vmovdqu (%0),%%ymm0 \n" + "vmovdqu (%0),%%xmm0 \n" + "vinserti128 $0x1,(%0,%3,1),%%ymm0,%%ymm0 \n" "lea (%0,%3,2),%0 \n" "vmovdqu %%ymm0,(%1) \n" "lea 0x20(%1),%1 \n" - "sub $0x10,%2 \n" - "jg 1b \n" + "sub $0x20,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%xmm0 \n" + "vmovdqu %%xmm0,(%1) \n" + + "99: \n" "vzeroupper \n" : "+r"(src), // %0 "+r"(dst), // %1 "+r"(width) // %2 : "r"(src_tile_stride) // %3 - : "cc", "memory", "xmm0"); + : "memory", "cc", "xmm0"); } -#endif // HAS_DETILEROW_16_AVX +#endif // HAS_DETILEROW_AVX2 -#ifdef HAS_DETILETOYUY2_SSE2 -// Read 16 Y, 8 UV, and write 8 YUYV. -void DetileToYUY2_SSE2(const uint8_t* src_y, +#ifdef HAS_DETILEROW_16_AVX2 +void DetileRow_16_AVX2(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width) { + asm volatile( + "sub $0x20,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu (%0,%3,2),%%ymm1 \n" + "lea (%0,%3,4),%0 \n" + "vmovdqu %%ymm0,(%1) \n" + "vmovdqu %%ymm1,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "r"(src_tile_stride) // %3 + : "memory", "cc", "xmm0", "xmm1"); +} +#endif // HAS_DETILEROW_16_AVX2 + +#ifdef HAS_DETILETOYUY2_AVX2 +// Read 32 Y, 16 UV, and write 16 YUYV (64 bytes). +void DetileToYUY2_AVX2(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, uint8_t* dst_yuy2, int width) { asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" // Load 16 Y - "sub $0x10,%3 \n" - "lea (%0,%4),%0 \n" - "movdqu (%1),%%xmm1 \n" // Load 8 UV - "lea (%1,%5),%1 \n" - "movdqu %%xmm0,%%xmm2 \n" - "punpcklbw %%xmm1,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm2 \n" - "movdqu %%xmm0,(%2) \n" - "movdqu %%xmm2,0x10(%2) \n" - "lea 0x20(%2),%2 \n" + "vmovdqu (%0),%%xmm0 \n" + "vinserti128 $0x1,(%0,%4,1),%%ymm0,%%ymm0 \n" + "lea (%0,%4,2),%0 \n" + "vmovdqu (%1),%%xmm1 \n" + "vinserti128 $0x1,(%1,%5,1),%%ymm1,%%ymm1 \n" + "lea (%1,%5,2),%1 \n" + "vpunpcklbw %%ymm1,%%ymm0,%%ymm2 \n" + "vpunpckhbw %%ymm1,%%ymm0,%%ymm3 \n" + "vperm2i128 $0x20,%%ymm3,%%ymm2,%%ymm0 \n" + "vperm2i128 $0x31,%%ymm3,%%ymm2,%%ymm1 \n" + "vmovdqu %%ymm0,(%2) \n" + "vmovdqu %%ymm1,0x20(%2) \n" + "lea 0x40(%2),%2 \n" + "subl $0x20,%3 \n" "jg 1b \n" - : "+r"(src_y), // %0 - "+r"(src_uv), // %1 - "+r"(dst_yuy2), // %2 - "+r"(width) // %3 - : "r"(src_y_tile_stride), // %4 - "r"(src_uv_tile_stride) // %5 - : "cc", "memory", "xmm0", "xmm1", "xmm2" // Clobber list - ); + "vzeroupper \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_yuy2), // %2 + "+rm"(width) // %3 + : "r"(src_y_tile_stride), // %4 + "r"(src_uv_tile_stride) // %5 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } -#endif // HAS_DETILETOYUY2_SSE2 +#endif // HAS_DETILETOYUY2_AVX2 -#ifdef HAS_DETILESPLITUVROW_SSSE3 -// TODO(greenjustin): Look into generating these constants instead of loading -// them since this can cause branch mispredicts for fPIC code on 32-bit -// machines. +#ifdef HAS_UNPACKMT2T_AVX2 +void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) { + static const uint32_t kMaskC0 = 0xc0c0c0c0u; + if (size >= 80) { + asm volatile( + "vpbroadcastd %[c0],%%ymm6 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vpsllw $2,%%xmm0,%%xmm1 \n" + "vinserti128 $0x1,%%xmm0,%%ymm1,%%ymm1 \n" + "vpsllw $4,%%ymm1,%%ymm0 \n" + "vpand %%ymm6,%%ymm0,%%ymm0 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vmovdqu 0x10(%[src]),%%ymm2 \n" + "vmovdqu 0x30(%[src]),%%ymm3 \n" + "lea 0x50(%[src]),%[src] \n" + "vpsrlw $2,%%ymm2,%%ymm4 \n" + "vpsrlw $2,%%ymm3,%%ymm5 \n" + "vpandn %%ymm4,%%ymm6,%%ymm4 \n" + "vpandn %%ymm5,%%ymm6,%%ymm5 \n" + "vpor %%ymm4,%%ymm0,%%ymm0 \n" + "vpor %%ymm5,%%ymm1,%%ymm1 \n" + + "vpunpcklbw %%ymm2,%%ymm0,%%ymm4 \n" + "vpunpckhbw %%ymm2,%%ymm0,%%ymm5 \n" + "vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm0 \n" + "vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm2 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "vmovdqu %%ymm2,0x20(%[dst]) \n" + + "vpunpcklbw %%ymm3,%%ymm1,%%ymm4 \n" + "vpunpckhbw %%ymm3,%%ymm1,%%ymm5 \n" + "vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm1 \n" + "vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm3 \n" + "vmovdqu %%ymm1,0x40(%[dst]) \n" + "vmovdqu %%ymm3,0x60(%[dst]) \n" + + "lea 0x80(%[dst]),%[dst] \n" + "sub $0x50,%[size] \n" + "cmp $0x50,%[size] \n" + "jae 1b \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [size] "+r"(size) + : [c0] "m"(kMaskC0) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6"); + } + if (size > 0) { + UnpackMT2T_C(src, dst, size); + } +} +#endif // HAS_UNPACKMT2T_AVX2 + +#ifdef HAS_DETILESPLITUVROW_AVX2 static const uvec8 kDeinterlaceUV = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15}; -// TODO(greenjustin): Research alternatives to pshufb, since pshufb can be very -// slow on older SSE2 processors. -void DetileSplitUVRow_SSSE3(const uint8_t* src_uv, - ptrdiff_t src_tile_stride, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { +void DetileSplitUVRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { asm volatile( - "movdqu %4,%%xmm1 \n" + "vbroadcasti128 %4,%%ymm1 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "lea (%0, %5),%0 \n" - "pshufb %%xmm1,%%xmm0 \n" - "movq %%xmm0,(%1) \n" - "lea 0x8(%1),%1 \n" - "movhps %%xmm0,(%2) \n" - "lea 0x8(%2),%2 \n" - "sub $0x10,%3 \n" - "jg 1b \n" + "vmovdqu (%0),%%xmm0 \n" + "vinserti128 $0x1,(%0,%5,1),%%ymm0,%%ymm0 \n" + "lea (%0,%5,2),%0 \n" + "vpshufb %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%1) \n" + "lea 0x10(%1),%1 \n" + "vextracti128 $0x1,%%ymm0,(%2) \n" + "lea 0x10(%2),%2 \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "jle 99f \n" + "vmovdqu (%0),%%xmm0 \n" + "vpshufb %%xmm1,%%xmm0,%%xmm0 \n" + "vmovq %%xmm0,(%1) \n" + "vmovhps %%xmm0,(%2) \n" + + "99: \n" + "vzeroupper \n" : "+r"(src_uv), // %0 "+r"(dst_u), // %1 "+r"(dst_v), // %2 "+r"(width) // %3 : "m"(kDeinterlaceUV), // %4 "r"(src_tile_stride) // %5 - : "cc", "memory", "xmm0", "xmm1"); + : "memory", "cc", "xmm0", "xmm1"); } -#endif // HAS_DETILESPLITUVROW_SSSE3 +#endif // HAS_DETILESPLITUVROW_AVX2 #ifdef HAS_MERGEUVROW_AVX512BW void MergeUVRow_AVX512BW(const uint8_t* src_u, @@ -6816,6 +7532,80 @@ void MergeUVRow_16_AVX2(const uint16_t* src_u, } #endif // HAS_MERGEUVROW_16_AVX2 +#ifdef HAS_MERGEUVROW_16_AVX512BW +void MergeUVRow_16_AVX512BW(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width) { + uintptr_t temp; + asm volatile( + "vmovd %[shift_u],%%xmm3 \n" + "vmovd %[shift_v],%%xmm4 \n" + "sub %0,%1 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + // 32 pixels per loop. + LABELALIGN + "1: \n" + "vpmovzxwd (%0),%%zmm0 \n" + "vpmovzxwd 0x20(%0),%%zmm2 \n" + "vpmovzxwd 0x00(%0,%1,1),%%zmm1 \n" + "vpmovzxwd 0x20(%0,%1,1),%%zmm5 \n" + "lea 0x40(%0),%0 \n" + "vpsllw %%xmm3,%%zmm0,%%zmm0 \n" + "vpsllw %%xmm3,%%zmm2,%%zmm2 \n" + "vpslld %%xmm4,%%zmm1,%%zmm1 \n" + "vpslld %%xmm4,%%zmm5,%%zmm5 \n" + "vpord %%zmm0,%%zmm1,%%zmm0 \n" + "vpord %%zmm2,%%zmm5,%%zmm2 \n" + "vmovdqu32 %%zmm0,(%2) \n" + "vmovdqu32 %%zmm2,0x40(%2) \n" + "lea 0x80(%2),%2 \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "je 99f \n" + "cmp $0x10,%3 \n" + "jl 3f \n" + "vpmovzxwd (%0),%%zmm0 \n" + "vpmovzxwd 0x00(%0,%1,1),%%zmm1 \n" + "lea 0x20(%0),%0 \n" + "vpsllw %%xmm3,%%zmm0,%%zmm0 \n" + "vpslld %%xmm4,%%zmm1,%%zmm1 \n" + "vpord %%zmm0,%%zmm1,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%2) \n" + "lea 0x40(%2),%2 \n" + "sub $0x10,%3 \n" + "je 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k3,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vpmovzxwd (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmovzxwd 0x00(%0,%1,1),%%zmm1%{%%k1%}%{z%} \n" + "vpsllw %%xmm3,%%zmm0,%%zmm0 \n" + "vpslld %%xmm4,%%zmm1,%%zmm1 \n" + "vpord %%zmm0,%%zmm1,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%2)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_u), // %0 + "+r"(src_v), // %1 + "+r"(dst_uv), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : [shift_u] "r"(16 - depth), // %[shift_u] + [shift_v] "r"(32 - depth) // %[shift_v] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1"); +} +#endif // HAS_MERGEUVROW_16_AVX512BW + #ifdef HAS_SPLITUVROW_16_AVX2 const uvec8 kSplitUVShuffle16 = {0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15}; @@ -6959,6 +7749,60 @@ void DivideRow_16_AVX2(const uint16_t* src_y, } #endif // HAS_DIVIDEROW_16_AVX2 +#ifdef HAS_DIVIDEROW_16_AVX512BW +void DivideRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + uintptr_t temp; + asm volatile( + "vpbroadcastw %[scale],%%zmm2 \n" + "sub $0x40,%2 \n" + "jl 2f \n" + + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vpmulhuw (%0),%%zmm2,%%zmm0 \n" + "vpmulhuw 0x40(%0),%%zmm2,%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "je 99f \n" + "cmp $0x20,%2 \n" + "jl 3f \n" + "vpmulhuw (%0),%%zmm2,%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + "je 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : [scale] "r"(scale) // %[scale] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1"); +} +#endif // HAS_DIVIDEROW_16_AVX512BW + // Use scale to convert lsb formats to msb, depending how many bits there are: // 32768 = 9 bits // 16384 = 10 bits @@ -7237,6 +8081,72 @@ void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, } #endif // HAS_HALFWIDTHROW_16TO8_AVX2 +#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX2) +void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + int n = (width - 1) & ~15; + if (n > 0) { + HalfWidthRow_16To8_AVX2(src_uv, src_uv_stride, dst_uv, scale, n); + src_uv += n * 2; + dst_uv += n; + width -= n; + } + HalfWidthRow_16To8_Odd_C(src_uv, src_uv_stride, dst_uv, scale, width); +} +#endif // HAS_HALFWIDTHROW_16TO8_ODD_AVX2 + +#ifdef HAS_HALFROW_AVX2 +void HalfRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width) { + asm volatile( + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpavgb (%0,%3,1),%%ymm0,%%ymm0 \n" + "lea 0x20(%0),%0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(width) // %2 + : "r"(src_uv_stride) // %3 + : "memory", "cc", "xmm0"); +} +#endif // HAS_HALFROW_AVX2 + +#ifdef HAS_HALFROW_16_AVX2 +void HalfRow_16_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width) { + asm volatile( + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpavgw (%0,%3,2),%%ymm0,%%ymm0 \n" + "lea 0x20(%0),%0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(width) // %2 + : "r"(src_uv_stride) // %3 + : "memory", "cc", "xmm0"); +} +#endif // HAS_HALFROW_16_AVX2 + #ifdef HAS_HALFROW_16TO8_AVX512BW void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, ptrdiff_t src_uv_stride, @@ -7616,7 +8526,7 @@ void SplitRGBRow_AVX2(const uint8_t* src_rgb, } #endif // HAS_SPLITRGBROW_AVX2 -#ifdef HAS_MERGERGBROW_SSSE3 +#ifdef HAS_MERGERGBROW_AVX2 // Shuffle table for converting Planar to RGB. static const uvec8 kMergeRGBShuffle[9] = { {0u, 128u, 128u, 1u, 128u, 128u, 2u, 128u, 128u, 3u, 128u, 128u, 4u, 128u, @@ -7638,58 +8548,69 @@ static const uvec8 kMergeRGBShuffle[9] = { {10u, 128u, 128u, 11u, 128u, 128u, 12u, 128u, 128u, 13u, 128u, 128u, 14u, 128u, 128u, 15u}}; -void MergeRGBRow_SSSE3(const uint8_t* src_r, - const uint8_t* src_g, - const uint8_t* src_b, - uint8_t* dst_rgb, - int width) { +void MergeRGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width) { asm volatile( + "sub %0,%1 \n" + "sub %0,%2 \n" + + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%1),%%xmm1 \n" - "movdqu (%2),%%xmm2 \n" - "pshufb (%5), %%xmm0 \n" - "pshufb 16(%5), %%xmm1 \n" - "pshufb 32(%5), %%xmm2 \n" - "por %%xmm1,%%xmm0 \n" - "por %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%3) \n" + "vmovdqu (%0),%%ymm3 \n" + "vmovdqu (%0,%1,1),%%ymm4 \n" + "vmovdqu (%0,%2,1),%%ymm5 \n" + "lea 0x20(%0),%0 \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%1),%%xmm1 \n" - "movdqu (%2),%%xmm2 \n" - "pshufb 48(%5), %%xmm0 \n" - "pshufb 64(%5), %%xmm1 \n" - "pshufb 80(%5), %%xmm2 \n" - "por %%xmm1,%%xmm0 \n" - "por %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,16(%3) \n" + "vbroadcasti128 (%5),%%ymm6 \n" + "vbroadcasti128 16(%5),%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm0 \n" + "vpshufb %%ymm7,%%ymm4,%%ymm6 \n" + "vbroadcasti128 32(%5),%%ymm7 \n" + "vpor %%ymm6,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm7,%%ymm5,%%ymm7 \n" + "vpor %%ymm7,%%ymm0,%%ymm0 \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%1),%%xmm1 \n" - "movdqu (%2),%%xmm2 \n" - "pshufb 96(%5), %%xmm0 \n" - "pshufb 112(%5), %%xmm1 \n" - "pshufb 128(%5), %%xmm2 \n" - "por %%xmm1,%%xmm0 \n" - "por %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,32(%3) \n" + "vbroadcasti128 48(%5),%%ymm6 \n" + "vbroadcasti128 64(%5),%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm1 \n" + "vpshufb %%ymm7,%%ymm4,%%ymm6 \n" + "vbroadcasti128 80(%5),%%ymm7 \n" + "vpor %%ymm6,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm7,%%ymm5,%%ymm7 \n" + "vpor %%ymm7,%%ymm1,%%ymm1 \n" - "lea 0x10(%0),%0 \n" - "lea 0x10(%1),%1 \n" - "lea 0x10(%2),%2 \n" - "lea 0x30(%3),%3 \n" - "sub $0x10,%4 \n" + "vbroadcasti128 96(%5),%%ymm6 \n" + "vbroadcasti128 112(%5),%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm2 \n" + "vpshufb %%ymm7,%%ymm4,%%ymm6 \n" + "vbroadcasti128 128(%5),%%ymm7 \n" + "vpor %%ymm6,%%ymm2,%%ymm2 \n" + "vpshufb %%ymm7,%%ymm5,%%ymm7 \n" + "vpor %%ymm7,%%ymm2,%%ymm2 \n" + + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm6 \n" + "vperm2i128 $0x30,%%ymm0,%%ymm2,%%ymm7 \n" + "vperm2i128 $0x31,%%ymm2,%%ymm1,%%ymm1 \n" + "vmovdqu %%ymm6,(%3) \n" + "vmovdqu %%ymm7,0x20(%3) \n" + "vmovdqu %%ymm1,0x40(%3) \n" + "lea 0x60(%3),%3 \n" + "subl $0x20,%4 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_r), // %0 "+r"(src_g), // %1 "+r"(src_b), // %2 "+r"(dst_rgb), // %3 - "+r"(width) // %4 + "+rm"(width) // %4 : "r"(&kMergeRGBShuffle[0]) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); } -#endif // HAS_MERGERGBROW_SSSE3 +#endif // HAS_MERGERGBROW_AVX2 #ifdef HAS_MERGEARGBROW_SSE2 void MergeARGBRow_SSE2(const uint8_t* src_r, @@ -8877,17 +9798,7 @@ void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBCOPYYTOALPHAROW_AVX512BW -#ifdef HAS_SETROW_X86 -void SetRow_X86(uint8_t* dst, uint8_t v8, int width) { - size_t width_tmp = (size_t)(width >> 2); - const uint32_t v32 = v8 * 0x01010101u; // Duplicate byte to all bytes. - asm volatile("rep stosl \n" - : "+D"(dst), // %0 - "+c"(width_tmp) // %1 - : "a"(v32) // %2 - : "memory", "cc"); -} - +#ifdef HAS_SETROW_ERMS void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width) { size_t width_tmp = (size_t)(width); asm volatile("rep stosb \n" @@ -8896,16 +9807,117 @@ void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width) { : "a"(v8) // %2 : "memory", "cc"); } +#endif // HAS_SETROW_ERMS -void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width) { - size_t width_tmp = (size_t)(width); - asm volatile("rep stosl \n" - : "+D"(dst_argb), // %0 - "+c"(width_tmp) // %1 - : "a"(v32) // %2 - : "memory", "cc"); +#ifdef HAS_SETROW_AVX2 +void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width) { + asm volatile( + "vmovd %2,%%xmm0 \n" + "vpbroadcastb %%xmm0,%%ymm0 \n" + "sub $0x40,%1 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu %%ymm0,(%0) \n" + "vmovdqu %%ymm0,0x20(%0) \n" + "lea 0x40(%0),%0 \n" + "sub $0x40,%1 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%1 \n" + "jle 99f \n" + "vmovdqu %%ymm0,(%0) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(dst), // %0 + "+r"(width) // %1 + : "r"((int)v8) // %2 + : "memory", "cc", "xmm0"); } -#endif // HAS_SETROW_X86 +#endif // HAS_SETROW_AVX2 + +#ifdef HAS_ARGBSETROW_AVX2 +void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width) { + asm volatile( + "vmovd %2,%%xmm0 \n" + "vpbroadcastd %%xmm0,%%ymm0 \n" + "sub $0x20,%1 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu %%ymm0,(%0) \n" + "vmovdqu %%ymm0,0x20(%0) \n" + "vmovdqu %%ymm0,0x40(%0) \n" + "vmovdqu %%ymm0,0x60(%0) \n" + "lea 0x80(%0),%0 \n" + "sub $0x20,%1 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%1 \n" + "jle 99f \n" + "cmp $0x10,%1 \n" + "jl 3f \n" + "vmovdqu %%ymm0,(%0) \n" + "vmovdqu %%ymm0,0x20(%0) \n" + "lea 0x40(%0),%0 \n" + "sub $0x10,%1 \n" + "jle 99f \n" + + "3: \n" + "vmovdqu %%ymm0,(%0) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(dst_argb), // %0 + "+r"(width) // %1 + : "r"(v32) // %2 + : "memory", "cc", "xmm0"); +} +#endif // HAS_ARGBSETROW_AVX2 + +#ifdef HAS_COPYROW_16_AVX2 +void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int width) { + asm volatile( + "sub $0x40,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vmovdqu 0x40(%0),%%ymm2 \n" + "vmovdqu 0x60(%0),%%ymm3 \n" + "lea 0x80(%0),%0 \n" + "vmovdqu %%ymm0,(%1) \n" + "vmovdqu %%ymm1,0x20(%1) \n" + "vmovdqu %%ymm2,0x40(%1) \n" + "vmovdqu %%ymm3,0x60(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vmovdqu %%ymm0,(%1) \n" + "vmovdqu %%ymm1,0x20(%1) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); +} +#endif // HAS_COPYROW_16_AVX2 #ifdef HAS_YUY2TOYROW_SSE2 void YUY2ToYRow_SSE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { @@ -9364,92 +10376,323 @@ void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy, } #endif // HAS_YUY2TOYROW_AVX2 -#ifdef HAS_ARGBBLENDROW_SSSE3 -// Shuffle table for isolating alpha. -static const uvec8 kShuffleAlpha = {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, - 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80}; - -// Blend 8 pixels at a time -void ARGBBlendRow_SSSE3(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { +#ifdef HAS_YUY2TOYROW_AVX512BW +void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + uintptr_t temp; asm volatile( - "pcmpeqb %%xmm7,%%xmm7 \n" - "psrlw $0xf,%%xmm7 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "psrlw $0x8,%%xmm6 \n" - "pcmpeqb %%xmm5,%%xmm5 \n" - "psllw $0x8,%%xmm5 \n" - "pcmpeqb %%xmm4,%%xmm4 \n" - "pslld $0x18,%%xmm4 \n" - "sub $0x4,%3 \n" - "jl 49f \n" + "sub $0x40,%2 \n" + "jl 2f \n" - // 4 pixel loop. LABELALIGN - "40: \n" - "movdqu (%0),%%xmm3 \n" - "lea 0x10(%0),%0 \n" - "movdqa %%xmm3,%%xmm0 \n" - "pxor %%xmm4,%%xmm3 \n" - "movdqu (%1),%%xmm2 \n" - "pshufb %4,%%xmm3 \n" - "pand %%xmm6,%%xmm2 \n" - "paddw %%xmm7,%%xmm3 \n" - "pmullw %%xmm3,%%xmm2 \n" - "movdqu (%1),%%xmm1 \n" + "1: \n" + "vmovdqu16 (%0),%%zmm0 \n" + "vmovdqu16 0x40(%0),%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vpmovwb %%zmm0,(%1) \n" + "vpmovwb %%zmm1,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "je 99f \n" + "cmp $0x20,%2 \n" + "jl 3f \n" + "vmovdqu16 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpmovwb %%zmm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x20,%2 \n" + "je 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmovwb %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_y), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_YUY2TOYROW_AVX512BW + +#ifdef HAS_YUY2TOUVROW_AVX512BW +void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + uintptr_t temp; + asm volatile( + "sub %1,%2 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%0),%%zmm0 \n" + "vpavgb 0x00(%0,%[stride],1),%%zmm0,%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x8,%%zmm0,%%zmm1 \n" + "vpsrld $0x18,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm1,(%1) \n" + "vpmovdb %%zmm0,0x00(%1,%2,1) \n" "lea 0x10(%1),%1 \n" - "psrlw $0x8,%%xmm1 \n" - "por %%xmm4,%%xmm0 \n" - "pmullw %%xmm3,%%xmm1 \n" - "psrlw $0x8,%%xmm2 \n" - "paddusb %%xmm2,%%xmm0 \n" - "pand %%xmm5,%%xmm1 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "je 99f \n" + "add $0x1,%k3 \n" + "shr $0x1,%k3 \n" + "mov $-1,%k[temp] \n" + "bzhi %k3,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x00(%0,%[stride],1),%%zmm1%{%%k1%}%{z%} \n" + "vpavgb %%zmm1,%%zmm0,%%zmm0 \n" + "vpsrld $0x8,%%zmm0,%%zmm1 \n" + "vpsrld $0x18,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm1,(%1)%{%%k1%} \n" + "vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : [stride] "r"((ptrdiff_t)(stride_yuy2)) // %[stride] + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_YUY2TOUVROW_AVX512BW + +#ifdef HAS_UYVYTOYROW_AVX512BW +void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + uintptr_t temp; + asm volatile( + "sub $0x40,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu16 (%0),%%zmm0 \n" + "vmovdqu16 0x40(%0),%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm1,%%zmm1 \n" + "vpmovwb %%zmm0,(%1) \n" + "vpmovwb %%zmm1,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "je 99f \n" + "cmp $0x20,%2 \n" + "jl 3f \n" + "vmovdqu16 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpmovwb %%zmm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x20,%2 \n" + "je 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpmovwb %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_y), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_UYVYTOYROW_AVX512BW + +#ifdef HAS_UYVYTOUVROW_AVX512BW +void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + uintptr_t temp; + asm volatile( + "sub %1,%2 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%0),%%zmm0 \n" + "vpavgb 0x00(%0,%[stride],1),%%zmm0,%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpmovdb %%zmm0,(%1) \n" + "vpsrld $0x10,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,0x00(%1,%2,1) \n" + "lea 0x10(%1),%1 \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "je 99f \n" + "add $0x1,%k3 \n" + "shr $0x1,%k3 \n" + "mov $-1,%k[temp] \n" + "bzhi %k3,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x00(%0,%[stride],1),%%zmm1%{%%k1%}%{z%} \n" + "vpavgb %%zmm1,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,(%1)%{%%k1%} \n" + "vpsrld $0x10,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : [stride] "r"((ptrdiff_t)(stride_uyvy)) // %[stride] + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_UYVYTOUVROW_AVX512BW + +#if defined(HAS_ARGBBLENDROW_AVX2) || defined(HAS_ARGBBLENDROW_AVX512BW) +// Shuffle table for isolating alpha. +SIMD_ALIGNED(static const uvec8 kShuffleAlpha[2]) = { + {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, 11u, 0x80, 11u, 0x80, 15u, 0x80, + 15u, 0x80}, + {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, 11u, 0x80, 11u, 0x80, 15u, 0x80, + 15u, 0x80}}; +#endif // defined(HAS_ARGBBLENDROW_AVX2) || defined(HAS_ARGBBLENDROW_AVX512BW) + +#ifdef HAS_ARGBBLENDROW_AVX2 +// Blend 8 pixels at a time, with 4-pixel and 1-pixel tails. +void ARGBBlendRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n" + "vpsrlw $0xf,%%ymm7,%%ymm7 \n" + "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrlw $0x8,%%ymm6,%%ymm6 \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsllw $0x8,%%ymm5,%%ymm5 \n" + "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n" + "vpslld $0x18,%%ymm4,%%ymm4 \n" + "sub $0x8,%3 \n" + "jl 2f \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm3 \n" + "vmovdqu (%1),%%ymm2 \n" + "lea 0x20(%0),%0 \n" + "lea 0x20(%1),%1 \n" + "vpxor %%ymm4,%%ymm3,%%ymm0 \n" + "vpshufb %4,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm7,%%ymm0,%%ymm0 \n" + "vpsrlw $0x8,%%ymm2,%%ymm1 \n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + "vpmullw %%ymm0,%%ymm2,%%ymm2 \n" + "vpmullw %%ymm0,%%ymm1,%%ymm1 \n" + "vpor %%ymm4,%%ymm3,%%ymm3 \n" + "vpsrlw $0x8,%%ymm2,%%ymm2 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpaddusb %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddusb %%ymm1,%%ymm3,%%ymm3 \n" + "vmovdqu %%ymm3,(%2) \n" + "lea 0x20(%2),%2 \n" + "sub $0x8,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x4,%3 \n" + "jl 3f \n" + "vmovdqu (%0),%%xmm3 \n" + "vmovdqu (%1),%%xmm2 \n" + "lea 0x10(%0),%0 \n" + "lea 0x10(%1),%1 \n" + "vpxor %%xmm4,%%xmm3,%%xmm0 \n" + "vpshufb %4,%%xmm0,%%xmm0 \n" + "vpaddw %%xmm7,%%xmm0,%%xmm0 \n" + "vpsrlw $0x8,%%xmm2,%%xmm1 \n" + "vpand %%xmm6,%%xmm2,%%xmm2 \n" + "vpmullw %%xmm0,%%xmm2,%%xmm2 \n" + "vpmullw %%xmm0,%%xmm1,%%xmm1 \n" + "vpor %%xmm4,%%xmm3,%%xmm3 \n" + "vpsrlw $0x8,%%xmm2,%%xmm2 \n" + "vpand %%xmm5,%%xmm1,%%xmm1 \n" + "vpaddusb %%xmm2,%%xmm3,%%xmm3 \n" + "vpaddusb %%xmm1,%%xmm3,%%xmm3 \n" + "vmovdqu %%xmm3,(%2) \n" "lea 0x10(%2),%2 \n" "sub $0x4,%3 \n" - "jge 40b \n" - "49: \n" + "3: \n" "add $0x3,%3 \n" "jl 99f \n" // 1 pixel loop. - "91: \n" - "movd (%0),%%xmm3 \n" + LABELALIGN + "4: \n" + "vmovd (%0),%%xmm3 \n" + "vmovd (%1),%%xmm2 \n" "lea 0x4(%0),%0 \n" - "movdqa %%xmm3,%%xmm0 \n" - "pxor %%xmm4,%%xmm3 \n" - "movd (%1),%%xmm2 \n" - "pshufb %4,%%xmm3 \n" - "pand %%xmm6,%%xmm2 \n" - "paddw %%xmm7,%%xmm3 \n" - "pmullw %%xmm3,%%xmm2 \n" - "movd (%1),%%xmm1 \n" "lea 0x4(%1),%1 \n" - "psrlw $0x8,%%xmm1 \n" - "por %%xmm4,%%xmm0 \n" - "pmullw %%xmm3,%%xmm1 \n" - "psrlw $0x8,%%xmm2 \n" - "paddusb %%xmm2,%%xmm0 \n" - "pand %%xmm5,%%xmm1 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movd %%xmm0,(%2) \n" + "vpxor %%xmm4,%%xmm3,%%xmm0 \n" + "vpshufb %4,%%xmm0,%%xmm0 \n" + "vpaddw %%xmm7,%%xmm0,%%xmm0 \n" + "vpsrlw $0x8,%%xmm2,%%xmm1 \n" + "vpand %%xmm6,%%xmm2,%%xmm2 \n" + "vpmullw %%xmm0,%%xmm2,%%xmm2 \n" + "vpmullw %%xmm0,%%xmm1,%%xmm1 \n" + "vpor %%xmm4,%%xmm3,%%xmm3 \n" + "vpsrlw $0x8,%%xmm2,%%xmm2 \n" + "vpand %%xmm5,%%xmm1,%%xmm1 \n" + "vpaddusb %%xmm2,%%xmm3,%%xmm3 \n" + "vpaddusb %%xmm1,%%xmm3,%%xmm3 \n" + "vmovd %%xmm3,(%2) \n" "lea 0x4(%2),%2 \n" "sub $0x1,%3 \n" - "jge 91b \n" + "jge 4b \n" + "99: \n" - : "+r"(src_argb), // %0 - "+r"(src_argb1), // %1 - "+r"(dst_argb), // %2 - "+r"(width) // %3 - : "m"(kShuffleAlpha) // %4 + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : "m"(kShuffleAlpha[0]) // %4 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } -#endif // HAS_ARGBBLENDROW_SSSE3 +#endif // HAS_ARGBBLENDROW_AVX2 #ifdef HAS_BLENDPLANEROW_SSSE3 // Blend 8 pixels at a time. @@ -9874,56 +11117,79 @@ void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBUNATTENUATEROW_AVX2 -#ifdef HAS_ARGBGRAYROW_SSSE3 -// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels -void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width) { +#if defined(HAS_ARGBGRAYROW_AVX2) || defined(HAS_ARGBGRAYROW_AVX512BW) || \ + defined(HAS_ARGBGRAYROW_AVX512VNNI) +static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, + 8, 8, 8, 0x80, 12, 12, 12, 0x80}; +#endif // defined(HAS_ARGBGRAYROW_AVX2) || defined(HAS_ARGBGRAYROW_AVX512BW) || + // defined(HAS_ARGBGRAYROW_AVX512VNNI) + +#ifdef HAS_ARGBGRAYROW_AVX2 +void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + static const uint32_t kCoeff = 0x004d961du; // {29, 150, 77, 0} + static const uint32_t kOne16 = 0x00010001u; + static const uint32_t kBias = 32896u; asm volatile( - "movdqa %3,%%xmm4 \n" - "movdqa %4,%%xmm5 \n" + "vpbroadcastd %[coeff],%%ymm2 \n" + "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n" + "vpsllw $7,%%ymm7,%%ymm3 \n" + "vpbroadcastb %%xmm3,%%ymm3 \n" // 0x80808080 + "vpslld $24,%%ymm7,%%ymm7 \n" // 0xff000000 + "vpbroadcastd %[one16],%%ymm4 \n" + "vpbroadcastd %[bias],%%ymm5 \n" + "vbroadcasti128 %[shuf],%%ymm6 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" - // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "psubb %%xmm5,%%xmm0 \n" - "psubb %%xmm5,%%xmm1 \n" - "movdqu %%xmm4,%%xmm6 \n" - "pmaddubsw %%xmm0,%%xmm6 \n" - "movdqu %%xmm4,%%xmm0 \n" - "pmaddubsw %%xmm1,%%xmm0 \n" - "phaddw %%xmm0,%%xmm6 \n" - "paddw %%xmm5,%%xmm6 \n" - "psrlw $0x8,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movdqu (%0),%%xmm2 \n" - "movdqu 0x10(%0),%%xmm3 \n" - "lea 0x20(%0),%0 \n" - "psrld $0x18,%%xmm2 \n" - "psrld $0x18,%%xmm3 \n" - "packuswb %%xmm3,%%xmm2 \n" - "packuswb %%xmm2,%%xmm2 \n" - "movdqa %%xmm6,%%xmm3 \n" - "punpcklbw %%xmm6,%%xmm6 \n" - "punpcklbw %%xmm2,%%xmm3 \n" - "movdqa %%xmm6,%%xmm1 \n" - "punpcklwd %%xmm3,%%xmm6 \n" - "punpckhwd %%xmm3,%%xmm1 \n" - "movdqu %%xmm6,(%1) \n" - "movdqu %%xmm1,0x10(%1) \n" - "lea 0x20(%1),%1 \n" - "sub $0x8,%2 \n" - "jg 1b \n" - : "+r"(src_argb), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "m"(kARGBToYJ), // %3 - "m"(kSub128) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); -} -#endif // HAS_ARGBGRAYROW_SSSE3 + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpxor %%ymm3,%%ymm0,%%ymm1 \n" + "vpmaddubsw %%ymm1,%%ymm2,%%ymm1 \n" + "vpmaddwd %%ymm4,%%ymm1,%%ymm1 \n" + "vpaddd %%ymm5,%%ymm1,%%ymm1 \n" + "vpsrld $8,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm6,%%ymm1,%%ymm1 \n" + "vpand %%ymm7,%%ymm0,%%ymm0 \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" -#ifdef HAS_ARGBSEPIAROW_SSSE3 + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovd (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vpxor %%xmm3,%%xmm0,%%xmm1 \n" + "vpmaddubsw %%xmm1,%%xmm2,%%xmm1 \n" + "vpmaddwd %%xmm4,%%xmm1,%%xmm1 \n" + "vpaddd %%xmm5,%%xmm1,%%xmm1 \n" + "vpsrld $8,%%xmm1,%%xmm1 \n" + "vpshufb %%xmm6,%%xmm1,%%xmm1 \n" + "vpand %%xmm7,%%xmm0,%%xmm0 \n" + "vpor %%xmm1,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width) + : [coeff] "m"(kCoeff), [one16] "m"(kOne16), [bias] "m"(kBias), + [shuf] "m"(kShufGrayDword) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_ARGBGRAYROW_AVX2 + +#if defined(HAS_ARGBSEPIAROW_AVX2) || defined(HAS_ARGBSEPIAROW_AVX512VNNI) // b = (r * 35 + g * 68 + b * 17) >> 7 // g = (r * 45 + g * 88 + b * 22) >> 7 // r = (r * 50 + g * 98 + b * 24) >> 7 @@ -9936,215 +11202,458 @@ static const vec8 kARGBToSepiaG = {22, 88, 45, 0, 22, 88, 45, 0, static const vec8 kARGBToSepiaR = {24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0}; +#endif -// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels. -void ARGBSepiaRow_SSSE3(uint8_t* dst_argb, int width) { +#ifdef HAS_ARGBSEPIAROW_AVX2 +// Convert ARGB pixels to Sepia ARGB pixels. +void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width) { asm volatile( - "movdqa %2,%%xmm2 \n" - "movdqa %3,%%xmm3 \n" - "movdqa %4,%%xmm4 \n" + "vbroadcasti128 %2,%%ymm2 \n" + "vbroadcasti128 %3,%%ymm3 \n" + "vbroadcasti128 %4,%%ymm4 \n" + "sub $0x8,%1 \n" + "jl 2f \n" // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm6 \n" - "pmaddubsw %%xmm2,%%xmm0 \n" - "pmaddubsw %%xmm2,%%xmm6 \n" - "phaddw %%xmm6,%%xmm0 \n" - "psrlw $0x7,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movdqu (%0),%%xmm5 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "pmaddubsw %%xmm3,%%xmm5 \n" - "pmaddubsw %%xmm3,%%xmm1 \n" - "phaddw %%xmm1,%%xmm5 \n" - "psrlw $0x7,%%xmm5 \n" - "packuswb %%xmm5,%%xmm5 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "movdqu (%0),%%xmm5 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "pmaddubsw %%xmm4,%%xmm5 \n" - "pmaddubsw %%xmm4,%%xmm1 \n" - "phaddw %%xmm1,%%xmm5 \n" - "psrlw $0x7,%%xmm5 \n" - "packuswb %%xmm5,%%xmm5 \n" - "movdqu (%0),%%xmm6 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "psrld $0x18,%%xmm6 \n" - "psrld $0x18,%%xmm1 \n" - "packuswb %%xmm1,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "punpcklbw %%xmm6,%%xmm5 \n" - "movdqa %%xmm0,%%xmm1 \n" - "punpcklwd %%xmm5,%%xmm0 \n" - "punpckhwd %%xmm5,%%xmm1 \n" - "movdqu %%xmm0,(%0) \n" - "movdqu %%xmm1,0x10(%0) \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpmaddubsw %%ymm2,%%ymm0,%%ymm1 \n" + "vpmaddubsw %%ymm3,%%ymm0,%%ymm5 \n" + "vpmaddubsw %%ymm4,%%ymm0,%%ymm6 \n" + "vphaddw %%ymm5,%%ymm1,%%ymm1 \n" + "vphaddw %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrlw $0x7,%%ymm1,%%ymm1 \n" + "vpsrlw $0x7,%%ymm6,%%ymm6 \n" + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm6,%%ymm6,%%ymm6 \n" + "vpshufd $0x01,%%ymm1,%%ymm5 \n" + "vpunpcklbw %%ymm5,%%ymm1,%%ymm1 \n" + "vpsrld $0x18,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm0,%%ymm6,%%ymm6 \n" + "vpunpcklwd %%ymm6,%%ymm1,%%ymm0 \n" + "vmovdqu %%ymm0,(%0) \n" "lea 0x20(%0),%0 \n" "sub $0x8,%1 \n" - "jg 1b \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%1 \n" + "jle 99f \n" + + // 1 pixel remainder loop. + LABELALIGN + "3: \n" + "vmovd (%0),%%xmm0 \n" + "vpmaddubsw %%xmm2,%%xmm0,%%xmm1 \n" + "vpmaddubsw %%xmm3,%%xmm0,%%xmm5 \n" + "vpmaddubsw %%xmm4,%%xmm0,%%xmm6 \n" + "vphaddw %%xmm5,%%xmm1,%%xmm1 \n" + "vphaddw %%xmm6,%%xmm6,%%xmm6 \n" + "vpsrlw $0x7,%%xmm1,%%xmm1 \n" + "vpsrlw $0x7,%%xmm6,%%xmm6 \n" + "vpackuswb %%xmm1,%%xmm1,%%xmm1 \n" + "vpackuswb %%xmm6,%%xmm6,%%xmm6 \n" + "vpshufd $0x01,%%xmm1,%%xmm5 \n" + "vpunpcklbw %%xmm5,%%xmm1,%%xmm1 \n" + "vpsrld $0x18,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vpunpcklbw %%xmm0,%%xmm6,%%xmm6 \n" + "vpunpcklwd %%xmm6,%%xmm1,%%xmm0 \n" + "vmovd %%xmm0,(%0) \n" + "lea 0x04(%0),%0 \n" + "sub $0x1,%1 \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" : "+r"(dst_argb), // %0 "+r"(width) // %1 : "m"(kARGBToSepiaB), // %2 "m"(kARGBToSepiaG), // %3 "m"(kARGBToSepiaR) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6"); } -#endif // HAS_ARGBSEPIAROW_SSSE3 +#endif // HAS_ARGBSEPIAROW_AVX2 -#ifdef HAS_ARGBCOLORMATRIXROW_SSSE3 -// Tranform 8 ARGB pixels (32 bytes) with color matrix. -// Same as Sepia except matrix is provided. -void ARGBColorMatrixRow_SSSE3(const uint8_t* src_argb, - uint8_t* dst_argb, - const int8_t* matrix_argb, - int width) { +#ifdef HAS_ARGBSEPIAROW_AVX512VNNI +void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) { + uintptr_t temp; asm volatile( - "movdqu (%3),%%xmm5 \n" - "pshufd $0x00,%%xmm5,%%xmm2 \n" - "pshufd $0x55,%%xmm5,%%xmm3 \n" - "pshufd $0xaa,%%xmm5,%%xmm4 \n" - "pshufd $0xff,%%xmm5,%%xmm5 \n" + "vpbroadcastd %[kSepiaB],%%zmm2 \n" + "vpbroadcastd %[kSepiaG],%%zmm3 \n" + "vpbroadcastd %[kSepiaR],%%zmm4 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrld $24,%%zmm5,%%zmm6 \n" // 0x000000ff + "vpslld $24,%%zmm5,%%zmm5 \n" // 0xff000000 + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[dst]),%%zmm0 \n" + "vpxord %%zmm1,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm7,%%zmm7 \n" + "vpxord %%zmm8,%%zmm8,%%zmm8 \n" + "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n" + "vpsrld $7,%%zmm1,%%zmm1 \n" + "vpsrld $7,%%zmm7,%%zmm7 \n" + "vpsrld $7,%%zmm8,%%zmm8 \n" + "vpminud %%zmm6,%%zmm7,%%zmm7 \n" + "vpminud %%zmm6,%%zmm8,%%zmm8 \n" + "vpslld $8,%%zmm7,%%zmm7 \n" + "vpslld $16,%%zmm8,%%zmm8 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpord %%zmm7,%%zmm1,%%zmm1 \n" + "vpord %%zmm8,%%zmm0,%%zmm0 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[dst]),%%zmm0%{%%k1%}%{z%} \n" + "vpxord %%zmm1,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm7,%%zmm7 \n" + "vpxord %%zmm8,%%zmm8,%%zmm8 \n" + "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n" + "vpsrld $7,%%zmm1,%%zmm1 \n" + "vpsrld $7,%%zmm7,%%zmm7 \n" + "vpsrld $7,%%zmm8,%%zmm8 \n" + "vpminud %%zmm6,%%zmm7,%%zmm7 \n" + "vpminud %%zmm6,%%zmm8,%%zmm8 \n" + "vpslld $8,%%zmm7,%%zmm7 \n" + "vpslld $16,%%zmm8,%%zmm8 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpord %%zmm7,%%zmm1,%%zmm1 \n" + "vpord %%zmm8,%%zmm0,%%zmm0 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [width] "+r"(width), [temp] "=&r"(temp) + : [kSepiaB] "m"(kARGBToSepiaB), [kSepiaG] "m"(kARGBToSepiaG), + [kSepiaR] "m"(kARGBToSepiaR) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8", "k1"); +} +#endif // HAS_ARGBSEPIAROW_AVX512VNNI + +#ifdef HAS_ARGBCOLORMATRIXROW_AVX2 +static const vec8 kShufTranspose4x4 = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; +// Transform ARGB pixels with color matrix. +void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + asm volatile( + "vbroadcasti128 (%3),%%ymm5 \n" + "vbroadcasti128 %4,%%ymm7 \n" + "vpshufd $0x00,%%ymm5,%%ymm2 \n" + "vpshufd $0x55,%%ymm5,%%ymm3 \n" + "vpshufd $0xaa,%%ymm5,%%ymm4 \n" + "vpshufd $0xff,%%ymm5,%%ymm5 \n" + "sub $0x8,%2 \n" + "jl 2f \n" // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm7 \n" - "pmaddubsw %%xmm2,%%xmm0 \n" - "pmaddubsw %%xmm2,%%xmm7 \n" - "movdqu (%0),%%xmm6 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "pmaddubsw %%xmm3,%%xmm6 \n" - "pmaddubsw %%xmm3,%%xmm1 \n" - "phaddsw %%xmm7,%%xmm0 \n" - "phaddsw %%xmm1,%%xmm6 \n" - "psraw $0x6,%%xmm0 \n" - "psraw $0x6,%%xmm6 \n" - "packuswb %%xmm0,%%xmm0 \n" - "packuswb %%xmm6,%%xmm6 \n" - "punpcklbw %%xmm6,%%xmm0 \n" - "movdqu (%0),%%xmm1 \n" - "movdqu 0x10(%0),%%xmm7 \n" - "pmaddubsw %%xmm4,%%xmm1 \n" - "pmaddubsw %%xmm4,%%xmm7 \n" - "phaddsw %%xmm7,%%xmm1 \n" - "movdqu (%0),%%xmm6 \n" - "movdqu 0x10(%0),%%xmm7 \n" - "pmaddubsw %%xmm5,%%xmm6 \n" - "pmaddubsw %%xmm5,%%xmm7 \n" - "phaddsw %%xmm7,%%xmm6 \n" - "psraw $0x6,%%xmm1 \n" - "psraw $0x6,%%xmm6 \n" - "packuswb %%xmm1,%%xmm1 \n" - "packuswb %%xmm6,%%xmm6 \n" - "punpcklbw %%xmm6,%%xmm1 \n" - "movdqa %%xmm0,%%xmm6 \n" - "punpcklwd %%xmm1,%%xmm0 \n" - "punpckhwd %%xmm1,%%xmm6 \n" - "movdqu %%xmm0,(%1) \n" - "movdqu %%xmm6,0x10(%1) \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" "lea 0x20(%0),%0 \n" + "vpmaddubsw %%ymm2,%%ymm0,%%ymm1 \n" + "vpmaddubsw %%ymm3,%%ymm0,%%ymm6 \n" + "vphaddsw %%ymm6,%%ymm1,%%ymm1 \n" + "vpmaddubsw %%ymm4,%%ymm0,%%ymm6 \n" + "vpmaddubsw %%ymm5,%%ymm0,%%ymm0 \n" + "vphaddsw %%ymm0,%%ymm6,%%ymm6 \n" + "vpsraw $0x6,%%ymm1,%%ymm1 \n" + "vpsraw $0x6,%%ymm6,%%ymm6 \n" + "vpackuswb %%ymm6,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm7,%%ymm1,%%ymm1 \n" + "vmovdqu %%ymm1,(%1) \n" "lea 0x20(%1),%1 \n" "sub $0x8,%2 \n" - "jg 1b \n" - : "+r"(src_argb), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "r"(matrix_argb) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); -} -#endif // HAS_ARGBCOLORMATRIXROW_SSSE3 + "jge 1b \n" -#ifdef HAS_ARGBQUANTIZEROW_SSE2 -// Quantize 4 ARGB pixels (16 bytes). -void ARGBQuantizeRow_SSE2(uint8_t* dst_argb, + "2: \n" + "add $0x8,%2 \n" + "jle 99f \n" + + // 1 pixel remainder loop. + LABELALIGN + "3: \n" + "vmovd (%0),%%xmm0 \n" + "lea 0x04(%0),%0 \n" + "vpmaddubsw %%xmm2,%%xmm0,%%xmm1 \n" + "vpmaddubsw %%xmm3,%%xmm0,%%xmm6 \n" + "vphaddsw %%xmm6,%%xmm1,%%xmm1 \n" + "vpmaddubsw %%xmm4,%%xmm0,%%xmm6 \n" + "vpmaddubsw %%xmm5,%%xmm0,%%xmm0 \n" + "vphaddsw %%xmm0,%%xmm6,%%xmm6 \n" + "vpsraw $0x6,%%xmm1,%%xmm1 \n" + "vpsraw $0x6,%%xmm6,%%xmm6 \n" + "vpackuswb %%xmm6,%%xmm1,%%xmm1 \n" + "vpshufb %%xmm7,%%xmm1,%%xmm1 \n" + "vmovd %%xmm1,(%1) \n" + "lea 0x04(%1),%1 \n" + "sub $0x1,%2 \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(matrix_argb), // %3 + "m"(kShufTranspose4x4) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#endif // HAS_ARGBCOLORMATRIXROW_AVX2 + +#ifdef HAS_ARGBCOLORMATRIXROW_AVX512VNNI +void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + uintptr_t temp; + asm volatile( + "vpbroadcastd (%[mat]),%%zmm2 \n" + "vpbroadcastd 0x04(%[mat]),%%zmm3 \n" + "vpbroadcastd 0x08(%[mat]),%%zmm4 \n" + "vpbroadcastd 0x0c(%[mat]),%%zmm5 \n" + "vpternlogd $0xff,%%zmm6,%%zmm6,%%zmm6 \n" + "vpsrld $24,%%zmm6,%%zmm6 \n" // 255 + "vpxord %%zmm9,%%zmm9,%%zmm9 \n" // 0 + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpxord %%zmm1,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm7,%%zmm7 \n" + "vpxord %%zmm8,%%zmm8,%%zmm8 \n" + "vpxord %%zmm10,%%zmm10,%%zmm10 \n" + "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n" + "vpdpbusd %%zmm5,%%zmm0,%%zmm10 \n" + "vpsrad $6,%%zmm1,%%zmm1 \n" + "vpsrad $6,%%zmm7,%%zmm7 \n" + "vpsrad $6,%%zmm8,%%zmm8 \n" + "vpsrad $6,%%zmm10,%%zmm10 \n" + "vpmaxsd %%zmm9,%%zmm1,%%zmm1 \n" + "vpmaxsd %%zmm9,%%zmm7,%%zmm7 \n" + "vpmaxsd %%zmm9,%%zmm8,%%zmm8 \n" + "vpmaxsd %%zmm9,%%zmm10,%%zmm10 \n" + "vpminud %%zmm6,%%zmm1,%%zmm1 \n" + "vpminud %%zmm6,%%zmm7,%%zmm7 \n" + "vpminud %%zmm6,%%zmm8,%%zmm8 \n" + "vpminud %%zmm6,%%zmm10,%%zmm10 \n" + "vpslld $8,%%zmm7,%%zmm7 \n" + "vpslld $16,%%zmm8,%%zmm8 \n" + "vpslld $24,%%zmm10,%%zmm10 \n" + "vpord %%zmm7,%%zmm1,%%zmm1 \n" + "vpord %%zmm10,%%zmm8,%%zmm8 \n" + "vpord %%zmm8,%%zmm1,%%zmm1 \n" + "vmovdqu32 %%zmm1,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpxord %%zmm1,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm7,%%zmm7 \n" + "vpxord %%zmm8,%%zmm8,%%zmm8 \n" + "vpxord %%zmm10,%%zmm10,%%zmm10 \n" + "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n" + "vpdpbusd %%zmm5,%%zmm0,%%zmm10 \n" + "vpsrad $6,%%zmm1,%%zmm1 \n" + "vpsrad $6,%%zmm7,%%zmm7 \n" + "vpsrad $6,%%zmm8,%%zmm8 \n" + "vpsrad $6,%%zmm10,%%zmm10 \n" + "vpmaxsd %%zmm9,%%zmm1,%%zmm1 \n" + "vpmaxsd %%zmm9,%%zmm7,%%zmm7 \n" + "vpmaxsd %%zmm9,%%zmm8,%%zmm8 \n" + "vpmaxsd %%zmm9,%%zmm10,%%zmm10 \n" + "vpminud %%zmm6,%%zmm1,%%zmm1 \n" + "vpminud %%zmm6,%%zmm7,%%zmm7 \n" + "vpminud %%zmm6,%%zmm8,%%zmm8 \n" + "vpminud %%zmm6,%%zmm10,%%zmm10 \n" + "vpslld $8,%%zmm7,%%zmm7 \n" + "vpslld $16,%%zmm8,%%zmm8 \n" + "vpslld $24,%%zmm10,%%zmm10 \n" + "vpord %%zmm7,%%zmm1,%%zmm1 \n" + "vpord %%zmm10,%%zmm8,%%zmm8 \n" + "vpord %%zmm8,%%zmm1,%%zmm1 \n" + "vmovdqu32 %%zmm1,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width), + [temp] "=&r"(temp) + : [mat] "r"(matrix_argb) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8", "xmm9", "xmm10", "k1"); +} +#endif // HAS_ARGBCOLORMATRIXROW_AVX512VNNI + +#ifdef HAS_ARGBQUANTIZEROW_AVX2 +// Quantize ARGB pixels. +void ARGBQuantizeRow_AVX2(uint8_t* dst_argb, int scale, int interval_size, int interval_offset, int width) { asm volatile( - "movd %2,%%xmm2 \n" - "movd %3,%%xmm3 \n" - "movd %4,%%xmm4 \n" - "pshuflw $0x40,%%xmm2,%%xmm2 \n" - "pshufd $0x44,%%xmm2,%%xmm2 \n" - "pshuflw $0x40,%%xmm3,%%xmm3 \n" - "pshufd $0x44,%%xmm3,%%xmm3 \n" - "pshuflw $0x40,%%xmm4,%%xmm4 \n" - "pshufd $0x44,%%xmm4,%%xmm4 \n" - "pxor %%xmm5,%%xmm5 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "pslld $0x18,%%xmm6 \n" + "vpbroadcastw %2,%%ymm2 \n" + "vpbroadcastw %3,%%ymm3 \n" + "vpbroadcastw %4,%%ymm4 \n" + "vpxor %%ymm5,%%ymm5,%%ymm5 \n" + "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n" + "vpslld $0x18,%%ymm6,%%ymm6 \n" + "sub $0x8,%1 \n" + "jl 2f \n" - // 4 pixel loop. + // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "pmulhuw %%xmm2,%%xmm0 \n" - "movdqu (%0),%%xmm1 \n" - "punpckhbw %%xmm5,%%xmm1 \n" - "pmulhuw %%xmm2,%%xmm1 \n" - "pmullw %%xmm3,%%xmm0 \n" - "movdqu (%0),%%xmm7 \n" - "pmullw %%xmm3,%%xmm1 \n" - "pand %%xmm6,%%xmm7 \n" - "paddw %%xmm4,%%xmm0 \n" - "paddw %%xmm4,%%xmm1 \n" - "packuswb %%xmm1,%%xmm0 \n" - "por %%xmm7,%%xmm0 \n" - "movdqu %%xmm0,(%0) \n" - "lea 0x10(%0),%0 \n" - "sub $0x4,%1 \n" - "jg 1b \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpunpckhbw %%ymm5,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm0,%%ymm7 \n" + "vpunpcklbw %%ymm5,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n" + "vpmullw %%ymm3,%%ymm0,%%ymm0 \n" + "vpmullw %%ymm3,%%ymm1,%%ymm1 \n" + "vpaddw %%ymm4,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm4,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpandn %%ymm0,%%ymm6,%%ymm0 \n" + "vpor %%ymm7,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%0) \n" + "lea 0x20(%0),%0 \n" + "sub $0x8,%1 \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%1 \n" + "jle 99f \n" + + // 1 pixel remainder loop. + LABELALIGN + "3: \n" + "vmovd (%0),%%xmm0 \n" + "vpand %%xmm6,%%xmm0,%%xmm7 \n" + "vpunpcklbw %%xmm5,%%xmm0,%%xmm0 \n" + "vpmulhuw %%xmm2,%%xmm0,%%xmm0 \n" + "vpmullw %%xmm3,%%xmm0,%%xmm0 \n" + "vpaddw %%xmm4,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vpandn %%xmm0,%%xmm6,%%xmm0 \n" + "vpor %%xmm7,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%0) \n" + "lea 0x04(%0),%0 \n" + "sub $0x1,%1 \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" : "+r"(dst_argb), // %0 "+r"(width) // %1 - : "r"(scale), // %2 - "r"(interval_size), // %3 - "r"(interval_offset) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + : "m"(scale), // %2 + "m"(interval_size), // %3 + "m"(interval_offset) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); } -#endif // HAS_ARGBQUANTIZEROW_SSE2 +#endif // HAS_ARGBQUANTIZEROW_AVX2 -#ifdef HAS_ARGBSHADEROW_SSE2 -// Shade 4 pixels at a time by specified value. -void ARGBShadeRow_SSE2(const uint8_t* src_argb, +#ifdef HAS_ARGBSHADEROW_AVX2 +// Shade ARGB pixels by specified value. +void ARGBShadeRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width, uint32_t value) { asm volatile( - "movd %3,%%xmm2 \n" - "punpcklbw %%xmm2,%%xmm2 \n" - "punpcklqdq %%xmm2,%%xmm2 \n" + "vpbroadcastd %3,%%ymm2 \n" + "vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n" + "sub $0x8,%2 \n" + "jl 2f \n" - // 4 pixel loop. + // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "lea 0x10(%0),%0 \n" - "movdqa %%xmm0,%%xmm1 \n" - "punpcklbw %%xmm0,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm1 \n" - "pmulhuw %%xmm2,%%xmm0 \n" - "pmulhuw %%xmm2,%%xmm1 \n" - "psrlw $0x8,%%xmm0 \n" - "psrlw $0x8,%%xmm1 \n" - "packuswb %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x4,%2 \n" - "jg 1b \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "lea 0x20(%0),%0 \n" + "vpunpckhbw %%ymm0,%%ymm0,%%ymm1 \n" + "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x8,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%2 \n" + "jle 99f \n" + + // 1 pixel remainder loop. + LABELALIGN + "3: \n" + "vmovd (%0),%%xmm0 \n" + "lea 0x04(%0),%0 \n" + "vpunpcklbw %%xmm0,%%xmm0,%%xmm0 \n" + "vpmulhuw %%xmm2,%%xmm0,%%xmm0 \n" + "vpsrlw $0x8,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%1) \n" + "lea 0x04(%1),%1 \n" + "sub $0x1,%2 \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" : "+r"(src_argb), // %0 "+r"(dst_argb), // %1 "+r"(width) // %2 - : "r"(value) // %3 + : "m"(value) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2"); } -#endif // HAS_ARGBSHADEROW_SSE2 +#endif // HAS_ARGBSHADEROW_AVX2 #ifdef HAS_ARGBMULTIPLYROW_SSE2 // Multiply 2 rows of ARGB pixels together, 4 pixels at a time. @@ -10341,12 +11850,12 @@ void ARGBSubtractRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBSUBTRACTROW_AVX2 -#ifdef HAS_SOBELXROW_SSE2 +#ifdef HAS_SOBELXROW_AVX2 // SobelX as a matrix is // -1 0 1 // -2 0 2 // -1 0 1 -void SobelXRow_SSE2(const uint8_t* src_y0, +void SobelXRow_AVX2(const uint8_t* src_y0, const uint8_t* src_y1, const uint8_t* src_y2, uint8_t* dst_sobelx, @@ -10355,440 +11864,201 @@ void SobelXRow_SSE2(const uint8_t* src_y0, "sub %0,%1 \n" "sub %0,%2 \n" "sub %0,%3 \n" - "pxor %%xmm5,%%xmm5 \n" - // 8 pixel loop. + // 16 pixel loop. LABELALIGN - "1: \n" - "movq (%0),%%xmm0 \n" - "movq 0x2(%0),%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "psubw %%xmm1,%%xmm0 \n" - "movq 0x00(%0,%1,1),%%xmm1 \n" - "movq 0x02(%0,%1,1),%%xmm2 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm2 \n" - "psubw %%xmm2,%%xmm1 \n" - "movq 0x00(%0,%2,1),%%xmm2 \n" - "movq 0x02(%0,%2,1),%%xmm3 \n" - "punpcklbw %%xmm5,%%xmm2 \n" - "punpcklbw %%xmm5,%%xmm3 \n" - "psubw %%xmm3,%%xmm2 \n" - "paddw %%xmm2,%%xmm0 \n" - "paddw %%xmm1,%%xmm0 \n" - "paddw %%xmm1,%%xmm0 \n" - "pxor %%xmm1,%%xmm1 \n" - "psubw %%xmm0,%%xmm1 \n" - "pmaxsw %%xmm1,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movq %%xmm0,0x00(%0,%3,1) \n" - "lea 0x8(%0),%0 \n" - "sub $0x8,%4 \n" + "1: \n" + "vpmovzxbw (%0),%%ymm0 \n" + "vpmovzxbw 0x02(%0),%%ymm1 \n" + "vpsubw %%ymm1,%%ymm0,%%ymm0 \n" + "vpmovzxbw 0x00(%0,%1,1),%%ymm1 \n" + "vpmovzxbw 0x02(%0,%1,1),%%ymm2 \n" + "vpsubw %%ymm2,%%ymm1,%%ymm1 \n" + "vpmovzxbw 0x00(%0,%2,1),%%ymm2 \n" + "vpmovzxbw 0x02(%0,%2,1),%%ymm3 \n" + "vpsubw %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddw %%ymm2,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm1,%%ymm0,%%ymm0 \n" + "vpabsw %%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,0x00(%0,%3,1) \n" + "lea 0x10(%0),%0 \n" + "sub $0x10,%4 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_y0), // %0 "+r"(src_y1), // %1 "+r"(src_y2), // %2 "+r"(dst_sobelx), // %3 "+r"(width) // %4 : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } -#endif // HAS_SOBELXROW_SSE2 +#endif // HAS_SOBELXROW_AVX2 -#ifdef HAS_SOBELYROW_SSE2 +#ifdef HAS_SOBELYROW_AVX2 // SobelY as a matrix is // -1 -2 -1 // 0 0 0 // 1 2 1 -void SobelYRow_SSE2(const uint8_t* src_y0, +void SobelYRow_AVX2(const uint8_t* src_y0, const uint8_t* src_y1, uint8_t* dst_sobely, int width) { asm volatile( "sub %0,%1 \n" "sub %0,%2 \n" - "pxor %%xmm5,%%xmm5 \n" - // 8 pixel loop. + // 16 pixel loop. LABELALIGN - "1: \n" - "movq (%0),%%xmm0 \n" - "movq 0x00(%0,%1,1),%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "psubw %%xmm1,%%xmm0 \n" - "movq 0x1(%0),%%xmm1 \n" - "movq 0x01(%0,%1,1),%%xmm2 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm2 \n" - "psubw %%xmm2,%%xmm1 \n" - "movq 0x2(%0),%%xmm2 \n" - "movq 0x02(%0,%1,1),%%xmm3 \n" - "punpcklbw %%xmm5,%%xmm2 \n" - "punpcklbw %%xmm5,%%xmm3 \n" - "psubw %%xmm3,%%xmm2 \n" - "paddw %%xmm2,%%xmm0 \n" - "paddw %%xmm1,%%xmm0 \n" - "paddw %%xmm1,%%xmm0 \n" - "pxor %%xmm1,%%xmm1 \n" - "psubw %%xmm0,%%xmm1 \n" - "pmaxsw %%xmm1,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movq %%xmm0,0x00(%0,%2,1) \n" - "lea 0x8(%0),%0 \n" - "sub $0x8,%3 \n" + "1: \n" + "vpmovzxbw (%0),%%ymm0 \n" + "vpmovzxbw 0x00(%0,%1,1),%%ymm1 \n" + "vpsubw %%ymm1,%%ymm0,%%ymm0 \n" + "vpmovzxbw 0x01(%0),%%ymm1 \n" + "vpmovzxbw 0x01(%0,%1,1),%%ymm2 \n" + "vpsubw %%ymm2,%%ymm1,%%ymm1 \n" + "vpmovzxbw 0x02(%0),%%ymm2 \n" + "vpmovzxbw 0x02(%0,%1,1),%%ymm3 \n" + "vpsubw %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddw %%ymm2,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm1,%%ymm0,%%ymm0 \n" + "vpabsw %%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,0x00(%0,%2,1) \n" + "lea 0x10(%0),%0 \n" + "sub $0x10,%3 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_y0), // %0 "+r"(src_y1), // %1 "+r"(dst_sobely), // %2 "+r"(width) // %3 : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } -#endif // HAS_SOBELYROW_SSE2 +#endif // HAS_SOBELYROW_AVX2 -#ifdef HAS_SOBELROW_SSE2 +#ifdef HAS_SOBELROW_AVX2 // Adds Sobel X and Sobel Y and stores Sobel into ARGB. // A = 255 // R = Sobel // G = Sobel // B = Sobel -void SobelRow_SSE2(const uint8_t* src_sobelx, +void SobelRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width) { asm volatile( "sub %0,%1 \n" - "pcmpeqb %%xmm5,%%xmm5 \n" - "pslld $0x18,%%xmm5 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpslld $0x18,%%ymm5,%%ymm5 \n" - // 8 pixel loop. + // 16 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x00(%0,%1,1),%%xmm1 \n" + "1: \n" + "vmovdqu (%0),%%xmm0 \n" + "vpaddusb 0x00(%0,%1,1),%%xmm0,%%xmm0 \n" "lea 0x10(%0),%0 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movdqa %%xmm0,%%xmm2 \n" - "punpcklbw %%xmm0,%%xmm2 \n" - "punpckhbw %%xmm0,%%xmm0 \n" - "movdqa %%xmm2,%%xmm1 \n" - "punpcklwd %%xmm2,%%xmm1 \n" - "punpckhwd %%xmm2,%%xmm2 \n" - "por %%xmm5,%%xmm1 \n" - "por %%xmm5,%%xmm2 \n" - "movdqa %%xmm0,%%xmm3 \n" - "punpcklwd %%xmm0,%%xmm3 \n" - "punpckhwd %%xmm0,%%xmm0 \n" - "por %%xmm5,%%xmm3 \n" - "por %%xmm5,%%xmm0 \n" - "movdqu %%xmm1,(%2) \n" - "movdqu %%xmm2,0x10(%2) \n" - "movdqu %%xmm3,0x20(%2) \n" - "movdqu %%xmm0,0x30(%2) \n" + "vpshufd $0xd8,%%xmm0,%%xmm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n" + "vpunpcklwd %%ymm0,%%ymm0,%%ymm1 \n" + "vpunpckhwd %%ymm0,%%ymm0,%%ymm0 \n" + "vpor %%ymm5,%%ymm1,%%ymm1 \n" + "vpor %%ymm5,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm1,(%2) \n" + "vmovdqu %%ymm0,0x20(%2) \n" "lea 0x40(%2),%2 \n" "sub $0x10,%3 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_sobelx), // %0 "+r"(src_sobely), // %1 "+r"(dst_argb), // %2 "+r"(width) // %3 : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm5"); } -#endif // HAS_SOBELROW_SSE2 +#endif // HAS_SOBELROW_AVX2 -#ifdef HAS_SOBELTOPLANEROW_SSE2 +#ifdef HAS_SOBELTOPLANEROW_AVX2 // Adds Sobel X and Sobel Y and stores Sobel into a plane. -void SobelToPlaneRow_SSE2(const uint8_t* src_sobelx, +void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, int width) { asm volatile("sub %0,%1 \n" - // 8 pixel loop. + // 32 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x00(%0,%1,1),%%xmm1 \n" - "lea 0x10(%0),%0 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x10,%3 \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpaddusb 0x00(%0,%1,1),%%ymm0,%%ymm0 \n" + "lea 0x20(%0),%0 \n" + "vmovdqu %%ymm0,(%2) \n" + "lea 0x20(%2),%2 \n" + "sub $0x20,%3 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_sobelx), // %0 "+r"(src_sobely), // %1 "+r"(dst_y), // %2 "+r"(width) // %3 : - : "memory", "cc", "xmm0", "xmm1"); + : "memory", "cc", "xmm0"); } -#endif // HAS_SOBELTOPLANEROW_SSE2 +#endif // HAS_SOBELTOPLANEROW_AVX2 -#ifdef HAS_SOBELXYROW_SSE2 +#ifdef HAS_SOBELXYROW_AVX2 // Mixes Sobel X, Sobel Y and Sobel into ARGB. // A = 255 // R = Sobel X // G = Sobel // B = Sobel Y -void SobelXYRow_SSE2(const uint8_t* src_sobelx, +void SobelXYRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width) { asm volatile( "sub %0,%1 \n" - "pcmpeqb %%xmm5,%%xmm5 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" - // 8 pixel loop. + // 16 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x00(%0,%1,1),%%xmm1 \n" + "1: \n" + "vmovdqu (%0),%%xmm0 \n" + "vmovdqu 0x00(%0,%1,1),%%xmm1 \n" "lea 0x10(%0),%0 \n" - "movdqa %%xmm0,%%xmm2 \n" - "paddusb %%xmm1,%%xmm2 \n" - "movdqa %%xmm0,%%xmm3 \n" - "punpcklbw %%xmm5,%%xmm3 \n" - "punpckhbw %%xmm5,%%xmm0 \n" - "movdqa %%xmm1,%%xmm4 \n" - "punpcklbw %%xmm2,%%xmm4 \n" - "punpckhbw %%xmm2,%%xmm1 \n" - "movdqa %%xmm4,%%xmm6 \n" - "punpcklwd %%xmm3,%%xmm6 \n" - "punpckhwd %%xmm3,%%xmm4 \n" - "movdqa %%xmm1,%%xmm7 \n" - "punpcklwd %%xmm0,%%xmm7 \n" - "punpckhwd %%xmm0,%%xmm1 \n" - "movdqu %%xmm6,(%2) \n" - "movdqu %%xmm4,0x10(%2) \n" - "movdqu %%xmm7,0x20(%2) \n" - "movdqu %%xmm1,0x30(%2) \n" + "vpaddusb %%xmm1,%%xmm0,%%xmm2 \n" + "vpshufd $0xd8,%%xmm0,%%xmm0 \n" + "vpshufd $0xd8,%%xmm1,%%xmm1 \n" + "vpshufd $0xd8,%%xmm2,%%xmm2 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vpunpcklbw %%ymm2,%%ymm1,%%ymm1 \n" + "vpunpcklbw %%ymm5,%%ymm0,%%ymm0 \n" + "vpunpcklwd %%ymm0,%%ymm1,%%ymm2 \n" + "vpunpckhwd %%ymm0,%%ymm1,%%ymm0 \n" + "vmovdqu %%ymm2,(%2) \n" + "vmovdqu %%ymm0,0x20(%2) \n" "lea 0x40(%2),%2 \n" "sub $0x10,%3 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_sobelx), // %0 "+r"(src_sobely), // %1 "+r"(dst_argb), // %2 "+r"(width) // %3 : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm5"); } -#endif // HAS_SOBELXYROW_SSE2 - -#ifdef HAS_COMPUTECUMULATIVESUMROW_SSE2 -// Creates a table of cumulative sums where each value is a sum of all values -// above and to the left of the value, inclusive of the value. -void ComputeCumulativeSumRow_SSE2(const uint8_t* row, - int32_t* cumsum, - const int32_t* previous_cumsum, - int width) { - asm volatile( - "pxor %%xmm0,%%xmm0 \n" - "pxor %%xmm1,%%xmm1 \n" - "sub $0x4,%3 \n" - "jl 49f \n" - "test $0xf,%1 \n" - "jne 49f \n" - - // 4 pixel loop. - LABELALIGN - "40: \n" - "movdqu (%0),%%xmm2 \n" - "lea 0x10(%0),%0 \n" - "movdqa %%xmm2,%%xmm4 \n" - "punpcklbw %%xmm1,%%xmm2 \n" - "movdqa %%xmm2,%%xmm3 \n" - "punpcklwd %%xmm1,%%xmm2 \n" - "punpckhwd %%xmm1,%%xmm3 \n" - "punpckhbw %%xmm1,%%xmm4 \n" - "movdqa %%xmm4,%%xmm5 \n" - "punpcklwd %%xmm1,%%xmm4 \n" - "punpckhwd %%xmm1,%%xmm5 \n" - "paddd %%xmm2,%%xmm0 \n" - "movdqu (%2),%%xmm2 \n" - "paddd %%xmm0,%%xmm2 \n" - "paddd %%xmm3,%%xmm0 \n" - "movdqu 0x10(%2),%%xmm3 \n" - "paddd %%xmm0,%%xmm3 \n" - "paddd %%xmm4,%%xmm0 \n" - "movdqu 0x20(%2),%%xmm4 \n" - "paddd %%xmm0,%%xmm4 \n" - "paddd %%xmm5,%%xmm0 \n" - "movdqu 0x30(%2),%%xmm5 \n" - "lea 0x40(%2),%2 \n" - "paddd %%xmm0,%%xmm5 \n" - "movdqu %%xmm2,(%1) \n" - "movdqu %%xmm3,0x10(%1) \n" - "movdqu %%xmm4,0x20(%1) \n" - "movdqu %%xmm5,0x30(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x4,%3 \n" - "jge 40b \n" - - "49: \n" - "add $0x3,%3 \n" - "jl 19f \n" - - // 1 pixel loop. - LABELALIGN - "10: \n" - "movd (%0),%%xmm2 \n" - "lea 0x4(%0),%0 \n" - "punpcklbw %%xmm1,%%xmm2 \n" - "punpcklwd %%xmm1,%%xmm2 \n" - "paddd %%xmm2,%%xmm0 \n" - "movdqu (%2),%%xmm2 \n" - "lea 0x10(%2),%2 \n" - "paddd %%xmm0,%%xmm2 \n" - "movdqu %%xmm2,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x1,%3 \n" - "jge 10b \n" - - "19: \n" - : "+r"(row), // %0 - "+r"(cumsum), // %1 - "+r"(previous_cumsum), // %2 - "+r"(width) // %3 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); -} -#endif // HAS_COMPUTECUMULATIVESUMROW_SSE2 - -#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 -void CumulativeSumToAverageRow_SSE2(const int32_t* topleft, - const int32_t* botleft, - int width, - int area, - uint8_t* dst, - int count) { - asm volatile( - "movd %5,%%xmm5 \n" - "cvtdq2ps %%xmm5,%%xmm5 \n" - "rcpss %%xmm5,%%xmm4 \n" - "pshufd $0x0,%%xmm4,%%xmm4 \n" - "sub $0x4,%3 \n" - "jl 49f \n" - "cmpl $0x80,%5 \n" - "ja 40f \n" - - "pshufd $0x0,%%xmm5,%%xmm5 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "psrld $0x10,%%xmm6 \n" - "cvtdq2ps %%xmm6,%%xmm6 \n" - "addps %%xmm6,%%xmm5 \n" - "mulps %%xmm4,%%xmm5 \n" - "cvtps2dq %%xmm5,%%xmm5 \n" - "packssdw %%xmm5,%%xmm5 \n" - - // 4 pixel small loop. - LABELALIGN - "4: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "movdqu 0x20(%0),%%xmm2 \n" - "movdqu 0x30(%0),%%xmm3 \n" - "psubd 0x00(%0,%4,4),%%xmm0 \n" - "psubd 0x10(%0,%4,4),%%xmm1 \n" - "psubd 0x20(%0,%4,4),%%xmm2 \n" - "psubd 0x30(%0,%4,4),%%xmm3 \n" - "lea 0x40(%0),%0 \n" - "psubd (%1),%%xmm0 \n" - "psubd 0x10(%1),%%xmm1 \n" - "psubd 0x20(%1),%%xmm2 \n" - "psubd 0x30(%1),%%xmm3 \n" - "paddd 0x00(%1,%4,4),%%xmm0 \n" - "paddd 0x10(%1,%4,4),%%xmm1 \n" - "paddd 0x20(%1,%4,4),%%xmm2 \n" - "paddd 0x30(%1,%4,4),%%xmm3 \n" - "lea 0x40(%1),%1 \n" - "packssdw %%xmm1,%%xmm0 \n" - "packssdw %%xmm3,%%xmm2 \n" - "pmulhuw %%xmm5,%%xmm0 \n" - "pmulhuw %%xmm5,%%xmm2 \n" - "packuswb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" - "jge 4b \n" - "jmp 49f \n" - - // 4 pixel loop - LABELALIGN - "40: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "movdqu 0x20(%0),%%xmm2 \n" - "movdqu 0x30(%0),%%xmm3 \n" - "psubd 0x00(%0,%4,4),%%xmm0 \n" - "psubd 0x10(%0,%4,4),%%xmm1 \n" - "psubd 0x20(%0,%4,4),%%xmm2 \n" - "psubd 0x30(%0,%4,4),%%xmm3 \n" - "lea 0x40(%0),%0 \n" - "psubd (%1),%%xmm0 \n" - "psubd 0x10(%1),%%xmm1 \n" - "psubd 0x20(%1),%%xmm2 \n" - "psubd 0x30(%1),%%xmm3 \n" - "paddd 0x00(%1,%4,4),%%xmm0 \n" - "paddd 0x10(%1,%4,4),%%xmm1 \n" - "paddd 0x20(%1,%4,4),%%xmm2 \n" - "paddd 0x30(%1,%4,4),%%xmm3 \n" - "lea 0x40(%1),%1 \n" - "cvtdq2ps %%xmm0,%%xmm0 \n" - "cvtdq2ps %%xmm1,%%xmm1 \n" - "mulps %%xmm4,%%xmm0 \n" - "mulps %%xmm4,%%xmm1 \n" - "cvtdq2ps %%xmm2,%%xmm2 \n" - "cvtdq2ps %%xmm3,%%xmm3 \n" - "mulps %%xmm4,%%xmm2 \n" - "mulps %%xmm4,%%xmm3 \n" - "cvtps2dq %%xmm0,%%xmm0 \n" - "cvtps2dq %%xmm1,%%xmm1 \n" - "cvtps2dq %%xmm2,%%xmm2 \n" - "cvtps2dq %%xmm3,%%xmm3 \n" - "packssdw %%xmm1,%%xmm0 \n" - "packssdw %%xmm3,%%xmm2 \n" - "packuswb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" - "jge 40b \n" - - "49: \n" - "add $0x3,%3 \n" - "jl 19f \n" - - // 1 pixel loop - LABELALIGN - "10: \n" - "movdqu (%0),%%xmm0 \n" - "psubd 0x00(%0,%4,4),%%xmm0 \n" - "lea 0x10(%0),%0 \n" - "psubd (%1),%%xmm0 \n" - "paddd 0x00(%1,%4,4),%%xmm0 \n" - "lea 0x10(%1),%1 \n" - "cvtdq2ps %%xmm0,%%xmm0 \n" - "mulps %%xmm4,%%xmm0 \n" - "cvtps2dq %%xmm0,%%xmm0 \n" - "packssdw %%xmm0,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movd %%xmm0,(%2) \n" - "lea 0x4(%2),%2 \n" - "sub $0x1,%3 \n" - "jge 10b \n" - "19: \n" - : "+r"(topleft), // %0 - "+r"(botleft), // %1 - "+r"(dst), // %2 - "+rm"(count) // %3 - : "r"((ptrdiff_t)(width)), // %4 - "rm"(area) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); -} -#endif // HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 +#endif // HAS_SOBELXYROW_AVX2 #ifdef HAS_COMPUTECUMULATIVESUMROW_AVX2 static const uvec8 kCumSumTranspose = {0, 4, 8, 12, 1, 5, 9, 13, @@ -10806,24 +12076,24 @@ void ComputeCumulativeSumRow_AVX2(const uint8_t* row, int width) { asm volatile( "vpxor %%xmm0,%%xmm0,%%xmm0 \n" - "vbroadcasti128 %4,%%ymm5 \n" + "vbroadcasti128 %4,%%ymm5 \n" "vmovdqu %5,%%ymm6 \n" "vmovdqu %6,%%ymm7 \n" - "vpcmpeqd %%ymm8,%%ymm8,%%ymm8 \n" - "vpsrlw $0xf,%%ymm8,%%ymm8 \n" + "vpcmpeqd %%ymm1,%%ymm1,%%ymm1 \n" + "vpsrlw $0xf,%%ymm1,%%ymm1 \n" "sub $0x4,%3 \n" "jl 49f \n" // 4 pixel loop. LABELALIGN "40: \n" - "vbroadcasti128 (%0),%%ymm2 \n" + "vbroadcasti128 (%0),%%ymm2 \n" "lea 0x10(%0),%0 \n" "vpshufb %%ymm5,%%ymm2,%%ymm2 \n" "vpmaddubsw %%ymm6,%%ymm2,%%ymm3 \n" "vpmaddubsw %%ymm7,%%ymm2,%%ymm4 \n" - "vpmaddwd %%ymm8,%%ymm3,%%ymm3 \n" - "vpmaddwd %%ymm8,%%ymm4,%%ymm4 \n" + "vpmaddwd %%ymm1,%%ymm3,%%ymm3 \n" + "vpmaddwd %%ymm1,%%ymm4,%%ymm4 \n" "vperm2i128 $0x11,%%ymm4,%%ymm4,%%ymm2 \n" "vpaddd (%2),%%ymm3,%%ymm3 \n" "vpaddd 0x20(%2),%%ymm4,%%ymm4 \n" @@ -10863,8 +12133,8 @@ void ComputeCumulativeSumRow_AVX2(const uint8_t* row, : "m"(kCumSumTranspose), // %4 "m"(kCumSumMaddLo), // %5 "m"(kCumSumMaddHi) // %6 - : "memory", "cc", "xmm0", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", - "xmm8"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); } #endif // HAS_COMPUTECUMULATIVESUMROW_AVX2 @@ -10880,7 +12150,7 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vcvtsi2ssl %5,%%xmm5,%%xmm5 \n" "vrcpss %%xmm5,%%xmm5,%%xmm4 \n" "vpshufd $0x0,%%xmm4,%%xmm4 \n" - "sub $0x4,%3 \n" + "subl $0x4,%3 \n" "jl 19f \n" "vinserti128 $0x1,%%xmm4,%%ymm4,%%ymm4 \n" "cmpl $0x80,%5 \n" @@ -10893,7 +12163,7 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vmulss %%xmm4,%%xmm6,%%xmm6 \n" "vcvtps2dq %%xmm6,%%xmm6 \n" "vpackssdw %%xmm6,%%xmm6,%%xmm6 \n" - "vpbroadcastw %%xmm6,%%ymm6 \n" + "vpbroadcastw %%xmm6,%%ymm6 \n" // 4 pixel small loop. LABELALIGN @@ -10910,12 +12180,12 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "lea 0x40(%1),%1 \n" "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" - "vextracti128 $0x1,%%ymm0,%%xmm1 \n" + "vextracti128 $0x1,%%ymm0,%%xmm1 \n" "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" "vpshufd $0xd8,%%xmm0,%%xmm0 \n" "vmovdqu %%xmm0,(%2) \n" "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" + "subl $0x4,%3 \n" "jge 4b \n" "vzeroupper \n" "jmp 19f \n" @@ -10940,17 +12210,17 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vcvtps2dq %%ymm0,%%ymm0 \n" "vcvtps2dq %%ymm1,%%ymm1 \n" "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" - "vextracti128 $0x1,%%ymm0,%%xmm1 \n" + "vextracti128 $0x1,%%ymm0,%%xmm1 \n" "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" "vpshufd $0xd8,%%xmm0,%%xmm0 \n" "vmovdqu %%xmm0,(%2) \n" "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" + "subl $0x4,%3 \n" "jge 10b \n" "vzeroupper \n" "19: \n" - "add $0x3,%3 \n" + "addl $0x3,%3 \n" "jl 99f \n" // 1 pixel loop. @@ -10970,106 +12240,96 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" "vmovd %%xmm0,(%2) \n" "lea 0x4(%2),%2 \n" - "sub $0x1,%3 \n" + "subl $0x1,%3 \n" "jge 20b \n" "99: \n" : "+r"(topleft), // %0 "+r"(botleft), // %1 "+r"(dst), // %2 - "+r"(count) // %3 + "+rm"(count) // %3 : "r"((ptrdiff_t)(width)), // %4 - "r"(area) // %5 + "rm"(area) // %5 : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5", "xmm6"); } #endif // HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 -#ifdef HAS_ARGBAFFINEROW_SSE2 +#ifdef HAS_ARGBAFFINEROW_AVX2 +static const float kAffineRamp[8] = {0.0f, 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, 7.0f}; +static const float kAffineStep8 = 8.0f; // Copy ARGB pixels from source image with slope to a row of destination. LIBYUV_API -void ARGBAffineRow_SSE2(const uint8_t* src_argb, +void ARGBAffineRow_AVX2(const uint8_t* src_argb, int src_argb_stride, uint8_t* dst_argb, const float* src_dudv, int width) { - ptrdiff_t src_argb_stride_temp = src_argb_stride; - intptr_t temp; asm volatile( - "movq (%3),%%xmm2 \n" - "movq 0x08(%3),%%xmm7 \n" - "shl $0x10,%1 \n" - "add $0x4,%1 \n" - "movd %1,%%xmm5 \n" - "sub $0x4,%4 \n" - "jl 49f \n" + "vbroadcastss (%[uv]),%%ymm2 \n" + "vbroadcastss 0x04(%[uv]),%%ymm3 \n" + "vbroadcastss 0x08(%[uv]),%%ymm4 \n" + "vbroadcastss 0x0c(%[uv]),%%ymm5 \n" + "vmovups %[ramp],%%ymm0 \n" + "vmulps %%ymm0,%%ymm4,%%ymm1 \n" + "vaddps %%ymm1,%%ymm2,%%ymm2 \n" + "vmulps %%ymm0,%%ymm5,%%ymm1 \n" + "vaddps %%ymm1,%%ymm3,%%ymm3 \n" + "vbroadcastss %[step8],%%ymm0 \n" + "vmulps %%ymm0,%%ymm4,%%ymm4 \n" + "vmulps %%ymm0,%%ymm5,%%ymm5 \n" + "vpbroadcastd %[stride],%%ymm6 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" - "pshufd $0x44,%%xmm7,%%xmm7 \n" - "pshufd $0x0,%%xmm5,%%xmm5 \n" - "movdqa %%xmm2,%%xmm0 \n" - "addps %%xmm7,%%xmm0 \n" - "movlhps %%xmm0,%%xmm2 \n" - "movdqa %%xmm7,%%xmm4 \n" - "addps %%xmm4,%%xmm4 \n" - "movdqa %%xmm2,%%xmm3 \n" - "addps %%xmm4,%%xmm3 \n" - "addps %%xmm4,%%xmm4 \n" - - // 4 pixel loop LABELALIGN - "40: \n" - "cvttps2dq %%xmm2,%%xmm0 \n" // x,y float->int first 2 - "cvttps2dq %%xmm3,%%xmm1 \n" // x,y float->int next 2 - "packssdw %%xmm1,%%xmm0 \n" // x, y as 8 shorts - "pmaddwd %%xmm5,%%xmm0 \n" // off = x*4 + y*stride - "movd %%xmm0,%k1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" - "movd %%xmm0,%k5 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" - "movd 0x00(%0,%1,1),%%xmm1 \n" - "movd 0x00(%0,%5,1),%%xmm6 \n" - "punpckldq %%xmm6,%%xmm1 \n" - "addps %%xmm4,%%xmm2 \n" - "movq %%xmm1,(%2) \n" - "movd %%xmm0,%k1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" - "movd %%xmm0,%k5 \n" - "movd 0x00(%0,%1,1),%%xmm0 \n" - "movd 0x00(%0,%5,1),%%xmm6 \n" - "punpckldq %%xmm6,%%xmm0 \n" - "addps %%xmm4,%%xmm3 \n" - "movq %%xmm0,0x08(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%4 \n" - "jge 40b \n" + "1: \n" + "vcvttps2dq %%ymm2,%%ymm0 \n" + "vcvttps2dq %%ymm3,%%ymm1 \n" + "vpslld $2,%%ymm0,%%ymm0 \n" + "vpmulld %%ymm6,%%ymm1,%%ymm1 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[src],%%ymm0,1),%%ymm1\n" + "vaddps %%ymm4,%%ymm2,%%ymm2 \n" + "vaddps %%ymm5,%%ymm3,%%ymm3 \n" + "vmovdqu %%ymm1,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" - "49: \n" - "add $0x3,%4 \n" - "jl 19f \n" + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + "vmovss 0x08(%[uv]),%%xmm4 \n" + "vmovss 0x0c(%[uv]),%%xmm5 \n" - // 1 pixel loop + // 1 pixel remainder loop. LABELALIGN - "10: \n" - "cvttps2dq %%xmm2,%%xmm0 \n" - "packssdw %%xmm0,%%xmm0 \n" - "pmaddwd %%xmm5,%%xmm0 \n" - "addps %%xmm7,%%xmm2 \n" - "movd %%xmm0,%k1 \n" - "movd 0x00(%0,%1,1),%%xmm0 \n" - "movd %%xmm0,(%2) \n" - "lea 0x04(%2),%2 \n" - "sub $0x1,%4 \n" - "jge 10b \n" - "19: \n" - : "+r"(src_argb), // %0 - "+r"(src_argb_stride_temp), // %1 - "+r"(dst_argb), // %2 - "+r"(src_dudv), // %3 - "+rm"(width), // %4 - "=&r"(temp) // %5 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + "3: \n" + "vcvttps2dq %%xmm2,%%xmm0 \n" + "vcvttps2dq %%xmm3,%%xmm1 \n" + "vpslld $2,%%xmm0,%%xmm0 \n" + "vpmulld %%xmm6,%%xmm1,%%xmm1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpcmpeqd %%xmm7,%%xmm7,%%xmm7 \n" + "vpsrldq $12,%%xmm7,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[src],%%xmm0,1),%%xmm1\n" + "vaddss %%xmm4,%%xmm2,%%xmm2 \n" + "vaddss %%xmm5,%%xmm3,%%xmm3 \n" + "vmovd %%xmm1,(%[dst]) \n" + "lea 0x04(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [width] "+r"(width) + : [src] "r"(src_argb), [uv] "r"(src_dudv), [stride] "m"(src_argb_stride), + [ramp] "m"(kAffineRamp), [step8] "m"(kAffineStep8) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); } -#endif // HAS_ARGBAFFINEROW_SSE2 +#endif // HAS_ARGBAFFINEROW_AVX2 #ifdef HAS_INTERPOLATEROW_AVX2 // Bilinear filter 32x2 -> 32x1 @@ -11701,11 +12961,14 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBPOLYNOMIALROW_AVX2 -#ifdef HAS_HALFFLOATROW_AVX2 +#if defined(HAS_HALFFLOATROW_AVX2) || defined(HAS_HALFFLOATROW_AVX512BW) // 2^-112 moves the float exponent into half float position. Applied as a // separate multiply after scale so no denormal constant is formed when // scale < 2^-14 (see HalfFloatRow_C). static const float kScaleBias = 1.9259299444e-34f; +#endif + +#ifdef HAS_HALFFLOATROW_AVX2 void HalfFloatRow_AVX2(const uint16_t* src, uint16_t* dst, float scale, @@ -11751,6 +13014,54 @@ void HalfFloatRow_AVX2(const uint16_t* src, } #endif // HAS_HALFFLOATROW_AVX2 +#ifdef HAS_HALFFLOATROW_AVX512BW +void HalfFloatRow_AVX512BW(const uint16_t* src, + uint16_t* dst, + float scale, + int width) { + uintptr_t temp; + asm volatile( + "vbroadcastss %[scale],%%zmm4 \n" + "vbroadcastss %[bias],%%zmm5 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vpmovzxwd (%[src]),%%zmm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vcvtdq2ps %%zmm0,%%zmm0 \n" + "vmulps %%zmm4,%%zmm0,%%zmm0 \n" + "vmulps %%zmm5,%%zmm0,%%zmm0 \n" + "vpsrld $0xd,%%zmm0,%%zmm0 \n" + "vpmovdw %%zmm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vpmovzxwd (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vcvtdq2ps %%zmm0,%%zmm0 \n" + "vmulps %%zmm4,%%zmm0,%%zmm0 \n" + "vmulps %%zmm5,%%zmm0,%%zmm0 \n" + "vpsrld $0xd,%%zmm0,%%zmm0 \n" + "vpmovdw %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : + [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [temp] "=&r"(temp) + : [scale] "m"(scale), [bias] "m"(kScaleBias) + : "memory", "cc", "xmm0", "xmm4", "xmm5", "k1"); +} +#endif // HAS_HALFFLOATROW_AVX512BW + #ifdef HAS_HALFFLOATROW_F16C void HalfFloatRow_F16C(const uint16_t* src, uint16_t* dst, @@ -11816,169 +13127,974 @@ void HalfFloat1Row_F16C(const uint16_t* src, uint16_t* dst, float, int width) { } #endif // HAS_HALFFLOATROW_F16C -#ifdef HAS_ARGBCOLORTABLEROW_X86 -// Tranform ARGB pixels with color table. -void ARGBColorTableRow_X86(uint8_t* dst_argb, +#ifdef HAS_BYTETOFLOATROW_AVX2 +void ByteToFloatRow_AVX2(const uint8_t* src, + float* dst, + float scale, + int width) { + asm volatile("vbroadcastss %3,%%ymm2 \n" + + LABELALIGN + "1: \n" + "vpmovzxbd (%0),%%ymm0 \n" + "vpmovzxbd 0x08(%0),%%ymm1 \n" + "lea 0x10(%0),%0 \n" + "vcvtdq2ps %%ymm0,%%ymm0 \n" + "vcvtdq2ps %%ymm1,%%ymm1 \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmulps %%ymm2,%%ymm1,%%ymm1 \n" + "vmovups %%ymm0,(%1) \n" + "vmovups %%ymm1,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 +#if defined(__x86_64__) + : "x"(scale) // %3 +#else + : "m"(scale) // %3 +#endif + : "memory", "cc", "xmm0", "xmm1", "xmm2"); +} +#endif // HAS_BYTETOFLOATROW_AVX2 + +#ifdef HAS_GAUSSROW_AVX2 +static const int32_t kGaussRound128 = 128; +static const int8_t kShufGauss32To16[16] = {0, 1, 4, 5, 8, 9, 12, 13, + -1, -1, -1, -1, -1, -1, -1, -1}; +void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width) { + asm volatile( + "vpbroadcastd %[k128],%%ymm4 \n" + "vbroadcasti128 %[kShuf],%%ymm5 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vmovdqu 0x04(%[src]),%%ymm1 \n" + "vmovdqu 0x08(%[src]),%%ymm2 \n" + "vpaddd 0x10(%[src]),%%ymm0,%%ymm0 \n" + "vpaddd 0x0c(%[src]),%%ymm1,%%ymm1 \n" + "lea 0x20(%[src]),%[src] \n" + "vpslld $2,%%ymm1,%%ymm1 \n" + "vpslld $1,%%ymm2,%%ymm3 \n" + "vpslld $2,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm4,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vpsrld $8,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovd (%[src]),%%xmm0 \n" + "vmovd 0x04(%[src]),%%xmm1 \n" + "vmovd 0x08(%[src]),%%xmm2 \n" + "vmovd 0x0c(%[src]),%%xmm3 \n" + "vpaddd %%xmm3,%%xmm1,%%xmm1 \n" + "vmovd 0x10(%[src]),%%xmm3 \n" + "vpaddd %%xmm3,%%xmm0,%%xmm0 \n" + "lea 0x04(%[src]),%[src] \n" + "vpslld $2,%%xmm1,%%xmm1 \n" + "vpslld $1,%%xmm2,%%xmm3 \n" + "vpslld $2,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm3,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm4,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm2,%%xmm0,%%xmm0 \n" + "vpsrld $8,%%xmm0,%%xmm0 \n" + "vpextrw $0,%%xmm0,(%[dst]) \n" + "lea 0x02(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) + : [k128] "m"(kGaussRound128), [kShuf] "m"(kShufGauss32To16) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +} +#endif // HAS_GAUSSROW_AVX2 + +#ifdef HAS_GAUSSCOL_AVX2 +void GaussCol_AVX2(const uint16_t* src0, + const uint16_t* src1, + const uint16_t* src2, + const uint16_t* src3, + const uint16_t* src4, + uint32_t* dst, + int width) { +#if defined(__x86_64__) + ptrdiff_t off1 = (ptrdiff_t)((uintptr_t)src1 - (uintptr_t)src0); + ptrdiff_t off2 = (ptrdiff_t)((uintptr_t)src2 - (uintptr_t)src0); + ptrdiff_t off3 = (ptrdiff_t)((uintptr_t)src3 - (uintptr_t)src0); + ptrdiff_t off4 = (ptrdiff_t)((uintptr_t)src4 - (uintptr_t)src0); + asm volatile( + "vpxor %%xmm5,%%xmm5,%%xmm5 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vpmovzxwd (%[p0]),%%ymm0 \n" + "vpmovzxwd (%[p0],%[off4],1),%%ymm4 \n" + "vpmovzxwd (%[p0],%[off1],1),%%ymm1 \n" + "vpmovzxwd (%[p0],%[off3],1),%%ymm3 \n" + "vpmovzxwd (%[p0],%[off2],1),%%ymm2 \n" + "lea 0x10(%[p0]),%[p0] \n" + "vpaddd %%ymm4,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm3,%%ymm1,%%ymm1 \n" + "vpslld $2,%%ymm1,%%ymm1 \n" + "vpslld $1,%%ymm2,%%ymm3 \n" + "vpslld $2,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dp]) \n" + "lea 0x20(%[dp]),%[dp] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vpinsrw $0,(%[p0]),%%xmm5,%%xmm0 \n" + "vpinsrw $0,(%[p0],%[off4],1),%%xmm5,%%xmm4\n" + "vpinsrw $0,(%[p0],%[off1],1),%%xmm5,%%xmm1\n" + "vpinsrw $0,(%[p0],%[off3],1),%%xmm5,%%xmm3\n" + "vpinsrw $0,(%[p0],%[off2],1),%%xmm5,%%xmm2\n" + "lea 0x02(%[p0]),%[p0] \n" + "vpaddd %%xmm4,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm3,%%xmm1,%%xmm1 \n" + "vpslld $2,%%xmm1,%%xmm1 \n" + "vpslld $1,%%xmm2,%%xmm3 \n" + "vpslld $2,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm3,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm2,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%[dp]) \n" + "lea 0x04(%[dp]),%[dp] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [p0] "+r"(src0), [dp] "+r"(dst), [width] "+r"(width) + : [off1] "r"(off1), [off2] "r"(off2), [off3] "r"(off3), + [off4] "r"(off4) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#else + int n = width & ~7; + ptrdiff_t idx = 0; + const uint16_t* tmp; + asm volatile( + "vpxor %%xmm5,%%xmm5,%%xmm5 \n" + "cmpl $0,%[n] \n" + "jle 2f \n" + + LABELALIGN + "1: \n" + "mov %[s0],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm0 \n" + "mov %[s4],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm4 \n" + "mov %[s1],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm1 \n" + "mov %[s3],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm3 \n" + "mov %[s2],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm2 \n" + "add $0x8,%[idx] \n" + "vpaddd %%ymm4,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm3,%%ymm1,%%ymm1 \n" + "vpslld $2,%%ymm1,%%ymm1 \n" + "vpslld $1,%%ymm2,%%ymm3 \n" + "vpslld $2,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dp]) \n" + "lea 0x20(%[dp]),%[dp] \n" + "cmpl %[n],%[idx] \n" + "jl 1b \n" + + "2: \n" + "cmpl %[width],%[idx] \n" + "jge 99f \n" + + LABELALIGN + "3: \n" + "mov %[s0],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm0\n" + "mov %[s4],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm4\n" + "mov %[s1],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm1\n" + "mov %[s3],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm3\n" + "mov %[s2],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm2\n" + "add $0x1,%[idx] \n" + "vpaddd %%xmm4,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm3,%%xmm1,%%xmm1 \n" + "vpslld $2,%%xmm1,%%xmm1 \n" + "vpslld $1,%%xmm2,%%xmm3 \n" + "vpslld $2,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm3,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm2,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%[dp]) \n" + "lea 0x04(%[dp]),%[dp] \n" + "cmpl %[width],%[idx] \n" + "jl 3b \n" + + "99: \n" + "vzeroupper \n" + : [idx] "+r"(idx), [tmp] "=&r"(tmp), [dp] "+r"(dst) + : [s0] "m"(src0), [s1] "m"(src1), [s2] "m"(src2), [s3] "m"(src3), + [s4] "m"(src4), [n] "m"(n), [width] "m"(width) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif +} +#endif // HAS_GAUSSCOL_AVX2 + +#if defined(HAS_GAUSSROW_F32_AVX2) || defined(HAS_GAUSSCOL_F32_AVX2) +static const float kGaussF32_4 = 4.0f; +static const float kGaussF32_6 = 6.0f; +#endif // defined(HAS_GAUSSROW_F32_AVX2) || defined(HAS_GAUSSCOL_F32_AVX2) + +#ifdef HAS_GAUSSROW_F32_AVX2 +static const float kGaussF32_Scale = 1.0f / 256.0f; +void GaussRow_F32_AVX2(const float* src, float* dst, int width) { + asm volatile( + "vbroadcastss %[k4],%%ymm4 \n" + "vbroadcastss %[k6],%%ymm5 \n" + "vbroadcastss %[kScale],%%ymm6 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "vmulps 0x04(%[src]),%%ymm4,%%ymm1 \n" + "vmulps 0x08(%[src]),%%ymm5,%%ymm2 \n" + "vmulps 0x0c(%[src]),%%ymm4,%%ymm3 \n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" + "vaddps %%ymm2,%%ymm0,%%ymm0 \n" + "vaddps %%ymm3,%%ymm0,%%ymm0 \n" + "vaddps 0x10(%[src]),%%ymm0,%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmulps %%ymm6,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "vmulss 0x04(%[src]),%%xmm4,%%xmm1 \n" + "vmulss 0x08(%[src]),%%xmm5,%%xmm2 \n" + "vmulss 0x0c(%[src]),%%xmm4,%%xmm3 \n" + "vaddss %%xmm1,%%xmm0,%%xmm0 \n" + "vaddss %%xmm2,%%xmm0,%%xmm0 \n" + "vaddss %%xmm3,%%xmm0,%%xmm0 \n" + "vaddss 0x10(%[src]),%%xmm0,%%xmm0 \n" + "lea 0x04(%[src]),%[src] \n" + "vmulss %%xmm6,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x04(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) + : [k4] "m"(kGaussF32_4), [k6] "m"(kGaussF32_6), + [kScale] "m"(kGaussF32_Scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6"); +} +#endif // HAS_GAUSSROW_F32_AVX2 + +#ifdef HAS_GAUSSCOL_F32_AVX2 +void GaussCol_F32_AVX2(const float* src0, + const float* src1, + const float* src2, + const float* src3, + const float* src4, + float* dst, + int width) { +#if defined(__x86_64__) + ptrdiff_t off1 = (ptrdiff_t)((uintptr_t)src1 - (uintptr_t)src0); + ptrdiff_t off2 = (ptrdiff_t)((uintptr_t)src2 - (uintptr_t)src0); + ptrdiff_t off3 = (ptrdiff_t)((uintptr_t)src3 - (uintptr_t)src0); + ptrdiff_t off4 = (ptrdiff_t)((uintptr_t)src4 - (uintptr_t)src0); + asm volatile( + "vbroadcastss %[k4],%%ymm4 \n" + "vbroadcastss %[k6],%%ymm5 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[p0]),%%ymm0 \n" + "vmulps (%[p0],%[off1],1),%%ymm4,%%ymm1\n" + "vmulps (%[p0],%[off2],1),%%ymm5,%%ymm2\n" + "vmulps (%[p0],%[off3],1),%%ymm4,%%ymm3\n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" + "vaddps %%ymm2,%%ymm0,%%ymm0 \n" + "vaddps %%ymm3,%%ymm0,%%ymm0 \n" + "vaddps (%[p0],%[off4],1),%%ymm0,%%ymm0\n" + "lea 0x20(%[p0]),%[p0] \n" + "vmovups %%ymm0,(%[dp]) \n" + "lea 0x20(%[dp]),%[dp] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[p0]),%%xmm0 \n" + "vmulss (%[p0],%[off1],1),%%xmm4,%%xmm1\n" + "vmulss (%[p0],%[off2],1),%%xmm5,%%xmm2\n" + "vmulss (%[p0],%[off3],1),%%xmm4,%%xmm3\n" + "vaddss %%xmm1,%%xmm0,%%xmm0 \n" + "vaddss %%xmm2,%%xmm0,%%xmm0 \n" + "vaddss %%xmm3,%%xmm0,%%xmm0 \n" + "vaddss (%[p0],%[off4],1),%%xmm0,%%xmm0\n" + "lea 0x04(%[p0]),%[p0] \n" + "vmovss %%xmm0,(%[dp]) \n" + "lea 0x04(%[dp]),%[dp] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [p0] "+r"(src0), [dp] "+r"(dst), [width] "+r"(width) + : [off1] "r"(off1), [off2] "r"(off2), [off3] "r"(off3), + [off4] "r"(off4), [k4] "m"(kGaussF32_4), [k6] "m"(kGaussF32_6) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#else + int n = width & ~7; + ptrdiff_t idx = 0; + const float* tmp; + asm volatile( + "vbroadcastss %[k4],%%ymm4 \n" + "vbroadcastss %[k6],%%ymm5 \n" + "cmpl $0,%[n] \n" + "jle 2f \n" + + LABELALIGN + "1: \n" + "mov %[s0],%[tmp] \n" + "vmovups (%[tmp],%[idx],4),%%ymm0 \n" + "mov %[s1],%[tmp] \n" + "vmulps (%[tmp],%[idx],4),%%ymm4,%%ymm1\n" + "mov %[s2],%[tmp] \n" + "vmulps (%[tmp],%[idx],4),%%ymm5,%%ymm2\n" + "mov %[s3],%[tmp] \n" + "vmulps (%[tmp],%[idx],4),%%ymm4,%%ymm3\n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" + "vaddps %%ymm2,%%ymm0,%%ymm0 \n" + "vaddps %%ymm3,%%ymm0,%%ymm0 \n" + "mov %[s4],%[tmp] \n" + "vaddps (%[tmp],%[idx],4),%%ymm0,%%ymm0\n" + "add $0x8,%[idx] \n" + "vmovups %%ymm0,(%[dp]) \n" + "lea 0x20(%[dp]),%[dp] \n" + "cmpl %[n],%[idx] \n" + "jl 1b \n" + + "2: \n" + "cmpl %[width],%[idx] \n" + "jge 99f \n" + + LABELALIGN + "3: \n" + "mov %[s0],%[tmp] \n" + "vmovss (%[tmp],%[idx],4),%%xmm0 \n" + "mov %[s1],%[tmp] \n" + "vmulss (%[tmp],%[idx],4),%%xmm4,%%xmm1\n" + "mov %[s2],%[tmp] \n" + "vmulss (%[tmp],%[idx],4),%%xmm5,%%xmm2\n" + "mov %[s3],%[tmp] \n" + "vmulss (%[tmp],%[idx],4),%%xmm4,%%xmm3\n" + "vaddss %%xmm1,%%xmm0,%%xmm0 \n" + "vaddss %%xmm2,%%xmm0,%%xmm0 \n" + "vaddss %%xmm3,%%xmm0,%%xmm0 \n" + "mov %[s4],%[tmp] \n" + "vaddss (%[tmp],%[idx],4),%%xmm0,%%xmm0\n" + "add $0x1,%[idx] \n" + "vmovss %%xmm0,(%[dp]) \n" + "lea 0x04(%[dp]),%[dp] \n" + "cmpl %[width],%[idx] \n" + "jl 3b \n" + + "99: \n" + "vzeroupper \n" + : [idx] "+r"(idx), [tmp] "=&r"(tmp), [dp] "+r"(dst) + : [s0] "m"(src0), [s1] "m"(src1), [s2] "m"(src2), [s3] "m"(src3), + [s4] "m"(src4), [n] "m"(n), [width] "m"(width), [k4] "m"(kGaussF32_4), + [k6] "m"(kGaussF32_6) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif +} +#endif // HAS_GAUSSCOL_F32_AVX2 + +#ifdef HAS_SCALEMAXSAMPLES_AVX2 +float ScaleMaxSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + float fmax; + asm volatile( + "vpxor %%xmm3,%%xmm3,%%xmm3 \n" + "vbroadcastss %[scale],%%ymm2 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmaxps %%ymm3,%%ymm0,%%ymm3 \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "vextractf128 $0x1,%%ymm3,%%xmm1 \n" + "vmaxps %%xmm3,%%xmm1,%%xmm3 \n" + "vpshufd $0x4e,%%xmm3,%%xmm1 \n" + "vmaxps %%xmm3,%%xmm1,%%xmm3 \n" + "vpshufd $0x55,%%xmm3,%%xmm1 \n" + "vmaxss %%xmm3,%%xmm1,%%xmm3 \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vmaxss %%xmm3,%%xmm0,%%xmm3 \n" + "vmulss %%xmm2,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vmovss %%xmm3,%[fmax] \n" + "vzeroupper \n" + : [src] "+r"(src), + [dst] "+r"(dst), + [width] "+r"(width), + [fmax] "=m"(fmax) + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + return fmax; +} +#endif // HAS_SCALEMAXSAMPLES_AVX2 + +#ifdef HAS_SCALESUMSAMPLES_AVX2 +float ScaleSumSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + float fsum; + asm volatile( + "vpxor %%xmm3,%%xmm3,%%xmm3 \n" + "vbroadcastss %[scale],%%ymm2 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmulps %%ymm0,%%ymm0,%%ymm1 \n" + "vaddps %%ymm1,%%ymm3,%%ymm3 \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "vextractf128 $0x1,%%ymm3,%%xmm1 \n" + "vaddps %%xmm1,%%xmm3,%%xmm3 \n" + "vhaddps %%xmm3,%%xmm3,%%xmm3 \n" + "vhaddps %%xmm3,%%xmm3,%%xmm3 \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vmulss %%xmm0,%%xmm0,%%xmm1 \n" + "vaddss %%xmm1,%%xmm3,%%xmm3 \n" + "vmulss %%xmm2,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vmovss %%xmm3,%[fsum] \n" + "vzeroupper \n" + : [src] "+r"(src), + [dst] "+r"(dst), + [width] "+r"(width), + [fsum] "=m"(fsum) + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + return fsum; +} +#endif // HAS_SCALESUMSAMPLES_AVX2 + +#ifdef HAS_SCALESAMPLES_AVX2 +void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) { + asm volatile( + "vbroadcastss %[scale],%%ymm2 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vmulss %%xmm2,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm2"); +} +#endif // HAS_SCALESAMPLES_AVX2 + +#ifdef HAS_ARGBCOLORTABLEROW_AVX2 +// Transform ARGB pixels with color table. +void ARGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + asm volatile( + "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrld $24,%%ymm6,%%ymm6 \n" // 0x000000ff + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[dst]),%%ymm0 \n" + "vpand %%ymm6,%%ymm0,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm2\n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + + "vpsrld $8,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $8,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vpsrld $16,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $16,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vpsrld $24,%%ymm0,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $24,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vmovdqu %%ymm2,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + "vpcmpeqd %%xmm5,%%xmm5,%%xmm5 \n" + "vpsrldq $12,%%xmm5,%%xmm5 \n" + + LABELALIGN + "3: \n" + "vmovd (%[dst]),%%xmm0 \n" + "vpand %%xmm6,%%xmm0,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm2\n" + "vpand %%xmm6,%%xmm2,%%xmm2 \n" + + "vpsrld $8,%%xmm0,%%xmm1 \n" + "vpand %%xmm6,%%xmm1,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $8,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vpsrld $16,%%xmm0,%%xmm1 \n" + "vpand %%xmm6,%%xmm1,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $16,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vpsrld $24,%%xmm0,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $24,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vmovd %%xmm2,(%[dst]) \n" + "lea 0x04(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [width] "+r"(width) + : [table] "r"(table_argb) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#endif // HAS_ARGBCOLORTABLEROW_AVX2 + +#ifdef HAS_RGBCOLORTABLEROW_AVX2 +// Transform RGB pixels with color table. +void RGBColorTableRow_AVX2(uint8_t* dst_argb, const uint8_t* table_argb, int width) { - uintptr_t pixel_temp; asm volatile( - // 1 pixel loop. - LABELALIGN - "1: \n" - "movzb (%0),%1 \n" - "lea 0x4(%0),%0 \n" - "movzb 0x00(%3,%1,4),%1 \n" - "mov %b1,-0x4(%0) \n" - "movzb -0x3(%0),%1 \n" - "movzb 0x01(%3,%1,4),%1 \n" - "mov %b1,-0x3(%0) \n" - "movzb -0x2(%0),%1 \n" - "movzb 0x02(%3,%1,4),%1 \n" - "mov %b1,-0x2(%0) \n" - "movzb -0x1(%0),%1 \n" - "movzb 0x03(%3,%1,4),%1 \n" - "mov %b1,-0x1(%0) \n" - "dec %2 \n" - "jg 1b \n" - : "+r"(dst_argb), // %0 - "=&d"(pixel_temp), // %1 - "+r"(width) // %2 - : "r"(table_argb) // %3 - : "memory", "cc"); -} -#endif // HAS_ARGBCOLORTABLEROW_X86 + "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrld $24,%%ymm6,%%ymm6 \n" // 0x000000ff + "sub $0x8,%[width] \n" + "jl 2f \n" -#ifdef HAS_RGBCOLORTABLEROW_X86 -// Tranform RGB pixels with color table. -void RGBColorTableRow_X86(uint8_t* dst_argb, - const uint8_t* table_argb, - int width) { - uintptr_t pixel_temp; - asm volatile( - // 1 pixel loop. LABELALIGN - "1: \n" - "movzb (%0),%1 \n" - "lea 0x4(%0),%0 \n" - "movzb 0x00(%3,%1,4),%1 \n" - "mov %b1,-0x4(%0) \n" - "movzb -0x3(%0),%1 \n" - "movzb 0x01(%3,%1,4),%1 \n" - "mov %b1,-0x3(%0) \n" - "movzb -0x2(%0),%1 \n" - "movzb 0x02(%3,%1,4),%1 \n" - "mov %b1,-0x2(%0) \n" - "dec %2 \n" - "jg 1b \n" - : "+r"(dst_argb), // %0 - "=&d"(pixel_temp), // %1 - "+r"(width) // %2 - : "r"(table_argb) // %3 - : "memory", "cc"); -} -#endif // HAS_RGBCOLORTABLEROW_X86 + "1: \n" + "vmovdqu (%[dst]),%%ymm0 \n" + "vpand %%ymm6,%%ymm0,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm2\n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" -#ifdef HAS_ARGBLUMACOLORTABLEROW_SSSE3 -// Tranform RGB pixels with luma table. -void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb, - uint8_t* dst_argb, - int width, - const uint8_t* luma, - uint32_t lumacoeff) { + "vpsrld $8,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $8,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vpsrld $16,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $16,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vpslld $24,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpor %%ymm0,%%ymm2,%%ymm2 \n" + + "vmovdqu %%ymm2,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + "vpcmpeqd %%xmm5,%%xmm5,%%xmm5 \n" + "vpsrldq $12,%%xmm5,%%xmm5 \n" + + LABELALIGN + "3: \n" + "vmovd (%[dst]),%%xmm0 \n" + "vpand %%xmm6,%%xmm0,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm2\n" + "vpand %%xmm6,%%xmm2,%%xmm2 \n" + + "vpsrld $8,%%xmm0,%%xmm1 \n" + "vpand %%xmm6,%%xmm1,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $8,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vpsrld $16,%%xmm0,%%xmm1 \n" + "vpand %%xmm6,%%xmm1,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $16,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vpslld $24,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm0,%%xmm0 \n" + "vpor %%xmm0,%%xmm2,%%xmm2 \n" + + "vmovd %%xmm2,(%[dst]) \n" + "lea 0x04(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [width] "+r"(width) + : [table] "r"(table_argb) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#endif // HAS_RGBCOLORTABLEROW_AVX2 + +#ifdef HAS_ARGBLUMACOLORTABLEROW_AVX2 +// Transform RGB pixels with luma table. +void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff) { + uint32_t coeff = lumacoeff & 0x00ffffffu; uintptr_t pixel_temp; uintptr_t table_temp; asm volatile( - "movd %6,%%xmm3 \n" - "pshufd $0x0,%%xmm3,%%xmm3 \n" - "pcmpeqb %%xmm4,%%xmm4 \n" - "psllw $0x8,%%xmm4 \n" - "pxor %%xmm5,%%xmm5 \n" + "vpbroadcastd %[coeff],%%ymm5 \n" + "vpcmpeqd %%xmm4,%%xmm4,%%xmm4 \n" + "vpsrlw $9,%%xmm4,%%xmm4 \n" + "vpsllw $8,%%xmm4,%%xmm4 \n" // 0x7f00 + "subl $0x8,%[width] \n" + "jl 2f \n" - // 4 pixel loop. LABELALIGN - "1: \n" - "movdqu (%2),%%xmm0 \n" - "pmaddubsw %%xmm3,%%xmm0 \n" - "phaddw %%xmm0,%%xmm0 \n" - "pand %%xmm4,%%xmm0 \n" - "punpcklwd %%xmm5,%%xmm0 \n" - "movd %%xmm0,%k1 \n" // 32 bit offset - "add %5,%1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vpmaddubsw %%ymm5,%%ymm0,%%ymm1 \n" + "vphaddw %%ymm1,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpand %%xmm4,%%xmm1,%%xmm1 \n" - "movzb (%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,(%3) \n" - "movzb 0x1(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x1(%3) \n" - "movzb 0x2(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x2(%3) \n" - "movzb 0x3(%2),%0 \n" - "mov %b0,0x3(%3) \n" + "vpextrw $0,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x00(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x00(%[dst]) \n" + "movzbl 0x01(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x01(%[dst]) \n" + "movzbl 0x02(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x02(%[dst]) \n" + "movzbl 0x03(%[src]),%k[t0] \n" + "mov %b[t0],0x03(%[dst]) \n" - "movd %%xmm0,%k1 \n" // 32 bit offset - "add %5,%1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" + "vpextrw $1,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x04(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x04(%[dst]) \n" + "movzbl 0x05(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x05(%[dst]) \n" + "movzbl 0x06(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x06(%[dst]) \n" + "movzbl 0x07(%[src]),%k[t0] \n" + "mov %b[t0],0x07(%[dst]) \n" - "movzb 0x4(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x4(%3) \n" - "movzb 0x5(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x5(%3) \n" - "movzb 0x6(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x6(%3) \n" - "movzb 0x7(%2),%0 \n" - "mov %b0,0x7(%3) \n" + "vpextrw $2,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x08(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x08(%[dst]) \n" + "movzbl 0x09(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x09(%[dst]) \n" + "movzbl 0x0a(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x0a(%[dst]) \n" + "movzbl 0x0b(%[src]),%k[t0] \n" + "mov %b[t0],0x0b(%[dst]) \n" - "movd %%xmm0,%k1 \n" // 32 bit offset - "add %5,%1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" + "vpextrw $3,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x0c(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x0c(%[dst]) \n" + "movzbl 0x0d(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x0d(%[dst]) \n" + "movzbl 0x0e(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x0e(%[dst]) \n" + "movzbl 0x0f(%[src]),%k[t0] \n" + "mov %b[t0],0x0f(%[dst]) \n" - "movzb 0x8(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x8(%3) \n" - "movzb 0x9(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x9(%3) \n" - "movzb 0xa(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0xa(%3) \n" - "movzb 0xb(%2),%0 \n" - "mov %b0,0xb(%3) \n" + "vpextrw $4,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x10(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x10(%[dst]) \n" + "movzbl 0x11(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x11(%[dst]) \n" + "movzbl 0x12(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x12(%[dst]) \n" + "movzbl 0x13(%[src]),%k[t0] \n" + "mov %b[t0],0x13(%[dst]) \n" - "movd %%xmm0,%k1 \n" // 32 bit offset - "add %5,%1 \n" + "vpextrw $5,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x14(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x14(%[dst]) \n" + "movzbl 0x15(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x15(%[dst]) \n" + "movzbl 0x16(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x16(%[dst]) \n" + "movzbl 0x17(%[src]),%k[t0] \n" + "mov %b[t0],0x17(%[dst]) \n" - "movzb 0xc(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0xc(%3) \n" - "movzb 0xd(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0xd(%3) \n" - "movzb 0xe(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0xe(%3) \n" - "movzb 0xf(%2),%0 \n" - "mov %b0,0xf(%3) \n" - "lea 0x10(%2),%2 \n" - "lea 0x10(%3),%3 \n" - "sub $0x4,%4 \n" - "jg 1b \n" - : "=&d"(pixel_temp), // %0 - "=&a"(table_temp), // %1 - "+r"(src_argb), // %2 - "+r"(dst_argb), // %3 - "+rm"(width) // %4 - : "r"(luma), // %5 - "rm"(lumacoeff) // %6 - : "memory", "cc", "xmm0", "xmm3", "xmm4", "xmm5"); + "vpextrw $6,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x18(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x18(%[dst]) \n" + "movzbl 0x19(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x19(%[dst]) \n" + "movzbl 0x1a(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x1a(%[dst]) \n" + "movzbl 0x1b(%[src]),%k[t0] \n" + "mov %b[t0],0x1b(%[dst]) \n" + + "vpextrw $7,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x1c(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x1c(%[dst]) \n" + "movzbl 0x1d(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x1d(%[dst]) \n" + "movzbl 0x1e(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x1e(%[dst]) \n" + "movzbl 0x1f(%[src]),%k[t0] \n" + "mov %b[t0],0x1f(%[dst]) \n" + + "lea 0x20(%[src]),%[src] \n" + "lea 0x20(%[dst]),%[dst] \n" + "subl $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "addl $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovd (%[src]),%%xmm0 \n" + "vpmaddubsw %%xmm5,%%xmm0,%%xmm1 \n" + "vphaddw %%xmm1,%%xmm1,%%xmm1 \n" + "vpand %%xmm4,%%xmm1,%%xmm1 \n" + "vpextrw $0,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x00(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x00(%[dst]) \n" + "movzbl 0x01(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x01(%[dst]) \n" + "movzbl 0x02(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x02(%[dst]) \n" + "movzbl 0x03(%[src]),%k[t0] \n" + "mov %b[t0],0x03(%[dst]) \n" + "lea 0x04(%[src]),%[src] \n" + "lea 0x04(%[dst]),%[dst] \n" + "subl $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), +#if defined(__i386__) + [width] "+m"(width), +#else + [width] "+rm"(width), +#endif + [t0] "=&q"(pixel_temp), [t1] "=&r"(table_temp) + : [luma] "r"(luma), [coeff] "m"(coeff) + : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5"); } -#endif // HAS_ARGBLUMACOLORTABLEROW_SSSE3 +#endif // HAS_ARGBLUMACOLORTABLEROW_AVX2 static const uvec8 kYUV24Shuffle[3] = { {8, 9, 0, 8, 9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12}, @@ -12278,7 +14394,7 @@ void ClampFloatToZero_SSE2(const float* src_x, float* dst_y, int width) { : "memory", "cc", "xmm0", "xmm1"); } -#ifdef HAS_CONVERT16TO8ROW_AVX2 +#ifdef HAS_CONVERT8TO8ROW_AVX2 void Convert8To8Row_AVX2(const uint8_t* src_y, uint8_t* dst_y, int scale, @@ -12315,7 +14431,62 @@ void Convert8To8Row_AVX2(const uint8_t* src_y, "r"(bias) // %4 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); } -#endif // HAS_CONVERT16TO8ROW_AVX2 +#endif // HAS_CONVERT8TO8ROW_AVX2 + +#ifdef HAS_CONVERT8TO8ROW_AVX512BW +void Convert8To8Row_AVX512BW(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width) { + uintptr_t temp; + asm volatile( + "vpbroadcastw %k[scale],%%zmm2 \n" + "vpbroadcastb %k[bias],%%zmm3 \n" + "vpxord %%zmm4,%%zmm4,%%zmm4 \n" + "vpsllw $8,%%zmm2,%%zmm2 \n" + "sub $0x40,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpunpckhbw %%zmm4,%%zmm0,%%zmm1 \n" + "vpunpcklbw %%zmm4,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddb %%zmm3,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "vmovdqu8 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpunpckhbw %%zmm4,%%zmm0,%%zmm1 \n" + "vpunpcklbw %%zmm4,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddb %%zmm3,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [width] "+r"(width), + [temp] "=&r"(temp) + : [scale] "r"(scale), [bias] "r"(bias) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "k1"); +} +#endif // HAS_CONVERT8TO8ROW_AVX512BW // =========================================================================== // Module 1: AYUV to Planar Y / UV / VU (AVX512VBMI) @@ -12491,10 +14662,155 @@ void OMITFP AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, } #endif // HAS_AYUVTOVUROW_AVX512VBMI +#ifdef HAS_AYUVTOYROW_AVX2 +void OMITFP AYUVToYRow_AVX2(const uint8_t* src_ayuv, + uint8_t* dst_y, + int width) { + asm volatile( + "vpcmpeqd %%ymm2,%%ymm2,%%ymm2 \n" + "vpsrld $24,%%ymm2,%%ymm2 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vmovdqu 0x20(%[src]),%%ymm1 \n" + "lea 0x40(%[src]),%[src] \n" + "vpsrld $16,%%ymm0,%%ymm0 \n" + "vpsrld $16,%%ymm1,%%ymm1 \n" + "vpand %%ymm2,%%ymm0,%%ymm0 \n" + "vpand %%ymm2,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vextracti128 $1,%%ymm0,%%xmm1 \n" + "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ayuv), [dst] "+r"(dst_y), [width] "+r"(width) + : + : "memory", "cc", "xmm0", "xmm1", "xmm2"); +} +#endif // HAS_AYUVTOYROW_AVX2 + +#if defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2) +static const uint8_t kShufAYUVToUV16[16] = { + 1, 5, 0, 4, 9, 13, 8, 12, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + +static const uint8_t kShufAYUVToVU16[16] = { + 0, 4, 1, 5, 8, 12, 9, 13, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + +static inline void AYUVToUVOrVURow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + const uint8_t* shuf_table, + int width) { + asm volatile( + "vbroadcasti128 (%[shuf]),%%ymm6 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpsrlw $15,%%ymm7,%%ymm7 \n" + "vpsllw $8,%%ymm7,%%ymm5 \n" + "vpor %%ymm5,%%ymm7,%%ymm7 \n" + "vpxor %%ymm5,%%ymm5,%%ymm5 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vmovdqu 0x20(%[src]),%%ymm1 \n" + "vmovdqu (%[src],%[stride]),%%ymm2 \n" + "vmovdqu 0x20(%[src],%[stride]),%%ymm3 \n" + "lea 0x40(%[src]),%[src] \n" + "vpshufb %%ymm6,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm6,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm6,%%ymm2,%%ymm2 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm3 \n" + "vpunpcklqdq %%ymm1,%%ymm0,%%ymm0 \n" + "vpunpcklqdq %%ymm3,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm7,%%ymm0,%%ymm0 \n" + "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n" + "vpaddw %%ymm2,%%ymm0,%%ymm0 \n" + "vpsrlw $1,%%ymm0,%%ymm0 \n" + "vpavgw %%ymm5,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vextracti128 $1,%%ymm0,%%xmm1 \n" + "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ayuv), [dst] "+r"(dst_uv), [width] "+rm"(width) + : [stride] "r"((ptrdiff_t)src_stride_ayuv), [shuf] "r"(shuf_table) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7"); +} +#endif // defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2) + +#ifdef HAS_AYUVTOUVROW_AVX2 +void OMITFP AYUVToUVRow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + AYUVToUVOrVURow_AVX2(src_ayuv, src_stride_ayuv, dst_uv, kShufAYUVToUV16, + width); +} +#endif // HAS_AYUVTOUVROW_AVX2 + +#ifdef HAS_AYUVTOVUROW_AVX2 +void OMITFP AYUVToVURow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + AYUVToUVOrVURow_AVX2(src_ayuv, src_stride_ayuv, dst_vu, kShufAYUVToVU16, + width); +} +#endif // HAS_AYUVTOVUROW_AVX2 + // =========================================================================== -// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX512BW) +// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX2 / AVX512BW) // =========================================================================== +#ifdef HAS_AR30TOARGBROW_AVX2 +void OMITFP AR30ToARGBRow_AVX2(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width) { + static const uint32_t kC0000000 = 0xc0000000u; + static const uint32_t k300000FF = 0x300000ffu; + static const uint32_t k0C00FF00 = 0x0c00ff00u; + static const uint32_t k03FF0000 = 0x03ff0000u; + asm volatile( + "vpbroadcastd %[m0],%%ymm4 \n" + "vpbroadcastd %[m1],%%ymm5 \n" + "vpbroadcastd %[m2],%%ymm6 \n" + "vpbroadcastd %[m3],%%ymm7 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpsrld $2,%%ymm0,%%ymm1 \n" + "vpsrld $4,%%ymm0,%%ymm2 \n" + "vpsrld $6,%%ymm0,%%ymm3 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + "vpand %%ymm7,%%ymm3,%%ymm3 \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + "vpor %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ar30), [dst] "+r"(dst_argb), [width] "+r"(width) + : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00), + [m3] "m"(k03FF0000) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_AR30TOARGBROW_AVX2 + #ifdef HAS_AR30TOARGBROW_AVX512BW void OMITFP AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_argb, @@ -12561,10 +14877,55 @@ void OMITFP AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, } #endif // HAS_AR30TOARGBROW_AVX512BW -#ifdef HAS_AR30TOABGRROW_AVX512BW +#if defined(HAS_AR30TOABGRROW_AVX2) || defined(HAS_AR30TOABGRROW_AVX512BW) static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15}; +#endif +#ifdef HAS_AR30TOABGRROW_AVX2 +void OMITFP AR30ToABGRRow_AVX2(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width) { + static const uint32_t kC0000000 = 0xc0000000u; + static const uint32_t k300000FF = 0x300000ffu; + static const uint32_t k0C00FF00 = 0x0c00ff00u; + static const uint32_t k03FF0000 = 0x03ff0000u; + asm volatile( + "vpbroadcastd %[m0],%%ymm4 \n" + "vpbroadcastd %[m1],%%ymm5 \n" + "vpbroadcastd %[m2],%%ymm6 \n" + "vbroadcasti128 %[shuf],%%ymm7 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpsrld $2,%%ymm0,%%ymm1 \n" + "vpsrld $4,%%ymm0,%%ymm2 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + "vpsrld $6,%%ymm0,%%ymm3 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vpbroadcastd %[m3],%%ymm1 \n" + "vpand %%ymm1,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + "vpor %%ymm2,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm7,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ar30), [dst] "+r"(dst_abgr), [width] "+r"(width) + : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00), + [m3] "m"(k03FF0000), [shuf] "m"(kShufSwapRB16) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_AR30TOABGRROW_AVX2 + +#ifdef HAS_AR30TOABGRROW_AVX512BW void OMITFP AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) { @@ -12633,6 +14994,40 @@ void OMITFP AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, } #endif // HAS_AR30TOABGRROW_AVX512BW +#ifdef HAS_AR30TOAB30ROW_AVX2 +void OMITFP AR30ToAB30Row_AVX2(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width) { + static const uint32_t kC00FFC00 = 0xc00ffc00u; + static const uint32_t k3FF00000 = 0x3ff00000u; + static const uint32_t k000003FF = 0x000003ffu; + asm volatile( + "vpbroadcastd %[m0],%%ymm3 \n" + "vpbroadcastd %[m1],%%ymm4 \n" + "vpbroadcastd %[m2],%%ymm5 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpslld $20,%%ymm0,%%ymm1 \n" + "vpsrld $20,%%ymm0,%%ymm2 \n" + "vpand %%ymm3,%%ymm0,%%ymm0 \n" + "vpand %%ymm4,%%ymm1,%%ymm1 \n" + "vpand %%ymm5,%%ymm2,%%ymm2 \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vpor %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ar30), [dst] "+r"(dst_ab30), [width] "+r"(width) + : [m0] "m"(kC00FFC00), [m1] "m"(k3FF00000), [m2] "m"(k000003FF) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +} +#endif // HAS_AR30TOAB30ROW_AVX2 + #ifdef HAS_AR30TOAB30ROW_AVX512BW void OMITFP AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_ab30, @@ -13864,12 +16259,6 @@ void OMITFP ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBMULTIPLYROW_AVX512BW -#if defined(HAS_ARGBGRAYROW_AVX512BW) || defined(HAS_ARGBGRAYROW_AVX512VNNI) -static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, - 8, 8, 8, 0x80, 12, 12, 12, 0x80}; -#endif // defined(HAS_ARGBGRAYROW_AVX512BW) || - // defined(HAS_ARGBGRAYROW_AVX512VNNI) - #ifdef HAS_ARGBGRAYROW_AVX512BW void OMITFP ARGBGrayRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, @@ -14490,6 +16879,1163 @@ void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBALPHABLENDROW_AVX512BW +// =========================================================================== +// Module 6: Extended YUV (I444, YUY2, UYVY, I210, I212, I410, P210, P410) +// to ARGB / RGBA / RGB24 / AR30 (AVX512BW / AVX512VBMI) +// =========================================================================== + +#define READYUV444_AVX512BW \ + "vpmovzxbw (%[u_buf]),%%zmm3 \n" \ + "vpmovzxbw 0x00(%[u_buf],%[v_buf],1),%%zmm1 \n" \ + "lea 0x20(%[u_buf]),%[u_buf] \n" \ + "vpsllw $8,%%zmm1,%%zmm1 \n" \ + "vpord %%zmm1,%%zmm3,%%zmm3 \n" \ + "vpmovzxbw (%[y_buf]),%%zmm4 \n" \ + "lea 0x20(%[y_buf]),%[y_buf] \n" \ + "vpsllw $8,%%zmm4,%%zmm1 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV444_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "mov $-1,%[temp] \n" \ + "vpmovzxbw (%[u_buf]),%%zmm3%{%%k1%}%{z%} \n" \ + "vpmovzxbw 0x00(%[u_buf],%[v_buf],1),%%zmm1%{%%k1%}%{z%} \n" \ + "vpsllw $8,%%zmm1,%%zmm1 \n" \ + "vpord %%zmm1,%%zmm3,%%zmm3 \n" \ + "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "vpsllw $8,%%zmm4,%%zmm1 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define STORERGBA_AVX512BW \ + "vpunpcklbw %%zmm0,%%zmm5,%%zmm3 \n" \ + "vpermq %%zmm3,%%zmm18,%%zmm3 \n" \ + "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n" \ + "vpermq %%zmm1,%%zmm18,%%zmm1 \n" \ + "vpunpcklwd %%zmm1,%%zmm3,%%zmm0 \n" \ + "vpunpckhwd %%zmm1,%%zmm3,%%zmm1 \n" \ + "vmovups %%zmm0,(%[dst_rgba]) \n" \ + "vmovups %%zmm1,0x40(%[dst_rgba]) \n" \ + "lea 0x80(%[dst_rgba]),%[dst_rgba] \n" + +#define STORERGBA_MASK_AVX512BW \ + "vpunpcklbw %%zmm0,%%zmm5,%%zmm3 \n" \ + "vpermq %%zmm3,%%zmm18,%%zmm3 \n" \ + "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n" \ + "vpermq %%zmm1,%%zmm18,%%zmm1 \n" \ + "vpunpcklwd %%zmm1,%%zmm3,%%zmm0 \n" \ + "vpunpckhwd %%zmm1,%%zmm3,%%zmm1 \n" \ + "kshiftrd $16,%%k1,%%k2 \n" \ + "vmovdqu32 %%zmm0,(%[dst_rgba])%{%%k1%} \n" \ + "vmovdqu32 %%zmm1,0x40(%[dst_rgba])%{%%k2%} \n" + +#define READYUV210_AVX512BW \ + "vmovdqu16 (%[u_buf]),%%ymm3 \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1 \n" \ + "lea 0x20(%[u_buf]),%[u_buf] \n" \ + "vpsrlw $2,%%ymm3,%%ymm3 \n" \ + "vpsrlw $2,%%ymm1,%%ymm1 \n" \ + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vpsllw $6,%%zmm4,%%zmm1 \n" \ + "vpsrlw $4,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV210_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "mov $-1,%[temp] \n" \ + "lea 1(%q[width]),%k[y_buf] \n" \ + "shr $1,%k[y_buf] \n" \ + "bzhi %q[y_buf],%[temp],%[y_buf] \n" \ + "kmovw %k[y_buf],%%k2 \n" \ + "vmovdqu16 (%[u_buf]),%%ymm3%{%%k2%}%{z%} \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1%{%%k2%}%{z%} \n" \ + "vpsrlw $2,%%ymm3,%%ymm3 \n" \ + "vpsrlw $2,%%ymm1,%%ymm1 \n" \ + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \ + "vpsllw $6,%%zmm4,%%zmm1 \n" \ + "vpsrlw $4,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV212_AVX512BW \ + "vmovdqu16 (%[u_buf]),%%ymm3 \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1 \n" \ + "lea 0x20(%[u_buf]),%[u_buf] \n" \ + "vpsraw $4,%%ymm3,%%ymm3 \n" \ + "vpsraw $4,%%ymm1,%%ymm1 \n" \ + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vpsllw $4,%%zmm4,%%zmm1 \n" \ + "vpsrlw $8,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV212_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "mov $-1,%[temp] \n" \ + "lea 1(%q[width]),%k[y_buf] \n" \ + "shr $1,%k[y_buf] \n" \ + "bzhi %q[y_buf],%[temp],%[y_buf] \n" \ + "kmovw %k[y_buf],%%k2 \n" \ + "vmovdqu16 (%[u_buf]),%%ymm3%{%%k2%}%{z%} \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1%{%%k2%}%{z%} \n" \ + "vpsraw $4,%%ymm3,%%ymm3 \n" \ + "vpsraw $4,%%ymm1,%%ymm1 \n" \ + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \ + "vpsllw $4,%%zmm4,%%zmm1 \n" \ + "vpsrlw $8,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV410_AVX512BW \ + "vmovdqu16 (%[u_buf]),%%zmm3 \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%zmm1 \n" \ + "lea 0x40(%[u_buf]),%[u_buf] \n" \ + "vpsrlw $2,%%zmm3,%%zmm3 \n" \ + "vpsrlw $2,%%zmm1,%%zmm1 \n" \ + "vpackuswb %%zmm3,%%zmm3,%%zmm3 \n" \ + "vpackuswb %%zmm1,%%zmm1,%%zmm1 \n" \ + "vpunpcklbw %%zmm1,%%zmm3,%%zmm3 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vpsllw $6,%%zmm4,%%zmm1 \n" \ + "vpsrlw $4,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV410_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vmovdqu16 (%[u_buf]),%%zmm3%{%%k1%}%{z%} \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%zmm1%{%%k1%}%{z%} \n" \ + "vpsrlw $2,%%zmm3,%%zmm3 \n" \ + "vpsrlw $2,%%zmm1,%%zmm1 \n" \ + "vpackuswb %%zmm3,%%zmm3,%%zmm3 \n" \ + "vpackuswb %%zmm1,%%zmm1,%%zmm1 \n" \ + "vpunpcklbw %%zmm1,%%zmm3,%%zmm3 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "vpsllw $6,%%zmm4,%%zmm1 \n" \ + "vpsrlw $4,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READP210_AVX512BW \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vmovdqu32 (%[uv_buf]),%%zmm3 \n" \ + "lea 0x40(%[uv_buf]),%[uv_buf] \n" \ + "vpshufb %%zmm14,%%zmm3,%%zmm3 \n" + +#define READP210_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "mov $-1,%[temp] \n" \ + "lea 1(%q[width]),%k[y_buf] \n" \ + "shr $1,%k[y_buf] \n" \ + "bzhi %q[y_buf],%[temp],%[y_buf] \n" \ + "kmovw %k[y_buf],%%k2 \n" \ + "vmovdqu32 (%[uv_buf]),%%zmm3%{%%k2%}%{z%} \n" \ + "vpshufb %%zmm14,%%zmm3,%%zmm3 \n" + +#define READP410_AVX512BW \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vmovdqu32 (%[uv_buf]),%%zmm3 \n" \ + "vmovdqu32 0x40(%[uv_buf]),%%zmm1 \n" \ + "lea 0x80(%[uv_buf]),%[uv_buf] \n" \ + "vpsrlw $8,%%zmm3,%%zmm3 \n" \ + "vpsrlw $8,%%zmm1,%%zmm1 \n" \ + "vpmovwb %%zmm3,%%ymm3 \n" \ + "vpmovwb %%zmm1,%%ymm1 \n" \ + "vinserti64x4 $1,%%ymm1,%%zmm3,%%zmm3 \n" + +#define READP410_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "kshiftrd $16,%%k1,%%k2 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "vmovdqu32 (%[uv_buf]),%%zmm3%{%%k1%}%{z%} \n" \ + "vmovdqu32 0x40(%[uv_buf]),%%zmm1%{%%k2%}%{z%} \n" \ + "vpsrlw $8,%%zmm3,%%zmm3 \n" \ + "vpsrlw $8,%%zmm1,%%zmm1 \n" \ + "vpmovwb %%zmm3,%%ymm3 \n" \ + "vpmovwb %%zmm1,%%ymm1 \n" \ + "vinserti64x4 $1,%%ymm1,%%zmm3,%%zmm3 \n" + +#ifdef HAS_I444TOARGBROW_AVX512BW +void OMITFP I444ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV444_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV444_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I444TOARGBROW_AVX512BW + +#ifdef HAS_I444ALPHATOARGBROW_AVX512BW +void OMITFP I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV444_AVX512BW + "vpmovzxbw (%[a_buf]),%%zmm5 \n" + "lea 0x20(%[a_buf]),%[a_buf] \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV444_MASK_AVX512BW + "vpmovzxbw (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [a_buf] "+r"(a_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I444ALPHATOARGBROW_AVX512BW + +#ifdef HAS_I444TORGB24ROW_AVX512VBMI +void OMITFP I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vmovdqu32 %[kMaskBG],%%zmm20 \n" + "vmovdqu32 %[kMaskDST0],%%zmm21 \n" + "vmovdqu32 %[kMaskDST1],%%zmm22 \n" + "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV444_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_AVX512VBMI + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV444_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_MASK_AVX512VBMI + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_rgb24] "+r"(dst_rgb24), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kMaskBG] "m"(kMaskBG), + [kMaskDST0] "m"(kMaskDST0), + [kMaskDST1] "m"(kMaskDST1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm20", "xmm21", "xmm22", "k1", "k3", "k4"); +} +#endif // HAS_I444TORGB24ROW_AVX512VBMI + +#ifdef HAS_I422TORGBAROW_AVX512BW +void OMITFP I422ToRGBARow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgba, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV422_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGBA_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV422_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGBA_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_rgba] "+r"(dst_rgba), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I422TORGBAROW_AVX512BW + +#ifdef HAS_YUY2TOARGBROW_AVX512BW +void OMITFP YUY2ToARGBRow_AVX512BW(const uint8_t* src_yuy2, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vbroadcasti32x4 %[kShufY],%%zmm14 \n" + "vbroadcasti32x4 %[kShufUV],%%zmm15 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src_yuy2]),%%zmm1 \n" + "lea 0x40(%[src_yuy2]),%[src_yuy2] \n" + "vpshufb %%zmm14,%%zmm1,%%zmm4 \n" + "vpshufb %%zmm15,%%zmm1,%%zmm3 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "mov $-1,%[temp] \n" + "add $1,%k[width] \n" + "shr $1,%k[width] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k2 \n" + "vmovdqu32 (%[src_yuy2]),%%zmm1%{%%k2%}%{z%}\n" + "vpshufb %%zmm14,%%zmm1,%%zmm4 \n" + "vpshufb %%zmm15,%%zmm1,%%zmm3 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [src_yuy2] "+r"(src_yuy2), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kShufY] "m"(kShuffleYUY2Y), + [kShufUV] "m"(kShuffleYUY2UV) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm14", "xmm15", "k1", "k2"); +} +#endif // HAS_YUY2TOARGBROW_AVX512BW + +#ifdef HAS_UYVYTOARGBROW_AVX512BW +void OMITFP UYVYToARGBRow_AVX512BW(const uint8_t* src_uyvy, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vbroadcasti32x4 %[kShufY],%%zmm14 \n" + "vbroadcasti32x4 %[kShufUV],%%zmm15 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src_uyvy]),%%zmm1 \n" + "lea 0x40(%[src_uyvy]),%[src_uyvy] \n" + "vpshufb %%zmm14,%%zmm1,%%zmm4 \n" + "vpshufb %%zmm15,%%zmm1,%%zmm3 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "mov $-1,%[temp] \n" + "add $1,%k[width] \n" + "shr $1,%k[width] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k2 \n" + "vmovdqu32 (%[src_uyvy]),%%zmm1%{%%k2%}%{z%}\n" + "vpshufb %%zmm14,%%zmm1,%%zmm4 \n" + "vpshufb %%zmm15,%%zmm1,%%zmm3 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [src_uyvy] "+r"(src_uyvy), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kShufY] "m"(kShuffleUYVYY), + [kShufUV] "m"(kShuffleUYVYUV) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm14", "xmm15", "k1", "k2"); +} +#endif // HAS_UYVYTOARGBROW_AVX512BW + +#ifdef HAS_I210TOARGBROW_AVX512BW +void OMITFP I210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV210_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV210_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I210TOARGBROW_AVX512BW + +#ifdef HAS_I210TOAR30ROW_AVX512BW +void OMITFP I210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV210_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV210_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2"); +} +#endif // HAS_I210TOAR30ROW_AVX512BW + +#ifdef HAS_I210ALPHATOARGBROW_AVX512BW +void OMITFP I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV210_AVX512BW + "vmovdqu16 (%[a_buf]),%%zmm5 \n" + "lea 0x40(%[a_buf]),%[a_buf] \n" + "vpsrlw $2,%%zmm5,%%zmm5 \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV210_MASK_AVX512BW + "vmovdqu16 (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" + "vpsrlw $2,%%zmm5,%%zmm5 \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [a_buf] "+r"(a_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I210ALPHATOARGBROW_AVX512BW + +#ifdef HAS_I212TOARGBROW_AVX512BW +void OMITFP I212ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV212_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV212_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I212TOARGBROW_AVX512BW + +#ifdef HAS_I212TOAR30ROW_AVX512BW +void OMITFP I212ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV212_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV212_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2"); +} +#endif // HAS_I212TOAR30ROW_AVX512BW + +#ifdef HAS_I410TOARGBROW_AVX512BW +void OMITFP I410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV410_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV410_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I410TOARGBROW_AVX512BW + +#ifdef HAS_I410TOAR30ROW_AVX512BW +void OMITFP I410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV410_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV410_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2"); +} +#endif // HAS_I410TOAR30ROW_AVX512BW + +#ifdef HAS_I410ALPHATOARGBROW_AVX512BW +void OMITFP I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV410_AVX512BW + "vmovdqu16 (%[a_buf]),%%zmm5 \n" + "lea 0x40(%[a_buf]),%[a_buf] \n" + "vpsrlw $2,%%zmm5,%%zmm5 \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV410_MASK_AVX512BW + "vmovdqu16 (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" + "vpsrlw $2,%%zmm5,%%zmm5 \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [a_buf] "+r"(a_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I410ALPHATOARGBROW_AVX512BW + +#ifdef HAS_P210TOARGBROW_AVX512BW +void OMITFP P210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vbroadcasti32x4 %[kShufUV],%%zmm14 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READP210_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READP210_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kShufUV] "m"(kShuffleYUY2UV) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm14", "k1", "k2"); +} +#endif // HAS_P210TOARGBROW_AVX512BW + +#ifdef HAS_P210TOAR30ROW_AVX512BW +void OMITFP P210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vbroadcasti32x4 %[kShufUV],%%zmm14 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READP210_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READP210_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1), + [kShufUV] "m"(kShuffleYUY2UV) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm14", "xmm19", "xmm20", "xmm21", + "k1", "k2"); +} +#endif // HAS_P210TOAR30ROW_AVX512BW + +#ifdef HAS_P410TOARGBROW_AVX512BW +void OMITFP P410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READP410_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READP410_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_P410TOARGBROW_AVX512BW + +#ifdef HAS_P410TOAR30ROW_AVX512BW +void OMITFP P410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READP410_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READP410_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2"); +} +#endif // HAS_P410TOAR30ROW_AVX512BW + #endif // !defined(LIBYUV_DISABLE_X86) && // (defined(__x86_64__) || defined(__i386__)) && // !defined(LIBYUV_ENABLE_ROWWIN) diff --git a/source/row_win.cc b/source/row_win.cc index da55bd12e..a986d79ef 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -20,6 +20,7 @@ #include #include // For AVX2 intrinsics +#include // For memcpy #include // For _mm_maddubs_epi16 #if defined(_MSC_VER) && !defined(__clang__) #include @@ -32,6 +33,7 @@ extern "C" { #if defined(__clang__) || defined(__GNUC__) #define LIBYUV_TARGET_SSSE3 __attribute__((target("ssse3"))) +#define LIBYUV_TARGET_AVX __attribute__((target("avx"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) @@ -43,6 +45,7 @@ extern "C" { __attribute__((target("avx512vnni,avx512vbmi,avx512bw,avx512vl,avx512f"))) #else #define LIBYUV_TARGET_SSSE3 +#define LIBYUV_TARGET_AVX #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW #define LIBYUV_TARGET_AVX512VBMI @@ -377,14 +380,16 @@ static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 3, 7, 0, 1}; static const uint32_t kPermd2_AVX2[8] = {5, 6, 3, 7, 0, 1, 2, 4}; #endif // defined(HAS_RAWTORGB24ROW_AVX2) -#if defined(HAS_RAWTORGB24ROW_AVX512BW) +#if defined(HAS_RAWTORGB24ROW_AVX512BW) || \ + defined(HAS_ARGBTORGB24ROW_AVX512BW) || \ + defined(HAS_ARGBTORAWROW_AVX512BW) static const uint32_t kPermdRGB24_0_AVX512BW[16] = { 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20}; static const uint32_t kPermdRGB24_1_AVX512BW[16] = { 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25}; static const uint32_t kPermdRGB24_2_AVX512BW[16] = { 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30}; -#endif // defined(HAS_RAWTORGB24ROW_AVX512BW) +#endif #if defined(HAS_RAWTORGB24ROW_AVX512VBMI) static const uint8_t kPermRAWToRGB24_0[64] = { 0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a, @@ -689,6 +694,9 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, #endif // HAS_RAWTORGB24ROW_AVX512VBMI #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \ defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) || \ defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) || \ defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) @@ -1458,21 +1466,101 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { } #endif // HAS_MIRRORUVROW_AVX2 +#ifdef HAS_MIRRORROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + const __m512i zmm_shuf = _mm512_broadcast_i32x4( + _mm_setr_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); + const __m512i zmm_rev = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + const uint8_t* s = src + width; + while (width >= 64) { + s -= 64; + __m512i v = _mm512_loadu_si512((const __m512i*)s); + v = _mm512_shuffle_epi8(v, zmm_shuf); + v = _mm512_shuffle_i64x2(v, v, 0x1b); + _mm512_storeu_si512((__m512i*)dst, v); + dst += 64; + width -= 64; + } + if (width > 0) { + __mmask64 k = (1ULL << width) - 1ULL; + __m512i v = _mm512_maskz_loadu_epi8(k, src); + v = _mm512_or_si512(_mm512_srli_epi16(v, 8), _mm512_slli_epi16(v, 8)); + __m512i idx = + _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)((64 - width) >> 1))); + v = _mm512_permutexvar_epi16(idx, v); + if (width & 1) { + __m512i v_hi = _mm512_alignr_epi64(v, v, 2); + v = _mm512_alignr_epi8(v_hi, v, 1); + } + _mm512_mask_storeu_epi8(dst, k, v); + } + _mm256_zeroupper(); +} +#endif // HAS_MIRRORROW_AVX512BW + +#ifdef HAS_MIRRORSPLITUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m512i zmm_shuf = _mm512_broadcast_i32x4( + _mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1)); + const __m512i zmm_perm = _mm512_setr_epi64(6, 4, 2, 0, 7, 5, 3, 1); + const __m512i zmm_rev = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + const uint8_t* s = src_uv + (ptrdiff_t)width * 2; + while (width >= 32) { + s -= 64; + __m512i v = _mm512_loadu_si512((const __m512i*)s); + v = _mm512_shuffle_epi8(v, zmm_shuf); + v = _mm512_permutexvar_epi64(zmm_perm, v); + _mm256_storeu_si256((__m256i*)dst_u, _mm512_castsi512_si256(v)); + _mm256_storeu_si256((__m256i*)dst_v, _mm512_extracti64x4_epi64(v, 1)); + dst_u += 32; + dst_v += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i idx = + _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)(32 - width))); + __m512i v = _mm512_maskz_loadu_epi16(k, src_uv); + v = _mm512_permutexvar_epi16(idx, v); + _mm256_mask_storeu_epi8(dst_u, k, _mm512_cvtepi16_epi8(v)); + _mm256_mask_storeu_epi8(dst_v, k, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8))); + } + _mm256_zeroupper(); +} +#endif // HAS_MIRRORSPLITUVROW_AVX512BW + #ifdef HAS_MIRRORUVROW_AVX512BW LIBYUV_TARGET_AVX512BW void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { const __m512i zmm_idx = _mm512_set_epi16( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); - src_uv += width * 2; - while (width > 0) { - src_uv -= 64; - __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_uv); + const uint8_t* s = src_uv + (ptrdiff_t)width * 2; + while (width >= 32) { + s -= 64; + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)s); zmm0 = _mm512_permutexvar_epi16(zmm_idx, zmm0); _mm512_storeu_si512((__m512i*)dst_uv, zmm0); dst_uv += 64; width -= 32; } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i idx = + _mm512_sub_epi16(zmm_idx, _mm512_set1_epi16((short)(32 - width))); + __m512i zmm0 = _mm512_maskz_loadu_epi16(k, src_uv); + _mm512_mask_storeu_epi16(dst_uv, k, _mm512_permutexvar_epi16(idx, zmm0)); + } _mm256_zeroupper(); } #endif // HAS_MIRRORUVROW_AVX512BW @@ -2500,13 +2588,18 @@ void NV21ToARGBRow_AVX2(const uint8_t* src_y, } #endif // HAS_NV21TOARGBROW_AVX2 -#if defined(HAS_I422TORGB24ROW_AVX2) || defined(HAS_I422TORGB24ROW_AVX512BW) +#if defined(HAS_I422TORGB24ROW_AVX2) || \ + defined(HAS_I422TORGB24ROW_AVX512BW) || \ + defined(HAS_ARGBTORGB24ROW_AVX2) || \ + defined(HAS_ARGBTORGB24ROW_AVX512BW) static const uint8_t kShuffleMaskARGBToRGB24[2][16] = { {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u}, {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u}}; +#endif +#if defined(HAS_I422TORGB24ROW_AVX2) LIBYUV_TARGET_AVX2 void I422ToRGB24Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, @@ -3016,53 +3109,56 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, } #endif // HAS_I422TOAR30ROW_AVX512BW -#ifdef HAS_ARGBBLENDROW_SSSE3 +#ifdef HAS_ARGBBLENDROW_AVX2 static const uint8_t kShuffleAlpha[16] = {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80}; -void ARGBBlendRow_SSSE3(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - __m128i shuff = _mm_loadu_si128((const __m128i*)kShuffleAlpha); - __m128i xmm_1 = _mm_set1_epi16(1); - __m128i mask_lo = _mm_set1_epi16(0x00ff); - __m128i mask_hi = _mm_set1_epi16((short)0xff00); - __m128i mask_alpha = _mm_set1_epi32((int)0xff000000); +LIBYUV_TARGET_AVX2 +void ARGBBlendRow_AVX2(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + __m256i shuff = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kShuffleAlpha)); + __m256i ymm_1 = _mm256_set1_epi16(1); + __m256i mask_lo = _mm256_set1_epi16(0x00ff); + __m256i mask_hi = _mm256_set1_epi16((short)0xff00); + __m256i mask_alpha = _mm256_set1_epi32((int)0xff000000); - while (width >= 4) { - __m128i xmm_s0 = _mm_loadu_si128((const __m128i*)src_argb); - __m128i xmm_s1 = _mm_loadu_si128((const __m128i*)src_argb1); - src_argb += 16; - src_argb1 += 16; + while (width >= 8) { + __m256i ymm_s0 = _mm256_loadu_si256((const __m256i*)src_argb0); + __m256i ymm_s1 = _mm256_loadu_si256((const __m256i*)src_argb1); + src_argb0 += 32; + src_argb1 += 32; - __m128i xmm_a = _mm_xor_si128(xmm_s0, mask_alpha); - xmm_a = _mm_shuffle_epi8(xmm_a, shuff); - xmm_a = _mm_add_epi16(xmm_a, xmm_1); + __m256i ymm_a = _mm256_xor_si256(ymm_s0, mask_alpha); + ymm_a = _mm256_shuffle_epi8(ymm_a, shuff); + ymm_a = _mm256_add_epi16(ymm_a, ymm_1); - __m128i xmm_s1_lo = _mm_and_si128(xmm_s1, mask_lo); - __m128i xmm_s1_hi = _mm_srli_epi16(xmm_s1, 8); + __m256i ymm_s1_lo = _mm256_and_si256(ymm_s1, mask_lo); + __m256i ymm_s1_hi = _mm256_srli_epi16(ymm_s1, 8); - xmm_s1_lo = _mm_mullo_epi16(xmm_s1_lo, xmm_a); - xmm_s1_hi = _mm_mullo_epi16(xmm_s1_hi, xmm_a); + ymm_s1_lo = _mm256_mullo_epi16(ymm_s1_lo, ymm_a); + ymm_s1_hi = _mm256_mullo_epi16(ymm_s1_hi, ymm_a); - xmm_s1_lo = _mm_srli_epi16(xmm_s1_lo, 8); - xmm_s1_hi = _mm_and_si128(xmm_s1_hi, mask_hi); + ymm_s1_lo = _mm256_srli_epi16(ymm_s1_lo, 8); + ymm_s1_hi = _mm256_and_si256(ymm_s1_hi, mask_hi); - __m128i res = _mm_or_si128(xmm_s0, mask_alpha); - res = _mm_adds_epu8(res, xmm_s1_lo); - res = _mm_adds_epu8(res, xmm_s1_hi); + __m256i res = _mm256_or_si256(ymm_s0, mask_alpha); + res = _mm256_adds_epu8(res, ymm_s1_lo); + res = _mm256_adds_epu8(res, ymm_s1_hi); - _mm_storeu_si128((__m128i*)dst_argb, res); - dst_argb += 16; - width -= 4; + _mm256_storeu_si256((__m256i*)dst_argb, res); + dst_argb += 32; + width -= 8; } + _mm256_zeroupper(); while (width > 0) { - uint32_t s0 = *(const uint32_t*)src_argb; + uint32_t s0 = *(const uint32_t*)src_argb0; uint32_t s1 = *(const uint32_t*)src_argb1; - src_argb += 4; + src_argb0 += 4; src_argb1 += 4; uint32_t a = 256 - (s0 >> 24); @@ -3079,7 +3175,7 @@ void ARGBBlendRow_SSSE3(const uint8_t* src_argb, --width; } } -#endif // HAS_ARGBBLENDROW_SSSE3 +#endif // HAS_ARGBBLENDROW_AVX2 #ifdef HAS_BLENDPLANEROW_SSSE3 void BlendPlaneRow_SSSE3(const uint8_t* src0, @@ -3423,9 +3519,101 @@ void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, #endif // HAS_RGB24MIRRORROW_AVX512VBMI // =========================================================================== -// Module 1: AYUV to Planar Y / UV / VU (AVX512VBMI) +// Module 1: AYUV to Planar Y / UV / VU (AVX2 / AVX512VBMI) // =========================================================================== +#ifdef HAS_AYUVTOYROW_AVX2 +LIBYUV_TARGET_AVX2 +void AYUVToYRow_AVX2(const uint8_t* src_ayuv, uint8_t* dst_y, int width) { + const __m256i ymm_mask = _mm256_set1_epi32(0x000000ff); + while (width >= 16) { + __m256i ymm0 = _mm256_and_si256( + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_ayuv), 16), + ymm_mask); + __m256i ymm1 = _mm256_and_si256( + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)(src_ayuv + 32)), + 16), + ymm_mask); + src_ayuv += 64; + ymm0 = _mm256_permute4x64_epi64(_mm256_packus_epi32(ymm0, ymm1), 0xd8); + __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0), + _mm256_extracti128_si256(ymm0, 1)); + _mm_storeu_si128((__m128i*)dst_y, out); + dst_y += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_AYUVTOYROW_AVX2 + +#if defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2) +static const uint8_t kShufAYUVToUV16_Win[16] = { + 1, 5, 0, 4, 9, 13, 8, 12, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + +static const uint8_t kShufAYUVToVU16_Win[16] = { + 0, 4, 1, 5, 8, 12, 9, 13, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + +LIBYUV_TARGET_AVX2 +static inline void AYUVToUVOrVURow_Win_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + const uint8_t* shuf_table, + int width) { + const __m256i ymm_shuf = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuf_table)); + const __m256i ymm_0101 = _mm256_set1_epi16(0x0101); + const __m256i ymm_zero = _mm256_setzero_si256(); + while (width >= 16) { + __m256i s00 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)src_ayuv), ymm_shuf); + __m256i s01 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ayuv + 32)), ymm_shuf); + __m256i s10 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ayuv + src_stride_ayuv)), + ymm_shuf); + __m256i s11 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ayuv + src_stride_ayuv + 32)), + ymm_shuf); + src_ayuv += 64; + __m256i p0 = + _mm256_maddubs_epi16(_mm256_unpacklo_epi64(s00, s01), ymm_0101); + __m256i p1 = + _mm256_maddubs_epi16(_mm256_unpacklo_epi64(s10, s11), ymm_0101); + p0 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(p0, p1), 1), + ymm_zero); + p0 = _mm256_permute4x64_epi64(p0, 0xd8); + __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(p0), + _mm256_extracti128_si256(p0, 1)); + _mm_storeu_si128((__m128i*)dst_uv, out); + dst_uv += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif + +#ifdef HAS_AYUVTOUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void AYUVToUVRow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + AYUVToUVOrVURow_Win_AVX2(src_ayuv, src_stride_ayuv, dst_uv, + kShufAYUVToUV16_Win, width); +} +#endif // HAS_AYUVTOUVROW_AVX2 + +#ifdef HAS_AYUVTOVUROW_AVX2 +LIBYUV_TARGET_AVX2 +void AYUVToVURow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + AYUVToUVOrVURow_Win_AVX2(src_ayuv, src_stride_ayuv, dst_vu, + kShufAYUVToVU16_Win, width); +} +#endif // HAS_AYUVTOVUROW_AVX2 + #ifdef HAS_AYUVTOYROW_AVX512VBMI LIBYUV_TARGET_AVX512VBMI void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width) { @@ -3548,9 +3736,34 @@ void AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, #endif // HAS_AYUVTOVUROW_AVX512VBMI // =========================================================================== -// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX512BW) +// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX2 / AVX512BW) // =========================================================================== +#ifdef HAS_AR30TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void AR30ToARGBRow_AVX2(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width) { + const __m256i m0 = _mm256_set1_epi32((int)0xc0000000u); + const __m256i m1 = _mm256_set1_epi32(0x300000ff); + const __m256i m2 = _mm256_set1_epi32(0x0c00ff00); + const __m256i m3 = _mm256_set1_epi32(0x03ff0000); + while (width >= 8) { + __m256i v = _mm256_loadu_si256((const __m256i*)src_ar30); + src_ar30 += 32; + __m256i out = _mm256_or_si256( + _mm256_or_si256(_mm256_and_si256(v, m0), + _mm256_and_si256(_mm256_srli_epi32(v, 2), m1)), + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v, 4), m2), + _mm256_and_si256(_mm256_srli_epi32(v, 6), m3))); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_AR30TOARGBROW_AVX2 + #ifdef HAS_AR30TOARGBROW_AVX512BW LIBYUV_TARGET_AVX512BW void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, @@ -3586,6 +3799,36 @@ void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, } #endif // HAS_AR30TOARGBROW_AVX512BW +#ifdef HAS_AR30TOABGRROW_AVX2 +LIBYUV_TARGET_AVX2 +void AR30ToABGRRow_AVX2(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width) { + static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7, + 10, 9, 8, 11, 14, 13, 12, 15}; + const __m256i m0 = _mm256_set1_epi32((int)0xc0000000u); + const __m256i m1 = _mm256_set1_epi32(0x300000ff); + const __m256i m2 = _mm256_set1_epi32(0x0c00ff00); + const __m256i m3 = _mm256_set1_epi32(0x03ff0000); + const __m256i shuf = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kShufSwapRB16)); + while (width >= 8) { + __m256i v = _mm256_loadu_si256((const __m256i*)src_ar30); + src_ar30 += 32; + __m256i out = _mm256_or_si256( + _mm256_or_si256(_mm256_and_si256(v, m0), + _mm256_and_si256(_mm256_srli_epi32(v, 2), m1)), + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v, 4), m2), + _mm256_and_si256(_mm256_srli_epi32(v, 6), m3))); + out = _mm256_shuffle_epi8(out, shuf); + _mm256_storeu_si256((__m256i*)dst_abgr, out); + dst_abgr += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_AR30TOABGRROW_AVX2 + #ifdef HAS_AR30TOABGRROW_AVX512BW LIBYUV_TARGET_AVX512BW void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, @@ -3627,6 +3870,29 @@ void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, } #endif // HAS_AR30TOABGRROW_AVX512BW +#ifdef HAS_AR30TOAB30ROW_AVX2 +LIBYUV_TARGET_AVX2 +void AR30ToAB30Row_AVX2(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width) { + const __m256i m0 = _mm256_set1_epi32((int)0xc00ffc00u); + const __m256i m1 = _mm256_set1_epi32(0x3ff00000); + const __m256i m2 = _mm256_set1_epi32(0x000003ff); + while (width >= 8) { + __m256i v = _mm256_loadu_si256((const __m256i*)src_ar30); + src_ar30 += 32; + __m256i out = _mm256_or_si256( + _mm256_and_si256(v, m0), + _mm256_or_si256(_mm256_and_si256(_mm256_slli_epi32(v, 20), m1), + _mm256_and_si256(_mm256_srli_epi32(v, 20), m2))); + _mm256_storeu_si256((__m256i*)dst_ab30, out); + dst_ab30 += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_AR30TOAB30ROW_AVX2 + #ifdef HAS_AR30TOAB30ROW_AVX512BW LIBYUV_TARGET_AVX512BW void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, @@ -5177,6 +5443,4653 @@ void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0, } #endif // HAS_ARGBALPHABLENDROW_AVX512BW +// =========================================================================== +// Module 6: AVX2 Row Functions (Minimum AVX2 Coverage) +// =========================================================================== + +#ifdef HAS_ARGBSEPIAROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width) { + static const int8_t kSepiaB[16] = {17, 68, 35, 0, 17, 68, 35, 0, + 17, 68, 35, 0, 17, 68, 35, 0}; + static const int8_t kSepiaG[16] = {22, 88, 45, 0, 22, 88, 45, 0, + 22, 88, 45, 0, 22, 88, 45, 0}; + static const int8_t kSepiaR[16] = {24, 98, 50, 0, 24, 98, 50, 0, + 24, 98, 50, 0, 24, 98, 50, 0}; + if (width >= 8) { + const __m256i vb = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaB)); + const __m256i vg = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaG)); + const __m256i vr = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaR)); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb); + __m256i b = _mm256_maddubs_epi16(v0, vb); + __m256i g = _mm256_maddubs_epi16(v0, vg); + __m256i r = _mm256_maddubs_epi16(v0, vr); + __m256i bg = _mm256_srli_epi16(_mm256_hadd_epi16(b, g), 7); + __m256i rr = _mm256_srli_epi16(_mm256_hadd_epi16(r, r), 7); + bg = _mm256_packus_epi16(bg, bg); + rr = _mm256_packus_epi16(rr, rr); + bg = _mm256_unpacklo_epi8(bg, _mm256_shuffle_epi32(bg, 0x01)); + __m256i a = _mm256_srli_epi32(v0, 24); + a = _mm256_packus_epi16(a, a); + a = _mm256_packus_epi16(a, a); + __m256i ra = _mm256_unpacklo_epi8(rr, a); + __m256i out = _mm256_unpacklo_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBSepiaRow_C(dst_argb, width); + } +} +#endif // HAS_ARGBSEPIAROW_AVX2 + +#ifdef HAS_ARGBCOLORMATRIXROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + static const int8_t kShufTranspose4x4[16] = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; + if (width >= 8) { + const __m256i m = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)matrix_argb)); + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufTranspose4x4)); + const __m256i m0 = _mm256_shuffle_epi32(m, 0x00); + const __m256i m1 = _mm256_shuffle_epi32(m, 0x55); + const __m256i m2 = _mm256_shuffle_epi32(m, 0xaa); + const __m256i m3 = _mm256_shuffle_epi32(m, 0xff); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + __m256i p01 = _mm256_hadds_epi16(_mm256_maddubs_epi16(v0, m0), + _mm256_maddubs_epi16(v0, m1)); + __m256i p23 = _mm256_hadds_epi16(_mm256_maddubs_epi16(v0, m2), + _mm256_maddubs_epi16(v0, m3)); + p01 = _mm256_srai_epi16(p01, 6); + p23 = _mm256_srai_epi16(p23, 6); + __m256i out = _mm256_shuffle_epi8(_mm256_packus_epi16(p01, p23), shuf); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBColorMatrixRow_C(src_argb, dst_argb, matrix_argb, width); + } +} +#endif // HAS_ARGBCOLORMATRIXROW_AVX2 + +#ifdef HAS_ARGBQUANTIZEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBQuantizeRow_AVX2(uint8_t* dst_argb, + int scale, + int interval_size, + int interval_offset, + int width) { + if (width >= 8) { + const __m256i v_scale = _mm256_set1_epi16((short)scale); + const __m256i v_size = _mm256_set1_epi16((short)interval_size); + const __m256i v_off = _mm256_set1_epi16((short)interval_offset); + const __m256i zero = _mm256_setzero_si256(); + const __m256i alpha_mask = _mm256_set1_epi32((int)0xff000000u); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb); + __m256i a = _mm256_and_si256(v0, alpha_mask); + __m256i lo = _mm256_unpacklo_epi8(v0, zero); + __m256i hi = _mm256_unpackhi_epi8(v0, zero); + lo = _mm256_add_epi16( + _mm256_mullo_epi16(_mm256_mulhi_epu16(lo, v_scale), v_size), v_off); + hi = _mm256_add_epi16( + _mm256_mullo_epi16(_mm256_mulhi_epu16(hi, v_scale), v_size), v_off); + __m256i out = _mm256_or_si256( + _mm256_andnot_si256(alpha_mask, _mm256_packus_epi16(lo, hi)), a); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBQuantizeRow_C(dst_argb, scale, interval_size, interval_offset, width); + } +} +#endif // HAS_ARGBQUANTIZEROW_AVX2 + +#ifdef HAS_ARGBSHADEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBShadeRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + uint32_t value) { + if (width >= 8) { + __m256i v_val = _mm256_set1_epi32((int)value); + v_val = _mm256_unpacklo_epi8(v_val, v_val); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + __m256i lo = _mm256_unpacklo_epi8(v0, v0); + __m256i hi = _mm256_unpackhi_epi8(v0, v0); + lo = _mm256_srli_epi16(_mm256_mulhi_epu16(lo, v_val), 8); + hi = _mm256_srli_epi16(_mm256_mulhi_epu16(hi, v_val), 8); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_packus_epi16(lo, hi)); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBShadeRow_C(src_argb, dst_argb, width, value); + } +} +#endif // HAS_ARGBSHADEROW_AVX2 + +#ifdef HAS_SOBELXROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelXRow_AVX2(const uint8_t* src_y0, + const uint8_t* src_y1, + const uint8_t* src_y2, + uint8_t* dst_sobelx, + int width) { + while (width > 0) { + __m256i a0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y0)); + __m256i b0 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 2))); + __m256i a1 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y1)); + __m256i b1 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 2))); + __m256i a2 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y2)); + __m256i b2 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y2 + 2))); + __m256i d0 = _mm256_sub_epi16(a0, b0); + __m256i d1 = _mm256_sub_epi16(a1, b1); + __m256i d2 = _mm256_sub_epi16(a2, b2); + __m256i sum = + _mm256_add_epi16(_mm256_add_epi16(d0, d2), _mm256_add_epi16(d1, d1)); + sum = _mm256_abs_epi16(sum); + sum = _mm256_permute4x64_epi64(_mm256_packus_epi16(sum, sum), 0xd8); + _mm_storeu_si128((__m128i*)dst_sobelx, _mm256_castsi256_si128(sum)); + src_y0 += 16; + src_y1 += 16; + src_y2 += 16; + dst_sobelx += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELXROW_AVX2 + +#ifdef HAS_SOBELYROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelYRow_AVX2(const uint8_t* src_y0, + const uint8_t* src_y1, + uint8_t* dst_sobely, + int width) { + while (width > 0) { + __m256i a0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y0)); + __m256i b0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y1)); + __m256i a1 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 1))); + __m256i b1 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 1))); + __m256i a2 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 2))); + __m256i b2 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 2))); + __m256i d0 = _mm256_sub_epi16(a0, b0); + __m256i d1 = _mm256_sub_epi16(a1, b1); + __m256i d2 = _mm256_sub_epi16(a2, b2); + __m256i sum = + _mm256_add_epi16(_mm256_add_epi16(d0, d2), _mm256_add_epi16(d1, d1)); + sum = _mm256_abs_epi16(sum); + sum = _mm256_permute4x64_epi64(_mm256_packus_epi16(sum, sum), 0xd8); + _mm_storeu_si128((__m128i*)dst_sobely, _mm256_castsi256_si128(sum)); + src_y0 += 16; + src_y1 += 16; + dst_sobely += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELYROW_AVX2 + +#ifdef HAS_SOBELROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelRow_AVX2(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + const __m256i v_alpha = _mm256_set1_epi32((int)0xff000000u); + while (width > 0) { + __m128i vx = _mm_loadu_si128((const __m128i*)src_sobelx); + __m128i vy = _mm_loadu_si128((const __m128i*)src_sobely); + __m256i s16 = _mm256_cvtepu8_epi16(_mm_adds_epu8(vx, vy)); + s16 = _mm256_or_si256(s16, _mm256_slli_epi16(s16, 8)); + s16 = _mm256_permute4x64_epi64(s16, 0xd8); + __m256i p0 = _mm256_or_si256(_mm256_unpacklo_epi16(s16, s16), v_alpha); + __m256i p1 = _mm256_or_si256(_mm256_unpackhi_epi16(s16, s16), v_alpha); + _mm256_storeu_si256((__m256i*)dst_argb, p0); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), p1); + src_sobelx += 16; + src_sobely += 16; + dst_argb += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELROW_AVX2 + +#ifdef HAS_SOBELTOPLANEROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_y, + int width) { + while (width > 0) { + __m256i vx = _mm256_loadu_si256((const __m256i*)src_sobelx); + __m256i vy = _mm256_loadu_si256((const __m256i*)src_sobely); + _mm256_storeu_si256((__m256i*)dst_y, _mm256_adds_epu8(vx, vy)); + src_sobelx += 32; + src_sobely += 32; + dst_y += 32; + width -= 32; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELTOPLANEROW_AVX2 + +#ifdef HAS_SOBELXYROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelXYRow_AVX2(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + const __m256i v_ff00 = _mm256_set1_epi16((short)0xff00u); + while (width > 0) { + __m128i vx = _mm_loadu_si128((const __m128i*)src_sobelx); + __m128i vy = _mm_loadu_si128((const __m128i*)src_sobely); + __m128i vs = _mm_adds_epu8(vx, vy); + __m256i bg = + _mm256_or_si256(_mm256_cvtepu8_epi16(vy), + _mm256_slli_epi16(_mm256_cvtepu8_epi16(vs), 8)); + __m256i ra = _mm256_or_si256(_mm256_cvtepu8_epi16(vx), v_ff00); + bg = _mm256_permute4x64_epi64(bg, 0xd8); + ra = _mm256_permute4x64_epi64(ra, 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi16(bg, ra)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_unpackhi_epi16(bg, ra)); + src_sobelx += 16; + src_sobely += 16; + dst_argb += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELXYROW_AVX2 + +#ifdef HAS_ARGBAFFINEROW_AVX2 +LIBYUV_API +LIBYUV_TARGET_AVX2 +void ARGBAffineRow_AVX2(const uint8_t* src_argb, + int src_argb_stride, + uint8_t* dst_argb, + const float* src_dudv, + int width) { + static const float kRamp[8] = {0.0f, 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, 7.0f}; + float uv_tail[4] = {src_dudv[0], src_dudv[1], src_dudv[2], src_dudv[3]}; + if (width >= 8) { + const __m256 ramp = _mm256_loadu_ps(kRamp); + const __m256 step8 = _mm256_set1_ps(8.0f); + __m256 du = _mm256_set1_ps(uv_tail[2]); + __m256 dv = _mm256_set1_ps(uv_tail[3]); + __m256 u = + _mm256_add_ps(_mm256_set1_ps(uv_tail[0]), _mm256_mul_ps(ramp, du)); + __m256 v = + _mm256_add_ps(_mm256_set1_ps(uv_tail[1]), _mm256_mul_ps(ramp, dv)); + du = _mm256_mul_ps(du, step8); + dv = _mm256_mul_ps(dv, step8); + const __m256i v_stride = _mm256_set1_epi32(src_argb_stride); + while (width >= 8) { + __m256i iu = _mm256_cvttps_epi32(u); + __m256i iv = _mm256_cvttps_epi32(v); + __m256i idx = _mm256_add_epi32(_mm256_slli_epi32(iu, 2), + _mm256_mullo_epi32(iv, v_stride)); + __m256i pix = _mm256_i32gather_epi32((const int*)src_argb, idx, 1); + _mm256_storeu_si256((__m256i*)dst_argb, pix); + u = _mm256_add_ps(u, du); + v = _mm256_add_ps(v, dv); + dst_argb += 32; + width -= 8; + } + uv_tail[0] = _mm256_cvtss_f32(u); + uv_tail[1] = _mm256_cvtss_f32(v); + _mm256_zeroupper(); + } + if (width > 0) { + ARGBAffineRow_C(src_argb, src_argb_stride, dst_argb, uv_tail, width); + } +} +#endif // HAS_ARGBAFFINEROW_AVX2 + +#ifdef HAS_ARGBCOLORTABLEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + if (width >= 8) { + const __m256i m0 = _mm256_set1_epi32(0x000000ff); + const __m256i m1 = _mm256_set1_epi32(0x0000ff00); + const __m256i m2 = _mm256_set1_epi32(0x00ff0000); + const __m256i m3 = _mm256_set1_epi32((int)0xff000000u); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb); + __m256i c0 = + _mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb, + _mm256_and_si256(v0, m0), 4), + m0); + __m256i c1 = _mm256_and_si256( + _mm256_i32gather_epi32((const int*)table_argb, + _mm256_and_si256(_mm256_srli_epi32(v0, 8), m0), + 4), + m1); + __m256i c2 = _mm256_and_si256( + _mm256_i32gather_epi32( + (const int*)table_argb, + _mm256_and_si256(_mm256_srli_epi32(v0, 16), m0), 4), + m2); + __m256i c3 = + _mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb, + _mm256_srli_epi32(v0, 24), 4), + m3); + __m256i out = + _mm256_or_si256(_mm256_or_si256(c0, c1), _mm256_or_si256(c2, c3)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBColorTableRow_C(dst_argb, table_argb, width); + } +} +#endif // HAS_ARGBCOLORTABLEROW_AVX2 + +#ifdef HAS_RGBCOLORTABLEROW_AVX2 +LIBYUV_TARGET_AVX2 +void RGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + if (width >= 8) { + const __m256i m0 = _mm256_set1_epi32(0x000000ff); + const __m256i m1 = _mm256_set1_epi32(0x0000ff00); + const __m256i m2 = _mm256_set1_epi32(0x00ff0000); + const __m256i m3 = _mm256_set1_epi32((int)0xff000000u); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb); + __m256i c0 = + _mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb, + _mm256_and_si256(v0, m0), 4), + m0); + __m256i c1 = _mm256_and_si256( + _mm256_i32gather_epi32((const int*)table_argb, + _mm256_and_si256(_mm256_srli_epi32(v0, 8), m0), + 4), + m1); + __m256i c2 = _mm256_and_si256( + _mm256_i32gather_epi32( + (const int*)table_argb, + _mm256_and_si256(_mm256_srli_epi32(v0, 16), m0), 4), + m2); + __m256i c3 = _mm256_and_si256(v0, m3); + __m256i out = + _mm256_or_si256(_mm256_or_si256(c0, c1), _mm256_or_si256(c2, c3)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + RGBColorTableRow_C(dst_argb, table_argb, width); + } +} +#endif // HAS_RGBCOLORTABLEROW_AVX2 + +#ifdef HAS_ARGBLUMACOLORTABLEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff) { + if (width >= 8) { + const __m256i v_coeff = _mm256_set1_epi32((int)(lumacoeff & 0x00ffffffu)); + const __m256i m_ff = _mm256_set1_epi32(0x000000ff); + const __m256i m_7f00 = _mm256_set1_epi32(0x00007f00); + const __m256i m_18 = _mm256_set1_epi32(0x00000018); + const __m256i m_alpha = _mm256_set1_epi32((int)0xff000000u); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + __m256i y = _mm256_maddubs_epi16(v0, v_coeff); + y = _mm256_hadd_epi16(y, y); + y = _mm256_permute4x64_epi64(y, 0xd8); + y = _mm256_and_si256(_mm256_cvtepu16_epi32(_mm256_castsi256_si128(y)), + m_7f00); + + __m256i b_idx = + _mm256_srli_epi32(_mm256_or_si256(_mm256_and_si256(v0, m_ff), y), 2); + __m256i b_val = _mm256_i32gather_epi32((const int*)luma, b_idx, 4); + __m256i b_sh = _mm256_and_si256(_mm256_slli_epi32(v0, 3), m_18); + b_val = _mm256_and_si256(_mm256_srlv_epi32(b_val, b_sh), m_ff); + + __m256i g_idx = _mm256_srli_epi32( + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v0, 8), m_ff), y), + 2); + __m256i g_val = _mm256_i32gather_epi32((const int*)luma, g_idx, 4); + __m256i g_sh = _mm256_and_si256(_mm256_srli_epi32(v0, 5), m_18); + g_val = _mm256_slli_epi32( + _mm256_and_si256(_mm256_srlv_epi32(g_val, g_sh), m_ff), 8); + + __m256i r_idx = _mm256_srli_epi32( + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v0, 16), m_ff), y), + 2); + __m256i r_val = _mm256_i32gather_epi32((const int*)luma, r_idx, 4); + __m256i r_sh = _mm256_and_si256(_mm256_srli_epi32(v0, 13), m_18); + r_val = _mm256_slli_epi32( + _mm256_and_si256(_mm256_srlv_epi32(r_val, r_sh), m_ff), 16); + + __m256i a_val = _mm256_and_si256(v0, m_alpha); + __m256i out = _mm256_or_si256(_mm256_or_si256(b_val, g_val), + _mm256_or_si256(r_val, a_val)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBLumaColorTableRow_C(src_argb, dst_argb, width, luma, lumacoeff); + } +} +#endif // HAS_ARGBLUMACOLORTABLEROW_AVX2 + +#ifdef HAS_BYTETOFLOATROW_AVX2 +LIBYUV_TARGET_AVX2 +void ByteToFloatRow_AVX2(const uint8_t* src, + float* dst, + float scale, + int width) { + const __m256 v_scale = _mm256_set1_ps(scale); + while (width > 0) { + __m256i i0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)src)); + __m256i i1 = + _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)(src + 8))); + _mm256_storeu_ps(dst, _mm256_mul_ps(_mm256_cvtepi32_ps(i0), v_scale)); + _mm256_storeu_ps(dst + 8, _mm256_mul_ps(_mm256_cvtepi32_ps(i1), v_scale)); + src += 16; + dst += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_BYTETOFLOATROW_AVX2 + +#ifdef HAS_GAUSSROW_AVX2 +LIBYUV_TARGET_AVX2 +void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width) { + static const int8_t kShuf32To16[16] = {0, 1, 4, 5, 8, 9, 12, 13, + -1, -1, -1, -1, -1, -1, -1, -1}; + if (width >= 8) { + const __m256i v_128 = _mm256_set1_epi32(128); + const __m256i v_shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShuf32To16)); + while (width >= 8) { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src); + __m256i s1 = _mm256_loadu_si256((const __m256i*)(src + 1)); + __m256i s2 = _mm256_loadu_si256((const __m256i*)(src + 2)); + __m256i s3 = _mm256_loadu_si256((const __m256i*)(src + 3)); + __m256i s4 = _mm256_loadu_si256((const __m256i*)(src + 4)); + src += 8; + __m256i sum04 = _mm256_add_epi32(s0, s4); + __m256i sum13 = _mm256_slli_epi32(_mm256_add_epi32(s1, s3), 2); + __m256i sum2 = + _mm256_add_epi32(_mm256_slli_epi32(s2, 2), _mm256_slli_epi32(s2, 1)); + __m256i acc = + _mm256_srli_epi32(_mm256_add_epi32(_mm256_add_epi32(sum04, sum13), + _mm256_add_epi32(sum2, v_128)), + 8); + acc = _mm256_permute4x64_epi64(_mm256_shuffle_epi8(acc, v_shuf), 0xd8); + _mm_storeu_si128((__m128i*)dst, _mm256_castsi256_si128(acc)); + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + GaussRow_C(src, dst, width); + } +} +#endif // HAS_GAUSSROW_AVX2 + +#ifdef HAS_GAUSSROW_F32_AVX2 +LIBYUV_TARGET_AVX2 +void GaussRow_F32_AVX2(const float* src, float* dst, int width) { + if (width >= 8) { + const __m256 v4 = _mm256_set1_ps(4.0f); + const __m256 v6 = _mm256_set1_ps(6.0f); + const __m256 v_scale = _mm256_set1_ps(1.0f / 256.0f); + while (width >= 8) { + __m256 acc = _mm256_loadu_ps(src); + acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 1), v4)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 2), v6)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 3), v4)); + acc = _mm256_add_ps(acc, _mm256_loadu_ps(src + 4)); + acc = _mm256_mul_ps(acc, v_scale); + _mm256_storeu_ps(dst, acc); + src += 8; + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + GaussRow_F32_C(src, dst, width); + } +} +#endif // HAS_GAUSSROW_F32_AVX2 + +#ifdef HAS_SETROW_AVX2 +LIBYUV_TARGET_AVX2 +void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width) { + const __m256i v = _mm256_set1_epi8((char)v8); + while (width >= 64) { + _mm256_storeu_si256((__m256i*)dst, v); + _mm256_storeu_si256((__m256i*)(dst + 32), v); + dst += 64; + width -= 64; + } + if (width > 0) { + _mm256_storeu_si256((__m256i*)dst, v); + } + _mm256_zeroupper(); +} +#endif // HAS_SETROW_AVX2 + +#ifdef HAS_ARGBSETROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width) { + const __m256i v = _mm256_set1_epi32((int)v32); + while (width >= 16) { + _mm256_storeu_si256((__m256i*)dst_argb, v); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), v); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + _mm256_storeu_si256((__m256i*)dst_argb, v); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBSETROW_AVX2 + +#ifdef HAS_COPYROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int width) { + while (width > 0) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)src); + __m256i v1 = _mm256_loadu_si256((const __m256i*)(src + 16)); + _mm256_storeu_si256((__m256i*)dst, v0); + _mm256_storeu_si256((__m256i*)(dst + 16), v1); + src += 32; + dst += 32; + width -= 32; + } + _mm256_zeroupper(); +} +#endif // HAS_COPYROW_16_AVX2 + +#ifdef HAS_DETILEROW_AVX2 +LIBYUV_TARGET_AVX2 +void DetileRow_AVX2(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width) { + while (width >= 32) { + __m128i t0 = _mm_loadu_si128((const __m128i*)src); + __m128i t1 = _mm_loadu_si128((const __m128i*)(src + src_tile_stride)); + src += 2 * src_tile_stride; + _mm256_storeu_si256((__m256i*)dst, _mm256_set_m128i(t1, t0)); + dst += 32; + width -= 32; + } + if (width > 0) { + _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); + } + _mm256_zeroupper(); +} +#endif // HAS_DETILEROW_AVX2 + +#ifdef HAS_DETILEROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void DetileRow_16_AVX2(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width) { + while (width >= 32) { + __m256i t0 = _mm256_loadu_si256((const __m256i*)src); + __m256i t1 = _mm256_loadu_si256((const __m256i*)(src + src_tile_stride)); + src += 2 * src_tile_stride; + _mm256_storeu_si256((__m256i*)dst, t0); + _mm256_storeu_si256((__m256i*)(dst + 16), t1); + dst += 32; + width -= 32; + } + if (width > 0) { + _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); + } + _mm256_zeroupper(); +} +#endif // HAS_DETILEROW_16_AVX2 + +#ifdef HAS_DETILESPLITUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void DetileSplitUVRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kDeinterlaceUV_Win[16] = {0, 2, 4, 6, 8, 10, 12, 14, + 1, 3, 5, 7, 9, 11, 13, 15}; + const __m128i shuf128 = _mm_loadu_si128((const __m128i*)kDeinterlaceUV_Win); + const __m256i shuf256 = _mm256_broadcastsi128_si256(shuf128); + while (width >= 32) { + __m128i t0 = _mm_loadu_si128((const __m128i*)src_uv); + __m128i t1 = _mm_loadu_si128((const __m128i*)(src_uv + src_tile_stride)); + src_uv += 2 * src_tile_stride; + __m256i v = _mm256_permute4x64_epi64( + _mm256_shuffle_epi8(_mm256_set_m128i(t1, t0), shuf256), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(v)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(v, 1)); + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + __m128i v = + _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)src_uv), shuf128); + _mm_storel_epi64((__m128i*)dst_u, v); + _mm_storel_epi64((__m128i*)dst_v, _mm_unpackhi_epi64(v, v)); + } + _mm256_zeroupper(); +} +#endif // HAS_DETILESPLITUVROW_AVX2 + +#if defined(__x86_64__) || defined(_M_X64) + +#ifdef HAS_ARGB1555TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width) { + const __m512i zmm_f800 = _mm512_set1_epi16((short)0xf800); + const __m512i zmm_03e0 = _mm512_set1_epi16(0x03e0); + const __m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00); + const __m512i zmm_0108 = _mm512_set1_epi16(0x0108); + const __m512i zmm_4200 = _mm512_set1_epi16(0x4200); + while (width >= 32) { + __m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb1555); + src_argb1555 += 64; + __m512i r = _mm512_mulhi_epu16( + _mm512_and_si512(_mm512_slli_epi16(v0, 1), zmm_f800), zmm_0108); + __m512i b = _mm512_mulhi_epu16(_mm512_slli_epi16(v0, 11), zmm_0108); + __m512i rb = _mm512_or_si512(_mm512_slli_epi16(r, 8), b); + __m512i g = _mm512_mulhi_epu16(_mm512_and_si512(v0, zmm_03e0), zmm_4200); + __m512i a = _mm512_and_si512(_mm512_srai_epi16(v0, 8), zmm_ff00); + __m512i ga = _mm512_or_si512(g, a); + __m512i lo = _mm512_unpacklo_epi8(rb, ga); + __m512i hi = _mm512_unpackhi_epi8(rb, ga); + __m512i out0 = _mm512_shuffle_i64x2(lo, hi, 0x44); + __m512i out1 = _mm512_shuffle_i64x2(lo, hi, 0xee); + out0 = _mm512_shuffle_i64x2(out0, out0, 0xd8); + out1 = _mm512_shuffle_i64x2(out1, out1, 0xd8); + _mm512_storeu_si512((__m512i*)dst_argb, out0); + _mm512_storeu_si512((__m512i*)(dst_argb + 64), out1); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1ULL << width) - 1ULL); + __m512i v0 = _mm512_maskz_loadu_epi16(k, src_argb1555); + __m512i r = _mm512_mulhi_epu16( + _mm512_and_si512(_mm512_slli_epi16(v0, 1), zmm_f800), zmm_0108); + __m512i b = _mm512_mulhi_epu16(_mm512_slli_epi16(v0, 11), zmm_0108); + __m512i rb = _mm512_or_si512(_mm512_slli_epi16(r, 8), b); + __m512i g = _mm512_mulhi_epu16(_mm512_and_si512(v0, zmm_03e0), zmm_4200); + __m512i a = _mm512_and_si512(_mm512_srai_epi16(v0, 8), zmm_ff00); + __m512i ga = _mm512_or_si512(g, a); + __m512i lo = _mm512_unpacklo_epi8(rb, ga); + __m512i hi = _mm512_unpackhi_epi8(rb, ga); + __m512i out0 = _mm512_shuffle_i64x2(lo, hi, 0x44); + __m512i out1 = _mm512_shuffle_i64x2(lo, hi, 0xee); + out0 = _mm512_shuffle_i64x2(out0, out0, 0xd8); + out1 = _mm512_shuffle_i64x2(out1, out1, 0xd8); + if (width > 16) { + _mm512_storeu_si512((__m512i*)dst_argb, out0); + _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(k >> 16), out1); + } else { + _mm512_mask_storeu_epi32(dst_argb, (__mmask16)k, out0); + } + } + _mm256_zeroupper(); +} +#endif // HAS_ARGB1555TOARGBROW_AVX512BW + +#ifdef HAS_ARGBTORGB565DITHERROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width) { + __m128i d128 = _mm_cvtsi32_si128((int)dither4); + d128 = _mm_unpacklo_epi8(d128, d128); + d128 = _mm_unpacklo_epi16(d128, d128); + const __m512i zmm_dither = _mm512_broadcast_i32x4(d128); + const __m512i zmm_001f = _mm512_set1_epi32(0x0000001f); + const __m512i zmm_07e0 = _mm512_set1_epi32(0x000007e0); + const __m512i zmm_f800 = _mm512_set1_epi32(0x0000f800); + while (width >= 16) { + __m512i v0 = _mm512_adds_epu8(_mm512_loadu_si512((const __m512i*)src_argb), + zmm_dither); + src_argb += 64; + __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f); + __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0); + __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800); + __m256i rgb16 = + _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r)); + _mm256_storeu_si256((__m256i*)dst_rgb, rgb16); + dst_rgb += 32; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v0 = + _mm512_adds_epu8(_mm512_maskz_loadu_epi32(k, src_argb), zmm_dither); + __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f); + __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0); + __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800); + __m256i rgb16 = + _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r)); + _mm256_mask_storeu_epi16(dst_rgb, k, rgb16); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTORGB565DITHERROW_AVX512BW + +#ifdef HAS_ARGBTORGB565ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToRGB565Row_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + const __m512i zmm_001f = _mm512_set1_epi32(0x0000001f); + const __m512i zmm_07e0 = _mm512_set1_epi32(0x000007e0); + const __m512i zmm_f800 = _mm512_set1_epi32(0x0000f800); + while (width >= 16) { + __m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb); + src_argb += 64; + __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f); + __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0); + __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800); + __m256i rgb16 = + _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r)); + _mm256_storeu_si256((__m256i*)dst_rgb, rgb16); + dst_rgb += 32; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v0 = _mm512_maskz_loadu_epi32(k, src_argb); + __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f); + __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0); + __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800); + __m256i rgb16 = + _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r)); + _mm256_mask_storeu_epi16(dst_rgb, k, rgb16); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTORGB565ROW_AVX512BW + +#ifdef HAS_MERGEUVROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW +void MergeUVRow_16_AVX512BW(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width) { + const __m128i sh_u = _mm_cvtsi32_si128(16 - depth); + const __m128i sh_v = _mm_cvtsi32_si128(32 - depth); + while (width >= 16) { + __m512i u32 = + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_u)); + __m512i v32 = + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_v)); + src_u += 16; + src_v += 16; + u32 = _mm512_sll_epi16(u32, sh_u); + v32 = _mm512_sll_epi32(v32, sh_v); + _mm512_storeu_si512((__m512i*)dst_uv, _mm512_or_si512(u32, v32)); + dst_uv += 32; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i u32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_u)); + __m512i v32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_v)); + u32 = _mm512_sll_epi16(u32, sh_u); + v32 = _mm512_sll_epi32(v32, sh_v); + _mm512_mask_storeu_epi32(dst_uv, k, _mm512_or_si512(u32, v32)); + } + _mm256_zeroupper(); +} +#endif // HAS_MERGEUVROW_16_AVX512BW + +#ifdef HAS_DIVIDEROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW +void DivideRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + while (width >= 32) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_y); + src_y += 32; + _mm512_storeu_si512((__m512i*)dst_y, _mm512_mulhi_epu16(v, zmm_scale)); + dst_y += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi16(k, src_y); + _mm512_mask_storeu_epi16(dst_y, k, _mm512_mulhi_epu16(v, zmm_scale)); + } + _mm256_zeroupper(); +} +#endif // HAS_DIVIDEROW_16_AVX512BW + +#ifdef HAS_YUY2TOYROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + while (width >= 32) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_yuy2); + src_yuy2 += 64; + _mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(v)); + dst_y += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi16(k, src_yuy2); + _mm256_mask_storeu_epi8(dst_y, k, _mm512_cvtepi16_epi8(v)); + } + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOYROW_AVX512BW + +#ifdef HAS_YUY2TOUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const ptrdiff_t stride = (ptrdiff_t)stride_yuy2; + while (width >= 32) { + __m512i r0 = _mm512_loadu_si512((const __m512i*)src_yuy2); + __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_yuy2 + stride)); + src_yuy2 += 64; + __m512i avg = _mm512_avg_epu8(r0, r1); + _mm_storeu_si128((__m128i*)dst_u, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 8))); + _mm_storeu_si128((__m128i*)dst_v, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 24))); + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i r0 = _mm512_maskz_loadu_epi32(k, src_yuy2); + __m512i r1 = _mm512_maskz_loadu_epi32(k, src_yuy2 + stride); + __m512i avg = _mm512_avg_epu8(r0, r1); + _mm_mask_storeu_epi8(dst_u, k, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 8))); + _mm_mask_storeu_epi8(dst_v, k, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 24))); + } + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOUVROW_AVX512BW + +#ifdef HAS_UYVYTOYROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + while (width >= 32) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_uyvy); + src_uyvy += 64; + _mm256_storeu_si256((__m256i*)dst_y, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8))); + dst_y += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi16(k, src_uyvy); + _mm256_mask_storeu_epi8(dst_y, k, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8))); + } + _mm256_zeroupper(); +} +#endif // HAS_UYVYTOYROW_AVX512BW + +#ifdef HAS_UYVYTOUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const ptrdiff_t stride = (ptrdiff_t)stride_uyvy; + while (width >= 32) { + __m512i r0 = _mm512_loadu_si512((const __m512i*)src_uyvy); + __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_uyvy + stride)); + src_uyvy += 64; + __m512i avg = _mm512_avg_epu8(r0, r1); + _mm_storeu_si128((__m128i*)dst_u, _mm512_cvtepi32_epi8(avg)); + _mm_storeu_si128((__m128i*)dst_v, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 16))); + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i r0 = _mm512_maskz_loadu_epi32(k, src_uyvy); + __m512i r1 = _mm512_maskz_loadu_epi32(k, src_uyvy + stride); + __m512i avg = _mm512_avg_epu8(r0, r1); + _mm_mask_storeu_epi8(dst_u, k, _mm512_cvtepi32_epi8(avg)); + _mm_mask_storeu_epi8(dst_v, k, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 16))); + } + _mm256_zeroupper(); +} +#endif // HAS_UYVYTOUVROW_AVX512BW + +#ifdef HAS_ARGBSEPIAROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI +void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) { + const __m512i zero = _mm512_setzero_si512(); + const __m512i c_b = _mm512_set1_epi32(0x00234411); + const __m512i c_g = _mm512_set1_epi32(0x002d5816); + const __m512i c_r = _mm512_set1_epi32(0x00326218); + const __m512i max255 = _mm512_set1_epi32(255); + const __m512i a_mask = _mm512_set1_epi32((int)0xff000000u); + while (width >= 16) { + __m512i v0 = _mm512_loadu_si512((const __m512i*)dst_argb); + __m512i b = _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_b), 7); + __m512i g = _mm512_min_epu32( + _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_g), 7), max255); + __m512i r = _mm512_min_epu32( + _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_r), 7), max255); + __m512i a = _mm512_and_si512(v0, a_mask); + __m512i out = _mm512_or_si512(_mm512_or_si512(b, _mm512_slli_epi32(g, 8)), + _mm512_or_si512(_mm512_slli_epi32(r, 16), a)); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v0 = _mm512_maskz_loadu_epi32(k, dst_argb); + __m512i b = _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_b), 7); + __m512i g = _mm512_min_epu32( + _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_g), 7), max255); + __m512i r = _mm512_min_epu32( + _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_r), 7), max255); + __m512i a = _mm512_and_si512(v0, a_mask); + __m512i out = _mm512_or_si512(_mm512_or_si512(b, _mm512_slli_epi32(g, 8)), + _mm512_or_si512(_mm512_slli_epi32(r, 16), a)); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBSEPIAROW_AVX512VNNI + +#ifdef HAS_ARGBCOLORMATRIXROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI +void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + const __m512i zero = _mm512_setzero_si512(); + const __m512i max255 = _mm512_set1_epi32(255); + const __m128i m128 = _mm_loadu_si128((const __m128i*)matrix_argb); + const __m512i m0 = _mm512_set1_epi32(_mm_cvtsi128_si32(m128)); + const __m512i m1 = + _mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 4))); + const __m512i m2 = + _mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 8))); + const __m512i m3 = + _mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 12))); + while (width >= 16) { + __m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb); + src_argb += 64; + __m512i b = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m0), 6), zero), + max255); + __m512i g = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m1), 6), zero), + max255); + __m512i r = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m2), 6), zero), + max255); + __m512i a = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m3), 6), zero), + max255); + __m512i out = _mm512_or_si512( + _mm512_or_si512(b, _mm512_slli_epi32(g, 8)), + _mm512_or_si512(_mm512_slli_epi32(r, 16), _mm512_slli_epi32(a, 24))); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v0 = _mm512_maskz_loadu_epi32(k, src_argb); + __m512i b = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m0), 6), zero), + max255); + __m512i g = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m1), 6), zero), + max255); + __m512i r = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m2), 6), zero), + max255); + __m512i a = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m3), 6), zero), + max255); + __m512i out = _mm512_or_si512( + _mm512_or_si512(b, _mm512_slli_epi32(g, 8)), + _mm512_or_si512(_mm512_slli_epi32(r, 16), _mm512_slli_epi32(a, 24))); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBCOLORMATRIXROW_AVX512VNNI + +#ifdef HAS_HALFFLOATROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void HalfFloatRow_AVX512BW(const uint16_t* src, + uint16_t* dst, + float scale, + int width) { + const __m512 vscale = _mm512_set1_ps(scale); + const __m512 vmult = _mm512_set1_ps(1.9259299444e-34f); + while (width >= 16) { + __m512i s32 = + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src)); + src += 16; + __m512 f = + _mm512_mul_ps(_mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale), vmult); + __m256i h16 = + _mm512_cvtepi32_epi16(_mm512_srli_epi32(_mm512_castps_si512(f), 13)); + _mm256_storeu_si256((__m256i*)dst, h16); + dst += 16; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src)); + __m512 f = + _mm512_mul_ps(_mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale), vmult); + __m256i h16 = + _mm512_cvtepi32_epi16(_mm512_srli_epi32(_mm512_castps_si512(f), 13)); + _mm256_mask_storeu_epi16(dst, k, h16); + } + _mm256_zeroupper(); +} +#endif // HAS_HALFFLOATROW_AVX512BW + +#ifdef HAS_CONVERT8TO8ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void Convert8To8Row_AVX512BW(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width) { + const __m512i zmm_scale = + _mm512_slli_epi16(_mm512_set1_epi16((short)scale), 8); + const __m512i zmm_bias = _mm512_set1_epi8((char)bias); + const __m512i zmm_zero = _mm512_setzero_si512(); + while (width >= 64) { + __m512i y8 = _mm512_loadu_si512((const __m512i*)src_y); + src_y += 64; + __m512i y_lo = + _mm512_mulhi_epu16(_mm512_unpacklo_epi8(y8, zmm_zero), zmm_scale); + __m512i y_hi = + _mm512_mulhi_epu16(_mm512_unpackhi_epi8(y8, zmm_zero), zmm_scale); + __m512i out8 = _mm512_add_epi8(_mm512_packus_epi16(y_lo, y_hi), zmm_bias); + _mm512_storeu_si512((__m512i*)dst_y, out8); + dst_y += 64; + width -= 64; + } + if (width > 0) { + __mmask64 k = (__mmask64)((1ULL << width) - 1ULL); + __m512i y8 = _mm512_maskz_loadu_epi8(k, src_y); + __m512i y_lo = + _mm512_mulhi_epu16(_mm512_unpacklo_epi8(y8, zmm_zero), zmm_scale); + __m512i y_hi = + _mm512_mulhi_epu16(_mm512_unpackhi_epi8(y8, zmm_zero), zmm_scale); + __m512i out8 = _mm512_add_epi8(_mm512_packus_epi16(y_lo, y_hi), zmm_bias); + _mm512_mask_storeu_epi8(dst_y, k, out8); + } + _mm256_zeroupper(); +} +#endif // HAS_CONVERT8TO8ROW_AVX512BW + +// =========================================================================== +// Extended YUV (I422, I444, YUY2, UYVY, I210, I212, I410, P210, P410) +// to ARGB / RGBA / RGB24 / AR30 (AVX512BW / AVX512VBMI) +// =========================================================================== + +static const uint16_t kSplitQuadWords_ExtWin[32] = { + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, + 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15}; +static const uint64_t kUnpermuteAVX512_ExtWin[8] = {0, 4, 1, 5, 2, 6, 3, 7}; +static const uint16_t kPermAR30_0_Win[32] = { + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, + 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47}; +static const uint16_t kPermAR30_1_Win[32] = { + 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, + 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63}; +static const uint8_t kShuffleYUY2Y_Win[16] = {0, 0, 2, 2, 4, 4, 6, 6, + 8, 8, 10, 10, 12, 12, 14, 14}; +static const uint8_t kShuffleYUY2UV_Win[16] = {1, 3, 1, 3, 5, 7, 5, 7, + 9, 11, 9, 11, 13, 15, 13, 15}; +static const uint8_t kShuffleUYVYY_Win[16] = {1, 1, 3, 3, 5, 5, 7, 7, + 9, 9, 11, 11, 13, 13, 15, 15}; +static const uint8_t kShuffleUYVYUV_Win[16] = {0, 2, 0, 2, 4, 6, 4, 6, + 8, 10, 8, 10, 12, 14, 12, 14}; + +#define YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants) \ + const __m512i zmm_kUVToB = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToB)); \ + const __m512i zmm_kUVToG = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToG)); \ + const __m512i zmm_kUVToR = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToR)); \ + const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kYToRgb)); \ + const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kYBiasToRgb)); \ + const __m512i zmm_128 = _mm512_set1_epi8((char)0x80) + +#define YUVTORGB16_WIN_AVX512BW \ + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \ + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), \ + zmm_kYBiasToRgb); \ + __m512i b16 = \ + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4); \ + __m512i g16 = \ + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)); \ + __m512i r16 = _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4) + +#define YUVTORGB8_WIN_AVX512BW \ + YUVTORGB16_WIN_AVX512BW; \ + __m512i zmm0 = _mm512_packus_epi16(_mm512_srai_epi16(b16, 6), zmm5); \ + __m512i zmm1 = _mm512_packus_epi16(_mm512_srai_epi16(g16, 6), zmm5); \ + __m512i zmm2 = _mm512_packus_epi16(_mm512_srai_epi16(r16, 6), zmm5) + +#define STOREARGB_WIN_AVX512BW(dst_ptr) \ + do { \ + __m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \ + _mm512_unpacklo_epi8(zmm0, zmm1)); \ + __m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \ + _mm512_unpacklo_epi8(zmm2, zmm5)); \ + _mm512_storeu_si512((__m512i*)(dst_ptr), _mm512_unpacklo_epi16(bg, ra)); \ + _mm512_storeu_si512((__m512i*)((dst_ptr) + 64), \ + _mm512_unpackhi_epi16(bg, ra)); \ + } while (0) + +#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \ + do { \ + __m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \ + _mm512_unpacklo_epi8(zmm0, zmm1)); \ + __m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \ + _mm512_unpacklo_epi8(zmm2, zmm5)); \ + _mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \ + _mm512_unpacklo_epi16(bg, ra)); \ + _mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \ + _mm512_unpackhi_epi16(bg, ra)); \ + } while (0) + +#define PACKAR30_WIN_AVX512BW(dst0, dst1) \ + b16 = _mm512_max_epi16(_mm512_min_epi16(b16, zmm_3ff0), zmm_zero); \ + g16 = _mm512_max_epi16(_mm512_min_epi16(g16, zmm_3ff0), zmm_zero); \ + r16 = _mm512_max_epi16(_mm512_min_epi16(r16, zmm_3ff0), zmm_zero); \ + __m512i lo16 = \ + _mm512_or_si512(_mm512_srli_epi16(b16, 4), \ + _mm512_and_si512(_mm512_slli_epi16(g16, 6), zmm_fc00)); \ + __m512i hi16 = \ + _mm512_or_si512(_mm512_or_si512(_mm512_and_si512(r16, zmm_3ff0), \ + _mm512_srli_epi16(g16, 10)), \ + zmm_c000); \ + __m512i dst0 = _mm512_permutex2var_epi16(lo16, zmm_perm0, hi16); \ + __m512i dst1 = _mm512_permutex2var_epi16(lo16, zmm_perm1, hi16) + +#ifdef HAS_I422TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I422ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m128i xu = _mm_loadu_si128((const __m128i*)u_buf); + __m128i xv = _mm_loadu_si128((const __m128i*)v_buf); + u_buf += 16; + v_buf += 16; + __m256i uv256 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + y_buf += 32; + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf); + __m128i xv = _mm_maskz_loadu_epi8(k_uv, v_buf); + __m256i uv256 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I422TOARGBROW_AVX512BW + +#ifdef HAS_I422TORGBAROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I422ToRGBARow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgba, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m128i xu = _mm_loadu_si128((const __m128i*)u_buf); + __m128i xv = _mm_loadu_si128((const __m128i*)v_buf); + u_buf += 16; + v_buf += 16; + __m256i uv256 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + y_buf += 32; + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + __m512i ab = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); + __m512i gr = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); + _mm512_storeu_si512((__m512i*)dst_rgba, _mm512_unpacklo_epi16(ab, gr)); + _mm512_storeu_si512((__m512i*)(dst_rgba + 64), + _mm512_unpackhi_epi16(ab, gr)); + dst_rgba += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf); + __m128i xv = _mm_maskz_loadu_epi8(k_uv, v_buf); + __m256i uv256 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + __m512i ab = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); + __m512i gr = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); + _mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, + _mm512_unpacklo_epi16(ab, gr)); + _mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16), + _mm512_unpackhi_epi16(ab, gr)); + } + _mm256_zeroupper(); +} +#endif // HAS_I422TORGBAROW_AVX512BW + +#ifdef HAS_I444TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I444ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i u16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf)); + __m512i v16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + u_buf += 32; + v_buf += 32; + y_buf += 32; + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf)); + __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf)); + __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf)); + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I444TOARGBROW_AVX512BW + +#ifdef HAS_I444ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + while (width >= 32) { + __m512i u16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf)); + __m512i v16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + __m512i a16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)a_buf)); + u_buf += 32; + v_buf += 32; + y_buf += 32; + a_buf += 32; + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf)); + __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf)); + __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf)); + __m512i a16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, a_buf)); + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I444ALPHATOARGBROW_AVX512BW + +#ifdef HAS_I444TORGB24ROW_AVX512VBMI +static const uint8_t kMaskBG_I444[64] = { + 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, + 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, + 0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, + 0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, + 0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, + 0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77}; +static const uint8_t kMaskDST0_I444[64] = { + 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, + 0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, + 0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, + 0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, + 0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, + 0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a}; +static const uint8_t kMaskDST1_I444[64] = { + 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, + 0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, + 0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +LIBYUV_TARGET_AVX512VBMI +void I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_I444); + const __m512i zmm_mask_DST0 = + _mm512_loadu_si512((const __m512i*)kMaskDST0_I444); + const __m512i zmm_mask_DST1 = + _mm512_loadu_si512((const __m512i*)kMaskDST1_I444); + while (width >= 32) { + __m512i u16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf)); + __m512i v16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + u_buf += 32; + v_buf += 32; + y_buf += 32; + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB16_WIN_AVX512BW; + __m512i zmm0 = _mm512_srai_epi16(b16, 6); + __m512i zmm1 = _mm512_srai_epi16(g16, 6); + __m512i zmm2 = _mm512_srai_epi16(r16, 6); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm1 = _mm512_packus_epi16(zmm1, zmm1); + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); + _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); + _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), + _mm512_castsi512_si256(zmm_dst1)); + dst_rgb24 += 96; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf)); + __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf)); + __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf)); + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB16_WIN_AVX512BW; + __m512i zmm0 = _mm512_srai_epi16(b16, 6); + __m512i zmm1 = _mm512_srai_epi16(g16, 6); + __m512i zmm2 = _mm512_srai_epi16(r16, 6); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm1 = _mm512_packus_epi16(zmm1, zmm1); + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); + int dst_bytes = width * 3; + __mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; + __mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; + _mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0); + _mm256_mask_storeu_epi8(dst_rgb24 + 64, k1, + _mm512_castsi512_si256(zmm_dst1)); + } + _mm256_zeroupper(); +} +#endif // HAS_I444TORGB24ROW_AVX512VBMI + +#ifdef HAS_YUY2TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void YUY2ToARGBRow_AVX512BW(const uint8_t* yuy2_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm_shuf_y = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleYUY2Y_Win)); + const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win)); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i in = _mm512_loadu_si512((const __m512i*)yuy2_buf); + yuy2_buf += 64; + __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y); + __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k2 = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i in = _mm512_maskz_loadu_epi32(k2, yuy2_buf); + __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y); + __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOARGBROW_AVX512BW + +#ifdef HAS_UYVYTOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void UYVYToARGBRow_AVX512BW(const uint8_t* uyvy_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm_shuf_y = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleUYVYY_Win)); + const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleUYVYUV_Win)); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i in = _mm512_loadu_si512((const __m512i*)uyvy_buf); + uyvy_buf += 64; + __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y); + __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k2 = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i in = _mm512_maskz_loadu_epi32(k2, uyvy_buf); + __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y); + __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_UYVYTOARGBROW_AVX512BW + +#ifdef HAS_I210TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m256i u_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2); + __m256i v_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2); + __m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I210TOARGBROW_AVX512BW + +#ifdef HAS_I210TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m256i u_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2); + __m256i v_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2); + __m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_I210TOAR30ROW_AVX512BW + +#ifdef HAS_I210ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + while (width >= 32) { + __m256i u_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2); + __m256i v_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i a16 = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)a_buf), 2); + u_buf += 16; + v_buf += 16; + y_buf += 32; + a_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2); + __m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i a16 = + _mm512_srli_epi16(_mm512_maskz_loadu_epi16((__mmask32)m32, a_buf), 2); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I210ALPHATOARGBROW_AVX512BW + +#ifdef HAS_I212TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I212ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m256i u_s = + _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 4); + __m256i v_s = + _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 4); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 4); + __m256i v_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 4); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I212TOARGBROW_AVX512BW + +#ifdef HAS_I212TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I212ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m256i u_s = + _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 4); + __m256i v_s = + _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 4); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 4); + __m256i v_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 4); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_I212TOAR30ROW_AVX512BW + +#ifdef HAS_I410TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i u_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2); + __m512i v_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 32; + v_buf += 32; + y_buf += 32; + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2); + __m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf); + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I410TOARGBROW_AVX512BW + +#ifdef HAS_I410TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m512i u_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2); + __m512i v_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 32; + v_buf += 32; + y_buf += 32; + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2); + __m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf); + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_I410TOAR30ROW_AVX512BW + +#ifdef HAS_I410ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + while (width >= 32) { + __m512i u_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2); + __m512i v_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i a16 = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)a_buf), 2); + u_buf += 32; + v_buf += 32; + y_buf += 32; + a_buf += 32; + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2); + __m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf); + __m512i a16 = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, a_buf), 2); + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I410ALPHATOARGBROW_AVX512BW + +#ifdef HAS_P210TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void P210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win)); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i zmm3 = _mm512_shuffle_epi8( + _mm512_loadu_si512((const __m512i*)uv_buf), zmm_shuf_uv); + y_buf += 32; + uv_buf += 32; + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i zmm3 = _mm512_shuffle_epi8(_mm512_maskz_loadu_epi32(k_uv, uv_buf), + zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_P210TOARGBROW_AVX512BW + +#ifdef HAS_P210TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void P210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win)); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i zmm3 = _mm512_shuffle_epi8( + _mm512_loadu_si512((const __m512i*)uv_buf), zmm_shuf_uv); + y_buf += 32; + uv_buf += 32; + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i zmm3 = _mm512_shuffle_epi8(_mm512_maskz_loadu_epi32(k_uv, uv_buf), + zmm_shuf_uv); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_P210TOAR30ROW_AVX512BW + +#ifdef HAS_P410TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void P410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i uv0 = _mm512_loadu_si512((const __m512i*)uv_buf); + __m512i uv1 = _mm512_loadu_si512((const __m512i*)(uv_buf + 32)); + y_buf += 32; + uv_buf += 64; + __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)); + __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)); + __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i uv0 = _mm512_maskz_loadu_epi32((__mmask16)m32, uv_buf); + __m512i uv1 = _mm512_maskz_loadu_epi32((__mmask16)(m32 >> 16), uv_buf + 32); + __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)); + __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)); + __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_P410TOARGBROW_AVX512BW + +#ifdef HAS_P410TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void P410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i uv0 = _mm512_loadu_si512((const __m512i*)uv_buf); + __m512i uv1 = _mm512_loadu_si512((const __m512i*)(uv_buf + 32)); + y_buf += 32; + uv_buf += 64; + __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)); + __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)); + __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i uv0 = _mm512_maskz_loadu_epi32((__mmask16)m32, uv_buf); + __m512i uv1 = _mm512_maskz_loadu_epi32((__mmask16)(m32 >> 16), uv_buf + 32); + __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)); + __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)); + __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_P410TOAR30ROW_AVX512BW + +#if defined(HAS_ARGBTORGB24ROW_AVX512BW) || defined(HAS_ARGBTORAWROW_AVX512BW) +LIBYUV_TARGET_AVX512BW +static inline void ARGBToRGBRow_AVX512BW(const uint8_t* src, + uint8_t* dst, + int width, + const uint8_t* shuffler) { + const __m512i zmm_shuf = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuffler)); + const __m512i zmm_p0 = + _mm512_loadu_si512((const __m512i*)kPermdRGB24_0_AVX512BW); + const __m512i zmm_p1 = + _mm512_loadu_si512((const __m512i*)kPermdRGB24_1_AVX512BW); + const __m512i zmm_p2 = + _mm512_loadu_si512((const __m512i*)kPermdRGB24_2_AVX512BW); + + while (width >= 64) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src + 64)); + __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src + 128)); + __m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src + 192)); + src += 256; + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + __m512i out0 = _mm512_permutex2var_epi32(zmm0, zmm_p0, zmm1); + __m512i out1 = _mm512_permutex2var_epi32(zmm1, zmm_p1, zmm2); + __m512i out2 = _mm512_permutex2var_epi32(zmm2, zmm_p2, zmm3); + + _mm512_storeu_si512((__m512i*)dst, out0); + _mm512_storeu_si512((__m512i*)(dst + 64), out1); + _mm512_storeu_si512((__m512i*)(dst + 128), out2); + dst += 192; + width -= 64; + } + + if (width > 0) { + uint64_t m_in = (1ULL << width) - 1ULL; + __m512i zmm0 = _mm512_maskz_loadu_epi32((__mmask16)m_in, src); + __m512i zmm1 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 16), src + 64); + __m512i zmm2 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 32), src + 128); + __m512i zmm3 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 48), src + 192); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + __m512i out0 = _mm512_permutex2var_epi32(zmm0, zmm_p0, zmm1); + __m512i out1 = _mm512_permutex2var_epi32(zmm1, zmm_p1, zmm2); + __m512i out2 = _mm512_permutex2var_epi32(zmm2, zmm_p2, zmm3); + + int bytes = width * 3; + __mmask64 k_out0 = + (bytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << bytes) - 1ULL); + __mmask64 k_out1 = + (bytes <= 64) + ? 0 + : ((bytes >= 128) ? ~(__mmask64)0 + : (__mmask64)((1ULL << (bytes - 64)) - 1ULL)); + __mmask64 k_out2 = + (bytes <= 128) ? 0 : (__mmask64)((1ULL << (bytes - 128)) - 1ULL); + + _mm512_mask_storeu_epi8(dst, k_out0, out0); + _mm512_mask_storeu_epi8(dst + 64, k_out1, out1); + _mm512_mask_storeu_epi8(dst + 128, k_out2, out2); + } + _mm256_zeroupper(); +} +#endif // defined(HAS_ARGBTORGB24ROW_AVX512BW) || + // defined(HAS_ARGBTORAWROW_AVX512BW) + +#ifdef HAS_ARGBTORGB24ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToRGB24Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRGB24[0]); +} +#endif // HAS_ARGBTORGB24ROW_AVX512BW + +#ifdef HAS_ARGBTORAWROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToRAWRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + static const uint8_t kShuffleMaskARGBToRAW_AVX512[16] = { + 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, + 128u}; + ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRAW_AVX512); +} +#endif // HAS_ARGBTORAWROW_AVX512BW + +#ifdef HAS_CONVERT16TO8ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void Convert16To8Row_AVX512BW(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width) { + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + const __m512i zmm_255 = _mm512_set1_epi16(255); + do { + __m512i y0 = _mm512_loadu_si512((const __m512i*)src_y); + __m512i y1 = _mm512_loadu_si512((const __m512i*)(src_y + 32)); + y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255); + y1 = _mm512_min_epu16(_mm512_mulhi_epu16(y1, zmm_scale), zmm_255); + _mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(y0)); + _mm256_storeu_si256((__m256i*)(dst_y + 32), _mm512_cvtepi16_epi8(y1)); + src_y += 64; + dst_y += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_CONVERT16TO8ROW_AVX512BW + +#ifdef HAS_COPYROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + while (width >= 128) { + __m512i z0 = _mm512_loadu_si512((const __m512i*)src); + __m512i z1 = _mm512_loadu_si512((const __m512i*)(src + 64)); + _mm512_storeu_si512((__m512i*)dst, z0); + _mm512_storeu_si512((__m512i*)(dst + 64), z1); + src += 128; + dst += 128; + width -= 128; + } + if (width >= 64) { + __m512i z0 = _mm512_loadu_si512((const __m512i*)src); + _mm512_storeu_si512((__m512i*)dst, z0); + src += 64; + dst += 64; + width -= 64; + } + if (width > 0) { + __mmask64 k = (__mmask64)((1ULL << width) - 1ULL); + __m512i z0 = _mm512_maskz_loadu_epi8(k, src); + _mm512_mask_storeu_epi8(dst, k, z0); + } + _mm256_zeroupper(); +} +#endif // HAS_COPYROW_AVX512BW + +#ifdef HAS_HALFROW_16TO8_AVX512BW +LIBYUV_TARGET_AVX512BW +void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + const __m512i zmm_255 = _mm512_set1_epi16(255); + do { + __m512i r0 = _mm512_loadu_si512((const __m512i*)src_uv); + __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); + __m512i s0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride)); + __m512i s1 = + _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride + 32)); + r0 = _mm512_avg_epu16(r0, s0); + r1 = _mm512_avg_epu16(r1, s1); + r0 = _mm512_min_epu16(_mm512_mulhi_epu16(r0, zmm_scale), zmm_255); + r1 = _mm512_min_epu16(_mm512_mulhi_epu16(r1, zmm_scale), zmm_255); + _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(r0)); + _mm256_storeu_si256((__m256i*)(dst_uv + 32), _mm512_cvtepi16_epi8(r1)); + src_uv += 64; + dst_uv += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFROW_16TO8_AVX512BW + +#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW +LIBYUV_TARGET_AVX512BW +void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const __m512i zmm_mask16 = _mm512_set1_epi32(0x0000ffff); + const __m512i zmm_2 = _mm512_set1_epi32(2); + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + const __m512i zmm_255 = _mm512_set1_epi16(255); + do { + __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv); + __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); + __m512i t0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride)); + __m512i t1 = + _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride + 32)); + __m512i sum0 = _mm512_add_epi32(_mm512_and_si512(s0, zmm_mask16), + _mm512_srli_epi32(s0, 16)); + sum0 = _mm512_add_epi32(sum0, _mm512_and_si512(t0, zmm_mask16)); + sum0 = _mm512_add_epi32(sum0, _mm512_srli_epi32(t0, 16)); + sum0 = _mm512_srli_epi32(_mm512_add_epi32(sum0, zmm_2), 2); + + __m512i sum1 = _mm512_add_epi32(_mm512_and_si512(s1, zmm_mask16), + _mm512_srli_epi32(s1, 16)); + sum1 = _mm512_add_epi32(sum1, _mm512_and_si512(t1, zmm_mask16)); + sum1 = _mm512_add_epi32(sum1, _mm512_srli_epi32(t1, 16)); + sum1 = _mm512_srli_epi32(_mm512_add_epi32(sum1, zmm_2), 2); + + __m512i avg16 = + _mm512_inserti64x4(_mm512_zextsi256_si512(_mm512_cvtepi32_epi16(sum0)), + _mm512_cvtepi32_epi16(sum1), 1); + avg16 = _mm512_min_epu16(_mm512_mulhi_epu16(avg16, zmm_scale), zmm_255); + _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(avg16)); + src_uv += 64; + dst_uv += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFWIDTHROW_16TO8_AVX512BW + +#ifdef HAS_SPLITUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void SplitUVRow_AVX512BW(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + do { + __m512i uv0 = _mm512_loadu_si512((const __m512i*)src_uv); + __m512i uv1 = _mm512_loadu_si512((const __m512i*)(src_uv + 64)); + _mm256_storeu_si256((__m256i*)dst_u, _mm512_cvtepi16_epi8(uv0)); + _mm256_storeu_si256((__m256i*)(dst_u + 32), _mm512_cvtepi16_epi8(uv1)); + _mm256_storeu_si256((__m256i*)dst_v, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8))); + _mm256_storeu_si256((__m256i*)(dst_v + 32), + _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8))); + src_uv += 128; + dst_u += 64; + dst_v += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITUVROW_AVX512BW + +#ifdef HAS_CONVERT8TO16ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void Convert8To16Row_AVX512BW(const uint8_t* src_y, + uint16_t* dst_y, + int bits, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(16 - bits); + do { + __m256i y0_8 = _mm256_loadu_si256((const __m256i*)src_y); + __m256i y1_8 = _mm256_loadu_si256((const __m256i*)(src_y + 32)); + __m512i y0 = _mm512_cvtepu8_epi16(y0_8); + __m512i y1 = _mm512_cvtepu8_epi16(y1_8); + y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)), + shift128); + y1 = _mm512_srl_epi16(_mm512_or_si512(y1, _mm512_slli_epi16(y1, 8)), + shift128); + _mm512_storeu_si512((__m512i*)dst_y, y0); + _mm512_storeu_si512((__m512i*)(dst_y + 32), y1); + src_y += 64; + dst_y += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_CONVERT8TO16ROW_AVX512BW + +#ifdef HAS_J400TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void J400ToARGBRow_AVX512BW(const uint8_t* src_y, + uint8_t* dst_argb, + int width) { + static const uint8_t kShuffleMaskJ400ToARGB_AVX512[64] = { + 0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, + 3u, 3u, 128u, 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, + 6u, 128u, 7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u, + 128u, 10u, 10u, 10u, 128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, + 13u, 13u, 13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u}; + const __m512i zmm_mask = + _mm512_loadu_si512((const __m512i*)kShuffleMaskJ400ToARGB_AVX512); + const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u); + do { + __m512i z0 = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)src_y)); + __m512i z1 = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)(src_y + 16))); + z0 = _mm512_or_si512(_mm512_shuffle_epi8(z0, zmm_mask), zmm_alpha); + z1 = _mm512_or_si512(_mm512_shuffle_epi8(z1, zmm_mask), zmm_alpha); + _mm512_storeu_si512((__m512i*)dst_argb, z0); + _mm512_storeu_si512((__m512i*)(dst_argb + 64), z1); + src_y += 32; + dst_argb += 128; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_J400TOARGBROW_AVX512BW + +#ifdef HAS_MERGEUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void MergeUVRow_AVX512BW(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width) { + do { + __m512i u16 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_u)); + __m512i v16 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_v)); + __m512i uv = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + _mm512_storeu_si512((__m512i*)dst_uv, uv); + src_u += 32; + src_v += 32; + dst_uv += 64; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEUVROW_AVX512BW + +#ifdef HAS_MULTIPLYROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW +void MultiplyRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + do { + __m512i z0 = _mm512_loadu_si512((const __m512i*)src_y); + __m512i z1 = _mm512_loadu_si512((const __m512i*)(src_y + 32)); + z0 = _mm512_mullo_epi16(z0, zmm_scale); + z1 = _mm512_mullo_epi16(z1, zmm_scale); + _mm512_storeu_si512((__m512i*)dst_y, z0); + _mm512_storeu_si512((__m512i*)(dst_y + 32), z1); + src_y += 64; + dst_y += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MULTIPLYROW_16_AVX512BW + +#ifdef HAS_ARGBTOYMATRIXROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + const __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + const __m512i zmm_80 = _mm512_set1_epi8((char)0x80); + const __m512i zmm_one = _mm512_set1_epi16(1); + const __m512i zmm_coeff = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY)); + __m512i zmm_add = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddY)); + + __m512i zmm_bias = _mm512_maddubs_epi16(zmm_coeff, zmm_80); + zmm_bias = _mm512_madd_epi16(zmm_bias, zmm_one); + zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias); + zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias); + + while (width > 0) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); + __m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192)); + + zmm0 = _mm512_xor_si512(zmm0, zmm_80); + zmm1 = _mm512_xor_si512(zmm1, zmm_80); + zmm2 = _mm512_xor_si512(zmm2, zmm_80); + zmm3 = _mm512_xor_si512(zmm3, zmm_80); + + zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one); + zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one); + zmm0 = _mm512_packs_epi32(zmm0, zmm1); + + zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one); + zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one); + zmm2 = _mm512_packs_epi32(zmm2, zmm3); + + zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8); + zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8); + + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); + _mm512_storeu_si512((__m512i*)dst_y, zmm0); + + src_argb += 256; + dst_y += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOYMATRIXROW_AVX512BW + +#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const __m512i zmm_u = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); + const __m512i zmm_v = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + const __m512i zmm_one = _mm512_set1_epi16(1); + const __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + const __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + + while (width > 0) { + __m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); + __m512i s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192)); + + __m512i u0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_u), zmm_one); + __m512i u1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_u), zmm_one); + __m512i u2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_u), zmm_one); + __m512i u3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_u), zmm_one); + __m512i u01 = + _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u0, u1)), + 8); + __m512i u23 = + _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u2, u3)), + 8); + _mm512_storeu_si512( + (__m512i*)dst_u, + _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(u01, u23))); + + __m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one); + __m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one); + __m512i v2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_v), zmm_one); + __m512i v3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_v), zmm_one); + __m512i v01 = + _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v0, v1)), + 8); + __m512i v23 = + _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v2, v3)), + 8); + _mm512_storeu_si512( + (__m512i*)dst_v, + _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(v01, v23))); + + src_argb += 256; + dst_u += 64; + dst_v += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW + +#ifdef HAS_ARGBTOUVMATRIXROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + static const uint32_t kPermdARGBToUV_AVX512BW[16] = { + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15}; + static const uint8_t kShuffleAARRGGBB_AVX512BW[16] = { + 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15}; + const __m512i zmm_u = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); + const __m512i zmm_v = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + const __m512i zmm_0101 = _mm512_set1_epi16(0x0101); + const __m512i zmm_one = _mm512_set1_epi16(1); + const __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_shuf = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512BW)); + const __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + const __m512i zmm_perm_uv = + _mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW); + + while (width >= 32) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i zmm2 = + _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb)); + __m512i zmm3 = + _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64)); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101); + zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101); + zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101); + zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101); + + zmm0 = _mm512_add_epi16(zmm0, zmm2); + zmm1 = _mm512_add_epi16(zmm1, zmm3); + + zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero); + zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero); + + zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); + + zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one); + zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one); + zmm0 = _mm512_packs_epi32(zmm1, zmm2); + zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); + + _mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0)); + _mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1)); + + src_argb += 128; + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + uint32_t mask = (1u << width) - 1u; + __mmask16 k1 = (__mmask16)mask; + __mmask16 k2 = (__mmask16)(mask >> 16); + __mmask16 k3 = ~k1; + __mmask16 k4 = ~k2; + __m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb); + __m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64); + __m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb); + __m512i zmm3 = + _mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64); + + zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0); + zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0); + zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0); + zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101); + zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101); + zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101); + zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101); + + zmm0 = _mm512_add_epi16(zmm0, zmm2); + zmm1 = _mm512_add_epi16(zmm1, zmm3); + + zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero); + zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero); + + zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); + + zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one); + zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one); + zmm0 = _mm512_packs_epi32(zmm1, zmm2); + zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); + + int halfwidth = (width + 1) >> 1; + __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u); + _mm_mask_storeu_epi8(dst_u, k_out, _mm512_castsi512_si128(zmm0)); + _mm_mask_storeu_epi8(dst_v, k_out, _mm512_extracti32x4_epi32(zmm0, 1)); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOUVMATRIXROW_AVX512BW + +#endif // defined(__x86_64__) || defined(_M_X64) + +#ifdef HAS_COPYROW_AVX +LIBYUV_TARGET_AVX +void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) { + do { + __m256i y0 = _mm256_loadu_si256((const __m256i*)src); + __m256i y1 = _mm256_loadu_si256((const __m256i*)(src + 32)); + _mm256_storeu_si256((__m256i*)dst, y0); + _mm256_storeu_si256((__m256i*)(dst + 32), y1); + src += 64; + dst += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_COPYROW_AVX + +#ifdef HAS_DETILETOYUY2_AVX2 +LIBYUV_TARGET_AVX2 +void DetileToYUY2_AVX2(const uint8_t* src_y, + ptrdiff_t src_y_tile_stride, + const uint8_t* src_uv, + ptrdiff_t src_uv_tile_stride, + uint8_t* dst_yuy2, + int width) { + do { + __m128i y0 = _mm_loadu_si128((const __m128i*)src_y); + __m128i y1 = _mm_loadu_si128((const __m128i*)(src_y + src_y_tile_stride)); + __m128i uv0 = _mm_loadu_si128((const __m128i*)src_uv); + __m128i uv1 = + _mm_loadu_si128((const __m128i*)(src_uv + src_uv_tile_stride)); + __m256i y = _mm256_inserti128_si256(_mm256_castsi128_si256(y0), y1, 1); + __m256i uv = _mm256_inserti128_si256(_mm256_castsi128_si256(uv0), uv1, 1); + __m256i lo = _mm256_unpacklo_epi8(y, uv); + __m256i hi = _mm256_unpackhi_epi8(y, uv); + _mm256_storeu_si256((__m256i*)dst_yuy2, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_yuy2 + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_y += 2 * src_y_tile_stride; + src_uv += 2 * src_uv_tile_stride; + dst_yuy2 += 64; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_DETILETOYUY2_AVX2 + +#ifdef HAS_UNPACKMT2T_AVX2 +LIBYUV_TARGET_AVX2 +void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) { + const __m256i mask_00c0 = _mm256_set1_epi16(0x00c0); + for (size_t i = 0; i < size; i += 80) { + __m256i low16 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src)); + __m256i l0 = _mm256_and_si256(_mm256_slli_epi16(low16, 6), mask_00c0); + __m256i l1 = _mm256_and_si256(_mm256_slli_epi16(low16, 4), mask_00c0); + __m256i l2 = _mm256_and_si256(_mm256_slli_epi16(low16, 2), mask_00c0); + __m256i l3 = _mm256_and_si256(low16, mask_00c0); + + __m256i u0 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16))); + __m256i u1 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 32))); + __m256i u2 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 48))); + __m256i u3 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 64))); + + u0 = _mm256_or_si256( + _mm256_or_si256(_mm256_slli_epi16(u0, 8), _mm256_srli_epi16(u0, 2)), + l0); + u1 = _mm256_or_si256( + _mm256_or_si256(_mm256_slli_epi16(u1, 8), _mm256_srli_epi16(u1, 2)), + l1); + u2 = _mm256_or_si256( + _mm256_or_si256(_mm256_slli_epi16(u2, 8), _mm256_srli_epi16(u2, 2)), + l2); + u3 = _mm256_or_si256( + _mm256_or_si256(_mm256_slli_epi16(u3, 8), _mm256_srli_epi16(u3, 2)), + l3); + + _mm256_storeu_si256((__m256i*)dst, u0); + _mm256_storeu_si256((__m256i*)(dst + 16), u1); + _mm256_storeu_si256((__m256i*)(dst + 32), u2); + _mm256_storeu_si256((__m256i*)(dst + 48), u3); + + src += 80; + dst += 64; + } + _mm256_zeroupper(); +} +#endif // HAS_UNPACKMT2T_AVX2 + +#ifdef HAS_ARGBGRAYROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, + 8, 8, 8, 0x80, 12, 12, 12, 0x80}; + const __m256i c_coeff = _mm256_set1_epi32(0x004d961d); + const __m256i c_sub128 = _mm256_set1_epi8((char)0x80); + const __m256i c_one16 = _mm256_set1_epi16(1); + const __m256i c_bias = _mm256_set1_epi32(32896); + const __m256i c_shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufGrayDword)); + const __m256i c_alpha = _mm256_set1_epi32((int)0xff000000u); + + while (width >= 8) { + __m256i s = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + __m256i y16 = _mm256_maddubs_epi16(c_coeff, _mm256_sub_epi8(s, c_sub128)); + __m256i y32 = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_madd_epi16(y16, c_one16), c_bias), 8); + __m256i out = _mm256_or_si256(_mm256_and_si256(s, c_alpha), + _mm256_shuffle_epi8(y32, c_shuf)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + if (width > 0) { + ARGBGrayRow_C(src_argb, dst_argb, width); + } +} +#endif // HAS_ARGBGRAYROW_AVX2 + +#ifdef HAS_MERGERGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeRGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width) { + static const uint8_t kMergeRGBShuf[5][16] = { + {0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u, 10u, 5u}, + {5u, 0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u, 10u}, + {10u, 5u, 0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u}, + {0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, + 0u}, + {0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, + 0u}}; + const __m256i shuf_r = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[0])); + const __m256i shuf_g = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[1])); + const __m256i shuf_b = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[2])); + const __m256i m1 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[3])); + const __m256i m2 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[4])); + + do { + __m256i r = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_r), shuf_r); + __m256i g = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_g), shuf_g); + __m256i b = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_b), shuf_b); + + __m256i c0 = _mm256_blendv_epi8(_mm256_blendv_epi8(r, g, m1), b, m2); + __m256i c1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g, b, m1), r, m2); + __m256i c2 = _mm256_blendv_epi8(_mm256_blendv_epi8(b, r, m1), g, m2); + + _mm256_storeu_si256((__m256i*)dst_rgb, + _mm256_permute2x128_si256(c0, c1, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_rgb + 32), + _mm256_permute2x128_si256(c2, c0, 0x30)); + _mm256_storeu_si256((__m256i*)(dst_rgb + 64), + _mm256_permute2x128_si256(c1, c2, 0x31)); + + src_r += 32; + src_g += 32; + src_b += 32; + dst_rgb += 96; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGERGBROW_AVX2 + +#ifdef HAS_RAWTORGBAROW_AVX2 +LIBYUV_TARGET_AVX2 +void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { + static const uint8_t kShufRAWToRGBA_Lo[16] = {0x80, 2u, 1u, 0u, 0x80, 5u, + 4u, 3u, 0x80, 8u, 7u, 6u, + 0x80, 11u, 10u, 9u}; + static const uint8_t kShufRAWToRGBA_Hi[16] = {0x80, 6u, 5u, 4u, 0x80, 9u, + 8u, 7u, 0x80, 12u, 11u, 10u, + 0x80, 15u, 14u, 13u}; + const __m256i shuf_lo = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufRAWToRGBA_Lo)); + const __m256i shuf_hi = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufRAWToRGBA_Hi)); + const __m256i alpha = _mm256_set1_epi32(0x000000ff); + + do { + __m128i s0 = _mm_loadu_si128((const __m128i*)src_raw); + __m128i s1 = _mm_loadu_si128((const __m128i*)(src_raw + 12)); + __m128i s2 = _mm_loadu_si128((const __m128i*)(src_raw + 20)); + __m128i s3 = _mm_loadu_si128((const __m128i*)(src_raw + 32)); + __m256i y0 = _mm256_inserti128_si256(_mm256_castsi128_si256(s0), s1, 1); + __m256i y1 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1); + y0 = _mm256_or_si256(_mm256_shuffle_epi8(y0, shuf_lo), alpha); + y1 = _mm256_or_si256(_mm256_shuffle_epi8(y1, shuf_hi), alpha); + _mm256_storeu_si256((__m256i*)dst_rgba, y0); + _mm256_storeu_si256((__m256i*)(dst_rgba + 32), y1); + src_raw += 48; + dst_rgba += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_RAWTORGBAROW_AVX2 + +#ifdef HAS_HALFROW_AVX2 +LIBYUV_TARGET_AVX2 +void HalfRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width) { + do { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride)); + _mm256_storeu_si256((__m256i*)dst_uv, _mm256_avg_epu8(r0, r1)); + src_uv += 32; + dst_uv += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFROW_AVX2 + +#ifdef HAS_HALFROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void HalfRow_16_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width) { + do { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride)); + _mm256_storeu_si256((__m256i*)dst_uv, _mm256_avg_epu16(r0, r1)); + src_uv += 16; + dst_uv += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFROW_16_AVX2 + +#ifdef HAS_CONVERT16TO8ROW_AVX2 +LIBYUV_TARGET_AVX2 +void Convert16To8Row_AVX2(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width) { + const __m256i ymm_scale = _mm256_set1_epi16((short)scale); + const __m256i ymm_255 = _mm256_set1_epi16(255); + do { + __m256i y0 = _mm256_loadu_si256((const __m256i*)src_y); + __m256i y1 = _mm256_loadu_si256((const __m256i*)(src_y + 16)); + y0 = _mm256_min_epu16(_mm256_mulhi_epu16(y0, ymm_scale), ymm_255); + y1 = _mm256_min_epu16(_mm256_mulhi_epu16(y1, ymm_scale), ymm_255); + __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(y0, y1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_y, out); + src_y += 32; + dst_y += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_CONVERT16TO8ROW_AVX2 + +#ifdef HAS_HALFROW_16TO8_AVX2 +LIBYUV_TARGET_AVX2 +void HalfRow_16To8_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const __m256i ymm_scale = _mm256_set1_epi16((short)scale); + const __m256i ymm_255 = _mm256_set1_epi16(255); + do { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16)); + __m256i s0 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride)); + __m256i s1 = + _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride + 16)); + r0 = _mm256_min_epu16(_mm256_mulhi_epu16(_mm256_avg_epu16(r0, s0), ymm_scale), + ymm_255); + r1 = _mm256_min_epu16(_mm256_mulhi_epu16(_mm256_avg_epu16(r1, s1), ymm_scale), + ymm_255); + __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(r0, r1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_uv, out); + src_uv += 32; + dst_uv += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFROW_16TO8_AVX2 + +#ifdef HAS_HALFWIDTHROW_16TO8_AVX2 +LIBYUV_TARGET_AVX2 +void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const __m256i ymm_mask16 = _mm256_set1_epi32(0x0000ffff); + const __m256i ymm_2 = _mm256_set1_epi32(2); + const __m256i ymm_scale = _mm256_set1_epi16((short)scale); + const __m256i ymm_255 = _mm256_set1_epi16(255); + do { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i s1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16)); + __m256i t0 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride)); + __m256i t1 = + _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride + 16)); + + __m256i sum0 = _mm256_add_epi32(_mm256_and_si256(s0, ymm_mask16), + _mm256_srli_epi32(s0, 16)); + sum0 = _mm256_add_epi32(sum0, _mm256_and_si256(t0, ymm_mask16)); + sum0 = _mm256_add_epi32(sum0, _mm256_srli_epi32(t0, 16)); + sum0 = _mm256_srli_epi32(_mm256_add_epi32(sum0, ymm_2), 2); + + __m256i sum1 = _mm256_add_epi32(_mm256_and_si256(s1, ymm_mask16), + _mm256_srli_epi32(s1, 16)); + sum1 = _mm256_add_epi32(sum1, _mm256_and_si256(t1, ymm_mask16)); + sum1 = _mm256_add_epi32(sum1, _mm256_srli_epi32(t1, 16)); + sum1 = _mm256_srli_epi32(_mm256_add_epi32(sum1, ymm_2), 2); + + __m256i avg16 = + _mm256_permute4x64_epi64(_mm256_packus_epi32(sum0, sum1), 0xd8); + avg16 = _mm256_min_epu16(_mm256_mulhi_epu16(avg16, ymm_scale), ymm_255); + __m256i out = + _mm256_permute4x64_epi64(_mm256_packus_epi16(avg16, avg16), 0xd8); + _mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(out)); + src_uv += 32; + dst_uv += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFWIDTHROW_16TO8_AVX2 + +#ifdef HAS_HALFWIDTHROW_16TO8_ODD_AVX2 +LIBYUV_TARGET_AVX2 +void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + int n = (width - 1) & ~15; + if (n > 0) { + HalfWidthRow_16To8_AVX2(src_uv, src_uv_stride, dst_uv, scale, n); + src_uv += n * 2; + dst_uv += n; + width -= n; + } + HalfWidthRow_16To8_Odd_C(src_uv, src_uv_stride, dst_uv, scale, width); +} +#endif // HAS_HALFWIDTHROW_16TO8_ODD_AVX2 + +#ifdef HAS_GAUSSCOL_AVX2 +LIBYUV_TARGET_AVX2 +void GaussCol_AVX2(const uint16_t* src0, + const uint16_t* src1, + const uint16_t* src2, + const uint16_t* src3, + const uint16_t* src4, + uint32_t* dst, + int width) { + while (width >= 8) { + __m256i v0 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src0)); + __m256i v1 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src1)); + __m256i v2 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src2)); + __m256i v3 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src3)); + __m256i v4 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src4)); + __m256i v04 = _mm256_add_epi32(v0, v4); + __m256i v13 = _mm256_slli_epi32(_mm256_add_epi32(v1, v3), 2); + __m256i v2_6 = + _mm256_add_epi32(_mm256_slli_epi32(v2, 2), _mm256_slli_epi32(v2, 1)); + _mm256_storeu_si256((__m256i*)dst, + _mm256_add_epi32(_mm256_add_epi32(v04, v13), v2_6)); + src0 += 8; + src1 += 8; + src2 += 8; + src3 += 8; + src4 += 8; + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + if (width > 0) { + GaussCol_C(src0, src1, src2, src3, src4, dst, width); + } +} +#endif // HAS_GAUSSCOL_AVX2 + +#ifdef HAS_GAUSSCOL_F32_AVX2 +LIBYUV_TARGET_AVX2 +void GaussCol_F32_AVX2(const float* src0, + const float* src1, + const float* src2, + const float* src3, + const float* src4, + float* dst, + int width) { + const __m256 c4 = _mm256_set1_ps(4.0f); + const __m256 c6 = _mm256_set1_ps(6.0f); + while (width >= 8) { + __m256 s0 = _mm256_loadu_ps(src0); + __m256 s1 = _mm256_loadu_ps(src1); + __m256 s2 = _mm256_loadu_ps(src2); + __m256 s3 = _mm256_loadu_ps(src3); + __m256 s4 = _mm256_loadu_ps(src4); + __m256 sum = _mm256_add_ps(s0, _mm256_mul_ps(s1, c4)); + sum = _mm256_add_ps(sum, _mm256_mul_ps(s2, c6)); + sum = _mm256_add_ps(sum, _mm256_mul_ps(s3, c4)); + sum = _mm256_add_ps(sum, s4); + _mm256_storeu_ps(dst, sum); + src0 += 8; + src1 += 8; + src2 += 8; + src3 += 8; + src4 += 8; + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + if (width > 0) { + GaussCol_F32_C(src0, src1, src2, src3, src4, dst, width); + } +} +#endif // HAS_GAUSSCOL_F32_AVX2 + +#ifdef HAS_SCALEMAXSAMPLES_AVX2 +LIBYUV_TARGET_AVX2 +float ScaleMaxSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + __m256 vmax = _mm256_setzero_ps(); + const __m256 vscale = _mm256_set1_ps(scale); + while (width >= 8) { + __m256 v = _mm256_loadu_ps(src); + src += 8; + vmax = _mm256_max_ps(v, vmax); + _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); + dst += 8; + width -= 8; + } + __m128 m128 = + _mm_max_ps(_mm256_extractf128_ps(vmax, 1), _mm256_castps256_ps128(vmax)); + m128 = _mm_max_ps(_mm_movehl_ps(m128, m128), m128); + m128 = _mm_max_ss(_mm_shuffle_ps(m128, m128, 1), m128); + float fmax = _mm_cvtss_f32(m128); + _mm256_zeroupper(); + for (; width > 0; --width) { + float v = *src++; + float vs = v * scale; + fmax = (v > fmax) ? v : fmax; + *dst++ = vs; + } + return fmax; +} +#endif // HAS_SCALEMAXSAMPLES_AVX2 + +#ifdef HAS_SCALESUMSAMPLES_AVX2 +LIBYUV_TARGET_AVX2 +float ScaleSumSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + __m256 vsum = _mm256_setzero_ps(); + const __m256 vscale = _mm256_set1_ps(scale); + while (width >= 8) { + __m256 v = _mm256_loadu_ps(src); + src += 8; + vsum = _mm256_add_ps(vsum, _mm256_mul_ps(v, v)); + _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); + dst += 8; + width -= 8; + } + __m128 s128 = + _mm_add_ps(_mm256_castps256_ps128(vsum), _mm256_extractf128_ps(vsum, 1)); + s128 = _mm_add_ps(s128, _mm_movehl_ps(s128, s128)); + s128 = _mm_add_ss(s128, _mm_shuffle_ps(s128, s128, 1)); + float fsum = _mm_cvtss_f32(s128); + _mm256_zeroupper(); + for (; width > 0; --width) { + float v = *src++; + fsum += v * v; + *dst++ = v * scale; + } + return fsum; +} +#endif // HAS_SCALESUMSAMPLES_AVX2 + +#ifdef HAS_SCALESAMPLES_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) { + const __m256 vscale = _mm256_set1_ps(scale); + while (width >= 8) { + __m256 v = _mm256_loadu_ps(src); + src += 8; + _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + if (width > 0) { + ScaleSamples_C(src, dst, scale, width); + } +} +#endif // HAS_SCALESAMPLES_AVX2 + +#if defined(HAS_ARGBTORGB24ROW_AVX2) || defined(HAS_ARGBTORAWROW_AVX2) +LIBYUV_TARGET_AVX2 +static inline void ARGBToRGBRow_AVX2(const uint8_t* src, + uint8_t* dst, + int width, + const uint8_t* shuffler) { + static const uint32_t kPermdRGB24_AVX[8] = {0, 1, 2, 4, 5, 6, 3, 7}; + const __m256i ymm_shuf = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuffler)); + const __m256i ymm_perm = + _mm256_loadu_si256((const __m256i*)kPermdRGB24_AVX); + do { + __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src); + __m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src + 32)); + __m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src + 64)); + __m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src + 96)); + src += 128; + + ymm0 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm0, ymm_shuf), + ymm_perm); + ymm1 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm1, ymm_shuf), + ymm_perm); + ymm2 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm2, ymm_shuf), + ymm_perm); + ymm3 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm3, ymm_shuf), + ymm_perm); + + _mm256_storeu_si256( + (__m256i*)dst, + _mm256_or_si256(ymm0, _mm256_permute4x64_epi64(ymm1, 0x3f))); + _mm256_storeu_si256( + (__m256i*)(dst + 32), + _mm256_or_si256(_mm256_permute4x64_epi64(ymm1, 0xf9), + _mm256_permute4x64_epi64(ymm2, 0x4f))); + _mm256_storeu_si256( + (__m256i*)(dst + 64), + _mm256_or_si256(_mm256_permute4x64_epi64(ymm2, 0xfe), + _mm256_permute4x64_epi64(ymm3, 0x93))); + dst += 96; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // defined(HAS_ARGBTORGB24ROW_AVX2) || defined(HAS_ARGBTORAWROW_AVX2) + +#ifdef HAS_ARGBTORGB24ROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { + ARGBToRGBRow_AVX2(src, dst, width, kShuffleMaskARGBToRGB24[0]); +} +#endif // HAS_ARGBTORGB24ROW_AVX2 + +#ifdef HAS_ARGBTORAWROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { + static const uint8_t kShuffleMaskARGBToRAW_AVX2[16] = { + 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, + 128u}; + ARGBToRGBRow_AVX2(src, dst, width, kShuffleMaskARGBToRAW_AVX2); +} +#endif // HAS_ARGBTORAWROW_AVX2 + +#ifdef HAS_SPLITUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void SplitUVRow_AVX2(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i uv0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i uv1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32)); + __m256i u = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_and_si256(uv0, mask_00ff), + _mm256_and_si256(uv1, mask_00ff)), + 0xd8); + __m256i v = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_srli_epi16(uv0, 8), + _mm256_srli_epi16(uv1, 8)), + 0xd8); + _mm256_storeu_si256((__m256i*)dst_u, u); + _mm256_storeu_si256((__m256i*)dst_v, v); + src_uv += 64; + dst_u += 32; + dst_v += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITUVROW_AVX2 + +#ifdef HAS_AR64TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void AR64ToARGBRow_AVX2(const uint16_t* src_ar64, + uint8_t* dst_argb, + int width) { + do { + __m256i s0 = _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_ar64), + 8); + __m256i s1 = _mm256_srli_epi16( + _mm256_loadu_si256((const __m256i*)(src_ar64 + 16)), 8); + __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, out); + src_ar64 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_AR64TOARGBROW_AVX2 + +#ifdef HAS_AB64TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void AB64ToARGBRow_AVX2(const uint16_t* src_ab64, + uint8_t* dst_argb, + int width) { + static const uint8_t kShufSwapRB[16] = {2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u, + 10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u}; + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufSwapRB)); + do { + __m256i s0 = _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_ab64), + 8); + __m256i s1 = _mm256_srli_epi16( + _mm256_loadu_si256((const __m256i*)(src_ab64 + 16)), 8); + __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + out = _mm256_shuffle_epi8(out, shuf); + _mm256_storeu_si256((__m256i*)dst_argb, out); + src_ab64 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_AB64TOARGBROW_AVX2 + +#ifdef HAS_ARGBTOAR64ROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBToAR64Row_AVX2(const uint8_t* src_argb, + uint16_t* dst_ar64, + int width) { + do { + __m256i s = _mm256_permute4x64_epi64( + _mm256_loadu_si256((const __m256i*)src_argb), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ar64, _mm256_unpacklo_epi8(s, s)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 16), _mm256_unpackhi_epi8(s, s)); + src_argb += 32; + dst_ar64 += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOAR64ROW_AVX2 + +#ifdef HAS_ARGBTOAB64ROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBToAB64Row_AVX2(const uint8_t* src_argb, + uint16_t* dst_ab64, + int width) { + static const uint8_t kShufSwapRB[16] = {2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u, + 10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u}; + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufSwapRB)); + do { + __m256i s = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)src_argb), shuf); + s = _mm256_permute4x64_epi64(s, 0xd8); + _mm256_storeu_si256((__m256i*)dst_ab64, _mm256_unpacklo_epi8(s, s)); + _mm256_storeu_si256((__m256i*)(dst_ab64 + 16), _mm256_unpackhi_epi8(s, s)); + src_argb += 32; + dst_ab64 += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOAB64ROW_AVX2 + +#ifdef HAS_ARGBADDROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBAddRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + do { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_argb); + __m256i s1 = _mm256_loadu_si256((const __m256i*)src_argb1); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_adds_epu8(s0, s1)); + src_argb += 32; + src_argb1 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBADDROW_AVX2 + +#ifdef HAS_ARGBSUBTRACTROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBSubtractRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + do { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_argb); + __m256i s1 = _mm256_loadu_si256((const __m256i*)src_argb1); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_subs_epu8(s0, s1)); + src_argb += 32; + src_argb1 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBSUBTRACTROW_AVX2 + +#ifdef HAS_ARGBCOPYALPHAROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { + const __m256i mask_alpha = _mm256_set1_epi32((int)0xff000000u); + do { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src); + __m256i s1 = _mm256_loadu_si256((const __m256i*)(src + 32)); + __m256i d0 = _mm256_loadu_si256((const __m256i*)dst); + __m256i d1 = _mm256_loadu_si256((const __m256i*)(dst + 32)); + _mm256_storeu_si256((__m256i*)dst, _mm256_blendv_epi8(d0, s0, mask_alpha)); + _mm256_storeu_si256((__m256i*)(dst + 32), + _mm256_blendv_epi8(d1, s1, mask_alpha)); + src += 64; + dst += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBCOPYALPHAROW_AVX2 + +#ifdef HAS_ARGBPOLYNOMIALROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + const float* poly, + int width) { + const __m256 c0 = _mm256_broadcast_ps((const __m128*)(poly + 0)); + const __m256 c1 = _mm256_broadcast_ps((const __m128*)(poly + 4)); + const __m256 c2 = _mm256_broadcast_ps((const __m128*)(poly + 8)); + const __m256 c3 = _mm256_broadcast_ps((const __m128*)(poly + 12)); + do { + __m128i s8 = _mm_loadl_epi64((const __m128i*)src_argb); + src_argb += 8; + __m256 x = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(s8)); + __m256 x2 = _mm256_mul_ps(x, x); + __m256 res = _mm256_add_ps(c0, _mm256_mul_ps(c1, x)); + res = _mm256_add_ps(res, _mm256_mul_ps(c2, x2)); + res = _mm256_add_ps(res, _mm256_mul_ps(_mm256_mul_ps(c3, x), x2)); + __m256i i32 = _mm256_cvttps_epi32(res); + __m256i u16 = + _mm256_permute4x64_epi64(_mm256_packus_epi32(i32, i32), 0xd8); + __m128i u8 = _mm_packus_epi16(_mm256_castsi256_si128(u16), + _mm256_castsi256_si128(u16)); + _mm_storel_epi64((__m128i*)dst_argb, u8); + dst_argb += 8; + width -= 2; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBPOLYNOMIALROW_AVX2 + +#ifdef HAS_ARGBUNATTENUATEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const uint8_t kUnattenShuffleAlpha[16] = { + 0u, 1u, 0u, 1u, 0u, 1u, 6u, 7u, 8u, 9u, 8u, 9u, 8u, 9u, 14u, 15u}; + const __m256i ymm_shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kUnattenShuffleAlpha)); + do { + __m256i ymm3 = _mm256_setr_epi32( + (int)fixed_invtbl8[src_argb[3]], (int)fixed_invtbl8[src_argb[7]], + (int)fixed_invtbl8[src_argb[11]], (int)fixed_invtbl8[src_argb[15]], + (int)fixed_invtbl8[src_argb[19]], (int)fixed_invtbl8[src_argb[23]], + (int)fixed_invtbl8[src_argb[27]], (int)fixed_invtbl8[src_argb[31]]); + __m256i ymm6 = _mm256_loadu_si256((const __m256i*)src_argb); + __m256i ymm0 = _mm256_unpacklo_epi8(ymm6, ymm6); + __m256i ymm1 = _mm256_unpackhi_epi8(ymm6, ymm6); + __m256i ymm2 = + _mm256_shuffle_epi8(_mm256_unpacklo_epi16(ymm3, ymm3), ymm_shuf); + ymm3 = _mm256_shuffle_epi8(_mm256_unpackhi_epi16(ymm3, ymm3), ymm_shuf); + ymm0 = _mm256_mulhi_epu16(ymm0, ymm2); + ymm1 = _mm256_mulhi_epu16(ymm1, ymm3); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_packus_epi16(ymm0, ymm1)); + src_argb += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBUNATTENUATEROW_AVX2 + +#ifdef HAS_COMPUTECUMULATIVESUMROW_AVX2 +LIBYUV_TARGET_AVX2 +void ComputeCumulativeSumRow_AVX2(const uint8_t* row, + int32_t* cumsum, + const int32_t* previous_cumsum, + int width) { + static const uint8_t kCumSumTranspose[16] = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; + static const int8_t kCumSumMaddLo[32] = { + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0}; + static const int8_t kCumSumMaddHi[32] = { + 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + __m256i ymm0 = _mm256_setzero_si256(); + const __m256i ymm5 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kCumSumTranspose)); + const __m256i ymm6 = _mm256_loadu_si256((const __m256i*)kCumSumMaddLo); + const __m256i ymm7 = _mm256_loadu_si256((const __m256i*)kCumSumMaddHi); + const __m256i ymm8 = _mm256_set1_epi16(1); + + while (width >= 4) { + __m256i ymm2 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)row)); + row += 16; + ymm2 = _mm256_shuffle_epi8(ymm2, ymm5); + __m256i ymm3 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm2, ymm6), ymm8); + __m256i ymm4 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm2, ymm7), ymm8); + __m256i sum4 = _mm256_permute2x128_si256(ymm4, ymm4, 0x11); + ymm3 = _mm256_add_epi32( + _mm256_add_epi32(ymm3, + _mm256_loadu_si256((const __m256i*)previous_cumsum)), + ymm0); + ymm4 = _mm256_add_epi32( + _mm256_add_epi32( + ymm4, _mm256_loadu_si256((const __m256i*)(previous_cumsum + 8))), + ymm0); + previous_cumsum += 16; + ymm0 = _mm256_add_epi32(ymm0, sum4); + _mm256_storeu_si256((__m256i*)cumsum, ymm3); + _mm256_storeu_si256((__m256i*)(cumsum + 8), ymm4); + cumsum += 16; + width -= 4; + } + + __m128i xmm0 = _mm256_castsi256_si128(ymm0); + while (width > 0) { + int32_t px; + memcpy(&px, row, sizeof(px)); + row += 4; + __m128i xmm2 = _mm_cvtepu8_epi32(_mm_cvtsi32_si128(px)); + xmm0 = _mm_add_epi32(xmm0, xmm2); + xmm2 = _mm_add_epi32(xmm0, + _mm_loadu_si128((const __m128i*)previous_cumsum)); + previous_cumsum += 4; + _mm_storeu_si128((__m128i*)cumsum, xmm2); + cumsum += 4; + --width; + } + _mm256_zeroupper(); +} +#endif // HAS_COMPUTECUMULATIVESUMROW_AVX2 + +#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 +LIBYUV_TARGET_AVX2 +void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, + const int32_t* botleft, + int width, + int area, + uint8_t* dst, + int count) { + __m128 xmm5 = _mm_cvtsi32_ss(_mm_setzero_ps(), area); + __m128 xmm4 = _mm_rcp_ss(xmm5); + xmm4 = _mm_shuffle_ps(xmm4, xmm4, 0); + + if (count >= 4) { + __m256 ymm4 = _mm256_insertf128_ps(_mm256_castps128_ps256(xmm4), xmm4, 1); + if ((uint32_t)area <= 0x80u) { + __m128 xmm6 = _mm_cvtepi32_ps(_mm_set1_epi32(0x0000ffff)); + xmm6 = _mm_mul_ss(_mm_add_ss(xmm5, xmm6), xmm4); + __m128i i6 = _mm_cvtps_epi32(xmm6); + i6 = _mm_packs_epi32(i6, i6); + __m256i ymm6 = _mm256_broadcastw_epi16(i6); + while (count >= 4) { + __m256i ymm0 = _mm256_sub_epi32( + _mm256_loadu_si256((const __m256i*)topleft), + _mm256_loadu_si256((const __m256i*)(topleft + width))); + __m256i ymm1 = _mm256_sub_epi32( + _mm256_loadu_si256((const __m256i*)(topleft + 8)), + _mm256_loadu_si256((const __m256i*)(topleft + width + 8))); + topleft += 16; + ymm0 = _mm256_sub_epi32(ymm0, + _mm256_loadu_si256((const __m256i*)botleft)); + ymm1 = _mm256_sub_epi32( + ymm1, _mm256_loadu_si256((const __m256i*)(botleft + 8))); + ymm0 = _mm256_add_epi32( + ymm0, _mm256_loadu_si256((const __m256i*)(botleft + width))); + ymm1 = _mm256_add_epi32( + ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8))); + botleft += 16; + ymm0 = _mm256_mulhi_epu16(_mm256_packs_epi32(ymm0, ymm1), ymm6); + __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0), + _mm256_extracti128_si256(ymm0, 1)); + _mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8)); + dst += 16; + count -= 4; + } + } else { + while (count >= 4) { + __m256i ymm0 = _mm256_sub_epi32( + _mm256_loadu_si256((const __m256i*)topleft), + _mm256_loadu_si256((const __m256i*)(topleft + width))); + __m256i ymm1 = _mm256_sub_epi32( + _mm256_loadu_si256((const __m256i*)(topleft + 8)), + _mm256_loadu_si256((const __m256i*)(topleft + width + 8))); + topleft += 16; + ymm0 = _mm256_sub_epi32(ymm0, + _mm256_loadu_si256((const __m256i*)botleft)); + ymm1 = _mm256_sub_epi32( + ymm1, _mm256_loadu_si256((const __m256i*)(botleft + 8))); + ymm0 = _mm256_add_epi32( + ymm0, _mm256_loadu_si256((const __m256i*)(botleft + width))); + ymm1 = _mm256_add_epi32( + ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8))); + botleft += 16; + ymm0 = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm0), ymm4)); + ymm1 = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm1), ymm4)); + ymm0 = _mm256_packs_epi32(ymm0, ymm1); + __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0), + _mm256_extracti128_si256(ymm0, 1)); + _mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8)); + dst += 16; + count -= 4; + } + } + _mm256_zeroupper(); + } + + while (count > 0) { + __m128i d0 = _mm_sub_epi32( + _mm_loadu_si128((const __m128i*)topleft), + _mm_loadu_si128((const __m128i*)(topleft + width))); + topleft += 4; + __m128i d1 = _mm_sub_epi32( + _mm_loadu_si128((const __m128i*)(botleft + width)), + _mm_loadu_si128((const __m128i*)botleft)); + botleft += 4; + d0 = _mm_add_epi32(d0, d1); + d0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(d0), xmm4)); + d0 = _mm_packs_epi32(d0, d0); + d0 = _mm_packus_epi16(d0, d0); + int32_t out32 = _mm_cvtsi128_si32(d0); + memcpy(dst, &out32, sizeof(out32)); + dst += 4; + --count; + } +} +#endif // HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 + +#ifdef HAS_I422TOYUY2ROW_AVX2 +LIBYUV_TARGET_AVX2 +void I422ToYUY2Row_AVX2(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_yuy2, + int width) { + do { + __m128i u = _mm_loadu_si128((const __m128i*)src_u); + __m128i v = _mm_loadu_si128((const __m128i*)src_v); + __m256i uv = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(u, v)), + _mm_unpackhi_epi8(u, v), 1); + __m256i y = _mm256_loadu_si256((const __m256i*)src_y); + __m256i lo = _mm256_unpacklo_epi8(y, uv); + __m256i hi = _mm256_unpackhi_epi8(y, uv); + _mm256_storeu_si256((__m256i*)dst_yuy2, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_yuy2 + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_y += 32; + src_u += 16; + src_v += 16; + dst_yuy2 += 64; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_I422TOYUY2ROW_AVX2 + +#ifdef HAS_I422TOUYVYROW_AVX2 +LIBYUV_TARGET_AVX2 +void I422ToUYVYRow_AVX2(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uyvy, + int width) { + do { + __m128i u = _mm_loadu_si128((const __m128i*)src_u); + __m128i v = _mm_loadu_si128((const __m128i*)src_v); + __m256i uv = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(u, v)), + _mm_unpackhi_epi8(u, v), 1); + __m256i y = _mm256_loadu_si256((const __m256i*)src_y); + __m256i lo = _mm256_unpacklo_epi8(uv, y); + __m256i hi = _mm256_unpackhi_epi8(uv, y); + _mm256_storeu_si256((__m256i*)dst_uyvy, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_uyvy + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_y += 32; + src_u += 16; + src_v += 16; + dst_uyvy += 64; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_I422TOUYVYROW_AVX2 + +#ifdef HAS_MERGEARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeARGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + const uint8_t* src_a, + uint8_t* dst_argb, + int width) { + do { + __m128i r = _mm_loadu_si128((const __m128i*)src_r); + __m128i g = _mm_loadu_si128((const __m128i*)src_g); + __m128i b = _mm_loadu_si128((const __m128i*)src_b); + __m128i a = _mm_loadu_si128((const __m128i*)src_a); + __m256i bg = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)), + _mm_unpackhi_epi8(b, g), 1); + __m256i ra = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)), + _mm_unpackhi_epi8(r, a), 1); + __m256i lo = _mm256_unpacklo_epi16(bg, ra); + __m256i hi = _mm256_unpackhi_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + src_a += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEARGBROW_AVX2 + +#ifdef HAS_MERGEXRGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeXRGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_argb, + int width) { + const __m128i a = _mm_set1_epi8((char)0xff); + do { + __m128i r = _mm_loadu_si128((const __m128i*)src_r); + __m128i g = _mm_loadu_si128((const __m128i*)src_g); + __m128i b = _mm_loadu_si128((const __m128i*)src_b); + __m256i bg = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)), + _mm_unpackhi_epi8(b, g), 1); + __m256i ra = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)), + _mm_unpackhi_epi8(r, a), 1); + __m256i lo = _mm256_unpacklo_epi16(bg, ra); + __m256i hi = _mm256_unpackhi_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEXRGBROW_AVX2 + +#ifdef HAS_MERGEARGB16TO8ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeARGB16To8Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + const uint16_t* src_a, + uint8_t* dst_argb, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(depth - 8); + const __m256i ymm_255 = _mm256_set1_epi16(255); + do { + __m256i r16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), shift128), + ymm_255); + __m256i g16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), shift128), + ymm_255); + __m256i b16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), shift128), + ymm_255); + __m256i a16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_a), shift128), + ymm_255); + __m256i br8 = + _mm256_permute4x64_epi64(_mm256_packus_epi16(b16, r16), 0xd8); + __m256i ga8 = + _mm256_permute4x64_epi64(_mm256_packus_epi16(g16, a16), 0xd8); + __m128i b = _mm256_castsi256_si128(br8); + __m128i r = _mm256_extracti128_si256(br8, 1); + __m128i g = _mm256_castsi256_si128(ga8); + __m128i a = _mm256_extracti128_si256(ga8, 1); + __m256i bg = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)), + _mm_unpackhi_epi8(b, g), 1); + __m256i ra = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)), + _mm_unpackhi_epi8(r, a), 1); + __m256i lo = _mm256_unpacklo_epi16(bg, ra); + __m256i hi = _mm256_unpackhi_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + src_a += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEARGB16TO8ROW_AVX2 + +#ifdef HAS_MERGEXRGB16TO8ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeXRGB16To8Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + uint8_t* dst_argb, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(depth - 8); + const __m256i ymm_255 = _mm256_set1_epi16(255); + const __m128i a = _mm_set1_epi8((char)0xff); + do { + __m256i r16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), shift128), + ymm_255); + __m256i g16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), shift128), + ymm_255); + __m256i b16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), shift128), + ymm_255); + __m256i br8 = + _mm256_permute4x64_epi64(_mm256_packus_epi16(b16, r16), 0xd8); + __m256i g8 = + _mm256_permute4x64_epi64(_mm256_packus_epi16(g16, g16), 0xd8); + __m128i b = _mm256_castsi256_si128(br8); + __m128i r = _mm256_extracti128_si256(br8, 1); + __m128i g = _mm256_castsi256_si128(g8); + __m256i bg = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)), + _mm_unpackhi_epi8(b, g), 1); + __m256i ra = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)), + _mm_unpackhi_epi8(r, a), 1); + __m256i lo = _mm256_unpacklo_epi16(bg, ra); + __m256i hi = _mm256_unpackhi_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEXRGB16TO8ROW_AVX2 + +#ifdef HAS_MERGEAR64ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeAR64Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + const uint16_t* src_a, + uint16_t* dst_ar64, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(16 - depth); + const __m256i ymm_max = + _mm256_set1_epi16((short)((1u << depth) - 1u)); + do { + __m256i r = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_r), ymm_max), + shift128); + __m256i g = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_g), ymm_max), + shift128); + __m256i b = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_b), ymm_max), + shift128); + __m256i a = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_a), ymm_max), + shift128); + __m256i bg_lo = _mm256_unpacklo_epi16(b, g); + __m256i bg_hi = _mm256_unpackhi_epi16(b, g); + __m256i ra_lo = _mm256_unpacklo_epi16(r, a); + __m256i ra_hi = _mm256_unpackhi_epi16(r, a); + __m256i p0_8 = _mm256_unpacklo_epi32(bg_lo, ra_lo); + __m256i p2_10 = _mm256_unpackhi_epi32(bg_lo, ra_lo); + __m256i p4_12 = _mm256_unpacklo_epi32(bg_hi, ra_hi); + __m256i p6_14 = _mm256_unpackhi_epi32(bg_hi, ra_hi); + _mm256_storeu_si256((__m256i*)dst_ar64, + _mm256_permute2x128_si256(p0_8, p2_10, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 16), + _mm256_permute2x128_si256(p4_12, p6_14, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 32), + _mm256_permute2x128_si256(p0_8, p2_10, 0x31)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 48), + _mm256_permute2x128_si256(p4_12, p6_14, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + src_a += 16; + dst_ar64 += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEAR64ROW_AVX2 + +#ifdef HAS_MERGEXR64ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeXR64Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + uint16_t* dst_ar64, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(16 - depth); + const __m256i ymm_max = + _mm256_set1_epi16((short)((1u << depth) - 1u)); + const __m256i a = _mm256_set1_epi16((short)0xffff); + do { + __m256i r = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_r), ymm_max), + shift128); + __m256i g = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_g), ymm_max), + shift128); + __m256i b = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_b), ymm_max), + shift128); + __m256i bg_lo = _mm256_unpacklo_epi16(b, g); + __m256i bg_hi = _mm256_unpackhi_epi16(b, g); + __m256i ra_lo = _mm256_unpacklo_epi16(r, a); + __m256i ra_hi = _mm256_unpackhi_epi16(r, a); + __m256i p0_8 = _mm256_unpacklo_epi32(bg_lo, ra_lo); + __m256i p2_10 = _mm256_unpackhi_epi32(bg_lo, ra_lo); + __m256i p4_12 = _mm256_unpacklo_epi32(bg_hi, ra_hi); + __m256i p6_14 = _mm256_unpackhi_epi32(bg_hi, ra_hi); + _mm256_storeu_si256((__m256i*)dst_ar64, + _mm256_permute2x128_si256(p0_8, p2_10, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 16), + _mm256_permute2x128_si256(p4_12, p6_14, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 32), + _mm256_permute2x128_si256(p0_8, p2_10, 0x31)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 48), + _mm256_permute2x128_si256(p4_12, p6_14, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + dst_ar64 += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEXR64ROW_AVX2 + +#ifdef HAS_NV21TOYUV24ROW_AVX2 +LIBYUV_TARGET_AVX2 +void NV21ToYUV24Row_AVX2(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_yuv24, + int width) { + static const uint8_t kYUV24Shuffle[3][16] = { + {8, 9, 0, 8, 9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12}, + {9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15}, + {2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15, 6, 14, 15, 7}}; + const __m256i ymm4 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kYUV24Shuffle[0])); + const __m256i ymm5 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kYUV24Shuffle[1])); + const __m256i ymm6 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kYUV24Shuffle[2])); + + do { + __m256 ymm2 = _mm256_castsi256_ps(_mm256_loadu_si256((const __m256i*)src_y)); + __m256 ymm3 = + _mm256_castsi256_ps(_mm256_loadu_si256((const __m256i*)src_vu)); + __m256i c0 = _mm256_shuffle_epi8( + _mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0x44)), ymm4); + __m256i c1 = _mm256_shuffle_epi8( + _mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0x99)), ymm5); + __m256i c2 = _mm256_shuffle_epi8( + _mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0xee)), ymm6); + _mm256_storeu_si256((__m256i*)dst_yuv24, + _mm256_permute2x128_si256(c0, c1, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_yuv24 + 32), + _mm256_permute2x128_si256(c2, c0, 0x30)); + _mm256_storeu_si256((__m256i*)(dst_yuv24 + 64), + _mm256_permute2x128_si256(c1, c2, 0x31)); + src_y += 32; + src_vu += 32; + dst_yuv24 += 96; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_NV21TOYUV24ROW_AVX2 + +#ifdef HAS_SPLITARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void SplitARGBRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + uint8_t* dst_a, + int width) { + static const uint8_t kShufSplitARGB[16] = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufSplitARGB)); + do { + __m256i t0 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)src_argb), shuf); + __m256i t1 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_argb + 32)), shuf); + __m256i p0 = _mm256_permute2x128_si256(t0, t1, 0x20); + __m256i p1 = _mm256_permute2x128_si256(t0, t1, 0x31); + __m256i bg = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(p0, p1), 0xd8); + __m256i ra = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(p0, p1), 0xd8); + _mm_storeu_si128((__m128i*)dst_b, _mm256_castsi256_si128(bg)); + _mm_storeu_si128((__m128i*)dst_g, _mm256_extracti128_si256(bg, 1)); + _mm_storeu_si128((__m128i*)dst_r, _mm256_castsi256_si128(ra)); + _mm_storeu_si128((__m128i*)dst_a, _mm256_extracti128_si256(ra, 1)); + src_argb += 64; + dst_r += 16; + dst_g += 16; + dst_b += 16; + dst_a += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITARGBROW_AVX2 + +#ifdef HAS_SPLITXRGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void SplitXRGBRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width) { + static const uint8_t kShufSplitARGB[16] = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufSplitARGB)); + do { + __m256i t0 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)src_argb), shuf); + __m256i t1 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_argb + 32)), shuf); + __m256i p0 = _mm256_permute2x128_si256(t0, t1, 0x20); + __m256i p1 = _mm256_permute2x128_si256(t0, t1, 0x31); + __m256i bg = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(p0, p1), 0xd8); + __m256i ra = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(p0, p1), 0xd8); + _mm_storeu_si128((__m128i*)dst_b, _mm256_castsi256_si128(bg)); + _mm_storeu_si128((__m128i*)dst_g, _mm256_extracti128_si256(bg, 1)); + _mm_storeu_si128((__m128i*)dst_r, _mm256_castsi256_si128(ra)); + src_argb += 64; + dst_r += 16; + dst_g += 16; + dst_b += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITXRGBROW_AVX2 + +#ifdef HAS_SPLITRGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void SplitRGBRow_AVX2(const uint8_t* src_rgb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width) { + static const uint8_t kSplitRGBShuffle[5][16] = { + {0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u}, + {1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u}, + {2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u}, + {0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, + 0u}, + {0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, + 0u}}; + const __m256i s0 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[0])); + const __m256i s1 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[1])); + const __m256i s2 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[2])); + const __m256i m0 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[3])); + const __m256i m1 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[4])); + + do { + __m256i ymm4 = _mm256_loadu_si256((const __m256i*)src_rgb); + __m256i ymm5 = _mm256_loadu_si256((const __m256i*)(src_rgb + 32)); + __m256i ymm6 = _mm256_loadu_si256((const __m256i*)(src_rgb + 64)); + src_rgb += 96; + __m256i ymm1 = _mm256_blend_epi32(ymm4, ymm5, 0xf0); + __m256i ymm2 = _mm256_permute2x128_si256(ymm4, ymm6, 0x21); + __m256i ymm3 = _mm256_blend_epi32(ymm5, ymm6, 0xf0); + ymm4 = _mm256_blendv_epi8(ymm1, ymm3, m0); + ymm5 = _mm256_blendv_epi8(ymm2, ymm1, m0); + ymm6 = _mm256_blendv_epi8(ymm3, ymm2, m0); + ymm1 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm4, ymm5, m1), s0); + ymm2 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm5, ymm6, m1), s1); + ymm3 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm6, ymm4, m1), s2); + _mm256_storeu_si256((__m256i*)dst_r, ymm1); + _mm256_storeu_si256((__m256i*)dst_g, ymm2); + _mm256_storeu_si256((__m256i*)dst_b, ymm3); + dst_r += 32; + dst_g += 32; + dst_b += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITRGBROW_AVX2 + +#ifdef HAS_SPLITUVROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void SplitUVRow_16_AVX2(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(16 - depth); + const __m256i mask16 = _mm256_set1_epi32(0x0000ffff); + do { + __m256i uv0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i uv1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16)); + __m256i u = _mm256_permute4x64_epi64( + _mm256_packus_epi32(_mm256_and_si256(uv0, mask16), + _mm256_and_si256(uv1, mask16)), + 0xd8); + __m256i v = _mm256_permute4x64_epi64( + _mm256_packus_epi32(_mm256_srli_epi32(uv0, 16), + _mm256_srli_epi32(uv1, 16)), + 0xd8); + _mm256_storeu_si256((__m256i*)dst_u, _mm256_srl_epi16(u, shift128)); + _mm256_storeu_si256((__m256i*)dst_v, _mm256_srl_epi16(v, shift128)); + src_uv += 32; + dst_u += 16; + dst_v += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITUVROW_16_AVX2 + +#ifdef HAS_YUY2TOYROW_AVX2 +LIBYUV_TARGET_AVX2 +void YUY2ToYRow_AVX2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i s0 = + _mm256_and_si256(_mm256_loadu_si256((const __m256i*)src_yuy2), + mask_00ff); + __m256i s1 = + _mm256_and_si256(_mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)), + mask_00ff); + _mm256_storeu_si256( + (__m256i*)dst_y, + _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8)); + src_yuy2 += 64; + dst_y += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOYROW_AVX2 + +#ifdef HAS_UYVYTOYROW_AVX2 +LIBYUV_TARGET_AVX2 +void UYVYToYRow_AVX2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + do { + __m256i s0 = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_uyvy), 8); + __m256i s1 = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)(src_uyvy + 32)), + 8); + _mm256_storeu_si256( + (__m256i*)dst_y, + _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8)); + src_uyvy += 64; + dst_y += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_UYVYTOYROW_AVX2 + +#ifdef HAS_YUY2TOUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void YUY2ToUVRow_AVX2(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i y0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_yuy2), + _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2))); + __m256i y1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)), + _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2 + 32))); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)), + 0xd8); + __m256i u = _mm256_and_si256(uv, mask_00ff); + __m256i v = _mm256_srli_epi16(uv, 8); + u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8); + v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v)); + src_yuy2 += 64; + dst_u += 16; + dst_v += 16; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOUVROW_AVX2 + +#ifdef HAS_YUY2TOUV422ROW_AVX2 +LIBYUV_TARGET_AVX2 +void YUY2ToUV422Row_AVX2(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i y0 = _mm256_loadu_si256((const __m256i*)src_yuy2); + __m256i y1 = _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)), + 0xd8); + __m256i u = _mm256_and_si256(uv, mask_00ff); + __m256i v = _mm256_srli_epi16(uv, 8); + u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8); + v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v)); + src_yuy2 += 64; + dst_u += 16; + dst_v += 16; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOUV422ROW_AVX2 + +#ifdef HAS_YUY2TONVUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void YUY2ToNVUVRow_AVX2(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_uv, + int width) { + do { + __m256i y0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_yuy2), + _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2))); + __m256i y1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)), + _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2 + 32))); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)), + 0xd8); + _mm256_storeu_si256((__m256i*)dst_uv, uv); + src_yuy2 += 64; + dst_uv += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_YUY2TONVUVROW_AVX2 + +#if defined(HAS_UYVYTOUVROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2) +LIBYUV_TARGET_AVX2 +void UYVYToUVRow_AVX2(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i y0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_uyvy), + _mm256_loadu_si256((const __m256i*)(src_uyvy + stride_uyvy))); + __m256i y1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_uyvy + 32)), + _mm256_loadu_si256((const __m256i*)(src_uyvy + stride_uyvy + 32))); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_and_si256(y0, mask_00ff), + _mm256_and_si256(y1, mask_00ff)), + 0xd8); + __m256i u = _mm256_and_si256(uv, mask_00ff); + __m256i v = _mm256_srli_epi16(uv, 8); + u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8); + v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v)); + src_uyvy += 64; + dst_u += 16; + dst_v += 16; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // defined(HAS_UYVYTOUVROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2) + +#if defined(HAS_UYVYTOUV422ROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2) +LIBYUV_TARGET_AVX2 +void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i y0 = _mm256_loadu_si256((const __m256i*)src_uyvy); + __m256i y1 = _mm256_loadu_si256((const __m256i*)(src_uyvy + 32)); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_and_si256(y0, mask_00ff), + _mm256_and_si256(y1, mask_00ff)), + 0xd8); + __m256i u = _mm256_and_si256(uv, mask_00ff); + __m256i v = _mm256_srli_epi16(uv, 8); + u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8); + v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v)); + src_uyvy += 64; + dst_u += 16; + dst_v += 16; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // defined(HAS_UYVYTOUV422ROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2) + +#ifdef HAS_I444TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void I444ToARGBRow_AVX2(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + const __m256i ymm_kUVToB = + _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); + const __m256i ymm_kUVToG = + _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); + const __m256i ymm_kUVToR = + _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); + const __m256i ymm_kYToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + const __m256i ymm_kYBiasToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); + const __m256i ymm_128 = _mm256_set1_epi8((char)0x80); + const __m256i ymm_alpha = _mm256_set1_epi8((char)0xff); + + while (width >= 16) { + __m128i xmm_u = _mm_loadu_si128((const __m128i*)u_buf); + __m128i xmm_v = _mm_loadu_si128((const __m128i*)v_buf); + __m128i xmm_y = _mm_loadu_si128((const __m128i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 16; + + __m256i ymm3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(xmm_u, xmm_v)), + _mm_unpackhi_epi8(xmm_u, xmm_v), 1); + __m256i ymm4 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(xmm_y, xmm_y)), + _mm_unpackhi_epi8(xmm_y, xmm_y), 1); + + ymm3 = _mm256_sub_epi8(ymm3, ymm_128); + ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb); + + __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3); + __m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3); + __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3); + + ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb); + + ymm0 = _mm256_srai_epi16(_mm256_adds_epi16(ymm0, ymm4), 6); + ymm1 = _mm256_srai_epi16(_mm256_subs_epi16(ymm4, ymm1), 6); + ymm2 = _mm256_srai_epi16(_mm256_adds_epi16(ymm2, ymm4), 6); + + ymm0 = _mm256_packus_epi16(ymm0, ymm0); + ymm1 = _mm256_packus_epi16(ymm1, ymm1); + ymm2 = _mm256_packus_epi16(ymm2, ymm2); + + ymm0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi8(ymm0, ymm1), 0xd8); + ymm2 = + _mm256_permute4x64_epi64(_mm256_unpacklo_epi8(ymm2, ymm_alpha), 0xd8); + + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi16(ymm0, ymm2)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_unpackhi_epi16(ymm0, ymm2)); + + dst_argb += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_I444TOARGBROW_AVX2 + +#ifdef HAS_MERGEXR30ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeXR30Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + uint8_t* dst_ar30, + int depth, + int width) { + int shift = depth - 10; + __m128i v_shift = _mm_cvtsi32_si128(shift); + __m256i v_alpha = _mm256_set1_epi16(0x0030); + __m256i v_max = _mm256_set1_epi16(1023); + while (width > 0) { + __m256i r = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), v_shift), + v_max); + __m256i g = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), v_shift), + v_max); + __m256i b = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), v_shift), + v_max); + r = _mm256_slli_epi16(_mm256_permute4x64_epi64(r, 0xd8), 4); + g = _mm256_permute4x64_epi64(g, 0xd8); + b = _mm256_permute4x64_epi64(b, 0xd8); + __m256i rb_lo = _mm256_unpacklo_epi16(b, r); + __m256i rb_hi = _mm256_unpackhi_epi16(b, r); + __m256i ag_lo = _mm256_slli_epi32(_mm256_unpacklo_epi16(g, v_alpha), 10); + __m256i ag_hi = _mm256_slli_epi32(_mm256_unpackhi_epi16(g, v_alpha), 10); + _mm256_storeu_si256((__m256i*)dst_ar30, _mm256_or_si256(rb_lo, ag_lo)); + _mm256_storeu_si256((__m256i*)(dst_ar30 + 32), + _mm256_or_si256(rb_hi, ag_hi)); + src_r += 16; + src_g += 16; + src_b += 16; + dst_ar30 += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_MERGEXR30ROW_AVX2 + +#ifdef HAS_MERGEUVROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void MergeUVRow_16_AVX2(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width) { + const __m128i sh_u = _mm_cvtsi32_si128(16 - depth); + const __m128i sh_v = _mm_cvtsi32_si128(32 - depth); + while (width > 0) { + __m256i u32 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_u)); + __m256i v32 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_v)); + src_u += 8; + src_v += 8; + u32 = _mm256_sll_epi16(u32, sh_u); + v32 = _mm256_sll_epi32(v32, sh_v); + _mm256_storeu_si256((__m256i*)dst_uv, _mm256_or_si256(u32, v32)); + dst_uv += 16; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_MERGEUVROW_16_AVX2 + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale.cc b/source/scale.cc index a9fdf79c8..f2157409c 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -206,6 +206,20 @@ static void ScalePlaneDown2_16(int src_width, src_stride = 0; } +#if defined(HAS_SCALEROWDOWN2_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleRowDown2 = + filtering == kFilterNone + ? ScaleRowDown2_16_Any_AVX2 + : (filtering == kFilterLinear ? ScaleRowDown2Linear_16_Any_AVX2 + : ScaleRowDown2Box_16_Any_AVX2); + if (IS_ALIGNED(dst_width, 16)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_AVX2 + : filtering == kFilterLinear ? ScaleRowDown2Linear_16_AVX2 + : ScaleRowDown2Box_16_AVX2; + } + } +#endif #if defined(HAS_SCALEROWDOWN2_16_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 16)) { ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_NEON @@ -220,15 +234,6 @@ static void ScalePlaneDown2_16(int src_width, : ScaleRowDown2Box_16_SME; } #endif -#if defined(HAS_SCALEROWDOWN2_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 16)) { - ScaleRowDown2 = - filtering == kFilterNone - ? ScaleRowDown2_16_SSE2 - : (filtering == kFilterLinear ? ScaleRowDown2Linear_16_SSE2 - : ScaleRowDown2Box_16_SSE2); - } -#endif #if defined(HAS_SCALEROWDOWN2_16_HVX) if (TestCpuFlag(kCpuHasHVX)) { ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_HVX @@ -360,18 +365,6 @@ static void ScalePlaneDown4_16(int src_width, src_ptr += src_stride * 2; // Point to row 2. src_stride = 0; } -#if defined(HAS_SCALEROWDOWN4_16_NEON) - if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 8)) { - ScaleRowDown4 = - filtering ? ScaleRowDown4Box_16_NEON : ScaleRowDown4_16_NEON; - } -#endif -#if defined(HAS_SCALEROWDOWN4_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleRowDown4 = - filtering ? ScaleRowDown4Box_16_SSE2 : ScaleRowDown4_16_SSE2; - } -#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -455,23 +448,23 @@ static void ScalePlaneDown34(int src_width, } } #endif -#if defined(HAS_SCALEROWDOWN34_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { +#if defined(HAS_SCALEROWDOWN34_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { if (dst_width % 24 == 0) { if (!filtering) { - ScaleRowDown34_0 = ScaleRowDown34_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_SSSE3; + ScaleRowDown34_0 = ScaleRowDown34_AVX2; + ScaleRowDown34_1 = ScaleRowDown34_AVX2; } else { - ScaleRowDown34_0 = ScaleRowDown34_0_Box_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_1_Box_SSSE3; + ScaleRowDown34_0 = ScaleRowDown34_0_Box_AVX2; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_AVX2; } } else { if (!filtering) { - ScaleRowDown34_0 = ScaleRowDown34_Any_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_Any_SSSE3; + ScaleRowDown34_0 = ScaleRowDown34_Any_AVX2; + ScaleRowDown34_1 = ScaleRowDown34_Any_AVX2; } else { - ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_SSSE3; + ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_AVX2; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_AVX2; } } } @@ -557,28 +550,6 @@ static void ScalePlaneDown34_16(int src_width, ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_C; ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_C; } -#if defined(HAS_SCALEROWDOWN34_16_NEON) - if (TestCpuFlag(kCpuHasNEON) && (dst_width % 24 == 0)) { - if (!filtering) { - ScaleRowDown34_0 = ScaleRowDown34_16_NEON; - ScaleRowDown34_1 = ScaleRowDown34_16_NEON; - } else { - ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_NEON; - ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_NEON; - } - } -#endif -#if defined(HAS_SCALEROWDOWN34_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) { - if (!filtering) { - ScaleRowDown34_0 = ScaleRowDown34_16_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_16_SSSE3; - } else { - ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_SSSE3; - } - } -#endif for (y = 0; y < dst_height - 2; y += 3) { ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width); @@ -664,22 +635,23 @@ static void ScalePlaneDown38(int src_width, } } #endif -#if defined(HAS_SCALEROWDOWN38_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { +#if defined(HAS_SCALEROWDOWN38_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { if (!filtering) { - ScaleRowDown38_3 = ScaleRowDown38_Any_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_Any_SSSE3; + ScaleRowDown38_3 = ScaleRowDown38_Any_AVX2; + ScaleRowDown38_2 = ScaleRowDown38_Any_AVX2; } else { - ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_SSSE3; + ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_AVX2; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_AVX2; } - if (dst_width % 12 == 0 && !filtering) { - ScaleRowDown38_3 = ScaleRowDown38_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_SSSE3; - } - if (dst_width % 6 == 0 && filtering) { - ScaleRowDown38_3 = ScaleRowDown38_3_Box_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_2_Box_SSSE3; + if (dst_width % 12 == 0) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_AVX2; + ScaleRowDown38_2 = ScaleRowDown38_AVX2; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_AVX2; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_AVX2; + } } } #endif @@ -783,28 +755,6 @@ static void ScalePlaneDown38_16(int src_width, ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_C; ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_C; } -#if defined(HAS_SCALEROWDOWN38_16_NEON) - if (TestCpuFlag(kCpuHasNEON) && (dst_width % 12 == 0)) { - if (!filtering) { - ScaleRowDown38_3 = ScaleRowDown38_16_NEON; - ScaleRowDown38_2 = ScaleRowDown38_16_NEON; - } else { - ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_NEON; - ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_NEON; - } - } -#endif -#if defined(HAS_SCALEROWDOWN38_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) { - if (!filtering) { - ScaleRowDown38_3 = ScaleRowDown38_16_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_16_SSSE3; - } else { - ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_SSSE3; - } - } -#endif for (y = 0; y < dst_height - 2; y += 3) { ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); @@ -1002,11 +952,15 @@ static int ScalePlaneBox(int src_width, ScaleAddRow = ScaleAddRow_AVX512BW; } #endif -#if defined(HAS_SCALEADDCOLS_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0) { - ScaleAddCols = (dx & 0xffff) ? ScaleAddCols2_AVX512BW - : ((dx != 0x10000) ? ScaleAddCols1_AVX512BW - : ScaleAddCols0_C); +#if defined(HAS_SCALEADDCOLS1_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0 && !(dx & 0xffff) && + dx != 0x10000) { + ScaleAddCols = ScaleAddCols1_AVX512BW; + } +#endif +#if defined(HAS_SCALEADDCOLS2_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0 && (dx & 0xffff)) { + ScaleAddCols = ScaleAddCols2_AVX512BW; } #endif #if defined(HAS_SCALEADDROW_NEON) @@ -1096,12 +1050,6 @@ static int ScalePlaneBox_16(int src_width, void (*ScaleAddRow)(const uint16_t* src_ptr, uint32_t* dst_ptr, int src_width) = ScaleAddRow_16_C; -#if defined(HAS_SCALEADDROW_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(src_width, 16)) { - ScaleAddRow = ScaleAddRow_16_SSE2; - } -#endif - uint32_t uy = y; for (j = 0; j < dst_height; ++j) { int boxheight; @@ -1224,9 +1172,9 @@ static int ScalePlaneBilinearDown(int src_width, } #endif -#if defined(HAS_SCALEFILTERCOLS_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleFilterCols = ScaleFilterCols_SSSE3; +#if defined(HAS_SCALEFILTERCOLS_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_AVX2; } #endif #if defined(HAS_SCALEFILTERCOLS_NEON) @@ -1307,14 +1255,6 @@ static int ScalePlaneBilinearDown_16(int src_width, &dx, &dy); src_width = Abs(src_width); -#if defined(HAS_INTERPOLATEROW_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - InterpolateRow = InterpolateRow_16_Any_SSSE3; - if (IS_ALIGNED(src_width, 16)) { - InterpolateRow = InterpolateRow_16_SSSE3; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_16_Any_AVX2; @@ -1355,12 +1295,6 @@ static int ScalePlaneBilinearDown_16(int src_width, } #endif -#if defined(HAS_SCALEFILTERCOLS_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleFilterCols = ScaleFilterCols_16_SSSE3; - } -#endif - uint32_t uy = y; for (j = 0; j < dst_height; ++j) { if (uy > max_y) { @@ -1468,9 +1402,9 @@ static int ScalePlaneBilinearUp(int src_width, if (filtering && src_width >= 32768) { ScaleFilterCols = ScaleFilterCols64_C; } -#if defined(HAS_SCALEFILTERCOLS_SSSE3) - if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleFilterCols = ScaleFilterCols_SSSE3; +#if defined(HAS_SCALEFILTERCOLS_AVX2) + if (filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_AVX2; } #endif #if defined(HAS_SCALEFILTERCOLS_NEON) @@ -1499,9 +1433,9 @@ static int ScalePlaneBilinearUp(int src_width, #endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleFilterCols = ScaleColsUp2_C; -#if defined(HAS_SCALECOLSUP2_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleFilterCols = ScaleColsUp2_SSE2; +#if defined(HAS_SCALECOLSUP2_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleColsUp2_AVX2; } #endif } @@ -1937,14 +1871,6 @@ static int ScalePlaneBilinearUp_16(int src_width, assert(dy <= 65536); src_width = Abs(src_width); -#if defined(HAS_INTERPOLATEROW_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - InterpolateRow = InterpolateRow_16_Any_SSSE3; - if (IS_ALIGNED(dst_width, 16)) { - InterpolateRow = InterpolateRow_16_SSSE3; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_16_Any_AVX2; @@ -1988,18 +1914,8 @@ static int ScalePlaneBilinearUp_16(int src_width, if (filtering && src_width >= 32768) { ScaleFilterCols = ScaleFilterCols64_16_C; } -#if defined(HAS_SCALEFILTERCOLS_16_SSSE3) - if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleFilterCols = ScaleFilterCols_16_SSSE3; - } -#endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleFilterCols = ScaleColsUp2_16_C; -#if defined(HAS_SCALECOLS_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleFilterCols = ScaleColsUp2_16_SSE2; - } -#endif } uint32_t uy = y; @@ -2093,9 +2009,9 @@ static void ScalePlaneSimple(int src_width, if (src_width * 2 == dst_width && x < 0x8000) { ScaleCols = ScaleColsUp2_C; -#if defined(HAS_SCALECOLSUP2_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleCols = ScaleColsUp2_SSE2; +#if defined(HAS_SCALECOLSUP2_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) { + ScaleCols = ScaleColsUp2_AVX2; } #endif } @@ -2131,11 +2047,6 @@ static void ScalePlaneSimple_16(int src_width, if (src_width * 2 == dst_width && x < 0x8000) { ScaleCols = ScaleColsUp2_16_C; -#if defined(HAS_SCALECOLS_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleCols = ScaleColsUp2_16_SSE2; - } -#endif } uint32_t uy = y; diff --git a/source/scale_any.cc b/source/scale_any.cc index 713f3f186..ec903768e 100644 --- a/source/scale_any.cc +++ b/source/scale_any.cc @@ -50,6 +50,19 @@ extern "C" { dst_ptr + np * BPP, r + 1); \ } +// Fixed scale down for 16-bit planar. +#define SDANY16(NAMEANY, SCALEROWDOWN_SIMD, SCALEROWDOWN_C, FACTOR, MASK) \ + void NAMEANY(const uint16_t* src_ptr, ptrdiff_t src_stride, \ + uint16_t* dst_ptr, int dst_width) { \ + int r = (int)((unsigned int)dst_width % (MASK + 1)); /* NOLINT */ \ + int n = dst_width - r; \ + if (n > 0) { \ + SCALEROWDOWN_SIMD(src_ptr, src_stride, dst_ptr, n); \ + } \ + ptrdiff_t np = n; \ + SCALEROWDOWN_C(src_ptr + (np * FACTOR), src_stride, dst_ptr + np, r); \ + } + #ifdef HAS_SCALEROWDOWN2_SSSE3 SDANY(ScaleRowDown2_Any_SSSE3, ScaleRowDown2_SSSE3, ScaleRowDown2_C, 2, 1, 15) SDANY(ScaleRowDown2Linear_Any_SSSE3, @@ -71,13 +84,21 @@ SDODD(ScaleRowDown2Box_Odd_SSSE3, 1, 15) #endif -#ifdef HAS_SCALEUVROWDOWN2BOX_SSSE3 -SDANY(ScaleUVRowDown2Box_Any_SSSE3, - ScaleUVRowDown2Box_SSSE3, - ScaleUVRowDown2Box_C, +#ifdef HAS_SCALEUVROWDOWN2_AVX2 +SDANY(ScaleUVRowDown2_Any_AVX2, + ScaleUVRowDown2_AVX2, + ScaleUVRowDown2_C, 2, 2, - 3) + 7) +#endif +#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2 +SDANY(ScaleUVRowDown2Linear_Any_AVX2, + ScaleUVRowDown2Linear_AVX2, + ScaleUVRowDown2Linear_C, + 2, + 2, + 7) #endif #ifdef HAS_SCALEUVROWDOWN2BOX_AVX2 SDANY(ScaleUVRowDown2Box_Any_AVX2, @@ -108,6 +129,23 @@ SDODD(ScaleRowDown2Box_Odd_AVX2, 1, 31) #endif +#ifdef HAS_SCALEROWDOWN2_16_AVX2 +SDANY16(ScaleRowDown2_16_Any_AVX2, + ScaleRowDown2_16_AVX2, + ScaleRowDown2_16_C, + 2, + 15) +SDANY16(ScaleRowDown2Linear_16_Any_AVX2, + ScaleRowDown2Linear_16_AVX2, + ScaleRowDown2Linear_16_C, + 2, + 15) +SDANY16(ScaleRowDown2Box_16_Any_AVX2, + ScaleRowDown2Box_16_AVX2, + ScaleRowDown2Box_16_C, + 2, + 15) +#endif #ifdef HAS_SCALEROWDOWN2_NEON SDANY(ScaleRowDown2_Any_NEON, ScaleRowDown2_NEON, ScaleRowDown2_C, 2, 1, 15) SDANY(ScaleRowDown2Linear_Any_NEON, @@ -204,21 +242,21 @@ SDANY(ScaleRowDown4Box_Any_LSX, 1, 7) #endif -#ifdef HAS_SCALEROWDOWN34_SSSE3 -SDANY(ScaleRowDown34_Any_SSSE3, - ScaleRowDown34_SSSE3, +#ifdef HAS_SCALEROWDOWN34_AVX2 +SDANY(ScaleRowDown34_Any_AVX2, + ScaleRowDown34_AVX2, ScaleRowDown34_C, 4 / 3, 1, 23) -SDANY(ScaleRowDown34_0_Box_Any_SSSE3, - ScaleRowDown34_0_Box_SSSE3, +SDANY(ScaleRowDown34_0_Box_Any_AVX2, + ScaleRowDown34_0_Box_AVX2, ScaleRowDown34_0_Box_C, 4 / 3, 1, 23) -SDANY(ScaleRowDown34_1_Box_Any_SSSE3, - ScaleRowDown34_1_Box_SSSE3, +SDANY(ScaleRowDown34_1_Box_Any_AVX2, + ScaleRowDown34_1_Box_AVX2, ScaleRowDown34_1_Box_C, 4 / 3, 1, @@ -285,25 +323,25 @@ SDANY(ScaleRowDown34_1_Box_Any_LSX, 1, 11) #endif -#ifdef HAS_SCALEROWDOWN38_SSSE3 -SDANY(ScaleRowDown38_Any_SSSE3, - ScaleRowDown38_SSSE3, +#ifdef HAS_SCALEROWDOWN38_AVX2 +SDANY(ScaleRowDown38_Any_AVX2, + ScaleRowDown38_AVX2, ScaleRowDown38_C, 8 / 3, 1, 11) -SDANY(ScaleRowDown38_3_Box_Any_SSSE3, - ScaleRowDown38_3_Box_SSSE3, +SDANY(ScaleRowDown38_3_Box_Any_AVX2, + ScaleRowDown38_3_Box_AVX2, ScaleRowDown38_3_Box_C, 8 / 3, 1, - 5) -SDANY(ScaleRowDown38_2_Box_Any_SSSE3, - ScaleRowDown38_2_Box_SSSE3, + 11) +SDANY(ScaleRowDown38_2_Box_Any_AVX2, + ScaleRowDown38_2_Box_AVX2, ScaleRowDown38_2_Box_C, 8 / 3, 1, - 5) + 11) #endif #ifdef HAS_SCALEROWDOWN38_NEON SDANY(ScaleRowDown38_Any_NEON, @@ -346,25 +384,25 @@ SDANY(ScaleRowDown38_2_Box_Any_LSX, 11) #endif -#ifdef HAS_SCALEARGBROWDOWN2_SSE2 -SDANY(ScaleARGBRowDown2_Any_SSE2, - ScaleARGBRowDown2_SSE2, +#ifdef HAS_SCALEARGBROWDOWN2_AVX2 +SDANY(ScaleARGBRowDown2_Any_AVX2, + ScaleARGBRowDown2_AVX2, ScaleARGBRowDown2_C, 2, 4, - 3) -SDANY(ScaleARGBRowDown2Linear_Any_SSE2, - ScaleARGBRowDown2Linear_SSE2, + 7) +SDANY(ScaleARGBRowDown2Linear_Any_AVX2, + ScaleARGBRowDown2Linear_AVX2, ScaleARGBRowDown2Linear_C, 2, 4, - 3) -SDANY(ScaleARGBRowDown2Box_Any_SSE2, - ScaleARGBRowDown2Box_SSE2, + 7) +SDANY(ScaleARGBRowDown2Box_Any_AVX2, + ScaleARGBRowDown2Box_AVX2, ScaleARGBRowDown2Box_C, 2, 4, - 3) + 7) #endif #ifdef HAS_SCALEARGBROWDOWN2_NEON SDANY(ScaleARGBRowDown2_Any_NEON, @@ -506,6 +544,8 @@ SDANY(ScaleARGBRowDown2Box_Any_WASMSIMD, 4, 3) #endif +#undef SDANY16 +#undef SDODD #undef SDANY // Scale down by even scale factor. @@ -522,17 +562,17 @@ SDANY(ScaleARGBRowDown2Box_Any_WASMSIMD, dst_ptr + np * BPP, r); \ } -#ifdef HAS_SCALEARGBROWDOWNEVEN_SSE2 -SDAANY(ScaleARGBRowDownEven_Any_SSE2, - ScaleARGBRowDownEven_SSE2, +#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2 +SDAANY(ScaleARGBRowDownEven_Any_AVX2, + ScaleARGBRowDownEven_AVX2, ScaleARGBRowDownEven_C, 4, - 3) -SDAANY(ScaleARGBRowDownEvenBox_Any_SSE2, - ScaleARGBRowDownEvenBox_SSE2, + 7) +SDAANY(ScaleARGBRowDownEvenBox_Any_AVX2, + ScaleARGBRowDownEvenBox_AVX2, ScaleARGBRowDownEvenBox_C, 4, - 3) + 7) #endif #ifdef HAS_SCALEARGBROWDOWNEVEN_NEON SDAANY(ScaleARGBRowDownEven_Any_NEON, @@ -558,6 +598,13 @@ SDAANY(ScaleARGBRowDownEvenBox_Any_LSX, 4, 3) #endif +#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2 +SDAANY(ScaleUVRowDownEven_Any_AVX2, + ScaleUVRowDownEven_AVX2, + ScaleUVRowDownEven_C, + 2, + 7) +#endif #ifdef HAS_SCALEUVROWDOWNEVEN_NEON SDAANY(ScaleUVRowDownEven_Any_NEON, ScaleUVRowDownEven_NEON, @@ -1107,7 +1154,7 @@ SBU2BLANY(ScaleUVRowUp2_Bilinear_Any_AVX2, SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_SSE41, ScaleUVRowUp2_Bilinear_16_SSE41, ScaleUVRowUp2_Bilinear_16_C, - 7, + 3, uint16_t) #endif diff --git a/source/scale_argb.cc b/source/scale_argb.cc index dd9df3612..53280ea16 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -67,19 +67,19 @@ static void ScaleARGBDown2(int src_width, src_argb += (y >> 16) * src_stride + ((x >> 16) - 1) * 4; } -#if defined(HAS_SCALEARGBROWDOWN2_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { +#if defined(HAS_SCALEARGBROWDOWN2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { ScaleARGBRowDown2 = filtering == kFilterNone - ? ScaleARGBRowDown2_Any_SSE2 - : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_SSE2 - : ScaleARGBRowDown2Box_Any_SSE2); - if (IS_ALIGNED(dst_width, 4)) { + ? ScaleARGBRowDown2_Any_AVX2 + : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_AVX2 + : ScaleARGBRowDown2Box_Any_AVX2); + if (IS_ALIGNED(dst_width, 8)) { ScaleARGBRowDown2 = filtering == kFilterNone - ? ScaleARGBRowDown2_SSE2 - : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_SSE2 - : ScaleARGBRowDown2Box_SSE2); + ? ScaleARGBRowDown2_AVX2 + : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_AVX2 + : ScaleARGBRowDown2Box_AVX2); } } #endif @@ -205,11 +205,11 @@ static int ScaleARGBDown4Box(int src_width, (void)dx; assert(dx == 65536 * 4); // Test scale factor of 4. assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4. -#if defined(HAS_SCALEARGBROWDOWN2_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_SSE2; - if (IS_ALIGNED(dst_width, 4)) { - ScaleARGBRowDown2 = ScaleARGBRowDown2Box_SSE2; +#if defined(HAS_SCALEARGBROWDOWN2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_AVX2; } } #endif @@ -284,13 +284,13 @@ static void ScaleARGBDownEven(int src_width, assert(IS_ALIGNED(src_width, 2)); assert(IS_ALIGNED(src_height, 2)); src_argb += (y >> 16) * src_stride + (x >> 16) * 4; -#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_SSE2 - : ScaleARGBRowDownEven_Any_SSE2; - if (IS_ALIGNED(dst_width, 4)) { +#if defined(HAS_SCALEARGBROWDOWNEVEN_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_AVX2 + : ScaleARGBRowDownEven_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { ScaleARGBRowDownEven = - filtering ? ScaleARGBRowDownEvenBox_SSE2 : ScaleARGBRowDownEven_SSE2; + filtering ? ScaleARGBRowDownEvenBox_AVX2 : ScaleARGBRowDownEven_AVX2; } } #endif @@ -444,9 +444,9 @@ static int ScaleARGBBilinearDown(int src_width, } } #endif -#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3; +#if defined(HAS_SCALEARGBFILTERCOLS_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_AVX2; } #endif #if defined(HAS_SCALEARGBFILTERCOLS_NEON) @@ -607,9 +607,9 @@ static int ScaleARGBBilinearUp(int src_width, ScaleARGBFilterCols = filtering ? ScaleARGBFilterCols64_C : ScaleARGBCols64_C; } -#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3) - if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3; +#if defined(HAS_SCALEARGBFILTERCOLS_AVX2) + if (filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_AVX2; } #endif #if defined(HAS_SCALEARGBFILTERCOLS_NEON) @@ -633,9 +633,9 @@ static int ScaleARGBBilinearUp(int src_width, ScaleARGBFilterCols = ScaleARGBFilterCols_RVV; } #endif -#if defined(HAS_SCALEARGBCOLS_SSE2) - if (!filtering && TestCpuFlag(kCpuHasSSE2) && src_width < 32768) { - ScaleARGBFilterCols = ScaleARGBCols_SSE2; +#if defined(HAS_SCALEARGBCOLS_AVX2) + if (!filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBCols_AVX2; } #endif #if defined(HAS_SCALEARGBCOLS_NEON) @@ -672,9 +672,9 @@ static int ScaleARGBBilinearUp(int src_width, #endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleARGBFilterCols = ScaleARGBColsUp2_C; -#if defined(HAS_SCALEARGBCOLSUP2_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleARGBFilterCols = ScaleARGBColsUp2_SSE2; +#if defined(HAS_SCALEARGBCOLSUP2_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBColsUp2_AVX2; } #endif #if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD) @@ -772,9 +772,9 @@ static void ScaleARGBSimple(int src_width, int dst_width, int x, int dx) = (src_width >= 32768) ? ScaleARGBCols64_C : ScaleARGBCols_C; const int64_t max_y = (int64_t)(src_height - 1) << 16; -#if defined(HAS_SCALEARGBCOLS_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && src_width < 32768) { - ScaleARGBCols = ScaleARGBCols_SSE2; +#if defined(HAS_SCALEARGBCOLS_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleARGBCols = ScaleARGBCols_AVX2; } #endif #if defined(HAS_SCALEARGBCOLS_NEON) @@ -803,9 +803,9 @@ static void ScaleARGBSimple(int src_width, #endif if (src_width * 2 == dst_width && x < 0x8000) { ScaleARGBCols = ScaleARGBColsUp2_C; -#if defined(HAS_SCALEARGBCOLSUP2_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleARGBCols = ScaleARGBColsUp2_SSE2; +#if defined(HAS_SCALEARGBCOLSUP2_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) { + ScaleARGBCols = ScaleARGBColsUp2_AVX2; } #endif #if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD) diff --git a/source/scale_common.cc b/source/scale_common.cc index 0f1e7c845..a07a2183e 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1155,6 +1155,86 @@ void ScaleUVRowDownEvenBox_C(const uint8_t* src_uv, } } +void ScaleUVRowDown2_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width) { + int x; + (void)src_stride; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = src_uv[2]; // Store the 2nd UV + dst_uv[1] = src_uv[3]; + src_uv += 4; + dst_uv += 2; + } +} + +void ScaleUVRowDown2Linear_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width) { + int x; + (void)src_stride; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] + 1) >> 1); + dst_uv[1] = (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + 1) >> 1); + src_uv += 4; + dst_uv += 2; + } +} + +void ScaleUVRowDown2Box_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width) { + int x; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] + + src_uv[src_stride] + src_uv[src_stride + 2] + 2) >> + 2); + dst_uv[1] = + (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + src_uv[src_stride + 1] + + src_uv[src_stride + 3] + 2) >> + 2); + src_uv += 4; + dst_uv += 2; + } +} + +void ScaleUVRowDownEven_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint16_t* dst_uv, + int dst_width) { + int x; + (void)src_stride; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = src_uv[0]; + dst_uv[1] = src_uv[1]; + src_uv += src_stepx * 2; + dst_uv += 2; + } +} + +void ScaleUVRowDownEvenBox_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint16_t* dst_uv, + int dst_width) { + int x; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] + + src_uv[src_stride] + src_uv[src_stride + 2] + 2) >> + 2); + dst_uv[1] = + (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + src_uv[src_stride + 1] + + src_uv[src_stride + 3] + 2) >> + 2); + src_uv += src_stepx * 2; + dst_uv += 2; + } +} + void ScaleUVRowUp2_Linear_C(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width) { @@ -1382,6 +1462,118 @@ void ScaleUVAddCols3_C(int dst_width, } } +// Scales a single row of 16-bit UV pixels using point sampling. +void ScaleUVCols_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx) { + int j; + for (j = 0; j < dst_width; ++j) { + const uint16_t* src = src_uv + (x >> 16) * 2; + dst_uv[0] = src[0]; + dst_uv[1] = src[1]; + dst_uv += 2; + x += dx; + } +} + +void ScaleUVCols64_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x32, + int dx) { + int64_t x = (int64_t)(x32); + int j; + for (j = 0; j < dst_width; ++j) { + const uint16_t* src = src_uv + (x >> 16) * 2; + dst_uv[0] = src[0]; + dst_uv[1] = src[1]; + dst_uv += 2; + x += dx; + } +} + +// Scales a single row of 16-bit UV pixels up by 2x using point sampling. +void ScaleUVColsUp2_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx) { + int j; + (void)x; + (void)dx; + for (j = 0; j < dst_width - 1; j += 2) { + dst_uv[0] = dst_uv[2] = src_uv[0]; + dst_uv[1] = dst_uv[3] = src_uv[1]; + src_uv += 2; + dst_uv += 4; + } + if (dst_width & 1) { + dst_uv[0] = src_uv[0]; + dst_uv[1] = src_uv[1]; + } +} + +#define BLENDER_16(a, b, f) \ + (uint16_t)((int)(a) + \ + (int)((((int64_t)((f)) * ((int64_t)(b) - (int)(a))) + 0x8000) >> \ + 16)) + +void ScaleUVFilterCols_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx) { + int j; + for (j = 0; j < dst_width; ++j) { + int xi = (x >> 16) * 2; + int xf = x & 0xffff; + const uint16_t* a = src_uv + xi; + const uint16_t* b = a + 2; + dst_uv[0] = BLENDER_16(a[0], b[0], xf); + dst_uv[1] = BLENDER_16(a[1], b[1], xf); + dst_uv += 2; + x += dx; + } +} + +void ScaleUVFilterCols64_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x32, + int dx) { + int64_t x = (int64_t)(x32); + int j; + for (j = 0; j < dst_width; ++j) { + int64_t xi = (x >> 16) * 2; + int xf = x & 0xffff; + const uint16_t* a = src_uv + xi; + const uint16_t* b = a + 2; + dst_uv[0] = BLENDER_16(a[0], b[0], xf); + dst_uv[1] = BLENDER_16(a[1], b[1], xf); + dst_uv += 2; + x += dx; + } +} +#undef BLENDER_16 + +// Scale 16-bit UV columns by 3 box filter. +void ScaleUVAddCols3_16_C(int dst_width, + int scaleval, + const uint32_t* src_uv, + uint16_t* dst_uv) { + int i; + for (i = 0; i < dst_width; ++i) { + uint32_t u = src_uv[0] + src_uv[2] + src_uv[4]; + uint32_t v = src_uv[1] + src_uv[3] + src_uv[5]; + src_uv += 6; + dst_uv[0] = (uint16_t)((u * (uint32_t)scaleval) >> 16); + dst_uv[1] = (uint16_t)((v * (uint32_t)scaleval) >> 16); + dst_uv += 2; + } +} + // Scale plane vertically with bilinear interpolation. void ScalePlaneVertical(int src_height, int dst_width, @@ -1536,14 +1728,6 @@ void ScalePlaneVertical_16(int src_height, assert(dst_width > 0); assert(dst_height > 0); src_argb += (x >> 16) * wpp; -#if defined(HAS_INTERPOLATEROW_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - InterpolateRow = InterpolateRow_16_Any_SSSE3; - if (IS_ALIGNED(dst_width_words, 16)) { - InterpolateRow = InterpolateRow_16_SSSE3; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_16_Any_AVX2; diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index b018af795..dc0383c37 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -28,6 +28,7 @@ extern "C" { (defined(__x86_64__) || defined(__i386__)) && \ !defined(LIBYUV_ENABLE_ROWWIN) +#ifdef HAS_SCALEROWDOWN34_AVX2 // Offsets for source bytes 0 to 9 static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9, 128, 128, 128, 128, 128, 128, 128, 128}; @@ -62,7 +63,9 @@ static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3}; // Coefficients for source bytes 21 to 31 static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2}; +#endif // HAS_SCALEROWDOWN34_AVX2 +#ifdef HAS_SCALEROWDOWN38_AVX2 static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}; @@ -96,6 +99,7 @@ static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128, // Scaling values for boxes of 3x2 and 2x2 static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3, 65536 / 3, 65536 / 2, 0, 0}; +#endif // HAS_SCALEROWDOWN38_AVX2 // GCC versions of row functions are verbatim conversions from Visual C. // Generated using gcc disassembly on Visual C object file: @@ -298,6 +302,118 @@ void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr, } #endif // HAS_SCALEROWDOWN2_AVX2 +#ifdef HAS_SCALEROWDOWN2_16_AVX2 +void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + (void)src_stride; + asm volatile( + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm0 \n" + "vpsrld $0x10,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "xmm0", "xmm1"); +} + +void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + (void)src_stride; + asm volatile( + "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n" + "vpsrld $0x10,%%ymm4,%%ymm4 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm2 \n" + "vpsrld $0x10,%%ymm1,%%ymm3 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpand %%ymm4,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpackusdw %%ymm3,%%ymm2,%%ymm2 \n" + "vpavgw %%ymm2,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); +} + +void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + asm volatile( + "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrld $0x1f,%%ymm6,%%ymm7 \n" + "vpslld $0x1,%%ymm7,%%ymm7 \n" + "vpsrld $0x10,%%ymm6,%%ymm6 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vmovdqu 0x00(%0,%3,2),%%ymm2 \n" + "vmovdqu 0x20(%0,%3,2),%%ymm3 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm4 \n" + "vpand %%ymm6,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm4,%%ymm0,%%ymm0 \n" + "vpsrld $0x10,%%ymm2,%%ymm4 \n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm4,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vpsrld $0x10,%%ymm1,%%ymm4 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpaddd %%ymm4,%%ymm1,%%ymm1 \n" + "vpsrld $0x10,%%ymm3,%%ymm4 \n" + "vpand %%ymm6,%%ymm3,%%ymm3 \n" + "vpaddd %%ymm4,%%ymm3,%%ymm3 \n" + "vpaddd %%ymm3,%%ymm1,%%ymm1 \n" + "vpaddd %%ymm7,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm7,%%ymm1,%%ymm1 \n" + "vpsrld $0x2,%%ymm0,%%ymm0 \n" + "vpsrld $0x2,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "r"(src_stride) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6", "xmm7"); +} +#endif // HAS_SCALEROWDOWN2_16_AVX2 + #ifdef HAS_SCALEROWDOWN2_AVX512BW void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -624,266 +740,418 @@ void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr, } #endif // HAS_SCALEROWDOWN4_AVX2 -void ScaleRowDown34_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +#ifdef HAS_SCALEROWDOWN34_AVX2 +void ScaleRowDown34_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { (void)src_stride; asm volatile( - "movdqa %[kShuf0],%%xmm3 \n" - "movdqa %[kShuf1],%%xmm4 \n" - "movdqa %[kShuf2],%%xmm5 \n" - "1: \n" - "movdqu (%[src_ptr]),%%xmm0 \n" - "movdqu 0x10(%[src_ptr]),%%xmm2 \n" - "lea 0x20(%[src_ptr]),%[src_ptr] \n" - "movdqa %%xmm2,%%xmm1 \n" - "palignr $0x8,%%xmm0,%%xmm1 \n" - "pshufb %%xmm3,%%xmm0 \n" - "pshufb %%xmm4,%%xmm1 \n" - "pshufb %%xmm5,%%xmm2 \n" - "movq %%xmm0,(%[dst_ptr]) \n" - "movq %%xmm1,0x8(%[dst_ptr]) \n" - "movq %%xmm2,0x10(%[dst_ptr]) \n" - "lea 0x18(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x18,%[dst_width] \n" - "jg 1b \n" - : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) - : [kShuf0] "m"(kShuf0), [kShuf1] "m"(kShuf1), [kShuf2] "m"(kShuf2) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); -} - -void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - asm volatile( - "movdqa %[kShuf01],%%xmm2 \n" - "movdqa %[kShuf11],%%xmm3 \n" - "movdqa %[kShuf21],%%xmm4 \n" - "movdqa %[kMadd01],%%xmm5 \n" - "movdqa %[kMadd11],%%xmm0 \n" - "movdqa %[kRound34],%%xmm1 \n" - "1: \n" - "movdqu (%[src_ptr]),%%xmm6 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm2,%%xmm6 \n" - "pmaddubsw %%xmm5,%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,(%[dst_ptr]) \n" - "movdqu 0x8(%[src_ptr]),%%xmm6 \n" - "movdqu 0x8(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm3,%%xmm6 \n" - "pmaddubsw %%xmm0,%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,0x8(%[dst_ptr]) \n" - "movdqu 0x10(%[src_ptr]),%%xmm6 \n" - "movdqu 0x10(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "lea 0x20(%[src_ptr]),%[src_ptr] \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm4,%%xmm6 \n" - "pmaddubsw %[kMadd21],%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,0x10(%[dst_ptr]) \n" - "lea 0x18(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x18,%[dst_width] \n" - "jg 1b \n" - : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) - : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), - [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), - [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); -} - -void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - asm volatile( - "movdqa %[kShuf01],%%xmm2 \n" // kShuf01 - "movdqa %[kShuf11],%%xmm3 \n" // kShuf11 - "movdqa %[kShuf21],%%xmm4 \n" // kShuf21 - "movdqa %[kMadd01],%%xmm5 \n" // kMadd01 - "movdqa %[kMadd11],%%xmm0 \n" // kMadd11 - "movdqa %[kRound34],%%xmm1 \n" // kRound34 - "1: \n" - "movdqu (%[src_ptr]),%%xmm6 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "pavgb %%xmm6,%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm2,%%xmm6 \n" - "pmaddubsw %%xmm5,%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,(%[dst_ptr]) \n" - "movdqu 0x8(%[src_ptr]),%%xmm6 \n" - "movdqu 0x8(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "pavgb %%xmm6,%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm3,%%xmm6 \n" - "pmaddubsw %%xmm0,%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,0x8(%[dst_ptr]) \n" - "movdqu 0x10(%[src_ptr]),%%xmm6 \n" - "movdqu 0x10(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "lea 0x20(%[src_ptr]),%[src_ptr] \n" - "pavgb %%xmm6,%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm4,%%xmm6 \n" - "pmaddubsw %[kMadd21],%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,0x10(%[dst_ptr]) \n" - "lea 0x18(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x18,%[dst_width] \n" - "jg 1b \n" - : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) - : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), - [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), - [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); -} - -void ScaleRowDown38_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - (void)src_stride; - asm volatile( - "movdqa %3,%%xmm4 \n" - "movdqa %4,%%xmm5 \n" + "vbroadcasti128 %[kShuf0],%%ymm3 \n" + "vbroadcasti128 %[kShuf1],%%ymm4 \n" + "vbroadcasti128 %[kShuf2],%%ymm5 \n" + "sub $0x30,%[dst_width] \n" + "jl 2f \n" LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "lea 0x20(%0),%0 \n" - "pshufb %%xmm4,%%xmm0 \n" - "pshufb %%xmm5,%%xmm1 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movq %%xmm0,(%1) \n" - "movhlps %%xmm0,%%xmm1 \n" - "movd %%xmm1,0x8(%1) \n" - "lea 0xc(%1),%1 \n" - "sub $0xc,%2 \n" - "jg 1b \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x20(%[src_ptr]),%%ymm1 \n" + "lea 0x40(%[src_ptr]),%[src_ptr] \n" + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n" + "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm6 \n" + "vpalignr $0x8,%%ymm2,%%ymm6,%%ymm1 \n" + "vpshufb %%ymm3,%%ymm2,%%ymm0 \n" + "vpshufb %%ymm4,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm5,%%ymm6,%%ymm2 \n" + "vpunpcklqdq %%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%[dst_ptr]) \n" + "vmovq %%xmm2,0x10(%[dst_ptr]) \n" + "vextracti128 $0x1,%%ymm0,%%xmm0 \n" + "vextracti128 $0x1,%%ymm2,%%xmm2 \n" + "vmovdqu %%xmm0,0x18(%[dst_ptr]) \n" + "vmovq %%xmm2,0x28(%[dst_ptr]) \n" + "lea 0x30(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x30,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x30,%[dst_width] \n" + "jle 99f \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm2 \n" + "vpalignr $0x8,%%xmm0,%%xmm2,%%xmm1 \n" + "vpshufb %%xmm3,%%xmm0,%%xmm0 \n" + "vpshufb %%xmm4,%%xmm1,%%xmm1 \n" + "vpshufb %%xmm5,%%xmm2,%%xmm2 \n" + "vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst_ptr]) \n" + "vmovq %%xmm2,0x10(%[dst_ptr]) \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), + [dst_width] "+r"(dst_width) + : [kShuf0] "m"(kShuf0), [kShuf1] "m"(kShuf1), [kShuf2] "m"(kShuf2) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); +} + +void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + asm volatile( + "vbroadcasti128 %[kRound34],%%ymm5 \n" + "sub $0x30,%[dst_width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm2 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu 0x20(%[src_ptr]),%%ymm1 \n" + "vmovdqu 0x20(%[src_ptr],%[src_stride],1),%%ymm3 \n" + "lea 0x40(%[src_ptr]),%[src_ptr] \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n" + "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm3 \n" + "vpalignr $0x8,%%ymm2,%%ymm3,%%ymm4 \n" + + "vbroadcasti128 %[kShuf01],%%ymm6 \n" + "vbroadcasti128 %[kMadd01],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n" + "vpaddsw %%ymm5,%%ymm2,%%ymm2 \n" + "vpsrlw $0x2,%%ymm2,%%ymm2 \n" + "vpackuswb %%ymm2,%%ymm2,%%ymm2 \n" + + "vbroadcasti128 %[kShuf11],%%ymm6 \n" + "vbroadcasti128 %[kMadd11],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm4,%%ymm4 \n" + "vpmaddubsw %%ymm7,%%ymm4,%%ymm4 \n" + "vpaddsw %%ymm5,%%ymm4,%%ymm4 \n" + "vpsrlw $0x2,%%ymm4,%%ymm4 \n" + "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n" + + "vbroadcasti128 %[kShuf21],%%ymm6 \n" + "vbroadcasti128 %[kMadd21],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm3 \n" + "vpmaddubsw %%ymm7,%%ymm3,%%ymm3 \n" + "vpaddsw %%ymm5,%%ymm3,%%ymm3 \n" + "vpsrlw $0x2,%%ymm3,%%ymm3 \n" + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" + + "vpunpcklqdq %%ymm4,%%ymm2,%%ymm2 \n" + "vmovdqu %%xmm2,(%[dst_ptr]) \n" + "vmovq %%xmm3,0x10(%[dst_ptr]) \n" + "vextracti128 $0x1,%%ymm2,%%xmm0 \n" + "vextracti128 $0x1,%%ymm3,%%xmm1 \n" + "vmovdqu %%xmm0,0x18(%[dst_ptr]) \n" + "vmovq %%xmm1,0x28(%[dst_ptr]) \n" + "lea 0x30(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x30,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x30,%[dst_width] \n" + "jle 99f \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm1 \n" + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm3 \n" + "vpalignr $0x8,%%xmm0,%%xmm3,%%xmm4 \n" + + "vpshufb %[kShuf01],%%xmm0,%%xmm0 \n" + "vpmaddubsw %[kMadd01],%%xmm0,%%xmm0 \n" + "vpaddsw %%xmm5,%%xmm0,%%xmm0 \n" + "vpsrlw $0x2,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + + "vpshufb %[kShuf11],%%xmm4,%%xmm4 \n" + "vpmaddubsw %[kMadd11],%%xmm4,%%xmm4 \n" + "vpaddsw %%xmm5,%%xmm4,%%xmm4 \n" + "vpsrlw $0x2,%%xmm4,%%xmm4 \n" + "vpackuswb %%xmm4,%%xmm4,%%xmm4 \n" + + "vpshufb %[kShuf21],%%xmm3,%%xmm3 \n" + "vpmaddubsw %[kMadd21],%%xmm3,%%xmm3 \n" + "vpaddsw %%xmm5,%%xmm3,%%xmm3 \n" + "vpsrlw $0x2,%%xmm3,%%xmm3 \n" + "vpackuswb %%xmm3,%%xmm3,%%xmm3 \n" + + "vpunpcklqdq %%xmm4,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst_ptr]) \n" + "vmovq %%xmm3,0x10(%[dst_ptr]) \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), + [dst_width] "+rm"(dst_width) + : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), + [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), + [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} + +void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + asm volatile( + "vbroadcasti128 %[kRound34],%%ymm5 \n" + "sub $0x30,%[dst_width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm2 \n" + "vpavgb %%ymm0,%%ymm2,%%ymm2 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu 0x20(%[src_ptr]),%%ymm1 \n" + "vmovdqu 0x20(%[src_ptr],%[src_stride],1),%%ymm3 \n" + "lea 0x40(%[src_ptr]),%[src_ptr] \n" + "vpavgb %%ymm1,%%ymm3,%%ymm3 \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n" + "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm3 \n" + "vpalignr $0x8,%%ymm2,%%ymm3,%%ymm4 \n" + + "vbroadcasti128 %[kShuf01],%%ymm6 \n" + "vbroadcasti128 %[kMadd01],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n" + "vpaddsw %%ymm5,%%ymm2,%%ymm2 \n" + "vpsrlw $0x2,%%ymm2,%%ymm2 \n" + "vpackuswb %%ymm2,%%ymm2,%%ymm2 \n" + + "vbroadcasti128 %[kShuf11],%%ymm6 \n" + "vbroadcasti128 %[kMadd11],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm4,%%ymm4 \n" + "vpmaddubsw %%ymm7,%%ymm4,%%ymm4 \n" + "vpaddsw %%ymm5,%%ymm4,%%ymm4 \n" + "vpsrlw $0x2,%%ymm4,%%ymm4 \n" + "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n" + + "vbroadcasti128 %[kShuf21],%%ymm6 \n" + "vbroadcasti128 %[kMadd21],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm3 \n" + "vpmaddubsw %%ymm7,%%ymm3,%%ymm3 \n" + "vpaddsw %%ymm5,%%ymm3,%%ymm3 \n" + "vpsrlw $0x2,%%ymm3,%%ymm3 \n" + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" + + "vpunpcklqdq %%ymm4,%%ymm2,%%ymm2 \n" + "vmovdqu %%xmm2,(%[dst_ptr]) \n" + "vmovq %%xmm3,0x10(%[dst_ptr]) \n" + "vextracti128 $0x1,%%ymm2,%%xmm0 \n" + "vextracti128 $0x1,%%ymm3,%%xmm1 \n" + "vmovdqu %%xmm0,0x18(%[dst_ptr]) \n" + "vmovq %%xmm1,0x28(%[dst_ptr]) \n" + "lea 0x30(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x30,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x30,%[dst_width] \n" + "jle 99f \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm1 \n" + "vpavgb %%ymm0,%%ymm1,%%ymm1 \n" + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm3 \n" + "vpalignr $0x8,%%xmm0,%%xmm3,%%xmm4 \n" + + "vpshufb %[kShuf01],%%xmm0,%%xmm0 \n" + "vpmaddubsw %[kMadd01],%%xmm0,%%xmm0 \n" + "vpaddsw %%xmm5,%%xmm0,%%xmm0 \n" + "vpsrlw $0x2,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + + "vpshufb %[kShuf11],%%xmm4,%%xmm4 \n" + "vpmaddubsw %[kMadd11],%%xmm4,%%xmm4 \n" + "vpaddsw %%xmm5,%%xmm4,%%xmm4 \n" + "vpsrlw $0x2,%%xmm4,%%xmm4 \n" + "vpackuswb %%xmm4,%%xmm4,%%xmm4 \n" + + "vpshufb %[kShuf21],%%xmm3,%%xmm3 \n" + "vpmaddubsw %[kMadd21],%%xmm3,%%xmm3 \n" + "vpaddsw %%xmm5,%%xmm3,%%xmm3 \n" + "vpsrlw $0x2,%%xmm3,%%xmm3 \n" + "vpackuswb %%xmm3,%%xmm3,%%xmm3 \n" + + "vpunpcklqdq %%xmm4,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst_ptr]) \n" + "vmovq %%xmm3,0x10(%[dst_ptr]) \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), + [dst_width] "+rm"(dst_width) + : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), + [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), + [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_SCALEROWDOWN34_AVX2 + +#ifdef HAS_SCALEROWDOWN38_AVX2 +void ScaleRowDown38_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + asm volatile( + "vmovdqu %3,%%xmm4 \n" + "vinserti128 $0x1,%4,%%ymm4,%%ymm4 \n" + "sub $0x18,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm4,%%ymm1,%%ymm1 \n" + "vextracti128 $0x1,%%ymm0,%%xmm2 \n" + "vextracti128 $0x1,%%ymm1,%%xmm3 \n" + "vpor %%xmm2,%%xmm0,%%xmm0 \n" + "vpor %%xmm3,%%xmm1,%%xmm1 \n" + "vmovq %%xmm0,(%1) \n" + "vpsrldq $0x8,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,0x8(%1) \n" + "vmovq %%xmm1,0xc(%1) \n" + "vpsrldq $0x8,%%xmm1,%%xmm1 \n" + "vmovd %%xmm1,0x14(%1) \n" + "lea 0x18(%1),%1 \n" + "sub $0x18,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x18,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm2 \n" + "vpor %%xmm2,%%xmm0,%%xmm0 \n" + "vmovq %%xmm0,(%1) \n" + "vpsrldq $0x8,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,0x8(%1) \n" + + "99: \n" + "vzeroupper \n" : "+r"(src_ptr), // %0 "+r"(dst_ptr), // %1 "+r"(dst_width) // %2 : "m"(kShuf38a), // %3 "m"(kShuf38b) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); } -void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { asm volatile( - "movdqa %[kShufAb0],%%xmm2 \n" - "movdqa %[kShufAb1],%%xmm3 \n" - "movdqa %[kShufAb2],%%xmm4 \n" - "movdqa %[kScaleAb2],%%xmm5 \n" + "vbroadcasti128 %[kShufAb0],%%ymm2 \n" + "vbroadcasti128 %[kShufAb1],%%ymm3 \n" + "vbroadcasti128 %[kShufAb2],%%ymm4 \n" + "vbroadcasti128 %[kScaleAb2],%%ymm5 \n" + "vpxor %%ymm7,%%ymm7,%%ymm7 \n" + + LABELALIGN "1: \n" - "movdqu (%[src_ptr]),%%xmm0 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm1 \n" - "lea 0x10(%[src_ptr]),%[src_ptr] \n" - "pavgb %%xmm1,%%xmm0 \n" - "movdqa %%xmm0,%%xmm1 \n" - "pshufb %%xmm2,%%xmm1 \n" - "movdqa %%xmm0,%%xmm6 \n" - "pshufb %%xmm3,%%xmm6 \n" - "paddusw %%xmm6,%%xmm1 \n" - "pshufb %%xmm4,%%xmm0 \n" - "paddusw %%xmm0,%%xmm1 \n" - "pmulhuw %%xmm5,%%xmm1 \n" - "packuswb %%xmm1,%%xmm1 \n" - "movd %%xmm1,(%[dst_ptr]) \n" - "psrlq $0x10,%%xmm1 \n" - "movd %%xmm1,0x2(%[dst_ptr]) \n" - "lea 0x6(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x6,%[dst_width] \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm1 \n" + "lea 0x20(%[src_ptr]),%[src_ptr] \n" + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm2,%%ymm0,%%ymm1 \n" + "vpshufb %%ymm3,%%ymm0,%%ymm6 \n" + "vpaddusw %%ymm6,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" + "vpaddusw %%ymm0,%%ymm1,%%ymm1 \n" + "vpmulhuw %%ymm5,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm7,%%ymm1,%%ymm1 \n" + "vextracti128 $0x1,%%ymm1,%%xmm0 \n" + "vpslldq $0x6,%%xmm0,%%xmm0 \n" + "vpor %%xmm0,%%xmm1,%%xmm1 \n" + "vmovq %%xmm1,(%[dst_ptr]) \n" + "vpsrldq $0x8,%%xmm1,%%xmm1 \n" + "vmovd %%xmm1,0x8(%[dst_ptr]) \n" + "lea 0xc(%[dst_ptr]),%[dst_ptr] \n" + "sub $0xc,%[dst_width] \n" "jg 1b \n" + "vzeroupper \n" : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) + [dst_width] "+rm"(dst_width) : [src_stride] "r"(src_stride), [kShufAb0] "m"(kShufAb0), [kShufAb1] "m"(kShufAb1), [kShufAb2] "m"(kShufAb2), [kScaleAb2] "m"(kScaleAb2) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); } -void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { asm volatile( - "movdqa %[kShufAc],%%xmm2 \n" - "movdqa %[kShufAc3],%%xmm3 \n" - "movdqa %[kScaleAc33],%%xmm4 \n" - "pxor %%xmm5,%%xmm5 \n" + "vbroadcasti128 %[kShufAc],%%ymm2 \n" + "vbroadcasti128 %[kShufAc3],%%ymm3 \n" + "vbroadcasti128 %[kScaleAc33],%%ymm4 \n" + "vpxor %%ymm5,%%ymm5,%%ymm5 \n" + + LABELALIGN "1: \n" - "movdqu (%[src_ptr]),%%xmm0 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm6 \n" - "movhlps %%xmm0,%%xmm1 \n" - "movhlps %%xmm6,%%xmm7 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm6 \n" - "punpcklbw %%xmm5,%%xmm7 \n" - "paddusw %%xmm6,%%xmm0 \n" - "paddusw %%xmm7,%%xmm1 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],2),%%xmm6 \n" - "lea 0x10(%[src_ptr]),%[src_ptr] \n" - "movhlps %%xmm6,%%xmm7 \n" - "punpcklbw %%xmm5,%%xmm6 \n" - "punpcklbw %%xmm5,%%xmm7 \n" - "paddusw %%xmm6,%%xmm0 \n" - "paddusw %%xmm7,%%xmm1 \n" - "movdqa %%xmm0,%%xmm6 \n" - "psrldq $0x2,%%xmm0 \n" - "paddusw %%xmm0,%%xmm6 \n" - "psrldq $0x2,%%xmm0 \n" - "paddusw %%xmm0,%%xmm6 \n" - "pshufb %%xmm2,%%xmm6 \n" - "movdqa %%xmm1,%%xmm7 \n" - "psrldq $0x2,%%xmm1 \n" - "paddusw %%xmm1,%%xmm7 \n" - "psrldq $0x2,%%xmm1 \n" - "paddusw %%xmm1,%%xmm7 \n" - "pshufb %%xmm3,%%xmm7 \n" - "paddusw %%xmm7,%%xmm6 \n" - "pmulhuw %%xmm4,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movd %%xmm6,(%[dst_ptr]) \n" - "psrlq $0x10,%%xmm6 \n" - "movd %%xmm6,0x2(%[dst_ptr]) \n" - "lea 0x6(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x6,%[dst_width] \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm6 \n" + "vpunpckhbw %%ymm5,%%ymm0,%%ymm1 \n" + "vpunpcklbw %%ymm5,%%ymm0,%%ymm0 \n" + "vpunpckhbw %%ymm5,%%ymm6,%%ymm7 \n" + "vpunpcklbw %%ymm5,%%ymm6,%%ymm6 \n" + "vpaddusw %%ymm6,%%ymm0,%%ymm0 \n" + "vpaddusw %%ymm7,%%ymm1,%%ymm1 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],2),%%ymm6 \n" + "lea 0x20(%[src_ptr]),%[src_ptr] \n" + "vpunpckhbw %%ymm5,%%ymm6,%%ymm7 \n" + "vpunpcklbw %%ymm5,%%ymm6,%%ymm6 \n" + "vpaddusw %%ymm6,%%ymm0,%%ymm0 \n" + "vpaddusw %%ymm7,%%ymm1,%%ymm1 \n" + "vpsrldq $0x2,%%ymm0,%%ymm6 \n" + "vpaddusw %%ymm0,%%ymm6,%%ymm6 \n" + "vpsrldq $0x4,%%ymm0,%%ymm0 \n" + "vpaddusw %%ymm0,%%ymm6,%%ymm6 \n" + "vpshufb %%ymm2,%%ymm6,%%ymm6 \n" + "vpsrldq $0x2,%%ymm1,%%ymm7 \n" + "vpaddusw %%ymm1,%%ymm7,%%ymm7 \n" + "vpsrldq $0x4,%%ymm1,%%ymm1 \n" + "vpaddusw %%ymm1,%%ymm7,%%ymm7 \n" + "vpshufb %%ymm3,%%ymm7,%%ymm7 \n" + "vpaddusw %%ymm7,%%ymm6,%%ymm6 \n" + "vpmulhuw %%ymm4,%%ymm6,%%ymm6 \n" + "vpackuswb %%ymm5,%%ymm6,%%ymm6 \n" + "vextracti128 $0x1,%%ymm6,%%xmm0 \n" + "vpslldq $0x6,%%xmm0,%%xmm0 \n" + "vpor %%xmm0,%%xmm6,%%xmm6 \n" + "vmovq %%xmm6,(%[dst_ptr]) \n" + "vpsrldq $0x8,%%xmm6,%%xmm6 \n" + "vmovd %%xmm6,0x8(%[dst_ptr]) \n" + "lea 0xc(%[dst_ptr]),%[dst_ptr] \n" + "sub $0xc,%[dst_width] \n" "jg 1b \n" + "vzeroupper \n" : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) + [dst_width] "+rm"(dst_width) : [src_stride] "r"(src_stride), [kShufAc] "m"(kShufAc), [kShufAc3] "m"(kShufAc3), [kScaleAc33] "m"(kScaleAc33) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } +#endif // HAS_SCALEROWDOWN38_AVX2 static const uvec8 kLinearShuffleFar = {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}; @@ -1245,8 +1513,13 @@ void ScaleRowUp2_Linear_16_SSE2(const uint16_t* src_ptr, "psrld $2,%%xmm0 \n" // 3/4*near+1/4*far (lo) "psrld $2,%%xmm1 \n" // 3/4*near+1/4*far (hi) - "packssdw %%xmm1,%%xmm0 \n" - "pshufd $0b11011000,%%xmm0,%%xmm0 \n" + "pshuflw $0x88,%%xmm0,%%xmm0 \n" + "pshufhw $0x88,%%xmm0,%%xmm0 \n" + "pshufd $0x88,%%xmm0,%%xmm0 \n" + "pshuflw $0x88,%%xmm1,%%xmm1 \n" + "pshufhw $0x88,%%xmm1,%%xmm1 \n" + "pshufd $0x88,%%xmm1,%%xmm1 \n" + "punpckldq %%xmm1,%%xmm0 \n" "movdqu %%xmm0,(%1) \n" "lea 0x8(%0),%0 \n" @@ -1335,11 +1608,21 @@ void ScaleRowUp2_Bilinear_16_SSE2(const uint16_t* src_ptr, "paddd %%xmm1,%%xmm2 \n" // 9 3 3 1 + 8 (2, hi) "psrld $4,%%xmm2 \n" // ^ div by 16 (2, hi) - "packssdw %%xmm0,%%xmm4 \n" - "pshufd $0b11011000,%%xmm4,%%xmm4 \n" + "pshuflw $0x88,%%xmm4,%%xmm4 \n" + "pshufhw $0x88,%%xmm4,%%xmm4 \n" + "pshufd $0x88,%%xmm4,%%xmm4 \n" + "pshuflw $0x88,%%xmm0,%%xmm0 \n" + "pshufhw $0x88,%%xmm0,%%xmm0 \n" + "pshufd $0x88,%%xmm0,%%xmm0 \n" + "punpckldq %%xmm0,%%xmm4 \n" "movdqu %%xmm4,(%1) \n" // store above - "packssdw %%xmm2,%%xmm5 \n" - "pshufd $0b11011000,%%xmm5,%%xmm5 \n" + "pshuflw $0x88,%%xmm5,%%xmm5 \n" + "pshufhw $0x88,%%xmm5,%%xmm5 \n" + "pshufd $0x88,%%xmm5,%%xmm5 \n" + "pshuflw $0x88,%%xmm2,%%xmm2 \n" + "pshufhw $0x88,%%xmm2,%%xmm2 \n" + "pshufd $0x88,%%xmm2,%%xmm2 \n" + "punpckldq %%xmm2,%%xmm5 \n" "movdqu %%xmm5,(%1,%4,2) \n" // store below "lea 0x8(%0),%0 \n" @@ -1970,10 +2253,12 @@ void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, } #endif // HAS_SCALEADDROW_AVX512BW -#ifdef HAS_SCALEADDCOLS_AVX512BW +#if defined(HAS_SCALEADDCOLS1_AVX512BW) || defined(HAS_SCALEADDCOLS2_AVX512BW) static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; +#endif +#ifdef HAS_SCALEADDCOLS1_AVX512BW void ScaleAddCols1_AVX512BW(int dst_width, int boxheight, int x, @@ -2223,7 +2508,9 @@ void ScaleAddCols1_AVX512BW(int dst_width, : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "k1", "k2", "k3", "k4"); } +#endif // HAS_SCALEADDCOLS1_AVX512BW +#ifdef HAS_SCALEADDCOLS2_AVX512BW void ScaleAddCols2_AVX512BW(int dst_width, int boxheight, int x, @@ -2581,110 +2868,114 @@ void ScaleAddCols2_AVX512BW(int dst_width, "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1", "k2", "k3", "k4"); } -#endif // HAS_SCALEADDCOLS_AVX512BW +#endif // HAS_SCALEADDCOLS2_AVX512BW -static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040, - 0x0040, 0x0040, 0x0040, 0x0040}; +#ifdef HAS_SCALEFILTERCOLS_AVX2 +static const int32_t kStep0_7_Filter[8] = {0, 1, 2, 3, 4, 5, 6, 7}; +static const uvec8 kShufFracCols = {0, 0, 4, 4, 8, 8, 12, 12, + 128, 128, 128, 128, 128, 128, 128, 128}; -// Bilinear column filtering. SSSE3 version. -void ScaleFilterCols_SSSE3(uint8_t* dst_ptr, - const uint8_t* src_ptr, - int dst_width, - int x, - int dx) { - intptr_t x0, x1, temp_pixel; - asm volatile( - "movd %6,%%xmm2 \n" - "movd %7,%%xmm3 \n" - "movl $0x04040000,%k2 \n" - "movd %k2,%%xmm5 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "psrlw $0x9,%%xmm6 \n" // 0x007f007f - "pcmpeqb %%xmm7,%%xmm7 \n" // 0x00ff mask for - // extracting 'a' - "psrlw $8,%%xmm7 \n" +// Bilinear column filtering. AVX2 version. +void ScaleFilterCols_AVX2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx) { + int n = dst_width & ~7; + int r = dst_width & 7; + if (n > 0) { + SIMD_ALIGNED(int32_t xi_buf[8]); + int32_t* xi_ptr = xi_buf; + uintptr_t tmp; + int loop_n = n; + asm volatile( + "vpbroadcastd %[x],%%ymm2 \n" + "vpbroadcastd %[dx],%%ymm3 \n" + "vpmulld %[kStep],%%ymm3,%%ymm0 \n" + "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" + "vpslld $0x3,%%ymm3,%%ymm3 \n" + "vbroadcasti128 %[kShufFrac],%%ymm5 \n" + "vpcmpeqb %%xmm6,%%xmm6,%%xmm6 \n" + "vpsrlw $0x8,%%xmm6,%%xmm7 \n" // 0x00ff + "vpsrlw $0x9,%%xmm6,%%xmm6 \n" // 0x007f - "pextrw $0x1,%%xmm2,%k3 \n" - "subl $0x2,%5 \n" - "jl 29f \n" - "movdqa %%xmm2,%%xmm0 \n" - "paddd %%xmm3,%%xmm0 \n" - "punpckldq %%xmm0,%%xmm2 \n" - "punpckldq %%xmm3,%%xmm3 \n" - "paddd %%xmm3,%%xmm3 \n" - "pextrw $0x3,%%xmm2,%k4 \n" - - LABELALIGN - "2: \n" - "movdqa %%xmm2,%%xmm1 \n" - "paddd %%xmm3,%%xmm2 \n" - "movzwl 0x00(%1,%3,1),%k2 \n" - "movd %k2,%%xmm0 \n" - "psrlw $0x9,%%xmm1 \n" - "movzwl 0x00(%1,%4,1),%k2 \n" - "movd %k2,%%xmm4 \n" - "pshufb %%xmm5,%%xmm1 \n" - "punpcklwd %%xmm4,%%xmm0 \n" - "pxor %%xmm6,%%xmm1 \n" - "movdqa %%xmm0,%%xmm4 \n" // Copy pixels. - "pmaddubsw %%xmm1,%%xmm0 \n" // a * (127 - f) + b * f - "pand %%xmm7,%%xmm4 \n" // Extract left pixels - // 'a'. - "paddw %%xmm4,%%xmm0 \n" // a * 128 + f * (b - a) - "pextrw $0x1,%%xmm2,%k3 \n" - "paddw %8,%%xmm0 \n" // add rounding bias 64. - "pextrw $0x3,%%xmm2,%k4 \n" - "psrlw $0x7,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movd %%xmm0,%k2 \n" - "mov %w2,(%0) \n" - "lea 0x2(%0),%0 \n" - "subl $0x2,%5 \n" - "jge 2b \n" - - LABELALIGN - "29: \n" - "addl $0x1,%5 \n" - "jl 99f \n" - "movzwl 0x00(%1,%3,1),%k2 \n" - "movd %k2,%%xmm0 \n" - "psrlw $0x9,%%xmm2 \n" - "pshufb %%xmm5,%%xmm2 \n" - "pxor %%xmm6,%%xmm2 \n" - "movdqa %%xmm0,%%xmm4 \n" // Copy pixels. - "pmaddubsw %%xmm2,%%xmm0 \n" // a * (127 - f) + b * f - "pand %%xmm7,%%xmm4 \n" // Extract left pixel 'a'. - "paddw %%xmm4,%%xmm0 \n" // a * 128 + f * (b - a) - "paddw %8,%%xmm0 \n" // add rounding bias 64. - "psrlw $0x7,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movd %%xmm0,%k2 \n" - "mov %b2,(%0) \n" - "99: \n" - : "+r"(dst_ptr), // %0 - "+r"(src_ptr), // %1 - "=&a"(temp_pixel), // %2 - "=&r"(x0), // %3 - "=&r"(x1), // %4 + LABELALIGN + "1: \n" + "vpsrad $0x10,%%ymm2,%%ymm0 \n" + "vmovdqu %%ymm0,(%[xi]) \n" + "vpsrlw $0x9,%%ymm2,%%ymm1 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpxor %%xmm6,%%xmm1,%%xmm1 \n" #if defined(__x86_64__) - "+rm"(dst_width) // %5 + "movslq 0x00(%[xi]),%q[tmp] \n" + "vpinsrw $0x0,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x04(%[xi]),%q[tmp] \n" + "vpinsrw $0x1,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x08(%[xi]),%q[tmp] \n" + "vpinsrw $0x2,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x0c(%[xi]),%q[tmp] \n" + "vpinsrw $0x3,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x10(%[xi]),%q[tmp] \n" + "vpinsrw $0x4,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x14(%[xi]),%q[tmp] \n" + "vpinsrw $0x5,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x18(%[xi]),%q[tmp] \n" + "vpinsrw $0x6,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x1c(%[xi]),%q[tmp] \n" + "vpinsrw $0x7,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" #else - "+m"(dst_width) // %5 + "mov 0x00(%[xi]),%[tmp] \n" + "vpinsrw $0x0,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x04(%[xi]),%[tmp] \n" + "vpinsrw $0x1,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x08(%[xi]),%[tmp] \n" + "vpinsrw $0x2,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x0c(%[xi]),%[tmp] \n" + "vpinsrw $0x3,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x10(%[xi]),%[tmp] \n" + "vpinsrw $0x4,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x14(%[xi]),%[tmp] \n" + "vpinsrw $0x5,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x18(%[xi]),%[tmp] \n" + "vpinsrw $0x6,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x1c(%[xi]),%[tmp] \n" + "vpinsrw $0x7,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" #endif - : "rm"(x), // %6 - "rm"(dx), // %7 -#if defined(__x86_64__) - "x"(kFadd40) // %8 -#else - "m"(kFadd40) // %8 -#endif - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + "vpand %%xmm7,%%xmm0,%%xmm4 \n" + "vpmaddubsw %%xmm1,%%xmm0,%%xmm0 \n" + "vpaddw %%xmm4,%%xmm0,%%xmm0 \n" + "vpsubw %%xmm6,%%xmm7,%%xmm4 \n" + "vpsrlw $0x1,%%xmm4,%%xmm4 \n" // 0x0040 + "vpaddw %%xmm4,%%xmm0,%%xmm0 \n" + "vpsrlw $0x7,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vmovq %%xmm0,(%[dst]) \n" + "lea 0x8(%[dst]),%[dst] \n" + "subl $0x8,%[n] \n" + "jg 1b \n" + "vzeroupper \n" + : [dst] "+r"(dst_ptr), [n] "+rm"(loop_n), [tmp] "=&r"(tmp) + : [src] "r"(src_ptr), [xi] "r"(xi_ptr), [x] "m"(x), [dx] "m"(dx), + [kStep] "m"(kStep0_7_Filter), [kShufFrac] "m"(kShufFracCols) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + int xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[j] = (uint8_t)(a + ((((x & 0xffff) >> 9) * (b - a) + 0x40) >> 7)); + x = (int)((uint32_t)x + (uint32_t)dx); + } } +#endif // HAS_SCALEFILTERCOLS_AVX2 -// Reads 4 pixels, duplicates them and writes 8 pixels. -// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned. -void ScaleColsUp2_SSE2(uint8_t* dst_ptr, +#ifdef HAS_SCALECOLSUP2_AVX2 +// Reads 16 pixels, duplicates them and writes 32 pixels (plus 16/8 tails). +void ScaleColsUp2_AVX2(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, int x, @@ -2692,40 +2983,67 @@ void ScaleColsUp2_SSE2(uint8_t* dst_ptr, (void)x; (void)dx; asm volatile( + "sub $0x20,%2 \n" + "jl 2f \n" + + LABELALIGN "1: \n" - "movdqu (%1),%%xmm0 \n" + "vmovdqu (%1),%%xmm0 \n" "lea 0x10(%1),%1 \n" - "movdqa %%xmm0,%%xmm1 \n" - "punpcklbw %%xmm0,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm1 \n" - "movdqu %%xmm0,(%0) \n" - "movdqu %%xmm1,0x10(%0) \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%0) \n" "lea 0x20(%0),%0 \n" "sub $0x20,%2 \n" - "jg 1b \n" + "jge 1b \n" + "2: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "test $0x10,%2 \n" + "je 3f \n" + "vmovq (%1),%%xmm0 \n" + "lea 0x8(%1),%1 \n" + "vpunpcklbw %%xmm0,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%0) \n" + "lea 0x10(%0),%0 \n" + + "3: \n" + "test $0x8,%2 \n" + "je 99f \n" + "vmovd (%1),%%xmm0 \n" + "vpunpcklbw %%xmm0,%%xmm0,%%xmm0 \n" + "vmovq %%xmm0,(%0) \n" + + "99: \n" + "vzeroupper \n" : "+r"(dst_ptr), // %0 "+r"(src_ptr), // %1 "+r"(dst_width) // %2 : - : "memory", "cc", "xmm0", "xmm1"); + : "memory", "cc", "xmm0"); } +#endif // HAS_SCALECOLSUP2_AVX2 -void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb, +#ifdef HAS_SCALEARGBROWDOWN2_AVX2 +void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { (void)src_stride; asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "lea 0x20(%0),%0 \n" - "shufps $0xdd,%%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x4,%2 \n" + "vmovups (%0),%%ymm0 \n" + "vmovups 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x8,%2 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_argb), // %0 "+r"(dst_argb), // %1 "+r"(dst_width) // %2 @@ -2733,24 +3051,26 @@ void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb, : "memory", "cc", "xmm0", "xmm1"); } -void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { (void)src_stride; asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "lea 0x20(%0),%0 \n" - "movdqa %%xmm0,%%xmm2 \n" - "shufps $0x88,%%xmm1,%%xmm0 \n" - "shufps $0xdd,%%xmm1,%%xmm2 \n" - "pavgb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x4,%2 \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vshufps $0x88,%%ymm1,%%ymm0,%%ymm2 \n" + "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm0,%%ymm2,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x8,%2 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_argb), // %0 "+r"(dst_argb), // %1 "+r"(dst_width) // %2 @@ -2758,188 +3078,165 @@ void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb, : "memory", "cc", "xmm0", "xmm1", "xmm2"); } -void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "movdqu 0x00(%0,%3,1),%%xmm2 \n" - "movdqu 0x10(%0,%3,1),%%xmm3 \n" - "lea 0x20(%0),%0 \n" - "pavgb %%xmm2,%%xmm0 \n" - "pavgb %%xmm3,%%xmm1 \n" - "movdqa %%xmm0,%%xmm2 \n" - "shufps $0x88,%%xmm1,%%xmm0 \n" - "shufps $0xdd,%%xmm1,%%xmm2 \n" - "pavgb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x4,%2 \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vmovdqu 0x00(%0,%3,1),%%ymm2 \n" + "vmovdqu 0x20(%0,%3,1),%%ymm3 \n" + "lea 0x40(%0),%0 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vshufps $0x88,%%ymm1,%%ymm0,%%ymm2 \n" + "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm0,%%ymm2,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x8,%2 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_argb), // %0 "+r"(dst_argb), // %1 "+r"(dst_width) // %2 : "r"(src_stride) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } +#endif // HAS_SCALEARGBROWDOWN2_AVX2 -// Reads 4 pixels at a time. -// Alignment requirement: dst_argb 16 byte aligned. -void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb, +#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2 +static const int32_t kStep0_7_Even[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + +// Reads 8 pixels at a time. +void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, int dst_width) { - ptrdiff_t src_stepx_x4 = (ptrdiff_t)(src_stepx); - ptrdiff_t src_stepx_x12; + ptrdiff_t src_stepx_x32 = (ptrdiff_t)src_stepx * 32; (void)src_stride; asm volatile( - "lea 0x00(,%1,4),%1 \n" - "lea 0x00(%1,%1,2),%4 \n" + "vpbroadcastd %[stepx],%%ymm2 \n" + "vpmulld %[kStep],%%ymm2,%%ymm2 \n" LABELALIGN "1: \n" - "movd (%0),%%xmm0 \n" - "movd 0x00(%0,%1,1),%%xmm1 \n" - "punpckldq %%xmm1,%%xmm0 \n" - "movd 0x00(%0,%1,2),%%xmm2 \n" - "movd 0x00(%0,%4,1),%%xmm3 \n" - "lea 0x00(%0,%1,4),%0 \n" - "punpckldq %%xmm3,%%xmm2 \n" - "punpcklqdq %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" + "vpcmpeqd %%ymm3,%%ymm3,%%ymm3 \n" + "vpgatherdd %%ymm3,(%[src],%%ymm2,4),%%ymm0 \n" + "add %[step32],%[src] \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" "jg 1b \n" - : "+r"(src_argb), // %0 - "+r"(src_stepx_x4), // %1 - "+r"(dst_argb), // %2 - "+r"(dst_width), // %3 - "=&r"(src_stepx_x12) // %4 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+rm"(dst_width) + : [step32] "r"(src_stepx_x32), [stepx] "m"(src_stepx), + [kStep] "m"(kStep0_7_Even) + : "memory", "cc", "xmm0", "xmm2", "xmm3"); } -// Blends four 2x2 to 4x1. -// Alignment requirement: dst_argb 16 byte aligned. -void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb, +// Blends eight 2x2 to 8x1. +void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, int dst_width) { - ptrdiff_t src_stepx_x4 = (ptrdiff_t)(src_stepx); - ptrdiff_t src_stepx_x12; + const uint8_t* row1 = src_argb + src_stride; + ptrdiff_t step4_bytes = (ptrdiff_t)src_stepx * 16; asm volatile( - "lea 0x00(,%1,4),%1 \n" - "lea 0x00(%1,%1,2),%4 \n" - "lea 0x00(%0,%5,1),%5 \n" + "vmovd %[stepx],%%xmm4 \n" + "vpshufd $0x0,%%xmm4,%%xmm4 \n" + "vpmulld %[kStep],%%xmm4,%%xmm4 \n" LABELALIGN "1: \n" - "movq (%0),%%xmm0 \n" - "movhps 0x00(%0,%1,1),%%xmm0 \n" - "movq 0x00(%0,%1,2),%%xmm1 \n" - "movhps 0x00(%0,%4,1),%%xmm1 \n" - "lea 0x00(%0,%1,4),%0 \n" - "movq (%5),%%xmm2 \n" - "movhps 0x00(%5,%1,1),%%xmm2 \n" - "movq 0x00(%5,%1,2),%%xmm3 \n" - "movhps 0x00(%5,%4,1),%%xmm3 \n" - "lea 0x00(%5,%1,4),%5 \n" - "pavgb %%xmm2,%%xmm0 \n" - "pavgb %%xmm3,%%xmm1 \n" - "movdqa %%xmm0,%%xmm2 \n" - "shufps $0x88,%%xmm1,%%xmm0 \n" - "shufps $0xdd,%%xmm1,%%xmm2 \n" - "pavgb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpgatherdq %%ymm5,(%[row0],%%xmm4,4),%%ymm0 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpgatherdq %%ymm5,(%[row1],%%xmm4,4),%%ymm2 \n" + "add %[step4],%[row0] \n" + "add %[step4],%[row1] \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpgatherdq %%ymm5,(%[row0],%%xmm4,4),%%ymm1 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpgatherdq %%ymm5,(%[row1],%%xmm4,4),%%ymm3 \n" + "add %[step4],%[row0] \n" + "add %[step4],%[row1] \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vshufps $0x88,%%ymm1,%%ymm0,%%ymm2 \n" + "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm0,%%ymm2,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "subl $0x8,%[width] \n" "jg 1b \n" - : "+r"(src_argb), // %0 - "+r"(src_stepx_x4), // %1 - "+r"(dst_argb), // %2 - "+rm"(dst_width), // %3 - "=&r"(src_stepx_x12), // %4 - "+r"(src_stride) // %5 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + "vzeroupper \n" + : [row0] "+r"(src_argb), [row1] "+r"(row1), [dst] "+r"(dst_argb), + [width] "+rm"(dst_width) + : [step4] "r"(step4_bytes), [stepx] "m"(src_stepx), + [kStep] "m"(kStep0_7_Even) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); } +#endif // HAS_SCALEARGBROWDOWNEVEN_AVX2 -void ScaleARGBCols_SSE2(uint8_t* dst_argb, +#ifdef HAS_SCALEARGBCOLS_AVX2 +static const int32_t kStep0_7_Cols[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + +void ScaleARGBCols_AVX2(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, int dx) { - intptr_t x0, x1; - asm volatile( - "movd %5,%%xmm2 \n" - "movd %6,%%xmm3 \n" - "pshufd $0x0,%%xmm2,%%xmm2 \n" - "pshufd $0x11,%%xmm3,%%xmm0 \n" - "paddd %%xmm0,%%xmm2 \n" - "paddd %%xmm3,%%xmm3 \n" - "pshufd $0x5,%%xmm3,%%xmm0 \n" - "paddd %%xmm0,%%xmm2 \n" - "paddd %%xmm3,%%xmm3 \n" - "pshufd $0x0,%%xmm3,%%xmm3 \n" - "pextrw $0x1,%%xmm2,%k0 \n" - "pextrw $0x3,%%xmm2,%k1 \n" - "cmp $0x0,%4 \n" - "jl 99f \n" - "sub $0x4,%4 \n" - "jl 49f \n" + int n = dst_width & ~7; + int r = dst_width & 7; + if (n > 0) { + int loop_n = n; + asm volatile( + "vpbroadcastd %[x],%%ymm2 \n" + "vpbroadcastd %[dx],%%ymm3 \n" + "vpmulld %[kStep],%%ymm3,%%ymm0 \n" + "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" + "vpslld $0x3,%%ymm3,%%ymm3 \n" - LABELALIGN - "40: \n" - "movd 0x00(%3,%0,4),%%xmm0 \n" - "movd 0x00(%3,%1,4),%%xmm1 \n" - "pextrw $0x5,%%xmm2,%k0 \n" - "pextrw $0x7,%%xmm2,%k1 \n" - "paddd %%xmm3,%%xmm2 \n" - "punpckldq %%xmm1,%%xmm0 \n" - "movd 0x00(%3,%0,4),%%xmm1 \n" - "movd 0x00(%3,%1,4),%%xmm4 \n" - "pextrw $0x1,%%xmm2,%k0 \n" - "pextrw $0x3,%%xmm2,%k1 \n" - "punpckldq %%xmm4,%%xmm1 \n" - "punpcklqdq %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%4 \n" - "jge 40b \n" - - "49: \n" - "test $0x2,%4 \n" - "je 29f \n" - "movd 0x00(%3,%0,4),%%xmm0 \n" - "movd 0x00(%3,%1,4),%%xmm1 \n" - "pextrw $0x5,%%xmm2,%k0 \n" - "punpckldq %%xmm1,%%xmm0 \n" - "movq %%xmm0,(%2) \n" - "lea 0x8(%2),%2 \n" - "29: \n" - "test $0x1,%4 \n" - "je 99f \n" - "movd 0x00(%3,%0,4),%%xmm0 \n" - "movd %%xmm0,(%2) \n" - "99: \n" - : "=&a"(x0), // %0 - "=&d"(x1), // %1 - "+r"(dst_argb), // %2 - "+r"(src_argb), // %3 - "+r"(dst_width) // %4 - : "rm"(x), // %5 - "rm"(dx) // %6 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); + LABELALIGN + "1: \n" + "vpsrad $0x10,%%ymm2,%%ymm0 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n" + "vpgatherdd %%ymm4,(%[src],%%ymm0,4),%%ymm1 \n" + "vmovdqu %%ymm1,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "subl $0x8,%[n] \n" + "jg 1b \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [n] "+rm"(loop_n) + : [src] "r"(src_argb), [x] "m"(x), [dx] "m"(dx), + [kStep] "m"(kStep0_7_Cols) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + const uint8_t* src = src_argb + (ptrdiff_t)(x >> 16) * 4; + dst_argb[0] = src[0]; + dst_argb[1] = src[1]; + dst_argb[2] = src[2]; + dst_argb[3] = src[3]; + dst_argb += 4; + x = (int)((uint32_t)x + (uint32_t)dx); + } } +#endif // HAS_SCALEARGBCOLS_AVX2 +#ifdef HAS_SCALEARGBCOLSUP2_AVX2 // Reads 4 pixels, duplicates them and writes 8 pixels. -// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned. -void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb, +void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, @@ -2947,24 +3244,27 @@ void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb, (void)x; (void)dx; asm volatile( + LABELALIGN "1: \n" - "movdqu (%1),%%xmm0 \n" + "vmovdqu (%1),%%xmm0 \n" "lea 0x10(%1),%1 \n" - "movdqa %%xmm0,%%xmm1 \n" - "punpckldq %%xmm0,%%xmm0 \n" - "punpckhdq %%xmm1,%%xmm1 \n" - "movdqu %%xmm0,(%0) \n" - "movdqu %%xmm1,0x10(%0) \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpunpckldq %%ymm0,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%0) \n" "lea 0x20(%0),%0 \n" "sub $0x8,%2 \n" "jg 1b \n" - + "vzeroupper \n" : "+r"(dst_argb), // %0 "+r"(src_argb), // %1 "+r"(dst_width) // %2 : - : "memory", "cc", "xmm0", "xmm1"); + : "memory", "cc", "xmm0"); } +#endif // HAS_SCALEARGBCOLSUP2_AVX2 + +#ifdef HAS_SCALEARGBFILTERCOLS_AVX2 +static const int32_t kStep0_3_ARGB[4] = {0, 1, 2, 3}; // Shuffle table for arranging 2 pixels into pairs for pmaddubsw static const uvec8 kShuffleColARGB = { @@ -2972,77 +3272,77 @@ static const uvec8 kShuffleColARGB = { 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u // bbggrraa 2nd pixel }; -// Shuffle table for duplicating 2 fractions into 8 bytes each -static const uvec8 kShuffleFractions = { - 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, +// Shuffle table for duplicating 4 fractions across 2x128-bit lanes +static const uint8_t kShuffleFractionsAVX2[32] = { + 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, + 8u, 8u, 8u, 8u, 8u, 8u, 8u, 8u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, }; -// Bilinear row filtering combines 4x2 -> 4x1. SSSE3 version -void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx) { - intptr_t x0, x1; - asm volatile( - "movdqa %[kShuffleColARGB],%%xmm4 \n" - "movdqa %[kShuffleFractions],%%xmm5 \n" - "movd %[x],%%xmm2 \n" - "movd %[dx],%%xmm3 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "psrlw $0x9,%%xmm6 \n" - "pextrw $0x1,%%xmm2,%k[x0] \n" - "sub $0x2,%[dst_width] \n" - "jl 29f \n" - "movdqa %%xmm2,%%xmm0 \n" - "paddd %%xmm3,%%xmm0 \n" - "punpckldq %%xmm0,%%xmm2 \n" - "punpckldq %%xmm3,%%xmm3 \n" - "paddd %%xmm3,%%xmm3 \n" - "pextrw $0x3,%%xmm2,%k[x1] \n" +// Bilinear row filtering combines 4x2 -> 4x1. AVX2 version. +void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + int n = dst_width & ~3; + int r = dst_width & 3; + if (n > 0) { + int loop_n = n; + asm volatile( + "vbroadcasti128 %[kShufARGB],%%ymm4 \n" + "vmovdqu %[kShufFrac],%%ymm5 \n" + "vmovd %[x],%%xmm2 \n" + "vpshufd $0x0,%%xmm2,%%xmm2 \n" + "vmovd %[dx],%%xmm3 \n" + "vpshufd $0x0,%%xmm3,%%xmm3 \n" + "vpmulld %[kStep],%%xmm3,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpslld $0x2,%%xmm3,%%xmm3 \n" + "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrlw $0x9,%%ymm6,%%ymm6 \n" - LABELALIGN - "2: \n" - "movdqa %%xmm2,%%xmm1 \n" - "paddd %%xmm3,%%xmm2 \n" - "movq 0x00(%[src_argb],%[x0],4),%%xmm0 \n" - "psrlw $0x9,%%xmm1 \n" - "movhps 0x00(%[src_argb],%[x1],4),%%xmm0 \n" - "pshufb %%xmm5,%%xmm1 \n" - "pshufb %%xmm4,%%xmm0 \n" - "pxor %%xmm6,%%xmm1 \n" - "pmaddubsw %%xmm1,%%xmm0 \n" - "psrlw $0x7,%%xmm0 \n" - "pextrw $0x1,%%xmm2,%k[x0] \n" - "pextrw $0x3,%%xmm2,%k[x1] \n" - "packuswb %%xmm0,%%xmm0 \n" - "movq %%xmm0,(%[dst_argb]) \n" - "lea 0x8(%[dst_argb]),%[dst_argb] \n" - "sub $0x2,%[dst_width] \n" - "jge 2b \n" - - LABELALIGN - "29: \n" - "add $0x1,%[dst_width] \n" - "jl 99f \n" - "psrlw $0x9,%%xmm2 \n" - "movq 0x00(%[src_argb],%[x0],4),%%xmm0 \n" - "pshufb %%xmm5,%%xmm2 \n" - "pshufb %%xmm4,%%xmm0 \n" - "pxor %%xmm6,%%xmm2 \n" - "pmaddubsw %%xmm2,%%xmm0 \n" - "psrlw $0x7,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movd %%xmm0,(%[dst_argb]) \n" - - LABELALIGN "99: \n" - - : [dst_argb] "+r"(dst_argb), [src_argb] "+r"(src_argb), - [dst_width] "+rm"(dst_width), [x0] "=&r"(x0), [x1] "=&r"(x1) - : [x] "rm"(x), [dx] "rm"(dx), [kShuffleColARGB] "m"(kShuffleColARGB), - [kShuffleFractions] "m"(kShuffleFractions) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + LABELALIGN + "1: \n" + "vpsrad $0x10,%%xmm2,%%xmm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdq %%ymm7,(%[src],%%xmm1,4),%%ymm0 \n" + "vpsrlw $0x9,%%xmm2,%%xmm1 \n" + "vpaddd %%xmm3,%%xmm2,%%xmm2 \n" + "vinserti128 $0x1,%%xmm1,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" + "vpxor %%ymm6,%%ymm1,%%ymm1 \n" + "vpmaddubsw %%ymm1,%%ymm0,%%ymm0 \n" + "vpsrlw $0x7,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "subl $0x4,%[n] \n" + "jg 1b \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [n] "+rm"(loop_n) + : [src] "r"(src_argb), [x] "m"(x), [dx] "m"(dx), + [kStep] "m"(kStep0_3_ARGB), [kShufARGB] "m"(kShuffleColARGB), + [kShufFrac] "m"(kShuffleFractionsAVX2) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + ptrdiff_t xi = (ptrdiff_t)(x >> 16) * 4; + int xf = (x >> 9) & 0x7f; + const uint8_t* a = src_argb + xi; + const uint8_t* b = a + 4; + dst_argb[0] = (uint8_t)((a[0] * (0x7f ^ xf) + b[0] * xf) >> 7); + dst_argb[1] = (uint8_t)((a[1] * (0x7f ^ xf) + b[1] * xf) >> 7); + dst_argb[2] = (uint8_t)((a[2] * (0x7f ^ xf) + b[2] * xf) >> 7); + dst_argb[3] = (uint8_t)((a[3] * (0x7f ^ xf) + b[3] * xf) >> 7); + dst_argb += 4; + x = (int)((uint32_t)x + (uint32_t)dx); + } } +#endif // HAS_SCALEARGBFILTERCOLS_AVX2 // Divide num by div and return as 16.16 fixed point result. int FixedDiv_X86(int num, int div) { @@ -3076,59 +3376,108 @@ int FixedDiv1_X86(int num, int div) { return num; } -#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3) || \ - defined(HAS_SCALEUVROWDOWN2BOX_AVX2) +#ifdef HAS_SCALEUVROWDOWN2_AVX2 +void ScaleUVRowDown2_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + asm volatile( + "sub $0x10,%2 \n" + "jl 2f \n" + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm0 \n" + "vpsrld $0x10,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vpsrld $0x10,%%ymm0,%%ymm0 \n" + "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%1) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "xmm0", "xmm1"); +} +#endif // HAS_SCALEUVROWDOWN2_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2 +void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + asm volatile( + "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n" + "vpsrld $0x10,%%ymm4,%%ymm4 \n" + "sub $0x10,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm2 \n" + "vpsrld $0x10,%%ymm1,%%ymm3 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpand %%ymm4,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vpsrld $0x10,%%ymm0,%%ymm2 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%1) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); +} +#endif // HAS_SCALEUVROWDOWN2LINEAR_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2 // Shuffle table for splitting UV into upper and lower part of register. static const uvec8 kShuffleSplitUV = {0u, 2u, 4u, 6u, 8u, 10u, 12u, 14u, 1u, 3u, 5u, 7u, 9u, 11u, 13u, 15u}; static const uvec8 kShuffleMergeUV = {0u, 8u, 2u, 10u, 4u, 12u, 6u, 14u, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; -#endif -#ifdef HAS_SCALEUVROWDOWN2BOX_SSSE3 - -void ScaleUVRowDown2Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - asm volatile( - "pcmpeqb %%xmm4,%%xmm4 \n" // 01010101 - "psrlw $0xf,%%xmm4 \n" - "packuswb %%xmm4,%%xmm4 \n" - "pxor %%xmm5, %%xmm5 \n" // zero - "movdqa %4,%%xmm1 \n" // split shuffler - "movdqa %5,%%xmm3 \n" // merge shuffler - - LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" // 8 UV row 0 - "movdqu 0x00(%0,%3,1),%%xmm2 \n" // 8 UV row 1 - "lea 0x10(%0),%0 \n" - "pshufb %%xmm1,%%xmm0 \n" // uuuuvvvv - "pshufb %%xmm1,%%xmm2 \n" - "pmaddubsw %%xmm4,%%xmm0 \n" // horizontal add - "pmaddubsw %%xmm4,%%xmm2 \n" - "paddw %%xmm2,%%xmm0 \n" // vertical add - "psrlw $0x1,%%xmm0 \n" // round - "pavgw %%xmm5,%%xmm0 \n" - "pshufb %%xmm3,%%xmm0 \n" // merge uv - "movq %%xmm0,(%1) \n" - "lea 0x8(%1),%1 \n" // 4 UV - "sub $0x4,%2 \n" - "jg 1b \n" - : "+r"(src_ptr), // %0 - "+r"(dst_ptr), // %1 - "+r"(dst_width) // %2 - : "r"(src_stride), // %3 - "m"(kShuffleSplitUV), // %4 - "m"(kShuffleMergeUV) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); -} -#endif // HAS_SCALEUVROWDOWN2BOX_SSSE3 - -#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2 void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -3169,6 +3518,40 @@ void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr, } #endif // HAS_SCALEUVROWDOWN2BOX_AVX2 +#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2 +void ScaleUVRowDownEven_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_uv, + int dst_width) { + ptrdiff_t step = (ptrdiff_t)src_stepx * 2; + ptrdiff_t step3 = step * 3; + (void)src_stride; + asm volatile( + "vpxor %%xmm0,%%xmm0,%%xmm0 \n" + + LABELALIGN + "1: \n" + "vpinsrw $0x0,(%[src]),%%xmm0,%%xmm0 \n" + "vpinsrw $0x1,0x00(%[src],%[step],1),%%xmm0,%%xmm0 \n" + "vpinsrw $0x2,0x00(%[src],%[step],2),%%xmm0,%%xmm0 \n" + "vpinsrw $0x3,0x00(%[src],%[step3],1),%%xmm0,%%xmm0 \n" + "lea 0x00(%[src],%[step],4),%[src] \n" + "vpinsrw $0x4,(%[src]),%%xmm0,%%xmm0 \n" + "vpinsrw $0x5,0x00(%[src],%[step],1),%%xmm0,%%xmm0 \n" + "vpinsrw $0x6,0x00(%[src],%[step],2),%%xmm0,%%xmm0 \n" + "vpinsrw $0x7,0x00(%[src],%[step3],1),%%xmm0,%%xmm0 \n" + "lea 0x00(%[src],%[step],4),%[src] \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "subl $0x8,%[width] \n" + "jg 1b \n" + : [src] "+r"(src_uv), [dst] "+r"(dst_uv), [width] "+rm"(dst_width) + : [step] "r"(step), [step3] "r"(step3) + : "memory", "cc", "xmm0"); +} +#endif // HAS_SCALEUVROWDOWNEVEN_AVX2 + static const uvec8 kUVLinearMadd31 = {3, 1, 3, 1, 1, 3, 1, 3, 3, 1, 3, 1, 1, 3, 1, 3}; diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 324822a0e..706f27a80 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -84,25 +84,25 @@ static void ScaleUVDown2(int src_width, assert(dx == 65536 * 2); // Test scale factor of 2. assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2. // Advance to odd row, even column. - if (filtering == kFilterBilinear) { + if (filtering) { src_uv += (y >> 16) * src_stride + (x >> 16) * 2; } else { src_uv += (y >> 16) * src_stride + ((x >> 16) - 1) * 2; } -#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && filtering) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_SSSE3; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_SSSE3; - } - } -#endif -#if defined(HAS_SCALEUVROWDOWN2BOX_AVX2) - if (TestCpuFlag(kCpuHasAVX2) && filtering) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_AVX2; +#if defined(HAS_SCALEUVROWDOWN2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleUVRowDown2 = + filtering == kFilterNone + ? ScaleUVRowDown2_Any_AVX2 + : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_AVX2 + : ScaleUVRowDown2Box_Any_AVX2); if (IS_ALIGNED(dst_width, 8)) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_AVX2; + ScaleUVRowDown2 = + filtering == kFilterNone + ? ScaleUVRowDown2_AVX2 + : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_AVX2 + : ScaleUVRowDown2Box_AVX2); } } #endif @@ -148,23 +148,48 @@ static void ScaleUVDown2(int src_width, } #endif -// This code is not enabled. Only box filter is available at this time. -#if defined(HAS_SCALEUVROWDOWN2_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ScaleUVRowDown2 = - filtering == kFilterNone - ? ScaleUVRowDown2_Any_SSSE3 - : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_SSSE3 - : ScaleUVRowDown2Box_Any_SSSE3); - if (IS_ALIGNED(dst_width, 2)) { - ScaleUVRowDown2 = - filtering == kFilterNone - ? ScaleUVRowDown2_SSSE3 - : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_SSSE3 - : ScaleUVRowDown2Box_SSSE3); - } + if (filtering == kFilterLinear) { + src_stride = 0; + } + for (j = 0; j < dst_height; ++j) { + ScaleUVRowDown2(src_uv, src_stride, dst_uv, dst_width); + src_uv += row_stride; + dst_uv += dst_stride; + } +} + +static void ScaleUVDown2_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy, + enum FilterMode filtering) { + int j; + ptrdiff_t row_stride = src_stride * (dy >> 16); + void (*ScaleUVRowDown2)(const uint16_t* src_uv, ptrdiff_t src_stride, + uint16_t* dst_uv, int dst_width) = + filtering == kFilterNone + ? ScaleUVRowDown2_16_C + : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_16_C + : ScaleUVRowDown2Box_16_C); + (void)src_width; + (void)src_height; + (void)dx; + assert(dx == 65536 * 2); // Test scale factor of 2. + assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2. + // Advance to odd row, even column. + if (filtering) { + src_uv += (y >> 16) * src_stride + (x >> 16) * 2; + } else { + src_uv += (y >> 16) * src_stride + ((x >> 16) - 1) * 2; } -#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -211,14 +236,6 @@ static int ScaleUVDown4Box(int src_width, assert(dx == 65536 * 4); // Test scale factor of 4. assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4. -#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_SSSE3; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_SSSE3; - } - } -#endif #if defined(HAS_SCALEUVROWDOWN2BOX_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_AVX2; @@ -262,6 +279,49 @@ static int ScaleUVDown4Box(int src_width, free_aligned_buffer_64(row); return 0; } + +static int ScaleUVDown4Box_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy) { + int j; + // Allocate 2 rows of 16-bit UV. + const int row_stride_words = (dst_width * 2 * 2 + 15) & ~15; + align_buffer_64(row, row_stride_words * 2 * (int)sizeof(uint16_t)); + if (!row) + return 1; + uint16_t* row16 = (uint16_t*)row; + ptrdiff_t row_stride = src_stride * (dy >> 16); + void (*ScaleUVRowDown2)(const uint16_t* src_uv, ptrdiff_t src_stride, + uint16_t* dst_uv, int dst_width) = + ScaleUVRowDown2Box_16_C; + // Advance to odd row, even column. + src_uv += (y >> 16) * src_stride + (x >> 16) * 2; + (void)src_width; + (void)src_height; + (void)dx; + assert(dx == 65536 * 4); // Test scale factor of 4. + assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4. + + for (j = 0; j < dst_height; ++j) { + ScaleUVRowDown2(src_uv, src_stride, row16, dst_width * 2); + ScaleUVRowDown2(src_uv + src_stride * 2, src_stride, + row16 + row_stride_words, dst_width * 2); + ScaleUVRowDown2(row16, row_stride_words, dst_uv, dst_width); + src_uv += row_stride; + dst_uv += dst_stride; + } + free_aligned_buffer_64(row); + return 0; +} #endif // HAS_SCALEUVDOWN4BOX // ScaleUV, any dimensions, box filter. @@ -493,6 +553,119 @@ static int ScaleUVBox(int src_width, } return 0; } + +static void ScaleUVAddCols1_16_C(int dst_width, + int boxheight, + int x, + int dx, + const uint32_t* src_uv, + uint16_t* dst_uv) { + int i, j; + int boxwidth = MIN1(dx >> 16); + uint32_t scaleval = 65536 / (boxwidth * boxheight); + x >>= 16; + for (i = 0; i < dst_width; ++i) { + uint32_t u = 0; + uint32_t v = 0; + for (j = 0; j < boxwidth; ++j) { + u += src_uv[(x + j) * 2]; + v += src_uv[(x + j) * 2 + 1]; + } + x += boxwidth; + *dst_uv++ = (uint16_t)((u * scaleval) >> 16); + *dst_uv++ = (uint16_t)((v * scaleval) >> 16); + } +} + +static void ScaleUVAddCols2_16_C(int dst_width, + int boxheight, + int x, + int dx, + const uint32_t* src_uv, + uint16_t* dst_uv) { + int i, j; + int minboxwidth = dx >> 16; + uint32_t scaletbl[2]; + scaletbl[0] = 65536 / (MIN1(minboxwidth) * boxheight); + scaletbl[1] = 65536 / (MIN1(minboxwidth + 1) * boxheight); + for (i = 0; i < dst_width; ++i) { + int ix = x >> 16; + uint32_t u = 0; + uint32_t v = 0; + int boxwidth; + uint32_t scaleval; + x += dx; + boxwidth = MIN1((x >> 16) - ix); + assert((boxwidth - minboxwidth == 0) || (boxwidth - minboxwidth == 1)); + scaleval = scaletbl[boxwidth - minboxwidth]; + for (j = 0; j < boxwidth; ++j) { + u += src_uv[(ix + j) * 2]; + v += src_uv[(ix + j) * 2 + 1]; + } + *dst_uv++ = (uint16_t)((u * scaleval) >> 16); + *dst_uv++ = (uint16_t)((v * scaleval) >> 16); + } +} + +static int ScaleUVBox_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy) { + int j, k; + const uint32_t max_y = (uint32_t)src_height << 16; + const int row_samples = src_width * 2; + { + // Row accumulator, two uint32_t per source UV pixel. + align_buffer_64(row32_buf, row_samples * (int)sizeof(uint32_t)); + if (!row32_buf) + return 1; + { + uint32_t* row32 = (uint32_t*)row32_buf; + void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx, + const uint32_t* src_uv, uint16_t* dst_uv) = + (dx & 0xffff) ? ScaleUVAddCols2_16_C : ScaleUVAddCols1_16_C; + void (*ScaleUVAddCols3)(int dst_width, int scaleval, + const uint32_t* src_uv, uint16_t* dst_uv) = + (dx == 0x30000) ? ScaleUVAddCols3_16_C : NULL; + void (*ScaleAddRow)(const uint16_t* src_ptr, uint32_t* dst_ptr, + int src_width) = ScaleAddRow_16_C; + + uint32_t uy = y; + for (j = 0; j < dst_height; ++j) { + int boxheight; + int iy = uy >> 16; + const uint16_t* src = src_uv + iy * src_stride; + uy += dy; + if (uy > max_y) { + uy = max_y; + } + boxheight = MIN1((int)(uy >> 16) - iy); + memset(row32, 0, row_samples * sizeof(uint32_t)); + for (k = 0; k < boxheight; ++k) { + ScaleAddRow(src, row32, row_samples); + src += src_stride; + } + if (ScaleUVAddCols3 && boxheight <= 21845) { + ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight), + row32 + (ptrdiff_t)(x >> 16) * 2, dst_uv); + } else { + ScaleUVAddCols(dst_width, boxheight, x, dx, row32, dst_uv); + } + dst_uv += dst_stride; + } + } + free_aligned_buffer_64(row32_buf); + } + return 0; +} #endif // HAS_SCALEUVBOX // ScaleUV Even @@ -523,13 +696,11 @@ static void ScaleUVDownEven(int src_width, assert(IS_ALIGNED(src_width, 2)); assert(IS_ALIGNED(src_height, 2)); src_uv += (y >> 16) * src_stride + (x >> 16) * 2; -#if defined(HAS_SCALEUVROWDOWNEVEN_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_SSSE3 - : ScaleUVRowDownEven_Any_SSSE3; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVRowDownEven = - filtering ? ScaleUVRowDownEvenBox_SSE2 : ScaleUVRowDownEven_SSSE3; +#if defined(HAS_SCALEUVROWDOWNEVEN_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && !filtering) { + ScaleUVRowDownEven = ScaleUVRowDownEven_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVRowDownEven = ScaleUVRowDownEven_AVX2; } } #endif @@ -541,16 +712,6 @@ static void ScaleUVDownEven(int src_width, } } #endif // TODO(fbarchard): Enable Box filter -#if defined(HAS_SCALEUVROWDOWNEVENBOX_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_NEON - : ScaleUVRowDownEven_Any_NEON; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVRowDownEven = - filtering ? ScaleUVRowDownEvenBox_NEON : ScaleUVRowDownEven_NEON; - } - } -#endif #if defined(HAS_SCALEUVROWDOWNEVEN_RVV) || defined(HAS_SCALEUVROWDOWN4_RVV) if (TestCpuFlag(kCpuHasRVV) && !filtering) { #if defined(HAS_SCALEUVROWDOWNEVEN_RVV) @@ -573,6 +734,41 @@ static void ScaleUVDownEven(int src_width, dst_uv += dst_stride; } } + +static void ScaleUVDownEven_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy, + enum FilterMode filtering) { + int j; + int col_step = dx >> 16; + ptrdiff_t row_stride = (dy >> 16) * src_stride; + void (*ScaleUVRowDownEven)(const uint16_t* src_uv, ptrdiff_t src_stride, + int src_step, uint16_t* dst_uv, int dst_width) = + filtering ? ScaleUVRowDownEvenBox_16_C : ScaleUVRowDownEven_16_C; + (void)src_width; + (void)src_height; + assert(IS_ALIGNED(src_width, 2)); + assert(IS_ALIGNED(src_height, 2)); + src_uv += (y >> 16) * src_stride + (x >> 16) * 2; + + if (filtering == kFilterLinear) { + src_stride = 0; + } + for (j = 0; j < dst_height; ++j) { + ScaleUVRowDownEven(src_uv, src_stride, col_step, dst_uv, dst_width); + src_uv += row_stride; + dst_uv += dst_stride; + } +} #endif // Scale UV down with bilinear interpolation. @@ -674,19 +870,6 @@ static int ScaleUVBilinearDown(int src_width, InterpolateRow = InterpolateRow_WASMSIMD; } } -#endif -#if defined(HAS_SCALEUVFILTERCOLS_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleUVFilterCols = ScaleUVFilterCols_SSSE3; - } -#endif -#if defined(HAS_SCALEUVFILTERCOLS_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVFilterCols = ScaleUVFilterCols_NEON; - } - } #endif // TODO(fbarchard): Consider not allocating row buffer for kFilterLinear. // Allocate a row of UV. @@ -720,6 +903,110 @@ static int ScaleUVBilinearDown(int src_width, } return 0; } + +static int ScaleUVBilinearDown_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy, + enum FilterMode filtering) { + int j; + void (*InterpolateRow)(uint16_t* dst_uv, const uint16_t* src_uv, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) = InterpolateRow_16_C; + void (*ScaleUVFilterCols)(uint16_t* dst_uv, const uint16_t* src_uv, + int dst_width, int x, int dx) = + (src_width >= 32768) ? ScaleUVFilterCols64_16_C : ScaleUVFilterCols_16_C; + int64_t xlast = x + (int64_t)(dst_width - 1) * dx; + int64_t xl = (dx >= 0) ? x : xlast; + int64_t xr = (dx >= 0) ? xlast : x; + int clip_src_width; + xl = (xl >> 16) & ~3; // Left edge aligned. + xr = (xr >> 16) + 1; // Right most pixel used. Bilinear uses 2 pixels. + xr = (xr + 1 + 3) & ~3; // 1 beyond 4 pixel aligned right most pixel. + if (xr > src_width) { + xr = src_width; + } + clip_src_width = (int)(xr - xl) * 2; // Width in uint16_t words. + src_uv += xl * 2; + x -= (int)(xl << 16); +#if defined(HAS_INTERPOLATEROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_16_Any_AVX2; + if (IS_ALIGNED(clip_src_width, 16)) { + InterpolateRow = InterpolateRow_16_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_16_AVX512BW; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_16_Any_NEON; + if (IS_ALIGNED(clip_src_width, 8)) { + InterpolateRow = InterpolateRow_16_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_SME) + if (TestCpuFlag(kCpuHasSME)) { + InterpolateRow = InterpolateRow_16_SME; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + InterpolateRow = InterpolateRow_16_HVX; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_16_Any_WASMSIMD; + if (IS_ALIGNED(clip_src_width, 8)) { + InterpolateRow = InterpolateRow_16_WASMSIMD; + } + } +#endif + // Allocate a row of 16-bit UV. + { + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; + uint32_t uy = y; + align_buffer_64(row, clip_src_width * (int)sizeof(uint16_t)); + if (!row) + return 1; + uint16_t* row16 = (uint16_t*)row; + if (uy > max_y) { + uy = max_y; + } + for (j = 0; j < dst_height; ++j) { + int yi = uy >> 16; + const uint16_t* src = src_uv + yi * src_stride; + if (filtering == kFilterLinear) { + ScaleUVFilterCols(dst_uv, src, dst_width, x, dx); + } else { + int yf = (uy >> 8) & 255; + InterpolateRow(row16, src, src_stride, clip_src_width, yf); + ScaleUVFilterCols(dst_uv, row16, dst_width, x, dx); + } + dst_uv += dst_stride; + uy += dy; + if (uy > max_y) { + uy = max_y; + } + } + free_aligned_buffer_64(row); + } + return 0; +} #endif // Scale UV up with bilinear interpolation. @@ -820,39 +1107,8 @@ static int ScaleUVBilinearUp(int src_width, if (src_width >= 32768) { ScaleUVFilterCols = filtering ? ScaleUVFilterCols64_C : ScaleUVCols64_C; } -#if defined(HAS_SCALEUVFILTERCOLS_SSSE3) - if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleUVFilterCols = ScaleUVFilterCols_SSSE3; - } -#endif -#if defined(HAS_SCALEUVFILTERCOLS_NEON) - if (filtering && TestCpuFlag(kCpuHasNEON)) { - ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON; - if (IS_ALIGNED(dst_width, 8)) { - ScaleUVFilterCols = ScaleUVFilterCols_NEON; - } - } -#endif -#if defined(HAS_SCALEUVCOLS_SSSE3) - if (!filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleUVFilterCols = ScaleUVCols_SSSE3; - } -#endif -#if defined(HAS_SCALEUVCOLS_NEON) - if (!filtering && TestCpuFlag(kCpuHasNEON)) { - ScaleUVFilterCols = ScaleUVCols_Any_NEON; - if (IS_ALIGNED(dst_width, 16)) { - ScaleUVFilterCols = ScaleUVCols_NEON; - } - } -#endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleUVFilterCols = ScaleUVColsUp2_C; -#if defined(HAS_SCALEUVCOLSUP2_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) { - ScaleUVFilterCols = ScaleUVColsUp2_SSSE3; - } -#endif } if (uy > max_y) { @@ -919,6 +1175,136 @@ static int ScaleUVBilinearUp(int src_width, } return 0; } + +static int ScaleUVBilinearUp_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy, + enum FilterMode filtering) { + assert(src_width > 0); + assert(src_height > 0); + assert(dst_width > 0); + assert(dst_height > 0); + assert(dy <= 65536); + + int j; + void (*InterpolateRow)(uint16_t* dst_uv, const uint16_t* src_uv, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) = InterpolateRow_16_C; + void (*ScaleUVFilterCols)(uint16_t* dst_uv, const uint16_t* src_uv, + int dst_width, int x, int dx) = + filtering ? ScaleUVFilterCols_16_C : ScaleUVCols_16_C; + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; + uint32_t uy = y; +#if defined(HAS_INTERPOLATEROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_16_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_16_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_16_AVX512BW; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_16_Any_NEON; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_16_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_SME) + if (TestCpuFlag(kCpuHasSME)) { + InterpolateRow = InterpolateRow_16_SME; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + InterpolateRow = InterpolateRow_16_HVX; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_16_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_16_WASMSIMD; + } + } +#endif + if (src_width >= 32768) { + ScaleUVFilterCols = + filtering ? ScaleUVFilterCols64_16_C : ScaleUVCols64_16_C; + } + if (!filtering && src_width * 2 == dst_width && x < 0x8000) { + ScaleUVFilterCols = ScaleUVColsUp2_16_C; + } + + if (uy > max_y) { + uy = max_y; + } + + { + int yi = uy >> 16; + const uint16_t* src = src_uv + yi * src_stride; + + // Allocate 2 rows of 16-bit UV. + const int row_stride_words = (dst_width * 2 + 15) & ~15; + align_buffer_64(row, row_stride_words * 2 * (int)sizeof(uint16_t)); + if (!row) + return 1; + + uint16_t* rowptr = (uint16_t*)row; + ptrdiff_t rowstride = row_stride_words; + int lasty = yi; + + ScaleUVFilterCols(rowptr, src, dst_width, x, dx); + if (src_height > 1) { + src += src_stride; + } + ScaleUVFilterCols(rowptr + rowstride, src, dst_width, x, dx); + if (src_height > 2) { + src += src_stride; + } + + for (j = 0; j < dst_height; ++j) { + if (uy > max_y) { + uy = max_y; + } + yi = uy >> 16; + if (yi != lasty) { + ScaleUVFilterCols(rowptr, src, dst_width, x, dx); + rowptr += rowstride; + rowstride = -rowstride; + lasty = yi; + if ((uy + 65536) < max_y) { + src += src_stride; + } + } + if (filtering == kFilterLinear) { + InterpolateRow(dst_uv, rowptr, 0, dst_width * 2, 0); + } else { + int yf = (uy >> 8) & 255; + InterpolateRow(dst_uv, rowptr, rowstride, dst_width * 2, yf); + } + dst_uv += dst_stride; + uy += dy; + } + free_aligned_buffer_64(row); + } + return 0; +} #endif // HAS_SCALEUVBILINEARUP // Scale UV, horizontally up by 2 times. @@ -1169,26 +1555,38 @@ static void ScaleUVSimple(int src_width, int x, int dx) = (src_width >= 32768) ? ScaleUVCols64_C : ScaleUVCols_C; (void)src_height; -#if defined(HAS_SCALEUVCOLS_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleUVCols = ScaleUVCols_SSSE3; - } -#endif -#if defined(HAS_SCALEUVCOLS_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ScaleUVCols = ScaleUVCols_Any_NEON; - if (IS_ALIGNED(dst_width, 8)) { - ScaleUVCols = ScaleUVCols_NEON; - } - } -#endif if (src_width * 2 == dst_width && x < 0x8000) { ScaleUVCols = ScaleUVColsUp2_C; -#if defined(HAS_SCALEUVCOLSUP2_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) { - ScaleUVCols = ScaleUVColsUp2_SSSE3; - } -#endif + } + + uint32_t uy = y; + for (j = 0; j < dst_height; ++j) { + int yi = uy >> 16; + ScaleUVCols(dst_uv, src_uv + yi * src_stride, dst_width, x, dx); + dst_uv += dst_stride; + uy += dy; + } +} + +static void ScaleUVSimple_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy) { + int j; + void (*ScaleUVCols)(uint16_t* dst_uv, const uint16_t* src_uv, int dst_width, + int x, int dx) = + (src_width >= 32768) ? ScaleUVCols64_16_C : ScaleUVCols_16_C; + (void)src_height; + if (src_width * 2 == dst_width && x < 0x8000) { + ScaleUVCols = ScaleUVColsUp2_16_C; } uint32_t uy = y; @@ -1402,8 +1800,166 @@ int UVScale(const uint8_t* src_uv, dst_height, filtering); } +// Scale a 16-bit UV plane (from P010/P210) +static int ScaleUV_16(const uint16_t* src, + int src_stride, + int src_width, + int src_height, + uint16_t* dst, + int dst_stride, + int dst_width, + int dst_height, + int clip_x, + int clip_y, + int clip_width, + int clip_height, + enum FilterMode filtering) { + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + const enum FilterMode orig_filtering = filtering; + // Simplify filtering when possible. + filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height, + filtering); + + // Negative src_height means invert the image. + if (src_height < 0) { + src_height = -src_height; + src = src + (src_height - 1) * (ptrdiff_t)src_stride; + src_stride = -src_stride; + } + ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y, + &dx, &dy); + if (clip_x) { + int64_t clipf = (int64_t)(clip_x)*dx; + x += (clipf & 0xffff); + src += (clipf >> 16) * 2; + dst += clip_x * 2; + } + if (clip_y) { + int64_t clipf = (int64_t)(clip_y)*dy; + y += (clipf & 0xffff); + src += (clipf >> 16) * (ptrdiff_t)src_stride; + dst += clip_y * (ptrdiff_t)dst_stride; + } + +#if HAS_SCALEUVBOX + // Exactly 4 goes to the faster ScaleUVDown4Box_16 below. Clipping is excluded + // because whole source rows are accumulated. + if (filtering == kFilterBox && !(dx == 0x40000 && dy == 0x40000) && + clip_x == 0 && clip_y == 0) { + return ScaleUVBox_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, x, dx, y, dy); + } +#endif + + // Special case for integer step values. + if (((dx | dy) & 0xffff) == 0) { + if (!dx || !dy) { // 1 pixel wide and/or tall. + filtering = kFilterNone; + } else { + // Optimized even scale down. ie 2, 4, 6, 8, 10x. + if (!(dx & 0x10000) && !(dy & 0x10000)) { +#if HAS_SCALEUVDOWN2 + if (dx == 0x20000 && dy == 0x20000) { + // Optimized 1/2 downsample. + ScaleUVDown2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, x, dx, y, dy, + filtering); + return 0; + } +#endif +#if HAS_SCALEUVDOWN4BOX + if (dx == 0x40000 && dy == 0x40000 && filtering == kFilterBox) { + // Optimized 1/4 box downsample. + return ScaleUVDown4Box_16(src_width, src_height, clip_width, + clip_height, src_stride, dst_stride, src, + dst, x, dx, y, dy); + } +#endif +#if HAS_SCALEUVDOWNEVEN + ScaleUVDownEven_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, x, dx, y, dy, + filtering); + return 0; +#endif + } + // Optimized odd scale down. ie 3, 5, 7, 9x. + if ((dx & 0x10000) && (dy & 0x10000)) { + filtering = kFilterNone; +#ifdef HAS_UVCOPY + if (dx == 0x10000 && dy == 0x10000) { + // Straight copy. + return UVCopy_16( + src + (y >> 16) * (ptrdiff_t)src_stride + (x >> 16) * 2, + src_stride, dst, dst_stride, clip_width, clip_height); + } +#endif + } + } + } + + // ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone to avoid + // reading a second source column. The Up2 kernels clamp their last pixel and + // only read the 1 source column when dst_width <= 2, so restore Linear or + // Bilinear for 2x upscales (e.g., P010ToP410 and P210ToP410 on 1-2 pixel wide + // images) when the caller requested filtering. + if (orig_filtering != kFilterNone && filtering == kFilterNone && + src_width == 1 && dst_width <= 2 && clip_x == 0 && clip_y == 0) { + if (clip_height == src_height && (dst_width + 1) / 2 == src_width) { + ScaleUVLinearUp2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst); + return 0; + } + if ((clip_height + 1) / 2 == src_height && + (clip_width + 1) / 2 == src_width && orig_filtering != kFilterLinear) { + ScaleUVBilinearUp2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst); + return 0; + } + } + + // HAS_SCALEPLANEVERTICAL + if (dx == 0x10000 && (x & 0xffff) == 0) { + // Arbitrary scale vertically, but unscaled horizontally. + ScalePlaneVertical_16(src_height, clip_width, clip_height, src_stride, + dst_stride, src, dst, x, y, dy, /*wpp=*/2, filtering); + return 0; + } + if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) { + ScaleUVLinearUp2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst); + return 0; + } + if ((clip_height + 1) / 2 == src_height && + (clip_width + 1) / 2 == src_width && + (filtering == kFilterBilinear || filtering == kFilterBox)) { + ScaleUVBilinearUp2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst); + return 0; + } +#if HAS_SCALEUVBILINEARUP + if (filtering && dy < 65536) { + return ScaleUVBilinearUp_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, x, dx, y, dy, + filtering); + } +#endif +#if HAS_SCALEUVBILINEARDOWN + if (filtering) { + return ScaleUVBilinearDown_16(src_width, src_height, clip_width, + clip_height, src_stride, dst_stride, src, dst, + x, dx, y, dy, filtering); + } +#endif + ScaleUVSimple_16(src_width, src_height, clip_width, clip_height, src_stride, + dst_stride, src, dst, x, dx, y, dy); + return 0; +} + // Scale a 16 bit UV image. -// This function is currently incomplete, it can't handle all cases. LIBYUV_API int UVScale_16(const uint16_t* src_uv, int src_stride_uv, @@ -1414,65 +1970,14 @@ int UVScale_16(const uint16_t* src_uv, int dst_width, int dst_height, enum FilterMode filtering) { - int dy = 0; - if (!src_uv || src_width <= 0 || src_height == 0 || src_width > 32768 || src_height < -32768 || src_height > 32768 || !dst_uv || dst_width <= 0 || dst_height <= 0) { return -1; } - - // UV does not support box filter yet, but allow the user to pass it. - // Simplify filtering when possible. - filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height, - filtering); - - // Negative src_height means invert the image. - if (src_height < 0) { - src_height = -src_height; - src_uv = src_uv + (src_height - 1) * (ptrdiff_t)src_stride_uv; - src_stride_uv = -src_stride_uv; - } - -#ifdef HAS_UVCOPY - if (!filtering && src_width == dst_width && (src_height % dst_height == 0)) { - if (dst_height == 1) { - return UVCopy_16( - src_uv + ((src_height - 1) / 2) * (ptrdiff_t)src_stride_uv, - src_stride_uv, dst_uv, dst_stride_uv, dst_width, dst_height); - } - dy = src_height / dst_height; - if (src_stride_uv > INT_MAX / dy) { - return -1; - } - return UVCopy_16(src_uv + ((dy - 1) / 2) * (ptrdiff_t)src_stride_uv, - dy * src_stride_uv, dst_uv, dst_stride_uv, dst_width, - dst_height); - } -#endif - - // ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone, which has - // no 16 bit implementation other than the copy above. The Up2 kernels only - // read the pixels they need, so restore Linear or Bilinear for a 2x upscale - // so P010ToP410 and P210ToP410 work on 1 pixel wide images. - if (filtering == kFilterNone && src_width == 1 && dst_width <= 2) { - filtering = (src_height == dst_height) ? kFilterLinear : kFilterBilinear; - } - - if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) { - ScaleUVLinearUp2_16(src_width, src_height, dst_width, dst_height, - src_stride_uv, dst_stride_uv, src_uv, dst_uv); - return 0; - } - - if ((dst_height + 1) / 2 == src_height && (dst_width + 1) / 2 == src_width && - (filtering == kFilterBilinear || filtering == kFilterBox)) { - ScaleUVBilinearUp2_16(src_width, src_height, dst_width, dst_height, - src_stride_uv, dst_stride_uv, src_uv, dst_uv); - return 0; - } - - return -1; + return ScaleUV_16(src_uv, src_stride_uv, src_width, src_height, dst_uv, + dst_stride_uv, dst_width, dst_height, 0, 0, dst_width, + dst_height, filtering); } #ifdef __cplusplus diff --git a/source/scale_win.cc b/source/scale_win.cc index f2ad7f078..d63877083 100644 --- a/source/scale_win.cc +++ b/source/scale_win.cc @@ -18,6 +18,7 @@ defined(LIBYUV_ENABLE_ROWWIN)) #include #include +#include #include #if defined(_MSC_VER) && !defined(__clang__) #include @@ -31,43 +32,9 @@ extern "C" { // This module is for 32 bit Visual C x86 #if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) && \ - (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN)) - -// Offsets for source bytes 0 to 9 -static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9, - 128, 128, 128, 128, 128, 128, 128, 128}; - -// Offsets for source bytes 11 to 20 with 8 subtracted = 3 to 12. -static const uvec8 kShuf1 = {3, 4, 5, 7, 8, 9, 11, 12, - 128, 128, 128, 128, 128, 128, 128, 128}; - -// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. -static const uvec8 kShuf2 = {5, 7, 8, 9, 11, 12, 13, 15, - 128, 128, 128, 128, 128, 128, 128, 128}; - -// Offsets for source bytes 0 to 10 -static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10}; - -// Offsets for source bytes 10 to 21 with 8 subtracted = 3 to 13. -static const uvec8 kShuf11 = {2, 3, 4, 5, 5, 6, 6, 7, - 8, 9, 9, 10, 10, 11, 12, 13}; - -// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. -static const uvec8 kShuf21 = {5, 6, 6, 7, 8, 9, 9, 10, - 10, 11, 12, 13, 13, 14, 14, 15}; - -// Coefficients for source bytes 0 to 10 -static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2}; - -// Coefficients for source bytes 10 to 21 -static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1}; - -// Coefficients for source bytes 21 to 31 -static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3}; - -// Coefficients for source bytes 21 to 31 -static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2}; + !defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) +#ifdef HAS_SCALEROWDOWN2_SSSE3 // Reads 32 pixels, throws half away and writes 16 pixels. __declspec(naked) void ScaleRowDown2_SSSE3(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -172,122 +139,9 @@ __declspec(naked) void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr, ret } } +#endif // HAS_SCALEROWDOWN2_SSSE3 -#ifdef HAS_SCALEROWDOWN2_AVX2 -// Reads 64 pixels, throws half away and writes 32 pixels. -__declspec(naked) void ScaleRowDown2_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_ptr - // src_stride ignored - mov edx, [esp + 12] // dst_ptr - mov ecx, [esp + 16] // dst_width - - wloop: - vmovdqu ymm0, [eax] - vmovdqu ymm1, [eax + 32] - lea eax, [eax + 64] - vpsrlw ymm0, ymm0, 8 // isolate odd pixels. - vpsrlw ymm1, ymm1, 8 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], ymm0 - lea edx, [edx + 32] - sub ecx, 32 - jg wloop - - vzeroupper - ret - } -} - -// Blends 64x1 rectangle to 32x1. -__declspec(naked) void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_ptr - // src_stride - mov edx, [esp + 12] // dst_ptr - mov ecx, [esp + 16] // dst_width - - vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b - vpsrlw ymm4, ymm4, 15 - vpackuswb ymm4, ymm4, ymm4 - vpxor ymm5, ymm5, ymm5 // constant 0 - - wloop: - vmovdqu ymm0, [eax] - vmovdqu ymm1, [eax + 32] - lea eax, [eax + 64] - vpmaddubsw ymm0, ymm0, ymm4 // horizontal add - vpmaddubsw ymm1, ymm1, ymm4 - vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2 - vpavgw ymm1, ymm1, ymm5 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], ymm0 - lea edx, [edx + 32] - sub ecx, 32 - jg wloop - - vzeroupper - ret - } -} - -// For rounding, average = (sum + 2) / 4 -// becomes average((sum >> 1), 0) -// Blends 64x2 rectangle to 32x1. -__declspec(naked) void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_ptr - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_ptr - mov ecx, [esp + 4 + 16] // dst_width - - vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b - vpsrlw ymm4, ymm4, 15 - vpackuswb ymm4, ymm4, ymm4 - vpxor ymm5, ymm5, ymm5 // constant 0 - - wloop: - vmovdqu ymm0, [eax] - vmovdqu ymm1, [eax + 32] - vmovdqu ymm2, [eax + esi] - vmovdqu ymm3, [eax + esi + 32] - lea eax, [eax + 64] - vpmaddubsw ymm0, ymm0, ymm4 // horizontal add - vpmaddubsw ymm1, ymm1, ymm4 - vpmaddubsw ymm2, ymm2, ymm4 - vpmaddubsw ymm3, ymm3, ymm4 - vpaddw ymm0, ymm0, ymm2 // vertical add - vpaddw ymm1, ymm1, ymm3 - vpsrlw ymm0, ymm0, 1 // (x + 2) / 4 = (x / 2 + 1) / 2 - vpsrlw ymm1, ymm1, 1 - vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2 - vpavgw ymm1, ymm1, ymm5 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], ymm0 - lea edx, [edx + 32] - sub ecx, 32 - jg wloop - - pop esi - vzeroupper - ret - } -} -#endif // HAS_SCALEROWDOWN2_AVX2 - +#ifdef HAS_SCALEROWDOWN4_SSSE3 // Point samples 32 pixels to 8 pixels. __declspec(naked) void ScaleRowDown4_SSSE3(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -377,755 +231,9 @@ __declspec(naked) void ScaleRowDown4Box_SSSE3(const uint8_t* src_ptr, ret } } +#endif // HAS_SCALEROWDOWN4_SSSE3 -#ifdef HAS_SCALEROWDOWN4_AVX2 -// Point samples 64 pixels to 16 pixels. -__declspec(naked) void ScaleRowDown4_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_ptr - // src_stride ignored - mov edx, [esp + 12] // dst_ptr - mov ecx, [esp + 16] // dst_width - vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff0000 - vpsrld ymm5, ymm5, 24 - vpslld ymm5, ymm5, 16 - - wloop: - vmovdqu ymm0, [eax] - vmovdqu ymm1, [eax + 32] - lea eax, [eax + 64] - vpand ymm0, ymm0, ymm5 - vpand ymm1, ymm1, ymm5 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vpsrlw ymm0, ymm0, 8 - vpackuswb ymm0, ymm0, ymm0 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 16 - jg wloop - - vzeroupper - ret - } -} - -// Blends 64x4 rectangle to 16x1. -__declspec(naked) void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - push edi - mov eax, [esp + 8 + 4] // src_ptr - mov esi, [esp + 8 + 8] // src_stride - mov edx, [esp + 8 + 12] // dst_ptr - mov ecx, [esp + 8 + 16] // dst_width - lea edi, [esi + esi * 2] // src_stride * 3 - vpcmpeqb ymm4, ymm4, ymm4 // constant 0x0101 - vpsrlw ymm4, ymm4, 15 - vpsllw ymm5, ymm4, 3 // constant 0x0008 - vpackuswb ymm4, ymm4, ymm4 - - wloop: - vmovdqu ymm0, [eax] // average rows - vmovdqu ymm1, [eax + 32] - vmovdqu ymm2, [eax + esi] - vmovdqu ymm3, [eax + esi + 32] - vpmaddubsw ymm0, ymm0, ymm4 // horizontal add - vpmaddubsw ymm1, ymm1, ymm4 - vpmaddubsw ymm2, ymm2, ymm4 - vpmaddubsw ymm3, ymm3, ymm4 - vpaddw ymm0, ymm0, ymm2 // vertical add rows 0, 1 - vpaddw ymm1, ymm1, ymm3 - vmovdqu ymm2, [eax + esi * 2] - vmovdqu ymm3, [eax + esi * 2 + 32] - vpmaddubsw ymm2, ymm2, ymm4 - vpmaddubsw ymm3, ymm3, ymm4 - vpaddw ymm0, ymm0, ymm2 // add row 2 - vpaddw ymm1, ymm1, ymm3 - vmovdqu ymm2, [eax + edi] - vmovdqu ymm3, [eax + edi + 32] - lea eax, [eax + 64] - vpmaddubsw ymm2, ymm2, ymm4 - vpmaddubsw ymm3, ymm3, ymm4 - vpaddw ymm0, ymm0, ymm2 // add row 3 - vpaddw ymm1, ymm1, ymm3 - vphaddw ymm0, ymm0, ymm1 // mutates - vpermq ymm0, ymm0, 0xd8 // unmutate vphaddw - vpaddw ymm0, ymm0, ymm5 // + 8 for round - vpsrlw ymm0, ymm0, 4 // /32 for average of 4 * 4 - vpackuswb ymm0, ymm0, ymm0 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 16 - jg wloop - - pop edi - pop esi - vzeroupper - ret - } -} -#endif // HAS_SCALEROWDOWN4_AVX2 - -// Point samples 32 pixels to 24 pixels. -// Produces three 8 byte values. For each 8 bytes, 16 bytes are read. -// Then shuffled to do the scaling. - -__declspec(naked) void ScaleRowDown34_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_ptr - // src_stride ignored - mov edx, [esp + 12] // dst_ptr - mov ecx, [esp + 16] // dst_width - movdqa xmm3, xmmword ptr kShuf0 - movdqa xmm4, xmmword ptr kShuf1 - movdqa xmm5, xmmword ptr kShuf2 - - wloop: - movdqu xmm0, [eax] - movdqu xmm1, [eax + 16] - lea eax, [eax + 32] - movdqa xmm2, xmm1 - palignr xmm1, xmm0, 8 - pshufb xmm0, xmm3 - pshufb xmm1, xmm4 - pshufb xmm2, xmm5 - movq qword ptr [edx], xmm0 - movq qword ptr [edx + 8], xmm1 - movq qword ptr [edx + 16], xmm2 - lea edx, [edx + 24] - sub ecx, 24 - jg wloop - - ret - } -} - -// Blends 32x2 rectangle to 24x1 -// Produces three 8 byte values. For each 8 bytes, 16 bytes are read. -// Then shuffled to do the scaling. - -// Register usage: -// xmm0 src_row 0 -// xmm1 src_row 1 -// xmm2 shuf 0 -// xmm3 shuf 1 -// xmm4 shuf 2 -// xmm5 madd 0 -// xmm6 madd 1 -// xmm7 kRound34 - -// Note that movdqa+palign may be better than movdqu. -__declspec(naked) void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_ptr - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_ptr - mov ecx, [esp + 4 + 16] // dst_width - movdqa xmm2, xmmword ptr kShuf01 - movdqa xmm3, xmmword ptr kShuf11 - movdqa xmm4, xmmword ptr kShuf21 - movdqa xmm5, xmmword ptr kMadd01 - movdqa xmm6, xmmword ptr kMadd11 - movdqa xmm7, xmmword ptr kRound34 - - wloop: - movdqu xmm0, [eax] // pixels 0..7 - movdqu xmm1, [eax + esi] - pavgb xmm0, xmm1 - pshufb xmm0, xmm2 - pmaddubsw xmm0, xmm5 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx], xmm0 - movdqu xmm0, [eax + 8] // pixels 8..15 - movdqu xmm1, [eax + esi + 8] - pavgb xmm0, xmm1 - pshufb xmm0, xmm3 - pmaddubsw xmm0, xmm6 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx + 8], xmm0 - movdqu xmm0, [eax + 16] // pixels 16..23 - movdqu xmm1, [eax + esi + 16] - lea eax, [eax + 32] - pavgb xmm0, xmm1 - pshufb xmm0, xmm4 - movdqa xmm1, xmmword ptr kMadd21 - pmaddubsw xmm0, xmm1 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx + 16], xmm0 - lea edx, [edx + 24] - sub ecx, 24 - jg wloop - - pop esi - ret - } -} - -// Note that movdqa+palign may be better than movdqu. -__declspec(naked) void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_ptr - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_ptr - mov ecx, [esp + 4 + 16] // dst_width - movdqa xmm2, xmmword ptr kShuf01 - movdqa xmm3, xmmword ptr kShuf11 - movdqa xmm4, xmmword ptr kShuf21 - movdqa xmm5, xmmword ptr kMadd01 - movdqa xmm6, xmmword ptr kMadd11 - movdqa xmm7, xmmword ptr kRound34 - - wloop: - movdqu xmm0, [eax] // pixels 0..7 - movdqu xmm1, [eax + esi] - pavgb xmm1, xmm0 - pavgb xmm0, xmm1 - pshufb xmm0, xmm2 - pmaddubsw xmm0, xmm5 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx], xmm0 - movdqu xmm0, [eax + 8] // pixels 8..15 - movdqu xmm1, [eax + esi + 8] - pavgb xmm1, xmm0 - pavgb xmm0, xmm1 - pshufb xmm0, xmm3 - pmaddubsw xmm0, xmm6 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx + 8], xmm0 - movdqu xmm0, [eax + 16] // pixels 16..23 - movdqu xmm1, [eax + esi + 16] - lea eax, [eax + 32] - pavgb xmm1, xmm0 - pavgb xmm0, xmm1 - pshufb xmm0, xmm4 - movdqa xmm1, xmmword ptr kMadd21 - pmaddubsw xmm0, xmm1 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx + 16], xmm0 - lea edx, [edx+24] - sub ecx, 24 - jg wloop - - pop esi - ret - } -} - -static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040, - 0x0040, 0x0040, 0x0040, 0x0040}; - -// Bilinear column filtering. SSSE3 version. -__declspec(naked) void ScaleFilterCols_SSSE3(uint8_t* dst_ptr, - const uint8_t* src_ptr, - int dst_width, - int x, - int dx) { - __asm { - push ebx - push esi - push edi - mov edi, [esp + 12 + 4] // dst_ptr - mov esi, [esp + 12 + 8] // src_ptr - mov ecx, [esp + 12 + 12] // dst_width - movd xmm2, [esp + 12 + 16] // x - movd xmm3, [esp + 12 + 20] // dx - mov eax, 0x04040000 // shuffle to line up fractions with pixel. - movd xmm5, eax - pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction. - psrlw xmm6, 9 - pcmpeqb xmm7, xmm7 // generate 0x00ff mask for extracting 'a' - psrlw xmm7, 8 - pextrw eax, xmm2, 1 // get x0 integer. preroll - sub ecx, 2 - jl xloop29 - - movdqa xmm0, xmm2 // x1 = x0 + dx - paddd xmm0, xmm3 - punpckldq xmm2, xmm0 // x0 x1 - punpckldq xmm3, xmm3 // dx dx - paddd xmm3, xmm3 // dx * 2, dx * 2 - pextrw edx, xmm2, 3 // get x1 integer. preroll - - // 2 Pixel loop. - xloop2: - movdqa xmm1, xmm2 // x0, x1 fractions. - paddd xmm2, xmm3 // x += dx - movzx ebx, word ptr [esi + eax] // 2 source x0 pixels - movd xmm0, ebx - psrlw xmm1, 9 // 7 bit fractions. - movzx ebx, word ptr [esi + edx] // 2 source x1 pixels - movd xmm4, ebx - pshufb xmm1, xmm5 // 0011 - punpcklwd xmm0, xmm4 - pxor xmm1, xmm6 // 0..7f and 7f..0 - movdqa xmm4, xmm0 // Copy pixels. - pmaddubsw xmm0, xmm1 // a * (127 - f) + b * f - pand xmm4, xmm7 // Extract left pixels 'a'. - paddw xmm0, xmm4 // a * 128 + f * (b - a) - pextrw eax, xmm2, 1 // get x0 integer. next iteration. - paddw xmm0, xmmword ptr kFadd40 // add rounding bias 64. - pextrw edx, xmm2, 3 // get x1 integer. next iteration. - psrlw xmm0, 7 // 8.7 fixed point to low 8 bits. - packuswb xmm0, xmm0 // 8 bits, 2 pixels. - movd ebx, xmm0 - mov [edi], bx - lea edi, [edi + 2] - sub ecx, 2 // 2 pixels - jge xloop2 - - xloop29: - add ecx, 2 - 1 - jl xloop99 - - // 1 pixel remainder - movzx ebx, word ptr [esi + eax] // 2 source x0 pixels - movd xmm0, ebx - psrlw xmm2, 9 // 7 bit fractions. - pshufb xmm2, xmm5 // 0011 - pxor xmm2, xmm6 // 0..7f and 7f..0 - movdqa xmm4, xmm0 // Copy pixels. - pmaddubsw xmm0, xmm2 // a * (127 - f) + b * f - pand xmm4, xmm7 // Extract left pixel 'a'. - paddw xmm0, xmm4 // a * 128 + f * (b - a) - paddw xmm0, xmmword ptr kFadd40 // add rounding bias 64. - psrlw xmm0, 7 // 8.7 fixed point to low 8 bits. - packuswb xmm0, xmm0 // 8 bits - movd ebx, xmm0 - mov [edi], bl - - xloop99: - - pop edi - pop esi - pop ebx - ret - } -} - -// Reads 16 pixels, duplicates them and writes 32 pixels. -__declspec(naked) void ScaleColsUp2_SSE2(uint8_t* dst_ptr, - const uint8_t* src_ptr, - int dst_width, - int x, - int dx) { - __asm { - mov edx, [esp + 4] // dst_ptr - mov eax, [esp + 8] // src_ptr - mov ecx, [esp + 12] // dst_width - - wloop: - movdqu xmm0, [eax] - lea eax, [eax + 16] - movdqa xmm1, xmm0 - punpcklbw xmm0, xmm0 - punpckhbw xmm1, xmm1 - movdqu [edx], xmm0 - movdqu [edx + 16], xmm1 - lea edx, [edx + 32] - sub ecx, 32 - jg wloop - - ret - } -} - -// Reads 8 pixels, throws half away and writes 4 even pixels (0, 2, 4, 6) -__declspec(naked) void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - uint8_t* dst_argb, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_argb - // src_stride ignored - mov edx, [esp + 12] // dst_argb - mov ecx, [esp + 16] // dst_width - - wloop: - movdqu xmm0, [eax] - movdqu xmm1, [eax + 16] - lea eax, [eax + 32] - shufps xmm0, xmm1, 0xdd - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - ret - } -} - -// Blends 8x1 rectangle to 4x1. -__declspec(naked) void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - uint8_t* dst_argb, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_argb - // src_stride ignored - mov edx, [esp + 12] // dst_argb - mov ecx, [esp + 16] // dst_width - - wloop: - movdqu xmm0, [eax] - movdqu xmm1, [eax + 16] - lea eax, [eax + 32] - movdqa xmm2, xmm0 - shufps xmm0, xmm1, 0x88 // even pixels - shufps xmm2, xmm1, 0xdd // odd pixels - pavgb xmm0, xmm2 - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - ret - } -} - -// Blends 8x2 rectangle to 4x1. -__declspec(naked) void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - uint8_t* dst_argb, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_argb - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_argb - mov ecx, [esp + 4 + 16] // dst_width - - wloop: - movdqu xmm0, [eax] - movdqu xmm1, [eax + 16] - movdqu xmm2, [eax + esi] - movdqu xmm3, [eax + esi + 16] - lea eax, [eax + 32] - pavgb xmm0, xmm2 // average rows - pavgb xmm1, xmm3 - movdqa xmm2, xmm0 // average columns (8 to 4 pixels) - shufps xmm0, xmm1, 0x88 // even pixels - shufps xmm2, xmm1, 0xdd // odd pixels - pavgb xmm0, xmm2 - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - pop esi - ret - } -} - -// Reads 4 pixels at a time. -__declspec(naked) void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - int src_stepx, - uint8_t* dst_argb, - int dst_width) { - __asm { - push ebx - push edi - mov eax, [esp + 8 + 4] // src_argb - // src_stride ignored - mov ebx, [esp + 8 + 12] // src_stepx - mov edx, [esp + 8 + 16] // dst_argb - mov ecx, [esp + 8 + 20] // dst_width - lea ebx, [ebx * 4] - lea edi, [ebx + ebx * 2] - - wloop: - movd xmm0, [eax] - movd xmm1, [eax + ebx] - punpckldq xmm0, xmm1 - movd xmm2, [eax + ebx * 2] - movd xmm3, [eax + edi] - lea eax, [eax + ebx * 4] - punpckldq xmm2, xmm3 - punpcklqdq xmm0, xmm2 - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - pop edi - pop ebx - ret - } -} - -// Blends four 2x2 to 4x1. -__declspec(naked) void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - int src_stepx, - uint8_t* dst_argb, - int dst_width) { - __asm { - push ebx - push esi - push edi - mov eax, [esp + 12 + 4] // src_argb - mov esi, [esp + 12 + 8] // src_stride - mov ebx, [esp + 12 + 12] // src_stepx - mov edx, [esp + 12 + 16] // dst_argb - mov ecx, [esp + 12 + 20] // dst_width - lea esi, [eax + esi] // row1 pointer - lea ebx, [ebx * 4] - lea edi, [ebx + ebx * 2] - - wloop: - movq xmm0, qword ptr [eax] // row0 4 pairs - movhps xmm0, qword ptr [eax + ebx] - movq xmm1, qword ptr [eax + ebx * 2] - movhps xmm1, qword ptr [eax + edi] - lea eax, [eax + ebx * 4] - movq xmm2, qword ptr [esi] // row1 4 pairs - movhps xmm2, qword ptr [esi + ebx] - movq xmm3, qword ptr [esi + ebx * 2] - movhps xmm3, qword ptr [esi + edi] - lea esi, [esi + ebx * 4] - pavgb xmm0, xmm2 // average rows - pavgb xmm1, xmm3 - movdqa xmm2, xmm0 // average columns (8 to 4 pixels) - shufps xmm0, xmm1, 0x88 // even pixels - shufps xmm2, xmm1, 0xdd // odd pixels - pavgb xmm0, xmm2 - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - pop edi - pop esi - pop ebx - ret - } -} - -// Column scaling unfiltered. SSE2 version. -__declspec(naked) void ScaleARGBCols_SSE2(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx) { - __asm { - push edi - push esi - mov edi, [esp + 8 + 4] // dst_argb - mov esi, [esp + 8 + 8] // src_argb - mov ecx, [esp + 8 + 12] // dst_width - movd xmm2, [esp + 8 + 16] // x - movd xmm3, [esp + 8 + 20] // dx - - pshufd xmm2, xmm2, 0 // x0 x0 x0 x0 - pshufd xmm0, xmm3, 0x11 // dx 0 dx 0 - paddd xmm2, xmm0 - paddd xmm3, xmm3 // 0, 0, 0, dx * 2 - pshufd xmm0, xmm3, 0x05 // dx * 2, dx * 2, 0, 0 - paddd xmm2, xmm0 // x3 x2 x1 x0 - paddd xmm3, xmm3 // 0, 0, 0, dx * 4 - pshufd xmm3, xmm3, 0 // dx * 4, dx * 4, dx * 4, dx * 4 - - pextrw eax, xmm2, 1 // get x0 integer. - pextrw edx, xmm2, 3 // get x1 integer. - - cmp ecx, 0 - jle xloop99 - sub ecx, 4 - jl xloop49 - - // 4 Pixel loop. - xloop4: - movd xmm0, [esi + eax * 4] // 1 source x0 pixels - movd xmm1, [esi + edx * 4] // 1 source x1 pixels - pextrw eax, xmm2, 5 // get x2 integer. - pextrw edx, xmm2, 7 // get x3 integer. - paddd xmm2, xmm3 // x += dx - punpckldq xmm0, xmm1 // x0 x1 - - movd xmm1, [esi + eax * 4] // 1 source x2 pixels - movd xmm4, [esi + edx * 4] // 1 source x3 pixels - pextrw eax, xmm2, 1 // get x0 integer. next iteration. - pextrw edx, xmm2, 3 // get x1 integer. next iteration. - punpckldq xmm1, xmm4 // x2 x3 - punpcklqdq xmm0, xmm1 // x0 x1 x2 x3 - movdqu [edi], xmm0 - lea edi, [edi + 16] - sub ecx, 4 // 4 pixels - jge xloop4 - - xloop49: - test ecx, 2 - je xloop29 - - // 2 Pixels. - movd xmm0, [esi + eax * 4] // 1 source x0 pixels - movd xmm1, [esi + edx * 4] // 1 source x1 pixels - pextrw eax, xmm2, 5 // get x2 integer. - punpckldq xmm0, xmm1 // x0 x1 - - movq qword ptr [edi], xmm0 - lea edi, [edi + 8] - - xloop29: - test ecx, 1 - je xloop99 - - // 1 Pixels. - movd xmm0, [esi + eax * 4] // 1 source x2 pixels - movd dword ptr [edi], xmm0 - xloop99: - - pop esi - pop edi - ret - } -} - -// Bilinear row filtering combines 2x1 -> 1x1. SSSE3 version. -// TODO(fbarchard): Port to Neon - -// Shuffle table for arranging 2 pixels into pairs for pmaddubsw -static const uvec8 kShuffleColARGB = { - 0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u, // bbggrraa 1st pixel - 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u // bbggrraa 2nd pixel -}; - -// Shuffle table for duplicating 2 fractions into 8 bytes each -static const uvec8 kShuffleFractions = { - 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, -}; - -__declspec(naked) void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx) { - __asm { - push esi - push edi - mov edi, [esp + 8 + 4] // dst_argb - mov esi, [esp + 8 + 8] // src_argb - mov ecx, [esp + 8 + 12] // dst_width - movd xmm2, [esp + 8 + 16] // x - movd xmm3, [esp + 8 + 20] // dx - movdqa xmm4, xmmword ptr kShuffleColARGB - movdqa xmm5, xmmword ptr kShuffleFractions - pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction. - psrlw xmm6, 9 - pextrw eax, xmm2, 1 // get x0 integer. preroll - sub ecx, 2 - jl xloop29 - - movdqa xmm0, xmm2 // x1 = x0 + dx - paddd xmm0, xmm3 - punpckldq xmm2, xmm0 // x0 x1 - punpckldq xmm3, xmm3 // dx dx - paddd xmm3, xmm3 // dx * 2, dx * 2 - pextrw edx, xmm2, 3 // get x1 integer. preroll - - // 2 Pixel loop. - xloop2: - movdqa xmm1, xmm2 // x0, x1 fractions. - paddd xmm2, xmm3 // x += dx - movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels - psrlw xmm1, 9 // 7 bit fractions. - movhps xmm0, qword ptr [esi + edx * 4] // 2 source x1 pixels - pshufb xmm1, xmm5 // 0000000011111111 - pshufb xmm0, xmm4 // arrange pixels into pairs - pxor xmm1, xmm6 // 0..7f and 7f..0 - pmaddubsw xmm0, xmm1 // argb_argb 16 bit, 2 pixels. - pextrw eax, xmm2, 1 // get x0 integer. next iteration. - pextrw edx, xmm2, 3 // get x1 integer. next iteration. - psrlw xmm0, 7 // argb 8.7 fixed point to low 8 bits. - packuswb xmm0, xmm0 // argb_argb 8 bits, 2 pixels. - movq qword ptr [edi], xmm0 - lea edi, [edi + 8] - sub ecx, 2 // 2 pixels - jge xloop2 - - xloop29: - - add ecx, 2 - 1 - jl xloop99 - - // 1 pixel remainder - psrlw xmm2, 9 // 7 bit fractions. - movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels - pshufb xmm2, xmm5 // 00000000 - pshufb xmm0, xmm4 // arrange pixels into pairs - pxor xmm2, xmm6 // 0..7f and 7f..0 - pmaddubsw xmm0, xmm2 // argb 16 bit, 1 pixel. - psrlw xmm0, 7 - packuswb xmm0, xmm0 // argb 8 bits, 1 pixel. - movd [edi], xmm0 - - xloop99: - - pop edi - pop esi - ret - } -} - -// Reads 4 pixels, duplicates them and writes 8 pixels. -__declspec(naked) void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx) { - __asm { - mov edx, [esp + 4] // dst_argb - mov eax, [esp + 8] // src_argb - mov ecx, [esp + 12] // dst_width - - wloop: - movdqu xmm0, [eax] - lea eax, [eax + 16] - movdqa xmm1, xmm0 - punpckldq xmm0, xmm0 - punpckhdq xmm1, xmm1 - movdqu [edx], xmm0 - movdqu [edx + 16], xmm1 - lea edx, [edx + 32] - sub ecx, 8 - jg wloop - - ret - } -} - +#ifdef HAS_FIXEDDIV_X86 // Divide num by div and return as 16.16 fixed point result. __declspec(naked) int FixedDiv_X86(int num, int div) { __asm { @@ -1153,7 +261,8 @@ __declspec(naked) int FixedDiv1_X86(int num, int div) { ret } } -#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) +#endif // HAS_FIXEDDIV_X86 +#endif // !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) // The following are available for Visual C 32/64 bit, and for clang with // LIBYUV_ENABLE_ROWWIN: @@ -1164,145 +273,739 @@ __declspec(naked) int FixedDiv1_X86(int num, int div) { defined(LIBYUV_ENABLE_ROWWIN)) #if defined(__clang__) || defined(__GNUC__) -#define LIBYUV_TARGET_SSSE3 __attribute__((target("ssse3"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) #define LIBYUV_TARGET_AVX512VBMI \ __attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f"))) #else -#define LIBYUV_TARGET_SSSE3 #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW #define LIBYUV_TARGET_AVX512VBMI #endif // defined(__clang__) || defined(__GNUC__) -#ifdef HAS_SCALEROWDOWN38_SSSE3 +static inline uint16_t LoadU16(const void* p) { + uint16_t v; + memcpy(&v, p, sizeof(v)); + return v; +} + +static inline uint32_t LoadU32(const void* p) { + uint32_t v; + memcpy(&v, p, sizeof(v)); + return v; +} + +static inline uint64_t LoadU64(const void* p) { + uint64_t v; + memcpy(&v, p, sizeof(v)); + return v; +} + +static inline void StoreU32(void* p, uint32_t v) { + memcpy(p, &v, sizeof(v)); +} + +#if defined(HAS_FIXEDDIV_X86) && \ + !(defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__) && \ + !defined(LIBYUV_ENABLE_ROWWIN)) +int FixedDiv_X86(int num, int div) { + return (int)(((int64_t)num * 65536) / div); +} +#endif + +#if defined(HAS_FIXEDDIV1_X86) && \ + !(defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__) && \ + !defined(LIBYUV_ENABLE_ROWWIN)) +int FixedDiv1_X86(int num, int div) { + return (int)((((int64_t)num * 65536) - 0x00010001) / (div - 1)); +} +#endif + +#ifdef HAS_SCALEROWDOWN2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowDown2_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + __m256i s0 = _mm256_srli_epi16( + _mm256_loadu_si256((const __m256i*)src_ptr), 8); + __m256i s1 = _mm256_srli_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), 8); + src_ptr += 64; + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 32; + dst_width -= 32; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m256i ones = _mm256_set1_epi8(1); + const __m256i zero = _mm256_setzero_si256(); + while (dst_width > 0) { + __m256i s0 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)src_ptr), ones); + __m256i s1 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones); + src_ptr += 64; + s0 = _mm256_avg_epu16(s0, zero); + s1 = _mm256_avg_epu16(s1, zero); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 32; + dst_width -= 32; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i ones = _mm256_set1_epi8(1); + const __m256i zero = _mm256_setzero_si256(); + while (dst_width > 0) { + __m256i s0 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)src_ptr), ones); + __m256i s1 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones); + __m256i s2 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), ones); + __m256i s3 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)), ones); + src_ptr += 64; + s0 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(s0, s2), 1), zero); + s1 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(s1, s3), 1), zero); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 32; + dst_width -= 32; + } +} +#endif // HAS_SCALEROWDOWN2_AVX2 + +#ifdef HAS_SCALEROWDOWN2_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + __m256i s0 = _mm256_srli_epi32( + _mm256_loadu_si256((const __m256i*)src_ptr), 16); + __m256i s1 = _mm256_srli_epi32( + _mm256_loadu_si256((const __m256i*)(src_ptr + 16)), 16); + src_ptr += 32; + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 16; + dst_width -= 16; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m256i mask = _mm256_set1_epi32(0x0000ffff); + while (dst_width > 0) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 16)); + src_ptr += 32; + __m256i e = _mm256_packus_epi32(_mm256_and_si256(r0, mask), + _mm256_and_si256(r1, mask)); + __m256i o = _mm256_packus_epi32(_mm256_srli_epi32(r0, 16), + _mm256_srli_epi32(r1, 16)); + __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu16(e, o), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 16; + dst_width -= 16; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + const __m256i mask = _mm256_set1_epi32(0x0000ffff); + const __m256i round2 = _mm256_set1_epi32(2); + while (dst_width > 0) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 16)); + __m256i r2 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)); + __m256i r3 = + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 16)); + src_ptr += 32; + __m256i s0 = _mm256_add_epi32( + _mm256_add_epi32(_mm256_and_si256(r0, mask), _mm256_srli_epi32(r0, 16)), + _mm256_add_epi32(_mm256_and_si256(r2, mask), + _mm256_srli_epi32(r2, 16))); + __m256i s1 = _mm256_add_epi32( + _mm256_add_epi32(_mm256_and_si256(r1, mask), _mm256_srli_epi32(r1, 16)), + _mm256_add_epi32(_mm256_and_si256(r3, mask), + _mm256_srli_epi32(r3, 16))); + s0 = _mm256_srli_epi32(_mm256_add_epi32(s0, round2), 2); + s1 = _mm256_srli_epi32(_mm256_add_epi32(s1, round2), 2); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWDOWN2_16_AVX2 + +#ifdef HAS_SCALEROWDOWN2_AVX512BW +LIBYUV_TARGET_AVX512BW +void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + while (dst_width >= 32) { + __m512i s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)src_ptr), 8); + src_ptr += 64; + _mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s)); + dst_ptr += 32; + dst_width -= 32; + } + if (dst_width > 0) { + __mmask32 k = (__mmask32)((1ULL << dst_width) - 1); + __m512i s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), 8); + _mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s)); + } +} + +LIBYUV_TARGET_AVX512BW +void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m512i ones = _mm512_set1_epi8(1); + const __m512i zero = _mm512_setzero_si512(); + while (dst_width >= 32) { + __m512i s = _mm512_avg_epu16( + _mm512_maddubs_epi16(_mm512_loadu_si512((const __m512i*)src_ptr), ones), + zero); + src_ptr += 64; + _mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s)); + dst_ptr += 32; + dst_width -= 32; + } + if (dst_width > 0) { + __mmask32 k = (__mmask32)((1ULL << dst_width) - 1); + __m512i s = _mm512_avg_epu16( + _mm512_maddubs_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), ones), + zero); + _mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s)); + } +} + +LIBYUV_TARGET_AVX512BW +void ScaleRowDown2Box_AVX512BW(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m512i ones = _mm512_set1_epi8(1); + const __m512i zero = _mm512_setzero_si512(); + while (dst_width >= 32) { + __m512i s0 = _mm512_maddubs_epi16( + _mm512_loadu_si512((const __m512i*)src_ptr), ones); + __m512i s1 = _mm512_maddubs_epi16( + _mm512_loadu_si512((const __m512i*)(src_ptr + src_stride)), ones); + src_ptr += 64; + __m512i s = + _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(s0, s1), 1), zero); + _mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s)); + dst_ptr += 32; + dst_width -= 32; + } + if (dst_width > 0) { + __mmask32 k = (__mmask32)((1ULL << dst_width) - 1); + __m512i s0 = + _mm512_maddubs_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), ones); + __m512i s1 = _mm512_maddubs_epi16( + _mm512_maskz_loadu_epi16(k, src_ptr + src_stride), ones); + __m512i s = + _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(s0, s1), 1), zero); + _mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s)); + } +} + +LIBYUV_TARGET_AVX512BW +void ScaleRowDown2Box_Odd_AVX512BW(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + dst_width -= 1; + if (dst_width > 0) { + ScaleRowDown2Box_AVX512BW(src_ptr, src_stride, dst_ptr, dst_width); + src_ptr += dst_width * 2; + dst_ptr += dst_width; + } + dst_ptr[0] = (uint8_t)((src_ptr[0] + src_ptr[src_stride] + 1) >> 1); +} +#endif // HAS_SCALEROWDOWN2_AVX512BW + +#ifdef HAS_SCALEROWDOWN4_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowDown4_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m256i mask = _mm256_set1_epi32(0x00ff0000); + while (dst_width > 0) { + __m256i s0 = + _mm256_and_si256(_mm256_loadu_si256((const __m256i*)src_ptr), mask); + __m256i s1 = _mm256_and_si256( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), mask); + src_ptr += 64; + __m256i t = _mm256_srli_epi16( + _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8), 8); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(t, t), 0xd8); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d)); + dst_ptr += 16; + dst_width -= 16; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i ones = _mm256_set1_epi8(1); + const __m256i round8 = _mm256_set1_epi16(8); + const ptrdiff_t stridex3 = src_stride * 3; + while (dst_width > 0) { + __m256i s0 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)src_ptr), ones); + __m256i s1 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones); + s0 = _mm256_add_epi16( + s0, _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), + ones)); + s1 = _mm256_add_epi16( + s1, _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)), + ones)); + s0 = _mm256_add_epi16( + s0, _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2)), + ones)); + s1 = _mm256_add_epi16( + s1, + _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2 + 32)), + ones)); + s0 = _mm256_add_epi16( + s0, + _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + stridex3)), ones)); + s1 = _mm256_add_epi16( + s1, _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + stridex3 + 32)), + ones)); + src_ptr += 64; + __m256i t = _mm256_permute4x64_epi64(_mm256_hadd_epi16(s0, s1), 0xd8); + t = _mm256_srli_epi16(_mm256_add_epi16(t, round8), 4); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(t, t), 0xd8); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d)); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWDOWN4_AVX2 + +#ifdef HAS_SCALEROWDOWN34_AVX2 +static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9, + 128, 128, 128, 128, 128, 128, 128, 128}; +static const uvec8 kShuf1 = {3, 4, 5, 7, 8, 9, 11, 12, + 128, 128, 128, 128, 128, 128, 128, 128}; +static const uvec8 kShuf2 = {5, 7, 8, 9, 11, 12, 13, 15, + 128, 128, 128, 128, 128, 128, 128, 128}; +static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10}; +static const uvec8 kShuf11 = {2, 3, 4, 5, 5, 6, 6, 7, + 8, 9, 9, 10, 10, 11, 12, 13}; +static const uvec8 kShuf21 = {5, 6, 6, 7, 8, 9, 9, 10, + 10, 11, 12, 13, 13, 14, 14, 15}; +static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2}; +static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1}; +static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3}; + +LIBYUV_TARGET_AVX2 +void ScaleRowDown34_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m256i shuf0 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf0)); + const __m256i shuf1 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf1)); + const __m256i shuf2 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf2)); + while (dst_width >= 48) { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i s1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 32)); + src_ptr += 64; + __m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20); + __m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31); + __m256i mid = _mm256_alignr_epi8(hi, lo, 8); + __m256i d0 = _mm256_shuffle_epi8(lo, shuf0); + __m256i d1 = _mm256_shuffle_epi8(mid, shuf1); + __m256i d2 = _mm256_shuffle_epi8(hi, shuf2); + __m256i d01 = _mm256_unpacklo_epi64(d0, d1); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(d2)); + _mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(d2, 1)); + dst_ptr += 48; + dst_width -= 48; + } + if (dst_width > 0) { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m128i lo = _mm256_castsi256_si128(s0); + __m128i hi = _mm256_extracti128_si256(s0, 1); + __m128i mid = _mm_alignr_epi8(hi, lo, 8); + __m128i d0 = _mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf0)); + __m128i d1 = _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf1)); + __m128i d2 = _mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf2)); + _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(d0, d1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), d2); + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i shuf01 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf01)); + const __m256i shuf11 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf11)); + const __m256i shuf21 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf21)); + const __m256i madd01 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd01)); + const __m256i madd11 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd11)); + const __m256i madd21 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd21)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width >= 48) { + __m256i s0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_ptr), + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride))); + __m256i s1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32))); + src_ptr += 64; + __m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20); + __m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31); + __m256i mid = _mm256_alignr_epi8(hi, lo, 8); + __m256i p0 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(lo, shuf01), madd01), + round2), + 2); + __m256i p1 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(mid, shuf11), madd11), + round2), + 2); + __m256i p2 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(hi, shuf21), madd21), + round2), + 2); + p0 = _mm256_packus_epi16(p0, p0); + p1 = _mm256_packus_epi16(p1, p1); + p2 = _mm256_packus_epi16(p2, p2); + __m256i d01 = _mm256_unpacklo_epi64(p0, p1); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(p2)); + _mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(p2, 1)); + dst_ptr += 48; + dst_width -= 48; + } + if (dst_width > 0) { + __m256i s0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_ptr), + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride))); + __m128i lo = _mm256_castsi256_si128(s0); + __m128i hi = _mm256_extracti128_si256(s0, 1); + __m128i mid = _mm_alignr_epi8(hi, lo, 8); + __m128i r2 = _mm256_castsi256_si128(round2); + __m128i p0 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16(_mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf01)), + _mm256_castsi256_si128(madd01)), + r2), + 2); + __m128i p1 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16( + _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf11)), + _mm256_castsi256_si128(madd11)), + r2), + 2); + __m128i p2 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16(_mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf21)), + _mm256_castsi256_si128(madd21)), + r2), + 2); + p0 = _mm_packus_epi16(p0, p0); + p1 = _mm_packus_epi16(p1, p1); + p2 = _mm_packus_epi16(p2, p2); + _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(p0, p1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), p2); + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i shuf01 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf01)); + const __m256i shuf11 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf11)); + const __m256i shuf21 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf21)); + const __m256i madd01 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd01)); + const __m256i madd11 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd11)); + const __m256i madd21 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd21)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width >= 48) { + __m256i a0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i b0 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)); + __m256i s0 = _mm256_avg_epu8(a0, _mm256_avg_epu8(a0, b0)); + __m256i a1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 32)); + __m256i b1 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)); + __m256i s1 = _mm256_avg_epu8(a1, _mm256_avg_epu8(a1, b1)); + src_ptr += 64; + __m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20); + __m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31); + __m256i mid = _mm256_alignr_epi8(hi, lo, 8); + __m256i p0 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(lo, shuf01), madd01), + round2), + 2); + __m256i p1 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(mid, shuf11), madd11), + round2), + 2); + __m256i p2 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(hi, shuf21), madd21), + round2), + 2); + p0 = _mm256_packus_epi16(p0, p0); + p1 = _mm256_packus_epi16(p1, p1); + p2 = _mm256_packus_epi16(p2, p2); + __m256i d01 = _mm256_unpacklo_epi64(p0, p1); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(p2)); + _mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(p2, 1)); + dst_ptr += 48; + dst_width -= 48; + } + if (dst_width > 0) { + __m256i a0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i b0 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)); + __m256i s0 = _mm256_avg_epu8(a0, _mm256_avg_epu8(a0, b0)); + __m128i lo = _mm256_castsi256_si128(s0); + __m128i hi = _mm256_extracti128_si256(s0, 1); + __m128i mid = _mm_alignr_epi8(hi, lo, 8); + __m128i r2 = _mm256_castsi256_si128(round2); + __m128i p0 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16(_mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf01)), + _mm256_castsi256_si128(madd01)), + r2), + 2); + __m128i p1 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16( + _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf11)), + _mm256_castsi256_si128(madd11)), + r2), + 2); + __m128i p2 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16(_mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf21)), + _mm256_castsi256_si128(madd21)), + r2), + 2); + p0 = _mm_packus_epi16(p0, p0); + p1 = _mm_packus_epi16(p1, p1); + p2 = _mm_packus_epi16(p2, p2); + _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(p0, p1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), p2); + } +} +#endif // HAS_SCALEROWDOWN34_AVX2 + +#ifdef HAS_SCALEROWDOWN38_AVX2 static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}; - static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0, 3, 6, 8, 11, 14, 128, 128, 128, 128}; - -// Arrange words 0,3,6 into 0,1,2 static const uvec8 kShufAc = {0, 1, 6, 7, 12, 13, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}; - -// Arrange words 0,3,6 into 3,4,5 static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0, 1, 6, 7, 12, 13, 128, 128, 128, 128}; - -// Scaling values for boxes of 3x3 and 2x3 static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9, 65536 / 9, 65536 / 6, 0, 0}; - -// Arrange first value for pixels 0,1,2,3,4,5 static const uvec8 kShufAb0 = {0, 128, 3, 128, 6, 128, 8, 128, 11, 128, 14, 128, 128, 128, 128, 128}; - -// Arrange second value for pixels 0,1,2,3,4,5 static const uvec8 kShufAb1 = {1, 128, 4, 128, 7, 128, 9, 128, 12, 128, 15, 128, 128, 128, 128, 128}; - -// Arrange third value for pixels 0,1,2,3,4,5 static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128, 13, 128, 128, 128, 128, 128, 128, 128}; - -// Scaling values for boxes of 3x2 and 2x2 static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3, 65536 / 3, 65536 / 2, 0, 0}; -// Scale 32 pixels to 12 -LIBYUV_TARGET_SSSE3 -void ScaleRowDown38_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +LIBYUV_TARGET_AVX2 +void ScaleRowDown38_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { (void)src_stride; - const __m128i shuf38a = _mm_loadu_si128((const __m128i*)&kShuf38a); - const __m128i shuf38b = _mm_loadu_si128((const __m128i*)&kShuf38b); + const __m256i shuf38 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf38a)), + _mm_loadu_si128((const __m128i*)&kShuf38b), 1); + while (dst_width >= 24) { + __m256i s0 = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), shuf38); + __m256i s1 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), shuf38); + src_ptr += 64; + __m128i d0 = + _mm_or_si128(_mm256_castsi256_si128(s0), _mm256_extracti128_si256(s0, 1)); + __m128i d1 = + _mm_or_si128(_mm256_castsi256_si128(s1), _mm256_extracti128_si256(s1, 1)); + _mm_storel_epi64((__m128i*)dst_ptr, d0); + StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d0, 8))); + _mm_storel_epi64((__m128i*)(dst_ptr + 12), d1); + StoreU32(dst_ptr + 20, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d1, 8))); + dst_ptr += 24; + dst_width -= 24; + } + if (dst_width > 0) { + __m256i s0 = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), shuf38); + __m128i d0 = + _mm_or_si128(_mm256_castsi256_si128(s0), _mm256_extracti128_si256(s0, 1)); + _mm_storel_epi64((__m128i*)dst_ptr, d0); + StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d0, 8))); + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i shuf_ab0 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb0)); + const __m256i shuf_ab1 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb1)); + const __m256i shuf_ab2 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb2)); + const __m256i scale_ab2 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kScaleAb2)); + const __m256i zero = _mm256_setzero_si256(); while (dst_width > 0) { - __m128i s0 = _mm_loadu_si128((const __m128i*)src_ptr); - __m128i s1 = _mm_loadu_si128((const __m128i*)(src_ptr + 16)); + __m256i r = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_ptr), + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride))); src_ptr += 32; - s0 = _mm_shuffle_epi8(s0, shuf38a); - s1 = _mm_shuffle_epi8(s1, shuf38b); - s0 = _mm_adds_epu8(s0, s1); - _mm_storel_epi64((__m128i*)dst_ptr, s0); - *(uint32_t*)(dst_ptr + 8) = - (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(s0, 8)); + __m256i t = _mm256_adds_epu16( + _mm256_adds_epu16(_mm256_shuffle_epi8(r, shuf_ab0), + _mm256_shuffle_epi8(r, shuf_ab1)), + _mm256_shuffle_epi8(r, shuf_ab2)); + t = _mm256_packus_epi16(_mm256_mulhi_epu16(t, scale_ab2), zero); + __m128i d = _mm_or_si128(_mm256_castsi256_si128(t), + _mm_slli_si128(_mm256_extracti128_si256(t, 1), 6)); + _mm_storel_epi64((__m128i*)dst_ptr, d); + StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d, 8))); dst_ptr += 12; dst_width -= 12; } } -// Scale 16x3 pixels to 6x1 with interpolation -LIBYUV_TARGET_SSSE3 -void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - const __m128i shuf_ac = _mm_loadu_si128((const __m128i*)&kShufAc); - const __m128i shuf_ac3 = _mm_loadu_si128((const __m128i*)&kShufAc3); - const __m128i scale_ac33 = _mm_loadu_si128((const __m128i*)&kScaleAc33); - const __m128i zero = _mm_setzero_si128(); +LIBYUV_TARGET_AVX2 +void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i shuf_ac = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAc)); + const __m256i shuf_ac3 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAc3)); + const __m256i scale_ac33 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kScaleAc33)); + const __m256i zero = _mm256_setzero_si256(); while (dst_width > 0) { - __m128i r0 = _mm_loadu_si128((const __m128i*)src_ptr); - __m128i r1 = _mm_loadu_si128((const __m128i*)(src_ptr + src_stride)); - __m128i r2 = _mm_loadu_si128((const __m128i*)(src_ptr + src_stride * 2)); - src_ptr += 16; - __m128i s0 = _mm_adds_epu16(_mm_adds_epu16(_mm_unpacklo_epi8(r0, zero), - _mm_unpacklo_epi8(r1, zero)), - _mm_unpacklo_epi8(r2, zero)); - __m128i s1 = _mm_adds_epu16(_mm_adds_epu16(_mm_unpackhi_epi8(r0, zero), - _mm_unpackhi_epi8(r1, zero)), - _mm_unpackhi_epi8(r2, zero)); - __m128i t0 = _mm_adds_epu16(_mm_adds_epu16(s0, _mm_srli_si128(s0, 2)), - _mm_srli_si128(s0, 4)); - __m128i t1 = _mm_adds_epu16(_mm_adds_epu16(s1, _mm_srli_si128(s1, 2)), - _mm_srli_si128(s1, 4)); - t0 = _mm_shuffle_epi8(t0, shuf_ac); - t1 = _mm_shuffle_epi8(t1, shuf_ac3); - t0 = _mm_adds_epu16(t0, t1); - t0 = _mm_mulhi_epu16(t0, scale_ac33); - t0 = _mm_packus_epi16(t0, t0); - *(uint32_t*)dst_ptr = (uint32_t)_mm_cvtsi128_si32(t0); - *(uint16_t*)(dst_ptr + 4) = - (uint16_t)_mm_cvtsi128_si32(_mm_srli_si128(t0, 4)); - dst_ptr += 6; - dst_width -= 6; + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)); + __m256i r2 = + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2)); + src_ptr += 32; + __m256i s0 = + _mm256_adds_epu16(_mm256_adds_epu16(_mm256_unpacklo_epi8(r0, zero), + _mm256_unpacklo_epi8(r1, zero)), + _mm256_unpacklo_epi8(r2, zero)); + __m256i s1 = + _mm256_adds_epu16(_mm256_adds_epu16(_mm256_unpackhi_epi8(r0, zero), + _mm256_unpackhi_epi8(r1, zero)), + _mm256_unpackhi_epi8(r2, zero)); + __m256i t0 = _mm256_shuffle_epi8( + _mm256_adds_epu16(_mm256_adds_epu16(s0, _mm256_srli_si256(s0, 2)), + _mm256_srli_si256(s0, 4)), + shuf_ac); + __m256i t1 = _mm256_shuffle_epi8( + _mm256_adds_epu16(_mm256_adds_epu16(s1, _mm256_srli_si256(s1, 2)), + _mm256_srli_si256(s1, 4)), + shuf_ac3); + __m256i t = _mm256_packus_epi16( + _mm256_mulhi_epu16(_mm256_adds_epu16(t0, t1), scale_ac33), zero); + __m128i d = _mm_or_si128(_mm256_castsi256_si128(t), + _mm_slli_si128(_mm256_extracti128_si256(t, 1), 6)); + _mm_storel_epi64((__m128i*)dst_ptr, d); + StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d, 8))); + dst_ptr += 12; + dst_width -= 12; } } - -// Scale 16x2 pixels to 6x1 with interpolation -LIBYUV_TARGET_SSSE3 -void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - const __m128i shuf_ab0 = _mm_loadu_si128((const __m128i*)&kShufAb0); - const __m128i shuf_ab1 = _mm_loadu_si128((const __m128i*)&kShufAb1); - const __m128i shuf_ab2 = _mm_loadu_si128((const __m128i*)&kShufAb2); - const __m128i scale_ab2 = _mm_loadu_si128((const __m128i*)&kScaleAb2); - while (dst_width > 0) { - __m128i r0 = _mm_loadu_si128((const __m128i*)src_ptr); - __m128i r1 = _mm_loadu_si128((const __m128i*)(src_ptr + src_stride)); - src_ptr += 16; - r0 = _mm_avg_epu8(r0, r1); - __m128i t0 = _mm_shuffle_epi8(r0, shuf_ab0); - __m128i t1 = _mm_shuffle_epi8(r0, shuf_ab1); - __m128i t2 = _mm_shuffle_epi8(r0, shuf_ab2); - t0 = _mm_adds_epu16(_mm_adds_epu16(t0, t1), t2); - t0 = _mm_mulhi_epu16(t0, scale_ab2); - t0 = _mm_packus_epi16(t0, t0); - *(uint32_t*)dst_ptr = (uint32_t)_mm_cvtsi128_si32(t0); - *(uint16_t*)(dst_ptr + 4) = - (uint16_t)_mm_cvtsi128_si32(_mm_srli_si128(t0, 4)); - dst_ptr += 6; - dst_width -= 6; - } -} -#endif // HAS_SCALEROWDOWN38_SSSE3 +#endif // HAS_SCALEROWDOWN38_AVX2 #ifdef HAS_SCALEADDROW_SSE2 // Reads 16 bytes and accumulates to 16 shorts at a time. @@ -1390,10 +1093,12 @@ void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, } #endif // HAS_SCALEADDROW_AVX512BW -#ifdef HAS_SCALEADDCOLS_AVX512BW +#if defined(HAS_SCALEADDCOLS1_AVX512BW) || defined(HAS_SCALEADDCOLS2_AVX512BW) static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; +#endif +#ifdef HAS_SCALEADDCOLS1_AVX512BW LIBYUV_TARGET_AVX512BW void ScaleAddCols1_AVX512BW(int dst_width, int boxheight, @@ -1543,7 +1248,9 @@ void ScaleAddCols1_AVX512BW(int dst_width, } } } +#endif // HAS_SCALEADDCOLS1_AVX512BW +#ifdef HAS_SCALEADDCOLS2_AVX512BW LIBYUV_TARGET_AVX512BW void ScaleAddCols2_AVX512BW(int dst_width, int boxheight, @@ -1745,7 +1452,7 @@ void ScaleAddCols2_AVX512BW(int dst_width, } } } -#endif // HAS_SCALEADDCOLS_AVX512BW +#endif // HAS_SCALEADDCOLS2_AVX512BW #ifdef HAS_SCALEUVADDCOLS3_AVX2 static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5}; @@ -1926,7 +1633,956 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, } #endif // HAS_SCALEUVADDCOLS3_AVX512VBMI -#undef LIBYUV_TARGET_SSSE3 +#if defined(HAS_SCALEROWUP2_LINEAR_AVX2) || \ + defined(HAS_SCALEROWUP2_BILINEAR_AVX2) +static const uvec8 kLinearMadd31 = {3, 1, 1, 3, 3, 1, 1, 3, + 3, 1, 1, 3, 3, 1, 1, 3}; +#endif + +#if defined(HAS_SCALEROWUP2_LINEAR_12_AVX2) || \ + defined(HAS_SCALEROWUP2_BILINEAR_12_AVX2) +static const uvec8 kLinearShuffleFar = {2, 3, 0, 1, 6, 7, 4, 5, + 10, 11, 8, 9, 14, 15, 12, 13}; +#endif + +#ifdef HAS_SCALEROWUP2_LINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Linear_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width) { + const __m256i madd31 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kLinearMadd31)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width > 0) { + __m256i s0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i s1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))), + 0xd8); + src_ptr += 16; + s0 = _mm256_unpacklo_epi16(s0, s0); + s1 = _mm256_unpacklo_epi16(s1, s1); + __m256i lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(s0, s1), madd31); + __m256i hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(s0, s1), madd31); + lo = _mm256_srli_epi16(_mm256_add_epi16(lo, round2), 2); + hi = _mm256_srli_epi16(_mm256_add_epi16(hi, round2), 2); + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(lo, hi)); + dst_ptr += 32; + dst_width -= 32; + } +} +#endif // HAS_SCALEROWUP2_LINEAR_AVX2 + +#ifdef HAS_SCALEROWUP2_BILINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Bilinear_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i madd31 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kLinearMadd31)); + const __m256i round8 = _mm256_set1_epi16(8); + while (dst_width > 0) { + __m256i a0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i a1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))), + 0xd8); + a0 = _mm256_unpacklo_epi16(a0, a0); + a1 = _mm256_unpacklo_epi16(a1, a1); + __m256i r0_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(a0, a1), madd31); + __m256i r0_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(a0, a1), madd31); + + __m256i b0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))), + 0xd8); + __m256i b1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))), + 0xd8); + src_ptr += 16; + b0 = _mm256_unpacklo_epi16(b0, b0); + b1 = _mm256_unpacklo_epi16(b1, b1); + __m256i r1_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(b0, b1), madd31); + __m256i r1_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(b0, b1), madd31); + + __m256i d0_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_lo, r0_lo), r0_lo), + _mm256_add_epi16(r1_lo, round8)), + 4); + __m256i d1_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_lo, r1_lo), r1_lo), + _mm256_add_epi16(r0_lo, round8)), + 4); + __m256i d0_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_hi, r0_hi), r0_hi), + _mm256_add_epi16(r1_hi, round8)), + 4); + __m256i d1_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_hi, r1_hi), r1_hi), + _mm256_add_epi16(r0_hi, round8)), + 4); + + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(d0_lo, d0_hi)); + _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), + _mm256_packus_epi16(d1_lo, d1_hi)); + dst_ptr += 32; + dst_width -= 32; + } +} +#endif // HAS_SCALEROWUP2_BILINEAR_AVX2 + +#ifdef HAS_SCALEROWUP2_LINEAR_12_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Linear_12_AVX2(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int dst_width) { + const __m256i shuf_far = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kLinearShuffleFar)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width > 0) { + __m256i s0 = + _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)src_ptr), + 0xd8); + __m256i s1 = _mm256_permute4x64_epi64( + _mm256_loadu_si256((const __m256i*)(src_ptr + 1)), 0xd8); + src_ptr += 16; + __m256i near_lo = _mm256_unpacklo_epi16(s0, s1); + __m256i near_hi = _mm256_unpackhi_epi16(s0, s1); + __m256i far_lo = _mm256_shuffle_epi8(near_lo, shuf_far); + __m256i far_hi = _mm256_shuffle_epi8(near_hi, shuf_far); + __m256i d_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(near_lo, near_lo), + _mm256_add_epi16(near_lo, _mm256_add_epi16(far_lo, round2))), + 2); + __m256i d_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(near_hi, near_hi), + _mm256_add_epi16(near_hi, _mm256_add_epi16(far_hi, round2))), + 2); + _mm256_storeu_si256((__m256i*)dst_ptr, d_lo); + _mm256_storeu_si256((__m256i*)(dst_ptr + 16), d_hi); + dst_ptr += 32; + dst_width -= 32; + } +} +#endif // HAS_SCALEROWUP2_LINEAR_12_AVX2 + +#ifdef HAS_SCALEROWUP2_BILINEAR_12_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Bilinear_12_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i shuf_far = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kLinearShuffleFar)); + const __m256i round8 = _mm256_set1_epi16(8); + while (dst_width > 0) { + __m256i a0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i a1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))), + 0xd8); + __m256i near0 = _mm256_unpacklo_epi16(a0, a1); + __m256i far0 = _mm256_shuffle_epi8(near0, shuf_far); + __m256i r0 = _mm256_add_epi16(_mm256_add_epi16(near0, near0), + _mm256_add_epi16(near0, far0)); + + __m256i b0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))), + 0xd8); + __m256i b1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))), + 0xd8); + src_ptr += 8; + __m256i near1 = _mm256_unpacklo_epi16(b0, b1); + __m256i far1 = _mm256_shuffle_epi8(near1, shuf_far); + __m256i r1 = _mm256_add_epi16(_mm256_add_epi16(near1, near1), + _mm256_add_epi16(near1, far1)); + + __m256i d0 = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0, r0), r0), + _mm256_add_epi16(r1, round8)), + 4); + __m256i d1 = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1, r1), r1), + _mm256_add_epi16(r0, round8)), + 4); + _mm256_storeu_si256((__m256i*)dst_ptr, d0); + _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), d1); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWUP2_BILINEAR_12_AVX2 + +#ifdef HAS_SCALEROWUP2_LINEAR_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Linear_16_AVX2(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int dst_width) { + const __m256i round2 = _mm256_set1_epi32(2); + while (dst_width > 0) { + __m256i near_lo = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr)); + __m256i near_hi = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 1))); + src_ptr += 8; + __m256i far_lo = _mm256_shuffle_epi32(near_lo, 0xb1); + __m256i far_hi = _mm256_shuffle_epi32(near_hi, 0xb1); + __m256i d_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(near_lo, near_lo), + _mm256_add_epi32(near_lo, _mm256_add_epi32(far_lo, round2))), + 2); + __m256i d_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(near_hi, near_hi), + _mm256_add_epi32(near_hi, _mm256_add_epi32(far_hi, round2))), + 2); + __m256i d = _mm256_shuffle_epi32(_mm256_packus_epi32(d_lo, d_hi), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWUP2_LINEAR_16_AVX2 + +#ifdef HAS_SCALEROWUP2_BILINEAR_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i round8 = _mm256_set1_epi32(8); + while (dst_width > 0) { + __m256i a0 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr)); + __m256i a1 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 1))); + __m256i r0_lo = _mm256_add_epi32( + _mm256_add_epi32(a0, a0), + _mm256_add_epi32(a0, _mm256_shuffle_epi32(a0, 0xb1))); + __m256i r0_hi = _mm256_add_epi32( + _mm256_add_epi32(a1, a1), + _mm256_add_epi32(a1, _mm256_shuffle_epi32(a1, 0xb1))); + + __m256i b0 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))); + __m256i b1 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))); + src_ptr += 8; + __m256i r1_lo = _mm256_add_epi32( + _mm256_add_epi32(b0, b0), + _mm256_add_epi32(b0, _mm256_shuffle_epi32(b0, 0xb1))); + __m256i r1_hi = _mm256_add_epi32( + _mm256_add_epi32(b1, b1), + _mm256_add_epi32(b1, _mm256_shuffle_epi32(b1, 0xb1))); + + __m256i d0_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_lo, r0_lo), r0_lo), + _mm256_add_epi32(r1_lo, round8)), + 4); + __m256i d1_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_lo, r1_lo), r1_lo), + _mm256_add_epi32(r0_lo, round8)), + 4); + __m256i d0_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_hi, r0_hi), r0_hi), + _mm256_add_epi32(r1_hi, round8)), + 4); + __m256i d1_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_hi, r1_hi), r1_hi), + _mm256_add_epi32(r0_hi, round8)), + 4); + + _mm256_storeu_si256( + (__m256i*)dst_ptr, + _mm256_shuffle_epi32(_mm256_packus_epi32(d0_lo, d0_hi), 0xd8)); + _mm256_storeu_si256( + (__m256i*)(dst_ptr + dst_stride), + _mm256_shuffle_epi32(_mm256_packus_epi32(d1_lo, d1_hi), 0xd8)); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWUP2_BILINEAR_16_AVX2 + +#ifdef HAS_SCALEFILTERCOLS_AVX2 +static const int32_t kStep0_7_Filter[8] = {0, 1, 2, 3, 4, 5, 6, 7}; +static const uvec8 kShufFracCols = {0, 0, 4, 4, 8, 8, 12, 12, + 128, 128, 128, 128, 128, 128, 128, 128}; + +LIBYUV_TARGET_AVX2 +void ScaleFilterCols_AVX2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx) { + int n = dst_width & ~7; + int r = dst_width & 7; + if (n > 0) { + SIMD_ALIGNED(int32_t xi_buf[8]); + SIMD_ALIGNED(uint16_t pix_buf[8]); + __m256i vx = _mm256_add_epi32( + _mm256_set1_epi32(x), + _mm256_mullo_epi32( + _mm256_set1_epi32(dx), + _mm256_loadu_si256((const __m256i*)kStep0_7_Filter))); + const __m256i vdx8 = _mm256_slli_epi32(_mm256_set1_epi32(dx), 3); + const __m256i shuf_frac = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kShufFracCols)); + const __m128i mask_7f = _mm_set1_epi16(0x007f); + const __m128i mask_ff = _mm_set1_epi16(0x00ff); + const __m128i round_40 = _mm_set1_epi16(0x0040); + for (int i = 0; i < n; i += 8) { + _mm256_store_si256((__m256i*)xi_buf, _mm256_srai_epi32(vx, 16)); + __m256i f256 = _mm256_permute4x64_epi64( + _mm256_shuffle_epi8(_mm256_srli_epi16(vx, 9), shuf_frac), 0xd8); + vx = _mm256_add_epi32(vx, vdx8); + __m128i weights = _mm_xor_si128(_mm256_castsi256_si128(f256), mask_7f); + pix_buf[0] = LoadU16(src_ptr + xi_buf[0]); + pix_buf[1] = LoadU16(src_ptr + xi_buf[1]); + pix_buf[2] = LoadU16(src_ptr + xi_buf[2]); + pix_buf[3] = LoadU16(src_ptr + xi_buf[3]); + pix_buf[4] = LoadU16(src_ptr + xi_buf[4]); + pix_buf[5] = LoadU16(src_ptr + xi_buf[5]); + pix_buf[6] = LoadU16(src_ptr + xi_buf[6]); + pix_buf[7] = LoadU16(src_ptr + xi_buf[7]); + __m128i pix = _mm_load_si128((const __m128i*)pix_buf); + __m128i a = _mm_and_si128(pix, mask_ff); + __m128i res = _mm_srli_epi16( + _mm_add_epi16(_mm_add_epi16(_mm_maddubs_epi16(pix, weights), a), + round_40), + 7); + _mm_storel_epi64((__m128i*)dst_ptr, _mm_packus_epi16(res, res)); + dst_ptr += 8; + } + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + int xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[j] = (uint8_t)(a + ((((x & 0xffff) >> 9) * (b - a) + 0x40) >> 7)); + x = (int)((uint32_t)x + (uint32_t)dx); + } +} +#endif // HAS_SCALEFILTERCOLS_AVX2 + +#ifdef HAS_SCALECOLSUP2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleColsUp2_AVX2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx) { + (void)x; + (void)dx; + while (dst_width >= 32) { + __m256i s = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + src_ptr += 16; + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_unpacklo_epi8(s, s)); + dst_ptr += 32; + dst_width -= 32; + } + if (dst_width & 16) { + __m128i s = _mm_loadl_epi64((const __m128i*)src_ptr); + src_ptr += 8; + _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi8(s, s)); + dst_ptr += 16; + } + if (dst_width & 8) { + __m128i s = _mm_cvtsi32_si128((int)LoadU32(src_ptr)); + _mm_storel_epi64((__m128i*)dst_ptr, _mm_unpacklo_epi8(s, s)); + } +} +#endif // HAS_SCALECOLSUP2_AVX2 + +#ifdef HAS_SCALEARGBROWDOWN2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + __m256 s0 = _mm256_loadu_ps((const float*)src_argb); + __m256 s1 = _mm256_loadu_ps((const float*)(src_argb + 32)); + src_argb += 64; + __m256i d = _mm256_permute4x64_epi64( + _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0xdd)), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, d); + dst_argb += 32; + dst_width -= 8; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + __m256 s0 = _mm256_loadu_ps((const float*)src_argb); + __m256 s1 = _mm256_loadu_ps((const float*)(src_argb + 32)); + src_argb += 64; + __m256i e = _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0x88)); + __m256i o = _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0xdd)); + __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, d); + dst_argb += 32; + dst_width -= 8; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + while (dst_width > 0) { + __m256i r0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_argb), + _mm256_loadu_si256((const __m256i*)(src_argb + src_stride))); + __m256i r1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_argb + 32)), + _mm256_loadu_si256((const __m256i*)(src_argb + src_stride + 32))); + src_argb += 64; + __m256i e = _mm256_castps_si256( + _mm256_shuffle_ps(_mm256_castsi256_ps(r0), _mm256_castsi256_ps(r1), + 0x88)); + __m256i o = _mm256_castps_si256( + _mm256_shuffle_ps(_mm256_castsi256_ps(r0), _mm256_castsi256_ps(r1), + 0xdd)); + __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, d); + dst_argb += 32; + dst_width -= 8; + } +} +#endif // HAS_SCALEARGBROWDOWN2_AVX2 + +#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; + SIMD_ALIGNED(uint32_t pix_buf[8]); + while (dst_width > 0) { + pix_buf[0] = LoadU32(src_argb); + pix_buf[1] = LoadU32(src_argb + step); + pix_buf[2] = LoadU32(src_argb + step * 2); + pix_buf[3] = LoadU32(src_argb + step * 3); + pix_buf[4] = LoadU32(src_argb + step * 4); + pix_buf[5] = LoadU32(src_argb + step * 5); + pix_buf[6] = LoadU32(src_argb + step * 6); + pix_buf[7] = LoadU32(src_argb + step * 7); + src_argb += step * 8; + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_load_si256((const __m256i*)pix_buf)); + dst_argb += 32; + dst_width -= 8; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width) { + const uint8_t* row1 = src_argb + src_stride; + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; + SIMD_ALIGNED(uint64_t r0_lo[4]); + SIMD_ALIGNED(uint64_t r0_hi[4]); + SIMD_ALIGNED(uint64_t r1_lo[4]); + SIMD_ALIGNED(uint64_t r1_hi[4]); + while (dst_width > 0) { + r0_lo[0] = LoadU64(src_argb); + r0_lo[1] = LoadU64(src_argb + step); + r0_lo[2] = LoadU64(src_argb + step * 2); + r0_lo[3] = LoadU64(src_argb + step * 3); + r0_hi[0] = LoadU64(src_argb + step * 4); + r0_hi[1] = LoadU64(src_argb + step * 5); + r0_hi[2] = LoadU64(src_argb + step * 6); + r0_hi[3] = LoadU64(src_argb + step * 7); + r1_lo[0] = LoadU64(row1); + r1_lo[1] = LoadU64(row1 + step); + r1_lo[2] = LoadU64(row1 + step * 2); + r1_lo[3] = LoadU64(row1 + step * 3); + r1_hi[0] = LoadU64(row1 + step * 4); + r1_hi[1] = LoadU64(row1 + step * 5); + r1_hi[2] = LoadU64(row1 + step * 6); + r1_hi[3] = LoadU64(row1 + step * 7); + src_argb += step * 8; + row1 += step * 8; + __m256i a0 = _mm256_avg_epu8(_mm256_load_si256((const __m256i*)r0_lo), + _mm256_load_si256((const __m256i*)r1_lo)); + __m256i a1 = _mm256_avg_epu8(_mm256_load_si256((const __m256i*)r0_hi), + _mm256_load_si256((const __m256i*)r1_hi)); + __m256i e = _mm256_castps_si256( + _mm256_shuffle_ps(_mm256_castsi256_ps(a0), _mm256_castsi256_ps(a1), + 0x88)); + __m256i o = _mm256_castps_si256( + _mm256_shuffle_ps(_mm256_castsi256_ps(a0), _mm256_castsi256_ps(a1), + 0xdd)); + __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, d); + dst_argb += 32; + dst_width -= 8; + } +} +#endif // HAS_SCALEARGBROWDOWNEVEN_AVX2 + +#ifdef HAS_SCALEARGBCOLS_AVX2 +static const int32_t kStep0_7_Cols[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + +LIBYUV_TARGET_AVX2 +void ScaleARGBCols_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + int n = dst_width & ~7; + int r = dst_width & 7; + if (n > 0) { + SIMD_ALIGNED(int32_t xi_buf[8]); + SIMD_ALIGNED(uint32_t pix_buf[8]); + __m256i vx = _mm256_add_epi32( + _mm256_set1_epi32(x), + _mm256_mullo_epi32( + _mm256_set1_epi32(dx), + _mm256_loadu_si256((const __m256i*)kStep0_7_Cols))); + const __m256i vdx8 = _mm256_slli_epi32(_mm256_set1_epi32(dx), 3); + for (int i = 0; i < n; i += 8) { + _mm256_store_si256((__m256i*)xi_buf, _mm256_srai_epi32(vx, 16)); + vx = _mm256_add_epi32(vx, vdx8); + pix_buf[0] = LoadU32(src_argb + (ptrdiff_t)xi_buf[0] * 4); + pix_buf[1] = LoadU32(src_argb + (ptrdiff_t)xi_buf[1] * 4); + pix_buf[2] = LoadU32(src_argb + (ptrdiff_t)xi_buf[2] * 4); + pix_buf[3] = LoadU32(src_argb + (ptrdiff_t)xi_buf[3] * 4); + pix_buf[4] = LoadU32(src_argb + (ptrdiff_t)xi_buf[4] * 4); + pix_buf[5] = LoadU32(src_argb + (ptrdiff_t)xi_buf[5] * 4); + pix_buf[6] = LoadU32(src_argb + (ptrdiff_t)xi_buf[6] * 4); + pix_buf[7] = LoadU32(src_argb + (ptrdiff_t)xi_buf[7] * 4); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_load_si256((const __m256i*)pix_buf)); + dst_argb += 32; + } + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + const uint8_t* src = src_argb + (ptrdiff_t)(x >> 16) * 4; + dst_argb[0] = src[0]; + dst_argb[1] = src[1]; + dst_argb[2] = src[2]; + dst_argb[3] = src[3]; + dst_argb += 4; + x = (int)((uint32_t)x + (uint32_t)dx); + } +} +#endif // HAS_SCALEARGBCOLS_AVX2 + +#ifdef HAS_SCALEARGBCOLSUP2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + (void)x; + (void)dx; + while (dst_width > 0) { + __m256i s = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_argb)), + 0xd8); + src_argb += 16; + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi32(s, s)); + dst_argb += 32; + dst_width -= 8; + } +} +#endif // HAS_SCALEARGBCOLSUP2_AVX2 + +#ifdef HAS_SCALEARGBFILTERCOLS_AVX2 +static const int32_t kStep0_3_ARGB[4] = {0, 1, 2, 3}; +static const uvec8 kShuffleColARGB = { + 0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u, + 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u}; +static const uint8_t kShuffleFractionsAVX2[32] = { + 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, + 8u, 8u, 8u, 8u, 8u, 8u, 8u, 8u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, +}; + +LIBYUV_TARGET_AVX2 +void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + int n = dst_width & ~3; + int r = dst_width & 3; + if (n > 0) { + SIMD_ALIGNED(int32_t xi_buf[4]); + SIMD_ALIGNED(uint64_t pix_buf[4]); + const __m256i shuf_argb = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kShuffleColARGB)); + const __m256i shuf_frac = + _mm256_loadu_si256((const __m256i*)kShuffleFractionsAVX2); + const __m256i mask_7f = _mm256_set1_epi16(0x007f); + __m128i vx = _mm_add_epi32( + _mm_set1_epi32(x), + _mm_mullo_epi32(_mm_set1_epi32(dx), + _mm_loadu_si128((const __m128i*)kStep0_3_ARGB))); + const __m128i vdx4 = _mm_slli_epi32(_mm_set1_epi32(dx), 2); + for (int i = 0; i < n; i += 4) { + _mm_store_si128((__m128i*)xi_buf, _mm_srai_epi32(vx, 16)); + __m128i f128 = _mm_srli_epi16(vx, 9); + vx = _mm_add_epi32(vx, vdx4); + pix_buf[0] = LoadU64(src_argb + (ptrdiff_t)xi_buf[0] * 4); + pix_buf[1] = LoadU64(src_argb + (ptrdiff_t)xi_buf[1] * 4); + pix_buf[2] = LoadU64(src_argb + (ptrdiff_t)xi_buf[2] * 4); + pix_buf[3] = LoadU64(src_argb + (ptrdiff_t)xi_buf[3] * 4); + __m256i pix = _mm256_shuffle_epi8( + _mm256_load_si256((const __m256i*)pix_buf), shuf_argb); + __m256i weights = _mm256_xor_si256( + _mm256_shuffle_epi8( + _mm256_inserti128_si256(_mm256_zextsi128_si256(f128), f128, 1), + shuf_frac), + mask_7f); + __m256i res = _mm256_srli_epi16(_mm256_maddubs_epi16(pix, weights), 7); + res = _mm256_permute4x64_epi64(_mm256_packus_epi16(res, res), 0xd8); + _mm_storeu_si128((__m128i*)dst_argb, _mm256_castsi256_si128(res)); + dst_argb += 16; + } + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + ptrdiff_t xi = (ptrdiff_t)(x >> 16) * 4; + int xf = (x >> 9) & 0x7f; + const uint8_t* a = src_argb + xi; + const uint8_t* b = a + 4; + dst_argb[0] = (uint8_t)((a[0] * (0x7f ^ xf) + b[0] * xf) >> 7); + dst_argb[1] = (uint8_t)((a[1] * (0x7f ^ xf) + b[1] * xf) >> 7); + dst_argb[2] = (uint8_t)((a[2] * (0x7f ^ xf) + b[2] * xf) >> 7); + dst_argb[3] = (uint8_t)((a[3] * (0x7f ^ xf) + b[3] * xf) >> 7); + dst_argb += 4; + x = (int)((uint32_t)x + (uint32_t)dx); + } +} +#endif // HAS_SCALEARGBFILTERCOLS_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowDown2_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + while (dst_width >= 16) { + __m256i s0 = + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_uv), 16); + __m256i s1 = _mm256_srli_epi32( + _mm256_loadu_si256((const __m256i*)(src_uv + 32)), 16); + src_uv += 64; + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_uv, d); + dst_uv += 32; + dst_width -= 16; + } + if (dst_width > 0) { + __m256i s0 = + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_uv), 16); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s0), 0xd8); + _mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(d)); + } +} +#endif // HAS_SCALEUVROWDOWN2_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + const __m256i mask = _mm256_set1_epi32(0x0000ffff); + while (dst_width >= 16) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32)); + src_uv += 64; + __m256i s0 = + _mm256_and_si256(_mm256_avg_epu8(r0, _mm256_srli_epi32(r0, 16)), mask); + __m256i s1 = + _mm256_and_si256(_mm256_avg_epu8(r1, _mm256_srli_epi32(r1, 16)), mask); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_uv, d); + dst_uv += 32; + dst_width -= 16; + } + if (dst_width > 0) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i s0 = + _mm256_and_si256(_mm256_avg_epu8(r0, _mm256_srli_epi32(r0, 16)), mask); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s0), 0xd8); + _mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(d)); + } +} +#endif // HAS_SCALEUVROWDOWN2LINEAR_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2 +static const uvec8 kShuffleSplitUV = {0u, 2u, 4u, 6u, 8u, 10u, 12u, 14u, + 1u, 3u, 5u, 7u, 9u, 11u, 13u, 15u}; +static const uvec8 kShuffleMergeUV = {0u, 8u, 2u, 10u, 4u, 12u, + 6u, 14u, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80}; + +LIBYUV_TARGET_AVX2 +void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i split = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kShuffleSplitUV)); + const __m256i merge = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kShuffleMergeUV)); + const __m256i ones = _mm256_set1_epi8(1); + const __m256i zero = _mm256_setzero_si256(); + while (dst_width > 0) { + __m256i r0 = _mm256_maddubs_epi16( + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), split), + ones); + __m256i r1 = _mm256_maddubs_epi16( + _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), split), + ones); + src_ptr += 32; + __m256i s = + _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(r0, r1), 1), zero); + __m256i d = _mm256_permute4x64_epi64(_mm256_shuffle_epi8(s, merge), 0xd8); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d)); + dst_ptr += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVROWDOWN2BOX_AVX2 + +#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowDownEven_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + const ptrdiff_t step = (ptrdiff_t)src_stepx * 2; + SIMD_ALIGNED(uint16_t uv_buf[8]); + while (dst_width > 0) { + uv_buf[0] = LoadU16(src_uv); + uv_buf[1] = LoadU16(src_uv + step); + uv_buf[2] = LoadU16(src_uv + step * 2); + uv_buf[3] = LoadU16(src_uv + step * 3); + uv_buf[4] = LoadU16(src_uv + step * 4); + uv_buf[5] = LoadU16(src_uv + step * 5); + uv_buf[6] = LoadU16(src_uv + step * 6); + uv_buf[7] = LoadU16(src_uv + step * 7); + src_uv += step * 8; + _mm_storeu_si128((__m128i*)dst_uv, _mm_load_si128((const __m128i*)uv_buf)); + dst_uv += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVROWDOWNEVEN_AVX2 + +#if defined(HAS_SCALEUVROWUP2_LINEAR_AVX2) || \ + defined(HAS_SCALEUVROWUP2_BILINEAR_AVX2) +static const uvec8 kUVLinearMadd31 = {3, 1, 3, 1, 1, 3, 1, 3, + 3, 1, 3, 1, 1, 3, 1, 3}; +#endif + +#ifdef HAS_SCALEUVROWUP2_LINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowUp2_Linear_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width) { + const __m256i madd31 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kUVLinearMadd31)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width > 0) { + __m256i s0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i s1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 2))), + 0xd8); + src_ptr += 16; + __m256i p = _mm256_unpacklo_epi8(s0, s1); + __m256i lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(p, p), madd31); + __m256i hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(p, p), madd31); + lo = _mm256_srli_epi16(_mm256_add_epi16(lo, round2), 2); + hi = _mm256_srli_epi16(_mm256_add_epi16(hi, round2), 2); + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(lo, hi)); + dst_ptr += 32; + dst_width -= 16; + } +} +#endif // HAS_SCALEUVROWUP2_LINEAR_AVX2 + +#ifdef HAS_SCALEUVROWUP2_BILINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowUp2_Bilinear_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i madd31 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kUVLinearMadd31)); + const __m256i round8 = _mm256_set1_epi16(8); + while (dst_width > 0) { + __m256i a0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i a1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 2))), + 0xd8); + __m256i pa = _mm256_unpacklo_epi8(a0, a1); + __m256i r0_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(pa, pa), madd31); + __m256i r0_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(pa, pa), madd31); + + __m256i b0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))), + 0xd8); + __m256i b1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 2))), + 0xd8); + src_ptr += 16; + __m256i pb = _mm256_unpacklo_epi8(b0, b1); + __m256i r1_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(pb, pb), madd31); + __m256i r1_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(pb, pb), madd31); + + __m256i d0_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_lo, r0_lo), r0_lo), + _mm256_add_epi16(r1_lo, round8)), + 4); + __m256i d1_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_lo, r1_lo), r1_lo), + _mm256_add_epi16(r0_lo, round8)), + 4); + __m256i d0_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_hi, r0_hi), r0_hi), + _mm256_add_epi16(r1_hi, round8)), + 4); + __m256i d1_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_hi, r1_hi), r1_hi), + _mm256_add_epi16(r0_hi, round8)), + 4); + + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(d0_lo, d0_hi)); + _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), + _mm256_packus_epi16(d1_lo, d1_hi)); + dst_ptr += 32; + dst_width -= 16; + } +} +#endif // HAS_SCALEUVROWUP2_BILINEAR_AVX2 + +#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowUp2_Linear_16_AVX2(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int dst_width) { + const __m256i round2 = _mm256_set1_epi32(2); + while (dst_width > 0) { + __m256i near_lo = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr)); + __m256i near_hi = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 2))); + src_ptr += 8; + __m256i far_lo = _mm256_shuffle_epi32(near_lo, 0x4e); + __m256i far_hi = _mm256_shuffle_epi32(near_hi, 0x4e); + __m256i d_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(near_lo, near_lo), + _mm256_add_epi32(near_lo, _mm256_add_epi32(far_lo, round2))), + 2); + __m256i d_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(near_hi, near_hi), + _mm256_add_epi32(near_hi, _mm256_add_epi32(far_hi, round2))), + 2); + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi32(d_lo, d_hi)); + dst_ptr += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVROWUP2_LINEAR_16_AVX2 + +#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i round8 = _mm256_set1_epi32(8); + while (dst_width > 0) { + __m256i a0 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr)); + __m256i a1 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 2))); + __m256i r0_lo = _mm256_add_epi32( + _mm256_add_epi32(a0, a0), + _mm256_add_epi32(a0, _mm256_shuffle_epi32(a0, 0x4e))); + __m256i r0_hi = _mm256_add_epi32( + _mm256_add_epi32(a1, a1), + _mm256_add_epi32(a1, _mm256_shuffle_epi32(a1, 0x4e))); + + __m256i b0 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))); + __m256i b1 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 2))); + src_ptr += 8; + __m256i r1_lo = _mm256_add_epi32( + _mm256_add_epi32(b0, b0), + _mm256_add_epi32(b0, _mm256_shuffle_epi32(b0, 0x4e))); + __m256i r1_hi = _mm256_add_epi32( + _mm256_add_epi32(b1, b1), + _mm256_add_epi32(b1, _mm256_shuffle_epi32(b1, 0x4e))); + + __m256i d0_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_lo, r0_lo), r0_lo), + _mm256_add_epi32(r1_lo, round8)), + 4); + __m256i d1_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_lo, r1_lo), r1_lo), + _mm256_add_epi32(r0_lo, round8)), + 4); + __m256i d0_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_hi, r0_hi), r0_hi), + _mm256_add_epi32(r1_hi, round8)), + 4); + __m256i d1_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_hi, r1_hi), r1_hi), + _mm256_add_epi32(r0_hi, round8)), + 4); + + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi32(d0_lo, d0_hi)); + _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), + _mm256_packus_epi32(d1_lo, d1_hi)); + dst_ptr += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVROWUP2_BILINEAR_16_AVX2 + #undef LIBYUV_TARGET_AVX2 #undef LIBYUV_TARGET_AVX512BW #undef LIBYUV_TARGET_AVX512VBMI diff --git a/unit_test/compare_test.cc b/unit_test/compare_test.cc index 2556bedaa..ba0bd272c 100644 --- a/unit_test/compare_test.cc +++ b/unit_test/compare_test.cc @@ -208,6 +208,61 @@ TEST_F(LibYUVCompareTest, BenchmarkARGBDetect_Unaligned) { free_aligned_buffer_page_end(src_a); } +TEST_F(LibYUVCompareTest, TestARGBDetectAnyWidth) { + const int kMaxWidth = 65; + const int kBufSize = kMaxWidth * 4 + 4; + align_buffer_page_end(src_a, kBufSize); + memset(src_a, 255, kBufSize); + + for (int width = 1; width <= kMaxWidth; ++width) { + for (int offset = 0; offset < 4; ++offset) { + uint8_t* row = src_a + kBufSize - width * 4 - offset; + // All 255 -> 0 (unknown). + MaskCpuFlags(disable_cpu_flags_); + uint32_t c_fourcc = ARGBDetect(row, width * 4, width, 1); + MaskCpuFlags(benchmark_cpu_info_); + uint32_t opt_fourcc = ARGBDetect(row, width * 4, width, 1); + EXPECT_EQ(0u, c_fourcc); + EXPECT_EQ(c_fourcc, opt_fourcc) << "width " << width << " off " << offset; + + for (int x = 0; x < width; ++x) { + // Byte 0 != 255 -> FOURCC_BGRA. + row[x * 4 + 0] = 128; + MaskCpuFlags(disable_cpu_flags_); + c_fourcc = ARGBDetect(row, width * 4, width, 1); + MaskCpuFlags(benchmark_cpu_info_); + opt_fourcc = ARGBDetect(row, width * 4, width, 1); + EXPECT_EQ(static_cast(FOURCC_BGRA), c_fourcc); + EXPECT_EQ(c_fourcc, opt_fourcc) + << "width " << width << " x " << x << " byte 0"; + + // Both byte 0 and byte 3 != 255 -> FOURCC_BGRA (byte 0 checked first). + row[x * 4 + 3] = 64; + MaskCpuFlags(disable_cpu_flags_); + c_fourcc = ARGBDetect(row, width * 4, width, 1); + MaskCpuFlags(benchmark_cpu_info_); + opt_fourcc = ARGBDetect(row, width * 4, width, 1); + EXPECT_EQ(static_cast(FOURCC_BGRA), c_fourcc); + EXPECT_EQ(c_fourcc, opt_fourcc) + << "width " << width << " x " << x << " byte 0+3"; + + // Byte 3 != 255 only -> FOURCC_ARGB. + row[x * 4 + 0] = 255; + MaskCpuFlags(disable_cpu_flags_); + c_fourcc = ARGBDetect(row, width * 4, width, 1); + MaskCpuFlags(benchmark_cpu_info_); + opt_fourcc = ARGBDetect(row, width * 4, width, 1); + EXPECT_EQ(static_cast(FOURCC_ARGB), c_fourcc); + EXPECT_EQ(c_fourcc, opt_fourcc) + << "width " << width << " x " << x << " byte 3"; + row[x * 4 + 3] = 255; + } + } + } + + free_aligned_buffer_page_end(src_a); +} + #ifdef ENABLE_ROW_TESTS TEST_F(LibYUVCompareTest, BenchmarkHammingDistance_Opt) { const int kMaxWidth = 4096 * 3; diff --git a/unit_test/cpu_test.cc b/unit_test/cpu_test.cc index 1a08f609b..3829f8dd2 100644 --- a/unit_test/cpu_test.cc +++ b/unit_test/cpu_test.cc @@ -207,6 +207,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) { int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8); int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8); int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM); + int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32); printf("Has X86 0x%x\n", has_x86); printf("Has SSE2 0x%x\n", has_sse2); printf("Has SSSE3 0x%x\n", has_ssse3); @@ -230,6 +231,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) { printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8); printf("Has AMXINT8 0x%x\n", has_amxint8); printf("Has AVX512BMM 0x%x\n", has_avx512bmm); + printf("Has AMXTF32 0x%x\n", has_amxtf32); } #endif // defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || // defined(_M_X64) diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 01312ff60..5f7cf116e 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -23,6 +23,11 @@ #include "libyuv/rotate.h" #include "libyuv/scale.h" +#if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \ + !defined(ENABLE_ROW_TESTS) +#define ENABLE_ROW_TESTS +#endif + #ifdef ENABLE_ROW_TESTS // row.h defines SIMD_ALIGNED, overriding unit_test.h // TODO(fbarchard): Remove row.h from unittests. Test public functions. @@ -1573,21 +1578,21 @@ TEST_F(LibYUVPlanarTest, TestAffine) { ASSERT_EQ(96u, interpolate_pixels_C[128][0]); ASSERT_EQ(191u, interpolate_pixels_C[255][3]); -#if defined(HAS_ARGBAFFINEROW_SSE2) - int has_sse2 = TestCpuFlag(kCpuHasSSE2); - if (has_sse2) { +#if defined(HAS_ARGBAFFINEROW_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { SIMD_ALIGNED(uint8_t interpolate_pixels_Opt[1280][4]); - ARGBAffineRow_SSE2(&orig_pixels_0[0][0][0], stride, + ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride, &interpolate_pixels_Opt[0][0], uv_step, 1280); ASSERT_EQ(0, memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1280 * 4)); - ARGBAffineRow_SSE2(&orig_pixels_0[0][0][0], stride, + ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride, &interpolate_pixels_Opt[0][0], uv_step, 1279); ASSERT_EQ(0, memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1279 * 4)); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { - ARGBAffineRow_SSE2(&orig_pixels_0[0][0][0], stride, + ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride, &interpolate_pixels_Opt[0][0], uv_step, 1280); } } @@ -3998,10 +4003,10 @@ TESTTPLANARTOP(MergeXR30, uint16_t, uint8_t, 12) TESTTPLANARTOP(MergeXR30, uint16_t, uint8_t, 16) // TODO(fbarchard): improve test for platforms and cpu detect -#ifdef HAS_MERGEUVROW_16_AVX2 +#if defined(HAS_MERGEUVROW_16_AVX512BW) || defined(HAS_MERGEUVROW_16_AVX2) TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) { - // Round count up to multiple of 8 - const int kPixels = (benchmark_width_ * benchmark_height_ + 7) & ~7; + // Round count up to multiple of 16 + const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15; align_buffer_page_end(src_pixels_u, kPixels * 2); align_buffer_page_end(src_pixels_v, kPixels * 2); @@ -4017,14 +4022,30 @@ TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) { reinterpret_cast(src_pixels_v), reinterpret_cast(dst_pixels_uv_c), 16, kPixels); +#if defined(HAS_MERGEUVROW_16_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_MERGEUVROW_16_AVX2) int has_avx2 = TestCpuFlag(kCpuHasAVX2); +#endif for (int i = 0; i < benchmark_iterations_; ++i) { - if (has_avx2) { +#if defined(HAS_MERGEUVROW_16_AVX512BW) + if (has_avx512) { + MergeUVRow_16_AVX512BW(reinterpret_cast(src_pixels_u), + reinterpret_cast(src_pixels_v), + reinterpret_cast(dst_pixels_uv_opt), 16, + kPixels); + } else +#endif +#if defined(HAS_MERGEUVROW_16_AVX2) + if (has_avx2) { MergeUVRow_16_AVX2(reinterpret_cast(src_pixels_u), reinterpret_cast(src_pixels_v), reinterpret_cast(dst_pixels_uv_opt), 16, kPixels); - } else { + } else +#endif + { MergeUVRow_16_C(reinterpret_cast(src_pixels_u), reinterpret_cast(src_pixels_v), reinterpret_cast(dst_pixels_uv_opt), 16, @@ -4331,10 +4352,10 @@ TEST_F(LibYUVPlanarTest, UYVYToY) { #ifdef ENABLE_ROW_TESTS // TODO(fbarchard): Improve test for more platforms. -#ifdef HAS_CONVERT16TO8ROW_AVX2 +#if defined(HAS_CONVERT16TO8ROW_AVX512BW) || defined(HAS_CONVERT16TO8ROW_AVX2) TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) { - // AVX2 does multiple of 32, so round count up - const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31; + // AVX512 does multiple of 64, so round count up + const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63; align_buffer_page_end(src_pixels_y, kPixels * 2); align_buffer_page_end(dst_pixels_y_opt, kPixels); align_buffer_page_end(dst_pixels_y_c, kPixels); @@ -4351,16 +4372,26 @@ TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) { Convert16To8Row_C(reinterpret_cast(src_pixels_y), dst_pixels_y_c, 16384, kPixels); +#if defined(HAS_CONVERT16TO8ROW_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_CONVERT16TO8ROW_AVX2) int has_avx2 = TestCpuFlag(kCpuHasAVX2); - int has_ssse3 = TestCpuFlag(kCpuHasSSSE3); +#endif for (int i = 0; i < benchmark_iterations_; ++i) { - if (has_avx2) { +#if defined(HAS_CONVERT16TO8ROW_AVX512BW) + if (has_avx512) { + Convert16To8Row_AVX512BW(reinterpret_cast(src_pixels_y), + dst_pixels_y_opt, 16384, kPixels); + } else +#endif +#if defined(HAS_CONVERT16TO8ROW_AVX2) + if (has_avx2) { Convert16To8Row_AVX2(reinterpret_cast(src_pixels_y), dst_pixels_y_opt, 16384, kPixels); - } else if (has_ssse3) { - Convert16To8Row_SSSE3(reinterpret_cast(src_pixels_y), - dst_pixels_y_opt, 16384, kPixels); - } else { + } else +#endif + { Convert16To8Row_C(reinterpret_cast(src_pixels_y), dst_pixels_y_opt, 16384, kPixels); } @@ -4374,7 +4405,7 @@ TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) { free_aligned_buffer_page_end(dst_pixels_y_opt); free_aligned_buffer_page_end(dst_pixels_y_c); } -#endif // HAS_CONVERT16TO8ROW_AVX2 +#endif // HAS_CONVERT16TO8ROW_AVX512BW || HAS_CONVERT16TO8ROW_AVX2 #if defined(HAS_HALFWIDTHROW_16TO8_AVX2) || \ defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \ @@ -4481,8 +4512,8 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) { #if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) if (has_avx512) { - HalfWidthRow_16To8_Any_AVX512BW(src_pixels_uv, kMaxPixels * 2, - dst_pixels_uv_opt, 16384, width); + HalfWidthRow_16To8_AVX512BW(src_pixels_uv, kMaxPixels * 2, + dst_pixels_uv_opt, 16384, width); } else #endif #if defined(HAS_HALFWIDTHROW_16TO8_AVX2) @@ -4519,10 +4550,11 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) { } #endif -#ifdef HAS_UYVYTOYROW_NEON +#if defined(HAS_UYVYTOYROW_AVX512BW) || defined(HAS_UYVYTOYROW_AVX2) || \ + defined(HAS_UYVYTOYROW_NEON) TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) { - // NEON does multiple of 16, so round count up - const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15; + // AVX2/AVX512 does multiple of 32, so round count up + const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31; align_buffer_page_end(src_pixels_y, kPixels * 2); align_buffer_page_end(dst_pixels_y_opt, kPixels); align_buffer_page_end(dst_pixels_y_c, kPixels); @@ -4533,8 +4565,34 @@ TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) { UYVYToYRow_C(src_pixels_y, dst_pixels_y_c, kPixels); +#if defined(HAS_UYVYTOYROW_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_UYVYTOYROW_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); +#endif +#if defined(HAS_UYVYTOYROW_NEON) + int has_neon = TestCpuFlag(kCpuHasNEON); +#endif for (int i = 0; i < benchmark_iterations_; ++i) { - UYVYToYRow_NEON(src_pixels_y, dst_pixels_y_opt, kPixels); +#if defined(HAS_UYVYTOYROW_AVX512BW) + if (has_avx512) { + UYVYToYRow_AVX512BW(src_pixels_y, dst_pixels_y_opt, kPixels); + } else +#endif +#if defined(HAS_UYVYTOYROW_AVX2) + if (has_avx2) { + UYVYToYRow_AVX2(src_pixels_y, dst_pixels_y_opt, kPixels); + } else +#endif +#if defined(HAS_UYVYTOYROW_NEON) + if (has_neon) { + UYVYToYRow_NEON(src_pixels_y, dst_pixels_y_opt, kPixels); + } else +#endif + { + UYVYToYRow_C(src_pixels_y, dst_pixels_y_opt, kPixels); + } } for (int i = 0; i < kPixels; ++i) { @@ -4545,7 +4603,99 @@ TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) { free_aligned_buffer_page_end(dst_pixels_y_opt); free_aligned_buffer_page_end(dst_pixels_y_c); } -#endif // HAS_UYVYTOYROW_NEON +#endif // HAS_UYVYTOYROW_... + +#if defined(HAS_RGB565TOYROW_AVX2) && defined(HAS_ARGB1555TOYROW_AVX2) && \ + defined(HAS_ARGB4444TOYROW_AVX2) +TEST_F(LibYUVPlanarTest, RGB16ToYUVRow_Opt) { + if (!TestCpuFlag(kCpuHasAVX2)) { + return; + } + const int kMaxWidth = 128; + SIMD_ALIGNED(uint8_t src0[kMaxWidth * 2]); + SIMD_ALIGNED(uint8_t src1[kMaxWidth * 2]); + SIMD_ALIGNED(uint8_t dst_y_c[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_y_opt[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_u_c[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_v_c[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_u_opt[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_v_opt[kMaxWidth]); + MemRandomize(src0, sizeof(src0)); + MemRandomize(src1, sizeof(src1)); + + for (int width : {1, 2, 7, 15, 16, 17, 31, 32, 33, 64, 127, 128}) { + const int halfwidth = (width + 1) / 2; + const int stride = kMaxWidth * 2; + + RGB565ToYRow_C(src0, dst_y_c, width); + if (IS_ALIGNED(width, 32)) { + RGB565ToYRow_AVX2(src0, dst_y_opt, width); + } else { + RGB565ToYRow_Any_AVX2(src0, dst_y_opt, width); + } + for (int i = 0; i < width; ++i) { + ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1); + } + + ARGB1555ToYRow_C(src0, dst_y_c, width); + if (IS_ALIGNED(width, 32)) { + ARGB1555ToYRow_AVX2(src0, dst_y_opt, width); + } else { + ARGB1555ToYRow_Any_AVX2(src0, dst_y_opt, width); + } + for (int i = 0; i < width; ++i) { + ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1); + } + + ARGB4444ToYRow_C(src0, dst_y_c, width); + if (IS_ALIGNED(width, 32)) { + ARGB4444ToYRow_AVX2(src0, dst_y_opt, width); + } else { + ARGB4444ToYRow_Any_AVX2(src0, dst_y_opt, width); + } + for (int i = 0; i < width; ++i) { + ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1); + } + + SIMD_ALIGNED(uint8_t src2rows[kMaxWidth * 2 * 2]); + memcpy(src2rows, src0, stride); + memcpy(src2rows + stride, src1, stride); + + RGB565ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width); + if (IS_ALIGNED(width, 32)) { + RGB565ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } else { + RGB565ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } + for (int i = 0; i < halfwidth; ++i) { + ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2); + ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2); + } + + ARGB1555ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width); + if (IS_ALIGNED(width, 32)) { + ARGB1555ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } else { + ARGB1555ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } + for (int i = 0; i < halfwidth; ++i) { + ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2); + ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2); + } + + ARGB4444ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width); + if (IS_ALIGNED(width, 32)) { + ARGB4444ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } else { + ARGB4444ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } + for (int i = 0; i < halfwidth; ++i) { + ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2); + ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2); + } + } +} +#endif #endif // ENABLE_ROW_TESTS @@ -4601,7 +4751,6 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) { int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); #endif int has_avx2 = TestCpuFlag(kCpuHasAVX2); - int has_sse2 = TestCpuFlag(kCpuHasSSE2); for (int i = 0; i < benchmark_iterations_; ++i) { #if defined(HAS_CONVERT8TO16ROW_AVX512BW) if (has_avx512) { @@ -4614,10 +4763,6 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) { Convert8To16Row_AVX2(src_pixels_y, reinterpret_cast(dst_pixels_y_opt), 10, kPixels); - } else if (has_sse2) { - Convert8To16Row_SSE2(src_pixels_y, - reinterpret_cast(dst_pixels_y_opt), 10, - kPixels); } else { Convert8To16Row_C(src_pixels_y, reinterpret_cast(dst_pixels_y_opt), 10, @@ -4666,6 +4811,16 @@ float TestScaleMaxSamples(int benchmark_width, max_opt = ScaleMaxSamples_NEON(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); +#elif defined(HAS_SCALEMAXSAMPLES_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + max_opt = ScaleMaxSamples_AVX2(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else { + max_opt = ScaleMaxSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #else max_opt = ScaleMaxSamples_C(reinterpret_cast(orig_y), @@ -4734,6 +4889,16 @@ float TestScaleSumSamples(int benchmark_width, sum_opt = ScaleSumSamples_NEON(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); +#elif defined(HAS_SCALESUMSAMPLES_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + sum_opt = ScaleSumSamples_AVX2(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else { + sum_opt = ScaleSumSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #else sum_opt = ScaleSumSamples_C(reinterpret_cast(orig_y), @@ -4811,6 +4976,14 @@ float TestScaleSamples(int benchmark_width, #ifdef HAS_SCALESUMSAMPLES_NEON ScaleSamples_NEON(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); +#elif defined(HAS_SCALESAMPLES_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleSamples_AVX2(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } else { + ScaleSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } #else ScaleSamples_C(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); @@ -4923,7 +5096,14 @@ TEST_F(LibYUVPlanarTest, TestGaussRow_Opt) { } GaussRow_C(&orig_pixels[0], &dst_pixels_c[0], 1280); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { -#if !defined(LIBYUV_DISABLE_NEON) && \ +#if defined(HAS_GAUSSROW_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + GaussRow_AVX2(&orig_pixels[0], &dst_pixels_opt[0], 1280); + } else { + GaussRow_C(&orig_pixels[0], &dst_pixels_opt[0], 1280); + } +#elif !defined(LIBYUV_DISABLE_NEON) && \ (defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON)) int has_neon = TestCpuFlag(kCpuHasNEON); if (has_neon) { @@ -4977,7 +5157,18 @@ TEST_F(LibYUVPlanarTest, TestGaussCol_Opt) { &orig_pixels[1280 * 3], &orig_pixels[1280 * 4], &dst_pixels_c[0], 1280); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { -#if !defined(LIBYUV_DISABLE_NEON) && \ +#if defined(HAS_GAUSSCOL_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + GaussCol_AVX2(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2], + &orig_pixels[1280 * 3], &orig_pixels[1280 * 4], + &dst_pixels_opt[0], 1280); + } else { + GaussCol_C(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2], + &orig_pixels[1280 * 3], &orig_pixels[1280 * 4], + &dst_pixels_opt[0], 1280); + } +#elif !defined(LIBYUV_DISABLE_NEON) && \ (defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON)) int has_neon = TestCpuFlag(kCpuHasNEON); if (has_neon) { @@ -5015,7 +5206,14 @@ TEST_F(LibYUVPlanarTest, TestGaussRow_F32_Opt) { } GaussRow_F32_C(&orig_pixels[0], &dst_pixels_c[0], 1280); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { -#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) +#if defined(HAS_GAUSSROW_F32_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + GaussRow_F32_AVX2(&orig_pixels[0], &dst_pixels_opt[0], 1280); + } else { + GaussRow_F32_C(&orig_pixels[0], &dst_pixels_opt[0], 1280); + } +#elif !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) int has_neon = TestCpuFlag(kCpuHasNEON); if (has_neon) { GaussRow_F32_NEON(&orig_pixels[0], &dst_pixels_opt[0], 1280); @@ -5049,7 +5247,18 @@ TEST_F(LibYUVPlanarTest, TestGaussCol_F32_Opt) { &orig_pixels[1280 * 3], &orig_pixels[1280 * 4], &dst_pixels_c[0], 1280); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { -#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) +#if defined(HAS_GAUSSCOL_F32_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + GaussCol_F32_AVX2(&orig_pixels[0], &orig_pixels[1280], + &orig_pixels[1280 * 2], &orig_pixels[1280 * 3], + &orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280); + } else { + GaussCol_F32_C(&orig_pixels[0], &orig_pixels[1280], + &orig_pixels[1280 * 2], &orig_pixels[1280 * 3], + &orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280); + } +#elif !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) int has_neon = TestCpuFlag(kCpuHasNEON); if (has_neon) { GaussCol_F32_NEON(&orig_pixels[0], &orig_pixels[1280], @@ -5129,6 +5338,151 @@ TEST_F(LibYUVPlanarTest, SwapUVRow) { free_aligned_buffer_page_end(src_pixels_vu); free_aligned_buffer_page_end(dst_pixels_uv); } + +TEST_F(LibYUVPlanarTest, HalfRow_Opt) { + const int kMaxPixels = 160; + align_buffer_page_end(src_pixels, kMaxPixels * 2); + align_buffer_page_end(dst_pixels_opt, kMaxPixels); + align_buffer_page_end(dst_pixels_c, kMaxPixels); + MemRandomize(src_pixels, kMaxPixels * 2); + + for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels}}) { + for (int width = 1; width <= 129; ++width) { + memset(dst_pixels_opt, 0, kMaxPixels); + memset(dst_pixels_c, 1, kMaxPixels); + HalfRow_C(src_pixels, stride, dst_pixels_c, width); +#if defined(HAS_HALFROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + if (IS_ALIGNED(width, 32)) { + HalfRow_AVX2(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_Any_AVX2(src_pixels, stride, dst_pixels_opt, width); + } + } else +#endif + { + HalfRow_C(src_pixels, stride, dst_pixels_opt, width); + } + ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width)) + << "stride=" << stride << " width=" << width; + } + } + + free_aligned_buffer_page_end(src_pixels); + free_aligned_buffer_page_end(dst_pixels_opt); + free_aligned_buffer_page_end(dst_pixels_c); +} + +TEST_F(LibYUVPlanarTest, HalfRow_16_Opt) { + const int kMaxPixels = 160; + align_buffer_page_end_16(src_pixels, kMaxPixels * 2); + align_buffer_page_end_16(dst_pixels_opt, kMaxPixels); + align_buffer_page_end_16(dst_pixels_c, kMaxPixels); + for (int i = 0; i < kMaxPixels * 2; ++i) { + src_pixels[i] = static_cast(fastrand()); + } + + for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels}}) { + for (int width = 1; width <= 129; ++width) { + memset(dst_pixels_opt, 0, kMaxPixels * 2); + memset(dst_pixels_c, 1, kMaxPixels * 2); + HalfRow_16_C(src_pixels, stride, dst_pixels_c, width); +#if defined(HAS_HALFROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + if (IS_ALIGNED(width, 16)) { + HalfRow_16_AVX2(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_16_Any_AVX2(src_pixels, stride, dst_pixels_opt, width); + } + } else +#endif + { + HalfRow_16_C(src_pixels, stride, dst_pixels_opt, width); + } + ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width * 2)) + << "stride=" << stride << " width=" << width; + } + } + + free_aligned_buffer_page_end_16(src_pixels); + free_aligned_buffer_page_end_16(dst_pixels_opt); + free_aligned_buffer_page_end_16(dst_pixels_c); +} + +TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Odd_Opt) { + const int kMaxPixels = 160; + align_buffer_page_end_16(src_pixels, kMaxPixels * 4); + align_buffer_page_end(dst_pixels_opt, kMaxPixels); + align_buffer_page_end(dst_pixels_c, kMaxPixels); + for (int i = 0; i < kMaxPixels * 4; ++i) { + src_pixels[i] = fastrand() & 1023; + } + + for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels * 2}}) { + for (int width = 1; width <= 129; ++width) { + memset(dst_pixels_opt, 0, kMaxPixels); + memset(dst_pixels_c, 1, kMaxPixels); + HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_c, 16384, width); +#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + HalfWidthRow_16To8_Odd_AVX2(src_pixels, stride, dst_pixels_opt, 16384, + width); + } else +#endif + { + HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_opt, 16384, + width); + } + ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width)) + << "stride=" << stride << " width=" << width; + } + } + + free_aligned_buffer_page_end_16(src_pixels); + free_aligned_buffer_page_end(dst_pixels_opt); + free_aligned_buffer_page_end(dst_pixels_c); +} + +TEST_F(LibYUVPlanarTest, ARGBChannelReorderRow_C_Test) { + const int kPixels = 65; + align_buffer_page_end(src_argb, kPixels * 4); + align_buffer_page_end(dst_row, kPixels * 4); + align_buffer_page_end(dst_api, kPixels * 4); + MemRandomize(src_argb, kPixels * 4); + + ARGBToABGRRow_C(src_argb, dst_row, kPixels); + ARGBToABGR(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4)); + + ARGBToBGRARow_C(src_argb, dst_row, kPixels); + ARGBToBGRA(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4)); + + ARGBToRGBARow_C(src_argb, dst_row, kPixels); + ARGBToRGBA(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4)); + + RGBAToARGBRow_C(src_argb, dst_row, kPixels); + RGBAToARGB(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4)); + + align_buffer_page_end_16(src_ar64, kPixels * 4); + align_buffer_page_end_16(dst_ab64_row, kPixels * 4); + align_buffer_page_end_16(dst_ab64_api, kPixels * 4); + for (int i = 0; i < kPixels * 4; ++i) { + src_ar64[i] = static_cast(fastrand()); + } + AR64ToAB64Row_C(src_ar64, dst_ab64_row, kPixels); + AR64ToAB64(src_ar64, kPixels * 4, dst_ab64_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_ab64_row, dst_ab64_api, kPixels * 8)); + + free_aligned_buffer_page_end_16(src_ar64); + free_aligned_buffer_page_end_16(dst_ab64_row); + free_aligned_buffer_page_end_16(dst_ab64_api); + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(dst_row); + free_aligned_buffer_page_end(dst_api); +} #endif // ENABLE_ROW_TESTS TEST_F(LibYUVPlanarTest, TestGaussPlane_F32) { diff --git a/unit_test/rotate_test.cc b/unit_test/rotate_test.cc index b9ece9409..5ddf899cb 100644 --- a/unit_test/rotate_test.cc +++ b/unit_test/rotate_test.cc @@ -14,6 +14,11 @@ #include "libyuv/cpu_id.h" #include "libyuv/rotate.h" +#if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \ + !defined(ENABLE_ROW_TESTS) +#define ENABLE_ROW_TESTS +#endif + #ifdef ENABLE_ROW_TESTS #include "libyuv/rotate_row.h" #endif @@ -892,9 +897,9 @@ TEST_F(LibYUVRotateTest, TransposeWx4_32_Test) { TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4, (uint8_t*)dst_pixels_opt, width * 4, width); } else -#elif defined(HAS_TRANSPOSEWX4_32_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - TransposeWx4_32_SSE2((const uint8_t*)src_pixels, height * 4, +#elif defined(HAS_TRANSPOSEWX4_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx4_32_AVX2((const uint8_t*)src_pixels, height * 4, (uint8_t*)dst_pixels_opt, width * 4, width); } else #endif @@ -933,9 +938,9 @@ TEST_F(LibYUVRotateTest, TransposeWx4_32_Opt) { TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4, (uint8_t*)dst_pixels_opt, width * 4, width); } else -#elif defined(HAS_TRANSPOSEWX4_32_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - TransposeWx4_32_SSE2((const uint8_t*)src_pixels, height * 4, +#elif defined(HAS_TRANSPOSEWX4_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx4_32_AVX2((const uint8_t*)src_pixels, height * 4, (uint8_t*)dst_pixels_opt, width * 4, width); } else #endif @@ -954,6 +959,97 @@ TEST_F(LibYUVRotateTest, TransposeWx4_32_Opt) { free_aligned_buffer_page_end(dst_pixels_opt); } +TEST_F(LibYUVRotateTest, TransposeWx4_32_Any) { + for (int width : {4, 8, 12, 16, 20, 28, 36}) { + const int height = 4; + align_buffer_page_end(src_pixels, height * width * 4); + align_buffer_page_end(dst_pixels_c, width * height * 4); + align_buffer_page_end(dst_pixels_opt, width * height * 4); + + MemRandomize(src_pixels, height * width * 4); + memset(dst_pixels_c, 1, width * height * 4); + memset(dst_pixels_opt, 2, width * height * 4); + + TransposeWx4_32_C(src_pixels, height * 4, dst_pixels_c, width * 4, width); +#if defined(HAS_TRANSPOSEWX4_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + if ((width & 7) == 0) { + TransposeWx4_32_AVX2(src_pixels, height * 4, dst_pixels_opt, width * 4, + width); + } else { + TransposeWx4_32_Any_AVX2(src_pixels, height * 4, dst_pixels_opt, + width * 4, width); + } + } else +#endif +#if defined(HAS_TRANSPOSEWX4_32_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + TransposeWx4_32_NEON(src_pixels, height * 4, dst_pixels_opt, width * 4, + width); + } else +#endif + { + TransposeWx4_32_C(src_pixels, height * 4, dst_pixels_opt, width * 4, + width); + } + + ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width * height * 4)) + << "width=" << width; + + free_aligned_buffer_page_end(src_pixels); + free_aligned_buffer_page_end(dst_pixels_c); + free_aligned_buffer_page_end(dst_pixels_opt); + } +} + +#if defined(HAS_TRANSPOSEWXH_32_AVX2) +TEST_F(LibYUVRotateTest, TransposeWxH_32_Test) { + if (!TestCpuFlag(kCpuHasAVX2)) { + return; + } + const int kMaxW = 35; + const int kMaxH = 25; + align_buffer_page_end(src_pixels, kMaxW * kMaxH * 4); + align_buffer_page_end(dst_pixels_c, kMaxW * kMaxH * 4); + align_buffer_page_end(dst_pixels_opt, kMaxW * kMaxH * 4); + MemRandomize(src_pixels, kMaxW * kMaxH * 4); + + for (int h = 1; h <= kMaxH; ++h) { + for (int w = 1; w <= kMaxW; ++w) { + const int src_stride = w * 4; + const int dst_stride = h * 4; + const int src_bytes = src_stride * h; + const int dst_bytes = dst_stride * w; + const uint8_t* src = src_pixels + kMaxW * kMaxH * 4 - src_bytes; + uint8_t* dst_c = dst_pixels_c + kMaxW * kMaxH * 4 - dst_bytes; + uint8_t* dst_opt = dst_pixels_opt + kMaxW * kMaxH * 4 - dst_bytes; + + memset(dst_c, 1, dst_bytes); + memset(dst_opt, 2, dst_bytes); + TransposeWxH_32_C(src, src_stride, dst_c, dst_stride, w, h); + TransposeWxH_32_AVX2(src, src_stride, dst_opt, dst_stride, w, h); + ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << w << "x" << h; + + // Also test negative src_stride (90-degree) and negative dst_stride + // (270-degree). + memset(dst_c, 3, dst_bytes); + memset(dst_opt, 4, dst_bytes); + TransposeWxH_32_C(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride, + dst_c + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride, w, + h); + TransposeWxH_32_AVX2(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride, + dst_opt + (ptrdiff_t)(w - 1) * dst_stride, + -dst_stride, w, h); + ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << "neg " << w << "x" << h; + } + } + + free_aligned_buffer_page_end(src_pixels); + free_aligned_buffer_page_end(dst_pixels_c); + free_aligned_buffer_page_end(dst_pixels_opt); +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX2) + #endif // ENABLE_ROW_TESTS } // namespace libyuv diff --git a/unit_test/scale_plane_test.cc b/unit_test/scale_plane_test.cc index bf2ec6b8a..012377d5c 100644 --- a/unit_test/scale_plane_test.cc +++ b/unit_test/scale_plane_test.cc @@ -267,6 +267,13 @@ TEST_F(LibYUVScaleTest, TestScaleRowDown2Box_16) { } else { ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280); } +#elif defined(HAS_SCALEROWDOWN2_16_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + ScaleRowDown2Box_16_AVX2(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280); + } else { + ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280); + } #else ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280); #endif diff --git a/unit_test/scale_uv_test.cc b/unit_test/scale_uv_test.cc index 5a8d160a9..bbb6bbc68 100644 --- a/unit_test/scale_uv_test.cc +++ b/unit_test/scale_uv_test.cc @@ -39,6 +39,9 @@ static int UVTestFilter(int src_width, if (!SizeValid(src_width, src_height, dst_width, dst_height)) { return 0; } + if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) { + return 0; + } int i; int64_t src_uv_plane_size = Abs(src_width) * Abs(src_height) * 2LL; @@ -259,6 +262,9 @@ static int UVTestBoxAgainstPlanes(int src_width, if (!SizeValid(src_width, src_height, dst_width, dst_height)) { return 0; } + if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) { + return 0; + } int64_t src_uv_plane_size = src_width * src_height * 2LL; int64_t src_plane_size = src_width * src_height * 1LL; int64_t dst_uv_plane_size = dst_width * dst_height * 2LL; @@ -369,4 +375,283 @@ TEST_F(LibYUVScaleTest, UVScaleBoxAverages) { free_aligned_buffer_page_end(src_uv); } +// Test 16-bit UVScale_16 with C vs Opt and return maximum pixel difference. +static int UVTestFilter_16(int src_width, + int src_height, + int dst_width, + int dst_height, + FilterMode f, + int benchmark_iterations, + int disable_cpu_flags, + int benchmark_cpu_info) { + if (!SizeValid(src_width, src_height, dst_width, dst_height)) { + return 0; + } + if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) { + return 0; + } + + int i; + int64_t src_uv_samples = Abs(src_width) * Abs(src_height) * 2LL; + int src_stride_uv = Abs(src_width) * 2; + int64_t dst_uv_samples = dst_width * dst_height * 2LL; + int dst_stride_uv = dst_width * 2; + + align_buffer_page_end(src_uv_buf, src_uv_samples * 2); + align_buffer_page_end(dst_uv_c_buf, dst_uv_samples * 2); + align_buffer_page_end(dst_uv_opt_buf, dst_uv_samples * 2); + + if (!src_uv_buf || !dst_uv_c_buf || !dst_uv_opt_buf) { + printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n"); + return 0; + } + uint16_t* src_uv = reinterpret_cast(src_uv_buf); + uint16_t* dst_uv_c = reinterpret_cast(dst_uv_c_buf); + uint16_t* dst_uv_opt = reinterpret_cast(dst_uv_opt_buf); + + MemRandomize(src_uv_buf, src_uv_samples * 2); + memset(dst_uv_c_buf, 2, dst_uv_samples * 2); + memset(dst_uv_opt_buf, 123, dst_uv_samples * 2); + + MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization. + int rc_c = UVScale_16(src_uv, src_stride_uv, src_width, src_height, dst_uv_c, + dst_stride_uv, dst_width, dst_height, f); + if (rc_c != 0) { + free_aligned_buffer_page_end(dst_uv_c_buf); + free_aligned_buffer_page_end(dst_uv_opt_buf); + free_aligned_buffer_page_end(src_uv_buf); + return 65536; + } + + MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization. + int rc_opt = 0; + for (i = 0; i < benchmark_iterations; ++i) { + rc_opt |= UVScale_16(src_uv, src_stride_uv, src_width, src_height, + dst_uv_opt, dst_stride_uv, dst_width, dst_height, f); + } + if (rc_opt != 0) { + free_aligned_buffer_page_end(dst_uv_c_buf); + free_aligned_buffer_page_end(dst_uv_opt_buf); + free_aligned_buffer_page_end(src_uv_buf); + return 65536; + } + + int max_diff = 0; + for (i = 0; i < dst_uv_samples; ++i) { + int abs_diff = Abs(static_cast(dst_uv_c[i]) - + static_cast(dst_uv_opt[i])); + if (abs_diff > max_diff) { + max_diff = abs_diff; + } + } + + free_aligned_buffer_page_end(dst_uv_c_buf); + free_aligned_buffer_page_end(dst_uv_opt_buf); + free_aligned_buffer_page_end(src_uv_buf); + return max_diff; +} + +#define DX(x, nom, denom) static_cast((Abs(x) / nom) * nom) +#define SX(x, nom, denom) static_cast((x / nom) * denom) + +#define TEST_FACTOR1_16(name, filter, nom, denom) \ + TEST_F(LibYUVScaleTest, UVScale_16_DownBy##name##_##filter) { \ + int diff = UVTestFilter_16( \ + SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \ + DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \ + kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \ + benchmark_cpu_info_); \ + ASSERT_EQ(0, diff); \ + } + +#if defined(ENABLE_FULL_TESTS) +#define TEST_FACTOR_16(name, nom, denom) \ + TEST_FACTOR1_16(name, None, nom, denom) \ + TEST_FACTOR1_16(name, Linear, nom, denom) \ + TEST_FACTOR1_16(name, Bilinear, nom, denom) \ + TEST_FACTOR1_16(name, Box, nom, denom) +#else +#define TEST_FACTOR_16(name, nom, denom) \ + TEST_FACTOR1_16(name, Bilinear, nom, denom) +#endif + +TEST_FACTOR_16(2, 1, 2) +TEST_FACTOR_16(4, 1, 4) +TEST_FACTOR_16(3by4, 3, 4) +TEST_FACTOR_16(3by8, 3, 8) +TEST_FACTOR_16(3, 1, 3) +#undef TEST_FACTOR1_16 +#undef TEST_FACTOR_16 +#undef SX +#undef DX + +#define TEST_SCALETO1_16(name, width, height, filter, max_diff) \ + TEST_F(LibYUVScaleTest, name##To##width##x##height##_##filter) { \ + int diff = UVTestFilter_16(benchmark_width_, benchmark_height_, width, \ + height, kFilter##filter, benchmark_iterations_, \ + disable_cpu_flags_, benchmark_cpu_info_); \ + ASSERT_LE(diff, max_diff); \ + } \ + TEST_F(LibYUVScaleTest, name##From##width##x##height##_##filter) { \ + int diff = UVTestFilter_16(width, height, Abs(benchmark_width_), \ + Abs(benchmark_height_), kFilter##filter, \ + benchmark_iterations_, disable_cpu_flags_, \ + benchmark_cpu_info_); \ + ASSERT_LE(diff, max_diff); \ + } + +#if defined(ENABLE_FULL_TESTS) +#define TEST_SCALETO_16(name, width, height) \ + TEST_SCALETO1_16(name, width, height, None, 0) \ + TEST_SCALETO1_16(name, width, height, Linear, 0) \ + TEST_SCALETO1_16(name, width, height, Bilinear, 0) +#else +#define TEST_SCALETO_16(name, width, height) \ + TEST_SCALETO1_16(name, width, height, Bilinear, 0) +#endif + +TEST_SCALETO_16(UVScale_16, 1, 1) +TEST_SCALETO_16(UVScale_16, 569, 480) +TEST_SCALETO_16(UVScale_16, 640, 360) +#undef TEST_SCALETO1_16 +#undef TEST_SCALETO_16 + +// Compare UVScale_16 against scaling U and V planes separately with +// ScalePlane_16 across Box, Bilinear, Linear, and None filters. +static int UVTestAgainstPlanes_16(int src_width, + int src_height, + int dst_width, + int dst_height, + FilterMode filtering, + int cpu_info) { + if (!SizeValid(src_width, src_height, dst_width, dst_height)) { + return 0; + } + if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) { + return 0; + } + const int abs_src_width = Abs(src_width); + const int abs_src_height = Abs(src_height); + int64_t src_uv_samples = abs_src_width * abs_src_height * 2LL; + int64_t src_plane_samples = abs_src_width * abs_src_height * 1LL; + int64_t dst_uv_samples = dst_width * dst_height * 2LL; + int64_t dst_plane_samples = dst_width * dst_height * 1LL; + + align_buffer_page_end(src_uv_buf, src_uv_samples * 2); + align_buffer_page_end(src_u_buf, src_plane_samples * 2); + align_buffer_page_end(src_v_buf, src_plane_samples * 2); + align_buffer_page_end(dst_uv_buf, dst_uv_samples * 2); + align_buffer_page_end(dst_u_buf, dst_plane_samples * 2); + align_buffer_page_end(dst_v_buf, dst_plane_samples * 2); + if (!src_uv_buf || !src_u_buf || !src_v_buf || !dst_uv_buf || !dst_u_buf || + !dst_v_buf) { + printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n"); + return 0; + } + uint16_t* src_uv = reinterpret_cast(src_uv_buf); + uint16_t* src_u = reinterpret_cast(src_u_buf); + uint16_t* src_v = reinterpret_cast(src_v_buf); + uint16_t* dst_uv = reinterpret_cast(dst_uv_buf); + uint16_t* dst_u = reinterpret_cast(dst_u_buf); + uint16_t* dst_v = reinterpret_cast(dst_v_buf); + + MemRandomize(src_uv_buf, src_uv_samples * 2); + memset(dst_uv_buf, 123, dst_uv_samples * 2); + for (int i = 0; i < abs_src_width * abs_src_height; ++i) { + src_u[i] = src_uv[i * 2 + 0]; + src_v[i] = src_uv[i * 2 + 1]; + } + + MaskCpuFlags(cpu_info); + int rc_uv = UVScale_16(src_uv, abs_src_width * 2, src_width, src_height, + dst_uv, dst_width * 2, dst_width, dst_height, + filtering); + int rc_u = ScalePlane_16(src_u, abs_src_width, src_width, src_height, dst_u, + dst_width, dst_width, dst_height, filtering); + int rc_v = ScalePlane_16(src_v, abs_src_width, src_width, src_height, dst_v, + dst_width, dst_width, dst_height, filtering); + int max_diff = (rc_uv != 0 || rc_u != 0 || rc_v != 0) ? 65536 : 0; + for (int i = 0; i < dst_width * dst_height; ++i) { + int du = Abs(static_cast(dst_uv[i * 2 + 0]) - + static_cast(dst_u[i])); + int dv = Abs(static_cast(dst_uv[i * 2 + 1]) - + static_cast(dst_v[i])); + if (du > max_diff) { + max_diff = du; + } + if (dv > max_diff) { + max_diff = dv; + } + } + + free_aligned_buffer_page_end(dst_v_buf); + free_aligned_buffer_page_end(dst_u_buf); + free_aligned_buffer_page_end(dst_uv_buf); + free_aligned_buffer_page_end(src_v_buf); + free_aligned_buffer_page_end(src_u_buf); + free_aligned_buffer_page_end(src_uv_buf); + return max_diff; +} + +#define TEST_UVPLANE16(name, swidth, sheight, dwidth, dheight, filter, \ + max_diff) \ + TEST_F(LibYUVScaleTest, UVScale_16_MatchesPlane_##name##_##filter) { \ + int diff_c = UVTestAgainstPlanes_16(swidth, sheight, dwidth, dheight, \ + kFilter##filter, disable_cpu_flags_); \ + ASSERT_LE(diff_c, max_diff); \ + int diff_opt = UVTestAgainstPlanes_16(swidth, sheight, dwidth, dheight, \ + kFilter##filter, \ + benchmark_cpu_info_); \ + ASSERT_LE(diff_opt, max_diff); \ + } + +TEST_UVPLANE16(By2, 480, 270, 240, 135, Box, 0) +TEST_UVPLANE16(By2, 480, 270, 240, 135, Bilinear, 0) +TEST_UVPLANE16(By2, 480, 270, 240, 135, None, 0) +TEST_UVPLANE16(By3, 480, 270, 160, 90, Box, 0) +TEST_UVPLANE16(By3Rounded, 320, 180, 107, 60, Box, 0) +TEST_UVPLANE16(By4, 480, 270, 120, 68, Box, 0) +TEST_UVPLANE16(By4Exact, 480, 272, 120, 68, Box, 1) +TEST_UVPLANE16(By6, 360, 180, 60, 30, Box, 0) +TEST_UVPLANE16(By6, 360, 180, 60, 30, None, 0) +TEST_UVPLANE16(OddTail, 33, 17, 7, 5, Box, 0) +TEST_UVPLANE16(By3OddTail, 69, 15, 23, 5, Box, 0) +TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, Bilinear, 0) +TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, Linear, 0) +TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, None, 0) +TEST_UVPLANE16(Up2x, 37, 23, 74, 46, Bilinear, 0) +TEST_UVPLANE16(Up2x, 37, 23, 74, 46, Linear, 0) +TEST_UVPLANE16(Up2x, 37, 23, 74, 46, None, 0) +TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, Bilinear, 0) +TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, Linear, 0) +TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, None, 0) +TEST_UVPLANE16(VerticalOnlyUp, 41, 19, 41, 53, Bilinear, 0) +TEST_UVPLANE16(VerticalOnlyDown, 41, 53, 41, 19, Bilinear, 0) +TEST_UVPLANE16(InvertCopy, 31, -17, 31, 17, None, 0) +TEST_UVPLANE16(InvertUp, 31, -17, 65, 39, Bilinear, 0) +#undef TEST_UVPLANE16 + +// Verify 2x kFilterLinear downscale averages pairs horizontally on odd rows +// without underflowing into the previous row. +TEST_F(LibYUVScaleTest, UVScale_16_DownBy2_LinearAveragesOddRows) { + const int kSrcWidth = 4; + const int kSrcHeight = 2; + const int kDstWidth = 2; + const int kDstHeight = 1; + uint16_t src_uv[kSrcWidth * kSrcHeight * 2] = { + // Row 0 (even row - must not be read): + 60000, 60000, 60000, 60000, 60000, 60000, 60000, 60000, + // Row 1 (odd row - horizontally averaged in pairs): + 100, 1000, 300, 3000, 500, 5000, 900, 7000, + }; + uint16_t dst_uv[kDstWidth * kDstHeight * 2] = {0}; + + ASSERT_EQ(0, UVScale_16(src_uv, kSrcWidth * 2, kSrcWidth, kSrcHeight, dst_uv, + kDstWidth * 2, kDstWidth, kDstHeight, kFilterLinear)); + EXPECT_EQ(200, dst_uv[0]); + EXPECT_EQ(2000, dst_uv[1]); + EXPECT_EQ(700, dst_uv[2]); + EXPECT_EQ(6000, dst_uv[3]); +} + } // namespace libyuv diff --git a/unit_test/unit_test.cc b/unit_test/unit_test.cc index ee1f63e64..c51b184e4 100644 --- a/unit_test/unit_test.cc +++ b/unit_test/unit_test.cc @@ -209,6 +209,12 @@ static int TestCpuEnv(int cpu_info) { if (TestEnv("LIBYUV_DISABLE_AMXINT8")) { cpu_info &= ~libyuv::kCpuHasAMXINT8; } + if (TestEnv("LIBYUV_DISABLE_AVX512BMM")) { + cpu_info &= ~libyuv::kCpuHasAVX512BMM; + } + if (TestEnv("LIBYUV_DISABLE_AMXTF32")) { + cpu_info &= ~libyuv::kCpuHasAMXTF32; + } #endif if (TestEnv("LIBYUV_DISABLE_ASM")) { cpu_info = libyuv::kCpuInitialized; diff --git a/util/cpuid.c b/util/cpuid.c index 33cd4201a..d85595297 100644 --- a/util/cpuid.c +++ b/util/cpuid.c @@ -217,6 +217,7 @@ int main(int argc, const char* argv[]) { int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8); int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8); int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM); + int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32); printf("Has X86 0x%x\n", has_x86); printf("Has SSE2 0x%x\n", has_sse2); printf("Has SSSE3 0x%x\n", has_ssse3); @@ -240,6 +241,7 @@ int main(int argc, const char* argv[]) { printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8); printf("Has AMXINT8 0x%x\n", has_amxint8); printf("Has AVX512BMM 0x%x\n", has_avx512bmm); + printf("Has AMXTF32 0x%x\n", has_amxtf32); #ifdef __linux__ // Test VDPPHPS instruction