coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-29 02:17:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

[libyuv] Replace AVX-512 _Any wrappers with k-mask tail handling

Browse Source 
Replace stack-buffer memcpy _Any wrappers with native AVX-512 k-mask
tail handling across 8 AVX-512 row kernels (unlocking composite
RGBToUVMatrixRow_AVX512BW as well) and remove IS_ALIGNED dispatch
constraints:
- MirrorSplitUVRow_AVX512BW: Masked 16-bit load (vmovdqu16 %{k1%}%{z%}),
  dynamic reverse index (vpsubw + vpermw via kMirrorUVPermute_AVX512),
  and vpmovwb %{k1%} U/V stores; unlocks unaligned SplitRotateUV180.
- ARGBToUVMatrixRow_AVX512BW: Fix MASK=63 vs 32-pixel loop bug;
  zero-load tail dwords per row, duplicate odd final pixel via knotw +
  vpshufd, and store masked U/V bytes.
- I422ToARGBRow_AVX512BW & I422ToAR30Row_AVX512BW: Add masked Y/U/V
  loads (READYUV422_MASK_AVX512BW) and masked ARGB/AR30 stores.
- CopyRow_AVX512BW: Add 64-byte full vector step + bzhi masked tail.
- BlendPlaneRow_AVX512BW, ARGBShuffleRow_AVX512BW, RGBToARGBRow_AVX512BW
  (RAWToARGBRow_AVX512BW / RGB24ToARGBRow_AVX512BW): Masked loads and
  stores in both row_gcc.cc inline assembly and row_win.cc intrinsics.

Reduces x86-64 binary .text size by 2,592 bytes with 0 bytes added to
.rodata.

Performance on Intel Sapphire Rapids (1275x1650, 1000 iterations):
Function (Test)          | Before  | After   | Speedup
-------------------------|---------|---------|--------
NV12Rotate180_Opt        | 429 ms  | 260 ms  | 1.65x
RGB24ToI420_Opt          | 615 ms  | 531 ms  | 1.16x
I420ToAR30_Opt           | 425 ms  | 383 ms  | 1.11x
ARGBToI420_Opt           | 427 ms  | 411 ms  | 1.04x
I420ToARGB_Opt           | 423 ms  | 411 ms  | 1.03x
RGB24ToARGB_Opt          | 493 ms  | 490 ms  | 1.01x
ARGBToBGRA_Opt           | 544 ms  | 542 ms  | 1.00x
CopyPlane_Opt            | 246 ms  | 245 ms  | 1.00x
BlendPlane_Opt           | 259 ms  | 258 ms  | 1.00x
RAWToARGB_Opt            | 486 ms  | 486 ms  | 1.00x

R=rrwinterton@gmail.com

Test: LIBYUV_WIDTH=1275 LIBYUV_HEIGHT=1650 LIBYUV_REPEAT=1000 out/Release/libyuv_unittest
Change-Id: Ib5e5ae52076352649108b20f8df2b12d81860fb1
This commit is contained in:
Frank Barchard 2026-10-12 13:37:50 -07:00
parent 1e2e3a3599
commit 4e0f6662c5
9 changed files with 537 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
include/libyuv/row.h
View File

@ -2059,12 +2059,6 @@ void RGBToUVMatrixRow_AVX512BW(const uint8_t* src_rgb,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void RGBToUVMatrixRow_Any_AVX512BW(const uint8_t* src_rgb,
int src_stride_rgb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_u,
@ -2418,12 +2412,6 @@ void ARGBToUVMatrixRow_Any_AVX2(const uint8_t* src_argb,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void ARGBToUVMatrixRow_Any_AVX512BW(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c);
void ARGBToUV444MatrixRow_Any_SSSE3(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
@ -3933,7 +3921,6 @@ void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_C(const uint8_t* src, uint8_t* dst, int count);
void CopyRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_AVX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_AVX512BW(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
@ -4061,10 +4048,6 @@ void ARGBShuffleRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const uint8_t* param,
int width);
void ARGBShuffleRow_Any_AVX512BW(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const uint8_t* param,
int width);
void ARGBShuffleRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_ptr,
const uint8_t* param,
@ -4192,15 +4175,9 @@ void RAWToARGBRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void RGB24ToARGBRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RAWToARGBRow_Any_AVX512BW(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width);
void RGB24ToARGBRow_Any_AVX512BW(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
void RAWToRGBARow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
@ -5042,18 +5019,6 @@ void I422ToARGBRow_Any_AVX2(const uint8_t* y_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
void I422ToARGBRow_Any_AVX512BW(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
void I422ToAR30Row_Any_AVX512BW(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
void I422ToRGBARow_Any_AVX2(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@ -5508,11 +5473,6 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0,
const uint8_t* alpha,
uint8_t* dst,
int width);
void BlendPlaneRow_Any_AVX512BW(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
uint8_t* dst_ptr,
int width);
void BlendPlaneRow_NEON(const uint8_t* src0,
const uint8_t* src1,
const uint8_t* alpha,

30
source/convert.cc
View File

@ -2926,10 +2926,7 @@ int ARGBToI420Matrix(const uint8_t* src_argb,
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
@ -3161,10 +3158,7 @@ int ARGBToI420Alpha(const uint8_t* src_argb,
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
@ -3416,10 +3410,7 @@ int RGB24ToI420(const uint8_t* src_rgb24,
#endif
#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_RGBTOUVMATRIXROW_NEON)
@ -3564,10 +3555,7 @@ int RGB24ToJ420(const uint8_t* src_rgb24,
#endif
#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_RGBTOUVMATRIXROW_NEON)
@ -3717,10 +3705,7 @@ int RAWToI420(const uint8_t* src_raw,
#endif
#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_RGBTOUVMATRIXROW_NEON)
@ -3855,10 +3840,7 @@ int RAWToJ420(const uint8_t* src_raw,
#endif
#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_RGBTOUVMATRIXROW_NEON)

30
source/convert_argb.cc
View File

@ -99,10 +99,7 @@ int I420ToARGBMatrix(const uint8_t* src_y,
#if defined(HAS_I422TOARGBROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
I422ToARGBRow = I422ToARGBRow_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I422ToARGBRow = I422ToARGBRow_AVX512BW;
}
I422ToARGBRow = I422ToARGBRow_AVX512BW;
}
#endif
#if defined(HAS_I422TOARGBROW_NEON)
@ -365,10 +362,7 @@ int I422ToARGBMatrix(const uint8_t* src_y,
#if defined(HAS_I422TOARGBROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
I422ToARGBRow = I422ToARGBRow_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I422ToARGBRow = I422ToARGBRow_AVX512BW;
}
I422ToARGBRow = I422ToARGBRow_AVX512BW;
}
#endif
#if defined(HAS_I422TOARGBROW_NEON)
@ -3874,10 +3868,7 @@ int RGB24ToARGB(const uint8_t* src_rgb24,
#endif
#if defined(HAS_RGB24TOARGBROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
RGB24ToARGBRow = RGB24ToARGBRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
RGB24ToARGBRow = RGB24ToARGBRow_AVX512BW;
}
RGB24ToARGBRow = RGB24ToARGBRow_AVX512BW;
}
#endif
#if defined(HAS_RGB24TOARGBROW_NEON)
@ -3975,10 +3966,7 @@ int RAWToARGB(const uint8_t* src_raw,
#endif
#if defined(HAS_RAWTOARGBROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
RAWToARGBRow = RAWToARGBRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
RAWToARGBRow = RAWToARGBRow_AVX512BW;
}
RAWToARGBRow = RAWToARGBRow_AVX512BW;
}
#endif
#if defined(HAS_RAWTOARGBROW_NEON)
@ -6689,10 +6677,7 @@ int I420ToRGB565Dither(const uint8_t* src_y,
#if defined(HAS_I422TOARGBROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
I422ToARGBRow = I422ToARGBRow_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I422ToARGBRow = I422ToARGBRow_AVX512BW;
}
I422ToARGBRow = I422ToARGBRow_AVX512BW;
}
#endif
#if defined(HAS_I422TOARGBROW_NEON)
@ -6865,10 +6850,7 @@ int I420ToAR30Matrix(const uint8_t* src_y,
#if defined(HAS_I422TOAR30ROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) ==
(kCpuHasAVX512BW | kCpuHasAVX512VL)) {
I422ToAR30Row = I422ToAR30Row_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
I422ToAR30Row = I422ToAR30Row_AVX512BW;
}
I422ToAR30Row = I422ToAR30Row_AVX512BW;
}
#endif
#if defined(HAS_I422TOAR30ROW_NEON)

35
source/convert_from_argb.cc
View File

@ -380,10 +380,7 @@ int ARGBToI422Matrix(const uint8_t* src_argb,
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
@ -856,10 +853,7 @@ int ARGBToNV16Matrix(const uint8_t* src_argb,
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
@ -1660,10 +1654,7 @@ int ARGBToYUY2Matrix(const uint8_t* src_argb,
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
@ -1829,10 +1820,7 @@ int ARGBToUYVYMatrix(const uint8_t* src_argb,
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_NEON)
@ -2095,10 +2083,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb,
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
@ -2496,10 +2481,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb,
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
ARGBToUVMatrixRow = ARGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBTOUVMATRIXROW_RVV)
@ -4206,10 +4188,7 @@ static int RAWToNVMatrix(const uint8_t* src_raw,
#endif
#if defined(HAS_RGBTOUVMATRIXROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
RGBToUVMatrixRow = RGBToUVMatrixRow_AVX512BW;
}
#endif
#if defined(HAS_RGBTOUVMATRIXROW_NEON)

22
source/planar_functions.cc
View File

@ -67,7 +67,7 @@ void CopyPlane(const uint8_t* src_y,
#endif
#if defined(HAS_COPYROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
CopyRow = IS_ALIGNED(width, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW;
CopyRow = CopyRow_AVX512BW;
}
#endif
#if defined(HAS_COPYROW_ERMS)
@ -3219,10 +3219,7 @@ int BlendPlane(const uint8_t* src_y0,
#endif
#if defined(HAS_BLENDPLANEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
BlendPlaneRow = BlendPlaneRow_Any_AVX512BW;
if (IS_ALIGNED(width, 64)) {
BlendPlaneRow = BlendPlaneRow_AVX512BW;
}
BlendPlaneRow = BlendPlaneRow_AVX512BW;
}
#endif
#if defined(HAS_BLENDPLANEROW_NEON)
@ -3326,10 +3323,7 @@ int I420Blend(const uint8_t* src_y0,
#endif
#if defined(HAS_BLENDPLANEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
BlendPlaneRow = BlendPlaneRow_Any_AVX512BW;
if (IS_ALIGNED(halfwidth, 64)) {
BlendPlaneRow = BlendPlaneRow_AVX512BW;
}
BlendPlaneRow = BlendPlaneRow_AVX512BW;
}
#endif
#if defined(HAS_BLENDPLANEROW_NEON)
@ -5008,10 +5002,7 @@ int ARGBShuffle(const uint8_t* src_argb,
#endif
#if defined(HAS_ARGBSHUFFLEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBShuffleRow = ARGBShuffleRow_Any_AVX512BW;
if (IS_ALIGNED(width, 32)) {
ARGBShuffleRow = ARGBShuffleRow_AVX512BW;
}
ARGBShuffleRow = ARGBShuffleRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBSHUFFLEROW_NEON)
@ -5103,10 +5094,7 @@ int AR64Shuffle(const uint16_t* src_ar64,
#endif
#if defined(HAS_ARGBSHUFFLEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
AR64ShuffleRow = ARGBShuffleRow_Any_AVX512BW;
if (IS_ALIGNED(width, 16)) {
AR64ShuffleRow = ARGBShuffleRow_AVX512BW;
}
AR64ShuffleRow = ARGBShuffleRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBSHUFFLEROW_NEON)

4
source/rotate.cc
View File

@ -258,7 +258,7 @@ void RotatePlane180(const uint8_t* src,
#endif
#if defined(HAS_COPYROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
CopyRow = IS_ALIGNED(width, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW;
CopyRow = CopyRow_AVX512BW;
}
#endif
#if defined(HAS_COPYROW_ERMS)
@ -499,7 +499,7 @@ void SplitRotateUV180(const uint8_t* src,
}
#endif
#if defined(HAS_MIRRORSPLITUVROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW) && IS_ALIGNED(width, 32)) {
if (TestCpuFlag(kCpuHasAVX512BW)) {
MirrorSplitUVRow = MirrorSplitUVRow_AVX512BW;
}
#endif

32
source/row_any.cc
View File

@ -340,9 +340,6 @@ ANY31(I422ToUYVYRow_Any_LASX, I422ToUYVYRow_LASX, 1, 1, 4, 31)
#ifdef HAS_BLENDPLANEROW_AVX2
ANY31(BlendPlaneRow_Any_AVX2, BlendPlaneRow_AVX2, 0, 0, 1, 31)
#endif
#ifdef HAS_BLENDPLANEROW_AVX512BW
ANY31(BlendPlaneRow_Any_AVX512BW, BlendPlaneRow_AVX512BW, 0, 0, 1, 63)
#endif
#ifdef HAS_BLENDPLANEROW_SSSE3
ANY31(BlendPlaneRow_Any_SSSE3, BlendPlaneRow_SSSE3, 0, 0, 1, 7)
#endif
@ -409,9 +406,6 @@ ANY31C(I422ToAR30Row_Any_SSSE3, I422ToAR30Row_SSSE3, 1, 0, 4, 7)
#ifdef HAS_I422TOAR30ROW_AVX2
ANY31C(I422ToAR30Row_Any_AVX2, I422ToAR30Row_AVX2, 1, 0, 4, 15)
#endif
#ifdef HAS_I422TOAR30ROW_AVX512BW
ANY31C(I422ToAR30Row_Any_AVX512BW, I422ToAR30Row_AVX512BW, 1, 0, 4, 31)
#endif
#ifdef HAS_I444TOARGBROW_SSSE3
ANY31C(I444ToARGBRow_Any_SSSE3, I444ToARGBRow_SSSE3, 0, 0, 4, 7)
#endif
@ -424,9 +418,6 @@ ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15)
#ifdef HAS_I422TOARGBROW_AVX2
ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15)
#endif
#ifdef HAS_I422TOARGBROW_AVX512BW
ANY31C(I422ToARGBRow_Any_AVX512BW, I422ToARGBRow_AVX512BW, 1, 0, 4, 31)
#endif
#ifdef HAS_I422TORGBAROW_AVX2
ANY31C(I422ToRGBARow_Any_AVX2, I422ToRGBARow_AVX2, 1, 0, 4, 15)
#endif
@ -1034,9 +1025,6 @@ ANY21PT(MergeUVRow_16_Any_NEON, MergeUVRow_16_NEON, uint16_t, 2, 7)
memcpy(dst_ptr + np * BPP, vout, r * BPP); \
}
#ifdef HAS_COPYROW_AVX512BW
ANY11(CopyRow_Any_AVX512BW, CopyRow_AVX512BW, 0, 1, 1, 127)
#endif
#ifdef HAS_COPYROW_AVX
ANY11(CopyRow_Any_AVX, CopyRow_AVX, 0, 1, 1, 63)
#endif
@ -1108,12 +1096,6 @@ ANY11(RAWToARGBRow_Any_AVX2, RAWToARGBRow_AVX2, 0, 3, 4, 31)
#if defined(HAS_RGB24TOARGBROW_AVX2)
ANY11(RGB24ToARGBRow_Any_AVX2, RGB24ToARGBRow_AVX2, 0, 3, 4, 31)
#endif
#if defined(HAS_RAWTOARGBROW_AVX512BW)
ANY11(RAWToARGBRow_Any_AVX512BW, RAWToARGBRow_AVX512BW, 0, 3, 4, 63)
#endif
#if defined(HAS_RGB24TOARGBROW_AVX512BW)
ANY11(RGB24ToARGBRow_Any_AVX512BW, RGB24ToARGBRow_AVX512BW, 0, 3, 4, 63)
#endif
#if defined(HAS_RAWTORGBAROW_SSSE3)
ANY11(RAWToRGBARow_Any_SSSE3, RAWToRGBARow_SSSE3, 0, 3, 4, 15)
#endif
@ -1503,14 +1485,6 @@ ANY11P(ARGBShuffleRow_Any_SSSE3, ARGBShuffleRow_SSSE3, const uint8_t*, 4, 4, 7)
#ifdef HAS_ARGBSHUFFLEROW_AVX2
ANY11P(ARGBShuffleRow_Any_AVX2, ARGBShuffleRow_AVX2, const uint8_t*, 4, 4, 15)
#endif
#ifdef HAS_ARGBSHUFFLEROW_AVX512BW
ANY11P(ARGBShuffleRow_Any_AVX512BW,
ARGBShuffleRow_AVX512BW,
const uint8_t*,
4,
4,
31)
#endif
#ifdef HAS_ARGBSHUFFLEROW_NEON
ANY11P(ARGBShuffleRow_Any_NEON, ARGBShuffleRow_NEON, const uint8_t*, 4, 4, 3)
#endif
@ -2465,12 +2439,6 @@ ANY12MS(ARGB1555ToUVMatrixRow_Any_AVX2, ARGB1555ToUVMatrixRow_AVX2, 0, 2, 31)
ANY12MS(ARGB4444ToUVMatrixRow_Any_AVX2, ARGB4444ToUVMatrixRow_AVX2, 0, 2, 31)
#endif
#endif
#ifdef HAS_ARGBTOUVMATRIXROW_AVX512BW
ANY12MS(ARGBToUVMatrixRow_Any_AVX512BW, ARGBToUVMatrixRow_AVX512BW, 0, 4, 63)
#endif
#ifdef HAS_RGBTOUVMATRIXROW_AVX512BW
ANY12MS(RGBToUVMatrixRow_Any_AVX512BW, RGBToUVMatrixRow_AVX512BW, 0, 3, 63)
#endif
#ifdef HAS_ARGBTOUVMATRIXROW_SSSE3
ANY12MS(ARGBToUVMatrixRow_Any_SSSE3, ARGBToUVMatrixRow_SSSE3, 0, 4, 7)
#endif

481
source/row_gcc.cc
View File

@ -458,21 +458,25 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
uint8_t* dst_argb,
const uint32_t* shuffler,
int width) {
uintptr_t temp;
uintptr_t mask;
asm volatile(
"vmovdqu64 %4,%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW
"vmovdqu64 %5,%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW
"vmovdqu64 %6,%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW
"vmovdqu64 %7,%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW
"vbroadcasti64x4 %3,%%zmm4 \n" // shuffler (32 bytes)
"vmovdqu64 %[q0],%%zmm20 \n" // kPermqRGB24ToARGB0_AVX512BW
"vmovdqu64 %[q1],%%zmm21 \n" // kPermqRGB24ToARGB1_AVX512BW
"vmovdqu64 %[q2],%%zmm22 \n" // kPermqRGB24ToARGB2_AVX512BW
"vmovdqu64 %[q3],%%zmm23 \n" // kPermqRGB24ToARGB3_AVX512BW
"vbroadcasti64x4 (%[shuffler]),%%zmm4 \n" // shuffler (32 bytes)
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n" // 0xffffffff
"vpslld $0x18,%%zmm16,%%zmm16 \n" // 0xff000000
"sub $0x40,%[width] \n"
"jl 2f \n"
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0 \n"
"vmovdqu8 64(%0),%%zmm1 \n"
"vmovdqu8 128(%0),%%zmm2 \n"
"lea 192(%0),%0 \n"
"vmovdqu8 (%[src_raw]),%%zmm0 \n"
"vmovdqu8 64(%[src_raw]),%%zmm1 \n"
"vmovdqu8 128(%[src_raw]),%%zmm2 \n"
"lea 192(%[src_raw]),%[src_raw] \n"
"vpermq %%zmm2,%%zmm23,%%zmm3 \n"
"vmovdqa64 %%zmm22,%%zmm18 \n"
"vpermi2q %%zmm2,%%zmm1,%%zmm18 \n"
@ -489,24 +493,71 @@ void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
"vpord %%zmm16,%%zmm1,%%zmm1 \n"
"vpord %%zmm16,%%zmm2,%%zmm2 \n"
"vpord %%zmm16,%%zmm3,%%zmm3 \n"
"vmovdqu32 %%zmm0,(%1) \n"
"vmovdqu32 %%zmm1,0x40(%1) \n"
"vmovdqu32 %%zmm2,0x80(%1) \n"
"vmovdqu32 %%zmm3,0xc0(%1) \n"
"lea 0x100(%1),%1 \n"
"sub $0x40,%2 \n"
"jg 1b \n"
"vmovdqu32 %%zmm0,(%[dst_argb]) \n"
"vmovdqu32 %%zmm1,0x40(%[dst_argb]) \n"
"vmovdqu32 %%zmm2,0x80(%[dst_argb]) \n"
"vmovdqu32 %%zmm3,0xc0(%[dst_argb]) \n"
"lea 0x100(%[dst_argb]),%[dst_argb]\n"
"sub $0x40,%[width] \n"
"jge 1b \n"
"2: \n"
"add $0x40,%[width] \n"
"je 99f \n"
"mov $-1,%[temp] \n"
"bzhi %q[width],%[temp],%[mask] \n"
"kmovq %[mask],%%k1 \n"
"kshiftrq $16,%%k1,%%k2 \n"
"kshiftrq $32,%%k1,%%k3 \n"
"kshiftrq $48,%%k1,%%k4 \n"
"lea (%q[width],%q[width],2),%k[width] \n"
"bzhi %q[width],%[temp],%[mask] \n"
"kmovq %[mask],%%k5 \n"
"vmovdqu8 (%[src_raw]),%%zmm0%{%%k5%}%{z%} \n"
"xor %k[mask],%k[mask] \n"
"sub $0x40,%k[width] \n"
"cmovg %[temp],%[mask] \n"
"bzhi %q[width],%[mask],%[mask] \n"
"kmovq %[mask],%%k5 \n"
"vmovdqu8 64(%[src_raw]),%%zmm1%{%%k5%}%{z%} \n"
"xor %k[mask],%k[mask] \n"
"sub $0x40,%k[width] \n"
"cmovg %[temp],%[mask] \n"
"bzhi %q[width],%[mask],%[mask] \n"
"kmovq %[mask],%%k5 \n"
"vmovdqu8 128(%[src_raw]),%%zmm2%{%%k5%}%{z%} \n"
"vpermq %%zmm2,%%zmm23,%%zmm3 \n"
"vpermi2q %%zmm2,%%zmm1,%%zmm22 \n"
"vpermi2q %%zmm1,%%zmm0,%%zmm21 \n"
"vpermq %%zmm0,%%zmm20,%%zmm0 \n"
"vpshufb %%zmm4,%%zmm0,%%zmm0 \n"
"vpshufb %%zmm4,%%zmm21,%%zmm1 \n"
"vpshufb %%zmm4,%%zmm22,%%zmm2 \n"
"vpshufb %%zmm4,%%zmm3,%%zmm3 \n"
"vpord %%zmm16,%%zmm0,%%zmm0 \n"
"vpord %%zmm16,%%zmm1,%%zmm1 \n"
"vpord %%zmm16,%%zmm2,%%zmm2 \n"
"vpord %%zmm16,%%zmm3,%%zmm3 \n"
"vmovdqu32 %%zmm0,(%[dst_argb])%{%%k1%} \n"
"vmovdqu32 %%zmm1,0x40(%[dst_argb])%{%%k2%} \n"
"vmovdqu32 %%zmm2,0x80(%[dst_argb])%{%%k3%} \n"
"vmovdqu32 %%zmm3,0xc0(%[dst_argb])%{%%k4%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(src_raw), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
: "m"(*shuffler), // %3
"m"(kPermqRGB24ToARGB0_AVX512BW), // %4
"m"(kPermqRGB24ToARGB1_AVX512BW), // %5
"m"(kPermqRGB24ToARGB2_AVX512BW), // %6
"m"(kPermqRGB24ToARGB3_AVX512BW) // %7
: [src_raw] "+r"(src_raw), // %[src_raw]
[dst_argb] "+r"(dst_argb), // %[dst_argb]
[width] "+r"(width), // %[width]
[temp] "=&r"(temp), // %[temp]
[mask] "=&r"(mask) // %[mask]
: [shuffler] "r"(shuffler), // %[shuffler]
[q0] "m"(kPermqRGB24ToARGB0_AVX512BW), // %[q0]
[q1] "m"(kPermqRGB24ToARGB1_AVX512BW), // %[q1]
[q2] "m"(kPermqRGB24ToARGB2_AVX512BW), // %[q2]
[q3] "m"(kPermqRGB24ToARGB3_AVX512BW) // %[q3]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16",
"xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23");
"xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1", "k2", "k3",
"k4", "k5");
}
void RAWToARGBRow_AVX512BW(const uint8_t* src_raw,
@ -2410,23 +2461,26 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
uint8_t* dst_v,
int width,
const struct ArgbConstants* c) {
uintptr_t temp;
asm volatile(
"vbroadcasti32x4 0x20(%5),%%zmm4 \n" // RGBToU
"vbroadcasti32x4 0x40(%5),%%zmm5 \n" // RGBToV
"vbroadcasti32x4 0x20(%[c]),%%zmm4 \n" // RGBToU
"vbroadcasti32x4 0x40(%[c]),%%zmm5 \n" // RGBToV
"vpternlogd $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
"vpabsb %%zmm16,%%zmm20 \n" // 0x0101
"vpsllw $15,%%zmm16,%%zmm17 \n" // 0x8000
"vbroadcasti32x4 %6,%%zmm21 \n" // kShuffleAARRGGBB
"vmovups %7,%%zmm18 \n" // kPermdARGBToY_AVX512BW
"vmovups %8,%%zmm19 \n" // kPermdARGBToUV_AVX512BW
"vbroadcasti32x4 %[shuf],%%zmm21 \n" // kShuffleAARRGGBB
"vmovups %[permy],%%zmm18 \n" // kPermdARGBToY_AVX512BW
"vmovups %[permuv],%%zmm19 \n" // kPermdARGBToUV_AVX512BW
"sub %1,%2 \n"
"subl $0x20,%3 \n"
"jl 2f \n"
LABELALIGN
"1: \n"
"vmovups (%0),%%zmm0 \n" // Read 32x2 ARGB Pixels
"vmovups 0x40(%0),%%zmm1 \n"
"vmovups 0x00(%0,%4,1),%%zmm2 \n"
"vmovups 0x40(%0,%4,1),%%zmm3 \n"
"vmovups 0x00(%0,%[stride],1),%%zmm2 \n"
"vmovups 0x40(%0,%[stride],1),%%zmm3 \n"
"vpshufb %%zmm21,%%zmm0,%%zmm0 \n" // aarrggbb
"vpshufb %%zmm21,%%zmm1,%%zmm1 \n"
"vpshufb %%zmm21,%%zmm2,%%zmm2 \n"
@ -2463,23 +2517,76 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
"lea 0x80(%0),%0 \n"
"lea 0x10(%1),%1 \n"
"subl $0x20,%3 \n"
"jg 1b \n"
"jge 1b \n"
"2: \n"
"addl $0x20,%3 \n"
"je 99f \n"
"mov $-1,%[temp] \n"
"bzhi %q3,%[temp],%[temp] \n"
"kmovd %k[temp],%%k1 \n"
"kshiftrd $16,%%k1,%%k2 \n"
"knotw %%k1,%%k3 \n"
"knotw %%k2,%%k4 \n"
"vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n"
"vmovdqu32 0x00(%0,%[stride],1),%%zmm2%{%%k1%}%{z%} \n"
"vmovdqu32 0x40(%0,%[stride],1),%%zmm3%{%%k2%}%{z%} \n"
"vpshufd $0xa0,%%zmm0,%%zmm0%{%%k3%} \n"
"vpshufd $0xa0,%%zmm1,%%zmm1%{%%k4%} \n"
"vpshufd $0xa0,%%zmm2,%%zmm2%{%%k3%} \n"
"vpshufd $0xa0,%%zmm3,%%zmm3%{%%k4%} \n"
"vpshufb %%zmm21,%%zmm0,%%zmm0 \n"
"vpshufb %%zmm21,%%zmm1,%%zmm1 \n"
"vpshufb %%zmm21,%%zmm2,%%zmm2 \n"
"vpshufb %%zmm21,%%zmm3,%%zmm3 \n"
"vpmaddubsw %%zmm20,%%zmm0,%%zmm0 \n"
"vpmaddubsw %%zmm20,%%zmm1,%%zmm1 \n"
"vpmaddubsw %%zmm20,%%zmm2,%%zmm2 \n"
"vpmaddubsw %%zmm20,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm0,%%zmm2,%%zmm0 \n"
"vpaddw %%zmm1,%%zmm3,%%zmm1 \n"
"vpxorq %%zmm2,%%zmm2,%%zmm2 \n"
"vpsrlw $1,%%zmm0,%%zmm0 \n"
"vpsrlw $1,%%zmm1,%%zmm1 \n"
"vpavgw %%zmm2,%%zmm0,%%zmm0 \n"
"vpavgw %%zmm2,%%zmm1,%%zmm1 \n"
"vpackuswb %%zmm1,%%zmm0,%%zmm0 \n"
"vpermd %%zmm0,%%zmm19,%%zmm0 \n"
"vpmaddubsw %%zmm4,%%zmm0,%%zmm1 \n"
"vpmaddubsw %%zmm5,%%zmm0,%%zmm0 \n"
"vpmaddwd %%zmm16,%%zmm1,%%zmm1 \n"
"vpmaddwd %%zmm16,%%zmm0,%%zmm0 \n"
"vpackssdw %%zmm0,%%zmm1,%%zmm0 \n"
"vpaddw %%zmm17,%%zmm0,%%zmm0 \n"
"vpsrlw $0x8,%%zmm0,%%zmm0 \n"
"vpackuswb %%zmm0,%%zmm0,%%zmm0 \n"
"vpermd %%zmm0,%%zmm18,%%zmm0 \n"
"addl $1,%3 \n"
"shrl $1,%3 \n"
"mov $-1,%[temp] \n"
"bzhi %q3,%[temp],%[temp] \n"
"kmovw %k[temp],%%k1 \n"
"vmovdqu8 %%xmm0,(%1)%{%%k1%} \n"
"vextracti32x4 $0x1,%%zmm0,%%xmm0 \n"
"vmovdqu8 %%xmm0,0x00(%1,%2,1)%{%%k1%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
#if defined(__i386__)
"+m"(width) // %3
#else
"+rm"(width) // %3
#endif
: "r"((ptrdiff_t)(src_stride_argb)), // %4
"r"(c), // %5
"m"(kShuffleAARRGGBB), // %6
"m"(kPermdARGBToY_AVX512BW), // %7
"m"(kPermdARGBToUV_AVX512BW) // %8
: "+r"(src_argb), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
"+r"(width), // %3
[temp] "=&r"(temp) // %[temp]
: [stride] "r"((ptrdiff_t)(src_stride_argb)), // %[stride]
[c] "r"(c), // %[c]
[shuf] "m"(kShuffleAARRGGBB), // %[shuf]
[permy] "m"(kPermdARGBToY_AVX512BW), // %[permy]
[permuv] "m"(kPermdARGBToUV_AVX512BW) // %[permuv]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm16",
"xmm17", "xmm18", "xmm19", "xmm20", "xmm21");
"xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "k1", "k2", "k3", "k4");
}
#endif // HAS_ARGBTOUVMATRIXROW_AVX512BW
@ -3899,6 +4006,18 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
"vmovups %%zmm0,0x40(%[dst_argb]) \n" \
"lea 0x80(%[dst_argb]), %[dst_argb] \n"
// Store masked ARGB values with AVX512BW.
#define STOREARGB_MASK_AVX512BW \
"vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \
"vpermq %%zmm0,%%zmm18,%%zmm0 \n" \
"vpunpcklbw %%zmm5,%%zmm2,%%zmm2 \n" \
"vpermq %%zmm2,%%zmm18,%%zmm2 \n" \
"vpunpcklwd %%zmm2,%%zmm0,%%zmm1 \n" \
"vpunpckhwd %%zmm2,%%zmm0,%%zmm0 \n" \
"kshiftrd $16,%%k1,%%k2 \n" \
"vmovdqu32 %%zmm1,(%[dst_argb])%{%%k1%} \n" \
"vmovdqu32 %%zmm0,0x40(%[dst_argb])%{%%k2%} \n"
// Store 16 RGB24 values.
#define STORERGB24_AVX2 \
"vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \
@ -4009,6 +4128,27 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
"vmovdqu32 %%zmm3,0x40(%[dst_ar30]) \n" \
"lea 0x80(%[dst_ar30]), %[dst_ar30] \n"
// Store masked AR30 values with AVX512BW.
#define STOREAR30_MASK_AVX512BW \
"vpminsw %%zmm7,%%zmm0,%%zmm0 \n" \
"vpminsw %%zmm7,%%zmm1,%%zmm1 \n" \
"vpminsw %%zmm7,%%zmm2,%%zmm2 \n" \
"vpmaxsw %%zmm6,%%zmm0,%%zmm0 \n" \
"vpmaxsw %%zmm6,%%zmm1,%%zmm1 \n" \
"vpmaxsw %%zmm6,%%zmm2,%%zmm2 \n" \
"vpsrlw $4,%%zmm0,%%zmm0 \n" \
"vpsrlw $10,%%zmm1,%%zmm3 \n" \
"vpsllw $6,%%zmm1,%%zmm1 \n" \
"vpternlogd $0xf8,%%zmm19,%%zmm1,%%zmm0 \n" \
"vpternlogd $0xe4,%%zmm7,%%zmm3,%%zmm2 \n" \
"vpord %%zmm5,%%zmm2,%%zmm2 \n" \
"vmovdqa64 %%zmm0,%%zmm3 \n" \
"vpermt2w %%zmm2,%%zmm20,%%zmm0 \n" \
"vpermt2w %%zmm2,%%zmm21,%%zmm3 \n" \
"kshiftrd $16,%%k1,%%k2 \n" \
"vmovdqu32 %%zmm0,(%[dst_ar30])%{%%k1%} \n" \
"vmovdqu32 %%zmm3,0x40(%[dst_ar30])%{%%k2%} \n"
// Store 16 AR30 values.
#define STOREAR30_AVX2 \
"vpminsw %%ymm7,%%ymm0,%%ymm0 \n" \
@ -4177,10 +4317,13 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
uintptr_t temp;
asm volatile (
YUVTORGB_SETUP_AVX512BW(yuvconstants)
"sub %[u_buf],%[v_buf] \n"
"vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n"
"sub $0x20,%[width] \n"
"jl 2f \n"
LABELALIGN
"1: \n"
@ -4188,20 +4331,31 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf,
YUVTORGB_AVX512BW(yuvconstants)
STOREARGB_AVX512BW
"sub $0x20,%[width] \n"
"jg 1b \n"
"jge 1b \n"
"2: \n"
"add $0x20,%[width] \n"
"je 99f \n"
READYUV422_MASK_AVX512BW
YUVTORGB_AVX512BW(yuvconstants)
STOREARGB_MASK_AVX512BW
"99: \n"
"vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
[dst_argb]"+r"(dst_argb), // %[dst_argb]
[width]"+rm"(width) // %[width]
[width]"+r"(width), // %[width]
[temp]"=&r"(temp) // %[temp]
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
[quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
[dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm]
[unperm]"r"(kUnpermuteAVX512) // %[unperm]
: "memory", "cc", YUVTORGB_REGS_AVX512BW
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"k1", "k2"
);
}
#endif // defined(HAS_I422TOARGBROW_AVX512BW)
@ -4361,6 +4515,7 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf,
uint8_t* dst_ar30,
const struct YuvConstants* yuvconstants,
int width) {
uintptr_t temp;
asm volatile (
YUVTORGB_SETUP_AR30_AVX512BW(yuvconstants)
"sub %[u_buf],%[v_buf] \n"
@ -4372,6 +4527,8 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf,
"vpxord %%zmm6,%%zmm6,%%zmm6 \n" // 0 for min
"vmovdqu16 (%[permar30_0]),%%zmm20 \n"
"vmovdqu16 (%[permar30_1]),%%zmm21 \n"
"sub $0x20,%[width] \n"
"jl 2f \n"
LABELALIGN
"1: \n"
@ -4379,14 +4536,24 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf,
YUVTORGB16_AVX512BW(yuvconstants)
STOREAR30_AVX512BW
"sub $0x20,%[width] \n"
"jg 1b \n"
"jge 1b \n"
"2: \n"
"add $0x20,%[width] \n"
"je 99f \n"
READYUV422_MASK_AVX512BW
YUVTORGB16_AVX512BW(yuvconstants)
STOREAR30_MASK_AVX512BW
"99: \n"
"vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
[dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
[width]"+rm"(width) // %[width]
[width]"+r"(width), // %[width]
[temp]"=&r"(temp) // %[temp]
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
[quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
[dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm]
@ -4395,7 +4562,7 @@ void OMITFP I422ToAR30Row_AVX512BW(const uint8_t* y_buf,
[permar30_1]"r"(kPermAR30_1) // %[permar30_1]
: "memory", "cc", YUVTORGB_REGS_AVX512BW
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
"xmm19", "xmm20", "xmm21"
"xmm19", "xmm20", "xmm21", "k1", "k2"
);
}
#endif // defined(HAS_I422TOAR30ROW_AVX512BW)
@ -5239,6 +5406,14 @@ static const uvec8 kShuffleMirrorSplitUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u,
#endif // defined(HAS_MIRRORSPLITUVROW_AVX2) ||
// defined(HAS_MIRRORSPLITUVROW_AVX512BW)
#if defined(HAS_MIRRORSPLITUVROW_AVX512BW) || defined(HAS_MIRRORUVROW_AVX512BW)
static const uint16_t kMirrorUVPermute_AVX512[32] = {
31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u,
20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u,
9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
#endif // defined(HAS_MIRRORSPLITUVROW_AVX512BW) ||
// defined(HAS_MIRRORUVROW_AVX512BW)
#ifdef HAS_MIRRORSPLITUVROW_AVX512BW
static const uint64_t kMirrorSplitUVPermute[8] = {6, 4, 2, 0, 7, 5, 3, 1};
@ -5247,16 +5422,17 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src,
uint8_t* dst_v,
int width) {
ptrdiff_t temp_width = (ptrdiff_t)(width);
uintptr_t temp;
asm volatile(
"vbroadcasti32x4 %4,%%zmm1 \n"
"lea -0x40(%0,%3,2),%0 \n"
"vbroadcasti32x4 %[shuf],%%zmm1 \n"
"sub %1,%2 \n"
"vmovdqu64 %5,%%zmm3 \n"
"vmovdqu64 %[perm],%%zmm3 \n"
"sub $0x20,%3 \n"
"jl 2f \n"
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0 \n"
"lea -0x40(%0),%0 \n"
"vmovdqu8 (%0,%3,2),%%zmm0 \n"
"vpshufb %%zmm1,%%zmm0,%%zmm0 \n"
"vpermq %%zmm0,%%zmm3,%%zmm0 \n"
"vextracti64x4 $0x1,%%zmm0,%%ymm2 \n"
@ -5264,15 +5440,36 @@ void MirrorSplitUVRow_AVX512BW(const uint8_t* src,
"vmovdqu %%ymm2,0x00(%1,%2,1) \n"
"lea 0x20(%1),%1 \n"
"sub $0x20,%3 \n"
"jg 1b \n"
"jge 1b \n"
"2: \n"
"add $0x20,%3 \n"
"je 99f \n"
"mov $-1,%[temp] \n"
"bzhi %q3,%[temp],%[temp] \n"
"kmovd %k[temp],%%k1 \n"
"vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n"
"mov $0x20,%k[temp] \n"
"sub %k3,%k[temp] \n"
"vpbroadcastw %k[temp],%%zmm2 \n"
"vmovdqu16 %[mirror_uv],%%zmm4 \n"
"vpsubw %%zmm2,%%zmm4,%%zmm2 \n"
"vpermw %%zmm0,%%zmm2,%%zmm0 \n"
"vpmovwb %%zmm0,(%1)%{%%k1%} \n"
"vpsrlw $0x8,%%zmm0,%%zmm0 \n"
"vpmovwb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(src), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
"+r"(temp_width) // %3
: "m"(kShuffleMirrorSplitUV), // %4
"m"(kMirrorSplitUVPermute) // %5
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
: "+r"(src), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
"+r"(temp_width), // %3
[temp] "=&r"(temp) // %[temp]
: [shuf] "m"(kShuffleMirrorSplitUV), // %[shuf]
[perm] "m"(kMirrorSplitUVPermute), // %[perm]
[mirror_uv] "m"(kMirrorUVPermute_AVX512) // %[mirror_uv]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "k1");
}
#endif // HAS_MIRRORSPLITUVROW_AVX512BW
@ -5358,10 +5555,6 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
#endif // HAS_MIRRORUVROW_AVX2
#ifdef HAS_MIRRORUVROW_AVX512BW
static const uint16_t kMirrorUVPermute_AVX512[32] = {
31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u,
20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u,
9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
ptrdiff_t temp_width = (ptrdiff_t)(width);
asm volatile("vmovdqu16 %3,%%zmm5 \n"
@ -7773,7 +7966,12 @@ void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) {
#ifdef HAS_COPYROW_AVX512BW
void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
uintptr_t temp;
asm volatile(
"sub $0x80,%2 \n"
"jl 2f \n"
LABELALIGN
"1: \n"
"vmovups (%0),%%zmm0 \n"
"vmovups 0x40(%0),%%zmm1 \n"
@ -7782,13 +7980,35 @@ void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
"vmovups %%zmm1,0x40(%1) \n"
"lea 0x80(%1),%1 \n"
"sub $0x80,%2 \n"
"jg 1b \n"
"jge 1b \n"
"2: \n"
"add $0x80,%2 \n"
"je 99f \n"
"cmp $0x40,%2 \n"
"jl 3f \n"
"vmovups (%0),%%zmm0 \n"
"lea 0x40(%0),%0 \n"
"vmovups %%zmm0,(%1) \n"
"lea 0x40(%1),%1 \n"
"sub $0x40,%2 \n"
"je 99f \n"
"3: \n"
"mov $-1,%[temp] \n"
"bzhi %q2,%[temp],%[temp] \n"
"kmovq %[temp],%%k1 \n"
"vmovdqu8 (%0),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu8 %%zmm0,(%1)%{%%k1%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width), // %2
[temp] "=&r"(temp) // %[temp]
:
: "memory", "cc", "xmm0", "xmm1");
: "memory", "cc", "xmm0", "xmm1", "k1");
}
#endif // HAS_COPYROW_AVX512BW
@ -8812,17 +9032,20 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0,
const uint8_t* alpha,
uint8_t* dst,
int width) {
uintptr_t temp;
asm volatile(
"vpternlogd $0xff,%%zmm17,%%zmm17,%%zmm17 \n"
"vpsllw $0x8,%%zmm17,%%zmm16 \n" // 0xff00
"vpabsb %%zmm17,%%zmm17 \n" // 0x0101
"vpsllw $7,%%zmm17,%%zmm17 \n" // 0x8080
"mov $0x807f807f,%%eax \n"
"vmovd %%eax,%%xmm18 \n"
"mov $0x807f807f,%k[temp] \n"
"vmovd %k[temp],%%xmm18 \n"
"vbroadcastss %%xmm18,%%zmm18 \n" // 0x807f
"sub %2,%0 \n"
"sub %2,%1 \n"
"sub %2,%3 \n"
"sub $64,%4 \n"
"jl 2f \n"
// 64 pixel loop.
LABELALIGN
@ -8848,16 +9071,45 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0,
"vmovdqu8 %%zmm0,(%3,%2,1) \n"
"add $64,%2 \n"
"sub $64,%4 \n"
"jg 1b \n"
"jge 1b \n"
"2: \n"
"add $64,%4 \n"
"je 99f \n"
"mov $-1,%[temp] \n"
"bzhi %q4,%[temp],%[temp] \n"
"kmovq %[temp],%%k1 \n"
"vmovdqu8 (%2),%%zmm0%{%%k1%}%{z%} \n"
"vpunpckhbw %%zmm0,%%zmm0,%%zmm3 \n"
"vpunpcklbw %%zmm0,%%zmm0,%%zmm0 \n"
"vpxorq %%zmm16,%%zmm3,%%zmm3 \n"
"vpxorq %%zmm16,%%zmm0,%%zmm0 \n"
"vmovdqu8 (%0,%2,1),%%zmm1%{%%k1%}%{z%} \n"
"vmovdqu8 (%1,%2,1),%%zmm2%{%%k1%}%{z%} \n"
"vpunpckhbw %%zmm2,%%zmm1,%%zmm4 \n"
"vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n"
"vpsubb %%zmm17,%%zmm4,%%zmm4 \n"
"vpsubb %%zmm17,%%zmm1,%%zmm1 \n"
"vpmaddubsw %%zmm4,%%zmm3,%%zmm3 \n"
"vpmaddubsw %%zmm1,%%zmm0,%%zmm0 \n"
"vpaddw %%zmm18,%%zmm3,%%zmm3 \n"
"vpaddw %%zmm18,%%zmm0,%%zmm0 \n"
"vpsrlw $0x8,%%zmm3,%%zmm3 \n"
"vpsrlw $0x8,%%zmm0,%%zmm0 \n"
"vpackuswb %%zmm3,%%zmm0,%%zmm0 \n"
"vmovdqu8 %%zmm0,(%3,%2,1)%{%%k1%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(src0), // %0
"+r"(src1), // %1
"+r"(alpha), // %2
"+r"(dst), // %3
"+rm"(width) // %4
: "+r"(src0), // %0
"+r"(src1), // %1
"+r"(alpha), // %2
"+r"(dst), // %3
"+r"(width), // %4
[temp] "=&r"(temp) // %[temp]
::"memory",
"cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", "xmm17",
"xmm18");
"cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", "xmm17", "xmm18",
"k1");
}
#endif // HAS_BLENDPLANEROW_AVX512BW
@ -10624,26 +10876,47 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
uint8_t* dst_argb,
const uint8_t* shuffler,
int width) {
asm volatile("vbroadcasti32x4 (%3),%%zmm5 \n"
uintptr_t temp;
asm volatile(
"vbroadcasti32x4 (%[shuffler]),%%zmm5 \n"
"sub $0x20,%2 \n"
"jl 2f \n"
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0 \n"
"vmovdqu8 0x40(%0),%%zmm1 \n"
"lea 0x80(%0),%0 \n"
"vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
"vpshufb %%zmm5,%%zmm1,%%zmm1 \n"
"vmovdqu8 %%zmm0,(%1) \n"
"vmovdqu8 %%zmm1,0x40(%1) \n"
"lea 0x80(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
: "r"(shuffler) // %3
: "memory", "cc", "xmm0", "xmm1", "xmm5");
LABELALIGN
"1: \n"
"vmovdqu8 (%0),%%zmm0 \n"
"vmovdqu8 0x40(%0),%%zmm1 \n"
"lea 0x80(%0),%0 \n"
"vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
"vpshufb %%zmm5,%%zmm1,%%zmm1 \n"
"vmovdqu8 %%zmm0,(%1) \n"
"vmovdqu8 %%zmm1,0x40(%1) \n"
"lea 0x80(%1),%1 \n"
"sub $0x20,%2 \n"
"jge 1b \n"
"2: \n"
"add $0x20,%2 \n"
"je 99f \n"
"mov $-1,%[temp] \n"
"bzhi %q2,%[temp],%[temp] \n"
"kmovd %k[temp],%%k1 \n"
"kshiftrd $16,%%k1,%%k2 \n"
"vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n"
"vmovdqu32 0x40(%0),%%zmm1%{%%k2%}%{z%} \n"
"vpshufb %%zmm5,%%zmm0,%%zmm0 \n"
"vpshufb %%zmm5,%%zmm1,%%zmm1 \n"
"vmovdqu32 %%zmm0,(%1)%{%%k1%} \n"
"vmovdqu32 %%zmm1,0x40(%1)%{%%k2%} \n"
"99: \n"
"vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width), // %2
[temp] "=&r"(temp) // %[temp]
: [shuffler] "r"(shuffler) // %[shuffler]
: "memory", "cc", "xmm0", "xmm1", "xmm5", "k1", "k2");
}
#endif // HAS_ARGBSHUFFLEROW_AVX512BW

136
source/row_win.cc
View File

@ -428,7 +428,7 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
__m512i zmm_shuf =
_mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)shuffler));
while (width > 0) {
while (width >= 64) {
__m512i zmm_raw0 = _mm512_loadu_si512((const __m512i*)src_raw);
__m512i zmm_raw1 = _mm512_loadu_si512((const __m512i*)(src_raw + 64));
__m512i zmm_raw2 = _mm512_loadu_si512((const __m512i*)(src_raw + 128));
@ -457,6 +457,42 @@ static void RGBToARGBRow_AVX512BW(const uint8_t* src_raw,
dst_argb += 256;
width -= 64;
}
if (width > 0) {
uint64_t dst_mask = (1ULL << width) - 1ULL;
int src_bytes = width * 3;
__mmask64 src_mask0 =
(src_bytes >= 64) ? ~0ULL : ((1ULL << src_bytes) - 1ULL);
__mmask64 src_mask1 =
(src_bytes >= 128)
? ~0ULL
: ((src_bytes > 64) ? ((1ULL << (src_bytes - 64)) - 1ULL) : 0ULL);
__mmask64 src_mask2 =
(src_bytes > 128) ? ((1ULL << (src_bytes - 128)) - 1ULL) : 0ULL;
__m512i zmm_raw0 = _mm512_maskz_loadu_epi8(src_mask0, src_raw);
__m512i zmm_raw1 = _mm512_maskz_loadu_epi8(src_mask1, src_raw + 64);
__m512i zmm_raw2 = _mm512_maskz_loadu_epi8(src_mask2, src_raw + 128);
__m512i zmm0 = _mm512_permutexvar_epi64(zmm_q0, zmm_raw0);
__m512i zmm1 = _mm512_permutex2var_epi64(zmm_raw0, zmm_q1, zmm_raw1);
__m512i zmm2 = _mm512_permutex2var_epi64(zmm_raw1, zmm_q2, zmm_raw2);
__m512i zmm3 = _mm512_permutexvar_epi64(zmm_q3, zmm_raw2);
zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf);
zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf);
zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf);
zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf);
zmm0 = _mm512_or_si512(zmm0, zmm_alpha);
zmm1 = _mm512_or_si512(zmm1, zmm_alpha);
zmm2 = _mm512_or_si512(zmm2, zmm_alpha);
zmm3 = _mm512_or_si512(zmm3, zmm_alpha);
_mm512_mask_storeu_epi32(dst_argb, (__mmask16)dst_mask, zmm0);
_mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(dst_mask >> 16), zmm1);
_mm512_mask_storeu_epi32(dst_argb + 128, (__mmask16)(dst_mask >> 32), zmm2);
_mm512_mask_storeu_epi32(dst_argb + 192, (__mmask16)(dst_mask >> 48), zmm3);
}
_mm256_zeroupper();
}
#endif // defined(HAS_RAWTOARGBROW_AVX512BW) ||
@ -1630,6 +1666,18 @@ void ARGBShuffleRow_AVX512BW(const uint8_t* src_argb,
dst_argb += 128;
width -= 32;
}
if (width > 0) {
uint32_t mask = (1u << width) - 1u;
__mmask16 mask0 = (__mmask16)mask;
__mmask16 mask1 = (__mmask16)(mask >> 16);
__m512i row = _mm512_maskz_loadu_epi32(mask0, src_argb);
__m512i row1 = _mm512_maskz_loadu_epi32(mask1, src_argb + 64);
row = _mm512_shuffle_epi8(row, control);
row1 = _mm512_shuffle_epi8(row1, control);
_mm512_mask_storeu_epi32(dst_argb, mask0, row);
_mm512_mask_storeu_epi32(dst_argb + 64, mask1, row1);
}
_mm256_zeroupper();
}
#endif // HAS_ARGBSHUFFLEROW_AVX512BW
@ -2401,6 +2449,62 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y,
dst_ar30 += 128;
width -= 32;
}
if (width > 0) {
__mmask16 mask_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u);
__mmask32 mask_y = (__mmask32)((1u << width) - 1u);
__m128i xmm_u = _mm_maskz_loadu_epi8(mask_uv, src_u);
__m128i xmm_v = _mm_maskz_loadu_epi8(mask_uv, src_u + offset);
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
__m256i ymm_y = _mm256_maskz_loadu_epi8(mask_y, src_y);
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
zmm0 = _mm512_min_epi16(zmm0, zmm_3ff0);
zmm1 = _mm512_min_epi16(zmm1, zmm_3ff0);
zmm2 = _mm512_min_epi16(zmm2, zmm_3ff0);
zmm0 = _mm512_max_epi16(zmm0, zmm_zero);
zmm1 = _mm512_max_epi16(zmm1, zmm_zero);
zmm2 = _mm512_max_epi16(zmm2, zmm_zero);
zmm0 = _mm512_srli_epi16(zmm0, 4);
zmm3 = _mm512_srli_epi16(zmm1, 10);
zmm1 = _mm512_slli_epi16(zmm1, 6);
zmm0 = _mm512_ternarylogic_epi32(zmm0, zmm1, zmm_fc00, 0xf8);
zmm2 = _mm512_ternarylogic_epi32(zmm2, zmm3, zmm_3ff0, 0xe4);
zmm2 = _mm512_or_si512(zmm2, zmm_c000);
__m512i zmm_dst0 = _mm512_permutex2var_epi16(zmm0, zmm_perm0, zmm2);
__m512i zmm_dst1 = _mm512_permutex2var_epi16(zmm0, zmm_perm1, zmm2);
_mm512_mask_storeu_epi32(dst_ar30, (__mmask16)mask_y, zmm_dst0);
_mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(mask_y >> 16),
zmm_dst1);
}
_mm256_zeroupper();
}
#endif // HAS_I422TOAR30ROW_AVX512BW
@ -2564,7 +2668,7 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0,
__m512i zmm_80 = _mm512_set1_epi8((char)0x80);
__m512i zmm_round = _mm512_set1_epi16((short)0x807f);
while (width > 0) {
while (width >= 64) {
__m512i zmm_a = _mm512_loadu_si512((const __m512i*)alpha);
__m512i zmm_s0 = _mm512_loadu_si512((const __m512i*)src0);
__m512i zmm_s1 = _mm512_loadu_si512((const __m512i*)src1);
@ -2596,6 +2700,34 @@ void BlendPlaneRow_AVX512BW(const uint8_t* src0,
dst += 64;
width -= 64;
}
if (width > 0) {
__mmask64 mask = (1ULL << width) - 1ULL;
__m512i zmm_a = _mm512_maskz_loadu_epi8(mask, alpha);
__m512i zmm_s0 = _mm512_maskz_loadu_epi8(mask, src0);
__m512i zmm_s1 = _mm512_maskz_loadu_epi8(mask, src1);
__m512i zmm_a_lo = _mm512_unpacklo_epi8(zmm_a, zmm_a);
__m512i zmm_a_hi = _mm512_unpackhi_epi8(zmm_a, zmm_a);
zmm_a_lo = _mm512_xor_si512(zmm_a_lo, zmm_ff00);
zmm_a_hi = _mm512_xor_si512(zmm_a_hi, zmm_ff00);
__m512i zmm_s_lo = _mm512_unpacklo_epi8(zmm_s0, zmm_s1);
__m512i zmm_s_hi = _mm512_unpackhi_epi8(zmm_s0, zmm_s1);
zmm_s_lo = _mm512_sub_epi8(zmm_s_lo, zmm_80);
zmm_s_hi = _mm512_sub_epi8(zmm_s_hi, zmm_80);
__m512i zmm_p_lo = _mm512_maddubs_epi16(zmm_a_lo, zmm_s_lo);
__m512i zmm_p_hi = _mm512_maddubs_epi16(zmm_a_hi, zmm_s_hi);
zmm_p_lo = _mm512_add_epi16(zmm_p_lo, zmm_round);
zmm_p_hi = _mm512_add_epi16(zmm_p_hi, zmm_round);
zmm_p_lo = _mm512_srli_epi16(zmm_p_lo, 8);
zmm_p_hi = _mm512_srli_epi16(zmm_p_hi, 8);
__m512i zmm_d = _mm512_packus_epi16(zmm_p_lo, zmm_p_hi);
_mm512_mask_storeu_epi8(dst, mask, zmm_d);
}
_mm256_zeroupper();
}
#endif // HAS_BLENDPLANEROW_AVX512BW