mirror of
https://chromium.googlesource.com/libyuv/libyuv
synced 2026-10-28 18:07:12 +08:00
STORERGB24 macros for avx512
- improves I420ToRAW performance
I420ToRAW on Skylake
Was AVX2
I420ToRAW_Opt (262 ms)
Now AVX512
I420ToRAW_Opt (212 ms)
I420ToRAW on Icelake with avx512vbmi
Was I420ToRAW_Opt (200 ms)
Now I420ToRAW_Opt (187 ms)
Bug: None
Change-Id: If5b171801ba931557d379696fc128d9b155f7c44
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8267746
Commit-Queue: Frank Barchard <fbarchard@google.com>
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard
libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent
26e56be0f9
commit
0f36c7f808
@ -349,6 +349,7 @@ extern "C" {
|
||||
#define HAS_RGB24TOARGBROW_AVX512BW
|
||||
#define HAS_ARGBSHUFFLEROW_AVX512BW
|
||||
#define HAS_I422TORGB24ROW_AVX512VBMI
|
||||
#define HAS_I422TORGB24ROW_AVX512BW
|
||||
#define HAS_RGBTOYMATRIXROW_AVX512BW
|
||||
#endif
|
||||
#define HAS_ARGBTOYMATRIXROW_AVX2
|
||||
@ -399,6 +400,7 @@ extern "C" {
|
||||
#define HAS_RGBTOUV444MATRIXROW_AVX512BW
|
||||
#define HAS_ARGBTOYMATRIXROW_AVX512BW
|
||||
#define HAS_I422TORGB24ROW_AVX512VBMI
|
||||
#define HAS_I422TORGB24ROW_AVX512BW
|
||||
#define HAS_ARGBTOUVMATRIXROW_AVX512BW
|
||||
#define HAS_J400TOARGBROW_AVX512BW
|
||||
#define HAS_MERGEUVROW_AVX512BW
|
||||
@ -4788,6 +4790,12 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width);
|
||||
void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width);
|
||||
void I422ToARGBRow_Any_AVX2(const uint8_t* y_buf,
|
||||
const uint8_t* u_buf,
|
||||
const uint8_t* v_buf,
|
||||
|
||||
@ -5557,6 +5557,14 @@ int I420ToRGB24Matrix(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI;
|
||||
@ -5770,6 +5778,14 @@ int I422ToRGB24Matrix(const uint8_t* src_y,
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
if (TestCpuFlag(kCpuHasAVX512BW)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW;
|
||||
if (IS_ALIGNED(width, 32)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_AVX512BW;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512VBMI)
|
||||
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
|
||||
I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI;
|
||||
|
||||
@ -413,6 +413,9 @@ ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15)
|
||||
#ifdef HAS_I422TORGB24ROW_AVX512VBMI
|
||||
ANY31C(I422ToRGB24Row_Any_AVX512VBMI, I422ToRGB24Row_AVX512VBMI, 1, 0, 3, 31)
|
||||
#endif
|
||||
#ifdef HAS_I422TORGB24ROW_AVX512BW
|
||||
ANY31C(I422ToRGB24Row_Any_AVX512BW, I422ToRGB24Row_AVX512BW, 1, 0, 3, 31)
|
||||
#endif
|
||||
#ifdef HAS_I422TOARGBROW_AVX2
|
||||
ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15)
|
||||
#endif
|
||||
|
||||
@ -3436,6 +3436,53 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf,
|
||||
"vmovups %%zmm0,0x40(%[dst_argb]) \n" \
|
||||
"lea 0x80(%[dst_argb]), %[dst_argb] \n"
|
||||
|
||||
// Store 16 RGB24 values.
|
||||
#define STORERGB24_AVX2 \
|
||||
"vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \
|
||||
"vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n" \
|
||||
"vmovdqa %%ymm0,%%ymm1 \n" \
|
||||
"vpunpcklwd %%ymm2,%%ymm0,%%ymm0 \n" \
|
||||
"vpunpckhwd %%ymm2,%%ymm1,%%ymm1 \n" \
|
||||
"vpshufb %%ymm5,%%ymm0,%%ymm0 \n" \
|
||||
"vpshufb %%ymm6,%%ymm1,%%ymm1 \n" \
|
||||
"vpalignr $0xc,%%ymm0,%%ymm1,%%ymm1 \n" \
|
||||
"vextracti128 $1,%%ymm0,%%xmm2 \n" \
|
||||
"vextracti128 $1,%%ymm1,%%xmm3 \n" \
|
||||
"vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n" \
|
||||
"vpalignr $8,%%xmm1,%%xmm2,%%xmm2 \n" \
|
||||
"vmovdqu %%xmm0,(%[dst_rgb24]) \n" \
|
||||
"vmovdqu %%xmm2,0x10(%[dst_rgb24]) \n" \
|
||||
"vmovdqu %%xmm3,0x20(%[dst_rgb24]) \n" \
|
||||
"lea 0x30(%[dst_rgb24]),%[dst_rgb24] \n"
|
||||
|
||||
// Store 32 RGB24 values with VBMI.
|
||||
#define STORERGB24_AVX512VBMI \
|
||||
"vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" \
|
||||
"vmovdqa64 %%zmm0,%%zmm3 \n" \
|
||||
"vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" \
|
||||
"vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" \
|
||||
"vmovdqu8 %%zmm3,(%[dst_rgb24]) \n" \
|
||||
"vmovdqu8 %%ymm0,0x40(%[dst_rgb24]) \n" \
|
||||
"lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n"
|
||||
|
||||
// Store 32 RGB24 values with AVX512BW.
|
||||
#define STORERGB24_AVX512BW \
|
||||
"vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \
|
||||
"vpunpcklbw %%zmm2,%%zmm2,%%zmm2 \n" \
|
||||
"vmovdqa64 %%zmm0,%%zmm1 \n" \
|
||||
"vpunpcklwd %%zmm2,%%zmm0,%%zmm0 \n" \
|
||||
"vpunpckhwd %%zmm2,%%zmm1,%%zmm1 \n" \
|
||||
"vpshufb %%zmm5,%%zmm0,%%zmm0 \n" \
|
||||
"vpshufb %%zmm6,%%zmm1,%%zmm1 \n" \
|
||||
"vpalignr $0xc,%%zmm0,%%zmm1,%%zmm1 \n" \
|
||||
"vmovdqa64 %%zmm20,%%zmm3 \n" \
|
||||
"vpermi2q %%zmm1,%%zmm0,%%zmm3 \n" \
|
||||
"vmovdqa64 %%zmm21,%%zmm4 \n" \
|
||||
"vpermi2q %%zmm1,%%zmm0,%%zmm4 \n" \
|
||||
"vmovdqu8 %%zmm3,(%[dst_rgb24]) \n" \
|
||||
"vmovdqu8 %%ymm4,0x40(%[dst_rgb24]) \n" \
|
||||
"lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n"
|
||||
|
||||
// Store 32 AR30 values.
|
||||
#define STOREAR30_AVX512BW \
|
||||
"vpsraw $0x4,%%zmm0,%%zmm0 \n" \
|
||||
@ -3578,22 +3625,7 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf,
|
||||
"1: \n"
|
||||
READYUV422_AVX2
|
||||
YUVTORGB_AVX2(yuvconstants)
|
||||
"vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n"
|
||||
"vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n"
|
||||
"vmovdqa %%ymm0,%%ymm1 \n"
|
||||
"vpunpcklwd %%ymm2,%%ymm0,%%ymm0 \n"
|
||||
"vpunpckhwd %%ymm2,%%ymm1,%%ymm1 \n"
|
||||
"vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
|
||||
"vpshufb %%ymm6,%%ymm1,%%ymm1 \n"
|
||||
"vpalignr $0xc,%%ymm0,%%ymm1,%%ymm1 \n"
|
||||
"vextracti128 $1,%%ymm0,%%xmm2 \n"
|
||||
"vextracti128 $1,%%ymm1,%%xmm3 \n"
|
||||
"vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n"
|
||||
"vpalignr $8,%%xmm1,%%xmm2,%%xmm2 \n"
|
||||
"vmovdqu %%xmm0,(%[dst_rgb24]) \n"
|
||||
"vmovdqu %%xmm2,0x10(%[dst_rgb24]) \n"
|
||||
"vmovdqu %%xmm3,0x20(%[dst_rgb24]) \n"
|
||||
"lea 0x30(%[dst_rgb24]),%[dst_rgb24] \n"
|
||||
STORERGB24_AVX2
|
||||
"subl $0x10,%[width] \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
@ -3702,13 +3734,7 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
|
||||
"1: \n"
|
||||
READYUV422_AVX512BW
|
||||
YUVTORGB_AVX512BW(yuvconstants)
|
||||
"vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" // zmm0 = BG
|
||||
"vmovdqa64 %%zmm0,%%zmm3 \n" // zmm3 = BG copy
|
||||
"vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" // zmm3 = dst0
|
||||
"vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" // zmm0 = dst1
|
||||
"vmovdqu8 %%zmm3,(%[dst_rgb24]) \n"
|
||||
"vmovdqu8 %%ymm0,0x40(%[dst_rgb24]) \n"
|
||||
"lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n"
|
||||
STORERGB24_AVX512VBMI
|
||||
"sub $0x20,%[width] \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
@ -3726,12 +3752,60 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf,
|
||||
[kMaskDST1]"m"(kMaskDST1) // %[kMaskDST1]
|
||||
: "memory", "cc", YUVTORGB_REGS_AVX512BW
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
|
||||
"xmm20", "xmm21", "xmm22",
|
||||
"xmm20", "xmm21", "xmm22"
|
||||
);
|
||||
}
|
||||
#endif // HAS_I422TORGB24ROW_AVX512VBMI
|
||||
|
||||
#if defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12};
|
||||
static const uint64_t kStitchRGB24_1[8] = {13, 6, 14, 15, 0, 0, 0, 0};
|
||||
|
||||
// 32 pixels
|
||||
// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96
|
||||
// bytes).
|
||||
void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf,
|
||||
const uint8_t* u_buf,
|
||||
const uint8_t* v_buf,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
asm volatile (
|
||||
YUVTORGB_SETUP_AVX512BW(yuvconstants)
|
||||
"vbroadcasti32x4 %[kShuffleMaskARGBToRGB24_1],%%zmm5 \n"
|
||||
"vbroadcasti32x4 %[kShuffleMaskARGBToRGB24_0],%%zmm6 \n"
|
||||
"vmovdqu64 %[kStitchRGB24_0],%%zmm20 \n"
|
||||
"vmovdqu64 %[kStitchRGB24_1],%%zmm21 \n"
|
||||
"sub %[u_buf],%[v_buf] \n"
|
||||
|
||||
LABELALIGN
|
||||
"1: \n"
|
||||
READYUV422_AVX512BW
|
||||
YUVTORGB_AVX512BW(yuvconstants)
|
||||
STORERGB24_AVX512BW
|
||||
"sub $0x20,%[width] \n"
|
||||
"jg 1b \n"
|
||||
"vzeroupper \n"
|
||||
: [y_buf]"+r"(y_buf), // %[y_buf]
|
||||
[u_buf]"+r"(u_buf), // %[u_buf]
|
||||
[v_buf]"+r"(v_buf), // %[v_buf]
|
||||
[dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24]
|
||||
[width]"+rm"(width) // %[width]
|
||||
: [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
|
||||
[quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm]
|
||||
[dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm]
|
||||
[unperm]"r"(kUnpermuteAVX512), // %[unperm]
|
||||
[kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]),
|
||||
[kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1]),
|
||||
[kStitchRGB24_0]"m"(kStitchRGB24_0), // %[kStitchRGB24_0]
|
||||
[kStitchRGB24_1]"m"(kStitchRGB24_1) // %[kStitchRGB24_1]
|
||||
: "memory", "cc", YUVTORGB_REGS_AVX512BW
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
|
||||
"xmm20", "xmm21"
|
||||
);
|
||||
}
|
||||
#endif // HAS_I422TORGB24ROW_AVX512BW
|
||||
|
||||
#if defined(HAS_I422TOAR30ROW_AVX512BW)
|
||||
// 32 pixels
|
||||
// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 AR30 (128
|
||||
|
||||
@ -1290,7 +1290,7 @@ void NV21ToARGBRow_AVX2(const uint8_t* src_y,
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAS_I422TORGB24ROW_AVX2
|
||||
#if defined(HAS_I422TORGB24ROW_AVX2) || defined(HAS_I422TORGB24ROW_AVX512BW)
|
||||
static const uint8_t kShuffleMaskARGBToRGB24[2][16] = {
|
||||
{0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u,
|
||||
128u},
|
||||
@ -1379,9 +1379,11 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y,
|
||||
__m128i xmm2_store = _mm256_extractf128_si256(ymm0_packed, 1);
|
||||
__m128i xmm3_store = _mm256_extractf128_si256(ymm1_packed, 1);
|
||||
|
||||
_mm_storel_epi64((__m128i*)dst_rgb24, xmm0_store);
|
||||
_mm_storeu_si128((__m128i*)(dst_rgb24 + 8), xmm1_store);
|
||||
_mm_storel_epi64((__m128i*)(dst_rgb24 + 24), xmm2_store);
|
||||
xmm0_store = _mm_unpacklo_epi64(xmm0_store, xmm1_store);
|
||||
xmm2_store = _mm_alignr_epi8(xmm2_store, xmm1_store, 8);
|
||||
|
||||
_mm_storeu_si128((__m128i*)dst_rgb24, xmm0_store);
|
||||
_mm_storeu_si128((__m128i*)(dst_rgb24 + 16), xmm2_store);
|
||||
_mm_storeu_si128((__m128i*)(dst_rgb24 + 32), xmm3_store);
|
||||
|
||||
dst_rgb24 += 48;
|
||||
@ -1501,6 +1503,105 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y,
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAS_I422TORGB24ROW_AVX512BW
|
||||
LIBYUV_TARGET_AVX512BW
|
||||
void I422ToRGB24Row_AVX512BW(const uint8_t* src_y,
|
||||
const uint8_t* src_u,
|
||||
const uint8_t* src_v,
|
||||
uint8_t* dst_rgb24,
|
||||
const struct YuvConstants* yuvconstants,
|
||||
int width) {
|
||||
static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12};
|
||||
static const uint64_t kStitchRGB24_1[8] = {13, 6, 14, 15, 0, 0, 0, 0};
|
||||
static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2};
|
||||
static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4};
|
||||
|
||||
// Constants
|
||||
__m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB));
|
||||
__m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG));
|
||||
__m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR));
|
||||
__m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb));
|
||||
__m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb));
|
||||
__m512i zmm_128 = _mm512_set1_epi8((char)0x80);
|
||||
|
||||
__m512i zmm_shuf0 = _mm512_broadcast_i32x4(
|
||||
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1]));
|
||||
__m512i zmm_shuf1 = _mm512_broadcast_i32x4(
|
||||
_mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0]));
|
||||
__m512i zmm_stitch0 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_0);
|
||||
__m512i zmm_stitch1 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_1);
|
||||
__m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords);
|
||||
__m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords);
|
||||
|
||||
ptrdiff_t offset = src_v - src_u;
|
||||
|
||||
while (width >= 32) {
|
||||
// READYUV422_AVX512BW
|
||||
__m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u);
|
||||
__m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset));
|
||||
src_u += 16;
|
||||
|
||||
__m512i zmm_u_val = _mm512_castsi128_si512(xmm_u);
|
||||
__m512i zmm_v_val = _mm512_castsi128_si512(xmm_v);
|
||||
|
||||
zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val);
|
||||
zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val);
|
||||
|
||||
__m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val);
|
||||
zmm3 = _mm512_permutex_epi64(zmm3, 0xd8);
|
||||
zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3);
|
||||
|
||||
__m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y);
|
||||
src_y += 32;
|
||||
__m512i zmm4 = _mm512_castsi256_si512(ymm_y);
|
||||
zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4);
|
||||
zmm4 = _mm512_permutex_epi64(zmm4, 0xd8);
|
||||
zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4);
|
||||
|
||||
// YUVTORGB_AVX512BW
|
||||
zmm3 = _mm512_sub_epi8(zmm3, zmm_128);
|
||||
zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb);
|
||||
|
||||
__m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3);
|
||||
__m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3);
|
||||
__m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3);
|
||||
|
||||
zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb);
|
||||
|
||||
zmm0 = _mm512_adds_epi16(zmm0, zmm4);
|
||||
zmm1 = _mm512_subs_epi16(zmm4, zmm1);
|
||||
zmm2 = _mm512_adds_epi16(zmm2, zmm4);
|
||||
|
||||
zmm0 = _mm512_srai_epi16(zmm0, 6);
|
||||
zmm1 = _mm512_srai_epi16(zmm1, 6);
|
||||
zmm2 = _mm512_srai_epi16(zmm2, 6);
|
||||
|
||||
zmm0 = _mm512_packus_epi16(zmm0, zmm0);
|
||||
zmm1 = _mm512_packus_epi16(zmm1, zmm1);
|
||||
zmm2 = _mm512_packus_epi16(zmm2, zmm2);
|
||||
|
||||
// STORERGB24_AVX512BW
|
||||
__m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1);
|
||||
__m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2);
|
||||
__m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr);
|
||||
__m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr);
|
||||
zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0);
|
||||
zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1);
|
||||
zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12);
|
||||
|
||||
__m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi);
|
||||
__m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi);
|
||||
|
||||
_mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0);
|
||||
_mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1));
|
||||
|
||||
dst_rgb24 += 96;
|
||||
width -= 32;
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
} // namespace libyuv
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user