diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 43f0b3273..8b4655207 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -349,6 +349,7 @@ extern "C" { #define HAS_RGB24TOARGBROW_AVX512BW #define HAS_ARGBSHUFFLEROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512VBMI +#define HAS_I422TORGB24ROW_AVX512BW #define HAS_RGBTOYMATRIXROW_AVX512BW #endif #define HAS_ARGBTOYMATRIXROW_AVX2 @@ -399,6 +400,7 @@ extern "C" { #define HAS_RGBTOUV444MATRIXROW_AVX512BW #define HAS_ARGBTOYMATRIXROW_AVX512BW #define HAS_I422TORGB24ROW_AVX512VBMI +#define HAS_I422TORGB24ROW_AVX512BW #define HAS_ARGBTOUVMATRIXROW_AVX512BW #define HAS_J400TOARGBROW_AVX512BW #define HAS_MERGEUVROW_AVX512BW @@ -4788,6 +4790,12 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width); +void I422ToRGB24Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGBRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, diff --git a/source/convert_argb.cc b/source/convert_argb.cc index 8dad827ea..a583dce3e 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -5557,6 +5557,14 @@ int I420ToRGB24Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB24ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW; + if (IS_ALIGNED(width, 32)) { + I422ToRGB24Row = I422ToRGB24Row_AVX512BW; + } + } +#endif #if defined(HAS_I422TORGB24ROW_AVX512VBMI) if (TestCpuFlag(kCpuHasAVX512VBMI)) { I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI; @@ -5770,6 +5778,14 @@ int I422ToRGB24Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB24ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I422ToRGB24Row = I422ToRGB24Row_Any_AVX512BW; + if (IS_ALIGNED(width, 32)) { + I422ToRGB24Row = I422ToRGB24Row_AVX512BW; + } + } +#endif #if defined(HAS_I422TORGB24ROW_AVX512VBMI) if (TestCpuFlag(kCpuHasAVX512VBMI)) { I422ToRGB24Row = I422ToRGB24Row_Any_AVX512VBMI; diff --git a/source/row_any.cc b/source/row_any.cc index 38784e144..1b41f3dad 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -413,6 +413,9 @@ ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 15) #ifdef HAS_I422TORGB24ROW_AVX512VBMI ANY31C(I422ToRGB24Row_Any_AVX512VBMI, I422ToRGB24Row_AVX512VBMI, 1, 0, 3, 31) #endif +#ifdef HAS_I422TORGB24ROW_AVX512BW +ANY31C(I422ToRGB24Row_Any_AVX512BW, I422ToRGB24Row_AVX512BW, 1, 0, 3, 31) +#endif #ifdef HAS_I422TOARGBROW_AVX2 ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15) #endif diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 37212b2a3..b53a7e613 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -3436,6 +3436,53 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vmovups %%zmm0,0x40(%[dst_argb]) \n" \ "lea 0x80(%[dst_argb]), %[dst_argb] \n" +// Store 16 RGB24 values. +#define STORERGB24_AVX2 \ + "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" \ + "vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n" \ + "vmovdqa %%ymm0,%%ymm1 \n" \ + "vpunpcklwd %%ymm2,%%ymm0,%%ymm0 \n" \ + "vpunpckhwd %%ymm2,%%ymm1,%%ymm1 \n" \ + "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" \ + "vpshufb %%ymm6,%%ymm1,%%ymm1 \n" \ + "vpalignr $0xc,%%ymm0,%%ymm1,%%ymm1 \n" \ + "vextracti128 $1,%%ymm0,%%xmm2 \n" \ + "vextracti128 $1,%%ymm1,%%xmm3 \n" \ + "vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n" \ + "vpalignr $8,%%xmm1,%%xmm2,%%xmm2 \n" \ + "vmovdqu %%xmm0,(%[dst_rgb24]) \n" \ + "vmovdqu %%xmm2,0x10(%[dst_rgb24]) \n" \ + "vmovdqu %%xmm3,0x20(%[dst_rgb24]) \n" \ + "lea 0x30(%[dst_rgb24]),%[dst_rgb24] \n" + +// Store 32 RGB24 values with VBMI. +#define STORERGB24_AVX512VBMI \ + "vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" \ + "vmovdqa64 %%zmm0,%%zmm3 \n" \ + "vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" \ + "vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" \ + "vmovdqu8 %%zmm3,(%[dst_rgb24]) \n" \ + "vmovdqu8 %%ymm0,0x40(%[dst_rgb24]) \n" \ + "lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n" + +// Store 32 RGB24 values with AVX512BW. +#define STORERGB24_AVX512BW \ + "vpunpcklbw %%zmm1,%%zmm0,%%zmm0 \n" \ + "vpunpcklbw %%zmm2,%%zmm2,%%zmm2 \n" \ + "vmovdqa64 %%zmm0,%%zmm1 \n" \ + "vpunpcklwd %%zmm2,%%zmm0,%%zmm0 \n" \ + "vpunpckhwd %%zmm2,%%zmm1,%%zmm1 \n" \ + "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" \ + "vpshufb %%zmm6,%%zmm1,%%zmm1 \n" \ + "vpalignr $0xc,%%zmm0,%%zmm1,%%zmm1 \n" \ + "vmovdqa64 %%zmm20,%%zmm3 \n" \ + "vpermi2q %%zmm1,%%zmm0,%%zmm3 \n" \ + "vmovdqa64 %%zmm21,%%zmm4 \n" \ + "vpermi2q %%zmm1,%%zmm0,%%zmm4 \n" \ + "vmovdqu8 %%zmm3,(%[dst_rgb24]) \n" \ + "vmovdqu8 %%ymm4,0x40(%[dst_rgb24]) \n" \ + "lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n" + // Store 32 AR30 values. #define STOREAR30_AVX512BW \ "vpsraw $0x4,%%zmm0,%%zmm0 \n" \ @@ -3578,22 +3625,7 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf, "1: \n" READYUV422_AVX2 YUVTORGB_AVX2(yuvconstants) - "vpunpcklbw %%ymm1,%%ymm0,%%ymm0 \n" - "vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n" - "vmovdqa %%ymm0,%%ymm1 \n" - "vpunpcklwd %%ymm2,%%ymm0,%%ymm0 \n" - "vpunpckhwd %%ymm2,%%ymm1,%%ymm1 \n" - "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" - "vpshufb %%ymm6,%%ymm1,%%ymm1 \n" - "vpalignr $0xc,%%ymm0,%%ymm1,%%ymm1 \n" - "vextracti128 $1,%%ymm0,%%xmm2 \n" - "vextracti128 $1,%%ymm1,%%xmm3 \n" - "vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n" - "vpalignr $8,%%xmm1,%%xmm2,%%xmm2 \n" - "vmovdqu %%xmm0,(%[dst_rgb24]) \n" - "vmovdqu %%xmm2,0x10(%[dst_rgb24]) \n" - "vmovdqu %%xmm3,0x20(%[dst_rgb24]) \n" - "lea 0x30(%[dst_rgb24]),%[dst_rgb24] \n" + STORERGB24_AVX2 "subl $0x10,%[width] \n" "jg 1b \n" "vzeroupper \n" @@ -3702,13 +3734,7 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, "1: \n" READYUV422_AVX512BW YUVTORGB_AVX512BW(yuvconstants) - "vpermt2b %%zmm1,%%zmm20,%%zmm0 \n" // zmm0 = BG - "vmovdqa64 %%zmm0,%%zmm3 \n" // zmm3 = BG copy - "vpermt2b %%zmm2,%%zmm21,%%zmm3 \n" // zmm3 = dst0 - "vpermt2b %%zmm2,%%zmm22,%%zmm0 \n" // zmm0 = dst1 - "vmovdqu8 %%zmm3,(%[dst_rgb24]) \n" - "vmovdqu8 %%ymm0,0x40(%[dst_rgb24]) \n" - "lea 0x60(%[dst_rgb24]),%[dst_rgb24] \n" + STORERGB24_AVX512VBMI "sub $0x20,%[width] \n" "jg 1b \n" "vzeroupper \n" @@ -3726,12 +3752,60 @@ void OMITFP I422ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, [kMaskDST1]"m"(kMaskDST1) // %[kMaskDST1] : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm20", "xmm21", "xmm22", "xmm20", "xmm21", "xmm22" ); } #endif // HAS_I422TORGB24ROW_AVX512VBMI +#if defined(HAS_I422TORGB24ROW_AVX512BW) +static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12}; +static const uint64_t kStitchRGB24_1[8] = {13, 6, 14, 15, 0, 0, 0, 0}; + +// 32 pixels +// 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96 +// bytes). +void OMITFP I422ToRGB24Row_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile ( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vbroadcasti32x4 %[kShuffleMaskARGBToRGB24_1],%%zmm5 \n" + "vbroadcasti32x4 %[kShuffleMaskARGBToRGB24_0],%%zmm6 \n" + "vmovdqu64 %[kStitchRGB24_0],%%zmm20 \n" + "vmovdqu64 %[kStitchRGB24_1],%%zmm21 \n" + "sub %[u_buf],%[v_buf] \n" + + LABELALIGN + "1: \n" + READYUV422_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_AVX512BW + "sub $0x20,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf]"+r"(y_buf), // %[y_buf] + [u_buf]"+r"(u_buf), // %[u_buf] + [v_buf]"+r"(v_buf), // %[v_buf] + [dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24] + [width]"+rm"(width) // %[width] + : [yuvconstants]"r"(yuvconstants), // %[yuvconstants] + [quadsplitperm]"r"(kSplitQuadWords), // %[quadsplitperm] + [dquadsplitperm]"r"(kSplitDoubleQuadWords), // %[dquadsplitperm] + [unperm]"r"(kUnpermuteAVX512), // %[unperm] + [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24[0]), + [kShuffleMaskARGBToRGB24_1]"m"(kShuffleMaskARGBToRGB24[1]), + [kStitchRGB24_0]"m"(kStitchRGB24_0), // %[kStitchRGB24_0] + [kStitchRGB24_1]"m"(kStitchRGB24_1) // %[kStitchRGB24_1] + : "memory", "cc", YUVTORGB_REGS_AVX512BW + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm20", "xmm21" + ); +} +#endif // HAS_I422TORGB24ROW_AVX512BW + #if defined(HAS_I422TOAR30ROW_AVX512BW) // 32 pixels // 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 AR30 (128 diff --git a/source/row_win.cc b/source/row_win.cc index 1f2c7e672..b6becd77e 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -1290,7 +1290,7 @@ void NV21ToARGBRow_AVX2(const uint8_t* src_y, } #endif -#ifdef HAS_I422TORGB24ROW_AVX2 +#if defined(HAS_I422TORGB24ROW_AVX2) || defined(HAS_I422TORGB24ROW_AVX512BW) static const uint8_t kShuffleMaskARGBToRGB24[2][16] = { {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u}, @@ -1379,9 +1379,11 @@ void I422ToRGB24Row_AVX2(const uint8_t* src_y, __m128i xmm2_store = _mm256_extractf128_si256(ymm0_packed, 1); __m128i xmm3_store = _mm256_extractf128_si256(ymm1_packed, 1); - _mm_storel_epi64((__m128i*)dst_rgb24, xmm0_store); - _mm_storeu_si128((__m128i*)(dst_rgb24 + 8), xmm1_store); - _mm_storel_epi64((__m128i*)(dst_rgb24 + 24), xmm2_store); + xmm0_store = _mm_unpacklo_epi64(xmm0_store, xmm1_store); + xmm2_store = _mm_alignr_epi8(xmm2_store, xmm1_store, 8); + + _mm_storeu_si128((__m128i*)dst_rgb24, xmm0_store); + _mm_storeu_si128((__m128i*)(dst_rgb24 + 16), xmm2_store); _mm_storeu_si128((__m128i*)(dst_rgb24 + 32), xmm3_store); dst_rgb24 += 48; @@ -1501,6 +1503,105 @@ void I422ToRGB24Row_AVX512VBMI(const uint8_t* src_y, } #endif +#ifdef HAS_I422TORGB24ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I422ToRGB24Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + static const uint64_t kStitchRGB24_0[8] = {0, 8, 9, 2, 10, 11, 4, 12}; + static const uint64_t kStitchRGB24_1[8] = {13, 6, 14, 15, 0, 0, 0, 0}; + static const uint64_t kSplitQuadWords[8] = {0, 2, 2, 2, 1, 2, 2, 2}; + static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; + + // Constants + __m512i zmm_kUVToB = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToB)); + __m512i zmm_kUVToG = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToG)); + __m512i zmm_kUVToR = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kUVToR)); + __m512i zmm_kYToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYToRgb)); + __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)yuvconstants->kYBiasToRgb)); + __m512i zmm_128 = _mm512_set1_epi8((char)0x80); + + __m512i zmm_shuf0 = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[1])); + __m512i zmm_shuf1 = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleMaskARGBToRGB24[0])); + __m512i zmm_stitch0 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_0); + __m512i zmm_stitch1 = _mm512_loadu_si512((const __m512i*)kStitchRGB24_1); + __m512i zmm_split = _mm512_loadu_si512((const __m512i*)kSplitQuadWords); + __m512i zmm_split_y = _mm512_loadu_si512((const __m512i*)kSplitDoubleQuadWords); + + ptrdiff_t offset = src_v - src_u; + + while (width >= 32) { + // READYUV422_AVX512BW + __m128i xmm_u = _mm_loadu_si128((const __m128i*)src_u); + __m128i xmm_v = _mm_loadu_si128((const __m128i*)(src_u + offset)); + src_u += 16; + + __m512i zmm_u_val = _mm512_castsi128_si512(xmm_u); + __m512i zmm_v_val = _mm512_castsi128_si512(xmm_v); + + zmm_u_val = _mm512_permutexvar_epi64(zmm_split, zmm_u_val); + zmm_v_val = _mm512_permutexvar_epi64(zmm_split, zmm_v_val); + + __m512i zmm3 = _mm512_unpacklo_epi8(zmm_u_val, zmm_v_val); + zmm3 = _mm512_permutex_epi64(zmm3, 0xd8); + zmm3 = _mm512_unpacklo_epi16(zmm3, zmm3); + + __m256i ymm_y = _mm256_loadu_si256((const __m256i*)src_y); + src_y += 32; + __m512i zmm4 = _mm512_castsi256_si512(ymm_y); + zmm4 = _mm512_permutexvar_epi64(zmm_split_y, zmm4); + zmm4 = _mm512_permutex_epi64(zmm4, 0xd8); + zmm4 = _mm512_unpacklo_epi8(zmm4, zmm4); + + // YUVTORGB_AVX512BW + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); + zmm4 = _mm512_mulhi_epu16(zmm4, zmm_kYToRgb); + + __m512i zmm0 = _mm512_maddubs_epi16(zmm_kUVToB, zmm3); + __m512i zmm1 = _mm512_maddubs_epi16(zmm_kUVToG, zmm3); + __m512i zmm2 = _mm512_maddubs_epi16(zmm_kUVToR, zmm3); + + zmm4 = _mm512_add_epi16(zmm4, zmm_kYBiasToRgb); + + zmm0 = _mm512_adds_epi16(zmm0, zmm4); + zmm1 = _mm512_subs_epi16(zmm4, zmm1); + zmm2 = _mm512_adds_epi16(zmm2, zmm4); + + zmm0 = _mm512_srai_epi16(zmm0, 6); + zmm1 = _mm512_srai_epi16(zmm1, 6); + zmm2 = _mm512_srai_epi16(zmm2, 6); + + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm1 = _mm512_packus_epi16(zmm1, zmm1); + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + + // STORERGB24_AVX512BW + __m512i zmm_bg = _mm512_unpacklo_epi8(zmm0, zmm1); + __m512i zmm_rr = _mm512_unpacklo_epi8(zmm2, zmm2); + __m512i zmm_lo = _mm512_unpacklo_epi16(zmm_bg, zmm_rr); + __m512i zmm_hi = _mm512_unpackhi_epi16(zmm_bg, zmm_rr); + zmm_lo = _mm512_shuffle_epi8(zmm_lo, zmm_shuf0); + zmm_hi = _mm512_shuffle_epi8(zmm_hi, zmm_shuf1); + zmm_hi = _mm512_alignr_epi8(zmm_hi, zmm_lo, 12); + + __m512i zmm_dst0 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch0, zmm_hi); + __m512i zmm_dst1 = _mm512_permutex2var_epi64(zmm_lo, zmm_stitch1, zmm_hi); + + _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); + _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), _mm512_castsi512_si256(zmm_dst1)); + + dst_rgb24 += 96; + width -= 32; + } + _mm256_zeroupper(); +} +#endif + #ifdef __cplusplus } // extern "C" } // namespace libyuv