diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 0202c4351..c05059207 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -314,8 +314,8 @@ extern "C" { #define HAS_COMPUTECUMULATIVESUMROW_AVX2 #define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 #define HAS_I422TOAR30ROW_AVX2 -#define HAS_RAWTORGB24ROW_AVX2 #endif +#define HAS_RAWTORGB24ROW_AVX2 #define HAS_I422TOUYVYROW_AVX2 #define HAS_I422TOYUY2ROW_AVX2 #define HAS_MERGEAR64ROW_AVX2 @@ -354,6 +354,8 @@ extern "C" { defined(_M_IX86)) && \ ((defined(_MSC_VER) && !defined(__clang__)) || \ defined(LIBYUV_ENABLE_ROWWIN)) +#define HAS_RAWTORGB24ROW_SSSE3 +#define HAS_RAWTORGB24ROW_AVX2 #define HAS_RAWTOARGBROW_AVX2 #define HAS_I422TORGB24ROW_AVX2 #define HAS_RGB24TOARGBROW_AVX2 @@ -372,7 +374,6 @@ extern "C" { #define HAS_CONVERT8TO16ROW_AVX2 #if defined(__x86_64__) || defined(_M_X64) #define HAS_I422TOAR30ROW_AVX2 -#define HAS_RAWTORGB24ROW_AVX2 #define HAS_RAWTOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW #define HAS_RAWTORGB24ROW_AVX512BW @@ -851,6 +852,7 @@ extern "C" { #define HAS_NV12TORGB565ROW_LASX #define HAS_NV21TOARGBROW_LASX #define HAS_RAWTOARGBROW_LASX +#define HAS_RAWTORGB24ROW_LASX #define HAS_RAWTOUVROW_LASX #define HAS_RGB24TOARGBROW_LASX #define HAS_RGB24TOUVROW_LASX @@ -4027,6 +4029,7 @@ void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); +void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RGB565ToARGBRow_NEON(const uint8_t* src_rgb565, uint8_t* dst_argb, @@ -4136,6 +4139,9 @@ void RAWToRGB24Row_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void RAWToRGB24Row_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void RAWToRGB24Row_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void RGB565ToARGBRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 1c9b08f48..edc72d70e 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -3625,7 +3625,7 @@ int RAWToRGB24(const uint8_t* src_raw, #if defined(HAS_RAWTORGB24ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { RAWToRGB24Row = RAWToRGB24Row_Any_NEON; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 16)) { RAWToRGB24Row = RAWToRGB24Row_NEON; } } @@ -3643,6 +3643,14 @@ int RAWToRGB24(const uint8_t* src_raw, } } #endif +#if defined(HAS_RAWTORGB24ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + RAWToRGB24Row = RAWToRGB24Row_Any_LASX; + if (IS_ALIGNED(width, 32)) { + RAWToRGB24Row = RAWToRGB24Row_LASX; + } + } +#endif #if defined(HAS_RAWTORGB24ROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { RAWToRGB24Row = RAWToRGB24Row_RVV; diff --git a/source/row_any.cc b/source/row_any.cc index f1c690ac5..c7704776f 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1125,11 +1125,14 @@ ANY11(ARGBToARGB4444Row_Any_LASX, ARGBToARGB4444Row_LASX, 0, 4, 2, 15) ANY11(J400ToARGBRow_Any_LSX, J400ToARGBRow_LSX, 0, 1, 4, 15) #endif #if defined(HAS_RAWTORGB24ROW_NEON) -ANY11(RAWToRGB24Row_Any_NEON, RAWToRGB24Row_NEON, 0, 3, 3, 7) +ANY11(RAWToRGB24Row_Any_NEON, RAWToRGB24Row_NEON, 0, 3, 3, 15) #endif #if defined(HAS_RAWTORGB24ROW_LSX) ANY11(RAWToRGB24Row_Any_LSX, RAWToRGB24Row_LSX, 0, 3, 3, 15) #endif +#if defined(HAS_RAWTORGB24ROW_LASX) +ANY11(RAWToRGB24Row_Any_LASX, RAWToRGB24Row_LASX, 0, 3, 3, 31) +#endif #ifdef HAS_UYVYTOYROW_AVX2 ANY11(UYVYToYRow_Any_AVX2, UYVYToYRow_AVX2, 0, 2, 1, 31) diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 75482806d..fd56f4959 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -85,11 +85,9 @@ static const uvec8 kShuffleMaskRAWToRGB24_AVX2[2] = { #endif // defined(HAS_RAWTORGB24ROW_AVX2) || // defined(HAS_RAWTORGB24ROW_AVX512BW) #if defined(HAS_RAWTORGB24ROW_AVX2) -static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 0, 0}; -static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 0, 0, 0, 0}; -static const uint32_t kPermd2_AVX2[8] = {0, 0, 0, 0, 0, 1, 2, 4}; -static const uint32_t kPermd3_AVX2[8] = {5, 6, 0, 0, 0, 0, 0, 0}; -static const uint32_t kPermd4_AVX2[8] = {0, 0, 0, 1, 2, 4, 5, 6}; +static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 3, 7}; +static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 3, 7, 0, 1}; +static const uint32_t kPermd2_AVX2[8] = {5, 6, 3, 7, 0, 1, 2, 4}; #endif // defined(HAS_RAWTORGB24ROW_AVX2) #if defined(HAS_RAWTORGB24ROW_AVX512BW) || \ defined(HAS_ARGBTORGB24ROW_AVX512BW) || \ @@ -598,12 +596,10 @@ void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, #ifdef HAS_RAWTORGB24ROW_AVX2 void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { asm volatile( - "vmovdqu %3,%%ymm7 \n" // shuffler (32 bytes) - "vmovdqu %4,%%ymm8 \n" // kPermd0_AVX2 - "vmovdqu %5,%%ymm9 \n" // kPermd1_AVX2 - "vmovdqu %6,%%ymm10 \n" // kPermd2_AVX2 - "vmovdqu %7,%%ymm11 \n" // kPermd3_AVX2 - "vmovdqu %8,%%ymm12 \n" // kPermd4_AVX2 + "vmovdqu %3,%%ymm4 \n" // shuffler (32 bytes) + "vmovdqu %4,%%ymm5 \n" // kPermd0_AVX2 + "vmovdqu %5,%%ymm6 \n" // kPermd1_AVX2 + "vmovdqu %6,%%ymm7 \n" // kPermd2_AVX2 LABELALIGN "1: \n" @@ -612,35 +608,35 @@ void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { "vmovdqu 64(%0),%%ymm2 \n" // raw2 "lea 96(%0),%0 \n" - "vpermq $0x94,%%ymm0,%%ymm3 \n" // b0 - "vperm2i128 $0x21,%%ymm1,%%ymm0,%%ymm4 \n" // m01 - "vperm2i128 $0x21,%%ymm2,%%ymm1,%%ymm5 \n" // m12 - "vpermq $0xe9,%%ymm2,%%ymm6 \n" // b3 - "vpermq $0xe9,%%ymm4,%%ymm4 \n" // b1 - "vpermq $0x94,%%ymm5,%%ymm5 \n" // b2 + "vperm2i128 $0x21,%%ymm1,%%ymm0,%%ymm3 \n" // m01 + "vpermq $0x94,%%ymm0,%%ymm0 \n" // b0 + "vperm2i128 $0x21,%%ymm2,%%ymm1,%%ymm1 \n" // m12 + "vpermq $0xe9,%%ymm2,%%ymm2 \n" // b3 + "vpermq $0xe9,%%ymm3,%%ymm3 \n" // b1 + "vpermq $0x94,%%ymm1,%%ymm1 \n" // b2 - "vpshufb %%ymm7,%%ymm3,%%ymm3 \n" - "vpshufb %%ymm7,%%ymm4,%%ymm4 \n" - "vpshufb %%ymm7,%%ymm5,%%ymm5 \n" - "vpshufb %%ymm7,%%ymm6,%%ymm6 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm4,%%ymm3,%%ymm3 \n" + "vpshufb %%ymm4,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm4,%%ymm2,%%ymm2 \n" + + "vpermd %%ymm0,%%ymm5,%%ymm0 \n" + "vpermd %%ymm3,%%ymm6,%%ymm3 \n" + "vpermd %%ymm1,%%ymm7,%%ymm1 \n" + "vpermd %%ymm2,%%ymm5,%%ymm2 \n" // Dst0 - "vpermd %%ymm3,%%ymm8,%%ymm0 \n" - "vpermq $0x00,%%ymm4,%%ymm1 \n" - "vpermd %%ymm4,%%ymm9,%%ymm2 \n" - "vpermd %%ymm5,%%ymm10,%%ymm3 \n" - "vpblendd $0xc0,%%ymm1,%%ymm0,%%ymm0 \n" + "vpblendd $0xc0,%%ymm3,%%ymm0,%%ymm0 \n" "vmovdqu %%ymm0,(%1) \n" // Dst1 - "vpblendd $0xf0,%%ymm3,%%ymm2,%%ymm2 \n" - "vpermd %%ymm5,%%ymm11,%%ymm4 \n" - "vpermd %%ymm6,%%ymm12,%%ymm5 \n" - "vmovdqu %%ymm2,32(%1) \n" + "vpblendd $0xf0,%%ymm1,%%ymm3,%%ymm3 \n" + "vpermq $0x90,%%ymm2,%%ymm2 \n" + "vmovdqu %%ymm3,32(%1) \n" // Dst2 - "vpblendd $0xfc,%%ymm5,%%ymm4,%%ymm4 \n" - "vmovdqu %%ymm4,64(%1) \n" + "vpblendd $0xfc,%%ymm2,%%ymm1,%%ymm1 \n" + "vmovdqu %%ymm1,64(%1) \n" "lea 96(%1),%1 \n" "sub $32,%2 \n" @@ -652,11 +648,9 @@ void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { : "m"(kShuffleMaskRAWToRGB24_AVX2[0]), // %3 "m"(kPermd0_AVX2), // %4 "m"(kPermd1_AVX2), // %5 - "m"(kPermd2_AVX2), // %6 - "m"(kPermd3_AVX2), // %7 - "m"(kPermd4_AVX2) // %8 + "m"(kPermd2_AVX2) // %6 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12"); + "xmm7"); } #endif // HAS_RAWTORGB24ROW_AVX2 @@ -681,24 +675,20 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, "vmovdqu8 128(%0),%%zmm2 \n" "lea 192(%0),%0 \n" "vpermq %%zmm2,%%zmm23,%%zmm3 \n" - "vmovdqa64 %%zmm22,%%zmm18 \n" - "vpermi2q %%zmm2,%%zmm1,%%zmm18 \n" - "vmovdqa64 %%zmm21,%%zmm19 \n" - "vpermi2q %%zmm1,%%zmm0,%%zmm19 \n" + "vmovdqa64 %%zmm0,%%zmm19 \n" + "vpermt2q %%zmm1,%%zmm21,%%zmm19 \n" + "vpermt2q %%zmm2,%%zmm22,%%zmm1 \n" "vpermq %%zmm0,%%zmm20,%%zmm0 \n" "vpshufb %%zmm4,%%zmm0,%%zmm0 \n" - "vpshufb %%zmm4,%%zmm19,%%zmm1 \n" - "vpshufb %%zmm4,%%zmm18,%%zmm2 \n" + "vpshufb %%zmm4,%%zmm19,%%zmm19 \n" + "vpshufb %%zmm4,%%zmm1,%%zmm2 \n" "vpshufb %%zmm4,%%zmm3,%%zmm3 \n" - "vmovdqa32 %%zmm24,%%zmm16 \n" - "vpermi2d %%zmm1,%%zmm0,%%zmm16 \n" - "vmovdqa32 %%zmm25,%%zmm17 \n" - "vpermi2d %%zmm2,%%zmm1,%%zmm17 \n" - "vmovdqa32 %%zmm26,%%zmm18 \n" - "vpermi2d %%zmm3,%%zmm2,%%zmm18 \n" - "vmovdqu8 %%zmm16,(%1) \n" - "vmovdqu8 %%zmm17,0x40(%1) \n" - "vmovdqu8 %%zmm18,0x80(%1) \n" + "vpermt2d %%zmm19,%%zmm24,%%zmm0 \n" + "vpermt2d %%zmm2,%%zmm25,%%zmm19 \n" + "vpermt2d %%zmm3,%%zmm26,%%zmm2 \n" + "vmovdqu8 %%zmm0,(%1) \n" + "vmovdqu8 %%zmm19,0x40(%1) \n" + "vmovdqu8 %%zmm2,0x80(%1) \n" "lea 0xc0(%1),%1 \n" "sub $0x40,%2 \n" "jg 1b \n" @@ -714,9 +704,8 @@ void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, "m"(kPermdRGB24_0_AVX512BW), // %8 "m"(kPermdRGB24_1_AVX512BW), // %9 "m"(kPermdRGB24_2_AVX512BW) // %10 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm16", - "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", - "xmm25", "xmm26"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm19", + "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26"); } #endif // HAS_RAWTORGB24ROW_AVX512BW diff --git a/source/row_lasx.cc b/source/row_lasx.cc index b3865918a..0ae67b12a 100644 --- a/source/row_lasx.cc +++ b/source/row_lasx.cc @@ -1422,6 +1422,42 @@ void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { } } +void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { + int x; + int len = width / 32; + __m256i reg0, reg1, reg2; + __m256i src0, src1, src2; + __m256i tmp0, tmp1, tmp2; + __m256i dst0, dst1, dst2; + __m256i shuf0 = {0x0708030405000102, 0x110C0D0E090A0B06, 0x0708030405000102, + 0x110C0D0E090A0B06}; + __m256i shuf1 = {0x1516171213140F10, 0x1F1E1B1C1D18191A, 0x1516171213140F10, + 0x1F1E1B1C1D18191A}; + __m256i shuf2 = {0x090405060102031E, 0x0D0E0F0A0B0C0708, 0x090405060102031E, + 0x0D0E0F0A0B0C0708}; + + for (x = 0; x < len; x++) { + reg0 = __lasx_xvld(src_raw, 0); + reg1 = __lasx_xvld(src_raw, 32); + reg2 = __lasx_xvld(src_raw, 64); + src0 = __lasx_xvpermi_q(reg1, reg0, 0x30); + src1 = __lasx_xvpermi_q(reg2, reg0, 0x21); + src2 = __lasx_xvpermi_q(reg2, reg1, 0x30); + DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuf0, src1, src0, shuf1, tmp0, + tmp1); + tmp2 = __lasx_xvshuf_b(src1, src2, shuf2); + tmp1 = __lasx_xvextrins_b(tmp1, src2, 0xE0); + dst0 = __lasx_xvpermi_q(tmp1, tmp0, 0x20); + dst1 = __lasx_xvpermi_q(tmp0, tmp2, 0x30); + dst2 = __lasx_xvpermi_q(tmp2, tmp1, 0x31); + __lasx_xvst(dst0, dst_rgb24, 0); + __lasx_xvst(dst1, dst_rgb24, 32); + __lasx_xvst(dst2, dst_rgb24, 64); + src_raw += 96; + dst_rgb24 += 96; + } +} + void ARGB1555ToYRow_LASX(const uint8_t* src_argb1555, uint8_t* dst_y, int width) { diff --git a/source/row_lsx.cc b/source/row_lsx.cc index 10eb88ae5..c54196100 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -2279,7 +2279,7 @@ void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { src2 = __lsx_vld(src_raw, 32); DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuf0, src1, src0, shuf1, dst0, dst1); dst2 = __lsx_vshuf_b(src1, src2, shuf2); - dst1 = __lsx_vinsgr2vr_b(dst1, src_raw[32], 0x0E); + dst1 = __lsx_vextrins_b(dst1, src2, 0xE0); __lsx_vst(dst0, dst_rgb24, 0); __lsx_vst(dst1, dst_rgb24, 16); __lsx_vst(dst2, dst_rgb24, 32); diff --git a/source/row_neon.cc b/source/row_neon.cc index 109a35223..0fff9ecd8 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -1372,17 +1372,18 @@ void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { asm volatile( "1: \n" - "vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RAW. - "subs %2, %2, #8 \n" // 8 processed per loop. - "vswp.u8 d1, d3 \n" // swap R, B - "vst3.8 {d1, d2, d3}, [%1]! \n" // store 8 pixels of - // RGB24. + "vld3.8 {d0, d2, d4}, [%0]! \n" // load 16 pixels of RAW. + "vld3.8 {d1, d3, d5}, [%0]! \n" + "subs %2, %2, #16 \n" // 16 processed per loop. + "vswp.u8 q0, q2 \n" // swap R, B + "vst3.8 {d0, d2, d4}, [%1]! \n" // store 16 pixels of + "vst3.8 {d1, d3, d5}, [%1]! \n" // RGB24. "bgt 1b \n" : "+r"(src_raw), // %0 "+r"(dst_rgb24), // %1 "+r"(width) // %2 : - : "cc", "memory", "d1", "d2", "d3" // Clobber List + : "cc", "memory", "q0", "q1", "q2" // Clobber List ); } diff --git a/source/row_neon64.cc b/source/row_neon64.cc index 368461e31..8b76f445c 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -2058,12 +2058,12 @@ void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { asm volatile( "1: \n" - "ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // read r g b - "subs %w2, %w2, #8 \n" // 8 processed per loop. - "mov v3.8b, v1.8b \n" // move g + "ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // read 16 r g b + "subs %w2, %w2, #16 \n" // 16 per loop. + "mov v3.16b, v1.16b \n" // move g "prfm pldl1keep, [%0, 448] \n" - "mov v4.8b, v0.8b \n" // move r - "st3 {v2.8b,v3.8b,v4.8b}, [%1], #24 \n" // store b g r + "mov v4.16b, v0.16b \n" // move r + "st3 {v2.16b,v3.16b,v4.16b}, [%1], #48 \n" // store 16 b g r "b.gt 1b \n" : "+r"(src_raw), // %0 "+r"(dst_rgb24), // %1 diff --git a/source/row_win.cc b/source/row_win.cc index c89aa9308..b319973c1 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -31,12 +31,14 @@ extern "C" { #endif #if defined(__clang__) || defined(__GNUC__) +#define LIBYUV_TARGET_SSSE3 __attribute__((target("ssse3"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) #define LIBYUV_TARGET_AVX512VBMI \ __attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f"))) #else +#define LIBYUV_TARGET_SSSE3 #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW #define LIBYUV_TARGET_AVX512VBMI @@ -338,6 +340,23 @@ static const uint8_t kShuffleMaskRAWToARGB_AVX512BW[32] = { 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, 13u, 128u}; #endif // defined(HAS_RAWTOARGBROW_AVX512BW) +#if defined(HAS_RAWTORGB24ROW_SSSE3) +// Shuffle table for converting RAW to RGB24. First 8. +static const uint8_t kShuffleMaskRAWToRGB24_0[16] = { + 2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, + 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u}; + +// Shuffle table for converting RAW to RGB24. Middle 8. +static const uint8_t kShuffleMaskRAWToRGB24_1[16] = { + 2u, 7u, 6u, 5u, 10u, 9u, 8u, 13u, + 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u}; + +// Shuffle table for converting RAW to RGB24. Last 8. +static const uint8_t kShuffleMaskRAWToRGB24_2[16] = { + 8u, 7u, 12u, 11u, 10u, 15u, 14u, 13u, + 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u}; +#endif // defined(HAS_RAWTORGB24ROW_SSSE3) + #if defined(HAS_RAWTORGB24ROW_AVX2) || defined(HAS_RAWTORGB24ROW_AVX512BW) static const uint8_t kShuffleMaskRAWToRGB24_AVX2[32] = { 2u, 1u, 0u, 5u, 4u, 3u, 8u, 7u, 6u, 11u, 10u, @@ -347,11 +366,9 @@ static const uint8_t kShuffleMaskRAWToRGB24_AVX2[32] = { // defined(HAS_RAWTORGB24ROW_AVX512BW) #if defined(HAS_RAWTORGB24ROW_AVX2) -static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 0, 0}; -static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 0, 0, 0, 0}; -static const uint32_t kPermd2_AVX2[8] = {0, 0, 0, 0, 0, 1, 2, 4}; -static const uint32_t kPermd3_AVX2[8] = {5, 6, 0, 0, 0, 0, 0, 0}; -static const uint32_t kPermd4_AVX2[8] = {0, 0, 0, 1, 2, 4, 5, 6}; +static const uint32_t kPermd0_AVX2[8] = {0, 1, 2, 4, 5, 6, 3, 7}; +static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 3, 7, 0, 1}; +static const uint32_t kPermd2_AVX2[8] = {5, 6, 3, 7, 0, 1, 2, 4}; #endif // defined(HAS_RAWTORGB24ROW_AVX2) #if defined(HAS_RAWTORGB24ROW_AVX512BW) @@ -466,6 +483,35 @@ void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, } #endif // HAS_RGB24TOARGBROW_AVX512BW +#ifdef HAS_RAWTORGB24ROW_SSSE3 +LIBYUV_TARGET_SSSE3 +void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, + uint8_t* dst_rgb24, + int width) { + __m128i shuf0 = _mm_loadu_si128((const __m128i*)kShuffleMaskRAWToRGB24_0); + __m128i shuf1 = _mm_loadu_si128((const __m128i*)kShuffleMaskRAWToRGB24_1); + __m128i shuf2 = _mm_loadu_si128((const __m128i*)kShuffleMaskRAWToRGB24_2); + + while (width > 0) { + __m128i raw0 = _mm_loadu_si128((const __m128i*)src_raw); + __m128i raw1 = _mm_loadu_si128((const __m128i*)(src_raw + 4)); + __m128i raw2 = _mm_loadu_si128((const __m128i*)(src_raw + 8)); + + raw0 = _mm_shuffle_epi8(raw0, shuf0); + raw1 = _mm_shuffle_epi8(raw1, shuf1); + raw2 = _mm_shuffle_epi8(raw2, shuf2); + + _mm_storel_epi64((__m128i*)dst_rgb24, raw0); + _mm_storel_epi64((__m128i*)(dst_rgb24 + 8), raw1); + _mm_storel_epi64((__m128i*)(dst_rgb24 + 16), raw2); + + src_raw += 24; + dst_rgb24 += 24; + width -= 8; + } +} +#endif // HAS_RAWTORGB24ROW_SSSE3 + #ifdef HAS_RAWTORGB24ROW_AVX2 LIBYUV_TARGET_AVX2 void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { @@ -474,44 +520,40 @@ void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { __m256i ymm_p0 = _mm256_loadu_si256((const __m256i*)kPermd0_AVX2); __m256i ymm_p1 = _mm256_loadu_si256((const __m256i*)kPermd1_AVX2); __m256i ymm_p2 = _mm256_loadu_si256((const __m256i*)kPermd2_AVX2); - __m256i ymm_p3 = _mm256_loadu_si256((const __m256i*)kPermd3_AVX2); - __m256i ymm_p4 = _mm256_loadu_si256((const __m256i*)kPermd4_AVX2); while (width > 0) { __m256i raw0 = _mm256_loadu_si256((const __m256i*)src_raw); __m256i raw1 = _mm256_loadu_si256((const __m256i*)(src_raw + 32)); __m256i raw2 = _mm256_loadu_si256((const __m256i*)(src_raw + 64)); - __m256i b0 = _mm256_permute4x64_epi64(raw0, 0x94); __m256i m01 = _mm256_permute2x128_si256(raw0, raw1, 0x21); - __m256i b1 = _mm256_permute4x64_epi64(m01, 0xe9); - + __m256i b0 = _mm256_permute4x64_epi64(raw0, 0x94); __m256i m12 = _mm256_permute2x128_si256(raw1, raw2, 0x21); - __m256i b2 = _mm256_permute4x64_epi64(m12, 0x94); __m256i b3 = _mm256_permute4x64_epi64(raw2, 0xe9); + __m256i b1 = _mm256_permute4x64_epi64(m01, 0xe9); + __m256i b2 = _mm256_permute4x64_epi64(m12, 0x94); b0 = _mm256_shuffle_epi8(b0, ymm_shuf); b1 = _mm256_shuffle_epi8(b1, ymm_shuf); b2 = _mm256_shuffle_epi8(b2, ymm_shuf); b3 = _mm256_shuffle_epi8(b3, ymm_shuf); - // Dst0 = blend(b0_dwords, b1_q0) __m256i d0 = _mm256_permutevar8x32_epi32(b0, ymm_p0); - __m256i b1_q0 = _mm256_permute4x64_epi64(b1, 0x00); - __m256i dst0 = _mm256_blend_epi32(d0, b1_q0, 0xc0); - - // Dst1 = blend(b1_dwords, b2_dwords) __m256i d1 = _mm256_permutevar8x32_epi32(b1, ymm_p1); __m256i d2 = _mm256_permutevar8x32_epi32(b2, ymm_p2); - __m256i dst1 = _mm256_blend_epi32(d1, d2, 0xf0); - - // Dst2 = blend(b2_dwords, b3_dwords) - __m256i d2_c45 = _mm256_permutevar8x32_epi32(b2, ymm_p3); - __m256i d3 = _mm256_permutevar8x32_epi32(b3, ymm_p4); - __m256i dst2 = _mm256_blend_epi32(d2_c45, d3, 0xfc); + __m256i d3 = _mm256_permutevar8x32_epi32(b3, ymm_p0); + // Dst0 = blend(d0, d1) + __m256i dst0 = _mm256_blend_epi32(d0, d1, 0xc0); _mm256_storeu_si256((__m256i*)dst_rgb24, dst0); + + // Dst1 = blend(d1, d2) + __m256i dst1 = _mm256_blend_epi32(d1, d2, 0xf0); + d3 = _mm256_permute4x64_epi64(d3, 0x90); _mm256_storeu_si256((__m256i*)(dst_rgb24 + 32), dst1); + + // Dst2 = blend(d2, d3) + __m256i dst2 = _mm256_blend_epi32(d2, d3, 0xfc); _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), dst2); src_raw += 96;