diff --git a/README.chromium b/README.chromium index 6e36e1cbc..5eb8a85c5 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1966 +Version: 1967 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/row.h b/include/libyuv/row.h index b748709c9..533734e6f 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -369,6 +369,7 @@ extern "C" { #define HAS_I422TOARGB1555ROW_AVX2 #define HAS_I422TOARGB4444ROW_AVX2 #define HAS_NV12TORGB565ROW_AVX2 +#define HAS_CONVERT8TO16ROW_AVX2 #if defined(__x86_64__) || defined(_M_X64) #define HAS_I422TOAR30ROW_AVX2 #define HAS_RAWTORGB24ROW_AVX2 @@ -418,6 +419,7 @@ extern "C" { #define HAS_RAWTORGB24ROW_AVX512VBMI #define HAS_ARGBTORGB24ROW_AVX512VBMI #define HAS_CONVERT16TO8ROW_AVX512BW +#define HAS_CONVERT8TO16ROW_AVX512BW #define HAS_HALFROW_16TO8_AVX512BW #define HAS_HALFWIDTHROW_16TO8_AVX512BW #endif @@ -485,6 +487,7 @@ extern "C" { #define HAS_AYUVTOYROW_NEON #define HAS_BYTETOFLOATROW_NEON #define HAS_CONVERT16TO8ROW_NEON +#define HAS_CONVERT8TO16ROW_NEON #define HAS_CONVERT8TO8ROW_NEON #define HAS_COPYROW_NEON #define HAS_DETILEROW_16_NEON @@ -583,7 +586,6 @@ extern "C" { #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #define HAS_ABGRTOAR30ROW_NEON #define HAS_ARGBTOAR30ROW_NEON -#define HAS_CONVERT8TO16ROW_NEON #define HAS_I210ALPHATOARGBROW_NEON #define HAS_I210TOAR30ROW_NEON #define HAS_I210TOARGBROW_NEON @@ -879,6 +881,7 @@ extern "C" { #define HAS_ARGBTOUVMATRIXROW_RVV #define HAS_ARGBTOYMATRIXROW_RVV #define HAS_CONVERT16TO8ROW_RVV +#define HAS_CONVERT8TO16ROW_RVV #define HAS_COPYROW_RVV #define HAS_HALFROW_16TO8_RVV #define HAS_HALFWIDTHROW_16TO8_RVV @@ -3579,6 +3582,10 @@ void Convert8To16Row_AVX2(const uint8_t* src_y, uint16_t* dst_y, int scale, int width); +void Convert8To16Row_AVX512BW(const uint8_t* src_y, + uint16_t* dst_y, + int scale, + int width); void Convert8To16Row_Any_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int scale, @@ -3587,6 +3594,10 @@ void Convert8To16Row_Any_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int scale, int width); +void Convert8To16Row_Any_AVX512BW(const uint8_t* src_ptr, + uint16_t* dst_ptr, + int scale, + int width); void Convert8To16Row_NEON(const uint8_t* src_y, uint16_t* dst_y, int scale, @@ -3599,6 +3610,10 @@ void Convert8To16Row_SME(const uint8_t* src_y, uint16_t* dst_y, int scale, int width); +void Convert8To16Row_RVV(const uint8_t* src_y, + uint16_t* dst_y, + int scale, + int width); void Convert16To8Row_C(const uint16_t* src_y, uint8_t* dst_y, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index 0c840bf66..f411b327a 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1967 +#define LIBYUV_VERSION 1968 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 39b6c150e..b63fc988a 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -235,6 +235,14 @@ void Convert8To16Plane(const uint8_t* src_y, } } #endif +#if defined(HAS_CONVERT8TO16ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + Convert8To16Row = Convert8To16Row_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + Convert8To16Row = Convert8To16Row_AVX512BW; + } + } +#endif #if defined(HAS_CONVERT8TO16ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { Convert8To16Row = Convert8To16Row_Any_NEON; @@ -248,6 +256,11 @@ void Convert8To16Plane(const uint8_t* src_y, Convert8To16Row = Convert8To16Row_SME; } #endif +#if defined(HAS_CONVERT8TO16ROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + Convert8To16Row = Convert8To16Row_RVV; + } +#endif // Convert plane for (y = 0; y < height; ++y) { diff --git a/source/row_any.cc b/source/row_any.cc index 8fe41b098..179ad5f1d 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1565,6 +1565,15 @@ ANY11C(Convert8To16Row_Any_AVX2, uint16_t, 31) #endif +#ifdef HAS_CONVERT8TO16ROW_AVX512BW +ANY11C(Convert8To16Row_Any_AVX512BW, + Convert8To16Row_AVX512BW, + 1, + 2, + uint8_t, + uint16_t, + 63) +#endif #ifdef HAS_CONVERT8TO16ROW_NEON ANY11C(Convert8To16Row_Any_NEON, Convert8To16Row_NEON, diff --git a/source/row_common.cc b/source/row_common.cc index ce302508b..dc823883c 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -3243,6 +3243,8 @@ void Convert16To8Row_C(const uint16_t* src_y, // Use scale to convert lsb formats to msb, depending how many bits there are: // 1024 = 10 bits +// 4096 = 12 bits +// 65536 = 16 bits void Convert8To16Row_C(const uint8_t* src_y, uint16_t* dst_y, int scale, diff --git a/source/row_gcc.cc b/source/row_gcc.cc index cf2473aa5..7252f8f44 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -6215,16 +6215,16 @@ void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, // 512 = 9 bits // 1024 = 10 bits // 4096 = 12 bits +// 65536 = 16 bits void Convert8To16Row_SSE2(const uint8_t* src_y, uint16_t* dst_y, int scale, int width) { + const int shift = __builtin_clz(scale) - 15; asm volatile( "movd %3,%%xmm2 \n" - "punpcklwd %%xmm2,%%xmm2 \n" - "pshufd $0x0,%%xmm2,%%xmm2 \n" - // 32 pixels per loop. + // 16 pixels per loop. LABELALIGN "1: \n" "movdqu (%0),%%xmm0 \n" @@ -6232,8 +6232,8 @@ void Convert8To16Row_SSE2(const uint8_t* src_y, "punpcklbw %%xmm0,%%xmm0 \n" "punpckhbw %%xmm1,%%xmm1 \n" "add $0x10,%0 \n" - "pmulhuw %%xmm2,%%xmm0 \n" - "pmulhuw %%xmm2,%%xmm1 \n" + "psrlw %%xmm2,%%xmm0 \n" + "psrlw %%xmm2,%%xmm1 \n" "movdqu %%xmm0,(%1) \n" "movdqu %%xmm1,0x10(%1) \n" "add $0x20,%1 \n" @@ -6242,7 +6242,7 @@ void Convert8To16Row_SSE2(const uint8_t* src_y, : "+r"(src_y), // %0 "+r"(dst_y), // %1 "+r"(width) // %2 - : "r"(scale) // %3 + : "r"(shift) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2"); } @@ -6278,6 +6278,46 @@ void Convert8To16Row_AVX2(const uint8_t* src_y, } #endif // HAS_CONVERT8TO16ROW_AVX2 +#ifdef HAS_CONVERT8TO16ROW_AVX512BW +// Use scale to convert to lsb formats depending how many bits there are: +// 512 = 9 bits +// 1024 = 10 bits +// 4096 = 12 bits +// 65536 = 16 bits +void Convert8To16Row_AVX512BW(const uint8_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + const int shift = __builtin_clz(scale) - 15; + asm volatile( + "vpbroadcastw %4,%%zmm2 \n" + "vmovd %3,%%xmm3 \n" + + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vpmovzxbw (%0),%%zmm0 \n" + "vpmovzxbw 0x20(%0),%%zmm1 \n" + "add $0x40,%0 \n" + "vpmullw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmullw %%zmm2,%%zmm1,%%zmm1 \n" + "vpsrlw %%xmm3,%%zmm0,%%zmm0 \n" + "vpsrlw %%xmm3,%%zmm1,%%zmm1 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "add $0x80,%1 \n" + "sub $0x40,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(shift), // %3 + "r"(0x0101) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); +} +#endif // HAS_CONVERT8TO16ROW_AVX512BW + #ifdef HAS_SPLITRGBROW_SSSE3 // Shuffle table for converting RGB to Planar. static const uvec8 kSplitRGBShuffle[9] = { diff --git a/source/row_neon.cc b/source/row_neon.cc index 3486c7d0a..69c42bf44 100644 --- a/source/row_neon.cc +++ b/source/row_neon.cc @@ -3831,6 +3831,37 @@ void Convert8To8Row_NEON(const uint8_t* src_y, : "cc", "memory", "q0", "q1", "q2", "q3", "d8", "q5"); } +// Use scale to convert lsb formats to msb, depending how many bits there are: +// 512 = 9 bits +// 1024 = 10 bits +// 4096 = 12 bits +// 65536 = 16 bits +void Convert8To16Row_NEON(const uint8_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + // (src * 0x0101 * scale) >> 16. + // Since scale is a power of two, compute the shift to use to avoid needing + // to widen to int32. + const int shift = 15 - __builtin_clz(scale); + asm volatile( + "vdup.16 q2, %3 \n" + "1: \n" + "vld1.8 {q0}, [%0]! \n" + "vmov q1, q0 \n" + "vzip.8 q0, q1 \n" + "subs %2, %2, #16 \n" + "vshl.u16 q0, q0, q2 \n" + "vshl.u16 q1, q1, q2 \n" + "vst1.16 {q0, q1}, [%1]! \n" + "bgt 1b \n" + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(shift) // %3 + : "cc", "memory", "q0", "q1", "q2"); +} + #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__).. #ifdef __cplusplus diff --git a/source/row_neon64.cc b/source/row_neon64.cc index a73108239..1fadf0844 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -5269,6 +5269,8 @@ void Convert8To8Row_NEON(const uint8_t* src_y, // Use scale to convert lsb formats to msb, depending how many bits there are: // 1024 = 10 bits +// 4096 = 12 bits +// 65536 = 16 bits void Convert8To16Row_NEON(const uint8_t* src_y, uint16_t* dst_y, int scale, diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 90b921e44..18cbe6a63 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -1813,6 +1813,32 @@ void Convert16To8Row_RVV(const uint16_t* src_y, } #endif +#ifdef HAS_CONVERT8TO16ROW_RVV +// Use scale to convert lsb formats to msb, depending how many bits there are: +// 512 = 9 bits +// 1024 = 10 bits +// 4096 = 12 bits +// 65536 = 16 bits +void Convert8To16Row_RVV(const uint8_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + size_t w = (size_t)width; + const int shift = __builtin_clz((int32_t)scale) - 15; + do { + size_t vl = __riscv_vsetvl_e8m2(w); + vuint8m2_t v_src = __riscv_vle8_v_u8m2(src_y, vl); + vuint16m4_t v_dst = __riscv_vwaddu_vx_u16m4(v_src, 0, vl); + v_dst = __riscv_vmul_vx_u16m4(v_dst, 0x0101, vl); + v_dst = __riscv_vsrl_vx_u16m4(v_dst, shift, vl); + __riscv_vse16_v_u16m4(dst_y, v_dst, vl); + w -= vl; + src_y += vl; + dst_y += vl; + } while (w > 0); +} +#endif + #ifdef HAS_HALFROW_16TO8_RVV void HalfRow_16To8_RVV(const uint16_t* src_uv, ptrdiff_t src_uv_stride, diff --git a/source/row_win.cc b/source/row_win.cc index ef1195f46..b0e1e634b 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -21,6 +21,9 @@ #include #include // For AVX2 intrinsics #include // For _mm_maddubs_epi16 +#if defined(_MSC_VER) && !defined(__clang__) +#include +#endif #ifdef __cplusplus namespace libyuv { @@ -762,6 +765,42 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBTOUVMATRIXROW_AVX2 +#ifdef HAS_CONVERT8TO16ROW_AVX2 +// Use scale to convert to lsb formats depending how many bits there are: +// 512 = 9 bits +// 1024 = 10 bits +// 4096 = 12 bits +// 65536 = 16 bits +LIBYUV_TARGET_AVX2 +void Convert8To16Row_AVX2(const uint8_t* src_y, + uint16_t* dst_y, + int scale, + int width) { +#if defined(_MSC_VER) && !defined(__clang__) + unsigned long index; + _BitScanReverse(&index, (unsigned long)scale); + const int shift = 16 - (int)index; +#else + const int shift = __builtin_clz(scale) - 15; +#endif + __m128i shift128 = _mm_cvtsi32_si128(shift); + do { + __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y); + ymm0 = _mm256_permute4x64_epi64(ymm0, 0xd8); + __m256i row0 = _mm256_unpacklo_epi8(ymm0, ymm0); + __m256i row1 = _mm256_unpackhi_epi8(ymm0, ymm0); + row0 = _mm256_srl_epi16(row0, shift128); + row1 = _mm256_srl_epi16(row1, shift128); + _mm256_storeu_si256((__m256i*)dst_y, row0); + _mm256_storeu_si256((__m256i*)(dst_y + 16), row1); + src_y += 32; + dst_y += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_CONVERT8TO16ROW_AVX2 + #ifdef HAS_MERGEUVROW_AVX2 LIBYUV_TARGET_AVX2 void MergeUVRow_AVX2(const uint8_t* src_u, diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 75f92c068..214081c0a 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -4181,9 +4181,9 @@ TEST_F(LibYUVPlanarTest, Convert8To16Plane) { #ifdef ENABLE_ROW_TESTS // TODO(fbarchard): Improve test for more platforms. -#ifdef HAS_CONVERT8TO16ROW_AVX2 +#if defined(HAS_CONVERT8TO16ROW_AVX512BW) || defined(HAS_CONVERT8TO16ROW_AVX2) TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) { - const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31; + const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63; align_buffer_page_end(src_pixels_y, kPixels); align_buffer_page_end(dst_pixels_y_opt, kPixels * 2); align_buffer_page_end(dst_pixels_y_c, kPixels * 2); @@ -4195,9 +4195,19 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) { Convert8To16Row_C(src_pixels_y, reinterpret_cast(dst_pixels_y_c), 1024, kPixels); +#if defined(HAS_CONVERT8TO16ROW_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif int has_avx2 = TestCpuFlag(kCpuHasAVX2); int has_sse2 = TestCpuFlag(kCpuHasSSE2); for (int i = 0; i < benchmark_iterations_; ++i) { +#if defined(HAS_CONVERT8TO16ROW_AVX512BW) + if (has_avx512) { + Convert8To16Row_AVX512BW(src_pixels_y, + reinterpret_cast(dst_pixels_y_opt), + 1024, kPixels); + } else +#endif if (has_avx2) { Convert8To16Row_AVX2(src_pixels_y, reinterpret_cast(dst_pixels_y_opt), 1024, @@ -4221,7 +4231,7 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) { free_aligned_buffer_page_end(dst_pixels_y_opt); free_aligned_buffer_page_end(dst_pixels_y_c); } -#endif // HAS_CONVERT8TO16ROW_AVX2 +#endif // HAS_CONVERT8TO16ROW_AVX512BW || HAS_CONVERT8TO16ROW_AVX2 float TestScaleMaxSamples(int benchmark_width, int benchmark_height,