diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 476c0ac4c..b0ce5d05d 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -404,6 +404,7 @@ extern "C" { #define HAS_ARGBMIRRORROW_AVX2 #define HAS_INTERPOLATEROW_AVX2 #define HAS_INTERPOLATEROW_16_AVX2 +#define HAS_MULTIPLYROW_16_AVX2 #endif // The following are available for AVX512 clang x64 platforms: @@ -422,6 +423,7 @@ extern "C" { #define HAS_CONVERT8TO16ROW_AVX512BW #define HAS_HALFROW_16TO8_AVX512BW #define HAS_HALFWIDTHROW_16TO8_AVX512BW +#define HAS_MULTIPLYROW_16_AVX512BW #endif // The following are available for AVX512 clang x64 platforms: @@ -887,6 +889,7 @@ extern "C" { #define HAS_HALFWIDTHROW_16TO8_RVV #define HAS_I422TOAR30ROW_RVV #define HAS_INTERPOLATEROW_RVV +#define HAS_MULTIPLYROW_16_RVV #define HAS_RGBTOUV444MATRIXROW_RVV #define HAS_RGBTOUVMATRIXROW_RVV #define HAS_RGBTOYMATRIXROW_RVV @@ -3539,6 +3542,14 @@ void MultiplyRow_16_Any_AVX2(const uint16_t* src_ptr, uint16_t* dst_ptr, int scale, int width); +void MultiplyRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); +void MultiplyRow_16_Any_AVX512BW(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int scale, + int width); void MultiplyRow_16_NEON(const uint16_t* src_y, uint16_t* dst_y, int scale, @@ -3551,6 +3562,10 @@ void MultiplyRow_16_SME(const uint16_t* src_y, uint16_t* dst_y, int scale, int width); +void MultiplyRow_16_RVV(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); void DivideRow_16_C(const uint16_t* src_y, uint16_t* dst_y, diff --git a/source/convert.cc b/source/convert.cc index 497b7b2bc..20445caf7 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -2395,7 +2395,7 @@ int ARGBToI420Matrix(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) - if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) { + if (TestCpuFlag(kCpuHasNeonI8MM)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; @@ -2614,7 +2614,7 @@ int ARGBToI420Alpha(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) - if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) { + if (TestCpuFlag(kCpuHasNeonI8MM)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index f87c5c054..7031cb36c 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -316,7 +316,7 @@ int ARGBToI422Matrix(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) - if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) { + if (TestCpuFlag(kCpuHasNeonI8MM)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; @@ -502,7 +502,7 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) - if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) { + if (TestCpuFlag(kCpuHasNeonI8MM)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; @@ -752,7 +752,7 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) - if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) { + if (TestCpuFlag(kCpuHasNeonI8MM)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; @@ -1230,7 +1230,7 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBTOUVMATRIXROW_NEON_I8MM) - if (TestCpuFlag(kCpuHasNEON) && TestCpuFlag(kCpuHasNeonI8MM)) { + if (TestCpuFlag(kCpuHasNeonI8MM)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_Any_NEON_I8MM; if (IS_ALIGNED(width, 16)) { ARGBToUVMatrixRow = ARGBToUVMatrixRow_NEON_I8MM; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 2c6cd09c0..2eb4d7b01 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -955,6 +955,14 @@ void ConvertToMSBPlane_16(const uint16_t* src_y, } } #endif +#if defined(HAS_MULTIPLYROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MultiplyRow_16 = MultiplyRow_16_Any_AVX512BW; + if (IS_ALIGNED(width, 64)) { + MultiplyRow_16 = MultiplyRow_16_AVX512BW; + } + } +#endif #if defined(HAS_MULTIPLYROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { MultiplyRow_16 = MultiplyRow_16_Any_NEON; @@ -968,6 +976,11 @@ void ConvertToMSBPlane_16(const uint16_t* src_y, MultiplyRow_16 = MultiplyRow_16_SME; } #endif +#if defined(HAS_MULTIPLYROW_16_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + MultiplyRow_16 = MultiplyRow_16_RVV; + } +#endif for (y = 0; y < height; ++y) { MultiplyRow_16(src_y, dst_y, scale, width); diff --git a/source/row_any.cc b/source/row_any.cc index 9daa51ae3..4f3a7a273 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -1583,6 +1583,15 @@ ANY11C(Convert8To16Row_Any_NEON, uint16_t, 15) #endif +#ifdef HAS_MULTIPLYROW_16_AVX512BW +ANY11C(MultiplyRow_16_Any_AVX512BW, + MultiplyRow_16_AVX512BW, + 2, + 2, + uint16_t, + uint16_t, + 63) +#endif #ifdef HAS_MULTIPLYROW_16_AVX2 ANY11C(MultiplyRow_16_Any_AVX2, MultiplyRow_16_AVX2, diff --git a/source/row_gcc.cc b/source/row_gcc.cc index ae69f6a2c..ec2b51ae9 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -5789,10 +5789,8 @@ void MultiplyRow_16_AVX2(const uint16_t* src_y, // 32 pixels per loop. LABELALIGN "1: \n" - "vmovdqu (%0),%%ymm0 \n" - "vmovdqu 0x20(%0),%%ymm1 \n" - "vpmullw %%ymm3,%%ymm0,%%ymm0 \n" - "vpmullw %%ymm3,%%ymm1,%%ymm1 \n" + "vpmullw (%0),%%ymm3,%%ymm0 \n" + "vpmullw 0x20(%0),%%ymm3,%%ymm1 \n" "vmovdqu %%ymm0,(%0,%1) \n" "vmovdqu %%ymm1,0x20(%0,%1) \n" "add $0x40,%0 \n" @@ -5807,6 +5805,34 @@ void MultiplyRow_16_AVX2(const uint16_t* src_y, } #endif // HAS_MULTIPLYROW_16_AVX2 +#ifdef HAS_MULTIPLYROW_16_AVX512BW +void MultiplyRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + asm volatile( + "vpbroadcastw %3,%%zmm2 \n" + + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vpmullw (%0),%%zmm2,%%zmm0 \n" + "vpmullw 0x40(%0),%%zmm2,%%zmm1 \n" + "add $0x80,%0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "add $0x80,%1 \n" + "sub $0x40,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width) // %2 + : "r"(scale) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm2"); +} +#endif // HAS_MULTIPLYROW_16_AVX512BW + // Use scale to convert msb formats to lsb, depending how many bits there are: // 512 = 9 bits // 1024 = 10 bits diff --git a/source/row_rvv.cc b/source/row_rvv.cc index 165b7be4e..9cc87ed28 100644 --- a/source/row_rvv.cc +++ b/source/row_rvv.cc @@ -1906,6 +1906,24 @@ void Convert8To16Row_RVV(const uint8_t* src_y, } #endif +#ifdef HAS_MULTIPLYROW_16_RVV +void MultiplyRow_16_RVV(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + size_t w = (size_t)width; + do { + size_t vl = __riscv_vsetvl_e16m8(w); + vuint16m8_t v_src = __riscv_vle16_v_u16m8(src_y, vl); + vuint16m8_t v_dst = __riscv_vmul_vx_u16m8(v_src, (uint16_t)scale, vl); + __riscv_vse16_v_u16m8(dst_y, v_dst, vl); + w -= vl; + src_y += vl; + dst_y += vl; + } while (w > 0); +} +#endif + #ifdef HAS_HALFROW_16TO8_RVV void HalfRow_16To8_RVV(const uint16_t* src_uv, ptrdiff_t src_uv_stride, diff --git a/source/row_win.cc b/source/row_win.cc index 927729d15..961235489 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -790,6 +790,28 @@ void Convert8To16Row_AVX2(const uint8_t* src_y, } #endif // HAS_CONVERT8TO16ROW_AVX2 +#ifdef HAS_MULTIPLYROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void MultiplyRow_16_AVX2(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + __m256i scale256 = _mm256_set1_epi16((short)scale); + do { + __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src_y); + __m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src_y + 16)); + ymm0 = _mm256_mullo_epi16(ymm0, scale256); + ymm1 = _mm256_mullo_epi16(ymm1, scale256); + _mm256_storeu_si256((__m256i*)dst_y, ymm0); + _mm256_storeu_si256((__m256i*)(dst_y + 16), ymm1); + src_y += 32; + dst_y += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MULTIPLYROW_16_AVX2 + #ifdef HAS_MERGEUVROW_AVX2 LIBYUV_TARGET_AVX2 void MergeUVRow_AVX2(const uint8_t* src_u, diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index df7747564..03a1c1883 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -3768,11 +3768,12 @@ TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) { } #endif -// TODO(fbarchard): Improve test for more platforms. -#ifdef HAS_MULTIPLYROW_16_AVX2 +#if defined(HAS_MULTIPLYROW_16_AVX512BW) || defined(HAS_MULTIPLYROW_16_AVX2) || \ + defined(HAS_MULTIPLYROW_16_NEON) || defined(HAS_MULTIPLYROW_16_SME) || \ + defined(HAS_MULTIPLYROW_16_RVV) TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) { - // Round count up to multiple of 32 - const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31; + // Round count up to multiple of 64 + const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63; align_buffer_page_end(src_pixels_y, kPixels * 2); align_buffer_page_end(dst_pixels_y_opt, kPixels * 2); @@ -3785,13 +3786,59 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) { MultiplyRow_16_C(reinterpret_cast(src_pixels_y), reinterpret_cast(dst_pixels_y_c), 64, kPixels); +#if defined(HAS_MULTIPLYROW_16_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_MULTIPLYROW_16_AVX2) int has_avx2 = TestCpuFlag(kCpuHasAVX2); +#endif +#if defined(HAS_MULTIPLYROW_16_NEON) + int has_neon = TestCpuFlag(kCpuHasNEON); +#endif +#if defined(HAS_MULTIPLYROW_16_SME) + int has_sme = TestCpuFlag(kCpuHasSME); +#endif +#if defined(HAS_MULTIPLYROW_16_RVV) + int has_rvv = TestCpuFlag(kCpuHasRVV); +#endif + for (int i = 0; i < benchmark_iterations_; ++i) { +#if defined(HAS_MULTIPLYROW_16_AVX512BW) + if (has_avx512) { + MultiplyRow_16_AVX512BW(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), 64, + kPixels); + } else +#endif +#if defined(HAS_MULTIPLYROW_16_AVX2) if (has_avx2) { MultiplyRow_16_AVX2(reinterpret_cast(src_pixels_y), reinterpret_cast(dst_pixels_y_opt), 64, kPixels); - } else { + } else +#endif +#if defined(HAS_MULTIPLYROW_16_SME) + if (has_sme) { + MultiplyRow_16_SME(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), 64, + kPixels); + } else +#endif +#if defined(HAS_MULTIPLYROW_16_NEON) + if (has_neon) { + MultiplyRow_16_NEON(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), 64, + kPixels); + } else +#endif +#if defined(HAS_MULTIPLYROW_16_RVV) + if (has_rvv) { + MultiplyRow_16_RVV(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), 64, + kPixels); + } else +#endif + { MultiplyRow_16_C(reinterpret_cast(src_pixels_y), reinterpret_cast(dst_pixels_y_opt), 64, kPixels); @@ -3806,7 +3853,73 @@ TEST_F(LibYUVPlanarTest, MultiplyRow_16_Opt) { free_aligned_buffer_page_end(dst_pixels_y_opt); free_aligned_buffer_page_end(dst_pixels_y_c); } -#endif // HAS_MULTIPLYROW_16_AVX2 + +#if defined(HAS_MULTIPLYROW_16_AVX512BW) || defined(HAS_MULTIPLYROW_16_AVX2) || \ + defined(HAS_MULTIPLYROW_16_NEON) +TEST_F(LibYUVPlanarTest, MultiplyRow_16_Any) { + const int kMaxPixels = 256; + align_buffer_page_end(src_pixels_y, kMaxPixels * 2); + align_buffer_page_end(dst_pixels_y_opt, kMaxPixels * 2); + align_buffer_page_end(dst_pixels_y_c, kMaxPixels * 2); + + for (int i = 0; i < kMaxPixels; ++i) { + reinterpret_cast(src_pixels_y)[i] = fastrand() & 1023; + } + +#if defined(HAS_MULTIPLYROW_16_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_MULTIPLYROW_16_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); +#endif +#if defined(HAS_MULTIPLYROW_16_NEON) + int has_neon = TestCpuFlag(kCpuHasNEON); +#endif + + for (int width = 1; width <= 129; ++width) { + memset(dst_pixels_y_opt, 0, kMaxPixels * 2); + memset(dst_pixels_y_c, 1, kMaxPixels * 2); + + MultiplyRow_16_C(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_c), 64, width); + +#if defined(HAS_MULTIPLYROW_16_AVX512BW) + if (has_avx512) { + MultiplyRow_16_Any_AVX512BW(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), + 64, width); + } else +#endif +#if defined(HAS_MULTIPLYROW_16_AVX2) + if (has_avx2) { + MultiplyRow_16_Any_AVX2(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), + 64, width); + } else +#endif +#if defined(HAS_MULTIPLYROW_16_NEON) + if (has_neon) { + MultiplyRow_16_Any_NEON(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), + 64, width); + } else +#endif + { + MultiplyRow_16_C(reinterpret_cast(src_pixels_y), + reinterpret_cast(dst_pixels_y_opt), 64, width); + } + + for (int i = 0; i < width * 2; ++i) { + ASSERT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]); + } + } + + free_aligned_buffer_page_end(src_pixels_y); + free_aligned_buffer_page_end(dst_pixels_y_opt); + free_aligned_buffer_page_end(dst_pixels_y_c); +} +#endif // HAS_MULTIPLYROW_16_AVX512BW || HAS_MULTIPLYROW_16_AVX2 || HAS_MULTIPLYROW_16_NEON +#endif // HAS_MULTIPLYROW_16_... TEST_F(LibYUVPlanarTest, Convert16To8Plane) { const int kPixels = benchmark_width_ * benchmark_height_;