From 2154de414c1ac41fb434348964c728ccc077ffd3 Mon Sep 17 00:00:00 2001 From: "fbarchard@google.com" Date: Tue, 3 Sep 2013 07:18:21 +0000 Subject: [PATCH] Port InterpolateRows to AVX2 BUG=264 TEST=ARGBInterpolate* R=changjun.yang@intel.com Review URL: https://webrtc-codereview.appspot.com/2160004 git-svn-id: http://libyuv.googlecode.com/svn/trunk@777 16f28f9a-4ce2-e073-06de-1de4eb20be90 --- README.chromium | 2 +- include/libyuv/row.h | 7 +++ include/libyuv/version.h | 2 +- source/planar_functions.cc | 8 +++ source/row_any.cc | 4 ++ source/row_win.cc | 103 +++++++++++++++++++++++++++++++++++++ source/scale.cc | 10 +++- source/scale_argb.cc | 24 +++++++++ source/scale_common.cc | 8 +++ 9 files changed, 165 insertions(+), 3 deletions(-) diff --git a/README.chromium b/README.chromium index df1bb03ea..a742a658b 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: http://code.google.com/p/libyuv/ -Version: 776 +Version: 777 License: BSD License File: LICENSE diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 2e31d94bb..91b034fbd 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -159,6 +159,7 @@ extern "C" { #define HAS_YUY2TOUV422ROW_AVX2 #define HAS_YUY2TOUVROW_AVX2 #define HAS_YUY2TOYROW_AVX2 +#define HAS_INTERPOLATEROW_AVX2 // Effects: #define HAS_ARGBADDROW_AVX2 @@ -1484,6 +1485,9 @@ void InterpolateRow_SSE2(uint8* dst_ptr, const uint8* src_ptr, void InterpolateRow_SSSE3(uint8* dst_ptr, const uint8* src_ptr, ptrdiff_t src_stride_ptr, int width, int source_y_fraction); +void InterpolateRow_AVX2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); void InterpolateRow_NEON(uint8* dst_ptr, const uint8* src_ptr, ptrdiff_t src_stride_ptr, int width, int source_y_fraction); @@ -1505,6 +1509,9 @@ void InterpolateRow_Any_SSE2(uint8* dst_ptr, const uint8* src_ptr, void InterpolateRow_Any_SSSE3(uint8* dst_ptr, const uint8* src_ptr, ptrdiff_t src_stride_ptr, int width, int source_y_fraction); +void InterpolateRow_Any_AVX2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride_ptr, int width, + int source_y_fraction); void InterpolateRows_Any_MIPS_DSPR2(uint8* dst_ptr, const uint8* src_ptr, ptrdiff_t src_stride_ptr, int width, int source_y_fraction); diff --git a/include/libyuv/version.h b/include/libyuv/version.h index 2ca2d3c3d..53483e7b9 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ // NOLINT #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 776 +#define LIBYUV_VERSION 777 #endif // INCLUDE_LIBYUV_VERSION_H_ NOLINT diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 8456c5043..87db9d05e 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -1708,6 +1708,14 @@ int ARGBInterpolate(const uint8* src_argb0, int src_stride_argb0, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && width >= 8) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON) && width >= 4) { InterpolateRow = InterpolateRow_Any_NEON; diff --git a/source/row_any.cc b/source/row_any.cc index d595871b5..a7e371fc1 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -501,6 +501,10 @@ YANY(ARGBShuffleRow_Any_NEON, ARGBShuffleRow_NEON, width & MASK, source_y_fraction); \ } +#ifdef HAS_INTERPOLATEROW_AVX2 +NANY(InterpolateRow_Any_AVX2, InterpolateRow_AVX2, + InterpolateRow_C, 1, 1, 32) +#endif #ifdef HAS_INTERPOLATEROW_SSSE3 NANY(InterpolateRow_Any_SSSE3, InterpolateRow_Unaligned_SSSE3, InterpolateRow_C, 1, 1, 15) diff --git a/source/row_win.cc b/source/row_win.cc index 87991eac1..55235e27f 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -5970,6 +5970,108 @@ void ARGBAffineRow_SSE2(const uint8* src_argb, int src_argb_stride, } #endif // HAS_ARGBAFFINEROW_SSE2 +#ifdef HAS_INTERPOLATEROW_AVX2 +// Bilinear filter 16x2 -> 16x1 +__declspec(naked) __declspec(align(16)) +void InterpolateRow_AVX2(uint8* dst_ptr, const uint8* src_ptr, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) { + __asm { + push esi + push edi + mov edi, [esp + 8 + 4] // dst_ptr + mov esi, [esp + 8 + 8] // src_ptr + mov edx, [esp + 8 + 12] // src_stride + mov ecx, [esp + 8 + 16] // dst_width + mov eax, [esp + 8 + 20] // source_y_fraction (0..255) + shr eax, 1 + // Dispatch to specialized filters if applicable. + cmp eax, 0 + je xloop100 // 0 / 128. Blend 100 / 0. + sub edi, esi + cmp eax, 32 + je xloop75 // 32 / 128 is 0.25. Blend 75 / 25. + cmp eax, 64 + je xloop50 // 64 / 128 is 0.50. Blend 50 / 50. + cmp eax, 96 + je xloop25 // 96 / 128 is 0.75. Blend 25 / 75. + + vmovd xmm0, eax // high fraction 0..127 + neg eax + add eax, 128 + vmovd xmm5, eax // low fraction 128..1 + vpunpcklbw xmm5, xmm5, xmm0 + vpunpcklwd xmm5, xmm5, xmm5 + vpxor ymm0, ymm0, ymm0 + vpermd ymm5, ymm0, ymm5 + + align 16 + xloop: + vmovdqu ymm0, [esi] + vmovdqu ymm2, [esi + edx] + vpunpckhbw ymm1, ymm0, ymm2 // mutates + vpunpcklbw ymm0, ymm0, ymm2 // mutates + vpmaddubsw ymm0, ymm0, ymm5 + vpmaddubsw ymm1, ymm1, ymm5 + vpsrlw ymm0, ymm0, 7 + vpsrlw ymm1, ymm1, 7 + vpackuswb ymm0, ymm0, ymm1 // unmutates + sub ecx, 32 + vmovdqu [esi + edi], ymm0 + lea esi, [esi + 32] + jg xloop + jmp xloop99 + + // Blend 25 / 75. + align 16 + xloop25: + vmovdqu ymm0, [esi] + vpavgb ymm0, ymm0, [esi + edx] + vpavgb ymm0, ymm0, [esi + edx] + sub ecx, 32 + vmovdqu [esi + edi], ymm0 + lea esi, [esi + 32] + jg xloop25 + jmp xloop99 + + // Blend 50 / 50. + align 16 + xloop50: + vmovdqu ymm0, [esi] + vpavgb ymm0, ymm0, [esi + edx] + sub ecx, 32 + vmovdqu [esi + edi], ymm0 + lea esi, [esi + 32] + jg xloop50 + jmp xloop99 + + // Blend 75 / 25. + align 16 + xloop75: + vmovdqu ymm0, [esi + edx] + vpavgb ymm0, ymm0, [esi] + vpavgb ymm0, ymm0, [esi] + sub ecx, 32 + vmovdqu [esi + edi], ymm0 + lea esi, [esi + 32] + jg xloop75 + jmp xloop99 + + // Blend 100 / 0 - Copy row unchanged. + align 16 + xloop100: + rep movsb + + xloop99: + pop edi + pop esi + vzeroupper + ret + } +} +#endif // HAS_INTERPOLATEROW_AVX2 + +#ifdef HAS_INTERPOLATEROW_SSSE3 // Bilinear filter 16x2 -> 16x1 __declspec(naked) __declspec(align(16)) void InterpolateRow_SSSE3(uint8* dst_ptr, const uint8* src_ptr, @@ -6074,6 +6176,7 @@ void InterpolateRow_SSSE3(uint8* dst_ptr, const uint8* src_ptr, ret } } +#endif // HAS_INTERPOLATEROW_SSSE3 #ifdef HAS_INTERPOLATEROW_SSE2 // Bilinear filter 16x2 -> 16x1 diff --git a/source/scale.cc b/source/scale.cc index 1c58b95e3..bdea38dc6 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -620,7 +620,7 @@ static void ScaleRowDown38_SSSE3(const uint8* src_ptr, ptrdiff_t src_stride, paddusb xmm0, xmm1 sub ecx, 12 - movq qword ptr [edx], xmm0 // write 12 pixels + movq qword ptr [edx], xmm0 // write 12 pixels movhlps xmm1, xmm0 movd [edx + 8], xmm1 lea edx, [edx + 12] @@ -2110,6 +2110,14 @@ void ScalePlaneBilinear(int src_width, int src_height, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && src_width >= 32) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(src_width, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON) && src_width >= 16) { InterpolateRow = InterpolateRow_Any_NEON; diff --git a/source/scale_argb.cc b/source/scale_argb.cc index 83e20c2f9..8180f4fb7 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -939,6 +939,14 @@ static void ScaleARGBBilinearDown(int src_height, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && clip_src_width >= 32) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(clip_src_width, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON) && clip_src_width >= 16) { InterpolateRow = InterpolateRow_Any_NEON; @@ -1014,6 +1022,14 @@ static void ScaleARGBBilinearUp(int src_width, int src_height, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && dst_width >= 8) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON) && dst_width >= 4) { InterpolateRow = InterpolateRow_Any_NEON; @@ -1157,6 +1173,14 @@ static void ScaleYUVToARGBBilinearUp(int src_width, int src_height, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && dst_width >= 8) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON) && dst_width >= 4) { InterpolateRow = InterpolateRow_Any_NEON; diff --git a/source/scale_common.cc b/source/scale_common.cc index 8f405dd97..2c13450ca 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -62,6 +62,14 @@ void ScalePlaneVertical(int src_height, } } #endif +#if defined(HAS_INTERPOLATEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && dst_widthx4 >= 32) { + InterpolateRow = InterpolateRow_Any_AVX2; + if (IS_ALIGNED(dst_widthx4, 32)) { + InterpolateRow = InterpolateRow_AVX2; + } + } +#endif #if defined(HAS_INTERPOLATEROW_NEON) if (TestCpuFlag(kCpuHasNEON) && dst_widthx4 >= 16) { InterpolateRow = InterpolateRow_Any_NEON;