From b5c764cfdd6a671aa837bf91ff53939402f9a730 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Thu, 24 Sep 2026 19:15:18 -0700 Subject: [PATCH] ARGBBlur AVX2 and NEON64 Add ComputeCumulativeSumRow_AVX2, CumulativeSumToAverageRow_AVX2, ComputeCumulativeSumRow_NEON, and CumulativeSumToAverageRow_NEON to accelerate ARGBBlur and ARGBComputeCumulativeSum on x86_64 and aarch64. Update ARGBBlurSmall_Opt to benchmark 320x180 with radius 1 and use a circular cumsum buffer of height radius * 2 + 2 in TestBlur. ARGBBlurSmall_Opt (320x180, r=1, 10000 iterations): CPU Was C/SSE2 Now AVX2/NEON Speedup -------------------- -------------- --------------- --------------- Intel Granite Rapids 29 ms (SSE2) 24 ms (AVX2) 1.21x Intel Emerald Rapids 30 ms (SSE2) 23 ms (AVX2) 1.30x Intel Sapphire Rapid 32 ms (SSE2) 28 ms (AVX2) 1.14x Intel Ice Lake 43 ms (SSE2) 34 ms (AVX2) 1.26x Intel Skylake-X 54 ms (SSE2) 45 ms (AVX2) 1.20x Intel Haswell 59 ms (SSE2) 54 ms (AVX2) 1.09x AMD Zen 5 32 ms (SSE2) 19 ms (AVX2) 1.68x AMD Zen 4 36 ms (SSE2) 26 ms (AVX2) 1.38x AMD Zen 3 42 ms (SSE2) 28 ms (AVX2) 1.50x ARM Neoverse-V2 256 ms (C) 40 ms (NEON) 6.40x ARGBBlur_Opt (1280x720, r=55, 1000 iterations): CPU Was C/SSE2 Now AVX2/NEON Speedup -------------------- -------------- --------------- --------------- Intel Granite Rapids 1150 ms (SSE2) 984 ms (AVX2) 1.17x Intel Emerald Rapids 1081 ms (SSE2) 953 ms (AVX2) 1.13x Intel Sapphire Rapid 1124 ms (SSE2) 973 ms (AVX2) 1.16x Intel Ice Lake 1450 ms (SSE2) 1167 ms (AVX2) 1.24x Intel Skylake-X 1935 ms (SSE2) 1630 ms (AVX2) 1.19x Intel Haswell 2211 ms (SSE2) 1982 ms (AVX2) 1.12x AMD Zen 5 1089 ms (SSE2) 694 ms (AVX2) 1.57x AMD Zen 4 1045 ms (SSE2) 929 ms (AVX2) 1.12x AMD Zen 3 1215 ms (SSE2) 967 ms (AVX2) 1.26x ARM Neoverse-V2 6529 ms (C) 1544 ms (NEON) 4.23x Bug: b/456384776 Change-Id: I39c463c791826d3adb05f0b0845b0b1f56f0e4da Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8464496 Commit-Queue: Frank Barchard Reviewed-by: richard winterton --- include/libyuv/row.h | 24 +++++ source/planar_functions.cc | 24 ++++- source/row_gcc.cc | 193 ++++++++++++++++++++++++++++++++++ source/row_neon64.cc | 208 +++++++++++++++++++++++++++++++++++++ unit_test/planar_test.cc | 26 +++-- 5 files changed, 463 insertions(+), 12 deletions(-) diff --git a/include/libyuv/row.h b/include/libyuv/row.h index 4c8b005d7..a282b1198 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -311,6 +311,8 @@ extern "C" { #define HAS_I410TOAR30ROW_AVX2 #define HAS_I410TOARGBROW_AVX2 #if defined(__x86_64__) +#define HAS_COMPUTECUMULATIVESUMROW_AVX2 +#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 #define HAS_I422TOAR30ROW_AVX2 #define HAS_RAWTORGB24ROW_AVX2 #endif @@ -587,6 +589,8 @@ extern "C" { #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #define HAS_ABGRTOAR30ROW_NEON #define HAS_ARGBTOAR30ROW_NEON +#define HAS_COMPUTECUMULATIVESUMROW_NEON +#define HAS_CUMULATIVESUMTOAVERAGEROW_NEON #define HAS_I210ALPHATOARGBROW_NEON #define HAS_I210TOAR30ROW_NEON #define HAS_I210TOARGBROW_NEON @@ -6612,6 +6616,26 @@ void ComputeCumulativeSumRow_SSE2(const uint8_t* row, int32_t* cumsum, const int32_t* previous_cumsum, int width); +void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, + const int32_t* botleft, + int width, + int area, + uint8_t* dst, + int count); +void ComputeCumulativeSumRow_AVX2(const uint8_t* row, + int32_t* cumsum, + const int32_t* previous_cumsum, + int width); +void CumulativeSumToAverageRow_NEON(const int32_t* topleft, + const int32_t* botleft, + int width, + int area, + uint8_t* dst, + int count); +void ComputeCumulativeSumRow_NEON(const uint8_t* row, + int32_t* cumsum, + const int32_t* previous_cumsum, + int width); void CumulativeSumToAverageRow_C(const int32_t* tl, const int32_t* bl, diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 61872ffa9..39585e1de 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -4354,11 +4354,21 @@ int ARGBComputeCumulativeSum(const uint8_t* src_argb, if (!dst_cumsum || !src_argb || width <= 0 || height <= 0) { return -1; } -#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_SSE2) +#if defined(HAS_COMPUTECUMULATIVESUMROW_SSE2) if (TestCpuFlag(kCpuHasSSE2)) { ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2; } #endif +#if defined(HAS_COMPUTECUMULATIVESUMROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2; + } +#endif +#if defined(HAS_COMPUTECUMULATIVESUMROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ComputeCumulativeSumRow = ComputeCumulativeSumRow_NEON; + } +#endif memset(dst_cumsum, 0, width * sizeof(dst_cumsum[0]) * 4); // 4 int per pixel. for (y = 0; y < height; ++y) { @@ -4418,6 +4428,18 @@ int ARGBBlur(const uint8_t* src_argb, ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2; CumulativeSumToAverageRow = CumulativeSumToAverageRow_SSE2; } +#endif +#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2; + CumulativeSumToAverageRow = CumulativeSumToAverageRow_AVX2; + } +#endif +#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ComputeCumulativeSumRow = ComputeCumulativeSumRow_NEON; + CumulativeSumToAverageRow = CumulativeSumToAverageRow_NEON; + } #endif // Compute enough CumulativeSum for first row to be blurred. After this // one row of CumulativeSum is updated at a time. diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 4b19e8353..0a432ed83 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -9629,6 +9629,199 @@ void CumulativeSumToAverageRow_SSE2(const int32_t* topleft, } #endif // HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 +#ifdef HAS_COMPUTECUMULATIVESUMROW_AVX2 +static const uvec8 kCumSumTranspose = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; +static const vec8 kCumSumMaddLo[2] = { + {1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0}, + {1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0}}; +static const vec8 kCumSumMaddHi[2] = { + {1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0}, + {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}}; + +void ComputeCumulativeSumRow_AVX2(const uint8_t* row, + int32_t* cumsum, + const int32_t* previous_cumsum, + int width) { + asm volatile( + "vpxor %%xmm0,%%xmm0,%%xmm0 \n" + "vbroadcasti128 %4,%%ymm5 \n" + "vmovdqu %5,%%ymm6 \n" + "vmovdqu %6,%%ymm7 \n" + "vpcmpeqd %%ymm8,%%ymm8,%%ymm8 \n" + "vpsrlw $0xf,%%ymm8,%%ymm8 \n" + "sub $0x4,%3 \n" + "jl 49f \n" + + // 4 pixel loop. + LABELALIGN + "40: \n" + "vbroadcasti128 (%0),%%ymm2 \n" + "lea 0x10(%0),%0 \n" + "vpshufb %%ymm5,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm6,%%ymm2,%%ymm3 \n" + "vpmaddubsw %%ymm7,%%ymm2,%%ymm4 \n" + "vpmaddwd %%ymm8,%%ymm3,%%ymm3 \n" + "vpmaddwd %%ymm8,%%ymm4,%%ymm4 \n" + "vperm2i128 $0x11,%%ymm4,%%ymm4,%%ymm2 \n" + "vpaddd (%2),%%ymm3,%%ymm3 \n" + "vpaddd 0x20(%2),%%ymm4,%%ymm4 \n" + "lea 0x40(%2),%2 \n" + "vpaddd %%ymm0,%%ymm3,%%ymm3 \n" + "vpaddd %%ymm0,%%ymm4,%%ymm4 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm3,(%1) \n" + "vmovdqu %%ymm4,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x4,%3 \n" + "jge 40b \n" + + "49: \n" + "add $0x3,%3 \n" + "jl 19f \n" + + // 1 pixel loop. + LABELALIGN + "10: \n" + "vpmovzxbd (%0),%%xmm2 \n" + "lea 0x4(%0),%0 \n" + "vpaddd %%xmm2,%%xmm0,%%xmm0 \n" + "vpaddd (%2),%%xmm0,%%xmm2 \n" + "lea 0x10(%2),%2 \n" + "vmovdqu %%xmm2,(%1) \n" + "lea 0x10(%1),%1 \n" + "sub $0x1,%3 \n" + "jge 10b \n" + + "19: \n" + "vzeroupper \n" + : "+r"(row), // %0 + "+r"(cumsum), // %1 + "+r"(previous_cumsum), // %2 + "+r"(width) // %3 + : "m"(kCumSumTranspose), // %4 + "m"(kCumSumMaddLo), // %5 + "m"(kCumSumMaddHi) // %6 + : "memory", "cc", "xmm0", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8"); +} +#endif // HAS_COMPUTECUMULATIVESUMROW_AVX2 + +#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 +void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, + const int32_t* botleft, + int width, + int area, + uint8_t* dst, + int count) { + asm volatile( + "vpxor %%xmm5,%%xmm5,%%xmm5 \n" + "vcvtsi2ssl %5,%%xmm5,%%xmm5 \n" + "vrcpss %%xmm5,%%xmm5,%%xmm4 \n" + "vpshufd $0x0,%%xmm4,%%xmm4 \n" + "sub $0x4,%3 \n" + "jl 19f \n" + "vinserti128 $0x1,%%xmm4,%%ymm4,%%ymm4 \n" + "cmpl $0x80,%5 \n" + "ja 10f \n" + + "vpcmpeqb %%xmm6,%%xmm6,%%xmm6 \n" + "vpsrld $0x10,%%xmm6,%%xmm6 \n" + "vcvtdq2ps %%xmm6,%%xmm6 \n" + "vaddss %%xmm6,%%xmm5,%%xmm6 \n" + "vmulss %%xmm4,%%xmm6,%%xmm6 \n" + "vcvtps2dq %%xmm6,%%xmm6 \n" + "vpackssdw %%xmm6,%%xmm6,%%xmm6 \n" + "vpbroadcastw %%xmm6,%%ymm6 \n" + + // 4 pixel small loop. + LABELALIGN + "4: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vpsubd 0x00(%0,%4,4),%%ymm0,%%ymm0\n" + "vpsubd 0x20(%0,%4,4),%%ymm1,%%ymm1\n" + "lea 0x40(%0),%0 \n" + "vpsubd (%1),%%ymm0,%%ymm0 \n" + "vpsubd 0x20(%1),%%ymm1,%%ymm1 \n" + "vpaddd 0x00(%1,%4,4),%%ymm0,%%ymm0\n" + "vpaddd 0x20(%1,%4,4),%%ymm1,%%ymm1\n" + "lea 0x40(%1),%1 \n" + "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm1 \n" + "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" + "vpshufd $0xd8,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%2) \n" + "lea 0x10(%2),%2 \n" + "sub $0x4,%3 \n" + "jge 4b \n" + "vzeroupper \n" + "jmp 19f \n" + + // 4 pixel loop. + LABELALIGN + "10: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vpsubd 0x00(%0,%4,4),%%ymm0,%%ymm0\n" + "vpsubd 0x20(%0,%4,4),%%ymm1,%%ymm1\n" + "lea 0x40(%0),%0 \n" + "vpsubd (%1),%%ymm0,%%ymm0 \n" + "vpsubd 0x20(%1),%%ymm1,%%ymm1 \n" + "vpaddd 0x00(%1,%4,4),%%ymm0,%%ymm0\n" + "vpaddd 0x20(%1,%4,4),%%ymm1,%%ymm1\n" + "lea 0x40(%1),%1 \n" + "vcvtdq2ps %%ymm0,%%ymm0 \n" + "vcvtdq2ps %%ymm1,%%ymm1 \n" + "vmulps %%ymm4,%%ymm0,%%ymm0 \n" + "vmulps %%ymm4,%%ymm1,%%ymm1 \n" + "vcvtps2dq %%ymm0,%%ymm0 \n" + "vcvtps2dq %%ymm1,%%ymm1 \n" + "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm1 \n" + "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" + "vpshufd $0xd8,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%2) \n" + "lea 0x10(%2),%2 \n" + "sub $0x4,%3 \n" + "jge 10b \n" + "vzeroupper \n" + + "19: \n" + "add $0x3,%3 \n" + "jl 99f \n" + + // 1 pixel loop. + LABELALIGN + "20: \n" + "vmovdqu (%0),%%xmm0 \n" + "vpsubd 0x00(%0,%4,4),%%xmm0,%%xmm0\n" + "lea 0x10(%0),%0 \n" + "vmovdqu 0x00(%1,%4,4),%%xmm1 \n" + "vpsubd (%1),%%xmm1,%%xmm1 \n" + "lea 0x10(%1),%1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vcvtdq2ps %%xmm0,%%xmm0 \n" + "vmulps %%xmm4,%%xmm0,%%xmm0 \n" + "vcvtps2dq %%xmm0,%%xmm0 \n" + "vpackssdw %%xmm0,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%2) \n" + "lea 0x4(%2),%2 \n" + "sub $0x1,%3 \n" + "jge 20b \n" + "99: \n" + : "+r"(topleft), // %0 + "+r"(botleft), // %1 + "+r"(dst), // %2 + "+r"(count) // %3 + : "r"((ptrdiff_t)(width)), // %4 + "r"(area) // %5 + : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5", "xmm6"); +} +#endif // HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 + #ifdef HAS_ARGBAFFINEROW_SSE2 // Copy ARGB pixels from source image with slope to a row of destination. LIBYUV_API diff --git a/source/row_neon64.cc b/source/row_neon64.cc index 53666bf41..2de205133 100644 --- a/source/row_neon64.cc +++ b/source/row_neon64.cc @@ -4414,6 +4414,214 @@ void SobelXYRow_NEON(const uint8_t* src_sobelx, : "cc", "memory", "v0", "v1", "v2", "v3"); } +#ifdef HAS_COMPUTECUMULATIVESUMROW_NEON +void ComputeCumulativeSumRow_NEON(const uint8_t* row, + int32_t* cumsum, + const int32_t* previous_cumsum, + int width) { + asm volatile( + "movi v0.4s, #0 \n" + "movi v1.16b, #0 \n" + "subs %w3, %w3, #4 \n" + "b.lt 49f \n" + + // 4 pixel loop. + "40: \n" + "ldr q2, [%0], #16 \n" + "ldp q16, q17, [%2] \n" + "ldp q18, q19, [%2, #32] \n" + "add %2, %2, #64 \n" + "ext v3.16b, v1.16b, v2.16b, #12 \n" + "uaddl v4.8h, v2.8b, v3.8b \n" + "uaddl2 v5.8h, v2.16b, v3.16b \n" + "add v5.8h, v5.8h, v4.8h \n" + "uaddw v16.4s, v16.4s, v4.4h \n" + "uaddw2 v17.4s, v17.4s, v4.8h \n" + "uaddw v18.4s, v18.4s, v5.4h \n" + "uaddw2 v19.4s, v19.4s, v5.8h \n" + "add v16.4s, v16.4s, v0.4s \n" + "add v17.4s, v17.4s, v0.4s \n" + "add v18.4s, v18.4s, v0.4s \n" + "add v19.4s, v19.4s, v0.4s \n" + "uaddw2 v0.4s, v0.4s, v5.8h \n" + "stp q16, q17, [%1] \n" + "stp q18, q19, [%1, #32] \n" + "add %1, %1, #64 \n" + "subs %w3, %w3, #4 \n" + "b.ge 40b \n" + + "49: \n" + "adds %w3, %w3, #3 \n" + "b.lt 19f \n" + + // 1 pixel loop. + "10: \n" + "ldr s2, [%0], #4 \n" + "ldr q16, [%2], #16 \n" + "ushll v2.8h, v2.8b, #0 \n" + "uaddw v0.4s, v0.4s, v2.4h \n" + "add v16.4s, v16.4s, v0.4s \n" + "str q16, [%1], #16 \n" + "subs %w3, %w3, #1 \n" + "b.ge 10b \n" + + "19: \n" + : "+r"(row), // %0 + "+r"(cumsum), // %1 + "+r"(previous_cumsum), // %2 + "+r"(width) // %3 + : + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", + "v18", "v19"); +} +#endif // HAS_COMPUTECUMULATIVESUMROW_NEON + +#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_NEON +void CumulativeSumToAverageRow_NEON(const int32_t* topleft, + const int32_t* botleft, + int width, + int area, + uint8_t* dst, + int count) { + const int32_t* topright; + const int32_t* botright; + int scale; + asm volatile( + "ucvtf s4, %w7 \n" + "fmov s5, #1.0 \n" + "fdiv s4, s5, s4 \n" + "subs %w3, %w3, #4 \n" + "b.lt 49f \n" + "add %4, %0, %8 \n" + "add %5, %1, %8 \n" + "sub %w6, %w7, #2 \n" + "cmp %w6, #126 \n" + "b.hi 40f \n" + "mov %w6, #32767 \n" + "add %w6, %w6, %w7 \n" + "udiv %w6, %w6, %w7 \n" + "dup v6.8h, %w6 \n" + + // 4 pixel small loop. + "4: \n" + "ldp q0, q1, [%0] \n" + "ldp q2, q3, [%0, #32] \n" + "add %0, %0, #64 \n" + "ldp q16, q17, [%4] \n" + "ldp q18, q19, [%4, #32] \n" + "add %4, %4, #64 \n" + "ldp q20, q21, [%1] \n" + "ldp q22, q23, [%1, #32] \n" + "add %1, %1, #64 \n" + "ldp q24, q25, [%5] \n" + "ldp q26, q27, [%5, #32] \n" + "add %5, %5, #64 \n" + "sub v0.4s, v0.4s, v16.4s \n" + "sub v1.4s, v1.4s, v17.4s \n" + "sub v2.4s, v2.4s, v18.4s \n" + "sub v3.4s, v3.4s, v19.4s \n" + "sub v24.4s, v24.4s, v20.4s \n" + "sub v25.4s, v25.4s, v21.4s \n" + "sub v26.4s, v26.4s, v22.4s \n" + "sub v27.4s, v27.4s, v23.4s \n" + "add v0.4s, v0.4s, v24.4s \n" + "add v1.4s, v1.4s, v25.4s \n" + "add v2.4s, v2.4s, v26.4s \n" + "add v3.4s, v3.4s, v27.4s \n" + "uzp1 v0.8h, v0.8h, v1.8h \n" + "uzp1 v2.8h, v2.8h, v3.8h \n" + "sqdmulh v0.8h, v0.8h, v6.8h \n" + "sqdmulh v2.8h, v2.8h, v6.8h \n" + "uzp1 v0.16b, v0.16b, v2.16b \n" + "str q0, [%2], #16 \n" + "subs %w3, %w3, #4 \n" + "b.ge 4b \n" + "b 49f \n" + + // 4 pixel loop. + "40: \n" + "ldp q0, q1, [%0] \n" + "ldp q2, q3, [%0, #32] \n" + "add %0, %0, #64 \n" + "ldp q16, q17, [%4] \n" + "ldp q18, q19, [%4, #32] \n" + "add %4, %4, #64 \n" + "ldp q20, q21, [%1] \n" + "ldp q22, q23, [%1, #32] \n" + "add %1, %1, #64 \n" + "ldp q24, q25, [%5] \n" + "ldp q26, q27, [%5, #32] \n" + "add %5, %5, #64 \n" + "sub v0.4s, v0.4s, v16.4s \n" + "sub v1.4s, v1.4s, v17.4s \n" + "sub v2.4s, v2.4s, v18.4s \n" + "sub v3.4s, v3.4s, v19.4s \n" + "sub v24.4s, v24.4s, v20.4s \n" + "sub v25.4s, v25.4s, v21.4s \n" + "sub v26.4s, v26.4s, v22.4s \n" + "sub v27.4s, v27.4s, v23.4s \n" + "add v0.4s, v0.4s, v24.4s \n" + "add v1.4s, v1.4s, v25.4s \n" + "add v2.4s, v2.4s, v26.4s \n" + "add v3.4s, v3.4s, v27.4s \n" + "scvtf v0.4s, v0.4s \n" + "scvtf v1.4s, v1.4s \n" + "scvtf v2.4s, v2.4s \n" + "scvtf v3.4s, v3.4s \n" + "fmul v0.4s, v0.4s, v4.s[0] \n" + "fmul v1.4s, v1.4s, v4.s[0] \n" + "fmul v2.4s, v2.4s, v4.s[0] \n" + "fmul v3.4s, v3.4s, v4.s[0] \n" + "fcvtzs v0.4s, v0.4s \n" + "fcvtzs v1.4s, v1.4s \n" + "fcvtzs v2.4s, v2.4s \n" + "fcvtzs v3.4s, v3.4s \n" + "uqxtn v0.4h, v0.4s \n" + "uqxtn2 v0.8h, v1.4s \n" + "uqxtn v2.4h, v2.4s \n" + "uqxtn2 v2.8h, v3.4s \n" + "uqxtn v0.8b, v0.8h \n" + "uqxtn2 v0.16b, v2.8h \n" + "str q0, [%2], #16 \n" + "subs %w3, %w3, #4 \n" + "b.ge 40b \n" + + "49: \n" + "adds %w3, %w3, #3 \n" + "b.lt 19f \n" + + // 1 pixel loop. + "10: \n" + "ldr q1, [%0, %8] \n" + "ldr q0, [%0], #16 \n" + "ldr q3, [%1, %8] \n" + "ldr q2, [%1], #16 \n" + "sub v0.4s, v0.4s, v1.4s \n" + "sub v3.4s, v3.4s, v2.4s \n" + "add v0.4s, v0.4s, v3.4s \n" + "scvtf v0.4s, v0.4s \n" + "fmul v0.4s, v0.4s, v4.s[0] \n" + "fcvtzs v0.4s, v0.4s \n" + "uqxtn v0.4h, v0.4s \n" + "uqxtn v0.8b, v0.8h \n" + "str s0, [%2], #4 \n" + "subs %w3, %w3, #1 \n" + "b.ge 10b \n" + "19: \n" + : "+r"(topleft), // %0 + "+r"(botleft), // %1 + "+r"(dst), // %2 + "+r"(count), // %3 + "=&r"(topright), // %4 + "=&r"(botright), // %5 + "=&r"(scale) // %6 + : "r"(area), // %7 + "r"((ptrdiff_t)width * 4) // %8 + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27"); +} +#endif // HAS_CUMULATIVESUMTOAVERAGEROW_NEON + // SobelX as a matrix is // -1 0 1 // -2 0 2 diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 0aecafdc8..5fd37ac27 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -2289,14 +2289,14 @@ static int TestBlur(int width, } const int kBpp = 4; const int kStride = width * kBpp; + const int kCumsumHeight = radius * 2 + 2; align_buffer_page_end(src_argb_a, (size_t)kStride * height + off); - align_buffer_page_end(dst_cumsum, (size_t)width * height * 16); + align_buffer_page_end(dst_cumsum, (size_t)width * kCumsumHeight * 16); align_buffer_page_end(dst_argb_c, (size_t)kStride * height); align_buffer_page_end(dst_argb_opt, (size_t)kStride * height); for (size_t i = 0; i < (size_t)kStride * height; ++i) { src_argb_a[i + off] = (fastrand() & 0xff); } - memset(dst_cumsum, 0, (size_t)width * height * 16); memset(dst_argb_c, 0, (size_t)kStride * height); memset(dst_argb_opt, 0, (size_t)kStride * height); @@ -2360,32 +2360,36 @@ TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlur_Opt)) { ASSERT_LE(max_diff, 1); } -static const int kBlurSmallSize = 5; +static const int kBlurSmallSize = 1; TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Any)) { int max_diff = - TestBlur(benchmark_width_ + 1, benchmark_height_, benchmark_iterations_, - disable_cpu_flags_, benchmark_cpu_info_, +1, 0, kBlurSmallSize); + TestBlur((benchmark_width_ + 3) / 4 + 1, (benchmark_height_ + 3) / 4, + benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_, + +1, 0, kBlurSmallSize); ASSERT_LE(max_diff, 1); } TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Unaligned)) { int max_diff = - TestBlur(benchmark_width_, benchmark_height_, benchmark_iterations_, - disable_cpu_flags_, benchmark_cpu_info_, +1, 1, kBlurSmallSize); + TestBlur((benchmark_width_ + 3) / 4, (benchmark_height_ + 3) / 4, + benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_, + +1, 1, kBlurSmallSize); ASSERT_LE(max_diff, 1); } TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Invert)) { int max_diff = - TestBlur(benchmark_width_, benchmark_height_, benchmark_iterations_, - disable_cpu_flags_, benchmark_cpu_info_, -1, 0, kBlurSmallSize); + TestBlur((benchmark_width_ + 3) / 4, (benchmark_height_ + 3) / 4, + benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_, + -1, 0, kBlurSmallSize); ASSERT_LE(max_diff, 1); } TEST_F(LibYUVPlanarTest, DISABLED_ARM(ARGBBlurSmall_Opt)) { int max_diff = - TestBlur(benchmark_width_, benchmark_height_, benchmark_iterations_, - disable_cpu_flags_, benchmark_cpu_info_, +1, 0, kBlurSmallSize); + TestBlur((benchmark_width_ + 3) / 4, (benchmark_height_ + 3) / 4, + benchmark_iterations_, disable_cpu_flags_, benchmark_cpu_info_, + +1, 0, kBlurSmallSize); ASSERT_LE(max_diff, 1); }