diff --git a/source/compare_neon64.cc b/source/compare_neon64.cc index 34fa3e494..0a4d52393 100644 --- a/source/compare_neon64.cc +++ b/source/compare_neon64.cc @@ -89,25 +89,30 @@ uint32_t SumSquareError_NEON(const uint8_t* src_a, uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a, const uint8_t* src_b, int count) { + // count is guaranteed to be a multiple of 32. uint32_t sse; asm volatile( - "movi v2.16b, #0 \n" + "movi v4.4s, #0 \n" + "movi v5.4s, #0 \n" "1: \n" - "ld1 {v0.16b}, [%0], #16 \n" - "ld1 {v1.16b}, [%1], #16 \n" - "subs %w2, %w2, #16 \n" + "ldp q0, q2, [%0], #32 \n" + "ldp q1, q3, [%1], #32 \n" + "subs %w2, %w2, #32 \n" "uabd v0.16b, v0.16b, v1.16b \n" + "uabd v1.16b, v2.16b, v3.16b \n" "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead - "udot v2.4s, v0.16b, v0.16b \n" + "udot v4.4s, v0.16b, v0.16b \n" + "udot v5.4s, v1.16b, v1.16b \n" "prfm pldl1keep, [%1, 448] \n" "b.gt 1b \n" - "addv s0, v2.4s \n" + "add v0.4s, v4.4s, v5.4s \n" + "addv s0, v0.4s \n" "fmov %w3, s0 \n" : "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(sse) : - : "memory", "cc", "v0", "v1", "v2"); + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5"); return sse; }