Fix missing inline asm register clobbers

Change-Id: Ic573c8d7d8921a1dd00e4481b854a8168d0c94f7
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8543110
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>

GEMINI.md

@@ -55,6 +55,17 @@ and compiler compatibility.
 
 1.  **Feature Macros**: Use the `HAS_` macros in `include/libyuv/row.h` to
     enable or disable specific instruction set extensions.
+2.  **Inline asm clobbers (all architectures)**: Every register written by an
+    inline asm block must be listed in its clobber list: vector registers,
+    general purpose registers, mask/predicate registers, `"cc"` when flags
+    are set (`subs`, `cmp`, `whilelt`, ...) and `"memory"` when the block
+    stores. Declare them even if the ABI treats them as caller-saved: with
+    LTO/LTCG the compiler may keep values live in any register across the
+    asm, and other OS ABIs (e.g. Windows) differ in which registers are
+    callee-saved. Never remove a clobber because it "is not needed by the
+    ABI". Use real register names, not numbers (`"v22"`, not `"22"`). Order
+    the list `"memory"` first (when stores occur), then `"cc"` (on
+    architectures with flags), then registers.
 
 ### x86 Architectures (32-bit and 64-bit)
 
@@ -63,6 +74,10 @@ and compiler compatibility.
 2.  **MSVC Intrinsics (`row_win.cc`)**: When converting `row_gcc.cc` to
     `row_win.cc`, use `_mm512_permutex2var_epi8`, not `_mm512_permi2var_epi8`
     intrinsic.
+3.  **Clobber names**: Always declare vector clobbers as `"xmmN"`, never
+    `"ymmN"` or `"zmmN"`, including for AVX2 and AVX512 code. Clang for
+    Windows must preserve xmm6-xmm15 and does not save them when they are
+    declared as ymm or zmm.
 
 ### AArch64
 

source/compare_gcc.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for GCC x86 and x64.
 #if !defined(LIBYUV_DISABLE_X86) &&               \
     (defined(__x86_64__) || defined(__i386__)) && \

source/compare_neon.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 #if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
     !defined(__aarch64__)
 

source/compare_neon64.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
 
 // 256 bits at a time

source/rotate_gcc.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for GCC and clang x64. 32 bit x86 uses rotate_win.cc.
 #if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
     !defined(LIBYUV_ENABLE_ROWWIN)

source/rotate_neon.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 #if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
     !defined(__aarch64__)
 

source/rotate_neon64.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for GCC Neon armv8 64 bit.
 #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
 

source/rotate_sme.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
     defined(__aarch64__)
 

source/row_gcc.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for GCC x86 and x64.
 #if !defined(LIBYUV_DISABLE_X86) &&               \
     (defined(__x86_64__) || defined(__i386__)) && \

source/row_lasx.cc

@@ -12,6 +12,13 @@
 
 #include "libyuv/row.h"
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 #if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
 #include "libyuv/loongson_intrinsics.h"
 
@@ -1960,7 +1967,9 @@ void ARGBToYMatrixRow_LASX(const uint8_t* src_argb,
         "+&r"(dst_y),     // %1
         "+&r"(width)      // %2
       : "r"(c), "r"(shuff)
-      : "memory");
+      : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+        "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr20",
+        "$xr21");
 }
 
 void RGBToYMatrixRow_LASX(const uint8_t* src_rgba,
@@ -2018,7 +2027,9 @@ void RGBToYMatrixRow_LASX(const uint8_t* src_rgba,
         "+&r"(width)      // %2
       : "r"(c),           // %3
         "r"(shuff)        // %4
-      : "memory");
+      : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
+        "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14",
+        "$xr15", "$xr16", "$xr17");
 }
 // undef for unified sources build
 #undef ALPHA_VAL

source/row_lsx.cc

@@ -12,6 +12,13 @@
 
 #include "libyuv/row.h"
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 #if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
 #include "libyuv/loongson_intrinsics.h"
 
@@ -2637,7 +2644,8 @@ void ARGBToYMatrixRow_LSX(const uint8_t* src_argb,
         "+&r"(dst_y),     // %1
         "+&r"(width)      // %2
       : "r"(c)
-      : "memory");
+      : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+        "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr20");
 }
 
 void RGBToYMatrixRow_LSX(const uint8_t* src_rgba,
@@ -2688,7 +2696,9 @@ void RGBToYMatrixRow_LSX(const uint8_t* src_rgba,
         "+&r"(width)      // %2
       : "r"(c),           // %3
         "r"(shuff)        // %4
-      : "memory");
+      : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6",
+        "$vr7", "$vr8", "$vr9", "$vr10", "$vr12", "$vr13", "$vr14", "$vr15",
+        "$vr16", "$vr17");
 }
 
 // undef for unified sources build

source/row_neon.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for GCC Neon
 #if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
     !defined(__aarch64__)
@@ -1670,8 +1677,8 @@ void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2,
         "+r"(dst_uv),       // %2
         "+r"(width)         // %3
       :
-      : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6",
-        "d7"  // Clobber List
+      : "memory", "cc", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8",
+        "d9"  // Clobber List
   );
 }
 
@@ -2981,8 +2988,8 @@ void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width) {
       : "+r"(dst_argb),  // %0
         "+r"(width)      // %1
       :
-      : "cc", "memory", "q0", "q1", "q2", "q3", "q10", "q11", "q12", "q13",
-        "q14", "q15");
+      : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q10", "q11", "q12",
+        "q13", "q14", "q15");
 }
 
 // Tranform 8 ARGB pixels (32 bytes) with color matrix.

source/row_neon64.cc

@@ -20,6 +20,13 @@ extern "C" {
 // STn over ZIP1+ST1
 // Exynos M1, M2, M3 are slow with ST2, ST3 and ST4 instructions.
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for GCC Neon armv8 64 bit.
 #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
 
@@ -2148,7 +2155,7 @@ void ARGB1555ToARGBRow_NEON(const uint8_t* src_argb1555,
         "+r"(dst_argb),      // %1
         "+r"(width)          // %2
       :
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v29"  // Clobber List
+      : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v29"  // Clobber List
   );
 }
 
@@ -3489,7 +3496,8 @@ void ARGB4444ToYRow_NEON(const uint8_t* src_argb4444,
         "+r"(dst_y),         // %1
         "+r"(width)          // %2
       :
-      : "cc", "memory", "v0", "v1", "v2", "v3", "v24", "v25", "v26", "v27");
+      : "memory", "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v24", "v25",
+        "v26", "v27");
 }
 
 // ARGB expects first 3 values to contain RGB and 4th value is ignored.

source/row_rvv.cc

@@ -17,10 +17,22 @@
 
 #include "libyuv/row.h"
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for RVV (RISC-V Vector extension)
 #if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
 #include <assert.h>
 
+// vsetvli/vsetivli write the vl and vtype CSRs. Only "vl" is declared, and
+// only for GCC; clang historically rejected it. The "vtype" register is
+// intentionally not declared, and whether it should be is unresolved.
+// TODO(fbarchard): consider adding "vtype" (and "vl" for clang) to the
+// clobber list once confirmed with compiler maintainers.
 #ifndef RVV_VL_CLOBBER
 #ifdef __clang__
 #define RVV_VL_CLOBBER

source/row_sme.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
     defined(__aarch64__)
 

source/row_sve.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
 
 #define RGBTOARGB8_SVE_2X                                 \

source/scale_gcc.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for GCC x86 and x64.
 #if !defined(LIBYUV_DISABLE_X86) &&               \
     (defined(__x86_64__) || defined(__i386__)) && \
@@ -2763,7 +2770,7 @@ void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb,
         "+r"(dst_argb),  // %1
         "+r"(dst_width)  // %2
       :
-      : "memory", "cc", "xmm0", "xmm1");
+      : "memory", "cc", "xmm0", "xmm1", "xmm2");
 }
 
 void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb,

source/scale_neon.cc

@@ -16,6 +16,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for GCC Neon.
 #if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
     !defined(__aarch64__)
@@ -40,7 +47,7 @@ void ScaleRowDown2_NEON(const uint8_t* src_ptr,
         "+r"(dst),       // %1
         "+r"(dst_width)  // %2
       :
-      : "q0", "q1"  // Clobber List
+      : "memory", "cc", "q0", "q1"  // Clobber List
   );
 }
 
@@ -61,7 +68,7 @@ void ScaleRowDown2Linear_NEON(const uint8_t* src_ptr,
         "+r"(dst),       // %1
         "+r"(dst_width)  // %2
       :
-      : "q0", "q1"  // Clobber List
+      : "memory", "cc", "q0", "q1"  // Clobber List
   );
 }
 
@@ -92,7 +99,7 @@ void ScaleRowDown2Box_NEON(const uint8_t* src_ptr,
         "+r"(dst),         // %2
         "+r"(dst_width)    // %3
       :
-      : "q0", "q1", "q2", "q3"  // Clobber List
+      : "memory", "cc", "q0", "q1", "q2", "q3"  // Clobber List
   );
 }
 
@@ -296,7 +303,7 @@ void ScaleRowDown38_NEON(const uint8_t* src_ptr,
         "+r"(dst_ptr),   // %1
         "+r"(dst_width)  // %2
       : "r"(&kShuf38)    // %3
-      : "d0", "d1", "d2", "d3", "d4", "d5", "memory", "cc");
+      : "memory", "cc", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7");
 }
 
 // 32x3 -> 12x1
@@ -725,7 +732,7 @@ void ScaleRowUp2_Linear_16_NEON(const uint16_t* src_ptr,
         "+r"(dst_width),  // %2
         "+r"(src_temp)    // %3
       :
-      : "memory", "cc", "q0", "q1", "q2", "q15"  // Clobber List
+      : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q15"  // Clobber List
   );
 }
 

source/scale_neon64.cc

@@ -17,6 +17,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for GCC Neon armv8 64 bit.
 #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
 
@@ -431,8 +438,8 @@ void ScaleRowDown38_3_Box_NEON(const uint8_t* src_ptr,
         [tblArray2] "r"(kScaleRowDown38_3_BoxIndices2),  // %[tblArray2]
         [tblArray3] "r"(kScaleRowDown38_3_BoxIndices3),  // %[tblArray3]
         [tblArray4] "r"(kScaleRowDown38_NarrowIndices)   // %[tblArray4]
-      : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v20", "v21", "22", "23",
-        "24", "v27", "v28", "v29", "v30", "v31");
+      : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v20", "v21", "v22",
+        "v23", "v24", "v25", "v27", "v28", "v29", "v30", "v31");
 }
 
 static const uvec8 kScaleRowDown38_2_BoxIndices1[] = {
@@ -721,7 +728,7 @@ void ScaleRowUp2_Linear_16_NEON(const uint16_t* src_ptr,
         "+r"(dst_ptr),   // %2
         "+r"(dst_width)  // %3
       :
-      : "memory", "cc", "v0", "v1", "v2", "v31"  // Clobber List
+      : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v31"  // Clobber List
   );
 }
 

source/scale_rvv.cc

@@ -18,6 +18,13 @@
 #include "libyuv/row.h"
 #include "libyuv/scale_row.h"
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 // This module is for RVV (RISC-V Vector extension)
 #if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
 #include <assert.h>
@@ -26,6 +33,11 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// vsetvli/vsetivli write the vl and vtype CSRs. Only "vl" is declared, and
+// only for GCC; clang historically rejected it. The "vtype" register is
+// intentionally not declared, and whether it should be is unresolved.
+// TODO(fbarchard): consider adding "vtype" (and "vl" for clang) to the
+// clobber list once confirmed with compiler maintainers.
 #ifndef RVV_VL_CLOBBER
 #ifdef __clang__
 #define RVV_VL_CLOBBER

source/scale_sme.cc

@@ -15,6 +15,13 @@ namespace libyuv {
 extern "C" {
 #endif
 
+// Note: every register written by an inline asm block (vector, general
+// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be
+// listed in its clobber list, even when the ABI treats it as caller-saved.
+// With LTO/LTCG the compiler may keep values live in any register across the
+// asm statement, and other OS ABIs (e.g. Windows) differ in which registers
+// are callee-saved. List "memory" first, then "cc", then registers.
+
 #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
     defined(__aarch64__)
 
@@ -356,7 +363,7 @@ __arm_locally_streaming void ScaleUVRowDown2Linear_SME(const uint8_t* src_uv,
         [dst_width] "+r"(dst_width),  // %[dst_width]
         [vl] "=r"(vl)                 // %[vl]
       :
-      : "z0", "z1", "p0", "p1");
+      : "memory", "cc", "z0", "z1", "p0", "p1");
 }
 
 #define SCALEUVROWDOWN2BOX_SVE                               \
@@ -408,7 +415,8 @@ __arm_locally_streaming void ScaleUVRowDown2Box_SME(const uint8_t* src_uv,
         [dst_width] "+r"(dst_width),  // %[dst_width]
         [vl] "=r"(vl)                 // %[vl]
       :
-      : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0", "p1");
+      : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
+        "p1");
 }
 
 #undef SCALEUVROWDOWN2BOX_SVE