coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-11-09 14:49:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

[libyuv] Implement 8, 16, and 32-bit transpose using x86 ACEv1

Browse Source 
Add kCpuHasACEV1 detection to CpuId using CPUID leaves 7.1, 0x1D.2,
and 0x24.1 along with XCR0 state validation (0x001600e6). Add
CLANG_HAS_ACEV1, GCC_HAS_ACEV1, and LIBYUV_TARGET_ACEV1 macros for
64-bit x86 builds on Clang >= 11 and GCC >= 11.

Implement TransposeWxH_ACEV1, TransposeUVWxH_ACEV1,
TransposeWxH_16_ACEV1, and TransposeWxH_32_ACEV1 in rotate_gcc.cc
using Palette 2 16x16 dword tiles (ldtilecfg, tilemovcol, tilemovrow,
and tilerelease) with k-register tail masking for arbitrary widths and
heights.

Bug: none
Test: sde64 -future-ag -- ./libyuv_unittest --gtest_filter=*Transpose*
Change-Id: I0091ac00b173a59183f84ff9555920caef965293
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8704075
Commit-Queue: Frank Barchard <fbarchard@google.com>
Reviewed-by: richard winterton <rrwinterton@gmail.com>
This commit is contained in:
Frank Barchard 2026-11-02 09:58:19 -08:00 committed by libyuv-scoped@luci-project-accounts.iam.gserviceaccount.com
parent 38ca05f1eb
commit c15967905d
12 changed files with 741 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/environment_variables.md
View File

@ -35,6 +35,7 @@ By default the cpu is detected and the most advanced form of SIMD is used. But
LIBYUV_DISABLE_AMXINT8
LIBYUV_DISABLE_AVX512BMM
LIBYUV_DISABLE_AMXTF32
LIBYUV_DISABLE_ACEV1
## Arm CPUs

1
include/libyuv/cpu_id.h
View File

@ -64,6 +64,7 @@ static const int kCpuHasAVXVNNIINT8 = 0x8000000;
static const int kCpuHasAMXINT8 = 0x10000000;
static const int kCpuHasAVX512BMM = 0x20000000;
static const int kCpuHasAMXTF32 = 0x40000000;
static const int kCpuHasACEV1 = 0x80000000; // NOLINT
// These flags are only valid on LOONGARCH processors.
static const int kCpuHasLOONGARCH = 0x20;

18
include/libyuv/cpu_support.h
View File

@ -69,6 +69,9 @@ extern "C" {
#if (__GNUC__ >= 9)
#define GCC_HAS_AVX512 1
#endif // GNUC >= 9
#if (__GNUC__ >= 11)
#define GCC_HAS_ACEV1 1
#endif // GNUC >= 11
#endif // __GNUC__
// Function target attributes so GCC enables zmm16..zmm31 and k1..k7 clobbers
@ -91,6 +94,14 @@ extern "C" {
#define LIBYUV_TARGET_AVX512VNNIVBMI
#endif
#if defined(GCC_HAS_ACEV1)
#define LIBYUV_TARGET_ACEV1 \
__attribute__((target("amx-tile,avx512bw,avx512vl,avx512f,bmi2")))
#elif !defined(LIBYUV_ENABLE_ROWWIN) && \
!(defined(_MSC_VER) && !defined(__clang__))
#define LIBYUV_TARGET_ACEV1
#endif
// clang >= 3.4.0 required for AVX2.
#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
(defined(__x86_64__) || defined(__i386__))
@ -109,6 +120,13 @@ extern "C" {
#endif // clang >= 7
#endif // __clang__
// clang >= 11.0.0 required for ACEV1.
#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && defined(__x86_64__)
#if (__clang_major__ >= 11) && !defined(__APPLE__)
#define CLANG_HAS_ACEV1 1
#endif // clang >= 11
#endif // __clang__
// Visual C 2012 (or clang >= 3.4 with LIBYUV_ENABLE_ROWWIN) required for AVX2.
#if ((defined(_M_IX86) || defined(_M_X64)) && !defined(__clang__) && \
defined(_MSC_VER) && _MSC_VER >= 1700) || \

38
include/libyuv/rotate_row.h
View File

@ -77,6 +77,17 @@ extern "C" {
#define HAS_TRANSPOSEWX16_32_AVX512BW
#endif
// The following are available for ACEV1 GCC/clang x64 platforms:
#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
(defined(CLANG_HAS_ACEV1) || defined(GCC_HAS_ACEV1)) && \
!defined(LIBYUV_ENABLE_ROWWIN) && \
(!defined(_MSC_VER) || defined(__clang__))
#define HAS_TRANSPOSEWXH_ACEV1
#define HAS_TRANSPOSEUVWXH_ACEV1
#define HAS_TRANSPOSEWXH_16_ACEV1
#define HAS_TRANSPOSEWXH_32_ACEV1
#endif
#if !defined(LIBYUV_DISABLE_NEON) && \
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
#if defined(__aarch64__)
@ -186,6 +197,12 @@ void TransposeWx16_AVX512BW(const uint8_t* src,
uint8_t* dst,
int dst_stride,
int width);
void TransposeWxH_ACEV1(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height);
void TransposeWx8_LSX(const uint8_t* src,
int src_stride,
uint8_t* dst,
@ -283,6 +300,14 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src,
uint8_t* dst_b,
int dst_stride_b,
int width);
void TransposeUVWxH_ACEV1(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width,
int height);
void TransposeUVWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
@ -405,6 +430,12 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src,
uint16_t* dst,
int dst_stride,
int width);
void TransposeWxH_16_ACEV1(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width,
int height);
void TransposeWx8_16_NEON(const uint16_t* src,
int src_stride,
uint16_t* dst,
@ -537,6 +568,13 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src,
int dst_stride,
int width);
void TransposeWxH_32_ACEV1(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height);
void TransposeWx4_32_RVV(const uint8_t* src,
int src_stride,
uint8_t* dst,

20
source/cpu_id.cc
View File

@ -405,8 +405,11 @@ static SAFEBUFFERS int GetCpuFlags(void) {
int cpu_info1[4] = {0, 0, 0, 0};
int cpu_info7[4] = {0, 0, 0, 0};
int cpu_einfo7[4] = {0, 0, 0, 0};
int cpu_info1d[4] = {0, 0, 0, 0};
int cpu_einfo1d_2[4] = {0, 0, 0, 0};
int cpu_einfo1e[4] = {0, 0, 0, 0};
int cpu_info24[4] = {0, 0, 0, 0};
int cpu_einfo24_1[4] = {0, 0, 0, 0};
int cpu_info21[4] = {0, 0, 0, 0};
int cpu_amdinfo21[4] = {0, 0, 0, 0};
int cpu_extinfo0[4] = {0, 0, 0, 0};
@ -426,6 +429,12 @@ static SAFEBUFFERS int GetCpuFlags(void) {
if ((unsigned)cpu_extinfo0[0] >= 0x80000021u) {
CpuId(0x80000021, 0, cpu_amdinfo21);
}
if (cpu_info0[0] >= 0x1d) {
CpuId(0x1d, 0, cpu_info1d);
if (cpu_info1d[0] >= 2) {
CpuId(0x1d, 2, cpu_einfo1d_2);
}
}
if (cpu_info0[0] >= 0x1e) {
CpuId(0x1e, 1, cpu_einfo1e);
}
@ -434,6 +443,9 @@ static SAFEBUFFERS int GetCpuFlags(void) {
}
if (cpu_info0[0] >= 0x24) {
CpuId(0x24, 0, cpu_info24);
if (cpu_info24[0] >= 1) {
CpuId(0x24, 1, cpu_einfo24_1);
}
}
cpu_info = kCpuHasX86 | ((cpu_info1[3] & 0x04000000) ? kCpuHasSSE2 : 0) |
((cpu_info1[2] & 0x00000200) ? kCpuHasSSSE3 : 0) |
@ -467,6 +479,14 @@ static SAFEBUFFERS int GetCpuFlags(void) {
((cpu_einfo1e[0] & 0x00000040) ? kCpuHasAMXTF32 : 0);
if (cpu_info0[0] >= 0x24 && (cpu_einfo7[3] & 0x00080000)) {
cpu_info |= ((cpu_info24[1] & 0xFF) >= 2) ? kCpuHasAVX10_2 : 0;
if ((GetXCR0() & 0x001600e6) == 0x001600e6 &&
(cpu_einfo7[2] & 0x00000800) && (cpu_einfo1d_2[0] & 0xff) >= 1 &&
((cpu_info24[1] & 0xff) >= 2 ||
((cpu_info24[1] & 0xff) >= 1 &&
(cpu_einfo24_1[2] & 0x00000004))) &&
(cpu_einfo24_1[2] & 0x00000008)) {
cpu_info |= kCpuHasACEV1;
}
}
}
}

31
source/rotate.cc
View File

@ -47,7 +47,8 @@ void TransposePlane(const uint8_t* src,
int width,
int height) {
int i = height;
#if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX)
#if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX) || \
defined(HAS_TRANSPOSEWXH_ACEV1)
void (*TransposeWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width, int height) = NULL;
#endif
@ -111,6 +112,11 @@ void TransposePlane(const uint8_t* src,
TransposeWx16 = TransposeWx16_AVX512BW;
}
#endif
#if defined(HAS_TRANSPOSEWXH_ACEV1)
if (TestCpuFlag(kCpuHasACEV1)) {
TransposeWxH_fn = TransposeWxH_ACEV1;
}
#endif
#if defined(HAS_TRANSPOSEWX8_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
TransposeWx8 = TransposeWx8_Any_LSX;
@ -144,7 +150,8 @@ void TransposePlane(const uint8_t* src,
}
#endif
#if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX)
#if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX) || \
defined(HAS_TRANSPOSEWXH_ACEV1)
if (TransposeWxH_fn) {
TransposeWxH_fn(src, src_stride, dst, dst_stride, width, height);
return;
@ -381,7 +388,8 @@ void SplitTransposeUV(const uint8_t* src,
int width,
int height) {
int i = height;
#if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX)
#if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX) || \
defined(HAS_TRANSPOSEUVWXH_ACEV1)
void (*TransposeUVWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst_a,
int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
int width, int height) = NULL;
@ -452,6 +460,11 @@ void SplitTransposeUV(const uint8_t* src,
TransposeUVWx16 = TransposeUVWx16_AVX512BW;
}
#endif
#if defined(HAS_TRANSPOSEUVWXH_ACEV1)
if (TestCpuFlag(kCpuHasACEV1)) {
TransposeUVWxH_fn = TransposeUVWxH_ACEV1;
}
#endif
#if defined(HAS_TRANSPOSEUVWX8_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
TransposeUVWx8 = TransposeUVWx8_Any_WASMSIMD;
@ -469,7 +482,8 @@ void SplitTransposeUV(const uint8_t* src,
}
#endif
#if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX)
#if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX) || \
defined(HAS_TRANSPOSEUVWXH_ACEV1)
if (TransposeUVWxH_fn) {
TransposeUVWxH_fn(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
width, i);
@ -733,7 +747,7 @@ static void TransposePlane_16(const uint16_t* src,
int height) {
int i = height;
#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \
defined(HAS_TRANSPOSEWXH_16_HVX)
defined(HAS_TRANSPOSEWXH_16_HVX) || defined(HAS_TRANSPOSEWXH_16_ACEV1)
void (*TransposeWxH_16_fn)(const uint16_t* src, int src_stride, uint16_t* dst,
int dst_stride, int width, int height) = NULL;
#endif
@ -797,6 +811,11 @@ static void TransposePlane_16(const uint16_t* src,
TransposeWx32_16 = TransposeWx32_16_AVX512BW;
}
#endif
#if defined(HAS_TRANSPOSEWXH_16_ACEV1)
if (TestCpuFlag(kCpuHasACEV1)) {
TransposeWxH_16_fn = TransposeWxH_16_ACEV1;
}
#endif
#if defined(HAS_TRANSPOSEWX8_16_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
TransposeWx8_16 = TransposeWx8_16_Any_LSX;
@ -823,7 +842,7 @@ static void TransposePlane_16(const uint16_t* src,
#endif
#if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \
defined(HAS_TRANSPOSEWXH_16_HVX)
defined(HAS_TRANSPOSEWXH_16_HVX) || defined(HAS_TRANSPOSEWXH_16_ACEV1)
if (TransposeWxH_16_fn) {
TransposeWxH_16_fn(src, src_stride, dst, dst_stride, width, height);
return;

12
source/rotate_argb.cc
View File

@ -32,7 +32,7 @@ static int ARGBTranspose(const uint8_t* src_argb,
int height) {
#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \
defined(HAS_TRANSPOSEWXH_32_AVX2)
defined(HAS_TRANSPOSEWXH_32_AVX2) || defined(HAS_TRANSPOSEWXH_32_ACEV1)
void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width, int height) = NULL;
#endif
@ -76,6 +76,14 @@ static int ARGBTranspose(const uint8_t* src_argb,
TransposeWx16_32 = TransposeWx16_32_AVX512BW;
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_ACEV1)
if (TestCpuFlag(kCpuHasACEV1)) {
TransposeWxH_32 = TransposeWxH_32_ACEV1;
#if defined(HAS_TRANSPOSEWX16_32_AVX512BW)
TransposeWx16_32 = NULL;
#endif
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
TransposeWxH_32 = TransposeWxH_32_NEON;
@ -142,7 +150,7 @@ static int ARGBTranspose(const uint8_t* src_argb,
#endif
#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \
defined(HAS_TRANSPOSEWXH_32_AVX2)
defined(HAS_TRANSPOSEWXH_32_AVX2) || defined(HAS_TRANSPOSEWXH_32_ACEV1)
if (TransposeWxH_32) {
if (height > 0) {
TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,

417
source/rotate_gcc.cc
View File

@ -397,6 +397,223 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src,
}
#endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW)
#if defined(HAS_TRANSPOSEWXH_ACEV1) || defined(HAS_TRANSPOSEUVWXH_ACEV1) || \
defined(HAS_TRANSPOSEWXH_16_ACEV1) || defined(HAS_TRANSPOSEWXH_32_ACEV1)
// Palette 2 tile configuration: 8 tiles of 16 rows x 64 bytes (16 dwords).
alignas(64) static const uint8_t kTileCfgPalette2[64] = {2};
#endif
#if defined(HAS_TRANSPOSEWXH_ACEV1)
LIBYUV_TARGET_ACEV1
void TransposeWxH_ACEV1(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const uint8_t* src2;
const uint8_t* src3;
uint8_t* dst2;
int w;
int cnt;
asm volatile(
"ldtilecfg %[tilecfg] \n"
// Process up to 16 rows per iteration of the outer loop.
"1: \n"
"movl $16, %k[cnt] \n"
"cmpl $16, %k[height] \n"
"cmovlel %k[height], %k[cnt] \n"
"movl $-1, %%eax \n"
"bzhil %k[cnt], %%eax, %%eax \n"
"kmovw %%eax, %%k2 \n"
"movq %[src], %[src2] \n"
"movq %[dst], %[dst2] \n"
"movl %[width], %k[w] \n"
// Process up to 16 columns per iteration of the inner loop.
"2: \n"
"movl $16, %%eax \n"
"cmpl $16, %k[w] \n"
"cmovlel %k[w], %%eax \n"
"movl $-1, %k[cnt] \n"
"bzhil %%eax, %k[cnt], %k[cnt] \n"
"kmovw %k[cnt], %%k1 \n"
"movq %[src2], %[src3] \n"
// Load H <= 16 rows into TMM0 columns.
"movl $16, %k[cnt] \n"
"cmpl $16, %k[height] \n"
"cmovlel %k[height], %k[cnt] \n"
"xorl %%eax, %%eax \n"
"3: \n"
"vpmovzxbd (%[src3]), %%zmm0%{%%k1%}%{z%} \n"
// tilemovcol %eax, %zmm0, %tmm0
".byte 0x62, 0xf2, 0xfd, 0x48, 0x4b, 0xc0 \n"
"addq %[src_stride], %[src3] \n"
"incl %%eax \n"
"cmpl %k[cnt], %%eax \n"
"jne 3b \n"
// Store W <= 16 rows from TMM0 rows.
"movl $16, %k[cnt] \n"
"cmpl $16, %k[w] \n"
"cmovlel %k[w], %k[cnt] \n"
"xorl %%eax, %%eax \n"
"4: \n"
// tilemovrow %eax, %tmm0, %zmm0
".byte 0x62, 0xf2, 0x7d, 0x48, 0x4a, 0xc0 \n"
"vpmovdb %%zmm0, (%[dst2])%{%%k2%} \n"
"addq %[dst_stride], %[dst2] \n"
"incl %%eax \n"
"cmpl %k[cnt], %%eax \n"
"jne 4b \n"
"addq $16, %[src2] \n"
"subl $16, %k[w] \n"
"jg 2b \n"
"movq %[src_stride], %%rax \n"
"salq $4, %%rax \n"
"addq %%rax, %[src] \n"
"addq $16, %[dst] \n"
"subl $16, %k[height] \n"
"jg 1b \n"
"tilerelease \n"
"vzeroupper \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[height] "+r"(height), // %[height]
[src2] "=&r"(src2), // %[src2]
[src3] "=&r"(src3), // %[src3]
[dst2] "=&r"(dst2), // %[dst2]
[w] "=&r"(w), // %[w]
[cnt] "=&r"(cnt) // %[cnt]
: [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
[dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride]
[width] "rm"(width), // %[width]
[tilecfg] "m"(kTileCfgPalette2) // %[tilecfg]
: "memory", "cc", "rax", "k1", "k2", "xmm0"
#if defined(__clang__)
,
"tmm0"
#endif
);
}
#endif // defined(HAS_TRANSPOSEWXH_ACEV1)
#if defined(HAS_TRANSPOSEUVWXH_ACEV1)
LIBYUV_TARGET_ACEV1
void TransposeUVWxH_ACEV1(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
int dst_stride_a,
uint8_t* dst_b,
int dst_stride_b,
int width,
int height) {
const uint8_t* src2;
const uint8_t* src3;
uint8_t* dst2_a;
uint8_t* dst2_b;
int w;
int cnt;
asm volatile(
"ldtilecfg %[tilecfg] \n"
// Process up to 16 rows per iteration of the outer loop.
"1: \n"
"movl $16, %k[cnt] \n"
"cmpl $16, %k[height] \n"
"cmovlel %k[height], %k[cnt] \n"
"movl $-1, %%eax \n"
"bzhil %k[cnt], %%eax, %%eax \n"
"kmovw %%eax, %%k2 \n"
"movq %[src], %[src2] \n"
"movq %[dst_a], %[dst2_a] \n"
"movq %[dst_b], %[dst2_b] \n"
"movl %[width], %k[w] \n"
// Process up to 16 UV pairs per iteration of the inner loop.
"2: \n"
"movl $16, %%eax \n"
"cmpl $16, %k[w] \n"
"cmovlel %k[w], %%eax \n"
"movl $-1, %k[cnt] \n"
"bzhil %%eax, %k[cnt], %k[cnt] \n"
"kmovw %k[cnt], %%k1 \n"
"movq %[src2], %[src3] \n"
// Load H <= 16 rows of UV pairs into TMM0 columns.
"movl $16, %k[cnt] \n"
"cmpl $16, %k[height] \n"
"cmovlel %k[height], %k[cnt] \n"
"xorl %%eax, %%eax \n"
"3: \n"
"vpmovzxwd (%[src3]), %%zmm0%{%%k1%}%{z%} \n"
// tilemovcol %eax, %zmm0, %tmm0
".byte 0x62, 0xf2, 0xfd, 0x48, 0x4b, 0xc0 \n"
"addq %[src_stride], %[src3] \n"
"incl %%eax \n"
"cmpl %k[cnt], %%eax \n"
"jne 3b \n"
// Store W <= 16 U and V rows from TMM0 rows.
"movl $16, %k[cnt] \n"
"cmpl $16, %k[w] \n"
"cmovlel %k[w], %k[cnt] \n"
"xorl %%eax, %%eax \n"
"4: \n"
// tilemovrow %eax, %tmm0, %zmm0
".byte 0x62, 0xf2, 0x7d, 0x48, 0x4a, 0xc0 \n"
"vpmovdb %%zmm0, (%[dst2_a])%{%%k2%} \n"
"vpsrld $8, %%zmm0, %%zmm0 \n"
"vpmovdb %%zmm0, (%[dst2_b])%{%%k2%} \n"
"addq %[dst_stride_a], %[dst2_a] \n"
"addq %[dst_stride_b], %[dst2_b] \n"
"incl %%eax \n"
"cmpl %k[cnt], %%eax \n"
"jne 4b \n"
"addq $32, %[src2] \n"
"subl $16, %k[w] \n"
"jg 2b \n"
"movq %[src_stride], %%rax \n"
"salq $4, %%rax \n"
"addq %%rax, %[src] \n"
"addq $16, %[dst_a] \n"
"addq $16, %[dst_b] \n"
"subl $16, %k[height] \n"
"jg 1b \n"
"tilerelease \n"
"vzeroupper \n"
: [src] "+m"(src), // %[src]
[dst_a] "+m"(dst_a), // %[dst_a]
[dst_b] "+m"(dst_b), // %[dst_b]
[height] "+r"(height), // %[height]
[src2] "=&r"(src2), // %[src2]
[src3] "=&r"(src3), // %[src3]
[dst2_a] "=&r"(dst2_a), // %[dst2_a]
[dst2_b] "=&r"(dst2_b), // %[dst2_b]
[w] "=&r"(w), // %[w]
[cnt] "=&r"(cnt) // %[cnt]
: [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
[dst_stride_a] "r"((ptrdiff_t)dst_stride_a), // %[dst_stride_a]
[dst_stride_b] "r"((ptrdiff_t)dst_stride_b), // %[dst_stride_b]
[width] "m"(width), // %[width]
[tilecfg] "m"(kTileCfgPalette2) // %[tilecfg]
: "memory", "cc", "rax", "k1", "k2", "xmm0"
#if defined(__clang__)
,
"tmm0"
#endif
);
}
#endif // defined(HAS_TRANSPOSEUVWXH_ACEV1)
#if defined(__x86_64__)
#if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2)
// Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8
@ -1020,6 +1237,106 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src,
}
#endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW)
#if defined(HAS_TRANSPOSEWXH_16_ACEV1)
LIBYUV_TARGET_ACEV1
void TransposeWxH_16_ACEV1(const uint16_t* src,
int src_stride,
uint16_t* dst,
int dst_stride,
int width,
int height) {
const uint16_t* src2;
const uint16_t* src3;
uint16_t* dst2;
int w;
int cnt;
asm volatile(
"ldtilecfg %[tilecfg] \n"
// Process up to 16 16-bit rows per iteration of the outer loop.
"1: \n"
"movl $16, %k[cnt] \n"
"cmpl $16, %k[height] \n"
"cmovlel %k[height], %k[cnt] \n"
"movl $-1, %%eax \n"
"bzhil %k[cnt], %%eax, %%eax \n"
"kmovw %%eax, %%k2 \n"
"movq %[src], %[src2] \n"
"movq %[dst], %[dst2] \n"
"movl %[width], %k[w] \n"
// Process up to 16 16-bit columns per iteration of the inner loop.
"2: \n"
"movl $16, %%eax \n"
"cmpl $16, %k[w] \n"
"cmovlel %k[w], %%eax \n"
"movl $-1, %k[cnt] \n"
"bzhil %%eax, %k[cnt], %k[cnt] \n"
"kmovw %k[cnt], %%k1 \n"
"movq %[src2], %[src3] \n"
// Load H <= 16 rows into TMM0 columns.
"movl $16, %k[cnt] \n"
"cmpl $16, %k[height] \n"
"cmovlel %k[height], %k[cnt] \n"
"xorl %%eax, %%eax \n"
"3: \n"
"vpmovzxwd (%[src3]), %%zmm0%{%%k1%}%{z%} \n"
// tilemovcol %eax, %zmm0, %tmm0
".byte 0x62, 0xf2, 0xfd, 0x48, 0x4b, 0xc0 \n"
"addq %[src_stride], %[src3] \n"
"incl %%eax \n"
"cmpl %k[cnt], %%eax \n"
"jne 3b \n"
// Store W <= 16 rows from TMM0 rows.
"movl $16, %k[cnt] \n"
"cmpl $16, %k[w] \n"
"cmovlel %k[w], %k[cnt] \n"
"xorl %%eax, %%eax \n"
"4: \n"
// tilemovrow %eax, %tmm0, %zmm0
".byte 0x62, 0xf2, 0x7d, 0x48, 0x4a, 0xc0 \n"
"vpmovdw %%zmm0, (%[dst2])%{%%k2%} \n"
"addq %[dst_stride], %[dst2] \n"
"incl %%eax \n"
"cmpl %k[cnt], %%eax \n"
"jne 4b \n"
"addq $32, %[src2] \n"
"subl $16, %k[w] \n"
"jg 2b \n"
"movq %[src_stride], %%rax \n"
"salq $4, %%rax \n"
"addq %%rax, %[src] \n"
"addq $32, %[dst] \n"
"subl $16, %k[height] \n"
"jg 1b \n"
"tilerelease \n"
"vzeroupper \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[height] "+r"(height), // %[height]
[src2] "=&r"(src2), // %[src2]
[src3] "=&r"(src3), // %[src3]
[dst2] "=&r"(dst2), // %[dst2]
[w] "=&r"(w), // %[w]
[cnt] "=&r"(cnt) // %[cnt]
: [src_stride] "r"((ptrdiff_t)src_stride * 2), // %[src_stride]
[dst_stride] "r"((ptrdiff_t)dst_stride * 2), // %[dst_stride]
[width] "rm"(width), // %[width]
[tilecfg] "m"(kTileCfgPalette2) // %[tilecfg]
: "memory", "cc", "rax", "k1", "k2", "xmm0"
#if defined(__clang__)
,
"tmm0"
#endif
);
}
#endif // defined(HAS_TRANSPOSEWXH_16_ACEV1)
#if defined(HAS_TRANSPOSEWX4_32_AVX2)
// Transpose 32 bit values (ARGB). Read a column of 4 pixels across 8 rows per
// AVX2 iteration (with a 4-row VEX 128-bit tail) and write 4 destination rows.
@ -1705,6 +2022,106 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src,
}
#endif // defined(HAS_TRANSPOSEWX16_32_AVX512BW)
#if defined(HAS_TRANSPOSEWXH_32_ACEV1)
LIBYUV_TARGET_ACEV1
void TransposeWxH_32_ACEV1(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const uint8_t* src2;
const uint8_t* src3;
uint8_t* dst2;
int w;
int cnt;
asm volatile(
"ldtilecfg %[tilecfg] \n"
// Process up to 16 32-bit rows per iteration of the outer loop.
"1: \n"
"movl $16, %k[cnt] \n"
"cmpl $16, %k[height] \n"
"cmovlel %k[height], %k[cnt] \n"
"movl $-1, %%eax \n"
"bzhil %k[cnt], %%eax, %%eax \n"
"kmovw %%eax, %%k2 \n"
"movq %[src], %[src2] \n"
"movq %[dst], %[dst2] \n"
"movl %[width], %k[w] \n"
// Process up to 16 32-bit columns per iteration of the inner loop.
"2: \n"
"movl $16, %%eax \n"
"cmpl $16, %k[w] \n"
"cmovlel %k[w], %%eax \n"
"movl $-1, %k[cnt] \n"
"bzhil %%eax, %k[cnt], %k[cnt] \n"
"kmovw %k[cnt], %%k1 \n"
"movq %[src2], %[src3] \n"
// Load H <= 16 rows into TMM0 columns.
"movl $16, %k[cnt] \n"
"cmpl $16, %k[height] \n"
"cmovlel %k[height], %k[cnt] \n"
"xorl %%eax, %%eax \n"
"3: \n"
"vmovdqu32 (%[src3]), %%zmm0%{%%k1%}%{z%} \n"
// tilemovcol %eax, %zmm0, %tmm0
".byte 0x62, 0xf2, 0xfd, 0x48, 0x4b, 0xc0 \n"
"addq %[src_stride], %[src3] \n"
"incl %%eax \n"
"cmpl %k[cnt], %%eax \n"
"jne 3b \n"
// Store W <= 16 rows from TMM0 rows.
"movl $16, %k[cnt] \n"
"cmpl $16, %k[w] \n"
"cmovlel %k[w], %k[cnt] \n"
"xorl %%eax, %%eax \n"
"4: \n"
// tilemovrow %eax, %tmm0, %zmm0
".byte 0x62, 0xf2, 0x7d, 0x48, 0x4a, 0xc0 \n"
"vmovdqu32 %%zmm0, (%[dst2])%{%%k2%} \n"
"addq %[dst_stride], %[dst2] \n"
"incl %%eax \n"
"cmpl %k[cnt], %%eax \n"
"jne 4b \n"
"addq $64, %[src2] \n"
"subl $16, %k[w] \n"
"jg 2b \n"
"movq %[src_stride], %%rax \n"
"salq $4, %%rax \n"
"addq %%rax, %[src] \n"
"addq $64, %[dst] \n"
"subl $16, %k[height] \n"
"jg 1b \n"
"tilerelease \n"
"vzeroupper \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[height] "+r"(height), // %[height]
[src2] "=&r"(src2), // %[src2]
[src3] "=&r"(src3), // %[src3]
[dst2] "=&r"(dst2), // %[dst2]
[w] "=&r"(w), // %[w]
[cnt] "=&r"(cnt) // %[cnt]
: [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
[dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride]
[width] "rm"(width), // %[width]
[tilecfg] "m"(kTileCfgPalette2) // %[tilecfg]
: "memory", "cc", "rax", "k1", "k2", "xmm0"
#if defined(__clang__)
,
"tmm0"
#endif
);
}
#endif // defined(HAS_TRANSPOSEWXH_32_ACEV1)
#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) ||
// defined(__i386__)) && !defined(LIBYUV_ENABLE_ROWWIN)

2
unit_test/cpu_test.cc
View File

@ -208,6 +208,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) {
int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8);
int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM);
int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32);
int has_acev1 = TestCpuFlag(kCpuHasACEV1);
printf("Has X86 0x%x\n", has_x86);
printf("Has SSE2 0x%x\n", has_sse2);
printf("Has SSSE3 0x%x\n", has_ssse3);
@ -232,6 +233,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) {
printf("Has AMXINT8 0x%x\n", has_amxint8);
printf("Has AVX512BMM 0x%x\n", has_avx512bmm);
printf("Has AMXTF32 0x%x\n", has_amxtf32);
printf("Has ACEV1 0x%x\n", has_acev1);
}
#endif // defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) ||
// defined(_M_X64)

204
unit_test/rotate_test.cc
View File

@ -1050,6 +1050,210 @@ TEST_F(LibYUVRotateTest, TransposeWxH_32_Test) {
}
#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
#if defined(HAS_TRANSPOSEWXH_ACEV1)
TEST_F(LibYUVRotateTest, TransposeWxH_ACEV1_Test) {
if (!TestCpuFlag(kCpuHasACEV1)) {
return;
}
const int kMaxW = 35;
const int kMaxH = 35;
align_buffer_page_end(src_pixels, kMaxW * kMaxH);
align_buffer_page_end(dst_pixels_c, kMaxW * kMaxH);
align_buffer_page_end(dst_pixels_opt, kMaxW * kMaxH);
MemRandomize(src_pixels, kMaxW * kMaxH);
for (int h = 1; h <= kMaxH; ++h) {
for (int w = 1; w <= kMaxW; ++w) {
const int src_stride = w;
const int dst_stride = h;
const int src_bytes = src_stride * h;
const int dst_bytes = dst_stride * w;
const uint8_t* src = src_pixels + kMaxW * kMaxH - src_bytes;
uint8_t* dst_c = dst_pixels_c + kMaxW * kMaxH - dst_bytes;
uint8_t* dst_opt = dst_pixels_opt + kMaxW * kMaxH - dst_bytes;
memset(dst_c, 1, dst_bytes);
memset(dst_opt, 2, dst_bytes);
MaskCpuFlags(disable_cpu_flags_);
TransposePlane(src, src_stride, dst_c, dst_stride, w, h);
MaskCpuFlags(benchmark_cpu_info_);
TransposeWxH_ACEV1(src, src_stride, dst_opt, dst_stride, w, h);
ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << w << "x" << h;
memset(dst_c, 3, dst_bytes);
memset(dst_opt, 4, dst_bytes);
MaskCpuFlags(disable_cpu_flags_);
TransposePlane(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
dst_c + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride, w,
h);
MaskCpuFlags(benchmark_cpu_info_);
TransposeWxH_ACEV1(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
dst_opt + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride,
w, h);
ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes))
<< "neg " << w << "x" << h;
}
}
free_aligned_buffer_page_end(src_pixels);
free_aligned_buffer_page_end(dst_pixels_c);
free_aligned_buffer_page_end(dst_pixels_opt);
}
#endif // defined(HAS_TRANSPOSEWXH_ACEV1)
#if defined(HAS_TRANSPOSEUVWXH_ACEV1)
TEST_F(LibYUVRotateTest, TransposeUVWxH_ACEV1_Test) {
if (!TestCpuFlag(kCpuHasACEV1)) {
return;
}
const int kMaxW = 35;
const int kMaxH = 35;
align_buffer_page_end(src_pixels, kMaxW * kMaxH * 2);
align_buffer_page_end(dst_u_c, kMaxW * kMaxH);
align_buffer_page_end(dst_v_c, kMaxW * kMaxH);
align_buffer_page_end(dst_u_opt, kMaxW * kMaxH);
align_buffer_page_end(dst_v_opt, kMaxW * kMaxH);
MemRandomize(src_pixels, kMaxW * kMaxH * 2);
for (int h = 1; h <= kMaxH; ++h) {
for (int w = 1; w <= kMaxW; ++w) {
const int src_stride = w * 2;
const int dst_stride = h;
const int src_bytes = src_stride * h;
const int dst_bytes = dst_stride * w;
const uint8_t* src = src_pixels + kMaxW * kMaxH * 2 - src_bytes;
uint8_t* u_c = dst_u_c + kMaxW * kMaxH - dst_bytes;
uint8_t* v_c = dst_v_c + kMaxW * kMaxH - dst_bytes;
uint8_t* u_opt = dst_u_opt + kMaxW * kMaxH - dst_bytes;
uint8_t* v_opt = dst_v_opt + kMaxW * kMaxH - dst_bytes;
memset(u_c, 1, dst_bytes);
memset(v_c, 2, dst_bytes);
memset(u_opt, 3, dst_bytes);
memset(v_opt, 4, dst_bytes);
MaskCpuFlags(disable_cpu_flags_);
SplitTransposeUV(src, src_stride, u_c, dst_stride, v_c, dst_stride, w, h);
MaskCpuFlags(benchmark_cpu_info_);
TransposeUVWxH_ACEV1(src, src_stride, u_opt, dst_stride, v_opt,
dst_stride, w, h);
ASSERT_EQ(0, memcmp(u_c, u_opt, dst_bytes)) << w << "x" << h;
ASSERT_EQ(0, memcmp(v_c, v_opt, dst_bytes)) << w << "x" << h;
memset(u_c, 5, dst_bytes);
memset(v_c, 6, dst_bytes);
memset(u_opt, 7, dst_bytes);
memset(v_opt, 8, dst_bytes);
MaskCpuFlags(disable_cpu_flags_);
SplitTransposeUV(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
u_c + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride,
v_c + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride, w,
h);
MaskCpuFlags(benchmark_cpu_info_);
TransposeUVWxH_ACEV1(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
u_opt + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride,
v_opt + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride,
w, h);
ASSERT_EQ(0, memcmp(u_c, u_opt, dst_bytes)) << "neg " << w << "x" << h;
ASSERT_EQ(0, memcmp(v_c, v_opt, dst_bytes)) << "neg " << w << "x" << h;
}
}
free_aligned_buffer_page_end(src_pixels);
free_aligned_buffer_page_end(dst_u_c);
free_aligned_buffer_page_end(dst_v_c);
free_aligned_buffer_page_end(dst_u_opt);
free_aligned_buffer_page_end(dst_v_opt);
}
#endif // defined(HAS_TRANSPOSEUVWXH_ACEV1)
#if defined(HAS_TRANSPOSEWXH_16_ACEV1)
TEST_F(LibYUVRotateTest, TransposeWxH_16_ACEV1_Test) {
if (!TestCpuFlag(kCpuHasACEV1)) {
return;
}
const int kMaxW = 35;
const int kMaxH = 35;
align_buffer_page_end_16(src_pixels, kMaxW * kMaxH);
align_buffer_page_end_16(dst_pixels_c, kMaxW * kMaxH);
align_buffer_page_end_16(dst_pixels_opt, kMaxW * kMaxH);
for (int i = 0; i < kMaxW * kMaxH; ++i) {
src_pixels[i] = fastrand() & 0xffff;
}
for (int h = 1; h <= kMaxH; ++h) {
for (int w = 1; w <= kMaxW; ++w) {
const int src_stride = w;
const int dst_stride = h;
const int src_elems = src_stride * h;
const int dst_elems = dst_stride * w;
const uint16_t* src = src_pixels + kMaxW * kMaxH - src_elems;
uint16_t* dst_c = dst_pixels_c + kMaxW * kMaxH - dst_elems;
uint16_t* dst_opt = dst_pixels_opt + kMaxW * kMaxH - dst_elems;
memset(dst_c, 1, dst_elems * 2);
memset(dst_opt, 2, dst_elems * 2);
MaskCpuFlags(disable_cpu_flags_);
RotatePlane_16(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride, dst_c,
dst_stride, w, h, kRotate90);
MaskCpuFlags(benchmark_cpu_info_);
TransposeWxH_16_ACEV1(src, src_stride, dst_opt, dst_stride, w, h);
ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_elems * 2)) << w << "x" << h;
}
}
free_aligned_buffer_page_end_16(src_pixels);
free_aligned_buffer_page_end_16(dst_pixels_c);
free_aligned_buffer_page_end_16(dst_pixels_opt);
}
#endif // defined(HAS_TRANSPOSEWXH_16_ACEV1)
#if defined(HAS_TRANSPOSEWXH_32_ACEV1)
TEST_F(LibYUVRotateTest, TransposeWxH_32_ACEV1_Test) {
if (!TestCpuFlag(kCpuHasACEV1)) {
return;
}
const int kMaxW = 35;
const int kMaxH = 35;
align_buffer_page_end(src_pixels, kMaxW * kMaxH * 4);
align_buffer_page_end(dst_pixels_c, kMaxW * kMaxH * 4);
align_buffer_page_end(dst_pixels_opt, kMaxW * kMaxH * 4);
MemRandomize(src_pixels, kMaxW * kMaxH * 4);
for (int h = 1; h <= kMaxH; ++h) {
for (int w = 1; w <= kMaxW; ++w) {
const int src_stride = w * 4;
const int dst_stride = h * 4;
const int src_bytes = src_stride * h;
const int dst_bytes = dst_stride * w;
const uint8_t* src = src_pixels + kMaxW * kMaxH * 4 - src_bytes;
uint8_t* dst_c = dst_pixels_c + kMaxW * kMaxH * 4 - dst_bytes;
uint8_t* dst_opt = dst_pixels_opt + kMaxW * kMaxH * 4 - dst_bytes;
memset(dst_c, 1, dst_bytes);
memset(dst_opt, 2, dst_bytes);
TransposeWxH_32_C(src, src_stride, dst_c, dst_stride, w, h);
TransposeWxH_32_ACEV1(src, src_stride, dst_opt, dst_stride, w, h);
ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << w << "x" << h;
memset(dst_c, 3, dst_bytes);
memset(dst_opt, 4, dst_bytes);
TransposeWxH_32_C(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
dst_c + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride, w,
h);
TransposeWxH_32_ACEV1(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride,
dst_opt + (ptrdiff_t)(w - 1) * dst_stride,
-dst_stride, w, h);
ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes))
<< "neg " << w << "x" << h;
}
}
free_aligned_buffer_page_end(src_pixels);
free_aligned_buffer_page_end(dst_pixels_c);
free_aligned_buffer_page_end(dst_pixels_opt);
}
#endif // defined(HAS_TRANSPOSEWXH_32_ACEV1)
#endif // ENABLE_ROW_TESTS
} // namespace libyuv

3
unit_test/unit_test.cc
View File

@ -215,6 +215,9 @@ static int TestCpuEnv(int cpu_info) {
if (TestEnv("LIBYUV_DISABLE_AMXTF32")) {
cpu_info &= ~libyuv::kCpuHasAMXTF32;
}
if (TestEnv("LIBYUV_DISABLE_ACEV1")) {
cpu_info &= ~libyuv::kCpuHasACEV1;
}
#endif
if (TestEnv("LIBYUV_DISABLE_ASM")) {
cpu_info = libyuv::kCpuInitialized;

2
util/cpuid.c
View File

@ -218,6 +218,7 @@ int main(int argc, const char* argv[]) {
int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8);
int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM);
int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32);
int has_acev1 = TestCpuFlag(kCpuHasACEV1);
printf("Has X86 0x%x\n", has_x86);
printf("Has SSE2 0x%x\n", has_sse2);
printf("Has SSSE3 0x%x\n", has_ssse3);
@ -242,6 +243,7 @@ int main(int argc, const char* argv[]) {
printf("Has AMXINT8 0x%x\n", has_amxint8);
printf("Has AVX512BMM 0x%x\n", has_avx512bmm);
printf("Has AMXTF32 0x%x\n", has_amxtf32);
printf("Has ACEV1 0x%x\n", has_acev1);
#ifdef __linux__
// Test VDPPHPS instruction