coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-28 18:07:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

[libyuv] Optimize ARGBRotate for Intel SSE2, AVX2, and AVX-512

Browse Source 
Optimize 32-bit ARGBRotate (0, 90, 180, 270) across Intel x86/x64
architectures (SSE2, AVX2, AVX512BW), with matching kernels for ARM
(NEON, SVE2, SME), WASM, LoongArch (LSX, LASX), and RISC-V (RVV):

- Replace column-by-column ScaleARGBRowDownEven gather loops in
  ARGBTranspose (90/270) with full-plane horizontal tile transposes:
  - AVX512BW (TransposeWxH_32_AVX512BW / TransposeNx4_32_AVX512BW):
    Transpose 16x16 blocks across zmm0..zmm31 using 4-stage
    vpunpckldq/hdq, vpunpcklqdq/hqdq, and vshufi32x4 butterflies with
    k-masked right and bottom tails, eliminating Any wrappers and scalar
    C edge fallbacks.
  - AVX2 (TransposeWxH_32_AVX2 / TransposeNx4_32_AVX2):
    Transpose 8x8 blocks across ymm0..ymm15 using vpunpck* and
    vperm2i128 with vpmaskmovd masked tails.
  - SSE2 (Transpose4x4_32_SSE2 / Transpose4x4_32_Any_SSE2):
    Transpose 4x4 blocks in xmm registers.
- Implement ARGBRotate180 directly via ARGBMirror with negative height,
  avoiding intermediate row buffer allocations and extra copies.
- Implement ARGBRotate0 via ARGBCopy.

Speedup vs Before (1275x1650, US Letter 150 DPI, 1000 iterations):
CPU                  | 0 deg   | 90 deg  | 180 deg | 270 deg
---------------------|---------|---------|---------|--------
Intel Ice Lake       | 1.02x   | 1.34x   | 1.18x   | 1.33x
Intel Skylake-X      | 1.02x   | 1.20x   | 1.07x   | 1.13x
Intel Emerald Rapids | 1.01x   | 1.07x   | 1.00x   | 1.07x
Intel AVX-512 vs C   | 1.00x   | 2.67x   | 2.41x   | 2.63x

R=rrwinterton@gmail.com

Bug: b/42280915
Test: LIBYUV_WIDTH=1275 LIBYUV_HEIGHT=1650 LIBYUV_REPEAT=1000 ./out/Release/libyuv_unittest --gtest_filter=*ARGBRotate*
Change-Id: Ib308a225c17e46eab083a9da4033598fcf0c303c
This commit is contained in:
Frank Barchard 2026-10-08 03:05:41 -07:00
parent 4ce4edd50e
commit c772fa496c
28 changed files with 2621 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Android.bp
View File

@ -54,6 +54,7 @@ cc_library {
"source/rotate_gcc.cc",
"source/rotate_neon.cc",
"source/rotate_neon64.cc",
"source/rotate_rvv.cc",
"source/row_any.cc",
"source/row_common.cc",
"source/row_gcc.cc",

1
Android.mk
View File

@ -30,6 +30,7 @@ LOCAL_SRC_FILES := \
source/rotate_gcc.cc \
source/rotate_neon.cc \
source/rotate_neon64.cc \
source/rotate_rvv.cc \
source/rotate_win.cc \
source/row_any.cc \
source/row_common.cc \

7
BUILD.gn
View File

@ -250,7 +250,10 @@ if (libyuv_use_neon) {
if (libyuv_use_sve) {
static_library("libyuv_sve") {
sources = [ "source/row_sve.cc" ]
sources = [
"source/rotate_sve.cc",
"source/row_sve.cc",
]
deps = [ ":libyuv_internal" ]
@ -288,6 +291,7 @@ if (libyuv_use_sme) {
if (libyuv_use_rvv && !libyuv_disable_rvv) {
static_library("libyuv_rvv") {
sources = [
"source/rotate_rvv.cc",
"source/row_rvv.cc",
"source/scale_rvv.cc",
]
@ -339,6 +343,7 @@ if (libyuv_use_lasx) {
static_library("libyuv_lasx") {
sources = [
# LASX Source Files
"source/rotate_lasx.cc",
"source/row_lasx.cc",
]

3
CMakeLists.txt
View File

@ -42,6 +42,7 @@ set ( ly_common_source_files
${ly_src_dir}/rotate_argb.cc
${ly_src_dir}/rotate_common.cc
${ly_src_dir}/rotate_gcc.cc
${ly_src_dir}/rotate_rvv.cc
${ly_src_dir}/rotate_wasm.cc
${ly_src_dir}/rotate_win.cc
${ly_src_dir}/row_any.cc
@ -114,6 +115,7 @@ if(NOT MSVC)
# Enable AArch64 SVE kernels.
add_library(${ly_lib_name}_sve OBJECT
${ly_src_dir}/rotate_sve.cc
${ly_src_dir}/row_sve.cc)
target_compile_options(${ly_lib_name}_sve PRIVATE -march=armv8.5-a+i8mm+sve2)
list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_sve>)
@ -215,6 +217,7 @@ if(LOONGARCH64)
endif()
if(SUPPORTS_LASX)
add_library(${ly_lib_name}_lasx OBJECT
${ly_src_dir}/rotate_lasx.cc
${ly_src_dir}/row_lasx.cc)
target_compile_options(${ly_lib_name}_lasx PRIVATE -mlasx)
list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_lasx>)

129
include/libyuv/rotate_row.h
View File

@ -32,12 +32,15 @@ extern "C" {
#define HAS_TRANSPOSEWX8_SSSE3
#define HAS_TRANSPOSEUVWX8_SSE2
#define HAS_TRANSPOSE4X4_32_SSE2
#define HAS_TRANSPOSE4X4_32_AVX2
#define HAS_TRANSPOSENX4_32_AVX2
#define HAS_TRANSPOSEWXH_32_AVX2
#define HAS_TRANSPOSEWX16_AVX2
#define HAS_TRANSPOSEUVWX16_AVX2
#if defined(__x86_64__) || defined(_M_X64)
#define HAS_TRANSPOSEWX16_AVX512BW
#define HAS_TRANSPOSEUVWX16_AVX512BW
#define HAS_TRANSPOSENX4_32_AVX512BW
#define HAS_TRANSPOSEWXH_32_AVX512BW
#endif
#endif
@ -48,7 +51,8 @@ extern "C" {
#define HAS_TRANSPOSEWX8_SSSE3
#define HAS_TRANSPOSEUVWX8_SSE2
#define HAS_TRANSPOSE4X4_32_SSE2
#define HAS_TRANSPOSE4X4_32_AVX2
#define HAS_TRANSPOSENX4_32_AVX2
#define HAS_TRANSPOSEWXH_32_AVX2
#define HAS_TRANSPOSEWX16_AVX2
#define HAS_TRANSPOSEUVWX16_AVX2
#endif
@ -58,6 +62,8 @@ extern "C" {
defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_TRANSPOSEWX16_AVX512BW
#define HAS_TRANSPOSEUVWX16_AVX512BW
#define HAS_TRANSPOSENX4_32_AVX512BW
#define HAS_TRANSPOSEWXH_32_AVX512BW
#endif
#if !defined(LIBYUV_DISABLE_NEON) && \
@ -68,18 +74,34 @@ extern "C" {
#define HAS_TRANSPOSEWX8_NEON
#endif
#define HAS_TRANSPOSEUVWX8_NEON
#define HAS_TRANSPOSE4X4_32_NEON
#define HAS_TRANSPOSENX4_32_NEON
#define HAS_TRANSPOSEWXH_32_NEON
#endif
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#define HAS_TRANSPOSENX4_32_SVE2
#endif
#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
defined(__aarch64__)
#define HAS_TRANSPOSEWXH_SME
#define HAS_TRANSPOSEUVWXH_SME
#define HAS_TRANSPOSEWXH_32_SME
#endif
#if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx)
#define HAS_TRANSPOSEWX16_LSX
#define HAS_TRANSPOSEUVWX16_LSX
#define HAS_TRANSPOSE4X4_32_LSX
#endif
#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
#define HAS_TRANSPOSE4X4_32_LASX
#endif
#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
#define HAS_TRANSPOSENX4_32_RVV
#define HAS_TRANSPOSEWXH_32_RVV
#endif
#if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__)
@ -331,8 +353,34 @@ void TransposeWx1_16_C(const uint16_t* src,
int dst_stride,
int width);
// Transpose 32 bit values (ARGB)
void Transpose4x4_32_NEON(const uint8_t* src,
void TransposeWxH_32_C(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height);
void TransposeWxH_32_SME(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height);
void TransposeNx4_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
void TransposeWxH_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height);
void TransposeNx4_32_SVE2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
@ -344,7 +392,52 @@ void Transpose4x4_32_SSE2(const uint8_t* src,
int dst_stride,
int width);
void Transpose4x4_32_AVX2(const uint8_t* src,
void TransposeNx4_32_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
void TransposeWxH_32_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height);
void TransposeNx4_32_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
void TransposeWxH_32_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height);
void TransposeNx4_32_RVV(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
void TransposeWxH_32_RVV(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height);
void Transpose4x4_32_LSX(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
void Transpose4x4_32_LASX(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
@ -362,6 +455,30 @@ void Transpose4x4_32_C(const uint8_t* src,
int dst_stride,
int width);
void Transpose4x4_32_Any_SSE2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
void Transpose4x4_32_Any_LSX(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
void Transpose4x4_32_Any_LASX(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
void Transpose4x4_32_Any_WASMSIMD(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width);
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

9
include/libyuv/row.h
View File

@ -818,6 +818,7 @@ extern "C" {
#define HAS_YUY2TOUV422ROW_LSX
#define HAS_YUY2TOYROW_LSX
#define HAS_ARGBTOYMATRIXROW_LSX
#define HAS_COPYROW_LSX
#define HAS_RGBTOYMATRIXROW_LSX
#endif
@ -846,6 +847,7 @@ extern "C" {
#define HAS_ARGBTORGB565DITHERROW_LASX
#define HAS_ARGBTORGB565ROW_LASX
#define HAS_ARGBTOYMATRIXROW_LASX
#define HAS_COPYROW_LASX
#define HAS_I422ALPHATOARGBROW_LASX
#define HAS_I422TOARGB1555ROW_LASX
#define HAS_I422TOARGB4444ROW_LASX
@ -884,6 +886,7 @@ extern "C" {
#define HAS_ARGBEXTRACTALPHAROW_RVV
#define HAS_ARGBMULTIPLYROW_RVV
#define HAS_ARGBTOAR64ROW_RVV
#define HAS_ARGBMIRRORROW_RVV
#define HAS_ARGBTOUV444MATRIXROW_RVV
#define HAS_ARGBTOUVMATRIXROW_RVV
#define HAS_ARGBTOYMATRIXROW_RVV
@ -2786,6 +2789,8 @@ void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
void ARGBMirrorRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width);
void ARGBMirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width);
void ARGBMirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width);
void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width);
void ARGBMirrorRow_WASMSIMD(const uint8_t* src, uint8_t* dst, int width);
void ARGBMirrorRow_C(const uint8_t* src, uint8_t* dst, int width);
void ARGBMirrorRow_Any_AVX2(const uint8_t* src_ptr,
uint8_t* dst_ptr,
@ -3912,11 +3917,15 @@ void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_SME(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_RVV(const uint8_t* src, uint8_t* dst, int count);
void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_C(const uint8_t* src, uint8_t* dst, int count);
void CopyRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_AVX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_AVX512BW(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count);

2
libyuv.gypi
View File

@ -59,9 +59,11 @@
'source/rotate_argb.cc',
'source/rotate_common.cc',
'source/rotate_gcc.cc',
'source/rotate_lasx.cc',
'source/rotate_lsx.cc',
'source/rotate_neon.cc',
'source/rotate_neon64.cc',
'source/rotate_rvv.cc',
'source/rotate_win.cc',
'source/row_any.cc',
'source/row_common.cc',

2
linux.mk
View File

@ -32,9 +32,11 @@ LOCAL_OBJ_FILES := \
source/rotate_argb.o \
source/rotate_common.o \
source/rotate_gcc.o \
source/rotate_lasx.o \
source/rotate_lsx.o \
source/rotate_neon.o \
source/rotate_neon64.o \
source/rotate_rvv.o \
source/rotate_win.o \
source/row_any.o \
source/row_common.o \

15
source/planar_functions.cc
View File

@ -95,6 +95,16 @@ void CopyPlane(const uint8_t* src_y,
CopyRow = CopyRow_RVV;
}
#endif
#if defined(HAS_COPYROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LSX : CopyRow_Any_LSX;
}
#endif
#if defined(HAS_COPYROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
CopyRow = IS_ALIGNED(width, 64) ? CopyRow_LASX : CopyRow_Any_LASX;
}
#endif
// Copy plane
for (y = 0; y < height; ++y) {
@ -2999,6 +3009,11 @@ int ARGBMirror(const uint8_t* src_argb,
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ARGBMirrorRow = ARGBMirrorRow_RVV;
}
#endif
// Mirror plane
for (y = 0; y < height; ++y) {

10
source/rotate.cc
View File

@ -286,6 +286,16 @@ void RotatePlane180(const uint8_t* src,
CopyRow = CopyRow_RVV;
}
#endif
#if defined(HAS_COPYROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LSX : CopyRow_Any_LSX;
}
#endif
#if defined(HAS_COPYROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
CopyRow = IS_ALIGNED(width, 64) ? CopyRow_LASX : CopyRow_Any_LASX;
}
#endif
// Odd height will harmlessly mirror the middle row twice.
for (y = 0; y < half_height; ++y) {

31
source/rotate_any.cc
View File

@ -91,6 +91,37 @@ TUVANY(TransposeUVWx16_Any_WASMSIMD,
#endif
#undef TUVANY
#define TANY32(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \
void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
int dst_stride, int width) { \
int r = width & MASK; \
int n = width - r; \
if (n > 0) { \
TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
} \
if (r > 0) { \
TPOS_C(src + (ptrdiff_t)n * src_stride, src_stride, \
dst + (ptrdiff_t)n * 4, dst_stride, 4, r); \
} \
}
#ifdef HAS_TRANSPOSE4X4_32_SSE2
TANY32(Transpose4x4_32_Any_SSE2, Transpose4x4_32_SSE2, TransposeWxH_32_C, 3)
#endif
#ifdef HAS_TRANSPOSE4X4_32_LSX
TANY32(Transpose4x4_32_Any_LSX, Transpose4x4_32_LSX, TransposeWxH_32_C, 3)
#endif
#ifdef HAS_TRANSPOSE4X4_32_LASX
TANY32(Transpose4x4_32_Any_LASX, Transpose4x4_32_LASX, TransposeWxH_32_C, 7)
#endif
#ifdef HAS_TRANSPOSE4X4_32_WASMSIMD
TANY32(Transpose4x4_32_Any_WASMSIMD,
Transpose4x4_32_WASMSIMD,
TransposeWxH_32_C,
3)
#endif
#undef TANY32
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

232
source/rotate_argb.cc
View File

@ -16,8 +16,8 @@
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
#include "libyuv/rotate.h"
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
#include "libyuv/scale_row.h" /* for ScaleARGBRowDownEven_ */
#ifdef __cplusplus
namespace libyuv {
@ -30,57 +30,104 @@ static int ARGBTranspose(const uint8_t* src_argb,
int dst_stride_argb,
int width,
int height) {
int i;
int src_pixel_step = src_stride_argb >> 2;
void (*ScaleARGBRowDownEven)(
const uint8_t* src_argb, ptrdiff_t src_stride_argb, int src_step,
uint8_t* dst_argb, int dst_width) = ScaleARGBRowDownEven_C;
#if defined(HAS_TRANSPOSEWXH_32_AVX2) || \
defined(HAS_TRANSPOSEWXH_32_AVX512BW) || \
defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
defined(HAS_TRANSPOSEWXH_32_RVV)
void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width, int height) = NULL;
#endif
void (*Transpose4x4_32)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = NULL;
// Check stride is a multiple of 4.
if (src_stride_argb & 3) {
return -1;
}
#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2)
#if defined(HAS_TRANSPOSE4X4_32_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_SSE2;
Transpose4x4_32 = Transpose4x4_32_Any_SSE2;
if (IS_ALIGNED(height, 4)) { // Width of dest.
ScaleARGBRowDownEven = ScaleARGBRowDownEven_SSE2;
Transpose4x4_32 = Transpose4x4_32_SSE2;
}
}
#endif
#if defined(HAS_SCALEARGBROWDOWNEVEN_NEON)
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
TransposeWxH_32 = TransposeWxH_32_AVX2;
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
TransposeWxH_32 = TransposeWxH_32_AVX512BW;
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_NEON;
if (IS_ALIGNED(height, 4)) { // Width of dest.
ScaleARGBRowDownEven = ScaleARGBRowDownEven_NEON;
}
TransposeWxH_32 = TransposeWxH_32_NEON;
}
#endif
#if defined(HAS_SCALEARGBROWDOWNEVEN_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_LSX;
if (IS_ALIGNED(height, 4)) { // Width of dest.
ScaleARGBRowDownEven = ScaleARGBRowDownEven_LSX;
}
#if defined(HAS_TRANSPOSENX4_32_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
Transpose4x4_32 = TransposeNx4_32_SVE2;
}
#endif
#if defined(HAS_SCALEARGBROWDOWNEVEN_RVV)
#if defined(HAS_TRANSPOSEWXH_32_SME)
if (TestCpuFlag(kCpuHasSME)) {
TransposeWxH_32 = TransposeWxH_32_SME;
}
#endif
#if defined(HAS_TRANSPOSEWXH_32_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_RVV;
TransposeWxH_32 = TransposeWxH_32_RVV;
}
#endif
#if defined(HAS_SCALEARGBROWDOWNEVEN_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_WASMSIMD;
#if defined(HAS_TRANSPOSE4X4_32_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
Transpose4x4_32 = Transpose4x4_32_Any_LSX;
if (IS_ALIGNED(height, 4)) { // Width of dest.
ScaleARGBRowDownEven = ScaleARGBRowDownEven_WASMSIMD;
Transpose4x4_32 = Transpose4x4_32_LSX;
}
}
#endif
#if defined(HAS_TRANSPOSE4X4_32_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
Transpose4x4_32 = Transpose4x4_32_Any_LASX;
if (IS_ALIGNED(height, 8)) { // Width of dest.
Transpose4x4_32 = Transpose4x4_32_LASX;
}
}
#endif
#if defined(HAS_TRANSPOSE4X4_32_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
Transpose4x4_32 = Transpose4x4_32_Any_WASMSIMD;
if (IS_ALIGNED(height, 4)) { // Width of dest.
Transpose4x4_32 = Transpose4x4_32_WASMSIMD;
}
}
#endif
for (i = 0; i < width; ++i) { // column of source to row of dest.
ScaleARGBRowDownEven(src_argb, 0, src_pixel_step, dst_argb, height);
dst_argb += dst_stride_argb;
src_argb += 4;
#if defined(HAS_TRANSPOSEWXH_32_AVX2) || \
defined(HAS_TRANSPOSEWXH_32_AVX512BW) || \
defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \
defined(HAS_TRANSPOSEWXH_32_RVV)
if (TransposeWxH_32) {
TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
height);
return 0;
}
#endif
if (Transpose4x4_32) {
while (width >= 4) {
Transpose4x4_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
height);
src_argb += 4 * 4;
dst_argb += 4 * (ptrdiff_t)dst_stride_argb;
width -= 4;
}
}
if (width > 0) {
TransposeWxH_32_C(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
}
return 0;
}
@ -121,128 +168,9 @@ static int ARGBRotate180(const uint8_t* src_argb,
int dst_stride_argb,
int width,
int height) {
// Swap first and last row and mirror the content. Uses a temporary row.
const uint8_t* src_bot = src_argb + (ptrdiff_t)src_stride_argb * (height - 1);
uint8_t* dst_bot = dst_argb + (ptrdiff_t)dst_stride_argb * (height - 1);
int half_height = (height + 1) >> 1;
int y;
void (*ARGBMirrorRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width) =
ARGBMirrorRow_C;
void (*CopyRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width) =
CopyRow_C;
if (width > INT_MAX / 4) {
return -1;
}
align_buffer_64(row, width * 4);
if (!row)
return 1;
#if defined(HAS_ARGBMIRRORROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBMirrorRow = ARGBMirrorRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGBMirrorRow = ARGBMirrorRow_NEON;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGBMirrorRow = ARGBMirrorRow_Any_SSE2;
if (IS_ALIGNED(width, 4)) {
ARGBMirrorRow = ARGBMirrorRow_SSE2;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBMirrorRow = ARGBMirrorRow_Any_AVX2;
if (IS_ALIGNED(width, 8)) {
ARGBMirrorRow = ARGBMirrorRow_AVX2;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBMirrorRow = ARGBMirrorRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBMIRRORROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBMirrorRow = ARGBMirrorRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
ARGBMirrorRow = ARGBMirrorRow_LSX;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
ARGBMirrorRow = ARGBMirrorRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
ARGBMirrorRow = ARGBMirrorRow_LASX;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
ARGBMirrorRow = ARGBMirrorRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 4)) {
ARGBMirrorRow = ARGBMirrorRow_WASMSIMD;
}
}
#endif
#if defined(HAS_COPYROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
}
#endif
#if defined(HAS_COPYROW_AVX)
if (TestCpuFlag(kCpuHasAVX)) {
CopyRow = IS_ALIGNED(width * 4, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
}
#endif
#if defined(HAS_COPYROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
CopyRow =
IS_ALIGNED(width * 4, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW;
}
#endif
#if defined(HAS_COPYROW_ERMS)
if (TestCpuFlag(kCpuHasERMS)) {
CopyRow = CopyRow_ERMS;
}
#endif
#if defined(HAS_COPYROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
}
#endif
#if defined(HAS_COPYROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
CopyRow = CopyRow_SVE2;
}
#endif
#if defined(HAS_COPYROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
CopyRow = CopyRow_SME;
}
#endif
#if defined(HAS_COPYROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
CopyRow = CopyRow_RVV;
}
#endif
// Odd height will harmlessly mirror the middle row twice.
for (y = 0; y < half_height; ++y) {
ARGBMirrorRow(src_argb, row, width); // Mirror first row into a buffer
ARGBMirrorRow(src_bot, dst_argb, width); // Mirror last row into first row
CopyRow(row, dst_bot, width * 4); // Copy first mirrored row into last
src_argb += src_stride_argb;
dst_argb += dst_stride_argb;
src_bot -= src_stride_argb;
dst_bot -= dst_stride_argb;
}
free_aligned_buffer_64(row);
return 0;
// Rotate by 180 is a vertical flip (negative height) and horizontal mirror.
return ARGBMirror(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width,
-height);
}
LIBYUV_API

16
source/rotate_common.cc
View File

@ -206,6 +206,22 @@ void Transpose4x4_32_C(const uint8_t* src,
}
}
void TransposeWxH_32_C(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
int i;
for (i = 0; i < width; ++i) {
int j;
for (j = 0; j < height; ++j) {
Copy4(dst + (ptrdiff_t)i * dst_stride + j * 4,
src + (ptrdiff_t)j * src_stride + i * 4);
}
}
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

874
source/rotate_gcc.cc
View File

@ -737,61 +737,845 @@ void Transpose4x4_32_SSE2(const uint8_t* src,
}
#endif // defined(HAS_TRANSPOSE4X4_32_SSE2)
#if defined(HAS_TRANSPOSE4X4_32_AVX2)
// Transpose 32 bit values (ARGB)
void Transpose4x4_32_AVX2(const uint8_t* src,
#if defined(HAS_TRANSPOSENX4_32_AVX2)
// Transpose 32 bit values (ARGB) in Nx4 tiles with 2 blocks of 4x4 at a time
// in ymm0..ymm3 and vpmaskmovd tail for arbitrary N.
void TransposeNx4_32_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
asm volatile(
// Main loop transpose 2 blocks of 4x4. Read a column, write a row.
"1: \n"
"vmovdqu (%0),%%xmm0 \n" // a b c d
"vmovdqu (%0,%3),%%xmm1 \n" // e f g h
"lea (%0,%3,2),%0 \n" // src += stride * 2
"vmovdqu (%0),%%xmm2 \n" // i j k l
"vmovdqu (%0,%3),%%xmm3 \n" // m n o p
"lea (%0,%3,2),%0 \n" // src += stride * 2
"sub $0x8,%[w] \n"
"jl 2f \n"
"vinserti128 $1,(%0),%%ymm0,%%ymm0 \n" // a b c d
"vinserti128 $1,(%0,%3),%%ymm1,%%ymm1 \n" // e f g h
"lea (%0,%3,2),%0 \n" // src += stride * 2
"vinserti128 $1,(%0),%%ymm2,%%ymm2 \n" // i j k l
"vinserti128 $1,(%0,%3),%%ymm3,%%ymm3 \n" // m n o p
"lea (%0,%3,2),%0 \n" // src += stride * 2
// Main loop: 2 blocks of 4x4 (8 source rows -> 8 dest columns / 32 B).
"1: \n"
"vmovdqu (%[src]),%%xmm0 \n" // a b c d
"vmovdqu (%[src],%[s]),%%xmm1 \n" // e f g h
"lea (%[src],%[s],2),%[src] \n" // src += stride * 2
"vmovdqu (%[src]),%%xmm2 \n" // i j k l
"vmovdqu (%[src],%[s]),%%xmm3 \n" // m n o p
"lea (%[src],%[s],2),%[src] \n" // src += stride * 2
// Transpose 2x2
"vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" // a e b f from row 0, 1
"vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" // i m j n from row 2, 3
"vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" // c g d h from row 0, 1
"vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" // k o l p from row 2, 3
"vinserti128 $1,(%[src]),%%ymm0,%%ymm0 \n"
"vinserti128 $1,(%[src],%[s]),%%ymm1,%%ymm1 \n"
"lea (%[src],%[s],2),%[src] \n"
"vinserti128 $1,(%[src]),%%ymm2,%%ymm2 \n"
"vinserti128 $1,(%[src],%[s]),%%ymm3,%%ymm3 \n"
"lea (%[src],%[s],2),%[src] \n"
// Transpose 4x4
"vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" // a e i m from row 0, 1
"vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" // b f j n from row 0, 1
"vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" // c g k o from row 2, 3
"vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" // d h l p from row 2, 3
// Transpose 2x2 across both 128-bit lanes
"vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n"
"vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n"
"vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n"
"vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n"
"vmovdqu %%ymm0,(%1) \n"
"lea 32(%1,%4),%1 \n" // dst += stride + 32
"vmovdqu %%ymm1,-32(%1) \n"
"vmovdqu %%ymm2,-32(%1,%4) \n"
"vmovdqu %%ymm3,-32(%1,%4,2) \n"
"sub %4,%1 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
"vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+rm"(width) // %2
: "r"((ptrdiff_t)(src_stride)), // %3
"r"((ptrdiff_t)(dst_stride)) // %4
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
// Transpose 4x4 across both 128-bit lanes
"vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n"
"vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n"
"vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n"
"vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n"
"vmovdqu %%ymm0,(%[dst0]) \n"
"vmovdqu %%ymm1,(%[dst1]) \n"
"vmovdqu %%ymm2,(%[dst2]) \n"
"vmovdqu %%ymm3,(%[dst3]) \n"
"add $0x20,%[dst0] \n"
"add $0x20,%[dst1] \n"
"add $0x20,%[dst2] \n"
"add $0x20,%[dst3] \n"
"sub $0x8,%[w] \n"
"jge 1b \n"
"2: \n"
"add $0x8,%[w] \n"
"jz 5f \n"
// 4x4 block for 4..7 remainder elements
"cmp $0x4,%[w] \n"
"jl 3f \n"
"vmovdqu (%[src]),%%xmm0 \n"
"vmovdqu (%[src],%[s]),%%xmm1 \n"
"lea (%[src],%[s],2),%[src] \n"
"vmovdqu (%[src]),%%xmm2 \n"
"vmovdqu (%[src],%[s]),%%xmm3 \n"
"lea (%[src],%[s],2),%[src] \n"
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
"vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
"vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n"
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
"vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n"
"vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n"
"vmovdqu %%xmm0,(%[dst0]) \n"
"vmovdqu %%xmm1,(%[dst1]) \n"
"vmovdqu %%xmm2,(%[dst2]) \n"
"vmovdqu %%xmm3,(%[dst3]) \n"
"add $0x10,%[dst0] \n"
"add $0x10,%[dst1] \n"
"add $0x10,%[dst2] \n"
"add $0x10,%[dst3] \n"
"sub $0x4,%[w] \n"
"jz 5f \n"
// Masked tail for 1..3 remainder elements using vpmaskmovd
"3: \n"
"vmovdqu (%[src]),%%xmm0 \n"
"vmovdqa %%xmm0,%%xmm1 \n"
"vmovdqa %%xmm0,%%xmm2 \n"
"mov $0x000000ff,%%eax \n"
"cmp $2,%[w] \n"
"jl 4f \n"
"vmovdqu (%[src],%[s]),%%xmm1 \n"
"mov $0x0000ffff,%%eax \n"
"je 4f \n"
"vmovdqu (%[src],%[s],2),%%xmm2 \n"
"mov $0x00ffffff,%%eax \n"
"4: \n"
"vmovd %%eax,%%xmm7 \n"
"vpmovsxbd %%xmm7,%%xmm7 \n"
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
"vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n"
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
"vpunpckhdq %%xmm2,%%xmm2,%%xmm3 \n"
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
"vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n"
"vpunpckhqdq %%xmm3,%%xmm6,%%xmm3 \n"
"vpmaskmovd %%xmm0,%%xmm7,(%[dst0]) \n"
"vpmaskmovd %%xmm1,%%xmm7,(%[dst1]) \n"
"vpmaskmovd %%xmm2,%%xmm7,(%[dst2]) \n"
"vpmaskmovd %%xmm3,%%xmm7,(%[dst3]) \n"
"5: \n"
"vzeroupper \n"
: [src] "+r"(src), // %[src]
[dst0] "+r"(dst), // %[dst0]
[dst1] "+r"(dst1), // %[dst1]
[dst2] "+r"(dst2), // %[dst2]
[dst3] "+r"(dst3), // %[dst3]
[w] "+r"(width) // %[w]
: [s] "r"(s) // %[s]
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7");
}
#endif // defined(HAS_TRANSPOSE4X4_32_AVX2)
#endif // defined(HAS_TRANSPOSENX4_32_AVX2)
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
// Transpose 32 bit plane of any width x height using AVX2 vpmaskmovd.
void TransposeWxH_32_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (height >= 8) {
const uint8_t* src_row = src;
uint8_t* dst_col = dst;
int w = width;
while (w >= 8) {
asm volatile(
"vmovdqu (%[s0]),%%ymm0 \n"
"vmovdqu (%[s0],%[s]),%%ymm1 \n"
"lea (%[s0],%[s],2),%%rax \n"
"vmovdqu (%%rax),%%ymm2 \n"
"vmovdqu (%%rax,%[s]),%%ymm3 \n"
"lea (%%rax,%[s],2),%%rax \n"
"vmovdqu (%%rax),%%ymm4 \n"
"vmovdqu (%%rax,%[s]),%%ymm5 \n"
"lea (%%rax,%[s],2),%%rax \n"
"vmovdqu (%%rax),%%ymm6 \n"
"vmovdqu (%%rax,%[s]),%%ymm7 \n"
"vpunpckldq %%ymm1,%%ymm0,%%ymm8 \n"
"vpunpckhdq %%ymm1,%%ymm0,%%ymm9 \n"
"vpunpckldq %%ymm3,%%ymm2,%%ymm10 \n"
"vpunpckhdq %%ymm3,%%ymm2,%%ymm11 \n"
"vpunpckldq %%ymm5,%%ymm4,%%ymm12 \n"
"vpunpckhdq %%ymm5,%%ymm4,%%ymm13 \n"
"vpunpckldq %%ymm7,%%ymm6,%%ymm14 \n"
"vpunpckhdq %%ymm7,%%ymm6,%%ymm15 \n"
"vpunpcklqdq %%ymm10,%%ymm8,%%ymm0 \n"
"vpunpckhqdq %%ymm10,%%ymm8,%%ymm1 \n"
"vpunpcklqdq %%ymm11,%%ymm9,%%ymm2 \n"
"vpunpckhqdq %%ymm11,%%ymm9,%%ymm3 \n"
"vpunpcklqdq %%ymm14,%%ymm12,%%ymm4 \n"
"vpunpckhqdq %%ymm14,%%ymm12,%%ymm5 \n"
"vpunpcklqdq %%ymm15,%%ymm13,%%ymm6 \n"
"vpunpckhqdq %%ymm15,%%ymm13,%%ymm7 \n"
"vperm2i128 $0x20,%%ymm4,%%ymm0,%%ymm8 \n"
"vperm2i128 $0x20,%%ymm5,%%ymm1,%%ymm9 \n"
"vperm2i128 $0x20,%%ymm6,%%ymm2,%%ymm10 \n"
"vperm2i128 $0x20,%%ymm7,%%ymm3,%%ymm11 \n"
"vperm2i128 $0x31,%%ymm4,%%ymm0,%%ymm12 \n"
"vperm2i128 $0x31,%%ymm5,%%ymm1,%%ymm13 \n"
"vperm2i128 $0x31,%%ymm6,%%ymm2,%%ymm14 \n"
"vperm2i128 $0x31,%%ymm7,%%ymm3,%%ymm15 \n"
"vmovdqu %%ymm8,(%[d0]) \n"
"vmovdqu %%ymm9,(%[d0],%[d]) \n"
"lea (%[d0],%[d],2),%%rax \n"
"vmovdqu %%ymm10,(%%rax) \n"
"vmovdqu %%ymm11,(%%rax,%[d]) \n"
"lea (%%rax,%[d],2),%%rax \n"
"vmovdqu %%ymm12,(%%rax) \n"
"vmovdqu %%ymm13,(%%rax,%[d]) \n"
"lea (%%rax,%[d],2),%%rax \n"
"vmovdqu %%ymm14,(%%rax) \n"
"vmovdqu %%ymm15,(%%rax,%[d]) \n"
:
: [s0] "r"(src_row), // %[s0]
[d0] "r"(dst_col), // %[d0]
[s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "rax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
"xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
"xmm13", "xmm14", "xmm15");
src_row += 32;
dst_col += 8 * d;
w -= 8;
}
if (w >= 4) {
TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, 8);
src_row += 16;
dst_col += 4 * d;
w -= 4;
}
if (w > 0) {
TransposeWxH_32_C(src_row, src_stride, dst_col, dst_stride, w, 8);
}
src += 8 * s;
dst += 32;
height -= 8;
}
asm volatile("vzeroupper \n");
if (height > 0) {
while (width >= 4) {
TransposeNx4_32_AVX2(src, src_stride, dst, dst_stride, height);
src += 16;
dst += 4 * d;
width -= 4;
}
if (width > 0) {
const uint8_t* src_rem = src;
uint8_t* dst0 = dst;
uint8_t* dst1 = (width > 1) ? dst0 + d : dst0;
uint8_t* dst2 = (width > 2) ? dst1 + d : dst1;
int col_mask_bits = (1 << (width * 8)) - 1;
int h_rem = height;
asm volatile(
"vmovd %[col_mask],%%xmm7 \n"
"vpmovsxbd %%xmm7,%%xmm7 \n"
"sub $0x4,%[h] \n"
"jl 2f \n"
"1: \n"
"vpmaskmovd (%[src]),%%xmm7,%%xmm0 \n"
"vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm1 \n"
"lea (%[src],%[s],2),%[src] \n"
"vpmaskmovd (%[src]),%%xmm7,%%xmm2 \n"
"vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm3 \n"
"lea (%[src],%[s],2),%[src] \n"
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
"vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
"vpunpckhdq %%xmm3,%%xmm2,%%xmm3 \n"
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
"vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n"
"vmovdqu %%xmm2,(%[dst2]) \n"
"vmovdqu %%xmm1,(%[dst1]) \n"
"vmovdqu %%xmm0,(%[dst0]) \n"
"add $0x10,%[dst2] \n"
"add $0x10,%[dst1] \n"
"add $0x10,%[dst0] \n"
"sub $0x4,%[h] \n"
"jge 1b \n"
"2: \n"
"add $0x4,%[h] \n"
"jz 4f \n"
"vpmaskmovd (%[src]),%%xmm7,%%xmm0 \n"
"vmovdqa %%xmm0,%%xmm1 \n"
"vmovdqa %%xmm0,%%xmm2 \n"
"mov $0x000000ff,%%eax \n"
"cmp $2,%[h] \n"
"jl 3f \n"
"vpmaskmovd (%[src],%[s]),%%xmm7,%%xmm1 \n"
"mov $0x0000ffff,%%eax \n"
"je 3f \n"
"vpmaskmovd (%[src],%[s],2),%%xmm7,%%xmm2 \n"
"mov $0x00ffffff,%%eax \n"
"3: \n"
"vmovd %%eax,%%xmm7 \n"
"vpmovsxbd %%xmm7,%%xmm7 \n"
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
"vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n"
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
"vpunpckhdq %%xmm2,%%xmm2,%%xmm3 \n"
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
"vpunpcklqdq %%xmm3,%%xmm6,%%xmm2 \n"
"vpmaskmovd %%xmm2,%%xmm7,(%[dst2]) \n"
"vpmaskmovd %%xmm1,%%xmm7,(%[dst1]) \n"
"vpmaskmovd %%xmm0,%%xmm7,(%[dst0]) \n"
"4: \n"
"vzeroupper \n"
: [src] "+r"(src_rem), // %[src]
[dst0] "+r"(dst0), // %[dst0]
[dst1] "+r"(dst1), // %[dst1]
[dst2] "+r"(dst2), // %[dst2]
[h] "+r"(h_rem) // %[h]
: [s] "r"(s), // %[s]
[col_mask] "r"(col_mask_bits) // %[col_mask]
: "memory", "cc", "eax", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
"xmm5", "xmm6", "xmm7");
}
}
}
#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
#if defined(HAS_TRANSPOSENX4_32_AVX512BW)
// Transpose 32 bit values (ARGB) in Nx4 tiles with 4 blocks of 4x4 at a time
// in zmm0..zmm3 and masked tail for arbitrary N.
void TransposeNx4_32_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
asm volatile(
"sub $0x10,%[w] \n"
"jl 2f \n"
// Main loop: 4 blocks of 4x4 (16 source rows -> 16 dest columns / 64 B).
"1: \n"
"vmovdqu (%[src]),%%xmm0 \n"
"vmovdqu (%[src],%[s]),%%xmm1 \n"
"lea (%[src],%[s],2),%[src] \n"
"vmovdqu (%[src]),%%xmm2 \n"
"vmovdqu (%[src],%[s]),%%xmm3 \n"
"lea (%[src],%[s],2),%[src] \n"
"vinserti32x4 $1,(%[src]),%%zmm0,%%zmm0 \n"
"vinserti32x4 $1,(%[src],%[s]),%%zmm1,%%zmm1 \n"
"lea (%[src],%[s],2),%[src] \n"
"vinserti32x4 $1,(%[src]),%%zmm2,%%zmm2 \n"
"vinserti32x4 $1,(%[src],%[s]),%%zmm3,%%zmm3 \n"
"lea (%[src],%[s],2),%[src] \n"
"vinserti32x4 $2,(%[src]),%%zmm0,%%zmm0 \n"
"vinserti32x4 $2,(%[src],%[s]),%%zmm1,%%zmm1 \n"
"lea (%[src],%[s],2),%[src] \n"
"vinserti32x4 $2,(%[src]),%%zmm2,%%zmm2 \n"
"vinserti32x4 $2,(%[src],%[s]),%%zmm3,%%zmm3 \n"
"lea (%[src],%[s],2),%[src] \n"
"vinserti32x4 $3,(%[src]),%%zmm0,%%zmm0 \n"
"vinserti32x4 $3,(%[src],%[s]),%%zmm1,%%zmm1 \n"
"lea (%[src],%[s],2),%[src] \n"
"vinserti32x4 $3,(%[src]),%%zmm2,%%zmm2 \n"
"vinserti32x4 $3,(%[src],%[s]),%%zmm3,%%zmm3 \n"
"lea (%[src],%[s],2),%[src] \n"
// Transpose 2x2 across all 4 128-bit lanes
"vpunpckldq %%zmm1,%%zmm0,%%zmm4 \n"
"vpunpckldq %%zmm3,%%zmm2,%%zmm5 \n"
"vpunpckhdq %%zmm1,%%zmm0,%%zmm6 \n"
"vpunpckhdq %%zmm3,%%zmm2,%%zmm7 \n"
// Transpose 4x4 across all 4 128-bit lanes
"vpunpcklqdq %%zmm5,%%zmm4,%%zmm0 \n"
"vpunpckhqdq %%zmm5,%%zmm4,%%zmm1 \n"
"vpunpcklqdq %%zmm7,%%zmm6,%%zmm2 \n"
"vpunpckhqdq %%zmm7,%%zmm6,%%zmm3 \n"
"vmovdqu32 %%zmm0,(%[dst0]) \n"
"vmovdqu32 %%zmm1,(%[dst1]) \n"
"vmovdqu32 %%zmm2,(%[dst2]) \n"
"vmovdqu32 %%zmm3,(%[dst3]) \n"
"add $0x40,%[dst0] \n"
"add $0x40,%[dst1] \n"
"add $0x40,%[dst2] \n"
"add $0x40,%[dst3] \n"
"sub $0x10,%[w] \n"
"jge 1b \n"
"2: \n"
"add $0x10,%[w] \n"
"jz 5f \n"
// 4x4 loop for 4..15 remainder elements
"sub $0x4,%[w] \n"
"jl 4f \n"
"3: \n"
"vmovdqu (%[src]),%%xmm0 \n"
"vmovdqu (%[src],%[s]),%%xmm1 \n"
"lea (%[src],%[s],2),%[src] \n"
"vmovdqu (%[src]),%%xmm2 \n"
"vmovdqu (%[src],%[s]),%%xmm3 \n"
"lea (%[src],%[s],2),%[src] \n"
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
"vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n"
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
"vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n"
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
"vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n"
"vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n"
"vmovdqu %%xmm0,(%[dst0]) \n"
"vmovdqu %%xmm1,(%[dst1]) \n"
"vmovdqu %%xmm2,(%[dst2]) \n"
"vmovdqu %%xmm3,(%[dst3]) \n"
"add $0x10,%[dst0] \n"
"add $0x10,%[dst1] \n"
"add $0x10,%[dst2] \n"
"add $0x10,%[dst3] \n"
"sub $0x4,%[w] \n"
"jge 3b \n"
"4: \n"
"add $0x4,%[w] \n"
"jz 5f \n"
// Masked tail for 1..3 remainder elements (in-bounds row loads + k1 mask)
"mov $1,%%eax \n"
"shlx %[w],%%eax,%%eax \n"
"dec %%eax \n"
"kmovw %%eax,%%k1 \n"
"vmovdqu (%[src]),%%xmm0 \n"
"vmovdqa %%xmm0,%%xmm1 \n"
"vmovdqa %%xmm0,%%xmm2 \n"
"cmp $1,%[w] \n"
"jle 6f \n"
"vmovdqu (%[src],%[s]),%%xmm1 \n"
"cmp $2,%[w] \n"
"jle 6f \n"
"vmovdqu (%[src],%[s],2),%%xmm2 \n"
"6: \n"
"vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n"
"vpunpckldq %%xmm2,%%xmm2,%%xmm5 \n"
"vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n"
"vpunpckhdq %%xmm2,%%xmm2,%%xmm7 \n"
"vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n"
"vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n"
"vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n"
"vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n"
"vmovdqu32 %%xmm0,(%[dst0])%{%%k1%} \n"
"vmovdqu32 %%xmm1,(%[dst1])%{%%k1%} \n"
"vmovdqu32 %%xmm2,(%[dst2])%{%%k1%} \n"
"vmovdqu32 %%xmm3,(%[dst3])%{%%k1%} \n"
"5: \n"
"vzeroupper \n"
: [src] "+r"(src), // %[src]
[dst0] "+r"(dst), // %[dst0]
[dst1] "+r"(dst1), // %[dst1]
[dst2] "+r"(dst2), // %[dst2]
[dst3] "+r"(dst3), // %[dst3]
[w] "+r"(width) // %[w]
: [s] "r"(s) // %[s]
: "memory", "cc", "eax", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
"xmm5", "xmm6", "xmm7");
}
#endif // defined(HAS_TRANSPOSENX4_32_AVX512BW)
#if defined(HAS_TRANSPOSEWXH_32_AVX512BW)
// Transpose 32 bit plane of any width x height using 16x16 ZMM horizontal tiles
// in zmm0..zmm31 with k-masked right and bottom tails.
void TransposeWxH_32_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (height > 0) {
const int bh = (height >= 16) ? 16 : height;
const uint8_t* src_row = src;
uint8_t* dst_col = dst;
int w = width;
if (bh == 16) {
while (w >= 16) {
const uint8_t* s_ptr = src_row;
uint8_t* d_ptr = dst_col;
asm volatile(
"vmovdqu32 (%[s_ptr]),%%zmm0 \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm1 \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm2 \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm3 \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm4 \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm5 \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm6 \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm7 \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm8 \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm9 \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm10 \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm11 \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm12 \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm13 \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm14 \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm15 \n"
// Stage 1: 32-bit unpack
"vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n"
"vpunpckhdq %%zmm1,%%zmm0,%%zmm17 \n"
"vpunpckldq %%zmm3,%%zmm2,%%zmm18 \n"
"vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n"
"vpunpckldq %%zmm5,%%zmm4,%%zmm20 \n"
"vpunpckhdq %%zmm5,%%zmm4,%%zmm21 \n"
"vpunpckldq %%zmm7,%%zmm6,%%zmm22 \n"
"vpunpckhdq %%zmm7,%%zmm6,%%zmm23 \n"
"vpunpckldq %%zmm9,%%zmm8,%%zmm24 \n"
"vpunpckhdq %%zmm9,%%zmm8,%%zmm25 \n"
"vpunpckldq %%zmm11,%%zmm10,%%zmm26 \n"
"vpunpckhdq %%zmm11,%%zmm10,%%zmm27 \n"
"vpunpckldq %%zmm13,%%zmm12,%%zmm28 \n"
"vpunpckhdq %%zmm13,%%zmm12,%%zmm29 \n"
"vpunpckldq %%zmm15,%%zmm14,%%zmm30 \n"
"vpunpckhdq %%zmm15,%%zmm14,%%zmm31 \n"
// Stage 2: 64-bit unpack
"vpunpcklqdq %%zmm18,%%zmm16,%%zmm0 \n"
"vpunpckhqdq %%zmm18,%%zmm16,%%zmm1 \n"
"vpunpcklqdq %%zmm19,%%zmm17,%%zmm2 \n"
"vpunpckhqdq %%zmm19,%%zmm17,%%zmm3 \n"
"vpunpcklqdq %%zmm22,%%zmm20,%%zmm4 \n"
"vpunpckhqdq %%zmm22,%%zmm20,%%zmm5 \n"
"vpunpcklqdq %%zmm23,%%zmm21,%%zmm6 \n"
"vpunpckhqdq %%zmm23,%%zmm21,%%zmm7 \n"
"vpunpcklqdq %%zmm26,%%zmm24,%%zmm8 \n"
"vpunpckhqdq %%zmm26,%%zmm24,%%zmm9 \n"
"vpunpcklqdq %%zmm27,%%zmm25,%%zmm10 \n"
"vpunpckhqdq %%zmm27,%%zmm25,%%zmm11 \n"
"vpunpcklqdq %%zmm30,%%zmm28,%%zmm12 \n"
"vpunpckhqdq %%zmm30,%%zmm28,%%zmm13 \n"
"vpunpcklqdq %%zmm31,%%zmm29,%%zmm14 \n"
"vpunpckhqdq %%zmm31,%%zmm29,%%zmm15 \n"
// Stage 3: 128-bit lane shuffle
"vshufi32x4 $0x88,%%zmm4,%%zmm0,%%zmm16 \n"
"vshufi32x4 $0x88,%%zmm5,%%zmm1,%%zmm17 \n"
"vshufi32x4 $0x88,%%zmm6,%%zmm2,%%zmm18 \n"
"vshufi32x4 $0x88,%%zmm7,%%zmm3,%%zmm19 \n"
"vshufi32x4 $0xdd,%%zmm4,%%zmm0,%%zmm20 \n"
"vshufi32x4 $0xdd,%%zmm5,%%zmm1,%%zmm21 \n"
"vshufi32x4 $0xdd,%%zmm6,%%zmm2,%%zmm22 \n"
"vshufi32x4 $0xdd,%%zmm7,%%zmm3,%%zmm23 \n"
"vshufi32x4 $0x88,%%zmm12,%%zmm8,%%zmm24 \n"
"vshufi32x4 $0x88,%%zmm13,%%zmm9,%%zmm25 \n"
"vshufi32x4 $0x88,%%zmm14,%%zmm10,%%zmm26 \n"
"vshufi32x4 $0x88,%%zmm15,%%zmm11,%%zmm27 \n"
"vshufi32x4 $0xdd,%%zmm12,%%zmm8,%%zmm28 \n"
"vshufi32x4 $0xdd,%%zmm13,%%zmm9,%%zmm29 \n"
"vshufi32x4 $0xdd,%%zmm14,%%zmm10,%%zmm30 \n"
"vshufi32x4 $0xdd,%%zmm15,%%zmm11,%%zmm31 \n"
// Stage 4: 256-bit half shuffle -> columns 0..15 in zmm0..zmm15
"vshufi32x4 $0x88,%%zmm24,%%zmm16,%%zmm0 \n"
"vshufi32x4 $0x88,%%zmm25,%%zmm17,%%zmm1 \n"
"vshufi32x4 $0x88,%%zmm26,%%zmm18,%%zmm2 \n"
"vshufi32x4 $0x88,%%zmm27,%%zmm19,%%zmm3 \n"
"vshufi32x4 $0x88,%%zmm28,%%zmm20,%%zmm4 \n"
"vshufi32x4 $0x88,%%zmm29,%%zmm21,%%zmm5 \n"
"vshufi32x4 $0x88,%%zmm30,%%zmm22,%%zmm6 \n"
"vshufi32x4 $0x88,%%zmm31,%%zmm23,%%zmm7 \n"
"vshufi32x4 $0xdd,%%zmm24,%%zmm16,%%zmm8 \n"
"vshufi32x4 $0xdd,%%zmm25,%%zmm17,%%zmm9 \n"
"vshufi32x4 $0xdd,%%zmm26,%%zmm18,%%zmm10 \n"
"vshufi32x4 $0xdd,%%zmm27,%%zmm19,%%zmm11 \n"
"vshufi32x4 $0xdd,%%zmm28,%%zmm20,%%zmm12 \n"
"vshufi32x4 $0xdd,%%zmm29,%%zmm21,%%zmm13 \n"
"vshufi32x4 $0xdd,%%zmm30,%%zmm22,%%zmm14 \n"
"vshufi32x4 $0xdd,%%zmm31,%%zmm23,%%zmm15 \n"
"vmovdqu32 %%zmm0,(%[d_ptr]) \n"
"vmovdqu32 %%zmm1,(%[d_ptr],%[d]) \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm2,(%[d_ptr]) \n"
"vmovdqu32 %%zmm3,(%[d_ptr],%[d]) \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm4,(%[d_ptr]) \n"
"vmovdqu32 %%zmm5,(%[d_ptr],%[d]) \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm6,(%[d_ptr]) \n"
"vmovdqu32 %%zmm7,(%[d_ptr],%[d]) \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm8,(%[d_ptr]) \n"
"vmovdqu32 %%zmm9,(%[d_ptr],%[d]) \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm10,(%[d_ptr]) \n"
"vmovdqu32 %%zmm11,(%[d_ptr],%[d]) \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm12,(%[d_ptr]) \n"
"vmovdqu32 %%zmm13,(%[d_ptr],%[d]) \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm14,(%[d_ptr]) \n"
"vmovdqu32 %%zmm15,(%[d_ptr],%[d]) \n"
: [s_ptr] "+r"(s_ptr), // %[s_ptr]
[d_ptr] "+r"(d_ptr) // %[d_ptr]
: [s] "r"(s), // %[s]
[d] "r"(d) // %[d]
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
"xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
"xmm13", "xmm14", "xmm15", "xmm16", "xmm17", "xmm18", "xmm19",
"xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26",
"xmm27", "xmm28", "xmm29", "xmm30", "xmm31");
src_row += 64;
dst_col += 16 * d;
w -= 16;
}
}
const uint32_t row_mask = (1u << bh) - 1;
while (w > 0) {
const int bw = (w >= 16) ? 16 : w;
const uint32_t col_mask = (1u << bw) - 1;
const uint8_t* s_ptr = src_row;
uint8_t* d_ptr = dst_col;
asm volatile(
"kmovw %[col_mask],%%k1 \n"
"kmovw %[row_mask],%%k2 \n"
"vpxord %%zmm0,%%zmm0,%%zmm0 \n"
"vmovdqa64 %%zmm0,%%zmm1 \n"
"vmovdqa64 %%zmm0,%%zmm2 \n"
"vmovdqa64 %%zmm0,%%zmm3 \n"
"vmovdqa64 %%zmm0,%%zmm4 \n"
"vmovdqa64 %%zmm0,%%zmm5 \n"
"vmovdqa64 %%zmm0,%%zmm6 \n"
"vmovdqa64 %%zmm0,%%zmm7 \n"
"vmovdqa64 %%zmm0,%%zmm8 \n"
"vmovdqa64 %%zmm0,%%zmm9 \n"
"vmovdqa64 %%zmm0,%%zmm10 \n"
"vmovdqa64 %%zmm0,%%zmm11 \n"
"vmovdqa64 %%zmm0,%%zmm12 \n"
"vmovdqa64 %%zmm0,%%zmm13 \n"
"vmovdqa64 %%zmm0,%%zmm14 \n"
"vmovdqa64 %%zmm0,%%zmm15 \n"
"vmovdqu32 (%[s_ptr]),%%zmm0%{%%k1%}%{z%} \n"
"cmp $1,%[bh] \n"
"jle 1f \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm1%{%%k1%}%{z%} \n"
"cmp $2,%[bh] \n"
"jle 1f \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm2%{%%k1%}%{z%} \n"
"cmp $3,%[bh] \n"
"jle 1f \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm3%{%%k1%}%{z%} \n"
"cmp $4,%[bh] \n"
"jle 1f \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm4%{%%k1%}%{z%} \n"
"cmp $5,%[bh] \n"
"jle 1f \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm5%{%%k1%}%{z%} \n"
"cmp $6,%[bh] \n"
"jle 1f \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm6%{%%k1%}%{z%} \n"
"cmp $7,%[bh] \n"
"jle 1f \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm7%{%%k1%}%{z%} \n"
"cmp $8,%[bh] \n"
"jle 1f \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm8%{%%k1%}%{z%} \n"
"cmp $9,%[bh] \n"
"jle 1f \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm9%{%%k1%}%{z%} \n"
"cmp $10,%[bh] \n"
"jle 1f \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm10%{%%k1%}%{z%} \n"
"cmp $11,%[bh] \n"
"jle 1f \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm11%{%%k1%}%{z%} \n"
"cmp $12,%[bh] \n"
"jle 1f \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm12%{%%k1%}%{z%} \n"
"cmp $13,%[bh] \n"
"jle 1f \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm13%{%%k1%}%{z%} \n"
"cmp $14,%[bh] \n"
"jle 1f \n"
"lea (%[s_ptr],%[s],2),%[s_ptr] \n"
"vmovdqu32 (%[s_ptr]),%%zmm14%{%%k1%}%{z%} \n"
"cmp $15,%[bh] \n"
"jle 1f \n"
"vmovdqu32 (%[s_ptr],%[s]),%%zmm15%{%%k1%}%{z%} \n"
"1: \n"
// Stage 1: 32-bit unpack
"vpunpckldq %%zmm1,%%zmm0,%%zmm16 \n"
"vpunpckhdq %%zmm1,%%zmm0,%%zmm17 \n"
"vpunpckldq %%zmm3,%%zmm2,%%zmm18 \n"
"vpunpckhdq %%zmm3,%%zmm2,%%zmm19 \n"
"vpunpckldq %%zmm5,%%zmm4,%%zmm20 \n"
"vpunpckhdq %%zmm5,%%zmm4,%%zmm21 \n"
"vpunpckldq %%zmm7,%%zmm6,%%zmm22 \n"
"vpunpckhdq %%zmm7,%%zmm6,%%zmm23 \n"
"vpunpckldq %%zmm9,%%zmm8,%%zmm24 \n"
"vpunpckhdq %%zmm9,%%zmm8,%%zmm25 \n"
"vpunpckldq %%zmm11,%%zmm10,%%zmm26 \n"
"vpunpckhdq %%zmm11,%%zmm10,%%zmm27 \n"
"vpunpckldq %%zmm13,%%zmm12,%%zmm28 \n"
"vpunpckhdq %%zmm13,%%zmm12,%%zmm29 \n"
"vpunpckldq %%zmm15,%%zmm14,%%zmm30 \n"
"vpunpckhdq %%zmm15,%%zmm14,%%zmm31 \n"
// Stage 2: 64-bit unpack
"vpunpcklqdq %%zmm18,%%zmm16,%%zmm0 \n"
"vpunpckhqdq %%zmm18,%%zmm16,%%zmm1 \n"
"vpunpcklqdq %%zmm19,%%zmm17,%%zmm2 \n"
"vpunpckhqdq %%zmm19,%%zmm17,%%zmm3 \n"
"vpunpcklqdq %%zmm22,%%zmm20,%%zmm4 \n"
"vpunpckhqdq %%zmm22,%%zmm20,%%zmm5 \n"
"vpunpcklqdq %%zmm23,%%zmm21,%%zmm6 \n"
"vpunpckhqdq %%zmm23,%%zmm21,%%zmm7 \n"
"vpunpcklqdq %%zmm26,%%zmm24,%%zmm8 \n"
"vpunpckhqdq %%zmm26,%%zmm24,%%zmm9 \n"
"vpunpcklqdq %%zmm27,%%zmm25,%%zmm10 \n"
"vpunpckhqdq %%zmm27,%%zmm25,%%zmm11 \n"
"vpunpcklqdq %%zmm30,%%zmm28,%%zmm12 \n"
"vpunpckhqdq %%zmm30,%%zmm28,%%zmm13 \n"
"vpunpcklqdq %%zmm31,%%zmm29,%%zmm14 \n"
"vpunpckhqdq %%zmm31,%%zmm29,%%zmm15 \n"
// Stage 3: 128-bit lane shuffle
"vshufi32x4 $0x88,%%zmm4,%%zmm0,%%zmm16 \n"
"vshufi32x4 $0x88,%%zmm5,%%zmm1,%%zmm17 \n"
"vshufi32x4 $0x88,%%zmm6,%%zmm2,%%zmm18 \n"
"vshufi32x4 $0x88,%%zmm7,%%zmm3,%%zmm19 \n"
"vshufi32x4 $0xdd,%%zmm4,%%zmm0,%%zmm20 \n"
"vshufi32x4 $0xdd,%%zmm5,%%zmm1,%%zmm21 \n"
"vshufi32x4 $0xdd,%%zmm6,%%zmm2,%%zmm22 \n"
"vshufi32x4 $0xdd,%%zmm7,%%zmm3,%%zmm23 \n"
"vshufi32x4 $0x88,%%zmm12,%%zmm8,%%zmm24 \n"
"vshufi32x4 $0x88,%%zmm13,%%zmm9,%%zmm25 \n"
"vshufi32x4 $0x88,%%zmm14,%%zmm10,%%zmm26 \n"
"vshufi32x4 $0x88,%%zmm15,%%zmm11,%%zmm27 \n"
"vshufi32x4 $0xdd,%%zmm12,%%zmm8,%%zmm28 \n"
"vshufi32x4 $0xdd,%%zmm13,%%zmm9,%%zmm29 \n"
"vshufi32x4 $0xdd,%%zmm14,%%zmm10,%%zmm30 \n"
"vshufi32x4 $0xdd,%%zmm15,%%zmm11,%%zmm31 \n"
// Stage 4: 256-bit half shuffle -> columns 0..15 in zmm0..zmm15
"vshufi32x4 $0x88,%%zmm24,%%zmm16,%%zmm0 \n"
"vshufi32x4 $0x88,%%zmm25,%%zmm17,%%zmm1 \n"
"vshufi32x4 $0x88,%%zmm26,%%zmm18,%%zmm2 \n"
"vshufi32x4 $0x88,%%zmm27,%%zmm19,%%zmm3 \n"
"vshufi32x4 $0x88,%%zmm28,%%zmm20,%%zmm4 \n"
"vshufi32x4 $0x88,%%zmm29,%%zmm21,%%zmm5 \n"
"vshufi32x4 $0x88,%%zmm30,%%zmm22,%%zmm6 \n"
"vshufi32x4 $0x88,%%zmm31,%%zmm23,%%zmm7 \n"
"vshufi32x4 $0xdd,%%zmm24,%%zmm16,%%zmm8 \n"
"vshufi32x4 $0xdd,%%zmm25,%%zmm17,%%zmm9 \n"
"vshufi32x4 $0xdd,%%zmm26,%%zmm18,%%zmm10 \n"
"vshufi32x4 $0xdd,%%zmm27,%%zmm19,%%zmm11 \n"
"vshufi32x4 $0xdd,%%zmm28,%%zmm20,%%zmm12 \n"
"vshufi32x4 $0xdd,%%zmm29,%%zmm21,%%zmm13 \n"
"vshufi32x4 $0xdd,%%zmm30,%%zmm22,%%zmm14 \n"
"vshufi32x4 $0xdd,%%zmm31,%%zmm23,%%zmm15 \n"
"vmovdqu32 %%zmm0,(%[d_ptr])%{%%k2%} \n"
"cmp $1,%[bw] \n"
"jle 2f \n"
"vmovdqu32 %%zmm1,(%[d_ptr],%[d])%{%%k2%} \n"
"cmp $2,%[bw] \n"
"jle 2f \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm2,(%[d_ptr])%{%%k2%} \n"
"cmp $3,%[bw] \n"
"jle 2f \n"
"vmovdqu32 %%zmm3,(%[d_ptr],%[d])%{%%k2%} \n"
"cmp $4,%[bw] \n"
"jle 2f \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm4,(%[d_ptr])%{%%k2%} \n"
"cmp $5,%[bw] \n"
"jle 2f \n"
"vmovdqu32 %%zmm5,(%[d_ptr],%[d])%{%%k2%} \n"
"cmp $6,%[bw] \n"
"jle 2f \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm6,(%[d_ptr])%{%%k2%} \n"
"cmp $7,%[bw] \n"
"jle 2f \n"
"vmovdqu32 %%zmm7,(%[d_ptr],%[d])%{%%k2%} \n"
"cmp $8,%[bw] \n"
"jle 2f \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm8,(%[d_ptr])%{%%k2%} \n"
"cmp $9,%[bw] \n"
"jle 2f \n"
"vmovdqu32 %%zmm9,(%[d_ptr],%[d])%{%%k2%} \n"
"cmp $10,%[bw] \n"
"jle 2f \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm10,(%[d_ptr])%{%%k2%} \n"
"cmp $11,%[bw] \n"
"jle 2f \n"
"vmovdqu32 %%zmm11,(%[d_ptr],%[d])%{%%k2%} \n"
"cmp $12,%[bw] \n"
"jle 2f \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm12,(%[d_ptr])%{%%k2%} \n"
"cmp $13,%[bw] \n"
"jle 2f \n"
"vmovdqu32 %%zmm13,(%[d_ptr],%[d])%{%%k2%} \n"
"cmp $14,%[bw] \n"
"jle 2f \n"
"lea (%[d_ptr],%[d],2),%[d_ptr] \n"
"vmovdqu32 %%zmm14,(%[d_ptr])%{%%k2%} \n"
"cmp $15,%[bw] \n"
"jle 2f \n"
"vmovdqu32 %%zmm15,(%[d_ptr],%[d])%{%%k2%} \n"
"2: \n"
: [s_ptr] "+r"(s_ptr), // %[s_ptr]
[d_ptr] "+r"(d_ptr) // %[d_ptr]
: [s] "r"(s), // %[s]
[d] "r"(d), // %[d]
[col_mask] "r"(col_mask), // %[col_mask]
[row_mask] "r"(row_mask), // %[row_mask]
[bh] "r"(bh), // %[bh]
[bw] "r"(bw) // %[bw]
: "memory", "cc", "k1", "k2", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
"xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
"xmm13", "xmm14", "xmm15", "xmm16", "xmm17", "xmm18", "xmm19",
"xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26",
"xmm27", "xmm28", "xmm29", "xmm30", "xmm31");
src_row += 64;
dst_col += 16 * d;
w -= 16;
}
src += 16 * s;
dst += 64;
height -= 16;
}
asm volatile("vzeroupper \n");
}
#endif // defined(HAS_TRANSPOSEWXH_32_AVX512BW)
#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) &&
// !defined(LIBYUV_ENABLE_ROWWIN)

103
source/rotate_lasx.cc Normal file
View File

@ -0,0 +1,103 @@
/*
* Copyright 2026 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_row.h"
#if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)
#include "libyuv/loongson_intrinsics.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
// Transpose 32 bit values (ARGB) in 2x 4x4 tiles per step (4 cols x 8 rows ->
// 4 rows x 8 cols), matching Transpose4x4_32_AVX2. Width is a multiple of 8.
void Transpose4x4_32_LASX(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
ptrdiff_t s = src_stride;
ptrdiff_t s2 = s * 2;
ptrdiff_t s3 = s + s2;
ptrdiff_t s4 = s * 4;
uint8_t* dst0 = dst;
uint8_t* dst1 = dst0 + dst_stride;
uint8_t* dst2 = dst1 + dst_stride;
uint8_t* dst3 = dst2 + dst_stride;
const uint8_t* src4;
asm volatile(
"srli.w %[w], %[w], 3 \n"
"beqz %[w], 2f \n"
"1: \n"
"vld $vr0, %[src], 0 \n"
"vldx $vr1, %[src], %[s] \n"
"vldx $vr2, %[src], %[s2] \n"
"vldx $vr3, %[src], %[s3] \n"
"add.d %[src4], %[src], %[s4] \n"
"vld $vr4, %[src4], 0 \n"
"vldx $vr5, %[src4], %[s] \n"
"vldx $vr6, %[src4], %[s2] \n"
"vldx $vr7, %[src4], %[s3] \n"
"add.d %[src], %[src4], %[s4] \n"
// Combine rows (0,4), (1,5), (2,6), (3,7) into 256-bit xr0..xr3
"xvpermi.q $xr0, $xr4, 0x02 \n"
"xvpermi.q $xr1, $xr5, 0x02 \n"
"xvpermi.q $xr2, $xr6, 0x02 \n"
"xvpermi.q $xr3, $xr7, 0x02 \n"
// Transpose 2x2 across both 128-bit lanes
"xvilvl.w $xr4, $xr1, $xr0 \n"
"xvilvh.w $xr5, $xr1, $xr0 \n"
"xvilvl.w $xr6, $xr3, $xr2 \n"
"xvilvh.w $xr7, $xr3, $xr2 \n"
// Transpose 4x4 across both 128-bit lanes
"xvilvl.d $xr0, $xr6, $xr4 \n"
"xvilvh.d $xr1, $xr6, $xr4 \n"
"xvilvl.d $xr2, $xr7, $xr5 \n"
"xvilvh.d $xr3, $xr7, $xr5 \n"
// Store 4 full 256-bit (32-byte) destination rows
"xvst $xr0, %[dst0], 0 \n"
"xvst $xr1, %[dst1], 0 \n"
"xvst $xr2, %[dst2], 0 \n"
"xvst $xr3, %[dst3], 0 \n"
"addi.d %[dst0], %[dst0], 32 \n"
"addi.d %[dst1], %[dst1], 32 \n"
"addi.d %[dst2], %[dst2], 32 \n"
"addi.d %[dst3], %[dst3], 32 \n"
"addi.w %[w], %[w], -1 \n"
"bnez %[w], 1b \n"
"2: \n"
: [src] "+r"(src), // %[src]
[src4] "=&r"(src4), // %[src4]
[dst0] "+r"(dst0), // %[dst0]
[dst1] "+r"(dst1), // %[dst1]
[dst2] "+r"(dst2), // %[dst2]
[dst3] "+r"(dst3), // %[dst3]
[w] "+r"(width) // %[w]
: [s] "r"(s), // %[s]
[s2] "r"(s2), // %[s2]
[s3] "r"(s3), // %[s3]
[s4] "r"(s4) // %[s4]
: "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6",
"$xr7");
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
#endif // !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx)

30
source/rotate_lsx.cc
View File

@ -212,6 +212,36 @@ void TransposeUVWx16_LSX(const uint8_t* src,
}
}
void Transpose4x4_32_LSX(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
int x;
int len = width / 4;
ptrdiff_t src_stride2 = src_stride * 2;
ptrdiff_t src_stride3 = src_stride + src_stride2;
ptrdiff_t src_stride4 = src_stride * 4;
ptrdiff_t dst_stride2 = dst_stride * 2;
ptrdiff_t dst_stride3 = dst_stride + dst_stride2;
__m128i src0, src1, src2, src3;
__m128i dst0, dst1, dst2, dst3;
for (x = 0; x < len; x++) {
src0 = __lsx_vld(src, 0);
src1 = __lsx_vldx(src, src_stride);
src2 = __lsx_vldx(src, src_stride2);
src3 = __lsx_vldx(src, src_stride3);
LSX_TRANSPOSE4x4_W(src0, src1, src2, src3, dst0, dst1, dst2, dst3);
__lsx_vst(dst0, dst, 0);
__lsx_vstx(dst1, dst, dst_stride);
__lsx_vstx(dst2, dst, dst_stride2);
__lsx_vstx(dst3, dst, dst_stride3);
dst += 16;
src += src_stride4;
}
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

247
source/rotate_neon.cc
View File

@ -177,45 +177,248 @@ void TransposeUVWx8_NEON(const uint8_t* src,
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
}
// Transpose 32 bit values (ARGB)
void Transpose4x4_32_NEON(const uint8_t* src,
// Transpose 32 bit values (ARGB) in Nx4 tiles using vld4.32 with lane insertion
// and lane stores for 1..3 remainder rows.
void TransposeNx4_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const uint8_t* src1 = src + src_stride;
const uint8_t* src2 = src1 + src_stride;
const uint8_t* src3 = src2 + src_stride;
uint8_t* dst1 = dst + dst_stride;
uint8_t* dst2 = dst1 + dst_stride;
uint8_t* dst3 = dst2 + dst_stride;
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
const ptrdiff_t d = dst_stride;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
asm volatile(
// Main loop transpose 4x4. Read a column, write a row.
"1: \n"
"subs %8, %8, #4 \n"
"blt 2f \n"
// Main loop transpose 4x4. Read a column, write a row.
"1: \n"
"vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0], %9 \n"
"vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1], %9 \n"
"vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2], %9 \n"
"vld4.32 {d1[1], d3[1], d5[1], d7[1]}, [%3], %9 \n"
"subs %8, %8, #4 \n" // w -= 4
"subs %8, %8, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"vst1.8 {q1}, [%5]! \n"
"vst1.8 {q2}, [%6]! \n"
"vst1.8 {q3}, [%7]! \n"
"bgt 1b \n"
"bge 1b \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(dst3), // %7
"+r"(width) // %8
: "r"((ptrdiff_t)src_stride * 4) // %9
"2: \n"
"adds %8, %8, #4 \n"
"beq 5f \n"
// Remainder of 1, 2, or 3 source rows.
"vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0] \n"
"cmp %8, #2 \n"
"blt 4f \n"
"vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"vst1.32 {d2}, [%5] \n"
"vst1.32 {d4}, [%6] \n"
"vst1.32 {d6}, [%7] \n"
"b 5f \n"
"3: \n"
"vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d2}, [%5]! \n"
"vst1.32 {d4}, [%6]! \n"
"vst1.32 {d6}, [%7]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"vst1.32 {d3[0]}, [%5] \n"
"vst1.32 {d5[0]}, [%6] \n"
"vst1.32 {d7[0]}, [%7] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"vst1.32 {d2[0]}, [%5] \n"
"vst1.32 {d4[0]}, [%6] \n"
"vst1.32 {d6[0]}, [%7] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(dst3), // %7
"+r"(width) // %8
: "r"(s * 4) // %9
: "memory", "cc", "q0", "q1", "q2", "q3");
}
void TransposeWxH_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const ptrdiff_t d = dst_stride;
while (width >= 4) {
TransposeNx4_32_NEON(src, src_stride, dst, dst_stride, height);
src += 16;
dst += 4 * d;
width -= 4;
}
if (width == 3) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
asm volatile(
"subs %7, %7, #4 \n"
"blt 2f \n"
"1: \n"
"vld3.32 {d0[0], d2[0], d4[0]}, [%0], %8 \n"
"vld3.32 {d0[1], d2[1], d4[1]}, [%1], %8 \n"
"vld3.32 {d1[0], d3[0], d5[0]}, [%2], %8 \n"
"vld3.32 {d1[1], d3[1], d5[1]}, [%3], %8 \n"
"subs %7, %7, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"vst1.8 {q1}, [%5]! \n"
"vst1.8 {q2}, [%6]! \n"
"bge 1b \n"
"2: \n"
"adds %7, %7, #4 \n"
"beq 5f \n"
"vld3.32 {d0[0], d2[0], d4[0]}, [%0] \n"
"cmp %7, #2 \n"
"blt 4f \n"
"vld3.32 {d0[1], d2[1], d4[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"vst1.32 {d2}, [%5] \n"
"vst1.32 {d4}, [%6] \n"
"b 5f \n"
"3: \n"
"vld3.32 {d1[0], d3[0], d5[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d2}, [%5]! \n"
"vst1.32 {d4}, [%6]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"vst1.32 {d3[0]}, [%5] \n"
"vst1.32 {d5[0]}, [%6] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"vst1.32 {d2[0]}, [%5] \n"
"vst1.32 {d4[0]}, [%6] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(height) // %7
: "r"(s * 4) // %8
: "memory", "cc", "q0", "q1", "q2");
} else if (width == 2) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
asm volatile(
"subs %6, %6, #4 \n"
"blt 2f \n"
"1: \n"
"vld2.32 {d0[0], d2[0]}, [%0], %7 \n"
"vld2.32 {d0[1], d2[1]}, [%1], %7 \n"
"vld2.32 {d1[0], d3[0]}, [%2], %7 \n"
"vld2.32 {d1[1], d3[1]}, [%3], %7 \n"
"subs %6, %6, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"vst1.8 {q1}, [%5]! \n"
"bge 1b \n"
"2: \n"
"adds %6, %6, #4 \n"
"beq 5f \n"
"vld2.32 {d0[0], d2[0]}, [%0] \n"
"cmp %6, #2 \n"
"blt 4f \n"
"vld2.32 {d0[1], d2[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"vst1.32 {d2}, [%5] \n"
"b 5f \n"
"3: \n"
"vld2.32 {d1[0], d3[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d2}, [%5]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"vst1.32 {d3[0]}, [%5] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"vst1.32 {d2[0]}, [%5] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(height) // %6
: "r"(s * 4) // %7
: "memory", "cc", "q0", "q1");
} else if (width == 1) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
asm volatile(
"subs %5, %5, #4 \n"
"blt 2f \n"
"1: \n"
"vld1.32 {d0[0]}, [%0], %6 \n"
"vld1.32 {d0[1]}, [%1], %6 \n"
"vld1.32 {d1[0]}, [%2], %6 \n"
"vld1.32 {d1[1]}, [%3], %6 \n"
"subs %5, %5, #4 \n"
"vst1.8 {q0}, [%4]! \n"
"bge 1b \n"
"2: \n"
"adds %5, %5, #4 \n"
"beq 5f \n"
"vld1.32 {d0[0]}, [%0] \n"
"cmp %5, #2 \n"
"blt 4f \n"
"vld1.32 {d0[1]}, [%1] \n"
"bgt 3f \n"
"vst1.32 {d0}, [%4] \n"
"b 5f \n"
"3: \n"
"vld1.32 {d1[0]}, [%2] \n"
"vst1.32 {d0}, [%4]! \n"
"vst1.32 {d1[0]}, [%4] \n"
"b 5f \n"
"4: \n"
"vst1.32 {d0[0]}, [%4] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(height) // %5
: "r"(s * 4) // %6
: "memory", "cc", "q0");
}
}
#endif // defined(__ARM_NEON__) && !defined(__aarch64__)
#ifdef __cplusplus

256
source/rotate_neon64.cc
View File

@ -232,44 +232,248 @@ void TransposeUVWx8_NEON(const uint8_t* src,
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v30", "v31");
}
// Transpose 32 bit values (ARGB)
void Transpose4x4_32_NEON(const uint8_t* src,
// Transpose 32 bit values (ARGB) in Nx4 tiles using ld4 with lane insertion
// and lane stores for 1..3 remainder rows.
void TransposeNx4_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const uint8_t* src1 = src + src_stride;
const uint8_t* src2 = src1 + src_stride;
const uint8_t* src3 = src2 + src_stride;
uint8_t* dst1 = dst + dst_stride;
uint8_t* dst2 = dst1 + dst_stride;
uint8_t* dst3 = dst2 + dst_stride;
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
const ptrdiff_t d = dst_stride;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
asm volatile(
// Main loop transpose 4x4. Read a column, write a row.
"1: \n"
"subs %w8, %w8, #4 \n"
"b.lt 2f \n"
// Main loop transpose 4x4. Read a column, write a row.
"1: \n"
"ld4 {v0.s, v1.s, v2.s, v3.s}[0], [%0], %9 \n"
"ld4 {v0.s, v1.s, v2.s, v3.s}[1], [%1], %9 \n"
"ld4 {v0.s, v1.s, v2.s, v3.s}[2], [%2], %9 \n"
"ld4 {v0.s, v1.s, v2.s, v3.s}[3], [%3], %9 \n"
"subs %w8, %w8, #4 \n" // w -= 4
"st1 {v0.4s}, [%4], 16 \n"
"st1 {v1.4s}, [%5], 16 \n"
"st1 {v2.4s}, [%6], 16 \n"
"st1 {v3.4s}, [%7], 16 \n"
"b.gt 1b \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(dst3), // %7
"+r"(width) // %8
: "r"((ptrdiff_t)src_stride * 4) // %9
"subs %w8, %w8, #4 \n"
"st1 {v0.4s}, [%4], 16 \n"
"st1 {v1.4s}, [%5], 16 \n"
"st1 {v2.4s}, [%6], 16 \n"
"st1 {v3.4s}, [%7], 16 \n"
"b.ge 1b \n"
"2: \n"
"adds %w8, %w8, #4 \n"
"b.eq 5f \n"
// Remainder of 1, 2, or 3 source rows.
"ld4 {v0.s, v1.s, v2.s, v3.s}[0], [%0] \n"
"cmp %w8, #2 \n"
"b.lt 4f \n"
"ld4 {v0.s, v1.s, v2.s, v3.s}[1], [%1] \n"
"b.gt 3f \n"
"st1 {v0.d}[0], [%4] \n"
"st1 {v1.d}[0], [%5] \n"
"st1 {v2.d}[0], [%6] \n"
"st1 {v3.d}[0], [%7] \n"
"b 5f \n"
"3: \n"
"ld4 {v0.s, v1.s, v2.s, v3.s}[2], [%2] \n"
"st1 {v0.d}[0], [%4], 8 \n"
"st1 {v1.d}[0], [%5], 8 \n"
"st1 {v2.d}[0], [%6], 8 \n"
"st1 {v3.d}[0], [%7], 8 \n"
"st1 {v0.s}[2], [%4] \n"
"st1 {v1.s}[2], [%5] \n"
"st1 {v2.s}[2], [%6] \n"
"st1 {v3.s}[2], [%7] \n"
"b 5f \n"
"4: \n"
"st1 {v0.s}[0], [%4] \n"
"st1 {v1.s}[0], [%5] \n"
"st1 {v2.s}[0], [%6] \n"
"st1 {v3.s}[0], [%7] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(dst3), // %7
"+r"(width) // %8
: "r"(s * 4) // %9
: "memory", "cc", "v0", "v1", "v2", "v3");
}
void TransposeWxH_32_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const ptrdiff_t d = dst_stride;
while (width >= 4) {
TransposeNx4_32_NEON(src, src_stride, dst, dst_stride, height);
src += 16;
dst += 4 * d;
width -= 4;
}
if (width == 3) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
asm volatile(
"subs %w7, %w7, #4 \n"
"b.lt 2f \n"
"1: \n"
"ld3 {v0.s, v1.s, v2.s}[0], [%0], %8 \n"
"ld3 {v0.s, v1.s, v2.s}[1], [%1], %8 \n"
"ld3 {v0.s, v1.s, v2.s}[2], [%2], %8 \n"
"ld3 {v0.s, v1.s, v2.s}[3], [%3], %8 \n"
"subs %w7, %w7, #4 \n"
"st1 {v0.4s}, [%4], 16 \n"
"st1 {v1.4s}, [%5], 16 \n"
"st1 {v2.4s}, [%6], 16 \n"
"b.ge 1b \n"
"2: \n"
"adds %w7, %w7, #4 \n"
"b.eq 5f \n"
"ld3 {v0.s, v1.s, v2.s}[0], [%0] \n"
"cmp %w7, #2 \n"
"b.lt 4f \n"
"ld3 {v0.s, v1.s, v2.s}[1], [%1] \n"
"b.gt 3f \n"
"st1 {v0.d}[0], [%4] \n"
"st1 {v1.d}[0], [%5] \n"
"st1 {v2.d}[0], [%6] \n"
"b 5f \n"
"3: \n"
"ld3 {v0.s, v1.s, v2.s}[2], [%2] \n"
"st1 {v0.d}[0], [%4], 8 \n"
"st1 {v1.d}[0], [%5], 8 \n"
"st1 {v2.d}[0], [%6], 8 \n"
"st1 {v0.s}[2], [%4] \n"
"st1 {v1.s}[2], [%5] \n"
"st1 {v2.s}[2], [%6] \n"
"b 5f \n"
"4: \n"
"st1 {v0.s}[0], [%4] \n"
"st1 {v1.s}[0], [%5] \n"
"st1 {v2.s}[0], [%6] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(dst2), // %6
"+r"(height) // %7
: "r"(s * 4) // %8
: "memory", "cc", "v0", "v1", "v2");
} else if (width == 2) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
asm volatile(
"subs %w6, %w6, #4 \n"
"b.lt 2f \n"
"1: \n"
"ld2 {v0.s, v1.s}[0], [%0], %7 \n"
"ld2 {v0.s, v1.s}[1], [%1], %7 \n"
"ld2 {v0.s, v1.s}[2], [%2], %7 \n"
"ld2 {v0.s, v1.s}[3], [%3], %7 \n"
"subs %w6, %w6, #4 \n"
"st1 {v0.4s}, [%4], 16 \n"
"st1 {v1.4s}, [%5], 16 \n"
"b.ge 1b \n"
"2: \n"
"adds %w6, %w6, #4 \n"
"b.eq 5f \n"
"ld2 {v0.s, v1.s}[0], [%0] \n"
"cmp %w6, #2 \n"
"b.lt 4f \n"
"ld2 {v0.s, v1.s}[1], [%1] \n"
"b.gt 3f \n"
"st1 {v0.d}[0], [%4] \n"
"st1 {v1.d}[0], [%5] \n"
"b 5f \n"
"3: \n"
"ld2 {v0.s, v1.s}[2], [%2] \n"
"st1 {v0.d}[0], [%4], 8 \n"
"st1 {v1.d}[0], [%5], 8 \n"
"st1 {v0.s}[2], [%4] \n"
"st1 {v1.s}[2], [%5] \n"
"b 5f \n"
"4: \n"
"st1 {v0.s}[0], [%4] \n"
"st1 {v1.s}[0], [%5] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(dst1), // %5
"+r"(height) // %6
: "r"(s * 4) // %7
: "memory", "cc", "v0", "v1");
} else if (width == 1) {
const ptrdiff_t s = src_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
asm volatile(
"subs %w5, %w5, #4 \n"
"b.lt 2f \n"
"1: \n"
"ld1 {v0.s}[0], [%0], %6 \n"
"ld1 {v0.s}[1], [%1], %6 \n"
"ld1 {v0.s}[2], [%2], %6 \n"
"ld1 {v0.s}[3], [%3], %6 \n"
"subs %w5, %w5, #4 \n"
"st1 {v0.4s}, [%4], 16 \n"
"b.ge 1b \n"
"2: \n"
"adds %w5, %w5, #4 \n"
"b.eq 5f \n"
"ld1 {v0.s}[0], [%0] \n"
"cmp %w5, #2 \n"
"b.lt 4f \n"
"ld1 {v0.s}[1], [%1] \n"
"b.gt 3f \n"
"st1 {v0.d}[0], [%4] \n"
"b 5f \n"
"3: \n"
"ld1 {v0.s}[2], [%2] \n"
"st1 {v0.d}[0], [%4], 8 \n"
"st1 {v0.s}[2], [%4] \n"
"b 5f \n"
"4: \n"
"st1 {v0.s}[0], [%4] \n"
"5: \n"
: "+r"(src), // %0
"+r"(src1), // %1
"+r"(src2), // %2
"+r"(src3), // %3
"+r"(dst), // %4
"+r"(height) // %5
: "r"(s * 4) // %6
: "memory", "cc", "v0");
}
}
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus

151
source/rotate_rvv.cc Normal file
View File

@ -0,0 +1,151 @@
/*
* Copyright 2026 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
// This module is for RVV (RISC-V Vector extension)
#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
#ifndef RVV_VL_CLOBBER
#ifdef __clang__
#define RVV_VL_CLOBBER
#else
#define RVV_VL_CLOBBER "vl",
#endif
#endif
#ifdef HAS_TRANSPOSENX4_32_RVV
// Transpose 32 bit values (ARGB) in Nx4 tiles, moving down N rows of source
// (VL at a time) and across N columns of destination (4 rows).
// Tuned for 512-bit VL (SiFive X280).
void TransposeNx4_32_RVV(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
int vl;
ptrdiff_t src_step;
ptrdiff_t dst_step;
uint8_t* dst1 = dst + (ptrdiff_t)dst_stride;
uint8_t* dst2 = dst1 + (ptrdiff_t)dst_stride;
uint8_t* dst3 = dst2 + (ptrdiff_t)dst_stride;
asm volatile(
"vsetvli %[vl], %[w], e32, m2, ta, ma \n"
"slli %[dst_step], %[vl], 2 \n"
"mul %[src_step], %[vl], %[src_stride] \n"
"sub %[w], %[w], %[vl] \n"
"bltz %[w], 2f \n"
"1: \n"
"vlsseg4e32.v v8, (%[src]), %[src_stride] \n"
"add %[src], %[src], %[src_step] \n"
"vse32.v v8, (%[dst0]) \n"
"vse32.v v10, (%[dst1]) \n"
"vse32.v v12, (%[dst2]) \n"
"vse32.v v14, (%[dst3]) \n"
"add %[dst0], %[dst0], %[dst_step] \n"
"add %[dst1], %[dst1], %[dst_step] \n"
"add %[dst2], %[dst2], %[dst_step] \n"
"add %[dst3], %[dst3], %[dst_step] \n"
"sub %[w], %[w], %[vl] \n"
"bgez %[w], 1b \n"
"2: \n"
"add %[w], %[w], %[vl] \n"
"beqz %[w], 3f \n"
"vsetvli %[vl], %[w], e32, m2, ta, ma \n"
"vlsseg4e32.v v8, (%[src]), %[src_stride] \n"
"vse32.v v8, (%[dst0]) \n"
"vse32.v v10, (%[dst1]) \n"
"vse32.v v12, (%[dst2]) \n"
"vse32.v v14, (%[dst3]) \n"
"3: \n"
: [w] "+r"(width), // %[w]
[src] "+r"(src), // %[src]
[dst0] "+r"(dst), // %[dst0]
[dst1] "+r"(dst1), // %[dst1]
[dst2] "+r"(dst2), // %[dst2]
[dst3] "+r"(dst3), // %[dst3]
[vl] "=&r"(vl), // %[vl]
[src_step] "=&r"(src_step), // %[src_step]
[dst_step] "=&r"(dst_step) // %[dst_step]
: [src_stride] "r"((ptrdiff_t)src_stride) // %[src_stride]
: RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12",
"v13", "v14", "v15");
}
#endif // HAS_TRANSPOSE4X4_32_RVV
#ifdef HAS_TRANSPOSEWXH_32_RVV
// Transpose 32 bit plane of any width x height using RVV without scalar
// fallback.
void TransposeWxH_32_RVV(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
while (width >= 4) {
TransposeNx4_32_RVV(src, src_stride, dst, dst_stride, height);
src += 16;
dst += 4 * (ptrdiff_t)dst_stride;
width -= 4;
}
while (width > 0) {
const uint8_t* s_col = src;
uint8_t* d_row = dst;
int h = height;
int vl;
ptrdiff_t src_step;
ptrdiff_t dst_step;
asm volatile(
"vsetvli %[vl], %[h], e32, m8, ta, ma \n"
"slli %[d_step], %[vl], 2 \n"
"mul %[s_step], %[vl], %[src_stride] \n"
"sub %[h], %[h], %[vl] \n"
"bltz %[h], 2f \n"
"1: \n"
"vlse32.v v8, (%[s]), %[src_stride] \n"
"add %[s], %[s], %[s_step] \n"
"vse32.v v8, (%[d]) \n"
"add %[d], %[d], %[d_step] \n"
"sub %[h], %[h], %[vl] \n"
"bgez %[h], 1b \n"
"2: \n"
"add %[h], %[h], %[vl] \n"
"beqz %[h], 3f \n"
"vsetvli %[vl], %[h], e32, m8, ta, ma \n"
"vlse32.v v8, (%[s]), %[src_stride] \n"
"vse32.v v8, (%[d]) \n"
"3: \n"
: [h] "+r"(h), // %[h]
[s] "+r"(s_col), // %[s]
[d] "+r"(d_row), // %[d]
[vl] "=&r"(vl), // %[vl]
[s_step] "=&r"(src_step), // %[s_step]
[d_step] "=&r"(dst_step) // %[d_step]
: [src_stride] "r"((ptrdiff_t)src_stride) // %[src_stride]
: RVV_VL_CLOBBER "vtype", "memory", "v8", "v9", "v10", "v11", "v12",
"v13", "v14", "v15");
src += 4;
dst += (ptrdiff_t)dst_stride;
width -= 1;
}
}
#endif // HAS_TRANSPOSEWXH_32_RVV
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
#endif // !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)

67
source/rotate_sme.cc
View File

@ -170,6 +170,73 @@ __arm_locally_streaming __arm_new("za") void TransposeUVWxH_SME(
} while (height > 0);
}
__arm_locally_streaming __arm_new("za") void TransposeWxH_32_SME(
const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
int vl;
asm("cntw %x0" : "=r"(vl));
do {
const uint8_t* src2 = src;
uint8_t* dst2 = dst;
// Process up to VL 32-bit rows per iteration of the outer loop.
int block_height = height > vl ? vl : height;
int width2 = width;
do {
const uint8_t* src3 = src2;
// Process up to VL 32-bit columns per iteration of the inner loop.
int block_width = width2 > vl ? vl : width2;
asm volatile(
"mov w12, #0 \n"
// Create a predicate to handle loading partial rows.
"whilelt p0.s, wzr, %w[block_width] \n"
// Load H <= VL rows into ZA0.S.
"1: \n"
"ld1w {za0h.s[w12, 0]}, p0/z, [%[src3]] \n"
"add %[src3], %[src3], %[src_stride] \n"
"add w12, w12, #1 \n"
"cmp w12, %w[block_height] \n"
"b.ne 1b \n"
// Create a predicate to handle storing partial columns.
"whilelt p0.s, wzr, %w[block_height] \n"
"mov w12, #0 \n"
// Store W <= VL columns from ZA0.S.
"2: \n"
"st1w {za0v.s[w12, 0]}, p0, [%[dst2]] \n"
"add %[dst2], %[dst2], %[dst_stride] \n"
"add w12, w12, #1 \n"
"cmp w12, %w[block_width] \n"
"b.ne 2b \n"
: [src3] "+r"(src3), // %[src3]
[dst2] "+r"(dst2) // %[dst2]
: [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
[dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride]
[block_width] "r"(block_width), // %[block_width]
[block_height] "r"(block_height) // %[block_height]
: "cc", "memory", "p0", "w12", "za");
src2 += vl * 4;
width2 -= vl;
} while (width2 > 0);
src += (ptrdiff_t)vl * src_stride;
dst += vl * 4;
height -= vl;
} while (height > 0);
}
#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
// defined(__aarch64__)

117
source/rotate_sve.cc Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright 2026 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_row.h"
#include "libyuv/row.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
// Transpose 32 bit values (ARGB) in Nx4 tiles using SVE2 with predication
// for tail widths.
void TransposeNx4_32_SVE2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t s4 = s * 4;
const ptrdiff_t d = dst_stride;
const uint8_t* src1 = src + s;
const uint8_t* src2 = src1 + s;
const uint8_t* src3 = src2 + s;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
asm volatile(
"ptrue p0.s, vl4 \n"
"subs %w[width], %w[width], #4 \n"
"b.lt 2f \n"
// Main loop: full 4x4 blocks.
"1: \n"
"ld1w {z0.s}, p0/z, [%[src0]] \n"
"ld1w {z1.s}, p0/z, [%[src1]] \n"
"ld1w {z2.s}, p0/z, [%[src2]] \n"
"ld1w {z3.s}, p0/z, [%[src3]] \n"
"add %[src0], %[src0], %[s4] \n"
"add %[src1], %[src1], %[s4] \n"
"add %[src2], %[src2], %[s4] \n"
"add %[src3], %[src3], %[s4] \n"
"trn1 z4.s, z0.s, z1.s \n"
"trn2 z5.s, z0.s, z1.s \n"
"trn1 z6.s, z2.s, z3.s \n"
"trn2 z7.s, z2.s, z3.s \n"
"trn1 z0.d, z4.d, z6.d \n"
"trn2 z1.d, z4.d, z6.d \n"
"trn1 z2.d, z5.d, z7.d \n"
"trn2 z3.d, z5.d, z7.d \n"
"subs %w[width], %w[width], #4 \n"
"st1w {z0.s}, p0, [%[dst0]] \n"
"st1w {z1.s}, p0, [%[dst1]] \n"
"st1w {z2.s}, p0, [%[dst2]] \n"
"st1w {z3.s}, p0, [%[dst3]] \n"
"add %[dst0], %[dst0], #16 \n"
"add %[dst1], %[dst1], #16 \n"
"add %[dst2], %[dst2], #16 \n"
"add %[dst3], %[dst3], #16 \n"
"b.ge 1b \n"
"2: \n"
"adds %w[width], %w[width], #4 \n"
"b.eq 3f \n"
// Tail loop: 1..3 remaining source rows / destination columns.
"cmp %w[width], #1 \n"
"csel %[src1], %[src0], %[src1], eq \n"
"cmp %w[width], #2 \n"
"csel %[src2], %[src1], %[src2], le \n"
"mov %[src3], %[src2] \n"
"ld1w {z0.s}, p0/z, [%[src0]] \n"
"ld1w {z1.s}, p0/z, [%[src1]] \n"
"ld1w {z2.s}, p0/z, [%[src2]] \n"
"ld1w {z3.s}, p0/z, [%[src3]] \n"
"trn1 z4.s, z0.s, z1.s \n"
"trn2 z5.s, z0.s, z1.s \n"
"trn1 z6.s, z2.s, z3.s \n"
"trn2 z7.s, z2.s, z3.s \n"
"trn1 z0.d, z4.d, z6.d \n"
"trn2 z1.d, z4.d, z6.d \n"
"trn1 z2.d, z5.d, z7.d \n"
"trn2 z3.d, z5.d, z7.d \n"
"whilelt p0.s, wzr, %w[width] \n"
"st1w {z0.s}, p0, [%[dst0]] \n"
"st1w {z1.s}, p0, [%[dst1]] \n"
"st1w {z2.s}, p0, [%[dst2]] \n"
"st1w {z3.s}, p0, [%[dst3]] \n"
"3: \n"
: [src0] "+r"(src), // %[src0]
[src1] "+r"(src1), // %[src1]
[src2] "+r"(src2), // %[src2]
[src3] "+r"(src3), // %[src3]
[dst0] "+r"(dst), // %[dst0]
[dst1] "+r"(dst1), // %[dst1]
[dst2] "+r"(dst2), // %[dst2]
[dst3] "+r"(dst3), // %[dst3]
[width] "+r"(width) // %[width]
: [s4] "r"(s4) // %[s4]
: "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
}
#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif

45
source/rotate_wasm.cc
View File

@ -201,6 +201,42 @@ void Transpose4x4_32_WASMSIMD(const uint8_t* src,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
uint8_t* dst1 = dst + d;
uint8_t* dst2 = dst1 + d;
uint8_t* dst3 = dst2 + d;
while (width >= 8) {
v128_t r0 = wasm_v128_load(src);
v128_t r1 = wasm_v128_load(src + s);
v128_t r2 = wasm_v128_load(src + s * 2);
v128_t r3 = wasm_v128_load(src + s * 3);
src += s * 4;
v128_t r4 = wasm_v128_load(src);
v128_t r5 = wasm_v128_load(src + s);
v128_t r6 = wasm_v128_load(src + s * 2);
v128_t r7 = wasm_v128_load(src + s * 3);
src += s * 4;
v128_t a0 = ZIPLO32(r0, r1);
v128_t a1 = ZIPHI32(r0, r1);
v128_t a2 = ZIPLO32(r2, r3);
v128_t a3 = ZIPHI32(r2, r3);
v128_t b0 = ZIPLO32(r4, r5);
v128_t b1 = ZIPHI32(r4, r5);
v128_t b2 = ZIPLO32(r6, r7);
v128_t b3 = ZIPHI32(r6, r7);
wasm_v128_store(dst, ZIPLO64(a0, a2));
wasm_v128_store(dst1, ZIPHI64(a0, a2));
wasm_v128_store(dst2, ZIPLO64(a1, a3));
wasm_v128_store(dst3, ZIPHI64(a1, a3));
wasm_v128_store(dst + 16, ZIPLO64(b0, b2));
wasm_v128_store(dst1 + 16, ZIPHI64(b0, b2));
wasm_v128_store(dst2 + 16, ZIPLO64(b1, b3));
wasm_v128_store(dst3 + 16, ZIPHI64(b1, b3));
dst += 32;
dst1 += 32;
dst2 += 32;
dst3 += 32;
width -= 8;
}
while (width > 0) {
v128_t r0 = wasm_v128_load(src);
v128_t r1 = wasm_v128_load(src + s);
@ -211,11 +247,14 @@ void Transpose4x4_32_WASMSIMD(const uint8_t* src,
v128_t a2 = ZIPLO32(r2, r3);
v128_t a3 = ZIPHI32(r2, r3);
wasm_v128_store(dst, ZIPLO64(a0, a2));
wasm_v128_store(dst + d, ZIPHI64(a0, a2));
wasm_v128_store(dst + d * 2, ZIPLO64(a1, a3));
wasm_v128_store(dst + d * 3, ZIPHI64(a1, a3));
wasm_v128_store(dst1, ZIPHI64(a0, a2));
wasm_v128_store(dst2, ZIPLO64(a1, a3));
wasm_v128_store(dst3, ZIPHI64(a1, a3));
src += s * 4;
dst += 16;
dst1 += 16;
dst2 += 16;
dst3 += 16;
width -= 4;
}
}

438
source/rotate_win.cc
View File

@ -447,17 +447,18 @@ void Transpose4x4_32_SSE2(const uint8_t* src,
}
#endif // defined(HAS_TRANSPOSE4X4_32_SSE2)
#if defined(HAS_TRANSPOSE4X4_32_AVX2)
// Transpose 32 bit values (ARGB). Read a column of 8 pixels, write a row.
#if defined(HAS_TRANSPOSENX4_32_AVX2)
// Transpose 32 bit values (ARGB) in Nx4 tiles with 2 blocks of 4x4 at a time
// in ymm0..ymm3 and _mm_maskstore_epi32 tail for arbitrary N.
LIBYUV_TARGET_AVX2
void Transpose4x4_32_AVX2(const uint8_t* src,
void TransposeNx4_32_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (width > 0) {
while (width >= 8) {
__m256i r0 = _mm256_inserti128_si256(
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))),
_mm_loadu_si128((const __m128i*)(src + s * 4)), 1);
@ -482,9 +483,436 @@ void Transpose4x4_32_AVX2(const uint8_t* src,
dst += 32;
width -= 8;
}
if (width >= 4) {
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
_mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1));
_mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1));
_mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3));
_mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3));
src += s * 4;
dst += 16;
width -= 4;
}
if (width > 0) {
const __m128i mask =
_mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (width * 8)) - 1));
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
__m128i r1 = (width > 1) ? _mm_loadu_si128((const __m128i*)(src + s)) : r0;
__m128i r2 =
(width > 2) ? _mm_loadu_si128((const __m128i*)(src + s * 2)) : r1;
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
__m128i t1 = _mm_unpacklo_epi32(r2, r2);
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
__m128i t3 = _mm_unpackhi_epi32(r2, r2);
_mm_maskstore_epi32((int*)(dst), mask, _mm_unpacklo_epi64(t0, t1));
_mm_maskstore_epi32((int*)(dst + d), mask, _mm_unpackhi_epi64(t0, t1));
_mm_maskstore_epi32((int*)(dst + d * 2), mask, _mm_unpacklo_epi64(t2, t3));
_mm_maskstore_epi32((int*)(dst + d * 3), mask, _mm_unpackhi_epi64(t2, t3));
}
_mm256_zeroupper();
}
#endif // defined(HAS_TRANSPOSE4X4_32_AVX2)
#endif // defined(HAS_TRANSPOSENX4_32_AVX2)
#if defined(HAS_TRANSPOSEWXH_32_AVX2)
LIBYUV_TARGET_AVX2
void TransposeWxH_32_AVX2(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (height >= 8) {
const uint8_t* src_row = src;
uint8_t* dst_col = dst;
int w = width;
while (w >= 8) {
__m256i r0 = _mm256_loadu_si256((const __m256i*)(src_row));
__m256i r1 = _mm256_loadu_si256((const __m256i*)(src_row + s));
__m256i r2 = _mm256_loadu_si256((const __m256i*)(src_row + s * 2));
__m256i r3 = _mm256_loadu_si256((const __m256i*)(src_row + s * 3));
__m256i r4 = _mm256_loadu_si256((const __m256i*)(src_row + s * 4));
__m256i r5 = _mm256_loadu_si256((const __m256i*)(src_row + s * 5));
__m256i r6 = _mm256_loadu_si256((const __m256i*)(src_row + s * 6));
__m256i r7 = _mm256_loadu_si256((const __m256i*)(src_row + s * 7));
__m256i t0 = _mm256_unpacklo_epi32(r0, r1);
__m256i t1 = _mm256_unpackhi_epi32(r0, r1);
__m256i t2 = _mm256_unpacklo_epi32(r2, r3);
__m256i t3 = _mm256_unpackhi_epi32(r2, r3);
__m256i t4 = _mm256_unpacklo_epi32(r4, r5);
__m256i t5 = _mm256_unpackhi_epi32(r4, r5);
__m256i t6 = _mm256_unpacklo_epi32(r6, r7);
__m256i t7 = _mm256_unpackhi_epi32(r6, r7);
__m256i u0 = _mm256_unpacklo_epi64(t0, t2);
__m256i u1 = _mm256_unpackhi_epi64(t0, t2);
__m256i u2 = _mm256_unpacklo_epi64(t1, t3);
__m256i u3 = _mm256_unpackhi_epi64(t1, t3);
__m256i u4 = _mm256_unpacklo_epi64(t4, t6);
__m256i u5 = _mm256_unpackhi_epi64(t4, t6);
__m256i u6 = _mm256_unpacklo_epi64(t5, t7);
__m256i u7 = _mm256_unpackhi_epi64(t5, t7);
_mm256_storeu_si256((__m256i*)(dst_col),
_mm256_permute2x128_si256(u0, u4, 0x20));
_mm256_storeu_si256((__m256i*)(dst_col + d),
_mm256_permute2x128_si256(u1, u5, 0x20));
_mm256_storeu_si256((__m256i*)(dst_col + d * 2),
_mm256_permute2x128_si256(u2, u6, 0x20));
_mm256_storeu_si256((__m256i*)(dst_col + d * 3),
_mm256_permute2x128_si256(u3, u7, 0x20));
_mm256_storeu_si256((__m256i*)(dst_col + d * 4),
_mm256_permute2x128_si256(u0, u4, 0x31));
_mm256_storeu_si256((__m256i*)(dst_col + d * 5),
_mm256_permute2x128_si256(u1, u5, 0x31));
_mm256_storeu_si256((__m256i*)(dst_col + d * 6),
_mm256_permute2x128_si256(u2, u6, 0x31));
_mm256_storeu_si256((__m256i*)(dst_col + d * 7),
_mm256_permute2x128_si256(u3, u7, 0x31));
src_row += 32;
dst_col += 8 * d;
w -= 8;
}
if (w >= 4) {
TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, 8);
src_row += 16;
dst_col += 4 * d;
w -= 4;
}
if (w > 0) {
const __m128i col_mask =
_mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (w * 8)) - 1));
uint8_t* dst0 = dst_col;
uint8_t* dst1 = (w > 1) ? dst0 + d : dst0;
uint8_t* dst2 = (w > 2) ? dst1 + d : dst1;
for (int y = 0; y < 8; y += 4) {
__m128i r0 =
_mm_maskload_epi32((const int*)(src_row + y * s), col_mask);
__m128i r1 =
_mm_maskload_epi32((const int*)(src_row + (y + 1) * s), col_mask);
__m128i r2 =
_mm_maskload_epi32((const int*)(src_row + (y + 2) * s), col_mask);
__m128i r3 =
_mm_maskload_epi32((const int*)(src_row + (y + 3) * s), col_mask);
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
_mm_storeu_si128((__m128i*)(dst2 + y * 4), _mm_unpacklo_epi64(t2, t3));
_mm_storeu_si128((__m128i*)(dst1 + y * 4), _mm_unpackhi_epi64(t0, t1));
_mm_storeu_si128((__m128i*)(dst0 + y * 4), _mm_unpacklo_epi64(t0, t1));
}
}
src += 8 * s;
dst += 32;
height -= 8;
}
while (height > 0) {
const int bh = (height >= 4) ? 4 : height;
const uint8_t* src_row = src;
uint8_t* dst_col = dst;
int w = width;
while (w >= 4) {
TransposeNx4_32_AVX2(src_row, src_stride, dst_col, dst_stride, bh);
src_row += 16;
dst_col += 4 * d;
w -= 4;
}
if (w > 0) {
const __m128i col_mask =
_mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (w * 8)) - 1));
const __m128i row_mask =
_mm_cvtepi8_epi32(_mm_cvtsi32_si128((1 << (bh * 8)) - 1));
uint8_t* dst0 = dst_col;
uint8_t* dst1 = (w > 1) ? dst0 + d : dst0;
uint8_t* dst2 = (w > 2) ? dst1 + d : dst1;
__m128i r0 = _mm_maskload_epi32((const int*)(src_row), col_mask);
__m128i r1 = (bh > 1)
? _mm_maskload_epi32((const int*)(src_row + s), col_mask)
: r0;
__m128i r2 =
(bh > 2) ? _mm_maskload_epi32((const int*)(src_row + s * 2), col_mask)
: r1;
__m128i r3 =
(bh > 3) ? _mm_maskload_epi32((const int*)(src_row + s * 3), col_mask)
: r2;
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
_mm_maskstore_epi32((int*)dst2, row_mask, _mm_unpacklo_epi64(t2, t3));
_mm_maskstore_epi32((int*)dst1, row_mask, _mm_unpackhi_epi64(t0, t1));
_mm_maskstore_epi32((int*)dst0, row_mask, _mm_unpacklo_epi64(t0, t1));
}
src += bh * s;
dst += bh * 4;
height -= bh;
}
_mm256_zeroupper();
}
#endif // defined(HAS_TRANSPOSEWXH_32_AVX2)
#if defined(HAS_TRANSPOSENX4_32_AVX512BW)
// Transpose 32 bit values (ARGB) in Nx4 tiles with 4 blocks of 4x4 at a time
// in zmm0..zmm3 and masked tail for arbitrary N.
LIBYUV_TARGET_AVX512BW
void TransposeNx4_32_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (width >= 16) {
__m512i r0 = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src)));
__m512i r1 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s)));
__m512i r2 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 2)));
__m512i r3 =
_mm512_castsi128_si512(_mm_loadu_si128((const __m128i*)(src + s * 3)));
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 4)),
1);
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 5)),
1);
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 6)),
1);
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 7)),
1);
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 8)),
2);
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 9)),
2);
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 10)),
2);
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 11)),
2);
r0 = _mm512_inserti32x4(r0, _mm_loadu_si128((const __m128i*)(src + s * 12)),
3);
r1 = _mm512_inserti32x4(r1, _mm_loadu_si128((const __m128i*)(src + s * 13)),
3);
r2 = _mm512_inserti32x4(r2, _mm_loadu_si128((const __m128i*)(src + s * 14)),
3);
r3 = _mm512_inserti32x4(r3, _mm_loadu_si128((const __m128i*)(src + s * 15)),
3);
__m512i t0 = _mm512_unpacklo_epi32(r0, r1);
__m512i t1 = _mm512_unpacklo_epi32(r2, r3);
__m512i t2 = _mm512_unpackhi_epi32(r0, r1);
__m512i t3 = _mm512_unpackhi_epi32(r2, r3);
_mm512_storeu_si512((__m512i*)(dst), _mm512_unpacklo_epi64(t0, t1));
_mm512_storeu_si512((__m512i*)(dst + d), _mm512_unpackhi_epi64(t0, t1));
_mm512_storeu_si512((__m512i*)(dst + d * 2), _mm512_unpacklo_epi64(t2, t3));
_mm512_storeu_si512((__m512i*)(dst + d * 3), _mm512_unpackhi_epi64(t2, t3));
src += s * 16;
dst += 64;
width -= 16;
}
while (width >= 4) {
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
__m128i r1 = _mm_loadu_si128((const __m128i*)(src + s));
__m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2));
__m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3));
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
__m128i t1 = _mm_unpacklo_epi32(r2, r3);
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
__m128i t3 = _mm_unpackhi_epi32(r2, r3);
_mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1));
_mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1));
_mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3));
_mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3));
src += s * 4;
dst += 16;
width -= 4;
}
if (width > 0) {
const __mmask8 mask = (1u << width) - 1;
__m128i r0 = _mm_loadu_si128((const __m128i*)(src));
__m128i r1 = (width > 1) ? _mm_loadu_si128((const __m128i*)(src + s)) : r0;
__m128i r2 =
(width > 2) ? _mm_loadu_si128((const __m128i*)(src + s * 2)) : r1;
__m128i t0 = _mm_unpacklo_epi32(r0, r1);
__m128i t1 = _mm_unpacklo_epi32(r2, r2);
__m128i t2 = _mm_unpackhi_epi32(r0, r1);
__m128i t3 = _mm_unpackhi_epi32(r2, r2);
_mm_mask_storeu_epi32(dst, mask, _mm_unpacklo_epi64(t0, t1));
_mm_mask_storeu_epi32(dst + d, mask, _mm_unpackhi_epi64(t0, t1));
_mm_mask_storeu_epi32(dst + d * 2, mask, _mm_unpacklo_epi64(t2, t3));
_mm_mask_storeu_epi32(dst + d * 3, mask, _mm_unpackhi_epi64(t2, t3));
}
_mm256_zeroupper();
}
#endif // defined(HAS_TRANSPOSENX4_32_AVX512BW)
#if defined(HAS_TRANSPOSEWXH_32_AVX512BW)
LIBYUV_TARGET_AVX512BW
void TransposeWxH_32_AVX512BW(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width,
int height) {
const ptrdiff_t s = src_stride;
const ptrdiff_t d = dst_stride;
while (height > 0) {
const int bh = (height >= 16) ? 16 : height;
const __mmask16 row_mask = (1u << bh) - 1;
const uint8_t* src_row = src;
uint8_t* dst_col = dst;
int w = width;
while (w > 0) {
const int bw = (w >= 16) ? 16 : w;
const __mmask16 col_mask = (1u << bw) - 1;
__m512i z = _mm512_setzero_si512();
__m512i r0 = _mm512_maskz_loadu_epi32(col_mask, src_row);
__m512i r1 =
(bh > 1) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s) : z;
__m512i r2 =
(bh > 2) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 2) : z;
__m512i r3 =
(bh > 3) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 3) : z;
__m512i r4 =
(bh > 4) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 4) : z;
__m512i r5 =
(bh > 5) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 5) : z;
__m512i r6 =
(bh > 6) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 6) : z;
__m512i r7 =
(bh > 7) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 7) : z;
__m512i r8 =
(bh > 8) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 8) : z;
__m512i r9 =
(bh > 9) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 9) : z;
__m512i r10 =
(bh > 10) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 10) : z;
__m512i r11 =
(bh > 11) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 11) : z;
__m512i r12 =
(bh > 12) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 12) : z;
__m512i r13 =
(bh > 13) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 13) : z;
__m512i r14 =
(bh > 14) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 14) : z;
__m512i r15 =
(bh > 15) ? _mm512_maskz_loadu_epi32(col_mask, src_row + s * 15) : z;
__m512i t0 = _mm512_unpacklo_epi32(r0, r1);
__m512i t1 = _mm512_unpackhi_epi32(r0, r1);
__m512i t2 = _mm512_unpacklo_epi32(r2, r3);
__m512i t3 = _mm512_unpackhi_epi32(r2, r3);
__m512i t4 = _mm512_unpacklo_epi32(r4, r5);
__m512i t5 = _mm512_unpackhi_epi32(r4, r5);
__m512i t6 = _mm512_unpacklo_epi32(r6, r7);
__m512i t7 = _mm512_unpackhi_epi32(r6, r7);
__m512i t8 = _mm512_unpacklo_epi32(r8, r9);
__m512i t9 = _mm512_unpackhi_epi32(r8, r9);
__m512i t10 = _mm512_unpacklo_epi32(r10, r11);
__m512i t11 = _mm512_unpackhi_epi32(r10, r11);
__m512i t12 = _mm512_unpacklo_epi32(r12, r13);
__m512i t13 = _mm512_unpackhi_epi32(r12, r13);
__m512i t14 = _mm512_unpacklo_epi32(r14, r15);
__m512i t15 = _mm512_unpackhi_epi32(r14, r15);
r0 = _mm512_unpacklo_epi64(t0, t2);
r1 = _mm512_unpackhi_epi64(t0, t2);
r2 = _mm512_unpacklo_epi64(t1, t3);
r3 = _mm512_unpackhi_epi64(t1, t3);
r4 = _mm512_unpacklo_epi64(t4, t6);
r5 = _mm512_unpackhi_epi64(t4, t6);
r6 = _mm512_unpacklo_epi64(t5, t7);
r7 = _mm512_unpackhi_epi64(t5, t7);
r8 = _mm512_unpacklo_epi64(t8, t10);
r9 = _mm512_unpackhi_epi64(t8, t10);
r10 = _mm512_unpacklo_epi64(t9, t11);
r11 = _mm512_unpackhi_epi64(t9, t11);
r12 = _mm512_unpacklo_epi64(t12, t14);
r13 = _mm512_unpackhi_epi64(t12, t14);
r14 = _mm512_unpacklo_epi64(t13, t15);
r15 = _mm512_unpackhi_epi64(t13, t15);
t0 = _mm512_shuffle_i32x4(r0, r4, 0x88);
t1 = _mm512_shuffle_i32x4(r1, r5, 0x88);
t2 = _mm512_shuffle_i32x4(r2, r6, 0x88);
t3 = _mm512_shuffle_i32x4(r3, r7, 0x88);
t4 = _mm512_shuffle_i32x4(r0, r4, 0xdd);
t5 = _mm512_shuffle_i32x4(r1, r5, 0xdd);
t6 = _mm512_shuffle_i32x4(r2, r6, 0xdd);
t7 = _mm512_shuffle_i32x4(r3, r7, 0xdd);
t8 = _mm512_shuffle_i32x4(r8, r12, 0x88);
t9 = _mm512_shuffle_i32x4(r9, r13, 0x88);
t10 = _mm512_shuffle_i32x4(r10, r14, 0x88);
t11 = _mm512_shuffle_i32x4(r11, r15, 0x88);
t12 = _mm512_shuffle_i32x4(r8, r12, 0xdd);
t13 = _mm512_shuffle_i32x4(r9, r13, 0xdd);
t14 = _mm512_shuffle_i32x4(r10, r14, 0xdd);
t15 = _mm512_shuffle_i32x4(r11, r15, 0xdd);
_mm512_mask_storeu_epi32(dst_col, row_mask,
_mm512_shuffle_i32x4(t0, t8, 0x88));
if (bw > 1)
_mm512_mask_storeu_epi32(dst_col + d, row_mask,
_mm512_shuffle_i32x4(t1, t9, 0x88));
if (bw > 2)
_mm512_mask_storeu_epi32(dst_col + d * 2, row_mask,
_mm512_shuffle_i32x4(t2, t10, 0x88));
if (bw > 3)
_mm512_mask_storeu_epi32(dst_col + d * 3, row_mask,
_mm512_shuffle_i32x4(t3, t11, 0x88));
if (bw > 4)
_mm512_mask_storeu_epi32(dst_col + d * 4, row_mask,
_mm512_shuffle_i32x4(t4, t12, 0x88));
if (bw > 5)
_mm512_mask_storeu_epi32(dst_col + d * 5, row_mask,
_mm512_shuffle_i32x4(t5, t13, 0x88));
if (bw > 6)
_mm512_mask_storeu_epi32(dst_col + d * 6, row_mask,
_mm512_shuffle_i32x4(t6, t14, 0x88));
if (bw > 7)
_mm512_mask_storeu_epi32(dst_col + d * 7, row_mask,
_mm512_shuffle_i32x4(t7, t15, 0x88));
if (bw > 8)
_mm512_mask_storeu_epi32(dst_col + d * 8, row_mask,
_mm512_shuffle_i32x4(t0, t8, 0xdd));
if (bw > 9)
_mm512_mask_storeu_epi32(dst_col + d * 9, row_mask,
_mm512_shuffle_i32x4(t1, t9, 0xdd));
if (bw > 10)
_mm512_mask_storeu_epi32(dst_col + d * 10, row_mask,
_mm512_shuffle_i32x4(t2, t10, 0xdd));
if (bw > 11)
_mm512_mask_storeu_epi32(dst_col + d * 11, row_mask,
_mm512_shuffle_i32x4(t3, t11, 0xdd));
if (bw > 12)
_mm512_mask_storeu_epi32(dst_col + d * 12, row_mask,
_mm512_shuffle_i32x4(t4, t12, 0xdd));
if (bw > 13)
_mm512_mask_storeu_epi32(dst_col + d * 13, row_mask,
_mm512_shuffle_i32x4(t5, t13, 0xdd));
if (bw > 14)
_mm512_mask_storeu_epi32(dst_col + d * 14, row_mask,
_mm512_shuffle_i32x4(t6, t14, 0xdd));
if (bw > 15)
_mm512_mask_storeu_epi32(dst_col + d * 15, row_mask,
_mm512_shuffle_i32x4(t7, t15, 0xdd));
src_row += 64;
dst_col += 16 * d;
w -= 16;
}
src += 16 * s;
dst += 64;
height -= 16;
}
_mm256_zeroupper();
}
#endif // defined(HAS_TRANSPOSEWXH_32_AVX512BW)
#ifdef __cplusplus
} // extern "C"

6
source/row_any.cc
View File

@ -1046,6 +1046,12 @@ ANY11(CopyRow_Any_SSE2, CopyRow_SSE2, 0, 1, 1, 31)
#ifdef HAS_COPYROW_NEON
ANY11(CopyRow_Any_NEON, CopyRow_NEON, 0, 1, 1, 31)
#endif
#ifdef HAS_COPYROW_LSX
ANY11(CopyRow_Any_LSX, CopyRow_LSX, 0, 1, 1, 31)
#endif
#ifdef HAS_COPYROW_LASX
ANY11(CopyRow_Any_LASX, CopyRow_LASX, 0, 1, 1, 63)
#endif
#if defined(HAS_ARGBTORGB24ROW_SSSE3)
ANY11(ARGBToRGB24Row_Any_SSSE3, ARGBToRGB24Row_SSSE3, 0, 4, 3, 15)
ANY11(ARGBToRAWRow_Any_SSSE3, ARGBToRAWRow_SSSE3, 0, 4, 3, 15)

14
source/row_lasx.cc
View File

@ -289,6 +289,20 @@ void ARGBMirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width) {
}
}
void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int width) {
int x;
int len = width / 64;
__m256i src0, src1;
for (x = 0; x < len; x++) {
DUP2_ARG2(__lasx_xvld, src, 0, src, 32, src0, src1);
__lasx_xvst(src0, dst, 0);
__lasx_xvst(src1, dst, 32);
src += 64;
dst += 64;
}
}
void I422ToYUY2Row_LASX(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,

14
source/row_lsx.cc
View File

@ -329,6 +329,20 @@ void ARGBMirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width) {
}
}
void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int width) {
int x;
int len = width / 32;
__m128i src0, src1;
for (x = 0; x < len; x++) {
DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
__lsx_vst(src0, dst, 0);
__lsx_vst(src1, dst, 16);
src += 32;
dst += 32;
}
}
void I422ToYUY2Row_LSX(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,

47
source/row_rvv.cc
View File

@ -2581,6 +2581,53 @@ void HalfWidthRow_16To8_RVV(const uint16_t* src_uv,
}
#endif
#ifdef HAS_ARGBMIRRORROW_RVV
void ARGBMirrorRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
int vl;
ptrdiff_t byte_vl;
src += (ptrdiff_t)width * 4;
asm volatile(
"vsetvli %[vl], %[w], e32, m8, ta, ma \n"
"slli %[byte_vl], %[vl], 2 \n"
"vid.v v16 \n"
"addi %[vl], %[vl], -1 \n"
"vrsub.vx v16, v16, %[vl] \n"
"addi %[vl], %[vl], 1 \n"
"sub %[w], %[w], %[vl] \n"
"bltz %[w], 2f \n"
"1: \n"
"sub %[src], %[src], %[byte_vl] \n"
"vle32.v v8, (%[src]) \n"
"vrgather.vv v0, v8, v16 \n"
"vse32.v v0, (%[dst]) \n"
"add %[dst], %[dst], %[byte_vl] \n"
"sub %[w], %[w], %[vl] \n"
"bgez %[w], 1b \n"
"2: \n"
"add %[w], %[w], %[vl] \n"
"beqz %[w], 3f \n"
"vsetvli %[vl], %[w], e32, m8, ta, ma \n"
"slli %[byte_vl], %[vl], 2 \n"
"vid.v v16 \n"
"addi %[vl], %[vl], -1 \n"
"vrsub.vx v16, v16, %[vl] \n"
"sub %[src], %[src], %[byte_vl] \n"
"vle32.v v8, (%[src]) \n"
"vrgather.vv v0, v8, v16 \n"
"vse32.v v0, (%[dst]) \n"
"3: \n"
: [src] "+r"(src), // %[src]
[dst] "+r"(dst), // %[dst]
[w] "+r"(width), // %[w]
[vl] "=&r"(vl), // %[vl]
[byte_vl] "=&r"(byte_vl) // %[byte_vl]
:
: RVV_VL_CLOBBER "vtype", "memory", "v0", "v1", "v2", "v3", "v4", "v5",
"v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
#endif // HAS_ARGBMIRRORROW_RVV
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv

14
unit_test/rotate_argb_test.cc
View File

@ -119,6 +119,20 @@ TEST_F(LibYUVRotateTest, ARGBRotate270_Opt) {
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, ARGBRotate90_Odd) {
ARGBTestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, ARGBRotate270_Odd) {
ARGBTestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
static void TestRotatePlane(int src_width,
int src_height,
int dst_width,