libyuv/source/rotate_argb.cc
Frank Barchard 1ddb18fb6b [libyuv] AVX512, NEON, WASM and RVV alpha copy and mirror rows
1275x1650 (US Letter 150 DPI, 1000 iterations) speedup:

CPU            ARGBExtractAlpha ARGBCopyYToAlpha ARGBMirror RGB24Mirror
-----------------------------------------------------------------------
Intel Granite Rapids      1.00x            1.00x      1.00x       1.04x
Intel Emerald Rapids      1.01x            0.97x      1.03x       1.07x
Intel Sapphire Rapids     0.97x            1.01x      1.08x       1.26x
Intel Ice Lake            0.99x            1.02x      0.97x       1.08x
Intel Skylake-X           1.04x            1.03x      1.01x          NA
Intel Haswell                NA               NA         NA          NA
AMD Turin (Zen 5)         1.01x            1.10x      1.25x       1.31x
AMD Genoa (Zen 4)         0.97x            1.04x      1.09x       1.42x
AMD Milan (Zen 3)            NA               NA         NA          NA
AMD Rome (Zen 2)             NA               NA         NA          NA
NVIDIA Grace (V2)         1.57x            2.51x         NA          NA
Google Axion 2            1.12x            3.17x         NA          NA
Ampere Siryn              1.06x            1.70x         NA          NA
ARM Neoverse-N1           1.49x            2.01x         NA          NA
Google Axion (Maple)      1.47x            2.03x         NA          NA
ARM Cortex-A53            1.06x            1.69x         NA          NA

Add AVX512BW ARGBExtractAlphaRow, ARGBCopyYToAlphaRow and
ARGBMirrorRow, and AVX512VBMI RGB24MirrorRow, in row_gcc.cc and as
intrinsics in row_win.cc. They are x64 only. The remainder is done
with a k-mask (bzhi) zero-masked load and a masked store, so they
handle any width without an Any wrapper and never access memory
outside the row.

RGB24MirrorRow_AVX512VBMI loads and stores 3 full zmm vectors per
64 pixels using 3 vpermi2b. The one output byte that comes from the
third input vector is merged with a masked vpbroadcastb. The 1 to 63
pixel tail rebases the same tables by 192 - 3 * r; indices >= 128 are
taken from the third vector with vpmovb2m and a masked vpermb.

Add ARGBCopyYToAlphaRow_NEON for Arm 32 bit (vld4 + vst4) and
AArch64 (zip + sli), and rewrite AArch64 ARGBExtractAlphaRow_NEON
with ldp + uzp2. Both prefetch source and destination, which is
2.5x faster on Neoverse-N1 and 1.2x to 1.5x faster on Cortex-A53
for 1280x720. Add WASM SIMD ARGBExtractAlphaRow and
ARGBCopyYToAlphaRow, using a swizzle table for CopyYToAlpha. Rework
the RVV versions to use unit stride loads with vnsrl and vzext.vf4.


R=rrwinterton@gmail.com

Bug: None
Change-Id: I436f27d70a3db0a2b176c91eb789a9491133cfcb
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8523531
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
2026-10-07 12:44:53 -07:00

292 lines
8.9 KiB
C++

/*
* Copyright 2012 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/rotate_argb.h"
#include <limits.h>
#include "libyuv/convert.h"
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
#include "libyuv/rotate.h"
#include "libyuv/row.h"
#include "libyuv/scale_row.h" /* for ScaleARGBRowDownEven_ */
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
static int ARGBTranspose(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
int i;
int src_pixel_step = src_stride_argb >> 2;
void (*ScaleARGBRowDownEven)(
const uint8_t* src_argb, ptrdiff_t src_stride_argb, int src_step,
uint8_t* dst_argb, int dst_width) = ScaleARGBRowDownEven_C;
// Check stride is a multiple of 4.
if (src_stride_argb & 3) {
return -1;
}
#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_SSE2;
if (IS_ALIGNED(height, 4)) { // Width of dest.
ScaleARGBRowDownEven = ScaleARGBRowDownEven_SSE2;
}
}
#endif
#if defined(HAS_SCALEARGBROWDOWNEVEN_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_NEON;
if (IS_ALIGNED(height, 4)) { // Width of dest.
ScaleARGBRowDownEven = ScaleARGBRowDownEven_NEON;
}
}
#endif
#if defined(HAS_SCALEARGBROWDOWNEVEN_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_LSX;
if (IS_ALIGNED(height, 4)) { // Width of dest.
ScaleARGBRowDownEven = ScaleARGBRowDownEven_LSX;
}
}
#endif
#if defined(HAS_SCALEARGBROWDOWNEVEN_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_RVV;
}
#endif
#if defined(HAS_SCALEARGBROWDOWNEVEN_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
ScaleARGBRowDownEven = ScaleARGBRowDownEven_Any_WASMSIMD;
if (IS_ALIGNED(height, 4)) { // Width of dest.
ScaleARGBRowDownEven = ScaleARGBRowDownEven_WASMSIMD;
}
}
#endif
for (i = 0; i < width; ++i) { // column of source to row of dest.
ScaleARGBRowDownEven(src_argb, 0, src_pixel_step, dst_argb, height);
dst_argb += dst_stride_argb;
src_argb += 4;
}
return 0;
}
static int ARGBRotate90(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
// Rotate by 90 is a ARGBTranspose with the source read
// from bottom to top. So set the source pointer to the end
// of the buffer and flip the sign of the source stride.
src_argb += (ptrdiff_t)src_stride_argb * (height - 1);
src_stride_argb = -src_stride_argb;
return ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
}
static int ARGBRotate270(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
// Rotate by 270 is a ARGBTranspose with the destination written
// from bottom to top. So set the destination pointer to the end
// of the buffer and flip the sign of the destination stride.
dst_argb += (ptrdiff_t)dst_stride_argb * (width - 1);
dst_stride_argb = -dst_stride_argb;
return ARGBTranspose(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
}
static int ARGBRotate180(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
// Swap first and last row and mirror the content. Uses a temporary row.
const uint8_t* src_bot = src_argb + (ptrdiff_t)src_stride_argb * (height - 1);
uint8_t* dst_bot = dst_argb + (ptrdiff_t)dst_stride_argb * (height - 1);
int half_height = (height + 1) >> 1;
int y;
void (*ARGBMirrorRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width) =
ARGBMirrorRow_C;
void (*CopyRow)(const uint8_t* src_argb, uint8_t* dst_argb, int width) =
CopyRow_C;
if (width > INT_MAX / 4) {
return -1;
}
align_buffer_64(row, width * 4);
if (!row)
return 1;
#if defined(HAS_ARGBMIRRORROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBMirrorRow = ARGBMirrorRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGBMirrorRow = ARGBMirrorRow_NEON;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGBMirrorRow = ARGBMirrorRow_Any_SSE2;
if (IS_ALIGNED(width, 4)) {
ARGBMirrorRow = ARGBMirrorRow_SSE2;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBMirrorRow = ARGBMirrorRow_Any_AVX2;
if (IS_ALIGNED(width, 8)) {
ARGBMirrorRow = ARGBMirrorRow_AVX2;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ARGBMirrorRow = ARGBMirrorRow_AVX512BW;
}
#endif
#if defined(HAS_ARGBMIRRORROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ARGBMirrorRow = ARGBMirrorRow_Any_LSX;
if (IS_ALIGNED(width, 8)) {
ARGBMirrorRow = ARGBMirrorRow_LSX;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
ARGBMirrorRow = ARGBMirrorRow_Any_LASX;
if (IS_ALIGNED(width, 16)) {
ARGBMirrorRow = ARGBMirrorRow_LASX;
}
}
#endif
#if defined(HAS_ARGBMIRRORROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
ARGBMirrorRow = ARGBMirrorRow_Any_WASMSIMD;
if (IS_ALIGNED(width, 4)) {
ARGBMirrorRow = ARGBMirrorRow_WASMSIMD;
}
}
#endif
#if defined(HAS_COPYROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
}
#endif
#if defined(HAS_COPYROW_AVX)
if (TestCpuFlag(kCpuHasAVX)) {
CopyRow = IS_ALIGNED(width * 4, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
}
#endif
#if defined(HAS_COPYROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
CopyRow =
IS_ALIGNED(width * 4, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW;
}
#endif
#if defined(HAS_COPYROW_ERMS)
if (TestCpuFlag(kCpuHasERMS)) {
CopyRow = CopyRow_ERMS;
}
#endif
#if defined(HAS_COPYROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
}
#endif
#if defined(HAS_COPYROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
CopyRow = CopyRow_SVE2;
}
#endif
#if defined(HAS_COPYROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
CopyRow = CopyRow_SME;
}
#endif
#if defined(HAS_COPYROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
CopyRow = CopyRow_RVV;
}
#endif
// Odd height will harmlessly mirror the middle row twice.
for (y = 0; y < half_height; ++y) {
ARGBMirrorRow(src_argb, row, width); // Mirror first row into a buffer
ARGBMirrorRow(src_bot, dst_argb, width); // Mirror last row into first row
CopyRow(row, dst_bot, width * 4); // Copy first mirrored row into last
src_argb += src_stride_argb;
dst_argb += dst_stride_argb;
src_bot -= src_stride_argb;
dst_bot -= dst_stride_argb;
}
free_aligned_buffer_64(row);
return 0;
}
LIBYUV_API
int ARGBRotate(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height,
enum RotationMode mode) {
if (!src_argb || width <= 0 || height == 0 || height == INT_MIN ||
!dst_argb) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
src_argb = src_argb + (ptrdiff_t)(height - 1) * src_stride_argb;
src_stride_argb = -src_stride_argb;
}
switch (mode) {
case kRotate0:
// copy frame
return ARGBCopy(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
case kRotate90:
return ARGBRotate90(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
case kRotate270:
return ARGBRotate270(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
case kRotate180:
return ARGBRotate180(src_argb, src_stride_argb, dst_argb, dst_stride_argb,
width, height);
default:
break;
}
return -1;
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif