libyuv/unit_test/rotate_argb_test.cc
Frank Barchard 16af162cf9 [libyuv] Optimize RotatePlane_16 SIMD and compact 32-bit transposes
Implement SIMD 16-bit plane rotation (TransposePlane_16,
RotatePlane90_16, RotatePlane180_16, RotatePlane270_16) used by
I010Rotate, I210Rotate, and I410Rotate, keeping tile sizes compact
with full-register writes and zero cross-lane shuffles:
- AVX2: 8x16 tile in ymm0..ymm15 with _Any tail (low-end x86 SIMD)
- AVX-512BW: 8x32 tile in zmm0..zmm15 with k1/k2 right/bottom masks
- NEON (ARM64/ARM32): 8x16 tile in v0..v23 / q0..q11 with _Any tail
- SME: 8xVL tile using ZA16 horizontal/vertical array slices
- RVV: 8xVL tile using widening unpacks with hardware vl tail
- WASM SIMD: 8x8 tile in v128 with _Any tail

Also update 32-bit ARGBRotate x86 transposes to compact vertical
in-lane tiles with full-register writes and zero cross-lane shuffles:
- TransposeWxH_32_AVX2: 4x8 tile in ymm0..ymm7 with vpmaskmovd tails
- TransposeWxH_32_AVX512BW: 4x16 tile in zmm0..zmm7 with k1/k2 tails

Add RotatePlane16_SplitMergeUV_Opt and RotatePlane16_SplitMergeUV_Odd
unit tests comparing 16-bit rotation against SplitUVPlane +
RotatePlane + MergeUVPlane across 0/90/180/270 degrees.

Performance on 1275x1650 (1000 iterations, before -> after):
CPU                     90 degree               180 degree
Coral RVV (Verilator)   7306 -> 1415c (5.16x)   2274 -> 719c (3.16x)
AMD Rome (Zen 2)        821 -> 215ms (3.82x)    259 -> 181ms (1.44x)
AMD Genoa (Zen 4)       483 -> 129ms (3.73x)    149 -> 111ms (1.34x)
AMD Turin (Zen 5)       321 -> 101ms (3.18x)    109 -> 78ms (1.40x)
AMD Milan (Zen 3)       505 -> 189ms (2.67x)    199 -> 118ms (1.69x)
Intel Skylake-X         1076 -> 447ms (2.41x)   353 -> 317ms (1.11x)
WASM SIMD (d8)          415 -> 203ms (2.04x)    370 -> 106ms (3.49x)
Intel Emerald Rapids    735 -> 422ms (1.74x)    238 -> 233ms (1.02x)
Intel Haswell           1120 -> 648ms (1.73x)   438 -> 393ms (1.11x)
Intel Ice Lake          774 -> 474ms (1.63x)    279 -> 290ms (0.96x)
ARM Neoverse-N1         864 -> 544ms (1.59x)    404 -> 266ms (1.52x)
Ampere Siryn            862 -> 576ms (1.50x)    568 -> 329ms (1.72x)
Google Axion (Maple)    554 -> 379ms (1.46x)    199 -> 142ms (1.41x)
Intel Sapphire Rapids   1126 -> 772ms (1.46x)   301 -> 288ms (1.04x)
Intel Granite Rapids    730 -> 565ms (1.29x)    280 -> 280ms (1.00x)
NVIDIA Grace            462 -> 365ms (1.26x)    192 -> 126ms (1.52x)
ARM Cortex-A53          28290 -> 24386ms (1.16x) 7178 -> 5075ms (1.41x)
Google Axion 2 (Tamar)  523 -> 453ms (1.15x)    217 -> 112ms (1.93x)

Test: libyuv_unittest --gtest_filter=*Rotate*
Bug: b/42280942
Change-Id: I91ba6cac22185b5d294fcb98344fdb8a23cc94ff
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8545294
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
2026-10-12 10:57:34 -07:00

442 lines
18 KiB
C++

/*
* Copyright 2012 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stdlib.h>
#include "../unit_test/unit_test.h"
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h"
#include "libyuv/rotate_argb.h"
namespace libyuv {
static void TestRotateBpp(int src_width,
int src_height,
int dst_width,
int dst_height,
libyuv::RotationMode mode,
int benchmark_iterations,
int disable_cpu_flags,
int benchmark_cpu_info,
const int kBpp) {
if (src_width < 1) {
src_width = 1;
}
if (src_height < 1) {
src_height = 1;
}
if (dst_width < 1) {
dst_width = 1;
}
if (dst_height < 1) {
dst_height = 1;
}
int src_stride_argb = src_width * kBpp;
int src_argb_plane_size = src_stride_argb * abs(src_height);
align_buffer_page_end(src_argb, src_argb_plane_size);
for (int i = 0; i < src_argb_plane_size; ++i) {
src_argb[i] = fastrand() & 0xff;
}
int dst_stride_argb = dst_width * kBpp;
int dst_argb_plane_size = dst_stride_argb * dst_height;
align_buffer_page_end(dst_argb_c, dst_argb_plane_size);
align_buffer_page_end(dst_argb_opt, dst_argb_plane_size);
memset(dst_argb_c, 2, dst_argb_plane_size);
memset(dst_argb_opt, 3, dst_argb_plane_size);
if (kBpp == 1) {
MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
RotatePlane(src_argb, src_stride_argb, dst_argb_c, dst_stride_argb,
src_width, src_height, mode);
MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
for (int i = 0; i < benchmark_iterations; ++i) {
RotatePlane(src_argb, src_stride_argb, dst_argb_opt, dst_stride_argb,
src_width, src_height, mode);
}
} else if (kBpp == 4) {
MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
ARGBRotate(src_argb, src_stride_argb, dst_argb_c, dst_stride_argb,
src_width, src_height, mode);
MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
for (int i = 0; i < benchmark_iterations; ++i) {
ARGBRotate(src_argb, src_stride_argb, dst_argb_opt, dst_stride_argb,
src_width, src_height, mode);
}
}
// Rotation should be exact.
for (int i = 0; i < dst_argb_plane_size; ++i) {
ASSERT_EQ(dst_argb_c[i], dst_argb_opt[i]);
}
free_aligned_buffer_page_end(dst_argb_c);
free_aligned_buffer_page_end(dst_argb_opt);
free_aligned_buffer_page_end(src_argb);
}
static void ARGBTestRotate(int src_width,
int src_height,
int dst_width,
int dst_height,
libyuv::RotationMode mode,
int benchmark_iterations,
int disable_cpu_flags,
int benchmark_cpu_info) {
TestRotateBpp(src_width, src_height, dst_width, dst_height, mode,
benchmark_iterations, disable_cpu_flags, benchmark_cpu_info, 4);
}
TEST_F(LibYUVRotateTest, ARGBRotate0_Opt) {
ARGBTestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
benchmark_height_, kRotate0, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, ARGBRotate90_Opt) {
ARGBTestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
benchmark_width_, kRotate90, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, ARGBRotate180_Opt) {
ARGBTestRotate(benchmark_width_, benchmark_height_, benchmark_width_,
benchmark_height_, kRotate180, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, ARGBRotate270_Opt) {
ARGBTestRotate(benchmark_width_, benchmark_height_, benchmark_height_,
benchmark_width_, kRotate270, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, ARGBRotate90_Odd) {
ARGBTestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, ARGBRotate270_Odd) {
ARGBTestRotate(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
static void TestRotatePlane(int src_width,
int src_height,
int dst_width,
int dst_height,
libyuv::RotationMode mode,
int benchmark_iterations,
int disable_cpu_flags,
int benchmark_cpu_info) {
TestRotateBpp(src_width, src_height, dst_width, dst_height, mode,
benchmark_iterations, disable_cpu_flags, benchmark_cpu_info, 1);
}
TEST_F(LibYUVRotateTest, RotatePlane0_Opt) {
TestRotatePlane(benchmark_width_, benchmark_height_, benchmark_width_,
benchmark_height_, kRotate0, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane90_Opt) {
TestRotatePlane(benchmark_width_, benchmark_height_, benchmark_height_,
benchmark_width_, kRotate90, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane180_Opt) {
TestRotatePlane(benchmark_width_, benchmark_height_, benchmark_width_,
benchmark_height_, kRotate180, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane270_Opt) {
TestRotatePlane(benchmark_width_, benchmark_height_, benchmark_height_,
benchmark_width_, kRotate270, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane0_Odd) {
TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane90_Odd) {
TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane180_Odd) {
TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, DISABLED_RotatePlane270_Odd) {
TestRotatePlane(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane90_TestStride) {
int argb_plane_size = benchmark_width_ * 4 * abs(benchmark_height_);
align_buffer_page_end(src_argb, argb_plane_size);
align_buffer_page_end(dst_argb, argb_plane_size);
ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4, dst_argb,
benchmark_width_ * 4, benchmark_width_,
benchmark_height_, kRotate0));
ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4 - 1, dst_argb,
benchmark_width_ * 4 - 1, benchmark_width_ - 1,
benchmark_height_, kRotate0));
ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4, dst_argb,
benchmark_width_ * 4, benchmark_width_,
benchmark_height_, kRotate180));
ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4 - 1, dst_argb,
benchmark_width_ * 4 - 1, benchmark_width_ - 1,
benchmark_height_, kRotate180));
ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4, dst_argb,
abs(benchmark_height_) * 4, benchmark_width_,
benchmark_height_, kRotate90));
ASSERT_EQ(-1, ARGBRotate(src_argb, benchmark_width_ * 4 - 1, dst_argb,
abs(benchmark_height_) * 4, benchmark_width_ - 1,
benchmark_height_, kRotate90));
ASSERT_EQ(0, ARGBRotate(src_argb, benchmark_width_ * 4, dst_argb,
abs(benchmark_height_) * 4, benchmark_width_,
benchmark_height_, kRotate270));
ASSERT_EQ(-1, ARGBRotate(src_argb, benchmark_width_ * 4 - 1, dst_argb,
abs(benchmark_height_) * 4, benchmark_width_ - 1,
benchmark_height_, kRotate270));
free_aligned_buffer_page_end(dst_argb);
free_aligned_buffer_page_end(src_argb);
}
static void TestRotatePlane_16(int src_width,
int src_height,
int dst_width,
int dst_height,
libyuv::RotationMode mode,
int benchmark_iterations,
int disable_cpu_flags,
int benchmark_cpu_info) {
if (src_width < 1) {
src_width = 1;
}
if (src_height < 1) {
src_height = 1;
}
if (dst_width < 1) {
dst_width = 1;
}
if (dst_height < 1) {
dst_height = 1;
}
int src_stride = src_width;
int src_plane_size = src_stride * abs(src_height);
align_buffer_page_end_16(src, src_plane_size);
for (int i = 0; i < src_plane_size; ++i) {
src[i] = fastrand() & 0xffff;
}
int dst_stride = dst_width;
int dst_plane_size = dst_stride * dst_height;
align_buffer_page_end_16(dst_c, dst_plane_size);
align_buffer_page_end_16(dst_opt, dst_plane_size);
memset(dst_c, 2, dst_plane_size * 2);
memset(dst_opt, 3, dst_plane_size * 2);
MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization.
RotatePlane_16(src, src_stride, dst_c, dst_stride, src_width, src_height,
mode);
MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization.
for (int i = 0; i < benchmark_iterations; ++i) {
RotatePlane_16(src, src_stride, dst_opt, dst_stride, src_width, src_height,
mode);
}
// Rotation should be exact.
for (int i = 0; i < dst_plane_size; ++i) {
ASSERT_EQ(dst_c[i], dst_opt[i]);
}
free_aligned_buffer_page_end_16(dst_c);
free_aligned_buffer_page_end_16(dst_opt);
free_aligned_buffer_page_end_16(src);
}
TEST_F(LibYUVRotateTest, RotatePlane0_16_Opt) {
TestRotatePlane_16(benchmark_width_, benchmark_height_, benchmark_width_,
benchmark_height_, kRotate0, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane90_16_Opt) {
TestRotatePlane_16(benchmark_width_, benchmark_height_, benchmark_height_,
benchmark_width_, kRotate90, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane180_16_Opt) {
TestRotatePlane_16(benchmark_width_, benchmark_height_, benchmark_width_,
benchmark_height_, kRotate180, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane270_16_Opt) {
TestRotatePlane_16(benchmark_width_, benchmark_height_, benchmark_height_,
benchmark_width_, kRotate270, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane0_16_Odd) {
TestRotatePlane_16(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate0,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane90_16_Odd) {
TestRotatePlane_16(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate90,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane180_16_Odd) {
TestRotatePlane_16(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_width_ + 1, benchmark_height_ + 1, kRotate180,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane270_16_Odd) {
TestRotatePlane_16(benchmark_width_ + 1, benchmark_height_ + 1,
benchmark_height_ + 1, benchmark_width_ + 1, kRotate270,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_);
}
static void TestRotatePlane16_SplitMergeUV(int src_width,
int src_height,
int dst_width,
int dst_height,
libyuv::RotationMode mode,
int benchmark_cpu_info) {
if (src_width < 1) {
src_width = 1;
}
if (src_height < 1) {
src_height = 1;
}
if (dst_width < 1) {
dst_width = 1;
}
if (dst_height < 1) {
dst_height = 1;
}
const int src_pixels = src_width * src_height;
const int dst_pixels = dst_width * dst_height;
align_buffer_page_end(src_16, src_pixels * 2);
align_buffer_page_end(dst_16, dst_pixels * 2);
align_buffer_page_end(ref_16, dst_pixels * 2);
align_buffer_page_end(src_lo, src_pixels);
align_buffer_page_end(src_hi, src_pixels);
align_buffer_page_end(dst_lo, dst_pixels);
align_buffer_page_end(dst_hi, dst_pixels);
for (int i = 0; i < src_pixels * 2; ++i) {
src_16[i] = fastrand() & 0xff;
}
memset(dst_16, 2, dst_pixels * 2);
memset(ref_16, 3, dst_pixels * 2);
MaskCpuFlags(benchmark_cpu_info);
SplitUVPlane(src_16, src_width * 2, src_lo, src_width, src_hi, src_width,
src_width, src_height);
RotatePlane(src_lo, src_width, dst_lo, dst_width, src_width, src_height,
mode);
RotatePlane(src_hi, src_width, dst_hi, dst_width, src_width, src_height,
mode);
MergeUVPlane(dst_lo, dst_width, dst_hi, dst_width, ref_16, dst_width * 2,
dst_width, dst_height);
I410Rotate(reinterpret_cast<const uint16_t*>(src_16), src_width,
reinterpret_cast<const uint16_t*>(src_16), src_width,
reinterpret_cast<const uint16_t*>(src_16), src_width,
reinterpret_cast<uint16_t*>(dst_16), dst_width,
reinterpret_cast<uint16_t*>(dst_16), dst_width,
reinterpret_cast<uint16_t*>(dst_16), dst_width, src_width,
src_height, mode);
EXPECT_EQ(0, memcmp(ref_16, dst_16, dst_pixels * 2));
free_aligned_buffer_page_end(dst_hi);
free_aligned_buffer_page_end(dst_lo);
free_aligned_buffer_page_end(src_hi);
free_aligned_buffer_page_end(src_lo);
free_aligned_buffer_page_end(ref_16);
free_aligned_buffer_page_end(dst_16);
free_aligned_buffer_page_end(src_16);
}
TEST_F(LibYUVRotateTest, RotatePlane16_SplitMergeUV_Opt) {
TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
benchmark_width_, benchmark_height_, kRotate0,
benchmark_cpu_info_);
TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
benchmark_height_, benchmark_width_, kRotate90,
benchmark_cpu_info_);
TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
benchmark_width_, benchmark_height_,
kRotate180, benchmark_cpu_info_);
TestRotatePlane16_SplitMergeUV(benchmark_width_, benchmark_height_,
benchmark_height_, benchmark_width_,
kRotate270, benchmark_cpu_info_);
}
TEST_F(LibYUVRotateTest, RotatePlane16_SplitMergeUV_Odd) {
TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
benchmark_width_ + 3, benchmark_height_ + 5,
kRotate0, benchmark_cpu_info_);
TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
benchmark_height_ + 5, benchmark_width_ + 3,
kRotate90, benchmark_cpu_info_);
TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
benchmark_width_ + 3, benchmark_height_ + 5,
kRotate180, benchmark_cpu_info_);
TestRotatePlane16_SplitMergeUV(benchmark_width_ + 3, benchmark_height_ + 5,
benchmark_height_ + 5, benchmark_width_ + 3,
kRotate270, benchmark_cpu_info_);
}
} // namespace libyuv